diff --git a/.arclint b/.arclint
new file mode 100644
index 0000000000000..01de1ee897ada
--- /dev/null
+++ b/.arclint
@@ -0,0 +1,12 @@
+{
+  "linters": {
+    "clang-format": {
+      "type": "script-and-regex",
+      "script-and-regex.script": "utils/arcanist/clang-format.sh",
+      "script-and-regex.regex": "/^(?P<severity>[[:alpha:]]+)\n(?P<message>[^\n]+)\n(====|(?P<line>\\d),(?P<char>\\d)\n(?P<original>.*)>>>>\n(?P<replacement>.*)<<<<\n)$/s",
+      "include": [
+        "(\\.(cc|cpp|h)$)"
+      ]
+    }
+  }
+}
diff --git a/clang-tools-extra/clang-doc/Representation.cpp b/clang-tools-extra/clang-doc/Representation.cpp
index 34d83b4381253..8c619d2a09632 100644
--- a/clang-tools-extra/clang-doc/Representation.cpp
+++ b/clang-tools-extra/clang-doc/Representation.cpp
@@ -187,7 +187,7 @@ void Info::mergeBase(Info &&Other) {
   // Unconditionally extend the description, since each decl may have a comment.
   std::move(Other.Description.begin(), Other.Description.end(),
             std::back_inserter(Description));
-  std::sort(Description.begin(), Description.end());
+  llvm::sort(Description);
   auto Last = std::unique(Description.begin(), Description.end());
   Description.erase(Last, Description.end());
 }
@@ -202,7 +202,7 @@ void SymbolInfo::merge(SymbolInfo &&Other) {
     DefLoc = std::move(Other.DefLoc);
   // Unconditionally extend the list of locations, since we want all of them.
   std::move(Other.Loc.begin(), Other.Loc.end(), std::back_inserter(Loc));
-  std::sort(Loc.begin(), Loc.end());
+  llvm::sort(Loc);
   auto Last = std::unique(Loc.begin(), Loc.end());
   Loc.erase(Last, Loc.end());
   mergeBase(std::move(Other));
@@ -314,7 +314,7 @@ bool Index::operator<(const Index &Other) const {
 }
 
 void Index::sort() {
-  std::sort(Children.begin(), Children.end());
+  llvm::sort(Children);
   for (auto &C : Children)
     C.sort();
 }
diff --git a/clang-tools-extra/clang-tidy/ClangTidy.cpp b/clang-tools-extra/clang-tidy/ClangTidy.cpp
index 52c217b649f57..05594f191a5fb 100644
--- a/clang-tools-extra/clang-tidy/ClangTidy.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidy.cpp
@@ -454,7 +454,7 @@ std::vector<std::string> ClangTidyASTConsumerFactory::getCheckNames() {
     CheckNames.push_back(AnalyzerCheckNamePrefix + AnalyzerCheck.first);
 #endif // CLANG_ENABLE_STATIC_ANALYZER
 
-  std::sort(CheckNames.begin(), CheckNames.end());
+  llvm::sort(CheckNames);
   return CheckNames;
 }
 
diff --git a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
index 068d56c1a8a8f..2e30a3b403e85 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
@@ -124,7 +124,6 @@ ClangTidyError::ClangTidyError(StringRef CheckName,
     : tooling::Diagnostic(CheckName, DiagLevel, BuildDirectory),
       IsWarningAsError(IsWarningAsError) {}
 
-
 class ClangTidyContext::CachedGlobList {
 public:
   CachedGlobList(StringRef Globs) : Globs(Globs) {}
@@ -297,57 +296,54 @@ static bool IsNOLINTFound(StringRef NolintDirectiveText, StringRef Line,
   return true;
 }
 
+static llvm::Optional<StringRef> getBuffer(const SourceManager &SM, FileID File,
+                                           bool AllowIO) {
+  // This is similar to the implementation of SourceManager::getBufferData(),
+  // but uses ContentCache::getRawBuffer() rather than getBuffer() if
+  // AllowIO=false, to avoid triggering file I/O if the file contents aren't
+  // already mapped.
+  bool CharDataInvalid = false;
+  const SrcMgr::SLocEntry &Entry = SM.getSLocEntry(File, &CharDataInvalid);
+  if (CharDataInvalid || !Entry.isFile())
+    return llvm::None;
+  const SrcMgr::ContentCache *Cache = Entry.getFile().getContentCache();
+  const llvm::MemoryBuffer *Buffer =
+      AllowIO ? Cache->getBuffer(SM.getDiagnostics(), SM.getFileManager(),
+                                 SourceLocation(), &CharDataInvalid)
+              : Cache->getRawBuffer();
+  if (!Buffer || CharDataInvalid)
+    return llvm::None;
+  return Buffer->getBuffer();
+}
+
 static bool LineIsMarkedWithNOLINT(const SourceManager &SM, SourceLocation Loc,
                                    unsigned DiagID,
-                                   const ClangTidyContext &Context) {
-  bool Invalid;
-  const char *CharacterData = SM.getCharacterData(Loc, &Invalid);
-  if (Invalid)
+                                   const ClangTidyContext &Context,
+                                   bool AllowIO) {
+  FileID File;
+  unsigned Offset;
+  std::tie(File, Offset) = SM.getDecomposedSpellingLoc(Loc);
+  llvm::Optional<StringRef> Buffer = getBuffer(SM, File, AllowIO);
+  if (!Buffer)
     return false;
 
   // Check if there's a NOLINT on this line.
-  const char *P = CharacterData;
-  while (*P != '\0' && *P != '\r' && *P != '\n')
-    ++P;
-  StringRef RestOfLine(CharacterData, P - CharacterData + 1);
+  StringRef RestOfLine = Buffer->substr(Offset).split('\n').first;
   if (IsNOLINTFound("NOLINT", RestOfLine, DiagID, Context))
     return true;
 
   // Check if there's a NOLINTNEXTLINE on the previous line.
-  const char *BufBegin =
-      SM.getCharacterData(SM.getLocForStartOfFile(SM.getFileID(Loc)), &Invalid);
-  if (Invalid || P == BufBegin)
-    return false;
-
-  // Scan backwards over the current line.
-  P = CharacterData;
-  while (P != BufBegin && *P != '\n')
-    --P;
-
-  // If we reached the begin of the file there is no line before it.
-  if (P == BufBegin)
-    return false;
-
-  // Skip over the newline.
-  --P;
-  const char *LineEnd = P;
-
-  // Now we're on the previous line. Skip to the beginning of it.
-  while (P != BufBegin && *P != '\n')
-    --P;
-
-  RestOfLine = StringRef(P, LineEnd - P + 1);
-  if (IsNOLINTFound("NOLINTNEXTLINE", RestOfLine, DiagID, Context))
-    return true;
-
-  return false;
+  StringRef PrevLine =
+      Buffer->substr(0, Offset).rsplit('\n').first.rsplit('\n').second;
+  return IsNOLINTFound("NOLINTNEXTLINE", PrevLine, DiagID, Context);
 }
 
 static bool LineIsMarkedWithNOLINTinMacro(const SourceManager &SM,
                                           SourceLocation Loc, unsigned DiagID,
-                                          const ClangTidyContext &Context) {
+                                          const ClangTidyContext &Context,
+                                          bool AllowIO) {
   while (true) {
-    if (LineIsMarkedWithNOLINT(SM, Loc, DiagID, Context))
+    if (LineIsMarkedWithNOLINT(SM, Loc, DiagID, Context, AllowIO))
       return true;
     if (!Loc.isMacroID())
       return false;
@@ -361,14 +357,13 @@ namespace tidy {
 
 bool shouldSuppressDiagnostic(DiagnosticsEngine::Level DiagLevel,
                               const Diagnostic &Info, ClangTidyContext &Context,
-                              bool CheckMacroExpansion) {
+                              bool AllowIO) {
   return Info.getLocation().isValid() &&
          DiagLevel != DiagnosticsEngine::Error &&
          DiagLevel != DiagnosticsEngine::Fatal &&
-         (CheckMacroExpansion ? LineIsMarkedWithNOLINTinMacro
-                              : LineIsMarkedWithNOLINT)(Info.getSourceManager(),
-                                                        Info.getLocation(),
-                                                        Info.getID(), Context);
+         LineIsMarkedWithNOLINTinMacro(Info.getSourceManager(),
+                                       Info.getLocation(), Info.getID(),
+                                       Context, AllowIO);
 }
 
 } // namespace tidy
@@ -680,7 +675,7 @@ void ClangTidyDiagnosticConsumer::removeIncompatibleErrors() {
   for (auto &FileAndEvents : FileEvents) {
     std::vector<Event> &Events = FileAndEvents.second;
     // Sweep.
-    std::sort(Events.begin(), Events.end());
+    llvm::sort(Events);
     int OpenIntervals = 0;
     for (const auto &Event : Events) {
       if (Event.Type == Event::ET_End)
@@ -710,9 +705,9 @@ struct LessClangTidyError {
     const tooling::DiagnosticMessage &M1 = LHS.Message;
     const tooling::DiagnosticMessage &M2 = RHS.Message;
 
-    return
-      std::tie(M1.FilePath, M1.FileOffset, LHS.DiagnosticName, M1.Message) <
-      std::tie(M2.FilePath, M2.FileOffset, RHS.DiagnosticName, M2.Message);
+    return std::tie(M1.FilePath, M1.FileOffset, LHS.DiagnosticName,
+                    M1.Message) <
+           std::tie(M2.FilePath, M2.FileOffset, RHS.DiagnosticName, M2.Message);
   }
 };
 struct EqualClangTidyError {
@@ -726,7 +721,7 @@ struct EqualClangTidyError {
 std::vector<ClangTidyError> ClangTidyDiagnosticConsumer::take() {
   finalizeLastError();
 
-  std::sort(Errors.begin(), Errors.end(), LessClangTidyError());
+  llvm::sort(Errors, LessClangTidyError());
   Errors.erase(std::unique(Errors.begin(), Errors.end(), EqualClangTidyError()),
                Errors.end());
   if (RemoveIncompatibleErrors)
diff --git a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h
index 18ce478805f8b..2bb3296b150dc 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h
+++ b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h
@@ -216,15 +216,12 @@ class ClangTidyContext {
 /// This does not handle suppression of notes following a suppressed diagnostic;
 /// that is left to the caller is it requires maintaining state in between calls
 /// to this function.
-/// The `CheckMacroExpansion` parameter determines whether the function should
-/// handle the case where the diagnostic is inside a macro expansion. A degree
-/// of control over this is needed because handling this case can require
-/// examining source files other than the one in which the diagnostic is
-/// located, and in some use cases we cannot rely on such other files being
-/// mapped in the SourceMapper.
+/// If `AllowIO` is false, the function does not attempt to read source files
+/// from disk which are not already mapped into memory; such files are treated
+/// as not containing a suppression comment.
 bool shouldSuppressDiagnostic(DiagnosticsEngine::Level DiagLevel,
                               const Diagnostic &Info, ClangTidyContext &Context,
-                              bool CheckMacroExpansion = true);
+                              bool AllowIO = true);
 
 /// A diagnostic consumer that turns each \c Diagnostic into a
 /// \c SourceManager-independent \c ClangTidyError.
diff --git a/clang-tools-extra/clang-tidy/modernize/UseDefaultMemberInitCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseDefaultMemberInitCheck.cpp
index cb275ab58f1be..04cc7aa9d4495 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseDefaultMemberInitCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseDefaultMemberInitCheck.cpp
@@ -17,6 +17,12 @@ namespace clang {
 namespace tidy {
 namespace modernize {
 
+namespace {
+AST_MATCHER_P(InitListExpr, initCountIs, unsigned, N) {
+  return Node.getNumInits() == N;
+}
+} // namespace
+
 static StringRef getValueOfValueInit(const QualType InitType) {
   switch (InitType->getScalarTypeKind()) {
   case Type::STK_CPointer:
@@ -190,7 +196,7 @@ void UseDefaultMemberInitCheck::storeOptions(
 }
 
 void UseDefaultMemberInitCheck::registerMatchers(MatchFinder *Finder) {
-  auto Init =
+  auto InitBase =
       anyOf(stringLiteral(), characterLiteral(), integerLiteral(),
             unaryOperator(hasAnyOperatorName("+", "-"),
                           hasUnaryOperand(integerLiteral())),
@@ -198,7 +204,13 @@ void UseDefaultMemberInitCheck::registerMatchers(MatchFinder *Finder) {
             unaryOperator(hasAnyOperatorName("+", "-"),
                           hasUnaryOperand(floatLiteral())),
             cxxBoolLiteral(), cxxNullPtrLiteralExpr(), implicitValueInitExpr(),
-            initListExpr(), declRefExpr(to(enumConstantDecl())));
+            declRefExpr(to(enumConstantDecl())));
+
+  auto Init =
+      anyOf(initListExpr(anyOf(
+                allOf(initCountIs(1), hasInit(0, ignoringImplicit(InitBase))),
+                initCountIs(0))),
+            InitBase);
 
   Finder->addMatcher(
       cxxConstructorDecl(
diff --git a/clang-tools-extra/clang-tidy/readability/RedundantStringCStrCheck.cpp b/clang-tools-extra/clang-tidy/readability/RedundantStringCStrCheck.cpp
index 8975f294373c3..e41cdfcc08d8e 100644
--- a/clang-tools-extra/clang-tidy/readability/RedundantStringCStrCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/RedundantStringCStrCheck.cpp
@@ -61,53 +61,8 @@ formatDereference(const ast_matchers::MatchFinder::MatchResult &Result,
   return (llvm::Twine("*") + Text).str();
 }
 
-// Trying to get CallExpr in which CxxConstructExpr is called.
-static const clang::CallExpr *
-tryGetCallExprAncestorForCxxConstructExpr(const Expr *TheExpr,
-                                          ASTContext &Context) {
-  // We skip nodes such as CXXBindTemporaryExpr, MaterializeTemporaryExpr.
-  for (ast_type_traits::DynTypedNode DynParent : Context.getParents(*TheExpr)) {
-    if (const auto *Parent = DynParent.get<Expr>()) {
-      if (const auto *TheCallExpr = dyn_cast<CallExpr>(Parent))
-        return TheCallExpr;
-
-      if (const clang::CallExpr *TheCallExpr =
-              tryGetCallExprAncestorForCxxConstructExpr(Parent, Context))
-        return TheCallExpr;
-    }
-  }
-
-  return nullptr;
-}
-
-// Check that ParamDecl of CallExprDecl has rvalue type.
-static bool checkParamDeclOfAncestorCallExprHasRValueRefType(
-    const Expr *TheCxxConstructExpr, ASTContext &Context) {
-  if (const clang::CallExpr *TheCallExpr =
-          tryGetCallExprAncestorForCxxConstructExpr(TheCxxConstructExpr,
-                                                    Context)) {
-    for (unsigned i = 0; i < TheCallExpr->getNumArgs(); ++i) {
-      const Expr *Arg = TheCallExpr->getArg(i);
-      if (Arg->getSourceRange() == TheCxxConstructExpr->getSourceRange()) {
-        if (const auto *TheCallExprFuncProto =
-                TheCallExpr->getCallee()
-                    ->getType()
-                    ->getPointeeType()
-                    ->getAs<FunctionProtoType>()) {
-          if (TheCallExprFuncProto->getParamType(i)->isRValueReferenceType())
-            return true;
-        }
-      }
-    }
-  }
-
-  return false;
-}
-
-AST_MATCHER(CXXConstructExpr,
-            matchedParamDeclOfAncestorCallExprHasRValueRefType) {
-  return checkParamDeclOfAncestorCallExprHasRValueRefType(
-      &Node, Finder->getASTContext());
+AST_MATCHER(MaterializeTemporaryExpr, isBoundToLValue) {
+  return Node.isBoundToLvalueReference();
 }
 
 } // end namespace
@@ -141,11 +96,11 @@ void RedundantStringCStrCheck::registerMatchers(
   // Detect redundant 'c_str()' calls through a string constructor.
   // If CxxConstructExpr is the part of some CallExpr we need to
   // check that matched ParamDecl of the ancestor CallExpr is not rvalue.
-  Finder->addMatcher(
-      cxxConstructExpr(
-          StringConstructorExpr, hasArgument(0, StringCStrCallExpr),
-          unless(matchedParamDeclOfAncestorCallExprHasRValueRefType())),
-      this);
+  Finder->addMatcher(cxxConstructExpr(StringConstructorExpr,
+                                      hasArgument(0, StringCStrCallExpr),
+                                      unless(hasParent(materializeTemporaryExpr(
+                                          unless(isBoundToLValue()))))),
+                     this);
 
   // Detect: 's == str.c_str()'  ->  's == str'
   Finder->addMatcher(
diff --git a/clang-tools-extra/clangd/ClangdLSPServer.cpp b/clang-tools-extra/clangd/ClangdLSPServer.cpp
index d0e8d139a40e8..19476d7fa0522 100644
--- a/clang-tools-extra/clangd/ClangdLSPServer.cpp
+++ b/clang-tools-extra/clangd/ClangdLSPServer.cpp
@@ -150,21 +150,6 @@ llvm::Error validateEdits(const DraftStore &DraftMgr, const FileEdits &FE) {
           llvm::to_string(InvalidFileCount - 1) + " others)");
 }
 
-// Converts a list of Ranges to a LinkedList of SelectionRange.
-SelectionRange render(const std::vector<Range> &Ranges) {
-  if (Ranges.empty())
-    return {};
-  SelectionRange Result;
-  Result.range = Ranges[0];
-  auto *Next = &Result.parent;
-  for (const auto &R : llvm::make_range(Ranges.begin() + 1, Ranges.end())) {
-    *Next = std::make_unique<SelectionRange>();
-    Next->get()->range = R;
-    Next = &Next->get()->parent;
-  }
-  return Result;
-}
-
 } // namespace
 
 // MessageHandler dispatches incoming LSP messages.
@@ -473,6 +458,14 @@ void ClangdLSPServer::notify(llvm::StringRef Method, llvm::json::Value Params) {
   Transp.notify(Method, std::move(Params));
 }
 
+static std::vector<llvm::StringRef> semanticTokenTypes() {
+  std::vector<llvm::StringRef> Types;
+  for (unsigned I = 0; I <= static_cast<unsigned>(HighlightingKind::LastKind);
+       ++I)
+    Types.push_back(toSemanticTokenType(static_cast<HighlightingKind>(I)));
+  return Types;
+}
+
 void ClangdLSPServer::onInitialize(const InitializeParams &Params,
                                    Callback<llvm::json::Value> Reply) {
   // Determine character encoding first as it affects constructed ClangdServer.
@@ -487,6 +480,13 @@ void ClangdLSPServer::onInitialize(const InitializeParams &Params,
 
   ClangdServerOpts.TheiaSemanticHighlighting =
       Params.capabilities.TheiaSemanticHighlighting;
+  if (Params.capabilities.TheiaSemanticHighlighting &&
+      Params.capabilities.SemanticTokens) {
+    log("Client supports legacy semanticHighlights notification and standard "
+        "semanticTokens request, choosing the latter (no notifications).");
+    ClangdServerOpts.TheiaSemanticHighlighting = false;
+  }
+
   if (Params.rootUri && *Params.rootUri)
     ClangdServerOpts.WorkspaceRoot = std::string(Params.rootUri->file());
   else if (Params.rootPath && !Params.rootPath->empty())
@@ -584,6 +584,14 @@ void ClangdLSPServer::onInitialize(const InitializeParams &Params,
                  // trigger on '->' and '::'.
                  {"triggerCharacters", {".", ">", ":"}},
              }},
+            {"semanticTokensProvider",
+             llvm::json::Object{
+                 {"documentProvider", llvm::json::Object{{"edits", true}}},
+                 {"rangeProvider", false},
+                 {"legend",
+                  llvm::json::Object{{"tokenTypes", semanticTokenTypes()},
+                                     {"tokenModifiers", llvm::json::Array()}}},
+             }},
             {"signatureHelpProvider",
              llvm::json::Object{
                  {"triggerCharacters", {"(", ","}},
@@ -611,7 +619,7 @@ void ClangdLSPServer::onInitialize(const InitializeParams &Params,
         }}}};
   if (NegotiatedOffsetEncoding)
     Result["offsetEncoding"] = *NegotiatedOffsetEncoding;
-  if (Params.capabilities.TheiaSemanticHighlighting)
+  if (ClangdServerOpts.TheiaSemanticHighlighting)
     Result.getObject("capabilities")
         ->insert(
             {"semanticHighlighting",
@@ -825,6 +833,10 @@ void ClangdLSPServer::onDocumentDidClose(
     std::lock_guard<std::mutex> HLock(HighlightingsMutex);
     FileToHighlightings.erase(File);
   }
+  {
+    std::lock_guard<std::mutex> HLock(SemanticTokensMutex);
+    LastSemanticTokens.erase(File);
+  }
   // clangd will not send updates for this file anymore, so we empty out the
   // list of diagnostics shown on the client (e.g. in the "Problems" pane of
   // VSCode). Note that this cannot race with actual diagnostics responses
@@ -1206,24 +1218,13 @@ void ClangdLSPServer::onSymbolInfo(const TextDocumentPositionParams &Params,
 void ClangdLSPServer::onSelectionRange(
     const SelectionRangeParams &Params,
     Callback<std::vector<SelectionRange>> Reply) {
-  if (Params.positions.size() != 1) {
-    elog("{0} positions provided to SelectionRange. Supports exactly one "
-         "position.",
-         Params.positions.size());
-    return Reply(llvm::make_error<LSPError>(
-        "SelectionRange supports exactly one position",
-        ErrorCode::InvalidRequest));
-  }
   Server->semanticRanges(
-      Params.textDocument.uri.file(), Params.positions[0],
+      Params.textDocument.uri.file(), Params.positions,
       [Reply = std::move(Reply)](
-          llvm::Expected<std::vector<Range>> Ranges) mutable {
-        if (!Ranges) {
+          llvm::Expected<std::vector<SelectionRange>> Ranges) mutable {
+        if (!Ranges)
           return Reply(Ranges.takeError());
-        }
-        std::vector<SelectionRange> Result;
-        Result.emplace_back(render(std::move(*Ranges)));
-        return Reply(std::move(Result));
+        return Reply(std::move(*Ranges));
       });
 }
 
@@ -1246,6 +1247,75 @@ void ClangdLSPServer::onDocumentLink(
       });
 }
 
+// Increment a numeric string: "" -> 1 -> 2 -> ... -> 9 -> 10 -> 11 ...
+static void increment(std::string &S) {
+  for (char &C : llvm::reverse(S)) {
+    if (C != '9') {
+      ++C;
+      return;
+    }
+    C = '0';
+  }
+  S.insert(S.begin(), '1');
+}
+
+void ClangdLSPServer::onSemanticTokens(const SemanticTokensParams &Params,
+                                       Callback<SemanticTokens> CB) {
+  Server->semanticHighlights(
+      Params.textDocument.uri.file(),
+      [this, File(Params.textDocument.uri.file().str()), CB(std::move(CB))](
+          llvm::Expected<std::vector<HighlightingToken>> HT) mutable {
+        if (!HT)
+          return CB(HT.takeError());
+        SemanticTokens Result;
+        Result.tokens = toSemanticTokens(*HT);
+        {
+          std::lock_guard<std::mutex> Lock(SemanticTokensMutex);
+          auto& Last = LastSemanticTokens[File];
+
+          Last.tokens = Result.tokens;
+          increment(Last.resultId);
+          Result.resultId = Last.resultId;
+        }
+        CB(std::move(Result));
+      });
+}
+
+void ClangdLSPServer::onSemanticTokensEdits(
+    const SemanticTokensEditsParams &Params,
+    Callback<SemanticTokensOrEdits> CB) {
+  Server->semanticHighlights(
+      Params.textDocument.uri.file(),
+      [this, PrevResultID(Params.previousResultId),
+       File(Params.textDocument.uri.file().str()), CB(std::move(CB))](
+          llvm::Expected<std::vector<HighlightingToken>> HT) mutable {
+        if (!HT)
+          return CB(HT.takeError());
+        std::vector<SemanticToken> Toks = toSemanticTokens(*HT);
+
+        SemanticTokensOrEdits Result;
+        {
+          std::lock_guard<std::mutex> Lock(SemanticTokensMutex);
+          auto& Last = LastSemanticTokens[File];
+
+          if (PrevResultID == Last.resultId) {
+            Result.edits = diffTokens(Last.tokens, Toks);
+          } else {
+            vlog("semanticTokens/edits: wanted edits vs {0} but last result "
+                 "had ID {1}. Returning full token list.",
+                 PrevResultID, Last.resultId);
+            Result.tokens = Toks;
+          }
+
+          Last.tokens = std::move(Toks);
+          increment(Last.resultId);
+          Result.resultId = Last.resultId;
+        }
+
+        CB(std::move(Result));
+      });
+}
+
 ClangdLSPServer::ClangdLSPServer(
     class Transport &Transp, const FileSystemProvider &FSProvider,
     const clangd::CodeCompleteOptions &CCOpts,
@@ -1293,6 +1363,8 @@ ClangdLSPServer::ClangdLSPServer(
   MsgHandler->bind("typeHierarchy/resolve", &ClangdLSPServer::onResolveTypeHierarchy);
   MsgHandler->bind("textDocument/selectionRange", &ClangdLSPServer::onSelectionRange);
   MsgHandler->bind("textDocument/documentLink", &ClangdLSPServer::onDocumentLink);
+  MsgHandler->bind("textDocument/semanticTokens", &ClangdLSPServer::onSemanticTokens);
+  MsgHandler->bind("textDocument/semanticTokens/edits", &ClangdLSPServer::onSemanticTokensEdits);
   // clang-format on
 }
 
diff --git a/clang-tools-extra/clangd/ClangdLSPServer.h b/clang-tools-extra/clangd/ClangdLSPServer.h
index c4e9e5fb679c5..e259ad04a8e99 100644
--- a/clang-tools-extra/clangd/ClangdLSPServer.h
+++ b/clang-tools-extra/clangd/ClangdLSPServer.h
@@ -119,6 +119,9 @@ class ClangdLSPServer : private ClangdServer::Callbacks {
                         Callback<std::vector<SelectionRange>>);
   void onDocumentLink(const DocumentLinkParams &,
                       Callback<std::vector<DocumentLink>>);
+  void onSemanticTokens(const SemanticTokensParams &, Callback<SemanticTokens>);
+  void onSemanticTokensEdits(const SemanticTokensEditsParams &,
+                             Callback<SemanticTokensOrEdits>);
 
   std::vector<Fix> getFixes(StringRef File, const clangd::Diagnostic &D);
 
@@ -161,6 +164,9 @@ class ClangdLSPServer : private ClangdServer::Callbacks {
   llvm::StringMap<DiagnosticToReplacementMap> FixItsMap;
   std::mutex HighlightingsMutex;
   llvm::StringMap<std::vector<HighlightingToken>> FileToHighlightings;
+  // Last semantic-tokens response, for incremental requests.
+  std::mutex SemanticTokensMutex;
+  llvm::StringMap<SemanticTokens> LastSemanticTokens;
 
   // Most code should not deal with Transport directly.
   // MessageHandler deals with incoming messages, use call() etc for outgoing.
diff --git a/clang-tools-extra/clangd/ClangdServer.cpp b/clang-tools-extra/clangd/ClangdServer.cpp
index 3d68f85b6487d..c1773258554ce 100644
--- a/clang-tools-extra/clangd/ClangdServer.cpp
+++ b/clang-tools-extra/clangd/ClangdServer.cpp
@@ -134,7 +134,8 @@ ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB,
                      : nullptr),
       GetClangTidyOptions(Opts.GetClangTidyOptions),
       SuggestMissingIncludes(Opts.SuggestMissingIncludes),
-      TweakFilter(Opts.TweakFilter), WorkspaceRoot(Opts.WorkspaceRoot),
+      BuildRecoveryAST(Opts.BuildRecoveryAST), TweakFilter(Opts.TweakFilter),
+      WorkspaceRoot(Opts.WorkspaceRoot),
       // Pass a callback into `WorkScheduler` to extract symbols from a newly
       // parsed file and rebuild the file index synchronously each time an AST
       // is parsed.
@@ -191,6 +192,7 @@ void ClangdServer::addDocument(PathRef File, llvm::StringRef Contents,
   Inputs.ForceRebuild = ForceRebuild;
   Inputs.Opts = std::move(Opts);
   Inputs.Index = Index;
+  Inputs.Opts.BuildRecoveryAST = BuildRecoveryAST;
   bool NewFile = WorkScheduler.update(File, Inputs, WantDiags);
   // If we loaded Foo.h, we want to make sure Foo.cpp is indexed.
   if (NewFile && BackgroundIdx)
@@ -269,9 +271,13 @@ void ClangdServer::signatureHelp(PathRef File, Position Pos,
     if (!IP)
       return CB(IP.takeError());
 
-    auto PreambleData = IP->Preamble;
-    CB(clangd::signatureHelp(File, IP->Command, PreambleData, IP->Contents, Pos,
-                             FS, Index));
+    const auto *PreambleData = IP->Preamble;
+    if (!PreambleData)
+      return CB(llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                        "Failed to parse includes"));
+
+    CB(clangd::signatureHelp(File, IP->Command, *PreambleData, IP->Contents,
+                             Pos, FS, Index));
   };
 
   // Unlike code completion, we wait for an up-to-date preamble here.
@@ -654,14 +660,22 @@ void ClangdServer::symbolInfo(PathRef File, Position Pos,
   WorkScheduler.runWithAST("SymbolInfo", File, std::move(Action));
 }
 
-void ClangdServer::semanticRanges(PathRef File, Position Pos,
-                                  Callback<std::vector<Range>> CB) {
-  auto Action =
-      [Pos, CB = std::move(CB)](llvm::Expected<InputsAndAST> InpAST) mutable {
-        if (!InpAST)
-          return CB(InpAST.takeError());
-        CB(clangd::getSemanticRanges(InpAST->AST, Pos));
-      };
+void ClangdServer::semanticRanges(PathRef File,
+                                  const std::vector<Position> &Positions,
+                                  Callback<std::vector<SelectionRange>> CB) {
+  auto Action = [Positions, CB = std::move(CB)](
+                    llvm::Expected<InputsAndAST> InpAST) mutable {
+    if (!InpAST)
+      return CB(InpAST.takeError());
+    std::vector<SelectionRange> Result;
+    for (const auto &Pos : Positions) {
+      if (auto Range = clangd::getSemanticRanges(InpAST->AST, Pos))
+        Result.push_back(std::move(*Range));
+      else
+        return CB(Range.takeError());
+    }
+    CB(std::move(Result));
+  };
   WorkScheduler.runWithAST("SemanticRanges", File, std::move(Action));
 }
 
@@ -677,6 +691,18 @@ void ClangdServer::documentLinks(PathRef File,
                            TUScheduler::InvalidateOnUpdate);
 }
 
+void ClangdServer::semanticHighlights(
+    PathRef File, Callback<std::vector<HighlightingToken>> CB) {
+  auto Action =
+      [CB = std::move(CB)](llvm::Expected<InputsAndAST> InpAST) mutable {
+        if (!InpAST)
+          return CB(InpAST.takeError());
+        CB(clangd::getSemanticHighlightings(InpAST->AST));
+      };
+  WorkScheduler.runWithAST("SemanticHighlights", File, std::move(Action),
+                           TUScheduler::InvalidateOnUpdate);
+}
+
 std::vector<std::pair<Path, std::size_t>>
 ClangdServer::getUsedBytesPerFile() const {
   return WorkScheduler.getUsedBytesPerFile();
diff --git a/clang-tools-extra/clangd/ClangdServer.h b/clang-tools-extra/clangd/ClangdServer.h
index ae3dd8a065d8a..ae3da84c42c80 100644
--- a/clang-tools-extra/clangd/ClangdServer.h
+++ b/clang-tools-extra/clangd/ClangdServer.h
@@ -118,6 +118,9 @@ class ClangdServer {
     /// enabled.
     ClangTidyOptionsBuilder GetClangTidyOptions;
 
+    /// If true, turn on the `-frecovery-ast` clang flag.
+    bool BuildRecoveryAST = false;
+
     /// Clangd's workspace root. Relevant for "workspace" operations not bound
     /// to a particular file.
     /// FIXME: If not set, should use the current working directory.
@@ -142,7 +145,7 @@ class ClangdServer {
     /// fetch system include path.
     std::vector<std::string> QueryDriverGlobs;
 
-    /// Enable semantic highlighting features.
+    /// Enable notification-based semantic highlighting.
     bool TheiaSemanticHighlighting = false;
 
     /// Returns true if the tweak should be enabled.
@@ -292,12 +295,15 @@ class ClangdServer {
                   Callback<std::vector<SymbolDetails>> CB);
 
   /// Get semantic ranges around a specified position in a file.
-  void semanticRanges(PathRef File, Position Pos,
-                      Callback<std::vector<Range>> CB);
+  void semanticRanges(PathRef File, const std::vector<Position> &Pos,
+                      Callback<std::vector<SelectionRange>> CB);
 
   /// Get all document links in a file.
   void documentLinks(PathRef File, Callback<std::vector<DocumentLink>> CB);
- 
+
+  void semanticHighlights(PathRef File,
+                          Callback<std::vector<HighlightingToken>>);
+
   /// Returns estimated memory usage for each of the currently open files.
   /// The order of results is unspecified.
   /// Overall memory usage of clangd may be significantly more than reported
@@ -342,6 +348,9 @@ class ClangdServer {
   // can be caused by missing includes (e.g. member access in incomplete type).
   bool SuggestMissingIncludes = false;
 
+  // If true, preserve expressions in AST for broken code.
+  bool BuildRecoveryAST = false;
+
   std::function<bool(const Tweak &)> TweakFilter;
 
   // GUARDED_BY(CachedCompletionFuzzyFindRequestMutex)
diff --git a/clang-tools-extra/clangd/CodeComplete.cpp b/clang-tools-extra/clangd/CodeComplete.cpp
index 344b90ecaa320..b544510ecea10 100644
--- a/clang-tools-extra/clangd/CodeComplete.cpp
+++ b/clang-tools-extra/clangd/CodeComplete.cpp
@@ -1022,7 +1022,7 @@ class SignatureHelpCollector final : public CodeCompleteConsumer {
 struct SemaCompleteInput {
   PathRef FileName;
   const tooling::CompileCommand &Command;
-  const PreambleData *Preamble;
+  const PreambleData &Preamble;
   llvm::StringRef Contents;
   size_t Offset;
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS;
@@ -1054,8 +1054,8 @@ bool semaCodeComplete(std::unique_ptr<CodeCompleteConsumer> Consumer,
                       IncludeStructure *Includes = nullptr) {
   trace::Span Tracer("Sema completion");
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS = Input.VFS;
-  if (Input.Preamble && Input.Preamble->StatCache)
-    VFS = Input.Preamble->StatCache->getConsumingFS(std::move(VFS));
+  if (Input.Preamble.StatCache)
+    VFS = Input.Preamble.StatCache->getConsumingFS(std::move(VFS));
   ParseInputs ParseInput;
   ParseInput.CompileCommand = Input.Command;
   ParseInput.FS = VFS;
@@ -1072,6 +1072,10 @@ bool semaCodeComplete(std::unique_ptr<CodeCompleteConsumer> Consumer,
   FrontendOpts.SkipFunctionBodies = true;
   // Disable typo correction in Sema.
   CI->getLangOpts()->SpellChecking = false;
+  // Code completion won't trigger in delayed template bodies.
+  // This is on-by-default in windows to allow parsing SDK headers; we're only
+  // disabling it for the main-file (not preamble).
+  CI->getLangOpts()->DelayedTemplateParsing = false;
   // Setup code completion.
   FrontendOpts.CodeCompleteOpts = Options;
   FrontendOpts.CodeCompletionAt.FileName = std::string(Input.FileName);
@@ -1095,9 +1099,7 @@ bool semaCodeComplete(std::unique_ptr<CodeCompleteConsumer> Consumer,
   // NOTE: we must call BeginSourceFile after prepareCompilerInstance. Otherwise
   // the remapped buffers do not get freed.
   auto Clang = prepareCompilerInstance(
-      std::move(CI),
-      (Input.Preamble && !CompletingInPreamble) ? &Input.Preamble->Preamble
-                                                : nullptr,
+      std::move(CI), !CompletingInPreamble ? &Input.Preamble.Preamble : nullptr,
       std::move(ContentsBuffer), std::move(VFS), IgnoreDiags);
   Clang->getPreprocessorOpts().SingleFileParseMode = CompletingInPreamble;
   Clang->setCodeCompletionConsumer(Consumer.release());
@@ -1114,8 +1116,7 @@ bool semaCodeComplete(std::unique_ptr<CodeCompleteConsumer> Consumer,
   //  - but Sema code complete won't see them: as part of the preamble, they're
   //    deserialized only when mentioned.
   // Force them to be deserialized so SemaCodeComplete sees them.
-  if (Input.Preamble)
-    loadMainFilePreambleMacros(Clang->getPreprocessor(), *Input.Preamble);
+  loadMainFilePreambleMacros(Clang->getPreprocessor(), Input.Preamble);
   if (Includes)
     Clang->getPreprocessor().addPPCallbacks(
         collectIncludeStructureCallback(Clang->getSourceManager(), Includes));
@@ -1754,12 +1755,12 @@ codeComplete(PathRef FileName, const tooling::CompileCommand &Command,
   return (!Preamble || Opts.RunParser == CodeCompleteOptions::NeverParse)
              ? std::move(Flow).runWithoutSema(Contents, *Offset, VFS)
              : std::move(Flow).run(
-                   {FileName, Command, Preamble, Contents, *Offset, VFS});
+                   {FileName, Command, *Preamble, Contents, *Offset, VFS});
 }
 
 SignatureHelp signatureHelp(PathRef FileName,
                             const tooling::CompileCommand &Command,
-                            const PreambleData *Preamble,
+                            const PreambleData &Preamble,
                             llvm::StringRef Contents, Position Pos,
                             llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS,
                             const SymbolIndex *Index) {
diff --git a/clang-tools-extra/clangd/CodeComplete.h b/clang-tools-extra/clangd/CodeComplete.h
index df06c156049fb..3adea47c89a11 100644
--- a/clang-tools-extra/clangd/CodeComplete.h
+++ b/clang-tools-extra/clangd/CodeComplete.h
@@ -276,7 +276,7 @@ CodeCompleteResult codeComplete(PathRef FileName,
 /// Get signature help at a specified \p Pos in \p FileName.
 SignatureHelp signatureHelp(PathRef FileName,
                             const tooling::CompileCommand &Command,
-                            const PreambleData *Preamble, StringRef Contents,
+                            const PreambleData &Preamble, StringRef Contents,
                             Position Pos,
                             IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS,
                             const SymbolIndex *Index);
diff --git a/clang-tools-extra/clangd/Compiler.h b/clang-tools-extra/clangd/Compiler.h
index ef5386bb0d17b..b7cc174455f31 100644
--- a/clang-tools-extra/clangd/Compiler.h
+++ b/clang-tools-extra/clangd/Compiler.h
@@ -38,6 +38,7 @@ class IgnoreDiagnostics : public DiagnosticConsumer {
 struct ParseOptions {
   tidy::ClangTidyOptions ClangTidyOpts;
   bool SuggestMissingIncludes = false;
+  bool BuildRecoveryAST = false;
 };
 
 /// Information required to run clang, e.g. to parse AST or do code completion.
diff --git a/clang-tools-extra/clangd/Hover.cpp b/clang-tools-extra/clangd/Hover.cpp
index 1d41f0a3e0461..63fe6b1d756c0 100644
--- a/clang-tools-extra/clangd/Hover.cpp
+++ b/clang-tools-extra/clangd/Hover.cpp
@@ -525,13 +525,13 @@ bool isParagraphLineBreak(llvm::StringRef Str, size_t LineBreakIndex) {
   return Str.substr(LineBreakIndex + 1)
       .drop_while([](auto C) { return C == ' ' || C == '\t'; })
       .startswith("\n");
-};
+}
 
 bool isPunctuationLineBreak(llvm::StringRef Str, size_t LineBreakIndex) {
   constexpr llvm::StringLiteral Punctuation = R"txt(.:,;!?)txt";
 
   return LineBreakIndex > 0 && Punctuation.contains(Str[LineBreakIndex - 1]);
-};
+}
 
 bool isFollowedByHardLineBreakIndicator(llvm::StringRef Str,
                                         size_t LineBreakIndex) {
@@ -556,7 +556,7 @@ bool isFollowedByHardLineBreakIndicator(llvm::StringRef Str,
        Str[NextNonSpaceCharIndex + 1] == ')');
 
   return FollowedBySingleCharIndicator || FollowedByNumberedListIndicator;
-};
+}
 
 bool isHardLineBreak(llvm::StringRef Str, size_t LineBreakIndex) {
   return isPunctuationLineBreak(Str, LineBreakIndex) ||
diff --git a/clang-tools-extra/clangd/ParsedAST.cpp b/clang-tools-extra/clangd/ParsedAST.cpp
index 91c71789dbe02..f6205879aa29c 100644
--- a/clang-tools-extra/clangd/ParsedAST.cpp
+++ b/clang-tools-extra/clangd/ParsedAST.cpp
@@ -253,6 +253,10 @@ ParsedAST::build(llvm::StringRef Version,
   const PrecompiledPreamble *PreamblePCH =
       Preamble ? &Preamble->Preamble : nullptr;
 
+  // Recovery expression currently only works for C++.
+  if (CI->getLangOpts()->CPlusPlus)
+    CI->getLangOpts()->RecoveryAST = Opts.BuildRecoveryAST;
+
   StoreDiags ASTDiags;
   std::string Content = std::string(Buffer->getBuffer());
   std::string Filename =
@@ -310,14 +314,13 @@ ParsedAST::build(llvm::StringRef Version,
           // Check for suppression comment. Skip the check for diagnostics not
           // in the main file, because we don't want that function to query the
           // source buffer for preamble files. For the same reason, we ask
-          // shouldSuppressDiagnostic not to follow macro expansions, since
-          // those might take us into a preamble file as well.
+          // shouldSuppressDiagnostic to avoid I/O.
           bool IsInsideMainFile =
               Info.hasSourceManager() &&
               isInsideMainFile(Info.getLocation(), Info.getSourceManager());
-          if (IsInsideMainFile && tidy::shouldSuppressDiagnostic(
-                                      DiagLevel, Info, *CTContext,
-                                      /* CheckMacroExpansion = */ false)) {
+          if (IsInsideMainFile &&
+              tidy::shouldSuppressDiagnostic(DiagLevel, Info, *CTContext,
+                                             /*AllowIO=*/false)) {
             return DiagnosticsEngine::Ignored;
           }
         }
@@ -548,8 +551,7 @@ buildAST(PathRef FileName, std::unique_ptr<CompilerInvocation> Invocation,
   }
 
   return ParsedAST::build(
-      Inputs.Version, std::make_unique<CompilerInvocation>(*Invocation),
-      CompilerInvocationDiags, Preamble,
+      Inputs.Version, std::move(Invocation), CompilerInvocationDiags, Preamble,
       llvm::MemoryBuffer::getMemBufferCopy(Inputs.Contents, FileName),
       std::move(VFS), Inputs.Index, Inputs.Opts);
 }
diff --git a/clang-tools-extra/clangd/Preamble.cpp b/clang-tools-extra/clangd/Preamble.cpp
index fdee71fd22449..48f15420032fa 100644
--- a/clang-tools-extra/clangd/Preamble.cpp
+++ b/clang-tools-extra/clangd/Preamble.cpp
@@ -132,6 +132,10 @@ buildPreamble(PathRef FileName, CompilerInvocation &CI,
   // to read back. We rely on dynamic index for the comments instead.
   CI.getPreprocessorOpts().WriteCommentListToPCH = false;
 
+  // Recovery expression currently only works for C++.
+  if (CI.getLangOpts()->CPlusPlus)
+    CI.getLangOpts()->RecoveryAST = Inputs.Opts.BuildRecoveryAST;
+
   CppFilePreambleCallbacks SerializedDeclsCollector(FileName, PreambleCallback);
   if (Inputs.FS->setCurrentWorkingDirectory(Inputs.CompileCommand.Directory)) {
     log("Couldn't set working directory when building the preamble.");
diff --git a/clang-tools-extra/clangd/Protocol.cpp b/clang-tools-extra/clangd/Protocol.cpp
index 462aebc4b0465..cd8e6bc10dcef 100644
--- a/clang-tools-extra/clangd/Protocol.cpp
+++ b/clang-tools-extra/clangd/Protocol.cpp
@@ -297,6 +297,8 @@ bool fromJSON(const llvm::json::Value &Params, ClientCapabilities &R) {
               SemanticHighlighting->getBoolean("semanticHighlighting"))
         R.TheiaSemanticHighlighting = *SemanticHighlightingSupport;
     }
+    if (auto *SemanticHighlighting = TextDocument->getObject("semanticTokens"))
+      R.SemanticTokens = true;
     if (auto *Diagnostics = TextDocument->getObject("publishDiagnostics")) {
       if (auto CategorySupport = Diagnostics->getBoolean("categorySupport"))
         R.DiagnosticCategory = *CategorySupport;
@@ -984,6 +986,59 @@ llvm::json::Value toJSON(const FileStatus &FStatus) {
   };
 }
 
+constexpr unsigned SemanticTokenEncodingSize = 5;
+static llvm::json::Value encodeTokens(llvm::ArrayRef<SemanticToken> Toks) {
+  llvm::json::Array Result;
+  for (const auto &Tok : Toks) {
+    Result.push_back(Tok.deltaLine);
+    Result.push_back(Tok.deltaStart);
+    Result.push_back(Tok.length);
+    Result.push_back(Tok.tokenType);
+    Result.push_back(Tok.tokenModifiers);
+  }
+  assert(Result.size() == SemanticTokenEncodingSize * Toks.size());
+  return Result;
+}
+
+bool operator==(const SemanticToken &L, const SemanticToken &R) {
+  return std::tie(L.deltaLine, L.deltaStart, L.length, L.tokenType,
+                  L.tokenModifiers) == std::tie(R.deltaLine, R.deltaStart,
+                                                R.length, R.tokenType,
+                                                R.tokenModifiers);
+}
+
+llvm::json::Value toJSON(const SemanticTokens &Tokens) {
+  return llvm::json::Object{{"resultId", Tokens.resultId},
+                            {"data", encodeTokens(Tokens.tokens)}};
+}
+
+llvm::json::Value toJSON(const SemanticTokensEdit &Edit) {
+  return llvm::json::Object{
+      {"start", SemanticTokenEncodingSize * Edit.startToken},
+      {"deleteCount", SemanticTokenEncodingSize * Edit.deleteTokens},
+      {"data", encodeTokens(Edit.tokens)}};
+}
+
+llvm::json::Value toJSON(const SemanticTokensOrEdits &TE) {
+  llvm::json::Object Result{{"resultId", TE.resultId}};
+  if (TE.edits)
+    Result["edits"] = *TE.edits;
+  if (TE.tokens)
+    Result["data"] = encodeTokens(*TE.tokens);
+  return Result;
+}
+
+bool fromJSON(const llvm::json::Value &Params, SemanticTokensParams &R) {
+  llvm::json::ObjectMapper O(Params);
+  return O && O.map("textDocument", R.textDocument);
+}
+
+bool fromJSON(const llvm::json::Value &Params, SemanticTokensEditsParams &R) {
+  llvm::json::ObjectMapper O(Params);
+  return O && O.map("textDocument", R.textDocument) &&
+         O.map("previousResultId", R.previousResultId);
+}
+
 llvm::raw_ostream &operator<<(llvm::raw_ostream &O,
                               const DocumentHighlight &V) {
   O << V.range;
diff --git a/clang-tools-extra/clangd/Protocol.h b/clang-tools-extra/clangd/Protocol.h
index d08e546e3ffb2..968b039c62cca 100644
--- a/clang-tools-extra/clangd/Protocol.h
+++ b/clang-tools-extra/clangd/Protocol.h
@@ -433,8 +433,13 @@ struct ClientCapabilities {
   /// textDocument.codeAction.codeActionLiteralSupport.
   bool CodeActionStructure = false;
 
+  /// Client advertises support for the semanticTokens feature.
+  /// We support the textDocument/semanticTokens request in any case.
+  /// textDocument.semanticTokens
+  bool SemanticTokens = false;
   /// Client supports Theia semantic highlighting extension.
   /// https://github.com/microsoft/vscode-languageserver-node/pull/367
+  /// This will be ignored if the client also supports semanticTokens.
   /// textDocument.semanticHighlightingCapabilities.semanticHighlighting
   /// FIXME: drop this support once clients support LSP 3.16 Semantic Tokens.
   bool TheiaSemanticHighlighting = false;
@@ -1340,7 +1345,77 @@ struct FileStatus {
   std::string state;
   // FIXME: add detail messages.
 };
-llvm::json::Value toJSON(const FileStatus &FStatus);
+llvm::json::Value toJSON(const FileStatus &);
+
+/// Specifies a single semantic token in the document.
+/// This struct is not part of LSP, which just encodes lists of tokens as
+/// arrays of numbers directly.
+struct SemanticToken {
+  /// token line number, relative to the previous token
+  unsigned deltaLine = 0;
+  /// token start character, relative to the previous token
+  /// (relative to 0 or the previous token's start if they are on the same line)
+  unsigned deltaStart = 0;
+  /// the length of the token. A token cannot be multiline
+  unsigned length = 0;
+  /// will be looked up in `SemanticTokensLegend.tokenTypes`
+  unsigned tokenType = 0;
+  /// each set bit will be looked up in `SemanticTokensLegend.tokenModifiers`
+  unsigned tokenModifiers = 0;
+};
+bool operator==(const SemanticToken &, const SemanticToken &);
+
+/// A versioned set of tokens.
+struct SemanticTokens {
+  // An optional result id. If provided and clients support delta updating
+  // the client will include the result id in the next semantic token request.
+  // A server can then instead of computing all semantic tokens again simply
+  // send a delta.
+  std::string resultId;
+
+  /// The actual tokens. For a detailed description about how the data is
+  /// structured pls see
+  /// https://github.com/microsoft/vscode-extension-samples/blob/5ae1f7787122812dcc84e37427ca90af5ee09f14/semantic-tokens-sample/vscode.proposed.d.ts#L71
+  std::vector<SemanticToken> tokens;
+};
+llvm::json::Value toJSON(const SemanticTokens &);
+
+struct SemanticTokensParams {
+  /// The text document.
+  TextDocumentIdentifier textDocument;
+};
+bool fromJSON(const llvm::json::Value &, SemanticTokensParams &);
+
+/// Requests the changes in semantic tokens since a previous response.
+struct SemanticTokensEditsParams {
+  /// The text document.
+  TextDocumentIdentifier textDocument;
+  /// The previous result id.
+  std::string previousResultId;
+};
+bool fromJSON(const llvm::json::Value &Params, SemanticTokensEditsParams &R);
+
+/// Describes a a replacement of a contiguous range of semanticTokens.
+struct SemanticTokensEdit {
+  // LSP specifies `start` and `deleteCount` which are relative to the array
+  // encoding of the previous tokens.
+  // We use token counts instead, and translate when serializing this struct.
+  unsigned startToken = 0;
+  unsigned deleteTokens = 0;
+  std::vector<SemanticToken> tokens;
+};
+llvm::json::Value toJSON(const SemanticTokensEdit &);
+
+/// This models LSP SemanticTokensEdits | SemanticTokens, which is the result of
+/// textDocument/semanticTokens/edits.
+struct SemanticTokensOrEdits {
+  std::string resultId;
+  /// Set if we computed edits relative to a previous set of tokens.
+  llvm::Optional<std::vector<SemanticTokensEdit>> edits;
+  /// Set if we computed a fresh set of tokens.
+  llvm::Optional<std::vector<SemanticToken>> tokens;
+};
+llvm::json::Value toJSON(const SemanticTokensOrEdits &);
 
 /// Represents a semantic highlighting information that has to be applied on a
 /// specific line of the text document.
diff --git a/clang-tools-extra/clangd/SemanticHighlighting.cpp b/clang-tools-extra/clangd/SemanticHighlighting.cpp
index b69f9e8f2710e..35eabfe00dd24 100644
--- a/clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ b/clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -445,6 +445,84 @@ bool operator==(const LineHighlightings &L, const LineHighlightings &R) {
   return std::tie(L.Line, L.Tokens) == std::tie(R.Line, R.Tokens);
 }
 
+std::vector<SemanticToken>
+toSemanticTokens(llvm::ArrayRef<HighlightingToken> Tokens) {
+  assert(std::is_sorted(Tokens.begin(), Tokens.end()));
+  std::vector<SemanticToken> Result;
+  const HighlightingToken *Last = nullptr;
+  for (const HighlightingToken &Tok : Tokens) {
+    // FIXME: support inactive code - we need to provide the actual bounds.
+    if (Tok.Kind == HighlightingKind::InactiveCode)
+      continue;
+    Result.emplace_back();
+    SemanticToken &Out = Result.back();
+    // deltaStart/deltaLine are relative if possible.
+    if (Last) {
+      assert(Tok.R.start.line >= Last->R.start.line);
+      Out.deltaLine = Tok.R.start.line - Last->R.start.line;
+      if (Out.deltaLine == 0) {
+        assert(Tok.R.start.character >= Last->R.start.character);
+        Out.deltaStart = Tok.R.start.character - Last->R.start.character;
+      } else {
+        Out.deltaStart = Tok.R.start.character;
+      }
+    } else {
+      Out.deltaLine = Tok.R.start.line;
+      Out.deltaStart = Tok.R.start.character;
+    }
+    assert(Tok.R.end.line == Tok.R.start.line);
+    Out.length = Tok.R.end.character - Tok.R.start.character;
+    Out.tokenType = static_cast<unsigned>(Tok.Kind);
+
+    Last = &Tok;
+  }
+  return Result;
+}
+llvm::StringRef toSemanticTokenType(HighlightingKind Kind) {
+  switch (Kind) {
+  case HighlightingKind::Variable:
+  case HighlightingKind::LocalVariable:
+  case HighlightingKind::StaticField:
+    return "variable";
+  case HighlightingKind::Parameter:
+    return "parameter";
+  case HighlightingKind::Function:
+    return "function";
+  case HighlightingKind::Method:
+    return "member";
+  case HighlightingKind::StaticMethod:
+    // FIXME: better function/member with static modifier?
+    return "function";
+  case HighlightingKind::Field:
+    return "member";
+  case HighlightingKind::Class:
+    return "class";
+  case HighlightingKind::Enum:
+    return "enum";
+  case HighlightingKind::EnumConstant:
+    return "enumConstant"; // nonstandard
+  case HighlightingKind::Typedef:
+    return "type";
+  case HighlightingKind::DependentType:
+    return "dependent"; // nonstandard
+  case HighlightingKind::DependentName:
+    return "dependent"; // nonstandard
+  case HighlightingKind::Namespace:
+    return "namespace";
+  case HighlightingKind::TemplateParameter:
+    return "typeParameter";
+  case HighlightingKind::Concept:
+    return "concept"; // nonstandard
+  case HighlightingKind::Primitive:
+    return "type";
+  case HighlightingKind::Macro:
+    return "macro";
+  case HighlightingKind::InactiveCode:
+    return "comment";
+  }
+  llvm_unreachable("unhandled HighlightingKind");
+}
+
 std::vector<TheiaSemanticHighlightingInformation>
 toTheiaSemanticHighlightingInformation(
     llvm::ArrayRef<LineHighlightings> Tokens) {
@@ -522,5 +600,31 @@ llvm::StringRef toTextMateScope(HighlightingKind Kind) {
   llvm_unreachable("unhandled HighlightingKind");
 }
 
+std::vector<SemanticTokensEdit>
+diffTokens(llvm::ArrayRef<SemanticToken> Old,
+           llvm::ArrayRef<SemanticToken> New) {
+  // For now, just replace everything from the first-last modification.
+  // FIXME: use a real diff instead, this is bad with include-insertion.
+
+  unsigned Offset = 0;
+  while (!Old.empty() && !New.empty() && Old.front() == New.front()) {
+    ++Offset;
+    Old = Old.drop_front();
+    New = New.drop_front();
+  }
+  while (!Old.empty() && !New.empty() && Old.back() == New.back()) {
+    Old = Old.drop_back();
+    New = New.drop_back();
+  }
+
+  if (Old.empty() && New.empty())
+    return {};
+  SemanticTokensEdit Edit;
+  Edit.startToken = Offset;
+  Edit.deleteTokens = Old.size();
+  Edit.tokens = New;
+  return {std::move(Edit)};
+}
+
 } // namespace clangd
 } // namespace clang
diff --git a/clang-tools-extra/clangd/SemanticHighlighting.h b/clang-tools-extra/clangd/SemanticHighlighting.h
index 31a97b81d6c24..9a96cc28c4f5e 100644
--- a/clang-tools-extra/clangd/SemanticHighlighting.h
+++ b/clang-tools-extra/clangd/SemanticHighlighting.h
@@ -6,8 +6,21 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// An implementation of semantic highlighting based on this proposal:
-// https://github.com/microsoft/vscode-languageserver-node/pull/367 in clangd.
+// This file supports semantic highlighting: categorizing tokens in the file so
+// that the editor can color/style them differently.
+//
+// This is particularly valuable for C++: its complex and context-dependent
+// grammar is a challenge for simple syntax-highlighting techniques.
+//
+// We support two protocols for providing highlights to the client:
+// - the `textDocument/semanticTokens` request from LSP 3.16
+//   https://github.com/microsoft/vscode-languageserver-node/blob/release/protocol/3.16.0-next.1/protocol/src/protocol.semanticTokens.proposed.ts
+// - the earlier proposed `textDocument/semanticHighlighting` notification
+//   https://github.com/microsoft/vscode-languageserver-node/pull/367
+//   This is referred to as "Theia" semantic highlighting in the code.
+//   It was supported from clangd 9 but should be considered deprecated as of
+//   clangd 11 and eventually removed.
+//
 // Semantic highlightings are calculated for an AST by visiting every AST node
 // and classifying nodes that are interesting to highlight (variables/function
 // calls etc.).
@@ -75,6 +88,11 @@ bool operator==(const LineHighlightings &L, const LineHighlightings &R);
 // main AST.
 std::vector<HighlightingToken> getSemanticHighlightings(ParsedAST &AST);
 
+std::vector<SemanticToken> toSemanticTokens(llvm::ArrayRef<HighlightingToken>);
+llvm::StringRef toSemanticTokenType(HighlightingKind Kind);
+std::vector<SemanticTokensEdit> diffTokens(llvm::ArrayRef<SemanticToken> Before,
+                                           llvm::ArrayRef<SemanticToken> After);
+
 /// Converts a HighlightingKind to a corresponding TextMate scope
 /// (https://manual.macromates.com/en/language_grammars).
 llvm::StringRef toTextMateScope(HighlightingKind Kind);
diff --git a/clang-tools-extra/clangd/SemanticSelection.cpp b/clang-tools-extra/clangd/SemanticSelection.cpp
index 49fc7703db057..a6b1ebfb83275 100644
--- a/clang-tools-extra/clangd/SemanticSelection.cpp
+++ b/clang-tools-extra/clangd/SemanticSelection.cpp
@@ -12,6 +12,7 @@
 #include "SourceCode.h"
 #include "clang/AST/DeclBase.h"
 #include "clang/Basic/SourceLocation.h"
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/Support/Error.h"
 
 namespace clang {
@@ -26,9 +27,8 @@ void addIfDistinct(const Range &R, std::vector<Range> &Result) {
 }
 } // namespace
 
-llvm::Expected<std::vector<Range>> getSemanticRanges(ParsedAST &AST,
-                                                     Position Pos) {
-  std::vector<Range> Result;
+llvm::Expected<SelectionRange> getSemanticRanges(ParsedAST &AST, Position Pos) {
+  std::vector<Range> Ranges;
   const auto &SM = AST.getSourceManager();
   const auto &LangOpts = AST.getLangOpts();
 
@@ -56,9 +56,29 @@ llvm::Expected<std::vector<Range>> getSemanticRanges(ParsedAST &AST,
     Range R;
     R.start = sourceLocToPosition(SM, SR->getBegin());
     R.end = sourceLocToPosition(SM, SR->getEnd());
-    addIfDistinct(R, Result);
+    addIfDistinct(R, Ranges);
   }
-  return Result;
+
+  if (Ranges.empty()) {
+    // LSP provides no way to signal "the point is not within a semantic range".
+    // Return an empty range at the point.
+    SelectionRange Empty;
+    Empty.range.start = Empty.range.end = Pos;
+    return std::move(Empty);
+  }
+
+  // Convert to the LSP linked-list representation.
+  SelectionRange Head;
+  Head.range = std::move(Ranges.front());
+  SelectionRange *Tail = &Head;
+  for (auto &Range :
+       llvm::makeMutableArrayRef(Ranges.data(), Ranges.size()).drop_front()) {
+    Tail->parent = std::make_unique<SelectionRange>();
+    Tail = Tail->parent.get();
+    Tail->range = std::move(Range);
+  }
+
+  return std::move(Head);
 }
 
 } // namespace clangd
diff --git a/clang-tools-extra/clangd/SemanticSelection.h b/clang-tools-extra/clangd/SemanticSelection.h
index b0d301e2efb89..810cc21d9a58f 100644
--- a/clang-tools-extra/clangd/SemanticSelection.h
+++ b/clang-tools-extra/clangd/SemanticSelection.h
@@ -22,10 +22,9 @@ namespace clangd {
 /// Returns the list of all interesting ranges around the Position \p Pos.
 /// The interesting ranges corresponds to the AST nodes in the SelectionTree
 /// containing \p Pos.
-/// Any range in the result strictly contains all the previous ranges in the
-/// result. front() is the inner most range. back() is the outermost range.
-llvm::Expected<std::vector<Range>> getSemanticRanges(ParsedAST &AST,
-                                                     Position Pos);
+/// If pos is not in any interesting range, return [Pos, Pos).
+llvm::Expected<SelectionRange> getSemanticRanges(ParsedAST &AST, Position Pos);
+
 } // namespace clangd
 } // namespace clang
 
diff --git a/clang-tools-extra/clangd/TUScheduler.cpp b/clang-tools-extra/clangd/TUScheduler.cpp
index e5a001997ecc5..a2df2c3ff62f4 100644
--- a/clang-tools-extra/clangd/TUScheduler.cpp
+++ b/clang-tools-extra/clangd/TUScheduler.cpp
@@ -569,12 +569,10 @@ void ASTWorker::runWithAST(
       vlog("ASTWorker rebuilding evicted AST to run {0}: {1} version {2}", Name,
            FileName, CurrentInputs->Version);
       llvm::Optional<ParsedAST> NewAST =
-          Invocation
-              ? buildAST(FileName,
-                         std::make_unique<CompilerInvocation>(*Invocation),
-                         CompilerInvocationDiagConsumer.take(), *CurrentInputs,
-                         getPossiblyStalePreamble())
-              : None;
+          Invocation ? buildAST(FileName, std::move(Invocation),
+                                CompilerInvocationDiagConsumer.take(),
+                                *CurrentInputs, getPossiblyStalePreamble())
+                     : None;
       AST = NewAST ? std::make_unique<ParsedAST>(std::move(*NewAST)) : nullptr;
     }
     // Make sure we put the AST back into the LRU cache.
diff --git a/clang-tools-extra/clangd/XRefs.cpp b/clang-tools-extra/clangd/XRefs.cpp
index 7d55a372905c0..2e2e6602c8d3e 100644
--- a/clang-tools-extra/clangd/XRefs.cpp
+++ b/clang-tools-extra/clangd/XRefs.cpp
@@ -43,6 +43,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/None.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Casting.h"
@@ -226,23 +227,21 @@ locateASTReferent(SourceLocation CurLoc, const syntax::Token *TouchedIdentifier,
   llvm::DenseMap<SymbolID, size_t> ResultIndex;
 
   auto AddResultDecl = [&](const NamedDecl *D) {
-    const NamedDecl *Def = getDefinition(D);
-    const NamedDecl *Preferred = Def ? Def : D;
-
-    auto Loc = makeLocation(AST.getASTContext(), nameLocation(*Preferred, SM),
-                            MainFilePath);
+    D = llvm::cast<NamedDecl>(D->getCanonicalDecl());
+    auto Loc =
+        makeLocation(AST.getASTContext(), nameLocation(*D, SM), MainFilePath);
     if (!Loc)
       return;
 
     Result.emplace_back();
-    Result.back().Name = printName(AST.getASTContext(), *Preferred);
+    Result.back().Name = printName(AST.getASTContext(), *D);
     Result.back().PreferredDeclaration = *Loc;
-    // Preferred is always a definition if possible, so this check works.
-    if (Def == Preferred)
-      Result.back().Definition = *Loc;
+    if (const NamedDecl *Def = getDefinition(D))
+      Result.back().Definition = makeLocation(
+          AST.getASTContext(), nameLocation(*Def, SM), MainFilePath);
 
     // Record SymbolID for index lookup later.
-    if (auto ID = getSymbolID(Preferred))
+    if (auto ID = getSymbolID(D))
       ResultIndex[*ID] = Result.size() - 1;
   };
 
@@ -585,13 +584,11 @@ class ReferenceFinder : public index::IndexDataConsumer {
                        SourceLocation Loc,
                        index::IndexDataConsumer::ASTNodeInfo ASTNode) override {
     assert(D->isCanonicalDecl() && "expect D to be a canonical declaration");
-    if (!CanonicalTargets.count(D))
+    const SourceManager &SM = AST.getSourceManager();
+    if (!CanonicalTargets.count(D) || !isInsideMainFile(Loc, SM))
       return true;
     const auto &TB = AST.getTokens();
-    const SourceManager &SM = AST.getSourceManager();
     Loc = SM.getFileLoc(Loc);
-    // We are only traversing decls *inside* the main file, so this should hold.
-    assert(isInsideMainFile(Loc, SM));
     if (const auto *Tok = TB.spelledTokenAt(Loc))
       References.push_back({*Tok, Roles});
     return true;
diff --git a/clang-tools-extra/clangd/clients/clangd-vscode/.vscode/launch.json b/clang-tools-extra/clangd/clients/clangd-vscode/.vscode/launch.json
index cd6b87bd05c0a..7d414bc00f32e 100644
--- a/clang-tools-extra/clangd/clients/clangd-vscode/.vscode/launch.json
+++ b/clang-tools-extra/clangd/clients/clangd-vscode/.vscode/launch.json
@@ -1,4 +1,4 @@
-// A launch configuration that compiles the extension and then opens it inside a new window
+// A launch configuration that compiles extension and opens it inside a new window.
 {
     "version": "0.1.0",
     "configurations": [
diff --git a/clang-tools-extra/clangd/clients/clangd-vscode/.vscode/tasks.json b/clang-tools-extra/clangd/clients/clangd-vscode/.vscode/tasks.json
index fb7f662e14d14..65b1c9598c0e1 100644
--- a/clang-tools-extra/clangd/clients/clangd-vscode/.vscode/tasks.json
+++ b/clang-tools-extra/clangd/clients/clangd-vscode/.vscode/tasks.json
@@ -6,25 +6,27 @@
 // ${fileExtname}: the current opened file's extension
 // ${cwd}: the current working directory of the spawned process
 
-// A task runner that calls a custom npm script that compiles the extension.
+// Task runner calls custom npm script to compile the extension.
 {
-    "version": "0.1.0",
+    "version": "2.0.0",
 
-    // we want to run npm
+    // Run NPM.
     "command": "npm",
 
-    // the command is a shell script
-    "isShellCommand": true,
+    // This command is a shell script.
+    "type": "shell",
 
     // show the output window only if unrecognized errors occur.
-    "showOutput": "silent",
+    "presentation": {
+        "reveal": "silent",
+    },
 
-    // we run the custom script "compile" as defined in package.json
+    // Run custom "compile" script as defined in package.json
     "args": ["run", "compile", "--loglevel", "silent"],
 
-    // The tsc compiler is started in watching mode
-    "isWatching": true,
+    // tsc compiler is kept alive and runs in the background.
+    "isBackground": true,
 
-    // use the standard tsc in watch mode problem matcher to find compile problems in the output.
+    // Find compilation problems in the output through tsc in watch mode.
     "problemMatcher": "$tsc-watch"
-}
\ No newline at end of file
+}
diff --git a/clang-tools-extra/clangd/clients/clangd-vscode/DEVELOPING.md b/clang-tools-extra/clangd/clients/clangd-vscode/DEVELOPING.md
index e888aba3ea205..15f2b930329e9 100644
--- a/clang-tools-extra/clangd/clients/clangd-vscode/DEVELOPING.md
+++ b/clang-tools-extra/clangd/clients/clangd-vscode/DEVELOPING.md
@@ -10,20 +10,20 @@ A guide of developing `vscode-clangd` extension.
 ## Steps
 
 1. Make sure you disable the installed `vscode-clangd` extension in VS Code.
-2. Make sure you have clangd in /usr/bin/clangd or edit src/extension.ts to
+2. Make sure you have clangd in `/usr/bin/clangd` or edit `src/extension.ts` to
 point to the binary.
-3. In order to start a development instance of VS code extended with this, run:
+3. To start a development instance of VS code extended with this, run:
 
 ```bash
    $ cd /path/to/clang-tools-extra/clangd/clients/clangd-vscode/
    $ npm install
    $ code .
-   # When VS Code starts, press <F5>.
+   # When VSCode starts, press <F5>.
 ```
 
 # Contributing
 
-Please follow the exsiting code style when contributing to the extension, we
+Please follow the existing code style when contributing to the extension, we
 recommend to run `npm run format` before sending a patch.
 
 # Publish to VS Code Marketplace
@@ -38,15 +38,15 @@ to the marketplace.
 * Bump the version in `package.json`, and commit the change to upstream
 
 The extension is published under `llvm-vs-code-extensions` account, which is
-currently maintained by clangd developers. If you want to make a new release,
-please contact clangd-dev@lists.llvm.org.
+maintained by clangd developers. If you want to make a new release, please
+contact clangd-dev@lists.llvm.org.
 
 ## Steps
 
 ```bash
   $ cd /path/to/clang-tools-extra/clangd/clients/clangd-vscode/
-  # For the first time, you need to login in the account. vsce will ask you for
-    the Personal Access Token, and remember it for future commands.
+  # For the first time, you need to login into the account. vsce will ask you
+    for the Personal Access Token and will remember it for future commands.
   $ vsce login llvm-vs-code-extensions
   # Publish the extension to the VSCode marketplace.
   $ npm run publish
diff --git a/clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts b/clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts
index 4749cd1bb5820..a7570b63e552e 100644
--- a/clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts
+++ b/clang-tools-extra/clangd/clients/clangd-vscode/src/extension.ts
@@ -3,7 +3,7 @@ import * as vscodelc from 'vscode-languageclient';
 import * as semanticHighlighting from './semantic-highlighting';
 
 /**
- * Method to get workspace configuration option
+ * Get an option from workspace configuration.
  * @param option name of the option (e.g. for clangd.path should be path)
  * @param defaultValue default value to return if option is not set
  */
@@ -75,8 +75,8 @@ class EnableEditsNearCursorFeature implements vscodelc.StaticFeature {
 }
 
 /**
- *  this method is called when your extension is activate
- *  your extension is activated the very first time the command is executed
+ *  This method is called when the extension is activated. The extension is
+ *  activated the very first time a command is executed.
  */
 export function activate(context: vscode.ExtensionContext) {
   const syncFileEvents = getConfig<boolean>('syncFileEvents', true);
@@ -97,7 +97,7 @@ export function activate(context: vscode.ExtensionContext) {
         documentSelector: [
             { scheme: 'file', language: 'c' },
             { scheme: 'file', language: 'cpp' },
-            // cuda is not supported by vscode, but our extension does.
+            // CUDA is not supported by vscode, but our extension does supports it.
             { scheme: 'file', language: 'cuda' },
             { scheme: 'file', language: 'objective-c'},
             { scheme: 'file', language: 'objective-cpp'}
@@ -106,7 +106,7 @@ export function activate(context: vscode.ExtensionContext) {
         // FIXME: send sync file events when clangd provides implementations.
         },
         initializationOptions: { clangdFileStatus: true },
-        // Do not switch to output window when clangd returns output
+        // Do not switch to output window when clangd returns output.
         revealOutputChannelOn: vscodelc.RevealOutputChannelOn.Never,
 
         // We hack up the completion items a bit to prevent VSCode from re-ranking them
@@ -126,7 +126,7 @@ export function activate(context: vscode.ExtensionContext) {
           provideCompletionItem: async (document, position, context, token, next) => {
             let list = await next(document, position, context, token);
             let items = (Array.isArray(list) ? list : list.items).map(item => {
-              // Gets the prefix used by vscode when doing fuzzymatch.
+              // Gets the prefix used by VSCode when doing fuzzymatch.
               let prefix = document.getText(new vscode.Range(item.range.start, position))
               if (prefix)
                 item.filterText = prefix + "_" + item.filterText;
diff --git a/clang-tools-extra/clangd/clients/clangd-vscode/tsconfig.json b/clang-tools-extra/clangd/clients/clangd-vscode/tsconfig.json
index 0b05f3090920a..71a62c71da025 100644
--- a/clang-tools-extra/clangd/clients/clangd-vscode/tsconfig.json
+++ b/clang-tools-extra/clangd/clients/clangd-vscode/tsconfig.json
@@ -26,4 +26,4 @@
         "node_modules",
         ".vscode-test"
     ]
-}
\ No newline at end of file
+}
diff --git a/clang-tools-extra/clangd/index/Relation.cpp b/clang-tools-extra/clangd/index/Relation.cpp
index e28591bb64b00..6daa10a6f95ea 100644
--- a/clang-tools-extra/clangd/index/Relation.cpp
+++ b/clang-tools-extra/clangd/index/Relation.cpp
@@ -26,7 +26,7 @@ RelationSlab::lookup(const SymbolID &Subject, RelationKind Predicate) const {
 
 RelationSlab RelationSlab::Builder::build() && {
   // Sort in SPO order.
-  std::sort(Relations.begin(), Relations.end());
+  llvm::sort(Relations);
 
   // Remove duplicates.
   Relations.erase(std::unique(Relations.begin(), Relations.end()),
diff --git a/clang-tools-extra/clangd/refactor/tweaks/AddUsing.cpp b/clang-tools-extra/clangd/refactor/tweaks/AddUsing.cpp
new file mode 100644
index 0000000000000..1ecec6674b022
--- /dev/null
+++ b/clang-tools-extra/clangd/refactor/tweaks/AddUsing.cpp
@@ -0,0 +1,286 @@
+//===--- AddUsing.cpp --------------------------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AST.h"
+#include "FindTarget.h"
+#include "Logger.h"
+#include "refactor/Tweak.h"
+#include "clang/AST/Decl.h"
+#include "clang/AST/RecursiveASTVisitor.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+// Tweak for removing full namespace qualifier under cursor on DeclRefExpr and
+// types and adding "using" statement instead.
+//
+// Only qualifiers that refer exclusively to namespaces (no record types) are
+// supported. There is some guessing of appropriate place to insert the using
+// declaration. If we find any existing usings, we insert it there. If not, we
+// insert right after the inner-most relevant namespace declaration. If there is
+// none, or there is, but it was declared via macro, we insert above the first
+// top level decl.
+//
+// Currently this only removes qualifier from under the cursor. In the future,
+// we should improve this to remove qualifier from all occurences of this
+// symbol.
+class AddUsing : public Tweak {
+public:
+  const char *id() const override;
+
+  bool prepare(const Selection &Inputs) override;
+  Expected<Effect> apply(const Selection &Inputs) override;
+  std::string title() const override;
+  Intent intent() const override { return Refactor; }
+
+private:
+  // The qualifier to remove. Set by prepare().
+  NestedNameSpecifierLoc QualifierToRemove;
+  // The name following QualifierToRemove. Set by prepare().
+  llvm::StringRef Name;
+};
+REGISTER_TWEAK(AddUsing)
+
+std::string AddUsing::title() const {
+  return std::string(llvm::formatv(
+      "Add using-declaration for {0} and remove qualifier.", Name));
+}
+
+// Locates all "using" statements relevant to SelectionDeclContext.
+class UsingFinder : public RecursiveASTVisitor<UsingFinder> {
+public:
+  UsingFinder(std::vector<const UsingDecl *> &Results,
+              const DeclContext *SelectionDeclContext, const SourceManager &SM)
+      : Results(Results), SelectionDeclContext(SelectionDeclContext), SM(SM) {}
+
+  bool VisitUsingDecl(UsingDecl *D) {
+    auto Loc = D->getUsingLoc();
+    if (SM.getFileID(Loc) != SM.getMainFileID()) {
+      return true;
+    }
+    if (D->getDeclContext()->Encloses(SelectionDeclContext)) {
+      Results.push_back(D);
+    }
+    return true;
+  }
+
+  bool TraverseDecl(Decl *Node) {
+    // There is no need to go deeper into nodes that do not enclose selection,
+    // since "using" there will not affect selection, nor would it make a good
+    // insertion point.
+    if (Node->getDeclContext()->Encloses(SelectionDeclContext)) {
+      return RecursiveASTVisitor<UsingFinder>::TraverseDecl(Node);
+    }
+    return true;
+  }
+
+private:
+  std::vector<const UsingDecl *> &Results;
+  const DeclContext *SelectionDeclContext;
+  const SourceManager &SM;
+};
+
+struct InsertionPointData {
+  // Location to insert the "using" statement. If invalid then the statement
+  // should not be inserted at all (it already exists).
+  SourceLocation Loc;
+  // Extra suffix to place after the "using" statement. Depending on what the
+  // insertion point is anchored to, we may need one or more \n to ensure
+  // proper formatting.
+  std::string Suffix;
+};
+
+// Finds the best place to insert the "using" statement. Returns invalid
+// SourceLocation if the "using" statement already exists.
+//
+// The insertion point might be a little awkward if the decl we're anchoring to
+// has a comment in an unfortunate place (e.g. directly above function or using
+// decl, or immediately following "namespace {". We should add some helpers for
+// dealing with that and use them in other code modifications as well.
+llvm::Expected<InsertionPointData>
+findInsertionPoint(const Tweak::Selection &Inputs,
+                   const NestedNameSpecifierLoc &QualifierToRemove,
+                   const llvm::StringRef Name) {
+  auto &SM = Inputs.AST->getSourceManager();
+
+  // Search for all using decls that affect this point in file. We need this for
+  // two reasons: to skip adding "using" if one already exists and to find best
+  // place to add it, if it doesn't exist.
+  SourceLocation LastUsingLoc;
+  std::vector<const UsingDecl *> Usings;
+  UsingFinder(Usings, &Inputs.ASTSelection.commonAncestor()->getDeclContext(),
+              SM)
+      .TraverseAST(Inputs.AST->getASTContext());
+
+  for (auto &U : Usings) {
+    if (SM.isBeforeInTranslationUnit(Inputs.Cursor, U->getUsingLoc()))
+      // "Usings" is sorted, so we're done.
+      break;
+    if (U->getQualifier()->getAsNamespace()->getCanonicalDecl() ==
+            QualifierToRemove.getNestedNameSpecifier()
+                ->getAsNamespace()
+                ->getCanonicalDecl() &&
+        U->getName() == Name) {
+      return InsertionPointData();
+    }
+    // Insertion point will be before last UsingDecl that affects cursor
+    // position. For most cases this should stick with the local convention of
+    // add using inside or outside namespace.
+    LastUsingLoc = U->getUsingLoc();
+  }
+  if (LastUsingLoc.isValid()) {
+    InsertionPointData Out;
+    Out.Loc = LastUsingLoc;
+    return Out;
+  }
+
+  // No relevant "using" statements. Try the nearest namespace level.
+  const auto *NS = Inputs.ASTSelection.commonAncestor()
+                       ->getDeclContext()
+                       .getEnclosingNamespaceContext();
+  if (auto *ND = dyn_cast<NamespaceDecl>(NS)) {
+    auto Toks = Inputs.AST->getTokens().expandedTokens(ND->getSourceRange());
+    const auto *Tok = llvm::find_if(Toks, [](const syntax::Token &Tok) {
+      return Tok.kind() == tok::l_brace;
+    });
+    if (Tok == Toks.end() || Tok->endLocation().isInvalid()) {
+      return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                     "Namespace with no {");
+    }
+    if (!Tok->endLocation().isMacroID()) {
+      InsertionPointData Out;
+      Out.Loc = Tok->endLocation();
+      Out.Suffix = "\n";
+      return Out;
+    }
+  }
+  // No using, no namespace, no idea where to insert. Try above the first
+  // top level decl.
+  auto TLDs = Inputs.AST->getLocalTopLevelDecls();
+  if (TLDs.empty()) {
+    return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                   "Cannot find place to insert \"using\"");
+  }
+  InsertionPointData Out;
+  Out.Loc = SM.getExpansionLoc(TLDs[0]->getBeginLoc());
+  Out.Suffix = "\n\n";
+  return Out;
+}
+
+bool AddUsing::prepare(const Selection &Inputs) {
+  auto &SM = Inputs.AST->getSourceManager();
+  auto *Node = Inputs.ASTSelection.commonAncestor();
+  if (Node == nullptr)
+    return false;
+
+  // If we're looking at a type or NestedNameSpecifier, walk up the tree until
+  // we find the "main" node we care about, which would be ElaboratedTypeLoc or
+  // DeclRefExpr.
+  for (; Node->Parent; Node = Node->Parent) {
+    if (Node->ASTNode.get<NestedNameSpecifierLoc>()) {
+      continue;
+    } else if (auto *T = Node->ASTNode.get<TypeLoc>()) {
+      if (T->getAs<ElaboratedTypeLoc>()) {
+        break;
+      } else if (Node->Parent->ASTNode.get<TypeLoc>() ||
+                 Node->Parent->ASTNode.get<NestedNameSpecifierLoc>()) {
+        // Node is TypeLoc, but it's parent is either TypeLoc or
+        // NestedNameSpecifier. In both cases, we want to go up, to find
+        // the outermost TypeLoc.
+        continue;
+      }
+    }
+    break;
+  }
+  if (Node == nullptr)
+    return false;
+
+  if (auto *D = Node->ASTNode.get<DeclRefExpr>()) {
+    QualifierToRemove = D->getQualifierLoc();
+    Name = D->getDecl()->getName();
+  } else if (auto *T = Node->ASTNode.get<TypeLoc>()) {
+    if (auto E = T->getAs<ElaboratedTypeLoc>()) {
+      QualifierToRemove = E.getQualifierLoc();
+      Name =
+          E.getType().getUnqualifiedType().getBaseTypeIdentifier()->getName();
+    }
+  }
+
+  // FIXME: This only supports removing qualifiers that are made up of just
+  // namespace names. If qualifier contains a type, we could take the longest
+  // namespace prefix and remove that.
+  if (!QualifierToRemove.hasQualifier() ||
+      !QualifierToRemove.getNestedNameSpecifier()->getAsNamespace() ||
+      Name.empty()) {
+    return false;
+  }
+
+  // Macros are difficult. We only want to offer code action when what's spelled
+  // under the cursor is a namespace qualifier. If it's a macro that expands to
+  // a qualifier, user would not know what code action will actually change.
+  // On the other hand, if the qualifier is part of the macro argument, we
+  // should still support that.
+  if (SM.isMacroBodyExpansion(QualifierToRemove.getBeginLoc()) ||
+      !SM.isWrittenInSameFile(QualifierToRemove.getBeginLoc(),
+                              QualifierToRemove.getEndLoc())) {
+    return false;
+  }
+
+  return true;
+}
+
+Expected<Tweak::Effect> AddUsing::apply(const Selection &Inputs) {
+  auto &SM = Inputs.AST->getSourceManager();
+  auto &TB = Inputs.AST->getTokens();
+
+  // Determine the length of the qualifier under the cursor, then remove it.
+  auto SpelledTokens = TB.spelledForExpanded(
+      TB.expandedTokens(QualifierToRemove.getSourceRange()));
+  if (!SpelledTokens) {
+    return llvm::createStringError(
+        llvm::inconvertibleErrorCode(),
+        "Could not determine length of the qualifier");
+  }
+  unsigned Length =
+      syntax::Token::range(SM, SpelledTokens->front(), SpelledTokens->back())
+          .length();
+  tooling::Replacements R;
+  if (auto Err = R.add(tooling::Replacement(
+          SM, SpelledTokens->front().location(), Length, ""))) {
+    return std::move(Err);
+  }
+
+  auto InsertionPoint = findInsertionPoint(Inputs, QualifierToRemove, Name);
+  if (!InsertionPoint) {
+    return InsertionPoint.takeError();
+  }
+
+  if (InsertionPoint->Loc.isValid()) {
+    // Add the using statement at appropriate location.
+    std::string UsingText;
+    llvm::raw_string_ostream UsingTextStream(UsingText);
+    UsingTextStream << "using ";
+    QualifierToRemove.getNestedNameSpecifier()->print(
+        UsingTextStream, Inputs.AST->getASTContext().getPrintingPolicy());
+    UsingTextStream << Name << ";" << InsertionPoint->Suffix;
+
+    assert(SM.getFileID(InsertionPoint->Loc) == SM.getMainFileID());
+    if (auto Err = R.add(tooling::Replacement(SM, InsertionPoint->Loc, 0,
+                                              UsingTextStream.str()))) {
+      return std::move(Err);
+    }
+  }
+
+  return Effect::mainFileEdit(Inputs.AST->getASTContext().getSourceManager(),
+                              std::move(R));
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/refactor/tweaks/CMakeLists.txt b/clang-tools-extra/clangd/refactor/tweaks/CMakeLists.txt
index 5817830b7ea37..995288bca2cf4 100644
--- a/clang-tools-extra/clangd/refactor/tweaks/CMakeLists.txt
+++ b/clang-tools-extra/clangd/refactor/tweaks/CMakeLists.txt
@@ -12,6 +12,7 @@ set(LLVM_LINK_COMPONENTS
 # $<TARGET_OBJECTS:obj.clangDaemonTweaks> to a list of sources, see
 # clangd/tool/CMakeLists.txt for an example.
 add_clang_library(clangDaemonTweaks OBJECT
+  AddUsing.cpp
   AnnotateHighlightings.cpp
   DumpAST.cpp
   DefineInline.cpp
diff --git a/clang-tools-extra/clangd/test/initialize-params.test b/clang-tools-extra/clangd/test/initialize-params.test
index 2b5c02fc8ce28..6c4b847a07ef2 100644
--- a/clang-tools-extra/clangd/test/initialize-params.test
+++ b/clang-tools-extra/clangd/test/initialize-params.test
@@ -38,6 +38,18 @@
 # CHECK-NEXT:      "referencesProvider": true,
 # CHECK-NEXT:      "renameProvider": true,
 # CHECK-NEXT:      "selectionRangeProvider": true,
+# CHECK-NEXT:      "semanticTokensProvider": {
+# CHECK-NEXT:        "documentProvider": {
+# CHECK-NEXT:          "edits": true
+# CHECK-NEXT:        },
+# CHECK-NEXT:        "legend": {
+# CHECK-NEXT:          "tokenModifiers": [],
+# CHECK-NEXT:          "tokenTypes": [
+# CHECK-NEXT:            "variable",
+# CHECK:               ]
+# CHECK-NEXT:        },
+# CHECK-NEXT:        "rangeProvider": false
+# CHECK-NEXT:      },
 # CHECK-NEXT:      "signatureHelpProvider": {
 # CHECK-NEXT:        "triggerCharacters": [
 # CHECK-NEXT:          "(",
diff --git a/clang-tools-extra/clangd/test/semantic-tokens.test b/clang-tools-extra/clangd/test/semantic-tokens.test
new file mode 100644
index 0000000000000..6fab10362b285
--- /dev/null
+++ b/clang-tools-extra/clangd/test/semantic-tokens.test
@@ -0,0 +1,87 @@
+# RUN: clangd -lit-test < %s | FileCheck -strict-whitespace %s -implicit-check-not=semanticHighlight
+# Send capabilities for both Theia semanticHighlight & standard semanticTokens.
+# clangd should not use/acknowledge the Theia protocol in this case.
+{"jsonrpc":"2.0","id":0,"method":"initialize","params":{"capabilities":{"textDocument":{
+  "semanticHighlightingCapabilities":{"semanticHighlighting":true},
+  "semanticTokens":{"dynamicRegistration":true}
+}}}}
+---
+{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{
+  "uri": "test:///foo.cpp",
+  "languageId": "cpp",
+  "text": "int x = 2;"
+}}}
+---
+# Non-incremental token request.
+{"jsonrpc":"2.0","id":1,"method":"textDocument/semanticTokens","params":{"textDocument":{"uri":"test:///foo.cpp"}}}
+# CHECK:       "id": 1,
+# CHECK-NEXT:  "jsonrpc": "2.0",
+# CHECK-NEXT:  "result": {
+# CHECK-NEXT:    "data": [
+#                  First line, char 5, variable, no modifiers.
+# CHECK-NEXT:      0,
+# CHECK-NEXT:      4,
+# CHECK-NEXT:      1,
+# CHECK-NEXT:      0,
+# CHECK-NEXT:      0
+# CHECK-NEXT:    ],
+# CHECK-NEXT:    "resultId": "1"
+# CHECK-NEXT:  }
+---
+{"jsonrpc":"2.0","method":"textDocument/didChange","params":{
+  "textDocument": {"uri":"test:///foo.cpp","version":2},
+  "contentChanges":[{"text":"int x = 2;\nint y = 3;"}]
+}}
+---
+# Incremental token request, based on previous response.
+{"jsonrpc":"2.0","id":2,"method":"textDocument/semanticTokens/edits","params":{
+  "textDocument": {"uri":"test:///foo.cpp"},
+  "previousResultId": "1"
+}}
+# CHECK:       "id": 2,
+# CHECK-NEXT:  "jsonrpc": "2.0",
+# CHECK-NEXT:  "result": {
+# CHECK-NEXT:    "edits": [
+# CHECK-NEXT:      {
+# CHECK-NEXT:        "data": [
+#                      Next line, char 5, variable, no modifiers
+# CHECK-NEXT:          1,
+# CHECK-NEXT:          4,
+# CHECK-NEXT:          1,
+# CHECK-NEXT:          0,
+# CHECK-NEXT:          0
+# CHECK-NEXT:        ],
+#                    Inserted at position 1
+# CHECK-NEXT:        "deleteCount": 0,
+# CHECK-NEXT:        "start": 5
+# CHECK-NEXT:      }
+# CHECK-NEXT:    ],
+# CHECK-NEXT:    "resultId": "2"
+# CHECK-NEXT:  }
+---
+# Incremental token request with incorrect baseline => full tokens list.
+{"jsonrpc":"2.0","id":2,"method":"textDocument/semanticTokens/edits","params":{
+  "textDocument": {"uri":"test:///foo.cpp"},
+  "previousResultId": "bogus"
+}}
+# CHECK:       "id": 2,
+# CHECK-NEXT:  "jsonrpc": "2.0",
+# CHECK-NEXT:  "result": {
+# CHECK-NEXT:    "data": [
+# CHECK-NEXT:      0,
+# CHECK-NEXT:      4,
+# CHECK-NEXT:      1,
+# CHECK-NEXT:      0,
+# CHECK-NEXT:      0,
+# CHECK-NEXT:      1,
+# CHECK-NEXT:      4,
+# CHECK-NEXT:      1,
+# CHECK-NEXT:      0,
+# CHECK-NEXT:      0
+# CHECK-NEXT:    ],
+# CHECK-NEXT:    "resultId": "3"
+# CHECK-NEXT:  }
+---
+{"jsonrpc":"2.0","id":3,"method":"shutdown"}
+---
+{"jsonrpc":"2.0","method":"exit"}
diff --git a/clang-tools-extra/clangd/tool/ClangdMain.cpp b/clang-tools-extra/clangd/tool/ClangdMain.cpp
index 7a7bb9b0718e0..4b3b565ac2b6d 100644
--- a/clang-tools-extra/clangd/tool/ClangdMain.cpp
+++ b/clang-tools-extra/clangd/tool/ClangdMain.cpp
@@ -281,6 +281,15 @@ opt<bool> CrossFileRename{
     Hidden,
 };
 
+opt<bool> RecoveryAST{
+    "recovery-ast",
+    cat(Features),
+    desc("Preserve expressions in AST for broken code (C++ only). Note that "
+         "this feature is experimental and may lead to crashes"),
+    init(false),
+    Hidden,
+};
+
 opt<unsigned> WorkerThreadsCount{
     "j",
     cat(Misc),
@@ -629,6 +638,7 @@ clangd accepts flags on the commandline, and in the CLANGD_FLAGS environment var
   }
   Opts.StaticIndex = StaticIdx.get();
   Opts.AsyncThreadsCount = WorkerThreadsCount;
+  Opts.BuildRecoveryAST = RecoveryAST;
 
   clangd::CodeCompleteOptions CCOpts;
   CCOpts.IncludeIneligibleResults = IncludeIneligibleResults;
@@ -681,18 +691,42 @@ clangd accepts flags on the commandline, and in the CLANGD_FLAGS environment var
   std::unique_ptr<tidy::ClangTidyOptionsProvider>
       ClangTidyOptProvider; /*GUARDED_BY(ClangTidyOptMu)*/
   if (EnableClangTidy) {
-    auto OverrideClangTidyOptions = tidy::ClangTidyOptions::getDefaults();
-    OverrideClangTidyOptions.Checks = ClangTidyChecks;
+    auto EmptyDefaults = tidy::ClangTidyOptions::getDefaults();
+    EmptyDefaults.Checks.reset(); // So we can tell if checks were ever set.
+    tidy::ClangTidyOptions OverrideClangTidyOptions;
+    if (!ClangTidyChecks.empty())
+      OverrideClangTidyOptions.Checks = ClangTidyChecks;
     ClangTidyOptProvider = std::make_unique<tidy::FileOptionsProvider>(
         tidy::ClangTidyGlobalOptions(),
-        /* Default */ tidy::ClangTidyOptions::getDefaults(),
+        /* Default */ EmptyDefaults,
         /* Override */ OverrideClangTidyOptions, FSProvider.getFileSystem());
     Opts.GetClangTidyOptions = [&](llvm::vfs::FileSystem &,
                                    llvm::StringRef File) {
       // This function must be thread-safe and tidy option providers are not.
-      std::lock_guard<std::mutex> Lock(ClangTidyOptMu);
-      // FIXME: use the FS provided to the function.
-      return ClangTidyOptProvider->getOptions(File);
+      tidy::ClangTidyOptions Opts;
+      {
+        std::lock_guard<std::mutex> Lock(ClangTidyOptMu);
+        // FIXME: use the FS provided to the function.
+        Opts = ClangTidyOptProvider->getOptions(File);
+      }
+      if (!Opts.Checks) {
+        // If the user hasn't configured clang-tidy checks at all, including
+        // via .clang-tidy, give them a nice set of checks.
+        // (This should be what the "default" options does, but it isn't...)
+        //
+        // These default checks are chosen for:
+        //  - low false-positive rate
+        //  - providing a lot of value
+        //  - being reasonably efficient
+        Opts.Checks = llvm::join_items(
+            ",", "readability-misleading-indentation",
+            "readability-deleted-default", "bugprone-integer-division",
+            "bugprone-sizeof-expression", "bugprone-suspicious-missing-comma",
+            "bugprone-unused-raii", "bugprone-unused-return-value",
+            "misc-unused-using-decls", "misc-unused-alias-decls",
+            "misc-definitions-in-headers");
+      }
+      return Opts;
     };
   }
   Opts.SuggestMissingIncludes = SuggestMissingIncludes;
diff --git a/clang-tools-extra/clangd/unittests/ClangdTests.cpp b/clang-tools-extra/clangd/unittests/ClangdTests.cpp
index 1e5fcf3d97e1a..d15eba80ae29e 100644
--- a/clang-tools-extra/clangd/unittests/ClangdTests.cpp
+++ b/clang-tools-extra/clangd/unittests/ClangdTests.cpp
@@ -552,15 +552,13 @@ TEST_F(ClangdVFSTest, InvalidCompileCommand) {
   EXPECT_ERROR(runFindDocumentHighlights(Server, FooCpp, Position()));
   EXPECT_ERROR(runRename(Server, FooCpp, Position(), "new_name",
                          clangd::RenameOptions()));
+  EXPECT_ERROR(runSignatureHelp(Server, FooCpp, Position()));
   // Identifier-based fallback completion.
   EXPECT_THAT(cantFail(runCodeComplete(Server, FooCpp, Position(),
                                        clangd::CodeCompleteOptions()))
                   .Completions,
               ElementsAre(Field(&CodeCompletion::Name, "int"),
                           Field(&CodeCompletion::Name, "main")));
-  auto SigHelp = runSignatureHelp(Server, FooCpp, Position());
-  ASSERT_TRUE(bool(SigHelp)) << "signatureHelp returned an error";
-  EXPECT_THAT(SigHelp->signatures, IsEmpty());
 }
 
 class ClangdThreadingTest : public ClangdVFSTest {};
diff --git a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
index f5c90a4677cb8..1084b15505799 100644
--- a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
+++ b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
@@ -93,8 +93,9 @@ std::unique_ptr<SymbolIndex> memIndex(std::vector<Symbol> Symbols) {
   return MemIndex::build(std::move(Slab).build(), RefSlab(), RelationSlab());
 }
 
-CodeCompleteResult completions(ClangdServer &Server, llvm::StringRef TestCode,
-                               Position Point,
+// Runs code completion.
+// If IndexSymbols is non-empty, an index will be built and passed to opts.
+CodeCompleteResult completions(const TestTU &TU, Position Point,
                                std::vector<Symbol> IndexSymbols = {},
                                clangd::CodeCompleteOptions Opts = {}) {
   std::unique_ptr<SymbolIndex> OverrideIndex;
@@ -104,49 +105,34 @@ CodeCompleteResult completions(ClangdServer &Server, llvm::StringRef TestCode,
     Opts.Index = OverrideIndex.get();
   }
 
-  auto File = testPath("foo.cpp");
-  runAddDocument(Server, File, TestCode);
-  auto CompletionList =
-      llvm::cantFail(runCodeComplete(Server, File, Point, Opts));
-  return CompletionList;
-}
-
-CodeCompleteResult completions(ClangdServer &Server, llvm::StringRef Text,
-                               std::vector<Symbol> IndexSymbols = {},
-                               clangd::CodeCompleteOptions Opts = {},
-                               PathRef FilePath = "foo.cpp") {
-  std::unique_ptr<SymbolIndex> OverrideIndex;
-  if (!IndexSymbols.empty()) {
-    assert(!Opts.Index && "both Index and IndexSymbols given!");
-    OverrideIndex = memIndex(std::move(IndexSymbols));
-    Opts.Index = OverrideIndex.get();
+  auto Inputs = TU.inputs();
+  IgnoreDiagnostics Diags;
+  auto CI = buildCompilerInvocation(Inputs, Diags);
+  if (!CI) {
+    ADD_FAILURE() << "Couldn't build CompilerInvocation";
+    return {};
   }
-
-  auto File = testPath(FilePath);
-  Annotations Test(Text);
-  runAddDocument(Server, File, Test.code());
-  auto CompletionList =
-      llvm::cantFail(runCodeComplete(Server, File, Test.point(), Opts));
-  return CompletionList;
+  auto Preamble =
+      buildPreamble(testPath(TU.Filename), *CI, /*OldPreamble=*/nullptr, Inputs,
+                    /*InMemory=*/true, /*Callback=*/nullptr);
+  return codeComplete(testPath(TU.Filename), Inputs.CompileCommand,
+                      Preamble.get(), TU.Code, Point, Inputs.FS, Opts);
 }
 
-// Builds a server and runs code completion.
-// If IndexSymbols is non-empty, an index will be built and passed to opts.
+// Runs code completion.
 CodeCompleteResult completions(llvm::StringRef Text,
                                std::vector<Symbol> IndexSymbols = {},
                                clangd::CodeCompleteOptions Opts = {},
                                PathRef FilePath = "foo.cpp") {
-  MockFSProvider FS;
-  MockCompilationDatabase CDB;
+  Annotations Test(Text);
+  auto TU = TestTU::withCode(Test.code());
   // To make sure our tests for completiopns inside templates work on Windows.
-  CDB.ExtraClangFlags = {"-fno-delayed-template-parsing"};
-  ClangdServer Server(CDB, FS, ClangdServer::optsForTest());
-  return completions(Server, Text, std::move(IndexSymbols), std::move(Opts),
-                     FilePath);
+  TU.Filename = FilePath.str();
+  return completions(TU, Test.point(), std::move(IndexSymbols),
+                     std::move(Opts));
 }
 
-// Builds a server and runs code completion.
-// If IndexSymbols is non-empty, an index will be built and passed to opts.
+// Runs code completion without the clang parser.
 CodeCompleteResult completionsNoCompile(llvm::StringRef Text,
                                         std::vector<Symbol> IndexSymbols = {},
                                         clangd::CodeCompleteOptions Opts = {},
@@ -669,53 +655,38 @@ TEST(CompletionTest, SemaIndexMergeWithLimit) {
 }
 
 TEST(CompletionTest, IncludeInsertionPreprocessorIntegrationTests) {
-  MockFSProvider FS;
-  MockCompilationDatabase CDB;
-  std::string Subdir = testPath("sub");
-  std::string SearchDirArg = (Twine("-I") + Subdir).str();
-  CDB.ExtraClangFlags = {SearchDirArg.c_str()};
-  std::string BarHeader = testPath("sub/bar.h");
-  FS.Files[BarHeader] = "";
+  TestTU TU;
+  TU.ExtraArgs.push_back("-I" + testPath("sub"));
+  TU.AdditionalFiles["sub/bar.h"] = "";
+  auto BarURI = URI::create(testPath("sub/bar.h")).toString();
 
-  ClangdServer Server(CDB, FS, ClangdServer::optsForTest());
-  auto BarURI = URI::create(BarHeader).toString();
   Symbol Sym = cls("ns::X");
   Sym.CanonicalDeclaration.FileURI = BarURI.c_str();
   Sym.IncludeHeaders.emplace_back(BarURI, 1);
   // Shoten include path based on search directory and insert.
-  auto Results = completions(Server,
-                             R"cpp(
-          int main() { ns::^ }
-      )cpp",
-                             {Sym});
+  Annotations Test("int main() { ns::^ }");
+  TU.Code = Test.code().str();
+  auto Results = completions(TU, Test.point(), {Sym});
   EXPECT_THAT(Results.Completions,
               ElementsAre(AllOf(Named("X"), InsertInclude("\"bar.h\""))));
   // Can be disabled via option.
   CodeCompleteOptions NoInsertion;
   NoInsertion.InsertIncludes = CodeCompleteOptions::NeverInsert;
-  Results = completions(Server,
-                        R"cpp(
-          int main() { ns::^ }
-      )cpp",
-                        {Sym}, NoInsertion);
+  Results = completions(TU, Test.point(), {Sym}, NoInsertion);
   EXPECT_THAT(Results.Completions,
               ElementsAre(AllOf(Named("X"), Not(InsertInclude()))));
   // Duplicate based on inclusions in preamble.
-  Results = completions(Server,
-                        R"cpp(
+  Test = Annotations(R"cpp(
           #include "sub/bar.h"  // not shortest, so should only match resolved.
           int main() { ns::^ }
-      )cpp",
-                        {Sym});
+      )cpp");
+  TU.Code = Test.code().str();
+  Results = completions(TU, Test.point(), {Sym});
   EXPECT_THAT(Results.Completions, ElementsAre(AllOf(Named("X"), Labeled("X"),
                                                      Not(InsertInclude()))));
 }
 
 TEST(CompletionTest, NoIncludeInsertionWhenDeclFoundInFile) {
-  MockFSProvider FS;
-  MockCompilationDatabase CDB;
-
-  ClangdServer Server(CDB, FS, ClangdServer::optsForTest());
   Symbol SymX = cls("ns::X");
   Symbol SymY = cls("ns::Y");
   std::string BarHeader = testPath("bar.h");
@@ -725,8 +696,7 @@ TEST(CompletionTest, NoIncludeInsertionWhenDeclFoundInFile) {
   SymX.IncludeHeaders.emplace_back("<bar>", 1);
   SymY.IncludeHeaders.emplace_back("<bar>", 1);
   // Shoten include path based on search directory and insert.
-  auto Results = completions(Server,
-                             R"cpp(
+  auto Results = completions(R"cpp(
           namespace ns {
             class X;
             class Y {};
@@ -740,34 +710,27 @@ TEST(CompletionTest, NoIncludeInsertionWhenDeclFoundInFile) {
 }
 
 TEST(CompletionTest, IndexSuppressesPreambleCompletions) {
-  MockFSProvider FS;
-  MockCompilationDatabase CDB;
-  ClangdServer Server(CDB, FS, ClangdServer::optsForTest());
-
-  FS.Files[testPath("bar.h")] =
-      R"cpp(namespace ns { struct preamble { int member; }; })cpp";
-  auto File = testPath("foo.cpp");
   Annotations Test(R"cpp(
       #include "bar.h"
       namespace ns { int local; }
       void f() { ns::^; }
       void f2() { ns::preamble().$2^; }
   )cpp");
-  runAddDocument(Server, File, Test.code());
-  clangd::CodeCompleteOptions Opts = {};
+  auto TU = TestTU::withCode(Test.code());
+  TU.AdditionalFiles["bar.h"] =
+      R"cpp(namespace ns { struct preamble { int member; }; })cpp";
 
+  clangd::CodeCompleteOptions Opts = {};
   auto I = memIndex({var("ns::index")});
   Opts.Index = I.get();
-  auto WithIndex = cantFail(runCodeComplete(Server, File, Test.point(), Opts));
+  auto WithIndex = completions(TU, Test.point(), {}, Opts);
   EXPECT_THAT(WithIndex.Completions,
               UnorderedElementsAre(Named("local"), Named("index")));
-  auto ClassFromPreamble =
-      cantFail(runCodeComplete(Server, File, Test.point("2"), Opts));
+  auto ClassFromPreamble = completions(TU, Test.point("2"), {}, Opts);
   EXPECT_THAT(ClassFromPreamble.Completions, Contains(Named("member")));
 
   Opts.Index = nullptr;
-  auto WithoutIndex =
-      cantFail(runCodeComplete(Server, File, Test.point(), Opts));
+  auto WithoutIndex = completions(TU, Test.point(), {}, Opts);
   EXPECT_THAT(WithoutIndex.Completions,
               UnorderedElementsAre(Named("local"), Named("preamble")));
 }
@@ -811,7 +774,14 @@ TEST(CompletionTest, DynamicIndexIncludeInsertion) {
   Server.addDocument(testPath("foo_impl.cpp"), FileContent);
   // Wait for the dynamic index being built.
   ASSERT_TRUE(Server.blockUntilIdleForTest());
-  EXPECT_THAT(completions(Server, "Foo^ foo;").Completions,
+
+  auto File = testPath("foo.cpp");
+  Annotations Test("Foo^ foo;");
+  runAddDocument(Server, File, Test.code());
+  auto CompletionList =
+      llvm::cantFail(runCodeComplete(Server, File, Test.point(), {}));
+
+  EXPECT_THAT(CompletionList.Completions,
               ElementsAre(AllOf(Named("Foo"), HasInclude("\"foo_header.h\""),
                                 InsertInclude())));
 }
@@ -892,13 +862,17 @@ TEST(CompletionTest, CommentsFromSystemHeaders) {
     int foo();
   )cpp";
 
-  auto Results = completions(Server,
-                             R"cpp(
+  auto File = testPath("foo.cpp");
+  Annotations Test(R"cpp(
 #include "foo.h"
 int x = foo^
      )cpp");
+  runAddDocument(Server, File, Test.code());
+  auto CompletionList =
+      llvm::cantFail(runCodeComplete(Server, File, Test.point(), {}));
+
   EXPECT_THAT(
-      Results.Completions,
+      CompletionList.Completions,
       Contains(AllOf(Named("foo"), Doc("This comment should be retained!"))));
 }
 
@@ -1064,15 +1038,23 @@ SignatureHelp signatures(llvm::StringRef Text, Position Point,
   if (!IndexSymbols.empty())
     Index = memIndex(IndexSymbols);
 
-  MockFSProvider FS;
-  MockCompilationDatabase CDB;
-  ClangdServer::Options Opts = ClangdServer::optsForTest();
-  Opts.StaticIndex = Index.get();
-
-  ClangdServer Server(CDB, FS, Opts);
-  auto File = testPath("foo.cpp");
-  runAddDocument(Server, File, Text);
-  return llvm::cantFail(runSignatureHelp(Server, File, Point));
+  auto TU = TestTU::withCode(Text);
+  auto Inputs = TU.inputs();
+  IgnoreDiagnostics Diags;
+  auto CI = buildCompilerInvocation(Inputs, Diags);
+  if (!CI) {
+    ADD_FAILURE() << "Couldn't build CompilerInvocation";
+    return {};
+  }
+  auto Preamble =
+      buildPreamble(testPath(TU.Filename), *CI, /*OldPreamble=*/nullptr, Inputs,
+                    /*InMemory=*/true, /*Callback=*/nullptr);
+  if (!Preamble) {
+    ADD_FAILURE() << "Couldn't build Preamble";
+    return {};
+  }
+  return signatureHelp(testPath(TU.Filename), Inputs.CompileCommand, *Preamble,
+                       Text, Point, Inputs.FS, Index.get());
 }
 
 SignatureHelp signatures(llvm::StringRef Text,
@@ -1195,9 +1177,7 @@ TEST(SignatureHelpTest, OpeningParen) {
     int foo(int a, int b, int c);
     int main() {
     #define ID(X) X
-      // FIXME: figure out why ID(foo (foo(10), )) doesn't work when preserving
-      // the recovery expression.
-      ID(foo $p^( 10, ^ ))
+      ID(foo $p^( foo(10), ^ ))
     })cpp"};
 
   for (auto Test : Tests) {
@@ -1548,14 +1528,7 @@ TEST(CompletionTest, DocumentationFromChangedFileCrash) {
 }
 
 TEST(CompletionTest, NonDocComments) {
-  MockFSProvider FS;
-  auto FooCpp = testPath("foo.cpp");
-  FS.Files[FooCpp] = "";
-
-  MockCompilationDatabase CDB;
-  ClangdServer Server(CDB, FS, ClangdServer::optsForTest());
-
-  Annotations Source(R"cpp(
+  const char *Text = R"cpp(
     // We ignore namespace comments, for rationale see CodeCompletionStrings.h.
     namespace comments_ns {
     }
@@ -1590,17 +1563,11 @@ TEST(CompletionTest, NonDocComments) {
     int Struct<T>::comments_quux() {
       int a = comments^;
     }
-  )cpp");
-  // FIXME: Auto-completion in a template requires disabling delayed template
-  // parsing.
-  CDB.ExtraClangFlags.push_back("-fno-delayed-template-parsing");
-  runAddDocument(Server, FooCpp, Source.code(), "null", WantDiagnostics::Yes);
-  CodeCompleteResult Completions = cantFail(runCodeComplete(
-      Server, FooCpp, Source.point(), clangd::CodeCompleteOptions()));
+  )cpp";
 
   // We should not get any of those comments in completion.
   EXPECT_THAT(
-      Completions.Completions,
+      completions(Text).Completions,
       UnorderedElementsAre(AllOf(Not(IsDocumented()), Named("comments_foo")),
                            AllOf(IsDocumented(), Named("comments_baz")),
                            AllOf(IsDocumented(), Named("comments_quux")),
@@ -1742,11 +1709,10 @@ TEST(CompletionTest, CodeCompletionContext) {
 TEST(CompletionTest, FixItForArrowToDot) {
   MockFSProvider FS;
   MockCompilationDatabase CDB;
-  ClangdServer Server(CDB, FS, ClangdServer::optsForTest());
 
   CodeCompleteOptions Opts;
   Opts.IncludeFixIts = true;
-  Annotations TestCode(
+  const char* Code =
       R"cpp(
         class Auxilary {
          public:
@@ -1762,13 +1728,12 @@ TEST(CompletionTest, FixItForArrowToDot) {
           ClassWithPtr x;
           x[[->]]^;
         }
-      )cpp");
-  auto Results =
-      completions(Server, TestCode.code(), TestCode.point(), {}, Opts);
+      )cpp";
+  auto Results = completions(Code, {}, Opts);
   EXPECT_EQ(Results.Completions.size(), 3u);
 
   TextEdit ReplacementEdit;
-  ReplacementEdit.range = TestCode.range();
+  ReplacementEdit.range = Annotations(Code).range();
   ReplacementEdit.newText = ".";
   for (const auto &C : Results.Completions) {
     EXPECT_TRUE(C.FixIts.size() == 1u || C.Name == "AuxFunction");
@@ -1779,13 +1744,9 @@ TEST(CompletionTest, FixItForArrowToDot) {
 }
 
 TEST(CompletionTest, FixItForDotToArrow) {
-  MockFSProvider FS;
-  MockCompilationDatabase CDB;
-  ClangdServer Server(CDB, FS, ClangdServer::optsForTest());
-
   CodeCompleteOptions Opts;
   Opts.IncludeFixIts = true;
-  Annotations TestCode(
+  const char* Code =
       R"cpp(
         class Auxilary {
          public:
@@ -1801,13 +1762,12 @@ TEST(CompletionTest, FixItForDotToArrow) {
           ClassWithPtr x;
           x[[.]]^;
         }
-      )cpp");
-  auto Results =
-      completions(Server, TestCode.code(), TestCode.point(), {}, Opts);
+      )cpp";
+  auto Results = completions(Code, {}, Opts);
   EXPECT_EQ(Results.Completions.size(), 3u);
 
   TextEdit ReplacementEdit;
-  ReplacementEdit.range = TestCode.range();
+  ReplacementEdit.range = Annotations(Code).range();
   ReplacementEdit.newText = "->";
   for (const auto &C : Results.Completions) {
     EXPECT_TRUE(C.FixIts.empty() || C.Name == "AuxFunction");
@@ -1860,7 +1820,8 @@ TEST(CompletionTest, RenderWithFixItNonMerged) {
 TEST(CompletionTest, CompletionTokenRange) {
   MockFSProvider FS;
   MockCompilationDatabase CDB;
-  ClangdServer Server(CDB, FS, ClangdServer::optsForTest());
+  TestTU TU;
+  TU.AdditionalFiles["foo/abc/foo.h"] = "";
 
   constexpr const char *TestCodes[] = {
       R"cpp(
@@ -1882,13 +1843,20 @@ TEST(CompletionTest, CompletionTokenRange) {
           Auxilary x;
           x.[[]]^;
         }
-      )cpp"};
+      )cpp",
+      R"cpp(
+        #include "foo/[[a^/]]foo.h"
+      )cpp",
+      R"cpp(
+        #include "foo/abc/[[fo^o.h"]]
+      )cpp",
+      };
   for (const auto &Text : TestCodes) {
     Annotations TestCode(Text);
-    auto Results = completions(Server, TestCode.code(), TestCode.point());
-
+    TU.Code = TestCode.code().str();
+    auto Results = completions(TU, TestCode.point());
     if (Results.Completions.size() != 1) {
-      ADD_FAILURE() << "Results.Completions.size() != 1";
+      ADD_FAILURE() << "Results.Completions.size() != 1" << Text;
       continue;
     }
     EXPECT_THAT(Results.Completions.front().CompletionTokenRange,
@@ -2241,13 +2209,12 @@ TEST(CompletionTest, InsertTheMostPopularHeader) {
 }
 
 TEST(CompletionTest, NoInsertIncludeIfOnePresent) {
-  MockFSProvider FS;
-  MockCompilationDatabase CDB;
-
-  std::string FooHeader = testPath("foo.h");
-  FS.Files[FooHeader] = "";
-
-  ClangdServer Server(CDB, FS, ClangdServer::optsForTest());
+  Annotations Test(R"cpp(
+    #include "foo.h"
+    Fun^
+  )cpp");
+  auto TU = TestTU::withCode(Test.code());
+  TU.AdditionalFiles["foo.h"] = "";
 
   std::string DeclFile = URI::create(testPath("foo")).toString();
   Symbol Sym = func("Func");
@@ -2256,7 +2223,7 @@ TEST(CompletionTest, NoInsertIncludeIfOnePresent) {
   Sym.IncludeHeaders.emplace_back("\"bar.h\"", 1000);
 
   EXPECT_THAT(
-      completions(Server, "#include \"foo.h\"\nFun^", {Sym}).Completions,
+      completions(TU, Test.point(), {Sym}).Completions,
       UnorderedElementsAre(
           AllOf(Named("Func"), HasInclude("\"foo.h\""), Not(InsertInclude()))));
 }
@@ -2273,20 +2240,15 @@ TEST(CompletionTest, MergeMacrosFromIndexAndSema) {
 }
 
 TEST(CompletionTest, MacroFromPreamble) {
-  MockFSProvider FS;
-  MockCompilationDatabase CDB;
-  std::string FooHeader = testPath("foo.h");
-  FS.Files[FooHeader] = "#define CLANGD_PREAMBLE_HEADER x\n";
-  ClangdServer Server(CDB, FS, ClangdServer::optsForTest());
-  auto Results = completions(
-      R"cpp(#include "foo.h"
-          #define CLANGD_PREAMBLE_MAIN x
+  Annotations Test(R"cpp(#define CLANGD_PREAMBLE_MAIN x
 
           int x = 0;
           #define CLANGD_MAIN x
           void f() { CLANGD_^ }
-      )cpp",
-      {func("CLANGD_INDEX")});
+      )cpp");
+  auto TU = TestTU::withCode(Test.code());
+  TU.HeaderCode = "#define CLANGD_PREAMBLE_HEADER x";
+  auto Results = completions(TU, Test.point(), {func("CLANGD_INDEX")});
   // We should get results from the main file, including the preamble section.
   // However no results from included files (the index should cover them).
   EXPECT_THAT(Results.Completions,
@@ -2399,29 +2361,22 @@ TEST(SignatureHelpTest, ConstructorInitializeFields) {
 }
 
 TEST(CompletionTest, IncludedCompletionKinds) {
-  MockFSProvider FS;
-  MockCompilationDatabase CDB;
-  std::string Subdir = testPath("sub");
-  std::string SearchDirArg = (Twine("-I") + Subdir).str();
-  CDB.ExtraClangFlags = {SearchDirArg.c_str()};
-  std::string BarHeader = testPath("sub/bar.h");
-  FS.Files[BarHeader] = "";
-  ClangdServer Server(CDB, FS, ClangdServer::optsForTest());
-  auto Results = completions(Server,
-                             R"cpp(
-        #include "^"
-      )cpp");
+  Annotations Test(R"cpp(#include "^")cpp");
+  auto TU = TestTU::withCode(Test.code());
+  TU.AdditionalFiles["sub/bar.h"] = "";
+  TU.ExtraArgs.push_back("-I" + testPath("sub"));
+
+  auto Results = completions(TU, Test.point());
   EXPECT_THAT(Results.Completions,
               AllOf(Has("sub/", CompletionItemKind::Folder),
                     Has("bar.h\"", CompletionItemKind::File)));
 }
 
 TEST(CompletionTest, NoCrashAtNonAlphaIncludeHeader) {
-  auto Results = completions(
+  completions(
       R"cpp(
         #include "./^"
       )cpp");
-  EXPECT_TRUE(Results.Completions.empty());
 }
 
 TEST(CompletionTest, NoAllScopesCompletionWhenQualified) {
@@ -2708,6 +2663,20 @@ TEST(CompletionTest, NoCrashWithIncompleteLambda) {
   EXPECT_THAT(Signatures, Contains(Sig("x() -> auto")));
 }
 
+TEST(CompletionTest, DelayedTemplateParsing) {
+  Annotations Test(R"cpp(
+    int xxx;
+    template <typename T> int foo() { return xx^; }
+  )cpp");
+  auto TU = TestTU::withCode(Test.code());
+  // Even though delayed-template-parsing is on, we will disable it to provide
+  // completion in templates.
+  TU.ExtraArgs.push_back("-fdelayed-template-parsing");
+
+  EXPECT_THAT(completions(TU, Test.point()).Completions,
+              Contains(Named("xxx")));
+}
+
 TEST(CompletionTest, CompletionRange) {
   const char *WithRange = "auto x = [[abc]]^";
   auto Completions = completions(WithRange);
diff --git a/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp b/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp
index 9334d9d17608d..00dd1f864813d 100644
--- a/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp
+++ b/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp
@@ -61,9 +61,7 @@ MATCHER_P3(Fix, Range, Replacement, Message,
          arg.Edits[0].range == Range && arg.Edits[0].newText == Replacement;
 }
 
-MATCHER_P(FixMessage, Message, "") {
-  return arg.Message == Message;
-}
+MATCHER_P(FixMessage, Message, "") { return arg.Message == Message; }
 
 MATCHER_P(EqualToLSPDiag, LSPDiag,
           "LSP diagnostic " + llvm::to_string(LSPDiag)) {
@@ -258,13 +256,13 @@ TEST(DiagnosticsTest, ClangTidy) {
           Diag(Test.range("macroarg"),
                "multiple unsequenced modifications to 'y'"),
           AllOf(
-            Diag(Test.range("main"),
-                 "use a trailing return type for this function"),
-            DiagSource(Diag::ClangTidy),
-            DiagName("modernize-use-trailing-return-type"),
-            // Verify that we don't have "[check-name]" suffix in the message.
-            WithFix(FixMessage("use a trailing return type for this function")))
-          ));
+              Diag(Test.range("main"),
+                   "use a trailing return type for this function"),
+              DiagSource(Diag::ClangTidy),
+              DiagName("modernize-use-trailing-return-type"),
+              // Verify that we don't have "[check-name]" suffix in the message.
+              WithFix(FixMessage(
+                  "use a trailing return type for this function")))));
 }
 
 TEST(DiagnosticTest, ClangTidySuppressionComment) {
@@ -274,7 +272,11 @@ TEST(DiagnosticTest, ClangTidySuppressionComment) {
       double d = 8 / i;  // NOLINT
       // NOLINTNEXTLINE
       double e = 8 / i;
-      double f = [[8]] / i;
+      #define BAD 8 / i
+      double f = BAD;  // NOLINT
+      double g = [[8]] / i;
+      #define BAD2 BAD
+      double h = BAD2;  // NOLINT
     }
   )cpp");
   TestTU TU = TestTU::withCode(Main.code());
@@ -678,7 +680,7 @@ void foo() {
 TEST(IncludeFixerTest, NoCrashMemebrAccess) {
   Annotations Test(R"cpp(// error-ok
     struct X { int  xyz; };
-    void g() { X x; x.$[[xy]] }
+    void g() { X x; x.$[[xy]]; }
   )cpp");
   auto TU = TestTU::withCode(Test.code());
   auto Index = buildIndexWithSymbol(
@@ -836,8 +838,9 @@ TEST(IncludeFixerTest, NoCrashOnTemplateInstantiations) {
   auto Index = buildIndexWithSymbol({});
   TU.ExternalIndex = Index.get();
 
-  EXPECT_THAT(TU.build().getDiagnostics(),
-              ElementsAre(Diag(Test.range(), "use of undeclared identifier 'a'")));
+  EXPECT_THAT(
+      TU.build().getDiagnostics(),
+      ElementsAre(Diag(Test.range(), "use of undeclared identifier 'a'")));
 }
 
 TEST(DiagsInHeaders, DiagInsideHeader) {
diff --git a/clang-tools-extra/clangd/unittests/FindTargetTests.cpp b/clang-tools-extra/clangd/unittests/FindTargetTests.cpp
index c38ccc3f9441d..7b6fff292e661 100644
--- a/clang-tools-extra/clangd/unittests/FindTargetTests.cpp
+++ b/clang-tools-extra/clangd/unittests/FindTargetTests.cpp
@@ -65,7 +65,7 @@ class TargetDeclTest : public ::testing::Test {
 protected:
   using Rel = DeclRelation;
   std::string Code;
-  std::vector<const char *> Flags;
+  std::vector<std::string> Flags;
 
   // Asserts that `Code` has a marked selection of a node `NodeType`,
   // and returns allTargetDecls() as PrintedDecl structs.
@@ -132,6 +132,16 @@ TEST_F(TargetDeclTest, Exprs) {
   EXPECT_DECLS("CXXOperatorCallExpr", "void operator()(int n)");
 }
 
+TEST_F(TargetDeclTest, Recovery) {
+  Code = R"cpp(
+    // error-ok: testing behavior on broken code
+    int f();
+    int f(int, int);
+    int x = [[f]](42);
+  )cpp";
+  EXPECT_DECLS("UnresolvedLookupExpr", "int f()", "int f(int, int)");
+}
+
 TEST_F(TargetDeclTest, UsingDecl) {
   Code = R"cpp(
     namespace foo {
@@ -685,6 +695,15 @@ TEST_F(FindExplicitReferencesTest, All) {
         )cpp",
         "0: targets = {x}\n"
         "1: targets = {X::a}\n"},
+       {R"cpp(
+        // error-ok: testing with broken code
+        int bar();
+        int foo() {
+          return $0^bar() + $1^bar(42);
+        }
+        )cpp",
+        "0: targets = {bar}\n"
+        "1: targets = {bar}\n"},
        // Namespaces and aliases.
        {R"cpp(
           namespace ns {}
diff --git a/clang-tools-extra/clangd/unittests/HoverTests.cpp b/clang-tools-extra/clangd/unittests/HoverTests.cpp
index 593fb16e21948..99fd1c5724476 100644
--- a/clang-tools-extra/clangd/unittests/HoverTests.cpp
+++ b/clang-tools-extra/clangd/unittests/HoverTests.cpp
@@ -1567,7 +1567,7 @@ TEST(Hover, All) {
             HI.Kind = index::SymbolKind::Variable;
             HI.NamespaceScope = "";
             HI.Name = "foo";
-            HI.Type = "cls<cls<cls<int> > >";
+            HI.Type = "cls<cls<cls<int>>>";
             HI.Value = "{}";
           }},
       {
@@ -1579,7 +1579,7 @@ TEST(Hover, All) {
             HI.Definition = "template <> struct cls<cls<cls<int>>> {}";
             HI.Kind = index::SymbolKind::Struct;
             HI.NamespaceScope = "";
-            HI.Name = "cls<cls<cls<int> > >";
+            HI.Name = "cls<cls<cls<int>>>";
             HI.Documentation = "type of nested templates.";
           }},
       {
diff --git a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
index 6a7f700e1f491..1cca429a4ea6e 100644
--- a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -14,8 +14,10 @@
 #include "TestFS.h"
 #include "TestTU.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Support/ScopedPrinter.h"
 #include "gmock/gmock.h"
 #include <algorithm>
 
@@ -23,6 +25,9 @@ namespace clang {
 namespace clangd {
 namespace {
 
+using testing::IsEmpty;
+using testing::SizeIs;
+
 MATCHER_P(LineNumber, L, "") { return arg.Line == L; }
 MATCHER(EmptyHighlightings, "") { return arg.Tokens.empty(); }
 
@@ -720,6 +725,77 @@ TEST(SemanticHighlighting, GeneratesHighlightsWhenFileChange) {
   ASSERT_EQ(Counter.Count, 1);
 }
 
+// Ranges are highlighted as variables, unless highlighted as $Function etc.
+std::vector<HighlightingToken> tokens(llvm::StringRef MarkedText) {
+  Annotations A(MarkedText);
+  std::vector<HighlightingToken> Results;
+  for (const Range& R : A.ranges())
+    Results.push_back({HighlightingKind::Variable, R});
+  for (unsigned I = 0; I < static_cast<unsigned>(HighlightingKind::LastKind); ++I) {
+    HighlightingKind Kind = static_cast<HighlightingKind>(I);
+    for (const Range& R : A.ranges(llvm::to_string(Kind)))
+      Results.push_back({Kind, R});
+  }
+  llvm::sort(Results);
+  return Results;
+}
+
+TEST(SemanticHighlighting, toSemanticTokens) {
+  auto Results = toSemanticTokens(tokens(R"(
+ [[blah]]
+
+    $Function[[big]] [[bang]]
+  )"));
+
+  ASSERT_THAT(Results, SizeIs(3));
+  EXPECT_EQ(Results[0].tokenType, unsigned(HighlightingKind::Variable));
+  EXPECT_EQ(Results[0].deltaLine, 1u);
+  EXPECT_EQ(Results[0].deltaStart, 1u);
+  EXPECT_EQ(Results[0].length, 4u);
+
+  EXPECT_EQ(Results[1].tokenType, unsigned(HighlightingKind::Function));
+  EXPECT_EQ(Results[1].deltaLine, 2u);
+  EXPECT_EQ(Results[1].deltaStart, 4u);
+  EXPECT_EQ(Results[1].length, 3u);
+
+  EXPECT_EQ(Results[2].tokenType, unsigned(HighlightingKind::Variable));
+  EXPECT_EQ(Results[2].deltaLine, 0u);
+  EXPECT_EQ(Results[2].deltaStart, 4u);
+  EXPECT_EQ(Results[2].length, 4u);
+}
+
+TEST(SemanticHighlighting, diffSemanticTokens) {
+  auto Before = toSemanticTokens(tokens(R"(
+    [[foo]] [[bar]] [[baz]]
+    [[one]] [[two]] [[three]]
+  )"));
+  EXPECT_THAT(diffTokens(Before, Before), IsEmpty());
+
+  auto After = toSemanticTokens(tokens(R"(
+    [[foo]] [[hello]] [[world]] [[baz]]
+    [[one]] [[two]] [[three]]
+  )"));
+
+  // Replace [bar, baz] with [hello, world, baz]
+  auto Diff = diffTokens(Before, After);
+  ASSERT_THAT(Diff, SizeIs(1));
+  EXPECT_EQ(1u, Diff.front().startToken);
+  EXPECT_EQ(2u, Diff.front().deleteTokens);
+  ASSERT_THAT(Diff.front().tokens, SizeIs(3));
+  // hello
+  EXPECT_EQ(0u, Diff.front().tokens[0].deltaLine);
+  EXPECT_EQ(4u, Diff.front().tokens[0].deltaStart);
+  EXPECT_EQ(5u, Diff.front().tokens[0].length);
+  // world
+  EXPECT_EQ(0u, Diff.front().tokens[1].deltaLine);
+  EXPECT_EQ(6u, Diff.front().tokens[1].deltaStart);
+  EXPECT_EQ(5u, Diff.front().tokens[1].length);
+  // baz
+  EXPECT_EQ(0u, Diff.front().tokens[2].deltaLine);
+  EXPECT_EQ(6u, Diff.front().tokens[2].deltaStart);
+  EXPECT_EQ(3u, Diff.front().tokens[2].length);
+}
+
 TEST(SemanticHighlighting, toTheiaSemanticHighlightingInformation) {
   auto CreatePosition = [](int Line, int Character) -> Position {
     Position Pos;
diff --git a/clang-tools-extra/clangd/unittests/SemanticSelectionTests.cpp b/clang-tools-extra/clangd/unittests/SemanticSelectionTests.cpp
index 3356a095515f6..0b0b7550cc525 100644
--- a/clang-tools-extra/clangd/unittests/SemanticSelectionTests.cpp
+++ b/clang-tools-extra/clangd/unittests/SemanticSelectionTests.cpp
@@ -24,8 +24,17 @@
 namespace clang {
 namespace clangd {
 namespace {
+using ::testing::ElementsAre;
 using ::testing::ElementsAreArray;
 
+// front() is SR.range, back() is outermost range.
+std::vector<Range> gatherRanges(const SelectionRange &SR) {
+  std::vector<Range> Ranges;
+  for (const SelectionRange *S = &SR; S; S = S->parent.get())
+    Ranges.push_back(S->range);
+  return Ranges;
+}
+
 TEST(SemanticSelection, All) {
   const char *Tests[] = {
       R"cpp( // Single statement in a function body.
@@ -78,7 +87,7 @@ TEST(SemanticSelection, All) {
         }]]]]
        )cpp",
       // Empty file.
-      "^",
+      "[[^]]",
       // FIXME: We should get the whole DeclStmt as a range.
       R"cpp( // Single statement in TU.
         [[int v = [[1^00]]]];
@@ -89,7 +98,7 @@ TEST(SemanticSelection, All) {
       // FIXME: No node found associated to the position.
       R"cpp( // Cursor in between spaces.
         void func() {
-          int v = 100 + ^  100;
+          int v = 100 + [[^]]  100;
         }
       )cpp",
       // Structs.
@@ -133,13 +142,13 @@ TEST(SemanticSelection, All) {
   for (const char *Test : Tests) {
     auto T = Annotations(Test);
     auto AST = TestTU::withCode(T.code()).build();
-    EXPECT_THAT(llvm::cantFail(getSemanticRanges(AST, T.point())),
+    EXPECT_THAT(gatherRanges(llvm::cantFail(getSemanticRanges(AST, T.point()))),
                 ElementsAreArray(T.ranges()))
         << Test;
   }
 }
 
-TEST(SemanticSelection, RunViaClangDServer) {
+TEST(SemanticSelection, RunViaClangdServer) {
   MockFSProvider FS;
   MockCompilationDatabase CDB;
   ClangdServer Server(CDB, FS, ClangdServer::optsForTest());
@@ -157,15 +166,20 @@ TEST(SemanticSelection, RunViaClangDServer) {
     // inp = HASH(foo(inp));
     [[inp = [[HASH([[foo([[in^p]])]])]]]];
   }]]]]
+  $empty[[^]]
   )cpp";
   Annotations SourceAnnotations(SourceContents);
   FS.Files[FooCpp] = std::string(SourceAnnotations.code());
   Server.addDocument(FooCpp, SourceAnnotations.code());
 
-  auto Ranges = runSemanticRanges(Server, FooCpp, SourceAnnotations.point());
+  auto Ranges = runSemanticRanges(Server, FooCpp, SourceAnnotations.points());
   ASSERT_TRUE(bool(Ranges))
       << "getSemanticRange returned an error: " << Ranges.takeError();
-  EXPECT_THAT(*Ranges, ElementsAreArray(SourceAnnotations.ranges()));
+  ASSERT_EQ(Ranges->size(), SourceAnnotations.points().size());
+  EXPECT_THAT(gatherRanges(Ranges->front()),
+              ElementsAreArray(SourceAnnotations.ranges()));
+  EXPECT_THAT(gatherRanges(Ranges->back()),
+              ElementsAre(SourceAnnotations.range("empty")));
 }
 } // namespace
 } // namespace clangd
diff --git a/clang-tools-extra/clangd/unittests/SyncAPI.cpp b/clang-tools-extra/clangd/unittests/SyncAPI.cpp
index 522aaed117e87..e976b5ab93894 100644
--- a/clang-tools-extra/clangd/unittests/SyncAPI.cpp
+++ b/clang-tools-extra/clangd/unittests/SyncAPI.cpp
@@ -146,9 +146,10 @@ RefSlab getRefs(const SymbolIndex &Index, SymbolID ID) {
   return std::move(Slab).build();
 }
 
-llvm::Expected<std::vector<Range>>
-runSemanticRanges(ClangdServer &Server, PathRef File, Position Pos) {
-  llvm::Optional<llvm::Expected<std::vector<Range>>> Result;
+llvm::Expected<std::vector<SelectionRange>>
+runSemanticRanges(ClangdServer &Server, PathRef File,
+                  const std::vector<Position> &Pos) {
+  llvm::Optional<llvm::Expected<std::vector<SelectionRange>>> Result;
   Server.semanticRanges(File, Pos, capture(Result));
   return std::move(*Result);
 }
diff --git a/clang-tools-extra/clangd/unittests/SyncAPI.h b/clang-tools-extra/clangd/unittests/SyncAPI.h
index 241b98d348948..22b03c6f1fa5e 100644
--- a/clang-tools-extra/clangd/unittests/SyncAPI.h
+++ b/clang-tools-extra/clangd/unittests/SyncAPI.h
@@ -56,8 +56,9 @@ SymbolSlab runFuzzyFind(const SymbolIndex &Index, StringRef Query);
 SymbolSlab runFuzzyFind(const SymbolIndex &Index, const FuzzyFindRequest &Req);
 RefSlab getRefs(const SymbolIndex &Index, SymbolID ID);
 
-llvm::Expected<std::vector<Range>>
-runSemanticRanges(ClangdServer &Server, PathRef File, Position Pos);
+llvm::Expected<std::vector<SelectionRange>>
+runSemanticRanges(ClangdServer &Server, PathRef File,
+                  const std::vector<Position> &Pos);
 
 llvm::Expected<llvm::Optional<clangd::Path>>
 runSwitchHeaderSource(ClangdServer &Server, PathRef File);
diff --git a/clang-tools-extra/clangd/unittests/TestTU.cpp b/clang-tools-extra/clangd/unittests/TestTU.cpp
index 909c125aed2e0..2adcfc338cc2e 100644
--- a/clang-tools-extra/clangd/unittests/TestTU.cpp
+++ b/clang-tools-extra/clangd/unittests/TestTU.cpp
@@ -20,7 +20,7 @@
 namespace clang {
 namespace clangd {
 
-ParsedAST TestTU::build() const {
+ParseInputs TestTU::inputs() const {
   std::string FullFilename = testPath(Filename),
               FullHeaderName = testPath(HeaderFilename),
               ImportThunk = testPath("import_thunk.h");
@@ -34,43 +34,48 @@ ParsedAST TestTU::build() const {
   Files[FullHeaderName] = HeaderCode;
   Files[ImportThunk] = ThunkContents;
 
-  std::vector<const char *> Cmd = {"clang"};
+  ParseInputs Inputs;
+  auto& Argv = Inputs.CompileCommand.CommandLine;
+  Argv = {"clang"};
   // FIXME: this shouldn't need to be conditional, but it breaks a
   // GoToDefinition test for some reason (getMacroArgExpandedLocation fails).
   if (!HeaderCode.empty()) {
-    Cmd.push_back("-include");
-    Cmd.push_back(ImplicitHeaderGuard ? ImportThunk.c_str()
-                                      : FullHeaderName.c_str());
+    Argv.push_back("-include");
+    Argv.push_back(ImplicitHeaderGuard ? ImportThunk : FullHeaderName);
     // ms-compatibility changes the meaning of #import.
     // The default is OS-dependent (on on windows), ensure it's off.
     if (ImplicitHeaderGuard)
-      Cmd.push_back("-fno-ms-compatibility");
+      Inputs.CompileCommand.CommandLine.push_back("-fno-ms-compatibility");
   }
-  Cmd.insert(Cmd.end(), ExtraArgs.begin(), ExtraArgs.end());
+  Argv.insert(Argv.end(), ExtraArgs.begin(), ExtraArgs.end());
   // Put the file name at the end -- this allows the extra arg (-xc++) to
   // override the language setting.
-  Cmd.push_back(FullFilename.c_str());
-  ParseInputs Inputs;
+  Argv.push_back(FullFilename);
   Inputs.CompileCommand.Filename = FullFilename;
-  Inputs.CompileCommand.CommandLine = {Cmd.begin(), Cmd.end()};
   Inputs.CompileCommand.Directory = testRoot();
   Inputs.Contents = Code;
   Inputs.FS = buildTestFS(Files);
   Inputs.Opts = ParseOptions();
+  Inputs.Opts.BuildRecoveryAST = true;
   Inputs.Opts.ClangTidyOpts.Checks = ClangTidyChecks;
   Inputs.Opts.ClangTidyOpts.WarningsAsErrors = ClangTidyWarningsAsErrors;
   Inputs.Index = ExternalIndex;
   if (Inputs.Index)
     Inputs.Opts.SuggestMissingIncludes = true;
+  return Inputs;
+}
+
+ParsedAST TestTU::build() const {
+  auto Inputs = inputs();
   StoreDiags Diags;
   auto CI = buildCompilerInvocation(Inputs, Diags);
   assert(CI && "Failed to build compilation invocation.");
   auto Preamble =
-      buildPreamble(FullFilename, *CI,
+      buildPreamble(testPath(Filename), *CI,
                     /*OldPreamble=*/nullptr, Inputs,
                     /*StoreInMemory=*/true, /*PreambleCallback=*/nullptr);
-  auto AST =
-      buildAST(FullFilename, std::move(CI), Diags.take(), Inputs, Preamble);
+  auto AST = buildAST(testPath(Filename), std::move(CI), Diags.take(), Inputs,
+                      Preamble);
   if (!AST.hasValue()) {
     ADD_FAILURE() << "Failed to build code:\n" << Code;
     llvm_unreachable("Failed to build TestTU!");
@@ -79,9 +84,17 @@ ParsedAST TestTU::build() const {
   // This guards against accidental syntax errors silently subverting tests.
   // error-ok is awfully primitive - using clang -verify would be nicer.
   // Ownership and layering makes it pretty hard.
-  if (llvm::none_of(Files, [](const auto &KV) {
-        return llvm::StringRef(KV.second).contains("error-ok");
-      })) {
+  bool ErrorOk = [&, this] {
+    llvm::StringLiteral Marker = "error-ok";
+    if (llvm::StringRef(Code).contains(Marker) ||
+        llvm::StringRef(HeaderCode).contains(Marker))
+      return true;
+    for (const auto& KV : this->AdditionalFiles)
+      if (llvm::StringRef(KV.second).contains(Marker))
+        return true;
+    return false;
+  }();
+  if (!ErrorOk) {
     for (const auto &D : AST->getDiagnostics())
       if (D.Severity >= DiagnosticsEngine::Error) {
         ADD_FAILURE()
diff --git a/clang-tools-extra/clangd/unittests/TestTU.h b/clang-tools-extra/clangd/unittests/TestTU.h
index 4668543d5b4db..229f65a4b95c7 100644
--- a/clang-tools-extra/clangd/unittests/TestTU.h
+++ b/clang-tools-extra/clangd/unittests/TestTU.h
@@ -17,6 +17,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_UNITTESTS_CLANGD_TESTTU_H
 #define LLVM_CLANG_TOOLS_EXTRA_UNITTESTS_CLANGD_TESTTU_H
 
+#include "Compiler.h"
 #include "ParsedAST.h"
 #include "Path.h"
 #include "index/Index.h"
@@ -54,7 +55,7 @@ struct TestTU {
   llvm::StringMap<std::string> AdditionalFiles;
 
   // Extra arguments for the compiler invocation.
-  std::vector<const char *> ExtraArgs;
+  std::vector<std::string> ExtraArgs;
 
   llvm::Optional<std::string> ClangTidyChecks;
   llvm::Optional<std::string> ClangTidyWarningsAsErrors;
@@ -67,6 +68,7 @@ struct TestTU {
   // By default, build() will report Error diagnostics as GTest errors.
   // Suppress this behavior by adding an 'error-ok' comment to the code.
   ParsedAST build() const;
+  ParseInputs inputs() const;
   SymbolSlab headerSymbols() const;
   std::unique_ptr<SymbolIndex> index() const;
 };
diff --git a/clang-tools-extra/clangd/unittests/TweakTesting.h b/clang-tools-extra/clangd/unittests/TweakTesting.h
index 10186f859bae2..c771149a72fc5 100644
--- a/clang-tools-extra/clangd/unittests/TweakTesting.h
+++ b/clang-tools-extra/clangd/unittests/TweakTesting.h
@@ -66,7 +66,7 @@ class TweakTest : public ::testing::Test {
   llvm::StringRef FileName = "TestTU.cpp";
 
   // Extra flags passed to the compilation in apply().
-  std::vector<const char *> ExtraArgs;
+  std::vector<std::string> ExtraArgs;
 
   // Context in which snippets of code should be placed to run tweaks.
   CodeContext Context = File;
diff --git a/clang-tools-extra/clangd/unittests/TweakTests.cpp b/clang-tools-extra/clangd/unittests/TweakTests.cpp
index cae922ffcb955..b5d6117217b63 100644
--- a/clang-tools-extra/clangd/unittests/TweakTests.cpp
+++ b/clang-tools-extra/clangd/unittests/TweakTests.cpp
@@ -2390,6 +2390,250 @@ TEST_F(DefineOutlineTest, FailsMacroSpecifier) {
     EXPECT_EQ(apply(Case.first), Case.second);
   }
 }
+
+TWEAK_TEST(AddUsing);
+TEST_F(AddUsingTest, Prepare) {
+  const std::string Header = R"cpp(
+#define NS(name) one::two::name
+namespace one {
+void oo() {}
+namespace two {
+enum ee {};
+void ff() {}
+class cc {
+public:
+  struct st {};
+  static void mm() {}
+};
+}
+})cpp";
+
+  EXPECT_AVAILABLE(Header + "void fun() { o^n^e^:^:^t^w^o^:^:^f^f(); }");
+  EXPECT_AVAILABLE(Header + "void fun() { o^n^e^::^o^o(); }");
+  EXPECT_AVAILABLE(Header + "void fun() { o^n^e^:^:^t^w^o^:^:^e^e E; }");
+  EXPECT_AVAILABLE(Header + "void fun() { o^n^e^:^:^t^w^o:^:^c^c C; }");
+  EXPECT_UNAVAILABLE(Header +
+                     "void fun() { o^n^e^:^:^t^w^o^:^:^c^c^:^:^m^m(); }");
+  EXPECT_UNAVAILABLE(Header +
+                     "void fun() { o^n^e^:^:^t^w^o^:^:^c^c^:^:^s^t inst; }");
+  EXPECT_UNAVAILABLE(Header +
+                     "void fun() { o^n^e^:^:^t^w^o^:^:^c^c^:^:^s^t inst; }");
+  EXPECT_UNAVAILABLE(Header + "void fun() { N^S(c^c) inst; }");
+}
+
+TEST_F(AddUsingTest, Apply) {
+  FileName = "test.cpp";
+  struct {
+    llvm::StringRef TestSource;
+    llvm::StringRef ExpectedSource;
+  } Cases[]{{
+                // Function, no other using, namespace.
+                R"cpp(
+#include "test.hpp"
+namespace {
+void fun() {
+  ^o^n^e^:^:^t^w^o^:^:^f^f();
+}
+})cpp",
+                R"cpp(
+#include "test.hpp"
+namespace {using one::two::ff;
+
+void fun() {
+  ff();
+}
+})cpp",
+            },
+            // Type, no other using, namespace.
+            {
+                R"cpp(
+#include "test.hpp"
+namespace {
+void fun() {
+  ::on^e::t^wo::c^c inst;
+}
+})cpp",
+                R"cpp(
+#include "test.hpp"
+namespace {using ::one::two::cc;
+
+void fun() {
+  cc inst;
+}
+})cpp",
+            },
+            // Type, no other using, no namespace.
+            {
+                R"cpp(
+#include "test.hpp"
+
+void fun() {
+  on^e::t^wo::e^e inst;
+})cpp",
+                R"cpp(
+#include "test.hpp"
+
+using one::two::ee;
+
+void fun() {
+  ee inst;
+})cpp"},
+            // Function, other usings.
+            {
+                R"cpp(
+#include "test.hpp"
+
+using one::two::cc;
+using one::two::ee;
+
+namespace {
+void fun() {
+  one::two::f^f();
+}
+})cpp",
+                R"cpp(
+#include "test.hpp"
+
+using one::two::cc;
+using one::two::ff;using one::two::ee;
+
+namespace {
+void fun() {
+  ff();
+}
+})cpp",
+            },
+            // Function, other usings inside namespace.
+            {
+                R"cpp(
+#include "test.hpp"
+
+using one::two::cc;
+
+namespace {
+
+using one::two::ff;
+
+void fun() {
+  o^ne::o^o();
+}
+})cpp",
+                R"cpp(
+#include "test.hpp"
+
+using one::two::cc;
+
+namespace {
+
+using one::oo;using one::two::ff;
+
+void fun() {
+  oo();
+}
+})cpp"},
+            // Using comes after cursor.
+            {
+                R"cpp(
+#include "test.hpp"
+
+namespace {
+
+void fun() {
+  one::t^wo::ff();
+}
+
+using one::two::cc;
+
+})cpp",
+                R"cpp(
+#include "test.hpp"
+
+namespace {using one::two::ff;
+
+
+void fun() {
+  ff();
+}
+
+using one::two::cc;
+
+})cpp"},
+            // Pointer type.
+            {R"cpp(
+#include "test.hpp"
+
+void fun() {
+  one::two::c^c *p;
+})cpp",
+             R"cpp(
+#include "test.hpp"
+
+using one::two::cc;
+
+void fun() {
+  cc *p;
+})cpp"},
+            // Namespace declared via macro.
+            {R"cpp(
+#include "test.hpp"
+#define NS_BEGIN(name) namespace name {
+
+NS_BEGIN(foo)
+
+void fun() {
+  one::two::f^f();
+}
+})cpp",
+             R"cpp(
+#include "test.hpp"
+#define NS_BEGIN(name) namespace name {
+
+using one::two::ff;
+
+NS_BEGIN(foo)
+
+void fun() {
+  ff();
+}
+})cpp"},
+            // Inside macro argument.
+            {R"cpp(
+#include "test.hpp"
+#define CALL(name) name()
+
+void fun() {
+  CALL(one::t^wo::ff);
+})cpp",
+             R"cpp(
+#include "test.hpp"
+#define CALL(name) name()
+
+using one::two::ff;
+
+void fun() {
+  CALL(ff);
+})cpp"}};
+  llvm::StringMap<std::string> EditedFiles;
+  for (const auto &Case : Cases) {
+    for (const auto &SubCase : expandCases(Case.TestSource)) {
+      ExtraFiles["test.hpp"] = R"cpp(
+namespace one {
+void oo() {}
+namespace two {
+enum ee {};
+void ff() {}
+class cc {
+public:
+  struct st { struct nested {}; };
+  static void mm() {}
+};
+}
+})cpp";
+      EXPECT_EQ(apply(SubCase, &EditedFiles), Case.ExpectedSource);
+    }
+  }
+}
+
 } // namespace
 } // namespace clangd
 } // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/XRefsTests.cpp b/clang-tools-extra/clangd/unittests/XRefsTests.cpp
index fc36dfa42d7f2..ce7f76ccf4f44 100644
--- a/clang-tools-extra/clangd/unittests/XRefsTests.cpp
+++ b/clang-tools-extra/clangd/unittests/XRefsTests.cpp
@@ -358,15 +358,15 @@ TEST(LocateSymbol, All) {
       )cpp",
 
       R"cpp(// Forward class declaration
-        class Foo;
-        class [[Foo]] {};
+        class $decl[[Foo]];
+        class $def[[Foo]] {};
         F^oo* foo();
       )cpp",
 
       R"cpp(// Function declaration
-        void foo();
+        void $decl[[foo]]();
         void g() { f^oo(); }
-        void [[foo]]() {}
+        void $def[[foo]]() {}
       )cpp",
 
       R"cpp(
@@ -1121,6 +1121,30 @@ TEST(FindReferences, WithinAST) {
   }
 }
 
+TEST(FindReferences, MainFileReferencesOnly) {
+  llvm::StringRef Test =
+      R"cpp(
+        void test() {
+          int [[fo^o]] = 1;
+          // refs not from main file should not be included.
+          #include "foo.inc"
+        })cpp";
+
+  Annotations Code(Test);
+  auto TU = TestTU::withCode(Code.code());
+  TU.AdditionalFiles["foo.inc"] = R"cpp(
+      foo = 3;
+    )cpp";
+  auto AST = TU.build();
+
+  std::vector<Matcher<Location>> ExpectedLocations;
+  for (const auto &R : Code.ranges())
+    ExpectedLocations.push_back(RangeIs(R));
+  EXPECT_THAT(findReferences(AST, Code.point(), 0).References,
+              ElementsAreArray(ExpectedLocations))
+      << Test;
+}
+
 TEST(FindReferences, ExplicitSymbols) {
   const char *Tests[] = {
       R"cpp(
diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability-convert-member-functions-to-static.rst b/clang-tools-extra/docs/clang-tidy/checks/readability-convert-member-functions-to-static.rst
index 891f6be637142..c2f05cf589eac 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/readability-convert-member-functions-to-static.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/readability-convert-member-functions-to-static.rst
@@ -10,5 +10,5 @@ After applying modifications as suggested by the check, runnnig the check again
 might find more opportunities to mark member functions ``static``.
 
 After making a member function ``static``, you might want to run the check
-`readability-static-accessed-through-instance` to replace calls like
+`readability-static-accessed-through-instance <readability-static-accessed-through-instance.html>`_ to replace calls like
 ``Instance.method()`` by ``Class::method()``.
diff --git a/clang-tools-extra/modularize/CoverageChecker.cpp b/clang-tools-extra/modularize/CoverageChecker.cpp
index 9df53d26c2e97..4246df9483d09 100644
--- a/clang-tools-extra/modularize/CoverageChecker.cpp
+++ b/clang-tools-extra/modularize/CoverageChecker.cpp
@@ -338,7 +338,7 @@ bool CoverageChecker::collectFileSystemHeaders() {
   }
 
   // Sort it, because different file systems might order the file differently.
-  std::sort(FileSystemHeaders.begin(), FileSystemHeaders.end());
+  llvm::sort(FileSystemHeaders);
 
   return true;
 }
diff --git a/clang-tools-extra/modularize/Modularize.cpp b/clang-tools-extra/modularize/Modularize.cpp
index 5076bd9ee601e..b1efb0e7996a1 100644
--- a/clang-tools-extra/modularize/Modularize.cpp
+++ b/clang-tools-extra/modularize/Modularize.cpp
@@ -509,7 +509,7 @@ class EntityMap : public StringMap<SmallVector<Entry, 2> > {
              HEnd = CurHeaderContents.end();
          H != HEnd; ++H) {
       // Sort contents.
-      std::sort(H->second.begin(), H->second.end());
+      llvm::sort(H->second);
 
       // Check whether we've seen this header before.
       DenseMap<const FileEntry *, HeaderContents>::iterator KnownH =
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize-use-default-member-init.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize-use-default-member-init.cpp
index 3570fcff4a5cc..60344e06dc31d 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize-use-default-member-init.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize-use-default-member-init.cpp
@@ -432,3 +432,17 @@ class FunctionTryBlock {
   // CHECK-MESSAGES: :[[@LINE-2]]:10: warning: use default member initializer for 'k' [modernize-use-default-member-init]
   // CHECK-FIXES: int i{5}, k{8};
 };
+
+struct PR45363 {
+  // Ensure no warning is emitted here
+  PR45363(int i = 0) : m_i{i} {}
+  int m_i;
+};
+
+struct EmptyBracedIntDefault {
+  EmptyBracedIntDefault() : m_i{} {}
+  int m_i;
+  // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: use default member initializer for 'm_i' [modernize-use-default-member-init]
+  // CHECK-FIXES:      {{^  }}EmptyBracedIntDefault()  {}
+  // CHECK-FIXES-NEXT: {{^  }}int m_i{};
+};
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize-use-using.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize-use-using.cpp
index eb3316a4a04e9..2eccc9066fa6b 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize-use-using.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize-use-using.cpp
@@ -247,19 +247,19 @@ typedef Q<T{0 < 0}.b> Q3_t;
 
 typedef TwoArgTemplate<TwoArgTemplate<int, Q<T{0 < 0}.b> >, S<(0 < 0), Q<b[0 < 0]> > > Nested_t;
 // CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use 'using' instead of 'typedef'
-// CHECK-FIXES: using Nested_t = TwoArgTemplate<TwoArgTemplate<int, Q<T{0 < 0}.b> >, S<(0 < 0), Q<b[0 < 0]> > >;
+// CHECK-FIXES: using Nested_t = TwoArgTemplate<TwoArgTemplate<int, Q<T{0 < 0}.b>>, S<(0 < 0), Q<b[0 < 0]>>>;
 
 template <typename... Args>
 class Variadic {};
 
 typedef Variadic<Variadic<int, bool, Q<T{0 < 0}.b> >, S<(0 < 0), Variadic<Q<b[0 < 0]> > > > Variadic_t;
 // CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use 'using' instead of 'typedef'
-// CHECK-FIXES: using Variadic_t = Variadic<Variadic<int, bool, Q<T{0 < 0}.b> >, S<(0 < 0), Variadic<Q<b[0 < 0]> > > >
+// CHECK-FIXES: using Variadic_t = Variadic<Variadic<int, bool, Q<T{0 < 0}.b>>, S<(0 < 0), Variadic<Q<b[0 < 0]>>>>
 
 typedef Variadic<Variadic<int, bool, Q<T{0 < 0}.b> >, S<(0 < 0), Variadic<Q<b[0 < 0]> > > > Variadic_t, *Variadic_p;
 // CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use 'using' instead of 'typedef'
 // CHECK-MESSAGES: :[[@LINE-2]]:103: warning: use 'using' instead of 'typedef'
-// CHECK-FIXES: using Variadic_t = Variadic<Variadic<int, bool, Q<T{0 < 0}.b> >, S<(0 < 0), Variadic<Q<b[0 < 0]> > > >;
+// CHECK-FIXES: using Variadic_t = Variadic<Variadic<int, bool, Q<T{0 < 0}.b>>, S<(0 < 0), Variadic<Q<b[0 < 0]>>>>;
 // CHECK-FIXES-NEXT: using Variadic_p = Variadic_t*;
 
 typedef struct { int a; } R_t, *R_p;
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability-redundant-string-cstr.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability-redundant-string-cstr.cpp
index 1773dc57a8d88..2561b81805bda 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability-redundant-string-cstr.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability-redundant-string-cstr.cpp
@@ -220,3 +220,27 @@ void m1(std::string&&) {
   m1tp m1p2 = m1;
   m1p2(s.c_str());  
 }
+
+namespace PR45286 {
+struct Foo {
+  void func(const std::string &) {}
+  void func2(std::string &&) {}
+};
+
+void bar() {
+  std::string Str{"aaa"};
+  Foo Foo;
+  Foo.func(Str.c_str());
+  // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: redundant call to 'c_str' [readability-redundant-string-cstr]
+  // CHECK-FIXES: {{^  }}Foo.func(Str);{{$}}
+
+  // Ensure it doesn't transform Binding to r values
+  Foo.func2(Str.c_str());
+
+  // Ensure its not confused by parens
+  Foo.func((Str.c_str()));
+  // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: redundant call to 'c_str' [readability-redundant-string-cstr]
+  // CHECK-FIXES: {{^  }}Foo.func((Str));{{$}}
+  Foo.func2((Str.c_str()));
+}
+} // namespace PR45286
diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt
index 9ddb633a2899d..75ab1a86a170f 100644
--- a/clang/CMakeLists.txt
+++ b/clang/CMakeLists.txt
@@ -226,7 +226,7 @@ set(C_INCLUDE_DIRS "" CACHE STRING
   "Colon separated list of directories clang will search for headers.")
 
 set(GCC_INSTALL_PREFIX "" CACHE PATH "Directory where gcc is installed." )
-set(DEFAULT_SYSROOT "" CACHE PATH
+set(DEFAULT_SYSROOT "" CACHE STRING
   "Default <path> to all compiler invocations for --sysroot=<path>." )
 
 set(ENABLE_LINKER_BUILD_ID OFF CACHE BOOL "pass --build-id to ld")
@@ -306,6 +306,8 @@ if (NOT DEFINED MATCHED_ARCH OR "${CMAKE_MATCH_1}" LESS 35)
     "Default architecture for OpenMP offloading to Nvidia GPUs." FORCE)
 endif()
 
+set(CLANG_SYSTEMZ_DEFAULT_ARCH "z10" CACHE STRING "SystemZ Default Arch")
+
 set(CLANG_VENDOR ${PACKAGE_VENDOR} CACHE STRING
   "Vendor-specific text for showing with version information.")
 
diff --git a/clang/cmake/caches/CrossWinToARMLinux.cmake b/clang/cmake/caches/CrossWinToARMLinux.cmake
index 0d359a1609a5e..3d1e961ada8d0 100644
--- a/clang/cmake/caches/CrossWinToARMLinux.cmake
+++ b/clang/cmake/caches/CrossWinToARMLinux.cmake
@@ -86,6 +86,8 @@ set(LIBCXXABI_TARGET_TRIPLE                 "${CMAKE_C_COMPILER_TARGET}" CACHE S
 set(LIBCXXABI_SYSROOT                       "${DEFAULT_SYSROOT}" CACHE STRING "")
 set(LIBCXXABI_LINK_TESTS_WITH_SHARED_LIBCXXABI OFF CACHE BOOL "")
 set(LIBCXXABI_LINK_TESTS_WITH_SHARED_LIBCXX    OFF CACHE BOOL "")
+set(LIBCXX_LINK_TESTS_WITH_SHARED_LIBCXXABI    OFF CACHE BOOL "")
+set(LIBCXX_LINK_TESTS_WITH_SHARED_LIBCXX       OFF CACHE BOOL "")
 
 set(LIBCXX_USE_COMPILER_RT                  ON CACHE BOOL "")
 set(LIBCXX_TARGET_TRIPLE                    "${CMAKE_C_COMPILER_TARGET}" CACHE STRING "")
diff --git a/clang/cmake/modules/AddClang.cmake b/clang/cmake/modules/AddClang.cmake
index 577cc11ab015b..c1bb386de6f77 100644
--- a/clang/cmake/modules/AddClang.cmake
+++ b/clang/cmake/modules/AddClang.cmake
@@ -17,7 +17,7 @@ function(clang_tablegen)
     message(FATAL_ERROR "SOURCE source-file required by clang_tablegen")
   endif()
 
-  set( CLANG_TABLEGEN_ARGUMENTS -I ${CLANG_SOURCE_DIR}/include )
+  set( CLANG_TABLEGEN_ARGUMENTS "" )
   set( LLVM_TARGET_DEFINITIONS ${CTG_SOURCE} )
   tablegen(CLANG ${CTG_UNPARSED_ARGUMENTS} ${CLANG_TABLEGEN_ARGUMENTS})
 
diff --git a/clang/docs/ClangCommandLineReference.rst b/clang/docs/ClangCommandLineReference.rst
index 3f0e471e852a9..511f3145e7e80 100644
--- a/clang/docs/ClangCommandLineReference.rst
+++ b/clang/docs/ClangCommandLineReference.rst
@@ -2625,6 +2625,10 @@ Use Intel MCU ABI
 
 Generate branches with extended addressability, usually via indirect jumps.
 
+.. option:: -mlvi-cfi, -mno-lvi-cfi
+
+Enable only control-flow mitigations for Load Value Injection (LVI)
+
 .. option:: -mmacosx-version-min=<arg>, -mmacos-version-min=<arg>
 
 Set Mac OS X deployment target
@@ -3145,6 +3149,8 @@ X86
 
 .. option:: -msahf, -mno-sahf
 
+.. option:: -mserialize, -mno-serialize
+
 .. option:: -msgx, -mno-sgx
 
 .. option:: -msha, -mno-sha
diff --git a/clang/docs/ClangPlugins.rst b/clang/docs/ClangPlugins.rst
index 7e33ea33c0df5..4194491d396ae 100644
--- a/clang/docs/ClangPlugins.rst
+++ b/clang/docs/ClangPlugins.rst
@@ -110,6 +110,9 @@ attribute, are:
  * ``existsInTarget``, which checks if the attribute is permitted for the given
    target.
 
+To see a working example of an attribute plugin, see `the Attribute.cpp example
+<https://github.com/llvm/llvm-project/blob/master/clang/examples/Attribute/Attribute.cpp>`_.
+
 Putting it all together
 =======================
 
diff --git a/clang/docs/OpenMPSupport.rst b/clang/docs/OpenMPSupport.rst
index 209a774405372..7fef9e867885a 100644
--- a/clang/docs/OpenMPSupport.rst
+++ b/clang/docs/OpenMPSupport.rst
@@ -229,7 +229,7 @@ implementation.
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
 | base language                | lambda support                                               | :good:`done`             |                                                                       |
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
-| misc extension               | array shaping                                                | :part:`worked on`        | D74144                                                                |
+| misc extension               | array shaping                                                | :good:`done`             | D74144                                                                |
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
 | misc extension               | library shutdown (omp_pause_resource[_all])                  | :none:`unclaimed parts`  | D55078                                                                |
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index ad13fb1b3e95f..a8163cad9fde0 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -111,7 +111,9 @@ New Pragmas in Clang
 Attribute Changes in Clang
 --------------------------
 
-- ...
+- Attributes can now be specified by clang plugins. See the
+  `Clang Plugins <ClangPlugins.html#defining-attributes>`_ documentation for
+  details.
 
 Windows Support
 ---------------
diff --git a/clang/examples/Attribute/Attribute.cpp b/clang/examples/Attribute/Attribute.cpp
new file mode 100644
index 0000000000000..998f175dae546
--- /dev/null
+++ b/clang/examples/Attribute/Attribute.cpp
@@ -0,0 +1,81 @@
+//===- Attribute.cpp ------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Example clang plugin which adds an an annotation to file-scope declarations
+// with the 'example' attribute.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/Attr.h"
+#include "clang/Sema/ParsedAttr.h"
+#include "clang/Sema/Sema.h"
+#include "clang/Sema/SemaDiagnostic.h"
+#include "llvm/IR/Attributes.h"
+using namespace clang;
+
+namespace {
+
+struct ExampleAttrInfo : public ParsedAttrInfo {
+  ExampleAttrInfo() {
+    // Can take an optional string argument (the check that the argument
+    // actually is a string happens in handleDeclAttribute).
+    OptArgs = 1;
+    // GNU-style __attribute__(("example")) and C++-style [[example]] and
+    // [[plugin::example]] supported.
+    static constexpr Spelling S[] = {{ParsedAttr::AS_GNU, "example"},
+                                     {ParsedAttr::AS_CXX11, "example"},
+                                     {ParsedAttr::AS_CXX11, "plugin::example"}};
+    Spellings = S;
+  }
+
+  bool diagAppertainsToDecl(Sema &S, const ParsedAttr &Attr,
+                            const Decl *D) const override {
+    // This attribute appertains to functions only.
+    if (!isa<FunctionDecl>(D)) {
+      S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type_str)
+        << Attr << "functions";
+      return false;
+    }
+    return true;
+  }
+
+  AttrHandling handleDeclAttribute(Sema &S, Decl *D,
+                                   const ParsedAttr &Attr) const override {
+    // Check if the decl is at file scope.
+    if (!D->getDeclContext()->isFileContext()) {
+      unsigned ID = S.getDiagnostics().getCustomDiagID(
+          DiagnosticsEngine::Error,
+          "'example' attribute only allowed at file scope");
+      S.Diag(Attr.getLoc(), ID);
+      return AttributeNotApplied;
+    }
+    // Check if we have an optional string argument.
+    StringRef Str = "";
+    if (Attr.getNumArgs() > 0) {
+      Expr *ArgExpr = Attr.getArgAsExpr(0);
+      StringLiteral *Literal =
+          dyn_cast<StringLiteral>(ArgExpr->IgnoreParenCasts());
+      if (Literal) {
+        Str = Literal->getString();
+      } else {
+        S.Diag(ArgExpr->getExprLoc(), diag::err_attribute_argument_type)
+            << Attr.getAttrName() << AANT_ArgumentString;
+        return AttributeNotApplied;
+      }
+    }
+    // Attach an annotate attribute to the Decl.
+    D->addAttr(AnnotateAttr::Create(S.Context, "example(" + Str.str() + ")",
+                                    Attr.getRange()));
+    return AttributeApplied;
+  }
+};
+
+} // namespace
+
+static ParsedAttrInfoRegistry::Add<ExampleAttrInfo> X("example", "");
diff --git a/clang/examples/Attribute/CMakeLists.txt b/clang/examples/Attribute/CMakeLists.txt
new file mode 100644
index 0000000000000..19323bb0962bc
--- /dev/null
+++ b/clang/examples/Attribute/CMakeLists.txt
@@ -0,0 +1,11 @@
+add_llvm_library(Attribute MODULE Attribute.cpp PLUGIN_TOOL clang)
+
+if(LLVM_ENABLE_PLUGINS AND (WIN32 OR CYGWIN))
+  target_link_libraries(AnnotateFunctions ${cmake_2_8_12_PRIVATE}
+    clangAST
+    clangBasic
+    clangFrontend
+    clangLex
+    LLVMSupport
+    )
+endif()
diff --git a/clang/examples/CMakeLists.txt b/clang/examples/CMakeLists.txt
index e4fedf3682e89..c014b3ddfe97b 100644
--- a/clang/examples/CMakeLists.txt
+++ b/clang/examples/CMakeLists.txt
@@ -6,3 +6,4 @@ endif()
 add_subdirectory(clang-interpreter)
 add_subdirectory(PrintFunctionNames)
 add_subdirectory(AnnotateFunctions)
+add_subdirectory(Attribute)
diff --git a/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h
index ad2bb6a76c438..0acd50021ed8d 100644
--- a/clang/include/clang-c/Index.h
+++ b/clang/include/clang-c/Index.h
@@ -35,22 +35,17 @@
 #define CINDEX_VERSION_MAJOR 0
 #define CINDEX_VERSION_MINOR 59
 
-#define CINDEX_VERSION_ENCODE(major, minor) ( \
-      ((major) * 10000)                       \
-    + ((minor) *     1))
+#define CINDEX_VERSION_ENCODE(major, minor) (((major)*10000) + ((minor)*1))
 
-#define CINDEX_VERSION CINDEX_VERSION_ENCODE( \
-    CINDEX_VERSION_MAJOR,                     \
-    CINDEX_VERSION_MINOR )
+#define CINDEX_VERSION                                                         \
+  CINDEX_VERSION_ENCODE(CINDEX_VERSION_MAJOR, CINDEX_VERSION_MINOR)
 
-#define CINDEX_VERSION_STRINGIZE_(major, minor)   \
-    #major"."#minor
-#define CINDEX_VERSION_STRINGIZE(major, minor)    \
-    CINDEX_VERSION_STRINGIZE_(major, minor)
+#define CINDEX_VERSION_STRINGIZE_(major, minor) #major "." #minor
+#define CINDEX_VERSION_STRINGIZE(major, minor)                                 \
+  CINDEX_VERSION_STRINGIZE_(major, minor)
 
-#define CINDEX_VERSION_STRING CINDEX_VERSION_STRINGIZE( \
-    CINDEX_VERSION_MAJOR,                               \
-    CINDEX_VERSION_MINOR)
+#define CINDEX_VERSION_STRING                                                  \
+  CINDEX_VERSION_STRINGIZE(CINDEX_VERSION_MAJOR, CINDEX_VERSION_MINOR)
 
 LLVM_CLANG_C_EXTERN_C_BEGIN
 
@@ -382,7 +377,7 @@ typedef struct {
  * \param outID stores the returned CXFileUniqueID.
  * \returns If there was a failure getting the unique ID, returns non-zero,
  * otherwise returns 0.
-*/
+ */
 CINDEX_LINKAGE int clang_getFileUniqueID(CXFile file, CXFileUniqueID *outID);
 
 /**
@@ -390,8 +385,8 @@ CINDEX_LINKAGE int clang_getFileUniqueID(CXFile file, CXFileUniqueID *outID);
  * multiple inclusions, either with the conventional
  * \#ifndef/\#define/\#endif macro guards or with \#pragma once.
  */
-CINDEX_LINKAGE unsigned
-clang_isFileMultipleIncludeGuarded(CXTranslationUnit tu, CXFile file);
+CINDEX_LINKAGE unsigned clang_isFileMultipleIncludeGuarded(CXTranslationUnit tu,
+                                                           CXFile file);
 
 /**
  * Retrieve a file handle within the given translation unit.
@@ -496,8 +491,7 @@ CINDEX_LINKAGE unsigned clang_equalLocations(CXSourceLocation loc1,
  * in a particular translation unit.
  */
 CINDEX_LINKAGE CXSourceLocation clang_getLocation(CXTranslationUnit tu,
-                                                  CXFile file,
-                                                  unsigned line,
+                                                  CXFile file, unsigned line,
                                                   unsigned column);
 /**
  * Retrieves the source location associated with a given character offset
@@ -566,8 +560,7 @@ CINDEX_LINKAGE int clang_Range_isNull(CXSourceRange range);
  * buffer to which the given source location points.
  */
 CINDEX_LINKAGE void clang_getExpansionLocation(CXSourceLocation location,
-                                               CXFile *file,
-                                               unsigned *line,
+                                               CXFile *file, unsigned *line,
                                                unsigned *column,
                                                unsigned *offset);
 
@@ -613,8 +606,7 @@ CINDEX_LINKAGE void clang_getExpansionLocation(CXSourceLocation location,
  */
 CINDEX_LINKAGE void clang_getPresumedLocation(CXSourceLocation location,
                                               CXString *filename,
-                                              unsigned *line,
-                                              unsigned *column);
+                                              unsigned *line, unsigned *column);
 
 /**
  * Legacy API to retrieve the file, line, column, and offset represented
@@ -625,8 +617,7 @@ CINDEX_LINKAGE void clang_getPresumedLocation(CXSourceLocation location,
  * details.
  */
 CINDEX_LINKAGE void clang_getInstantiationLocation(CXSourceLocation location,
-                                                   CXFile *file,
-                                                   unsigned *line,
+                                                   CXFile *file, unsigned *line,
                                                    unsigned *column,
                                                    unsigned *offset);
 
@@ -653,8 +644,7 @@ CINDEX_LINKAGE void clang_getInstantiationLocation(CXSourceLocation location,
  * buffer to which the given source location points.
  */
 CINDEX_LINKAGE void clang_getSpellingLocation(CXSourceLocation location,
-                                              CXFile *file,
-                                              unsigned *line,
+                                              CXFile *file, unsigned *line,
                                               unsigned *column,
                                               unsigned *offset);
 
@@ -682,10 +672,8 @@ CINDEX_LINKAGE void clang_getSpellingLocation(CXSourceLocation location,
  * buffer to which the given source location points.
  */
 CINDEX_LINKAGE void clang_getFileLocation(CXSourceLocation location,
-                                          CXFile *file,
-                                          unsigned *line,
-                                          unsigned *column,
-                                          unsigned *offset);
+                                          CXFile *file, unsigned *line,
+                                          unsigned *column, unsigned *offset);
 
 /**
  * Retrieve a source location representing the first character within a
@@ -727,7 +715,8 @@ CINDEX_LINKAGE CXSourceRangeList *clang_getSkippedRanges(CXTranslationUnit tu,
  * The preprocessor will skip lines when they are surrounded by an
  * if/ifdef/ifndef directive whose condition does not evaluate to true.
  */
-CINDEX_LINKAGE CXSourceRangeList *clang_getAllSkippedRanges(CXTranslationUnit tu);
+CINDEX_LINKAGE CXSourceRangeList *
+clang_getAllSkippedRanges(CXTranslationUnit tu);
 
 /**
  * Destroy the given \c CXSourceRangeList.
@@ -758,7 +747,7 @@ enum CXDiagnosticSeverity {
    * This diagnostic is a note that should be attached to the
    * previous (non-note) diagnostic.
    */
-  CXDiagnostic_Note    = 1,
+  CXDiagnostic_Note = 1,
 
   /**
    * This diagnostic indicates suspicious code that may not be
@@ -769,14 +758,14 @@ enum CXDiagnosticSeverity {
   /**
    * This diagnostic indicates that the code is ill-formed.
    */
-  CXDiagnostic_Error   = 3,
+  CXDiagnostic_Error = 3,
 
   /**
    * This diagnostic indicates that the code is ill-formed such
    * that future parser recovery is unlikely to produce useful
    * results.
    */
-  CXDiagnostic_Fatal   = 4
+  CXDiagnostic_Fatal = 4
 };
 
 /**
@@ -849,9 +838,8 @@ enum CXLoadDiag_Error {
  * \returns A loaded CXDiagnosticSet if successful, and NULL otherwise.  These
  * diagnostics should be released using clang_disposeDiagnosticSet().
  */
-CINDEX_LINKAGE CXDiagnosticSet clang_loadDiagnostics(const char *file,
-                                                  enum CXLoadDiag_Error *error,
-                                                  CXString *errorString);
+CINDEX_LINKAGE CXDiagnosticSet clang_loadDiagnostics(
+    const char *file, enum CXLoadDiag_Error *error, CXString *errorString);
 
 /**
  * Release a CXDiagnosticSet and all of its contained diagnostics.
@@ -891,7 +879,7 @@ CINDEX_LINKAGE CXDiagnostic clang_getDiagnostic(CXTranslationUnit Unit,
  * \param Unit the translation unit to query.
  */
 CINDEX_LINKAGE CXDiagnosticSet
-  clang_getDiagnosticSetFromTU(CXTranslationUnit Unit);
+clang_getDiagnosticSetFromTU(CXTranslationUnit Unit);
 
 /**
  * Destroy a diagnostic.
@@ -997,7 +985,7 @@ CINDEX_LINKAGE unsigned clang_defaultDiagnosticDisplayOptions(void);
  * Determine the severity of the given diagnostic.
  */
 CINDEX_LINKAGE enum CXDiagnosticSeverity
-clang_getDiagnosticSeverity(CXDiagnostic);
+    clang_getDiagnosticSeverity(CXDiagnostic);
 
 /**
  * Retrieve the source location of the given diagnostic.
@@ -1049,8 +1037,8 @@ CINDEX_LINKAGE unsigned clang_getDiagnosticCategory(CXDiagnostic);
  *
  * \returns The name of the given diagnostic category.
  */
-CINDEX_DEPRECATED CINDEX_LINKAGE
-CXString clang_getDiagnosticCategoryName(unsigned Category);
+CINDEX_DEPRECATED CINDEX_LINKAGE CXString
+clang_getDiagnosticCategoryName(unsigned Category);
 
 /**
  * Retrieve the diagnostic category text for a given diagnostic.
@@ -1112,9 +1100,8 @@ CINDEX_LINKAGE unsigned clang_getDiagnosticNumFixIts(CXDiagnostic Diagnostic);
  * \returns A string containing text that should be replace the source
  * code indicated by the \c ReplacementRange.
  */
-CINDEX_LINKAGE CXString clang_getDiagnosticFixIt(CXDiagnostic Diagnostic,
-                                                 unsigned FixIt,
-                                               CXSourceRange *ReplacementRange);
+CINDEX_LINKAGE CXString clang_getDiagnosticFixIt(
+    CXDiagnostic Diagnostic, unsigned FixIt, CXSourceRange *ReplacementRange);
 
 /**
  * @}
@@ -1177,12 +1164,9 @@ clang_getTranslationUnitSpelling(CXTranslationUnit CTUnit);
  * guarantee their validity until the call to this function returns.
  */
 CINDEX_LINKAGE CXTranslationUnit clang_createTranslationUnitFromSourceFile(
-                                         CXIndex CIdx,
-                                         const char *source_filename,
-                                         int num_clang_command_line_args,
-                                   const char * const *clang_command_line_args,
-                                         unsigned num_unsaved_files,
-                                         struct CXUnsavedFile *unsaved_files);
+    CXIndex CIdx, const char *source_filename, int num_clang_command_line_args,
+    const char *const *clang_command_line_args, unsigned num_unsaved_files,
+    struct CXUnsavedFile *unsaved_files);
 
 /**
  * Same as \c clang_createTranslationUnit2, but returns
@@ -1190,9 +1174,8 @@ CINDEX_LINKAGE CXTranslationUnit clang_createTranslationUnitFromSourceFile(
  * routine returns a \c NULL \c CXTranslationUnit, without further detailed
  * error codes.
  */
-CINDEX_LINKAGE CXTranslationUnit clang_createTranslationUnit(
-    CXIndex CIdx,
-    const char *ast_filename);
+CINDEX_LINKAGE CXTranslationUnit
+clang_createTranslationUnit(CXIndex CIdx, const char *ast_filename);
 
 /**
  * Create a translation unit from an AST file (\c -emit-ast).
@@ -1202,10 +1185,9 @@ CINDEX_LINKAGE CXTranslationUnit clang_createTranslationUnit(
  *
  * \returns Zero on success, otherwise returns an error code.
  */
-CINDEX_LINKAGE enum CXErrorCode clang_createTranslationUnit2(
-    CXIndex CIdx,
-    const char *ast_filename,
-    CXTranslationUnit *out_TU);
+CINDEX_LINKAGE enum CXErrorCode
+clang_createTranslationUnit2(CXIndex CIdx, const char *ast_filename,
+                             CXTranslationUnit *out_TU);
 
 /**
  * Flags that control the creation of translation units.
@@ -1383,14 +1365,11 @@ CINDEX_LINKAGE unsigned clang_defaultEditingTranslationUnitOptions(void);
  * routine returns a \c NULL \c CXTranslationUnit, without further detailed
  * error codes.
  */
-CINDEX_LINKAGE CXTranslationUnit
-clang_parseTranslationUnit(CXIndex CIdx,
-                           const char *source_filename,
-                           const char *const *command_line_args,
-                           int num_command_line_args,
-                           struct CXUnsavedFile *unsaved_files,
-                           unsigned num_unsaved_files,
-                           unsigned options);
+CINDEX_LINKAGE CXTranslationUnit clang_parseTranslationUnit(
+    CXIndex CIdx, const char *source_filename,
+    const char *const *command_line_args, int num_command_line_args,
+    struct CXUnsavedFile *unsaved_files, unsigned num_unsaved_files,
+    unsigned options);
 
 /**
  * Parse the given source file and the translation unit corresponding
@@ -1436,15 +1415,11 @@ clang_parseTranslationUnit(CXIndex CIdx,
  *
  * \returns Zero on success, otherwise returns an error code.
  */
-CINDEX_LINKAGE enum CXErrorCode
-clang_parseTranslationUnit2(CXIndex CIdx,
-                            const char *source_filename,
-                            const char *const *command_line_args,
-                            int num_command_line_args,
-                            struct CXUnsavedFile *unsaved_files,
-                            unsigned num_unsaved_files,
-                            unsigned options,
-                            CXTranslationUnit *out_TU);
+CINDEX_LINKAGE enum CXErrorCode clang_parseTranslationUnit2(
+    CXIndex CIdx, const char *source_filename,
+    const char *const *command_line_args, int num_command_line_args,
+    struct CXUnsavedFile *unsaved_files, unsigned num_unsaved_files,
+    unsigned options, CXTranslationUnit *out_TU);
 
 /**
  * Same as clang_parseTranslationUnit2 but requires a full command line
@@ -1623,14 +1598,14 @@ CINDEX_LINKAGE unsigned clang_defaultReparseOptions(CXTranslationUnit TU);
  * \c clang_disposeTranslationUnit(TU).  The error codes returned by this
  * routine are described by the \c CXErrorCode enum.
  */
-CINDEX_LINKAGE int clang_reparseTranslationUnit(CXTranslationUnit TU,
-                                                unsigned num_unsaved_files,
-                                          struct CXUnsavedFile *unsaved_files,
-                                                unsigned options);
+CINDEX_LINKAGE int
+clang_reparseTranslationUnit(CXTranslationUnit TU, unsigned num_unsaved_files,
+                             struct CXUnsavedFile *unsaved_files,
+                             unsigned options);
 
 /**
-  * Categorizes how memory is being used by a translation unit.
-  */
+ * Categorizes how memory is being used by a translation unit.
+ */
 enum CXTUResourceUsageKind {
   CXTUResourceUsage_AST = 1,
   CXTUResourceUsage_Identifiers = 2,
@@ -1648,16 +1623,16 @@ enum CXTUResourceUsageKind {
   CXTUResourceUsage_Preprocessor_HeaderSearch = 14,
   CXTUResourceUsage_MEMORY_IN_BYTES_BEGIN = CXTUResourceUsage_AST,
   CXTUResourceUsage_MEMORY_IN_BYTES_END =
-    CXTUResourceUsage_Preprocessor_HeaderSearch,
+      CXTUResourceUsage_Preprocessor_HeaderSearch,
 
   CXTUResourceUsage_First = CXTUResourceUsage_AST,
   CXTUResourceUsage_Last = CXTUResourceUsage_Preprocessor_HeaderSearch
 };
 
 /**
-  * Returns the human-readable null-terminated C string that represents
-  *  the name of the memory category.  This string should never be freed.
-  */
+ * Returns the human-readable null-terminated C string that represents
+ *  the name of the memory category.  This string should never be freed.
+ */
 CINDEX_LINKAGE
 const char *clang_getTUResourceUsageName(enum CXTUResourceUsageKind kind);
 
@@ -1670,8 +1645,8 @@ typedef struct CXTUResourceUsageEntry {
 } CXTUResourceUsageEntry;
 
 /**
-  * The memory usage of a CXTranslationUnit, broken into categories.
-  */
+ * The memory usage of a CXTranslationUnit, broken into categories.
+ */
 typedef struct CXTUResourceUsage {
   /* Private data member, used for queries. */
   void *data;
@@ -1686,10 +1661,11 @@ typedef struct CXTUResourceUsage {
 } CXTUResourceUsage;
 
 /**
-  * Return the memory usage of a translation unit.  This object
-  *  should be released with clang_disposeCXTUResourceUsage().
-  */
-CINDEX_LINKAGE CXTUResourceUsage clang_getCXTUResourceUsage(CXTranslationUnit TU);
+ * Return the memory usage of a translation unit.  This object
+ *  should be released with clang_disposeCXTUResourceUsage().
+ */
+CINDEX_LINKAGE CXTUResourceUsage
+clang_getCXTUResourceUsage(CXTranslationUnit TU);
 
 CINDEX_LINKAGE void clang_disposeCXTUResourceUsage(CXTUResourceUsage usage);
 
@@ -1704,24 +1680,21 @@ clang_getTranslationUnitTargetInfo(CXTranslationUnit CTUnit);
 /**
  * Destroy the CXTargetInfo object.
  */
-CINDEX_LINKAGE void
-clang_TargetInfo_dispose(CXTargetInfo Info);
+CINDEX_LINKAGE void clang_TargetInfo_dispose(CXTargetInfo Info);
 
 /**
  * Get the normalized target triple as a string.
  *
  * Returns the empty string in case of any error.
  */
-CINDEX_LINKAGE CXString
-clang_TargetInfo_getTriple(CXTargetInfo Info);
+CINDEX_LINKAGE CXString clang_TargetInfo_getTriple(CXTargetInfo Info);
 
 /**
  * Get the pointer width of the target in bits.
  *
  * Returns -1 in case of error.
  */
-CINDEX_LINKAGE int
-clang_TargetInfo_getPointerWidth(CXTargetInfo Info);
+CINDEX_LINKAGE int clang_TargetInfo_getPointerWidth(CXTargetInfo Info);
 
 /**
  * @}
@@ -1741,95 +1714,95 @@ enum CXCursorKind {
    * spelling, find their definitions, etc. However, the specific kind
    * of the declaration is not reported.
    */
-  CXCursor_UnexposedDecl                 = 1,
+  CXCursor_UnexposedDecl = 1,
   /** A C or C++ struct. */
-  CXCursor_StructDecl                    = 2,
+  CXCursor_StructDecl = 2,
   /** A C or C++ union. */
-  CXCursor_UnionDecl                     = 3,
+  CXCursor_UnionDecl = 3,
   /** A C++ class. */
-  CXCursor_ClassDecl                     = 4,
+  CXCursor_ClassDecl = 4,
   /** An enumeration. */
-  CXCursor_EnumDecl                      = 5,
+  CXCursor_EnumDecl = 5,
   /**
    * A field (in C) or non-static data member (in C++) in a
    * struct, union, or C++ class.
    */
-  CXCursor_FieldDecl                     = 6,
+  CXCursor_FieldDecl = 6,
   /** An enumerator constant. */
-  CXCursor_EnumConstantDecl              = 7,
+  CXCursor_EnumConstantDecl = 7,
   /** A function. */
-  CXCursor_FunctionDecl                  = 8,
+  CXCursor_FunctionDecl = 8,
   /** A variable. */
-  CXCursor_VarDecl                       = 9,
+  CXCursor_VarDecl = 9,
   /** A function or method parameter. */
-  CXCursor_ParmDecl                      = 10,
+  CXCursor_ParmDecl = 10,
   /** An Objective-C \@interface. */
-  CXCursor_ObjCInterfaceDecl             = 11,
+  CXCursor_ObjCInterfaceDecl = 11,
   /** An Objective-C \@interface for a category. */
-  CXCursor_ObjCCategoryDecl              = 12,
+  CXCursor_ObjCCategoryDecl = 12,
   /** An Objective-C \@protocol declaration. */
-  CXCursor_ObjCProtocolDecl              = 13,
+  CXCursor_ObjCProtocolDecl = 13,
   /** An Objective-C \@property declaration. */
-  CXCursor_ObjCPropertyDecl              = 14,
+  CXCursor_ObjCPropertyDecl = 14,
   /** An Objective-C instance variable. */
-  CXCursor_ObjCIvarDecl                  = 15,
+  CXCursor_ObjCIvarDecl = 15,
   /** An Objective-C instance method. */
-  CXCursor_ObjCInstanceMethodDecl        = 16,
+  CXCursor_ObjCInstanceMethodDecl = 16,
   /** An Objective-C class method. */
-  CXCursor_ObjCClassMethodDecl           = 17,
+  CXCursor_ObjCClassMethodDecl = 17,
   /** An Objective-C \@implementation. */
-  CXCursor_ObjCImplementationDecl        = 18,
+  CXCursor_ObjCImplementationDecl = 18,
   /** An Objective-C \@implementation for a category. */
-  CXCursor_ObjCCategoryImplDecl          = 19,
+  CXCursor_ObjCCategoryImplDecl = 19,
   /** A typedef. */
-  CXCursor_TypedefDecl                   = 20,
+  CXCursor_TypedefDecl = 20,
   /** A C++ class method. */
-  CXCursor_CXXMethod                     = 21,
+  CXCursor_CXXMethod = 21,
   /** A C++ namespace. */
-  CXCursor_Namespace                     = 22,
+  CXCursor_Namespace = 22,
   /** A linkage specification, e.g. 'extern "C"'. */
-  CXCursor_LinkageSpec                   = 23,
+  CXCursor_LinkageSpec = 23,
   /** A C++ constructor. */
-  CXCursor_Constructor                   = 24,
+  CXCursor_Constructor = 24,
   /** A C++ destructor. */
-  CXCursor_Destructor                    = 25,
+  CXCursor_Destructor = 25,
   /** A C++ conversion function. */
-  CXCursor_ConversionFunction            = 26,
+  CXCursor_ConversionFunction = 26,
   /** A C++ template type parameter. */
-  CXCursor_TemplateTypeParameter         = 27,
+  CXCursor_TemplateTypeParameter = 27,
   /** A C++ non-type template parameter. */
-  CXCursor_NonTypeTemplateParameter      = 28,
+  CXCursor_NonTypeTemplateParameter = 28,
   /** A C++ template template parameter. */
-  CXCursor_TemplateTemplateParameter     = 29,
+  CXCursor_TemplateTemplateParameter = 29,
   /** A C++ function template. */
-  CXCursor_FunctionTemplate              = 30,
+  CXCursor_FunctionTemplate = 30,
   /** A C++ class template. */
-  CXCursor_ClassTemplate                 = 31,
+  CXCursor_ClassTemplate = 31,
   /** A C++ class template partial specialization. */
   CXCursor_ClassTemplatePartialSpecialization = 32,
   /** A C++ namespace alias declaration. */
-  CXCursor_NamespaceAlias                = 33,
+  CXCursor_NamespaceAlias = 33,
   /** A C++ using directive. */
-  CXCursor_UsingDirective                = 34,
+  CXCursor_UsingDirective = 34,
   /** A C++ using declaration. */
-  CXCursor_UsingDeclaration              = 35,
+  CXCursor_UsingDeclaration = 35,
   /** A C++ alias declaration */
-  CXCursor_TypeAliasDecl                 = 36,
+  CXCursor_TypeAliasDecl = 36,
   /** An Objective-C \@synthesize definition. */
-  CXCursor_ObjCSynthesizeDecl            = 37,
+  CXCursor_ObjCSynthesizeDecl = 37,
   /** An Objective-C \@dynamic definition. */
-  CXCursor_ObjCDynamicDecl               = 38,
+  CXCursor_ObjCDynamicDecl = 38,
   /** An access specifier. */
-  CXCursor_CXXAccessSpecifier            = 39,
+  CXCursor_CXXAccessSpecifier = 39,
 
-  CXCursor_FirstDecl                     = CXCursor_UnexposedDecl,
-  CXCursor_LastDecl                      = CXCursor_CXXAccessSpecifier,
+  CXCursor_FirstDecl = CXCursor_UnexposedDecl,
+  CXCursor_LastDecl = CXCursor_CXXAccessSpecifier,
 
   /* References */
-  CXCursor_FirstRef                      = 40, /* Decl references */
-  CXCursor_ObjCSuperClassRef             = 40,
-  CXCursor_ObjCProtocolRef               = 41,
-  CXCursor_ObjCClassRef                  = 42,
+  CXCursor_FirstRef = 40, /* Decl references */
+  CXCursor_ObjCSuperClassRef = 40,
+  CXCursor_ObjCProtocolRef = 41,
+  CXCursor_ObjCClassRef = 42,
   /**
    * A reference to a type declaration.
    *
@@ -1845,22 +1818,22 @@ enum CXCursorKind {
    * while the type of the variable "size" is referenced. The cursor
    * referenced by the type of size is the typedef for size_type.
    */
-  CXCursor_TypeRef                       = 43,
-  CXCursor_CXXBaseSpecifier              = 44,
+  CXCursor_TypeRef = 43,
+  CXCursor_CXXBaseSpecifier = 44,
   /**
    * A reference to a class template, function template, template
    * template parameter, or class template partial specialization.
    */
-  CXCursor_TemplateRef                   = 45,
+  CXCursor_TemplateRef = 45,
   /**
    * A reference to a namespace or namespace alias.
    */
-  CXCursor_NamespaceRef                  = 46,
+  CXCursor_NamespaceRef = 46,
   /**
    * A reference to a member of a struct, union, or class that occurs in
    * some non-expression context, e.g., a designated initializer.
    */
-  CXCursor_MemberRef                     = 47,
+  CXCursor_MemberRef = 47,
   /**
    * A reference to a labeled statement.
    *
@@ -1876,7 +1849,7 @@ enum CXCursorKind {
    *
    * A label reference cursor refers to a label statement.
    */
-  CXCursor_LabelRef                      = 48,
+  CXCursor_LabelRef = 48,
 
   /**
    * A reference to a set of overloaded functions or function templates
@@ -1914,26 +1887,26 @@ enum CXCursorKind {
    * \c clang_getOverloadedDecl() can be used to retrieve the definitions
    * referenced by this cursor.
    */
-  CXCursor_OverloadedDeclRef             = 49,
+  CXCursor_OverloadedDeclRef = 49,
 
   /**
    * A reference to a variable that occurs in some non-expression
    * context, e.g., a C++ lambda capture list.
    */
-  CXCursor_VariableRef                   = 50,
+  CXCursor_VariableRef = 50,
 
-  CXCursor_LastRef                       = CXCursor_VariableRef,
+  CXCursor_LastRef = CXCursor_VariableRef,
 
   /* Error conditions */
-  CXCursor_FirstInvalid                  = 70,
-  CXCursor_InvalidFile                   = 70,
-  CXCursor_NoDeclFound                   = 71,
-  CXCursor_NotImplemented                = 72,
-  CXCursor_InvalidCode                   = 73,
-  CXCursor_LastInvalid                   = CXCursor_InvalidCode,
+  CXCursor_FirstInvalid = 70,
+  CXCursor_InvalidFile = 70,
+  CXCursor_NoDeclFound = 71,
+  CXCursor_NotImplemented = 72,
+  CXCursor_InvalidCode = 73,
+  CXCursor_LastInvalid = CXCursor_InvalidCode,
 
   /* Expressions */
-  CXCursor_FirstExpr                     = 100,
+  CXCursor_FirstExpr = 100,
 
   /**
    * An expression whose specific kind is not exposed via this
@@ -1944,104 +1917,104 @@ enum CXCursorKind {
    * spelling, children, etc. However, the specific kind of the
    * expression is not reported.
    */
-  CXCursor_UnexposedExpr                 = 100,
+  CXCursor_UnexposedExpr = 100,
 
   /**
    * An expression that refers to some value declaration, such
    * as a function, variable, or enumerator.
    */
-  CXCursor_DeclRefExpr                   = 101,
+  CXCursor_DeclRefExpr = 101,
 
   /**
    * An expression that refers to a member of a struct, union,
    * class, Objective-C class, etc.
    */
-  CXCursor_MemberRefExpr                 = 102,
+  CXCursor_MemberRefExpr = 102,
 
   /** An expression that calls a function. */
-  CXCursor_CallExpr                      = 103,
+  CXCursor_CallExpr = 103,
 
   /** An expression that sends a message to an Objective-C
    object or class. */
-  CXCursor_ObjCMessageExpr               = 104,
+  CXCursor_ObjCMessageExpr = 104,
 
   /** An expression that represents a block literal. */
-  CXCursor_BlockExpr                     = 105,
+  CXCursor_BlockExpr = 105,
 
   /** An integer literal.
    */
-  CXCursor_IntegerLiteral                = 106,
+  CXCursor_IntegerLiteral = 106,
 
   /** A floating point number literal.
    */
-  CXCursor_FloatingLiteral               = 107,
+  CXCursor_FloatingLiteral = 107,
 
   /** An imaginary number literal.
    */
-  CXCursor_ImaginaryLiteral              = 108,
+  CXCursor_ImaginaryLiteral = 108,
 
   /** A string literal.
    */
-  CXCursor_StringLiteral                 = 109,
+  CXCursor_StringLiteral = 109,
 
   /** A character literal.
    */
-  CXCursor_CharacterLiteral              = 110,
+  CXCursor_CharacterLiteral = 110,
 
   /** A parenthesized expression, e.g. "(1)".
    *
    * This AST node is only formed if full location information is requested.
    */
-  CXCursor_ParenExpr                     = 111,
+  CXCursor_ParenExpr = 111,
 
   /** This represents the unary-expression's (except sizeof and
    * alignof).
    */
-  CXCursor_UnaryOperator                 = 112,
+  CXCursor_UnaryOperator = 112,
 
   /** [C99 6.5.2.1] Array Subscripting.
    */
-  CXCursor_ArraySubscriptExpr            = 113,
+  CXCursor_ArraySubscriptExpr = 113,
 
   /** A builtin binary operation expression such as "x + y" or
    * "x <= y".
    */
-  CXCursor_BinaryOperator                = 114,
+  CXCursor_BinaryOperator = 114,
 
   /** Compound assignment such as "+=".
    */
-  CXCursor_CompoundAssignOperator        = 115,
+  CXCursor_CompoundAssignOperator = 115,
 
   /** The ?: ternary operator.
    */
-  CXCursor_ConditionalOperator           = 116,
+  CXCursor_ConditionalOperator = 116,
 
   /** An explicit cast in C (C99 6.5.4) or a C-style cast in C++
    * (C++ [expr.cast]), which uses the syntax (Type)expr.
    *
    * For example: (int)f.
    */
-  CXCursor_CStyleCastExpr                = 117,
+  CXCursor_CStyleCastExpr = 117,
 
   /** [C99 6.5.2.5]
    */
-  CXCursor_CompoundLiteralExpr           = 118,
+  CXCursor_CompoundLiteralExpr = 118,
 
   /** Describes an C or C++ initializer list.
    */
-  CXCursor_InitListExpr                  = 119,
+  CXCursor_InitListExpr = 119,
 
   /** The GNU address of label extension, representing &&label.
    */
-  CXCursor_AddrLabelExpr                 = 120,
+  CXCursor_AddrLabelExpr = 120,
 
   /** This is the GNU Statement Expression extension: ({int X=4; X;})
    */
-  CXCursor_StmtExpr                      = 121,
+  CXCursor_StmtExpr = 121,
 
   /** Represents a C11 generic selection.
    */
-  CXCursor_GenericSelectionExpr          = 122,
+  CXCursor_GenericSelectionExpr = 122,
 
   /** Implements the GNU __null extension, which is a name for a null
    * pointer constant that has integral type (e.g., int or long) and is the same
@@ -2051,23 +2024,23 @@ enum CXCursorKind {
    * NULL as __null in C++ rather than using 0 (which is an integer that may not
    * match the size of a pointer).
    */
-  CXCursor_GNUNullExpr                   = 123,
+  CXCursor_GNUNullExpr = 123,
 
   /** C++'s static_cast<> expression.
    */
-  CXCursor_CXXStaticCastExpr             = 124,
+  CXCursor_CXXStaticCastExpr = 124,
 
   /** C++'s dynamic_cast<> expression.
    */
-  CXCursor_CXXDynamicCastExpr            = 125,
+  CXCursor_CXXDynamicCastExpr = 125,
 
   /** C++'s reinterpret_cast<> expression.
    */
-  CXCursor_CXXReinterpretCastExpr        = 126,
+  CXCursor_CXXReinterpretCastExpr = 126,
 
   /** C++'s const_cast<> expression.
    */
-  CXCursor_CXXConstCastExpr              = 127,
+  CXCursor_CXXConstCastExpr = 127,
 
   /** Represents an explicit C++ type conversion that uses "functional"
    * notion (C++ [expr.type.conv]).
@@ -2077,60 +2050,60 @@ enum CXCursorKind {
    *   x = int(0.5);
    * \endcode
    */
-  CXCursor_CXXFunctionalCastExpr         = 128,
+  CXCursor_CXXFunctionalCastExpr = 128,
 
   /** A C++ typeid expression (C++ [expr.typeid]).
    */
-  CXCursor_CXXTypeidExpr                 = 129,
+  CXCursor_CXXTypeidExpr = 129,
 
   /** [C++ 2.13.5] C++ Boolean Literal.
    */
-  CXCursor_CXXBoolLiteralExpr            = 130,
+  CXCursor_CXXBoolLiteralExpr = 130,
 
   /** [C++0x 2.14.7] C++ Pointer Literal.
    */
-  CXCursor_CXXNullPtrLiteralExpr         = 131,
+  CXCursor_CXXNullPtrLiteralExpr = 131,
 
   /** Represents the "this" expression in C++
    */
-  CXCursor_CXXThisExpr                   = 132,
+  CXCursor_CXXThisExpr = 132,
 
   /** [C++ 15] C++ Throw Expression.
    *
    * This handles 'throw' and 'throw' assignment-expression. When
    * assignment-expression isn't present, Op will be null.
    */
-  CXCursor_CXXThrowExpr                  = 133,
+  CXCursor_CXXThrowExpr = 133,
 
   /** A new expression for memory allocation and constructor calls, e.g:
    * "new CXXNewExpr(foo)".
    */
-  CXCursor_CXXNewExpr                    = 134,
+  CXCursor_CXXNewExpr = 134,
 
   /** A delete expression for memory deallocation and destructor calls,
    * e.g. "delete[] pArray".
    */
-  CXCursor_CXXDeleteExpr                 = 135,
+  CXCursor_CXXDeleteExpr = 135,
 
   /** A unary expression. (noexcept, sizeof, or other traits)
    */
-  CXCursor_UnaryExpr                     = 136,
+  CXCursor_UnaryExpr = 136,
 
   /** An Objective-C string literal i.e. @"foo".
    */
-  CXCursor_ObjCStringLiteral             = 137,
+  CXCursor_ObjCStringLiteral = 137,
 
   /** An Objective-C \@encode expression.
    */
-  CXCursor_ObjCEncodeExpr                = 138,
+  CXCursor_ObjCEncodeExpr = 138,
 
   /** An Objective-C \@selector expression.
    */
-  CXCursor_ObjCSelectorExpr              = 139,
+  CXCursor_ObjCSelectorExpr = 139,
 
   /** An Objective-C \@protocol expression.
    */
-  CXCursor_ObjCProtocolExpr              = 140,
+  CXCursor_ObjCProtocolExpr = 140,
 
   /** An Objective-C "bridged" cast expression, which casts between
    * Objective-C pointers and C pointers, transferring ownership in the process.
@@ -2139,7 +2112,7 @@ enum CXCursorKind {
    *   NSString *str = (__bridge_transfer NSString *)CFCreateString();
    * \endcode
    */
-  CXCursor_ObjCBridgedCastExpr           = 141,
+  CXCursor_ObjCBridgedCastExpr = 141,
 
   /** Represents a C++0x pack expansion that produces a sequence of
    * expressions.
@@ -2154,7 +2127,7 @@ enum CXCursorKind {
    * }
    * \endcode
    */
-  CXCursor_PackExpansionExpr             = 142,
+  CXCursor_PackExpansionExpr = 142,
 
   /** Represents an expression that computes the length of a parameter
    * pack.
@@ -2166,7 +2139,7 @@ enum CXCursorKind {
    * };
    * \endcode
    */
-  CXCursor_SizeOfPackExpr                = 143,
+  CXCursor_SizeOfPackExpr = 143,
 
   /* Represents a C++ lambda expression that produces a local function
    * object.
@@ -2180,33 +2153,42 @@ enum CXCursorKind {
    * }
    * \endcode
    */
-  CXCursor_LambdaExpr                    = 144,
+  CXCursor_LambdaExpr = 144,
 
   /** Objective-c Boolean Literal.
    */
-  CXCursor_ObjCBoolLiteralExpr           = 145,
+  CXCursor_ObjCBoolLiteralExpr = 145,
 
   /** Represents the "self" expression in an Objective-C method.
    */
-  CXCursor_ObjCSelfExpr                  = 146,
+  CXCursor_ObjCSelfExpr = 146,
 
   /** OpenMP 4.0 [2.4, Array Section].
    */
-  CXCursor_OMPArraySectionExpr           = 147,
+  CXCursor_OMPArraySectionExpr = 147,
 
   /** Represents an @available(...) check.
    */
-  CXCursor_ObjCAvailabilityCheckExpr     = 148,
+  CXCursor_ObjCAvailabilityCheckExpr = 148,
 
   /**
    * Fixed point literal
    */
-  CXCursor_FixedPointLiteral             = 149,
+  CXCursor_FixedPointLiteral = 149,
 
-  CXCursor_LastExpr                      = CXCursor_FixedPointLiteral,
+  /** OpenMP 5.0 [2.1.4, Array Shaping].
+   */
+  CXCursor_OMPArrayShapingExpr = 150,
+
+  /**
+   * OpenMP 5.0 [2.1.6 Iterators]
+   */
+  CXCursor_OMPIteratorExpr = 151,
+
+  CXCursor_LastExpr = CXCursor_OMPIteratorExpr,
 
   /* Statements */
-  CXCursor_FirstStmt                     = 200,
+  CXCursor_FirstStmt = 200,
   /**
    * A statement whose specific kind is not exposed via this
    * interface.
@@ -2216,7 +2198,7 @@ enum CXCursorKind {
    * children, etc. However, the specific kind of the statement is not
    * reported.
    */
-  CXCursor_UnexposedStmt                 = 200,
+  CXCursor_UnexposedStmt = 200,
 
   /** A labelled statement in a function.
    *
@@ -2229,226 +2211,226 @@ enum CXCursorKind {
    * \endcode
    *
    */
-  CXCursor_LabelStmt                     = 201,
+  CXCursor_LabelStmt = 201,
 
   /** A group of statements like { stmt stmt }.
    *
    * This cursor kind is used to describe compound statements, e.g. function
    * bodies.
    */
-  CXCursor_CompoundStmt                  = 202,
+  CXCursor_CompoundStmt = 202,
 
   /** A case statement.
    */
-  CXCursor_CaseStmt                      = 203,
+  CXCursor_CaseStmt = 203,
 
   /** A default statement.
    */
-  CXCursor_DefaultStmt                   = 204,
+  CXCursor_DefaultStmt = 204,
 
   /** An if statement
    */
-  CXCursor_IfStmt                        = 205,
+  CXCursor_IfStmt = 205,
 
   /** A switch statement.
    */
-  CXCursor_SwitchStmt                    = 206,
+  CXCursor_SwitchStmt = 206,
 
   /** A while statement.
    */
-  CXCursor_WhileStmt                     = 207,
+  CXCursor_WhileStmt = 207,
 
   /** A do statement.
    */
-  CXCursor_DoStmt                        = 208,
+  CXCursor_DoStmt = 208,
 
   /** A for statement.
    */
-  CXCursor_ForStmt                       = 209,
+  CXCursor_ForStmt = 209,
 
   /** A goto statement.
    */
-  CXCursor_GotoStmt                      = 210,
+  CXCursor_GotoStmt = 210,
 
   /** An indirect goto statement.
    */
-  CXCursor_IndirectGotoStmt              = 211,
+  CXCursor_IndirectGotoStmt = 211,
 
   /** A continue statement.
    */
-  CXCursor_ContinueStmt                  = 212,
+  CXCursor_ContinueStmt = 212,
 
   /** A break statement.
    */
-  CXCursor_BreakStmt                     = 213,
+  CXCursor_BreakStmt = 213,
 
   /** A return statement.
    */
-  CXCursor_ReturnStmt                    = 214,
+  CXCursor_ReturnStmt = 214,
 
   /** A GCC inline assembly statement extension.
    */
-  CXCursor_GCCAsmStmt                    = 215,
-  CXCursor_AsmStmt                       = CXCursor_GCCAsmStmt,
+  CXCursor_GCCAsmStmt = 215,
+  CXCursor_AsmStmt = CXCursor_GCCAsmStmt,
 
   /** Objective-C's overall \@try-\@catch-\@finally statement.
    */
-  CXCursor_ObjCAtTryStmt                 = 216,
+  CXCursor_ObjCAtTryStmt = 216,
 
   /** Objective-C's \@catch statement.
    */
-  CXCursor_ObjCAtCatchStmt               = 217,
+  CXCursor_ObjCAtCatchStmt = 217,
 
   /** Objective-C's \@finally statement.
    */
-  CXCursor_ObjCAtFinallyStmt             = 218,
+  CXCursor_ObjCAtFinallyStmt = 218,
 
   /** Objective-C's \@throw statement.
    */
-  CXCursor_ObjCAtThrowStmt               = 219,
+  CXCursor_ObjCAtThrowStmt = 219,
 
   /** Objective-C's \@synchronized statement.
    */
-  CXCursor_ObjCAtSynchronizedStmt        = 220,
+  CXCursor_ObjCAtSynchronizedStmt = 220,
 
   /** Objective-C's autorelease pool statement.
    */
-  CXCursor_ObjCAutoreleasePoolStmt       = 221,
+  CXCursor_ObjCAutoreleasePoolStmt = 221,
 
   /** Objective-C's collection statement.
    */
-  CXCursor_ObjCForCollectionStmt         = 222,
+  CXCursor_ObjCForCollectionStmt = 222,
 
   /** C++'s catch statement.
    */
-  CXCursor_CXXCatchStmt                  = 223,
+  CXCursor_CXXCatchStmt = 223,
 
   /** C++'s try statement.
    */
-  CXCursor_CXXTryStmt                    = 224,
+  CXCursor_CXXTryStmt = 224,
 
   /** C++'s for (* : *) statement.
    */
-  CXCursor_CXXForRangeStmt               = 225,
+  CXCursor_CXXForRangeStmt = 225,
 
   /** Windows Structured Exception Handling's try statement.
    */
-  CXCursor_SEHTryStmt                    = 226,
+  CXCursor_SEHTryStmt = 226,
 
   /** Windows Structured Exception Handling's except statement.
    */
-  CXCursor_SEHExceptStmt                 = 227,
+  CXCursor_SEHExceptStmt = 227,
 
   /** Windows Structured Exception Handling's finally statement.
    */
-  CXCursor_SEHFinallyStmt                = 228,
+  CXCursor_SEHFinallyStmt = 228,
 
   /** A MS inline assembly statement extension.
    */
-  CXCursor_MSAsmStmt                     = 229,
+  CXCursor_MSAsmStmt = 229,
 
   /** The null statement ";": C99 6.8.3p3.
    *
    * This cursor kind is used to describe the null statement.
    */
-  CXCursor_NullStmt                      = 230,
+  CXCursor_NullStmt = 230,
 
   /** Adaptor class for mixing declarations with statements and
    * expressions.
    */
-  CXCursor_DeclStmt                      = 231,
+  CXCursor_DeclStmt = 231,
 
   /** OpenMP parallel directive.
    */
-  CXCursor_OMPParallelDirective          = 232,
+  CXCursor_OMPParallelDirective = 232,
 
   /** OpenMP SIMD directive.
    */
-  CXCursor_OMPSimdDirective              = 233,
+  CXCursor_OMPSimdDirective = 233,
 
   /** OpenMP for directive.
    */
-  CXCursor_OMPForDirective               = 234,
+  CXCursor_OMPForDirective = 234,
 
   /** OpenMP sections directive.
    */
-  CXCursor_OMPSectionsDirective          = 235,
+  CXCursor_OMPSectionsDirective = 235,
 
   /** OpenMP section directive.
    */
-  CXCursor_OMPSectionDirective           = 236,
+  CXCursor_OMPSectionDirective = 236,
 
   /** OpenMP single directive.
    */
-  CXCursor_OMPSingleDirective            = 237,
+  CXCursor_OMPSingleDirective = 237,
 
   /** OpenMP parallel for directive.
    */
-  CXCursor_OMPParallelForDirective       = 238,
+  CXCursor_OMPParallelForDirective = 238,
 
   /** OpenMP parallel sections directive.
    */
-  CXCursor_OMPParallelSectionsDirective  = 239,
+  CXCursor_OMPParallelSectionsDirective = 239,
 
   /** OpenMP task directive.
    */
-  CXCursor_OMPTaskDirective              = 240,
+  CXCursor_OMPTaskDirective = 240,
 
   /** OpenMP master directive.
    */
-  CXCursor_OMPMasterDirective            = 241,
+  CXCursor_OMPMasterDirective = 241,
 
   /** OpenMP critical directive.
    */
-  CXCursor_OMPCriticalDirective          = 242,
+  CXCursor_OMPCriticalDirective = 242,
 
   /** OpenMP taskyield directive.
    */
-  CXCursor_OMPTaskyieldDirective         = 243,
+  CXCursor_OMPTaskyieldDirective = 243,
 
   /** OpenMP barrier directive.
    */
-  CXCursor_OMPBarrierDirective           = 244,
+  CXCursor_OMPBarrierDirective = 244,
 
   /** OpenMP taskwait directive.
    */
-  CXCursor_OMPTaskwaitDirective          = 245,
+  CXCursor_OMPTaskwaitDirective = 245,
 
   /** OpenMP flush directive.
    */
-  CXCursor_OMPFlushDirective             = 246,
+  CXCursor_OMPFlushDirective = 246,
 
   /** Windows Structured Exception Handling's leave statement.
    */
-  CXCursor_SEHLeaveStmt                  = 247,
+  CXCursor_SEHLeaveStmt = 247,
 
   /** OpenMP ordered directive.
    */
-  CXCursor_OMPOrderedDirective           = 248,
+  CXCursor_OMPOrderedDirective = 248,
 
   /** OpenMP atomic directive.
    */
-  CXCursor_OMPAtomicDirective            = 249,
+  CXCursor_OMPAtomicDirective = 249,
 
   /** OpenMP for SIMD directive.
    */
-  CXCursor_OMPForSimdDirective           = 250,
+  CXCursor_OMPForSimdDirective = 250,
 
   /** OpenMP parallel for SIMD directive.
    */
-  CXCursor_OMPParallelForSimdDirective   = 251,
+  CXCursor_OMPParallelForSimdDirective = 251,
 
   /** OpenMP target directive.
    */
-  CXCursor_OMPTargetDirective            = 252,
+  CXCursor_OMPTargetDirective = 252,
 
   /** OpenMP teams directive.
    */
-  CXCursor_OMPTeamsDirective             = 253,
+  CXCursor_OMPTeamsDirective = 253,
 
   /** OpenMP taskgroup directive.
    */
-  CXCursor_OMPTaskgroupDirective         = 254,
+  CXCursor_OMPTaskgroupDirective = 254,
 
   /** OpenMP cancellation point directive.
    */
@@ -2456,35 +2438,35 @@ enum CXCursorKind {
 
   /** OpenMP cancel directive.
    */
-  CXCursor_OMPCancelDirective            = 256,
+  CXCursor_OMPCancelDirective = 256,
 
   /** OpenMP target data directive.
    */
-  CXCursor_OMPTargetDataDirective        = 257,
+  CXCursor_OMPTargetDataDirective = 257,
 
   /** OpenMP taskloop directive.
    */
-  CXCursor_OMPTaskLoopDirective          = 258,
+  CXCursor_OMPTaskLoopDirective = 258,
 
   /** OpenMP taskloop simd directive.
    */
-  CXCursor_OMPTaskLoopSimdDirective      = 259,
+  CXCursor_OMPTaskLoopSimdDirective = 259,
 
   /** OpenMP distribute directive.
    */
-  CXCursor_OMPDistributeDirective        = 260,
+  CXCursor_OMPDistributeDirective = 260,
 
   /** OpenMP target enter data directive.
    */
-  CXCursor_OMPTargetEnterDataDirective   = 261,
+  CXCursor_OMPTargetEnterDataDirective = 261,
 
   /** OpenMP target exit data directive.
    */
-  CXCursor_OMPTargetExitDataDirective    = 262,
+  CXCursor_OMPTargetExitDataDirective = 262,
 
   /** OpenMP target parallel directive.
    */
-  CXCursor_OMPTargetParallelDirective    = 263,
+  CXCursor_OMPTargetParallelDirective = 263,
 
   /** OpenMP target parallel for directive.
    */
@@ -2492,7 +2474,7 @@ enum CXCursorKind {
 
   /** OpenMP target update directive.
    */
-  CXCursor_OMPTargetUpdateDirective      = 265,
+  CXCursor_OMPTargetUpdateDirective = 265,
 
   /** OpenMP distribute parallel for directive.
    */
@@ -2564,23 +2546,23 @@ enum CXCursorKind {
 
   /** OpenMP master taskloop simd directive.
    */
-  CXCursor_OMPMasterTaskLoopSimdDirective      = 283,
+  CXCursor_OMPMasterTaskLoopSimdDirective = 283,
 
   /** OpenMP parallel master taskloop simd directive.
    */
-  CXCursor_OMPParallelMasterTaskLoopSimdDirective      = 284,
+  CXCursor_OMPParallelMasterTaskLoopSimdDirective = 284,
 
   /** OpenMP parallel master directive.
    */
-  CXCursor_OMPParallelMasterDirective      = 285,
+  CXCursor_OMPParallelMasterDirective = 285,
 
   /** OpenMP depobj directive.
    */
-  CXCursor_OMPDepobjDirective             = 286,
+  CXCursor_OMPDepobjDirective = 286,
 
   /** OpenMP scan directive.
    */
-  CXCursor_OMPScanDirective               = 287,
+  CXCursor_OMPScanDirective = 287,
 
   CXCursor_LastStmt = CXCursor_OMPScanDirective,
 
@@ -2590,89 +2572,89 @@ enum CXCursorKind {
    * The translation unit cursor exists primarily to act as the root
    * cursor for traversing the contents of a translation unit.
    */
-  CXCursor_TranslationUnit               = 300,
+  CXCursor_TranslationUnit = 300,
 
   /* Attributes */
-  CXCursor_FirstAttr                     = 400,
+  CXCursor_FirstAttr = 400,
   /**
    * An attribute whose specific kind is not exposed via this
    * interface.
    */
-  CXCursor_UnexposedAttr                 = 400,
-
-  CXCursor_IBActionAttr                  = 401,
-  CXCursor_IBOutletAttr                  = 402,
-  CXCursor_IBOutletCollectionAttr        = 403,
-  CXCursor_CXXFinalAttr                  = 404,
-  CXCursor_CXXOverrideAttr               = 405,
-  CXCursor_AnnotateAttr                  = 406,
-  CXCursor_AsmLabelAttr                  = 407,
-  CXCursor_PackedAttr                    = 408,
-  CXCursor_PureAttr                      = 409,
-  CXCursor_ConstAttr                     = 410,
-  CXCursor_NoDuplicateAttr               = 411,
-  CXCursor_CUDAConstantAttr              = 412,
-  CXCursor_CUDADeviceAttr                = 413,
-  CXCursor_CUDAGlobalAttr                = 414,
-  CXCursor_CUDAHostAttr                  = 415,
-  CXCursor_CUDASharedAttr                = 416,
-  CXCursor_VisibilityAttr                = 417,
-  CXCursor_DLLExport                     = 418,
-  CXCursor_DLLImport                     = 419,
-  CXCursor_NSReturnsRetained             = 420,
-  CXCursor_NSReturnsNotRetained          = 421,
-  CXCursor_NSReturnsAutoreleased         = 422,
-  CXCursor_NSConsumesSelf                = 423,
-  CXCursor_NSConsumed                    = 424,
-  CXCursor_ObjCException                 = 425,
-  CXCursor_ObjCNSObject                  = 426,
-  CXCursor_ObjCIndependentClass          = 427,
-  CXCursor_ObjCPreciseLifetime           = 428,
-  CXCursor_ObjCReturnsInnerPointer       = 429,
-  CXCursor_ObjCRequiresSuper             = 430,
-  CXCursor_ObjCRootClass                 = 431,
-  CXCursor_ObjCSubclassingRestricted     = 432,
-  CXCursor_ObjCExplicitProtocolImpl      = 433,
-  CXCursor_ObjCDesignatedInitializer     = 434,
-  CXCursor_ObjCRuntimeVisible            = 435,
-  CXCursor_ObjCBoxable                   = 436,
-  CXCursor_FlagEnum                      = 437,
-  CXCursor_ConvergentAttr                = 438,
-  CXCursor_WarnUnusedAttr                = 439,
-  CXCursor_WarnUnusedResultAttr          = 440,
-  CXCursor_AlignedAttr                   = 441,
-  CXCursor_LastAttr                      = CXCursor_AlignedAttr,
+  CXCursor_UnexposedAttr = 400,
+
+  CXCursor_IBActionAttr = 401,
+  CXCursor_IBOutletAttr = 402,
+  CXCursor_IBOutletCollectionAttr = 403,
+  CXCursor_CXXFinalAttr = 404,
+  CXCursor_CXXOverrideAttr = 405,
+  CXCursor_AnnotateAttr = 406,
+  CXCursor_AsmLabelAttr = 407,
+  CXCursor_PackedAttr = 408,
+  CXCursor_PureAttr = 409,
+  CXCursor_ConstAttr = 410,
+  CXCursor_NoDuplicateAttr = 411,
+  CXCursor_CUDAConstantAttr = 412,
+  CXCursor_CUDADeviceAttr = 413,
+  CXCursor_CUDAGlobalAttr = 414,
+  CXCursor_CUDAHostAttr = 415,
+  CXCursor_CUDASharedAttr = 416,
+  CXCursor_VisibilityAttr = 417,
+  CXCursor_DLLExport = 418,
+  CXCursor_DLLImport = 419,
+  CXCursor_NSReturnsRetained = 420,
+  CXCursor_NSReturnsNotRetained = 421,
+  CXCursor_NSReturnsAutoreleased = 422,
+  CXCursor_NSConsumesSelf = 423,
+  CXCursor_NSConsumed = 424,
+  CXCursor_ObjCException = 425,
+  CXCursor_ObjCNSObject = 426,
+  CXCursor_ObjCIndependentClass = 427,
+  CXCursor_ObjCPreciseLifetime = 428,
+  CXCursor_ObjCReturnsInnerPointer = 429,
+  CXCursor_ObjCRequiresSuper = 430,
+  CXCursor_ObjCRootClass = 431,
+  CXCursor_ObjCSubclassingRestricted = 432,
+  CXCursor_ObjCExplicitProtocolImpl = 433,
+  CXCursor_ObjCDesignatedInitializer = 434,
+  CXCursor_ObjCRuntimeVisible = 435,
+  CXCursor_ObjCBoxable = 436,
+  CXCursor_FlagEnum = 437,
+  CXCursor_ConvergentAttr = 438,
+  CXCursor_WarnUnusedAttr = 439,
+  CXCursor_WarnUnusedResultAttr = 440,
+  CXCursor_AlignedAttr = 441,
+  CXCursor_LastAttr = CXCursor_AlignedAttr,
 
   /* Preprocessing */
-  CXCursor_PreprocessingDirective        = 500,
-  CXCursor_MacroDefinition               = 501,
-  CXCursor_MacroExpansion                = 502,
-  CXCursor_MacroInstantiation            = CXCursor_MacroExpansion,
-  CXCursor_InclusionDirective            = 503,
-  CXCursor_FirstPreprocessing            = CXCursor_PreprocessingDirective,
-  CXCursor_LastPreprocessing             = CXCursor_InclusionDirective,
+  CXCursor_PreprocessingDirective = 500,
+  CXCursor_MacroDefinition = 501,
+  CXCursor_MacroExpansion = 502,
+  CXCursor_MacroInstantiation = CXCursor_MacroExpansion,
+  CXCursor_InclusionDirective = 503,
+  CXCursor_FirstPreprocessing = CXCursor_PreprocessingDirective,
+  CXCursor_LastPreprocessing = CXCursor_InclusionDirective,
 
   /* Extra Declarations */
   /**
    * A module import declaration.
    */
-  CXCursor_ModuleImportDecl              = 600,
-  CXCursor_TypeAliasTemplateDecl         = 601,
+  CXCursor_ModuleImportDecl = 600,
+  CXCursor_TypeAliasTemplateDecl = 601,
   /**
    * A static_assert or _Static_assert node
    */
-  CXCursor_StaticAssert                  = 602,
+  CXCursor_StaticAssert = 602,
   /**
    * a friend declaration.
    */
-  CXCursor_FriendDecl                    = 603,
-  CXCursor_FirstExtraDecl                = CXCursor_ModuleImportDecl,
-  CXCursor_LastExtraDecl                 = CXCursor_FriendDecl,
+  CXCursor_FriendDecl = 603,
+  CXCursor_FirstExtraDecl = CXCursor_ModuleImportDecl,
+  CXCursor_LastExtraDecl = CXCursor_FriendDecl,
 
   /**
    * A code completion overload candidate.
    */
-  CXCursor_OverloadCandidate             = 700
+  CXCursor_OverloadCandidate = 700
 };
 
 /**
@@ -2943,14 +2925,10 @@ typedef struct CXPlatformAvailability {
  * platform-availability structures returned. There are
  * \c min(N, availability_size) such structures.
  */
-CINDEX_LINKAGE int
-clang_getCursorPlatformAvailability(CXCursor cursor,
-                                    int *always_deprecated,
-                                    CXString *deprecated_message,
-                                    int *always_unavailable,
-                                    CXString *unavailable_message,
-                                    CXPlatformAvailability *availability,
-                                    int availability_size);
+CINDEX_LINKAGE int clang_getCursorPlatformAvailability(
+    CXCursor cursor, int *always_deprecated, CXString *deprecated_message,
+    int *always_unavailable, CXString *unavailable_message,
+    CXPlatformAvailability *availability, int availability_size);
 
 /**
  * Free the memory associated with a \c CXPlatformAvailability structure.
@@ -2977,11 +2955,7 @@ CINDEX_LINKAGE enum CXLanguageKind clang_getCursorLanguage(CXCursor cursor);
  * Describe the "thread-local storage (TLS) kind" of the declaration
  * referred to by a cursor.
  */
-enum CXTLSKind {
-  CXTLS_None = 0,
-  CXTLS_Dynamic,
-  CXTLS_Static
-};
+enum CXTLSKind { CXTLS_None = 0, CXTLS_Dynamic, CXTLS_Static };
 
 /**
  * Determine the "thread-local storage (TLS) kind" of the declaration
@@ -3013,7 +2987,7 @@ CINDEX_LINKAGE void clang_disposeCXCursorSet(CXCursorSet cset);
  * Queries a CXCursorSet to see if it contains a specific CXCursor.
  *
  * \returns non-zero if the set contains the specified cursor.
-*/
+ */
 CINDEX_LINKAGE unsigned clang_CXCursorSet_contains(CXCursorSet cset,
                                                    CXCursor cursor);
 
@@ -3021,7 +2995,7 @@ CINDEX_LINKAGE unsigned clang_CXCursorSet_contains(CXCursorSet cset,
  * Inserts a CXCursor into a CXCursorSet.
  *
  * \returns zero if the CXCursor was already in the set, and non-zero otherwise.
-*/
+ */
 CINDEX_LINKAGE unsigned clang_CXCursorSet_insert(CXCursorSet cset,
                                                  CXCursor cursor);
 
@@ -3441,9 +3415,9 @@ CINDEX_LINKAGE CXType clang_getEnumDeclIntegerType(CXCursor C);
  * Retrieve the integer value of an enum constant declaration as a signed
  *  long long.
  *
- * If the cursor does not reference an enum constant declaration, LLONG_MIN is returned.
- * Since this is also potentially a valid constant value, the kind of the cursor
- * must be verified before calling this function.
+ * If the cursor does not reference an enum constant declaration, LLONG_MIN is
+ * returned. Since this is also potentially a valid constant value, the kind of
+ * the cursor must be verified before calling this function.
  */
 CINDEX_LINKAGE long long clang_getEnumConstantDeclValue(CXCursor C);
 
@@ -3451,11 +3425,12 @@ CINDEX_LINKAGE long long clang_getEnumConstantDeclValue(CXCursor C);
  * Retrieve the integer value of an enum constant declaration as an unsigned
  *  long long.
  *
- * If the cursor does not reference an enum constant declaration, ULLONG_MAX is returned.
- * Since this is also potentially a valid constant value, the kind of the cursor
- * must be verified before calling this function.
+ * If the cursor does not reference an enum constant declaration, ULLONG_MAX is
+ * returned. Since this is also potentially a valid constant value, the kind of
+ * the cursor must be verified before calling this function.
  */
-CINDEX_LINKAGE unsigned long long clang_getEnumConstantDeclUnsignedValue(CXCursor C);
+CINDEX_LINKAGE unsigned long long
+clang_getEnumConstantDeclUnsignedValue(CXCursor C);
 
 /**
  * Retrieve the bit width of a bit field declaration as an integer.
@@ -3536,8 +3511,8 @@ CINDEX_LINKAGE int clang_Cursor_getNumTemplateArguments(CXCursor C);
  * For I = 0, 1, and 2, Type, Integral, and Integral will be returned,
  * respectively.
  */
-CINDEX_LINKAGE enum CXTemplateArgumentKind clang_Cursor_getTemplateArgumentKind(
-    CXCursor C, unsigned I);
+CINDEX_LINKAGE enum CXTemplateArgumentKind
+clang_Cursor_getTemplateArgumentKind(CXCursor C, unsigned I);
 
 /**
  * Retrieve a CXType representing the type of a TemplateArgument of a
@@ -3597,8 +3572,8 @@ CINDEX_LINKAGE long long clang_Cursor_getTemplateArgumentValue(CXCursor C,
  * If called with I = 1 or 2, 2147483649 or true will be returned, respectively.
  * For I == 0, this function's behavior is undefined.
  */
-CINDEX_LINKAGE unsigned long long clang_Cursor_getTemplateArgumentUnsignedValue(
-    CXCursor C, unsigned I);
+CINDEX_LINKAGE unsigned long long
+clang_Cursor_getTemplateArgumentUnsignedValue(CXCursor C, unsigned I);
 
 /**
  * Determine whether two CXTypes represent the same type.
@@ -3783,7 +3758,8 @@ CINDEX_LINKAGE CXType clang_getCursorResultType(CXCursor C);
  * Retrieve the exception specification type associated with a given cursor.
  * This is a value of type CXCursor_ExceptionSpecificationKind.
  *
- * This only returns a valid result if the cursor refers to a function or method.
+ * This only returns a valid result if the cursor refers to a function or
+ * method.
  */
 CINDEX_LINKAGE int clang_getCursorExceptionSpecificationType(CXCursor C);
 
@@ -4011,7 +3987,8 @@ CINDEX_LINKAGE int clang_Type_getNumTemplateArguments(CXType T);
  * This function only returns template type arguments and does not handle
  * template template arguments or variadic packs.
  */
-CINDEX_LINKAGE CXType clang_Type_getTemplateArgumentAsType(CXType T, unsigned i);
+CINDEX_LINKAGE CXType clang_Type_getTemplateArgumentAsType(CXType T,
+                                                           unsigned i);
 
 /**
  * Retrieve the ref-qualifier kind of a function or method.
@@ -4047,9 +4024,9 @@ enum CX_CXXAccessSpecifier {
 /**
  * Returns the access control level for the referenced object.
  *
- * If the cursor refers to a C++ declaration, its access control level within its
- * parent scope is returned. Otherwise, if the cursor refers to a base specifier or
- * access specifier, the specifier itself is returned.
+ * If the cursor refers to a C++ declaration, its access control level within
+ * its parent scope is returned. Otherwise, if the cursor refers to a base
+ * specifier or access specifier, the specifier itself is returned.
  */
 CINDEX_LINKAGE enum CX_CXXAccessSpecifier clang_getCXXAccessSpecifier(CXCursor);
 
@@ -4200,7 +4177,7 @@ CINDEX_LINKAGE unsigned clang_visitChildren(CXCursor parent,
                                             CXCursorVisitor visitor,
                                             CXClientData client_data);
 #ifdef __has_feature
-#  if __has_feature(blocks)
+#if __has_feature(blocks)
 /**
  * Visitor invoked for each cursor found by a traversal.
  *
@@ -4211,16 +4188,16 @@ CINDEX_LINKAGE unsigned clang_visitChildren(CXCursor parent,
  * The visitor should return one of the \c CXChildVisitResult values
  * to direct clang_visitChildrenWithBlock().
  */
-typedef enum CXChildVisitResult
-     (^CXCursorVisitorBlock)(CXCursor cursor, CXCursor parent);
+typedef enum CXChildVisitResult (^CXCursorVisitorBlock)(CXCursor cursor,
+                                                        CXCursor parent);
 
 /**
  * Visits the children of a cursor using the specified block.  Behaves
  * identically to clang_visitChildren() in all other respects.
  */
-CINDEX_LINKAGE unsigned clang_visitChildrenWithBlock(CXCursor parent,
-                                                    CXCursorVisitorBlock block);
-#  endif
+CINDEX_LINKAGE unsigned
+clang_visitChildrenWithBlock(CXCursor parent, CXCursorVisitorBlock block);
+#endif
 #endif
 
 /**
@@ -4257,15 +4234,14 @@ CINDEX_LINKAGE CXString clang_constructUSR_ObjCClass(const char *class_name);
 /**
  * Construct a USR for a specified Objective-C category.
  */
-CINDEX_LINKAGE CXString
-  clang_constructUSR_ObjCCategory(const char *class_name,
-                                 const char *category_name);
+CINDEX_LINKAGE CXString clang_constructUSR_ObjCCategory(
+    const char *class_name, const char *category_name);
 
 /**
  * Construct a USR for a specified Objective-C protocol.
  */
 CINDEX_LINKAGE CXString
-  clang_constructUSR_ObjCProtocol(const char *protocol_name);
+clang_constructUSR_ObjCProtocol(const char *protocol_name);
 
 /**
  * Construct a USR for a specified Objective-C instance variable and
@@ -4305,9 +4281,8 @@ CINDEX_LINKAGE CXString clang_getCursorSpelling(CXCursor);
  *
  * \param options Reserved.
  */
-CINDEX_LINKAGE CXSourceRange clang_Cursor_getSpellingNameRange(CXCursor,
-                                                          unsigned pieceIndex,
-                                                          unsigned options);
+CINDEX_LINKAGE CXSourceRange clang_Cursor_getSpellingNameRange(
+    CXCursor, unsigned pieceIndex, unsigned options);
 
 /**
  * Opaque pointer representing a policy that controls pretty printing
@@ -4361,9 +4336,10 @@ clang_PrintingPolicy_getProperty(CXPrintingPolicy Policy,
 /**
  * Set a property value for the given printing policy.
  */
-CINDEX_LINKAGE void clang_PrintingPolicy_setProperty(CXPrintingPolicy Policy,
-                                                     enum CXPrintingPolicyProperty Property,
-                                                     unsigned Value);
+CINDEX_LINKAGE void
+clang_PrintingPolicy_setProperty(CXPrintingPolicy Policy,
+                                 enum CXPrintingPolicyProperty Property,
+                                 unsigned Value);
 
 /**
  * Retrieve the default policy for the cursor.
@@ -4511,18 +4487,18 @@ CINDEX_LINKAGE CXType clang_Cursor_getReceiverType(CXCursor C);
  * Property attributes for a \c CXCursor_ObjCPropertyDecl.
  */
 typedef enum {
-  CXObjCPropertyAttr_noattr    = 0x00,
-  CXObjCPropertyAttr_readonly  = 0x01,
-  CXObjCPropertyAttr_getter    = 0x02,
-  CXObjCPropertyAttr_assign    = 0x04,
+  CXObjCPropertyAttr_noattr = 0x00,
+  CXObjCPropertyAttr_readonly = 0x01,
+  CXObjCPropertyAttr_getter = 0x02,
+  CXObjCPropertyAttr_assign = 0x04,
   CXObjCPropertyAttr_readwrite = 0x08,
-  CXObjCPropertyAttr_retain    = 0x10,
-  CXObjCPropertyAttr_copy      = 0x20,
+  CXObjCPropertyAttr_retain = 0x10,
+  CXObjCPropertyAttr_copy = 0x20,
   CXObjCPropertyAttr_nonatomic = 0x40,
-  CXObjCPropertyAttr_setter    = 0x80,
-  CXObjCPropertyAttr_atomic    = 0x100,
-  CXObjCPropertyAttr_weak      = 0x200,
-  CXObjCPropertyAttr_strong    = 0x400,
+  CXObjCPropertyAttr_setter = 0x80,
+  CXObjCPropertyAttr_atomic = 0x100,
+  CXObjCPropertyAttr_weak = 0x200,
+  CXObjCPropertyAttr_strong = 0x400,
   CXObjCPropertyAttr_unsafe_unretained = 0x800,
   CXObjCPropertyAttr_class = 0x1000
 } CXObjCPropertyAttrKind;
@@ -4534,8 +4510,8 @@ typedef enum {
  *
  * \param reserved Reserved for future use, pass 0.
  */
-CINDEX_LINKAGE unsigned clang_Cursor_getObjCPropertyAttributes(CXCursor C,
-                                                             unsigned reserved);
+CINDEX_LINKAGE unsigned
+clang_Cursor_getObjCPropertyAttributes(CXCursor C, unsigned reserved);
 
 /**
  * Given a cursor that represents a property declaration, return the
@@ -4597,8 +4573,9 @@ CINDEX_LINKAGE unsigned clang_Cursor_isVariadic(CXCursor C);
  * non-zero if the 'generated_declaration' is set in the attribute.
  */
 CINDEX_LINKAGE unsigned clang_Cursor_isExternalSymbol(CXCursor C,
-                                       CXString *language, CXString *definedIn,
-                                       unsigned *isGenerated);
+                                                      CXString *language,
+                                                      CXString *definedIn,
+                                                      unsigned *isGenerated);
 
 /**
  * Given a cursor that represents a declaration, return the associated
@@ -4724,8 +4701,8 @@ CINDEX_LINKAGE unsigned clang_Module_getNumTopLevelHeaders(CXTranslationUnit,
  * \returns the specified top level header associated with the module.
  */
 CINDEX_LINKAGE
-CXFile clang_Module_getTopLevelHeader(CXTranslationUnit,
-                                      CXModule Module, unsigned Index);
+CXFile clang_Module_getTopLevelHeader(CXTranslationUnit, CXModule Module,
+                                      unsigned Index);
 
 /**
  * @}
@@ -4743,7 +4720,8 @@ CXFile clang_Module_getTopLevelHeader(CXTranslationUnit,
 /**
  * Determine if a C++ constructor is a converting constructor.
  */
-CINDEX_LINKAGE unsigned clang_CXXConstructor_isConvertingConstructor(CXCursor C);
+CINDEX_LINKAGE unsigned
+clang_CXXConstructor_isConvertingConstructor(CXCursor C);
 
 /**
  * Determine if a C++ constructor is a copy constructor.
@@ -4873,9 +4851,8 @@ CINDEX_LINKAGE CXCursor clang_getSpecializedCursorTemplate(CXCursor C);
  * \returns The piece of the name pointed to by the given cursor. If there is no
  * name, or if the PieceIndex is out-of-range, a null-cursor will be returned.
  */
-CINDEX_LINKAGE CXSourceRange clang_getCursorReferenceNameRange(CXCursor C,
-                                                unsigned NameFlags,
-                                                unsigned PieceIndex);
+CINDEX_LINKAGE CXSourceRange clang_getCursorReferenceNameRange(
+    CXCursor C, unsigned NameFlags, unsigned PieceIndex);
 
 enum CXNameRefFlags {
   /**
@@ -5043,15 +5020,14 @@ CINDEX_LINKAGE void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range,
  * \param Cursors an array of \p NumTokens cursors, whose contents will be
  * replaced with the cursors corresponding to each token.
  */
-CINDEX_LINKAGE void clang_annotateTokens(CXTranslationUnit TU,
-                                         CXToken *Tokens, unsigned NumTokens,
-                                         CXCursor *Cursors);
+CINDEX_LINKAGE void clang_annotateTokens(CXTranslationUnit TU, CXToken *Tokens,
+                                         unsigned NumTokens, CXCursor *Cursors);
 
 /**
  * Free the given set of tokens.
  */
-CINDEX_LINKAGE void clang_disposeTokens(CXTranslationUnit TU,
-                                        CXToken *Tokens, unsigned NumTokens);
+CINDEX_LINKAGE void clang_disposeTokens(CXTranslationUnit TU, CXToken *Tokens,
+                                        unsigned NumTokens);
 
 /**
  * @}
@@ -5068,15 +5044,11 @@ CINDEX_LINKAGE void clang_disposeTokens(CXTranslationUnit TU,
 
 /* for debug/testing */
 CINDEX_LINKAGE CXString clang_getCursorKindSpelling(enum CXCursorKind Kind);
-CINDEX_LINKAGE void clang_getDefinitionSpellingAndExtent(CXCursor,
-                                          const char **startBuf,
-                                          const char **endBuf,
-                                          unsigned *startLine,
-                                          unsigned *startColumn,
-                                          unsigned *endLine,
-                                          unsigned *endColumn);
+CINDEX_LINKAGE void clang_getDefinitionSpellingAndExtent(
+    CXCursor, const char **startBuf, const char **endBuf, unsigned *startLine,
+    unsigned *startColumn, unsigned *endLine, unsigned *endColumn);
 CINDEX_LINKAGE void clang_enableStackTraces(void);
-CINDEX_LINKAGE void clang_executeOnThread(void (*fn)(void*), void *user_data,
+CINDEX_LINKAGE void clang_executeOnThread(void (*fn)(void *), void *user_data,
                                           unsigned stack_size);
 
 /**
@@ -5327,9 +5299,8 @@ clang_getCompletionChunkKind(CXCompletionString completion_string,
  *
  * \returns the text associated with the chunk at index \c chunk_number.
  */
-CINDEX_LINKAGE CXString
-clang_getCompletionChunkText(CXCompletionString completion_string,
-                             unsigned chunk_number);
+CINDEX_LINKAGE CXString clang_getCompletionChunkText(
+    CXCompletionString completion_string, unsigned chunk_number);
 
 /**
  * Retrieve the completion string associated with a particular chunk
@@ -5342,9 +5313,8 @@ clang_getCompletionChunkText(CXCompletionString completion_string,
  * \returns the completion string associated with the chunk at index
  * \c chunk_number.
  */
-CINDEX_LINKAGE CXCompletionString
-clang_getCompletionChunkCompletionString(CXCompletionString completion_string,
-                                         unsigned chunk_number);
+CINDEX_LINKAGE CXCompletionString clang_getCompletionChunkCompletionString(
+    CXCompletionString completion_string, unsigned chunk_number);
 
 /**
  * Retrieve the number of chunks in the given code-completion string.
@@ -5401,9 +5371,8 @@ clang_getCompletionNumAnnotations(CXCompletionString completion_string);
  * \returns annotation string associated with the completion at index
  * \c annotation_number, or a NULL string if that annotation is not available.
  */
-CINDEX_LINKAGE CXString
-clang_getCompletionAnnotation(CXCompletionString completion_string,
-                              unsigned annotation_number);
+CINDEX_LINKAGE CXString clang_getCompletionAnnotation(
+    CXCompletionString completion_string, unsigned annotation_number);
 
 /**
  * Retrieve the parent context of the given completion string.
@@ -5421,9 +5390,8 @@ clang_getCompletionAnnotation(CXCompletionString completion_string,
  * \returns The name of the completion parent, e.g., "NSObject" if
  * the completion string represents a method in the NSObject class.
  */
-CINDEX_LINKAGE CXString
-clang_getCompletionParent(CXCompletionString completion_string,
-                          enum CXCursorKind *kind);
+CINDEX_LINKAGE CXString clang_getCompletionParent(
+    CXCompletionString completion_string, enum CXCursorKind *kind);
 
 /**
  * Retrieve the brief documentation comment attached to the declaration
@@ -5779,13 +5747,11 @@ CINDEX_LINKAGE unsigned clang_defaultCodeCompleteOptions(void);
  * completion fails, returns NULL.
  */
 CINDEX_LINKAGE
-CXCodeCompleteResults *clang_codeCompleteAt(CXTranslationUnit TU,
-                                            const char *complete_filename,
-                                            unsigned complete_line,
-                                            unsigned complete_column,
-                                            struct CXUnsavedFile *unsaved_files,
-                                            unsigned num_unsaved_files,
-                                            unsigned options);
+CXCodeCompleteResults *
+clang_codeCompleteAt(CXTranslationUnit TU, const char *complete_filename,
+                     unsigned complete_line, unsigned complete_column,
+                     struct CXUnsavedFile *unsaved_files,
+                     unsigned num_unsaved_files, unsigned options);
 
 /**
  * Sort the code-completion results in case-insensitive alphabetical
@@ -5834,8 +5800,8 @@ CXDiagnostic clang_codeCompleteGetDiagnostic(CXCodeCompleteResults *Results,
  * along with the given code completion results.
  */
 CINDEX_LINKAGE
-unsigned long long clang_codeCompleteGetContexts(
-                                                CXCodeCompleteResults *Results);
+unsigned long long
+clang_codeCompleteGetContexts(CXCodeCompleteResults *Results);
 
 /**
  * Returns the cursor kind for the container for the current code
@@ -5854,9 +5820,9 @@ unsigned long long clang_codeCompleteGetContexts(
  * container
  */
 CINDEX_LINKAGE
-enum CXCursorKind clang_codeCompleteGetContainerKind(
-                                                 CXCodeCompleteResults *Results,
-                                                     unsigned *IsIncomplete);
+enum CXCursorKind
+clang_codeCompleteGetContainerKind(CXCodeCompleteResults *Results,
+                                   unsigned *IsIncomplete);
 
 /**
  * Returns the USR for the container for the current code completion
@@ -5908,19 +5874,19 @@ CINDEX_LINKAGE CXString clang_getClangVersion(void);
  */
 CINDEX_LINKAGE void clang_toggleCrashRecovery(unsigned isEnabled);
 
- /**
-  * Visitor invoked for each file in a translation unit
-  *        (used with clang_getInclusions()).
-  *
-  * This visitor function will be invoked by clang_getInclusions() for each
-  * file included (either at the top-level or by \#include directives) within
-  * a translation unit.  The first argument is the file being included, and
-  * the second and third arguments provide the inclusion stack.  The
-  * array is sorted in order of immediate inclusion.  For example,
-  * the first element refers to the location that included 'included_file'.
-  */
+/**
+ * Visitor invoked for each file in a translation unit
+ *        (used with clang_getInclusions()).
+ *
+ * This visitor function will be invoked by clang_getInclusions() for each
+ * file included (either at the top-level or by \#include directives) within
+ * a translation unit.  The first argument is the file being included, and
+ * the second and third arguments provide the inclusion stack.  The
+ * array is sorted in order of immediate inclusion.  For example,
+ * the first element refers to the location that included 'included_file'.
+ */
 typedef void (*CXInclusionVisitor)(CXFile included_file,
-                                   CXSourceLocation* inclusion_stack,
+                                   CXSourceLocation *inclusion_stack,
                                    unsigned include_len,
                                    CXClientData client_data);
 
@@ -5935,7 +5901,7 @@ CINDEX_LINKAGE void clang_getInclusions(CXTranslationUnit tu,
                                         CXClientData client_data);
 
 typedef enum {
-  CXEval_Int = 1 ,
+  CXEval_Int = 1,
   CXEval_Float = 2,
   CXEval_ObjCStrLiteral = 3,
   CXEval_StrLiteral = 4,
@@ -5944,12 +5910,12 @@ typedef enum {
 
   CXEval_UnExposed = 0
 
-} CXEvalResultKind ;
+} CXEvalResultKind;
 
 /**
  * Evaluation result of a cursor
  */
-typedef void * CXEvalResult;
+typedef void *CXEvalResult;
 
 /**
  * If cursor is a statement declaration tries to evaluate the
@@ -5986,7 +5952,8 @@ CINDEX_LINKAGE unsigned clang_EvalResult_isUnsignedInt(CXEvalResult E);
  * Returns the evaluation result as an unsigned integer if
  * the kind is Int and clang_EvalResult_isUnsignedInt is non-zero.
  */
-CINDEX_LINKAGE unsigned long long clang_EvalResult_getAsUnsigned(CXEvalResult E);
+CINDEX_LINKAGE unsigned long long
+clang_EvalResult_getAsUnsigned(CXEvalResult E);
 
 /**
  * Returns the evaluation result as double if the
@@ -6000,7 +5967,7 @@ CINDEX_LINKAGE double clang_EvalResult_getAsDouble(CXEvalResult E);
  * instead call clang_EvalResult_dispose on the CXEvalResult returned
  * by clang_Cursor_Evaluate.
  */
-CINDEX_LINKAGE const char* clang_EvalResult_getAsStr(CXEvalResult E);
+CINDEX_LINKAGE const char *clang_EvalResult_getAsStr(CXEvalResult E);
 
 /**
  * Disposes the created Eval memory.
@@ -6058,7 +6025,8 @@ CINDEX_LINKAGE unsigned clang_remap_getNumFiles(CXRemapping);
  * is associated with.
  */
 CINDEX_LINKAGE void clang_remap_getFilenames(CXRemapping, unsigned index,
-                                     CXString *original, CXString *transformed);
+                                             CXString *original,
+                                             CXString *transformed);
 
 /**
  * Dispose the remapping.
@@ -6074,10 +6042,7 @@ CINDEX_LINKAGE void clang_remap_dispose(CXRemapping);
  * @{
  */
 
-enum CXVisitorResult {
-  CXVisit_Break,
-  CXVisit_Continue
-};
+enum CXVisitorResult { CXVisit_Break, CXVisit_Continue };
 
 typedef struct CXCursorAndRangeVisitor {
   void *context;
@@ -6115,8 +6080,8 @@ typedef enum {
  *
  * \returns one of the CXResult enumerators.
  */
-CINDEX_LINKAGE CXResult clang_findReferencesInFile(CXCursor cursor, CXFile file,
-                                               CXCursorAndRangeVisitor visitor);
+CINDEX_LINKAGE CXResult clang_findReferencesInFile(
+    CXCursor cursor, CXFile file, CXCursorAndRangeVisitor visitor);
 
 /**
  * Find #import/#include directives in a specific file.
@@ -6130,15 +6095,14 @@ CINDEX_LINKAGE CXResult clang_findReferencesInFile(CXCursor cursor, CXFile file,
  *
  * \returns one of the CXResult enumerators.
  */
-CINDEX_LINKAGE CXResult clang_findIncludesInFile(CXTranslationUnit TU,
-                                                 CXFile file,
-                                              CXCursorAndRangeVisitor visitor);
+CINDEX_LINKAGE CXResult clang_findIncludesInFile(
+    CXTranslationUnit TU, CXFile file, CXCursorAndRangeVisitor visitor);
 
 #ifdef __has_feature
-#  if __has_feature(blocks)
+#if __has_feature(blocks)
 
-typedef enum CXVisitorResult
-    (^CXCursorAndRangeVisitorBlock)(CXCursor, CXSourceRange);
+typedef enum CXVisitorResult (^CXCursorAndRangeVisitorBlock)(CXCursor,
+                                                             CXSourceRange);
 
 CINDEX_LINKAGE
 CXResult clang_findReferencesInFileWithBlock(CXCursor, CXFile,
@@ -6148,7 +6112,7 @@ CINDEX_LINKAGE
 CXResult clang_findIncludesInFileWithBlock(CXTranslationUnit, CXFile,
                                            CXCursorAndRangeVisitorBlock);
 
-#  endif
+#endif
 #endif
 
 /**
@@ -6231,46 +6195,46 @@ typedef struct {
 } CXIdxImportedASTFileInfo;
 
 typedef enum {
-  CXIdxEntity_Unexposed     = 0,
-  CXIdxEntity_Typedef       = 1,
-  CXIdxEntity_Function      = 2,
-  CXIdxEntity_Variable      = 3,
-  CXIdxEntity_Field         = 4,
-  CXIdxEntity_EnumConstant  = 5,
+  CXIdxEntity_Unexposed = 0,
+  CXIdxEntity_Typedef = 1,
+  CXIdxEntity_Function = 2,
+  CXIdxEntity_Variable = 3,
+  CXIdxEntity_Field = 4,
+  CXIdxEntity_EnumConstant = 5,
 
-  CXIdxEntity_ObjCClass     = 6,
-  CXIdxEntity_ObjCProtocol  = 7,
-  CXIdxEntity_ObjCCategory  = 8,
+  CXIdxEntity_ObjCClass = 6,
+  CXIdxEntity_ObjCProtocol = 7,
+  CXIdxEntity_ObjCCategory = 8,
 
   CXIdxEntity_ObjCInstanceMethod = 9,
-  CXIdxEntity_ObjCClassMethod    = 10,
-  CXIdxEntity_ObjCProperty  = 11,
-  CXIdxEntity_ObjCIvar      = 12,
-
-  CXIdxEntity_Enum          = 13,
-  CXIdxEntity_Struct        = 14,
-  CXIdxEntity_Union         = 15,
-
-  CXIdxEntity_CXXClass              = 16,
-  CXIdxEntity_CXXNamespace          = 17,
-  CXIdxEntity_CXXNamespaceAlias     = 18,
-  CXIdxEntity_CXXStaticVariable     = 19,
-  CXIdxEntity_CXXStaticMethod       = 20,
-  CXIdxEntity_CXXInstanceMethod     = 21,
-  CXIdxEntity_CXXConstructor        = 22,
-  CXIdxEntity_CXXDestructor         = 23,
+  CXIdxEntity_ObjCClassMethod = 10,
+  CXIdxEntity_ObjCProperty = 11,
+  CXIdxEntity_ObjCIvar = 12,
+
+  CXIdxEntity_Enum = 13,
+  CXIdxEntity_Struct = 14,
+  CXIdxEntity_Union = 15,
+
+  CXIdxEntity_CXXClass = 16,
+  CXIdxEntity_CXXNamespace = 17,
+  CXIdxEntity_CXXNamespaceAlias = 18,
+  CXIdxEntity_CXXStaticVariable = 19,
+  CXIdxEntity_CXXStaticMethod = 20,
+  CXIdxEntity_CXXInstanceMethod = 21,
+  CXIdxEntity_CXXConstructor = 22,
+  CXIdxEntity_CXXDestructor = 23,
   CXIdxEntity_CXXConversionFunction = 24,
-  CXIdxEntity_CXXTypeAlias          = 25,
-  CXIdxEntity_CXXInterface          = 26
+  CXIdxEntity_CXXTypeAlias = 25,
+  CXIdxEntity_CXXInterface = 26
 
 } CXIdxEntityKind;
 
 typedef enum {
   CXIdxEntityLang_None = 0,
-  CXIdxEntityLang_C    = 1,
+  CXIdxEntityLang_C = 1,
   CXIdxEntityLang_ObjC = 2,
-  CXIdxEntityLang_CXX  = 3,
-  CXIdxEntityLang_Swift  = 4
+  CXIdxEntityLang_CXX = 3,
+  CXIdxEntityLang_Swift = 4
 } CXIdxEntityLanguage;
 
 /**
@@ -6284,16 +6248,16 @@ typedef enum {
  * CXIdxEntity_CXXTypeAlias
  */
 typedef enum {
-  CXIdxEntity_NonTemplate   = 0,
-  CXIdxEntity_Template      = 1,
+  CXIdxEntity_NonTemplate = 0,
+  CXIdxEntity_Template = 1,
   CXIdxEntity_TemplatePartialSpecialization = 2,
   CXIdxEntity_TemplateSpecialization = 3
 } CXIdxEntityCXXTemplateKind;
 
 typedef enum {
-  CXIdxAttr_Unexposed     = 0,
-  CXIdxAttr_IBAction      = 1,
-  CXIdxAttr_IBOutlet      = 2,
+  CXIdxAttr_Unexposed = 0,
+  CXIdxAttr_IBAction = 1,
+  CXIdxAttr_IBOutlet = 2,
   CXIdxAttr_IBOutletCollection = 3
 } CXIdxAttrKind;
 
@@ -6325,9 +6289,7 @@ typedef struct {
   CXIdxLoc classLoc;
 } CXIdxIBOutletCollectionAttrInfo;
 
-typedef enum {
-  CXIdxDeclFlag_Skipped = 0x1
-} CXIdxDeclInfoFlags;
+typedef enum { CXIdxDeclFlag_Skipped = 0x1 } CXIdxDeclInfoFlags;
 
 typedef struct {
   const CXIdxEntityInfo *entityInfo;
@@ -6496,11 +6458,10 @@ typedef struct {
   /**
    * Called at the end of indexing; passes the complete diagnostic set.
    */
-  void (*diagnostic)(CXClientData client_data,
-                     CXDiagnosticSet, void *reserved);
+  void (*diagnostic)(CXClientData client_data, CXDiagnosticSet, void *reserved);
 
-  CXIdxClientFile (*enteredMainFile)(CXClientData client_data,
-                                     CXFile mainFile, void *reserved);
+  CXIdxClientFile (*enteredMainFile)(CXClientData client_data, CXFile mainFile,
+                                     void *reserved);
 
   /**
    * Called when a file gets \#included/\#imported.
@@ -6525,8 +6486,7 @@ typedef struct {
   CXIdxClientContainer (*startedTranslationUnit)(CXClientData client_data,
                                                  void *reserved);
 
-  void (*indexDeclaration)(CXClientData client_data,
-                           const CXIdxDeclInfo *);
+  void (*indexDeclaration)(CXClientData client_data, const CXIdxDeclInfo *);
 
   /**
    * Called to index a reference of an entity.
@@ -6570,8 +6530,8 @@ clang_index_getClientContainer(const CXIdxContainerInfo *);
  * For setting a custom CXIdxClientContainer attached to a
  * container.
  */
-CINDEX_LINKAGE void
-clang_index_setClientContainer(const CXIdxContainerInfo *,CXIdxClientContainer);
+CINDEX_LINKAGE void clang_index_setClientContainer(const CXIdxContainerInfo *,
+                                                   CXIdxClientContainer);
 
 /**
  * For retrieving a custom CXIdxClientEntity attached to an entity.
@@ -6582,8 +6542,8 @@ clang_index_getClientEntity(const CXIdxEntityInfo *);
 /**
  * For setting a custom CXIdxClientEntity attached to an entity.
  */
-CINDEX_LINKAGE void
-clang_index_setClientEntity(const CXIdxEntityInfo *, CXIdxClientEntity);
+CINDEX_LINKAGE void clang_index_setClientEntity(const CXIdxEntityInfo *,
+                                                CXIdxClientEntity);
 
 /**
  * An indexing action/session, to be applied to one or multiple
@@ -6671,18 +6631,12 @@ typedef enum {
  *
  * The rest of the parameters are the same as #clang_parseTranslationUnit.
  */
-CINDEX_LINKAGE int clang_indexSourceFile(CXIndexAction,
-                                         CXClientData client_data,
-                                         IndexerCallbacks *index_callbacks,
-                                         unsigned index_callbacks_size,
-                                         unsigned index_options,
-                                         const char *source_filename,
-                                         const char * const *command_line_args,
-                                         int num_command_line_args,
-                                         struct CXUnsavedFile *unsaved_files,
-                                         unsigned num_unsaved_files,
-                                         CXTranslationUnit *out_TU,
-                                         unsigned TU_options);
+CINDEX_LINKAGE int clang_indexSourceFile(
+    CXIndexAction, CXClientData client_data, IndexerCallbacks *index_callbacks,
+    unsigned index_callbacks_size, unsigned index_options,
+    const char *source_filename, const char *const *command_line_args,
+    int num_command_line_args, struct CXUnsavedFile *unsaved_files,
+    unsigned num_unsaved_files, CXTranslationUnit *out_TU, unsigned TU_options);
 
 /**
  * Same as clang_indexSourceFile but requires a full command line
@@ -6712,12 +6666,9 @@ CINDEX_LINKAGE int clang_indexSourceFileFullArgv(
  * \returns If there is a failure from which there is no recovery, returns
  * non-zero, otherwise returns 0.
  */
-CINDEX_LINKAGE int clang_indexTranslationUnit(CXIndexAction,
-                                              CXClientData client_data,
-                                              IndexerCallbacks *index_callbacks,
-                                              unsigned index_callbacks_size,
-                                              unsigned index_options,
-                                              CXTranslationUnit);
+CINDEX_LINKAGE int clang_indexTranslationUnit(
+    CXIndexAction, CXClientData client_data, IndexerCallbacks *index_callbacks,
+    unsigned index_callbacks_size, unsigned index_options, CXTranslationUnit);
 
 /**
  * Retrieve the CXIdxFile, file, line, column, and offset represented by
@@ -6729,8 +6680,7 @@ CINDEX_LINKAGE int clang_indexTranslationUnit(CXIndexAction,
  */
 CINDEX_LINKAGE void clang_indexLoc_getFileLocation(CXIdxLoc loc,
                                                    CXIdxClientFile *indexFile,
-                                                   CXFile *file,
-                                                   unsigned *line,
+                                                   CXFile *file, unsigned *line,
                                                    unsigned *column,
                                                    unsigned *offset);
 
@@ -6773,8 +6723,7 @@ typedef enum CXVisitorResult (*CXFieldVisitor)(CXCursor C,
  * \returns a non-zero value if the traversal was terminated
  * prematurely by the visitor returning \c CXFieldVisit_Break.
  */
-CINDEX_LINKAGE unsigned clang_Type_visitFields(CXType T,
-                                               CXFieldVisitor visitor,
+CINDEX_LINKAGE unsigned clang_Type_visitFields(CXType T, CXFieldVisitor visitor,
                                                CXClientData client_data);
 
 /**
diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h
index ca0f991c24e31..6813ab58874ee 100644
--- a/clang/include/clang/AST/ASTContext.h
+++ b/clang/include/clang/AST/ASTContext.h
@@ -970,7 +970,7 @@ class ASTContext : public RefCountedBase<ASTContext> {
 #include "clang/Basic/OpenCLImageTypes.def"
   CanQualType OCLSamplerTy, OCLEventTy, OCLClkEventTy;
   CanQualType OCLQueueTy, OCLReserveIDTy;
-  CanQualType OMPArraySectionTy;
+  CanQualType OMPArraySectionTy, OMPArrayShapingTy, OMPIteratorTy;
 #define EXT_OPAQUE_TYPE(ExtType, Id, Ext) \
   CanQualType Id##Ty;
 #include "clang/Basic/OpenCLExtensionTypes.def"
diff --git a/clang/include/clang/AST/Attr.h b/clang/include/clang/AST/Attr.h
index 5f6035cb0c282..735056e2a31e0 100644
--- a/clang/include/clang/AST/Attr.h
+++ b/clang/include/clang/AST/Attr.h
@@ -16,10 +16,11 @@
 #include "clang/AST/ASTFwd.h"
 #include "clang/AST/AttrIterator.h"
 #include "clang/AST/Decl.h"
-#include "clang/AST/OpenMPClause.h"
+#include "clang/AST/Expr.h"
 #include "clang/AST/Type.h"
 #include "clang/Basic/AttrKinds.h"
 #include "clang/Basic/AttributeCommonInfo.h"
+#include "clang/Basic/LangOptions.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/OpenMPKinds.h"
 #include "clang/Basic/Sanitizers.h"
@@ -40,6 +41,7 @@ class Expr;
 class QualType;
 class FunctionDecl;
 class TypeSourceInfo;
+class OMPTraitInfo;
 
 /// Attr - This represents one attribute.
 class Attr : public AttributeCommonInfo {
diff --git a/clang/include/clang/AST/BuiltinTypes.def b/clang/include/clang/AST/BuiltinTypes.def
index 74a45ee4ccc0f..f8eb4ec19c8f1 100644
--- a/clang/include/clang/AST/BuiltinTypes.def
+++ b/clang/include/clang/AST/BuiltinTypes.def
@@ -313,8 +313,14 @@ PLACEHOLDER_TYPE(ARCUnbridgedCast, ARCUnbridgedCastTy)
 // A placeholder type for OpenMP array sections.
 PLACEHOLDER_TYPE(OMPArraySection, OMPArraySectionTy)
 
+// A placeholder type for OpenMP array shaping operation.
+PLACEHOLDER_TYPE(OMPArrayShaping, OMPArrayShapingTy)
+
+// A placeholder type for OpenMP iterators.
+PLACEHOLDER_TYPE(OMPIterator, OMPIteratorTy)
+
 #ifdef LAST_BUILTIN_TYPE
-LAST_BUILTIN_TYPE(OMPArraySection)
+LAST_BUILTIN_TYPE(OMPIterator)
 #undef LAST_BUILTIN_TYPE
 #endif
 
diff --git a/clang/include/clang/AST/ComputeDependence.h b/clang/include/clang/AST/ComputeDependence.h
index 02f826438d4df..ab742c9b70ddf 100644
--- a/clang/include/clang/AST/ComputeDependence.h
+++ b/clang/include/clang/AST/ComputeDependence.h
@@ -87,6 +87,8 @@ class ParenListExpr;
 class PseudoObjectExpr;
 class AtomicExpr;
 class OMPArraySectionExpr;
+class OMPArrayShapingExpr;
+class OMPIteratorExpr;
 class ObjCArrayLiteral;
 class ObjCDictionaryLiteral;
 class ObjCBoxedExpr;
@@ -172,6 +174,8 @@ ExprDependence computeDependence(PseudoObjectExpr *E);
 ExprDependence computeDependence(AtomicExpr *E);
 
 ExprDependence computeDependence(OMPArraySectionExpr *E);
+ExprDependence computeDependence(OMPArrayShapingExpr *E);
+ExprDependence computeDependence(OMPIteratorExpr *E);
 
 ExprDependence computeDependence(ObjCArrayLiteral *E);
 ExprDependence computeDependence(ObjCDictionaryLiteral *E);
diff --git a/clang/include/clang/AST/Decl.h b/clang/include/clang/AST/Decl.h
index 0ce8b852b2ff3..2af1189511a30 100644
--- a/clang/include/clang/AST/Decl.h
+++ b/clang/include/clang/AST/Decl.h
@@ -2030,7 +2030,7 @@ class FunctionDecl : public DeclaratorDecl,
   /// declaration to the declaration that is a definition (if there is one).
   bool isDefined(const FunctionDecl *&Definition) const;
 
-  virtual bool isDefined() const {
+  bool isDefined() const {
     const FunctionDecl* Definition;
     return isDefined(Definition);
   }
@@ -4542,6 +4542,13 @@ inline bool IsEnumDeclScoped(EnumDecl *ED) {
   return ED->isScoped();
 }
 
+/// OpenMP variants are mangled early based on their OpenMP context selector.
+/// The new name looks likes this:
+///  <name> + OpenMPVariantManglingSeparatorStr + <mangled OpenMP context>
+static constexpr StringRef getOpenMPVariantManglingSeparatorStr() {
+  return ".ompvariant";
+}
+
 } // namespace clang
 
 #endif // LLVM_CLANG_AST_DECL_H
diff --git a/clang/include/clang/AST/DependenceFlags.h b/clang/include/clang/AST/DependenceFlags.h
index 788227156c4d9..0b24bae6df9b1 100644
--- a/clang/include/clang/AST/DependenceFlags.h
+++ b/clang/include/clang/AST/DependenceFlags.h
@@ -36,7 +36,6 @@ struct ExprDependenceScope {
   };
 };
 using ExprDependence = ExprDependenceScope::ExprDependence;
-static constexpr unsigned ExprDependenceBits = 5;
 
 struct TypeDependenceScope {
   enum TypeDependence : uint8_t {
@@ -51,18 +50,19 @@ struct TypeDependenceScope {
     /// Whether this type is a variably-modified type (C99 6.7.5).
     VariablyModified = 8,
 
-    // FIXME: add Error bit.
+    /// Whether this type references an error, e.g. decltype(err-expression)
+    /// yields an error type.
+    Error = 16,
 
     None = 0,
-    All = 15,
+    All = 31,
 
     DependentInstantiation = Dependent | Instantiation,
 
-    LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/VariablyModified)
+    LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/Error)
   };
 };
 using TypeDependence = TypeDependenceScope::TypeDependence;
-static constexpr unsigned TypeDependenceBits = 4;
 
 #define LLVM_COMMON_DEPENDENCE(NAME)                                           \
   struct NAME##Scope {                                                         \
@@ -78,37 +78,134 @@ static constexpr unsigned TypeDependenceBits = 4;
       LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/Dependent)                    \
     };                                                                         \
   };                                                                           \
-  using NAME = NAME##Scope::NAME;                                              \
-  static constexpr unsigned NAME##Bits = 3;
+  using NAME = NAME##Scope::NAME;
 
 LLVM_COMMON_DEPENDENCE(NestedNameSpecifierDependence)
 LLVM_COMMON_DEPENDENCE(TemplateNameDependence)
 LLVM_COMMON_DEPENDENCE(TemplateArgumentDependence)
 #undef LLVM_COMMON_DEPENDENCE
 
+// A combined space of all dependence concepts for all node types.
+// Used when aggregating dependence of nodes of different types.
+class Dependence {
+public:
+  enum Bits : uint8_t {
+    None = 0,
+
+    // Contains a template parameter pack that wasn't expanded.
+    UnexpandedPack = 1,
+    // Uses a template parameter, even if it doesn't affect the result.
+    // Validity depends on the template parameter.
+    Instantiation = 2,
+    // Expression type depends on template context.
+    // Value and Instantiation should also be set.
+    Type = 4,
+    // Expression value depends on template context.
+    // Instantiation should also be set.
+    Value = 8,
+    // Depends on template context.
+    // The type/value distinction is only meaningful for expressions.
+    Dependent = Type | Value,
+    // Includes an error, and depends on how it is resolved.
+    Error = 16,
+    // Type depends on a runtime value (variable-length array).
+    VariablyModified = 32,
+
+    LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/VariablyModified)
+  };
+
+  Dependence() : V(None) {}
+
+  Dependence(TypeDependence D)
+      : V(translate(D, TypeDependence::UnexpandedPack, UnexpandedPack) |
+             translate(D, TypeDependence::Instantiation, Instantiation) |
+             translate(D, TypeDependence::Dependent, Dependent) |
+             translate(D, TypeDependence::VariablyModified, VariablyModified)) {
+  }
+
+  Dependence(ExprDependence D)
+      : V(translate(D, ExprDependence::UnexpandedPack, UnexpandedPack) |
+             translate(D, ExprDependence::Instantiation, Instantiation) |
+             translate(D, ExprDependence::Type, Type) |
+             translate(D, ExprDependence::Value, Value) |
+             translate(D, ExprDependence::Error, Error)) {}
+
+  Dependence(NestedNameSpecifierDependence D) :
+    V ( translate(D, NNSDependence::UnexpandedPack, UnexpandedPack) |
+            translate(D, NNSDependence::Instantiation, Instantiation) |
+            translate(D, NNSDependence::Dependent, Dependent)){}
+
+  Dependence(TemplateArgumentDependence D)
+      : V(translate(D, TADependence::UnexpandedPack, UnexpandedPack) |
+             translate(D, TADependence::Instantiation, Instantiation) |
+             translate(D, TADependence::Dependent, Dependent)) {}
+
+  Dependence(TemplateNameDependence D)
+      : V(translate(D, TNDependence::UnexpandedPack, UnexpandedPack) |
+             translate(D, TNDependence::Instantiation, Instantiation) |
+             translate(D, TNDependence::Dependent, Dependent)) {}
+
+  TypeDependence type() const {
+    return translate(V, UnexpandedPack, TypeDependence::UnexpandedPack) |
+           translate(V, Instantiation, TypeDependence::Instantiation) |
+           translate(V, Dependent, TypeDependence::Dependent) |
+           translate(V, Error, TypeDependence::Error) |
+           translate(V, VariablyModified, TypeDependence::VariablyModified);
+  }
+
+  ExprDependence expr() const {
+    return translate(V, UnexpandedPack, ExprDependence::UnexpandedPack) |
+           translate(V, Instantiation, ExprDependence::Instantiation) |
+           translate(V, Type, ExprDependence::Type) |
+           translate(V, Value, ExprDependence::Value) |
+           translate(V, Error, ExprDependence::Error);
+  }
+
+  NestedNameSpecifierDependence nestedNameSpecifier() const {
+    return translate(V, UnexpandedPack, NNSDependence::UnexpandedPack) |
+           translate(V, Instantiation, NNSDependence::Instantiation) |
+           translate(V, Dependent, NNSDependence::Dependent);
+  }
+
+  TemplateArgumentDependence templateArgument() const {
+    return translate(V, UnexpandedPack, TADependence::UnexpandedPack) |
+           translate(V, Instantiation, TADependence::Instantiation) |
+           translate(V, Dependent, TADependence::Dependent);
+  }
+
+  TemplateNameDependence templateName() const {
+    return translate(V, UnexpandedPack, TNDependence::UnexpandedPack) |
+           translate(V, Instantiation, TNDependence::Instantiation) |
+           translate(V, Dependent, TNDependence::Dependent);
+  }
+
+private:
+  Bits V;
+
+  template <typename T, typename U>
+  static U translate(T Bits, T FromBit, U ToBit) {
+    return (Bits & FromBit) ? ToBit : static_cast<U>(0);
+  }
+
+  // Abbreviations to make conversions more readable.
+  using NNSDependence = NestedNameSpecifierDependence;
+  using TADependence = TemplateArgumentDependence;
+  using TNDependence = TemplateNameDependence;
+};
+
 /// Computes dependencies of a reference with the name having template arguments
 /// with \p TA dependencies.
 inline ExprDependence toExprDependence(TemplateArgumentDependence TA) {
-  auto D = ExprDependence::None;
-  if (TA & TemplateArgumentDependence::UnexpandedPack)
-    D |= ExprDependence::UnexpandedPack;
-  if (TA & TemplateArgumentDependence::Instantiation)
-    D |= ExprDependence::Instantiation;
-  if (TA & TemplateArgumentDependence::Dependent)
-    D |= ExprDependence::Type | ExprDependence::Value;
-  return D;
+  return Dependence(TA).expr();
 }
-inline ExprDependence toExprDependence(TypeDependence TD) {
-  // This hack works because TypeDependence and TemplateArgumentDependence
-  // share the same bit representation, apart from variably-modified.
-  return toExprDependence(static_cast<TemplateArgumentDependence>(
-      TD & ~TypeDependence::VariablyModified));
+inline ExprDependence toExprDependence(TypeDependence D) {
+  return Dependence(D).expr();
 }
-inline ExprDependence toExprDependence(NestedNameSpecifierDependence NSD) {
-  // This hack works because TypeDependence and TemplateArgumentDependence
-  // share the same bit representation.
-  return toExprDependence(static_cast<TemplateArgumentDependence>(NSD)) &
-         ~ExprDependence::TypeValue;
+// Note: it's often necessary to strip `Dependent` from qualifiers.
+// If V<T>:: refers to the current instantiation, NNS is considered dependent
+// but the containing V<T>::foo likely isn't.
+inline ExprDependence toExprDependence(NestedNameSpecifierDependence D) {
+  return Dependence(D).expr();
 }
 inline ExprDependence turnTypeToValueDependence(ExprDependence D) {
   // Type-dependent expressions are always be value-dependent, so we simply drop
@@ -124,57 +221,39 @@ inline ExprDependence turnValueToTypeDependence(ExprDependence D) {
 
 // Returned type-dependence will never have VariablyModified set.
 inline TypeDependence toTypeDependence(ExprDependence D) {
-  // Supported bits all have the same representation.
-  return static_cast<TypeDependence>(D & (ExprDependence::UnexpandedPack |
-                                          ExprDependence::Instantiation |
-                                          ExprDependence::Type));
+  return Dependence(D).type();
 }
 inline TypeDependence toTypeDependence(NestedNameSpecifierDependence D) {
-  // Supported bits all have the same representation.
-  return static_cast<TypeDependence>(D);
+  return Dependence(D).type();
 }
 inline TypeDependence toTypeDependence(TemplateNameDependence D) {
-  // Supported bits all have the same representation.
-  return static_cast<TypeDependence>(D);
+  return Dependence(D).type();
 }
 inline TypeDependence toTypeDependence(TemplateArgumentDependence D) {
-  // Supported bits all have the same representation.
-  return static_cast<TypeDependence>(D);
+  return Dependence(D).type();
 }
 
 inline NestedNameSpecifierDependence
 toNestedNameSpecifierDependendence(TypeDependence D) {
-  // This works because both classes share the same bit representation.
-  return static_cast<NestedNameSpecifierDependence>(
-      D & ~TypeDependence::VariablyModified);
+  return Dependence(D).nestedNameSpecifier();
 }
 
 inline TemplateArgumentDependence
 toTemplateArgumentDependence(TypeDependence D) {
-  // This works because both classes share the same bit representation.
-  return static_cast<TemplateArgumentDependence>(
-      D & ~TypeDependence::VariablyModified);
+  return Dependence(D).templateArgument();
 }
 inline TemplateArgumentDependence
 toTemplateArgumentDependence(TemplateNameDependence D) {
-  // This works because both classes share the same bit representation.
-  return static_cast<TemplateArgumentDependence>(D);
+  return Dependence(D).templateArgument();
 }
 inline TemplateArgumentDependence
-toTemplateArgumentDependence(ExprDependence ED) {
-  TemplateArgumentDependence TAD = TemplateArgumentDependence::None;
-  if (ED & (ExprDependence::Type | ExprDependence::Value))
-    TAD |= TemplateArgumentDependence::Dependent;
-  if (ED & ExprDependence::Instantiation)
-    TAD |= TemplateArgumentDependence::Instantiation;
-  if (ED & ExprDependence::UnexpandedPack)
-    TAD |= TemplateArgumentDependence::UnexpandedPack;
-  return TAD;
+toTemplateArgumentDependence(ExprDependence D) {
+  return Dependence(D).templateArgument();
 }
 
 inline TemplateNameDependence
 toTemplateNameDependence(NestedNameSpecifierDependence D) {
-  return static_cast<TemplateNameDependence>(D);
+  return Dependence(D).templateName();
 }
 
 LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
diff --git a/clang/include/clang/AST/Expr.h b/clang/include/clang/AST/Expr.h
index fa49182728194..e566e3c6ab899 100644
--- a/clang/include/clang/AST/Expr.h
+++ b/clang/include/clang/AST/Expr.h
@@ -2026,7 +2026,8 @@ class PredefinedExpr final
 
   static StringRef getIdentKindName(IdentKind IK);
   static std::string ComputeName(IdentKind IK, const Decl *CurrentDecl);
-  static std::string ComputeName(ASTContext &Ctx, IdentKind IK, const QualType Ty);
+  static std::string ComputeName(ASTContext &Context, IdentKind IK,
+                                 const QualType Ty);
 
   SourceLocation getBeginLoc() const { return getLocation(); }
   SourceLocation getEndLoc() const { return getLocation(); }
diff --git a/clang/include/clang/AST/ExprOpenMP.h b/clang/include/clang/AST/ExprOpenMP.h
index f971ed8457bcd..dc6e061604b31 100644
--- a/clang/include/clang/AST/ExprOpenMP.h
+++ b/clang/include/clang/AST/ExprOpenMP.h
@@ -116,6 +116,258 @@ class OMPArraySectionExpr : public Expr {
     return const_child_range(&SubExprs[BASE], &SubExprs[END_EXPR]);
   }
 };
+
+/// An explicit cast in C or a C-style cast in C++, which uses the syntax
+/// ([s1][s2]...[sn])expr. For example: @c ([3][3])f.
+class OMPArrayShapingExpr final
+    : public Expr,
+      private llvm::TrailingObjects<OMPArrayShapingExpr, Expr *, SourceRange> {
+  friend TrailingObjects;
+  friend class ASTStmtReader;
+  friend class ASTStmtWriter;
+  /// Base node.
+  SourceLocation LPLoc; /// The location of the left paren
+  SourceLocation RPLoc; /// The location of the right paren
+  unsigned NumDims = 0; /// Number of dimensions in the shaping expression.
+
+  /// Construct full expression.
+  OMPArrayShapingExpr(QualType ExprTy, Expr *Op, SourceLocation L,
+                      SourceLocation R, ArrayRef<Expr *> Dims);
+
+  /// Construct an empty expression.
+  explicit OMPArrayShapingExpr(EmptyShell Shell, unsigned NumDims)
+      : Expr(OMPArrayShapingExprClass, Shell), NumDims(NumDims) {}
+
+  /// Sets the dimensions for the array shaping.
+  void setDimensions(ArrayRef<Expr *> Dims);
+
+  /// Sets the base expression for array shaping operation.
+  void setBase(Expr *Op) { getTrailingObjects<Expr *>()[NumDims] = Op; }
+
+  /// Sets source ranges for the brackets in the array shaping operation.
+  void setBracketsRanges(ArrayRef<SourceRange> BR);
+
+  unsigned numTrailingObjects(OverloadToken<Expr *>) const {
+    // Add an extra one for the base expression.
+    return NumDims + 1;
+  }
+
+  unsigned numTrailingObjects(OverloadToken<SourceRange>) const {
+    return NumDims;
+  }
+
+public:
+  static OMPArrayShapingExpr *Create(const ASTContext &Context, QualType T,
+                                     Expr *Op, SourceLocation L,
+                                     SourceLocation R, ArrayRef<Expr *> Dims,
+                                     ArrayRef<SourceRange> BracketRanges);
+
+  static OMPArrayShapingExpr *CreateEmpty(const ASTContext &Context,
+                                          unsigned NumDims);
+
+  SourceLocation getLParenLoc() const { return LPLoc; }
+  void setLParenLoc(SourceLocation L) { LPLoc = L; }
+
+  SourceLocation getRParenLoc() const { return RPLoc; }
+  void setRParenLoc(SourceLocation L) { RPLoc = L; }
+
+  SourceLocation getBeginLoc() const LLVM_READONLY { return LPLoc; }
+  SourceLocation getEndLoc() const LLVM_READONLY {
+    return getBase()->getEndLoc();
+  }
+
+  /// Fetches the dimensions for array shaping expression.
+  ArrayRef<Expr *> getDimensions() const {
+    return llvm::makeArrayRef(getTrailingObjects<Expr *>(), NumDims);
+  }
+
+  /// Fetches source ranges for the brackets os the array shaping expression.
+  ArrayRef<SourceRange> getBracketsRanges() const {
+    return llvm::makeArrayRef(getTrailingObjects<SourceRange>(), NumDims);
+  }
+
+  /// Fetches base expression of array shaping expression.
+  Expr *getBase() { return getTrailingObjects<Expr *>()[NumDims]; }
+  const Expr *getBase() const { return getTrailingObjects<Expr *>()[NumDims]; }
+
+  static bool classof(const Stmt *T) {
+    return T->getStmtClass() == OMPArrayShapingExprClass;
+  }
+
+  // Iterators
+  child_range children() {
+    Stmt **Begin = reinterpret_cast<Stmt **>(getTrailingObjects<Expr *>());
+    return child_range(Begin, Begin + NumDims + 1);
+  }
+  const_child_range children() const {
+    Stmt *const *Begin =
+        reinterpret_cast<Stmt *const *>(getTrailingObjects<Expr *>());
+    return const_child_range(Begin, Begin + NumDims + 1);
+  }
+};
+
+/// OpenMP 5.0 [2.1.6 Iterators]
+/// Iterators are identifiers that expand to multiple values in the clause on
+/// which they appear.
+/// The syntax of the iterator modifier is as follows:
+/// \code
+/// iterator(iterators-definition)
+/// \endcode
+/// where iterators-definition is one of the following:
+/// \code
+/// iterator-specifier [, iterators-definition ]
+/// \endcode
+/// where iterator-specifier is one of the following:
+/// \code
+/// [ iterator-type ] identifier = range-specification
+/// \endcode
+/// where identifier is a base language identifier.
+/// iterator-type is a type name.
+/// range-specification is of the form begin:end[:step], where begin and end are
+/// expressions for which their types can be converted to iterator-type and step
+/// is an integral expression.
+/// In an iterator-specifier, if the iterator-type is not specified then the
+/// type of that iterator is of int type.
+/// The iterator-type must be an integral or pointer type.
+/// The iterator-type must not be const qualified.
+class OMPIteratorExpr final
+    : public Expr,
+      private llvm::TrailingObjects<OMPIteratorExpr, Decl *, Expr *,
+                                    SourceLocation> {
+public:
+  /// Iterator range representation begin:end[:step].
+  struct IteratorRange {
+    Expr *Begin = nullptr;
+    Expr *End = nullptr;
+    Expr *Step = nullptr;
+  };
+  /// Iterator definition representation.
+  struct IteratorDefinition {
+    Decl *IteratorDecl = nullptr;
+    IteratorRange Range;
+    SourceLocation AssignmentLoc;
+    SourceLocation ColonLoc, SecondColonLoc;
+  };
+
+private:
+  friend TrailingObjects;
+  friend class ASTStmtReader;
+  friend class ASTStmtWriter;
+
+  /// Offset in the list of expressions for subelements of the ranges.
+  enum class RangeExprOffset {
+    Begin = 0,
+    End = 1,
+    Step = 2,
+    Total = 3,
+  };
+  /// Offset in the list of locations for subelements of colon symbols
+  /// locations.
+  enum class RangeLocOffset {
+    AssignLoc = 0,
+    FirstColonLoc = 1,
+    SecondColonLoc = 2,
+    Total = 3,
+  };
+  /// Location of 'iterator' keyword.
+  SourceLocation IteratorKwLoc;
+  /// Location of '('.
+  SourceLocation LPLoc;
+  /// Location of ')'.
+  SourceLocation RPLoc;
+  /// Number of iterator definitions.
+  unsigned NumIterators = 0;
+
+  OMPIteratorExpr(QualType ExprTy, SourceLocation IteratorKwLoc,
+                  SourceLocation L, SourceLocation R,
+                  ArrayRef<OMPIteratorExpr::IteratorDefinition> Data);
+
+  /// Construct an empty expression.
+  explicit OMPIteratorExpr(EmptyShell Shell, unsigned NumIterators)
+      : Expr(OMPIteratorExprClass, Shell), NumIterators(NumIterators) {}
+
+  /// Sets basic declaration for the specified iterator definition.
+  void setIteratorDeclaration(unsigned I, Decl *D);
+
+  /// Sets the location of the assignment symbol for the specified iterator
+  /// definition.
+  void setAssignmentLoc(unsigned I, SourceLocation Loc);
+
+  /// Sets begin, end and optional step expressions for specified iterator
+  /// definition.
+  void setIteratorRange(unsigned I, Expr *Begin, SourceLocation ColonLoc,
+                        Expr *End, SourceLocation SecondColonLoc, Expr *Step);
+
+  unsigned numTrailingObjects(OverloadToken<Decl *>) const {
+    return NumIterators;
+  }
+
+  unsigned numTrailingObjects(OverloadToken<Expr *>) const {
+    return NumIterators * static_cast<int>(RangeExprOffset::Total);
+  }
+
+public:
+  static OMPIteratorExpr *Create(const ASTContext &Context, QualType T,
+                                 SourceLocation IteratorKwLoc, SourceLocation L,
+                                 SourceLocation R,
+                                 ArrayRef<IteratorDefinition> Data);
+
+  static OMPIteratorExpr *CreateEmpty(const ASTContext &Context,
+                                      unsigned NumIterators);
+
+  SourceLocation getLParenLoc() const { return LPLoc; }
+  void setLParenLoc(SourceLocation L) { LPLoc = L; }
+
+  SourceLocation getRParenLoc() const { return RPLoc; }
+  void setRParenLoc(SourceLocation L) { RPLoc = L; }
+
+  SourceLocation getIteratorKwLoc() const { return IteratorKwLoc; }
+  void setIteratorKwLoc(SourceLocation L) { IteratorKwLoc = L; }
+  SourceLocation getBeginLoc() const LLVM_READONLY { return IteratorKwLoc; }
+  SourceLocation getEndLoc() const LLVM_READONLY { return RPLoc; }
+
+  /// Gets the iterator declaration for the given iterator.
+  Decl *getIteratorDecl(unsigned I);
+  const Decl *getIteratorDecl(unsigned I) const {
+    return const_cast<OMPIteratorExpr *>(this)->getIteratorDecl(I);
+  }
+
+  /// Gets the iterator range for the given iterator.
+  IteratorRange getIteratorRange(unsigned I);
+  const IteratorRange getIteratorRange(unsigned I) const {
+    return const_cast<OMPIteratorExpr *>(this)->getIteratorRange(I);
+  }
+
+  /// Gets the location of '=' for the given iterator definition.
+  SourceLocation getAssignLoc(unsigned I) const;
+  /// Gets the location of the first ':' in the range for the given iterator
+  /// definition.
+  SourceLocation getColonLoc(unsigned I) const;
+  /// Gets the location of the second ':' (if any) in the range for the given
+  /// iteratori definition.
+  SourceLocation getSecondColonLoc(unsigned I) const;
+
+  /// Returns number of iterator definitions.
+  unsigned numOfIterators() const { return NumIterators; }
+
+  static bool classof(const Stmt *T) {
+    return T->getStmtClass() == OMPIteratorExprClass;
+  }
+
+  // Iterators
+  child_range children() {
+    Stmt **Begin = reinterpret_cast<Stmt **>(getTrailingObjects<Expr *>());
+    return child_range(
+        Begin, Begin + NumIterators * static_cast<int>(RangeExprOffset::Total));
+  }
+  const_child_range children() const {
+    Stmt *const *Begin =
+        reinterpret_cast<Stmt *const *>(getTrailingObjects<Expr *>());
+    return const_child_range(
+        Begin, Begin + NumIterators * static_cast<int>(RangeExprOffset::Total));
+  }
+};
+
 } // end namespace clang
 
 #endif
diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 29c251ef7ee69..efa6d0554a7ca 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -4372,6 +4372,9 @@ class OMPDependClause final
   /// Set colon location.
   void setColonLoc(SourceLocation Loc) { ColonLoc = Loc; }
 
+  /// Sets optional dependency modifier.
+  void setModifier(Expr *DepModifier);
+
 public:
   /// Creates clause with a list of variables \a VL.
   ///
@@ -4387,7 +4390,7 @@ class OMPDependClause final
   /// clause.
   static OMPDependClause *Create(const ASTContext &C, SourceLocation StartLoc,
                                  SourceLocation LParenLoc,
-                                 SourceLocation EndLoc,
+                                 SourceLocation EndLoc, Expr *DepModifier,
                                  OpenMPDependClauseKind DepKind,
                                  SourceLocation DepLoc, SourceLocation ColonLoc,
                                  ArrayRef<Expr *> VL, unsigned NumLoops);
@@ -4404,6 +4407,12 @@ class OMPDependClause final
   /// Get dependency type.
   OpenMPDependClauseKind getDependencyKind() const { return DepKind; }
 
+  /// Return optional depend modifier.
+  Expr *getModifier();
+  const Expr *getModifier() const {
+    return const_cast<OMPDependClause *>(this)->getModifier();
+  }
+
   /// Get dependency type location.
   SourceLocation getDependencyLoc() const { return DepLoc; }
 
@@ -5240,19 +5249,14 @@ class OMPMapClause final : public OMPMappableExprListClause<OMPMapClause>,
     return getUniqueDeclarationsNum() + getTotalComponentListNum();
   }
 
-public:
-  /// Number of allowed map-type-modifiers.
-  static constexpr unsigned NumberOfModifiers =
-      OMPC_MAP_MODIFIER_last - OMPC_MAP_MODIFIER_unknown - 1;
-
 private:
   /// Map-type-modifiers for the 'map' clause.
-  OpenMPMapModifierKind MapTypeModifiers[NumberOfModifiers] = {
+  OpenMPMapModifierKind MapTypeModifiers[NumberOfOMPMapClauseModifiers] = {
       OMPC_MAP_MODIFIER_unknown, OMPC_MAP_MODIFIER_unknown,
       OMPC_MAP_MODIFIER_unknown};
 
   /// Location of map-type-modifiers for the 'map' clause.
-  SourceLocation MapTypeModifiersLoc[NumberOfModifiers];
+  SourceLocation MapTypeModifiersLoc[NumberOfOMPMapClauseModifiers];
 
   /// Map type for the 'map' clause.
   OpenMPMapClauseKind MapType = OMPC_MAP_unknown;
@@ -5321,7 +5325,7 @@ class OMPMapClause final : public OMPMappableExprListClause<OMPMapClause>,
   /// \param I index for map-type-modifier.
   /// \param T map-type-modifier for the clause.
   void setMapTypeModifier(unsigned I, OpenMPMapModifierKind T) {
-    assert(I < NumberOfModifiers &&
+    assert(I < NumberOfOMPMapClauseModifiers &&
            "Unexpected index to store map type modifier, exceeds array size.");
     MapTypeModifiers[I] = T;
   }
@@ -5331,7 +5335,7 @@ class OMPMapClause final : public OMPMappableExprListClause<OMPMapClause>,
   /// \param I index for map-type-modifier location.
   /// \param TLoc map-type-modifier location.
   void setMapTypeModifierLoc(unsigned I, SourceLocation TLoc) {
-    assert(I < NumberOfModifiers &&
+    assert(I < NumberOfOMPMapClauseModifiers &&
            "Index to store map type modifier location exceeds array size.");
     MapTypeModifiersLoc[I] = TLoc;
   }
@@ -5406,7 +5410,7 @@ class OMPMapClause final : public OMPMappableExprListClause<OMPMapClause>,
   ///
   /// \param Cnt index for map-type-modifier.
   OpenMPMapModifierKind getMapTypeModifier(unsigned Cnt) const LLVM_READONLY {
-    assert(Cnt < NumberOfModifiers &&
+    assert(Cnt < NumberOfOMPMapClauseModifiers &&
            "Requested modifier exceeds the total number of modifiers.");
     return MapTypeModifiers[Cnt];
   }
@@ -5416,7 +5420,7 @@ class OMPMapClause final : public OMPMappableExprListClause<OMPMapClause>,
   ///
   /// \param Cnt index for map-type-modifier location.
   SourceLocation getMapTypeModifierLoc(unsigned Cnt) const LLVM_READONLY {
-    assert(Cnt < NumberOfModifiers &&
+    assert(Cnt < NumberOfOMPMapClauseModifiers &&
            "Requested modifier location exceeds total number of modifiers.");
     return MapTypeModifiersLoc[Cnt];
   }
@@ -7135,6 +7139,19 @@ class OMPClausePrinter final : public OMPClauseVisitor<OMPClausePrinter> {
 #include "clang/Basic/OpenMPKinds.def"
 };
 
+struct OMPTraitProperty {
+  llvm::omp::TraitProperty Kind = llvm::omp::TraitProperty::invalid;
+};
+struct OMPTraitSelector {
+  Expr *ScoreOrCondition = nullptr;
+  llvm::omp::TraitSelector Kind = llvm::omp::TraitSelector::invalid;
+  llvm::SmallVector<OMPTraitProperty, 1> Properties;
+};
+struct OMPTraitSet {
+  llvm::omp::TraitSet Kind = llvm::omp::TraitSet::invalid;
+  llvm::SmallVector<OMPTraitSelector, 2> Selectors;
+};
+
 /// Helper data structure representing the traits in a match clause of an
 /// `declare variant` or `metadirective`. The outer level is an ordered
 /// collection of selector sets, each with an associated kind and an ordered
@@ -7146,27 +7163,17 @@ class OMPTraitInfo {
   friend class ASTContext;
 
 public:
-  struct OMPTraitProperty {
-    llvm::omp::TraitProperty Kind = llvm::omp::TraitProperty::invalid;
-  };
-  struct OMPTraitSelector {
-    Expr *ScoreOrCondition = nullptr;
-    llvm::omp::TraitSelector Kind = llvm::omp::TraitSelector::invalid;
-    llvm::SmallVector<OMPTraitProperty, 1> Properties;
-  };
-  struct OMPTraitSet {
-    llvm::omp::TraitSet Kind = llvm::omp::TraitSet::invalid;
-    llvm::SmallVector<OMPTraitSelector, 2> Selectors;
-  };
+  /// Reconstruct a (partial) OMPTraitInfo object from a mangled name.
+  OMPTraitInfo(StringRef MangledName);
 
   /// The outermost level of selector sets.
   llvm::SmallVector<OMPTraitSet, 2> Sets;
 
   bool anyScoreOrCondition(
       llvm::function_ref<bool(Expr *&, bool /* IsScore */)> Cond) {
-    return llvm::any_of(Sets, [&](OMPTraitInfo::OMPTraitSet &Set) {
+    return llvm::any_of(Sets, [&](OMPTraitSet &Set) {
       return llvm::any_of(
-          Set.Selectors, [&](OMPTraitInfo::OMPTraitSelector &Selector) {
+          Set.Selectors, [&](OMPTraitSelector &Selector) {
             return Cond(Selector.ScoreOrCondition,
                         /* IsScore */ Selector.Kind !=
                             llvm::omp::TraitSelector::user_condition);
@@ -7178,9 +7185,14 @@ class OMPTraitInfo {
   /// former is a flat representation the actual main difference is that the
   /// latter uses clang::Expr to store the score/condition while the former is
   /// independent of clang. Thus, expressions and conditions are evaluated in
-  /// this method.
+  /// this method. If \p DeviceSetOnly is true, only the device selector set, if
+  /// present, is put in \p VMI, otherwise all selector sets are put in \p VMI.
   void getAsVariantMatchInfo(ASTContext &ASTCtx,
-                             llvm::omp::VariantMatchInfo &VMI) const;
+                             llvm::omp::VariantMatchInfo &VMI,
+                             bool DeviceSetOnly) const;
+
+  /// Return a string representation identifying this context selector.
+  std::string getMangledName() const;
 
   /// Print a human readable representation into \p OS.
   void print(llvm::raw_ostream &OS, const PrintingPolicy &Policy) const;
diff --git a/clang/include/clang/AST/PrettyPrinter.h b/clang/include/clang/AST/PrettyPrinter.h
index dedfccda265da..d00deab95d9fd 100644
--- a/clang/include/clang/AST/PrettyPrinter.h
+++ b/clang/include/clang/AST/PrettyPrinter.h
@@ -57,7 +57,8 @@ struct PrintingPolicy {
         SuppressLifetimeQualifiers(false), SuppressTypedefs(false),
         SuppressTemplateArgsInCXXConstructors(false), Bool(LO.Bool),
         Restrict(LO.C99), Alignof(LO.CPlusPlus11), UnderscoreAlignof(LO.C11),
-        UseVoidForZeroParams(!LO.CPlusPlus), TerseOutput(false),
+        UseVoidForZeroParams(!LO.CPlusPlus),
+        SplitTemplateClosers(!LO.CPlusPlus11), TerseOutput(false),
         PolishForDeclaration(false), Half(LO.Half),
         MSWChar(LO.MicrosoftExt && !LO.WChar), IncludeNewlines(true),
         MSVCFormatting(false), ConstantsAsWritten(false),
@@ -206,6 +207,10 @@ struct PrintingPolicy {
   /// with zero parameters.
   unsigned UseVoidForZeroParams : 1;
 
+  /// Whether nested templates must be closed like 'a<b<c> >' rather than
+  /// 'a<b<c>>'.
+  unsigned SplitTemplateClosers : 1;
+
   /// Provide a 'terse' output.
   ///
   /// For example, in this mode we don't print function bodies, class members,
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index 2f598564f3077..3453338496599 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -2551,6 +2551,8 @@ DEF_TRAVERSE_STMT(CXXMemberCallExpr, {})
 DEF_TRAVERSE_STMT(AddrLabelExpr, {})
 DEF_TRAVERSE_STMT(ArraySubscriptExpr, {})
 DEF_TRAVERSE_STMT(OMPArraySectionExpr, {})
+DEF_TRAVERSE_STMT(OMPArrayShapingExpr, {})
+DEF_TRAVERSE_STMT(OMPIteratorExpr, {})
 
 DEF_TRAVERSE_STMT(BlockExpr, {
   TRY_TO(TraverseDecl(S->getBlockDecl()));
diff --git a/clang/include/clang/AST/Stmt.h b/clang/include/clang/AST/Stmt.h
index 87e1d7f26a168..8c7f0ecad3ef6 100644
--- a/clang/include/clang/AST/Stmt.h
+++ b/clang/include/clang/AST/Stmt.h
@@ -21,6 +21,7 @@
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/SourceLocation.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/BitmaskEnum.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/iterator.h"
@@ -310,9 +311,9 @@ class alignas(void *) Stmt {
 
     unsigned ValueKind : 2;
     unsigned ObjectKind : 3;
-    unsigned /*ExprDependence*/ Dependent : ExprDependenceBits;
+    unsigned /*ExprDependence*/ Dependent : llvm::BitWidth<ExprDependence>;
   };
-  enum { NumExprBits = NumStmtBits + 5 + ExprDependenceBits };
+  enum { NumExprBits = NumStmtBits + 5 + llvm::BitWidth<ExprDependence> };
 
   class ConstantExprBitfields {
     friend class ASTStmtReader;
diff --git a/clang/include/clang/AST/TextNodeDumper.h b/clang/include/clang/AST/TextNodeDumper.h
index b08d1e482be8b..0fb4baf0ca4d5 100644
--- a/clang/include/clang/AST/TextNodeDumper.h
+++ b/clang/include/clang/AST/TextNodeDumper.h
@@ -274,6 +274,7 @@ class TextNodeDumper
   void VisitObjCSubscriptRefExpr(const ObjCSubscriptRefExpr *Node);
   void VisitObjCIvarRefExpr(const ObjCIvarRefExpr *Node);
   void VisitObjCBoolLiteralExpr(const ObjCBoolLiteralExpr *Node);
+  void VisitOMPIteratorExpr(const OMPIteratorExpr *Node);
 
   void VisitRValueReferenceType(const ReferenceType *T);
   void VisitArrayType(const ArrayType *T);
diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h
index 85a7d682f5c8c..283ed4b2a6243 100644
--- a/clang/include/clang/AST/Type.h
+++ b/clang/include/clang/AST/Type.h
@@ -1475,7 +1475,7 @@ class alignas(8) Type : public ExtQualsTypeCommonBase {
     unsigned TC : 8;
 
     /// Store information on the type dependency.
-    /*TypeDependence*/ unsigned Dependence : TypeDependenceBits;
+    unsigned Dependence : llvm::BitWidth<TypeDependence>;
 
     /// True if the cache (i.e. the bitfields here starting with
     /// 'Cache') is valid.
@@ -1504,7 +1504,7 @@ class alignas(8) Type : public ExtQualsTypeCommonBase {
       return CachedLocalOrUnnamed;
     }
   };
-  enum { NumTypeBits = 8 + TypeDependenceBits + 6 };
+  enum { NumTypeBits = 8 + llvm::BitWidth<TypeDependence> + 6 };
 
 protected:
   // These classes allow subclasses to somewhat cleanly pack bitfields
@@ -1658,11 +1658,8 @@ class alignas(8) Type : public ExtQualsTypeCommonBase {
     /// The kind of vector, either a generic vector type or some
     /// target-specific vector type such as for AltiVec or Neon.
     unsigned VecKind : 3;
-
     /// The number of elements in the vector.
-    unsigned NumElements : 29 - NumTypeBits;
-
-    enum { MaxNumElements = (1 << (29 - NumTypeBits)) - 1 };
+    uint32_t NumElements;
   };
 
   class AttributedTypeBitfields {
@@ -2119,6 +2116,11 @@ class alignas(8) Type : public ExtQualsTypeCommonBase {
   /// than implicitly __strong.
   bool isObjCARCImplicitlyUnretainedType() const;
 
+  /// Check if the type is the CUDA device builtin surface type.
+  bool isCUDADeviceBuiltinSurfaceType() const;
+  /// Check if the type is the CUDA device builtin texture type.
+  bool isCUDADeviceBuiltinTextureType() const;
+
   /// Return the implicit lifetime for this type, which must not be dependent.
   Qualifiers::ObjCLifetime getObjCARCImplicitLifetime() const;
 
@@ -2142,6 +2144,11 @@ class alignas(8) Type : public ExtQualsTypeCommonBase {
     return static_cast<TypeDependence>(TypeBits.Dependence);
   }
 
+  /// Whether this type is an error type.
+  bool containsErrors() const {
+    return getDependence() & TypeDependence::Error;
+  }
+
   /// Whether this type is a dependent type, meaning that its definition
   /// somehow depends on a template parameter (C++ [temp.dep.type]).
   bool isDependentType() const {
@@ -3247,10 +3254,6 @@ class VectorType : public Type, public llvm::FoldingSetNode {
   QualType getElementType() const { return ElementType; }
   unsigned getNumElements() const { return VectorTypeBits.NumElements; }
 
-  static bool isVectorSizeTooLarge(unsigned NumElements) {
-    return NumElements > VectorTypeBitfields::MaxNumElements;
-  }
-
   bool isSugared() const { return false; }
   QualType desugar() const { return QualType(this, 0); }
 
@@ -6682,9 +6685,9 @@ inline bool Type::isTemplateTypeParmType() const {
 }
 
 inline bool Type::isSpecificBuiltinType(unsigned K) const {
-  if (const BuiltinType *BT = getAs<BuiltinType>())
-    if (BT->getKind() == (BuiltinType::Kind) K)
-      return true;
+  if (const BuiltinType *BT = getAs<BuiltinType>()) {
+    return BT->getKind() == static_cast<BuiltinType::Kind>(K);
+  }
   return false;
 }
 
@@ -6703,9 +6706,7 @@ inline const BuiltinType *Type::getAsPlaceholderType() const {
 
 inline bool Type::isSpecificPlaceholderType(unsigned K) const {
   assert(BuiltinType::isPlaceholderTypeKind((BuiltinType::Kind) K));
-  if (const auto *BT = dyn_cast<BuiltinType>(this))
-    return (BT->getKind() == (BuiltinType::Kind) K);
-  return false;
+  return isSpecificBuiltinType(K);
 }
 
 inline bool Type::isNonOverloadPlaceholderType() const {
@@ -6715,34 +6716,24 @@ inline bool Type::isNonOverloadPlaceholderType() const {
 }
 
 inline bool Type::isVoidType() const {
-  if (const auto *BT = dyn_cast<BuiltinType>(CanonicalType))
-    return BT->getKind() == BuiltinType::Void;
-  return false;
+  return isSpecificBuiltinType(BuiltinType::Void);
 }
 
 inline bool Type::isHalfType() const {
-  if (const auto *BT = dyn_cast<BuiltinType>(CanonicalType))
-    return BT->getKind() == BuiltinType::Half;
   // FIXME: Should we allow complex __fp16? Probably not.
-  return false;
+  return isSpecificBuiltinType(BuiltinType::Half);
 }
 
 inline bool Type::isFloat16Type() const {
-  if (const auto *BT = dyn_cast<BuiltinType>(CanonicalType))
-    return BT->getKind() == BuiltinType::Float16;
-  return false;
+  return isSpecificBuiltinType(BuiltinType::Float16);
 }
 
 inline bool Type::isFloat128Type() const {
-  if (const auto *BT = dyn_cast<BuiltinType>(CanonicalType))
-    return BT->getKind() == BuiltinType::Float128;
-  return false;
+  return isSpecificBuiltinType(BuiltinType::Float128);
 }
 
 inline bool Type::isNullPtrType() const {
-  if (const auto *BT = getAs<BuiltinType>())
-    return BT->getKind() == BuiltinType::NullPtr;
-  return false;
+  return isSpecificBuiltinType(BuiltinType::NullPtr);
 }
 
 bool IsEnumDeclComplete(EnumDecl *);
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index dd873ab2595c9..6f7a4b7949b08 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -1073,16 +1073,20 @@ def CUDADeviceBuiltin : IgnoredAttr {
   let LangOpts = [CUDA];
 }
 
-def CUDADeviceBuiltinSurfaceType : IgnoredAttr {
+def CUDADeviceBuiltinSurfaceType : InheritableAttr {
   let Spellings = [GNU<"device_builtin_surface_type">,
                    Declspec<"__device_builtin_surface_type__">];
   let LangOpts = [CUDA];
+  let Subjects = SubjectList<[CXXRecord]>;
+  let Documentation = [CUDADeviceBuiltinSurfaceTypeDocs];
 }
 
-def CUDADeviceBuiltinTextureType : IgnoredAttr {
+def CUDADeviceBuiltinTextureType : InheritableAttr {
   let Spellings = [GNU<"device_builtin_texture_type">,
                    Declspec<"__device_builtin_texture_type__">];
   let LangOpts = [CUDA];
+  let Subjects = SubjectList<[CXXRecord]>;
+  let Documentation = [CUDADeviceBuiltinTextureTypeDocs];
 }
 
 def CUDAGlobal : InheritableAttr {
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index e8cc79967f059..7d375ac502976 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -140,7 +140,8 @@ def NoEscapeDocs : Documentation {
 the compiler that the pointer cannot escape: that is, no reference to the object
 the pointer points to that is derived from the parameter value will survive
 after the function returns. Users are responsible for making sure parameters
-annotated with ``noescape`` do not actuallly escape.
+annotated with ``noescape`` do not actually escape. Calling ``free()`` on such
+a parameter does not constitute an escape.
 
 For example:
 
@@ -484,7 +485,7 @@ parameter.
 Note that this attribute merely informs the compiler that a function always
 returns a sufficiently aligned pointer. It does not cause the compiler to
 emit code to enforce that alignment.  The behavior is undefined if the returned
-poitner is not sufficiently aligned.
+pointer is not sufficiently aligned.
   }];
 }
 
@@ -942,11 +943,11 @@ The behavior of a function with respect to reference counting for Foundation
 convention (e.g. functions starting with "get" are assumed to return at
 ``+0``).
 
-It can be overriden using a family of the following attributes.  In
+It can be overridden using a family of the following attributes.  In
 Objective-C, the annotation ``__attribute__((ns_returns_retained))`` applied to
 a function communicates that the object is returned at ``+1``, and the caller
 is responsible for freeing it.
-Similiarly, the annotation ``__attribute__((ns_returns_not_retained))``
+Similarly, the annotation ``__attribute__((ns_returns_not_retained))``
 specifies that the object is returned at ``+0`` and the ownership remains with
 the callee.
 The annotation ``__attribute__((ns_consumes_self))`` specifies that
@@ -1290,7 +1291,7 @@ correspond to different platforms. For most platforms, the availability
 attribute with the platform corresponding to the target platform will be used;
 any others will be ignored. However, the availability for ``watchOS`` and
 ``tvOS`` can be implicitly inferred from an ``iOS`` availability attribute.
-Any explicit availability attributes for those platforms are still prefered over
+Any explicit availability attributes for those platforms are still preferred over
 the implicitly inferred availability attributes. If no availability attribute
 specifies availability for the current target platform, the availability
 attributes are ignored. Supported platforms are:
@@ -1398,7 +1399,7 @@ pragma rather than using the inferred ``iOS`` availability from the declaration:
   void getsThePragmaTVOSAvailabilityAttribute(void) __attribute__((availability(iOS,introduced=11.0)));
   #pragma clang attribute pop
 
-The compiler is also able to apply implicly inferred attributes from a pragma
+The compiler is also able to apply implicitly inferred attributes from a pragma
 as well. For example, when targeting ``tvOS``, the function below will receive
 a ``tvOS`` availability attribute that is implicitly inferred from the ``iOS``
 availability attribute applied by the pragma:
@@ -1716,7 +1717,7 @@ def BPFPreserveAccessIndexDocs : Documentation {
 Clang supports the ``__attribute__((preserve_access_index))``
 attribute for the BPF target. This attribute may be attached to a
 struct or union declaration, where if -g is specified, it enables
-preserving struct or union member access debuginfo indicies of this
+preserving struct or union member access debuginfo indices of this
 struct or union, similar to clang ``__builtin_preserve_acceess_index()``.
   }];
 }
@@ -1732,7 +1733,7 @@ directly as an interrupt service routine.
 
 By default, the compiler will produce a function prologue and epilogue suitable for
 an interrupt service routine that handles an External Interrupt Controller (eic)
-generated interrupt. This behaviour can be explicitly requested with the "eic"
+generated interrupt. This behavior can be explicitly requested with the "eic"
 argument.
 
 Otherwise, for use with vectored interrupt mode, the argument passed should be
@@ -4118,7 +4119,7 @@ using the Swift calling convention for a function or function pointer.
 The lowering for the Swift calling convention, as described by the Swift
 ABI documentation, occurs in multiple phases.  The first, "high-level"
 phase breaks down the formal parameters and results into innately direct
-and indirect components, adds implicit paraameters for the generic
+and indirect components, adds implicit parameters for the generic
 signature, and assigns the context and error ABI treatments to parameters
 where applicable.  The second phase breaks down the direct parameters
 and results from the first phase and assigns them to registers or the
@@ -4160,7 +4161,7 @@ of the first phase, as follows:
 ``swiftcall`` does not support variadic arguments or unprototyped functions.
 
 The parameter ABI treatment attributes are aspects of the function type.
-A function type which which applies an ABI treatment attribute to a
+A function type which applies an ABI treatment attribute to a
 parameter is a different type from an otherwise-identical function type
 that does not.  A single parameter may not have multiple ABI treatment
 attributes.
@@ -4289,7 +4290,7 @@ with different ABI versions supported. For example, a newer version of a class
 could have a different set of data members and thus have a different size. Using
 the ``abi_tag`` attribute, it is possible to have different mangled names for
 a global variable of the class type. Therefore, the old code could keep using
-the old manged name and the new code will use the new mangled name with tags.
+the old mangled name and the new code will use the new mangled name with tags.
   }];
 }
 
@@ -4443,9 +4444,9 @@ takes precedence over the command line option ``-fpatchable-function-entry=N,M``
 def TransparentUnionDocs : Documentation {
   let Category = DocCatDecl;
   let Content = [{
-This attribute can be applied to a union to change the behaviour of calls to
+This attribute can be applied to a union to change the behavior of calls to
 functions that have an argument with a transparent union type. The compiler
-behaviour is changed in the following manner:
+behavior is changed in the following manner:
 
 - A value whose type is any member of the transparent union can be passed as an
   argument without the need to cast that value.
@@ -4977,7 +4978,7 @@ When applied to the definition of a function, method, or block, every parameter
 of the function with implicit strong retainable object pointer type is
 considered externally-retained, and becomes ``const``. By explicitly annotating
 a parameter with ``__strong``, you can opt back into the default
-non-externally-retained behaviour for that parameter. For instance,
+non-externally-retained behavior for that parameter. For instance,
 ``first_param`` is externally-retained below, but not ``second_param``:
 
 .. code-block:: objc
@@ -4999,7 +5000,7 @@ def MIGConventionDocs : Documentation {
   The Mach Interface Generator release-on-success convention dictates
 functions that follow it to only release arguments passed to them when they
 return "success" (a ``kern_return_t`` error code that indicates that
-no errors have occured). Otherwise the release is performed by the MIG client
+no errors have occurred). Otherwise the release is performed by the MIG client
 that called the function. The annotation ``__attribute__((mig_server_routine))``
 is applied in order to specify which functions are expected to follow the
 convention. This allows the Static Analyzer to find bugs caused by violations of
@@ -5063,6 +5064,28 @@ the initializer on host side.
   }];
 }
 
+def CUDADeviceBuiltinSurfaceTypeDocs : Documentation {
+  let Category = DocCatType;
+  let Content = [{
+The ``device_builtin_surface_type`` attribute can be applied to a class
+template when declaring the surface reference. A surface reference variable
+could be accessed on the host side and, on the device side, might be translated
+into an internal surface object, which is established through surface bind and
+unbind runtime APIs.
+  }];
+}
+
+def CUDADeviceBuiltinTextureTypeDocs : Documentation {
+  let Category = DocCatType;
+  let Content = [{
+The ``device_builtin_texture_type`` attribute can be applied to a class
+template when declaring the texture reference. A texture reference variable
+could be accessed on the host side and, on the device side, might be translated
+into an internal texture object, which is established through texture bind and
+unbind runtime APIs.
+  }];
+}
+
 def LifetimeOwnerDocs : Documentation {
   let Category = DocCatDecl;
   let Content = [{
diff --git a/clang/include/clang/Basic/Builtins.def b/clang/include/clang/Basic/Builtins.def
index 0aa11fa6ac177..0b411d9bd8e45 100644
--- a/clang/include/clang/Basic/Builtins.def
+++ b/clang/include/clang/Basic/Builtins.def
@@ -472,7 +472,7 @@ BUILTIN(__builtin___NSStringMakeConstantString, "FC*cC*", "nc")
 BUILTIN(__builtin_va_start, "vA.", "nt")
 BUILTIN(__builtin_va_end, "vA", "n")
 BUILTIN(__builtin_va_copy, "vAA", "n")
-BUILTIN(__builtin_stdarg_start, "vA.", "n")
+BUILTIN(__builtin_stdarg_start, "vA.", "nt")
 BUILTIN(__builtin_assume_aligned, "v*vC*z.", "nc")
 BUILTIN(__builtin_bcmp, "ivC*vC*z", "Fn")
 BUILTIN(__builtin_bcopy, "vv*v*z", "n")
diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.def b/clang/include/clang/Basic/BuiltinsAMDGPU.def
index a9143ad8292c3..b42c8a77c4bc3 100644
--- a/clang/include/clang/Basic/BuiltinsAMDGPU.def
+++ b/clang/include/clang/Basic/BuiltinsAMDGPU.def
@@ -33,6 +33,10 @@ BUILTIN(__builtin_amdgcn_workitem_id_x, "Ui", "nc")
 BUILTIN(__builtin_amdgcn_workitem_id_y, "Ui", "nc")
 BUILTIN(__builtin_amdgcn_workitem_id_z, "Ui", "nc")
 
+BUILTIN(__builtin_amdgcn_workgroup_size_x, "Us", "nc")
+BUILTIN(__builtin_amdgcn_workgroup_size_y, "Us", "nc")
+BUILTIN(__builtin_amdgcn_workgroup_size_z, "Us", "nc")
+
 BUILTIN(__builtin_amdgcn_mbcnt_hi, "UiUiUi", "nc")
 BUILTIN(__builtin_amdgcn_mbcnt_lo, "UiUiUi", "nc")
 
diff --git a/clang/include/clang/Basic/BuiltinsX86.def b/clang/include/clang/Basic/BuiltinsX86.def
index d6aa46de02f95..ccb6f341d63da 100644
--- a/clang/include/clang/Basic/BuiltinsX86.def
+++ b/clang/include/clang/Basic/BuiltinsX86.def
@@ -1900,6 +1900,9 @@ TARGET_BUILTIN(__builtin_ia32_invpcid, "vUiv*", "nc", "invpcid")
 TARGET_BUILTIN(__builtin_ia32_enqcmd, "Ucv*vC*", "n", "enqcmd")
 TARGET_BUILTIN(__builtin_ia32_enqcmds, "Ucv*vC*", "n", "enqcmd")
 
+// SERIALIZE
+TARGET_BUILTIN(__builtin_ia32_serialize, "v", "n", "serialize")
+
 // MSVC
 TARGET_HEADER_BUILTIN(_BitScanForward, "UcUNi*UNi", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_BitScanReverse, "UcUNi*UNi", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
diff --git a/clang/include/clang/Basic/CodeGenOptions.def b/clang/include/clang/Basic/CodeGenOptions.def
index 5b59954fae7bb..0faa013ac4975 100644
--- a/clang/include/clang/Basic/CodeGenOptions.def
+++ b/clang/include/clang/Basic/CodeGenOptions.def
@@ -254,6 +254,7 @@ CODEGENOPT(UnwindTables      , 1, 0) ///< Emit unwind tables.
 CODEGENOPT(VectorizeLoop     , 1, 0) ///< Run loop vectorizer.
 CODEGENOPT(VectorizeSLP      , 1, 0) ///< Run SLP vectorizer.
 CODEGENOPT(ProfileSampleAccurate, 1, 0) ///< Sample profile is accurate.
+CODEGENOPT(CallGraphProfile  , 1, 0) ///< Run call graph profile.
 
   /// Attempt to use register sized accesses to bit-fields in structures, when
   /// possible.
@@ -386,10 +387,6 @@ CODEGENOPT(ForceEmitVTables, 1, 0)
 /// Whether to emit an address-significance table into the object file.
 CODEGENOPT(Addrsig, 1, 0)
 
-ENUM_CODEGENOPT(SignReturnAddress, SignReturnAddressScope, 2, SignReturnAddressScope::None)
-ENUM_CODEGENOPT(SignReturnAddressKey, SignReturnAddressKeyValue, 1, SignReturnAddressKeyValue::AKey)
-CODEGENOPT(BranchTargetEnforcement, 1, 0)
-
 /// Whether to emit unused static constants.
 CODEGENOPT(KeepStaticConsts, 1, 0)
 
@@ -399,4 +396,3 @@ CODEGENOPT(ForceAAPCSBitfieldLoad, 1, 0)
 #undef CODEGENOPT
 #undef ENUM_CODEGENOPT
 #undef VALUE_CODEGENOPT
-
diff --git a/clang/include/clang/Basic/CodeGenOptions.h b/clang/include/clang/Basic/CodeGenOptions.h
index d435bebcdcf97..60d418688710e 100644
--- a/clang/include/clang/Basic/CodeGenOptions.h
+++ b/clang/include/clang/Basic/CodeGenOptions.h
@@ -110,14 +110,6 @@ class CodeGenOptions : public CodeGenOptionsBase {
     Embed_Marker    // Embed a marker as a placeholder for bitcode.
   };
 
-  enum class SignReturnAddressScope {
-    None,    // No signing for any function
-    NonLeaf, // Sign the return address of functions that spill LR
-    All      // Sign the return address of all functions
-  };
-
-  enum class SignReturnAddressKeyValue { AKey, BKey };
-
   enum class FramePointerKind {
     None,        // Omit all frame pointers.
     NonLeaf,     // Keep non-leaf frame pointers.
diff --git a/clang/include/clang/Basic/Cuda.h b/clang/include/clang/Basic/Cuda.h
index da572957d10dd..c2ebf87343047 100644
--- a/clang/include/clang/Basic/Cuda.h
+++ b/clang/include/clang/Basic/Cuda.h
@@ -117,6 +117,7 @@ enum class CudaFeature {
   CUDA_USES_FATBIN_REGISTER_END,
 };
 
+CudaVersion ToCudaVersion(llvm::VersionTuple);
 bool CudaFeatureEnabled(llvm::VersionTuple, CudaFeature);
 bool CudaFeatureEnabled(CudaVersion, CudaFeature);
 
diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index 0b3a1a2575b64..8cbce0572818c 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -366,6 +366,8 @@ def err_analyzer_checker_option_unknown : Error<
   "checker '%0' has no option called '%1'">;
 def err_analyzer_checker_option_invalid_input : Error<
   "invalid input for checker option '%0', that expects %1">;
+def err_analyzer_checker_incompatible_analyzer_option : Error<
+  "checker cannot be enabled with analyzer option '%0' == %1">;
 
 def err_drv_invalid_hvx_length : Error<
   "-mhvx-length is not supported without a -mhvx/-mhvx= flag">;
@@ -472,13 +474,13 @@ def note_drv_verify_prefix_spelling : Note<
   "-verify prefixes must start with a letter and contain only alphanumeric"
   " characters, hyphens, and underscores">;
 
-def warn_drv_experimental_isel_incomplete : Warning<
-  "-fexperimental-isel support for the '%0' architecture is incomplete">,
-  InGroup<ExperimentalISel>;
+def warn_drv_global_isel_incomplete : Warning<
+  "-fglobal-isel support for the '%0' architecture is incomplete">,
+  InGroup<GlobalISel>;
 
-def warn_drv_experimental_isel_incomplete_opt : Warning<
-  "-fexperimental-isel support is incomplete for this architecture at the current optimization level">,
-  InGroup<ExperimentalISel>;
+def warn_drv_global_isel_incomplete_opt : Warning<
+  "-fglobal-isel support is incomplete for this architecture at the current optimization level">,
+  InGroup<GlobalISel>;
 
 def warn_drv_moutline_unsupported_opt : Warning<
   "The '%0' architecture does not support -moutline; flag ignored">,
diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td
index 7c7bd7eae3740..84810e657d969 100644
--- a/clang/include/clang/Basic/DiagnosticGroups.td
+++ b/clang/include/clang/Basic/DiagnosticGroups.td
@@ -1160,8 +1160,8 @@ def UnknownArgument : DiagGroup<"unknown-argument">;
 // compiling OpenCL C/C++ but which is not compatible with the SPIR spec.
 def SpirCompat : DiagGroup<"spir-compat">;
 
-// Warning for the experimental-isel options.
-def ExperimentalISel : DiagGroup<"experimental-isel">;
+// Warning for the GlobalISel options.
+def GlobalISel : DiagGroup<"global-isel">;
 
 // A warning group specifically for warnings related to function
 // multiversioning.
diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td
index 469af4167f8aa..d5e480d557ba9 100644
--- a/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -1220,6 +1220,10 @@ def err_omp_expected_identifier_for_critical : Error<
   "expected identifier specifying the name of the 'omp critical' directive">;
 def err_omp_expected_reduction_identifier : Error<
   "expected identifier or one of the following operators: '+', '-', '*', '&', '|', '^', '&&', or '||'">;
+def err_omp_expected_equal_in_iterator : Error<
+  "expected '=' in iterator specifier">;
+def err_omp_expected_punc_after_iterator : Error<
+  "expected ',' or ')' after iterator specifier">;
 def err_omp_decl_in_declare_simd_variant : Error<
   "function declaration is expected after 'declare %select{simd|variant}0' directive">;
 def err_omp_unknown_map_type : Error<
@@ -1232,8 +1236,11 @@ def err_omp_map_type_modifier_missing : Error<
   "missing map type modifier">;
 def err_omp_declare_simd_inbranch_notinbranch : Error<
   "unexpected '%0' clause, '%1' is specified already">;
-def err_expected_end_declare_target : Error<
-  "expected '#pragma omp end declare target'">;
+def err_expected_end_declare_target_or_variant : Error<
+  "expected '#pragma omp end declare %select{target|variant}0'">;
+def err_expected_begin_declare_variant
+    : Error<"'#pragma omp end declare variant' with no matching '#pragma omp "
+            "begin declare variant'">;
 def err_omp_declare_target_unexpected_clause: Error<
   "unexpected '%0' clause, only %select{'to' or 'link'|'to', 'link' or 'device_type'}1 clauses expected">;
 def err_omp_expected_clause: Error<
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index b13492fb85152..5d8daa27b5bff 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -4957,9 +4957,13 @@ def note_using_value_decl_missing_typename : Note<
   "add 'typename' to treat this using declaration as a type">;
 
 def err_template_kw_refers_to_non_template : Error<
-  "%0 following the 'template' keyword does not refer to a template">;
+  "%0%select{| following the 'template' keyword}1 "
+  "does not refer to a template">;
 def note_template_kw_refers_to_non_template : Note<
   "declared as a non-template here">;
+def err_template_kw_refers_to_dependent_non_template : Error<
+  "%0%select{| following the 'template' keyword}1 "
+  "cannot refer to a dependent template">;
 def err_template_kw_refers_to_class_template : Error<
   "'%0%1' instantiated to a class template, not a function template">;
 def note_referenced_class_template : Note<
@@ -5983,7 +5987,7 @@ def err_func_def_incomplete_result : Error<
 def err_atomic_specifier_bad_type : Error<
   "_Atomic cannot be applied to "
   "%select{incomplete |array |function |reference |atomic |qualified |sizeless |}0type "
-  "%1 %select{||||||which is not trivially copyable}0">;
+  "%1 %select{|||||||which is not trivially copyable}0">;
 
 // Expressions.
 def select_unary_expr_or_type_trait_kind : TextSubstitution<
@@ -8008,6 +8012,22 @@ def err_cuda_ovl_target : Error<
 def note_cuda_ovl_candidate_target_mismatch : Note<
     "candidate template ignored: target attributes do not match">;
 
+def err_cuda_device_builtin_surftex_cls_template : Error<
+    "illegal device builtin %select{surface|texture}0 reference "
+    "class template %1 declared here">;
+def note_cuda_device_builtin_surftex_cls_should_have_n_args : Note<
+    "%0 needs to have exactly %1 template parameters">;
+def note_cuda_device_builtin_surftex_cls_should_have_match_arg : Note<
+    "the %select{1st|2nd|3rd}1 template parameter of %0 needs to be "
+    "%select{a type|an integer or enum value}2">;
+
+def err_cuda_device_builtin_surftex_ref_decl : Error<
+    "illegal device builtin %select{surface|texture}0 reference "
+    "type %1 declared here">;
+def note_cuda_device_builtin_surftex_should_be_template_class : Note<
+    "%0 needs to be instantiated from a class template with proper "
+    "template arguments">;
+
 def warn_non_pod_vararg_with_format_string : Warning<
   "cannot pass %select{non-POD|non-trivial}0 object of type %1 to variadic "
   "%select{function|block|method|constructor}2; expected type from format "
@@ -9706,7 +9726,7 @@ def err_omp_expected_var_name_member_expr : Error<
 def err_omp_expected_var_name_member_expr_or_array_item : Error<
   "expected variable name%select{|, data member of current class}0, array element or array section">;
 def err_omp_expected_addressable_lvalue_or_array_item : Error<
-  "expected addressable lvalue expression, array element or array section%select{| of non 'omp_depend_t' type}0">;
+  "expected addressable lvalue expression, array element%select{ or array section|, array section or array shaping expression}0%select{| of non 'omp_depend_t' type}1">;
 def err_omp_expected_named_var_member_or_array_expression: Error<
   "expected expression containing only member accesses and/or array sections based on named variables">;
 def err_omp_bit_fields_forbidden_in_clause : Error<
@@ -9781,6 +9801,12 @@ def note_omp_conversion_here : Note<
 def err_omp_ambiguous_conversion : Error<
   "ambiguous conversion from type %0 to an integral or unscoped "
   "enumeration type">;
+def err_omp_iterator_not_integral_or_pointer : Error<
+  "expected integral or pointer type as the iterator-type, not %0">;
+def err_omp_iterator_step_not_integral : Error<
+  "iterator step expression %0 is not the integral expression">;
+def err_omp_iterator_step_constant_zero : Error<
+  "iterator step expression %0 evaluates to 0">;
 def err_omp_required_access : Error<
   "%0 variable must be %1">;
 def err_omp_const_variable : Error<
@@ -9869,8 +9895,6 @@ def err_omp_reduction_in_task : Error<
   "reduction variables may not be accessed in an explicit task">;
 def err_omp_reduction_id_not_compatible : Error<
   "list item of type %0 is not valid for specified reduction operation: unable to provide default initialization value">;
-def err_omp_in_reduction_not_task_reduction : Error<
-  "in_reduction variable must appear in a task_reduction clause">;
 def err_omp_reduction_identifier_mismatch : Error<
   "in_reduction variable must have the same reduction operation as in a task_reduction clause">;
 def note_omp_previous_reduction_identifier : Note<
@@ -9881,7 +9905,7 @@ def err_omp_prohibited_region : Error<
   "; perhaps you forget to enclose 'omp %3' directive into a for or a parallel for region with 'ordered' clause?|"
   "; perhaps you forget to enclose 'omp %3' directive into a target region?|"
   "; perhaps you forget to enclose 'omp %3' directive into a teams region?|"
-  "; perhaps you forget to enclose 'omp %3' directive into a for, simd, or for simd region?}2">;
+  "; perhaps you forget to enclose 'omp %3' directive into a for, simd, for simd, parallel for, or parallel for simd region?}2">;
 def err_omp_prohibited_region_simd : Error<
   "OpenMP constructs may not be nested inside a simd region%select{| except for ordered simd, simd, scan, or atomic directive}0">;
 def err_omp_prohibited_region_atomic : Error<
@@ -9975,10 +9999,16 @@ def err_omp_declare_mapper_redefinition : Error<
 def err_omp_invalid_mapper: Error<
   "cannot find a valid user-defined mapper for type %0 with name %1">;
 def err_omp_array_section_use : Error<"OpenMP array section is not allowed here">;
+def err_omp_array_shaping_use : Error<"OpenMP array shaping operation is not allowed here">;
+def err_omp_iterator_use : Error<"OpenMP iterator is not allowed here">;
 def err_omp_typecheck_section_value : Error<
   "subscripted value is not an array or pointer">;
 def err_omp_typecheck_section_not_integer : Error<
   "array section %select{lower bound|length}0 is not an integer">;
+def err_omp_typecheck_shaping_not_integer : Error<
+  "array shaping operation dimension is not an integer">;
+def err_omp_shaping_dimension_not_positive : Error<
+  "array shaping dimension is evaluated to a non-positive value %0">;
 def err_omp_section_function_type : Error<
   "section of pointer to function type %0">;
 def warn_omp_section_is_char : Warning<"array section %select{lower bound|length}0 is of type 'char'">,
@@ -10049,6 +10079,10 @@ def err_omp_depend_sink_source_not_allowed : Error<
   "'depend(%select{source|sink:vec}0)' clause%select{|s}0 cannot be mixed with 'depend(%select{sink:vec|source}0)' clause%select{s|}0">;
 def err_omp_depend_zero_length_array_section_not_allowed : Error<
   "zero-length array section is not allowed in 'depend' clause">;
+def err_omp_depend_sink_source_with_modifier : Error<
+  "depend modifier cannot be used with 'sink' or 'source' depend type">;
+def err_omp_depend_modifier_not_iterator : Error<
+  "expected iterator specification as depend modifier">;
 def err_omp_linear_ordered : Error<
   "'linear' clause cannot be specified along with 'ordered' clause with a parameter">;
 def err_omp_unexpected_schedule_modifier : Error<
@@ -10078,7 +10112,8 @@ def warn_omp_nesting_simd : Warning<
   InGroup<SourceUsesOpenMP>;
 def err_omp_orphaned_device_directive : Error<
   "orphaned 'omp %0' directives are prohibited"
-  "; perhaps you forget to enclose the directive into a %select{|||target |teams|for, simd, or for simd }1region?">;
+  "; perhaps you forget to enclose the directive into a "
+  "%select{|||target |teams|for, simd, for simd, parallel for, or parallel for simd }1region?">;
 def err_omp_reduction_non_addressable_expression : Error<
   "expected addressable reduction item for the task-based directives">;
 def err_omp_reduction_with_nogroup : Error<
@@ -10118,6 +10153,19 @@ def err_omp_depobj_single_clause_expected : Error<
   "exactly one of 'depend', 'destroy', or 'update' clauses is expected">;
 def err_omp_scan_single_clause_expected : Error<
   "exactly one of 'inclusive' or 'exclusive' clauses is expected">;
+def err_omp_inclusive_exclusive_not_reduction : Error<
+  "the list item must appear in 'reduction' clause with the 'inscan' modifier "
+  "of the parent directive">;
+def err_omp_reduction_not_inclusive_exclusive : Error<
+  "the inscan reduction list item must appear as a list item in an 'inclusive' or"
+  " 'exclusive' clause on an inner 'omp scan' directive">;
+def err_omp_wrong_inscan_reduction : Error<
+  "'inscan' modifier can be used only in 'omp for', 'omp simd', 'omp for simd',"
+  " 'omp parallel for', or 'omp parallel for simd' directive">;
+def err_omp_inscan_reduction_expected : Error<
+  "expected 'reduction' clause with the 'inscan' modifier">;
+def note_omp_previous_inscan_reduction : Note<
+  "'reduction' clause with 'inscan' modifier is used here">;
 def err_omp_expected_predefined_allocator : Error<
   "expected one of the predefined allocators for the variables with the static "
   "storage: 'omp_default_mem_alloc', 'omp_large_cap_mem_alloc', "
@@ -10199,10 +10247,16 @@ def err_omp_flush_order_clause_and_list : Error<
 def note_omp_flush_order_clause_here : Note<
   "memory order clause '%0' is specified here">;
 def err_omp_non_lvalue_in_map_or_motion_clauses: Error<
-  "expected addressable lvalue in '%0' clause"
-  >;
+  "expected addressable lvalue in '%0' clause">;
 def err_omp_event_var_expected : Error<
   "expected variable of the 'omp_event_handle_t' type%select{|, not %1}0">;
+def warn_nested_declare_variant
+    : Warning<"nesting `omp begin/end declare variant` is not supported yet; "
+              "nested context ignored">,
+      InGroup<SourceUsesOpenMP>;
+def err_omp_non_pointer_type_array_shaping_base : Error<
+  "expected expression with a pointer to a complete type as a base of an array "
+  "shaping operation">;
 } // end of OpenMP category
 
 let CategoryName = "Related Result Type Issue" in {
diff --git a/clang/include/clang/Basic/IdentifierTable.h b/clang/include/clang/Basic/IdentifierTable.h
index e5ae83331f4c1..f23e7276b0306 100644
--- a/clang/include/clang/Basic/IdentifierTable.h
+++ b/clang/include/clang/Basic/IdentifierTable.h
@@ -108,7 +108,10 @@ class alignas(IdentifierInfoAlignment) IdentifierInfo {
   // True if this is the 'import' contextual keyword.
   unsigned IsModulesImport : 1;
 
-  // 29 bits left in a 64-bit word.
+  // True if this is a mangled OpenMP variant name.
+  unsigned IsMangledOpenMPVariantName : 1;
+
+  // 28 bits left in a 64-bit word.
 
   // Managed by the language front-end.
   void *FETokenInfo = nullptr;
@@ -121,7 +124,7 @@ class alignas(IdentifierInfoAlignment) IdentifierInfo {
         IsPoisoned(false), IsCPPOperatorKeyword(false),
         NeedsHandleIdentifier(false), IsFromAST(false), ChangedAfterLoad(false),
         FEChangedAfterLoad(false), RevertedTokenID(false), OutOfDate(false),
-        IsModulesImport(false) {}
+        IsModulesImport(false), IsMangledOpenMPVariantName(false) {}
 
 public:
   IdentifierInfo(const IdentifierInfo &) = delete;
@@ -371,6 +374,12 @@ class alignas(IdentifierInfoAlignment) IdentifierInfo {
       RecomputeNeedsHandleIdentifier();
   }
 
+  /// Determine whether this is the mangled name of an OpenMP variant.
+  bool isMangledOpenMPVariantName() const { return IsMangledOpenMPVariantName; }
+
+  /// Set whether this is the mangled name of an OpenMP variant.
+  void setMangledOpenMPVariantName(bool I) { IsMangledOpenMPVariantName = I; }
+
   /// Return true if this identifier is an editor placeholder.
   ///
   /// Editor placeholders are produced by the code-completion engine and are
diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def
index 42a9230998f6c..982525d1e4679 100644
--- a/clang/include/clang/Basic/LangOptions.def
+++ b/clang/include/clang/Basic/LangOptions.def
@@ -148,7 +148,7 @@ LANGOPT(RelaxedTemplateTemplateArgs, 1, 0, "C++17 relaxed matching of template t
 
 LANGOPT(DoubleSquareBracketAttributes, 1, 0, "'[[]]' attributes extension for all language standard modes")
 
-COMPATIBLE_LANGOPT(RecoveryAST, 1, CPlusPlus, "Preserve expressions in AST when encountering errors")
+COMPATIBLE_LANGOPT(RecoveryAST, 1, 0, "Preserve expressions in AST when encountering errors")
 
 BENIGN_LANGOPT(ThreadsafeStatics , 1, 1, "thread-safe static initializers")
 LANGOPT(POSIXThreads      , 1, 0, "POSIX thread support")
@@ -358,6 +358,12 @@ LANGOPT(RegisterStaticDestructors, 1, 1, "Register C++ static destructors")
 
 COMPATIBLE_VALUE_LANGOPT(MaxTokens, 32, 0, "Max number of tokens per TU or 0")
 
+ENUM_LANGOPT(SignReturnAddressScope, SignReturnAddressScopeKind, 2, SignReturnAddressScopeKind::None,
+             "Scope of return address signing")
+ENUM_LANGOPT(SignReturnAddressKey, SignReturnAddressKeyKind, 1, SignReturnAddressKeyKind::AKey,
+             "Key used for return address signing")
+LANGOPT(BranchTargetEnforcement, 1, 0, "Branch-target enforcement enabled")
+
 #undef LANGOPT
 #undef COMPATIBLE_LANGOPT
 #undef BENIGN_LANGOPT
diff --git a/clang/include/clang/Basic/LangOptions.h b/clang/include/clang/Basic/LangOptions.h
index 0c0db4e03fb84..339f448a9632c 100644
--- a/clang/include/clang/Basic/LangOptions.h
+++ b/clang/include/clang/Basic/LangOptions.h
@@ -234,6 +234,22 @@ class LangOptions : public LangOptionsBase {
     All,
   };
 
+  enum class SignReturnAddressScopeKind {
+    /// No signing for any function.
+    None,
+    /// Sign the return address of functions that spill LR.
+    NonLeaf,
+    /// Sign the return address of all functions,
+    All
+  };
+
+  enum class SignReturnAddressKeyKind {
+    /// Return address signing uses APIA key.
+    AKey,
+    /// Return address signing uses APIB key.
+    BKey
+  };
+
 public:
   /// Set of enabled sanitizers.
   SanitizerSet Sanitize;
@@ -359,6 +375,21 @@ class LangOptions : public LangOptionsBase {
 
   /// Return the OpenCL C or C++ version as a VersionTuple.
   VersionTuple getOpenCLVersionTuple() const;
+
+  /// Check if return address signing is enabled.
+  bool hasSignReturnAddress() const {
+    return getSignReturnAddressScope() != SignReturnAddressScopeKind::None;
+  }
+
+  /// Check if return address signing uses AKey.
+  bool isSignReturnAddressWithAKey() const {
+    return getSignReturnAddressKey() == SignReturnAddressKeyKind::AKey;
+  }
+
+  /// Check if leaf functions are also signed.
+  bool isSignReturnAddressScopeAll() const {
+    return getSignReturnAddressScope() == SignReturnAddressScopeKind::All;
+  }
 };
 
 /// Floating point control options
diff --git a/clang/include/clang/Basic/Module.h b/clang/include/clang/Basic/Module.h
index 9c2bc155cd4f6..c47eb4587a574 100644
--- a/clang/include/clang/Basic/Module.h
+++ b/clang/include/clang/Basic/Module.h
@@ -662,7 +662,7 @@ class ASTSourceDescriptor {
   StringRef Path;
   StringRef ASTFile;
   ASTFileSignature Signature;
-  const Module *ClangModule = nullptr;
+  Module *ClangModule = nullptr;
 
 public:
   ASTSourceDescriptor() = default;
@@ -670,13 +670,13 @@ class ASTSourceDescriptor {
                       ASTFileSignature Signature)
       : PCHModuleName(std::move(Name)), Path(std::move(Path)),
         ASTFile(std::move(ASTFile)), Signature(Signature) {}
-  ASTSourceDescriptor(const Module &M);
+  ASTSourceDescriptor(Module &M);
 
   std::string getModuleName() const;
   StringRef getPath() const { return Path; }
   StringRef getASTFile() const { return ASTFile; }
   ASTFileSignature getSignature() const { return Signature; }
-  const Module *getModuleOrNull() const { return ClangModule; }
+  Module *getModuleOrNull() const { return ClangModule; }
 };
 
 
diff --git a/clang/include/clang/Basic/OpenMPKinds.def b/clang/include/clang/Basic/OpenMPKinds.def
index bfb41ab105ea5..3cf92ead9560e 100644
--- a/clang/include/clang/Basic/OpenMPKinds.def
+++ b/clang/include/clang/Basic/OpenMPKinds.def
@@ -1112,6 +1112,7 @@ OPENMP_DEPOBJ_CLAUSE(update)
 
 // Modifiers for 'reduction' clause.
 OPENMP_REDUCTION_MODIFIER(default)
+OPENMP_REDUCTION_MODIFIER(inscan)
 
 #undef OPENMP_REDUCTION_MODIFIER
 #undef OPENMP_SCAN_CLAUSE
diff --git a/clang/include/clang/Basic/OpenMPKinds.h b/clang/include/clang/Basic/OpenMPKinds.h
index b567f89b986e0..f12b5545d0c14 100644
--- a/clang/include/clang/Basic/OpenMPKinds.h
+++ b/clang/include/clang/Basic/OpenMPKinds.h
@@ -91,6 +91,10 @@ enum OpenMPMapModifierKind {
   OMPC_MAP_MODIFIER_last
 };
 
+  /// Number of allowed map-type-modifiers.
+static constexpr unsigned NumberOfOMPMapClauseModifiers =
+    OMPC_MAP_MODIFIER_last - OMPC_MAP_MODIFIER_unknown - 1;
+
 /// OpenMP modifier kind for 'to' clause.
 enum OpenMPToModifierKind {
 #define OPENMP_TO_MODIFIER_KIND(Name) \
diff --git a/clang/include/clang/Basic/StmtNodes.td b/clang/include/clang/Basic/StmtNodes.td
index 478179d4131f4..aecf24f890331 100644
--- a/clang/include/clang/Basic/StmtNodes.td
+++ b/clang/include/clang/Basic/StmtNodes.td
@@ -70,6 +70,7 @@ def OffsetOfExpr : StmtNode<Expr>;
 def UnaryExprOrTypeTraitExpr : StmtNode<Expr>;
 def ArraySubscriptExpr : StmtNode<Expr>;
 def OMPArraySectionExpr : StmtNode<Expr>;
+def OMPIteratorExpr : StmtNode<Expr>;
 def CallExpr : StmtNode<Expr>;
 def MemberExpr : StmtNode<Expr>;
 def CastExpr : StmtNode<Expr, 1>;
@@ -81,6 +82,7 @@ def BinaryConditionalOperator : StmtNode<AbstractConditionalOperator>;
 def ImplicitCastExpr : StmtNode<CastExpr>;
 def ExplicitCastExpr : StmtNode<CastExpr, 1>;
 def CStyleCastExpr : StmtNode<ExplicitCastExpr>;
+def OMPArrayShapingExpr : StmtNode<Expr>;
 def CompoundLiteralExpr : StmtNode<Expr>;
 def ExtVectorElementExpr : StmtNode<Expr>;
 def InitListExpr : StmtNode<Expr>;
diff --git a/clang/include/clang/Basic/TargetInfo.h b/clang/include/clang/Basic/TargetInfo.h
index cbfcece1b666d..ab47954040719 100644
--- a/clang/include/clang/Basic/TargetInfo.h
+++ b/clang/include/clang/Basic/TargetInfo.h
@@ -16,7 +16,7 @@
 
 #include "clang/Basic/AddressSpaces.h"
 #include "clang/Basic/LLVM.h"
-#include "clang/Basic/CodeGenOptions.h"
+#include "clang/Basic/LangOptions.h"
 #include "clang/Basic/Specifiers.h"
 #include "clang/Basic/TargetCXXABI.h"
 #include "clang/Basic/TargetOptions.h"
@@ -212,6 +212,8 @@ class TargetInfo : public virtual TransferrableTargetInfo,
 
   unsigned ARMCDECoprocMask : 8;
 
+  unsigned MaxOpenCLWorkGroupSize;
+
   // TargetInfo Constructor.  Default initializes all fields.
   TargetInfo(const llvm::Triple &T);
 
@@ -273,7 +275,14 @@ class TargetInfo : public virtual TransferrableTargetInfo,
     //     void *__overflow_arg_area;
     //     void *__reg_save_area;
     //   } va_list[1];
-    SystemZBuiltinVaList
+    SystemZBuiltinVaList,
+
+    // typedef struct __va_list_tag {
+    //    void *__current_saved_reg_area_pointer;
+    //    void *__saved_reg_area_end_pointer;
+    //    void *__overflow_area_pointer;
+    //} va_list;
+    HexagonBuiltinVaList
   };
 
 protected:
@@ -656,6 +665,8 @@ class TargetInfo : public virtual TransferrableTargetInfo,
   /// types for the given target.
   unsigned getSimdDefaultAlign() const { return SimdDefaultAlign; }
 
+  unsigned getMaxOpenCLWorkGroupSize() const { return MaxOpenCLWorkGroupSize; }
+
   /// Return the alignment (in bits) of the thrown exception object. This is
   /// only meaningful for targets that allocate C++ exceptions in a system
   /// runtime, such as those using the Itanium C++ ABI.
@@ -1127,10 +1138,10 @@ class TargetInfo : public virtual TransferrableTargetInfo,
   }
 
   struct BranchProtectionInfo {
-    CodeGenOptions::SignReturnAddressScope SignReturnAddr =
-        CodeGenOptions::SignReturnAddressScope::None;
-    CodeGenOptions::SignReturnAddressKeyValue SignKey =
-        CodeGenOptions::SignReturnAddressKeyValue::AKey;
+    LangOptions::SignReturnAddressScopeKind SignReturnAddr =
+        LangOptions::SignReturnAddressScopeKind::None;
+    LangOptions::SignReturnAddressKeyKind SignKey =
+        LangOptions::SignReturnAddressKeyKind::AKey;
     bool BranchTargetEnforcement = false;
   };
 
@@ -1206,6 +1217,10 @@ class TargetInfo : public virtual TransferrableTargetInfo,
         "cpu_specific Multiversioning not implemented on this target");
   }
 
+  // Get the cache line size of a given cpu. This method switches over
+  // the given cpu and returns "None" if the CPU is not found.
+  virtual Optional<unsigned> getCPUCacheLineSize() const { return None; }
+
   // Returns maximal number of args passed in registers.
   unsigned getRegParmMax() const {
     assert(RegParmMax < 7 && "RegParmMax value is larger than AST can handle");
diff --git a/clang/include/clang/Basic/arm_cde.td b/clang/include/clang/Basic/arm_cde.td
index e258bf5ee83e5..6a00e669864c8 100644
--- a/clang/include/clang/Basic/arm_cde.td
+++ b/clang/include/clang/Basic/arm_cde.td
@@ -189,6 +189,40 @@ def vcx3qa : FunctionMacro<
   "__arm_vcx3qa_impl((cp), (acc), __arm_vreinterpretq_u8(n), "
                      "__arm_vreinterpretq_u8(m), (imm))">;
 
+class CDEIntrinsicMasked<string irname, dag argsReg, dag imm, dag cgArgs>
+  : CDEIntrinsic<Vector,
+      !con((args imm_coproc:$cp, Vector:$inactive_or_acc),
+           argsReg, imm, (args Predicate:$pred)),
+      !con((CDEIRInt<irname # "_predicated", [Vector,Predicate]>
+            $cp, $inactive_or_acc), cgArgs, (? $imm, $pred))> {
+  let params = T.All;
+  let polymorphicOnly = 1;
+}
+
+def vcx1q_m : CDEIntrinsicMasked<"vcx1q", (args), (args imm_12b:$imm), (?)>;
+def vcx1qa_m : CDEIntrinsicMasked<"vcx1qa", (args), (args imm_12b:$imm), (?)>;
+
+multiclass VCXPredicated<dag argsReg, dag imm, dag cgArgs,
+                         list<string> macroArgs, string macro> {
+  def _m_impl : CDEIntrinsicMasked<NAME, argsReg, imm, cgArgs>;
+  def a_m_impl : CDEIntrinsicMasked<NAME#"a", argsReg, imm, cgArgs>;
+
+  def _m: FunctionMacro<
+    !listconcat(["cp", "inactive"], macroArgs, ["imm", "pred"]),
+    "__arm_"#NAME#"_m_impl((cp), (inactive), "#macro#" (imm), (pred))">;
+  def a_m: FunctionMacro<
+    !listconcat(["cp", "acc"], macroArgs, ["imm", "pred"]),
+    "__arm_"#NAME#"a_m_impl((cp), (acc), "#macro#" (imm), (pred))">;
+}
+
+defm vcx2q :
+  VCXPredicated<(args v16u8:$n), (args imm_7b:$imm), (? $n), ["n"],
+                "__arm_vreinterpretq_u8(n),">;
+defm vcx3q :
+  VCXPredicated<(args v16u8:$n, v16u8:$m), (args imm_4b:$imm), (? $n, $m),
+                ["n", "m"], "__arm_vreinterpretq_u8(n), "
+                            "__arm_vreinterpretq_u8(m),">;
+
 // vreinterpretq intrinsics required by the ACLE CDE specification
 
 foreach desttype = [/* no u8 */ s8, u16, s16, u32, s32, u64, s64, f16, f32] in {
diff --git a/clang/include/clang/CodeGen/CodeGenABITypes.h b/clang/include/clang/CodeGen/CodeGenABITypes.h
index 31f0cea572324..5f4af7fd2a367 100644
--- a/clang/include/clang/CodeGen/CodeGenABITypes.h
+++ b/clang/include/clang/CodeGen/CodeGenABITypes.h
@@ -28,11 +28,12 @@
 #include "clang/CodeGen/CGFunctionInfo.h"
 
 namespace llvm {
-  class DataLayout;
-  class Module;
-  class Function;
-  class FunctionType;
-  class Type;
+class Constant;
+class DataLayout;
+class Module;
+class Function;
+class FunctionType;
+class Type;
 }
 
 namespace clang {
@@ -44,6 +45,7 @@ class CoverageSourceInfo;
 class DiagnosticsEngine;
 class HeaderSearchOptions;
 class ObjCMethodDecl;
+class ObjCProtocolDecl;
 class PreprocessorOptions;
 
 namespace CodeGen {
@@ -137,6 +139,13 @@ llvm::Function *getNonTrivialCStructDestructor(CodeGenModule &CGM,
                                                CharUnits DstAlignment,
                                                bool IsVolatile, QualType QT);
 
+/// Get a pointer to a protocol object for the given declaration, emitting it if
+/// it hasn't already been emitted in this translation unit. Note that the ABI
+/// for emitting a protocol reference in code (e.g. for a protocol expression)
+/// in most runtimes is not as simple as just materializing a pointer to this
+/// object.
+llvm::Constant *emitObjCProtocolObject(CodeGenModule &CGM,
+                                       const ObjCProtocolDecl *p);
 }  // end namespace CodeGen
 }  // end namespace clang
 
diff --git a/clang/include/clang/Config/config.h.cmake b/clang/include/clang/Config/config.h.cmake
index 261b3841b86f8..26e9d5c4eb4d3 100644
--- a/clang/include/clang/Config/config.h.cmake
+++ b/clang/include/clang/Config/config.h.cmake
@@ -35,6 +35,9 @@
 /* Default architecture for OpenMP offloading to Nvidia GPUs. */
 #define CLANG_OPENMP_NVPTX_DEFAULT_ARCH "${CLANG_OPENMP_NVPTX_DEFAULT_ARCH}"
 
+/* Default architecture for SystemZ. */
+#define CLANG_SYSTEMZ_DEFAULT_ARCH "${CLANG_SYSTEMZ_DEFAULT_ARCH}"
+
 /* Multilib suffix for libdir. */
 #define CLANG_LIBDIR_SUFFIX "${CLANG_LIBDIR_SUFFIX}"
 
diff --git a/clang/include/clang/Driver/CC1Options.td b/clang/include/clang/Driver/CC1Options.td
index 2224c152f6266..218404e264095 100644
--- a/clang/include/clang/Driver/CC1Options.td
+++ b/clang/include/clang/Driver/CC1Options.td
@@ -282,8 +282,6 @@ def no_struct_path_tbaa : Flag<["-"], "no-struct-path-tbaa">,
   HelpText<"Turn off struct-path aware Type Based Alias Analysis">;
 def new_struct_path_tbaa : Flag<["-"], "new-struct-path-tbaa">,
   HelpText<"Enable enhanced struct-path aware Type Based Alias Analysis">;
-def masm_verbose : Flag<["-"], "masm-verbose">,
-  HelpText<"Generate verbose assembly output">;
 def mdebug_pass : Separate<["-"], "mdebug-pass">,
   HelpText<"Enable additional debug output">;
 def mframe_pointer_EQ : Joined<["-"], "mframe-pointer=">,
@@ -455,8 +453,6 @@ def fspell_checking_limit : Separate<["-"], "fspell-checking-limit">, MetaVarNam
 def fcaret_diagnostics_max_lines :
   Separate<["-"], "fcaret-diagnostics-max-lines">, MetaVarName<"<N>">,
   HelpText<"Set the maximum number of source lines to show in a caret diagnostic">;
-def fmessage_length : Separate<["-"], "fmessage-length">, MetaVarName<"<N>">,
-  HelpText<"Format message diagnostics so that they fit within N columns or fewer, when possible.">;
 def verify_EQ : CommaJoined<["-"], "verify=">,
   MetaVarName<"<prefixes>">,
   HelpText<"Verify diagnostic output using comment directives that start with"
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 383a37bf57a1a..9b08a624682d2 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -590,13 +590,17 @@ def cuda_include_ptx_EQ : Joined<["--"], "cuda-include-ptx=">, Flags<[DriverOpti
   HelpText<"Include PTX for the following GPU architecture (e.g. sm_35) or 'all'. May be specified more than once.">;
 def no_cuda_include_ptx_EQ : Joined<["--"], "no-cuda-include-ptx=">, Flags<[DriverOption]>,
   HelpText<"Do not include PTX for the following GPU architecture (e.g. sm_35) or 'all'. May be specified more than once.">;
+def offload_arch_EQ : Joined<["--"], "offload-arch=">, Flags<[DriverOption]>,
+  HelpText<"CUDA/HIP offloading device architecture (e.g. sm_35, gfx906).  May be specified more than once.">;
 def cuda_gpu_arch_EQ : Joined<["--"], "cuda-gpu-arch=">, Flags<[DriverOption]>,
-  HelpText<"CUDA GPU architecture (e.g. sm_35).  May be specified more than once.">;
+  Alias<offload_arch_EQ>;
 def hip_link : Flag<["--"], "hip-link">,
   HelpText<"Link clang-offload-bundler bundles for HIP">;
-def no_cuda_gpu_arch_EQ : Joined<["--"], "no-cuda-gpu-arch=">, Flags<[DriverOption]>,
-  HelpText<"Remove GPU architecture (e.g. sm_35) from the list of GPUs to compile for. "
+def no_offload_arch_EQ : Joined<["--"], "no-offload-arch=">, Flags<[DriverOption]>,
+  HelpText<"Remove CUDA/HIP offloading device architecture (e.g. sm_35, gfx906) from the list of devices to compile for. "
            "'all' resets the list to its default value.">;
+def no_cuda_gpu_arch_EQ : Joined<["--"], "no-cuda-gpu-arch=">, Flags<[DriverOption]>,
+  Alias<no_offload_arch_EQ>;
 def cuda_noopt_device_debug : Flag<["--"], "cuda-noopt-device-debug">,
   HelpText<"Enable device-side debug info generation. Disables ptxas optimizations.">;
 def no_cuda_version_check : Flag<["--"], "no-cuda-version-check">,
@@ -914,7 +918,7 @@ def fdiagnostics_hotness_threshold_EQ : Joined<["-"], "fdiagnostics-hotness-thre
     Group<f_Group>, Flags<[CC1Option]>, MetaVarName<"<number>">,
     HelpText<"Prevent optimization remarks from being output if they do not have at least this profile count">;
 def fdiagnostics_show_option : Flag<["-"], "fdiagnostics-show-option">, Group<f_Group>,
-    Flags<[CC1Option]>, HelpText<"Print option name with mappable diagnostics">;
+    HelpText<"Print option name with mappable diagnostics">;
 def fdiagnostics_show_note_include_stack : Flag<["-"], "fdiagnostics-show-note-include-stack">,
     Group<f_Group>,  Flags<[CC1Option]>, HelpText<"Display include stacks for diagnostic notes">;
 def fdiagnostics_format_EQ : Joined<["-"], "fdiagnostics-format=">, Group<f_clang_Group>;
@@ -1264,8 +1268,10 @@ def finline_functions : Flag<["-"], "finline-functions">, Group<f_clang_Group>,
 def finline_hint_functions: Flag<["-"], "finline-hint-functions">, Group<f_clang_Group>, Flags<[CC1Option]>,
   HelpText<"Inline functions which are (explicitly or implicitly) marked inline">;
 def finline : Flag<["-"], "finline">, Group<clang_ignored_f_Group>;
+def fglobal_isel : Flag<["-"], "fglobal-isel">, Group<f_clang_Group>, 
+  HelpText<"Enables the global instruction selector">;
 def fexperimental_isel : Flag<["-"], "fexperimental-isel">, Group<f_clang_Group>,
-  HelpText<"Enables the experimental global instruction selector">;
+  Alias<fglobal_isel>;
 def fexperimental_new_pass_manager : Flag<["-"], "fexperimental-new-pass-manager">,
   Group<f_clang_Group>, Flags<[CC1Option]>,
   HelpText<"Enables an experimental new pass manager in LLVM.">;
@@ -1377,7 +1383,8 @@ def fmacro_backtrace_limit_EQ : Joined<["-"], "fmacro-backtrace-limit=">,
                                 Group<f_Group>, Flags<[DriverOption, CoreOption]>;
 def fmerge_all_constants : Flag<["-"], "fmerge-all-constants">, Group<f_Group>,
   Flags<[CC1Option, CoreOption]>, HelpText<"Allow merging of constants">;
-def fmessage_length_EQ : Joined<["-"], "fmessage-length=">, Group<f_Group>;
+def fmessage_length_EQ : Joined<["-"], "fmessage-length=">, Group<f_Group>, Flags<[CC1Option]>,
+  HelpText<"Format message diagnostics so that they fit within N columns">;
 def fms_extensions : Flag<["-"], "fms-extensions">, Group<f_Group>, Flags<[CC1Option, CoreOption]>,
   HelpText<"Accept some non-standard constructs supported by the Microsoft compiler">;
 def fms_compatibility : Flag<["-"], "fms-compatibility">, Group<f_Group>, Flags<[CC1Option, CoreOption]>,
@@ -1527,7 +1534,7 @@ def fno_cxx_modules : Flag <["-"], "fno-cxx-modules">, Group<f_Group>,
 def fno_diagnostics_fixit_info : Flag<["-"], "fno-diagnostics-fixit-info">, Group<f_Group>,
   Flags<[CC1Option]>, HelpText<"Do not include fixit information in diagnostics">;
 def fno_diagnostics_show_hotness : Flag<["-"], "fno-diagnostics-show-hotness">, Group<f_Group>;
-def fno_diagnostics_show_option : Flag<["-"], "fno-diagnostics-show-option">, Group<f_Group>;
+def fno_diagnostics_show_option : Flag<["-"], "fno-diagnostics-show-option">, Group<f_Group>, Flags<[CC1Option]>;
 def fno_diagnostics_show_note_include_stack : Flag<["-"], "fno-diagnostics-show-note-include-stack">,
     Flags<[CC1Option]>, Group<f_Group>;
 def fdigraphs : Flag<["-"], "fdigraphs">, Group<f_Group>, Flags<[CC1Option]>,
@@ -1545,8 +1552,10 @@ def fno_exceptions : Flag<["-"], "fno-exceptions">, Group<f_Group>;
 def fno_gnu_keywords : Flag<["-"], "fno-gnu-keywords">, Group<f_Group>, Flags<[CC1Option]>;
 def fno_inline_functions : Flag<["-"], "fno-inline-functions">, Group<f_clang_Group>, Flags<[CC1Option]>;
 def fno_inline : Flag<["-"], "fno-inline">, Group<f_clang_Group>, Flags<[CC1Option]>;
+def fno_global_isel : Flag<["-"], "fno-global-isel">, Group<f_clang_Group>,
+  HelpText<"Disables the global instruction selector">;
 def fno_experimental_isel : Flag<["-"], "fno-experimental-isel">, Group<f_clang_Group>,
-  HelpText<"Disables the experimental global instruction selector">;
+  Alias<fno_global_isel>;
 def fno_experimental_new_pass_manager : Flag<["-"], "fno-experimental-new-pass-manager">,
   Group<f_clang_Group>, Flags<[CC1Option]>,
   HelpText<"Disables an experimental new pass manager in LLVM.">;
@@ -1618,7 +1627,7 @@ def fno_register_global_dtors_with_atexit : Flag<["-"], "fno-register-global-dto
   HelpText<"Don't use atexit or __cxa_atexit to register global destructors">;
 def fno_unit_at_a_time : Flag<["-"], "fno-unit-at-a-time">, Group<f_Group>;
 def fno_unwind_tables : Flag<["-"], "fno-unwind-tables">, Group<f_Group>;
-def fno_verbose_asm : Flag<["-"], "fno-verbose-asm">, Group<f_Group>;
+def fno_verbose_asm : Flag<["-"], "fno-verbose-asm">, Group<f_Group>, Flags<[CC1Option]>;
 def fno_working_directory : Flag<["-"], "fno-working-directory">, Group<f_Group>;
 def fno_wrapv : Flag<["-"], "fno-wrapv">, Group<f_Group>;
 def fno_zero_initialized_in_bss : Flag<["-"], "fno-zero-initialized-in-bss">, Group<f_Group>;
@@ -1975,7 +1984,8 @@ def fuse_init_array : Flag<["-"], "fuse-init-array">, Group<f_Group>,
 def fno_use_init_array : Flag<["-"], "fno-use-init-array">, Group<f_Group>, Flags<[CC1Option]>,
   HelpText<"Don't use .init_array instead of .ctors">;
 def fno_var_tracking : Flag<["-"], "fno-var-tracking">, Group<clang_ignored_f_Group>;
-def fverbose_asm : Flag<["-"], "fverbose-asm">, Group<f_Group>;
+def fverbose_asm : Flag<["-"], "fverbose-asm">, Group<f_Group>,
+  HelpText<"Generate verbose assembly output">;
 def dA : Flag<["-"], "dA">, Alias<fverbose_asm>;
 def fvisibility_EQ : Joined<["-"], "fvisibility=">, Group<f_Group>,
   HelpText<"Set the default symbol visibility for all global declarations">, Values<"hidden,default">;
@@ -2356,6 +2366,14 @@ def mspeculative_load_hardening : Flag<["-"], "mspeculative-load-hardening">,
   Group<m_Group>, Flags<[CoreOption,CC1Option]>;
 def mno_speculative_load_hardening : Flag<["-"], "mno-speculative-load-hardening">,
   Group<m_Group>, Flags<[CoreOption]>;
+def mlvi_hardening : Flag<["-"], "mlvi-hardening">, Group<m_Group>, Flags<[CoreOption,DriverOption]>,
+  HelpText<"Enable all mitigations for Load Value Injection (LVI)">;
+def mno_lvi_hardening : Flag<["-"], "mno-lvi-hardening">, Group<m_Group>, Flags<[CoreOption,DriverOption]>,
+  HelpText<"Disable mitigations for Load Value Injection (LVI)">;
+def mlvi_cfi : Flag<["-"], "mlvi-cfi">, Group<m_Group>, Flags<[CoreOption,DriverOption]>,
+  HelpText<"Enable only control-flow mitigations for Load Value Injection (LVI)">;
+def mno_lvi_cfi : Flag<["-"], "mno-lvi-cfi">, Group<m_Group>, Flags<[CoreOption,DriverOption]>,
+  HelpText<"Disable control-flow mitigations for Load Value Injection (LVI)">;
 
 def mrelax : Flag<["-"], "mrelax">, Group<m_riscv_Features_Group>,
   HelpText<"Enable linker relaxation">;
@@ -3265,6 +3283,8 @@ def mrdseed : Flag<["-"], "mrdseed">, Group<m_x86_Features_Group>;
 def mno_rdseed : Flag<["-"], "mno-rdseed">, Group<m_x86_Features_Group>;
 def msahf : Flag<["-"], "msahf">, Group<m_x86_Features_Group>;
 def mno_sahf : Flag<["-"], "mno-sahf">, Group<m_x86_Features_Group>;
+def mserialize : Flag<["-"], "mserialize">, Group<m_x86_Features_Group>;
+def mno_serialize : Flag<["-"], "mno-serialize">, Group<m_x86_Features_Group>;
 def msgx : Flag<["-"], "msgx">, Group<m_x86_Features_Group>;
 def mno_sgx : Flag<["-"], "mno-sgx">, Group<m_x86_Features_Group>;
 def msha : Flag<["-"], "msha">, Group<m_x86_Features_Group>;
diff --git a/clang/include/clang/Frontend/CompilerInvocation.h b/clang/include/clang/Frontend/CompilerInvocation.h
index f3253d5b40e3c..25476f78a6a00 100644
--- a/clang/include/clang/Frontend/CompilerInvocation.h
+++ b/clang/include/clang/Frontend/CompilerInvocation.h
@@ -59,8 +59,7 @@ class TargetOptions;
 /// report the error(s).
 bool ParseDiagnosticArgs(DiagnosticOptions &Opts, llvm::opt::ArgList &Args,
                          DiagnosticsEngine *Diags = nullptr,
-                         bool DefaultDiagColor = true,
-                         bool DefaultShowOpt = true);
+                         bool DefaultDiagColor = true);
 
 class CompilerInvocationBase {
 public:
diff --git a/clang/include/clang/Frontend/FrontendAction.h b/clang/include/clang/Frontend/FrontendAction.h
index e994e24cf5afa..c9f9f080c1413 100644
--- a/clang/include/clang/Frontend/FrontendAction.h
+++ b/clang/include/clang/Frontend/FrontendAction.h
@@ -312,6 +312,7 @@ class WrapperFrontendAction : public FrontendAction {
   bool BeginSourceFileAction(CompilerInstance &CI) override;
   void ExecuteAction() override;
   void EndSourceFileAction() override;
+  bool shouldEraseOutputFiles() override;
 
 public:
   /// Construct a WrapperFrontendAction from an existing action, taking
diff --git a/clang/include/clang/Frontend/FrontendActions.h b/clang/include/clang/Frontend/FrontendActions.h
index 89ac20075fa45..9ca2bfda21389 100644
--- a/clang/include/clang/Frontend/FrontendActions.h
+++ b/clang/include/clang/Frontend/FrontendActions.h
@@ -119,17 +119,13 @@ class GenerateModuleAction : public ASTFrontendAction {
   bool hasASTFileSupport() const override { return false; }
 };
 
-class GenerateInterfaceStubAction : public ASTFrontendAction {
-protected:
-  TranslationUnitKind getTranslationUnitKind() override { return TU_Module; }
-
-  bool hasASTFileSupport() const override { return false; }
-};
-
-class GenerateInterfaceIfsExpV1Action : public GenerateInterfaceStubAction {
+class GenerateInterfaceStubsAction : public ASTFrontendAction {
 protected:
   std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
                                                  StringRef InFile) override;
+
+  TranslationUnitKind getTranslationUnitKind() override { return TU_Module; }
+  bool hasASTFileSupport() const override { return false; }
 };
 
 class GenerateModuleFromModuleMapAction : public GenerateModuleAction {
diff --git a/clang/include/clang/Frontend/FrontendOptions.h b/clang/include/clang/Frontend/FrontendOptions.h
index 66fec6436a404..6069b5eea2650 100644
--- a/clang/include/clang/Frontend/FrontendOptions.h
+++ b/clang/include/clang/Frontend/FrontendOptions.h
@@ -90,7 +90,7 @@ enum ActionKind {
   GeneratePCH,
 
   /// Generate Interface Stub Files.
-  GenerateInterfaceIfsExpV1,
+  GenerateInterfaceStubs,
 
   /// Only execute frontend initialization.
   InitOnly,
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index b6f65227ab2eb..8abcc73e83d39 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -13,7 +13,6 @@
 #ifndef LLVM_CLANG_PARSE_PARSER_H
 #define LLVM_CLANG_PARSE_PARSER_H
 
-#include "clang/AST/OpenMPClause.h"
 #include "clang/AST/Availability.h"
 #include "clang/Basic/BitmaskEnum.h"
 #include "clang/Basic/OpenMPKinds.h"
@@ -24,6 +23,7 @@
 #include "clang/Sema/DeclSpec.h"
 #include "clang/Sema/Sema.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/Frontend/OpenMP/OMPContext.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/PrettyStackTrace.h"
 #include "llvm/Support/SaveAndRestore.h"
@@ -49,6 +49,10 @@ namespace clang {
   class OMPClause;
   class ObjCTypeParamList;
   class ObjCTypeParameter;
+  struct OMPTraitProperty;
+  struct OMPTraitSelector;
+  struct OMPTraitSet;
+  class OMPTraitInfo;
 
 /// Parser - This implements a parser for the C family of languages.  After
 /// parsing units of the grammar, productions are invoked to handle whatever has
@@ -763,13 +767,17 @@ class Parser : public CodeCompletionHandler {
   }
 
   /// getTypeAnnotation - Read a parsed type out of an annotation token.
-  static ParsedType getTypeAnnotation(const Token &Tok) {
+  static TypeResult getTypeAnnotation(const Token &Tok) {
+    if (!Tok.getAnnotationValue())
+      return TypeError();
     return ParsedType::getFromOpaquePtr(Tok.getAnnotationValue());
   }
 
 private:
-  static void setTypeAnnotation(Token &Tok, ParsedType T) {
-    Tok.setAnnotationValue(T.getAsOpaquePtr());
+  static void setTypeAnnotation(Token &Tok, TypeResult T) {
+    assert((T.isInvalid() || T.get()) &&
+           "produced a valid-but-null type annotation?");
+    Tok.setAnnotationValue(T.isInvalid() ? nullptr : T.get().getAsOpaquePtr());
   }
 
   static NamedDecl *getNonTypeAnnotation(const Token &Tok) {
@@ -2940,45 +2948,66 @@ class Parser : public CodeCompletionHandler {
 
   /// Parse a property kind into \p TIProperty for the selector set \p Set and
   /// selector \p Selector.
-  void parseOMPTraitPropertyKind(OMPTraitInfo::OMPTraitProperty &TIProperty,
+  void parseOMPTraitPropertyKind(OMPTraitProperty &TIProperty,
                                  llvm::omp::TraitSet Set,
                                  llvm::omp::TraitSelector Selector,
                                  llvm::StringMap<SourceLocation> &Seen);
 
   /// Parse a selector kind into \p TISelector for the selector set \p Set.
-  void parseOMPTraitSelectorKind(OMPTraitInfo::OMPTraitSelector &TISelector,
+  void parseOMPTraitSelectorKind(OMPTraitSelector &TISelector,
                                  llvm::omp::TraitSet Set,
                                  llvm::StringMap<SourceLocation> &Seen);
 
   /// Parse a selector set kind into \p TISet.
-  void parseOMPTraitSetKind(OMPTraitInfo::OMPTraitSet &TISet,
+  void parseOMPTraitSetKind(OMPTraitSet &TISet,
                             llvm::StringMap<SourceLocation> &Seen);
 
   /// Parses an OpenMP context property.
-  void parseOMPContextProperty(OMPTraitInfo::OMPTraitSelector &TISelector,
+  void parseOMPContextProperty(OMPTraitSelector &TISelector,
                                llvm::omp::TraitSet Set,
                                llvm::StringMap<SourceLocation> &Seen);
 
   /// Parses an OpenMP context selector.
-  void parseOMPContextSelector(OMPTraitInfo::OMPTraitSelector &TISelector,
+  void parseOMPContextSelector(OMPTraitSelector &TISelector,
                                llvm::omp::TraitSet Set,
                                llvm::StringMap<SourceLocation> &SeenSelectors);
 
   /// Parses an OpenMP context selector set.
-  void parseOMPContextSelectorSet(OMPTraitInfo::OMPTraitSet &TISet,
+  void parseOMPContextSelectorSet(OMPTraitSet &TISet,
                                   llvm::StringMap<SourceLocation> &SeenSets);
 
   /// Parses OpenMP context selectors.
   bool parseOMPContextSelectors(SourceLocation Loc, OMPTraitInfo &TI);
 
+  /// Parse a `match` clause for an '#pragma omp declare variant'. Return true
+  /// if there was an error.
+  bool parseOMPDeclareVariantMatchClause(SourceLocation Loc, OMPTraitInfo &TI);
+
   /// Parse clauses for '#pragma omp declare variant'.
   void ParseOMPDeclareVariantClauses(DeclGroupPtrTy Ptr, CachedTokens &Toks,
                                      SourceLocation Loc);
+
   /// Parse clauses for '#pragma omp declare target'.
   DeclGroupPtrTy ParseOMPDeclareTargetClauses();
   /// Parse '#pragma omp end declare target'.
   void ParseOMPEndDeclareTargetDirective(OpenMPDirectiveKind DKind,
                                          SourceLocation Loc);
+
+  /// Skip tokens until a `annot_pragma_openmp_end` was found. Emit a warning if
+  /// it is not the current token.
+  void skipUntilPragmaOpenMPEnd(OpenMPDirectiveKind DKind);
+
+  /// Check the \p FoundKind against the \p ExpectedKind, if not issue an error
+  /// that the "end" matching the "begin" directive of kind \p BeginKind was not
+  /// found. Finally, if the expected kind was found or if \p SkipUntilOpenMPEnd
+  /// is set, skip ahead using the helper `skipUntilPragmaOpenMPEnd`.
+  void parseOMPEndDirective(OpenMPDirectiveKind BeginKind,
+                            OpenMPDirectiveKind ExpectedKind,
+                            OpenMPDirectiveKind FoundKind,
+                            SourceLocation MatchingLoc,
+                            SourceLocation FoundLoc,
+                            bool SkipUntilOpenMPEnd);
+
   /// Parses declarative OpenMP directives.
   DeclGroupPtrTy ParseOpenMPDeclarativeDirectiveWithExtDecl(
       AccessSpecifier &AS, ParsedAttributesWithRange &Attrs,
@@ -2997,6 +3026,10 @@ class Parser : public CodeCompletionHandler {
                                              DeclarationName &Name,
                                              AccessSpecifier AS = AS_none);
 
+  /// Tries to parse cast part of OpenMP array shaping operation:
+  /// '[' expression ']' { '[' expression ']' } ')'.
+  bool tryParseOpenMPArrayShapingCastPart();
+
   /// Parses simple list of variables.
   ///
   /// \param Kind Kind of the directive.
@@ -3065,6 +3098,11 @@ class Parser : public CodeCompletionHandler {
   OMPClause *ParseOpenMPVarListClause(OpenMPDirectiveKind DKind,
                                       OpenMPClauseKind Kind, bool ParseOnly);
 
+  /// Parses and creates OpenMP 5.0 iterators expression:
+  /// <iterators> = 'iterator' '(' { [ <iterator-type> ] identifier =
+  /// <range-specification> }+ ')'
+  ExprResult ParseOpenMPIteratorsExpr();
+
 public:
   /// Parses simple expression in parens for single-expression clauses of OpenMP
   /// constructs.
@@ -3074,16 +3112,16 @@ class Parser : public CodeCompletionHandler {
 
   /// Data used for parsing list of variables in OpenMP clauses.
   struct OpenMPVarListDataTy {
-    Expr *TailExpr = nullptr;
+    Expr *DepModOrTailExpr = nullptr;
     SourceLocation ColonLoc;
     SourceLocation RLoc;
     CXXScopeSpec ReductionOrMapperIdScopeSpec;
     DeclarationNameInfo ReductionOrMapperId;
     int ExtraModifier = -1; ///< Additional modifier for linear, map, depend or
                             ///< lastprivate clause.
-    SmallVector<OpenMPMapModifierKind, OMPMapClause::NumberOfModifiers>
+    SmallVector<OpenMPMapModifierKind, NumberOfOMPMapClauseModifiers>
     MapTypeModifiers;
-    SmallVector<SourceLocation, OMPMapClause::NumberOfModifiers>
+    SmallVector<SourceLocation, NumberOfOMPMapClauseModifiers>
     MapTypeModifiersLoc;
     bool IsMapTypeImplicit = false;
     SourceLocation ExtraModifierLoc;
@@ -3147,7 +3185,8 @@ class Parser : public CodeCompletionHandler {
   // C++ 14.3: Template arguments [temp.arg]
   typedef SmallVector<ParsedTemplateArgument, 16> TemplateArgList;
 
-  bool ParseGreaterThanInTemplateList(SourceLocation &RAngleLoc,
+  bool ParseGreaterThanInTemplateList(SourceLocation LAngleLoc,
+                                      SourceLocation &RAngleLoc,
                                       bool ConsumeLastToken,
                                       bool ObjCGenericList);
   bool ParseTemplateIdAfterTemplateName(bool ConsumeLastToken,
diff --git a/clang/include/clang/Sema/DeclSpec.h b/clang/include/clang/Sema/DeclSpec.h
index 5a6bedcd6f59d..78010b7bb46e6 100644
--- a/clang/include/clang/Sema/DeclSpec.h
+++ b/clang/include/clang/Sema/DeclSpec.h
@@ -667,6 +667,13 @@ class DeclSpec {
   bool SetTypeSpecType(TST T, SourceLocation Loc, const char *&PrevSpec,
                        unsigned &DiagID, ParsedType Rep,
                        const PrintingPolicy &Policy);
+  bool SetTypeSpecType(TST T, SourceLocation Loc, const char *&PrevSpec,
+                       unsigned &DiagID, TypeResult Rep,
+                       const PrintingPolicy &Policy) {
+    if (Rep.isInvalid())
+      return SetTypeSpecError();
+    return SetTypeSpecType(T, Loc, PrevSpec, DiagID, Rep.get(), Policy);
+  }
   bool SetTypeSpecType(TST T, SourceLocation Loc, const char *&PrevSpec,
                        unsigned &DiagID, Decl *Rep, bool Owned,
                        const PrintingPolicy &Policy);
diff --git a/clang/include/clang/Sema/Ownership.h b/clang/include/clang/Sema/Ownership.h
index 4de3397d51b45..7c7b1d35c9fd5 100644
--- a/clang/include/clang/Sema/Ownership.h
+++ b/clang/include/clang/Sema/Ownership.h
@@ -278,6 +278,7 @@ namespace clang {
 
   inline ExprResult ExprError() { return ExprResult(true); }
   inline StmtResult StmtError() { return StmtResult(true); }
+  inline TypeResult TypeError() { return TypeResult(true); }
 
   inline ExprResult ExprError(const DiagnosticBuilder&) { return ExprError(); }
   inline StmtResult StmtError(const DiagnosticBuilder&) { return StmtError(); }
diff --git a/clang/include/clang/Sema/ParsedAttr.h b/clang/include/clang/Sema/ParsedAttr.h
index e1e63ed87896b..98671bf06ab12 100644
--- a/clang/include/clang/Sema/ParsedAttr.h
+++ b/clang/include/clang/Sema/ParsedAttr.h
@@ -63,9 +63,9 @@ struct ParsedAttrInfo {
   /// The syntaxes supported by this attribute and how they're spelled.
   struct Spelling {
     AttributeCommonInfo::Syntax Syntax;
-    std::string NormalizedFullName;
+    const char *NormalizedFullName;
   };
-  std::vector<Spelling> Spellings;
+  ArrayRef<Spelling> Spellings;
 
   ParsedAttrInfo(AttributeCommonInfo::Kind AttrKind =
                      AttributeCommonInfo::NoSemaHandlerAttribute)
diff --git a/clang/include/clang/Sema/ParsedTemplate.h b/clang/include/clang/Sema/ParsedTemplate.h
index 82d00494b0d6b..f0245b93c7eb2 100644
--- a/clang/include/clang/Sema/ParsedTemplate.h
+++ b/clang/include/clang/Sema/ParsedTemplate.h
@@ -169,7 +169,9 @@ namespace clang {
     /// template-name.
     ParsedTemplateTy Template;
 
-    /// The kind of template that Template refers to.
+    /// The kind of template that Template refers to. If this is
+    /// TNK_Non_template, an error was encountered and diagnosed
+    /// when parsing or looking up the template name.
     TemplateNameKind Kind;
 
     /// The location of the '<' before the template argument
@@ -183,6 +185,10 @@ namespace clang {
     /// NumArgs - The number of template arguments.
     unsigned NumArgs;
 
+    /// Whether an error was encountered in the template arguments.
+    /// If so, NumArgs and the trailing arguments are best-effort.
+    bool ArgsInvalid;
+
     /// Retrieves a pointer to the template arguments
     ParsedTemplateArgument *getTemplateArgs() {
       return getTrailingObjects<ParsedTemplateArgument>();
@@ -195,13 +201,13 @@ namespace clang {
            IdentifierInfo *Name, OverloadedOperatorKind OperatorKind,
            ParsedTemplateTy OpaqueTemplateName, TemplateNameKind TemplateKind,
            SourceLocation LAngleLoc, SourceLocation RAngleLoc,
-           ArrayRef<ParsedTemplateArgument> TemplateArgs,
+           ArrayRef<ParsedTemplateArgument> TemplateArgs, bool ArgsInvalid,
            SmallVectorImpl<TemplateIdAnnotation *> &CleanupList) {
       TemplateIdAnnotation *TemplateId = new (llvm::safe_malloc(
           totalSizeToAlloc<ParsedTemplateArgument>(TemplateArgs.size())))
           TemplateIdAnnotation(TemplateKWLoc, TemplateNameLoc, Name,
                                OperatorKind, OpaqueTemplateName, TemplateKind,
-                               LAngleLoc, RAngleLoc, TemplateArgs);
+                               LAngleLoc, RAngleLoc, TemplateArgs, ArgsInvalid);
       CleanupList.push_back(TemplateId);
       return TemplateId;
     }
@@ -213,6 +219,20 @@ namespace clang {
       this->~TemplateIdAnnotation();
       free(this);
     }
+
+    /// Determine whether this might be a type template.
+    bool mightBeType() const {
+      return Kind == TNK_Non_template ||
+             Kind == TNK_Type_template ||
+             Kind == TNK_Dependent_template_name ||
+             Kind == TNK_Undeclared_template;
+    }
+
+    bool hasInvalidName() const { return Kind == TNK_Non_template; }
+    bool hasInvalidArgs() const { return ArgsInvalid; }
+
+    bool isInvalid() const { return hasInvalidName() || hasInvalidArgs(); }
+
   private:
     TemplateIdAnnotation(const TemplateIdAnnotation &) = delete;
 
@@ -222,11 +242,12 @@ namespace clang {
                          ParsedTemplateTy OpaqueTemplateName,
                          TemplateNameKind TemplateKind,
                          SourceLocation LAngleLoc, SourceLocation RAngleLoc,
-                         ArrayRef<ParsedTemplateArgument> TemplateArgs) noexcept
+                         ArrayRef<ParsedTemplateArgument> TemplateArgs,
+                         bool ArgsInvalid) noexcept
         : TemplateKWLoc(TemplateKWLoc), TemplateNameLoc(TemplateNameLoc),
           Name(Name), Operator(OperatorKind), Template(OpaqueTemplateName),
           Kind(TemplateKind), LAngleLoc(LAngleLoc), RAngleLoc(RAngleLoc),
-          NumArgs(TemplateArgs.size()) {
+          NumArgs(TemplateArgs.size()), ArgsInvalid(ArgsInvalid) {
 
       std::uninitialized_copy(TemplateArgs.begin(), TemplateArgs.end(),
                               getTemplateArgs());
diff --git a/clang/include/clang/Sema/Scope.h b/clang/include/clang/Sema/Scope.h
index 6133425a42a62..169ca175eed22 100644
--- a/clang/include/clang/Sema/Scope.h
+++ b/clang/include/clang/Sema/Scope.h
@@ -320,9 +320,7 @@ class Scope {
 
   /// isDeclScope - Return true if this is the scope that the specified decl is
   /// declared in.
-  bool isDeclScope(Decl *D) {
-    return DeclsInScope.count(D) != 0;
-  }
+  bool isDeclScope(const Decl *D) const { return DeclsInScope.count(D) != 0; }
 
   DeclContext *getEntity() const { return Entity; }
   void setEntity(DeclContext *E) { Entity = E; }
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 17f90403b0ddb..6a4940539d3fc 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -25,6 +25,7 @@
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/ExprConcepts.h"
 #include "clang/AST/ExprObjC.h"
+#include "clang/AST/ExprOpenMP.h"
 #include "clang/AST/ExternalASTSource.h"
 #include "clang/AST/LocInfoType.h"
 #include "clang/AST/MangleNumberingContext.h"
@@ -58,6 +59,7 @@
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallBitVector.h"
+#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/TinyPtrVector.h"
@@ -4976,7 +4978,7 @@ class Sema final {
                                        SourceLocation RParen, ParsedType Ty);
   ExprResult ActOnUniqueStableNameExpr(SourceLocation OpLoc,
                                        SourceLocation LParen,
-                                       SourceLocation RParen, Expr *Operand);
+                                       SourceLocation RParen, Expr *E);
 
   bool CheckLoopHintExpr(Expr *E, SourceLocation Loc);
 
@@ -5051,6 +5053,25 @@ class Sema final {
   ExprResult ActOnOMPArraySectionExpr(Expr *Base, SourceLocation LBLoc,
                                       Expr *LowerBound, SourceLocation ColonLoc,
                                       Expr *Length, SourceLocation RBLoc);
+  ExprResult ActOnOMPArrayShapingExpr(Expr *Base, SourceLocation LParenLoc,
+                                      SourceLocation RParenLoc,
+                                      ArrayRef<Expr *> Dims,
+                                      ArrayRef<SourceRange> Brackets);
+
+  /// Data structure for iterator expression.
+  struct OMPIteratorData {
+    IdentifierInfo *DeclIdent = nullptr;
+    SourceLocation DeclIdentLoc;
+    ParsedType Type;
+    OMPIteratorExpr::IteratorRange Range;
+    SourceLocation AssignLoc;
+    SourceLocation ColonLoc;
+    SourceLocation SecColonLoc;
+  };
+
+  ExprResult ActOnOMPIteratorExpr(Scope *S, SourceLocation IteratorKwLoc,
+                                  SourceLocation LLoc, SourceLocation RLoc,
+                                  ArrayRef<OMPIteratorData> Data);
 
   // This struct is for use by ActOnMemberAccess to allow
   // BuildMemberReferenceExpr to be able to reinvoke ActOnMemberAccess after
@@ -7151,6 +7172,27 @@ class Sema final {
                                    bool AllowFunctionTemplates = true,
                                    bool AllowDependent = true);
 
+  enum TemplateNameIsRequiredTag { TemplateNameIsRequired };
+  /// Whether and why a template name is required in this lookup.
+  class RequiredTemplateKind {
+  public:
+    /// Template name is required if TemplateKWLoc is valid.
+    RequiredTemplateKind(SourceLocation TemplateKWLoc = SourceLocation())
+        : TemplateKW(TemplateKWLoc) {}
+    /// Template name is unconditionally required.
+    RequiredTemplateKind(TemplateNameIsRequiredTag) : TemplateKW() {}
+
+    SourceLocation getTemplateKeywordLoc() const {
+      return TemplateKW.getValueOr(SourceLocation());
+    }
+    bool hasTemplateKeyword() const { return getTemplateKeywordLoc().isValid(); }
+    bool isRequired() const { return TemplateKW != SourceLocation(); }
+    explicit operator bool() const { return isRequired(); }
+
+  private:
+    llvm::Optional<SourceLocation> TemplateKW;
+  };
+
   enum class AssumedTemplateKind {
     /// This is not assumed to be a template name.
     None,
@@ -7160,12 +7202,11 @@ class Sema final {
     /// functions (but no function templates).
     FoundFunctions,
   };
-  bool LookupTemplateName(LookupResult &R, Scope *S, CXXScopeSpec &SS,
-                          QualType ObjectType, bool EnteringContext,
-                          bool &MemberOfUnknownSpecialization,
-                          SourceLocation TemplateKWLoc = SourceLocation(),
-                          AssumedTemplateKind *ATK = nullptr,
-                          bool Disambiguation = false);
+  bool LookupTemplateName(
+      LookupResult &R, Scope *S, CXXScopeSpec &SS, QualType ObjectType,
+      bool EnteringContext, bool &MemberOfUnknownSpecialization,
+      RequiredTemplateKind RequiredTemplate = SourceLocation(),
+      AssumedTemplateKind *ATK = nullptr, bool AllowTypoCorrection = true);
 
   TemplateNameKind isTemplateName(Scope *S,
                                   CXXScopeSpec &SS,
@@ -7378,7 +7419,7 @@ class Sema final {
                                const DeclarationNameInfo &NameInfo,
                                const TemplateArgumentListInfo *TemplateArgs);
 
-  TemplateNameKind ActOnDependentTemplateName(
+  TemplateNameKind ActOnTemplateName(
       Scope *S, CXXScopeSpec &SS, SourceLocation TemplateKWLoc,
       const UnqualifiedId &Name, ParsedType ObjectType, bool EnteringContext,
       TemplateTy &Template, bool AllowInjectedClassName = false);
@@ -9979,14 +10020,54 @@ class Sema final {
                                      MapT &Map, unsigned Selector = 0,
                                      SourceRange SrcRange = SourceRange());
 
-  /// Marks all the functions that might be required for the currently active
-  /// OpenMP context.
-  void markOpenMPDeclareVariantFuncsReferenced(SourceLocation Loc,
-                                               FunctionDecl *Func,
-                                               bool MightBeOdrUse);
+  /// Helper to keep information about the current `omp begin/end declare
+  /// variant` nesting.
+  struct OMPDeclareVariantScope {
+    /// The associated OpenMP context selector.
+    OMPTraitInfo *TI;
+
+    /// The associated OpenMP context selector mangling.
+    std::string NameSuffix;
+
+    OMPDeclareVariantScope(OMPTraitInfo &TI);
+  };
+
+  /// The current `omp begin/end declare variant` scopes.
+  SmallVector<OMPDeclareVariantScope, 4> OMPDeclareVariantScopes;
+
+  /// The declarator \p D defines a function in the scope \p S which is nested
+  /// in an `omp begin/end declare variant` scope. In this method we create a
+  /// declaration for \p D and rename \p D according to the OpenMP context
+  /// selector of the surrounding scope.
+  FunctionDecl *
+  ActOnStartOfFunctionDefinitionInOpenMPDeclareVariantScope(Scope *S,
+                                                            Declarator &D);
+
+  /// Register \p FD as specialization of \p BaseFD in the current `omp
+  /// begin/end declare variant` scope.
+  void ActOnFinishedFunctionDefinitionInOpenMPDeclareVariantScope(
+      FunctionDecl *FD, FunctionDecl *BaseFD);
 
 public:
-  /// Struct to store the context selectors info for declare variant directive.
+
+  /// Can we exit a scope at the moment.
+  bool isInOpenMPDeclareVariantScope() {
+    return !OMPDeclareVariantScopes.empty();
+  }
+
+  /// Given the potential call expression \p Call, determine if there is a
+  /// specialization via the OpenMP declare variant mechanism available. If
+  /// there is, return the specialized call expression, otherwise return the
+  /// original \p Call.
+  ExprResult ActOnOpenMPCall(Sema &S, ExprResult Call, Scope *Scope,
+                             SourceLocation LParenLoc, MultiExprArg ArgExprs,
+                             SourceLocation RParenLoc, Expr *ExecConfig);
+
+  /// Handle a `omp begin declare variant`.
+  void ActOnOpenMPBeginDeclareVariant(SourceLocation Loc, OMPTraitInfo &TI);
+
+  /// Handle a `omp end declare variant`.
+  void ActOnOpenMPEndDeclareVariant();
 
   /// Checks if the variant/multiversion functions are compatible.
   bool areMultiversionVariantFunctionsCompatible(
@@ -10674,7 +10755,7 @@ class Sema final {
       SourceLocation StartLoc, SourceLocation LParenLoc, SourceLocation EndLoc);
 
   OMPClause *ActOnOpenMPVarListClause(
-      OpenMPClauseKind Kind, ArrayRef<Expr *> Vars, Expr *TailExpr,
+      OpenMPClauseKind Kind, ArrayRef<Expr *> Vars, Expr *DepModOrTailExpr,
       const OMPVarListLocTy &Locs, SourceLocation ColonLoc,
       CXXScopeSpec &ReductionOrMapperIdScopeSpec,
       DeclarationNameInfo &ReductionOrMapperId, int ExtraModifier,
@@ -10772,10 +10853,10 @@ class Sema final {
                                      SourceLocation EndLoc);
   /// Called on well-formed 'depend' clause.
   OMPClause *
-  ActOnOpenMPDependClause(OpenMPDependClauseKind DepKind, SourceLocation DepLoc,
-                          SourceLocation ColonLoc, ArrayRef<Expr *> VarList,
-                          SourceLocation StartLoc, SourceLocation LParenLoc,
-                          SourceLocation EndLoc);
+  ActOnOpenMPDependClause(Expr *DepModifier, OpenMPDependClauseKind DepKind,
+                          SourceLocation DepLoc, SourceLocation ColonLoc,
+                          ArrayRef<Expr *> VarList, SourceLocation StartLoc,
+                          SourceLocation LParenLoc, SourceLocation EndLoc);
   /// Called on well-formed 'device' clause.
   OMPClause *ActOnOpenMPDeviceClause(OpenMPDeviceClauseModifier Modifier,
                                      Expr *Device, SourceLocation StartLoc,
diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h
index f185c1a16834b..e6e9c8570cc8b 100644
--- a/clang/include/clang/Serialization/ASTBitCodes.h
+++ b/clang/include/clang/Serialization/ASTBitCodes.h
@@ -1016,6 +1016,12 @@ namespace serialization {
       /// \brief The '_Sat unsigned long _Fract' type
       PREDEF_TYPE_SAT_ULONG_FRACT_ID = 69,
 
+      /// The placeholder type for OpenMP array shaping operation.
+      PREDEF_TYPE_OMP_ARRAY_SHAPING = 70,
+
+      /// The placeholder type for OpenMP iterator expression.
+      PREDEF_TYPE_OMP_ITERATOR = 71,
+
       /// OpenCL image types with auto numeration
 #define IMAGE_TYPE(ImgType, Id, SingletonId, Access, Suffix) \
       PREDEF_TYPE_##Id##_ID,
@@ -1868,6 +1874,8 @@ namespace serialization {
       STMT_OMP_TARGET_TEAMS_DISTRIBUTE_PARALLEL_FOR_SIMD_DIRECTIVE,
       STMT_OMP_TARGET_TEAMS_DISTRIBUTE_SIMD_DIRECTIVE,
       EXPR_OMP_ARRAY_SECTION,
+      EXPR_OMP_ARRAY_SHAPING,
+      EXPR_OMP_ITERATOR,
 
       // ARC
       EXPR_OBJC_BRIDGED_CAST,     // ObjCBridgedCastExpr
diff --git a/clang/include/clang/StaticAnalyzer/Checkers/BuiltinCheckerRegistration.h b/clang/include/clang/StaticAnalyzer/Checkers/BuiltinCheckerRegistration.h
index c7732333d9baf..e2be957821b99 100644
--- a/clang/include/clang/StaticAnalyzer/Checkers/BuiltinCheckerRegistration.h
+++ b/clang/include/clang/StaticAnalyzer/Checkers/BuiltinCheckerRegistration.h
@@ -28,7 +28,7 @@ class CheckerRegistry;
 #define GET_CHECKERS
 #define CHECKER(FULLNAME, CLASS, HELPTEXT, DOC_URI, IS_HIDDEN)                 \
   void register##CLASS(CheckerManager &mgr);                                   \
-  bool shouldRegister##CLASS(const LangOptions &LO);
+  bool shouldRegister##CLASS(const CheckerManager &mgr);
 #include "clang/StaticAnalyzer/Checkers/Checkers.inc"
 #undef CHECKER
 #undef GET_CHECKERS
diff --git a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
index a21107cd4c2df..6a577940e3138 100644
--- a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
+++ b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
@@ -299,7 +299,7 @@ def StdCLibraryFunctionArgsChecker : Checker<"StdCLibraryFunctionArgs">,
   HelpText<"Check constraints of arguments of C standard library functions, "
            "such as whether the parameter of isalpha is in the range [0, 255] "
            "or is EOF.">,
-  Dependencies<[StdCLibraryFunctionsChecker]>,
+  Dependencies<[StdCLibraryFunctionsChecker, CallAndMessageChecker]>,
   Documentation<NotDocumented>;
 
 def TrustNonnullChecker : Checker<"TrustNonnull">,
@@ -1422,6 +1422,12 @@ def DebugIteratorModeling : Checker<"DebugIteratorModeling">,
   Dependencies<[DebugContainerModeling, IteratorModeling]>,
   Documentation<NotDocumented>;
 
+def StdCLibraryFunctionsTesterChecker : Checker<"StdCLibraryFunctionsTester">,
+  HelpText<"Add test functions to the summary map, so testing of individual "
+           "summary constituents becomes possible.">,
+  Dependencies<[StdCLibraryFunctionsChecker]>,
+  Documentation<NotDocumented>;
+
 } // end "debug"
 
 
diff --git a/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h b/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h
index 4454d7603b27f..f34f5c2392902 100644
--- a/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h
+++ b/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h
@@ -14,6 +14,7 @@
 #define LLVM_CLANG_STATICANALYZER_CORE_CHECKERMANAGER_H
 
 #include "clang/Analysis/ProgramPoint.h"
+#include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/LangOptions.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState_Fwd.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/Store.h"
@@ -121,14 +122,38 @@ enum class ObjCMessageVisitKind {
 };
 
 class CheckerManager {
-  ASTContext &Context;
+  ASTContext *Context = nullptr;
   const LangOptions LangOpts;
-  AnalyzerOptions &AOptions;
+  const AnalyzerOptions &AOptions;
+  const Preprocessor *PP = nullptr;
   CheckerNameRef CurrentCheckerName;
+  DiagnosticsEngine &Diags;
+  std::unique_ptr<CheckerRegistry> Registry;
 
 public:
-  CheckerManager(ASTContext &Context, AnalyzerOptions &AOptions)
-      : Context(Context), LangOpts(Context.getLangOpts()), AOptions(AOptions) {}
+  // These constructors are defined in the Frontend library, because
+  // CheckerRegistry, a crucial component of the initialization is in there.
+  // CheckerRegistry cannot be moved to the Core library, because the checker
+  // registration functions are defined in the Checkers library, and the library
+  // dependencies look like this: Core -> Checkers -> Frontend.
+
+  CheckerManager(
+      ASTContext &Context, AnalyzerOptions &AOptions, const Preprocessor &PP,
+      ArrayRef<std::string> plugins,
+      ArrayRef<std::function<void(CheckerRegistry &)>> checkerRegistrationFns);
+
+  /// Constructs a CheckerManager that ignores all non TblGen-generated
+  /// checkers. Useful for unit testing, unless the checker infrastructure
+  /// itself is tested.
+  CheckerManager(ASTContext &Context, AnalyzerOptions &AOptions,
+                 const Preprocessor &PP)
+      : CheckerManager(Context, AOptions, PP, {}, {}) {}
+
+  /// Constructs a CheckerManager without requiring an AST. No checker
+  /// registration will take place. Only useful for retrieving the
+  /// CheckerRegistry and print for help flags where the AST is unavalaible.
+  CheckerManager(AnalyzerOptions &AOptions, const LangOptions &LangOpts,
+                 DiagnosticsEngine &Diags, ArrayRef<std::string> plugins);
 
   ~CheckerManager();
 
@@ -140,8 +165,17 @@ class CheckerManager {
   void finishedCheckerRegistration();
 
   const LangOptions &getLangOpts() const { return LangOpts; }
-  AnalyzerOptions &getAnalyzerOptions() const { return AOptions; }
-  ASTContext &getASTContext() const { return Context; }
+  const AnalyzerOptions &getAnalyzerOptions() const { return AOptions; }
+  const Preprocessor &getPreprocessor() const {
+    assert(PP);
+    return *PP;
+  }
+  const CheckerRegistry &getCheckerRegistry() const { return *Registry; }
+  DiagnosticsEngine &getDiagnostics() const { return Diags; }
+  ASTContext &getASTContext() const {
+    assert(Context);
+    return *Context;
+  }
 
   /// Emits an error through a DiagnosticsEngine about an invalid user supplied
   /// checker option value.
diff --git a/clang/include/clang/StaticAnalyzer/Frontend/AnalysisConsumer.h b/clang/include/clang/StaticAnalyzer/Frontend/AnalysisConsumer.h
index 2d24e6a9586b7..bcc29a60ad703 100644
--- a/clang/include/clang/StaticAnalyzer/Frontend/AnalysisConsumer.h
+++ b/clang/include/clang/StaticAnalyzer/Frontend/AnalysisConsumer.h
@@ -55,7 +55,7 @@ class AnalysisASTConsumer : public ASTConsumer {
 std::unique_ptr<AnalysisASTConsumer>
 CreateAnalysisConsumer(CompilerInstance &CI);
 
-} // end GR namespace
+} // namespace ento
 
 } // end clang namespace
 
diff --git a/clang/include/clang/StaticAnalyzer/Frontend/AnalyzerHelpFlags.h b/clang/include/clang/StaticAnalyzer/Frontend/AnalyzerHelpFlags.h
new file mode 100644
index 0000000000000..a30c241e13502
--- /dev/null
+++ b/clang/include/clang/StaticAnalyzer/Frontend/AnalyzerHelpFlags.h
@@ -0,0 +1,30 @@
+//===-- AnalyzerHelpFlags.h - Query functions for --help flags --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_STATICANALYZER_FRONTEND_ANALYZERHELPFLAGS_H
+#define LLVM_CLANG_STATICANALYZER_FRONTEND_ANALYZERHELPFLAGS_H
+
+namespace llvm {
+class raw_ostream;
+} // namespace llvm
+
+namespace clang {
+
+class CompilerInstance;
+
+namespace ento {
+
+void printCheckerHelp(llvm::raw_ostream &OS, CompilerInstance &CI);
+void printEnabledCheckerList(llvm::raw_ostream &OS, CompilerInstance &CI);
+void printAnalyzerConfigList(llvm::raw_ostream &OS);
+void printCheckerConfigList(llvm::raw_ostream &OS, CompilerInstance &CI);
+
+} // namespace ento
+} // namespace clang
+
+#endif
diff --git a/clang/include/clang/StaticAnalyzer/Frontend/CheckerRegistration.h b/clang/include/clang/StaticAnalyzer/Frontend/CheckerRegistration.h
deleted file mode 100644
index 52a534499002c..0000000000000
--- a/clang/include/clang/StaticAnalyzer/Frontend/CheckerRegistration.h
+++ /dev/null
@@ -1,38 +0,0 @@
-//===-- CheckerRegistration.h - Checker Registration Function ---*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_STATICANALYZER_FRONTEND_CHECKERREGISTRATION_H
-#define LLVM_CLANG_STATICANALYZER_FRONTEND_CHECKERREGISTRATION_H
-
-#include "clang/AST/ASTContext.h"
-#include "clang/Basic/LLVM.h"
-#include <functional>
-#include <memory>
-#include <string>
-
-namespace clang {
-  class AnalyzerOptions;
-  class LangOptions;
-  class DiagnosticsEngine;
-
-namespace ento {
-  class CheckerManager;
-  class CheckerRegistry;
-
-  std::unique_ptr<CheckerManager> createCheckerManager(
-      ASTContext &context,
-      AnalyzerOptions &opts,
-      ArrayRef<std::string> plugins,
-      ArrayRef<std::function<void(CheckerRegistry &)>> checkerRegistrationFns,
-      DiagnosticsEngine &diags);
-
-} // end ento namespace
-
-} // end namespace clang
-
-#endif
diff --git a/clang/include/clang/StaticAnalyzer/Frontend/CheckerRegistry.h b/clang/include/clang/StaticAnalyzer/Frontend/CheckerRegistry.h
index 8830542f27d82..4e98ba2e10d23 100644
--- a/clang/include/clang/StaticAnalyzer/Frontend/CheckerRegistry.h
+++ b/clang/include/clang/StaticAnalyzer/Frontend/CheckerRegistry.h
@@ -10,7 +10,6 @@
 #define LLVM_CLANG_STATICANALYZER_CORE_CHECKERREGISTRY_H
 
 #include "clang/Basic/LLVM.h"
-#include "clang/StaticAnalyzer/Core/CheckerManager.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
@@ -70,10 +69,11 @@ namespace clang {
 
 class AnalyzerOptions;
 class DiagnosticsEngine;
-class LangOptions;
 
 namespace ento {
 
+class CheckerManager;
+
 /// Manages a set of available checkers for running a static analysis.
 /// The checkers are organized into packages by full name, where including
 /// a package will recursively include all subpackages and checkers within it.
@@ -83,14 +83,19 @@ namespace ento {
 class CheckerRegistry {
 public:
   CheckerRegistry(ArrayRef<std::string> plugins, DiagnosticsEngine &diags,
-                  AnalyzerOptions &AnOpts, const LangOptions &LangOpts,
+                  AnalyzerOptions &AnOpts,
                   ArrayRef<std::function<void(CheckerRegistry &)>>
                       checkerRegistrationFns = {});
 
+  /// Collects all enabled checkers in the field EnabledCheckers. It preserves
+  /// the order of insertion, as dependencies have to be enabled before the
+  /// checkers that depend on them.
+  void initializeRegistry(const CheckerManager &Mgr);
+
   /// Initialization functions perform any necessary setup for a checker.
   /// They should include a call to CheckerManager::registerChecker.
   using InitializationFunction = void (*)(CheckerManager &);
-  using ShouldRegisterFunction = bool (*)(const LangOptions &);
+  using ShouldRegisterFunction = bool (*)(const CheckerManager &);
 
   /// Specifies a command line option. It may either belong to a checker or a
   /// package.
@@ -162,12 +167,12 @@ class CheckerRegistry {
 
     ConstCheckerInfoList Dependencies;
 
-    bool isEnabled(const LangOptions &LO) const {
-      return State == StateFromCmdLine::State_Enabled && ShouldRegister(LO);
+    bool isEnabled(const CheckerManager &mgr) const {
+      return State == StateFromCmdLine::State_Enabled && ShouldRegister(mgr);
     }
 
-    bool isDisabled(const LangOptions &LO) const {
-      return State == StateFromCmdLine::State_Disabled || !ShouldRegister(LO);
+    bool isDisabled(const CheckerManager &mgr) const {
+      return State == StateFromCmdLine::State_Disabled || !ShouldRegister(mgr);
     }
 
     // Since each checker must have a different full name, we can identify
@@ -205,14 +210,20 @@ class CheckerRegistry {
 
   using PackageInfoList = llvm::SmallVector<PackageInfo, 0>;
 
-  template <typename T> static void addToCheckerMgr(CheckerManager &mgr) {
-    mgr.registerChecker<T>();
+private:
+  /// Default initialization function for checkers -- since CheckerManager
+  /// includes this header, we need to make it a template parameter, and since
+  /// the checker must be a template parameter as well, we can't put this in the
+  /// cpp file.
+  template <typename MGR, typename T> static void initializeManager(MGR &mgr) {
+    mgr.template registerChecker<T>();
   }
 
-  static bool returnTrue(const LangOptions &LO) {
+  template <typename T> static bool returnTrue(const CheckerManager &mgr) {
     return true;
   }
 
+public:
   /// Adds a checker to the registry. Use this non-templated overload when your
   /// checker requires custom initialization.
   void addChecker(InitializationFunction Fn, ShouldRegisterFunction sfn,
@@ -221,13 +232,17 @@ class CheckerRegistry {
 
   /// Adds a checker to the registry. Use this templated overload when your
   /// checker does not require any custom initialization.
+  /// This function isn't really needed and probably causes more headaches than
+  /// the tiny convenience that it provides, but external plugins might use it,
+  /// and there isn't a strong incentive to remove it.
   template <class T>
   void addChecker(StringRef FullName, StringRef Desc, StringRef DocsUri,
                   bool IsHidden = false) {
     // Avoid MSVC's Compiler Error C2276:
     // http://msdn.microsoft.com/en-us/library/850cstw1(v=VS.80).aspx
-    addChecker(&CheckerRegistry::addToCheckerMgr<T>,
-               &CheckerRegistry::returnTrue, FullName, Desc, DocsUri, IsHidden);
+    addChecker(&CheckerRegistry::initializeManager<CheckerManager, T>,
+               &CheckerRegistry::returnTrue<T>, FullName, Desc, DocsUri,
+               IsHidden);
   }
 
   /// Makes the checker with the full name \p fullName depends on the checker
@@ -263,7 +278,7 @@ class CheckerRegistry {
   void addPackageOption(StringRef OptionType, StringRef PackageFullName,
                         StringRef OptionName, StringRef DefaultValStr,
                         StringRef Description, StringRef DevelopmentStatus,
-                         bool IsHidden = false);
+                        bool IsHidden = false);
 
   // FIXME: This *really* should be added to the frontend flag descriptions.
   /// Initializes a CheckerManager by calling the initialization functions for
@@ -283,11 +298,6 @@ class CheckerRegistry {
   void printCheckerOptionList(raw_ostream &Out) const;
 
 private:
-  /// Collect all enabled checkers. The returned container preserves the order
-  /// of insertion, as dependencies have to be enabled before the checkers that
-  /// depend on them.
-  CheckerInfoSet getEnabledCheckers() const;
-
   /// Return an iterator range of mutable CheckerInfos \p CmdLineArg applies to.
   /// For example, it'll return the checkers for the core package, if
   /// \p CmdLineArg is "core".
@@ -314,7 +324,7 @@ class CheckerRegistry {
 
   DiagnosticsEngine &Diags;
   AnalyzerOptions &AnOpts;
-  const LangOptions &LangOpts;
+  CheckerInfoSet EnabledCheckers;
 };
 
 } // namespace ento
diff --git a/clang/include/clang/StaticAnalyzer/Frontend/FrontendActions.h b/clang/include/clang/StaticAnalyzer/Frontend/FrontendActions.h
index 878b65a1b143c..2b12330e4f2db 100644
--- a/clang/include/clang/StaticAnalyzer/Frontend/FrontendActions.h
+++ b/clang/include/clang/StaticAnalyzer/Frontend/FrontendActions.h
@@ -20,6 +20,8 @@ class AnalyzerOptions;
 
 namespace ento {
 
+class CheckerManager;
+
 //===----------------------------------------------------------------------===//
 // AST Consumer Actions
 //===----------------------------------------------------------------------===//
@@ -51,23 +53,7 @@ class ParseModelFileAction : public ASTFrontendAction {
   llvm::StringMap<Stmt *> &Bodies;
 };
 
-void printCheckerHelp(raw_ostream &OS,
-                      ArrayRef<std::string> plugins,
-                      AnalyzerOptions &opts,
-                      DiagnosticsEngine &diags,
-                      const LangOptions &LangOpts);
-void printEnabledCheckerList(raw_ostream &OS, ArrayRef<std::string> plugins,
-                             AnalyzerOptions &opts,
-                             DiagnosticsEngine &diags,
-                             const LangOptions &LangOpts);
-void printAnalyzerConfigList(raw_ostream &OS);
-void printCheckerConfigList(raw_ostream &OS, ArrayRef<std::string> plugins,
-                            AnalyzerOptions &opts,
-                            DiagnosticsEngine &diags,
-                            const LangOptions &LangOpts);
-
-} // end GR namespace
-
+} // namespace ento
 } // end namespace clang
 
 #endif
diff --git a/clang/include/clang/Tooling/Syntax/Tokens.h b/clang/include/clang/Tooling/Syntax/Tokens.h
index 2ee8400748108..7e50892284f4b 100644
--- a/clang/include/clang/Tooling/Syntax/Tokens.h
+++ b/clang/include/clang/Tooling/Syntax/Tokens.h
@@ -176,6 +176,12 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Token &T);
 class TokenBuffer {
 public:
   TokenBuffer(const SourceManager &SourceMgr) : SourceMgr(&SourceMgr) {}
+
+  TokenBuffer(TokenBuffer &&) = default;
+  TokenBuffer(const TokenBuffer &) = delete;
+  TokenBuffer &operator=(TokenBuffer &&) = default;
+  TokenBuffer &operator=(const TokenBuffer &) = delete;
+
   /// All tokens produced by the preprocessor after all macro replacements,
   /// directives, etc. Source locations found in the clang AST will always
   /// point to one of these tokens.
@@ -191,18 +197,20 @@ class TokenBuffer {
   /// token range R.
   llvm::ArrayRef<syntax::Token> expandedTokens(SourceRange R) const;
 
-  /// Find the subrange of spelled tokens that produced the corresponding \p
-  /// Expanded tokens.
+  /// Returns the subrange of spelled tokens corresponding to AST node spanning
+  /// \p Expanded. This is the text that should be replaced if a refactoring
+  /// were to rewrite the node. If \p Expanded is empty, the returned value is
+  /// llvm::None.
   ///
-  /// EXPECTS: \p Expanded is a subrange of expandedTokens().
-  ///
-  /// Will fail if the expanded tokens do not correspond to a
-  /// sequence of spelled tokens. E.g. for the following example:
+  /// Will fail if the expanded tokens do not correspond to a sequence of
+  /// spelled tokens. E.g. for the following example:
   ///
   ///   #define FIRST f1 f2 f3
   ///   #define SECOND s1 s2 s3
+  ///   #define ID2(X, Y) X Y
   ///
   ///   a FIRST b SECOND c // expanded tokens are: a f1 f2 f3 b s1 s2 s3 c
+  ///   d ID2(e f g, h) i  // expanded tokens are: d e f g h i
   ///
   /// the results would be:
   ///   expanded   => spelled
@@ -212,8 +220,10 @@ class TokenBuffer {
   ///   a f1 f2 f3 => a FIRST
   ///         a f1 => can't map
   ///        s1 s2 => can't map
+  ///         e f  => e f
+  ///         g h  => can't map
   ///
-  /// If \p Expanded is empty, the returned value is llvm::None.
+  /// EXPECTS: \p Expanded is a subrange of expandedTokens().
   /// Complexity is logarithmic.
   llvm::Optional<llvm::ArrayRef<syntax::Token>>
   spelledForExpanded(llvm::ArrayRef<syntax::Token> Expanded) const;
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 29f028bfc2020..29b4c848ae2ce 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -1391,8 +1391,11 @@ void ASTContext::InitBuiltinTypes(const TargetInfo &Target,
   InitBuiltinType(BuiltinFnTy,  BuiltinType::BuiltinFn);
 
   // Placeholder type for OMP array sections.
-  if (LangOpts.OpenMP)
+  if (LangOpts.OpenMP) {
     InitBuiltinType(OMPArraySectionTy, BuiltinType::OMPArraySection);
+    InitBuiltinType(OMPArrayShapingTy, BuiltinType::OMPArrayShaping);
+    InitBuiltinType(OMPIteratorTy, BuiltinType::OMPIterator);
+  }
 
   // C99 6.2.5p11.
   FloatComplexTy      = getComplexType(FloatTy);
@@ -2166,6 +2169,11 @@ TypeInfo ASTContext::getTypeInfoImpl(const Type *T) const {
     return getTypeInfo(cast<AdjustedType>(T)->getAdjustedType().getTypePtr());
   case Type::ObjCInterface: {
     const auto *ObjCI = cast<ObjCInterfaceType>(T);
+    if (ObjCI->getDecl()->isInvalidDecl()) {
+      Width = 8;
+      Align = 8;
+      break;
+    }
     const ASTRecordLayout &Layout = getASTObjCInterfaceLayout(ObjCI->getDecl());
     Width = toBits(Layout.getSize());
     Align = toBits(Layout.getAlignment());
@@ -7800,6 +7808,57 @@ CreateSystemZBuiltinVaListDecl(const ASTContext *Context) {
   return Context->buildImplicitTypedef(VaListTagArrayType, "__builtin_va_list");
 }
 
+static TypedefDecl *CreateHexagonBuiltinVaListDecl(const ASTContext *Context) {
+  // typedef struct __va_list_tag {
+  RecordDecl *VaListTagDecl;
+  VaListTagDecl = Context->buildImplicitRecord("__va_list_tag");
+  VaListTagDecl->startDefinition();
+
+  const size_t NumFields = 3;
+  QualType FieldTypes[NumFields];
+  const char *FieldNames[NumFields];
+
+  //   void *CurrentSavedRegisterArea;
+  FieldTypes[0] = Context->getPointerType(Context->VoidTy);
+  FieldNames[0] = "__current_saved_reg_area_pointer";
+
+  //   void *SavedRegAreaEnd;
+  FieldTypes[1] = Context->getPointerType(Context->VoidTy);
+  FieldNames[1] = "__saved_reg_area_end_pointer";
+
+  //   void *OverflowArea;
+  FieldTypes[2] = Context->getPointerType(Context->VoidTy);
+  FieldNames[2] = "__overflow_area_pointer";
+
+  // Create fields
+  for (unsigned i = 0; i < NumFields; ++i) {
+    FieldDecl *Field = FieldDecl::Create(
+        const_cast<ASTContext &>(*Context), VaListTagDecl, SourceLocation(),
+        SourceLocation(), &Context->Idents.get(FieldNames[i]), FieldTypes[i],
+        /*TInfo=*/0,
+        /*BitWidth=*/0,
+        /*Mutable=*/false, ICIS_NoInit);
+    Field->setAccess(AS_public);
+    VaListTagDecl->addDecl(Field);
+  }
+  VaListTagDecl->completeDefinition();
+  Context->VaListTagDecl = VaListTagDecl;
+  QualType VaListTagType = Context->getRecordType(VaListTagDecl);
+
+  // } __va_list_tag;
+  TypedefDecl *VaListTagTypedefDecl =
+      Context->buildImplicitTypedef(VaListTagType, "__va_list_tag");
+
+  QualType VaListTagTypedefType = Context->getTypedefType(VaListTagTypedefDecl);
+
+  // typedef __va_list_tag __builtin_va_list[1];
+  llvm::APInt Size(Context->getTypeSize(Context->getSizeType()), 1);
+  QualType VaListTagArrayType = Context->getConstantArrayType(
+      VaListTagTypedefType, Size, nullptr, ArrayType::Normal, 0);
+
+  return Context->buildImplicitTypedef(VaListTagArrayType, "__builtin_va_list");
+}
+
 static TypedefDecl *CreateVaListDecl(const ASTContext *Context,
                                      TargetInfo::BuiltinVaListKind Kind) {
   switch (Kind) {
@@ -7819,6 +7878,8 @@ static TypedefDecl *CreateVaListDecl(const ASTContext *Context,
     return CreateAAPCSABIBuiltinVaListDecl(Context);
   case TargetInfo::SystemZBuiltinVaList:
     return CreateSystemZBuiltinVaListDecl(Context);
+  case TargetInfo::HexagonBuiltinVaList:
+    return CreateHexagonBuiltinVaListDecl(Context);
   }
 
   llvm_unreachable("Unhandled __builtin_va_list type kind");
diff --git a/clang/lib/AST/AttrImpl.cpp b/clang/lib/AST/AttrImpl.cpp
index a5ff68c187788..bc0db1ba10a2a 100644
--- a/clang/lib/AST/AttrImpl.cpp
+++ b/clang/lib/AST/AttrImpl.cpp
@@ -151,6 +151,11 @@ OMPDeclareTargetDeclAttr::getDeviceType(const ValueDecl *VD) {
   return llvm::None;
 }
 
+namespace clang {
+llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const OMPTraitInfo &TI);
+llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const OMPTraitInfo *TI);
+}
+
 void OMPDeclareVariantAttr::printPrettyPragma(
     raw_ostream &OS, const PrintingPolicy &Policy) const {
   if (const Expr *E = getVariantFuncRef()) {
@@ -158,9 +163,7 @@ void OMPDeclareVariantAttr::printPrettyPragma(
     E->printPretty(OS, nullptr, Policy);
     OS << ")";
   }
-  OS << " match(";
-  traitInfos->print(OS, Policy);
-  OS << ")";
+  OS << " match(" << traitInfos << ")";
 }
 
 #include "clang/AST/AttrImpl.inc"
diff --git a/clang/lib/AST/ComputeDependence.cpp b/clang/lib/AST/ComputeDependence.cpp
index a6ccf9aad321e..3a326f62a2ec3 100644
--- a/clang/lib/AST/ComputeDependence.cpp
+++ b/clang/lib/AST/ComputeDependence.cpp
@@ -172,7 +172,7 @@ ExprDependence clang::computeDependence(VAArgExpr *E) {
 
 ExprDependence clang::computeDependence(NoInitExpr *E) {
   return toExprDependence(E->getType()->getDependence()) &
-         (ExprDependence::Instantiation & ExprDependence::Error);
+         (ExprDependence::Instantiation | ExprDependence::Error);
 }
 
 ExprDependence clang::computeDependence(ArrayInitLoopExpr *E) {
@@ -363,6 +363,31 @@ ExprDependence clang::computeDependence(OMPArraySectionExpr *E) {
   return D;
 }
 
+ExprDependence clang::computeDependence(OMPArrayShapingExpr *E) {
+  auto D = E->getBase()->getDependence() |
+           toExprDependence(E->getType()->getDependence());
+  for (Expr *Dim: E->getDimensions())
+    if (Dim)
+      D |= Dim->getDependence();
+  return D;
+}
+
+ExprDependence clang::computeDependence(OMPIteratorExpr *E) {
+  auto D = toExprDependence(E->getType()->getDependence());
+  for (unsigned I = 0, End = E->numOfIterators(); I < End; ++I) {
+    if (auto *VD = cast_or_null<ValueDecl>(E->getIteratorDecl(I)))
+      D |= toExprDependence(VD->getType()->getDependence());
+    OMPIteratorExpr::IteratorRange IR = E->getIteratorRange(I);
+    if (Expr *BE = IR.Begin)
+      D |= BE->getDependence();
+    if (Expr *EE = IR.End)
+      D |= EE->getDependence();
+    if (Expr *SE = IR.Step)
+      D |= SE->getDependence();
+  }
+  return D;
+}
+
 /// Compute the type-, value-, and instantiation-dependence of a
 /// declaration reference
 /// based on the declaration being referenced.
@@ -370,7 +395,8 @@ ExprDependence clang::computeDependence(DeclRefExpr *E, const ASTContext &Ctx) {
   auto Deps = ExprDependence::None;
 
   if (auto *NNS = E->getQualifier())
-    Deps |= toExprDependence(NNS->getDependence());
+    Deps |= toExprDependence(NNS->getDependence() &
+                             ~NestedNameSpecifierDependence::Dependent);
 
   if (auto *FirstArg = E->getTemplateArgs()) {
     unsigned NumArgs = E->getNumTemplateArgs();
@@ -590,7 +616,8 @@ ExprDependence clang::computeDependence(CXXPseudoDestructorExpr *E) {
     D |= turnTypeToValueDependence(
         toExprDependence(ST->getType()->getDependence()));
   if (auto *Q = E->getQualifier())
-    D |= toExprDependence(Q->getDependence());
+    D |= toExprDependence(Q->getDependence() &
+                          ~NestedNameSpecifierDependence::Dependent);
   return D;
 }
 
@@ -616,7 +643,8 @@ clang::computeDependence(OverloadExpr *E, bool KnownDependent,
     Deps |= ExprDependence::UnexpandedPack;
   Deps |= getDependenceInExpr(E->getNameInfo());
   if (auto *Q = E->getQualifier())
-    Deps |= toExprDependence(Q->getDependence());
+    Deps |= toExprDependence(Q->getDependence() &
+                             ~NestedNameSpecifierDependence::Dependent);
   for (auto *D : E->decls()) {
     if (D->getDeclContext()->isDependentContext() ||
         isa<UnresolvedUsingValueDecl>(D))
diff --git a/clang/lib/AST/DeclarationName.cpp b/clang/lib/AST/DeclarationName.cpp
index 4eb11bc57e521..ecf676c9936d1 100644
--- a/clang/lib/AST/DeclarationName.cpp
+++ b/clang/lib/AST/DeclarationName.cpp
@@ -17,6 +17,7 @@
 #include "clang/AST/DeclBase.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclTemplate.h"
+#include "clang/AST/OpenMPClause.h"
 #include "clang/AST/PrettyPrinter.h"
 #include "clang/AST/Type.h"
 #include "clang/AST/TypeLoc.h"
@@ -138,8 +139,19 @@ void DeclarationName::print(raw_ostream &OS,
                             const PrintingPolicy &Policy) const {
   switch (getNameKind()) {
   case DeclarationName::Identifier:
-    if (const IdentifierInfo *II = getAsIdentifierInfo())
-      OS << II->getName();
+    if (const IdentifierInfo *II = getAsIdentifierInfo()) {
+      StringRef Name = II->getName();
+      // If this is a mangled OpenMP variant name we strip off the mangling for
+      // printing. It should not be visible to the user at all.
+      if (II->isMangledOpenMPVariantName()) {
+        std::pair<StringRef, StringRef> NameContextPair =
+            Name.split(getOpenMPVariantManglingSeparatorStr());
+        OS << NameContextPair.first << "["
+           << OMPTraitInfo(NameContextPair.second) << "]";
+      } else {
+        OS << Name;
+      }
+    }
     return;
 
   case DeclarationName::ObjCZeroArgSelector:
diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp
index b603d2ab29eed..d82381d611efa 100644
--- a/clang/lib/AST/Expr.cpp
+++ b/clang/lib/AST/Expr.cpp
@@ -522,7 +522,7 @@ PredefinedExpr::PredefinedExpr(SourceLocation L, QualType FnTy, IdentKind IK,
 }
 
 PredefinedExpr::PredefinedExpr(SourceLocation L, QualType FnTy, IdentKind IK,
-                               Expr *Info)
+                               Expr *E)
     : Expr(PredefinedExprClass, FnTy, VK_LValue, OK_Ordinary) {
   PredefinedExprBits.Kind = IK;
   assert((getIdentKind() == IK) &&
@@ -531,7 +531,7 @@ PredefinedExpr::PredefinedExpr(SourceLocation L, QualType FnTy, IdentKind IK,
          "Constructor only valid with UniqueStableNameExpr");
   PredefinedExprBits.HasFunctionName = false;
   PredefinedExprBits.Loc = L;
-  setExpr(Info);
+  setExpr(E);
   setDependence(computeDependence(this));
 }
 
@@ -3398,6 +3398,8 @@ bool Expr::HasSideEffects(const ASTContext &Ctx,
   case ParenExprClass:
   case ArraySubscriptExprClass:
   case OMPArraySectionExprClass:
+  case OMPArrayShapingExprClass:
+  case OMPIteratorExprClass:
   case MemberExprClass:
   case ConditionalOperatorClass:
   case BinaryConditionalOperatorClass:
@@ -4569,3 +4571,165 @@ RecoveryExpr *RecoveryExpr::CreateEmpty(ASTContext &Ctx, unsigned NumSubExprs) {
                            alignof(RecoveryExpr));
   return new (Mem) RecoveryExpr(EmptyShell());
 }
+
+void OMPArrayShapingExpr::setDimensions(ArrayRef<Expr *> Dims) {
+  assert(
+      NumDims == Dims.size() &&
+      "Preallocated number of dimensions is different from the provided one.");
+  llvm::copy(Dims, getTrailingObjects<Expr *>());
+}
+
+void OMPArrayShapingExpr::setBracketsRanges(ArrayRef<SourceRange> BR) {
+  assert(
+      NumDims == BR.size() &&
+      "Preallocated number of dimensions is different from the provided one.");
+  llvm::copy(BR, getTrailingObjects<SourceRange>());
+}
+
+OMPArrayShapingExpr::OMPArrayShapingExpr(QualType ExprTy, Expr *Op,
+                                         SourceLocation L, SourceLocation R,
+                                         ArrayRef<Expr *> Dims)
+    : Expr(OMPArrayShapingExprClass, ExprTy, VK_LValue, OK_Ordinary), LPLoc(L),
+      RPLoc(R), NumDims(Dims.size()) {
+  setBase(Op);
+  setDimensions(Dims);
+  setDependence(computeDependence(this));
+}
+
+OMPArrayShapingExpr *
+OMPArrayShapingExpr::Create(const ASTContext &Context, QualType T, Expr *Op,
+                            SourceLocation L, SourceLocation R,
+                            ArrayRef<Expr *> Dims,
+                            ArrayRef<SourceRange> BracketRanges) {
+  assert(Dims.size() == BracketRanges.size() &&
+         "Different number of dimensions and brackets ranges.");
+  void *Mem = Context.Allocate(
+      totalSizeToAlloc<Expr *, SourceRange>(Dims.size() + 1, Dims.size()),
+      alignof(OMPArrayShapingExpr));
+  auto *E = new (Mem) OMPArrayShapingExpr(T, Op, L, R, Dims);
+  E->setBracketsRanges(BracketRanges);
+  return E;
+}
+
+OMPArrayShapingExpr *OMPArrayShapingExpr::CreateEmpty(const ASTContext &Context,
+                                                      unsigned NumDims) {
+  void *Mem = Context.Allocate(
+      totalSizeToAlloc<Expr *, SourceRange>(NumDims + 1, NumDims),
+      alignof(OMPArrayShapingExpr));
+  return new (Mem) OMPArrayShapingExpr(EmptyShell(), NumDims);
+}
+
+void OMPIteratorExpr::setIteratorDeclaration(unsigned I, Decl *D) {
+  assert(I < NumIterators &&
+         "Idx is greater or equal the number of iterators definitions.");
+  getTrailingObjects<Decl *>()[I] = D;
+}
+
+void OMPIteratorExpr::setAssignmentLoc(unsigned I, SourceLocation Loc) {
+  assert(I < NumIterators &&
+         "Idx is greater or equal the number of iterators definitions.");
+  getTrailingObjects<
+      SourceLocation>()[I * static_cast<int>(RangeLocOffset::Total) +
+                        static_cast<int>(RangeLocOffset::AssignLoc)] = Loc;
+}
+
+void OMPIteratorExpr::setIteratorRange(unsigned I, Expr *Begin,
+                                       SourceLocation ColonLoc, Expr *End,
+                                       SourceLocation SecondColonLoc,
+                                       Expr *Step) {
+  assert(I < NumIterators &&
+         "Idx is greater or equal the number of iterators definitions.");
+  getTrailingObjects<Expr *>()[I * static_cast<int>(RangeExprOffset::Total) +
+                               static_cast<int>(RangeExprOffset::Begin)] =
+      Begin;
+  getTrailingObjects<Expr *>()[I * static_cast<int>(RangeExprOffset::Total) +
+                               static_cast<int>(RangeExprOffset::End)] = End;
+  getTrailingObjects<Expr *>()[I * static_cast<int>(RangeExprOffset::Total) +
+                               static_cast<int>(RangeExprOffset::Step)] = Step;
+  getTrailingObjects<
+      SourceLocation>()[I * static_cast<int>(RangeLocOffset::Total) +
+                        static_cast<int>(RangeLocOffset::FirstColonLoc)] =
+      ColonLoc;
+  getTrailingObjects<
+      SourceLocation>()[I * static_cast<int>(RangeLocOffset::Total) +
+                        static_cast<int>(RangeLocOffset::SecondColonLoc)] =
+      SecondColonLoc;
+}
+
+Decl *OMPIteratorExpr::getIteratorDecl(unsigned I) {
+  return getTrailingObjects<Decl *>()[I];
+}
+
+OMPIteratorExpr::IteratorRange OMPIteratorExpr::getIteratorRange(unsigned I) {
+  IteratorRange Res;
+  Res.Begin =
+      getTrailingObjects<Expr *>()[I * static_cast<int>(
+                                           RangeExprOffset::Total) +
+                                   static_cast<int>(RangeExprOffset::Begin)];
+  Res.End =
+      getTrailingObjects<Expr *>()[I * static_cast<int>(
+                                           RangeExprOffset::Total) +
+                                   static_cast<int>(RangeExprOffset::End)];
+  Res.Step =
+      getTrailingObjects<Expr *>()[I * static_cast<int>(
+                                           RangeExprOffset::Total) +
+                                   static_cast<int>(RangeExprOffset::Step)];
+  return Res;
+}
+
+SourceLocation OMPIteratorExpr::getAssignLoc(unsigned I) const {
+  return getTrailingObjects<
+      SourceLocation>()[I * static_cast<int>(RangeLocOffset::Total) +
+                        static_cast<int>(RangeLocOffset::AssignLoc)];
+}
+
+SourceLocation OMPIteratorExpr::getColonLoc(unsigned I) const {
+  return getTrailingObjects<
+      SourceLocation>()[I * static_cast<int>(RangeLocOffset::Total) +
+                        static_cast<int>(RangeLocOffset::FirstColonLoc)];
+}
+
+SourceLocation OMPIteratorExpr::getSecondColonLoc(unsigned I) const {
+  return getTrailingObjects<
+      SourceLocation>()[I * static_cast<int>(RangeLocOffset::Total) +
+                        static_cast<int>(RangeLocOffset::SecondColonLoc)];
+}
+
+OMPIteratorExpr::OMPIteratorExpr(
+    QualType ExprTy, SourceLocation IteratorKwLoc, SourceLocation L,
+    SourceLocation R, ArrayRef<OMPIteratorExpr::IteratorDefinition> Data)
+    : Expr(OMPIteratorExprClass, ExprTy, VK_LValue, OK_Ordinary),
+      IteratorKwLoc(IteratorKwLoc), LPLoc(L), RPLoc(R),
+      NumIterators(Data.size()) {
+  for (unsigned I = 0, E = Data.size(); I < E; ++I) {
+    const IteratorDefinition &D = Data[I];
+    setIteratorDeclaration(I, D.IteratorDecl);
+    setAssignmentLoc(I, D.AssignmentLoc);
+    setIteratorRange(I, D.Range.Begin, D.ColonLoc, D.Range.End,
+                     D.SecondColonLoc, D.Range.Step);
+  }
+  setDependence(computeDependence(this));
+}
+
+OMPIteratorExpr *
+OMPIteratorExpr::Create(const ASTContext &Context, QualType T,
+                        SourceLocation IteratorKwLoc, SourceLocation L,
+                        SourceLocation R,
+                        ArrayRef<OMPIteratorExpr::IteratorDefinition> Data) {
+  void *Mem = Context.Allocate(
+      totalSizeToAlloc<Decl *, Expr *, SourceLocation>(
+          Data.size(), Data.size() * static_cast<int>(RangeExprOffset::Total),
+          Data.size() * static_cast<int>(RangeLocOffset::Total)),
+      alignof(OMPIteratorExpr));
+  return new (Mem) OMPIteratorExpr(T, IteratorKwLoc, L, R, Data);
+}
+
+OMPIteratorExpr *OMPIteratorExpr::CreateEmpty(const ASTContext &Context,
+                                              unsigned NumIterators) {
+  void *Mem = Context.Allocate(
+      totalSizeToAlloc<Decl *, Expr *, SourceLocation>(
+          NumIterators, NumIterators * static_cast<int>(RangeExprOffset::Total),
+          NumIterators * static_cast<int>(RangeLocOffset::Total)),
+      alignof(OMPIteratorExpr));
+  return new (Mem) OMPIteratorExpr(EmptyShell(), NumIterators);
+}
diff --git a/clang/lib/AST/ExprClassification.cpp b/clang/lib/AST/ExprClassification.cpp
index 58e70347292c7..8587a476a5d90 100644
--- a/clang/lib/AST/ExprClassification.cpp
+++ b/clang/lib/AST/ExprClassification.cpp
@@ -140,6 +140,8 @@ static Cl::Kinds ClassifyInternal(ASTContext &Ctx, const Expr *E) {
   case Expr::MSPropertyRefExprClass:
   case Expr::MSPropertySubscriptExprClass:
   case Expr::OMPArraySectionExprClass:
+  case Expr::OMPArrayShapingExprClass:
+  case Expr::OMPIteratorExprClass:
     return Cl::CL_LValue;
 
     // C99 6.5.2.5p5 says that compound literals are lvalues.
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index 06f4885e47d6c..c6e1cc7b67df3 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -8677,6 +8677,10 @@ bool PointerExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E,
 static bool EvaluateArrayNewInitList(EvalInfo &Info, LValue &This,
                                      APValue &Result, const InitListExpr *ILE,
                                      QualType AllocType);
+static bool EvaluateArrayNewConstructExpr(EvalInfo &Info, LValue &This,
+                                          APValue &Result,
+                                          const CXXConstructExpr *CCE,
+                                          QualType AllocType);
 
 bool PointerExprEvaluator::VisitCXXNewExpr(const CXXNewExpr *E) {
   if (!Info.getLangOpts().CPlusPlus2a)
@@ -8726,6 +8730,7 @@ bool PointerExprEvaluator::VisitCXXNewExpr(const CXXNewExpr *E) {
 
   const Expr *Init = E->getInitializer();
   const InitListExpr *ResizedArrayILE = nullptr;
+  const CXXConstructExpr *ResizedArrayCCE = nullptr;
 
   QualType AllocType = E->getAllocatedType();
   if (Optional<const Expr*> ArraySize = E->getArraySize()) {
@@ -8769,7 +8774,7 @@ bool PointerExprEvaluator::VisitCXXNewExpr(const CXXNewExpr *E) {
     //   -- the new-initializer is a braced-init-list and the number of
     //      array elements for which initializers are provided [...]
     //      exceeds the number of elements to initialize
-    if (Init) {
+    if (Init && !isa<CXXConstructExpr>(Init)) {
       auto *CAT = Info.Ctx.getAsConstantArrayType(Init->getType());
       assert(CAT && "unexpected type for array initializer");
 
@@ -8792,6 +8797,8 @@ bool PointerExprEvaluator::VisitCXXNewExpr(const CXXNewExpr *E) {
       // special handling for this case when we initialize.
       if (InitBound != AllocBound)
         ResizedArrayILE = cast<InitListExpr>(Init);
+    } else if (Init) {
+      ResizedArrayCCE = cast<CXXConstructExpr>(Init);
     }
 
     AllocType = Info.Ctx.getConstantArrayType(AllocType, ArrayBound, nullptr,
@@ -8856,6 +8863,10 @@ bool PointerExprEvaluator::VisitCXXNewExpr(const CXXNewExpr *E) {
     if (!EvaluateArrayNewInitList(Info, Result, *Val, ResizedArrayILE,
                                   AllocType))
       return false;
+  } else if (ResizedArrayCCE) {
+    if (!EvaluateArrayNewConstructExpr(Info, Result, *Val, ResizedArrayCCE,
+                                       AllocType))
+      return false;
   } else if (Init) {
     if (!EvaluateInPlace(*Val, Info, Result, Init))
       return false;
@@ -9683,6 +9694,16 @@ static bool EvaluateArrayNewInitList(EvalInfo &Info, LValue &This,
       .VisitInitListExpr(ILE, AllocType);
 }
 
+static bool EvaluateArrayNewConstructExpr(EvalInfo &Info, LValue &This,
+                                          APValue &Result,
+                                          const CXXConstructExpr *CCE,
+                                          QualType AllocType) {
+  assert(CCE->isRValue() && CCE->getType()->isArrayType() &&
+         "not an array rvalue");
+  return ArrayExprEvaluator(Info, This, Result)
+      .VisitCXXConstructExpr(CCE, This, &Result, AllocType);
+}
+
 // Return true iff the given array filler may depend on the element index.
 static bool MaybeElementDependentArrayFiller(const Expr *FillerExpr) {
   // For now, just whitelist non-class value-initialization and initialization
@@ -14159,6 +14180,8 @@ static ICEDiag CheckICE(const Expr* E, const ASTContext &Ctx) {
   case Expr::StringLiteralClass:
   case Expr::ArraySubscriptExprClass:
   case Expr::OMPArraySectionExprClass:
+  case Expr::OMPArrayShapingExprClass:
+  case Expr::OMPIteratorExprClass:
   case Expr::MemberExprClass:
   case Expr::CompoundAssignOperatorClass:
   case Expr::CompoundLiteralExprClass:
diff --git a/clang/lib/AST/ItaniumMangle.cpp b/clang/lib/AST/ItaniumMangle.cpp
index 823c085c4aea8..0495a44a38370 100644
--- a/clang/lib/AST/ItaniumMangle.cpp
+++ b/clang/lib/AST/ItaniumMangle.cpp
@@ -3714,6 +3714,8 @@ void CXXNameMangler::mangleExpression(const Expr *E, unsigned Arity) {
   case Expr::TypoExprClass: // This should no longer exist in the AST by now.
   case Expr::RecoveryExprClass:
   case Expr::OMPArraySectionExprClass:
+  case Expr::OMPArrayShapingExprClass:
+  case Expr::OMPIteratorExprClass:
   case Expr::CXXInheritedCtorInitExprClass:
     llvm_unreachable("unexpected statement kind");
 
diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp
index 4c32093bc9dc9..0b3c9b738f9c4 100644
--- a/clang/lib/AST/MicrosoftMangle.cpp
+++ b/clang/lib/AST/MicrosoftMangle.cpp
@@ -2940,11 +2940,17 @@ void MicrosoftCXXNameMangler::mangleType(const AtomicType *T, Qualifiers,
 
 void MicrosoftCXXNameMangler::mangleType(const PipeType *T, Qualifiers,
                                          SourceRange Range) {
-  DiagnosticsEngine &Diags = Context.getDiags();
-  unsigned DiagID = Diags.getCustomDiagID(DiagnosticsEngine::Error,
-    "cannot mangle this OpenCL pipe type yet");
-  Diags.Report(Range.getBegin(), DiagID)
-    << Range;
+  QualType ElementType = T->getElementType();
+
+  llvm::SmallString<64> TemplateMangling;
+  llvm::raw_svector_ostream Stream(TemplateMangling);
+  MicrosoftCXXNameMangler Extra(Context, Stream);
+  Stream << "?$";
+  Extra.mangleSourceName("ocl_pipe");
+  Extra.mangleType(ElementType, Range, QMM_Escape);
+  Extra.mangleIntegerLiteral(llvm::APSInt::get(T->isReadOnly()), true);
+
+  mangleArtificialTagType(TTK_Struct, TemplateMangling, {"__clang"});
 }
 
 void MicrosoftMangleContextImpl::mangleCXXName(GlobalDecl GD,
diff --git a/clang/lib/AST/NSAPI.cpp b/clang/lib/AST/NSAPI.cpp
index ae6ff04f5126d..3bb3605ea9361 100644
--- a/clang/lib/AST/NSAPI.cpp
+++ b/clang/lib/AST/NSAPI.cpp
@@ -483,6 +483,8 @@ NSAPI::getNSNumberFactoryMethodKind(QualType T) const {
   case BuiltinType::PseudoObject:
   case BuiltinType::BuiltinFn:
   case BuiltinType::OMPArraySection:
+  case BuiltinType::OMPArrayShaping:
+  case BuiltinType::OMPIterator:
     break;
   }
 
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index fc7912d6fdcac..a205a1bca1b98 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -24,6 +24,8 @@
 #include <cassert>
 
 using namespace clang;
+using namespace llvm;
+using namespace omp;
 
 OMPClause::child_range OMPClause::children() {
   switch (getClauseKind()) {
@@ -898,16 +900,19 @@ OMPDepobjClause *OMPDepobjClause::CreateEmpty(const ASTContext &C) {
 OMPDependClause *
 OMPDependClause::Create(const ASTContext &C, SourceLocation StartLoc,
                         SourceLocation LParenLoc, SourceLocation EndLoc,
-                        OpenMPDependClauseKind DepKind, SourceLocation DepLoc,
-                        SourceLocation ColonLoc, ArrayRef<Expr *> VL,
-                        unsigned NumLoops) {
-  void *Mem = C.Allocate(totalSizeToAlloc<Expr *>(VL.size() + NumLoops));
+                        Expr *DepModifier, OpenMPDependClauseKind DepKind,
+                        SourceLocation DepLoc, SourceLocation ColonLoc,
+                        ArrayRef<Expr *> VL, unsigned NumLoops) {
+  void *Mem = C.Allocate(
+      totalSizeToAlloc<Expr *>(VL.size() + /*depend-modifier*/ 1 + NumLoops),
+      alignof(OMPDependClause));
   OMPDependClause *Clause = new (Mem)
       OMPDependClause(StartLoc, LParenLoc, EndLoc, VL.size(), NumLoops);
   Clause->setVarRefs(VL);
   Clause->setDependencyKind(DepKind);
   Clause->setDependencyLoc(DepLoc);
   Clause->setColonLoc(ColonLoc);
+  Clause->setModifier(DepModifier);
   for (unsigned I = 0 ; I < NumLoops; ++I)
     Clause->setLoopData(I, nullptr);
   return Clause;
@@ -915,7 +920,9 @@ OMPDependClause::Create(const ASTContext &C, SourceLocation StartLoc,
 
 OMPDependClause *OMPDependClause::CreateEmpty(const ASTContext &C, unsigned N,
                                               unsigned NumLoops) {
-  void *Mem = C.Allocate(totalSizeToAlloc<Expr *>(N + NumLoops));
+  void *Mem =
+      C.Allocate(totalSizeToAlloc<Expr *>(N + /*depend-modifier*/ 1 + NumLoops),
+                 alignof(OMPDependClause));
   return new (Mem) OMPDependClause(N, NumLoops);
 }
 
@@ -925,7 +932,7 @@ void OMPDependClause::setLoopData(unsigned NumLoop, Expr *Cnt) {
          NumLoop < NumLoops &&
          "Expected sink or source depend + loop index must be less number of "
          "loops.");
-  auto It = std::next(getVarRefs().end(), NumLoop);
+  auto *It = std::next(getVarRefs().end(), NumLoop + 1);
   *It = Cnt;
 }
 
@@ -935,7 +942,7 @@ Expr *OMPDependClause::getLoopData(unsigned NumLoop) {
          NumLoop < NumLoops &&
          "Expected sink or source depend + loop index must be less number of "
          "loops.");
-  auto It = std::next(getVarRefs().end(), NumLoop);
+  auto *It = std::next(getVarRefs().end(), NumLoop + 1);
   return *It;
 }
 
@@ -945,10 +952,15 @@ const Expr *OMPDependClause::getLoopData(unsigned NumLoop) const {
          NumLoop < NumLoops &&
          "Expected sink or source depend + loop index must be less number of "
          "loops.");
-  auto It = std::next(getVarRefs().end(), NumLoop);
+  const auto *It = std::next(getVarRefs().end(), NumLoop + 1);
   return *It;
 }
 
+void OMPDependClause::setModifier(Expr *DepModifier) {
+  *getVarRefs().end() = DepModifier;
+}
+Expr *OMPDependClause::getModifier() { return *getVarRefs().end(); }
+
 unsigned OMPClauseMappableExprCommon::getComponentsTotalNumber(
     MappableExprComponentListsRef ComponentLists) {
   unsigned TotalNum = 0u;
@@ -1296,7 +1308,7 @@ OMPExclusiveClause *OMPExclusiveClause::CreateEmpty(const ASTContext &C,
 
 void OMPClausePrinter::VisitOMPIfClause(OMPIfClause *Node) {
   OS << "if(";
-  if (Node->getNameModifier() != llvm::omp::OMPD_unknown)
+  if (Node->getNameModifier() != OMPD_unknown)
     OS << getOpenMPDirectiveName(Node->getNameModifier()) << ": ";
   Node->getCondition()->printPretty(OS, nullptr, Policy, 0);
   OS << ")";
@@ -1725,6 +1737,10 @@ void OMPClausePrinter::VisitOMPDepobjClause(OMPDepobjClause *Node) {
 
 void OMPClausePrinter::VisitOMPDependClause(OMPDependClause *Node) {
   OS << "depend(";
+  if (Expr *DepModifier = Node->getModifier()) {
+    DepModifier->printPretty(OS, nullptr, Policy);
+    OS << ", ";
+  }
   OS << getOpenMPSimpleClauseTypeName(Node->getClauseKind(),
                                       Node->getDependencyKind());
   if (!Node->varlist_empty()) {
@@ -1738,7 +1754,7 @@ void OMPClausePrinter::VisitOMPMapClause(OMPMapClause *Node) {
   if (!Node->varlist_empty()) {
     OS << "map(";
     if (Node->getMapType() != OMPC_MAP_unknown) {
-      for (unsigned I = 0; I < OMPMapClause::NumberOfModifiers; ++I) {
+      for (unsigned I = 0; I < NumberOfOMPMapClauseModifiers; ++I) {
         if (Node->getMapTypeModifier(I) != OMPC_MAP_MODIFIER_unknown) {
           OS << getOpenMPSimpleClauseTypeName(OMPC_map,
                                               Node->getMapTypeModifier(I));
@@ -1866,25 +1882,28 @@ void OMPClausePrinter::VisitOMPExclusiveClause(OMPExclusiveClause *Node) {
   }
 }
 
-void OMPTraitInfo::getAsVariantMatchInfo(
-    ASTContext &ASTCtx, llvm::omp::VariantMatchInfo &VMI) const {
+void OMPTraitInfo::getAsVariantMatchInfo(ASTContext &ASTCtx,
+                                         VariantMatchInfo &VMI,
+                                         bool DeviceSetOnly) const {
   for (const OMPTraitSet &Set : Sets) {
+    if (DeviceSetOnly && Set.Kind != TraitSet::device)
+      continue;
     for (const OMPTraitSelector &Selector : Set.Selectors) {
 
       // User conditions are special as we evaluate the condition here.
-      if (Selector.Kind == llvm::omp::TraitSelector::user_condition) {
+      if (Selector.Kind == TraitSelector::user_condition) {
         assert(Selector.ScoreOrCondition &&
                "Ill-formed user condition, expected condition expression!");
         assert(Selector.Properties.size() == 1 &&
                Selector.Properties.front().Kind ==
-                   llvm::omp::TraitProperty::user_condition_unknown &&
+                   TraitProperty::user_condition_unknown &&
                "Ill-formed user condition, expected unknown trait property!");
 
         llvm::APInt CondVal =
             Selector.ScoreOrCondition->EvaluateKnownConstInt(ASTCtx);
         VMI.addTrait(CondVal.isNullValue()
-                         ? llvm::omp::TraitProperty::user_condition_false
-                         : llvm::omp::TraitProperty::user_condition_true);
+                         ? TraitProperty::user_condition_false
+                         : TraitProperty::user_condition_true);
         continue;
       }
 
@@ -1897,13 +1916,13 @@ void OMPTraitInfo::getAsVariantMatchInfo(
       for (const OMPTraitProperty &Property : Selector.Properties)
         VMI.addTrait(Set.Kind, Property.Kind, ScorePtr);
 
-      if (Set.Kind != llvm::omp::TraitSet::construct)
+      if (Set.Kind != TraitSet::construct)
         continue;
 
       // TODO: This might not hold once we implement SIMD properly.
       assert(Selector.Properties.size() == 1 &&
              Selector.Properties.front().Kind ==
-                 llvm::omp::getOpenMPContextTraitPropertyForSelector(
+                 getOpenMPContextTraitPropertyForSelector(
                      Selector.Kind) &&
              "Ill-formed construct selector!");
 
@@ -1915,29 +1934,29 @@ void OMPTraitInfo::getAsVariantMatchInfo(
 void OMPTraitInfo::print(llvm::raw_ostream &OS,
                          const PrintingPolicy &Policy) const {
   bool FirstSet = true;
-  for (const OMPTraitInfo::OMPTraitSet &Set : Sets) {
+  for (const OMPTraitSet &Set : Sets) {
     if (!FirstSet)
       OS << ", ";
     FirstSet = false;
-    OS << llvm::omp::getOpenMPContextTraitSetName(Set.Kind) << "={";
+    OS << getOpenMPContextTraitSetName(Set.Kind) << "={";
 
     bool FirstSelector = true;
-    for (const OMPTraitInfo::OMPTraitSelector &Selector : Set.Selectors) {
+    for (const OMPTraitSelector &Selector : Set.Selectors) {
       if (!FirstSelector)
         OS << ", ";
       FirstSelector = false;
-      OS << llvm::omp::getOpenMPContextTraitSelectorName(Selector.Kind);
+      OS << getOpenMPContextTraitSelectorName(Selector.Kind);
 
       bool AllowsTraitScore = false;
       bool RequiresProperty = false;
-      llvm::omp::isValidTraitSelectorForTraitSet(
+      isValidTraitSelectorForTraitSet(
           Selector.Kind, Set.Kind, AllowsTraitScore, RequiresProperty);
 
       if (!RequiresProperty)
         continue;
 
       OS << "(";
-      if (Selector.Kind == llvm::omp::TraitSelector::user_condition) {
+      if (Selector.Kind == TraitSelector::user_condition) {
         Selector.ScoreOrCondition->printPretty(OS, nullptr, Policy);
       } else {
 
@@ -1948,12 +1967,11 @@ void OMPTraitInfo::print(llvm::raw_ostream &OS,
         }
 
         bool FirstProperty = true;
-        for (const OMPTraitInfo::OMPTraitProperty &Property :
-             Selector.Properties) {
+        for (const OMPTraitProperty &Property : Selector.Properties) {
           if (!FirstProperty)
             OS << ", ";
           FirstProperty = false;
-          OS << llvm::omp::getOpenMPContextTraitPropertyName(Property.Kind);
+          OS << getOpenMPContextTraitPropertyName(Property.Kind);
         }
       }
       OS << ")";
@@ -1962,6 +1980,63 @@ void OMPTraitInfo::print(llvm::raw_ostream &OS,
   }
 }
 
+std::string OMPTraitInfo::getMangledName() const {
+  std::string MangledName;
+  llvm::raw_string_ostream OS(MangledName);
+  for (const OMPTraitSet &Set : Sets) {
+    OS << '.' << 'S' << unsigned(Set.Kind);
+    for (const OMPTraitSelector &Selector : Set.Selectors) {
+
+      bool AllowsTraitScore = false;
+      bool RequiresProperty = false;
+      isValidTraitSelectorForTraitSet(
+          Selector.Kind, Set.Kind, AllowsTraitScore, RequiresProperty);
+      OS << '.' << 's' << unsigned(Selector.Kind);
+
+      if (!RequiresProperty ||
+          Selector.Kind == TraitSelector::user_condition)
+        continue;
+
+      for (const OMPTraitProperty &Property : Selector.Properties)
+        OS << '.' << 'P'
+           << getOpenMPContextTraitPropertyName(Property.Kind);
+    }
+  }
+  return OS.str();
+}
+
+OMPTraitInfo::OMPTraitInfo(StringRef MangledName) {
+  unsigned long U;
+  do {
+    if (!MangledName.consume_front(".S"))
+      break;
+    if (MangledName.consumeInteger(10, U))
+      break;
+    Sets.push_back(OMPTraitSet());
+    OMPTraitSet &Set = Sets.back();
+    Set.Kind = TraitSet(U);
+    do {
+      if (!MangledName.consume_front(".s"))
+        break;
+      if (MangledName.consumeInteger(10, U))
+        break;
+      Set.Selectors.push_back(OMPTraitSelector());
+      OMPTraitSelector &Selector = Set.Selectors.back();
+      Selector.Kind = TraitSelector(U);
+      do {
+        if (!MangledName.consume_front(".P"))
+          break;
+        Selector.Properties.push_back(OMPTraitProperty());
+        OMPTraitProperty &Property = Selector.Properties.back();
+        std::pair<StringRef, StringRef> PropRestPair = MangledName.split('.');
+        Property.Kind =
+            getOpenMPContextTraitPropertyKind(Set.Kind, PropRestPair.first);
+        MangledName = PropRestPair.second;
+      } while (true);
+    } while (true);
+  } while (true);
+}
+
 llvm::raw_ostream &clang::operator<<(llvm::raw_ostream &OS,
                                      const OMPTraitInfo &TI) {
   LangOptions LO;
diff --git a/clang/lib/AST/PrintfFormatString.cpp b/clang/lib/AST/PrintfFormatString.cpp
index bae60d4644078..d13ff0a02d171 100644
--- a/clang/lib/AST/PrintfFormatString.cpp
+++ b/clang/lib/AST/PrintfFormatString.cpp
@@ -11,10 +11,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "FormatStringParsing.h"
 #include "clang/AST/FormatString.h"
 #include "clang/AST/OSLog.h"
-#include "FormatStringParsing.h"
 #include "clang/Basic/TargetInfo.h"
+#include "llvm/Support/Regex.h"
 
 using clang::analyze_format_string::ArgType;
 using clang::analyze_format_string::FormatStringHandler;
diff --git a/clang/lib/AST/RecordLayoutBuilder.cpp b/clang/lib/AST/RecordLayoutBuilder.cpp
index 932edb75bd584..cd54b61ca3d89 100644
--- a/clang/lib/AST/RecordLayoutBuilder.cpp
+++ b/clang/lib/AST/RecordLayoutBuilder.cpp
@@ -3226,7 +3226,8 @@ ASTContext::getObjCLayout(const ObjCInterfaceDecl *D,
   if (D->hasExternalLexicalStorage() && !D->getDefinition())
     getExternalSource()->CompleteType(const_cast<ObjCInterfaceDecl*>(D));
   D = D->getDefinition();
-  assert(D && D->isThisDeclarationADefinition() && "Invalid interface decl!");
+  assert(D && !D->isInvalidDecl() && D->isThisDeclarationADefinition() &&
+         "Invalid interface decl!");
 
   // Look up this layout, if already laid out, return what we have.
   const ObjCContainerDecl *Key =
diff --git a/clang/lib/AST/StmtPrinter.cpp b/clang/lib/AST/StmtPrinter.cpp
index 80fdc09a8a6cf..6537ca98667e3 100644
--- a/clang/lib/AST/StmtPrinter.cpp
+++ b/clang/lib/AST/StmtPrinter.cpp
@@ -1350,6 +1350,37 @@ void StmtPrinter::VisitOMPArraySectionExpr(OMPArraySectionExpr *Node) {
   OS << "]";
 }
 
+void StmtPrinter::VisitOMPArrayShapingExpr(OMPArrayShapingExpr *Node) {
+  OS << "(";
+  for (Expr *E : Node->getDimensions()) {
+    OS << "[";
+    PrintExpr(E);
+    OS << "]";
+  }
+  OS << ")";
+  PrintExpr(Node->getBase());
+}
+
+void StmtPrinter::VisitOMPIteratorExpr(OMPIteratorExpr *Node) {
+  OS << "iterator(";
+  for (unsigned I = 0, E = Node->numOfIterators(); I < E; ++I) {
+    auto *VD = cast<ValueDecl>(Node->getIteratorDecl(I));
+    VD->getType().print(OS, Policy);
+    const OMPIteratorExpr::IteratorRange Range = Node->getIteratorRange(I);
+    OS << " " << VD->getName() << " = ";
+    PrintExpr(Range.Begin);
+    OS << ":";
+    PrintExpr(Range.End);
+    if (Range.Step) {
+      OS << ":";
+      PrintExpr(Range.Step);
+    }
+    if (I < E - 1)
+      OS << ", ";
+  }
+  OS << ")";
+}
+
 void StmtPrinter::PrintCallArgs(CallExpr *Call) {
   for (unsigned i = 0, e = Call->getNumArgs(); i != e; ++i) {
     if (isa<CXXDefaultArgExpr>(Call->getArg(i))) {
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index 5e87eb3e237c3..fec12ac98b4ea 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -1194,6 +1194,16 @@ void StmtProfiler::VisitOMPArraySectionExpr(const OMPArraySectionExpr *S) {
   VisitExpr(S);
 }
 
+void StmtProfiler::VisitOMPArrayShapingExpr(const OMPArrayShapingExpr *S) {
+  VisitExpr(S);
+}
+
+void StmtProfiler::VisitOMPIteratorExpr(const OMPIteratorExpr *S) {
+  VisitExpr(S);
+  for (unsigned I = 0, E = S->numOfIterators(); I < E; ++I)
+    VisitDecl(S->getIteratorDecl(I));
+}
+
 void StmtProfiler::VisitCallExpr(const CallExpr *S) {
   VisitExpr(S);
 }
diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index 6a6d8692228af..dc0dd92f09df6 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -1086,6 +1086,23 @@ void TextNodeDumper::VisitObjCBoolLiteralExpr(const ObjCBoolLiteralExpr *Node) {
   OS << " " << (Node->getValue() ? "__objc_yes" : "__objc_no");
 }
 
+void TextNodeDumper::VisitOMPIteratorExpr(const OMPIteratorExpr *Node) {
+  OS << " ";
+  for (unsigned I = 0, E = Node->numOfIterators(); I < E; ++I) {
+    Visit(Node->getIteratorDecl(I));
+    OS << " = ";
+    const OMPIteratorExpr::IteratorRange Range = Node->getIteratorRange(I);
+    OS << " begin ";
+    Visit(Range.Begin);
+    OS << " end ";
+    Visit(Range.End);
+    if (Range.Step) {
+      OS << " step ";
+      Visit(Range.Step);
+    }
+  }
+}
+
 void TextNodeDumper::VisitRValueReferenceType(const ReferenceType *T) {
   if (T->isSpelledAsLValue())
     OS << " written as lvalue reference";
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index 69c942e46f729..3428437c31462 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -2249,6 +2249,9 @@ bool QualType::isTrivialType(const ASTContext &Context) const {
   if ((*this)->isArrayType())
     return Context.getBaseElementType(*this).isTrivialType(Context);
 
+  if ((*this)->isSizelessBuiltinType())
+    return true;
+
   // Return false for incomplete types after skipping any incomplete array
   // types which are expressly allowed by the standard and thus our API.
   if ((*this)->isIncompleteType())
@@ -2303,6 +2306,9 @@ bool QualType::isTriviallyCopyableType(const ASTContext &Context) const {
   if (CanonicalType->isDependentType())
     return false;
 
+  if (CanonicalType->isSizelessBuiltinType())
+    return true;
+
   // Return false for incomplete types after skipping any incomplete array types
   // which are expressly allowed by the standard and thus our API.
   if (CanonicalType->isIncompleteType())
@@ -2496,6 +2502,9 @@ bool QualType::isCXX11PODType(const ASTContext &Context) const {
   const Type *BaseTy = ty->getBaseElementTypeUnsafe();
   assert(BaseTy && "NULL element type");
 
+  if (BaseTy->isSizelessBuiltinType())
+    return true;
+
   // Return false for incomplete types after skipping any incomplete array
   // types which are expressly allowed by the standard and thus our API.
   if (BaseTy->isIncompleteType())
@@ -2899,6 +2908,10 @@ StringRef BuiltinType::getName(const PrintingPolicy &Policy) const {
     return "reserve_id_t";
   case OMPArraySection:
     return "<OpenMP array section type>";
+  case OMPArrayShaping:
+    return "<OpenMP array shaping type>";
+  case OMPIterator:
+    return "<OpenMP iterator type>";
 #define EXT_OPAQUE_TYPE(ExtType, Id, Ext) \
   case Id: \
     return #ExtType;
@@ -3905,6 +3918,8 @@ bool Type::canHaveNullability(bool ResultIfUnknown) const {
     case BuiltinType::BuiltinFn:
     case BuiltinType::NullPtr:
     case BuiltinType::OMPArraySection:
+    case BuiltinType::OMPArrayShaping:
+    case BuiltinType::OMPIterator:
       return false;
     }
     llvm_unreachable("unknown builtin type");
@@ -4084,6 +4099,20 @@ bool Type::isCARCBridgableType() const {
   return Pointee->isVoidType() || Pointee->isRecordType();
 }
 
+/// Check if the specified type is the CUDA device builtin surface type.
+bool Type::isCUDADeviceBuiltinSurfaceType() const {
+  if (const auto *RT = getAs<RecordType>())
+    return RT->getDecl()->hasAttr<CUDADeviceBuiltinSurfaceTypeAttr>();
+  return false;
+}
+
+/// Check if the specified type is the CUDA device builtin texture type.
+bool Type::isCUDADeviceBuiltinTextureType() const {
+  if (const auto *RT = getAs<RecordType>())
+    return RT->getDecl()->hasAttr<CUDADeviceBuiltinTextureTypeAttr>();
+  return false;
+}
+
 bool Type::hasSizedVLAType() const {
   if (!isVariablyModifiedType()) return false;
 
diff --git a/clang/lib/AST/TypeLoc.cpp b/clang/lib/AST/TypeLoc.cpp
index 665a86f2c1432..50391dba2a05a 100644
--- a/clang/lib/AST/TypeLoc.cpp
+++ b/clang/lib/AST/TypeLoc.cpp
@@ -404,6 +404,8 @@ TypeSpecifierType BuiltinTypeLoc::getWrittenTypeSpec() const {
 #include "clang/Basic/AArch64SVEACLETypes.def"
   case BuiltinType::BuiltinFn:
   case BuiltinType::OMPArraySection:
+  case BuiltinType::OMPArrayShaping:
+  case BuiltinType::OMPIterator:
     return TST_unspecified;
   }
 
diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
index 28244fea55b24..8df93e8c51de8 100644
--- a/clang/lib/AST/TypePrinter.cpp
+++ b/clang/lib/AST/TypePrinter.cpp
@@ -1739,13 +1739,13 @@ static void printTo(raw_ostream &OS, ArrayRef<TA> Args,
 
     OS << ArgString;
 
-    NeedSpace = (!ArgString.empty() && ArgString.back() == '>');
+    // If the last character of our string is '>', add another space to
+    // keep the two '>''s separate tokens.
+    NeedSpace = Policy.SplitTemplateClosers && !ArgString.empty() &&
+                ArgString.back() == '>';
     FirstArg = false;
   }
 
-  // If the last character of our string is '>', add another space to
-  // keep the two '>''s separate tokens. We don't *have* to do this in
-  // C++0x, but it's still good hygiene.
   if (NeedSpace)
     OS << ' ';
 
diff --git a/clang/lib/Analysis/RetainSummaryManager.cpp b/clang/lib/Analysis/RetainSummaryManager.cpp
index 00bc854a88049..9f45a8efe546f 100644
--- a/clang/lib/Analysis/RetainSummaryManager.cpp
+++ b/clang/lib/Analysis/RetainSummaryManager.cpp
@@ -146,7 +146,9 @@ static bool isSubclass(const Decl *D,
 }
 
 static bool isOSObjectSubclass(const Decl *D) {
-  return D && isSubclass(D, "OSMetaClassBase");
+  // OSSymbols are particular OSObjects that are allocated globally
+  // and therefore aren't really refcounted, so we ignore them.
+  return D && isSubclass(D, "OSMetaClassBase") && !isSubclass(D, "OSSymbol");
 }
 
 static bool isOSObjectDynamicCast(StringRef S) {
diff --git a/clang/lib/Analysis/ThreadSafety.cpp b/clang/lib/Analysis/ThreadSafety.cpp
index 252083f377d94..e0ff23df5ab4b 100644
--- a/clang/lib/Analysis/ThreadSafety.cpp
+++ b/clang/lib/Analysis/ThreadSafety.cpp
@@ -2139,12 +2139,14 @@ void BuildLockset::VisitDeclStmt(const DeclStmt *S) {
 
       // handle constructors that involve temporaries
       if (auto *EWC = dyn_cast<ExprWithCleanups>(E))
-        E = EWC->getSubExpr();
-      if (auto *ICE = dyn_cast<ImplicitCastExpr>(E))
-        if (ICE->getCastKind() == CK_NoOp)
-          E = ICE->getSubExpr();
+        E = EWC->getSubExpr()->IgnoreParens();
+      if (auto *CE = dyn_cast<CastExpr>(E))
+        if (CE->getCastKind() == CK_NoOp ||
+            CE->getCastKind() == CK_ConstructorConversion ||
+            CE->getCastKind() == CK_UserDefinedConversion)
+          E = CE->getSubExpr()->IgnoreParens();
       if (auto *BTE = dyn_cast<CXXBindTemporaryExpr>(E))
-        E = BTE->getSubExpr();
+        E = BTE->getSubExpr()->IgnoreParens();
 
       if (const auto *CE = dyn_cast<CXXConstructExpr>(E)) {
         const auto *CtorD = dyn_cast_or_null<NamedDecl>(CE->getConstructor());
diff --git a/clang/lib/Analysis/plugins/CheckerOptionHandling/CheckerOptionHandling.cpp b/clang/lib/Analysis/plugins/CheckerOptionHandling/CheckerOptionHandling.cpp
index 77de3630ae7ec..32fba9c93752c 100644
--- a/clang/lib/Analysis/plugins/CheckerOptionHandling/CheckerOptionHandling.cpp
+++ b/clang/lib/Analysis/plugins/CheckerOptionHandling/CheckerOptionHandling.cpp
@@ -21,7 +21,7 @@ void registerMyChecker(CheckerManager &Mgr) {
                << '\n';
 }
 
-bool shouldRegisterMyChecker(const LangOptions &LO) { return true; }
+bool shouldRegisterMyChecker(const CheckerManager &mgr) { return true; }
 
 } // end anonymous namespace
 
diff --git a/clang/lib/Basic/Attributes.cpp b/clang/lib/Basic/Attributes.cpp
index a860c9dba2ef8..ff6dbf870fcf4 100644
--- a/clang/lib/Basic/Attributes.cpp
+++ b/clang/lib/Basic/Attributes.cpp
@@ -85,12 +85,12 @@ static SmallString<64> normalizeName(const IdentifierInfo *Name,
   StringRef ScopeName = normalizeAttrScopeName(Scope, SyntaxUsed);
   StringRef AttrName = normalizeAttrName(Name, ScopeName, SyntaxUsed);
 
-  // Ensure that in the case of C++11 attributes, we look for '::foo' if it is
-  // unscoped.
   SmallString<64> FullName = ScopeName;
-  if (Scope || SyntaxUsed == AttributeCommonInfo::AS_CXX11 ||
-      SyntaxUsed == AttributeCommonInfo::AS_C2x)
+  if (!ScopeName.empty()) {
+    assert(SyntaxUsed == AttributeCommonInfo::AS_CXX11 ||
+           SyntaxUsed == AttributeCommonInfo::AS_C2x);
     FullName += "::";
+  }
   FullName += AttrName;
 
   return FullName;
diff --git a/clang/lib/Basic/Cuda.cpp b/clang/lib/Basic/Cuda.cpp
index e06d120c58bfc..74eb5473b71d2 100644
--- a/clang/lib/Basic/Cuda.cpp
+++ b/clang/lib/Basic/Cuda.cpp
@@ -362,7 +362,7 @@ CudaVersion MaxVersionForCudaArch(CudaArch A) {
   }
 }
 
-static CudaVersion ToCudaVersion(llvm::VersionTuple Version) {
+CudaVersion ToCudaVersion(llvm::VersionTuple Version) {
   int IVer =
       Version.getMajor() * 10 + Version.getMinor().getValueOr(0);
   switch(IVer) {
diff --git a/clang/lib/Basic/Module.cpp b/clang/lib/Basic/Module.cpp
index dd8f11101107a..5fd7d304f8f42 100644
--- a/clang/lib/Basic/Module.cpp
+++ b/clang/lib/Basic/Module.cpp
@@ -659,7 +659,7 @@ void VisibleModuleSet::setVisible(Module *M, SourceLocation Loc,
   VisitModule({M, nullptr});
 }
 
-ASTSourceDescriptor::ASTSourceDescriptor(const Module &M)
+ASTSourceDescriptor::ASTSourceDescriptor(Module &M)
     : Signature(M.Signature), ClangModule(&M) {
   if (M.Directory)
     Path = M.Directory->getName();
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index e8bf41dc07be2..7a7c66c8e5872 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -1025,6 +1025,8 @@ bool clang::isAllowedClauseForDirective(OpenMPDirectiveKind DKind,
       break;
     }
     break;
+  case OMPD_begin_declare_variant:
+  case OMPD_end_declare_variant:
   case OMPD_declare_target:
   case OMPD_end_declare_target:
   case OMPD_unknown:
@@ -1288,6 +1290,8 @@ void clang::getOpenMPCaptureRegions(
   case OMPD_end_declare_target:
   case OMPD_requires:
   case OMPD_declare_variant:
+  case OMPD_begin_declare_variant:
+  case OMPD_end_declare_variant:
     llvm_unreachable("OpenMP Directive is not allowed");
   case OMPD_unknown:
     llvm_unreachable("Unknown OpenMP directive");
diff --git a/clang/lib/Basic/TargetInfo.cpp b/clang/lib/Basic/TargetInfo.cpp
index 2330339bedfb2..2f1e044bb106d 100644
--- a/clang/lib/Basic/TargetInfo.cpp
+++ b/clang/lib/Basic/TargetInfo.cpp
@@ -133,6 +133,8 @@ TargetInfo::TargetInfo(const llvm::Triple &T) : TargetOpts(), Triple(T) {
   // Default to an unknown platform name.
   PlatformName = "unknown";
   PlatformMinVersion = VersionTuple();
+
+  MaxOpenCLWorkGroupSize = 1024;
 }
 
 // Out of line virtual dtor for TargetInfo.
diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp
index 74bb6900b19e3..8ceb7f2b515e4 100644
--- a/clang/lib/Basic/Targets/AArch64.cpp
+++ b/clang/lib/Basic/Targets/AArch64.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "AArch64.h"
+#include "clang/Basic/LangOptions.h"
 #include "clang/Basic/TargetBuiltins.h"
 #include "clang/Basic/TargetInfo.h"
 #include "llvm/ADT/ArrayRef.h"
@@ -121,15 +122,15 @@ bool AArch64TargetInfo::validateBranchProtection(StringRef Spec,
     return false;
 
   BPI.SignReturnAddr =
-      llvm::StringSwitch<CodeGenOptions::SignReturnAddressScope>(PBP.Scope)
-          .Case("non-leaf", CodeGenOptions::SignReturnAddressScope::NonLeaf)
-          .Case("all", CodeGenOptions::SignReturnAddressScope::All)
-          .Default(CodeGenOptions::SignReturnAddressScope::None);
+      llvm::StringSwitch<LangOptions::SignReturnAddressScopeKind>(PBP.Scope)
+          .Case("non-leaf", LangOptions::SignReturnAddressScopeKind::NonLeaf)
+          .Case("all", LangOptions::SignReturnAddressScopeKind::All)
+          .Default(LangOptions::SignReturnAddressScopeKind::None);
 
   if (PBP.Key == "a_key")
-    BPI.SignKey = CodeGenOptions::SignReturnAddressKeyValue::AKey;
+    BPI.SignKey = LangOptions::SignReturnAddressKeyKind::AKey;
   else
-    BPI.SignKey = CodeGenOptions::SignReturnAddressKeyValue::BKey;
+    BPI.SignKey = LangOptions::SignReturnAddressKeyKind::BKey;
 
   BPI.BranchTargetEnforcement = PBP.BranchTargetEnforcement;
   return true;
@@ -151,6 +152,7 @@ void AArch64TargetInfo::fillValidCPUList(
 
 void AArch64TargetInfo::getTargetDefinesARMV81A(const LangOptions &Opts,
                                                 MacroBuilder &Builder) const {
+  // FIXME: Armv8.1 makes __ARM_FEATURE_CRC32 mandatory. Handle it here.
   Builder.defineMacro("__ARM_FEATURE_QRDMX", "1");
 }
 
@@ -171,17 +173,26 @@ void AArch64TargetInfo::getTargetDefinesARMV83A(const LangOptions &Opts,
 void AArch64TargetInfo::getTargetDefinesARMV84A(const LangOptions &Opts,
                                                 MacroBuilder &Builder) const {
   // Also include the Armv8.3 defines
-  // FIXME: Armv8.4 makes some extensions mandatory. Handle them here.
+  // FIXME: Armv8.4 makes __ARM_FEATURE_ATOMICS, defined in GCC, mandatory.
+  // Add and handle it here.
   getTargetDefinesARMV83A(Opts, Builder);
 }
 
 void AArch64TargetInfo::getTargetDefinesARMV85A(const LangOptions &Opts,
                                                 MacroBuilder &Builder) const {
   // Also include the Armv8.4 defines
-  // FIXME: Armv8.5 makes some extensions mandatory. Handle them here.
   getTargetDefinesARMV84A(Opts, Builder);
 }
 
+void AArch64TargetInfo::getTargetDefinesARMV86A(const LangOptions &Opts,
+                                                MacroBuilder &Builder) const {
+  // Also include the Armv8.5 defines
+  // FIXME: Armv8.6 makes the following extensions mandatory:
+  // - __ARM_FEATURE_BF16
+  // - __ARM_FEATURE_MATMUL_INT8
+  // Handle them here.
+  getTargetDefinesARMV85A(Opts, Builder);
+}
 
 void AArch64TargetInfo::getTargetDefines(const LangOptions &Opts,
                                          MacroBuilder &Builder) const {
@@ -272,6 +283,27 @@ void AArch64TargetInfo::getTargetDefines(const LangOptions &Opts,
   if ((FPU & NeonMode) && HasFP16FML)
     Builder.defineMacro("__ARM_FEATURE_FP16FML", "1");
 
+  if (Opts.hasSignReturnAddress()) {
+    // Bitmask:
+    // 0: Protection using the A key
+    // 1: Protection using the B key
+    // 2: Protection including leaf functions
+    unsigned Value = 0;
+
+    if (Opts.isSignReturnAddressWithAKey())
+      Value |= (1 << 0);
+    else
+      Value |= (1 << 1);
+
+    if (Opts.isSignReturnAddressScopeAll())
+      Value |= (1 << 2);
+
+    Builder.defineMacro("__ARM_FEATURE_PAC_DEFAULT", std::to_string(Value));
+  }
+
+  if (Opts.BranchTargetEnforcement)
+    Builder.defineMacro("__ARM_FEATURE_BTI_DEFAULT", "1");
+
   switch (ArchKind) {
   default:
     break;
@@ -290,6 +322,9 @@ void AArch64TargetInfo::getTargetDefines(const LangOptions &Opts,
   case llvm::AArch64::ArchKind::ARMV8_5A:
     getTargetDefinesARMV85A(Opts, Builder);
     break;
+  case llvm::AArch64::ArchKind::ARMV8_6A:
+    getTargetDefinesARMV86A(Opts, Builder);
+    break;
   }
 
   // All of the __sync_(bool|val)_compare_and_swap_(1|2|4|8) builtins work.
@@ -344,6 +379,8 @@ bool AArch64TargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
       ArchKind = llvm::AArch64::ArchKind::ARMV8_4A;
     if (Feature == "+v8.5a")
       ArchKind = llvm::AArch64::ArchKind::ARMV8_5A;
+    if (Feature == "+v8.6a")
+      ArchKind = llvm::AArch64::ArchKind::ARMV8_6A;
     if (Feature == "+fullfp16")
       HasFullFP16 = true;
     if (Feature == "+dotprod")
diff --git a/clang/lib/Basic/Targets/AArch64.h b/clang/lib/Basic/Targets/AArch64.h
index 41970e5d716e7..bc73d195b7d4f 100644
--- a/clang/lib/Basic/Targets/AArch64.h
+++ b/clang/lib/Basic/Targets/AArch64.h
@@ -70,6 +70,8 @@ class LLVM_LIBRARY_VISIBILITY AArch64TargetInfo : public TargetInfo {
                                MacroBuilder &Builder) const;
   void getTargetDefinesARMV85A(const LangOptions &Opts,
                                MacroBuilder &Builder) const;
+  void getTargetDefinesARMV86A(const LangOptions &Opts,
+                               MacroBuilder &Builder) const;
   void getTargetDefines(const LangOptions &Opts,
                         MacroBuilder &Builder) const override;
 
diff --git a/clang/lib/Basic/Targets/AMDGPU.cpp b/clang/lib/Basic/Targets/AMDGPU.cpp
index 8017bc3fd5c4f..8e948f22a7366 100644
--- a/clang/lib/Basic/Targets/AMDGPU.cpp
+++ b/clang/lib/Basic/Targets/AMDGPU.cpp
@@ -133,7 +133,36 @@ const char *const AMDGPUTargetInfo::GCCRegNames[] = {
   "s113", "s114", "s115", "s116", "s117", "s118", "s119", "s120", "s121",
   "s122", "s123", "s124", "s125", "s126", "s127", "exec", "vcc", "scc",
   "m0", "flat_scratch", "exec_lo", "exec_hi", "vcc_lo", "vcc_hi",
-  "flat_scratch_lo", "flat_scratch_hi"
+  "flat_scratch_lo", "flat_scratch_hi",
+  "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7", "a8",
+  "a9", "a10", "a11", "a12", "a13", "a14", "a15", "a16", "a17",
+  "a18", "a19", "a20", "a21", "a22", "a23", "a24", "a25", "a26",
+  "a27", "a28", "a29", "a30", "a31", "a32", "a33", "a34", "a35",
+  "a36", "a37", "a38", "a39", "a40", "a41", "a42", "a43", "a44",
+  "a45", "a46", "a47", "a48", "a49", "a50", "a51", "a52", "a53",
+  "a54", "a55", "a56", "a57", "a58", "a59", "a60", "a61", "a62",
+  "a63", "a64", "a65", "a66", "a67", "a68", "a69", "a70", "a71",
+  "a72", "a73", "a74", "a75", "a76", "a77", "a78", "a79", "a80",
+  "a81", "a82", "a83", "a84", "a85", "a86", "a87", "a88", "a89",
+  "a90", "a91", "a92", "a93", "a94", "a95", "a96", "a97", "a98",
+  "a99", "a100", "a101", "a102", "a103", "a104", "a105", "a106", "a107",
+  "a108", "a109", "a110", "a111", "a112", "a113", "a114", "a115", "a116",
+  "a117", "a118", "a119", "a120", "a121", "a122", "a123", "a124", "a125",
+  "a126", "a127", "a128", "a129", "a130", "a131", "a132", "a133", "a134",
+  "a135", "a136", "a137", "a138", "a139", "a140", "a141", "a142", "a143",
+  "a144", "a145", "a146", "a147", "a148", "a149", "a150", "a151", "a152",
+  "a153", "a154", "a155", "a156", "a157", "a158", "a159", "a160", "a161",
+  "a162", "a163", "a164", "a165", "a166", "a167", "a168", "a169", "a170",
+  "a171", "a172", "a173", "a174", "a175", "a176", "a177", "a178", "a179",
+  "a180", "a181", "a182", "a183", "a184", "a185", "a186", "a187", "a188",
+  "a189", "a190", "a191", "a192", "a193", "a194", "a195", "a196", "a197",
+  "a198", "a199", "a200", "a201", "a202", "a203", "a204", "a205", "a206",
+  "a207", "a208", "a209", "a210", "a211", "a212", "a213", "a214", "a215",
+  "a216", "a217", "a218", "a219", "a220", "a221", "a222", "a223", "a224",
+  "a225", "a226", "a227", "a228", "a229", "a230", "a231", "a232", "a233",
+  "a234", "a235", "a236", "a237", "a238", "a239", "a240", "a241", "a242",
+  "a243", "a244", "a245", "a246", "a247", "a248", "a249", "a250", "a251",
+  "a252", "a253", "a254", "a255"
 };
 
 ArrayRef<const char *> AMDGPUTargetInfo::getGCCRegNames() const {
@@ -242,28 +271,6 @@ bool AMDGPUTargetInfo::initFeatureMap(
   return TargetInfo::initFeatureMap(Features, Diags, CPU, FeatureVec);
 }
 
-void AMDGPUTargetInfo::adjustTargetOptions(const CodeGenOptions &CGOpts,
-                                           TargetOptions &TargetOpts) const {
-  bool hasFP32Denormals = false;
-  bool hasFP64Denormals = false;
-
-  for (auto &I : TargetOpts.FeaturesAsWritten) {
-    if (I == "+fp32-denormals" || I == "-fp32-denormals")
-      hasFP32Denormals = true;
-    if (I == "+fp64-fp16-denormals" || I == "-fp64-fp16-denormals")
-      hasFP64Denormals = true;
-  }
-  if (!hasFP32Denormals)
-    TargetOpts.Features.push_back(
-      (Twine(hasFastFMAF() && hasFullRateDenormalsF32() &&
-             CGOpts.FP32DenormalMode.Output == llvm::DenormalMode::IEEE
-             ? '+' : '-') + Twine("fp32-denormals"))
-            .str());
-  // Always do not flush fp64 or fp16 denorms.
-  if (!hasFP64Denormals && hasFP64())
-    TargetOpts.Features.push_back("+fp64-fp16-denormals");
-}
-
 void AMDGPUTargetInfo::fillValidCPUList(
     SmallVectorImpl<StringRef> &Values) const {
   if (isAMDGCN(getTriple()))
diff --git a/clang/lib/Basic/Targets/AMDGPU.h b/clang/lib/Basic/Targets/AMDGPU.h
index ed6929214f6e0..548cadcab9ab8 100644
--- a/clang/lib/Basic/Targets/AMDGPU.h
+++ b/clang/lib/Basic/Targets/AMDGPU.h
@@ -114,11 +114,14 @@ class LLVM_LIBRARY_VISIBILITY AMDGPUTargetInfo final : public TargetInfo {
   /// Accepted register names: (n, m is unsigned integer, n < m)
   /// v
   /// s
+  /// a
   /// {vn}, {v[n]}
   /// {sn}, {s[n]}
+  /// {an}, {a[n]}
   /// {S} , where S is a special register name
   ////{v[n:m]}
   /// {s[n:m]}
+  /// {a[n:m]}
   bool validateAsmConstraint(const char *&Name,
                              TargetInfo::ConstraintInfo &Info) const override {
     static const ::llvm::StringSet<> SpecialRegs({
@@ -135,7 +138,7 @@ class LLVM_LIBRARY_VISIBILITY AMDGPUTargetInfo final : public TargetInfo {
     }
     if (S.empty())
       return false;
-    if (S.front() != 'v' && S.front() != 's') {
+    if (S.front() != 'v' && S.front() != 's' && S.front() != 'a') {
       if (!HasLeftParen)
         return false;
       auto E = S.find('}');
@@ -153,7 +156,7 @@ class LLVM_LIBRARY_VISIBILITY AMDGPUTargetInfo final : public TargetInfo {
     if (!HasLeftParen) {
       if (!S.empty())
         return false;
-      // Found s or v.
+      // Found s, v or a.
       Info.setAllowsRegister();
       Name = S.data() - 1;
       return true;
@@ -184,7 +187,8 @@ class LLVM_LIBRARY_VISIBILITY AMDGPUTargetInfo final : public TargetInfo {
     S = S.drop_front();
     if (!S.empty())
       return false;
-    // Found {vn}, {sn}, {v[n]}, {s[n]}, {v[n:m]}, or {s[n:m]}.
+    // Found {vn}, {sn}, {an}, {v[n]}, {s[n]}, {a[n]}, {v[n:m]}, {s[n:m]}
+    // or {a[n:m]}.
     Info.setAllowsRegister();
     Name = S.data() - 1;
     return true;
@@ -208,9 +212,6 @@ class LLVM_LIBRARY_VISIBILITY AMDGPUTargetInfo final : public TargetInfo {
                  StringRef CPU,
                  const std::vector<std::string> &FeatureVec) const override;
 
-  void adjustTargetOptions(const CodeGenOptions &CGOpts,
-                           TargetOptions &TargetOpts) const override;
-
   ArrayRef<Builtin::Info> getTargetBuiltins() const override;
 
   void getTargetDefines(const LangOptions &Opts,
diff --git a/clang/lib/Basic/Targets/ARM.cpp b/clang/lib/Basic/Targets/ARM.cpp
index 4ab794ef3c091..8815161815386 100644
--- a/clang/lib/Basic/Targets/ARM.cpp
+++ b/clang/lib/Basic/Targets/ARM.cpp
@@ -201,6 +201,8 @@ StringRef ARMTargetInfo::getCPUAttr() const {
     return "8_4A";
   case llvm::ARM::ArchKind::ARMV8_5A:
     return "8_5A";
+  case llvm::ARM::ArchKind::ARMV8_6A:
+    return "8_6A";
   case llvm::ARM::ArchKind::ARMV8MBaseline:
     return "8M_BASE";
   case llvm::ARM::ArchKind::ARMV8MMainline:
@@ -830,6 +832,7 @@ void ARMTargetInfo::getTargetDefines(const LangOptions &Opts,
   case llvm::ARM::ArchKind::ARMV8_3A:
   case llvm::ARM::ArchKind::ARMV8_4A:
   case llvm::ARM::ArchKind::ARMV8_5A:
+  case llvm::ARM::ArchKind::ARMV8_6A:
     getTargetDefinesARMV83A(Opts, Builder);
     break;
   }
diff --git a/clang/lib/Basic/Targets/BPF.h b/clang/lib/Basic/Targets/BPF.h
index b2f1831e960e6..43e55dfbfb2b2 100644
--- a/clang/lib/Basic/Targets/BPF.h
+++ b/clang/lib/Basic/Targets/BPF.h
@@ -35,9 +35,9 @@ class LLVM_LIBRARY_VISIBILITY BPFTargetInfo : public TargetInfo {
     Int64Type = SignedLong;
     RegParmMax = 5;
     if (Triple.getArch() == llvm::Triple::bpfeb) {
-      resetDataLayout("E-m:e-p:64:64-i64:64-n32:64-S128");
+      resetDataLayout("E-m:e-p:64:64-i64:64-i128:128-n32:64-S128");
     } else {
-      resetDataLayout("e-m:e-p:64:64-i64:64-n32:64-S128");
+      resetDataLayout("e-m:e-p:64:64-i64:64-i128:128-n32:64-S128");
     }
     MaxAtomicPromoteWidth = 64;
     MaxAtomicInlineWidth = 64;
diff --git a/clang/lib/Basic/Targets/Hexagon.h b/clang/lib/Basic/Targets/Hexagon.h
index f58f594b104fe..7e173df816839 100644
--- a/clang/lib/Basic/Targets/Hexagon.h
+++ b/clang/lib/Basic/Targets/Hexagon.h
@@ -57,6 +57,7 @@ class LLVM_LIBRARY_VISIBILITY HexagonTargetInfo : public TargetInfo {
     LargeArrayAlign = 64;
     UseBitFieldTypeAlignment = true;
     ZeroLengthBitfieldBoundary = 32;
+    MaxAtomicPromoteWidth = MaxAtomicInlineWidth = 64;
 
     // These are the default values anyway, but explicitly make sure
     // that the size of the boolean type is 8 bits. Bool vectors are used
@@ -103,6 +104,8 @@ class LLVM_LIBRARY_VISIBILITY HexagonTargetInfo : public TargetInfo {
                             DiagnosticsEngine &Diags) override;
 
   BuiltinVaListKind getBuiltinVaListKind() const override {
+    if (getTriple().isMusl())
+      return TargetInfo::HexagonBuiltinVaList;
     return TargetInfo::CharPtrBuiltinVaList;
   }
 
diff --git a/clang/lib/Basic/Targets/X86.cpp b/clang/lib/Basic/Targets/X86.cpp
index 7b575f52bb22d..def23a65edde7 100644
--- a/clang/lib/Basic/Targets/X86.cpp
+++ b/clang/lib/Basic/Targets/X86.cpp
@@ -857,6 +857,8 @@ bool X86TargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
       HasINVPCID = true;
     } else if (Feature == "+enqcmd") {
       HasENQCMD = true;
+    } else if (Feature == "+serialize") {
+      HasSERIALIZE = true;
     }
 
     X86SSEEnum Level = llvm::StringSwitch<X86SSEEnum>(Feature)
@@ -1247,6 +1249,8 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("__INVPCID__");
   if (HasENQCMD)
     Builder.defineMacro("__ENQCMD__");
+  if (HasSERIALIZE)
+    Builder.defineMacro("__SERIALIZE__");
 
   // Each case falls through to the previous one here.
   switch (SSELevel) {
@@ -1390,6 +1394,7 @@ bool X86TargetInfo::isValidFeatureName(StringRef Name) const {
       .Case("rdseed", true)
       .Case("rtm", true)
       .Case("sahf", true)
+      .Case("serialize", true)
       .Case("sgx", true)
       .Case("sha", true)
       .Case("shstk", true)
@@ -1474,6 +1479,7 @@ bool X86TargetInfo::hasFeature(StringRef Feature) const {
       .Case("retpoline-external-thunk", HasRetpolineExternalThunk)
       .Case("rtm", HasRTM)
       .Case("sahf", HasLAHFSAHF)
+      .Case("serialize", HasSERIALIZE)
       .Case("sgx", HasSGX)
       .Case("sha", HasSHA)
       .Case("shstk", HasSHSTK)
@@ -1731,6 +1737,120 @@ bool X86TargetInfo::validateAsmConstraint(
   }
 }
 
+// Below is based on the following information:
+// +------------------------------------+-------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------+
+// |           Processor Name           | Cache Line Size (Bytes) |                                                                            Source                                                                            |
+// +------------------------------------+-------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------+
+// | i386                               |                      64 | https://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-optimization-manual.pdf                                          |
+// | i486                               |                      16 | "four doublewords" (doubleword = 32 bits, 4 bits * 32 bits = 16 bytes) https://en.wikichip.org/w/images/d/d3/i486_MICROPROCESSOR_HARDWARE_REFERENCE_MANUAL_%281990%29.pdf and http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.126.4216&rep=rep1&type=pdf (page 29) |
+// | i586/Pentium MMX                   |                      32 | https://www.7-cpu.com/cpu/P-MMX.html                                                                                                                         |
+// | i686/Pentium                       |                      32 | https://www.7-cpu.com/cpu/P6.html                                                                                                                            |
+// | Netburst/Pentium4                  |                      64 | https://www.7-cpu.com/cpu/P4-180.html                                                                                                                        |
+// | Atom                               |                      64 | https://www.7-cpu.com/cpu/Atom.html                                                                                                                          |
+// | Westmere                           |                      64 | https://en.wikichip.org/wiki/intel/microarchitectures/sandy_bridge_(client) "Cache Architecture"                                                             |
+// | Sandy Bridge                       |                      64 | https://en.wikipedia.org/wiki/Sandy_Bridge and https://www.7-cpu.com/cpu/SandyBridge.html                                                                    |
+// | Ivy Bridge                         |                      64 | https://blog.stuffedcow.net/2013/01/ivb-cache-replacement/ and https://www.7-cpu.com/cpu/IvyBridge.html                                                      |
+// | Haswell                            |                      64 | https://www.7-cpu.com/cpu/Haswell.html                                                                                                                       |
+// | Boadwell                           |                      64 | https://www.7-cpu.com/cpu/Broadwell.html                                                                                                                     |
+// | Skylake (including skylake-avx512) |                      64 | https://www.nas.nasa.gov/hecc/support/kb/skylake-processors_550.html "Cache Hierarchy"                                                                       |
+// | Cascade Lake                       |                      64 | https://www.nas.nasa.gov/hecc/support/kb/cascade-lake-processors_579.html "Cache Hierarchy"                                                                  |
+// | Skylake                            |                      64 | https://en.wikichip.org/wiki/intel/microarchitectures/kaby_lake "Memory Hierarchy"                                                                           |
+// | Ice Lake                           |                      64 | https://www.7-cpu.com/cpu/Ice_Lake.html                                                                                                                      |
+// | Knights Landing                    |                      64 | https://software.intel.com/en-us/articles/intel-xeon-phi-processor-7200-family-memory-management-optimizations "The Intel® Xeon Phi™ Processor Architecture" |
+// | Knights Mill                       |                      64 | https://software.intel.com/sites/default/files/managed/9e/bc/64-ia-32-architectures-optimization-manual.pdf?countrylabel=Colombia "2.5.5.2 L1 DCache "       |
+// +------------------------------------+-------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------+
+Optional<unsigned> X86TargetInfo::getCPUCacheLineSize() const {
+  switch (CPU) {
+    // i386
+    case CK_i386:
+    // i486
+    case CK_i486:
+    case CK_WinChipC6:
+    case CK_WinChip2:
+    case CK_C3:
+    // Lakemont
+    case CK_Lakemont:
+      return 16;
+
+    // i586
+    case CK_i586:
+    case CK_Pentium:
+    case CK_PentiumMMX:
+    // i686
+    case CK_PentiumPro:
+    case CK_i686:
+    case CK_Pentium2:
+    case CK_Pentium3:
+    case CK_PentiumM:
+    case CK_C3_2:
+    // K6
+    case CK_K6:
+    case CK_K6_2:
+    case CK_K6_3:
+    // Geode
+    case CK_Geode:
+      return 32;
+
+    // Netburst
+    case CK_Pentium4:
+    case CK_Prescott:
+    case CK_Nocona:
+    // Atom
+    case CK_Bonnell:
+    case CK_Silvermont:
+    case CK_Goldmont:
+    case CK_GoldmontPlus:
+    case CK_Tremont:
+
+    case CK_Westmere:
+    case CK_SandyBridge:
+    case CK_IvyBridge:
+    case CK_Haswell:
+    case CK_Broadwell:
+    case CK_SkylakeClient:
+    case CK_SkylakeServer:
+    case CK_Cascadelake:
+    case CK_Nehalem:
+    case CK_Cooperlake:
+    case CK_Cannonlake:
+    case CK_Tigerlake:
+    case CK_IcelakeClient:
+    case CK_IcelakeServer:
+    case CK_KNL:
+    case CK_KNM:
+    // K7
+    case CK_Athlon:
+    case CK_AthlonXP:
+    // K8
+    case CK_K8:
+    case CK_K8SSE3:
+    case CK_AMDFAM10:
+    // Bobcat
+    case CK_BTVER1:
+    case CK_BTVER2:
+    // Bulldozer
+    case CK_BDVER1:
+    case CK_BDVER2:
+    case CK_BDVER3:
+    case CK_BDVER4:
+    // Zen
+    case CK_ZNVER1:
+    case CK_ZNVER2:
+    // Deprecated
+    case CK_x86_64:
+    case CK_Yonah:
+    case CK_Penryn:
+    case CK_Core2:
+      return 64;
+
+    // The following currently have unknown cache line sizes (but they are probably all 64):
+    // Core
+    case CK_Generic:
+      return None;
+  }
+  llvm_unreachable("Unknown CPU kind");
+}
+
 bool X86TargetInfo::validateOutputSize(const llvm::StringMap<bool> &FeatureMap,
                                        StringRef Constraint,
                                        unsigned Size) const {
diff --git a/clang/lib/Basic/Targets/X86.h b/clang/lib/Basic/Targets/X86.h
index 3166e2cce080b..155918dfc9335 100644
--- a/clang/lib/Basic/Targets/X86.h
+++ b/clang/lib/Basic/Targets/X86.h
@@ -129,6 +129,7 @@ class LLVM_LIBRARY_VISIBILITY X86TargetInfo : public TargetInfo {
   bool HasPTWRITE = false;
   bool HasINVPCID = false;
   bool HasENQCMD = false;
+  bool HasSERIALIZE = false;
 
 protected:
   /// Enumeration of all of the X86 CPUs supported by Clang.
@@ -187,6 +188,8 @@ class LLVM_LIBRARY_VISIBILITY X86TargetInfo : public TargetInfo {
       StringRef Name,
       llvm::SmallVectorImpl<StringRef> &Features) const override;
 
+  Optional<unsigned> getCPUCacheLineSize() const override;
+
   bool validateAsmConstraint(const char *&Name,
                              TargetInfo::ConstraintInfo &info) const override;
 
diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index b8a59bcdab391..14cc06fbef1ca 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -1117,6 +1117,7 @@ void EmitAssemblyHelper::EmitAssemblyWithNewPassManager(
   PTO.LoopInterleaving = CodeGenOpts.UnrollLoops;
   PTO.LoopVectorization = CodeGenOpts.VectorizeLoop;
   PTO.SLPVectorization = CodeGenOpts.VectorizeSLP;
+  PTO.CallGraphProfile = CodeGenOpts.CallGraphProfile;
   PTO.Coroutines = LangOpts.Coroutines;
 
   PassInstrumentationCallbacks PIC;
@@ -1530,6 +1531,7 @@ static void runThinLTOBackend(ModuleSummaryIndex *CombinedIndex, Module *M,
   Conf.PTO.LoopInterleaving = CGOpts.UnrollLoops;
   Conf.PTO.LoopVectorization = CGOpts.VectorizeLoop;
   Conf.PTO.SLPVectorization = CGOpts.VectorizeSLP;
+  Conf.PTO.CallGraphProfile = CGOpts.CallGraphProfile;
 
   // Context sensitive profile.
   if (CGOpts.hasProfileCSIRInstr()) {
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index b9327197c8423..50430197a70ab 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -13413,6 +13413,48 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
   }
 }
 
+namespace {
+// If \p E is not null pointer, insert address space cast to match return
+// type of \p E if necessary.
+Value *EmitAMDGPUDispatchPtr(CodeGenFunction &CGF,
+                             const CallExpr *E = nullptr) {
+  auto *F = CGF.CGM.getIntrinsic(Intrinsic::amdgcn_dispatch_ptr);
+  auto *Call = CGF.Builder.CreateCall(F);
+  Call->addAttribute(
+      AttributeList::ReturnIndex,
+      Attribute::getWithDereferenceableBytes(Call->getContext(), 64));
+  Call->addAttribute(AttributeList::ReturnIndex,
+                     Attribute::getWithAlignment(Call->getContext(), Align(4)));
+  if (!E)
+    return Call;
+  QualType BuiltinRetType = E->getType();
+  auto *RetTy = cast<llvm::PointerType>(CGF.ConvertType(BuiltinRetType));
+  if (RetTy == Call->getType())
+    return Call;
+  return CGF.Builder.CreateAddrSpaceCast(Call, RetTy);
+}
+
+// \p Index is 0, 1, and 2 for x, y, and z dimension, respectively.
+Value *EmitAMDGPUWorkGroupSize(CodeGenFunction &CGF, unsigned Index) {
+  const unsigned XOffset = 4;
+  auto *DP = EmitAMDGPUDispatchPtr(CGF);
+  // Indexing the HSA kernel_dispatch_packet struct.
+  auto *Offset = llvm::ConstantInt::get(CGF.Int32Ty, XOffset + Index * 2);
+  auto *GEP = CGF.Builder.CreateGEP(DP, Offset);
+  auto *DstTy =
+      CGF.Int16Ty->getPointerTo(GEP->getType()->getPointerAddressSpace());
+  auto *Cast = CGF.Builder.CreateBitCast(GEP, DstTy);
+  auto *LD = CGF.Builder.CreateLoad(Address(Cast, CharUnits::fromQuantity(2)));
+  llvm::MDBuilder MDHelper(CGF.getLLVMContext());
+  llvm::MDNode *RNode = MDHelper.createRange(APInt(16, 1),
+      APInt(16, CGF.getTarget().getMaxOpenCLWorkGroupSize() + 1));
+  LD->setMetadata(llvm::LLVMContext::MD_range, RNode);
+  LD->setMetadata(llvm::LLVMContext::MD_invariant_load,
+      llvm::MDNode::get(CGF.getLLVMContext(), None));
+  return LD;
+}
+} // namespace
+
 Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
                                               const CallExpr *E) {
   switch (BuiltinID) {
@@ -13495,21 +13537,8 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
   case AMDGPU::BI__builtin_amdgcn_cosf:
   case AMDGPU::BI__builtin_amdgcn_cosh:
     return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_cos);
-  case AMDGPU::BI__builtin_amdgcn_dispatch_ptr: {
-    auto *F = CGM.getIntrinsic(Intrinsic::amdgcn_dispatch_ptr);
-    auto *Call = Builder.CreateCall(F);
-    Call->addAttribute(
-        AttributeList::ReturnIndex,
-        Attribute::getWithDereferenceableBytes(Call->getContext(), 64));
-    Call->addAttribute(
-        AttributeList::ReturnIndex,
-        Attribute::getWithAlignment(Call->getContext(), Align(4)));
-    QualType BuiltinRetType = E->getType();
-    auto *RetTy = cast<llvm::PointerType>(ConvertType(BuiltinRetType));
-    if (RetTy == Call->getType())
-      return Call;
-    return Builder.CreateAddrSpaceCast(Call, RetTy);
-  }
+  case AMDGPU::BI__builtin_amdgcn_dispatch_ptr:
+    return EmitAMDGPUDispatchPtr(*this, E);
   case AMDGPU::BI__builtin_amdgcn_log_clampf:
     return emitUnaryBuiltin(*this, E, Intrinsic::amdgcn_log_clamp);
   case AMDGPU::BI__builtin_amdgcn_ldexp:
@@ -13605,6 +13634,14 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
   case AMDGPU::BI__builtin_amdgcn_workitem_id_z:
     return emitRangedBuiltin(*this, Intrinsic::amdgcn_workitem_id_z, 0, 1024);
 
+  // amdgcn workgroup size
+  case AMDGPU::BI__builtin_amdgcn_workgroup_size_x:
+    return EmitAMDGPUWorkGroupSize(*this, 0);
+  case AMDGPU::BI__builtin_amdgcn_workgroup_size_y:
+    return EmitAMDGPUWorkGroupSize(*this, 1);
+  case AMDGPU::BI__builtin_amdgcn_workgroup_size_z:
+    return EmitAMDGPUWorkGroupSize(*this, 2);
+
   // r600 intrinsics
   case AMDGPU::BI__builtin_r600_recipsqrt_ieee:
   case AMDGPU::BI__builtin_r600_recipsqrt_ieeef:
diff --git a/clang/lib/CodeGen/CGCUDANV.cpp b/clang/lib/CodeGen/CGCUDANV.cpp
index 5d8e545050d9b..351c5058aa4c9 100644
--- a/clang/lib/CodeGen/CGCUDANV.cpp
+++ b/clang/lib/CodeGen/CGCUDANV.cpp
@@ -50,7 +50,7 @@ class CGNVCUDARuntime : public CGCUDARuntime {
   struct VarInfo {
     llvm::GlobalVariable *Var;
     const VarDecl *D;
-    unsigned Flag;
+    DeviceVarFlags Flags;
   };
   llvm::SmallVector<VarInfo, 16> DeviceVars;
   /// Keeps track of variable containing handle of GPU binary. Populated by
@@ -124,8 +124,25 @@ class CGNVCUDARuntime : public CGCUDARuntime {
 
   void emitDeviceStub(CodeGenFunction &CGF, FunctionArgList &Args) override;
   void registerDeviceVar(const VarDecl *VD, llvm::GlobalVariable &Var,
-                         unsigned Flags) override {
-    DeviceVars.push_back({&Var, VD, Flags});
+                         bool Extern, bool Constant) override {
+    DeviceVars.push_back({&Var,
+                          VD,
+                          {DeviceVarFlags::Variable, Extern, Constant,
+                           /*Normalized*/ false, /*Type*/ 0}});
+  }
+  void registerDeviceSurf(const VarDecl *VD, llvm::GlobalVariable &Var,
+                          bool Extern, int Type) override {
+    DeviceVars.push_back({&Var,
+                          VD,
+                          {DeviceVarFlags::Surface, Extern, /*Constant*/ false,
+                           /*Normalized*/ false, Type}});
+  }
+  void registerDeviceTex(const VarDecl *VD, llvm::GlobalVariable &Var,
+                         bool Extern, int Type, bool Normalized) override {
+    DeviceVars.push_back({&Var,
+                          VD,
+                          {DeviceVarFlags::Texture, Extern, /*Constant*/ false,
+                           Normalized, Type}});
   }
 
   /// Creates module constructor function
@@ -423,30 +440,70 @@ llvm::Function *CGNVCUDARuntime::makeRegisterGlobalsFn() {
     Builder.CreateCall(RegisterFunc, Args);
   }
 
+  llvm::Type *VarSizeTy = IntTy;
+  // For HIP or CUDA 9.0+, device variable size is type of `size_t`.
+  if (CGM.getLangOpts().HIP ||
+      ToCudaVersion(CGM.getTarget().getSDKVersion()) >= CudaVersion::CUDA_90)
+    VarSizeTy = SizeTy;
+
   // void __cudaRegisterVar(void **, char *, char *, const char *,
   //                        int, int, int, int)
   llvm::Type *RegisterVarParams[] = {VoidPtrPtrTy, CharPtrTy, CharPtrTy,
-                                     CharPtrTy,    IntTy,     IntTy,
+                                     CharPtrTy,    IntTy,     VarSizeTy,
                                      IntTy,        IntTy};
   llvm::FunctionCallee RegisterVar = CGM.CreateRuntimeFunction(
-      llvm::FunctionType::get(IntTy, RegisterVarParams, false),
+      llvm::FunctionType::get(VoidTy, RegisterVarParams, false),
       addUnderscoredPrefixToName("RegisterVar"));
+  // void __cudaRegisterSurface(void **, const struct surfaceReference *,
+  //                            const void **, const char *, int, int);
+  llvm::FunctionCallee RegisterSurf = CGM.CreateRuntimeFunction(
+      llvm::FunctionType::get(
+          VoidTy, {VoidPtrPtrTy, VoidPtrTy, CharPtrTy, CharPtrTy, IntTy, IntTy},
+          false),
+      addUnderscoredPrefixToName("RegisterSurface"));
+  // void __cudaRegisterTexture(void **, const struct textureReference *,
+  //                            const void **, const char *, int, int, int)
+  llvm::FunctionCallee RegisterTex = CGM.CreateRuntimeFunction(
+      llvm::FunctionType::get(
+          VoidTy,
+          {VoidPtrPtrTy, VoidPtrTy, CharPtrTy, CharPtrTy, IntTy, IntTy, IntTy},
+          false),
+      addUnderscoredPrefixToName("RegisterTexture"));
   for (auto &&Info : DeviceVars) {
     llvm::GlobalVariable *Var = Info.Var;
-    unsigned Flags = Info.Flag;
     llvm::Constant *VarName = makeConstantString(getDeviceSideName(Info.D));
-    uint64_t VarSize =
-        CGM.getDataLayout().getTypeAllocSize(Var->getValueType());
-    llvm::Value *Args[] = {
-        &GpuBinaryHandlePtr,
-        Builder.CreateBitCast(Var, VoidPtrTy),
-        VarName,
-        VarName,
-        llvm::ConstantInt::get(IntTy, (Flags & ExternDeviceVar) ? 1 : 0),
-        llvm::ConstantInt::get(IntTy, VarSize),
-        llvm::ConstantInt::get(IntTy, (Flags & ConstantDeviceVar) ? 1 : 0),
-        llvm::ConstantInt::get(IntTy, 0)};
-    Builder.CreateCall(RegisterVar, Args);
+    switch (Info.Flags.getKind()) {
+    case DeviceVarFlags::Variable: {
+      uint64_t VarSize =
+          CGM.getDataLayout().getTypeAllocSize(Var->getValueType());
+      llvm::Value *Args[] = {
+          &GpuBinaryHandlePtr,
+          Builder.CreateBitCast(Var, VoidPtrTy),
+          VarName,
+          VarName,
+          llvm::ConstantInt::get(IntTy, Info.Flags.isExtern()),
+          llvm::ConstantInt::get(VarSizeTy, VarSize),
+          llvm::ConstantInt::get(IntTy, Info.Flags.isConstant()),
+          llvm::ConstantInt::get(IntTy, 0)};
+      Builder.CreateCall(RegisterVar, Args);
+      break;
+    }
+    case DeviceVarFlags::Surface:
+      Builder.CreateCall(
+          RegisterSurf,
+          {&GpuBinaryHandlePtr, Builder.CreateBitCast(Var, VoidPtrTy), VarName,
+           VarName, llvm::ConstantInt::get(IntTy, Info.Flags.getSurfTexType()),
+           llvm::ConstantInt::get(IntTy, Info.Flags.isExtern())});
+      break;
+    case DeviceVarFlags::Texture:
+      Builder.CreateCall(
+          RegisterTex,
+          {&GpuBinaryHandlePtr, Builder.CreateBitCast(Var, VoidPtrTy), VarName,
+           VarName, llvm::ConstantInt::get(IntTy, Info.Flags.getSurfTexType()),
+           llvm::ConstantInt::get(IntTy, Info.Flags.isNormalized()),
+           llvm::ConstantInt::get(IntTy, Info.Flags.isExtern())});
+      break;
+    }
   }
 
   Builder.CreateRetVoid();
diff --git a/clang/lib/CodeGen/CGCUDARuntime.h b/clang/lib/CodeGen/CGCUDARuntime.h
index 330e950c98eb9..19e70a2022a51 100644
--- a/clang/lib/CodeGen/CGCUDARuntime.h
+++ b/clang/lib/CodeGen/CGCUDARuntime.h
@@ -42,9 +42,30 @@ class CGCUDARuntime {
 
 public:
   // Global variable properties that must be passed to CUDA runtime.
-  enum DeviceVarFlags {
-    ExternDeviceVar = 0x01,   // extern
-    ConstantDeviceVar = 0x02, // __constant__
+  class DeviceVarFlags {
+  public:
+    enum DeviceVarKind {
+      Variable, // Variable
+      Surface,  // Builtin surface
+      Texture,  // Builtin texture
+    };
+
+  private:
+    unsigned Kind : 2;
+    unsigned Extern : 1;
+    unsigned Constant : 1;   // Constant variable.
+    unsigned Normalized : 1; // Normalized texture.
+    int SurfTexType;         // Type of surface/texutre.
+
+  public:
+    DeviceVarFlags(DeviceVarKind K, bool E, bool C, bool N, int T)
+        : Kind(K), Extern(E), Constant(C), Normalized(N), SurfTexType(T) {}
+
+    DeviceVarKind getKind() const { return static_cast<DeviceVarKind>(Kind); }
+    bool isExtern() const { return Extern; }
+    bool isConstant() const { return Constant; }
+    bool isNormalized() const { return Normalized; }
+    int getSurfTexType() const { return SurfTexType; }
   };
 
   CGCUDARuntime(CodeGenModule &CGM) : CGM(CGM) {}
@@ -57,7 +78,11 @@ class CGCUDARuntime {
   /// Emits a kernel launch stub.
   virtual void emitDeviceStub(CodeGenFunction &CGF, FunctionArgList &Args) = 0;
   virtual void registerDeviceVar(const VarDecl *VD, llvm::GlobalVariable &Var,
-                                 unsigned Flags) = 0;
+                                 bool Extern, bool Constant) = 0;
+  virtual void registerDeviceSurf(const VarDecl *VD, llvm::GlobalVariable &Var,
+                                  bool Extern, int Type) = 0;
+  virtual void registerDeviceTex(const VarDecl *VD, llvm::GlobalVariable &Var,
+                                 bool Extern, int Type, bool Normalized) = 0;
 
   /// Constructs and returns a module initialization function or nullptr if it's
   /// not needed. Must be called after all kernels have been emitted.
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index da6cb458982ba..6d3c2ad66cdca 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -1817,9 +1817,10 @@ CGDebugInfo::CollectTemplateParams(const TemplateParameterList *TPList,
         if (auto *templateType =
                 dyn_cast_or_null<NonTypeTemplateParmDecl>(TPList->getParam(i)))
           if (templateType->hasDefaultArgument())
-            defaultParameter =
+            defaultParameter = llvm::APSInt::isSameValue(
                 templateType->getDefaultArgument()->EvaluateKnownConstInt(
-                    CGM.getContext()) == TA.getAsIntegral();
+                    CGM.getContext()),
+                TA.getAsIntegral());
 
       TemplateParams.push_back(DBuilder.createTemplateValueParameter(
           TheCU, Name, TTy, defaultParameter,
@@ -2483,6 +2484,17 @@ llvm::DIModule *CGDebugInfo::getOrCreateModuleRef(ASTSourceDescriptor Mod,
     assert(StringRef(M->Name).startswith(CGM.getLangOpts().ModuleName) &&
            "clang module without ASTFile must be specified by -fmodule-name");
 
+  // Return a StringRef to the remapped Path.
+  auto RemapPath = [this](StringRef Path) -> std::string {
+    std::string Remapped = remapDIPath(Path);
+    StringRef Relative(Remapped);
+    StringRef CompDir = TheCU->getDirectory();
+    if (Relative.consume_front(CompDir))
+      Relative.consume_front(llvm::sys::path::get_separator());
+
+    return Relative.str();
+  };
+
   if (CreateSkeletonCU && IsRootModule && !Mod.getASTFile().empty()) {
     // PCH files don't have a signature field in the control block,
     // but LLVM detects skeleton CUs by looking for a non-zero DWO id.
@@ -2496,16 +2508,12 @@ llvm::DIModule *CGDebugInfo::getOrCreateModuleRef(ASTSourceDescriptor Mod,
     if (!llvm::sys::path::is_absolute(Mod.getASTFile()))
       PCM = Mod.getPath();
     llvm::sys::path::append(PCM, Mod.getASTFile());
-    std::string RemappedPCM = remapDIPath(PCM);
-    StringRef RelativePCM(RemappedPCM);
-    StringRef CompDir = TheCU->getDirectory();
-    if (RelativePCM.consume_front(CompDir))
-      RelativePCM.consume_front(llvm::sys::path::get_separator());
-    DIB.createCompileUnit(TheCU->getSourceLanguage(),
-                          // TODO: Support "Source" from external AST providers?
-                          DIB.createFile(Mod.getModuleName(), CompDir),
-                          TheCU->getProducer(), false, StringRef(), 0, RelativePCM,
-                          llvm::DICompileUnit::FullDebug, Signature);
+    DIB.createCompileUnit(
+        TheCU->getSourceLanguage(),
+        // TODO: Support "Source" from external AST providers?
+        DIB.createFile(Mod.getModuleName(), TheCU->getDirectory()),
+        TheCU->getProducer(), false, StringRef(), 0, RemapPath(PCM),
+        llvm::DICompileUnit::FullDebug, Signature);
     DIB.finalize();
   }
 
@@ -2513,9 +2521,10 @@ llvm::DIModule *CGDebugInfo::getOrCreateModuleRef(ASTSourceDescriptor Mod,
       IsRootModule ? nullptr
                    : getOrCreateModuleRef(ASTSourceDescriptor(*M->Parent),
                                           CreateSkeletonCU);
+  std::string IncludePath = Mod.getPath().str();
   llvm::DIModule *DIMod =
       DBuilder.createModule(Parent, Mod.getModuleName(), ConfigMacros,
-                            Mod.getPath());
+                            RemapPath(IncludePath));
   ModuleCache[M].reset(DIMod);
   return DIMod;
 }
diff --git a/clang/lib/CodeGen/CGDecl.cpp b/clang/lib/CodeGen/CGDecl.cpp
index 74a63ea9b899b..f15049ab0b5df 100644
--- a/clang/lib/CodeGen/CGDecl.cpp
+++ b/clang/lib/CodeGen/CGDecl.cpp
@@ -1102,7 +1102,9 @@ static llvm::Constant *constWithPadding(CodeGenModule &CGM, IsPattern isPattern,
     auto *NewArrayTy = llvm::ArrayType::get(NewElemTy, Size);
     return llvm::ConstantArray::get(NewArrayTy, Values);
   }
-  // FIXME: Do we need to handle tail padding in vectors?
+  // FIXME: Add handling for tail padding in vectors. Vectors don't
+  // have padding between or inside elements, but the total amount of
+  // data can be less than the allocated size.
   return constant;
 }
 
diff --git a/clang/lib/CodeGen/CGDeclCXX.cpp b/clang/lib/CodeGen/CGDeclCXX.cpp
index 0c6f0e71942fb..e501e53e4cb1b 100644
--- a/clang/lib/CodeGen/CGDeclCXX.cpp
+++ b/clang/lib/CodeGen/CGDeclCXX.cpp
@@ -16,7 +16,7 @@
 #include "CodeGenFunction.h"
 #include "TargetInfo.h"
 #include "clang/AST/Attr.h"
-#include "clang/Basic/CodeGenOptions.h"
+#include "clang/Basic/LangOptions.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/MDBuilder.h"
@@ -396,20 +396,20 @@ llvm::Function *CodeGenModule::CreateGlobalInitOrDestructFunction(
       !isInSanitizerBlacklist(SanitizerKind::ShadowCallStack, Fn, Loc))
     Fn->addFnAttr(llvm::Attribute::ShadowCallStack);
 
-  auto RASignKind = getCodeGenOpts().getSignReturnAddress();
-  if (RASignKind != CodeGenOptions::SignReturnAddressScope::None) {
+  auto RASignKind = getLangOpts().getSignReturnAddressScope();
+  if (RASignKind != LangOptions::SignReturnAddressScopeKind::None) {
     Fn->addFnAttr("sign-return-address",
-                  RASignKind == CodeGenOptions::SignReturnAddressScope::All
+                  RASignKind == LangOptions::SignReturnAddressScopeKind::All
                       ? "all"
                       : "non-leaf");
-    auto RASignKey = getCodeGenOpts().getSignReturnAddressKey();
+    auto RASignKey = getLangOpts().getSignReturnAddressKey();
     Fn->addFnAttr("sign-return-address-key",
-                  RASignKey == CodeGenOptions::SignReturnAddressKeyValue::AKey
+                  RASignKey == LangOptions::SignReturnAddressKeyKind::AKey
                       ? "a_key"
                       : "b_key");
   }
 
-  if (getCodeGenOpts().BranchTargetEnforcement)
+  if (getLangOpts().BranchTargetEnforcement)
     Fn->addFnAttr("branch-target-enforcement");
 
   return Fn;
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 4b5b2a5454062..1d4ba22d14c3f 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -1674,7 +1674,7 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile,
 
         // Shuffle vector to get vec3.
         V = Builder.CreateShuffleVector(V, llvm::UndefValue::get(vec4Ty),
-                                        {0, 1, 2}, "extractVec");
+                                        ArrayRef<int>{0, 1, 2}, "extractVec");
         return EmitFromMemory(V, Ty);
       }
     }
diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp
index df576decd69d6..fa2d228b7eeb0 100644
--- a/clang/lib/CodeGen/CGExprAgg.cpp
+++ b/clang/lib/CodeGen/CGExprAgg.cpp
@@ -15,6 +15,7 @@
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
 #include "ConstantEmitter.h"
+#include "TargetInfo.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Attr.h"
 #include "clang/AST/DeclCXX.h"
@@ -1946,6 +1947,18 @@ void CodeGenFunction::EmitAggregateCopy(LValue Dest, LValue Src, QualType Ty,
     }
   }
 
+  if (getLangOpts().CUDAIsDevice) {
+    if (Ty->isCUDADeviceBuiltinSurfaceType()) {
+      if (getTargetHooks().emitCUDADeviceBuiltinSurfaceDeviceCopy(*this, Dest,
+                                                                  Src))
+        return;
+    } else if (Ty->isCUDADeviceBuiltinTextureType()) {
+      if (getTargetHooks().emitCUDADeviceBuiltinTextureDeviceCopy(*this, Dest,
+                                                                  Src))
+        return;
+    }
+  }
+
   // Aggregate assignment turns into llvm.memcpy.  This is almost valid per
   // C99 6.5.16.1p3, which states "If the value being stored in an object is
   // read from another object that overlaps in anyway the storage of the first
diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp
index e17c1c5f7ac4f..da5d778a4922b 100644
--- a/clang/lib/CodeGen/CGExprConstant.cpp
+++ b/clang/lib/CodeGen/CGExprConstant.cpp
@@ -589,19 +589,21 @@ bool ConstStructBuilder::AppendBytes(CharUnits FieldOffsetInChars,
 bool ConstStructBuilder::AppendBitField(
     const FieldDecl *Field, uint64_t FieldOffset, llvm::ConstantInt *CI,
     bool AllowOverwrite) {
-  uint64_t FieldSize = Field->getBitWidthValue(CGM.getContext());
+  const CGRecordLayout &RL =
+      CGM.getTypes().getCGRecordLayout(Field->getParent());
+  const CGBitFieldInfo &Info = RL.getBitFieldInfo(Field);
   llvm::APInt FieldValue = CI->getValue();
 
   // Promote the size of FieldValue if necessary
   // FIXME: This should never occur, but currently it can because initializer
   // constants are cast to bool, and because clang is not enforcing bitfield
   // width limits.
-  if (FieldSize > FieldValue.getBitWidth())
-    FieldValue = FieldValue.zext(FieldSize);
+  if (Info.Size > FieldValue.getBitWidth())
+    FieldValue = FieldValue.zext(Info.Size);
 
   // Truncate the size of FieldValue to the bit field size.
-  if (FieldSize < FieldValue.getBitWidth())
-    FieldValue = FieldValue.trunc(FieldSize);
+  if (Info.Size < FieldValue.getBitWidth())
+    FieldValue = FieldValue.trunc(Info.Size);
 
   return Builder.addBits(FieldValue,
                          CGM.getContext().toBits(StartOffset) + FieldOffset,
diff --git a/clang/lib/CodeGen/CGObjCGNU.cpp b/clang/lib/CodeGen/CGObjCGNU.cpp
index db78309e9fd9f..35b9268084922 100644
--- a/clang/lib/CodeGen/CGObjCGNU.cpp
+++ b/clang/lib/CodeGen/CGObjCGNU.cpp
@@ -617,6 +617,13 @@ class CGObjCGNU : public CGObjCRuntime {
   llvm::Value *GenerateProtocolRef(CodeGenFunction &CGF,
                                    const ObjCProtocolDecl *PD) override;
   void GenerateProtocol(const ObjCProtocolDecl *PD) override;
+
+  virtual llvm::Constant *GenerateProtocolRef(const ObjCProtocolDecl *PD);
+
+  llvm::Constant *GetOrEmitProtocol(const ObjCProtocolDecl *PD) override {
+    return GenerateProtocolRef(PD);
+  }
+
   llvm::Function *ModuleInitFunction() override;
   llvm::FunctionCallee GetPropertyGetFunction() override;
   llvm::FunctionCallee GetPropertySetFunction() override;
@@ -1348,7 +1355,7 @@ class CGObjCGNUstep2 : public CGObjCGNUstep {
   void GenerateProtocol(const ObjCProtocolDecl *PD) override {
     // Do nothing - we only emit referenced protocols.
   }
-  llvm::Constant *GenerateProtocolRef(const ObjCProtocolDecl *PD) {
+  llvm::Constant *GenerateProtocolRef(const ObjCProtocolDecl *PD) override {
     std::string ProtocolName = PD->getNameAsString();
     auto *&Protocol = ExistingProtocols[ProtocolName];
     if (Protocol)
@@ -3039,13 +3046,18 @@ CGObjCGNU::GenerateProtocolList(ArrayRef<std::string> Protocols) {
 
 llvm::Value *CGObjCGNU::GenerateProtocolRef(CodeGenFunction &CGF,
                                             const ObjCProtocolDecl *PD) {
+  auto protocol = GenerateProtocolRef(PD);
+  llvm::Type *T =
+      CGM.getTypes().ConvertType(CGM.getContext().getObjCProtoType());
+  return CGF.Builder.CreateBitCast(protocol, llvm::PointerType::getUnqual(T));
+}
+
+llvm::Constant *CGObjCGNU::GenerateProtocolRef(const ObjCProtocolDecl *PD) {
   llvm::Constant *&protocol = ExistingProtocols[PD->getNameAsString()];
   if (!protocol)
     GenerateProtocol(PD);
   assert(protocol && "Unknown protocol");
-  llvm::Type *T =
-    CGM.getTypes().ConvertType(CGM.getContext().getObjCProtoType());
-  return CGF.Builder.CreateBitCast(protocol, llvm::PointerType::getUnqual(T));
+  return protocol;
 }
 
 llvm::Constant *
diff --git a/clang/lib/CodeGen/CGObjCMac.cpp b/clang/lib/CodeGen/CGObjCMac.cpp
index 87fd51b5d8b11..3986310eaa706 100644
--- a/clang/lib/CodeGen/CGObjCMac.cpp
+++ b/clang/lib/CodeGen/CGObjCMac.cpp
@@ -1107,11 +1107,6 @@ class CGObjCCommonMac : public CodeGen::CGObjCRuntime {
 
   void GenerateProtocol(const ObjCProtocolDecl *PD) override;
 
-  /// GetOrEmitProtocol - Get the protocol object for the given
-  /// declaration, emitting it if necessary. The return value has type
-  /// ProtocolPtrTy.
-  virtual llvm::Constant *GetOrEmitProtocol(const ObjCProtocolDecl *PD)=0;
-
   /// GetOrEmitProtocolRef - Get a forward reference to the protocol
   /// object for the given declaration, emitting it if needed. These
   /// forward references will be filled in with empty bodies if no
diff --git a/clang/lib/CodeGen/CGObjCRuntime.cpp b/clang/lib/CodeGen/CGObjCRuntime.cpp
index c34758c7e3b3f..39efe040302d3 100644
--- a/clang/lib/CodeGen/CGObjCRuntime.cpp
+++ b/clang/lib/CodeGen/CGObjCRuntime.cpp
@@ -13,14 +13,15 @@
 //===----------------------------------------------------------------------===//
 
 #include "CGObjCRuntime.h"
-#include "CGCleanup.h"
 #include "CGCXXABI.h"
+#include "CGCleanup.h"
 #include "CGRecordLayout.h"
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
 #include "clang/AST/RecordLayout.h"
 #include "clang/AST/StmtObjC.h"
 #include "clang/CodeGen/CGFunctionInfo.h"
+#include "clang/CodeGen/CodeGenABITypes.h"
 #include "llvm/Support/SaveAndRestore.h"
 
 using namespace clang;
@@ -383,3 +384,9 @@ CGObjCRuntime::getMessageSendInfo(const ObjCMethodDecl *method,
     CGM.getTypes().GetFunctionType(argsInfo)->getPointerTo();
   return MessageSendInfo(argsInfo, signatureType);
 }
+
+llvm::Constant *
+clang::CodeGen::emitObjCProtocolObject(CodeGenModule &CGM,
+                                       const ObjCProtocolDecl *protocol) {
+  return CGM.getObjCRuntime().GetOrEmitProtocol(protocol);
+}
diff --git a/clang/lib/CodeGen/CGObjCRuntime.h b/clang/lib/CodeGen/CGObjCRuntime.h
index f0b3525cfde25..a2c189585f7bc 100644
--- a/clang/lib/CodeGen/CGObjCRuntime.h
+++ b/clang/lib/CodeGen/CGObjCRuntime.h
@@ -211,6 +211,11 @@ class CGObjCRuntime {
   /// implementations.
   virtual void GenerateProtocol(const ObjCProtocolDecl *OPD) = 0;
 
+  /// GetOrEmitProtocol - Get the protocol object for the given
+  /// declaration, emitting it if necessary. The return value has type
+  /// ProtocolPtrTy.
+  virtual llvm::Constant *GetOrEmitProtocol(const ObjCProtocolDecl *PD) = 0;
+
   /// Generate a function preamble for a method with the specified
   /// types.
 
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 58c5334776f3f..6642851a56bc6 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -1275,52 +1275,6 @@ CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM, StringRef FirstSeparator,
   loadOffloadInfoMetadata();
 }
 
-bool CGOpenMPRuntime::tryEmitDeclareVariant(const GlobalDecl &NewGD,
-                                            const GlobalDecl &OldGD,
-                                            llvm::GlobalValue *OrigAddr,
-                                            bool IsForDefinition) {
-  // Emit at least a definition for the aliasee if the the address of the
-  // original function is requested.
-  if (IsForDefinition || OrigAddr)
-    (void)CGM.GetAddrOfGlobal(NewGD);
-  StringRef NewMangledName = CGM.getMangledName(NewGD);
-  llvm::GlobalValue *Addr = CGM.GetGlobalValue(NewMangledName);
-  if (Addr && !Addr->isDeclaration()) {
-    const auto *D = cast<FunctionDecl>(OldGD.getDecl());
-    const CGFunctionInfo &FI = CGM.getTypes().arrangeGlobalDeclaration(NewGD);
-    llvm::Type *DeclTy = CGM.getTypes().GetFunctionType(FI);
-
-    // Create a reference to the named value.  This ensures that it is emitted
-    // if a deferred decl.
-    llvm::GlobalValue::LinkageTypes LT = CGM.getFunctionLinkage(OldGD);
-
-    // Create the new alias itself, but don't set a name yet.
-    auto *GA =
-        llvm::GlobalAlias::create(DeclTy, 0, LT, "", Addr, &CGM.getModule());
-
-    if (OrigAddr) {
-      assert(OrigAddr->isDeclaration() && "Expected declaration");
-
-      GA->takeName(OrigAddr);
-      OrigAddr->replaceAllUsesWith(
-          llvm::ConstantExpr::getBitCast(GA, OrigAddr->getType()));
-      OrigAddr->eraseFromParent();
-    } else {
-      GA->setName(CGM.getMangledName(OldGD));
-    }
-
-    // Set attributes which are particular to an alias; this is a
-    // specialization of the attributes which may be set on a global function.
-    if (D->hasAttr<WeakAttr>() || D->hasAttr<WeakRefAttr>() ||
-        D->isWeakImported())
-      GA->setLinkage(llvm::Function::WeakAnyLinkage);
-
-    CGM.SetCommonAttributes(OldGD, GA);
-    return true;
-  }
-  return false;
-}
-
 void CGOpenMPRuntime::clear() {
   InternalVars.clear();
   // Clean non-target variable declarations possibly used only in debug info.
@@ -1334,14 +1288,6 @@ void CGOpenMPRuntime::clear() {
       continue;
     GV->eraseFromParent();
   }
-  // Emit aliases for the deferred aliasees.
-  for (const auto &Pair : DeferredVariantFunction) {
-    StringRef MangledName = CGM.getMangledName(Pair.second.second);
-    llvm::GlobalValue *Addr = CGM.GetGlobalValue(MangledName);
-    // If not able to emit alias, just emit original declaration.
-    (void)tryEmitDeclareVariant(Pair.second.first, Pair.second.second, Addr,
-                                /*IsForDefinition=*/false);
-  }
 }
 
 std::string CGOpenMPRuntime::getName(ArrayRef<StringRef> Parts) const {
@@ -5417,17 +5363,32 @@ std::pair<llvm::Value *, Address> CGOpenMPRuntime::emitDependClause(
       if (Dependencies[I].first == OMPC_DEPEND_depobj)
         continue;
       const Expr *E = Dependencies[I].second;
-      LValue Addr = CGF.EmitLValue(E);
+      const auto *OASE = dyn_cast<OMPArrayShapingExpr>(E);
+      llvm::Value *Addr;
+      if (OASE) {
+        const Expr *Base = OASE->getBase();
+        Addr = CGF.EmitScalarExpr(Base);
+      } else {
+        Addr = CGF.EmitLValue(E).getPointer(CGF);
+      }
       llvm::Value *Size;
       QualType Ty = E->getType();
-      if (const auto *ASE =
-              dyn_cast<OMPArraySectionExpr>(E->IgnoreParenImpCasts())) {
+      if (OASE) {
+        Size = CGF.getTypeSize(OASE->getBase()->getType()->getPointeeType());
+        for (const Expr *SE : OASE->getDimensions()) {
+           llvm::Value *Sz = CGF.EmitScalarExpr(SE);
+           Sz = CGF.EmitScalarConversion(Sz, SE->getType(),
+                                    CGF.getContext().getSizeType(),
+                                    SE->getExprLoc());
+           Size = CGF.Builder.CreateNUWMul(Size, Sz);
+        }
+      } else if (const auto *ASE =
+                     dyn_cast<OMPArraySectionExpr>(E->IgnoreParenImpCasts())) {
         LValue UpAddrLVal =
             CGF.EmitOMPArraySectionExpr(ASE, /*IsLowerBound=*/false);
         llvm::Value *UpAddr = CGF.Builder.CreateConstGEP1_32(
             UpAddrLVal.getPointer(CGF), /*Idx0=*/1);
-        llvm::Value *LowIntPtr =
-            CGF.Builder.CreatePtrToInt(Addr.getPointer(CGF), CGM.SizeTy);
+        llvm::Value *LowIntPtr = CGF.Builder.CreatePtrToInt(Addr, CGM.SizeTy);
         llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGM.SizeTy);
         Size = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
       } else {
@@ -5446,9 +5407,8 @@ std::pair<llvm::Value *, Address> CGOpenMPRuntime::emitDependClause(
       // deps[i].base_addr = &<Dependencies[i].second>;
       LValue BaseAddrLVal = CGF.EmitLValueForField(
           Base, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
-      CGF.EmitStoreOfScalar(
-          CGF.Builder.CreatePtrToInt(Addr.getPointer(CGF), CGF.IntPtrTy),
-          BaseAddrLVal);
+      CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy),
+                            BaseAddrLVal);
       // deps[i].len = sizeof(<Dependencies[i].second>);
       LValue LenLVal = CGF.EmitLValueForField(
           Base, *std::next(KmpDependInfoRD->field_begin(), Len));
@@ -7008,6 +6968,8 @@ emitNumTeamsForTargetDirective(CodeGenFunction &CGF,
   case OMPD_target_update:
   case OMPD_declare_simd:
   case OMPD_declare_variant:
+  case OMPD_begin_declare_variant:
+  case OMPD_end_declare_variant:
   case OMPD_declare_target:
   case OMPD_end_declare_target:
   case OMPD_declare_reduction:
@@ -7321,6 +7283,8 @@ emitNumThreadsForTargetDirective(CodeGenFunction &CGF,
   case OMPD_target_update:
   case OMPD_declare_simd:
   case OMPD_declare_variant:
+  case OMPD_begin_declare_variant:
+  case OMPD_end_declare_variant:
   case OMPD_declare_target:
   case OMPD_end_declare_target:
   case OMPD_declare_reduction:
@@ -7484,6 +7448,20 @@ class MappableExprsHandler {
   llvm::Value *getExprTypeSize(const Expr *E) const {
     QualType ExprTy = E->getType().getCanonicalType();
 
+    // Calculate the size for array shaping expression.
+    if (const auto *OAE = dyn_cast<OMPArrayShapingExpr>(E)) {
+      llvm::Value *Size =
+          CGF.getTypeSize(OAE->getBase()->getType()->getPointeeType());
+      for (const Expr *SE : OAE->getDimensions()) {
+        llvm::Value *Sz = CGF.EmitScalarExpr(SE);
+        Sz = CGF.EmitScalarConversion(Sz, SE->getType(),
+                                      CGF.getContext().getSizeType(),
+                                      SE->getExprLoc());
+        Size = CGF.Builder.CreateNUWMul(Size, Sz);
+      }
+      return Size;
+    }
+
     // Reference types are ignored for mapping purposes.
     if (const auto *RefTy = ExprTy->getAs<ReferenceType>())
       ExprTy = RefTy->getPointeeType().getCanonicalType();
@@ -7815,6 +7793,7 @@ class MappableExprsHandler {
     const Expr *AssocExpr = I->getAssociatedExpression();
     const auto *AE = dyn_cast<ArraySubscriptExpr>(AssocExpr);
     const auto *OASE = dyn_cast<OMPArraySectionExpr>(AssocExpr);
+    const auto *OAShE = dyn_cast<OMPArrayShapingExpr>(AssocExpr);
 
     if (isa<MemberExpr>(AssocExpr)) {
       // The base is the 'this' pointer. The content of the pointer is going
@@ -7824,6 +7803,11 @@ class MappableExprsHandler {
                (OASE &&
                 isa<CXXThisExpr>(OASE->getBase()->IgnoreParenImpCasts()))) {
       BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress(CGF);
+    } else if (OAShE &&
+               isa<CXXThisExpr>(OAShE->getBase()->IgnoreParenCasts())) {
+      BP = Address(
+          CGF.EmitScalarExpr(OAShE->getBase()),
+          CGF.getContext().getTypeAlignInChars(OAShE->getBase()->getType()));
     } else {
       // The base is the reference to the variable.
       // BP = &Var.
@@ -7906,9 +7890,12 @@ class MappableExprsHandler {
       // types.
       const auto *OASE =
           dyn_cast<OMPArraySectionExpr>(I->getAssociatedExpression());
+      const auto *OAShE =
+          dyn_cast<OMPArrayShapingExpr>(I->getAssociatedExpression());
       const auto *UO = dyn_cast<UnaryOperator>(I->getAssociatedExpression());
       const auto *BO = dyn_cast<BinaryOperator>(I->getAssociatedExpression());
       bool IsPointer =
+          OAShE ||
           (OASE && OMPArraySectionExpr::getBaseOriginalType(OASE)
                        .getCanonicalType()
                        ->isAnyPointerType()) ||
@@ -7926,8 +7913,15 @@ class MappableExprsHandler {
                 isa<BinaryOperator>(Next->getAssociatedExpression())) &&
                "Unexpected expression");
 
-        Address LB = CGF.EmitOMPSharedLValue(I->getAssociatedExpression())
-                         .getAddress(CGF);
+        Address LB = Address::invalid();
+        if (OAShE) {
+          LB = Address(CGF.EmitScalarExpr(OAShE->getBase()),
+                       CGF.getContext().getTypeAlignInChars(
+                           OAShE->getBase()->getType()));
+        } else {
+          LB = CGF.EmitOMPSharedLValue(I->getAssociatedExpression())
+                   .getAddress(CGF);
+        }
 
         // If this component is a pointer inside the base struct then we don't
         // need to create any entry for it - it will be combined with the object
@@ -9107,6 +9101,8 @@ getNestedDistributeDirective(ASTContext &Ctx, const OMPExecutableDirective &D) {
     case OMPD_target_update:
     case OMPD_declare_simd:
     case OMPD_declare_variant:
+    case OMPD_begin_declare_variant:
+    case OMPD_end_declare_variant:
     case OMPD_declare_target:
     case OMPD_end_declare_target:
     case OMPD_declare_reduction:
@@ -9886,6 +9882,8 @@ void CGOpenMPRuntime::scanForTargetRegionsFunctions(const Stmt *S,
     case OMPD_target_update:
     case OMPD_declare_simd:
     case OMPD_declare_variant:
+    case OMPD_begin_declare_variant:
+    case OMPD_end_declare_variant:
     case OMPD_declare_target:
     case OMPD_end_declare_target:
     case OMPD_declare_reduction:
@@ -10525,6 +10523,8 @@ void CGOpenMPRuntime::emitTargetDataStandAloneCall(
     case OMPD_teams_distribute_parallel_for_simd:
     case OMPD_declare_simd:
     case OMPD_declare_variant:
+    case OMPD_begin_declare_variant:
+    case OMPD_end_declare_variant:
     case OMPD_declare_target:
     case OMPD_end_declare_target:
     case OMPD_declare_reduction:
@@ -11383,56 +11383,6 @@ Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF,
   return Address(Addr, Align);
 }
 
-/// Finds the variant function that matches current context with its context
-/// selector.
-static const FunctionDecl *getDeclareVariantFunction(CodeGenModule &CGM,
-                                                     const FunctionDecl *FD) {
-  if (!FD->hasAttrs() || !FD->hasAttr<OMPDeclareVariantAttr>())
-    return FD;
-
-  SmallVector<Expr *, 8> VariantExprs;
-  SmallVector<VariantMatchInfo, 8> VMIs;
-  for (const auto *A : FD->specific_attrs<OMPDeclareVariantAttr>()) {
-    const OMPTraitInfo &TI = *A->getTraitInfos();
-    VMIs.push_back(VariantMatchInfo());
-    TI.getAsVariantMatchInfo(CGM.getContext(), VMIs.back());
-    VariantExprs.push_back(A->getVariantFuncRef());
-  }
-
-  OMPContext Ctx(CGM.getLangOpts().OpenMPIsDevice, CGM.getTriple());
-  // FIXME: Keep the context in the OMPIRBuilder so we can add constructs as we
-  //        build them.
-
-  int BestMatchIdx = getBestVariantMatchForContext(VMIs, Ctx);
-  if (BestMatchIdx < 0)
-    return FD;
-
-  return cast<FunctionDecl>(
-      cast<DeclRefExpr>(VariantExprs[BestMatchIdx]->IgnoreParenImpCasts())
-          ->getDecl());
-}
-
-bool CGOpenMPRuntime::emitDeclareVariant(GlobalDecl GD, bool IsForDefinition) {
-  const auto *D = cast<FunctionDecl>(GD.getDecl());
-  // If the original function is defined already, use its definition.
-  StringRef MangledName = CGM.getMangledName(GD);
-  llvm::GlobalValue *Orig = CGM.GetGlobalValue(MangledName);
-  if (Orig && !Orig->isDeclaration())
-    return false;
-  const FunctionDecl *NewFD = getDeclareVariantFunction(CGM, D);
-  // Emit original function if it does not have declare variant attribute or the
-  // context does not match.
-  if (NewFD == D)
-    return false;
-  GlobalDecl NewGD = GD.getWithDecl(NewFD);
-  if (tryEmitDeclareVariant(NewGD, GD, Orig, IsForDefinition)) {
-    DeferredVariantFunction.erase(D);
-    return true;
-  }
-  DeferredVariantFunction.insert(std::make_pair(D, std::make_pair(NewGD, GD)));
-  return true;
-}
-
 CGOpenMPRuntime::NontemporalDeclsRAII::NontemporalDeclsRAII(
     CodeGenModule &CGM, const OMPLoopDirective &S)
     : CGM(CGM), NeedToPush(S.hasClausesOfKind<OMPNontemporalClause>()) {
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h
index 99b201bcf6fea..122996030e868 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.h
@@ -334,17 +334,6 @@ class CGOpenMPRuntime {
   /// default location.
   virtual unsigned getDefaultLocationReserved2Flags() const { return 0; }
 
-  /// Tries to emit declare variant function for \p OldGD from \p NewGD.
-  /// \param OrigAddr LLVM IR value for \p OldGD.
-  /// \param IsForDefinition true, if requested emission for the definition of
-  /// \p OldGD.
-  /// \returns true, was able to emit a definition function for \p OldGD, which
-  /// points to \p NewGD.
-  virtual bool tryEmitDeclareVariant(const GlobalDecl &NewGD,
-                                     const GlobalDecl &OldGD,
-                                     llvm::GlobalValue *OrigAddr,
-                                     bool IsForDefinition);
-
   /// Returns default flags for the barriers depending on the directive, for
   /// which this barier is going to be emitted.
   static unsigned getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind);
@@ -686,12 +675,6 @@ class CGOpenMPRuntime {
   /// must be emitted.
   llvm::SmallDenseSet<const VarDecl *> DeferredGlobalVariables;
 
-  /// Mapping of the original functions to their variants and original global
-  /// decl.
-  llvm::MapVector<CanonicalDeclPtr<const FunctionDecl>,
-                  std::pair<GlobalDecl, GlobalDecl>>
-      DeferredVariantFunction;
-
   using NontemporalDeclsSet = llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>>;
   /// Stack for list of declarations in current context marked as nontemporal.
   /// The set is the union of all current stack elements.
@@ -1726,9 +1709,6 @@ class CGOpenMPRuntime {
   /// Return whether the unified_shared_memory has been specified.
   bool hasRequiresUnifiedSharedMemory() const;
 
-  /// Emits the definition of the declare variant function.
-  virtual bool emitDeclareVariant(GlobalDecl GD, bool IsForDefinition);
-
   /// Checks if the \p VD variable is marked as nontemporal declaration in
   /// current context.
   bool isNontemporalDecl(const ValueDecl *VD) const;
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp
index 4b5c85541a7a5..aa6b6bc64c328 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp
@@ -803,6 +803,8 @@ static bool hasNestedSPMDDirective(ASTContext &Ctx,
     case OMPD_target_update:
     case OMPD_declare_simd:
     case OMPD_declare_variant:
+    case OMPD_begin_declare_variant:
+    case OMPD_end_declare_variant:
     case OMPD_declare_target:
     case OMPD_end_declare_target:
     case OMPD_declare_reduction:
@@ -881,6 +883,8 @@ static bool supportsSPMDExecutionMode(ASTContext &Ctx,
   case OMPD_target_update:
   case OMPD_declare_simd:
   case OMPD_declare_variant:
+  case OMPD_begin_declare_variant:
+  case OMPD_end_declare_variant:
   case OMPD_declare_target:
   case OMPD_end_declare_target:
   case OMPD_declare_reduction:
@@ -1052,6 +1056,8 @@ static bool hasNestedLightweightDirective(ASTContext &Ctx,
     case OMPD_target_update:
     case OMPD_declare_simd:
     case OMPD_declare_variant:
+    case OMPD_begin_declare_variant:
+    case OMPD_end_declare_variant:
     case OMPD_declare_target:
     case OMPD_end_declare_target:
     case OMPD_declare_reduction:
@@ -1136,6 +1142,8 @@ static bool supportsLightweightRuntime(ASTContext &Ctx,
   case OMPD_target_update:
   case OMPD_declare_simd:
   case OMPD_declare_variant:
+  case OMPD_begin_declare_variant:
+  case OMPD_end_declare_variant:
   case OMPD_declare_target:
   case OMPD_end_declare_target:
   case OMPD_declare_reduction:
@@ -1923,19 +1931,6 @@ unsigned CGOpenMPRuntimeNVPTX::getDefaultLocationReserved2Flags() const {
   llvm_unreachable("Unknown flags are requested.");
 }
 
-bool CGOpenMPRuntimeNVPTX::tryEmitDeclareVariant(const GlobalDecl &NewGD,
-                                                 const GlobalDecl &OldGD,
-                                                 llvm::GlobalValue *OrigAddr,
-                                                 bool IsForDefinition) {
-  // Emit the function in OldGD with the body from NewGD, if NewGD is defined.
-  auto *NewFD = cast<FunctionDecl>(NewGD.getDecl());
-  if (NewFD->isDefined()) {
-    CGM.emitOpenMPDeviceFunctionRedefinition(OldGD, NewGD, OrigAddr);
-    return true;
-  }
-  return false;
-}
-
 CGOpenMPRuntimeNVPTX::CGOpenMPRuntimeNVPTX(CodeGenModule &CGM)
     : CGOpenMPRuntime(CGM, "_", "$") {
   if (!CGM.getLangOpts().OpenMPIsDevice)
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.h b/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.h
index 834adb3782a09..c52ae43817c75 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.h
@@ -193,18 +193,6 @@ class CGOpenMPRuntimeNVPTX : public CGOpenMPRuntime {
   /// Full/Lightweight runtime mode. Used for better optimization.
   unsigned getDefaultLocationReserved2Flags() const override;
 
-  /// Tries to emit declare variant function for \p OldGD from \p NewGD.
-  /// \param OrigAddr LLVM IR value for \p OldGD.
-  /// \param IsForDefinition true, if requested emission for the definition of
-  /// \p OldGD.
-  /// \returns true, was able to emit a definition function for \p OldGD, which
-  /// points to \p NewGD.
-  /// NVPTX backend does not support global aliases, so just use the function,
-  /// emitted for \p NewGD instead of \p OldGD.
-  bool tryEmitDeclareVariant(const GlobalDecl &NewGD, const GlobalDecl &OldGD,
-                             llvm::GlobalValue *OrigAddr,
-                             bool IsForDefinition) override;
-
 public:
   explicit CGOpenMPRuntimeNVPTX(CodeGenModule &CGM);
   void clear() override;
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index bef36bf4693b2..af7056a10f00d 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -24,6 +24,7 @@
 #include "clang/Basic/OpenMPKinds.h"
 #include "clang/Basic/PrettyStackTrace.h"
 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
+#include "llvm/IR/Constants.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/Support/AtomicOrdering.h"
 using namespace clang;
@@ -3535,9 +3536,13 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
         // initializer/combiner/finalizer.
         CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
                                                            RedCG, Cnt);
-        llvm::Value *ReductionsPtr =
-            CGF.EmitLoadOfScalar(CGF.EmitLValue(TaskgroupDescriptors[Cnt]),
-                                 TaskgroupDescriptors[Cnt]->getExprLoc());
+        llvm::Value *ReductionsPtr;
+        if (const Expr *TRExpr = TaskgroupDescriptors[Cnt]) {
+          ReductionsPtr = CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr),
+                                               TRExpr->getExprLoc());
+        } else {
+          ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+        }
         Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
             CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
         Replacement = Address(
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 1f06514a38c13..024fc068c217c 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -848,55 +848,54 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
           FD->getBody()->getStmtClass() == Stmt::CoroutineBodyStmtClass)
         SanOpts.Mask &= ~SanitizerKind::Null;
 
-  if (D) {
-    // Apply xray attributes to the function (as a string, for now)
-    if (const auto *XRayAttr = D->getAttr<XRayInstrumentAttr>()) {
-      if (CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
-              XRayInstrKind::FunctionEntry) ||
-          CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
-              XRayInstrKind::FunctionExit)) {
-        if (XRayAttr->alwaysXRayInstrument() && ShouldXRayInstrumentFunction())
-          Fn->addFnAttr("function-instrument", "xray-always");
-        if (XRayAttr->neverXRayInstrument())
-          Fn->addFnAttr("function-instrument", "xray-never");
-        if (const auto *LogArgs = D->getAttr<XRayLogArgsAttr>())
-          if (ShouldXRayInstrumentFunction())
-            Fn->addFnAttr("xray-log-args",
-                          llvm::utostr(LogArgs->getArgumentCount()));
-      }
-    } else {
-      if (ShouldXRayInstrumentFunction() && !CGM.imbueXRayAttrs(Fn, Loc))
-        Fn->addFnAttr(
-            "xray-instruction-threshold",
-            llvm::itostr(CGM.getCodeGenOpts().XRayInstructionThreshold));
+  // Apply xray attributes to the function (as a string, for now)
+  if (const auto *XRayAttr = D ? D->getAttr<XRayInstrumentAttr>() : nullptr) {
+    if (CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
+            XRayInstrKind::FunctionEntry) ||
+        CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
+            XRayInstrKind::FunctionExit)) {
+      if (XRayAttr->alwaysXRayInstrument() && ShouldXRayInstrumentFunction())
+        Fn->addFnAttr("function-instrument", "xray-always");
+      if (XRayAttr->neverXRayInstrument())
+        Fn->addFnAttr("function-instrument", "xray-never");
+      if (const auto *LogArgs = D->getAttr<XRayLogArgsAttr>())
+        if (ShouldXRayInstrumentFunction())
+          Fn->addFnAttr("xray-log-args",
+                        llvm::utostr(LogArgs->getArgumentCount()));
     }
+  } else {
+    if (ShouldXRayInstrumentFunction() && !CGM.imbueXRayAttrs(Fn, Loc))
+      Fn->addFnAttr(
+          "xray-instruction-threshold",
+          llvm::itostr(CGM.getCodeGenOpts().XRayInstructionThreshold));
+  }
 
-    if (ShouldXRayInstrumentFunction()) {
-      if (CGM.getCodeGenOpts().XRayIgnoreLoops)
-        Fn->addFnAttr("xray-ignore-loops");
+  if (ShouldXRayInstrumentFunction()) {
+    if (CGM.getCodeGenOpts().XRayIgnoreLoops)
+      Fn->addFnAttr("xray-ignore-loops");
 
-      if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
-              XRayInstrKind::FunctionExit))
-        Fn->addFnAttr("xray-skip-exit");
+    if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
+            XRayInstrKind::FunctionExit))
+      Fn->addFnAttr("xray-skip-exit");
 
-      if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
-              XRayInstrKind::FunctionEntry))
-        Fn->addFnAttr("xray-skip-entry");
-    }
+    if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
+            XRayInstrKind::FunctionEntry))
+      Fn->addFnAttr("xray-skip-entry");
+  }
 
-    unsigned Count, Offset;
-    if (const auto *Attr = D->getAttr<PatchableFunctionEntryAttr>()) {
-      Count = Attr->getCount();
-      Offset = Attr->getOffset();
-    } else {
-      Count = CGM.getCodeGenOpts().PatchableFunctionEntryCount;
-      Offset = CGM.getCodeGenOpts().PatchableFunctionEntryOffset;
-    }
-    if (Count && Offset <= Count) {
-      Fn->addFnAttr("patchable-function-entry", std::to_string(Count - Offset));
-      if (Offset)
-        Fn->addFnAttr("patchable-function-prefix", std::to_string(Offset));
-    }
+  unsigned Count, Offset;
+  if (const auto *Attr =
+          D ? D->getAttr<PatchableFunctionEntryAttr>() : nullptr) {
+    Count = Attr->getCount();
+    Offset = Attr->getOffset();
+  } else {
+    Count = CGM.getCodeGenOpts().PatchableFunctionEntryCount;
+    Offset = CGM.getCodeGenOpts().PatchableFunctionEntryOffset;
+  }
+  if (Count && Offset <= Count) {
+    Fn->addFnAttr("patchable-function-entry", std::to_string(Count - Offset));
+    if (Offset)
+      Fn->addFnAttr("patchable-function-prefix", std::to_string(Offset));
   }
 
   // Add no-jump-tables value.
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index cef4a6751491a..82a1f935e00a1 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -748,6 +748,19 @@ llvm::MDNode *CodeGenModule::getTBAATypeInfo(QualType QTy) {
 TBAAAccessInfo CodeGenModule::getTBAAAccessInfo(QualType AccessType) {
   if (!TBAA)
     return TBAAAccessInfo();
+  if (getLangOpts().CUDAIsDevice) {
+    // As CUDA builtin surface/texture types are replaced, skip generating TBAA
+    // access info.
+    if (AccessType->isCUDADeviceBuiltinSurfaceType()) {
+      if (getTargetCodeGenInfo().getCUDADeviceBuiltinSurfaceDeviceType() !=
+          nullptr)
+        return TBAAAccessInfo();
+    } else if (AccessType->isCUDADeviceBuiltinTextureType()) {
+      if (getTargetCodeGenInfo().getCUDADeviceBuiltinTextureDeviceType() !=
+          nullptr)
+        return TBAAAccessInfo();
+    }
+  }
   return TBAA->getAccessInfo(AccessType);
 }
 
@@ -2542,7 +2555,9 @@ void CodeGenModule::EmitGlobal(GlobalDecl GD) {
           !Global->hasAttr<CUDAGlobalAttr>() &&
           !Global->hasAttr<CUDAConstantAttr>() &&
           !Global->hasAttr<CUDASharedAttr>() &&
-          !(LangOpts.HIP && Global->hasAttr<HIPPinnedShadowAttr>()))
+          !(LangOpts.HIP && Global->hasAttr<HIPPinnedShadowAttr>()) &&
+          !Global->getType()->isCUDADeviceBuiltinSurfaceType() &&
+          !Global->getType()->isCUDADeviceBuiltinTextureType())
         return;
     } else {
       // We need to emit host-side 'shadows' for all global
@@ -2640,11 +2655,6 @@ void CodeGenModule::EmitGlobal(GlobalDecl GD) {
     return;
   }
 
-    // Check if this must be emitted as declare variant.
-  if (LangOpts.OpenMP && isa<FunctionDecl>(Global) && OpenMPRuntime &&
-      OpenMPRuntime->emitDeclareVariant(GD, /*IsForDefinition=*/false))
-    return;
-
   // If we're deferring emission of a C++ variable with an
   // initializer, remember the order in which it appeared in the file.
   if (getLangOpts().CPlusPlus && isa<VarDecl>(Global) &&
@@ -2850,50 +2860,6 @@ void CodeGenModule::EmitMultiVersionFunctionDefinition(GlobalDecl GD,
     EmitGlobalFunctionDefinition(GD, GV);
 }
 
-void CodeGenModule::emitOpenMPDeviceFunctionRedefinition(
-    GlobalDecl OldGD, GlobalDecl NewGD, llvm::GlobalValue *GV) {
-  assert(getLangOpts().OpenMP && getLangOpts().OpenMPIsDevice &&
-         OpenMPRuntime && "Expected OpenMP device mode.");
-  const auto *D = cast<FunctionDecl>(OldGD.getDecl());
-
-  // Compute the function info and LLVM type.
-  const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(OldGD);
-  llvm::FunctionType *Ty = getTypes().GetFunctionType(FI);
-
-  // Get or create the prototype for the function.
-  if (!GV || (GV->getType()->getElementType() != Ty)) {
-    GV = cast<llvm::GlobalValue>(GetOrCreateLLVMFunction(
-        getMangledName(OldGD), Ty, GlobalDecl(), /*ForVTable=*/false,
-        /*DontDefer=*/true, /*IsThunk=*/false, llvm::AttributeList(),
-        ForDefinition));
-    SetFunctionAttributes(OldGD, cast<llvm::Function>(GV),
-                          /*IsIncompleteFunction=*/false,
-                          /*IsThunk=*/false);
-  }
-  // We need to set linkage and visibility on the function before
-  // generating code for it because various parts of IR generation
-  // want to propagate this information down (e.g. to local static
-  // declarations).
-  auto *Fn = cast<llvm::Function>(GV);
-  setFunctionLinkage(OldGD, Fn);
-
-  // FIXME: this is redundant with part of
-  // setFunctionDefinitionAttributes
-  setGVProperties(Fn, OldGD);
-
-  MaybeHandleStaticInExternC(D, Fn);
-
-  maybeSetTrivialComdat(*D, *Fn);
-
-  CodeGenFunction(*this).GenerateCode(NewGD, Fn, FI);
-
-  setNonAliasAttributes(OldGD, Fn);
-  SetLLVMFunctionAttributesForDefinition(D, Fn);
-
-  if (D->hasAttr<AnnotateAttr>())
-    AddGlobalAnnotations(D, Fn);
-}
-
 void CodeGenModule::EmitGlobalDefinition(GlobalDecl GD, llvm::GlobalValue *GV) {
   const auto *D = cast<ValueDecl>(GD.getDecl());
 
@@ -3208,10 +3174,6 @@ llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction(
         EmitGlobal(GDDef);
       }
     }
-    // Check if this must be emitted as declare variant and emit reference to
-    // the the declare variant function.
-    if (LangOpts.OpenMP && OpenMPRuntime)
-      (void)OpenMPRuntime->emitDeclareVariant(GD, /*IsForDefinition=*/true);
 
     if (FD->isMultiVersion()) {
       if (FD->hasAttr<TargetAttr>())
@@ -4154,12 +4116,16 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D,
       !getLangOpts().CUDAIsDevice &&
       (D->hasAttr<CUDAConstantAttr>() || D->hasAttr<CUDADeviceAttr>() ||
        D->hasAttr<CUDASharedAttr>());
+  bool IsCUDADeviceShadowVar =
+      getLangOpts().CUDAIsDevice &&
+      (D->getType()->isCUDADeviceBuiltinSurfaceType() ||
+       D->getType()->isCUDADeviceBuiltinTextureType());
   // HIP pinned shadow of initialized host-side global variables are also
   // left undefined.
   bool IsHIPPinnedShadowVar =
       getLangOpts().CUDAIsDevice && D->hasAttr<HIPPinnedShadowAttr>();
-  if (getLangOpts().CUDA &&
-      (IsCUDASharedVar || IsCUDAShadowVar || IsHIPPinnedShadowVar))
+  if (getLangOpts().CUDA && (IsCUDASharedVar || IsCUDAShadowVar ||
+                             IsCUDADeviceShadowVar || IsHIPPinnedShadowVar))
     Init = llvm::UndefValue::get(getTypes().ConvertType(ASTTy));
   else if (D->hasAttr<LoaderUninitializedAttr>())
     Init = llvm::UndefValue::get(getTypes().ConvertType(ASTTy));
@@ -4285,25 +4251,48 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D,
       if (D->hasAttr<CUDADeviceAttr>() || D->hasAttr<CUDAConstantAttr>() ||
           D->hasAttr<HIPPinnedShadowAttr>()) {
         Linkage = llvm::GlobalValue::InternalLinkage;
-
-        // Shadow variables and their properties must be registered
-        // with CUDA runtime.
-        unsigned Flags = 0;
-        if (!D->hasDefinition())
-          Flags |= CGCUDARuntime::ExternDeviceVar;
-        if (D->hasAttr<CUDAConstantAttr>())
-          Flags |= CGCUDARuntime::ConstantDeviceVar;
-        // Extern global variables will be registered in the TU where they are
-        // defined.
+        // Shadow variables and their properties must be registered with CUDA
+        // runtime. Skip Extern global variables, which will be registered in
+        // the TU where they are defined.
         if (!D->hasExternalStorage())
-          getCUDARuntime().registerDeviceVar(D, *GV, Flags);
-      } else if (D->hasAttr<CUDASharedAttr>())
+          getCUDARuntime().registerDeviceVar(D, *GV, !D->hasDefinition(),
+                                             D->hasAttr<CUDAConstantAttr>());
+      } else if (D->hasAttr<CUDASharedAttr>()) {
         // __shared__ variables are odd. Shadows do get created, but
         // they are not registered with the CUDA runtime, so they
         // can't really be used to access their device-side
         // counterparts. It's not clear yet whether it's nvcc's bug or
         // a feature, but we've got to do the same for compatibility.
         Linkage = llvm::GlobalValue::InternalLinkage;
+      } else if (D->getType()->isCUDADeviceBuiltinSurfaceType() ||
+                 D->getType()->isCUDADeviceBuiltinTextureType()) {
+        // Builtin surfaces and textures and their template arguments are
+        // also registered with CUDA runtime.
+        Linkage = llvm::GlobalValue::InternalLinkage;
+        const ClassTemplateSpecializationDecl *TD =
+            cast<ClassTemplateSpecializationDecl>(
+                D->getType()->getAs<RecordType>()->getDecl());
+        const TemplateArgumentList &Args = TD->getTemplateArgs();
+        if (TD->hasAttr<CUDADeviceBuiltinSurfaceTypeAttr>()) {
+          assert(Args.size() == 2 &&
+                 "Unexpected number of template arguments of CUDA device "
+                 "builtin surface type.");
+          auto SurfType = Args[1].getAsIntegral();
+          if (!D->hasExternalStorage())
+            getCUDARuntime().registerDeviceSurf(D, *GV, !D->hasDefinition(),
+                                                SurfType.getSExtValue());
+        } else {
+          assert(Args.size() == 3 &&
+                 "Unexpected number of template arguments of CUDA device "
+                 "builtin texture type.");
+          auto TexType = Args[1].getAsIntegral();
+          auto Normalized = Args[2].getAsIntegral();
+          if (!D->hasExternalStorage())
+            getCUDARuntime().registerDeviceTex(D, *GV, !D->hasDefinition(),
+                                               TexType.getSExtValue(),
+                                               Normalized.getZExtValue());
+        }
+      }
     }
   }
 
@@ -4688,11 +4677,6 @@ void CodeGenModule::HandleCXXStaticMemberVarInstantiation(VarDecl *VD) {
 
 void CodeGenModule::EmitGlobalFunctionDefinition(GlobalDecl GD,
                                                  llvm::GlobalValue *GV) {
-  // Check if this must be emitted as declare variant.
-  if (LangOpts.OpenMP && OpenMPRuntime &&
-      OpenMPRuntime->emitDeclareVariant(GD, /*IsForDefinition=*/true))
-    return;
-
   const auto *D = cast<FunctionDecl>(GD.getDecl());
 
   // Compute the function info and LLVM type.
diff --git a/clang/lib/CodeGen/CodeGenModule.h b/clang/lib/CodeGen/CodeGenModule.h
index a538884a97f6e..4565b924ae27a 100644
--- a/clang/lib/CodeGen/CodeGenModule.h
+++ b/clang/lib/CodeGen/CodeGenModule.h
@@ -1310,11 +1310,6 @@ class CodeGenModule : public CodeGenTypeCache {
   /// \param D Requires declaration
   void EmitOMPRequiresDecl(const OMPRequiresDecl *D);
 
-  /// Emits the definition of \p OldGD function with body from \p NewGD.
-  /// Required for proper handling of declare variant directive on the GPU.
-  void emitOpenMPDeviceFunctionRedefinition(GlobalDecl OldGD, GlobalDecl NewGD,
-                                            llvm::GlobalValue *GV);
-
   /// Returns whether the given record has hidden LTO visibility and therefore
   /// may participate in (single-module) CFI and whole-program vtable
   /// optimization.
diff --git a/clang/lib/CodeGen/CodeGenTBAA.cpp b/clang/lib/CodeGen/CodeGenTBAA.cpp
index 7d730cb1ed154..8cc8c162dfbe7 100644
--- a/clang/lib/CodeGen/CodeGenTBAA.cpp
+++ b/clang/lib/CodeGen/CodeGenTBAA.cpp
@@ -141,6 +141,34 @@ llvm::MDNode *CodeGenTBAA::getTypeInfoHelper(const Type *Ty) {
     case BuiltinType::UInt128:
       return getTypeInfo(Context.Int128Ty);
 
+    case BuiltinType::UShortFract:
+      return getTypeInfo(Context.ShortFractTy);
+    case BuiltinType::UFract:
+      return getTypeInfo(Context.FractTy);
+    case BuiltinType::ULongFract:
+      return getTypeInfo(Context.LongFractTy);
+
+    case BuiltinType::SatUShortFract:
+      return getTypeInfo(Context.SatShortFractTy);
+    case BuiltinType::SatUFract:
+      return getTypeInfo(Context.SatFractTy);
+    case BuiltinType::SatULongFract:
+      return getTypeInfo(Context.SatLongFractTy);
+
+    case BuiltinType::UShortAccum:
+      return getTypeInfo(Context.ShortAccumTy);
+    case BuiltinType::UAccum:
+      return getTypeInfo(Context.AccumTy);
+    case BuiltinType::ULongAccum:
+      return getTypeInfo(Context.LongAccumTy);
+
+    case BuiltinType::SatUShortAccum:
+      return getTypeInfo(Context.SatShortAccumTy);
+    case BuiltinType::SatUAccum:
+      return getTypeInfo(Context.SatAccumTy);
+    case BuiltinType::SatULongAccum:
+      return getTypeInfo(Context.SatLongAccumTy);
+
     // Treat all other builtin types as distinct types. This includes
     // treating wchar_t, char16_t, and char32_t as distinct from their
     // "underlying types".
diff --git a/clang/lib/CodeGen/CodeGenTypes.cpp b/clang/lib/CodeGen/CodeGenTypes.cpp
index aa32d83de0af0..caa71b10f2311 100644
--- a/clang/lib/CodeGen/CodeGenTypes.cpp
+++ b/clang/lib/CodeGen/CodeGenTypes.cpp
@@ -397,6 +397,20 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
 
   const Type *Ty = T.getTypePtr();
 
+  // For the device-side compilation, CUDA device builtin surface/texture types
+  // may be represented in different types.
+  if (Context.getLangOpts().CUDAIsDevice) {
+    if (T->isCUDADeviceBuiltinSurfaceType()) {
+      if (auto *Ty = CGM.getTargetCodeGenInfo()
+                         .getCUDADeviceBuiltinSurfaceDeviceType())
+        return Ty;
+    } else if (T->isCUDADeviceBuiltinTextureType()) {
+      if (auto *Ty = CGM.getTargetCodeGenInfo()
+                         .getCUDADeviceBuiltinTextureDeviceType())
+        return Ty;
+    }
+  }
+
   // RecordTypes are cached and processed specially.
   if (const RecordType *RT = dyn_cast<RecordType>(Ty))
     return ConvertRecordDeclType(RT->getDecl());
@@ -595,7 +609,11 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
     llvm::Type *PointeeType = ConvertTypeForMem(ETy);
     if (PointeeType->isVoidTy())
       PointeeType = llvm::Type::getInt8Ty(getLLVMContext());
-    unsigned AS = Context.getTargetAddressSpace(ETy);
+
+    unsigned AS = PointeeType->isFunctionTy()
+                      ? getDataLayout().getProgramAddressSpace()
+                      : Context.getTargetAddressSpace(ETy);
+
     ResultType = llvm::PointerType::get(PointeeType, AS);
     break;
   }
diff --git a/clang/lib/CodeGen/TargetInfo.cpp b/clang/lib/CodeGen/TargetInfo.cpp
index 9033853d619f7..24c102acd98d4 100644
--- a/clang/lib/CodeGen/TargetInfo.cpp
+++ b/clang/lib/CodeGen/TargetInfo.cpp
@@ -28,6 +28,7 @@
 #include "llvm/ADT/Triple.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/IR/DataLayout.h"
+#include "llvm/IR/IntrinsicsNVPTX.h"
 #include "llvm/IR/Type.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm> // std::sort
@@ -5106,9 +5107,11 @@ class AArch64TargetCodeGenInfo : public TargetCodeGenInfo {
     if (!FD)
       return;
 
-    CodeGenOptions::SignReturnAddressScope Scope = CGM.getCodeGenOpts().getSignReturnAddress();
-    CodeGenOptions::SignReturnAddressKeyValue Key = CGM.getCodeGenOpts().getSignReturnAddressKey();
-    bool BranchTargetEnforcement = CGM.getCodeGenOpts().BranchTargetEnforcement;
+    LangOptions::SignReturnAddressScopeKind Scope =
+        CGM.getLangOpts().getSignReturnAddressScope();
+    LangOptions::SignReturnAddressKeyKind Key =
+        CGM.getLangOpts().getSignReturnAddressKey();
+    bool BranchTargetEnforcement = CGM.getLangOpts().BranchTargetEnforcement;
     if (const auto *TA = FD->getAttr<TargetAttr>()) {
       ParsedTargetAttr Attr = TA->parse();
       if (!Attr.BranchProtection.empty()) {
@@ -5124,14 +5127,14 @@ class AArch64TargetCodeGenInfo : public TargetCodeGenInfo {
     }
 
     auto *Fn = cast<llvm::Function>(GV);
-    if (Scope != CodeGenOptions::SignReturnAddressScope::None) {
+    if (Scope != LangOptions::SignReturnAddressScopeKind::None) {
       Fn->addFnAttr("sign-return-address",
-                    Scope == CodeGenOptions::SignReturnAddressScope::All
+                    Scope == LangOptions::SignReturnAddressScopeKind::All
                         ? "all"
                         : "non-leaf");
 
       Fn->addFnAttr("sign-return-address-key",
-                    Key == CodeGenOptions::SignReturnAddressKeyValue::AKey
+                    Key == LangOptions::SignReturnAddressKeyKind::AKey
                         ? "a_key"
                         : "b_key");
     }
@@ -6414,9 +6417,14 @@ Address ARMABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
 
 namespace {
 
+class NVPTXTargetCodeGenInfo;
+
 class NVPTXABIInfo : public ABIInfo {
+  NVPTXTargetCodeGenInfo &CGInfo;
+
 public:
-  NVPTXABIInfo(CodeGenTypes &CGT) : ABIInfo(CGT) {}
+  NVPTXABIInfo(CodeGenTypes &CGT, NVPTXTargetCodeGenInfo &Info)
+      : ABIInfo(CGT), CGInfo(Info) {}
 
   ABIArgInfo classifyReturnType(QualType RetTy) const;
   ABIArgInfo classifyArgumentType(QualType Ty) const;
@@ -6429,16 +6437,61 @@ class NVPTXABIInfo : public ABIInfo {
 class NVPTXTargetCodeGenInfo : public TargetCodeGenInfo {
 public:
   NVPTXTargetCodeGenInfo(CodeGenTypes &CGT)
-    : TargetCodeGenInfo(new NVPTXABIInfo(CGT)) {}
+      : TargetCodeGenInfo(new NVPTXABIInfo(CGT, *this)) {}
 
   void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
                            CodeGen::CodeGenModule &M) const override;
   bool shouldEmitStaticExternCAliases() const override;
 
+  llvm::Type *getCUDADeviceBuiltinSurfaceDeviceType() const override {
+    // On the device side, surface reference is represented as an object handle
+    // in 64-bit integer.
+    return llvm::Type::getInt64Ty(getABIInfo().getVMContext());
+  }
+
+  llvm::Type *getCUDADeviceBuiltinTextureDeviceType() const override {
+    // On the device side, texture reference is represented as an object handle
+    // in 64-bit integer.
+    return llvm::Type::getInt64Ty(getABIInfo().getVMContext());
+  }
+
+  bool emitCUDADeviceBuiltinSurfaceDeviceCopy(CodeGenFunction &CGF, LValue Dst,
+                                              LValue Src) const override {
+    emitBuiltinSurfTexDeviceCopy(CGF, Dst, Src);
+    return true;
+  }
+
+  bool emitCUDADeviceBuiltinTextureDeviceCopy(CodeGenFunction &CGF, LValue Dst,
+                                              LValue Src) const override {
+    emitBuiltinSurfTexDeviceCopy(CGF, Dst, Src);
+    return true;
+  }
+
 private:
-  // Adds a NamedMDNode with F, Name, and Operand as operands, and adds the
+  // Adds a NamedMDNode with GV, Name, and Operand as operands, and adds the
   // resulting MDNode to the nvvm.annotations MDNode.
-  static void addNVVMMetadata(llvm::Function *F, StringRef Name, int Operand);
+  static void addNVVMMetadata(llvm::GlobalValue *GV, StringRef Name,
+                              int Operand);
+
+  static void emitBuiltinSurfTexDeviceCopy(CodeGenFunction &CGF, LValue Dst,
+                                           LValue Src) {
+    llvm::Value *Handle = nullptr;
+    llvm::Constant *C =
+        llvm::dyn_cast<llvm::Constant>(Src.getAddress(CGF).getPointer());
+    // Lookup `addrspacecast` through the constant pointer if any.
+    if (auto *ASC = llvm::dyn_cast_or_null<llvm::AddrSpaceCastOperator>(C))
+      C = llvm::cast<llvm::Constant>(ASC->getPointerOperand());
+    if (auto *GV = llvm::dyn_cast_or_null<llvm::GlobalVariable>(C)) {
+      // Load the handle from the specific global variable using
+      // `nvvm.texsurf.handle.internal` intrinsic.
+      Handle = CGF.EmitRuntimeCall(
+          CGF.CGM.getIntrinsic(llvm::Intrinsic::nvvm_texsurf_handle_internal,
+                               {GV->getType()}),
+          {GV}, "texsurf_handle");
+    } else
+      Handle = CGF.EmitLoadOfScalar(Src, SourceLocation());
+    CGF.EmitStoreOfScalar(Handle, Dst);
+  }
 };
 
 /// Checks if the type is unsupported directly by the current target.
@@ -6511,8 +6564,19 @@ ABIArgInfo NVPTXABIInfo::classifyArgumentType(QualType Ty) const {
     Ty = EnumTy->getDecl()->getIntegerType();
 
   // Return aggregates type as indirect by value
-  if (isAggregateTypeForABI(Ty))
+  if (isAggregateTypeForABI(Ty)) {
+    // Under CUDA device compilation, tex/surf builtin types are replaced with
+    // object types and passed directly.
+    if (getContext().getLangOpts().CUDAIsDevice) {
+      if (Ty->isCUDADeviceBuiltinSurfaceType())
+        return ABIArgInfo::getDirect(
+            CGInfo.getCUDADeviceBuiltinSurfaceDeviceType());
+      if (Ty->isCUDADeviceBuiltinTextureType())
+        return ABIArgInfo::getDirect(
+            CGInfo.getCUDADeviceBuiltinTextureDeviceType());
+    }
     return getNaturalAlignIndirect(Ty, /* byval */ true);
+  }
 
   return (Ty->isPromotableIntegerType() ? ABIArgInfo::getExtend(Ty)
                                         : ABIArgInfo::getDirect());
@@ -6540,6 +6604,17 @@ void NVPTXTargetCodeGenInfo::setTargetAttributes(
     const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const {
   if (GV->isDeclaration())
     return;
+  const VarDecl *VD = dyn_cast_or_null<VarDecl>(D);
+  if (VD) {
+    if (M.getLangOpts().CUDA) {
+      if (VD->getType()->isCUDADeviceBuiltinSurfaceType())
+        addNVVMMetadata(GV, "surface", 1);
+      else if (VD->getType()->isCUDADeviceBuiltinTextureType())
+        addNVVMMetadata(GV, "texture", 1);
+      return;
+    }
+  }
+
   const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);
   if (!FD) return;
 
@@ -6588,16 +6663,16 @@ void NVPTXTargetCodeGenInfo::setTargetAttributes(
   }
 }
 
-void NVPTXTargetCodeGenInfo::addNVVMMetadata(llvm::Function *F, StringRef Name,
-                                             int Operand) {
-  llvm::Module *M = F->getParent();
+void NVPTXTargetCodeGenInfo::addNVVMMetadata(llvm::GlobalValue *GV,
+                                             StringRef Name, int Operand) {
+  llvm::Module *M = GV->getParent();
   llvm::LLVMContext &Ctx = M->getContext();
 
   // Get "nvvm.annotations" metadata node
   llvm::NamedMDNode *MD = M->getOrInsertNamedMetadata("nvvm.annotations");
 
   llvm::Metadata *MDVals[] = {
-      llvm::ConstantAsMetadata::get(F), llvm::MDString::get(Ctx, Name),
+      llvm::ConstantAsMetadata::get(GV), llvm::MDString::get(Ctx, Name),
       llvm::ConstantAsMetadata::get(
           llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), Operand))};
   // Append metadata to nvvm.annotations
@@ -7511,18 +7586,25 @@ void TCETargetCodeGenInfo::setTargetAttributes(
 
 namespace {
 
-class HexagonABIInfo : public ABIInfo {
+class HexagonABIInfo : public DefaultABIInfo {
 public:
-  HexagonABIInfo(CodeGenTypes &CGT) : ABIInfo(CGT) {}
+  HexagonABIInfo(CodeGenTypes &CGT) : DefaultABIInfo(CGT) {}
 
 private:
   ABIArgInfo classifyReturnType(QualType RetTy) const;
   ABIArgInfo classifyArgumentType(QualType RetTy) const;
+  ABIArgInfo classifyArgumentType(QualType RetTy, unsigned *RegsLeft) const;
 
   void computeInfo(CGFunctionInfo &FI) const override;
 
   Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
                     QualType Ty) const override;
+  Address EmitVAArgFromMemory(CodeGenFunction &CFG, Address VAListAddr,
+                              QualType Ty) const;
+  Address EmitVAArgForHexagon(CodeGenFunction &CFG, Address VAListAddr,
+                              QualType Ty) const;
+  Address EmitVAArgForHexagonLinux(CodeGenFunction &CFG, Address VAListAddr,
+                                   QualType Ty) const;
 };
 
 class HexagonTargetCodeGenInfo : public TargetCodeGenInfo {
@@ -7533,23 +7615,63 @@ class HexagonTargetCodeGenInfo : public TargetCodeGenInfo {
   int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
     return 29;
   }
+
+  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
+                           CodeGen::CodeGenModule &GCM) const override {
+    if (GV->isDeclaration())
+      return;
+    const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);
+    if (!FD)
+      return;
+  }
 };
 
 } // namespace
 
 void HexagonABIInfo::computeInfo(CGFunctionInfo &FI) const {
+  unsigned RegsLeft = 6;
   if (!getCXXABI().classifyReturnType(FI))
     FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
   for (auto &I : FI.arguments())
-    I.info = classifyArgumentType(I.type);
+    I.info = classifyArgumentType(I.type, &RegsLeft);
+}
+
+static bool HexagonAdjustRegsLeft(uint64_t Size, unsigned *RegsLeft) {
+  assert(Size <= 64 && "Not expecting to pass arguments larger than 64 bits"
+                       " through registers");
+
+  if (*RegsLeft == 0)
+    return false;
+
+  if (Size <= 32) {
+    (*RegsLeft)--;
+    return true;
+  }
+
+  if (2 <= (*RegsLeft & (~1U))) {
+    *RegsLeft = (*RegsLeft & (~1U)) - 2;
+    return true;
+  }
+
+  // Next available register was r5 but candidate was greater than 32-bits so it
+  // has to go on the stack. However we still consume r5
+  if (*RegsLeft == 1)
+    *RegsLeft = 0;
+
+  return false;
 }
 
-ABIArgInfo HexagonABIInfo::classifyArgumentType(QualType Ty) const {
+ABIArgInfo HexagonABIInfo::classifyArgumentType(QualType Ty,
+                                                unsigned *RegsLeft) const {
   if (!isAggregateTypeForABI(Ty)) {
     // Treat an enum type as its underlying type.
     if (const EnumType *EnumTy = Ty->getAs<EnumType>())
       Ty = EnumTy->getDecl()->getIntegerType();
 
+    uint64_t Size = getContext().getTypeSize(Ty);
+    if (Size <= 64)
+      HexagonAdjustRegsLeft(Size, RegsLeft);
+
     return Ty->isPromotableIntegerType() ? ABIArgInfo::getExtend(Ty)
                                          : ABIArgInfo::getDirect();
   }
@@ -7562,13 +7684,20 @@ ABIArgInfo HexagonABIInfo::classifyArgumentType(QualType Ty) const {
     return ABIArgInfo::getIgnore();
 
   uint64_t Size = getContext().getTypeSize(Ty);
-  if (Size <= 64) {
+  unsigned Align = getContext().getTypeAlign(Ty);
+
+  if (Size > 64)
+    return getNaturalAlignIndirect(Ty, /*ByVal=*/true);
+
+  if (HexagonAdjustRegsLeft(Size, RegsLeft))
+    Align = Size <= 32 ? 32 : 64;
+  if (Size <= Align) {
     // Pass in the smallest viable integer type.
     if (!llvm::isPowerOf2_64(Size))
       Size = llvm::NextPowerOf2(Size);
     return ABIArgInfo::getDirect(llvm::Type::getIntNTy(getVMContext(), Size));
   }
-  return getNaturalAlignIndirect(Ty, /*ByVal=*/true);
+  return DefaultABIInfo::classifyArgumentType(Ty);
 }
 
 ABIArgInfo HexagonABIInfo::classifyReturnType(QualType RetTy) const {
@@ -7586,7 +7715,6 @@ ABIArgInfo HexagonABIInfo::classifyReturnType(QualType RetTy) const {
       if (Size == VecSize || Size == 2*VecSize)
         return ABIArgInfo::getDirectInReg();
     }
-
     // Large vector types should be returned via memory.
     if (Size > 64)
       return getNaturalAlignIndirect(RetTy);
@@ -7615,13 +7743,242 @@ ABIArgInfo HexagonABIInfo::classifyReturnType(QualType RetTy) const {
   return getNaturalAlignIndirect(RetTy, /*ByVal=*/true);
 }
 
+Address HexagonABIInfo::EmitVAArgFromMemory(CodeGenFunction &CGF,
+                                            Address VAListAddr,
+                                            QualType Ty) const {
+  // Load the overflow area pointer.
+  Address __overflow_area_pointer_p =
+      CGF.Builder.CreateStructGEP(VAListAddr, 2, "__overflow_area_pointer_p");
+  llvm::Value *__overflow_area_pointer = CGF.Builder.CreateLoad(
+      __overflow_area_pointer_p, "__overflow_area_pointer");
+
+  uint64_t Align = CGF.getContext().getTypeAlign(Ty) / 8;
+  if (Align > 4) {
+    // Alignment should be a power of 2.
+    assert((Align & (Align - 1)) == 0 && "Alignment is not power of 2!");
+
+    // overflow_arg_area = (overflow_arg_area + align - 1) & -align;
+    llvm::Value *Offset = llvm::ConstantInt::get(CGF.Int64Ty, Align - 1);
+
+    // Add offset to the current pointer to access the argument.
+    __overflow_area_pointer =
+        CGF.Builder.CreateGEP(__overflow_area_pointer, Offset);
+    llvm::Value *AsInt =
+        CGF.Builder.CreatePtrToInt(__overflow_area_pointer, CGF.Int32Ty);
+
+    // Create a mask which should be "AND"ed
+    // with (overflow_arg_area + align - 1)
+    llvm::Value *Mask = llvm::ConstantInt::get(CGF.Int32Ty, -(int)Align);
+    __overflow_area_pointer = CGF.Builder.CreateIntToPtr(
+        CGF.Builder.CreateAnd(AsInt, Mask), __overflow_area_pointer->getType(),
+        "__overflow_area_pointer.align");
+  }
+
+  // Get the type of the argument from memory and bitcast
+  // overflow area pointer to the argument type.
+  llvm::Type *PTy = CGF.ConvertTypeForMem(Ty);
+  Address AddrTyped = CGF.Builder.CreateBitCast(
+      Address(__overflow_area_pointer, CharUnits::fromQuantity(Align)),
+      llvm::PointerType::getUnqual(PTy));
+
+  // Round up to the minimum stack alignment for varargs which is 4 bytes.
+  uint64_t Offset = llvm::alignTo(CGF.getContext().getTypeSize(Ty) / 8, 4);
+
+  __overflow_area_pointer = CGF.Builder.CreateGEP(
+      __overflow_area_pointer, llvm::ConstantInt::get(CGF.Int32Ty, Offset),
+      "__overflow_area_pointer.next");
+  CGF.Builder.CreateStore(__overflow_area_pointer, __overflow_area_pointer_p);
+
+  return AddrTyped;
+}
+
+Address HexagonABIInfo::EmitVAArgForHexagon(CodeGenFunction &CGF,
+                                            Address VAListAddr,
+                                            QualType Ty) const {
+  // FIXME: Need to handle alignment
+  llvm::Type *BP = CGF.Int8PtrTy;
+  llvm::Type *BPP = CGF.Int8PtrPtrTy;
+  CGBuilderTy &Builder = CGF.Builder;
+  Address VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, BPP, "ap");
+  llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur");
+  // Handle address alignment for type alignment > 32 bits
+  uint64_t TyAlign = CGF.getContext().getTypeAlign(Ty) / 8;
+  if (TyAlign > 4) {
+    assert((TyAlign & (TyAlign - 1)) == 0 && "Alignment is not power of 2!");
+    llvm::Value *AddrAsInt = Builder.CreatePtrToInt(Addr, CGF.Int32Ty);
+    AddrAsInt = Builder.CreateAdd(AddrAsInt, Builder.getInt32(TyAlign - 1));
+    AddrAsInt = Builder.CreateAnd(AddrAsInt, Builder.getInt32(~(TyAlign - 1)));
+    Addr = Builder.CreateIntToPtr(AddrAsInt, BP);
+  }
+  llvm::Type *PTy = llvm::PointerType::getUnqual(CGF.ConvertType(Ty));
+  Address AddrTyped = Builder.CreateBitCast(
+      Address(Addr, CharUnits::fromQuantity(TyAlign)), PTy);
+
+  uint64_t Offset = llvm::alignTo(CGF.getContext().getTypeSize(Ty) / 8, 4);
+  llvm::Value *NextAddr = Builder.CreateGEP(
+      Addr, llvm::ConstantInt::get(CGF.Int32Ty, Offset), "ap.next");
+  Builder.CreateStore(NextAddr, VAListAddrAsBPP);
+
+  return AddrTyped;
+}
+
+Address HexagonABIInfo::EmitVAArgForHexagonLinux(CodeGenFunction &CGF,
+                                                 Address VAListAddr,
+                                                 QualType Ty) const {
+  int ArgSize = CGF.getContext().getTypeSize(Ty) / 8;
+
+  if (ArgSize > 8)
+    return EmitVAArgFromMemory(CGF, VAListAddr, Ty);
+
+  // Here we have check if the argument is in register area or
+  // in overflow area.
+  // If the saved register area pointer + argsize rounded up to alignment >
+  // saved register area end pointer, argument is in overflow area.
+  unsigned RegsLeft = 6;
+  Ty = CGF.getContext().getCanonicalType(Ty);
+  (void)classifyArgumentType(Ty, &RegsLeft);
+
+  llvm::BasicBlock *MaybeRegBlock = CGF.createBasicBlock("vaarg.maybe_reg");
+  llvm::BasicBlock *InRegBlock = CGF.createBasicBlock("vaarg.in_reg");
+  llvm::BasicBlock *OnStackBlock = CGF.createBasicBlock("vaarg.on_stack");
+  llvm::BasicBlock *ContBlock = CGF.createBasicBlock("vaarg.end");
+
+  // Get rounded size of the argument.GCC does not allow vararg of
+  // size < 4 bytes. We follow the same logic here.
+  ArgSize = (CGF.getContext().getTypeSize(Ty) <= 32) ? 4 : 8;
+  int ArgAlign = (CGF.getContext().getTypeSize(Ty) <= 32) ? 4 : 8;
+
+  // Argument may be in saved register area
+  CGF.EmitBlock(MaybeRegBlock);
+
+  // Load the current saved register area pointer.
+  Address __current_saved_reg_area_pointer_p = CGF.Builder.CreateStructGEP(
+      VAListAddr, 0, "__current_saved_reg_area_pointer_p");
+  llvm::Value *__current_saved_reg_area_pointer = CGF.Builder.CreateLoad(
+      __current_saved_reg_area_pointer_p, "__current_saved_reg_area_pointer");
+
+  // Load the saved register area end pointer.
+  Address __saved_reg_area_end_pointer_p = CGF.Builder.CreateStructGEP(
+      VAListAddr, 1, "__saved_reg_area_end_pointer_p");
+  llvm::Value *__saved_reg_area_end_pointer = CGF.Builder.CreateLoad(
+      __saved_reg_area_end_pointer_p, "__saved_reg_area_end_pointer");
+
+  // If the size of argument is > 4 bytes, check if the stack
+  // location is aligned to 8 bytes
+  if (ArgAlign > 4) {
+
+    llvm::Value *__current_saved_reg_area_pointer_int =
+        CGF.Builder.CreatePtrToInt(__current_saved_reg_area_pointer,
+                                   CGF.Int32Ty);
+
+    __current_saved_reg_area_pointer_int = CGF.Builder.CreateAdd(
+        __current_saved_reg_area_pointer_int,
+        llvm::ConstantInt::get(CGF.Int32Ty, (ArgAlign - 1)),
+        "align_current_saved_reg_area_pointer");
+
+    __current_saved_reg_area_pointer_int =
+        CGF.Builder.CreateAnd(__current_saved_reg_area_pointer_int,
+                              llvm::ConstantInt::get(CGF.Int32Ty, -ArgAlign),
+                              "align_current_saved_reg_area_pointer");
+
+    __current_saved_reg_area_pointer =
+        CGF.Builder.CreateIntToPtr(__current_saved_reg_area_pointer_int,
+                                   __current_saved_reg_area_pointer->getType(),
+                                   "align_current_saved_reg_area_pointer");
+  }
+
+  llvm::Value *__new_saved_reg_area_pointer =
+      CGF.Builder.CreateGEP(__current_saved_reg_area_pointer,
+                            llvm::ConstantInt::get(CGF.Int32Ty, ArgSize),
+                            "__new_saved_reg_area_pointer");
+
+  llvm::Value *UsingStack = 0;
+  UsingStack = CGF.Builder.CreateICmpSGT(__new_saved_reg_area_pointer,
+                                         __saved_reg_area_end_pointer);
+
+  CGF.Builder.CreateCondBr(UsingStack, OnStackBlock, InRegBlock);
+
+  // Argument in saved register area
+  // Implement the block where argument is in register saved area
+  CGF.EmitBlock(InRegBlock);
+
+  llvm::Type *PTy = CGF.ConvertType(Ty);
+  llvm::Value *__saved_reg_area_p = CGF.Builder.CreateBitCast(
+      __current_saved_reg_area_pointer, llvm::PointerType::getUnqual(PTy));
+
+  CGF.Builder.CreateStore(__new_saved_reg_area_pointer,
+                          __current_saved_reg_area_pointer_p);
+
+  CGF.EmitBranch(ContBlock);
+
+  // Argument in overflow area
+  // Implement the block where the argument is in overflow area.
+  CGF.EmitBlock(OnStackBlock);
+
+  // Load the overflow area pointer
+  Address __overflow_area_pointer_p =
+      CGF.Builder.CreateStructGEP(VAListAddr, 2, "__overflow_area_pointer_p");
+  llvm::Value *__overflow_area_pointer = CGF.Builder.CreateLoad(
+      __overflow_area_pointer_p, "__overflow_area_pointer");
+
+  // Align the overflow area pointer according to the alignment of the argument
+  if (ArgAlign > 4) {
+    llvm::Value *__overflow_area_pointer_int =
+        CGF.Builder.CreatePtrToInt(__overflow_area_pointer, CGF.Int32Ty);
+
+    __overflow_area_pointer_int =
+        CGF.Builder.CreateAdd(__overflow_area_pointer_int,
+                              llvm::ConstantInt::get(CGF.Int32Ty, ArgAlign - 1),
+                              "align_overflow_area_pointer");
+
+    __overflow_area_pointer_int =
+        CGF.Builder.CreateAnd(__overflow_area_pointer_int,
+                              llvm::ConstantInt::get(CGF.Int32Ty, -ArgAlign),
+                              "align_overflow_area_pointer");
+
+    __overflow_area_pointer = CGF.Builder.CreateIntToPtr(
+        __overflow_area_pointer_int, __overflow_area_pointer->getType(),
+        "align_overflow_area_pointer");
+  }
+
+  // Get the pointer for next argument in overflow area and store it
+  // to overflow area pointer.
+  llvm::Value *__new_overflow_area_pointer = CGF.Builder.CreateGEP(
+      __overflow_area_pointer, llvm::ConstantInt::get(CGF.Int32Ty, ArgSize),
+      "__overflow_area_pointer.next");
+
+  CGF.Builder.CreateStore(__new_overflow_area_pointer,
+                          __overflow_area_pointer_p);
+
+  CGF.Builder.CreateStore(__new_overflow_area_pointer,
+                          __current_saved_reg_area_pointer_p);
+
+  // Bitcast the overflow area pointer to the type of argument.
+  llvm::Type *OverflowPTy = CGF.ConvertTypeForMem(Ty);
+  llvm::Value *__overflow_area_p = CGF.Builder.CreateBitCast(
+      __overflow_area_pointer, llvm::PointerType::getUnqual(OverflowPTy));
+
+  CGF.EmitBranch(ContBlock);
+
+  // Get the correct pointer to load the variable argument
+  // Implement the ContBlock
+  CGF.EmitBlock(ContBlock);
+
+  llvm::Type *MemPTy = llvm::PointerType::getUnqual(CGF.ConvertTypeForMem(Ty));
+  llvm::PHINode *ArgAddr = CGF.Builder.CreatePHI(MemPTy, 2, "vaarg.addr");
+  ArgAddr->addIncoming(__saved_reg_area_p, InRegBlock);
+  ArgAddr->addIncoming(__overflow_area_p, OnStackBlock);
+
+  return Address(ArgAddr, CharUnits::fromQuantity(ArgAlign));
+}
+
 Address HexagonABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
                                   QualType Ty) const {
-  // FIXME: Someone needs to audit that this handle alignment correctly.
-  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*indirect*/ false,
-                          getContext().getTypeInfoInChars(Ty),
-                          CharUnits::fromQuantity(4),
-                          /*AllowHigherAlign*/ true);
+
+  if (getTarget().getTriple().isMusl())
+    return EmitVAArgForHexagonLinux(CGF, VAListAddr, Ty);
+
+  return EmitVAArgForHexagon(CGF, VAListAddr, Ty);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/CodeGen/TargetInfo.h b/clang/lib/CodeGen/TargetInfo.h
index e1e90e73cb587..e7c842bae4a98 100644
--- a/clang/lib/CodeGen/TargetInfo.h
+++ b/clang/lib/CodeGen/TargetInfo.h
@@ -315,6 +315,32 @@ class TargetCodeGenInfo {
   virtual bool shouldEmitStaticExternCAliases() const { return true; }
 
   virtual void setCUDAKernelCallingConvention(const FunctionType *&FT) const {}
+
+  /// Return the device-side type for the CUDA device builtin surface type.
+  virtual llvm::Type *getCUDADeviceBuiltinSurfaceDeviceType() const {
+    // By default, no change from the original one.
+    return nullptr;
+  }
+  /// Return the device-side type for the CUDA device builtin texture type.
+  virtual llvm::Type *getCUDADeviceBuiltinTextureDeviceType() const {
+    // By default, no change from the original one.
+    return nullptr;
+  }
+
+  /// Emit the device-side copy of the builtin surface type.
+  virtual bool emitCUDADeviceBuiltinSurfaceDeviceCopy(CodeGenFunction &CGF,
+                                                      LValue Dst,
+                                                      LValue Src) const {
+    // DO NOTHING by default.
+    return false;
+  }
+  /// Emit the device-side copy of the builtin texture type.
+  virtual bool emitCUDADeviceBuiltinTextureDeviceCopy(CodeGenFunction &CGF,
+                                                      LValue Dst,
+                                                      LValue Src) const {
+    // DO NOTHING by default.
+    return false;
+  }
 };
 
 } // namespace CodeGen
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 31115e51dea16..029ba62b5209f 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -141,6 +141,13 @@ Driver::Driver(StringRef ClangExecutable, StringRef TargetTriple,
   Dir = std::string(llvm::sys::path::parent_path(ClangExecutable));
   InstalledDir = Dir; // Provide a sensible default installed dir.
 
+  if ((!SysRoot.empty()) && llvm::sys::path::is_relative(SysRoot)) {
+    // Prepend InstalledDir if SysRoot is relative
+    SmallString<128> P(InstalledDir);
+    llvm::sys::path::append(P, SysRoot);
+    SysRoot = std::string(P);
+  }
+
 #if defined(CLANG_CONFIG_FILE_SYSTEM_DIR)
   SystemConfigDir = CLANG_CONFIG_FILE_SYSTEM_DIR;
 #endif
@@ -1454,10 +1461,6 @@ void Driver::generateCompilationDiagnostics(
   // Print the version of the compiler.
   PrintVersion(C, llvm::errs());
 
-  Diag(clang::diag::note_drv_command_failed_diag_msg)
-      << "PLEASE submit a bug report to " BUG_REPORT_URL " and include the "
-         "crash backtrace, preprocessed source, and associated run script.";
-
   // Suppress driver output and emit preprocessor output to temp file.
   Mode = CPPMode;
   CCGenDiagnostics = true;
@@ -2976,13 +2979,13 @@ class OffloadingActionBuilder final {
       std::set<CudaArch> GpuArchs;
       bool Error = false;
       for (Arg *A : Args) {
-        if (!(A->getOption().matches(options::OPT_cuda_gpu_arch_EQ) ||
-              A->getOption().matches(options::OPT_no_cuda_gpu_arch_EQ)))
+        if (!(A->getOption().matches(options::OPT_offload_arch_EQ) ||
+              A->getOption().matches(options::OPT_no_offload_arch_EQ)))
           continue;
         A->claim();
 
         const StringRef ArchStr = A->getValue();
-        if (A->getOption().matches(options::OPT_no_cuda_gpu_arch_EQ) &&
+        if (A->getOption().matches(options::OPT_no_offload_arch_EQ) &&
             ArchStr == "all") {
           GpuArchs.clear();
           continue;
@@ -2991,9 +2994,9 @@ class OffloadingActionBuilder final {
         if (Arch == CudaArch::UNKNOWN) {
           C.getDriver().Diag(clang::diag::err_drv_cuda_bad_gpu_arch) << ArchStr;
           Error = true;
-        } else if (A->getOption().matches(options::OPT_cuda_gpu_arch_EQ))
+        } else if (A->getOption().matches(options::OPT_offload_arch_EQ))
           GpuArchs.insert(Arch);
-        else if (A->getOption().matches(options::OPT_no_cuda_gpu_arch_EQ))
+        else if (A->getOption().matches(options::OPT_no_offload_arch_EQ))
           GpuArchs.erase(Arch);
         else
           llvm_unreachable("Unexpected option.");
@@ -3982,7 +3985,7 @@ class OffloadingActionBuilder final {
         A->claim();
         auto ParsedArg = Opts.ParseOneArg(Args, Index);
         // TODO: Support --no-cuda-gpu-arch, --{,no-}cuda-gpu-arch=all.
-        if (ParsedArg->getOption().matches(options::OPT_cuda_gpu_arch_EQ)) {
+        if (ParsedArg->getOption().matches(options::OPT_offload_arch_EQ)) {
           ParsedArg->claim();
           GpuArchList.push_back(StringToCudaArch(ParsedArg->getValue(0)));
         }
diff --git a/clang/lib/Driver/ToolChains/AMDGPU.cpp b/clang/lib/Driver/ToolChains/AMDGPU.cpp
index 06e4686ac2b98..2cec0dc9de22c 100644
--- a/clang/lib/Driver/ToolChains/AMDGPU.cpp
+++ b/clang/lib/Driver/ToolChains/AMDGPU.cpp
@@ -103,6 +103,18 @@ AMDGPUToolChain::TranslateArgs(const DerivedArgList &Args, StringRef BoundArch,
   return DAL;
 }
 
+bool AMDGPUToolChain::getDefaultDenormsAreZeroForTarget(
+    llvm::AMDGPU::GPUKind Kind) {
+  const unsigned ArchAttr = llvm::AMDGPU::getArchAttrAMDGCN(Kind);
+
+  // Default to enabling f32 denormals by default on subtargets where fma is
+  // fast with denormals
+  const bool BothDenormAndFMAFast =
+      (ArchAttr & llvm::AMDGPU::FEATURE_FAST_FMA_F32) &&
+      (ArchAttr & llvm::AMDGPU::FEATURE_FAST_DENORMAL_F32);
+  return !BothDenormAndFMAFast;
+}
+
 llvm::DenormalMode AMDGPUToolChain::getDefaultDenormalModeForType(
     const llvm::opt::ArgList &DriverArgs, Action::OffloadKind DeviceOffloadKind,
     const llvm::fltSemantics *FPType) const {
@@ -121,18 +133,10 @@ llvm::DenormalMode AMDGPUToolChain::getDefaultDenormalModeForType(
   const StringRef GpuArch = DriverArgs.getLastArgValue(options::OPT_mcpu_EQ);
   auto Kind = llvm::AMDGPU::parseArchAMDGCN(GpuArch);
 
-  // Default to enabling f32 denormals by default on subtargets where fma is
-  // fast with denormals
-
-  const unsigned ArchAttr = llvm::AMDGPU::getArchAttrAMDGCN(Kind);
-  const bool DefaultDenormsAreZeroForTarget =
-    (ArchAttr & llvm::AMDGPU::FEATURE_FAST_FMA_F32) &&
-    (ArchAttr & llvm::AMDGPU::FEATURE_FAST_DENORMAL_F32);
-
   // TODO: There are way too many flags that change this. Do we need to check
   // them all?
   bool DAZ = DriverArgs.hasArg(options::OPT_cl_denorms_are_zero) ||
-             !DefaultDenormsAreZeroForTarget;
+             getDefaultDenormsAreZeroForTarget(Kind);
   // Outputs are flushed to zero, preserving sign
   return DAZ ? llvm::DenormalMode::getPreserveSign() :
                llvm::DenormalMode::getIEEE();
diff --git a/clang/lib/Driver/ToolChains/AMDGPU.h b/clang/lib/Driver/ToolChains/AMDGPU.h
index 78c40580b3020..e7a873efb0085 100644
--- a/clang/lib/Driver/ToolChains/AMDGPU.h
+++ b/clang/lib/Driver/ToolChains/AMDGPU.h
@@ -13,6 +13,8 @@
 #include "clang/Driver/Options.h"
 #include "clang/Driver/Tool.h"
 #include "clang/Driver/ToolChain.h"
+#include "llvm/Support/TargetParser.h"
+
 #include <map>
 
 namespace clang {
@@ -67,6 +69,10 @@ class LLVM_LIBRARY_VISIBILITY AMDGPUToolChain : public Generic_ELF {
                              llvm::opt::ArgStringList &CC1Args,
                              Action::OffloadKind DeviceOffloadKind) const override;
 
+  /// Return whether denormals should be flushed, and treated as 0 by default
+  /// for the subtarget.
+  static bool getDefaultDenormsAreZeroForTarget(llvm::AMDGPU::GPUKind GPUKind);
+
   llvm::DenormalMode getDefaultDenormalModeForType(
       const llvm::opt::ArgList &DriverArgs,
       Action::OffloadKind DeviceOffloadKind,
diff --git a/clang/lib/Driver/ToolChains/Arch/AArch64.cpp b/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
index dc31a5ab7b0f0..b21cfac6e7ed7 100644
--- a/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
@@ -140,7 +140,8 @@ getAArch64MicroArchFeaturesFromMtune(const Driver &D, StringRef Mtune,
   // Handle CPU name is 'native'.
   if (MtuneLowerCase == "native")
     MtuneLowerCase = std::string(llvm::sys::getHostCPUName());
-  if (MtuneLowerCase == "cyclone" || MtuneLowerCase.find("apple") == 0) {
+  if (MtuneLowerCase == "cyclone" ||
+      StringRef(MtuneLowerCase).startswith("apple")) {
     Features.push_back("+zcm");
     Features.push_back("+zcz");
   }
diff --git a/clang/lib/Driver/ToolChains/Arch/SystemZ.cpp b/clang/lib/Driver/ToolChains/Arch/SystemZ.cpp
index 4d871104c95ac..f81bf68172de1 100644
--- a/clang/lib/Driver/ToolChains/Arch/SystemZ.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/SystemZ.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "SystemZ.h"
+#include "clang/Config/config.h"
 #include "clang/Driver/DriverDiagnostic.h"
 #include "clang/Driver/Options.h"
 #include "llvm/Option/ArgList.h"
@@ -47,7 +48,7 @@ std::string systemz::getSystemZTargetCPU(const ArgList &Args) {
 
     return std::string(CPUName);
   }
-  return "z10";
+  return CLANG_SYSTEMZ_DEFAULT_ARCH;
 }
 
 void systemz::getSystemZTargetFeatures(const Driver &D, const ArgList &Args,
diff --git a/clang/lib/Driver/ToolChains/Arch/X86.cpp b/clang/lib/Driver/ToolChains/Arch/X86.cpp
index 44a636dcfd1ce..dbbc025de38c0 100644
--- a/clang/lib/Driver/ToolChains/Arch/X86.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/X86.cpp
@@ -146,6 +146,7 @@ void x86::getX86TargetFeatures(const Driver &D, const llvm::Triple &Triple,
   // flags). This is a bit hacky but keeps existing usages working. We should
   // consider deprecating this and instead warn if the user requests external
   // retpoline thunks and *doesn't* request some form of retpolines.
+  auto SpectreOpt = clang::driver::options::ID::OPT_INVALID;
   if (Args.hasArgNoClaim(options::OPT_mretpoline, options::OPT_mno_retpoline,
                          options::OPT_mspeculative_load_hardening,
                          options::OPT_mno_speculative_load_hardening)) {
@@ -153,12 +154,14 @@ void x86::getX86TargetFeatures(const Driver &D, const llvm::Triple &Triple,
                      false)) {
       Features.push_back("+retpoline-indirect-calls");
       Features.push_back("+retpoline-indirect-branches");
+      SpectreOpt = options::OPT_mretpoline;
     } else if (Args.hasFlag(options::OPT_mspeculative_load_hardening,
                             options::OPT_mno_speculative_load_hardening,
                             false)) {
       // On x86, speculative load hardening relies on at least using retpolines
       // for indirect calls.
       Features.push_back("+retpoline-indirect-calls");
+      SpectreOpt = options::OPT_mspeculative_load_hardening;
     }
   } else if (Args.hasFlag(options::OPT_mretpoline_external_thunk,
                           options::OPT_mno_retpoline_external_thunk, false)) {
@@ -166,6 +169,26 @@ void x86::getX86TargetFeatures(const Driver &D, const llvm::Triple &Triple,
     // eventually switch to an error here.
     Features.push_back("+retpoline-indirect-calls");
     Features.push_back("+retpoline-indirect-branches");
+    SpectreOpt = options::OPT_mretpoline_external_thunk;
+  }
+
+  auto LVIOpt = clang::driver::options::ID::OPT_INVALID;
+  if (Args.hasFlag(options::OPT_mlvi_hardening, options::OPT_mno_lvi_hardening,
+                   false)) {
+    Features.push_back("+lvi-load-hardening");
+    Features.push_back("+lvi-cfi"); // load hardening implies CFI protection
+    LVIOpt = options::OPT_mlvi_hardening;
+  } else if (Args.hasFlag(options::OPT_mlvi_cfi, options::OPT_mno_lvi_cfi,
+                          false)) {
+    Features.push_back("+lvi-cfi");
+    LVIOpt = options::OPT_mlvi_cfi;
+  }
+
+  if (SpectreOpt != clang::driver::options::ID::OPT_INVALID &&
+      LVIOpt != clang::driver::options::ID::OPT_INVALID) {
+    D.Diag(diag::err_drv_argument_not_allowed_with)
+        << D.getOpts().getOptionName(SpectreOpt)
+        << D.getOpts().getOptionName(LVIOpt);
   }
 
   // Now add any that the user explicitly requested on the command line,
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 12d5345a64af5..603d04f0a9b3d 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -3579,9 +3579,9 @@ static void RenderDiagnosticsOptions(const Driver &D, const ArgList &Args,
     CmdArgs.push_back("-fno-diagnostics-fixit-info");
 
   // Enable -fdiagnostics-show-option by default.
-  if (Args.hasFlag(options::OPT_fdiagnostics_show_option,
-                   options::OPT_fno_diagnostics_show_option))
-    CmdArgs.push_back("-fdiagnostics-show-option");
+  if (!Args.hasFlag(options::OPT_fdiagnostics_show_option,
+                    options::OPT_fno_diagnostics_show_option, true))
+    CmdArgs.push_back("-fno-diagnostics-show-option");
 
   if (const Arg *A =
           Args.getLastArg(options::OPT_fdiagnostics_show_category_EQ)) {
@@ -4273,7 +4273,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
       StringRef ArgStr =
           Args.hasArg(options::OPT_interface_stub_version_EQ)
               ? Args.getLastArgValue(options::OPT_interface_stub_version_EQ)
-              : "experimental-ifs-v1";
+              : "experimental-ifs-v2";
       CmdArgs.push_back("-emit-interface-stubs");
       CmdArgs.push_back(
           Args.MakeArgString(Twine("-interface-stub-version=") + ArgStr.str()));
@@ -4733,9 +4733,9 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   // Decide whether to use verbose asm. Verbose assembly is the default on
   // toolchains which have the integrated assembler on by default.
   bool IsIntegratedAssemblerDefault = TC.IsIntegratedAssemblerDefault();
-  if (Args.hasFlag(options::OPT_fverbose_asm, options::OPT_fno_verbose_asm,
-                   IsIntegratedAssemblerDefault))
-    CmdArgs.push_back("-masm-verbose");
+  if (!Args.hasFlag(options::OPT_fverbose_asm, options::OPT_fno_verbose_asm,
+                    IsIntegratedAssemblerDefault))
+    CmdArgs.push_back("-fno-verbose-asm");
 
   if (!TC.useIntegratedAs())
     CmdArgs.push_back("-no-integrated-as");
@@ -5220,15 +5220,20 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   }
 
   // Pass -fmessage-length=.
-  CmdArgs.push_back("-fmessage-length");
+  unsigned MessageLength = 0;
   if (Arg *A = Args.getLastArg(options::OPT_fmessage_length_EQ)) {
-    CmdArgs.push_back(A->getValue());
+    StringRef V(A->getValue());
+    if (V.getAsInteger(0, MessageLength))
+      D.Diag(diag::err_drv_invalid_argument_to_option)
+          << V << A->getOption().getName();
   } else {
     // If -fmessage-length=N was not specified, determine whether this is a
     // terminal and, if so, implicitly define -fmessage-length appropriately.
-    unsigned N = llvm::sys::Process::StandardErrColumns();
-    CmdArgs.push_back(Args.MakeArgString(Twine(N)));
+    MessageLength = llvm::sys::Process::StandardErrColumns();
   }
+  if (MessageLength != 0)
+    CmdArgs.push_back(
+        Args.MakeArgString("-fmessage-length=" + Twine(MessageLength)));
 
   // -fvisibility= and -fvisibility-ms-compat are of a piece.
   if (const Arg *A = Args.getLastArg(options::OPT_fvisibility_EQ,
@@ -5701,7 +5706,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   Args.AddLastArg(CmdArgs, options::OPT_fexperimental_new_pass_manager,
                   options::OPT_fno_experimental_new_pass_manager);
 
-  ObjCRuntime Runtime = AddObjCRuntimeArgs(Args, CmdArgs, rewriteKind);
+  ObjCRuntime Runtime = AddObjCRuntimeArgs(Args, Inputs, CmdArgs, rewriteKind);
   RenderObjCOptions(TC, D, RawTriple, Args, Runtime, rewriteKind != RK_None,
                     Input, CmdArgs);
 
@@ -6218,10 +6223,10 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (SplitLTOUnit)
     CmdArgs.push_back("-fsplit-lto-unit");
 
-  if (Arg *A = Args.getLastArg(options::OPT_fexperimental_isel,
-                               options::OPT_fno_experimental_isel)) {
+  if (Arg *A = Args.getLastArg(options::OPT_fglobal_isel,
+                               options::OPT_fno_global_isel)) {
     CmdArgs.push_back("-mllvm");
-    if (A->getOption().matches(options::OPT_fexperimental_isel)) {
+    if (A->getOption().matches(options::OPT_fglobal_isel)) {
       CmdArgs.push_back("-global-isel=1");
 
       // GISel is on by default on AArch64 -O0, so don't bother adding
@@ -6240,9 +6245,9 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
         CmdArgs.push_back("-global-isel-abort=2");
 
         if (!IsArchSupported)
-          D.Diag(diag::warn_drv_experimental_isel_incomplete) << Triple.getArchName();
+          D.Diag(diag::warn_drv_global_isel_incomplete) << Triple.getArchName();
         else
-          D.Diag(diag::warn_drv_experimental_isel_incomplete_opt);
+          D.Diag(diag::warn_drv_global_isel_incomplete_opt);
       }
     } else {
       CmdArgs.push_back("-global-isel=0");
@@ -6408,6 +6413,7 @@ Clang::~Clang() {}
 ///
 /// Returns true if the runtime is non-fragile.
 ObjCRuntime Clang::AddObjCRuntimeArgs(const ArgList &args,
+                                      const InputInfoList &inputs,
                                       ArgStringList &cmdArgs,
                                       RewriteKind rewriteKind) const {
   // Look for the controlling runtime option.
@@ -6531,8 +6537,11 @@ ObjCRuntime Clang::AddObjCRuntimeArgs(const ArgList &args,
       runtime = ObjCRuntime(ObjCRuntime::GCC, VersionTuple());
   }
 
-  cmdArgs.push_back(
-      args.MakeArgString("-fobjc-runtime=" + runtime.getAsString()));
+  if (llvm::any_of(inputs, [](const InputInfo &input) {
+        return types::isObjC(input.getType());
+      }))
+    cmdArgs.push_back(
+        args.MakeArgString("-fobjc-runtime=" + runtime.getAsString()));
   return runtime;
 }
 
diff --git a/clang/lib/Driver/ToolChains/Clang.h b/clang/lib/Driver/ToolChains/Clang.h
index 1552515c1461a..64af2fdd5115c 100644
--- a/clang/lib/Driver/ToolChains/Clang.h
+++ b/clang/lib/Driver/ToolChains/Clang.h
@@ -77,6 +77,7 @@ class LLVM_LIBRARY_VISIBILITY Clang : public Tool {
   enum RewriteKind { RK_None, RK_Fragile, RK_NonFragile };
 
   ObjCRuntime AddObjCRuntimeArgs(const llvm::opt::ArgList &args,
+                                 const InputInfoList &inputs,
                                  llvm::opt::ArgStringList &cmdArgs,
                                  RewriteKind rewrite) const;
 
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index ebdb22fae3963..2ca50fb948bb9 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -50,6 +50,7 @@
 #include "llvm/Support/Program.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include "llvm/Support/TargetParser.h"
+#include "llvm/Support/Threading.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include "llvm/Support/YAMLParser.h"
 
@@ -346,14 +347,14 @@ std::string tools::getCPUName(const ArgList &Args, const llvm::Triple &T,
   }
 }
 
-unsigned tools::getLTOParallelism(const ArgList &Args, const Driver &D) {
-  unsigned Parallelism = 0;
+llvm::StringRef tools::getLTOParallelism(const ArgList &Args, const Driver &D) {
   Arg *LtoJobsArg = Args.getLastArg(options::OPT_flto_jobs_EQ);
-  if (LtoJobsArg &&
-      StringRef(LtoJobsArg->getValue()).getAsInteger(10, Parallelism))
-    D.Diag(diag::err_drv_invalid_int_value) << LtoJobsArg->getAsString(Args)
-                                            << LtoJobsArg->getValue();
-  return Parallelism;
+  if (!LtoJobsArg)
+    return {};
+  if (!llvm::get_threadpool_strategy(LtoJobsArg->getValue()))
+    D.Diag(diag::err_drv_invalid_int_value)
+        << LtoJobsArg->getAsString(Args) << LtoJobsArg->getValue();
+  return LtoJobsArg->getValue();
 }
 
 // CloudABI uses -ffunction-sections and -fdata-sections by default.
@@ -418,7 +419,8 @@ void tools::addLTOOptions(const ToolChain &ToolChain, const ArgList &Args,
   if (IsThinLTO)
     CmdArgs.push_back("-plugin-opt=thinlto");
 
-  if (unsigned Parallelism = getLTOParallelism(Args, ToolChain.getDriver()))
+  StringRef Parallelism = getLTOParallelism(Args, ToolChain.getDriver());
+  if (!Parallelism.empty())
     CmdArgs.push_back(
         Args.MakeArgString("-plugin-opt=jobs=" + Twine(Parallelism)));
 
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.h b/clang/lib/Driver/ToolChains/CommonArgs.h
index 984f3ee98af1e..c94b2b828c9b8 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.h
+++ b/clang/lib/Driver/ToolChains/CommonArgs.h
@@ -88,7 +88,8 @@ llvm::opt::Arg *getLastProfileSampleUseArg(const llvm::opt::ArgList &Args);
 
 bool isObjCAutoRefCount(const llvm::opt::ArgList &Args);
 
-unsigned getLTOParallelism(const llvm::opt::ArgList &Args, const Driver &D);
+llvm::StringRef getLTOParallelism(const llvm::opt::ArgList &Args,
+                                  const Driver &D);
 
 bool areOptimizationsEnabled(const llvm::opt::ArgList &Args);
 
diff --git a/clang/lib/Driver/ToolChains/Darwin.cpp b/clang/lib/Driver/ToolChains/Darwin.cpp
index d1aec649cdc71..451d0d206d07d 100644
--- a/clang/lib/Driver/ToolChains/Darwin.cpp
+++ b/clang/lib/Driver/ToolChains/Darwin.cpp
@@ -23,6 +23,7 @@
 #include "llvm/Support/Path.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include "llvm/Support/TargetParser.h"
+#include "llvm/Support/Threading.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include <cstdlib> // ::getenv
 
@@ -605,10 +606,12 @@ void darwin::Linker::ConstructJob(Compilation &C, const JobAction &JA,
 
   getMachOToolChain().addProfileRTLibs(Args, CmdArgs);
 
-  if (unsigned Parallelism =
-          getLTOParallelism(Args, getToolChain().getDriver())) {
+  StringRef Parallelism = getLTOParallelism(Args, getToolChain().getDriver());
+  if (!Parallelism.empty()) {
     CmdArgs.push_back("-mllvm");
-    CmdArgs.push_back(Args.MakeArgString("-threads=" + Twine(Parallelism)));
+    unsigned NumThreads =
+        llvm::get_threadpool_strategy(Parallelism)->compute_thread_count();
+    CmdArgs.push_back(Args.MakeArgString("-threads=" + Twine(NumThreads)));
   }
 
   if (getToolChain().ShouldLinkCXXStdlib(Args))
diff --git a/clang/lib/Driver/ToolChains/HIP.cpp b/clang/lib/Driver/ToolChains/HIP.cpp
index 157dca7e0c8db..e4ace81dbac79 100644
--- a/clang/lib/Driver/ToolChains/HIP.cpp
+++ b/clang/lib/Driver/ToolChains/HIP.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "HIP.h"
+#include "AMDGPU.h"
 #include "CommonArgs.h"
 #include "InputInfo.h"
 #include "clang/Basic/Cuda.h"
@@ -16,6 +17,7 @@
 #include "clang/Driver/Options.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/TargetParser.h"
 
 using namespace clang::driver;
 using namespace clang::driver::toolchains;
@@ -266,7 +268,7 @@ void AMDGCN::Linker::ConstructJob(Compilation &C, const JobAction &JA,
 
 HIPToolChain::HIPToolChain(const Driver &D, const llvm::Triple &Triple,
                              const ToolChain &HostTC, const ArgList &Args)
-    : ToolChain(D, Triple, Args), HostTC(HostTC) {
+    : AMDGPUToolChain(D, Triple, Args), HostTC(HostTC) {
   // Lookup binaries into the driver directory, this is used to
   // discover the clang-offload-bundler executable.
   getProgramPaths().push_back(getDriver().Dir);
@@ -283,6 +285,7 @@ void HIPToolChain::addClangTargetOptions(
   (void) GpuArch;
   assert(DeviceOffloadingKind == Action::OFK_HIP &&
          "Only HIP offloading kinds are supported for GPUs.");
+  auto Kind = llvm::AMDGPU::parseArchAMDGCN(GpuArch);
 
   CC1Args.push_back("-target-cpu");
   CC1Args.push_back(DriverArgs.MakeArgStringRef(GpuArch));
@@ -343,11 +346,14 @@ void HIPToolChain::addClangTargetOptions(
     std::string GFXVersion = GpuArch.drop_front(3).str();
     std::string ISAVerBC = "oclc_isa_version_" + GFXVersion + ".amdgcn.bc";
 
-    llvm::StringRef FlushDenormalControlBC;
-    if (DriverArgs.hasArg(options::OPT_fcuda_flush_denormals_to_zero))
-      FlushDenormalControlBC = "oclc_daz_opt_on.amdgcn.bc";
-    else
-      FlushDenormalControlBC = "oclc_daz_opt_off.amdgcn.bc";
+    bool FTZDAZ = DriverArgs.hasFlag(
+      options::OPT_fcuda_flush_denormals_to_zero,
+      options::OPT_fno_cuda_flush_denormals_to_zero,
+      getDefaultDenormsAreZeroForTarget(Kind));
+
+    std::string FlushDenormalControlBC = FTZDAZ ?
+      "oclc_daz_opt_on.amdgcn.bc" :
+      "oclc_daz_opt_off.amdgcn.bc";
 
     llvm::StringRef WaveFrontSizeBC;
     if (stoi(GFXVersion) < 1000)
@@ -357,7 +363,7 @@ void HIPToolChain::addClangTargetOptions(
 
     BCLibs.append({"hip.amdgcn.bc", "ocml.amdgcn.bc", "ockl.amdgcn.bc",
                    "oclc_finite_only_off.amdgcn.bc",
-                   std::string(FlushDenormalControlBC),
+                   FlushDenormalControlBC,
                    "oclc_correctly_rounded_sqrt_on.amdgcn.bc",
                    "oclc_unsafe_math_off.amdgcn.bc", ISAVerBC,
                    std::string(WaveFrontSizeBC)});
diff --git a/clang/lib/Driver/ToolChains/HIP.h b/clang/lib/Driver/ToolChains/HIP.h
index c4f944e458bf2..b6a3a27186353 100644
--- a/clang/lib/Driver/ToolChains/HIP.h
+++ b/clang/lib/Driver/ToolChains/HIP.h
@@ -11,6 +11,7 @@
 
 #include "clang/Driver/ToolChain.h"
 #include "clang/Driver/Tool.h"
+#include "AMDGPU.h"
 
 namespace clang {
 namespace driver {
@@ -72,7 +73,7 @@ class LLVM_LIBRARY_VISIBILITY Linker : public Tool {
 
 namespace toolchains {
 
-class LLVM_LIBRARY_VISIBILITY HIPToolChain : public ToolChain {
+class LLVM_LIBRARY_VISIBILITY HIPToolChain final : public AMDGPUToolChain {
 public:
   HIPToolChain(const Driver &D, const llvm::Triple &Triple,
                 const ToolChain &HostTC, const llvm::opt::ArgList &Args);
diff --git a/clang/lib/Driver/ToolChains/PS4CPU.cpp b/clang/lib/Driver/ToolChains/PS4CPU.cpp
index 9ecbb7241d454..ebeed3803e06d 100644
--- a/clang/lib/Driver/ToolChains/PS4CPU.cpp
+++ b/clang/lib/Driver/ToolChains/PS4CPU.cpp
@@ -434,3 +434,17 @@ SanitizerMask toolchains::PS4CPU::getSupportedSanitizers() const {
   Res |= SanitizerKind::Vptr;
   return Res;
 }
+
+void toolchains::PS4CPU::addClangTargetOptions(
+      const ArgList &DriverArgs,
+      ArgStringList &CC1Args,
+      Action::OffloadKind DeviceOffloadingKind) const {
+  // PS4 does not use init arrays.
+  if (DriverArgs.hasArg(options::OPT_fuse_init_array)) {
+    Arg *A = DriverArgs.getLastArg(options::OPT_fuse_init_array);
+    getDriver().Diag(clang::diag::err_drv_unsupported_opt_for_target)
+        << A->getAsString(DriverArgs) << getTriple().str();
+  }
+
+  CC1Args.push_back("-fno-use-init-array");
+}
diff --git a/clang/lib/Driver/ToolChains/PS4CPU.h b/clang/lib/Driver/ToolChains/PS4CPU.h
index c82b0c3a1b56a..8fedb6eda0eff 100644
--- a/clang/lib/Driver/ToolChains/PS4CPU.h
+++ b/clang/lib/Driver/ToolChains/PS4CPU.h
@@ -88,6 +88,11 @@ class LLVM_LIBRARY_VISIBILITY PS4CPU : public Generic_ELF {
   // capable of unit splitting.
   bool canSplitThinLTOUnit() const override { return false; }
 
+  void addClangTargetOptions(
+    const llvm::opt::ArgList &DriverArgs,
+    llvm::opt::ArgStringList &CC1Args,
+    Action::OffloadKind DeviceOffloadingKind) const override;
+
   llvm::DenormalMode getDefaultDenormalModeForType(
     const llvm::opt::ArgList &DriverArgs,
     Action::OffloadKind DeviceOffloadKind,
diff --git a/clang/lib/Format/ContinuationIndenter.cpp b/clang/lib/Format/ContinuationIndenter.cpp
index d2397dbfeb87b..ba42ba0ca0500 100644
--- a/clang/lib/Format/ContinuationIndenter.cpp
+++ b/clang/lib/Format/ContinuationIndenter.cpp
@@ -346,6 +346,11 @@ bool ContinuationIndenter::mustBreak(const LineState &State) {
       Current.startsSequence(TT_SelectorName, tok::colon, tok::caret)) {
     return true;
   }
+  // Avoid producing inconsistent states by requiring breaks where they are not
+  // permitted for C# generic type constraints.
+  if (State.Stack.back().IsCSharpGenericTypeConstraint &&
+      Previous.isNot(TT_CSharpGenericTypeConstraintComma))
+    return false;
   if ((startsNextParameter(Current, Style) || Previous.is(tok::semi) ||
        (Previous.is(TT_TemplateCloser) && Current.is(TT_StartOfName) &&
         Style.isCpp() &&
diff --git a/clang/lib/Format/FormatToken.h b/clang/lib/Format/FormatToken.h
index 10a5f0e96f96f..48ec7602c21c2 100644
--- a/clang/lib/Format/FormatToken.h
+++ b/clang/lib/Format/FormatToken.h
@@ -910,9 +910,64 @@ struct AdditionalKeywords {
   /// Returns \c true if \p Tok is a true JavaScript identifier, returns
   /// \c false if it is a keyword or a pseudo keyword.
   bool IsJavaScriptIdentifier(const FormatToken &Tok) const {
-    return Tok.is(tok::identifier) &&
-           JsExtraKeywords.find(Tok.Tok.getIdentifierInfo()) ==
-               JsExtraKeywords.end();
+    // Based on the list of JavaScript & TypeScript keywords here:
+    // https://github.com/microsoft/TypeScript/blob/master/src/compiler/scanner.ts#L74
+    switch (Tok.Tok.getKind()) {
+    case tok::kw_break:
+    case tok::kw_case:
+    case tok::kw_catch:
+    case tok::kw_class:
+    case tok::kw_continue:
+    case tok::kw_const:
+    case tok::kw_default:
+    case tok::kw_delete:
+    case tok::kw_do:
+    case tok::kw_else:
+    case tok::kw_enum:
+    case tok::kw_export:
+    case tok::kw_false:
+    case tok::kw_for:
+    case tok::kw_if:
+    case tok::kw_import:
+    case tok::kw_module:
+    case tok::kw_new:
+    case tok::kw_private:
+    case tok::kw_protected:
+    case tok::kw_public:
+    case tok::kw_return:
+    case tok::kw_static:
+    case tok::kw_switch:
+    case tok::kw_this:
+    case tok::kw_throw:
+    case tok::kw_true:
+    case tok::kw_try:
+    case tok::kw_typeof:
+    case tok::kw_void:
+    case tok::kw_while:
+      // These are JS keywords that are lexed by LLVM/clang as keywords.
+      return false;
+    case tok::identifier:
+      // For identifiers, make sure they are true identifiers, excluding the
+      // JavaScript pseudo-keywords (not lexed by LLVM/clang as keywords).
+      return JsExtraKeywords.find(Tok.Tok.getIdentifierInfo()) ==
+             JsExtraKeywords.end();
+    default:
+      // Other keywords are handled in the switch below, to avoid problems due
+      // to duplicate case labels when using the #include trick.
+      break;
+    }
+
+    switch (Tok.Tok.getKind()) {
+      // Handle C++ keywords not included above: these are all JS identifiers.
+#define KEYWORD(X, Y) case tok::kw_##X:
+#include "clang/Basic/TokenKinds.def"
+      // #undef KEYWORD is not needed -- it's #undef-ed at the end of
+      // TokenKinds.def
+      return true;
+    default:
+      // All other tokens (punctuation etc) are not JS identifiers.
+      return false;
+    }
   }
 
   /// Returns \c true if \p Tok is a C# keyword, returns
diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index f2666a8bd1717..029741c3dce70 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -1047,7 +1047,7 @@ class AnnotatingParser {
                        Keywords.kw___has_include_next)) {
         parseHasInclude();
       }
-      if (Tok->is(Keywords.kw_where) && Tok->Next &&
+      if (Style.isCSharp() && Tok->is(Keywords.kw_where) && Tok->Next &&
           Tok->Next->isNot(tok::l_paren)) {
         Tok->Type = TT_CSharpGenericTypeConstraint;
         parseCSharpGenericTypeConstraint();
@@ -1060,15 +1060,20 @@ class AnnotatingParser {
   }
 
   void parseCSharpGenericTypeConstraint() {
+    int OpenAngleBracketsCount = 0;
     while (CurrentToken) {
       if (CurrentToken->is(tok::less)) {
         // parseAngle is too greedy and will consume the whole line.
         CurrentToken->Type = TT_TemplateOpener;
+        ++OpenAngleBracketsCount;
         next();
       } else if (CurrentToken->is(tok::greater)) {
         CurrentToken->Type = TT_TemplateCloser;
+        --OpenAngleBracketsCount;
         next();
-      } else if (CurrentToken->is(tok::comma)) {
+      } else if (CurrentToken->is(tok::comma) && OpenAngleBracketsCount == 0) {
+        // We allow line breaks after GenericTypeConstraintComma's
+        // so do not flag commas in Generics as GenericTypeConstraintComma's.
         CurrentToken->Type = TT_CSharpGenericTypeConstraintComma;
         next();
       } else if (CurrentToken->is(Keywords.kw_where)) {
@@ -1517,9 +1522,9 @@ class AnnotatingParser {
     if (Style.Language == FormatStyle::LK_JavaScript) {
       if (Current.is(tok::exclaim)) {
         if (Current.Previous &&
-            (Current.Previous->isOneOf(tok::identifier, tok::kw_namespace,
-                                       tok::r_paren, tok::r_square,
-                                       tok::r_brace) ||
+            (Keywords.IsJavaScriptIdentifier(*Current.Previous) ||
+             Current.Previous->isOneOf(tok::kw_namespace, tok::r_paren,
+                                       tok::r_square, tok::r_brace) ||
              Current.Previous->Tok.isLiteral())) {
           Current.Type = TT_JsNonNullAssertion;
           return;
@@ -2799,20 +2804,38 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
                                     tok::l_square));
   if (Right.is(tok::star) && Left.is(tok::l_paren))
     return false;
-  if (Right.isOneOf(tok::star, tok::amp, tok::ampamp) &&
-      (Left.is(tok::identifier) || Left.isSimpleTypeSpecifier()) &&
-      // Space between the type and the * in:
-      //   operator void*()
-      //   operator char*()
-      //   operator /*comment*/ const char*()
-      //   operator volatile /*comment*/ char*()
-      //   operator Foo*()
-      // dependent on PointerAlignment style.
-      Left.Previous &&
-      (Left.Previous->endsSequence(tok::kw_operator) ||
-       Left.Previous->endsSequence(tok::kw_const, tok::kw_operator) ||
-       Left.Previous->endsSequence(tok::kw_volatile, tok::kw_operator)))
-    return (Style.PointerAlignment != FormatStyle::PAS_Left);
+  if (Right.isOneOf(tok::star, tok::amp, tok::ampamp)) {
+    const FormatToken *Previous = &Left;
+    while (Previous && !Previous->is(tok::kw_operator)) {
+      if (Previous->is(tok::identifier) || Previous->isSimpleTypeSpecifier()) {
+        Previous = Previous->getPreviousNonComment();
+        continue;
+      }
+      if (Previous->is(TT_TemplateCloser) && Previous->MatchingParen) {
+        Previous = Previous->MatchingParen->getPreviousNonComment();
+        continue;
+      }
+      if (Previous->is(tok::coloncolon)) {
+        Previous = Previous->getPreviousNonComment();
+        continue;
+      }
+      break;
+    }
+    // Space between the type and the * in:
+    //   operator void*()
+    //   operator char*()
+    //   operator /*comment*/ const char*()
+    //   operator volatile /*comment*/ char*()
+    //   operator Foo*()
+    //   operator C<T>*()
+    //   operator std::Foo*()
+    //   operator C<T>::D<U>*()
+    // dependent on PointerAlignment style.
+    if (Previous && (Previous->endsSequence(tok::kw_operator) ||
+       Previous->endsSequence(tok::kw_const, tok::kw_operator) ||
+       Previous->endsSequence(tok::kw_volatile, tok::kw_operator)))
+      return (Style.PointerAlignment != FormatStyle::PAS_Left);
+  }
   const auto SpaceRequiredForArrayInitializerLSquare =
       [](const FormatToken &LSquareTok, const FormatStyle &Style) {
         return Style.SpacesInContainerLiterals ||
@@ -3047,10 +3070,8 @@ bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line,
         (Right.is(TT_TemplateString) && Right.TokenText.startswith("}")))
       return false;
     // In tagged template literals ("html`bar baz`"), there is no space between
-    // the tag identifier and the template string. getIdentifierInfo makes sure
-    // that the identifier is not a pseudo keyword like `yield`, either.
-    if (Left.is(tok::identifier) && Keywords.IsJavaScriptIdentifier(Left) &&
-        Right.is(TT_TemplateString))
+    // the tag identifier and the template string.
+    if (Keywords.IsJavaScriptIdentifier(Left) && Right.is(TT_TemplateString))
       return false;
     if (Right.is(tok::star) &&
         Left.isOneOf(Keywords.kw_function, Keywords.kw_yield))
diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp
index d8202bd614580..18899314512e3 100644
--- a/clang/lib/Format/UnwrappedLineParser.cpp
+++ b/clang/lib/Format/UnwrappedLineParser.cpp
@@ -2340,6 +2340,12 @@ void UnwrappedLineParser::parseRecord(bool ParseAsExpr) {
       }
       if (FormatTok->Tok.is(tok::semi))
         return;
+      if (Style.isCSharp() && FormatTok->is(Keywords.kw_where)) {
+        addUnwrappedLine();
+        nextToken();
+        parseCSharpGenericTypeConstraint();
+        break;
+      }
       nextToken();
     }
   }
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index 646ce7852da47..75cd7ea34e63f 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -827,6 +827,7 @@ static bool ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args, InputKind IK,
   Opts.RerollLoops = Args.hasArg(OPT_freroll_loops);
 
   Opts.DisableIntegratedAS = Args.hasArg(OPT_fno_integrated_as);
+  Opts.CallGraphProfile = !Opts.DisableIntegratedAS;
   Opts.Autolink = !Args.hasArg(OPT_fno_autolink);
   Opts.SampleProfileFile =
       std::string(Args.getLastArgValue(OPT_fprofile_sample_use_EQ));
@@ -858,7 +859,7 @@ static bool ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args, InputKind IK,
   Opts.CoverageMapping =
       Args.hasFlag(OPT_fcoverage_mapping, OPT_fno_coverage_mapping, false);
   Opts.DumpCoverageMapping = Args.hasArg(OPT_dump_coverage_mapping);
-  Opts.AsmVerbose = Args.hasArg(OPT_masm_verbose);
+  Opts.AsmVerbose = !Args.hasArg(OPT_fno_verbose_asm);
   Opts.PreserveAsmComments = !Args.hasArg(OPT_fno_preserve_as_comments);
   Opts.AssumeSaneOperatorNew = !Args.hasArg(OPT_fno_assume_sane_operator_new);
   Opts.ObjCAutoRefCountExceptions = Args.hasArg(OPT_fobjc_arc_exceptions);
@@ -1390,38 +1391,6 @@ static bool ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args, InputKind IK,
 
   Opts.Addrsig = Args.hasArg(OPT_faddrsig);
 
-  if (Arg *A = Args.getLastArg(OPT_msign_return_address_EQ)) {
-    StringRef SignScope = A->getValue();
-
-    if (SignScope.equals_lower("none"))
-      Opts.setSignReturnAddress(CodeGenOptions::SignReturnAddressScope::None);
-    else if (SignScope.equals_lower("all"))
-      Opts.setSignReturnAddress(CodeGenOptions::SignReturnAddressScope::All);
-    else if (SignScope.equals_lower("non-leaf"))
-      Opts.setSignReturnAddress(
-          CodeGenOptions::SignReturnAddressScope::NonLeaf);
-    else
-      Diags.Report(diag::err_drv_invalid_value)
-          << A->getAsString(Args) << SignScope;
-
-    if (Arg *A = Args.getLastArg(OPT_msign_return_address_key_EQ)) {
-      StringRef SignKey = A->getValue();
-      if (!SignScope.empty() && !SignKey.empty()) {
-        if (SignKey.equals_lower("a_key"))
-          Opts.setSignReturnAddressKey(
-              CodeGenOptions::SignReturnAddressKeyValue::AKey);
-        else if (SignKey.equals_lower("b_key"))
-          Opts.setSignReturnAddressKey(
-              CodeGenOptions::SignReturnAddressKeyValue::BKey);
-        else
-          Diags.Report(diag::err_drv_invalid_value)
-              << A->getAsString(Args) << SignKey;
-      }
-    }
-  }
-
-  Opts.BranchTargetEnforcement = Args.hasArg(OPT_mbranch_target_enforce);
-
   Opts.KeepStaticConsts = Args.hasArg(OPT_fkeep_static_consts);
 
   Opts.SpeculativeLoadHardening = Args.hasArg(OPT_mspeculative_load_hardening);
@@ -1552,7 +1521,7 @@ static bool checkVerifyPrefixes(const std::vector<std::string> &VerifyPrefixes,
 
 bool clang::ParseDiagnosticArgs(DiagnosticOptions &Opts, ArgList &Args,
                                 DiagnosticsEngine *Diags,
-                                bool DefaultDiagColor, bool DefaultShowOpt) {
+                                bool DefaultDiagColor) {
   bool Success = true;
 
   Opts.DiagnosticLogFile =
@@ -1570,9 +1539,7 @@ bool clang::ParseDiagnosticArgs(DiagnosticOptions &Opts, ArgList &Args,
   Opts.ShowFixits = !Args.hasArg(OPT_fno_diagnostics_fixit_info);
   Opts.ShowLocation = !Args.hasArg(OPT_fno_show_source_location);
   Opts.AbsolutePath = Args.hasArg(OPT_fdiagnostics_absolute_paths);
-  Opts.ShowOptionNames =
-      Args.hasFlag(OPT_fdiagnostics_show_option,
-                   OPT_fno_diagnostics_show_option, DefaultShowOpt);
+  Opts.ShowOptionNames = !Args.hasArg(OPT_fno_diagnostics_show_option);
 
   llvm::sys::Process::UseANSIEscapeCodes(Args.hasArg(OPT_fansi_escape_codes));
 
@@ -1680,7 +1647,8 @@ bool clang::ParseDiagnosticArgs(DiagnosticOptions &Opts, ArgList &Args,
       Diags->Report(diag::warn_ignoring_ftabstop_value)
       << Opts.TabStop << DiagnosticOptions::DefaultTabStop;
   }
-  Opts.MessageLength = getLastArgIntValue(Args, OPT_fmessage_length, 0, Diags);
+  Opts.MessageLength =
+      getLastArgIntValue(Args, OPT_fmessage_length_EQ, 0, Diags);
   addDiagnosticArgs(Args, OPT_W_Group, OPT_W_value_Group, Opts.Warnings);
   addDiagnosticArgs(Args, OPT_R_Group, OPT_R_value_Group, Opts.Remarks);
 
@@ -1786,25 +1754,26 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
       StringRef ArgStr =
           Args.hasArg(OPT_interface_stub_version_EQ)
               ? Args.getLastArgValue(OPT_interface_stub_version_EQ)
-              : "experimental-ifs-v1";
+              : "experimental-ifs-v2";
       if (ArgStr == "experimental-yaml-elf-v1" ||
+          ArgStr == "experimental-ifs-v1" ||
           ArgStr == "experimental-tapi-elf-v1") {
         std::string ErrorMessage =
             "Invalid interface stub format: " + ArgStr.str() +
             " is deprecated.";
         Diags.Report(diag::err_drv_invalid_value)
             << "Must specify a valid interface stub format type, ie: "
-               "-interface-stub-version=experimental-ifs-v1"
+               "-interface-stub-version=experimental-ifs-v2"
             << ErrorMessage;
-      } else if (ArgStr != "experimental-ifs-v1") {
+      } else if (!ArgStr.startswith("experimental-ifs-")) {
         std::string ErrorMessage =
             "Invalid interface stub format: " + ArgStr.str() + ".";
         Diags.Report(diag::err_drv_invalid_value)
             << "Must specify a valid interface stub format type, ie: "
-               "-interface-stub-version=experimental-ifs-v1"
+               "-interface-stub-version=experimental-ifs-v2"
             << ErrorMessage;
       } else {
-        Opts.ProgramAction = frontend::GenerateInterfaceIfsExpV1;
+        Opts.ProgramAction = frontend::GenerateInterfaceStubs;
       }
       break;
     }
@@ -2926,7 +2895,7 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
   if (Args.hasArg(OPT_fconcepts_ts))
     Diags.Report(diag::warn_fe_concepts_ts_flag);
   Opts.RecoveryAST =
-      Args.hasFlag(OPT_frecovery_ast, OPT_fno_recovery_ast, Opts.CPlusPlus);
+      Args.hasFlag(OPT_frecovery_ast, OPT_fno_recovery_ast, false);
   Opts.HeinousExtensions = Args.hasArg(OPT_fheinous_gnu_extensions);
   Opts.AccessControl = !Args.hasArg(OPT_fno_access_control);
   Opts.ElideConstructors = !Args.hasArg(OPT_fno_elide_constructors);
@@ -3364,6 +3333,40 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
   Opts.BuildingPCHWithObjectFile = Args.hasArg(OPT_building_pch_with_obj);
 
   Opts.MaxTokens = getLastArgIntValue(Args, OPT_fmax_tokens_EQ, 0, Diags);
+
+  if (Arg *A = Args.getLastArg(OPT_msign_return_address_EQ)) {
+    StringRef SignScope = A->getValue();
+
+    if (SignScope.equals_lower("none"))
+      Opts.setSignReturnAddressScope(
+          LangOptions::SignReturnAddressScopeKind::None);
+    else if (SignScope.equals_lower("all"))
+      Opts.setSignReturnAddressScope(
+          LangOptions::SignReturnAddressScopeKind::All);
+    else if (SignScope.equals_lower("non-leaf"))
+      Opts.setSignReturnAddressScope(
+          LangOptions::SignReturnAddressScopeKind::NonLeaf);
+    else
+      Diags.Report(diag::err_drv_invalid_value)
+          << A->getAsString(Args) << SignScope;
+
+    if (Arg *A = Args.getLastArg(OPT_msign_return_address_key_EQ)) {
+      StringRef SignKey = A->getValue();
+      if (!SignScope.empty() && !SignKey.empty()) {
+        if (SignKey.equals_lower("a_key"))
+          Opts.setSignReturnAddressKey(
+              LangOptions::SignReturnAddressKeyKind::AKey);
+        else if (SignKey.equals_lower("b_key"))
+          Opts.setSignReturnAddressKey(
+              LangOptions::SignReturnAddressKeyKind::BKey);
+        else
+          Diags.Report(diag::err_drv_invalid_value)
+              << A->getAsString(Args) << SignKey;
+      }
+    }
+  }
+
+  Opts.BranchTargetEnforcement = Args.hasArg(OPT_mbranch_target_enforce);
 }
 
 static bool isStrictlyPreprocessorAction(frontend::ActionKind Action) {
@@ -3384,7 +3387,7 @@ static bool isStrictlyPreprocessorAction(frontend::ActionKind Action) {
   case frontend::GenerateModuleInterface:
   case frontend::GenerateHeaderModule:
   case frontend::GeneratePCH:
-  case frontend::GenerateInterfaceIfsExpV1:
+  case frontend::GenerateInterfaceStubs:
   case frontend::ParseSyntaxOnly:
   case frontend::ModuleFileInfo:
   case frontend::VerifyPCH:
@@ -3612,9 +3615,8 @@ bool CompilerInvocation::CreateFromArgs(CompilerInvocation &Res,
     Diags.Report(diag::err_fe_dependency_file_requires_MT);
     Success = false;
   }
-  Success &=
-      ParseDiagnosticArgs(Res.getDiagnosticOpts(), Args, &Diags,
-                          false /*DefaultDiagColor*/, false /*DefaultShowOpt*/);
+  Success &= ParseDiagnosticArgs(Res.getDiagnosticOpts(), Args, &Diags,
+                                 /*DefaultDiagColor=*/false);
   ParseCommentArgs(LangOpts.CommentOpts, Args);
   ParseFileSystemArgs(Res.getFileSystemOpts(), Args);
   // FIXME: We shouldn't have to pass the DashX option around here
diff --git a/clang/lib/Frontend/FrontendAction.cpp b/clang/lib/Frontend/FrontendAction.cpp
index 1dc85d967ca01..0155238dd0a82 100644
--- a/clang/lib/Frontend/FrontendAction.cpp
+++ b/clang/lib/Frontend/FrontendAction.cpp
@@ -1081,6 +1081,9 @@ void WrapperFrontendAction::ExecuteAction() {
 void WrapperFrontendAction::EndSourceFileAction() {
   WrappedAction->EndSourceFileAction();
 }
+bool WrapperFrontendAction::shouldEraseOutputFiles() {
+  return WrappedAction->shouldEraseOutputFiles();
+}
 
 bool WrapperFrontendAction::usesPreprocessorOnly() const {
   return WrappedAction->usesPreprocessorOnly();
diff --git a/clang/lib/Frontend/InterfaceStubFunctionsConsumer.cpp b/clang/lib/Frontend/InterfaceStubFunctionsConsumer.cpp
index 2b7f0f8f9b663..b7c1e693413b7 100644
--- a/clang/lib/Frontend/InterfaceStubFunctionsConsumer.cpp
+++ b/clang/lib/Frontend/InterfaceStubFunctionsConsumer.cpp
@@ -290,7 +290,7 @@ class InterfaceStubFunctionsConsumer : public ASTConsumer {
                              const ASTContext &context, StringRef Format,
                              raw_ostream &OS) -> void {
       OS << "--- !" << Format << "\n";
-      OS << "IfsVersion: 1.0\n";
+      OS << "IfsVersion: 2.0\n";
       OS << "Triple: " << T.str() << "\n";
       OS << "ObjectFileFormat: "
          << "ELF"
@@ -299,11 +299,11 @@ class InterfaceStubFunctionsConsumer : public ASTConsumer {
       for (const auto &E : Symbols) {
         const MangledSymbol &Symbol = E.second;
         for (auto Name : Symbol.Names) {
-          OS << "  \""
+          OS << "  - { Name: \""
              << (Symbol.ParentName.empty() || Instance.getLangOpts().CPlusPlus
                      ? ""
                      : (Symbol.ParentName + "."))
-             << Name << "\" : { Type: ";
+             << Name << "\", Type: ";
           switch (Symbol.Type) {
           default:
             llvm_unreachable(
@@ -330,15 +330,15 @@ class InterfaceStubFunctionsConsumer : public ASTConsumer {
       OS.flush();
     };
 
-    assert(Format == "experimental-ifs-v1" && "Unexpected IFS Format.");
+    assert(Format == "experimental-ifs-v2" && "Unexpected IFS Format.");
     writeIfsV1(Instance.getTarget().getTriple(), Symbols, context, Format, *OS);
   }
 };
 } // namespace
 
 std::unique_ptr<ASTConsumer>
-GenerateInterfaceIfsExpV1Action::CreateASTConsumer(CompilerInstance &CI,
-                                                   StringRef InFile) {
+GenerateInterfaceStubsAction::CreateASTConsumer(CompilerInstance &CI,
+                                                StringRef InFile) {
   return std::make_unique<InterfaceStubFunctionsConsumer>(
-      CI, InFile, "experimental-ifs-v1");
+      CI, InFile, "experimental-ifs-v2");
 }
diff --git a/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp b/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
index 70735ab2d7b2e..7c59ae42d2a28 100644
--- a/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
+++ b/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
@@ -23,6 +23,7 @@
 #include "clang/Frontend/Utils.h"
 #include "clang/FrontendTool/Utils.h"
 #include "clang/Rewrite/Frontend/FrontendActions.h"
+#include "clang/StaticAnalyzer/Frontend/AnalyzerHelpFlags.h"
 #include "clang/StaticAnalyzer/Frontend/FrontendActions.h"
 #include "llvm/Option/OptTable.h"
 #include "llvm/Option/Option.h"
@@ -64,8 +65,8 @@ CreateFrontendBaseAction(CompilerInstance &CI) {
   case GenerateHeaderModule:
     return std::make_unique<GenerateHeaderModuleAction>();
   case GeneratePCH:            return std::make_unique<GeneratePCHAction>();
-  case GenerateInterfaceIfsExpV1:
-    return std::make_unique<GenerateInterfaceIfsExpV1Action>();
+  case GenerateInterfaceStubs:
+    return std::make_unique<GenerateInterfaceStubsAction>();
   case InitOnly:               return std::make_unique<InitOnlyAction>();
   case ParseSyntaxOnly:        return std::make_unique<SyntaxOnlyAction>();
   case ModuleFileInfo:         return std::make_unique<DumpModuleInfoAction>();
@@ -243,35 +244,24 @@ bool ExecuteCompilerInvocation(CompilerInstance *Clang) {
   // These should happen AFTER plugins have been loaded!
 
   AnalyzerOptions &AnOpts = *Clang->getAnalyzerOpts();
+
   // Honor -analyzer-checker-help and -analyzer-checker-help-hidden.
   if (AnOpts.ShowCheckerHelp || AnOpts.ShowCheckerHelpAlpha ||
       AnOpts.ShowCheckerHelpDeveloper) {
-    ento::printCheckerHelp(llvm::outs(),
-                           Clang->getFrontendOpts().Plugins,
-                           AnOpts,
-                           Clang->getDiagnostics(),
-                           Clang->getLangOpts());
+    ento::printCheckerHelp(llvm::outs(), *Clang);
     return true;
   }
 
   // Honor -analyzer-checker-option-help.
   if (AnOpts.ShowCheckerOptionList || AnOpts.ShowCheckerOptionAlphaList ||
       AnOpts.ShowCheckerOptionDeveloperList) {
-    ento::printCheckerConfigList(llvm::outs(),
-                                 Clang->getFrontendOpts().Plugins,
-                                 *Clang->getAnalyzerOpts(),
-                                 Clang->getDiagnostics(),
-                                 Clang->getLangOpts());
+    ento::printCheckerConfigList(llvm::outs(), *Clang);
     return true;
   }
 
   // Honor -analyzer-list-enabled-checkers.
   if (AnOpts.ShowEnabledCheckerList) {
-    ento::printEnabledCheckerList(llvm::outs(),
-                                  Clang->getFrontendOpts().Plugins,
-                                  AnOpts,
-                                  Clang->getDiagnostics(),
-                                  Clang->getLangOpts());
+    ento::printEnabledCheckerList(llvm::outs(), *Clang);
     return true;
   }
 
diff --git a/clang/lib/Headers/CMakeLists.txt b/clang/lib/Headers/CMakeLists.txt
index 28d43cb7ed35a..6851957600e05 100644
--- a/clang/lib/Headers/CMakeLists.txt
+++ b/clang/lib/Headers/CMakeLists.txt
@@ -37,6 +37,7 @@ set(files
   bmi2intrin.h
   bmiintrin.h
   __clang_cuda_builtin_vars.h
+  __clang_cuda_math.h
   __clang_cuda_cmath.h
   __clang_cuda_complex_builtins.h
   __clang_cuda_device_functions.h
@@ -87,6 +88,7 @@ set(files
   ptwriteintrin.h
   rdseedintrin.h
   rtmintrin.h
+  serializeintrin.h
   sgxintrin.h
   s390intrin.h
   shaintrin.h
@@ -109,6 +111,7 @@ set(files
   vecintrin.h
   vpclmulqdqintrin.h
   waitpkgintrin.h
+  wasm_simd128.h
   wbnoinvdintrin.h
   wmmintrin.h
   __wmmintrin_aes.h
diff --git a/clang/lib/Headers/__clang_cuda_device_functions.h b/clang/lib/Headers/__clang_cuda_device_functions.h
index 50ad674f94830..d15f6b61d6ef2 100644
--- a/clang/lib/Headers/__clang_cuda_device_functions.h
+++ b/clang/lib/Headers/__clang_cuda_device_functions.h
@@ -26,26 +26,6 @@
 #define __DEVICE__ static __device__ __forceinline__
 #endif
 
-// libdevice provides fast low precision and slow full-recision implementations
-// for some functions. Which one gets selected depends on
-// __CLANG_CUDA_APPROX_TRANSCENDENTALS__ which gets defined by clang if
-// -ffast-math or -fcuda-approx-transcendentals are in effect.
-#pragma push_macro("__FAST_OR_SLOW")
-#if defined(__CLANG_CUDA_APPROX_TRANSCENDENTALS__)
-#define __FAST_OR_SLOW(fast, slow) fast
-#else
-#define __FAST_OR_SLOW(fast, slow) slow
-#endif
-
-// For C++ 17 we need to include noexcept attribute to be compatible
-// with the header-defined version. This may be removed once
-// variant is supported.
-#if defined(_OPENMP) && defined(__cplusplus) && __cplusplus >= 201703L
-#define __NOEXCEPT noexcept
-#else
-#define __NOEXCEPT
-#endif
-
 __DEVICE__ int __all(int __a) { return __nvvm_vote_all(__a); }
 __DEVICE__ int __any(int __a) { return __nvvm_vote_any(__a); }
 __DEVICE__ unsigned int __ballot(int __a) { return __nvvm_vote_ballot(__a); }
@@ -359,10 +339,10 @@ __DEVICE__ int __iAtomicAdd(int *__p, int __v) {
   return __nvvm_atom_add_gen_i(__p, __v);
 }
 __DEVICE__ int __iAtomicAdd_block(int *__p, int __v) {
-  __nvvm_atom_cta_add_gen_i(__p, __v);
+  return __nvvm_atom_cta_add_gen_i(__p, __v);
 }
 __DEVICE__ int __iAtomicAdd_system(int *__p, int __v) {
-  __nvvm_atom_sys_add_gen_i(__p, __v);
+  return __nvvm_atom_sys_add_gen_i(__p, __v);
 }
 __DEVICE__ int __iAtomicAnd(int *__p, int __v) {
   return __nvvm_atom_and_gen_i(__p, __v);
@@ -1483,149 +1463,14 @@ __DEVICE__ unsigned int __vsubus4(unsigned int __a, unsigned int __b) {
   return r;
 }
 #endif // CUDA_VERSION >= 9020
-__DEVICE__ int abs(int __a) __NOEXCEPT { return __nv_abs(__a); }
-__DEVICE__ double fabs(double __a) __NOEXCEPT { return __nv_fabs(__a); }
-__DEVICE__ double acos(double __a) { return __nv_acos(__a); }
-__DEVICE__ float acosf(float __a) { return __nv_acosf(__a); }
-__DEVICE__ double acosh(double __a) { return __nv_acosh(__a); }
-__DEVICE__ float acoshf(float __a) { return __nv_acoshf(__a); }
-__DEVICE__ double asin(double __a) { return __nv_asin(__a); }
-__DEVICE__ float asinf(float __a) { return __nv_asinf(__a); }
-__DEVICE__ double asinh(double __a) { return __nv_asinh(__a); }
-__DEVICE__ float asinhf(float __a) { return __nv_asinhf(__a); }
-__DEVICE__ double atan(double __a) { return __nv_atan(__a); }
-__DEVICE__ double atan2(double __a, double __b) { return __nv_atan2(__a, __b); }
-__DEVICE__ float atan2f(float __a, float __b) { return __nv_atan2f(__a, __b); }
-__DEVICE__ float atanf(float __a) { return __nv_atanf(__a); }
-__DEVICE__ double atanh(double __a) { return __nv_atanh(__a); }
-__DEVICE__ float atanhf(float __a) { return __nv_atanhf(__a); }
-__DEVICE__ double cbrt(double __a) { return __nv_cbrt(__a); }
-__DEVICE__ float cbrtf(float __a) { return __nv_cbrtf(__a); }
-__DEVICE__ double ceil(double __a) { return __nv_ceil(__a); }
-__DEVICE__ float ceilf(float __a) { return __nv_ceilf(__a); }
+
+// For OpenMP we require the user to include <time.h> as we need to know what
+// clock_t is on the system.
 #ifndef _OPENMP
-__DEVICE__ int clock() { return __nvvm_read_ptx_sreg_clock(); }
-__DEVICE__ long long clock64() { return __nvvm_read_ptx_sreg_clock64(); }
-#endif
-__DEVICE__ double copysign(double __a, double __b) {
-  return __nv_copysign(__a, __b);
-}
-__DEVICE__ float copysignf(float __a, float __b) {
-  return __nv_copysignf(__a, __b);
-}
-__DEVICE__ double cos(double __a) { return __nv_cos(__a); }
-__DEVICE__ float cosf(float __a) {
-  return __FAST_OR_SLOW(__nv_fast_cosf, __nv_cosf)(__a);
-}
-__DEVICE__ double cosh(double __a) { return __nv_cosh(__a); }
-__DEVICE__ float coshf(float __a) { return __nv_coshf(__a); }
-__DEVICE__ double cospi(double __a) { return __nv_cospi(__a); }
-__DEVICE__ float cospif(float __a) { return __nv_cospif(__a); }
-__DEVICE__ double cyl_bessel_i0(double __a) { return __nv_cyl_bessel_i0(__a); }
-__DEVICE__ float cyl_bessel_i0f(float __a) { return __nv_cyl_bessel_i0f(__a); }
-__DEVICE__ double cyl_bessel_i1(double __a) { return __nv_cyl_bessel_i1(__a); }
-__DEVICE__ float cyl_bessel_i1f(float __a) { return __nv_cyl_bessel_i1f(__a); }
-__DEVICE__ double erf(double __a) { return __nv_erf(__a); }
-__DEVICE__ double erfc(double __a) { return __nv_erfc(__a); }
-__DEVICE__ float erfcf(float __a) { return __nv_erfcf(__a); }
-__DEVICE__ double erfcinv(double __a) { return __nv_erfcinv(__a); }
-__DEVICE__ float erfcinvf(float __a) { return __nv_erfcinvf(__a); }
-__DEVICE__ double erfcx(double __a) { return __nv_erfcx(__a); }
-__DEVICE__ float erfcxf(float __a) { return __nv_erfcxf(__a); }
-__DEVICE__ float erff(float __a) { return __nv_erff(__a); }
-__DEVICE__ double erfinv(double __a) { return __nv_erfinv(__a); }
-__DEVICE__ float erfinvf(float __a) { return __nv_erfinvf(__a); }
-__DEVICE__ double exp(double __a) { return __nv_exp(__a); }
-__DEVICE__ double exp10(double __a) { return __nv_exp10(__a); }
-__DEVICE__ float exp10f(float __a) { return __nv_exp10f(__a); }
-__DEVICE__ double exp2(double __a) { return __nv_exp2(__a); }
-__DEVICE__ float exp2f(float __a) { return __nv_exp2f(__a); }
-__DEVICE__ float expf(float __a) { return __nv_expf(__a); }
-__DEVICE__ double expm1(double __a) { return __nv_expm1(__a); }
-__DEVICE__ float expm1f(float __a) { return __nv_expm1f(__a); }
-__DEVICE__ float fabsf(float __a) { return __nv_fabsf(__a); }
-__DEVICE__ double fdim(double __a, double __b) { return __nv_fdim(__a, __b); }
-__DEVICE__ float fdimf(float __a, float __b) { return __nv_fdimf(__a, __b); }
-__DEVICE__ double fdivide(double __a, double __b) { return __a / __b; }
-__DEVICE__ float fdividef(float __a, float __b) {
-#if __FAST_MATH__ && !__CUDA_PREC_DIV
-  return __nv_fast_fdividef(__a, __b);
-#else
-  return __a / __b;
+__DEVICE__ /* clock_t= */ int clock() { return __nvvm_read_ptx_sreg_clock(); }
 #endif
-}
-__DEVICE__ double floor(double __f) { return __nv_floor(__f); }
-__DEVICE__ float floorf(float __f) { return __nv_floorf(__f); }
-__DEVICE__ double fma(double __a, double __b, double __c) {
-  return __nv_fma(__a, __b, __c);
-}
-__DEVICE__ float fmaf(float __a, float __b, float __c) {
-  return __nv_fmaf(__a, __b, __c);
-}
-__DEVICE__ double fmax(double __a, double __b) { return __nv_fmax(__a, __b); }
-__DEVICE__ float fmaxf(float __a, float __b) { return __nv_fmaxf(__a, __b); }
-__DEVICE__ double fmin(double __a, double __b) { return __nv_fmin(__a, __b); }
-__DEVICE__ float fminf(float __a, float __b) { return __nv_fminf(__a, __b); }
-__DEVICE__ double fmod(double __a, double __b) { return __nv_fmod(__a, __b); }
-__DEVICE__ float fmodf(float __a, float __b) { return __nv_fmodf(__a, __b); }
-__DEVICE__ double frexp(double __a, int *__b) { return __nv_frexp(__a, __b); }
-__DEVICE__ float frexpf(float __a, int *__b) { return __nv_frexpf(__a, __b); }
-__DEVICE__ double hypot(double __a, double __b) { return __nv_hypot(__a, __b); }
-__DEVICE__ float hypotf(float __a, float __b) { return __nv_hypotf(__a, __b); }
-__DEVICE__ int ilogb(double __a) { return __nv_ilogb(__a); }
-__DEVICE__ int ilogbf(float __a) { return __nv_ilogbf(__a); }
-__DEVICE__ double j0(double __a) { return __nv_j0(__a); }
-__DEVICE__ float j0f(float __a) { return __nv_j0f(__a); }
-__DEVICE__ double j1(double __a) { return __nv_j1(__a); }
-__DEVICE__ float j1f(float __a) { return __nv_j1f(__a); }
-__DEVICE__ double jn(int __n, double __a) { return __nv_jn(__n, __a); }
-__DEVICE__ float jnf(int __n, float __a) { return __nv_jnf(__n, __a); }
-#if defined(__LP64__) || defined(_WIN64)
-__DEVICE__ long labs(long __a) __NOEXCEPT { return __nv_llabs(__a); };
-#else
-__DEVICE__ long labs(long __a) __NOEXCEPT { return __nv_abs(__a); };
-#endif
-__DEVICE__ double ldexp(double __a, int __b) { return __nv_ldexp(__a, __b); }
-__DEVICE__ float ldexpf(float __a, int __b) { return __nv_ldexpf(__a, __b); }
-__DEVICE__ double lgamma(double __a) { return __nv_lgamma(__a); }
-__DEVICE__ float lgammaf(float __a) { return __nv_lgammaf(__a); }
-__DEVICE__ long long llabs(long long __a) __NOEXCEPT { return __nv_llabs(__a); }
-__DEVICE__ long long llmax(long long __a, long long __b) {
-  return __nv_llmax(__a, __b);
-}
-__DEVICE__ long long llmin(long long __a, long long __b) {
-  return __nv_llmin(__a, __b);
-}
-__DEVICE__ long long llrint(double __a) { return __nv_llrint(__a); }
-__DEVICE__ long long llrintf(float __a) { return __nv_llrintf(__a); }
-__DEVICE__ long long llround(double __a) { return __nv_llround(__a); }
-__DEVICE__ long long llroundf(float __a) { return __nv_llroundf(__a); }
-__DEVICE__ double log(double __a) { return __nv_log(__a); }
-__DEVICE__ double log10(double __a) { return __nv_log10(__a); }
-__DEVICE__ float log10f(float __a) { return __nv_log10f(__a); }
-__DEVICE__ double log1p(double __a) { return __nv_log1p(__a); }
-__DEVICE__ float log1pf(float __a) { return __nv_log1pf(__a); }
-__DEVICE__ double log2(double __a) { return __nv_log2(__a); }
-__DEVICE__ float log2f(float __a) {
-  return __FAST_OR_SLOW(__nv_fast_log2f, __nv_log2f)(__a);
-}
-__DEVICE__ double logb(double __a) { return __nv_logb(__a); }
-__DEVICE__ float logbf(float __a) { return __nv_logbf(__a); }
-__DEVICE__ float logf(float __a) {
-  return __FAST_OR_SLOW(__nv_fast_logf, __nv_logf)(__a);
-}
-#if defined(__LP64__) || defined(_WIN64)
-__DEVICE__ long lrint(double __a) { return llrint(__a); }
-__DEVICE__ long lrintf(float __a) { return __float2ll_rn(__a); }
-__DEVICE__ long lround(double __a) { return llround(__a); }
-__DEVICE__ long lroundf(float __a) { return llroundf(__a); }
-#else
-__DEVICE__ long lrint(double __a) { return (long)rint(__a); }
-__DEVICE__ long lrintf(float __a) { return __float2int_rn(__a); }
-__DEVICE__ long lround(double __a) { return round(__a); }
-__DEVICE__ long lroundf(float __a) { return roundf(__a); }
-#endif
-__DEVICE__ int max(int __a, int __b) { return __nv_max(__a, __b); }
+__DEVICE__ long long clock64() { return __nvvm_read_ptx_sreg_clock64(); }
+
 // These functions shouldn't be declared when including this header
 // for math function resolution purposes.
 #ifndef _OPENMP
@@ -1636,158 +1481,6 @@ __DEVICE__ void *memset(void *__a, int __b, size_t __c) {
   return __builtin_memset(__a, __b, __c);
 }
 #endif
-__DEVICE__ int min(int __a, int __b) { return __nv_min(__a, __b); }
-__DEVICE__ double modf(double __a, double *__b) { return __nv_modf(__a, __b); }
-__DEVICE__ float modff(float __a, float *__b) { return __nv_modff(__a, __b); }
-__DEVICE__ double nearbyint(double __a) { return __nv_nearbyint(__a); }
-__DEVICE__ float nearbyintf(float __a) { return __nv_nearbyintf(__a); }
-__DEVICE__ double nextafter(double __a, double __b) {
-  return __nv_nextafter(__a, __b);
-}
-__DEVICE__ float nextafterf(float __a, float __b) {
-  return __nv_nextafterf(__a, __b);
-}
-__DEVICE__ double norm(int __dim, const double *__t) {
-  return __nv_norm(__dim, __t);
-}
-__DEVICE__ double norm3d(double __a, double __b, double __c) {
-  return __nv_norm3d(__a, __b, __c);
-}
-__DEVICE__ float norm3df(float __a, float __b, float __c) {
-  return __nv_norm3df(__a, __b, __c);
-}
-__DEVICE__ double norm4d(double __a, double __b, double __c, double __d) {
-  return __nv_norm4d(__a, __b, __c, __d);
-}
-__DEVICE__ float norm4df(float __a, float __b, float __c, float __d) {
-  return __nv_norm4df(__a, __b, __c, __d);
-}
-__DEVICE__ double normcdf(double __a) { return __nv_normcdf(__a); }
-__DEVICE__ float normcdff(float __a) { return __nv_normcdff(__a); }
-__DEVICE__ double normcdfinv(double __a) { return __nv_normcdfinv(__a); }
-__DEVICE__ float normcdfinvf(float __a) { return __nv_normcdfinvf(__a); }
-__DEVICE__ float normf(int __dim, const float *__t) {
-  return __nv_normf(__dim, __t);
-}
-__DEVICE__ double pow(double __a, double __b) { return __nv_pow(__a, __b); }
-__DEVICE__ float powf(float __a, float __b) { return __nv_powf(__a, __b); }
-__DEVICE__ double powi(double __a, int __b) { return __nv_powi(__a, __b); }
-__DEVICE__ float powif(float __a, int __b) { return __nv_powif(__a, __b); }
-__DEVICE__ double rcbrt(double __a) { return __nv_rcbrt(__a); }
-__DEVICE__ float rcbrtf(float __a) { return __nv_rcbrtf(__a); }
-__DEVICE__ double remainder(double __a, double __b) {
-  return __nv_remainder(__a, __b);
-}
-__DEVICE__ float remainderf(float __a, float __b) {
-  return __nv_remainderf(__a, __b);
-}
-__DEVICE__ double remquo(double __a, double __b, int *__c) {
-  return __nv_remquo(__a, __b, __c);
-}
-__DEVICE__ float remquof(float __a, float __b, int *__c) {
-  return __nv_remquof(__a, __b, __c);
-}
-__DEVICE__ double rhypot(double __a, double __b) {
-  return __nv_rhypot(__a, __b);
-}
-__DEVICE__ float rhypotf(float __a, float __b) {
-  return __nv_rhypotf(__a, __b);
-}
-__DEVICE__ double rint(double __a) { return __nv_rint(__a); }
-__DEVICE__ float rintf(float __a) { return __nv_rintf(__a); }
-__DEVICE__ double rnorm(int __a, const double *__b) {
-  return __nv_rnorm(__a, __b);
-}
-__DEVICE__ double rnorm3d(double __a, double __b, double __c) {
-  return __nv_rnorm3d(__a, __b, __c);
-}
-__DEVICE__ float rnorm3df(float __a, float __b, float __c) {
-  return __nv_rnorm3df(__a, __b, __c);
-}
-__DEVICE__ double rnorm4d(double __a, double __b, double __c, double __d) {
-  return __nv_rnorm4d(__a, __b, __c, __d);
-}
-__DEVICE__ float rnorm4df(float __a, float __b, float __c, float __d) {
-  return __nv_rnorm4df(__a, __b, __c, __d);
-}
-__DEVICE__ float rnormf(int __dim, const float *__t) {
-  return __nv_rnormf(__dim, __t);
-}
-__DEVICE__ double round(double __a) { return __nv_round(__a); }
-__DEVICE__ float roundf(float __a) { return __nv_roundf(__a); }
-__DEVICE__ double rsqrt(double __a) { return __nv_rsqrt(__a); }
-__DEVICE__ float rsqrtf(float __a) { return __nv_rsqrtf(__a); }
-__DEVICE__ double scalbn(double __a, int __b) { return __nv_scalbn(__a, __b); }
-__DEVICE__ float scalbnf(float __a, int __b) { return __nv_scalbnf(__a, __b); }
-// TODO: remove once variant is supported
-#ifndef _OPENMP
-__DEVICE__ double scalbln(double __a, long __b) {
-  if (__b > INT_MAX)
-    return __a > 0 ? HUGE_VAL : -HUGE_VAL;
-  if (__b < INT_MIN)
-    return __a > 0 ? 0.0 : -0.0;
-  return scalbn(__a, (int)__b);
-}
-__DEVICE__ float scalblnf(float __a, long __b) {
-  if (__b > INT_MAX)
-    return __a > 0 ? HUGE_VALF : -HUGE_VALF;
-  if (__b < INT_MIN)
-    return __a > 0 ? 0.f : -0.f;
-  return scalbnf(__a, (int)__b);
-}
-#endif
-__DEVICE__ double sin(double __a) { return __nv_sin(__a); }
-__DEVICE__ void sincos(double __a, double *__s, double *__c) {
-  return __nv_sincos(__a, __s, __c);
-}
-__DEVICE__ void sincosf(float __a, float *__s, float *__c) {
-  return __FAST_OR_SLOW(__nv_fast_sincosf, __nv_sincosf)(__a, __s, __c);
-}
-__DEVICE__ void sincospi(double __a, double *__s, double *__c) {
-  return __nv_sincospi(__a, __s, __c);
-}
-__DEVICE__ void sincospif(float __a, float *__s, float *__c) {
-  return __nv_sincospif(__a, __s, __c);
-}
-__DEVICE__ float sinf(float __a) {
-  return __FAST_OR_SLOW(__nv_fast_sinf, __nv_sinf)(__a);
-}
-__DEVICE__ double sinh(double __a) { return __nv_sinh(__a); }
-__DEVICE__ float sinhf(float __a) { return __nv_sinhf(__a); }
-__DEVICE__ double sinpi(double __a) { return __nv_sinpi(__a); }
-__DEVICE__ float sinpif(float __a) { return __nv_sinpif(__a); }
-__DEVICE__ double sqrt(double __a) { return __nv_sqrt(__a); }
-__DEVICE__ float sqrtf(float __a) { return __nv_sqrtf(__a); }
-__DEVICE__ double tan(double __a) { return __nv_tan(__a); }
-__DEVICE__ float tanf(float __a) { return __nv_tanf(__a); }
-__DEVICE__ double tanh(double __a) { return __nv_tanh(__a); }
-__DEVICE__ float tanhf(float __a) { return __nv_tanhf(__a); }
-__DEVICE__ double tgamma(double __a) { return __nv_tgamma(__a); }
-__DEVICE__ float tgammaf(float __a) { return __nv_tgammaf(__a); }
-__DEVICE__ double trunc(double __a) { return __nv_trunc(__a); }
-__DEVICE__ float truncf(float __a) { return __nv_truncf(__a); }
-__DEVICE__ unsigned long long ullmax(unsigned long long __a,
-                                     unsigned long long __b) {
-  return __nv_ullmax(__a, __b);
-}
-__DEVICE__ unsigned long long ullmin(unsigned long long __a,
-                                     unsigned long long __b) {
-  return __nv_ullmin(__a, __b);
-}
-__DEVICE__ unsigned int umax(unsigned int __a, unsigned int __b) {
-  return __nv_umax(__a, __b);
-}
-__DEVICE__ unsigned int umin(unsigned int __a, unsigned int __b) {
-  return __nv_umin(__a, __b);
-}
-__DEVICE__ double y0(double __a) { return __nv_y0(__a); }
-__DEVICE__ float y0f(float __a) { return __nv_y0f(__a); }
-__DEVICE__ double y1(double __a) { return __nv_y1(__a); }
-__DEVICE__ float y1f(float __a) { return __nv_y1f(__a); }
-__DEVICE__ double yn(int __a, double __b) { return __nv_yn(__a, __b); }
-__DEVICE__ float ynf(int __a, float __b) { return __nv_ynf(__a, __b); }
 
-#undef __NOEXCEPT
 #pragma pop_macro("__DEVICE__")
-#pragma pop_macro("__FAST_OR_SLOW")
 #endif // __CLANG_CUDA_DEVICE_FUNCTIONS_H__
diff --git a/clang/lib/Headers/__clang_cuda_math.h b/clang/lib/Headers/__clang_cuda_math.h
new file mode 100644
index 0000000000000..7956135bfad59
--- /dev/null
+++ b/clang/lib/Headers/__clang_cuda_math.h
@@ -0,0 +1,345 @@
+/*===---- __clang_cuda_math.h - Device-side CUDA math support --------------===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===-----------------------------------------------------------------------===
+ */
+#ifndef __CLANG_CUDA_MATH_H__
+#define __CLANG_CUDA_MATH_H__
+#ifndef __CUDA__
+#error "This file is for CUDA compilation only."
+#endif
+
+#ifndef _OPENMP
+#if CUDA_VERSION < 9000
+#error This file is intended to be used with CUDA-9+ only.
+#endif
+#endif
+
+// __DEVICE__ is a helper macro with common set of attributes for the wrappers
+// we implement in this file. We need static in order to avoid emitting unused
+// functions and __forceinline__ helps inlining these wrappers at -O1.
+#pragma push_macro("__DEVICE__")
+#ifdef _OPENMP
+#define __DEVICE__ static __inline__ __attribute__((always_inline))
+#else
+#define __DEVICE__ static __device__ __forceinline__
+#endif
+
+// libdevice provides fast low precision and slow full-recision implementations
+// for some functions. Which one gets selected depends on
+// __CLANG_CUDA_APPROX_TRANSCENDENTALS__ which gets defined by clang if
+// -ffast-math or -fcuda-approx-transcendentals are in effect.
+#pragma push_macro("__FAST_OR_SLOW")
+#if defined(__CLANG_CUDA_APPROX_TRANSCENDENTALS__)
+#define __FAST_OR_SLOW(fast, slow) fast
+#else
+#define __FAST_OR_SLOW(fast, slow) slow
+#endif
+
+// For C++ 17 we need to include noexcept attribute to be compatible
+// with the header-defined version. This may be removed once
+// variant is supported.
+#if defined(_OPENMP) && defined(__cplusplus) && __cplusplus >= 201703L
+#define __NOEXCEPT noexcept
+#else
+#define __NOEXCEPT
+#endif
+
+__DEVICE__ int abs(int __a) __NOEXCEPT { return __nv_abs(__a); }
+__DEVICE__ double fabs(double __a) __NOEXCEPT { return __nv_fabs(__a); }
+__DEVICE__ double acos(double __a) { return __nv_acos(__a); }
+__DEVICE__ float acosf(float __a) { return __nv_acosf(__a); }
+__DEVICE__ double acosh(double __a) { return __nv_acosh(__a); }
+__DEVICE__ float acoshf(float __a) { return __nv_acoshf(__a); }
+__DEVICE__ double asin(double __a) { return __nv_asin(__a); }
+__DEVICE__ float asinf(float __a) { return __nv_asinf(__a); }
+__DEVICE__ double asinh(double __a) { return __nv_asinh(__a); }
+__DEVICE__ float asinhf(float __a) { return __nv_asinhf(__a); }
+__DEVICE__ double atan(double __a) { return __nv_atan(__a); }
+__DEVICE__ double atan2(double __a, double __b) { return __nv_atan2(__a, __b); }
+__DEVICE__ float atan2f(float __a, float __b) { return __nv_atan2f(__a, __b); }
+__DEVICE__ float atanf(float __a) { return __nv_atanf(__a); }
+__DEVICE__ double atanh(double __a) { return __nv_atanh(__a); }
+__DEVICE__ float atanhf(float __a) { return __nv_atanhf(__a); }
+__DEVICE__ double cbrt(double __a) { return __nv_cbrt(__a); }
+__DEVICE__ float cbrtf(float __a) { return __nv_cbrtf(__a); }
+__DEVICE__ double ceil(double __a) { return __nv_ceil(__a); }
+__DEVICE__ float ceilf(float __a) { return __nv_ceilf(__a); }
+__DEVICE__ double copysign(double __a, double __b) {
+  return __nv_copysign(__a, __b);
+}
+__DEVICE__ float copysignf(float __a, float __b) {
+  return __nv_copysignf(__a, __b);
+}
+__DEVICE__ double cos(double __a) { return __nv_cos(__a); }
+__DEVICE__ float cosf(float __a) {
+  return __FAST_OR_SLOW(__nv_fast_cosf, __nv_cosf)(__a);
+}
+__DEVICE__ double cosh(double __a) { return __nv_cosh(__a); }
+__DEVICE__ float coshf(float __a) { return __nv_coshf(__a); }
+__DEVICE__ double cospi(double __a) { return __nv_cospi(__a); }
+__DEVICE__ float cospif(float __a) { return __nv_cospif(__a); }
+__DEVICE__ double cyl_bessel_i0(double __a) { return __nv_cyl_bessel_i0(__a); }
+__DEVICE__ float cyl_bessel_i0f(float __a) { return __nv_cyl_bessel_i0f(__a); }
+__DEVICE__ double cyl_bessel_i1(double __a) { return __nv_cyl_bessel_i1(__a); }
+__DEVICE__ float cyl_bessel_i1f(float __a) { return __nv_cyl_bessel_i1f(__a); }
+__DEVICE__ double erf(double __a) { return __nv_erf(__a); }
+__DEVICE__ double erfc(double __a) { return __nv_erfc(__a); }
+__DEVICE__ float erfcf(float __a) { return __nv_erfcf(__a); }
+__DEVICE__ double erfcinv(double __a) { return __nv_erfcinv(__a); }
+__DEVICE__ float erfcinvf(float __a) { return __nv_erfcinvf(__a); }
+__DEVICE__ double erfcx(double __a) { return __nv_erfcx(__a); }
+__DEVICE__ float erfcxf(float __a) { return __nv_erfcxf(__a); }
+__DEVICE__ float erff(float __a) { return __nv_erff(__a); }
+__DEVICE__ double erfinv(double __a) { return __nv_erfinv(__a); }
+__DEVICE__ float erfinvf(float __a) { return __nv_erfinvf(__a); }
+__DEVICE__ double exp(double __a) { return __nv_exp(__a); }
+__DEVICE__ double exp10(double __a) { return __nv_exp10(__a); }
+__DEVICE__ float exp10f(float __a) { return __nv_exp10f(__a); }
+__DEVICE__ double exp2(double __a) { return __nv_exp2(__a); }
+__DEVICE__ float exp2f(float __a) { return __nv_exp2f(__a); }
+__DEVICE__ float expf(float __a) { return __nv_expf(__a); }
+__DEVICE__ double expm1(double __a) { return __nv_expm1(__a); }
+__DEVICE__ float expm1f(float __a) { return __nv_expm1f(__a); }
+__DEVICE__ float fabsf(float __a) __NOEXCEPT { return __nv_fabsf(__a); }
+__DEVICE__ double fdim(double __a, double __b) { return __nv_fdim(__a, __b); }
+__DEVICE__ float fdimf(float __a, float __b) { return __nv_fdimf(__a, __b); }
+__DEVICE__ double fdivide(double __a, double __b) { return __a / __b; }
+__DEVICE__ float fdividef(float __a, float __b) {
+#if __FAST_MATH__ && !__CUDA_PREC_DIV
+  return __nv_fast_fdividef(__a, __b);
+#else
+  return __a / __b;
+#endif
+}
+__DEVICE__ double floor(double __f) { return __nv_floor(__f); }
+__DEVICE__ float floorf(float __f) { return __nv_floorf(__f); }
+__DEVICE__ double fma(double __a, double __b, double __c) {
+  return __nv_fma(__a, __b, __c);
+}
+__DEVICE__ float fmaf(float __a, float __b, float __c) {
+  return __nv_fmaf(__a, __b, __c);
+}
+__DEVICE__ double fmax(double __a, double __b) { return __nv_fmax(__a, __b); }
+__DEVICE__ float fmaxf(float __a, float __b) { return __nv_fmaxf(__a, __b); }
+__DEVICE__ double fmin(double __a, double __b) { return __nv_fmin(__a, __b); }
+__DEVICE__ float fminf(float __a, float __b) { return __nv_fminf(__a, __b); }
+__DEVICE__ double fmod(double __a, double __b) { return __nv_fmod(__a, __b); }
+__DEVICE__ float fmodf(float __a, float __b) { return __nv_fmodf(__a, __b); }
+__DEVICE__ double frexp(double __a, int *__b) { return __nv_frexp(__a, __b); }
+__DEVICE__ float frexpf(float __a, int *__b) { return __nv_frexpf(__a, __b); }
+__DEVICE__ double hypot(double __a, double __b) { return __nv_hypot(__a, __b); }
+__DEVICE__ float hypotf(float __a, float __b) { return __nv_hypotf(__a, __b); }
+__DEVICE__ int ilogb(double __a) { return __nv_ilogb(__a); }
+__DEVICE__ int ilogbf(float __a) { return __nv_ilogbf(__a); }
+__DEVICE__ double j0(double __a) { return __nv_j0(__a); }
+__DEVICE__ float j0f(float __a) { return __nv_j0f(__a); }
+__DEVICE__ double j1(double __a) { return __nv_j1(__a); }
+__DEVICE__ float j1f(float __a) { return __nv_j1f(__a); }
+__DEVICE__ double jn(int __n, double __a) { return __nv_jn(__n, __a); }
+__DEVICE__ float jnf(int __n, float __a) { return __nv_jnf(__n, __a); }
+#if defined(__LP64__) || defined(_WIN64)
+__DEVICE__ long labs(long __a) __NOEXCEPT { return __nv_llabs(__a); };
+#else
+__DEVICE__ long labs(long __a) __NOEXCEPT { return __nv_abs(__a); };
+#endif
+__DEVICE__ double ldexp(double __a, int __b) { return __nv_ldexp(__a, __b); }
+__DEVICE__ float ldexpf(float __a, int __b) { return __nv_ldexpf(__a, __b); }
+__DEVICE__ double lgamma(double __a) { return __nv_lgamma(__a); }
+__DEVICE__ float lgammaf(float __a) { return __nv_lgammaf(__a); }
+__DEVICE__ long long llabs(long long __a) __NOEXCEPT { return __nv_llabs(__a); }
+__DEVICE__ long long llmax(long long __a, long long __b) {
+  return __nv_llmax(__a, __b);
+}
+__DEVICE__ long long llmin(long long __a, long long __b) {
+  return __nv_llmin(__a, __b);
+}
+__DEVICE__ long long llrint(double __a) { return __nv_llrint(__a); }
+__DEVICE__ long long llrintf(float __a) { return __nv_llrintf(__a); }
+__DEVICE__ long long llround(double __a) { return __nv_llround(__a); }
+__DEVICE__ long long llroundf(float __a) { return __nv_llroundf(__a); }
+__DEVICE__ double log(double __a) { return __nv_log(__a); }
+__DEVICE__ double log10(double __a) { return __nv_log10(__a); }
+__DEVICE__ float log10f(float __a) { return __nv_log10f(__a); }
+__DEVICE__ double log1p(double __a) { return __nv_log1p(__a); }
+__DEVICE__ float log1pf(float __a) { return __nv_log1pf(__a); }
+__DEVICE__ double log2(double __a) { return __nv_log2(__a); }
+__DEVICE__ float log2f(float __a) {
+  return __FAST_OR_SLOW(__nv_fast_log2f, __nv_log2f)(__a);
+}
+__DEVICE__ double logb(double __a) { return __nv_logb(__a); }
+__DEVICE__ float logbf(float __a) { return __nv_logbf(__a); }
+__DEVICE__ float logf(float __a) {
+  return __FAST_OR_SLOW(__nv_fast_logf, __nv_logf)(__a);
+}
+#if defined(__LP64__) || defined(_WIN64)
+__DEVICE__ long lrint(double __a) { return llrint(__a); }
+__DEVICE__ long lrintf(float __a) { return __float2ll_rn(__a); }
+__DEVICE__ long lround(double __a) { return llround(__a); }
+__DEVICE__ long lroundf(float __a) { return llroundf(__a); }
+#else
+__DEVICE__ long lrint(double __a) { return (long)rint(__a); }
+__DEVICE__ long lrintf(float __a) { return __float2int_rn(__a); }
+__DEVICE__ long lround(double __a) { return round(__a); }
+__DEVICE__ long lroundf(float __a) { return roundf(__a); }
+#endif
+__DEVICE__ int max(int __a, int __b) { return __nv_max(__a, __b); }
+__DEVICE__ int min(int __a, int __b) { return __nv_min(__a, __b); }
+__DEVICE__ double modf(double __a, double *__b) { return __nv_modf(__a, __b); }
+__DEVICE__ float modff(float __a, float *__b) { return __nv_modff(__a, __b); }
+__DEVICE__ double nearbyint(double __a) { return __nv_nearbyint(__a); }
+__DEVICE__ float nearbyintf(float __a) { return __nv_nearbyintf(__a); }
+__DEVICE__ double nextafter(double __a, double __b) {
+  return __nv_nextafter(__a, __b);
+}
+__DEVICE__ float nextafterf(float __a, float __b) {
+  return __nv_nextafterf(__a, __b);
+}
+__DEVICE__ double norm(int __dim, const double *__t) {
+  return __nv_norm(__dim, __t);
+}
+__DEVICE__ double norm3d(double __a, double __b, double __c) {
+  return __nv_norm3d(__a, __b, __c);
+}
+__DEVICE__ float norm3df(float __a, float __b, float __c) {
+  return __nv_norm3df(__a, __b, __c);
+}
+__DEVICE__ double norm4d(double __a, double __b, double __c, double __d) {
+  return __nv_norm4d(__a, __b, __c, __d);
+}
+__DEVICE__ float norm4df(float __a, float __b, float __c, float __d) {
+  return __nv_norm4df(__a, __b, __c, __d);
+}
+__DEVICE__ double normcdf(double __a) { return __nv_normcdf(__a); }
+__DEVICE__ float normcdff(float __a) { return __nv_normcdff(__a); }
+__DEVICE__ double normcdfinv(double __a) { return __nv_normcdfinv(__a); }
+__DEVICE__ float normcdfinvf(float __a) { return __nv_normcdfinvf(__a); }
+__DEVICE__ float normf(int __dim, const float *__t) {
+  return __nv_normf(__dim, __t);
+}
+__DEVICE__ double pow(double __a, double __b) { return __nv_pow(__a, __b); }
+__DEVICE__ float powf(float __a, float __b) { return __nv_powf(__a, __b); }
+__DEVICE__ double powi(double __a, int __b) { return __nv_powi(__a, __b); }
+__DEVICE__ float powif(float __a, int __b) { return __nv_powif(__a, __b); }
+__DEVICE__ double rcbrt(double __a) { return __nv_rcbrt(__a); }
+__DEVICE__ float rcbrtf(float __a) { return __nv_rcbrtf(__a); }
+__DEVICE__ double remainder(double __a, double __b) {
+  return __nv_remainder(__a, __b);
+}
+__DEVICE__ float remainderf(float __a, float __b) {
+  return __nv_remainderf(__a, __b);
+}
+__DEVICE__ double remquo(double __a, double __b, int *__c) {
+  return __nv_remquo(__a, __b, __c);
+}
+__DEVICE__ float remquof(float __a, float __b, int *__c) {
+  return __nv_remquof(__a, __b, __c);
+}
+__DEVICE__ double rhypot(double __a, double __b) {
+  return __nv_rhypot(__a, __b);
+}
+__DEVICE__ float rhypotf(float __a, float __b) {
+  return __nv_rhypotf(__a, __b);
+}
+__DEVICE__ double rint(double __a) { return __nv_rint(__a); }
+__DEVICE__ float rintf(float __a) { return __nv_rintf(__a); }
+__DEVICE__ double rnorm(int __a, const double *__b) {
+  return __nv_rnorm(__a, __b);
+}
+__DEVICE__ double rnorm3d(double __a, double __b, double __c) {
+  return __nv_rnorm3d(__a, __b, __c);
+}
+__DEVICE__ float rnorm3df(float __a, float __b, float __c) {
+  return __nv_rnorm3df(__a, __b, __c);
+}
+__DEVICE__ double rnorm4d(double __a, double __b, double __c, double __d) {
+  return __nv_rnorm4d(__a, __b, __c, __d);
+}
+__DEVICE__ float rnorm4df(float __a, float __b, float __c, float __d) {
+  return __nv_rnorm4df(__a, __b, __c, __d);
+}
+__DEVICE__ float rnormf(int __dim, const float *__t) {
+  return __nv_rnormf(__dim, __t);
+}
+__DEVICE__ double round(double __a) { return __nv_round(__a); }
+__DEVICE__ float roundf(float __a) { return __nv_roundf(__a); }
+__DEVICE__ double rsqrt(double __a) { return __nv_rsqrt(__a); }
+__DEVICE__ float rsqrtf(float __a) { return __nv_rsqrtf(__a); }
+__DEVICE__ double scalbn(double __a, int __b) { return __nv_scalbn(__a, __b); }
+__DEVICE__ float scalbnf(float __a, int __b) { return __nv_scalbnf(__a, __b); }
+// TODO: remove once variant is supported
+#ifndef _OPENMP
+__DEVICE__ double scalbln(double __a, long __b) {
+  if (__b > INT_MAX)
+    return __a > 0 ? HUGE_VAL : -HUGE_VAL;
+  if (__b < INT_MIN)
+    return __a > 0 ? 0.0 : -0.0;
+  return scalbn(__a, (int)__b);
+}
+__DEVICE__ float scalblnf(float __a, long __b) {
+  if (__b > INT_MAX)
+    return __a > 0 ? HUGE_VALF : -HUGE_VALF;
+  if (__b < INT_MIN)
+    return __a > 0 ? 0.f : -0.f;
+  return scalbnf(__a, (int)__b);
+}
+#endif
+__DEVICE__ double sin(double __a) { return __nv_sin(__a); }
+__DEVICE__ void sincos(double __a, double *__s, double *__c) {
+  return __nv_sincos(__a, __s, __c);
+}
+__DEVICE__ void sincosf(float __a, float *__s, float *__c) {
+  return __FAST_OR_SLOW(__nv_fast_sincosf, __nv_sincosf)(__a, __s, __c);
+}
+__DEVICE__ void sincospi(double __a, double *__s, double *__c) {
+  return __nv_sincospi(__a, __s, __c);
+}
+__DEVICE__ void sincospif(float __a, float *__s, float *__c) {
+  return __nv_sincospif(__a, __s, __c);
+}
+__DEVICE__ float sinf(float __a) {
+  return __FAST_OR_SLOW(__nv_fast_sinf, __nv_sinf)(__a);
+}
+__DEVICE__ double sinh(double __a) { return __nv_sinh(__a); }
+__DEVICE__ float sinhf(float __a) { return __nv_sinhf(__a); }
+__DEVICE__ double sinpi(double __a) { return __nv_sinpi(__a); }
+__DEVICE__ float sinpif(float __a) { return __nv_sinpif(__a); }
+__DEVICE__ double sqrt(double __a) { return __nv_sqrt(__a); }
+__DEVICE__ float sqrtf(float __a) { return __nv_sqrtf(__a); }
+__DEVICE__ double tan(double __a) { return __nv_tan(__a); }
+__DEVICE__ float tanf(float __a) { return __nv_tanf(__a); }
+__DEVICE__ double tanh(double __a) { return __nv_tanh(__a); }
+__DEVICE__ float tanhf(float __a) { return __nv_tanhf(__a); }
+__DEVICE__ double tgamma(double __a) { return __nv_tgamma(__a); }
+__DEVICE__ float tgammaf(float __a) { return __nv_tgammaf(__a); }
+__DEVICE__ double trunc(double __a) { return __nv_trunc(__a); }
+__DEVICE__ float truncf(float __a) { return __nv_truncf(__a); }
+__DEVICE__ unsigned long long ullmax(unsigned long long __a,
+                                     unsigned long long __b) {
+  return __nv_ullmax(__a, __b);
+}
+__DEVICE__ unsigned long long ullmin(unsigned long long __a,
+                                     unsigned long long __b) {
+  return __nv_ullmin(__a, __b);
+}
+__DEVICE__ unsigned int umax(unsigned int __a, unsigned int __b) {
+  return __nv_umax(__a, __b);
+}
+__DEVICE__ unsigned int umin(unsigned int __a, unsigned int __b) {
+  return __nv_umin(__a, __b);
+}
+__DEVICE__ double y0(double __a) { return __nv_y0(__a); }
+__DEVICE__ float y0f(float __a) { return __nv_y0f(__a); }
+__DEVICE__ double y1(double __a) { return __nv_y1(__a); }
+__DEVICE__ float y1f(float __a) { return __nv_y1f(__a); }
+__DEVICE__ double yn(int __a, double __b) { return __nv_yn(__a, __b); }
+__DEVICE__ float ynf(int __a, float __b) { return __nv_ynf(__a, __b); }
+
+#pragma pop_macro("__DEVICE__")
+#pragma pop_macro("__FAST_OR_SLOW")
+#undef __NOEXCEPT
+
+#endif // __CLANG_CUDA_DEVICE_FUNCTIONS_H__
diff --git a/clang/lib/Headers/__clang_cuda_runtime_wrapper.h b/clang/lib/Headers/__clang_cuda_runtime_wrapper.h
index e91de3c81dbdc..63404c9bdeb5c 100644
--- a/clang/lib/Headers/__clang_cuda_runtime_wrapper.h
+++ b/clang/lib/Headers/__clang_cuda_runtime_wrapper.h
@@ -83,13 +83,15 @@
 #if CUDA_VERSION < 9000
 #define __CUDABE__
 #else
+#define __CUDACC__
 #define __CUDA_LIBDEVICE__
 #endif
 // Disables definitions of device-side runtime support stubs in
 // cuda_device_runtime_api.h
+#include "host_defines.h"
+#undef __CUDACC__
 #include "driver_types.h"
 #include "host_config.h"
-#include "host_defines.h"
 
 // Temporarily replace "nv_weak" with weak, so __attribute__((nv_weak)) in
 // cuda_device_runtime_api.h ends up being __attribute__((weak)) which is the
@@ -141,11 +143,12 @@ inline __host__ double __signbitd(double x) {
 // to provide our own.
 #include <__clang_cuda_libdevice_declares.h>
 
-// Wrappers for many device-side standard library functions became compiler
-// builtins in CUDA-9 and have been removed from the CUDA headers. Clang now
-// provides its own implementation of the wrappers.
+// Wrappers for many device-side standard library functions, incl. math
+// functions, became compiler builtins in CUDA-9 and have been removed from the
+// CUDA headers. Clang now provides its own implementation of the wrappers.
 #if CUDA_VERSION >= 9000
 #include <__clang_cuda_device_functions.h>
+#include <__clang_cuda_math.h>
 #endif
 
 // __THROW is redefined to be empty by device_functions_decls.h in CUDA. Clang's
diff --git a/clang/lib/Headers/cpuid.h b/clang/lib/Headers/cpuid.h
index 4ddd64847c32e..b06c37fa08ccf 100644
--- a/clang/lib/Headers/cpuid.h
+++ b/clang/lib/Headers/cpuid.h
@@ -182,6 +182,7 @@
 /* Features in %edx for leaf 7 sub-leaf 0 */
 #define bit_AVX5124VNNIW  0x00000004
 #define bit_AVX5124FMAPS  0x00000008
+#define bit_SERIALIZE     0x00004000
 #define bit_PCONFIG       0x00040000
 #define bit_IBT           0x00100000
 
diff --git a/clang/lib/Headers/immintrin.h b/clang/lib/Headers/immintrin.h
index edf8c42ec4914..164b1f40478dc 100644
--- a/clang/lib/Headers/immintrin.h
+++ b/clang/lib/Headers/immintrin.h
@@ -434,6 +434,10 @@ _storebe_i64(void * __P, long long __D) {
 #include <enqcmdintrin.h>
 #endif
 
+#if !defined(_MSC_VER) || __has_feature(modules) || defined(__SERIALIZE__)
+#include <serializeintrin.h>
+#endif
+
 #if defined(_MSC_VER) && __has_extension(gnu_asm)
 /* Define the default attributes for these intrinsics */
 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__))
diff --git a/clang/lib/Headers/openmp_wrappers/__clang_openmp_math_declares.h b/clang/lib/Headers/openmp_wrappers/__clang_openmp_math_declares.h
index a422c98bf97d9..dd97faca6932c 100644
--- a/clang/lib/Headers/openmp_wrappers/__clang_openmp_math_declares.h
+++ b/clang/lib/Headers/openmp_wrappers/__clang_openmp_math_declares.h
@@ -26,6 +26,7 @@
 #include <__clang_cuda_libdevice_declares.h>
 /// Provide definitions for these functions.
 #include <__clang_cuda_device_functions.h>
+#include <__clang_cuda_math.h>
 
 #undef __CUDA__
 
diff --git a/clang/lib/Headers/serializeintrin.h b/clang/lib/Headers/serializeintrin.h
new file mode 100644
index 0000000000000..b774e5a24a0ba
--- /dev/null
+++ b/clang/lib/Headers/serializeintrin.h
@@ -0,0 +1,30 @@
+/*===--------------- serializeintrin.h - serialize intrinsics --------------===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===-----------------------------------------------------------------------===
+ */
+
+#ifndef __IMMINTRIN_H
+#error "Never use <serializeintrin.h> directly; include <immintrin.h> instead."
+#endif
+
+#ifndef __SERIALIZEINTRIN_H
+#define __SERIALIZEINTRIN_H
+
+/// Serialize instruction fetch and execution.
+///
+/// \headerfile <x86intrin.h>
+///
+/// This intrinsic corresponds to the <c> SERIALIZE </c> instruction.
+///
+static __inline__ void
+__attribute__((__always_inline__, __nodebug__, __target__("serialize")))
+_serialize (void)
+{
+  __builtin_ia32_serialize ();
+}
+
+#endif /* __SERIALIZEINTRIN_H */
diff --git a/clang/lib/Headers/wasm_simd128.h b/clang/lib/Headers/wasm_simd128.h
new file mode 100644
index 0000000000000..c2c57cadfdf24
--- /dev/null
+++ b/clang/lib/Headers/wasm_simd128.h
@@ -0,0 +1,1145 @@
+/*===---- wasm_simd128.h - WebAssembly portable SIMD intrinsics ------------===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===-----------------------------------------------------------------------===
+ */
+
+#ifndef __WASM_SIMD128_H
+#define __WASM_SIMD128_H
+
+#include <stdbool.h>
+#include <stdint.h>
+
+// User-facing type
+typedef int32_t v128_t __attribute__((__vector_size__(16), __aligned__(16)));
+
+// Internal types determined by clang builtin definitions
+typedef int32_t __v128_u __attribute__((__vector_size__(16), __aligned__(1)));
+typedef char __i8x16 __attribute__((__vector_size__(16), __aligned__(16)));
+typedef signed char __s8x16
+    __attribute__((__vector_size__(16), __aligned__(16)));
+typedef unsigned char __u8x16
+    __attribute__((__vector_size__(16), __aligned__(16)));
+typedef short __i16x8 __attribute__((__vector_size__(16), __aligned__(16)));
+typedef unsigned short __u16x8
+    __attribute__((__vector_size__(16), __aligned__(16)));
+typedef int __i32x4 __attribute__((__vector_size__(16), __aligned__(16)));
+typedef unsigned int __u32x4
+    __attribute__((__vector_size__(16), __aligned__(16)));
+typedef long long __i64x2 __attribute__((__vector_size__(16), __aligned__(16)));
+typedef unsigned long long __u64x2
+    __attribute__((__vector_size__(16), __aligned__(16)));
+typedef float __f32x4 __attribute__((__vector_size__(16), __aligned__(16)));
+typedef double __f64x2 __attribute__((__vector_size__(16), __aligned__(16)));
+
+#define __DEFAULT_FN_ATTRS                                                     \
+  __attribute__((__always_inline__, __nodebug__, __target__("simd128"),        \
+                 __min_vector_width__(128)))
+
+#define __REQUIRE_CONSTANT(e)                                                  \
+  _Static_assert(__builtin_constant_p(e), "Expected constant")
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_v128_load(const void *__mem) {
+  // UB-free unaligned access copied from xmmintrin.h
+  struct __wasm_v128_load_struct {
+    __v128_u __v;
+  } __attribute__((__packed__, __may_alias__));
+  return ((const struct __wasm_v128_load_struct *)__mem)->__v;
+}
+
+#ifdef __wasm_unimplemented_simd128__
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_v8x16_load_splat(const void *__mem) {
+  struct __wasm_v8x16_load_splat_struct {
+    uint8_t __v;
+  } __attribute__((__packed__, __may_alias__));
+  uint8_t __v = ((const struct __wasm_v8x16_load_splat_struct *)__mem)->__v;
+  return (v128_t)(__u8x16){__v, __v, __v, __v, __v, __v, __v, __v,
+                           __v, __v, __v, __v, __v, __v, __v, __v};
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_v16x8_load_splat(const void *__mem) {
+  struct __wasm_v16x8_load_splat_struct {
+    uint16_t __v;
+  } __attribute__((__packed__, __may_alias__));
+  uint16_t __v = ((const struct __wasm_v16x8_load_splat_struct *)__mem)->__v;
+  return (v128_t)(__u16x8){__v, __v, __v, __v, __v, __v, __v, __v};
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_v32x4_load_splat(const void *__mem) {
+  struct __wasm_v32x4_load_splat_struct {
+    uint32_t __v;
+  } __attribute__((__packed__, __may_alias__));
+  uint32_t __v = ((const struct __wasm_v32x4_load_splat_struct *)__mem)->__v;
+  return (v128_t)(__u32x4){__v, __v, __v, __v};
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_v64x2_load_splat(const void *__mem) {
+  struct __wasm_v64x2_load_splat_struct {
+    uint64_t __v;
+  } __attribute__((__packed__, __may_alias__));
+  uint64_t __v = ((const struct __wasm_v64x2_load_splat_struct *)__mem)->__v;
+  return (v128_t)(__u64x2){__v, __v};
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_i16x8_load_8x8(const void *__mem) {
+  typedef int8_t __i8x8 __attribute__((__vector_size__(8), __aligned__(8)));
+  struct __wasm_i16x8_load_8x8_struct {
+    __i8x8 __v;
+  } __attribute__((__packed__, __may_alias__));
+  __i8x8 __v = ((const struct __wasm_i16x8_load_8x8_struct *)__mem)->__v;
+  return (v128_t) __builtin_convertvector(__v, __i16x8);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_u16x8_load_8x8(const void *__mem) {
+  typedef uint8_t __u8x8 __attribute__((__vector_size__(8), __aligned__(8)));
+  struct __wasm_u16x8_load_8x8_struct {
+    __u8x8 __v;
+  } __attribute__((__packed__, __may_alias__));
+  __u8x8 __v = ((const struct __wasm_u16x8_load_8x8_struct *)__mem)->__v;
+  return (v128_t) __builtin_convertvector(__v, __u16x8);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_i32x4_load_16x4(const void *__mem) {
+  typedef int16_t __i16x4 __attribute__((__vector_size__(8), __aligned__(8)));
+  struct __wasm_i32x4_load_16x4_struct {
+    __i16x4 __v;
+  } __attribute__((__packed__, __may_alias__));
+  __i16x4 __v = ((const struct __wasm_i32x4_load_16x4_struct *)__mem)->__v;
+  return (v128_t) __builtin_convertvector(__v, __i32x4);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_u32x4_load_16x4(const void *__mem) {
+  typedef uint16_t __u16x4 __attribute__((__vector_size__(8), __aligned__(8)));
+  struct __wasm_u32x4_load_16x4_struct {
+    __u16x4 __v;
+  } __attribute__((__packed__, __may_alias__));
+  __u16x4 __v = ((const struct __wasm_u32x4_load_16x4_struct *)__mem)->__v;
+  return (v128_t) __builtin_convertvector(__v, __u32x4);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_i64x2_load_32x2(const void *__mem) {
+  typedef int32_t __i32x2 __attribute__((__vector_size__(8), __aligned__(8)));
+  struct __wasm_i64x2_load_32x2_struct {
+    __i32x2 __v;
+  } __attribute__((__packed__, __may_alias__));
+  __i32x2 __v = ((const struct __wasm_i64x2_load_32x2_struct *)__mem)->__v;
+  return (v128_t) __builtin_convertvector(__v, __i64x2);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_u64x2_load_32x2(const void *__mem) {
+  typedef uint32_t __u32x2 __attribute__((__vector_size__(8), __aligned__(8)));
+  struct __wasm_u64x2_load_32x2_struct {
+    __u32x2 __v;
+  } __attribute__((__packed__, __may_alias__));
+  __u32x2 __v = ((const struct __wasm_u64x2_load_32x2_struct *)__mem)->__v;
+  return (v128_t) __builtin_convertvector(__v, __u64x2);
+}
+
+#endif // __wasm_unimplemented_simd128__
+
+static __inline__ void __DEFAULT_FN_ATTRS wasm_v128_store(void *__mem,
+                                                          v128_t __a) {
+  // UB-free unaligned access copied from xmmintrin.h
+  struct __wasm_v128_store_struct {
+    __v128_u __v;
+  } __attribute__((__packed__, __may_alias__));
+  ((struct __wasm_v128_store_struct *)__mem)->__v = __a;
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_i8x16_make(int8_t __c0, int8_t __c1, int8_t __c2, int8_t __c3, int8_t __c4,
+                int8_t __c5, int8_t __c6, int8_t __c7, int8_t __c8, int8_t __c9,
+                int8_t __c10, int8_t __c11, int8_t __c12, int8_t __c13,
+                int8_t __c14, int8_t __c15) {
+  return (v128_t)(__i8x16){__c0,  __c1,  __c2,  __c3, __c4,  __c5,
+                           __c6,  __c7,  __c8,  __c9, __c10, __c11,
+                           __c12, __c13, __c14, __c15};
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_i16x8_make(int16_t __c0, int16_t __c1, int16_t __c2, int16_t __c3,
+                int16_t __c4, int16_t __c5, int16_t __c6, int16_t __c7) {
+  return (v128_t)(__i16x8){__c0, __c1, __c2, __c3, __c4, __c5, __c6, __c7};
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_make(int32_t __c0,
+                                                            int32_t __c1,
+                                                            int32_t __c2,
+                                                            int32_t __c3) {
+  return (v128_t)(__i32x4){__c0, __c1, __c2, __c3};
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f32x4_make(float __c0,
+                                                            float __c1,
+                                                            float __c2,
+                                                            float __c3) {
+  return (v128_t)(__f32x4){__c0, __c1, __c2, __c3};
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i64x2_make(int64_t __c0,
+                                                            int64_t __c1) {
+  return (v128_t)(__i64x2){__c0, __c1};
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f64x2_make(double __c0,
+                                                            double __c1) {
+  return (v128_t)(__f64x2){__c0, __c1};
+}
+
+#define wasm_i8x16_const(__c0, __c1, __c2, __c3, __c4, __c5, __c6, __c7, __c8, \
+                         __c9, __c10, __c11, __c12, __c13, __c14, __c15)       \
+  __extension__({                                                              \
+    __REQUIRE_CONSTANT(__c0);                                                  \
+    __REQUIRE_CONSTANT(__c1);                                                  \
+    __REQUIRE_CONSTANT(__c2);                                                  \
+    __REQUIRE_CONSTANT(__c3);                                                  \
+    __REQUIRE_CONSTANT(__c4);                                                  \
+    __REQUIRE_CONSTANT(__c5);                                                  \
+    __REQUIRE_CONSTANT(__c6);                                                  \
+    __REQUIRE_CONSTANT(__c7);                                                  \
+    __REQUIRE_CONSTANT(__c8);                                                  \
+    __REQUIRE_CONSTANT(__c9);                                                  \
+    __REQUIRE_CONSTANT(__c10);                                                 \
+    __REQUIRE_CONSTANT(__c11);                                                 \
+    __REQUIRE_CONSTANT(__c12);                                                 \
+    __REQUIRE_CONSTANT(__c13);                                                 \
+    __REQUIRE_CONSTANT(__c14);                                                 \
+    __REQUIRE_CONSTANT(__c15);                                                 \
+    (v128_t)(__i8x16){__c0, __c1, __c2,  __c3,  __c4,  __c5,  __c6,  __c7,     \
+                      __c8, __c9, __c10, __c11, __c12, __c13, __c14, __c15};   \
+  })
+
+#define wasm_i16x8_const(__c0, __c1, __c2, __c3, __c4, __c5, __c6, __c7)       \
+  __extension__({                                                              \
+    __REQUIRE_CONSTANT(__c0);                                                  \
+    __REQUIRE_CONSTANT(__c1);                                                  \
+    __REQUIRE_CONSTANT(__c2);                                                  \
+    __REQUIRE_CONSTANT(__c3);                                                  \
+    __REQUIRE_CONSTANT(__c4);                                                  \
+    __REQUIRE_CONSTANT(__c5);                                                  \
+    __REQUIRE_CONSTANT(__c6);                                                  \
+    __REQUIRE_CONSTANT(__c7);                                                  \
+    (v128_t)(__i16x8){__c0, __c1, __c2, __c3, __c4, __c5, __c6, __c7};         \
+  })
+
+#define wasm_i32x4_const(__c0, __c1, __c2, __c3)                               \
+  __extension__({                                                              \
+    __REQUIRE_CONSTANT(__c0);                                                  \
+    __REQUIRE_CONSTANT(__c1);                                                  \
+    __REQUIRE_CONSTANT(__c2);                                                  \
+    __REQUIRE_CONSTANT(__c3);                                                  \
+    (v128_t)(__i32x4){__c0, __c1, __c2, __c3};                                 \
+  })
+
+#define wasm_f32x4_const(__c0, __c1, __c2, __c3)                               \
+  __extension__({                                                              \
+    __REQUIRE_CONSTANT(__c0);                                                  \
+    __REQUIRE_CONSTANT(__c1);                                                  \
+    __REQUIRE_CONSTANT(__c2);                                                  \
+    __REQUIRE_CONSTANT(__c3);                                                  \
+    (v128_t)(__f32x4){__c0, __c1, __c2, __c3};                                 \
+  })
+
+#define wasm_i64x2_const(__c0, __c1)                                           \
+  __extension__({                                                              \
+    __REQUIRE_CONSTANT(__c0);                                                  \
+    __REQUIRE_CONSTANT(__c1);                                                  \
+    (v128_t)(__i64x2){__c0, __c1};                                             \
+  })
+
+#define wasm_f64x2_const(__c0, __c1)                                           \
+  __extension__({                                                              \
+    __REQUIRE_CONSTANT(__c0);                                                  \
+    __REQUIRE_CONSTANT(__c1);                                                  \
+    (v128_t)(__f64x2){__c0, __c1};                                             \
+  })
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_splat(int8_t __a) {
+  return (v128_t)(__i8x16){__a, __a, __a, __a, __a, __a, __a, __a,
+                           __a, __a, __a, __a, __a, __a, __a, __a};
+}
+
+#define wasm_i8x16_extract_lane(__a, __i)                                      \
+  (__builtin_wasm_extract_lane_s_i8x16((__i8x16)(__a), __i))
+
+#define wasm_u8x16_extract_lane(__a, __i)                                      \
+  (__builtin_wasm_extract_lane_u_i8x16((__i8x16)(__a), __i))
+
+#define wasm_i8x16_replace_lane(__a, __i, __b)                                 \
+  ((v128_t)__builtin_wasm_replace_lane_i8x16((__i8x16)(__a), __i, __b))
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_splat(int16_t __a) {
+  return (v128_t)(__i16x8){__a, __a, __a, __a, __a, __a, __a, __a};
+}
+
+#define wasm_i16x8_extract_lane(__a, __i)                                      \
+  (__builtin_wasm_extract_lane_s_i16x8((__i16x8)(__a), __i))
+
+#define wasm_u16x8_extract_lane(__a, __i)                                      \
+  (__builtin_wasm_extract_lane_u_i16x8((__i16x8)(__a), __i))
+
+#define wasm_i16x8_replace_lane(__a, __i, __b)                                 \
+  ((v128_t)__builtin_wasm_replace_lane_i16x8((__i16x8)(__a), __i, __b))
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_splat(int32_t __a) {
+  return (v128_t)(__i32x4){__a, __a, __a, __a};
+}
+
+#define wasm_i32x4_extract_lane(__a, __i)                                      \
+  (__builtin_wasm_extract_lane_i32x4((__i32x4)(__a), __i))
+
+#define wasm_i32x4_replace_lane(__a, __i, __b)                                 \
+  ((v128_t)__builtin_wasm_replace_lane_i32x4((__i32x4)(__a), __i, __b))
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i64x2_splat(int64_t __a) {
+  return (v128_t)(__i64x2){__a, __a};
+}
+
+#define wasm_i64x2_extract_lane(__a, __i)                                      \
+  (__builtin_wasm_extract_lane_i64x2((__i64x2)(__a), __i))
+
+#define wasm_i64x2_replace_lane(__a, __i, __b)                                 \
+  ((v128_t)__builtin_wasm_replace_lane_i64x2((__i64x2)(__a), __i, __b))
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f32x4_splat(float __a) {
+  return (v128_t)(__f32x4){__a, __a, __a, __a};
+}
+
+#define wasm_f32x4_extract_lane(__a, __i)                                      \
+  (__builtin_wasm_extract_lane_f32x4((__f32x4)(__a), __i))
+
+#define wasm_f32x4_replace_lane(__a, __i, __b)                                 \
+  ((v128_t)__builtin_wasm_replace_lane_f32x4((__f32x4)(__a), __i, __b))
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f64x2_splat(double __a) {
+  return (v128_t)(__f64x2){__a, __a};
+}
+
+#define wasm_f64x2_extract_lane(__a, __i)                                      \
+  (__builtin_wasm_extract_lane_f64x2((__f64x2)(__a), __i))
+
+#define wasm_f64x2_replace_lane(__a, __i, __b)                                 \
+  ((v128_t)__builtin_wasm_replace_lane_f64x2((__f64x2)(__a), __i, __b))
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_eq(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__s8x16)__a == (__s8x16)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_ne(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__s8x16)__a != (__s8x16)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_lt(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__s8x16)__a < (__s8x16)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_lt(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__u8x16)__a < (__u8x16)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_gt(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__s8x16)__a > (__s8x16)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_gt(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__u8x16)__a > (__u8x16)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_le(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__s8x16)__a <= (__s8x16)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_le(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__u8x16)__a <= (__u8x16)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_ge(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__s8x16)__a >= (__s8x16)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_ge(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__u8x16)__a >= (__u8x16)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_eq(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__i16x8)__a == (__i16x8)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_ne(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__u16x8)__a != (__u16x8)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_lt(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__i16x8)__a < (__i16x8)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u16x8_lt(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__u16x8)__a < (__u16x8)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_gt(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__i16x8)__a > (__i16x8)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u16x8_gt(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__u16x8)__a > (__u16x8)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_le(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__i16x8)__a <= (__i16x8)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u16x8_le(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__u16x8)__a <= (__u16x8)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_ge(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__i16x8)__a >= (__i16x8)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u16x8_ge(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__u16x8)__a >= (__u16x8)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_eq(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__i32x4)__a == (__i32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_ne(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__i32x4)__a != (__i32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_lt(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__i32x4)__a < (__i32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u32x4_lt(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__u32x4)__a < (__u32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_gt(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__i32x4)__a > (__i32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u32x4_gt(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__u32x4)__a > (__u32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_le(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__i32x4)__a <= (__i32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u32x4_le(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__u32x4)__a <= (__u32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_ge(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__i32x4)__a >= (__i32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u32x4_ge(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__u32x4)__a >= (__u32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f32x4_eq(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__f32x4)__a == (__f32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f32x4_ne(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__f32x4)__a != (__f32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f32x4_lt(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__f32x4)__a < (__f32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f32x4_gt(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__f32x4)__a > (__f32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f32x4_le(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__f32x4)__a <= (__f32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f32x4_ge(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__f32x4)__a >= (__f32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f64x2_eq(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__f64x2)__a == (__f64x2)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f64x2_ne(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__f64x2)__a != (__f64x2)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f64x2_lt(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__f64x2)__a < (__f64x2)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f64x2_gt(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__f64x2)__a > (__f64x2)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f64x2_le(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__f64x2)__a <= (__f64x2)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f64x2_ge(v128_t __a,
+                                                          v128_t __b) {
+  return (v128_t)((__f64x2)__a >= (__f64x2)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_v128_not(v128_t __a) {
+  return ~__a;
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_v128_and(v128_t __a,
+                                                          v128_t __b) {
+  return __a & __b;
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_v128_or(v128_t __a,
+                                                         v128_t __b) {
+  return __a | __b;
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_v128_xor(v128_t __a,
+                                                          v128_t __b) {
+  return __a ^ __b;
+}
+
+#ifdef __wasm_unimplemented_simd128__
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_v128_andnot(v128_t __a,
+                                                             v128_t __b) {
+  return __a & ~__b;
+}
+
+#endif // __wasm_unimplemented_simd128__
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_v128_bitselect(v128_t __a,
+                                                                v128_t __b,
+                                                                v128_t __mask) {
+  return (v128_t)__builtin_wasm_bitselect((__i32x4)__a, (__i32x4)__b,
+                                          (__i32x4)__mask);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_abs(v128_t __a) {
+  return (v128_t)__builtin_wasm_abs_i8x16((__i8x16)__a);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_neg(v128_t __a) {
+  return (v128_t)(-(__u8x16)__a);
+}
+
+static __inline__ bool __DEFAULT_FN_ATTRS wasm_i8x16_any_true(v128_t __a) {
+  return __builtin_wasm_any_true_i8x16((__i8x16)__a);
+}
+
+static __inline__ bool __DEFAULT_FN_ATTRS wasm_i8x16_all_true(v128_t __a) {
+  return __builtin_wasm_all_true_i8x16((__i8x16)__a);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_shl(v128_t __a,
+                                                           int32_t __b) {
+  return (v128_t)((__i8x16)__a << __b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_shr(v128_t __a,
+                                                           int32_t __b) {
+  return (v128_t)((__s8x16)__a >> __b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_shr(v128_t __a,
+                                                           int32_t __b) {
+  return (v128_t)((__u8x16)__a >> __b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_add(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)((__u8x16)__a + (__u8x16)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_i8x16_add_saturate(v128_t __a, v128_t __b) {
+  return (v128_t)__builtin_wasm_add_saturate_s_i8x16((__i8x16)__a,
+                                                     (__i8x16)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_u8x16_add_saturate(v128_t __a, v128_t __b) {
+  return (v128_t)__builtin_wasm_add_saturate_u_i8x16((__i8x16)__a,
+                                                     (__i8x16)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_sub(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)((__u8x16)__a - (__u8x16)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_i8x16_sub_saturate(v128_t __a, v128_t __b) {
+  return (v128_t)__builtin_wasm_sub_saturate_s_i8x16((__i8x16)__a,
+                                                     (__i8x16)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_u8x16_sub_saturate(v128_t __a, v128_t __b) {
+  return (v128_t)__builtin_wasm_sub_saturate_u_i8x16((__i8x16)__a,
+                                                     (__i8x16)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_mul(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)((__u8x16)__a * (__u8x16)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_min(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)__builtin_wasm_min_s_i8x16((__i8x16)__a, (__i8x16)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_min(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)__builtin_wasm_min_u_i8x16((__i8x16)__a, (__i8x16)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_max(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)__builtin_wasm_max_s_i8x16((__i8x16)__a, (__i8x16)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_max(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)__builtin_wasm_max_u_i8x16((__i8x16)__a, (__i8x16)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_avgr(v128_t __a,
+                                                            v128_t __b) {
+  return (v128_t)__builtin_wasm_avgr_u_i8x16((__i8x16)__a, (__i8x16)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_abs(v128_t __a) {
+  return (v128_t)__builtin_wasm_abs_i16x8((__i16x8)__a);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_neg(v128_t __a) {
+  return (v128_t)(-(__u16x8)__a);
+}
+
+static __inline__ bool __DEFAULT_FN_ATTRS wasm_i16x8_any_true(v128_t __a) {
+  return __builtin_wasm_any_true_i16x8((__i16x8)__a);
+}
+
+static __inline__ bool __DEFAULT_FN_ATTRS wasm_i16x8_all_true(v128_t __a) {
+  return __builtin_wasm_all_true_i16x8((__i16x8)__a);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_shl(v128_t __a,
+                                                           int32_t __b) {
+  return (v128_t)((__i16x8)__a << __b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_shr(v128_t __a,
+                                                           int32_t __b) {
+  return (v128_t)((__i16x8)__a >> __b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u16x8_shr(v128_t __a,
+                                                           int32_t __b) {
+  return (v128_t)((__u16x8)__a >> __b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_add(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)((__u16x8)__a + (__u16x8)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_i16x8_add_saturate(v128_t __a, v128_t __b) {
+  return (v128_t)__builtin_wasm_add_saturate_s_i16x8((__i16x8)__a,
+                                                     (__i16x8)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_u16x8_add_saturate(v128_t __a, v128_t __b) {
+  return (v128_t)__builtin_wasm_add_saturate_u_i16x8((__i16x8)__a,
+                                                     (__i16x8)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_sub(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)((__i16x8)__a - (__i16x8)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_i16x8_sub_saturate(v128_t __a, v128_t __b) {
+  return (v128_t)__builtin_wasm_sub_saturate_s_i16x8((__i16x8)__a,
+                                                     (__i16x8)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_u16x8_sub_saturate(v128_t __a, v128_t __b) {
+  return (v128_t)__builtin_wasm_sub_saturate_u_i16x8((__i16x8)__a,
+                                                     (__i16x8)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_mul(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)((__u16x8)__a * (__u16x8)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_min(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)__builtin_wasm_min_s_i16x8((__i16x8)__a, (__i16x8)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u16x8_min(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)__builtin_wasm_min_u_i16x8((__i16x8)__a, (__i16x8)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_max(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)__builtin_wasm_max_s_i16x8((__i16x8)__a, (__i16x8)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u16x8_max(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)__builtin_wasm_max_u_i16x8((__i16x8)__a, (__i16x8)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u16x8_avgr(v128_t __a,
+                                                            v128_t __b) {
+  return (v128_t)__builtin_wasm_avgr_u_i16x8((__i16x8)__a, (__i16x8)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_abs(v128_t __a) {
+  return (v128_t)__builtin_wasm_abs_i32x4((__i32x4)__a);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_neg(v128_t __a) {
+  return (v128_t)(-(__u32x4)__a);
+}
+
+static __inline__ bool __DEFAULT_FN_ATTRS wasm_i32x4_any_true(v128_t __a) {
+  return __builtin_wasm_any_true_i32x4((__i32x4)__a);
+}
+
+static __inline__ bool __DEFAULT_FN_ATTRS wasm_i32x4_all_true(v128_t __a) {
+  return __builtin_wasm_all_true_i32x4((__i32x4)__a);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_shl(v128_t __a,
+                                                           int32_t __b) {
+  return (v128_t)((__i32x4)__a << __b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_shr(v128_t __a,
+                                                           int32_t __b) {
+  return (v128_t)((__i32x4)__a >> __b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u32x4_shr(v128_t __a,
+                                                           int32_t __b) {
+  return (v128_t)((__u32x4)__a >> __b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_add(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)((__u32x4)__a + (__u32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_sub(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)((__u32x4)__a - (__u32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_mul(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)((__u32x4)__a * (__u32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_min(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)__builtin_wasm_min_s_i32x4((__i32x4)__a, (__i32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u32x4_min(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)__builtin_wasm_min_u_i32x4((__i32x4)__a, (__i32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_max(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)__builtin_wasm_max_s_i32x4((__i32x4)__a, (__i32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u32x4_max(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)__builtin_wasm_max_u_i32x4((__i32x4)__a, (__i32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i64x2_neg(v128_t __a) {
+  return (v128_t)(-(__u64x2)__a);
+}
+
+#ifdef __wasm_unimplemented_simd128__
+
+static __inline__ bool __DEFAULT_FN_ATTRS wasm_i64x2_any_true(v128_t __a) {
+  return __builtin_wasm_any_true_i64x2((__i64x2)__a);
+}
+
+static __inline__ bool __DEFAULT_FN_ATTRS wasm_i64x2_all_true(v128_t __a) {
+  return __builtin_wasm_all_true_i64x2((__i64x2)__a);
+}
+
+#endif // __wasm_unimplemented_simd128__
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i64x2_shl(v128_t __a,
+                                                           int32_t __b) {
+  return (v128_t)((__i64x2)__a << (int64_t)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i64x2_shr(v128_t __a,
+                                                           int32_t __b) {
+  return (v128_t)((__i64x2)__a >> (int64_t)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u64x2_shr(v128_t __a,
+                                                           int32_t __b) {
+  return (v128_t)((__u64x2)__a >> (int64_t)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i64x2_add(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)((__u64x2)__a + (__u64x2)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i64x2_sub(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)((__u64x2)__a - (__u64x2)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f32x4_abs(v128_t __a) {
+  return (v128_t)__builtin_wasm_abs_f32x4((__f32x4)__a);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f32x4_neg(v128_t __a) {
+  return (v128_t)(-(__f32x4)__a);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f32x4_sqrt(v128_t __a) {
+  return (v128_t)__builtin_wasm_sqrt_f32x4((__f32x4)__a);
+}
+
+#ifdef __wasm_unimplemented_simd128__
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f32x4_qfma(v128_t __a,
+                                                            v128_t __b,
+                                                            v128_t __c) {
+  return (v128_t)__builtin_wasm_qfma_f32x4((__f32x4)__a, (__f32x4)__b,
+                                           (__f32x4)__c);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f32x4_qfms(v128_t __a,
+                                                            v128_t __b,
+                                                            v128_t __c) {
+  return (v128_t)__builtin_wasm_qfms_f32x4((__f32x4)__a, (__f32x4)__b,
+                                           (__f32x4)__c);
+}
+
+#endif // __wasm_unimplemented_simd128__
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f32x4_add(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)((__f32x4)__a + (__f32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f32x4_sub(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)((__f32x4)__a - (__f32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f32x4_mul(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)((__f32x4)__a * (__f32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f32x4_div(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)((__f32x4)__a / (__f32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f32x4_min(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)__builtin_wasm_min_f32x4((__f32x4)__a, (__f32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f32x4_max(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)__builtin_wasm_max_f32x4((__f32x4)__a, (__f32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f64x2_abs(v128_t __a) {
+  return (v128_t)__builtin_wasm_abs_f64x2((__f64x2)__a);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f64x2_neg(v128_t __a) {
+  return (v128_t)(-(__f64x2)__a);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f64x2_sqrt(v128_t __a) {
+  return (v128_t)__builtin_wasm_sqrt_f64x2((__f64x2)__a);
+}
+
+#ifdef __wasm_unimplemented_simd128__
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f64x2_qfma(v128_t __a,
+                                                            v128_t __b,
+                                                            v128_t __c) {
+  return (v128_t)__builtin_wasm_qfma_f64x2((__f64x2)__a, (__f64x2)__b,
+                                           (__f64x2)__c);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f64x2_qfms(v128_t __a,
+                                                            v128_t __b,
+                                                            v128_t __c) {
+  return (v128_t)__builtin_wasm_qfms_f64x2((__f64x2)__a, (__f64x2)__b,
+                                           (__f64x2)__c);
+}
+
+#endif // __wasm_unimplemented_simd128__
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f64x2_add(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)((__f64x2)__a + (__f64x2)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f64x2_sub(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)((__f64x2)__a - (__f64x2)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f64x2_mul(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)((__f64x2)__a * (__f64x2)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f64x2_div(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)((__f64x2)__a / (__f64x2)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f64x2_min(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)__builtin_wasm_min_f64x2((__f64x2)__a, (__f64x2)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_f64x2_max(v128_t __a,
+                                                           v128_t __b) {
+  return (v128_t)__builtin_wasm_max_f64x2((__f64x2)__a, (__f64x2)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_i32x4_trunc_saturate_f32x4(v128_t __a) {
+  return (v128_t)__builtin_wasm_trunc_saturate_s_i32x4_f32x4((__f32x4)__a);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_u32x4_trunc_saturate_f32x4(v128_t __a) {
+  return (v128_t)__builtin_wasm_trunc_saturate_u_i32x4_f32x4((__f32x4)__a);
+}
+
+#ifdef __wasm_unimplemented_simd128__
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_i64x2_trunc_saturate_f64x2(v128_t __a) {
+  return (v128_t)__builtin_wasm_trunc_saturate_s_i64x2_f64x2((__f64x2)__a);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_u64x2_trunc_saturate_f64x2(v128_t __a) {
+  return (v128_t)__builtin_wasm_trunc_saturate_s_i64x2_f64x2((__f64x2)__a);
+}
+
+#endif // __wasm_unimplemented_simd128__
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_f32x4_convert_i32x4(v128_t __a) {
+  return (v128_t) __builtin_convertvector((__i32x4)__a, __f32x4);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_f32x4_convert_u32x4(v128_t __a) {
+  return (v128_t) __builtin_convertvector((__u32x4)__a, __f32x4);
+}
+
+#ifdef __wasm_unimplemented_simd128__
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_f64x2_convert_i64x2(v128_t __a) {
+  return (v128_t) __builtin_convertvector((__i64x2)__a, __f64x2);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_f64x2_convert_u64x2(v128_t __a) {
+  return (v128_t) __builtin_convertvector((__u64x2)__a, __f64x2);
+}
+
+#endif // __wasm_unimplemented_simd128__
+
+#define wasm_v8x16_shuffle(__a, __b, __c0, __c1, __c2, __c3, __c4, __c5, __c6, \
+                           __c7, __c8, __c9, __c10, __c11, __c12, __c13,       \
+                           __c14, __c15)                                       \
+  ((v128_t)(__builtin_shufflevector(                                           \
+      (__u8x16)(__a), (__u8x16)(__b), __c0, __c1, __c2, __c3, __c4, __c5,      \
+      __c6, __c7, __c8, __c9, __c10, __c11, __c12, __c13, __c14, __c15)))
+
+#define wasm_v16x8_shuffle(__a, __b, __c0, __c1, __c2, __c3, __c4, __c5, __c6, \
+                           __c7)                                               \
+  ((v128_t)(__builtin_shufflevector((__u16x8)(__a), (__u16x8)(__b), __c0,      \
+                                    __c1, __c2, __c3, __c4, __c5, __c6,        \
+                                    __c7)))
+
+#define wasm_v32x4_shuffle(__a, __b, __c0, __c1, __c2, __c3)                   \
+  ((v128_t)(__builtin_shufflevector((__u32x4)(__a), (__u32x4)(__b), __c0,      \
+                                    __c1, __c2, __c3)))
+
+#define wasm_v64x2_shuffle(__a, __b, __c0, __c1)                               \
+  ((v128_t)(                                                                   \
+      __builtin_shufflevector((__u64x2)(__a), (__u64x2)(__b), __c0, __c1)))
+
+#ifdef __wasm_unimplemented_simd128__
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_v8x16_swizzle(v128_t __a,
+                                                               v128_t __b) {
+  return (v128_t)__builtin_wasm_swizzle_v8x16((__i8x16)__a, (__i8x16)__b);
+}
+
+#endif // __wasm_unimplemented_simd128__
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_i8x16_narrow_i16x8(v128_t __a, v128_t __b) {
+  return (v128_t)__builtin_wasm_narrow_s_i8x16_i16x8((__i16x8)__a,
+                                                     (__i16x8)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_u8x16_narrow_i16x8(v128_t __a, v128_t __b) {
+  return (v128_t)__builtin_wasm_narrow_u_i8x16_i16x8((__i16x8)__a,
+                                                     (__i16x8)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_i16x8_narrow_i32x4(v128_t __a, v128_t __b) {
+  return (v128_t)__builtin_wasm_narrow_s_i16x8_i32x4((__i32x4)__a,
+                                                     (__i32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_u16x8_narrow_i32x4(v128_t __a, v128_t __b) {
+  return (v128_t)__builtin_wasm_narrow_u_i16x8_i32x4((__i32x4)__a,
+                                                     (__i32x4)__b);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_i16x8_widen_low_i8x16(v128_t __a) {
+  return (v128_t)__builtin_wasm_widen_low_s_i16x8_i8x16((__i8x16)__a);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_i16x8_widen_high_i8x16(v128_t __a) {
+  return (v128_t)__builtin_wasm_widen_high_s_i16x8_i8x16((__i8x16)__a);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_i16x8_widen_low_u8x16(v128_t __a) {
+  return (v128_t)__builtin_wasm_widen_low_u_i16x8_i8x16((__i8x16)__a);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_i16x8_widen_high_u8x16(v128_t __a) {
+  return (v128_t)__builtin_wasm_widen_high_u_i16x8_i8x16((__i8x16)__a);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_i32x4_widen_low_i16x8(v128_t __a) {
+  return (v128_t)__builtin_wasm_widen_low_s_i32x4_i16x8((__i16x8)__a);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_i32x4_widen_high_i16x8(v128_t __a) {
+  return (v128_t)__builtin_wasm_widen_high_s_i32x4_i16x8((__i16x8)__a);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_i32x4_widen_low_u16x8(v128_t __a) {
+  return (v128_t)__builtin_wasm_widen_low_u_i32x4_i16x8((__i16x8)__a);
+}
+
+static __inline__ v128_t __DEFAULT_FN_ATTRS
+wasm_i32x4_widen_high_u16x8(v128_t __a) {
+  return (v128_t)__builtin_wasm_widen_high_u_i32x4_i16x8((__i16x8)__a);
+}
+
+// Undefine helper macros
+#undef __DEFAULT_FN_ATTRS
+
+#endif // __WASM_SIMD128_H
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index a51745697b115..f0d2d4332b373 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -29,6 +29,7 @@
 #include "clang/Basic/TokenKinds.h"
 #include "llvm/ADT/None.h"
 #include "llvm/ADT/Optional.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/ADT/StringRef.h"
@@ -2092,7 +2093,8 @@ void Lexer::codeCompleteIncludedFile(const char *PathStart,
                                      bool IsAngled) {
   // Completion only applies to the filename, after the last slash.
   StringRef PartialPath(PathStart, CompletionPoint - PathStart);
-  auto Slash = PartialPath.find_last_of(LangOpts.MSVCCompat ? "/\\" : "/");
+  llvm::StringRef SlashChars = LangOpts.MSVCCompat ? "/\\" : "/";
+  auto Slash = PartialPath.find_last_of(SlashChars);
   StringRef Dir =
       (Slash == StringRef::npos) ? "" : PartialPath.take_front(Slash);
   const char *StartOfFilename =
@@ -2100,7 +2102,8 @@ void Lexer::codeCompleteIncludedFile(const char *PathStart,
   // Code completion filter range is the filename only, up to completion point.
   PP->setCodeCompletionIdentifierInfo(&PP->getIdentifierTable().get(
       StringRef(StartOfFilename, CompletionPoint - StartOfFilename)));
-  // We should replace the characters up to the closing quote, if any.
+  // We should replace the characters up to the closing quote or closest slash,
+  // if any.
   while (CompletionPoint < BufferEnd) {
     char Next = *(CompletionPoint + 1);
     if (Next == 0 || Next == '\r' || Next == '\n')
@@ -2108,7 +2111,10 @@ void Lexer::codeCompleteIncludedFile(const char *PathStart,
     ++CompletionPoint;
     if (Next == (IsAngled ? '>' : '"'))
       break;
+    if (llvm::is_contained(SlashChars, Next))
+      break;
   }
+
   PP->setCodeCompletionTokenRange(
       FileLoc.getLocWithOffset(StartOfFilename - BufferStart),
       FileLoc.getLocWithOffset(CompletionPoint - BufferStart));
diff --git a/clang/lib/Parse/ParseCXXInlineMethods.cpp b/clang/lib/Parse/ParseCXXInlineMethods.cpp
index a75965784168c..06e02ed0e11c6 100644
--- a/clang/lib/Parse/ParseCXXInlineMethods.cpp
+++ b/clang/lib/Parse/ParseCXXInlineMethods.cpp
@@ -1114,17 +1114,14 @@ bool Parser::ConsumeAndStoreInitializer(CachedTokens &Toks,
           break;
         }
 
+        // Put the token stream back and undo any annotations we performed
+        // after the comma. They may reflect a different parse than the one
+        // we will actually perform at the end of the class.
+        PA.RevertAnnotations();
+
         // If what follows could be a declaration, it is a declaration.
-        if (Result != TPResult::False && Result != TPResult::Error) {
-          PA.Revert();
+        if (Result != TPResult::False && Result != TPResult::Error)
           return true;
-        }
-
-        // In the uncommon case that we decide the following tokens are part
-        // of a template argument, revert any annotations we've performed in
-        // those tokens. We're not going to look them up until we've parsed
-        // the rest of the class, and that might add more declarations.
-        PA.RevertAnnotations();
       }
 
       // Keep going. We know we're inside a template argument list now.
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index b7cfe21da95d4..dde48bb64c961 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -3163,9 +3163,19 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
 
       // We are looking for a qualified typename.
       Token Next = NextToken();
-      if (Next.is(tok::annot_template_id) &&
-          static_cast<TemplateIdAnnotation *>(Next.getAnnotationValue())
-            ->Kind == TNK_Type_template) {
+
+      TemplateIdAnnotation *TemplateId = Next.is(tok::annot_template_id)
+                                             ? takeTemplateIdAnnotation(Next)
+                                             : nullptr;
+      if (TemplateId && TemplateId->hasInvalidName()) {
+        // We found something like 'T::U<Args> x', but U is not a template.
+        // Assume it was supposed to be a type.
+        DS.SetTypeSpecError();
+        ConsumeAnnotationToken();
+        break;
+      }
+
+      if (TemplateId && TemplateId->Kind == TNK_Type_template) {
         // We have a qualified template-id, e.g., N::A<int>
 
         // If this would be a valid constructor declaration with template
@@ -3175,7 +3185,6 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
         //
         // To improve diagnostics for this case, parse the declaration as a
         // constructor (and reject the extra template arguments later).
-        TemplateIdAnnotation *TemplateId = takeTemplateIdAnnotation(Next);
         if ((DSContext == DeclSpecContext::DSC_top_level ||
              DSContext == DeclSpecContext::DSC_class) &&
             TemplateId->Name &&
@@ -3196,9 +3205,7 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
         continue;
       }
 
-      if (Next.is(tok::annot_template_id) &&
-          static_cast<TemplateIdAnnotation *>(Next.getAnnotationValue())
-            ->Kind == TNK_Concept_template &&
+      if (TemplateId && TemplateId->Kind == TNK_Concept_template &&
           GetLookAheadToken(2).isOneOf(tok::kw_auto, tok::kw_decltype)) {
         DS.getTypeSpecScope() = SS;
         // This is a qualified placeholder-specifier, e.g., ::C<int> auto ...
@@ -3211,16 +3218,12 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
       if (Next.is(tok::annot_typename)) {
         DS.getTypeSpecScope() = SS;
         ConsumeAnnotationToken(); // The C++ scope.
-        if (Tok.getAnnotationValue()) {
-          ParsedType T = getTypeAnnotation(Tok);
-          isInvalid = DS.SetTypeSpecType(DeclSpec::TST_typename,
-                                         Tok.getAnnotationEndLoc(),
-                                         PrevSpec, DiagID, T, Policy);
-          if (isInvalid)
-            break;
-        }
-        else
-          DS.SetTypeSpecError();
+        TypeResult T = getTypeAnnotation(Tok);
+        isInvalid = DS.SetTypeSpecType(DeclSpec::TST_typename,
+                                       Tok.getAnnotationEndLoc(),
+                                       PrevSpec, DiagID, T, Policy);
+        if (isInvalid)
+          break;
         DS.SetRangeEnd(Tok.getAnnotationEndLoc());
         ConsumeAnnotationToken(); // The typename
       }
@@ -3288,13 +3291,9 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
       if (DS.hasTypeSpecifier() && DS.hasTagDefinition())
         goto DoneWithDeclSpec;
 
-      if (Tok.getAnnotationValue()) {
-        ParsedType T = getTypeAnnotation(Tok);
-        isInvalid = DS.SetTypeSpecType(DeclSpec::TST_typename, Loc, PrevSpec,
-                                       DiagID, T, Policy);
-      } else
-        DS.SetTypeSpecError();
-
+      TypeResult T = getTypeAnnotation(Tok);
+      isInvalid = DS.SetTypeSpecType(DeclSpec::TST_typename, Loc, PrevSpec,
+                                     DiagID, T, Policy);
       if (isInvalid)
         break;
 
@@ -3460,7 +3459,18 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
       // type-name or placeholder-specifier
     case tok::annot_template_id: {
       TemplateIdAnnotation *TemplateId = takeTemplateIdAnnotation(Tok);
+
+      if (TemplateId->hasInvalidName()) {
+        DS.SetTypeSpecError();
+        break;
+      }
+
       if (TemplateId->Kind == TNK_Concept_template) {
+        // If we've already diagnosed that this type-constraint has invalid
+        // arguemnts, drop it and just form 'auto' or 'decltype(auto)'.
+        if (TemplateId->hasInvalidArgs())
+          TemplateId = nullptr;
+
         if (NextToken().is(tok::identifier)) {
           Diag(Loc, diag::err_placeholder_expected_auto_or_decltype_auto)
               << FixItHint::CreateInsertion(NextToken().getLocation(), "auto");
@@ -5198,14 +5208,30 @@ bool Parser::isDeclarationSpecifier(bool DisambiguatingWithExpression) {
 
     // placeholder-type-specifier
   case tok::annot_template_id: {
+    TemplateIdAnnotation *TemplateId = takeTemplateIdAnnotation(Tok);
+    if (TemplateId->hasInvalidName())
+      return true;
+    // FIXME: What about type templates that have only been annotated as
+    // annot_template_id, not as annot_typename?
     return isTypeConstraintAnnotation() &&
-        (NextToken().is(tok::kw_auto) || NextToken().is(tok::kw_decltype));
+           (NextToken().is(tok::kw_auto) || NextToken().is(tok::kw_decltype));
   }
-  case tok::annot_cxxscope:
+
+  case tok::annot_cxxscope: {
+    TemplateIdAnnotation *TemplateId =
+        NextToken().is(tok::annot_template_id)
+            ? takeTemplateIdAnnotation(NextToken())
+            : nullptr;
+    if (TemplateId && TemplateId->hasInvalidName())
+      return true;
+    // FIXME: What about type templates that have only been annotated as
+    // annot_template_id, not as annot_typename?
     if (NextToken().is(tok::identifier) && TryAnnotateTypeConstraint())
       return true;
     return isTypeConstraintAnnotation() &&
         GetLookAheadToken(2).isOneOf(tok::kw_auto, tok::kw_decltype);
+  }
+
   case tok::kw___declspec:
   case tok::kw___cdecl:
   case tok::kw___stdcall:
diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp
index 86a9a8208b2eb..710632379ace9 100644
--- a/clang/lib/Parse/ParseDeclCXX.cpp
+++ b/clang/lib/Parse/ParseDeclCXX.cpp
@@ -1147,19 +1147,14 @@ TypeResult Parser::ParseBaseTypeSpecifier(SourceLocation &BaseLoc,
   // Check whether we have a template-id that names a type.
   if (Tok.is(tok::annot_template_id)) {
     TemplateIdAnnotation *TemplateId = takeTemplateIdAnnotation(Tok);
-    if (TemplateId->Kind == TNK_Type_template ||
-        TemplateId->Kind == TNK_Dependent_template_name ||
-        TemplateId->Kind == TNK_Undeclared_template) {
+    if (TemplateId->mightBeType()) {
       AnnotateTemplateIdTokenAsType(SS, /*IsClassName*/true);
 
       assert(Tok.is(tok::annot_typename) && "template-id -> type failed");
-      ParsedType Type = getTypeAnnotation(Tok);
+      TypeResult Type = getTypeAnnotation(Tok);
       EndLocation = Tok.getAnnotationEndLoc();
       ConsumeAnnotationToken();
-
-      if (Type)
-        return Type;
-      return true;
+      return Type;
     }
 
     // Fall through to produce an error below.
@@ -1176,7 +1171,9 @@ TypeResult Parser::ParseBaseTypeSpecifier(SourceLocation &BaseLoc,
   if (Tok.is(tok::less)) {
     // It looks the user intended to write a template-id here, but the
     // template-name was wrong. Try to fix that.
-    TemplateNameKind TNK = TNK_Type_template;
+    // FIXME: Invoke ParseOptionalCXXScopeSpecifier in a "'template' is neither
+    // required nor permitted" mode, and do this there.
+    TemplateNameKind TNK = TNK_Non_template;
     TemplateTy Template;
     if (!Actions.DiagnoseUnknownTemplateName(*Id, IdLoc, getCurScope(),
                                              &SS, Template, TNK)) {
@@ -1184,14 +1181,6 @@ TypeResult Parser::ParseBaseTypeSpecifier(SourceLocation &BaseLoc,
         << Id;
     }
 
-    if (!Template) {
-      TemplateArgList TemplateArgs;
-      SourceLocation LAngleLoc, RAngleLoc;
-      ParseTemplateIdAfterTemplateName(true, LAngleLoc, TemplateArgs,
-                                       RAngleLoc);
-      return true;
-    }
-
     // Form the template name
     UnqualifiedId TemplateName;
     TemplateName.setIdentifier(Id, IdLoc);
@@ -1200,7 +1189,8 @@ TypeResult Parser::ParseBaseTypeSpecifier(SourceLocation &BaseLoc,
     if (AnnotateTemplateIdToken(Template, TNK, SS, SourceLocation(),
                                 TemplateName))
       return true;
-    if (TNK == TNK_Type_template || TNK == TNK_Dependent_template_name)
+    if (Tok.is(tok::annot_template_id) &&
+        takeTemplateIdAnnotation(Tok)->mightBeType())
       AnnotateTemplateIdTokenAsType(SS, /*IsClassName*/true);
 
     // If we didn't end up with a typename token, there's nothing more we
@@ -1211,7 +1201,7 @@ TypeResult Parser::ParseBaseTypeSpecifier(SourceLocation &BaseLoc,
     // Retrieve the type from the annotation token, consume that token, and
     // return.
     EndLocation = Tok.getAnnotationEndLoc();
-    ParsedType Type = getTypeAnnotation(Tok);
+    TypeResult Type = getTypeAnnotation(Tok);
     ConsumeAnnotationToken();
     return Type;
   }
@@ -1630,9 +1620,9 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
     NameLoc = ConsumeAnnotationToken();
 
     if (TemplateId->Kind == TNK_Undeclared_template) {
-      // Try to resolve the template name to a type template.
-      Actions.ActOnUndeclaredTypeTemplateName(getCurScope(), TemplateId->Template,
-                                              TemplateId->Kind, NameLoc, Name);
+      // Try to resolve the template name to a type template. May update Kind.
+      Actions.ActOnUndeclaredTypeTemplateName(
+          getCurScope(), TemplateId->Template, TemplateId->Kind, NameLoc, Name);
       if (TemplateId->Kind == TNK_Undeclared_template) {
         RecoverFromUndeclaredTemplateName(
             Name, NameLoc,
@@ -1641,10 +1631,9 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
       }
     }
 
-    if (TemplateId && TemplateId->Kind != TNK_Type_template &&
-        TemplateId->Kind != TNK_Dependent_template_name) {
+    if (TemplateId && !TemplateId->mightBeType()) {
       // The template-name in the simple-template-id refers to
-      // something other than a class template. Give an appropriate
+      // something other than a type template. Give an appropriate
       // error message and skip to the ';'.
       SourceRange Range(NameLoc);
       if (SS.isNotEmpty())
@@ -1816,7 +1805,9 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
     // or explicit instantiation.
     ASTTemplateArgsPtr TemplateArgsPtr(TemplateId->getTemplateArgs(),
                                        TemplateId->NumArgs);
-    if (TemplateInfo.Kind == ParsedTemplateInfo::ExplicitInstantiation &&
+    if (TemplateId->isInvalid()) {
+      // Can't build the declaration.
+    } else if (TemplateInfo.Kind == ParsedTemplateInfo::ExplicitInstantiation &&
         TUK == Sema::TUK_Declaration) {
       // This is an explicit instantiation of a class template.
       ProhibitAttributes(attrs);
@@ -3516,7 +3507,7 @@ MemInitResult Parser::ParseMemInitializer(Decl *ConstructorDecl) {
   // : declype(...)
   DeclSpec DS(AttrFactory);
   // : template_name<...>
-  ParsedType TemplateTypeTy;
+  TypeResult TemplateTypeTy;
 
   if (Tok.is(tok::identifier)) {
     // Get the identifier. This may be a member name or a class name,
@@ -3533,15 +3524,11 @@ MemInitResult Parser::ParseMemInitializer(Decl *ConstructorDecl) {
     TemplateIdAnnotation *TemplateId = Tok.is(tok::annot_template_id)
                                            ? takeTemplateIdAnnotation(Tok)
                                            : nullptr;
-    if (TemplateId && (TemplateId->Kind == TNK_Type_template ||
-                       TemplateId->Kind == TNK_Dependent_template_name ||
-                       TemplateId->Kind == TNK_Undeclared_template)) {
+    if (TemplateId && TemplateId->mightBeType()) {
       AnnotateTemplateIdTokenAsType(SS, /*IsClassName*/true);
       assert(Tok.is(tok::annot_typename) && "template-id -> type failed");
       TemplateTypeTy = getTypeAnnotation(Tok);
       ConsumeAnnotationToken();
-      if (!TemplateTypeTy)
-        return true;
     } else {
       Diag(Tok, diag::err_expected_member_or_base_name);
       return true;
@@ -3560,8 +3547,10 @@ MemInitResult Parser::ParseMemInitializer(Decl *ConstructorDecl) {
     SourceLocation EllipsisLoc;
     TryConsumeToken(tok::ellipsis, EllipsisLoc);
 
+    if (TemplateTypeTy.isInvalid())
+      return true;
     return Actions.ActOnMemInitializer(ConstructorDecl, getCurScope(), SS, II,
-                                       TemplateTypeTy, DS, IdLoc,
+                                       TemplateTypeTy.get(), DS, IdLoc,
                                        InitList.get(), EllipsisLoc);
   } else if(Tok.is(tok::l_paren)) {
     BalancedDelimiterTracker T(*this, tok::l_paren);
@@ -3571,8 +3560,10 @@ MemInitResult Parser::ParseMemInitializer(Decl *ConstructorDecl) {
     ExprVector ArgExprs;
     CommaLocsTy CommaLocs;
     auto RunSignatureHelp = [&] {
+      if (TemplateTypeTy.isInvalid())
+        return QualType();
       QualType PreferredType = Actions.ProduceCtorInitMemberSignatureHelp(
-          getCurScope(), ConstructorDecl, SS, TemplateTypeTy, ArgExprs, II,
+          getCurScope(), ConstructorDecl, SS, TemplateTypeTy.get(), ArgExprs, II,
           T.getOpenLocation());
       CalledSignatureHelp = true;
       return PreferredType;
@@ -3593,12 +3584,17 @@ MemInitResult Parser::ParseMemInitializer(Decl *ConstructorDecl) {
     SourceLocation EllipsisLoc;
     TryConsumeToken(tok::ellipsis, EllipsisLoc);
 
+    if (TemplateTypeTy.isInvalid())
+      return true;
     return Actions.ActOnMemInitializer(ConstructorDecl, getCurScope(), SS, II,
-                                       TemplateTypeTy, DS, IdLoc,
+                                       TemplateTypeTy.get(), DS, IdLoc,
                                        T.getOpenLocation(), ArgExprs,
                                        T.getCloseLocation(), EllipsisLoc);
   }
 
+  if (TemplateTypeTy.isInvalid())
+    return true;
+
   if (getLangOpts().CPlusPlus11)
     return Diag(Tok, diag::err_expected_either) << tok::l_paren << tok::l_brace;
   else
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index bcd5679cb43f2..fc07cfe5ad03f 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -1455,7 +1455,7 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
     break;
   case tok::annot_typename:
     if (isStartOfObjCClassMessageMissingOpenBracket()) {
-      ParsedType Type = getTypeAnnotation(Tok);
+      TypeResult Type = getTypeAnnotation(Tok);
 
       // Fake up a Declarator to use with ActOnTypeName.
       DeclSpec DS(AttrFactory);
@@ -2102,8 +2102,14 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
                                             OpKind, SS, TemplateKWLoc, Name,
                                  CurParsedObjCImpl ? CurParsedObjCImpl->Dcl
                                                    : nullptr);
-      if (!LHS.isInvalid() && Tok.is(tok::less))
-        checkPotentialAngleBracket(LHS);
+      if (!LHS.isInvalid()) {
+        if (Tok.is(tok::less))
+          checkPotentialAngleBracket(LHS);
+      } else if (OrigLHS && Name.isValid()) {
+        // Preserve the LHS if the RHS is an invalid member.
+        LHS = Actions.CreateRecoveryExpr(OrigLHS->getBeginLoc(),
+                                         Name.getEndLoc(), {OrigLHS});
+      }
       break;
     }
     case tok::plusplus:    // postfix-expression: postfix-expression '++'
@@ -2642,6 +2648,33 @@ ExprResult Parser::ParseBuiltinPrimaryExpression() {
   return ParsePostfixExpressionSuffix(Res.get());
 }
 
+bool Parser::tryParseOpenMPArrayShapingCastPart() {
+  assert(Tok.is(tok::l_square) && "Expected open bracket");
+  bool ErrorFound = true;
+  TentativeParsingAction TPA(*this);
+  do {
+    if (Tok.isNot(tok::l_square))
+      break;
+    // Consume '['
+    ConsumeBracket();
+    // Skip inner expression.
+    while (!SkipUntil(tok::r_square, tok::annot_pragma_openmp_end,
+                      StopAtSemi | StopBeforeMatch))
+      ;
+    if (Tok.isNot(tok::r_square))
+      break;
+    // Consume ']'
+    ConsumeBracket();
+    // Found ')' - done.
+    if (Tok.is(tok::r_paren)) {
+      ErrorFound = false;
+      break;
+    }
+  } while (Tok.isNot(tok::annot_pragma_openmp_end));
+  TPA.Revert();
+  return !ErrorFound;
+}
+
 /// ParseParenExpression - This parses the unit that starts with a '(' token,
 /// based on what is allowed by ExprType.  The actual thing parsed is returned
 /// in ExprType. If stopIfCastExpr is true, it will only return the parsed type,
@@ -2666,6 +2699,8 @@ ExprResult Parser::ParseBuiltinPrimaryExpression() {
 ///         '(' '...' fold-operator cast-expression ')'
 ///         '(' cast-expression fold-operator '...'
 ///                 fold-operator cast-expression ')'
+/// [OPENMP] Array shaping operation
+///       '(' '[' expression ']' { '[' expression ']' } cast-expression
 /// \endverbatim
 ExprResult
 Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
@@ -2942,6 +2977,38 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
       Result = Actions.ActOnParenListExpr(OpenLoc, Tok.getLocation(),
                                           ArgExprs);
     }
+  } else if (getLangOpts().OpenMP >= 50 && OpenMPDirectiveParsing &&
+             ExprType == CastExpr && Tok.is(tok::l_square) &&
+             tryParseOpenMPArrayShapingCastPart()) {
+    bool ErrorFound = false;
+    SmallVector<Expr *, 4> OMPDimensions;
+    SmallVector<SourceRange, 4> OMPBracketsRanges;
+    do {
+      BalancedDelimiterTracker TS(*this, tok::l_square);
+      TS.consumeOpen();
+      ExprResult NumElements =
+          Actions.CorrectDelayedTyposInExpr(ParseExpression());
+      if (!NumElements.isUsable()) {
+        ErrorFound = true;
+        while (!SkipUntil(tok::r_square, tok::r_paren,
+                          StopAtSemi | StopBeforeMatch))
+          ;
+      }
+      TS.consumeClose();
+      OMPDimensions.push_back(NumElements.get());
+      OMPBracketsRanges.push_back(TS.getRange());
+    } while (Tok.isNot(tok::r_paren));
+    // Match the ')'.
+    T.consumeClose();
+    RParenLoc = T.getCloseLocation();
+    Result = Actions.CorrectDelayedTyposInExpr(ParseAssignmentExpression());
+    if (ErrorFound) {
+      Result = ExprError();
+    } else if (!Result.isInvalid()) {
+      Result = Actions.ActOnOMPArrayShapingExpr(
+          Result.get(), OpenLoc, RParenLoc, OMPDimensions, OMPBracketsRanges);
+    }
+    return Result;
   } else {
     InMessageExpressionRAIIObject InMessage(*this, false);
 
diff --git a/clang/lib/Parse/ParseExprCXX.cpp b/clang/lib/Parse/ParseExprCXX.cpp
index a0b97ea7514dc..c5e895d090a55 100644
--- a/clang/lib/Parse/ParseExprCXX.cpp
+++ b/clang/lib/Parse/ParseExprCXX.cpp
@@ -134,7 +134,7 @@ void Parser::CheckForTemplateAndDigraph(Token &Next, ParsedType ObjectType,
 /// \param MayBePseudoDestructor When non-NULL, points to a flag that
 /// indicates whether this nested-name-specifier may be part of a
 /// pseudo-destructor name. In this case, the flag will be set false
-/// if we don't actually end up parsing a destructor name. Moreorover,
+/// if we don't actually end up parsing a destructor name. Moreover,
 /// if we do end up determining that we are parsing a destructor name,
 /// the last component of the nested-name-specifier is not parsed as
 /// part of the scope specifier.
@@ -316,13 +316,11 @@ bool Parser::ParseOptionalCXXScopeSpecifier(
       // Commit to parsing the template-id.
       TPA.Commit();
       TemplateTy Template;
-      if (TemplateNameKind TNK = Actions.ActOnDependentTemplateName(
-              getCurScope(), SS, TemplateKWLoc, TemplateName, ObjectType,
-              EnteringContext, Template, /*AllowInjectedClassName*/ true)) {
-        if (AnnotateTemplateIdToken(Template, TNK, SS, TemplateKWLoc,
-                                    TemplateName, false))
-          return true;
-      } else
+      TemplateNameKind TNK = Actions.ActOnTemplateName(
+          getCurScope(), SS, TemplateKWLoc, TemplateName, ObjectType,
+          EnteringContext, Template, /*AllowInjectedClassName*/ true);
+      if (AnnotateTemplateIdToken(Template, TNK, SS, TemplateKWLoc,
+                                  TemplateName, false))
         return true;
 
       continue;
@@ -356,7 +354,8 @@ bool Parser::ParseOptionalCXXScopeSpecifier(
       ASTTemplateArgsPtr TemplateArgsPtr(TemplateId->getTemplateArgs(),
                                          TemplateId->NumArgs);
 
-      if (Actions.ActOnCXXNestedNameSpecifier(getCurScope(),
+      if (TemplateId->isInvalid() ||
+          Actions.ActOnCXXNestedNameSpecifier(getCurScope(),
                                               SS,
                                               TemplateId->TemplateKWLoc,
                                               TemplateId->Template,
@@ -512,7 +511,7 @@ bool Parser::ParseOptionalCXXScopeSpecifier(
       if (MemberOfUnknownSpecialization && (ObjectType || SS.isSet()) &&
           (IsTypename || isTemplateArgumentList(1) == TPResult::True)) {
         // If we had errors before, ObjectType can be dependent even without any
-        // templates, do not report missing template keyword in that case.
+        // templates. Do not report missing template keyword in that case.
         if (!ObjectHadErrors) {
           // We have something like t::getAs<T>, where getAs is a
           // member of an unknown specialization. However, this will only
@@ -527,16 +526,13 @@ bool Parser::ParseOptionalCXXScopeSpecifier(
               << FixItHint::CreateInsertion(Tok.getLocation(), "template ");
         }
 
-        if (TemplateNameKind TNK = Actions.ActOnDependentTemplateName(
-                getCurScope(), SS, Tok.getLocation(), TemplateName, ObjectType,
-                EnteringContext, Template, /*AllowInjectedClassName*/ true)) {
-          // Consume the identifier.
-          ConsumeToken();
-          if (AnnotateTemplateIdToken(Template, TNK, SS, SourceLocation(),
-                                      TemplateName, false))
-            return true;
-        }
-        else
+        SourceLocation TemplateNameLoc = ConsumeToken();
+
+        TemplateNameKind TNK = Actions.ActOnTemplateName(
+            getCurScope(), SS, TemplateNameLoc, TemplateName, ObjectType,
+            EnteringContext, Template, /*AllowInjectedClassName*/ true);
+        if (AnnotateTemplateIdToken(Template, TNK, SS, SourceLocation(),
+                                    TemplateName, false))
           return true;
 
         continue;
@@ -1738,8 +1734,11 @@ Parser::ParseCXXPseudoDestructor(Expr *Base, SourceLocation OpLoc,
     assert(Tok.is(tok::coloncolon) &&"ParseOptionalCXXScopeSpecifier fail");
     CCLoc = ConsumeToken();
   } else if (Tok.is(tok::annot_template_id)) {
-    FirstTypeName.setTemplateId(
-                              (TemplateIdAnnotation *)Tok.getAnnotationValue());
+    TemplateIdAnnotation *TemplateId = takeTemplateIdAnnotation(Tok);
+    // FIXME: Carry on and build an AST representation for tooling.
+    if (TemplateId->isInvalid())
+      return ExprError();
+    FirstTypeName.setTemplateId(TemplateId);
     ConsumeAnnotationToken();
     assert(Tok.is(tok::coloncolon) &&"ParseOptionalCXXScopeSpecifier fail");
     CCLoc = ConsumeToken();
@@ -1774,6 +1773,12 @@ Parser::ParseCXXPseudoDestructor(Expr *Base, SourceLocation OpLoc,
 
   // If there is a '<', the second type name is a template-id. Parse
   // it as such.
+  //
+  // FIXME: This is not a context in which a '<' is assumed to start a template
+  // argument list. This affects examples such as
+  //   void f(auto *p) { p->~X<int>(); }
+  // ... but there's no ambiguity, and nowhere to write 'template' in such an
+  // example, so we accept it anyway.
   if (Tok.is(tok::less) &&
       ParseUnqualifiedIdTemplateId(
           SS, ObjectType, Base && Base->containsErrors(), SourceLocation(),
@@ -2142,12 +2147,8 @@ void Parser::ParseCXXSimpleTypeSpecifier(DeclSpec &DS) {
 
   // type-name
   case tok::annot_typename: {
-    if (getTypeAnnotation(Tok))
-      DS.SetTypeSpecType(DeclSpec::TST_typename, Loc, PrevSpec, DiagID,
-                         getTypeAnnotation(Tok), Policy);
-    else
-      DS.SetTypeSpecError();
-
+    DS.SetTypeSpecType(DeclSpec::TST_typename, Loc, PrevSpec, DiagID,
+                       getTypeAnnotation(Tok), Policy);
     DS.SetRangeEnd(Tok.getAnnotationEndLoc());
     ConsumeAnnotationToken();
 
@@ -2303,11 +2304,9 @@ bool Parser::ParseUnqualifiedIdTemplateId(
     if (AssumeTemplateId) {
       // We defer the injected-class-name checks until we've found whether
       // this template-id is used to form a nested-name-specifier or not.
-      TNK = Actions.ActOnDependentTemplateName(
-          getCurScope(), SS, TemplateKWLoc, Id, ObjectType, EnteringContext,
-          Template, /*AllowInjectedClassName*/ true);
-      if (TNK == TNK_Non_template)
-        return true;
+      TNK = Actions.ActOnTemplateName(getCurScope(), SS, TemplateKWLoc, Id,
+                                      ObjectType, EnteringContext, Template,
+                                      /*AllowInjectedClassName*/ true);
     } else {
       bool MemberOfUnknownSpecialization;
       TNK = Actions.isTemplateName(getCurScope(), SS,
@@ -2344,11 +2343,11 @@ bool Parser::ParseUnqualifiedIdTemplateId(
               << Name
               << FixItHint::CreateInsertion(Id.StartLocation, "template ");
         }
-        TNK = Actions.ActOnDependentTemplateName(
+        TNK = Actions.ActOnTemplateName(
             getCurScope(), SS, TemplateKWLoc, Id, ObjectType, EnteringContext,
             Template, /*AllowInjectedClassName*/ true);
-        if (TNK == TNK_Non_template)
-          return true;
+      } else if (TNK == TNK_Non_template) {
+        return false;
       }
     }
     break;
@@ -2361,6 +2360,8 @@ bool Parser::ParseUnqualifiedIdTemplateId(
                                  TemplateName, ObjectType,
                                  EnteringContext, Template,
                                  MemberOfUnknownSpecialization);
+    if (TNK == TNK_Non_template)
+      return false;
     break;
   }
 
@@ -2369,11 +2370,9 @@ bool Parser::ParseUnqualifiedIdTemplateId(
     bool MemberOfUnknownSpecialization;
     TemplateName.setIdentifier(Name, NameLoc);
     if (ObjectType) {
-      TNK = Actions.ActOnDependentTemplateName(
+      TNK = Actions.ActOnTemplateName(
           getCurScope(), SS, TemplateKWLoc, TemplateName, ObjectType,
           EnteringContext, Template, /*AllowInjectedClassName*/ true);
-      if (TNK == TNK_Non_template)
-        return true;
     } else {
       TNK = Actions.isTemplateName(getCurScope(), SS, TemplateKWLoc.isValid(),
                                    TemplateName, ObjectType,
@@ -2383,7 +2382,7 @@ bool Parser::ParseUnqualifiedIdTemplateId(
       if (TNK == TNK_Non_template && !Id.DestructorName.get()) {
         Diag(NameLoc, diag::err_destructor_template_id)
           << Name << SS.getRange();
-        return true;
+        // Carry on to parse the template arguments before bailing out.
       }
     }
     break;
@@ -2393,9 +2392,6 @@ bool Parser::ParseUnqualifiedIdTemplateId(
     return false;
   }
 
-  if (TNK == TNK_Non_template)
-    return false;
-
   // Parse the enclosed template argument list.
   SourceLocation LAngleLoc, RAngleLoc;
   TemplateArgList TemplateArgs;
@@ -2403,6 +2399,10 @@ bool Parser::ParseUnqualifiedIdTemplateId(
                                        RAngleLoc))
     return true;
 
+  // If this is a non-template, we already issued a diagnostic.
+  if (TNK == TNK_Non_template)
+    return true;
+
   if (Id.getKind() == UnqualifiedIdKind::IK_Identifier ||
       Id.getKind() == UnqualifiedIdKind::IK_OperatorFunctionId ||
       Id.getKind() == UnqualifiedIdKind::IK_LiteralOperatorId) {
@@ -2420,7 +2420,7 @@ bool Parser::ParseUnqualifiedIdTemplateId(
 
     TemplateIdAnnotation *TemplateId = TemplateIdAnnotation::Create(
         TemplateKWLoc, Id.StartLocation, TemplateII, OpKind, Template, TNK,
-        LAngleLoc, RAngleLoc, TemplateArgs, TemplateIds);
+        LAngleLoc, RAngleLoc, TemplateArgs, /*ArgsInvalid*/false, TemplateIds);
 
     Id.setTemplateId(TemplateId);
     return false;
@@ -2786,7 +2786,7 @@ bool Parser::ParseUnqualifiedId(CXXScopeSpec &SS, ParsedType ObjectType,
           TemplateKWLoc ? *TemplateKWLoc : SourceLocation(), Id, IdLoc,
           EnteringContext, Result, TemplateSpecified);
     else if (TemplateSpecified &&
-             Actions.ActOnDependentTemplateName(
+             Actions.ActOnTemplateName(
                  getCurScope(), SS, *TemplateKWLoc, Result, ObjectType,
                  EnteringContext, Template,
                  /*AllowInjectedClassName*/ true) == TNK_Non_template)
@@ -2800,6 +2800,13 @@ bool Parser::ParseUnqualifiedId(CXXScopeSpec &SS, ParsedType ObjectType,
   if (Tok.is(tok::annot_template_id)) {
     TemplateIdAnnotation *TemplateId = takeTemplateIdAnnotation(Tok);
 
+    // FIXME: Consider passing invalid template-ids on to callers; they may
+    // be able to recover better than we can.
+    if (TemplateId->isInvalid()) {
+      ConsumeAnnotationToken();
+      return true;
+    }
+
     // If the template-name names the current class, then this is a constructor
     if (AllowConstructorName && TemplateId->Name &&
         Actions.isCurrentClassName(*TemplateId->Name, getCurScope(), &SS)) {
@@ -2865,7 +2872,7 @@ bool Parser::ParseUnqualifiedId(CXXScopeSpec &SS, ParsedType ObjectType,
           TemplateKWLoc ? *TemplateKWLoc : SourceLocation(), nullptr,
           SourceLocation(), EnteringContext, Result, TemplateSpecified);
     else if (TemplateSpecified &&
-             Actions.ActOnDependentTemplateName(
+             Actions.ActOnTemplateName(
                  getCurScope(), SS, *TemplateKWLoc, Result, ObjectType,
                  EnteringContext, Template,
                  /*AllowInjectedClassName*/ true) == TNK_Non_template)
@@ -3094,10 +3101,14 @@ Parser::ParseCXXNewExpression(bool UseGlobal, SourceLocation Start) {
       auto RunSignatureHelp = [&]() {
         ParsedType TypeRep =
             Actions.ActOnTypeName(getCurScope(), DeclaratorInfo).get();
-        assert(TypeRep && "invalid types should be handled before");
-        QualType PreferredType = Actions.ProduceConstructorSignatureHelp(
-            getCurScope(), TypeRep.get()->getCanonicalTypeInternal(),
-            DeclaratorInfo.getEndLoc(), ConstructorArgs, ConstructorLParen);
+        QualType PreferredType;
+        // ActOnTypeName might adjust DeclaratorInfo and return a null type even
+        // the passing DeclaratorInfo is valid, e.g. running SignatureHelp on
+        // `new decltype(invalid) (^)`.
+        if (TypeRep)
+          PreferredType = Actions.ProduceConstructorSignatureHelp(
+              getCurScope(), TypeRep.get()->getCanonicalTypeInternal(),
+              DeclaratorInfo.getEndLoc(), ConstructorArgs, ConstructorLParen);
         CalledSignatureHelp = true;
         return PreferredType;
       };
@@ -3541,6 +3552,8 @@ ExprResult Parser::ParseRequiresExpression() {
             } else {
               TemplateId = takeTemplateIdAnnotation(Tok);
               ConsumeAnnotationToken();
+              if (TemplateId->isInvalid())
+                break;
             }
 
             if (auto *Req = Actions.ActOnTypeRequirement(TypenameKWLoc, SS,
diff --git a/clang/lib/Parse/ParseObjc.cpp b/clang/lib/Parse/ParseObjc.cpp
index 1c61631513867..10252b08a8d16 100644
--- a/clang/lib/Parse/ParseObjc.cpp
+++ b/clang/lib/Parse/ParseObjc.cpp
@@ -523,10 +523,9 @@ ObjCTypeParamList *Parser::parseObjCTypeParamListOrProtocolRefs(
     SkipUntil(tok::greater, tok::at, StopBeforeMatch);
     if (Tok.is(tok::greater))
       ConsumeToken();
-  } else if (ParseGreaterThanInTemplateList(rAngleLoc,
+  } else if (ParseGreaterThanInTemplateList(lAngleLoc, rAngleLoc,
                                             /*ConsumeLastToken=*/true,
                                             /*ObjCGenericList=*/true)) {
-    Diag(lAngleLoc, diag::note_matching) << "'<'";
     SkipUntil({tok::greater, tok::greaterequal, tok::at, tok::minus,
                tok::minus, tok::plus, tok::colon, tok::l_paren, tok::l_brace,
                tok::comma, tok::semi },
@@ -1551,7 +1550,7 @@ ParseObjCProtocolReferences(SmallVectorImpl<Decl *> &Protocols,
   }
 
   // Consume the '>'.
-  if (ParseGreaterThanInTemplateList(EndLoc, consumeLastToken,
+  if (ParseGreaterThanInTemplateList(LAngleLoc, EndLoc, consumeLastToken,
                                      /*ObjCGenericList=*/false))
     return true;
 
@@ -1649,7 +1648,7 @@ void Parser::parseObjCTypeArgsOrProtocolQualifiers(
   if (allSingleIdentifiers) {
     // Parse the closing '>'.
     SourceLocation rAngleLoc;
-    (void)ParseGreaterThanInTemplateList(rAngleLoc, consumeLastToken,
+    (void)ParseGreaterThanInTemplateList(lAngleLoc, rAngleLoc, consumeLastToken,
                                          /*ObjCGenericList=*/true);
 
     // Let Sema figure out what we parsed.
@@ -1755,7 +1754,7 @@ void Parser::parseObjCTypeArgsOrProtocolQualifiers(
 
   // Parse the closing '>'.
   SourceLocation rAngleLoc;
-  (void)ParseGreaterThanInTemplateList(rAngleLoc, consumeLastToken,
+  (void)ParseGreaterThanInTemplateList(lAngleLoc, rAngleLoc, consumeLastToken,
                                        /*ObjCGenericList=*/true);
 
   if (invalid) {
@@ -2979,7 +2978,7 @@ bool Parser::isStartOfObjCClassMessageMissingOpenBracket() {
       InMessageExpression)
     return false;
 
-  ParsedType Type;
+  TypeResult Type;
 
   if (Tok.is(tok::annot_typename))
     Type = getTypeAnnotation(Tok);
@@ -2989,7 +2988,8 @@ bool Parser::isStartOfObjCClassMessageMissingOpenBracket() {
   else
     return false;
 
-  if (!Type.get().isNull() && Type.get()->isObjCObjectOrInterfaceType()) {
+  // FIXME: Should not be querying properties of types from the parser.
+  if (Type.isUsable() && Type.get().get()->isObjCObjectOrInterfaceType()) {
     const Token &AfterNext = GetLookAheadToken(2);
     if (AfterNext.isOneOf(tok::colon, tok::r_square)) {
       if (Tok.is(tok::identifier))
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 7ae9885abe2bd..4bec5f557ab8b 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -11,8 +11,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/OpenMPClause.h"
 #include "clang/AST/StmtOpenMP.h"
 #include "clang/Basic/OpenMPKinds.h"
+#include "clang/Basic/TargetInfo.h"
 #include "clang/Basic/TokenKinds.h"
 #include "clang/Parse/ParseDiagnostic.h"
 #include "clang/Parse/Parser.h"
@@ -48,6 +50,8 @@ enum OpenMPDirectiveKindEx {
   OMPD_target_teams_distribute_parallel,
   OMPD_mapper,
   OMPD_variant,
+  OMPD_begin,
+  OMPD_begin_declare,
 };
 
 // Helper to unify the enum class OpenMPDirectiveKind with its extension
@@ -101,6 +105,7 @@ static unsigned getOpenMPDirectiveKindEx(StringRef S) {
       .Case("update", OMPD_update)
       .Case("mapper", OMPD_mapper)
       .Case("variant", OMPD_variant)
+      .Case("begin", OMPD_begin)
       .Default(OMPD_unknown);
 }
 
@@ -109,18 +114,21 @@ static OpenMPDirectiveKindExWrapper parseOpenMPDirectiveKind(Parser &P) {
   // E.g.: OMPD_for OMPD_simd ===> OMPD_for_simd
   // TODO: add other combined directives in topological order.
   static const OpenMPDirectiveKindExWrapper F[][3] = {
+      {OMPD_begin, OMPD_declare, OMPD_begin_declare},
+      {OMPD_end, OMPD_declare, OMPD_end_declare},
       {OMPD_cancellation, OMPD_point, OMPD_cancellation_point},
       {OMPD_declare, OMPD_reduction, OMPD_declare_reduction},
       {OMPD_declare, OMPD_mapper, OMPD_declare_mapper},
       {OMPD_declare, OMPD_simd, OMPD_declare_simd},
       {OMPD_declare, OMPD_target, OMPD_declare_target},
       {OMPD_declare, OMPD_variant, OMPD_declare_variant},
+      {OMPD_begin_declare, OMPD_variant, OMPD_begin_declare_variant},
+      {OMPD_end_declare, OMPD_variant, OMPD_end_declare_variant},
       {OMPD_distribute, OMPD_parallel, OMPD_distribute_parallel},
       {OMPD_distribute_parallel, OMPD_for, OMPD_distribute_parallel_for},
       {OMPD_distribute_parallel_for, OMPD_simd,
        OMPD_distribute_parallel_for_simd},
       {OMPD_distribute, OMPD_simd, OMPD_distribute_simd},
-      {OMPD_end, OMPD_declare, OMPD_end_declare},
       {OMPD_end_declare, OMPD_target, OMPD_end_declare_target},
       {OMPD_target, OMPD_data, OMPD_target_data},
       {OMPD_target, OMPD_enter, OMPD_target_enter},
@@ -748,7 +756,8 @@ static bool parseDeclareSimdClauses(
                                  getOpenMPClauseKind(ClauseName), *Vars, Data))
           IsError = true;
         if (CKind == OMPC_aligned) {
-          Alignments.append(Aligneds.size() - Alignments.size(), Data.TailExpr);
+          Alignments.append(Aligneds.size() - Alignments.size(),
+                            Data.DepModOrTailExpr);
         } else if (CKind == OMPC_linear) {
           assert(0 <= Data.ExtraModifier &&
                  Data.ExtraModifier <= OMPC_LINEAR_unknown &&
@@ -759,7 +768,7 @@ static bool parseDeclareSimdClauses(
             Data.ExtraModifier = OMPC_LINEAR_val;
           LinModifiers.append(Linears.size() - LinModifiers.size(),
                               Data.ExtraModifier);
-          Steps.append(Linears.size() - Steps.size(), Data.TailExpr);
+          Steps.append(Linears.size() - Steps.size(), Data.DepModOrTailExpr);
         }
       } else
         // TODO: add parsing of other clauses.
@@ -796,13 +805,7 @@ Parser::ParseOMPDeclareSimdClauses(Parser::DeclGroupPtrTy Ptr,
   bool IsError =
       parseDeclareSimdClauses(*this, BS, Simdlen, Uniforms, Aligneds,
                               Alignments, Linears, LinModifiers, Steps);
-  // Need to check for extra tokens.
-  if (Tok.isNot(tok::annot_pragma_openmp_end)) {
-    Diag(Tok, diag::warn_omp_extra_tokens_at_eol)
-        << getOpenMPDirectiveName(OMPD_declare_simd);
-    while (Tok.isNot(tok::annot_pragma_openmp_end))
-      ConsumeAnyToken();
-  }
+  skipUntilPragmaOpenMPEnd(OMPD_declare_simd);
   // Skip the last annot_pragma_openmp_end.
   SourceLocation EndLoc = ConsumeAnnotationToken();
   if (IsError)
@@ -863,7 +866,7 @@ static bool checkForDuplicates(Parser &P, StringRef Name,
 } // namespace
 
 void Parser::parseOMPTraitPropertyKind(
-    OMPTraitInfo::OMPTraitProperty &TIProperty, llvm::omp::TraitSet Set,
+    OMPTraitProperty &TIProperty, llvm::omp::TraitSet Set,
     llvm::omp::TraitSelector Selector, llvm::StringMap<SourceLocation> &Seen) {
   TIProperty.Kind = TraitProperty::invalid;
 
@@ -932,14 +935,14 @@ void Parser::parseOMPTraitPropertyKind(
       << CONTEXT_TRAIT_LVL << listOpenMPContextTraitProperties(Set, Selector);
 }
 
-void Parser::parseOMPContextProperty(OMPTraitInfo::OMPTraitSelector &TISelector,
+void Parser::parseOMPContextProperty(OMPTraitSelector &TISelector,
                                      llvm::omp::TraitSet Set,
                                      llvm::StringMap<SourceLocation> &Seen) {
   assert(TISelector.Kind != TraitSelector::user_condition &&
          "User conditions are special properties not handled here!");
 
   SourceLocation PropertyLoc = Tok.getLocation();
-  OMPTraitInfo::OMPTraitProperty TIProperty;
+  OMPTraitProperty TIProperty;
   parseOMPTraitPropertyKind(TIProperty, Set, TISelector.Kind, Seen);
 
   // If we have an invalid property here we already issued a warning.
@@ -973,7 +976,7 @@ void Parser::parseOMPContextProperty(OMPTraitInfo::OMPTraitSelector &TISelector,
 }
 
 void Parser::parseOMPTraitSelectorKind(
-    OMPTraitInfo::OMPTraitSelector &TISelector, llvm::omp::TraitSet Set,
+    OMPTraitSelector &TISelector, llvm::omp::TraitSet Set,
     llvm::StringMap<SourceLocation> &Seen) {
   TISelector.Kind = TraitSelector::invalid;
 
@@ -1052,7 +1055,7 @@ static ExprResult parseContextScore(Parser &P) {
 ///
 /// <trait-selector-name> ['('[<trait-score>] <trait-property> [, <t-p>]* ')']
 void Parser::parseOMPContextSelector(
-    OMPTraitInfo::OMPTraitSelector &TISelector, llvm::omp::TraitSet Set,
+    OMPTraitSelector &TISelector, llvm::omp::TraitSet Set,
     llvm::StringMap<SourceLocation> &SeenSelectors) {
   unsigned short OuterPC = ParenCount;
 
@@ -1130,13 +1133,19 @@ void Parser::parseOMPContextSelector(
   // Parse '('.
   (void)BDT.consumeOpen();
 
+  SourceLocation ScoreLoc = Tok.getLocation();
   ExprResult Score = parseContextScore(*this);
 
-  if (!AllowsTraitScore && Score.isUsable()) {
-    Diag(Score.get()->getBeginLoc(),
-         diag::warn_omp_ctx_incompatible_score_for_property)
-        << getOpenMPContextTraitSelectorName(TISelector.Kind)
-        << getOpenMPContextTraitSetName(Set) << Score.get();
+  if (!AllowsTraitScore && !Score.isUnset()) {
+    if (Score.isUsable()) {
+      Diag(ScoreLoc, diag::warn_omp_ctx_incompatible_score_for_property)
+          << getOpenMPContextTraitSelectorName(TISelector.Kind)
+          << getOpenMPContextTraitSetName(Set) << Score.get();
+    } else {
+      Diag(ScoreLoc, diag::warn_omp_ctx_incompatible_score_for_property)
+          << getOpenMPContextTraitSelectorName(TISelector.Kind)
+          << getOpenMPContextTraitSetName(Set) << "<invalid>";
+    }
     Score = ExprResult();
   }
 
@@ -1152,7 +1161,7 @@ void Parser::parseOMPContextSelector(
   BDT.consumeClose();
 }
 
-void Parser::parseOMPTraitSetKind(OMPTraitInfo::OMPTraitSet &TISet,
+void Parser::parseOMPTraitSetKind(OMPTraitSet &TISet,
                                   llvm::StringMap<SourceLocation> &Seen) {
   TISet.Kind = TraitSet::invalid;
 
@@ -1216,7 +1225,7 @@ void Parser::parseOMPTraitSetKind(OMPTraitInfo::OMPTraitSet &TISet,
 ///
 /// <trait-set-selector-name> '=' '{' <trait-selector> [, <trait-selector>]* '}'
 void Parser::parseOMPContextSelectorSet(
-    OMPTraitInfo::OMPTraitSet &TISet,
+    OMPTraitSet &TISet,
     llvm::StringMap<SourceLocation> &SeenSets) {
   auto OuterBC = BraceCount;
 
@@ -1271,7 +1280,7 @@ void Parser::parseOMPContextSelectorSet(
 
   llvm::StringMap<SourceLocation> SeenSelectors;
   do {
-    OMPTraitInfo::OMPTraitSelector TISelector;
+    OMPTraitSelector TISelector;
     parseOMPContextSelector(TISelector, TISet.Kind, SeenSelectors);
     if (TISelector.Kind != TraitSelector::invalid &&
         !TISelector.Properties.empty())
@@ -1293,10 +1302,10 @@ void Parser::parseOMPContextSelectorSet(
 /// Parse OpenMP context selectors:
 ///
 /// <trait-set-selector> [, <trait-set-selector>]*
-bool Parser::parseOMPContextSelectors(SourceLocation Loc, OMPTraitInfo &TI) {
+bool Parser::parseOMPContextSelectors(SourceLocation Loc, OMPTraitInfo& TI) {
   llvm::StringMap<SourceLocation> SeenSets;
   do {
-    OMPTraitInfo::OMPTraitSet TISet;
+    OMPTraitSet TISet;
     parseOMPContextSelectorSet(TISet, SeenSets);
     if (TISet.Kind != TraitSet::invalid && !TISet.Selectors.empty())
       TI.Sets.push_back(TISet);
@@ -1340,6 +1349,29 @@ void Parser::ParseOMPDeclareVariantClauses(Parser::DeclGroupPtrTy Ptr,
     return;
   }
 
+  OMPTraitInfo &TI = Actions.getASTContext().getNewOMPTraitInfo();
+  if (parseOMPDeclareVariantMatchClause(Loc, TI))
+    return;
+
+  Optional<std::pair<FunctionDecl *, Expr *>> DeclVarData =
+      Actions.checkOpenMPDeclareVariantFunction(
+          Ptr, AssociatedFunction.get(), TI,
+          SourceRange(Loc, Tok.getLocation()));
+
+  // Skip last tokens.
+  while (Tok.isNot(tok::annot_pragma_openmp_end))
+    ConsumeAnyToken();
+  if (DeclVarData && !TI.Sets.empty())
+    Actions.ActOnOpenMPDeclareVariantDirective(
+        DeclVarData->first, DeclVarData->second, TI,
+        SourceRange(Loc, Tok.getLocation()));
+
+  // Skip the last annot_pragma_openmp_end.
+  (void)ConsumeAnnotationToken();
+}
+
+bool Parser::parseOMPDeclareVariantMatchClause(SourceLocation Loc,
+                                               OMPTraitInfo &TI) {
   // Parse 'match'.
   OpenMPClauseKind CKind = Tok.isAnnotation()
                                ? OMPC_unknown
@@ -1351,7 +1383,7 @@ void Parser::ParseOMPDeclareVariantClauses(Parser::DeclGroupPtrTy Ptr,
       ;
     // Skip the last annot_pragma_openmp_end.
     (void)ConsumeAnnotationToken();
-    return;
+    return true;
   }
   (void)ConsumeToken();
   // Parse '('.
@@ -1362,31 +1394,15 @@ void Parser::ParseOMPDeclareVariantClauses(Parser::DeclGroupPtrTy Ptr,
       ;
     // Skip the last annot_pragma_openmp_end.
     (void)ConsumeAnnotationToken();
-    return;
+    return true;
   }
 
   // Parse inner context selectors.
-  OMPTraitInfo &TI = Actions.getASTContext().getNewOMPTraitInfo();
   parseOMPContextSelectors(Loc, TI);
 
   // Parse ')'
   (void)T.consumeClose();
-
-  Optional<std::pair<FunctionDecl *, Expr *>> DeclVarData =
-      Actions.checkOpenMPDeclareVariantFunction(
-          Ptr, AssociatedFunction.get(), TI,
-          SourceRange(Loc, Tok.getLocation()));
-
-  // Skip last tokens.
-  while (Tok.isNot(tok::annot_pragma_openmp_end))
-    ConsumeAnyToken();
-  if (DeclVarData.hasValue() && !TI.Sets.empty())
-    Actions.ActOnOpenMPDeclareVariantDirective(
-        DeclVarData.getValue().first, DeclVarData.getValue().second, TI,
-        SourceRange(Loc, Tok.getLocation()));
-
-  // Skip the last annot_pragma_openmp_end.
-  (void)ConsumeAnnotationToken();
+  return false;
 }
 
 /// Parsing of simple OpenMP clauses like 'default' or 'proc_bind'.
@@ -1524,21 +1540,48 @@ Parser::DeclGroupPtrTy Parser::ParseOMPDeclareTargetClauses() {
   return Actions.BuildDeclaratorGroup(Decls);
 }
 
-void Parser::ParseOMPEndDeclareTargetDirective(OpenMPDirectiveKind DKind,
-                                               SourceLocation DTLoc) {
-  if (DKind != OMPD_end_declare_target) {
-    Diag(Tok, diag::err_expected_end_declare_target);
-    Diag(DTLoc, diag::note_matching) << "'#pragma omp declare target'";
+void Parser::skipUntilPragmaOpenMPEnd(OpenMPDirectiveKind DKind) {
+  // The last seen token is annot_pragma_openmp_end - need to check for
+  // extra tokens.
+  if (Tok.is(tok::annot_pragma_openmp_end))
+    return;
+
+  Diag(Tok, diag::warn_omp_extra_tokens_at_eol)
+      << getOpenMPDirectiveName(DKind);
+  while (Tok.isNot(tok::annot_pragma_openmp_end))
+    ConsumeAnyToken();
+}
+
+void Parser::parseOMPEndDirective(OpenMPDirectiveKind BeginKind,
+                                  OpenMPDirectiveKind ExpectedKind,
+                                  OpenMPDirectiveKind FoundKind,
+                                  SourceLocation BeginLoc,
+                                  SourceLocation FoundLoc,
+                                  bool SkipUntilOpenMPEnd) {
+  int DiagSelection = ExpectedKind == OMPD_end_declare_target ? 0 : 1;
+
+  if (FoundKind == ExpectedKind) {
+    ConsumeAnyToken();
+    skipUntilPragmaOpenMPEnd(ExpectedKind);
     return;
   }
-  ConsumeAnyToken();
-  if (Tok.isNot(tok::annot_pragma_openmp_end)) {
-    Diag(Tok, diag::warn_omp_extra_tokens_at_eol)
-        << getOpenMPDirectiveName(OMPD_end_declare_target);
+
+  Diag(FoundLoc, diag::err_expected_end_declare_target_or_variant)
+      << DiagSelection;
+  Diag(BeginLoc, diag::note_matching)
+      << ("'#pragma omp " + getOpenMPDirectiveName(BeginKind) + "'").str();
+  if (SkipUntilOpenMPEnd)
     SkipUntil(tok::annot_pragma_openmp_end, StopBeforeMatch);
-  }
+}
+
+void Parser::ParseOMPEndDeclareTargetDirective(OpenMPDirectiveKind DKind,
+                                               SourceLocation DKLoc) {
+  parseOMPEndDirective(OMPD_declare_target, OMPD_end_declare_target, DKind,
+                       DKLoc, Tok.getLocation(),
+                       /* SkipUntilOpenMPEnd */ false);
   // Skip the last annot_pragma_openmp_end.
-  ConsumeAnyToken();
+  if (Tok.is(tok::annot_pragma_openmp_end))
+    ConsumeAnnotationToken();
 }
 
 /// Parsing of declarative OpenMP directives.
@@ -1616,13 +1659,7 @@ Parser::DeclGroupPtrTy Parser::ParseOpenMPDeclarativeDirectiveWithExtDecl(
     DeclDirectiveListParserHelper Helper(this, DKind);
     if (!ParseOpenMPSimpleVarList(DKind, Helper,
                                   /*AllowScopeSpecifier=*/true)) {
-      // The last seen token is annot_pragma_openmp_end - need to check for
-      // extra tokens.
-      if (Tok.isNot(tok::annot_pragma_openmp_end)) {
-        Diag(Tok, diag::warn_omp_extra_tokens_at_eol)
-            << getOpenMPDirectiveName(DKind);
-        SkipUntil(tok::annot_pragma_openmp_end, StopBeforeMatch);
-      }
+      skipUntilPragmaOpenMPEnd(DKind);
       // Skip the last annot_pragma_openmp_end.
       ConsumeAnnotationToken();
       return Actions.ActOnOpenMPThreadprivateDirective(Loc,
@@ -1661,13 +1698,7 @@ Parser::DeclGroupPtrTy Parser::ParseOpenMPDeclarativeDirectiveWithExtDecl(
             ConsumeToken();
           Actions.EndOpenMPClause();
         }
-        // The last seen token is annot_pragma_openmp_end - need to check for
-        // extra tokens.
-        if (Tok.isNot(tok::annot_pragma_openmp_end)) {
-          Diag(Tok, diag::warn_omp_extra_tokens_at_eol)
-              << getOpenMPDirectiveName(DKind);
-          SkipUntil(tok::annot_pragma_openmp_end, StopBeforeMatch);
-        }
+        skipUntilPragmaOpenMPEnd(DKind);
       }
       // Skip the last annot_pragma_openmp_end.
       ConsumeAnnotationToken();
@@ -1720,14 +1751,7 @@ Parser::DeclGroupPtrTy Parser::ParseOpenMPDeclarativeDirectiveWithExtDecl(
   case OMPD_declare_reduction:
     ConsumeToken();
     if (DeclGroupPtrTy Res = ParseOpenMPDeclareReductionDirective(AS)) {
-      // The last seen token is annot_pragma_openmp_end - need to check for
-      // extra tokens.
-      if (Tok.isNot(tok::annot_pragma_openmp_end)) {
-        Diag(Tok, diag::warn_omp_extra_tokens_at_eol)
-            << getOpenMPDirectiveName(OMPD_declare_reduction);
-        while (Tok.isNot(tok::annot_pragma_openmp_end))
-          ConsumeAnyToken();
-      }
+      skipUntilPragmaOpenMPEnd(OMPD_declare_reduction);
       // Skip the last annot_pragma_openmp_end.
       ConsumeAnnotationToken();
       return Res;
@@ -1742,6 +1766,61 @@ Parser::DeclGroupPtrTy Parser::ParseOpenMPDeclarativeDirectiveWithExtDecl(
     }
     break;
   }
+  case OMPD_begin_declare_variant: {
+    // The syntax is:
+    // { #pragma omp begin declare variant clause }
+    // <function-declaration-or-definition-sequence>
+    // { #pragma omp end declare variant }
+    //
+    ConsumeToken();
+    OMPTraitInfo &TI = Actions.getASTContext().getNewOMPTraitInfo();
+    if (parseOMPDeclareVariantMatchClause(Loc, TI))
+      break;
+
+    // Skip last tokens.
+    skipUntilPragmaOpenMPEnd(OMPD_begin_declare_variant);
+
+    VariantMatchInfo VMI;
+    ASTContext &ASTCtx = Actions.getASTContext();
+    TI.getAsVariantMatchInfo(ASTCtx, VMI, /* DeviceSetOnly */ true);
+    OMPContext OMPCtx(ASTCtx.getLangOpts().OpenMPIsDevice,
+                      ASTCtx.getTargetInfo().getTriple());
+
+    if (isVariantApplicableInContext(VMI, OMPCtx)) {
+      Actions.ActOnOpenMPBeginDeclareVariant(Loc, TI);
+      break;
+    }
+
+    // Elide all the code till the matching end declare variant was found.
+    unsigned Nesting = 1;
+    SourceLocation DKLoc;
+    OpenMPDirectiveKind DK = OMPD_unknown;
+    do {
+      DKLoc = Tok.getLocation();
+      DK = parseOpenMPDirectiveKind(*this);
+      if (DK == OMPD_end_declare_variant)
+        --Nesting;
+      else if (DK == OMPD_begin_declare_variant)
+        ++Nesting;
+      if (!Nesting || isEofOrEom())
+        break;
+      ConsumeAnyToken();
+    } while (true);
+
+    parseOMPEndDirective(OMPD_begin_declare_variant, OMPD_end_declare_variant,
+                         DK, Loc, DKLoc, /* SkipUntilOpenMPEnd */ true);
+    if (isEofOrEom())
+      return nullptr;
+    break;
+  }
+  case OMPD_end_declare_variant: {
+    if (Actions.isInOpenMPDeclareVariantScope())
+      Actions.ActOnOpenMPEndDeclareVariant();
+    else
+      Diag(Loc, diag::err_expected_begin_declare_variant);
+    ConsumeToken();
+    break;
+  }
   case OMPD_declare_variant:
   case OMPD_declare_simd: {
     // The syntax is:
@@ -1970,13 +2049,7 @@ Parser::ParseOpenMPDeclarativeOrExecutableDirective(ParsedStmtContext StmtCtx) {
     DeclDirectiveListParserHelper Helper(this, DKind);
     if (!ParseOpenMPSimpleVarList(DKind, Helper,
                                   /*AllowScopeSpecifier=*/false)) {
-      // The last seen token is annot_pragma_openmp_end - need to check for
-      // extra tokens.
-      if (Tok.isNot(tok::annot_pragma_openmp_end)) {
-        Diag(Tok, diag::warn_omp_extra_tokens_at_eol)
-            << getOpenMPDirectiveName(DKind);
-        SkipUntil(tok::annot_pragma_openmp_end, StopBeforeMatch);
-      }
+      skipUntilPragmaOpenMPEnd(DKind);
       DeclGroupPtrTy Res = Actions.ActOnOpenMPThreadprivateDirective(
           Loc, Helper.getIdentifiers());
       Directive = Actions.ActOnDeclStmt(Res, Loc, Tok.getLocation());
@@ -2021,13 +2094,7 @@ Parser::ParseOpenMPDeclarativeOrExecutableDirective(ParsedStmtContext StmtCtx) {
             ConsumeToken();
           Actions.EndOpenMPClause();
         }
-        // The last seen token is annot_pragma_openmp_end - need to check for
-        // extra tokens.
-        if (Tok.isNot(tok::annot_pragma_openmp_end)) {
-          Diag(Tok, diag::warn_omp_extra_tokens_at_eol)
-              << getOpenMPDirectiveName(DKind);
-          SkipUntil(tok::annot_pragma_openmp_end, StopBeforeMatch);
-        }
+        skipUntilPragmaOpenMPEnd(DKind);
       }
       DeclGroupPtrTy Res = Actions.ActOnOpenMPAllocateDirective(
           Loc, Helper.getIdentifiers(), Clauses);
@@ -2040,14 +2107,7 @@ Parser::ParseOpenMPDeclarativeOrExecutableDirective(ParsedStmtContext StmtCtx) {
     ConsumeToken();
     if (DeclGroupPtrTy Res =
             ParseOpenMPDeclareReductionDirective(/*AS=*/AS_none)) {
-      // The last seen token is annot_pragma_openmp_end - need to check for
-      // extra tokens.
-      if (Tok.isNot(tok::annot_pragma_openmp_end)) {
-        Diag(Tok, diag::warn_omp_extra_tokens_at_eol)
-            << getOpenMPDirectiveName(OMPD_declare_reduction);
-        while (Tok.isNot(tok::annot_pragma_openmp_end))
-          ConsumeAnyToken();
-      }
+      skipUntilPragmaOpenMPEnd(OMPD_declare_reduction);
       ConsumeAnyToken();
       Directive = Actions.ActOnDeclStmt(Res, Loc, Tok.getLocation());
     } else {
@@ -2251,6 +2311,8 @@ Parser::ParseOpenMPDeclarativeOrExecutableDirective(ParsedStmtContext StmtCtx) {
   case OMPD_declare_target:
   case OMPD_end_declare_target:
   case OMPD_requires:
+  case OMPD_begin_declare_variant:
+  case OMPD_end_declare_variant:
   case OMPD_declare_variant:
     Diag(Tok, diag::err_omp_unexpected_directive)
         << 1 << getOpenMPDirectiveName(DKind);
@@ -2521,9 +2583,7 @@ OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind,
     break;
   case OMPC_device_type:
   case OMPC_unknown:
-    Diag(Tok, diag::warn_omp_extra_tokens_at_eol)
-        << getOpenMPDirectiveName(DKind);
-    SkipUntil(tok::annot_pragma_openmp_end, StopBeforeMatch);
+    skipUntilPragmaOpenMPEnd(DKind);
     break;
   case OMPC_threadprivate:
   case OMPC_uniform:
@@ -2996,6 +3056,114 @@ static void parseMapType(Parser &P, Parser::OpenMPVarListDataTy &Data) {
   P.ConsumeToken();
 }
 
+/// Parses simple expression in parens for single-expression clauses of OpenMP
+/// constructs.
+/// \param RLoc Returned location of right paren.
+ExprResult Parser::ParseOpenMPIteratorsExpr() {
+  assert(Tok.is(tok::identifier) && PP.getSpelling(Tok) == "iterator" &&
+         "Expected 'iterator' token.");
+  SourceLocation IteratorKwLoc = ConsumeToken();
+
+  BalancedDelimiterTracker T(*this, tok::l_paren, tok::annot_pragma_openmp_end);
+  if (T.expectAndConsume(diag::err_expected_lparen_after, "iterator"))
+    return ExprError();
+
+  SourceLocation LLoc = T.getOpenLocation();
+  SmallVector<Sema::OMPIteratorData, 4> Data;
+  while (Tok.isNot(tok::r_paren) && Tok.isNot(tok::annot_pragma_openmp_end)) {
+    // Check if the type parsing is required.
+    ParsedType IteratorType;
+    if (Tok.isNot(tok::identifier) || NextToken().isNot(tok::equal)) {
+      // identifier '=' is not found - parse type.
+      TypeResult TR = ParseTypeName();
+      if (TR.isInvalid()) {
+        T.skipToEnd();
+        return ExprError();
+      }
+      IteratorType = TR.get();
+    }
+
+    // Parse identifier.
+    IdentifierInfo *II = nullptr;
+    SourceLocation IdLoc;
+    if (Tok.is(tok::identifier)) {
+      II = Tok.getIdentifierInfo();
+      IdLoc = ConsumeToken();
+    } else {
+      Diag(Tok, diag::err_expected_unqualified_id) << 0;
+    }
+
+    // Parse '='.
+    SourceLocation AssignLoc;
+    if (Tok.is(tok::equal))
+      AssignLoc = ConsumeToken();
+    else
+      Diag(Tok, diag::err_omp_expected_equal_in_iterator);
+
+    // Parse range-specification - <begin> ':' <end> [ ':' <step> ]
+    ColonProtectionRAIIObject ColonRAII(*this);
+    // Parse <begin>
+    SourceLocation Loc = Tok.getLocation();
+    ExprResult LHS = ParseCastExpression(AnyCastExpr);
+    ExprResult Begin = Actions.CorrectDelayedTyposInExpr(
+        ParseRHSOfBinaryExpression(LHS, prec::Conditional));
+    Begin = Actions.ActOnFinishFullExpr(Begin.get(), Loc,
+                                        /*DiscardedValue=*/false);
+    // Parse ':'.
+    SourceLocation ColonLoc;
+    if (Tok.is(tok::colon))
+      ColonLoc = ConsumeToken();
+
+    // Parse <end>
+    Loc = Tok.getLocation();
+    LHS = ParseCastExpression(AnyCastExpr);
+    ExprResult End = Actions.CorrectDelayedTyposInExpr(
+        ParseRHSOfBinaryExpression(LHS, prec::Conditional));
+    End = Actions.ActOnFinishFullExpr(End.get(), Loc,
+                                      /*DiscardedValue=*/false);
+
+    SourceLocation SecColonLoc;
+    ExprResult Step;
+    // Parse optional step.
+    if (Tok.is(tok::colon)) {
+      // Parse ':'
+      SecColonLoc = ConsumeToken();
+      // Parse <step>
+      Loc = Tok.getLocation();
+      LHS = ParseCastExpression(AnyCastExpr);
+      Step = Actions.CorrectDelayedTyposInExpr(
+          ParseRHSOfBinaryExpression(LHS, prec::Conditional));
+      Step = Actions.ActOnFinishFullExpr(Step.get(), Loc,
+                                         /*DiscardedValue=*/false);
+    }
+
+    // Parse ',' or ')'
+    if (Tok.isNot(tok::comma) && Tok.isNot(tok::r_paren))
+      Diag(Tok, diag::err_omp_expected_punc_after_iterator);
+    if (Tok.is(tok::comma))
+      ConsumeToken();
+
+    Sema::OMPIteratorData &D = Data.emplace_back();
+    D.DeclIdent = II;
+    D.DeclIdentLoc = IdLoc;
+    D.Type = IteratorType;
+    D.AssignLoc = AssignLoc;
+    D.ColonLoc = ColonLoc;
+    D.SecColonLoc = SecColonLoc;
+    D.Range.Begin = Begin.get();
+    D.Range.End = End.get();
+    D.Range.Step = Step.get();
+  }
+
+  // Parse ')'.
+  SourceLocation RLoc = Tok.getLocation();
+  if (!T.consumeClose())
+    RLoc = T.getCloseLocation();
+
+  return Actions.ActOnOMPIteratorExpr(getCurScope(), IteratorKwLoc, LLoc, RLoc,
+                                      Data);
+}
+
 /// Parses clauses with list.
 bool Parser::ParseOpenMPVarList(OpenMPDirectiveKind DKind,
                                 OpenMPClauseKind Kind,
@@ -3011,6 +3179,7 @@ bool Parser::ParseOpenMPVarList(OpenMPDirectiveKind DKind,
                          getOpenMPClauseName(Kind)))
     return true;
 
+  bool DependWithIterator = false;
   bool NeedRParenForLinear = false;
   BalancedDelimiterTracker LinearT(*this, tok::l_paren,
                                   tok::annot_pragma_openmp_end);
@@ -3048,6 +3217,22 @@ bool Parser::ParseOpenMPVarList(OpenMPDirectiveKind DKind,
       Data.ReductionOrMapperId =
           Actions.GetNameFromUnqualifiedId(UnqualifiedReductionId);
   } else if (Kind == OMPC_depend) {
+    if (getLangOpts().OpenMP >= 50) {
+      if (Tok.is(tok::identifier) && PP.getSpelling(Tok) == "iterator") {
+        // Handle optional dependence modifier.
+        // iterator(iterators-definition)
+        // where iterators-definition is iterator-specifier [,
+        // iterators-definition ]
+        // where iterator-specifier is [ iterator-type ] identifier =
+        // range-specification
+        DependWithIterator = true;
+        EnterScope(Scope::OpenMPDirectiveScope | Scope::DeclScope);
+        ExprResult IteratorRes = ParseOpenMPIteratorsExpr();
+        Data.DepModOrTailExpr = IteratorRes.get();
+        // Parse ','
+        ExpectAndConsume(tok::comma);
+      }
+    }
     // Handle dependency type for depend clause.
     ColonProtectionRAIIObject ColonRAII(*this);
     Data.ExtraModifier = getOpenMPSimpleClauseType(
@@ -3169,7 +3354,7 @@ bool Parser::ParseOpenMPVarList(OpenMPDirectiveKind DKind,
                                        /*DiscardedValue=*/false);
     if (Tail.isUsable()) {
       if (Tok.is(tok::colon)) {
-        Data.TailExpr = Tail.get();
+        Data.DepModOrTailExpr = Tail.get();
         Data.ColonLoc = ConsumeToken();
         TPA.Commit();
       } else {
@@ -3195,6 +3380,7 @@ bool Parser::ParseOpenMPVarList(OpenMPDirectiveKind DKind,
   const bool MayHaveTail = (Kind == OMPC_linear || Kind == OMPC_aligned);
   while (IsComma || (Tok.isNot(tok::r_paren) && Tok.isNot(tok::colon) &&
                      Tok.isNot(tok::annot_pragma_openmp_end))) {
+    ParseScope OMPListScope(this, Scope::OpenMPDirectiveScope);
     ColonProtectionRAIIObject ColonRAII(*this, MayHaveTail);
     // Parse variable
     ExprResult VarExpr =
@@ -3231,7 +3417,7 @@ bool Parser::ParseOpenMPVarList(OpenMPDirectiveKind DKind,
     Tail =
         Actions.ActOnFinishFullExpr(Tail.get(), ELoc, /*DiscardedValue*/ false);
     if (Tail.isUsable())
-      Data.TailExpr = Tail.get();
+      Data.DepModOrTailExpr = Tail.get();
     else
       SkipUntil(tok::comma, tok::r_paren, tok::annot_pragma_openmp_end,
                 StopBeforeMatch);
@@ -3241,8 +3427,11 @@ bool Parser::ParseOpenMPVarList(OpenMPDirectiveKind DKind,
   Data.RLoc = Tok.getLocation();
   if (!T.consumeClose())
     Data.RLoc = T.getCloseLocation();
+  // Exit from scope when the iterator is used in depend clause.
+  if (DependWithIterator)
+    ExitScope();
   return (Kind != OMPC_depend && Kind != OMPC_map && Vars.empty()) ||
-         (MustHaveTail && !Data.TailExpr) || InvalidReductionId ||
+         (MustHaveTail && !Data.DepModOrTailExpr) || InvalidReductionId ||
          IsInvalidMapperModifier;
 }
 
@@ -3314,7 +3503,7 @@ OMPClause *Parser::ParseOpenMPVarListClause(OpenMPDirectiveKind DKind,
     return nullptr;
   OMPVarListLocTy Locs(Loc, LOpen, Data.RLoc);
   return Actions.ActOnOpenMPVarListClause(
-      Kind, Vars, Data.TailExpr, Locs, Data.ColonLoc,
+      Kind, Vars, Data.DepModOrTailExpr, Locs, Data.ColonLoc,
       Data.ReductionOrMapperIdScopeSpec, Data.ReductionOrMapperId,
       Data.ExtraModifier, Data.MapTypeModifiers, Data.MapTypeModifiersLoc,
       Data.IsMapTypeImplicit, Data.ExtraModifierLoc);
diff --git a/clang/lib/Parse/ParseTemplate.cpp b/clang/lib/Parse/ParseTemplate.cpp
index 802fe35d4f62a..5025ea5c85feb 100644
--- a/clang/lib/Parse/ParseTemplate.cpp
+++ b/clang/lib/Parse/ParseTemplate.cpp
@@ -1031,7 +1031,8 @@ void Parser::DiagnoseMisplacedEllipsisInDeclarator(SourceLocation EllipsisLoc,
 /// or argument list.
 ///
 /// \returns true, if current token does not start with '>', false otherwise.
-bool Parser::ParseGreaterThanInTemplateList(SourceLocation &RAngleLoc,
+bool Parser::ParseGreaterThanInTemplateList(SourceLocation LAngleLoc,
+                                            SourceLocation &RAngleLoc,
                                             bool ConsumeLastToken,
                                             bool ObjCGenericList) {
   // What will be left once we've consumed the '>'.
@@ -1041,7 +1042,8 @@ bool Parser::ParseGreaterThanInTemplateList(SourceLocation &RAngleLoc,
 
   switch (Tok.getKind()) {
   default:
-    Diag(Tok.getLocation(), diag::err_expected) << tok::greater;
+    Diag(getEndOfPreviousToken(), diag::err_expected) << tok::greater;
+    Diag(LAngleLoc, diag::note_matching) << tok::less;
     return true;
 
   case tok::greater:
@@ -1220,16 +1222,17 @@ Parser::ParseTemplateIdAfterTemplateName(bool ConsumeLastToken,
 
     if (Invalid) {
       // Try to find the closing '>'.
-      if (ConsumeLastToken)
-        SkipUntil(tok::greater, StopAtSemi);
+      if (getLangOpts().CPlusPlus11)
+        SkipUntil(tok::greater, tok::greatergreater,
+                  tok::greatergreatergreater, StopAtSemi | StopBeforeMatch);
       else
         SkipUntil(tok::greater, StopAtSemi | StopBeforeMatch);
-      return true;
     }
   }
 
-  return ParseGreaterThanInTemplateList(RAngleLoc, ConsumeLastToken,
-                                        /*ObjCGenericList=*/false);
+  return ParseGreaterThanInTemplateList(LAngleLoc, RAngleLoc, ConsumeLastToken,
+                                        /*ObjCGenericList=*/false) ||
+         Invalid;
 }
 
 /// Replace the tokens that form a simple-template-id with an
@@ -1280,12 +1283,13 @@ bool Parser::AnnotateTemplateIdToken(TemplateTy Template, TemplateNameKind TNK,
                                      bool AllowTypeAnnotation,
                                      bool TypeConstraint) {
   assert(getLangOpts().CPlusPlus && "Can only annotate template-ids in C++");
-  assert(Template && (Tok.is(tok::less) || TypeConstraint) &&
+  assert((Tok.is(tok::less) || TypeConstraint) &&
          "Parser isn't at the beginning of a template-id");
   assert(!(TypeConstraint && AllowTypeAnnotation) && "type-constraint can't be "
                                                      "a type annotation");
   assert((!TypeConstraint || TNK == TNK_Concept_template) && "type-constraint "
          "must accompany a concept name");
+  assert((Template || TNK == TNK_Non_template) && "missing template name");
 
   // Consume the template-name.
   SourceLocation TemplateNameLoc = TemplateName.getSourceRange().getBegin();
@@ -1293,40 +1297,31 @@ bool Parser::AnnotateTemplateIdToken(TemplateTy Template, TemplateNameKind TNK,
   // Parse the enclosed template argument list.
   SourceLocation LAngleLoc, RAngleLoc;
   TemplateArgList TemplateArgs;
+  bool ArgsInvalid = false;
   if (!TypeConstraint || Tok.is(tok::less)) {
-    bool Invalid = ParseTemplateIdAfterTemplateName(false, LAngleLoc,
-                                                    TemplateArgs,
-                                                    RAngleLoc);
-
-    if (Invalid) {
-      // If we failed to parse the template ID but skipped ahead to a >, we're not
-      // going to be able to form a token annotation.  Eat the '>' if present.
-      TryConsumeToken(tok::greater);
-      // FIXME: Annotate the token stream so we don't produce the same errors
-      // again if we're doing this annotation as part of a tentative parse.
+    ArgsInvalid = ParseTemplateIdAfterTemplateName(false, LAngleLoc,
+                                                   TemplateArgs, RAngleLoc);
+    // If we couldn't recover from invalid arguments, don't form an annotation
+    // token -- we don't know how much to annotate.
+    // FIXME: This can lead to duplicate diagnostics if we retry parsing this
+    // template-id in another context. Try to annotate anyway?
+    if (RAngleLoc.isInvalid())
       return true;
-    }
   }
 
   ASTTemplateArgsPtr TemplateArgsPtr(TemplateArgs);
 
   // Build the annotation token.
   if (TNK == TNK_Type_template && AllowTypeAnnotation) {
-    TypeResult Type = Actions.ActOnTemplateIdType(
-        getCurScope(), SS, TemplateKWLoc, Template, TemplateName.Identifier,
-        TemplateNameLoc, LAngleLoc, TemplateArgsPtr, RAngleLoc);
-    if (Type.isInvalid()) {
-      // If we failed to parse the template ID but skipped ahead to a >, we're
-      // not going to be able to form a token annotation.  Eat the '>' if
-      // present.
-      TryConsumeToken(tok::greater);
-      // FIXME: Annotate the token stream so we don't produce the same errors
-      // again if we're doing this annotation as part of a tentative parse.
-      return true;
-    }
+    TypeResult Type = ArgsInvalid
+                          ? TypeError()
+                          : Actions.ActOnTemplateIdType(
+                                getCurScope(), SS, TemplateKWLoc, Template,
+                                TemplateName.Identifier, TemplateNameLoc,
+                                LAngleLoc, TemplateArgsPtr, RAngleLoc);
 
     Tok.setKind(tok::annot_typename);
-    setTypeAnnotation(Tok, Type.get());
+    setTypeAnnotation(Tok, Type);
     if (SS.isNotEmpty())
       Tok.setLocation(SS.getBeginLoc());
     else if (TemplateKWLoc.isValid())
@@ -1350,7 +1345,7 @@ bool Parser::AnnotateTemplateIdToken(TemplateTy Template, TemplateNameKind TNK,
 
     TemplateIdAnnotation *TemplateId = TemplateIdAnnotation::Create(
         TemplateKWLoc, TemplateNameLoc, TemplateII, OpKind, Template, TNK,
-        LAngleLoc, RAngleLoc, TemplateArgs, TemplateIds);
+        LAngleLoc, RAngleLoc, TemplateArgs, ArgsInvalid, TemplateIds);
 
     Tok.setAnnotationValue(TemplateId);
     if (TemplateKWLoc.isValid())
@@ -1386,29 +1381,24 @@ void Parser::AnnotateTemplateIdTokenAsType(CXXScopeSpec &SS,
   assert(Tok.is(tok::annot_template_id) && "Requires template-id tokens");
 
   TemplateIdAnnotation *TemplateId = takeTemplateIdAnnotation(Tok);
-  assert((TemplateId->Kind == TNK_Type_template ||
-          TemplateId->Kind == TNK_Dependent_template_name ||
-          TemplateId->Kind == TNK_Undeclared_template) &&
+  assert(TemplateId->mightBeType() &&
          "Only works for type and dependent templates");
 
   ASTTemplateArgsPtr TemplateArgsPtr(TemplateId->getTemplateArgs(),
                                      TemplateId->NumArgs);
 
-  TypeResult Type
-    = Actions.ActOnTemplateIdType(getCurScope(),
-                                  SS,
-                                  TemplateId->TemplateKWLoc,
-                                  TemplateId->Template,
-                                  TemplateId->Name,
-                                  TemplateId->TemplateNameLoc,
-                                  TemplateId->LAngleLoc,
-                                  TemplateArgsPtr,
-                                  TemplateId->RAngleLoc,
-                                  /*IsCtorOrDtorName*/false,
-                                  IsClassName);
+  TypeResult Type =
+      TemplateId->isInvalid()
+          ? TypeError()
+          : Actions.ActOnTemplateIdType(
+                getCurScope(), SS, TemplateId->TemplateKWLoc,
+                TemplateId->Template, TemplateId->Name,
+                TemplateId->TemplateNameLoc, TemplateId->LAngleLoc,
+                TemplateArgsPtr, TemplateId->RAngleLoc,
+                /*IsCtorOrDtorName*/ false, IsClassName);
   // Create the new "type" annotation token.
   Tok.setKind(tok::annot_typename);
-  setTypeAnnotation(Tok, Type.isInvalid() ? nullptr : Type.get());
+  setTypeAnnotation(Tok, Type);
   if (SS.isNotEmpty()) // it was a C++ qualified type name.
     Tok.setLocation(SS.getBeginLoc());
   // End location stays the same
@@ -1420,7 +1410,9 @@ void Parser::AnnotateTemplateIdTokenAsType(CXXScopeSpec &SS,
 
 /// Determine whether the given token can end a template argument.
 static bool isEndOfTemplateArgument(Token Tok) {
-  return Tok.isOneOf(tok::comma, tok::greater, tok::greatergreater);
+  // FIXME: Handle '>>>'.
+  return Tok.isOneOf(tok::comma, tok::greater, tok::greatergreater,
+                     tok::greatergreatergreater);
 }
 
 /// Parse a C++ template template argument.
@@ -1460,15 +1452,14 @@ ParsedTemplateArgument Parser::ParseTemplateTemplateArgument() {
 
       TryConsumeToken(tok::ellipsis, EllipsisLoc);
 
-      // If the next token signals the end of a template argument,
-      // then we have a dependent template name that could be a template
-      // template argument.
+      // If the next token signals the end of a template argument, then we have
+      // a (possibly-dependent) template name that could be a template template
+      // argument.
       TemplateTy Template;
       if (isEndOfTemplateArgument(Tok) &&
-          Actions.ActOnDependentTemplateName(
-              getCurScope(), SS, TemplateKWLoc, Name,
-              /*ObjectType=*/nullptr,
-              /*EnteringContext=*/false, Template))
+          Actions.ActOnTemplateName(getCurScope(), SS, TemplateKWLoc, Name,
+                                    /*ObjectType=*/nullptr,
+                                    /*EnteringContext=*/false, Template))
         Result = ParsedTemplateArgument(SS, Template, Name.StartLocation);
     }
   } else if (Tok.is(tok::identifier)) {
@@ -1572,10 +1563,8 @@ Parser::ParseTemplateArgumentList(TemplateArgList &TemplateArgs) {
     if (TryConsumeToken(tok::ellipsis, EllipsisLoc))
       Arg = Actions.ActOnPackExpansion(Arg, EllipsisLoc);
 
-    if (Arg.isInvalid()) {
-      SkipUntil(tok::comma, tok::greater, StopAtSemi | StopBeforeMatch);
+    if (Arg.isInvalid())
       return true;
-    }
 
     // Save this template argument.
     TemplateArgs.push_back(Arg);
@@ -1755,7 +1744,7 @@ bool Parser::diagnoseUnknownTemplateId(ExprResult LHS, SourceLocation Less) {
     TPA.Commit();
 
     SourceLocation Greater;
-    ParseGreaterThanInTemplateList(Greater, true, false);
+    ParseGreaterThanInTemplateList(Less, Greater, true, false);
     Actions.diagnoseExprIntendedAsTemplateName(getCurScope(), LHS,
                                                Less, Greater);
     return true;
@@ -1784,7 +1773,7 @@ void Parser::checkPotentialAngleBracket(ExprResult &PotentialTemplateName) {
        NextToken().isOneOf(tok::greatergreater, tok::greatergreatergreater))) {
     SourceLocation Less = ConsumeToken();
     SourceLocation Greater;
-    ParseGreaterThanInTemplateList(Greater, true, false);
+    ParseGreaterThanInTemplateList(Less, Greater, true, false);
     Actions.diagnoseExprIntendedAsTemplateName(
         getCurScope(), PotentialTemplateName, Less, Greater);
     // FIXME: Perform error recovery.
diff --git a/clang/lib/Parse/ParseTentative.cpp b/clang/lib/Parse/ParseTentative.cpp
index bca7d021da0a2..529e3f3210544 100644
--- a/clang/lib/Parse/ParseTentative.cpp
+++ b/clang/lib/Parse/ParseTentative.cpp
@@ -984,10 +984,16 @@ Parser::TPResult Parser::TryParseDeclarator(bool mayBeAbstract,
                                         NextToken().is(tok::kw_operator)))) &&
       mayHaveIdentifier) {
     // declarator-id
-    if (Tok.is(tok::annot_cxxscope))
+    if (Tok.is(tok::annot_cxxscope)) {
+      CXXScopeSpec SS;
+      Actions.RestoreNestedNameSpecifierAnnotation(
+          Tok.getAnnotationValue(), Tok.getAnnotationRange(), SS);
+      if (SS.isInvalid())
+        return TPResult::Error;
       ConsumeAnnotationToken();
-    else if (Tok.is(tok::identifier))
+    } else if (Tok.is(tok::identifier)) {
       TentativelyDeclaredIdentifiers.push_back(Tok.getIdentifierInfo());
+    }
     if (Tok.is(tok::kw_operator)) {
       if (TryParseOperatorId() == TPResult::Error)
         return TPResult::Error;
@@ -1539,7 +1545,9 @@ Parser::isCXXDeclarationSpecifier(Parser::TPResult BracedCastResult,
     TemplateIdAnnotation *TemplateId = takeTemplateIdAnnotation(Tok);
     // If lookup for the template-name found nothing, don't assume we have a
     // definitive disambiguation result yet.
-    if (TemplateId->Kind == TNK_Undeclared_template && InvalidAsDeclSpec) {
+    if ((TemplateId->hasInvalidName() ||
+         TemplateId->Kind == TNK_Undeclared_template) &&
+        InvalidAsDeclSpec) {
       // 'template-id(' can be a valid expression but not a valid decl spec if
       // the template-name is not declared, but we don't consider this to be a
       // definitive disambiguation. In any other context, it's an error either
@@ -1547,6 +1555,8 @@ Parser::isCXXDeclarationSpecifier(Parser::TPResult BracedCastResult,
       *InvalidAsDeclSpec = NextToken().is(tok::l_paren);
       return TPResult::Ambiguous;
     }
+    if (TemplateId->hasInvalidName())
+      return TPResult::Error;
     if (IsPlaceholderSpecifier(TemplateId, /*Lookahead=*/0))
       return TPResult::True;
     if (TemplateId->Kind != TNK_Type_template)
@@ -1566,6 +1576,13 @@ Parser::isCXXDeclarationSpecifier(Parser::TPResult BracedCastResult,
           NextToken().is(tok::annot_template_id)) {
         TemplateIdAnnotation *TemplateId =
             takeTemplateIdAnnotation(NextToken());
+        if (TemplateId->hasInvalidName()) {
+          if (InvalidAsDeclSpec) {
+            *InvalidAsDeclSpec = NextToken().is(tok::l_paren);
+            return TPResult::Ambiguous;
+          }
+          return TPResult::Error;
+        }
         if (IsPlaceholderSpecifier(TemplateId, /*Lookahead=*/1))
           return TPResult::True;
       }
@@ -2012,17 +2029,14 @@ Parser::TryParseParameterDeclarationClause(bool *InvalidAsDeclaration,
     //   (a) the previous parameter did, and
     //   (b) this must be the first declaration of the function, so we can't
     //       inherit any default arguments from elsewhere.
-    // If we see an ')', then we've reached the end of a
-    // parameter-declaration-clause, and the last param is missing its default
-    // argument.
+    // FIXME: If we reach a ')' without consuming any '>'s, then this must
+    // also be a function parameter (that's missing its default argument).
     if (VersusTemplateArgument)
-      return Tok.isOneOf(tok::equal, tok::r_paren) ? TPResult::True
-                                                   : TPResult::False;
+      return Tok.is(tok::equal) ? TPResult::True : TPResult::False;
 
     if (Tok.is(tok::equal)) {
       // '=' assignment-expression
       // Parse through assignment-expression.
-      // FIXME: assignment-expression may contain an unparenthesized comma.
       if (!SkipUntil(tok::comma, tok::r_paren, StopAtSemi | StopBeforeMatch))
         return TPResult::Error;
     }
diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp
index 0a63ac2d5e1bc..e9dbd50c3ac10 100644
--- a/clang/lib/Parse/Parser.cpp
+++ b/clang/lib/Parse/Parser.cpp
@@ -1878,9 +1878,7 @@ bool Parser::TryAnnotateTypeOrScopeToken() {
                                      Tok.getLocation());
     } else if (Tok.is(tok::annot_template_id)) {
       TemplateIdAnnotation *TemplateId = takeTemplateIdAnnotation(Tok);
-      if (TemplateId->Kind != TNK_Type_template &&
-          TemplateId->Kind != TNK_Dependent_template_name &&
-          TemplateId->Kind != TNK_Undeclared_template) {
+      if (!TemplateId->mightBeType()) {
         Diag(Tok, diag::err_typename_refers_to_non_type_template)
           << Tok.getAnnotationRange();
         return true;
@@ -1889,14 +1887,13 @@ bool Parser::TryAnnotateTypeOrScopeToken() {
       ASTTemplateArgsPtr TemplateArgsPtr(TemplateId->getTemplateArgs(),
                                          TemplateId->NumArgs);
 
-      Ty = Actions.ActOnTypenameType(getCurScope(), TypenameLoc, SS,
-                                     TemplateId->TemplateKWLoc,
-                                     TemplateId->Template,
-                                     TemplateId->Name,
-                                     TemplateId->TemplateNameLoc,
-                                     TemplateId->LAngleLoc,
-                                     TemplateArgsPtr,
-                                     TemplateId->RAngleLoc);
+      Ty = TemplateId->isInvalid()
+               ? TypeError()
+               : Actions.ActOnTypenameType(
+                     getCurScope(), TypenameLoc, SS, TemplateId->TemplateKWLoc,
+                     TemplateId->Template, TemplateId->Name,
+                     TemplateId->TemplateNameLoc, TemplateId->LAngleLoc,
+                     TemplateArgsPtr, TemplateId->RAngleLoc);
     } else {
       Diag(Tok, diag::err_expected_type_name_after_typename)
         << SS.getRange();
@@ -1905,7 +1902,7 @@ bool Parser::TryAnnotateTypeOrScopeToken() {
 
     SourceLocation EndLoc = Tok.getLastLoc();
     Tok.setKind(tok::annot_typename);
-    setTypeAnnotation(Tok, Ty.isInvalid() ? nullptr : Ty.get());
+    setTypeAnnotation(Tok, Ty);
     Tok.setAnnotationEndLoc(EndLoc);
     Tok.setLocation(TypenameLoc);
     PP.AnnotateCachedTokens(Tok);
diff --git a/clang/lib/Sema/CodeCompleteConsumer.cpp b/clang/lib/Sema/CodeCompleteConsumer.cpp
index b88ff9dd64cde..2402d896faac2 100644
--- a/clang/lib/Sema/CodeCompleteConsumer.cpp
+++ b/clang/lib/Sema/CodeCompleteConsumer.cpp
@@ -570,29 +570,10 @@ void PrintingCodeCompleteConsumer::ProcessCodeCompleteResults(
         if (const char *BriefComment = CCS->getBriefComment())
           OS << " : " << BriefComment;
       }
-      for (const FixItHint &FixIt : Results[I].FixIts) {
-        const SourceLocation BLoc = FixIt.RemoveRange.getBegin();
-        const SourceLocation ELoc = FixIt.RemoveRange.getEnd();
-
-        SourceManager &SM = SemaRef.SourceMgr;
-        std::pair<FileID, unsigned> BInfo = SM.getDecomposedLoc(BLoc);
-        std::pair<FileID, unsigned> EInfo = SM.getDecomposedLoc(ELoc);
-        // Adjust for token ranges.
-        if (FixIt.RemoveRange.isTokenRange())
-          EInfo.second += Lexer::MeasureTokenLength(ELoc, SM, SemaRef.LangOpts);
-
-        OS << " (requires fix-it:"
-           << " {" << SM.getLineNumber(BInfo.first, BInfo.second) << ':'
-           << SM.getColumnNumber(BInfo.first, BInfo.second) << '-'
-           << SM.getLineNumber(EInfo.first, EInfo.second) << ':'
-           << SM.getColumnNumber(EInfo.first, EInfo.second) << "}"
-           << " to \"" << FixIt.CodeToInsert << "\")";
-      }
-      OS << '\n';
       break;
 
     case CodeCompletionResult::RK_Keyword:
-      OS << Results[I].Keyword << '\n';
+      OS << Results[I].Keyword;
       break;
 
     case CodeCompletionResult::RK_Macro:
@@ -602,13 +583,31 @@ void PrintingCodeCompleteConsumer::ProcessCodeCompleteResults(
               includeBriefComments())) {
         OS << " : " << CCS->getAsString();
       }
-      OS << '\n';
       break;
 
     case CodeCompletionResult::RK_Pattern:
-      OS << "Pattern : " << Results[I].Pattern->getAsString() << '\n';
+      OS << "Pattern : " << Results[I].Pattern->getAsString();
       break;
     }
+    for (const FixItHint &FixIt : Results[I].FixIts) {
+      const SourceLocation BLoc = FixIt.RemoveRange.getBegin();
+      const SourceLocation ELoc = FixIt.RemoveRange.getEnd();
+
+      SourceManager &SM = SemaRef.SourceMgr;
+      std::pair<FileID, unsigned> BInfo = SM.getDecomposedLoc(BLoc);
+      std::pair<FileID, unsigned> EInfo = SM.getDecomposedLoc(ELoc);
+      // Adjust for token ranges.
+      if (FixIt.RemoveRange.isTokenRange())
+        EInfo.second += Lexer::MeasureTokenLength(ELoc, SM, SemaRef.LangOpts);
+
+      OS << " (requires fix-it:"
+         << " {" << SM.getLineNumber(BInfo.first, BInfo.second) << ':'
+         << SM.getColumnNumber(BInfo.first, BInfo.second) << '-'
+         << SM.getLineNumber(EInfo.first, EInfo.second) << ':'
+         << SM.getColumnNumber(EInfo.first, EInfo.second) << "}"
+         << " to \"" << FixIt.CodeToInsert << "\")";
+    }
+    OS << '\n';
   }
 }
 
diff --git a/clang/lib/Sema/DeclSpec.cpp b/clang/lib/Sema/DeclSpec.cpp
index 2da300a62dc3c..ae4a78a4556de 100644
--- a/clang/lib/Sema/DeclSpec.cpp
+++ b/clang/lib/Sema/DeclSpec.cpp
@@ -30,6 +30,9 @@ using namespace clang;
 
 void UnqualifiedId::setTemplateId(TemplateIdAnnotation *TemplateId) {
   assert(TemplateId && "NULL template-id annotation?");
+  assert(!TemplateId->isInvalid() &&
+         "should not convert invalid template-ids to unqualified-ids");
+
   Kind = UnqualifiedIdKind::IK_TemplateId;
   this->TemplateId = TemplateId;
   StartLocation = TemplateId->TemplateNameLoc;
@@ -38,6 +41,9 @@ void UnqualifiedId::setTemplateId(TemplateIdAnnotation *TemplateId) {
 
 void UnqualifiedId::setConstructorTemplateId(TemplateIdAnnotation *TemplateId) {
   assert(TemplateId && "NULL template-id annotation?");
+  assert(!TemplateId->isInvalid() &&
+         "should not convert invalid template-ids to unqualified-ids");
+
   Kind = UnqualifiedIdKind::IK_ConstructorTemplateId;
   this->TemplateId = TemplateId;
   StartLocation = TemplateId->TemplateNameLoc;
diff --git a/clang/lib/Sema/OpenCLBuiltins.td b/clang/lib/Sema/OpenCLBuiltins.td
index 651ca8aefc3cc..f3397f69824b9 100644
--- a/clang/lib/Sema/OpenCLBuiltins.td
+++ b/clang/lib/Sema/OpenCLBuiltins.td
@@ -1275,15 +1275,6 @@ def : Builtin<"get_default_queue", [Queue]>;
 // TODO: ndrange functions
 
 
-// OpenCL v2.0 s9.17.3: Additions to section 6.13.1: Work-Item Functions
-let MinVersion = CL20 in {
-  let Extension = FuncExtKhrSubgroups in {
-    def get_sub_group_size : Builtin<"get_sub_group_size", [UInt]>;
-    def get_max_sub_group_size : Builtin<"get_max_sub_group_size", [UInt]>;
-    def get_num_sub_groups : Builtin<"get_num_sub_groups", [UInt]>;
-  }
-}
-
 //--------------------------------------------------------------------
 // End of the builtin functions defined in the OpenCL C specification.
 // Builtin functions defined in the OpenCL C Extension are below.
@@ -1458,6 +1449,41 @@ let Extension = FuncExtKhrGlMsaaSharing in {
   }
 }
 
+//--------------------------------------------------------------------
+// OpenCL Extension v2.0 s28 - Subgroups
+// --- Table 28.2.1 ---
+let Extension = FuncExtKhrSubgroups in {
+  foreach name = ["get_sub_group_size", "get_max_sub_group_size",
+                  "get_num_sub_groups", "get_sub_group_id",
+                  "get_sub_group_local_id"] in {
+    def : Builtin<name, [UInt]>;
+  }
+  let MinVersion = CL20 in {
+    foreach name = ["get_enqueued_num_sub_groups"] in {
+      def : Builtin<name, [UInt]>;
+    }
+  }
+}
+
+// --- Table 28.2.2 ---
+// TODO: sub_group_barrier
+
+// --- Table 28.2.4 ---
+let Extension = FuncExtKhrSubgroups in {
+  foreach name = ["sub_group_all", "sub_group_any"] in {
+    def : Builtin<name, [Int, Int], Attr.Convergent>;
+  }
+  foreach name = ["sub_group_broadcast"] in {
+    def : Builtin<name, [IntLongFloatGenType1, IntLongFloatGenType1, UInt], Attr.Convergent>;
+  }
+  foreach name = ["sub_group_reduce_", "sub_group_scan_exclusive_",
+                  "sub_group_scan_inclusive_"] in {
+    foreach op = ["add", "min", "max"] in {
+      def : Builtin<name # op, [IntLongFloatGenType1, IntLongFloatGenType1], Attr.Convergent>;
+    }
+  }
+}
+
 //--------------------------------------------------------------------
 // Arm extensions.
 let Extension = ArmIntegerDotProductInt8 in {
diff --git a/clang/lib/Sema/ParsedAttr.cpp b/clang/lib/Sema/ParsedAttr.cpp
index 6d96ea96cd371..d45777ca127d6 100644
--- a/clang/lib/Sema/ParsedAttr.cpp
+++ b/clang/lib/Sema/ParsedAttr.cpp
@@ -115,7 +115,7 @@ const ParsedAttrInfo &ParsedAttrInfo::get(const AttributeCommonInfo &A) {
     return *AttrInfoMap[A.getParsedKind()];
 
   // If this is an ignored attribute then return an appropriate ParsedAttrInfo.
-  static ParsedAttrInfo IgnoredParsedAttrInfo(
+  static const ParsedAttrInfo IgnoredParsedAttrInfo(
       AttributeCommonInfo::IgnoredAttribute);
   if (A.getParsedKind() == AttributeCommonInfo::IgnoredAttribute)
     return IgnoredParsedAttrInfo;
@@ -140,7 +140,7 @@ const ParsedAttrInfo &ParsedAttrInfo::get(const AttributeCommonInfo &A) {
         return *Ptr;
 
   // If we failed to find a match then return a default ParsedAttrInfo.
-  static ParsedAttrInfo DefaultParsedAttrInfo(
+  static const ParsedAttrInfo DefaultParsedAttrInfo(
       AttributeCommonInfo::UnknownAttribute);
   return DefaultParsedAttrInfo;
 }
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index ff11e97c5783b..a4131e16d3f5a 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -1522,48 +1522,63 @@ class DeferredDiagnosticsEmitter
   }
 
   void visitUsedDecl(SourceLocation Loc, Decl *D) {
-    if (auto *FD = dyn_cast<FunctionDecl>(D)) {
-      FunctionDecl *Caller = UseStack.empty() ? nullptr : UseStack.back();
-      auto IsKnownEmitted = S.getEmissionStatus(FD, /*Final=*/true) ==
-                            Sema::FunctionEmissionStatus::Emitted;
-      if (!Caller)
-        ShouldEmit = IsKnownEmitted;
-      if ((!ShouldEmit && !S.getLangOpts().OpenMP && !Caller) ||
-          S.shouldIgnoreInHostDeviceCheck(FD) || Visited.count(D))
-        return;
-      // Finalize analysis of OpenMP-specific constructs.
-      if (Caller && S.LangOpts.OpenMP && UseStack.size() == 1)
-        S.finalizeOpenMPDelayedAnalysis(Caller, FD, Loc);
-      // Finalize analysis of SYCL-specific constructs.
-      if (Caller && S.LangOpts.SYCLIsDevice)
-        S.finalizeSYCLDelayedAnalysis(Caller, FD, Loc);
-      if (Caller)
-        S.DeviceKnownEmittedFns[FD] = {Caller, Loc};
-      if (ShouldEmit || InOMPDeviceContext)
-        S.emitDeferredDiags(FD, Caller);
-      Visited.insert(D);
-      UseStack.push_back(FD);
-      if (auto *S = FD->getBody()) {
-        this->Visit(S);
-      }
-      UseStack.pop_back();
-      Visited.erase(D);
-    } else if (auto *VD = dyn_cast<VarDecl>(D)) {
-      if (auto *Init = VD->getInit()) {
-        if (S.LangOpts.SYCLIsDevice)
-          return;
-        auto DevTy = OMPDeclareTargetDeclAttr::getDeviceType(VD);
-        bool IsDev = DevTy && (*DevTy == OMPDeclareTargetDeclAttr::DT_NoHost ||
-                               *DevTy == OMPDeclareTargetDeclAttr::DT_Any);
-        if (IsDev)
-          ++InOMPDeviceContext;
-        this->Visit(Init);
-        if (IsDev)
-          --InOMPDeviceContext;
-      }
-    } else
+    if (isa<VarDecl>(D))
+      return;
+    if (auto *FD = dyn_cast<FunctionDecl>(D))
+      checkFunc(Loc, FD);
+    else
       Inherited::visitUsedDecl(Loc, D);
   }
+
+  void checkVar(VarDecl *VD) {
+    assert(VD->isFileVarDecl() &&
+           "Should only check file-scope variables");
+    if (auto *Init = VD->getInit()) {
+      auto DevTy = OMPDeclareTargetDeclAttr::getDeviceType(VD);
+      bool IsDev = DevTy && (*DevTy == OMPDeclareTargetDeclAttr::DT_NoHost ||
+                             *DevTy == OMPDeclareTargetDeclAttr::DT_Any);
+      if (IsDev)
+        ++InOMPDeviceContext;
+      this->Visit(Init);
+      if (IsDev)
+        --InOMPDeviceContext;
+    }
+  }
+
+  void checkFunc(SourceLocation Loc, FunctionDecl *FD) {
+    FunctionDecl *Caller = UseStack.empty() ? nullptr : UseStack.back();
+    auto IsKnownEmitted = S.getEmissionStatus(FD, /*Final=*/true) ==
+                          Sema::FunctionEmissionStatus::Emitted;
+    if (!Caller)
+      ShouldEmit = IsKnownEmitted;
+    if ((!ShouldEmit && !S.getLangOpts().OpenMP && !Caller) ||
+        S.shouldIgnoreInHostDeviceCheck(FD) || Visited.count(FD))
+      return;
+    // Finalize analysis of OpenMP-specific constructs.
+    if (Caller && S.LangOpts.OpenMP && UseStack.size() == 1)
+      S.finalizeOpenMPDelayedAnalysis(Caller, FD, Loc);
+    // Finalize analysis of SYCL-specific constructs.
+    if (Caller && S.LangOpts.SYCLIsDevice)
+      S.finalizeSYCLDelayedAnalysis(Caller, FD, Loc);
+    if (Caller)
+      S.DeviceKnownEmittedFns[FD] = {Caller, Loc};
+    if (ShouldEmit || InOMPDeviceContext)
+      S.emitDeferredDiags(FD, Caller);
+    Visited.insert(FD);
+    UseStack.push_back(FD);
+    if (auto *S = FD->getBody()) {
+      this->Visit(S);
+    }
+    UseStack.pop_back();
+    Visited.erase(FD);
+  }
+
+  void checkRecordedDecl(Decl *D) {
+    if (auto *FD = dyn_cast<FunctionDecl>(D))
+      checkFunc(SourceLocation(), FD);
+    else
+      checkVar(cast<VarDecl>(D));
+  }
 };
 } // namespace
 
@@ -1579,7 +1594,7 @@ void Sema::emitDeferredDiags() {
 
   DeferredDiagnosticsEmitter DDE(*this);
   for (auto D : DeclsToCheckForDeferredDiags)
-    DDE.visitUsedDecl(SourceLocation(), D);
+    DDE.checkRecordedDecl(D);
 }
 
 // In CUDA, there are some constructs which may appear in semantically-valid
diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp
index 6ba404034a86b..00d47faec8a5b 100644
--- a/clang/lib/Sema/SemaCodeComplete.cpp
+++ b/clang/lib/Sema/SemaCodeComplete.cpp
@@ -9,6 +9,7 @@
 //  This file defines the code-completion semantic actions.
 //
 //===----------------------------------------------------------------------===//
+#include "clang/AST/ASTConcept.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclBase.h"
 #include "clang/AST/DeclCXX.h"
@@ -16,8 +17,11 @@
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/ExprCXX.h"
+#include "clang/AST/ExprConcepts.h"
 #include "clang/AST/ExprObjC.h"
+#include "clang/AST/NestedNameSpecifier.h"
 #include "clang/AST/QualTypeNames.h"
+#include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/AST/Type.h"
 #include "clang/Basic/CharInfo.h"
 #include "clang/Basic/Specifiers.h"
@@ -4746,6 +4750,369 @@ static RecordDecl *getAsRecordDecl(const QualType BaseType) {
   return nullptr;
 }
 
+namespace {
+// Collects completion-relevant information about a concept-constrainted type T.
+// In particular, examines the constraint expressions to find members of T.
+//
+// The design is very simple: we walk down each constraint looking for
+// expressions of the form T.foo().
+// If we're extra lucky, the return type is specified.
+// We don't do any clever handling of && or || in constraint expressions, we
+// take members from both branches.
+//
+// For example, given:
+//   template <class T> concept X = requires (T t, string& s) { t.print(s); };
+//   template <X U> void foo(U u) { u.^ }
+// We want to suggest the inferred member function 'print(string)'.
+// We see that u has type U, so X<U> holds.
+// X<U> requires t.print(s) to be valid, where t has type U (substituted for T).
+// By looking at the CallExpr we find the signature of print().
+//
+// While we tend to know in advance which kind of members (access via . -> ::)
+// we want, it's simpler just to gather them all and post-filter.
+//
+// FIXME: some of this machinery could be used for non-concept type-parms too,
+// enabling completion for type parameters based on other uses of that param.
+//
+// FIXME: there are other cases where a type can be constrained by a concept,
+// e.g. inside `if constexpr(ConceptSpecializationExpr) { ... }`
+class ConceptInfo {
+public:
+  // Describes a likely member of a type, inferred by concept constraints.
+  // Offered as a code completion for T. T-> and T:: contexts.
+  struct Member {
+    // Always non-null: we only handle members with ordinary identifier names.
+    const IdentifierInfo *Name = nullptr;
+    // Set for functions we've seen called.
+    // We don't have the declared parameter types, only the actual types of
+    // arguments we've seen. These are still valuable, as it's hard to render
+    // a useful function completion with neither parameter types nor names!
+    llvm::Optional<SmallVector<QualType, 1>> ArgTypes;
+    // Whether this is accessed as T.member, T->member, or T::member.
+    enum AccessOperator {
+      Colons,
+      Arrow,
+      Dot,
+    } Operator = Dot;
+    // What's known about the type of a variable or return type of a function.
+    const TypeConstraint *ResultType = nullptr;
+    // FIXME: also track:
+    //   - kind of entity (function/variable/type), to expose structured results
+    //   - template args kinds/types, as a proxy for template params
+
+    // For now we simply return these results as "pattern" strings.
+    CodeCompletionString *render(Sema &S, CodeCompletionAllocator &Alloc,
+                                 CodeCompletionTUInfo &Info) const {
+      CodeCompletionBuilder B(Alloc, Info);
+      // Result type
+      if (ResultType) {
+        std::string AsString;
+        {
+          llvm::raw_string_ostream OS(AsString);
+          QualType ExactType = deduceType(*ResultType);
+          if (!ExactType.isNull())
+            ExactType.print(OS, getCompletionPrintingPolicy(S));
+          else
+            ResultType->print(OS, getCompletionPrintingPolicy(S));
+        }
+        B.AddResultTypeChunk(Alloc.CopyString(AsString));
+      }
+      // Member name
+      B.AddTypedTextChunk(Alloc.CopyString(Name->getName()));
+      // Function argument list
+      if (ArgTypes) {
+        B.AddChunk(clang::CodeCompletionString::CK_LeftParen);
+        bool First = true;
+        for (QualType Arg : *ArgTypes) {
+          if (First)
+            First = false;
+          else {
+            B.AddChunk(clang::CodeCompletionString::CK_Comma);
+            B.AddChunk(clang::CodeCompletionString::CK_HorizontalSpace);
+          }
+          B.AddPlaceholderChunk(Alloc.CopyString(
+              Arg.getAsString(getCompletionPrintingPolicy(S))));
+        }
+        B.AddChunk(clang::CodeCompletionString::CK_RightParen);
+      }
+      return B.TakeString();
+    }
+  };
+
+  // BaseType is the type parameter T to infer members from.
+  // T must be accessible within S, as we use it to find the template entity
+  // that T is attached to in order to gather the relevant constraints.
+  ConceptInfo(const TemplateTypeParmType &BaseType, Scope *S) {
+    auto *TemplatedEntity = getTemplatedEntity(BaseType.getDecl(), S);
+    for (const Expr *E : constraintsForTemplatedEntity(TemplatedEntity))
+      believe(E, &BaseType);
+  }
+
+  std::vector<Member> members() {
+    std::vector<Member> Results;
+    for (const auto &E : this->Results)
+      Results.push_back(E.second);
+    llvm::sort(Results, [](const Member &L, const Member &R) {
+      return L.Name->getName() < R.Name->getName();
+    });
+    return Results;
+  }
+
+private:
+  // Infer members of T, given that the expression E (dependent on T) is true.
+  void believe(const Expr *E, const TemplateTypeParmType *T) {
+    if (!E || !T)
+      return;
+    if (auto *CSE = dyn_cast<ConceptSpecializationExpr>(E)) {
+      // If the concept is
+      //   template <class A, class B> concept CD = f<A, B>();
+      // And the concept specialization is
+      //   CD<int, T>
+      // Then we're substituting T for B, so we want to make f<A, B>() true
+      // by adding members to B - i.e. believe(f<A, B>(), B);
+      //
+      // For simplicity:
+      // - we don't attempt to substitute int for A
+      // - when T is used in other ways (like CD<T*>) we ignore it
+      ConceptDecl *CD = CSE->getNamedConcept();
+      TemplateParameterList *Params = CD->getTemplateParameters();
+      unsigned Index = 0;
+      for (const auto &Arg : CSE->getTemplateArguments()) {
+        if (Index >= Params->size())
+          break; // Won't happen in valid code.
+        if (isApprox(Arg, T)) {
+          auto *TTPD = dyn_cast<TemplateTypeParmDecl>(Params->getParam(Index));
+          if (!TTPD)
+            continue;
+          // T was used as an argument, and bound to the parameter TT.
+          auto *TT = cast<TemplateTypeParmType>(TTPD->getTypeForDecl());
+          // So now we know the constraint as a function of TT is true.
+          believe(CD->getConstraintExpr(), TT);
+          // (concepts themselves have no associated constraints to require)
+        }
+
+        ++Index;
+      }
+    } else if (auto *BO = dyn_cast<BinaryOperator>(E)) {
+      // For A && B, we can infer members from both branches.
+      // For A || B, the union is still more useful than the intersection.
+      if (BO->getOpcode() == BO_LAnd || BO->getOpcode() == BO_LOr) {
+        believe(BO->getLHS(), T);
+        believe(BO->getRHS(), T);
+      }
+    } else if (auto *RE = dyn_cast<RequiresExpr>(E)) {
+      // A requires(){...} lets us infer members from each requirement.
+      for (const concepts::Requirement *Req : RE->getRequirements()) {
+        if (!Req->isDependent())
+          continue; // Can't tell us anything about T.
+        // Now Req cannot a substitution-error: those aren't dependent.
+
+        if (auto *TR = dyn_cast<concepts::TypeRequirement>(Req)) {
+          // Do a full traversal so we get `foo` from `typename T::foo::bar`.
+          QualType AssertedType = TR->getType()->getType();
+          ValidVisitor(this, T).TraverseType(AssertedType);
+        } else if (auto *ER = dyn_cast<concepts::ExprRequirement>(Req)) {
+          ValidVisitor Visitor(this, T);
+          // If we have a type constraint on the value of the expression,
+          // AND the whole outer expression describes a member, then we'll
+          // be able to use the constraint to provide the return type.
+          if (ER->getReturnTypeRequirement().isTypeConstraint()) {
+            Visitor.OuterType =
+                ER->getReturnTypeRequirement().getTypeConstraint();
+            Visitor.OuterExpr = ER->getExpr();
+          }
+          Visitor.TraverseStmt(ER->getExpr());
+        } else if (auto *NR = dyn_cast<concepts::NestedRequirement>(Req)) {
+          believe(NR->getConstraintExpr(), T);
+        }
+      }
+    }
+  }
+
+  // This visitor infers members of T based on traversing expressions/types
+  // that involve T. It is invoked with code known to be valid for T.
+  class ValidVisitor : public RecursiveASTVisitor<ValidVisitor> {
+    ConceptInfo *Outer;
+    const TemplateTypeParmType *T;
+
+    CallExpr *Caller = nullptr;
+    Expr *Callee = nullptr;
+
+  public:
+    // If set, OuterExpr is constrained by OuterType.
+    Expr *OuterExpr = nullptr;
+    const TypeConstraint *OuterType = nullptr;
+
+    ValidVisitor(ConceptInfo *Outer, const TemplateTypeParmType *T)
+        : Outer(Outer), T(T) {
+      assert(T);
+    }
+
+    // In T.foo or T->foo, `foo` is a member function/variable.
+    bool VisitCXXDependentScopeMemberExpr(CXXDependentScopeMemberExpr *E) {
+      const Type *Base = E->getBaseType().getTypePtr();
+      bool IsArrow = E->isArrow();
+      if (Base->isPointerType() && IsArrow) {
+        IsArrow = false;
+        Base = Base->getPointeeType().getTypePtr();
+      }
+      if (isApprox(Base, T))
+        addValue(E, E->getMember(), IsArrow ? Member::Arrow : Member::Dot);
+      return true;
+    }
+
+    // In T::foo, `foo` is a static member function/variable.
+    bool VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *E) {
+      if (E->getQualifier() && isApprox(E->getQualifier()->getAsType(), T))
+        addValue(E, E->getDeclName(), Member::Colons);
+      return true;
+    }
+
+    // In T::typename foo, `foo` is a type.
+    bool VisitDependentNameType(DependentNameType *DNT) {
+      const auto *Q = DNT->getQualifier();
+      if (Q && isApprox(Q->getAsType(), T))
+        addType(DNT->getIdentifier());
+      return true;
+    }
+
+    // In T::foo::bar, `foo` must be a type.
+    // VisitNNS() doesn't exist, and TraverseNNS isn't always called :-(
+    bool TraverseNestedNameSpecifierLoc(NestedNameSpecifierLoc NNSL) {
+      if (NNSL) {
+        NestedNameSpecifier *NNS = NNSL.getNestedNameSpecifier();
+        const auto *Q = NNS->getPrefix();
+        if (Q && isApprox(Q->getAsType(), T))
+          addType(NNS->getAsIdentifier());
+      }
+      // FIXME: also handle T::foo<X>::bar
+      return RecursiveASTVisitor::TraverseNestedNameSpecifierLoc(NNSL);
+    }
+
+    // FIXME also handle T::foo<X>
+
+    // Track the innermost caller/callee relationship so we can tell if a
+    // nested expr is being called as a function.
+    bool VisitCallExpr(CallExpr *CE) {
+      Caller = CE;
+      Callee = CE->getCallee();
+      return true;
+    }
+
+  private:
+    void addResult(Member &&M) {
+      auto R = Outer->Results.try_emplace(M.Name);
+      Member &O = R.first->second;
+      // Overwrite existing if the new member has more info.
+      // The preference of . vs :: vs -> is fairly arbitrary.
+      if (/*Inserted*/ R.second ||
+          std::make_tuple(M.ArgTypes.hasValue(), M.ResultType != nullptr,
+                          M.Operator) > std::make_tuple(O.ArgTypes.hasValue(),
+                                                        O.ResultType != nullptr,
+                                                        O.Operator))
+        O = std::move(M);
+    }
+
+    void addType(const IdentifierInfo *Name) {
+      if (!Name)
+        return;
+      Member M;
+      M.Name = Name;
+      M.Operator = Member::Colons;
+      addResult(std::move(M));
+    }
+
+    void addValue(Expr *E, DeclarationName Name,
+                  Member::AccessOperator Operator) {
+      if (!Name.isIdentifier())
+        return;
+      Member Result;
+      Result.Name = Name.getAsIdentifierInfo();
+      Result.Operator = Operator;
+      // If this is the callee of an immediately-enclosing CallExpr, then
+      // treat it as a method, otherwise it's a variable.
+      if (Caller != nullptr && Callee == E) {
+        Result.ArgTypes.emplace();
+        for (const auto *Arg : Caller->arguments())
+          Result.ArgTypes->push_back(Arg->getType());
+        if (Caller == OuterExpr) {
+          Result.ResultType = OuterType;
+        }
+      } else {
+        if (E == OuterExpr)
+          Result.ResultType = OuterType;
+      }
+      addResult(std::move(Result));
+    }
+  };
+
+  static bool isApprox(const TemplateArgument &Arg, const Type *T) {
+    return Arg.getKind() == TemplateArgument::Type &&
+           isApprox(Arg.getAsType().getTypePtr(), T);
+  }
+
+  static bool isApprox(const Type *T1, const Type *T2) {
+    return T1 && T2 &&
+           T1->getCanonicalTypeUnqualified() ==
+               T2->getCanonicalTypeUnqualified();
+  }
+
+  // Returns the DeclContext immediately enclosed by the template parameter
+  // scope. For primary templates, this is the templated (e.g.) CXXRecordDecl.
+  // For specializations, this is e.g. ClassTemplatePartialSpecializationDecl.
+  static DeclContext *getTemplatedEntity(const TemplateTypeParmDecl *D,
+                                         Scope *S) {
+    if (D == nullptr)
+      return nullptr;
+    Scope *Inner = nullptr;
+    while (S) {
+      if (S->isTemplateParamScope() && S->isDeclScope(D))
+        return Inner ? Inner->getEntity() : nullptr;
+      Inner = S;
+      S = S->getParent();
+    }
+    return nullptr;
+  }
+
+  // Gets all the type constraint expressions that might apply to the type
+  // variables associated with DC (as returned by getTemplatedEntity()).
+  static SmallVector<const Expr *, 1>
+  constraintsForTemplatedEntity(DeclContext *DC) {
+    SmallVector<const Expr *, 1> Result;
+    if (DC == nullptr)
+      return Result;
+    // Primary templates can have constraints.
+    if (const auto *TD = cast<Decl>(DC)->getDescribedTemplate())
+      TD->getAssociatedConstraints(Result);
+    // Partial specializations may have constraints.
+    if (const auto *CTPSD =
+            dyn_cast<ClassTemplatePartialSpecializationDecl>(DC))
+      CTPSD->getAssociatedConstraints(Result);
+    if (const auto *VTPSD = dyn_cast<VarTemplatePartialSpecializationDecl>(DC))
+      VTPSD->getAssociatedConstraints(Result);
+    return Result;
+  }
+
+  // Attempt to find the unique type satisfying a constraint.
+  // This lets us show e.g. `int` instead of `std::same_as<int>`.
+  static QualType deduceType(const TypeConstraint &T) {
+    // Assume a same_as<T> return type constraint is std::same_as or equivalent.
+    // In this case the return type is T.
+    DeclarationName DN = T.getNamedConcept()->getDeclName();
+    if (DN.isIdentifier() && DN.getAsIdentifierInfo()->isStr("same_as"))
+      if (const auto *Args = T.getTemplateArgsAsWritten())
+        if (Args->getNumTemplateArgs() == 1) {
+          const auto &Arg = Args->arguments().front().getArgument();
+          if (Arg.getKind() == TemplateArgument::Type)
+            return Arg.getAsType();
+        }
+    return {};
+  }
+
+  llvm::DenseMap<const IdentifierInfo *, Member> Results;
+};
+} // namespace
+
 void Sema::CodeCompleteMemberReferenceExpr(Scope *S, Expr *Base,
                                            Expr *OtherOpBase,
                                            SourceLocation OpLoc, bool IsArrow,
@@ -4802,15 +5169,31 @@ void Sema::CodeCompleteMemberReferenceExpr(Scope *S, Expr *Base,
       if (const PointerType *Ptr = BaseType->getAs<PointerType>()) {
         BaseType = Ptr->getPointeeType();
         BaseKind = VK_LValue;
-      } else if (BaseType->isObjCObjectPointerType())
-        /*Do nothing*/;
-      else
+      } else if (BaseType->isObjCObjectPointerType() ||
+                 BaseType->isTemplateTypeParmType()) {
+        // Both cases (dot/arrow) handled below.
+      } else {
         return false;
+      }
     }
 
     if (RecordDecl *RD = getAsRecordDecl(BaseType)) {
       AddRecordMembersCompletionResults(*this, Results, S, BaseType, BaseKind,
                                         RD, std::move(AccessOpFixIt));
+    } else if (const auto *TTPT =
+                   dyn_cast<TemplateTypeParmType>(BaseType.getTypePtr())) {
+      auto Operator =
+          IsArrow ? ConceptInfo::Member::Arrow : ConceptInfo::Member::Dot;
+      for (const auto &R : ConceptInfo(*TTPT, S).members()) {
+        if (R.Operator != Operator)
+          continue;
+        CodeCompletionResult Result(
+            R.render(*this, CodeCompleter->getAllocator(),
+                     CodeCompleter->getCodeCompletionTUInfo()));
+        if (AccessOpFixIt)
+          Result.FixIts.push_back(*AccessOpFixIt);
+        Results.AddResult(std::move(Result));
+      }
     } else if (!IsArrow && BaseType->isObjCObjectPointerType()) {
       // Objective-C property reference.
       AddedPropertiesSet AddedProperties;
@@ -5446,13 +5829,14 @@ void Sema::CodeCompleteQualifiedId(Scope *S, CXXScopeSpec &SS,
   // Always pretend to enter a context to ensure that a dependent type
   // resolves to a dependent record.
   DeclContext *Ctx = computeDeclContext(SS, /*EnteringContext=*/true);
-  if (!Ctx)
-    return;
 
   // Try to instantiate any non-dependent declaration contexts before
-  // we look in them.
-  if (!isDependentScopeSpecifier(SS) && RequireCompleteDeclContext(SS, Ctx))
-    return;
+  // we look in them. Bail out if we fail.
+  NestedNameSpecifier *NNS = SS.getScopeRep();
+  if (NNS != nullptr && SS.isValid() && !NNS->isDependent()) {
+    if (Ctx == nullptr || RequireCompleteDeclContext(SS, Ctx))
+      return;
+  }
 
   ResultBuilder Results(*this, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(), CC);
@@ -5462,21 +5846,34 @@ void Sema::CodeCompleteQualifiedId(Scope *S, CXXScopeSpec &SS,
 
   // The "template" keyword can follow "::" in the grammar, but only
   // put it into the grammar if the nested-name-specifier is dependent.
-  NestedNameSpecifier *NNS = SS.getScopeRep();
+  // FIXME: results is always empty, this appears to be dead.
   if (!Results.empty() && NNS->isDependent())
     Results.AddResult("template");
 
+  // If the scope is a concept-constrained type parameter, infer nested
+  // members based on the constraints.
+  if (const auto *TTPT =
+          dyn_cast_or_null<TemplateTypeParmType>(NNS->getAsType())) {
+    for (const auto &R : ConceptInfo(*TTPT, S).members()) {
+      if (R.Operator != ConceptInfo::Member::Colons)
+        continue;
+      Results.AddResult(CodeCompletionResult(
+          R.render(*this, CodeCompleter->getAllocator(),
+                   CodeCompleter->getCodeCompletionTUInfo())));
+    }
+  }
+
   // Add calls to overridden virtual functions, if there are any.
   //
   // FIXME: This isn't wonderful, because we don't know whether we're actually
   // in a context that permits expressions. This is a general issue with
   // qualified-id completions.
-  if (!EnteringContext)
+  if (Ctx && !EnteringContext)
     MaybeAddOverrideCalls(*this, Ctx, Results);
   Results.ExitScope();
 
-  if (CodeCompleter->includeNamespaceLevelDecls() ||
-      (!Ctx->isNamespace() && !Ctx->isTranslationUnit())) {
+  if (Ctx &&
+      (CodeCompleter->includeNamespaceLevelDecls() || !Ctx->isFileContext())) {
     CodeCompletionDeclConsumer Consumer(Results, Ctx, BaseType);
     LookupVisibleDecls(Ctx, LookupOrdinaryName, Consumer,
                        /*IncludeGlobalScope=*/true,
diff --git a/clang/lib/Sema/SemaCoroutine.cpp b/clang/lib/Sema/SemaCoroutine.cpp
index 6dc9e342beb92..5ed0bbd6041d1 100644
--- a/clang/lib/Sema/SemaCoroutine.cpp
+++ b/clang/lib/Sema/SemaCoroutine.cpp
@@ -502,8 +502,9 @@ VarDecl *Sema::buildCoroutinePromise(SourceLocation Loc) {
     return nullptr;
 
   auto *ScopeInfo = getCurFunction();
-  // Build a list of arguments, based on the coroutine functions arguments,
-  // that will be passed to the promise type's constructor.
+
+  // Build a list of arguments, based on the coroutine function's arguments,
+  // that if present will be passed to the promise type's constructor.
   llvm::SmallVector<Expr *, 4> CtorArgExprs;
 
   // Add implicit object parameter.
@@ -519,6 +520,7 @@ VarDecl *Sema::buildCoroutinePromise(SourceLocation Loc) {
     }
   }
 
+  // Add the coroutine function's parameters.
   auto &Moves = ScopeInfo->CoroutineParameterMoves;
   for (auto *PD : FD->parameters()) {
     if (PD->getType()->isDependentType())
@@ -540,28 +542,33 @@ VarDecl *Sema::buildCoroutinePromise(SourceLocation Loc) {
     CtorArgExprs.push_back(RefExpr.get());
   }
 
-  // Create an initialization sequence for the promise type using the
-  // constructor arguments, wrapped in a parenthesized list expression.
-  Expr *PLE = ParenListExpr::Create(Context, FD->getLocation(),
-                                    CtorArgExprs, FD->getLocation());
-  InitializedEntity Entity = InitializedEntity::InitializeVariable(VD);
-  InitializationKind Kind = InitializationKind::CreateForInit(
-      VD->getLocation(), /*DirectInit=*/true, PLE);
-  InitializationSequence InitSeq(*this, Entity, Kind, CtorArgExprs,
-                                 /*TopLevelOfInitList=*/false,
-                                 /*TreatUnavailableAsInvalid=*/false);
-
-  // Attempt to initialize the promise type with the arguments.
-  // If that fails, fall back to the promise type's default constructor.
-  if (InitSeq) {
-    ExprResult Result = InitSeq.Perform(*this, Entity, Kind, CtorArgExprs);
-    if (Result.isInvalid()) {
-      VD->setInvalidDecl();
-    } else if (Result.get()) {
-      VD->setInit(MaybeCreateExprWithCleanups(Result.get()));
-      VD->setInitStyle(VarDecl::CallInit);
-      CheckCompleteVariableDeclaration(VD);
-    }
+  // If we have a non-zero number of constructor arguments, try to use them.
+  // Otherwise, fall back to the promise type's default constructor.
+  if (!CtorArgExprs.empty()) {
+    // Create an initialization sequence for the promise type using the
+    // constructor arguments, wrapped in a parenthesized list expression.
+    Expr *PLE = ParenListExpr::Create(Context, FD->getLocation(),
+                                      CtorArgExprs, FD->getLocation());
+    InitializedEntity Entity = InitializedEntity::InitializeVariable(VD);
+    InitializationKind Kind = InitializationKind::CreateForInit(
+        VD->getLocation(), /*DirectInit=*/true, PLE);
+    InitializationSequence InitSeq(*this, Entity, Kind, CtorArgExprs,
+                                   /*TopLevelOfInitList=*/false,
+                                   /*TreatUnavailableAsInvalid=*/false);
+
+    // Attempt to initialize the promise type with the arguments.
+    // If that fails, fall back to the promise type's default constructor.
+    if (InitSeq) {
+      ExprResult Result = InitSeq.Perform(*this, Entity, Kind, CtorArgExprs);
+      if (Result.isInvalid()) {
+        VD->setInvalidDecl();
+      } else if (Result.get()) {
+        VD->setInit(MaybeCreateExprWithCleanups(Result.get()));
+        VD->setInitStyle(VarDecl::CallInit);
+        CheckCompleteVariableDeclaration(VD);
+      }
+    } else
+      ActOnUninitializedDecl(VD);
   } else
     ActOnUninitializedDecl(VD);
 
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 04b231109c9d4..98de799415a0e 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -2734,6 +2734,11 @@ static void checkNewAttributesAfterDef(Sema &S, Decl *New, const Decl *Old) {
       // honored it.
       ++I;
       continue;
+    } else if (isa<OMPDeclareVariantAttr>(NewAttribute)) {
+      // We allow to add OMP[Begin]DeclareVariantAttr to be added to
+      // declarations after defintions.
+      ++I;
+      continue;
     }
 
     S.Diag(NewAttribute->getLocation(),
@@ -12265,7 +12270,7 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
     VDecl->setInitStyle(VarDecl::ListInit);
   }
 
-  if (LangOpts.OpenMP && VDecl->hasGlobalStorage())
+  if (LangOpts.OpenMP && VDecl->isFileVarDecl())
     DeclsToCheckForDeferredDiags.push_back(VDecl);
   CheckCompleteVariableDeclaration(VDecl);
 }
@@ -13582,9 +13587,28 @@ Sema::ActOnStartOfFunctionDef(Scope *FnBodyScope, Declarator &D,
   assert(D.isFunctionDeclarator() && "Not a function declarator!");
   Scope *ParentScope = FnBodyScope->getParent();
 
+  // Check if we are in an `omp begin/end declare variant` scope. If we are, and
+  // we define a non-templated function definition, we will create a declaration
+  // instead (=BaseFD), and emit the definition with a mangled name afterwards.
+  // The base function declaration will have the equivalent of an `omp declare
+  // variant` annotation which specifies the mangled definition as a
+  // specialization function under the OpenMP context defined as part of the
+  // `omp begin declare variant`.
+  FunctionDecl *BaseFD = nullptr;
+  if (LangOpts.OpenMP && isInOpenMPDeclareVariantScope() &&
+      TemplateParameterLists.empty())
+    BaseFD = ActOnStartOfFunctionDefinitionInOpenMPDeclareVariantScope(
+        ParentScope, D);
+
   D.setFunctionDefinitionKind(FDK_Definition);
   Decl *DP = HandleDeclarator(ParentScope, D, TemplateParameterLists);
-  return ActOnStartOfFunctionDef(FnBodyScope, DP, SkipBody);
+  Decl *Dcl = ActOnStartOfFunctionDef(FnBodyScope, DP, SkipBody);
+
+  if (BaseFD)
+    ActOnFinishedFunctionDefinitionInOpenMPDeclareVariantScope(
+        cast<FunctionDecl>(Dcl), BaseFD);
+
+  return Dcl;
 }
 
 void Sema::ActOnFinishInlineFunctionDef(FunctionDecl *D) {
@@ -16161,6 +16185,10 @@ ExprResult Sema::VerifyBitField(SourceLocation FieldLoc,
                                 IdentifierInfo *FieldName,
                                 QualType FieldTy, bool IsMsStruct,
                                 Expr *BitWidth, bool *ZeroWidth) {
+  assert(BitWidth);
+  if (BitWidth->containsErrors())
+    return ExprError();
+
   // Default to true; that shouldn't confuse checks for emptiness
   if (ZeroWidth)
     *ZeroWidth = true;
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index deb3a66630244..71d923431ad6f 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -7523,6 +7523,16 @@ static void ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D,
     handleSimpleAttributeWithExclusions<HIPPinnedShadowAttr, CUDADeviceAttr,
                                         CUDAConstantAttr>(S, D, AL);
     break;
+  case ParsedAttr::AT_CUDADeviceBuiltinSurfaceType:
+    handleSimpleAttributeWithExclusions<CUDADeviceBuiltinSurfaceTypeAttr,
+                                        CUDADeviceBuiltinTextureTypeAttr>(S, D,
+                                                                          AL);
+    break;
+  case ParsedAttr::AT_CUDADeviceBuiltinTextureType:
+    handleSimpleAttributeWithExclusions<CUDADeviceBuiltinTextureTypeAttr,
+                                        CUDADeviceBuiltinSurfaceTypeAttr>(S, D,
+                                                                          AL);
+    break;
   case ParsedAttr::AT_GNUInline:
     handleGNUInlineAttr(S, D, AL);
     break;
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index 23d099a9f2e88..2b4f445bfa215 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -5877,6 +5877,123 @@ static void checkForMultipleExportedDefaultConstructors(Sema &S,
   }
 }
 
+static void checkCUDADeviceBuiltinSurfaceClassTemplate(Sema &S,
+                                                       CXXRecordDecl *Class) {
+  bool ErrorReported = false;
+  auto reportIllegalClassTemplate = [&ErrorReported](Sema &S,
+                                                     ClassTemplateDecl *TD) {
+    if (ErrorReported)
+      return;
+    S.Diag(TD->getLocation(),
+           diag::err_cuda_device_builtin_surftex_cls_template)
+        << /*surface*/ 0 << TD;
+    ErrorReported = true;
+  };
+
+  ClassTemplateDecl *TD = Class->getDescribedClassTemplate();
+  if (!TD) {
+    auto *SD = dyn_cast<ClassTemplateSpecializationDecl>(Class);
+    if (!SD) {
+      S.Diag(Class->getLocation(),
+             diag::err_cuda_device_builtin_surftex_ref_decl)
+          << /*surface*/ 0 << Class;
+      S.Diag(Class->getLocation(),
+             diag::note_cuda_device_builtin_surftex_should_be_template_class)
+          << Class;
+      return;
+    }
+    TD = SD->getSpecializedTemplate();
+  }
+
+  TemplateParameterList *Params = TD->getTemplateParameters();
+  unsigned N = Params->size();
+
+  if (N != 2) {
+    reportIllegalClassTemplate(S, TD);
+    S.Diag(TD->getLocation(),
+           diag::note_cuda_device_builtin_surftex_cls_should_have_n_args)
+        << TD << 2;
+  }
+  if (N > 0 && !isa<TemplateTypeParmDecl>(Params->getParam(0))) {
+    reportIllegalClassTemplate(S, TD);
+    S.Diag(TD->getLocation(),
+           diag::note_cuda_device_builtin_surftex_cls_should_have_match_arg)
+        << TD << /*1st*/ 0 << /*type*/ 0;
+  }
+  if (N > 1) {
+    auto *NTTP = dyn_cast<NonTypeTemplateParmDecl>(Params->getParam(1));
+    if (!NTTP || !NTTP->getType()->isIntegralOrEnumerationType()) {
+      reportIllegalClassTemplate(S, TD);
+      S.Diag(TD->getLocation(),
+             diag::note_cuda_device_builtin_surftex_cls_should_have_match_arg)
+          << TD << /*2nd*/ 1 << /*integer*/ 1;
+    }
+  }
+}
+
+static void checkCUDADeviceBuiltinTextureClassTemplate(Sema &S,
+                                                       CXXRecordDecl *Class) {
+  bool ErrorReported = false;
+  auto reportIllegalClassTemplate = [&ErrorReported](Sema &S,
+                                                     ClassTemplateDecl *TD) {
+    if (ErrorReported)
+      return;
+    S.Diag(TD->getLocation(),
+           diag::err_cuda_device_builtin_surftex_cls_template)
+        << /*texture*/ 1 << TD;
+    ErrorReported = true;
+  };
+
+  ClassTemplateDecl *TD = Class->getDescribedClassTemplate();
+  if (!TD) {
+    auto *SD = dyn_cast<ClassTemplateSpecializationDecl>(Class);
+    if (!SD) {
+      S.Diag(Class->getLocation(),
+             diag::err_cuda_device_builtin_surftex_ref_decl)
+          << /*texture*/ 1 << Class;
+      S.Diag(Class->getLocation(),
+             diag::note_cuda_device_builtin_surftex_should_be_template_class)
+          << Class;
+      return;
+    }
+    TD = SD->getSpecializedTemplate();
+  }
+
+  TemplateParameterList *Params = TD->getTemplateParameters();
+  unsigned N = Params->size();
+
+  if (N != 3) {
+    reportIllegalClassTemplate(S, TD);
+    S.Diag(TD->getLocation(),
+           diag::note_cuda_device_builtin_surftex_cls_should_have_n_args)
+        << TD << 3;
+  }
+  if (N > 0 && !isa<TemplateTypeParmDecl>(Params->getParam(0))) {
+    reportIllegalClassTemplate(S, TD);
+    S.Diag(TD->getLocation(),
+           diag::note_cuda_device_builtin_surftex_cls_should_have_match_arg)
+        << TD << /*1st*/ 0 << /*type*/ 0;
+  }
+  if (N > 1) {
+    auto *NTTP = dyn_cast<NonTypeTemplateParmDecl>(Params->getParam(1));
+    if (!NTTP || !NTTP->getType()->isIntegralOrEnumerationType()) {
+      reportIllegalClassTemplate(S, TD);
+      S.Diag(TD->getLocation(),
+             diag::note_cuda_device_builtin_surftex_cls_should_have_match_arg)
+          << TD << /*2nd*/ 1 << /*integer*/ 1;
+    }
+  }
+  if (N > 2) {
+    auto *NTTP = dyn_cast<NonTypeTemplateParmDecl>(Params->getParam(2));
+    if (!NTTP || !NTTP->getType()->isIntegralOrEnumerationType()) {
+      reportIllegalClassTemplate(S, TD);
+      S.Diag(TD->getLocation(),
+             diag::note_cuda_device_builtin_surftex_cls_should_have_match_arg)
+          << TD << /*3rd*/ 2 << /*integer*/ 1;
+    }
+  }
+}
+
 void Sema::checkClassLevelCodeSegAttribute(CXXRecordDecl *Class) {
   // Mark any compiler-generated routines with the implicit code_seg attribute.
   for (auto *Method : Class->methods()) {
@@ -6657,6 +6774,13 @@ void Sema::CheckCompletedCXXClass(Scope *S, CXXRecordDecl *Record) {
     // is especially required for cases like vtable assumption loads.
     MarkVTableUsed(Record->getInnerLocStart(), Record);
   }
+
+  if (getLangOpts().CUDA) {
+    if (Record->hasAttr<CUDADeviceBuiltinSurfaceTypeAttr>())
+      checkCUDADeviceBuiltinSurfaceClassTemplate(*this, Record);
+    else if (Record->hasAttr<CUDADeviceBuiltinTextureTypeAttr>())
+      checkCUDADeviceBuiltinTextureClassTemplate(*this, Record);
+  }
 }
 
 /// Look up the special member function that would be called by a special
diff --git a/clang/lib/Sema/SemaExceptionSpec.cpp b/clang/lib/Sema/SemaExceptionSpec.cpp
index 53c62a1a40177..0dc0c68205fbb 100644
--- a/clang/lib/Sema/SemaExceptionSpec.cpp
+++ b/clang/lib/Sema/SemaExceptionSpec.cpp
@@ -1299,6 +1299,8 @@ CanThrowResult Sema::canThrow(const Stmt *S) {
     // Some might be dependent for other reasons.
   case Expr::ArraySubscriptExprClass:
   case Expr::OMPArraySectionExprClass:
+  case Expr::OMPArrayShapingExprClass:
+  case Expr::OMPIteratorExprClass:
   case Expr::BinaryOperatorClass:
   case Expr::DependentCoawaitExprClass:
   case Expr::CompoundAssignOperatorClass:
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index bd7db4a9aca18..7a0db866cb0e8 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -4816,6 +4816,206 @@ ExprResult Sema::ActOnOMPArraySectionExpr(Expr *Base, SourceLocation LBLoc,
                           VK_LValue, OK_Ordinary, ColonLoc, RBLoc);
 }
 
+ExprResult Sema::ActOnOMPArrayShapingExpr(Expr *Base, SourceLocation LParenLoc,
+                                          SourceLocation RParenLoc,
+                                          ArrayRef<Expr *> Dims,
+                                          ArrayRef<SourceRange> Brackets) {
+  if (Base->getType()->isPlaceholderType()) {
+    ExprResult Result = CheckPlaceholderExpr(Base);
+    if (Result.isInvalid())
+      return ExprError();
+    Result = DefaultLvalueConversion(Result.get());
+    if (Result.isInvalid())
+      return ExprError();
+    Base = Result.get();
+  }
+  QualType BaseTy = Base->getType();
+  // Delay analysis of the types/expressions if instantiation/specialization is
+  // required.
+  if (!BaseTy->isPointerType() && Base->isTypeDependent())
+    return OMPArrayShapingExpr::Create(Context, Context.DependentTy, Base,
+                                       LParenLoc, RParenLoc, Dims, Brackets);
+  if (!BaseTy->isPointerType() ||
+      (!Base->isTypeDependent() &&
+       BaseTy->getPointeeType()->isIncompleteType()))
+    return ExprError(Diag(Base->getExprLoc(),
+                          diag::err_omp_non_pointer_type_array_shaping_base)
+                     << Base->getSourceRange());
+
+  SmallVector<Expr *, 4> NewDims;
+  bool ErrorFound = false;
+  for (Expr *Dim : Dims) {
+    if (Dim->getType()->isPlaceholderType()) {
+      ExprResult Result = CheckPlaceholderExpr(Dim);
+      if (Result.isInvalid()) {
+        ErrorFound = true;
+        continue;
+      }
+      Result = DefaultLvalueConversion(Result.get());
+      if (Result.isInvalid()) {
+        ErrorFound = true;
+        continue;
+      }
+      Dim = Result.get();
+    }
+    if (!Dim->isTypeDependent()) {
+      ExprResult Result =
+          PerformOpenMPImplicitIntegerConversion(Dim->getExprLoc(), Dim);
+      if (Result.isInvalid()) {
+        ErrorFound = true;
+        Diag(Dim->getExprLoc(), diag::err_omp_typecheck_shaping_not_integer)
+            << Dim->getSourceRange();
+        continue;
+      }
+      Dim = Result.get();
+      Expr::EvalResult EvResult;
+      if (!Dim->isValueDependent() && Dim->EvaluateAsInt(EvResult, Context)) {
+        // OpenMP 5.0, [2.1.4 Array Shaping]
+        // Each si is an integral type expression that must evaluate to a
+        // positive integer.
+        llvm::APSInt Value = EvResult.Val.getInt();
+        if (!Value.isStrictlyPositive()) {
+          Diag(Dim->getExprLoc(), diag::err_omp_shaping_dimension_not_positive)
+              << Value.toString(/*Radix=*/10, /*Signed=*/true)
+              << Dim->getSourceRange();
+          ErrorFound = true;
+          continue;
+        }
+      }
+    }
+    NewDims.push_back(Dim);
+  }
+  if (ErrorFound)
+    return ExprError();
+  return OMPArrayShapingExpr::Create(Context, Context.OMPArrayShapingTy, Base,
+                                     LParenLoc, RParenLoc, NewDims, Brackets);
+}
+
+ExprResult Sema::ActOnOMPIteratorExpr(Scope *S, SourceLocation IteratorKwLoc,
+                                      SourceLocation LLoc, SourceLocation RLoc,
+                                      ArrayRef<OMPIteratorData> Data) {
+  SmallVector<OMPIteratorExpr::IteratorDefinition, 4> ID;
+  bool IsCorrect = true;
+  for (const OMPIteratorData &D : Data) {
+    TypeSourceInfo *TInfo = nullptr;
+    SourceLocation StartLoc;
+    QualType DeclTy;
+    if (!D.Type.getAsOpaquePtr()) {
+      // OpenMP 5.0, 2.1.6 Iterators
+      // In an iterator-specifier, if the iterator-type is not specified then
+      // the type of that iterator is of int type.
+      DeclTy = Context.IntTy;
+      StartLoc = D.DeclIdentLoc;
+    } else {
+      DeclTy = GetTypeFromParser(D.Type, &TInfo);
+      StartLoc = TInfo->getTypeLoc().getBeginLoc();
+    }
+
+    bool IsDeclTyDependent = DeclTy->isDependentType() ||
+                             DeclTy->containsUnexpandedParameterPack() ||
+                             DeclTy->isInstantiationDependentType();
+    if (!IsDeclTyDependent) {
+      if (!DeclTy->isIntegralType(Context) && !DeclTy->isAnyPointerType()) {
+        // OpenMP 5.0, 2.1.6 Iterators, Restrictions, C/C++
+        // The iterator-type must be an integral or pointer type.
+        Diag(StartLoc, diag::err_omp_iterator_not_integral_or_pointer)
+            << DeclTy;
+        IsCorrect = false;
+        continue;
+      }
+      if (DeclTy.isConstant(Context)) {
+        // OpenMP 5.0, 2.1.6 Iterators, Restrictions, C/C++
+        // The iterator-type must not be const qualified.
+        Diag(StartLoc, diag::err_omp_iterator_not_integral_or_pointer)
+            << DeclTy;
+        IsCorrect = false;
+        continue;
+      }
+    }
+
+    // Iterator declaration.
+    assert(D.DeclIdent && "Identifier expected.");
+    // Always try to create iterator declarator to avoid extra error messages
+    // about unknown declarations use.
+    auto *VD = VarDecl::Create(Context, CurContext, StartLoc, D.DeclIdentLoc,
+                               D.DeclIdent, DeclTy, TInfo, SC_None);
+    VD->setImplicit();
+    if (S) {
+      // Check for conflicting previous declaration.
+      DeclarationNameInfo NameInfo(VD->getDeclName(), D.DeclIdentLoc);
+      LookupResult Previous(*this, NameInfo, LookupOrdinaryName,
+                            ForVisibleRedeclaration);
+      Previous.suppressDiagnostics();
+      LookupName(Previous, S);
+
+      FilterLookupForScope(Previous, CurContext, S, /*ConsiderLinkage=*/false,
+                           /*AllowInlineNamespace=*/false);
+      if (!Previous.empty()) {
+        NamedDecl *Old = Previous.getRepresentativeDecl();
+        Diag(D.DeclIdentLoc, diag::err_redefinition) << VD->getDeclName();
+        Diag(Old->getLocation(), diag::note_previous_definition);
+      } else {
+        PushOnScopeChains(VD, S);
+      }
+    } else {
+      CurContext->addDecl(VD);
+    }
+    Expr *Begin = D.Range.Begin;
+    if (!IsDeclTyDependent && Begin && !Begin->isTypeDependent()) {
+      ExprResult BeginRes =
+          PerformImplicitConversion(Begin, DeclTy, AA_Converting);
+      Begin = BeginRes.get();
+    }
+    Expr *End = D.Range.End;
+    if (!IsDeclTyDependent && End && !End->isTypeDependent()) {
+      ExprResult EndRes = PerformImplicitConversion(End, DeclTy, AA_Converting);
+      End = EndRes.get();
+    }
+    Expr *Step = D.Range.Step;
+    if (!IsDeclTyDependent && Step && !Step->isTypeDependent()) {
+      if (!Step->getType()->isIntegralType(Context)) {
+        Diag(Step->getExprLoc(), diag::err_omp_iterator_step_not_integral)
+            << Step << Step->getSourceRange();
+        IsCorrect = false;
+        continue;
+      }
+      llvm::APSInt Result;
+      bool IsConstant = Step->isIntegerConstantExpr(Result, Context);
+      // OpenMP 5.0, 2.1.6 Iterators, Restrictions
+      // If the step expression of a range-specification equals zero, the
+      // behavior is unspecified.
+      if (IsConstant && Result.isNullValue()) {
+        Diag(Step->getExprLoc(), diag::err_omp_iterator_step_constant_zero)
+            << Step << Step->getSourceRange();
+        IsCorrect = false;
+        continue;
+      }
+    }
+    if (!Begin || !End || !IsCorrect) {
+      IsCorrect = false;
+      continue;
+    }
+    OMPIteratorExpr::IteratorDefinition &IDElem = ID.emplace_back();
+    IDElem.IteratorDecl = VD;
+    IDElem.AssignmentLoc = D.AssignLoc;
+    IDElem.Range.Begin = Begin;
+    IDElem.Range.End = End;
+    IDElem.Range.Step = Step;
+    IDElem.ColonLoc = D.ColonLoc;
+    IDElem.SecondColonLoc = D.SecColonLoc;
+  }
+  if (!IsCorrect) {
+    // Invalidate all created iterator declarations if error is found.
+    for (const OMPIteratorExpr::IteratorDefinition &D : ID) {
+      if (Decl *ID = D.IteratorDecl)
+        ID->setInvalidDecl();
+    }
+    return ExprError();
+  }
+  return OMPIteratorExpr::Create(Context, Context.OMPIteratorTy, IteratorKwLoc,
+                                 LLoc, RLoc, ID);
+}
+
 ExprResult
 Sema::CreateBuiltinArraySubscriptExpr(Expr *Base, SourceLocation LLoc,
                                       Expr *Idx, SourceLocation RLoc) {
@@ -5576,6 +5776,8 @@ static bool isPlaceholderToRemoveAsArg(QualType type) {
   case BuiltinType::BoundMember:
   case BuiltinType::BuiltinFn:
   case BuiltinType::OMPArraySection:
+  case BuiltinType::OMPArrayShaping:
+  case BuiltinType::OMPIterator:
     return true;
 
   }
@@ -5812,6 +6014,10 @@ ExprResult Sema::ActOnCallExpr(Scope *Scope, Expr *Fn, SourceLocation LParenLoc,
     }
   }
 
+  if (LangOpts.OpenMP)
+    Call = ActOnOpenMPCall(*this, Call, Scope, LParenLoc, ArgExprs, RParenLoc,
+                           ExecConfig);
+
   return Call;
 }
 
@@ -9219,7 +9425,13 @@ static bool tryGCCVectorConvertAndSplat(Sema &S, ExprResult *Scalar,
       // Reject cases where the scalar type is not a constant and has a higher
       // Order than the vector element type.
       llvm::APFloat Result(0.0);
-      bool CstScalar = Scalar->get()->EvaluateAsFloat(Result, S.Context);
+
+      // Determine whether this is a constant scalar. In the event that the
+      // value is dependent (and thus cannot be evaluated by the constant
+      // evaluator), skip the evaluation. This will then diagnose once the
+      // expression is instantiated.
+      bool CstScalar = Scalar->get()->isValueDependent() ||
+                       Scalar->get()->EvaluateAsFloat(Result, S.Context);
       int Order = S.Context.getFloatingTypeOrder(VectorEltTy, ScalarTy);
       if (!CstScalar && Order < 0)
         return true;
@@ -16125,9 +16337,6 @@ void Sema::MarkFunctionReferenced(SourceLocation Loc, FunctionDecl *Func,
 
     Func->markUsed(Context);
   }
-
-  if (LangOpts.OpenMP)
-    markOpenMPDeclareVariantFuncsReferenced(Loc, Func, MightBeOdrUse);
 }
 
 /// Directly mark a variable odr-used. Given a choice, prefer to use
@@ -18446,6 +18655,13 @@ ExprResult Sema::CheckPlaceholderExpr(Expr *E) {
     Diag(E->getBeginLoc(), diag::err_omp_array_section_use);
     return ExprError();
 
+  // Expressions of unknown type.
+  case BuiltinType::OMPArrayShaping:
+    return ExprError(Diag(E->getBeginLoc(), diag::err_omp_array_shaping_use));
+
+  case BuiltinType::OMPIterator:
+    return ExprError(Diag(E->getBeginLoc(), diag::err_omp_iterator_use));
+
   // Everything else should be impossible.
 #define IMAGE_TYPE(ImgType, Id, SingletonId, Access, Suffix) \
   case BuiltinType::Id:
@@ -18526,8 +18742,8 @@ bool Sema::IsDependentFunctionNameExpr(Expr *E) {
 
 ExprResult Sema::CreateRecoveryExpr(SourceLocation Begin, SourceLocation End,
                                     ArrayRef<Expr *> SubExprs) {
-  // RecoveryExpr is type-dependent to suppress bogus diagnostics and this trick
-  // does not work in C.
+  // FIXME: enable it for C++, RecoveryExpr is type-dependent to suppress
+  // bogus diagnostics and this trick does not work in C.
   // FIXME: use containsErrors() to suppress unwanted diags in C.
   if (!Context.getLangOpts().RecoveryAST)
     return ExprError();
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 11cc43a16db1a..f663b1d436594 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -35,6 +35,8 @@
 #include "llvm/ADT/PointerEmbeddedInt.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Frontend/OpenMP/OMPConstants.h"
+#include <set>
+
 using namespace clang;
 using namespace llvm::omp;
 
@@ -62,14 +64,15 @@ class DSAStackTy {
   struct DSAVarData {
     OpenMPDirectiveKind DKind = OMPD_unknown;
     OpenMPClauseKind CKind = OMPC_unknown;
+    unsigned Modifier = 0;
     const Expr *RefExpr = nullptr;
     DeclRefExpr *PrivateCopy = nullptr;
     SourceLocation ImplicitDSALoc;
     DSAVarData() = default;
     DSAVarData(OpenMPDirectiveKind DKind, OpenMPClauseKind CKind,
                const Expr *RefExpr, DeclRefExpr *PrivateCopy,
-               SourceLocation ImplicitDSALoc)
-        : DKind(DKind), CKind(CKind), RefExpr(RefExpr),
+               SourceLocation ImplicitDSALoc, unsigned Modifier)
+        : DKind(DKind), CKind(CKind), Modifier(Modifier), RefExpr(RefExpr),
           PrivateCopy(PrivateCopy), ImplicitDSALoc(ImplicitDSALoc) {}
   };
   using OperatorOffsetTy =
@@ -80,6 +83,7 @@ class DSAStackTy {
 private:
   struct DSAInfo {
     OpenMPClauseKind Attributes = OMPC_unknown;
+    unsigned Modifier = 0;
     /// Pointer to a reference expression and a flag which shows that the
     /// variable is marked as lastprivate(true) or not (false).
     llvm::PointerIntPair<const Expr *, 1, bool> RefExpr;
@@ -164,6 +168,8 @@ class DSAStackTy {
     /// List of globals marked as declare target link in this target region
     /// (isOpenMPTargetExecutionDirective(Directive) == true).
     llvm::SmallVector<DeclRefExpr *, 4> DeclareTargetLinkVarDecls;
+    /// List of decls used in inclusive/exclusive clauses of the scan directive.
+    llvm::DenseSet<CanonicalDeclPtr<Decl>> UsedInScanDirective;
     SharingMapTy(OpenMPDirectiveKind DKind, DeclarationNameInfo Name,
                  Scope *CurScope, SourceLocation Loc)
         : Directive(DKind), DirectiveName(Name), CurScope(CurScope),
@@ -469,9 +475,22 @@ class DSAStackTy {
   /// parent directive.
   const ValueDecl *getParentLoopControlVariable(unsigned I) const;
 
+  /// Marks the specified decl \p D as used in scan directive.
+  void markDeclAsUsedInScanDirective(ValueDecl *D) {
+    if (SharingMapTy *Stack = getSecondOnStackOrNull())
+      Stack->UsedInScanDirective.insert(D);
+  }
+
+  /// Checks if the specified declaration was used in the inner scan directive.
+  bool isUsedInScanDirective(ValueDecl *D) const {
+    if (const SharingMapTy *Stack = getTopOfStackOrNull())
+      return Stack->UsedInScanDirective.count(D) > 0;
+    return false;
+  }
+
   /// Adds explicit data sharing attribute to the specified declaration.
   void addDSA(const ValueDecl *D, const Expr *E, OpenMPClauseKind A,
-              DeclRefExpr *PrivateCopy = nullptr);
+              DeclRefExpr *PrivateCopy = nullptr, unsigned Modifier = 0);
 
   /// Adds additional information for the reduction items with the reduction id
   /// represented as an operator.
@@ -1079,6 +1098,7 @@ DSAStackTy::DSAVarData DSAStackTy::getDSA(const_iterator &Iter,
     DVar.PrivateCopy = Data.PrivateCopy;
     DVar.CKind = Data.Attributes;
     DVar.ImplicitDSALoc = Iter->DefaultAttrLoc;
+    DVar.Modifier = Data.Modifier;
     return DVar;
   }
 
@@ -1226,19 +1246,21 @@ const ValueDecl *DSAStackTy::getParentLoopControlVariable(unsigned I) const {
 }
 
 void DSAStackTy::addDSA(const ValueDecl *D, const Expr *E, OpenMPClauseKind A,
-                        DeclRefExpr *PrivateCopy) {
+                        DeclRefExpr *PrivateCopy, unsigned Modifier) {
   D = getCanonicalDecl(D);
   if (A == OMPC_threadprivate) {
     DSAInfo &Data = Threadprivates[D];
     Data.Attributes = A;
     Data.RefExpr.setPointer(E);
     Data.PrivateCopy = nullptr;
+    Data.Modifier = Modifier;
   } else {
     DSAInfo &Data = getTopOfStack().SharingMap[D];
     assert(Data.Attributes == OMPC_unknown || (A == Data.Attributes) ||
            (A == OMPC_firstprivate && Data.Attributes == OMPC_lastprivate) ||
            (A == OMPC_lastprivate && Data.Attributes == OMPC_firstprivate) ||
            (isLoopControlVariable(D).first && A == OMPC_private));
+    Data.Modifier = Modifier;
     if (A == OMPC_lastprivate && Data.Attributes == OMPC_firstprivate) {
       Data.RefExpr.setInt(/*IntVal=*/true);
       return;
@@ -1250,6 +1272,7 @@ void DSAStackTy::addDSA(const ValueDecl *D, const Expr *E, OpenMPClauseKind A,
     Data.PrivateCopy = PrivateCopy;
     if (PrivateCopy) {
       DSAInfo &Data = getTopOfStack().SharingMap[PrivateCopy->getDecl()];
+      Data.Modifier = Modifier;
       Data.Attributes = A;
       Data.RefExpr.setPointerAndInt(PrivateCopy, IsLastprivate);
       Data.PrivateCopy = nullptr;
@@ -1355,7 +1378,7 @@ const DSAStackTy::DSAVarData DSAStackTy::getTopMostTaskgroupReductionData(
                                        "set.");
     TaskgroupDescriptor = I->TaskgroupReductionRef;
     return DSAVarData(OMPD_taskgroup, OMPC_reduction, Data.RefExpr.getPointer(),
-                      Data.PrivateCopy, I->DefaultAttrLoc);
+                      Data.PrivateCopy, I->DefaultAttrLoc, /*Modifier=*/0);
   }
   return DSAVarData();
 }
@@ -1380,7 +1403,7 @@ const DSAStackTy::DSAVarData DSAStackTy::getTopMostTaskgroupReductionData(
                                        "set.");
     TaskgroupDescriptor = I->TaskgroupReductionRef;
     return DSAVarData(OMPD_taskgroup, OMPC_reduction, Data.RefExpr.getPointer(),
-                      Data.PrivateCopy, I->DefaultAttrLoc);
+                      Data.PrivateCopy, I->DefaultAttrLoc, /*Modifier=*/0);
   }
   return DSAVarData();
 }
@@ -1455,6 +1478,7 @@ const DSAStackTy::DSAVarData DSAStackTy::getTopDSA(ValueDecl *D,
   if (TI != Threadprivates.end()) {
     DVar.RefExpr = TI->getSecond().RefExpr.getPointer();
     DVar.CKind = OMPC_threadprivate;
+    DVar.Modifier = TI->getSecond().Modifier;
     return DVar;
   }
   if (VD && VD->hasAttr<OMPThreadPrivateDeclAttr>()) {
@@ -1538,15 +1562,18 @@ const DSAStackTy::DSAVarData DSAStackTy::getTopDSA(ValueDecl *D,
     const_iterator EndI = end();
     if (FromParent && I != EndI)
       ++I;
-    auto It = I->SharingMap.find(D);
-    if (It != I->SharingMap.end()) {
-      const DSAInfo &Data = It->getSecond();
-      DVar.RefExpr = Data.RefExpr.getPointer();
-      DVar.PrivateCopy = Data.PrivateCopy;
-      DVar.CKind = Data.Attributes;
-      DVar.ImplicitDSALoc = I->DefaultAttrLoc;
-      DVar.DKind = I->Directive;
-      return DVar;
+    if (I != EndI) {
+      auto It = I->SharingMap.find(D);
+      if (It != I->SharingMap.end()) {
+        const DSAInfo &Data = It->getSecond();
+        DVar.RefExpr = Data.RefExpr.getPointer();
+        DVar.PrivateCopy = Data.PrivateCopy;
+        DVar.CKind = Data.Attributes;
+        DVar.ImplicitDSALoc = I->DefaultAttrLoc;
+        DVar.DKind = I->Directive;
+        DVar.Modifier = Data.Modifier;
+        return DVar;
+      }
     }
 
     DVar.CKind = OMPC_shared;
@@ -1584,6 +1611,8 @@ const DSAStackTy::DSAVarData DSAStackTy::getTopDSA(ValueDecl *D,
   const_iterator EndI = end();
   if (FromParent && I != EndI)
     ++I;
+  if (I == EndI)
+    return DVar;
   auto It = I->SharingMap.find(D);
   if (It != I->SharingMap.end()) {
     const DSAInfo &Data = It->getSecond();
@@ -1592,6 +1621,7 @@ const DSAStackTy::DSAVarData DSAStackTy::getTopDSA(ValueDecl *D,
     DVar.CKind = Data.Attributes;
     DVar.ImplicitDSALoc = I->DefaultAttrLoc;
     DVar.DKind = I->Directive;
+    DVar.Modifier = Data.Modifier;
   }
 
   return DVar;
@@ -1915,7 +1945,8 @@ bool Sema::isOpenMPCapturedByRef(const ValueDecl *D, unsigned Level,
 
           if (isa<ArraySubscriptExpr>(EI->getAssociatedExpression()) ||
               isa<OMPArraySectionExpr>(EI->getAssociatedExpression()) ||
-              isa<MemberExpr>(EI->getAssociatedExpression())) {
+              isa<MemberExpr>(EI->getAssociatedExpression()) ||
+              isa<OMPArrayShapingExpr>(EI->getAssociatedExpression())) {
             IsVariableAssociatedWithSection = true;
             // There is nothing more we need to know about this variable.
             return true;
@@ -2244,6 +2275,22 @@ bool Sema::isOpenMPGlobalCapturedDecl(ValueDecl *D, unsigned Level,
 
 void Sema::DestroyDataSharingAttributesStack() { delete DSAStack; }
 
+void Sema::ActOnOpenMPBeginDeclareVariant(SourceLocation Loc,
+                                          OMPTraitInfo &TI) {
+  if (!OMPDeclareVariantScopes.empty()) {
+    Diag(Loc, diag::warn_nested_declare_variant);
+    return;
+  }
+  OMPDeclareVariantScopes.push_back(OMPDeclareVariantScope(TI));
+}
+
+void Sema::ActOnOpenMPEndDeclareVariant() {
+  assert(isInOpenMPDeclareVariantScope() &&
+         "Not in OpenMP declare variant scope!");
+
+  OMPDeclareVariantScopes.pop_back();
+}
+
 void Sema::finalizeOpenMPDelayedAnalysis(const FunctionDecl *Caller,
                                          const FunctionDecl *Callee,
                                          SourceLocation Loc) {
@@ -2299,11 +2346,64 @@ void Sema::EndOpenMPClause() {
   DSAStack->setClauseParsingMode(/*K=*/OMPC_unknown);
 }
 
-static void checkAllocateClauses(Sema &S, DSAStackTy *Stack,
-                                 ArrayRef<OMPClause *> Clauses);
 static std::pair<ValueDecl *, bool>
 getPrivateItem(Sema &S, Expr *&RefExpr, SourceLocation &ELoc,
                SourceRange &ERange, bool AllowArraySection = false);
+
+/// Check consistency of the reduction clauses.
+static void checkReductionClauses(Sema &S, DSAStackTy *Stack,
+                                 ArrayRef<OMPClause *> Clauses) {
+  bool InscanFound = false;
+  SourceLocation InscanLoc;
+  // OpenMP 5.0, 2.19.5.4 reduction Clause, Restrictions.
+  // A reduction clause without the inscan reduction-modifier may not appear on
+  // a construct on which a reduction clause with the inscan reduction-modifier
+  // appears.
+  for (OMPClause *C : Clauses) {
+    if (C->getClauseKind() != OMPC_reduction)
+      continue;
+    auto *RC = cast<OMPReductionClause>(C);
+    if (RC->getModifier() == OMPC_REDUCTION_inscan) {
+      InscanFound = true;
+      InscanLoc = RC->getModifierLoc();
+      break;
+    }
+  }
+  if (InscanFound) {
+    for (OMPClause *C : Clauses) {
+      if (C->getClauseKind() != OMPC_reduction)
+        continue;
+      auto *RC = cast<OMPReductionClause>(C);
+      if (RC->getModifier() != OMPC_REDUCTION_inscan) {
+        S.Diag(RC->getModifier() == OMPC_REDUCTION_unknown
+                   ? RC->getBeginLoc()
+                   : RC->getModifierLoc(),
+               diag::err_omp_inscan_reduction_expected);
+        S.Diag(InscanLoc, diag::note_omp_previous_inscan_reduction);
+        continue;
+      }
+      for (Expr *Ref : RC->varlists()) {
+        assert(Ref && "NULL expr in OpenMP nontemporal clause.");
+        SourceLocation ELoc;
+        SourceRange ERange;
+        Expr *SimpleRefExpr = Ref;
+        auto Res = getPrivateItem(S, SimpleRefExpr, ELoc, ERange,
+                                  /*AllowArraySection=*/true);
+        ValueDecl *D = Res.first;
+        if (!D)
+          continue;
+        if (!Stack->isUsedInScanDirective(getCanonicalDecl(D))) {
+          S.Diag(Ref->getExprLoc(),
+                 diag::err_omp_reduction_not_inclusive_exclusive)
+              << Ref->getSourceRange();
+        }
+      }
+    }
+  }
+}
+
+static void checkAllocateClauses(Sema &S, DSAStackTy *Stack,
+                                 ArrayRef<OMPClause *> Clauses);
 static DeclRefExpr *buildCapture(Sema &S, ValueDecl *D, Expr *CaptureExpr,
                                  bool WithInit);
 
@@ -2380,6 +2480,7 @@ void Sema::EndOpenMPDSABlock(Stmt *CurDirective) {
     // Check allocate clauses.
     if (!CurContext->isDependentContext())
       checkAllocateClauses(*this, DSAStack, D->clauses());
+    checkReductionClauses(*this, DSAStack, D->clauses());
   }
 
   DSAStack->pop();
@@ -3127,7 +3228,7 @@ class DSAAttrChecker final : public StmtVisitor<DSAAttrChecker, void> {
                        StackComponents,
                    OpenMPClauseKind) {
                   // Variable is used if it has been marked as an array, array
-                  // section or the variable iself.
+                  // section, array shaping or the variable iself.
                   return StackComponents.size() == 1 ||
                          std::all_of(
                              std::next(StackComponents.rbegin()),
@@ -3138,6 +3239,8 @@ class DSAAttrChecker final : public StmtVisitor<DSAAttrChecker, void> {
                                           nullptr &&
                                       (isa<OMPArraySectionExpr>(
                                            MC.getAssociatedExpression()) ||
+                                       isa<OMPArrayShapingExpr>(
+                                           MC.getAssociatedExpression()) ||
                                        isa<ArraySubscriptExpr>(
                                            MC.getAssociatedExpression()));
                              });
@@ -3295,8 +3398,10 @@ class DSAAttrChecker final : public StmtVisitor<DSAAttrChecker, void> {
                   // Do both expressions have the same kind?
                   if (CCI->getAssociatedExpression()->getStmtClass() !=
                       SC.getAssociatedExpression()->getStmtClass())
-                    if (!(isa<OMPArraySectionExpr>(
-                              SC.getAssociatedExpression()) &&
+                    if (!((isa<OMPArraySectionExpr>(
+                               SC.getAssociatedExpression()) ||
+                           isa<OMPArrayShapingExpr>(
+                               SC.getAssociatedExpression())) &&
                           isa<ArraySubscriptExpr>(
                               CCI->getAssociatedExpression())))
                       return false;
@@ -3788,6 +3893,8 @@ void Sema::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind, Scope *CurScope) {
   case OMPD_end_declare_target:
   case OMPD_requires:
   case OMPD_declare_variant:
+  case OMPD_begin_declare_variant:
+  case OMPD_end_declare_variant:
     llvm_unreachable("OpenMP Directive is not allowed");
   case OMPD_unknown:
     llvm_unreachable("Unknown OpenMP directive");
@@ -4177,7 +4284,7 @@ static bool checkNestingOfRegions(Sema &SemaRef, const DSAStackTy *Stack,
     if (ParentRegion == OMPD_unknown &&
         !isOpenMPNestingTeamsDirective(CurrentRegion) &&
         CurrentRegion != OMPD_cancellation_point &&
-        CurrentRegion != OMPD_cancel)
+        CurrentRegion != OMPD_cancel && CurrentRegion != OMPD_scan)
       return false;
     if (CurrentRegion == OMPD_cancellation_point ||
         CurrentRegion == OMPD_cancel) {
@@ -4298,7 +4405,8 @@ static bool checkNestingOfRegions(Sema &SemaRef, const DSAStackTy *Stack,
       NestingProhibited =
           SemaRef.LangOpts.OpenMP < 50 ||
           (ParentRegion != OMPD_simd && ParentRegion != OMPD_for &&
-           ParentRegion != OMPD_for_simd);
+           ParentRegion != OMPD_for_simd && ParentRegion != OMPD_parallel_for &&
+           ParentRegion != OMPD_parallel_for_simd);
       OrphanSeen = ParentRegion == OMPD_unknown;
       Recommend = ShouldBeInLoopSimdRegion;
     }
@@ -5014,6 +5122,8 @@ StmtResult Sema::ActOnOpenMPExecutableDirective(
   case OMPD_declare_simd:
   case OMPD_requires:
   case OMPD_declare_variant:
+  case OMPD_begin_declare_variant:
+  case OMPD_end_declare_variant:
     llvm_unreachable("OpenMP Directive is not allowed");
   case OMPD_unknown:
     llvm_unreachable("Unknown OpenMP directive");
@@ -5435,6 +5545,131 @@ static void setPrototype(Sema &S, FunctionDecl *FD, FunctionDecl *FDWithProto,
   FD->setParams(Params);
 }
 
+Sema::OMPDeclareVariantScope::OMPDeclareVariantScope(OMPTraitInfo &TI)
+    : TI(&TI), NameSuffix(TI.getMangledName()) {}
+
+FunctionDecl *
+Sema::ActOnStartOfFunctionDefinitionInOpenMPDeclareVariantScope(Scope *S,
+                                                                Declarator &D) {
+  auto *BaseFD = cast<FunctionDecl>(ActOnDeclarator(S, D));
+  OMPDeclareVariantScope &DVScope = OMPDeclareVariantScopes.back();
+  std::string MangledName;
+  MangledName += D.getIdentifier()->getName();
+  MangledName += getOpenMPVariantManglingSeparatorStr();
+  MangledName += DVScope.NameSuffix;
+  IdentifierInfo &VariantII = Context.Idents.get(MangledName);
+
+  VariantII.setMangledOpenMPVariantName(true);
+  D.SetIdentifier(&VariantII, D.getBeginLoc());
+  return BaseFD;
+}
+
+void Sema::ActOnFinishedFunctionDefinitionInOpenMPDeclareVariantScope(
+    FunctionDecl *FD, FunctionDecl *BaseFD) {
+  // Do not mark function as is used to prevent its emission if this is the
+  // only place where it is used.
+  EnterExpressionEvaluationContext Unevaluated(
+      *this, Sema::ExpressionEvaluationContext::Unevaluated);
+
+  Expr *VariantFuncRef = DeclRefExpr::Create(
+      Context, NestedNameSpecifierLoc(), SourceLocation(), FD,
+      /* RefersToEnclosingVariableOrCapture */ false,
+      /* NameLoc */ FD->getLocation(), FD->getType(), ExprValueKind::VK_RValue);
+
+  OMPDeclareVariantScope &DVScope = OMPDeclareVariantScopes.back();
+  auto *OMPDeclareVariantA = OMPDeclareVariantAttr::CreateImplicit(
+      Context, VariantFuncRef, DVScope.TI);
+  BaseFD->addAttr(OMPDeclareVariantA);
+
+  BaseFD->setImplicit(true);
+}
+
+ExprResult Sema::ActOnOpenMPCall(Sema &S, ExprResult Call, Scope *Scope,
+                                 SourceLocation LParenLoc,
+                                 MultiExprArg ArgExprs,
+                                 SourceLocation RParenLoc, Expr *ExecConfig) {
+  // The common case is a regular call we do not want to specialize at all. Try
+  // to make that case fast by bailing early.
+  CallExpr *CE = dyn_cast<CallExpr>(Call.get());
+  if (!CE)
+    return Call;
+
+  FunctionDecl *CalleeFnDecl = CE->getDirectCallee();
+  if (!CalleeFnDecl)
+    return Call;
+
+  if (!CalleeFnDecl->hasAttr<OMPDeclareVariantAttr>())
+    return Call;
+
+  ASTContext &Context = S.getASTContext();
+  OMPContext OMPCtx(S.getLangOpts().OpenMPIsDevice,
+                    Context.getTargetInfo().getTriple());
+
+  SmallVector<Expr *, 4> Exprs;
+  SmallVector<VariantMatchInfo, 4> VMIs;
+  while (CalleeFnDecl) {
+    for (OMPDeclareVariantAttr *A :
+         CalleeFnDecl->specific_attrs<OMPDeclareVariantAttr>()) {
+      Expr *VariantRef = A->getVariantFuncRef();
+
+      VariantMatchInfo VMI;
+      OMPTraitInfo &TI = A->getTraitInfo();
+      TI.getAsVariantMatchInfo(Context, VMI, /* DeviceSetOnly */ false);
+      if (!isVariantApplicableInContext(VMI, OMPCtx))
+        continue;
+
+      VMIs.push_back(VMI);
+      Exprs.push_back(VariantRef);
+    }
+
+    CalleeFnDecl = CalleeFnDecl->getPreviousDecl();
+  }
+
+  ExprResult NewCall;
+  do {
+    int BestIdx = getBestVariantMatchForContext(VMIs, OMPCtx);
+    if (BestIdx < 0)
+      return Call;
+    Expr *BestExpr = cast<DeclRefExpr>(Exprs[BestIdx]);
+    Decl *BestDecl = cast<DeclRefExpr>(BestExpr)->getDecl();
+
+    {
+      // Try to build a (member) call expression for the current best applicable
+      // variant expression. We allow this to fail in which case we continue
+      // with the next best variant expression. The fail case is part of the
+      // implementation defined behavior in the OpenMP standard when it talks
+      // about what differences in the function prototypes: "Any differences
+      // that the specific OpenMP context requires in the prototype of the
+      // variant from the base function prototype are implementation defined."
+      // This wording is there to allow the specialized variant to have a
+      // different type than the base function. This is intended and OK but if
+      // we cannot create a call the difference is not in the "implementation
+      // defined range" we allow.
+      Sema::TentativeAnalysisScope Trap(*this);
+
+      if (auto *SpecializedMethod = dyn_cast<CXXMethodDecl>(BestDecl)) {
+        auto *MemberCall = dyn_cast<CXXMemberCallExpr>(CE);
+        BestExpr = MemberExpr::CreateImplicit(
+            S.Context, MemberCall->getImplicitObjectArgument(),
+            /* IsArrow */ false, SpecializedMethod, S.Context.BoundMemberTy,
+            MemberCall->getValueKind(), MemberCall->getObjectKind());
+      }
+      NewCall = S.BuildCallExpr(Scope, BestExpr, LParenLoc, ArgExprs, RParenLoc,
+                                ExecConfig);
+      if (NewCall.isUsable())
+        break;
+    }
+
+    VMIs.erase(VMIs.begin() + BestIdx);
+    Exprs.erase(Exprs.begin() + BestIdx);
+  } while (!VMIs.empty());
+
+  if (!NewCall.isUsable())
+    return Call;
+
+  return PseudoObjectExpr::Create(Context, CE, {NewCall.get()}, 0);
+}
+
 Optional<std::pair<FunctionDecl *, Expr *>>
 Sema::checkOpenMPDeclareVariantFunction(Sema::DeclGroupPtrTy DG,
                                         Expr *VariantRef, OMPTraitInfo &TI,
@@ -5707,25 +5942,6 @@ void Sema::ActOnOpenMPDeclareVariantDirective(FunctionDecl *FD,
   FD->addAttr(NewAttr);
 }
 
-void Sema::markOpenMPDeclareVariantFuncsReferenced(SourceLocation Loc,
-                                                   FunctionDecl *Func,
-                                                   bool MightBeOdrUse) {
-  assert(LangOpts.OpenMP && "Expected OpenMP mode.");
-
-  if (!Func->isDependentContext() && Func->hasAttrs()) {
-    for (OMPDeclareVariantAttr *A :
-         Func->specific_attrs<OMPDeclareVariantAttr>()) {
-      // TODO: add checks for active OpenMP context where possible.
-      Expr *VariantRef = A->getVariantFuncRef();
-      auto *DRE = cast<DeclRefExpr>(VariantRef->IgnoreParenImpCasts());
-      auto *F = cast<FunctionDecl>(DRE->getDecl());
-      if (!F->isDefined() && F->isTemplateInstantiation())
-        InstantiateFunctionDefinition(Loc, F->getFirstDecl());
-      MarkFunctionReferenced(Loc, F, MightBeOdrUse);
-    }
-  }
-}
-
 StmtResult Sema::ActOnOpenMPParallelDirective(ArrayRef<OMPClause *> Clauses,
                                               Stmt *AStmt,
                                               SourceLocation StartLoc,
@@ -11221,6 +11437,8 @@ static OpenMPDirectiveKind getOpenMPCaptureRegionForClause(
     case OMPD_declare_mapper:
     case OMPD_declare_simd:
     case OMPD_declare_variant:
+    case OMPD_begin_declare_variant:
+    case OMPD_end_declare_variant:
     case OMPD_declare_target:
     case OMPD_end_declare_target:
     case OMPD_teams:
@@ -11293,6 +11511,8 @@ static OpenMPDirectiveKind getOpenMPCaptureRegionForClause(
     case OMPD_declare_mapper:
     case OMPD_declare_simd:
     case OMPD_declare_variant:
+    case OMPD_begin_declare_variant:
+    case OMPD_end_declare_variant:
     case OMPD_declare_target:
     case OMPD_end_declare_target:
     case OMPD_teams:
@@ -11370,6 +11590,8 @@ static OpenMPDirectiveKind getOpenMPCaptureRegionForClause(
     case OMPD_declare_mapper:
     case OMPD_declare_simd:
     case OMPD_declare_variant:
+    case OMPD_begin_declare_variant:
+    case OMPD_end_declare_variant:
     case OMPD_declare_target:
     case OMPD_end_declare_target:
     case OMPD_simd:
@@ -11444,6 +11666,8 @@ static OpenMPDirectiveKind getOpenMPCaptureRegionForClause(
     case OMPD_declare_mapper:
     case OMPD_declare_simd:
     case OMPD_declare_variant:
+    case OMPD_begin_declare_variant:
+    case OMPD_end_declare_variant:
     case OMPD_declare_target:
     case OMPD_end_declare_target:
     case OMPD_simd:
@@ -11519,6 +11743,8 @@ static OpenMPDirectiveKind getOpenMPCaptureRegionForClause(
     case OMPD_declare_mapper:
     case OMPD_declare_simd:
     case OMPD_declare_variant:
+    case OMPD_begin_declare_variant:
+    case OMPD_end_declare_variant:
     case OMPD_declare_target:
     case OMPD_end_declare_target:
     case OMPD_simd:
@@ -11593,6 +11819,8 @@ static OpenMPDirectiveKind getOpenMPCaptureRegionForClause(
     case OMPD_declare_mapper:
     case OMPD_declare_simd:
     case OMPD_declare_variant:
+    case OMPD_begin_declare_variant:
+    case OMPD_end_declare_variant:
     case OMPD_declare_target:
     case OMPD_end_declare_target:
     case OMPD_simd:
@@ -11666,6 +11894,8 @@ static OpenMPDirectiveKind getOpenMPCaptureRegionForClause(
     case OMPD_declare_mapper:
     case OMPD_declare_simd:
     case OMPD_declare_variant:
+    case OMPD_begin_declare_variant:
+    case OMPD_end_declare_variant:
     case OMPD_declare_target:
     case OMPD_end_declare_target:
     case OMPD_simd:
@@ -11742,6 +11972,8 @@ static OpenMPDirectiveKind getOpenMPCaptureRegionForClause(
     case OMPD_declare_mapper:
     case OMPD_declare_simd:
     case OMPD_declare_variant:
+    case OMPD_begin_declare_variant:
+    case OMPD_end_declare_variant:
     case OMPD_declare_target:
     case OMPD_end_declare_target:
     case OMPD_simd:
@@ -12826,7 +13058,7 @@ OMPClause *Sema::ActOnOpenMPDestroyClause(SourceLocation StartLoc,
 }
 
 OMPClause *Sema::ActOnOpenMPVarListClause(
-    OpenMPClauseKind Kind, ArrayRef<Expr *> VarList, Expr *TailExpr,
+    OpenMPClauseKind Kind, ArrayRef<Expr *> VarList, Expr *DepModOrTailExpr,
     const OMPVarListLocTy &Locs, SourceLocation ColonLoc,
     CXXScopeSpec &ReductionOrMapperIdScopeSpec,
     DeclarationNameInfo &ReductionOrMapperId, int ExtraModifier,
@@ -12876,13 +13108,13 @@ OMPClause *Sema::ActOnOpenMPVarListClause(
     assert(0 <= ExtraModifier && ExtraModifier <= OMPC_LINEAR_unknown &&
            "Unexpected linear modifier.");
     Res = ActOnOpenMPLinearClause(
-        VarList, TailExpr, StartLoc, LParenLoc,
+        VarList, DepModOrTailExpr, StartLoc, LParenLoc,
         static_cast<OpenMPLinearClauseKind>(ExtraModifier), ExtraModifierLoc,
         ColonLoc, EndLoc);
     break;
   case OMPC_aligned:
-    Res = ActOnOpenMPAlignedClause(VarList, TailExpr, StartLoc, LParenLoc,
-                                   ColonLoc, EndLoc);
+    Res = ActOnOpenMPAlignedClause(VarList, DepModOrTailExpr, StartLoc,
+                                   LParenLoc, ColonLoc, EndLoc);
     break;
   case OMPC_copyin:
     Res = ActOnOpenMPCopyinClause(VarList, StartLoc, LParenLoc, EndLoc);
@@ -12897,8 +13129,8 @@ OMPClause *Sema::ActOnOpenMPVarListClause(
     assert(0 <= ExtraModifier && ExtraModifier <= OMPC_DEPEND_unknown &&
            "Unexpected depend modifier.");
     Res = ActOnOpenMPDependClause(
-        static_cast<OpenMPDependClauseKind>(ExtraModifier), ExtraModifierLoc,
-        ColonLoc, VarList, StartLoc, LParenLoc, EndLoc);
+        DepModOrTailExpr, static_cast<OpenMPDependClauseKind>(ExtraModifier),
+        ExtraModifierLoc, ColonLoc, VarList, StartLoc, LParenLoc, EndLoc);
     break;
   case OMPC_map:
     assert(0 <= ExtraModifier && ExtraModifier <= OMPC_MAP_unknown &&
@@ -12923,8 +13155,8 @@ OMPClause *Sema::ActOnOpenMPVarListClause(
     Res = ActOnOpenMPIsDevicePtrClause(VarList, Locs);
     break;
   case OMPC_allocate:
-    Res = ActOnOpenMPAllocateClause(TailExpr, VarList, StartLoc, LParenLoc,
-                                    ColonLoc, EndLoc);
+    Res = ActOnOpenMPAllocateClause(DepModOrTailExpr, VarList, StartLoc,
+                                    LParenLoc, ColonLoc, EndLoc);
     break;
   case OMPC_nontemporal:
     Res = ActOnOpenMPNontemporalClause(VarList, StartLoc, LParenLoc, EndLoc);
@@ -13971,9 +14203,11 @@ struct ReductionData {
   SmallVector<Decl *, 4> ExprCaptures;
   /// List of postupdate expressions.
   SmallVector<Expr *, 4> ExprPostUpdates;
+  /// Reduction modifier.
+  unsigned RedModifier = 0;
   ReductionData() = delete;
   /// Reserves required memory for the reduction data.
-  ReductionData(unsigned Size) {
+  ReductionData(unsigned Size, unsigned Modifier = 0) : RedModifier(Modifier) {
     Vars.reserve(Size);
     Privates.reserve(Size);
     LHSs.reserve(Size);
@@ -14613,8 +14847,8 @@ static bool actOnOMPReductionKindClause(
     if (ClauseKind == OMPC_in_reduction) {
       SourceRange ParentSR;
       BinaryOperatorKind ParentBOK;
-      const Expr *ParentReductionOp;
-      Expr *ParentBOKTD, *ParentReductionOpTD;
+      const Expr *ParentReductionOp = nullptr;
+      Expr *ParentBOKTD = nullptr, *ParentReductionOpTD = nullptr;
       DSAStackTy::DSAVarData ParentBOKDSA =
           Stack->getTopMostTaskgroupReductionData(D, ParentSR, ParentBOK,
                                                   ParentBOKTD);
@@ -14623,13 +14857,9 @@ static bool actOnOMPReductionKindClause(
               D, ParentSR, ParentReductionOp, ParentReductionOpTD);
       bool IsParentBOK = ParentBOKDSA.DKind != OMPD_unknown;
       bool IsParentReductionOp = ParentReductionOpDSA.DKind != OMPD_unknown;
-      if (!IsParentBOK && !IsParentReductionOp) {
-        S.Diag(ELoc, diag::err_omp_in_reduction_not_task_reduction);
-        continue;
-      }
       if ((DeclareReductionRef.isUnset() && IsParentReductionOp) ||
-          (DeclareReductionRef.isUsable() && IsParentBOK) || BOK != ParentBOK ||
-          IsParentReductionOp) {
+          (DeclareReductionRef.isUsable() && IsParentBOK) ||
+          (IsParentBOK && BOK != ParentBOK) || IsParentReductionOp) {
         bool EmitError = true;
         if (IsParentReductionOp && DeclareReductionRef.isUsable()) {
           llvm::FoldingSetNodeID RedId, ParentRedId;
@@ -14652,7 +14882,6 @@ static bool actOnOMPReductionKindClause(
         }
       }
       TaskgroupDescriptor = IsParentBOK ? ParentBOKTD : ParentReductionOpTD;
-      assert(TaskgroupDescriptor && "Taskgroup descriptor must be defined.");
     }
 
     DeclRefExpr *Ref = nullptr;
@@ -14691,7 +14920,8 @@ static bool actOnOMPReductionKindClause(
     }
     // All reduction items are still marked as reduction (to do not increase
     // code base size).
-    Stack->addDSA(D, RefExpr->IgnoreParens(), OMPC_reduction, Ref);
+    Stack->addDSA(D, RefExpr->IgnoreParens(), OMPC_reduction, Ref,
+                  RD.RedModifier);
     if (CurrDir == OMPD_taskgroup) {
       if (DeclareReductionRef.isUsable())
         Stack->addTaskgroupReductionData(D, ReductionIdRange,
@@ -14718,8 +14948,22 @@ OMPClause *Sema::ActOnOpenMPReductionClause(
         << getOpenMPClauseName(OMPC_reduction);
     return nullptr;
   }
+  // OpenMP 5.0, 2.19.5.4 reduction Clause, Restrictions
+  // A reduction clause with the inscan reduction-modifier may only appear on a
+  // worksharing-loop construct, a worksharing-loop SIMD construct, a simd
+  // construct, a parallel worksharing-loop construct or a parallel
+  // worksharing-loop SIMD construct.
+  if (Modifier == OMPC_REDUCTION_inscan &&
+      (DSAStack->getCurrentDirective() != OMPD_for &&
+       DSAStack->getCurrentDirective() != OMPD_for_simd &&
+       DSAStack->getCurrentDirective() != OMPD_simd &&
+       DSAStack->getCurrentDirective() != OMPD_parallel_for &&
+       DSAStack->getCurrentDirective() != OMPD_parallel_for_simd)) {
+    Diag(ModifierLoc, diag::err_omp_wrong_inscan_reduction);
+    return nullptr;
+  }
 
-  ReductionData RD(VarList.size());
+  ReductionData RD(VarList.size(), Modifier);
   if (actOnOMPReductionKindClause(*this, DSAStack, OMPC_reduction, VarList,
                                   StartLoc, LParenLoc, ColonLoc, EndLoc,
                                   ReductionIdScopeSpec, ReductionId,
@@ -15403,7 +15647,7 @@ OMPClause *Sema::ActOnOpenMPDepobjClause(Expr *Depobj, SourceLocation StartLoc,
 }
 
 OMPClause *
-Sema::ActOnOpenMPDependClause(OpenMPDependClauseKind DepKind,
+Sema::ActOnOpenMPDependClause(Expr *DepModifier, OpenMPDependClauseKind DepKind,
                               SourceLocation DepLoc, SourceLocation ColonLoc,
                               ArrayRef<Expr *> VarList, SourceLocation StartLoc,
                               SourceLocation LParenLoc, SourceLocation EndLoc) {
@@ -15425,12 +15669,26 @@ Sema::ActOnOpenMPDependClause(OpenMPDependClauseKind DepKind,
     Except.push_back(OMPC_DEPEND_sink);
     if (LangOpts.OpenMP < 50 || DSAStack->getCurrentDirective() == OMPD_depobj)
       Except.push_back(OMPC_DEPEND_depobj);
+    std::string Expected = (LangOpts.OpenMP >= 50 && !DepModifier)
+                               ? "depend modifier(iterator) or "
+                               : "";
     Diag(DepLoc, diag::err_omp_unexpected_clause_value)
-        << getListOfPossibleValues(OMPC_depend, /*First=*/0,
-                                   /*Last=*/OMPC_DEPEND_unknown, Except)
+        << Expected + getListOfPossibleValues(OMPC_depend, /*First=*/0,
+                                              /*Last=*/OMPC_DEPEND_unknown,
+                                              Except)
         << getOpenMPClauseName(OMPC_depend);
     return nullptr;
   }
+  if (DepModifier &&
+      (DepKind == OMPC_DEPEND_source || DepKind == OMPC_DEPEND_sink)) {
+    Diag(DepModifier->getExprLoc(),
+         diag::err_omp_depend_sink_source_with_modifier);
+    return nullptr;
+  }
+  if (DepModifier &&
+      !DepModifier->getType()->isSpecificBuiltinType(BuiltinType::OMPIterator))
+    Diag(DepModifier->getExprLoc(), diag::err_omp_depend_modifier_not_iterator);
+
   SmallVector<Expr *, 8> Vars;
   DSAStackTy::OperatorOffsetTy OpsOffs;
   llvm::APSInt DepCounter(/*BitWidth=*/32);
@@ -15590,7 +15848,8 @@ Sema::ActOnOpenMPDependClause(OpenMPDependClauseKind DepKind,
             (OMPDependTFound &&
              DSAStack->getOMPDependT().getTypePtr() == ExprTy.getTypePtr())) {
           Diag(ELoc, diag::err_omp_expected_addressable_lvalue_or_array_item)
-              << 1 << RefExpr->getSourceRange();
+              << (LangOpts.OpenMP >= 50 ? 1 : 0) << 1
+              << RefExpr->getSourceRange();
           continue;
         }
 
@@ -15603,6 +15862,7 @@ Sema::ActOnOpenMPDependClause(OpenMPDependClauseKind DepKind,
                   ->isPointerType() &&
              !ASE->getBase()->getType().getNonReferenceType()->isArrayType())) {
           Diag(ELoc, diag::err_omp_expected_addressable_lvalue_or_array_item)
+              << (LangOpts.OpenMP >= 50 ? 1 : 0)
               << (LangOpts.OpenMP >= 50 ? 1 : 0) << RefExpr->getSourceRange();
           continue;
         }
@@ -15613,8 +15873,10 @@ Sema::ActOnOpenMPDependClause(OpenMPDependClauseKind DepKind,
           Res = CreateBuiltinUnaryOp(ELoc, UO_AddrOf,
                                      RefExpr->IgnoreParenImpCasts());
         }
-        if (!Res.isUsable() && !isa<OMPArraySectionExpr>(SimpleExpr)) {
+        if (!Res.isUsable() && !isa<OMPArraySectionExpr>(SimpleExpr) &&
+            !isa<OMPArrayShapingExpr>(SimpleExpr)) {
           Diag(ELoc, diag::err_omp_expected_addressable_lvalue_or_array_item)
+              << (LangOpts.OpenMP >= 50 ? 1 : 0)
               << (LangOpts.OpenMP >= 50 ? 1 : 0) << RefExpr->getSourceRange();
           continue;
         }
@@ -15635,8 +15897,8 @@ Sema::ActOnOpenMPDependClause(OpenMPDependClauseKind DepKind,
     return nullptr;
 
   auto *C = OMPDependClause::Create(Context, StartLoc, LParenLoc, EndLoc,
-                                    DepKind, DepLoc, ColonLoc, Vars,
-                                    TotalDepCount.getZExtValue());
+                                    DepModifier, DepKind, DepLoc, ColonLoc,
+                                    Vars, TotalDepCount.getZExtValue());
   if ((DepKind == OMPC_DEPEND_sink || DepKind == OMPC_DEPEND_source) &&
       DSAStack->isParentOrderedRegion())
     DSAStack->addDoacrossDependClause(C, OpsOffs);
@@ -16046,6 +16308,15 @@ class MapBaseChecker final : public StmtVisitor<MapBaseChecker, bool> {
     Components.emplace_back(OASE, nullptr);
     return RelevantExpr || Visit(E);
   }
+  bool VisitOMPArrayShapingExpr(OMPArrayShapingExpr *E) {
+    Expr *Base = E->getBase();
+
+    // Record the component - we don't have any declaration associated.
+    Components.emplace_back(E, nullptr);
+
+    return Visit(Base->IgnoreParenImpCasts());
+  }
+
   bool VisitUnaryOperator(UnaryOperator *UO) {
     if (SemaRef.getLangOpts().OpenMP < 50 || !UO->isLValue() ||
         UO->getOpcode() != UO_Deref) {
@@ -16171,9 +16442,11 @@ static bool checkMapConflicts(
           //  variable in map clauses of the same construct.
           if (CurrentRegionOnly &&
               (isa<ArraySubscriptExpr>(CI->getAssociatedExpression()) ||
-               isa<OMPArraySectionExpr>(CI->getAssociatedExpression())) &&
+               isa<OMPArraySectionExpr>(CI->getAssociatedExpression()) ||
+               isa<OMPArrayShapingExpr>(CI->getAssociatedExpression())) &&
               (isa<ArraySubscriptExpr>(SI->getAssociatedExpression()) ||
-               isa<OMPArraySectionExpr>(SI->getAssociatedExpression()))) {
+               isa<OMPArraySectionExpr>(SI->getAssociatedExpression()) ||
+               isa<OMPArrayShapingExpr>(SI->getAssociatedExpression()))) {
             SemaRef.Diag(CI->getAssociatedExpression()->getExprLoc(),
                          diag::err_omp_multiple_array_items_in_map_clause)
                 << CI->getAssociatedExpression()->getSourceRange();
@@ -16205,6 +16478,9 @@ static bool checkMapConflicts(
             const Expr *E = OASE->getBase()->IgnoreParenImpCasts();
             Type =
                 OMPArraySectionExpr::getBaseOriginalType(E).getCanonicalType();
+          } else if (const auto *OASE = dyn_cast<OMPArrayShapingExpr>(
+                         SI->getAssociatedExpression())) {
+            Type = OASE->getBase()->getType()->getPointeeType();
           }
           if (Type.isNull() || Type->isAnyPointerType() ||
               checkArrayExpressionDoesNotReferToWholeSize(
@@ -16667,6 +16943,7 @@ static void checkMappableExpressionList(
     QualType Type;
     auto *ASE = dyn_cast<ArraySubscriptExpr>(VE->IgnoreParens());
     auto *OASE = dyn_cast<OMPArraySectionExpr>(VE->IgnoreParens());
+    auto *OAShE = dyn_cast<OMPArrayShapingExpr>(VE->IgnoreParens());
     if (ASE) {
       Type = ASE->getType().getNonReferenceType();
     } else if (OASE) {
@@ -16677,6 +16954,8 @@ static void checkMappableExpressionList(
       else
         Type = BaseType->getPointeeType();
       Type = Type.getNonReferenceType();
+    } else if (OAShE) {
+      Type = OAShE->getBase()->getType()->getPointeeType();
     } else {
       Type = VE->getType();
     }
@@ -16780,7 +17059,7 @@ OMPClause *Sema::ActOnOpenMPMapClause(
   OpenMPMapModifierKind Modifiers[] = {OMPC_MAP_MODIFIER_unknown,
                                        OMPC_MAP_MODIFIER_unknown,
                                        OMPC_MAP_MODIFIER_unknown};
-  SourceLocation ModifiersLoc[OMPMapClause::NumberOfModifiers];
+  SourceLocation ModifiersLoc[NumberOfOMPMapClauseModifiers];
 
   // Process map-type-modifiers, flag errors for duplicate modifiers.
   unsigned Count = 0;
@@ -16790,7 +17069,7 @@ OMPClause *Sema::ActOnOpenMPMapClause(
       Diag(MapTypeModifiersLoc[I], diag::err_omp_duplicate_map_type_modifier);
       continue;
     }
-    assert(Count < OMPMapClause::NumberOfModifiers &&
+    assert(Count < NumberOfOMPMapClauseModifiers &&
            "Modifiers exceed the allowed number of map type modifiers");
     Modifiers[Count] = MapTypeModifiers[I];
     ModifiersLoc[Count] = MapTypeModifiersLoc[I];
@@ -18021,6 +18300,19 @@ OMPClause *Sema::ActOnOpenMPInclusiveClause(ArrayRef<Expr *> VarList,
     if (!D)
       continue;
 
+    const DSAStackTy::DSAVarData DVar =
+        DSAStack->getTopDSA(D, /*FromParent=*/true);
+    // OpenMP 5.0, 2.9.6, scan Directive, Restrictions.
+    // A list item that appears in the inclusive or exclusive clause must appear
+    // in a reduction clause with the inscan modifier on the enclosing
+    // worksharing-loop, worksharing-loop SIMD, or simd construct.
+    if (DVar.CKind != OMPC_reduction ||
+        DVar.Modifier != OMPC_REDUCTION_inscan)
+      Diag(ELoc, diag::err_omp_inclusive_exclusive_not_reduction)
+          << RefExpr->getSourceRange();
+
+    if (DSAStack->getParentDirective() != OMPD_unknown)
+      DSAStack->markDeclAsUsedInScanDirective(D);
     Vars.push_back(RefExpr);
   }
 
@@ -18049,6 +18341,21 @@ OMPClause *Sema::ActOnOpenMPExclusiveClause(ArrayRef<Expr *> VarList,
     if (!D)
       continue;
 
+    OpenMPDirectiveKind ParentDirective = DSAStack->getParentDirective();
+    DSAStackTy::DSAVarData DVar;
+    if (ParentDirective != OMPD_unknown)
+      DVar = DSAStack->getTopDSA(D, /*FromParent=*/true);
+    // OpenMP 5.0, 2.9.6, scan Directive, Restrictions.
+    // A list item that appears in the inclusive or exclusive clause must appear
+    // in a reduction clause with the inscan modifier on the enclosing
+    // worksharing-loop, worksharing-loop SIMD, or simd construct.
+    if (ParentDirective == OMPD_unknown || DVar.CKind != OMPC_reduction ||
+        DVar.Modifier != OMPC_REDUCTION_inscan) {
+      Diag(ELoc, diag::err_omp_inclusive_exclusive_not_reduction)
+          << RefExpr->getSourceRange();
+    } else {
+      DSAStack->markDeclAsUsedInScanDirective(D);
+    }
     Vars.push_back(RefExpr);
   }
 
diff --git a/clang/lib/Sema/SemaPseudoObject.cpp b/clang/lib/Sema/SemaPseudoObject.cpp
index 87c3c264b472e..6a541f9693ec2 100644
--- a/clang/lib/Sema/SemaPseudoObject.cpp
+++ b/clang/lib/Sema/SemaPseudoObject.cpp
@@ -1663,6 +1663,8 @@ Expr *Sema::recreateSyntacticForm(PseudoObjectExpr *E) {
                                         bop->getType(), bop->getValueKind(),
                                         bop->getObjectKind(),
                                         bop->getOperatorLoc(), FPOptions());
+  } else if (isa<CallExpr>(syntax)) {
+    return syntax;
   } else {
     assert(syntax->hasPlaceholderType(BuiltinType::PseudoObject));
     return stripOpaqueValuesFromPseudoObjectRef(*this, syntax);
diff --git a/clang/lib/Sema/SemaStmt.cpp b/clang/lib/Sema/SemaStmt.cpp
index 72b8b4765aa64..b2179c78176e5 100644
--- a/clang/lib/Sema/SemaStmt.cpp
+++ b/clang/lib/Sema/SemaStmt.cpp
@@ -334,6 +334,11 @@ void Sema::DiagnoseUnusedExprResult(const Stmt *S) {
     }
   } else if (const PseudoObjectExpr *POE = dyn_cast<PseudoObjectExpr>(E)) {
     const Expr *Source = POE->getSyntacticForm();
+    // Handle the actually selected call of an OpenMP specialized call.
+    if (LangOpts.OpenMP && isa<CallExpr>(Source) &&
+        POE->getNumSemanticExprs() == 1 &&
+        isa<CallExpr>(POE->getSemanticExpr(0)))
+      return DiagnoseUnusedExprResult(POE->getSemanticExpr(0));
     if (isa<ObjCSubscriptRefExpr>(Source))
       DiagID = diag::warn_unused_container_subscript_expr;
     else
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index f95b16c1cbaed..342262522e615 100755
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -205,7 +205,8 @@ TemplateNameKind Sema::isTemplateName(Scope *S,
   LookupResult R(*this, TName, Name.getBeginLoc(), LookupOrdinaryName);
   if (LookupTemplateName(R, S, SS, ObjectType, EnteringContext,
                          MemberOfUnknownSpecialization, SourceLocation(),
-                         &AssumedTemplate, Disambiguation))
+                         &AssumedTemplate,
+                         /*AllowTypoCorrection=*/!Disambiguation))
     return TNK_Non_template;
 
   if (AssumedTemplate != AssumedTemplateKind::None) {
@@ -371,12 +372,15 @@ bool Sema::LookupTemplateName(LookupResult &Found,
                               QualType ObjectType,
                               bool EnteringContext,
                               bool &MemberOfUnknownSpecialization,
-                              SourceLocation TemplateKWLoc,
+                              RequiredTemplateKind RequiredTemplate,
                               AssumedTemplateKind *ATK,
-                              bool Disambiguation) {
+                              bool AllowTypoCorrection) {
   if (ATK)
     *ATK = AssumedTemplateKind::None;
 
+  if (SS.isInvalid())
+    return true;
+
   Found.setTemplateNameLookup(true);
 
   // Determine where to perform name lookup
@@ -386,7 +390,7 @@ bool Sema::LookupTemplateName(LookupResult &Found,
   if (!ObjectType.isNull()) {
     // This nested-name-specifier occurs in a member access expression, e.g.,
     // x->B::f, and we are looking into the type of the object.
-    assert(!SS.isSet() && "ObjectType and scope specifier cannot coexist");
+    assert(SS.isEmpty() && "ObjectType and scope specifier cannot coexist");
     LookupCtx = computeDeclContext(ObjectType);
     IsDependent = !LookupCtx && ObjectType->isDependentType();
     assert((IsDependent || !ObjectType->isIncompleteType() ||
@@ -412,11 +416,11 @@ bool Sema::LookupTemplateName(LookupResult &Found,
       Found.clear();
       return false;
     }
-  } else if (SS.isSet()) {
+  } else if (SS.isNotEmpty()) {
     // This nested-name-specifier occurs after another nested-name-specifier,
     // so long into the context associated with the prior nested-name-specifier.
     LookupCtx = computeDeclContext(SS, EnteringContext);
-    IsDependent = !LookupCtx;
+    IsDependent = !LookupCtx && isDependentScopeSpecifier(SS);
 
     // The declaration context must be complete.
     if (LookupCtx && RequireCompleteDeclContext(SS, LookupCtx))
@@ -443,7 +447,7 @@ bool Sema::LookupTemplateName(LookupResult &Found,
     IsDependent |= Found.wasNotFoundInCurrentInstantiation();
   }
 
-  if (!SS.isSet() && (ObjectType.isNull() || Found.empty())) {
+  if (SS.isEmpty() && (ObjectType.isNull() || Found.empty())) {
     // C++ [basic.lookup.classref]p1:
     //   In a class member access expression (5.2.5), if the . or -> token is
     //   immediately followed by an identifier followed by a <, the
@@ -470,7 +474,8 @@ bool Sema::LookupTemplateName(LookupResult &Found,
   if (Found.isAmbiguous())
     return false;
 
-  if (ATK && !SS.isSet() && ObjectType.isNull() && TemplateKWLoc.isInvalid()) {
+  if (ATK && SS.isEmpty() && ObjectType.isNull() &&
+      !RequiredTemplate.hasTemplateKeyword()) {
     // C++2a [temp.names]p2:
     //   A name is also considered to refer to a template if it is an
     //   unqualified-id followed by a < and name lookup finds either one or more
@@ -496,7 +501,7 @@ bool Sema::LookupTemplateName(LookupResult &Found,
     }
   }
 
-  if (Found.empty() && !IsDependent && !Disambiguation) {
+  if (Found.empty() && !IsDependent && AllowTypoCorrection) {
     // If we did not find any names, and this is not a disambiguation, attempt
     // to correct any typos.
     DeclarationName Name = Found.getLookupName();
@@ -542,9 +547,11 @@ bool Sema::LookupTemplateName(LookupResult &Found,
 
     // If a 'template' keyword was used, a lookup that finds only non-template
     // names is an error.
-    if (ExampleLookupResult && TemplateKWLoc.isValid()) {
+    if (ExampleLookupResult && RequiredTemplate) {
       Diag(Found.getNameLoc(), diag::err_template_kw_refers_to_non_template)
-        << Found.getLookupName() << SS.getRange();
+          << Found.getLookupName() << SS.getRange()
+          << RequiredTemplate.hasTemplateKeyword()
+          << RequiredTemplate.getTemplateKeywordLoc();
       Diag(ExampleLookupResult->getUnderlyingDecl()->getLocation(),
            diag::note_template_kw_refers_to_non_template)
           << Found.getLookupName();
@@ -3470,6 +3477,10 @@ QualType Sema::CheckTemplateIdType(TemplateName Name,
                                                           DTN->getIdentifier(),
                                                           TemplateArgs);
 
+  if (Name.getAsAssumedTemplateName() &&
+      resolveAssumedTemplateNameAsType(/*Scope*/nullptr, Name, TemplateLoc))
+    return QualType();
+
   TemplateDecl *Template = Name.getAsTemplateDecl();
   if (!Template || isa<FunctionTemplateDecl>(Template) ||
       isa<VarTemplateDecl>(Template) || isa<ConceptDecl>(Template)) {
@@ -4623,21 +4634,28 @@ Sema::BuildQualifiedTemplateIdExpr(CXXScopeSpec &SS,
   return BuildTemplateIdExpr(SS, TemplateKWLoc, R, /*ADL*/ false, TemplateArgs);
 }
 
-/// Form a dependent template name.
+/// Form a template name from a name that is syntactically required to name a
+/// template, either due to use of the 'template' keyword or because a name in
+/// this syntactic context is assumed to name a template (C++ [temp.names]p2-4).
+///
+/// This action forms a template name given the name of the template and its
+/// optional scope specifier. This is used when the 'template' keyword is used
+/// or when the parsing context unambiguously treats a following '<' as
+/// introducing a template argument list. Note that this may produce a
+/// non-dependent template name if we can perform the lookup now and identify
+/// the named template.
 ///
-/// This action forms a dependent template name given the template
-/// name and its (presumably dependent) scope specifier. For
-/// example, given "MetaFun::template apply", the scope specifier \p
-/// SS will be "MetaFun::", \p TemplateKWLoc contains the location
+/// For example, given "x.MetaFun::template apply", the scope specifier
+/// \p SS will be "MetaFun::", \p TemplateKWLoc contains the location
 /// of the "template" keyword, and "apply" is the \p Name.
-TemplateNameKind Sema::ActOnDependentTemplateName(Scope *S,
-                                                  CXXScopeSpec &SS,
-                                                  SourceLocation TemplateKWLoc,
-                                                  const UnqualifiedId &Name,
-                                                  ParsedType ObjectType,
-                                                  bool EnteringContext,
-                                                  TemplateTy &Result,
-                                                  bool AllowInjectedClassName) {
+TemplateNameKind Sema::ActOnTemplateName(Scope *S,
+                                         CXXScopeSpec &SS,
+                                         SourceLocation TemplateKWLoc,
+                                         const UnqualifiedId &Name,
+                                         ParsedType ObjectType,
+                                         bool EnteringContext,
+                                         TemplateTy &Result,
+                                         bool AllowInjectedClassName) {
   if (TemplateKWLoc.isValid() && S && !S->getTemplateParamParent())
     Diag(TemplateKWLoc,
          getLangOpts().CPlusPlus11 ?
@@ -4645,95 +4663,115 @@ TemplateNameKind Sema::ActOnDependentTemplateName(Scope *S,
            diag::ext_template_outside_of_template)
       << FixItHint::CreateRemoval(TemplateKWLoc);
 
+  if (SS.isInvalid())
+    return TNK_Non_template;
+
+  // Figure out where isTemplateName is going to look.
   DeclContext *LookupCtx = nullptr;
-  if (SS.isSet())
+  if (SS.isNotEmpty())
     LookupCtx = computeDeclContext(SS, EnteringContext);
-  if (!LookupCtx && ObjectType)
-    LookupCtx = computeDeclContext(ObjectType.get());
-  if (LookupCtx) {
-    // C++0x [temp.names]p5:
-    //   If a name prefixed by the keyword template is not the name of
-    //   a template, the program is ill-formed. [Note: the keyword
-    //   template may not be applied to non-template members of class
-    //   templates. -end note ] [ Note: as is the case with the
-    //   typename prefix, the template prefix is allowed in cases
-    //   where it is not strictly necessary; i.e., when the
-    //   nested-name-specifier or the expression on the left of the ->
-    //   or . is not dependent on a template-parameter, or the use
-    //   does not appear in the scope of a template. -end note]
-    //
-    // Note: C++03 was more strict here, because it banned the use of
-    // the "template" keyword prior to a template-name that was not a
-    // dependent name. C++ DR468 relaxed this requirement (the
-    // "template" keyword is now permitted). We follow the C++0x
-    // rules, even in C++03 mode with a warning, retroactively applying the DR.
-    bool MemberOfUnknownSpecialization;
-    TemplateNameKind TNK = isTemplateName(S, SS, TemplateKWLoc.isValid(), Name,
-                                          ObjectType, EnteringContext, Result,
-                                          MemberOfUnknownSpecialization);
-    if (TNK == TNK_Non_template && MemberOfUnknownSpecialization) {
-      // This is a dependent template. Handle it below.
-    } else if (TNK == TNK_Non_template) {
-      // Do the lookup again to determine if this is a "nothing found" case or
-      // a "not a template" case. FIXME: Refactor isTemplateName so we don't
-      // need to do this.
-      DeclarationNameInfo DNI = GetNameFromUnqualifiedId(Name);
-      LookupResult R(*this, DNI.getName(), Name.getBeginLoc(),
-                     LookupOrdinaryName);
-      bool MOUS;
-      if (!LookupTemplateName(R, S, SS, ObjectType.get(), EnteringContext,
-                              MOUS, TemplateKWLoc) && !R.isAmbiguous())
+  else if (ObjectType)
+    LookupCtx = computeDeclContext(GetTypeFromParser(ObjectType));
+
+  // C++0x [temp.names]p5:
+  //   If a name prefixed by the keyword template is not the name of
+  //   a template, the program is ill-formed. [Note: the keyword
+  //   template may not be applied to non-template members of class
+  //   templates. -end note ] [ Note: as is the case with the
+  //   typename prefix, the template prefix is allowed in cases
+  //   where it is not strictly necessary; i.e., when the
+  //   nested-name-specifier or the expression on the left of the ->
+  //   or . is not dependent on a template-parameter, or the use
+  //   does not appear in the scope of a template. -end note]
+  //
+  // Note: C++03 was more strict here, because it banned the use of
+  // the "template" keyword prior to a template-name that was not a
+  // dependent name. C++ DR468 relaxed this requirement (the
+  // "template" keyword is now permitted). We follow the C++0x
+  // rules, even in C++03 mode with a warning, retroactively applying the DR.
+  bool MemberOfUnknownSpecialization;
+  TemplateNameKind TNK = isTemplateName(S, SS, TemplateKWLoc.isValid(), Name,
+                                        ObjectType, EnteringContext, Result,
+                                        MemberOfUnknownSpecialization);
+  if (TNK != TNK_Non_template) {
+    // We resolved this to a (non-dependent) template name. Return it.
+    auto *LookupRD = dyn_cast_or_null<CXXRecordDecl>(LookupCtx);
+    if (!AllowInjectedClassName && SS.isNotEmpty() && LookupRD &&
+        Name.getKind() == UnqualifiedIdKind::IK_Identifier &&
+        Name.Identifier && LookupRD->getIdentifier() == Name.Identifier) {
+      // C++14 [class.qual]p2:
+      //   In a lookup in which function names are not ignored and the
+      //   nested-name-specifier nominates a class C, if the name specified
+      //   [...] is the injected-class-name of C, [...] the name is instead
+      //   considered to name the constructor
+      //
+      // We don't get here if naming the constructor would be valid, so we
+      // just reject immediately and recover by treating the
+      // injected-class-name as naming the template.
+      Diag(Name.getBeginLoc(),
+           diag::ext_out_of_line_qualified_id_type_names_constructor)
+          << Name.Identifier
+          << 0 /*injected-class-name used as template name*/
+          << TemplateKWLoc.isValid();
+    }
+    return TNK;
+  }
+
+  if (!MemberOfUnknownSpecialization) {
+    // Didn't find a template name, and the lookup wasn't dependent.
+    // Do the lookup again to determine if this is a "nothing found" case or
+    // a "not a template" case. FIXME: Refactor isTemplateName so we don't
+    // need to do this.
+    DeclarationNameInfo DNI = GetNameFromUnqualifiedId(Name);
+    LookupResult R(*this, DNI.getName(), Name.getBeginLoc(),
+                   LookupOrdinaryName);
+    bool MOUS;
+    // Tell LookupTemplateName that we require a template so that it diagnoses
+    // cases where it finds a non-template.
+    RequiredTemplateKind RTK = TemplateKWLoc.isValid()
+                                   ? RequiredTemplateKind(TemplateKWLoc)
+                                   : TemplateNameIsRequired;
+    if (!LookupTemplateName(R, S, SS, ObjectType.get(), EnteringContext, MOUS,
+                            RTK, nullptr, /*AllowTypoCorrection=*/false) &&
+        !R.isAmbiguous()) {
+      if (LookupCtx)
         Diag(Name.getBeginLoc(), diag::err_no_member)
             << DNI.getName() << LookupCtx << SS.getRange();
-      return TNK_Non_template;
-    } else {
-      // We found something; return it.
-      auto *LookupRD = dyn_cast<CXXRecordDecl>(LookupCtx);
-      if (!AllowInjectedClassName && SS.isSet() && LookupRD &&
-          Name.getKind() == UnqualifiedIdKind::IK_Identifier &&
-          Name.Identifier && LookupRD->getIdentifier() == Name.Identifier) {
-        // C++14 [class.qual]p2:
-        //   In a lookup in which function names are not ignored and the
-        //   nested-name-specifier nominates a class C, if the name specified
-        //   [...] is the injected-class-name of C, [...] the name is instead
-        //   considered to name the constructor
-        //
-        // We don't get here if naming the constructor would be valid, so we
-        // just reject immediately and recover by treating the
-        // injected-class-name as naming the template.
-        Diag(Name.getBeginLoc(),
-             diag::ext_out_of_line_qualified_id_type_names_constructor)
-            << Name.Identifier
-            << 0 /*injected-class-name used as template name*/
-            << 1 /*'template' keyword was used*/;
-      }
-      return TNK;
+      else
+        Diag(Name.getBeginLoc(), diag::err_undeclared_use)
+            << DNI.getName() << SS.getRange();
     }
+    return TNK_Non_template;
   }
 
   NestedNameSpecifier *Qualifier = SS.getScopeRep();
 
   switch (Name.getKind()) {
   case UnqualifiedIdKind::IK_Identifier:
-    Result = TemplateTy::make(Context.getDependentTemplateName(Qualifier,
-                                                              Name.Identifier));
+    Result = TemplateTy::make(
+        Context.getDependentTemplateName(Qualifier, Name.Identifier));
     return TNK_Dependent_template_name;
 
   case UnqualifiedIdKind::IK_OperatorFunctionId:
-    Result = TemplateTy::make(Context.getDependentTemplateName(Qualifier,
-                                             Name.OperatorFunctionId.Operator));
+    Result = TemplateTy::make(Context.getDependentTemplateName(
+        Qualifier, Name.OperatorFunctionId.Operator));
     return TNK_Function_template;
 
   case UnqualifiedIdKind::IK_LiteralOperatorId:
-    llvm_unreachable("literal operator id cannot have a dependent scope");
+    // This is a kind of template name, but can never occur in a dependent
+    // scope (literal operators can only be declared at namespace scope).
+    break;
 
   default:
     break;
   }
 
-  Diag(Name.getBeginLoc(), diag::err_template_kw_refers_to_non_template)
+  // This name cannot possibly name a dependent template. Diagnose this now
+  // rather than building a dependent template name that can never be valid.
+  Diag(Name.getBeginLoc(),
+       diag::err_template_kw_refers_to_dependent_non_template)
       << GetNameFromUnqualifiedId(Name).getName() << Name.getSourceRange()
-      << TemplateKWLoc;
+      << TemplateKWLoc.isValid() << TemplateKWLoc;
   return TNK_Non_template;
 }
 
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index f84d9b010d739..589ff1ac7565f 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -1693,6 +1693,12 @@ static QualType ConvertDeclSpecToType(TypeProcessingState &state) {
     break;
   }
 
+  // FIXME: we want resulting declarations to be marked invalid, but claiming
+  // the type is invalid is too strong - e.g. it causes ActOnTypeName to return
+  // a null type.
+  if (Result->containsErrors())
+    declarator.setInvalidType();
+
   if (S.getLangOpts().OpenCL &&
       S.checkOpenCLDisabledTypeDeclSpec(DS, Result))
     declarator.setInvalidType(true);
@@ -2453,28 +2459,34 @@ QualType Sema::BuildVectorType(QualType CurType, Expr *SizeExpr,
     return Context.getDependentVectorType(CurType, SizeExpr, AttrLoc,
                                                VectorType::GenericVector);
 
-  unsigned VectorSize = static_cast<unsigned>(VecSize.getZExtValue() * 8);
+  // vecSize is specified in bytes - convert to bits.
+  if (!VecSize.isIntN(61)) {
+    // Bit size will overflow uint64.
+    Diag(AttrLoc, diag::err_attribute_size_too_large)
+        << SizeExpr->getSourceRange();
+    return QualType();
+  }
+  uint64_t VectorSizeBits = VecSize.getZExtValue() * 8;
   unsigned TypeSize = static_cast<unsigned>(Context.getTypeSize(CurType));
 
-  if (VectorSize == 0) {
+  if (VectorSizeBits == 0) {
     Diag(AttrLoc, diag::err_attribute_zero_size) << SizeExpr->getSourceRange();
     return QualType();
   }
 
-  // vecSize is specified in bytes - convert to bits.
-  if (VectorSize % TypeSize) {
+  if (VectorSizeBits % TypeSize) {
     Diag(AttrLoc, diag::err_attribute_invalid_size)
         << SizeExpr->getSourceRange();
     return QualType();
   }
 
-  if (VectorType::isVectorSizeTooLarge(VectorSize / TypeSize)) {
+  if (VectorSizeBits / TypeSize > std::numeric_limits<uint32_t>::max()) {
     Diag(AttrLoc, diag::err_attribute_size_too_large)
         << SizeExpr->getSourceRange();
     return QualType();
   }
 
-  return Context.getVectorType(CurType, VectorSize / TypeSize,
+  return Context.getVectorType(CurType, VectorSizeBits / TypeSize,
                                VectorType::GenericVector);
 }
 
@@ -2506,6 +2518,11 @@ QualType Sema::BuildExtVectorType(QualType T, Expr *ArraySize,
       return QualType();
     }
 
+    if (!vecSize.isIntN(32)) {
+      Diag(AttrLoc, diag::err_attribute_size_too_large)
+          << ArraySize->getSourceRange();
+      return QualType();
+    }
     // Unlike gcc's vector_size attribute, the size is specified as the
     // number of elements, not the number of bytes.
     unsigned vectorSize = static_cast<unsigned>(vecSize.getZExtValue());
@@ -2516,12 +2533,6 @@ QualType Sema::BuildExtVectorType(QualType T, Expr *ArraySize,
       return QualType();
     }
 
-    if (VectorType::isVectorSizeTooLarge(vectorSize)) {
-      Diag(AttrLoc, diag::err_attribute_size_too_large)
-        << ArraySize->getSourceRange();
-      return QualType();
-    }
-
     return Context.getExtVectorType(T, vectorSize);
   }
 
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 552723d06d942..0b38b958151c2 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -1846,12 +1846,13 @@ class TreeTransform {
   /// By default, performs semantic analysis to build the new OpenMP clause.
   /// Subclasses may override this routine to provide different behavior.
   OMPClause *
-  RebuildOMPDependClause(OpenMPDependClauseKind DepKind, SourceLocation DepLoc,
-                         SourceLocation ColonLoc, ArrayRef<Expr *> VarList,
-                         SourceLocation StartLoc, SourceLocation LParenLoc,
-                         SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPDependClause(DepKind, DepLoc, ColonLoc, VarList,
-                                             StartLoc, LParenLoc, EndLoc);
+  RebuildOMPDependClause(Expr *DepModifier, OpenMPDependClauseKind DepKind,
+                         SourceLocation DepLoc, SourceLocation ColonLoc,
+                         ArrayRef<Expr *> VarList, SourceLocation StartLoc,
+                         SourceLocation LParenLoc, SourceLocation EndLoc) {
+    return getSema().ActOnOpenMPDependClause(DepModifier, DepKind, DepLoc,
+                                             ColonLoc, VarList, StartLoc,
+                                             LParenLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'device' clause.
@@ -2383,6 +2384,29 @@ class TreeTransform {
                                               ColonLoc, Length, RBracketLoc);
   }
 
+  /// Build a new array shaping expression.
+  ///
+  /// By default, performs semantic analysis to build the new expression.
+  /// Subclasses may override this routine to provide different behavior.
+  ExprResult RebuildOMPArrayShapingExpr(Expr *Base, SourceLocation LParenLoc,
+                                        SourceLocation RParenLoc,
+                                        ArrayRef<Expr *> Dims,
+                                        ArrayRef<SourceRange> BracketsRanges) {
+    return getSema().ActOnOMPArrayShapingExpr(Base, LParenLoc, RParenLoc, Dims,
+                                              BracketsRanges);
+  }
+
+  /// Build a new iterator expression.
+  ///
+  /// By default, performs semantic analysis to build the new expression.
+  /// Subclasses may override this routine to provide different behavior.
+  ExprResult RebuildOMPIteratorExpr(
+      SourceLocation IteratorKwLoc, SourceLocation LLoc, SourceLocation RLoc,
+      ArrayRef<Sema::OMPIteratorData> Data) {
+    return getSema().ActOnOMPIteratorExpr(/*Scope=*/nullptr, IteratorKwLoc,
+                                          LLoc, RLoc, Data);
+  }
+
   /// Build a new call expression.
   ///
   /// By default, performs semantic analysis to build the new expression.
@@ -9286,6 +9310,13 @@ template <typename Derived>
 OMPClause *
 TreeTransform<Derived>::TransformOMPDependClause(OMPDependClause *C) {
   llvm::SmallVector<Expr *, 16> Vars;
+  Expr *DepModifier = C->getModifier();
+  if (DepModifier) {
+    ExprResult DepModRes = getDerived().TransformExpr(DepModifier);
+    if (DepModRes.isInvalid())
+      return nullptr;
+    DepModifier = DepModRes.get();
+  }
   Vars.reserve(C->varlist_size());
   for (auto *VE : C->varlists()) {
     ExprResult EVar = getDerived().TransformExpr(cast<Expr>(VE));
@@ -9294,8 +9325,9 @@ TreeTransform<Derived>::TransformOMPDependClause(OMPDependClause *C) {
     Vars.push_back(EVar.get());
   }
   return getDerived().RebuildOMPDependClause(
-      C->getDependencyKind(), C->getDependencyLoc(), C->getColonLoc(), Vars,
-      C->getBeginLoc(), C->getLParenLoc(), C->getEndLoc());
+      DepModifier, C->getDependencyKind(), C->getDependencyLoc(),
+      C->getColonLoc(), Vars, C->getBeginLoc(), C->getLParenLoc(),
+      C->getEndLoc());
 }
 
 template <typename Derived>
@@ -10024,6 +10056,90 @@ TreeTransform<Derived>::TransformOMPArraySectionExpr(OMPArraySectionExpr *E) {
       Length.get(), E->getRBracketLoc());
 }
 
+template <typename Derived>
+ExprResult
+TreeTransform<Derived>::TransformOMPArrayShapingExpr(OMPArrayShapingExpr *E) {
+  ExprResult Base = getDerived().TransformExpr(E->getBase());
+  if (Base.isInvalid())
+    return ExprError();
+
+  SmallVector<Expr *, 4> Dims;
+  bool ErrorFound = false;
+  for (Expr *Dim : E->getDimensions()) {
+    ExprResult DimRes = getDerived().TransformExpr(Dim);
+    if (DimRes.isInvalid()) {
+      ErrorFound = true;
+      continue;
+    }
+    Dims.push_back(DimRes.get());
+  }
+
+  if (ErrorFound)
+    return ExprError();
+  return getDerived().RebuildOMPArrayShapingExpr(Base.get(), E->getLParenLoc(),
+                                                 E->getRParenLoc(), Dims,
+                                                 E->getBracketsRanges());
+}
+
+template <typename Derived>
+ExprResult
+TreeTransform<Derived>::TransformOMPIteratorExpr(OMPIteratorExpr *E) {
+  unsigned NumIterators = E->numOfIterators();
+  SmallVector<Sema::OMPIteratorData, 4> Data(NumIterators);
+
+  bool ErrorFound = false;
+  bool NeedToRebuild = getDerived().AlwaysRebuild();
+  for (unsigned I = 0; I < NumIterators; ++I) {
+    auto *D = cast<VarDecl>(E->getIteratorDecl(I));
+    Data[I].DeclIdent = D->getIdentifier();
+    Data[I].DeclIdentLoc = D->getLocation();
+    if (D->getLocation() == D->getBeginLoc()) {
+      assert(SemaRef.Context.hasSameType(D->getType(), SemaRef.Context.IntTy) &&
+             "Implicit type must be int.");
+    } else {
+      TypeSourceInfo *TSI = getDerived().TransformType(D->getTypeSourceInfo());
+      QualType DeclTy = getDerived().TransformType(D->getType());
+      Data[I].Type = SemaRef.CreateParsedType(DeclTy, TSI);
+    }
+    OMPIteratorExpr::IteratorRange Range = E->getIteratorRange(I);
+    ExprResult Begin = getDerived().TransformExpr(Range.Begin);
+    ExprResult End = getDerived().TransformExpr(Range.End);
+    ExprResult Step = getDerived().TransformExpr(Range.Step);
+    ErrorFound = ErrorFound ||
+                 !(!D->getTypeSourceInfo() || (Data[I].Type.getAsOpaquePtr() &&
+                                               !Data[I].Type.get().isNull())) ||
+                 Begin.isInvalid() || End.isInvalid() || Step.isInvalid();
+    if (ErrorFound)
+      continue;
+    Data[I].Range.Begin = Begin.get();
+    Data[I].Range.End = End.get();
+    Data[I].Range.Step = Step.get();
+    Data[I].AssignLoc = E->getAssignLoc(I);
+    Data[I].ColonLoc = E->getColonLoc(I);
+    Data[I].SecColonLoc = E->getSecondColonLoc(I);
+    NeedToRebuild =
+        NeedToRebuild ||
+        (D->getTypeSourceInfo() && Data[I].Type.get().getTypePtrOrNull() !=
+                                       D->getType().getTypePtrOrNull()) ||
+        Range.Begin != Data[I].Range.Begin || Range.End != Data[I].Range.End ||
+        Range.Step != Data[I].Range.Step;
+  }
+  if (ErrorFound)
+    return ExprError();
+  if (!NeedToRebuild)
+    return E;
+
+  ExprResult Res = getDerived().RebuildOMPIteratorExpr(
+      E->getIteratorKwLoc(), E->getLParenLoc(), E->getRParenLoc(), Data);
+  if (!Res.isUsable())
+    return Res;
+  auto *IE = cast<OMPIteratorExpr>(Res.get());
+  for (unsigned I = 0; I < NumIterators; ++I)
+    getDerived().transformedLocalDecl(E->getIteratorDecl(I),
+                                      IE->getIteratorDecl(I));
+  return Res;
+}
+
 template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformCallExpr(CallExpr *E) {
@@ -13688,11 +13804,10 @@ TreeTransform<Derived>::RebuildTemplateName(CXXScopeSpec &SS,
   UnqualifiedId TemplateName;
   TemplateName.setIdentifier(&Name, NameLoc);
   Sema::TemplateTy Template;
-  getSema().ActOnDependentTemplateName(/*Scope=*/nullptr,
-                                       SS, TemplateKWLoc, TemplateName,
-                                       ParsedType::make(ObjectType),
-                                       /*EnteringContext=*/false,
-                                       Template, AllowInjectedClassName);
+  getSema().ActOnTemplateName(/*Scope=*/nullptr, SS, TemplateKWLoc,
+                              TemplateName, ParsedType::make(ObjectType),
+                              /*EnteringContext=*/false, Template,
+                              AllowInjectedClassName);
   return Template.get();
 }
 
@@ -13709,11 +13824,9 @@ TreeTransform<Derived>::RebuildTemplateName(CXXScopeSpec &SS,
   SourceLocation SymbolLocations[3] = { NameLoc, NameLoc, NameLoc };
   Name.setOperatorFunctionId(NameLoc, Operator, SymbolLocations);
   Sema::TemplateTy Template;
-  getSema().ActOnDependentTemplateName(/*Scope=*/nullptr,
-                                       SS, TemplateKWLoc, Name,
-                                       ParsedType::make(ObjectType),
-                                       /*EnteringContext=*/false,
-                                       Template, AllowInjectedClassName);
+  getSema().ActOnTemplateName(
+      /*Scope=*/nullptr, SS, TemplateKWLoc, Name, ParsedType::make(ObjectType),
+      /*EnteringContext=*/false, Template, AllowInjectedClassName);
   return Template.get();
 }
 
diff --git a/clang/lib/Serialization/ASTCommon.cpp b/clang/lib/Serialization/ASTCommon.cpp
index f93f1f77405d3..566bda2d71f70 100644
--- a/clang/lib/Serialization/ASTCommon.cpp
+++ b/clang/lib/Serialization/ASTCommon.cpp
@@ -243,6 +243,12 @@ serialization::TypeIdxFromBuiltin(const BuiltinType *BT) {
   case BuiltinType::OMPArraySection:
     ID = PREDEF_TYPE_OMP_ARRAY_SECTION;
     break;
+  case BuiltinType::OMPArrayShaping:
+    ID = PREDEF_TYPE_OMP_ARRAY_SHAPING;
+    break;
+  case BuiltinType::OMPIterator:
+    ID = PREDEF_TYPE_OMP_ITERATOR;
+    break;
   }
 
   return TypeIdx(ID);
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index c2d28f4362418..7d84f1108c854 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -6957,6 +6957,12 @@ QualType ASTReader::GetType(TypeID ID) {
     case PREDEF_TYPE_OMP_ARRAY_SECTION:
       T = Context.OMPArraySectionTy;
       break;
+    case PREDEF_TYPE_OMP_ARRAY_SHAPING:
+      T = Context.OMPArraySectionTy;
+      break;
+    case PREDEF_TYPE_OMP_ITERATOR:
+      T = Context.OMPIteratorTy;
+      break;
 #define SVE_TYPE(Name, Id, SingletonId) \
     case PREDEF_TYPE_##Id##_ID: \
       T = Context.SingletonId; \
@@ -8511,7 +8517,7 @@ unsigned ASTReader::getModuleFileID(ModuleFile *F) {
 
 llvm::Optional<ASTSourceDescriptor>
 ASTReader::getSourceDescriptor(unsigned ID) {
-  if (const Module *M = getSubmodule(ID))
+  if (Module *M = getSubmodule(ID))
     return ASTSourceDescriptor(*M);
 
   // If there is only a single PCH, return it instead.
@@ -12304,6 +12310,7 @@ void OMPClauseReader::VisitOMPDepobjClause(OMPDepobjClause *C) {
 
 void OMPClauseReader::VisitOMPDependClause(OMPDependClause *C) {
   C->setLParenLoc(Record.readSourceLocation());
+  C->setModifier(Record.readSubExpr());
   C->setDependencyKind(
       static_cast<OpenMPDependClauseKind>(Record.readInt()));
   C->setDependencyLoc(Record.readSourceLocation());
@@ -12328,7 +12335,7 @@ void OMPClauseReader::VisitOMPDeviceClause(OMPDeviceClause *C) {
 
 void OMPClauseReader::VisitOMPMapClause(OMPMapClause *C) {
   C->setLParenLoc(Record.readSourceLocation());
-  for (unsigned I = 0; I < OMPMapClause::NumberOfModifiers; ++I) {
+  for (unsigned I = 0; I < NumberOfOMPMapClauseModifiers; ++I) {
     C->setMapTypeModifier(
         I, static_cast<OpenMPMapModifierKind>(Record.readInt()));
     C->setMapTypeModifierLoc(I, Record.readSourceLocation());
diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp
index 5d9033e379777..2c915659ad7c5 100644
--- a/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -50,6 +50,7 @@
 #include "clang/Lex/Token.h"
 #include "clang/Serialization/ASTBitCodes.h"
 #include "clang/Serialization/ASTRecordReader.h"
+#include "llvm/ADT/BitmaskEnum.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
@@ -107,7 +108,7 @@ namespace clang {
     /// The number of record fields required for the Expr class
     /// itself.
     static const unsigned NumExprFields =
-        NumStmtFields + ExprDependenceBits + 3;
+        NumStmtFields + llvm::BitWidth<ExprDependence> + 3;
 
     /// Read and initialize a ExplicitTemplateArgumentList structure.
     void ReadTemplateKWAndArgsInfo(ASTTemplateKWAndArgsInfo &Args,
@@ -910,6 +911,42 @@ void ASTStmtReader::VisitOMPArraySectionExpr(OMPArraySectionExpr *E) {
   E->setRBracketLoc(readSourceLocation());
 }
 
+void ASTStmtReader::VisitOMPArrayShapingExpr(OMPArrayShapingExpr *E) {
+  VisitExpr(E);
+  unsigned NumDims = Record.readInt();
+  E->setBase(Record.readSubExpr());
+  SmallVector<Expr *, 4> Dims(NumDims);
+  for (unsigned I = 0; I < NumDims; ++I)
+    Dims[I] = Record.readSubExpr();
+  E->setDimensions(Dims);
+  SmallVector<SourceRange, 4> SRs(NumDims);
+  for (unsigned I = 0; I < NumDims; ++I)
+    SRs[I] = readSourceRange();
+  E->setBracketsRanges(SRs);
+  E->setLParenLoc(readSourceLocation());
+  E->setRParenLoc(readSourceLocation());
+}
+
+void ASTStmtReader::VisitOMPIteratorExpr(OMPIteratorExpr *E) {
+  VisitExpr(E);
+  unsigned NumIters = Record.readInt();
+  E->setIteratorKwLoc(readSourceLocation());
+  E->setLParenLoc(readSourceLocation());
+  E->setRParenLoc(readSourceLocation());
+  for (unsigned I = 0; I < NumIters; ++I) {
+    E->setIteratorDeclaration(I, Record.readDeclRef());
+    E->setAssignmentLoc(I, readSourceLocation());
+    Expr *Begin = Record.readSubExpr();
+    Expr *End = Record.readSubExpr();
+    Expr *Step = Record.readSubExpr();
+    SourceLocation ColonLoc = readSourceLocation();
+    SourceLocation SecColonLoc;
+    if (Step)
+      SecColonLoc = readSourceLocation();
+    E->setIteratorRange(I, Begin, ColonLoc, End, SecColonLoc, Step);
+  }
+}
+
 void ASTStmtReader::VisitCallExpr(CallExpr *E) {
   VisitExpr(E);
   unsigned NumArgs = Record.readInt();
@@ -2865,6 +2902,16 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) {
       S = new (Context) OMPArraySectionExpr(Empty);
       break;
 
+    case EXPR_OMP_ARRAY_SHAPING:
+      S = OMPArrayShapingExpr::CreateEmpty(
+          Context, Record[ASTStmtReader::NumExprFields]);
+      break;
+
+    case EXPR_OMP_ITERATOR:
+      S = OMPIteratorExpr::CreateEmpty(Context,
+                                       Record[ASTStmtReader::NumExprFields]);
+      break;
+
     case EXPR_CALL:
       S = CallExpr::CreateEmpty(
           Context, /*NumArgs=*/Record[ASTStmtReader::NumExprFields], Empty);
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index f7c58ed11d9f4..89ba7e6239657 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -6379,6 +6379,7 @@ void OMPClauseWriter::VisitOMPDependClause(OMPDependClause *C) {
   Record.push_back(C->varlist_size());
   Record.push_back(C->getNumLoops());
   Record.AddSourceLocation(C->getLParenLoc());
+  Record.AddStmt(C->getModifier());
   Record.push_back(C->getDependencyKind());
   Record.AddSourceLocation(C->getDependencyLoc());
   Record.AddSourceLocation(C->getColonLoc());
@@ -6402,7 +6403,7 @@ void OMPClauseWriter::VisitOMPMapClause(OMPMapClause *C) {
   Record.push_back(C->getTotalComponentListNum());
   Record.push_back(C->getTotalComponentsNum());
   Record.AddSourceLocation(C->getLParenLoc());
-  for (unsigned I = 0; I < OMPMapClause::NumberOfModifiers; ++I) {
+  for (unsigned I = 0; I < NumberOfOMPMapClauseModifiers; ++I) {
     Record.push_back(C->getMapTypeModifier(I));
     Record.AddSourceLocation(C->getMapTypeModifierLoc(I));
   }
diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp
index d64e2330850a1..ee8bb3e1d4626 100644
--- a/clang/lib/Serialization/ASTWriterStmt.cpp
+++ b/clang/lib/Serialization/ASTWriterStmt.cpp
@@ -774,6 +774,39 @@ void ASTStmtWriter::VisitOMPArraySectionExpr(OMPArraySectionExpr *E) {
   Code = serialization::EXPR_OMP_ARRAY_SECTION;
 }
 
+void ASTStmtWriter::VisitOMPArrayShapingExpr(OMPArrayShapingExpr *E) {
+  VisitExpr(E);
+  Record.push_back(E->getDimensions().size());
+  Record.AddStmt(E->getBase());
+  for (Expr *Dim : E->getDimensions())
+    Record.AddStmt(Dim);
+  for (SourceRange SR : E->getBracketsRanges())
+    Record.AddSourceRange(SR);
+  Record.AddSourceLocation(E->getLParenLoc());
+  Record.AddSourceLocation(E->getRParenLoc());
+  Code = serialization::EXPR_OMP_ARRAY_SHAPING;
+}
+
+void ASTStmtWriter::VisitOMPIteratorExpr(OMPIteratorExpr *E) {
+  VisitExpr(E);
+  Record.push_back(E->numOfIterators());
+  Record.AddSourceLocation(E->getIteratorKwLoc());
+  Record.AddSourceLocation(E->getLParenLoc());
+  Record.AddSourceLocation(E->getRParenLoc());
+  for (unsigned I = 0, End = E->numOfIterators(); I < End; ++I) {
+    Record.AddDeclRef(E->getIteratorDecl(I));
+    Record.AddSourceLocation(E->getAssignLoc(I));
+    OMPIteratorExpr::IteratorRange Range = E->getIteratorRange(I);
+    Record.AddStmt(Range.Begin);
+    Record.AddStmt(Range.End);
+    Record.AddStmt(Range.Step);
+    Record.AddSourceLocation(E->getColonLoc(I));
+    if (Range.Step)
+      Record.AddSourceLocation(E->getSecondColonLoc(I));
+  }
+  Code = serialization::EXPR_OMP_ITERATOR;
+}
+
 void ASTStmtWriter::VisitCallExpr(CallExpr *E) {
   VisitExpr(E);
   Record.push_back(E->getNumArgs());
diff --git a/clang/lib/Serialization/GeneratePCH.cpp b/clang/lib/Serialization/GeneratePCH.cpp
index 002233e49bb06..d869796b82c12 100644
--- a/clang/lib/Serialization/GeneratePCH.cpp
+++ b/clang/lib/Serialization/GeneratePCH.cpp
@@ -57,6 +57,11 @@ void PCHGenerator::HandleTranslationUnit(ASTContext &Ctx) {
     }
   }
 
+  // Errors that do not prevent the PCH from being written should not cause the
+  // overall compilation to fail either.
+  if (AllowASTWithErrors)
+    PP.getDiagnostics().getClient()->clear();
+
   // Emit the PCH file to the Buffer.
   assert(SemaPtr && "No Sema?");
   Buffer->Signature =
diff --git a/clang/lib/StaticAnalyzer/Checkers/AnalysisOrderChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/AnalysisOrderChecker.cpp
index 2ef50a727ece4..fb1a407dbdf7f 100644
--- a/clang/lib/StaticAnalyzer/Checkers/AnalysisOrderChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/AnalysisOrderChecker.cpp
@@ -186,6 +186,6 @@ void ento::registerAnalysisOrderChecker(CheckerManager &mgr) {
   mgr.registerChecker<AnalysisOrderChecker>();
 }
 
-bool ento::shouldRegisterAnalysisOrderChecker(const LangOptions &LO) {
+bool ento::shouldRegisterAnalysisOrderChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/AnalyzerStatsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/AnalyzerStatsChecker.cpp
index 634492655228a..c06604b6cffef 100644
--- a/clang/lib/StaticAnalyzer/Checkers/AnalyzerStatsChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/AnalyzerStatsChecker.cpp
@@ -140,6 +140,6 @@ void ento::registerAnalyzerStatsChecker(CheckerManager &mgr) {
   mgr.registerChecker<AnalyzerStatsChecker>();
 }
 
-bool ento::shouldRegisterAnalyzerStatsChecker(const LangOptions &LO) {
+bool ento::shouldRegisterAnalyzerStatsChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/ArrayBoundChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ArrayBoundChecker.cpp
index 716571c592771..59163c1f31fa9 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ArrayBoundChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ArrayBoundChecker.cpp
@@ -92,6 +92,6 @@ void ento::registerArrayBoundChecker(CheckerManager &mgr) {
   mgr.registerChecker<ArrayBoundChecker>();
 }
 
-bool ento::shouldRegisterArrayBoundChecker(const LangOptions &LO) {
+bool ento::shouldRegisterArrayBoundChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/ArrayBoundCheckerV2.cpp b/clang/lib/StaticAnalyzer/Checkers/ArrayBoundCheckerV2.cpp
index 192a3a1a805f8..7c264bba4b6a7 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ArrayBoundCheckerV2.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ArrayBoundCheckerV2.cpp
@@ -356,6 +356,6 @@ void ento::registerArrayBoundCheckerV2(CheckerManager &mgr) {
   mgr.registerChecker<ArrayBoundCheckerV2>();
 }
 
-bool ento::shouldRegisterArrayBoundCheckerV2(const LangOptions &LO) {
+bool ento::shouldRegisterArrayBoundCheckerV2(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/BasicObjCFoundationChecks.cpp b/clang/lib/StaticAnalyzer/Checkers/BasicObjCFoundationChecks.cpp
index 325952fe4ed4d..918c6e361381e 100644
--- a/clang/lib/StaticAnalyzer/Checkers/BasicObjCFoundationChecks.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/BasicObjCFoundationChecks.cpp
@@ -1243,7 +1243,7 @@ void ento::registerNilArgChecker(CheckerManager &mgr) {
   mgr.registerChecker<NilArgChecker>();
 }
 
-bool ento::shouldRegisterNilArgChecker(const LangOptions &LO) {
+bool ento::shouldRegisterNilArgChecker(const CheckerManager &mgr) {
   return true;
 }
 
@@ -1251,7 +1251,7 @@ void ento::registerCFNumberChecker(CheckerManager &mgr) {
   mgr.registerChecker<CFNumberChecker>();
 }
 
-bool ento::shouldRegisterCFNumberChecker(const LangOptions &LO) {
+bool ento::shouldRegisterCFNumberChecker(const CheckerManager &mgr) {
   return true;
 }
 
@@ -1259,7 +1259,7 @@ void ento::registerCFRetainReleaseChecker(CheckerManager &mgr) {
   mgr.registerChecker<CFRetainReleaseChecker>();
 }
 
-bool ento::shouldRegisterCFRetainReleaseChecker(const LangOptions &LO) {
+bool ento::shouldRegisterCFRetainReleaseChecker(const CheckerManager &mgr) {
   return true;
 }
 
@@ -1267,7 +1267,7 @@ void ento::registerClassReleaseChecker(CheckerManager &mgr) {
   mgr.registerChecker<ClassReleaseChecker>();
 }
 
-bool ento::shouldRegisterClassReleaseChecker(const LangOptions &LO) {
+bool ento::shouldRegisterClassReleaseChecker(const CheckerManager &mgr) {
   return true;
 }
 
@@ -1275,7 +1275,7 @@ void ento::registerVariadicMethodTypeChecker(CheckerManager &mgr) {
   mgr.registerChecker<VariadicMethodTypeChecker>();
 }
 
-bool ento::shouldRegisterVariadicMethodTypeChecker(const LangOptions &LO) {
+bool ento::shouldRegisterVariadicMethodTypeChecker(const CheckerManager &mgr) {
   return true;
 }
 
@@ -1283,7 +1283,7 @@ void ento::registerObjCLoopChecker(CheckerManager &mgr) {
   mgr.registerChecker<ObjCLoopChecker>();
 }
 
-bool ento::shouldRegisterObjCLoopChecker(const LangOptions &LO) {
+bool ento::shouldRegisterObjCLoopChecker(const CheckerManager &mgr) {
   return true;
 }
 
@@ -1291,6 +1291,6 @@ void ento::registerObjCNonNilReturnValueChecker(CheckerManager &mgr) {
   mgr.registerChecker<ObjCNonNilReturnValueChecker>();
 }
 
-bool ento::shouldRegisterObjCNonNilReturnValueChecker(const LangOptions &LO) {
+bool ento::shouldRegisterObjCNonNilReturnValueChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/BlockInCriticalSectionChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/BlockInCriticalSectionChecker.cpp
index 0eb3c3d1d0e6d..2752b37f9b3f6 100644
--- a/clang/lib/StaticAnalyzer/Checkers/BlockInCriticalSectionChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/BlockInCriticalSectionChecker.cpp
@@ -184,6 +184,6 @@ void ento::registerBlockInCriticalSectionChecker(CheckerManager &mgr) {
   mgr.registerChecker<BlockInCriticalSectionChecker>();
 }
 
-bool ento::shouldRegisterBlockInCriticalSectionChecker(const LangOptions &LO) {
+bool ento::shouldRegisterBlockInCriticalSectionChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/BoolAssignmentChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/BoolAssignmentChecker.cpp
index a64b1213ba291..6c0caf3c4e78a 100644
--- a/clang/lib/StaticAnalyzer/Checkers/BoolAssignmentChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/BoolAssignmentChecker.cpp
@@ -96,6 +96,6 @@ void ento::registerBoolAssignmentChecker(CheckerManager &mgr) {
     mgr.registerChecker<BoolAssignmentChecker>();
 }
 
-bool ento::shouldRegisterBoolAssignmentChecker(const LangOptions &LO) {
+bool ento::shouldRegisterBoolAssignmentChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp
index 51236437329c6..3eeb1e9a3502e 100644
--- a/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp
@@ -135,6 +135,6 @@ void ento::registerBuiltinFunctionChecker(CheckerManager &mgr) {
   mgr.registerChecker<BuiltinFunctionChecker>();
 }
 
-bool ento::shouldRegisterBuiltinFunctionChecker(const LangOptions &LO) {
+bool ento::shouldRegisterBuiltinFunctionChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/CStringChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/CStringChecker.cpp
index 15004134bfe68..dcad2264e8cd9 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CStringChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/CStringChecker.cpp
@@ -2430,7 +2430,7 @@ void ento::registerCStringModeling(CheckerManager &Mgr) {
   Mgr.registerChecker<CStringChecker>();
 }
 
-bool ento::shouldRegisterCStringModeling(const LangOptions &LO) {
+bool ento::shouldRegisterCStringModeling(const CheckerManager &mgr) {
   return true;
 }
 
@@ -2441,7 +2441,7 @@ bool ento::shouldRegisterCStringModeling(const LangOptions &LO) {
     checker->Filter.CheckName##name = mgr.getCurrentCheckerName();             \
   }                                                                            \
                                                                                \
-  bool ento::shouldRegister##name(const LangOptions &LO) { return true; }
+  bool ento::shouldRegister##name(const CheckerManager &mgr) { return true; }
 
 REGISTER_CHECKER(CStringNullArg)
 REGISTER_CHECKER(CStringOutOfBounds)
diff --git a/clang/lib/StaticAnalyzer/Checkers/CStringSyntaxChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/CStringSyntaxChecker.cpp
index d84fcc69a4925..888724f7ea3b2 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CStringSyntaxChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/CStringSyntaxChecker.cpp
@@ -291,6 +291,6 @@ void ento::registerCStringSyntaxChecker(CheckerManager &mgr) {
   mgr.registerChecker<CStringSyntaxChecker>();
 }
 
-bool ento::shouldRegisterCStringSyntaxChecker(const LangOptions &LO) {
+bool ento::shouldRegisterCStringSyntaxChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/CXXSelfAssignmentChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/CXXSelfAssignmentChecker.cpp
index aada05db2cbc4..24776338ce10b 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CXXSelfAssignmentChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/CXXSelfAssignmentChecker.cpp
@@ -76,6 +76,6 @@ void ento::registerCXXSelfAssignmentChecker(CheckerManager &Mgr) {
   Mgr.registerChecker<CXXSelfAssignmentChecker>();
 }
 
-bool ento::shouldRegisterCXXSelfAssignmentChecker(const LangOptions &LO) {
+bool ento::shouldRegisterCXXSelfAssignmentChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp
index 2fcb765cd4eef..82bc200ba4ce5 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp
@@ -605,7 +605,7 @@ void ento::registerCallAndMessageChecker(CheckerManager &mgr) {
   mgr.registerChecker<CallAndMessageChecker>();
 }
 
-bool ento::shouldRegisterCallAndMessageChecker(const LangOptions &LO) {
+bool ento::shouldRegisterCallAndMessageChecker(const CheckerManager &mgr) {
   return true;
 }
 
@@ -615,6 +615,6 @@ void ento::registerCallAndMessageUnInitRefArg(CheckerManager &mgr) {
   Checker->CheckName_CallAndMessageUnInitRefArg = mgr.getCurrentCheckerName();
 }
 
-bool ento::shouldRegisterCallAndMessageUnInitRefArg(const LangOptions &LO) {
+bool ento::shouldRegisterCallAndMessageUnInitRefArg(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/CastSizeChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/CastSizeChecker.cpp
index b3432901e9369..a498f252e693c 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CastSizeChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/CastSizeChecker.cpp
@@ -146,10 +146,11 @@ void ento::registerCastSizeChecker(CheckerManager &mgr) {
   mgr.registerChecker<CastSizeChecker>();
 }
 
-bool ento::shouldRegisterCastSizeChecker(const LangOptions &LO) {
+bool ento::shouldRegisterCastSizeChecker(const CheckerManager &mgr) {
   // PR31226: C++ is more complicated than what this checker currently supports.
   // There are derived-to-base casts, there are different rules for 0-size
   // structures, no flexible arrays, etc.
   // FIXME: Disabled on C++ for now.
+  const LangOptions &LO = mgr.getLangOpts();
   return !LO.CPlusPlus;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/CastToStructChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/CastToStructChecker.cpp
index 93665596be298..e674ec43bcd9d 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CastToStructChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/CastToStructChecker.cpp
@@ -120,6 +120,6 @@ void ento::registerCastToStructChecker(CheckerManager &mgr) {
   mgr.registerChecker<CastToStructChecker>();
 }
 
-bool ento::shouldRegisterCastToStructChecker(const LangOptions &LO) {
+bool ento::shouldRegisterCastToStructChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/CastValueChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/CastValueChecker.cpp
index 1c3ff3e1b2d9f..85e2b71e53ea1 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CastValueChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/CastValueChecker.cpp
@@ -436,6 +436,6 @@ void ento::registerCastValueChecker(CheckerManager &Mgr) {
   Mgr.registerChecker<CastValueChecker>();
 }
 
-bool ento::shouldRegisterCastValueChecker(const LangOptions &LO) {
+bool ento::shouldRegisterCastValueChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/CheckObjCDealloc.cpp b/clang/lib/StaticAnalyzer/Checkers/CheckObjCDealloc.cpp
index 50b872bd8682f..13836f08a61ef 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CheckObjCDealloc.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/CheckObjCDealloc.cpp
@@ -1088,7 +1088,8 @@ void ento::registerObjCDeallocChecker(CheckerManager &Mgr) {
   Mgr.registerChecker<ObjCDeallocChecker>();
 }
 
-bool ento::shouldRegisterObjCDeallocChecker(const LangOptions &LO) {
+bool ento::shouldRegisterObjCDeallocChecker(const CheckerManager &mgr) {
   // These checker only makes sense under MRR.
+  const LangOptions &LO = mgr.getLangOpts();
   return LO.getGC() != LangOptions::GCOnly && !LO.ObjCAutoRefCount;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/CheckObjCInstMethSignature.cpp b/clang/lib/StaticAnalyzer/Checkers/CheckObjCInstMethSignature.cpp
index 1694c237cda42..175dfcef0df45 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CheckObjCInstMethSignature.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/CheckObjCInstMethSignature.cpp
@@ -138,6 +138,6 @@ void ento::registerObjCMethSigsChecker(CheckerManager &mgr) {
   mgr.registerChecker<ObjCMethSigsChecker>();
 }
 
-bool ento::shouldRegisterObjCMethSigsChecker(const LangOptions &LO) {
+bool ento::shouldRegisterObjCMethSigsChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/CheckPlacementNew.cpp b/clang/lib/StaticAnalyzer/Checkers/CheckPlacementNew.cpp
index be1a65c474d82..43ed0ffb238d2 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CheckPlacementNew.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/CheckPlacementNew.cpp
@@ -1,3 +1,15 @@
+//==- CheckPlacementNew.cpp - Check for placement new operation --*- C++ -*-==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines a check for misuse of the default placement new operator.
+//
+//===----------------------------------------------------------------------===//
+
 #include "clang/StaticAnalyzer/Checkers/BuiltinCheckerRegistration.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
@@ -117,6 +129,6 @@ void ento::registerPlacementNewChecker(CheckerManager &mgr) {
   mgr.registerChecker<PlacementNewChecker>();
 }
 
-bool ento::shouldRegisterPlacementNewChecker(const LangOptions &LO) {
+bool ento::shouldRegisterPlacementNewChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/CheckSecuritySyntaxOnly.cpp b/clang/lib/StaticAnalyzer/Checkers/CheckSecuritySyntaxOnly.cpp
index d9ffa562c0aaa..d06c87631bfb5 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CheckSecuritySyntaxOnly.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/CheckSecuritySyntaxOnly.cpp
@@ -1076,7 +1076,7 @@ void ento::registerSecuritySyntaxChecker(CheckerManager &mgr) {
   mgr.registerChecker<SecuritySyntaxChecker>();
 }
 
-bool ento::shouldRegisterSecuritySyntaxChecker(const LangOptions &LO) {
+bool ento::shouldRegisterSecuritySyntaxChecker(const CheckerManager &mgr) {
   return true;
 }
 
@@ -1087,7 +1087,7 @@ bool ento::shouldRegisterSecuritySyntaxChecker(const LangOptions &LO) {
     checker->filter.checkName_##name = mgr.getCurrentCheckerName();            \
   }                                                                            \
                                                                                \
-  bool ento::shouldRegister##name(const LangOptions &LO) { return true; }
+  bool ento::shouldRegister##name(const CheckerManager &mgr) { return true; }
 
 REGISTER_CHECKER(bcmp)
 REGISTER_CHECKER(bcopy)
diff --git a/clang/lib/StaticAnalyzer/Checkers/CheckSizeofPointer.cpp b/clang/lib/StaticAnalyzer/Checkers/CheckSizeofPointer.cpp
index ec401cfa89859..0d2551f11583e 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CheckSizeofPointer.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/CheckSizeofPointer.cpp
@@ -91,6 +91,6 @@ void ento::registerSizeofPointerChecker(CheckerManager &mgr) {
   mgr.registerChecker<SizeofPointerChecker>();
 }
 
-bool ento::shouldRegisterSizeofPointerChecker(const LangOptions &LO) {
+bool ento::shouldRegisterSizeofPointerChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/ChrootChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ChrootChecker.cpp
index 7a41a7b6b216e..fd53c04f4bbf0 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ChrootChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ChrootChecker.cpp
@@ -136,6 +136,6 @@ void ento::registerChrootChecker(CheckerManager &mgr) {
   mgr.registerChecker<ChrootChecker>();
 }
 
-bool ento::shouldRegisterChrootChecker(const LangOptions &LO) {
+bool ento::shouldRegisterChrootChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/CloneChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/CloneChecker.cpp
index ce45b5be34c9f..7968aed85e1bc 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CloneChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/CloneChecker.cpp
@@ -208,6 +208,6 @@ void ento::registerCloneChecker(CheckerManager &Mgr) {
     .getCheckerStringOption(Checker, "IgnoredFilesPattern");
 }
 
-bool ento::shouldRegisterCloneChecker(const LangOptions &LO) {
+bool ento::shouldRegisterCloneChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/ContainerModeling.cpp b/clang/lib/StaticAnalyzer/Checkers/ContainerModeling.cpp
index 7d8dc8b8a0ab6..73c6517fd0ebf 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ContainerModeling.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ContainerModeling.cpp
@@ -12,6 +12,7 @@
 
 #include "clang/StaticAnalyzer/Checkers/BuiltinCheckerRegistration.h"
 #include "clang/AST/DeclTemplate.h"
+#include "clang/Driver/DriverDiagnostic.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
 #include "clang/StaticAnalyzer/Core/Checker.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h"
@@ -31,47 +32,43 @@ namespace {
 class ContainerModeling
   : public Checker<check::PostCall, check::LiveSymbols, check::DeadSymbols> {
 
-  void handleBegin(CheckerContext &C, const Expr *CE, const SVal &RetVal,
-                   const SVal &Cont) const;
-  void handleEnd(CheckerContext &C, const Expr *CE, const SVal &RetVal,
-                 const SVal &Cont) const;
-  void handleAssignment(CheckerContext &C, const SVal &Cont,
-                        const Expr *CE = nullptr,
-                        const SVal &OldCont = UndefinedVal()) const;
-  void handleAssign(CheckerContext &C, const SVal &Cont) const;
-  void handleClear(CheckerContext &C, const SVal &Cont) const;
-  void handlePushBack(CheckerContext &C, const SVal &Cont) const;
-  void handlePopBack(CheckerContext &C, const SVal &Cont) const;
-  void handlePushFront(CheckerContext &C, const SVal &Cont) const;
-  void handlePopFront(CheckerContext &C, const SVal &Cont) const;
-  void handleInsert(CheckerContext &C, const SVal &Cont,
-                    const SVal &Iter) const;
-  void handleErase(CheckerContext &C, const SVal &Cont, const SVal &Iter) const;
-  void handleErase(CheckerContext &C, const SVal &Cont, const SVal &Iter1,
-                   const SVal &Iter2) const;
-  void handleEraseAfter(CheckerContext &C, const SVal &Cont,
-                        const SVal &Iter) const;
-  void handleEraseAfter(CheckerContext &C, const SVal &Cont, const SVal &Iter1,
-                        const SVal &Iter2) const;
+  void handleBegin(CheckerContext &C, const Expr *CE, SVal RetVal,
+                   SVal Cont) const;
+  void handleEnd(CheckerContext &C, const Expr *CE, SVal RetVal,
+                 SVal Cont) const;
+  void handleAssignment(CheckerContext &C, SVal Cont, const Expr *CE = nullptr,
+                        SVal OldCont = UndefinedVal()) const;
+  void handleAssign(CheckerContext &C, SVal Cont, const Expr *ContE) const;
+  void handleClear(CheckerContext &C, SVal Cont, const Expr *ContE) const;
+  void handlePushBack(CheckerContext &C, SVal Cont, const Expr *ContE) const;
+  void handlePopBack(CheckerContext &C, SVal Cont, const Expr *ContE) const;
+  void handlePushFront(CheckerContext &C, SVal Cont, const Expr *ContE) const;
+  void handlePopFront(CheckerContext &C, SVal Cont, const Expr *ContE) const;
+  void handleInsert(CheckerContext &C, SVal Cont, SVal Iter) const;
+  void handleErase(CheckerContext &C, SVal Cont, SVal Iter) const;
+  void handleErase(CheckerContext &C, SVal Cont, SVal Iter1, SVal Iter2) const;
+  void handleEraseAfter(CheckerContext &C, SVal Cont, SVal Iter) const;
+  void handleEraseAfter(CheckerContext &C, SVal Cont, SVal Iter1,
+                        SVal Iter2) const;
+  const NoteTag *getChangeTag(CheckerContext &C, StringRef Text,
+                              const MemRegion *ContReg,
+                              const Expr *ContE) const;
   void printState(raw_ostream &Out, ProgramStateRef State, const char *NL,
                   const char *Sep) const override;
 
 public:
-  ContainerModeling() {}
+  ContainerModeling() = default;
 
   void checkPostCall(const CallEvent &Call, CheckerContext &C) const;
   void checkLiveSymbols(ProgramStateRef State, SymbolReaper &SR) const;
   void checkDeadSymbols(SymbolReaper &SR, CheckerContext &C) const;
 
-  typedef void (ContainerModeling::*NoItParamFn)(CheckerContext &,
-                                                 const SVal &) const;
-  typedef void (ContainerModeling::*OneItParamFn)(CheckerContext &,
-                                                  const SVal &,
-                                                  const SVal &) const;
-  typedef void (ContainerModeling::*TwoItParamFn)(CheckerContext &,
-                                                  const SVal &,
-                                                  const SVal &,
-                                                  const SVal &) const;
+  using NoItParamFn = void (ContainerModeling::*)(CheckerContext &, SVal,
+                                                  const Expr *) const;
+  using OneItParamFn = void (ContainerModeling::*)(CheckerContext &, SVal,
+                                                   SVal) const;
+  using TwoItParamFn = void (ContainerModeling::*)(CheckerContext &, SVal, SVal,
+                                                   SVal) const;
 
   CallDescriptionMap<NoItParamFn> NoIterParamFunctions = {
     {{0, "clear", 0},
@@ -184,7 +181,8 @@ void ContainerModeling::checkPostCall(const CallEvent &Call,
     if (const auto *InstCall = dyn_cast<CXXInstanceCall>(&Call)) {
       const NoItParamFn *Handler0 = NoIterParamFunctions.lookup(Call);
       if (Handler0) {
-        (this->**Handler0)(C, InstCall->getCXXThisVal());
+        (this->**Handler0)(C, InstCall->getCXXThisVal(),
+                           InstCall->getCXXThisExpr());
         return;
       }
 
@@ -259,7 +257,7 @@ void ContainerModeling::checkDeadSymbols(SymbolReaper &SR,
 }
 
 void ContainerModeling::handleBegin(CheckerContext &C, const Expr *CE,
-                                   const SVal &RetVal, const SVal &Cont) const {
+                                   SVal RetVal, SVal Cont) const {
   const auto *ContReg = Cont.getAsRegion();
   if (!ContReg)
     return;
@@ -281,7 +279,7 @@ void ContainerModeling::handleBegin(CheckerContext &C, const Expr *CE,
 }
 
 void ContainerModeling::handleEnd(CheckerContext &C, const Expr *CE,
-                                 const SVal &RetVal, const SVal &Cont) const {
+                                 SVal RetVal, SVal Cont) const {
   const auto *ContReg = Cont.getAsRegion();
   if (!ContReg)
     return;
@@ -302,9 +300,8 @@ void ContainerModeling::handleEnd(CheckerContext &C, const Expr *CE,
   C.addTransition(State);
 }
 
-void ContainerModeling::handleAssignment(CheckerContext &C, const SVal &Cont,
-                                         const Expr *CE,
-                                         const SVal &OldCont) const {
+void ContainerModeling::handleAssignment(CheckerContext &C, SVal Cont,
+                                         const Expr *CE, SVal OldCont) const {
   const auto *ContReg = Cont.getAsRegion();
   if (!ContReg)
     return;
@@ -379,8 +376,8 @@ void ContainerModeling::handleAssignment(CheckerContext &C, const SVal &Cont,
   C.addTransition(State);
 }
 
-void ContainerModeling::handleAssign(CheckerContext &C,
-                                     const SVal &Cont) const {
+void ContainerModeling::handleAssign(CheckerContext &C, SVal Cont,
+                                     const Expr *ContE) const {
   const auto *ContReg = Cont.getAsRegion();
   if (!ContReg)
     return;
@@ -393,7 +390,8 @@ void ContainerModeling::handleAssign(CheckerContext &C,
   C.addTransition(State);
 }
 
-void ContainerModeling::handleClear(CheckerContext &C, const SVal &Cont) const {
+void ContainerModeling::handleClear(CheckerContext &C, SVal Cont,
+                                    const Expr *ContE) const {
   const auto *ContReg = Cont.getAsRegion();
   if (!ContReg)
     return;
@@ -415,12 +413,14 @@ void ContainerModeling::handleClear(CheckerContext &C, const SVal &Cont) const {
       }
     }
   }
+  const NoteTag *ChangeTag =
+    getChangeTag(C, "became empty", ContReg, ContE);
   State = invalidateAllIteratorPositions(State, ContReg);
-  C.addTransition(State);
+  C.addTransition(State, ChangeTag);
 }
 
-void ContainerModeling::handlePushBack(CheckerContext &C,
-                                      const SVal &Cont) const {
+void ContainerModeling::handlePushBack(CheckerContext &C, SVal Cont,
+                                       const Expr *ContE) const {
   const auto *ContReg = Cont.getAsRegion();
   if (!ContReg)
     return;
@@ -452,13 +452,15 @@ void ContainerModeling::handlePushBack(CheckerContext &C,
                     nonloc::SymbolVal(EndSym),
                     nonloc::ConcreteInt(BVF.getValue(llvm::APSInt::get(1))),
                     SymMgr.getType(EndSym)).getAsSymbol();
+    const NoteTag *ChangeTag =
+      getChangeTag(C, "extended to the back by 1 position", ContReg, ContE);
     State = setContainerData(State, ContReg, CData->newEnd(newEndSym));
+    C.addTransition(State, ChangeTag);
   }
-  C.addTransition(State);
 }
 
-void ContainerModeling::handlePopBack(CheckerContext &C,
-                                     const SVal &Cont) const {
+void ContainerModeling::handlePopBack(CheckerContext &C, SVal Cont,
+                                      const Expr *ContE) const {
   const auto *ContReg = Cont.getAsRegion();
   if (!ContReg)
     return;
@@ -479,6 +481,8 @@ void ContainerModeling::handlePopBack(CheckerContext &C,
                     nonloc::SymbolVal(EndSym),
                     nonloc::ConcreteInt(BVF.getValue(llvm::APSInt::get(1))),
                     SymMgr.getType(EndSym)).getAsSymbol();
+    const NoteTag *ChangeTag =
+      getChangeTag(C, "shrank from the back by 1 position", ContReg, ContE);
     // For vector-like and deque-like containers invalidate the last and the
     // past-end iterator positions. For list-like containers only invalidate
     // the last position
@@ -491,12 +495,12 @@ void ContainerModeling::handlePopBack(CheckerContext &C,
     }
     auto newEndSym = BackSym;
     State = setContainerData(State, ContReg, CData->newEnd(newEndSym));
-    C.addTransition(State);
+    C.addTransition(State, ChangeTag);
   }
 }
 
-void ContainerModeling::handlePushFront(CheckerContext &C,
-                                       const SVal &Cont) const {
+void ContainerModeling::handlePushFront(CheckerContext &C, SVal Cont,
+                                        const Expr *ContE) const {
   const auto *ContReg = Cont.getAsRegion();
   if (!ContReg)
     return;
@@ -522,14 +526,16 @@ void ContainerModeling::handlePushFront(CheckerContext &C,
                       nonloc::SymbolVal(BeginSym),
                       nonloc::ConcreteInt(BVF.getValue(llvm::APSInt::get(1))),
                       SymMgr.getType(BeginSym)).getAsSymbol();
+      const NoteTag *ChangeTag =
+        getChangeTag(C, "extended to the front by 1 position", ContReg, ContE);
       State = setContainerData(State, ContReg, CData->newBegin(newBeginSym));
-      C.addTransition(State);
+      C.addTransition(State, ChangeTag);
     }
   }
 }
 
-void ContainerModeling::handlePopFront(CheckerContext &C,
-                                      const SVal &Cont) const {
+void ContainerModeling::handlePopFront(CheckerContext &C, SVal Cont,
+                                       const Expr *ContE) const {
   const auto *ContReg = Cont.getAsRegion();
   if (!ContReg)
     return;
@@ -557,13 +563,15 @@ void ContainerModeling::handlePopFront(CheckerContext &C,
                     nonloc::SymbolVal(BeginSym),
                     nonloc::ConcreteInt(BVF.getValue(llvm::APSInt::get(1))),
                     SymMgr.getType(BeginSym)).getAsSymbol();
+    const NoteTag *ChangeTag =
+      getChangeTag(C, "shrank from the front by 1 position", ContReg, ContE);
     State = setContainerData(State, ContReg, CData->newBegin(newBeginSym));
-    C.addTransition(State);
+    C.addTransition(State, ChangeTag);
   }
 }
 
-void ContainerModeling::handleInsert(CheckerContext &C, const SVal &Cont,
-                                     const SVal &Iter) const {
+void ContainerModeling::handleInsert(CheckerContext &C, SVal Cont,
+                                     SVal Iter) const {
   const auto *ContReg = Cont.getAsRegion();
   if (!ContReg)
     return;
@@ -593,8 +601,8 @@ void ContainerModeling::handleInsert(CheckerContext &C, const SVal &Cont,
   }
 }
 
-void ContainerModeling::handleErase(CheckerContext &C, const SVal &Cont,
-                                    const SVal &Iter) const {
+void ContainerModeling::handleErase(CheckerContext &C, SVal Cont,
+                                    SVal Iter) const {
   const auto *ContReg = Cont.getAsRegion();
   if (!ContReg)
     return;
@@ -627,9 +635,8 @@ void ContainerModeling::handleErase(CheckerContext &C, const SVal &Cont,
   C.addTransition(State);
 }
 
-void ContainerModeling::handleErase(CheckerContext &C, const SVal &Cont,
-                                    const SVal &Iter1,
-                                    const SVal &Iter2) const {
+void ContainerModeling::handleErase(CheckerContext &C, SVal Cont, SVal Iter1,
+                                    SVal Iter2) const {
   const auto *ContReg = Cont.getAsRegion();
   if (!ContReg)
     return;
@@ -664,8 +671,8 @@ void ContainerModeling::handleErase(CheckerContext &C, const SVal &Cont,
   C.addTransition(State);
 }
 
-void ContainerModeling::handleEraseAfter(CheckerContext &C, const SVal &Cont,
-                                        const SVal &Iter) const {
+void ContainerModeling::handleEraseAfter(CheckerContext &C, SVal Cont,
+                                        SVal Iter) const {
   auto State = C.getState();
   const auto *Pos = getIteratorPosition(State, Iter);
   if (!Pos)
@@ -685,9 +692,8 @@ void ContainerModeling::handleEraseAfter(CheckerContext &C, const SVal &Cont,
   C.addTransition(State);
 }
 
-void ContainerModeling::handleEraseAfter(CheckerContext &C, const SVal &Cont,
-                                         const SVal &Iter1,
-                                         const SVal &Iter2) const {
+void ContainerModeling::handleEraseAfter(CheckerContext &C, SVal Cont,
+                                         SVal Iter1, SVal Iter2) const {
   auto State = C.getState();
   const auto *Pos1 = getIteratorPosition(State, Iter1);
   const auto *Pos2 = getIteratorPosition(State, Iter2);
@@ -700,6 +706,33 @@ void ContainerModeling::handleEraseAfter(CheckerContext &C, const SVal &Cont,
   C.addTransition(State);
 }
 
+const NoteTag *ContainerModeling::getChangeTag(CheckerContext &C,
+                                               StringRef Text,
+                                               const MemRegion *ContReg,
+                                               const Expr *ContE) const {
+  StringRef Name;
+  // First try to get the name of the variable from the region
+  if (const auto *DR = dyn_cast<DeclRegion>(ContReg)) {
+    Name = DR->getDecl()->getName();
+  // If the region is not a `DeclRegion` then use the expression instead
+  } else if (const auto *DRE =
+             dyn_cast<DeclRefExpr>(ContE->IgnoreParenCasts())) {
+    Name = DRE->getDecl()->getName();
+  }
+
+  return C.getNoteTag(
+      [Text, Name, ContReg](PathSensitiveBugReport &BR) -> std::string {
+        if (!BR.isInteresting(ContReg))
+          return "";
+
+        SmallString<256> Msg;
+        llvm::raw_svector_ostream Out(Msg);
+        Out << "Container " << (!Name.empty() ? ("'" + Name.str() + "' ") : "" )
+            << Text;
+        return std::string(Out.str());
+      });
+}
+
 void ContainerModeling::printState(raw_ostream &Out, ProgramStateRef State,
                                   const char *NL, const char *Sep) const {
   auto ContMap = State->get<ContainerMap>();
@@ -1035,6 +1068,16 @@ void ento::registerContainerModeling(CheckerManager &mgr) {
   mgr.registerChecker<ContainerModeling>();
 }
 
-bool ento::shouldRegisterContainerModeling(const LangOptions &LO) {
+bool ento::shouldRegisterContainerModeling(const CheckerManager &mgr) {
+  if (!mgr.getLangOpts().CPlusPlus)
+    return false;
+
+  if (!mgr.getAnalyzerOptions().ShouldAggressivelySimplifyBinaryOperation) {
+    mgr.getASTContext().getDiagnostics().Report(
+        diag::err_analyzer_checker_incompatible_analyzer_option)
+      << "aggressive-binary-operation-simplification" << "false";
+    return false;
+  }
+
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/ConversionChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ConversionChecker.cpp
index 8dd3132f07e2b..4216a68831192 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ConversionChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ConversionChecker.cpp
@@ -196,6 +196,6 @@ void ento::registerConversionChecker(CheckerManager &mgr) {
   mgr.registerChecker<ConversionChecker>();
 }
 
-bool ento::shouldRegisterConversionChecker(const LangOptions &LO) {
+bool ento::shouldRegisterConversionChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/DeadStoresChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/DeadStoresChecker.cpp
index 61441889fc649..6bc186aa27558 100644
--- a/clang/lib/StaticAnalyzer/Checkers/DeadStoresChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/DeadStoresChecker.cpp
@@ -540,6 +540,6 @@ void ento::registerDeadStoresChecker(CheckerManager &Mgr) {
       AnOpts.getCheckerBooleanOption(Chk, "ShowFixIts");
 }
 
-bool ento::shouldRegisterDeadStoresChecker(const LangOptions &LO) {
+bool ento::shouldRegisterDeadStoresChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/DebugCheckers.cpp b/clang/lib/StaticAnalyzer/Checkers/DebugCheckers.cpp
index 0cb4be2c7fdc3..e93c1e9b0a6a1 100644
--- a/clang/lib/StaticAnalyzer/Checkers/DebugCheckers.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/DebugCheckers.cpp
@@ -47,7 +47,7 @@ void ento::registerDominatorsTreeDumper(CheckerManager &mgr) {
   mgr.registerChecker<DominatorsTreeDumper>();
 }
 
-bool ento::shouldRegisterDominatorsTreeDumper(const LangOptions &LO) {
+bool ento::shouldRegisterDominatorsTreeDumper(const CheckerManager &mgr) {
   return true;
 }
 
@@ -73,7 +73,7 @@ void ento::registerPostDominatorsTreeDumper(CheckerManager &mgr) {
   mgr.registerChecker<PostDominatorsTreeDumper>();
 }
 
-bool ento::shouldRegisterPostDominatorsTreeDumper(const LangOptions &LO) {
+bool ento::shouldRegisterPostDominatorsTreeDumper(const CheckerManager &mgr) {
   return true;
 }
 
@@ -98,7 +98,7 @@ void ento::registerControlDependencyTreeDumper(CheckerManager &mgr) {
   mgr.registerChecker<ControlDependencyTreeDumper>();
 }
 
-bool ento::shouldRegisterControlDependencyTreeDumper(const LangOptions &LO) {
+bool ento::shouldRegisterControlDependencyTreeDumper(const CheckerManager &mgr) {
   return true;
 }
 
@@ -122,7 +122,7 @@ void ento::registerLiveVariablesDumper(CheckerManager &mgr) {
   mgr.registerChecker<LiveVariablesDumper>();
 }
 
-bool ento::shouldRegisterLiveVariablesDumper(const LangOptions &LO) {
+bool ento::shouldRegisterLiveVariablesDumper(const CheckerManager &mgr) {
   return true;
 }
 
@@ -145,7 +145,7 @@ void ento::registerLiveStatementsDumper(CheckerManager &mgr) {
   mgr.registerChecker<LiveStatementsDumper>();
 }
 
-bool ento::shouldRegisterLiveStatementsDumper(const LangOptions &LO) {
+bool ento::shouldRegisterLiveStatementsDumper(const CheckerManager &mgr) {
   return true;
 }
 
@@ -169,7 +169,7 @@ void ento::registerCFGViewer(CheckerManager &mgr) {
   mgr.registerChecker<CFGViewer>();
 }
 
-bool ento::shouldRegisterCFGViewer(const LangOptions &LO) {
+bool ento::shouldRegisterCFGViewer(const CheckerManager &mgr) {
   return true;
 }
 
@@ -199,7 +199,7 @@ void ento::registerCFGDumper(CheckerManager &mgr) {
   mgr.registerChecker<CFGDumper>();
 }
 
-bool ento::shouldRegisterCFGDumper(const LangOptions &LO) {
+bool ento::shouldRegisterCFGDumper(const CheckerManager &mgr) {
   return true;
 }
 
@@ -223,7 +223,7 @@ void ento::registerCallGraphViewer(CheckerManager &mgr) {
   mgr.registerChecker<CallGraphViewer>();
 }
 
-bool ento::shouldRegisterCallGraphViewer(const LangOptions &LO) {
+bool ento::shouldRegisterCallGraphViewer(const CheckerManager &mgr) {
   return true;
 }
 
@@ -247,7 +247,7 @@ void ento::registerCallGraphDumper(CheckerManager &mgr) {
   mgr.registerChecker<CallGraphDumper>();
 }
 
-bool ento::shouldRegisterCallGraphDumper(const LangOptions &LO) {
+bool ento::shouldRegisterCallGraphDumper(const CheckerManager &mgr) {
   return true;
 }
 
@@ -291,7 +291,7 @@ void ento::registerConfigDumper(CheckerManager &mgr) {
   mgr.registerChecker<ConfigDumper>();
 }
 
-bool ento::shouldRegisterConfigDumper(const LangOptions &LO) {
+bool ento::shouldRegisterConfigDumper(const CheckerManager &mgr) {
   return true;
 }
 
@@ -314,7 +314,7 @@ void ento::registerExplodedGraphViewer(CheckerManager &mgr) {
   mgr.registerChecker<ExplodedGraphViewer>();
 }
 
-bool ento::shouldRegisterExplodedGraphViewer(const LangOptions &LO) {
+bool ento::shouldRegisterExplodedGraphViewer(const CheckerManager &mgr) {
   return true;
 }
 
@@ -346,6 +346,6 @@ void ento::registerReportStmts(CheckerManager &mgr) {
   mgr.registerChecker<ReportStmts>();
 }
 
-bool ento::shouldRegisterReportStmts(const LangOptions &LO) {
+bool ento::shouldRegisterReportStmts(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/DebugContainerModeling.cpp b/clang/lib/StaticAnalyzer/Checkers/DebugContainerModeling.cpp
index 8d0572723991a..6fed999ffc80c 100644
--- a/clang/lib/StaticAnalyzer/Checkers/DebugContainerModeling.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/DebugContainerModeling.cpp
@@ -92,7 +92,19 @@ void DebugContainerModeling::analyzerContainerDataField(const CallExpr *CE,
       if (Field) {
         State = State->BindExpr(CE, C.getLocationContext(),
                                 nonloc::SymbolVal(Field));
-        C.addTransition(State);
+
+        // Progpagate interestingness from the container's data (marked
+        // interesting by an `ExprInspection` debug call to the container
+        // itself.
+        const NoteTag *InterestingTag =
+          C.getNoteTag(
+              [Cont, Field](PathSensitiveBugReport &BR) -> std::string {
+                if (BR.isInteresting(Field)) {
+                  BR.markInteresting(Cont);
+                }
+                return "";
+              });
+        C.addTransition(State, InterestingTag);
         return;
       }
     }
@@ -133,6 +145,6 @@ void ento::registerDebugContainerModeling(CheckerManager &mgr) {
   mgr.registerChecker<DebugContainerModeling>();
 }
 
-bool ento::shouldRegisterDebugContainerModeling(const LangOptions &LO) {
+bool ento::shouldRegisterDebugContainerModeling(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/DebugIteratorModeling.cpp b/clang/lib/StaticAnalyzer/Checkers/DebugIteratorModeling.cpp
index 254e51094b2a7..5833eea56da88 100644
--- a/clang/lib/StaticAnalyzer/Checkers/DebugIteratorModeling.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/DebugIteratorModeling.cpp
@@ -139,6 +139,6 @@ void ento::registerDebugIteratorModeling(CheckerManager &mgr) {
   mgr.registerChecker<DebugIteratorModeling>();
 }
 
-bool ento::shouldRegisterDebugIteratorModeling(const LangOptions &LO) {
+bool ento::shouldRegisterDebugIteratorModeling(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/DeleteWithNonVirtualDtorChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/DeleteWithNonVirtualDtorChecker.cpp
index 45c1984c5e157..97556ca856a0f 100644
--- a/clang/lib/StaticAnalyzer/Checkers/DeleteWithNonVirtualDtorChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/DeleteWithNonVirtualDtorChecker.cpp
@@ -148,6 +148,6 @@ void ento::registerDeleteWithNonVirtualDtorChecker(CheckerManager &mgr) {
 }
 
 bool ento::shouldRegisterDeleteWithNonVirtualDtorChecker(
-                                                        const LangOptions &LO) {
+                                                    const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/DereferenceChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/DereferenceChecker.cpp
index 46100cd1dace0..2411f0e2d058f 100644
--- a/clang/lib/StaticAnalyzer/Checkers/DereferenceChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/DereferenceChecker.cpp
@@ -304,6 +304,6 @@ void ento::registerDereferenceChecker(CheckerManager &mgr) {
   mgr.registerChecker<DereferenceChecker>();
 }
 
-bool ento::shouldRegisterDereferenceChecker(const LangOptions &LO) {
+bool ento::shouldRegisterDereferenceChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/DirectIvarAssignment.cpp b/clang/lib/StaticAnalyzer/Checkers/DirectIvarAssignment.cpp
index 0c46447e19853..d09f0da3faacd 100644
--- a/clang/lib/StaticAnalyzer/Checkers/DirectIvarAssignment.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/DirectIvarAssignment.cpp
@@ -222,7 +222,7 @@ void ento::registerDirectIvarAssignment(CheckerManager &mgr) {
   mgr.registerChecker<DirectIvarAssignment>();
 }
 
-bool ento::shouldRegisterDirectIvarAssignment(const LangOptions &LO) {
+bool ento::shouldRegisterDirectIvarAssignment(const CheckerManager &mgr) {
   return true;
 }
 
@@ -232,6 +232,6 @@ void ento::registerDirectIvarAssignmentForAnnotatedFunctions(
 }
 
 bool ento::shouldRegisterDirectIvarAssignmentForAnnotatedFunctions(
-                                                        const LangOptions &LO) {
+                                                    const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/DivZeroChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/DivZeroChecker.cpp
index 8798bde88dcd6..2b3164ba4a2c8 100644
--- a/clang/lib/StaticAnalyzer/Checkers/DivZeroChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/DivZeroChecker.cpp
@@ -101,6 +101,6 @@ void ento::registerDivZeroChecker(CheckerManager &mgr) {
   mgr.registerChecker<DivZeroChecker>();
 }
 
-bool ento::shouldRegisterDivZeroChecker(const LangOptions &LO) {
+bool ento::shouldRegisterDivZeroChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/DynamicTypeChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/DynamicTypeChecker.cpp
index 8cc38f9735f33..dbc930d7d37b7 100644
--- a/clang/lib/StaticAnalyzer/Checkers/DynamicTypeChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/DynamicTypeChecker.cpp
@@ -203,6 +203,6 @@ void ento::registerDynamicTypeChecker(CheckerManager &mgr) {
   mgr.registerChecker<DynamicTypeChecker>();
 }
 
-bool ento::shouldRegisterDynamicTypeChecker(const LangOptions &LO) {
+bool ento::shouldRegisterDynamicTypeChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/DynamicTypePropagation.cpp b/clang/lib/StaticAnalyzer/Checkers/DynamicTypePropagation.cpp
index cce3449b8873f..71681594a0a14 100644
--- a/clang/lib/StaticAnalyzer/Checkers/DynamicTypePropagation.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/DynamicTypePropagation.cpp
@@ -981,7 +981,7 @@ void ento::registerObjCGenericsChecker(CheckerManager &mgr) {
   checker->CheckGenerics = true;
 }
 
-bool ento::shouldRegisterObjCGenericsChecker(const LangOptions &LO) {
+bool ento::shouldRegisterObjCGenericsChecker(const CheckerManager &mgr) {
   return true;
 }
 
@@ -989,6 +989,6 @@ void ento::registerDynamicTypePropagation(CheckerManager &mgr) {
   mgr.registerChecker<DynamicTypePropagation>();
 }
 
-bool ento::shouldRegisterDynamicTypePropagation(const LangOptions &LO) {
+bool ento::shouldRegisterDynamicTypePropagation(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/EnumCastOutOfRangeChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/EnumCastOutOfRangeChecker.cpp
index 481a5685a71f6..0e94b915a468f 100644
--- a/clang/lib/StaticAnalyzer/Checkers/EnumCastOutOfRangeChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/EnumCastOutOfRangeChecker.cpp
@@ -142,6 +142,6 @@ void ento::registerEnumCastOutOfRangeChecker(CheckerManager &mgr) {
   mgr.registerChecker<EnumCastOutOfRangeChecker>();
 }
 
-bool ento::shouldRegisterEnumCastOutOfRangeChecker(const LangOptions &LO) {
+bool ento::shouldRegisterEnumCastOutOfRangeChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/ExprInspectionChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ExprInspectionChecker.cpp
index 10b27831d89f8..4225d890c47a5 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ExprInspectionChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ExprInspectionChecker.cpp
@@ -52,9 +52,12 @@ class ExprInspectionChecker : public Checker<eval::Call, check::DeadSymbols,
   typedef void (ExprInspectionChecker::*FnCheck)(const CallExpr *,
                                                  CheckerContext &C) const;
 
-  ExplodedNode *reportBug(llvm::StringRef Msg, CheckerContext &C) const;
+  // Optional parameter `ExprVal` for expression value to be marked interesting.
+  ExplodedNode *reportBug(llvm::StringRef Msg, CheckerContext &C,
+                          Optional<SVal> ExprVal = None) const;
   ExplodedNode *reportBug(llvm::StringRef Msg, BugReporter &BR,
-                          ExplodedNode *N) const;
+                          ExplodedNode *N,
+                          Optional<SVal> ExprVal = None) const;
 
 public:
   bool evalCall(const CallEvent &Call, CheckerContext &C) const;
@@ -144,22 +147,28 @@ static const char *getArgumentValueString(const CallExpr *CE,
 }
 
 ExplodedNode *ExprInspectionChecker::reportBug(llvm::StringRef Msg,
-                                               CheckerContext &C) const {
+                                               CheckerContext &C,
+                                               Optional<SVal> ExprVal) const {
   ExplodedNode *N = C.generateNonFatalErrorNode();
-  reportBug(Msg, C.getBugReporter(), N);
+  reportBug(Msg, C.getBugReporter(), N, ExprVal);
   return N;
 }
 
 ExplodedNode *ExprInspectionChecker::reportBug(llvm::StringRef Msg,
                                                BugReporter &BR,
-                                               ExplodedNode *N) const {
+                                               ExplodedNode *N,
+                                               Optional<SVal> ExprVal) const {
   if (!N)
     return nullptr;
 
   if (!BT)
     BT.reset(new BugType(this, "Checking analyzer assumptions", "debug"));
 
-  BR.emitReport(std::make_unique<PathSensitiveBugReport>(*BT, Msg, N));
+  auto R = std::make_unique<PathSensitiveBugReport>(*BT, Msg, N);
+  if (ExprVal) {
+    R->markInteresting(*ExprVal);
+  }
+  BR.emitReport(std::move(R));
   return N;
 }
 
@@ -406,7 +415,8 @@ void ExprInspectionChecker::analyzerExpress(const CallExpr *CE,
     return;
   }
 
-  SymbolRef Sym = C.getSVal(CE->getArg(0)).getAsSymbol();
+  SVal ArgVal = C.getSVal(CE->getArg(0));
+  SymbolRef Sym = ArgVal.getAsSymbol();
   if (!Sym) {
     reportBug("Not a symbol", C);
     return;
@@ -419,7 +429,7 @@ void ExprInspectionChecker::analyzerExpress(const CallExpr *CE,
     return;
   }
 
-  reportBug(*Str, C);
+  reportBug(*Str, C, ArgVal);
 }
 
 void ExprInspectionChecker::analyzerIsTainted(const CallExpr *CE,
@@ -437,6 +447,6 @@ void ento::registerExprInspectionChecker(CheckerManager &Mgr) {
   Mgr.registerChecker<ExprInspectionChecker>();
 }
 
-bool ento::shouldRegisterExprInspectionChecker(const LangOptions &LO) {
+bool ento::shouldRegisterExprInspectionChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/FixedAddressChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/FixedAddressChecker.cpp
index b315a84522859..6275e49e51ae4 100644
--- a/clang/lib/StaticAnalyzer/Checkers/FixedAddressChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/FixedAddressChecker.cpp
@@ -66,6 +66,6 @@ void ento::registerFixedAddressChecker(CheckerManager &mgr) {
   mgr.registerChecker<FixedAddressChecker>();
 }
 
-bool ento::shouldRegisterFixedAddressChecker(const LangOptions &LO) {
+bool ento::shouldRegisterFixedAddressChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/FuchsiaHandleChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/FuchsiaHandleChecker.cpp
index dee2212cd5d83..78c51f3eab010 100644
--- a/clang/lib/StaticAnalyzer/Checkers/FuchsiaHandleChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/FuchsiaHandleChecker.cpp
@@ -565,7 +565,7 @@ void ento::registerFuchsiaHandleChecker(CheckerManager &mgr) {
   mgr.registerChecker<FuchsiaHandleChecker>();
 }
 
-bool ento::shouldRegisterFuchsiaHandleChecker(const LangOptions &LO) {
+bool ento::shouldRegisterFuchsiaHandleChecker(const CheckerManager &mgr) {
   return true;
 }
 
diff --git a/clang/lib/StaticAnalyzer/Checkers/GCDAntipatternChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/GCDAntipatternChecker.cpp
index 76fa56406443a..63fbe75fd4983 100644
--- a/clang/lib/StaticAnalyzer/Checkers/GCDAntipatternChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/GCDAntipatternChecker.cpp
@@ -225,6 +225,6 @@ void ento::registerGCDAntipattern(CheckerManager &Mgr) {
   Mgr.registerChecker<GCDAntipatternChecker>();
 }
 
-bool ento::shouldRegisterGCDAntipattern(const LangOptions &LO) {
+bool ento::shouldRegisterGCDAntipattern(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/GTestChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/GTestChecker.cpp
index f4308f510f0ba..8d9afbe88aa84 100644
--- a/clang/lib/StaticAnalyzer/Checkers/GTestChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/GTestChecker.cpp
@@ -291,8 +291,9 @@ void ento::registerGTestChecker(CheckerManager &Mgr) {
   Mgr.registerChecker<GTestChecker>();
 }
 
-bool ento::shouldRegisterGTestChecker(const LangOptions &LO) {
+bool ento::shouldRegisterGTestChecker(const CheckerManager &mgr) {
   // gtest is a C++ API so there is no sense running the checker
   // if not compiling for C++.
+  const LangOptions &LO = mgr.getLangOpts();
   return LO.CPlusPlus;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/GenericTaintChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/GenericTaintChecker.cpp
index 3dca242f15c3d..1f3e749892293 100644
--- a/clang/lib/StaticAnalyzer/Checkers/GenericTaintChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/GenericTaintChecker.cpp
@@ -948,6 +948,6 @@ void ento::registerGenericTaintChecker(CheckerManager &Mgr) {
     Checker->parseConfiguration(Mgr, Option, std::move(Config.getValue()));
 }
 
-bool ento::shouldRegisterGenericTaintChecker(const LangOptions &LO) {
+bool ento::shouldRegisterGenericTaintChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/IdenticalExprChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/IdenticalExprChecker.cpp
index cc2cfb7742270..1cf81b54e77d3 100644
--- a/clang/lib/StaticAnalyzer/Checkers/IdenticalExprChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/IdenticalExprChecker.cpp
@@ -351,6 +351,8 @@ static bool isIdenticalStmt(const ASTContext &Ctx, const Stmt *Stmt1,
   case Stmt::CallExprClass:
   case Stmt::ArraySubscriptExprClass:
   case Stmt::OMPArraySectionExprClass:
+  case Stmt::OMPArrayShapingExprClass:
+  case Stmt::OMPIteratorExprClass:
   case Stmt::ImplicitCastExprClass:
   case Stmt::ParenExprClass:
   case Stmt::BreakStmtClass:
@@ -513,6 +515,6 @@ void ento::registerIdenticalExprChecker(CheckerManager &Mgr) {
   Mgr.registerChecker<FindIdenticalExprChecker>();
 }
 
-bool ento::shouldRegisterIdenticalExprChecker(const LangOptions &LO) {
+bool ento::shouldRegisterIdenticalExprChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/InnerPointerChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/InnerPointerChecker.cpp
index dd89c53478e8a..65e52e139ee41 100644
--- a/clang/lib/StaticAnalyzer/Checkers/InnerPointerChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/InnerPointerChecker.cpp
@@ -307,6 +307,6 @@ void ento::registerInnerPointerChecker(CheckerManager &Mgr) {
   Mgr.registerChecker<InnerPointerChecker>();
 }
 
-bool ento::shouldRegisterInnerPointerChecker(const LangOptions &LO) {
+bool ento::shouldRegisterInnerPointerChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/InvalidatedIteratorChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/InvalidatedIteratorChecker.cpp
index d1a9a7df071d7..c6cd11b09a4ca 100644
--- a/clang/lib/StaticAnalyzer/Checkers/InvalidatedIteratorChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/InvalidatedIteratorChecker.cpp
@@ -90,6 +90,6 @@ void ento::registerInvalidatedIteratorChecker(CheckerManager &mgr) {
   mgr.registerChecker<InvalidatedIteratorChecker>();
 }
 
-bool ento::shouldRegisterInvalidatedIteratorChecker(const LangOptions &LO) {
+bool ento::shouldRegisterInvalidatedIteratorChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/IteratorModeling.cpp b/clang/lib/StaticAnalyzer/Checkers/IteratorModeling.cpp
index 9a813b0a1a456..2850c6d393226 100644
--- a/clang/lib/StaticAnalyzer/Checkers/IteratorModeling.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/IteratorModeling.cpp
@@ -717,6 +717,6 @@ void ento::registerIteratorModeling(CheckerManager &mgr) {
   mgr.registerChecker<IteratorModeling>();
 }
 
-bool ento::shouldRegisterIteratorModeling(const LangOptions &LO) {
+bool ento::shouldRegisterIteratorModeling(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/IteratorRangeChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/IteratorRangeChecker.cpp
index f9b493bf9bb03..6e86ef0c552a9 100644
--- a/clang/lib/StaticAnalyzer/Checkers/IteratorRangeChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/IteratorRangeChecker.cpp
@@ -221,7 +221,12 @@ void IteratorRangeChecker::reportBug(const StringRef &Message, SVal Val,
                                      ExplodedNode *ErrNode) const {
   auto R = std::make_unique<PathSensitiveBugReport>(*OutOfRangeBugType, Message,
                                                     ErrNode);
+
+  const auto *Pos = getIteratorPosition(C.getState(), Val);
+  assert(Pos && "Iterator without known position cannot be out-of-range.");
+
   R->markInteresting(Val);
+  R->markInteresting(Pos->getContainer());
   C.emitReport(std::move(R));
 }
 
@@ -304,6 +309,6 @@ void ento::registerIteratorRangeChecker(CheckerManager &mgr) {
   mgr.registerChecker<IteratorRangeChecker>();
 }
 
-bool ento::shouldRegisterIteratorRangeChecker(const LangOptions &LO) {
+bool ento::shouldRegisterIteratorRangeChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/IvarInvalidationChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/IvarInvalidationChecker.cpp
index 0d64fbd6f62ec..3e6756efe0e69 100644
--- a/clang/lib/StaticAnalyzer/Checkers/IvarInvalidationChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/IvarInvalidationChecker.cpp
@@ -739,7 +739,7 @@ void ento::registerIvarInvalidationModeling(CheckerManager &mgr) {
   mgr.registerChecker<IvarInvalidationChecker>();
 }
 
-bool ento::shouldRegisterIvarInvalidationModeling(const LangOptions &LO) {
+bool ento::shouldRegisterIvarInvalidationModeling(const CheckerManager &mgr) {
   return true;
 }
 
@@ -751,7 +751,7 @@ bool ento::shouldRegisterIvarInvalidationModeling(const LangOptions &LO) {
     checker->Filter.checkName_##name = mgr.getCurrentCheckerName();            \
   }                                                                            \
                                                                                \
-  bool ento::shouldRegister##name(const LangOptions &LO) { return true; }
+  bool ento::shouldRegister##name(const CheckerManager &mgr) { return true; }
 
 REGISTER_CHECKER(InstanceVariableInvalidation)
 REGISTER_CHECKER(MissingInvalidationMethod)
diff --git a/clang/lib/StaticAnalyzer/Checkers/LLVMConventionsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/LLVMConventionsChecker.cpp
index 7522fdd0a99b3..1f3d8844d330e 100644
--- a/clang/lib/StaticAnalyzer/Checkers/LLVMConventionsChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/LLVMConventionsChecker.cpp
@@ -314,6 +314,6 @@ void ento::registerLLVMConventionsChecker(CheckerManager &mgr) {
   mgr.registerChecker<LLVMConventionsChecker>();
 }
 
-bool ento::shouldRegisterLLVMConventionsChecker(const LangOptions &LO) {
+bool ento::shouldRegisterLLVMConventionsChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/LocalizationChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/LocalizationChecker.cpp
index 79de1844e7452..252377f24bd7b 100644
--- a/clang/lib/StaticAnalyzer/Checkers/LocalizationChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/LocalizationChecker.cpp
@@ -1403,7 +1403,7 @@ void ento::registerNonLocalizedStringChecker(CheckerManager &mgr) {
           checker, "AggressiveReport");
 }
 
-bool ento::shouldRegisterNonLocalizedStringChecker(const LangOptions &LO) {
+bool ento::shouldRegisterNonLocalizedStringChecker(const CheckerManager &mgr) {
   return true;
 }
 
@@ -1412,7 +1412,7 @@ void ento::registerEmptyLocalizationContextChecker(CheckerManager &mgr) {
 }
 
 bool ento::shouldRegisterEmptyLocalizationContextChecker(
-                                                        const LangOptions &LO) {
+                                                    const CheckerManager &mgr) {
   return true;
 }
 
@@ -1420,6 +1420,6 @@ void ento::registerPluralMisuseChecker(CheckerManager &mgr) {
   mgr.registerChecker<PluralMisuseChecker>();
 }
 
-bool ento::shouldRegisterPluralMisuseChecker(const LangOptions &LO) {
+bool ento::shouldRegisterPluralMisuseChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/MIGChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MIGChecker.cpp
index d7555b78ac40b..837213875a60c 100644
--- a/clang/lib/StaticAnalyzer/Checkers/MIGChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/MIGChecker.cpp
@@ -293,6 +293,6 @@ void ento::registerMIGChecker(CheckerManager &Mgr) {
   Mgr.registerChecker<MIGChecker>();
 }
 
-bool ento::shouldRegisterMIGChecker(const LangOptions &LO) {
+bool ento::shouldRegisterMIGChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/MPI-Checker/MPIChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MPI-Checker/MPIChecker.cpp
index cde5ee3cbac39..7ac7a38dacf33 100644
--- a/clang/lib/StaticAnalyzer/Checkers/MPI-Checker/MPIChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/MPI-Checker/MPIChecker.cpp
@@ -190,6 +190,6 @@ void clang::ento::registerMPIChecker(CheckerManager &MGR) {
   MGR.registerChecker<clang::ento::mpi::MPIChecker>();
 }
 
-bool clang::ento::shouldRegisterMPIChecker(const LangOptions &LO) {
+bool clang::ento::shouldRegisterMPIChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/MacOSKeychainAPIChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MacOSKeychainAPIChecker.cpp
index e064ca6bd88f6..43f7dcd14b014 100644
--- a/clang/lib/StaticAnalyzer/Checkers/MacOSKeychainAPIChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/MacOSKeychainAPIChecker.cpp
@@ -667,6 +667,6 @@ void ento::registerMacOSKeychainAPIChecker(CheckerManager &mgr) {
   mgr.registerChecker<MacOSKeychainAPIChecker>();
 }
 
-bool ento::shouldRegisterMacOSKeychainAPIChecker(const LangOptions &LO) {
+bool ento::shouldRegisterMacOSKeychainAPIChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/MacOSXAPIChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MacOSXAPIChecker.cpp
index 410721d8b6ff6..04e7f8dec8d77 100644
--- a/clang/lib/StaticAnalyzer/Checkers/MacOSXAPIChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/MacOSXAPIChecker.cpp
@@ -176,6 +176,6 @@ void ento::registerMacOSXAPIChecker(CheckerManager &mgr) {
   mgr.registerChecker<MacOSXAPIChecker>();
 }
 
-bool ento::shouldRegisterMacOSXAPIChecker(const LangOptions &LO) {
+bool ento::shouldRegisterMacOSXAPIChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
index 23b4abc670798..52786bcaf072f 100644
--- a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
@@ -58,19 +58,23 @@
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerHelpers.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/DynamicSize.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ProgramStateTrait.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState_Fwd.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/SymbolManager.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/ErrorHandling.h"
 #include <climits>
+#include <functional>
 #include <utility>
 
 using namespace clang;
 using namespace ento;
+using namespace std::placeholders;
 
 //===----------------------------------------------------------------------===//
 // The types of allocation we're modeling. This is used to check whether a
@@ -92,8 +96,6 @@ enum AllocationFamily {
   AF_InnerBuffer
 };
 
-struct MemFunctionInfoTy;
-
 } // end of anonymous namespace
 
 /// Print names of allocators and deallocators.
@@ -262,47 +264,6 @@ struct ReallocPair {
 
 REGISTER_MAP_WITH_PROGRAMSTATE(ReallocPairs, SymbolRef, ReallocPair)
 
-//===----------------------------------------------------------------------===//
-// Kinds of memory operations, information about resource managing functions.
-//===----------------------------------------------------------------------===//
-
-namespace {
-
-struct MemFunctionInfoTy {
-  /// The value of the MallocChecker:Optimistic is stored in this variable.
-  ///
-  /// In pessimistic mode, the checker assumes that it does not know which
-  /// functions might free the memory.
-  /// In optimistic mode, the checker assumes that all user-defined functions
-  /// which might free a pointer are annotated.
-  DefaultBool ShouldIncludeOwnershipAnnotatedFunctions;
-
-  CallDescription CD_alloca{{"alloca"}, 1}, CD_win_alloca{{"_alloca"}, 1},
-      CD_malloc{{"malloc"}, 1}, CD_BSD_malloc{{"malloc"}, 3},
-      CD_free{{"free"}, 1}, CD_realloc{{"realloc"}, 2},
-      CD_calloc{{"calloc"}, 2}, CD_valloc{{"valloc"}, 1},
-      CD_reallocf{{"reallocf"}, 2}, CD_strndup{{"strndup"}, 2},
-      CD_strdup{{"strdup"}, 1}, CD_win_strdup{{"_strdup"}, 1},
-      CD_kmalloc{{"kmalloc"}, 2}, CD_if_nameindex{{"if_nameindex"}, 1},
-      CD_if_freenameindex{{"if_freenameindex"}, 1}, CD_wcsdup{{"wcsdup"}, 1},
-      CD_win_wcsdup{{"_wcsdup"}, 1}, CD_kfree{{"kfree"}, 2},
-      CD_g_malloc{{"g_malloc"}, 1}, CD_g_malloc0{{"g_malloc0"}, 1},
-      CD_g_realloc{{"g_realloc"}, 2}, CD_g_try_malloc{{"g_try_malloc"}, 1},
-      CD_g_try_malloc0{{"g_try_malloc0"}, 1},
-      CD_g_try_realloc{{"g_try_realloc"}, 2}, CD_g_free{{"g_free"}, 1},
-      CD_g_memdup{{"g_memdup"}, 2}, CD_g_malloc_n{{"g_malloc_n"}, 2},
-      CD_g_malloc0_n{{"g_malloc0_n"}, 2}, CD_g_realloc_n{{"g_realloc_n"}, 3},
-      CD_g_try_malloc_n{{"g_try_malloc_n"}, 2},
-      CD_g_try_malloc0_n{{"g_try_malloc0_n"}, 2},
-      CD_g_try_realloc_n{{"g_try_realloc_n"}, 3};
-
-  bool isMemFunction(const CallEvent &Call) const;
-  bool isCMemFunction(const CallEvent &Call) const;
-  bool isCMemFreeFunction(const CallEvent &Call) const;
-  bool isCMemAllocFunction(const CallEvent &Call) const;
-};
-} // end of anonymous namespace
-
 /// Tells if the callee is one of the builtin new/delete operators, including
 /// placement operators and other standard overloads.
 static bool isStandardNewDelete(const FunctionDecl *FD);
@@ -326,7 +287,11 @@ class MallocChecker
                      check::PreStmt<CXXDeleteExpr>, check::PostStmt<BlockExpr>,
                      check::PostObjCMessage, check::Location, eval::Assume> {
 public:
-  MemFunctionInfoTy MemFunctionInfo;
+  /// In pessimistic mode, the checker assumes that it does not know which
+  /// functions might free the memory.
+  /// In optimistic mode, the checker assumes that all user-defined functions
+  /// which might free a pointer are annotated.
+  DefaultBool ShouldIncludeOwnershipAnnotatedFunctions;
 
   /// Many checkers are essentially built into this one, so enabling them will
   /// make MallocChecker perform additional modeling and reporting.
@@ -386,9 +351,91 @@ class MallocChecker
   mutable std::unique_ptr<BugType> BT_OffsetFree[CK_NumCheckKinds];
   mutable std::unique_ptr<BugType> BT_UseZerroAllocated[CK_NumCheckKinds];
 
+#define CHECK_FN(NAME)                                                         \
+  void NAME(CheckerContext &C, const CallExpr *CE, ProgramStateRef State) const;
+
+  CHECK_FN(checkFree)
+  CHECK_FN(checkIfNameIndex)
+  CHECK_FN(checkBasicAlloc)
+  CHECK_FN(checkKernelMalloc)
+  CHECK_FN(checkCalloc)
+  CHECK_FN(checkAlloca)
+  CHECK_FN(checkStrdup)
+  CHECK_FN(checkIfFreeNameIndex)
+  CHECK_FN(checkCXXNewOrCXXDelete)
+  CHECK_FN(checkGMalloc0)
+  CHECK_FN(checkGMemdup)
+  CHECK_FN(checkGMallocN)
+  CHECK_FN(checkGMallocN0)
+  CHECK_FN(checkReallocN)
+  CHECK_FN(checkOwnershipAttr)
+
+  void checkRealloc(CheckerContext &C, const CallExpr *CE,
+                    ProgramStateRef State, bool ShouldFreeOnFail) const;
+
+  using CheckFn =
+      std::function<void(const MallocChecker *, CheckerContext &C,
+                         const CallExpr *CE, ProgramStateRef State)>;
+
+  const CallDescriptionMap<CheckFn> FreeingMemFnMap{
+      {{"free", 1}, &MallocChecker::checkFree},
+      {{"if_freenameindex", 1}, &MallocChecker::checkIfFreeNameIndex},
+      {{"kfree", 1}, &MallocChecker::checkFree},
+      {{"g_free", 1}, &MallocChecker::checkFree},
+  };
+
+  bool isFreeingCall(const CallEvent &Call) const;
+
+  CallDescriptionMap<CheckFn> AllocatingMemFnMap{
+      {{"alloca", 1}, &MallocChecker::checkAlloca},
+      {{"_alloca", 1}, &MallocChecker::checkAlloca},
+      {{"malloc", 1}, &MallocChecker::checkBasicAlloc},
+      {{"malloc", 3}, &MallocChecker::checkKernelMalloc},
+      {{"calloc", 2}, &MallocChecker::checkCalloc},
+      {{"valloc", 1}, &MallocChecker::checkBasicAlloc},
+      {{CDF_MaybeBuiltin, "strndup", 2}, &MallocChecker::checkStrdup},
+      {{CDF_MaybeBuiltin, "strdup", 1}, &MallocChecker::checkStrdup},
+      {{"_strdup", 1}, &MallocChecker::checkStrdup},
+      {{"kmalloc", 2}, &MallocChecker::checkKernelMalloc},
+      {{"if_nameindex", 1}, &MallocChecker::checkIfNameIndex},
+      {{CDF_MaybeBuiltin, "wcsdup", 1}, &MallocChecker::checkStrdup},
+      {{CDF_MaybeBuiltin, "_wcsdup", 1}, &MallocChecker::checkStrdup},
+      {{"g_malloc", 1}, &MallocChecker::checkBasicAlloc},
+      {{"g_malloc0", 1}, &MallocChecker::checkGMalloc0},
+      {{"g_try_malloc", 1}, &MallocChecker::checkBasicAlloc},
+      {{"g_try_malloc0", 1}, &MallocChecker::checkGMalloc0},
+      {{"g_memdup", 2}, &MallocChecker::checkGMemdup},
+      {{"g_malloc_n", 2}, &MallocChecker::checkGMallocN},
+      {{"g_malloc0_n", 2}, &MallocChecker::checkGMallocN0},
+      {{"g_try_malloc_n", 2}, &MallocChecker::checkGMallocN},
+      {{"g_try_malloc0_n", 2}, &MallocChecker::checkGMallocN0},
+  };
+
+  CallDescriptionMap<CheckFn> ReallocatingMemFnMap{
+      {{"realloc", 2},
+       std::bind(&MallocChecker::checkRealloc, _1, _2, _3, _4, false)},
+      {{"reallocf", 2},
+       std::bind(&MallocChecker::checkRealloc, _1, _2, _3, _4, true)},
+      {{"g_realloc", 2},
+       std::bind(&MallocChecker::checkRealloc, _1, _2, _3, _4, false)},
+      {{"g_try_realloc", 2},
+       std::bind(&MallocChecker::checkRealloc, _1, _2, _3, _4, false)},
+      {{"g_realloc_n", 3}, &MallocChecker::checkReallocN},
+      {{"g_try_realloc_n", 3}, &MallocChecker::checkReallocN},
+  };
+
+  bool isMemCall(const CallEvent &Call) const;
+
   // TODO: Remove mutable by moving the initializtaion to the registry function.
   mutable Optional<uint64_t> KernelZeroFlagVal;
 
+  using KernelZeroSizePtrValueTy = Optional<int>;
+  /// Store the value of macro called `ZERO_SIZE_PTR`.
+  /// The value is initialized at first use, before first use the outer
+  /// Optional is empty, afterwards it contains another Optional that indicates
+  /// if the macro value could be determined, and if yes the value itself.
+  mutable Optional<KernelZeroSizePtrValueTy> KernelZeroSizePtrValue;
+
   /// Process C++ operator new()'s allocation, which is the part of C++
   /// new-expression that goes before the constructor.
   void processNewAllocation(const CXXNewExpr *NE, CheckerContext &C,
@@ -658,6 +705,10 @@ class MallocChecker
                                     CheckerContext &C);
 
   void reportLeak(SymbolRef Sym, ExplodedNode *N, CheckerContext &C) const;
+
+  /// Test if value in ArgVal equals to value in macro `ZERO_SIZE_PTR`.
+  bool isArgZERO_SIZE_PTR(ProgramStateRef State, CheckerContext &C,
+                          SVal ArgVal) const;
 };
 
 //===----------------------------------------------------------------------===//
@@ -810,28 +861,31 @@ class StopTrackingCallback final : public SymbolVisitor {
 };
 } // end anonymous namespace
 
-//===----------------------------------------------------------------------===//
-// Methods of MemFunctionInfoTy.
-//===----------------------------------------------------------------------===//
+static bool isStandardNewDelete(const FunctionDecl *FD) {
+  if (!FD)
+    return false;
 
-bool MemFunctionInfoTy::isMemFunction(const CallEvent &Call) const {
-  return isCMemFunction(Call) || isStandardNewDelete(Call);
-}
+  OverloadedOperatorKind Kind = FD->getOverloadedOperator();
+  if (Kind != OO_New && Kind != OO_Array_New && Kind != OO_Delete &&
+      Kind != OO_Array_Delete)
+    return false;
 
-bool MemFunctionInfoTy::isCMemFunction(const CallEvent &Call) const {
-  return isCMemFreeFunction(Call) || isCMemAllocFunction(Call);
+  // This is standard if and only if it's not defined in a user file.
+  SourceLocation L = FD->getLocation();
+  // If the header for operator delete is not included, it's still defined
+  // in an invalid source location. Check to make sure we don't crash.
+  return !L.isValid() ||
+         FD->getASTContext().getSourceManager().isInSystemHeader(L);
 }
 
-bool MemFunctionInfoTy::isCMemFreeFunction(const CallEvent &Call) const {
-  if (Call.isCalled(CD_free, CD_realloc, CD_reallocf, CD_g_free, CD_kfree))
-    return true;
+//===----------------------------------------------------------------------===//
+// Methods of MallocChecker and MallocBugVisitor.
+//===----------------------------------------------------------------------===//
 
-  if (Call.isCalled(CD_if_freenameindex))
+bool MallocChecker::isFreeingCall(const CallEvent &Call) const {
+  if (FreeingMemFnMap.lookup(Call) || ReallocatingMemFnMap.lookup(Call))
     return true;
 
-  if (!ShouldIncludeOwnershipAnnotatedFunctions)
-    return false;
-
   const auto *Func = dyn_cast<FunctionDecl>(Call.getDecl());
   if (Func && Func->hasAttrs()) {
     for (const auto *I : Func->specific_attrs<OwnershipAttr>()) {
@@ -843,60 +897,21 @@ bool MemFunctionInfoTy::isCMemFreeFunction(const CallEvent &Call) const {
   return false;
 }
 
-bool MemFunctionInfoTy::isCMemAllocFunction(const CallEvent &Call) const {
-  if (Call.isCalled(CD_malloc, CD_realloc, CD_reallocf, CD_calloc, CD_valloc,
-                    CD_strdup, CD_win_strdup, CD_strndup, CD_wcsdup,
-                    CD_win_wcsdup, CD_kmalloc, CD_g_malloc, CD_g_malloc0,
-                    CD_g_realloc, CD_g_try_malloc, CD_g_try_malloc0,
-                    CD_g_try_realloc, CD_g_memdup, CD_g_malloc_n,
-                    CD_g_malloc0_n, CD_g_realloc_n, CD_g_try_malloc_n,
-                    CD_g_try_malloc0_n, CD_g_try_realloc_n))
-    return true;
-
-  if (Call.isCalled(CD_if_nameindex))
-    return true;
-
-  if (Call.isCalled(CD_alloca, CD_win_alloca))
+bool MallocChecker::isMemCall(const CallEvent &Call) const {
+  if (FreeingMemFnMap.lookup(Call) || AllocatingMemFnMap.lookup(Call) ||
+      ReallocatingMemFnMap.lookup(Call))
     return true;
 
   if (!ShouldIncludeOwnershipAnnotatedFunctions)
     return false;
 
   const auto *Func = dyn_cast<FunctionDecl>(Call.getDecl());
-  if (Func && Func->hasAttrs()) {
-    for (const auto *I : Func->specific_attrs<OwnershipAttr>()) {
-      OwnershipAttr::OwnershipKind OwnKind = I->getOwnKind();
-      if (OwnKind == OwnershipAttr::Returns)
-        return true;
-    }
-  }
-
-  return false;
-}
-
-static bool isStandardNewDelete(const FunctionDecl *FD) {
-  if (!FD)
-    return false;
-
-  OverloadedOperatorKind Kind = FD->getOverloadedOperator();
-  if (Kind != OO_New && Kind != OO_Array_New && Kind != OO_Delete &&
-      Kind != OO_Array_Delete)
-    return false;
-
-  // This is standard if and only if it's not defined in a user file.
-  SourceLocation L = FD->getLocation();
-  // If the header for operator delete is not included, it's still defined
-  // in an invalid source location. Check to make sure we don't crash.
-  return !L.isValid() ||
-         FD->getASTContext().getSourceManager().isInSystemHeader(L);
+  return Func && Func->hasAttr<OwnershipAttr>();
 }
 
-//===----------------------------------------------------------------------===//
-// Methods of MallocChecker and MallocBugVisitor.
-//===----------------------------------------------------------------------===//
-
-llvm::Optional<ProgramStateRef> MallocChecker::performKernelMalloc(
-  const CallExpr *CE, CheckerContext &C, const ProgramStateRef &State) const {
+llvm::Optional<ProgramStateRef>
+MallocChecker::performKernelMalloc(const CallExpr *CE, CheckerContext &C,
+                                   const ProgramStateRef &State) const {
   // 3-argument malloc(), as commonly used in {Free,Net,Open}BSD Kernels:
   //
   // void *malloc(unsigned long size, struct malloc_type *mtp, int flags);
@@ -984,173 +999,169 @@ SVal MallocChecker::evalMulForBufferSize(CheckerContext &C, const Expr *Blocks,
   return TotalSize;
 }
 
-void MallocChecker::checkPostCall(const CallEvent &Call,
-                                  CheckerContext &C) const {
-  if (C.wasInlined)
-    return;
+void MallocChecker::checkBasicAlloc(CheckerContext &C, const CallExpr *CE,
+                                    ProgramStateRef State) const {
+  State = MallocMemAux(C, CE, CE->getArg(0), UndefinedVal(), State, AF_Malloc);
+  State = ProcessZeroAllocCheck(C, CE, 0, State);
+  C.addTransition(State);
+}
 
-  const auto *CE = dyn_cast_or_null<CallExpr>(Call.getOriginExpr());
-  if (!CE)
-    return;
+void MallocChecker::checkKernelMalloc(CheckerContext &C, const CallExpr *CE,
+                                      ProgramStateRef State) const {
+  llvm::Optional<ProgramStateRef> MaybeState =
+      performKernelMalloc(CE, C, State);
+  if (MaybeState.hasValue())
+    State = MaybeState.getValue();
+  else
+    State =
+        MallocMemAux(C, CE, CE->getArg(0), UndefinedVal(), State, AF_Malloc);
+  C.addTransition(State);
+}
 
-  const FunctionDecl *FD = C.getCalleeDecl(CE);
-  if (!FD)
+void MallocChecker::checkRealloc(CheckerContext &C, const CallExpr *CE,
+                                 ProgramStateRef State,
+                                 bool ShouldFreeOnFail) const {
+  State = ReallocMemAux(C, CE, ShouldFreeOnFail, State, AF_Malloc);
+  State = ProcessZeroAllocCheck(C, CE, 1, State);
+  C.addTransition(State);
+}
+
+void MallocChecker::checkCalloc(CheckerContext &C, const CallExpr *CE,
+                                ProgramStateRef State) const {
+  State = CallocMem(C, CE, State);
+  State = ProcessZeroAllocCheck(C, CE, 0, State);
+  State = ProcessZeroAllocCheck(C, CE, 1, State);
+  C.addTransition(State);
+}
+
+void MallocChecker::checkFree(CheckerContext &C, const CallExpr *CE,
+                              ProgramStateRef State) const {
+  bool IsKnownToBeAllocatedMemory = false;
+  if (suppressDeallocationsInSuspiciousContexts(CE, C))
     return;
+  State =
+      FreeMemAux(C, CE, State, 0, false, IsKnownToBeAllocatedMemory, AF_Malloc);
+  C.addTransition(State);
+}
 
-  ProgramStateRef State = C.getState();
+void MallocChecker::checkAlloca(CheckerContext &C, const CallExpr *CE,
+                                ProgramStateRef State) const {
+  State = MallocMemAux(C, CE, CE->getArg(0), UndefinedVal(), State, AF_Alloca);
+  State = ProcessZeroAllocCheck(C, CE, 0, State);
+  C.addTransition(State);
+}
+
+void MallocChecker::checkStrdup(CheckerContext &C, const CallExpr *CE,
+                                ProgramStateRef State) const {
+  State = MallocUpdateRefState(C, CE, State, AF_Malloc);
+
+  C.addTransition(State);
+}
+
+void MallocChecker::checkIfNameIndex(CheckerContext &C, const CallExpr *CE,
+                                     ProgramStateRef State) const {
+  // Should we model this differently? We can allocate a fixed number of
+  // elements with zeros in the last one.
+  State =
+      MallocMemAux(C, CE, UnknownVal(), UnknownVal(), State, AF_IfNameIndex);
+
+  C.addTransition(State);
+}
+
+void MallocChecker::checkIfFreeNameIndex(CheckerContext &C, const CallExpr *CE,
+                                         ProgramStateRef State) const {
   bool IsKnownToBeAllocatedMemory = false;
+  State = FreeMemAux(C, CE, State, 0, false, IsKnownToBeAllocatedMemory,
+                     AF_IfNameIndex);
+  C.addTransition(State);
+}
 
-  if (FD->getKind() == Decl::Function) {
-    if (Call.isCalled(MemFunctionInfo.CD_malloc, MemFunctionInfo.CD_BSD_malloc,
-                      MemFunctionInfo.CD_g_malloc,
-                      MemFunctionInfo.CD_g_try_malloc)) {
-      switch (CE->getNumArgs()) {
-      default:
-        return;
-      case 1:
-        State = MallocMemAux(C, CE, CE->getArg(0), UndefinedVal(), State,
-                             AF_Malloc);
-        State = ProcessZeroAllocCheck(C, CE, 0, State);
-        break;
-      case 2:
-        llvm_unreachable("There shouldn't be a 2-argument malloc!");
-        break;
-      case 3:
-        llvm::Optional<ProgramStateRef> MaybeState =
-          performKernelMalloc(CE, C, State);
-        if (MaybeState.hasValue())
-          State = MaybeState.getValue();
-        else
-          State = MallocMemAux(C, CE, CE->getArg(0), UndefinedVal(), State,
-                               AF_Malloc);
-        break;
-      }
-    } else if (Call.isCalled(MemFunctionInfo.CD_kmalloc)) {
-      if (CE->getNumArgs() < 1)
-        return;
-      llvm::Optional<ProgramStateRef> MaybeState =
-          performKernelMalloc(CE, C, State);
-      if (MaybeState.hasValue())
-        State = MaybeState.getValue();
-      else
-        State = MallocMemAux(C, CE, CE->getArg(0), UndefinedVal(), State,
-                             AF_Malloc);
-    } else if (Call.isCalled(MemFunctionInfo.CD_valloc)) {
-      if (CE->getNumArgs() < 1)
-        return;
-      State =
-          MallocMemAux(C, CE, CE->getArg(0), UndefinedVal(), State, AF_Malloc);
-      State = ProcessZeroAllocCheck(C, CE, 0, State);
-    } else if (Call.isCalled(MemFunctionInfo.CD_realloc,
-                             MemFunctionInfo.CD_g_realloc,
-                             MemFunctionInfo.CD_g_try_realloc)) {
-      State =
-          ReallocMemAux(C, CE, /*ShouldFreeOnFail*/ false, State, AF_Malloc);
-      State = ProcessZeroAllocCheck(C, CE, 1, State);
-    } else if (Call.isCalled(MemFunctionInfo.CD_reallocf)) {
-      State = ReallocMemAux(C, CE, /*ShouldFreeOnFail*/ true, State, AF_Malloc);
-      State = ProcessZeroAllocCheck(C, CE, 1, State);
-    } else if (Call.isCalled(MemFunctionInfo.CD_calloc)) {
-      State = CallocMem(C, CE, State);
-      State = ProcessZeroAllocCheck(C, CE, 0, State);
-      State = ProcessZeroAllocCheck(C, CE, 1, State);
-    } else if (Call.isCalled(MemFunctionInfo.CD_free, MemFunctionInfo.CD_g_free,
-                             MemFunctionInfo.CD_kfree)) {
-      if (suppressDeallocationsInSuspiciousContexts(CE, C))
-        return;
+void MallocChecker::checkCXXNewOrCXXDelete(CheckerContext &C,
+                                           const CallExpr *CE,
+                                           ProgramStateRef State) const {
+  bool IsKnownToBeAllocatedMemory = false;
 
-      State = FreeMemAux(C, CE, State, 0, false, IsKnownToBeAllocatedMemory,
-                         AF_Malloc);
-    } else if (Call.isCalled(
-                   MemFunctionInfo.CD_strdup, MemFunctionInfo.CD_win_strdup,
-                   MemFunctionInfo.CD_wcsdup, MemFunctionInfo.CD_win_wcsdup)) {
-      State = MallocUpdateRefState(C, CE, State, AF_Malloc);
-    } else if (Call.isCalled(MemFunctionInfo.CD_strndup)) {
-      State = MallocUpdateRefState(C, CE, State, AF_Malloc);
-    } else if (Call.isCalled(MemFunctionInfo.CD_alloca,
-                             MemFunctionInfo.CD_win_alloca)) {
-      if (CE->getNumArgs() < 1)
-        return;
-      State =
-          MallocMemAux(C, CE, CE->getArg(0), UndefinedVal(), State, AF_Alloca);
-      State = ProcessZeroAllocCheck(C, CE, 0, State);
-    } else if (isStandardNewDelete(FD)) {
-      // Process direct calls to operator new/new[]/delete/delete[] functions
-      // as distinct from new/new[]/delete/delete[] expressions that are
-      // processed by the checkPostStmt callbacks for CXXNewExpr and
-      // CXXDeleteExpr.
-      switch (FD->getOverloadedOperator()) {
-      case OO_New:
-        State = MallocMemAux(C, CE, CE->getArg(0), UndefinedVal(), State,
-                             AF_CXXNew);
-        State = ProcessZeroAllocCheck(C, CE, 0, State);
-        break;
-      case OO_Array_New:
-        State = MallocMemAux(C, CE, CE->getArg(0), UndefinedVal(), State,
-                             AF_CXXNewArray);
-        State = ProcessZeroAllocCheck(C, CE, 0, State);
-        break;
-      case OO_Delete:
-        State = FreeMemAux(C, CE, State, 0, false, IsKnownToBeAllocatedMemory,
-                           AF_CXXNew);
-        break;
-      case OO_Array_Delete:
-        State = FreeMemAux(C, CE, State, 0, false, IsKnownToBeAllocatedMemory,
-                           AF_CXXNewArray);
-        break;
-      default:
-        llvm_unreachable("not a new/delete operator");
-      }
-    } else if (Call.isCalled(MemFunctionInfo.CD_if_nameindex)) {
-      // Should we model this differently? We can allocate a fixed number of
-      // elements with zeros in the last one.
-      State = MallocMemAux(C, CE, UnknownVal(), UnknownVal(), State,
-                           AF_IfNameIndex);
-    } else if (Call.isCalled(MemFunctionInfo.CD_if_freenameindex)) {
-      State = FreeMemAux(C, CE, State, 0, false, IsKnownToBeAllocatedMemory,
-                         AF_IfNameIndex);
-    } else if (Call.isCalled(MemFunctionInfo.CD_g_malloc0,
-                             MemFunctionInfo.CD_g_try_malloc0)) {
-      if (CE->getNumArgs() < 1)
-        return;
-      SValBuilder &svalBuilder = C.getSValBuilder();
-      SVal zeroVal = svalBuilder.makeZeroVal(svalBuilder.getContext().CharTy);
-      State = MallocMemAux(C, CE, CE->getArg(0), zeroVal, State, AF_Malloc);
-      State = ProcessZeroAllocCheck(C, CE, 0, State);
-    } else if (Call.isCalled(MemFunctionInfo.CD_g_memdup)) {
-      if (CE->getNumArgs() < 2)
-        return;
-      State =
-          MallocMemAux(C, CE, CE->getArg(1), UndefinedVal(), State, AF_Malloc);
-      State = ProcessZeroAllocCheck(C, CE, 1, State);
-    } else if (Call.isCalled(MemFunctionInfo.CD_g_malloc_n,
-                             MemFunctionInfo.CD_g_try_malloc_n,
-                             MemFunctionInfo.CD_g_malloc0_n,
-                             MemFunctionInfo.CD_g_try_malloc0_n)) {
-      if (CE->getNumArgs() < 2)
-        return;
-      SVal Init = UndefinedVal();
-      if (Call.isCalled(MemFunctionInfo.CD_g_malloc0_n,
-                        MemFunctionInfo.CD_g_try_malloc0_n)) {
-        SValBuilder &SB = C.getSValBuilder();
-        Init = SB.makeZeroVal(SB.getContext().CharTy);
-      }
-      SVal TotalSize = evalMulForBufferSize(C, CE->getArg(0), CE->getArg(1));
-      State = MallocMemAux(C, CE, TotalSize, Init, State, AF_Malloc);
-      State = ProcessZeroAllocCheck(C, CE, 0, State);
-      State = ProcessZeroAllocCheck(C, CE, 1, State);
-    } else if (Call.isCalled(MemFunctionInfo.CD_g_realloc_n,
-                             MemFunctionInfo.CD_g_try_realloc_n)) {
-      if (CE->getNumArgs() < 3)
-        return;
-      State = ReallocMemAux(C, CE, /*ShouldFreeOnFail*/ false, State, AF_Malloc,
-                            /*SuffixWithN*/ true);
-      State = ProcessZeroAllocCheck(C, CE, 1, State);
-      State = ProcessZeroAllocCheck(C, CE, 2, State);
-    }
+  const FunctionDecl *FD = C.getCalleeDecl(CE);
+  // Process direct calls to operator new/new[]/delete/delete[] functions
+  // as distinct from new/new[]/delete/delete[] expressions that are
+  // processed by the checkPostStmt callbacks for CXXNewExpr and
+  // CXXDeleteExpr.
+  switch (FD->getOverloadedOperator()) {
+  case OO_New:
+    State =
+        MallocMemAux(C, CE, CE->getArg(0), UndefinedVal(), State, AF_CXXNew);
+    State = ProcessZeroAllocCheck(C, CE, 0, State);
+    break;
+  case OO_Array_New:
+    State = MallocMemAux(C, CE, CE->getArg(0), UndefinedVal(), State,
+                         AF_CXXNewArray);
+    State = ProcessZeroAllocCheck(C, CE, 0, State);
+    break;
+  case OO_Delete:
+    State = FreeMemAux(C, CE, State, 0, false, IsKnownToBeAllocatedMemory,
+                       AF_CXXNew);
+    break;
+  case OO_Array_Delete:
+    State = FreeMemAux(C, CE, State, 0, false, IsKnownToBeAllocatedMemory,
+                       AF_CXXNewArray);
+    break;
+  default:
+    llvm_unreachable("not a new/delete operator");
   }
 
-  if (MemFunctionInfo.ShouldIncludeOwnershipAnnotatedFunctions ||
+  C.addTransition(State);
+}
+
+void MallocChecker::checkGMalloc0(CheckerContext &C, const CallExpr *CE,
+                                  ProgramStateRef State) const {
+  SValBuilder &svalBuilder = C.getSValBuilder();
+  SVal zeroVal = svalBuilder.makeZeroVal(svalBuilder.getContext().CharTy);
+  State = MallocMemAux(C, CE, CE->getArg(0), zeroVal, State, AF_Malloc);
+  State = ProcessZeroAllocCheck(C, CE, 0, State);
+  C.addTransition(State);
+}
+
+void MallocChecker::checkGMemdup(CheckerContext &C, const CallExpr *CE,
+                                 ProgramStateRef State) const {
+  State = MallocMemAux(C, CE, CE->getArg(1), UndefinedVal(), State, AF_Malloc);
+  State = ProcessZeroAllocCheck(C, CE, 1, State);
+  C.addTransition(State);
+}
+
+void MallocChecker::checkGMallocN(CheckerContext &C, const CallExpr *CE,
+                                  ProgramStateRef State) const {
+  SVal Init = UndefinedVal();
+  SVal TotalSize = evalMulForBufferSize(C, CE->getArg(0), CE->getArg(1));
+  State = MallocMemAux(C, CE, TotalSize, Init, State, AF_Malloc);
+  State = ProcessZeroAllocCheck(C, CE, 0, State);
+  State = ProcessZeroAllocCheck(C, CE, 1, State);
+  C.addTransition(State);
+}
+
+void MallocChecker::checkGMallocN0(CheckerContext &C, const CallExpr *CE,
+                                   ProgramStateRef State) const {
+  SValBuilder &SB = C.getSValBuilder();
+  SVal Init = SB.makeZeroVal(SB.getContext().CharTy);
+  SVal TotalSize = evalMulForBufferSize(C, CE->getArg(0), CE->getArg(1));
+  State = MallocMemAux(C, CE, TotalSize, Init, State, AF_Malloc);
+  State = ProcessZeroAllocCheck(C, CE, 0, State);
+  State = ProcessZeroAllocCheck(C, CE, 1, State);
+  C.addTransition(State);
+}
+
+void MallocChecker::checkReallocN(CheckerContext &C, const CallExpr *CE,
+                                  ProgramStateRef State) const {
+  State = ReallocMemAux(C, CE, /*ShouldFreeOnFail=*/false, State, AF_Malloc,
+                        /*SuffixWithN=*/true);
+  State = ProcessZeroAllocCheck(C, CE, 1, State);
+  State = ProcessZeroAllocCheck(C, CE, 2, State);
+  C.addTransition(State);
+}
+
+void MallocChecker::checkOwnershipAttr(CheckerContext &C, const CallExpr *CE,
+                                       ProgramStateRef State) const {
+  const FunctionDecl *FD = C.getCalleeDecl(CE);
+  if (ShouldIncludeOwnershipAnnotatedFunctions ||
       ChecksEnabled[CK_MismatchedDeallocatorChecker]) {
     // Check all the attributes, if there are any.
     // There can be multiple of these attributes.
@@ -1170,6 +1181,44 @@ void MallocChecker::checkPostCall(const CallEvent &Call,
   C.addTransition(State);
 }
 
+void MallocChecker::checkPostCall(const CallEvent &Call,
+                                  CheckerContext &C) const {
+  if (C.wasInlined)
+    return;
+
+  const auto *CE = dyn_cast_or_null<CallExpr>(Call.getOriginExpr());
+  if (!CE)
+    return;
+
+  const FunctionDecl *FD = C.getCalleeDecl(CE);
+  if (!FD)
+    return;
+
+  ProgramStateRef State = C.getState();
+
+  if (const CheckFn *Callback = FreeingMemFnMap.lookup(Call)) {
+    (*Callback)(this, C, CE, State);
+    return;
+  }
+
+  if (const CheckFn *Callback = AllocatingMemFnMap.lookup(Call)) {
+    (*Callback)(this, C, CE, State);
+    return;
+  }
+
+  if (const CheckFn *Callback = ReallocatingMemFnMap.lookup(Call)) {
+    (*Callback)(this, C, CE, State);
+    return;
+  }
+
+  if (isStandardNewDelete(Call)) {
+    checkCXXNewOrCXXDelete(C, CE, State);
+    return;
+  }
+
+  checkOwnershipAttr(C, CE, State);
+}
+
 // Performs a 0-sized allocations check.
 ProgramStateRef MallocChecker::ProcessZeroAllocCheck(
     CheckerContext &C, const Expr *E, const unsigned IndexOfSizeArg,
@@ -1436,8 +1485,7 @@ MallocChecker::MallocMemReturnsAttr(CheckerContext &C, const CallExpr *CE,
   if (!State)
     return nullptr;
 
-  if (Att->getModule()->getName() !=
-      MemFunctionInfo.CD_malloc.getFunctionName())
+  if (Att->getModule()->getName() != "malloc")
     return nullptr;
 
   OwnershipAttr::args_iterator I = Att->args_begin(), E = Att->args_end();
@@ -1533,8 +1581,7 @@ ProgramStateRef MallocChecker::FreeMemAttr(CheckerContext &C,
   if (!State)
     return nullptr;
 
-  if (Att->getModule()->getName() !=
-      MemFunctionInfo.CD_malloc.getFunctionName())
+  if (Att->getModule()->getName() != "malloc")
     return nullptr;
 
   bool IsKnownToBeAllocated = false;
@@ -1677,7 +1724,13 @@ ProgramStateRef MallocChecker::FreeMemAux(
   // Nonlocs can't be freed, of course.
   // Non-region locations (labels and fixed addresses) also shouldn't be freed.
   if (!R) {
-    ReportBadFree(C, ArgVal, ArgExpr->getSourceRange(), ParentExpr, Family);
+    // Exception:
+    // If the macro ZERO_SIZE_PTR is defined, this could be a kernel source
+    // code. In that case, the ZERO_SIZE_PTR defines a special value used for a
+    // zero-sized memory block which is allowed to be freed, despite not being a
+    // null pointer.
+    if (Family != AF_Malloc || !isArgZERO_SIZE_PTR(State, C, ArgVal))
+      ReportBadFree(C, ArgVal, ArgExpr->getSourceRange(), ParentExpr, Family);
     return nullptr;
   }
 
@@ -2579,8 +2632,7 @@ void MallocChecker::checkPreCall(const CallEvent &Call,
     if (!FD)
       return;
 
-    if (ChecksEnabled[CK_MallocChecker] &&
-        (MemFunctionInfo.isCMemFreeFunction(Call)))
+    if (ChecksEnabled[CK_MallocChecker] && isFreeingCall(Call))
       return;
   }
 
@@ -2879,7 +2931,7 @@ bool MallocChecker::mayFreeAnyEscapedMemoryOrIsModeledExplicitly(
 
   // If it's one of the allocation functions we can reason about, we model
   // its behavior explicitly.
-  if (MemFunctionInfo.isMemFunction(*Call))
+  if (isMemCall(*Call))
     return false;
 
   // If it's not a system call, assume it frees memory.
@@ -3023,6 +3075,18 @@ ProgramStateRef MallocChecker::checkPointerEscapeAux(
   return State;
 }
 
+bool MallocChecker::isArgZERO_SIZE_PTR(ProgramStateRef State, CheckerContext &C,
+                                       SVal ArgVal) const {
+  if (!KernelZeroSizePtrValue)
+    KernelZeroSizePtrValue =
+        tryExpandAsInteger("ZERO_SIZE_PTR", C.getPreprocessor());
+
+  const llvm::APSInt *ArgValKnown =
+      C.getSValBuilder().getKnownValue(State, ArgVal);
+  return ArgValKnown && *KernelZeroSizePtrValue &&
+         ArgValKnown->getSExtValue() == **KernelZeroSizePtrValue;
+}
+
 static SymbolRef findFailedReallocSymbol(ProgramStateRef currState,
                                          ProgramStateRef prevState) {
   ReallocPairsTy currMap = currState->get<ReallocPairs>();
@@ -3285,11 +3349,11 @@ void ento::registerInnerPointerCheckerAux(CheckerManager &mgr) {
 
 void ento::registerDynamicMemoryModeling(CheckerManager &mgr) {
   auto *checker = mgr.registerChecker<MallocChecker>();
-  checker->MemFunctionInfo.ShouldIncludeOwnershipAnnotatedFunctions =
+  checker->ShouldIncludeOwnershipAnnotatedFunctions =
       mgr.getAnalyzerOptions().getCheckerBooleanOption(checker, "Optimistic");
 }
 
-bool ento::shouldRegisterDynamicMemoryModeling(const LangOptions &LO) {
+bool ento::shouldRegisterDynamicMemoryModeling(const CheckerManager &mgr) {
   return true;
 }
 
@@ -3301,7 +3365,7 @@ bool ento::shouldRegisterDynamicMemoryModeling(const LangOptions &LO) {
         mgr.getCurrentCheckerName();                                           \
   }                                                                            \
                                                                                \
-  bool ento::shouldRegister##name(const LangOptions &LO) { return true; }
+  bool ento::shouldRegister##name(const CheckerManager &mgr) { return true; }
 
 REGISTER_CHECKER(MallocChecker)
 REGISTER_CHECKER(NewDeleteChecker)
diff --git a/clang/lib/StaticAnalyzer/Checkers/MallocOverflowSecurityChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MallocOverflowSecurityChecker.cpp
index 4fd06f24c5bc6..e31630f63b5ac 100644
--- a/clang/lib/StaticAnalyzer/Checkers/MallocOverflowSecurityChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/MallocOverflowSecurityChecker.cpp
@@ -337,6 +337,6 @@ void ento::registerMallocOverflowSecurityChecker(CheckerManager &mgr) {
   mgr.registerChecker<MallocOverflowSecurityChecker>();
 }
 
-bool ento::shouldRegisterMallocOverflowSecurityChecker(const LangOptions &LO) {
+bool ento::shouldRegisterMallocOverflowSecurityChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/MallocSizeofChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MallocSizeofChecker.cpp
index b5881a9e65338..71f593cb2b561 100644
--- a/clang/lib/StaticAnalyzer/Checkers/MallocSizeofChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/MallocSizeofChecker.cpp
@@ -250,6 +250,6 @@ void ento::registerMallocSizeofChecker(CheckerManager &mgr) {
   mgr.registerChecker<MallocSizeofChecker>();
 }
 
-bool ento::shouldRegisterMallocSizeofChecker(const LangOptions &LO) {
+bool ento::shouldRegisterMallocSizeofChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/MismatchedIteratorChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MismatchedIteratorChecker.cpp
index 143910588959e..d170b875856c2 100644
--- a/clang/lib/StaticAnalyzer/Checkers/MismatchedIteratorChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/MismatchedIteratorChecker.cpp
@@ -290,6 +290,6 @@ void ento::registerMismatchedIteratorChecker(CheckerManager &mgr) {
   mgr.registerChecker<MismatchedIteratorChecker>();
 }
 
-bool ento::shouldRegisterMismatchedIteratorChecker(const LangOptions &LO) {
+bool ento::shouldRegisterMismatchedIteratorChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/MmapWriteExecChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MmapWriteExecChecker.cpp
index ceea62160545e..5d63d6efd2341 100644
--- a/clang/lib/StaticAnalyzer/Checkers/MmapWriteExecChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/MmapWriteExecChecker.cpp
@@ -88,6 +88,6 @@ void ento::registerMmapWriteExecChecker(CheckerManager &mgr) {
       .getCheckerIntegerOption(Mwec, "MmapProtRead");
 }
 
-bool ento::shouldRegisterMmapWriteExecChecker(const LangOptions &LO) {
+bool ento::shouldRegisterMmapWriteExecChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/MoveChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MoveChecker.cpp
index 40eb113e3f8e9..7f0519c695b0f 100644
--- a/clang/lib/StaticAnalyzer/Checkers/MoveChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/MoveChecker.cpp
@@ -757,6 +757,6 @@ void ento::registerMoveChecker(CheckerManager &mgr) {
       mgr.getAnalyzerOptions().getCheckerStringOption(chk, "WarnOn"), mgr);
 }
 
-bool ento::shouldRegisterMoveChecker(const LangOptions &LO) {
+bool ento::shouldRegisterMoveChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/NSAutoreleasePoolChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/NSAutoreleasePoolChecker.cpp
index 41b7fe5e43b6f..be17e401fb534 100644
--- a/clang/lib/StaticAnalyzer/Checkers/NSAutoreleasePoolChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/NSAutoreleasePoolChecker.cpp
@@ -80,6 +80,7 @@ void ento::registerNSAutoreleasePoolChecker(CheckerManager &mgr) {
   mgr.registerChecker<NSAutoreleasePoolChecker>();
 }
 
-bool ento::shouldRegisterNSAutoreleasePoolChecker(const LangOptions &LO) {
+bool ento::shouldRegisterNSAutoreleasePoolChecker(const CheckerManager &mgr) { 
+  const LangOptions &LO = mgr.getLangOpts();
   return LO.getGC() != LangOptions::NonGC;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/NSErrorChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/NSErrorChecker.cpp
index 85370bf133cd7..2dee1e459a060 100644
--- a/clang/lib/StaticAnalyzer/Checkers/NSErrorChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/NSErrorChecker.cpp
@@ -312,7 +312,7 @@ void ento::registerNSOrCFErrorDerefChecker(CheckerManager &mgr) {
   mgr.registerChecker<NSOrCFErrorDerefChecker>();
 }
 
-bool ento::shouldRegisterNSOrCFErrorDerefChecker(const LangOptions &LO) {
+bool ento::shouldRegisterNSOrCFErrorDerefChecker(const CheckerManager &mgr) {
   return true;
 }
 
@@ -322,7 +322,7 @@ void ento::registerNSErrorChecker(CheckerManager &mgr) {
   checker->ShouldCheckNSError = true;
 }
 
-bool ento::shouldRegisterNSErrorChecker(const LangOptions &LO) {
+bool ento::shouldRegisterNSErrorChecker(const CheckerManager &mgr) {
   return true;
 }
 
@@ -332,6 +332,6 @@ void ento::registerCFErrorChecker(CheckerManager &mgr) {
   checker->ShouldCheckCFError = true;
 }
 
-bool ento::shouldRegisterCFErrorChecker(const LangOptions &LO) {
+bool ento::shouldRegisterCFErrorChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/NoReturnFunctionChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/NoReturnFunctionChecker.cpp
index fc34255bf6c95..af208e8673187 100644
--- a/clang/lib/StaticAnalyzer/Checkers/NoReturnFunctionChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/NoReturnFunctionChecker.cpp
@@ -143,6 +143,6 @@ void ento::registerNoReturnFunctionChecker(CheckerManager &mgr) {
   mgr.registerChecker<NoReturnFunctionChecker>();
 }
 
-bool ento::shouldRegisterNoReturnFunctionChecker(const LangOptions &LO) {
+bool ento::shouldRegisterNoReturnFunctionChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/NonNullParamChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/NonNullParamChecker.cpp
index 6ffc89745365c..8c7975a513a07 100644
--- a/clang/lib/StaticAnalyzer/Checkers/NonNullParamChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/NonNullParamChecker.cpp
@@ -226,6 +226,6 @@ void ento::registerNonNullParamChecker(CheckerManager &mgr) {
   mgr.registerChecker<NonNullParamChecker>();
 }
 
-bool ento::shouldRegisterNonNullParamChecker(const LangOptions &LO) {
+bool ento::shouldRegisterNonNullParamChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/NonnullGlobalConstantsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/NonnullGlobalConstantsChecker.cpp
index 6efba433eed22..80b705fb73926 100644
--- a/clang/lib/StaticAnalyzer/Checkers/NonnullGlobalConstantsChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/NonnullGlobalConstantsChecker.cpp
@@ -147,6 +147,6 @@ void ento::registerNonnullGlobalConstantsChecker(CheckerManager &Mgr) {
   Mgr.registerChecker<NonnullGlobalConstantsChecker>();
 }
 
-bool ento::shouldRegisterNonnullGlobalConstantsChecker(const LangOptions &LO) {
+bool ento::shouldRegisterNonnullGlobalConstantsChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/NullabilityChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/NullabilityChecker.cpp
index 922048733c7c4..0b8d100992a22 100644
--- a/clang/lib/StaticAnalyzer/Checkers/NullabilityChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/NullabilityChecker.cpp
@@ -508,13 +508,7 @@ void NullabilityChecker::checkEvent(ImplicitNullDerefEvent Event) const {
 /// return expressions of ObjC types when the return type of the function or
 /// method is non-null but the express is not.
 static const Expr *lookThroughImplicitCasts(const Expr *E) {
-  assert(E);
-
-  while (auto *ICE = dyn_cast<ImplicitCastExpr>(E)) {
-    E = ICE->getSubExpr();
-  }
-
-  return E;
+  return E->IgnoreImpCasts();
 }
 
 /// This method check when nullable pointer or null value is returned from a
@@ -1188,7 +1182,7 @@ void ento::registerNullabilityBase(CheckerManager &mgr) {
   mgr.registerChecker<NullabilityChecker>();
 }
 
-bool ento::shouldRegisterNullabilityBase(const LangOptions &LO) {
+bool ento::shouldRegisterNullabilityBase(const CheckerManager &mgr) {
   return true;
 }
 
@@ -1204,7 +1198,7 @@ bool ento::shouldRegisterNullabilityBase(const LangOptions &LO) {
             checker, "NoDiagnoseCallsToSystemHeaders", true);                  \
   }                                                                            \
                                                                                \
-  bool ento::shouldRegister##name##Checker(const LangOptions &LO) {            \
+  bool ento::shouldRegister##name##Checker(const CheckerManager &mgr) {            \
     return true;                                                               \
   }
 
diff --git a/clang/lib/StaticAnalyzer/Checkers/NumberObjectConversionChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/NumberObjectConversionChecker.cpp
index 1053424ae6faa..df69254ead263 100644
--- a/clang/lib/StaticAnalyzer/Checkers/NumberObjectConversionChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/NumberObjectConversionChecker.cpp
@@ -349,6 +349,6 @@ void ento::registerNumberObjectConversionChecker(CheckerManager &Mgr) {
       Mgr.getAnalyzerOptions().getCheckerBooleanOption(Chk, "Pedantic");
 }
 
-bool ento::shouldRegisterNumberObjectConversionChecker(const LangOptions &LO) {
+bool ento::shouldRegisterNumberObjectConversionChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/OSObjectCStyleCast.cpp b/clang/lib/StaticAnalyzer/Checkers/OSObjectCStyleCast.cpp
index 90d69b81305c8..53ed0e187a4ce 100644
--- a/clang/lib/StaticAnalyzer/Checkers/OSObjectCStyleCast.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/OSObjectCStyleCast.cpp
@@ -84,6 +84,6 @@ void ento::registerOSObjectCStyleCast(CheckerManager &Mgr) {
   Mgr.registerChecker<OSObjectCStyleCastChecker>();
 }
 
-bool ento::shouldRegisterOSObjectCStyleCast(const LangOptions &LO) {
+bool ento::shouldRegisterOSObjectCStyleCast(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/ObjCAtSyncChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ObjCAtSyncChecker.cpp
index 0e25817c87932..43af4bb142867 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ObjCAtSyncChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ObjCAtSyncChecker.cpp
@@ -91,6 +91,7 @@ void ento::registerObjCAtSyncChecker(CheckerManager &mgr) {
   mgr.registerChecker<ObjCAtSyncChecker>();
 }
 
-bool ento::shouldRegisterObjCAtSyncChecker(const LangOptions &LO) {
+bool ento::shouldRegisterObjCAtSyncChecker(const CheckerManager &mgr) {
+  const LangOptions &LO = mgr.getLangOpts();
   return LO.ObjC;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/ObjCAutoreleaseWriteChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ObjCAutoreleaseWriteChecker.cpp
index 9d587c5856500..f36e971bb8ba6 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ObjCAutoreleaseWriteChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ObjCAutoreleaseWriteChecker.cpp
@@ -206,6 +206,6 @@ void ento::registerAutoreleaseWriteChecker(CheckerManager &Mgr) {
   Mgr.registerChecker<ObjCAutoreleaseWriteChecker>();
 }
 
-bool ento::shouldRegisterAutoreleaseWriteChecker(const LangOptions &LO) {
+bool ento::shouldRegisterAutoreleaseWriteChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/ObjCContainersASTChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ObjCContainersASTChecker.cpp
index 4450c464f89d3..8428b2294ba6f 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ObjCContainersASTChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ObjCContainersASTChecker.cpp
@@ -172,6 +172,6 @@ void ento::registerObjCContainersASTChecker(CheckerManager &mgr) {
   mgr.registerChecker<ObjCContainersASTChecker>();
 }
 
-bool ento::shouldRegisterObjCContainersASTChecker(const LangOptions &LO) {
+bool ento::shouldRegisterObjCContainersASTChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/ObjCContainersChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ObjCContainersChecker.cpp
index 8abb926d4862f..ff19d7a2b9d4c 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ObjCContainersChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ObjCContainersChecker.cpp
@@ -188,6 +188,6 @@ void ento::registerObjCContainersChecker(CheckerManager &mgr) {
   mgr.registerChecker<ObjCContainersChecker>();
 }
 
-bool ento::shouldRegisterObjCContainersChecker(const LangOptions &LO) {
+bool ento::shouldRegisterObjCContainersChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/ObjCMissingSuperCallChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ObjCMissingSuperCallChecker.cpp
index 1870c08432de3..24e2a4dea922e 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ObjCMissingSuperCallChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ObjCMissingSuperCallChecker.cpp
@@ -221,7 +221,7 @@ void ento::registerObjCSuperCallChecker(CheckerManager &Mgr) {
   Mgr.registerChecker<ObjCSuperCallChecker>();
 }
 
-bool ento::shouldRegisterObjCSuperCallChecker(const LangOptions &LO) {
+bool ento::shouldRegisterObjCSuperCallChecker(const CheckerManager &mgr) {
   return true;
 }
 
diff --git a/clang/lib/StaticAnalyzer/Checkers/ObjCPropertyChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ObjCPropertyChecker.cpp
index 9a49200545e37..4636fd1605118 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ObjCPropertyChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ObjCPropertyChecker.cpp
@@ -79,6 +79,6 @@ void ento::registerObjCPropertyChecker(CheckerManager &Mgr) {
   Mgr.registerChecker<ObjCPropertyChecker>();
 }
 
-bool ento::shouldRegisterObjCPropertyChecker(const LangOptions &LO) {
+bool ento::shouldRegisterObjCPropertyChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/ObjCSelfInitChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ObjCSelfInitChecker.cpp
index 344285750f0e8..17d3c042ac403 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ObjCSelfInitChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ObjCSelfInitChecker.cpp
@@ -437,6 +437,6 @@ void ento::registerObjCSelfInitChecker(CheckerManager &mgr) {
   mgr.registerChecker<ObjCSelfInitChecker>();
 }
 
-bool ento::shouldRegisterObjCSelfInitChecker(const LangOptions &LO) {
+bool ento::shouldRegisterObjCSelfInitChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/ObjCSuperDeallocChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ObjCSuperDeallocChecker.cpp
index 0575be8453743..39ffbca8d393a 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ObjCSuperDeallocChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ObjCSuperDeallocChecker.cpp
@@ -284,6 +284,6 @@ void ento::registerObjCSuperDeallocChecker(CheckerManager &Mgr) {
   Mgr.registerChecker<ObjCSuperDeallocChecker>();
 }
 
-bool ento::shouldRegisterObjCSuperDeallocChecker(const LangOptions &LO) {
+bool ento::shouldRegisterObjCSuperDeallocChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/ObjCUnusedIVarsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ObjCUnusedIVarsChecker.cpp
index cb47704515723..c9828c36a06a0 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ObjCUnusedIVarsChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ObjCUnusedIVarsChecker.cpp
@@ -186,6 +186,6 @@ void ento::registerObjCUnusedIvarsChecker(CheckerManager &mgr) {
   mgr.registerChecker<ObjCUnusedIvarsChecker>();
 }
 
-bool ento::shouldRegisterObjCUnusedIvarsChecker(const LangOptions &LO) {
+bool ento::shouldRegisterObjCUnusedIvarsChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/PaddingChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/PaddingChecker.cpp
index 4a3c2b8cd40e2..0b00664c7c104 100644
--- a/clang/lib/StaticAnalyzer/Checkers/PaddingChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/PaddingChecker.cpp
@@ -353,6 +353,6 @@ void ento::registerPaddingChecker(CheckerManager &Mgr) {
         Checker, "AllowedPad", "a non-negative value");
 }
 
-bool ento::shouldRegisterPaddingChecker(const LangOptions &LO) {
+bool ento::shouldRegisterPaddingChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/PointerArithChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/PointerArithChecker.cpp
index 259f23abdc958..d3e2849a0ce69 100644
--- a/clang/lib/StaticAnalyzer/Checkers/PointerArithChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/PointerArithChecker.cpp
@@ -343,6 +343,6 @@ void ento::registerPointerArithChecker(CheckerManager &mgr) {
   mgr.registerChecker<PointerArithChecker>();
 }
 
-bool ento::shouldRegisterPointerArithChecker(const LangOptions &LO) {
+bool ento::shouldRegisterPointerArithChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/PointerIterationChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/PointerIterationChecker.cpp
index 307e59b8eebc4..8aca6d009cdb0 100644
--- a/clang/lib/StaticAnalyzer/Checkers/PointerIterationChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/PointerIterationChecker.cpp
@@ -95,6 +95,7 @@ void ento::registerPointerIterationChecker(CheckerManager &Mgr) {
   Mgr.registerChecker<PointerIterationChecker>();
 }
 
-bool ento::shouldRegisterPointerIterationChecker(const LangOptions &LO) {
+bool ento::shouldRegisterPointerIterationChecker(const CheckerManager &mgr) {
+  const LangOptions &LO = mgr.getLangOpts();
   return LO.CPlusPlus;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/PointerSortingChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/PointerSortingChecker.cpp
index a3bfac97e40a7..8eee4df473e8f 100644
--- a/clang/lib/StaticAnalyzer/Checkers/PointerSortingChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/PointerSortingChecker.cpp
@@ -108,6 +108,7 @@ void ento::registerPointerSortingChecker(CheckerManager &Mgr) {
   Mgr.registerChecker<PointerSortingChecker>();
 }
 
-bool ento::shouldRegisterPointerSortingChecker(const LangOptions &LO) {
+bool ento::shouldRegisterPointerSortingChecker(const CheckerManager &mgr) {
+  const LangOptions &LO = mgr.getLangOpts();
   return LO.CPlusPlus;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/PointerSubChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/PointerSubChecker.cpp
index 88d0eb2ae7484..81c19d9a09400 100644
--- a/clang/lib/StaticAnalyzer/Checkers/PointerSubChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/PointerSubChecker.cpp
@@ -74,6 +74,6 @@ void ento::registerPointerSubChecker(CheckerManager &mgr) {
   mgr.registerChecker<PointerSubChecker>();
 }
 
-bool ento::shouldRegisterPointerSubChecker(const LangOptions &LO) {
+bool ento::shouldRegisterPointerSubChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/PthreadLockChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/PthreadLockChecker.cpp
index b89e1671cfadc..285d2da104f1a 100644
--- a/clang/lib/StaticAnalyzer/Checkers/PthreadLockChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/PthreadLockChecker.cpp
@@ -725,7 +725,7 @@ void ento::registerPthreadLockBase(CheckerManager &mgr) {
   mgr.registerChecker<PthreadLockChecker>();
 }
 
-bool ento::shouldRegisterPthreadLockBase(const LangOptions &LO) { return true; }
+bool ento::shouldRegisterPthreadLockBase(const CheckerManager &mgr) { return true; }
 
 #define REGISTER_CHECKER(name)                                                 \
   void ento::register##name(CheckerManager &mgr) {                             \
@@ -735,7 +735,7 @@ bool ento::shouldRegisterPthreadLockBase(const LangOptions &LO) { return true; }
         mgr.getCurrentCheckerName();                                           \
   }                                                                            \
                                                                                \
-  bool ento::shouldRegister##name(const LangOptions &LO) { return true; }
+  bool ento::shouldRegister##name(const CheckerManager &mgr) { return true; }
 
 REGISTER_CHECKER(PthreadLockChecker)
 REGISTER_CHECKER(FuchsiaLockChecker)
diff --git a/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.cpp
index 6f8cb1432bb11..4bf9beb365f66 100644
--- a/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.cpp
@@ -1477,7 +1477,7 @@ void ento::registerRetainCountBase(CheckerManager &Mgr) {
   Mgr.registerChecker<RetainCountChecker>();
 }
 
-bool ento::shouldRegisterRetainCountBase(const LangOptions &LO) {
+bool ento::shouldRegisterRetainCountBase(const CheckerManager &mgr) {
   return true;
 }
 
@@ -1485,7 +1485,7 @@ bool ento::shouldRegisterRetainCountBase(const LangOptions &LO) {
 // it should be possible to enable the NS/CF retain count checker as
 // osx.cocoa.RetainCount, and it should be possible to disable
 // osx.OSObjectRetainCount using osx.cocoa.RetainCount:CheckOSObject=false.
-static bool getOption(AnalyzerOptions &Options,
+static bool getOption(const AnalyzerOptions &Options,
                       StringRef Postfix,
                       StringRef Value) {
   auto I = Options.Config.find(
@@ -1503,7 +1503,7 @@ void ento::registerRetainCountChecker(CheckerManager &Mgr) {
                                        "true");
 }
 
-bool ento::shouldRegisterRetainCountChecker(const LangOptions &LO) {
+bool ento::shouldRegisterRetainCountChecker(const CheckerManager &mgr) {
   return true;
 }
 
@@ -1515,6 +1515,6 @@ void ento::registerOSObjectRetainCountChecker(CheckerManager &Mgr) {
     Chk->TrackOSObjects = true;
 }
 
-bool ento::shouldRegisterOSObjectRetainCountChecker(const LangOptions &LO) {
+bool ento::shouldRegisterOSObjectRetainCountChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/ReturnPointerRangeChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ReturnPointerRangeChecker.cpp
index 8a89649aae777..599d4f306aa13 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ReturnPointerRangeChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ReturnPointerRangeChecker.cpp
@@ -91,6 +91,6 @@ void ento::registerReturnPointerRangeChecker(CheckerManager &mgr) {
   mgr.registerChecker<ReturnPointerRangeChecker>();
 }
 
-bool ento::shouldRegisterReturnPointerRangeChecker(const LangOptions &LO) {
+bool ento::shouldRegisterReturnPointerRangeChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/ReturnUndefChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ReturnUndefChecker.cpp
index fbd15d864424e..5266cbf86b44f 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ReturnUndefChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ReturnUndefChecker.cpp
@@ -122,6 +122,6 @@ void ento::registerReturnUndefChecker(CheckerManager &mgr) {
   mgr.registerChecker<ReturnUndefChecker>();
 }
 
-bool ento::shouldRegisterReturnUndefChecker(const LangOptions &LO) {
+bool ento::shouldRegisterReturnUndefChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/ReturnValueChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ReturnValueChecker.cpp
index f98065492726b..14ecede170832 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ReturnValueChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ReturnValueChecker.cpp
@@ -165,6 +165,6 @@ void ento::registerReturnValueChecker(CheckerManager &Mgr) {
   Mgr.registerChecker<ReturnValueChecker>();
 }
 
-bool ento::shouldRegisterReturnValueChecker(const LangOptions &LO) {
+bool ento::shouldRegisterReturnValueChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/RunLoopAutoreleaseLeakChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/RunLoopAutoreleaseLeakChecker.cpp
index 5e305aa709b64..d9dc72ddaa218 100644
--- a/clang/lib/StaticAnalyzer/Checkers/RunLoopAutoreleaseLeakChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/RunLoopAutoreleaseLeakChecker.cpp
@@ -203,6 +203,6 @@ void ento::registerRunLoopAutoreleaseLeakChecker(CheckerManager &mgr) {
   mgr.registerChecker<RunLoopAutoreleaseLeakChecker>();
 }
 
-bool ento::shouldRegisterRunLoopAutoreleaseLeakChecker(const LangOptions &LO) {
+bool ento::shouldRegisterRunLoopAutoreleaseLeakChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/STLAlgorithmModeling.cpp b/clang/lib/StaticAnalyzer/Checkers/STLAlgorithmModeling.cpp
index 9452ccdc46c96..933e0146ff59b 100644
--- a/clang/lib/StaticAnalyzer/Checkers/STLAlgorithmModeling.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/STLAlgorithmModeling.cpp
@@ -174,7 +174,7 @@ void ento::registerSTLAlgorithmModeling(CheckerManager &Mgr) {
                                                   "AggressiveStdFindModeling");
 }
 
-bool ento::shouldRegisterSTLAlgorithmModeling(const LangOptions &LO) {
+bool ento::shouldRegisterSTLAlgorithmModeling(const CheckerManager &mgr) {
   return true;
 }
 
diff --git a/clang/lib/StaticAnalyzer/Checkers/SimpleStreamChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/SimpleStreamChecker.cpp
index 8193bcbef4cdf..8d380ed1b93de 100644
--- a/clang/lib/StaticAnalyzer/Checkers/SimpleStreamChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/SimpleStreamChecker.cpp
@@ -271,6 +271,6 @@ void ento::registerSimpleStreamChecker(CheckerManager &mgr) {
 }
 
 // This checker should be enabled regardless of how language options are set.
-bool ento::shouldRegisterSimpleStreamChecker(const LangOptions &LO) {
+bool ento::shouldRegisterSimpleStreamChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/SmartPtrModeling.cpp b/clang/lib/StaticAnalyzer/Checkers/SmartPtrModeling.cpp
index fd372aafa50d9..8250fd46b9266 100644
--- a/clang/lib/StaticAnalyzer/Checkers/SmartPtrModeling.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/SmartPtrModeling.cpp
@@ -67,6 +67,7 @@ void ento::registerSmartPtrModeling(CheckerManager &Mgr) {
   Mgr.registerChecker<SmartPtrModeling>();
 }
 
-bool ento::shouldRegisterSmartPtrModeling(const LangOptions &LO) {
+bool ento::shouldRegisterSmartPtrModeling(const CheckerManager &mgr) {
+  const LangOptions &LO = mgr.getLangOpts();
   return LO.CPlusPlus;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/StackAddrEscapeChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/StackAddrEscapeChecker.cpp
index 7285d27495a7c..21fc65a8e4f1a 100644
--- a/clang/lib/StaticAnalyzer/Checkers/StackAddrEscapeChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/StackAddrEscapeChecker.cpp
@@ -365,7 +365,7 @@ void ento::registerStackAddrEscapeBase(CheckerManager &mgr) {
   mgr.registerChecker<StackAddrEscapeChecker>();
 }
 
-bool ento::shouldRegisterStackAddrEscapeBase(const LangOptions &LO) {
+bool ento::shouldRegisterStackAddrEscapeBase(const CheckerManager &mgr) {
   return true;
 }
 
@@ -376,7 +376,7 @@ bool ento::shouldRegisterStackAddrEscapeBase(const LangOptions &LO) {
     Chk->ChecksEnabled[StackAddrEscapeChecker::CK_##name] = true;              \
   }                                                                            \
                                                                                \
-  bool ento::shouldRegister##name(const LangOptions &LO) {                     \
+  bool ento::shouldRegister##name(const CheckerManager &mgr) {                     \
     return true;                                                               \
   }
 
diff --git a/clang/lib/StaticAnalyzer/Checkers/StdLibraryFunctionsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/StdLibraryFunctionsChecker.cpp
index ff296e7ea46b2..f03696daab0b0 100644
--- a/clang/lib/StaticAnalyzer/Checkers/StdLibraryFunctionsChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/StdLibraryFunctionsChecker.cpp
@@ -190,6 +190,9 @@ class StdLibraryFunctionsChecker
     ProgramStateRef apply(ProgramStateRef State, const CallEvent &Call,
                           const Summary &Summary) const override {
       SVal V = getArgSVal(Call, getArgNo());
+      if (V.isUndef())
+        return State;
+
       DefinedOrUnknownSVal L = V.castAs<DefinedOrUnknownSVal>();
       if (!L.getAs<Loc>())
         return State;
@@ -303,7 +306,11 @@ class StdLibraryFunctionsChecker
   void checkPostCall(const CallEvent &Call, CheckerContext &C) const;
   bool evalCall(const CallEvent &Call, CheckerContext &C) const;
 
-  enum CheckKind { CK_StdCLibraryFunctionArgsChecker, CK_NumCheckKinds };
+  enum CheckKind {
+    CK_StdCLibraryFunctionArgsChecker,
+    CK_StdCLibraryFunctionsTesterChecker,
+    CK_NumCheckKinds
+  };
   DefaultBool ChecksEnabled[CK_NumCheckKinds];
   CheckerNameRef CheckNames[CK_NumCheckKinds];
 
@@ -452,23 +459,26 @@ void StdLibraryFunctionsChecker::checkPreCall(const CallEvent &Call,
   const Summary &Summary = *FoundSummary;
   ProgramStateRef State = C.getState();
 
+  ProgramStateRef NewState = State;
   for (const ValueConstraintPtr& VC : Summary.ArgConstraints) {
-    ProgramStateRef SuccessSt = VC->apply(State, Call, Summary);
-    ProgramStateRef FailureSt = VC->negate()->apply(State, Call, Summary);
+    ProgramStateRef SuccessSt = VC->apply(NewState, Call, Summary);
+    ProgramStateRef FailureSt = VC->negate()->apply(NewState, Call, Summary);
     // The argument constraint is not satisfied.
     if (FailureSt && !SuccessSt) {
-      if (ExplodedNode *N = C.generateErrorNode(State))
+      if (ExplodedNode *N = C.generateErrorNode(NewState))
         reportBug(Call, N, C);
       break;
     } else {
-      // Apply the constraint even if we cannot reason about the argument. This
-      // means both SuccessSt and FailureSt can be true. If we weren't applying
-      // the constraint that would mean that symbolic execution continues on a
-      // code whose behaviour is undefined.
+      // We will apply the constraint even if we cannot reason about the
+      // argument. This means both SuccessSt and FailureSt can be true. If we
+      // weren't applying the constraint that would mean that symbolic
+      // execution continues on a code whose behaviour is undefined.
       assert(SuccessSt);
-      C.addTransition(SuccessSt);
+      NewState = SuccessSt;
     }
   }
+  if (NewState && NewState != State)
+    C.addTransition(NewState);
 }
 
 void StdLibraryFunctionsChecker::checkPostCall(const CallEvent &Call,
@@ -933,13 +943,39 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
       {"getdelim", Summaries{Getline(IntTy, IntMax), Getline(LongTy, LongMax),
                              Getline(LongLongTy, LongLongMax)}},
   };
+
+  // Functions for testing.
+  if (ChecksEnabled[CK_StdCLibraryFunctionsTesterChecker]) {
+    llvm::StringMap<Summaries> TestFunctionSummaryMap = {
+        {"__two_constrained_args",
+         Summaries{
+             Summary(ArgTypes{IntTy, IntTy}, RetType{IntTy}, EvalCallAsPure)
+                 .ArgConstraint(
+                     ArgumentCondition(0U, WithinRange, SingleValue(1)))
+                 .ArgConstraint(
+                     ArgumentCondition(1U, WithinRange, SingleValue(1)))}},
+        {"__arg_constrained_twice",
+         Summaries{Summary(ArgTypes{IntTy}, RetType{IntTy}, EvalCallAsPure)
+                       .ArgConstraint(
+                           ArgumentCondition(0U, OutOfRange, SingleValue(1)))
+                       .ArgConstraint(
+                           ArgumentCondition(0U, OutOfRange, SingleValue(2)))}},
+    };
+    for (auto &E : TestFunctionSummaryMap) {
+      auto InsertRes =
+          FunctionSummaryMap.insert({std::string(E.getKey()), E.getValue()});
+      assert(InsertRes.second &&
+             "Test functions must not clash with modeled functions");
+      (void)InsertRes;
+    }
+  }
 }
 
 void ento::registerStdCLibraryFunctionsChecker(CheckerManager &mgr) {
   mgr.registerChecker<StdLibraryFunctionsChecker>();
 }
 
-bool ento::shouldRegisterStdCLibraryFunctionsChecker(const LangOptions &LO) {
+bool ento::shouldRegisterStdCLibraryFunctionsChecker(const CheckerManager &mgr) {
   return true;
 }
 
@@ -952,6 +988,7 @@ bool ento::shouldRegisterStdCLibraryFunctionsChecker(const LangOptions &LO) {
         mgr.getCurrentCheckerName();                                           \
   }                                                                            \
                                                                                \
-  bool ento::shouldRegister##name(const LangOptions &LO) { return true; }
+  bool ento::shouldRegister##name(const CheckerManager &mgr) { return true; }
 
 REGISTER_CHECKER(StdCLibraryFunctionArgsChecker)
+REGISTER_CHECKER(StdCLibraryFunctionsTesterChecker)
diff --git a/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp
index 7e5bf7c376f0f..4b20fcdc9a15f 100644
--- a/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp
@@ -441,4 +441,6 @@ void ento::registerStreamChecker(CheckerManager &mgr) {
   mgr.registerChecker<StreamChecker>();
 }
 
-bool ento::shouldRegisterStreamChecker(const LangOptions &LO) { return true; }
+bool ento::shouldRegisterStreamChecker(const CheckerManager &mgr) {
+  return true;
+}
diff --git a/clang/lib/StaticAnalyzer/Checkers/TaintTesterChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/TaintTesterChecker.cpp
index f81705304f3ab..916977c10c0c1 100644
--- a/clang/lib/StaticAnalyzer/Checkers/TaintTesterChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/TaintTesterChecker.cpp
@@ -63,6 +63,6 @@ void ento::registerTaintTesterChecker(CheckerManager &mgr) {
   mgr.registerChecker<TaintTesterChecker>();
 }
 
-bool ento::shouldRegisterTaintTesterChecker(const LangOptions &LO) {
+bool ento::shouldRegisterTaintTesterChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/TestAfterDivZeroChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/TestAfterDivZeroChecker.cpp
index 3663b09636924..eeec807ccee40 100644
--- a/clang/lib/StaticAnalyzer/Checkers/TestAfterDivZeroChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/TestAfterDivZeroChecker.cpp
@@ -261,6 +261,6 @@ void ento::registerTestAfterDivZeroChecker(CheckerManager &mgr) {
   mgr.registerChecker<TestAfterDivZeroChecker>();
 }
 
-bool ento::shouldRegisterTestAfterDivZeroChecker(const LangOptions &LO) {
+bool ento::shouldRegisterTestAfterDivZeroChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/TraversalChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/TraversalChecker.cpp
index 73183aa468f65..2f316bd3b20db 100644
--- a/clang/lib/StaticAnalyzer/Checkers/TraversalChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/TraversalChecker.cpp
@@ -64,7 +64,7 @@ void ento::registerTraversalDumper(CheckerManager &mgr) {
   mgr.registerChecker<TraversalDumper>();
 }
 
-bool ento::shouldRegisterTraversalDumper(const LangOptions &LO) {
+bool ento::shouldRegisterTraversalDumper(const CheckerManager &mgr) {
   return true;
 }
 
@@ -116,6 +116,6 @@ void ento::registerCallDumper(CheckerManager &mgr) {
   mgr.registerChecker<CallDumper>();
 }
 
-bool ento::shouldRegisterCallDumper(const LangOptions &LO) {
+bool ento::shouldRegisterCallDumper(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/TrustNonnullChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/TrustNonnullChecker.cpp
index 62a4c2ab0209c..5cc7131725274 100644
--- a/clang/lib/StaticAnalyzer/Checkers/TrustNonnullChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/TrustNonnullChecker.cpp
@@ -252,6 +252,6 @@ void ento::registerTrustNonnullChecker(CheckerManager &Mgr) {
   Mgr.registerChecker<TrustNonnullChecker>(Mgr.getASTContext());
 }
 
-bool ento::shouldRegisterTrustNonnullChecker(const LangOptions &LO) {
+bool ento::shouldRegisterTrustNonnullChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/UndefBranchChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/UndefBranchChecker.cpp
index 247cba7dc9333..3e0caaf79ca09 100644
--- a/clang/lib/StaticAnalyzer/Checkers/UndefBranchChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/UndefBranchChecker.cpp
@@ -110,6 +110,6 @@ void ento::registerUndefBranchChecker(CheckerManager &mgr) {
   mgr.registerChecker<UndefBranchChecker>();
 }
 
-bool ento::shouldRegisterUndefBranchChecker(const LangOptions &LO) {
+bool ento::shouldRegisterUndefBranchChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/UndefCapturedBlockVarChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/UndefCapturedBlockVarChecker.cpp
index 7b581bef39001..e457513d8de44 100644
--- a/clang/lib/StaticAnalyzer/Checkers/UndefCapturedBlockVarChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/UndefCapturedBlockVarChecker.cpp
@@ -101,6 +101,6 @@ void ento::registerUndefCapturedBlockVarChecker(CheckerManager &mgr) {
   mgr.registerChecker<UndefCapturedBlockVarChecker>();
 }
 
-bool ento::shouldRegisterUndefCapturedBlockVarChecker(const LangOptions &LO) {
+bool ento::shouldRegisterUndefCapturedBlockVarChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/UndefResultChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/UndefResultChecker.cpp
index 9a37786037a29..392da48180983 100644
--- a/clang/lib/StaticAnalyzer/Checkers/UndefResultChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/UndefResultChecker.cpp
@@ -187,6 +187,6 @@ void ento::registerUndefResultChecker(CheckerManager &mgr) {
   mgr.registerChecker<UndefResultChecker>();
 }
 
-bool ento::shouldRegisterUndefResultChecker(const LangOptions &LO) {
+bool ento::shouldRegisterUndefResultChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/UndefinedArraySubscriptChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/UndefinedArraySubscriptChecker.cpp
index 2f075eaeb03bf..fdefe75e8201d 100644
--- a/clang/lib/StaticAnalyzer/Checkers/UndefinedArraySubscriptChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/UndefinedArraySubscriptChecker.cpp
@@ -62,6 +62,6 @@ void ento::registerUndefinedArraySubscriptChecker(CheckerManager &mgr) {
   mgr.registerChecker<UndefinedArraySubscriptChecker>();
 }
 
-bool ento::shouldRegisterUndefinedArraySubscriptChecker(const LangOptions &LO) {
+bool ento::shouldRegisterUndefinedArraySubscriptChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/UndefinedAssignmentChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/UndefinedAssignmentChecker.cpp
index 277a8a1433282..05f8f6084c0b6 100644
--- a/clang/lib/StaticAnalyzer/Checkers/UndefinedAssignmentChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/UndefinedAssignmentChecker.cpp
@@ -120,6 +120,6 @@ void ento::registerUndefinedAssignmentChecker(CheckerManager &mgr) {
   mgr.registerChecker<UndefinedAssignmentChecker>();
 }
 
-bool ento::shouldRegisterUndefinedAssignmentChecker(const LangOptions &LO) {
+bool ento::shouldRegisterUndefinedAssignmentChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/UninitializedObject/UninitializedObjectChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/UninitializedObject/UninitializedObjectChecker.cpp
index 14f551403d98d..e0be718decb29 100644
--- a/clang/lib/StaticAnalyzer/Checkers/UninitializedObject/UninitializedObjectChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/UninitializedObject/UninitializedObjectChecker.cpp
@@ -608,7 +608,7 @@ std::string clang::ento::getVariableName(const FieldDecl *Field) {
 void ento::registerUninitializedObjectChecker(CheckerManager &Mgr) {
   auto Chk = Mgr.registerChecker<UninitializedObjectChecker>();
 
-  AnalyzerOptions &AnOpts = Mgr.getAnalyzerOptions();
+  const AnalyzerOptions &AnOpts = Mgr.getAnalyzerOptions();
   UninitObjCheckerOptions &ChOpts = Chk->Opts;
 
   ChOpts.IsPedantic = AnOpts.getCheckerBooleanOption(Chk, "Pedantic");
@@ -628,6 +628,6 @@ void ento::registerUninitializedObjectChecker(CheckerManager &Mgr) {
         "\"" + ErrorMsg + "\"");
 }
 
-bool ento::shouldRegisterUninitializedObjectChecker(const LangOptions &LO) {
+bool ento::shouldRegisterUninitializedObjectChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp
index f4e225d836f38..6ea8451512f3a 100644
--- a/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp
@@ -503,7 +503,7 @@ void UnixAPIPortabilityChecker::checkPreStmt(const CallExpr *CE,
     mgr.registerChecker<CHECKERNAME>();                                        \
   }                                                                            \
                                                                                \
-  bool ento::shouldRegister##CHECKERNAME(const LangOptions &LO) {              \
+  bool ento::shouldRegister##CHECKERNAME(const CheckerManager &mgr) {              \
     return true;                                                               \
   }
 
diff --git a/clang/lib/StaticAnalyzer/Checkers/UnreachableCodeChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/UnreachableCodeChecker.cpp
index 65dd82675df9b..74eec81ffb3e5 100644
--- a/clang/lib/StaticAnalyzer/Checkers/UnreachableCodeChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/UnreachableCodeChecker.cpp
@@ -257,6 +257,6 @@ void ento::registerUnreachableCodeChecker(CheckerManager &mgr) {
   mgr.registerChecker<UnreachableCodeChecker>();
 }
 
-bool ento::shouldRegisterUnreachableCodeChecker(const LangOptions &LO) {
+bool ento::shouldRegisterUnreachableCodeChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/VLASizeChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/VLASizeChecker.cpp
index 92e81edafbf30..21e8b9653039b 100644
--- a/clang/lib/StaticAnalyzer/Checkers/VLASizeChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/VLASizeChecker.cpp
@@ -187,6 +187,6 @@ void ento::registerVLASizeChecker(CheckerManager &mgr) {
   mgr.registerChecker<VLASizeChecker>();
 }
 
-bool ento::shouldRegisterVLASizeChecker(const LangOptions &LO) {
+bool ento::shouldRegisterVLASizeChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/ValistChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ValistChecker.cpp
index a3610514a924b..dde5912b6d6ee 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ValistChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ValistChecker.cpp
@@ -404,7 +404,7 @@ void ento::registerValistBase(CheckerManager &mgr) {
   mgr.registerChecker<ValistChecker>();
 }
 
-bool ento::shouldRegisterValistBase(const LangOptions &LO) {
+bool ento::shouldRegisterValistBase(const CheckerManager &mgr) {
   return true;
 }
 
@@ -416,7 +416,7 @@ bool ento::shouldRegisterValistBase(const LangOptions &LO) {
         mgr.getCurrentCheckerName();                                           \
   }                                                                            \
                                                                                \
-  bool ento::shouldRegister##name##Checker(const LangOptions &LO) {            \
+  bool ento::shouldRegister##name##Checker(const CheckerManager &mgr) {            \
     return true;                                                               \
   }
 
diff --git a/clang/lib/StaticAnalyzer/Checkers/VforkChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/VforkChecker.cpp
index 077a34810db77..8f147026ae192 100644
--- a/clang/lib/StaticAnalyzer/Checkers/VforkChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/VforkChecker.cpp
@@ -217,6 +217,6 @@ void ento::registerVforkChecker(CheckerManager &mgr) {
   mgr.registerChecker<VforkChecker>();
 }
 
-bool ento::shouldRegisterVforkChecker(const LangOptions &LO) {
+bool ento::shouldRegisterVforkChecker(const CheckerManager &mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/VirtualCallChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/VirtualCallChecker.cpp
index fd93fc33115f5..f49ee5fa5ad37 100644
--- a/clang/lib/StaticAnalyzer/Checkers/VirtualCallChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/VirtualCallChecker.cpp
@@ -224,14 +224,17 @@ void ento::registerVirtualCallChecker(CheckerManager &Mgr) {
   }
 }
 
-bool ento::shouldRegisterVirtualCallModeling(const LangOptions &LO) {
+bool ento::shouldRegisterVirtualCallModeling(const CheckerManager &mgr) {
+  const LangOptions &LO = mgr.getLangOpts();
   return LO.CPlusPlus;
 }
 
-bool ento::shouldRegisterPureVirtualCallChecker(const LangOptions &LO) {
+bool ento::shouldRegisterPureVirtualCallChecker(const CheckerManager &mgr) {
+  const LangOptions &LO = mgr.getLangOpts();
   return LO.CPlusPlus;
 }
 
-bool ento::shouldRegisterVirtualCallChecker(const LangOptions &LO) {
+bool ento::shouldRegisterVirtualCallChecker(const CheckerManager &mgr) {
+  const LangOptions &LO = mgr.getLangOpts();
   return LO.CPlusPlus;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/cert/PutenvWithAutoChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/cert/PutenvWithAutoChecker.cpp
index b712e77cf6d2b..1c67bbd77ec88 100644
--- a/clang/lib/StaticAnalyzer/Checkers/cert/PutenvWithAutoChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/cert/PutenvWithAutoChecker.cpp
@@ -63,4 +63,4 @@ void ento::registerPutenvWithAuto(CheckerManager &Mgr) {
   Mgr.registerChecker<PutenvWithAutoChecker>();
 }
 
-bool ento::shouldRegisterPutenvWithAuto(const LangOptions &) { return true; }
+bool ento::shouldRegisterPutenvWithAuto(const CheckerManager &) { return true; }
diff --git a/clang/lib/StaticAnalyzer/Core/CheckerHelpers.cpp b/clang/lib/StaticAnalyzer/Core/CheckerHelpers.cpp
index e43b172d018d4..4b63ebc40edeb 100644
--- a/clang/lib/StaticAnalyzer/Core/CheckerHelpers.cpp
+++ b/clang/lib/StaticAnalyzer/Core/CheckerHelpers.cpp
@@ -126,9 +126,6 @@ llvm::Optional<int> tryExpandAsInteger(StringRef Macro,
     if (!T.isOneOf(tok::l_paren, tok::r_paren))
       FilteredTokens.push_back(T);
 
-  if (FilteredTokens.size() > 2)
-    return llvm::None;
-
   // Parse an integer at the end of the macro definition.
   const Token &T = FilteredTokens.back();
   if (!T.isLiteral())
@@ -140,11 +137,10 @@ llvm::Optional<int> tryExpandAsInteger(StringRef Macro,
     return llvm::None;
 
   // Parse an optional minus sign.
-  if (FilteredTokens.size() == 2) {
-    if (FilteredTokens.front().is(tok::minus))
+  size_t Size = FilteredTokens.size();
+  if (Size >= 2) {
+    if (FilteredTokens[Size - 2].is(tok::minus))
       IntValue = -IntValue;
-    else
-      return llvm::None;
   }
 
   return IntValue.getSExtValue();
diff --git a/clang/lib/StaticAnalyzer/Core/CheckerManager.cpp b/clang/lib/StaticAnalyzer/Core/CheckerManager.cpp
index ce5e4a46d3e2c..17c66cd232791 100644
--- a/clang/lib/StaticAnalyzer/Core/CheckerManager.cpp
+++ b/clang/lib/StaticAnalyzer/Core/CheckerManager.cpp
@@ -64,10 +64,9 @@ void CheckerManager::reportInvalidCheckerOptionValue(
     const CheckerBase *C, StringRef OptionName,
     StringRef ExpectedValueDesc) const {
 
-  Context.getDiagnostics()
-      .Report(diag::err_analyzer_checker_option_invalid_input)
-          << (llvm::Twine() + C->getTagDescription() + ":" + OptionName).str()
-          << ExpectedValueDesc;
+  getDiagnostics().Report(diag::err_analyzer_checker_option_invalid_input)
+      << (llvm::Twine() + C->getTagDescription() + ":" + OptionName).str()
+      << ExpectedValueDesc;
 }
 
 //===----------------------------------------------------------------------===//
@@ -903,8 +902,3 @@ CheckerManager::getCachedStmtCheckersFor(const Stmt *S, bool isPreVisit) {
       Checkers.push_back(Info.CheckFn);
   return Checkers;
 }
-
-CheckerManager::~CheckerManager() {
-  for (const auto &CheckerDtor : CheckerDtors)
-    CheckerDtor();
-}
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
index b9adee87436a9..1f0d89d59120e 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
@@ -1413,6 +1413,8 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
     case Stmt::SubstNonTypeTemplateParmExprClass:
     case Stmt::CXXNullPtrLiteralExprClass:
     case Stmt::OMPArraySectionExprClass:
+    case Stmt::OMPArrayShapingExprClass:
+    case Stmt::OMPIteratorExprClass:
     case Stmt::TypeTraitExprClass: {
       Bldr.takeNodes(Pred);
       ExplodedNodeSet preVisit;
@@ -3173,9 +3175,10 @@ std::string ExprEngine::DumpGraph(bool trim, StringRef Filename) {
                             /*Title=*/"Exploded Graph",
                             /*Filename=*/std::string(Filename));
   }
-#endif
+#else
   llvm::errs() << "Warning: dumping graph requires assertions" << "\n";
   return "";
+#endif
 }
 
 std::string ExprEngine::DumpGraph(ArrayRef<const ExplodedNode*> Nodes,
diff --git a/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp b/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp
index 2e3aa0669061f..74c689730e58d 100644
--- a/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp
+++ b/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp
@@ -33,7 +33,6 @@
 #include "clang/StaticAnalyzer/Core/PathDiagnosticConsumers.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/AnalysisManager.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
-#include "clang/StaticAnalyzer/Frontend/CheckerRegistration.h"
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Support/FileSystem.h"
@@ -209,8 +208,8 @@ class AnalysisConsumer : public AnalysisASTConsumer,
 
   void Initialize(ASTContext &Context) override {
     Ctx = &Context;
-    checkerMgr = createCheckerManager(
-        *Ctx, *Opts, Plugins, CheckerRegistrationFns, PP.getDiagnostics());
+    checkerMgr = std::make_unique<CheckerManager>(*Ctx, *Opts, PP, Plugins,
+                                                  CheckerRegistrationFns);
 
     Mgr = std::make_unique<AnalysisManager>(*Ctx, PP, PathConsumers,
                                             CreateStoreMgr, CreateConstraintMgr,
diff --git a/clang/lib/StaticAnalyzer/Frontend/CheckerRegistration.cpp b/clang/lib/StaticAnalyzer/Frontend/AnalyzerHelpFlags.cpp
similarity index 69%
rename from clang/lib/StaticAnalyzer/Frontend/CheckerRegistration.cpp
rename to clang/lib/StaticAnalyzer/Frontend/AnalyzerHelpFlags.cpp
index f4f06e32cd1d7..d589e69bdf347 100644
--- a/clang/lib/StaticAnalyzer/Frontend/CheckerRegistration.cpp
+++ b/clang/lib/StaticAnalyzer/Frontend/AnalyzerHelpFlags.cpp
@@ -10,8 +10,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/StaticAnalyzer/Frontend/CheckerRegistration.h"
+#include "clang/StaticAnalyzer/Frontend/AnalyzerHelpFlags.h"
 #include "clang/Basic/Diagnostic.h"
+#include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
 #include "clang/StaticAnalyzer/Core/AnalyzerOptions.h"
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
@@ -24,53 +25,34 @@
 using namespace clang;
 using namespace ento;
 
-std::unique_ptr<CheckerManager> ento::createCheckerManager(
-    ASTContext &context,
-    AnalyzerOptions &opts,
-    ArrayRef<std::string> plugins,
-    ArrayRef<std::function<void(CheckerRegistry &)>> checkerRegistrationFns,
-    DiagnosticsEngine &diags) {
-  auto checkerMgr = std::make_unique<CheckerManager>(context, opts);
-
-  CheckerRegistry allCheckers(plugins, diags, opts, context.getLangOpts(),
-                              checkerRegistrationFns);
-
-  allCheckers.initializeManager(*checkerMgr);
-  allCheckers.validateCheckerOptions();
-  checkerMgr->finishedCheckerRegistration();
-
-  return checkerMgr;
-}
-
-void ento::printCheckerHelp(raw_ostream &out, ArrayRef<std::string> plugins,
-                            AnalyzerOptions &anopts,
-                            DiagnosticsEngine &diags,
-                            const LangOptions &langOpts) {
+void ento::printCheckerHelp(raw_ostream &out, CompilerInstance &CI) {
   out << "OVERVIEW: Clang Static Analyzer Checkers List\n\n";
   out << "USAGE: -analyzer-checker <CHECKER or PACKAGE,...>\n\n";
 
-  CheckerRegistry(plugins, diags, anopts, langOpts)
-      .printCheckerWithDescList(out);
+  auto CheckerMgr = std::make_unique<CheckerManager>(
+      *CI.getAnalyzerOpts(), CI.getLangOpts(), CI.getDiagnostics(),
+      CI.getFrontendOpts().Plugins);
+
+  CheckerMgr->getCheckerRegistry().printCheckerWithDescList(out);
 }
 
-void ento::printEnabledCheckerList(raw_ostream &out,
-                                   ArrayRef<std::string> plugins,
-                                   AnalyzerOptions &anopts,
-                                   DiagnosticsEngine &diags,
-                                   const LangOptions &langOpts) {
+void ento::printEnabledCheckerList(raw_ostream &out, CompilerInstance &CI) {
   out << "OVERVIEW: Clang Static Analyzer Enabled Checkers List\n\n";
 
-  CheckerRegistry(plugins, diags, anopts, langOpts)
-      .printEnabledCheckerList(out);
+  auto CheckerMgr = std::make_unique<CheckerManager>(
+      *CI.getAnalyzerOpts(), CI.getLangOpts(), CI.getDiagnostics(),
+      CI.getFrontendOpts().Plugins);
+
+  CheckerMgr->getCheckerRegistry().printEnabledCheckerList(out);
 }
 
-void ento::printCheckerConfigList(raw_ostream &OS,
-                                  ArrayRef<std::string> plugins,
-                                  AnalyzerOptions &opts,
-                                  DiagnosticsEngine &diags,
-                                  const LangOptions &LangOpts) {
-  CheckerRegistry(plugins, diags, opts, LangOpts)
-      .printCheckerOptionList(OS);
+void ento::printCheckerConfigList(raw_ostream &out, CompilerInstance &CI) {
+
+  auto CheckerMgr = std::make_unique<CheckerManager>(
+      *CI.getAnalyzerOpts(), CI.getLangOpts(), CI.getDiagnostics(),
+      CI.getFrontendOpts().Plugins);
+
+  CheckerMgr->getCheckerRegistry().printCheckerOptionList(out);
 }
 
 void ento::printAnalyzerConfigList(raw_ostream &out) {
diff --git a/clang/lib/StaticAnalyzer/Frontend/CMakeLists.txt b/clang/lib/StaticAnalyzer/Frontend/CMakeLists.txt
index 5e7dd8f18cd73..0c100a121f777 100644
--- a/clang/lib/StaticAnalyzer/Frontend/CMakeLists.txt
+++ b/clang/lib/StaticAnalyzer/Frontend/CMakeLists.txt
@@ -6,14 +6,16 @@ set(LLVM_LINK_COMPONENTS
 
 add_clang_library(clangStaticAnalyzerFrontend
   AnalysisConsumer.cpp
-  CheckerRegistration.cpp
+  AnalyzerHelpFlags.cpp
   CheckerRegistry.cpp
+  CreateCheckerManager.cpp
   FrontendActions.cpp
   ModelConsumer.cpp
   ModelInjector.cpp
 
   LINK_LIBS
   clangAST
+  clangASTMatchers
   clangAnalysis
   clangBasic
   clangCrossTU
diff --git a/clang/lib/StaticAnalyzer/Frontend/CheckerRegistry.cpp b/clang/lib/StaticAnalyzer/Frontend/CheckerRegistry.cpp
index 4af204474494f..e5583549a01f4 100644
--- a/clang/lib/StaticAnalyzer/Frontend/CheckerRegistry.cpp
+++ b/clang/lib/StaticAnalyzer/Frontend/CheckerRegistry.cpp
@@ -109,9 +109,9 @@ CheckerRegistry::getMutableCheckersForCmdLineArg(StringRef CmdLineArg) {
 
 CheckerRegistry::CheckerRegistry(
     ArrayRef<std::string> Plugins, DiagnosticsEngine &Diags,
-    AnalyzerOptions &AnOpts, const LangOptions &LangOpts,
+    AnalyzerOptions &AnOpts,
     ArrayRef<std::function<void(CheckerRegistry &)>> CheckerRegistrationFns)
-    : Diags(Diags), AnOpts(AnOpts), LangOpts(LangOpts) {
+    : Diags(Diags), AnOpts(AnOpts) {
 
   // Register builtin checkers.
 #define GET_CHECKERS
@@ -179,12 +179,16 @@ CheckerRegistry::CheckerRegistry(
   addDependency(FULLNAME, DEPENDENCY);
 
 #define GET_CHECKER_OPTIONS
-#define CHECKER_OPTION(TYPE, FULLNAME, CMDFLAG, DESC, DEFAULT_VAL, DEVELOPMENT_STATUS, IS_HIDDEN)  \
-  addCheckerOption(TYPE, FULLNAME, CMDFLAG, DEFAULT_VAL, DESC, DEVELOPMENT_STATUS, IS_HIDDEN);
+#define CHECKER_OPTION(TYPE, FULLNAME, CMDFLAG, DESC, DEFAULT_VAL,             \
+                       DEVELOPMENT_STATUS, IS_HIDDEN)                          \
+  addCheckerOption(TYPE, FULLNAME, CMDFLAG, DEFAULT_VAL, DESC,                 \
+                   DEVELOPMENT_STATUS, IS_HIDDEN);
 
 #define GET_PACKAGE_OPTIONS
-#define PACKAGE_OPTION(TYPE, FULLNAME, CMDFLAG, DESC, DEFAULT_VAL, DEVELOPMENT_STATUS, IS_HIDDEN)  \
-  addPackageOption(TYPE, FULLNAME, CMDFLAG, DEFAULT_VAL, DESC, DEVELOPMENT_STATUS, IS_HIDDEN);
+#define PACKAGE_OPTION(TYPE, FULLNAME, CMDFLAG, DESC, DEFAULT_VAL,             \
+                       DEVELOPMENT_STATUS, IS_HIDDEN)                          \
+  addPackageOption(TYPE, FULLNAME, CMDFLAG, DEFAULT_VAL, DESC,                 \
+                   DEVELOPMENT_STATUS, IS_HIDDEN);
 
 #include "clang/StaticAnalyzer/Checkers/Checkers.inc"
 #undef CHECKER_DEPENDENCY
@@ -213,24 +217,53 @@ CheckerRegistry::CheckerRegistry(
                                  : StateFromCmdLine::State_Disabled;
     }
   }
+  validateCheckerOptions();
+}
+
+/// Collects dependenies in \p enabledCheckers. Return None on failure.
+LLVM_NODISCARD
+static llvm::Optional<CheckerRegistry::CheckerInfoSet>
+collectDependencies(const CheckerRegistry::CheckerInfo &checker,
+                    const CheckerManager &Mgr);
+
+void CheckerRegistry::initializeRegistry(const CheckerManager &Mgr) {
+  for (const CheckerInfo &Checker : Checkers) {
+    if (!Checker.isEnabled(Mgr))
+      continue;
+
+    // Recursively enable its dependencies.
+    llvm::Optional<CheckerInfoSet> Deps = collectDependencies(Checker, Mgr);
+
+    if (!Deps) {
+      // If we failed to enable any of the dependencies, don't enable this
+      // checker.
+      continue;
+    }
+
+    // Note that set_union also preserves the order of insertion.
+    EnabledCheckers.set_union(*Deps);
+
+    // Enable the checker.
+    EnabledCheckers.insert(&Checker);
+  }
 }
 
 /// Collects dependencies in \p ret, returns false on failure.
 static bool
 collectDependenciesImpl(const CheckerRegistry::ConstCheckerInfoList &Deps,
-                        const LangOptions &LO,
+                        const CheckerManager &Mgr,
                         CheckerRegistry::CheckerInfoSet &Ret);
 
 /// Collects dependenies in \p enabledCheckers. Return None on failure.
 LLVM_NODISCARD
 static llvm::Optional<CheckerRegistry::CheckerInfoSet>
 collectDependencies(const CheckerRegistry::CheckerInfo &checker,
-                    const LangOptions &LO) {
+                    const CheckerManager &Mgr) {
 
   CheckerRegistry::CheckerInfoSet Ret;
   // Add dependencies to the enabled checkers only if all of them can be
   // enabled.
-  if (!collectDependenciesImpl(checker.Dependencies, LO, Ret))
+  if (!collectDependenciesImpl(checker.Dependencies, Mgr, Ret))
     return None;
 
   return Ret;
@@ -238,16 +271,16 @@ collectDependencies(const CheckerRegistry::CheckerInfo &checker,
 
 static bool
 collectDependenciesImpl(const CheckerRegistry::ConstCheckerInfoList &Deps,
-                        const LangOptions &LO,
+                        const CheckerManager &Mgr,
                         CheckerRegistry::CheckerInfoSet &Ret) {
 
   for (const CheckerRegistry::CheckerInfo *Dependency : Deps) {
 
-    if (Dependency->isDisabled(LO))
+    if (Dependency->isDisabled(Mgr))
       return false;
 
     // Collect dependencies recursively.
-    if (!collectDependenciesImpl(Dependency->Dependencies, LO, Ret))
+    if (!collectDependenciesImpl(Dependency->Dependencies, Mgr, Ret))
       return false;
 
     Ret.insert(Dependency);
@@ -256,34 +289,6 @@ collectDependenciesImpl(const CheckerRegistry::ConstCheckerInfoList &Deps,
   return true;
 }
 
-CheckerRegistry::CheckerInfoSet CheckerRegistry::getEnabledCheckers() const {
-
-  CheckerInfoSet EnabledCheckers;
-
-  for (const CheckerInfo &Checker : Checkers) {
-    if (!Checker.isEnabled(LangOpts))
-      continue;
-
-    // Recursively enable its dependencies.
-    llvm::Optional<CheckerInfoSet> Deps =
-        collectDependencies(Checker, LangOpts);
-
-    if (!Deps) {
-      // If we failed to enable any of the dependencies, don't enable this
-      // checker.
-      continue;
-    }
-
-    // Note that set_union also preserves the order of insertion.
-    EnabledCheckers.set_union(*Deps);
-
-    // Enable the checker.
-    EnabledCheckers.insert(&Checker);
-  }
-
-  return EnabledCheckers;
-}
-
 void CheckerRegistry::resolveDependencies() {
   for (const std::pair<StringRef, StringRef> &Entry : Dependencies) {
     auto CheckerIt = binaryFind(Checkers, Entry.first);
@@ -298,8 +303,6 @@ void CheckerRegistry::resolveDependencies() {
 
     CheckerIt->Dependencies.emplace_back(&*DependencyIt);
   }
-
-  Dependencies.clear();
 }
 
 void CheckerRegistry::addDependency(StringRef FullName, StringRef Dependency) {
@@ -378,14 +381,12 @@ void CheckerRegistry::resolveCheckerAndPackageOptions() {
     insertOptionToCollection(CheckerOptEntry.first, Checkers,
                              CheckerOptEntry.second, AnOpts, Diags);
   }
-  CheckerOptions.clear();
 
   for (const std::pair<StringRef, CmdLineOption> &PackageOptEntry :
        PackageOptions) {
     insertOptionToCollection(PackageOptEntry.first, Packages,
                              PackageOptEntry.second, AnOpts, Diags);
   }
-  PackageOptions.clear();
 }
 
 void CheckerRegistry::addPackage(StringRef FullName) {
@@ -432,11 +433,8 @@ void CheckerRegistry::addCheckerOption(StringRef OptionType,
 }
 
 void CheckerRegistry::initializeManager(CheckerManager &CheckerMgr) const {
-  // Collect checkers enabled by the options.
-  CheckerInfoSet enabledCheckers = getEnabledCheckers();
-
   // Initialize the CheckerManager with all enabled checkers.
-  for (const auto *Checker : enabledCheckers) {
+  for (const auto *Checker : EnabledCheckers) {
     CheckerMgr.setCurrentCheckerName(CheckerNameRef(Checker->FullName));
     Checker->Initialize(CheckerMgr);
   }
@@ -505,6 +503,10 @@ void CheckerRegistry::validateCheckerOptions() const {
   }
 }
 
+//===----------------------------------------------------------------------===//
+// Printing functions.
+//===----------------------------------------------------------------------===//
+
 void CheckerRegistry::printCheckerWithDescList(raw_ostream &Out,
                                                size_t MaxNameChars) const {
   // FIXME: Print available packages.
@@ -556,9 +558,6 @@ void CheckerRegistry::printCheckerWithDescList(raw_ostream &Out,
 }
 
 void CheckerRegistry::printEnabledCheckerList(raw_ostream &Out) const {
-  // Collect checkers enabled by the options.
-  CheckerInfoSet EnabledCheckers = getEnabledCheckers();
-
   for (const auto *i : EnabledCheckers)
     Out << i->FullName << '\n';
 }
diff --git a/clang/lib/StaticAnalyzer/Frontend/CreateCheckerManager.cpp b/clang/lib/StaticAnalyzer/Frontend/CreateCheckerManager.cpp
new file mode 100644
index 0000000000000..140f807514439
--- /dev/null
+++ b/clang/lib/StaticAnalyzer/Frontend/CreateCheckerManager.cpp
@@ -0,0 +1,49 @@
+//===- CheckerManager.h - Static Analyzer Checker Manager -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines the Static Analyzer Checker Manager.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Frontend/CheckerRegistry.h"
+#include <memory>
+
+namespace clang {
+namespace ento {
+
+CheckerManager::CheckerManager(
+    ASTContext &Context, AnalyzerOptions &AOptions, const Preprocessor &PP,
+    ArrayRef<std::string> plugins,
+    ArrayRef<std::function<void(CheckerRegistry &)>> checkerRegistrationFns)
+    : Context(&Context), LangOpts(Context.getLangOpts()), AOptions(AOptions),
+      PP(&PP), Diags(Context.getDiagnostics()),
+      Registry(
+          std::make_unique<CheckerRegistry>(plugins, Context.getDiagnostics(),
+                                            AOptions, checkerRegistrationFns)) {
+  Registry->initializeRegistry(*this);
+  Registry->initializeManager(*this);
+  finishedCheckerRegistration();
+}
+
+CheckerManager::CheckerManager(AnalyzerOptions &AOptions,
+                               const LangOptions &LangOpts,
+                               DiagnosticsEngine &Diags,
+                               ArrayRef<std::string> plugins)
+    : LangOpts(LangOpts), AOptions(AOptions), Diags(Diags),
+      Registry(std::make_unique<CheckerRegistry>(plugins, Diags, AOptions)) {
+  Registry->initializeRegistry(*this);
+}
+
+CheckerManager::~CheckerManager() {
+  for (const auto &CheckerDtor : CheckerDtors)
+    CheckerDtor();
+}
+
+} // namespace ento
+} // namespace clang
diff --git a/clang/lib/Tooling/Syntax/BuildTree.cpp b/clang/lib/Tooling/Syntax/BuildTree.cpp
index 82c87ba02b744..11058edec615d 100644
--- a/clang/lib/Tooling/Syntax/BuildTree.cpp
+++ b/clang/lib/Tooling/Syntax/BuildTree.cpp
@@ -44,15 +44,6 @@ using namespace clang;
 LLVM_ATTRIBUTE_UNUSED
 static bool isImplicitExpr(clang::Expr *E) { return E->IgnoreImplicit() != E; }
 
-static SourceLocation getQualifiedNameStart(DeclaratorDecl *D) {
-  auto DN = D->getDeclName();
-  bool IsAnonymous = DN.isIdentifier() && !DN.getAsIdentifierInfo();
-  if (IsAnonymous)
-    return SourceLocation();
-  return D->getQualifierLoc() ? D->getQualifierLoc().getBeginLoc()
-                              : D->getLocation();
-}
-
 namespace {
 /// Get start location of the Declarator from the TypeLoc.
 /// E.g.:
@@ -142,8 +133,9 @@ static SourceRange getDeclaratorRange(const SourceManager &SM, TypeLoc T,
       End = Name;
   }
   if (Initializer.isValid()) {
-    assert(SM.isBeforeInTranslationUnit(End, Initializer.getEnd()));
-    End = Initializer.getEnd();
+    auto InitializerEnd = Initializer.getEnd();
+    assert(SM.isBeforeInTranslationUnit(End, InitializerEnd) || End == InitializerEnd);
+    End = InitializerEnd;
   }
   return SourceRange(Start, End);
 }
@@ -212,10 +204,6 @@ class syntax::TreeBuilder {
     foldNode(Range, New, nullptr);
   }
 
-  /// Must be called with the range of each `DeclaratorDecl`. Ensures the
-  /// corresponding declarator nodes are covered by `SimpleDeclaration`.
-  void noticeDeclRange(llvm::ArrayRef<syntax::Token> Range);
-
   /// Notifies that we should not consume trailing semicolon when computing
   /// token range of \p D.
   void noticeDeclWithoutSemicolon(Decl *D);
@@ -237,11 +225,6 @@ class syntax::TreeBuilder {
   void markChild(syntax::Node *N, NodeRole R);
   /// Set role for the syntax node matching \p N.
   void markChild(ASTPtr N, NodeRole R);
-  /// Set role for the delayed node that spans exactly \p Range.
-  void markDelayedChild(llvm::ArrayRef<syntax::Token> Range, NodeRole R);
-  /// Set role for the node that may or may not be delayed. Node must span
-  /// exactly \p Range.
-  void markMaybeDelayedChild(llvm::ArrayRef<syntax::Token> Range, NodeRole R);
 
   /// Finish building the tree and consume the root node.
   syntax::TranslationUnit *finalize() && {
@@ -285,7 +268,38 @@ class syntax::TreeBuilder {
     return maybeAppendSemicolon(Tokens, D);
   }
 
-  llvm::ArrayRef<syntax::Token> getDeclRange(const Decl *D) const {
+  /// Returns true if \p D is the last declarator in a chain and is thus
+  /// reponsible for creating SimpleDeclaration for the whole chain.
+  template <class T>
+  bool isResponsibleForCreatingDeclaration(const T *D) const {
+    static_assert((std::is_base_of<DeclaratorDecl, T>::value ||
+                   std::is_base_of<TypedefNameDecl, T>::value),
+                  "only DeclaratorDecl and TypedefNameDecl are supported.");
+
+    const Decl *Next = D->getNextDeclInContext();
+
+    // There's no next sibling, this one is responsible.
+    if (Next == nullptr) {
+      return true;
+    }
+    const auto *NextT = llvm::dyn_cast<T>(Next);
+
+    // Next sibling is not the same type, this one is responsible.
+    if (NextT == nullptr) {
+      return true;
+    }
+    // Next sibling doesn't begin at the same loc, it must be a different
+    // declaration, so this declarator is responsible.
+    if (NextT->getBeginLoc() != D->getBeginLoc()) {
+      return true;
+    }
+
+    // NextT is a member of the same declaration, and we need the last member to
+    // create declaration. This one is not responsible.
+    return false;
+  }
+
+  llvm::ArrayRef<syntax::Token> getDeclarationRange(Decl *D) {
     llvm::ArrayRef<clang::syntax::Token> Tokens;
     // We want to drop the template parameters for specializations.
     if (const auto *S = llvm::dyn_cast<TagDecl>(D))
@@ -294,9 +308,11 @@ class syntax::TreeBuilder {
       Tokens = getRange(D->getSourceRange());
     return maybeAppendSemicolon(Tokens, D);
   }
+
   llvm::ArrayRef<syntax::Token> getExprRange(const Expr *E) const {
     return getRange(E->getSourceRange());
   }
+
   /// Find the adjusted range for the statement, consuming the trailing
   /// semicolon when needed.
   llvm::ArrayRef<syntax::Token> getStmtRange(const Stmt *S) const {
@@ -359,25 +375,6 @@ class syntax::TreeBuilder {
       }
     }
 
-    ~Forest() { assert(DelayedFolds.empty()); }
-
-    void assignRoleDelayed(llvm::ArrayRef<syntax::Token> Range,
-                           syntax::NodeRole Role) {
-      auto It = DelayedFolds.find(Range.begin());
-      assert(It != DelayedFolds.end());
-      assert(It->second.End == Range.end());
-      It->second.Role = Role;
-    }
-
-    void assignRoleMaybeDelayed(llvm::ArrayRef<syntax::Token> Range,
-                                syntax::NodeRole Role) {
-      auto It = DelayedFolds.find(Range.begin());
-      if (It == DelayedFolds.end())
-        return assignRole(Range, Role);
-      assert(It->second.End == Range.end());
-      It->second.Role = Role;
-    }
-
     void assignRole(llvm::ArrayRef<syntax::Token> Range,
                     syntax::NodeRole Role) {
       assert(!Range.empty());
@@ -396,50 +393,34 @@ class syntax::TreeBuilder {
     void foldChildren(const syntax::Arena &A,
                       llvm::ArrayRef<syntax::Token> Tokens,
                       syntax::Tree *Node) {
-      // Execute delayed folds inside `Tokens`.
-      auto BeginFolds = DelayedFolds.lower_bound(Tokens.begin());
-      auto EndFolds = BeginFolds;
-      for (; EndFolds != DelayedFolds.end() &&
-             EndFolds->second.End <= Tokens.end();
-           ++EndFolds)
-        ;
-      // We go in reverse order to ensure we fold deeper nodes first.
-      for (auto RevIt = EndFolds; RevIt != BeginFolds; --RevIt) {
-        auto It = std::prev(RevIt);
-        foldChildrenEager(A, llvm::makeArrayRef(It->first, It->second.End),
-                          It->second.Node);
-      }
-      DelayedFolds.erase(BeginFolds, EndFolds);
-
       // Attach children to `Node`.
-      foldChildrenEager(A, Tokens, Node);
-    }
+      assert(Node->firstChild() == nullptr && "node already has children");
 
-    /// Schedule a call to `foldChildren` that will only be executed when
-    /// containing node is folded. The range of delayed nodes can be extended by
-    /// calling `extendDelayedFold`. Only one delayed node for each starting
-    /// token is allowed.
-    void foldChildrenDelayed(llvm::ArrayRef<syntax::Token> Tokens,
-                             syntax::Tree *Node) {
-      assert(!Tokens.empty());
-      bool Inserted =
-          DelayedFolds.insert({Tokens.begin(), DelayedFold{Tokens.end(), Node}})
-              .second;
-      (void)Inserted;
-      assert(Inserted && "Multiple delayed folds start at the same token");
-    }
+      auto *FirstToken = Tokens.begin();
+      auto BeginChildren = Trees.lower_bound(FirstToken);
 
-    /// If there a delayed fold, starting at `ExtendedRange.begin()`, extends
-    /// its endpoint to `ExtendedRange.end()` and returns true.
-    /// Otherwise, returns false.
-    bool extendDelayedFold(llvm::ArrayRef<syntax::Token> ExtendedRange) {
-      assert(!ExtendedRange.empty());
-      auto It = DelayedFolds.find(ExtendedRange.data());
-      if (It == DelayedFolds.end())
-        return false;
-      assert(It->second.End <= ExtendedRange.end());
-      It->second.End = ExtendedRange.end();
-      return true;
+      assert((BeginChildren == Trees.end() ||
+              BeginChildren->first == FirstToken) &&
+             "fold crosses boundaries of existing subtrees");
+      auto EndChildren = Trees.lower_bound(Tokens.end());
+      assert(
+          (EndChildren == Trees.end() || EndChildren->first == Tokens.end()) &&
+          "fold crosses boundaries of existing subtrees");
+
+      // We need to go in reverse order, because we can only prepend.
+      for (auto It = EndChildren; It != BeginChildren; --It) {
+        auto *C = std::prev(It)->second;
+        if (C->role() == NodeRole::Detached)
+          C->setRole(NodeRole::Unknown);
+        Node->prependChildLowLevel(C);
+      }
+
+      // Mark that this node came from the AST and is backed by the source code.
+      Node->Original = true;
+      Node->CanModify = A.tokenBuffer().spelledForExpanded(Tokens).hasValue();
+
+      Trees.erase(BeginChildren, EndChildren);
+      Trees.insert({FirstToken, Node});
     }
 
     // EXPECTS: all tokens were consumed and are owned by a single root node.
@@ -467,51 +448,10 @@ class syntax::TreeBuilder {
     }
 
   private:
-    /// Implementation detail of `foldChildren`, does acutal folding ignoring
-    /// delayed folds.
-    void foldChildrenEager(const syntax::Arena &A,
-                           llvm::ArrayRef<syntax::Token> Tokens,
-                           syntax::Tree *Node) {
-      assert(Node->firstChild() == nullptr && "node already has children");
-
-      auto *FirstToken = Tokens.begin();
-      auto BeginChildren = Trees.lower_bound(FirstToken);
-      assert((BeginChildren == Trees.end() ||
-              BeginChildren->first == FirstToken) &&
-             "fold crosses boundaries of existing subtrees");
-      auto EndChildren = Trees.lower_bound(Tokens.end());
-      assert(
-          (EndChildren == Trees.end() || EndChildren->first == Tokens.end()) &&
-          "fold crosses boundaries of existing subtrees");
-
-      // We need to go in reverse order, because we can only prepend.
-      for (auto It = EndChildren; It != BeginChildren; --It) {
-        auto *C = std::prev(It)->second;
-        if (C->role() == NodeRole::Detached)
-          C->setRole(NodeRole::Unknown);
-        Node->prependChildLowLevel(C);
-      }
-
-      // Mark that this node came from the AST and is backed by the source code.
-      Node->Original = true;
-      Node->CanModify = A.tokenBuffer().spelledForExpanded(Tokens).hasValue();
-
-      Trees.erase(BeginChildren, EndChildren);
-      Trees.insert({FirstToken, Node});
-    }
-
     /// Maps from the start token to a subtree starting at that token.
     /// Keys in the map are pointers into the array of expanded tokens, so
     /// pointer order corresponds to the order of preprocessor tokens.
     std::map<const syntax::Token *, syntax::Node *> Trees;
-
-    /// See documentation of `foldChildrenDelayed` for details.
-    struct DelayedFold {
-      const syntax::Token *End = nullptr;
-      syntax::Tree *Node = nullptr;
-      NodeRole Role = NodeRole::Unknown;
-    };
-    std::map<const syntax::Token *, DelayedFold> DelayedFolds;
   };
 
   /// For debugging purposes.
@@ -535,47 +475,16 @@ class BuildTreeVisitor : public RecursiveASTVisitor<BuildTreeVisitor> {
   bool shouldTraversePostOrder() const { return true; }
 
   bool WalkUpFromDeclaratorDecl(DeclaratorDecl *DD) {
-    // Ensure declarators are covered by SimpleDeclaration.
-    Builder.noticeDeclRange(Builder.getDeclRange(DD));
-
-    // Build the declarator node.
-    SourceRange Initializer;
-    if (auto *V = llvm::dyn_cast<VarDecl>(DD)) {
-      auto *I = V->getInit();
-      // Initializers in range-based-for are not part of the declarator
-      if (I && !V->isCXXForRangeDecl())
-        Initializer = I->getSourceRange();
-    }
-    auto Declarator = getDeclaratorRange(
-        Builder.sourceManager(), DD->getTypeSourceInfo()->getTypeLoc(),
-        getQualifiedNameStart(DD), Initializer);
-    if (Declarator.isValid()) {
-      auto *N = new (allocator()) syntax::SimpleDeclarator;
-      Builder.foldNode(Builder.getRange(Declarator), N, DD);
-      Builder.markChild(N, syntax::NodeRole::SimpleDeclaration_declarator);
-    }
-
-    return true;
+    return processDeclaratorAndDeclaration(DD);
   }
 
-  bool WalkUpFromTypedefNameDecl(TypedefNameDecl *D) {
-    // Ensure declarators are covered by SimpleDeclaration.
-    Builder.noticeDeclRange(Builder.getDeclRange(D));
-
-    auto R = getDeclaratorRange(
-        Builder.sourceManager(), D->getTypeSourceInfo()->getTypeLoc(),
-        /*Name=*/D->getLocation(), /*Initializer=*/SourceRange());
-    if (R.isValid()) {
-      auto *N = new (allocator()) syntax::SimpleDeclarator;
-      Builder.foldNode(Builder.getRange(R), N, D);
-      Builder.markChild(N, syntax::NodeRole::SimpleDeclaration_declarator);
-    }
-    return true;
+  bool WalkUpFromTypedefNameDecl(TypedefNameDecl *TD) {
+    return processDeclaratorAndDeclaration(TD);
   }
 
   bool VisitDecl(Decl *D) {
     assert(!D->isImplicit());
-    Builder.foldNode(Builder.getDeclRange(D),
+    Builder.foldNode(Builder.getDeclarationRange(D),
                      new (allocator()) syntax::UnknownDeclaration(), D);
     return true;
   }
@@ -599,9 +508,9 @@ class BuildTreeVisitor : public RecursiveASTVisitor<BuildTreeVisitor> {
 
   bool WalkUpFromTemplateDecl(TemplateDecl *S) {
     foldTemplateDeclaration(
-        Builder.getDeclRange(S),
+        Builder.getDeclarationRange(S),
         Builder.findToken(S->getTemplateParameters()->getTemplateLoc()),
-        Builder.getDeclRange(S->getTemplatedDecl()), S);
+        Builder.getDeclarationRange(S->getTemplatedDecl()), S);
     return true;
   }
 
@@ -618,7 +527,7 @@ class BuildTreeVisitor : public RecursiveASTVisitor<BuildTreeVisitor> {
   syntax::Declaration *handleFreeStandingTagDecl(TagDecl *C) {
     assert(C->isFreeStanding());
     // Class is a declaration specifier and needs a spanning declaration node.
-    auto DeclarationRange = Builder.getDeclRange(C);
+    auto DeclarationRange = Builder.getDeclarationRange(C);
     syntax::Declaration *Result = new (allocator()) syntax::SimpleDeclaration;
     Builder.foldNode(DeclarationRange, Result, nullptr);
 
@@ -702,7 +611,7 @@ class BuildTreeVisitor : public RecursiveASTVisitor<BuildTreeVisitor> {
   }
 
   bool WalkUpFromNamespaceDecl(NamespaceDecl *S) {
-    auto Tokens = Builder.getDeclRange(S);
+    auto Tokens = Builder.getDeclarationRange(S);
     if (Tokens.front().kind() == tok::coloncolon) {
       // Handle nested namespace definitions. Those start at '::' token, e.g.
       // namespace a^::b {}
@@ -741,10 +650,9 @@ class BuildTreeVisitor : public RecursiveASTVisitor<BuildTreeVisitor> {
 
   bool WalkUpFromFunctionTypeLoc(FunctionTypeLoc L) {
     Builder.markChildToken(L.getLParenLoc(), syntax::NodeRole::OpenParen);
-    for (auto *P : L.getParams())
-      Builder.markDelayedChild(
-          Builder.getDeclRange(P),
-          syntax::NodeRole::ParametersAndQualifiers_parameter);
+    for (auto *P : L.getParams()) {
+      Builder.markChild(P, syntax::NodeRole::ParametersAndQualifiers_parameter);
+    }
     Builder.markChildToken(L.getRParenLoc(), syntax::NodeRole::CloseParen);
     Builder.foldNode(Builder.getRange(L.getLParenLoc(), L.getEndLoc()),
                      new (allocator()) syntax::ParametersAndQualifiers, L);
@@ -755,7 +663,7 @@ class BuildTreeVisitor : public RecursiveASTVisitor<BuildTreeVisitor> {
     if (!L.getTypePtr()->hasTrailingReturn())
       return WalkUpFromFunctionTypeLoc(L);
 
-    auto TrailingReturnTokens = BuildTrailingReturn(L);
+    auto *TrailingReturnTokens = BuildTrailingReturn(L);
     // Finish building the node for parameters.
     Builder.markChild(TrailingReturnTokens,
                       syntax::NodeRole::ParametersAndQualifiers_trailingReturn);
@@ -877,7 +785,7 @@ class BuildTreeVisitor : public RecursiveASTVisitor<BuildTreeVisitor> {
   }
 
   bool WalkUpFromEmptyDecl(EmptyDecl *S) {
-    Builder.foldNode(Builder.getDeclRange(S),
+    Builder.foldNode(Builder.getDeclarationRange(S),
                      new (allocator()) syntax::EmptyDeclaration, S);
     return true;
   }
@@ -887,55 +795,108 @@ class BuildTreeVisitor : public RecursiveASTVisitor<BuildTreeVisitor> {
                           syntax::NodeRole::StaticAssertDeclaration_condition);
     Builder.markExprChild(S->getMessage(),
                           syntax::NodeRole::StaticAssertDeclaration_message);
-    Builder.foldNode(Builder.getDeclRange(S),
+    Builder.foldNode(Builder.getDeclarationRange(S),
                      new (allocator()) syntax::StaticAssertDeclaration, S);
     return true;
   }
 
   bool WalkUpFromLinkageSpecDecl(LinkageSpecDecl *S) {
-    Builder.foldNode(Builder.getDeclRange(S),
+    Builder.foldNode(Builder.getDeclarationRange(S),
                      new (allocator()) syntax::LinkageSpecificationDeclaration,
                      S);
     return true;
   }
 
   bool WalkUpFromNamespaceAliasDecl(NamespaceAliasDecl *S) {
-    Builder.foldNode(Builder.getDeclRange(S),
+    Builder.foldNode(Builder.getDeclarationRange(S),
                      new (allocator()) syntax::NamespaceAliasDefinition, S);
     return true;
   }
 
   bool WalkUpFromUsingDirectiveDecl(UsingDirectiveDecl *S) {
-    Builder.foldNode(Builder.getDeclRange(S),
+    Builder.foldNode(Builder.getDeclarationRange(S),
                      new (allocator()) syntax::UsingNamespaceDirective, S);
     return true;
   }
 
   bool WalkUpFromUsingDecl(UsingDecl *S) {
-    Builder.foldNode(Builder.getDeclRange(S),
+    Builder.foldNode(Builder.getDeclarationRange(S),
                      new (allocator()) syntax::UsingDeclaration, S);
     return true;
   }
 
   bool WalkUpFromUnresolvedUsingValueDecl(UnresolvedUsingValueDecl *S) {
-    Builder.foldNode(Builder.getDeclRange(S),
+    Builder.foldNode(Builder.getDeclarationRange(S),
                      new (allocator()) syntax::UsingDeclaration, S);
     return true;
   }
 
   bool WalkUpFromUnresolvedUsingTypenameDecl(UnresolvedUsingTypenameDecl *S) {
-    Builder.foldNode(Builder.getDeclRange(S),
+    Builder.foldNode(Builder.getDeclarationRange(S),
                      new (allocator()) syntax::UsingDeclaration, S);
     return true;
   }
 
   bool WalkUpFromTypeAliasDecl(TypeAliasDecl *S) {
-    Builder.foldNode(Builder.getDeclRange(S),
+    Builder.foldNode(Builder.getDeclarationRange(S),
                      new (allocator()) syntax::TypeAliasDeclaration, S);
     return true;
   }
 
 private:
+  template <class T> SourceLocation getQualifiedNameStart(T *D) {
+    static_assert((std::is_base_of<DeclaratorDecl, T>::value ||
+                   std::is_base_of<TypedefNameDecl, T>::value),
+                  "only DeclaratorDecl and TypedefNameDecl are supported.");
+
+    auto DN = D->getDeclName();
+    bool IsAnonymous = DN.isIdentifier() && !DN.getAsIdentifierInfo();
+    if (IsAnonymous)
+      return SourceLocation();
+
+    if (const auto *DD = llvm::dyn_cast<DeclaratorDecl>(D)) {
+      if (DD->getQualifierLoc()) {
+        return DD->getQualifierLoc().getBeginLoc();
+      }
+    }
+
+    return D->getLocation();
+  }
+
+  SourceRange getInitializerRange(Decl *D) {
+    if (auto *V = llvm::dyn_cast<VarDecl>(D)) {
+      auto *I = V->getInit();
+      // Initializers in range-based-for are not part of the declarator
+      if (I && !V->isCXXForRangeDecl())
+        return I->getSourceRange();
+    }
+
+    return SourceRange();
+  }
+
+  /// Folds SimpleDeclarator node (if present) and in case this is the last
+  /// declarator in the chain it also folds SimpleDeclaration node.
+  template <class T> bool processDeclaratorAndDeclaration(T *D) {
+    SourceRange Initializer = getInitializerRange(D);
+    auto Range = getDeclaratorRange(Builder.sourceManager(),
+                                    D->getTypeSourceInfo()->getTypeLoc(),
+                                    getQualifiedNameStart(D), Initializer);
+
+    // There doesn't have to be a declarator (e.g. `void foo(int)` only has
+    // declaration, but no declarator).
+    if (Range.getBegin().isValid()) {
+      auto *N = new (allocator()) syntax::SimpleDeclarator;
+      Builder.foldNode(Builder.getRange(Range), N, nullptr);
+      Builder.markChild(N, syntax::NodeRole::SimpleDeclaration_declarator);
+    }
+
+    if (Builder.isResponsibleForCreatingDeclaration(D)) {
+      Builder.foldNode(Builder.getDeclarationRange(D),
+                       new (allocator()) syntax::SimpleDeclaration, D);
+    }
+    return true;
+  }
+
   /// Returns the range of the built node.
   syntax::TrailingReturnType *BuildTrailingReturn(FunctionProtoTypeLoc L) {
     assert(L.getTypePtr()->hasTrailingReturn());
@@ -963,7 +924,7 @@ class BuildTreeVisitor : public RecursiveASTVisitor<BuildTreeVisitor> {
       Builder.markChild(ReturnDeclarator,
                         syntax::NodeRole::TrailingReturnType_declarator);
     auto *R = new (allocator()) syntax::TrailingReturnType;
-    Builder.foldNode(Tokens, R, nullptr);
+    Builder.foldNode(Tokens, R, L);
     return R;
   }
 
@@ -989,12 +950,10 @@ class BuildTreeVisitor : public RecursiveASTVisitor<BuildTreeVisitor> {
       ArrayRef<syntax::Token> TemplatedDeclaration, Decl *From) {
     assert(TemplateKW && TemplateKW->kind() == tok::kw_template);
     Builder.markChildToken(TemplateKW, syntax::NodeRole::IntroducerKeyword);
-    Builder.markMaybeDelayedChild(
-        TemplatedDeclaration,
-        syntax::NodeRole::TemplateDeclaration_declaration);
 
     auto *N = new (allocator()) syntax::TemplateDeclaration;
     Builder.foldNode(Range, N, From);
+    Builder.markChild(N, syntax::NodeRole::TemplateDeclaration_declaration);
     return N;
   }
 
@@ -1006,13 +965,6 @@ class BuildTreeVisitor : public RecursiveASTVisitor<BuildTreeVisitor> {
 };
 } // namespace
 
-void syntax::TreeBuilder::noticeDeclRange(llvm::ArrayRef<syntax::Token> Range) {
-  if (Pending.extendDelayedFold(Range))
-    return;
-  Pending.foldChildrenDelayed(Range,
-                              new (allocator()) syntax::SimpleDeclaration);
-}
-
 void syntax::TreeBuilder::noticeDeclWithoutSemicolon(Decl *D) {
   DeclsWithoutSemicolons.insert(D);
 }
@@ -1040,16 +992,6 @@ void syntax::TreeBuilder::markChild(ASTPtr N, NodeRole R) {
   setRole(SN, R);
 }
 
-void syntax::TreeBuilder::markDelayedChild(llvm::ArrayRef<syntax::Token> Range,
-                                           NodeRole R) {
-  Pending.assignRoleDelayed(Range, R);
-}
-
-void syntax::TreeBuilder::markMaybeDelayedChild(
-    llvm::ArrayRef<syntax::Token> Range, NodeRole R) {
-  Pending.assignRoleMaybeDelayed(Range, R);
-}
-
 void syntax::TreeBuilder::markStmtChild(Stmt *Child, NodeRole Role) {
   if (!Child)
     return;
diff --git a/clang/lib/Tooling/Syntax/Tokens.cpp b/clang/lib/Tooling/Syntax/Tokens.cpp
index 9e12d8b603bfe..af11f25b10581 100644
--- a/clang/lib/Tooling/Syntax/Tokens.cpp
+++ b/clang/lib/Tooling/Syntax/Tokens.cpp
@@ -35,6 +35,69 @@
 using namespace clang;
 using namespace clang::syntax;
 
+namespace {
+// Finds the smallest consecutive subsuquence of Toks that covers R.
+llvm::ArrayRef<syntax::Token>
+getTokensCovering(llvm::ArrayRef<syntax::Token> Toks, SourceRange R,
+                  const SourceManager &SM) {
+  if (R.isInvalid())
+    return {};
+  const syntax::Token *Begin =
+      llvm::partition_point(Toks, [&](const syntax::Token &T) {
+        return SM.isBeforeInTranslationUnit(T.location(), R.getBegin());
+      });
+  const syntax::Token *End =
+      llvm::partition_point(Toks, [&](const syntax::Token &T) {
+        return !SM.isBeforeInTranslationUnit(R.getEnd(), T.location());
+      });
+  if (Begin > End)
+    return {};
+  return {Begin, End};
+}
+
+// Finds the smallest expansion range that contains expanded tokens First and
+// Last, e.g.:
+// #define ID(x) x
+// ID(ID(ID(a1) a2))
+//          ~~       -> a1
+//              ~~   -> a2
+//       ~~~~~~~~~   -> a1 a2
+SourceRange findCommonRangeForMacroArgs(const syntax::Token &First,
+                                        const syntax::Token &Last,
+                                        const SourceManager &SM) {
+  SourceRange Res;
+  auto FirstLoc = First.location(), LastLoc = Last.location();
+  // Keep traversing up the spelling chain as longs as tokens are part of the
+  // same expansion.
+  while (!FirstLoc.isFileID() && !LastLoc.isFileID()) {
+    auto ExpInfoFirst = SM.getSLocEntry(SM.getFileID(FirstLoc)).getExpansion();
+    auto ExpInfoLast = SM.getSLocEntry(SM.getFileID(LastLoc)).getExpansion();
+    // Stop if expansions have diverged.
+    if (ExpInfoFirst.getExpansionLocStart() !=
+        ExpInfoLast.getExpansionLocStart())
+      break;
+    // Do not continue into macro bodies.
+    if (!ExpInfoFirst.isMacroArgExpansion() ||
+        !ExpInfoLast.isMacroArgExpansion())
+      break;
+    FirstLoc = SM.getImmediateSpellingLoc(FirstLoc);
+    LastLoc = SM.getImmediateSpellingLoc(LastLoc);
+    // Update the result afterwards, as we want the tokens that triggered the
+    // expansion.
+    Res = {FirstLoc, LastLoc};
+  }
+  // Normally mapping back to expansion location here only changes FileID, as
+  // we've already found some tokens expanded from the same macro argument, and
+  // they should map to a consecutive subset of spelled tokens. Unfortunately
+  // SourceManager::isBeforeInTranslationUnit discriminates sourcelocations
+  // based on their FileID in addition to offsets. So even though we are
+  // referring to same tokens, SourceManager might tell us that one is before
+  // the other if they've got different FileIDs.
+  return SM.getExpansionRange(CharSourceRange(Res, true)).getAsRange();
+}
+
+} // namespace
+
 syntax::Token::Token(SourceLocation Location, unsigned Length,
                      tok::TokenKind Kind)
     : Location(Location), Length(Length), Kind(Kind) {
@@ -121,19 +184,7 @@ llvm::StringRef FileRange::text(const SourceManager &SM) const {
 }
 
 llvm::ArrayRef<syntax::Token> TokenBuffer::expandedTokens(SourceRange R) const {
-  if (R.isInvalid())
-    return {};
-  const Token *Begin =
-      llvm::partition_point(expandedTokens(), [&](const syntax::Token &T) {
-        return SourceMgr->isBeforeInTranslationUnit(T.location(), R.getBegin());
-      });
-  const Token *End =
-      llvm::partition_point(expandedTokens(), [&](const syntax::Token &T) {
-        return !SourceMgr->isBeforeInTranslationUnit(R.getEnd(), T.location());
-      });
-  if (Begin > End)
-    return {};
-  return {Begin, End};
+  return getTokensCovering(expandedTokens(), R, *SourceMgr);
 }
 
 CharSourceRange FileRange::toCharRange(const SourceManager &SM) const {
@@ -206,8 +257,6 @@ TokenBuffer::spelledForExpanded(llvm::ArrayRef<syntax::Token> Expanded) const {
   if (Expanded.empty())
     return llvm::None;
 
-  // FIXME: also allow changes uniquely mapping to macro arguments.
-
   const syntax::Token *BeginSpelled;
   const Mapping *BeginMapping;
   std::tie(BeginSpelled, BeginMapping) =
@@ -225,12 +274,28 @@ TokenBuffer::spelledForExpanded(llvm::ArrayRef<syntax::Token> Expanded) const {
 
   const MarkedFile &File = Files.find(FID)->second;
 
-  // Do not allow changes that cross macro expansion boundaries.
+  // If both tokens are coming from a macro argument expansion, try and map to
+  // smallest part of the macro argument. BeginMapping && LastMapping check is
+  // only for performance, they are a prerequisite for Expanded.front() and
+  // Expanded.back() being part of a macro arg expansion.
+  if (BeginMapping && LastMapping &&
+      SourceMgr->isMacroArgExpansion(Expanded.front().location()) &&
+      SourceMgr->isMacroArgExpansion(Expanded.back().location())) {
+    auto CommonRange = findCommonRangeForMacroArgs(Expanded.front(),
+                                                   Expanded.back(), *SourceMgr);
+    // It might be the case that tokens are arguments of different macro calls,
+    // in that case we should continue with the logic below instead of returning
+    // an empty range.
+    if (CommonRange.isValid())
+      return getTokensCovering(File.SpelledTokens, CommonRange, *SourceMgr);
+  }
+
+  // Do not allow changes that doesn't cover full expansion.
   unsigned BeginExpanded = Expanded.begin() - ExpandedTokens.data();
   unsigned EndExpanded = Expanded.end() - ExpandedTokens.data();
-  if (BeginMapping && BeginMapping->BeginExpanded < BeginExpanded)
+  if (BeginMapping && BeginExpanded != BeginMapping->BeginExpanded)
     return llvm::None;
-  if (LastMapping && EndExpanded < LastMapping->EndExpanded)
+  if (LastMapping && LastMapping->EndExpanded != EndExpanded)
     return llvm::None;
   // All is good, return the result.
   return llvm::makeArrayRef(
@@ -645,8 +710,8 @@ std::string syntax::Token::str() const {
 }
 
 std::string syntax::Token::dumpForTests(const SourceManager &SM) const {
-  return std::string(
-      llvm::formatv("{0}   {1}", tok::getTokenName(kind()), text(SM)));
+  return std::string(llvm::formatv("Token(`{0}`, {1}, length = {2})", text(SM),
+                                   tok::getTokenName(kind()), length()));
 }
 
 std::string TokenBuffer::dumpForTests() const {
diff --git a/clang/lib/Tooling/Transformer/SourceCode.cpp b/clang/lib/Tooling/Transformer/SourceCode.cpp
index 38b331b0a1c3b..0530b3fa0e959 100644
--- a/clang/lib/Tooling/Transformer/SourceCode.cpp
+++ b/clang/lib/Tooling/Transformer/SourceCode.cpp
@@ -20,6 +20,7 @@
 #include "clang/Basic/SourceManager.h"
 #include "clang/Lex/Lexer.h"
 #include "llvm/Support/Errc.h"
+#include <set>
 
 using namespace clang;
 
diff --git a/clang/test/AST/ast-dump-expr-errors.cpp b/clang/test/AST/ast-dump-expr-errors.cpp
index e623fad04f4c8..9334b73a43545 100644
--- a/clang/test/AST/ast-dump-expr-errors.cpp
+++ b/clang/test/AST/ast-dump-expr-errors.cpp
@@ -42,5 +42,9 @@ int d = static_cast<int>(bar() + 1);
 
 // FIXME: store initializer even when 'auto' could not be deduced.
 // Expressions with errors currently do not keep initializers around.
-// CHECK:     `-VarDecl {{.*}} invalid e 'auto'
+// CHECK: -VarDecl {{.*}} invalid e 'auto'
 auto e = bar();
+
+// Error type should result in an invalid decl.
+// CHECK: -VarDecl {{.*}} invalid f 'decltype(<recovery-expr>(bar))'
+decltype(bar()) f;
diff --git a/clang/test/AST/ast-dump-openmp-begin-declare-variant_1.c b/clang/test/AST/ast-dump-openmp-begin-declare-variant_1.c
new file mode 100644
index 0000000000000..f427049807c61
--- /dev/null
+++ b/clang/test/AST/ast-dump-openmp-begin-declare-variant_1.c
@@ -0,0 +1,55 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -fopenmp -verify -ast-dump %s       | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -fopenmp -verify -ast-dump %s -x c++| FileCheck %s
+// expected-no-diagnostics
+
+int also_before(void) {
+  return 0;
+}
+
+#pragma omp begin declare variant match(device={kind(gpu)})
+int also_after(void) {
+  return 1;
+}
+int also_before(void) {
+  return 2;
+}
+#pragma omp end declare variant
+
+
+#pragma omp begin declare variant match(device={kind(fpga)})
+
+This text is never parsed!
+
+#pragma omp end declare variant
+
+int also_after(void) {
+  return 0;
+}
+
+int test() {
+  // Should return 0.
+  return also_after() + also_before();
+}
+
+// Make sure:
+//  - we do not see the ast nodes for the gpu kind
+//  - we do not choke on the text in the kind(fpga) guarded scopes
+
+// CHECK:      |-FunctionDecl [[ADDR_0:0x[a-z0-9]*]] <{{.*}}, line:7:1> line:5:5 used also_before 'int ({{.*}})'
+// CHECK-NEXT: | `-CompoundStmt [[ADDR_1:0x[a-z0-9]*]] <col:23, line:7:1>
+// CHECK-NEXT: |   `-ReturnStmt [[ADDR_2:0x[a-z0-9]*]] <line:6:3, col:10>
+// CHECK-NEXT: |     `-IntegerLiteral [[ADDR_3:0x[a-z0-9]*]] <col:10> 'int' 0
+// CHECK-NEXT: |-FunctionDecl [[ADDR_4:0x[a-z0-9]*]] <line:25:1, line:27:1> line:25:5 used also_after 'int ({{.*}})'
+// CHECK-NEXT: | `-CompoundStmt [[ADDR_5:0x[a-z0-9]*]] <col:22, line:27:1>
+// CHECK-NEXT: |   `-ReturnStmt [[ADDR_6:0x[a-z0-9]*]] <line:26:3, col:10>
+// CHECK-NEXT: |     `-IntegerLiteral [[ADDR_7:0x[a-z0-9]*]] <col:10> 'int' 0
+// CHECK-NEXT: `-FunctionDecl [[ADDR_8:0x[a-z0-9]*]] <line:29:1, line:32:1> line:29:5 test 'int ({{.*}})'
+// CHECK-NEXT:   `-CompoundStmt [[ADDR_9:0x[a-z0-9]*]] <col:12, line:32:1>
+// CHECK-NEXT:     `-ReturnStmt [[ADDR_10:0x[a-z0-9]*]] <line:31:3, col:37>
+// CHECK-NEXT:       `-BinaryOperator [[ADDR_11:0x[a-z0-9]*]] <col:10, col:37> 'int' '+'
+// CHECK-NEXT:         |-CallExpr [[ADDR_12:0x[a-z0-9]*]] <col:10, col:21> 'int'
+// CHECK-NEXT:         | `-ImplicitCastExpr [[ADDR_13:0x[a-z0-9]*]] <col:10> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CHECK-NEXT:         |   `-DeclRefExpr [[ADDR_14:0x[a-z0-9]*]] <col:10> 'int ({{.*}})' {{.*}}Function [[ADDR_4]] 'also_after' 'int ({{.*}})'
+// CHECK-NEXT:         `-CallExpr [[ADDR_15:0x[a-z0-9]*]] <col:25, col:37> 'int'
+// CHECK-NEXT:           `-ImplicitCastExpr [[ADDR_16:0x[a-z0-9]*]] <col:25> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CHECK-NEXT:             `-DeclRefExpr [[ADDR_17:0x[a-z0-9]*]] <col:25> 'int ({{.*}})' {{.*}}Function [[ADDR_0]] 'also_before' 'int ({{.*}})'
diff --git a/clang/test/AST/ast-dump-openmp-begin-declare-variant_2.c b/clang/test/AST/ast-dump-openmp-begin-declare-variant_2.c
new file mode 100644
index 0000000000000..fda973b1e6c4c
--- /dev/null
+++ b/clang/test/AST/ast-dump-openmp-begin-declare-variant_2.c
@@ -0,0 +1,82 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -fopenmp -verify -ast-dump %s       | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -fopenmp -verify -ast-dump %s -x c++| FileCheck %s
+// expected-no-diagnostics
+
+#pragma omp begin declare variant match(device={kind(cpu)})
+int also_before(void) {
+  return 0;
+}
+#pragma omp end declare variant
+
+#pragma omp begin declare variant match(implementation={vendor(score(100):llvm)})
+int also_after(void) {
+  return 0;
+}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(implementation={vendor(score(0):llvm)})
+int also_before(void) {
+  return 1;
+}
+#pragma omp end declare variant
+
+int also_after(void) {
+  return 2;
+}
+
+int test() {
+  // Should return 0.
+  return also_after() + also_before();
+}
+
+// Make sure:
+//  - we do see the ast nodes for the cpu kind
+//  - we do see the ast nodes for the llvm vendor
+//  - we pick the right callees
+
+// CHECK:      |-FunctionDecl [[ADDR_0:0x[a-z0-9]*]] <{{.*}}, col:21> col:5 implicit used also_before 'int ({{.*}})'
+// CHECK-NEXT: | `-OMPDeclareVariantAttr [[ADDR_1:0x[a-z0-9]*]] <<invalid sloc>> Implicit device={kind(cpu)}
+// CHECK-NEXT: |   `-DeclRefExpr [[ADDR_2:0x[a-z0-9]*]] <col:1> 'int ({{.*}})' Function [[ADDR_3:0x[a-z0-9]*]] 'also_before[device={kind(cpu)}]' 'int ({{.*}})'
+// CHECK-NEXT: |-FunctionDecl [[ADDR_3]] <col:1, line:8:1> line:6:1 also_before[device={kind(cpu)}] 'int ({{.*}})'
+// CHECK-NEXT: | `-CompoundStmt [[ADDR_4:0x[a-z0-9]*]] <col:23, line:8:1>
+// CHECK-NEXT: |   `-ReturnStmt [[ADDR_5:0x[a-z0-9]*]] <line:7:3, col:10>
+// CHECK-NEXT: |     `-IntegerLiteral [[ADDR_6:0x[a-z0-9]*]] <col:10> 'int' 0
+// CHECK-NEXT: |-FunctionDecl [[ADDR_7:0x[a-z0-9]*]] <line:12:1, col:20> col:5 implicit used also_after 'int ({{.*}})'
+// CHECK-NEXT: | `-OMPDeclareVariantAttr [[ADDR_8:0x[a-z0-9]*]] <<invalid sloc>> Implicit implementation={vendor(score(100): llvm)}
+// CHECK-NEXT: |   `-DeclRefExpr [[ADDR_9:0x[a-z0-9]*]] <col:1> 'int ({{.*}})' Function [[ADDR_10:0x[a-z0-9]*]] 'also_after[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// CHECK-NEXT: |-FunctionDecl [[ADDR_10]] <col:1, line:14:1> line:12:1 also_after[implementation={vendor(llvm)}] 'int ({{.*}})'
+// CHECK-NEXT: | `-CompoundStmt [[ADDR_11:0x[a-z0-9]*]] <col:22, line:14:1>
+// CHECK-NEXT: |   `-ReturnStmt [[ADDR_12:0x[a-z0-9]*]] <line:13:3, col:10>
+// CHECK-NEXT: |     `-IntegerLiteral [[ADDR_13:0x[a-z0-9]*]] <col:10> 'int' 0
+// CHECK-NEXT: |-FunctionDecl [[ADDR_14:0x[a-z0-9]*]] prev [[ADDR_0]] <line:17:1, col:21> col:5 implicit used also_before 'int ({{.*}})'
+// CHECK-NEXT: | |-OMPDeclareVariantAttr [[ADDR_15:0x[a-z0-9]*]] <<invalid sloc>> Inherited Implicit device={kind(cpu)}
+// CHECK-NEXT: | | `-DeclRefExpr [[ADDR_2]] <line:6:1> 'int ({{.*}})' Function [[ADDR_3]] 'also_before[device={kind(cpu)}]' 'int ({{.*}})'
+// CHECK-NEXT: | `-OMPDeclareVariantAttr [[ADDR_16:0x[a-z0-9]*]] <<invalid sloc>> Implicit implementation={vendor(score(0): llvm)}
+// CHECK-NEXT: |   `-DeclRefExpr [[ADDR_17:0x[a-z0-9]*]] <line:17:1> 'int ({{.*}})' Function [[ADDR_18:0x[a-z0-9]*]] 'also_before[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// CHECK-NEXT: |-FunctionDecl [[ADDR_18]] <col:1, line:19:1> line:17:1 also_before[implementation={vendor(llvm)}] 'int ({{.*}})'
+// CHECK-NEXT: | `-CompoundStmt [[ADDR_19:0x[a-z0-9]*]] <col:23, line:19:1>
+// CHECK-NEXT: |   `-ReturnStmt [[ADDR_20:0x[a-z0-9]*]] <line:18:3, col:10>
+// CHECK-NEXT: |     `-IntegerLiteral [[ADDR_21:0x[a-z0-9]*]] <col:10> 'int' 1
+// CHECK-NEXT: |-FunctionDecl [[ADDR_22:0x[a-z0-9]*]] prev [[ADDR_7]] <line:22:1, line:24:1> line:22:5 used also_after 'int ({{.*}})'
+// CHECK-NEXT: | |-CompoundStmt [[ADDR_23:0x[a-z0-9]*]] <col:22, line:24:1>
+// CHECK-NEXT: | | `-ReturnStmt [[ADDR_24:0x[a-z0-9]*]] <line:23:3, col:10>
+// CHECK-NEXT: | |   `-IntegerLiteral [[ADDR_25:0x[a-z0-9]*]] <col:10> 'int' 2
+// CHECK-NEXT: | `-OMPDeclareVariantAttr [[ADDR_26:0x[a-z0-9]*]] <<invalid sloc>> Inherited Implicit implementation={vendor(score(100): llvm)}
+// CHECK-NEXT: |   `-DeclRefExpr [[ADDR_9]] <line:12:1> 'int ({{.*}})' Function [[ADDR_10]] 'also_after[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// CHECK-NEXT: `-FunctionDecl [[ADDR_27:0x[a-z0-9]*]] <line:26:1, line:29:1> line:26:5 test 'int ({{.*}})'
+// CHECK-NEXT:   `-CompoundStmt [[ADDR_28:0x[a-z0-9]*]] <col:12, line:29:1>
+// CHECK-NEXT:     `-ReturnStmt [[ADDR_29:0x[a-z0-9]*]] <line:28:3, col:37>
+// CHECK-NEXT:       `-BinaryOperator [[ADDR_30:0x[a-z0-9]*]] <col:10, col:37> 'int' '+'
+// CHECK-NEXT:         |-PseudoObjectExpr [[ADDR_31:0x[a-z0-9]*]] <col:10, col:21> 'int'
+// CHECK-NEXT:         | |-CallExpr [[ADDR_32:0x[a-z0-9]*]] <col:10, col:21> 'int'
+// CHECK-NEXT:         | | `-ImplicitCastExpr [[ADDR_33:0x[a-z0-9]*]] <col:10> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CHECK-NEXT:         | |   `-DeclRefExpr [[ADDR_34:0x[a-z0-9]*]] <col:10> 'int ({{.*}})' {{.*}}Function [[ADDR_22]] 'also_after' 'int ({{.*}})'
+// CHECK-NEXT:         | `-CallExpr [[ADDR_35:0x[a-z0-9]*]] <line:12:1, line:28:21> 'int'
+// CHECK-NEXT:         |   `-ImplicitCastExpr [[ADDR_36:0x[a-z0-9]*]] <line:12:1> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CHECK-NEXT:         |     `-DeclRefExpr [[ADDR_9]] <col:1> 'int ({{.*}})' Function [[ADDR_10]] 'also_after[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// CHECK-NEXT:         `-PseudoObjectExpr [[ADDR_37:0x[a-z0-9]*]] <line:28:25, col:37> 'int'
+// CHECK-NEXT:           |-CallExpr [[ADDR_38:0x[a-z0-9]*]] <col:25, col:37> 'int'
+// CHECK-NEXT:           | `-ImplicitCastExpr [[ADDR_39:0x[a-z0-9]*]] <col:25> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CHECK-NEXT:           |   `-DeclRefExpr [[ADDR_40:0x[a-z0-9]*]] <col:25> 'int ({{.*}})' {{.*}}Function [[ADDR_14]] 'also_before' 'int ({{.*}})'
+// CHECK-NEXT:           `-CallExpr [[ADDR_41:0x[a-z0-9]*]] <line:6:1, line:28:37> 'int'
+// CHECK-NEXT:             `-ImplicitCastExpr [[ADDR_42:0x[a-z0-9]*]] <line:6:1> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CHECK-NEXT:               `-DeclRefExpr [[ADDR_2]] <col:1> 'int ({{.*}})' Function [[ADDR_3]] 'also_before[device={kind(cpu)}]' 'int ({{.*}})'
diff --git a/clang/test/AST/ast-dump-openmp-begin-declare-variant_3.c b/clang/test/AST/ast-dump-openmp-begin-declare-variant_3.c
new file mode 100644
index 0000000000000..efedd2f9bb746
--- /dev/null
+++ b/clang/test/AST/ast-dump-openmp-begin-declare-variant_3.c
@@ -0,0 +1,82 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -fopenmp -verify -ast-dump %s       | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -fopenmp -verify -ast-dump %s -x c++| FileCheck %s
+// expected-no-diagnostics
+
+#pragma omp begin declare variant match(device={kind(cpu)})
+int also_before(void) {
+  return 1;
+}
+#pragma omp end declare variant
+
+#pragma omp begin declare variant match(implementation={vendor(score(0):llvm)})
+int also_after(void) {
+  return 0;
+}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(implementation={vendor(score(100):llvm)})
+int also_before(void) {
+  return 0;
+}
+#pragma omp end declare variant
+
+int also_after(void) {
+  return 2;
+}
+
+int test() {
+  // Should return 0.
+  return also_after() + also_before();
+}
+
+// Make sure:
+//  - we do see the ast nodes for the cpu kind
+//  - we do see the ast nodes for the llvm vendor
+//  - we pick the right callees
+
+// CHECK:      |-FunctionDecl [[ADDR_0:0x[a-z0-9]*]] <{{.*}}, col:21> col:5 implicit used also_before 'int ({{.*}})'
+// CHECK-NEXT: | `-OMPDeclareVariantAttr [[ADDR_1:0x[a-z0-9]*]] <<invalid sloc>> Implicit device={kind(cpu)}
+// CHECK-NEXT: |   `-DeclRefExpr [[ADDR_2:0x[a-z0-9]*]] <col:1> 'int ({{.*}})' Function [[ADDR_3:0x[a-z0-9]*]] 'also_before[device={kind(cpu)}]' 'int ({{.*}})'
+// CHECK-NEXT: |-FunctionDecl [[ADDR_3]] <col:1, line:8:1> line:6:1 also_before[device={kind(cpu)}] 'int ({{.*}})'
+// CHECK-NEXT: | `-CompoundStmt [[ADDR_4:0x[a-z0-9]*]] <col:23, line:8:1>
+// CHECK-NEXT: |   `-ReturnStmt [[ADDR_5:0x[a-z0-9]*]] <line:7:3, col:10>
+// CHECK-NEXT: |     `-IntegerLiteral [[ADDR_6:0x[a-z0-9]*]] <col:10> 'int' 1
+// CHECK-NEXT: |-FunctionDecl [[ADDR_7:0x[a-z0-9]*]] <line:12:1, col:20> col:5 implicit used also_after 'int ({{.*}})'
+// CHECK-NEXT: | `-OMPDeclareVariantAttr [[ADDR_8:0x[a-z0-9]*]] <<invalid sloc>> Implicit implementation={vendor(score(0): llvm)}
+// CHECK-NEXT: |   `-DeclRefExpr [[ADDR_9:0x[a-z0-9]*]] <col:1> 'int ({{.*}})' Function [[ADDR_10:0x[a-z0-9]*]] 'also_after[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// CHECK-NEXT: |-FunctionDecl [[ADDR_10]] <col:1, line:14:1> line:12:1 also_after[implementation={vendor(llvm)}] 'int ({{.*}})'
+// CHECK-NEXT: | `-CompoundStmt [[ADDR_11:0x[a-z0-9]*]] <col:22, line:14:1>
+// CHECK-NEXT: |   `-ReturnStmt [[ADDR_12:0x[a-z0-9]*]] <line:13:3, col:10>
+// CHECK-NEXT: |     `-IntegerLiteral [[ADDR_13:0x[a-z0-9]*]] <col:10> 'int' 0
+// CHECK-NEXT: |-FunctionDecl [[ADDR_14:0x[a-z0-9]*]] prev [[ADDR_0]] <line:17:1, col:21> col:5 implicit used also_before 'int ({{.*}})'
+// CHECK-NEXT: | |-OMPDeclareVariantAttr [[ADDR_15:0x[a-z0-9]*]] <<invalid sloc>> Inherited Implicit device={kind(cpu)}
+// CHECK-NEXT: | | `-DeclRefExpr [[ADDR_2]] <line:6:1> 'int ({{.*}})' Function [[ADDR_3]] 'also_before[device={kind(cpu)}]' 'int ({{.*}})'
+// CHECK-NEXT: | `-OMPDeclareVariantAttr [[ADDR_16:0x[a-z0-9]*]] <<invalid sloc>> Implicit implementation={vendor(score(100): llvm)}
+// CHECK-NEXT: |   `-DeclRefExpr [[ADDR_17:0x[a-z0-9]*]] <line:17:1> 'int ({{.*}})' Function [[ADDR_18:0x[a-z0-9]*]] 'also_before[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// CHECK-NEXT: |-FunctionDecl [[ADDR_18]] <col:1, line:19:1> line:17:1 also_before[implementation={vendor(llvm)}] 'int ({{.*}})'
+// CHECK-NEXT: | `-CompoundStmt [[ADDR_19:0x[a-z0-9]*]] <col:23, line:19:1>
+// CHECK-NEXT: |   `-ReturnStmt [[ADDR_20:0x[a-z0-9]*]] <line:18:3, col:10>
+// CHECK-NEXT: |     `-IntegerLiteral [[ADDR_21:0x[a-z0-9]*]] <col:10> 'int' 0
+// CHECK-NEXT: |-FunctionDecl [[ADDR_22:0x[a-z0-9]*]] prev [[ADDR_7]] <line:22:1, line:24:1> line:22:5 used also_after 'int ({{.*}})'
+// CHECK-NEXT: | |-CompoundStmt [[ADDR_23:0x[a-z0-9]*]] <col:22, line:24:1>
+// CHECK-NEXT: | | `-ReturnStmt [[ADDR_24:0x[a-z0-9]*]] <line:23:3, col:10>
+// CHECK-NEXT: | |   `-IntegerLiteral [[ADDR_25:0x[a-z0-9]*]] <col:10> 'int' 2
+// CHECK-NEXT: | `-OMPDeclareVariantAttr [[ADDR_26:0x[a-z0-9]*]] <<invalid sloc>> Inherited Implicit implementation={vendor(score(0): llvm)}
+// CHECK-NEXT: |   `-DeclRefExpr [[ADDR_9]] <line:12:1> 'int ({{.*}})' Function [[ADDR_10]] 'also_after[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// CHECK-NEXT: `-FunctionDecl [[ADDR_27:0x[a-z0-9]*]] <line:26:1, line:29:1> line:26:5 test 'int ({{.*}})'
+// CHECK-NEXT:   `-CompoundStmt [[ADDR_28:0x[a-z0-9]*]] <col:12, line:29:1>
+// CHECK-NEXT:     `-ReturnStmt [[ADDR_29:0x[a-z0-9]*]] <line:28:3, col:37>
+// CHECK-NEXT:       `-BinaryOperator [[ADDR_30:0x[a-z0-9]*]] <col:10, col:37> 'int' '+'
+// CHECK-NEXT:         |-PseudoObjectExpr [[ADDR_31:0x[a-z0-9]*]] <col:10, col:21> 'int'
+// CHECK-NEXT:         | |-CallExpr [[ADDR_32:0x[a-z0-9]*]] <col:10, col:21> 'int'
+// CHECK-NEXT:         | | `-ImplicitCastExpr [[ADDR_33:0x[a-z0-9]*]] <col:10> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CHECK-NEXT:         | |   `-DeclRefExpr [[ADDR_34:0x[a-z0-9]*]] <col:10> 'int ({{.*}})' {{.*}}Function [[ADDR_22]] 'also_after' 'int ({{.*}})'
+// CHECK-NEXT:         | `-CallExpr [[ADDR_35:0x[a-z0-9]*]] <line:12:1, line:28:21> 'int'
+// CHECK-NEXT:         |   `-ImplicitCastExpr [[ADDR_36:0x[a-z0-9]*]] <line:12:1> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CHECK-NEXT:         |     `-DeclRefExpr [[ADDR_9]] <col:1> 'int ({{.*}})' Function [[ADDR_10]] 'also_after[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// CHECK-NEXT:         `-PseudoObjectExpr [[ADDR_37:0x[a-z0-9]*]] <line:28:25, col:37> 'int'
+// CHECK-NEXT:           |-CallExpr [[ADDR_38:0x[a-z0-9]*]] <col:25, col:37> 'int'
+// CHECK-NEXT:           | `-ImplicitCastExpr [[ADDR_39:0x[a-z0-9]*]] <col:25> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CHECK-NEXT:           |   `-DeclRefExpr [[ADDR_40:0x[a-z0-9]*]] <col:25> 'int ({{.*}})' {{.*}}Function [[ADDR_14]] 'also_before' 'int ({{.*}})'
+// CHECK-NEXT:           `-CallExpr [[ADDR_41:0x[a-z0-9]*]] <line:17:1, line:28:37> 'int'
+// CHECK-NEXT:             `-ImplicitCastExpr [[ADDR_42:0x[a-z0-9]*]] <line:17:1> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CHECK-NEXT:               `-DeclRefExpr [[ADDR_17]] <col:1> 'int ({{.*}})' Function [[ADDR_18]] 'also_before[implementation={vendor(llvm)}]' 'int ({{.*}})'
diff --git a/clang/test/AST/ast-dump-openmp-begin-declare-variant_4.c b/clang/test/AST/ast-dump-openmp-begin-declare-variant_4.c
new file mode 100644
index 0000000000000..b35433478e49b
--- /dev/null
+++ b/clang/test/AST/ast-dump-openmp-begin-declare-variant_4.c
@@ -0,0 +1,48 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -fopenmp -verify -ast-dump %s       | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -fopenmp -verify -ast-dump %s -x c++| FileCheck %s
+// expected-no-diagnostics
+
+#pragma omp begin declare variant match(device={kind(cpu)})
+int also_before(void) {
+  return 0;
+}
+#pragma omp end declare variant
+
+int also_after(void) {
+  return 0;
+}
+
+int test() {
+  // Should return 0.
+  return also_after() + also_before();
+}
+
+// Make sure:
+//  - we do see the ast nodes for the cpu kind
+//  - we pick the right callees
+
+// CHECK:      |-FunctionDecl [[ADDR_0:0x[a-z0-9]*]] <{{.*}}, col:21> col:5 implicit used also_before 'int ({{.*}})'
+// CHECK-NEXT: | `-OMPDeclareVariantAttr [[ADDR_1:0x[a-z0-9]*]] <<invalid sloc>> Implicit device={kind(cpu)}
+// CHECK-NEXT: |   `-DeclRefExpr [[ADDR_2:0x[a-z0-9]*]] <col:1> 'int ({{.*}})' Function [[ADDR_3:0x[a-z0-9]*]] 'also_before[device={kind(cpu)}]' 'int ({{.*}})'
+// CHECK-NEXT: |-FunctionDecl [[ADDR_3]] <col:1, line:8:1> line:6:1 also_before[device={kind(cpu)}] 'int ({{.*}})'
+// CHECK-NEXT: | `-CompoundStmt [[ADDR_4:0x[a-z0-9]*]] <col:23, line:8:1>
+// CHECK-NEXT: |   `-ReturnStmt [[ADDR_5:0x[a-z0-9]*]] <line:7:3, col:10>
+// CHECK-NEXT: |     `-IntegerLiteral [[ADDR_6:0x[a-z0-9]*]] <col:10> 'int' 0
+// CHECK-NEXT: |-FunctionDecl [[ADDR_7:0x[a-z0-9]*]] <line:11:1, line:13:1> line:11:5 used also_after 'int ({{.*}})'
+// CHECK-NEXT: | `-CompoundStmt [[ADDR_8:0x[a-z0-9]*]] <col:22, line:13:1>
+// CHECK-NEXT: |   `-ReturnStmt [[ADDR_9:0x[a-z0-9]*]] <line:12:3, col:10>
+// CHECK-NEXT: |     `-IntegerLiteral [[ADDR_10:0x[a-z0-9]*]] <col:10> 'int' 0
+// CHECK-NEXT: `-FunctionDecl [[ADDR_11:0x[a-z0-9]*]] <line:15:1, line:18:1> line:15:5 test 'int ({{.*}})'
+// CHECK-NEXT:   `-CompoundStmt [[ADDR_12:0x[a-z0-9]*]] <col:12, line:18:1>
+// CHECK-NEXT:     `-ReturnStmt [[ADDR_13:0x[a-z0-9]*]] <line:17:3, col:37>
+// CHECK-NEXT:       `-BinaryOperator [[ADDR_14:0x[a-z0-9]*]] <col:10, col:37> 'int' '+'
+// CHECK-NEXT:         |-CallExpr [[ADDR_15:0x[a-z0-9]*]] <col:10, col:21> 'int'
+// CHECK-NEXT:         | `-ImplicitCastExpr [[ADDR_16:0x[a-z0-9]*]] <col:10> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CHECK-NEXT:         |   `-DeclRefExpr [[ADDR_17:0x[a-z0-9]*]] <col:10> 'int ({{.*}})' {{.*}}Function [[ADDR_7]] 'also_after' 'int ({{.*}})'
+// CHECK-NEXT:         `-PseudoObjectExpr [[ADDR_18:0x[a-z0-9]*]] <col:25, col:37> 'int'
+// CHECK-NEXT:           |-CallExpr [[ADDR_19:0x[a-z0-9]*]] <col:25, col:37> 'int'
+// CHECK-NEXT:           | `-ImplicitCastExpr [[ADDR_20:0x[a-z0-9]*]] <col:25> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CHECK-NEXT:           |   `-DeclRefExpr [[ADDR_21:0x[a-z0-9]*]] <col:25> 'int ({{.*}})' {{.*}}Function [[ADDR_0]] 'also_before' 'int ({{.*}})'
+// CHECK-NEXT:           `-CallExpr [[ADDR_22:0x[a-z0-9]*]] <line:6:1, line:17:37> 'int'
+// CHECK-NEXT:             `-ImplicitCastExpr [[ADDR_23:0x[a-z0-9]*]] <line:6:1> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CHECK-NEXT:               `-DeclRefExpr [[ADDR_2]] <col:1> 'int ({{.*}})' Function [[ADDR_3]] 'also_before[device={kind(cpu)}]' 'int ({{.*}})'
diff --git a/clang/test/AST/ast-dump-openmp-begin-declare-variant_5.c b/clang/test/AST/ast-dump-openmp-begin-declare-variant_5.c
new file mode 100644
index 0000000000000..b2a1cb0b3c86a
--- /dev/null
+++ b/clang/test/AST/ast-dump-openmp-begin-declare-variant_5.c
@@ -0,0 +1,158 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -fopenmp -verify -ast-dump %s       | FileCheck %s --check-prefix=C
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -fopenmp -verify -ast-dump %s -x c++| FileCheck %s --check-prefix=CXX
+// expected-no-diagnostics
+
+int also_before(void) {
+  return 1;
+}
+
+#pragma omp begin declare variant match(implementation={vendor(llvm)})
+int also_after(void) {
+  return 0;
+}
+int also_before(void) {
+  return 0;
+}
+#pragma omp end declare variant
+
+int also_after(void) {
+  return 2;
+}
+
+int main() {
+  // Should return 0.
+  return (also_after)() +
+         (also_before)() +
+         (&also_after)() +
+         (&also_before)();
+}
+
+// Make sure:
+//  - we see the specialization in the AST
+//  - we pick the right callees
+
+// C:      |-FunctionDecl [[ADDR_0:0x[a-z0-9]*]] <{{.*}}, line:7:1> line:5:5 used also_before 'int ({{.*}})'
+// C-NEXT: | `-CompoundStmt [[ADDR_1:0x[a-z0-9]*]] <col:23, line:7:1>
+// C-NEXT: |   `-ReturnStmt [[ADDR_2:0x[a-z0-9]*]] <line:6:3, col:10>
+// C-NEXT: |     `-IntegerLiteral [[ADDR_3:0x[a-z0-9]*]] <col:10> 'int' 1
+// C-NEXT: |-FunctionDecl [[ADDR_4:0x[a-z0-9]*]] <line:10:1, col:20> col:5 implicit used also_after 'int ({{.*}})'
+// C-NEXT: | `-OMPDeclareVariantAttr [[ADDR_5:0x[a-z0-9]*]] <<invalid sloc>> Implicit implementation={vendor(llvm)}
+// C-NEXT: |   `-DeclRefExpr [[ADDR_6:0x[a-z0-9]*]] <col:1> 'int ({{.*}})' Function [[ADDR_7:0x[a-z0-9]*]] 'also_after[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// C-NEXT: |-FunctionDecl [[ADDR_7]] <col:1, line:12:1> line:10:1 also_after[implementation={vendor(llvm)}] 'int ({{.*}})'
+// C-NEXT: | `-CompoundStmt [[ADDR_8:0x[a-z0-9]*]] <col:22, line:12:1>
+// C-NEXT: |   `-ReturnStmt [[ADDR_9:0x[a-z0-9]*]] <line:11:3, col:10>
+// C-NEXT: |     `-IntegerLiteral [[ADDR_10:0x[a-z0-9]*]] <col:10> 'int' 0
+// C-NEXT: |-FunctionDecl [[ADDR_11:0x[a-z0-9]*]] prev [[ADDR_0]] <line:13:1, col:21> col:5 implicit used also_before 'int ({{.*}})'
+// C-NEXT: | `-OMPDeclareVariantAttr [[ADDR_12:0x[a-z0-9]*]] <<invalid sloc>> Implicit implementation={vendor(llvm)}
+// C-NEXT: |   `-DeclRefExpr [[ADDR_13:0x[a-z0-9]*]] <col:1> 'int ({{.*}})' Function [[ADDR_14:0x[a-z0-9]*]] 'also_before[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// C-NEXT: |-FunctionDecl [[ADDR_14]] <col:1, line:15:1> line:13:1 also_before[implementation={vendor(llvm)}] 'int ({{.*}})'
+// C-NEXT: | `-CompoundStmt [[ADDR_15:0x[a-z0-9]*]] <col:23, line:15:1>
+// C-NEXT: |   `-ReturnStmt [[ADDR_16:0x[a-z0-9]*]] <line:14:3, col:10>
+// C-NEXT: |     `-IntegerLiteral [[ADDR_17:0x[a-z0-9]*]] <col:10> 'int' 0
+// C-NEXT: |-FunctionDecl [[ADDR_18:0x[a-z0-9]*]] prev [[ADDR_4]] <line:18:1, line:20:1> line:18:5 used also_after 'int ({{.*}})'
+// C-NEXT: | |-CompoundStmt [[ADDR_19:0x[a-z0-9]*]] <col:22, line:20:1>
+// C-NEXT: | | `-ReturnStmt [[ADDR_20:0x[a-z0-9]*]] <line:19:3, col:10>
+// C-NEXT: | |   `-IntegerLiteral [[ADDR_21:0x[a-z0-9]*]] <col:10> 'int' 2
+// C-NEXT: | `-OMPDeclareVariantAttr [[ADDR_22:0x[a-z0-9]*]] <<invalid sloc>> Inherited Implicit implementation={vendor(llvm)}
+// C-NEXT: |   `-DeclRefExpr [[ADDR_6]] <line:10:1> 'int ({{.*}})' Function [[ADDR_7]] 'also_after[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// C-NEXT: `-FunctionDecl [[ADDR_23:0x[a-z0-9]*]] <line:22:1, line:28:1> line:22:5 main 'int ({{.*}})'
+// C-NEXT:   `-CompoundStmt [[ADDR_24:0x[a-z0-9]*]] <col:12, line:28:1>
+// C-NEXT:     `-ReturnStmt [[ADDR_25:0x[a-z0-9]*]] <line:24:3, line:27:25>
+// C-NEXT:       `-BinaryOperator [[ADDR_26:0x[a-z0-9]*]] <line:24:10, line:27:25> 'int' '+'
+// C-NEXT:         |-BinaryOperator [[ADDR_27:0x[a-z0-9]*]] <line:24:10, line:26:24> 'int' '+'
+// C-NEXT:         | |-BinaryOperator [[ADDR_28:0x[a-z0-9]*]] <line:24:10, line:25:24> 'int' '+'
+// C-NEXT:         | | |-PseudoObjectExpr [[ADDR_29:0x[a-z0-9]*]] <line:24:10, col:23> 'int'
+// C-NEXT:         | | | |-CallExpr [[ADDR_30:0x[a-z0-9]*]] <col:10, col:23> 'int'
+// C-NEXT:         | | | | `-ImplicitCastExpr [[ADDR_31:0x[a-z0-9]*]] <col:10, col:21> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// C-NEXT:         | | | |   `-ParenExpr [[ADDR_32:0x[a-z0-9]*]] <col:10, col:21> 'int ({{.*}})'
+// C-NEXT:         | | | |     `-DeclRefExpr [[ADDR_33:0x[a-z0-9]*]] <col:11> 'int ({{.*}})' Function [[ADDR_18]] 'also_after' 'int ({{.*}})'
+// C-NEXT:         | | | `-CallExpr [[ADDR_34:0x[a-z0-9]*]] <line:10:1, line:24:23> 'int'
+// C-NEXT:         | | |   `-ImplicitCastExpr [[ADDR_35:0x[a-z0-9]*]] <line:10:1> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// C-NEXT:         | | |     `-DeclRefExpr [[ADDR_6]] <col:1> 'int ({{.*}})' Function [[ADDR_7]] 'also_after[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// C-NEXT:         | | `-PseudoObjectExpr [[ADDR_36:0x[a-z0-9]*]] <line:25:10, col:24> 'int'
+// C-NEXT:         | |   |-CallExpr [[ADDR_37:0x[a-z0-9]*]] <col:10, col:24> 'int'
+// C-NEXT:         | |   | `-ImplicitCastExpr [[ADDR_38:0x[a-z0-9]*]] <col:10, col:22> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// C-NEXT:         | |   |   `-ParenExpr [[ADDR_39:0x[a-z0-9]*]] <col:10, col:22> 'int ({{.*}})'
+// C-NEXT:         | |   |     `-DeclRefExpr [[ADDR_40:0x[a-z0-9]*]] <col:11> 'int ({{.*}})' Function [[ADDR_11]] 'also_before' 'int ({{.*}})'
+// C-NEXT:         | |   `-CallExpr [[ADDR_41:0x[a-z0-9]*]] <line:13:1, line:25:24> 'int'
+// C-NEXT:         | |     `-ImplicitCastExpr [[ADDR_42:0x[a-z0-9]*]] <line:13:1> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// C-NEXT:         | |       `-DeclRefExpr [[ADDR_13]] <col:1> 'int ({{.*}})' Function [[ADDR_14]] 'also_before[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// C-NEXT:         | `-PseudoObjectExpr [[ADDR_43:0x[a-z0-9]*]] <line:26:10, col:24> 'int'
+// C-NEXT:         |   |-CallExpr [[ADDR_44:0x[a-z0-9]*]] <col:10, col:24> 'int'
+// C-NEXT:         |   | `-ParenExpr [[ADDR_45:0x[a-z0-9]*]] <col:10, col:22> 'int (*)({{.*}})'
+// C-NEXT:         |   |   `-UnaryOperator [[ADDR_46:0x[a-z0-9]*]] <col:11, col:12> 'int (*)({{.*}})' prefix '&' cannot overflow
+// C-NEXT:         |   |     `-DeclRefExpr [[ADDR_47:0x[a-z0-9]*]] <col:12> 'int ({{.*}})' Function [[ADDR_18]] 'also_after' 'int ({{.*}})'
+// C-NEXT:         |   `-CallExpr [[ADDR_48:0x[a-z0-9]*]] <line:10:1, line:26:24> 'int'
+// C-NEXT:         |     `-ImplicitCastExpr [[ADDR_49:0x[a-z0-9]*]] <line:10:1> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// C-NEXT:         |       `-DeclRefExpr [[ADDR_6]] <col:1> 'int ({{.*}})' Function [[ADDR_7]] 'also_after[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// C-NEXT:         `-PseudoObjectExpr [[ADDR_50:0x[a-z0-9]*]] <line:27:10, col:25> 'int'
+// C-NEXT:           |-CallExpr [[ADDR_51:0x[a-z0-9]*]] <col:10, col:25> 'int'
+// C-NEXT:           | `-ParenExpr [[ADDR_52:0x[a-z0-9]*]] <col:10, col:23> 'int (*)({{.*}})'
+// C-NEXT:           |   `-UnaryOperator [[ADDR_53:0x[a-z0-9]*]] <col:11, col:12> 'int (*)({{.*}})' prefix '&' cannot overflow
+// C-NEXT:           |     `-DeclRefExpr [[ADDR_54:0x[a-z0-9]*]] <col:12> 'int ({{.*}})' Function [[ADDR_11]] 'also_before' 'int ({{.*}})'
+// C-NEXT:           `-CallExpr [[ADDR_55:0x[a-z0-9]*]] <line:13:1, line:27:25> 'int'
+// C-NEXT:             `-ImplicitCastExpr [[ADDR_56:0x[a-z0-9]*]] <line:13:1> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// C-NEXT:               `-DeclRefExpr [[ADDR_13]] <col:1> 'int ({{.*}})' Function [[ADDR_14]] 'also_before[implementation={vendor(llvm)}]' 'int ({{.*}})'
+
+// CXX:      |-FunctionDecl [[ADDR_0:0x[a-z0-9]*]] <{{.*}}, line:7:1> line:5:5 used also_before 'int ({{.*}})'
+// CXX-NEXT: | `-CompoundStmt [[ADDR_1:0x[a-z0-9]*]] <col:23, line:7:1>
+// CXX-NEXT: |   `-ReturnStmt [[ADDR_2:0x[a-z0-9]*]] <line:6:3, col:10>
+// CXX-NEXT: |     `-IntegerLiteral [[ADDR_3:0x[a-z0-9]*]] <col:10> 'int' 1
+// CXX-NEXT: |-FunctionDecl [[ADDR_4:0x[a-z0-9]*]] <line:10:1, col:20> col:5 implicit used also_after 'int ({{.*}})'
+// CXX-NEXT: | `-OMPDeclareVariantAttr [[ADDR_5:0x[a-z0-9]*]] <<invalid sloc>> Implicit implementation={vendor(llvm)}
+// CXX-NEXT: |   `-DeclRefExpr [[ADDR_6:0x[a-z0-9]*]] <col:1> 'int ({{.*}})' Function [[ADDR_7:0x[a-z0-9]*]] 'also_after[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// CXX-NEXT: |-FunctionDecl [[ADDR_7]] <col:1, line:12:1> line:10:1 also_after[implementation={vendor(llvm)}] 'int ({{.*}})'
+// CXX-NEXT: | `-CompoundStmt [[ADDR_8:0x[a-z0-9]*]] <col:22, line:12:1>
+// CXX-NEXT: |   `-ReturnStmt [[ADDR_9:0x[a-z0-9]*]] <line:11:3, col:10>
+// CXX-NEXT: |     `-IntegerLiteral [[ADDR_10:0x[a-z0-9]*]] <col:10> 'int' 0
+// CXX-NEXT: |-FunctionDecl [[ADDR_11:0x[a-z0-9]*]] prev [[ADDR_0]] <line:13:1, col:21> col:5 implicit used also_before 'int ({{.*}})'
+// CXX-NEXT: | `-OMPDeclareVariantAttr [[ADDR_12:0x[a-z0-9]*]] <<invalid sloc>> Implicit implementation={vendor(llvm)}
+// CXX-NEXT: |   `-DeclRefExpr [[ADDR_13:0x[a-z0-9]*]] <col:1> 'int ({{.*}})' Function [[ADDR_14:0x[a-z0-9]*]] 'also_before[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// CXX-NEXT: |-FunctionDecl [[ADDR_14]] <col:1, line:15:1> line:13:1 also_before[implementation={vendor(llvm)}] 'int ({{.*}})'
+// CXX-NEXT: | `-CompoundStmt [[ADDR_15:0x[a-z0-9]*]] <col:23, line:15:1>
+// CXX-NEXT: |   `-ReturnStmt [[ADDR_16:0x[a-z0-9]*]] <line:14:3, col:10>
+// CXX-NEXT: |     `-IntegerLiteral [[ADDR_17:0x[a-z0-9]*]] <col:10> 'int' 0
+// CXX-NEXT: |-FunctionDecl [[ADDR_18:0x[a-z0-9]*]] prev [[ADDR_4]] <line:18:1, line:20:1> line:18:5 used also_after 'int ({{.*}})'
+// CXX-NEXT: | |-CompoundStmt [[ADDR_19:0x[a-z0-9]*]] <col:22, line:20:1>
+// CXX-NEXT: | | `-ReturnStmt [[ADDR_20:0x[a-z0-9]*]] <line:19:3, col:10>
+// CXX-NEXT: | |   `-IntegerLiteral [[ADDR_21:0x[a-z0-9]*]] <col:10> 'int' 2
+// CXX-NEXT: | `-OMPDeclareVariantAttr [[ADDR_22:0x[a-z0-9]*]] <<invalid sloc>> Inherited Implicit implementation={vendor(llvm)}
+// CXX-NEXT: |   `-DeclRefExpr [[ADDR_6]] <line:10:1> 'int ({{.*}})' Function [[ADDR_7]] 'also_after[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// CXX-NEXT: `-FunctionDecl [[ADDR_23:0x[a-z0-9]*]] <line:22:1, line:28:1> line:22:5 main 'int ({{.*}})'
+// CXX-NEXT:   `-CompoundStmt [[ADDR_24:0x[a-z0-9]*]] <col:12, line:28:1>
+// CXX-NEXT:     `-ReturnStmt [[ADDR_25:0x[a-z0-9]*]] <line:24:3, line:27:25>
+// CXX-NEXT:       `-BinaryOperator [[ADDR_26:0x[a-z0-9]*]] <line:24:10, line:27:25> 'int' '+'
+// CXX-NEXT:         |-BinaryOperator [[ADDR_27:0x[a-z0-9]*]] <line:24:10, line:26:24> 'int' '+'
+// CXX-NEXT:         | |-BinaryOperator [[ADDR_28:0x[a-z0-9]*]] <line:24:10, line:25:24> 'int' '+'
+// CXX-NEXT:         | | |-PseudoObjectExpr [[ADDR_29:0x[a-z0-9]*]] <line:24:10, col:23> 'int'
+// CXX-NEXT:         | | | |-CallExpr [[ADDR_30:0x[a-z0-9]*]] <col:10, col:23> 'int'
+// CXX-NEXT:         | | | | `-ImplicitCastExpr [[ADDR_31:0x[a-z0-9]*]] <col:10, col:21> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CXX-NEXT:         | | | |   `-ParenExpr [[ADDR_32:0x[a-z0-9]*]] <col:10, col:21> 'int ({{.*}})' lvalue
+// CXX-NEXT:         | | | |     `-DeclRefExpr [[ADDR_33:0x[a-z0-9]*]] <col:11> 'int ({{.*}})' {{.*}}Function [[ADDR_18]] 'also_after' 'int ({{.*}})'
+// CXX-NEXT:         | | | `-CallExpr [[ADDR_34:0x[a-z0-9]*]] <line:10:1, line:24:23> 'int'
+// CXX-NEXT:         | | |   `-ImplicitCastExpr [[ADDR_35:0x[a-z0-9]*]] <line:10:1> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CXX-NEXT:         | | |     `-DeclRefExpr [[ADDR_6]] <col:1> 'int ({{.*}})' Function [[ADDR_7]] 'also_after[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// CXX-NEXT:         | | `-PseudoObjectExpr [[ADDR_36:0x[a-z0-9]*]] <line:25:10, col:24> 'int'
+// CXX-NEXT:         | |   |-CallExpr [[ADDR_37:0x[a-z0-9]*]] <col:10, col:24> 'int'
+// CXX-NEXT:         | |   | `-ImplicitCastExpr [[ADDR_38:0x[a-z0-9]*]] <col:10, col:22> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CXX-NEXT:         | |   |   `-ParenExpr [[ADDR_39:0x[a-z0-9]*]] <col:10, col:22> 'int ({{.*}})' lvalue
+// CXX-NEXT:         | |   |     `-DeclRefExpr [[ADDR_40:0x[a-z0-9]*]] <col:11> 'int ({{.*}})' {{.*}}Function [[ADDR_11]] 'also_before' 'int ({{.*}})'
+// CXX-NEXT:         | |   `-CallExpr [[ADDR_41:0x[a-z0-9]*]] <line:13:1, line:25:24> 'int'
+// CXX-NEXT:         | |     `-ImplicitCastExpr [[ADDR_42:0x[a-z0-9]*]] <line:13:1> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CXX-NEXT:         | |       `-DeclRefExpr [[ADDR_13]] <col:1> 'int ({{.*}})' Function [[ADDR_14]] 'also_before[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// CXX-NEXT:         | `-PseudoObjectExpr [[ADDR_43:0x[a-z0-9]*]] <line:26:10, col:24> 'int'
+// CXX-NEXT:         |   |-CallExpr [[ADDR_44:0x[a-z0-9]*]] <col:10, col:24> 'int'
+// CXX-NEXT:         |   | `-ParenExpr [[ADDR_45:0x[a-z0-9]*]] <col:10, col:22> 'int (*)({{.*}})'
+// CXX-NEXT:         |   |   `-UnaryOperator [[ADDR_46:0x[a-z0-9]*]] <col:11, col:12> 'int (*)({{.*}})' prefix '&' cannot overflow
+// CXX-NEXT:         |   |     `-DeclRefExpr [[ADDR_47:0x[a-z0-9]*]] <col:12> 'int ({{.*}})' {{.*}}Function [[ADDR_18]] 'also_after' 'int ({{.*}})'
+// CXX-NEXT:         |   `-CallExpr [[ADDR_48:0x[a-z0-9]*]] <line:10:1, line:26:24> 'int'
+// CXX-NEXT:         |     `-ImplicitCastExpr [[ADDR_49:0x[a-z0-9]*]] <line:10:1> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CXX-NEXT:         |       `-DeclRefExpr [[ADDR_6]] <col:1> 'int ({{.*}})' Function [[ADDR_7]] 'also_after[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// CXX-NEXT:         `-PseudoObjectExpr [[ADDR_50:0x[a-z0-9]*]] <line:27:10, col:25> 'int'
+// CXX-NEXT:           |-CallExpr [[ADDR_51:0x[a-z0-9]*]] <col:10, col:25> 'int'
+// CXX-NEXT:           | `-ParenExpr [[ADDR_52:0x[a-z0-9]*]] <col:10, col:23> 'int (*)({{.*}})'
+// CXX-NEXT:           |   `-UnaryOperator [[ADDR_53:0x[a-z0-9]*]] <col:11, col:12> 'int (*)({{.*}})' prefix '&' cannot overflow
+// CXX-NEXT:           |     `-DeclRefExpr [[ADDR_54:0x[a-z0-9]*]] <col:12> 'int ({{.*}})' {{.*}}Function [[ADDR_11]] 'also_before' 'int ({{.*}})'
+// CXX-NEXT:           `-CallExpr [[ADDR_55:0x[a-z0-9]*]] <line:13:1, line:27:25> 'int'
+// CXX-NEXT:             `-ImplicitCastExpr [[ADDR_56:0x[a-z0-9]*]] <line:13:1> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CXX-NEXT:               `-DeclRefExpr [[ADDR_13]] <col:1> 'int ({{.*}})' Function [[ADDR_14]] 'also_before[implementation={vendor(llvm)}]' 'int ({{.*}})'
diff --git a/clang/test/AST/ast-dump-openmp-begin-declare-variant_6.c b/clang/test/AST/ast-dump-openmp-begin-declare-variant_6.c
new file mode 100644
index 0000000000000..1e5241bc2c589
--- /dev/null
+++ b/clang/test/AST/ast-dump-openmp-begin-declare-variant_6.c
@@ -0,0 +1,64 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -fopenmp -verify -ast-dump %s       | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -fopenmp -verify -ast-dump %s -x c++| FileCheck %s
+// expected-no-diagnostics
+
+int also_before(void) {
+  return 0;
+}
+
+#pragma omp begin declare variant match(implementation={vendor(ibm)})
+int also_after(void) {
+  return 1;
+}
+int also_before(void) {
+  return 2;
+}
+#pragma omp end declare variant
+
+int also_after(void) {
+  return 0;
+}
+
+int main() {
+  // Should return 0.
+  return also_after() + also_before();
+}
+
+// Make sure:
+//  - we see the specialization in the AST
+//  - we do use the original pointers for the calls as the variants are not applicable (this is not the ibm compiler).
+
+// CHECK:      |-FunctionDecl [[ADDR_0:0x[a-z0-9]*]] <{{.*}}, line:7:1> line:5:5 used also_before 'int ({{.*}})'
+// CHECK-NEXT: | `-CompoundStmt [[ADDR_1:0x[a-z0-9]*]] <col:23, line:7:1>
+// CHECK-NEXT: |   `-ReturnStmt [[ADDR_2:0x[a-z0-9]*]] <line:6:3, col:10>
+// CHECK-NEXT: |     `-IntegerLiteral [[ADDR_3:0x[a-z0-9]*]] <col:10> 'int' 0
+// CHECK-NEXT: |-FunctionDecl [[ADDR_4:0x[a-z0-9]*]] <line:10:1, col:20> col:5 implicit used also_after 'int ({{.*}})'
+// CHECK-NEXT: | `-OMPDeclareVariantAttr [[ADDR_5:0x[a-z0-9]*]] <<invalid sloc>> Implicit implementation={vendor(ibm)}
+// CHECK-NEXT: |   `-DeclRefExpr [[ADDR_6:0x[a-z0-9]*]] <col:1> 'int ({{.*}})' Function [[ADDR_7:0x[a-z0-9]*]] 'also_after[implementation={vendor(ibm)}]' 'int ({{.*}})'
+// CHECK-NEXT: |-FunctionDecl [[ADDR_7]] <col:1, line:12:1> line:10:1 also_after[implementation={vendor(ibm)}] 'int ({{.*}})'
+// CHECK-NEXT: | `-CompoundStmt [[ADDR_8:0x[a-z0-9]*]] <col:22, line:12:1>
+// CHECK-NEXT: |   `-ReturnStmt [[ADDR_9:0x[a-z0-9]*]] <line:11:3, col:10>
+// CHECK-NEXT: |     `-IntegerLiteral [[ADDR_10:0x[a-z0-9]*]] <col:10> 'int' 1
+// CHECK-NEXT: |-FunctionDecl [[ADDR_11:0x[a-z0-9]*]] prev [[ADDR_0]] <line:13:1, col:21> col:5 implicit used also_before 'int ({{.*}})'
+// CHECK-NEXT: | `-OMPDeclareVariantAttr [[ADDR_12:0x[a-z0-9]*]] <<invalid sloc>> Implicit implementation={vendor(ibm)}
+// CHECK-NEXT: |   `-DeclRefExpr [[ADDR_13:0x[a-z0-9]*]] <col:1> 'int ({{.*}})' Function [[ADDR_14:0x[a-z0-9]*]] 'also_before[implementation={vendor(ibm)}]' 'int ({{.*}})'
+// CHECK-NEXT: |-FunctionDecl [[ADDR_14]] <col:1, line:15:1> line:13:1 also_before[implementation={vendor(ibm)}] 'int ({{.*}})'
+// CHECK-NEXT: | `-CompoundStmt [[ADDR_15:0x[a-z0-9]*]] <col:23, line:15:1>
+// CHECK-NEXT: |   `-ReturnStmt [[ADDR_16:0x[a-z0-9]*]] <line:14:3, col:10>
+// CHECK-NEXT: |     `-IntegerLiteral [[ADDR_17:0x[a-z0-9]*]] <col:10> 'int' 2
+// CHECK-NEXT: |-FunctionDecl [[ADDR_18:0x[a-z0-9]*]] prev [[ADDR_4]] <line:18:1, line:20:1> line:18:5 used also_after 'int ({{.*}})'
+// CHECK-NEXT: | |-CompoundStmt [[ADDR_19:0x[a-z0-9]*]] <col:22, line:20:1>
+// CHECK-NEXT: | | `-ReturnStmt [[ADDR_20:0x[a-z0-9]*]] <line:19:3, col:10>
+// CHECK-NEXT: | |   `-IntegerLiteral [[ADDR_21:0x[a-z0-9]*]] <col:10> 'int' 0
+// CHECK-NEXT: | `-OMPDeclareVariantAttr [[ADDR_22:0x[a-z0-9]*]] <<invalid sloc>> Inherited Implicit implementation={vendor(ibm)}
+// CHECK-NEXT: |   `-DeclRefExpr [[ADDR_6]] <line:10:1> 'int ({{.*}})' Function [[ADDR_7]] 'also_after[implementation={vendor(ibm)}]' 'int ({{.*}})'
+// CHECK-NEXT: `-FunctionDecl [[ADDR_23:0x[a-z0-9]*]] <line:22:1, line:25:1> line:22:5 main 'int ({{.*}})'
+// CHECK-NEXT:   `-CompoundStmt [[ADDR_24:0x[a-z0-9]*]] <col:12, line:25:1>
+// CHECK-NEXT:     `-ReturnStmt [[ADDR_25:0x[a-z0-9]*]] <line:24:3, col:37>
+// CHECK-NEXT:       `-BinaryOperator [[ADDR_26:0x[a-z0-9]*]] <col:10, col:37> 'int' '+'
+// CHECK-NEXT:         |-CallExpr [[ADDR_27:0x[a-z0-9]*]] <col:10, col:21> 'int'
+// CHECK-NEXT:         | `-ImplicitCastExpr [[ADDR_28:0x[a-z0-9]*]] <col:10> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CHECK-NEXT:         |   `-DeclRefExpr [[ADDR_29:0x[a-z0-9]*]] <col:10> 'int ({{.*}})' {{.*}}Function [[ADDR_18]] 'also_after' 'int ({{.*}})'
+// CHECK-NEXT:         `-CallExpr [[ADDR_30:0x[a-z0-9]*]] <col:25, col:37> 'int'
+// CHECK-NEXT:           `-ImplicitCastExpr [[ADDR_31:0x[a-z0-9]*]] <col:25> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CHECK-NEXT:             `-DeclRefExpr [[ADDR_32:0x[a-z0-9]*]] <col:25> 'int ({{.*}})' {{.*}}Function [[ADDR_11]] 'also_before' 'int ({{.*}})'
diff --git a/clang/test/AST/ast-dump-openmp-begin-declare-variant_7.c b/clang/test/AST/ast-dump-openmp-begin-declare-variant_7.c
new file mode 100644
index 0000000000000..cf5c1f3f62c13
--- /dev/null
+++ b/clang/test/AST/ast-dump-openmp-begin-declare-variant_7.c
@@ -0,0 +1,82 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -fopenmp -verify -ast-dump %s       | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -fopenmp -verify -ast-dump %s -x c++| FileCheck %s
+// expected-no-diagnostics
+
+int OK_1(void);
+
+#pragma omp begin declare variant match(implementation={vendor(intel)})
+int OK_1(void) {
+  return 1;
+}
+int OK_2(void) {
+  return 1;
+}
+int not_OK(void) {
+  return 1;
+}
+int OK_3(void) {
+  return 1;
+}
+#pragma omp end declare variant
+
+int OK_3(void);
+
+int test() {
+  // Should cause an error due to not_OK()
+  return OK_1() + not_OK() + OK_3();
+}
+
+// Make sure:
+//  - we see a single error for `not_OK`
+//  - we do not see errors for OK_{1,2,3}
+//  FIXME: We actually do not see there error here.
+//         This case is unlikely to happen in practise and hard to diagnose during SEMA.
+//         We will issue an error during code generation instead. This is similar to the
+//         diagnosis in other multi-versioning schemes.
+
+// CHECK:      |-FunctionDecl [[ADDR_0:0x[a-z0-9]*]] <{{.*}}, col:14> col:5 used OK_1 'int ({{.*}})'
+// CHECK-NEXT: |-FunctionDecl [[ADDR_1:0x[a-z0-9]*]] prev [[ADDR_0]] <line:8:1, col:14> col:5 implicit used OK_1 'int ({{.*}})'
+// CHECK-NEXT: | `-OMPDeclareVariantAttr [[ADDR_2:0x[a-z0-9]*]] <<invalid sloc>> Implicit implementation={vendor(intel)}
+// CHECK-NEXT: |   `-DeclRefExpr [[ADDR_3:0x[a-z0-9]*]] <col:1> 'int ({{.*}})' Function [[ADDR_4:0x[a-z0-9]*]] 'OK_1[implementation={vendor(intel)}]' 'int ({{.*}})'
+// CHECK-NEXT: |-FunctionDecl [[ADDR_4]] <col:1, line:10:1> line:8:1 OK_1[implementation={vendor(intel)}] 'int ({{.*}})'
+// CHECK-NEXT: | `-CompoundStmt [[ADDR_5:0x[a-z0-9]*]] <col:16, line:10:1>
+// CHECK-NEXT: |   `-ReturnStmt [[ADDR_6:0x[a-z0-9]*]] <line:9:3, col:10>
+// CHECK-NEXT: |     `-IntegerLiteral [[ADDR_7:0x[a-z0-9]*]] <col:10> 'int' 1
+// CHECK-NEXT: |-FunctionDecl [[ADDR_8:0x[a-z0-9]*]] <line:11:1, col:14> col:5 implicit OK_2 'int ({{.*}})'
+// CHECK-NEXT: | `-OMPDeclareVariantAttr [[ADDR_9:0x[a-z0-9]*]] <<invalid sloc>> Implicit implementation={vendor(intel)}
+// CHECK-NEXT: |   `-DeclRefExpr [[ADDR_10:0x[a-z0-9]*]] <col:1> 'int ({{.*}})' Function [[ADDR_11:0x[a-z0-9]*]] 'OK_2[implementation={vendor(intel)}]' 'int ({{.*}})'
+// CHECK-NEXT: |-FunctionDecl [[ADDR_11]] <col:1, line:13:1> line:11:1 OK_2[implementation={vendor(intel)}] 'int ({{.*}})'
+// CHECK-NEXT: | `-CompoundStmt [[ADDR_12:0x[a-z0-9]*]] <col:16, line:13:1>
+// CHECK-NEXT: |   `-ReturnStmt [[ADDR_13:0x[a-z0-9]*]] <line:12:3, col:10>
+// CHECK-NEXT: |     `-IntegerLiteral [[ADDR_14:0x[a-z0-9]*]] <col:10> 'int' 1
+// CHECK-NEXT: |-FunctionDecl [[ADDR_15:0x[a-z0-9]*]] <line:14:1, col:16> col:5 implicit used not_OK 'int ({{.*}})'
+// CHECK-NEXT: | `-OMPDeclareVariantAttr [[ADDR_16:0x[a-z0-9]*]] <<invalid sloc>> Implicit implementation={vendor(intel)}
+// CHECK-NEXT: |   `-DeclRefExpr [[ADDR_17:0x[a-z0-9]*]] <col:1> 'int ({{.*}})' Function [[ADDR_18:0x[a-z0-9]*]] 'not_OK[implementation={vendor(intel)}]' 'int ({{.*}})'
+// CHECK-NEXT: |-FunctionDecl [[ADDR_18]] <col:1, line:16:1> line:14:1 not_OK[implementation={vendor(intel)}] 'int ({{.*}})'
+// CHECK-NEXT: | `-CompoundStmt [[ADDR_19:0x[a-z0-9]*]] <col:18, line:16:1>
+// CHECK-NEXT: |   `-ReturnStmt [[ADDR_20:0x[a-z0-9]*]] <line:15:3, col:10>
+// CHECK-NEXT: |     `-IntegerLiteral [[ADDR_21:0x[a-z0-9]*]] <col:10> 'int' 1
+// CHECK-NEXT: |-FunctionDecl [[ADDR_22:0x[a-z0-9]*]] <line:17:1, col:14> col:5 implicit used OK_3 'int ({{.*}})'
+// CHECK-NEXT: | `-OMPDeclareVariantAttr [[ADDR_23:0x[a-z0-9]*]] <<invalid sloc>> Implicit implementation={vendor(intel)}
+// CHECK-NEXT: |   `-DeclRefExpr [[ADDR_24:0x[a-z0-9]*]] <col:1> 'int ({{.*}})' Function [[ADDR_25:0x[a-z0-9]*]] 'OK_3[implementation={vendor(intel)}]' 'int ({{.*}})'
+// CHECK-NEXT: |-FunctionDecl [[ADDR_25]] <col:1, line:19:1> line:17:1 OK_3[implementation={vendor(intel)}] 'int ({{.*}})'
+// CHECK-NEXT: | `-CompoundStmt [[ADDR_26:0x[a-z0-9]*]] <col:16, line:19:1>
+// CHECK-NEXT: |   `-ReturnStmt [[ADDR_27:0x[a-z0-9]*]] <line:18:3, col:10>
+// CHECK-NEXT: |     `-IntegerLiteral [[ADDR_28:0x[a-z0-9]*]] <col:10> 'int' 1
+// CHECK-NEXT: |-FunctionDecl [[ADDR_29:0x[a-z0-9]*]] prev [[ADDR_22]] <line:22:1, col:14> col:5 used OK_3 'int ({{.*}})'
+// CHECK-NEXT: | `-OMPDeclareVariantAttr [[ADDR_30:0x[a-z0-9]*]] <<invalid sloc>> Inherited Implicit implementation={vendor(intel)}
+// CHECK-NEXT: |   `-DeclRefExpr [[ADDR_24]] <line:17:1> 'int ({{.*}})' Function [[ADDR_25]] 'OK_3[implementation={vendor(intel)}]' 'int ({{.*}})'
+// CHECK-NEXT: `-FunctionDecl [[ADDR_31:0x[a-z0-9]*]] <line:24:1, line:27:1> line:24:5 test 'int ({{.*}})'
+// CHECK-NEXT:   `-CompoundStmt [[ADDR_32:0x[a-z0-9]*]] <col:12, line:27:1>
+// CHECK-NEXT:     `-ReturnStmt [[ADDR_33:0x[a-z0-9]*]] <line:26:3, col:35>
+// CHECK-NEXT:       `-BinaryOperator [[ADDR_34:0x[a-z0-9]*]] <col:10, col:35> 'int' '+'
+// CHECK-NEXT:         |-BinaryOperator [[ADDR_35:0x[a-z0-9]*]] <col:10, col:26> 'int' '+'
+// CHECK-NEXT:         | |-CallExpr [[ADDR_36:0x[a-z0-9]*]] <col:10, col:15> 'int'
+// CHECK-NEXT:         | | `-ImplicitCastExpr [[ADDR_37:0x[a-z0-9]*]] <col:10> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CHECK-NEXT:         | |   `-DeclRefExpr [[ADDR_38:0x[a-z0-9]*]] <col:10> 'int ({{.*}})' {{.*}}Function [[ADDR_1]] 'OK_1' 'int ({{.*}})'
+// CHECK-NEXT:         | `-CallExpr [[ADDR_39:0x[a-z0-9]*]] <col:19, col:26> 'int'
+// CHECK-NEXT:         |   `-ImplicitCastExpr [[ADDR_40:0x[a-z0-9]*]] <col:19> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CHECK-NEXT:         |     `-DeclRefExpr [[ADDR_41:0x[a-z0-9]*]] <col:19> 'int ({{.*}})' {{.*}}Function [[ADDR_15]] 'not_OK' 'int ({{.*}})'
+// CHECK-NEXT:         `-CallExpr [[ADDR_42:0x[a-z0-9]*]] <col:30, col:35> 'int'
+// CHECK-NEXT:           `-ImplicitCastExpr [[ADDR_43:0x[a-z0-9]*]] <col:30> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CHECK-NEXT:             `-DeclRefExpr [[ADDR_44:0x[a-z0-9]*]] <col:30> 'int ({{.*}})' {{.*}}Function [[ADDR_29]] 'OK_3' 'int ({{.*}})'
diff --git a/clang/test/AST/ast-dump-openmp-begin-declare-variant_8.c b/clang/test/AST/ast-dump-openmp-begin-declare-variant_8.c
new file mode 100644
index 0000000000000..efedd2f9bb746
--- /dev/null
+++ b/clang/test/AST/ast-dump-openmp-begin-declare-variant_8.c
@@ -0,0 +1,82 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -fopenmp -verify -ast-dump %s       | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -fopenmp -verify -ast-dump %s -x c++| FileCheck %s
+// expected-no-diagnostics
+
+#pragma omp begin declare variant match(device={kind(cpu)})
+int also_before(void) {
+  return 1;
+}
+#pragma omp end declare variant
+
+#pragma omp begin declare variant match(implementation={vendor(score(0):llvm)})
+int also_after(void) {
+  return 0;
+}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(implementation={vendor(score(100):llvm)})
+int also_before(void) {
+  return 0;
+}
+#pragma omp end declare variant
+
+int also_after(void) {
+  return 2;
+}
+
+int test() {
+  // Should return 0.
+  return also_after() + also_before();
+}
+
+// Make sure:
+//  - we do see the ast nodes for the cpu kind
+//  - we do see the ast nodes for the llvm vendor
+//  - we pick the right callees
+
+// CHECK:      |-FunctionDecl [[ADDR_0:0x[a-z0-9]*]] <{{.*}}, col:21> col:5 implicit used also_before 'int ({{.*}})'
+// CHECK-NEXT: | `-OMPDeclareVariantAttr [[ADDR_1:0x[a-z0-9]*]] <<invalid sloc>> Implicit device={kind(cpu)}
+// CHECK-NEXT: |   `-DeclRefExpr [[ADDR_2:0x[a-z0-9]*]] <col:1> 'int ({{.*}})' Function [[ADDR_3:0x[a-z0-9]*]] 'also_before[device={kind(cpu)}]' 'int ({{.*}})'
+// CHECK-NEXT: |-FunctionDecl [[ADDR_3]] <col:1, line:8:1> line:6:1 also_before[device={kind(cpu)}] 'int ({{.*}})'
+// CHECK-NEXT: | `-CompoundStmt [[ADDR_4:0x[a-z0-9]*]] <col:23, line:8:1>
+// CHECK-NEXT: |   `-ReturnStmt [[ADDR_5:0x[a-z0-9]*]] <line:7:3, col:10>
+// CHECK-NEXT: |     `-IntegerLiteral [[ADDR_6:0x[a-z0-9]*]] <col:10> 'int' 1
+// CHECK-NEXT: |-FunctionDecl [[ADDR_7:0x[a-z0-9]*]] <line:12:1, col:20> col:5 implicit used also_after 'int ({{.*}})'
+// CHECK-NEXT: | `-OMPDeclareVariantAttr [[ADDR_8:0x[a-z0-9]*]] <<invalid sloc>> Implicit implementation={vendor(score(0): llvm)}
+// CHECK-NEXT: |   `-DeclRefExpr [[ADDR_9:0x[a-z0-9]*]] <col:1> 'int ({{.*}})' Function [[ADDR_10:0x[a-z0-9]*]] 'also_after[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// CHECK-NEXT: |-FunctionDecl [[ADDR_10]] <col:1, line:14:1> line:12:1 also_after[implementation={vendor(llvm)}] 'int ({{.*}})'
+// CHECK-NEXT: | `-CompoundStmt [[ADDR_11:0x[a-z0-9]*]] <col:22, line:14:1>
+// CHECK-NEXT: |   `-ReturnStmt [[ADDR_12:0x[a-z0-9]*]] <line:13:3, col:10>
+// CHECK-NEXT: |     `-IntegerLiteral [[ADDR_13:0x[a-z0-9]*]] <col:10> 'int' 0
+// CHECK-NEXT: |-FunctionDecl [[ADDR_14:0x[a-z0-9]*]] prev [[ADDR_0]] <line:17:1, col:21> col:5 implicit used also_before 'int ({{.*}})'
+// CHECK-NEXT: | |-OMPDeclareVariantAttr [[ADDR_15:0x[a-z0-9]*]] <<invalid sloc>> Inherited Implicit device={kind(cpu)}
+// CHECK-NEXT: | | `-DeclRefExpr [[ADDR_2]] <line:6:1> 'int ({{.*}})' Function [[ADDR_3]] 'also_before[device={kind(cpu)}]' 'int ({{.*}})'
+// CHECK-NEXT: | `-OMPDeclareVariantAttr [[ADDR_16:0x[a-z0-9]*]] <<invalid sloc>> Implicit implementation={vendor(score(100): llvm)}
+// CHECK-NEXT: |   `-DeclRefExpr [[ADDR_17:0x[a-z0-9]*]] <line:17:1> 'int ({{.*}})' Function [[ADDR_18:0x[a-z0-9]*]] 'also_before[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// CHECK-NEXT: |-FunctionDecl [[ADDR_18]] <col:1, line:19:1> line:17:1 also_before[implementation={vendor(llvm)}] 'int ({{.*}})'
+// CHECK-NEXT: | `-CompoundStmt [[ADDR_19:0x[a-z0-9]*]] <col:23, line:19:1>
+// CHECK-NEXT: |   `-ReturnStmt [[ADDR_20:0x[a-z0-9]*]] <line:18:3, col:10>
+// CHECK-NEXT: |     `-IntegerLiteral [[ADDR_21:0x[a-z0-9]*]] <col:10> 'int' 0
+// CHECK-NEXT: |-FunctionDecl [[ADDR_22:0x[a-z0-9]*]] prev [[ADDR_7]] <line:22:1, line:24:1> line:22:5 used also_after 'int ({{.*}})'
+// CHECK-NEXT: | |-CompoundStmt [[ADDR_23:0x[a-z0-9]*]] <col:22, line:24:1>
+// CHECK-NEXT: | | `-ReturnStmt [[ADDR_24:0x[a-z0-9]*]] <line:23:3, col:10>
+// CHECK-NEXT: | |   `-IntegerLiteral [[ADDR_25:0x[a-z0-9]*]] <col:10> 'int' 2
+// CHECK-NEXT: | `-OMPDeclareVariantAttr [[ADDR_26:0x[a-z0-9]*]] <<invalid sloc>> Inherited Implicit implementation={vendor(score(0): llvm)}
+// CHECK-NEXT: |   `-DeclRefExpr [[ADDR_9]] <line:12:1> 'int ({{.*}})' Function [[ADDR_10]] 'also_after[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// CHECK-NEXT: `-FunctionDecl [[ADDR_27:0x[a-z0-9]*]] <line:26:1, line:29:1> line:26:5 test 'int ({{.*}})'
+// CHECK-NEXT:   `-CompoundStmt [[ADDR_28:0x[a-z0-9]*]] <col:12, line:29:1>
+// CHECK-NEXT:     `-ReturnStmt [[ADDR_29:0x[a-z0-9]*]] <line:28:3, col:37>
+// CHECK-NEXT:       `-BinaryOperator [[ADDR_30:0x[a-z0-9]*]] <col:10, col:37> 'int' '+'
+// CHECK-NEXT:         |-PseudoObjectExpr [[ADDR_31:0x[a-z0-9]*]] <col:10, col:21> 'int'
+// CHECK-NEXT:         | |-CallExpr [[ADDR_32:0x[a-z0-9]*]] <col:10, col:21> 'int'
+// CHECK-NEXT:         | | `-ImplicitCastExpr [[ADDR_33:0x[a-z0-9]*]] <col:10> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CHECK-NEXT:         | |   `-DeclRefExpr [[ADDR_34:0x[a-z0-9]*]] <col:10> 'int ({{.*}})' {{.*}}Function [[ADDR_22]] 'also_after' 'int ({{.*}})'
+// CHECK-NEXT:         | `-CallExpr [[ADDR_35:0x[a-z0-9]*]] <line:12:1, line:28:21> 'int'
+// CHECK-NEXT:         |   `-ImplicitCastExpr [[ADDR_36:0x[a-z0-9]*]] <line:12:1> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CHECK-NEXT:         |     `-DeclRefExpr [[ADDR_9]] <col:1> 'int ({{.*}})' Function [[ADDR_10]] 'also_after[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// CHECK-NEXT:         `-PseudoObjectExpr [[ADDR_37:0x[a-z0-9]*]] <line:28:25, col:37> 'int'
+// CHECK-NEXT:           |-CallExpr [[ADDR_38:0x[a-z0-9]*]] <col:25, col:37> 'int'
+// CHECK-NEXT:           | `-ImplicitCastExpr [[ADDR_39:0x[a-z0-9]*]] <col:25> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CHECK-NEXT:           |   `-DeclRefExpr [[ADDR_40:0x[a-z0-9]*]] <col:25> 'int ({{.*}})' {{.*}}Function [[ADDR_14]] 'also_before' 'int ({{.*}})'
+// CHECK-NEXT:           `-CallExpr [[ADDR_41:0x[a-z0-9]*]] <line:17:1, line:28:37> 'int'
+// CHECK-NEXT:             `-ImplicitCastExpr [[ADDR_42:0x[a-z0-9]*]] <line:17:1> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CHECK-NEXT:               `-DeclRefExpr [[ADDR_17]] <col:1> 'int ({{.*}})' Function [[ADDR_18]] 'also_before[implementation={vendor(llvm)}]' 'int ({{.*}})'
diff --git a/clang/test/AST/ast-dump-openmp-begin-declare-variant_9.c b/clang/test/AST/ast-dump-openmp-begin-declare-variant_9.c
new file mode 100644
index 0000000000000..af61859ae3068
--- /dev/null
+++ b/clang/test/AST/ast-dump-openmp-begin-declare-variant_9.c
@@ -0,0 +1,184 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -fopenmp -verify -ast-dump %s       | FileCheck %s --check-prefix=C
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -fopenmp -verify -ast-dump %s -x c++| FileCheck %s --check-prefix=CXX
+// expected-no-diagnostics
+
+int also_before(void) {
+  return 0;
+}
+
+#pragma omp begin declare variant match(implementation={vendor(llvm)})
+int also_after(void) {
+  return 1;
+}
+int also_before(void) {
+  return 2;
+}
+#pragma omp end declare variant
+
+int also_after(void) {
+  return 0;
+}
+
+void foo();
+typedef int(*fd)(void);
+int main() {
+  // Should return 0.
+  fd fns[2];
+  fns[0] = &also_before;
+  fns[1] = also_after;
+  return (foo(), also_after)() +
+         (fns[0])() +
+         (1[fns])();
+}
+
+// Make sure:
+//  - we see the specialization in the AST
+//  - we pick the right callees
+
+// C:      |-FunctionDecl [[ADDR_0:0x[a-z0-9]*]] <{{.*}}, line:7:1> line:5:5 used also_before 'int ({{.*}})'
+// C-NEXT: | `-CompoundStmt [[ADDR_1:0x[a-z0-9]*]] <col:23, line:7:1>
+// C-NEXT: |   `-ReturnStmt [[ADDR_2:0x[a-z0-9]*]] <line:6:3, col:10>
+// C-NEXT: |     `-IntegerLiteral [[ADDR_3:0x[a-z0-9]*]] <col:10> 'int' 0
+// C-NEXT: |-FunctionDecl [[ADDR_4:0x[a-z0-9]*]] <line:10:1, col:20> col:5 implicit used also_after 'int ({{.*}})'
+// C-NEXT: | `-OMPDeclareVariantAttr [[ADDR_5:0x[a-z0-9]*]] <<invalid sloc>> Implicit implementation={vendor(llvm)}
+// C-NEXT: |   `-DeclRefExpr [[ADDR_6:0x[a-z0-9]*]] <col:1> 'int ({{.*}})' Function [[ADDR_7:0x[a-z0-9]*]] 'also_after[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// C-NEXT: |-FunctionDecl [[ADDR_7]] <col:1, line:12:1> line:10:1 also_after[implementation={vendor(llvm)}] 'int ({{.*}})'
+// C-NEXT: | `-CompoundStmt [[ADDR_8:0x[a-z0-9]*]] <col:22, line:12:1>
+// C-NEXT: |   `-ReturnStmt [[ADDR_9:0x[a-z0-9]*]] <line:11:3, col:10>
+// C-NEXT: |     `-IntegerLiteral [[ADDR_10:0x[a-z0-9]*]] <col:10> 'int' 1
+// C-NEXT: |-FunctionDecl [[ADDR_11:0x[a-z0-9]*]] prev [[ADDR_0]] <line:13:1, col:21> col:5 implicit used also_before 'int ({{.*}})'
+// C-NEXT: | `-OMPDeclareVariantAttr [[ADDR_12:0x[a-z0-9]*]] <<invalid sloc>> Implicit implementation={vendor(llvm)}
+// C-NEXT: |   `-DeclRefExpr [[ADDR_13:0x[a-z0-9]*]] <col:1> 'int ({{.*}})' Function [[ADDR_14:0x[a-z0-9]*]] 'also_before[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// C-NEXT: |-FunctionDecl [[ADDR_14]] <col:1, line:15:1> line:13:1 also_before[implementation={vendor(llvm)}] 'int ({{.*}})'
+// C-NEXT: | `-CompoundStmt [[ADDR_15:0x[a-z0-9]*]] <col:23, line:15:1>
+// C-NEXT: |   `-ReturnStmt [[ADDR_16:0x[a-z0-9]*]] <line:14:3, col:10>
+// C-NEXT: |     `-IntegerLiteral [[ADDR_17:0x[a-z0-9]*]] <col:10> 'int' 2
+// C-NEXT: |-FunctionDecl [[ADDR_18:0x[a-z0-9]*]] prev [[ADDR_4]] <line:18:1, line:20:1> line:18:5 used also_after 'int ({{.*}})'
+// C-NEXT: | |-CompoundStmt [[ADDR_19:0x[a-z0-9]*]] <col:22, line:20:1>
+// C-NEXT: | | `-ReturnStmt [[ADDR_20:0x[a-z0-9]*]] <line:19:3, col:10>
+// C-NEXT: | |   `-IntegerLiteral [[ADDR_21:0x[a-z0-9]*]] <col:10> 'int' 0
+// C-NEXT: | `-OMPDeclareVariantAttr [[ADDR_22:0x[a-z0-9]*]] <<invalid sloc>> Inherited Implicit implementation={vendor(llvm)}
+// C-NEXT: |   `-DeclRefExpr [[ADDR_6]] <line:10:1> 'int ({{.*}})' Function [[ADDR_7]] 'also_after[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// C-NEXT: |-FunctionDecl [[ADDR_23:0x[a-z0-9]*]] <line:22:1, col:10> col:6 used foo 'void ({{.*}})'
+// C-NEXT: |-TypedefDecl [[ADDR_24:0x[a-z0-9]*]] <line:23:1, col:22> col:14 referenced fd 'int (*)({{.*}})'
+// C-NEXT: | `-PointerType [[ADDR_25:0x[a-z0-9]*]] 'int (*)({{.*}})'
+// C-NEXT: |   `-ParenType [[ADDR_26:0x[a-z0-9]*]] 'int ({{.*}})' sugar
+// C-NEXT: |     `-FunctionProtoType [[ADDR_27:0x[a-z0-9]*]] 'int ({{.*}})' cdecl
+// C-NEXT: |       `-BuiltinType [[ADDR_28:0x[a-z0-9]*]] 'int'
+// C-NEXT: `-FunctionDecl [[ADDR_29:0x[a-z0-9]*]] <line:24:1, line:32:1> line:24:5 main 'int ({{.*}})'
+// C-NEXT:   `-CompoundStmt [[ADDR_30:0x[a-z0-9]*]] <col:12, line:32:1>
+// C-NEXT:     |-DeclStmt [[ADDR_31:0x[a-z0-9]*]] <line:26:3, col:12>
+// C-NEXT:     | `-VarDecl [[ADDR_32:0x[a-z0-9]*]] <col:3, col:11> col:6 used fns 'fd [2]'
+// C-NEXT:     |-BinaryOperator [[ADDR_33:0x[a-z0-9]*]] <line:27:3, col:13> 'fd':'int (*)({{.*}})' '='
+// C-NEXT:     | |-ArraySubscriptExpr [[ADDR_34:0x[a-z0-9]*]] <col:3, col:8> 'fd':'int (*)({{.*}})' lvalue
+// C-NEXT:     | | |-ImplicitCastExpr [[ADDR_35:0x[a-z0-9]*]] <col:3> 'fd *' <ArrayToPointerDecay>
+// C-NEXT:     | | | `-DeclRefExpr [[ADDR_36:0x[a-z0-9]*]] <col:3> 'fd [2]' {{.*}}Var [[ADDR_32]] 'fns' 'fd [2]'
+// C-NEXT:     | | `-IntegerLiteral [[ADDR_37:0x[a-z0-9]*]] <col:7> 'int' 0
+// C-NEXT:     | `-UnaryOperator [[ADDR_38:0x[a-z0-9]*]] <col:12, col:13> 'int (*)({{.*}})' prefix '&' cannot overflow
+// C-NEXT:     |   `-DeclRefExpr [[ADDR_39:0x[a-z0-9]*]] <col:13> 'int ({{.*}})' Function [[ADDR_11]] 'also_before' 'int ({{.*}})'
+// C-NEXT:     |-BinaryOperator [[ADDR_40:0x[a-z0-9]*]] <line:28:3, col:12> 'fd':'int (*)({{.*}})' '='
+// C-NEXT:     | |-ArraySubscriptExpr [[ADDR_41:0x[a-z0-9]*]] <col:3, col:8> 'fd':'int (*)({{.*}})' lvalue
+// C-NEXT:     | | |-ImplicitCastExpr [[ADDR_42:0x[a-z0-9]*]] <col:3> 'fd *' <ArrayToPointerDecay>
+// C-NEXT:     | | | `-DeclRefExpr [[ADDR_43:0x[a-z0-9]*]] <col:3> 'fd [2]' {{.*}}Var [[ADDR_32]] 'fns' 'fd [2]'
+// C-NEXT:     | | `-IntegerLiteral [[ADDR_44:0x[a-z0-9]*]] <col:7> 'int' 1
+// C-NEXT:     | `-ImplicitCastExpr [[ADDR_45:0x[a-z0-9]*]] <col:12> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// C-NEXT:     |   `-DeclRefExpr [[ADDR_46:0x[a-z0-9]*]] <col:12> 'int ({{.*}})' Function [[ADDR_18]] 'also_after' 'int ({{.*}})'
+// C-NEXT:     `-ReturnStmt [[ADDR_47:0x[a-z0-9]*]] <line:29:3, line:31:19>
+// C-NEXT:       `-BinaryOperator [[ADDR_48:0x[a-z0-9]*]] <line:29:10, line:31:19> 'int' '+'
+// C-NEXT:         |-BinaryOperator [[ADDR_49:0x[a-z0-9]*]] <line:29:10, line:30:19> 'int' '+'
+// C-NEXT:         | |-CallExpr [[ADDR_50:0x[a-z0-9]*]] <line:29:10, col:30> 'int'
+// C-NEXT:         | | `-ParenExpr [[ADDR_51:0x[a-z0-9]*]] <col:10, col:28> 'int (*)({{.*}})'
+// C-NEXT:         | |   `-BinaryOperator [[ADDR_52:0x[a-z0-9]*]] <col:11, col:18> 'int (*)({{.*}})' ','
+// C-NEXT:         | |     |-CallExpr [[ADDR_53:0x[a-z0-9]*]] <col:11, col:15> 'void'
+// C-NEXT:         | |     | `-ImplicitCastExpr [[ADDR_54:0x[a-z0-9]*]] <col:11> 'void (*)({{.*}})' <FunctionToPointerDecay>
+// C-NEXT:         | |     |   `-DeclRefExpr [[ADDR_55:0x[a-z0-9]*]] <col:11> 'void ({{.*}})' Function [[ADDR_23]] 'foo' 'void ({{.*}})'
+// C-NEXT:         | |     `-ImplicitCastExpr [[ADDR_56:0x[a-z0-9]*]] <col:18> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// C-NEXT:         | |       `-DeclRefExpr [[ADDR_57:0x[a-z0-9]*]] <col:18> 'int ({{.*}})' Function [[ADDR_18]] 'also_after' 'int ({{.*}})'
+// C-NEXT:         | `-CallExpr [[ADDR_58:0x[a-z0-9]*]] <line:30:10, col:19> 'int'
+// C-NEXT:         |   `-ImplicitCastExpr [[ADDR_59:0x[a-z0-9]*]] <col:10, col:17> 'fd':'int (*)({{.*}})' <LValueToRValue>
+// C-NEXT:         |     `-ParenExpr [[ADDR_60:0x[a-z0-9]*]] <col:10, col:17> 'fd':'int (*)({{.*}})' lvalue
+// C-NEXT:         |       `-ArraySubscriptExpr [[ADDR_61:0x[a-z0-9]*]] <col:11, col:16> 'fd':'int (*)({{.*}})' lvalue
+// C-NEXT:         |         |-ImplicitCastExpr [[ADDR_62:0x[a-z0-9]*]] <col:11> 'fd *' <ArrayToPointerDecay>
+// C-NEXT:         |         | `-DeclRefExpr [[ADDR_63:0x[a-z0-9]*]] <col:11> 'fd [2]' {{.*}}Var [[ADDR_32]] 'fns' 'fd [2]'
+// C-NEXT:         |         `-IntegerLiteral [[ADDR_64:0x[a-z0-9]*]] <col:15> 'int' 0
+// C-NEXT:         `-CallExpr [[ADDR_65:0x[a-z0-9]*]] <line:31:10, col:19> 'int'
+// C-NEXT:           `-ImplicitCastExpr [[ADDR_66:0x[a-z0-9]*]] <col:10, col:17> 'fd':'int (*)({{.*}})' <LValueToRValue>
+// C-NEXT:             `-ParenExpr [[ADDR_67:0x[a-z0-9]*]] <col:10, col:17> 'fd':'int (*)({{.*}})' lvalue
+// C-NEXT:               `-ArraySubscriptExpr [[ADDR_68:0x[a-z0-9]*]] <col:11, col:16> 'fd':'int (*)({{.*}})' lvalue
+// C-NEXT:                 |-IntegerLiteral [[ADDR_69:0x[a-z0-9]*]] <col:11> 'int' 1
+// C-NEXT:                 `-ImplicitCastExpr [[ADDR_70:0x[a-z0-9]*]] <col:13> 'fd *' <ArrayToPointerDecay>
+// C-NEXT:                   `-DeclRefExpr [[ADDR_71:0x[a-z0-9]*]] <col:13> 'fd [2]' {{.*}}Var [[ADDR_32]] 'fns' 'fd [2]'
+
+// CXX:      |-FunctionDecl [[ADDR_0:0x[a-z0-9]*]] <{{.*}}, line:7:1> line:5:5 used also_before 'int ({{.*}})'
+// CXX-NEXT: | `-CompoundStmt [[ADDR_1:0x[a-z0-9]*]] <col:23, line:7:1>
+// CXX-NEXT: |   `-ReturnStmt [[ADDR_2:0x[a-z0-9]*]] <line:6:3, col:10>
+// CXX-NEXT: |     `-IntegerLiteral [[ADDR_3:0x[a-z0-9]*]] <col:10> 'int' 0
+// CXX-NEXT: |-FunctionDecl [[ADDR_4:0x[a-z0-9]*]] <line:10:1, col:20> col:5 implicit used also_after 'int ({{.*}})'
+// CXX-NEXT: | `-OMPDeclareVariantAttr [[ADDR_5:0x[a-z0-9]*]] <<invalid sloc>> Implicit implementation={vendor(llvm)}
+// CXX-NEXT: |   `-DeclRefExpr [[ADDR_6:0x[a-z0-9]*]] <col:1> 'int ({{.*}})' Function [[ADDR_7:0x[a-z0-9]*]] 'also_after[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// CXX-NEXT: |-FunctionDecl [[ADDR_7]] <col:1, line:12:1> line:10:1 also_after[implementation={vendor(llvm)}] 'int ({{.*}})'
+// CXX-NEXT: | `-CompoundStmt [[ADDR_8:0x[a-z0-9]*]] <col:22, line:12:1>
+// CXX-NEXT: |   `-ReturnStmt [[ADDR_9:0x[a-z0-9]*]] <line:11:3, col:10>
+// CXX-NEXT: |     `-IntegerLiteral [[ADDR_10:0x[a-z0-9]*]] <col:10> 'int' 1
+// CXX-NEXT: |-FunctionDecl [[ADDR_11:0x[a-z0-9]*]] prev [[ADDR_0]] <line:13:1, col:21> col:5 implicit used also_before 'int ({{.*}})'
+// CXX-NEXT: | `-OMPDeclareVariantAttr [[ADDR_12:0x[a-z0-9]*]] <<invalid sloc>> Implicit implementation={vendor(llvm)}
+// CXX-NEXT: |   `-DeclRefExpr [[ADDR_13:0x[a-z0-9]*]] <col:1> 'int ({{.*}})' Function [[ADDR_14:0x[a-z0-9]*]] 'also_before[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// CXX-NEXT: |-FunctionDecl [[ADDR_14]] <col:1, line:15:1> line:13:1 also_before[implementation={vendor(llvm)}] 'int ({{.*}})'
+// CXX-NEXT: | `-CompoundStmt [[ADDR_15:0x[a-z0-9]*]] <col:23, line:15:1>
+// CXX-NEXT: |   `-ReturnStmt [[ADDR_16:0x[a-z0-9]*]] <line:14:3, col:10>
+// CXX-NEXT: |     `-IntegerLiteral [[ADDR_17:0x[a-z0-9]*]] <col:10> 'int' 2
+// CXX-NEXT: |-FunctionDecl [[ADDR_18:0x[a-z0-9]*]] prev [[ADDR_4]] <line:18:1, line:20:1> line:18:5 used also_after 'int ({{.*}})'
+// CXX-NEXT: | |-CompoundStmt [[ADDR_19:0x[a-z0-9]*]] <col:22, line:20:1>
+// CXX-NEXT: | | `-ReturnStmt [[ADDR_20:0x[a-z0-9]*]] <line:19:3, col:10>
+// CXX-NEXT: | |   `-IntegerLiteral [[ADDR_21:0x[a-z0-9]*]] <col:10> 'int' 0
+// CXX-NEXT: | `-OMPDeclareVariantAttr [[ADDR_22:0x[a-z0-9]*]] <<invalid sloc>> Inherited Implicit implementation={vendor(llvm)}
+// CXX-NEXT: |   `-DeclRefExpr [[ADDR_6]] <line:10:1> 'int ({{.*}})' Function [[ADDR_7]] 'also_after[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// CXX-NEXT: |-FunctionDecl [[ADDR_23:0x[a-z0-9]*]] <line:22:1, col:10> col:6 used foo 'void ({{.*}})'
+// CXX-NEXT: |-TypedefDecl [[ADDR_24:0x[a-z0-9]*]] <line:23:1, col:22> col:14 referenced fd 'int (*)({{.*}})'
+// CXX-NEXT: | `-PointerType [[ADDR_25:0x[a-z0-9]*]] 'int (*)({{.*}})'
+// CXX-NEXT: |   `-ParenType [[ADDR_26:0x[a-z0-9]*]] 'int ({{.*}})' sugar
+// CXX-NEXT: |     `-FunctionProtoType [[ADDR_27:0x[a-z0-9]*]] 'int ({{.*}})' cdecl
+// CXX-NEXT: |       `-BuiltinType [[ADDR_28:0x[a-z0-9]*]] 'int'
+// CXX-NEXT: `-FunctionDecl [[ADDR_29:0x[a-z0-9]*]] <line:24:1, line:32:1> line:24:5 main 'int ({{.*}})'
+// CXX-NEXT:   `-CompoundStmt [[ADDR_30:0x[a-z0-9]*]] <col:12, line:32:1>
+// CXX-NEXT:     |-DeclStmt [[ADDR_31:0x[a-z0-9]*]] <line:26:3, col:12>
+// CXX-NEXT:     | `-VarDecl [[ADDR_32:0x[a-z0-9]*]] <col:3, col:11> col:6 used fns 'fd [2]'
+// CXX-NEXT:     |-BinaryOperator [[ADDR_33:0x[a-z0-9]*]] <line:27:3, col:13> 'fd':'int (*)({{.*}})' {{.*}}'='
+// CXX-NEXT:     | |-ArraySubscriptExpr [[ADDR_34:0x[a-z0-9]*]] <col:3, col:8> 'fd':'int (*)({{.*}})' lvalue
+// CXX-NEXT:     | | |-ImplicitCastExpr [[ADDR_35:0x[a-z0-9]*]] <col:3> 'fd *' <ArrayToPointerDecay>
+// CXX-NEXT:     | | | `-DeclRefExpr [[ADDR_36:0x[a-z0-9]*]] <col:3> 'fd [2]' {{.*}}Var [[ADDR_32]] 'fns' 'fd [2]'
+// CXX-NEXT:     | | `-IntegerLiteral [[ADDR_37:0x[a-z0-9]*]] <col:7> 'int' 0
+// CXX-NEXT:     | `-UnaryOperator [[ADDR_38:0x[a-z0-9]*]] <col:12, col:13> 'int (*)({{.*}})' prefix '&' cannot overflow
+// CXX-NEXT:     |   `-DeclRefExpr [[ADDR_39:0x[a-z0-9]*]] <col:13> 'int ({{.*}})' {{.*}}Function [[ADDR_11]] 'also_before' 'int ({{.*}})'
+// CXX-NEXT:     |-BinaryOperator [[ADDR_40:0x[a-z0-9]*]] <line:28:3, col:12> 'fd':'int (*)({{.*}})' {{.*}}'='
+// CXX-NEXT:     | |-ArraySubscriptExpr [[ADDR_41:0x[a-z0-9]*]] <col:3, col:8> 'fd':'int (*)({{.*}})' lvalue
+// CXX-NEXT:     | | |-ImplicitCastExpr [[ADDR_42:0x[a-z0-9]*]] <col:3> 'fd *' <ArrayToPointerDecay>
+// CXX-NEXT:     | | | `-DeclRefExpr [[ADDR_43:0x[a-z0-9]*]] <col:3> 'fd [2]' {{.*}}Var [[ADDR_32]] 'fns' 'fd [2]'
+// CXX-NEXT:     | | `-IntegerLiteral [[ADDR_44:0x[a-z0-9]*]] <col:7> 'int' 1
+// CXX-NEXT:     | `-ImplicitCastExpr [[ADDR_45:0x[a-z0-9]*]] <col:12> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CXX-NEXT:     |   `-DeclRefExpr [[ADDR_46:0x[a-z0-9]*]] <col:12> 'int ({{.*}})' {{.*}}Function [[ADDR_18]] 'also_after' 'int ({{.*}})'
+// CXX-NEXT:     `-ReturnStmt [[ADDR_47:0x[a-z0-9]*]] <line:29:3, line:31:19>
+// CXX-NEXT:       `-BinaryOperator [[ADDR_48:0x[a-z0-9]*]] <line:29:10, line:31:19> 'int' '+'
+// CXX-NEXT:         |-BinaryOperator [[ADDR_49:0x[a-z0-9]*]] <line:29:10, line:30:19> 'int' '+'
+// CXX-NEXT:         | |-CallExpr [[ADDR_50:0x[a-z0-9]*]] <line:29:10, col:30> 'int'
+// CXX-NEXT:         | | `-ImplicitCastExpr [[ADDR_51:0x[a-z0-9]*]] <col:10, col:28> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CXX-NEXT:         | |   `-ParenExpr [[ADDR_52:0x[a-z0-9]*]] <col:10, col:28> 'int ({{.*}})' lvalue
+// CXX-NEXT:         | |     `-BinaryOperator [[ADDR_53:0x[a-z0-9]*]] <col:11, col:18> 'int ({{.*}})' {{.*}}','
+// CXX-NEXT:         | |       |-CallExpr [[ADDR_54:0x[a-z0-9]*]] <col:11, col:15> 'void'
+// CXX-NEXT:         | |       | `-ImplicitCastExpr [[ADDR_55:0x[a-z0-9]*]] <col:11> 'void (*)({{.*}})' <FunctionToPointerDecay>
+// CXX-NEXT:         | |       |   `-DeclRefExpr [[ADDR_56:0x[a-z0-9]*]] <col:11> 'void ({{.*}})' {{.*}}Function [[ADDR_23]] 'foo' 'void ({{.*}})'
+// CXX-NEXT:         | |       `-DeclRefExpr [[ADDR_57:0x[a-z0-9]*]] <col:18> 'int ({{.*}})' {{.*}}Function [[ADDR_18]] 'also_after' 'int ({{.*}})'
+// CXX-NEXT:         | `-CallExpr [[ADDR_58:0x[a-z0-9]*]] <line:30:10, col:19> 'int'
+// CXX-NEXT:         |   `-ImplicitCastExpr [[ADDR_59:0x[a-z0-9]*]] <col:10, col:17> 'fd':'int (*)({{.*}})' <LValueToRValue>
+// CXX-NEXT:         |     `-ParenExpr [[ADDR_60:0x[a-z0-9]*]] <col:10, col:17> 'fd':'int (*)({{.*}})' lvalue
+// CXX-NEXT:         |       `-ArraySubscriptExpr [[ADDR_61:0x[a-z0-9]*]] <col:11, col:16> 'fd':'int (*)({{.*}})' lvalue
+// CXX-NEXT:         |         |-ImplicitCastExpr [[ADDR_62:0x[a-z0-9]*]] <col:11> 'fd *' <ArrayToPointerDecay>
+// CXX-NEXT:         |         | `-DeclRefExpr [[ADDR_63:0x[a-z0-9]*]] <col:11> 'fd [2]' {{.*}}Var [[ADDR_32]] 'fns' 'fd [2]'
+// CXX-NEXT:         |         `-IntegerLiteral [[ADDR_64:0x[a-z0-9]*]] <col:15> 'int' 0
+// CXX-NEXT:         `-CallExpr [[ADDR_65:0x[a-z0-9]*]] <line:31:10, col:19> 'int'
+// CXX-NEXT:           `-ImplicitCastExpr [[ADDR_66:0x[a-z0-9]*]] <col:10, col:17> 'fd':'int (*)({{.*}})' <LValueToRValue>
+// CXX-NEXT:             `-ParenExpr [[ADDR_67:0x[a-z0-9]*]] <col:10, col:17> 'fd':'int (*)({{.*}})' lvalue
+// CXX-NEXT:               `-ArraySubscriptExpr [[ADDR_68:0x[a-z0-9]*]] <col:11, col:16> 'fd':'int (*)({{.*}})' lvalue
+// CXX-NEXT:                 |-IntegerLiteral [[ADDR_69:0x[a-z0-9]*]] <col:11> 'int' 1
+// CXX-NEXT:                 `-ImplicitCastExpr [[ADDR_70:0x[a-z0-9]*]] <col:13> 'fd *' <ArrayToPointerDecay>
+// CXX-NEXT:                   `-DeclRefExpr [[ADDR_71:0x[a-z0-9]*]] <col:13> 'fd [2]' {{.*}}Var [[ADDR_32]] 'fns' 'fd [2]'
diff --git a/clang/test/AST/ast-dump-openmp-begin-declare-variant_addr_1.c b/clang/test/AST/ast-dump-openmp-begin-declare-variant_addr_1.c
new file mode 100644
index 0000000000000..1cb39dac3c7f6
--- /dev/null
+++ b/clang/test/AST/ast-dump-openmp-begin-declare-variant_addr_1.c
@@ -0,0 +1,151 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -fopenmp -verify -ast-dump %s       | FileCheck %s --check-prefix=C
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -fopenmp -verify -ast-dump %s -x c++| FileCheck %s --check-prefix=CXX
+// expected-no-diagnostics
+
+int also_before(void) {
+  return 0;
+}
+
+#pragma omp begin declare variant match(implementation={vendor(llvm)})
+int also_after(void) {
+  return 1;
+}
+int also_before(void) {
+  return 2;
+}
+#pragma omp end declare variant
+
+int also_after(void) {
+  return 0;
+}
+
+int test(int (*fd)(void)) {
+  return fd();
+}
+int main() {
+  // Should return 0.
+  return test(also_after) +
+         test(also_before) +
+         test(&also_after) +
+         test(&also_before);
+}
+
+// Make sure:
+//  - we see the specialization in the AST
+//  - we pick the right callees
+
+// CXX:      |-FunctionDecl [[ADDR_0:0x[a-z0-9]*]] <{{.*}}, line:7:1> line:5:5 used also_before 'int ({{.*}})'
+// CXX-NEXT: | `-CompoundStmt [[ADDR_1:0x[a-z0-9]*]] <col:23, line:7:1>
+// CXX-NEXT: |   `-ReturnStmt [[ADDR_2:0x[a-z0-9]*]] <line:6:3, col:10>
+// CXX-NEXT: |     `-IntegerLiteral [[ADDR_3:0x[a-z0-9]*]] <col:10> 'int' 0
+// CXX-NEXT: |-FunctionDecl [[ADDR_4:0x[a-z0-9]*]] <line:10:1, col:20> col:5 implicit used also_after 'int ({{.*}})'
+// CXX-NEXT: | `-OMPDeclareVariantAttr [[ADDR_5:0x[a-z0-9]*]] <<invalid sloc>> Implicit implementation={vendor(llvm)}
+// CXX-NEXT: |   `-DeclRefExpr [[ADDR_6:0x[a-z0-9]*]] <col:1> 'int ({{.*}})' Function [[ADDR_7:0x[a-z0-9]*]] 'also_after[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// CXX-NEXT: |-FunctionDecl [[ADDR_7]] <col:1, line:12:1> line:10:1 also_after[implementation={vendor(llvm)}] 'int ({{.*}})'
+// CXX-NEXT: | `-CompoundStmt [[ADDR_8:0x[a-z0-9]*]] <col:22, line:12:1>
+// CXX-NEXT: |   `-ReturnStmt [[ADDR_9:0x[a-z0-9]*]] <line:11:3, col:10>
+// CXX-NEXT: |     `-IntegerLiteral [[ADDR_10:0x[a-z0-9]*]] <col:10> 'int' 1
+// CXX-NEXT: |-FunctionDecl [[ADDR_11:0x[a-z0-9]*]] prev [[ADDR_0]] <line:13:1, col:21> col:5 implicit used also_before 'int ({{.*}})'
+// CXX-NEXT: | `-OMPDeclareVariantAttr [[ADDR_12:0x[a-z0-9]*]] <<invalid sloc>> Implicit implementation={vendor(llvm)}
+// CXX-NEXT: |   `-DeclRefExpr [[ADDR_13:0x[a-z0-9]*]] <col:1> 'int ({{.*}})' Function [[ADDR_14:0x[a-z0-9]*]] 'also_before[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// CXX-NEXT: |-FunctionDecl [[ADDR_14]] <col:1, line:15:1> line:13:1 also_before[implementation={vendor(llvm)}] 'int ({{.*}})'
+// CXX-NEXT: | `-CompoundStmt [[ADDR_15:0x[a-z0-9]*]] <col:23, line:15:1>
+// CXX-NEXT: |   `-ReturnStmt [[ADDR_16:0x[a-z0-9]*]] <line:14:3, col:10>
+// CXX-NEXT: |     `-IntegerLiteral [[ADDR_17:0x[a-z0-9]*]] <col:10> 'int' 2
+// CXX-NEXT: |-FunctionDecl [[ADDR_18:0x[a-z0-9]*]] prev [[ADDR_4]] <line:18:1, line:20:1> line:18:5 used also_after 'int ({{.*}})'
+// CXX-NEXT: | |-CompoundStmt [[ADDR_19:0x[a-z0-9]*]] <col:22, line:20:1>
+// CXX-NEXT: | | `-ReturnStmt [[ADDR_20:0x[a-z0-9]*]] <line:19:3, col:10>
+// CXX-NEXT: | |   `-IntegerLiteral [[ADDR_21:0x[a-z0-9]*]] <col:10> 'int' 0
+// CXX-NEXT: | `-OMPDeclareVariantAttr [[ADDR_22:0x[a-z0-9]*]] <<invalid sloc>> Inherited Implicit implementation={vendor(llvm)}
+// CXX-NEXT: |   `-DeclRefExpr [[ADDR_6]] <line:10:1> 'int ({{.*}})' Function [[ADDR_7]] 'also_after[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// CXX-NEXT: |-FunctionDecl [[ADDR_23:0x[a-z0-9]*]] <line:22:1, line:24:1> line:22:5 used test 'int (int (*)({{.*}}))'
+// CXX-NEXT: | |-ParmVarDecl [[ADDR_24:0x[a-z0-9]*]] <col:10, col:24> col:16 used fd 'int (*)({{.*}})'
+// CXX-NEXT: | `-CompoundStmt [[ADDR_25:0x[a-z0-9]*]] <col:27, line:24:1>
+// CXX-NEXT: |   `-ReturnStmt [[ADDR_26:0x[a-z0-9]*]] <line:23:3, col:13>
+// CXX-NEXT: |     `-CallExpr [[ADDR_27:0x[a-z0-9]*]] <col:10, col:13> 'int'
+// CXX-NEXT: |       `-ImplicitCastExpr [[ADDR_28:0x[a-z0-9]*]] <col:10> 'int (*)({{.*}})' <LValueToRValue>
+// CXX-NEXT: |         `-DeclRefExpr [[ADDR_29:0x[a-z0-9]*]] <col:10> 'int (*)({{.*}})' {{.*}}ParmVar [[ADDR_24]] 'fd' 'int (*)({{.*}})'
+// CXX-NEXT: `-FunctionDecl [[ADDR_30:0x[a-z0-9]*]] <line:25:1, line:31:1> line:25:5 main 'int ({{.*}})'
+// CXX-NEXT:   `-CompoundStmt [[ADDR_31:0x[a-z0-9]*]] <col:12, line:31:1>
+// CXX-NEXT:     `-ReturnStmt [[ADDR_32:0x[a-z0-9]*]] <line:27:3, line:30:27>
+// CXX-NEXT:       `-BinaryOperator [[ADDR_33:0x[a-z0-9]*]] <line:27:10, line:30:27> 'int' '+'
+// CXX-NEXT:         |-BinaryOperator [[ADDR_34:0x[a-z0-9]*]] <line:27:10, line:29:26> 'int' '+'
+// CXX-NEXT:         | |-BinaryOperator [[ADDR_35:0x[a-z0-9]*]] <line:27:10, line:28:26> 'int' '+'
+// CXX-NEXT:         | | |-CallExpr [[ADDR_36:0x[a-z0-9]*]] <line:27:10, col:25> 'int'
+// CXX-NEXT:         | | | |-ImplicitCastExpr [[ADDR_37:0x[a-z0-9]*]] <col:10> 'int (*)(int (*)({{.*}}))' <FunctionToPointerDecay>
+// CXX-NEXT:         | | | | `-DeclRefExpr [[ADDR_38:0x[a-z0-9]*]] <col:10> 'int (int (*)({{.*}}))' {{.*}}Function [[ADDR_23]] 'test' 'int (int (*)({{.*}}))'
+// CXX-NEXT:         | | | `-ImplicitCastExpr [[ADDR_39:0x[a-z0-9]*]] <col:15> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CXX-NEXT:         | | |   `-DeclRefExpr [[ADDR_40:0x[a-z0-9]*]] <col:15> 'int ({{.*}})' {{.*}}Function [[ADDR_18]] 'also_after' 'int ({{.*}})'
+// CXX-NEXT:         | | `-CallExpr [[ADDR_41:0x[a-z0-9]*]] <line:28:10, col:26> 'int'
+// CXX-NEXT:         | |   |-ImplicitCastExpr [[ADDR_42:0x[a-z0-9]*]] <col:10> 'int (*)(int (*)({{.*}}))' <FunctionToPointerDecay>
+// CXX-NEXT:         | |   | `-DeclRefExpr [[ADDR_43:0x[a-z0-9]*]] <col:10> 'int (int (*)({{.*}}))' {{.*}}Function [[ADDR_23]] 'test' 'int (int (*)({{.*}}))'
+// CXX-NEXT:         | |   `-ImplicitCastExpr [[ADDR_44:0x[a-z0-9]*]] <col:15> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CXX-NEXT:         | |     `-DeclRefExpr [[ADDR_45:0x[a-z0-9]*]] <col:15> 'int ({{.*}})' {{.*}}Function [[ADDR_11]] 'also_before' 'int ({{.*}})'
+// CXX-NEXT:         | `-CallExpr [[ADDR_46:0x[a-z0-9]*]] <line:29:10, col:26> 'int'
+// CXX-NEXT:         |   |-ImplicitCastExpr [[ADDR_47:0x[a-z0-9]*]] <col:10> 'int (*)(int (*)({{.*}}))' <FunctionToPointerDecay>
+// CXX-NEXT:         |   | `-DeclRefExpr [[ADDR_48:0x[a-z0-9]*]] <col:10> 'int (int (*)({{.*}}))' {{.*}}Function [[ADDR_23]] 'test' 'int (int (*)({{.*}}))'
+// CXX-NEXT:         |   `-UnaryOperator [[ADDR_49:0x[a-z0-9]*]] <col:15, col:16> 'int (*)({{.*}})' prefix '&' cannot overflow
+// CXX-NEXT:         |     `-DeclRefExpr [[ADDR_50:0x[a-z0-9]*]] <col:16> 'int ({{.*}})' {{.*}}Function [[ADDR_18]] 'also_after' 'int ({{.*}})'
+// CXX-NEXT:         `-CallExpr [[ADDR_51:0x[a-z0-9]*]] <line:30:10, col:27> 'int'
+// CXX-NEXT:           |-ImplicitCastExpr [[ADDR_52:0x[a-z0-9]*]] <col:10> 'int (*)(int (*)({{.*}}))' <FunctionToPointerDecay>
+// CXX-NEXT:           | `-DeclRefExpr [[ADDR_53:0x[a-z0-9]*]] <col:10> 'int (int (*)({{.*}}))' {{.*}}Function [[ADDR_23]] 'test' 'int (int (*)({{.*}}))'
+// CXX-NEXT:           `-UnaryOperator [[ADDR_54:0x[a-z0-9]*]] <col:15, col:16> 'int (*)({{.*}})' prefix '&' cannot overflow
+// CXX-NEXT:             `-DeclRefExpr [[ADDR_55:0x[a-z0-9]*]] <col:16> 'int ({{.*}})' {{.*}}Function [[ADDR_11]] 'also_before' 'int ({{.*}})'
+
+// C:      |-FunctionDecl [[ADDR_0:0x[a-z0-9]*]] <{{.*}}, line:7:1> line:5:5 used also_before 'int ({{.*}})'
+// C-NEXT: | `-CompoundStmt [[ADDR_1:0x[a-z0-9]*]] <col:23, line:7:1>
+// C-NEXT: |   `-ReturnStmt [[ADDR_2:0x[a-z0-9]*]] <line:6:3, col:10>
+// C-NEXT: |     `-IntegerLiteral [[ADDR_3:0x[a-z0-9]*]] <col:10> 'int' 0
+// C-NEXT: |-FunctionDecl [[ADDR_4:0x[a-z0-9]*]] <line:10:1, col:20> col:5 implicit used also_after 'int ({{.*}})'
+// C-NEXT: | `-OMPDeclareVariantAttr [[ADDR_5:0x[a-z0-9]*]] <<invalid sloc>> Implicit implementation={vendor(llvm)}
+// C-NEXT: |   `-DeclRefExpr [[ADDR_6:0x[a-z0-9]*]] <col:1> 'int ({{.*}})' Function [[ADDR_7:0x[a-z0-9]*]] 'also_after[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// C-NEXT: |-FunctionDecl [[ADDR_7]] <col:1, line:12:1> line:10:1 also_after[implementation={vendor(llvm)}] 'int ({{.*}})'
+// C-NEXT: | `-CompoundStmt [[ADDR_8:0x[a-z0-9]*]] <col:22, line:12:1>
+// C-NEXT: |   `-ReturnStmt [[ADDR_9:0x[a-z0-9]*]] <line:11:3, col:10>
+// C-NEXT: |     `-IntegerLiteral [[ADDR_10:0x[a-z0-9]*]] <col:10> 'int' 1
+// C-NEXT: |-FunctionDecl [[ADDR_11:0x[a-z0-9]*]] prev [[ADDR_0]] <line:13:1, col:21> col:5 implicit used also_before 'int ({{.*}})'
+// C-NEXT: | `-OMPDeclareVariantAttr [[ADDR_12:0x[a-z0-9]*]] <<invalid sloc>> Implicit implementation={vendor(llvm)}
+// C-NEXT: |   `-DeclRefExpr [[ADDR_13:0x[a-z0-9]*]] <col:1> 'int ({{.*}})' Function [[ADDR_14:0x[a-z0-9]*]] 'also_before[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// C-NEXT: |-FunctionDecl [[ADDR_14]] <col:1, line:15:1> line:13:1 also_before[implementation={vendor(llvm)}] 'int ({{.*}})'
+// C-NEXT: | `-CompoundStmt [[ADDR_15:0x[a-z0-9]*]] <col:23, line:15:1>
+// C-NEXT: |   `-ReturnStmt [[ADDR_16:0x[a-z0-9]*]] <line:14:3, col:10>
+// C-NEXT: |     `-IntegerLiteral [[ADDR_17:0x[a-z0-9]*]] <col:10> 'int' 2
+// C-NEXT: |-FunctionDecl [[ADDR_18:0x[a-z0-9]*]] prev [[ADDR_4]] <line:18:1, line:20:1> line:18:5 used also_after 'int ({{.*}})'
+// C-NEXT: | |-CompoundStmt [[ADDR_19:0x[a-z0-9]*]] <col:22, line:20:1>
+// C-NEXT: | | `-ReturnStmt [[ADDR_20:0x[a-z0-9]*]] <line:19:3, col:10>
+// C-NEXT: | |   `-IntegerLiteral [[ADDR_21:0x[a-z0-9]*]] <col:10> 'int' 0
+// C-NEXT: | `-OMPDeclareVariantAttr [[ADDR_22:0x[a-z0-9]*]] <<invalid sloc>> Inherited Implicit implementation={vendor(llvm)}
+// C-NEXT: |   `-DeclRefExpr [[ADDR_6]] <line:10:1> 'int ({{.*}})' Function [[ADDR_7]] 'also_after[implementation={vendor(llvm)}]' 'int ({{.*}})'
+// C-NEXT: |-FunctionDecl [[ADDR_23:0x[a-z0-9]*]] <line:22:1, line:24:1> line:22:5 used test 'int (int (*)({{.*}}))'
+// C-NEXT: | |-ParmVarDecl [[ADDR_24:0x[a-z0-9]*]] <col:10, col:24> col:16 used fd 'int (*)({{.*}})'
+// C-NEXT: | `-CompoundStmt [[ADDR_25:0x[a-z0-9]*]] <col:27, line:24:1>
+// C-NEXT: |   `-ReturnStmt [[ADDR_26:0x[a-z0-9]*]] <line:23:3, col:13>
+// C-NEXT: |     `-CallExpr [[ADDR_27:0x[a-z0-9]*]] <col:10, col:13> 'int'
+// C-NEXT: |       `-ImplicitCastExpr [[ADDR_28:0x[a-z0-9]*]] <col:10> 'int (*)({{.*}})' <LValueToRValue>
+// C-NEXT: |         `-DeclRefExpr [[ADDR_29:0x[a-z0-9]*]] <col:10> 'int (*)({{.*}})' {{.*}}ParmVar [[ADDR_24]] 'fd' 'int (*)({{.*}})'
+// C-NEXT: `-FunctionDecl [[ADDR_30:0x[a-z0-9]*]] <line:25:1, line:31:1> line:25:5 main 'int ({{.*}})'
+// C-NEXT:   `-CompoundStmt [[ADDR_31:0x[a-z0-9]*]] <col:12, line:31:1>
+// C-NEXT:     `-ReturnStmt [[ADDR_32:0x[a-z0-9]*]] <line:27:3, line:30:27>
+// C-NEXT:       `-BinaryOperator [[ADDR_33:0x[a-z0-9]*]] <line:27:10, line:30:27> 'int' '+'
+// C-NEXT:         |-BinaryOperator [[ADDR_34:0x[a-z0-9]*]] <line:27:10, line:29:26> 'int' '+'
+// C-NEXT:         | |-BinaryOperator [[ADDR_35:0x[a-z0-9]*]] <line:27:10, line:28:26> 'int' '+'
+// C-NEXT:         | | |-CallExpr [[ADDR_36:0x[a-z0-9]*]] <line:27:10, col:25> 'int'
+// C-NEXT:         | | | |-ImplicitCastExpr [[ADDR_37:0x[a-z0-9]*]] <col:10> 'int (*)(int (*)({{.*}}))' <FunctionToPointerDecay>
+// C-NEXT:         | | | | `-DeclRefExpr [[ADDR_38:0x[a-z0-9]*]] <col:10> 'int (int (*)({{.*}}))' Function [[ADDR_23]] 'test' 'int (int (*)({{.*}}))'
+// C-NEXT:         | | | `-ImplicitCastExpr [[ADDR_39:0x[a-z0-9]*]] <col:15> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// C-NEXT:         | | |   `-DeclRefExpr [[ADDR_40:0x[a-z0-9]*]] <col:15> 'int ({{.*}})' Function [[ADDR_18]] 'also_after' 'int ({{.*}})'
+// C-NEXT:         | | `-CallExpr [[ADDR_41:0x[a-z0-9]*]] <line:28:10, col:26> 'int'
+// C-NEXT:         | |   |-ImplicitCastExpr [[ADDR_42:0x[a-z0-9]*]] <col:10> 'int (*)(int (*)({{.*}}))' <FunctionToPointerDecay>
+// C-NEXT:         | |   | `-DeclRefExpr [[ADDR_43:0x[a-z0-9]*]] <col:10> 'int (int (*)({{.*}}))' Function [[ADDR_23]] 'test' 'int (int (*)({{.*}}))'
+// C-NEXT:         | |   `-ImplicitCastExpr [[ADDR_44:0x[a-z0-9]*]] <col:15> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// C-NEXT:         | |     `-DeclRefExpr [[ADDR_45:0x[a-z0-9]*]] <col:15> 'int ({{.*}})' Function [[ADDR_11]] 'also_before' 'int ({{.*}})'
+// C-NEXT:         | `-CallExpr [[ADDR_46:0x[a-z0-9]*]] <line:29:10, col:26> 'int'
+// C-NEXT:         |   |-ImplicitCastExpr [[ADDR_47:0x[a-z0-9]*]] <col:10> 'int (*)(int (*)({{.*}}))' <FunctionToPointerDecay>
+// C-NEXT:         |   | `-DeclRefExpr [[ADDR_48:0x[a-z0-9]*]] <col:10> 'int (int (*)({{.*}}))' Function [[ADDR_23]] 'test' 'int (int (*)({{.*}}))'
+// C-NEXT:         |   `-UnaryOperator [[ADDR_49:0x[a-z0-9]*]] <col:15, col:16> 'int (*)({{.*}})' prefix '&' cannot overflow
+// C-NEXT:         |     `-DeclRefExpr [[ADDR_50:0x[a-z0-9]*]] <col:16> 'int ({{.*}})' Function [[ADDR_18]] 'also_after' 'int ({{.*}})'
+// C-NEXT:         `-CallExpr [[ADDR_51:0x[a-z0-9]*]] <line:30:10, col:27> 'int'
+// C-NEXT:           |-ImplicitCastExpr [[ADDR_52:0x[a-z0-9]*]] <col:10> 'int (*)(int (*)({{.*}}))' <FunctionToPointerDecay>
+// C-NEXT:           | `-DeclRefExpr [[ADDR_53:0x[a-z0-9]*]] <col:10> 'int (int (*)({{.*}}))' Function [[ADDR_23]] 'test' 'int (int (*)({{.*}}))'
+// C-NEXT:           `-UnaryOperator [[ADDR_54:0x[a-z0-9]*]] <col:15, col:16> 'int (*)({{.*}})' prefix '&' cannot overflow
+// C-NEXT:             `-DeclRefExpr [[ADDR_55:0x[a-z0-9]*]] <col:16> 'int ({{.*}})' Function [[ADDR_11]] 'also_before' 'int ({{.*}})'
diff --git a/clang/test/AST/ast-dump-openmp-begin-declare-variant_decl_1.c b/clang/test/AST/ast-dump-openmp-begin-declare-variant_decl_1.c
new file mode 100644
index 0000000000000..2cf28058703c6
--- /dev/null
+++ b/clang/test/AST/ast-dump-openmp-begin-declare-variant_decl_1.c
@@ -0,0 +1,53 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -fopenmp -verify -ast-dump %s       | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -fopenmp -verify -ast-dump %s -x c++| FileCheck %s
+// expected-no-diagnostics
+// FIXME: We have to improve the warnings here as nothing is impacted by the declare variant.
+int also_before(void) {
+  return 0;
+}
+
+#pragma omp begin declare variant match(device={kind(cpu)})
+int also_before(void);
+#pragma omp end declare variant
+#pragma omp begin declare variant match(implementation={vendor(score(100):llvm)})
+int also_after(void);
+#pragma omp end declare variant
+#pragma omp begin declare variant match(implementation={vendor(score(0):llvm)})
+int also_before(void);
+#pragma omp end declare variant
+
+int also_after(void) {
+  return 0;
+}
+
+int test() {
+  // Should return 0.
+  return also_after() + also_before();
+}
+
+// Make sure:
+//  - we do see the ast nodes for the cpu kind
+//  - we do see the ast nodes for the llvm vendor
+//  - we pick the right callees
+
+// CHECK:      |-FunctionDecl [[ADDR_0:0x[a-z0-9]*]] <{{.*}}, line:7:1> line:5:5 used also_before 'int ({{.*}})'
+// CHECK-NEXT: | `-CompoundStmt [[ADDR_1:0x[a-z0-9]*]] <col:23, line:7:1>
+// CHECK-NEXT: |   `-ReturnStmt [[ADDR_2:0x[a-z0-9]*]] <line:6:3, col:10>
+// CHECK-NEXT: |     `-IntegerLiteral [[ADDR_3:0x[a-z0-9]*]] <col:10> 'int' 0
+// CHECK-NEXT: |-FunctionDecl [[ADDR_4:0x[a-z0-9]*]] prev [[ADDR_0]] <line:10:1, col:21> col:5 used also_before 'int ({{.*}})'
+// CHECK-NEXT: |-FunctionDecl [[ADDR_5:0x[a-z0-9]*]] <line:13:1, col:20> col:5 used also_after 'int ({{.*}})'
+// CHECK-NEXT: |-FunctionDecl [[ADDR_6:0x[a-z0-9]*]] prev [[ADDR_4]] <line:16:1, col:21> col:5 used also_before 'int ({{.*}})'
+// CHECK-NEXT: |-FunctionDecl [[ADDR_7:0x[a-z0-9]*]] prev [[ADDR_5]] <line:19:1, line:21:1> line:19:5 used also_after 'int ({{.*}})'
+// CHECK-NEXT: | `-CompoundStmt [[ADDR_8:0x[a-z0-9]*]] <col:22, line:21:1>
+// CHECK-NEXT: |   `-ReturnStmt [[ADDR_9:0x[a-z0-9]*]] <line:20:3, col:10>
+// CHECK-NEXT: |     `-IntegerLiteral [[ADDR_10:0x[a-z0-9]*]] <col:10> 'int' 0
+// CHECK-NEXT: `-FunctionDecl [[ADDR_11:0x[a-z0-9]*]] <line:23:1, line:26:1> line:23:5 test 'int ({{.*}})'
+// CHECK-NEXT:   `-CompoundStmt [[ADDR_12:0x[a-z0-9]*]] <col:12, line:26:1>
+// CHECK-NEXT:     `-ReturnStmt [[ADDR_13:0x[a-z0-9]*]] <line:25:3, col:37>
+// CHECK-NEXT:       `-BinaryOperator [[ADDR_14:0x[a-z0-9]*]] <col:10, col:37> 'int' '+'
+// CHECK-NEXT:         |-CallExpr [[ADDR_15:0x[a-z0-9]*]] <col:10, col:21> 'int'
+// CHECK-NEXT:         | `-ImplicitCastExpr [[ADDR_16:0x[a-z0-9]*]] <col:10> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CHECK-NEXT:         |   `-DeclRefExpr [[ADDR_17:0x[a-z0-9]*]] <col:10> 'int ({{.*}})' {{.*}}Function [[ADDR_7]] 'also_after' 'int ({{.*}})'
+// CHECK-NEXT:         `-CallExpr [[ADDR_18:0x[a-z0-9]*]] <col:25, col:37> 'int'
+// CHECK-NEXT:           `-ImplicitCastExpr [[ADDR_19:0x[a-z0-9]*]] <col:25> 'int (*)({{.*}})' <FunctionToPointerDecay>
+// CHECK-NEXT:             `-DeclRefExpr [[ADDR_20:0x[a-z0-9]*]] <col:25> 'int ({{.*}})' {{.*}}Function [[ADDR_6]] 'also_before' 'int ({{.*}})'
diff --git a/clang/test/AST/ast-dump-recovery.cpp b/clang/test/AST/ast-dump-recovery.cpp
index beb409edc4a6a..9ccfc5a106f89 100644
--- a/clang/test/AST/ast-dump-recovery.cpp
+++ b/clang/test/AST/ast-dump-recovery.cpp
@@ -83,3 +83,18 @@ int binary = a + nullptr;
 // CHECK-NEXT:  `-DeclRefExpr {{.*}} 'a'
 // DISABLED-NOT: -RecoveryExpr {{.*}} contains-errors
 int ternary = a ? nullptr : a;
+
+// CHECK:     FunctionDecl
+// CHECK-NEXT:|-ParmVarDecl {{.*}} x
+// CHECK-NEXT:`-CompoundStmt
+// CHECK-NEXT: |-RecoveryExpr {{.*}} contains-errors
+// CHECK-NEXT: | `-DeclRefExpr {{.*}} 'foo'
+// CHECK-NEXT: `-CallExpr {{.*}} contains-errors
+// CHECK-NEXT:  |-RecoveryExpr {{.*}} contains-errors
+// CHECK-NEXT:  | `-DeclRefExpr {{.*}} 'foo'
+// CHECK-NEXT:  `-DeclRefExpr {{.*}} 'x'
+struct Foo {} foo;
+void test(int x) {
+  foo.abc;
+  foo->func(x);
+}
\ No newline at end of file
diff --git a/clang/test/Analysis/analyzer-enabled-checkers.c b/clang/test/Analysis/analyzer-enabled-checkers.c
index 55f15f6d37c8f..6f719ec15d4f5 100644
--- a/clang/test/Analysis/analyzer-enabled-checkers.c
+++ b/clang/test/Analysis/analyzer-enabled-checkers.c
@@ -7,11 +7,11 @@
 // CHECK:      OVERVIEW: Clang Static Analyzer Enabled Checkers List
 // CHECK-EMPTY:
 // CHECK-NEXT: apiModeling.StdCLibraryFunctions
+// CHECK-NEXT: core.CallAndMessage
 // CHECK-NEXT: apiModeling.StdCLibraryFunctionArgs
 // CHECK-NEXT: apiModeling.TrustNonnull
 // CHECK-NEXT: apiModeling.llvm.CastValue
 // CHECK-NEXT: apiModeling.llvm.ReturnValue
-// CHECK-NEXT: core.CallAndMessage
 // CHECK-NEXT: core.DivideZero
 // CHECK-NEXT: core.DynamicTypePropagation
 // CHECK-NEXT: core.NonNullParamChecker
diff --git a/clang/test/Analysis/container-modeling-no-aggressive-binary-operation-simplification-warn.cpp b/clang/test/Analysis/container-modeling-no-aggressive-binary-operation-simplification-warn.cpp
new file mode 100644
index 0000000000000..1a55e878f9ef0
--- /dev/null
+++ b/clang/test/Analysis/container-modeling-no-aggressive-binary-operation-simplification-warn.cpp
@@ -0,0 +1,7 @@
+// RUN: %clang_analyze_cc1 -std=c++11\
+// RUN: -analyzer-checker=core,cplusplus,alpha.cplusplus.ContainerModeling\
+// RUN: %s 2>&1 | FileCheck %s
+
+// XFAIL: *
+
+// CHECK: checker cannot be enabled with analyzer option 'aggressive-binary-operation-simplification' == false
diff --git a/clang/test/Analysis/container-modeling.cpp b/clang/test/Analysis/container-modeling.cpp
index 8162ba9039e80..bf4a12a0e0fe2 100644
--- a/clang/test/Analysis/container-modeling.cpp
+++ b/clang/test/Analysis/container-modeling.cpp
@@ -1,6 +1,6 @@
-// RUN: %clang_analyze_cc1 -std=c++11 -analyzer-checker=core,cplusplus,debug.DebugIteratorModeling,debug.ExprInspection -analyzer-config aggressive-binary-operation-simplification=true -analyzer-config c++-container-inlining=false %s -verify
+// RUN: %clang_analyze_cc1 -std=c++11 -analyzer-checker=core,cplusplus,debug.DebugIteratorModeling,debug.ExprInspection -analyzer-config aggressive-binary-operation-simplification=true -analyzer-config c++-container-inlining=false %s -analyzer-output=text -verify
 
-// RUN: %clang_analyze_cc1 -std=c++11 -analyzer-checker=core,cplusplus,debug.DebugIteratorModeling,debug.ExprInspection -analyzer-config aggressive-binary-operation-simplification=true -analyzer-config c++-container-inlining=true -DINLINE=1 %s -verify
+// RUN: %clang_analyze_cc1 -std=c++11 -analyzer-checker=core,cplusplus,debug.DebugIteratorModeling,debug.ExprInspection -analyzer-config aggressive-binary-operation-simplification=true -analyzer-config c++-container-inlining=true -DINLINE=1 %s -analyzer-output=text -verify
 
 // RUN: %clang_analyze_cc1 -std=c++11 -analyzer-checker=core,cplusplus,alpha.cplusplus.IteratorModeling,debug.ExprInspection -analyzer-config aggressive-binary-operation-simplification=true %s 2>&1 | FileCheck %s
 
@@ -20,14 +20,16 @@ void begin(const std::vector<int> &V) {
   V.begin();
 
   clang_analyzer_denote(clang_analyzer_container_begin(V), "$V.begin()");
-  clang_analyzer_express(clang_analyzer_container_begin(V)); //expected-warning{{$V.begin()}}
+  clang_analyzer_express(clang_analyzer_container_begin(V)); // expected-warning{{$V.begin()}}
+                                                             // expected-note@-1{{$V.begin()}}
 }
 
 void end(const std::vector<int> &V) {
   V.end();
 
   clang_analyzer_denote(clang_analyzer_container_end(V), "$V.end()");
-  clang_analyzer_express(clang_analyzer_container_end(V)); //expected-warning{{$V.end()}}
+  clang_analyzer_express(clang_analyzer_container_end(V)); // expected-warning{{$V.end()}}
+                                                           // expected-note@-1{{$V.end()}}
 }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -48,8 +50,10 @@ void move_assignment(std::vector<int> &V1, std::vector<int> &V2) {
   long B2 = clang_analyzer_container_begin(V2);
   long E2 = clang_analyzer_container_end(V2);
   V1 = std::move(V2);
-  clang_analyzer_eval(clang_analyzer_container_begin(V1) == B2); //expected-warning{{TRUE}}
-  clang_analyzer_eval(clang_analyzer_container_end(V2) == E2); //expected-warning{{TRUE}}
+  clang_analyzer_eval(clang_analyzer_container_begin(V1) == B2); // expected-warning{{TRUE}}
+                                                                 // expected-note@-1{{TRUE}}
+  clang_analyzer_eval(clang_analyzer_container_end(V2) == E2); // expected-warning{{TRUE}}
+                                                               // expected-note@-1{{TRUE}}
 }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -60,9 +64,11 @@ void move_assignment(std::vector<int> &V1, std::vector<int> &V2) {
 
 /// push_back()
 ///
-/// Design decision: extends containers to the ->RIGHT-> (i.e. the
+/// Design decision: extends containers to the ->BACK-> (i.e. the
 /// past-the-end position of the container is incremented).
 
+void clang_analyzer_dump(void*);
+
 void push_back(std::vector<int> &V, int n) {
   V.cbegin();
   V.cend();
@@ -70,15 +76,17 @@ void push_back(std::vector<int> &V, int n) {
   clang_analyzer_denote(clang_analyzer_container_begin(V), "$V.begin()");
   clang_analyzer_denote(clang_analyzer_container_end(V), "$V.end()");
 
-  V.push_back(n);
+  V.push_back(n); // expected-note 2{{Container 'V' extended to the back by 1 position}}
 
   clang_analyzer_express(clang_analyzer_container_begin(V)); // expected-warning{{$V.begin()}}
+                                                             // expected-note@-1{{$V.begin()}}
   clang_analyzer_express(clang_analyzer_container_end(V)); // expected-warning{{$V.end() + 1}}
+                                                           // expected-note@-1{{$V.end() + 1}}
 }
 
 /// emplace_back()
 ///
-/// Design decision: extends containers to the ->RIGHT-> (i.e. the
+/// Design decision: extends containers to the ->BACK-> (i.e. the
 /// past-the-end position of the container is incremented).
 
 void emplace_back(std::vector<int> &V, int n) {
@@ -88,15 +96,17 @@ void emplace_back(std::vector<int> &V, int n) {
   clang_analyzer_denote(clang_analyzer_container_begin(V), "$V.begin()");
   clang_analyzer_denote(clang_analyzer_container_end(V), "$V.end()");
 
-  V.emplace_back(n);
+  V.emplace_back(n); // expected-note 2{{Container 'V' extended to the back by 1 position}}
 
   clang_analyzer_express(clang_analyzer_container_begin(V)); // expected-warning{{$V.begin()}}
+                                                             // expected-note@-1{{$V.begin()}}
   clang_analyzer_express(clang_analyzer_container_end(V)); // expected-warning{{$V.end() + 1}}
+                                                           // expected-note@-1{{$V.end() + 1}}
 }
 
 /// pop_back()
 ///
-/// Design decision: shrinks containers to the <-LEFT<- (i.e. the
+/// Design decision: shrinks containers to the <-FRONT<- (i.e. the
 /// past-the-end position of the container is decremented).
 
 void pop_back(std::vector<int> &V, int n) {
@@ -106,66 +116,132 @@ void pop_back(std::vector<int> &V, int n) {
   clang_analyzer_denote(clang_analyzer_container_begin(V), "$V.begin()");
   clang_analyzer_denote(clang_analyzer_container_end(V), "$V.end()");
 
-  V.pop_back();
+  V.pop_back(); // expected-note 2{{Container 'V' shrank from the back by 1 position}}
+
 
   clang_analyzer_express(clang_analyzer_container_begin(V)); // expected-warning{{$V.begin()}}
+                                                             // expected-note@-1{{$V.begin()}}
   clang_analyzer_express(clang_analyzer_container_end(V)); // expected-warning{{$V.end() - 1}}
+                                                           // expected-note@-1{{$V.end() - 1}}
 }
 
 /// push_front()
 ///
-/// Design decision: extends containers to the <-LEFT<- (i.e. the first
+/// Design decision: extends containers to the <-FRONT<- (i.e. the first
 /// position of the container is decremented).
 
-void push_front(std::deque<int> &D, int n) {
-  D.cbegin();
-  D.cend();
+void push_front(std::list<int> &L, int n) {
+  L.cbegin();
+  L.cend();
 
-  clang_analyzer_denote(clang_analyzer_container_begin(D), "$D.begin()");
-  clang_analyzer_denote(clang_analyzer_container_end(D), "$D.end()");
+  clang_analyzer_denote(clang_analyzer_container_begin(L), "$L.begin()");
+  clang_analyzer_denote(clang_analyzer_container_end(L), "$L.end()");
 
-  D.push_front(n);
+  L.push_front(n); // expected-note 2{{Container 'L' extended to the front by 1 position}}
 
-  clang_analyzer_express(clang_analyzer_container_begin(D)); // expected-warning{{$D.begin()}} FIXME: Should be $D.begin() - 1 (to correctly track the container's size)
-  clang_analyzer_express(clang_analyzer_container_end(D)); // expected-warning{{$D.end()}}
+  clang_analyzer_express(clang_analyzer_container_begin(L)); // expected-warning{{$L.begin() - 1}}
+                                                             // expected-note@-1{{$L.begin() - 1}}
+  clang_analyzer_express(clang_analyzer_container_end(L)); // expected-warning{{$L.end()}}
+                                                           // expected-note@-1{{$L.end()}}
 }
 
 /// emplace_front()
 ///
-/// Design decision: extends containers to the <-LEFT<- (i.e. the first
+/// Design decision: extends containers to the <-FRONT<- (i.e. the first
 /// position of the container is decremented).
 
-void deque_emplace_front(std::deque<int> &D, int n) {
-  D.cbegin();
-  D.cend();
+void emplace_front(std::list<int> &L, int n) {
+  L.cbegin();
+  L.cend();
 
-  clang_analyzer_denote(clang_analyzer_container_begin(D), "$D.begin()");
-  clang_analyzer_denote(clang_analyzer_container_end(D), "$D.end()");
+  clang_analyzer_denote(clang_analyzer_container_begin(L), "$L.begin()");
+  clang_analyzer_denote(clang_analyzer_container_end(L), "$L.end()");
 
-  D.emplace_front(n);
+  L.emplace_front(n); // expected-note 2{{Container 'L' extended to the front by 1 position}}
 
-  clang_analyzer_express(clang_analyzer_container_begin(D)); // expected-warning{{$D.begin()}} FIXME: Should be $D.begin - 1 (to correctly track the container's size)
-  clang_analyzer_express(clang_analyzer_container_end(D)); // expected-warning{{$D.end()}}
+  clang_analyzer_express(clang_analyzer_container_begin(L)); // expected-warning{{$L.begin() - 1}}
+                                                             // expected-note@-1{{$L.begin() - 1}}
+  clang_analyzer_express(clang_analyzer_container_end(L)); // expected-warning{{$L.end()}}
+                                                           // expected-note@-1{{$L.end()}}
 }
 
 /// pop_front()
 ///
-/// Design decision: shrinks containers to the ->RIGHT-> (i.e. the first
+/// Design decision: shrinks containers to the ->BACK-> (i.e. the first
 /// position of the container is incremented).
 
-void deque_pop_front(std::deque<int> &D, int n) {
-  D.cbegin();
-  D.cend();
+void pop_front(std::list<int> &L, int n) {
+  L.cbegin();
+  L.cend();
+
+  clang_analyzer_denote(clang_analyzer_container_begin(L), "$L.begin()");
+  clang_analyzer_denote(clang_analyzer_container_end(L), "$L.end()");
+
+  L.pop_front(); // expected-note 2{{Container 'L' shrank from the front by 1 position}}
+
+  clang_analyzer_express(clang_analyzer_container_begin(L)); // expected-warning{{$L.begin() + 1}}
+                                                             // expected-note@-1{{$L.begin() + 1}}
+  clang_analyzer_express(clang_analyzer_container_end(L)); // expected-warning{{$L.end()}}
+                                                           // expected-note@-1{{$L.end()}}
+}
+
+////////////////////////////////////////////////////////////////////////////////
+///
+/// O T H E R   T E S T S
+///
+////////////////////////////////////////////////////////////////////////////////
+
+/// Track local variable
+
+void push_back() {
+  std::vector<int> V;
+  V.end();
+  
+  clang_analyzer_denote(clang_analyzer_container_end(V), "$V.end()");
+
+  V.push_back(1); // expected-note{{Container 'V' extended to the back by 1 position}}
+
+  clang_analyzer_express(clang_analyzer_container_end(V)); // expected-warning{{$V.end() + 1}}
+                                                           // expected-note@-1{{$V.end() + 1}}
+}
+
+/// Track the right container only
+
+void push_back1(std::vector<int> &V1, std::vector<int> &V2, int n) {
+  V1.cbegin();
+  V1.cend();
+  V2.cbegin();
+  V2.cend();
 
-  clang_analyzer_denote(clang_analyzer_container_begin(D), "$D.begin()");
-  clang_analyzer_denote(clang_analyzer_container_end(D), "$D.end()");
+  clang_analyzer_denote(clang_analyzer_container_begin(V1), "$V1.begin()");
 
-  D.pop_front();
+  V2.push_back(n); // no-note
 
-  clang_analyzer_express(clang_analyzer_container_begin(D)); // expected-warning{{$D.begin() + 1}}
-  clang_analyzer_express(clang_analyzer_container_end(D)); // expected-warning{{$D.end()}}
+  clang_analyzer_express(clang_analyzer_container_begin(V1)); // expected-warning{{$V1.begin()}}
+                                                              // expected-note@-1{{$V1.begin()}}
 }
 
+void push_back2(std::vector<int> &V1, std::vector<int> &V2, int n) {
+  V1.cbegin();
+  V1.cend();
+  V2.cbegin();
+  V2.cend();
+
+  clang_analyzer_denote(clang_analyzer_container_begin(V1), "$V1.begin()");
+  clang_analyzer_denote(clang_analyzer_container_begin(V2), "$V2.begin()");
+
+  V1.push_back(n); // expected-note{{Container 'V1' extended to the back by 1 position}}
+                   // Only once!
+
+  clang_analyzer_express(clang_analyzer_container_begin(V1)); // expected-warning{{$V1.begin()}}
+                                                              // expected-note@-1{{$V1.begin()}}
+
+  clang_analyzer_express(clang_analyzer_container_begin(V2)); // expected-warning{{$V2.begin()}}
+                                                              // expected-note@-1{{$V2.begin()}}
+}
+
+/// Print Container Data as Part of the Program State
+
 void clang_analyzer_printState();
 
 void print_state(std::vector<int> &V) {
diff --git a/clang/test/Analysis/iterator-modeling-no-aggressive-binary-operation-simplification-no-crash.cpp b/clang/test/Analysis/iterator-modeling-no-aggressive-binary-operation-simplification-no-crash.cpp
new file mode 100644
index 0000000000000..4b7c52db54627
--- /dev/null
+++ b/clang/test/Analysis/iterator-modeling-no-aggressive-binary-operation-simplification-no-crash.cpp
@@ -0,0 +1,15 @@
+// RUN: %clang_analyze_cc1 -std=c++11\
+// RUN: -analyzer-checker=core,cplusplus,debug.DebugIteratorModeling,debug.ExprInspection\
+// RUN: %s 2>&1 | FileCheck %s
+
+// XFAIL: *
+
+// CHECK: checker cannot be enabled with analyzer option 'aggressive-binary-operation-simplification' == false
+
+#include "Inputs/system-header-simulator-cxx.h"
+
+void clang_analyzer_eval(bool);
+
+void comparison(std::vector<int> &V) {
+  clang_analyzer_eval(V.begin() == V.end()); // no-crash
+}
diff --git a/clang/test/Analysis/iterator-modelling.cpp b/clang/test/Analysis/iterator-modeling.cpp
similarity index 100%
rename from clang/test/Analysis/iterator-modelling.cpp
rename to clang/test/Analysis/iterator-modeling.cpp
diff --git a/clang/test/Analysis/iterator-range.cpp b/clang/test/Analysis/iterator-range.cpp
index babcfdec99d66..bdfb04ba3a8ca 100644
--- a/clang/test/Analysis/iterator-range.cpp
+++ b/clang/test/Analysis/iterator-range.cpp
@@ -1,5 +1,5 @@
-// RUN: %clang_analyze_cc1 -std=c++11 -analyzer-checker=core,cplusplus,alpha.cplusplus.IteratorRange -analyzer-config aggressive-binary-operation-simplification=true -analyzer-config c++-container-inlining=false %s -verify
-// RUN: %clang_analyze_cc1 -std=c++11 -analyzer-checker=core,cplusplus,alpha.cplusplus.IteratorRange -analyzer-config aggressive-binary-operation-simplification=true -analyzer-config c++-container-inlining=true -DINLINE=1 %s -verify
+// RUN: %clang_analyze_cc1 -std=c++11 -analyzer-checker=core,cplusplus,alpha.cplusplus.IteratorRange -analyzer-config aggressive-binary-operation-simplification=true -analyzer-config c++-container-inlining=false -analyzer-output=text %s -verify
+// RUN: %clang_analyze_cc1 -std=c++11 -analyzer-checker=core,cplusplus,alpha.cplusplus.IteratorRange -analyzer-config aggressive-binary-operation-simplification=true -analyzer-config c++-container-inlining=true -DINLINE=1 -analyzer-output=text %s -verify
 
 #include "Inputs/system-header-simulator-cxx.h"
 
@@ -32,6 +32,7 @@ void deref_ahead_of_end(const std::vector<int> &V) {
 void deref_end(const std::vector<int> &V) {
   auto i = V.end();
   *i; // expected-warning{{Past-the-end iterator dereferenced}}
+      // expected-note@-1{{Past-the-end iterator dereferenced}}
 }
 
 // Prefix increment - operator++()
@@ -59,6 +60,7 @@ void incr_ahead_of_end(const std::vector<int> &V) {
 void incr_end(const std::vector<int> &V) {
   auto i = V.end();
   ++i; // expected-warning{{Iterator incremented behind the past-the-end iterator}}
+       // expected-note@-1{{Iterator incremented behind the past-the-end iterator}}
 }
 
 // Postfix increment - operator++(int)
@@ -86,6 +88,7 @@ void ahead_of_end_incr(const std::vector<int> &V) {
 void end_incr(const std::vector<int> &V) {
   auto i = V.end();
   i++; // expected-warning{{Iterator incremented behind the past-the-end iterator}}
+       // expected-note@-1{{Iterator incremented behind the past-the-end iterator}}
 }
 
 // Prefix decrement - operator--()
@@ -93,6 +96,7 @@ void end_incr(const std::vector<int> &V) {
 void decr_begin(const std::vector<int> &V) {
   auto i = V.begin();
   --i; // expected-warning{{Iterator decremented ahead of its valid range}}
+       // expected-note@-1{{Iterator decremented ahead of its valid range}}
 }
 
 void decr_behind_begin(const std::vector<int> &V) {
@@ -120,6 +124,7 @@ void decr_end(const std::vector<int> &V) {
 void begin_decr(const std::vector<int> &V) {
   auto i = V.begin();
   i--; // expected-warning{{Iterator decremented ahead of its valid range}}
+       // expected-note@-1{{Iterator decremented ahead of its valid range}}
 }
 
 void behind_begin_decr(const std::vector<int> &V) {
@@ -168,11 +173,13 @@ void incr_by_2_ahead_by_2_of_end(const std::vector<int> &V) {
 void incr_by_2_ahead_of_end(const std::vector<int> &V) {
   auto i = --V.end();
   i += 2; // expected-warning{{Iterator incremented behind the past-the-end iterator}}
+          // expected-note@-1{{Iterator incremented behind the past-the-end iterator}}
 }
 
 void incr_by_2_end(const std::vector<int> &V) {
   auto i = V.end();
   i += 2; // expected-warning{{Iterator incremented behind the past-the-end iterator}}
+          // expected-note@-1{{Iterator incremented behind the past-the-end iterator}}
 }
 
 // Addition - operator+(int)
@@ -201,11 +208,13 @@ void incr_by_2_copy_ahead_by_2_of_end(const std::vector<int> &V) {
 void incr_by_2_copy_ahead_of_end(const std::vector<int> &V) {
   auto i = --V.end();
   auto j = i + 2; // expected-warning{{Iterator incremented behind the past-the-end iterator}}
+                  // expected-note@-1{{Iterator incremented behind the past-the-end iterator}}
 }
 
 void incr_by_2_copy_end(const std::vector<int> &V) {
   auto i = V.end();
   auto j = i + 2; // expected-warning{{Iterator incremented behind the past-the-end iterator}}
+                  // expected-note@-1{{Iterator incremented behind the past-the-end iterator}}
 }
 
 // Subtraction assignment - operator-=(int)
@@ -213,11 +222,13 @@ void incr_by_2_copy_end(const std::vector<int> &V) {
 void decr_by_2_begin(const std::vector<int> &V) {
   auto i = V.begin();
   i -= 2; // expected-warning{{Iterator decremented ahead of its valid range}}
+          // expected-note@-1{{Iterator decremented ahead of its valid range}}
 }
 
 void decr_by_2_behind_begin(const std::vector<int> &V) {
   auto i = ++V.begin();
   i -= 2; // expected-warning{{Iterator decremented ahead of its valid range}}
+          // expected-note@-1{{Iterator decremented ahead of its valid range}}
 }
 
 void decr_by_2_behind_begin_by_2(const std::vector<int> &V) {
@@ -246,11 +257,13 @@ void decr_by_2_end(const std::vector<int> &V) {
 void decr_by_2_copy_begin(const std::vector<int> &V) {
   auto i = V.begin();
   auto j = i - 2; // expected-warning{{Iterator decremented ahead of its valid range}}
+                  // expected-note@-1{{Iterator decremented ahead of its valid range}}
 }
 
 void decr_by_2_copy_behind_begin(const std::vector<int> &V) {
   auto i = ++V.begin();
   auto j = i - 2; // expected-warning{{Iterator decremented ahead of its valid range}}
+                  // expected-note@-1{{Iterator decremented ahead of its valid range}}
 }
 
 void decr_by_2_copy_behind_begin_by_2(const std::vector<int> &V) {
@@ -303,6 +316,7 @@ void subscript_zero_ahead_of_end(const std::vector<int> &V) {
 void subscript_zero_end(const std::vector<int> &V) {
   auto i = V.end();
   auto j = i[0]; // expected-warning{{Past-the-end iterator dereferenced}}
+                 // expected-note@-1{{Past-the-end iterator dereferenced}}
 }
 
 // By negative number
@@ -329,7 +343,8 @@ void subscript_negative_ahead_of_end(const std::vector<int> &V) {
 
 void subscript_negative_end(const std::vector<int> &V) {
   auto i = V.end();
-  auto j = i[-1]; // // expected-warning{{Past-the-end iterator dereferenced}} FIXME: expect no warning
+  auto j = i[-1]; // expected-warning{{Past-the-end iterator dereferenced}} FIXME: expect no warning
+                  // expected-note@-1{{Past-the-end iterator dereferenced}}
 }
 
 // By positive number
@@ -357,6 +372,7 @@ void subscript_positive_ahead_of_end(const std::vector<int> &V) {
 void subscript_positive_end(const std::vector<int> &V) {
   auto i = V.end();
   auto j = i[1]; // expected-warning{{Past-the-end iterator dereferenced}} FIXME: expect warning Iterator incremented behind the past-the-end iterator
+                 // expected-note@-1{{Past-the-end iterator dereferenced}} FIXME: expect note@-1 Iterator incremented behind the past-the-end iterator
 }
 
 //
@@ -388,6 +404,7 @@ void advance_plus_1_ahead_of_end(const std::vector<int> &V) {
 void advance_plus_1_end(const std::vector<int> &V) {
   auto i = V.end();
   std::advance(i, 1); // expected-warning{{Iterator incremented behind the past-the-end iterator}}
+                      // expected-note@-1{{Iterator incremented behind the past-the-end iterator}}
 }
 
 // std::advance() by -1
@@ -395,6 +412,7 @@ void advance_plus_1_end(const std::vector<int> &V) {
 void advance_minus_1_begin(const std::vector<int> &V) {
   auto i = V.begin();
   std::advance(i, -1); // expected-warning{{Iterator decremented ahead of its valid range}}
+                       // expected-note@-1{{Iterator decremented ahead of its valid range}}
 }
 
 void advance_minus_1_behind_begin(const std::vector<int> &V) {
@@ -437,11 +455,13 @@ void advance_plus_2_unknown(const std::vector<int> &V) {
 void advance_plus_2_ahead_of_end(const std::vector<int> &V) {
   auto i = --V.end();
   std::advance(i, 2); // expected-warning{{Iterator incremented behind the past-the-end iterator}}
+                      // expected-note@-1{{Iterator incremented behind the past-the-end iterator}}
 }
 
 void advance_plus_2_end(const std::vector<int> &V) {
   auto i = V.end();
   std::advance(i, 2); // expected-warning{{Iterator incremented behind the past-the-end iterator}}
+                      // expected-note@-1{{Iterator incremented behind the past-the-end iterator}}
 }
 
 // std::advance() by -2
@@ -449,11 +469,13 @@ void advance_plus_2_end(const std::vector<int> &V) {
 void advance_minus_2_begin(const std::vector<int> &V) {
   auto i = V.begin();
   std::advance(i, -2); // expected-warning{{Iterator decremented ahead of its valid range}}
+                       // expected-note@-1{{Iterator decremented ahead of its valid range}}
 }
 
 void advance_minus_2_behind_begin(const std::vector<int> &V) {
   auto i = ++V.begin();
   std::advance(i, -2); // expected-warning{{Iterator decremented ahead of its valid range}}
+                       // expected-note@-1{{Iterator decremented ahead of its valid range}}
 }
 
 void advance_minus_2_unknown(const std::vector<int> &V) {
@@ -527,6 +549,7 @@ void next_plus_1_ahead_of_end(const std::vector<int> &V) {
 void next_plus_1_end(const std::vector<int> &V) {
   auto i = V.end();
   auto j = std::next(i); // expected-warning{{Iterator incremented behind the past-the-end iterator}}
+                         // expected-note@-1{{Iterator incremented behind the past-the-end iterator}}
 }
 
 // std::next() by -1
@@ -534,6 +557,7 @@ void next_plus_1_end(const std::vector<int> &V) {
 void next_minus_1_begin(const std::vector<int> &V) {
   auto i = V.begin();
   auto j = std::next(i, -1); // expected-warning{{Iterator decremented ahead of its valid range}}
+                             // expected-note@-1{{Iterator decremented ahead of its valid range}}
 }
 
 void next_minus_1_behind_begin(const std::vector<int> &V) {
@@ -576,11 +600,13 @@ void next_plus_2_unknown(const std::vector<int> &V) {
 void next_plus_2_ahead_of_end(const std::vector<int> &V) {
   auto i = --V.end();
   auto j = std::next(i, 2); // expected-warning{{Iterator incremented behind the past-the-end iterator}}
+                            // expected-note@-1{{Iterator incremented behind the past-the-end iterator}}
 }
 
 void next_plus_2_end(const std::vector<int> &V) {
   auto i = V.end();
   auto j = std::next(i, 2); // expected-warning{{Iterator incremented behind the past-the-end iterator}}
+                            // expected-note@-1{{Iterator incremented behind the past-the-end iterator}}
 }
 
 // std::next() by -2
@@ -588,11 +614,13 @@ void next_plus_2_end(const std::vector<int> &V) {
 void next_minus_2_begin(const std::vector<int> &V) {
   auto i = V.begin();
   auto j = std::next(i, -2); // expected-warning{{Iterator decremented ahead of its valid range}}
+                             // expected-note@-1{{Iterator decremented ahead of its valid range}}
 }
 
 void next_minus_2_behind_begin(const std::vector<int> &V) {
   auto i = ++V.begin();
   auto j = std::next(i, -2); // expected-warning{{Iterator decremented ahead of its valid range}}
+                             // expected-note@-1{{Iterator decremented ahead of its valid range}}
 }
 
 void next_minus_2_unknown(const std::vector<int> &V) {
@@ -646,6 +674,7 @@ void next_0_end(const std::vector<int> &V) {
 void prev_plus_1_begin(const std::vector<int> &V) {
   auto i = V.begin();
   auto j = std::prev(i); // expected-warning{{Iterator decremented ahead of its valid range}}
+                         // expected-note@-1{{Iterator decremented ahead of its valid range}}
 }
 
 void prev_plus_1_behind_begin(const std::vector<int> &V) {
@@ -693,6 +722,7 @@ void prev_minus_1_ahead_of_end(const std::vector<int> &V) {
 void prev_minus_1_end(const std::vector<int> &V) {
   auto i = V.end();
   auto j = std::prev(i, -1); // expected-warning{{Iterator incremented behind the past-the-end iterator}}
+                             // expected-note@-1{{Iterator incremented behind the past-the-end iterator}}
 }
 
 // std::prev() by +2
@@ -700,11 +730,13 @@ void prev_minus_1_end(const std::vector<int> &V) {
 void prev_plus_2_begin(const std::vector<int> &V) {
   auto i = V.begin();
   auto j = std::prev(i, 2); // expected-warning{{Iterator decremented ahead of its valid range}}
+                            // expected-note@-1{{Iterator decremented ahead of its valid range}}
 }
 
 void prev_plus_2_behind_begin(const std::vector<int> &V) {
   auto i = ++V.begin();
   auto j = std::prev(i, 2); // expected-warning{{Iterator decremented ahead of its valid range}}
+                            // expected-note@-1{{Iterator decremented ahead of its valid range}}
 }
 
 void prev_plus_2_unknown(const std::vector<int> &V) {
@@ -742,11 +774,13 @@ void prev_minus_2_unknown(const std::vector<int> &V) {
 void prev_minus_2_ahead_of_end(const std::vector<int> &V) {
   auto i = --V.end();
   auto j = std::prev(i, -2); // expected-warning{{Iterator incremented behind the past-the-end iterator}}
+                             // expected-note@-1{{Iterator incremented behind the past-the-end iterator}}
 }
 
 void prev_minus_2_end(const std::vector<int> &V) {
   auto i = V.end();
   auto j = std::prev(i, -2); // expected-warning{{Iterator incremented behind the past-the-end iterator}}
+                             // expected-note@-1{{Iterator incremented behind the past-the-end iterator}}
 }
 
 // std::prev() by 0
@@ -793,5 +827,17 @@ void arrow_deref_begin(const std::vector<S> &V) {
 
 void arrow_deref_end(const std::vector<S> &V) {
   auto i = V.end();
-  int n = i->n; //  expected-warning{{Past-the-end iterator dereferenced}}
+  int n = i->n; // expected-warning{{Past-the-end iterator dereferenced}}
+                // expected-note@-1{{Past-the-end iterator dereferenced}}
+}
+
+// Container modification - test path notes
+
+void deref_end_after_pop_back(std::vector<int> &V) {
+  const auto i = --V.end();
+
+  V.pop_back(); // expected-note{{Container 'V' shrank from the back by 1 position}}
+
+  *i; // expected-warning{{Past-the-end iterator dereferenced}}
+      // expected-note@-1{{Past-the-end iterator dereferenced}}
 }
diff --git a/clang/test/Analysis/kmalloc-linux.c b/clang/test/Analysis/kmalloc-linux.c
index ccb6188a51b3d..4cca5712b0967 100644
--- a/clang/test/Analysis/kmalloc-linux.c
+++ b/clang/test/Analysis/kmalloc-linux.c
@@ -1,4 +1,7 @@
-// RUN: %clang_analyze_cc1 -triple x86_64-unknown-linux %s
+// RUN: %clang_analyze_cc1 -triple x86_64-unknown-linux %s -verify \
+// RUN:   -Wno-incompatible-library-redeclaration \
+// RUN:   -analyzer-checker=core \
+// RUN:   -analyzer-checker=unix.Malloc
 
 #define __GFP_ZERO 0x8000
 #define NULL ((void *)0)
@@ -6,6 +9,7 @@
 typedef __typeof(sizeof(int)) size_t;
 
 void *kmalloc(size_t, int);
+void kfree(void *);
 
 struct test {
 };
@@ -61,6 +65,8 @@ typedef unsigned long long uint64_t;
 
 struct malloc_type;
 
+// 3 parameter malloc:
+// https://www.freebsd.org/cgi/man.cgi?query=malloc&sektion=9
 void *malloc(unsigned long size, struct malloc_type *mtp, int flags);
 
 void test_3arg_malloc(struct malloc_type *mtp) {
@@ -97,7 +103,7 @@ void test_3arg_malloc_indeterminate(struct malloc_type *mtp, int flags) {
   struct test **list, *t;
   int i;
 
-  list = alloc(sizeof(*list) * 10, mtp, flags);
+  list = malloc(sizeof(*list) * 10, mtp, flags);
   if (list == NULL)
     return;
 
@@ -107,3 +113,25 @@ void test_3arg_malloc_indeterminate(struct malloc_type *mtp, int flags) {
   }
   kfree(list);
 }
+
+void test_3arg_malloc_leak(struct malloc_type *mtp, int flags) {
+  struct test **list;
+
+  list = malloc(sizeof(*list) * 10, mtp, flags);
+  if (list == NULL)
+    return;
+} // expected-warning{{Potential leak of memory pointed to by 'list'}}
+
+// kmalloc can return a constant value defined in ZERO_SIZE_PTR
+// if a block of size 0 is requested
+#define ZERO_SIZE_PTR ((void *)16)
+
+void test_kfree_ZERO_SIZE_PTR() {
+  void *ptr = ZERO_SIZE_PTR;
+  kfree(ptr); // no warning about freeing this value
+}
+
+void test_kfree_other_constant_value() {
+  void *ptr = (void *)1;
+  kfree(ptr); // expected-warning{{Argument to kfree() is a constant address (1)}}
+}
diff --git a/clang/test/Analysis/loop-widening-notes.cpp b/clang/test/Analysis/loop-widening-notes.cpp
index 2c26a1490e5c2..0ba71d030d058 100644
--- a/clang/test/Analysis/loop-widening-notes.cpp
+++ b/clang/test/Analysis/loop-widening-notes.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_analyze_cc1 -analyzer-checker=core,alpha -analyzer-max-loop 2 -analyzer-config widen-loops=true -analyzer-output=text -verify -analyzer-config eagerly-assume=false %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,alpha.core -analyzer-max-loop 2 -analyzer-config widen-loops=true -analyzer-output=text -verify -analyzer-config eagerly-assume=false %s
 
 int *p_a;
 int bar();
diff --git a/clang/test/Analysis/malloc.cpp b/clang/test/Analysis/malloc.cpp
index 6e5a0e4d59715..cc7fefe23d75a 100644
--- a/clang/test/Analysis/malloc.cpp
+++ b/clang/test/Analysis/malloc.cpp
@@ -164,3 +164,11 @@ void test(A a) {
   (void)a.getName();
 }
 } // namespace argument_leak
+
+#define ZERO_SIZE_PTR ((void *)16)
+
+void test_delete_ZERO_SIZE_PTR() {
+  int *Ptr = (int *)ZERO_SIZE_PTR;
+  // ZERO_SIZE_PTR is specially handled but only for malloc family
+  delete Ptr; // expected-warning{{Argument to 'delete' is a constant address (16)}}
+}
diff --git a/clang/test/Analysis/osobject-retain-release.cpp b/clang/test/Analysis/osobject-retain-release.cpp
index 41606a30c39f4..d88349dcd807e 100644
--- a/clang/test/Analysis/osobject-retain-release.cpp
+++ b/clang/test/Analysis/osobject-retain-release.cpp
@@ -53,6 +53,9 @@ struct MyArray : public OSArray {
   OSObject *generateObject(OSObject *input) override;
 };
 
+// These are never refcounted.
+struct OSSymbol : OSObject {};
+
 struct OtherStruct {
   static void doNothingToArray(OSArray *array);
   OtherStruct(OSArray *arr);
@@ -754,3 +757,10 @@ void test() {
   b(0);
 }
 } // namespace inherited_constructor_crash
+
+namespace ossymbol_suppression {
+OSSymbol *createSymbol();
+void test() {
+  OSSymbol *sym = createSymbol(); // no-warning
+}
+} // namespace ossymbol_suppression
diff --git a/clang/test/Analysis/std-c-library-functions-arg-constraints.c b/clang/test/Analysis/std-c-library-functions-arg-constraints.c
index a20b90ad1ccb1..9753f9eb00cca 100644
--- a/clang/test/Analysis/std-c-library-functions-arg-constraints.c
+++ b/clang/test/Analysis/std-c-library-functions-arg-constraints.c
@@ -3,6 +3,7 @@
 // RUN:   -analyzer-checker=core \
 // RUN:   -analyzer-checker=apiModeling.StdCLibraryFunctions \
 // RUN:   -analyzer-checker=apiModeling.StdCLibraryFunctionArgs \
+// RUN:   -analyzer-checker=debug.StdCLibraryFunctionsTester \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -triple x86_64-unknown-linux-gnu \
 // RUN:   -verify=report
@@ -12,6 +13,7 @@
 // RUN:   -analyzer-checker=core \
 // RUN:   -analyzer-checker=apiModeling.StdCLibraryFunctions \
 // RUN:   -analyzer-checker=apiModeling.StdCLibraryFunctionArgs \
+// RUN:   -analyzer-checker=debug.StdCLibraryFunctionsTester \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -triple x86_64-unknown-linux-gnu \
 // RUN:   -analyzer-output=text \
@@ -85,3 +87,30 @@ void test_notnull_symbolic2(FILE *fp, int *buf) {
     // bugpath-warning{{Function argument constraint is not satisfied}} \
     // bugpath-note{{Function argument constraint is not satisfied}}
 }
+
+int __two_constrained_args(int, int);
+void test_constraints_on_multiple_args(int x, int y) {
+  // State split should not happen here. I.e. x == 1 should not be evaluated
+  // FALSE.
+  __two_constrained_args(x, y);
+  clang_analyzer_eval(x == 1); // \
+  // report-warning{{TRUE}} \
+  // bugpath-warning{{TRUE}} \
+  // bugpath-note{{TRUE}}
+  clang_analyzer_eval(y == 1); // \
+  // report-warning{{TRUE}} \
+  // bugpath-warning{{TRUE}} \
+  // bugpath-note{{TRUE}}
+}
+
+int __arg_constrained_twice(int);
+void test_multiple_constraints_on_same_arg(int x) {
+  __arg_constrained_twice(x);
+  // Check that both constraints are applied and only one branch is there.
+  clang_analyzer_eval(x < 1 || x > 2); // \
+  // report-warning{{TRUE}} \
+  // bugpath-warning{{TRUE}} \
+  // bugpath-note{{TRUE}} \
+  // bugpath-note{{Assuming 'x' is < 1}} \
+  // bugpath-note{{Left side of '||' is true}}
+}
diff --git a/clang/test/Analysis/std-c-library-functions.c b/clang/test/Analysis/std-c-library-functions.c
index 3f700a7c39a45..a275ea6720ade 100644
--- a/clang/test/Analysis/std-c-library-functions.c
+++ b/clang/test/Analysis/std-c-library-functions.c
@@ -89,6 +89,14 @@ void test_fread_fwrite(FILE *fp, int *buf) {
   clang_analyzer_eval(z <= y); // expected-warning{{TRUE}}
 }
 
+void test_fread_uninitialized(void) {
+  void *ptr;
+  size_t sz;
+  size_t nmem;
+  FILE *fp;
+  (void)fread(ptr, sz, nmem, fp); // expected-warning {{1st function call argument is an uninitialized value}}
+}
+
 ssize_t getline(char **, size_t *, FILE *);
 void test_getline(FILE *fp) {
   char *line = 0;
diff --git a/clang/test/CMakeLists.txt b/clang/test/CMakeLists.txt
index 5a09a8fb067a2..21584b6e58aa5 100644
--- a/clang/test/CMakeLists.txt
+++ b/clang/test/CMakeLists.txt
@@ -35,6 +35,16 @@ configure_lit_site_cfg(
   ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
   MAIN_CONFIG
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py
+  PATHS
+  "LLVM_SOURCE_DIR"
+  "LLVM_BINARY_DIR"
+  "LLVM_TOOLS_DIR"
+  "LLVM_LIBS_DIR"
+  "SHLIBDIR"
+  "LLVM_LIT_TOOLS_DIR"
+  "CLANG_BINARY_DIR"
+  "CLANG_SOURCE_DIR"
+  "CLANG_TOOLS_DIR"
   )
 
 configure_lit_site_cfg(
@@ -42,6 +52,13 @@ configure_lit_site_cfg(
   ${CMAKE_CURRENT_BINARY_DIR}/Unit/lit.site.cfg.py
   MAIN_CONFIG
   ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.cfg.py
+  PATHS
+  "LLVM_SOURCE_DIR"
+  "LLVM_BINARY_DIR"
+  "LLVM_TOOLS_DIR"
+  "LLVM_LIBS_DIR"
+  "CLANG_BINARY_DIR"
+  "SHLIBDIR"
   )
 
 option(CLANG_TEST_USE_VG "Run Clang tests under Valgrind" OFF)
@@ -81,6 +98,7 @@ endif ()
 
 if (CLANG_BUILD_EXAMPLES)
   list(APPEND CLANG_TEST_DEPS
+    Attribute
     AnnotateFunctions
     clang-interpreter
     PrintFunctionNames
diff --git a/clang/test/CXX/basic/basic.lookup/basic.lookup.classref/p1-cxx11.cpp b/clang/test/CXX/basic/basic.lookup/basic.lookup.classref/p1-cxx11.cpp
index f812ea1bd8bef..1afea99e8895c 100644
--- a/clang/test/CXX/basic/basic.lookup/basic.lookup.classref/p1-cxx11.cpp
+++ b/clang/test/CXX/basic/basic.lookup/basic.lookup.classref/p1-cxx11.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++11 -fsyntax-only -fdiagnostics-show-option -verify %s
+// RUN: %clang_cc1 -std=c++11 -fsyntax-only -verify %s
 
 template<typename T>
 struct set{};
diff --git a/clang/test/CXX/basic/basic.lookup/basic.lookup.classref/p1.cpp b/clang/test/CXX/basic/basic.lookup/basic.lookup.classref/p1.cpp
index bb6bb73ec7029..e3599db18350b 100644
--- a/clang/test/CXX/basic/basic.lookup/basic.lookup.classref/p1.cpp
+++ b/clang/test/CXX/basic/basic.lookup/basic.lookup.classref/p1.cpp
@@ -1,6 +1,6 @@
-// RUN: %clang_cc1 -fsyntax-only -fdiagnostics-show-option -verify %s
-// RUN: %clang_cc1 -fsyntax-only -fdiagnostics-show-option -verify -std=c++98 %s
-// RUN: %clang_cc1 -fsyntax-only -fdiagnostics-show-option -verify -std=c++11 %s
+// RUN: %clang_cc1 -fsyntax-only -verify %s
+// RUN: %clang_cc1 -fsyntax-only -verify -std=c++98 %s
+// RUN: %clang_cc1 -fsyntax-only -verify -std=c++11 %s
 
 // C++98 [basic.lookup.classref]p1:
 //   In a class member access expression (5.2.5), if the . or -> token is
diff --git a/clang/test/CXX/basic/basic.lookup/basic.lookup.unqual/p3.cpp b/clang/test/CXX/basic/basic.lookup/basic.lookup.unqual/p3.cpp
index ef3b127ef82aa..8ac4e9733ebb3 100644
--- a/clang/test/CXX/basic/basic.lookup/basic.lookup.unqual/p3.cpp
+++ b/clang/test/CXX/basic/basic.lookup/basic.lookup.unqual/p3.cpp
@@ -60,7 +60,7 @@ namespace AnnexD_example {
     A a;
     f < a;
 #if __cplusplus > 201703L
-    // expected-error@-2 {{expected '>'}}
+    // expected-error@-2 {{expected '>'}} expected-note@-2 {{to match this '<'}}
 #endif
     (f) < a;
   }
diff --git a/clang/test/CXX/dcl.decl/dcl.init/dcl.init.ref/p5-cxx03-extra-copy.cpp b/clang/test/CXX/dcl.decl/dcl.init/dcl.init.ref/p5-cxx03-extra-copy.cpp
index 7a5caef36e737..e3190245d240f 100644
--- a/clang/test/CXX/dcl.decl/dcl.init/dcl.init.ref/p5-cxx03-extra-copy.cpp
+++ b/clang/test/CXX/dcl.decl/dcl.init/dcl.init.ref/p5-cxx03-extra-copy.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fsyntax-only -std=c++03 -fdiagnostics-show-option -Wbind-to-temporary-copy -verify %s
+// RUN: %clang_cc1 -fsyntax-only -std=c++03 -Wbind-to-temporary-copy -verify %s
 
 // C++03 requires that we check for a copy constructor when binding a
 // reference to a temporary, since we are allowed to make a copy, Even
diff --git a/clang/test/CXX/drs/dr3xx.cpp b/clang/test/CXX/drs/dr3xx.cpp
index a5edc32fa48c9..a1501fa563b10 100644
--- a/clang/test/CXX/drs/dr3xx.cpp
+++ b/clang/test/CXX/drs/dr3xx.cpp
@@ -20,7 +20,7 @@ namespace dr301 { // dr301: yes
              (void(*)(S, S))operator+<S>;
     bool b = (void(*)(S, S))operator- <
              (void(*)(S, S))operator-;
-    bool c = (void(*)(S, S))operator+ <
+    bool c = (void(*)(S, S))operator+ < // expected-note {{to match this '<'}}
              (void(*)(S, S))operator-; // expected-error {{expected '>'}}
   }
 
@@ -123,7 +123,7 @@ namespace dr305 { // dr305: no
     template<typename T> using T2 = T;
   };
   void k(Z *z) {
-    z->~T1<int>(); // expected-error {{no member named 'T1' in 'dr305::Z'}} expected-error +{{}}
+    z->~T1<int>(); // expected-error {{no member named 'T1' in 'dr305::Z'}}
     z->~T2<int>(); // expected-error {{no member named '~int'}}
     z->~T2<Z>();
   }
diff --git a/clang/test/CXX/expr/expr.prim/expr.prim.req/type-requirement.cpp b/clang/test/CXX/expr/expr.prim/expr.prim.req/type-requirement.cpp
index 71c87ffff8bce..b8903b884e0a3 100644
--- a/clang/test/CXX/expr/expr.prim/expr.prim.req/type-requirement.cpp
+++ b/clang/test/CXX/expr/expr.prim/expr.prim.req/type-requirement.cpp
@@ -63,7 +63,7 @@ template<typename T> struct invalid { typename T::type x; };
 // expected-error@-1 {{typename specifier refers to non-type member 'type' in 'D'}}
 using r1i5 = r1<invalid<D>>;
 // expected-error@-1 {{constraints not satisfied for class template 'r1' [with T = invalid<D>]}}
-// expected-note@-2 {{while checking constraint satisfaction for template 'r1<invalid<D> >' required here}}
+// expected-note@-2 {{while checking constraint satisfaction for template 'r1<invalid<D>>' required here}}
 
 // mismatching template arguments
 
@@ -191,4 +191,4 @@ namespace std_example {
   using c2 = C1_check<has_type>; // expected-error{{constraints not satisfied for class template 'C1_check' [with T = std_example::has_type]}}
   using c3 = C2_check<has_inner>; // expected-error{{constraints not satisfied for class template 'C2_check' [with T = std_example::has_inner]}}
   using c4 = C3_check<void>; // expected-error{{constraints not satisfied for class template 'C3_check' [with T = void]}}
-}
\ No newline at end of file
+}
diff --git a/clang/test/CXX/temp/p3.cpp b/clang/test/CXX/temp/p3.cpp
index e9fd8a3090e86..90f6397857aec 100644
--- a/clang/test/CXX/temp/p3.cpp
+++ b/clang/test/CXX/temp/p3.cpp
@@ -7,9 +7,7 @@ template<typename T> struct S {
 template<typename T> int S<T>::a, S<T>::b; // expected-error {{can only declare a single entity}}
 
 template<typename T> struct A { static A a; } A<T>::a; // expected-error {{expected ';' after struct}} \
-                                                          expected-error {{use of undeclared identifier 'T'}} \
-                                                          expected-error {{no member named 'a'}} \
-                                                          expected-warning {{extra qualification}}
+                                                          expected-error {{use of undeclared identifier 'T'}}
 
 template<typename T> struct B { } f(); // expected-error {{expected ';' after struct}} \
                                           expected-error {{requires a type specifier}}
diff --git a/clang/test/CXX/temp/temp.arg/temp.arg.template/p3-0x.cpp b/clang/test/CXX/temp/temp.arg/temp.arg.template/p3-0x.cpp
index 1c13bffa212f7..0c555b53f95e6 100644
--- a/clang/test/CXX/temp/temp.arg/temp.arg.template/p3-0x.cpp
+++ b/clang/test/CXX/temp/temp.arg/temp.arg.template/p3-0x.cpp
@@ -13,9 +13,9 @@ template <class T1, class T2, int N = 17> struct E;
 
 eval<A<int>> eA;
 eval<B<int, float>> eB;
-eval<C<17>> eC; // expected-error{{implicit instantiation of undefined template 'eval<C<17> >'}}
-eval<D<int, 17>> eD; // expected-error{{implicit instantiation of undefined template 'eval<D<int, 17> >'}}
-eval<E<int, float>> eE; // expected-error{{implicit instantiation of undefined template 'eval<E<int, float, 17> >}}
+eval<C<17>> eC; // expected-error{{implicit instantiation of undefined template 'eval<C<17>>'}}
+eval<D<int, 17>> eD; // expected-error{{implicit instantiation of undefined template 'eval<D<int, 17>>'}}
+eval<E<int, float>> eE; // expected-error{{implicit instantiation of undefined template 'eval<E<int, float, 17>>}}
 
 template<template <int ...N> class TT> struct X0 { }; // expected-note{{previous non-type template parameter with type 'int' is here}}
 template<int I, int J, int ...Rest> struct X0a;
diff --git a/clang/test/CodeCompletion/concepts.cpp b/clang/test/CodeCompletion/concepts.cpp
new file mode 100644
index 0000000000000..64aad240c1ecf
--- /dev/null
+++ b/clang/test/CodeCompletion/concepts.cpp
@@ -0,0 +1,59 @@
+template <typename T, typename U> concept convertible_to = true;
+template <typename T, typename U> concept same_as = true;
+template <typename T> concept integral = true;
+
+template <typename A, typename B>
+concept W = requires(A a, B b) {
+  { b.www } noexcept -> integral;
+};
+
+template <typename T> concept X = requires(T t) {
+  t.xxx(42);
+  typename T::xxx_t;
+  T::xyz::member;
+};
+
+template <typename T, typename U>
+concept Y = requires(T t, U u) { t.yyy(u); };
+
+template <typename T>
+concept Z = requires(T t) {
+  { t.zzz() } -> same_as<int>;
+  requires W<int, T>;
+};
+
+// Concept constraints in all three slots require X, Y, Z, and ad-hoc stuff.
+template <X T>
+requires Y<T, int> && requires(T *t) { { t->aaa() } -> convertible_to<double>; }
+void foo(T t) requires Z<T> || requires(T &t) { t.bbb(); t->bb(); } {
+  t.x;
+  t->x;
+  T::x;
+
+  // RUN: %clang_cc1 -std=c++2a -code-completion-with-fixits -code-completion-at=%s:29:5 %s \
+  // RUN: | FileCheck %s -check-prefix=DOT -implicit-check-not=xxx_t
+  // DOT: Pattern : [#convertible_to<double>#]aaa()
+  // DOT: Pattern : bb() (requires fix-it: {{.*}} to "->")
+  // DOT: Pattern : bbb()
+  // DOT: Pattern : [#integral#]www
+  // DOT: Pattern : xxx(<#int#>)
+  // FIXME: it would be nice to have int instead of U here.
+  // DOT: Pattern : yyy(<#U#>)
+  // DOT: Pattern : [#int#]zzz()
+
+  // RUN: %clang_cc1 -std=c++2a -code-completion-with-fixits -code-completion-at=%s:30:6 %s \
+  // RUN: | FileCheck %s -check-prefix=ARROW -implicit-check-not=xxx_t
+  // ARROW: Pattern : [#convertible_to<double>#]aaa() (requires fix-it: {{.*}} to ".")
+  // ARROW: Pattern : bb()
+  // ARROW: Pattern : bbb() (requires fix-it
+  // ARROW: Pattern : [#integral#]www (requires fix-it
+  // ARROW: Pattern : xxx(<#int#>) (requires fix-it
+  // ARROW: Pattern : yyy(<#U#>) (requires fix-it
+  // ARROW: Pattern : [#int#]zzz() (requires fix-it
+
+  // RUN: %clang_cc1 -std=c++2a -code-completion-with-fixits -code-completion-at=%s:31:6 %s \
+  // RUN: | FileCheck %s -check-prefix=COLONS -implicit-check-not=yyy
+  // COLONS: Pattern : xxx_t
+  // COLONS: Pattern : xyz
+}
+
diff --git a/clang/test/CodeGen/arm-cde-reinterpret.c b/clang/test/CodeGen/arm-cde-reinterpret.c
index 569b51bdfdbe8..c169a88bc4e39 100644
--- a/clang/test/CodeGen/arm-cde-reinterpret.c
+++ b/clang/test/CodeGen/arm-cde-reinterpret.c
@@ -1,8 +1,11 @@
-// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi \
 // RUN:   -target-feature +cdecp0 -target-feature +mve.fp \
 // RUN:   -mfloat-abi hard -O0 -disable-O0-optnone \
-// RUN:   -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s
+// RUN:   -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s --check-prefixes=CHECK,CHECK-LE
+// RUN: %clang_cc1 -triple thumbebv8.1m.main-arm-none-eabi \
+// RUN:   -target-feature +cdecp0 -target-feature +mve.fp \
+// RUN:   -mfloat-abi hard -O0 -disable-O0-optnone \
+// RUN:   -S -emit-llvm -o - %s | opt -S -mem2reg | FileCheck %s --check-prefixes=CHECK,CHECK-BE
 
 #include <arm_cde.h>
 
@@ -16,7 +19,8 @@ int8x16_t test_s8(uint8x16_t x) {
 
 // CHECK-LABEL: @test_u16(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[X:%.*]] to <8 x i16>
+// CHECK-LE-NEXT: [[TMP0:%.*]] = bitcast <16 x i8> [[X:%.*]] to <8 x i16>
+// CHECK-BE-NEXT: [[TMP0:%.*]] = call <8 x i16> @llvm.arm.mve.vreinterpretq.v8i16.v16i8(<16 x i8> [[X:%.*]])
 // CHECK-NEXT:    ret <8 x i16> [[TMP0]]
 //
 uint16x8_t test_u16(uint8x16_t x) {
@@ -25,7 +29,8 @@ uint16x8_t test_u16(uint8x16_t x) {
 
 // CHECK-LABEL: @test_s32(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[X:%.*]] to <4 x i32>
+// CHECK-LE-NEXT: [[TMP0:%.*]] = bitcast <16 x i8> [[X:%.*]] to <4 x i32>
+// CHECK-BE-NEXT: [[TMP0:%.*]] = call <4 x i32> @llvm.arm.mve.vreinterpretq.v4i32.v16i8(<16 x i8> [[X:%.*]])
 // CHECK-NEXT:    ret <4 x i32> [[TMP0]]
 //
 int32x4_t test_s32(uint8x16_t x) {
@@ -34,7 +39,8 @@ int32x4_t test_s32(uint8x16_t x) {
 
 // CHECK-LABEL: @test_u32(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[X:%.*]] to <4 x i32>
+// CHECK-LE-NEXT: [[TMP0:%.*]] = bitcast <16 x i8> [[X:%.*]] to <4 x i32>
+// CHECK-BE-NEXT: [[TMP0:%.*]] = call <4 x i32> @llvm.arm.mve.vreinterpretq.v4i32.v16i8(<16 x i8> [[X:%.*]])
 // CHECK-NEXT:    ret <4 x i32> [[TMP0]]
 //
 uint32x4_t test_u32(uint8x16_t x) {
@@ -43,7 +49,8 @@ uint32x4_t test_u32(uint8x16_t x) {
 
 // CHECK-LABEL: @test_s64(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[X:%.*]] to <2 x i64>
+// CHECK-LE-NEXT: [[TMP0:%.*]] = bitcast <16 x i8> [[X:%.*]] to <2 x i64>
+// CHECK-BE-NEXT: [[TMP0:%.*]] = call <2 x i64> @llvm.arm.mve.vreinterpretq.v2i64.v16i8(<16 x i8> [[X:%.*]])
 // CHECK-NEXT:    ret <2 x i64> [[TMP0]]
 //
 int64x2_t test_s64(uint8x16_t x) {
@@ -52,7 +59,8 @@ int64x2_t test_s64(uint8x16_t x) {
 
 // CHECK-LABEL: @test_u64(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[X:%.*]] to <2 x i64>
+// CHECK-LE-NEXT: [[TMP0:%.*]] = bitcast <16 x i8> [[X:%.*]] to <2 x i64>
+// CHECK-BE-NEXT: [[TMP0:%.*]] = call <2 x i64> @llvm.arm.mve.vreinterpretq.v2i64.v16i8(<16 x i8> [[X:%.*]])
 // CHECK-NEXT:    ret <2 x i64> [[TMP0]]
 //
 uint64x2_t test_u64(uint8x16_t x) {
@@ -61,7 +69,8 @@ uint64x2_t test_u64(uint8x16_t x) {
 
 // CHECK-LABEL: @test_f16(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[X:%.*]] to <8 x half>
+// CHECK-LE-NEXT: [[TMP0:%.*]] = bitcast <16 x i8> [[X:%.*]] to <8 x half>
+// CHECK-BE-NEXT: [[TMP0:%.*]] = call <8 x half> @llvm.arm.mve.vreinterpretq.v8f16.v16i8(<16 x i8> [[X:%.*]])
 // CHECK-NEXT:    ret <8 x half> [[TMP0]]
 //
 float16x8_t test_f16(uint8x16_t x) {
@@ -70,7 +79,8 @@ float16x8_t test_f16(uint8x16_t x) {
 
 // CHECK-LABEL: @test_f32(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <16 x i8> [[X:%.*]] to <4 x float>
+// CHECK-LE-NEXT: [[TMP0:%.*]] = bitcast <16 x i8> [[X:%.*]] to <4 x float>
+// CHECK-BE-NEXT: [[TMP0:%.*]] = call <4 x float> @llvm.arm.mve.vreinterpretq.v4f32.v16i8(<16 x i8> [[X:%.*]])
 // CHECK-NEXT:    ret <4 x float> [[TMP0]]
 //
 float32x4_t test_f32(uint8x16_t x) {
diff --git a/clang/test/CodeGen/arm-cde-vec.c b/clang/test/CodeGen/arm-cde-vec.c
index fcf9270d87f06..ea186a46be005 100644
--- a/clang/test/CodeGen/arm-cde-vec.c
+++ b/clang/test/CodeGen/arm-cde-vec.c
@@ -102,3 +102,75 @@ uint64x2_t test_vcx3q(uint64x2_t n, float32x4_t m) {
 int8x16_t test_vcx3qa(int8x16_t acc, uint16x8_t n, float32x4_t m) {
   return __arm_vcx3qa(1, acc, n, m, 13);
 }
+
+// CHECK-LABEL: @test_vcx1q_m(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i16> @llvm.arm.cde.vcx1q.predicated.v8i16.v8i1(i32 0, <8 x i16> [[INACTIVE:%.*]], i32 1111, <8 x i1> [[TMP1]])
+// CHECK-NEXT:    ret <8 x i16> [[TMP2]]
+//
+uint16x8_t test_vcx1q_m(uint16x8_t inactive, mve_pred16_t p) {
+  return __arm_vcx1q_m(0, inactive, 1111, p);
+}
+
+// CHECK-LABEL: @test_vcx1qa_m(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i8> @llvm.arm.cde.vcx1qa.predicated.v16i8.v16i1(i32 1, <16 x i8> [[ACC:%.*]], i32 1112, <16 x i1> [[TMP1]])
+// CHECK-NEXT:    ret <16 x i8> [[TMP2]]
+//
+uint8x16_t test_vcx1qa_m(uint8x16_t acc, mve_pred16_t p) {
+  return __arm_vcx1qa_m(1, acc, 1112, p);
+}
+
+// CHECK-LABEL: @test_vcx2q_m(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x float> [[N:%.*]] to <16 x i8>
+// CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i32> @llvm.arm.cde.vcx2q.predicated.v4i32.v4i1(i32 0, <4 x i32> [[INACTIVE:%.*]], <16 x i8> [[TMP0]], i32 111, <4 x i1> [[TMP2]])
+// CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+//
+int32x4_t test_vcx2q_m(int32x4_t inactive, float32x4_t n, mve_pred16_t p) {
+  return __arm_vcx2q_m(0, inactive, n, 111, p);
+}
+
+// CHECK-LABEL: @test_vcx2qa_m(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[N:%.*]] to <16 x i8>
+// CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = call <4 x float> @llvm.arm.cde.vcx2qa.predicated.v4f32.v4i1(i32 0, <4 x float> [[ACC:%.*]], <16 x i8> [[TMP0]], i32 112, <4 x i1> [[TMP2]])
+// CHECK-NEXT:    ret <4 x float> [[TMP3]]
+//
+float32x4_t test_vcx2qa_m(float32x4_t acc, float16x8_t n, mve_pred16_t p) {
+  return __arm_vcx2qa_m(0, acc, n, 112, p);
+}
+
+// CHECK-LABEL: @test_vcx3q_m(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x float> [[N:%.*]] to <16 x i8>
+// CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
+// CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i64> @llvm.arm.cde.vcx3q.predicated.v2i64.v4i1(i32 1, <2 x i64> [[INACTIVE:%.*]], <16 x i8> [[TMP0]], <16 x i8> [[M:%.*]], i32 11, <4 x i1> [[TMP2]])
+// CHECK-NEXT:    ret <2 x i64> [[TMP3]]
+//
+int64x2_t test_vcx3q_m(int64x2_t inactive, float32x4_t n, int8x16_t m, mve_pred16_t p) {
+  return __arm_vcx3q_m(1, inactive, n, m, 11, p);
+}
+
+// CHECK-LABEL: @test_vcx3qa_m(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[N:%.*]] to <16 x i8>
+// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[M:%.*]] to <16 x i8>
+// CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[P:%.*]] to i32
+// CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP2]])
+// CHECK-NEXT:    [[TMP4:%.*]] = call <4 x float> @llvm.arm.cde.vcx3qa.predicated.v4f32.v4i1(i32 1, <4 x float> [[INACTIVE:%.*]], <16 x i8> [[TMP0]], <16 x i8> [[TMP1]], i32 12, <4 x i1> [[TMP3]])
+// CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x float> [[TMP4]] to <8 x half>
+// CHECK-NEXT:    ret <8 x half> [[TMP5]]
+//
+float16x8_t test_vcx3qa_m(float32x4_t inactive, float16x8_t n, uint32x4_t m, mve_pred16_t p) {
+  return __arm_vcx3qa_m(1, inactive, n, m, 12, p);
+}
diff --git a/clang/test/CodeGen/avx512cdintrin.c b/clang/test/CodeGen/avx512cdintrin.c
index 6483d7e8dda56..b15126808b801 100644
--- a/clang/test/CodeGen/avx512cdintrin.c
+++ b/clang/test/CodeGen/avx512cdintrin.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512cd -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512cd -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 
 #include <immintrin.h>
diff --git a/clang/test/CodeGen/avx512vbmivl-builtin.c b/clang/test/CodeGen/avx512vbmivl-builtin.c
index 0bf9165f6c6af..2562da26d6ce7 100644
--- a/clang/test/CodeGen/avx512vbmivl-builtin.c
+++ b/clang/test/CodeGen/avx512vbmivl-builtin.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vbmi -target-feature +avx512vl -target-feature +avx512bw -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vbmi -target-feature +avx512vl -target-feature +avx512bw -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 
 #include <immintrin.h>
diff --git a/clang/test/CodeGen/avx512vl-builtins-constrained-cmp.c b/clang/test/CodeGen/avx512vl-builtins-constrained-cmp.c
index ae052dfffbab7..6be4f614561a6 100644
--- a/clang/test/CodeGen/avx512vl-builtins-constrained-cmp.c
+++ b/clang/test/CodeGen/avx512vl-builtins-constrained-cmp.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -ffreestanding %s -fexperimental-new-pass-manager -triple=x86_64-apple-darwin -target-feature +avx512f -target-feature +avx512vl -emit-llvm -ffp-exception-behavior=strict -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -fexperimental-new-pass-manager -triple=x86_64-apple-darwin -target-feature +avx512f -target-feature +avx512vl -emit-llvm -ffp-exception-behavior=strict -o - -Wall -Werror | FileCheck %s
 
 #include <immintrin.h>
 
diff --git a/clang/test/CodeGen/avx512vpopcntdqintrin.c b/clang/test/CodeGen/avx512vpopcntdqintrin.c
index e7c797c195499..22c44bc21edfc 100644
--- a/clang/test/CodeGen/avx512vpopcntdqintrin.c
+++ b/clang/test/CodeGen/avx512vpopcntdqintrin.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vpopcntdq -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vpopcntdq -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 #include <immintrin.h>
 
diff --git a/clang/test/CodeGen/avx512vpopcntdqvlintrin.c b/clang/test/CodeGen/avx512vpopcntdqvlintrin.c
index 010cb6b4f3447..686bfb986ceb3 100644
--- a/clang/test/CodeGen/avx512vpopcntdqvlintrin.c
+++ b/clang/test/CodeGen/avx512vpopcntdqvlintrin.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vpopcntdq -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vpopcntdq -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 #include <immintrin.h>
 
diff --git a/clang/test/CodeGen/fixed-point-tbaa.c b/clang/test/CodeGen/fixed-point-tbaa.c
new file mode 100644
index 0000000000000..0698603316731
--- /dev/null
+++ b/clang/test/CodeGen/fixed-point-tbaa.c
@@ -0,0 +1,109 @@
+// RUN: %clang_cc1 -triple x86_64-apple-darwin -O1 -ffixed-point %s -emit-llvm -o - | FileCheck %s  -check-prefixes=CHECK
+//
+// Check that we generate correct TBAA metadata for fixed-point types.
+
+void sfract(unsigned short _Fract *p, short _Fract *q,
+            unsigned _Sat short _Fract *r, _Sat short _Fract *s) {
+  // CHECK-LABEL: define void @sfract
+  // CHECK: store i8 -128, i8* %p, align 1, !tbaa [[TAG_sf:!.*]]
+  // CHECK: store i8 -64, i8* %q, align 1, !tbaa [[TAG_sf]]
+  // CHECK: store i8 -128, i8* %r, align 1, !tbaa [[TAG_sat_sf:!.*]]
+  // CHECK: store i8 -64, i8* %s, align 1, !tbaa [[TAG_sat_sf]]
+  *p = 0.5hur;
+  *q = -0.5hr;
+  *r = 0.5hur;
+  *s = -0.5hr;
+}
+
+void fract(unsigned _Fract *p, _Fract *q,
+           unsigned _Sat _Fract *r, _Sat _Fract *s) {
+  // CHECK-LABEL: define void @fract
+  // CHECK: store i16 -32768, i16* %p, align 2, !tbaa [[TAG_f:!.*]]
+  // CHECK: store i16 -16384, i16* %q, align 2, !tbaa [[TAG_f]]
+  // CHECK: store i16 -32768, i16* %r, align 2, !tbaa [[TAG_sat_f:!.*]]
+  // CHECK: store i16 -16384, i16* %s, align 2, !tbaa [[TAG_sat_f]]
+  *p = 0.5ur;
+  *q = -0.5r;
+  *r = 0.5ur;
+  *s = -0.5r;
+}
+
+void lfract(unsigned long _Fract *p, long _Fract *q,
+            unsigned _Sat long _Fract *r, _Sat long _Fract *s) {
+  // CHECK-LABEL: define void @lfract
+  // CHECK: store i32 -2147483648, i32* %p, align 4, !tbaa [[TAG_lf:!.*]]
+  // CHECK: store i32 -1073741824, i32* %q, align 4, !tbaa [[TAG_lf]]
+  // CHECK: store i32 -2147483648, i32* %r, align 4, !tbaa [[TAG_sat_lf:!.*]]
+  // CHECK: store i32 -1073741824, i32* %s, align 4, !tbaa [[TAG_sat_lf]]
+  *p = 0.5ulr;
+  *q = -0.5lr;
+  *r = 0.5ulr;
+  *s = -0.5lr;
+}
+
+void saccum(unsigned short _Accum *p, short _Accum *q,
+            unsigned _Sat short _Accum *r, _Sat short _Accum *s) {
+  // CHECK-LABEL: define void @saccum
+  // CHECK: store i16 128, i16* %p, align 2, !tbaa [[TAG_sk:!.*]]
+  // CHECK: store i16 -64, i16* %q, align 2, !tbaa [[TAG_sk]]
+  // CHECK: store i16 128, i16* %r, align 2, !tbaa [[TAG_sat_sk:!.*]]
+  // CHECK: store i16 -64, i16* %s, align 2, !tbaa [[TAG_sat_sk]]
+  *p = 0.5huk;
+  *q = -0.5hk;
+  *r = 0.5huk;
+  *s = -0.5hk;
+}
+
+void accum(unsigned _Accum *p, _Accum *q,
+           unsigned _Sat _Accum *r, _Sat _Accum *s) {
+  // CHECK-LABEL: define void @accum
+  // CHECK: store i32 32768, i32* %p, align 4, !tbaa [[TAG_k:!.*]]
+  // CHECK: store i32 -16384, i32* %q, align 4, !tbaa [[TAG_k]]
+  // CHECK: store i32 32768, i32* %r, align 4, !tbaa [[TAG_sat_k:!.*]]
+  // CHECK: store i32 -16384, i32* %s, align 4, !tbaa [[TAG_sat_k]]
+  *p = 0.5uk;
+  *q = -0.5k;
+  *r = 0.5uk;
+  *s = -0.5k;
+}
+
+void laccum(unsigned long _Accum *p, long _Accum *q,
+            unsigned _Sat long _Accum *r, _Sat long _Accum *s) {
+  // CHECK-LABEL: define void @laccum
+  // CHECK: store i64 2147483648, i64* %p, align 8, !tbaa [[TAG_lk:!.*]]
+  // CHECK: store i64 -1073741824, i64* %q, align 8, !tbaa [[TAG_lk]]
+  // CHECK: store i64 2147483648, i64* %r, align 8, !tbaa [[TAG_sat_lk:!.*]]
+  // CHECK: store i64 -1073741824, i64* %s, align 8, !tbaa [[TAG_sat_lk]]
+  *p = 0.5ulk;
+  *q = -0.5lk;
+  *r = 0.5ulk;
+  *s = -0.5lk;
+}
+
+// CHECK-DAG: [[TAG_sf]] = !{[[TYPE_sf:!.*]], [[TYPE_sf]], i64 0}
+// CHECK-DAG: [[TYPE_sf]] = !{!"short _Fract"
+// CHECK-DAG: [[TAG_f]] = !{[[TYPE_f:!.*]], [[TYPE_f]], i64 0}
+// CHECK-DAG: [[TYPE_f]] = !{!"_Fract"
+// CHECK-DAG: [[TAG_lf]] = !{[[TYPE_lf:!.*]], [[TYPE_lf]], i64 0}
+// CHECK-DAG: [[TYPE_lf]] = !{!"long _Fract"
+
+// CHECK-DAG: [[TAG_sat_sf]] = !{[[TYPE_sat_sf:!.*]], [[TYPE_sat_sf]], i64 0}
+// CHECK-DAG: [[TYPE_sat_sf]] = !{!"_Sat short _Fract"
+// CHECK-DAG: [[TAG_sat_f]] = !{[[TYPE_sat_f:!.*]], [[TYPE_sat_f]], i64 0}
+// CHECK-DAG: [[TYPE_sat_f]] = !{!"_Sat _Fract"
+// CHECK-DAG: [[TAG_sat_lf]] = !{[[TYPE_sat_lf:!.*]], [[TYPE_sat_lf]], i64 0}
+// CHECK-DAG: [[TYPE_sat_lf]] = !{!"_Sat long _Fract"
+
+// CHECK-DAG: [[TAG_sk]] = !{[[TYPE_sk:!.*]], [[TYPE_sk]], i64 0}
+// CHECK-DAG: [[TYPE_sk]] = !{!"short _Accum"
+// CHECK-DAG: [[TAG_k]] = !{[[TYPE_k:!.*]], [[TYPE_k]], i64 0}
+// CHECK-DAG: [[TYPE_k]] = !{!"_Accum"
+// CHECK-DAG: [[TAG_lk]] = !{[[TYPE_lk:!.*]], [[TYPE_lk]], i64 0}
+// CHECK-DAG: [[TYPE_lk]] = !{!"long _Accum"
+
+// CHECK-DAG: [[TAG_sat_sk]] = !{[[TYPE_sat_sk:!.*]], [[TYPE_sat_sk]], i64 0}
+// CHECK-DAG: [[TYPE_sat_sk]] = !{!"_Sat short _Accum"
+// CHECK-DAG: [[TAG_sat_k]] = !{[[TYPE_sat_k:!.*]], [[TYPE_sat_k]], i64 0}
+// CHECK-DAG: [[TYPE_sat_k]] = !{!"_Sat _Accum"
+// CHECK-DAG: [[TAG_sat_lk]] = !{[[TYPE_sat_lk:!.*]], [[TYPE_sat_lk]], i64 0}
+// CHECK-DAG: [[TYPE_sat_lk]] = !{!"_Sat long _Accum"
diff --git a/clang/test/CodeGen/gfni-builtins.c b/clang/test/CodeGen/gfni-builtins.c
index 0d10d7aeacd6a..61d957d59e768 100644
--- a/clang/test/CodeGen/gfni-builtins.c
+++ b/clang/test/CodeGen/gfni-builtins.c
@@ -1,6 +1,6 @@
-// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +gfni -emit-llvm -o - | FileCheck %s --check-prefix SSE
-// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +gfni -target-feature +avx -emit-llvm -o - | FileCheck %s --check-prefixes SSE,AVX
-// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +gfni -target-feature +avx512bw -target-feature +avx512vl -emit-llvm -o - | FileCheck %s --check-prefixes SSE,AVX,AVX512
+// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +gfni -emit-llvm -o - | FileCheck %s --check-prefix SSE
+// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +gfni -target-feature +avx -emit-llvm -o - | FileCheck %s --check-prefixes SSE,AVX
+// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +gfni -target-feature +avx512bw -target-feature +avx512vl -emit-llvm -o - | FileCheck %s --check-prefixes SSE,AVX,AVX512
 
 #include <immintrin.h>
 
diff --git a/clang/test/CodeGen/hexagon-linux-vararg.c b/clang/test/CodeGen/hexagon-linux-vararg.c
new file mode 100644
index 0000000000000..53c716c517449
--- /dev/null
+++ b/clang/test/CodeGen/hexagon-linux-vararg.c
@@ -0,0 +1,81 @@
+// REQUIRES: hexagon-registered-target
+// RUN: %clang_cc1 -emit-llvm -triple hexagon-unknown-linux-musl %s -o - | FileCheck %s
+#include <stdarg.h>
+
+struct AAA {
+  int x;
+  int y;
+  int z;
+  int d;
+};
+
+// CHECK:   call void @llvm.va_start(i8* %arraydecay1)
+// CHECK:   %arraydecay2 = getelementptr inbounds [1 x %struct.__va_list_tag],
+// [1 x %struct.__va_list_tag]* %ap, i32 0, i32 0
+// CHECK:   br label %vaarg.maybe_reg
+
+// CHECK: vaarg.maybe_reg:                                  ; preds = %entry
+// CHECK:   %__current_saved_reg_area_pointer_p = getelementptr inbounds
+// %struct.__va_list_tag, %struct.__va_list_tag* %arraydecay2, i32 0, i32 0
+// CHECK:   %__current_saved_reg_area_pointer = load i8*, i8**
+// %__current_saved_reg_area_pointer_p
+// CHECK:   %__saved_reg_area_end_pointer_p = getelementptr inbounds
+// %struct.__va_list_tag, %struct.__va_list_tag* %arraydecay2, i32 0, i32 1
+// CHECK:   %__saved_reg_area_end_pointer = load i8*, i8**
+// %__saved_reg_area_end_pointer_p
+// CHECK:   %__new_saved_reg_area_pointer = getelementptr i8, i8*
+// %__current_saved_reg_area_pointer, i32 4
+// CHECK:   %0 = icmp sgt i8* %__new_saved_reg_area_pointer,
+// %__saved_reg_area_end_pointer
+// CHECK:   br i1 %0, label %vaarg.on_stack, label %vaarg.in_reg
+
+// CHECK: vaarg.in_reg:                                     ; preds =
+// %vaarg.maybe_reg
+// CHECK:   %1 = bitcast i8* %__current_saved_reg_area_pointer to i32*
+// CHECK:   store i8* %__new_saved_reg_area_pointer, i8**
+// %__current_saved_reg_area_pointer_p
+// CHECK:   br label %vaarg.end
+
+// CHECK: vaarg.on_stack:                                   ; preds =
+// %vaarg.maybe_reg
+// CHECK:   %__overflow_area_pointer_p = getelementptr inbounds
+// %struct.__va_list_tag, %struct.__va_list_tag* %arraydecay2, i32 0, i32 2
+// CHECK:   %__overflow_area_pointer = load i8*, i8** %__overflow_area_pointer_p
+// CHECK:   %__overflow_area_pointer.next = getelementptr i8, i8*
+// %__overflow_area_pointer, i32 4
+// CHECK:   store i8* %__overflow_area_pointer.next, i8**
+// %__overflow_area_pointer_p
+// CHECK:   store i8* %__overflow_area_pointer.next, i8**
+// %__current_saved_reg_area_pointer_p
+// CHECK:   %2 = bitcast i8* %__overflow_area_pointer to i32*
+// CHECK:   br label %vaarg.end
+
+// CHECK: vaarg.end:                                        ; preds =
+// %vaarg.on_stack, %vaarg.in_reg
+// CHECK:   %vaarg.addr = phi i32* [ %1, %vaarg.in_reg ], [ %2, %vaarg.on_stack
+// ]
+// CHECK:   %3 = load i32, i32* %vaarg.addr
+
+struct AAA aaa = {100, 200, 300, 400};
+
+int foo(int xx, ...) {
+  va_list ap;
+  int d;
+  int ret = 0;
+  struct AAA bbb;
+  va_start(ap, xx);
+  d = va_arg(ap, int);
+  ret += d;
+  bbb = va_arg(ap, struct AAA);
+  ret += bbb.d;
+  d = va_arg(ap, int);
+  ret += d;
+  va_end(ap);
+  return ret;
+}
+
+int main(void) {
+  int x;
+  x = foo(1, 2, aaa, 4);
+  return x;
+}
diff --git a/clang/test/CodeGen/intel-avx512vlvp2intersect.c b/clang/test/CodeGen/intel-avx512vlvp2intersect.c
index c607a6996928e..1b360d1d7c5ba 100644
--- a/clang/test/CodeGen/intel-avx512vlvp2intersect.c
+++ b/clang/test/CodeGen/intel-avx512vlvp2intersect.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +avx512vp2intersect -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
-// RUN: %clang_cc1 %s -ffreestanding -triple=i386-unknown-unknown -target-feature +avx512vp2intersect -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 %s -flax-vector-conversions=none -ffreestanding -triple=x86_64-unknown-unknown -target-feature +avx512vp2intersect -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 %s -flax-vector-conversions=none -ffreestanding -triple=i386-unknown-unknown -target-feature +avx512vp2intersect -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 #include <immintrin.h>
 
diff --git a/clang/test/CodeGen/intel-avx512vp2intersect.c b/clang/test/CodeGen/intel-avx512vp2intersect.c
index bcbf6076eec33..2a3d38acfbf2c 100644
--- a/clang/test/CodeGen/intel-avx512vp2intersect.c
+++ b/clang/test/CodeGen/intel-avx512vp2intersect.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +avx512vp2intersect -emit-llvm -o - -Wall -Werror | FileCheck %s
-// RUN: %clang_cc1 %s -ffreestanding -triple=i386-unknown-unknown -target-feature +avx512vp2intersect -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 %s -flax-vector-conversions=none -ffreestanding -triple=x86_64-unknown-unknown -target-feature +avx512vp2intersect -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 %s -flax-vector-conversions=none -ffreestanding -triple=i386-unknown-unknown -target-feature +avx512vp2intersect -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 #include <immintrin.h>
 
diff --git a/clang/test/CodeGen/target-data.c b/clang/test/CodeGen/target-data.c
index e49f8453e360e..e619843f4bdbd 100644
--- a/clang/test/CodeGen/target-data.c
+++ b/clang/test/CodeGen/target-data.c
@@ -245,8 +245,8 @@
 
 // RUN: %clang_cc1 -triple bpfel -o - -emit-llvm %s | \
 // RUN: FileCheck %s -check-prefix=BPFEL
-// BPFEL: target datalayout = "e-m:e-p:64:64-i64:64-n32:64-S128"
+// BPFEL: target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"
 
 // RUN: %clang_cc1 -triple bpfeb -o - -emit-llvm %s | \
 // RUN: FileCheck %s -check-prefix=BPFEB
-// BPFEB: target datalayout = "E-m:e-p:64:64-i64:64-n32:64-S128"
+// BPFEB: target datalayout = "E-m:e-p:64:64-i64:64-i128:128-n32:64-S128"
diff --git a/clang/test/CodeGen/thinlto-distributed-newpm.ll b/clang/test/CodeGen/thinlto-distributed-newpm.ll
index bac9c4322045e..5740c6f616be8 100644
--- a/clang/test/CodeGen/thinlto-distributed-newpm.ll
+++ b/clang/test/CodeGen/thinlto-distributed-newpm.ll
@@ -34,7 +34,7 @@
 ; CHECK-O: Running analysis: OptimizationRemarkEmitterAnalysis on main
 ; CHECK-O: Running analysis: PassInstrumentationAnalysis on main
 ; CHECK-O: Running pass: InferFunctionAttrsPass
-; CHECK-O: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager<{{.*}}Function> >
+; CHECK-O: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager<{{.*}}Function>{{ ?}}>
 ; CHECK-O: Starting {{.*}}Function pass manager run.
 ; CHECK-O: Running pass: SimplifyCFGPass on main
 ; CHECK-O: Running analysis: TargetIRAnalysis on main
@@ -57,7 +57,7 @@
 ; CHECK-O: Running analysis: PassInstrumentationAnalysis on main
 ; CHECK-O: Running analysis: AssumptionAnalysis on main
 ; CHECK-O: Running pass: DeadArgumentEliminationPass
-; CHECK-O: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager<{{.*}}Function> >
+; CHECK-O: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager<{{.*}}Function>{{ ?}}>
 ; CHECK-O: Starting {{.*}}Function pass manager run.
 ; CHECK-O: Running pass: InstCombinePass on main
 ; CHECK-O: Running analysis: TargetLibraryAnalysis on main
@@ -179,7 +179,7 @@
 ; CHECK-O: Running pass: ReversePostOrderFunctionAttrsPass
 ; CHECK-O: Running analysis: CallGraphAnalysis
 ; CHECK-O: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
-; CHECK-O: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager<{{.*}}Function> >
+; CHECK-O: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager<{{.*}}Function>{{ ?}}>
 ; CHECK-O: Starting {{.*}}Function pass manager run.
 ; CHECK-O: Running pass: Float2IntPass on main
 ; CHECK-O: Running pass: LowerConstantIntrinsicsPass on main
diff --git a/clang/test/CodeGen/vaes-builtins.c b/clang/test/CodeGen/vaes-builtins.c
index 1dd5784814d3a..eea1f408b3ce8 100644
--- a/clang/test/CodeGen/vaes-builtins.c
+++ b/clang/test/CodeGen/vaes-builtins.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +vaes -emit-llvm -o - | FileCheck %s --check-prefix AVX
-// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +vaes -target-feature +avx512f -emit-llvm -o - | FileCheck %s --check-prefixes AVX,AVX512
+// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +vaes -emit-llvm -o - | FileCheck %s --check-prefix AVX
+// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +vaes -target-feature +avx512f -emit-llvm -o - | FileCheck %s --check-prefixes AVX,AVX512
 
 #include <immintrin.h>
 
diff --git a/clang/test/CodeGen/vpclmulqdq-builtins.c b/clang/test/CodeGen/vpclmulqdq-builtins.c
index 15eef399bcdf7..aa2b8bca91268 100644
--- a/clang/test/CodeGen/vpclmulqdq-builtins.c
+++ b/clang/test/CodeGen/vpclmulqdq-builtins.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +vpclmulqdq -emit-llvm -o - | FileCheck %s --check-prefix AVX
-// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +vpclmulqdq -target-feature +avx512f -emit-llvm -o - | FileCheck %s --check-prefixes AVX,AVX512
+// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +vpclmulqdq -emit-llvm -o - | FileCheck %s --check-prefix AVX
+// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +vpclmulqdq -target-feature +avx512f -emit-llvm -o - | FileCheck %s --check-prefixes AVX,AVX512
 
 #include <immintrin.h>
 
diff --git a/clang/test/CodeGen/x86-serialize-intrin.c b/clang/test/CodeGen/x86-serialize-intrin.c
new file mode 100644
index 0000000000000..a3587efc88835
--- /dev/null
+++ b/clang/test/CodeGen/x86-serialize-intrin.c
@@ -0,0 +1,11 @@
+// RUN: %clang_cc1 %s -ffreestanding -triple x86_64-unknown-unknown -target-feature +serialize -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 %s -ffreestanding -triple i386-unknown-unknown -target-feature +serialize -emit-llvm -o - | FileCheck %s
+
+#include <immintrin.h>
+
+void test_serialize(void)
+{
+// CHECK-LABEL: test_serialize
+// CHECK: call void @llvm.x86.serialize()
+  _serialize();
+}
diff --git a/clang/test/CodeGen/xray-global-init.cpp b/clang/test/CodeGen/xray-global-init.cpp
new file mode 100644
index 0000000000000..588be8a45a507
--- /dev/null
+++ b/clang/test/CodeGen/xray-global-init.cpp
@@ -0,0 +1,13 @@
+// RUN: %clang_cc1 -triple=x86_64-linux-gnu -emit-llvm -fxray-instrument -fxray-instruction-threshold=1 %s -o - \
+// RUN:   | FileCheck %s
+
+struct A {
+  A();
+  ~A();
+};
+
+A a;
+
+// Check that the xray-instruction-threshold was applied
+// CHECK: define internal void @_GLOBAL__sub_I_xray_global_init.cpp() [[NUX:#[0-9]+]] section ".text.startup" {
+// CHECK: attributes [[NUX]] = { noinline nounwind {{.*}}"xray-instruction-threshold"="1"{{.*}} }
diff --git a/clang/test/CodeGenCUDA/amdgpu-workgroup-size.cu b/clang/test/CodeGenCUDA/amdgpu-workgroup-size.cu
new file mode 100644
index 0000000000000..5928320b89f00
--- /dev/null
+++ b/clang/test/CodeGenCUDA/amdgpu-workgroup-size.cu
@@ -0,0 +1,25 @@
+// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa \
+// RUN:     -fcuda-is-device -emit-llvm -o - -x hip %s \
+// RUN:     | FileCheck %s
+
+#include "Inputs/cuda.h"
+
+// CHECK-LABEL: test_get_workgroup_size
+// CHECK: call align 4 dereferenceable(64) i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr()
+// CHECK: getelementptr i8, i8 addrspace(4)* %{{.*}}, i32 4
+// CHECK: load i16, i16 addrspace(4)* %{{.*}}, align 2, !range [[$WS_RANGE:![0-9]*]], !invariant.load
+// CHECK: getelementptr i8, i8 addrspace(4)* %{{.*}}, i32 6
+// CHECK: load i16, i16 addrspace(4)* %{{.*}}, align 2, !range [[$WS_RANGE:![0-9]*]], !invariant.load
+// CHECK: getelementptr i8, i8 addrspace(4)* %{{.*}}, i32 8
+// CHECK: load i16, i16 addrspace(4)* %{{.*}}, align 2, !range [[$WS_RANGE:![0-9]*]], !invariant.load
+__device__ void test_get_workgroup_size(int d, int *out)
+{
+  switch (d) {
+  case 0: *out = __builtin_amdgcn_workgroup_size_x(); break;
+  case 1: *out = __builtin_amdgcn_workgroup_size_y(); break;
+  case 2: *out = __builtin_amdgcn_workgroup_size_z(); break;
+  default: *out = 0;
+  }
+}
+
+// CHECK-DAG: [[$WS_RANGE]] = !{i16 1, i16 1025}
diff --git a/clang/test/CodeGenCUDA/device-stub.cu b/clang/test/CodeGenCUDA/device-stub.cu
index 9db5738cdedec..0f4a5644fd48f 100644
--- a/clang/test/CodeGenCUDA/device-stub.cu
+++ b/clang/test/CodeGenCUDA/device-stub.cu
@@ -181,10 +181,10 @@ void hostfunc(void) { kernelfunc<<<1, 1>>>(1, 1, 1); }
 // Test that we've built a function to register kernels and global vars.
 // ALL: define internal void @__[[PREFIX]]_register_globals
 // ALL: call{{.*}}[[PREFIX]]RegisterFunction(i8** %0, {{.*}}kernelfunc{{[^,]*}}, {{[^@]*}}@0
-// ALL-DAG: call{{.*}}[[PREFIX]]RegisterVar(i8** %0, {{.*}}device_var{{[^,]*}}, {{[^@]*}}@1, {{.*}}i32 0, i32 4, i32 0, i32 0
-// ALL-DAG: call{{.*}}[[PREFIX]]RegisterVar(i8** %0, {{.*}}constant_var{{[^,]*}}, {{[^@]*}}@2, {{.*}}i32 0, i32 4, i32 1, i32 0
-// ALL-DAG: call{{.*}}[[PREFIX]]RegisterVar(i8** %0, {{.*}}ext_device_var_def{{[^,]*}}, {{[^@]*}}@3, {{.*}}i32 0, i32 4, i32 0, i32 0
-// ALL-DAG: call{{.*}}[[PREFIX]]RegisterVar(i8** %0, {{.*}}ext_constant_var_def{{[^,]*}}, {{[^@]*}}@4, {{.*}}i32 0, i32 4, i32 1, i32 0
+// ALL-DAG: call void {{.*}}[[PREFIX]]RegisterVar(i8** %0, {{.*}}device_var{{[^,]*}}, {{[^@]*}}@1, {{.*}}i32 0, {{i32|i64}} 4, i32 0, i32 0
+// ALL-DAG: call void {{.*}}[[PREFIX]]RegisterVar(i8** %0, {{.*}}constant_var{{[^,]*}}, {{[^@]*}}@2, {{.*}}i32 0, {{i32|i64}} 4, i32 1, i32 0
+// ALL-DAG: call void {{.*}}[[PREFIX]]RegisterVar(i8** %0, {{.*}}ext_device_var_def{{[^,]*}}, {{[^@]*}}@3, {{.*}}i32 0, {{i32|i64}} 4, i32 0, i32 0
+// ALL-DAG: call void {{.*}}[[PREFIX]]RegisterVar(i8** %0, {{.*}}ext_constant_var_def{{[^,]*}}, {{[^@]*}}@4, {{.*}}i32 0, {{i32|i64}} 4, i32 1, i32 0
 // ALL: ret void
 
 // Test that we've built a constructor.
diff --git a/clang/test/CodeGenCUDA/flush-denormals.cu b/clang/test/CodeGenCUDA/flush-denormals.cu
index 8577fca928668..e5adb2aa6d25b 100644
--- a/clang/test/CodeGenCUDA/flush-denormals.cu
+++ b/clang/test/CodeGenCUDA/flush-denormals.cu
@@ -1,26 +1,26 @@
 // RUN: %clang_cc1 -fcuda-is-device \
 // RUN:   -triple nvptx-nvidia-cuda -emit-llvm -o - %s | \
-// RUN:   FileCheck -check-prefix=NOFTZ %s
+// RUN:   FileCheck -check-prefixes=NOFTZ,PTXNOFTZ %s
 
 // RUN: %clang_cc1 -fcuda-is-device -fdenormal-fp-math-f32=ieee \
 // RUN:   -triple nvptx-nvidia-cuda -emit-llvm -o - %s | \
-// RUN:   FileCheck -check-prefix=NOFTZ %s
+// RUN:   FileCheck -check-prefixes=NOFTZ,PTXNOFTZ %s
 
 // RUN: %clang_cc1 -fcuda-is-device -fdenormal-fp-math-f32=preserve-sign \
 // RUN:   -triple nvptx-nvidia-cuda -emit-llvm -o - %s | \
-// RUN:   FileCheck -check-prefix=FTZ %s
+// RUN:   FileCheck -check-prefixes=FTZ,PTXFTZ %s
 
 // RUN: %clang_cc1 -fcuda-is-device -x hip \
 // RUN:   -triple amdgcn-amd-amdhsa -target-cpu gfx900 -emit-llvm -o - %s | \
-// RUN:   FileCheck -check-prefix=AMDNOFTZ %s
+// RUN:   FileCheck -check-prefix=NOFTZ %s
 
 // RUN: %clang_cc1 -fcuda-is-device -x hip \
 // RUN:   -triple amdgcn-amd-amdhsa -target-cpu gfx900 -fdenormal-fp-math-f32=ieee -emit-llvm -o - %s | \
-// RUN:   FileCheck -check-prefix=AMDNOFTZ %s
+// RUN:   FileCheck -check-prefix=NOFTZ %s
 
 // RUN: %clang_cc1 -fcuda-is-device -x hip -fdenormal-fp-math-f32=preserve-sign \
 // RUN:   -triple amdgcn-amd-amdhsa -target-cpu gfx900 -emit-llvm -o - %s | \
-// RUN:   FileCheck -check-prefix=AMDFTZ %s
+// RUN:   FileCheck -check-prefix=FTZ %s
 
 #include "Inputs/cuda.h"
 
@@ -29,10 +29,13 @@
 // -fdenormal-fp-math-f32. Further, check that we reflect the presence or
 // absence of -fcuda-flush-denormals-to-zero in a module flag.
 
-// AMDGCN targets always have +fp64-fp16-denormals.
-// AMDGCN targets without fast FMAF (e.g. gfx803) always have +fp32-denormals.
-// For AMDGCN target with fast FMAF (e.g. gfx900), it has +fp32-denormals
-// by default and -fp32-denormals when there is option
+// AMDGCN targets always have f64/f16 denormals enabled.
+//
+// AMDGCN targets without fast FMAF (e.g. gfx803) always have f32 denormal
+// flushing by default.
+//
+// For AMDGCN target with fast FMAF (e.g. gfx900), it has ieee denormals by
+// default and preserve-sign when there with the option
 // -fcuda-flush-denormals-to-zero.
 
 // CHECK-LABEL: define void @foo() #0
@@ -41,11 +44,8 @@ extern "C" __device__ void foo() {}
 // FTZ: attributes #0 = {{.*}} "denormal-fp-math-f32"="preserve-sign,preserve-sign"
 // NOFTZ-NOT: "denormal-fp-math-f32"
 
-// AMDNOFTZ: attributes #0 = {{.*}}+fp32-denormals{{.*}}+fp64-fp16-denormals
-// AMDFTZ: attributes #0 = {{.*}}+fp64-fp16-denormals{{.*}}-fp32-denormals
-
-// FTZ:!llvm.module.flags = !{{{.*}}[[MODFLAG:![0-9]+]]}
-// FTZ:[[MODFLAG]] = !{i32 4, !"nvvm-reflect-ftz", i32 1}
+// PTXFTZ:!llvm.module.flags = !{{{.*}}[[MODFLAG:![0-9]+]]}
+// PTXFTZ:[[MODFLAG]] = !{i32 4, !"nvvm-reflect-ftz", i32 1}
 
-// NOFTZ:!llvm.module.flags = !{{{.*}}[[MODFLAG:![0-9]+]]}
-// NOFTZ:[[MODFLAG]] = !{i32 4, !"nvvm-reflect-ftz", i32 0}
+// PTXNOFTZ:!llvm.module.flags = !{{{.*}}[[MODFLAG:![0-9]+]]}
+// PTXNOFTZ:[[MODFLAG]] = !{i32 4, !"nvvm-reflect-ftz", i32 0}
diff --git a/clang/test/CodeGenCUDA/surface.cu b/clang/test/CodeGenCUDA/surface.cu
new file mode 100644
index 0000000000000..a28e052fc2b74
--- /dev/null
+++ b/clang/test/CodeGenCUDA/surface.cu
@@ -0,0 +1,42 @@
+// REQUIRES: x86-registered-target
+// REQUIRES: nvptx-registered-target
+
+// RUN: %clang_cc1 -std=c++11 -fcuda-is-device -triple nvptx64-nvidia-cuda -emit-llvm -o - %s | FileCheck --check-prefix=DEVICE %s
+// RUN: echo "GPU binary would be here" > %t
+// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-linux-gnu -target-sdk-version=8.0 -fcuda-include-gpubinary %t -emit-llvm -o - %s | FileCheck --check-prefix=HOST %s
+
+struct surfaceReference {
+  int desc;
+};
+
+template <typename T, int dim = 1>
+struct __attribute__((device_builtin_surface_type)) surface : public surfaceReference {
+};
+
+// Partial specialization over `void`.
+template<int dim>
+struct __attribute__((device_builtin_surface_type)) surface<void, dim> : public surfaceReference {
+};
+
+// On the device side, surface references are represented as `i64` handles.
+// DEVICE: @surf = addrspace(1) global i64 undef, align 4
+// On the host side, they remain in the original type.
+// HOST: @surf = internal global %struct.surface
+// HOST: @0 = private unnamed_addr constant [5 x i8] c"surf\00"
+surface<void, 2> surf;
+
+__attribute__((device)) int suld_2d_zero(surface<void, 2>, int, int) asm("llvm.nvvm.suld.2d.i32.zero");
+
+// DEVICE-LABEL: i32 @_Z3fooii(i32 %x, i32 %y)
+// DEVICE: call i64 @llvm.nvvm.texsurf.handle.internal.p1i64(i64 addrspace(1)* @surf)
+// DEVICE: call i32 @llvm.nvvm.suld.2d.i32.zero(i64 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
+__attribute__((device)) int foo(int x, int y) {
+  return suld_2d_zero(surf, x, y);
+}
+
+// HOST: define internal void @[[PREFIX:__cuda]]_register_globals
+// Texture references need registering with correct arguments.
+// HOST: call void @[[PREFIX]]RegisterSurface(i8** %0, i8*{{.*}}({{.*}}@surf{{.*}}), i8*{{.*}}({{.*}}@0{{.*}}), i8*{{.*}}({{.*}}@0{{.*}}), i32 2, i32 0)
+
+// They also need annotating in metadata.
+// DEVICE: !0 = !{i64 addrspace(1)* @surf, !"surface", i32 1}
diff --git a/clang/test/CodeGenCUDA/texture.cu b/clang/test/CodeGenCUDA/texture.cu
new file mode 100644
index 0000000000000..7838eeb110b0a
--- /dev/null
+++ b/clang/test/CodeGenCUDA/texture.cu
@@ -0,0 +1,55 @@
+// REQUIRES: x86-registered-target
+// REQUIRES: nvptx-registered-target
+
+// RUN: %clang_cc1 -std=c++11 -fcuda-is-device -triple nvptx64-nvidia-cuda -emit-llvm -o - %s | FileCheck --check-prefix=DEVICE %s
+// RUN: echo "GPU binary would be here" > %t
+// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-linux-gnu -target-sdk-version=8.0 -fcuda-include-gpubinary %t -emit-llvm -o - %s | FileCheck --check-prefix=HOST %s
+
+struct textureReference {
+  int desc;
+};
+
+enum ReadMode {
+  ElementType = 0,
+  NormalizedFloat = 1
+};
+
+template <typename T, int dim = 1, enum ReadMode mode = ElementType>
+struct __attribute__((device_builtin_texture_type)) texture : public textureReference {
+};
+
+// On the device side, texture references are represented as `i64` handles.
+// DEVICE: @tex = addrspace(1) global i64 undef, align 4
+// DEVICE: @norm = addrspace(1) global i64 undef, align 4
+// On the host side, they remain in the original type.
+// HOST: @tex = internal global %struct.texture
+// HOST: @norm = internal global %struct.texture
+// HOST: @0 = private unnamed_addr constant [4 x i8] c"tex\00"
+// HOST: @1 = private unnamed_addr constant [5 x i8] c"norm\00"
+texture<float, 2, ElementType> tex;
+texture<float, 2, NormalizedFloat> norm;
+
+struct v4f {
+  float x, y, z, w;
+};
+
+__attribute__((device)) v4f tex2d_ld(texture<float, 2, ElementType>, float, float) asm("llvm.nvvm.tex.unified.2d.v4f32.f32");
+__attribute__((device)) v4f tex2d_ld(texture<float, 2, NormalizedFloat>, int, int) asm("llvm.nvvm.tex.unified.2d.v4f32.s32");
+
+// DEVICE-LABEL: float @_Z3fooff(float %x, float %y)
+// DEVICE: call i64 @llvm.nvvm.texsurf.handle.internal.p1i64(i64 addrspace(1)* @tex)
+// DEVICE: call %struct.v4f @llvm.nvvm.tex.unified.2d.v4f32.f32(i64 %{{.*}}, float %{{.*}}, float %{{.*}})
+// DEVICE: call i64 @llvm.nvvm.texsurf.handle.internal.p1i64(i64 addrspace(1)* @norm)
+// DEVICE: call %struct.v4f @llvm.nvvm.tex.unified.2d.v4f32.s32(i64 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
+__attribute__((device)) float foo(float x, float y) {
+  return tex2d_ld(tex, x, y).x + tex2d_ld(norm, int(x), int(y)).x;
+}
+
+// HOST: define internal void @[[PREFIX:__cuda]]_register_globals
+// Texture references need registering with correct arguments.
+// HOST: call void @[[PREFIX]]RegisterTexture(i8** %0, i8*{{.*}}({{.*}}@tex{{.*}}), i8*{{.*}}({{.*}}@0{{.*}}), i8*{{.*}}({{.*}}@0{{.*}}), i32 2, i32 0, i32 0)
+// HOST: call void @[[PREFIX]]RegisterTexture(i8** %0, i8*{{.*}}({{.*}}@norm{{.*}}), i8*{{.*}}({{.*}}@1{{.*}}), i8*{{.*}}({{.*}}@1{{.*}}), i32 2, i32 1, i32 0)
+
+// They also need annotating in metadata.
+// DEVICE: !0 = !{i64 addrspace(1)* @tex, !"texture", i32 1}
+// DEVICE: !1 = !{i64 addrspace(1)* @norm, !"texture", i32 1}
diff --git a/clang/test/CodeGenCXX/bitfield-layout.cpp b/clang/test/CodeGenCXX/bitfield-layout.cpp
index 46f41111a1de2..d8f8c87eb28b7 100644
--- a/clang/test/CodeGenCXX/bitfield-layout.cpp
+++ b/clang/test/CodeGenCXX/bitfield-layout.cpp
@@ -1,11 +1,14 @@
-// RUN: %clang_cc1 %s -triple=x86_64-apple-darwin10 -emit-llvm -o - -O3 | FileCheck -check-prefix CHECK-LP64 %s
-// RUN: %clang_cc1 %s -triple=i386-apple-darwin10 -emit-llvm -o - -O3 | FileCheck -check-prefix CHECK-LP32 %s
+// RUN: %clang_cc1 %s -triple=x86_64-apple-darwin10 -emit-llvm -o - -O3 | FileCheck -check-prefix=CHECK-LP64 -check-prefix=CHECK %s
+// RUN: %clang_cc1 %s -triple=i386-apple-darwin10 -emit-llvm -o - -O3 | FileCheck -check-prefix CHECK-LP32 -check-prefix=CHECK %s
+// RUN: %clang_cc1 %s -triple=aarch64_be-none-eabi -emit-llvm -o - -O3 | FileCheck -check-prefix CHECK-A64BE -check-prefix=CHECK %s
+// RUN: %clang_cc1 %s -triple=thumbv7_be-none-eabi -emit-llvm -o - -O3 | FileCheck -check-prefix CHECK-A32BE -check-prefix=CHECK %s
 
 // CHECK-LP64: %union.Test1 = type { i32, [4 x i8] }
 union Test1 {
   int a;
   int b: 39;
-} t1;
+};
+Test1 t1;
 
 // CHECK-LP64: %union.Test2 = type { i8 }
 union Test2 {
@@ -17,10 +20,16 @@ union Test3 {
   int : 9;
 } t3;
 
+// CHECK: %union.Test4 = type { i8, i8 }
+union Test4 {
+  char val : 16;
+};
+Test4 t4;
 
 #define CHECK(x) if (!(x)) return __LINE__
 
-int f() {
+// CHECK: define i32 @_Z11test_assignv()
+int test_assign() {
   struct {
     int a;
 
@@ -37,7 +46,41 @@ int f() {
   CHECK(c.b == (unsigned long long)-1);
   CHECK(c.c == 0);
 
-// CHECK-LP64: ret i32 0
-// CHECK-LP32: ret i32 0
+  Test1 u1;
+  Test4 u2;
+
+  u1.b = 1;
+  u2.val = 42;
+
+  CHECK(u1.b == 1);
+  CHECK(u2.val == 42);
+
+  // CHECK: ret i32 0
+  return 0;
+}
+
+// CHECK: define i32 @_Z9test_initv()
+int test_init() {
+  struct S {
+    int a;
+
+    unsigned long long b : 65;
+
+    int c;
+  };
+  S s1 = {1, 42, 0};
+
+  CHECK(s1.a == 1);
+  CHECK(s1.b == (unsigned long long)42);
+  CHECK(s1.c == 0);
+
+  Test1 u1 = {1};
+  Test4 u2 = {42};
+
+  CHECK(u1.a == 1);
+  CHECK(u1.b == 1);
+  CHECK(u2.val == 42);
+
+  // CHECK: ret i32 0
   return 0;
 }
diff --git a/clang/test/CodeGenCXX/debug-info-template-explicit-specialization.cpp b/clang/test/CodeGenCXX/debug-info-template-explicit-specialization.cpp
index 4e41c4092bf4e..d97d82b769657 100644
--- a/clang/test/CodeGenCXX/debug-info-template-explicit-specialization.cpp
+++ b/clang/test/CodeGenCXX/debug-info-template-explicit-specialization.cpp
@@ -110,7 +110,7 @@ struct j_wrap {
 };
 j_wrap<j<int>> j_wrap_j;
 // CHECK: DICompositeType(tag: DW_TAG_structure_type, name: "j<int, int>"
-// CHECK: DICompositeType(tag: DW_TAG_structure_type, name: "j_wrap<j<int, int> >"
+// CHECK: DICompositeType(tag: DW_TAG_structure_type, name: "j_wrap<j<int, int>>"
 
 template <typename T>
 struct k {
diff --git a/clang/test/CodeGenCXX/debug-info-template-parameter.cpp b/clang/test/CodeGenCXX/debug-info-template-parameter.cpp
index 95e7a187fe102..c38c535d8b069 100644
--- a/clang/test/CodeGenCXX/debug-info-template-parameter.cpp
+++ b/clang/test/CodeGenCXX/debug-info-template-parameter.cpp
@@ -8,22 +8,24 @@
 
 // CHECK: DILocalVariable(name: "f1", {{.*}}, type: ![[TEMPLATE_TYPE:[0-9]+]]
 // CHECK: [[TEMPLATE_TYPE]] = {{.*}}!DICompositeType({{.*}}, templateParams: ![[F1_TYPE:[0-9]+]]
-// CHECK: [[F1_TYPE]] = !{![[FIRST:[0-9]+]], ![[SECOND:[0-9]+]]}
+// CHECK: [[F1_TYPE]] = !{![[FIRST:[0-9]+]], ![[SECOND:[0-9]+]], ![[THIRD:[0-9]+]]}
 // CHECK: [[FIRST]] = !DITemplateTypeParameter(name: "T", type: !{{[0-9]*}})
 // CHECK: [[SECOND]] = !DITemplateValueParameter(name: "i", type: !{{[0-9]*}}, value: i32 6)
+// CHECK: [[THIRD]] = !DITemplateValueParameter(name: "b", type: !{{[0-9]*}}, value: i8 0)
 
 // CHECK: DILocalVariable(name: "f2", {{.*}}, type: ![[TEMPLATE_TYPE:[0-9]+]]
 // CHECK: [[TEMPLATE_TYPE]] = {{.*}}!DICompositeType({{.*}}, templateParams: ![[F2_TYPE:[0-9]+]]
-// CHECK: [[F2_TYPE]] = !{![[FIRST:[0-9]+]], ![[SECOND:[0-9]+]]}
+// CHECK: [[F2_TYPE]] = !{![[FIRST:[0-9]+]], ![[SECOND:[0-9]+]], ![[THIRD:[0-9]+]]}
 // CHECK: [[FIRST]] = !DITemplateTypeParameter(name: "T", type: !{{[0-9]*}}, defaulted: true)
 // CHECK: [[SECOND]] = !DITemplateValueParameter(name: "i", type: !{{[0-9]*}}, defaulted: true, value: i32 3)
+// CHECK: [[THIRD]] = !DITemplateValueParameter(name: "b", type: !{{[0-9]*}}, defaulted: true, value: i8 1)
 
-template <typename T = char, int i = 3>
+template <typename T = char, int i = 3, bool b = true>
 class foo {
 };
 
 int main() {
-  foo<int, 6> f1;
+  foo<int, 6, false> f1;
   foo<> f2;
   return 0;
 }
diff --git a/clang/test/CodeGenObjC/debug-info-class-extension.m b/clang/test/CodeGenObjC/debug-info-class-extension.m
index a27810cce7433..db654e6a60a55 100644
--- a/clang/test/CodeGenObjC/debug-info-class-extension.m
+++ b/clang/test/CodeGenObjC/debug-info-class-extension.m
@@ -1,5 +1,5 @@
 // FIXME: Check IR rather than asm, then triple is not needed.
-// RUN: %clang_cc1 -triple %itanium_abi_triple -masm-verbose -S -debug-info-kind=limited %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple %itanium_abi_triple -S -debug-info-kind=limited %s -o - | FileCheck %s
 
 // CHECK: AT_APPLE_objc_complete_type
 
diff --git a/clang/test/CodeGenObjC/debug-info-class-extension2.m b/clang/test/CodeGenObjC/debug-info-class-extension2.m
index d4750c120f602..ea7865b4ac5b6 100644
--- a/clang/test/CodeGenObjC/debug-info-class-extension2.m
+++ b/clang/test/CodeGenObjC/debug-info-class-extension2.m
@@ -1,5 +1,5 @@
 // FIXME: Check IR rather than asm, then triple is not needed.
-// RUN: %clang_cc1 -triple %itanium_abi_triple -masm-verbose -S -debug-info-kind=limited %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple %itanium_abi_triple -S -debug-info-kind=limited %s -o - | FileCheck %s
 // CHECK: AT_APPLE_objc_complete_type
 
 @interface Foo {} @end
diff --git a/clang/test/CodeGenObjC/debug-info-class-extension3.m b/clang/test/CodeGenObjC/debug-info-class-extension3.m
index a9cf6f6a5c592..f81445b47a212 100644
--- a/clang/test/CodeGenObjC/debug-info-class-extension3.m
+++ b/clang/test/CodeGenObjC/debug-info-class-extension3.m
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -masm-verbose -S -debug-info-kind=limited %s -o - | FileCheck %s
+// RUN: %clang_cc1 -S -debug-info-kind=limited %s -o - | FileCheck %s
 
 // CHECK-NOT: AT_APPLE_objc_complete_type
 
diff --git a/clang/test/CodeGenObjC/debug-info-property.m b/clang/test/CodeGenObjC/debug-info-property.m
index 9b471be23dbdc..ca013b24be421 100644
--- a/clang/test/CodeGenObjC/debug-info-property.m
+++ b/clang/test/CodeGenObjC/debug-info-property.m
@@ -1,5 +1,5 @@
 // FIXME: Check IR rather than asm, then triple is not needed.
-// RUN: %clang_cc1 -triple %itanium_abi_triple -masm-verbose -S -debug-info-kind=limited %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple %itanium_abi_triple -S -debug-info-kind=limited %s -o - | FileCheck %s
 
 // CHECK: AT_APPLE_property_name
 // CHECK: AT_APPLE_property_attribute
diff --git a/clang/test/CodeGenObjC/debug-info-property2.m b/clang/test/CodeGenObjC/debug-info-property2.m
index 6a15922c932c2..7e0a5e9f954ba 100644
--- a/clang/test/CodeGenObjC/debug-info-property2.m
+++ b/clang/test/CodeGenObjC/debug-info-property2.m
@@ -1,5 +1,5 @@
 // FIXME: Check IR rather than asm, then triple is not needed.
-// RUN: %clang_cc1 -triple %itanium_abi_triple -masm-verbose -S -debug-info-kind=limited %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple %itanium_abi_triple -S -debug-info-kind=limited %s -o - | FileCheck %s
 
 // CHECK: AT_APPLE_property_name
 @interface C {
diff --git a/clang/test/CodeGenObjC/debug-info-property4.m b/clang/test/CodeGenObjC/debug-info-property4.m
index f862c85b344d1..1f489f2f6b637 100644
--- a/clang/test/CodeGenObjC/debug-info-property4.m
+++ b/clang/test/CodeGenObjC/debug-info-property4.m
@@ -1,5 +1,5 @@
 // FIXME: Check IR rather than asm, then triple is not needed.
-// RUN: %clang_cc1 -triple %itanium_abi_triple -masm-verbose -S -debug-info-kind=limited %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple %itanium_abi_triple -S -debug-info-kind=limited %s -o - | FileCheck %s
 
 // CHECK: AT_APPLE_property_name
 // CHECK-NOT: AT_APPLE_property_getter
diff --git a/clang/test/CodeGenObjC/debug-info-property5.m b/clang/test/CodeGenObjC/debug-info-property5.m
index 191da9c16fcc7..8b70f1ff20824 100644
--- a/clang/test/CodeGenObjC/debug-info-property5.m
+++ b/clang/test/CodeGenObjC/debug-info-property5.m
@@ -1,5 +1,5 @@
 // FIXME: Check IR rather than asm, then triple is not needed.
-// RUN: %clang_cc1 -triple %itanium_abi_triple -masm-verbose -S -debug-info-kind=limited %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple %itanium_abi_triple -S -debug-info-kind=limited %s -o - | FileCheck %s
 
 // CHECK: AT_APPLE_property_name
 // CHECK: AT_APPLE_property_getter
diff --git a/clang/test/CodeGenObjC/property-dbg.m b/clang/test/CodeGenObjC/property-dbg.m
index fb70747f5db89..f15213131ccc0 100644
--- a/clang/test/CodeGenObjC/property-dbg.m
+++ b/clang/test/CodeGenObjC/property-dbg.m
@@ -1,5 +1,5 @@
 // FIXME: Check IR rather than asm, then triple is not needed.
-// RUN: %clang_cc1 -triple %itanium_abi_triple -S -debug-info-kind=limited -masm-verbose -x objective-c < %s | grep DW_AT_name
+// RUN: %clang_cc1 -triple %itanium_abi_triple -S -debug-info-kind=limited -x objective-c < %s | grep DW_AT_name
 @interface Foo {
   int i;
 }
diff --git a/clang/test/CodeGenOpenCL/amdgpu-features.cl b/clang/test/CodeGenOpenCL/amdgpu-features.cl
index b4ee2edc5443b..7529a4d4abb15 100644
--- a/clang/test/CodeGenOpenCL/amdgpu-features.cl
+++ b/clang/test/CodeGenOpenCL/amdgpu-features.cl
@@ -14,15 +14,15 @@
 // RUN: %clang_cc1 -triple amdgcn -target-cpu gfx600 -S -emit-llvm -o - %s | FileCheck --check-prefix=GFX600 %s
 // RUN: %clang_cc1 -triple amdgcn -target-cpu gfx601 -S -emit-llvm -o - %s | FileCheck --check-prefix=GFX601 %s
 
-// GFX904: "target-features"="+16-bit-insts,+ci-insts,+dpp,+flat-address-space,+fp32-denormals,+fp64-fp16-denormals,+gfx8-insts,+gfx9-insts,+s-memrealtime"
-// GFX906: "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dot1-insts,+dot2-insts,+dpp,+flat-address-space,+fp32-denormals,+fp64-fp16-denormals,+gfx8-insts,+gfx9-insts,+s-memrealtime"
-// GFX908: "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dot1-insts,+dot2-insts,+dot3-insts,+dot4-insts,+dot5-insts,+dot6-insts,+dpp,+flat-address-space,+fp32-denormals,+fp64-fp16-denormals,+gfx8-insts,+gfx9-insts,+mai-insts,+s-memrealtime"
-// GFX1010: "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dpp,+flat-address-space,+fp32-denormals,+fp64-fp16-denormals,+gfx10-insts,+gfx8-insts,+gfx9-insts,+s-memrealtime"
-// GFX1011: "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dot1-insts,+dot2-insts,+dot5-insts,+dot6-insts,+dpp,+flat-address-space,+fp32-denormals,+fp64-fp16-denormals,+gfx10-insts,+gfx8-insts,+gfx9-insts,+s-memrealtime"
-// GFX1012: "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dot1-insts,+dot2-insts,+dot5-insts,+dot6-insts,+dpp,+flat-address-space,+fp32-denormals,+fp64-fp16-denormals,+gfx10-insts,+gfx8-insts,+gfx9-insts,+s-memrealtime"
-// GFX801: "target-features"="+16-bit-insts,+ci-insts,+dpp,+flat-address-space,+fp32-denormals,+fp64-fp16-denormals,+gfx8-insts,+s-memrealtime"
-// GFX700: "target-features"="+ci-insts,+flat-address-space,+fp64-fp16-denormals,-fp32-denormals"
-// GFX600: "target-features"="+fp64-fp16-denormals,-fp32-denormals"
-// GFX601: "target-features"="+fp64-fp16-denormals,-fp32-denormals"
+// GFX904: "target-features"="+16-bit-insts,+ci-insts,+dpp,+flat-address-space,+gfx8-insts,+gfx9-insts,+s-memrealtime"
+// GFX906: "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dot1-insts,+dot2-insts,+dpp,+flat-address-space,+gfx8-insts,+gfx9-insts,+s-memrealtime"
+// GFX908: "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dot1-insts,+dot2-insts,+dot3-insts,+dot4-insts,+dot5-insts,+dot6-insts,+dpp,+flat-address-space,+gfx8-insts,+gfx9-insts,+mai-insts,+s-memrealtime"
+// GFX1010: "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dpp,+flat-address-space,+gfx10-insts,+gfx8-insts,+gfx9-insts,+s-memrealtime"
+// GFX1011: "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dot1-insts,+dot2-insts,+dot5-insts,+dot6-insts,+dpp,+flat-address-space,+gfx10-insts,+gfx8-insts,+gfx9-insts,+s-memrealtime"
+// GFX1012: "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dot1-insts,+dot2-insts,+dot5-insts,+dot6-insts,+dpp,+flat-address-space,+gfx10-insts,+gfx8-insts,+gfx9-insts,+s-memrealtime"
+// GFX801: "target-features"="+16-bit-insts,+ci-insts,+dpp,+flat-address-space,+gfx8-insts,+s-memrealtime"
+// GFX700: "target-features"="+ci-insts,+flat-address-space"
+// GFX600-NOT: "target-features"
+// GFX601-NOT: "target-features"
 
 kernel void test() {}
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn.cl
index 0aa3e4144c525..8f2f149103b34 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn.cl
@@ -527,6 +527,24 @@ void test_get_local_id(int d, global int *out)
 	}
 }
 
+// CHECK-LABEL: @test_get_workgroup_size(
+// CHECK: call align 4 dereferenceable(64) i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr()
+// CHECK: getelementptr i8, i8 addrspace(4)* %{{.*}}, i64 4
+// CHECK: load i16, i16 addrspace(4)* %{{.*}}, align 4, !range [[$WS_RANGE:![0-9]*]], !invariant.load
+// CHECK: getelementptr i8, i8 addrspace(4)* %{{.*}}, i64 6
+// CHECK: load i16, i16 addrspace(4)* %{{.*}}, align 2, !range [[$WS_RANGE:![0-9]*]], !invariant.load
+// CHECK: getelementptr i8, i8 addrspace(4)* %{{.*}}, i64 8
+// CHECK: load i16, i16 addrspace(4)* %{{.*}}, align 4, !range [[$WS_RANGE:![0-9]*]], !invariant.load
+void test_get_workgroup_size(int d, global int *out)
+{
+	switch (d) {
+	case 0: *out = __builtin_amdgcn_workgroup_size_x() + 1; break;
+	case 1: *out = __builtin_amdgcn_workgroup_size_y(); break;
+	case 2: *out = __builtin_amdgcn_workgroup_size_z(); break;
+	default: *out = 0;
+	}
+}
+
 // CHECK-LABEL: @test_fmed3_f32
 // CHECK: call float @llvm.amdgcn.fmed3.f32(
 void test_fmed3_f32(global float* out, float a, float b, float c)
@@ -698,6 +716,7 @@ kernel void test_mqsad_u32_u8(global uint4* out, ulong src0, uint src1, uint4 sr
 }
 
 // CHECK-DAG: [[$WI_RANGE]] = !{i32 0, i32 1024}
+// CHECK-DAG: [[$WS_RANGE]] = !{i16 1, i16 1025}
 // CHECK-DAG: attributes #[[$NOUNWIND_READONLY:[0-9]+]] = { nounwind readonly }
 // CHECK-DAG: attributes #[[$READ_EXEC_ATTRS]] = { convergent }
 // CHECK-DAG: ![[$EXEC]] = !{!"exec"}
diff --git a/clang/test/CodeGenOpenCL/inline-asm-amdgcn.cl b/clang/test/CodeGenOpenCL/inline-asm-amdgcn.cl
index ccd98210d3a39..37090772f6646 100644
--- a/clang/test/CodeGenOpenCL/inline-asm-amdgcn.cl
+++ b/clang/test/CodeGenOpenCL/inline-asm-amdgcn.cl
@@ -1,8 +1,35 @@
 // REQUIRES: amdgpu-registered-target
-// RUN: %clang_cc1 -emit-llvm -o - -triple amdgcn %s | FileCheck %s
+// RUN: %clang_cc1 -emit-llvm -O0 -o - -triple amdgcn %s | FileCheck %s
+
+typedef float float32 __attribute__((ext_vector_type(32)));
 
 kernel void test_long(int arg0) {
   long v15_16;
-  // CHECK: tail call i64 asm sideeffect "v_lshlrev_b64 v[15:16], 0, $0", "={v[15:16]},v"(i32 %arg0)
+  // CHECK: call i64 asm sideeffect "v_lshlrev_b64 v[15:16], 0, $0", "={v[15:16]},v"
   __asm volatile("v_lshlrev_b64 v[15:16], 0, %0" : "={v[15:16]}"(v15_16) : "v"(arg0));
 }
+
+kernel void test_agpr() {
+  float32 acc_c;
+  float reg_a;
+  float reg_b;
+  float32 reg_c;
+  // CHECK:  call <32 x float> asm "v_mfma_f32_32x32x1f32 $0, $1, $2, $3", "=a,v,v,a,~{a0},~{a1},~{a2},~{a3},~{a4},~{a5},~{a6},~{a7},~{a8},~{a9},~{a10},~{a11},~{a12},~{a13},~{a14},~{a15},~{a16},~{a17},~{a18},~{a19},~{a20},~{a21},~{a22},~{a23},~{a24},~{a25},~{a26},~{a27},~{a28},~{a29},~{a30},~{a31}"
+  __asm ("v_mfma_f32_32x32x1f32 %0, %1, %2, %3"
+         : "=a"(acc_c)
+         : "v"(reg_a), "v"(reg_b), "a"(reg_c)
+         : "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7",
+           "a8", "a9", "a10", "a11", "a12", "a13", "a14", "a15",
+           "a16", "a17", "a18", "a19", "a20", "a21", "a22", "a23",
+           "a24", "a25", "a26", "a27", "a28", "a29", "a30", "a31");
+
+   // CHECK: call <32 x float> asm sideeffect "v_mfma_f32_32x32x1f32 a[0:31], $0, $1, a[0:31]", "={a[0:31]},v,v,{a[0:31]}"
+  __asm volatile("v_mfma_f32_32x32x1f32 a[0:31], %0, %1, a[0:31]"
+                 : "={a[0:31]}"(acc_c)
+                 : "v"(reg_a),"v"(reg_b), "{a[0:31]}"(reg_c));
+
+  // CHECK: call float asm "v_accvgpr_read_b32 $0, $1", "={a1},{a1}"
+  __asm ("v_accvgpr_read_b32 %0, %1"
+         : "={a1}"(reg_a)
+         : "{a1}"(reg_b));
+}
diff --git a/clang/test/CodeGenOpenCL/pipe_types_mangling.cl b/clang/test/CodeGenOpenCL/pipe_types_mangling.cl
new file mode 100644
index 0000000000000..a7439f378b817
--- /dev/null
+++ b/clang/test/CodeGenOpenCL/pipe_types_mangling.cl
@@ -0,0 +1,77 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm -O0 -cl-std=clc++ -o - %s | FileCheck %s --check-prefixes=LINUX
+// RUN: %clang_cc1 -triple x86_64-unknown-windows-pc -emit-llvm -O0 -cl-std=clc++ -o - %s -DWIN| FileCheck %s --check-prefixes=WINDOWS
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm -O0 -cl-std=CL2.0 -o - %s | FileCheck %s --check-prefixes=LINUX
+// RUN: %clang_cc1 -triple x86_64-unknown-windows-pc -emit-llvm -O0 -cl-std=CL2.0 -o - %s -DWIN| FileCheck %s --check-prefixes=OCLWINDOWS
+
+typedef unsigned char __attribute__((ext_vector_type(3))) uchar3;
+typedef int __attribute__((ext_vector_type(4))) int4;
+
+__attribute__((overloadable))
+void test1(read_only pipe int p) {
+// LINUX: define void @_Z5test18ocl_pipe
+// WINDOWS: define dso_local void @"?test1@@YAXU?$ocl_pipe@H$00@__clang@@@Z"
+// OCLWINDOWS: define dso_local void @"?test1@@$$J0YAXU?$ocl_pipe@H$00@__clang@@@Z"
+}
+
+__attribute__((overloadable))
+void test2(write_only pipe float p) {
+// LINUX: define void @_Z5test28ocl_pipe
+// WINDOWS: define dso_local void @"?test2@@YAXU?$ocl_pipe@M$0A@@__clang@@@Z"
+// OCLWINDOWS: define dso_local void @"?test2@@$$J0YAXU?$ocl_pipe@M$0A@@__clang@@@Z"
+}
+
+#ifdef WIN
+// It isn't possible to overload on pipe types in Linux mode
+// because the OCL specification on the Itanium ABI has a specified mangling
+// for the entire class of types, and thus doesn't take element type or read/write
+// into account. Thus, both would result in the same mangling, which is an IR-CodeGen
+// error. Our windows implementation of this mangling doesn't have that problem,
+// so we can test it here.
+__attribute__((overloadable))
+void test2(read_only pipe int p) {
+// WINDOWS: define dso_local void @"?test2@@YAXU?$ocl_pipe@H$00@__clang@@@Z"
+// OCLWINDOWS: define dso_local void @"?test2@@$$J0YAXU?$ocl_pipe@H$00@__clang@@@Z"
+}
+#endif
+
+__attribute__((overloadable))
+void test3(read_only pipe const int p) {
+// LINUX: define void @_Z5test38ocl_pipe
+// WINDOWS: define dso_local void @"?test3@@YAXU?$ocl_pipe@$$CBH$00@__clang@@@Z"
+// OCLWINDOWS: define dso_local void @"?test3@@$$J0YAXU?$ocl_pipe@$$CBH$00@__clang@@@Z"
+}
+
+__attribute__((overloadable))
+void test4(read_only pipe uchar3 p) {
+// LINUX: define void @_Z5test48ocl_pipe
+// WINDOWS: define dso_local void @"?test4@@YAXU?$ocl_pipe@T?$__vector@E$02@__clang@@$00@__clang@@@Z"
+// OCLWINDOWS: define dso_local void @"?test4@@$$J0YAXU?$ocl_pipe@T?$__vector@E$02@__clang@@$00@__clang@@@Z"
+}
+
+__attribute__((overloadable))
+void test5(read_only pipe int4 p) {
+// LINUX: define void @_Z5test58ocl_pipe
+// WINDOWS: define dso_local void @"?test5@@YAXU?$ocl_pipe@T?$__vector@H$03@__clang@@$00@__clang@@@Z"
+// OCLWINDOWS: define dso_local void @"?test5@@$$J0YAXU?$ocl_pipe@T?$__vector@H$03@__clang@@$00@__clang@@@Z"
+}
+
+typedef read_only pipe int MyPipe;
+kernel void test6(MyPipe p) {
+// LINUX: define spir_kernel void @test6
+// WINDOWS: define dso_local spir_kernel void @test6
+// OCLWINDOWS: define dso_local spir_kernel void @test6
+}
+
+struct Person {
+  const char *Name;
+  bool isFemale;
+  int ID;
+};
+
+__attribute__((overloadable))
+void test_reserved_read_pipe(global struct Person *SDst,
+                             read_only pipe struct Person SPipe) {
+// LINUX: define void @_Z23test_reserved_read_pipePU8CLglobal6Person8ocl_pipe
+// WINDOWS: define dso_local void @"?test_reserved_read_pipe@@YAXPEAU?$_ASCLglobal@$$CAUPerson@@@__clang@@U?$ocl_pipe@UPerson@@$00@2@@Z"
+// OCLWINDOWS: define dso_local void @"?test_reserved_read_pipe@@$$J0YAXPEAU?$_ASCLglobal@$$CAUPerson@@@__clang@@U?$ocl_pipe@UPerson@@$00@2@@Z"
+}
diff --git a/clang/test/Driver/aarch64-cpus.c b/clang/test/Driver/aarch64-cpus.c
index 36444f02e9554..971f3cd83521d 100644
--- a/clang/test/Driver/aarch64-cpus.c
+++ b/clang/test/Driver/aarch64-cpus.c
@@ -603,6 +603,39 @@
 // RUN: %clang -target aarch64 -march=armv8.5-a+fp16 -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV85A-FP16 %s
 // GENERICV85A-FP16: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-cpu" "generic" "-target-feature" "+neon" "-target-feature" "+v8.5a" "-target-feature" "+fullfp16"
 
+// RUN: %clang -target aarch64 -march=armv8.6a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV86A %s
+// RUN: %clang -target aarch64 -march=armv8.6-a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV86A %s
+// RUN: %clang -target aarch64 -mlittle-endian -march=armv8.6a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV86A %s
+// RUN: %clang -target aarch64 -mlittle-endian -march=armv8.6-a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV86A %s
+// RUN: %clang -target aarch64_be -mlittle-endian -march=armv8.6a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV86A %s
+// RUN: %clang -target aarch64_be -mlittle-endian -march=armv8.6-a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV86A %s
+// GENERICV86A: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-cpu" "generic" "-target-feature" "+neon" "-target-feature" "+v8.6a"
+
+// RUN: %clang -target aarch64_be -march=armv8.6a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV86A-BE %s
+// RUN: %clang -target aarch64_be -march=armv8.6-a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV86A-BE %s
+// RUN: %clang -target aarch64 -mbig-endian -march=armv8.6a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV86A-BE %s
+// RUN: %clang -target aarch64 -mbig-endian -march=armv8.6-a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV86A-BE %s
+// RUN: %clang -target aarch64_be -mbig-endian -march=armv8.6a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV86A-BE %s
+// RUN: %clang -target aarch64_be -mbig-endian -march=armv8.6-a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV86A-BE %s
+// GENERICV86A-BE: "-cc1"{{.*}} "-triple" "aarch64_be{{.*}}" "-target-cpu" "generic" "-target-feature" "+neon" "-target-feature" "+v8.6a"
+
+// The SVE extension is an optional extension for Armv8-A.
+// RUN: %clang -target aarch64 -march=armv8a+sve -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV8A-SVE %s
+// RUN: %clang -target aarch64 -march=armv8.6a+sve -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV8A-SVE %s
+// GENERICV8A-SVE: "-target-feature" "+sve"
+// RUN: %clang -target aarch64 -march=armv8a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV8A-NOSVE %s
+// RUN: %clang -target aarch64 -march=armv8.6a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV8A-NOSVE %s
+// GENERICV8A-NOSVE-NOT: "-target-feature" "+sve"
+
+// The BFloat16 extension is a mandatory component of the Armv8.6-A extensions, but is permitted as an
+// optional feature for any implementation of Armv8.2-A to Armv8.5-A (inclusive)
+// RUN: %clang -target aarch64 -march=armv8.5a+bf16 -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV85A-BF16 %s
+// GENERICV85A-BF16: "-target-feature" "+bf16"
+// RUN: %clang -target aarch64 -march=armv8.5a+bf16+nobf16 -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV85A-BF16-NO-BF16 %s
+// GENERICV85A-BF16-NO-BF16: "-target-feature" "-bf16"
+// RUN: %clang -target aarch64 -march=armv8.5a+bf16+sve -### -c %s 2>&1 | FileCheck -check-prefixes=GENERICV85A-BF16-SVE %s
+// GENERICV85A-BF16-SVE: "-target-feature" "+bf16" "-target-feature" "+sve"
+
 // fullfp16 is off by default for v8a, feature must not be mentioned
 // RUN: %clang -target aarch64 -march=armv8a  -### -c %s 2>&1 | FileCheck -check-prefix=V82ANOFP16 -check-prefix=GENERIC %s
 // RUN: %clang -target aarch64 -march=armv8-a -### -c %s 2>&1 | FileCheck -check-prefix=V82ANOFP16 -check-prefix=GENERIC %s
diff --git a/clang/test/Driver/arm-cortex-cpus.c b/clang/test/Driver/arm-cortex-cpus.c
index 12129e4ee25fc..c1ce9c08ca874 100644
--- a/clang/test/Driver/arm-cortex-cpus.c
+++ b/clang/test/Driver/arm-cortex-cpus.c
@@ -335,6 +335,23 @@
 // RUN: %clang -target arm -march=armebv8.5-a -mbig-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-V85A %s
 // CHECK-BE-V85A: "-cc1"{{.*}} "-triple" "armebv8.5{{.*}}" "-target-cpu" "generic"
 
+// RUN: %clang -target armv8.6a -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-V86A %s
+// RUN: %clang -target arm -march=armv8.6a -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-V86A %s
+// RUN: %clang -target arm -march=armv8.6-a -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-V86A %s
+// RUN: %clang -target arm -march=armv8.6a -mlittle-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-V86A %s
+// RUN: %clang -target armv8.6a -mlittle-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-V86A %s
+// RUN: %clang -target arm -march=armv8.6a -mlittle-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-V86A %s
+// RUN: %clang -target arm -mlittle-endian -march=armv8.6-a -mlittle-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-V86A %s
+// CHECK-V86A: "-cc1"{{.*}} "-triple" "armv8.6{{.*}}" "-target-cpu" "generic"
+
+// RUN: %clang -target armebv8.6a -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-V86A %s
+// RUN: %clang -target armv8.6a -mbig-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-V86A %s
+// RUN: %clang -target armeb -march=armebv8.6a -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-V86A %s
+// RUN: %clang -target armeb -march=armebv8.6-a -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-V86A %s
+// RUN: %clang -target arm -march=armebv8.6a -mbig-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-V86A %s
+// RUN: %clang -target arm -march=armebv8.6-a -mbig-endian -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-BE-V86A %s
+// CHECK-BE-V86A: "-cc1"{{.*}} "-triple" "armebv8.6{{.*}}" "-target-cpu" "generic"
+
 // Once we have CPUs with optional v8.2-A FP16, we will need a way to turn it
 // on and off. Cortex-A53 is a placeholder for now.
 // RUN: %clang -target armv8a-linux-eabi -mcpu=cortex-a53+fp16 -### -c %s 2>&1 | FileCheck --check-prefix CHECK-CORTEX-A53-FP16 %s
@@ -432,6 +449,9 @@
 // RUN: %clang -target armv8a-linux-eabi -march=armv8.5-a+fp16 -### -c %s 2>&1 | FileCheck --check-prefix CHECK-V85A-FP16 %s
 // CHECK-V85A-FP16: "-cc1"{{.*}} "-triple" "armv8.5{{.*}}" "-target-cpu" "generic" {{.*}}"-target-feature" "+fullfp16"
 
+// RUN: %clang -target armv8a-linux-eabi -march=armv8.6-a+bf16 -### -c %s 2>&1 | FileCheck --check-prefix CHECK-V86A-BF16 %s
+// CHECK-V86A-BF16: "-cc1"{{.*}} "-triple" "armv8.6{{.*}}" "-target-cpu" "generic" {{.*}}"-target-feature" "+bf16"
+
 // RUN: %clang -target arm -march=armv8.2-a+fp16 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-FULLFP16-SOFT %s
 // RUN: %clang -target arm -march=armv8.2-a+fp16fml -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-FULLFP16-SOFT %s
 // RUN: %clang -target arm -march=armv8.2-a+fp16+fp16fml -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-FULLFP16-SOFT %s
diff --git a/clang/test/Driver/clang-translation.c b/clang/test/Driver/clang-translation.c
index 79d8f6f18ab03..2f02970a2a8ee 100644
--- a/clang/test/Driver/clang-translation.c
+++ b/clang/test/Driver/clang-translation.c
@@ -4,7 +4,6 @@
 // I386: "-disable-free"
 // I386: "-mrelocation-model" "static"
 // I386: "-mframe-pointer=all"
-// I386: "-masm-verbose"
 // I386: "-munwind-tables"
 // I386: "-Os"
 // I386: "-fvisibility"
diff --git a/clang/test/Driver/cuda-flush-denormals-to-zero.cu b/clang/test/Driver/cuda-flush-denormals-to-zero.cu
index 84ef358758fd7..5b1046b0cb129 100644
--- a/clang/test/Driver/cuda-flush-denormals-to-zero.cu
+++ b/clang/test/Driver/cuda-flush-denormals-to-zero.cu
@@ -2,10 +2,20 @@
 // -fcuda-flush-denormals-to-zero. This should be translated to
 // -fdenormal-fp-math-f32=preserve-sign
 
-// RUN: %clang -no-canonical-prefixes -### -target x86_64-linux-gnu -c -march=haswell--cuda-gpu-arch=sm_20 -fcuda-flush-denormals-to-zero -nocudainc -nocudalib %s 2>&1 | FileCheck -check-prefix=FTZ %s
-// RUN: %clang -no-canonical-prefixes -### -target x86_64-linux-gnu -c -march=haswell--cuda-gpu-arch=sm_20 -fno-cuda-flush-denormals-to-zero -nocudainc -nocudalib %s 2>&1 | FileCheck -check-prefix=NOFTZ %s
-// RUN: %clang -no-canonical-prefixes -### -target x86_64-linux-gnu -c -march=haswell--cuda-gpu-arch=sm_10 -fcuda-flush-denormals-to-zero -nocudainc -nocudalib %s 2>&1 | FileCheck -check-prefix=FTZ %s
-// RUN: %clang -no-canonical-prefixes -### -target x86_64-linux-gnu -c -march=haswell--cuda-gpu-arch=sm_10 -fno-cuda-flush-denormals-to-zero -nocudainc -nocudalib %s 2>&1 | FileCheck -check-prefix=NOFTZ %s
+// RUN: %clang -no-canonical-prefixes -### -target x86_64-linux-gnu -c -march=haswell --cuda-gpu-arch=sm_20 -fcuda-flush-denormals-to-zero -nocudainc -nocudalib %s 2>&1 | FileCheck -check-prefix=FTZ %s
+// RUN: %clang -no-canonical-prefixes -### -target x86_64-linux-gnu -c -march=haswell --cuda-gpu-arch=sm_20 -fno-cuda-flush-denormals-to-zero -nocudainc -nocudalib %s 2>&1 | FileCheck -check-prefix=NOFTZ %s
+// RUN: %clang -no-canonical-prefixes -### -target x86_64-linux-gnu -c -march=haswell --cuda-gpu-arch=sm_70 -fcuda-flush-denormals-to-zero -nocudainc -nocudalib %s 2>&1 | FileCheck -check-prefix=FTZ %s
+// RUN: %clang -no-canonical-prefixes -### -target x86_64-linux-gnu -c -march=haswell --cuda-gpu-arch=sm_70 -fno-cuda-flush-denormals-to-zero -nocudainc -nocudalib %s 2>&1 | FileCheck -check-prefix=NOFTZ %s
+
+// Test explicit argument.
+// RUN: %clang -no-canonical-prefixes -### -target x86_64-linux-gnu -c -march=haswell --cuda-gpu-arch=gfx803 -fcuda-flush-denormals-to-zero -nocudainc -nogpulib %s 2>&1 | FileCheck -check-prefix=FTZ %s
+// RUN: %clang -no-canonical-prefixes -### -target x86_64-linux-gnu -c -march=haswell --cuda-gpu-arch=gfx803 -fno-cuda-flush-denormals-to-zero -nocudainc -nogpulib %s 2>&1 | FileCheck -check-prefix=NOFTZ %s
+// RUN: %clang -x hip -no-canonical-prefixes -### -target x86_64-linux-gnu -c -march=haswell --cuda-gpu-arch=gfx900 -fcuda-flush-denormals-to-zero -nocudainc -nogpulib %s 2>&1 | FileCheck -check-prefix=FTZ %s
+// RUN: %clang -x hip -no-canonical-prefixes -### -target x86_64-linux-gnu -c -march=haswell --cuda-gpu-arch=gfx900 -fno-cuda-flush-denormals-to-zero -nocudainc -nogpulib %s 2>&1 | FileCheck -check-prefix=NOFTZ %s
+
+// Test the default changing with no argument based on the subtarget.
+// RUN: %clang -x hip -no-canonical-prefixes -### -target x86_64-linux-gnu -c -march=haswell --cuda-gpu-arch=gfx803 -nocudainc -nogpulib %s 2>&1 | FileCheck -check-prefix=FTZ %s
+// RUN: %clang -x hip -no-canonical-prefixes -### -target x86_64-linux-gnu -c -march=haswell --cuda-gpu-arch=gfx900 -nocudainc -nogpulib %s 2>&1 | FileCheck -check-prefix=NOFTZ %s
 
 // CPUFTZ-NOT: -fdenormal-fp-math
 
diff --git a/clang/test/Driver/darwin-objc-options.m b/clang/test/Driver/darwin-objc-options.m
index 3e21fb38c0a95..60827f2937ed6 100644
--- a/clang/test/Driver/darwin-objc-options.m
+++ b/clang/test/Driver/darwin-objc-options.m
@@ -31,5 +31,12 @@
 // CHECK-CHECK-I386_IOS-NOT: -fobjc-dispatch-method
 // CHECK-CHECK-I386_IOS: darwin-objc-options
 
+/// Don't add -fobjc-runtime for non-ObjC input.
+// RUN: touch %t.c
+// RUN: %clang -target x86_64-apple-darwin -x objective-c -S -### %t.c 2>&1 | FileCheck --check-prefix=F %s
+// RUN: %clang -target x86_64-apple-darwin -S -### %t.c 2>&1 | FileCheck --check-prefix=NO_F %s
+// F: -fobjc-runtime=
+// NO_F-NOT: -fobjc-runtime=
+
 // Don't crash with an unexpected target triple.
 // RUN: %clang -target i386-apple-ios7 -S -### %s
diff --git a/clang/test/Driver/fdiagnostics-show-option.c b/clang/test/Driver/fdiagnostics-show-option.c
new file mode 100644
index 0000000000000..a574503caceef
--- /dev/null
+++ b/clang/test/Driver/fdiagnostics-show-option.c
@@ -0,0 +1,7 @@
+/// -fdiagnostics-show-option is the default
+// RUN: %clang -### -c %s 2>&1 | FileCheck --check-prefix=ENABLED %s
+// ENABLED-NOT: "-fno-diagnostics-show-option"
+
+// RUN: %clang -### -c %s -fdiagnostics-show-option -fno-diagnostics-show-option 2>&1 | \
+// RUN:   FileCheck --check-prefix=DISABLED %s
+// DISABLED: "-fno-diagnostics-show-option"
diff --git a/clang/test/Driver/fmessage-length.c b/clang/test/Driver/fmessage-length.c
new file mode 100644
index 0000000000000..638add05b2e52
--- /dev/null
+++ b/clang/test/Driver/fmessage-length.c
@@ -0,0 +1,9 @@
+// RUN: %clang -### -c %s -fmessage-length=80 2>&1 | FileCheck %s
+// CHECK: "-fmessage-length=80"
+
+/// Omit -fmessage-length=0 to simplify common CC1 command lines.
+// RUN: %clang -### -c %s -fmessage-length=0 2>&1 | FileCheck --check-prefix=ZERO %s
+// ZERO-NOT: "-fmessage-length=0"
+
+// RUN: %clang -### -c %s -fmessage-length=nan 2>&1 | FileCheck --check-prefix=ERR %s
+// ERR: error: invalid argument 'nan' to -fmessage-length=
diff --git a/clang/test/Driver/global-isel.c b/clang/test/Driver/global-isel.c
index f4fc6a758ac25..66f196b03c1ea 100644
--- a/clang/test/Driver/global-isel.c
+++ b/clang/test/Driver/global-isel.c
@@ -1,24 +1,35 @@
 // REQUIRES: x86-registered-target,aarch64-registered-target
 
+// RUN: %clang -fglobal-isel -S -### %s 2>&1 | FileCheck --check-prefix=ENABLED %s
+// RUN: %clang -fno-global-isel -S -### %s 2>&1 | FileCheck --check-prefix=DISABLED %s
+
+// RUN: %clang -target aarch64 -fglobal-isel -S %s -### 2>&1 | FileCheck --check-prefix=ARM64-DEFAULT %s
+// RUN: %clang -target aarch64 -fglobal-isel -S -O0 %s -### 2>&1 | FileCheck --check-prefix=ARM64-O0 %s
+// RUN: %clang -target aarch64 -fglobal-isel -S -O2 %s -### 2>&1 | FileCheck --check-prefix=ARM64-O2 %s
+// RUN: %clang -target aarch64 -fglobal-isel -Wno-global-isel -S -O2 %s -### 2>&1 | FileCheck --check-prefix=ARM64-O2-NOWARN %s
+
+// RUN: %clang -target x86_64 -fglobal-isel -S %s -### 2>&1 | FileCheck --check-prefix=X86_64 %s
+
+// Now test the aliases.
+
 // RUN: %clang -fexperimental-isel -S -### %s 2>&1 | FileCheck --check-prefix=ENABLED %s
 // RUN: %clang -fno-experimental-isel -S -### %s 2>&1 | FileCheck --check-prefix=DISABLED %s
 
 // RUN: %clang -target aarch64 -fexperimental-isel -S %s -### 2>&1 | FileCheck --check-prefix=ARM64-DEFAULT %s
 // RUN: %clang -target aarch64 -fexperimental-isel -S -O0 %s -### 2>&1 | FileCheck --check-prefix=ARM64-O0 %s
 // RUN: %clang -target aarch64 -fexperimental-isel -S -O2 %s -### 2>&1 | FileCheck --check-prefix=ARM64-O2 %s
-// RUN: %clang -target aarch64 -fexperimental-isel -Wno-experimental-isel -S -O2 %s -### 2>&1 | FileCheck --check-prefix=ARM64-O2-NOWARN %s
 
 // RUN: %clang -target x86_64 -fexperimental-isel -S %s -### 2>&1 | FileCheck --check-prefix=X86_64 %s
 
 // ENABLED: "-mllvm" "-global-isel=1"
 // DISABLED: "-mllvm" "-global-isel=0"
 
-// ARM64-DEFAULT-NOT: warning: -fexperimental-sel
+// ARM64-DEFAULT-NOT: warning: -fglobal-isel
 // ARM64-DEFAULT-NOT: "-global-isel-abort=2"
-// ARM64-O0-NOT: warning: -fexperimental-sel
-// ARM64-O2: warning: -fexperimental-isel support is incomplete for this architecture at the current optimization level
+// ARM64-O0-NOT: warning: -fglobal-isel
+// ARM64-O2: warning: -fglobal-isel support is incomplete for this architecture at the current optimization level
 // ARM64-O2: "-mllvm" "-global-isel-abort=2"
-// ARM64-O2-NOWARN-NOT: warning: -fexperimental-isel
+// ARM64-O2-NOWARN-NOT: warning: -fglobal-isel
 
-// X86_64: -fexperimental-isel support for the 'x86_64' architecture is incomplete
+// X86_64: -fglobal-isel support for the 'x86_64' architecture is incomplete
 // X86_64: "-mllvm" "-global-isel-abort=2"
diff --git a/clang/test/Driver/hip-device-libs.hip b/clang/test/Driver/hip-device-libs.hip
index b79cb70cbe683..cb1747c2d7984 100644
--- a/clang/test/Driver/hip-device-libs.hip
+++ b/clang/test/Driver/hip-device-libs.hip
@@ -2,23 +2,94 @@
 // REQUIRES: x86-registered-target
 // REQUIRES: amdgpu-registered-target
 
-// Test flush-denormals-to-zero enabled uses oclc_daz_opt_on
+// Test if oclc_daz_opt_on or if oclc_daz_opt_off is linked depending on
+// expected denormal mode.
 
+// Test subtarget with flushing on by default.
+// RUN: %clang -### -target x86_64-linux-gnu \
+// RUN:   --cuda-gpu-arch=gfx803 \
+// RUN:   --hip-device-lib-path=%S/Inputs/hip_dev_lib   \
+// RUN:   %S/Inputs/hip_multiple_inputs/b.hip \
+// RUN: 2>&1 | FileCheck %s --check-prefixes=COM,FLUSHD
+
+
+// Test subtarget with flushing off by ddefault.
 // RUN: %clang -### -target x86_64-linux-gnu \
 // RUN:   --cuda-gpu-arch=gfx900 \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_dev_lib \
+// RUN:   %S/Inputs/hip_multiple_inputs/b.hip \
+// RUN: 2>&1 | FileCheck %s --check-prefixes=COM,NOFLUSHD
+
+
+// Test explicit flag, opposite of target default.
+// RUN: %clang -### -target x86_64-linux-gnu \
+// RUN:   --cuda-gpu-arch=gfx900 \
 // RUN:   -fcuda-flush-denormals-to-zero \
+// RUN:   --hip-device-lib-path=%S/Inputs/hip_dev_lib \
 // RUN:   %S/Inputs/hip_multiple_inputs/b.hip \
 // RUN: 2>&1 | FileCheck %s --check-prefixes=COM,FLUSHD
 
-// Test flush-denormals-to-zero disabled uses oclc_daz_opt_off
 
+// Test explicit flag, opposite of target default.
+// RUN: %clang -### -target x86_64-linux-gnu \
+// RUN:   --cuda-gpu-arch=gfx803 \
+// RUN:   -fno-cuda-flush-denormals-to-zero \
+// RUN:   --hip-device-lib-path=%S/Inputs/hip_dev_lib \
+// RUN:   %S/Inputs/hip_multiple_inputs/b.hip \
+// RUN: 2>&1 | FileCheck %s --check-prefixes=COM,NOFLUSHD
+
+
+// Test explicit flag, same as target default.
 // RUN: %clang -### -target x86_64-linux-gnu \
 // RUN:   --cuda-gpu-arch=gfx900 \
+// RUN:   -fno-cuda-flush-denormals-to-zero \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_dev_lib \
 // RUN:   %S/Inputs/hip_multiple_inputs/b.hip \
 // RUN: 2>&1 | FileCheck %s --check-prefixes=COM,NOFLUSHD
 
+
+// Test explicit flag, same as target default.
+// RUN: %clang -### -target x86_64-linux-gnu \
+// RUN:   --cuda-gpu-arch=gfx803 \
+// RUN:   -fcuda-flush-denormals-to-zero \
+// RUN:   --hip-device-lib-path=%S/Inputs/hip_dev_lib \
+// RUN:   %S/Inputs/hip_multiple_inputs/b.hip \
+// RUN: 2>&1 | FileCheck %s --check-prefixes=COM,FLUSHD
+
+
+// Test last flag wins, not flushing
+// RUN: %clang -### -target x86_64-linux-gnu \
+// RUN:   --cuda-gpu-arch=gfx803 \
+// RUN:   -fcuda-flush-denormals-to-zero -fno-cuda-flush-denormals-to-zero \
+// RUN:   --hip-device-lib-path=%S/Inputs/hip_dev_lib \
+// RUN:   %S/Inputs/hip_multiple_inputs/b.hip \
+// RUN: 2>&1 | FileCheck %s --check-prefixes=COM,NOFLUSHD
+
+
+// RUN: %clang -### -target x86_64-linux-gnu \
+// RUN:   --cuda-gpu-arch=gfx900 \
+// RUN:   -fcuda-flush-denormals-to-zero -fno-cuda-flush-denormals-to-zero \
+// RUN:   --hip-device-lib-path=%S/Inputs/hip_dev_lib   \
+// RUN:   %S/Inputs/hip_multiple_inputs/b.hip \
+// RUN: 2>&1 | FileCheck %s --check-prefixes=COM,NOFLUSHD
+
+
+// RUN: %clang -### -target x86_64-linux-gnu \
+// RUN:   --cuda-gpu-arch=gfx900 \
+// RUN:   -fno-cuda-flush-denormals-to-zero -fcuda-flush-denormals-to-zero \
+// RUN:   --hip-device-lib-path=%S/Inputs/hip_dev_lib   \
+// RUN:   %S/Inputs/hip_multiple_inputs/b.hip \
+// RUN: 2>&1 | FileCheck %s --check-prefixes=COM,FLUSHD
+
+
+// RUN: %clang -### -target x86_64-linux-gnu \
+// RUN:   --cuda-gpu-arch=gfx803 \
+// RUN:   -fno-cuda-flush-denormals-to-zero -fcuda-flush-denormals-to-zero \
+// RUN:   --hip-device-lib-path=%S/Inputs/hip_dev_lib \
+// RUN:   %S/Inputs/hip_multiple_inputs/b.hip \
+// RUN: 2>&1 | FileCheck %s --check-prefixes=COM,FLUSHD
+
+
 // Test environment variable HIP_DEVICE_LIB_PATH
 
 // RUN: env HIP_DEVICE_LIB_PATH=%S/Inputs/hip_dev_lib \
@@ -33,4 +104,3 @@
 // COM-SAME: "-mlink-builtin-bitcode" "{{.*}}ockl.amdgcn.bc"
 // FLUSHD-SAME: "-mlink-builtin-bitcode" "{{.*}}oclc_daz_opt_on.amdgcn.bc"
 // NOFLUSHD-SAME: "-mlink-builtin-bitcode" "{{.*}}oclc_daz_opt_off.amdgcn.bc"
-
diff --git a/clang/test/Driver/integrated-as.c b/clang/test/Driver/integrated-as.c
index df5cf1a17ecce..170515579b1ac 100644
--- a/clang/test/Driver/integrated-as.c
+++ b/clang/test/Driver/integrated-as.c
@@ -12,6 +12,7 @@
 
 // NOFIAS-NOT: cc1as
 // NOFIAS: -cc1
+// NOFIAS: "-fno-verbose-asm"
 // NOFIAS: -no-integrated-as
 
 // RUN: %clang -target arm-linux-androideabi -### \
diff --git a/clang/test/Driver/lto-jobs.c b/clang/test/Driver/lto-jobs.c
index 539867713b07c..c28d0ad300f4e 100644
--- a/clang/test/Driver/lto-jobs.c
+++ b/clang/test/Driver/lto-jobs.c
@@ -8,4 +8,4 @@
 // RUN: %clang -target x86_64-apple-darwin13.3.0 -### %s -flto=thin -flto-jobs=5 2> %t
 // RUN: FileCheck -check-prefix=CHECK-LINK-THIN-JOBS2-ACTION < %t %s
 //
-// CHECK-LINK-THIN-JOBS2-ACTION: "-mllvm" "-threads=5"
+// CHECK-LINK-THIN-JOBS2-ACTION: "-mllvm" "-threads={{[0-9]+}}"
diff --git a/clang/test/Driver/ps4cpu.c b/clang/test/Driver/ps4cpu.c
new file mode 100644
index 0000000000000..ac5f3fcdc8441
--- /dev/null
+++ b/clang/test/Driver/ps4cpu.c
@@ -0,0 +1,17 @@
+// REQUIRES: x86-registered-target
+
+// Test that the driver always emits -fno-use-init-array on the PS4 target
+// since its ABI does not support the .init_array section.
+
+// RUN: %clang -c %s -target x86_64-scei-ps4 -### 2>&1                      \
+// RUN:   | FileCheck %s
+// RUN: %clang -c %s -target x86_64-scei-ps4 -fno-use-init-array -### 2>&1  \
+// RUN:   | FileCheck %s
+// RUN: %clang -c %s -target x86_64-scei-ps4 -fuse-init-array -### 2>&1     \
+// RUN:   | FileCheck %s --check-prefix=CHECK-ERROR
+
+// CHECK: "-fno-use-init-array"
+// CHECK-NOT: "-fuse-init-array"
+
+// CHECK-ERROR: unsupported option '-fuse-init-array' for target 'x86_64-scei-ps4'
+
diff --git a/clang/test/Driver/rewrite-legacy-objc.m b/clang/test/Driver/rewrite-legacy-objc.m
index dc92dd4bf1073..fb7df4b97c061 100644
--- a/clang/test/Driver/rewrite-legacy-objc.m
+++ b/clang/test/Driver/rewrite-legacy-objc.m
@@ -3,11 +3,11 @@
 // TEST0: clang{{.*}}" "-cc1"
 // TEST0: "-rewrite-objc"
 // FIXME: CHECK-NOT is broken somehow, it doesn't work here. Check adjacency instead.
-// TEST0: "-fmessage-length" "0" "-stack-protector" "1" "-fblocks" "-fencode-extended-block-signature" "-fregister-global-dtors-with-atexit" "-fgnuc-version=4.2.1" "-fobjc-runtime=macosx-fragile" "-fno-objc-infer-related-result-type" "-fobjc-exceptions" "-fexceptions" "-fmax-type-align=16" "-fdiagnostics-show-option"
+// TEST0: "-stack-protector" "1" "-fblocks" "-fencode-extended-block-signature" "-fregister-global-dtors-with-atexit" "-fgnuc-version=4.2.1" "-fobjc-runtime=macosx-fragile" "-fno-objc-infer-related-result-type" "-fobjc-exceptions" "-fexceptions" "-fmax-type-align=16"
 // TEST0: rewrite-legacy-objc.m"
 // RUN: %clang -no-canonical-prefixes -target i386-apple-macosx10.9.0 -rewrite-legacy-objc %s -o - -### 2>&1 | \
 // RUN:   FileCheck -check-prefix=TEST1 %s
 // RUN: %clang -no-canonical-prefixes -target i386-apple-macosx10.6.0 -rewrite-legacy-objc %s -o - -### 2>&1 | \
 // RUN:   FileCheck -check-prefix=TEST2 %s
-// TEST1: "-fmessage-length" "0" "-stack-protector" "1" "-fblocks" "-fencode-extended-block-signature" "-fregister-global-dtors-with-atexit" "-fgnuc-version=4.2.1" "-fobjc-runtime=macosx-fragile" "-fobjc-subscripting-legacy-runtime" "-fno-objc-infer-related-result-type" "-fobjc-exceptions" "-fmax-type-align=16" "-fdiagnostics-show-option"
-// TEST2: "-fmessage-length" "0" "-stack-protector" "1" "-fblocks" "-fencode-extended-block-signature" "-fregister-global-dtors-with-atexit" "-fgnuc-version=4.2.1" "-fobjc-runtime=macosx-fragile" "-fobjc-subscripting-legacy-runtime" "-fno-objc-infer-related-result-type" "-fobjc-exceptions" "-fmax-type-align=16" "-fdiagnostics-show-option"
+// TEST1: "-stack-protector" "1" "-fblocks" "-fencode-extended-block-signature" "-fregister-global-dtors-with-atexit" "-fgnuc-version=4.2.1" "-fobjc-runtime=macosx-fragile" "-fobjc-subscripting-legacy-runtime" "-fno-objc-infer-related-result-type" "-fobjc-exceptions" "-fmax-type-align=16"
+// TEST2: "-stack-protector" "1" "-fblocks" "-fencode-extended-block-signature" "-fregister-global-dtors-with-atexit" "-fgnuc-version=4.2.1" "-fobjc-runtime=macosx-fragile" "-fobjc-subscripting-legacy-runtime" "-fno-objc-infer-related-result-type" "-fobjc-exceptions" "-fmax-type-align=16"
diff --git a/clang/test/Driver/rewrite-objc.m b/clang/test/Driver/rewrite-objc.m
index b04062992b7f4..6073dcdfdafe5 100644
--- a/clang/test/Driver/rewrite-objc.m
+++ b/clang/test/Driver/rewrite-objc.m
@@ -3,4 +3,4 @@
 // TEST0: clang{{.*}}" "-cc1"
 // TEST0: "-rewrite-objc"
 // FIXME: CHECK-NOT is broken somehow, it doesn't work here. Check adjacency instead.
-// TEST0: "-fmessage-length" "0" "-stack-protector" "1" "-fblocks" "-fencode-extended-block-signature" "-fregister-global-dtors-with-atexit" "-fgnuc-version=4.2.1" "-fobjc-runtime=macosx" "-fno-objc-infer-related-result-type" "-fobjc-exceptions" "-fexceptions" "-fmax-type-align=16" "-fdiagnostics-show-option"
+// TEST0: "-stack-protector" "1" "-fblocks" "-fencode-extended-block-signature" "-fregister-global-dtors-with-atexit" "-fgnuc-version=4.2.1" "-fobjc-runtime=macosx" "-fno-objc-infer-related-result-type" "-fobjc-exceptions" "-fexceptions" "-fmax-type-align=16"
diff --git a/clang/test/Driver/show-option-names.c b/clang/test/Driver/show-option-names.c
deleted file mode 100644
index 9843a4371f14b..0000000000000
--- a/clang/test/Driver/show-option-names.c
+++ /dev/null
@@ -1,7 +0,0 @@
-// REQUIRES: x86-registered-target
-
-// RUN: %clang -target x86_64-apple-darwin -fsyntax-only -isysroot /FOO %s 2>&1 | FileCheck --check-prefix=CHECK-SHOW-OPTION-NAMES %s
-// CHECK-SHOW-OPTION-NAMES: warning: no such sysroot directory: '{{([A-Za-z]:.*)?}}/FOO' [-Wmissing-sysroot]
-
-// RUN: %clang -target x86_64-apple-darwin -fsyntax-only -fno-diagnostics-show-option -isysroot /FOO %s 2>&1 | FileCheck --check-prefix=CHECK-NO-SHOW-OPTION-NAMES %s
-// CHECK-NO-SHOW-OPTION-NAMES: warning: no such sysroot directory: '{{([A-Za-z]:.*)?}}/FOO'{{$}}
diff --git a/clang/test/Driver/x86-target-features.c b/clang/test/Driver/x86-target-features.c
index 9a406b504b24e..44a3d2e1164bc 100644
--- a/clang/test/Driver/x86-target-features.c
+++ b/clang/test/Driver/x86-target-features.c
@@ -154,6 +154,16 @@
 // SLH: "-mspeculative-load-hardening"
 // NO-SLH-NOT: retpoline
 
+// RUN: %clang -target i386-linux-gnu -mlvi-cfi %s -### -o %t.o 2>&1 | FileCheck -check-prefix=LVICFI %s
+// RUN: %clang -target i386-linux-gnu -mno-lvi-cfi %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-LVICFI %s
+// LVICFI: "-target-feature" "+lvi-cfi"
+// NO-LVICFI-NOT: lvi-cfi
+
+// RUN: %clang -target i386-linux-gnu -mlvi-hardening %s -### -o %t.o 2>&1 | FileCheck -check-prefix=LVIHARDENING %s
+// RUN: %clang -target i386-linux-gnu -mno-lvi-hardening %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-LVIHARDENING %s
+// LVIHARDENING: "-target-feature" "+lvi-load-hardening" "-target-feature" "+lvi-cfi"
+// NO-LVIHARDENING-NOT: lvi
+
 // RUN: %clang -target i386-linux-gnu -mwaitpkg %s -### -o %t.o 2>&1 | FileCheck -check-prefix=WAITPKG %s
 // RUN: %clang -target i386-linux-gnu -mno-waitpkg %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-WAITPKG %s
 // WAITPKG: "-target-feature" "+waitpkg"
@@ -198,3 +208,8 @@
 // RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mno-vzeroupper %s -### -o %t.o 2>&1 | FileCheck --check-prefix=NO-VZEROUPPER %s
 // VZEROUPPER: "-target-feature" "+vzeroupper"
 // NO-VZEROUPPER: "-target-feature" "-vzeroupper"
+
+// RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mserialize %s -### -o %t.o 2>&1 | FileCheck -check-prefix=SERIALIZE %s
+// RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mno-serialize %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-SERIALIZE %s
+// SERIALIZE: "-target-feature" "+serialize"
+// NO-SERIALIZE: "-target-feature" "-serialize"
diff --git a/clang/test/Frontend/diagnostics-option-names.c b/clang/test/Frontend/diagnostics-option-names.c
index ed0d2ed8ef9e0..71455be0a75b9 100644
--- a/clang/test/Frontend/diagnostics-option-names.c
+++ b/clang/test/Frontend/diagnostics-option-names.c
@@ -1,4 +1,4 @@
-// RUN: not %clang_cc1 -fdiagnostics-show-option -Werror -Weverything %s 2> %t
+// RUN: not %clang_cc1 -Werror -Weverything %s 2> %t
 // RUN: FileCheck < %t %s
 
 int f0(int, unsigned);
diff --git a/clang/test/Frontend/plugin-attribute.cpp b/clang/test/Frontend/plugin-attribute.cpp
new file mode 100644
index 0000000000000..571ede3dc0b11
--- /dev/null
+++ b/clang/test/Frontend/plugin-attribute.cpp
@@ -0,0 +1,25 @@
+// RUN: %clang -fplugin=%llvmshlibdir/Attribute%pluginext -emit-llvm -S %s -o - 2>&1 | FileCheck %s --check-prefix=ATTRIBUTE
+// RUN: not %clang -fplugin=%llvmshlibdir/Attribute%pluginext -emit-llvm -DBAD_ATTRIBUTE -S %s -o - 2>&1 | FileCheck %s --check-prefix=BADATTRIBUTE
+// REQUIRES: plugins, examples
+
+void fn1a() __attribute__((example)) { }
+[[example]] void fn1b() { }
+[[plugin::example]] void fn1c() { }
+void fn2() __attribute__((example("somestring"))) { }
+// ATTRIBUTE: warning: 'example' attribute only applies to functions
+int var1 __attribute__((example("otherstring"))) = 1;
+
+// ATTRIBUTE: [[STR1_VAR:@.+]] = private unnamed_addr constant [10 x i8] c"example()\00"
+// ATTRIBUTE: [[STR2_VAR:@.+]] = private unnamed_addr constant [20 x i8] c"example(somestring)\00"
+// ATTRIBUTE: @llvm.global.annotations = {{.*}}@{{.*}}fn1a{{.*}}[[STR1_VAR]]{{.*}}@{{.*}}fn1b{{.*}}[[STR1_VAR]]{{.*}}@{{.*}}fn1c{{.*}}[[STR1_VAR]]{{.*}}@{{.*}}fn2{{.*}}[[STR2_VAR]]
+
+#ifdef BAD_ATTRIBUTE
+class Example {
+  // BADATTRIBUTE: error: 'example' attribute only allowed at file scope
+  void __attribute__((example)) fn3();
+};
+// BADATTRIBUTE: error: 'example' attribute requires a string
+void fn4() __attribute__((example(123))) { }
+// BADATTRIBUTE: error: 'example' attribute takes no more than 1 argument
+void fn5() __attribute__((example("a","b"))) { }
+#endif
diff --git a/clang/test/Frontend/source-col-map.c b/clang/test/Frontend/source-col-map.c
index 1c8078998c561..b257261b8b2bc 100644
--- a/clang/test/Frontend/source-col-map.c
+++ b/clang/test/Frontend/source-col-map.c
@@ -1,4 +1,4 @@
-// RUN: not %clang_cc1 -fsyntax-only -fmessage-length 75 -o /dev/null -x c < %s 2>&1 | FileCheck %s -strict-whitespace
+// RUN: not %clang_cc1 -fsyntax-only -fmessage-length=75 -o /dev/null -x c < %s 2>&1 | FileCheck %s -strict-whitespace
 // REQUIRES: utf8-capable-terminal
 
 // Test case for the text diagnostics source column conversion crash.
diff --git a/clang/test/Headers/arm-cde-header.c b/clang/test/Headers/arm-cde-header.c
index e9ff649051620..b2f721a6ff9ce 100644
--- a/clang/test/Headers/arm-cde-header.c
+++ b/clang/test/Headers/arm-cde-header.c
@@ -1,7 +1,9 @@
 // RUN: %clang_cc1 -std=c89 -triple thumbv8.1m.main-arm-none-eabi -fallow-half-arguments-and-returns -target-feature +mve.fp -target-feature +cdecp0 -fsyntax-only %s
-// RUN: %clang_cc1 -std=c99 -triple thumbv8.1m.main-arm-none-eabi -fallow-half-arguments-and-returns -target-feature +mve.fp -target-feature +cdecp0 -fsyntax-only %s
-// RUN: %clang_cc1 -std=c11 -triple thumbv8.1m.main-arm-none-eabi -fallow-half-arguments-and-returns -target-feature +mve.fp -target-feature +cdecp0 -fsyntax-only %s
+// RUN: %clang_cc1 -std=c17 -triple thumbv8.1m.main-arm-none-eabi -fallow-half-arguments-and-returns -target-feature +mve -target-feature +cdecp0 -fsyntax-only %s
 // RUN: %clang_cc1 -triple thumbv8.1m.main-arm-none-eabi -fallow-half-arguments-and-returns -target-feature +mve.fp -target-feature +cdecp0 -fsyntax-only %s
+// RUN: %clang_cc1 -xc++ -std=c++98 -triple thumbv8.1m.main-arm-none-eabi -fallow-half-arguments-and-returns -target-feature +mve -target-feature +cdecp0 -fsyntax-only %s
+// RUN: %clang_cc1 -xc++ -std=c++20 -triple thumbv8.1m.main-arm-none-eabi -fallow-half-arguments-and-returns -target-feature +mve.fp -target-feature +cdecp0 -fsyntax-only %s
+// RUN: %clang_cc1 -xc++ -triple thumbv8.1m.main-arm-none-eabi -fallow-half-arguments-and-returns -target-feature +mve.fp -target-feature +cdecp0 -fsyntax-only %s
 
 // Check that the headers don't conflict with each other
 #include <arm_cde.h>
diff --git a/clang/test/Index/pch-with-errors.c b/clang/test/Index/pch-with-errors.c
index 5c94a8a8e4d38..e8711c8e26a9b 100644
--- a/clang/test/Index/pch-with-errors.c
+++ b/clang/test/Index/pch-with-errors.c
@@ -42,3 +42,6 @@ void foo(void) {
 
 // RUN: not c-index-test -write-pch %t.pch foobar.c 2>&1 | FileCheck -check-prefix=NONEXISTENT %s
 // NONEXISTENT: Unable to load translation unit
+
+// RUN: %clang -x c-header %s -o %t-clang.h.pch -Xclang -detailed-preprocessing-record -Xclang -fallow-pch-with-compiler-errors
+// RUN: c-index-test -index-file %s -include %t-clang.h -Xclang -detailed-preprocessing-record | FileCheck -check-prefix=CHECK-INDEX %s
diff --git a/clang/test/InterfaceStubs/bad-format.cpp b/clang/test/InterfaceStubs/bad-format.cpp
index 4d51ac867eb2a..1289067a365ac 100644
--- a/clang/test/InterfaceStubs/bad-format.cpp
+++ b/clang/test/InterfaceStubs/bad-format.cpp
@@ -7,6 +7,9 @@
 // RUN: not %clang -emit-interface-stubs -interface-stub-version=experimental-yaml-elf-v1 %s 2>&1 | \
 // RUN: FileCheck -check-prefix=CHECK-YAML-DEPRECATED %s
 
+// RUN: not %clang -emit-interface-stubs -interface-stub-version=experimental-ifs-v1 %s 2>&1 | \
+// RUN: FileCheck -check-prefix=CHECK-V1-DEPRECATED %s
+
 // RUN: not %clang -emit-interface-stubs -interface-stub-version=bad-format %s 2>&1 | \
 // RUN: FileCheck %s
 
@@ -21,16 +24,22 @@
 // CHECK: error: invalid value
 // CHECK: 'Invalid interface stub format: bad-format.' in 'Must specify a
 // CHECK: valid interface stub format type, ie:
-// CHECK: -interface-stub-version=experimental-ifs-v1'
+// CHECK: -interface-stub-version=experimental-ifs-v2'
 
 // CHECK-TAPI-DEPRECATED: error: invalid value
 // CHECK-TAPI-DEPRECATED: 'Invalid interface stub format:
 // CHECK-TAPI-DEPRECATED: experimental-tapi-elf-v1 is deprecated.' in 'Must
 // CHECK-TAPI-DEPRECATED: specify a valid interface stub format type, ie:
-// CHECK-TAPI-DEPRECATED: -interface-stub-version=experimental-ifs-v1'
+// CHECK-TAPI-DEPRECATED: -interface-stub-version=experimental-ifs-v2'
 
 // CHECK-YAML-DEPRECATED: error: invalid value
 // CHECK-YAML-DEPRECATED: 'Invalid interface stub format:
 // CHECK-YAML-DEPRECATED: experimental-yaml-elf-v1 is deprecated.' in 'Must
 // CHECK-YAML-DEPRECATED: specify a valid interface stub format type, ie:
-// CHECK-YAML-DEPRECATED: -interface-stub-version=experimental-ifs-v1'
+// CHECK-YAML-DEPRECATED: -interface-stub-version=experimental-ifs-v2'
+
+// CHECK-V1-DEPRECATED: error: invalid value
+// CHECK-V1-DEPRECATED: 'Invalid interface stub format:
+// CHECK-V1-DEPRECATED: experimental-ifs-v1 is deprecated.' in 'Must
+// CHECK-V1-DEPRECATED: specify a valid interface stub format type, ie:
+// CHECK-V1-DEPRECATED: -interface-stub-version=experimental-ifs-v2'
diff --git a/clang/test/InterfaceStubs/blocks.c b/clang/test/InterfaceStubs/blocks.c
index 927f2bf288696..8e2a01159aaba 100644
--- a/clang/test/InterfaceStubs/blocks.c
+++ b/clang/test/InterfaceStubs/blocks.c
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -emit-interface-stubs -fblocks -o - %s | FileCheck %s
 
-// CHECK: --- !experimental-ifs-v1
-// CHECK-NEXT: IfsVersion: 1.0
+// CHECK: --- !experimental-ifs-v2
+// CHECK-NEXT: IfsVersion: 2.0
 // CHECK-NEXT: Triple:
 // CHECK-NEXT: ObjectFileFormat: ELF
 // CHECK-NEXT: Symbols:
diff --git a/clang/test/InterfaceStubs/class-template-partial-specialization.cpp b/clang/test/InterfaceStubs/class-template-partial-specialization.cpp
index 4c0edaa2dd8f7..b6580861de8bf 100644
--- a/clang/test/InterfaceStubs/class-template-partial-specialization.cpp
+++ b/clang/test/InterfaceStubs/class-template-partial-specialization.cpp
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -o - -emit-interface-stubs %s | FileCheck %s
 
-// CHECK:      --- !experimental-ifs-v1
-// CHECK-NEXT: IfsVersion: 1.0
+// CHECK:      --- !experimental-ifs-v2
+// CHECK-NEXT: IfsVersion: 2.0
 // CHECK-NEXT: Triple:
 // CHECK-NEXT: ObjectFileFormat: ELF
 // CHECK-NEXT: Symbols:
diff --git a/clang/test/InterfaceStubs/conflict-type.ifs b/clang/test/InterfaceStubs/conflict-type.ifs
index aaa04775e3176..cc6191900a30d 100644
--- a/clang/test/InterfaceStubs/conflict-type.ifs
+++ b/clang/test/InterfaceStubs/conflict-type.ifs
@@ -7,10 +7,10 @@
 # CHECK-IFS-NEXT: Filename:
 # CHECK-IFS-NEXT: Type Values: Object Func
 
---- !experimental-ifs-v1
-IfsVersion: 1.0
+--- !experimental-ifs-v2
+IfsVersion: 2.0
 Triple: x86_64-linux-gnu
 ObjectFileFormat: ELF
 Symbols:
-  a: { Type: Object, Size: 1 }
+  - { Name: a, Type: Object, Size: 1 }
 ...
diff --git a/clang/test/InterfaceStubs/constructor-using-shadow.cpp b/clang/test/InterfaceStubs/constructor-using-shadow.cpp
index d4b85ac73e56d..e806cc323ee70 100644
--- a/clang/test/InterfaceStubs/constructor-using-shadow.cpp
+++ b/clang/test/InterfaceStubs/constructor-using-shadow.cpp
@@ -1,12 +1,12 @@
 // RUN: %clang_cc1 -o - -emit-interface-stubs %s | FileCheck %s
 
-// CHECK:      --- !experimental-ifs-v1
-// CHECK-NEXT: IfsVersion: 1.0
+// CHECK:      --- !experimental-ifs-v2
+// CHECK-NEXT: IfsVersion: 2.0
 // CHECK-NEXT: Triple:
 // CHECK-NEXT: ObjectFileFormat: ELF
 // CHECK-NEXT: Symbols:
 // CHECK-NEXT: ...
 
- // ConstructorUsingShadowDecl
+// ConstructorUsingShadowDecl
 struct Base { Base(int); };
 struct Derived : public Base { using Base::Base; };
diff --git a/clang/test/InterfaceStubs/cxx-conversion.cpp b/clang/test/InterfaceStubs/cxx-conversion.cpp
index 96425a42b6fc2..f9de07d178502 100644
--- a/clang/test/InterfaceStubs/cxx-conversion.cpp
+++ b/clang/test/InterfaceStubs/cxx-conversion.cpp
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -o - -emit-interface-stubs %s | FileCheck %s
 
-// CHECK:      --- !experimental-ifs-v1
-// CHECK-NEXT: IfsVersion: 1.0
+// CHECK:      --- !experimental-ifs-v2
+// CHECK-NEXT: IfsVersion: 2.0
 // CHECK-NEXT: Triple:
 // CHECK-NEXT: ObjectFileFormat: ELF
 // CHECK-NEXT: Symbols:
diff --git a/clang/test/InterfaceStubs/cxxdeduction-guide.cpp b/clang/test/InterfaceStubs/cxxdeduction-guide.cpp
index f09b9d929ca3e..4d9f24bae5b30 100644
--- a/clang/test/InterfaceStubs/cxxdeduction-guide.cpp
+++ b/clang/test/InterfaceStubs/cxxdeduction-guide.cpp
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -o - -emit-interface-stubs -std=c++17 %s | FileCheck %s
 
-// CHECK:      --- !experimental-ifs-v1
-// CHECK-NEXT: IfsVersion: 1.0
+// CHECK:      --- !experimental-ifs-v2
+// CHECK-NEXT: IfsVersion: 2.0
 // CHECK-NEXT: Triple:
 // CHECK-NEXT: ObjectFileFormat: ELF
 // CHECK-NEXT: Symbols:
diff --git a/clang/test/InterfaceStubs/driver-test3.c b/clang/test/InterfaceStubs/driver-test3.c
index bccd1c9bccd45..a3f3966dbe8f5 100644
--- a/clang/test/InterfaceStubs/driver-test3.c
+++ b/clang/test/InterfaceStubs/driver-test3.c
@@ -8,12 +8,12 @@
 
 // CHECK-OBJ: bar
 
-// CHECK-IFS: --- !experimental-ifs-v1
+// CHECK-IFS: --- !experimental-ifs-v2
 // CHECK-IFS-NEXT: IfsVersion:
 // CHECK-IFS-NEXT: Triple:
 // CHECK-IFS-NEXT: ObjectFileFormat:
 // CHECK-IFS-NEXT: Symbols:
-// CHECK-IFS-NEXT:   "bar" : { Type: Func }
+// CHECK-IFS-NEXT:   - { Name: "bar", Type: Func }
 // CHECK-IFS-NEXT: ...
 
 int bar(int a) { return a; }
diff --git a/clang/test/InterfaceStubs/empty.c b/clang/test/InterfaceStubs/empty.c
new file mode 100644
index 0000000000000..c68c124e513e6
--- /dev/null
+++ b/clang/test/InterfaceStubs/empty.c
@@ -0,0 +1,8 @@
+// RUN: %clang_cc1 -o - -emit-interface-stubs %s | FileCheck %s
+
+// CHECK:      --- !experimental-ifs-v2
+// CHECK-NEXT: IfsVersion: 2.0
+// CHECK-NEXT: Triple:
+// CHECK-NEXT: ObjectFileFormat:
+// CHECK-NEXT: Symbols:
+// CHECK-NEXT: ...
diff --git a/clang/test/InterfaceStubs/func.ifs b/clang/test/InterfaceStubs/func.ifs
index d115523bfda49..9de5213de9d6c 100644
--- a/clang/test/InterfaceStubs/func.ifs
+++ b/clang/test/InterfaceStubs/func.ifs
@@ -7,13 +7,13 @@
 # RUN: %clang -emit-interface-stubs -o - %s %s -emit-merged-ifs | \
 # RUN: FileCheck %s --check-prefixes=CHECK-MERGE-IFS
 
-# CHECK-IFS: --- !experimental-ifs-v1
-# CHECK-IFS-NEXT: IfsVersion:      1.0
+# CHECK-IFS: --- !experimental-ifs-v2
+# CHECK-IFS-NEXT: IfsVersion: 2.0
 # CHECK-IFS-NEXT: Triple:          x86_64-linux-gnu
 # CHECK-IFS-NEXT: ObjectFileFormat: ELF
 # CHECK-IFS-NEXT: Symbols:
-# CHECK-IFS-DAG:   a:               { Type: Func }
-# CHECK-IFS-DAG:   b:               { Type: Object, Size: 4 }
+# CHECK-IFS-DAG:   - { Name: a, Type: Func }
+# CHECK-IFS-DAG:   - { Name: b, Type: Object, Size: 4 }
 # CHECK-IFS: ...
 
 # CHECK-ELF: ELF Header:
@@ -23,18 +23,18 @@
 # CHECK-ELF:   OBJECT  GLOBAL DEFAULT  1 b
 
 # Here we are testing to see if two identical symbols will merge.
-# CHECK-MERGE-IFS: --- !experimental-ifs-v1
-# CHECK-MERGE-IFS-NEXT: IfsVersion:      1.0
+# CHECK-MERGE-IFS: --- !experimental-ifs-v2
+# CHECK-MERGE-IFS-NEXT: IfsVersion: 2.0
 # CHECK-MERGE-IFS-NEXT: Triple:          x86_64-linux-gnu
 # CHECK-MERGE-IFS-NEXT: ObjectFileFormat: ELF
 # CHECK-MERGE-IFS-NEXT: Symbols:
-# CHECK-MERGE-IFS-NEXT:   a:               { Type: Func }
+# CHECK-MERGE-IFS-NEXT:   - { Name: a, Type: Func }
 # CHECK-MERGE-IFS-NEXT: ...
 
---- !experimental-ifs-v1
-IfsVersion: 1.0
+--- !experimental-ifs-v2
+IfsVersion: 2.0
 Triple: x86_64-linux-gnu
 ObjectFileFormat: ELF
 Symbols:
-  a: { Type: Func }
+  - { Name: a, Type: Func }
 ...
diff --git a/clang/test/InterfaceStubs/hidden-class-inheritance.cpp b/clang/test/InterfaceStubs/hidden-class-inheritance.cpp
index 19ba579608ecd..2219fd5b2e8ae 100644
--- a/clang/test/InterfaceStubs/hidden-class-inheritance.cpp
+++ b/clang/test/InterfaceStubs/hidden-class-inheritance.cpp
@@ -14,7 +14,7 @@
 // RUN: -DPARENT_METHOD_VISIBILITY="" -DCHILD_METHOD_VISIBILITY="" %s | \
 // RUN: FileCheck -check-prefix=CHECK-HP %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -o - -emit-interface-stubs \
-// RUN: -interface-stub-version=experimental-ifs-v1 \
+// RUN: -interface-stub-version=experimental-ifs-v2 \
 // RUN: -DPARENT_CLASS_VISIBILITY=HIDDEN -DCHILD_CLASS_VISIBILITY="" \
 // RUN: -DPARENT_METHOD_VISIBILITY="" -DCHILD_METHOD_VISIBILITY="" %s | \
 // RUN: FileCheck -check-prefix=CHECK-HP2 %s
diff --git a/clang/test/InterfaceStubs/indirect-field-decl.cpp b/clang/test/InterfaceStubs/indirect-field-decl.cpp
index d0e5fd26e4b73..2c30b0ee4005a 100644
--- a/clang/test/InterfaceStubs/indirect-field-decl.cpp
+++ b/clang/test/InterfaceStubs/indirect-field-decl.cpp
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -o - -emit-interface-stubs %s | FileCheck %s
 
-// CHECK:      --- !experimental-ifs-v1
-// CHECK-NEXT: IfsVersion: 1.0
+// CHECK:      --- !experimental-ifs-v2
+// CHECK-NEXT: IfsVersion: 2.0
 // CHECK-NEXT: Triple:
 // CHECK-NEXT: ObjectFileFormat: ELF
 // CHECK-NEXT: Symbols:
diff --git a/clang/test/InterfaceStubs/inline.c b/clang/test/InterfaceStubs/inline.c
index 0b0ac83726ad1..1dec4ae677d76 100644
--- a/clang/test/InterfaceStubs/inline.c
+++ b/clang/test/InterfaceStubs/inline.c
@@ -55,8 +55,8 @@ INLINE int foo() {
 // RUN: -c -std=gnu89 -xc %s | llvm-nm - 2>&1 | \
 // RUN: FileCheck -check-prefix=CHECK-SYMBOLS %s
 
-// CHECK-TAPI-DAG: foo" : { Type: Func }
-// CHECK-TAPI-DAG: foo.var" : { Type: Object, Size: 4 }
+// CHECK-TAPI-DAG: foo", Type: Func }
+// CHECK-TAPI-DAG: foo.var",  Type: Object, Size: 4 }
 // CHECK-SYMBOLS-DAG: foo
 // CHECK-SYMBOLS-DAG: foo.var
 #include "inline.h"
diff --git a/clang/test/InterfaceStubs/lambda.cpp b/clang/test/InterfaceStubs/lambda.cpp
index e892f1eee11c6..a167f6556b942 100644
--- a/clang/test/InterfaceStubs/lambda.cpp
+++ b/clang/test/InterfaceStubs/lambda.cpp
@@ -1,11 +1,11 @@
 // RUN: %clang_cc1 -triple %itanium_abi_triple -emit-interface-stubs -o - %s \
 // RUN:     | FileCheck %s
 
-// CHECK: --- !experimental-ifs-v1
-// CHECK-NEXT: IfsVersion: 1.0
+// CHECK: --- !experimental-ifs-v2
+// CHECK-NEXT: IfsVersion: 2.0
 // CHECK-NEXT: Triple:
 // CHECK-NEXT: ObjectFileFormat: ELF
 // CHECK-NEXT: Symbols:
-// CHECK-NEXT:   f" : { Type: Object, Size: 1 }
+// CHECK-NEXT:   f", Type: Object, Size: 1 }
 // CHECK-NEXT: ...
 auto f = [](void* data) { int i; };
diff --git a/clang/test/InterfaceStubs/namespace-alias.cpp b/clang/test/InterfaceStubs/namespace-alias.cpp
index 6a7f27c9b7b0a..a4e05f9047019 100644
--- a/clang/test/InterfaceStubs/namespace-alias.cpp
+++ b/clang/test/InterfaceStubs/namespace-alias.cpp
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -o - -emit-interface-stubs %s | FileCheck %s
 
-// CHECK:      --- !experimental-ifs-v1
-// CHECK-NEXT: IfsVersion: 1.0
+// CHECK:      --- !experimental-ifs-v2
+// CHECK-NEXT: IfsVersion: 2.0
 // CHECK-NEXT: Triple:
 // CHECK-NEXT: ObjectFileFormat: ELF
 // CHECK-NEXT: Symbols:
diff --git a/clang/test/InterfaceStubs/namespace.cpp b/clang/test/InterfaceStubs/namespace.cpp
index 1c62346d22fc1..ad4db24ff7de3 100644
--- a/clang/test/InterfaceStubs/namespace.cpp
+++ b/clang/test/InterfaceStubs/namespace.cpp
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -o - -emit-interface-stubs %s | FileCheck %s
 
-// CHECK:      --- !experimental-ifs-v1
-// CHECK-NEXT: IfsVersion: 1.0
+// CHECK:      --- !experimental-ifs-v2
+// CHECK-NEXT: IfsVersion: 2.0
 // CHECK-NEXT: Triple:
 // CHECK-NEXT: ObjectFileFormat: ELF
 // CHECK-NEXT: Symbols:
diff --git a/clang/test/InterfaceStubs/non-type-template-parm-decl.cpp b/clang/test/InterfaceStubs/non-type-template-parm-decl.cpp
index 51176ac0ba0b1..6390099dee5f6 100644
--- a/clang/test/InterfaceStubs/non-type-template-parm-decl.cpp
+++ b/clang/test/InterfaceStubs/non-type-template-parm-decl.cpp
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -o - -emit-interface-stubs %s | FileCheck %s
 
-// CHECK:      --- !experimental-ifs-v1
-// CHECK-NEXT: IfsVersion: 1.0
+// CHECK:      --- !experimental-ifs-v2
+// CHECK-NEXT: IfsVersion: 2.0
 // CHECK-NEXT: Triple:
 // CHECK-NEXT: ObjectFileFormat: ELF
 // CHECK-NEXT: Symbols:
diff --git a/clang/test/InterfaceStubs/object.c b/clang/test/InterfaceStubs/object.c
index d6e28c5f884af..45e2d38ba3e9c 100644
--- a/clang/test/InterfaceStubs/object.c
+++ b/clang/test/InterfaceStubs/object.c
@@ -1,6 +1,6 @@
 // RUN: %clang_cc1 -fvisibility default -o - -emit-interface-stubs %s | FileCheck -check-prefix=CHECK-TAPI %s
 // RUN: %clang -fvisibility=default -c -o - %s | llvm-nm - 2>&1 | FileCheck -check-prefix=CHECK-SYMBOLS %s
 
-// CHECK-TAPI: data" : { Type: Object, Size: 4 }
+// CHECK-TAPI: data", Type: Object, Size: 4 }
 // CHECK-SYMBOLS: data
 int data = 42;
diff --git a/clang/test/InterfaceStubs/object.ifs b/clang/test/InterfaceStubs/object.ifs
index 7dc1134bac937..3afdf4e65eef7 100644
--- a/clang/test/InterfaceStubs/object.ifs
+++ b/clang/test/InterfaceStubs/object.ifs
@@ -4,12 +4,12 @@
 # RUN: %clang -emit-interface-stubs -o - %s | llvm-readelf --all | \
 # RUN: FileCheck %s --check-prefixes=CHECK-ELF
 
-# CHECK-IFS: --- !experimental-ifs-v1
-# CHECK-IFS-NEXT: IfsVersion:      1.0
+# CHECK-IFS: --- !experimental-ifs-v2
+# CHECK-IFS-NEXT: IfsVersion: 2.0
 # CHECK-IFS-NEXT: Triple: x86_64-linux-gnu
 # CHECK-IFS-NEXT: ObjectFileFormat: ELF
 # CHECK-IFS-NEXT: Symbols:
-# CHECK-IFS-NEXT:   b:               { Type: Object, Size: 4 }
+# CHECK-IFS-NEXT:   - { Name: b, Type: Object, Size: 4 }
 # CHECK-IFS-NEXT: ...
 
 # CHECK-ELF: ELF Header:
@@ -19,10 +19,10 @@
 # CHECK-ELF-NOT:   FUNC    GLOBAL DEFAULT  1 a
 # CHECK-ELF:   OBJECT  GLOBAL DEFAULT  1 b
 
---- !experimental-ifs-v1
-IfsVersion: 1.0
+--- !experimental-ifs-v2
+IfsVersion: 2.0
 Triple: x86_64-linux-gnu
 ObjectFileFormat: ELF
 Symbols:
-  b: { Type: Object, Size: 4 }
+  - { Name: b, Type: Object, Size: 4 }
 ...
diff --git a/clang/test/InterfaceStubs/ppc.cpp b/clang/test/InterfaceStubs/ppc.cpp
index 9a91697d95063..8b7a276bb054f 100644
--- a/clang/test/InterfaceStubs/ppc.cpp
+++ b/clang/test/InterfaceStubs/ppc.cpp
@@ -4,11 +4,11 @@
 // RUN:   -emit-interface-stubs -emit-merged-ifs -S | \
 // RUN: FileCheck -check-prefix=CHECK-IFS %s
 
- // CHECK-IFS: --- !experimental-ifs-v1
- // CHECK-IFS: IfsVersion:      1.0
- // CHECK-IFS: Triple: powerpc64le
- // CHECK-IFS: Symbols:
- // CHECK-IFS:   _Z8helloPPCv: { Type: Func }
- // CHECK-IFS: ...
+// CHECK-IFS: --- !experimental-ifs-v2
+// CHECK-IFS: IfsVersion: 2.0
+// CHECK-IFS: Triple: powerpc64le
+// CHECK-IFS: Symbols:
+// CHECK-IFS:   - { Name: _Z8helloPPCv, Type: Func }
+// CHECK-IFS: ...
 
 int helloPPC();
diff --git a/clang/test/InterfaceStubs/template-constexpr.cpp b/clang/test/InterfaceStubs/template-constexpr.cpp
index c4c7afa42f1ec..f59a55b2bb457 100644
--- a/clang/test/InterfaceStubs/template-constexpr.cpp
+++ b/clang/test/InterfaceStubs/template-constexpr.cpp
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -o - -emit-interface-stubs %s | FileCheck %s
 
-// CHECK:      --- !experimental-ifs-v1
-// CHECK-NEXT: IfsVersion: 1.0
+// CHECK:      --- !experimental-ifs-v2
+// CHECK-NEXT: IfsVersion: 2.0
 // CHECK-NEXT: Triple:
 // CHECK-NEXT: ObjectFileFormat: ELF
 // CHECK-NEXT: Symbols:
diff --git a/clang/test/InterfaceStubs/template-namespace-function.cpp b/clang/test/InterfaceStubs/template-namespace-function.cpp
index 47788d4a3e0a1..68f017c4d5ece 100644
--- a/clang/test/InterfaceStubs/template-namespace-function.cpp
+++ b/clang/test/InterfaceStubs/template-namespace-function.cpp
@@ -6,10 +6,10 @@
 // RUN: FileCheck -check-prefix=CHECK-SYMBOLS %s
 
 // CHECK: Symbols:
-// CHECK-DAG:  "_ZN3qux3barEii" : { Type: Func }
-// CHECK-DAG:  "_ZN3baz3addIiEET_S1_S1_" : { Type: Func }
-// CHECK-DAG:  "_Z4fbarff" : { Type: Func }
-// CHECK-DAG:  "_ZN3baz3addIfEET_S1_S1_" : { Type: Func }
+// CHECK-DAG:  - { Name: "_ZN3qux3barEii", Type: Func }
+// CHECK-DAG:  - { Name: "_ZN3baz3addIiEET_S1_S1_",  Type: Func }
+// CHECK-DAG:  - { Name: "_Z4fbarff", Type: Func }
+// CHECK-DAG:  - { Name: "_ZN3baz3addIfEET_S1_S1_", Type: Func }
 
 // Same symbols just different order.
 // CHECK-SYMBOLS-DAG:  _Z4fbarff
diff --git a/clang/test/InterfaceStubs/template-template-parm-decl.cpp b/clang/test/InterfaceStubs/template-template-parm-decl.cpp
index 63883536a816d..5451ec6178e2c 100644
--- a/clang/test/InterfaceStubs/template-template-parm-decl.cpp
+++ b/clang/test/InterfaceStubs/template-template-parm-decl.cpp
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -o - -emit-interface-stubs %s | FileCheck %s
 
-// CHECK:      --- !experimental-ifs-v1
-// CHECK-NEXT: IfsVersion: 1.0
+// CHECK:      --- !experimental-ifs-v2
+// CHECK-NEXT: IfsVersion: 2.0
 // CHECK-NEXT: Triple:
 // CHECK-NEXT: ObjectFileFormat: ELF
 // CHECK-NEXT: Symbols:
diff --git a/clang/test/InterfaceStubs/trycatch.cpp b/clang/test/InterfaceStubs/trycatch.cpp
index 57076a097cb59..dac7806926a55 100644
--- a/clang/test/InterfaceStubs/trycatch.cpp
+++ b/clang/test/InterfaceStubs/trycatch.cpp
@@ -2,13 +2,12 @@
 
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fcxx-exceptions -o - -emit-interface-stubs %s | FileCheck %s
 
-
-// CHECK:      --- !experimental-ifs-v1
-// CHECK-NEXT: IfsVersion: 1.0
+// CHECK:      --- !experimental-ifs-v2
+// CHECK-NEXT: IfsVersion: 2.0
 // CHECK-NEXT: Triple: x86_64-unknown-linux-gnu
 // CHECK-NEXT: ObjectFileFormat: ELF
 // CHECK-NEXT: Symbols:
-// CHECK-NEXT: "_Z1fv" : { Type: Func }
+// CHECK-NEXT: - { Name: "_Z1fv", Type: Func }
 // CHECK-NEXT: ...
 
 class C5 {};
diff --git a/clang/test/InterfaceStubs/unresolved-using-typename.cpp b/clang/test/InterfaceStubs/unresolved-using-typename.cpp
index e6afc781412a1..d4aad84d72118 100644
--- a/clang/test/InterfaceStubs/unresolved-using-typename.cpp
+++ b/clang/test/InterfaceStubs/unresolved-using-typename.cpp
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -o - -emit-interface-stubs %s | FileCheck %s
 
-// CHECK:      --- !experimental-ifs-v1
-// CHECK-NEXT: IfsVersion: 1.0
+// CHECK:      --- !experimental-ifs-v2
+// CHECK-NEXT: IfsVersion: 2.0
 // CHECK-NEXT: Triple:
 // CHECK-NEXT: ObjectFileFormat: ELF
 // CHECK-NEXT: Symbols:
diff --git a/clang/test/InterfaceStubs/usings.cpp b/clang/test/InterfaceStubs/usings.cpp
index 735a040c91dc1..2ef83207fcb34 100644
--- a/clang/test/InterfaceStubs/usings.cpp
+++ b/clang/test/InterfaceStubs/usings.cpp
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -o - -emit-interface-stubs %s | FileCheck %s
 
-// CHECK:      --- !experimental-ifs-v1
-// CHECK-NEXT: IfsVersion: 1.0
+// CHECK:      --- !experimental-ifs-v2
+// CHECK-NEXT: IfsVersion: 2.0
 // CHECK-NEXT: Triple:
 // CHECK-NEXT: ObjectFileFormat: ELF
 // CHECK-NEXT: Symbols:
diff --git a/clang/test/InterfaceStubs/var-template-specialization-decl.cpp b/clang/test/InterfaceStubs/var-template-specialization-decl.cpp
index bbb5ae888977e..9b67dac9865c1 100644
--- a/clang/test/InterfaceStubs/var-template-specialization-decl.cpp
+++ b/clang/test/InterfaceStubs/var-template-specialization-decl.cpp
@@ -1,12 +1,12 @@
 // REQUIRES: x86-registered-target
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -o - -emit-interface-stubs %s | FileCheck %s
 
-// CHECK:      --- !experimental-ifs-v1
-// CHECK-NEXT: IfsVersion: 1.0
+// CHECK:      --- !experimental-ifs-v2
+// CHECK-NEXT: IfsVersion: 2.0
 // CHECK-NEXT: Triple: x86_64-unknown-linux-gnu
 // CHECK-NEXT: ObjectFileFormat: ELF
 // CHECK-NEXT: Symbols:
-// CHECK-NEXT: "a" : { Type: Object, Size: 4 }
+// CHECK-NEXT: - { Name: "a", Type: Object, Size: 4 }
 // CHECK-NEXT: ...
 
 template<typename T, T v> struct S9 {
diff --git a/clang/test/InterfaceStubs/weak.cpp b/clang/test/InterfaceStubs/weak.cpp
index 1581ffa9d5d70..e3c0413b65112 100644
--- a/clang/test/InterfaceStubs/weak.cpp
+++ b/clang/test/InterfaceStubs/weak.cpp
@@ -1,14 +1,14 @@
 // REQUIRES: x86-registered-target
 // RUN: %clang_cc1 -triple x86_64-linux-gnu -o - -emit-interface-stubs \
-// RUN: -interface-stub-version=experimental-ifs-v1 %s | \
+// RUN: -interface-stub-version=experimental-ifs-v2 %s | \
 // RUN: FileCheck %s
 
 // RUN: %clang -target x86_64-unknown-linux-gnu -o - -c %s | llvm-nm - 2>&1 | \
 // RUN: FileCheck -check-prefix=CHECK-SYMBOLS %s
 
 // CHECK: Symbols:
-// CHECK-DAG:  "_Z8weakFuncv" : { Type: Func, Weak: true }
-// CHECK-DAG:  "_Z10strongFuncv" : { Type: Func }
+// CHECK-DAG:  - { Name: "_Z8weakFuncv", Type: Func, Weak: true }
+// CHECK-DAG:  - { Name: "_Z10strongFuncv", Type: Func }
 
 // CHECK-SYMBOLS-DAG: _Z10strongFuncv
 // CHECK-SYMBOLS-DAG: _Z8weakFuncv
diff --git a/clang/test/InterfaceStubs/windows.cpp b/clang/test/InterfaceStubs/windows.cpp
index c81c702861e4c..73f3ed5d39acf 100644
--- a/clang/test/InterfaceStubs/windows.cpp
+++ b/clang/test/InterfaceStubs/windows.cpp
@@ -6,11 +6,11 @@
 // CHECK-CC1: Symbols:
 // CHECK-CC1-NEXT: ?helloWindowsMsvc@@YAHXZ
 
- // CHECK-IFS: --- !experimental-ifs-v1
- // CHECK-IFS: IfsVersion:      1.0
- // CHECK-IFS: Triple:
- // CHECK-IFS: Symbols:
- // CHECK-IFS:   ?helloWindowsMsvc@@YAHXZ: { Type: Func }
- // CHECK-IFS: ...
+// CHECK-IFS: --- !experimental-ifs-v2
+// CHECK-IFS: IfsVersion: 2.0
+// CHECK-IFS: Triple:
+// CHECK-IFS: Symbols:
+// CHECK-IFS:   - { Name: '?helloWindowsMsvc@@YAHXZ', Type: Func }
+// CHECK-IFS: ...
 
 int helloWindowsMsvc();
diff --git a/clang/test/Misc/diag-aka-types.cpp b/clang/test/Misc/diag-aka-types.cpp
index c386504a6c563..3e051bdddec66 100644
--- a/clang/test/Misc/diag-aka-types.cpp
+++ b/clang/test/Misc/diag-aka-types.cpp
@@ -50,9 +50,9 @@ template <typename T>
 class A {};
 
 int a1 = A<decltype(1 + 2)>(); // expected-error{{no viable conversion from 'A<decltype(1 + 2)>' (aka 'A<int>') to 'int'}}
-int a2 = A<A<decltype(1 + 2)>>(); // expected-error{{no viable conversion from 'A<A<decltype(1 + 2)> >' (aka 'A<A<int> >') to 'int'}}
+int a2 = A<A<decltype(1 + 2)>>(); // expected-error{{no viable conversion from 'A<A<decltype(1 + 2)>>' (aka 'A<A<int>>') to 'int'}}
 int a3 = A<__typeof(1 + 2)>(); // expected-error{{no viable conversion from 'A<typeof (1 + 2)>' (aka 'A<int>') to 'int'}}
-int a4 = A<A<__typeof(1 + 2)>>(); // expected-error{{no viable conversion from 'A<A<typeof (1 + 2)> >' (aka 'A<A<int> >') to 'int'}}
+int a4 = A<A<__typeof(1 + 2)>>(); // expected-error{{no viable conversion from 'A<A<typeof (1 + 2)>>' (aka 'A<A<int>>') to 'int'}}
 
 using B = A<decltype(1+2)>;
 int a5 = B(); // expected-error{{no viable conversion from 'B' (aka 'A<int>') to 'int'}}
diff --git a/clang/test/Misc/diag-line-wrapping.cpp b/clang/test/Misc/diag-line-wrapping.cpp
index 2bcb03f9781c4..9e8cb9b53da55 100644
--- a/clang/test/Misc/diag-line-wrapping.cpp
+++ b/clang/test/Misc/diag-line-wrapping.cpp
@@ -1,5 +1,5 @@
-// RUN: not %clang_cc1 -fsyntax-only -fmessage-length 60 %s 2>&1 | FileCheck %s
-// RUN: not %clang_cc1 -fsyntax-only -fmessage-length 0 %s 2>&1 | FileCheck %s
+// RUN: not %clang_cc1 -fsyntax-only -fmessage-length=60 %s 2>&1 | FileCheck %s
+// RUN: not %clang_cc1 -fsyntax-only -fmessage-length=0 %s 2>&1 | FileCheck %s
 
 struct B { void f(); };
 struct D1 : B {};
diff --git a/clang/test/Misc/message-length.c b/clang/test/Misc/message-length.c
index a6f4f44e6b9c0..1e0b4edb7c032 100644
--- a/clang/test/Misc/message-length.c
+++ b/clang/test/Misc/message-length.c
@@ -1,6 +1,6 @@
-// RUN: not %clang_cc1 -fmessage-length 72 %s 2>&1 | FileCheck -strict-whitespace %s
-// RUN: not %clang_cc1 -fmessage-length 1 %s
-// RUN: not %clang_cc1 -fmessage-length 8 %s 2>&1 | FileCheck -check-prefix=CHECK-DOT %s
+// RUN: not %clang_cc1 -fmessage-length=72 %s 2>&1 | FileCheck -strict-whitespace %s
+// RUN: not %clang_cc1 -fmessage-length=1 %s
+// RUN: not %clang_cc1 -fmessage-length=8 %s 2>&1 | FileCheck -check-prefix=CHECK-DOT %s
 // Hack so we can check things better, force the file name and line.
 # 1 "FILE" 1
 
diff --git a/clang/test/Misc/pragma-attribute-supported-attributes-list.test b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
index 9132e55aa69ef..8bc2b1bc53832 100644
--- a/clang/test/Misc/pragma-attribute-supported-attributes-list.test
+++ b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
@@ -30,6 +30,8 @@
 // CHECK-NEXT: CPUSpecific (SubjectMatchRule_function)
 // CHECK-NEXT: CUDAConstant (SubjectMatchRule_variable)
 // CHECK-NEXT: CUDADevice (SubjectMatchRule_function, SubjectMatchRule_variable)
+// CHECK-NEXT: CUDADeviceBuiltinSurfaceType (SubjectMatchRule_record)
+// CHECK-NEXT: CUDADeviceBuiltinTextureType (SubjectMatchRule_record)
 // CHECK-NEXT: CUDAGlobal (SubjectMatchRule_function)
 // CHECK-NEXT: CUDAHost (SubjectMatchRule_function)
 // CHECK-NEXT: CUDALaunchBounds (SubjectMatchRule_objc_method, SubjectMatchRule_hasType_functionType)
diff --git a/clang/test/Misc/show-diag-options.c b/clang/test/Misc/show-diag-options.c
index 8f05fbc76b560..4e98d63195f10 100644
--- a/clang/test/Misc/show-diag-options.c
+++ b/clang/test/Misc/show-diag-options.c
@@ -1,16 +1,16 @@
-// RUN: %clang_cc1 -fsyntax-only %s 2>&1 \
+// RUN: %clang_cc1 -fsyntax-only -fno-diagnostics-show-option %s 2>&1 \
 // RUN:   | FileCheck %s -check-prefix=BASE
-// RUN: %clang_cc1 -fsyntax-only -fdiagnostics-show-option %s 2>&1 \
+// RUN: %clang_cc1 -fsyntax-only %s 2>&1 \
 // RUN:   | FileCheck %s -check-prefix=OPTION
-// RUN: not %clang_cc1 -fsyntax-only -fdiagnostics-show-option -Werror %s 2>&1 \
+// RUN: not %clang_cc1 -fsyntax-only -Werror %s 2>&1 \
 // RUN:   | FileCheck %s -check-prefix=OPTION_ERROR
-// RUN: %clang_cc1 -fsyntax-only -std=c89 -pedantic -fdiagnostics-show-option %s 2>&1 \
+// RUN: %clang_cc1 -fsyntax-only -std=c89 -pedantic %s 2>&1 \
 // RUN:   | FileCheck %s -check-prefix=OPTION_PEDANTIC
-// RUN: %clang_cc1 -fsyntax-only -fdiagnostics-show-category id %s 2>&1 \
+// RUN: %clang_cc1 -fsyntax-only -fno-diagnostics-show-option -fdiagnostics-show-category id %s 2>&1 \
 // RUN:   | FileCheck %s -check-prefix=CATEGORY_ID
-// RUN: %clang_cc1 -fsyntax-only -fdiagnostics-show-category name %s 2>&1 \
+// RUN: %clang_cc1 -fsyntax-only -fno-diagnostics-show-option -fdiagnostics-show-category name %s 2>&1 \
 // RUN:   | FileCheck %s -check-prefix=CATEGORY_NAME
-// RUN: not %clang_cc1 -fsyntax-only -fdiagnostics-show-option -fdiagnostics-show-category name -Werror %s 2>&1 \
+// RUN: not %clang_cc1 -fsyntax-only -fdiagnostics-show-category name -Werror %s 2>&1 \
 // RUN:   | FileCheck %s -check-prefix=OPTION_ERROR_CATEGORY
 
 void test(int x, int y) {
diff --git a/clang/test/Misc/unnecessary-elipses.cpp b/clang/test/Misc/unnecessary-elipses.cpp
index 2ee725869b5c3..c8c178c37f6c6 100644
--- a/clang/test/Misc/unnecessary-elipses.cpp
+++ b/clang/test/Misc/unnecessary-elipses.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fsyntax-only -fmessage-length 80 %s 2>&1 | FileCheck -strict-whitespace %s
+// RUN: %clang_cc1 -fsyntax-only -fmessage-length=80 %s 2>&1 | FileCheck -strict-whitespace %s
 
 int main() {
     "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
@@ -12,4 +12,4 @@ int main() {
 
         "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"        ;
 // CHECK: {{^  ..."xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"...}}
-}
\ No newline at end of file
+}
diff --git a/clang/test/Misc/unprintable.c b/clang/test/Misc/unprintable.c
index eaa4f34d80283..30e4494566300 100644
--- a/clang/test/Misc/unprintable.c
+++ b/clang/test/Misc/unprintable.c
@@ -1,4 +1,4 @@
-// RUN: not %clang_cc1 %s -fmessage-length 40 2>&1 | FileCheck -strict-whitespace %s
+// RUN: not %clang_cc1 %s -fmessage-length=40 2>&1 | FileCheck -strict-whitespace %s
 
 int main() {
     int i;
diff --git a/clang/test/Misc/wrong-encoding2.c b/clang/test/Misc/wrong-encoding2.c
index 43a0f4e900ed8..b60ed7f92b861 100644
--- a/clang/test/Misc/wrong-encoding2.c
+++ b/clang/test/Misc/wrong-encoding2.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fsyntax-only -fmessage-length 100 %s 2>&1 | FileCheck -strict-whitespace %s
+// RUN: %clang_cc1 -fsyntax-only -fmessage-length=100 %s 2>&1 | FileCheck -strict-whitespace %s
 // REQUIRES: asserts
 
 int main() {
diff --git a/clang/test/Modules/ExtDebugInfo.cpp b/clang/test/Modules/ExtDebugInfo.cpp
index 5d2921cd99599..a66c678227fe7 100644
--- a/clang/test/Modules/ExtDebugInfo.cpp
+++ b/clang/test/Modules/ExtDebugInfo.cpp
@@ -85,14 +85,14 @@ void foo() {
 
 // This type is not anchored in the module by an explicit template instantiation.
 // CHECK: !DICompositeType(tag: DW_TAG_class_type,
-// CHECK-SAME:             name: "Template<long, DebugCXX::traits<long> >",
+// CHECK-SAME:             name: "Template<long, DebugCXX::traits<long>>",
 // CHECK-SAME:             scope: ![[NS]],
 // CHECK-SAME:             elements:
 // CHECK-SAME:             identifier: "_ZTSN8DebugCXX8TemplateIlNS_6traitsIlEEEE")
 
 // This type is anchored in the module by an explicit template instantiation.
 // CHECK: !DICompositeType(tag: DW_TAG_class_type,
-// CHECK-SAME:             name: "Template<int, DebugCXX::traits<int> >",
+// CHECK-SAME:             name: "Template<int, DebugCXX::traits<int>>",
 // CHECK-SAME:             scope: ![[NS]],
 // CHECK-SAME:             flags: DIFlagFwdDecl,
 // CHECK-SAME:             identifier: "_ZTSN8DebugCXX8TemplateIiNS_6traitsIiEEEE")
@@ -103,7 +103,7 @@ void foo() {
 
 // This one isn't.
 // CHECK: !DICompositeType(tag: DW_TAG_class_type,
-// CHECK-SAME:             name: "Template<float, DebugCXX::traits<float> >",
+// CHECK-SAME:             name: "Template<float, DebugCXX::traits<float>>",
 // CHECK-SAME:             scope: ![[NS]],
 // CHECK-SAME:             elements:
 // CHECK-SAME:             templateParams:
diff --git a/clang/test/Modules/ModuleDebugInfo.cpp b/clang/test/Modules/ModuleDebugInfo.cpp
index 40136978017cb..c361053d03cd6 100644
--- a/clang/test/Modules/ModuleDebugInfo.cpp
+++ b/clang/test/Modules/ModuleDebugInfo.cpp
@@ -65,7 +65,7 @@
 
 // This type is anchored by an explicit template instantiation.
 // CHECK: !DICompositeType(tag: DW_TAG_class_type,
-// CHECK-SAME:             name: "Template<int, DebugCXX::traits<int> >"
+// CHECK-SAME:             name: "Template<int, DebugCXX::traits<int>>"
 // CHECK-SAME:             elements:
 // CHECK-SAME:             templateParams:
 // CHECK-SAME:             identifier: "_ZTSN8DebugCXX8TemplateIiNS_6traitsIiEEEE")
@@ -80,7 +80,7 @@
 // CHECK-SAME:             identifier: "_ZTSN8DebugCXX6traitsIfEE")
 
 // CHECK: !DICompositeType(tag: DW_TAG_class_type,
-// CHECK-SAME:             name: "Template<long, DebugCXX::traits<long> >"
+// CHECK-SAME:             name: "Template<long, DebugCXX::traits<long>>"
 // CHECK-SAME:             elements:
 // CHECK-SAME:             templateParams:
 // CHECK-SAME:             identifier: "_ZTSN8DebugCXX8TemplateIlNS_6traitsIlEEEE")
@@ -89,7 +89,7 @@
 // no mangled name here yet.
 
 // CHECK: !DICompositeType(tag: DW_TAG_class_type,
-// CHECK-SAME:             name: "Template<float, DebugCXX::traits<float> >"
+// CHECK-SAME:             name: "Template<float, DebugCXX::traits<float>>"
 // CHECK-SAME:             flags: DIFlagFwdDecl
 // CHECK-SAME:             identifier: "_ZTSN8DebugCXX8TemplateIfNS_6traitsIfEEEE")
 
diff --git a/clang/test/Modules/debug-info-moduleimport.m b/clang/test/Modules/debug-info-moduleimport.m
index 9dee9964b538e..5787ffe227513 100644
--- a/clang/test/Modules/debug-info-moduleimport.m
+++ b/clang/test/Modules/debug-info-moduleimport.m
@@ -34,9 +34,11 @@
 
 // RUN: %clang_cc1 -debug-info-kind=limited -fmodules -fimplicit-module-maps \
 // RUN:   -fmodules-cache-path=%t -fdebug-prefix-map=%t=/MODULE-CACHE \
+// RUN:   -fdebug-prefix-map=%S=/SRCDIR \
 // RUN:   -fmodule-format=obj -dwarf-ext-refs \
 // RUN:   %s -I %S/Inputs -isysroot /tmp/.. -I %t -emit-llvm -o - \
 // RUN:     | FileCheck %s --check-prefix=SKEL-CHECK
+// SKEL-CHECK: includePath: "/SRCDIR/Inputs"
 // SKEL-CHECK: distinct !DICompileUnit({{.*}}file: ![[CUFILE:[0-9]+]]
 // SKEL-CHECK: ![[CUFILE]] = !DIFile({{.*}}directory: "[[COMP_DIR:.*]]"
 // SKEL-CHECK: distinct !DICompileUnit({{.*}}file: ![[DWOFILE:[0-9]+]]{{.*}}splitDebugFilename: "/MODULE-CACHE{{.*}}dwoId
diff --git a/clang/test/OpenMP/barrier_codegen.cpp b/clang/test/OpenMP/barrier_codegen.cpp
index 3f2e7e346aed3..e61ca186a697f 100644
--- a/clang/test/OpenMP/barrier_codegen.cpp
+++ b/clang/test/OpenMP/barrier_codegen.cpp
@@ -46,7 +46,7 @@ int main(int argc, char **argv) {
 // CLANGCG-NOT:        #
 // IRBUILDER:          ; Function Attrs: nounwind
 // IRBUILDER-NEXT:     declare i32 @__kmpc_global_thread_num(%struct.ident_t*) #
-// IRBUILDER_OPT:      ; Function Attrs: nofree nosync nounwind readonly
+// IRBUILDER_OPT:      ; Function Attrs: inaccessiblememonly nofree nosync nounwind readonly
 // IRBUILDER_OPT-NEXT: declare i32 @__kmpc_global_thread_num(%struct.ident_t*) #
 
 // CHECK: define {{.+}} [[TMAIN_INT]](
diff --git a/clang/test/OpenMP/begin_declare_variant_elided_range_withouth_end.c b/clang/test/OpenMP/begin_declare_variant_elided_range_withouth_end.c
new file mode 100644
index 0000000000000..d58dd000a5039
--- /dev/null
+++ b/clang/test/OpenMP/begin_declare_variant_elided_range_withouth_end.c
@@ -0,0 +1,31 @@
+// RUN: %clang_cc1 -triple=x86_64-pc-win32 -verify -fopenmp -x c -std=c99 -fms-extensions -Wno-pragma-pack %s
+// RUN: %clang_cc1 -triple=x86_64-pc-win32 -verify -fopenmp-simd -x c -std=c99 -fms-extensions -Wno-pragma-pack %s
+
+#pragma omp begin declare variant match(device={kind(gpu)}) // expected-note {{to match this '#pragma omp begin declare variant'}}
+int also_before(void) {
+  return 0;
+}
+
+#pragma omp begin declare variant match(device={kind(gpu)})
+int also_after(void) {
+  return 2;
+}
+int also_before(void) {
+  return 2;
+}
+#pragma omp end declare variant
+
+
+#pragma omp begin declare variant match(device={kind(fpga)})
+
+This text is never parsed!
+
+#pragma omp end declare variant
+
+int also_after(void) {
+  return 0;
+}
+
+int test() {
+  return also_after() + also_before();
+} // expected-error {{expected '#pragma omp end declare variant'}}
diff --git a/clang/test/OpenMP/begin_declare_variant_messages.c b/clang/test/OpenMP/begin_declare_variant_messages.c
new file mode 100644
index 0000000000000..63aee2799b784
--- /dev/null
+++ b/clang/test/OpenMP/begin_declare_variant_messages.c
@@ -0,0 +1,173 @@
+// RUN: %clang_cc1 -triple=x86_64-pc-win32 -verify -fopenmp -x c -std=c99 -fms-extensions -Wno-pragma-pack %s
+
+// RUN: %clang_cc1 -triple=x86_64-pc-win32 -verify -fopenmp-simd -x c -std=c99 -fms-extensions -Wno-pragma-pack %s
+
+
+#pragma omp begin // expected-error {{expected an OpenMP directive}}
+#pragma omp end declare variant // expected-error {{'#pragma omp end declare variant' with no matching '#pragma omp begin declare variant'}}
+#pragma omp begin declare // expected-error {{expected an OpenMP directive}}
+#pragma omp end declare variant // expected-error {{'#pragma omp end declare variant' with no matching '#pragma omp begin declare variant'}}
+#pragma omp begin variant // expected-error {{expected an OpenMP directive}}
+#pragma omp end declare variant // expected-error {{'#pragma omp end declare variant' with no matching '#pragma omp begin declare variant'}}
+#pragma omp variant begin // expected-error {{expected an OpenMP directive}}
+#pragma omp declare variant end // expected-error {{function declaration is expected after 'declare variant' directive}}
+#pragma omp begin declare variant // expected-error {{expected 'match' clause on 'omp declare variant' directive}}
+#pragma omp end declare variant
+// TODO: Issue an error message
+#pragma omp end declare variant // expected-error {{'#pragma omp end declare variant' with no matching '#pragma omp begin declare variant'}}
+#pragma omp end declare variant // expected-error {{'#pragma omp end declare variant' with no matching '#pragma omp begin declare variant'}}
+#pragma omp end declare variant // expected-error {{'#pragma omp end declare variant' with no matching '#pragma omp begin declare variant'}}
+#pragma omp end declare variant // expected-error {{'#pragma omp end declare variant' with no matching '#pragma omp begin declare variant'}}
+
+int foo(void);
+const int var;
+
+#pragma omp begin declare variant // expected-error {{expected 'match' clause on 'omp declare variant' directive}}
+#pragma omp end declare variant
+#pragma omp begin declare variant xxx // expected-error {{expected 'match' clause on 'omp declare variant' directive}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match // expected-error {{expected '(' after 'match'}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match( // expected-error {{expected ')'}} expected-warning {{expected identifier or string literal describing a context set; set skipped}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}} expected-note {{to match this '('}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match() // expected-warning {{expected identifier or string literal describing a context set; set skipped}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(xxx) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(xxx=) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(xxx=yyy) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(xxx=yyy}) // expected-error {{expected ')'}} expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-warning {{extra tokens at the end of '#pragma omp begin declare variant' are ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}} expected-note {{to match this '('}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(xxx={) // expected-error {{expected ')'}} expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}} expected-note {{to match this '('}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(xxx={}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(xxx={vvv, vvv}) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(xxx={vvv} xxx) // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(xxx={vvv}) xxx // expected-warning {{'xxx' is not a valid context set in a `declare variant`; set ignored}} expected-warning {{extra tokens at the end of '#pragma omp begin declare variant' are ignored}} expected-note {{context set options are: 'construct' 'device' 'implementation' 'user'}} expected-note {{the ignored set spans until here}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(implementation={xxx}) // expected-warning {{'xxx' is not a valid context selector for the context set 'implementation'; selector ignored}} expected-note {{context selector options are: 'vendor' 'extension' 'unified_address' 'unified_shared_memory' 'reverse_offload' 'dynamic_allocators' 'atomic_default_mem_order'}} expected-note {{the ignored selector spans until here}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(implementation={vendor}) // expected-warning {{the context selector 'vendor' in context set 'implementation' requires a context property defined in parentheses; selector ignored}} expected-note {{the ignored selector spans until here}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(implementation={vendor(}) // expected-error {{expected ')'}} expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{context property options are: 'amd' 'arm' 'bsc' 'cray' 'fujitsu' 'gnu' 'ibm' 'intel' 'llvm' 'pgi' 'ti' 'unknown'}} expected-note {{to match this '('}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(implementation={vendor()}) // expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{context property options are: 'amd' 'arm' 'bsc' 'cray' 'fujitsu' 'gnu' 'ibm' 'intel' 'llvm' 'pgi' 'ti' 'unknown'}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(implementation={vendor(score ibm)}) // expected-error {{expected '(' after 'score'}} expected-warning {{expected '':'' after the score expression; '':'' assumed}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(implementation={vendor(score( ibm)}) // expected-error {{use of undeclared identifier 'ibm'}} expected-error {{expected ')'}} expected-warning {{expected '':'' after the score expression; '':'' assumed}} expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{context property options are: 'amd' 'arm' 'bsc' 'cray' 'fujitsu' 'gnu' 'ibm' 'intel' 'llvm' 'pgi' 'ti' 'unknown'}} expected-note {{to match this '('}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(implementation={vendor(score(2 ibm)}) // expected-error {{expected ')'}} expected-error {{expected ')'}} expected-warning {{expected '':'' after the score expression; '':'' assumed}} expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{to match this '('}} expected-note {{context property options are: 'amd' 'arm' 'bsc' 'cray' 'fujitsu' 'gnu' 'ibm' 'intel' 'llvm' 'pgi' 'ti' 'unknown'}} expected-note {{to match this '('}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(implementation={vendor(score(foo()) ibm)}) // expected-warning {{expected '':'' after the score expression; '':'' assumed}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(implementation={vendor(score(5): ibm), vendor(llvm)}) // expected-warning {{the context selector 'vendor' was used already in the same 'omp declare variant' directive; selector ignored}} expected-note {{the previous context selector 'vendor' used here}} expected-note {{the ignored selector spans until here}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(implementation={vendor(score(5): ibm), kind(cpu)}) // expected-warning {{the context selector 'kind' is not valid for the context set 'implementation'; selector ignored}} expected-note {{the context selector 'kind' can be nested in the context set 'device'; try 'match(device={kind(property)})'}} expected-note {{the ignored selector spans until here}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(device={xxx}) // expected-warning {{'xxx' is not a valid context selector for the context set 'device'; selector ignored}} expected-note {{context selector options are: 'kind' 'isa' 'arch'}} expected-note {{the ignored selector spans until here}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(device={kind}) // expected-warning {{the context selector 'kind' in context set 'device' requires a context property defined in parentheses; selector ignored}} expected-note {{the ignored selector spans until here}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(device={kind(}) // expected-error {{expected ')'}} expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{context property options are: 'host' 'nohost' 'cpu' 'gpu' 'fpga' 'any'}} expected-note {{to match this '('}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(device={kind()}) // expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{context property options are: 'host' 'nohost' 'cpu' 'gpu' 'fpga' 'any'}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(device={kind(score cpu)}) // expected-error {{expected '(' after 'score'}} expected-warning {{expected '':'' after the score expression; '':'' assumed}} expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('<invalid>'); score ignored}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(device={kind(score( ibm)}) // expected-error {{use of undeclared identifier 'ibm'}} expected-error {{expected ')'}} expected-warning {{expected '':'' after the score expression; '':'' assumed}} expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('<invalid>'); score ignored}} expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{context property options are: 'host' 'nohost' 'cpu' 'gpu' 'fpga' 'any'}} expected-note {{to match this '('}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(device={kind(score(2 gpu)}) // expected-error {{expected ')'}} expected-error {{expected ')'}} expected-warning {{expected '':'' after the score expression; '':'' assumed}} expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('2'); score ignored}} expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{to match this '('}} expected-note {{context property options are: 'host' 'nohost' 'cpu' 'gpu' 'fpga' 'any'}} expected-note {{to match this '('}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(device={kind(score(foo()) ibm)}) // expected-warning {{expected '':'' after the score expression; '':'' assumed}} expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('foo()'); score ignored}} expected-warning {{'ibm' is not a valid context property for the context selector 'kind' and the context set 'device'; property ignored}} expected-note {{try 'match(implementation={vendor(ibm)})'}} expected-note {{the ignored property spans until here}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(device={kind(score(5): host), kind(llvm)}) // expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('5'); score ignored}} expected-warning {{the context selector 'kind' was used already in the same 'omp declare variant' directive; selector ignored}} expected-note {{the previous context selector 'kind' used here}} expected-note {{the ignored selector spans until here}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(device={kind(score(5): nohost), vendor(llvm)}) // expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('5'); score ignored}} expected-warning {{the context selector 'vendor' is not valid for the context set 'device'; selector ignored}} expected-note {{the context selector 'vendor' can be nested in the context set 'implementation'; try 'match(implementation={vendor(property)})'}} expected-note {{the ignored selector spans until here}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(device = {kind(score(foo()): cpu}) // expected-error {{expected ')'}} expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('foo()'); score ignored}} expected-note {{to match this '('}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(device = {kind(score(foo()): cpu)) // expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('foo()'); score ignored}} expected-warning {{expected '}' after the context selectors for the context set "device"; '}' assumed}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(device = {kind(score(foo()): cpu)} // expected-error {{expected ')'}} expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('foo()'); score ignored}} expected-note {{to match this '('}}
+#pragma omp end declare variant
+
+#pragma omp begin declare variant match(implementation = {vendor(score(foo) :llvm)})
+#pragma omp end declare variant
+#pragma omp begin declare variant match(implementation = {vendor(score(foo()) :llvm)})
+#pragma omp end declare variant
+#pragma omp begin declare variant match(implementation = {vendor(score(<expr>) :llvm)}) // expected-error {{expected expression}} expected-error {{use of undeclared identifier 'expr'}} expected-error {{expected expression}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(user = {condition(foo)})
+#pragma omp end declare variant
+#pragma omp begin declare variant match(user = {condition(foo())})
+#pragma omp end declare variant
+#pragma omp begin declare variant match(user = {condition(<expr>)}) // expected-error {{expected expression}} expected-error {{use of undeclared identifier 'expr'}} expected-error {{expected expression}} expected-note {{the ignored selector spans until here}}
+#pragma omp end declare variant
+
+#pragma omp begin declare variant match(device = {kind(score(&var): cpu)}) // expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('&var'); score ignored}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(device = {kind(score(var): cpu)}) // expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('var'); score ignored}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(device = {kind(score(foo): cpu)}) // expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('foo'); score ignored}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(device = {kind(score(foo()): cpu)}) // expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('foo()'); score ignored}}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(device = {kind(score(<expr>): cpu)}) // expected-error {{expected expression}} expected-error {{use of undeclared identifier 'expr'}} expected-error {{expected expression}} expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('<invalid>'); score ignored}}
+#pragma omp end declare variant
+
+#pragma omp begin declare variant match(device={kind(cpu)})
+static int defined_twice_a(void) { // expected-note {{previous definition is here}}
+  return 0;
+}
+int defined_twice_b(void) { // expected-note {{previous definition is here}}
+  return 0;
+}
+inline int defined_twice_c(void) { // expected-note {{previous definition is here}}
+  return 0;
+}
+#pragma omp end declare variant
+#pragma omp begin declare variant match(device={kind(cpu)})
+static int defined_twice_a(void) { // expected-error {{redefinition of 'defined_twice_a[device={kind(cpu)}]'}}
+  return 1;
+}
+int defined_twice_b(void) { // expected-error {{redefinition of 'defined_twice_b[device={kind(cpu)}]'}}
+  return 1;
+}
+inline int defined_twice_c(void) { // expected-error {{redefinition of 'defined_twice_c[device={kind(cpu)}]'}}
+  return 1;
+}
+#pragma omp end declare variant
+
+
+// TODO: Issue an error message
+#pragma omp begin declare variant match(device={kind(cpu)})
+// The matching end is missing. Since the device clause is matching we will
+// emit and error.
+int also_before(void) {
+  return 0;
+}
+
+
+#pragma omp begin declare variant match(device={kind(gpu)}) // expected-note {{to match this '#pragma omp begin declare variant'}}
+// The matching end is missing. Since the device clause is not matching we will
+// cause us to elide the rest of the file and emit and error.
+int also_after(void) {
+  return 2;
+}
+int also_before(void) {
+  return 2;
+}
+
+#pragma omp begin declare variant match(device={kind(fpga)})
+
+This text is never parsed!
+
+#pragma omp end declare variant
+
+This text is also not parsed! // expected-error {{expected '#pragma omp end declare variant'}}
diff --git a/clang/test/OpenMP/begin_declare_variant_no_end_for_matching_selector.c b/clang/test/OpenMP/begin_declare_variant_no_end_for_matching_selector.c
new file mode 100644
index 0000000000000..406776b6d4a9c
--- /dev/null
+++ b/clang/test/OpenMP/begin_declare_variant_no_end_for_matching_selector.c
@@ -0,0 +1,33 @@
+// RUN: %clang_cc1 -triple=x86_64-pc-win32 -verify -fopenmp -x c -std=c99 -fms-extensions -Wno-pragma-pack %s
+// RUN: %clang_cc1 -triple=x86_64-pc-win32 -verify -fopenmp-simd -x c -std=c99 -fms-extensions -Wno-pragma-pack %s
+
+#pragma omp begin declare variant match(device={kind(cpu)})
+int also_before(void) {
+  return 0;
+}
+#pragma omp end declare variant
+
+#pragma omp begin declare variant match(device={kind(gpu)}) // expected-note {{to match this '#pragma omp begin declare variant'}}
+// The matching end is missing. Since the device clause is not matching we will
+// cause us to elide the rest of the file and emit and error.
+int also_after(void) {
+  return 2;
+}
+int also_before(void) {
+  return 2;
+}
+
+
+#pragma omp begin declare variant match(device={kind(fpga)})
+
+This text is never parsed!
+
+#pragma omp end declare variant
+
+int also_after(void) {
+  return 0;
+}
+
+int test() {
+  return also_after() + also_before();
+} // expected-error {{expected '#pragma omp end declare variant'}}
diff --git a/clang/test/OpenMP/begin_declare_variant_range_withouth_end.c b/clang/test/OpenMP/begin_declare_variant_range_withouth_end.c
new file mode 100644
index 0000000000000..30dd0c25fccf1
--- /dev/null
+++ b/clang/test/OpenMP/begin_declare_variant_range_withouth_end.c
@@ -0,0 +1,33 @@
+// RUN: %clang_cc1 -triple=x86_64-pc-win32 -verify -fopenmp -x c -std=c99 -fms-extensions -Wno-pragma-pack %s
+// RUN: %clang_cc1 -triple=x86_64-pc-win32 -verify -fopenmp-simd -x c -std=c99 -fms-extensions -Wno-pragma-pack %s
+
+// TODO: Issue an eror message as the end is missing
+// expected-no-diagnostics
+#pragma omp begin declare variant match(device={kind(cpu)})
+int also_before(void) {
+  return 0;
+}
+
+#pragma omp begin declare variant match(device={kind(gpu)})
+int also_after(void) {
+  return 2;
+}
+int also_before(void) {
+  return 2;
+}
+#pragma omp end declare variant
+
+
+#pragma omp begin declare variant match(device={kind(fpga)})
+
+This text is never parsed!
+
+#pragma omp end declare variant
+
+int also_after(void) {
+  return 0;
+}
+
+int test() {
+  return also_after() + also_before();
+}
diff --git a/clang/test/OpenMP/begin_declare_variant_using_messages.cpp b/clang/test/OpenMP/begin_declare_variant_using_messages.cpp
new file mode 100644
index 0000000000000..174eea4243e5c
--- /dev/null
+++ b/clang/test/OpenMP/begin_declare_variant_using_messages.cpp
@@ -0,0 +1,61 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -fopenmp -verify %s
+
+namespace BEFORE_AND_1 {
+void before_and_1();
+}
+namespace AFTER_AND_2 {
+void after_and_2(); // expected-note {{'AFTER_AND_2::after_and_2' declared here}} expected-note {{'AFTER_AND_2::after_and_2' declared here}}
+}
+namespace ONLY_1 {
+void only_1(); // expected-note {{'ONLY_1::only_1' declared here}}
+}
+namespace BEFORE_1_AND_2 {
+void before_1_and_2();
+}
+
+using BEFORE_1_AND_2::before_1_and_2;
+using BEFORE_AND_1::before_and_1;
+
+void test_before() {
+  before_and_1();
+  after_and_2(); // expected-error {{use of undeclared identifier 'after_and_2'; did you mean 'AFTER_AND_2::after_and_2'?}}
+  only_1(); // expected-error {{use of undeclared identifier 'only_1'; did you mean 'ONLY_1::only_1'?}}
+  before_1_and_2();
+}
+
+#pragma omp begin declare variant match(implementation = {vendor(llvm)})
+using BEFORE_1_AND_2::before_1_and_2;
+using BEFORE_AND_1::before_and_1;
+using ONLY_1::only_1;
+void test_1() {
+  before_and_1();
+  after_and_2(); // expected-error {{use of undeclared identifier 'after_and_2'; did you mean 'AFTER_AND_2::after_and_2'?}}
+  only_1();
+  before_1_and_2();
+}
+#pragma omp end declare variant
+
+#pragma omp begin declare variant match(implementation = {vendor(llvm)})
+using AFTER_AND_2::after_and_2;
+using BEFORE_1_AND_2::before_1_and_2;
+void test_2() {
+  before_and_1();
+  after_and_2();
+  only_1();
+  before_1_and_2();
+}
+#pragma omp end declare variant
+
+void test_after() {
+  before_and_1();
+  after_and_2();
+  only_1();
+  before_1_and_2();
+}
+
+using AFTER_AND_2::after_and_2;
+
+// Make sure:
+//  - we do not see the ast nodes for the gpu kind
+//  - we do not choke on the text in the kind(fpga) guarded scopes
+//  - we pick the right cbefore_1_and_2ees
diff --git a/clang/test/OpenMP/declare_variant_ast_print.cpp b/clang/test/OpenMP/declare_variant_ast_print.cpp
index fdc6d18ca1340..7382bc89f5b7a 100644
--- a/clang/test/OpenMP/declare_variant_ast_print.cpp
+++ b/clang/test/OpenMP/declare_variant_ast_print.cpp
@@ -64,8 +64,7 @@ int baz() {
 // CHECK-NEXT: }
 // CHECK-NEXT: template<> void h_ref<double>(double *hp, double *hp2, double *hq, double *lin) {
 // CHECK-NEXT: }
-// CHECK-NEXT: template<> void h_ref<float>(float *hp, float *hp2, float *hq, float *lin) {
-// CHECK-NEXT: }
+// CHECK-NEXT: template<> void h_ref<float>(float *hp, float *hp2, float *hq, float *lin)
 template <class C>
 void h_ref(C *hp, C *hp2, C *hq, C *lin) {
 }
diff --git a/clang/test/OpenMP/declare_variant_device_kind_codegen.cpp b/clang/test/OpenMP/declare_variant_device_kind_codegen.cpp
index 55195ffd43b26..99f33741960e8 100644
--- a/clang/test/OpenMP/declare_variant_device_kind_codegen.cpp
+++ b/clang/test/OpenMP/declare_variant_device_kind_codegen.cpp
@@ -40,20 +40,11 @@
 
 // expected-no-diagnostics
 
+// CHECK-NOT: alias
+
 // CHECK-NOT: ret i32 {{1|4|81|84}}
-// CHECK-DAG: @_Z3barv = {{.*}}alias i32 (), i32 ()* @_Z3foov
-// CHECK-DAG: @_ZN16SpecSpecialFuncs6MethodEv = {{.*}}alias i32 (%struct.SpecSpecialFuncs*), i32 (%struct.SpecSpecialFuncs*)* @_ZN16SpecSpecialFuncs7method_Ev
-// CHECK-DAG: @_ZN16SpecSpecialFuncs6methodEv = linkonce_odr {{.*}}alias i32 (%struct.SpecSpecialFuncs*), i32 (%struct.SpecSpecialFuncs*)* @_ZN16SpecSpecialFuncs7method_Ev
-// CHECK-DAG: @_ZN12SpecialFuncs6methodEv = linkonce_odr {{.*}}alias i32 (%struct.SpecialFuncs*), i32 (%struct.SpecialFuncs*)* @_ZN12SpecialFuncs7method_Ev
-// CHECK-DAG: @_Z5prio_v = {{.*}}alias i32 (), i32 ()* @_Z5prio1v
-// CHECK-DAG: @_ZL6prio1_v = internal alias i32 (), i32 ()* @_ZL5prio2v
-// CHECK-DAG: @_Z4callv = {{.*}}alias i32 (), i32 ()* @_Z4testv
-// CHECK-DAG: @_ZL9stat_usedv = internal alias i32 (), i32 ()* @_ZL10stat_used_v
-// CHECK-DAG: @_ZN12SpecialFuncs6MethodEv = {{.*}}alias i32 (%struct.SpecialFuncs*), i32 (%struct.SpecialFuncs*)* @_ZN12SpecialFuncs7method_Ev
-// CHECK-DAG: @fn_linkage = {{.*}}alias i32 (), i32 ()* @_Z18fn_linkage_variantv
-// CHECK-DAG: @_Z11fn_linkage1v = {{.*}}alias i32 (), i32 ()* @fn_linkage_variant1
 // CHECK-DAG: declare {{.*}}i32 @_Z5bazzzv()
-// CHECK-DAG: declare {{.*}}i32 @_Z3bazv()
+// CHECK-DAG: define {{.*}}i32 @_Z3bazv()
 // CHECK-DAG: ret i32 2
 // CHECK-DAG: ret i32 3
 // CHECK-DAG: ret i32 5
@@ -64,7 +55,7 @@
 // CHECK-DAG: ret i32 85
 // CHECK-DAG: ret i32 86
 // CHECK-DAG: ret i32 87
-// CHECK-NOT: ret i32 {{1|4|81|84}}
+// CHECK-NOT: ret i32 {{4|81|84}}
 
 #ifndef HEADER
 #define HEADER
diff --git a/clang/test/OpenMP/declare_variant_implementation_vendor_codegen.cpp b/clang/test/OpenMP/declare_variant_implementation_vendor_codegen.cpp
index a8f6a2689b297..bc34d174c1491 100644
--- a/clang/test/OpenMP/declare_variant_implementation_vendor_codegen.cpp
+++ b/clang/test/OpenMP/declare_variant_implementation_vendor_codegen.cpp
@@ -1,33 +1,23 @@
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple %itanium_abi_triple -emit-llvm %s -fexceptions -fcxx-exceptions -o - -fsanitize-address-use-after-scope | FileCheck %s
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple %itanium_abi_triple -emit-llvm %s -fexceptions -fcxx-exceptions -o - -fsanitize-address-use-after-scope | FileCheck %s --implicit-check-not='ret i32 {{6|7|8|9|10|12|13|14|15|19|21|22|23|24}}'
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple %itanium_abi_triple -fexceptions -fcxx-exceptions -emit-pch -o %t -fopenmp-version=50 %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple %itanium_abi_triple -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - -fopenmp-version=50 | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -x c++ -triple %itanium_abi_triple -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - -fopenmp-version=50 | FileCheck %s --implicit-check-not='ret i32 {{6|7|8|9|10|12|13|14|15|19|21|22|23|24}}'
 // expected-no-diagnostics
 
-// CHECK-NOT: ret i32 {{1|4|81|84}}
-// CHECK-DAG: @_Z3barv = {{.*}}alias i32 (), i32 ()* @_Z3foov
-// CHECK-DAG: @_ZN16SpecSpecialFuncs6MethodEv = {{.*}}alias i32 (%struct.SpecSpecialFuncs*), i32 (%struct.SpecSpecialFuncs*)* @_ZN16SpecSpecialFuncs7method_Ev
-// CHECK-DAG: @_ZN16SpecSpecialFuncs6methodEv = linkonce_odr {{.*}}alias i32 (%struct.SpecSpecialFuncs*), i32 (%struct.SpecSpecialFuncs*)* @_ZN16SpecSpecialFuncs7method_Ev
-// CHECK-DAG: @_ZN12SpecialFuncs6methodEv = linkonce_odr {{.*}}alias i32 (%struct.SpecialFuncs*), i32 (%struct.SpecialFuncs*)* @_ZN12SpecialFuncs7method_Ev
-// CHECK-DAG: @_Z5prio_v = {{.*}}alias i32 (), i32 ()* @_Z5prio1v
-// CHECK-DAG: @_ZL6prio1_v = internal alias i32 (), i32 ()* @_ZL5prio2v
-// CHECK-DAG: @_Z4callv = {{.*}}alias i32 (), i32 ()* @_Z4testv
-// CHECK-DAG: @_ZL9stat_usedv = internal alias i32 (), i32 ()* @_ZL10stat_used_v
-// CHECK-DAG: @_ZN12SpecialFuncs6MethodEv = {{.*}}alias i32 (%struct.SpecialFuncs*), i32 (%struct.SpecialFuncs*)* @_ZN12SpecialFuncs7method_Ev
-// CHECK-DAG: @fn_linkage = {{.*}}alias i32 (), i32 ()* @_Z18fn_linkage_variantv
-// CHECK-DAG: @_Z11fn_linkage1v = {{.*}}alias i32 (), i32 ()* @fn_linkage_variant1
-// CHECK-DAG: declare {{.*}}i32 @_Z5bazzzv()
-// CHECK-DAG: declare {{.*}}i32 @_Z3bazv()
-// CHECK-DAG: ret i32 2
-// CHECK-DAG: ret i32 3
-// CHECK-DAG: ret i32 5
-// CHECK-DAG: ret i32 6
-// CHECK-DAG: ret i32 7
-// CHECK-DAG: ret i32 82
-// CHECK-DAG: ret i32 83
-// CHECK-DAG: ret i32 85
-// CHECK-DAG: ret i32 86
-// CHECK-DAG: ret i32 87
-// CHECK-NOT: ret i32 {{1|4|81|84}}
+// CHECK-DAG:  ret i32 2
+// CHECK-DAG:  ret i32 3
+// CHECK-DAG:  ret i32 4
+// CHECK-DAG:  ret i32 5
+// CHECK-DAG:  ret i32 11
+// CHECK-DAG:  ret i32 16
+// CHECK-DAG:  ret i32 17
+// CHECK-DAG:  ret i32 18
+// CHECK-DAG:  ret i32 19
+// CHECK-DAG:  ret i32 20
+// CHECK-DAG:  ret i32 25
+// CHECK-DAG:  ret i32 26
+// CHECK-DAG:  ret i32 27
+// CHECK-DAG:  ret i32 28
+// CHECK-DAG:  ret i32 29
 
 #ifndef HEADER
 #define HEADER
@@ -35,45 +25,45 @@
 int foo() { return 2; }
 
 #pragma omp declare variant(foo) match(implementation = {vendor(llvm)})
-int bar() { return 1; }
+int bar() { return 3; }
 
 int bazzz();
 #pragma omp declare variant(bazzz) match(implementation = {vendor(llvm)})
-int baz() { return 1; }
+int baz() { return 4; }
 
 int test();
 #pragma omp declare variant(test) match(implementation = {vendor(llvm)})
-int call() { return 1; }
+int call() { return 5; }
 
 static int stat_unused_();
 #pragma omp declare variant(stat_unused_) match(implementation = {vendor(llvm)})
-static int stat_unused() { return 1; }
+static int stat_unused() { return 6; }
 
 static int stat_used_();
 #pragma omp declare variant(stat_used_) match(implementation = {vendor(llvm)})
-static int stat_used() { return 1; }
+static int stat_used() { return 7; }
 
 int main() { return bar() + baz() + call() + stat_used(); }
 
-int test() { return 3; }
-static int stat_unused_() { return 4; }
-static int stat_used_() { return 5; }
+int test() { return 8; }
+static int stat_unused_() { return 9; }
+static int stat_used_() { return 10; }
 
 struct SpecialFuncs {
   void vd() {}
   SpecialFuncs();
   ~SpecialFuncs();
 
-  int method_() { return 6; }
+  int method_() { return 11; }
 #pragma omp declare variant(SpecialFuncs::method_)                             \
     match(implementation = {vendor(llvm)})
-  int method() { return 1; }
+  int method() { return 12; }
 #pragma omp declare variant(SpecialFuncs::method_)                             \
     match(implementation = {vendor(llvm)})
   int Method();
 } s;
 
-int SpecialFuncs::Method() { return 1; }
+int SpecialFuncs::Method() { return 13; }
 
 struct SpecSpecialFuncs {
   void vd() {}
@@ -83,50 +73,50 @@ struct SpecSpecialFuncs {
   int method_();
 #pragma omp declare variant(SpecSpecialFuncs::method_)                         \
     match(implementation = {vendor(llvm)})
-  int method() { return 1; }
+  int method() { return 14; }
 #pragma omp declare variant(SpecSpecialFuncs::method_)                         \
     match(implementation = {vendor(llvm)})
   int Method();
 } s1;
 
-int SpecSpecialFuncs::method_() { return 7; }
-int SpecSpecialFuncs::Method() { return 1; }
+int SpecSpecialFuncs::method_() { return 15; }
+int SpecSpecialFuncs::Method() { return 16; }
 
 void xxx() {
   (void)s.method();
   (void)s1.method();
 }
 
-int prio() { return 81; }
-int prio1() { return 82; }
+int prio() { return 17; }
+int prio1() { return 18; }
 
 #pragma omp declare variant(prio) match(implementation = {vendor(llvm)})
 #pragma omp declare variant(prio1) match(implementation = {vendor(score(1): llvm)})
-int prio_() { return 1; }
+int prio_() { return 19; }
 
-static int prio2() { return 83; }
-static int prio3() { return 84; }
-static int prio4() { return 84; }
+static int prio2() { return 20; }
+static int prio3() { return 21; }
+static int prio4() { return 22; }
 
 #pragma omp declare variant(prio4) match(implementation = {vendor(score(3): llvm)})
 #pragma omp declare variant(prio2) match(implementation = {vendor(score(5): llvm)})
 #pragma omp declare variant(prio3) match(implementation = {vendor(score(1): llvm)})
-static int prio1_() { return 1; }
+static int prio1_() { return 23; }
 
 int int_fn() { return prio1_(); }
 
-int fn_linkage_variant() { return 85; }
+int fn_linkage_variant() { return 24; }
 extern "C" {
 #pragma omp declare variant(fn_linkage_variant) match(implementation = {vendor(llvm)})
-int fn_linkage() { return 1; }
+int fn_linkage() { return 25; }
 }
 
-extern "C" int fn_linkage_variant1() { return 86; }
+extern "C" int fn_linkage_variant1() { return 26; }
 #pragma omp declare variant(fn_linkage_variant1) match(implementation = {vendor(llvm)})
-int fn_linkage1() { return 1; }
+int fn_linkage1() { return 27; }
 
-int fn_variant2() { return 1; }
+int fn_variant2() { return 28; }
 #pragma omp declare variant(fn_variant2) match(implementation = {vendor(llvm, ibm)})
-int fn2() { return 87; }
+int fn2() { return 29; }
 
 #endif // HEADER
diff --git a/clang/test/OpenMP/declare_variant_messages.c b/clang/test/OpenMP/declare_variant_messages.c
index 7b87e696152bd..fc9f5eb32cf26 100644
--- a/clang/test/OpenMP/declare_variant_messages.c
+++ b/clang/test/OpenMP/declare_variant_messages.c
@@ -40,12 +40,13 @@ int foo(void);
 #pragma omp declare variant(foo) match(device={kind}) // expected-warning {{the context selector 'kind' in context set 'device' requires a context property defined in parentheses; selector ignored}} expected-note {{the ignored selector spans until here}}
 #pragma omp declare variant(foo) match(device={kind(}) // expected-error {{expected ')'}} expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{context property options are: 'host' 'nohost' 'cpu' 'gpu' 'fpga' 'any'}} expected-note {{to match this '('}}
 #pragma omp declare variant(foo) match(device={kind()}) // expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{context property options are: 'host' 'nohost' 'cpu' 'gpu' 'fpga' 'any'}}
-#pragma omp declare variant(foo) match(device={kind(score cpu)}) // expected-error {{expected '(' after 'score'}} expected-warning {{expected '':'' after the score expression; '':'' assumed}}
-#pragma omp declare variant(foo) match(device={kind(score( ibm)}) // expected-error {{use of undeclared identifier 'ibm'}} expected-error {{expected ')'}} expected-warning {{expected '':'' after the score expression; '':'' assumed}} expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{context property options are: 'host' 'nohost' 'cpu' 'gpu' 'fpga' 'any'}} expected-note {{to match this '('}}
+#pragma omp declare variant(foo) match(device={kind(score cpu)}) // expected-error {{expected '(' after 'score'}} expected-warning {{expected '':'' after the score expression; '':'' assumed}} expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('<invalid>'); score ignored}}
+#pragma omp declare variant(foo) match(device={kind(score( ibm)}) // expected-error {{use of undeclared identifier 'ibm'}} expected-error {{expected ')'}} expected-warning {{expected '':'' after the score expression; '':'' assumed}} expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('<invalid>'); score ignored}} expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{context property options are: 'host' 'nohost' 'cpu' 'gpu' 'fpga' 'any'}} expected-note {{to match this '('}}
 #pragma omp declare variant(foo) match(device={kind(score(2 gpu)}) // expected-error {{expected ')'}} expected-error {{expected ')'}} expected-warning {{expected '':'' after the score expression; '':'' assumed}} expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('2'); score ignored}} expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{to match this '('}} expected-note {{context property options are: 'host' 'nohost' 'cpu' 'gpu' 'fpga' 'any'}} expected-note {{to match this '('}}
 #pragma omp declare variant(foo) match(device={kind(score(foo()) ibm)}) // expected-warning {{expected '':'' after the score expression; '':'' assumed}} expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('foo()'); score ignored}} expected-warning {{'ibm' is not a valid context property for the context selector 'kind' and the context set 'device'; property ignored}} expected-note {{try 'match(implementation={vendor(ibm)})'}} expected-note {{the ignored property spans until here}}
 #pragma omp declare variant(foo) match(device={kind(score(5): host), kind(llvm)}) // expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('5'); score ignored}} expected-warning {{the context selector 'kind' was used already in the same 'omp declare variant' directive; selector ignored}} expected-note {{the previous context selector 'kind' used here}} expected-note {{the ignored selector spans until here}}
 #pragma omp declare variant(foo) match(device={kind(score(5): nohost), vendor(llvm)}) // expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('5'); score ignored}} expected-warning {{the context selector 'vendor' is not valid for the context set 'device'; selector ignored}} expected-note {{the context selector 'vendor' can be nested in the context set 'implementation'; try 'match(implementation={vendor(property)})'}} expected-note {{the ignored selector spans until here}}
+#pragma omp declare variant(foo) match(implementation={extension("aaa")}) // expected-warning {{'aaa' is not a valid context property for the context selector 'extension' and the context set 'implementation'; property ignored}} expected-note {{context property options are: <none>}} expected-note {{the ignored property spans until here}}
 int bar(void);
 
 #pragma omp declare variant(foo) match(implementation = {vendor(score(foo) :llvm)}) // expected-warning {{score expressions in the OpenMP context selector need to be constant; foo is not and will be ignored}}
@@ -77,8 +78,8 @@ int main();
 
 
 
-#pragma omp declare variant(foo) match(xxx={}) // expected-error {{function declaration is expected after 'declare variant' directive}}
-#pragma omp declare variant(foo) match(xxx={}) // expected-error {{function declaration is expected after 'declare variant' directive}}
+#pragma omp declare variant(foo) match(implementation={vendor(llvm)}) // expected-error {{function declaration is expected after 'declare variant' directive}}
+#pragma omp declare variant(foo) match(implementation={vendor(llvm)}) // expected-error {{function declaration is expected after 'declare variant' directive}}
 #pragma init_seg(compiler)
 int main();
 
diff --git a/clang/test/OpenMP/declare_variant_messages.cpp b/clang/test/OpenMP/declare_variant_messages.cpp
index f9950a88241c4..f8271c999de3e 100644
--- a/clang/test/OpenMP/declare_variant_messages.cpp
+++ b/clang/test/OpenMP/declare_variant_messages.cpp
@@ -43,8 +43,8 @@ T foofoo();
 #pragma omp declare variant(foofoo <int>) match(device={kind}) // expected-warning {{the context selector 'kind' in context set 'device' requires a context property defined in parentheses; selector ignored}} expected-note {{the ignored selector spans until here}}
 #pragma omp declare variant(foofoo <int>) match(device={kind(}) // expected-error {{expected ')'}} expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{context property options are: 'host' 'nohost' 'cpu' 'gpu' 'fpga' 'any'}} expected-note {{to match this '('}}
 #pragma omp declare variant(foofoo <int>) match(device={kind()}) // expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{context property options are: 'host' 'nohost' 'cpu' 'gpu' 'fpga' 'any'}}
-#pragma omp declare variant(foofoo <int>) match(device={kind(score cpu)}) // expected-error {{expected '(' after 'score'}} expected-warning {{expected '':'' after the score expression; '':'' assumed}}
-#pragma omp declare variant(foofoo <int>) match(device={kind(score( ibm)}) // expected-error {{use of undeclared identifier 'ibm'}} expected-error {{expected ')'}} expected-warning {{expected '':'' after the score expression; '':'' assumed}} expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{context property options are: 'host' 'nohost' 'cpu' 'gpu' 'fpga' 'any'}} expected-note {{to match this '('}}
+#pragma omp declare variant(foofoo <int>) match(device={kind(score cpu)}) // expected-error {{expected '(' after 'score'}} expected-warning {{expected '':'' after the score expression; '':'' assumed}} expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('<invalid>'); score ignored}}
+#pragma omp declare variant(foofoo <int>) match(device={kind(score( ibm)}) // expected-error {{use of undeclared identifier 'ibm'}} expected-error {{expected ')'}} expected-warning {{expected '':'' after the score expression; '':'' assumed}} expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('<invalid>'); score ignored}} expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{context property options are: 'host' 'nohost' 'cpu' 'gpu' 'fpga' 'any'}} expected-note {{to match this '('}}
 #pragma omp declare variant(foofoo <int>) match(device={kind(score(2 gpu)}) // expected-error {{expected ')'}} expected-error {{expected ')'}} expected-warning {{expected '':'' after the score expression; '':'' assumed}} expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('2'); score ignored}} expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{to match this '('}} expected-note {{context property options are: 'host' 'nohost' 'cpu' 'gpu' 'fpga' 'any'}} expected-note {{to match this '('}}
 #pragma omp declare variant(foofoo <int>) match(device={kind(score(foofoo <int>()) ibm)}) // expected-warning {{expected '':'' after the score expression; '':'' assumed}} expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('foofoo<int>()'); score ignored}} expected-warning {{'ibm' is not a valid context property for the context selector 'kind' and the context set 'device'; property ignored}} expected-note {{try 'match(implementation={vendor(ibm)})'}} expected-note {{the ignored property spans until here}}
 #pragma omp declare variant(foofoo <int>) match(device={kind(score(5): host), kind(llvm)}) // expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('5'); score ignored}} expected-warning {{the context selector 'kind' was used already in the same 'omp declare variant' directive; selector ignored}} expected-note {{the previous context selector 'kind' used here}} expected-note {{the ignored selector spans until here}}
@@ -83,8 +83,8 @@ int bar();
 #pragma omp declare variant(foofoo <int>) match(device={kind}) // expected-warning {{the context selector 'kind' in context set 'device' requires a context property defined in parentheses; selector ignored}} expected-note {{the ignored selector spans until here}}
 #pragma omp declare variant(foofoo <int>) match(device={kind(}) // expected-error {{expected ')'}} expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{context property options are: 'host' 'nohost' 'cpu' 'gpu' 'fpga' 'any'}} expected-note {{to match this '('}}
 #pragma omp declare variant(foofoo <int>) match(device={kind()}) // expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{context property options are: 'host' 'nohost' 'cpu' 'gpu' 'fpga' 'any'}}
-#pragma omp declare variant(foofoo <int>) match(device={kind(score cpu)}) // expected-error {{expected '(' after 'score'}} expected-warning {{expected '':'' after the score expression; '':'' assumed}}
-#pragma omp declare variant(foofoo <int>) match(device={kind(score( ibm)}) // expected-error {{use of undeclared identifier 'ibm'}} expected-error {{expected ')'}} expected-warning {{expected '':'' after the score expression; '':'' assumed}} expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{context property options are: 'host' 'nohost' 'cpu' 'gpu' 'fpga' 'any'}} expected-note {{to match this '('}}
+#pragma omp declare variant(foofoo <int>) match(device={kind(score cpu)}) // expected-error {{expected '(' after 'score'}} expected-warning {{expected '':'' after the score expression; '':'' assumed}} expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('<invalid>'); score ignored}}
+#pragma omp declare variant(foofoo <int>) match(device={kind(score( ibm)}) // expected-error {{use of undeclared identifier 'ibm'}} expected-error {{expected ')'}} expected-warning {{expected '':'' after the score expression; '':'' assumed}} expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('<invalid>'); score ignored}} expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{context property options are: 'host' 'nohost' 'cpu' 'gpu' 'fpga' 'any'}} expected-note {{to match this '('}}
 #pragma omp declare variant(foofoo <int>) match(device={kind(score(C gpu)}) // expected-error {{expected ')'}} expected-error {{expected ')'}} expected-warning {{expected '':'' after the score expression; '':'' assumed}} expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('C'); score ignored}} expected-warning {{expected identifier or string literal describing a context property; property skipped}} expected-note {{to match this '('}} expected-note {{context property options are: 'host' 'nohost' 'cpu' 'gpu' 'fpga' 'any'}} expected-note {{to match this '('}}
 #pragma omp declare variant(foofoo <int>) match(device={kind(score(foofoo <int>()) ibm)}) // expected-warning {{expected '':'' after the score expression; '':'' assumed}} expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('foofoo<int>()'); score ignored}} expected-warning {{'ibm' is not a valid context property for the context selector 'kind' and the context set 'device'; property ignored}} expected-note {{try 'match(implementation={vendor(ibm)})'}} expected-note {{the ignored property spans until here}}
 #pragma omp declare variant(foofoo <int>) match(device={kind(score(C+5): host), kind(llvm)}) // expected-warning {{the context selector 'kind' in the context set 'device' cannot have a score ('C + 5'); score ignored}} expected-warning {{the context selector 'kind' was used already in the same 'omp declare variant' directive; selector ignored}} expected-note {{the previous context selector 'kind' used here}} expected-note {{the ignored selector spans until here}}
@@ -130,10 +130,16 @@ int score_and_cond_const();
 template <int C>
 int score_and_cond_const_inst();
 
+__attribute__((pure)) int pure() { return 0; }
+
+#pragma omp declare variant(pure) match(user = {condition(1)}) // expected-warning {{ignoring return value of function declared with pure attribute}}
+int unused_warning_after_specialization() { return foo(); }
+
 void score_and_cond_inst() {
   score_and_cond_non_const();
   score_and_cond_non_const_inst<8>(); // expected-note {{in instantiation of function template specialization 'score_and_cond_non_const_inst<8>' requested here}}
   score_and_cond_const_inst<9>();
+  unused_warning_after_specialization();
 }
 
 #pragma omp declare variant(barbar <int>) match(implementation = {}) // expected-warning {{expected identifier or string literal describing a context selector; selector skipped}} expected-note {{context selector options are: 'vendor' 'extension' 'unified_address' 'unified_shared_memory' 'reverse_offload' 'dynamic_allocators' 'atomic_default_mem_order'}} expected-note {{the ignored selector spans until here}}
diff --git a/clang/test/OpenMP/declare_variant_mixed_codegen.c b/clang/test/OpenMP/declare_variant_mixed_codegen.c
index 63457095b934d..9eaa35fe3c05b 100644
--- a/clang/test/OpenMP/declare_variant_mixed_codegen.c
+++ b/clang/test/OpenMP/declare_variant_mixed_codegen.c
@@ -7,7 +7,6 @@
 // RUN: %clang_cc1 -verify -fopenmp -x c -triple nvptx64-unknown-unknown -aux-triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -o - -fopenmp-version=50 | FileCheck %s --check-prefix GPU
 // expected-no-diagnostics
 
-// HOST: @base = alias i32 (double), i32 (double)* @hst
 #ifndef HEADER
 #define HEADER
 
@@ -20,9 +19,9 @@ int hst(double i) { return 1; }
 int base();
 
 // HOST-LABEL: define void @foo()
-// HOST: call i32 (double, ...) bitcast (i32 (double)* @base to i32 (double, ...)*)(double -1.000000e+00)
+// HOST: call i32 @hst(double -1.000000e+00)
 // HOST: call i32 @hst(double -2.000000e+00)
-// HOST: call void [[OFFL:@.+_foo_l29]]()
+// HOST: call void [[OFFL:@.+_foo_l28]]()
 void foo() {
   base(-1);
   hst(-2);
@@ -34,15 +33,14 @@ void foo() {
 }
 
 // HOST: define {{.*}}void [[OFFL]]()
-// HOST: call i32 (double, ...) bitcast (i32 (double)* @base to i32 (double, ...)*)(double -3.000000e+00)
+// HOST: call i32 @hst(double -3.000000e+00)
 // HOST: call i32 @dev(double -4.000000e+00)
 
-// GPU: define {{.*}}void @__omp_offloading_{{.+}}_foo_l29()
-// GPU: call i32 @base(double -3.000000e+00)
+// GPU: define {{.*}}void @__omp_offloading_{{.+}}_foo_l28()
+// GPU: call i32 @dev(double -3.000000e+00)
 // GPU: call i32 @dev(double -4.000000e+00)
 
-// GPU: define {{.*}}i32 @base(double
-// GPU: ret i32 0
+// GPU-NOT: @base
 // GPU: define {{.*}}i32 @dev(double
 // GPU: ret i32 0
 
diff --git a/clang/test/OpenMP/declare_variant_mixed_codegen.cpp b/clang/test/OpenMP/declare_variant_mixed_codegen.cpp
index 4609a4f77728f..ce48ff8e9be7d 100644
--- a/clang/test/OpenMP/declare_variant_mixed_codegen.cpp
+++ b/clang/test/OpenMP/declare_variant_mixed_codegen.cpp
@@ -1,34 +1,30 @@
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-unknown-linux -emit-llvm %s -fexceptions -fcxx-exceptions -o - -fsanitize-address-use-after-scope | FileCheck %s
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-unknown-linux -emit-llvm %s -fexceptions -fcxx-exceptions -o - -fsanitize-address-use-after-scope | FileCheck %s --implicit-check-not='ret i32 {{6|7|8|10|13|15|19|22|23|24}}'
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-unknown-linux -fexceptions -fcxx-exceptions -emit-pch -o %t -fopenmp-version=50 %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-unknown-linux -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - -fopenmp-version=50 | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-unknown-linux -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - -fopenmp-version=50 | FileCheck %s --implicit-check-not='ret i32 {{6|7|8|10|13|15|19|22|23|24}}'
+
 // expected-no-diagnostics
 
-// CHECK-NOT: ret i32 {{1|4|81|84}}
-// CHECK-DAG: @_Z3barv = {{.*}}alias i32 (), i32 ()* @_Z3foov
-// CHECK-DAG: @_ZN16SpecSpecialFuncs6MethodEv = {{.*}}alias i32 (%struct.SpecSpecialFuncs*), i32 (%struct.SpecSpecialFuncs*)* @_ZN16SpecSpecialFuncs7method_Ev
-// CHECK-DAG: @_ZN16SpecSpecialFuncs6methodEv = linkonce_odr {{.*}}alias i32 (%struct.SpecSpecialFuncs*), i32 (%struct.SpecSpecialFuncs*)* @_ZN16SpecSpecialFuncs7method_Ev
-// CHECK-DAG: @_ZN12SpecialFuncs6methodEv = linkonce_odr {{.*}}alias i32 (%struct.SpecialFuncs*), i32 (%struct.SpecialFuncs*)* @_ZN12SpecialFuncs7method_Ev
-// CHECK-DAG: @_Z5prio_v = {{.*}}alias i32 (), i32 ()* @_Z5prio1v
-// CHECK-DAG: @_ZL6prio1_v = internal alias i32 (), i32 ()* @_ZL5prio2v
-// CHECK-DAG: @_Z4callv = {{.*}}alias i32 (), i32 ()* @_Z4testv
-// CHECK-DAG: @_ZL9stat_usedv = internal alias i32 (), i32 ()* @_ZL10stat_used_v
-// CHECK-DAG: @_ZN12SpecialFuncs6MethodEv = {{.*}}alias i32 (%struct.SpecialFuncs*), i32 (%struct.SpecialFuncs*)* @_ZN12SpecialFuncs7method_Ev
-// CHECK-DAG: @fn_linkage = {{.*}}alias i32 (), i32 ()* @_Z18fn_linkage_variantv
-// CHECK-DAG: @_Z11fn_linkage1v = {{.*}}alias i32 (), i32 ()* @fn_linkage_variant1
-// CHECK-DAG: declare {{.*}}i32 @_Z5bazzzv()
-// CHECK-DAG: declare {{.*}}i32 @_Z3bazv()
-// CHECK-DAG: ret i32 2
-// CHECK-DAG: ret i32 3
-// CHECK-DAG: ret i32 5
-// CHECK-DAG: ret i32 6
-// CHECK-DAG: ret i32 7
-// CHECK-DAG: ret i32 82
-// CHECK-DAG: ret i32 83
-// CHECK-DAG: ret i32 85
-// CHECK-DAG: ret i32 86
-// CHECK-DAG: ret i32 87
-// CHECK-DAG: ret i32 88
-// CHECK-NOT: ret i32 {{1|4|81|84}}
+// CHECK-DAG:  ret i32 2
+// CHECK-DAG:  ret i32 3
+// CHECK-DAG:  ret i32 4
+// CHECK-DAG:  ret i32 5
+// CHECK-DAG:  ret i32 9
+// CHECK-DAG:  ret i32 11
+// CHECK-DAG:  ret i32 12
+// CHECK-DAG:  ret i32 14
+// CHECK-DAG:  ret i32 16
+// CHECK-DAG:  ret i32 17
+// CHECK-DAG:  ret i32 18
+// CHECK-DAG:  ret i32 19
+// CHECK-DAG:  ret i32 20
+// CHECK-DAG:  ret i32 21
+// CHECK-DAG:  ret i32 25
+// CHECK-DAG:  ret i32 26
+// CHECK-DAG:  ret i32 27
+// CHECK-DAG:  ret i32 28
+// CHECK-DAG:  ret i32 29
+// CHECK-DAG:  ret i32 30
+// CHECK-DAG:  ret i32 31
 
 #ifndef HEADER
 #define HEADER
@@ -36,47 +32,47 @@
 int foo() { return 2; }
 
 #pragma omp declare variant(foo) match(implementation = {vendor(llvm)}, device={kind(cpu)})
-int bar() { return 1; }
+int bar() { return 3; }
 
 int bazzz();
 #pragma omp declare variant(bazzz) match(implementation = {vendor(llvm)}, device={kind(host)})
-int baz() { return 1; }
+int baz() { return 4; }
 
 int test();
 #pragma omp declare variant(test) match(implementation = {vendor(llvm)}, device={kind(cpu)})
-int call() { return 1; }
+int call() { return 5; }
 
-static int stat_unused_no_emit() { return 1; }
+static int stat_unused_no_emit() { return 6; }
 static int stat_unused_();
 #pragma omp declare variant(stat_unused_) match(implementation = {vendor(llvm)}, device={kind(cpu)})
 #pragma omp declare variant(stat_unused_no_emit) match(implementation = {vendor(unknown)}, device = {kind(gpu)})
-static int stat_unused() { return 1; }
+static int stat_unused() { return 7; }
 
 static int stat_used_();
 #pragma omp declare variant(stat_used_) match(implementation = {vendor(llvm)}, device={kind(host)})
-static int stat_used() { return 1; }
+static int stat_used() { return 8; }
 
 int main() { return bar() + baz() + call() + stat_used(); }
 
-int test() { return 3; }
-static int stat_unused_() { return 4; }
-static int stat_used_() { return 5; }
+int test() { return 9; }
+static int stat_unused_() { return 10; }
+static int stat_used_() { return 11; }
 
 struct SpecialFuncs {
   void vd() {}
   SpecialFuncs();
   ~SpecialFuncs();
 
-  int method_() { return 6; }
+  int method_() { return 12; }
 #pragma omp declare variant(SpecialFuncs::method_)                             \
     match(implementation = {vendor(llvm)}, device={kind(cpu)})
-  int method() { return 1; }
+  int method() { return 13; }
 #pragma omp declare variant(SpecialFuncs::method_)                             \
     match(implementation = {vendor(llvm)}, device={kind(host)})
   int Method();
 } s;
 
-int SpecialFuncs::Method() { return 1; }
+int SpecialFuncs::Method() { return 14; }
 
 struct SpecSpecialFuncs {
   void vd() {}
@@ -86,60 +82,60 @@ struct SpecSpecialFuncs {
   int method_();
 #pragma omp declare variant(SpecSpecialFuncs::method_)                         \
     match(implementation = {vendor(llvm)}, device={kind(cpu)})
-  int method() { return 1; }
+  int method() { return 15; }
 #pragma omp declare variant(SpecSpecialFuncs::method_)                         \
     match(implementation = {vendor(llvm)}, device={kind(host)})
   int Method();
 } s1;
 
-int SpecSpecialFuncs::method_() { return 7; }
-int SpecSpecialFuncs::Method() { return 1; }
+int SpecSpecialFuncs::method_() { return 16; }
+int SpecSpecialFuncs::Method() { return 17; }
 
 void xxx() {
   (void)s.method();
   (void)s1.method();
 }
 
-int prio() { return 81; }
-int prio1() { return 82; }
+int prio() { return 18; }
+int prio1() { return 19; }
 
 #pragma omp declare variant(prio1) match(implementation = {vendor(score(2): llvm)}, device={kind(cpu,host)})
 #pragma omp declare variant(prio) match(implementation = {vendor(score(1): llvm)}, device={kind(cpu)})
-int prio_() { return 1; }
+int prio_() { return 20; }
 
-static int prio2() { return 83; }
-static int prio3() { return 84; }
-static int prio4() { return 84; }
+static int prio2() { return 21; }
+static int prio3() { return 22; }
+static int prio4() { return 23; }
 
 #pragma omp declare variant(prio4) match(implementation = {vendor(score(5): llvm)})
 #pragma omp declare variant(prio2) match(implementation = {vendor(score(8): llvm)}, device={kind(cpu,host)})
 #pragma omp declare variant(prio3) match(implementation = {vendor(score(7): llvm)}, device={kind(cpu)})
-static int prio1_() { return 1; }
+static int prio1_() { return 24; }
 
 int int_fn() { return prio1_(); }
 
-int fn_linkage_variant() { return 85; }
+int fn_linkage_variant() { return 25; }
 extern "C" {
 #pragma omp declare variant(fn_linkage_variant) match(implementation = {vendor(llvm)}, device={kind(cpu)})
-int fn_linkage() { return 1; }
+int fn_linkage() { return 26; }
 }
 
-extern "C" int fn_linkage_variant1() { return 86; }
+extern "C" int fn_linkage_variant1() { return 27; }
 #pragma omp declare variant(fn_linkage_variant1) match(implementation = {vendor(llvm)}, device={kind(host)})
-int fn_linkage1() { return 1; }
+int fn_linkage1() { return 28; }
 
-int fn_variant2() { return 1; }
+int fn_variant2() { return 29; }
 #pragma omp declare variant(fn_variant2) match(implementation = {vendor(llvm, ibm)}, device={kind(cpu)})
 #pragma omp declare variant(fn_variant2) match(implementation = {vendor(llvm)}, device={kind(cpu,gpu)})
 #pragma omp declare variant(fn_variant2) match(implementation = {vendor(llvm)}, device={kind(nohost)})
 #pragma omp declare variant(fn_variant2) match(implementation = {vendor(llvm)}, device={kind(cpu,nohost)})
 #pragma omp declare variant(fn_variant2) match(implementation = {vendor(llvm)}, device={kind(gpu)})
 #pragma omp declare variant(fn_variant2) match(implementation = {vendor(llvm)}, device={kind(fpga)})
-int fn2() { return 87; }
+int fn2() { return 30; }
 
 #pragma omp declare variant(stat_unused_no_emit) match(implementation = {vendor(unknown)}, device = {kind(gpu)})
 template <typename T>
-static T stat_unused_T() { return 88; }
+static T stat_unused_T() { return 31; }
 
 int bazzzzzzzz() {
   return stat_unused_T<int>();
diff --git a/clang/test/OpenMP/depobj_ast_print.cpp b/clang/test/OpenMP/depobj_ast_print.cpp
index 8b6586ca2b260..f3076646a25e0 100644
--- a/clang/test/OpenMP/depobj_ast_print.cpp
+++ b/clang/test/OpenMP/depobj_ast_print.cpp
@@ -17,17 +17,20 @@ void foo() {}
 template <class T>
 T tmain(T argc) {
   static T a;
-#pragma omp depobj(a) depend(in:argc)
+  int *b;
+#pragma omp depobj(a) depend(in:argc, ([4][*b][4])b)
 #pragma omp depobj(argc) destroy
 #pragma omp depobj(argc) update(inout)
   return argc;
 }
 // CHECK:      static T a;
-// CHECK-NEXT: #pragma omp depobj (a) depend(in : argc){{$}}
+// CHECK-NEXT: int *b;
+// CHECK-NEXT: #pragma omp depobj (a) depend(in : argc,([4][*b][4])b){{$}}
 // CHECK-NEXT: #pragma omp depobj (argc) destroy{{$}}
 // CHECK-NEXT: #pragma omp depobj (argc) update(inout){{$}}
 // CHECK:      static void *a;
-// CHECK-NEXT: #pragma omp depobj (a) depend(in : argc){{$}}
+// CHECK-NEXT: int *b;
+// CHECK-NEXT: #pragma omp depobj (a) depend(in : argc,([4][*b][4])b){{$}}
 // CHECK-NEXT: #pragma omp depobj (argc) destroy{{$}}
 // CHECK-NEXT: #pragma omp depobj (argc) update(inout){{$}}
 
diff --git a/clang/test/OpenMP/depobj_codegen.cpp b/clang/test/OpenMP/depobj_codegen.cpp
index 61b97d0b84796..e51c607ac55a3 100644
--- a/clang/test/OpenMP/depobj_codegen.cpp
+++ b/clang/test/OpenMP/depobj_codegen.cpp
@@ -21,8 +21,8 @@ void foo() {}
 template <class T>
 T tmain(T argc) {
   static T a;
-  void *argv;
-#pragma omp depobj(a) depend(in:argv)
+  int *argv;
+#pragma omp depobj(a) depend(in:argv, ([3][*(int*)argv][4])argv)
 #pragma omp depobj(argc) destroy
 #pragma omp depobj(argc) update(inout)
   return argc;
@@ -87,19 +87,30 @@ int main(int argc, char **argv) {
 // CHECK-LABEL: tmain
 // CHECK: [[ARGC_ADDR:%.+]] = alloca i8*,
 // CHECK: [[GTID:%.+]] = call i32 @__kmpc_global_thread_num(
-// CHECK: [[DEP_ADDR_VOID:%.+]] = call i8* @__kmpc_alloc(i32 [[GTID]], i64 48, i8* null)
-// CHECK: [[DEP_ADDR:%.+]] = bitcast i8* [[DEP_ADDR_VOID]] to [2 x %struct.kmp_depend_info]*
-// CHECK: [[BASE_ADDR:%.+]] = getelementptr inbounds [2 x %struct.kmp_depend_info], [2 x %struct.kmp_depend_info]* [[DEP_ADDR]], i{{.+}} 0, i{{.+}} 0
+// CHECK: [[DEP_ADDR_VOID:%.+]] = call i8* @__kmpc_alloc(i32 [[GTID]], i64 72, i8* null)
+// CHECK: [[DEP_ADDR:%.+]] = bitcast i8* [[DEP_ADDR_VOID]] to [3 x %struct.kmp_depend_info]*
+// CHECK: [[BASE_ADDR:%.+]] = getelementptr inbounds [3 x %struct.kmp_depend_info], [3 x %struct.kmp_depend_info]* [[DEP_ADDR]], i{{.+}} 0, i{{.+}} 0
 // CHECK: [[SZ_BASE:%.+]] = getelementptr inbounds %struct.kmp_depend_info, %struct.kmp_depend_info* [[BASE_ADDR]], i{{.+}} 0, i{{.+}} 0
-// CHECK: store i64 1, i64* [[SZ_BASE]],
-// CHECK: [[BASE_ADDR:%.+]] = getelementptr inbounds [2 x %struct.kmp_depend_info], [2 x %struct.kmp_depend_info]* [[DEP_ADDR]], i{{.+}} 0, i{{.+}} 1
+// CHECK: store i64 2, i64* [[SZ_BASE]],
+// CHECK: [[BASE_ADDR:%.+]] = getelementptr inbounds [3 x %struct.kmp_depend_info], [3 x %struct.kmp_depend_info]* [[DEP_ADDR]], i{{.+}} 0, i{{.+}} 1
 // CHECK: [[ADDR:%.+]] = getelementptr inbounds %struct.kmp_depend_info, %struct.kmp_depend_info* [[BASE_ADDR]], i{{.+}} 0, i{{.+}} 0
 // CHECK: store i64 %{{.+}}, i64* [[ADDR]],
 // CHECK: [[SZ_ADDR:%.+]] = getelementptr inbounds %struct.kmp_depend_info, %struct.kmp_depend_info* [[BASE_ADDR]], i{{.+}} 0, i{{.+}} 1
 // CHECK: store i64 8, i64* [[SZ_ADDR]],
 // CHECK: [[FLAGS_ADDR:%.+]] = getelementptr inbounds %struct.kmp_depend_info, %struct.kmp_depend_info* [[BASE_ADDR]], i{{.+}} 0, i{{.+}} 2
 // CHECK: store i8 1, i8* [[FLAGS_ADDR]],
-// CHECK: [[BASE_ADDR:%.+]] = getelementptr inbounds [2 x %struct.kmp_depend_info], [2 x %struct.kmp_depend_info]* [[DEP_ADDR]], i{{.+}} 0, i{{.+}} 1
+// CHECK: [[SHAPE_ADDR:%.+]] = load i32*, i32** [[ARGV_ADDR:%.+]],
+// CHECK: [[SZ1:%.+]] = mul nuw i64 12, %{{.+}}
+// CHECK: [[SZ:%.+]] = mul nuw i64 [[SZ1]], 4
+// CHECK: [[BASE_ADDR:%.+]] = getelementptr inbounds [3 x %struct.kmp_depend_info], [3 x %struct.kmp_depend_info]* [[DEP_ADDR]], i{{.+}} 0, i{{.+}} 2
+// CHECK: [[ADDR:%.+]] = getelementptr inbounds %struct.kmp_depend_info, %struct.kmp_depend_info* [[BASE_ADDR]], i{{.+}} 0, i{{.+}} 0
+// CHECK: [[SHAPE:%.+]] = ptrtoint i32* [[SHAPE_ADDR]] to i64
+// CHECK: store i64 [[SHAPE]], i64* [[ADDR]],
+// CHECK: [[SZ_ADDR:%.+]] = getelementptr inbounds %struct.kmp_depend_info, %struct.kmp_depend_info* [[BASE_ADDR]], i{{.+}} 0, i{{.+}} 1
+// CHECK: store i64 [[SZ]], i64* [[SZ_ADDR]],
+// CHECK: [[FLAGS_ADDR:%.+]] = getelementptr inbounds %struct.kmp_depend_info, %struct.kmp_depend_info* [[BASE_ADDR]], i{{.+}} 0, i{{.+}} 2
+// CHECK: store i8 1, i8* [[FLAGS_ADDR]],
+// CHECK: [[BASE_ADDR:%.+]] = getelementptr inbounds [3 x %struct.kmp_depend_info], [3 x %struct.kmp_depend_info]* [[DEP_ADDR]], i{{.+}} 0, i{{.+}} 1
 // CHECK: [[DEP:%.+]] = bitcast %struct.kmp_depend_info* [[BASE_ADDR]] to i8*
 // CHECK: store i8* [[DEP]], i8** [[TMAIN_A]],
 // CHECK: [[ARGC:%.+]] = load i8*, i8** [[ARGC_ADDR]],
diff --git a/clang/test/OpenMP/depobj_messages.cpp b/clang/test/OpenMP/depobj_messages.cpp
index ad28dffe184a8..1b20b1c525e77 100644
--- a/clang/test/OpenMP/depobj_messages.cpp
+++ b/clang/test/OpenMP/depobj_messages.cpp
@@ -142,7 +142,7 @@ label1 : {
 #pragma omp parallel depobj(argc) // expected-warning {{extra tokens at the end of '#pragma omp parallel' are ignored}}
   ;
 #pragma omp depobj(x) seq_cst // expected-error {{unexpected OpenMP clause 'seq_cst' in directive '#pragma omp depobj'}}
-#pragma omp depobj(x) depend(source: x) // expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
+#pragma omp depobj(x) depend(source: x) // expected-error {{expected depend modifier(iterator) or 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
 #pragma omp depobj(x) update // expected-error {{expected '(' after 'update'}}
 #pragma omp depobj(x) update( // expected-error {{expected ')'}} expected-note {{to match this '('}} expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'update'}}
 #pragma omp depobj(x) update(sink // expected-error {{expected ')'}} expected-note {{to match this '('}} expected-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'update'}}
@@ -158,7 +158,7 @@ label1 : {
 #pragma omp depend(out:x) depobj(x) // expected-error {{expected an OpenMP directive}}
 #pragma omp destroy depobj(x) // expected-error {{expected an OpenMP directive}}
 #pragma omp update(out) depobj(x) // expected-error {{expected an OpenMP directive}}
-#pragma omp depobj depend(in:x) (x) // expected-error {{expected depobj expression}} expected-warning {{extra tokens at the end of '#pragma omp depobj' are ignored}} expected-error {{expected addressable lvalue expression, array element or array section of non 'omp_depend_t' type}}
+#pragma omp depobj depend(in:x) (x) // expected-error {{expected depobj expression}} expected-warning {{extra tokens at the end of '#pragma omp depobj' are ignored}} expected-error {{expected addressable lvalue expression, array element, array section or array shaping expression of non 'omp_depend_t' type}}
 #pragma omp depobj destroy (x) // expected-error {{expected depobj expression}} expected-warning {{extra tokens at the end of '#pragma omp depobj' are ignored}}
 #pragma omp depobj update(in) (x) // expected-error {{expected depobj expression}} expected-warning {{extra tokens at the end of '#pragma omp depobj' are ignored}}
   return tmain(argc); // expected-note {{in instantiation of function template specialization 'tmain<int>' requested here}}
diff --git a/clang/test/OpenMP/distribute_parallel_for_private_messages.cpp b/clang/test/OpenMP/distribute_parallel_for_private_messages.cpp
index 05c9dbb07a364..d25598e46f816 100644
--- a/clang/test/OpenMP/distribute_parallel_for_private_messages.cpp
+++ b/clang/test/OpenMP/distribute_parallel_for_private_messages.cpp
@@ -312,7 +312,7 @@ int main(int argc, char **argv) {
     m = k + 2;
 
   s6 = s6_0; // expected-note {{in instantiation of member function 'S6<float>::operator=' requested here}}
-  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float> >::operator=' requested here}}
+  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float>>::operator=' requested here}}
   return foomain(argc, argv); // expected-note {{in instantiation of function template specialization 'foomain<int, char>' requested here}}
 }
 
diff --git a/clang/test/OpenMP/distribute_parallel_for_simd_private_messages.cpp b/clang/test/OpenMP/distribute_parallel_for_simd_private_messages.cpp
index d279d2fb9adc4..43bc6ad8e6372 100644
--- a/clang/test/OpenMP/distribute_parallel_for_simd_private_messages.cpp
+++ b/clang/test/OpenMP/distribute_parallel_for_simd_private_messages.cpp
@@ -312,7 +312,7 @@ int main(int argc, char **argv) {
     m = k + 2;
 
   s6 = s6_0; // expected-note {{in instantiation of member function 'S6<float>::operator=' requested here}}
-  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float> >::operator=' requested here}}
+  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float>>::operator=' requested here}}
   return foomain(argc, argv); // expected-note {{in instantiation of function template specialization 'foomain<int, char>' requested here}}
 }
 
diff --git a/clang/test/OpenMP/distribute_simd_private_messages.cpp b/clang/test/OpenMP/distribute_simd_private_messages.cpp
index 17ef7a0fbcfdd..261a46ac6099b 100644
--- a/clang/test/OpenMP/distribute_simd_private_messages.cpp
+++ b/clang/test/OpenMP/distribute_simd_private_messages.cpp
@@ -312,7 +312,7 @@ int main(int argc, char **argv) {
     m = k + 2;
 
   s6 = s6_0; // expected-note {{in instantiation of member function 'S6<float>::operator=' requested here}}
-  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float> >::operator=' requested here}}
+  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float>>::operator=' requested here}}
   return foomain(argc, argv); // expected-note {{in instantiation of function template specialization 'foomain<int, char>' requested here}}
 }
 
diff --git a/clang/test/OpenMP/for_private_messages.cpp b/clang/test/OpenMP/for_private_messages.cpp
index 877d683ce626b..14219882f0183 100644
--- a/clang/test/OpenMP/for_private_messages.cpp
+++ b/clang/test/OpenMP/for_private_messages.cpp
@@ -244,7 +244,7 @@ int main(int argc, char **argv) {
     si = k + 1;
 
   s6 = s6_0; // expected-note {{in instantiation of member function 'S6<float>::operator=' requested here}}
-  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float> >::operator=' requested here}}
+  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float>>::operator=' requested here}}
   return foomain(argc, argv); // expected-note {{in instantiation of function template specialization 'foomain<int, char>' requested here}}
 }
 
diff --git a/clang/test/OpenMP/for_simd_private_messages.cpp b/clang/test/OpenMP/for_simd_private_messages.cpp
index 0bf33797e0b9b..c2e2ef5e5bb29 100644
--- a/clang/test/OpenMP/for_simd_private_messages.cpp
+++ b/clang/test/OpenMP/for_simd_private_messages.cpp
@@ -234,7 +234,7 @@ int main(int argc, char **argv) {
     m = k + 2;
 
   s6 = s6_0; // expected-note {{in instantiation of member function 'S6<float>::operator=' requested here}}
-  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float> >::operator=' requested here}}
+  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float>>::operator=' requested here}}
   return foomain(argc, argv); // expected-note {{in instantiation of function template specialization 'foomain<int, char>' requested here}}
 }
 
diff --git a/clang/test/OpenMP/master_taskloop_in_reduction_messages.cpp b/clang/test/OpenMP/master_taskloop_in_reduction_messages.cpp
index b9fa587ca2ec6..d985c84579b87 100644
--- a/clang/test/OpenMP/master_taskloop_in_reduction_messages.cpp
+++ b/clang/test/OpenMP/master_taskloop_in_reduction_messages.cpp
@@ -54,7 +54,7 @@ void foobar3(int &ref) {
 }
 
 void foobar4(int &ref) {
-#pragma omp master taskloop in_reduction(min:ref) // expected-error {{in_reduction variable must appear in a task_reduction clause}}
+#pragma omp master taskloop in_reduction(min:ref)
   for (int i = 0; i < 10; ++i)
   foo();
 }
@@ -181,7 +181,7 @@ T tmain(T argc) {
   for (int i = 0; i < 10; ++i)
   foo();
 #pragma omp taskgroup task_reduction(+:c)
-#pragma omp master taskloop in_reduction(+ : a, b, c, d, f) // expected-error {{a reduction list item with incomplete type 'S1'}} expected-error 3 {{const-qualified variable cannot be in_reduction}} expected-error 2 {{'operator+' is a private member of 'S2'}} expected-error 2 {{in_reduction variable must appear in a task_reduction clause}}
+#pragma omp master taskloop in_reduction(+ : a, b, c, d, f) // expected-error {{a reduction list item with incomplete type 'S1'}} expected-error 3 {{const-qualified variable cannot be in_reduction}} expected-error 2 {{'operator+' is a private member of 'S2'}}
   for (int i = 0; i < 10; ++i)
   foo();
 #pragma omp master taskloop in_reduction(min : a, b, c, d, f) // expected-error {{a reduction list item with incomplete type 'S1'}} expected-error 4 {{arguments of OpenMP clause 'in_reduction' for 'min' or 'max' must be of arithmetic type}} expected-error 3 {{const-qualified variable cannot be in_reduction}}
@@ -199,7 +199,7 @@ T tmain(T argc) {
 #pragma omp master taskloop in_reduction(- : da) // expected-error {{const-qualified variable cannot be in_reduction}} expected-error {{const-qualified variable cannot be in_reduction}}
   for (int i = 0; i < 10; ++i)
   foo();
-#pragma omp master taskloop in_reduction(^ : fl) // expected-error {{invalid operands to binary expression ('float' and 'float')}} expected-error {{in_reduction variable must appear in a task_reduction clause}}
+#pragma omp master taskloop in_reduction(^ : fl) // expected-error {{invalid operands to binary expression ('float' and 'float')}}
   for (int i = 0; i < 10; ++i)
   foo();
 #pragma omp master taskloop in_reduction(&& : S2::S2s) // expected-error {{shared variable cannot be reduction}}
@@ -316,7 +316,7 @@ int main(int argc, char **argv) {
   for (int i = 0; i < 10; ++i)
   foo();
 #pragma omp taskgroup task_reduction(+:c)
-#pragma omp master taskloop in_reduction(+ : a, b, c, d, f) // expected-error {{a reduction list item with incomplete type 'S1'}} expected-error 2 {{const-qualified variable cannot be in_reduction}} expected-error {{'operator+' is a private member of 'S2'}} expected-error {{in_reduction variable must appear in a task_reduction clause}}
+#pragma omp master taskloop in_reduction(+ : a, b, c, d, f) // expected-error {{a reduction list item with incomplete type 'S1'}} expected-error 2 {{const-qualified variable cannot be in_reduction}} expected-error {{'operator+' is a private member of 'S2'}}
   for (int i = 0; i < 10; ++i)
   foo();
 #pragma omp master taskloop in_reduction(min : a, b, c, d, f) // expected-error {{a reduction list item with incomplete type 'S1'}} expected-error 2 {{arguments of OpenMP clause 'in_reduction' for 'min' or 'max' must be of arithmetic type}} expected-error 2 {{const-qualified variable cannot be in_reduction}}
diff --git a/clang/test/OpenMP/master_taskloop_private_messages.cpp b/clang/test/OpenMP/master_taskloop_private_messages.cpp
index 30220cb294e47..66540ecdee03b 100644
--- a/clang/test/OpenMP/master_taskloop_private_messages.cpp
+++ b/clang/test/OpenMP/master_taskloop_private_messages.cpp
@@ -253,7 +253,7 @@ int main(int argc, char **argv) {
     si = k + 1;
 
   s6 = s6_0; // expected-note {{in instantiation of member function 'S6<float>::operator=' requested here}}
-  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float> >::operator=' requested here}}
+  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float>>::operator=' requested here}}
   return foomain(argc, argv); // expected-note {{in instantiation of function template specialization 'foomain<int, char>' requested here}}
 }
 
diff --git a/clang/test/OpenMP/master_taskloop_simd_in_reduction_messages.cpp b/clang/test/OpenMP/master_taskloop_simd_in_reduction_messages.cpp
index bc69bd3f9a548..a434cda9760eb 100644
--- a/clang/test/OpenMP/master_taskloop_simd_in_reduction_messages.cpp
+++ b/clang/test/OpenMP/master_taskloop_simd_in_reduction_messages.cpp
@@ -54,7 +54,7 @@ void foobar3(int &ref) {
 }
 
 void foobar4(int &ref) {
-#pragma omp master taskloop simd in_reduction(min:ref) // expected-error {{in_reduction variable must appear in a task_reduction clause}}
+#pragma omp master taskloop simd in_reduction(min:ref)
   for (int i = 0; i < 10; ++i)
   foo();
 }
@@ -181,7 +181,7 @@ T tmain(T argc) {
   for (int i = 0; i < 10; ++i)
   foo();
 #pragma omp taskgroup task_reduction(+:c)
-#pragma omp master taskloop simd in_reduction(+ : a, b, c, d, f) // expected-error {{a reduction list item with incomplete type 'S1'}} expected-error 3 {{const-qualified variable cannot be in_reduction}} expected-error 2 {{'operator+' is a private member of 'S2'}} expected-error 2 {{in_reduction variable must appear in a task_reduction clause}}
+#pragma omp master taskloop simd in_reduction(+ : a, b, c, d, f) // expected-error {{a reduction list item with incomplete type 'S1'}} expected-error 3 {{const-qualified variable cannot be in_reduction}} expected-error 2 {{'operator+' is a private member of 'S2'}}
   for (int i = 0; i < 10; ++i)
   foo();
 #pragma omp master taskloop simd in_reduction(min : a, b, c, d, f) // expected-error {{a reduction list item with incomplete type 'S1'}} expected-error 4 {{arguments of OpenMP clause 'in_reduction' for 'min' or 'max' must be of arithmetic type}} expected-error 3 {{const-qualified variable cannot be in_reduction}}
@@ -199,7 +199,7 @@ T tmain(T argc) {
 #pragma omp master taskloop simd in_reduction(- : da) // expected-error {{const-qualified variable cannot be in_reduction}} expected-error {{const-qualified variable cannot be in_reduction}}
   for (int i = 0; i < 10; ++i)
   foo();
-#pragma omp master taskloop simd in_reduction(^ : fl) // expected-error {{invalid operands to binary expression ('float' and 'float')}} expected-error {{in_reduction variable must appear in a task_reduction clause}}
+#pragma omp master taskloop simd in_reduction(^ : fl) // expected-error {{invalid operands to binary expression ('float' and 'float')}}
   for (int i = 0; i < 10; ++i)
   foo();
 #pragma omp master taskloop simd in_reduction(&& : S2::S2s) // expected-error {{shared variable cannot be reduction}}
@@ -316,7 +316,7 @@ int main(int argc, char **argv) {
   for (int i = 0; i < 10; ++i)
   foo();
 #pragma omp taskgroup task_reduction(+:c)
-#pragma omp master taskloop simd in_reduction(+ : a, b, c, d, f) // expected-error {{a reduction list item with incomplete type 'S1'}} expected-error 2 {{const-qualified variable cannot be in_reduction}} expected-error {{'operator+' is a private member of 'S2'}} expected-error {{in_reduction variable must appear in a task_reduction clause}}
+#pragma omp master taskloop simd in_reduction(+ : a, b, c, d, f) // expected-error {{a reduction list item with incomplete type 'S1'}} expected-error 2 {{const-qualified variable cannot be in_reduction}} expected-error {{'operator+' is a private member of 'S2'}}
   for (int i = 0; i < 10; ++i)
   foo();
 #pragma omp master taskloop simd in_reduction(min : a, b, c, d, f) // expected-error {{a reduction list item with incomplete type 'S1'}} expected-error 2 {{arguments of OpenMP clause 'in_reduction' for 'min' or 'max' must be of arithmetic type}} expected-error 2 {{const-qualified variable cannot be in_reduction}}
diff --git a/clang/test/OpenMP/master_taskloop_simd_private_messages.cpp b/clang/test/OpenMP/master_taskloop_simd_private_messages.cpp
index 721c649bee17a..ea209c53b54b9 100644
--- a/clang/test/OpenMP/master_taskloop_simd_private_messages.cpp
+++ b/clang/test/OpenMP/master_taskloop_simd_private_messages.cpp
@@ -253,7 +253,7 @@ int main(int argc, char **argv) {
     si = k + 1;
 
   s6 = s6_0; // expected-note {{in instantiation of member function 'S6<float>::operator=' requested here}}
-  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float> >::operator=' requested here}}
+  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float>>::operator=' requested here}}
   return foomain(argc, argv); // expected-note {{in instantiation of function template specialization 'foomain<int, char>' requested here}}
 }
 
diff --git a/clang/test/OpenMP/nesting_of_regions.cpp b/clang/test/OpenMP/nesting_of_regions.cpp
index d987d84c79e39..9824c1df5310e 100644
--- a/clang/test/OpenMP/nesting_of_regions.cpp
+++ b/clang/test/OpenMP/nesting_of_regions.cpp
@@ -84,7 +84,7 @@ void foo() {
   }
 #pragma omp parallel
   {
-#pragma omp scan // expected-error {{region cannot be closely nested inside 'parallel' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // expected-error {{region cannot be closely nested inside 'parallel' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     bar();
   }
 #pragma omp parallel
@@ -618,7 +618,7 @@ void foo() {
   }
 #pragma omp for
   for (int i = 0; i < 10; ++i) {
-#pragma omp scan // omp45-error {{region cannot be closely nested inside 'for' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}} omp50-error {{exactly one of 'inclusive' or 'exclusive' clauses is expected}}
+#pragma omp scan // omp45-error {{region cannot be closely nested inside 'for' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}} omp50-error {{exactly one of 'inclusive' or 'exclusive' clauses is expected}}
     bar();
   }
 #pragma omp for
@@ -2618,7 +2618,7 @@ void foo() {
   }
 #pragma omp parallel for
   for (int i = 0; i < 10; ++i) {
-#pragma omp scan // expected-error {{region cannot be closely nested inside 'parallel for' region}}
+#pragma omp scan // omp45-error {{region cannot be closely nested inside 'parallel for' region}} omp50-error {{exactly one of 'inclusive' or 'exclusive' clauses is expected}}
     bar();
   }
 #pragma omp parallel for
@@ -2892,7 +2892,7 @@ void foo() {
   }
 #pragma omp parallel for simd
   for (int i = 0; i < 10; ++i) {
-#pragma omp scan // omp45-error {{OpenMP constructs may not be nested inside a simd region}} omp50-error {{region cannot be closely nested inside 'parallel for simd' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // omp45-error {{OpenMP constructs may not be nested inside a simd region}} omp50-error {{exactly one of 'inclusive' or 'exclusive' clauses is expected}}
     bar();
   }
 #pragma omp parallel for simd
@@ -4489,7 +4489,7 @@ void foo() {
   }
 #pragma omp target
   {
-#pragma omp scan // expected-error {{region cannot be closely nested inside 'target' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // expected-error {{region cannot be closely nested inside 'target' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     bar();
   }
 #pragma omp target
@@ -4728,7 +4728,7 @@ void foo() {
   }
 #pragma omp target parallel
   {
-#pragma omp scan // expected-error {{region cannot be closely nested inside 'target parallel' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // expected-error {{region cannot be closely nested inside 'target parallel' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     bar();
   }
 #pragma omp target parallel
@@ -5240,7 +5240,7 @@ void foo() {
 #pragma omp target
 #pragma omp teams
   {
-#pragma omp scan // expected-error {{region cannot be closely nested inside 'teams' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // expected-error {{region cannot be closely nested inside 'teams' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     bar();
   }
 #pragma omp target
@@ -5872,7 +5872,7 @@ void foo() {
 #pragma omp teams
 #pragma omp distribute
   for (int i = 0; i < 10; ++i) {
-#pragma omp scan // expected-error {{region cannot be closely nested inside 'distribute' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // expected-error {{region cannot be closely nested inside 'distribute' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     bar();
   }
 #pragma omp target
@@ -6510,7 +6510,7 @@ void foo() {
 #pragma omp teams
 #pragma omp distribute parallel for simd
   for (int i = 0; i < 10; ++i) {
-#pragma omp scan // omp45-error {{OpenMP constructs may not be nested inside a simd region}} omp50-error {{region cannot be closely nested inside 'distribute parallel for simd' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // omp45-error {{OpenMP constructs may not be nested inside a simd region}} omp50-error {{region cannot be closely nested inside 'distribute parallel for simd' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     bar();
   }
 #pragma omp target
@@ -6799,7 +6799,7 @@ void foo() {
   }
 #pragma omp target simd
   for (int i = 0; i < 10; ++i) {
-#pragma omp scan // omp45-error {{OpenMP constructs may not be nested inside a simd region}} omp50-error {{region cannot be closely nested inside 'target simd' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // omp45-error {{OpenMP constructs may not be nested inside a simd region}} omp50-error {{region cannot be closely nested inside 'target simd' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     bar();
   }
 #pragma omp target simd
@@ -7049,7 +7049,7 @@ void foo() {
 #pragma omp target
 #pragma omp teams distribute
   for (int i = 0; i < 10; ++i) {
-#pragma omp scan // expected-error {{region cannot be closely nested inside 'teams distribute' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // expected-error {{region cannot be closely nested inside 'teams distribute' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     bar();
   }
 #pragma omp target
@@ -7318,7 +7318,7 @@ void foo() {
 #pragma omp target
 #pragma omp teams distribute
   for (int i = 0; i < 10; ++i) {
-#pragma omp scan // expected-error {{region cannot be closely nested inside 'teams distribute' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // expected-error {{region cannot be closely nested inside 'teams distribute' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     bar();
   }
 #pragma omp target
@@ -7607,7 +7607,7 @@ void foo() {
 #pragma omp target
 #pragma omp teams distribute simd
   for (int i = 0; i < 10; ++i) {
-#pragma omp scan // omp45-error {{OpenMP constructs may not be nested inside a simd region}} omp50-error {{region cannot be closely nested inside 'teams distribute simd' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // omp45-error {{OpenMP constructs may not be nested inside a simd region}} omp50-error {{region cannot be closely nested inside 'teams distribute simd' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     bar();
   }
 #pragma omp target
@@ -7896,7 +7896,7 @@ void foo() {
 #pragma omp target
 #pragma omp teams distribute parallel for simd
   for (int i = 0; i < 10; ++i) {
-#pragma omp scan // omp45-error {{OpenMP constructs may not be nested inside a simd region}} omp50-error {{region cannot be closely nested inside 'teams distribute parallel for simd' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // omp45-error {{OpenMP constructs may not be nested inside a simd region}} omp50-error {{region cannot be closely nested inside 'teams distribute parallel for simd' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     bar();
   }
 #pragma omp target
@@ -8402,7 +8402,7 @@ void foo() {
   }
 #pragma omp target teams
   {
-#pragma omp scan // expected-error {{region cannot be closely nested inside 'target teams' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // expected-error {{region cannot be closely nested inside 'target teams' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     bar();
   }
 #pragma omp target teams
@@ -8681,7 +8681,7 @@ void foo() {
   }
 #pragma omp target teams distribute
   for (int i = 0; i < 10; ++i) {
-#pragma omp scan // expected-error {{region cannot be closely nested inside 'target teams distribute' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // expected-error {{region cannot be closely nested inside 'target teams distribute' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     bar();
   }
 #pragma omp target teams distribute
@@ -9177,7 +9177,7 @@ void foo() {
   }
 #pragma omp target teams distribute parallel for simd
   for (int i = 0; i < 10; ++i) {
-#pragma omp scan // omp45-error {{OpenMP constructs may not be nested inside a simd region}} omp50-error {{region cannot be closely nested inside 'target teams distribute parallel for simd' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // omp45-error {{OpenMP constructs may not be nested inside a simd region}} omp50-error {{region cannot be closely nested inside 'target teams distribute parallel for simd' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     bar();
   }
 #pragma omp target teams distribute parallel for simd
@@ -9197,7 +9197,7 @@ void foo() {
   }
 #pragma omp target teams distribute parallel for simd
   for (int i = 0; i < 10; ++i) {
-#pragma omp atomic // omp45-error {{OpenMP constructs may not be nested inside a simd region}}OK
+#pragma omp atomic // omp45-error {{OpenMP constructs may not be nested inside a simd region}}
     ++a;
   }
 #pragma omp target teams distribute parallel for simd
@@ -9425,7 +9425,7 @@ void foo() {
   }
 #pragma omp target teams distribute simd
   for (int i = 0; i < 10; ++i) {
-#pragma omp scan // omp45-error {{OpenMP constructs may not be nested inside a simd region}} omp50-error {{region cannot be closely nested inside 'target teams distribute simd' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // omp45-error {{OpenMP constructs may not be nested inside a simd region}} omp50-error {{region cannot be closely nested inside 'target teams distribute simd' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     bar();
   }
 #pragma omp target teams distribute simd
@@ -9445,7 +9445,7 @@ void foo() {
   }
 #pragma omp target teams distribute simd
   for (int i = 0; i < 10; ++i) {
-#pragma omp atomic // omp45-error {{OpenMP constructs may not be nested inside a simd region}}OK
+#pragma omp atomic // omp45-error {{OpenMP constructs may not be nested inside a simd region}}
     ++a;
   }
 #pragma omp target teams distribute simd
@@ -9623,7 +9623,7 @@ void foo() {
   }
 #pragma omp parallel
   {
-#pragma omp scan // expected-error {{region cannot be closely nested inside 'parallel' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // expected-error {{region cannot be closely nested inside 'parallel' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     bar();
   }
 #pragma omp parallel
@@ -12035,7 +12035,7 @@ void foo() {
   }
 #pragma omp parallel for
   for (int i = 0; i < 10; ++i) {
-#pragma omp scan // expected-error {{region cannot be closely nested inside 'parallel for' region}}
+#pragma omp scan // omp45-error {{region cannot be closely nested inside 'parallel for' region}} omp50-error {{exactly one of 'inclusive' or 'exclusive' clauses is expected}}
     bar();
   }
 #pragma omp parallel for
@@ -12310,7 +12310,7 @@ void foo() {
   }
 #pragma omp parallel for simd
   for (int i = 0; i < 10; ++i) {
-#pragma omp scan // omp45-error {{OpenMP constructs may not be nested inside a simd region}} omp50-error {{region cannot be closely nested inside 'parallel for simd' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // omp45-error {{OpenMP constructs may not be nested inside a simd region}} omp50-error {{exactly one of 'inclusive' or 'exclusive' clauses is expected}}
     bar();
   }
 #pragma omp parallel for simd
@@ -13331,7 +13331,7 @@ void foo() {
   }
 #pragma omp target
   {
-#pragma omp scan // expected-error {{region cannot be closely nested inside 'target' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // expected-error {{region cannot be closely nested inside 'target' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     bar();
   }
 #pragma omp target
@@ -13561,7 +13561,7 @@ void foo() {
   }
 #pragma omp target parallel
   {
-#pragma omp scan // expected-error {{region cannot be closely nested inside 'target parallel' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // expected-error {{region cannot be closely nested inside 'target parallel' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     bar();
   }
 #pragma omp target parallel
@@ -14072,7 +14072,7 @@ void foo() {
 #pragma omp target
 #pragma omp teams
   {
-#pragma omp scan // expected-error {{region cannot be closely nested inside 'teams' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // expected-error {{region cannot be closely nested inside 'teams' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     bar();
   }
 #pragma omp target
@@ -14678,7 +14678,7 @@ void foo() {
 #pragma omp teams
 #pragma omp distribute
   for (int i = 0; i < 10; ++i) {
-#pragma omp scan // expected-error {{region cannot be closely nested inside 'distribute' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // expected-error {{region cannot be closely nested inside 'distribute' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     bar();
   }
 #pragma omp target
@@ -15334,7 +15334,7 @@ void foo() {
 #pragma omp teams
 #pragma omp distribute parallel for simd
   for (int i = 0; i < 10; ++i) {
-#pragma omp scan // omp45-error {{OpenMP constructs may not be nested inside a simd region}} omp50-error {{region cannot be closely nested inside 'distribute parallel for simd' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // omp45-error {{OpenMP constructs may not be nested inside a simd region}} omp50-error {{region cannot be closely nested inside 'distribute parallel for simd' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     bar();
   }
 #pragma omp target
@@ -15653,7 +15653,7 @@ void foo() {
 #pragma omp teams
 #pragma omp distribute simd
   for (int i = 0; i < 10; ++i) {
-#pragma omp scan // omp45-error {{OpenMP constructs may not be nested inside a simd region}} omp50-error {{region cannot be closely nested inside 'distribute simd' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // omp45-error {{OpenMP constructs may not be nested inside a simd region}} omp50-error {{region cannot be closely nested inside 'distribute simd' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     bar();
   }
 #pragma omp target
@@ -15934,7 +15934,7 @@ void foo() {
   }
 #pragma omp target simd
   for (int i = 0; i < 10; ++i) {
-#pragma omp scan // omp45-error {{OpenMP constructs may not be nested inside a simd region}} omp50-error {{region cannot be closely nested inside 'target simd' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // omp45-error {{OpenMP constructs may not be nested inside a simd region}} omp50-error {{region cannot be closely nested inside 'target simd' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     bar();
   }
 #pragma omp target simd
@@ -16195,7 +16195,7 @@ void foo() {
 #pragma omp target
 #pragma omp teams distribute
   for (int i = 0; i < 10; ++i) {
-#pragma omp scan // expected-error {{region cannot be closely nested inside 'teams distribute' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // expected-error {{region cannot be closely nested inside 'teams distribute' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     bar();
   }
 #pragma omp target
@@ -16484,7 +16484,7 @@ void foo() {
 #pragma omp target
 #pragma omp teams distribute simd
   for (int i = 0; i < 10; ++i) {
-#pragma omp scan // omp45-error {{OpenMP constructs may not be nested inside a simd region}} omp50-error {{region cannot be closely nested inside 'teams distribute simd' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // omp45-error {{OpenMP constructs may not be nested inside a simd region}} omp50-error {{region cannot be closely nested inside 'teams distribute simd' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     bar();
   }
 #pragma omp target
@@ -16773,7 +16773,7 @@ void foo() {
 #pragma omp target
 #pragma omp teams distribute parallel for simd
   for (int i = 0; i < 10; ++i) {
-#pragma omp scan // omp45-error {{OpenMP constructs may not be nested inside a simd region}} omp50-error {{region cannot be closely nested inside 'teams distribute parallel for simd' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // omp45-error {{OpenMP constructs may not be nested inside a simd region}} omp50-error {{region cannot be closely nested inside 'teams distribute parallel for simd' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     bar();
   }
 #pragma omp target
@@ -17279,7 +17279,7 @@ void foo() {
   }
 #pragma omp target teams
   {
-#pragma omp scan // expected-error {{region cannot be closely nested inside 'target teams' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // expected-error {{region cannot be closely nested inside 'target teams' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     bar();
   }
 #pragma omp target teams
@@ -17558,7 +17558,7 @@ void foo() {
   }
 #pragma omp target teams distribute
   for (int i = 0; i < 10; ++i) {
-#pragma omp scan // expected-error {{region cannot be closely nested inside 'target teams distribute' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // expected-error {{region cannot be closely nested inside 'target teams distribute' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     bar();
   }
 #pragma omp target teams distribute
@@ -18054,7 +18054,7 @@ void foo() {
   }
 #pragma omp target teams distribute parallel for simd
   for (int i = 0; i < 10; ++i) {
-#pragma omp scan // omp45-error {{OpenMP constructs may not be nested inside a simd region}} omp50-error {{region cannot be closely nested inside 'target teams distribute parallel for simd' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // omp45-error {{OpenMP constructs may not be nested inside a simd region}} omp50-error {{region cannot be closely nested inside 'target teams distribute parallel for simd' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     bar();
   }
 #pragma omp target teams distribute parallel for simd
@@ -18074,7 +18074,7 @@ void foo() {
   }
 #pragma omp target teams distribute parallel for simd
   for (int i = 0; i < 10; ++i) {
-#pragma omp atomic // omp45-error {{OpenMP constructs may not be nested inside a simd region}}OK
+#pragma omp atomic // omp45-error {{OpenMP constructs may not be nested inside a simd region}}
     ++a;
   }
 #pragma omp target teams distribute parallel for simd
@@ -18302,7 +18302,7 @@ void foo() {
   }
 #pragma omp target teams distribute simd
   for (int i = 0; i < 10; ++i) {
-#pragma omp scan // omp45-error {{OpenMP constructs may not be nested inside a simd region}} omp50-error {{region cannot be closely nested inside 'target teams distribute simd' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, or for simd region?}}
+#pragma omp scan // omp45-error {{OpenMP constructs may not be nested inside a simd region}} omp50-error {{region cannot be closely nested inside 'target teams distribute simd' region; perhaps you forget to enclose 'omp scan' directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     bar();
   }
 #pragma omp target teams distribute simd
@@ -18322,7 +18322,7 @@ void foo() {
   }
 #pragma omp target teams distribute simd
   for (int i = 0; i < 10; ++i) {
-#pragma omp atomic // omp45-error {{OpenMP constructs may not be nested inside a simd region}}OK
+#pragma omp atomic // omp45-error {{OpenMP constructs may not be nested inside a simd region}}
     ++a;
   }
 #pragma omp target teams distribute simd
diff --git a/clang/test/OpenMP/nvptx_declare_variant_device_kind_codegen.cpp b/clang/test/OpenMP/nvptx_declare_variant_device_kind_codegen.cpp
index a9ed8f7486822..9881eab8d656f 100644
--- a/clang/test/OpenMP/nvptx_declare_variant_device_kind_codegen.cpp
+++ b/clang/test/OpenMP/nvptx_declare_variant_device_kind_codegen.cpp
@@ -1,43 +1,28 @@
 // RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm-bc %s -o %t-ppc-host.bc -fopenmp-version=50 -DGPU
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -aux-triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -fopenmp-version=50 -DGPU | FileCheck %s --implicit-check-not='ret i32 {{1|81|84}}'
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -aux-triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -fopenmp-version=50 -DGPU | FileCheck %s --implicit-check-not='ret i32 {{6|7|9|10|12|14|17|18|20|21|22|23|24|26}}'
 // RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -aux-triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -emit-pch -o %t -fopenmp-version=50 -DGPU
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -aux-triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -o - -fopenmp-version=50 -DGPU | FileCheck %s --implicit-check-not='ret i32 {{1|81|84}}'
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -aux-triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -o - -fopenmp-version=50 -DGPU | FileCheck %s --implicit-check-not='ret i32 {{6|7|9|10|12|14|17|18|20|21|22|23|24|26}}'
 
 // RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm-bc %s -o %t-ppc-host.bc -fopenmp-version=50 -DNOHOST
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -aux-triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -fopenmp-version=50 -DNOHOST | FileCheck %s --implicit-check-not='ret i32 {{1|81|84}}'
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -aux-triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -fopenmp-version=50 -DNOHOST | FileCheck %s --implicit-check-not='ret i32 {{6|7|9|10|12|14|17|18|20|21|22|23|24|26}}'
 // RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -aux-triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -emit-pch -o %t -fopenmp-version=50 -DNOHOST
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -aux-triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -o - -fopenmp-version=50 -DNOHOST | FileCheck %s --implicit-check-not='ret i32 {{1|81|84}}'
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -aux-triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -o - -fopenmp-version=50 -DNOHOST | FileCheck %s --implicit-check-not='ret i32 {{6|7|9|10|12|14|17|18|20|21|22|23|24|26}}'
 // expected-no-diagnostics
 
-// CHECK-NOT: ret i32 {{1|81|84}}
-// CHECK-DAG: define {{.*}}i32 @_Z3barv()
-// CHECK-DAG: define {{.*}}i32 @_ZN16SpecSpecialFuncs6MethodEv(%struct.SpecSpecialFuncs* %{{.+}})
-// CHECK-DAG: define {{.*}}i32 @_ZN12SpecialFuncs6MethodEv(%struct.SpecialFuncs* %{{.+}})
-// CHECK-DAG: define linkonce_odr {{.*}}i32 @_ZN16SpecSpecialFuncs6methodEv(%struct.SpecSpecialFuncs* %{{.+}})
-// CHECK-DAG: define linkonce_odr {{.*}}i32 @_ZN12SpecialFuncs6methodEv(%struct.SpecialFuncs* %{{.+}})
-// CHECK-DAG: define {{.*}}i32 @_Z5prio_v()
-// CHECK-DAG: define internal i32 @_ZL6prio1_v()
-// CHECK-DAG: define {{.*}}i32 @_Z4callv()
-// CHECK-DAG: define internal i32 @_ZL9stat_usedv()
-// CHECK-DAG: define {{.*}}i32 @fn_linkage()
-// CHECK-DAG: define {{.*}}i32 @_Z11fn_linkage1v()
-
 // CHECK-DAG: ret i32 2
 // CHECK-DAG: ret i32 3
 // CHECK-DAG: ret i32 4
 // CHECK-DAG: ret i32 5
-// CHECK-DAG: ret i32 6
-// CHECK-DAG: ret i32 7
-// CHECK-DAG: ret i32 82
-// CHECK-DAG: ret i32 83
-// CHECK-DAG: ret i32 85
-// CHECK-DAG: ret i32 86
-// CHECK-DAG: ret i32 87
-
-// Outputs for function members
-// CHECK-DAG: ret i32 6
-// CHECK-DAG: ret i32 7
-// CHECK-NOT: ret i32 {{1|81|84}}
+// CHECK-DAG: ret i32 8
+// CHECK-DAG: ret i32 11
+// CHECK-DAG: ret i32 13
+// CHECK-DAG: ret i32 15
+// CHECK-DAG: ret i32 16
+// CHECK-DAG: ret i32 19
+// CHECK-DAG: ret i32 25
+
+// Outputs for function members checked via implicit filecheck flag
+
 
 #ifndef HEADER
 #define HEADER
@@ -62,19 +47,19 @@ static int stat_used_();
 #pragma omp declare target
 
 #pragma omp declare variant(foo) match(device = {kind(CORRECT)})
-int bar() { return 1; }
+int bar() { return 3; }
 
 #pragma omp declare variant(bazzz) match(device = {kind(CORRECT)})
-int baz() { return 1; }
+int baz() { return 4; }
 
 #pragma omp declare variant(test) match(device = {kind(CORRECT)})
-int call() { return 1; }
+int call() { return 5; }
 
 #pragma omp declare variant(stat_unused_) match(device = {kind(CORRECT)})
-static int stat_unused() { return 1; }
+static int stat_unused() { return 6; }
 
 #pragma omp declare variant(stat_used_) match(device = {kind(CORRECT)})
-static int stat_used() { return 1; }
+static int stat_used() { return 7; }
 
 #pragma omp end declare target
 
@@ -86,9 +71,9 @@ int main() {
   return res;
 }
 
-int test() { return 3; }
-static int stat_unused_() { return 4; }
-static int stat_used_() { return 5; }
+int test() { return 8; }
+static int stat_unused_() { return 9; }
+static int stat_used_() { return 10; }
 
 #pragma omp declare target
 
@@ -96,17 +81,16 @@ struct SpecialFuncs {
   void vd() {}
   SpecialFuncs();
   ~SpecialFuncs();
-
-  int method_() { return 6; }
+  int method_() { return 11; }
 #pragma omp declare variant(SpecialFuncs::method_) \
     match(device = {kind(CORRECT)})
-  int method() { return 1; }
+  int method() { return 12; }
 #pragma omp declare variant(SpecialFuncs::method_) \
     match(device = {kind(CORRECT)})
   int Method();
 } s;
 
-int SpecialFuncs::Method() { return 1; }
+int SpecialFuncs::Method() { return 13; }
 
 struct SpecSpecialFuncs {
   void vd() {}
@@ -116,7 +100,7 @@ struct SpecSpecialFuncs {
   int method_();
 #pragma omp declare variant(SpecSpecialFuncs::method_) \
     match(device = {kind(CORRECT)})
-  int method() { return 1; }
+  int method() { return 14; }
 #pragma omp declare variant(SpecSpecialFuncs::method_) \
     match(device = {kind(CORRECT)})
   int Method();
@@ -124,17 +108,17 @@ struct SpecSpecialFuncs {
 
 #pragma omp end declare target
 
-int SpecSpecialFuncs::method_() { return 7; }
-int SpecSpecialFuncs::Method() { return 1; }
+int SpecSpecialFuncs::method_() { return 15; }
+int SpecSpecialFuncs::Method() { return 16; }
 
-int prio() { return 81; }
-int prio1() { return 82; }
-static int prio2() { return 83; }
-static int prio3() { return 84; }
-static int prio4() { return 84; }
-int fn_linkage_variant() { return 85; }
-extern "C" int fn_linkage_variant1() { return 86; }
-int fn_variant2() { return 1; }
+int prio() { return 17; }
+int prio1() { return 18; }
+static int prio2() { return 19; }
+static int prio3() { return 20; }
+static int prio4() { return 21; }
+int fn_linkage_variant() { return 22; }
+extern "C" int fn_linkage_variant1() { return 23; }
+int fn_variant2() { return 24; }
 
 #pragma omp declare target
 
@@ -145,25 +129,25 @@ void xxx() {
 
 #pragma omp declare variant(prio) match(device = {kind(SUBSET)})
 #pragma omp declare variant(prio1) match(device = {kind(CORRECT)})
-int prio_() { return 1; }
+int prio_() { return 25; }
 
 #pragma omp declare variant(prio4) match(device = {kind(SUBSET)})
 #pragma omp declare variant(prio2) match(device = {kind(CORRECT)})
 #pragma omp declare variant(prio3) match(device = {kind(SUBSET)})
-static int prio1_() { return 1; }
+static int prio1_() { return 26; }
 
 int int_fn() { return prio1_(); }
 
 extern "C" {
 #pragma omp declare variant(fn_linkage_variant) match(device = {kind(CORRECT)})
-int fn_linkage() { return 1; }
+int fn_linkage() { return 27; }
 }
 
 #pragma omp declare variant(fn_linkage_variant1) match(device = {kind(CORRECT)})
-int fn_linkage1() { return 1; }
+int fn_linkage1() { return 28; }
 
 #pragma omp declare variant(fn_variant2) match(device = {kind(WRONG)})
-int fn2() { return 87; }
+int fn2() { return 29; }
 
 #pragma omp end declare target
 
diff --git a/clang/test/OpenMP/nvptx_declare_variant_implementation_vendor_codegen.cpp b/clang/test/OpenMP/nvptx_declare_variant_implementation_vendor_codegen.cpp
index 04870f0845982..4cece48a067bc 100644
--- a/clang/test/OpenMP/nvptx_declare_variant_implementation_vendor_codegen.cpp
+++ b/clang/test/OpenMP/nvptx_declare_variant_implementation_vendor_codegen.cpp
@@ -1,38 +1,20 @@
 // RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm-bc %s -o %t-ppc-host.bc -fopenmp-version=50
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -aux-triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -fopenmp-version=50 | FileCheck %s --implicit-check-not='ret i32 {{1|81|84}}'
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -aux-triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -fopenmp-version=50 | FileCheck %s --implicit-check-not='ret i32 {{6|7|8|9|10|12|13|14|15|17|18|19|20|21|22|23|24|26}}'
 // RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -aux-triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -emit-pch -o %t -fopenmp-version=50
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -aux-triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -o - -fopenmp-version=50 | FileCheck %s --implicit-check-not='ret i32 {{1|81|84}}'
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -aux-triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -o - -fopenmp-version=50 | FileCheck %s --implicit-check-not='ret i32 {{6|7|8|9|10|12|13|14|15|17|18|19|20|21|22|23|24|26}}'
 // expected-no-diagnostics
 
-// CHECK-NOT: ret i32 {{1|81|84}}
-// CHECK-DAG: define {{.*}}i32 @_Z3barv()
-// CHECK-DAG: define {{.*}}i32 @_ZN16SpecSpecialFuncs6MethodEv(%struct.SpecSpecialFuncs* %{{.+}})
-// CHECK-DAG: define {{.*}}i32 @_ZN12SpecialFuncs6MethodEv(%struct.SpecialFuncs* %{{.+}})
-// CHECK-DAG: define linkonce_odr {{.*}}i32 @_ZN16SpecSpecialFuncs6methodEv(%struct.SpecSpecialFuncs* %{{.+}})
-// CHECK-DAG: define linkonce_odr {{.*}}i32 @_ZN12SpecialFuncs6methodEv(%struct.SpecialFuncs* %{{.+}})
-// CHECK-DAG: define {{.*}}i32 @_Z5prio_v()
-// CHECK-DAG: define internal i32 @_ZL6prio1_v()
-// CHECK-DAG: define {{.*}}i32 @_Z4callv()
-// CHECK-DAG: define internal i32 @_ZL9stat_usedv()
-// CHECK-DAG: define {{.*}}i32 @fn_linkage()
-// CHECK-DAG: define {{.*}}i32 @_Z11fn_linkage1v()
-
-// CHECK-DAG: ret i32 2
-// CHECK-DAG: ret i32 3
-// CHECK-DAG: ret i32 4
-// CHECK-DAG: ret i32 5
-// CHECK-DAG: ret i32 6
-// CHECK-DAG: ret i32 7
-// CHECK-DAG: ret i32 82
-// CHECK-DAG: ret i32 83
-// CHECK-DAG: ret i32 85
-// CHECK-DAG: ret i32 86
-// CHECK-DAG: ret i32 87
-
-// Outputs for function members
-// CHECK-DAG: ret i32 6
-// CHECK-DAG: ret i32 7
-// CHECK-NOT: ret i32 {{1|81|84}}
+// CHECK-DAG:  ret i32 2
+// CHECK-DAG:  ret i32 3
+// CHECK-DAG:  ret i32 4
+// CHECK-DAG:  ret i32 5
+// CHECK-DAG:  ret i32 11
+// CHECK-DAG:  ret i32 16
+// CHECK-DAG:  ret i32 19
+// CHECK-DAG:  ret i32 25
+// CHECK-DAG:  ret i32 27
+// CHECK-DAG:  ret i32 28
+// CHECK-DAG:  ret i32 29
 
 #ifndef HEADER
 #define HEADER
@@ -46,19 +28,19 @@ static int stat_used_();
 #pragma omp declare target
 
 #pragma omp declare variant(foo) match(implementation = {vendor(llvm)})
-int bar() { return 1; }
+int bar() { return 3; }
 
 #pragma omp declare variant(bazzz) match(implementation = {vendor(llvm)})
-int baz() { return 1; }
+int baz() { return 4; }
 
 #pragma omp declare variant(test) match(implementation = {vendor(llvm)})
-int call() { return 1; }
+int call() { return 5; }
 
 #pragma omp declare variant(stat_unused_) match(implementation = {vendor(llvm)})
-static int stat_unused() { return 1; }
+static int stat_unused() { return 6; }
 
 #pragma omp declare variant(stat_used_) match(implementation = {vendor(llvm)})
-static int stat_used() { return 1; }
+static int stat_used() { return 7; }
 
 #pragma omp end declare target
 
@@ -70,9 +52,9 @@ int main() {
   return res;
 }
 
-int test() { return 3; }
-static int stat_unused_() { return 4; }
-static int stat_used_() { return 5; }
+int test() { return 8; }
+static int stat_unused_() { return 9; }
+static int stat_used_() { return 10; }
 
 #pragma omp declare target
 
@@ -81,16 +63,16 @@ struct SpecialFuncs {
   SpecialFuncs();
   ~SpecialFuncs();
 
-  int method_() { return 6; }
+  int method_() { return 11; }
 #pragma omp declare variant(SpecialFuncs::method_) \
     match(implementation = {vendor(llvm)})
-  int method() { return 1; }
+  int method() { return 12; }
 #pragma omp declare variant(SpecialFuncs::method_) \
     match(implementation = {vendor(llvm)})
   int Method();
 } s;
 
-int SpecialFuncs::Method() { return 1; }
+int SpecialFuncs::Method() { return 13; }
 
 struct SpecSpecialFuncs {
   void vd() {}
@@ -100,7 +82,7 @@ struct SpecSpecialFuncs {
   int method_();
 #pragma omp declare variant(SpecSpecialFuncs::method_) \
     match(implementation = {vendor(llvm)})
-  int method() { return 1; }
+  int method() { return 14; }
 #pragma omp declare variant(SpecSpecialFuncs::method_) \
     match(implementation = {vendor(llvm)})
   int Method();
@@ -108,17 +90,17 @@ struct SpecSpecialFuncs {
 
 #pragma omp end declare target
 
-int SpecSpecialFuncs::method_() { return 7; }
-int SpecSpecialFuncs::Method() { return 1; }
+int SpecSpecialFuncs::method_() { return 15; }
+int SpecSpecialFuncs::Method() { return 16; }
 
-int prio() { return 81; }
-int prio1() { return 82; }
-static int prio2() { return 83; }
-static int prio3() { return 84; }
-static int prio4() { return 84; }
-int fn_linkage_variant() { return 85; }
-extern "C" int fn_linkage_variant1() { return 86; }
-int fn_variant2() { return 1; }
+int prio() { return 17; }
+int prio1() { return 18; }
+static int prio2() { return 19; }
+static int prio3() { return 20; }
+static int prio4() { return 21; }
+int fn_linkage_variant() { return 22; }
+extern "C" int fn_linkage_variant1() { return 23; }
+int fn_variant2() { return 24; }
 
 #pragma omp declare target
 
@@ -130,7 +112,7 @@ void xxx() {
 #pragma omp declare variant(prio) match(implementation = {vendor(llvm)})
 #pragma omp declare variant(prio1) match(implementation = {vendor(score(1) \
                                                                   : llvm)})
-int prio_() { return 1; }
+int prio_() { return 25; }
 
 #pragma omp declare variant(prio4) match(implementation = {vendor(score(3) \
                                                                   : llvm)})
@@ -138,20 +120,20 @@ int prio_() { return 1; }
                                                                   : llvm)})
 #pragma omp declare variant(prio3) match(implementation = {vendor(score(1) \
                                                                   : llvm)})
-static int prio1_() { return 1; }
+static int prio1_() { return 26; }
 
 int int_fn() { return prio1_(); }
 
 extern "C" {
 #pragma omp declare variant(fn_linkage_variant) match(implementation = {vendor(llvm)})
-int fn_linkage() { return 1; }
+int fn_linkage() { return 27; }
 }
 
 #pragma omp declare variant(fn_linkage_variant1) match(implementation = {vendor(llvm)})
-int fn_linkage1() { return 1; }
+int fn_linkage1() { return 28; }
 
 #pragma omp declare variant(fn_variant2) match(implementation = {vendor(llvm, ibm)})
-int fn2() { return 87; }
+int fn2() { return 29; }
 
 #pragma omp end declare target
 
diff --git a/clang/test/OpenMP/nvptx_target_exceptions_messages.cpp b/clang/test/OpenMP/nvptx_target_exceptions_messages.cpp
index faff77e0a43b7..c71615d2521fc 100644
--- a/clang/test/OpenMP/nvptx_target_exceptions_messages.cpp
+++ b/clang/test/OpenMP/nvptx_target_exceptions_messages.cpp
@@ -1,5 +1,10 @@
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm-bc %s -o %t-ppc-host.bc -fexceptions -fcxx-exceptions
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -fexceptions -fcxx-exceptions -ferror-limit 100
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown \
+// RUN:   -verify=host -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm-bc \
+// RUN:   %s -o %t-ppc-host.bc -fexceptions -fcxx-exceptions
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown \
+// RUN:   -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s \
+// RUN:   -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - \
+// RUN:   -fexceptions -fcxx-exceptions -ferror-limit 100
 
 #ifndef HEADER
 #define HEADER
@@ -81,4 +86,17 @@ int (*B)() = &foobar2;
 int foobar1() { throw 1; }
 int foobar2() { throw 1; } // expected-error {{cannot use 'throw' with exceptions disabled}}
 
+
+int foobar3();
+int (*C)() = &foobar3; // expected-warning {{declaration is not declared in any declare target region}}
+                       // host-warning@-1 {{declaration is not declared in any declare target region}}
+#pragma omp declare target
+int (*D)() = C; // expected-note {{used here}}
+                // host-note@-1 {{used here}}
+#pragma omp end declare target
+int foobar3() { throw 1; }
+
+// Check no infinite recursion in deferred diagnostic emitter.
+long E = (long)&E;
+
 #endif // HEADER
diff --git a/clang/test/OpenMP/parallel_for_private_messages.cpp b/clang/test/OpenMP/parallel_for_private_messages.cpp
index 3f742600e1a67..4c888c0eab220 100644
--- a/clang/test/OpenMP/parallel_for_private_messages.cpp
+++ b/clang/test/OpenMP/parallel_for_private_messages.cpp
@@ -234,7 +234,7 @@ int main(int argc, char **argv) {
     m = k + 2;
 
   s6 = s6_0; // expected-note {{in instantiation of member function 'S6<float>::operator=' requested here}}
-  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float> >::operator=' requested here}}
+  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float>>::operator=' requested here}}
   return foomain(argc, argv); // expected-note {{in instantiation of function template specialization 'foomain<int, char>' requested here}}
 }
 
diff --git a/clang/test/OpenMP/parallel_for_reduction_messages.cpp b/clang/test/OpenMP/parallel_for_reduction_messages.cpp
index b28fe6c379992..50a93129b1fa2 100644
--- a/clang/test/OpenMP/parallel_for_reduction_messages.cpp
+++ b/clang/test/OpenMP/parallel_for_reduction_messages.cpp
@@ -1,10 +1,10 @@
-// RUN: %clang_cc1 -verify -fopenmp -ferror-limit 150 -o - %s -Wuninitialized
-// RUN: %clang_cc1 -verify -fopenmp -std=c++98 -ferror-limit 150 -o - %s -Wuninitialized
-// RUN: %clang_cc1 -verify -fopenmp -std=c++11 -ferror-limit 150 -o - %s -Wuninitialized
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=50 -ferror-limit 150 -o - %s -Wuninitialized
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=50 -std=c++98 -ferror-limit 150 -o - %s -Wuninitialized
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=50 -std=c++11 -ferror-limit 150 -o - %s -Wuninitialized
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 150 -o - %s -Wuninitialized
-// RUN: %clang_cc1 -verify -fopenmp-simd -std=c++98 -ferror-limit 150 -o - %s -Wuninitialized
-// RUN: %clang_cc1 -verify -fopenmp-simd -std=c++11 -ferror-limit 150 -o - %s -Wuninitialized
+// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=50 -ferror-limit 150 -o - %s -Wuninitialized
+// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=50 -std=c++98 -ferror-limit 150 -o - %s -Wuninitialized
+// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=50 -std=c++11 -ferror-limit 150 -o - %s -Wuninitialized
 
 extern int omp_default_mem_alloc;
 void xxx(int argc) {
@@ -78,6 +78,14 @@ class S5 {
 #pragma omp for reduction(+:a) // expected-error {{reduction variable must be shared}}
   for (int i = 0; i < 10; ++i)
     ::foo();
+#pragma omp parallel for reduction(inscan, +:a)
+  for (int i = 0; i < 10; ++i) {
+#pragma omp scan inclusive(a)
+  }
+#pragma omp parallel for reduction(inscan, +:a)
+  for (int i = 0; i < 10; ++i) {
+#pragma omp scan exclusive(a)
+  }
   }
 };
 class S6 { // expected-note 3 {{candidate function (the implicit copy assignment operator) not viable: no known conversion from 'int' to 'const S6' for 1st argument}}
@@ -333,5 +341,21 @@ int main(int argc, char **argv) {
   for (int i = 0; i < 10; ++i)
     m++;
 
+#pragma omp parallel for reduction(inscan, + : m) reduction(*: fl) reduction(default, &&: j) // expected-error 2 {{expected 'reduction' clause with the 'inscan' modifier}} expected-note 2 {{'reduction' clause with 'inscan' modifier is used here}}
+  for (int i = 0; i < 10; ++i) {
+#pragma omp scan exclusive(m)
+    m++;
+  }
+#pragma omp parallel for reduction(inscan, + : m, fl, j) // expected-error 2 {{the inscan reduction list item must appear as a list item in an 'inclusive' or 'exclusive' clause on an inner 'omp scan' directive}}
+  for (int i = 0; i < 10; ++i) {
+#pragma omp scan exclusive(m)
+    m++;
+  }
+#pragma omp parallel for reduction(inscan, + : m, fl, j) // expected-error 2 {{the inscan reduction list item must appear as a list item in an 'inclusive' or 'exclusive' clause on an inner 'omp scan' directive}}
+  for (int i = 0; i < 10; ++i) {
+#pragma omp scan inclusive(m)
+    m++;
+  }
+
   return tmain(argc) + tmain(fl); // expected-note {{in instantiation of function template specialization 'tmain<int>' requested here}} expected-note {{in instantiation of function template specialization 'tmain<float>' requested here}}
 }
diff --git a/clang/test/OpenMP/parallel_for_simd_private_messages.cpp b/clang/test/OpenMP/parallel_for_simd_private_messages.cpp
index 40ea8ecdc1595..05de18e9f7dca 100644
--- a/clang/test/OpenMP/parallel_for_simd_private_messages.cpp
+++ b/clang/test/OpenMP/parallel_for_simd_private_messages.cpp
@@ -234,7 +234,7 @@ int main(int argc, char **argv) {
     m = k + 3;
 
   s6 = s6_0; // expected-note {{in instantiation of member function 'S6<float>::operator=' requested here}}
-  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float> >::operator=' requested here}}
+  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float>>::operator=' requested here}}
   return foomain(argc, argv); // expected-note {{in instantiation of function template specialization 'foomain<int, char>' requested here}}
 }
 
diff --git a/clang/test/OpenMP/parallel_master_private_messages.cpp b/clang/test/OpenMP/parallel_master_private_messages.cpp
index 2e21a1acd3fda..93716dab69ae3 100644
--- a/clang/test/OpenMP/parallel_master_private_messages.cpp
+++ b/clang/test/OpenMP/parallel_master_private_messages.cpp
@@ -278,7 +278,7 @@ int main(int argc, char **argv) {
   }
 
   s6 = s6_0; // expected-note {{in instantiation of member function 'S6<float>::operator=' requested here}}
-  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float> >::operator=' requested here}}
+  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float>>::operator=' requested here}}
   return foomain(argc, argv); // expected-note {{in instantiation of function template specialization 'foomain<int, char>' requested here}}
 }
 
diff --git a/clang/test/OpenMP/parallel_master_taskloop_private_messages.cpp b/clang/test/OpenMP/parallel_master_taskloop_private_messages.cpp
index 47b40f91b746d..c6d2ca59515f2 100644
--- a/clang/test/OpenMP/parallel_master_taskloop_private_messages.cpp
+++ b/clang/test/OpenMP/parallel_master_taskloop_private_messages.cpp
@@ -253,7 +253,7 @@ int main(int argc, char **argv) {
     si = k + 1;
 
   s6 = s6_0; // expected-note {{in instantiation of member function 'S6<float>::operator=' requested here}}
-  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float> >::operator=' requested here}}
+  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float>>::operator=' requested here}}
   return foomain(argc, argv); // expected-note {{in instantiation of function template specialization 'foomain<int, char>' requested here}}
 }
 
diff --git a/clang/test/OpenMP/parallel_master_taskloop_simd_private_messages.cpp b/clang/test/OpenMP/parallel_master_taskloop_simd_private_messages.cpp
index eb1b5c7a1eb9f..2feee9aaf372c 100644
--- a/clang/test/OpenMP/parallel_master_taskloop_simd_private_messages.cpp
+++ b/clang/test/OpenMP/parallel_master_taskloop_simd_private_messages.cpp
@@ -253,7 +253,7 @@ int main(int argc, char **argv) {
     si = k + 1;
 
   s6 = s6_0; // expected-note {{in instantiation of member function 'S6<float>::operator=' requested here}}
-  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float> >::operator=' requested here}}
+  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float>>::operator=' requested here}}
   return foomain(argc, argv); // expected-note {{in instantiation of function template specialization 'foomain<int, char>' requested here}}
 }
 
diff --git a/clang/test/OpenMP/parallel_reduction_messages.c b/clang/test/OpenMP/parallel_reduction_messages.c
index 61a3e9300436c..5d6010d59d500 100644
--- a/clang/test/OpenMP/parallel_reduction_messages.c
+++ b/clang/test/OpenMP/parallel_reduction_messages.c
@@ -2,7 +2,7 @@
 
 int incomplete[];
 
-void test() {
+void test(int *p) {
   int a;
 #pragma omp parallel reduction( // expected-error {{expected identifier}} expected-error {{expected ')'}} expected-warning {{missing ':' after reduction identifier - ignoring}} expected-note {{to match this '('}}
   ;
@@ -10,11 +10,13 @@ void test() {
   ;
 #pragma omp parallel reduction(default, // expected-error {{expected identifier}} expected-error {{expected ')'}} expected-warning {{missing ':' after reduction identifier - ignoring}} expected-note {{to match this '('}}
   ;
-#pragma omp parallel reduction(unknown, +: a) // expected-error {{expected 'default' in OpenMP clause 'reduction'}}
+#pragma omp parallel reduction(unknown, +: a) // expected-error {{expected 'default' or 'inscan' in OpenMP clause 'reduction'}}
   ;
 #pragma omp parallel reduction(default, + : a)
   ;
-#pragma omp parallel reduction(+ : incomplete) // expected-error {{a reduction list item with incomplete type 'int []'}}
+#pragma omp parallel reduction(inscan, + : a) // expected-error {{'inscan' modifier can be used only in 'omp for', 'omp simd', 'omp for simd', 'omp parallel for', or 'omp parallel for simd' directive}}
+  ;
+#pragma omp parallel reduction(+ : incomplete, ([10])p) // expected-error {{a reduction list item with incomplete type 'int []'}} expected-error {{expected variable name, array element or array section}}
   ;
 }
 
diff --git a/clang/test/OpenMP/parallel_sections_private_messages.cpp b/clang/test/OpenMP/parallel_sections_private_messages.cpp
index 7e9a6e41b7383..728e418158109 100644
--- a/clang/test/OpenMP/parallel_sections_private_messages.cpp
+++ b/clang/test/OpenMP/parallel_sections_private_messages.cpp
@@ -278,7 +278,7 @@ int main(int argc, char **argv) {
   }
 
   s6 = s6_0; // expected-note {{in instantiation of member function 'S6<float>::operator=' requested here}}
-  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float> >::operator=' requested here}}
+  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float>>::operator=' requested here}}
   return foomain(argc, argv); // expected-note {{in instantiation of function template specialization 'foomain<int, char>' requested here}}
 }
 
diff --git a/clang/test/OpenMP/scan_ast_print.cpp b/clang/test/OpenMP/scan_ast_print.cpp
index 4b9eca6f7ec9f..0bee5d9f744a8 100644
--- a/clang/test/OpenMP/scan_ast_print.cpp
+++ b/clang/test/OpenMP/scan_ast_print.cpp
@@ -15,33 +15,35 @@ void foo() {}
 template <class T>
 T tmain(T argc) {
   static T a;
-#pragma omp for
+#pragma omp for reduction(inscan, +: a)
   for (int i = 0; i < 10; ++i) {
 #pragma omp scan inclusive(a)
   }
   return a + argc;
 }
 // CHECK:      static T a;
-// CHECK-NEXT: #pragma omp for
+// CHECK-NEXT: #pragma omp for reduction(inscan, +: a)
 // CHECK-NEXT: for (int i = 0; i < 10; ++i) {
 // CHECK-NEXT: #pragma omp scan inclusive(a){{$}}
 // CHECK:      static int a;
-// CHECK-NEXT: #pragma omp for
+// CHECK-NEXT: #pragma omp for reduction(inscan, +: a)
 // CHECK-NEXT: for (int i = 0; i < 10; ++i) {
 // CHECK-NEXT: #pragma omp scan inclusive(a)
 // CHECK:      static char a;
-// CHECK-NEXT: #pragma omp for
+// CHECK-NEXT: #pragma omp for reduction(inscan, +: a)
 // CHECK-NEXT: for (int i = 0; i < 10; ++i) {
 // CHECK-NEXT: #pragma omp scan inclusive(a)
 
 int main(int argc, char **argv) {
   static int a;
 // CHECK: static int a;
-#pragma omp for simd
+#pragma omp parallel
+#pragma omp for simd reduction(inscan, ^: a, argc)
   for (int i = 0; i < 10; ++i) {
 #pragma omp scan exclusive(a, argc)
   }
-// CHECK-NEXT: #pragma omp for simd
+// CHECK-NEXT: #pragma omp parallel
+// CHECK-NEXT: #pragma omp for simd reduction(inscan, ^: a,argc)
 // CHECK-NEXT: for (int i = 0; i < 10; ++i) {
 // CHECK-NEXT: #pragma omp scan exclusive(a,argc){{$}}
   return tmain(argc) + tmain(argv[0][0]) + a;
diff --git a/clang/test/OpenMP/scan_messages.cpp b/clang/test/OpenMP/scan_messages.cpp
index 9f093858c2490..9ce8314511c1f 100644
--- a/clang/test/OpenMP/scan_messages.cpp
+++ b/clang/test/OpenMP/scan_messages.cpp
@@ -1,9 +1,10 @@
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=50 -ferror-limit 100 %s
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=50 -ferror-limit 150 %s
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 %s
+// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=50 -ferror-limit 150 %s
 
 template <class T>
-T tmain(T argc) {
+T tmain() {
+  static T argc;
 #pragma omp for
   for (int i = 0; i < 10; ++i) {
 #pragma omp scan // expected-error {{exactly one of 'inclusive' or 'exclusive' clauses is expected}}
@@ -15,43 +16,43 @@ T tmain(T argc) {
 #pragma omp scan untied  // expected-error {{unexpected OpenMP clause 'untied' in directive '#pragma omp scan'}} expected-error {{exactly one of 'inclusive' or 'exclusive' clauses is expected}}
 #pragma omp scan unknown // expected-warning {{extra tokens at the end of '#pragma omp scan' are ignored}} expected-error {{exactly one of 'inclusive' or 'exclusive' clauses is expected}}
   }
-#pragma omp for simd
+#pragma omp for simd reduction(inscan, +: argc)
   for (int i = 0; i < 10; ++i)
     if (argc)
 #pragma omp scan inclusive(argc) // expected-error {{'#pragma omp scan' cannot be an immediate substatement}}
     if (argc) {
-#pragma omp scan inclusive(argc)
+#pragma omp scan inclusive(argc) // expected-error {{orphaned 'omp scan' directives are prohibited; perhaps you forget to enclose the directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     }
-#pragma omp simd
+#pragma omp simd reduction(inscan, +: argc)
   for (int i = 0; i < 10; ++i)
   while (argc)
 #pragma omp scan inclusive(argc) // expected-error {{'#pragma omp scan' cannot be an immediate substatement}}
     while (argc) {
-#pragma omp scan inclusive(argc)
+#pragma omp scan inclusive(argc) // expected-error {{orphaned 'omp scan' directives are prohibited; perhaps you forget to enclose the directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     }
-#pragma omp simd
+#pragma omp simd reduction(inscan, +: argc)
   for (int i = 0; i < 10; ++i)
   do
 #pragma omp scan inclusive(argc) // expected-error {{'#pragma omp scan' cannot be an immediate substatement}}
     while (argc)
       ;
-#pragma omp simd
+#pragma omp simd reduction(inscan, +: argc)
   for (int i = 0; i < 10; ++i)
   do {
 #pragma omp scan inclusive(argc)
   } while (argc);
-#pragma omp simd
+#pragma omp simd reduction(inscan, +: argc)
   for (int i = 0; i < 10; ++i)
   switch (argc)
 #pragma omp scan inclusive(argc) // expected-error {{'#pragma omp scan' cannot be an immediate substatement}}
     switch (argc)
     case 1:
-#pragma omp scan inclusive(argc) // expected-error {{'#pragma omp scan' cannot be an immediate substatement}}
+#pragma omp scan inclusive(argc) // expected-error {{'#pragma omp scan' cannot be an immediate substatement}} expected-error {{orphaned 'omp scan' directives are prohibited; perhaps you forget to enclose the directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
   switch (argc)
   case 1: {
-#pragma omp scan inclusive(argc)
+#pragma omp scan inclusive(argc) // expected-error {{orphaned 'omp scan' directives are prohibited; perhaps you forget to enclose the directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
   }
-#pragma omp simd
+#pragma omp simd reduction(inscan, +: argc)
   for (int i = 0; i < 10; ++i)
   switch (argc) {
 #pragma omp scan exclusive(argc) // expected-note 2 {{previous 'scan' directive used here}}
@@ -62,19 +63,19 @@ T tmain(T argc) {
 #pragma omp scan exclusive(argc) // expected-error {{exactly one 'scan' directive must appear in the loop body of an enclosing directive}}
   } break;
   }
-#pragma omp simd
+#pragma omp simd reduction(inscan, +: argc)
   for (int i = 0; i < 10; ++i)
   for (;;)
 #pragma omp scan exclusive(argc) // expected-error {{'#pragma omp scan' cannot be an immediate substatement}}
     for (;;) {
-#pragma omp scan exclusive(argc)
+#pragma omp scan exclusive(argc) // expected-error {{orphaned 'omp scan' directives are prohibited; perhaps you forget to enclose the directive into a for, simd, for simd, parallel for, or parallel for simd region?}}
     }
-#pragma omp simd
+#pragma omp simd reduction(inscan, +: argc)
   for (int i = 0; i < 10; ++i) {
 label:
 #pragma omp scan exclusive(argc)
   }
-#pragma omp simd
+#pragma omp simd reduction(inscan, +: argc)
   for (int i = 0; i < 10; ++i) {
 label1 : {
 #pragma omp scan inclusive(argc)
@@ -83,64 +84,65 @@ label1 : {
   return T();
 }
 
-int main(int argc, char **argv) {
-#pragma omp simd
+int main() {
+  static int argc;
+#pragma omp simd reduction(inscan, +: argc)
   for (int i = 0; i < 10; ++i) {
 #pragma omp scan inclusive(argc) inclusive(argc) // expected-error {{exactly one of 'inclusive' or 'exclusive' clauses is expected}}
   ;
   }
-#pragma omp simd
+#pragma omp simd reduction(inscan, +: argc)
   for (int i = 0; i < 10; ++i) {
 #pragma omp scan exclusive(argc) inclusive(argc) // expected-error {{exactly one of 'inclusive' or 'exclusive' clauses is expected}}
   ;
   }
-#pragma omp simd
+#pragma omp simd reduction(inscan, +: argc)
   for (int i = 0; i < 10; ++i) {
 #pragma omp scan exclusive(argc) exclusive(argc) // expected-error {{exactly one of 'inclusive' or 'exclusive' clauses is expected}}
   ;
   }
-#pragma omp simd
+#pragma omp simd reduction(inscan, +: argc) // expected-error {{the inscan reduction list item must appear as a list item in an 'inclusive' or 'exclusive' clause on an inner 'omp scan' directive}}
   for (int i = 0; i < 10; ++i) {
 #pragma omp scan untied  // expected-error {{unexpected OpenMP clause 'untied' in directive '#pragma omp scan'}} expected-error {{exactly one of 'inclusive' or 'exclusive' clauses is expected}}
 #pragma omp scan unknown // expected-warning {{extra tokens at the end of '#pragma omp scan' are ignored}} expected-error {{exactly one of 'inclusive' or 'exclusive' clauses is expected}}
   }
-#pragma omp simd
+#pragma omp simd reduction(inscan, +: argc)
   for (int i = 0; i < 10; ++i)
   if (argc)
 #pragma omp scan inclusive(argc) // expected-error {{'#pragma omp scan' cannot be an immediate substatement}}
     if (argc) {
-#pragma omp scan inclusive(argc)
+#pragma omp scan inclusive(argc) // expected-error {{orphaned 'omp scan' directives are prohibited; perhaps you forget to enclose the directive into a for, simd, for simd, parallel for, or parallel for simd region?}} expected-error {{the list item must appear in 'reduction' clause with the 'inscan' modifier of the parent directive}}
     }
-#pragma omp simd
+#pragma omp simd reduction(inscan, +: argc)
   for (int i = 0; i < 10; ++i)
   while (argc)
 #pragma omp scan inclusive(argc) // expected-error {{'#pragma omp scan' cannot be an immediate substatement}}
     while (argc) {
-#pragma omp scan inclusive(argc)
+#pragma omp scan inclusive(argc) // expected-error {{orphaned 'omp scan' directives are prohibited; perhaps you forget to enclose the directive into a for, simd, for simd, parallel for, or parallel for simd region?}} expected-error {{the list item must appear in 'reduction' clause with the 'inscan' modifier of the parent directive}}
     }
-#pragma omp simd
+#pragma omp simd reduction(inscan, +: argc)
   for (int i = 0; i < 10; ++i)
   do
 #pragma omp scan inclusive(argc) // expected-error {{'#pragma omp scan' cannot be an immediate substatement}}
     while (argc)
       ;
-#pragma omp simd
+#pragma omp simd reduction(inscan, +: argc)
   for (int i = 0; i < 10; ++i)
   do {
 #pragma omp scan exclusive(argc)
   } while (argc);
-#pragma omp simd
+#pragma omp simd reduction(inscan, +: argc)
   for (int i = 0; i < 10; ++i)
   switch (argc)
 #pragma omp scan exclusive(argc) // expected-error {{'#pragma omp scan' cannot be an immediate substatement}}
     switch (argc)
     case 1:
-#pragma omp scan exclusive(argc) // expected-error {{'#pragma omp scan' cannot be an immediate substatement}}
+#pragma omp scan exclusive(argc) // expected-error {{'#pragma omp scan' cannot be an immediate substatement}} expected-error {{orphaned 'omp scan' directives are prohibited; perhaps you forget to enclose the directive into a for, simd, for simd, parallel for, or parallel for simd region?}} expected-error {{the list item must appear in 'reduction' clause with the 'inscan' modifier of the parent directive}}
   switch (argc)
   case 1: {
-#pragma omp scan exclusive(argc)
+#pragma omp scan exclusive(argc) // expected-error {{orphaned 'omp scan' directives are prohibited; perhaps you forget to enclose the directive into a for, simd, for simd, parallel for, or parallel for simd region?}} expected-error {{the list item must appear in 'reduction' clause with the 'inscan' modifier of the parent directive}}
   }
-#pragma omp simd
+#pragma omp simd reduction(inscan, +: argc)
   for (int i = 0; i < 10; ++i)
   switch (argc) {
 #pragma omp scan inclusive(argc) // expected-note 2 {{previous 'scan' directive used here}}
@@ -151,24 +153,24 @@ int main(int argc, char **argv) {
 #pragma omp scan inclusive(argc) // expected-error {{exactly one 'scan' directive must appear in the loop body of an enclosing directive}}
   } break;
   }
-#pragma omp simd
+#pragma omp simd reduction(inscan, +: argc)
   for (int i = 0; i < 10; ++i)
   for (;;)
 #pragma omp scan inclusive(argc) // expected-error {{'#pragma omp scan' cannot be an immediate substatement}}
     for (;;) {
-#pragma omp scan inclusive(argc)
+#pragma omp scan inclusive(argc) // expected-error {{orphaned 'omp scan' directives are prohibited; perhaps you forget to enclose the directive into a for, simd, for simd, parallel for, or parallel for simd region?}} expected-error {{the list item must appear in 'reduction' clause with the 'inscan' modifier of the parent directive}}
     }
-#pragma omp simd
+#pragma omp simd reduction(inscan, +: argc)
   for (int i = 0; i < 10; ++i) {
 label:
 #pragma omp scan inclusive(argc)
   }
-#pragma omp simd
+#pragma omp simd reduction(inscan, +: argc)
   for (int i = 0; i < 10; ++i) {
 label1 : {
 #pragma omp scan inclusive(argc)
 }
 }
 
-  return tmain(argc);
+  return tmain<int>();
 }
diff --git a/clang/test/OpenMP/sections_private_messages.cpp b/clang/test/OpenMP/sections_private_messages.cpp
index e032c7a5eceaa..c89effd85a6e2 100644
--- a/clang/test/OpenMP/sections_private_messages.cpp
+++ b/clang/test/OpenMP/sections_private_messages.cpp
@@ -278,7 +278,7 @@ int main(int argc, char **argv) {
   }
 
   s6 = s6_0; // expected-note {{in instantiation of member function 'S6<float>::operator=' requested here}}
-  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float> >::operator=' requested here}}
+  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float>>::operator=' requested here}}
   return foomain(argc, argv); // expected-note {{in instantiation of function template specialization 'foomain<int, char>' requested here}}
 }
 
diff --git a/clang/test/OpenMP/simd_private_messages.cpp b/clang/test/OpenMP/simd_private_messages.cpp
index 12927b07d1999..dea0574d75aa0 100644
--- a/clang/test/OpenMP/simd_private_messages.cpp
+++ b/clang/test/OpenMP/simd_private_messages.cpp
@@ -191,7 +191,7 @@ int main(int argc, char **argv) {
   for (int k = 0; k < argc; ++k) ++k;
 
   s6 = s6_0; // expected-note {{in instantiation of member function 'S6<float>::operator=' requested here}}
-  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float> >::operator=' requested here}}
+  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float>>::operator=' requested here}}
   return foomain(argc, argv); // expected-note {{in instantiation of function template specialization 'foomain<int, char>' requested here}}
 }
 
diff --git a/clang/test/OpenMP/single_private_messages.cpp b/clang/test/OpenMP/single_private_messages.cpp
index e809f0732a1b3..ff742ac26f246 100644
--- a/clang/test/OpenMP/single_private_messages.cpp
+++ b/clang/test/OpenMP/single_private_messages.cpp
@@ -200,7 +200,7 @@ int main(int argc, char **argv) {
   foo();
 
   s6 = s6_0; // expected-note {{in instantiation of member function 'S6<float>::operator=' requested here}}
-  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float> >::operator=' requested here}}
+  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float>>::operator=' requested here}}
   return foomain(argc, argv); // expected-note {{in instantiation of function template specialization 'foomain<int, char>' requested here}}
 }
 
diff --git a/clang/test/OpenMP/target_data_ast_print.cpp b/clang/test/OpenMP/target_data_ast_print.cpp
index fa67c1834aa42..fcd6e928655c2 100644
--- a/clang/test/OpenMP/target_data_ast_print.cpp
+++ b/clang/test/OpenMP/target_data_ast_print.cpp
@@ -1,10 +1,10 @@
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -ast-print %s | FileCheck %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=50 -ast-print %s | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -x c++ -std=c++11 -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -ast-print %s | FileCheck %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s
+// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=50 -ast-print %s | FileCheck %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s
 // expected-no-diagnostics
 
 #ifndef HEADER
@@ -139,6 +139,8 @@ int main (int argc, char **argv) {
   static int a;
 // CHECK: static int a;
 
+#pragma omp target data map(to: ([argc][3][a])argv)
+  // CHECK: #pragma omp target data map(to: ([argc][3][a])argv)
 #pragma omp target data map(to: c)
 // CHECK:      #pragma omp target data map(to: c)
   a=2;
diff --git a/clang/test/OpenMP/target_firstprivate_messages.cpp b/clang/test/OpenMP/target_firstprivate_messages.cpp
index f5e0c4c9143d0..0a3079bbf83e7 100644
--- a/clang/test/OpenMP/target_firstprivate_messages.cpp
+++ b/clang/test/OpenMP/target_firstprivate_messages.cpp
@@ -211,7 +211,7 @@ int main(int argc, char **argv) {
 #pragma omp target map(i) firstprivate(i) // expected-error {{firstprivate variable cannot be in a map clause in '#pragma omp target' directive}}
   {}
   s6 = s6_0; // expected-note {{in instantiation of member function 'S6<float>::operator=' requested here}}
-  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float> >::operator=' requested here}}
+  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float>>::operator=' requested here}}
   return foomain(argc, argv); // expected-note {{in instantiation of function template specialization 'foomain<int, char>' requested here}}
 }
 
diff --git a/clang/test/OpenMP/target_map_codegen.cpp b/clang/test/OpenMP/target_map_codegen.cpp
index b9766e82ce03d..ecfe50c01ea6d 100644
--- a/clang/test/OpenMP/target_map_codegen.cpp
+++ b/clang/test/OpenMP/target_map_codegen.cpp
@@ -5353,5 +5353,81 @@ void explicit_maps_single (int ii){
 // CK31: define {{.+}}[[CALL00]]
 // CK31: define {{.+}}[[CALL01]]
 
+#endif
+///==========================================================================///
+// RUN: %clang_cc1 -DCK32 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK32 --check-prefix CK32-64
+// RUN: %clang_cc1 -DCK32 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK32 --check-prefix CK32-64
+// RUN: %clang_cc1 -DCK32 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK32 --check-prefix CK32-32
+// RUN: %clang_cc1 -DCK32 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK32 --check-prefix CK32-32
+
+// RUN: %clang_cc1 -DCK32 -verify -fopenmp-simd -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY32 %s
+// RUN: %clang_cc1 -DCK32 -fopenmp-simd -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY32 %s
+// RUN: %clang_cc1 -DCK32 -verify -fopenmp-simd -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY32 %s
+// RUN: %clang_cc1 -DCK32 -fopenmp-simd -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY32 %s
+// SIMD-ONLY32-NOT: {{__kmpc|__tgt}}
+#ifdef CK32
+
+// CK32-DAG: [[MTYPE_TO:@.+]] = {{.+}}constant [1 x i64] [i64 33]
+// CK32-DAG: [[MTYPE_FROM:@.+]] = {{.+}}constant [1 x i64] [i64 34]
+
+void array_shaping(float *f, int sa) {
+
+  // CK32-DAG: call i32 @__tgt_target(i64 -1, i8* @{{.+}}, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE_TO]]{{.+}})
+  // CK32-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
+  // CK32-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
+  // CK32-DAG: [[GEPS]] = getelementptr inbounds {{.+}}[[S:%[^,]+]]
+
+  // CK32-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
+  // CK32-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
+  // CK32-DAG: [[S0:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 0
+
+  // CK32-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to float**
+  // CK32-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to float**
+
+  // CK32-DAG: store float* [[F1:%.+]], float** [[BPC0]],
+  // CK32-DAG: store float* [[F2:%.+]], float** [[PC0]],
+  // CK32-DAG: store i64 [[SIZE:%.+]], i64* [[S0]],
+
+  // CK32-DAG: [[F1]] = load float*, float** [[F_ADDR:%.+]],
+  // CK32-DAG: [[F2]] = load float*, float** [[F_ADDR]],
+  // CK32-64-DAG: [[SIZE]] = mul nuw i64 [[SZ1:%.+]], 4
+  // CK32-64-DAG: [[SZ1]] = mul nuw i64 12, %{{.+}}
+  // CK32-32-DAG: [[SIZE]] = sext i32 [[SZ1:%.+]] to i64
+  // CK32-32-DAG: [[SZ1]] = mul nuw i32 [[SZ2:%.+]], 4
+  // CK32-32-DAG: [[SZ2]] = mul nuw i32 12, %{{.+}}
+  #pragma omp target map(to:([3][sa][4])f)
+  f[0] = 1;
+  sa = 1;
+  // CK32-DAG: call i32 @__tgt_target(i64 -1, i8* @{{.+}}, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE_FROM]]{{.+}})
+  // CK32-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
+  // CK32-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
+  // CK32-DAG: [[GEPS]] = getelementptr inbounds {{.+}}[[S:%[^,]+]]
+
+  // CK32-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
+  // CK32-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
+  // CK32-DAG: [[S0:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 0
+
+  // CK32-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to float**
+  // CK32-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to float**
+
+  // CK32-DAG: store float* [[F1:%.+]], float** [[BPC0]],
+  // CK32-DAG: store float* [[F2:%.+]], float** [[PC0]],
+  // CK32-DAG: store i64 [[SIZE:%.+]], i64* [[S0]],
+
+  // CK32-DAG: [[F1]] = load float*, float** [[F_ADDR:%.+]],
+  // CK32-DAG: [[F2]] = load float*, float** [[F_ADDR]],
+  // CK32-64-DAG: [[SIZE]] = mul nuw i64 [[SZ1:%.+]], 5
+  // CK32-64-DAG: [[SZ1]] = mul nuw i64 4, %{{.+}}
+  // CK32-32-DAG: [[SIZE]] = sext i32 [[SZ1:%.+]] to i64
+  // CK32-32-DAG: [[SZ1]] = mul nuw i32 [[SZ2:%.+]], 5
+  // CK32-32-DAG: [[SZ2]] = mul nuw i32 4, %{{.+}}
+  #pragma omp target map(from: ([sa][5])f)
+  f[0] = 1;
+}
+
 #endif
 #endif
diff --git a/clang/test/OpenMP/target_map_messages.cpp b/clang/test/OpenMP/target_map_messages.cpp
index 96932af6a04c5..a18590fc85fea 100644
--- a/clang/test/OpenMP/target_map_messages.cpp
+++ b/clang/test/OpenMP/target_map_messages.cpp
@@ -140,6 +140,8 @@ struct SA {
     {}
     #pragma omp target map(close bf: a)   // expected-error {{incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'}}
     {}
+    #pragma omp target map(([b[I]][bf])f)  // le45-error {{expected ',' or ']' in lambda capture list}} le45-error {{expected ')'}} le45-note {{to match this '('}}
+    {}
     return;
   }
 };
@@ -189,203 +191,209 @@ void SAclient(int arg) {
 
   SD u;
   SC r(p),t(p);
-  #pragma omp target map(r)
+#pragma omp target map(r)
   {}
-  #pragma omp target map(marr[2][0:2][0:2]) // expected-error {{array section does not specify contiguous storage}}
+#pragma omp target map(marr[2] [0:2] [0:2]) // expected-error {{array section does not specify contiguous storage}}
   {}
-  #pragma omp target map(marr[:][0:2][0:2]) // expected-error {{array section does not specify contiguous storage}}
+#pragma omp target map(marr[:] [0:2] [0:2]) // expected-error {{array section does not specify contiguous storage}}
   {}
-  #pragma omp target map(marr[2][3][0:2])
+#pragma omp target map(marr[2][3] [0:2])
   {}
-  #pragma omp target map(marr[:][:][:])
+#pragma omp target map(marr[:][:][:])
   {}
-  #pragma omp target map(marr[:2][:][:])
+#pragma omp target map(marr[:2][:][:])
   {}
-  #pragma omp target map(marr[arg:][:][:])
+#pragma omp target map(marr [arg:][:][:])
   {}
-  #pragma omp target map(marr[arg:])
+#pragma omp target map(marr [arg:])
   {}
-  #pragma omp target map(marr[arg:][:arg][:]) // correct if arg is the size of dimension 2
+#pragma omp target map(marr [arg:][:arg][:]) // correct if arg is the size of dimension 2
   {}
-  #pragma omp target map(marr[:arg][:])
+#pragma omp target map(marr[:arg][:])
   {}
-  #pragma omp target map(marr[:arg][n:])
+#pragma omp target map(marr[:arg] [n:])
   {}
-  #pragma omp target map(marr[:][:arg][n:]) // correct if arg is the size of  dimension 2
+#pragma omp target map(marr[:][:arg] [n:]) // correct if arg is the size of  dimension 2
   {}
-  #pragma omp target map(marr[:][:m][n:]) // expected-error {{array section does not specify contiguous storage}}
+#pragma omp target map(marr[:][:m] [n:])   // expected-error {{array section does not specify contiguous storage}}
   {}
-  #pragma omp target map(marr[n:m][:arg][n:])
+#pragma omp target map(marr [n:m][:arg] [n:])
   {}
-  #pragma omp target map(marr[:2][:1][:]) // expected-error {{array section does not specify contiguous storage}}
+#pragma omp target map(marr[:2][:1][:])  // expected-error {{array section does not specify contiguous storage}}
   {}
-  #pragma omp target map(marr[:2][1:][:]) // expected-error {{array section does not specify contiguous storage}}
+#pragma omp target map(marr[:2] [1:][:]) // expected-error {{array section does not specify contiguous storage}}
   {}
-  #pragma omp target map(marr[:2][:][:1]) // expected-error {{array section does not specify contiguous storage}}
+#pragma omp target map(marr[:2][:][:1])  // expected-error {{array section does not specify contiguous storage}}
   {}
-  #pragma omp target map(marr[:2][:][1:]) // expected-error {{array section does not specify contiguous storage}}
+#pragma omp target map(marr[:2][:] [1:]) // expected-error {{array section does not specify contiguous storage}}
   {}
-  #pragma omp target map(marr[:1][:2][:])
+#pragma omp target map(marr[:1][:2][:])
   {}
-  #pragma omp target map(marr[:1][0][:])
+#pragma omp target map(marr[:1][0][:])
   {}
-  #pragma omp target map(marr[:arg][:2][:]) // correct if arg is 1
+#pragma omp target map(marr[:arg][:2][:]) // correct if arg is 1
   {}
-  #pragma omp target map(marr[:1][3:1][:2])
+#pragma omp target map(marr[:1] [3:1][:2])
   {}
-  #pragma omp target map(marr[:1][3:arg][:2]) // correct if arg is 1
+#pragma omp target map(marr[:1] [3:arg][:2]) // correct if arg is 1
   {}
-  #pragma omp target map(marr[:1][3:2][:2]) // expected-error {{array section does not specify contiguous storage}}
+#pragma omp target map(marr[:1] [3:2][:2])   // expected-error {{array section does not specify contiguous storage}}
   {}
-  #pragma omp target map(marr[:2][:10][:])
+#pragma omp target map(marr[:2][:10][:])
   {}
-  #pragma omp target map(marr[:2][:][:5+5])
+#pragma omp target map(marr[:2][:][:5 + 5])
   {}
-  #pragma omp target map(marr[:2][2+2-4:][0:5+5])
+#pragma omp target map(marr[:2] [2 + 2 - 4:] [0:5 + 5])
   {}
 
-  #pragma omp target map(marr[:1][:2][0]) // expected-error {{array section does not specify contiguous storage}}
+#pragma omp target map(marr[:1][:2][0]) // expected-error {{array section does not specify contiguous storage}}
   {}
-  #pragma omp target map(marr2[:1][:2][0])
+#pragma omp target map(marr2[:1][:2][0])
   {}
 
-  #pragma omp target map(mvla[:1][:][0]) // correct if the size of dimension 2 is 1.
+#pragma omp target map(mvla[:1][:][0])    // correct if the size of dimension 2 is 1.
   {}
-  #pragma omp target map(mvla[:2][:arg][:]) // correct if arg is the size of dimension 2.
+#pragma omp target map(mvla[:2][:arg][:]) // correct if arg is the size of dimension 2.
   {}
-  #pragma omp target map(mvla[:1][:2][0]) // expected-error {{array section does not specify contiguous storage}}
+#pragma omp target map(mvla[:1][:2][0])   // expected-error {{array section does not specify contiguous storage}}
    {}
-  #pragma omp target map(mvla[1][2:arg][:])
+#pragma omp target map(mvla[1] [2:arg][:])
   {}
-  #pragma omp target map(mvla[:1][:][:])
+#pragma omp target map(mvla[:1][:][:])
   {}
-  #pragma omp target map(mvla2[:1][:2][:11])
+#pragma omp target map(mvla2[:1][:2][:11])
   {}
-  #pragma omp target map(mvla2[:1][:2][:10]) // expected-error {{array section does not specify contiguous storage}}
+#pragma omp target map(mvla2[:1][:2][:10]) // expected-error {{array section does not specify contiguous storage}}
   {}
 
-  #pragma omp target map(mptr[:2][2+2-4:1][0:5+5]) // expected-error {{array section does not specify contiguous storage}}
+#pragma omp target map(mptr[:2] [2 + 2 - 4:1] [0:5 + 5]) // expected-error {{array section does not specify contiguous storage}}
   {}
-  #pragma omp target map(mptr[:1][:2-1][2:4-3])
+#pragma omp target map(mptr[:1][:2 - 1] [2:4 - 3])
   {}
-  #pragma omp target map(mptr[:1][:arg][2:4-3]) // correct if arg is 1.
+#pragma omp target map(mptr[:1][:arg] [2:4 - 3]) // correct if arg is 1.
   {}
-  #pragma omp target map(mptr[:1][:2-1][0:2])
+#pragma omp target map(mptr[:1][:2 - 1] [0:2])
   {}
-  #pragma omp target map(mptr[:1][:2][0:2]) // expected-error {{array section does not specify contiguous storage}}
+#pragma omp target map(mptr[:1][:2] [0:2]) // expected-error {{array section does not specify contiguous storage}}
   {}
-  #pragma omp target map(mptr[:1][:][0:2]) // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
+#pragma omp target map(mptr[:1][:] [0:2])  // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
   {}
-  #pragma omp target map(mptr[:2][:1][0:2]) // expected-error {{array section does not specify contiguous storage}}
+#pragma omp target map(mptr[:2][:1] [0:2]) // expected-error {{array section does not specify contiguous storage}}
   {}
 
-  #pragma omp target map(r.ArrS[0].B)
+#pragma omp target map(r.ArrS[0].B)
   {}
-  #pragma omp target map(r.ArrS[:1].B) // expected-error {{OpenMP array section is not allowed here}}
+#pragma omp target map(r.ArrS[:1].B)   // expected-error {{OpenMP array section is not allowed here}}
   {}
-  #pragma omp target map(r.ArrS[:arg].B) // expected-error {{OpenMP array section is not allowed here}}
+#pragma omp target map(r.ArrS[:arg].B) // expected-error {{OpenMP array section is not allowed here}}
   {}
-  #pragma omp target map(r.ArrS[0].Arr[1:23])
+#pragma omp target map(r.ArrS[0].Arr [1:23])
   {}
-  #pragma omp target map(r.ArrS[0].Arr[1:arg])
+#pragma omp target map(r.ArrS[0].Arr [1:arg])
   {}
-  #pragma omp target map(r.ArrS[0].Arr[arg:23])
+#pragma omp target map(r.ArrS[0].Arr [arg:23])
   {}
-  #pragma omp target map(r.ArrS[0].Error) // expected-error {{no member named 'Error' in 'SB'}}
+#pragma omp target map(r.ArrS[0].Error)          // expected-error {{no member named 'Error' in 'SB'}}
   {}
-  #pragma omp target map(r.ArrS[0].A, r.ArrS[1].A) // expected-error {{multiple array elements associated with the same variable are not allowed in map clauses of the same construct}} expected-note {{used here}}
+#pragma omp target map(r.ArrS[0].A, r.ArrS[1].A) // expected-error {{multiple array elements associated with the same variable are not allowed in map clauses of the same construct}} expected-note {{used here}}
   {}
-  #pragma omp target map(r.ArrS[0].A, t.ArrS[1].A)
+#pragma omp target map(r.ArrS[0].A, t.ArrS[1].A)
   {}
-  #pragma omp target map(r.PtrS[0], r.PtrS->B) // expected-error {{same pointer dereferenced in multiple different ways in map clause expressions}} expected-note {{used here}}
+#pragma omp target map(r.PtrS[0], r.PtrS->B) // expected-error {{same pointer dereferenced in multiple different ways in map clause expressions}} expected-note {{used here}}
   {}
-  #pragma omp target map(r.PtrS, r.PtrS->B) // expected-error {{pointer cannot be mapped along with a section derived from itself}} expected-note {{used here}}
+#pragma omp target map(r.PtrS, r.PtrS->B)    // expected-error {{pointer cannot be mapped along with a section derived from itself}} expected-note {{used here}}
   {}
-  #pragma omp target map(r.PtrS->A, r.PtrS->B)
+#pragma omp target map(r.PtrS->A, r.PtrS->B)
   {}
-  #pragma omp target map(r.RPtrS[0], r.RPtrS->B) // expected-error {{same pointer dereferenced in multiple different ways in map clause expressions}} expected-note {{used here}}
+#pragma omp target map(r.RPtrS[0], r.RPtrS->B) // expected-error {{same pointer dereferenced in multiple different ways in map clause expressions}} expected-note {{used here}}
   {}
-  #pragma omp target map(r.RPtrS, r.RPtrS->B) // expected-error {{pointer cannot be mapped along with a section derived from itself}} expected-note {{used here}}
+#pragma omp target map(r.RPtrS, r.RPtrS->B)    // expected-error {{pointer cannot be mapped along with a section derived from itself}} expected-note {{used here}}
   {}
-  #pragma omp target map(r.RPtrS->A, r.RPtrS->B)
+#pragma omp target map(r.RPtrS->A, r.RPtrS->B)
   {}
-  #pragma omp target map(r.S.Arr[:12])
+#pragma omp target map(r.S.Arr[:12])
   {}
-  #pragma omp target map(r.S.foo()[:12]) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}} le50-error {{expected addressable lvalue in 'map' clause}}
+#pragma omp target map(r.S.foo() [:12]) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}} le50-error {{expected addressable lvalue in 'map' clause}}
   {}
-  #pragma omp target map(r.C, r.D)
+#pragma omp target map(r.C, r.D)
   {}
-  #pragma omp target map(r.C, r.C) // expected-error {{variable already marked as mapped in current construct}} expected-note {{used here}}
+#pragma omp target map(r.C, r.C)     // expected-error {{variable already marked as mapped in current construct}} expected-note {{used here}}
   {}
-  #pragma omp target map(r.C) map(r.C) // expected-error {{variable already marked as mapped in current construct}} expected-note {{used here}}
+#pragma omp target map(r.C) map(r.C) // expected-error {{variable already marked as mapped in current construct}} expected-note {{used here}}
   {}
-  #pragma omp target map(r.C, r.S)  // this would be an error only caught at runtime - Sema would have to make sure there is not way for the missing data between fields to be mapped somewhere else.
+#pragma omp target map(r.C, r.S)     // this would be an error only caught at runtime - Sema would have to make sure there is not way for the missing data between fields to be mapped somewhere else.
   {}
-  #pragma omp target map(r, r.S)  // expected-error {{variable already marked as mapped in current construct}} expected-note {{used here}}
+#pragma omp target map(r, r.S)       // expected-error {{variable already marked as mapped in current construct}} expected-note {{used here}}
   {}
-  #pragma omp target map(r.C, t.C)
+#pragma omp target map(r.C, t.C)
   {}
-  #pragma omp target map(r.A)   // expected-error {{bit fields cannot be used to specify storage in a 'map' clause}}
+#pragma omp target map(r.A) // expected-error {{bit fields cannot be used to specify storage in a 'map' clause}}
   {}
-  #pragma omp target map(r.Arr)
+#pragma omp target map(r.Arr)
   {}
-  #pragma omp target map(r.Arr[3:5])
+#pragma omp target map(r.Arr [3:5])
   {}
-  #pragma omp target map(r.Ptr[3:5])
+#pragma omp target map(r.Ptr [3:5])
   {}
-  #pragma omp target map(r.ArrS[3:5].A)   // expected-error {{OpenMP array section is not allowed here}}
+#pragma omp target map(r.ArrS [3:5].A)         // expected-error {{OpenMP array section is not allowed here}}
   {}
-  #pragma omp target map(r.ArrS[3:5].Arr[6:7])   // expected-error {{OpenMP array section is not allowed here}}
+#pragma omp target map(r.ArrS [3:5].Arr [6:7]) // expected-error {{OpenMP array section is not allowed here}}
   {}
-  #pragma omp target map(r.ArrS[3].Arr[6:7])
+#pragma omp target map(r.ArrS[3].Arr [6:7])
   {}
-  #pragma omp target map(r.S.Arr[4:5])
+#pragma omp target map(r.S.Arr [4:5])
   {}
-  #pragma omp target map(r.S.Ptr[4:5])
+#pragma omp target map(r.S.Ptr [4:5])
   {}
-  #pragma omp target map(r.S.Ptr[:])  // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
+#pragma omp target map(r.S.Ptr[:]) // expected-error {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
   {}
-  #pragma omp target map((p+1)->A)  // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}}
+#pragma omp target map((p + 1)->A) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}}
   {}
-  #pragma omp target map(u.B)  // expected-error {{mapping of union members is not allowed}}
+#pragma omp target map(u.B)        // expected-error {{mapping of union members is not allowed}}
   {}
-  #pragma omp target
+#pragma omp target
   {
     u.B = 0;
     r.S.foo();
   }
 
-  #pragma omp target data map(to: r.C) //expected-note {{used here}}
+#pragma omp target data map(to \
+                            : r.C) //expected-note {{used here}}
   {
-    #pragma omp target map(r.D)  // expected-error {{original storage of expression in data environment is shared but data environment do not fully contain mapped expression storage}}
+#pragma omp target map(r.D)        // expected-error {{original storage of expression in data environment is shared but data environment do not fully contain mapped expression storage}}
     {}
   }
 
-  #pragma omp target data map(to: t.Ptr) //expected-note {{used here}}
+#pragma omp target data map(to \
+                            : t.Ptr) //expected-note {{used here}}
   {
-    #pragma omp target map(t.Ptr[:23])  // expected-error {{pointer cannot be mapped along with a section derived from itself}}
+#pragma omp target map(t.Ptr[:23])   // expected-error {{pointer cannot be mapped along with a section derived from itself}}
     {}
   }
 
-  #pragma omp target data map(to: t.C, t.D)
+#pragma omp target data map(to \
+                            : t.C, t.D)
   {
-  #pragma omp target data map(to: t.C)
+#pragma omp target data map(to \
+                            : t.C)
   {
-    #pragma omp target map(t.D)
+#pragma omp target map(t.D)
     {}
   }
   }
-  #pragma omp target data map(marr[:][:][:])
+#pragma omp target data map(marr[:][:][:])
   {
-    #pragma omp target data map(marr)
+#pragma omp target data map(marr)
     {}
   }
 
-  #pragma omp target data map(to: t)
+#pragma omp target data map(to \
+                            : t)
   {
-  #pragma omp target data map(to: t.C)
+#pragma omp target data map(to \
+                            : t.C)
   {
-    #pragma omp target map(t.D)
+#pragma omp target map(t.D)
     {}
   }
   }
diff --git a/clang/test/OpenMP/target_parallel_for_private_messages.cpp b/clang/test/OpenMP/target_parallel_for_private_messages.cpp
index b11981e066353..124eadb61de7d 100644
--- a/clang/test/OpenMP/target_parallel_for_private_messages.cpp
+++ b/clang/test/OpenMP/target_parallel_for_private_messages.cpp
@@ -237,7 +237,7 @@ int main(int argc, char **argv) {
     m = k + 2;
 
   s6 = s6_0; // expected-note {{in instantiation of member function 'S6<float>::operator=' requested here}}
-  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float> >::operator=' requested here}}
+  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float>>::operator=' requested here}}
   return foomain(argc, argv); // expected-note {{in instantiation of function template specialization 'foomain<int, char>' requested here}}
 }
 
diff --git a/clang/test/OpenMP/target_parallel_for_simd_private_messages.cpp b/clang/test/OpenMP/target_parallel_for_simd_private_messages.cpp
index 5f54dfa2f987f..6a0f40e285396 100644
--- a/clang/test/OpenMP/target_parallel_for_simd_private_messages.cpp
+++ b/clang/test/OpenMP/target_parallel_for_simd_private_messages.cpp
@@ -237,7 +237,7 @@ int main(int argc, char **argv) {
     m = k + 2;
 
   s6 = s6_0; // expected-note {{in instantiation of member function 'S6<float>::operator=' requested here}}
-  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float> >::operator=' requested here}}
+  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float>>::operator=' requested here}}
   return foomain(argc, argv); // expected-note {{in instantiation of function template specialization 'foomain<int, char>' requested here}}
 }
 
diff --git a/clang/test/OpenMP/target_private_messages.cpp b/clang/test/OpenMP/target_private_messages.cpp
index d8d96b5476cb0..2b9fd35c05c66 100644
--- a/clang/test/OpenMP/target_private_messages.cpp
+++ b/clang/test/OpenMP/target_private_messages.cpp
@@ -204,7 +204,7 @@ int main(int argc, char **argv) {
 #pragma omp target map(i) private(i) // expected-error {{private variable cannot be in a map clause in '#pragma omp target' directive}}
   {}
   s6 = s6_0; // expected-note {{in instantiation of member function 'S6<float>::operator=' requested here}}
-  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float> >::operator=' requested here}}
+  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float>>::operator=' requested here}}
   return foomain(argc, argv); // expected-note {{in instantiation of function template specialization 'foomain<int, char>' requested here}}
 }
 
diff --git a/clang/test/OpenMP/target_simd_private_messages.cpp b/clang/test/OpenMP/target_simd_private_messages.cpp
index 44732863b4e21..ee0122f2da50f 100644
--- a/clang/test/OpenMP/target_simd_private_messages.cpp
+++ b/clang/test/OpenMP/target_simd_private_messages.cpp
@@ -237,7 +237,7 @@ int main(int argc, char **argv) {
     m = k + 2;
 
   s6 = s6_0; // expected-note {{in instantiation of member function 'S6<float>::operator=' requested here}}
-  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float> >::operator=' requested here}}
+  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float>>::operator=' requested here}}
   return foomain(argc, argv); // expected-note {{in instantiation of function template specialization 'foomain<int, char>' requested here}}
 }
 
diff --git a/clang/test/OpenMP/target_update_ast_print.cpp b/clang/test/OpenMP/target_update_ast_print.cpp
index e60e081b32104..fb6440b87cea3 100644
--- a/clang/test/OpenMP/target_update_ast_print.cpp
+++ b/clang/test/OpenMP/target_update_ast_print.cpp
@@ -1,10 +1,10 @@
-// RUN: %clang_cc1 -verify -fopenmp -ast-print %s | FileCheck %s
-// RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=50 -ast-print %s | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -x c++ -std=c++11 -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -ast-print %s | FileCheck %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s
+// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=50 -ast-print %s | FileCheck %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s
 // expected-no-diagnostics
 
 #ifndef HEADER
@@ -14,29 +14,29 @@ void foo() {}
 
 template <class T, class U>
 T foo(T targ, U uarg) {
-  static T a;
+  static T a, *p;
   U b;
   int l;
-#pragma omp target update to(a) if(l>5) device(l) nowait depend(inout:l)
+#pragma omp target update to(([a][targ])p, a) if(l>5) device(l) nowait depend(inout:l)
 
-#pragma omp target update from(b) if(l<5) device(l-1) nowait depend(inout:l)
+#pragma omp target update from(b, ([a][targ])p) if(l<5) device(l-1) nowait depend(inout:l)
   return a + targ + (T)b;
 }
-// CHECK:      static T a;
+// CHECK:      static T a, *p;
 // CHECK-NEXT: U b;
 // CHECK-NEXT: int l;
-// CHECK-NEXT: #pragma omp target update to(a) if(l > 5) device(l) nowait depend(inout : l){{$}}
-// CHECK-NEXT: #pragma omp target update from(b) if(l < 5) device(l - 1) nowait depend(inout : l)
-// CHECK:      static int a;
+// CHECK-NEXT: #pragma omp target update to(([a][targ])p,a) if(l > 5) device(l) nowait depend(inout : l){{$}}
+// CHECK-NEXT: #pragma omp target update from(b,([a][targ])p) if(l < 5) device(l - 1) nowait depend(inout : l)
+// CHECK:      static int a, *p;
 // CHECK-NEXT: float b;
 // CHECK-NEXT: int l;
-// CHECK-NEXT: #pragma omp target update to(a) if(l > 5) device(l) nowait depend(inout : l)
-// CHECK-NEXT: #pragma omp target update from(b) if(l < 5) device(l - 1) nowait depend(inout : l)
-// CHECK:      static char a;
+// CHECK-NEXT: #pragma omp target update to(([a][targ])p,a) if(l > 5) device(l) nowait depend(inout : l)
+// CHECK-NEXT: #pragma omp target update from(b,([a][targ])p) if(l < 5) device(l - 1) nowait depend(inout : l)
+// CHECK:      static char a, *p;
 // CHECK-NEXT: float b;
 // CHECK-NEXT: int l;
-// CHECK-NEXT: #pragma omp target update to(a) if(l > 5) device(l) nowait depend(inout : l)
-// CHECK-NEXT: #pragma omp target update from(b) if(l < 5) device(l - 1) nowait depend(inout : l)
+// CHECK-NEXT: #pragma omp target update to(([a][targ])p,a) if(l > 5) device(l) nowait depend(inout : l)
+// CHECK-NEXT: #pragma omp target update from(b,([a][targ])p) if(l < 5) device(l - 1) nowait depend(inout : l)
 
 int main(int argc, char **argv) {
   static int a;
diff --git a/clang/test/OpenMP/target_update_codegen.cpp b/clang/test/OpenMP/target_update_codegen.cpp
index 479461e7ca80e..fd5a62a8067c7 100644
--- a/clang/test/OpenMP/target_update_codegen.cpp
+++ b/clang/test/OpenMP/target_update_codegen.cpp
@@ -984,5 +984,80 @@ void lvalue_find_base(float **f, SSA *sa) {
   #pragma omp target update from(*(sa->sa->i+*(1+sa->i+f)))
 }
 
+#endif
+///==========================================================================///
+// RUN: %clang_cc1 -DCK18 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK18 --check-prefix CK18-64
+// RUN: %clang_cc1 -DCK18 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK18 --check-prefix CK18-64
+// RUN: %clang_cc1 -DCK18 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK18 --check-prefix CK18-32
+// RUN: %clang_cc1 -DCK18 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK18 --check-prefix CK18-32
+
+// RUN: %clang_cc1 -DCK18 -verify -fopenmp-simd -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -DCK18 -fopenmp-simd -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -DCK18 -verify -fopenmp-simd -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -DCK18 -fopenmp-simd -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY18 %s
+// SIMD-ONLY18-NOT: {{__kmpc|__tgt}}
+#ifdef CK18
+
+// CK18-DAG: [[MTYPE_TO:@.+]] = {{.+}}constant [1 x i64] [i64 33]
+// CK18-DAG: [[MTYPE_FROM:@.+]] = {{.+}}constant [1 x i64] [i64 34]
+
+//CK18-LABEL: array_shaping
+void array_shaping(float *f, int sa) {
+
+  // CK18-DAG: call void @__tgt_target_data_update(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE_TO]]{{.+}})
+  // CK18-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
+  // CK18-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
+  // CK18-DAG: [[GEPS]] = getelementptr inbounds {{.+}}[[S:%[^,]+]]
+
+  // CK18-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
+  // CK18-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
+  // CK18-DAG: [[S0:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 0
+
+  // CK18-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to float**
+  // CK18-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to float**
+
+  // CK18-DAG: store float* [[F1:%.+]], float** [[BPC0]],
+  // CK18-DAG: store float* [[F2:%.+]], float** [[PC0]],
+  // CK18-DAG: store i64 [[SIZE:%.+]], i64* [[S0]],
+
+  // CK18-DAG: [[F1]] = load float*, float** [[F_ADDR:%.+]],
+  // CK18-DAG: [[F2]] = load float*, float** [[F_ADDR]],
+  // CK18-64-DAG: [[SIZE]] = mul nuw i64 [[SZ1:%.+]], 4
+  // CK18-64-DAG: [[SZ1]] = mul nuw i64 12, %{{.+}}
+  // CK18-32-DAG: [[SIZE]] = sext i32 [[SZ1:%.+]] to i64
+  // CK18-32-DAG: [[SZ1]] = mul nuw i32 [[SZ2:%.+]], 4
+  // CK18-32-DAG: [[SZ2]] = mul nuw i32 12, %{{.+}}
+  #pragma omp target update to(([3][sa][4])f)
+  sa = 1;
+  // CK18-DAG: call void @__tgt_target_data_update(i64 -1, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], i64* [[GEPS:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE_FROM]]{{.+}})
+  // CK18-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
+  // CK18-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
+  // CK18-DAG: [[GEPS]] = getelementptr inbounds {{.+}}[[S:%[^,]+]]
+
+  // CK18-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
+  // CK18-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
+  // CK18-DAG: [[S0:%.+]] = getelementptr inbounds {{.+}}[[S]], i{{.+}} 0, i{{.+}} 0
+
+  // CK18-DAG: [[BPC0:%.+]] = bitcast i8** [[BP0]] to float**
+  // CK18-DAG: [[PC0:%.+]] = bitcast i8** [[P0]] to float**
+
+  // CK18-DAG: store float* [[F1:%.+]], float** [[BPC0]],
+  // CK18-DAG: store float* [[F2:%.+]], float** [[PC0]],
+  // CK18-DAG: store i64 [[SIZE:%.+]], i64* [[S0]],
+
+  // CK18-DAG: [[F1]] = load float*, float** [[F_ADDR:%.+]],
+  // CK18-DAG: [[F2]] = load float*, float** [[F_ADDR]],
+  // CK18-64-DAG: [[SIZE]] = mul nuw i64 [[SZ1:%.+]], 5
+  // CK18-64-DAG: [[SZ1]] = mul nuw i64 4, %{{.+}}
+  // CK18-32-DAG: [[SIZE]] = sext i32 [[SZ1:%.+]] to i64
+  // CK18-32-DAG: [[SZ1]] = mul nuw i32 [[SZ2:%.+]], 5
+  // CK18-32-DAG: [[SZ2]] = mul nuw i32 4, %{{.+}}
+  #pragma omp target update from(([sa][5])f)
+}
+
 #endif
 #endif
diff --git a/clang/test/OpenMP/target_update_from_messages.cpp b/clang/test/OpenMP/target_update_from_messages.cpp
index 141cfc38ffeb1..eaf862a105827 100644
--- a/clang/test/OpenMP/target_update_from_messages.cpp
+++ b/clang/test/OpenMP/target_update_from_messages.cpp
@@ -74,7 +74,7 @@ struct S8 {
 #pragma omp target update from(*(this->S->i+this->S->s6[0].pp)) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}} le45-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
 #pragma omp target update from(*(a+this->ptr)) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}} le45-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
 #pragma omp target update from(*(*(this->ptr)+a+this->ptr)) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}} le45-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
-#pragma omp target update from(*(this+this)) // expected-error {{invalid operands to binary expression ('S8 *' and 'S8 *')}}
+#pragma omp target update from(*(this+this)) // expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}} expected-error {{invalid operands to binary expression ('S8 *' and 'S8 *')}}
   }
 };
 
@@ -198,8 +198,8 @@ int main(int argc, char **argv) {
 #pragma omp target update from(**(-(*offset)+BB+*m)) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}} le45-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
 #pragma omp target update from(**(*(*(&offset))+BB-*m)) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}} le45-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
 #pragma omp target update from(*(x+*(y+*(**BB+BBB)+s7.i))) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}} le45-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
-#pragma omp target update from(*(m+(m))) // expected-error {{invalid operands to binary expression ('int *' and 'int *')}}
-#pragma omp target update from(*(1+y+y)) // expected-error {{indirection requires pointer operand ('int' invalid)}}
+#pragma omp target update from(*(m+(m))) // expected-error {{invalid operands to binary expression ('int *' and 'int *')}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+#pragma omp target update from(*(1+y+y)) // expected-error {{indirection requires pointer operand ('int' invalid)}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
 #pragma omp target data map(to: s7.i)
   {
 #pragma omp target update from(s7.x)
diff --git a/clang/test/OpenMP/target_update_to_messages.cpp b/clang/test/OpenMP/target_update_to_messages.cpp
index 832adc0dd4d09..66ba9d3bb23f3 100644
--- a/clang/test/OpenMP/target_update_to_messages.cpp
+++ b/clang/test/OpenMP/target_update_to_messages.cpp
@@ -77,7 +77,7 @@ struct S8 {
 #pragma omp target update to(*(this->S->i+this->S->s6[0].pp)) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}} le45-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
 #pragma omp target update to(*(a+this->ptr)) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}} le45-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
 #pragma omp target update to(*(*(this->ptr)+a+this->ptr)) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}} le45-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
-#pragma omp target update to(*(this+this)) // expected-error {{invalid operands to binary expression ('S8 *' and 'S8 *')}}
+#pragma omp target update to(*(this+this)) // expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}} expected-error {{invalid operands to binary expression ('S8 *' and 'S8 *')}}
     {}
   }
 };
@@ -205,8 +205,8 @@ int main(int argc, char **argv) {
 #pragma omp target update to(**(*offset+BB+*m)) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}} le45-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
 #pragma omp target update to(**(*(*(&offset))+BB+*m)) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}} le45-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
 #pragma omp target update to(*(x+*(y+*(**BB+BBB)+s7.i))) // le45-error {{expected expression containing only member accesses and/or array sections based on named variables}} le45-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
-#pragma omp target update to(*(m+(m))) // expected-error {{invalid operands to binary expression ('int *' and 'int *')}}
-#pragma omp target update to(*(1+y+y)) // expected-error {{indirection requires pointer operand ('int' invalid)}}
+#pragma omp target update to(*(m+(m))) // expected-error {{invalid operands to binary expression ('int *' and 'int *')}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+#pragma omp target update to(*(1+y+y)) // expected-error {{indirection requires pointer operand ('int' invalid)}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
   {}
   return tmain<int, 3>(argc)+tmain<from, 4>(argc); // expected-note {{in instantiation of function template specialization 'tmain<int, 3>' requested here}} expected-note {{in instantiation of function template specialization 'tmain<int, 4>' requested here}}
 }
diff --git a/clang/test/OpenMP/task_ast_print.cpp b/clang/test/OpenMP/task_ast_print.cpp
index 73323c6c2ae57..cc569a6aab845 100644
--- a/clang/test/OpenMP/task_ast_print.cpp
+++ b/clang/test/OpenMP/task_ast_print.cpp
@@ -26,7 +26,7 @@ struct S1 {
 template <typename T>
 class S7 : public T {
 protected:
-  T a, b;
+  T a, b, c[10], d[10];
   S7() : a(0) {}
 
 public:
@@ -34,7 +34,7 @@ class S7 : public T {
     omp_depend_t x;
     omp_event_handle_t evt;
 #pragma omp taskgroup allocate(b) task_reduction(+:b)
-#pragma omp task private(a) private(this->a) private(T::a) in_reduction(+:this->b) allocate(b) depend(depobj:x) detach(evt)
+#pragma omp task private(a) private(this->a) private(T::a) in_reduction(+:this->b) allocate(b) depend(depobj:x) detach(evt) depend(iterator(i=0:10:1, T *k = &a:&b), in: c[i], d[(int)(k-&a)])
     for (int k = 0; k < a.a; ++k)
       ++this->a.a;
   }
@@ -47,9 +47,9 @@ class S7 : public T {
 };
 
 // CHECK: #pragma omp taskgroup allocate(this->b) task_reduction(+: this->b)
-// CHECK: #pragma omp task private(this->a) private(this->a) private(T::a) in_reduction(+: this->b) allocate(this->b) depend(depobj : x) detach(evt){{$}}
+// CHECK: #pragma omp task private(this->a) private(this->a) private(T::a) in_reduction(+: this->b) allocate(this->b) depend(depobj : x) detach(evt) depend(iterator(int i = 0:10:1, T * k = &this->a:&this->b), in : this->c[i],this->d[(int)(k - &this->a)]){{$}}
 // CHECK: #pragma omp task private(this->a) private(this->a)
-// CHECK: #pragma omp task private(this->a) private(this->a) private(this->S1::a)
+// CHECK: #pragma omp task private(this->a) private(this->a) private(this->S1::a) in_reduction(+: this->b) allocate(this->b) depend(depobj : x) detach(evt) depend(iterator(int i = 0:10:1, S1 * k = &this->a:&this->b), in : this->c[i],this->d[(int)(k - &this->a)])
 
 class S8 : public S7<S1> {
   S8() {}
@@ -99,7 +99,7 @@ T tmain(T argc, T *argv) {
   T arr[argc];
   omp_depend_t x;
   omp_event_handle_t evt;
-#pragma omp task untied depend(in : argc, argv[b:argc], arr[:]) if (task : argc > 0) depend(depobj: x) detach(evt)
+#pragma omp task untied depend(in : argc, argv[b:argc], arr[:], ([argc][sizeof(T)])argv) if (task : argc > 0) depend(depobj: x) detach(evt)
   a = 2;
 #pragma omp task default(none), private(argc, b) firstprivate(argv) shared(d) if (argc > 0) final(S<T>::TS > 0) priority(argc)
   foo();
@@ -116,7 +116,7 @@ T tmain(T argc, T *argv) {
 // CHECK-NEXT: T arr[argc];
 // CHECK-NEXT: omp_depend_t x;
 // CHECK-NEXT: omp_event_handle_t evt;
-// CHECK-NEXT: #pragma omp task untied depend(in : argc,argv[b:argc],arr[:]) if(task: argc > 0) depend(depobj : x) detach(evt)
+// CHECK-NEXT: #pragma omp task untied depend(in : argc,argv[b:argc],arr[:],([argc][sizeof(T)])argv) if(task: argc > 0) depend(depobj : x) detach(evt)
 // CHECK-NEXT: a = 2;
 // CHECK-NEXT: #pragma omp task default(none) private(argc,b) firstprivate(argv) shared(d) if(argc > 0) final(S<T>::TS > 0) priority(argc)
 // CHECK-NEXT: foo()
@@ -130,7 +130,7 @@ T tmain(T argc, T *argv) {
 // CHECK-NEXT: int arr[argc];
 // CHECK-NEXT: omp_depend_t x;
 // CHECK-NEXT: omp_event_handle_t evt;
-// CHECK-NEXT: #pragma omp task untied depend(in : argc,argv[b:argc],arr[:]) if(task: argc > 0) depend(depobj : x) detach(evt)
+// CHECK-NEXT: #pragma omp task untied depend(in : argc,argv[b:argc],arr[:],([argc][sizeof(int)])argv) if(task: argc > 0) depend(depobj : x) detach(evt)
 // CHECK-NEXT: a = 2;
 // CHECK-NEXT: #pragma omp task default(none) private(argc,b) firstprivate(argv) shared(d) if(argc > 0) final(S<int>::TS > 0) priority(argc)
 // CHECK-NEXT: foo()
@@ -144,7 +144,7 @@ T tmain(T argc, T *argv) {
 // CHECK-NEXT: long arr[argc];
 // CHECK-NEXT: omp_depend_t x;
 // CHECK-NEXT: omp_event_handle_t evt;
-// CHECK-NEXT: #pragma omp task untied depend(in : argc,argv[b:argc],arr[:]) if(task: argc > 0) depend(depobj : x) detach(evt)
+// CHECK-NEXT: #pragma omp task untied depend(in : argc,argv[b:argc],arr[:],([argc][sizeof(long)])argv) if(task: argc > 0) depend(depobj : x) detach(evt)
 // CHECK-NEXT: a = 2;
 // CHECK-NEXT: #pragma omp task default(none) private(argc,b) firstprivate(argv) shared(d) if(argc > 0) final(S<long>::TS > 0) priority(argc)
 // CHECK-NEXT: foo()
@@ -164,22 +164,26 @@ int main(int argc, char **argv) {
 #pragma omp threadprivate(a)
   Enum ee;
 // CHECK: Enum ee;
-#pragma omp task untied mergeable depend(out:argv[:a][1], (arr)[0:]) if(task: argc > 0) priority(f) depend(depobj:y)
-  // CHECK-NEXT: #pragma omp task untied mergeable depend(out : argv[:a][1],(arr)[0:]) if(task: argc > 0) priority(f) depend(depobj : y)
+#pragma omp task untied mergeable depend(out:argv[:a][1], (arr)[0:],([argc][10])argv,b) if(task: argc > 0) priority(f) depend(depobj:y)
+  // CHECK-NEXT: #pragma omp task untied mergeable depend(out : argv[:a][1],(arr)[0:],([argc][10])argv,b) if(task: argc > 0) priority(f) depend(depobj : y)
   a = 2;
 // CHECK-NEXT: a = 2;
 #pragma omp taskgroup task_reduction(min: arr1)
-#pragma omp task default(none), private(argc, b) firstprivate(argv, evt) if (argc > 0) final(a > 0) depend(inout : a, argv[:argc],arr[:a]) priority(23) in_reduction(min: arr1), detach(evt)
+#pragma omp task default(none), private(argc, b) firstprivate(argv, evt) if (argc > 0) final(a > 0) depend(inout : a, argv[:argc],arr[:a], ([10][argc])argv) priority(23) in_reduction(min: arr1), detach(evt)
   // CHECK-NEXT: #pragma omp taskgroup task_reduction(min: arr1)
-  // CHECK-NEXT: #pragma omp task default(none) private(argc,b) firstprivate(argv,evt) if(argc > 0) final(a > 0) depend(inout : a,argv[:argc],arr[:a]) priority(23) in_reduction(min: arr1) detach(evt)
+  // CHECK-NEXT: #pragma omp task default(none) private(argc,b) firstprivate(argv,evt) if(argc > 0) final(a > 0) depend(inout : a,argv[:argc],arr[:a],([10][argc])argv) priority(23) in_reduction(min: arr1) detach(evt)
   foo();
   // CHECK-NEXT: foo();
 #pragma omp taskgroup task_reduction(min: arr1)
 #pragma omp parallel reduction(+:arr1)
-#pragma omp task in_reduction(min: arr1)
+#pragma omp task in_reduction(min: arr1) depend(iterator(i=0:argc, unsigned j=argc:0:a), out: argv[i][j])
   // CHECK-NEXT: #pragma omp taskgroup task_reduction(min: arr1)
   // CHECK-NEXT: #pragma omp parallel reduction(+: arr1)
-  // CHECK-NEXT: #pragma omp task in_reduction(min: arr1)
+  // CHECK-NEXT: #pragma omp task in_reduction(min: arr1) depend(iterator(int i = 0:argc, unsigned int j = argc:0:a), out : argv[i][j])
+  foo();
+  // CHECK-NEXT: foo();
+  // CHECK-NEXT: #pragma omp task in_reduction(+: arr1)
+#pragma omp task in_reduction(+: arr1)
   foo();
   // CHECK-NEXT: foo();
   return tmain<int, 5>(b, &b) + tmain<long, 1>(x, &x);
diff --git a/clang/test/OpenMP/task_codegen.c b/clang/test/OpenMP/task_codegen.c
index d0bbe5670d691..0f01f11be8b31 100644
--- a/clang/test/OpenMP/task_codegen.c
+++ b/clang/test/OpenMP/task_codegen.c
@@ -19,7 +19,7 @@ void foo();
 int main() {
   omp_depend_t d, x;
   omp_event_handle_t evt;
-  int a;
+  int a, *b;
   // CHECK: [[D_ADDR:%.+]] = alloca i8*,
   // CHECK: [[X_ADDR:%.+]] = alloca i8*,
   // CHECK: [[EVT_ADDR:%.+]] = alloca i64,
@@ -42,7 +42,7 @@ int main() {
   // CHECK: [[X_DEP_BASE_SIZE:%.+]] = getelementptr inbounds %struct.kmp_depend_info, %struct.kmp_depend_info* [[X_DEP_BASE]], i{{.+}} 0, i{{.+}} 0
   // CHECK: [[SIZE2:%.+]] = load i64, i64* [[X_DEP_BASE_SIZE]],
   // CHECK: [[SIZE3:%.+]] = add nuw i64 [[SIZE]], [[SIZE2]]
-  // CHECK: [[SIZE:%.+]] = add nuw i64 [[SIZE3]], 1
+  // CHECK: [[SIZE:%.+]] = add nuw i64 [[SIZE3]], 2
   // CHECK: [[SIZE32:%.+]] = trunc i64 [[SIZE]] to i32
   // CHECK: [[SIZE64:%.+]] = zext i32 [[SIZE32]] to i64
   // CHECK: [[SV:%.+]] = call i8* @llvm.stacksave()
@@ -56,11 +56,25 @@ int main() {
   // CHECK: store i64 4, i64* [[SIZE_ADDR]],
   // CHECK: [[FLAGS_ADDR:%.+]] = getelementptr inbounds %struct.kmp_depend_info, %struct.kmp_depend_info* [[VLA0]], i{{.+}} 0, i{{.+}} 2
   // CHECK: store i8 1, i8* [[FLAGS_ADDR]],
-  // CHECK: [[VLA_D:%.+]] = getelementptr %struct.kmp_depend_info, %struct.kmp_depend_info* [[VLA]], i64 1
+  // CHECK: [[A:%.+]] = load i32, i32* [[A_ADDR]],
+  // CHECK: [[A_CAST:%.+]] = sext i32 [[A]] to i64
+  // CHECK: [[SZ1:%.+]] = mul nuw i64 24, [[A_CAST]]
+  // CHECK: [[A:%.+]] = load i32, i32* [[A_ADDR]],
+  // CHECK: [[A_CAST:%.+]] = sext i32 [[A]] to i64
+  // CHECK: [[SZ:%.+]] = mul nuw i64 [[SZ1]], [[A_CAST]]
+  // CHECK: [[VLA1:%.+]] = getelementptr %struct.kmp_depend_info, %struct.kmp_depend_info* [[VLA]], i64 1
+  // CHECK: [[BASE_ADDR:%.+]] = getelementptr inbounds %struct.kmp_depend_info, %struct.kmp_depend_info* [[VLA1]], i{{.+}} 0, i{{.+}} 0
+  // CHECK: [[B_ADDR_CAST:%.+]] = ptrtoint i32** %{{.+}} to i64
+  // CHECK: store i64 [[B_ADDR_CAST]], i64* [[BASE_ADDR]],
+  // CHECK: [[SIZE_ADDR:%.+]] = getelementptr inbounds %struct.kmp_depend_info, %struct.kmp_depend_info* [[VLA1]], i{{.+}} 0, i{{.+}} 1
+  // CHECK: store i64 [[SZ]], i64* [[SIZE_ADDR]],
+  // CHECK: [[FLAGS_ADDR:%.+]] = getelementptr inbounds %struct.kmp_depend_info, %struct.kmp_depend_info* [[VLA1]], i{{.+}} 0, i{{.+}} 2
+  // CHECK: store i8 1, i8* [[FLAGS_ADDR]],
+  // CHECK: [[VLA_D:%.+]] = getelementptr %struct.kmp_depend_info, %struct.kmp_depend_info* [[VLA]], i64 2
   // CHECK: [[D_SIZE:%.+]] = mul nuw i64 24, [[SIZE1]]
   // CHECK: [[DEST:%.+]] = bitcast %struct.kmp_depend_info* [[VLA_D]] to i8*
   // CHECK: [[SRC:%.+]] = bitcast %struct.kmp_depend_info* [[D_DEP]] to i8*
-  // CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 [[DEST]], i8* align 8 [[SRC]], i64 [[D_SIZE]], i1 false)
+  // CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align {{.+}} [[DEST]], i8* align {{.+}} [[SRC]], i64 [[D_SIZE]], i1 false)
   // CHECK: [[VLA_X:%.+]] = getelementptr %struct.kmp_depend_info, %struct.kmp_depend_info* [[VLA_D]], i64 [[SIZE1]]
   // CHECK: [[X_SIZE:%.+]] = mul nuw i64 24, [[SIZE2]]
   // CHECK: [[DEST:%.+]] = bitcast %struct.kmp_depend_info* [[VLA_X]] to i8*
@@ -70,7 +84,7 @@ int main() {
   // CHECK: call i32 @__kmpc_omp_task_with_deps(%struct.ident_t* @{{.+}}, i32 [[GTID]], i8* [[ALLOC]], i32 [[SIZE32]], i8* [[BC]], i32 0, i8* null)
   // CHECK: [[SV:%.+]] = load i8*, i8** [[SV_ADDR]],
   // CHECK: call void @llvm.stackrestore(i8* [[SV]])
-#pragma omp task depend(in: a) depend(depobj: d, x) detach(evt)
+#pragma omp task depend(in: a, ([3][a][a])&b) depend(depobj: d, x) detach(evt)
   {
 #pragma omp taskgroup
     {
diff --git a/clang/test/OpenMP/task_depend_messages.cpp b/clang/test/OpenMP/task_depend_messages.cpp
index ea55cdaa84af4..bfb0e771b2ca7 100644
--- a/clang/test/OpenMP/task_depend_messages.cpp
+++ b/clang/test/OpenMP/task_depend_messages.cpp
@@ -28,21 +28,21 @@ int main(int argc, char **argv, char *env[]) {
 
   #pragma omp task depend(in : arr[0])
   #pragma omp task depend // expected-error {{expected '(' after 'depend'}}
-  #pragma omp task depend ( // omp45-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}} omp50-error {{expected 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
-  #pragma omp task depend () // omp45-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} omp50-error {{expected 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
-  #pragma omp task depend (argc // omp45-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}} omp50-error {{expected 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
-  #pragma omp task depend (source : argc) // omp45-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp50-error {{expected 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
-  #pragma omp task depend (source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} omp45-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp50-error {{expected 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
+  #pragma omp task depend ( // omp45-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-error {{expected ')'}} expected-note {{to match this '('}} expected-warning {{missing ':' after dependency type - ignoring}} omp50-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
+  #pragma omp task depend () // omp45-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} omp50-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
+  #pragma omp task depend (argc // omp45-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected ')'}} expected-note {{to match this '('}} omp50-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
+  #pragma omp task depend (source : argc) // omp45-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp50-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
+  #pragma omp task depend (source) // expected-error {{expected expression}} expected-warning {{missing ':' after dependency type - ignoring}} omp45-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp50-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
   #pragma omp task depend (in : argc)) // expected-warning {{extra tokens at the end of '#pragma omp task' are ignored}}
   #pragma omp task depend (out: ) // expected-error {{expected expression}}
-  #pragma omp task depend (inout : foobool(argc)), depend (in, argc) // expected-error {{expected addressable lvalue expression, array element or array section}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}}
+  #pragma omp task depend (inout : foobool(argc)), depend (in, argc) // omp50-error {{expected addressable lvalue expression, array element, array section or array shaping expression}} omp45-error {{expected addressable lvalue expression, array element or array section}} expected-warning {{missing ':' after dependency type - ignoring}} expected-error {{expected expression}}
   #pragma omp task depend (out :S1) // expected-error {{'S1' does not refer to a value}}
   #pragma omp task depend(in : argv[1][1] = '2')
-  #pragma omp task depend (in : vec[1]) // expected-error {{expected addressable lvalue expression, array element or array section}}
+  #pragma omp task depend (in : vec[1]) // omp50-error {{expected addressable lvalue expression, array element, array section or array shaping expression}} omp45-error {{expected addressable lvalue expression, array element or array section}}
   #pragma omp task depend (in : argv[0])
   #pragma omp task depend (in : ) // expected-error {{expected expression}}
   #pragma omp task depend (in : main)
-  #pragma omp task depend(in : a[0]) // expected-error{{expected addressable lvalue expression, array element or array section}}
+  #pragma omp task depend(in : a[0]) // omp50-error {{expected addressable lvalue expression, array element, array section or array shaping expression}} omp45-error {{expected addressable lvalue expression, array element or array section}}
   #pragma omp task depend (in : vec[1:2]) // expected-error {{ value is not an array or pointer}}
   #pragma omp task depend (in : argv[ // expected-error {{expected expression}} expected-error {{expected ']'}} expected-error {{expected ')'}} expected-note {{to match this '['}} expected-note {{to match this '('}}
   #pragma omp task depend (in : argv[: // expected-error {{expected expression}} expected-error {{expected ']'}} expected-error {{expected ')'}} expected-note {{to match this '['}} expected-note {{to match this '('}}
@@ -62,7 +62,27 @@ int main(int argc, char **argv, char *env[]) {
   #pragma omp task depend(depobj:argc) // omp45-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp50-error {{expected lvalue expression of 'omp_depend_t' type, not 'int'}}
   #pragma omp task depend(depobj : argv[ : argc][1 : argc - 1]) // omp45-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp50-error {{expected lvalue expression of 'omp_depend_t' type, not '<OpenMP array section type>'}}
   #pragma omp task depend(depobj : arr[0]) // omp45-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
+  #pragma omp task depend(in : ([ // expected-error {{expected variable name or 'this' in lambda capture list}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+  #pragma omp task depend(in : ([] // expected-error {{expected body of lambda expression}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+  #pragma omp task depend(in : ([]) // omp45-error {{expected body of lambda expression}} expected-error {{expected ')'}} expected-note {{to match this '('}} omp50-error 2 {{expected expression}}
+  #pragma omp task depend(in : ([])a // omp45-error {{expected body of lambda expression}} expected-error {{expected ')'}} expected-note {{to match this '('}} omp50-error {{expected expression}}
+  #pragma omp task depend(in : ([])a) // omp45-error {{expected body of lambda expression}} omp50-error {{expected expression}}
+  #pragma omp task depend(in : ([a])a) // omp45-error {{expected body of lambda expression}} omp50-error {{expected expression with a pointer to a complete type as a base of an array shaping operation}}
+  #pragma omp task depend(in : ([a])argc) // omp45-error {{expected body of lambda expression}} omp50-error {{expected expression with a pointer to a complete type as a base of an array shaping operation}}
+  #pragma omp task depend(in : ([-1][0])argv) // omp45-error {{expected variable name or 'this' in lambda capture list}} omp45-error {{expected ')'}} omp45-note {{to match this '('}} omp50-error {{array shaping dimension is evaluated to a non-positive value -1}} omp50-error {{array shaping dimension is evaluated to a non-positive value 0}}
+  #pragma omp task depend(iterator // expected-error {{expected ')'}} omp45-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-note {{to match this '('}} omp50-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}} omp50-error {{expected '(' after 'iterator'}} omp50-error {{expected ','}}
+  #pragma omp task depend(iterator():argc) // omp45-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp50-error {{expected ','}} omp50-error {{expected 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}}
+  #pragma omp task depend(iterator(argc // expected-error {{expected ')'}} omp45-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-note {{to match this '('}} omp50-error {{unknown type name 'argc'}} omp50-error {{expected ')'}} omp50-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}} omp50-error {{expected ','}} omp50-note {{to match this '('}}
+  #pragma omp task depend(iterator(unsigned argc: // expected-error {{expected ')'}} omp45-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-note {{to match this '('}} omp50-error {{expected '=' in iterator specifier}} omp50-error 2 {{expected expression}} omp50-error {{expected ',' or ')' after iterator specifier}} omp50-error {{expected ')'}} omp50-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}} omp50-error {{expected ','}} omp50-note {{to match this '('}}
+  #pragma omp task depend(iterator(unsigned argc = // expected-error {{expected ')'}} omp45-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} expected-warning {{missing ':' after dependency type - ignoring}} expected-note {{to match this '('}} omp50-error 2 {{expected expression}} omp50-error {{expected ',' or ')' after iterator specifier}} omp50-error {{expected ')'}} omp50-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}} omp50-error {{expected ','}} omp50-note {{to match this '('}}
+  #pragma omp task depend(iterator(vector argc = 0:2):argc) // omp45-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp50-error {{expected integral or pointer type as the iterator-type, not 'vector'}} omp50-error {{expected depend modifier(iterator) or 'in', 'out', 'inout', 'mutexinoutset' or 'depobj' in OpenMP clause 'depend'}} omp50-error {{expected ','}}
+  #pragma omp task depend(iterator(vector *argc = nullptr:nullptr+2:0), in:argc) // omp45-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp50-error {{invalid operands to binary expression ('nullptr_t' and 'int')}} omp50-error {{iterator step expression 0 evaluates to 0}}
+  #pragma omp task depend(iterator(vector *argc = 0:vector():argc), in:argc) // omp45-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp50-error {{converting 'vector' to incompatible type 'vector *'}}
   foo();
+#pragma omp task depend(iterator(unsigned argc = 0:10), in : argc) // omp45-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}}
+  argc = 0;
+#pragma omp task depend(iterator(i = 0:10, i = 0:10), in : argv[i]) // omp45-error {{expected 'in', 'out', 'inout' or 'mutexinoutset' in OpenMP clause 'depend'}} omp45-error {{use of undeclared identifier 'i'}} omp50-error {{redefinition of 'i'}} omp50-note {{previous definition is here}}
+  i = 0; // expected-error {{use of undeclared identifier 'i'}}
 
   return 0;
 }
diff --git a/clang/test/OpenMP/task_in_reduction_codegen.cpp b/clang/test/OpenMP/task_in_reduction_codegen.cpp
index 8b16e85768723..07a701d84ad6a 100644
--- a/clang/test/OpenMP/task_in_reduction_codegen.cpp
+++ b/clang/test/OpenMP/task_in_reduction_codegen.cpp
@@ -44,6 +44,8 @@ int main(int argc, char **argv) {
 #pragma omp task in_reduction(+:a) in_reduction(-:d) allocate(omp_high_bw_mem_alloc: d)
     a += d[a];
   }
+#pragma omp task in_reduction(+:a)
+  ++a;
   return 0;
 }
 
@@ -91,4 +93,10 @@ int main(int argc, char **argv) {
 // CHECK:       add nsw i32
 // CHECK:       store i32 %
 // CHECK-NOT:   call i8* @__kmpc_threadprivate_cached(
+
+// CHECK: [[A_PTR:%.+]] = call i8* @__kmpc_task_reduction_get_th_data(i32 %{{.+}}, i8* null, i8* %{{.+}})
+// CHECK-NEXT: [[A_ADDR:%.+]] = bitcast i8* [[A_PTR]] to i32*
+// CHECK-NEXT: [[A:%.+]] = load i32, i32* [[A_ADDR]],
+// CHECK-NEXT: [[NEW:%.+]] = add nsw i32 [[A]], 1
+// CHECK-NEXT: store i32 [[NEW]], i32* [[A_ADDR]],
 #endif
diff --git a/clang/test/OpenMP/task_in_reduction_message.cpp b/clang/test/OpenMP/task_in_reduction_message.cpp
index d064a5f6252e5..6b4ad06551e6e 100644
--- a/clang/test/OpenMP/task_in_reduction_message.cpp
+++ b/clang/test/OpenMP/task_in_reduction_message.cpp
@@ -51,7 +51,7 @@ void foobar3(int &ref) {
 }
 
 void foobar4(int &ref) {
-#pragma omp task in_reduction(min:ref) // expected-error {{in_reduction variable must appear in a task_reduction clause}}
+#pragma omp task in_reduction(min:ref)
   foo();
 }
 
@@ -164,7 +164,7 @@ T tmain(T argc) {
 #pragma omp task in_reduction(^ : T) // expected-error {{'T' does not refer to a value}}
   foo();
 #pragma omp taskgroup task_reduction(+:c)
-#pragma omp task in_reduction(+ : a, b, c, d, f) // expected-error {{a reduction list item with incomplete type 'S1'}} expected-error 3 {{const-qualified variable cannot be in_reduction}} expected-error 2 {{'operator+' is a private member of 'S2'}} expected-error 2 {{in_reduction variable must appear in a task_reduction clause}}
+#pragma omp task in_reduction(+ : a, b, c, d, f) // expected-error {{a reduction list item with incomplete type 'S1'}} expected-error 3 {{const-qualified variable cannot be in_reduction}} expected-error 2 {{'operator+' is a private member of 'S2'}}
   foo();
 #pragma omp task in_reduction(min : a, b, c, d, f) // expected-error {{a reduction list item with incomplete type 'S1'}} expected-error 4 {{arguments of OpenMP clause 'in_reduction' for 'min' or 'max' must be of arithmetic type}} expected-error 3 {{const-qualified variable cannot be in_reduction}}
   foo();
@@ -176,7 +176,7 @@ T tmain(T argc) {
   foo();
 #pragma omp task in_reduction(- : da) // expected-error {{const-qualified variable cannot be in_reduction}} expected-error {{const-qualified variable cannot be in_reduction}}
   foo();
-#pragma omp task in_reduction(^ : fl) // expected-error {{invalid operands to binary expression ('float' and 'float')}} expected-error {{in_reduction variable must appear in a task_reduction clause}}
+#pragma omp task in_reduction(^ : fl) // expected-error {{invalid operands to binary expression ('float' and 'float')}}
   foo();
 #pragma omp task in_reduction(&& : S2::S2s) // expected-error {{shared variable cannot be reduction}}
   foo();
@@ -268,7 +268,7 @@ int main(int argc, char **argv) {
 #pragma omp task in_reduction(^ : S1) // expected-error {{'S1' does not refer to a value}}
   foo();
 #pragma omp taskgroup task_reduction(+:c)
-#pragma omp task in_reduction(+ : a, b, c, d, f) // expected-error {{a reduction list item with incomplete type 'S1'}} expected-error 2 {{const-qualified variable cannot be in_reduction}} expected-error {{'operator+' is a private member of 'S2'}} expected-error {{in_reduction variable must appear in a task_reduction clause}}
+#pragma omp task in_reduction(+ : a, b, c, d, f) // expected-error {{a reduction list item with incomplete type 'S1'}} expected-error 2 {{const-qualified variable cannot be in_reduction}} expected-error {{'operator+' is a private member of 'S2'}}
   foo();
 #pragma omp task in_reduction(min : a, b, c, d, f) // expected-error {{a reduction list item with incomplete type 'S1'}} expected-error 2 {{arguments of OpenMP clause 'in_reduction' for 'min' or 'max' must be of arithmetic type}} expected-error 2 {{const-qualified variable cannot be in_reduction}}
   foo();
diff --git a/clang/test/OpenMP/taskloop_in_reduction_messages.cpp b/clang/test/OpenMP/taskloop_in_reduction_messages.cpp
index 1441637ed280a..ad091267c33d2 100644
--- a/clang/test/OpenMP/taskloop_in_reduction_messages.cpp
+++ b/clang/test/OpenMP/taskloop_in_reduction_messages.cpp
@@ -54,7 +54,7 @@ void foobar3(int &ref) {
 }
 
 void foobar4(int &ref) {
-#pragma omp taskloop in_reduction(min:ref) // expected-error {{in_reduction variable must appear in a task_reduction clause}}
+#pragma omp taskloop in_reduction(min:ref)
   for (int i = 0; i < 10; ++i)
   foo();
 }
@@ -181,7 +181,7 @@ T tmain(T argc) {
   for (int i = 0; i < 10; ++i)
   foo();
 #pragma omp taskgroup task_reduction(+:c)
-#pragma omp taskloop in_reduction(+ : a, b, c, d, f) // expected-error {{a reduction list item with incomplete type 'S1'}} expected-error 3 {{const-qualified variable cannot be in_reduction}} expected-error 2 {{'operator+' is a private member of 'S2'}} expected-error 2 {{in_reduction variable must appear in a task_reduction clause}}
+#pragma omp taskloop in_reduction(+ : a, b, c, d, f) // expected-error {{a reduction list item with incomplete type 'S1'}} expected-error 3 {{const-qualified variable cannot be in_reduction}} expected-error 2 {{'operator+' is a private member of 'S2'}}
   for (int i = 0; i < 10; ++i)
   foo();
 #pragma omp taskloop in_reduction(min : a, b, c, d, f) // expected-error {{a reduction list item with incomplete type 'S1'}} expected-error 4 {{arguments of OpenMP clause 'in_reduction' for 'min' or 'max' must be of arithmetic type}} expected-error 3 {{const-qualified variable cannot be in_reduction}}
@@ -199,7 +199,7 @@ T tmain(T argc) {
 #pragma omp taskloop in_reduction(- : da) // expected-error {{const-qualified variable cannot be in_reduction}} expected-error {{const-qualified variable cannot be in_reduction}}
   for (int i = 0; i < 10; ++i)
   foo();
-#pragma omp taskloop in_reduction(^ : fl) // expected-error {{invalid operands to binary expression ('float' and 'float')}} expected-error {{in_reduction variable must appear in a task_reduction clause}}
+#pragma omp taskloop in_reduction(^ : fl) // expected-error {{invalid operands to binary expression ('float' and 'float')}}
   for (int i = 0; i < 10; ++i)
   foo();
 #pragma omp taskloop in_reduction(&& : S2::S2s) // expected-error {{shared variable cannot be reduction}}
@@ -316,7 +316,7 @@ int main(int argc, char **argv) {
   for (int i = 0; i < 10; ++i)
   foo();
 #pragma omp taskgroup task_reduction(+:c)
-#pragma omp taskloop in_reduction(+ : a, b, c, d, f) // expected-error {{a reduction list item with incomplete type 'S1'}} expected-error 2 {{const-qualified variable cannot be in_reduction}} expected-error {{'operator+' is a private member of 'S2'}} expected-error {{in_reduction variable must appear in a task_reduction clause}}
+#pragma omp taskloop in_reduction(+ : a, b, c, d, f) // expected-error {{a reduction list item with incomplete type 'S1'}} expected-error 2 {{const-qualified variable cannot be in_reduction}} expected-error {{'operator+' is a private member of 'S2'}}
   for (int i = 0; i < 10; ++i)
   foo();
 #pragma omp taskloop in_reduction(min : a, b, c, d, f) // expected-error {{a reduction list item with incomplete type 'S1'}} expected-error 2 {{arguments of OpenMP clause 'in_reduction' for 'min' or 'max' must be of arithmetic type}} expected-error 2 {{const-qualified variable cannot be in_reduction}}
diff --git a/clang/test/OpenMP/taskloop_private_messages.cpp b/clang/test/OpenMP/taskloop_private_messages.cpp
index bc563ca112cb6..d15d60c4b7518 100644
--- a/clang/test/OpenMP/taskloop_private_messages.cpp
+++ b/clang/test/OpenMP/taskloop_private_messages.cpp
@@ -253,7 +253,7 @@ int main(int argc, char **argv) {
     si = k + 1;
 
   s6 = s6_0; // expected-note {{in instantiation of member function 'S6<float>::operator=' requested here}}
-  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float> >::operator=' requested here}}
+  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float>>::operator=' requested here}}
   return foomain(argc, argv); // expected-note {{in instantiation of function template specialization 'foomain<int, char>' requested here}}
 }
 
diff --git a/clang/test/OpenMP/taskloop_simd_in_reduction_messages.cpp b/clang/test/OpenMP/taskloop_simd_in_reduction_messages.cpp
index 77243696911bf..c097d28aaa0a0 100644
--- a/clang/test/OpenMP/taskloop_simd_in_reduction_messages.cpp
+++ b/clang/test/OpenMP/taskloop_simd_in_reduction_messages.cpp
@@ -54,7 +54,7 @@ void foobar3(int &ref) {
 }
 
 void foobar4(int &ref) {
-#pragma omp taskloop simd in_reduction(min:ref) // expected-error {{in_reduction variable must appear in a task_reduction clause}}
+#pragma omp taskloop simd in_reduction(min:ref)
   for (int i = 0; i < 10; ++i)
   foo();
 }
@@ -181,7 +181,7 @@ T tmain(T argc) {
   for (int i = 0; i < 10; ++i)
   foo();
 #pragma omp taskgroup task_reduction(+:c)
-#pragma omp taskloop simd in_reduction(+ : a, b, c, d, f) // expected-error {{a reduction list item with incomplete type 'S1'}} expected-error 3 {{const-qualified variable cannot be in_reduction}} expected-error 2 {{'operator+' is a private member of 'S2'}} expected-error 2 {{in_reduction variable must appear in a task_reduction clause}}
+#pragma omp taskloop simd in_reduction(+ : a, b, c, d, f) // expected-error {{a reduction list item with incomplete type 'S1'}} expected-error 3 {{const-qualified variable cannot be in_reduction}} expected-error 2 {{'operator+' is a private member of 'S2'}}
   for (int i = 0; i < 10; ++i)
   foo();
 #pragma omp taskloop simd in_reduction(min : a, b, c, d, f) // expected-error {{a reduction list item with incomplete type 'S1'}} expected-error 4 {{arguments of OpenMP clause 'in_reduction' for 'min' or 'max' must be of arithmetic type}} expected-error 3 {{const-qualified variable cannot be in_reduction}}
@@ -199,7 +199,7 @@ T tmain(T argc) {
 #pragma omp taskloop simd in_reduction(- : da) // expected-error {{const-qualified variable cannot be in_reduction}} expected-error {{const-qualified variable cannot be in_reduction}}
   for (int i = 0; i < 10; ++i)
   foo();
-#pragma omp taskloop simd in_reduction(^ : fl) // expected-error {{invalid operands to binary expression ('float' and 'float')}} expected-error {{in_reduction variable must appear in a task_reduction clause}}
+#pragma omp taskloop simd in_reduction(^ : fl) // expected-error {{invalid operands to binary expression ('float' and 'float')}}
   for (int i = 0; i < 10; ++i)
   foo();
 #pragma omp taskloop simd in_reduction(&& : S2::S2s) // expected-error {{shared variable cannot be reduction}}
@@ -316,7 +316,7 @@ int main(int argc, char **argv) {
   for (int i = 0; i < 10; ++i)
   foo();
 #pragma omp taskgroup task_reduction(+:c)
-#pragma omp taskloop simd in_reduction(+ : a, b, c, d, f) // expected-error {{a reduction list item with incomplete type 'S1'}} expected-error 2 {{const-qualified variable cannot be in_reduction}} expected-error {{'operator+' is a private member of 'S2'}} expected-error {{in_reduction variable must appear in a task_reduction clause}}
+#pragma omp taskloop simd in_reduction(+ : a, b, c, d, f) // expected-error {{a reduction list item with incomplete type 'S1'}} expected-error 2 {{const-qualified variable cannot be in_reduction}} expected-error {{'operator+' is a private member of 'S2'}}
   for (int i = 0; i < 10; ++i)
   foo();
 #pragma omp taskloop simd in_reduction(min : a, b, c, d, f) // expected-error {{a reduction list item with incomplete type 'S1'}} expected-error 2 {{arguments of OpenMP clause 'in_reduction' for 'min' or 'max' must be of arithmetic type}} expected-error 2 {{const-qualified variable cannot be in_reduction}}
diff --git a/clang/test/OpenMP/taskloop_simd_private_messages.cpp b/clang/test/OpenMP/taskloop_simd_private_messages.cpp
index c7b7ba05426bf..be697b9ed8a3e 100644
--- a/clang/test/OpenMP/taskloop_simd_private_messages.cpp
+++ b/clang/test/OpenMP/taskloop_simd_private_messages.cpp
@@ -253,7 +253,7 @@ int main(int argc, char **argv) {
     si = k + 1;
 
   s6 = s6_0; // expected-note {{in instantiation of member function 'S6<float>::operator=' requested here}}
-  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float> >::operator=' requested here}}
+  s7 = s7_0; // expected-note {{in instantiation of member function 'S7<S6<float>>::operator=' requested here}}
   return foomain(argc, argv); // expected-note {{in instantiation of function template specialization 'foomain<int, char>' requested here}}
 }
 
diff --git a/clang/test/Parser/cxx-ambig-init-templ.cpp b/clang/test/Parser/cxx-ambig-init-templ.cpp
index a64760f3f447e..eb90c0e02b8ff 100644
--- a/clang/test/Parser/cxx-ambig-init-templ.cpp
+++ b/clang/test/Parser/cxx-ambig-init-templ.cpp
@@ -2,6 +2,7 @@
 
 template<int> struct c { c(int) = delete; typedef void val; operator int() const; };
 
+int f;
 int val;
 int foobar;
 struct S {
@@ -29,6 +30,12 @@ struct S {
     int k6 = a < b, c < d > (n) // expected-error {{undeclared identifier 'n'}}
   );
 
+  static void f1b(
+    int k6 = a < b, c < d > (f)
+  );
+  using f1b_T = decltype(f1b(0)); // only one parameter, because second param
+                                  // would be missing its default argument
+
   void f2a(
     // T3<int> here is a parameter type, so must be declared before it is used.
     int k1 = c < b, T3 < int > x = 0 // expected-error {{no template named 'T3'}}
@@ -56,14 +63,21 @@ struct S {
     int missing_default // expected-error {{missing default argument on parameter}}
   );
 
+  // FIXME: We should ideally disambiguate this as two parameters.
   void f6(
-    int k = b < c,
-    unsigned int (missing_default) // expected-error {{missing default argument on parameter}}
+    int k = b < c, // expected-error {{unexpected end of default argument}}
+    unsigned int (missing_default)
   );
 
-  template<int, int=0> struct a { static const int val = 0; operator int(); }; // expected-note {{here}}
+  template<int, int = 0> struct a { // expected-note {{here}}
+    a();
+    a(int);
+    static const int val = 0;
+    operator int();
+  };
   static const int b = 0, c = 1, d = 2, goobar = 3;
   template<int, typename> struct e { operator int(); };
+  static const int f = 0;
 
   int mp1 = 0 < 1,
       a<b<c,b<c>::*mp2,
diff --git a/clang/test/Parser/cxx-class.cpp b/clang/test/Parser/cxx-class.cpp
index e73c8740a3efa..576c6d7e8b976 100644
--- a/clang/test/Parser/cxx-class.cpp
+++ b/clang/test/Parser/cxx-class.cpp
@@ -211,9 +211,9 @@ namespace DtorErrors {
   };
 
   struct T {};
-  T t1 = t1.T::~T<int>; // expected-error {{destructor name 'T' does not refer to a template}} expected-error {{expected '(' for function-style cast or type construction}} expected-error {{expected expression}}
+  T t1 = t1.T::~T<int>; // expected-error {{destructor name 'T' does not refer to a template}}
   // Emit the same diagnostic as for the previous case, plus something about ~.
-  T t2 = t2.~T::T<int>; // expected-error {{'~' in destructor name should be after nested name specifier}} expected-error {{destructor name 'T' does not refer to a template}} expected-error {{expected '(' for function-style cast or type construction}} expected-error {{expected expression}}
+  T t2 = t2.~T::T<int>; // expected-error {{'~' in destructor name should be after nested name specifier}} expected-error {{destructor name 'T' does not refer to a template}}
 }
 
 namespace BadFriend {
@@ -298,6 +298,11 @@ namespace ArrayMemberAccess {
   }
 }
 
+namespace UndeclaredBaseTemplate {
+  template<class> struct imp;
+  template<class T> struct is_member_function_pointer : undeclared<imp<T>::member> {}; // expected-error {{unknown template name 'undeclared'}}
+}
+
 // PR11109 must appear at the end of the source file
 class pr11109r3 { // expected-note{{to match this '{'}}
   public // expected-error{{expected ':'}} expected-error{{expected '}'}} expected-error{{expected ';' after class}}
diff --git a/clang/test/Parser/cxx-decl.cpp b/clang/test/Parser/cxx-decl.cpp
index ba1cce419a463..dbf3a3e70bb03 100644
--- a/clang/test/Parser/cxx-decl.cpp
+++ b/clang/test/Parser/cxx-decl.cpp
@@ -240,8 +240,7 @@ namespace PR17255 {
 void foo() {
   typename A::template B<> c; // expected-error {{use of undeclared identifier 'A'}}
 #if __cplusplus <= 199711L
-  // expected-error@-2 {{'typename' occurs outside of a template}}
-  // expected-error@-3 {{'template' keyword outside of a template}}
+  // expected-error@-2 {{'template' keyword outside of a template}}
 #endif
 }
 }
diff --git a/clang/test/Parser/cxx-member-initializers.cpp b/clang/test/Parser/cxx-member-initializers.cpp
index 8e52adbbfd335..f5b09752d72b1 100644
--- a/clang/test/Parser/cxx-member-initializers.cpp
+++ b/clang/test/Parser/cxx-member-initializers.cpp
@@ -62,7 +62,7 @@ namespace PR16480 {
 
   // FIXME: This should be valid in the union of C99 and C++11.
   struct F : X<0> {
-    F() : X<A<T>().n + (T){}.n>{} {} // expected-error +{{}}
+    F() : X<A<T>().n + (T){}.n>{} {} // expected-error +{{}} expected-note {{to match}}
 
     struct T { int n; };
     template<typename> struct A { int n; };
@@ -70,7 +70,7 @@ namespace PR16480 {
 
   // FIXME: This is valid now, but may be made ill-formed by DR1607.
   struct G : X<0> {
-    G() : X<0 && [](){return 0;}()>{} // expected-error +{{}}
+    G() : X<0 && [](){return 0;}()>{} // expected-error +{{}} expected-note {{to match}}
   };
 
   struct Errs : X<0> {
@@ -103,5 +103,5 @@ class G {
   void l(int x = C<int, C<int, int>::C1>().f()) {}
 
   // This isn't, but it shouldn't crash. The diagnostics don't matter much.
-  void m(int x = C<int, union int>().f()) {} // expected-error {{declaration of anonymous union must be a definition}} expected-error {{expected a type}}
+  void m(int x = C<int, union int>().f()) {} // expected-error {{declaration of anonymous union must be a definition}} expected-error {{expected a type}} expected-error {{expected '>'}} expected-note {{to match}}
 };
diff --git a/clang/test/Parser/cxx-template-argument.cpp b/clang/test/Parser/cxx-template-argument.cpp
index 70945e3aa746d..0e72cbfbfb175 100644
--- a/clang/test/Parser/cxx-template-argument.cpp
+++ b/clang/test/Parser/cxx-template-argument.cpp
@@ -9,8 +9,8 @@ template<typename T> struct A {};
 
 // Check for template argument lists followed by junk
 // FIXME: The diagnostics here aren't great...
-A<int+> int x; // expected-error {{expected '>'}} expected-error {{expected unqualified-id}}
-A<int x; // expected-error {{expected '>'}}
+A<int+> int x; // expected-error {{expected '>'}} expected-note {{to match this '<'}} expected-error {{expected unqualified-id}}
+A<int x; // expected-error {{expected '>'}} expected-note {{to match this '<'}}
 
 // PR8912
 template <bool> struct S {};
diff --git a/clang/test/Parser/cxx-template-decl.cpp b/clang/test/Parser/cxx-template-decl.cpp
index 0d52ad8fb50f0..14b97f12e3572 100644
--- a/clang/test/Parser/cxx-template-decl.cpp
+++ b/clang/test/Parser/cxx-template-decl.cpp
@@ -233,12 +233,12 @@ namespace broken_baseclause {
 template<typename T>
 struct base { };
 
-struct t1 : base<int,
-  public:  // expected-error {{expected expression}}
+struct t1 : base<int, // expected-note {{to match this '<'}}
+  public:  // expected-error {{expected expression}} expected-error {{expected '>'}}
 };
 // expected-error@-1 {{expected '{' after base class list}}
-struct t2 : base<int,
-  public  // expected-error {{expected expression}}
+struct t2 : base<int, // expected-note {{to match this '<'}}
+  public  // expected-error {{expected expression}} expected-error {{expected '>'}}
 };
 // expected-error@-1 {{expected '{' after base class list}}
 
diff --git a/clang/test/Parser/eof2.cpp b/clang/test/Parser/eof2.cpp
index 6a8b862e60999..44e2b58bb6d1c 100644
--- a/clang/test/Parser/eof2.cpp
+++ b/clang/test/Parser/eof2.cpp
@@ -1,11 +1,12 @@
 // RUN: not %clang_cc1 %s -fsyntax-only 2>&1 | FileCheck %s
 
 // CHECK: error: expected expression
+// CHECK: error: expected '>'
 // CHECK: error: expected member name or ';' after declaration specifiers
 // CHECK: error: expected '}'
 // CHECK: note: to match this '{'
 // CHECK: error: expected ';' after class
-// CHECK: 4 errors generated.
+// CHECK: 5 errors generated.
 
 // Do not add anything to the end of this file.  This requires the whitespace
 // plus EOF after the '<' token.
diff --git a/clang/test/Parser/objc-error-qualified-implementation.m b/clang/test/Parser/objc-error-qualified-implementation.m
index 179e2d27479bd..684c388a584ea 100644
--- a/clang/test/Parser/objc-error-qualified-implementation.m
+++ b/clang/test/Parser/objc-error-qualified-implementation.m
@@ -17,13 +17,13 @@ @implementation J < P,P > // expected-error {{@implementation declaration cannot
 
 @interface K @end
 
-@implementation K <P // expected-error {{@implementation declaration cannot be protocol qualified}} expected-note{{to match this '<'}}
-@end // expected-error {{expected '>'}}
+@implementation K <P // expected-error {{@implementation declaration cannot be protocol qualified}} expected-error {{expected '>'}} expected-note {{to match this '<'}}
+@end
 
 // rdar://13920026
 @implementation I (Cat) <P>  // expected-error {{@implementation declaration cannot be protocol qualified}}
 - (void) Meth {}
 @end
 
-@implementation I (Cat1) <P // expected-error {{@implementation declaration cannot be protocol qualified}}
-@end // expected-error {{expected '>'}}
+@implementation I (Cat1) <P // expected-error {{@implementation declaration cannot be protocol qualified}} expected-error {{expected '>'}} expected-note {{to match this '<'}}
+@end
diff --git a/clang/test/Parser/objcxx0x-lambda-expressions.mm b/clang/test/Parser/objcxx0x-lambda-expressions.mm
index 396313816fe5d..0f3e9481a7220 100644
--- a/clang/test/Parser/objcxx0x-lambda-expressions.mm
+++ b/clang/test/Parser/objcxx0x-lambda-expressions.mm
@@ -11,8 +11,7 @@ void f() {
 
     []; // expected-error {{expected body of lambda expression}}
     [=,foo+] {}; // expected-error {{expected ',' or ']' in lambda capture list}}
-    [&this] {}; // expected-error {{cannot take the address of an rvalue of type 'C *'}} \
-                // expected-error {{expected identifier}}
+    [&this] {}; // expected-error {{cannot take the address of an rvalue of type 'C *'}}
     [] {}; 
     [=] (int i) {}; 
     [&] (int) mutable -> void {}; 
@@ -25,8 +24,7 @@ void f() {
     [foo{bar}] () {};
     [foo = {bar}] () {}; // expected-error {{<initializer_list>}}
 
-    [foo(bar) baz] () {}; // expected-error {{called object type 'int' is not a function}} \
-                          // expected-error {{expected ';'}}
+    [foo(bar) baz] () {}; // expected-error {{called object type 'int' is not a function}}
     [foo(bar), baz] () {}; // ok
 
     [foo = bar baz]; // expected-warning {{receiver type 'int'}} expected-warning {{instance method '-baz'}}
diff --git a/clang/test/Parser/objcxx11-invalid-lambda.cpp b/clang/test/Parser/objcxx11-invalid-lambda.cpp
index 221cae9b90709..bdb4e880fd0b7 100644
--- a/clang/test/Parser/objcxx11-invalid-lambda.cpp
+++ b/clang/test/Parser/objcxx11-invalid-lambda.cpp
@@ -1,11 +1,10 @@
 // RUN: %clang_cc1 -fsyntax-only -verify -x objective-c++ -std=c++11 %s
 
-void foo() {
+void foo() {  // expected-note {{to match this '{'}}
   int bar;
   auto baz = [
-      bar(  // expected-note 2{{to match this '('}}\
-            // expected-warning {{captures are a C++14 extension}}
+      bar(  // expected-note {{to match this '('}} expected-note {{to match this '('}}
         foo_undeclared() // expected-error{{use of undeclared identifier 'foo_undeclared'}}
       /* ) */
-    ] () { };   // expected-error 2{{expected ')'}}
-}
\ No newline at end of file
+    ] () { };   // expected-error{{expected ')'}}
+}               // expected-error{{expected ')'}} expected-error {{expected ',' or ']'}} expected-error{{expected ';' at end of declaration}} expected-error{{expected '}'}}
diff --git a/clang/test/Parser/recovery.cpp b/clang/test/Parser/recovery.cpp
index 2d5b518c691fe..28c8817ea0276 100644
--- a/clang/test/Parser/recovery.cpp
+++ b/clang/test/Parser/recovery.cpp
@@ -133,7 +133,10 @@ Foo<int> missingSemiBeforeFunctionReturningTemplateId2();
 namespace PR17084 {
 enum class EnumID {};
 template <typename> struct TempID;
-template <> struct TempID<BadType> : BadType, EnumID::Garbage; // expected-error{{use of undeclared identifier 'BadType'}}
+template <> struct TempID<BadType> // expected-error{{use of undeclared identifier 'BadType'}}
+  : BadType, // expected-error {{expected class name}}
+    EnumID::Garbage // expected-error {{expected class name}}
+  ; // expected-error@-1 {{expected '{' after base class list}}
 }
 
 namespace pr15133 {
diff --git a/clang/test/Preprocessor/aarch64-target-features.c b/clang/test/Preprocessor/aarch64-target-features.c
index c62c82a0c96a7..9cb12f8afb329 100644
--- a/clang/test/Preprocessor/aarch64-target-features.c
+++ b/clang/test/Preprocessor/aarch64-target-features.c
@@ -39,6 +39,8 @@
 // CHECK-NOT: __ARM_SIZEOF_WCHAR_T 2
 // CHECK-NOT: __ARM_FEATURE_SVE
 // CHECK-NOT: __ARM_FEATURE_DOTPROD
+// CHECK-NOT: __ARM_FEATURE_PAC_DEFAULT
+// CHECK-NOT: __ARM_FEATURE_BTI_DEFAULT
 
 // RUN: %clang -target aarch64-none-linux-gnu -march=armv8-a+crypto -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-CRYPTO %s
 // RUN: %clang -target arm64-none-linux-gnu -march=armv8-a+crypto -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-CRYPTO %s
@@ -334,3 +336,33 @@
 // ================== Check Memory Tagging Extensions (MTE).
 // RUN: %clang -target arm64-none-linux-gnu -march=armv8.5-a+memtag -x c -E -dM %s -o - 2>&1 | FileCheck -check-prefix=CHECK-MEMTAG %s
 // CHECK-MEMTAG: __ARM_FEATURE_MEMORY_TAGGING 1
+
+// ================== Check Pointer Authentication Extension (PAuth).
+// RUN: %clang -target arm64-none-linux-gnu -march=armv8-a -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-PAUTH-OFF %s
+// RUN: %clang -target arm64-none-linux-gnu -march=armv8.5-a -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-PAUTH-OFF %s
+// RUN: %clang -target arm64-none-linux-gnu -march=armv8-a -mbranch-protection=none -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-PAUTH-OFF %s
+// RUN: %clang -target arm64-none-linux-gnu -march=armv8-a -mbranch-protection=bti -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-PAUTH-OFF %s
+// RUN: %clang -target arm64-none-linux-gnu -march=armv8-a -mbranch-protection=standard -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-PAUTH %s
+// RUN: %clang -target arm64-none-linux-gnu -march=armv8-a -mbranch-protection=pac-ret -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-PAUTH %s
+// RUN: %clang -target arm64-none-linux-gnu -march=armv8-a -mbranch-protection=pac-ret+b-key -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-PAUTH-BKEY %s
+// RUN: %clang -target arm64-none-linux-gnu -march=armv8-a -mbranch-protection=pac-ret+leaf -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-PAUTH-ALL %s
+// RUN: %clang -target arm64-none-linux-gnu -march=armv8-a -mbranch-protection=pac-ret+leaf+b-key -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-PAUTH-BKEY-ALL %s
+// CHECK-PAUTH-OFF-NOT:  __ARM_FEATURE_PAC_DEFAULT
+// CHECK-PAUTH:          #define __ARM_FEATURE_PAC_DEFAULT 1
+// CHECK-PAUTH-BKEY:     #define __ARM_FEATURE_PAC_DEFAULT 2
+// CHECK-PAUTH-ALL:      #define __ARM_FEATURE_PAC_DEFAULT 5
+// CHECK-PAUTH-BKEY-ALL: #define __ARM_FEATURE_PAC_DEFAULT 6
+
+// ================== Check Branch Target Identification (BTI).
+// RUN: %clang -target arm64-none-linux-gnu -march=armv8-a -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-BTI-OFF %s
+// RUN: %clang -target arm64-none-linux-gnu -march=armv8.5-a -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-BTI-OFF %s
+// RUN: %clang -target arm64-none-linux-gnu -march=armv8-a -mbranch-protection=none -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-BTI-OFF %s
+// RUN: %clang -target arm64-none-linux-gnu -march=armv8-a -mbranch-protection=pac-ret -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-BTI-OFF %s
+// RUN: %clang -target arm64-none-linux-gnu -march=armv8-a -mbranch-protection=pac-ret+leaf -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-BTI-OFF %s
+// RUN: %clang -target arm64-none-linux-gnu -march=armv8-a -mbranch-protection=pac-ret+b-key -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-BTI-OFF %s
+// RUN: %clang -target arm64-none-linux-gnu -march=armv8-a -mbranch-protection=pac-ret+leaf+b-key -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-BTI-OFF %s
+// RUN: %clang -target arm64-none-linux-gnu -march=armv8-a -mbranch-protection=standard -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-BTI %s
+// RUN: %clang -target arm64-none-linux-gnu -march=armv8-a -mbranch-protection=bti -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-BTI %s
+// RUN: %clang -target arm64-none-linux-gnu -march=armv8-a -mbranch-protection=pac-ret+bti -x c -E -dM %s -o - | FileCheck -check-prefix=CHECK-BTI %s
+// CHECK-BTI-OFF-NOT: __ARM_FEATURE_BTI_DEFAULT
+// CHECK-BTI:         #define __ARM_FEATURE_BTI_DEFAULT 1
diff --git a/clang/test/Preprocessor/arm-target-features.c b/clang/test/Preprocessor/arm-target-features.c
index ab18036c2cf9b..363dfdd68c400 100644
--- a/clang/test/Preprocessor/arm-target-features.c
+++ b/clang/test/Preprocessor/arm-target-features.c
@@ -841,5 +841,10 @@
 // CHECK-V85A: #define __ARM_ARCH_8_5A__ 1
 // CHECK-V85A: #define __ARM_ARCH_PROFILE 'A'
 
+// RUN: %clang -target armv8.6a-none-none-eabi -x c -E -dM %s -o - | FileCheck -match-full-lines --check-prefix=CHECK-V86A %s
+// CHECK-V86A: #define __ARM_ARCH 8
+// CHECK-V86A: #define __ARM_ARCH_8_6A__ 1
+// CHECK-V86A: #define __ARM_ARCH_PROFILE 'A'
+
 // RUN: %clang -target arm-none-none-eabi -march=armv7-m -mfpu=softvfp -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-SOFTVFP %s
 // CHECK-SOFTVFP-NOT: #define __ARM_FP 0x
diff --git a/clang/test/Preprocessor/hexagon-predefines.c b/clang/test/Preprocessor/hexagon-predefines.c
index 54013ceffa645..7979d567134bb 100644
--- a/clang/test/Preprocessor/hexagon-predefines.c
+++ b/clang/test/Preprocessor/hexagon-predefines.c
@@ -113,3 +113,18 @@
 // CHECK-LINUX: #define __unix__ 1
 // CHECK-LINUX: #define linux 1
 // CHECK-LINUX: #define unix 1
+
+// RUN: %clang_cc1 -E -dM -triple hexagon-unknown-linux-musl \
+// RUN: -target-cpu hexagonv67 -target-feature +hvxv67 \
+// RUN: -target-feature +hvx-length128b %s | FileCheck \
+// RUN: %s -check-prefix CHECK-ATOMIC
+// CHECK-ATOMIC: #define __CLANG_ATOMIC_BOOL_LOCK_FREE 2
+// CHECK-ATOMIC: #define __CLANG_ATOMIC_CHAR16_T_LOCK_FREE 2
+// CHECK-ATOMIC: #define __CLANG_ATOMIC_CHAR32_T_LOCK_FREE 2
+// CHECK-ATOMIC: #define __CLANG_ATOMIC_CHAR_LOCK_FREE 2
+// CHECK-ATOMIC: #define __CLANG_ATOMIC_INT_LOCK_FREE 2
+// CHECK-ATOMIC: #define __CLANG_ATOMIC_LLONG_LOCK_FREE 2
+// CHECK-ATOMIC: #define __CLANG_ATOMIC_LONG_LOCK_FREE 2
+// CHECK-ATOMIC: #define __CLANG_ATOMIC_POINTER_LOCK_FREE 2
+// CHECK-ATOMIC: #define __CLANG_ATOMIC_SHORT_LOCK_FREE 2
+// CHECK-ATOMIC: #define __CLANG_ATOMIC_WCHAR_T_LOCK_FREE 2
diff --git a/clang/test/Preprocessor/x86_target_features.c b/clang/test/Preprocessor/x86_target_features.c
index 8d55c5d29e12f..e3bfdc5270899 100644
--- a/clang/test/Preprocessor/x86_target_features.c
+++ b/clang/test/Preprocessor/x86_target_features.c
@@ -483,3 +483,11 @@
 // RUN: %clang -target i386-unknown-unknown -march=atom -mno-enqcmd -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=NOENQCMD %s
 
 // NOENQCMD-NOT: #define __ENQCMD__ 1
+
+// RUN: %clang -target i386-unknown-unknown -march=atom -mserialize -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=SERIALIZE %s
+
+// SERIALIZE: #define __SERIALIZE__ 1
+
+// RUN: %clang -target i386-unknown-unknown -march=atom -mno-serialize -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=NOSERIALIZE %s
+
+// NOSERIALIZE-NOT: #define __SERIALIZE__ 1
diff --git a/clang/test/Sema/invalid-bitwidth-expr.mm b/clang/test/Sema/invalid-bitwidth-expr.mm
new file mode 100644
index 0000000000000..fe93cac683ae8
--- /dev/null
+++ b/clang/test/Sema/invalid-bitwidth-expr.mm
@@ -0,0 +1,13 @@
+// RUN: %clang_cc1 -fobjc-runtime=gcc -frecovery-ast -verify %s
+// RUN: %clang_cc1 -fobjc-runtime=gcc -fno-recovery-ast -verify %s
+
+@interface Ivar
+{
+  int Foo : foo(); // expected-error {{use of undeclared identifier}}
+};
+@end
+
+struct X { int Y: foo(); }; // expected-error {{use of undeclared identifier}}
+
+constexpr int s = sizeof(Ivar);
+constexpr int ss = sizeof(X);
diff --git a/clang/test/Sema/invalid-member.cpp b/clang/test/Sema/invalid-member.cpp
new file mode 100644
index 0000000000000..544979980fc97
--- /dev/null
+++ b/clang/test/Sema/invalid-member.cpp
@@ -0,0 +1,15 @@
+// RUN: %clang_cc1 -verify -fsyntax-only -fno-recovery-ast %s
+// RUN: %clang_cc1 -verify -fsyntax-only -frecovery-ast %s
+
+void foo(); // expected-note 2{{requires 0 arguments}}
+class X {
+  decltype(foo(42)) invalid; // expected-error {{no matching function}}
+};
+// Should be able to evaluate sizeof without crashing.
+static_assert(sizeof(X) == 1, "No valid members");
+
+class Y {
+  typeof(foo(42)) invalid; // expected-error {{no matching function}}
+};
+// Should be able to evaluate sizeof without crashing.
+static_assert(sizeof(Y) == 1, "No valid members");
diff --git a/clang/test/Sema/parentheses.c b/clang/test/Sema/parentheses.c
index 047bcbfe6cafd..164fe4c0f12d5 100644
--- a/clang/test/Sema/parentheses.c
+++ b/clang/test/Sema/parentheses.c
@@ -117,5 +117,5 @@ void conditional_op(int x, int y, _Bool b, void* p) {
   (void)(x && b ? 1 : 2);  // no warning, logical operator
 }
 
-// RUN: not %clang_cc1 -fsyntax-only -Wparentheses -Werror -fdiagnostics-show-option %s 2>&1 | FileCheck %s -check-prefix=CHECK-FLAG
+// RUN: not %clang_cc1 -fsyntax-only -Wparentheses -Werror %s 2>&1 | FileCheck %s -check-prefix=CHECK-FLAG
 // CHECK-FLAG: error: using the result of an assignment as a condition without parentheses [-Werror,-Wparentheses]
diff --git a/clang/test/Sema/types.c b/clang/test/Sema/types.c
index 8869b3427dc58..177e5fbd9704d 100644
--- a/clang/test/Sema/types.c
+++ b/clang/test/Sema/types.c
@@ -69,9 +69,15 @@ void test2(int i) {
   char c = (char __attribute__((may_alias))) i;
 }
 
-// vector size too large
-int __attribute__ ((vector_size(8192))) x1; // expected-error {{vector size too large}}
-typedef int __attribute__ ((ext_vector_type(8192))) x2; // expected-error {{vector size too large}}
+// vector size
+int __attribute__((vector_size(123456))) v1;
+int __attribute__((vector_size(0x1000000000))) v2;         // expected-error {{vector size too large}}
+int __attribute__((vector_size((__int128_t)1 << 100))) v3; // expected-error {{vector size too large}}
+int __attribute__((vector_size(0))) v4;                    // expected-error {{zero vector size}}
+typedef int __attribute__((ext_vector_type(123456))) e1;
+typedef int __attribute__((ext_vector_type(0x100000000))) e2;      // expected-error {{vector size too large}}
+typedef int __attribute__((vector_size((__int128_t)1 << 100))) e3; // expected-error {{vector size too large}}
+typedef int __attribute__((ext_vector_type(0))) e4;                // expected-error {{zero vector size}}
 
 // no support for vector enum type
 enum { e_2 } x3 __attribute__((vector_size(64))); // expected-error {{invalid vector element type}}
diff --git a/clang/test/SemaCUDA/attr-declspec.cu b/clang/test/SemaCUDA/attr-declspec.cu
index dda12ce8a51f2..bad86c65c23a5 100644
--- a/clang/test/SemaCUDA/attr-declspec.cu
+++ b/clang/test/SemaCUDA/attr-declspec.cu
@@ -6,16 +6,19 @@
 // RUN: %clang_cc1 -DEXPECT_WARNINGS -fms-extensions -fsyntax-only -verify -x c %s
 
 #if defined(EXPECT_WARNINGS)
-// expected-warning@+12 {{'__device__' attribute ignored}}
-// expected-warning@+12 {{'__global__' attribute ignored}}
-// expected-warning@+12 {{'__constant__' attribute ignored}}
-// expected-warning@+12 {{'__shared__' attribute ignored}}
-// expected-warning@+12 {{'__host__' attribute ignored}}
+// expected-warning@+15 {{'__device__' attribute ignored}}
+// expected-warning@+15 {{'__global__' attribute ignored}}
+// expected-warning@+15 {{'__constant__' attribute ignored}}
+// expected-warning@+15 {{'__shared__' attribute ignored}}
+// expected-warning@+15 {{'__host__' attribute ignored}}
+// expected-warning@+20 {{'__device_builtin_surface_type__' attribute ignored}}
+// expected-warning@+20 {{'__device_builtin_texture_type__' attribute ignored}}
 //
 // (Currently we don't for the other attributes. They are implemented with
 // IgnoredAttr, which is ignored irrespective of any LangOpts.)
 #else
-// expected-no-diagnostics
+// expected-warning@+14 {{'__device_builtin_surface_type__' attribute only applies to classes}}
+// expected-warning@+14 {{'__device_builtin_texture_type__' attribute only applies to classes}}
 #endif
 
 __declspec(__device__) void f_device();
diff --git a/clang/test/SemaCUDA/attributes-on-non-cuda.cu b/clang/test/SemaCUDA/attributes-on-non-cuda.cu
index e9e32ce658cbe..215721d273f44 100644
--- a/clang/test/SemaCUDA/attributes-on-non-cuda.cu
+++ b/clang/test/SemaCUDA/attributes-on-non-cuda.cu
@@ -7,16 +7,19 @@
 // RUN: %clang_cc1 -DEXPECT_WARNINGS -fsyntax-only -verify -x c %s
 
 #if defined(EXPECT_WARNINGS)
-// expected-warning@+12 {{'device' attribute ignored}}
-// expected-warning@+12 {{'global' attribute ignored}}
-// expected-warning@+12 {{'constant' attribute ignored}}
-// expected-warning@+12 {{'shared' attribute ignored}}
-// expected-warning@+12 {{'host' attribute ignored}}
+// expected-warning@+15 {{'device' attribute ignored}}
+// expected-warning@+15 {{'global' attribute ignored}}
+// expected-warning@+15 {{'constant' attribute ignored}}
+// expected-warning@+15 {{'shared' attribute ignored}}
+// expected-warning@+15 {{'host' attribute ignored}}
+// expected-warning@+21 {{'device_builtin_surface_type' attribute ignored}}
+// expected-warning@+21 {{'device_builtin_texture_type' attribute ignored}}
 //
 // NOTE: IgnoredAttr in clang which is used for the rest of
 // attributes ignores LangOpts, so there are no warnings.
 #else
-// expected-no-diagnostics
+// expected-warning@+15 {{'device_builtin_surface_type' attribute only applies to classes}}
+// expected-warning@+15 {{'device_builtin_texture_type' attribute only applies to classes}}
 #endif
 
 __attribute__((device)) void f_device();
diff --git a/clang/test/SemaCUDA/bad-attributes.cu b/clang/test/SemaCUDA/bad-attributes.cu
index 5aaff9759673c..d72f74471c237 100644
--- a/clang/test/SemaCUDA/bad-attributes.cu
+++ b/clang/test/SemaCUDA/bad-attributes.cu
@@ -70,3 +70,27 @@ void host_fn() {
 __device__ void device_fn() {
   __constant__ int c; // expected-error {{__constant__ variables must be global}}
 }
+
+typedef __attribute__((device_builtin_surface_type)) unsigned long long s0_ty; // expected-warning {{'device_builtin_surface_type' attribute only applies to classes}}
+typedef __attribute__((device_builtin_texture_type)) unsigned long long t0_ty; // expected-warning {{'device_builtin_texture_type' attribute only applies to classes}}
+
+struct __attribute__((device_builtin_surface_type)) s1_ref {}; // expected-error {{illegal device builtin surface reference type 's1_ref' declared here}}
+// expected-note@-1 {{'s1_ref' needs to be instantiated from a class template with proper template arguments}}
+struct __attribute__((device_builtin_texture_type)) t1_ref {}; // expected-error {{illegal device builtin texture reference type 't1_ref' declared here}}
+// expected-note@-1 {{'t1_ref' needs to be instantiated from a class template with proper template arguments}}
+
+template <typename T>
+struct __attribute__((device_builtin_surface_type)) s2_cls_template {}; // expected-error {{illegal device builtin surface reference class template 's2_cls_template' declared here}}
+// expected-note@-1 {{'s2_cls_template' needs to have exactly 2 template parameters}}
+template <typename T>
+struct __attribute__((device_builtin_texture_type)) t2_cls_template {}; // expected-error {{illegal device builtin texture reference class template 't2_cls_template' declared here}}
+// expected-note@-1 {{'t2_cls_template' needs to have exactly 3 template parameters}}
+
+template <int val, void *ptr>
+struct __attribute__((device_builtin_surface_type)) s3_cls_template {}; // expected-error {{illegal device builtin surface reference class template 's3_cls_template' declared here}}
+// expected-note@-1 {{the 1st template parameter of 's3_cls_template' needs to be a type}}
+// expected-note@-2 {{the 2nd template parameter of 's3_cls_template' needs to be an integer or enum value}}
+template <int val, int type, typename T>
+struct __attribute__((device_builtin_texture_type)) t3_cls_template {}; // expected-error {{illegal device builtin texture reference class template 't3_cls_template' declared here}}
+// expected-note@-1 {{the 1st template parameter of 't3_cls_template' needs to be a type}}
+// expected-note@-2 {{the 3rd template parameter of 't3_cls_template' needs to be an integer or enum value}}
diff --git a/clang/test/SemaCXX/PR20705.cpp b/clang/test/SemaCXX/PR20705.cpp
index be2676e563563..6b558b893d247 100644
--- a/clang/test/SemaCXX/PR20705.cpp
+++ b/clang/test/SemaCXX/PR20705.cpp
@@ -4,8 +4,7 @@ template <typename T>
 struct X {};
 auto b = []() {
   struct S {
-    static typename X<decltype(int)>::type Run(){};
-    // expected-error@-1 4{{}}
+    static typename X<decltype(int)>::type Run(){}; // expected-error {{expected '('}}
   };
   return 5;
 }();
@@ -16,6 +15,5 @@ class PC {
 
 template <typename T>
 class P {
-  static typename PC<T, Invalid>::Type Foo();
-  // expected-error@-1 4{{}}
+  static typename PC<T, Invalid>::Type Foo(); // expected-error {{undeclared identifier 'Invalid'}}
 };
diff --git a/clang/test/SemaCXX/PR9459.cpp b/clang/test/SemaCXX/PR9459.cpp
index 2b96f34e8a72a..010e228f67628 100644
--- a/clang/test/SemaCXX/PR9459.cpp
+++ b/clang/test/SemaCXX/PR9459.cpp
@@ -2,6 +2,6 @@
 
 // Don't crash.
 
-template<typename>struct ae_same;
+template<typename>struct ae_same; // expected-note {{}}
 template<typename>struct ts{}ap() // expected-error {{expected ';' after struct}} expected-error {{requires a type specifier}}
-{ts<a>::ap<ae_same<int>::&ae_same<>>::p(a); }; // expected-error {{use of undeclared identifier 'a'}}
+{ts<a>::ap<ae_same<int>::&ae_same<>>::p(a); }; // expected-error {{use of undeclared identifier 'a'}} expected-error 5{{}}
diff --git a/clang/test/SemaCXX/atomic-type.cpp b/clang/test/SemaCXX/atomic-type.cpp
index 1ed321e47b9af..d7d8bbba50ca3 100644
--- a/clang/test/SemaCXX/atomic-type.cpp
+++ b/clang/test/SemaCXX/atomic-type.cpp
@@ -103,3 +103,11 @@ namespace copy_init {
 bool PR21836(_Atomic(int) *x) { // expected-warning {{'_Atomic' is a C11 extension}}
     return *x;
 }
+
+namespace non_trivially_copyable {
+  struct S {
+    ~S() {}
+  };
+  _Atomic S s;  // expected-error {{_Atomic cannot be applied to type 'non_trivially_copyable::S' which is not trivially copyable}} \
+                // expected-warning {{'_Atomic' is a C11 extension}}
+}
diff --git a/clang/test/SemaCXX/builtins.cpp b/clang/test/SemaCXX/builtins.cpp
index 5306f4fad83b2..d08e673c04b77 100644
--- a/clang/test/SemaCXX/builtins.cpp
+++ b/clang/test/SemaCXX/builtins.cpp
@@ -14,7 +14,6 @@ template<int (*Compare)(const char *s1, const char *s2)>
 int equal(const char *s1, const char *s2) {
   return Compare(s1, s2) == 0;
 }
-
 template int equal<&__builtin_strcmp>(const char*, const char*); // expected-error {{builtin functions must be directly called}}
 
 // PR13195
diff --git a/clang/test/SemaCXX/cast-conversion.cpp b/clang/test/SemaCXX/cast-conversion.cpp
index e48d918f889bc..4d5abfdcfb1e3 100644
--- a/clang/test/SemaCXX/cast-conversion.cpp
+++ b/clang/test/SemaCXX/cast-conversion.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fsyntax-only -triple x86_64-unknown-unknown -verify %s -std=c++11 -Wno-unused
+// RUN: %clang_cc1 -fsyntax-only -triple x86_64-unknown-unknown -verify %s -std=c++11
 
 struct R {
   R(int);
diff --git a/clang/test/SemaCXX/constant-expression-cxx2a.cpp b/clang/test/SemaCXX/constant-expression-cxx2a.cpp
index 42d7300c5fdec..f66f380b635fd 100644
--- a/clang/test/SemaCXX/constant-expression-cxx2a.cpp
+++ b/clang/test/SemaCXX/constant-expression-cxx2a.cpp
@@ -1381,3 +1381,23 @@ namespace PR45133 {
   constinit V v2 = X<2>();
   constinit V v3 = X<3>(); // expected-error {{constant init}} expected-note {{constinit}} expected-note {{in call}}
 }
+
+namespace PR45350 {
+  int q;
+  struct V { int n; int *p = &n; constexpr ~V() { *p = *p * 10 + n; }};
+  constexpr int f(int n) {
+    int k = 0;
+    V *p = new V[n];
+    for (int i = 0; i != n; ++i) {
+      if (p[i].p != &p[i].n) return -1;
+      p[i].n = i;
+      p[i].p = &k;
+    }
+    delete[] p;
+    return k;
+  }
+  // [expr.delete]p6:
+  //   In the case of an array, the elements will be destroyed in order of
+  //   decreasing address
+  static_assert(f(6) == 543210);
+}
diff --git a/clang/test/SemaCXX/cxx1z-copy-omission.cpp b/clang/test/SemaCXX/cxx1z-copy-omission.cpp
index eceac810e72a5..a7133d79b463f 100644
--- a/clang/test/SemaCXX/cxx1z-copy-omission.cpp
+++ b/clang/test/SemaCXX/cxx1z-copy-omission.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++1z -verify -Wno-unused %s
+// RUN: %clang_cc1 -std=c++1z -verify %s
 
 struct Noncopyable {
   Noncopyable();
@@ -107,10 +107,8 @@ void test_expressions(bool b) {
   sizeof(make_indestructible()); // expected-error {{deleted}}
   sizeof(make_incomplete()); // expected-error {{incomplete}}
   typeid(Indestructible{}); // expected-error {{deleted}}
-  typeid(make_indestructible()); // expected-error {{deleted}} \
-                                 // expected-error {{need to include <typeinfo>}}
-  typeid(make_incomplete()); // expected-error {{incomplete}} \
-                             // expected-error {{need to include <typeinfo>}}
+  typeid(make_indestructible()); // expected-error {{deleted}}
+  typeid(make_incomplete()); // expected-error {{incomplete}}
 
   // FIXME: The first two cases here are now also valid in C++17 onwards.
   using I = decltype(Indestructible()); // expected-error {{deleted}}
diff --git a/clang/test/SemaCXX/decltype-crash.cpp b/clang/test/SemaCXX/decltype-crash.cpp
index 1ffc525a27037..1cebfcd72c861 100644
--- a/clang/test/SemaCXX/decltype-crash.cpp
+++ b/clang/test/SemaCXX/decltype-crash.cpp
@@ -3,8 +3,5 @@
 int& a();
 
 void f() {
-  decltype(a()) c; // expected-warning {{'decltype' is a keyword in C++11}} \
-                   // expected-error {{use of undeclared identifier 'decltype'}} \
-                   // expected-error {{expected ';' after expression}} \
-                   // expected-error {{use of undeclared identifier 'c'}}
+  decltype(a()) c; // expected-warning {{'decltype' is a keyword in C++11}} expected-error {{use of undeclared identifier 'decltype'}}
 }
diff --git a/clang/test/SemaCXX/decltype.cpp b/clang/test/SemaCXX/decltype.cpp
index f06ca226bca2b..32c61bbccc842 100644
--- a/clang/test/SemaCXX/decltype.cpp
+++ b/clang/test/SemaCXX/decltype.cpp
@@ -105,5 +105,6 @@ template<typename>
 class conditional {
 };
 
-void foo(conditional<decltype((1),int>) {  // expected-note 2 {{to match this '('}} expected-error {{expected ')'}}
+// FIXME: The diagnostics here are produced twice.
+void foo(conditional<decltype((1),int>) {  // expected-note 2 {{to match this '('}} expected-error {{expected ')'}} expected-note 2{{to match this '<'}}
 } // expected-error {{expected function body after function declarator}} expected-error 2 {{expected '>'}} expected-error {{expected ')'}}
diff --git a/clang/test/SemaCXX/implicit-exception-spec.cpp b/clang/test/SemaCXX/implicit-exception-spec.cpp
index 19d645d6d3d15..8545e8422b2a5 100644
--- a/clang/test/SemaCXX/implicit-exception-spec.cpp
+++ b/clang/test/SemaCXX/implicit-exception-spec.cpp
@@ -56,7 +56,7 @@ namespace ExceptionSpecification {
 namespace DefaultArgument {
   struct Default {
     struct T {
-      T(int = ExceptionIf<noexcept(Default())::f()); // expected-error {{call to implicitly-deleted default constructor}}
+      T(int = ExceptionIf<noexcept(Default())::f()); // expected-error {{call to implicitly-deleted default constructor}} expected-error {{expected '>'}} expected-note {{to match this '<'}}
     } t; // expected-note {{has no default constructor}}
   };
 }
diff --git a/clang/test/SemaCXX/injected-class-name-crash.cpp b/clang/test/SemaCXX/injected-class-name-crash.cpp
index 2996a7cdb48c2..a044ba064b58e 100644
--- a/clang/test/SemaCXX/injected-class-name-crash.cpp
+++ b/clang/test/SemaCXX/injected-class-name-crash.cpp
@@ -1,9 +1,9 @@
 // RUN: %clang_cc1 -fsyntax-only -verify %s
 
 template <class T>
-struct X : public Foo<Bar { // expected-error {{unknown template name 'Foo'}} expected-error {{use of undeclared identifier 'Bar'}}
+struct X : public Foo<Bar { // expected-error {{unknown template name 'Foo'}} expected-error {{use of undeclared identifier 'Bar'}} expected-note {{to match this '<'}}
   X();
-}; // expected-error {{expected '{' after base class list}}
+}; // expected-error {{expected '>'}} expected-error {{expected '{' after base class list}}
 
 
 template <class T>
diff --git a/clang/test/SemaCXX/invalid-member-expr.cpp b/clang/test/SemaCXX/invalid-member-expr.cpp
index e0b40794a0ce7..6ef33ea575a12 100644
--- a/clang/test/SemaCXX/invalid-member-expr.cpp
+++ b/clang/test/SemaCXX/invalid-member-expr.cpp
@@ -54,10 +54,7 @@ namespace test3 {
 namespace rdar11293995 {
 
 struct Length {
-  // FIXME: We try to annotate the template-id here during tentative parsing,
-  // and fail, then try again during the actual parse. This results in the same
-  // diagnostic being produced twice. :(
-  explicit Length(PassRefPtr<CalculationValue>); // expected-error 2{{undeclared identifier 'CalculationValue'}}
+  explicit Length(PassRefPtr<CalculationValue>); // expected-error {{undeclared identifier 'CalculationValue'}}
 };
 
 struct LengthSize {
diff --git a/clang/test/SemaCXX/invalid-template-specifier.cpp b/clang/test/SemaCXX/invalid-template-specifier.cpp
index 140ad00c9e6f3..f8f97ded5da5d 100644
--- a/clang/test/SemaCXX/invalid-template-specifier.cpp
+++ b/clang/test/SemaCXX/invalid-template-specifier.cpp
@@ -7,6 +7,5 @@ const template basic_istream<char>; // expected-error {{expected unqualified-id}
 
 namespace S {}
 template <class X> class Y {
-  void x() { S::template y<char>(1); } // expected-error {{no member named 'y' in namespace 'S'}} \
-                                       // expected-error {{unqualified-id}}
+  void x() { S::template y<char>(1); } // expected-error {{no member named 'y' in namespace 'S'}}
 };
diff --git a/clang/test/SemaCXX/literal-operators.cpp b/clang/test/SemaCXX/literal-operators.cpp
index 304aa7cab7f30..834d5ec7923e6 100644
--- a/clang/test/SemaCXX/literal-operators.cpp
+++ b/clang/test/SemaCXX/literal-operators.cpp
@@ -47,3 +47,7 @@ template <unsigned long long...> void operator "" _invalid();  // expected-error
 _Complex float operator""if(long double); // expected-warning {{reserved}}
 _Complex float test_if_1() { return 2.0f + 1.5if; };
 void test_if_2() { "foo"if; } // expected-error {{no matching literal operator for call to 'operator""if'}}
+
+template<typename T> void dependent_member_template() {
+  T().template operator""_foo<int>(); // expected-error {{'operator""_foo' following the 'template' keyword cannot refer to a dependent template}}
+}
diff --git a/clang/test/SemaCXX/pseudo-destructors.cpp b/clang/test/SemaCXX/pseudo-destructors.cpp
index b71b523de683e..7a5c540794e2d 100644
--- a/clang/test/SemaCXX/pseudo-destructors.cpp
+++ b/clang/test/SemaCXX/pseudo-destructors.cpp
@@ -119,3 +119,67 @@ void test2(Foo d) {
   d.~Derived(); // expected-error {{member reference type 'dotPointerAccess::Foo' (aka 'dotPointerAccess::Derived *') is a pointer; did you mean to use '->'}}
 }
 }
+
+int pr45294 = 1 .~undeclared_tempate_name<>(); // expected-error {{use of undeclared 'undeclared_tempate_name'}}
+
+namespace TwoPhaseLookup {
+  namespace NonTemplate {
+    struct Y {};
+    using G = Y;
+    template<typename T> void f(T *p) { p->~G(); } // expected-error {{no member named '~Y'}}
+    void h1(Y *p) { p->~G(); }
+    void h2(Y *p) { f(p); }
+    namespace N { struct G{}; }
+    void h3(N::G *p) { p->~G(); }
+    void h4(N::G *p) { f(p); } // expected-note {{instantiation of}}
+  }
+
+  namespace NonTemplateUndeclared {
+    struct Y {};
+    template<typename T> void f(T *p) { p->~G(); } // expected-error {{undeclared identifier 'G' in destructor name}}
+    using G = Y;
+    void h1(Y *p) { p->~G(); }
+    void h2(Y *p) { f(p); } // expected-note {{instantiation of}}
+    namespace N { struct G{}; }
+    void h3(N::G *p) { p->~G(); }
+    void h4(N::G *p) { f(p); }
+  }
+
+  namespace Template {
+    template<typename T> struct Y {};
+    template<class U> using G = Y<U>;
+    template<typename T> void f(T *p) { p->~G<int>(); } // expected-error {{no member named '~Y'}}
+    void h1(Y<int> *p) { p->~G<int>(); }
+    void h2(Y<int> *p) { f(p); }
+    namespace N { template<typename T> struct G {}; }
+    void h3(N::G<int> *p) { p->~G<int>(); }
+    void h4(N::G<int> *p) { f(p); } // expected-note {{instantiation of}}
+  }
+
+  namespace TemplateUndeclared {
+    template<typename T> struct Y {};
+    // FIXME: Formally, this is ill-formed before we hit any instantiation,
+    // because we aren't supposed to treat the '<' as introducing a template
+    // name.
+    template<typename T> void f(T *p) { p->~G<int>(); } // expected-error {{no member named 'G'}}
+    template<class U> using G = Y<U>;
+    void h1(Y<int> *p) { p->~G<int>(); }
+    void h2(Y<int> *p) { f(p); } // expected-note {{instantiation of}}
+    namespace N { template<typename T> struct G {}; }
+    void h3(N::G<int> *p) { p->~G<int>(); }
+    void h4(N::G<int> *p) { f(p); }
+  }
+
+  namespace TemplateNamesNonTemplate {
+    int A; // expected-note 2{{non-template here}}
+    template<typename> int B; // expected-note 2{{variable template 'B' declared here}} expected-warning {{extension}}
+    using C = int; // expected-note 2{{non-template here}}
+
+    template<typename T> void f1(int *p) { p->~A<int>(); } // expected-error {{'A' does not refer to a template}}
+    template<typename T> void f2(int *p) { p->~B<int>(); } // expected-error {{template name refers to non-type template 'B'}}
+    template<typename T> void f3(int *p) { p->~C<int>(); } // expected-error {{'C' does not refer to a template}}
+    template<typename T> void f4(int *p) { p->TemplateNamesNonTemplate::C::~A<int>(); } // expected-error {{'A' does not refer to a template}}
+    template<typename T> void f5(int *p) { p->TemplateNamesNonTemplate::C::~B<int>(); } // expected-error {{template name refers to non-type template 'TemplateNamesNonTemplate::B'}}
+    template<typename T> void f6(int *p) { p->TemplateNamesNonTemplate::C::~C<int>(); } // expected-error {{'C' does not refer to a template}}
+  }
+}
diff --git a/clang/test/SemaCXX/sizeless-1.cpp b/clang/test/SemaCXX/sizeless-1.cpp
index 5cbf1923d7055..a7c02343cb489 100644
--- a/clang/test/SemaCXX/sizeless-1.cpp
+++ b/clang/test/SemaCXX/sizeless-1.cpp
@@ -266,6 +266,7 @@ int vararg_receiver(int count, svint8_t first, ...) {
   __builtin_va_list va;
 
   __builtin_va_start(va, first);
+  __builtin_va_arg(va, svint8_t);
   __builtin_va_end(va);
   return count;
 }
@@ -477,6 +478,7 @@ void cxx_only(int sel) {
   (void)typeid(ref_int8);
   (void)typeid(static_int8_ptr);
 
+  _Static_assert(__is_trivially_copyable(svint8_t), "");
   _Static_assert(__is_trivially_destructible(svint8_t), "");
   _Static_assert(!__is_nothrow_assignable(svint8_t, svint8_t), "");
   _Static_assert(__is_nothrow_assignable(svint8_t &, svint8_t), "");
@@ -491,6 +493,16 @@ void cxx_only(int sel) {
   _Static_assert(!__is_assignable(svint8_t, svint8_t), "");
   _Static_assert(__is_assignable(svint8_t &, svint8_t), "");
   _Static_assert(!__is_assignable(svint8_t &, svint16_t), "");
+  _Static_assert(__has_nothrow_assign(svint8_t), "");
+  _Static_assert(__has_nothrow_move_assign(svint8_t), "");
+  _Static_assert(__has_nothrow_copy(svint8_t), "");
+  _Static_assert(__has_nothrow_constructor(svint8_t), "");
+  _Static_assert(__has_trivial_assign(svint8_t), "");
+  _Static_assert(__has_trivial_move_assign(svint8_t), "");
+  _Static_assert(__has_trivial_copy(svint8_t), "");
+  _Static_assert(__has_trivial_constructor(svint8_t), "");
+  _Static_assert(__has_trivial_move_constructor(svint8_t), "");
+  _Static_assert(__has_trivial_destructor(svint8_t), "");
   _Static_assert(!__has_virtual_destructor(svint8_t), "");
   _Static_assert(!__is_abstract(svint8_t), "");
   _Static_assert(!__is_aggregate(svint8_t), "");
@@ -502,7 +514,9 @@ void cxx_only(int sel) {
   _Static_assert(!__is_enum(svint8_t), "");
   _Static_assert(!__is_final(svint8_t), "");
   _Static_assert(!__is_literal(svint8_t), "");
+  _Static_assert(__is_pod(svint8_t), "");
   _Static_assert(!__is_polymorphic(svint8_t), "");
+  _Static_assert(__is_trivial(svint8_t), "");
   _Static_assert(__is_object(svint8_t), "");
   _Static_assert(!__is_arithmetic(svint8_t), "");
   _Static_assert(!__is_floating_point(svint8_t), "");
@@ -579,9 +593,7 @@ void cxx_only(int sel) {
   for (const svint8_t &x : wrapper<svint8_t>()) { // expected-warning {{loop variable 'x' binds to a temporary value produced by a range of type 'wrapper<svint8_t>'}} expected-note {{use non-reference type}}
     (void)x;
   }
-  // This warning is bogus and will be removed by a later patch.
-  // The point is to show that it's being removed for the right reasons.
-  for (const svint8_t x : wrapper<const svint8_t &>()) { // expected-warning {{loop variable 'x' creates a copy from type 'const svint8_t'}} expected-note {{use reference type}}
+  for (const svint8_t x : wrapper<const svint8_t &>()) {
     (void)x;
   }
 #endif
diff --git a/clang/test/SemaCXX/vararg-non-pod.cpp b/clang/test/SemaCXX/vararg-non-pod.cpp
index 1b7f3b68dc970..a1bbe748d12da 100644
--- a/clang/test/SemaCXX/vararg-non-pod.cpp
+++ b/clang/test/SemaCXX/vararg-non-pod.cpp
@@ -164,6 +164,13 @@ void t6(Foo somearg, ... ) {
   __builtin_va_start(list, somearg);
 }
 
+// __builtin_stdarg_start is a compatibility alias for __builtin_va_start,
+// it should behave the same
+void t6b(Foo somearg, ... ) {
+  __builtin_va_list list;
+  __builtin_stdarg_start(list, somearg); // second argument to 'va_start' is not the last named parameter [-Wvarargs]
+}
+
 void t7(int n, ...) {
   __builtin_va_list list;
   __builtin_va_start(list, n);
diff --git a/clang/test/SemaCXX/varargs.cpp b/clang/test/SemaCXX/varargs.cpp
index 625f1dce18f3d..f2f53dc2001fd 100644
--- a/clang/test/SemaCXX/varargs.cpp
+++ b/clang/test/SemaCXX/varargs.cpp
@@ -22,8 +22,7 @@ void no_params(...) {
 // default ctor.
 void record_context(int a, ...) {
   struct Foo {
-    // expected-error@+2 {{'va_start' cannot be used outside a function}}
-    // expected-error@+1 {{default argument references parameter 'a'}}
+    // expected-error@+1 {{'va_start' cannot be used outside a function}}
     void meth(int a, int b = (__builtin_va_start(ap, a), 0)) {}
   };
 }
diff --git a/clang/test/SemaCXX/vector.cpp b/clang/test/SemaCXX/vector.cpp
index 67e5a94cb108c..caa840596d7db 100644
--- a/clang/test/SemaCXX/vector.cpp
+++ b/clang/test/SemaCXX/vector.cpp
@@ -335,7 +335,7 @@ const int &reference_to_vec_element = vi4(1).x;
 typedef bool bad __attribute__((__vector_size__(16)));  // expected-error {{invalid vector element type 'bool'}}
 
 namespace Templates {
-template <typename Elt, unsigned Size>
+template <typename Elt, unsigned long long Size>
 struct TemplateVectorType {
   typedef Elt __attribute__((__vector_size__(Size))) type; // #1
 };
@@ -343,7 +343,7 @@ struct TemplateVectorType {
 template <int N, typename T>
 struct PR15730 {
   typedef T __attribute__((vector_size(N * sizeof(T)))) type;
-  typedef T __attribute__((vector_size(8192))) type2; // #2
+  typedef T __attribute__((vector_size(0x1000000000))) type2; // #2
   typedef T __attribute__((vector_size(3))) type3; // #3
 };
 
@@ -352,19 +352,20 @@ void Init() {
   const TemplateVectorType<int, 32>::type Works2 = {};
   // expected-error@#1 {{invalid vector element type 'bool'}}
   // expected-note@+1 {{in instantiation of template class 'Templates::TemplateVectorType<bool, 32>' requested here}}
-  const TemplateVectorType<bool, 32>::type NoBool;
+  const TemplateVectorType<bool, 32>::type NoBool = {};
   // expected-error@#1 {{invalid vector element type 'int __attribute__((ext_vector_type(4)))' (vector of 4 'int' values)}}
   // expected-note@+1 {{in instantiation of template class 'Templates::TemplateVectorType<int __attribute__((ext_vector_type(4))), 32>' requested here}}
-  const TemplateVectorType<vi4, 32>::type NoComplex;
+  const TemplateVectorType<vi4, 32>::type NoComplex = {};
   // expected-error@#1 {{vector size not an integral multiple of component size}}
   // expected-note@+1 {{in instantiation of template class 'Templates::TemplateVectorType<int, 33>' requested here}}
-  const TemplateVectorType<int, 33>::type BadSize;
+  const TemplateVectorType<int, 33>::type BadSize = {};
+  const TemplateVectorType<int, 3200>::type Large = {};
   // expected-error@#1 {{vector size too large}}
-  // expected-note@+1 {{in instantiation of template class 'Templates::TemplateVectorType<int, 8192>' requested here}}
-  const TemplateVectorType<int, 8192>::type TooLarge;
+  // expected-note@+1 {{in instantiation of template class 'Templates::TemplateVectorType<int, 68719476736>' requested here}}
+  const TemplateVectorType<int, 0x1000000000>::type TooLarge = {};
   // expected-error@#1 {{zero vector size}}
   // expected-note@+1 {{in instantiation of template class 'Templates::TemplateVectorType<int, 0>' requested here}}
-  const TemplateVectorType<int, 0>::type Zero;
+  const TemplateVectorType<int, 0>::type Zero = {};
 
   // expected-error@#2 {{vector size too large}}
   // expected-error@#3 {{vector size not an integral multiple of component size}}
@@ -400,3 +401,77 @@ namespace swizzle_typo_correction {
     return A.xyzw < B.x && B.y > A.y; // OK, not a typo for 'xyzv'
   }
 }
+
+namespace PR45299 {
+typedef float float4 __attribute__((vector_size(16)));
+
+// In this example, 'k' is value dependent. PR45299 reported that this asserted
+// because of that, since the truncation check attempted to constant evaluate k,
+// which it could not do because it is dependent.
+template <typename T>
+struct NormalMember {
+  float4 f(float4 x) {
+    return k * x;
+  }
+  float k;
+};
+
+#if __cplusplus >= 201103L
+// This should not diagnose, since the constant evaluator (during instantiation)
+// can tell that this isn't a truncation.
+template <typename T>
+struct ConstantValueNoDiag {
+  float4 f(float4 x) {
+    return k * x;
+  }
+  static constexpr double k = 1;
+};
+
+// The following two both diagnose because they cause a truncation.  Test both
+// the dependent type and non-dependent type versions.
+template <typename T>
+struct DiagTrunc {
+  float4 f(float4 x) {
+    // expected-error@+1{{as implicit conversion would cause truncation}}
+    return k * x;
+  }
+  static constexpr double k = 1340282346638528859811704183484516925443.000000;
+};
+template <typename T>
+struct DiagTruncDependentType {
+  float4 f(float4 x) {
+    // expected-error@+1{{as implicit conversion would cause truncation}}
+    return k * x;
+  }
+  static constexpr T k = 1340282346638528859811704183484516925443.000000;
+};
+
+template <typename T>
+struct PR45298 {
+    T k1 = T(0);
+};
+
+// Ensure this no longer asserts.
+template <typename T>
+struct PR45298Consumer {
+  float4 f(float4 x) {
+    return (float)s.k1 * x;
+  }
+
+  PR45298<T> s;
+};
+#endif // __cplusplus >= 201103L
+
+void use() {
+  float4 theFloat4;
+  NormalMember<double>().f(theFloat4);
+#if __cplusplus >= 201103L
+  ConstantValueNoDiag<double>().f(theFloat4);
+  // expected-note@+1{{in instantiation of member function}}
+  DiagTrunc<double>().f(theFloat4);
+  // expected-note@+1{{in instantiation of member function}}
+  DiagTruncDependentType<double>().f(theFloat4);
+  PR45298Consumer<double>().f(theFloat4);
+#endif // __cplusplus >= 201103L
+}
+}
diff --git a/clang/test/SemaCXX/warn-thread-safety-analysis.cpp b/clang/test/SemaCXX/warn-thread-safety-analysis.cpp
index 23255a53eae7c..cdb22cd22a996 100644
--- a/clang/test/SemaCXX/warn-thread-safety-analysis.cpp
+++ b/clang/test/SemaCXX/warn-thread-safety-analysis.cpp
@@ -5648,6 +5648,22 @@ namespace ReturnScopedLockable {
     auto ptr = get();
     return ptr->f();
   }
+  void use_constructor() {
+    auto ptr = ReadLockedPtr<Object>(nullptr);
+    ptr->f();
+    auto ptr2 = ReadLockedPtr<Object>{nullptr};
+    ptr2->f();
+    auto ptr3 = (ReadLockedPtr<Object>{nullptr});
+    ptr3->f();
+  }
+  struct Convertible {
+    Convertible();
+    operator ReadLockedPtr<Object>();
+  };
+  void use_conversion() {
+    ReadLockedPtr<Object> ptr = Convertible();
+    ptr->f();
+  }
 }
 
 namespace PR38640 {
diff --git a/clang/test/SemaObjC/crash-on-type-args-protocols.m b/clang/test/SemaObjC/crash-on-type-args-protocols.m
index f7d847851faf6..0d902631a28c9 100644
--- a/clang/test/SemaObjC/crash-on-type-args-protocols.m
+++ b/clang/test/SemaObjC/crash-on-type-args-protocols.m
@@ -14,27 +14,27 @@ @interface X : NSObject <X>
 @class A;
 
 #ifdef FIRST
-id<X> F1(id<[P> v) { // expected-error {{expected a type}} // expected-error {{use of undeclared identifier 'P'}} // expected-error {{use of undeclared identifier 'v'}} // expected-note {{to match this '('}}
+id<X> F1(id<[P> v) { // expected-error {{expected a type}} expected-error {{use of undeclared identifier 'P'}} expected-error {{use of undeclared identifier 'v'}} expected-note {{to match this '('}} expected-note {{to match this '<'}}
   return 0;
-}
+} // expected-error {{expected '>'}}
 #endif
 
 #ifdef SECOND
-id<X> F2(id<P[X> v) { // expected-error {{unknown type name 'P'}} // expected-error {{unexpected interface name 'X': expected expression}} // expected-error {{use of undeclared identifier 'v'}} // expected-note {{to match this '('}}
+id<X> F2(id<P[X> v) { // expected-error {{unknown type name 'P'}} expected-error {{unexpected interface name 'X': expected expression}} expected-error {{use of undeclared identifier 'v'}} expected-note {{to match this '('}} expected-note {{to match this '<'}}
   return 0;
-}
+} // expected-error {{expected '>'}}
 #endif
 
 #ifdef THIRD
-id<X> F3(id<P, P *[> v) { // expected-error {{unknown type name 'P'}} // expected-error {{expected expression}} // expected-error {{use of undeclared identifier 'v'}} // expected-note {{to match this '('}}
+id<X> F3(id<P, P *[> v) { // expected-error {{unknown type name 'P'}} expected-error {{expected expression}} expected-error {{use of undeclared identifier 'v'}} expected-note {{to match this '('}} expected-note {{to match this '<'}}
   return 0;
-}
+} // expected-error {{expected '>'}}
 #endif
 
 #ifdef FOURTH
-id<X> F4(id<P, P *(> v { // expected-error {{unknown type name 'P'}} // expected-error {{expected ')'}} // expected-note {{to match this '('}} // expected-note {{to match this '('}}
+id<X> F4(id<P, P *(> v { // expected-error {{unknown type name 'P'}} expected-error {{expected ')'}} expected-note {{to match this '('}} expected-note {{to match this '('}} expected-note {{to match this '<'}}
   return 0;
-}
+} // expected-error {{expected '>'}}
 #endif
 
-// expected-error {{expected '>'}} // expected-error {{expected parameter declarator}} // expected-error {{expected ')'}} // expected-error {{expected function body after function declarator}}
+// expected-error {{expected parameter declarator}} // expected-error {{expected ')'}} // expected-error {{expected function body after function declarator}}
diff --git a/clang/test/SemaObjCXX/parameterized_classes_subst.mm b/clang/test/SemaObjCXX/parameterized_classes_subst.mm
index 2eb630aeedf49..8aacf21faf091 100644
--- a/clang/test/SemaObjCXX/parameterized_classes_subst.mm
+++ b/clang/test/SemaObjCXX/parameterized_classes_subst.mm
@@ -427,7 +427,6 @@ void testVariadicInstantiation() {
 template<typename K, typename V>
 struct NonDependentTemplate {
   typedef NSMutableDictionaryBuilder::template apply<NSString *, NSObject *> type; // expected-error{{'apply' following the 'template' keyword does not refer to a template}}
-  // expected-error@-1{{expected member name or }}
 };
 
 // However, one can use an alias template to turn a parameterized
diff --git a/clang/test/SemaOpenCL/fdeclare-opencl-builtins.cl b/clang/test/SemaOpenCL/fdeclare-opencl-builtins.cl
index e593b21ec4591..d1dcdfe8cb359 100644
--- a/clang/test/SemaOpenCL/fdeclare-opencl-builtins.cl
+++ b/clang/test/SemaOpenCL/fdeclare-opencl-builtins.cl
@@ -140,14 +140,11 @@ kernel void basic_image_writeonly(write_only image1d_buffer_t image_write_only_i
 
 kernel void basic_subgroup(global uint *out) {
   out[0] = get_sub_group_size();
-#if !defined(__OPENCL_CPP_VERSION__) && __OPENCL_C_VERSION__ < CL_VERSION_2_0
-// expected-error@-2{{implicit declaration of function 'get_sub_group_size' is invalid in OpenCL}}
-// expected-error@-3{{implicit conversion changes signedness: 'int' to 'uint' (aka 'unsigned int')}}
-#elif defined(__OPENCL_CPP_VERSION__)
-// expected-error@-5{{no matching function for call to 'get_sub_group_size'}}
-// expected-note@-6{{candidate unavailable as it requires OpenCL extension 'cl_khr_subgroups' to be enabled}}
+#if defined(__OPENCL_CPP_VERSION__)
+  // expected-error@-2{{no matching function for call to 'get_sub_group_size'}}
+  // expected-note@-3{{candidate unavailable as it requires OpenCL extension 'cl_khr_subgroups' to be enabled}}
 #else
-// expected-error@-8{{use of declaration 'get_sub_group_size' requires cl_khr_subgroups extension to be enabled}}
+  // expected-error@-5{{use of declaration 'get_sub_group_size' requires cl_khr_subgroups extension to be enabled}}
 #endif
 }
 
diff --git a/clang/test/SemaOpenCLCXX/address-space-references.cl b/clang/test/SemaOpenCLCXX/address-space-references.cl
index 66cd1c02e32f9..068318dfa1415 100644
--- a/clang/test/SemaOpenCLCXX/address-space-references.cl
+++ b/clang/test/SemaOpenCLCXX/address-space-references.cl
@@ -11,7 +11,7 @@ int bar(const __global unsigned int &i); // expected-note{{passing argument to p
 int bar(const unsigned int &i);
 
 void foo() {
-  bar(1); // expected-error{{binding reference of type 'const __global unsigned int' to value of type 'int' changes address space}}
+  bar(1) // expected-error{{binding reference of type 'const __global unsigned int' to value of type 'int' changes address space}}
 }
 
 // Test addr space conversion with nested pointers
diff --git a/clang/test/SemaSYCL/wrapped-accessor.cpp b/clang/test/SemaSYCL/wrapped-accessor.cpp
index b111b6771342f..83bb3ff2448fb 100644
--- a/clang/test/SemaSYCL/wrapped-accessor.cpp
+++ b/clang/test/SemaSYCL/wrapped-accessor.cpp
@@ -23,10 +23,10 @@ int main() {
 }
 
 // Check declaration of the kernel
-// CHECK: wrapped_access{{.*}} 'void (AccWrapper<cl::sycl::accessor<int, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::global_buffer, cl::sycl::access::placeholder::false_t> >, __global int *, cl::sycl::range<1>, cl::sycl::range<1>, cl::sycl::id<1>)'
+// CHECK: wrapped_access{{.*}} 'void (AccWrapper<cl::sycl::accessor<int, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::global_buffer, cl::sycl::access::placeholder::false_t>>, __global int *, cl::sycl::range<1>, cl::sycl::range<1>, cl::sycl::id<1>)'
 
 // Check parameters of the kernel
-// CHECK: ParmVarDecl {{.*}} used _arg_ 'AccWrapper<cl::sycl::accessor<int, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::global_buffer, cl::sycl::access::placeholder::false_t> >':'AccWrapper<cl::sycl::accessor<int, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::global_buffer, cl::sycl::access::placeholder::false_t> >'
+// CHECK: ParmVarDecl {{.*}} used _arg_ 'AccWrapper<cl::sycl::accessor<int, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::global_buffer, cl::sycl::access::placeholder::false_t>>':'AccWrapper<cl::sycl::accessor<int, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::global_buffer, cl::sycl::access::placeholder::false_t>>'
 // CHECK: ParmVarDecl {{.*}} used _arg_accessor '__global int *'
 // CHECK: ParmVarDecl {{.*}} used [[_arg_AccessRange:[0-9a-zA-Z_]+]] 'cl::sycl::range<1>'
 // CHECK: ParmVarDecl {{.*}} used [[_arg_MemRange:[0-9a-zA-Z_]+]] 'cl::sycl::range<1>'
@@ -37,14 +37,14 @@ int main() {
 // CHECK: VarDecl {{.*}}'(lambda at {{.*}}wrapped-accessor.cpp{{.*}})'
 // CHECK-NEXT: InitListExpr
 // CHECK-NEXT: CXXConstructExpr {{.*}}AccWrapper<cl::sycl::accessor
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'const AccWrapper<cl::sycl::accessor<int, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::global_buffer, cl::sycl::access::placeholder::false_t> >' lvalue <NoOp>
-// CHECK-NEXT: DeclRefExpr {{.*}} 'AccWrapper<cl::sycl::accessor<int, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::global_buffer, cl::sycl::access::placeholder::false_t> >':'AccWrapper<cl::sycl::accessor<int, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::global_buffer, cl::sycl::access::placeholder::false_t> >' lvalue ParmVar {{.*}} '_arg_' 'AccWrapper<cl::sycl::accessor<int, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::global_buffer, cl::sycl::access::placeholder::false_t> >':'AccWrapper<cl::sycl::accessor<int, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::global_buffer, cl::sycl::access::placeholder::false_t> >'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'const AccWrapper<cl::sycl::accessor<int, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::global_buffer, cl::sycl::access::placeholder::false_t>>' lvalue <NoOp>
+// CHECK-NEXT: DeclRefExpr {{.*}} 'AccWrapper<cl::sycl::accessor<int, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::global_buffer, cl::sycl::access::placeholder::false_t>>':'AccWrapper<cl::sycl::accessor<int, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::global_buffer, cl::sycl::access::placeholder::false_t>>' lvalue ParmVar {{.*}} '_arg_' 'AccWrapper<cl::sycl::accessor<int, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::global_buffer, cl::sycl::access::placeholder::false_t>>':'AccWrapper<cl::sycl::accessor<int, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::global_buffer, cl::sycl::access::placeholder::false_t>>'
 
 // Check that accessor field of the wrapper object is initialized using __init method
 // CHECK-NEXT: CXXMemberCallExpr {{.*}} 'void'
 // CHECK-NEXT: MemberExpr {{.*}} 'void ({{.*}}PtrType, range<1>, range<1>, id<1>)' lvalue .__init
 // CHECK-NEXT: MemberExpr {{.*}} 'cl::sycl::accessor<int, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::global_buffer, cl::sycl::access::placeholder::false_t>':'cl::sycl::accessor<int, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::global_buffer, cl::sycl::access::placeholder::false_t>' lvalue .accessor {{.*}}
-// CHECK-NEXT: MemberExpr {{.*}} 'AccWrapper<cl::sycl::accessor<int, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::global_buffer, cl::sycl::access::placeholder::false_t> >':'AccWrapper<cl::sycl::accessor<int, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::global_buffer, cl::sycl::access::placeholder::false_t> >' lvalue .
+// CHECK-NEXT: MemberExpr {{.*}} 'AccWrapper<cl::sycl::accessor<int, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::global_buffer, cl::sycl::access::placeholder::false_t>>':'AccWrapper<cl::sycl::accessor<int, 1, cl::sycl::access::mode::read_write, cl::sycl::access::target::global_buffer, cl::sycl::access::placeholder::false_t>>' lvalue .
 // CHECK-NEXT: DeclRefExpr {{.*}} '(lambda at {{.*}}wrapped-accessor.cpp{{.*}})' lvalue Var {{.*}} '(lambda at {{.*}}wrapped-accessor.cpp{{.*}})'
 
 // Parameters of the _init method
diff --git a/clang/test/SemaTemplate/dependent-base-classes.cpp b/clang/test/SemaTemplate/dependent-base-classes.cpp
index bb4f3ca1f17c8..f8f36b2a077d4 100644
--- a/clang/test/SemaTemplate/dependent-base-classes.cpp
+++ b/clang/test/SemaTemplate/dependent-base-classes.cpp
@@ -55,8 +55,7 @@ namespace PR6031 {
   struct NoDepBase {
     int foo() {
       class NoDepBase::Nested nested; // expected-error{{no class named 'Nested' in 'NoDepBase<T>'}}
-      typedef typename NoDepBase::template MemberTemplate<T>::type type; // expected-error{{no member named 'MemberTemplate' in 'NoDepBase<T>'}} \
-      // FIXME: expected-error{{unqualified-id}}
+      typedef typename NoDepBase::template MemberTemplate<T>::type type; // expected-error{{no member named 'MemberTemplate' in 'NoDepBase<T>'}}
       return NoDepBase::a; // expected-error{{no member named 'a' in 'NoDepBase<T>'}}
     }
   };
@@ -103,10 +102,7 @@ namespace PR6081 {
     template< class X >
     void f0(const X & k)
     {
-      this->template f1<int>()(k); // expected-error{{no member named 'f1' in 'C<T>'}} \
-      // FIXME: expected-error{{unqualified-id}} \
-      // expected-error{{function-style cast or type construction}} \
-      // expected-error{{expected expression}}
+      this->template f1<int>()(k); // expected-error{{no member named 'f1' in 'C<T>'}}
     }
   };
 }
diff --git a/clang/test/SemaTemplate/instantiate-init.cpp b/clang/test/SemaTemplate/instantiate-init.cpp
index 69f4d5fc4cc46..99b29c77d55af 100644
--- a/clang/test/SemaTemplate/instantiate-init.cpp
+++ b/clang/test/SemaTemplate/instantiate-init.cpp
@@ -100,7 +100,7 @@ namespace PR7985 {
     integral_c<1> ic1 = array_lengthof(Description<int>::data);
     (void)sizeof(array_lengthof(Description<float>::data));
 
-    (void)sizeof(array_lengthof( // expected-error{{no matching function for call to 'array_lengthof'}}
+    sizeof(array_lengthof( // expected-error{{no matching function for call to 'array_lengthof'}}
                           Description<int*>::data // expected-note{{in instantiation of static data member 'PR7985::Description<int *>::data' requested here}}
                           ));
 
diff --git a/clang/test/SemaTemplate/instantiate-member-expr.cpp b/clang/test/SemaTemplate/instantiate-member-expr.cpp
index 6ba94b28df622..abe7ec8f5445f 100644
--- a/clang/test/SemaTemplate/instantiate-member-expr.cpp
+++ b/clang/test/SemaTemplate/instantiate-member-expr.cpp
@@ -16,7 +16,7 @@ class ExprEngine {
 
  template <typename CHECKER>
  void registerCheck(CHECKER *check) {
-   Checkers.push_back(S<void *>()); // expected-note {{in instantiation of member function 'vector<S<void *> >::push_back' requested here}}
+   Checkers.push_back(S<void *>()); // expected-note {{in instantiation of member function 'vector<S<void *>>::push_back' requested here}}
  }
 };
 
diff --git a/clang/test/SemaTemplate/ms-delayed-default-template-args.cpp b/clang/test/SemaTemplate/ms-delayed-default-template-args.cpp
index b2350c900e639..3c46905955245 100644
--- a/clang/test/SemaTemplate/ms-delayed-default-template-args.cpp
+++ b/clang/test/SemaTemplate/ms-delayed-default-template-args.cpp
@@ -49,9 +49,7 @@ template <typename T> struct Bar { T x; };
 template <typename T = Bar<Weber>>  // expected-error {{use of undeclared identifier 'Weber'}}
 struct Foo {
   static_assert(sizeof(T) == 4, "Bar should have gotten int");
-  // FIXME: These diagnostics are bad.
-}; // expected-error {{expected ',' or '>' in template-parameter-list}}
-// expected-warning@-1 {{does not declare anything}}
+};
 typedef int Weber;
 }
 
diff --git a/clang/test/SemaTemplate/nested-name-spec-template.cpp b/clang/test/SemaTemplate/nested-name-spec-template.cpp
index e9d0eb202033d..07dd41bfe27bf 100644
--- a/clang/test/SemaTemplate/nested-name-spec-template.cpp
+++ b/clang/test/SemaTemplate/nested-name-spec-template.cpp
@@ -58,8 +58,7 @@ struct ::N::A<int>::X {
 
 template<typename T>
 struct TestA {
-  typedef typename N::template B<T>::type type; // expected-error{{'B' following the 'template' keyword does not refer to a template}} \
-                                                // expected-error{{expected member name}}
+  typedef typename N::template B<T>::type type; // expected-error{{'B' following the 'template' keyword does not refer to a template}}
 };
 
 // Reduced from a Boost failure.
@@ -143,8 +142,7 @@ namespace PR9449 {
 
   template <typename T>
   void f() {
-    int s<T>::template n<T>::* f; // expected-error{{implicit instantiation of undefined template 'PR9449::s<int>'}} \
-    // expected-error{{no member named 'n'}}
+    int s<T>::template n<T>::* f; // expected-error{{implicit instantiation of undefined template 'PR9449::s<int>'}}
   }
 
   template void f<int>(); // expected-note{{in instantiation of}}
diff --git a/clang/test/SemaTemplate/temp_arg.cpp b/clang/test/SemaTemplate/temp_arg.cpp
index df5bf875f3bed..72c172f001bd2 100644
--- a/clang/test/SemaTemplate/temp_arg.cpp
+++ b/clang/test/SemaTemplate/temp_arg.cpp
@@ -15,6 +15,6 @@ A a4; // expected-error{{use of class template 'A' requires template arguments}}
 namespace test0 {
   template <class t> class foo {};
   template <class t> class bar {
-    bar(::test0::foo<tee> *ptr) {} // FIXME(redundant): expected-error 2 {{use of undeclared identifier 'tee'}}
+    bar(::test0::foo<tee> *ptr) {} // expected-error {{use of undeclared identifier 'tee'}}
   };
 }
diff --git a/clang/test/SemaTemplate/temp_arg_nontype_cxx1z.cpp b/clang/test/SemaTemplate/temp_arg_nontype_cxx1z.cpp
index 253ad7bc56065..fdcc5006f5890 100644
--- a/clang/test/SemaTemplate/temp_arg_nontype_cxx1z.cpp
+++ b/clang/test/SemaTemplate/temp_arg_nontype_cxx1z.cpp
@@ -102,7 +102,7 @@ namespace PtrMem {
   static_assert(!is_same<Ab, Abce>, ""); // expected-error {{undeclared}} expected-error {{must be a type}}
   static_assert(!is_same<Ab, Abde>, ""); // expected-error {{undeclared}} expected-error {{must be a type}}
   static_assert(!is_same<Abce, Abde>, ""); // expected-error 2{{undeclared}} expected-error {{must be a type}}
-  static_assert(is_same<Abce, A<int B::*, (int B::*)(int C::*)&E::e>, ""); // expected-error {{undeclared}} expected-error {{not supported}}
+  static_assert(is_same<Abce, A<int B::*, (int B::*)(int C::*)&E::e>>, ""); // expected-error {{undeclared}} expected-error {{not supported}}
 
   using Ae = A<int E::*, e>;
   using Ae = A<int E::*, &E::e>;
@@ -111,7 +111,7 @@ namespace PtrMem {
   static_assert(!is_same<Ae, Aecb>, ""); // expected-error {{undeclared}} expected-error {{must be a type}}
   static_assert(!is_same<Ae, Aedb>, ""); // expected-error {{undeclared}} expected-error {{must be a type}}
   static_assert(!is_same<Aecb, Aedb>, ""); // expected-error 2{{undeclared}} expected-error {{must be a type}}
-  static_assert(is_same<Aecb, A<int E::*, (int E::*)(int C::*)&B::b>, ""); // expected-error {{undeclared}} expected-error {{not supported}}
+  static_assert(is_same<Aecb, A<int E::*, (int E::*)(int C::*)&B::b>>, ""); // expected-error {{undeclared}} expected-error {{not supported}}
 
   using An = A<int E::*, nullptr>;
   using A0 = A<int E::*, (int E::*)0>;
diff --git a/clang/test/Unit/lit.site.cfg.py.in b/clang/test/Unit/lit.site.cfg.py.in
index 715b4d93f58f6..400a9c05e58c5 100644
--- a/clang/test/Unit/lit.site.cfg.py.in
+++ b/clang/test/Unit/lit.site.cfg.py.in
@@ -2,14 +2,14 @@
 
 import sys
 
-config.llvm_src_root = "@LLVM_SOURCE_DIR@"
-config.llvm_obj_root = "@LLVM_BINARY_DIR@"
-config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
-config.llvm_libs_dir = "@LLVM_LIBS_DIR@"
+config.llvm_src_root = path("@LLVM_SOURCE_DIR@")
+config.llvm_obj_root = path("@LLVM_BINARY_DIR@")
+config.llvm_tools_dir = path("@LLVM_TOOLS_DIR@")
+config.llvm_libs_dir = path("@LLVM_LIBS_DIR@")
 config.llvm_build_mode = "@LLVM_BUILD_MODE@"
-config.clang_obj_root = "@CLANG_BINARY_DIR@"
+config.clang_obj_root = path("@CLANG_BINARY_DIR@")
 config.enable_shared = @ENABLE_SHARED@
-config.shlibdir = "@SHLIBDIR@"
+config.shlibdir = path("@SHLIBDIR@")
 config.target_triple = "@TARGET_TRIPLE@"
 
 # Support substitution of the tools_dir, libs_dirs, and build_mode with user
@@ -25,4 +25,5 @@ except KeyError:
     lit_config.fatal("unable to find %r parameter, use '--param=%s=VALUE'" % (key,key))
 
 # Let the main config do the real work.
-lit_config.load_config(config, "@CLANG_SOURCE_DIR@/test/Unit/lit.cfg.py")
+lit_config.load_config(
+    config, os.path.join(path("@CLANG_SOURCE_DIR@"), "test/Unit/lit.cfg.py"))
diff --git a/clang/test/lit.site.cfg.py.in b/clang/test/lit.site.cfg.py.in
index 62616d9a2b959..84dd0f8846cf9 100644
--- a/clang/test/lit.site.cfg.py.in
+++ b/clang/test/lit.site.cfg.py.in
@@ -2,16 +2,16 @@
 
 import sys
 
-config.llvm_src_root = "@LLVM_SOURCE_DIR@"
-config.llvm_obj_root = "@LLVM_BINARY_DIR@"
-config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
-config.llvm_libs_dir = "@LLVM_LIBS_DIR@"
-config.llvm_shlib_dir = "@SHLIBDIR@"
+config.llvm_src_root = path("@LLVM_SOURCE_DIR@")
+config.llvm_obj_root = path("@LLVM_BINARY_DIR@")
+config.llvm_tools_dir = path("@LLVM_TOOLS_DIR@")
+config.llvm_libs_dir = path("@LLVM_LIBS_DIR@")
+config.llvm_shlib_dir = path("@SHLIBDIR@")
 config.llvm_plugin_ext = "@LLVM_PLUGIN_EXT@"
-config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@"
-config.clang_obj_root = "@CLANG_BINARY_DIR@"
-config.clang_src_dir = "@CLANG_SOURCE_DIR@"
-config.clang_tools_dir = "@CLANG_TOOLS_DIR@"
+config.lit_tools_dir = path("@LLVM_LIT_TOOLS_DIR@")
+config.clang_obj_root = path("@CLANG_BINARY_DIR@")
+config.clang_src_dir = path("@CLANG_SOURCE_DIR@")
+config.clang_tools_dir = path("@CLANG_TOOLS_DIR@")
 config.host_triple = "@LLVM_HOST_TRIPLE@"
 config.target_triple = "@TARGET_TRIPLE@"
 config.host_cxx = "@CMAKE_CXX_COMPILER@"
@@ -50,4 +50,5 @@ if not "@CLANG_DEFAULT_LINKER@":
     config.available_features.add('platform-linker')
 
 # Let the main config do the real work.
-lit_config.load_config(config, "@CLANG_SOURCE_DIR@/test/lit.cfg.py")
+lit_config.load_config(
+    config, os.path.join(config.clang_src_dir, "test/lit.cfg.py"))
diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
index 9105c616786fb..91659e76219ab 100644
--- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
+++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
@@ -218,16 +218,15 @@ static llvm::json::Array toJSONSorted(const llvm::StringSet<> &Set) {
   std::vector<llvm::StringRef> Strings;
   for (auto &&I : Set)
     Strings.push_back(I.getKey());
-  std::sort(Strings.begin(), Strings.end());
+  llvm::sort(Strings);
   return llvm::json::Array(Strings);
 }
 
 static llvm::json::Array toJSONSorted(std::vector<ClangModuleDep> V) {
-  std::sort(V.begin(), V.end(),
-            [](const ClangModuleDep &A, const ClangModuleDep &B) {
-              return std::tie(A.ModuleName, A.ContextHash) <
-                     std::tie(B.ModuleName, B.ContextHash);
-            });
+  llvm::sort(V, [](const ClangModuleDep &A, const ClangModuleDep &B) {
+    return std::tie(A.ModuleName, A.ContextHash) <
+           std::tie(B.ModuleName, B.ContextHash);
+  });
 
   llvm::json::Array Ret;
   for (const ClangModuleDep &CMD : V)
@@ -275,16 +274,15 @@ class FullDeps {
     std::vector<ContextModulePair> ModuleNames;
     for (auto &&M : Modules)
       ModuleNames.push_back(M.first);
-    std::sort(ModuleNames.begin(), ModuleNames.end(),
-              [](const ContextModulePair &A, const ContextModulePair &B) {
-                return std::tie(A.ModuleName, A.InputIndex) <
-                       std::tie(B.ModuleName, B.InputIndex);
-              });
-
-    std::sort(Inputs.begin(), Inputs.end(),
-              [](const InputDeps &A, const InputDeps &B) {
-                return A.FileName < B.FileName;
-              });
+    llvm::sort(ModuleNames,
+               [](const ContextModulePair &A, const ContextModulePair &B) {
+                 return std::tie(A.ModuleName, A.InputIndex) <
+                        std::tie(B.ModuleName, B.InputIndex);
+               });
+
+    llvm::sort(Inputs, [](const InputDeps &A, const InputDeps &B) {
+      return A.FileName < B.FileName;
+    });
 
     using namespace llvm::json;
 
diff --git a/clang/tools/driver/cc1_main.cpp b/clang/tools/driver/cc1_main.cpp
index d8c22f21706b3..33477852d2854 100644
--- a/clang/tools/driver/cc1_main.cpp
+++ b/clang/tools/driver/cc1_main.cpp
@@ -177,7 +177,7 @@ static int PrintSupportedCPUs(std::string TargetStr) {
   // the target machine will handle the mcpu printing
   llvm::TargetOptions Options;
   std::unique_ptr<llvm::TargetMachine> TheTargetMachine(
-      TheTarget->createTargetMachine(TargetStr, "", "+cpuHelp", Options, None));
+      TheTarget->createTargetMachine(TargetStr, "", "+cpuhelp", Options, None));
   return 0;
 }
 
diff --git a/clang/tools/driver/driver.cpp b/clang/tools/driver/driver.cpp
index c59752178b66a..f037a821f9d7d 100644
--- a/clang/tools/driver/driver.cpp
+++ b/clang/tools/driver/driver.cpp
@@ -38,6 +38,7 @@
 #include "llvm/Support/Host.h"
 #include "llvm/Support/InitLLVM.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/PrettyStackTrace.h"
 #include "llvm/Support/Process.h"
 #include "llvm/Support/Program.h"
 #include "llvm/Support/Regex.h"
@@ -342,6 +343,9 @@ static int ExecuteCC1Tool(SmallVectorImpl<const char *> &ArgV) {
 int main(int argc_, const char **argv_) {
   noteBottomOfStack();
   llvm::InitLLVM X(argc_, argv_);
+  llvm::setBugReportMsg("PLEASE submit a bug report to " BUG_REPORT_URL
+                        " and include the crash backtrace, preprocessed "
+                        "source, and associated run script.\n");
   SmallVector<const char *, 256> argv(argv_, argv_ + argc_);
 
   if (llvm::sys::Process::FixupStandardFileDescriptors())
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index ae7fd4271799f..14814f89d97d7 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -105,8 +105,7 @@ cxtu::CXTUOwner::~CXTUOwner() {
 
 /// Compare two source ranges to determine their relative position in
 /// the translation unit.
-static RangeComparisonResult RangeCompare(SourceManager &SM,
-                                          SourceRange R1,
+static RangeComparisonResult RangeCompare(SourceManager &SM, SourceRange R1,
                                           SourceRange R2) {
   assert(R1.isValid() && "First range is invalid?");
   assert(R2.isValid() && "Second range is invalid?");
@@ -147,22 +146,20 @@ CXSourceRange cxloc::translateSourceRange(const SourceManager &SM,
   // location accordingly.
   SourceLocation EndLoc = R.getEnd();
   bool IsTokenRange = R.isTokenRange();
-  if (EndLoc.isValid() && EndLoc.isMacroID() && !SM.isMacroArgExpansion(EndLoc)) {
+  if (EndLoc.isValid() && EndLoc.isMacroID() &&
+      !SM.isMacroArgExpansion(EndLoc)) {
     CharSourceRange Expansion = SM.getExpansionRange(EndLoc);
     EndLoc = Expansion.getEnd();
     IsTokenRange = Expansion.isTokenRange();
   }
   if (IsTokenRange && EndLoc.isValid()) {
-    unsigned Length = Lexer::MeasureTokenLength(SM.getSpellingLoc(EndLoc),
-                                                SM, LangOpts);
+    unsigned Length =
+        Lexer::MeasureTokenLength(SM.getSpellingLoc(EndLoc), SM, LangOpts);
     EndLoc = EndLoc.getLocWithOffset(Length);
   }
 
   CXSourceRange Result = {
-    { &SM, &LangOpts },
-    R.getBegin().getRawEncoding(),
-    EndLoc.getRawEncoding()
-  };
+      {&SM, &LangOpts}, R.getBegin().getRawEncoding(), EndLoc.getRawEncoding()};
   return Result;
 }
 
@@ -197,7 +194,7 @@ bool CursorVisitor::Visit(CXCursor Cursor, bool CheckedRegionOfInterest) {
       assert(0 && "Invalid declaration cursor");
       return true; // abort.
     }
-    
+
     // Ignore implicit declarations, unless it's an objc method because
     // currently we should report implicit methods for properties when indexing.
     if (D->isImplicit() && !isa<ObjCMethodDecl>(D))
@@ -236,10 +233,11 @@ static bool visitPreprocessedEntitiesInRange(SourceRange R,
                                              CursorVisitor &Visitor) {
   SourceManager &SM = Visitor.getASTUnit()->getSourceManager();
   FileID FID;
-  
+
   if (!Visitor.shouldVisitIncludedEntities()) {
     // If the begin/end of the range lie in the same FileID, do the optimization
-    // where we skip preprocessed entities that do not come from the same FileID.
+    // where we skip preprocessed entities that do not come from the same
+    // FileID.
     FID = SM.getFileID(SM.getFileLoc(R.getBegin()));
     if (FID != SM.getFileID(SM.getFileLoc(R.getEnd())))
       FID = FileID();
@@ -256,10 +254,11 @@ bool CursorVisitor::visitFileRegion() {
 
   ASTUnit *Unit = cxtu::getASTUnit(TU);
   SourceManager &SM = Unit->getSourceManager();
-  
-  std::pair<FileID, unsigned>
-    Begin = SM.getDecomposedLoc(SM.getFileLoc(RegionOfInterest.getBegin())), 
-    End = SM.getDecomposedLoc(SM.getFileLoc(RegionOfInterest.getEnd())); 
+
+  std::pair<FileID, unsigned> Begin = SM.getDecomposedLoc(
+                                  SM.getFileLoc(RegionOfInterest.getBegin())),
+                              End = SM.getDecomposedLoc(
+                                  SM.getFileLoc(RegionOfInterest.getEnd()));
 
   if (End.first != Begin.first) {
     // If the end does not reside in the same file, try to recover by
@@ -271,7 +270,7 @@ bool CursorVisitor::visitFileRegion() {
   assert(Begin.first == End.first);
   if (Begin.second > End.second)
     return false;
-  
+
   FileID File = Begin.first;
   unsigned Offset = Begin.second;
   unsigned Length = End.second - Begin.second;
@@ -293,16 +292,16 @@ static bool isInLexicalContext(Decl *D, DeclContext *DC) {
   if (!DC)
     return false;
 
-  for (DeclContext *DeclDC = D->getLexicalDeclContext();
-         DeclDC; DeclDC = DeclDC->getLexicalParent()) {
+  for (DeclContext *DeclDC = D->getLexicalDeclContext(); DeclDC;
+       DeclDC = DeclDC->getLexicalParent()) {
     if (DeclDC == DC)
       return true;
   }
   return false;
 }
 
-bool CursorVisitor::visitDeclsFromFileRegion(FileID File,
-                                             unsigned Offset, unsigned Length) {
+bool CursorVisitor::visitDeclsFromFileRegion(FileID File, unsigned Offset,
+                                             unsigned Length) {
   ASTUnit *Unit = cxtu::getASTUnit(TU);
   SourceManager &SM = Unit->getSourceManager();
   SourceRange Range = RegionOfInterest;
@@ -350,7 +349,8 @@ bool CursorVisitor::visitDeclsFromFileRegion(FileID File,
       if (!TD->isFreeStanding())
         continue;
 
-    RangeComparisonResult CompRes = RangeCompare(SM, D->getSourceRange(),Range);
+    RangeComparisonResult CompRes =
+        RangeCompare(SM, D->getSourceRange(), Range);
     if (CompRes == RangeBefore)
       continue;
     if (CompRes == RangeAfter)
@@ -376,8 +376,9 @@ bool CursorVisitor::visitDeclsFromFileRegion(FileID File,
   // No Decls overlapped with the range. Move up the lexical context until there
   // is a context that contains the range or we reach the translation unit
   // level.
-  DeclContext *DC = DIt == Decls.begin() ? (*DIt)->getLexicalDeclContext()
-                                         : (*(DIt-1))->getLexicalDeclContext();
+  DeclContext *DC = DIt == Decls.begin()
+                        ? (*DIt)->getLexicalDeclContext()
+                        : (*(DIt - 1))->getLexicalDeclContext();
 
   while (DC && !DC->isTranslationUnit()) {
     Decl *D = cast<Decl>(DC);
@@ -400,10 +401,9 @@ bool CursorVisitor::visitPreprocessedEntitiesInRegion() {
   if (!AU->getPreprocessor().getPreprocessingRecord())
     return false;
 
-  PreprocessingRecord &PPRec
-    = *AU->getPreprocessor().getPreprocessingRecord();
+  PreprocessingRecord &PPRec = *AU->getPreprocessor().getPreprocessingRecord();
   SourceManager &SM = AU->getSourceManager();
-  
+
   if (RegionOfInterest.isValid()) {
     SourceRange MappedRange = AU->mapRangeToPreamble(RegionOfInterest);
     SourceLocation B = MappedRange.getBegin();
@@ -411,8 +411,8 @@ bool CursorVisitor::visitPreprocessedEntitiesInRegion() {
 
     if (AU->isInPreambleFileID(B)) {
       if (SM.isLoadedSourceLocation(E))
-        return visitPreprocessedEntitiesInRange(SourceRange(B, E),
-                                                 PPRec, *this);
+        return visitPreprocessedEntitiesInRange(SourceRange(B, E), PPRec,
+                                                *this);
 
       // Beginning of range lies in the preamble but it also extends beyond
       // it into the main file. Split the range into 2 parts, one covering
@@ -420,22 +420,19 @@ bool CursorVisitor::visitPreprocessedEntitiesInRegion() {
       // calls to visitPreprocessedEntitiesInRange to accept a source range that
       // lies in the same FileID, allowing it to skip preprocessed entities that
       // do not come from the same FileID.
-      bool breaked =
-        visitPreprocessedEntitiesInRange(
-                                   SourceRange(B, AU->getEndOfPreambleFileID()),
-                                          PPRec, *this);
-      if (breaked) return true;
+      bool breaked = visitPreprocessedEntitiesInRange(
+          SourceRange(B, AU->getEndOfPreambleFileID()), PPRec, *this);
+      if (breaked)
+        return true;
       return visitPreprocessedEntitiesInRange(
-                                    SourceRange(AU->getStartOfMainFileID(), E),
-                                        PPRec, *this);
+          SourceRange(AU->getStartOfMainFileID(), E), PPRec, *this);
     }
 
     return visitPreprocessedEntitiesInRange(SourceRange(B, E), PPRec, *this);
   }
 
-  bool OnlyLocalDecls
-    = !AU->isMainFileAST() && AU->getOnlyLocalDecls(); 
-  
+  bool OnlyLocalDecls = !AU->isMainFileAST() && AU->getOnlyLocalDecls();
+
   if (OnlyLocalDecls)
     return visitPreprocessedEntities(PPRec.local_begin(), PPRec.local_end(),
                                      PPRec);
@@ -443,7 +440,7 @@ bool CursorVisitor::visitPreprocessedEntitiesInRegion() {
   return visitPreprocessedEntities(PPRec.begin(), PPRec.end(), PPRec);
 }
 
-template<typename InputIterator>
+template <typename InputIterator>
 bool CursorVisitor::visitPreprocessedEntities(InputIterator First,
                                               InputIterator Last,
                                               PreprocessingRecord &PPRec,
@@ -469,11 +466,11 @@ bool CursorVisitor::visitPreprocessedEntities(InputIterator First,
 
       continue;
     }
-    
+
     if (InclusionDirective *ID = dyn_cast<InclusionDirective>(PPE)) {
       if (Visit(MakeInclusionDirectiveCursor(ID, TU)))
         return true;
-      
+
       continue;
     }
   }
@@ -482,11 +479,11 @@ bool CursorVisitor::visitPreprocessedEntities(InputIterator First,
 }
 
 /// Visit the children of the given cursor.
-/// 
+///
 /// \returns true if the visitation should be aborted, false if it
 /// should continue.
 bool CursorVisitor::VisitChildren(CXCursor Cursor) {
-  if (clang_isReference(Cursor.kind) && 
+  if (clang_isReference(Cursor.kind) &&
       Cursor.kind != CXCursor_CXXBaseSpecifier) {
     // By definition, references have no children.
     return false;
@@ -521,14 +518,14 @@ bool CursorVisitor::VisitChildren(CXCursor Cursor) {
   if (clang_isTranslationUnit(Cursor.kind)) {
     CXTranslationUnit TU = getCursorTU(Cursor);
     ASTUnit *CXXUnit = cxtu::getASTUnit(TU);
-    
-    int VisitOrder[2] = { VisitPreprocessorLast, !VisitPreprocessorLast };
+
+    int VisitOrder[2] = {VisitPreprocessorLast, !VisitPreprocessorLast};
     for (unsigned I = 0; I != 2; ++I) {
       if (VisitOrder[I]) {
         if (!CXXUnit->isMainFileAST() && CXXUnit->getOnlyLocalDecls() &&
             RegionOfInterest.isInvalid()) {
           for (ASTUnit::top_level_iterator TL = CXXUnit->top_level_begin(),
-                                        TLEnd = CXXUnit->top_level_end();
+                                           TLEnd = CXXUnit->top_level_end();
                TL != TLEnd; ++TL) {
             const Optional<bool> V = handleDeclForVisitation(*TL);
             if (!V.hasValue())
@@ -536,7 +533,7 @@ bool CursorVisitor::VisitChildren(CXCursor Cursor) {
             return V.getValue();
           }
         } else if (VisitDeclContext(
-                                CXXUnit->getASTContext().getTranslationUnitDecl()))
+                       CXXUnit->getASTContext().getTranslationUnitDecl()))
           return true;
         continue;
       }
@@ -545,7 +542,7 @@ bool CursorVisitor::VisitChildren(CXCursor Cursor) {
       if (CXXUnit->getPreprocessor().getPreprocessingRecord())
         visitPreprocessedEntitiesInRegion();
     }
-    
+
     return false;
   }
 
@@ -559,7 +556,7 @@ bool CursorVisitor::VisitChildren(CXCursor Cursor) {
 
   if (Cursor.kind == CXCursor_IBOutletCollectionAttr) {
     const IBOutletCollectionAttr *A =
-      cast<IBOutletCollectionAttr>(cxcursor::getCursorAttr(Cursor));
+        cast<IBOutletCollectionAttr>(cxcursor::getCursorAttr(Cursor));
     if (const ObjCObjectType *ObjT = A->getInterface()->getAs<ObjCObjectType>())
       return Visit(cxcursor::MakeCursorObjCClassRef(
           ObjT->getInterface(),
@@ -587,7 +584,7 @@ bool CursorVisitor::VisitChildren(CXCursor Cursor) {
 bool CursorVisitor::VisitBlockDecl(BlockDecl *B) {
   if (TypeSourceInfo *TSInfo = B->getSignatureAsWritten())
     if (Visit(TSInfo->getTypeLoc()))
-        return true;
+      return true;
 
   if (Stmt *Body = B->getBody())
     return Visit(MakeCXCursor(Body, StmtParent, TU, RegionOfInterest));
@@ -600,7 +597,7 @@ Optional<bool> CursorVisitor::shouldVisitCursor(CXCursor Cursor) {
     SourceRange Range = getFullCursorExtent(Cursor, AU->getSourceManager());
     if (Range.isInvalid())
       return None;
-    
+
     switch (CompareRegionOfInterest(Range)) {
     case RangeBefore:
       // This declaration comes before the region of interest; skip it.
@@ -623,10 +620,10 @@ bool CursorVisitor::VisitDeclContext(DeclContext *DC) {
 
   // FIXME: Eventually remove.  This part of a hack to support proper
   // iteration over all Decls contained lexically within an ObjC container.
-  SaveAndRestore<DeclContext::decl_iterator*> DI_saved(DI_current, &I);
+  SaveAndRestore<DeclContext::decl_iterator *> DI_saved(DI_current, &I);
   SaveAndRestore<DeclContext::decl_iterator> DE_saved(DE_current, E);
 
-  for ( ; I != E; ++I) {
+  for (; I != E; ++I) {
     Decl *D = *I;
     if (D->getLexicalDeclContext() != DC)
       continue;
@@ -684,10 +681,10 @@ bool CursorVisitor::VisitTranslationUnitDecl(TranslationUnitDecl *D) {
 }
 
 bool CursorVisitor::VisitTypeAliasTemplateDecl(TypeAliasTemplateDecl *D) {
-    if (VisitTemplateParameters(D->getTemplateParameters()))
-        return true;
+  if (VisitTemplateParameters(D->getTemplateParameters()))
+    return true;
 
-    return Visit(MakeCXCursor(D->getTemplatedDecl(), TU, RegionOfInterest));
+  return Visit(MakeCXCursor(D->getTemplatedDecl(), TU, RegionOfInterest));
 }
 
 bool CursorVisitor::VisitTypeAliasDecl(TypeAliasDecl *D) {
@@ -704,28 +701,26 @@ bool CursorVisitor::VisitTypedefDecl(TypedefDecl *D) {
   return false;
 }
 
-bool CursorVisitor::VisitTagDecl(TagDecl *D) {
-  return VisitDeclContext(D);
-}
+bool CursorVisitor::VisitTagDecl(TagDecl *D) { return VisitDeclContext(D); }
 
 bool CursorVisitor::VisitClassTemplateSpecializationDecl(
-                                          ClassTemplateSpecializationDecl *D) {
+    ClassTemplateSpecializationDecl *D) {
   bool ShouldVisitBody = false;
   switch (D->getSpecializationKind()) {
   case TSK_Undeclared:
   case TSK_ImplicitInstantiation:
     // Nothing to visit
     return false;
-      
+
   case TSK_ExplicitInstantiationDeclaration:
   case TSK_ExplicitInstantiationDefinition:
     break;
-      
+
   case TSK_ExplicitSpecialization:
     ShouldVisitBody = true;
     break;
   }
-  
+
   // Visit the template arguments used in the specialization.
   if (TypeSourceInfo *SpecType = D->getTypeAsWritten()) {
     TypeLoc TL = SpecType->getTypeLoc();
@@ -741,7 +736,7 @@ bool CursorVisitor::VisitClassTemplateSpecializationDecl(
 }
 
 bool CursorVisitor::VisitClassTemplatePartialSpecializationDecl(
-                                   ClassTemplatePartialSpecializationDecl *D) {
+    ClassTemplatePartialSpecializationDecl *D) {
   // FIXME: Visit the "outer" template parameter lists on the TagDecl
   // before visiting these template parameters.
   if (VisitTemplateParameters(D->getTemplateParameters()))
@@ -753,7 +748,7 @@ bool CursorVisitor::VisitClassTemplatePartialSpecializationDecl(
   for (unsigned I = 0, N = Info->NumTemplateArgs; I != N; ++I)
     if (VisitTemplateArgumentLoc(TemplateArgs[I]))
       return true;
-  
+
   return VisitCXXRecordDecl(D);
 }
 
@@ -768,7 +763,7 @@ bool CursorVisitor::VisitTemplateTypeParmDecl(TemplateTypeParmDecl *D) {
     if (TypeSourceInfo *DefArg = D->getDefaultArgumentInfo())
       if (Visit(DefArg->getTypeLoc()))
         return true;
-  
+
   return false;
 }
 
@@ -781,7 +776,7 @@ bool CursorVisitor::VisitEnumConstantDecl(EnumConstantDecl *D) {
 bool CursorVisitor::VisitDeclaratorDecl(DeclaratorDecl *DD) {
   unsigned NumParamList = DD->getNumTemplateParameterLists();
   for (unsigned i = 0; i < NumParamList; i++) {
-    TemplateParameterList* Params = DD->getTemplateParameterList(i);
+    TemplateParameterList *Params = DD->getTemplateParameterList(i);
     if (VisitTemplateParameters(Params))
       return true;
   }
@@ -817,7 +812,7 @@ static int CompareCXXCtorInitializers(CXXCtorInitializer *const *X,
 bool CursorVisitor::VisitFunctionDecl(FunctionDecl *ND) {
   unsigned NumParamList = ND->getNumTemplateParameterLists();
   for (unsigned i = 0; i < NumParamList; i++) {
-    TemplateParameterList* Params = ND->getTemplateParameterList(i);
+    TemplateParameterList *Params = ND->getTemplateParameterList(i);
     if (VisitTemplateParameters(Params))
       return true;
   }
@@ -828,7 +823,7 @@ bool CursorVisitor::VisitFunctionDecl(FunctionDecl *ND) {
     TypeLoc TL = TSInfo->getTypeLoc().IgnoreParens();
     FunctionTypeLoc FTL = TL.getAs<FunctionTypeLoc>();
     const bool HasTrailingRT = HasTrailingReturnType(ND);
-    
+
     // If we have a function declared directly (without the use of a typedef),
     // visit just the return type. Otherwise, just visit the function's type
     // now.
@@ -841,14 +836,14 @@ bool CursorVisitor::VisitFunctionDecl(FunctionDecl *ND) {
     if (NestedNameSpecifierLoc QualifierLoc = ND->getQualifierLoc())
       if (VisitNestedNameSpecifierLoc(QualifierLoc))
         return true;
-    
+
     // Visit the declaration name.
     if (!isa<CXXDestructorDecl>(ND))
       if (VisitDeclarationNameInfo(ND->getNameInfo()))
         return true;
-    
+
     // FIXME: Visit explicitly-specified template arguments!
-    
+
     // Visit the function parameters, if we have a function type.
     if (FTL && VisitFunctionTypeLoc(FTL, true))
       return true;
@@ -859,7 +854,7 @@ bool CursorVisitor::VisitFunctionDecl(FunctionDecl *ND) {
 
     // FIXME: Attributes?
   }
-  
+
   if (ND->doesThisDeclarationHaveABody() && !ND->isLateTemplateParsed()) {
     if (CXXConstructorDecl *Constructor = dyn_cast<CXXConstructorDecl>(ND)) {
       // Find the initializers that were written in the source.
@@ -867,14 +862,14 @@ bool CursorVisitor::VisitFunctionDecl(FunctionDecl *ND) {
       for (auto *I : Constructor->inits()) {
         if (!I->isWritten())
           continue;
-      
+
         WrittenInits.push_back(I);
       }
-      
+
       // Sort the initializers in source order
       llvm::array_pod_sort(WrittenInits.begin(), WrittenInits.end(),
                            &CompareCXXCtorInitializers);
-      
+
       // Visit the initializers in source order
       for (unsigned I = 0, N = WrittenInits.size(); I != N; ++I) {
         CXXCtorInitializer *Init = WrittenInits[I];
@@ -886,14 +881,14 @@ bool CursorVisitor::VisitFunctionDecl(FunctionDecl *ND) {
           if (Visit(TInfo->getTypeLoc()))
             return true;
         }
-        
+
         // Visit the initializer value.
         if (Expr *Initializer = Init->getInit())
           if (Visit(MakeCXCursor(Initializer, ND, TU, RegionOfInterest)))
             return true;
-      } 
+      }
     }
-    
+
     if (Visit(MakeCXCursor(ND->getBody(), StmtParent, TU, RegionOfInterest)))
       return true;
   }
@@ -927,12 +922,12 @@ bool CursorVisitor::VisitVarDecl(VarDecl *D) {
 bool CursorVisitor::VisitNonTypeTemplateParmDecl(NonTypeTemplateParmDecl *D) {
   if (VisitDeclaratorDecl(D))
     return true;
-  
+
   if (D->hasDefaultArgument() && !D->defaultArgumentWasInherited())
     if (Expr *DefArg = D->getDefaultArgument())
       return Visit(MakeCXCursor(DefArg, StmtParent, TU, RegionOfInterest));
-  
-  return false;  
+
+  return false;
 }
 
 bool CursorVisitor::VisitFunctionTemplateDecl(FunctionTemplateDecl *D) {
@@ -940,8 +935,8 @@ bool CursorVisitor::VisitFunctionTemplateDecl(FunctionTemplateDecl *D) {
   // before visiting these template parameters.
   if (VisitTemplateParameters(D->getTemplateParameters()))
     return true;
-  
-  auto* FD = D->getTemplatedDecl();
+
+  auto *FD = D->getTemplatedDecl();
   return VisitAttributes(FD) || VisitFunctionDecl(FD);
 }
 
@@ -950,19 +945,19 @@ bool CursorVisitor::VisitClassTemplateDecl(ClassTemplateDecl *D) {
   // before visiting these template parameters.
   if (VisitTemplateParameters(D->getTemplateParameters()))
     return true;
-  
-  auto* CD = D->getTemplatedDecl();
+
+  auto *CD = D->getTemplatedDecl();
   return VisitAttributes(CD) || VisitCXXRecordDecl(CD);
 }
 
 bool CursorVisitor::VisitTemplateTemplateParmDecl(TemplateTemplateParmDecl *D) {
   if (VisitTemplateParameters(D->getTemplateParameters()))
     return true;
-  
+
   if (D->hasDefaultArgument() && !D->defaultArgumentWasInherited() &&
       VisitTemplateArgumentLoc(D->getDefaultArgument()))
     return true;
-  
+
   return false;
 }
 
@@ -1052,18 +1047,18 @@ bool CursorVisitor::VisitObjCContainerDecl(ObjCContainerDecl *D) {
   }
 
   // Now sort the Decls so that they appear in lexical order.
-  llvm::sort(DeclsInContainer,
-             [&SM](Decl *A, Decl *B) {
-               SourceLocation L_A = A->getBeginLoc();
-               SourceLocation L_B = B->getBeginLoc();
-               return L_A != L_B ? SM.isBeforeInTranslationUnit(L_A, L_B)
-                                 : SM.isBeforeInTranslationUnit(A->getEndLoc(),
-                                                                B->getEndLoc());
-             });
+  llvm::sort(DeclsInContainer, [&SM](Decl *A, Decl *B) {
+    SourceLocation L_A = A->getBeginLoc();
+    SourceLocation L_B = B->getBeginLoc();
+    return L_A != L_B
+               ? SM.isBeforeInTranslationUnit(L_A, L_B)
+               : SM.isBeforeInTranslationUnit(A->getEndLoc(), B->getEndLoc());
+  });
 
   // Now visit the decls.
-  for (SmallVectorImpl<Decl*>::iterator I = DeclsInContainer.begin(),
-         E = DeclsInContainer.end(); I != E; ++I) {
+  for (SmallVectorImpl<Decl *>::iterator I = DeclsInContainer.begin(),
+                                         E = DeclsInContainer.end();
+       I != E; ++I) {
     CXCursor Cursor = MakeCXCursor(*I, TU, RegionOfInterest);
     const Optional<bool> &V = shouldVisitCursor(Cursor);
     if (!V.hasValue())
@@ -1086,7 +1081,8 @@ bool CursorVisitor::VisitObjCCategoryDecl(ObjCCategoryDecl *ND) {
 
   ObjCCategoryDecl::protocol_loc_iterator PL = ND->protocol_loc_begin();
   for (ObjCCategoryDecl::protocol_iterator I = ND->protocol_begin(),
-         E = ND->protocol_end(); I != E; ++I, ++PL)
+                                           E = ND->protocol_end();
+       I != E; ++I, ++PL)
     if (Visit(MakeCursorObjCProtocolRef(*I, *PL, TU)))
       return true;
 
@@ -1096,10 +1092,11 @@ bool CursorVisitor::VisitObjCCategoryDecl(ObjCCategoryDecl *ND) {
 bool CursorVisitor::VisitObjCProtocolDecl(ObjCProtocolDecl *PID) {
   if (!PID->isThisDeclarationADefinition())
     return Visit(MakeCursorObjCProtocolRef(PID, PID->getLocation(), TU));
-  
+
   ObjCProtocolDecl::protocol_loc_iterator PL = PID->protocol_loc_begin();
   for (ObjCProtocolDecl::protocol_iterator I = PID->protocol_begin(),
-       E = PID->protocol_end(); I != E; ++I, ++PL)
+                                           E = PID->protocol_end();
+       I != E; ++I, ++PL)
     if (Visit(MakeCursorObjCProtocolRef(*I, *PL, TU)))
       return true;
 
@@ -1122,9 +1119,8 @@ bool CursorVisitor::VisitObjCPropertyDecl(ObjCPropertyDecl *PD) {
     return false;
 
   IdentifierInfo *PropertyId = PD->getIdentifier();
-  ObjCPropertyDecl *prevDecl =
-    ObjCPropertyDecl::findPropertyDecl(cast<DeclContext>(ID), PropertyId,
-                                       PD->getQueryKind());
+  ObjCPropertyDecl *prevDecl = ObjCPropertyDecl::findPropertyDecl(
+      cast<DeclContext>(ID), PropertyId, PD->getQueryKind());
 
   if (!prevDecl)
     return false;
@@ -1168,10 +1164,8 @@ bool CursorVisitor::VisitObjCInterfaceDecl(ObjCInterfaceDecl *D) {
     return true;
 
   // Issue callbacks for super class.
-  if (D->getSuperClass() &&
-      Visit(MakeCursorObjCSuperClassRef(D->getSuperClass(),
-                                        D->getSuperClassLoc(),
-                                        TU)))
+  if (D->getSuperClass() && Visit(MakeCursorObjCSuperClassRef(
+                                D->getSuperClass(), D->getSuperClassLoc(), TU)))
     return true;
 
   if (TypeSourceInfo *SuperClassTInfo = D->getSuperClassTInfo())
@@ -1180,7 +1174,8 @@ bool CursorVisitor::VisitObjCInterfaceDecl(ObjCInterfaceDecl *D) {
 
   ObjCInterfaceDecl::protocol_loc_iterator PL = D->protocol_loc_begin();
   for (ObjCInterfaceDecl::protocol_iterator I = D->protocol_begin(),
-         E = D->protocol_end(); I != E; ++I, ++PL)
+                                            E = D->protocol_end();
+       I != E; ++I, ++PL)
     if (Visit(MakeCursorObjCProtocolRef(*I, *PL, TU)))
       return true;
 
@@ -1218,7 +1213,7 @@ bool CursorVisitor::VisitObjCPropertyImplDecl(ObjCPropertyImplDecl *PD) {
   if (ObjCIvarDecl *Ivar = PD->getPropertyIvarDecl())
     if (PD->isIvarNameSpecified())
       return Visit(MakeCursorMemberRef(Ivar, PD->getPropertyIvarDeclLoc(), TU));
-  
+
   return false;
 }
 
@@ -1231,8 +1226,8 @@ bool CursorVisitor::VisitNamespaceAliasDecl(NamespaceAliasDecl *D) {
   if (NestedNameSpecifierLoc QualifierLoc = D->getQualifierLoc())
     if (VisitNestedNameSpecifierLoc(QualifierLoc))
       return true;
-  
-  return Visit(MakeCursorNamespaceRef(D->getAliasedNamespace(), 
+
+  return Visit(MakeCursorNamespaceRef(D->getAliasedNamespace(),
                                       D->getTargetNameLoc(), TU));
 }
 
@@ -1242,10 +1237,10 @@ bool CursorVisitor::VisitUsingDecl(UsingDecl *D) {
     if (VisitNestedNameSpecifierLoc(QualifierLoc))
       return true;
   }
-  
+
   if (Visit(MakeCursorOverloadedDeclRef(D, D->getLocation(), TU)))
     return true;
-    
+
   return VisitDeclarationNameInfo(D->getNameInfo());
 }
 
@@ -1270,12 +1265,12 @@ bool CursorVisitor::VisitUnresolvedUsingValueDecl(UnresolvedUsingValueDecl *D) {
 }
 
 bool CursorVisitor::VisitUnresolvedUsingTypenameDecl(
-                                               UnresolvedUsingTypenameDecl *D) {
+    UnresolvedUsingTypenameDecl *D) {
   // Visit nested-name-specifier.
   if (NestedNameSpecifierLoc QualifierLoc = D->getQualifierLoc())
     if (VisitNestedNameSpecifierLoc(QualifierLoc))
       return true;
-  
+
   return false;
 }
 
@@ -1325,7 +1320,7 @@ bool CursorVisitor::VisitDeclarationNameInfo(DeclarationNameInfo Name) {
   llvm_unreachable("Invalid DeclarationName::Kind!");
 }
 
-bool CursorVisitor::VisitNestedNameSpecifier(NestedNameSpecifier *NNS, 
+bool CursorVisitor::VisitNestedNameSpecifier(NestedNameSpecifier *NNS,
                                              SourceRange Range) {
   // FIXME: This whole routine is a hack to work around the lack of proper
   // source information in nested-name-specifiers (PR5791). Since we do have
@@ -1333,18 +1328,18 @@ bool CursorVisitor::VisitNestedNameSpecifier(NestedNameSpecifier *NNS,
   // nested-name-specifier, if it's a single-token component.
   if (!NNS)
     return false;
-  
+
   // Get the first component in the nested-name-specifier.
   while (NestedNameSpecifier *Prefix = NNS->getPrefix())
     NNS = Prefix;
-  
+
   switch (NNS->getKind()) {
   case NestedNameSpecifier::Namespace:
-    return Visit(MakeCursorNamespaceRef(NNS->getAsNamespace(), Range.getBegin(),
-                                        TU));
+    return Visit(
+        MakeCursorNamespaceRef(NNS->getAsNamespace(), Range.getBegin(), TU));
 
   case NestedNameSpecifier::NamespaceAlias:
-    return Visit(MakeCursorNamespaceRef(NNS->getAsNamespaceAlias(), 
+    return Visit(MakeCursorNamespaceRef(NNS->getAsNamespaceAlias(),
                                         Range.getBegin(), TU));
 
   case NestedNameSpecifier::TypeSpec: {
@@ -1356,77 +1351,75 @@ bool CursorVisitor::VisitNestedNameSpecifier(NestedNameSpecifier *NNS,
       return Visit(MakeCursorTypeRef(Typedef->getDecl(), Range.getBegin(), TU));
     if (const TagType *Tag = dyn_cast<TagType>(T))
       return Visit(MakeCursorTypeRef(Tag->getDecl(), Range.getBegin(), TU));
-    if (const TemplateSpecializationType *TST
-                                      = dyn_cast<TemplateSpecializationType>(T))
+    if (const TemplateSpecializationType *TST =
+            dyn_cast<TemplateSpecializationType>(T))
       return VisitTemplateName(TST->getTemplateName(), Range.getBegin());
     break;
   }
-      
+
   case NestedNameSpecifier::TypeSpecWithTemplate:
   case NestedNameSpecifier::Global:
   case NestedNameSpecifier::Identifier:
   case NestedNameSpecifier::Super:
-    break;      
+    break;
   }
-  
+
   return false;
 }
 
-bool 
-CursorVisitor::VisitNestedNameSpecifierLoc(NestedNameSpecifierLoc Qualifier) {
+bool CursorVisitor::VisitNestedNameSpecifierLoc(
+    NestedNameSpecifierLoc Qualifier) {
   SmallVector<NestedNameSpecifierLoc, 4> Qualifiers;
   for (; Qualifier; Qualifier = Qualifier.getPrefix())
     Qualifiers.push_back(Qualifier);
-  
+
   while (!Qualifiers.empty()) {
     NestedNameSpecifierLoc Q = Qualifiers.pop_back_val();
     NestedNameSpecifier *NNS = Q.getNestedNameSpecifier();
     switch (NNS->getKind()) {
     case NestedNameSpecifier::Namespace:
-      if (Visit(MakeCursorNamespaceRef(NNS->getAsNamespace(), 
-                                       Q.getLocalBeginLoc(),
-                                       TU)))
+      if (Visit(MakeCursorNamespaceRef(NNS->getAsNamespace(),
+                                       Q.getLocalBeginLoc(), TU)))
         return true;
-        
+
       break;
-      
+
     case NestedNameSpecifier::NamespaceAlias:
-      if (Visit(MakeCursorNamespaceRef(NNS->getAsNamespaceAlias(), 
-                                       Q.getLocalBeginLoc(),
-                                       TU)))
+      if (Visit(MakeCursorNamespaceRef(NNS->getAsNamespaceAlias(),
+                                       Q.getLocalBeginLoc(), TU)))
         return true;
-        
+
       break;
-        
+
     case NestedNameSpecifier::TypeSpec:
     case NestedNameSpecifier::TypeSpecWithTemplate:
       if (Visit(Q.getTypeLoc()))
         return true;
-        
+
       break;
-        
+
     case NestedNameSpecifier::Global:
     case NestedNameSpecifier::Identifier:
     case NestedNameSpecifier::Super:
-      break;              
+      break;
     }
   }
-  
+
   return false;
 }
 
 bool CursorVisitor::VisitTemplateParameters(
-                                          const TemplateParameterList *Params) {
+    const TemplateParameterList *Params) {
   if (!Params)
     return false;
-  
+
   for (TemplateParameterList::const_iterator P = Params->begin(),
-                                          PEnd = Params->end();
+                                             PEnd = Params->end();
        P != PEnd; ++P) {
     if (Visit(MakeCXCursor(*P, TU, RegionOfInterest)))
       return true;
   }
-  
+
   return false;
 }
 
@@ -1449,22 +1442,20 @@ bool CursorVisitor::VisitTemplateName(TemplateName Name, SourceLocation Loc) {
   case TemplateName::DependentTemplate:
     // FIXME: Visit nested-name-specifier.
     return false;
-      
+
   case TemplateName::QualifiedTemplate:
     // FIXME: Visit nested-name-specifier.
     return Visit(MakeCursorTemplateRef(
-                                  Name.getAsQualifiedTemplateName()->getDecl(), 
-                                       Loc, TU));
+        Name.getAsQualifiedTemplateName()->getDecl(), Loc, TU));
 
   case TemplateName::SubstTemplateTemplateParm:
     return Visit(MakeCursorTemplateRef(
-                         Name.getAsSubstTemplateTemplateParm()->getParameter(),
-                                       Loc, TU));
-      
+        Name.getAsSubstTemplateTemplateParm()->getParameter(), Loc, TU));
+
   case TemplateName::SubstTemplateTemplateParmPack:
     return Visit(MakeCursorTemplateRef(
-                  Name.getAsSubstTemplateTemplateParmPack()->getParameterPack(),
-                                       Loc, TU));
+        Name.getAsSubstTemplateTemplateParmPack()->getParameterPack(), Loc,
+        TU));
   }
 
   llvm_unreachable("Invalid TemplateName::Kind!");
@@ -1476,12 +1467,12 @@ bool CursorVisitor::VisitTemplateArgumentLoc(const TemplateArgumentLoc &TAL) {
   case TemplateArgument::Integral:
   case TemplateArgument::Pack:
     return false;
-      
+
   case TemplateArgument::Type:
     if (TypeSourceInfo *TSInfo = TAL.getTypeSourceInfo())
       return Visit(TSInfo->getTypeLoc());
     return false;
-      
+
   case TemplateArgument::Declaration:
     if (Expr *E = TAL.getSourceDeclExpression())
       return Visit(MakeCXCursor(E, StmtParent, TU, RegionOfInterest));
@@ -1496,13 +1487,13 @@ bool CursorVisitor::VisitTemplateArgumentLoc(const TemplateArgumentLoc &TAL) {
     if (Expr *E = TAL.getSourceExpression())
       return Visit(MakeCXCursor(E, StmtParent, TU, RegionOfInterest));
     return false;
-  
+
   case TemplateArgument::Template:
   case TemplateArgument::TemplateExpansion:
     if (VisitNestedNameSpecifierLoc(TAL.getTemplateQualifierLoc()))
       return true;
-      
-    return VisitTemplateName(TAL.getArgument().getAsTemplateOrTemplatePattern(), 
+
+    return VisitTemplateName(TAL.getArgument().getAsTemplateOrTemplatePattern(),
                              TAL.getTemplateNameLoc());
   }
 
@@ -1528,19 +1519,17 @@ bool CursorVisitor::VisitBuiltinTypeLoc(BuiltinTypeLoc TL) {
   case BuiltinType::Void:
   case BuiltinType::NullPtr:
   case BuiltinType::Dependent:
-#define IMAGE_TYPE(ImgType, Id, SingletonId, Access, Suffix) \
+#define IMAGE_TYPE(ImgType, Id, SingletonId, Access, Suffix)                   \
   case BuiltinType::Id:
 #include "clang/Basic/OpenCLImageTypes.def"
-#define EXT_OPAQUE_TYPE(ExtTYpe, Id, Ext) \
-  case BuiltinType::Id:
+#define EXT_OPAQUE_TYPE(ExtTYpe, Id, Ext) case BuiltinType::Id:
 #include "clang/Basic/OpenCLExtensionTypes.def"
   case BuiltinType::OCLSampler:
   case BuiltinType::OCLEvent:
   case BuiltinType::OCLClkEvent:
   case BuiltinType::OCLQueue:
   case BuiltinType::OCLReserveID:
-#define SVE_TYPE(Name, Id, SingletonId) \
-  case BuiltinType::Id:
+#define SVE_TYPE(Name, Id, SingletonId) case BuiltinType::Id:
 #include "clang/Basic/AArch64SVEACLETypes.def"
 #define BUILTIN_TYPE(Id, SingletonId)
 #define SIGNED_TYPE(Id, SingletonId) case BuiltinType::Id:
@@ -1565,8 +1554,8 @@ bool CursorVisitor::VisitBuiltinTypeLoc(BuiltinTypeLoc TL) {
 
   if (!VisitType.isNull()) {
     if (const TypedefType *Typedef = VisitType->getAs<TypedefType>())
-      return Visit(MakeCursorTypeRef(Typedef->getDecl(), TL.getBuiltinLoc(),
-                                     TU));
+      return Visit(
+          MakeCursorTypeRef(Typedef->getDecl(), TL.getBuiltinLoc(), TU));
   }
 
   return false;
@@ -1661,7 +1650,7 @@ bool CursorVisitor::VisitAttributedTypeLoc(AttributedTypeLoc TL) {
   return Visit(TL.getModifiedLoc());
 }
 
-bool CursorVisitor::VisitFunctionTypeLoc(FunctionTypeLoc TL, 
+bool CursorVisitor::VisitFunctionTypeLoc(FunctionTypeLoc TL,
                                          bool SkipResultType) {
   if (!SkipResultType && Visit(TL.getReturnLoc()))
     return true;
@@ -1694,25 +1683,25 @@ bool CursorVisitor::VisitAdjustedTypeLoc(AdjustedTypeLoc TL) {
 
 bool CursorVisitor::VisitDeducedTemplateSpecializationTypeLoc(
     DeducedTemplateSpecializationTypeLoc TL) {
-  if (VisitTemplateName(TL.getTypePtr()->getTemplateName(), 
+  if (VisitTemplateName(TL.getTypePtr()->getTemplateName(),
                         TL.getTemplateNameLoc()))
     return true;
-  
+
   return false;
 }
 
 bool CursorVisitor::VisitTemplateSpecializationTypeLoc(
-                                             TemplateSpecializationTypeLoc TL) {
+    TemplateSpecializationTypeLoc TL) {
   // Visit the template name.
-  if (VisitTemplateName(TL.getTypePtr()->getTemplateName(), 
+  if (VisitTemplateName(TL.getTypePtr()->getTemplateName(),
                         TL.getTemplateNameLoc()))
     return true;
-  
+
   // Visit the template arguments.
   for (unsigned I = 0, N = TL.getNumArgs(); I != N; ++I)
     if (VisitTemplateArgumentLoc(TL.getArgLoc(I)))
       return true;
-  
+
   return false;
 }
 
@@ -1739,12 +1728,11 @@ bool CursorVisitor::VisitDependentNameTypeLoc(DependentNameTypeLoc TL) {
 }
 
 bool CursorVisitor::VisitDependentTemplateSpecializationTypeLoc(
-                                    DependentTemplateSpecializationTypeLoc TL) {
+    DependentTemplateSpecializationTypeLoc TL) {
   // Visit the nested-name-specifier, if there is one.
-  if (TL.getQualifierLoc() &&
-      VisitNestedNameSpecifierLoc(TL.getQualifierLoc()))
+  if (TL.getQualifierLoc() && VisitNestedNameSpecifierLoc(TL.getQualifierLoc()))
     return true;
-  
+
   // Visit the template arguments.
   for (unsigned I = 0, N = TL.getNumArgs(); I != N; ++I)
     if (VisitTemplateArgumentLoc(TL.getArgLoc(I)))
@@ -1756,7 +1744,7 @@ bool CursorVisitor::VisitDependentTemplateSpecializationTypeLoc(
 bool CursorVisitor::VisitElaboratedTypeLoc(ElaboratedTypeLoc TL) {
   if (VisitNestedNameSpecifierLoc(TL.getQualifierLoc()))
     return true;
-  
+
   return Visit(TL.getNamedTypeLoc());
 }
 
@@ -1783,10 +1771,10 @@ bool CursorVisitor::VisitPipeTypeLoc(PipeTypeLoc TL) {
   return Visit(TL.getValueLoc());
 }
 
-#define DEFAULT_TYPELOC_IMPL(CLASS, PARENT) \
-bool CursorVisitor::Visit##CLASS##TypeLoc(CLASS##TypeLoc TL) { \
-  return Visit##PARENT##Loc(TL); \
-}
+#define DEFAULT_TYPELOC_IMPL(CLASS, PARENT)                                    \
+  bool CursorVisitor::Visit##CLASS##TypeLoc(CLASS##TypeLoc TL) {               \
+    return Visit##PARENT##Loc(TL);                                             \
+  }
 
 DEFAULT_TYPELOC_IMPL(Complex, Type)
 DEFAULT_TYPELOC_IMPL(ConstantArray, ArrayType)
@@ -1827,7 +1815,7 @@ bool CursorVisitor::VisitAttributes(Decl *D) {
     if ((TU->ParsingOptions & CXTranslationUnit_VisitImplicitAttributes ||
          !I->isImplicit()) &&
         Visit(MakeCXCursor(I, D, TU)))
-        return true;
+      return true;
 
   return false;
 }
@@ -1837,14 +1825,16 @@ bool CursorVisitor::VisitAttributes(Decl *D) {
 //===----------------------------------------------------------------------===//
 
 namespace {
-#define DEF_JOB(NAME, DATA, KIND)\
-class NAME : public VisitorJob {\
-public:\
-  NAME(const DATA *d, CXCursor parent) : \
-      VisitorJob(parent, VisitorJob::KIND, d) {} \
-  static bool classof(const VisitorJob *VJ) { return VJ->getKind() == KIND; }\
-  const DATA *get() const { return static_cast<const DATA*>(data[0]); }\
-};
+#define DEF_JOB(NAME, DATA, KIND)                                              \
+  class NAME : public VisitorJob {                                             \
+  public:                                                                      \
+    NAME(const DATA *d, CXCursor parent)                                       \
+        : VisitorJob(parent, VisitorJob::KIND, d) {}                           \
+    static bool classof(const VisitorJob *VJ) {                                \
+      return VJ->getKind() == KIND;                                            \
+    }                                                                          \
+    const DATA *get() const { return static_cast<const DATA *>(data[0]); }     \
+  };
 
 DEF_JOB(StmtVisit, Stmt, StmtVisitKind)
 DEF_JOB(MemberExprParts, MemberExpr, MemberExprPartsKind)
@@ -1873,9 +1863,9 @@ class ExplicitTemplateArgsVisit : public VisitorJob {
 };
 class DeclVisit : public VisitorJob {
 public:
-  DeclVisit(const Decl *D, CXCursor parent, bool isFirst) :
-    VisitorJob(parent, VisitorJob::DeclVisitKind,
-               D, isFirst ? (void*) 1 : (void*) nullptr) {}
+  DeclVisit(const Decl *D, CXCursor parent, bool isFirst)
+      : VisitorJob(parent, VisitorJob::DeclVisitKind, D,
+                   isFirst ? (void *)1 : (void *)nullptr) {}
   static bool classof(const VisitorJob *VJ) {
     return VJ->getKind() == DeclVisitKind;
   }
@@ -1884,15 +1874,15 @@ class DeclVisit : public VisitorJob {
 };
 class TypeLocVisit : public VisitorJob {
 public:
-  TypeLocVisit(TypeLoc tl, CXCursor parent) :
-    VisitorJob(parent, VisitorJob::TypeLocVisitKind,
-               tl.getType().getAsOpaquePtr(), tl.getOpaqueData()) {}
+  TypeLocVisit(TypeLoc tl, CXCursor parent)
+      : VisitorJob(parent, VisitorJob::TypeLocVisitKind,
+                   tl.getType().getAsOpaquePtr(), tl.getOpaqueData()) {}
 
   static bool classof(const VisitorJob *VJ) {
     return VJ->getKind() == TypeLocVisitKind;
   }
 
-  TypeLoc get() const { 
+  TypeLoc get() const {
     QualType T = QualType::getFromOpaquePtr(data[0]);
     return TypeLoc(T, const_cast<void *>(data[1]));
   }
@@ -1901,42 +1891,43 @@ class TypeLocVisit : public VisitorJob {
 class LabelRefVisit : public VisitorJob {
 public:
   LabelRefVisit(LabelDecl *LD, SourceLocation labelLoc, CXCursor parent)
-    : VisitorJob(parent, VisitorJob::LabelRefVisitKind, LD,
-                 labelLoc.getPtrEncoding()) {}
-  
+      : VisitorJob(parent, VisitorJob::LabelRefVisitKind, LD,
+                   labelLoc.getPtrEncoding()) {}
+
   static bool classof(const VisitorJob *VJ) {
     return VJ->getKind() == VisitorJob::LabelRefVisitKind;
   }
   const LabelDecl *get() const {
     return static_cast<const LabelDecl *>(data[0]);
   }
-  SourceLocation getLoc() const { 
-    return SourceLocation::getFromPtrEncoding(data[1]); }
+  SourceLocation getLoc() const {
+    return SourceLocation::getFromPtrEncoding(data[1]);
+  }
 };
-  
+
 class NestedNameSpecifierLocVisit : public VisitorJob {
 public:
   NestedNameSpecifierLocVisit(NestedNameSpecifierLoc Qualifier, CXCursor parent)
-    : VisitorJob(parent, VisitorJob::NestedNameSpecifierLocVisitKind,
-                 Qualifier.getNestedNameSpecifier(),
-                 Qualifier.getOpaqueData()) { }
-  
+      : VisitorJob(parent, VisitorJob::NestedNameSpecifierLocVisitKind,
+                   Qualifier.getNestedNameSpecifier(),
+                   Qualifier.getOpaqueData()) {}
+
   static bool classof(const VisitorJob *VJ) {
     return VJ->getKind() == VisitorJob::NestedNameSpecifierLocVisitKind;
   }
-  
+
   NestedNameSpecifierLoc get() const {
     return NestedNameSpecifierLoc(
-            const_cast<NestedNameSpecifier *>(
-              static_cast<const NestedNameSpecifier *>(data[0])),
-            const_cast<void *>(data[1]));
+        const_cast<NestedNameSpecifier *>(
+            static_cast<const NestedNameSpecifier *>(data[0])),
+        const_cast<void *>(data[1]));
   }
 };
-  
+
 class DeclarationNameInfoVisit : public VisitorJob {
 public:
   DeclarationNameInfoVisit(const Stmt *S, CXCursor parent)
-    : VisitorJob(parent, VisitorJob::DeclarationNameInfoVisitKind, S) {}
+      : VisitorJob(parent, VisitorJob::DeclarationNameInfoVisitKind, S) {}
   static bool classof(const VisitorJob *VJ) {
     return VJ->getKind() == VisitorJob::DeclarationNameInfoVisitKind;
   }
@@ -1959,8 +1950,8 @@ class DeclarationNameInfoVisit : public VisitorJob {
 class MemberRefVisit : public VisitorJob {
 public:
   MemberRefVisit(const FieldDecl *D, SourceLocation L, CXCursor parent)
-    : VisitorJob(parent, VisitorJob::MemberRefVisitKind, D,
-                 L.getPtrEncoding()) {}
+      : VisitorJob(parent, VisitorJob::MemberRefVisitKind, D,
+                   L.getPtrEncoding()) {}
   static bool classof(const VisitorJob *VJ) {
     return VJ->getKind() == VisitorJob::MemberRefVisitKind;
   }
@@ -1968,22 +1959,24 @@ class MemberRefVisit : public VisitorJob {
     return static_cast<const FieldDecl *>(data[0]);
   }
   SourceLocation getLoc() const {
-    return SourceLocation::getFromRawEncoding((unsigned)(uintptr_t) data[1]);
+    return SourceLocation::getFromRawEncoding((unsigned)(uintptr_t)data[1]);
   }
 };
 class EnqueueVisitor : public ConstStmtVisitor<EnqueueVisitor, void> {
   friend class OMPClauseEnqueue;
   VisitorWorkList &WL;
   CXCursor Parent;
+
 public:
   EnqueueVisitor(VisitorWorkList &wl, CXCursor parent)
-    : WL(wl), Parent(parent) {}
+      : WL(wl), Parent(parent) {}
 
   void VisitAddrLabelExpr(const AddrLabelExpr *E);
   void VisitBlockExpr(const BlockExpr *B);
   void VisitCompoundLiteralExpr(const CompoundLiteralExpr *E);
   void VisitCompoundStmt(const CompoundStmt *S);
-  void VisitCXXDefaultArgExpr(const CXXDefaultArgExpr *E) { /* Do nothing. */ }
+  void VisitCXXDefaultArgExpr(const CXXDefaultArgExpr *E) { /* Do nothing. */
+  }
   void VisitMSDependentExistsStmt(const MSDependentExistsStmt *S);
   void VisitCXXDependentScopeMemberExpr(const CXXDependentScopeMemberExpr *E);
   void VisitCXXNewExpr(const CXXNewExpr *E);
@@ -2106,7 +2099,7 @@ class EnqueueVisitor : public ConstStmtVisitor<EnqueueVisitor, void> {
   void EnqueueChildren(const Stmt *S);
   void EnqueueChildren(const OMPClause *S);
 };
-} // end anonyous namespace
+} // namespace
 
 void EnqueueVisitor::AddDeclarationNameInfo(const Stmt *S) {
   // 'S' should always be non-null, since it comes from the
@@ -2114,8 +2107,8 @@ void EnqueueVisitor::AddDeclarationNameInfo(const Stmt *S) {
   WL.push_back(DeclarationNameInfoVisit(S, Parent));
 }
 
-void 
-EnqueueVisitor::AddNestedNameSpecifierLoc(NestedNameSpecifierLoc Qualifier) {
+void EnqueueVisitor::AddNestedNameSpecifierLoc(
+    NestedNameSpecifierLoc Qualifier) {
   if (Qualifier)
     WL.push_back(NestedNameSpecifierLocVisit(Qualifier, Parent));
 }
@@ -2139,7 +2132,7 @@ void EnqueueVisitor::AddMemberRef(const FieldDecl *D, SourceLocation L) {
 void EnqueueVisitor::AddTypeLoc(TypeSourceInfo *TI) {
   if (TI)
     WL.push_back(TypeLocVisit(TI->getTypeLoc(), Parent));
- }
+}
 void EnqueueVisitor::EnqueueChildren(const Stmt *S) {
   unsigned size = WL.size();
   for (const Stmt *SubStmt : S->children()) {
@@ -2156,12 +2149,11 @@ namespace {
 class OMPClauseEnqueue : public ConstOMPClauseVisitor<OMPClauseEnqueue> {
   EnqueueVisitor *Visitor;
   /// Process clauses with list of variables.
-  template <typename T>
-  void VisitOMPClauseList(T *Node);
+  template <typename T> void VisitOMPClauseList(T *Node);
+
 public:
-  OMPClauseEnqueue(EnqueueVisitor *Visitor) : Visitor(Visitor) { }
-#define OPENMP_CLAUSE(Name, Class)                                             \
-  void Visit##Class(const Class *C);
+  OMPClauseEnqueue(EnqueueVisitor *Visitor) : Visitor(Visitor) {}
+#define OPENMP_CLAUSE(Name, Class) void Visit##Class(const Class *C);
 #include "clang/Basic/OpenMPKinds.def"
   void VisitOMPClauseWithPreInit(const OMPClauseWithPreInit *C);
   void VisitOMPClauseWithPostUpdate(const OMPClauseWithPostUpdate *C);
@@ -2208,9 +2200,9 @@ void OMPClauseEnqueue::VisitOMPCollapseClause(const OMPCollapseClause *C) {
   Visitor->AddStmt(C->getNumForLoops());
 }
 
-void OMPClauseEnqueue::VisitOMPDefaultClause(const OMPDefaultClause *C) { }
+void OMPClauseEnqueue::VisitOMPDefaultClause(const OMPDefaultClause *C) {}
 
-void OMPClauseEnqueue::VisitOMPProcBindClause(const OMPProcBindClause *C) { }
+void OMPClauseEnqueue::VisitOMPProcBindClause(const OMPProcBindClause *C) {}
 
 void OMPClauseEnqueue::VisitOMPScheduleClause(const OMPScheduleClause *C) {
   VisitOMPClauseWithPreInit(C);
@@ -2281,7 +2273,8 @@ void OMPClauseEnqueue::VisitOMPNumTeamsClause(const OMPNumTeamsClause *C) {
   Visitor->AddStmt(C->getNumTeams());
 }
 
-void OMPClauseEnqueue::VisitOMPThreadLimitClause(const OMPThreadLimitClause *C) {
+void OMPClauseEnqueue::VisitOMPThreadLimitClause(
+    const OMPThreadLimitClause *C) {
   VisitOMPClauseWithPreInit(C);
   Visitor->AddStmt(C->getThreadLimit());
 }
@@ -2302,8 +2295,7 @@ void OMPClauseEnqueue::VisitOMPHintClause(const OMPHintClause *C) {
   Visitor->AddStmt(C->getHint());
 }
 
-template<typename T>
-void OMPClauseEnqueue::VisitOMPClauseList(T *Node) {
+template <typename T> void OMPClauseEnqueue::VisitOMPClauseList(T *Node) {
   for (const auto *I : Node->varlists()) {
     Visitor->AddStmt(I);
   }
@@ -2326,7 +2318,7 @@ void OMPClauseEnqueue::VisitOMPPrivateClause(const OMPPrivateClause *C) {
   }
 }
 void OMPClauseEnqueue::VisitOMPFirstprivateClause(
-                                        const OMPFirstprivateClause *C) {
+    const OMPFirstprivateClause *C) {
   VisitOMPClauseList(C);
   VisitOMPClauseWithPreInit(C);
   for (const auto *E : C->private_copies()) {
@@ -2337,7 +2329,7 @@ void OMPClauseEnqueue::VisitOMPFirstprivateClause(
   }
 }
 void OMPClauseEnqueue::VisitOMPLastprivateClause(
-                                        const OMPLastprivateClause *C) {
+    const OMPLastprivateClause *C) {
   VisitOMPClauseList(C);
   VisitOMPClauseWithPostUpdate(C);
   for (auto *E : C->private_copies()) {
@@ -2442,8 +2434,8 @@ void OMPClauseEnqueue::VisitOMPCopyinClause(const OMPCopyinClause *C) {
     Visitor->AddStmt(E);
   }
 }
-void
-OMPClauseEnqueue::VisitOMPCopyprivateClause(const OMPCopyprivateClause *C) {
+void OMPClauseEnqueue::VisitOMPCopyprivateClause(
+    const OMPCopyprivateClause *C) {
   VisitOMPClauseList(C);
   for (auto *E : C->source_exprs()) {
     Visitor->AddStmt(E);
@@ -2480,10 +2472,12 @@ void OMPClauseEnqueue::VisitOMPToClause(const OMPToClause *C) {
 void OMPClauseEnqueue::VisitOMPFromClause(const OMPFromClause *C) {
   VisitOMPClauseList(C);
 }
-void OMPClauseEnqueue::VisitOMPUseDevicePtrClause(const OMPUseDevicePtrClause *C) {
+void OMPClauseEnqueue::VisitOMPUseDevicePtrClause(
+    const OMPUseDevicePtrClause *C) {
   VisitOMPClauseList(C);
 }
-void OMPClauseEnqueue::VisitOMPIsDevicePtrClause(const OMPIsDevicePtrClause *C) {
+void OMPClauseEnqueue::VisitOMPIsDevicePtrClause(
+    const OMPIsDevicePtrClause *C) {
   VisitOMPClauseList(C);
 }
 void OMPClauseEnqueue::VisitOMPNontemporalClause(
@@ -2493,7 +2487,7 @@ void OMPClauseEnqueue::VisitOMPNontemporalClause(
     Visitor->AddStmt(E);
 }
 void OMPClauseEnqueue::VisitOMPOrderClause(const OMPOrderClause *C) {}
-}
+} // namespace
 
 void EnqueueVisitor::EnqueueChildren(const OMPClause *S) {
   unsigned size = WL.size();
@@ -2520,16 +2514,16 @@ void EnqueueVisitor::VisitCompoundStmt(const CompoundStmt *S) {
   for (auto &I : llvm::reverse(S->body()))
     AddStmt(I);
 }
-void EnqueueVisitor::
-VisitMSDependentExistsStmt(const MSDependentExistsStmt *S) {
+void EnqueueVisitor::VisitMSDependentExistsStmt(
+    const MSDependentExistsStmt *S) {
   AddStmt(S->getSubStmt());
   AddDeclarationNameInfo(S);
   if (NestedNameSpecifierLoc QualifierLoc = S->getQualifierLoc())
     AddNestedNameSpecifierLoc(QualifierLoc);
 }
 
-void EnqueueVisitor::
-VisitCXXDependentScopeMemberExpr(const CXXDependentScopeMemberExpr *E) {
+void EnqueueVisitor::VisitCXXDependentScopeMemberExpr(
+    const CXXDependentScopeMemberExpr *E) {
   if (E->hasExplicitTemplateArgs())
     AddExplicitTemplateArgs(E->getTemplateArgs(), E->getNumTemplateArgs());
   AddDeclarationNameInfo(E);
@@ -2547,16 +2541,16 @@ void EnqueueVisitor::VisitCXXNewExpr(const CXXNewExpr *E) {
   AddTypeLoc(E->getAllocatedTypeSourceInfo());
   // Enqueue the placement arguments.
   for (unsigned I = E->getNumPlacementArgs(); I > 0; --I)
-    AddStmt(E->getPlacementArg(I-1));
+    AddStmt(E->getPlacementArg(I - 1));
 }
 void EnqueueVisitor::VisitCXXOperatorCallExpr(const CXXOperatorCallExpr *CE) {
   for (unsigned I = CE->getNumArgs(); I > 1 /* Yes, this is 1 */; --I)
-    AddStmt(CE->getArg(I-1));
+    AddStmt(CE->getArg(I - 1));
   AddStmt(CE->getCallee());
   AddStmt(CE->getArg(0));
 }
 void EnqueueVisitor::VisitCXXPseudoDestructorExpr(
-                                        const CXXPseudoDestructorExpr *E) {
+    const CXXPseudoDestructorExpr *E) {
   // Visit the name of the type being destroyed.
   AddTypeLoc(E->getDestroyedTypeInfo());
   // Visit the scope type that looks disturbingly like the nested-name-specifier
@@ -2569,11 +2563,11 @@ void EnqueueVisitor::VisitCXXPseudoDestructorExpr(
   AddStmt(E->getBase());
 }
 void EnqueueVisitor::VisitCXXScalarValueInitExpr(
-                                        const CXXScalarValueInitExpr *E) {
+    const CXXScalarValueInitExpr *E) {
   AddTypeLoc(E->getTypeSourceInfo());
 }
 void EnqueueVisitor::VisitCXXTemporaryObjectExpr(
-                                        const CXXTemporaryObjectExpr *E) {
+    const CXXTemporaryObjectExpr *E) {
   EnqueueChildren(E);
   AddTypeLoc(E->getTypeSourceInfo());
 }
@@ -2584,7 +2578,7 @@ void EnqueueVisitor::VisitCXXTypeidExpr(const CXXTypeidExpr *E) {
 }
 
 void EnqueueVisitor::VisitCXXUnresolvedConstructExpr(
-                                        const CXXUnresolvedConstructExpr *E) {
+    const CXXUnresolvedConstructExpr *E) {
   EnqueueChildren(E);
   AddTypeLoc(E->getTypeSourceInfo());
 }
@@ -2611,7 +2605,7 @@ void EnqueueVisitor::VisitDeclRefExpr(const DeclRefExpr *DR) {
   WL.push_back(DeclRefExprParts(DR, Parent));
 }
 void EnqueueVisitor::VisitDependentScopeDeclRefExpr(
-                                        const DependentScopeDeclRefExpr *E) {
+    const DependentScopeDeclRefExpr *E) {
   if (E->hasExplicitTemplateArgs())
     AddExplicitTemplateArgs(E->getTemplateArgs(), E->getNumTemplateArgs());
   AddDeclarationNameInfo(E);
@@ -2677,7 +2671,7 @@ void EnqueueVisitor::VisitInitListExpr(const InitListExpr *IE) {
 }
 void EnqueueVisitor::VisitMemberExpr(const MemberExpr *M) {
   WL.push_back(MemberExprParts(M, Parent));
-  
+
   // If the base of the member access expression is an implicit 'this', don't
   // visit it.
   // FIXME: If we ever want to show these implicit accesses, this will be
@@ -2708,7 +2702,7 @@ void EnqueueVisitor::VisitObjCMessageExpr(const ObjCMessageExpr *M) {
 void EnqueueVisitor::VisitOffsetOfExpr(const OffsetOfExpr *E) {
   // Visit the components of the offsetof expression.
   for (unsigned N = E->getNumComponents(), I = N; I > 0; --I) {
-    const OffsetOfNode &Node = E->getComponent(I-1);
+    const OffsetOfNode &Node = E->getComponent(I - 1);
     switch (Node.getKind()) {
     case OffsetOfNode::Array:
       AddStmt(E->getIndexExpr(Node.getArrayExprIndex()));
@@ -2730,14 +2724,12 @@ void EnqueueVisitor::VisitOverloadExpr(const OverloadExpr *E) {
   WL.push_back(OverloadExprParts(E, Parent));
 }
 void EnqueueVisitor::VisitUnaryExprOrTypeTraitExpr(
-                                        const UnaryExprOrTypeTraitExpr *E) {
+    const UnaryExprOrTypeTraitExpr *E) {
   EnqueueChildren(E);
   if (E->isArgumentType())
     AddTypeLoc(E->getArgumentTypeInfo());
 }
-void EnqueueVisitor::VisitStmt(const Stmt *S) {
-  EnqueueChildren(S);
-}
+void EnqueueVisitor::VisitStmt(const Stmt *S) { EnqueueChildren(S); }
 void EnqueueVisitor::VisitSwitchStmt(const SwitchStmt *S) {
   AddStmt(S->getBody());
   AddStmt(S->getCond());
@@ -2752,7 +2744,7 @@ void EnqueueVisitor::VisitWhileStmt(const WhileStmt *W) {
 
 void EnqueueVisitor::VisitTypeTraitExpr(const TypeTraitExpr *E) {
   for (unsigned I = E->getNumArgs(); I > 0; --I)
-    AddTypeLoc(E->getArg(I-1));
+    AddTypeLoc(E->getArg(I - 1));
 }
 
 void EnqueueVisitor::VisitArrayTypeTraitExpr(const ArrayTypeTraitExpr *E) {
@@ -2791,7 +2783,7 @@ void EnqueueVisitor::VisitPseudoObjectExpr(const PseudoObjectExpr *E) {
 }
 
 void EnqueueVisitor::VisitOMPExecutableDirective(
-  const OMPExecutableDirective *D) {
+    const OMPExecutableDirective *D) {
   EnqueueChildren(D);
   for (ArrayRef<OMPClause *>::iterator I = D->clauses().begin(),
                                        E = D->clauses().end();
@@ -2840,8 +2832,8 @@ void EnqueueVisitor::VisitOMPCriticalDirective(const OMPCriticalDirective *D) {
   AddDeclarationNameInfo(D);
 }
 
-void
-EnqueueVisitor::VisitOMPParallelForDirective(const OMPParallelForDirective *D) {
+void EnqueueVisitor::VisitOMPParallelForDirective(
+    const OMPParallelForDirective *D) {
   VisitOMPLoopDirective(D);
 }
 
@@ -2864,8 +2856,8 @@ void EnqueueVisitor::VisitOMPTaskDirective(const OMPTaskDirective *D) {
   VisitOMPExecutableDirective(D);
 }
 
-void
-EnqueueVisitor::VisitOMPTaskyieldDirective(const OMPTaskyieldDirective *D) {
+void EnqueueVisitor::VisitOMPTaskyieldDirective(
+    const OMPTaskyieldDirective *D) {
   VisitOMPExecutableDirective(D);
 }
 
@@ -3051,7 +3043,8 @@ void EnqueueVisitor::VisitOMPTargetTeamsDistributeSimdDirective(
 }
 
 void CursorVisitor::EnqueueWorkList(VisitorWorkList &WL, const Stmt *S) {
-  EnqueueVisitor(WL, MakeCXCursor(S, StmtParent, TU,RegionOfInterest)).Visit(S);
+  EnqueueVisitor(WL, MakeCXCursor(S, StmtParent, TU, RegionOfInterest))
+      .Visit(S);
 }
 
 bool CursorVisitor::IsInRegionOfInterest(CXCursor C) {
@@ -3070,199 +3063,199 @@ bool CursorVisitor::RunVisitorWorkList(VisitorWorkList &WL) {
 
     // Set the Parent field, then back to its old value once we're done.
     SetParentRAII SetParent(Parent, StmtParent, LI.getParent());
-  
+
     switch (LI.getKind()) {
-      case VisitorJob::DeclVisitKind: {
-        const Decl *D = cast<DeclVisit>(&LI)->get();
-        if (!D)
-          continue;
+    case VisitorJob::DeclVisitKind: {
+      const Decl *D = cast<DeclVisit>(&LI)->get();
+      if (!D)
+        continue;
 
-        // For now, perform default visitation for Decls.
-        if (Visit(MakeCXCursor(D, TU, RegionOfInterest,
-                               cast<DeclVisit>(&LI)->isFirst())))
-            return true;
+      // For now, perform default visitation for Decls.
+      if (Visit(MakeCXCursor(D, TU, RegionOfInterest,
+                             cast<DeclVisit>(&LI)->isFirst())))
+        return true;
 
-        continue;
-      }
-      case VisitorJob::ExplicitTemplateArgsVisitKind: {
-        for (const TemplateArgumentLoc &Arg :
-             *cast<ExplicitTemplateArgsVisit>(&LI)) {
-          if (VisitTemplateArgumentLoc(Arg))
-            return true;
-        }
-        continue;
-      }
-      case VisitorJob::TypeLocVisitKind: {
-        // Perform default visitation for TypeLocs.
-        if (Visit(cast<TypeLocVisit>(&LI)->get()))
+      continue;
+    }
+    case VisitorJob::ExplicitTemplateArgsVisitKind: {
+      for (const TemplateArgumentLoc &Arg :
+           *cast<ExplicitTemplateArgsVisit>(&LI)) {
+        if (VisitTemplateArgumentLoc(Arg))
           return true;
-        continue;
       }
-      case VisitorJob::LabelRefVisitKind: {
-        const LabelDecl *LS = cast<LabelRefVisit>(&LI)->get();
-        if (LabelStmt *stmt = LS->getStmt()) {
-          if (Visit(MakeCursorLabelRef(stmt, cast<LabelRefVisit>(&LI)->getLoc(),
-                                       TU))) {
-            return true;
-          }
+      continue;
+    }
+    case VisitorJob::TypeLocVisitKind: {
+      // Perform default visitation for TypeLocs.
+      if (Visit(cast<TypeLocVisit>(&LI)->get()))
+        return true;
+      continue;
+    }
+    case VisitorJob::LabelRefVisitKind: {
+      const LabelDecl *LS = cast<LabelRefVisit>(&LI)->get();
+      if (LabelStmt *stmt = LS->getStmt()) {
+        if (Visit(MakeCursorLabelRef(stmt, cast<LabelRefVisit>(&LI)->getLoc(),
+                                     TU))) {
+          return true;
         }
-        continue;
       }
+      continue;
+    }
 
-      case VisitorJob::NestedNameSpecifierLocVisitKind: {
-        NestedNameSpecifierLocVisit *V = cast<NestedNameSpecifierLocVisit>(&LI);
-        if (VisitNestedNameSpecifierLoc(V->get()))
-          return true;
+    case VisitorJob::NestedNameSpecifierLocVisitKind: {
+      NestedNameSpecifierLocVisit *V = cast<NestedNameSpecifierLocVisit>(&LI);
+      if (VisitNestedNameSpecifierLoc(V->get()))
+        return true;
+      continue;
+    }
+
+    case VisitorJob::DeclarationNameInfoVisitKind: {
+      if (VisitDeclarationNameInfo(cast<DeclarationNameInfoVisit>(&LI)->get()))
+        return true;
+      continue;
+    }
+    case VisitorJob::MemberRefVisitKind: {
+      MemberRefVisit *V = cast<MemberRefVisit>(&LI);
+      if (Visit(MakeCursorMemberRef(V->get(), V->getLoc(), TU)))
+        return true;
+      continue;
+    }
+    case VisitorJob::StmtVisitKind: {
+      const Stmt *S = cast<StmtVisit>(&LI)->get();
+      if (!S)
         continue;
-      }
-        
-      case VisitorJob::DeclarationNameInfoVisitKind: {
-        if (VisitDeclarationNameInfo(cast<DeclarationNameInfoVisit>(&LI)
-                                     ->get()))
-          return true;
+
+      // Update the current cursor.
+      CXCursor Cursor = MakeCXCursor(S, StmtParent, TU, RegionOfInterest);
+      if (!IsInRegionOfInterest(Cursor))
         continue;
+      switch (Visitor(Cursor, Parent, ClientData)) {
+      case CXChildVisit_Break:
+        return true;
+      case CXChildVisit_Continue:
+        break;
+      case CXChildVisit_Recurse:
+        if (PostChildrenVisitor)
+          WL.push_back(PostChildrenVisit(nullptr, Cursor));
+        EnqueueWorkList(WL, S);
+        break;
       }
-      case VisitorJob::MemberRefVisitKind: {
-        MemberRefVisit *V = cast<MemberRefVisit>(&LI);
-        if (Visit(MakeCursorMemberRef(V->get(), V->getLoc(), TU)))
+      continue;
+    }
+    case VisitorJob::MemberExprPartsKind: {
+      // Handle the other pieces in the MemberExpr besides the base.
+      const MemberExpr *M = cast<MemberExprParts>(&LI)->get();
+
+      // Visit the nested-name-specifier
+      if (NestedNameSpecifierLoc QualifierLoc = M->getQualifierLoc())
+        if (VisitNestedNameSpecifierLoc(QualifierLoc))
           return true;
-        continue;
-      }
-      case VisitorJob::StmtVisitKind: {
-        const Stmt *S = cast<StmtVisit>(&LI)->get();
-        if (!S)
-          continue;
 
-        // Update the current cursor.
-        CXCursor Cursor = MakeCXCursor(S, StmtParent, TU, RegionOfInterest);
-        if (!IsInRegionOfInterest(Cursor))
-          continue;
-        switch (Visitor(Cursor, Parent, ClientData)) {
-          case CXChildVisit_Break: return true;
-          case CXChildVisit_Continue: break;
-          case CXChildVisit_Recurse:
-            if (PostChildrenVisitor)
-              WL.push_back(PostChildrenVisit(nullptr, Cursor));
-            EnqueueWorkList(WL, S);
-            break;
-        }
-        continue;
-      }
-      case VisitorJob::MemberExprPartsKind: {
-        // Handle the other pieces in the MemberExpr besides the base.
-        const MemberExpr *M = cast<MemberExprParts>(&LI)->get();
-        
-        // Visit the nested-name-specifier
-        if (NestedNameSpecifierLoc QualifierLoc = M->getQualifierLoc())
-          if (VisitNestedNameSpecifierLoc(QualifierLoc))
+      // Visit the declaration name.
+      if (VisitDeclarationNameInfo(M->getMemberNameInfo()))
+        return true;
+
+      // Visit the explicitly-specified template arguments, if any.
+      if (M->hasExplicitTemplateArgs()) {
+        for (const TemplateArgumentLoc *Arg = M->getTemplateArgs(),
+                                       *ArgEnd = Arg + M->getNumTemplateArgs();
+             Arg != ArgEnd; ++Arg) {
+          if (VisitTemplateArgumentLoc(*Arg))
             return true;
-        
-        // Visit the declaration name.
-        if (VisitDeclarationNameInfo(M->getMemberNameInfo()))
-          return true;
-        
-        // Visit the explicitly-specified template arguments, if any.
-        if (M->hasExplicitTemplateArgs()) {
-          for (const TemplateArgumentLoc *Arg = M->getTemplateArgs(),
-               *ArgEnd = Arg + M->getNumTemplateArgs();
-               Arg != ArgEnd; ++Arg) {
-            if (VisitTemplateArgumentLoc(*Arg))
-              return true;
-          }
         }
-        continue;
       }
-      case VisitorJob::DeclRefExprPartsKind: {
-        const DeclRefExpr *DR = cast<DeclRefExprParts>(&LI)->get();
-        // Visit nested-name-specifier, if present.
-        if (NestedNameSpecifierLoc QualifierLoc = DR->getQualifierLoc())
-          if (VisitNestedNameSpecifierLoc(QualifierLoc))
-            return true;
-        // Visit declaration name.
-        if (VisitDeclarationNameInfo(DR->getNameInfo()))
+      continue;
+    }
+    case VisitorJob::DeclRefExprPartsKind: {
+      const DeclRefExpr *DR = cast<DeclRefExprParts>(&LI)->get();
+      // Visit nested-name-specifier, if present.
+      if (NestedNameSpecifierLoc QualifierLoc = DR->getQualifierLoc())
+        if (VisitNestedNameSpecifierLoc(QualifierLoc))
+          return true;
+      // Visit declaration name.
+      if (VisitDeclarationNameInfo(DR->getNameInfo()))
+        return true;
+      continue;
+    }
+    case VisitorJob::OverloadExprPartsKind: {
+      const OverloadExpr *O = cast<OverloadExprParts>(&LI)->get();
+      // Visit the nested-name-specifier.
+      if (NestedNameSpecifierLoc QualifierLoc = O->getQualifierLoc())
+        if (VisitNestedNameSpecifierLoc(QualifierLoc))
+          return true;
+      // Visit the declaration name.
+      if (VisitDeclarationNameInfo(O->getNameInfo()))
+        return true;
+      // Visit the overloaded declaration reference.
+      if (Visit(MakeCursorOverloadedDeclRef(O, TU)))
+        return true;
+      continue;
+    }
+    case VisitorJob::SizeOfPackExprPartsKind: {
+      const SizeOfPackExpr *E = cast<SizeOfPackExprParts>(&LI)->get();
+      NamedDecl *Pack = E->getPack();
+      if (isa<TemplateTypeParmDecl>(Pack)) {
+        if (Visit(MakeCursorTypeRef(cast<TemplateTypeParmDecl>(Pack),
+                                    E->getPackLoc(), TU)))
           return true;
+
         continue;
       }
-      case VisitorJob::OverloadExprPartsKind: {
-        const OverloadExpr *O = cast<OverloadExprParts>(&LI)->get();
-        // Visit the nested-name-specifier.
-        if (NestedNameSpecifierLoc QualifierLoc = O->getQualifierLoc())
-          if (VisitNestedNameSpecifierLoc(QualifierLoc))
-            return true;
-        // Visit the declaration name.
-        if (VisitDeclarationNameInfo(O->getNameInfo()))
-          return true;
-        // Visit the overloaded declaration reference.
-        if (Visit(MakeCursorOverloadedDeclRef(O, TU)))
+
+      if (isa<TemplateTemplateParmDecl>(Pack)) {
+        if (Visit(MakeCursorTemplateRef(cast<TemplateTemplateParmDecl>(Pack),
+                                        E->getPackLoc(), TU)))
           return true;
+
         continue;
       }
-      case VisitorJob::SizeOfPackExprPartsKind: {
-        const SizeOfPackExpr *E = cast<SizeOfPackExprParts>(&LI)->get();
-        NamedDecl *Pack = E->getPack();
-        if (isa<TemplateTypeParmDecl>(Pack)) {
-          if (Visit(MakeCursorTypeRef(cast<TemplateTypeParmDecl>(Pack),
-                                      E->getPackLoc(), TU)))
-            return true;
-          
-          continue;
-        }
-          
-        if (isa<TemplateTemplateParmDecl>(Pack)) {
-          if (Visit(MakeCursorTemplateRef(cast<TemplateTemplateParmDecl>(Pack),
-                                          E->getPackLoc(), TU)))
-            return true;
-          
+
+      // Non-type template parameter packs and function parameter packs are
+      // treated like DeclRefExpr cursors.
+      continue;
+    }
+
+    case VisitorJob::LambdaExprPartsKind: {
+      // Visit non-init captures.
+      const LambdaExpr *E = cast<LambdaExprParts>(&LI)->get();
+      for (LambdaExpr::capture_iterator C = E->explicit_capture_begin(),
+                                        CEnd = E->explicit_capture_end();
+           C != CEnd; ++C) {
+        if (!C->capturesVariable())
           continue;
-        }
-        
-        // Non-type template parameter packs and function parameter packs are
-        // treated like DeclRefExpr cursors.
-        continue;
+
+        if (Visit(MakeCursorVariableRef(C->getCapturedVar(), C->getLocation(),
+                                        TU)))
+          return true;
       }
-        
-      case VisitorJob::LambdaExprPartsKind: {
-        // Visit non-init captures.
-        const LambdaExpr *E = cast<LambdaExprParts>(&LI)->get();
-        for (LambdaExpr::capture_iterator C = E->explicit_capture_begin(),
-                                       CEnd = E->explicit_capture_end();
-             C != CEnd; ++C) {
-          if (!C->capturesVariable())
-            continue;
-
-          if (Visit(MakeCursorVariableRef(C->getCapturedVar(),
-                                          C->getLocation(),
-                                          TU)))
-            return true;
+      // Visit init captures
+      for (auto InitExpr : E->capture_inits()) {
+        if (Visit(InitExpr))
+          return true;
+      }
+
+      TypeLoc TL = E->getCallOperator()->getTypeSourceInfo()->getTypeLoc();
+      // Visit parameters and return type, if present.
+      if (FunctionTypeLoc Proto = TL.getAs<FunctionProtoTypeLoc>()) {
+        if (E->hasExplicitParameters()) {
+          // Visit parameters.
+          for (unsigned I = 0, N = Proto.getNumParams(); I != N; ++I)
+            if (Visit(MakeCXCursor(Proto.getParam(I), TU)))
+              return true;
         }
-        // Visit init captures
-        for (auto InitExpr : E->capture_inits()) {
-          if (Visit(InitExpr))
+        if (E->hasExplicitResultType()) {
+          // Visit result type.
+          if (Visit(Proto.getReturnLoc()))
             return true;
         }
-        
-        TypeLoc TL = E->getCallOperator()->getTypeSourceInfo()->getTypeLoc();
-        // Visit parameters and return type, if present.
-        if (FunctionTypeLoc Proto = TL.getAs<FunctionProtoTypeLoc>()) {
-          if (E->hasExplicitParameters()) {
-            // Visit parameters.
-            for (unsigned I = 0, N = Proto.getNumParams(); I != N; ++I)
-              if (Visit(MakeCXCursor(Proto.getParam(I), TU)))
-                return true;
-          }
-          if (E->hasExplicitResultType()) {
-            // Visit result type.
-            if (Visit(Proto.getReturnLoc()))
-              return true;
-          }
-        }
-        break;
       }
+      break;
+    }
 
-      case VisitorJob::PostChildrenVisitKind:
-        if (PostChildrenVisitor(Parent, ClientData))
-          return true;
-        break;
+    case VisitorJob::PostChildrenVisitKind:
+      if (PostChildrenVisitor(Parent, ClientData))
+        return true;
+      break;
     }
   }
   return false;
@@ -3274,8 +3267,7 @@ bool CursorVisitor::Visit(const Stmt *S) {
     WL = WorkListFreeList.back();
     WL->clear();
     WorkListFreeList.pop_back();
-  }
-  else {
+  } else {
     WL = new VisitorWorkList();
     WorkListCache.push_back(WL);
   }
@@ -3293,14 +3285,14 @@ RefNamePieces buildPieces(unsigned NameFlags, bool IsMemberRefExpr,
   const bool WantQualifier = NameFlags & CXNameRange_WantQualifier;
   const bool WantTemplateArgs = NameFlags & CXNameRange_WantTemplateArgs;
   const bool WantSinglePiece = NameFlags & CXNameRange_WantSinglePiece;
-  
+
   const DeclarationName::NameKind Kind = NI.getName().getNameKind();
-  
+
   RefNamePieces Pieces;
 
   if (WantQualifier && QLoc.isValid())
     Pieces.push_back(QLoc);
-  
+
   if (Kind != DeclarationName::CXXOperatorName || IsMemberRefExpr)
     Pieces.push_back(NI.getLoc());
 
@@ -3309,24 +3301,24 @@ RefNamePieces buildPieces(unsigned NameFlags, bool IsMemberRefExpr,
 
   if (Kind == DeclarationName::CXXOperatorName) {
     Pieces.push_back(SourceLocation::getFromRawEncoding(
-                       NI.getInfo().CXXOperatorName.BeginOpNameLoc));
+        NI.getInfo().CXXOperatorName.BeginOpNameLoc));
     Pieces.push_back(SourceLocation::getFromRawEncoding(
-                       NI.getInfo().CXXOperatorName.EndOpNameLoc));
+        NI.getInfo().CXXOperatorName.EndOpNameLoc));
   }
-  
+
   if (WantSinglePiece) {
     SourceRange R(Pieces.front().getBegin(), Pieces.back().getEnd());
     Pieces.clear();
     Pieces.push_back(R);
-  }  
+  }
 
-  return Pieces;  
-}
+  return Pieces;
 }
+} // namespace
 
 //===----------------------------------------------------------------------===//
 // Misc. API hooks.
-//===----------------------------------------------------------------------===//               
+//===----------------------------------------------------------------------===//
 
 namespace {
 struct RegisterFatalErrorHandler {
@@ -3334,9 +3326,10 @@ struct RegisterFatalErrorHandler {
     clang_install_aborting_llvm_fatal_error_handler();
   }
 };
-}
+} // namespace
 
-static llvm::ManagedStatic<RegisterFatalErrorHandler> RegisterFatalErrorHandlerOnce;
+static llvm::ManagedStatic<RegisterFatalErrorHandler>
+    RegisterFatalErrorHandlerOnce;
 
 CXIndex clang_createIndex(int excludeDeclarationsFromPCH,
                           int displayDiagnostics) {
@@ -3422,9 +3415,7 @@ enum CXErrorCode clang_createTranslationUnit2(CXIndex CIdx,
   if (!CIdx || !ast_filename || !out_TU)
     return CXError_InvalidArguments;
 
-  LOG_FUNC_SECTION {
-    *Log << ast_filename;
-  }
+  LOG_FUNC_SECTION { *Log << ast_filename; }
 
   CIndexer *CXXIdx = static_cast<CIndexer *>(CIdx);
   FileSystemOptions FileSystemOpts;
@@ -3433,10 +3424,8 @@ enum CXErrorCode clang_createTranslationUnit2(CXIndex CIdx,
       CompilerInstance::createDiagnostics(new DiagnosticOptions());
   std::unique_ptr<ASTUnit> AU = ASTUnit::LoadFromASTFile(
       ast_filename, CXXIdx->getPCHContainerOperations()->getRawReader(),
-      ASTUnit::LoadEverything, Diags,
-      FileSystemOpts, /*UseDebugInfo=*/false,
-      CXXIdx->getOnlyLocalDecls(), None,
-      CaptureDiagsKind::All,
+      ASTUnit::LoadEverything, Diags, FileSystemOpts, /*UseDebugInfo=*/false,
+      CXXIdx->getOnlyLocalDecls(), None, CaptureDiagsKind::All,
       /*AllowPCHWithCompilerErrors=*/true,
       /*UserFilesAreVolatile=*/true);
   *out_TU = MakeCXTranslationUnit(CXXIdx, std::move(AU));
@@ -3444,22 +3433,18 @@ enum CXErrorCode clang_createTranslationUnit2(CXIndex CIdx,
 }
 
 unsigned clang_defaultEditingTranslationUnitOptions() {
-  return CXTranslationUnit_PrecompiledPreamble | 
+  return CXTranslationUnit_PrecompiledPreamble |
          CXTranslationUnit_CacheCompletionResults;
 }
 
-CXTranslationUnit
-clang_createTranslationUnitFromSourceFile(CXIndex CIdx,
-                                          const char *source_filename,
-                                          int num_command_line_args,
-                                          const char * const *command_line_args,
-                                          unsigned num_unsaved_files,
-                                          struct CXUnsavedFile *unsaved_files) {
+CXTranslationUnit clang_createTranslationUnitFromSourceFile(
+    CXIndex CIdx, const char *source_filename, int num_command_line_args,
+    const char *const *command_line_args, unsigned num_unsaved_files,
+    struct CXUnsavedFile *unsaved_files) {
   unsigned Options = CXTranslationUnit_DetailedPreprocessingRecord;
-  return clang_parseTranslationUnit(CIdx, source_filename,
-                                    command_line_args, num_command_line_args,
-                                    unsaved_files, num_unsaved_files,
-                                    Options);
+  return clang_parseTranslationUnit(CIdx, source_filename, command_line_args,
+                                    num_command_line_args, unsaved_files,
+                                    num_unsaved_files, Options);
 }
 
 static CXErrorCode
@@ -3485,17 +3470,18 @@ clang_parseTranslationUnit_Impl(CXIndex CIdx, const char *source_filename,
   bool CreatePreambleOnFirstParse =
       options & CXTranslationUnit_CreatePreambleOnFirstParse;
   // FIXME: Add a flag for modules.
-  TranslationUnitKind TUKind
-    = (options & (CXTranslationUnit_Incomplete |
-                  CXTranslationUnit_SingleFileParse))? TU_Prefix : TU_Complete;
-  bool CacheCodeCompletionResults
-    = options & CXTranslationUnit_CacheCompletionResults;
-  bool IncludeBriefCommentsInCodeCompletion
-    = options & CXTranslationUnit_IncludeBriefCommentsInCodeCompletion;
+  TranslationUnitKind TUKind = (options & (CXTranslationUnit_Incomplete |
+                                           CXTranslationUnit_SingleFileParse))
+                                   ? TU_Prefix
+                                   : TU_Complete;
+  bool CacheCodeCompletionResults =
+      options & CXTranslationUnit_CacheCompletionResults;
+  bool IncludeBriefCommentsInCodeCompletion =
+      options & CXTranslationUnit_IncludeBriefCommentsInCodeCompletion;
   bool SingleFileParse = options & CXTranslationUnit_SingleFileParse;
   bool ForSerialization = options & CXTranslationUnit_ForSerialization;
-  bool RetainExcludedCB = options &
-    CXTranslationUnit_RetainExcludedConditionalBlocks;
+  bool RetainExcludedCB =
+      options & CXTranslationUnit_RetainExcludedConditionalBlocks;
   SkipFunctionBodiesScope SkipFunctionBodies = SkipFunctionBodiesScope::None;
   if (options & CXTranslationUnit_SkipFunctionBodies) {
     SkipFunctionBodies =
@@ -3505,8 +3491,8 @@ clang_parseTranslationUnit_Impl(CXIndex CIdx, const char *source_filename,
   }
 
   // Configure the diagnostics.
-  IntrusiveRefCntPtr<DiagnosticsEngine>
-    Diags(CompilerInstance::createDiagnostics(new DiagnosticOptions));
+  IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
+      CompilerInstance::createDiagnostics(new DiagnosticOptions));
 
   if (options & CXTranslationUnit_KeepGoing)
     Diags->setFatalsAsError(true);
@@ -3516,16 +3502,17 @@ clang_parseTranslationUnit_Impl(CXIndex CIdx, const char *source_filename,
     CaptureDiagnostics = CaptureDiagsKind::AllWithoutNonErrorsFromIncludes;
 
   // Recover resources if we crash before exiting this function.
-  llvm::CrashRecoveryContextCleanupRegistrar<DiagnosticsEngine,
-    llvm::CrashRecoveryContextReleaseRefCleanup<DiagnosticsEngine> >
-    DiagCleanup(Diags.get());
+  llvm::CrashRecoveryContextCleanupRegistrar<
+      DiagnosticsEngine,
+      llvm::CrashRecoveryContextReleaseRefCleanup<DiagnosticsEngine>>
+      DiagCleanup(Diags.get());
 
   std::unique_ptr<std::vector<ASTUnit::RemappedFile>> RemappedFiles(
       new std::vector<ASTUnit::RemappedFile>());
 
   // Recover resources if we crash before exiting this function.
-  llvm::CrashRecoveryContextCleanupRegistrar<
-    std::vector<ASTUnit::RemappedFile> > RemappedCleanup(RemappedFiles.get());
+  llvm::CrashRecoveryContextCleanupRegistrar<std::vector<ASTUnit::RemappedFile>>
+      RemappedCleanup(RemappedFiles.get());
 
   for (auto &UF : unsaved_files) {
     std::unique_ptr<llvm::MemoryBuffer> MB =
@@ -3537,12 +3524,12 @@ clang_parseTranslationUnit_Impl(CXIndex CIdx, const char *source_filename,
       new std::vector<const char *>());
 
   // Recover resources if we crash before exiting this method.
-  llvm::CrashRecoveryContextCleanupRegistrar<std::vector<const char*> >
-    ArgsCleanup(Args.get());
+  llvm::CrashRecoveryContextCleanupRegistrar<std::vector<const char *>>
+      ArgsCleanup(Args.get());
 
   // Since the Clang C library is primarily used by batch tools dealing with
   // (often very broken) source code, where spell-checking can have a
-  // significant negative impact on performance (particularly when 
+  // significant negative impact on performance (particularly when
   // precompiled headers are involved), we disable it by default.
   // Only do this if we haven't found a spell-checking-related argument.
   bool FoundSpellCheckingArgument = false;
@@ -3625,13 +3612,11 @@ clang_parseTranslationUnit_Impl(CXIndex CIdx, const char *source_filename,
 }
 
 CXTranslationUnit
-clang_parseTranslationUnit(CXIndex CIdx,
-                           const char *source_filename,
+clang_parseTranslationUnit(CXIndex CIdx, const char *source_filename,
                            const char *const *command_line_args,
                            int num_command_line_args,
                            struct CXUnsavedFile *unsaved_files,
-                           unsigned num_unsaved_files,
-                           unsigned options) {
+                           unsigned num_unsaved_files, unsigned options) {
   CXTranslationUnit TU;
   enum CXErrorCode Result = clang_parseTranslationUnit2(
       CIdx, source_filename, command_line_args, num_command_line_args,
@@ -3714,8 +3699,7 @@ CXString clang_Type_getObjCEncoding(CXType CT) {
   CXTranslationUnit tu = static_cast<CXTranslationUnit>(CT.data[1]);
   ASTContext &Ctx = getASTUnit(tu)->getASTContext();
   std::string encoding;
-  Ctx.getObjCEncodingForType(QualType::getFromOpaquePtr(CT.data[0]),
-                             encoding);
+  Ctx.getObjCEncodingForType(QualType::getFromOpaquePtr(CT.data[0]), encoding);
 
   return cxstring::createDup(encoding);
 }
@@ -3764,7 +3748,7 @@ unsigned clang_Cursor_isFunctionInlined(CXCursor C) {
   return FD->isInlined();
 }
 
-static StringLiteral* getCFSTR_value(CallExpr *callExpr) {
+static StringLiteral *getCFSTR_value(CallExpr *callExpr) {
   if (callExpr->getNumArgs() != 1) {
     return nullptr;
   }
@@ -3775,7 +3759,7 @@ static StringLiteral* getCFSTR_value(CallExpr *callExpr) {
     ImplicitCastExpr *I = static_cast<ImplicitCastExpr *>(arg);
     auto *subExpr = I->getSubExprAsWritten();
 
-    if(subExpr->getStmtClass() != Stmt::StringLiteralClass){
+    if (subExpr->getStmtClass() != Stmt::StringLiteralClass) {
       return nullptr;
     }
 
@@ -3824,7 +3808,7 @@ long long clang_EvalResult_getAsLongLong(CXEvalResult E) {
   if (!E) {
     return 0;
   }
-  ExprEvalResult *Result = (ExprEvalResult*)E;
+  ExprEvalResult *Result = (ExprEvalResult *)E;
   if (Result->IsUnsignedInt)
     return Result->EvalData.unsignedVal;
   return Result->EvalData.intVal;
@@ -3839,7 +3823,7 @@ unsigned long long clang_EvalResult_getAsUnsigned(CXEvalResult E) {
     return 0;
   }
 
-  ExprEvalResult *Result = (ExprEvalResult*)E;
+  ExprEvalResult *Result = (ExprEvalResult *)E;
   if (Result->IsUnsignedInt)
     return Result->EvalData.unsignedVal;
   return Result->EvalData.intVal;
@@ -3852,14 +3836,14 @@ double clang_EvalResult_getAsDouble(CXEvalResult E) {
   return ((ExprEvalResult *)E)->EvalData.floatVal;
 }
 
-const char* clang_EvalResult_getAsStr(CXEvalResult E) {
+const char *clang_EvalResult_getAsStr(CXEvalResult E) {
   if (!E) {
     return nullptr;
   }
   return ((ExprEvalResult *)E)->EvalData.stringVal;
 }
 
-static const ExprEvalResult* evaluateExpr(Expr *expr, CXCursor C) {
+static const ExprEvalResult *evaluateExpr(Expr *expr, CXCursor C) {
   Expr::EvalResult ER;
   ASTContext &ctx = getCursorContext(C);
   if (!expr)
@@ -3880,7 +3864,7 @@ static const ExprEvalResult* evaluateExpr(Expr *expr, CXCursor C) {
   if (ER.Val.isInt()) {
     result->EvalType = CXEval_Int;
 
-    auto& val = ER.Val.getInt();
+    auto &val = ER.Val.getInt();
     if (val.isUnsigned()) {
       result->IsUnsignedInt = true;
       result->EvalData.unsignedVal = val.getZExtValue();
@@ -4053,7 +4037,7 @@ unsigned clang_Cursor_hasAttrs(CXCursor C) {
 }
 unsigned clang_defaultSaveOptions(CXTranslationUnit TU) {
   return CXSaveTranslationUnit_None;
-}  
+}
 
 static CXSaveError clang_saveTranslationUnit_Impl(CXTranslationUnit TU,
                                                   const char *FileName,
@@ -4068,9 +4052,7 @@ static CXSaveError clang_saveTranslationUnit_Impl(CXTranslationUnit TU,
 
 int clang_saveTranslationUnit(CXTranslationUnit TU, const char *FileName,
                               unsigned options) {
-  LOG_FUNC_SECTION {
-    *Log << TU << ' ' << FileName;
-  }
+  LOG_FUNC_SECTION { *Log << TU << ' ' << FileName; }
 
   if (isNotUsableTU(TU)) {
     LOG_BAD_TU(TU);
@@ -4162,7 +4144,7 @@ clang_reparseTranslationUnit_Impl(CXTranslationUnit TU,
   }
 
   // Reset the associated diagnostics.
-  delete static_cast<CXDiagnosticSetImpl*>(TU->Diagnostics);
+  delete static_cast<CXDiagnosticSetImpl *>(TU->Diagnostics);
   TU->Diagnostics = nullptr;
 
   CIndexer *CXXIdx = TU->CIdx;
@@ -4176,8 +4158,8 @@ clang_reparseTranslationUnit_Impl(CXTranslationUnit TU,
       new std::vector<ASTUnit::RemappedFile>());
 
   // Recover resources if we crash before exiting this function.
-  llvm::CrashRecoveryContextCleanupRegistrar<
-    std::vector<ASTUnit::RemappedFile> > RemappedCleanup(RemappedFiles.get());
+  llvm::CrashRecoveryContextCleanupRegistrar<std::vector<ASTUnit::RemappedFile>>
+      RemappedCleanup(RemappedFiles.get());
 
   for (auto &UF : unsaved_files) {
     std::unique_ptr<llvm::MemoryBuffer> MB =
@@ -4197,9 +4179,7 @@ int clang_reparseTranslationUnit(CXTranslationUnit TU,
                                  unsigned num_unsaved_files,
                                  struct CXUnsavedFile *unsaved_files,
                                  unsigned options) {
-  LOG_FUNC_SECTION {
-    *Log << TU;
-  }
+  LOG_FUNC_SECTION { *Log << TU; }
 
   if (num_unsaved_files && !unsaved_files)
     return CXError_InvalidArguments;
@@ -4222,7 +4202,6 @@ int clang_reparseTranslationUnit(CXTranslationUnit TU,
   return result;
 }
 
-
 CXString clang_getTranslationUnitSpelling(CXTranslationUnit CTUnit) {
   if (isNotUsableTU(CTUnit)) {
     LOG_BAD_TU(CTUnit);
@@ -4249,7 +4228,7 @@ CXTargetInfo clang_getTranslationUnitTargetInfo(CXTranslationUnit CTUnit) {
     return nullptr;
   }
 
-  CXTargetInfoImpl* impl = new CXTargetInfoImpl();
+  CXTargetInfoImpl *impl = new CXTargetInfoImpl();
   impl->TranslationUnit = CTUnit;
   return impl;
 }
@@ -4264,7 +4243,7 @@ CXString clang_TargetInfo_getTriple(CXTargetInfo TargetInfo) {
 
   ASTUnit *CXXUnit = cxtu::getASTUnit(CTUnit);
   std::string Triple =
-    CXXUnit->getASTContext().getTargetInfo().getTriple().normalize();
+      CXXUnit->getASTContext().getTargetInfo().getTriple().normalize();
   return cxstring::createDup(Triple);
 }
 
@@ -4343,8 +4322,7 @@ const char *clang_getFileContents(CXTranslationUnit TU, CXFile file,
   return buf->getBufferStart();
 }
 
-unsigned clang_isFileMultipleIncludeGuarded(CXTranslationUnit TU,
-                                            CXFile file) {
+unsigned clang_isFileMultipleIncludeGuarded(CXTranslationUnit TU, CXFile file) {
   if (isNotUsableTU(TU)) {
     LOG_BAD_TU(TU);
     return 0;
@@ -4355,8 +4333,9 @@ unsigned clang_isFileMultipleIncludeGuarded(CXTranslationUnit TU,
 
   ASTUnit *CXXUnit = cxtu::getASTUnit(TU);
   FileEntry *FEnt = static_cast<FileEntry *>(file);
-  return CXXUnit->getPreprocessor().getHeaderSearchInfo()
-                                          .isFileMultipleIncludeGuarded(FEnt);
+  return CXXUnit->getPreprocessor()
+      .getHeaderSearchInfo()
+      .isFileMultipleIncludeGuarded(FEnt);
 }
 
 int clang_getFileUniqueID(CXFile file, CXFileUniqueID *outID) {
@@ -4421,12 +4400,12 @@ static const Decl *getDeclFromExpr(const Stmt *E) {
   if (const OpaqueValueExpr *OVE = dyn_cast<OpaqueValueExpr>(E))
     if (Expr *Src = OVE->getSourceExpr())
       return getDeclFromExpr(Src);
-      
+
   if (const CallExpr *CE = dyn_cast<CallExpr>(E))
     return getDeclFromExpr(CE->getCallee());
   if (const CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(E))
     if (!CE->isElidable())
-    return CE->getConstructor();
+      return CE->getConstructor();
   if (const CXXInheritedCtorInitExpr *CE =
           dyn_cast<CXXInheritedCtorInitExpr>(E))
     return CE->getConstructor();
@@ -4435,11 +4414,11 @@ static const Decl *getDeclFromExpr(const Stmt *E) {
 
   if (const ObjCProtocolExpr *PE = dyn_cast<ObjCProtocolExpr>(E))
     return PE->getProtocol();
-  if (const SubstNonTypeTemplateParmPackExpr *NTTP
-                              = dyn_cast<SubstNonTypeTemplateParmPackExpr>(E))
+  if (const SubstNonTypeTemplateParmPackExpr *NTTP =
+          dyn_cast<SubstNonTypeTemplateParmPackExpr>(E))
     return NTTP->getParameterPack();
   if (const SizeOfPackExpr *SizeOfPack = dyn_cast<SizeOfPackExpr>(E))
-    if (isa<NonTypeTemplateParmDecl>(SizeOfPack->getPack()) || 
+    if (isa<NonTypeTemplateParmDecl>(SizeOfPack->getPack()) ||
         isa<ParmVarDecl>(SizeOfPack->getPack()))
       return SizeOfPack->getPack();
 
@@ -4451,7 +4430,7 @@ static SourceLocation getLocationFromExpr(const Expr *E) {
     return getLocationFromExpr(CE->getSubExpr());
 
   if (const ObjCMessageExpr *Msg = dyn_cast<ObjCMessageExpr>(E))
-    return /*FIXME:*/Msg->getLeftLoc();
+    return /*FIXME:*/ Msg->getLeftLoc();
   if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(E))
     return DRE->getLocation();
   if (const MemberExpr *Member = dyn_cast<MemberExpr>(E))
@@ -4468,8 +4447,7 @@ static SourceLocation getLocationFromExpr(const Expr *E) {
 
 extern "C" {
 
-unsigned clang_visitChildren(CXCursor parent,
-                             CXCursorVisitor visitor,
+unsigned clang_visitChildren(CXCursor parent, CXCursorVisitor visitor,
                              CXClientData client_data) {
   CursorVisitor CursorVis(getCursorTU(parent), visitor, client_data,
                           /*VisitPreprocessorLast=*/false);
@@ -4480,34 +4458,32 @@ unsigned clang_visitChildren(CXCursor parent,
 #define __has_feature(x) 0
 #endif
 #if __has_feature(blocks)
-typedef enum CXChildVisitResult 
-     (^CXCursorVisitorBlock)(CXCursor cursor, CXCursor parent);
+typedef enum CXChildVisitResult (^CXCursorVisitorBlock)(CXCursor cursor,
+                                                        CXCursor parent);
 
 static enum CXChildVisitResult visitWithBlock(CXCursor cursor, CXCursor parent,
-    CXClientData client_data) {
+                                              CXClientData client_data) {
   CXCursorVisitorBlock block = (CXCursorVisitorBlock)client_data;
   return block(cursor, parent);
 }
 #else
 // If we are compiled with a compiler that doesn't have native blocks support,
-// define and call the block manually, so the 
-typedef struct _CXChildVisitResult
-{
-	void *isa;
-	int flags;
-	int reserved;
-	enum CXChildVisitResult(*invoke)(struct _CXChildVisitResult*, CXCursor,
-                                         CXCursor);
-} *CXCursorVisitorBlock;
+// define and call the block manually, so the
+typedef struct _CXChildVisitResult {
+  void *isa;
+  int flags;
+  int reserved;
+  enum CXChildVisitResult (*invoke)(struct _CXChildVisitResult *, CXCursor,
+                                    CXCursor);
+} * CXCursorVisitorBlock;
 
 static enum CXChildVisitResult visitWithBlock(CXCursor cursor, CXCursor parent,
-    CXClientData client_data) {
+                                              CXClientData client_data) {
   CXCursorVisitorBlock block = (CXCursorVisitorBlock)client_data;
   return block->invoke(block, cursor, parent);
 }
 #endif
 
-
 unsigned clang_visitChildrenWithBlock(CXCursor parent,
                                       CXCursorVisitorBlock block) {
   return clang_visitChildren(parent, visitWithBlock, block);
@@ -4523,14 +4499,14 @@ static CXString getDeclSpelling(const Decl *D) {
             dyn_cast<ObjCPropertyImplDecl>(D))
       if (ObjCPropertyDecl *Property = PropImpl->getPropertyDecl())
         return cxstring::createDup(Property->getIdentifier()->getName());
-    
+
     if (const ImportDecl *ImportD = dyn_cast<ImportDecl>(D))
       if (Module *Mod = ImportD->getImportedModule())
         return cxstring::createDup(Mod->getFullModuleName());
 
     return cxstring::createEmpty();
   }
-  
+
   if (const ObjCMethodDecl *OMD = dyn_cast<ObjCMethodDecl>(ND))
     return cxstring::createDup(OMD->getSelector().getAsString());
 
@@ -4542,11 +4518,11 @@ static CXString getDeclSpelling(const Decl *D) {
 
   if (isa<UsingDirectiveDecl>(D))
     return cxstring::createEmpty();
-  
+
   SmallString<1024> S;
   llvm::raw_svector_ostream os(S);
   ND->printName(os);
-  
+
   return cxstring::createDup(os.str());
 }
 
@@ -4577,34 +4553,34 @@ CXString clang_getCursorSpelling(CXCursor C) {
       const TypeDecl *Type = getCursorTypeRef(C).first;
       assert(Type && "Missing type decl");
 
-      return cxstring::createDup(getCursorContext(C).getTypeDeclType(Type).
-                              getAsString());
+      return cxstring::createDup(
+          getCursorContext(C).getTypeDeclType(Type).getAsString());
     }
     case CXCursor_TemplateRef: {
       const TemplateDecl *Template = getCursorTemplateRef(C).first;
       assert(Template && "Missing template decl");
-      
+
       return cxstring::createDup(Template->getNameAsString());
     }
-        
+
     case CXCursor_NamespaceRef: {
       const NamedDecl *NS = getCursorNamespaceRef(C).first;
       assert(NS && "Missing namespace decl");
-      
+
       return cxstring::createDup(NS->getNameAsString());
     }
 
     case CXCursor_MemberRef: {
       const FieldDecl *Field = getCursorMemberRef(C).first;
       assert(Field && "Missing member decl");
-      
+
       return cxstring::createDup(Field->getNameAsString());
     }
 
     case CXCursor_LabelRef: {
       const LabelStmt *Label = getCursorLabelRef(C).first;
       assert(Label && "Missing label");
-      
+
       return cxstring::createRef(Label->getName());
     }
 
@@ -4617,20 +4593,20 @@ CXString clang_getCursorSpelling(CXCursor C) {
       }
       if (const OverloadExpr *E = Storage.dyn_cast<const OverloadExpr *>())
         return cxstring::createDup(E->getName().getAsString());
-      OverloadedTemplateStorage *Ovl
-        = Storage.get<OverloadedTemplateStorage*>();
+      OverloadedTemplateStorage *Ovl =
+          Storage.get<OverloadedTemplateStorage *>();
       if (Ovl->size() == 0)
         return cxstring::createEmpty();
       return cxstring::createDup((*Ovl->begin())->getNameAsString());
     }
-        
+
     case CXCursor_VariableRef: {
       const VarDecl *Var = getCursorVariableRef(C).first;
       assert(Var && "Missing variable decl");
-      
+
       return cxstring::createDup(Var->getNameAsString());
     }
-        
+
     default:
       return cxstring::createRef("<not implemented>");
     }
@@ -4666,18 +4642,18 @@ CXString clang_getCursorSpelling(CXCursor C) {
 
     return cxstring::createEmpty();
   }
-  
+
   if (C.kind == CXCursor_MacroExpansion)
-    return cxstring::createRef(getCursorMacroExpansion(C).getName()
-                                                           ->getNameStart());
+    return cxstring::createRef(
+        getCursorMacroExpansion(C).getName()->getNameStart());
 
   if (C.kind == CXCursor_MacroDefinition)
-    return cxstring::createRef(getCursorMacroDefinition(C)->getName()
-                                                           ->getNameStart());
+    return cxstring::createRef(
+        getCursorMacroDefinition(C)->getName()->getNameStart());
 
   if (C.kind == CXCursor_InclusionDirective)
     return cxstring::createDup(getCursorInclusionDirective(C)->getFileName());
-      
+
   if (clang_isDeclaration(C.kind))
     return getDeclSpelling(getCursorDecl(C));
 
@@ -4711,8 +4687,7 @@ CXString clang_getCursorSpelling(CXCursor C) {
   return cxstring::createEmpty();
 }
 
-CXSourceRange clang_Cursor_getSpellingNameRange(CXCursor C,
-                                                unsigned pieceIndex,
+CXSourceRange clang_Cursor_getSpellingNameRange(CXCursor C, unsigned pieceIndex,
                                                 unsigned options) {
   if (clang_Cursor_isNull(C))
     return clang_getNullRange();
@@ -4731,8 +4706,8 @@ CXSourceRange clang_Cursor_getSpellingNameRange(CXCursor C,
   }
 
   if (C.kind == CXCursor_ObjCMessageExpr) {
-    if (const ObjCMessageExpr *
-          ME = dyn_cast_or_null<ObjCMessageExpr>(getCursorExpr(C))) {
+    if (const ObjCMessageExpr *ME =
+            dyn_cast_or_null<ObjCMessageExpr>(getCursorExpr(C))) {
       if (pieceIndex >= ME->getNumSelectorLocs())
         return clang_getNullRange();
       return cxloc::translateSourceRange(Ctx, ME->getSelectorLoc(pieceIndex));
@@ -4741,8 +4716,8 @@ CXSourceRange clang_Cursor_getSpellingNameRange(CXCursor C,
 
   if (C.kind == CXCursor_ObjCInstanceMethodDecl ||
       C.kind == CXCursor_ObjCClassMethodDecl) {
-    if (const ObjCMethodDecl *
-          MD = dyn_cast_or_null<ObjCMethodDecl>(getCursorDecl(C))) {
+    if (const ObjCMethodDecl *MD =
+            dyn_cast_or_null<ObjCMethodDecl>(getCursorDecl(C))) {
       if (pieceIndex >= MD->getNumSelectorLocs())
         return clang_getNullRange();
       return cxloc::translateSourceRange(Ctx, MD->getSelectorLoc(pieceIndex));
@@ -4753,11 +4728,11 @@ CXSourceRange clang_Cursor_getSpellingNameRange(CXCursor C,
       C.kind == CXCursor_ObjCCategoryImplDecl) {
     if (pieceIndex > 0)
       return clang_getNullRange();
-    if (const ObjCCategoryDecl *
-          CD = dyn_cast_or_null<ObjCCategoryDecl>(getCursorDecl(C)))
+    if (const ObjCCategoryDecl *CD =
+            dyn_cast_or_null<ObjCCategoryDecl>(getCursorDecl(C)))
       return cxloc::translateSourceRange(Ctx, CD->getCategoryNameLoc());
-    if (const ObjCCategoryImplDecl *
-          CID = dyn_cast_or_null<ObjCCategoryImplDecl>(getCursorDecl(C)))
+    if (const ObjCCategoryImplDecl *CID =
+            dyn_cast_or_null<ObjCCategoryImplDecl>(getCursorDecl(C)))
       return cxloc::translateSourceRange(Ctx, CID->getCategoryNameLoc());
   }
 
@@ -4768,8 +4743,8 @@ CXSourceRange clang_Cursor_getSpellingNameRange(CXCursor C,
             dyn_cast_or_null<ImportDecl>(getCursorDecl(C))) {
       ArrayRef<SourceLocation> Locs = ImportD->getIdentifierLocs();
       if (!Locs.empty())
-        return cxloc::translateSourceRange(Ctx,
-                                         SourceRange(Locs.front(), Locs.back()));
+        return cxloc::translateSourceRange(
+            Ctx, SourceRange(Locs.front(), Locs.back()));
     }
     return clang_getNullRange();
   }
@@ -5040,7 +5015,7 @@ CXString clang_getCursorPrettyPrinted(CXCursor C, CXPrintingPolicy cxPolicy) {
 CXString clang_getCursorDisplayName(CXCursor C) {
   if (!clang_isDeclaration(C.kind))
     return clang_getCursorSpelling(C);
-  
+
   const Decl *D = getCursorDecl(C);
   if (!D)
     return cxstring::createEmpty();
@@ -5048,7 +5023,7 @@ CXString clang_getCursorDisplayName(CXCursor C) {
   PrintingPolicy Policy = getCursorContext(C).getPrintingPolicy();
   if (const FunctionTemplateDecl *FunTmpl = dyn_cast<FunctionTemplateDecl>(D))
     D = FunTmpl->getTemplatedDecl();
-  
+
   if (const FunctionDecl *Function = dyn_cast<FunctionDecl>(D)) {
     SmallString<64> Str;
     llvm::raw_svector_ostream OS(Str);
@@ -5061,7 +5036,7 @@ CXString clang_getCursorDisplayName(CXCursor C) {
         OS << ", ";
       OS << Function->getParamDecl(I)->getType().getAsString(Policy);
     }
-    
+
     if (Function->isVariadic()) {
       if (Function->getNumParams())
         OS << ", ";
@@ -5070,7 +5045,7 @@ CXString clang_getCursorDisplayName(CXCursor C) {
     OS << ")";
     return cxstring::createDup(OS.str());
   }
-  
+
   if (const ClassTemplateDecl *ClassTemplate = dyn_cast<ClassTemplateDecl>(D)) {
     SmallString<64> Str;
     llvm::raw_svector_ostream OS(Str);
@@ -5080,13 +5055,13 @@ CXString clang_getCursorDisplayName(CXCursor C) {
     for (unsigned I = 0, N = Params->size(); I != N; ++I) {
       if (I)
         OS << ", ";
-      
+
       NamedDecl *Param = Params->getParam(I);
       if (Param->getIdentifier()) {
         OS << Param->getIdentifier()->getName();
         continue;
       }
-      
+
       // There is no parameter name, which makes this tricky. Try to come up
       // with something useful that isn't too long.
       if (TemplateTypeParmDecl *TTP = dyn_cast<TemplateTypeParmDecl>(Param))
@@ -5095,20 +5070,20 @@ CXString clang_getCursorDisplayName(CXCursor C) {
           if (TC->hasExplicitTemplateArgs())
             OS << "<...>";
         } else
-          OS << (TTP->wasDeclaredWithTypename()? "typename" : "class");
-      else if (NonTypeTemplateParmDecl *NTTP
-                                    = dyn_cast<NonTypeTemplateParmDecl>(Param))
+          OS << (TTP->wasDeclaredWithTypename() ? "typename" : "class");
+      else if (NonTypeTemplateParmDecl *NTTP =
+                   dyn_cast<NonTypeTemplateParmDecl>(Param))
         OS << NTTP->getType().getAsString(Policy);
       else
         OS << "template<...> class";
     }
-    
+
     OS << ">";
     return cxstring::createDup(OS.str());
   }
-  
-  if (const ClassTemplateSpecializationDecl *ClassSpec
-                              = dyn_cast<ClassTemplateSpecializationDecl>(D)) {
+
+  if (const ClassTemplateSpecializationDecl *ClassSpec =
+          dyn_cast<ClassTemplateSpecializationDecl>(D)) {
     // If the type was explicitly written, use that.
     if (TypeSourceInfo *TSInfo = ClassSpec->getTypeAsWritten())
       return cxstring::createDup(TSInfo->getType().getAsString(Policy));
@@ -5120,64 +5095,64 @@ CXString clang_getCursorDisplayName(CXCursor C) {
                               Policy);
     return cxstring::createDup(OS.str());
   }
-  
+
   return clang_getCursorSpelling(C);
 }
-  
+
 CXString clang_getCursorKindSpelling(enum CXCursorKind Kind) {
   switch (Kind) {
   case CXCursor_FunctionDecl:
-      return cxstring::createRef("FunctionDecl");
+    return cxstring::createRef("FunctionDecl");
   case CXCursor_TypedefDecl:
-      return cxstring::createRef("TypedefDecl");
+    return cxstring::createRef("TypedefDecl");
   case CXCursor_EnumDecl:
-      return cxstring::createRef("EnumDecl");
+    return cxstring::createRef("EnumDecl");
   case CXCursor_EnumConstantDecl:
-      return cxstring::createRef("EnumConstantDecl");
+    return cxstring::createRef("EnumConstantDecl");
   case CXCursor_StructDecl:
-      return cxstring::createRef("StructDecl");
+    return cxstring::createRef("StructDecl");
   case CXCursor_UnionDecl:
-      return cxstring::createRef("UnionDecl");
+    return cxstring::createRef("UnionDecl");
   case CXCursor_ClassDecl:
-      return cxstring::createRef("ClassDecl");
+    return cxstring::createRef("ClassDecl");
   case CXCursor_FieldDecl:
-      return cxstring::createRef("FieldDecl");
+    return cxstring::createRef("FieldDecl");
   case CXCursor_VarDecl:
-      return cxstring::createRef("VarDecl");
+    return cxstring::createRef("VarDecl");
   case CXCursor_ParmDecl:
-      return cxstring::createRef("ParmDecl");
+    return cxstring::createRef("ParmDecl");
   case CXCursor_ObjCInterfaceDecl:
-      return cxstring::createRef("ObjCInterfaceDecl");
+    return cxstring::createRef("ObjCInterfaceDecl");
   case CXCursor_ObjCCategoryDecl:
-      return cxstring::createRef("ObjCCategoryDecl");
+    return cxstring::createRef("ObjCCategoryDecl");
   case CXCursor_ObjCProtocolDecl:
-      return cxstring::createRef("ObjCProtocolDecl");
+    return cxstring::createRef("ObjCProtocolDecl");
   case CXCursor_ObjCPropertyDecl:
-      return cxstring::createRef("ObjCPropertyDecl");
+    return cxstring::createRef("ObjCPropertyDecl");
   case CXCursor_ObjCIvarDecl:
-      return cxstring::createRef("ObjCIvarDecl");
+    return cxstring::createRef("ObjCIvarDecl");
   case CXCursor_ObjCInstanceMethodDecl:
-      return cxstring::createRef("ObjCInstanceMethodDecl");
+    return cxstring::createRef("ObjCInstanceMethodDecl");
   case CXCursor_ObjCClassMethodDecl:
-      return cxstring::createRef("ObjCClassMethodDecl");
+    return cxstring::createRef("ObjCClassMethodDecl");
   case CXCursor_ObjCImplementationDecl:
-      return cxstring::createRef("ObjCImplementationDecl");
+    return cxstring::createRef("ObjCImplementationDecl");
   case CXCursor_ObjCCategoryImplDecl:
-      return cxstring::createRef("ObjCCategoryImplDecl");
+    return cxstring::createRef("ObjCCategoryImplDecl");
   case CXCursor_CXXMethod:
-      return cxstring::createRef("CXXMethod");
+    return cxstring::createRef("CXXMethod");
   case CXCursor_UnexposedDecl:
-      return cxstring::createRef("UnexposedDecl");
+    return cxstring::createRef("UnexposedDecl");
   case CXCursor_ObjCSuperClassRef:
-      return cxstring::createRef("ObjCSuperClassRef");
+    return cxstring::createRef("ObjCSuperClassRef");
   case CXCursor_ObjCProtocolRef:
-      return cxstring::createRef("ObjCProtocolRef");
+    return cxstring::createRef("ObjCProtocolRef");
   case CXCursor_ObjCClassRef:
-      return cxstring::createRef("ObjCClassRef");
+    return cxstring::createRef("ObjCClassRef");
   case CXCursor_TypeRef:
-      return cxstring::createRef("TypeRef");
+    return cxstring::createRef("TypeRef");
   case CXCursor_TemplateRef:
-      return cxstring::createRef("TemplateRef");
+    return cxstring::createRef("TemplateRef");
   case CXCursor_NamespaceRef:
     return cxstring::createRef("NamespaceRef");
   case CXCursor_MemberRef:
@@ -5189,195 +5164,199 @@ CXString clang_getCursorKindSpelling(enum CXCursorKind Kind) {
   case CXCursor_VariableRef:
     return cxstring::createRef("VariableRef");
   case CXCursor_IntegerLiteral:
-      return cxstring::createRef("IntegerLiteral");
+    return cxstring::createRef("IntegerLiteral");
   case CXCursor_FixedPointLiteral:
-      return cxstring::createRef("FixedPointLiteral");
+    return cxstring::createRef("FixedPointLiteral");
   case CXCursor_FloatingLiteral:
-      return cxstring::createRef("FloatingLiteral");
+    return cxstring::createRef("FloatingLiteral");
   case CXCursor_ImaginaryLiteral:
-      return cxstring::createRef("ImaginaryLiteral");
+    return cxstring::createRef("ImaginaryLiteral");
   case CXCursor_StringLiteral:
-      return cxstring::createRef("StringLiteral");
+    return cxstring::createRef("StringLiteral");
   case CXCursor_CharacterLiteral:
-      return cxstring::createRef("CharacterLiteral");
+    return cxstring::createRef("CharacterLiteral");
   case CXCursor_ParenExpr:
-      return cxstring::createRef("ParenExpr");
+    return cxstring::createRef("ParenExpr");
   case CXCursor_UnaryOperator:
-      return cxstring::createRef("UnaryOperator");
+    return cxstring::createRef("UnaryOperator");
   case CXCursor_ArraySubscriptExpr:
-      return cxstring::createRef("ArraySubscriptExpr");
+    return cxstring::createRef("ArraySubscriptExpr");
   case CXCursor_OMPArraySectionExpr:
-      return cxstring::createRef("OMPArraySectionExpr");
+    return cxstring::createRef("OMPArraySectionExpr");
+  case CXCursor_OMPArrayShapingExpr:
+    return cxstring::createRef("OMPArrayShapingExpr");
+  case CXCursor_OMPIteratorExpr:
+    return cxstring::createRef("OMPIteratorExpr");
   case CXCursor_BinaryOperator:
-      return cxstring::createRef("BinaryOperator");
+    return cxstring::createRef("BinaryOperator");
   case CXCursor_CompoundAssignOperator:
-      return cxstring::createRef("CompoundAssignOperator");
+    return cxstring::createRef("CompoundAssignOperator");
   case CXCursor_ConditionalOperator:
-      return cxstring::createRef("ConditionalOperator");
+    return cxstring::createRef("ConditionalOperator");
   case CXCursor_CStyleCastExpr:
-      return cxstring::createRef("CStyleCastExpr");
+    return cxstring::createRef("CStyleCastExpr");
   case CXCursor_CompoundLiteralExpr:
-      return cxstring::createRef("CompoundLiteralExpr");
+    return cxstring::createRef("CompoundLiteralExpr");
   case CXCursor_InitListExpr:
-      return cxstring::createRef("InitListExpr");
+    return cxstring::createRef("InitListExpr");
   case CXCursor_AddrLabelExpr:
-      return cxstring::createRef("AddrLabelExpr");
+    return cxstring::createRef("AddrLabelExpr");
   case CXCursor_StmtExpr:
-      return cxstring::createRef("StmtExpr");
+    return cxstring::createRef("StmtExpr");
   case CXCursor_GenericSelectionExpr:
-      return cxstring::createRef("GenericSelectionExpr");
+    return cxstring::createRef("GenericSelectionExpr");
   case CXCursor_GNUNullExpr:
-      return cxstring::createRef("GNUNullExpr");
+    return cxstring::createRef("GNUNullExpr");
   case CXCursor_CXXStaticCastExpr:
-      return cxstring::createRef("CXXStaticCastExpr");
+    return cxstring::createRef("CXXStaticCastExpr");
   case CXCursor_CXXDynamicCastExpr:
-      return cxstring::createRef("CXXDynamicCastExpr");
+    return cxstring::createRef("CXXDynamicCastExpr");
   case CXCursor_CXXReinterpretCastExpr:
-      return cxstring::createRef("CXXReinterpretCastExpr");
+    return cxstring::createRef("CXXReinterpretCastExpr");
   case CXCursor_CXXConstCastExpr:
-      return cxstring::createRef("CXXConstCastExpr");
+    return cxstring::createRef("CXXConstCastExpr");
   case CXCursor_CXXFunctionalCastExpr:
-      return cxstring::createRef("CXXFunctionalCastExpr");
+    return cxstring::createRef("CXXFunctionalCastExpr");
   case CXCursor_CXXTypeidExpr:
-      return cxstring::createRef("CXXTypeidExpr");
+    return cxstring::createRef("CXXTypeidExpr");
   case CXCursor_CXXBoolLiteralExpr:
-      return cxstring::createRef("CXXBoolLiteralExpr");
+    return cxstring::createRef("CXXBoolLiteralExpr");
   case CXCursor_CXXNullPtrLiteralExpr:
-      return cxstring::createRef("CXXNullPtrLiteralExpr");
+    return cxstring::createRef("CXXNullPtrLiteralExpr");
   case CXCursor_CXXThisExpr:
-      return cxstring::createRef("CXXThisExpr");
+    return cxstring::createRef("CXXThisExpr");
   case CXCursor_CXXThrowExpr:
-      return cxstring::createRef("CXXThrowExpr");
+    return cxstring::createRef("CXXThrowExpr");
   case CXCursor_CXXNewExpr:
-      return cxstring::createRef("CXXNewExpr");
+    return cxstring::createRef("CXXNewExpr");
   case CXCursor_CXXDeleteExpr:
-      return cxstring::createRef("CXXDeleteExpr");
+    return cxstring::createRef("CXXDeleteExpr");
   case CXCursor_UnaryExpr:
-      return cxstring::createRef("UnaryExpr");
+    return cxstring::createRef("UnaryExpr");
   case CXCursor_ObjCStringLiteral:
-      return cxstring::createRef("ObjCStringLiteral");
+    return cxstring::createRef("ObjCStringLiteral");
   case CXCursor_ObjCBoolLiteralExpr:
-      return cxstring::createRef("ObjCBoolLiteralExpr");
+    return cxstring::createRef("ObjCBoolLiteralExpr");
   case CXCursor_ObjCAvailabilityCheckExpr:
-      return cxstring::createRef("ObjCAvailabilityCheckExpr");
+    return cxstring::createRef("ObjCAvailabilityCheckExpr");
   case CXCursor_ObjCSelfExpr:
-      return cxstring::createRef("ObjCSelfExpr");
+    return cxstring::createRef("ObjCSelfExpr");
   case CXCursor_ObjCEncodeExpr:
-      return cxstring::createRef("ObjCEncodeExpr");
+    return cxstring::createRef("ObjCEncodeExpr");
   case CXCursor_ObjCSelectorExpr:
-      return cxstring::createRef("ObjCSelectorExpr");
+    return cxstring::createRef("ObjCSelectorExpr");
   case CXCursor_ObjCProtocolExpr:
-      return cxstring::createRef("ObjCProtocolExpr");
+    return cxstring::createRef("ObjCProtocolExpr");
   case CXCursor_ObjCBridgedCastExpr:
-      return cxstring::createRef("ObjCBridgedCastExpr");
+    return cxstring::createRef("ObjCBridgedCastExpr");
   case CXCursor_BlockExpr:
-      return cxstring::createRef("BlockExpr");
+    return cxstring::createRef("BlockExpr");
   case CXCursor_PackExpansionExpr:
-      return cxstring::createRef("PackExpansionExpr");
+    return cxstring::createRef("PackExpansionExpr");
   case CXCursor_SizeOfPackExpr:
-      return cxstring::createRef("SizeOfPackExpr");
+    return cxstring::createRef("SizeOfPackExpr");
   case CXCursor_LambdaExpr:
     return cxstring::createRef("LambdaExpr");
   case CXCursor_UnexposedExpr:
-      return cxstring::createRef("UnexposedExpr");
+    return cxstring::createRef("UnexposedExpr");
   case CXCursor_DeclRefExpr:
-      return cxstring::createRef("DeclRefExpr");
+    return cxstring::createRef("DeclRefExpr");
   case CXCursor_MemberRefExpr:
-      return cxstring::createRef("MemberRefExpr");
+    return cxstring::createRef("MemberRefExpr");
   case CXCursor_CallExpr:
-      return cxstring::createRef("CallExpr");
+    return cxstring::createRef("CallExpr");
   case CXCursor_ObjCMessageExpr:
-      return cxstring::createRef("ObjCMessageExpr");
+    return cxstring::createRef("ObjCMessageExpr");
   case CXCursor_BuiltinBitCastExpr:
     return cxstring::createRef("BuiltinBitCastExpr");
   case CXCursor_UnexposedStmt:
-      return cxstring::createRef("UnexposedStmt");
+    return cxstring::createRef("UnexposedStmt");
   case CXCursor_DeclStmt:
-      return cxstring::createRef("DeclStmt");
+    return cxstring::createRef("DeclStmt");
   case CXCursor_LabelStmt:
-      return cxstring::createRef("LabelStmt");
+    return cxstring::createRef("LabelStmt");
   case CXCursor_CompoundStmt:
-      return cxstring::createRef("CompoundStmt");
+    return cxstring::createRef("CompoundStmt");
   case CXCursor_CaseStmt:
-      return cxstring::createRef("CaseStmt");
+    return cxstring::createRef("CaseStmt");
   case CXCursor_DefaultStmt:
-      return cxstring::createRef("DefaultStmt");
+    return cxstring::createRef("DefaultStmt");
   case CXCursor_IfStmt:
-      return cxstring::createRef("IfStmt");
+    return cxstring::createRef("IfStmt");
   case CXCursor_SwitchStmt:
-      return cxstring::createRef("SwitchStmt");
+    return cxstring::createRef("SwitchStmt");
   case CXCursor_WhileStmt:
-      return cxstring::createRef("WhileStmt");
+    return cxstring::createRef("WhileStmt");
   case CXCursor_DoStmt:
-      return cxstring::createRef("DoStmt");
+    return cxstring::createRef("DoStmt");
   case CXCursor_ForStmt:
-      return cxstring::createRef("ForStmt");
+    return cxstring::createRef("ForStmt");
   case CXCursor_GotoStmt:
-      return cxstring::createRef("GotoStmt");
+    return cxstring::createRef("GotoStmt");
   case CXCursor_IndirectGotoStmt:
-      return cxstring::createRef("IndirectGotoStmt");
+    return cxstring::createRef("IndirectGotoStmt");
   case CXCursor_ContinueStmt:
-      return cxstring::createRef("ContinueStmt");
+    return cxstring::createRef("ContinueStmt");
   case CXCursor_BreakStmt:
-      return cxstring::createRef("BreakStmt");
+    return cxstring::createRef("BreakStmt");
   case CXCursor_ReturnStmt:
-      return cxstring::createRef("ReturnStmt");
+    return cxstring::createRef("ReturnStmt");
   case CXCursor_GCCAsmStmt:
-      return cxstring::createRef("GCCAsmStmt");
+    return cxstring::createRef("GCCAsmStmt");
   case CXCursor_MSAsmStmt:
-      return cxstring::createRef("MSAsmStmt");
+    return cxstring::createRef("MSAsmStmt");
   case CXCursor_ObjCAtTryStmt:
-      return cxstring::createRef("ObjCAtTryStmt");
+    return cxstring::createRef("ObjCAtTryStmt");
   case CXCursor_ObjCAtCatchStmt:
-      return cxstring::createRef("ObjCAtCatchStmt");
+    return cxstring::createRef("ObjCAtCatchStmt");
   case CXCursor_ObjCAtFinallyStmt:
-      return cxstring::createRef("ObjCAtFinallyStmt");
+    return cxstring::createRef("ObjCAtFinallyStmt");
   case CXCursor_ObjCAtThrowStmt:
-      return cxstring::createRef("ObjCAtThrowStmt");
+    return cxstring::createRef("ObjCAtThrowStmt");
   case CXCursor_ObjCAtSynchronizedStmt:
-      return cxstring::createRef("ObjCAtSynchronizedStmt");
+    return cxstring::createRef("ObjCAtSynchronizedStmt");
   case CXCursor_ObjCAutoreleasePoolStmt:
-      return cxstring::createRef("ObjCAutoreleasePoolStmt");
+    return cxstring::createRef("ObjCAutoreleasePoolStmt");
   case CXCursor_ObjCForCollectionStmt:
-      return cxstring::createRef("ObjCForCollectionStmt");
+    return cxstring::createRef("ObjCForCollectionStmt");
   case CXCursor_CXXCatchStmt:
-      return cxstring::createRef("CXXCatchStmt");
+    return cxstring::createRef("CXXCatchStmt");
   case CXCursor_CXXTryStmt:
-      return cxstring::createRef("CXXTryStmt");
+    return cxstring::createRef("CXXTryStmt");
   case CXCursor_CXXForRangeStmt:
-      return cxstring::createRef("CXXForRangeStmt");
+    return cxstring::createRef("CXXForRangeStmt");
   case CXCursor_SEHTryStmt:
-      return cxstring::createRef("SEHTryStmt");
+    return cxstring::createRef("SEHTryStmt");
   case CXCursor_SEHExceptStmt:
-      return cxstring::createRef("SEHExceptStmt");
+    return cxstring::createRef("SEHExceptStmt");
   case CXCursor_SEHFinallyStmt:
-      return cxstring::createRef("SEHFinallyStmt");
+    return cxstring::createRef("SEHFinallyStmt");
   case CXCursor_SEHLeaveStmt:
-      return cxstring::createRef("SEHLeaveStmt");
+    return cxstring::createRef("SEHLeaveStmt");
   case CXCursor_NullStmt:
-      return cxstring::createRef("NullStmt");
+    return cxstring::createRef("NullStmt");
   case CXCursor_InvalidFile:
-      return cxstring::createRef("InvalidFile");
+    return cxstring::createRef("InvalidFile");
   case CXCursor_InvalidCode:
     return cxstring::createRef("InvalidCode");
   case CXCursor_NoDeclFound:
-      return cxstring::createRef("NoDeclFound");
+    return cxstring::createRef("NoDeclFound");
   case CXCursor_NotImplemented:
-      return cxstring::createRef("NotImplemented");
+    return cxstring::createRef("NotImplemented");
   case CXCursor_TranslationUnit:
-      return cxstring::createRef("TranslationUnit");
+    return cxstring::createRef("TranslationUnit");
   case CXCursor_UnexposedAttr:
-      return cxstring::createRef("UnexposedAttr");
+    return cxstring::createRef("UnexposedAttr");
   case CXCursor_IBActionAttr:
-      return cxstring::createRef("attribute(ibaction)");
+    return cxstring::createRef("attribute(ibaction)");
   case CXCursor_IBOutletAttr:
-     return cxstring::createRef("attribute(iboutlet)");
+    return cxstring::createRef("attribute(iboutlet)");
   case CXCursor_IBOutletCollectionAttr:
-      return cxstring::createRef("attribute(iboutletcollection)");
+    return cxstring::createRef("attribute(iboutletcollection)");
   case CXCursor_CXXFinalAttr:
-      return cxstring::createRef("attribute(final)");
+    return cxstring::createRef("attribute(final)");
   case CXCursor_CXXOverrideAttr:
-      return cxstring::createRef("attribute(override)");
+    return cxstring::createRef("attribute(override)");
   case CXCursor_AnnotateAttr:
     return cxstring::createRef("attribute(annotate)");
   case CXCursor_AsmLabelAttr:
@@ -5433,7 +5412,8 @@ CXString clang_getCursorKindSpelling(enum CXCursorKind Kind) {
   case CXCursor_ObjCSubclassingRestricted:
     return cxstring::createRef("attribute(objc_subclassing_restricted)");
   case CXCursor_ObjCExplicitProtocolImpl:
-    return cxstring::createRef("attribute(objc_protocol_requires_explicit_implementation)");
+    return cxstring::createRef(
+        "attribute(objc_protocol_requires_explicit_implementation)");
   case CXCursor_ObjCDesignatedInitializer:
     return cxstring::createRef("attribute(objc_designated_initializer)");
   case CXCursor_ObjCRuntimeVisible:
@@ -5600,21 +5580,21 @@ CXString clang_getCursorKindSpelling(enum CXCursorKind Kind) {
   case CXCursor_OMPTargetTeamsDistributeSimdDirective:
     return cxstring::createRef("OMPTargetTeamsDistributeSimdDirective");
   case CXCursor_OverloadCandidate:
-      return cxstring::createRef("OverloadCandidate");
+    return cxstring::createRef("OverloadCandidate");
   case CXCursor_TypeAliasTemplateDecl:
-      return cxstring::createRef("TypeAliasTemplateDecl");
+    return cxstring::createRef("TypeAliasTemplateDecl");
   case CXCursor_StaticAssert:
-      return cxstring::createRef("StaticAssert");
+    return cxstring::createRef("StaticAssert");
   case CXCursor_FriendDecl:
-      return cxstring::createRef("FriendDecl");
+    return cxstring::createRef("FriendDecl");
   case CXCursor_ConvergentAttr:
-      return cxstring::createRef("attribute(convergent)");
+    return cxstring::createRef("attribute(convergent)");
   case CXCursor_WarnUnusedAttr:
-      return cxstring::createRef("attribute(warn_unused)");
+    return cxstring::createRef("attribute(warn_unused)");
   case CXCursor_WarnUnusedResultAttr:
-      return cxstring::createRef("attribute(warn_unused_result)");
+    return cxstring::createRef("attribute(warn_unused_result)");
   case CXCursor_AlignedAttr:
-      return cxstring::createRef("attribute(aligned)");
+    return cxstring::createRef("attribute(aligned)");
   }
 
   llvm_unreachable("Unhandled CXCursorKind");
@@ -5627,17 +5607,16 @@ struct GetCursorData {
   SourceLocation VisitedDeclaratorDeclStartLoc;
   CXCursor &BestCursor;
 
-  GetCursorData(SourceManager &SM,
-                SourceLocation tokenBegin, CXCursor &outputCursor)
-    : TokenBeginLoc(tokenBegin), BestCursor(outputCursor) {
+  GetCursorData(SourceManager &SM, SourceLocation tokenBegin,
+                CXCursor &outputCursor)
+      : TokenBeginLoc(tokenBegin), BestCursor(outputCursor) {
     PointsAtMacroArgExpansion = SM.isMacroArgExpansion(tokenBegin);
     VisitedObjCPropertyImplDecl = false;
   }
 };
 
-static enum CXChildVisitResult GetCursorVisitor(CXCursor cursor,
-                                                CXCursor parent,
-                                                CXClientData client_data) {
+static enum CXChildVisitResult
+GetCursorVisitor(CXCursor cursor, CXCursor parent, CXClientData client_data) {
   GetCursorData *Data = static_cast<GetCursorData *>(client_data);
   CXCursor *BestCursor = &Data->BestCursor;
 
@@ -5646,16 +5625,16 @@ static enum CXChildVisitResult GetCursorVisitor(CXCursor cursor,
   // cursor.
   if (cursor.kind == CXCursor_MacroExpansion && Data->PointsAtMacroArgExpansion)
     return CXChildVisit_Recurse;
-  
+
   if (clang_isDeclaration(cursor.kind)) {
     // Avoid having the implicit methods override the property decls.
-    if (const ObjCMethodDecl *MD
-          = dyn_cast_or_null<ObjCMethodDecl>(getCursorDecl(cursor))) {
+    if (const ObjCMethodDecl *MD =
+            dyn_cast_or_null<ObjCMethodDecl>(getCursorDecl(cursor))) {
       if (MD->isImplicit())
         return CXChildVisit_Break;
 
-    } else if (const ObjCInterfaceDecl *ID
-                 = dyn_cast_or_null<ObjCInterfaceDecl>(getCursorDecl(cursor))) {
+    } else if (const ObjCInterfaceDecl *ID =
+                   dyn_cast_or_null<ObjCInterfaceDecl>(getCursorDecl(cursor))) {
       // Check that when we have multiple @class references in the same line,
       // that later ones do not override the previous ones.
       // If we have:
@@ -5664,16 +5643,16 @@ static enum CXChildVisitResult GetCursorVisitor(CXCursor cursor,
       // 'Foo' even though the cursor location was at 'Foo'.
       if (BestCursor->kind == CXCursor_ObjCInterfaceDecl ||
           BestCursor->kind == CXCursor_ObjCClassRef)
-        if (const ObjCInterfaceDecl *PrevID
-             = dyn_cast_or_null<ObjCInterfaceDecl>(getCursorDecl(*BestCursor))){
-         if (PrevID != ID &&
-             !PrevID->isThisDeclarationADefinition() &&
-             !ID->isThisDeclarationADefinition())
-           return CXChildVisit_Break;
+        if (const ObjCInterfaceDecl *PrevID =
+                dyn_cast_or_null<ObjCInterfaceDecl>(
+                    getCursorDecl(*BestCursor))) {
+          if (PrevID != ID && !PrevID->isThisDeclarationADefinition() &&
+              !ID->isThisDeclarationADefinition())
+            return CXChildVisit_Break;
         }
 
-    } else if (const DeclaratorDecl *DD
-                    = dyn_cast_or_null<DeclaratorDecl>(getCursorDecl(cursor))) {
+    } else if (const DeclaratorDecl *DD =
+                   dyn_cast_or_null<DeclaratorDecl>(getCursorDecl(cursor))) {
       SourceLocation StartLoc = DD->getSourceRange().getBegin();
       // Check that when we have multiple declarators in the same line,
       // that later ones do not override the previous ones.
@@ -5685,8 +5664,9 @@ static enum CXChildVisitResult GetCursorVisitor(CXCursor cursor,
         return CXChildVisit_Break;
       Data->VisitedDeclaratorDeclStartLoc = StartLoc;
 
-    } else if (const ObjCPropertyImplDecl *PropImp
-              = dyn_cast_or_null<ObjCPropertyImplDecl>(getCursorDecl(cursor))) {
+    } else if (const ObjCPropertyImplDecl *PropImp =
+                   dyn_cast_or_null<ObjCPropertyImplDecl>(
+                       getCursorDecl(cursor))) {
       (void)PropImp;
       // Check that when we have multiple @synthesize in the same line,
       // that later ones do not override the previous ones.
@@ -5707,15 +5687,16 @@ static enum CXChildVisitResult GetCursorVisitor(CXCursor cursor,
       // when the expression source range overlaps the declaration range.
       // This can happen for C++ constructor expressions whose range generally
       // include the variable declaration, e.g.:
-      //  MyCXXClass foo; // Make sure pointing at 'foo' returns a VarDecl cursor.
+      //  MyCXXClass foo; // Make sure pointing at 'foo' returns a VarDecl
+      //  cursor.
       if (D->getLocation().isValid() && Data->TokenBeginLoc.isValid() &&
           D->getLocation() == Data->TokenBeginLoc)
         return CXChildVisit_Break;
     }
   }
 
-  // If our current best cursor is the construction of a temporary object, 
-  // don't replace that cursor with a type reference, because we want 
+  // If our current best cursor is the construction of a temporary object,
+  // don't replace that cursor with a type reference, because we want
   // clang_getCursor() to point at the constructor.
   if (clang_isExpression(BestCursor->kind) &&
       isa<CXXTemporaryObjectExpr>(getCursorExpr(*BestCursor)) &&
@@ -5753,42 +5734,42 @@ CXCursor clang_getCursor(CXTranslationUnit TU, CXSourceLocation Loc) {
     CXFile ResultFile;
     unsigned ResultLine, ResultColumn;
     CXString SearchFileName, ResultFileName, KindSpelling, USR;
-    const char *IsDef = clang_isCursorDefinition(Result)? " (Definition)" : "";
+    const char *IsDef = clang_isCursorDefinition(Result) ? " (Definition)" : "";
     CXSourceLocation ResultLoc = clang_getCursorLocation(Result);
 
     clang_getFileLocation(Loc, &SearchFile, &SearchLine, &SearchColumn,
                           nullptr);
-    clang_getFileLocation(ResultLoc, &ResultFile, &ResultLine,
-                          &ResultColumn, nullptr);
+    clang_getFileLocation(ResultLoc, &ResultFile, &ResultLine, &ResultColumn,
+                          nullptr);
     SearchFileName = clang_getFileName(SearchFile);
     ResultFileName = clang_getFileName(ResultFile);
     KindSpelling = clang_getCursorKindSpelling(Result.kind);
     USR = clang_getCursorUSR(Result);
-    *Log << llvm::format("(%s:%d:%d) = %s",
-                   clang_getCString(SearchFileName), SearchLine, SearchColumn,
-                   clang_getCString(KindSpelling))
-        << llvm::format("(%s:%d:%d):%s%s",
-                     clang_getCString(ResultFileName), ResultLine, ResultColumn,
-                     clang_getCString(USR), IsDef);
+    *Log << llvm::format("(%s:%d:%d) = %s", clang_getCString(SearchFileName),
+                         SearchLine, SearchColumn,
+                         clang_getCString(KindSpelling))
+         << llvm::format("(%s:%d:%d):%s%s", clang_getCString(ResultFileName),
+                         ResultLine, ResultColumn, clang_getCString(USR),
+                         IsDef);
     clang_disposeString(SearchFileName);
     clang_disposeString(ResultFileName);
     clang_disposeString(KindSpelling);
     clang_disposeString(USR);
-    
+
     CXCursor Definition = clang_getCursorDefinition(Result);
     if (!clang_equalCursors(Definition, clang_getNullCursor())) {
       CXSourceLocation DefinitionLoc = clang_getCursorLocation(Definition);
-      CXString DefinitionKindSpelling
-                                = clang_getCursorKindSpelling(Definition.kind);
+      CXString DefinitionKindSpelling =
+          clang_getCursorKindSpelling(Definition.kind);
       CXFile DefinitionFile;
       unsigned DefinitionLine, DefinitionColumn;
-      clang_getFileLocation(DefinitionLoc, &DefinitionFile,
-                            &DefinitionLine, &DefinitionColumn, nullptr);
+      clang_getFileLocation(DefinitionLoc, &DefinitionFile, &DefinitionLine,
+                            &DefinitionColumn, nullptr);
       CXString DefinitionFileName = clang_getFileName(DefinitionFile);
       *Log << llvm::format("  -> %s(%s:%d:%d)",
-                     clang_getCString(DefinitionKindSpelling),
-                     clang_getCString(DefinitionFileName),
-                     DefinitionLine, DefinitionColumn);
+                           clang_getCString(DefinitionKindSpelling),
+                           clang_getCString(DefinitionFileName), DefinitionLine,
+                           DefinitionColumn);
       clang_disposeString(DefinitionFileName);
       clang_disposeString(DefinitionKindSpelling);
     }
@@ -5821,9 +5802,9 @@ unsigned clang_hashCursor(CXCursor C) {
   unsigned Index = 0;
   if (clang_isExpression(C.kind) || clang_isStatement(C.kind))
     Index = 1;
-  
-  return llvm::DenseMapInfo<std::pair<unsigned, const void*> >::getHashValue(
-                                        std::make_pair(C.kind, C.data[Index]));
+
+  return llvm::DenseMapInfo<std::pair<unsigned, const void *>>::getHashValue(
+      std::make_pair(C.kind, C.data[Index]));
 }
 
 unsigned clang_isInvalid(enum CXCursorKind K) {
@@ -5857,7 +5838,7 @@ unsigned clang_isStatement(enum CXCursorKind K) {
 }
 
 unsigned clang_isAttribute(enum CXCursorKind K) {
-    return K >= CXCursor_FirstAttr && K <= CXCursor_LastAttr;
+  return K >= CXCursor_FirstAttr && K <= CXCursor_LastAttr;
 }
 
 unsigned clang_isTranslationUnit(enum CXCursorKind K) {
@@ -5867,41 +5848,39 @@ unsigned clang_isTranslationUnit(enum CXCursorKind K) {
 unsigned clang_isPreprocessing(enum CXCursorKind K) {
   return K >= CXCursor_FirstPreprocessing && K <= CXCursor_LastPreprocessing;
 }
-  
+
 unsigned clang_isUnexposed(enum CXCursorKind K) {
   switch (K) {
-    case CXCursor_UnexposedDecl:
-    case CXCursor_UnexposedExpr:
-    case CXCursor_UnexposedStmt:
-    case CXCursor_UnexposedAttr:
-      return true;
-    default:
-      return false;
+  case CXCursor_UnexposedDecl:
+  case CXCursor_UnexposedExpr:
+  case CXCursor_UnexposedStmt:
+  case CXCursor_UnexposedAttr:
+    return true;
+  default:
+    return false;
   }
 }
 
-CXCursorKind clang_getCursorKind(CXCursor C) {
-  return C.kind;
-}
+CXCursorKind clang_getCursorKind(CXCursor C) { return C.kind; }
 
 CXSourceLocation clang_getCursorLocation(CXCursor C) {
   if (clang_isReference(C.kind)) {
     switch (C.kind) {
     case CXCursor_ObjCSuperClassRef: {
-      std::pair<const ObjCInterfaceDecl *, SourceLocation> P
-        = getCursorObjCSuperClassRef(C);
+      std::pair<const ObjCInterfaceDecl *, SourceLocation> P =
+          getCursorObjCSuperClassRef(C);
       return cxloc::translateSourceLocation(P.first->getASTContext(), P.second);
     }
 
     case CXCursor_ObjCProtocolRef: {
-      std::pair<const ObjCProtocolDecl *, SourceLocation> P
-        = getCursorObjCProtocolRef(C);
+      std::pair<const ObjCProtocolDecl *, SourceLocation> P =
+          getCursorObjCProtocolRef(C);
       return cxloc::translateSourceLocation(P.first->getASTContext(), P.second);
     }
 
     case CXCursor_ObjCClassRef: {
-      std::pair<const ObjCInterfaceDecl *, SourceLocation> P
-        = getCursorObjCClassRef(C);
+      std::pair<const ObjCInterfaceDecl *, SourceLocation> P =
+          getCursorObjCClassRef(C);
       return cxloc::translateSourceLocation(P.first->getASTContext(), P.second);
     }
 
@@ -5935,10 +5914,10 @@ CXSourceLocation clang_getCursorLocation(CXCursor C) {
       const CXXBaseSpecifier *BaseSpec = getCursorCXXBaseSpecifier(C);
       if (!BaseSpec)
         return clang_getNullLocation();
-      
+
       if (TypeSourceInfo *TSInfo = BaseSpec->getTypeSourceInfo())
-        return cxloc::translateSourceLocation(getCursorContext(C),
-                                            TSInfo->getTypeLoc().getBeginLoc());
+        return cxloc::translateSourceLocation(
+            getCursorContext(C), TSInfo->getTypeLoc().getBeginLoc());
 
       return cxloc::translateSourceLocation(getCursorContext(C),
                                             BaseSpec->getBeginLoc());
@@ -5950,8 +5929,8 @@ CXSourceLocation clang_getCursorLocation(CXCursor C) {
     }
 
     case CXCursor_OverloadedDeclRef:
-      return cxloc::translateSourceLocation(getCursorContext(C),
-                                          getCursorOverloadedDeclRef(C).second);
+      return cxloc::translateSourceLocation(
+          getCursorContext(C), getCursorOverloadedDeclRef(C).second);
 
     default:
       // FIXME: Need a way to enumerate all non-reference cases.
@@ -5960,8 +5939,8 @@ CXSourceLocation clang_getCursorLocation(CXCursor C) {
   }
 
   if (clang_isExpression(C.kind))
-    return cxloc::translateSourceLocation(getCursorContext(C),
-                                   getLocationFromExpr(getCursorExpr(C)));
+    return cxloc::translateSourceLocation(
+        getCursorContext(C), getLocationFromExpr(getCursorExpr(C)));
 
   if (clang_isStatement(C.kind))
     return cxloc::translateSourceLocation(getCursorContext(C),
@@ -5973,8 +5952,8 @@ CXSourceLocation clang_getCursorLocation(CXCursor C) {
   }
 
   if (C.kind == CXCursor_MacroExpansion) {
-    SourceLocation L
-      = cxcursor::getCursorMacroExpansion(C).getSourceRange().getBegin();
+    SourceLocation L =
+        cxcursor::getCursorMacroExpansion(C).getSourceRange().getBegin();
     return cxloc::translateSourceLocation(getCursorContext(C), L);
   }
 
@@ -5984,14 +5963,13 @@ CXSourceLocation clang_getCursorLocation(CXCursor C) {
   }
 
   if (C.kind == CXCursor_InclusionDirective) {
-    SourceLocation L
-      = cxcursor::getCursorInclusionDirective(C)->getSourceRange().getBegin();
+    SourceLocation L =
+        cxcursor::getCursorInclusionDirective(C)->getSourceRange().getBegin();
     return cxloc::translateSourceLocation(getCursorContext(C), L);
   }
 
   if (clang_isAttribute(C.kind)) {
-    SourceLocation L
-      = cxcursor::getCursorAttr(C)->getLocation();
+    SourceLocation L = cxcursor::getCursorAttr(C)->getLocation();
     return cxloc::translateSourceLocation(getCursorContext(C), L);
   }
 
@@ -6036,12 +6014,12 @@ CXCursor cxcursor::getCursor(CXTranslationUnit TU, SourceLocation SLoc) {
   // the token under the cursor.
   SLoc = Lexer::GetBeginningOfToken(SLoc, CXXUnit->getSourceManager(),
                                     CXXUnit->getASTContext().getLangOpts());
-  
+
   CXCursor Result = MakeCXCursorInvalid(CXCursor_NoDeclFound);
   if (SLoc.isValid()) {
     GetCursorData ResultData(CXXUnit->getSourceManager(), SLoc, Result);
     CursorVisitor CursorVis(TU, GetCursorVisitor, &ResultData,
-                            /*VisitPreprocessorLast=*/true, 
+                            /*VisitPreprocessorLast=*/true,
                             /*VisitIncludedEntities=*/false,
                             SourceLocation(SLoc));
     CursorVis.visitFileRegion();
@@ -6054,7 +6032,7 @@ static SourceRange getRawCursorExtent(CXCursor C) {
   if (clang_isReference(C.kind)) {
     switch (C.kind) {
     case CXCursor_ObjCSuperClassRef:
-      return  getCursorObjCSuperClassRef(C).second;
+      return getCursorObjCSuperClassRef(C).second;
 
     case CXCursor_ObjCProtocolRef:
       return getCursorObjCProtocolRef(C).second;
@@ -6085,7 +6063,7 @@ static SourceRange getRawCursorExtent(CXCursor C) {
 
     case CXCursor_VariableRef:
       return getCursorVariableRef(C).second;
-        
+
     default:
       // FIXME: Need a way to enumerate all non-reference cases.
       llvm_unreachable("Missed a reference kind");
@@ -6118,7 +6096,8 @@ static SourceRange getRawCursorExtent(CXCursor C) {
 
   if (C.kind == CXCursor_InclusionDirective) {
     ASTUnit *TU = getCursorASTUnit(C);
-    SourceRange Range = cxcursor::getCursorInclusionDirective(C)->getSourceRange();
+    SourceRange Range =
+        cxcursor::getCursorInclusionDirective(C)->getSourceRange();
     return TU->mapRangeFromPreamble(Range);
   }
 
@@ -6185,9 +6164,9 @@ static SourceRange getFullCursorExtent(CXCursor C, SourceManager &SrcMgr) {
         R.setBegin(VD->getLocation());
     }
 
-    return R;    
+    return R;
   }
-  
+
   return getRawCursorExtent(C);
 }
 
@@ -6214,10 +6193,10 @@ CXCursor clang_getCursorReferenced(CXCursor C) {
             dyn_cast<ObjCPropertyImplDecl>(D))
       if (ObjCPropertyDecl *Property = PropImpl->getPropertyDecl())
         return MakeCXCursor(Property, tu);
-    
+
     return C;
   }
-  
+
   if (clang_isExpression(C.kind)) {
     const Expr *E = getCursorExpr(C);
     const Decl *D = getDeclFromExpr(E);
@@ -6227,10 +6206,10 @@ CXCursor clang_getCursorReferenced(CXCursor C) {
                                                declCursor);
       return declCursor;
     }
-    
+
     if (const OverloadExpr *Ovl = dyn_cast_or_null<OverloadExpr>(E))
       return MakeCursorOverloadedDeclRef(Ovl, tu);
-        
+
     return clang_getNullCursor();
   }
 
@@ -6239,7 +6218,7 @@ CXCursor clang_getCursorReferenced(CXCursor C) {
     if (const GotoStmt *Goto = dyn_cast_or_null<GotoStmt>(S))
       if (LabelDecl *label = Goto->getLabel())
         if (LabelStmt *labelS = label->getStmt())
-        return MakeCXCursor(labelS, getCursorDecl(C), tu);
+          return MakeCXCursor(labelS, getCursorDecl(C), tu);
 
     return clang_getNullCursor();
   }
@@ -6254,60 +6233,58 @@ CXCursor clang_getCursorReferenced(CXCursor C) {
     return clang_getNullCursor();
 
   switch (C.kind) {
-    case CXCursor_ObjCSuperClassRef:
-      return MakeCXCursor(getCursorObjCSuperClassRef(C).first, tu);
+  case CXCursor_ObjCSuperClassRef:
+    return MakeCXCursor(getCursorObjCSuperClassRef(C).first, tu);
 
-    case CXCursor_ObjCProtocolRef: {
-      const ObjCProtocolDecl *Prot = getCursorObjCProtocolRef(C).first;
-      if (const ObjCProtocolDecl *Def = Prot->getDefinition())
-        return MakeCXCursor(Def, tu);
+  case CXCursor_ObjCProtocolRef: {
+    const ObjCProtocolDecl *Prot = getCursorObjCProtocolRef(C).first;
+    if (const ObjCProtocolDecl *Def = Prot->getDefinition())
+      return MakeCXCursor(Def, tu);
 
-      return MakeCXCursor(Prot, tu);
-    }
+    return MakeCXCursor(Prot, tu);
+  }
 
-    case CXCursor_ObjCClassRef: {
-      const ObjCInterfaceDecl *Class = getCursorObjCClassRef(C).first;
-      if (const ObjCInterfaceDecl *Def = Class->getDefinition())
-        return MakeCXCursor(Def, tu);
+  case CXCursor_ObjCClassRef: {
+    const ObjCInterfaceDecl *Class = getCursorObjCClassRef(C).first;
+    if (const ObjCInterfaceDecl *Def = Class->getDefinition())
+      return MakeCXCursor(Def, tu);
 
-      return MakeCXCursor(Class, tu);
-    }
+    return MakeCXCursor(Class, tu);
+  }
 
-    case CXCursor_TypeRef:
-      return MakeCXCursor(getCursorTypeRef(C).first, tu );
+  case CXCursor_TypeRef:
+    return MakeCXCursor(getCursorTypeRef(C).first, tu);
 
-    case CXCursor_TemplateRef:
-      return MakeCXCursor(getCursorTemplateRef(C).first, tu );
+  case CXCursor_TemplateRef:
+    return MakeCXCursor(getCursorTemplateRef(C).first, tu);
 
-    case CXCursor_NamespaceRef:
-      return MakeCXCursor(getCursorNamespaceRef(C).first, tu );
+  case CXCursor_NamespaceRef:
+    return MakeCXCursor(getCursorNamespaceRef(C).first, tu);
 
-    case CXCursor_MemberRef:
-      return MakeCXCursor(getCursorMemberRef(C).first, tu );
+  case CXCursor_MemberRef:
+    return MakeCXCursor(getCursorMemberRef(C).first, tu);
 
-    case CXCursor_CXXBaseSpecifier: {
-      const CXXBaseSpecifier *B = cxcursor::getCursorCXXBaseSpecifier(C);
-      return clang_getTypeDeclaration(cxtype::MakeCXType(B->getType(),
-                                                         tu ));
-    }
+  case CXCursor_CXXBaseSpecifier: {
+    const CXXBaseSpecifier *B = cxcursor::getCursorCXXBaseSpecifier(C);
+    return clang_getTypeDeclaration(cxtype::MakeCXType(B->getType(), tu));
+  }
 
-    case CXCursor_LabelRef:
-      // FIXME: We end up faking the "parent" declaration here because we
-      // don't want to make CXCursor larger.
-      return MakeCXCursor(getCursorLabelRef(C).first,
-                          cxtu::getASTUnit(tu)->getASTContext()
-                              .getTranslationUnitDecl(),
-                          tu);
+  case CXCursor_LabelRef:
+    // FIXME: We end up faking the "parent" declaration here because we
+    // don't want to make CXCursor larger.
+    return MakeCXCursor(
+        getCursorLabelRef(C).first,
+        cxtu::getASTUnit(tu)->getASTContext().getTranslationUnitDecl(), tu);
 
-    case CXCursor_OverloadedDeclRef:
-      return C;
-      
-    case CXCursor_VariableRef:
-      return MakeCXCursor(getCursorVariableRef(C).first, tu);
+  case CXCursor_OverloadedDeclRef:
+    return C;
 
-    default:
-      // We would prefer to enumerate all non-reference cursor kinds here.
-      llvm_unreachable("Unhandled reference cursor kind");
+  case CXCursor_VariableRef:
+    return MakeCXCursor(getCursorVariableRef(C).first, tu);
+
+  default:
+    // We would prefer to enumerate all non-reference cursor kinds here.
+    llvm_unreachable("Unhandled reference cursor kind");
   }
 }
 
@@ -6363,7 +6340,7 @@ CXCursor clang_getCursorDefinition(CXCursor C) {
   case Decl::Block:
   case Decl::Captured:
   case Decl::OMPCapturedExpr:
-  case Decl::Label:  // FIXME: Is this right??
+  case Decl::Label: // FIXME: Is this right??
   case Decl::ClassScopeFunctionSpecialization:
   case Decl::CXXDeductionGuide:
   case Decl::Import:
@@ -6439,8 +6416,8 @@ CXCursor clang_getCursorDefinition(CXCursor C) {
   }
 
   case Decl::ClassTemplate: {
-    if (RecordDecl *Def = cast<ClassTemplateDecl>(D)->getTemplatedDecl()
-                                                            ->getDefinition())
+    if (RecordDecl *Def =
+            cast<ClassTemplateDecl>(D)->getTemplatedDecl()->getDefinition())
       return MakeCXCursor(cast<CXXRecordDecl>(Def)->getDescribedClassTemplate(),
                           TU);
     return clang_getNullCursor();
@@ -6454,14 +6431,13 @@ CXCursor clang_getCursorDefinition(CXCursor C) {
   }
 
   case Decl::Using:
-    return MakeCursorOverloadedDeclRef(cast<UsingDecl>(D), 
-                                       D->getLocation(), TU);
+    return MakeCursorOverloadedDeclRef(cast<UsingDecl>(D), D->getLocation(),
+                                       TU);
 
   case Decl::UsingShadow:
   case Decl::ConstructorUsingShadow:
     return clang_getCursorDefinition(
-                       MakeCXCursor(cast<UsingShadowDecl>(D)->getTargetDecl(),
-                                    TU));
+        MakeCXCursor(cast<UsingShadowDecl>(D)->getTargetDecl(), TU));
 
   case Decl::ObjCMethod: {
     const ObjCMethodDecl *Method = cast<ObjCMethodDecl>(D);
@@ -6471,11 +6447,11 @@ CXCursor clang_getCursorDefinition(CXCursor C) {
     // Dig out the method definition in the associated
     // @implementation, if we have it.
     // FIXME: The ASTs should make finding the definition easier.
-    if (const ObjCInterfaceDecl *Class
-                       = dyn_cast<ObjCInterfaceDecl>(Method->getDeclContext()))
+    if (const ObjCInterfaceDecl *Class =
+            dyn_cast<ObjCInterfaceDecl>(Method->getDeclContext()))
       if (ObjCImplementationDecl *ClassImpl = Class->getImplementation())
-        if (ObjCMethodDecl *Def = ClassImpl->getMethod(Method->getSelector(),
-                                                  Method->isInstanceMethod()))
+        if (ObjCMethodDecl *Def = ClassImpl->getMethod(
+                Method->getSelector(), Method->isInstanceMethod()))
           if (Def->isThisDeclarationADefinition())
             return MakeCXCursor(Def, TU);
 
@@ -6483,13 +6459,14 @@ CXCursor clang_getCursorDefinition(CXCursor C) {
   }
 
   case Decl::ObjCCategory:
-    if (ObjCCategoryImplDecl *Impl
-                               = cast<ObjCCategoryDecl>(D)->getImplementation())
+    if (ObjCCategoryImplDecl *Impl =
+            cast<ObjCCategoryDecl>(D)->getImplementation())
       return MakeCXCursor(Impl, TU);
     return clang_getNullCursor();
 
   case Decl::ObjCProtocol:
-    if (const ObjCProtocolDecl *Def = cast<ObjCProtocolDecl>(D)->getDefinition())
+    if (const ObjCProtocolDecl *Def =
+            cast<ObjCProtocolDecl>(D)->getDefinition())
       return MakeCXCursor(Def, TU);
     return clang_getNullCursor();
 
@@ -6514,8 +6491,8 @@ CXCursor clang_getCursorDefinition(CXCursor C) {
     return clang_getNullCursor();
 
   case Decl::ObjCCompatibleAlias:
-    if (const ObjCInterfaceDecl *Class
-          = cast<ObjCCompatibleAliasDecl>(D)->getClassInterface())
+    if (const ObjCInterfaceDecl *Class =
+            cast<ObjCCompatibleAliasDecl>(D)->getClassInterface())
       if (const ObjCInterfaceDecl *Def = Class->getDefinition())
         return MakeCXCursor(Def, TU);
 
@@ -6545,9 +6522,10 @@ unsigned clang_isCursorDefinition(CXCursor C) {
 CXCursor clang_getCanonicalCursor(CXCursor C) {
   if (!clang_isDeclaration(C.kind))
     return C;
-  
+
   if (const Decl *D = getCursorDecl(C)) {
-    if (const ObjCCategoryImplDecl *CatImplD = dyn_cast<ObjCCategoryImplDecl>(D))
+    if (const ObjCCategoryImplDecl *CatImplD =
+            dyn_cast<ObjCCategoryImplDecl>(D))
       if (ObjCCategoryDecl *CatD = CatImplD->getCategoryDecl())
         return MakeCXCursor(CatD, getCursorTU(C));
 
@@ -6557,30 +6535,30 @@ CXCursor clang_getCanonicalCursor(CXCursor C) {
 
     return MakeCXCursor(D->getCanonicalDecl(), getCursorTU(C));
   }
-  
+
   return C;
 }
 
 int clang_Cursor_getObjCSelectorIndex(CXCursor cursor) {
   return cxcursor::getSelectorIdentifierIndexAndLoc(cursor).first;
 }
-  
+
 unsigned clang_getNumOverloadedDecls(CXCursor C) {
   if (C.kind != CXCursor_OverloadedDeclRef)
     return 0;
-  
+
   OverloadedDeclRefStorage Storage = getCursorOverloadedDeclRef(C).first;
   if (const OverloadExpr *E = Storage.dyn_cast<const OverloadExpr *>())
     return E->getNumDecls();
-  
-  if (OverloadedTemplateStorage *S
-                              = Storage.dyn_cast<OverloadedTemplateStorage*>())
+
+  if (OverloadedTemplateStorage *S =
+          Storage.dyn_cast<OverloadedTemplateStorage *>())
     return S->size();
-  
+
   const Decl *D = Storage.get<const Decl *>();
   if (const UsingDecl *Using = dyn_cast<UsingDecl>(D))
     return Using->shadow_size();
-  
+
   return 0;
 }
 
@@ -6590,16 +6568,16 @@ CXCursor clang_getOverloadedDecl(CXCursor cursor, unsigned index) {
 
   if (index >= clang_getNumOverloadedDecls(cursor))
     return clang_getNullCursor();
-  
+
   CXTranslationUnit TU = getCursorTU(cursor);
   OverloadedDeclRefStorage Storage = getCursorOverloadedDeclRef(cursor).first;
   if (const OverloadExpr *E = Storage.dyn_cast<const OverloadExpr *>())
     return MakeCXCursor(E->decls_begin()[index], TU);
-  
-  if (OverloadedTemplateStorage *S
-                              = Storage.dyn_cast<OverloadedTemplateStorage*>())
+
+  if (OverloadedTemplateStorage *S =
+          Storage.dyn_cast<OverloadedTemplateStorage *>())
     return MakeCXCursor(S->begin()[index], TU);
-  
+
   const Decl *D = Storage.get<const Decl *>();
   if (const UsingDecl *Using = dyn_cast<UsingDecl>(D)) {
     // FIXME: This is, unfortunately, linear time.
@@ -6607,17 +6585,13 @@ CXCursor clang_getOverloadedDecl(CXCursor cursor, unsigned index) {
     std::advance(Pos, index);
     return MakeCXCursor(cast<UsingShadowDecl>(*Pos)->getTargetDecl(), TU);
   }
-  
+
   return clang_getNullCursor();
 }
-  
-void clang_getDefinitionSpellingAndExtent(CXCursor C,
-                                          const char **startBuf,
-                                          const char **endBuf,
-                                          unsigned *startLine,
-                                          unsigned *startColumn,
-                                          unsigned *endLine,
-                                          unsigned *endColumn) {
+
+void clang_getDefinitionSpellingAndExtent(
+    CXCursor C, const char **startBuf, const char **endBuf, unsigned *startLine,
+    unsigned *startColumn, unsigned *endLine, unsigned *endColumn) {
   assert(getCursorDecl(C) && "CXCursor has null decl");
   const FunctionDecl *FD = dyn_cast<FunctionDecl>(getCursorDecl(C));
   CompoundStmt *Body = dyn_cast<CompoundStmt>(FD->getBody());
@@ -6631,18 +6605,17 @@ void clang_getDefinitionSpellingAndExtent(CXCursor C,
   *endColumn = SM.getSpellingColumnNumber(Body->getRBracLoc());
 }
 
-
 CXSourceRange clang_getCursorReferenceNameRange(CXCursor C, unsigned NameFlags,
                                                 unsigned PieceIndex) {
   RefNamePieces Pieces;
-  
+
   switch (C.kind) {
   case CXCursor_MemberRefExpr:
     if (const MemberExpr *E = dyn_cast<MemberExpr>(getCursorExpr(C)))
       Pieces = buildPieces(NameFlags, true, E->getMemberNameInfo(),
                            E->getQualifierLoc().getSourceRange());
     break;
-  
+
   case CXCursor_DeclRefExpr:
     if (const DeclRefExpr *E = dyn_cast<DeclRefExpr>(getCursorExpr(C))) {
       SourceRange TemplateArgLoc(E->getLAngleLoc(), E->getRAngleLoc());
@@ -6651,10 +6624,10 @@ CXSourceRange clang_getCursorReferenceNameRange(CXCursor C, unsigned NameFlags,
                       E->getQualifierLoc().getSourceRange(), &TemplateArgLoc);
     }
     break;
-    
+
   case CXCursor_CallExpr:
-    if (const CXXOperatorCallExpr *OCE = 
-        dyn_cast<CXXOperatorCallExpr>(getCursorExpr(C))) {
+    if (const CXXOperatorCallExpr *OCE =
+            dyn_cast<CXXOperatorCallExpr>(getCursorExpr(C))) {
       const Expr *Callee = OCE->getCallee();
       if (const ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(Callee))
         Callee = ICE->getSubExpr();
@@ -6664,7 +6637,7 @@ CXSourceRange clang_getCursorReferenceNameRange(CXCursor C, unsigned NameFlags,
                              DRE->getQualifierLoc().getSourceRange());
     }
     break;
-    
+
   default:
     break;
   }
@@ -6673,11 +6646,11 @@ CXSourceRange clang_getCursorReferenceNameRange(CXCursor C, unsigned NameFlags,
     if (PieceIndex == 0)
       return clang_getCursorExtent(C);
   } else if (PieceIndex < Pieces.size()) {
-      SourceRange R = Pieces[PieceIndex];
-      if (R.isValid())
-        return cxloc::translateSourceRange(getCursorContext(C), R);
+    SourceRange R = Pieces[PieceIndex];
+    if (R.isValid())
+      return cxloc::translateSourceRange(getCursorContext(C), R);
   }
-  
+
   return clang_getNullRange();
 }
 
@@ -6686,7 +6659,7 @@ void clang_enableStackTraces(void) {
   llvm::sys::PrintStackTraceOnErrorSignal(StringRef());
 }
 
-void clang_executeOnThread(void (*fn)(void*), void *user_data,
+void clang_executeOnThread(void (*fn)(void *), void *user_data,
                            unsigned stack_size) {
   llvm::llvm_execute_on_thread(fn, user_data,
                                stack_size == 0
@@ -6715,8 +6688,8 @@ CXString clang_getTokenSpelling(CXTranslationUnit TU, CXToken CXTok) {
   case CXToken_Identifier:
   case CXToken_Keyword:
     // We know we have an IdentifierInfo*, so use that.
-    return cxstring::createRef(static_cast<IdentifierInfo *>(CXTok.ptr_data)
-                            ->getNameStart());
+    return cxstring::createRef(
+        static_cast<IdentifierInfo *>(CXTok.ptr_data)->getNameStart());
 
   case CXToken_Literal: {
     // We have stashed the starting pointer in the ptr_data field. Use it.
@@ -6741,11 +6714,11 @@ CXString clang_getTokenSpelling(CXTranslationUnit TU, CXToken CXTok) {
     return cxstring::createEmpty();
 
   SourceLocation Loc = SourceLocation::getFromRawEncoding(CXTok.int_data[1]);
-  std::pair<FileID, unsigned> LocInfo
-    = CXXUnit->getSourceManager().getDecomposedSpellingLoc(Loc);
+  std::pair<FileID, unsigned> LocInfo =
+      CXXUnit->getSourceManager().getDecomposedSpellingLoc(Loc);
   bool Invalid = false;
-  StringRef Buffer
-    = CXXUnit->getSourceManager().getBufferData(LocInfo.first, &Invalid);
+  StringRef Buffer =
+      CXXUnit->getSourceManager().getBufferData(LocInfo.first, &Invalid);
   if (Invalid)
     return cxstring::createEmpty();
 
@@ -6762,8 +6735,9 @@ CXSourceLocation clang_getTokenLocation(CXTranslationUnit TU, CXToken CXTok) {
   if (!CXXUnit)
     return clang_getNullLocation();
 
-  return cxloc::translateSourceLocation(CXXUnit->getASTContext(),
-                        SourceLocation::getFromRawEncoding(CXTok.int_data[1]));
+  return cxloc::translateSourceLocation(
+      CXXUnit->getASTContext(),
+      SourceLocation::getFromRawEncoding(CXTok.int_data[1]));
 }
 
 CXSourceRange clang_getTokenExtent(CXTranslationUnit TU, CXToken CXTok) {
@@ -6776,17 +6750,18 @@ CXSourceRange clang_getTokenExtent(CXTranslationUnit TU, CXToken CXTok) {
   if (!CXXUnit)
     return clang_getNullRange();
 
-  return cxloc::translateSourceRange(CXXUnit->getASTContext(),
-                        SourceLocation::getFromRawEncoding(CXTok.int_data[1]));
+  return cxloc::translateSourceRange(
+      CXXUnit->getASTContext(),
+      SourceLocation::getFromRawEncoding(CXTok.int_data[1]));
 }
 
 static void getTokens(ASTUnit *CXXUnit, SourceRange Range,
                       SmallVectorImpl<CXToken> &CXTokens) {
   SourceManager &SourceMgr = CXXUnit->getSourceManager();
-  std::pair<FileID, unsigned> BeginLocInfo
-    = SourceMgr.getDecomposedSpellingLoc(Range.getBegin());
-  std::pair<FileID, unsigned> EndLocInfo
-    = SourceMgr.getDecomposedSpellingLoc(Range.getEnd());
+  std::pair<FileID, unsigned> BeginLocInfo =
+      SourceMgr.getDecomposedSpellingLoc(Range.getBegin());
+  std::pair<FileID, unsigned> EndLocInfo =
+      SourceMgr.getDecomposedSpellingLoc(Range.getEnd());
 
   // Cannot tokenize across files.
   if (BeginLocInfo.first != EndLocInfo.first)
@@ -6794,14 +6769,13 @@ static void getTokens(ASTUnit *CXXUnit, SourceRange Range,
 
   // Create a lexer
   bool Invalid = false;
-  StringRef Buffer
-    = SourceMgr.getBufferData(BeginLocInfo.first, &Invalid);
+  StringRef Buffer = SourceMgr.getBufferData(BeginLocInfo.first, &Invalid);
   if (Invalid)
     return;
-  
+
   Lexer Lex(SourceMgr.getLocForStartOfFile(BeginLocInfo.first),
-            CXXUnit->getASTContext().getLangOpts(),
-            Buffer.begin(), Buffer.data() + BeginLocInfo.second, Buffer.end());
+            CXXUnit->getASTContext().getLangOpts(), Buffer.begin(),
+            Buffer.data() + BeginLocInfo.second, Buffer.end());
   Lex.SetCommentRetentionState(true);
 
   // Lex tokens until we hit the end of the range.
@@ -6828,16 +6802,13 @@ static void getTokens(ASTUnit *CXXUnit, SourceRange Range,
       CXTok.ptr_data = const_cast<char *>(Tok.getLiteralData());
     } else if (Tok.is(tok::raw_identifier)) {
       // Lookup the identifier to determine whether we have a keyword.
-      IdentifierInfo *II
-        = CXXUnit->getPreprocessor().LookUpIdentifierInfo(Tok);
+      IdentifierInfo *II = CXXUnit->getPreprocessor().LookUpIdentifierInfo(Tok);
 
       if ((II->getObjCKeywordID() != tok::objc_not_keyword) && previousWasAt) {
         CXTok.int_data[0] = CXToken_Keyword;
-      }
-      else {
-        CXTok.int_data[0] = Tok.is(tok::identifier)
-          ? CXToken_Identifier
-          : CXToken_Keyword;
+      } else {
+        CXTok.int_data[0] =
+            Tok.is(tok::identifier) ? CXToken_Identifier : CXToken_Keyword;
       }
       CXTok.ptr_data = II;
     } else if (Tok.is(tok::comment)) {
@@ -6853,9 +6824,7 @@ static void getTokens(ASTUnit *CXXUnit, SourceRange Range,
 }
 
 CXToken *clang_getToken(CXTranslationUnit TU, CXSourceLocation Location) {
-  LOG_FUNC_SECTION {
-    *Log << TU << ' ' << Location;
-  }
+  LOG_FUNC_SECTION { *Log << TU << ' ' << Location; }
 
   if (isNotUsableTU(TU)) {
     LOG_BAD_TU(TU);
@@ -6871,9 +6840,11 @@ CXToken *clang_getToken(CXTranslationUnit TU, CXSourceLocation Location) {
     return NULL;
   SourceManager &SM = CXXUnit->getSourceManager();
   std::pair<FileID, unsigned> DecomposedEnd = SM.getDecomposedLoc(Begin);
-  DecomposedEnd.second += Lexer::MeasureTokenLength(Begin, SM, CXXUnit->getLangOpts());
+  DecomposedEnd.second +=
+      Lexer::MeasureTokenLength(Begin, SM, CXXUnit->getLangOpts());
 
-  SourceLocation End = SM.getComposedLoc(DecomposedEnd.first, DecomposedEnd.second);
+  SourceLocation End =
+      SM.getComposedLoc(DecomposedEnd.first, DecomposedEnd.second);
 
   SmallVector<CXToken, 32> CXTokens;
   getTokens(CXXUnit, SourceRange(Begin, End), CXTokens);
@@ -6888,11 +6859,9 @@ CXToken *clang_getToken(CXTranslationUnit TU, CXSourceLocation Location) {
   return Token;
 }
 
-void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range,
-                    CXToken **Tokens, unsigned *NumTokens) {
-  LOG_FUNC_SECTION {
-    *Log << TU << ' ' << Range;
-  }
+void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range, CXToken **Tokens,
+                    unsigned *NumTokens) {
+  LOG_FUNC_SECTION { *Log << TU << ' ' << Range; }
 
   if (Tokens)
     *Tokens = nullptr;
@@ -6909,7 +6878,7 @@ void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range,
     return;
 
   ASTUnit::ConcurrencyCheck Check(*CXXUnit);
-  
+
   SourceRange R = cxloc::translateCXSourceRange(Range);
   if (R.isInvalid())
     return;
@@ -6926,8 +6895,8 @@ void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range,
   *NumTokens = CXTokens.size();
 }
 
-void clang_disposeTokens(CXTranslationUnit TU,
-                         CXToken *Tokens, unsigned NumTokens) {
+void clang_disposeTokens(CXTranslationUnit TU, CXToken *Tokens,
+                         unsigned NumTokens) {
   free(Tokens);
 }
 
@@ -6995,17 +6964,15 @@ class AnnotateTokensWorker {
 public:
   AnnotateTokensWorker(CXToken *tokens, CXCursor *cursors, unsigned numTokens,
                        CXTranslationUnit TU, SourceRange RegionOfInterest)
-    : Tokens(tokens), Cursors(cursors),
-      NumTokens(numTokens), TokIdx(0), PreprocessingTokIdx(0),
-      AnnotateVis(TU,
-                  AnnotateTokensVisitor, this,
-                  /*VisitPreprocessorLast=*/true,
-                  /*VisitIncludedEntities=*/false,
-                  RegionOfInterest,
-                  /*VisitDeclsOnly=*/false,
-                  AnnotateTokensPostChildrenVisitor),
-      SrcMgr(cxtu::getASTUnit(TU)->getSourceManager()),
-      HasContextSensitiveKeywords(false) { }
+      : Tokens(tokens), Cursors(cursors), NumTokens(numTokens), TokIdx(0),
+        PreprocessingTokIdx(0),
+        AnnotateVis(TU, AnnotateTokensVisitor, this,
+                    /*VisitPreprocessorLast=*/true,
+                    /*VisitIncludedEntities=*/false, RegionOfInterest,
+                    /*VisitDeclsOnly=*/false,
+                    AnnotateTokensPostChildrenVisitor),
+        SrcMgr(cxtu::getASTUnit(TU)->getSourceManager()),
+        HasContextSensitiveKeywords(false) {}
 
   void VisitChildren(CXCursor C) { AnnotateVis.VisitChildren(C); }
   enum CXChildVisitResult Visit(CXCursor cursor, CXCursor parent);
@@ -7017,18 +6984,16 @@ class AnnotateTokensWorker {
   void HandlePostPonedChildCursor(CXCursor Cursor, unsigned StartTokenIndex);
 
   void AnnotateTokens();
-  
-  /// Determine whether the annotator saw any cursors that have 
+
+  /// Determine whether the annotator saw any cursors that have
   /// context-sensitive keywords.
   bool hasContextSensitiveKeywords() const {
     return HasContextSensitiveKeywords;
   }
 
-  ~AnnotateTokensWorker() {
-    assert(PostChildrenInfos.empty());
-  }
+  ~AnnotateTokensWorker() { assert(PostChildrenInfos.empty()); }
 };
-}
+} // namespace
 
 void AnnotateTokensWorker::AnnotateTokens() {
   // Walk the AST within the region of interest, annotating tokens
@@ -7110,9 +7075,8 @@ static inline void updateCursorAnnotation(CXCursor &Cursor,
 ///
 /// Pass RangeBefore to annotate tokens with a cursor until a range is reached.
 /// Pass RangeOverlap to annotate tokens inside a range.
-void AnnotateTokensWorker::annotateAndAdvanceTokens(CXCursor updateC,
-                                               RangeComparisonResult compResult,
-                                               SourceRange range) {
+void AnnotateTokensWorker::annotateAndAdvanceTokens(
+    CXCursor updateC, RangeComparisonResult compResult, SourceRange range) {
   while (MoreTokens()) {
     const unsigned I = NextToken();
     if (isFunctionMacroToken(I))
@@ -7132,9 +7096,7 @@ void AnnotateTokensWorker::annotateAndAdvanceTokens(CXCursor updateC,
 /// Special annotation handling for macro argument tokens.
 /// \returns true if it advanced beyond all macro tokens, false otherwise.
 bool AnnotateTokensWorker::annotateAndAdvanceFunctionMacroTokens(
-                                               CXCursor updateC,
-                                               RangeComparisonResult compResult,
-                                               SourceRange range) {
+    CXCursor updateC, RangeComparisonResult compResult, SourceRange range) {
   assert(MoreTokens());
   assert(isFunctionMacroToken(NextToken()) &&
          "Should be called only for macro arg tokens");
@@ -7148,7 +7110,7 @@ bool AnnotateTokensWorker::annotateAndAdvanceFunctionMacroTokens(
   // annotated by a subsequent annotation try.
 
   bool atLeastOneCompFail = false;
-  
+
   unsigned I = NextToken();
   for (; I < NumTokens && isFunctionMacroToken(I); ++I) {
     SourceLocation TokLoc = getFunctionMacroTokenLoc(I);
@@ -7168,8 +7130,8 @@ bool AnnotateTokensWorker::annotateAndAdvanceFunctionMacroTokens(
   return true;
 }
 
-enum CXChildVisitResult
-AnnotateTokensWorker::Visit(CXCursor cursor, CXCursor parent) {  
+enum CXChildVisitResult AnnotateTokensWorker::Visit(CXCursor cursor,
+                                                    CXCursor parent) {
   SourceRange cursorRange = getRawCursorExtent(cursor);
   if (cursorRange.isInvalid())
     return CXChildVisit_Recurse;
@@ -7180,15 +7142,16 @@ AnnotateTokensWorker::Visit(CXCursor cursor, CXCursor parent) {
   if (!HasContextSensitiveKeywords) {
     // Objective-C properties can have context-sensitive keywords.
     if (cursor.kind == CXCursor_ObjCPropertyDecl) {
-      if (const ObjCPropertyDecl *Property
-                  = dyn_cast_or_null<ObjCPropertyDecl>(getCursorDecl(cursor)))
-        HasContextSensitiveKeywords = Property->getPropertyAttributesAsWritten() != 0;
+      if (const ObjCPropertyDecl *Property =
+              dyn_cast_or_null<ObjCPropertyDecl>(getCursorDecl(cursor)))
+        HasContextSensitiveKeywords =
+            Property->getPropertyAttributesAsWritten() != 0;
     }
     // Objective-C methods can have context-sensitive keywords.
     else if (cursor.kind == CXCursor_ObjCInstanceMethodDecl ||
              cursor.kind == CXCursor_ObjCClassMethodDecl) {
-      if (const ObjCMethodDecl *Method
-            = dyn_cast_or_null<ObjCMethodDecl>(getCursorDecl(cursor))) {
+      if (const ObjCMethodDecl *Method =
+              dyn_cast_or_null<ObjCMethodDecl>(getCursorDecl(cursor))) {
         if (Method->getObjCDeclQualifier())
           HasContextSensitiveKeywords = true;
         else {
@@ -7200,11 +7163,11 @@ AnnotateTokensWorker::Visit(CXCursor cursor, CXCursor parent) {
           }
         }
       }
-    }    
+    }
     // C++ methods can have context-sensitive keywords.
     else if (cursor.kind == CXCursor_CXXMethod) {
-      if (const CXXMethodDecl *Method
-                  = dyn_cast_or_null<CXXMethodDecl>(getCursorDecl(cursor))) {
+      if (const CXXMethodDecl *Method =
+              dyn_cast_or_null<CXXMethodDecl>(getCursorDecl(cursor))) {
         if (Method->hasAttr<FinalAttr>() || Method->hasAttr<OverrideAttr>())
           HasContextSensitiveKeywords = true;
       }
@@ -7224,8 +7187,8 @@ AnnotateTokensWorker::Visit(CXCursor cursor, CXCursor parent) {
   if (cursor.kind == CXCursor_ObjCInstanceMethodDecl &&
       parent.kind == CXCursor_ObjCPropertyDecl)
     return CXChildVisit_Continue;
-  
-  if (clang_isPreprocessing(cursor.kind)) {    
+
+  if (clang_isPreprocessing(cursor.kind)) {
     // Items in the preprocessing record are kept separate from items in
     // declarations, so we keep a separate token index.
     unsigned SavedTokIdx = TokIdx;
@@ -7246,7 +7209,7 @@ AnnotateTokensWorker::Visit(CXCursor cursor, CXCursor parent) {
       }
       break;
     }
-    
+
     // Look at all of the tokens within this range.
     while (MoreTokens()) {
       const unsigned I = NextToken();
@@ -7288,10 +7251,11 @@ AnnotateTokensWorker::Visit(CXCursor cursor, CXCursor parent) {
   const enum CXCursorKind cursorK = clang_getCursorKind(cursor);
   const enum CXCursorKind K = clang_getCursorKind(parent);
   const CXCursor updateC =
-    (clang_isInvalid(K) || K == CXCursor_TranslationUnit ||
-     // Attributes are annotated out-of-order, skip tokens until we reach it.
-     clang_isAttribute(cursor.kind))
-     ? clang_getNullCursor() : parent;
+      (clang_isInvalid(K) || K == CXCursor_TranslationUnit ||
+       // Attributes are annotated out-of-order, skip tokens until we reach it.
+       clang_isAttribute(cursor.kind))
+          ? clang_getNullCursor()
+          : parent;
 
   annotateAndAdvanceTokens(updateC, RangeBefore, cursorRange);
 
@@ -7417,13 +7381,14 @@ void AnnotateTokensWorker::HandlePostPonedChildCursor(
 static enum CXChildVisitResult AnnotateTokensVisitor(CXCursor cursor,
                                                      CXCursor parent,
                                                      CXClientData client_data) {
-  return static_cast<AnnotateTokensWorker*>(client_data)->Visit(cursor, parent);
+  return static_cast<AnnotateTokensWorker *>(client_data)
+      ->Visit(cursor, parent);
 }
 
 static bool AnnotateTokensPostChildrenVisitor(CXCursor cursor,
                                               CXClientData client_data) {
-  return static_cast<AnnotateTokensWorker*>(client_data)->
-                                                      postVisitChildren(cursor);
+  return static_cast<AnnotateTokensWorker *>(client_data)
+      ->postVisitChildren(cursor);
 }
 
 namespace {
@@ -7436,11 +7401,11 @@ class MarkMacroArgTokensVisitor {
   CXToken *Tokens;
   unsigned NumTokens;
   unsigned CurIdx;
-  
+
 public:
-  MarkMacroArgTokensVisitor(SourceManager &SM,
-                            CXToken *tokens, unsigned numTokens)
-    : SM(SM), Tokens(tokens), NumTokens(numTokens), CurIdx(0) { }
+  MarkMacroArgTokensVisitor(SourceManager &SM, CXToken *tokens,
+                            unsigned numTokens)
+      : SM(SM), Tokens(tokens), NumTokens(numTokens), CurIdx(0) {}
 
   CXChildVisitResult visit(CXCursor cursor, CXCursor parent) {
     if (cursor.kind != CXCursor_MacroExpansion)
@@ -7455,7 +7420,7 @@ class MarkMacroArgTokensVisitor {
                                         macroRange.getBegin()))
         break;
     }
-    
+
     if (CurIdx == NumTokens)
       return CXChildVisit_Break;
 
@@ -7499,14 +7464,14 @@ class MarkMacroArgTokensVisitor {
 static CXChildVisitResult
 MarkMacroArgTokensVisitorDelegate(CXCursor cursor, CXCursor parent,
                                   CXClientData client_data) {
-  return static_cast<MarkMacroArgTokensVisitor*>(client_data)->visit(cursor,
-                                                                     parent);
+  return static_cast<MarkMacroArgTokensVisitor *>(client_data)
+      ->visit(cursor, parent);
 }
 
 /// Used by \c annotatePreprocessorTokens.
 /// \returns true if lexing was finished, false otherwise.
-static bool lexNext(Lexer &Lex, Token &Tok,
-                   unsigned &NextIdx, unsigned NumTokens) {
+static bool lexNext(Lexer &Lex, Token &Tok, unsigned &NextIdx,
+                    unsigned NumTokens) {
   if (NextIdx >= NumTokens)
     return true;
 
@@ -7517,17 +7482,16 @@ static bool lexNext(Lexer &Lex, Token &Tok,
 
 static void annotatePreprocessorTokens(CXTranslationUnit TU,
                                        SourceRange RegionOfInterest,
-                                       CXCursor *Cursors,
-                                       CXToken *Tokens,
+                                       CXCursor *Cursors, CXToken *Tokens,
                                        unsigned NumTokens) {
   ASTUnit *CXXUnit = cxtu::getASTUnit(TU);
 
   Preprocessor &PP = CXXUnit->getPreprocessor();
   SourceManager &SourceMgr = CXXUnit->getSourceManager();
-  std::pair<FileID, unsigned> BeginLocInfo
-    = SourceMgr.getDecomposedSpellingLoc(RegionOfInterest.getBegin());
-  std::pair<FileID, unsigned> EndLocInfo
-    = SourceMgr.getDecomposedSpellingLoc(RegionOfInterest.getEnd());
+  std::pair<FileID, unsigned> BeginLocInfo =
+      SourceMgr.getDecomposedSpellingLoc(RegionOfInterest.getBegin());
+  std::pair<FileID, unsigned> EndLocInfo =
+      SourceMgr.getDecomposedSpellingLoc(RegionOfInterest.getEnd());
 
   if (BeginLocInfo.first != EndLocInfo.first)
     return;
@@ -7539,11 +7503,10 @@ static void annotatePreprocessorTokens(CXTranslationUnit TU,
     return;
 
   Lexer Lex(SourceMgr.getLocForStartOfFile(BeginLocInfo.first),
-            CXXUnit->getASTContext().getLangOpts(),
-            Buffer.begin(), Buffer.data() + BeginLocInfo.second,
-            Buffer.end());
+            CXXUnit->getASTContext().getLangOpts(), Buffer.begin(),
+            Buffer.data() + BeginLocInfo.second, Buffer.end());
   Lex.SetCommentRetentionState(true);
-  
+
   unsigned NextIdx = 0;
   // Lex tokens in raw mode until we hit the end of the range, to avoid
   // entering #includes or expanding macros.
@@ -7551,10 +7514,10 @@ static void annotatePreprocessorTokens(CXTranslationUnit TU,
     Token Tok;
     if (lexNext(Lex, Tok, NextIdx, NumTokens))
       break;
-    unsigned TokIdx = NextIdx-1;
+    unsigned TokIdx = NextIdx - 1;
     assert(Tok.getLocation() ==
-             SourceLocation::getFromRawEncoding(Tokens[TokIdx].int_data[1]));
-    
+           SourceLocation::getFromRawEncoding(Tokens[TokIdx].int_data[1]));
+
   reprocess:
     if (Tok.is(tok::hash) && Tok.isAtStartOfLine()) {
       // We have found a preprocessing directive. Annotate the tokens
@@ -7601,7 +7564,7 @@ static void annotatePreprocessorTokens(CXTranslationUnit TU,
         }
       } while (!Tok.isAtStartOfLine());
 
-      unsigned LastIdx = finished ? NextIdx-1 : NextIdx-2;
+      unsigned LastIdx = finished ? NextIdx - 1 : NextIdx - 2;
       assert(TokIdx <= LastIdx);
       SourceLocation EndLoc =
           SourceLocation::getFromRawEncoding(Tokens[LastIdx].int_data[1]);
@@ -7610,7 +7573,7 @@ static void annotatePreprocessorTokens(CXTranslationUnit TU,
 
       for (; TokIdx <= LastIdx; ++TokIdx)
         updateCursorAnnotation(Cursors[TokIdx], Cursor);
-      
+
       if (finished)
         break;
       goto reprocess;
@@ -7629,10 +7592,9 @@ static void clang_annotateTokensImpl(CXTranslationUnit TU, ASTUnit *CXXUnit,
   // Determine the region of interest, which contains all of the tokens.
   SourceRange RegionOfInterest;
   RegionOfInterest.setBegin(
-    cxloc::translateSourceLocation(clang_getTokenLocation(TU, Tokens[0])));
-  RegionOfInterest.setEnd(
-    cxloc::translateSourceLocation(clang_getTokenLocation(TU,
-                                                         Tokens[NumTokens-1])));
+      cxloc::translateSourceLocation(clang_getTokenLocation(TU, Tokens[0])));
+  RegionOfInterest.setEnd(cxloc::translateSourceLocation(
+      clang_getTokenLocation(TU, Tokens[NumTokens - 1])));
 
   // Relex the tokens within the source range to look for preprocessing
   // directives.
@@ -7650,20 +7612,19 @@ static void clang_annotateTokensImpl(CXTranslationUnit TU, ASTUnit *CXXUnit,
 
   if (CXXUnit->getPreprocessor().getPreprocessingRecord()) {
     // Search and mark tokens that are macro argument expansions.
-    MarkMacroArgTokensVisitor Visitor(CXXUnit->getSourceManager(),
-                                      Tokens, NumTokens);
-    CursorVisitor MacroArgMarker(TU,
-                                 MarkMacroArgTokensVisitorDelegate, &Visitor,
-                                 /*VisitPreprocessorLast=*/true,
-                                 /*VisitIncludedEntities=*/false,
-                                 RegionOfInterest);
+    MarkMacroArgTokensVisitor Visitor(CXXUnit->getSourceManager(), Tokens,
+                                      NumTokens);
+    CursorVisitor MacroArgMarker(
+        TU, MarkMacroArgTokensVisitorDelegate, &Visitor,
+        /*VisitPreprocessorLast=*/true,
+        /*VisitIncludedEntities=*/false, RegionOfInterest);
     MacroArgMarker.visitPreprocessedEntitiesInRegion();
   }
-  
+
   // Annotate all of the source locations in the region of interest that map to
   // a specific cursor.
   AnnotateTokensWorker W(Tokens, Cursors, NumTokens, TU, RegionOfInterest);
-  
+
   // FIXME: We use a ridiculous stack size here because the data-recursion
   // algorithm uses a large stack frame than the non-data recursive version,
   // and AnnotationTokensWorker currently transforms the data-recursion
@@ -7677,43 +7638,43 @@ static void clang_annotateTokensImpl(CXTranslationUnit TU, ASTUnit *CXXUnit,
     for (unsigned I = 0; I != NumTokens; ++I) {
       if (clang_getTokenKind(Tokens[I]) != CXToken_Identifier)
         continue;
-      
+
       if (Cursors[I].kind == CXCursor_ObjCPropertyDecl) {
         IdentifierInfo *II = static_cast<IdentifierInfo *>(Tokens[I].ptr_data);
-        if (const ObjCPropertyDecl *Property
-            = dyn_cast_or_null<ObjCPropertyDecl>(getCursorDecl(Cursors[I]))) {
+        if (const ObjCPropertyDecl *Property =
+                dyn_cast_or_null<ObjCPropertyDecl>(getCursorDecl(Cursors[I]))) {
           if (Property->getPropertyAttributesAsWritten() != 0 &&
               llvm::StringSwitch<bool>(II->getName())
-              .Case("readonly", true)
-              .Case("assign", true)
-              .Case("unsafe_unretained", true)
-              .Case("readwrite", true)
-              .Case("retain", true)
-              .Case("copy", true)
-              .Case("nonatomic", true)
-              .Case("atomic", true)
-              .Case("getter", true)
-              .Case("setter", true)
-              .Case("strong", true)
-              .Case("weak", true)
-              .Case("class", true)
-              .Default(false))
+                  .Case("readonly", true)
+                  .Case("assign", true)
+                  .Case("unsafe_unretained", true)
+                  .Case("readwrite", true)
+                  .Case("retain", true)
+                  .Case("copy", true)
+                  .Case("nonatomic", true)
+                  .Case("atomic", true)
+                  .Case("getter", true)
+                  .Case("setter", true)
+                  .Case("strong", true)
+                  .Case("weak", true)
+                  .Case("class", true)
+                  .Default(false))
             Tokens[I].int_data[0] = CXToken_Keyword;
         }
         continue;
       }
-      
+
       if (Cursors[I].kind == CXCursor_ObjCInstanceMethodDecl ||
           Cursors[I].kind == CXCursor_ObjCClassMethodDecl) {
         IdentifierInfo *II = static_cast<IdentifierInfo *>(Tokens[I].ptr_data);
         if (llvm::StringSwitch<bool>(II->getName())
-            .Case("in", true)
-            .Case("out", true)
-            .Case("inout", true)
-            .Case("oneway", true)
-            .Case("bycopy", true)
-            .Case("byref", true)
-            .Default(false))
+                .Case("in", true)
+                .Case("out", true)
+                .Case("inout", true)
+                .Case("oneway", true)
+                .Case("bycopy", true)
+                .Case("byref", true)
+                .Default(false))
           Tokens[I].int_data[0] = CXToken_Keyword;
         continue;
       }
@@ -7727,9 +7688,8 @@ static void clang_annotateTokensImpl(CXTranslationUnit TU, ASTUnit *CXXUnit,
   }
 }
 
-void clang_annotateTokens(CXTranslationUnit TU,
-                          CXToken *Tokens, unsigned NumTokens,
-                          CXCursor *Cursors) {
+void clang_annotateTokens(CXTranslationUnit TU, CXToken *Tokens,
+                          unsigned NumTokens, CXCursor *Cursors) {
   if (isNotUsableTU(TU)) {
     LOG_BAD_TU(TU);
     return;
@@ -7742,7 +7702,7 @@ void clang_annotateTokens(CXTranslationUnit TU,
   LOG_FUNC_SECTION {
     *Log << TU << ' ';
     CXSourceLocation bloc = clang_getTokenLocation(TU, Tokens[0]);
-    CXSourceLocation eloc = clang_getTokenLocation(TU, Tokens[NumTokens-1]);
+    CXSourceLocation eloc = clang_getTokenLocation(TU, Tokens[NumTokens - 1]);
     *Log << clang_getRange(bloc, eloc);
   }
 
@@ -7777,13 +7737,17 @@ CXLinkageKind clang_getCursorLinkage(CXCursor cursor) {
   const Decl *D = cxcursor::getCursorDecl(cursor);
   if (const NamedDecl *ND = dyn_cast_or_null<NamedDecl>(D))
     switch (ND->getLinkageInternal()) {
-      case NoLinkage:
-      case VisibleNoLinkage: return CXLinkage_NoLinkage;
-      case ModuleInternalLinkage:
-      case InternalLinkage: return CXLinkage_Internal;
-      case UniqueExternalLinkage: return CXLinkage_UniqueExternal;
-      case ModuleLinkage:
-      case ExternalLinkage: return CXLinkage_External;
+    case NoLinkage:
+    case VisibleNoLinkage:
+      return CXLinkage_NoLinkage;
+    case ModuleInternalLinkage:
+    case InternalLinkage:
+      return CXLinkage_Internal;
+    case UniqueExternalLinkage:
+      return CXLinkage_UniqueExternal;
+    case ModuleLinkage:
+    case ExternalLinkage:
+      return CXLinkage_External;
     };
 
   return CXLinkage_Invalid;
@@ -7800,9 +7764,12 @@ CXVisibilityKind clang_getCursorVisibility(CXCursor cursor) {
   const Decl *D = cxcursor::getCursorDecl(cursor);
   if (const NamedDecl *ND = dyn_cast_or_null<NamedDecl>(D))
     switch (ND->getVisibility()) {
-      case HiddenVisibility: return CXVisibility_Hidden;
-      case ProtectedVisibility: return CXVisibility_Protected;
-      case DefaultVisibility: return CXVisibility_Default;
+    case HiddenVisibility:
+      return CXVisibility_Hidden;
+    case ProtectedVisibility:
+      return CXVisibility_Protected;
+    case DefaultVisibility:
+      return CXVisibility_Default;
     };
 
   return CXVisibility_Invalid;
@@ -7817,46 +7784,46 @@ static CXLanguageKind getDeclLanguage(const Decl *D) {
     return CXLanguage_C;
 
   switch (D->getKind()) {
-    default:
-      break;
-    case Decl::ImplicitParam:
-    case Decl::ObjCAtDefsField:
-    case Decl::ObjCCategory:
-    case Decl::ObjCCategoryImpl:
-    case Decl::ObjCCompatibleAlias:
-    case Decl::ObjCImplementation:
-    case Decl::ObjCInterface:
-    case Decl::ObjCIvar:
-    case Decl::ObjCMethod:
-    case Decl::ObjCProperty:
-    case Decl::ObjCPropertyImpl:
-    case Decl::ObjCProtocol:
-    case Decl::ObjCTypeParam:
-      return CXLanguage_ObjC;
-    case Decl::CXXConstructor:
-    case Decl::CXXConversion:
-    case Decl::CXXDestructor:
-    case Decl::CXXMethod:
-    case Decl::CXXRecord:
-    case Decl::ClassTemplate:
-    case Decl::ClassTemplatePartialSpecialization:
-    case Decl::ClassTemplateSpecialization:
-    case Decl::Friend:
-    case Decl::FriendTemplate:
-    case Decl::FunctionTemplate:
-    case Decl::LinkageSpec:
-    case Decl::Namespace:
-    case Decl::NamespaceAlias:
-    case Decl::NonTypeTemplateParm:
-    case Decl::StaticAssert:
-    case Decl::TemplateTemplateParm:
-    case Decl::TemplateTypeParm:
-    case Decl::UnresolvedUsingTypename:
-    case Decl::UnresolvedUsingValue:
-    case Decl::Using:
-    case Decl::UsingDirective:
-    case Decl::UsingShadow:
-      return CXLanguage_CPlusPlus;
+  default:
+    break;
+  case Decl::ImplicitParam:
+  case Decl::ObjCAtDefsField:
+  case Decl::ObjCCategory:
+  case Decl::ObjCCategoryImpl:
+  case Decl::ObjCCompatibleAlias:
+  case Decl::ObjCImplementation:
+  case Decl::ObjCInterface:
+  case Decl::ObjCIvar:
+  case Decl::ObjCMethod:
+  case Decl::ObjCProperty:
+  case Decl::ObjCPropertyImpl:
+  case Decl::ObjCProtocol:
+  case Decl::ObjCTypeParam:
+    return CXLanguage_ObjC;
+  case Decl::CXXConstructor:
+  case Decl::CXXConversion:
+  case Decl::CXXDestructor:
+  case Decl::CXXMethod:
+  case Decl::CXXRecord:
+  case Decl::ClassTemplate:
+  case Decl::ClassTemplatePartialSpecialization:
+  case Decl::ClassTemplateSpecialization:
+  case Decl::Friend:
+  case Decl::FriendTemplate:
+  case Decl::FunctionTemplate:
+  case Decl::LinkageSpec:
+  case Decl::Namespace:
+  case Decl::NamespaceAlias:
+  case Decl::NonTypeTemplateParm:
+  case Decl::StaticAssert:
+  case Decl::TemplateTemplateParm:
+  case Decl::TemplateTypeParm:
+  case Decl::UnresolvedUsingTypename:
+  case Decl::UnresolvedUsingValue:
+  case Decl::Using:
+  case Decl::UsingDirective:
+  case Decl::UsingShadow:
+    return CXLanguage_CPlusPlus;
   }
 
   return CXLanguage_C;
@@ -7865,7 +7832,7 @@ static CXLanguageKind getDeclLanguage(const Decl *D) {
 static CXAvailabilityKind getCursorAvailabilityForDecl(const Decl *D) {
   if (isa<FunctionDecl>(D) && cast<FunctionDecl>(D)->isDeleted())
     return CXAvailability_NotAvailable;
-  
+
   switch (D->getAvailability()) {
   case AR_Available:
   case AR_NotYetIntroduced:
@@ -7893,12 +7860,12 @@ enum CXAvailabilityKind clang_getCursorAvailability(CXCursor cursor) {
 }
 
 static CXVersion convertVersion(VersionTuple In) {
-  CXVersion Out = { -1, -1, -1 };
+  CXVersion Out = {-1, -1, -1};
   if (In.empty())
     return Out;
 
   Out.Major = In.getMajor();
-  
+
   Optional<unsigned> Minor = In.getMinor();
   if (Minor.hasValue())
     Out.Minor = *Minor;
@@ -7908,7 +7875,7 @@ static CXVersion convertVersion(VersionTuple In) {
   Optional<unsigned> Subminor = In.getSubminor();
   if (Subminor.hasValue())
     Out.Subminor = *Subminor;
-  
+
   return Out;
 }
 
@@ -7956,11 +7923,10 @@ static void getCursorPlatformAvailabilityForDecl(
   if (AvailabilityAttrs.empty())
     return;
 
-  llvm::sort(AvailabilityAttrs,
-             [](AvailabilityAttr *LHS, AvailabilityAttr *RHS) {
-               return LHS->getPlatform()->getName() <
-                      RHS->getPlatform()->getName();
-             });
+  llvm::sort(
+      AvailabilityAttrs, [](AvailabilityAttr *LHS, AvailabilityAttr *RHS) {
+        return LHS->getPlatform()->getName() < RHS->getPlatform()->getName();
+      });
   ASTContext &Ctx = D->getASTContext();
   auto It = std::unique(
       AvailabilityAttrs.begin(), AvailabilityAttrs.end(),
@@ -8077,9 +8043,9 @@ CXTLSKind clang_getCursorTLSKind(CXCursor cursor) {
   return CXTLS_None;
 }
 
- /// If the given cursor is the "templated" declaration
- /// describing a class or function template, return the class or
- /// function template.
+/// If the given cursor is the "templated" declaration
+/// describing a class or function template, return the class or
+/// function template.
 static const Decl *maybeGetTemplateCursor(const Decl *D) {
   if (!D)
     return nullptr;
@@ -8095,7 +8061,6 @@ static const Decl *maybeGetTemplateCursor(const Decl *D) {
   return D;
 }
 
-
 enum CX_StorageClass clang_Cursor_getStorageClass(CXCursor C) {
   StorageClass sc = SC_None;
   const Decl *D = getCursorDecl(C);
@@ -8134,16 +8099,16 @@ CXCursor clang_getCursorSemanticParent(CXCursor cursor) {
       if (!DC)
         return clang_getNullCursor();
 
-      return MakeCXCursor(maybeGetTemplateCursor(cast<Decl>(DC)), 
+      return MakeCXCursor(maybeGetTemplateCursor(cast<Decl>(DC)),
                           getCursorTU(cursor));
     }
   }
-  
+
   if (clang_isStatement(cursor.kind) || clang_isExpression(cursor.kind)) {
     if (const Decl *D = getCursorDecl(cursor))
       return MakeCXCursor(D, getCursorTU(cursor));
   }
-  
+
   return clang_getNullCursor();
 }
 
@@ -8154,12 +8119,12 @@ CXCursor clang_getCursorLexicalParent(CXCursor cursor) {
       if (!DC)
         return clang_getNullCursor();
 
-      return MakeCXCursor(maybeGetTemplateCursor(cast<Decl>(DC)), 
+      return MakeCXCursor(maybeGetTemplateCursor(cast<Decl>(DC)),
                           getCursorTU(cursor));
     }
   }
 
-  // FIXME: Note that we can't easily compute the lexical context of a 
+  // FIXME: Note that we can't easily compute the lexical context of a
   // statement or expression, so we return nothing.
   return clang_getNullCursor();
 }
@@ -8181,9 +8146,9 @@ unsigned clang_Cursor_getObjCPropertyAttributes(CXCursor C, unsigned reserved) {
   ObjCPropertyDecl::PropertyAttributeKind Attr =
       PD->getPropertyAttributesAsWritten();
 
-#define SET_CXOBJCPROP_ATTR(A) \
-  if (Attr & ObjCPropertyDecl::OBJC_PR_##A) \
-    Result |= CXObjCPropertyAttr_##A
+#define SET_CXOBJCPROP_ATTR(A)                                                 \
+  if (Attr & ObjCPropertyDecl::OBJC_PR_##A)                                    \
+  Result |= CXObjCPropertyAttr_##A
   SET_CXOBJCPROP_ATTR(readonly);
   SET_CXOBJCPROP_ATTR(getter);
   SET_CXOBJCPROP_ATTR(assign);
@@ -8240,12 +8205,18 @@ unsigned clang_Cursor_getObjCDeclQualifiers(CXCursor C) {
     return CXObjCDeclQualifier_None;
 
   unsigned Result = CXObjCDeclQualifier_None;
-  if (QT & Decl::OBJC_TQ_In) Result |= CXObjCDeclQualifier_In;
-  if (QT & Decl::OBJC_TQ_Inout) Result |= CXObjCDeclQualifier_Inout;
-  if (QT & Decl::OBJC_TQ_Out) Result |= CXObjCDeclQualifier_Out;
-  if (QT & Decl::OBJC_TQ_Bycopy) Result |= CXObjCDeclQualifier_Bycopy;
-  if (QT & Decl::OBJC_TQ_Byref) Result |= CXObjCDeclQualifier_Byref;
-  if (QT & Decl::OBJC_TQ_Oneway) Result |= CXObjCDeclQualifier_Oneway;
+  if (QT & Decl::OBJC_TQ_In)
+    Result |= CXObjCDeclQualifier_In;
+  if (QT & Decl::OBJC_TQ_Inout)
+    Result |= CXObjCDeclQualifier_Inout;
+  if (QT & Decl::OBJC_TQ_Out)
+    Result |= CXObjCDeclQualifier_Out;
+  if (QT & Decl::OBJC_TQ_Bycopy)
+    Result |= CXObjCDeclQualifier_Bycopy;
+  if (QT & Decl::OBJC_TQ_Byref)
+    Result |= CXObjCDeclQualifier_Byref;
+  if (QT & Decl::OBJC_TQ_Oneway)
+    Result |= CXObjCDeclQualifier_Oneway;
 
   return Result;
 }
@@ -8276,9 +8247,9 @@ unsigned clang_Cursor_isVariadic(CXCursor C) {
   return 0;
 }
 
-unsigned clang_Cursor_isExternalSymbol(CXCursor C,
-                                     CXString *language, CXString *definedIn,
-                                     unsigned *isGenerated) {
+unsigned clang_Cursor_isExternalSymbol(CXCursor C, CXString *language,
+                                       CXString *definedIn,
+                                       unsigned *isGenerated) {
   if (!clang_isDeclaration(C.kind))
     return 0;
 
@@ -8316,8 +8287,8 @@ CXString clang_Cursor_getRawCommentText(CXCursor C) {
   const Decl *D = getCursorDecl(C);
   ASTContext &Context = getCursorContext(C);
   const RawComment *RC = Context.getRawCommentForAnyRedecl(D);
-  StringRef RawText = RC ? RC->getRawText(Context.getSourceManager()) :
-                           StringRef();
+  StringRef RawText =
+      RC ? RC->getRawText(Context.getSourceManager()) : StringRef();
 
   // Don't duplicate the string because RawText points directly into source
   // code.
@@ -8361,46 +8332,46 @@ CXModule clang_getModuleForFile(CXTranslationUnit TU, CXFile File) {
   if (!File)
     return nullptr;
   FileEntry *FE = static_cast<FileEntry *>(File);
-  
+
   ASTUnit &Unit = *cxtu::getASTUnit(TU);
   HeaderSearch &HS = Unit.getPreprocessor().getHeaderSearchInfo();
   ModuleMap::KnownHeader Header = HS.findModuleForHeader(FE);
-  
+
   return Header.getModule();
 }
 
 CXFile clang_Module_getASTFile(CXModule CXMod) {
   if (!CXMod)
     return nullptr;
-  Module *Mod = static_cast<Module*>(CXMod);
+  Module *Mod = static_cast<Module *>(CXMod);
   return const_cast<FileEntry *>(Mod->getASTFile());
 }
 
 CXModule clang_Module_getParent(CXModule CXMod) {
   if (!CXMod)
     return nullptr;
-  Module *Mod = static_cast<Module*>(CXMod);
+  Module *Mod = static_cast<Module *>(CXMod);
   return Mod->Parent;
 }
 
 CXString clang_Module_getName(CXModule CXMod) {
   if (!CXMod)
     return cxstring::createEmpty();
-  Module *Mod = static_cast<Module*>(CXMod);
+  Module *Mod = static_cast<Module *>(CXMod);
   return cxstring::createDup(Mod->Name);
 }
 
 CXString clang_Module_getFullName(CXModule CXMod) {
   if (!CXMod)
     return cxstring::createEmpty();
-  Module *Mod = static_cast<Module*>(CXMod);
+  Module *Mod = static_cast<Module *>(CXMod);
   return cxstring::createDup(Mod->getFullModuleName());
 }
 
 int clang_Module_isSystem(CXModule CXMod) {
   if (!CXMod)
     return 0;
-  Module *Mod = static_cast<Module*>(CXMod);
+  Module *Mod = static_cast<Module *>(CXMod);
   return Mod->IsSystem;
 }
 
@@ -8412,21 +8383,21 @@ unsigned clang_Module_getNumTopLevelHeaders(CXTranslationUnit TU,
   }
   if (!CXMod)
     return 0;
-  Module *Mod = static_cast<Module*>(CXMod);
+  Module *Mod = static_cast<Module *>(CXMod);
   FileManager &FileMgr = cxtu::getASTUnit(TU)->getFileManager();
   ArrayRef<const FileEntry *> TopHeaders = Mod->getTopHeaders(FileMgr);
   return TopHeaders.size();
 }
 
-CXFile clang_Module_getTopLevelHeader(CXTranslationUnit TU,
-                                      CXModule CXMod, unsigned Index) {
+CXFile clang_Module_getTopLevelHeader(CXTranslationUnit TU, CXModule CXMod,
+                                      unsigned Index) {
   if (isNotUsableTU(TU)) {
     LOG_BAD_TU(TU);
     return nullptr;
   }
   if (!CXMod)
     return nullptr;
-  Module *Mod = static_cast<Module*>(CXMod);
+  Module *Mod = static_cast<Module *>(CXMod);
   FileManager &FileMgr = cxtu::getASTUnit(TU)->getFileManager();
 
   ArrayRef<const FileEntry *> TopHeaders = Mod->getTopHeaders(FileMgr);
@@ -8524,7 +8495,7 @@ unsigned clang_CXXMethod_isDefaulted(CXCursor C) {
 unsigned clang_CXXMethod_isStatic(CXCursor C) {
   if (!clang_isDeclaration(C.kind))
     return 0;
-  
+
   const Decl *D = cxcursor::getCursorDecl(C);
   const CXXMethodDecl *Method =
       D ? dyn_cast_or_null<CXXMethodDecl>(D->getAsFunction()) : nullptr;
@@ -8534,7 +8505,7 @@ unsigned clang_CXXMethod_isStatic(CXCursor C) {
 unsigned clang_CXXMethod_isVirtual(CXCursor C) {
   if (!clang_isDeclaration(C.kind))
     return 0;
-  
+
   const Decl *D = cxcursor::getCursorDecl(C);
   const CXXMethodDecl *Method =
       D ? dyn_cast_or_null<CXXMethodDecl>(D->getAsFunction()) : nullptr;
@@ -8568,11 +8539,11 @@ unsigned clang_EnumDecl_isScoped(CXCursor C) {
 CXType clang_getIBOutletCollectionType(CXCursor C) {
   if (C.kind != CXCursor_IBOutletCollectionAttr)
     return cxtype::MakeCXType(QualType(), cxcursor::getCursorTU(C));
-  
+
   const IBOutletCollectionAttr *A =
-    cast<IBOutletCollectionAttr>(cxcursor::getCursorAttr(C));
-  
-  return cxtype::MakeCXType(A->getInterface(), cxcursor::getCursorTU(C));  
+      cast<IBOutletCollectionAttr>(cxcursor::getCursorAttr(C));
+
+  return cxtype::MakeCXType(A->getInterface(), cxcursor::getCursorTU(C));
 }
 
 //===----------------------------------------------------------------------===//
@@ -8582,57 +8553,57 @@ CXType clang_getIBOutletCollectionType(CXCursor C) {
 typedef std::vector<CXTUResourceUsageEntry> MemUsageEntries;
 
 static inline void createCXTUResourceUsageEntry(MemUsageEntries &entries,
-                                              enum CXTUResourceUsageKind k,
-                                              unsigned long amount) {
-  CXTUResourceUsageEntry entry = { k, amount };
+                                                enum CXTUResourceUsageKind k,
+                                                unsigned long amount) {
+  CXTUResourceUsageEntry entry = {k, amount};
   entries.push_back(entry);
 }
 
 const char *clang_getTUResourceUsageName(CXTUResourceUsageKind kind) {
   const char *str = "";
   switch (kind) {
-    case CXTUResourceUsage_AST:
-      str = "ASTContext: expressions, declarations, and types"; 
-      break;
-    case CXTUResourceUsage_Identifiers:
-      str = "ASTContext: identifiers";
-      break;
-    case CXTUResourceUsage_Selectors:
-      str = "ASTContext: selectors";
-      break;
-    case CXTUResourceUsage_GlobalCompletionResults:
-      str = "Code completion: cached global results";
-      break;
-    case CXTUResourceUsage_SourceManagerContentCache:
-      str = "SourceManager: content cache allocator";
-      break;
-    case CXTUResourceUsage_AST_SideTables:
-      str = "ASTContext: side tables";
-      break;
-    case CXTUResourceUsage_SourceManager_Membuffer_Malloc:
-      str = "SourceManager: malloc'ed memory buffers";
-      break;
-    case CXTUResourceUsage_SourceManager_Membuffer_MMap:
-      str = "SourceManager: mmap'ed memory buffers";
-      break;
-    case CXTUResourceUsage_ExternalASTSource_Membuffer_Malloc:
-      str = "ExternalASTSource: malloc'ed memory buffers";
-      break;
-    case CXTUResourceUsage_ExternalASTSource_Membuffer_MMap:
-      str = "ExternalASTSource: mmap'ed memory buffers";
-      break;
-    case CXTUResourceUsage_Preprocessor:
-      str = "Preprocessor: malloc'ed memory";
-      break;
-    case CXTUResourceUsage_PreprocessingRecord:
-      str = "Preprocessor: PreprocessingRecord";
-      break;
-    case CXTUResourceUsage_SourceManager_DataStructures:
-      str = "SourceManager: data structures and tables";
-      break;
-    case CXTUResourceUsage_Preprocessor_HeaderSearch:
-      str = "Preprocessor: header search tables";
-      break;
+  case CXTUResourceUsage_AST:
+    str = "ASTContext: expressions, declarations, and types";
+    break;
+  case CXTUResourceUsage_Identifiers:
+    str = "ASTContext: identifiers";
+    break;
+  case CXTUResourceUsage_Selectors:
+    str = "ASTContext: selectors";
+    break;
+  case CXTUResourceUsage_GlobalCompletionResults:
+    str = "Code completion: cached global results";
+    break;
+  case CXTUResourceUsage_SourceManagerContentCache:
+    str = "SourceManager: content cache allocator";
+    break;
+  case CXTUResourceUsage_AST_SideTables:
+    str = "ASTContext: side tables";
+    break;
+  case CXTUResourceUsage_SourceManager_Membuffer_Malloc:
+    str = "SourceManager: malloc'ed memory buffers";
+    break;
+  case CXTUResourceUsage_SourceManager_Membuffer_MMap:
+    str = "SourceManager: mmap'ed memory buffers";
+    break;
+  case CXTUResourceUsage_ExternalASTSource_Membuffer_Malloc:
+    str = "ExternalASTSource: malloc'ed memory buffers";
+    break;
+  case CXTUResourceUsage_ExternalASTSource_Membuffer_MMap:
+    str = "ExternalASTSource: mmap'ed memory buffers";
+    break;
+  case CXTUResourceUsage_Preprocessor:
+    str = "Preprocessor: malloc'ed memory";
+    break;
+  case CXTUResourceUsage_PreprocessingRecord:
+    str = "Preprocessor: PreprocessingRecord";
+    break;
+  case CXTUResourceUsage_SourceManager_DataStructures:
+    str = "SourceManager: data structures and tables";
+    break;
+  case CXTUResourceUsage_Preprocessor_HeaderSearch:
+    str = "Preprocessor: header search tables";
+    break;
   }
   return str;
 }
@@ -8640,99 +8611,99 @@ const char *clang_getTUResourceUsageName(CXTUResourceUsageKind kind) {
 CXTUResourceUsage clang_getCXTUResourceUsage(CXTranslationUnit TU) {
   if (isNotUsableTU(TU)) {
     LOG_BAD_TU(TU);
-    CXTUResourceUsage usage = { (void*) nullptr, 0, nullptr };
+    CXTUResourceUsage usage = {(void *)nullptr, 0, nullptr};
     return usage;
   }
-  
+
   ASTUnit *astUnit = cxtu::getASTUnit(TU);
   std::unique_ptr<MemUsageEntries> entries(new MemUsageEntries());
   ASTContext &astContext = astUnit->getASTContext();
-  
+
   // How much memory is used by AST nodes and types?
-  createCXTUResourceUsageEntry(*entries, CXTUResourceUsage_AST,
-    (unsigned long) astContext.getASTAllocatedMemory());
+  createCXTUResourceUsageEntry(
+      *entries, CXTUResourceUsage_AST,
+      (unsigned long)astContext.getASTAllocatedMemory());
 
   // How much memory is used by identifiers?
-  createCXTUResourceUsageEntry(*entries, CXTUResourceUsage_Identifiers,
-    (unsigned long) astContext.Idents.getAllocator().getTotalMemory());
+  createCXTUResourceUsageEntry(
+      *entries, CXTUResourceUsage_Identifiers,
+      (unsigned long)astContext.Idents.getAllocator().getTotalMemory());
 
   // How much memory is used for selectors?
-  createCXTUResourceUsageEntry(*entries, CXTUResourceUsage_Selectors,
-    (unsigned long) astContext.Selectors.getTotalMemory());
-  
+  createCXTUResourceUsageEntry(
+      *entries, CXTUResourceUsage_Selectors,
+      (unsigned long)astContext.Selectors.getTotalMemory());
+
   // How much memory is used by ASTContext's side tables?
-  createCXTUResourceUsageEntry(*entries, CXTUResourceUsage_AST_SideTables,
-    (unsigned long) astContext.getSideTableAllocatedMemory());
-  
+  createCXTUResourceUsageEntry(
+      *entries, CXTUResourceUsage_AST_SideTables,
+      (unsigned long)astContext.getSideTableAllocatedMemory());
+
   // How much memory is used for caching global code completion results?
   unsigned long completionBytes = 0;
   if (GlobalCodeCompletionAllocator *completionAllocator =
-      astUnit->getCachedCompletionAllocator().get()) {
+          astUnit->getCachedCompletionAllocator().get()) {
     completionBytes = completionAllocator->getTotalMemory();
   }
-  createCXTUResourceUsageEntry(*entries,
-                               CXTUResourceUsage_GlobalCompletionResults,
-                               completionBytes);
-  
+  createCXTUResourceUsageEntry(
+      *entries, CXTUResourceUsage_GlobalCompletionResults, completionBytes);
+
   // How much memory is being used by SourceManager's content cache?
-  createCXTUResourceUsageEntry(*entries,
-          CXTUResourceUsage_SourceManagerContentCache,
-          (unsigned long) astContext.getSourceManager().getContentCacheSize());
-  
+  createCXTUResourceUsageEntry(
+      *entries, CXTUResourceUsage_SourceManagerContentCache,
+      (unsigned long)astContext.getSourceManager().getContentCacheSize());
+
   // How much memory is being used by the MemoryBuffer's in SourceManager?
   const SourceManager::MemoryBufferSizes &srcBufs =
-    astUnit->getSourceManager().getMemoryBufferSizes();
-  
+      astUnit->getSourceManager().getMemoryBufferSizes();
+
   createCXTUResourceUsageEntry(*entries,
                                CXTUResourceUsage_SourceManager_Membuffer_Malloc,
-                               (unsigned long) srcBufs.malloc_bytes);
+                               (unsigned long)srcBufs.malloc_bytes);
   createCXTUResourceUsageEntry(*entries,
                                CXTUResourceUsage_SourceManager_Membuffer_MMap,
-                               (unsigned long) srcBufs.mmap_bytes);
-  createCXTUResourceUsageEntry(*entries,
-                               CXTUResourceUsage_SourceManager_DataStructures,
-                               (unsigned long) astContext.getSourceManager()
-                                .getDataStructureSizes());
-  
+                               (unsigned long)srcBufs.mmap_bytes);
+  createCXTUResourceUsageEntry(
+      *entries, CXTUResourceUsage_SourceManager_DataStructures,
+      (unsigned long)astContext.getSourceManager().getDataStructureSizes());
+
   // How much memory is being used by the ExternalASTSource?
   if (ExternalASTSource *esrc = astContext.getExternalSource()) {
     const ExternalASTSource::MemoryBufferSizes &sizes =
-      esrc->getMemoryBufferSizes();
-    
-    createCXTUResourceUsageEntry(*entries,
-      CXTUResourceUsage_ExternalASTSource_Membuffer_Malloc,
-                                 (unsigned long) sizes.malloc_bytes);
-    createCXTUResourceUsageEntry(*entries,
-      CXTUResourceUsage_ExternalASTSource_Membuffer_MMap,
-                                 (unsigned long) sizes.mmap_bytes);
+        esrc->getMemoryBufferSizes();
+
+    createCXTUResourceUsageEntry(
+        *entries, CXTUResourceUsage_ExternalASTSource_Membuffer_Malloc,
+        (unsigned long)sizes.malloc_bytes);
+    createCXTUResourceUsageEntry(
+        *entries, CXTUResourceUsage_ExternalASTSource_Membuffer_MMap,
+        (unsigned long)sizes.mmap_bytes);
   }
-  
+
   // How much memory is being used by the Preprocessor?
   Preprocessor &pp = astUnit->getPreprocessor();
-  createCXTUResourceUsageEntry(*entries,
-                               CXTUResourceUsage_Preprocessor,
+  createCXTUResourceUsageEntry(*entries, CXTUResourceUsage_Preprocessor,
                                pp.getTotalMemory());
-  
+
   if (PreprocessingRecord *pRec = pp.getPreprocessingRecord()) {
     createCXTUResourceUsageEntry(*entries,
                                  CXTUResourceUsage_PreprocessingRecord,
-                                 pRec->getTotalMemory());    
+                                 pRec->getTotalMemory());
   }
-  
+
   createCXTUResourceUsageEntry(*entries,
                                CXTUResourceUsage_Preprocessor_HeaderSearch,
                                pp.getHeaderSearchInfo().getTotalMemory());
 
-  CXTUResourceUsage usage = { (void*) entries.get(),
-                            (unsigned) entries->size(),
-                            !entries->empty() ? &(*entries)[0] : nullptr };
+  CXTUResourceUsage usage = {(void *)entries.get(), (unsigned)entries->size(),
+                             !entries->empty() ? &(*entries)[0] : nullptr};
   (void)entries.release();
   return usage;
 }
 
 void clang_disposeCXTUResourceUsage(CXTUResourceUsage usage) {
   if (usage.data)
-    delete (MemUsageEntries*) usage.data;
+    delete (MemUsageEntries *)usage.data;
 }
 
 CXSourceRangeList *clang_getSkippedRanges(CXTranslationUnit TU, CXFile file) {
@@ -8749,7 +8720,8 @@ CXSourceRangeList *clang_getSkippedRanges(CXTranslationUnit TU, CXFile file) {
     return skipped;
 
   ASTUnit *astUnit = cxtu::getASTUnit(TU);
-  PreprocessingRecord *ppRec = astUnit->getPreprocessor().getPreprocessingRecord();
+  PreprocessingRecord *ppRec =
+      astUnit->getPreprocessor().getPreprocessingRecord();
   if (!ppRec)
     return skipped;
 
@@ -8761,11 +8733,14 @@ CXSourceRangeList *clang_getSkippedRanges(CXTranslationUnit TU, CXFile file) {
 
   const std::vector<SourceRange> &SkippedRanges = ppRec->getSkippedRanges();
   std::vector<SourceRange> wantedRanges;
-  for (std::vector<SourceRange>::const_iterator i = SkippedRanges.begin(), ei = SkippedRanges.end();
+  for (std::vector<SourceRange>::const_iterator i = SkippedRanges.begin(),
+                                                ei = SkippedRanges.end();
        i != ei; ++i) {
-    if (sm.getFileID(i->getBegin()) == wantedFileID || sm.getFileID(i->getEnd()) == wantedFileID)
+    if (sm.getFileID(i->getBegin()) == wantedFileID ||
+        sm.getFileID(i->getEnd()) == wantedFileID)
       wantedRanges.push_back(*i);
-    else if (isMainFile && (astUnit->isInPreambleFileID(i->getBegin()) || astUnit->isInPreambleFileID(i->getEnd())))
+    else if (isMainFile && (astUnit->isInPreambleFileID(i->getBegin()) ||
+                            astUnit->isInPreambleFileID(i->getEnd())))
       wantedRanges.push_back(*i);
   }
 
@@ -8786,9 +8761,10 @@ CXSourceRangeList *clang_getAllSkippedRanges(CXTranslationUnit TU) {
     LOG_BAD_TU(TU);
     return skipped;
   }
-    
+
   ASTUnit *astUnit = cxtu::getASTUnit(TU);
-  PreprocessingRecord *ppRec = astUnit->getPreprocessor().getPreprocessingRecord();
+  PreprocessingRecord *ppRec =
+      astUnit->getPreprocessor().getPreprocessingRecord();
   if (!ppRec)
     return skipped;
 
@@ -8814,10 +8790,10 @@ void clang_disposeSourceRangeList(CXSourceRangeList *ranges) {
 void clang::PrintLibclangResourceUsage(CXTranslationUnit TU) {
   CXTUResourceUsage Usage = clang_getCXTUResourceUsage(TU);
   for (unsigned I = 0; I != Usage.numEntries; ++I)
-    fprintf(stderr, "  %s: %lu\n", 
+    fprintf(stderr, "  %s: %lu\n",
             clang_getTUResourceUsageName(Usage.entries[I].kind),
             Usage.entries[I].amount);
-  
+
   clang_disposeCXTUResourceUsage(Usage);
 }
 
@@ -8839,15 +8815,11 @@ bool RunSafely(llvm::CrashRecoveryContext &CRC, llvm::function_ref<void()> Fn,
   return CRC.RunSafely(Fn);
 }
 
-unsigned GetSafetyThreadStackSize() {
-  return SafetyStackThreadSize;
-}
+unsigned GetSafetyThreadStackSize() { return SafetyStackThreadSize; }
 
-void SetSafetyThreadStackSize(unsigned Value) {
-  SafetyStackThreadSize = Value;
-}
+void SetSafetyThreadStackSize(unsigned Value) { SafetyStackThreadSize = Value; }
 
-}
+} // namespace clang
 
 void clang::setThreadBackgroundPriority() {
   if (getenv("LIBCLANG_BGPRIO_DISABLE"))
@@ -8862,12 +8834,12 @@ void cxindex::printDiagsToStderr(ASTUnit *Unit) {
   if (!Unit)
     return;
 
-  for (ASTUnit::stored_diag_iterator D = Unit->stored_diag_begin(), 
-                                  DEnd = Unit->stored_diag_end();
+  for (ASTUnit::stored_diag_iterator D = Unit->stored_diag_begin(),
+                                     DEnd = Unit->stored_diag_end();
        D != DEnd; ++D) {
     CXStoredDiagnostic Diag(*D, Unit->getLangOpts());
-    CXString Msg = clang_formatDiagnostic(&Diag,
-                                clang_defaultDiagnosticDisplayOptions());
+    CXString Msg =
+        clang_formatDiagnostic(&Diag, clang_defaultDiagnosticDisplayOptions());
     fprintf(stderr, "%s\n", clang_getCString(Msg));
     clang_disposeString(Msg);
   }
@@ -8881,7 +8853,7 @@ void cxindex::printDiagsToStderr(ASTUnit *Unit) {
 
 MacroInfo *cxindex::getMacroInfo(const IdentifierInfo &II,
                                  SourceLocation MacroDefLoc,
-                                 CXTranslationUnit TU){
+                                 CXTranslationUnit TU) {
   if (MacroDefLoc.isInvalid() || !TU)
     return nullptr;
   if (!II.hadMacroDefinition())
@@ -8891,8 +8863,8 @@ MacroInfo *cxindex::getMacroInfo(const IdentifierInfo &II,
   Preprocessor &PP = Unit->getPreprocessor();
   MacroDirective *MD = PP.getLocalMacroDirectiveHistory(&II);
   if (MD) {
-    for (MacroDirective::DefInfo
-           Def = MD->getDefinition(); Def; Def = Def.getPreviousDefinition()) {
+    for (MacroDirective::DefInfo Def = MD->getDefinition(); Def;
+         Def = Def.getPreviousDefinition()) {
       if (MacroDefLoc == Def.getMacroInfo()->getDefinitionLoc())
         return Def.getMacroInfo();
     }
@@ -9028,13 +9000,13 @@ Logger &cxindex::Logger::operator<<(CXSourceRange range) {
   CXString BFileName = clang_getFileName(BFile);
   if (BFile == EFile) {
     *this << llvm::format("[%s %d:%d-%d:%d]", clang_getCString(BFileName),
-                         BLine, BColumn, ELine, EColumn);
+                          BLine, BColumn, ELine, EColumn);
   } else {
     CXString EFileName = clang_getFileName(EFile);
-    *this << llvm::format("[%s:%d:%d - ", clang_getCString(BFileName),
-                          BLine, BColumn)
-          << llvm::format("%s:%d:%d]", clang_getCString(EFileName),
-                          ELine, EColumn);
+    *this << llvm::format("[%s:%d:%d - ", clang_getCString(BFileName), BLine,
+                          BColumn)
+          << llvm::format("%s:%d:%d]", clang_getCString(EFileName), ELine,
+                          EColumn);
     clang_disposeString(EFileName);
   }
   clang_disposeString(BFileName);
diff --git a/clang/tools/libclang/CXCursor.cpp b/clang/tools/libclang/CXCursor.cpp
index 147e3eaf4762c..3bcc7cc7af058 100644
--- a/clang/tools/libclang/CXCursor.cpp
+++ b/clang/tools/libclang/CXCursor.cpp
@@ -12,9 +12,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CXTranslationUnit.h"
 #include "CXCursor.h"
 #include "CXString.h"
+#include "CXTranslationUnit.h"
 #include "CXType.h"
 #include "clang-c/Index.h"
 #include "clang/AST/Attr.h"
@@ -33,55 +33,97 @@ using namespace cxcursor;
 
 CXCursor cxcursor::MakeCXCursorInvalid(CXCursorKind K, CXTranslationUnit TU) {
   assert(K >= CXCursor_FirstInvalid && K <= CXCursor_LastInvalid);
-  CXCursor C = { K, 0, { nullptr, nullptr, TU } };
+  CXCursor C = {K, 0, {nullptr, nullptr, TU}};
   return C;
 }
 
 static CXCursorKind GetCursorKind(const Attr *A) {
   assert(A && "Invalid arguments!");
   switch (A->getKind()) {
-    default: break;
-    case attr::IBAction: return CXCursor_IBActionAttr;
-    case attr::IBOutlet: return CXCursor_IBOutletAttr;
-    case attr::IBOutletCollection: return CXCursor_IBOutletCollectionAttr;
-    case attr::Final: return CXCursor_CXXFinalAttr;
-    case attr::Override: return CXCursor_CXXOverrideAttr;
-    case attr::Annotate: return CXCursor_AnnotateAttr;
-    case attr::AsmLabel: return CXCursor_AsmLabelAttr;
-    case attr::Packed: return CXCursor_PackedAttr;
-    case attr::Pure: return CXCursor_PureAttr;
-    case attr::Const: return CXCursor_ConstAttr;
-    case attr::NoDuplicate: return CXCursor_NoDuplicateAttr;
-    case attr::CUDAConstant: return CXCursor_CUDAConstantAttr;
-    case attr::CUDADevice: return CXCursor_CUDADeviceAttr;
-    case attr::CUDAGlobal: return CXCursor_CUDAGlobalAttr;
-    case attr::CUDAHost: return CXCursor_CUDAHostAttr;
-    case attr::CUDAShared: return CXCursor_CUDASharedAttr;
-    case attr::Visibility: return CXCursor_VisibilityAttr;
-    case attr::DLLExport: return CXCursor_DLLExport;
-    case attr::DLLImport: return CXCursor_DLLImport;
-    case attr::NSReturnsRetained: return CXCursor_NSReturnsRetained;
-    case attr::NSReturnsNotRetained: return CXCursor_NSReturnsNotRetained;
-    case attr::NSReturnsAutoreleased: return CXCursor_NSReturnsAutoreleased;
-    case attr::NSConsumesSelf: return CXCursor_NSConsumesSelf;
-    case attr::NSConsumed: return CXCursor_NSConsumed;
-    case attr::ObjCException: return CXCursor_ObjCException;
-    case attr::ObjCNSObject: return CXCursor_ObjCNSObject;
-    case attr::ObjCIndependentClass: return CXCursor_ObjCIndependentClass;
-    case attr::ObjCPreciseLifetime: return CXCursor_ObjCPreciseLifetime;
-    case attr::ObjCReturnsInnerPointer: return CXCursor_ObjCReturnsInnerPointer;
-    case attr::ObjCRequiresSuper: return CXCursor_ObjCRequiresSuper;
-    case attr::ObjCRootClass: return CXCursor_ObjCRootClass;
-    case attr::ObjCSubclassingRestricted: return CXCursor_ObjCSubclassingRestricted;
-    case attr::ObjCExplicitProtocolImpl: return CXCursor_ObjCExplicitProtocolImpl;
-    case attr::ObjCDesignatedInitializer: return CXCursor_ObjCDesignatedInitializer;
-    case attr::ObjCRuntimeVisible: return CXCursor_ObjCRuntimeVisible;
-    case attr::ObjCBoxable: return CXCursor_ObjCBoxable;
-    case attr::FlagEnum: return CXCursor_FlagEnum;
-    case attr::Convergent: return CXCursor_ConvergentAttr;
-    case attr::WarnUnused: return CXCursor_WarnUnusedAttr;
-    case attr::WarnUnusedResult: return CXCursor_WarnUnusedResultAttr;
-    case attr::Aligned: return CXCursor_AlignedAttr;
+  default:
+    break;
+  case attr::IBAction:
+    return CXCursor_IBActionAttr;
+  case attr::IBOutlet:
+    return CXCursor_IBOutletAttr;
+  case attr::IBOutletCollection:
+    return CXCursor_IBOutletCollectionAttr;
+  case attr::Final:
+    return CXCursor_CXXFinalAttr;
+  case attr::Override:
+    return CXCursor_CXXOverrideAttr;
+  case attr::Annotate:
+    return CXCursor_AnnotateAttr;
+  case attr::AsmLabel:
+    return CXCursor_AsmLabelAttr;
+  case attr::Packed:
+    return CXCursor_PackedAttr;
+  case attr::Pure:
+    return CXCursor_PureAttr;
+  case attr::Const:
+    return CXCursor_ConstAttr;
+  case attr::NoDuplicate:
+    return CXCursor_NoDuplicateAttr;
+  case attr::CUDAConstant:
+    return CXCursor_CUDAConstantAttr;
+  case attr::CUDADevice:
+    return CXCursor_CUDADeviceAttr;
+  case attr::CUDAGlobal:
+    return CXCursor_CUDAGlobalAttr;
+  case attr::CUDAHost:
+    return CXCursor_CUDAHostAttr;
+  case attr::CUDAShared:
+    return CXCursor_CUDASharedAttr;
+  case attr::Visibility:
+    return CXCursor_VisibilityAttr;
+  case attr::DLLExport:
+    return CXCursor_DLLExport;
+  case attr::DLLImport:
+    return CXCursor_DLLImport;
+  case attr::NSReturnsRetained:
+    return CXCursor_NSReturnsRetained;
+  case attr::NSReturnsNotRetained:
+    return CXCursor_NSReturnsNotRetained;
+  case attr::NSReturnsAutoreleased:
+    return CXCursor_NSReturnsAutoreleased;
+  case attr::NSConsumesSelf:
+    return CXCursor_NSConsumesSelf;
+  case attr::NSConsumed:
+    return CXCursor_NSConsumed;
+  case attr::ObjCException:
+    return CXCursor_ObjCException;
+  case attr::ObjCNSObject:
+    return CXCursor_ObjCNSObject;
+  case attr::ObjCIndependentClass:
+    return CXCursor_ObjCIndependentClass;
+  case attr::ObjCPreciseLifetime:
+    return CXCursor_ObjCPreciseLifetime;
+  case attr::ObjCReturnsInnerPointer:
+    return CXCursor_ObjCReturnsInnerPointer;
+  case attr::ObjCRequiresSuper:
+    return CXCursor_ObjCRequiresSuper;
+  case attr::ObjCRootClass:
+    return CXCursor_ObjCRootClass;
+  case attr::ObjCSubclassingRestricted:
+    return CXCursor_ObjCSubclassingRestricted;
+  case attr::ObjCExplicitProtocolImpl:
+    return CXCursor_ObjCExplicitProtocolImpl;
+  case attr::ObjCDesignatedInitializer:
+    return CXCursor_ObjCDesignatedInitializer;
+  case attr::ObjCRuntimeVisible:
+    return CXCursor_ObjCRuntimeVisible;
+  case attr::ObjCBoxable:
+    return CXCursor_ObjCBoxable;
+  case attr::FlagEnum:
+    return CXCursor_FlagEnum;
+  case attr::Convergent:
+    return CXCursor_ConvergentAttr;
+  case attr::WarnUnused:
+    return CXCursor_WarnUnusedAttr;
+  case attr::WarnUnusedResult:
+    return CXCursor_WarnUnusedResultAttr;
+  case attr::Aligned:
+    return CXCursor_AlignedAttr;
   }
 
   return CXCursor_UnexposedAttr;
@@ -90,7 +132,7 @@ static CXCursorKind GetCursorKind(const Attr *A) {
 CXCursor cxcursor::MakeCXCursor(const Attr *A, const Decl *Parent,
                                 CXTranslationUnit TU) {
   assert(A && Parent && TU && "Invalid arguments!");
-  CXCursor C = { GetCursorKind(A), 0, { Parent, A, TU } };
+  CXCursor C = {GetCursorKind(A), 0, {Parent, A, TU}};
   return C;
 }
 
@@ -114,12 +156,13 @@ CXCursor cxcursor::MakeCXCursor(const Decl *D, CXTranslationUnit TU,
       if (I != SelLocs.end())
         SelectorIdIndex = I - SelLocs.begin();
     }
-    CXCursor C = { K, SelectorIdIndex,
-                   { D, (void*)(intptr_t) (FirstInDeclGroup ? 1 : 0), TU }};
+    CXCursor C = {K,
+                  SelectorIdIndex,
+                  {D, (void *)(intptr_t)(FirstInDeclGroup ? 1 : 0), TU}};
     return C;
   }
-  
-  CXCursor C = { K, 0, { D, (void*)(intptr_t) (FirstInDeclGroup ? 1 : 0), TU }};
+
+  CXCursor C = {K, 0, {D, (void *)(intptr_t)(FirstInDeclGroup ? 1 : 0), TU}};
   return C;
 }
 
@@ -128,59 +171,59 @@ CXCursor cxcursor::MakeCXCursor(const Stmt *S, const Decl *Parent,
                                 SourceRange RegionOfInterest) {
   assert(S && TU && "Invalid arguments!");
   CXCursorKind K = CXCursor_NotImplemented;
-  
+
   switch (S->getStmtClass()) {
   case Stmt::NoStmtClass:
     break;
-  
+
   case Stmt::CaseStmtClass:
     K = CXCursor_CaseStmt;
     break;
-  
+
   case Stmt::DefaultStmtClass:
     K = CXCursor_DefaultStmt;
     break;
-  
+
   case Stmt::IfStmtClass:
     K = CXCursor_IfStmt;
     break;
-  
+
   case Stmt::SwitchStmtClass:
     K = CXCursor_SwitchStmt;
     break;
-  
+
   case Stmt::WhileStmtClass:
     K = CXCursor_WhileStmt;
     break;
-  
+
   case Stmt::DoStmtClass:
     K = CXCursor_DoStmt;
     break;
-  
+
   case Stmt::ForStmtClass:
     K = CXCursor_ForStmt;
     break;
-  
+
   case Stmt::GotoStmtClass:
     K = CXCursor_GotoStmt;
     break;
-  
+
   case Stmt::IndirectGotoStmtClass:
     K = CXCursor_IndirectGotoStmt;
     break;
-  
+
   case Stmt::ContinueStmtClass:
     K = CXCursor_ContinueStmt;
     break;
-  
+
   case Stmt::BreakStmtClass:
     K = CXCursor_BreakStmt;
     break;
-  
+
   case Stmt::ReturnStmtClass:
     K = CXCursor_ReturnStmt;
     break;
-  
+
   case Stmt::GCCAsmStmtClass:
     K = CXCursor_GCCAsmStmt;
     break;
@@ -188,55 +231,55 @@ CXCursor cxcursor::MakeCXCursor(const Stmt *S, const Decl *Parent,
   case Stmt::MSAsmStmtClass:
     K = CXCursor_MSAsmStmt;
     break;
-  
+
   case Stmt::ObjCAtTryStmtClass:
     K = CXCursor_ObjCAtTryStmt;
     break;
-  
+
   case Stmt::ObjCAtCatchStmtClass:
     K = CXCursor_ObjCAtCatchStmt;
     break;
-  
+
   case Stmt::ObjCAtFinallyStmtClass:
     K = CXCursor_ObjCAtFinallyStmt;
     break;
-  
+
   case Stmt::ObjCAtThrowStmtClass:
     K = CXCursor_ObjCAtThrowStmt;
     break;
-  
+
   case Stmt::ObjCAtSynchronizedStmtClass:
     K = CXCursor_ObjCAtSynchronizedStmt;
     break;
-  
+
   case Stmt::ObjCAutoreleasePoolStmtClass:
     K = CXCursor_ObjCAutoreleasePoolStmt;
     break;
-  
+
   case Stmt::ObjCForCollectionStmtClass:
     K = CXCursor_ObjCForCollectionStmt;
     break;
-  
+
   case Stmt::CXXCatchStmtClass:
     K = CXCursor_CXXCatchStmt;
     break;
-  
+
   case Stmt::CXXTryStmtClass:
     K = CXCursor_CXXTryStmt;
     break;
-  
+
   case Stmt::CXXForRangeStmtClass:
     K = CXCursor_CXXForRangeStmt;
     break;
-  
+
   case Stmt::SEHTryStmtClass:
     K = CXCursor_SEHTryStmt;
     break;
-  
+
   case Stmt::SEHExceptStmtClass:
     K = CXCursor_SEHExceptStmt;
     break;
-  
+
   case Stmt::SEHFinallyStmtClass:
     K = CXCursor_SEHFinallyStmt;
     break;
@@ -303,8 +346,8 @@ CXCursor cxcursor::MakeCXCursor(const Stmt *S, const Decl *Parent,
     break;
 
   case Stmt::PseudoObjectExprClass:
-    return MakeCXCursor(cast<PseudoObjectExpr>(S)->getSyntacticForm(),
-                        Parent, TU, RegionOfInterest);
+    return MakeCXCursor(cast<PseudoObjectExpr>(S)->getSyntacticForm(), Parent,
+                        TU, RegionOfInterest);
 
   case Stmt::CompoundStmtClass:
     K = CXCursor_CompoundStmt;
@@ -355,8 +398,8 @@ CXCursor cxcursor::MakeCXCursor(const Stmt *S, const Decl *Parent,
     break;
 
   case Stmt::ConstantExprClass:
-    return MakeCXCursor(cast<ConstantExpr>(S)->getSubExpr(),
-                        Parent, TU, RegionOfInterest);
+    return MakeCXCursor(cast<ConstantExpr>(S)->getSubExpr(), Parent, TU,
+                        RegionOfInterest);
 
   case Stmt::ParenExprClass:
     K = CXCursor_ParenExpr;
@@ -380,6 +423,14 @@ CXCursor cxcursor::MakeCXCursor(const Stmt *S, const Decl *Parent,
     K = CXCursor_OMPArraySectionExpr;
     break;
 
+  case Stmt::OMPArrayShapingExprClass:
+    K = CXCursor_OMPArrayShapingExpr;
+    break;
+
+  case Stmt::OMPIteratorExprClass:
+    K = CXCursor_OMPIteratorExpr;
+    break;
+
   case Stmt::BinaryOperatorClass:
     K = CXCursor_BinaryOperator;
     break;
@@ -483,7 +534,7 @@ CXCursor cxcursor::MakeCXCursor(const Stmt *S, const Decl *Parent,
   case Stmt::ObjCProtocolExprClass:
     K = CXCursor_ObjCProtocolExpr;
     break;
-      
+
   case Stmt::ObjCBoolLiteralExprClass:
     K = CXCursor_ObjCBoolLiteralExpr;
     break;
@@ -509,10 +560,10 @@ CXCursor cxcursor::MakeCXCursor(const Stmt *S, const Decl *Parent,
     break;
 
   case Stmt::DeclRefExprClass:
-    if (const ImplicitParamDecl *IPD =
-         dyn_cast_or_null<ImplicitParamDecl>(cast<DeclRefExpr>(S)->getDecl())) {
+    if (const ImplicitParamDecl *IPD = dyn_cast_or_null<ImplicitParamDecl>(
+            cast<DeclRefExpr>(S)->getDecl())) {
       if (const ObjCMethodDecl *MD =
-            dyn_cast<ObjCMethodDecl>(IPD->getDeclContext())) {
+              dyn_cast<ObjCMethodDecl>(IPD->getDeclContext())) {
         if (MD->getSelfDecl() == IPD) {
           K = CXCursor_ObjCSelfExpr;
           break;
@@ -531,34 +582,34 @@ CXCursor cxcursor::MakeCXCursor(const Stmt *S, const Decl *Parent,
   case Stmt::TypoExprClass: // A typo could actually be a DeclRef or a MemberRef
     K = CXCursor_DeclRefExpr;
     break;
-      
+
   case Stmt::CXXDependentScopeMemberExprClass:
   case Stmt::CXXPseudoDestructorExprClass:
-  case Stmt::MemberExprClass:            
+  case Stmt::MemberExprClass:
   case Stmt::MSPropertyRefExprClass:
   case Stmt::ObjCIsaExprClass:
-  case Stmt::ObjCIvarRefExprClass:    
-  case Stmt::ObjCPropertyRefExprClass: 
+  case Stmt::ObjCIvarRefExprClass:
+  case Stmt::ObjCPropertyRefExprClass:
   case Stmt::UnresolvedMemberExprClass:
     K = CXCursor_MemberRefExpr;
     break;
-      
-  case Stmt::CallExprClass:              
+
+  case Stmt::CallExprClass:
   case Stmt::CXXOperatorCallExprClass:
   case Stmt::CXXMemberCallExprClass:
   case Stmt::CUDAKernelCallExprClass:
-  case Stmt::CXXConstructExprClass:  
-  case Stmt::CXXInheritedCtorInitExprClass:  
+  case Stmt::CXXConstructExprClass:
+  case Stmt::CXXInheritedCtorInitExprClass:
   case Stmt::CXXTemporaryObjectExprClass:
   case Stmt::CXXUnresolvedConstructExprClass:
   case Stmt::UserDefinedLiteralClass:
     K = CXCursor_CallExpr;
     break;
-      
+
   case Stmt::LambdaExprClass:
     K = CXCursor_LambdaExpr;
     break;
-      
+
   case Stmt::ObjCMessageExprClass: {
     K = CXCursor_ObjCMessageExpr;
     int SelectorIdIndex = -1;
@@ -572,10 +623,10 @@ CXCursor cxcursor::MakeCXCursor(const Stmt *S, const Decl *Parent,
       if (I != SelLocs.end())
         SelectorIdIndex = I - SelLocs.begin();
     }
-    CXCursor C = { K, 0, { Parent, S, TU } };
+    CXCursor C = {K, 0, {Parent, S, TU}};
     return getSelectorIdentifierCursor(SelectorIdIndex, C);
   }
-      
+
   case Stmt::MSDependentExistsStmtClass:
     K = CXCursor_UnexposedStmt;
     break;
@@ -745,17 +796,17 @@ CXCursor cxcursor::MakeCXCursor(const Stmt *S, const Decl *Parent,
     K = CXCursor_BuiltinBitCastExpr;
   }
 
-  CXCursor C = { K, 0, { Parent, S, TU } };
+  CXCursor C = {K, 0, {Parent, S, TU}};
   return C;
 }
 
-CXCursor cxcursor::MakeCursorObjCSuperClassRef(ObjCInterfaceDecl *Super, 
-                                               SourceLocation Loc, 
+CXCursor cxcursor::MakeCursorObjCSuperClassRef(ObjCInterfaceDecl *Super,
+                                               SourceLocation Loc,
                                                CXTranslationUnit TU) {
   assert(Super && TU && "Invalid arguments!");
   void *RawLoc = Loc.getPtrEncoding();
-  CXCursor C = { CXCursor_ObjCSuperClassRef, 0, { Super, RawLoc, TU } };
-  return C;    
+  CXCursor C = {CXCursor_ObjCSuperClassRef, 0, {Super, RawLoc, TU}};
+  return C;
 }
 
 std::pair<const ObjCInterfaceDecl *, SourceLocation>
@@ -765,13 +816,13 @@ cxcursor::getCursorObjCSuperClassRef(CXCursor C) {
                         SourceLocation::getFromPtrEncoding(C.data[1]));
 }
 
-CXCursor cxcursor::MakeCursorObjCProtocolRef(const ObjCProtocolDecl *Proto, 
-                                             SourceLocation Loc, 
+CXCursor cxcursor::MakeCursorObjCProtocolRef(const ObjCProtocolDecl *Proto,
+                                             SourceLocation Loc,
                                              CXTranslationUnit TU) {
   assert(Proto && TU && "Invalid arguments!");
   void *RawLoc = Loc.getPtrEncoding();
-  CXCursor C = { CXCursor_ObjCProtocolRef, 0, { Proto, RawLoc, TU } };
-  return C;    
+  CXCursor C = {CXCursor_ObjCProtocolRef, 0, {Proto, RawLoc, TU}};
+  return C;
 }
 
 std::pair<const ObjCProtocolDecl *, SourceLocation>
@@ -781,16 +832,16 @@ cxcursor::getCursorObjCProtocolRef(CXCursor C) {
                         SourceLocation::getFromPtrEncoding(C.data[1]));
 }
 
-CXCursor cxcursor::MakeCursorObjCClassRef(const ObjCInterfaceDecl *Class, 
-                                          SourceLocation Loc, 
+CXCursor cxcursor::MakeCursorObjCClassRef(const ObjCInterfaceDecl *Class,
+                                          SourceLocation Loc,
                                           CXTranslationUnit TU) {
   // 'Class' can be null for invalid code.
   if (!Class)
     return MakeCXCursorInvalid(CXCursor_InvalidCode);
   assert(TU && "Invalid arguments!");
   void *RawLoc = Loc.getPtrEncoding();
-  CXCursor C = { CXCursor_ObjCClassRef, 0, { Class, RawLoc, TU } };
-  return C;    
+  CXCursor C = {CXCursor_ObjCClassRef, 0, {Class, RawLoc, TU}};
+  return C;
 }
 
 std::pair<const ObjCInterfaceDecl *, SourceLocation>
@@ -800,12 +851,12 @@ cxcursor::getCursorObjCClassRef(CXCursor C) {
                         SourceLocation::getFromPtrEncoding(C.data[1]));
 }
 
-CXCursor cxcursor::MakeCursorTypeRef(const TypeDecl *Type, SourceLocation Loc, 
+CXCursor cxcursor::MakeCursorTypeRef(const TypeDecl *Type, SourceLocation Loc,
                                      CXTranslationUnit TU) {
   assert(Type && TU && "Invalid arguments!");
   void *RawLoc = Loc.getPtrEncoding();
-  CXCursor C = { CXCursor_TypeRef, 0, { Type, RawLoc, TU } };
-  return C;    
+  CXCursor C = {CXCursor_TypeRef, 0, {Type, RawLoc, TU}};
+  return C;
 }
 
 std::pair<const TypeDecl *, SourceLocation>
@@ -815,13 +866,13 @@ cxcursor::getCursorTypeRef(CXCursor C) {
                         SourceLocation::getFromPtrEncoding(C.data[1]));
 }
 
-CXCursor cxcursor::MakeCursorTemplateRef(const TemplateDecl *Template, 
+CXCursor cxcursor::MakeCursorTemplateRef(const TemplateDecl *Template,
                                          SourceLocation Loc,
                                          CXTranslationUnit TU) {
   assert(Template && TU && "Invalid arguments!");
   void *RawLoc = Loc.getPtrEncoding();
-  CXCursor C = { CXCursor_TemplateRef, 0, { Template, RawLoc, TU } };
-  return C;    
+  CXCursor C = {CXCursor_TemplateRef, 0, {Template, RawLoc, TU}};
+  return C;
 }
 
 std::pair<const TemplateDecl *, SourceLocation>
@@ -832,14 +883,14 @@ cxcursor::getCursorTemplateRef(CXCursor C) {
 }
 
 CXCursor cxcursor::MakeCursorNamespaceRef(const NamedDecl *NS,
-                                          SourceLocation Loc, 
+                                          SourceLocation Loc,
                                           CXTranslationUnit TU) {
-  
+
   assert(NS && (isa<NamespaceDecl>(NS) || isa<NamespaceAliasDecl>(NS)) && TU &&
          "Invalid arguments!");
   void *RawLoc = Loc.getPtrEncoding();
-  CXCursor C = { CXCursor_NamespaceRef, 0, { NS, RawLoc, TU } };
-  return C;    
+  CXCursor C = {CXCursor_NamespaceRef, 0, {NS, RawLoc, TU}};
+  return C;
 }
 
 std::pair<const NamedDecl *, SourceLocation>
@@ -849,12 +900,12 @@ cxcursor::getCursorNamespaceRef(CXCursor C) {
                         SourceLocation::getFromPtrEncoding(C.data[1]));
 }
 
-CXCursor cxcursor::MakeCursorVariableRef(const VarDecl *Var, SourceLocation Loc, 
+CXCursor cxcursor::MakeCursorVariableRef(const VarDecl *Var, SourceLocation Loc,
                                          CXTranslationUnit TU) {
-  
+
   assert(Var && TU && "Invalid arguments!");
   void *RawLoc = Loc.getPtrEncoding();
-  CXCursor C = { CXCursor_VariableRef, 0, { Var, RawLoc, TU } };
+  CXCursor C = {CXCursor_VariableRef, 0, {Var, RawLoc, TU}};
   return C;
 }
 
@@ -865,13 +916,14 @@ cxcursor::getCursorVariableRef(CXCursor C) {
                         SourceLocation::getFromPtrEncoding(C.data[1]));
 }
 
-CXCursor cxcursor::MakeCursorMemberRef(const FieldDecl *Field, SourceLocation Loc, 
+CXCursor cxcursor::MakeCursorMemberRef(const FieldDecl *Field,
+                                       SourceLocation Loc,
                                        CXTranslationUnit TU) {
-  
+
   assert(Field && TU && "Invalid arguments!");
   void *RawLoc = Loc.getPtrEncoding();
-  CXCursor C = { CXCursor_MemberRef, 0, { Field, RawLoc, TU } };
-  return C;    
+  CXCursor C = {CXCursor_MemberRef, 0, {Field, RawLoc, TU}};
+  return C;
 }
 
 std::pair<const FieldDecl *, SourceLocation>
@@ -882,23 +934,22 @@ cxcursor::getCursorMemberRef(CXCursor C) {
 }
 
 CXCursor cxcursor::MakeCursorCXXBaseSpecifier(const CXXBaseSpecifier *B,
-                                              CXTranslationUnit TU){
-  CXCursor C = { CXCursor_CXXBaseSpecifier, 0, { B, nullptr, TU } };
-  return C;  
+                                              CXTranslationUnit TU) {
+  CXCursor C = {CXCursor_CXXBaseSpecifier, 0, {B, nullptr, TU}};
+  return C;
 }
 
 const CXXBaseSpecifier *cxcursor::getCursorCXXBaseSpecifier(CXCursor C) {
   assert(C.kind == CXCursor_CXXBaseSpecifier);
-  return static_cast<const CXXBaseSpecifier*>(C.data[0]);
+  return static_cast<const CXXBaseSpecifier *>(C.data[0]);
 }
 
-CXCursor cxcursor::MakePreprocessingDirectiveCursor(SourceRange Range, 
+CXCursor cxcursor::MakePreprocessingDirectiveCursor(SourceRange Range,
                                                     CXTranslationUnit TU) {
-  CXCursor C = { CXCursor_PreprocessingDirective, 0,
-                 { Range.getBegin().getPtrEncoding(),
-                   Range.getEnd().getPtrEncoding(),
-                   TU }
-               };
+  CXCursor C = {
+      CXCursor_PreprocessingDirective,
+      0,
+      {Range.getBegin().getPtrEncoding(), Range.getEnd().getPtrEncoding(), TU}};
   return C;
 }
 
@@ -923,7 +974,7 @@ const MacroDefinitionRecord *cxcursor::getCursorMacroDefinition(CXCursor C) {
 
 CXCursor cxcursor::MakeMacroExpansionCursor(MacroExpansion *MI,
                                             CXTranslationUnit TU) {
-  CXCursor C = { CXCursor_MacroExpansion, 0, { MI, nullptr, TU } };
+  CXCursor C = {CXCursor_MacroExpansion, 0, {MI, nullptr, TU}};
   return C;
 }
 
@@ -952,9 +1003,9 @@ SourceRange cxcursor::MacroExpansionCursor::getSourceRange() const {
   return getAsMacroExpansion()->getSourceRange();
 }
 
-CXCursor cxcursor::MakeInclusionDirectiveCursor(InclusionDirective *ID, 
+CXCursor cxcursor::MakeInclusionDirectiveCursor(InclusionDirective *ID,
                                                 CXTranslationUnit TU) {
-  CXCursor C = { CXCursor_InclusionDirective, 0, { ID, nullptr, TU } };
+  CXCursor C = {CXCursor_InclusionDirective, 0, {ID, nullptr, TU}};
   return C;
 }
 
@@ -963,13 +1014,13 @@ const InclusionDirective *cxcursor::getCursorInclusionDirective(CXCursor C) {
   return static_cast<const InclusionDirective *>(C.data[0]);
 }
 
-CXCursor cxcursor::MakeCursorLabelRef(LabelStmt *Label, SourceLocation Loc, 
+CXCursor cxcursor::MakeCursorLabelRef(LabelStmt *Label, SourceLocation Loc,
                                       CXTranslationUnit TU) {
-  
+
   assert(Label && TU && "Invalid arguments!");
   void *RawLoc = Loc.getPtrEncoding();
-  CXCursor C = { CXCursor_LabelRef, 0, { Label, RawLoc, TU } };
-  return C;    
+  CXCursor C = {CXCursor_LabelRef, 0, {Label, RawLoc, TU}};
+  return C;
 }
 
 std::pair<const LabelStmt *, SourceLocation>
@@ -984,11 +1035,9 @@ CXCursor cxcursor::MakeCursorOverloadedDeclRef(const OverloadExpr *E,
   assert(E && TU && "Invalid arguments!");
   OverloadedDeclRefStorage Storage(E);
   void *RawLoc = E->getNameLoc().getPtrEncoding();
-  CXCursor C = { 
-                 CXCursor_OverloadedDeclRef, 0,
-                 { Storage.getOpaqueValue(), RawLoc, TU } 
-               };
-  return C;    
+  CXCursor C = {
+      CXCursor_OverloadedDeclRef, 0, {Storage.getOpaqueValue(), RawLoc, TU}};
+  return C;
 }
 
 CXCursor cxcursor::MakeCursorOverloadedDeclRef(const Decl *D,
@@ -997,31 +1046,27 @@ CXCursor cxcursor::MakeCursorOverloadedDeclRef(const Decl *D,
   assert(D && TU && "Invalid arguments!");
   void *RawLoc = Loc.getPtrEncoding();
   OverloadedDeclRefStorage Storage(D);
-  CXCursor C = { 
-    CXCursor_OverloadedDeclRef, 0,
-    { Storage.getOpaqueValue(), RawLoc, TU }
-  };
-  return C;    
+  CXCursor C = {
+      CXCursor_OverloadedDeclRef, 0, {Storage.getOpaqueValue(), RawLoc, TU}};
+  return C;
 }
 
-CXCursor cxcursor::MakeCursorOverloadedDeclRef(TemplateName Name, 
+CXCursor cxcursor::MakeCursorOverloadedDeclRef(TemplateName Name,
                                                SourceLocation Loc,
                                                CXTranslationUnit TU) {
   assert(Name.getAsOverloadedTemplate() && TU && "Invalid arguments!");
   void *RawLoc = Loc.getPtrEncoding();
   OverloadedDeclRefStorage Storage(Name.getAsOverloadedTemplate());
-  CXCursor C = { 
-    CXCursor_OverloadedDeclRef, 0,
-    { Storage.getOpaqueValue(), RawLoc, TU } 
-  };
-  return C;    
+  CXCursor C = {
+      CXCursor_OverloadedDeclRef, 0, {Storage.getOpaqueValue(), RawLoc, TU}};
+  return C;
 }
 
 std::pair<cxcursor::OverloadedDeclRefStorage, SourceLocation>
 cxcursor::getCursorOverloadedDeclRef(CXCursor C) {
   assert(C.kind == CXCursor_OverloadedDeclRef);
   return std::make_pair(OverloadedDeclRefStorage::getFromOpaqueValue(
-                                       const_cast<void *>(C.data[0])),
+                            const_cast<void *>(C.data[0])),
                         SourceLocation::getFromPtrEncoding(C.data[1]));
 }
 
@@ -1058,11 +1103,11 @@ ASTUnit *cxcursor::getCursorASTUnit(CXCursor Cursor) {
 }
 
 CXTranslationUnit cxcursor::getCursorTU(CXCursor Cursor) {
-  return static_cast<CXTranslationUnit>(const_cast<void*>(Cursor.data[2]));
+  return static_cast<CXTranslationUnit>(const_cast<void *>(Cursor.data[2]));
 }
 
 void cxcursor::getOverriddenCursors(CXCursor cursor,
-                                    SmallVectorImpl<CXCursor> &overridden) { 
+                                    SmallVectorImpl<CXCursor> &overridden) {
   assert(clang_isDeclaration(cursor.kind));
   const NamedDecl *D = dyn_cast_or_null<NamedDecl>(getCursorDecl(cursor));
   if (!D)
@@ -1072,8 +1117,9 @@ void cxcursor::getOverriddenCursors(CXCursor cursor,
   SmallVector<const NamedDecl *, 8> OverDecls;
   D->getASTContext().getOverriddenMethods(D, OverDecls);
 
-  for (SmallVectorImpl<const NamedDecl *>::iterator
-         I = OverDecls.begin(), E = OverDecls.end(); I != E; ++I) {
+  for (SmallVectorImpl<const NamedDecl *>::iterator I = OverDecls.begin(),
+                                                    E = OverDecls.end();
+       I != E; ++I) {
     overridden.push_back(MakeCXCursor(*I, TU));
   }
 }
@@ -1084,13 +1130,13 @@ cxcursor::getSelectorIdentifierIndexAndLoc(CXCursor cursor) {
     if (cursor.xdata != -1)
       return std::make_pair(cursor.xdata,
                             cast<ObjCMessageExpr>(getCursorExpr(cursor))
-                                                ->getSelectorLoc(cursor.xdata));
+                                ->getSelectorLoc(cursor.xdata));
   } else if (cursor.kind == CXCursor_ObjCClassMethodDecl ||
              cursor.kind == CXCursor_ObjCInstanceMethodDecl) {
     if (cursor.xdata != -1)
       return std::make_pair(cursor.xdata,
                             cast<ObjCMethodDecl>(getCursorDecl(cursor))
-                                                ->getSelectorLoc(cursor.xdata));
+                                ->getSelectorLoc(cursor.xdata));
   }
 
   return std::make_pair(-1, SourceLocation());
@@ -1101,16 +1147,16 @@ CXCursor cxcursor::getSelectorIdentifierCursor(int SelIdx, CXCursor cursor) {
 
   if (cursor.kind == CXCursor_ObjCMessageExpr) {
     if (SelIdx == -1 ||
-        unsigned(SelIdx) >= cast<ObjCMessageExpr>(getCursorExpr(cursor))
-                                                         ->getNumSelectorLocs())
+        unsigned(SelIdx) >=
+            cast<ObjCMessageExpr>(getCursorExpr(cursor))->getNumSelectorLocs())
       newCursor.xdata = -1;
     else
       newCursor.xdata = SelIdx;
   } else if (cursor.kind == CXCursor_ObjCClassMethodDecl ||
              cursor.kind == CXCursor_ObjCInstanceMethodDecl) {
     if (SelIdx == -1 ||
-        unsigned(SelIdx) >= cast<ObjCMethodDecl>(getCursorDecl(cursor))
-                                                         ->getNumSelectorLocs())
+        unsigned(SelIdx) >=
+            cast<ObjCMethodDecl>(getCursorDecl(cursor))->getNumSelectorLocs())
       newCursor.xdata = -1;
     else
       newCursor.xdata = SelIdx;
@@ -1128,11 +1174,11 @@ CXCursor cxcursor::getTypeRefCursor(CXCursor cursor) {
 
   const Expr *E = getCursorExpr(cursor);
   TypeSourceInfo *Type = nullptr;
-  if (const CXXUnresolvedConstructExpr *
-        UnCtor = dyn_cast<CXXUnresolvedConstructExpr>(E)) {
+  if (const CXXUnresolvedConstructExpr *UnCtor =
+          dyn_cast<CXXUnresolvedConstructExpr>(E)) {
     Type = UnCtor->getTypeSourceInfo();
   } else if (const CXXTemporaryObjectExpr *Tmp =
-                 dyn_cast<CXXTemporaryObjectExpr>(E)){
+                 dyn_cast<CXXTemporaryObjectExpr>(E)) {
     Type = Tmp->getTypeSourceInfo();
   }
 
@@ -1169,7 +1215,7 @@ bool cxcursor::operator==(CXCursor X, CXCursor Y) {
 // properly in the ASTs.
 bool cxcursor::isFirstInDeclGroup(CXCursor C) {
   assert(clang_isDeclaration(C.kind));
-  return ((uintptr_t) (C.data[1])) != 0;
+  return ((uintptr_t)(C.data[1])) != 0;
 }
 
 //===----------------------------------------------------------------------===//
@@ -1224,15 +1270,13 @@ CXCursor clang_Cursor_getArgument(CXCursor C, unsigned i) {
     const Expr *E = cxcursor::getCursorExpr(C);
     if (const CallExpr *CE = dyn_cast<CallExpr>(E)) {
       if (i < CE->getNumArgs()) {
-        return cxcursor::MakeCXCursor(CE->getArg(i),
-                                      getCursorDecl(C),
+        return cxcursor::MakeCXCursor(CE->getArg(i), getCursorDecl(C),
                                       cxcursor::getCursorTU(C));
       }
     }
     if (const CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(E)) {
       if (i < CE->getNumArgs()) {
-        return cxcursor::MakeCXCursor(CE->getArg(i),
-                                      getCursorDecl(C),
+        return cxcursor::MakeCXCursor(CE->getArg(i), getCursorDecl(C),
                                       cxcursor::getCursorTU(C));
       }
     }
@@ -1246,13 +1290,13 @@ int clang_Cursor_getNumTemplateArguments(CXCursor C) {
     return -1;
   }
 
-  const FunctionDecl *FD = llvm::dyn_cast_or_null<clang::FunctionDecl>(
-      getCursorDecl(C));
+  const FunctionDecl *FD =
+      llvm::dyn_cast_or_null<clang::FunctionDecl>(getCursorDecl(C));
   if (!FD) {
     return -1;
   }
 
-  const FunctionTemplateSpecializationInfo* SpecInfo =
+  const FunctionTemplateSpecializationInfo *SpecInfo =
       FD->getTemplateSpecializationInfo();
   if (!SpecInfo) {
     return -1;
@@ -1278,19 +1322,19 @@ enum CXGetTemplateArgumentStatus {
   CXGetTemplateArgumentStatus_InvalidIndex = -4
 };
 
-static int clang_Cursor_getTemplateArgument(
-    CXCursor C, unsigned I, TemplateArgument *TA) {
+static int clang_Cursor_getTemplateArgument(CXCursor C, unsigned I,
+                                            TemplateArgument *TA) {
   if (clang_getCursorKind(C) != CXCursor_FunctionDecl) {
     return CXGetTemplateArgumentStatus_CursorNotFunctionDecl;
   }
 
-  const FunctionDecl *FD = llvm::dyn_cast_or_null<clang::FunctionDecl>(
-      getCursorDecl(C));
+  const FunctionDecl *FD =
+      llvm::dyn_cast_or_null<clang::FunctionDecl>(getCursorDecl(C));
   if (!FD) {
     return CXGetTemplateArgumentStatus_BadFunctionDeclCast;
   }
 
-  const FunctionTemplateSpecializationInfo* SpecInfo =
+  const FunctionTemplateSpecializationInfo *SpecInfo =
       FD->getTemplateSpecializationInfo();
   if (!SpecInfo) {
     return CXGetTemplateArgumentStatus_NullTemplSpecInfo;
@@ -1312,17 +1356,24 @@ enum CXTemplateArgumentKind clang_Cursor_getTemplateArgumentKind(CXCursor C,
   }
 
   switch (TA.getKind()) {
-    case TemplateArgument::Null: return CXTemplateArgumentKind_Null;
-    case TemplateArgument::Type: return CXTemplateArgumentKind_Type;
-    case TemplateArgument::Declaration:
-      return CXTemplateArgumentKind_Declaration;
-    case TemplateArgument::NullPtr: return CXTemplateArgumentKind_NullPtr;
-    case TemplateArgument::Integral: return CXTemplateArgumentKind_Integral;
-    case TemplateArgument::Template: return CXTemplateArgumentKind_Template;
-    case TemplateArgument::TemplateExpansion:
-      return CXTemplateArgumentKind_TemplateExpansion;
-    case TemplateArgument::Expression: return CXTemplateArgumentKind_Expression;
-    case TemplateArgument::Pack: return CXTemplateArgumentKind_Pack;
+  case TemplateArgument::Null:
+    return CXTemplateArgumentKind_Null;
+  case TemplateArgument::Type:
+    return CXTemplateArgumentKind_Type;
+  case TemplateArgument::Declaration:
+    return CXTemplateArgumentKind_Declaration;
+  case TemplateArgument::NullPtr:
+    return CXTemplateArgumentKind_NullPtr;
+  case TemplateArgument::Integral:
+    return CXTemplateArgumentKind_Integral;
+  case TemplateArgument::Template:
+    return CXTemplateArgumentKind_Template;
+  case TemplateArgument::TemplateExpansion:
+    return CXTemplateArgumentKind_TemplateExpansion;
+  case TemplateArgument::Expression:
+    return CXTemplateArgumentKind_Expression;
+  case TemplateArgument::Pack:
+    return CXTemplateArgumentKind_Pack;
   }
 
   return CXTemplateArgumentKind_Invalid;
@@ -1382,13 +1433,13 @@ unsigned long long clang_Cursor_getTemplateArgumentUnsignedValue(CXCursor C,
 typedef llvm::DenseMap<CXCursor, unsigned> CXCursorSet_Impl;
 
 static inline CXCursorSet packCXCursorSet(CXCursorSet_Impl *setImpl) {
-  return (CXCursorSet) setImpl;
+  return (CXCursorSet)setImpl;
 }
 static inline CXCursorSet_Impl *unpackCXCursorSet(CXCursorSet set) {
-  return (CXCursorSet_Impl*) set;
+  return (CXCursorSet_Impl *)set;
 }
 namespace llvm {
-template<> struct DenseMapInfo<CXCursor> {
+template <> struct DenseMapInfo<CXCursor> {
 public:
   static inline CXCursor getEmptyKey() {
     return MakeCXCursorInvalid(CXCursor_InvalidFile);
@@ -1397,16 +1448,14 @@ template<> struct DenseMapInfo<CXCursor> {
     return MakeCXCursorInvalid(CXCursor_NoDeclFound);
   }
   static inline unsigned getHashValue(const CXCursor &cursor) {
-    return llvm::DenseMapInfo<std::pair<const void *, const void *> >
-      ::getHashValue(std::make_pair(cursor.data[0], cursor.data[1]));
+    return llvm::DenseMapInfo<std::pair<const void *, const void *>>::
+        getHashValue(std::make_pair(cursor.data[0], cursor.data[1]));
   }
   static inline bool isEqual(const CXCursor &x, const CXCursor &y) {
-    return x.kind == y.kind &&
-           x.data[0] == y.data[0] &&
-           x.data[1] == y.data[1];
+    return x.kind == y.kind && x.data[0] == y.data[0] && x.data[1] == y.data[1];
   }
 };
-}
+} // namespace llvm
 
 CXCursorSet clang_createCXCursorSet() {
   return packCXCursorSet(new CXCursorSet_Impl());
@@ -1437,7 +1486,7 @@ unsigned clang_CXCursorSet_insert(CXCursorSet set, CXCursor cursor) {
   entry = 1;
   return flag;
 }
-  
+
 CXCompletionString clang_getCursorCompletionString(CXCursor cursor) {
   enum CXCursorKind kind = clang_getCursorKind(cursor);
   if (clang_isDeclaration(kind)) {
@@ -1445,13 +1494,11 @@ CXCompletionString clang_getCursorCompletionString(CXCursor cursor) {
     if (const NamedDecl *namedDecl = dyn_cast_or_null<NamedDecl>(decl)) {
       ASTUnit *unit = getCursorASTUnit(cursor);
       CodeCompletionResult Result(namedDecl, CCP_Declaration);
-      CodeCompletionString *String
-        = Result.CreateCodeCompletionString(unit->getASTContext(),
-                                            unit->getPreprocessor(),
-                                            CodeCompletionContext::CCC_Other,
-                                 unit->getCodeCompletionTUInfo().getAllocator(),
-                                 unit->getCodeCompletionTUInfo(),
-                                 true);
+      CodeCompletionString *String = Result.CreateCodeCompletionString(
+          unit->getASTContext(), unit->getPreprocessor(),
+          CodeCompletionContext::CCC_Other,
+          unit->getCodeCompletionTUInfo().getAllocator(),
+          unit->getCodeCompletionTUInfo(), true);
       return String;
     }
   } else if (kind == CXCursor_MacroDefinition) {
@@ -1472,58 +1519,57 @@ CXCompletionString clang_getCursorCompletionString(CXCursor cursor) {
 }
 
 namespace {
-  struct OverridenCursorsPool {
-    typedef SmallVector<CXCursor, 2> CursorVec;
-    std::vector<CursorVec*> AllCursors;
-    std::vector<CursorVec*> AvailableCursors;
-    
-    ~OverridenCursorsPool() {
-      for (std::vector<CursorVec*>::iterator I = AllCursors.begin(),
-           E = AllCursors.end(); I != E; ++I) {
-        delete *I;
-      }
+struct OverridenCursorsPool {
+  typedef SmallVector<CXCursor, 2> CursorVec;
+  std::vector<CursorVec *> AllCursors;
+  std::vector<CursorVec *> AvailableCursors;
+
+  ~OverridenCursorsPool() {
+    for (std::vector<CursorVec *>::iterator I = AllCursors.begin(),
+                                            E = AllCursors.end();
+         I != E; ++I) {
+      delete *I;
     }
-  };
-}
+  }
+};
+} // namespace
 
 void *cxcursor::createOverridenCXCursorsPool() {
   return new OverridenCursorsPool();
 }
-  
+
 void cxcursor::disposeOverridenCXCursorsPool(void *pool) {
-  delete static_cast<OverridenCursorsPool*>(pool);
+  delete static_cast<OverridenCursorsPool *>(pool);
 }
- 
-void clang_getOverriddenCursors(CXCursor cursor,
-                                CXCursor **overridden,
+
+void clang_getOverriddenCursors(CXCursor cursor, CXCursor **overridden,
                                 unsigned *num_overridden) {
   if (overridden)
     *overridden = nullptr;
   if (num_overridden)
     *num_overridden = 0;
-  
+
   CXTranslationUnit TU = cxcursor::getCursorTU(cursor);
-  
+
   if (!overridden || !num_overridden || !TU)
     return;
 
   if (!clang_isDeclaration(cursor.kind))
     return;
-    
+
   OverridenCursorsPool &pool =
-    *static_cast<OverridenCursorsPool*>(TU->OverridenCursorsPool);
+      *static_cast<OverridenCursorsPool *>(TU->OverridenCursorsPool);
 
   OverridenCursorsPool::CursorVec *Vec = nullptr;
 
   if (!pool.AvailableCursors.empty()) {
     Vec = pool.AvailableCursors.back();
     pool.AvailableCursors.pop_back();
-  }
-  else {
+  } else {
     Vec = new OverridenCursorsPool::CursorVec();
     pool.AllCursors.push_back(Vec);
   }
-  
+
   // Clear out the vector, but don't free the memory contents.  This
   // reduces malloc() traffic.
   Vec->clear();
@@ -1537,7 +1583,7 @@ void clang_getOverriddenCursors(CXCursor cursor,
 
   // Get the overridden cursors.
   cxcursor::getOverriddenCursors(cursor, *Vec);
-  
+
   // Did we get any overridden cursors?  If not, return Vec to the pool
   // of available cursor vectors.
   if (Vec->size() == 1) {
@@ -1554,7 +1600,7 @@ void clang_getOverriddenCursors(CXCursor cursor,
 void clang_disposeOverriddenCursors(CXCursor *overridden) {
   if (!overridden)
     return;
-  
+
   // Use pointer arithmetic to get back the first faux entry
   // which has a back-reference to the TU and the vector.
   --overridden;
@@ -1562,12 +1608,12 @@ void clang_disposeOverriddenCursors(CXCursor *overridden) {
       static_cast<OverridenCursorsPool::CursorVec *>(
           const_cast<void *>(overridden->data[0]));
   CXTranslationUnit TU = getCursorTU(*overridden);
-  
+
   assert(Vec && TU);
 
   OverridenCursorsPool &pool =
-    *static_cast<OverridenCursorsPool*>(TU->OverridenCursorsPool);
-  
+      *static_cast<OverridenCursorsPool *>(TU->OverridenCursorsPool);
+
   pool.AvailableCursors.push_back(Vec);
 }
 
@@ -1600,8 +1646,8 @@ int clang_Cursor_isDynamicCall(CXCursor C) {
     ME = dyn_cast_or_null<MemberExpr>(CE->getCallee());
 
   if (ME) {
-    if (const CXXMethodDecl *
-          MD = dyn_cast_or_null<CXXMethodDecl>(ME->getMemberDecl()))
+    if (const CXXMethodDecl *MD =
+            dyn_cast_or_null<CXXMethodDecl>(ME->getMemberDecl()))
       return MD->isVirtual() &&
              ME->performsVirtualDispatch(
                  cxcursor::getCursorContext(C).getLangOpts());
diff --git a/clang/tools/libclang/CXCursor.h b/clang/tools/libclang/CXCursor.h
index 78e597b2bbea7..1e4c0a05c9ef7 100644
--- a/clang/tools/libclang/CXCursor.h
+++ b/clang/tools/libclang/CXCursor.h
@@ -58,39 +58,36 @@ CXCursor MakeCXCursor(const clang::Stmt *S, const clang::Decl *Parent,
 CXCursor MakeCXCursorInvalid(CXCursorKind K, CXTranslationUnit TU = nullptr);
 
 /// Create an Objective-C superclass reference at the given location.
-CXCursor MakeCursorObjCSuperClassRef(ObjCInterfaceDecl *Super, 
-                                     SourceLocation Loc, 
-                                     CXTranslationUnit TU);
+CXCursor MakeCursorObjCSuperClassRef(ObjCInterfaceDecl *Super,
+                                     SourceLocation Loc, CXTranslationUnit TU);
 
 /// Unpack an ObjCSuperClassRef cursor into the interface it references
 /// and optionally the location where the reference occurred.
 std::pair<const ObjCInterfaceDecl *, SourceLocation>
-  getCursorObjCSuperClassRef(CXCursor C);
+getCursorObjCSuperClassRef(CXCursor C);
 
 /// Create an Objective-C protocol reference at the given location.
 CXCursor MakeCursorObjCProtocolRef(const ObjCProtocolDecl *Proto,
-                                   SourceLocation Loc, 
-                                   CXTranslationUnit TU);
+                                   SourceLocation Loc, CXTranslationUnit TU);
 
 /// Unpack an ObjCProtocolRef cursor into the protocol it references
 /// and optionally the location where the reference occurred.
 std::pair<const ObjCProtocolDecl *, SourceLocation>
-  getCursorObjCProtocolRef(CXCursor C);
+getCursorObjCProtocolRef(CXCursor C);
 
 /// Create an Objective-C class reference at the given location.
 CXCursor MakeCursorObjCClassRef(const ObjCInterfaceDecl *Class,
-                                SourceLocation Loc, 
-                                CXTranslationUnit TU);
+                                SourceLocation Loc, CXTranslationUnit TU);
 
 /// Unpack an ObjCClassRef cursor into the class it references
 /// and optionally the location where the reference occurred.
 std::pair<const ObjCInterfaceDecl *, SourceLocation>
-  getCursorObjCClassRef(CXCursor C);
+getCursorObjCClassRef(CXCursor C);
 
 /// Create a type reference at the given location.
 CXCursor MakeCursorTypeRef(const TypeDecl *Type, SourceLocation Loc,
                            CXTranslationUnit TU);
-                               
+
 /// Unpack a TypeRef cursor into the class it references
 /// and optionally the location where the reference occurred.
 std::pair<const TypeDecl *, SourceLocation> getCursorTypeRef(CXCursor C);
@@ -102,9 +99,9 @@ CXCursor MakeCursorTemplateRef(const TemplateDecl *Template, SourceLocation Loc,
 /// Unpack a TemplateRef cursor into the template it references and
 /// the location where the reference occurred.
 std::pair<const TemplateDecl *, SourceLocation>
-  getCursorTemplateRef(CXCursor C);
+getCursorTemplateRef(CXCursor C);
 
-/// Create a reference to a namespace or namespace alias at the given 
+/// Create a reference to a namespace or namespace alias at the given
 /// location.
 CXCursor MakeCursorNamespaceRef(const NamedDecl *NS, SourceLocation Loc,
                                 CXTranslationUnit TU);
@@ -114,7 +111,7 @@ CXCursor MakeCursorNamespaceRef(const NamedDecl *NS, SourceLocation Loc,
 std::pair<const NamedDecl *, SourceLocation> getCursorNamespaceRef(CXCursor C);
 
 /// Create a reference to a variable at the given location.
-CXCursor MakeCursorVariableRef(const VarDecl *Var, SourceLocation Loc, 
+CXCursor MakeCursorVariableRef(const VarDecl *Var, SourceLocation Loc,
                                CXTranslationUnit TU);
 
 /// Unpack a VariableRef cursor into the variable it references and the
@@ -122,10 +119,10 @@ CXCursor MakeCursorVariableRef(const VarDecl *Var, SourceLocation Loc,
 std::pair<const VarDecl *, SourceLocation> getCursorVariableRef(CXCursor C);
 
 /// Create a reference to a field at the given location.
-CXCursor MakeCursorMemberRef(const FieldDecl *Field, SourceLocation Loc, 
+CXCursor MakeCursorMemberRef(const FieldDecl *Field, SourceLocation Loc,
                              CXTranslationUnit TU);
 
-/// Unpack a MemberRef cursor into the field it references and the 
+/// Unpack a MemberRef cursor into the field it references and the
 /// location where the reference occurred.
 std::pair<const FieldDecl *, SourceLocation> getCursorMemberRef(CXCursor C);
 
@@ -223,7 +220,7 @@ CXCursor MakeCursorOverloadedDeclRef(const Decl *D, SourceLocation Location,
                                      CXTranslationUnit TU);
 
 /// Create a overloaded declaration reference cursor for a template name.
-CXCursor MakeCursorOverloadedDeclRef(TemplateName Template, 
+CXCursor MakeCursorOverloadedDeclRef(TemplateName Template,
                                      SourceLocation Location,
                                      CXTranslationUnit TU);
 
@@ -235,7 +232,7 @@ typedef llvm::PointerUnion<const OverloadExpr *, const Decl *,
 /// Unpack an overloaded declaration reference into an expression,
 /// declaration, or template name along with the source location.
 std::pair<OverloadedDeclRefStorage, SourceLocation>
-  getCursorOverloadedDeclRef(CXCursor C);
+getCursorOverloadedDeclRef(CXCursor C);
 
 const Decl *getCursorDecl(CXCursor Cursor);
 const Expr *getCursorExpr(CXCursor Cursor);
@@ -284,14 +281,13 @@ bool getDeclCursorUSR(const Decl *D, SmallVectorImpl<char> &Buf);
 
 bool operator==(CXCursor X, CXCursor Y);
 
-inline bool operator!=(CXCursor X, CXCursor Y) {
-  return !(X == Y);
-}
+inline bool operator!=(CXCursor X, CXCursor Y) { return !(X == Y); }
 
 /// Return true if the cursor represents a declaration that is the
 /// first in a declaration group.
 bool isFirstInDeclGroup(CXCursor C);
 
-}} // end namespace: clang::cxcursor
+} // namespace cxcursor
+} // namespace clang
 
 #endif
diff --git a/clang/tools/libclang/CursorVisitor.h b/clang/tools/libclang/CursorVisitor.h
index b0afa5a0b591a..3337fecd0db39 100644
--- a/clang/tools/libclang/CursorVisitor.h
+++ b/clang/tools/libclang/CursorVisitor.h
@@ -16,43 +16,52 @@
 #include "clang/AST/TypeLocVisitor.h"
 
 namespace clang {
-  class PreprocessingRecord;
-  class ASTUnit;
+class PreprocessingRecord;
+class ASTUnit;
 
 namespace cxcursor {
 
 class VisitorJob {
 public:
-  enum Kind { DeclVisitKind, StmtVisitKind, MemberExprPartsKind,
-              TypeLocVisitKind, OverloadExprPartsKind,
-              DeclRefExprPartsKind, LabelRefVisitKind,
-              ExplicitTemplateArgsVisitKind,
-              NestedNameSpecifierLocVisitKind,
-              DeclarationNameInfoVisitKind,
-              MemberRefVisitKind, SizeOfPackExprPartsKind,
-              LambdaExprPartsKind, PostChildrenVisitKind };
+  enum Kind {
+    DeclVisitKind,
+    StmtVisitKind,
+    MemberExprPartsKind,
+    TypeLocVisitKind,
+    OverloadExprPartsKind,
+    DeclRefExprPartsKind,
+    LabelRefVisitKind,
+    ExplicitTemplateArgsVisitKind,
+    NestedNameSpecifierLocVisitKind,
+    DeclarationNameInfoVisitKind,
+    MemberRefVisitKind,
+    SizeOfPackExprPartsKind,
+    LambdaExprPartsKind,
+    PostChildrenVisitKind
+  };
+
 protected:
   const void *data[3];
   CXCursor parent;
   Kind K;
   VisitorJob(CXCursor C, Kind k, const void *d1, const void *d2 = nullptr,
              const void *d3 = nullptr)
-    : parent(C), K(k) {
+      : parent(C), K(k) {
     data[0] = d1;
     data[1] = d2;
     data[2] = d3;
   }
+
 public:
   Kind getKind() const { return K; }
   const CXCursor &getParent() const { return parent; }
 };
-  
+
 typedef SmallVector<VisitorJob, 10> VisitorWorkList;
 
 // Cursor visitor.
 class CursorVisitor : public DeclVisitor<CursorVisitor, bool>,
-                      public TypeLocVisitor<CursorVisitor, bool>
-{
+                      public TypeLocVisitor<CursorVisitor, bool> {
 public:
   /// Callback called after child nodes of a cursor have been visited.
   /// Return true to break visitation or false to continue.
@@ -79,14 +88,14 @@ class CursorVisitor : public DeclVisitor<CursorVisitor, bool>,
   /// The opaque client data, to be passed along to the visitor.
   CXClientData ClientData;
 
-  /// Whether we should visit the preprocessing record entries last, 
+  /// Whether we should visit the preprocessing record entries last,
   /// after visiting other declarations.
   bool VisitPreprocessorLast;
 
   /// Whether we should visit declarations or preprocessing record
   /// entries that are #included inside the \arg RegionOfInterest.
   bool VisitIncludedEntities;
-  
+
   /// When valid, a source range to which the cursor should restrict
   /// its search.
   SourceRange RegionOfInterest;
@@ -103,8 +112,8 @@ class CursorVisitor : public DeclVisitor<CursorVisitor, bool>,
   SmallVectorImpl<Decl *>::iterator FileDE_current;
 
   // Cache of pre-allocated worklists for data-recursion walk of Stmts.
-  SmallVector<VisitorWorkList*, 5> WorkListFreeList;
-  SmallVector<VisitorWorkList*, 5> WorkListCache;
+  SmallVector<VisitorWorkList *, 5> WorkListFreeList;
+  SmallVector<VisitorWorkList *, 5> WorkListCache;
 
   using DeclVisitor<CursorVisitor, bool>::Visit;
   using TypeLocVisitor<CursorVisitor, bool>::Visit;
@@ -123,10 +132,8 @@ class CursorVisitor : public DeclVisitor<CursorVisitor, bool>,
     CXCursor OldParent;
 
   public:
-    SetParentRAII(CXCursor &Parent, const Decl *&StmtParent,
-                  CXCursor NewParent)
-      : Parent(Parent), StmtParent(StmtParent), OldParent(Parent)
-    {
+    SetParentRAII(CXCursor &Parent, const Decl *&StmtParent, CXCursor NewParent)
+        : Parent(Parent), StmtParent(StmtParent), OldParent(Parent) {
       Parent = NewParent;
       if (clang_isDeclaration(Parent.kind))
         StmtParent = getCursorDecl(Parent);
@@ -141,21 +148,17 @@ class CursorVisitor : public DeclVisitor<CursorVisitor, bool>,
 
 public:
   CursorVisitor(CXTranslationUnit TU, CXCursorVisitor Visitor,
-                CXClientData ClientData,
-                bool VisitPreprocessorLast,
+                CXClientData ClientData, bool VisitPreprocessorLast,
                 bool VisitIncludedPreprocessingEntries = false,
                 SourceRange RegionOfInterest = SourceRange(),
                 bool VisitDeclsOnly = false,
                 PostChildrenVisitorTy PostChildrenVisitor = nullptr)
-    : TU(TU), AU(cxtu::getASTUnit(TU)),
-      Visitor(Visitor), PostChildrenVisitor(PostChildrenVisitor),
-      ClientData(ClientData),
-      VisitPreprocessorLast(VisitPreprocessorLast),
-      VisitIncludedEntities(VisitIncludedPreprocessingEntries),
-      RegionOfInterest(RegionOfInterest),
-      VisitDeclsOnly(VisitDeclsOnly),
-      DI_current(nullptr), FileDI_current(nullptr)
-  {
+      : TU(TU), AU(cxtu::getASTUnit(TU)), Visitor(Visitor),
+        PostChildrenVisitor(PostChildrenVisitor), ClientData(ClientData),
+        VisitPreprocessorLast(VisitPreprocessorLast),
+        VisitIncludedEntities(VisitIncludedPreprocessingEntries),
+        RegionOfInterest(RegionOfInterest), VisitDeclsOnly(VisitDeclsOnly),
+        DI_current(nullptr), FileDI_current(nullptr) {
     Parent.kind = CXCursor_NoDeclFound;
     Parent.data[0] = nullptr;
     Parent.data[1] = nullptr;
@@ -165,8 +168,9 @@ class CursorVisitor : public DeclVisitor<CursorVisitor, bool>,
 
   ~CursorVisitor() {
     // Free the pre-allocated worklists for data-recursion.
-    for (SmallVectorImpl<VisitorWorkList*>::iterator
-          I = WorkListCache.begin(), E = WorkListCache.end(); I != E; ++I) {
+    for (SmallVectorImpl<VisitorWorkList *>::iterator I = WorkListCache.begin(),
+                                                      E = WorkListCache.end();
+         I != E; ++I) {
       delete *I;
     }
   }
@@ -179,14 +183,12 @@ class CursorVisitor : public DeclVisitor<CursorVisitor, bool>,
   /// Visit declarations and preprocessed entities for the file region
   /// designated by \see RegionOfInterest.
   bool visitFileRegion();
-  
+
   bool visitPreprocessedEntitiesInRegion();
 
-  bool shouldVisitIncludedEntities() const {
-    return VisitIncludedEntities;
-  }
+  bool shouldVisitIncludedEntities() const { return VisitIncludedEntities; }
 
-  template<typename InputIterator>
+  template <typename InputIterator>
   bool visitPreprocessedEntities(InputIterator First, InputIterator Last,
                                  PreprocessingRecord &PPRec,
                                  FileID FID = FileID());
@@ -206,7 +208,7 @@ class CursorVisitor : public DeclVisitor<CursorVisitor, bool>,
   bool VisitTagDecl(TagDecl *D);
   bool VisitClassTemplateSpecializationDecl(ClassTemplateSpecializationDecl *D);
   bool VisitClassTemplatePartialSpecializationDecl(
-                                     ClassTemplatePartialSpecializationDecl *D);
+      ClassTemplatePartialSpecializationDecl *D);
   bool VisitTemplateTypeParmDecl(TemplateTypeParmDecl *D);
   bool VisitEnumConstantDecl(EnumConstantDecl *D);
   bool VisitDeclaratorDecl(DeclaratorDecl *DD);
@@ -244,16 +246,15 @@ class CursorVisitor : public DeclVisitor<CursorVisitor, bool>,
   bool VisitDeclarationNameInfo(DeclarationNameInfo Name);
   bool VisitNestedNameSpecifier(NestedNameSpecifier *NNS, SourceRange Range);
   bool VisitNestedNameSpecifierLoc(NestedNameSpecifierLoc NNS);
-  
+
   // Template visitors
   bool VisitTemplateParameters(const TemplateParameterList *Params);
   bool VisitTemplateName(TemplateName Name, SourceLocation Loc);
   bool VisitTemplateArgumentLoc(const TemplateArgumentLoc &TAL);
-  
+
   // Type visitors
 #define ABSTRACT_TYPELOC(CLASS, PARENT)
-#define TYPELOC(CLASS, PARENT) \
-  bool Visit##CLASS##TypeLoc(CLASS##TypeLoc TyLoc);
+#define TYPELOC(CLASS, PARENT) bool Visit##CLASS##TypeLoc(CLASS##TypeLoc TyLoc);
 #include "clang/AST/TypeLocNodes.def"
 
   bool VisitTagTypeLoc(TagTypeLoc TL);
@@ -270,8 +271,7 @@ class CursorVisitor : public DeclVisitor<CursorVisitor, bool>,
   Optional<bool> handleDeclForVisitation(const Decl *D);
 };
 
-}
-}
+} // namespace cxcursor
+} // namespace clang
 
 #endif
-
diff --git a/clang/unittests/AST/DeclPrinterTest.cpp b/clang/unittests/AST/DeclPrinterTest.cpp
index e6732cc3f3152..018e99237ae94 100644
--- a/clang/unittests/AST/DeclPrinterTest.cpp
+++ b/clang/unittests/AST/DeclPrinterTest.cpp
@@ -1158,8 +1158,8 @@ TEST(DeclPrinter, TestTemplateArgumentList4) {
     "template<typename T> struct X {};"
     "Z<X<int>> A;",
     "A",
-    "Z<X<int> > A"));
-    // Should be: with semicolon, without extra space in "> >"
+    "Z<X<int>> A"));
+    // Should be: with semicolon
 }
 
 TEST(DeclPrinter, TestTemplateArgumentList5) {
diff --git a/clang/unittests/Analysis/CloneDetectionTest.cpp b/clang/unittests/Analysis/CloneDetectionTest.cpp
index 03b63c400434b..f8f3602f5a2a7 100644
--- a/clang/unittests/Analysis/CloneDetectionTest.cpp
+++ b/clang/unittests/Analysis/CloneDetectionTest.cpp
@@ -42,7 +42,7 @@ class NoBarFunctionConstraint {
           for (const StmtSequence &Arg : {A, B}) {
             if (const auto *D =
                     dyn_cast<const FunctionDecl>(Arg.getContainingDecl())) {
-              if (D->getNameAsString().find("bar") == 0)
+              if (D->getName().startswith("bar"))
                 return false;
             }
           }
diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp
index a6f35ba270696..fe16eeaba53cd 100644
--- a/clang/unittests/Format/FormatTest.cpp
+++ b/clang/unittests/Format/FormatTest.cpp
@@ -15386,6 +15386,13 @@ TEST_F(FormatTest, OperatorSpacing) {
   verifyFormat("Foo::operator&&();", Style);
   verifyFormat("operator&&(int(&&)(), class Foo);", Style);
 
+  // PR45107
+  verifyFormat("operator Vector<String>&();", Style);
+  verifyFormat("operator foo::Bar*();", Style);
+  verifyFormat("operator const Foo<X>::Bar<Y>*();", Style);
+  verifyFormat("operator/*a*/ const /*b*/ Foo /*c*/<X> /*d*/ ::Bar<Y>*();",
+               Style);
+
   Style.PointerAlignment = FormatStyle::PAS_Middle;
   verifyFormat("Foo::operator*();", Style);
   verifyFormat("Foo::operator void *();", Style);
diff --git a/clang/unittests/Format/FormatTestCSharp.cpp b/clang/unittests/Format/FormatTestCSharp.cpp
index 17b8e070c36a5..b0e4e76cefe70 100644
--- a/clang/unittests/Format/FormatTestCSharp.cpp
+++ b/clang/unittests/Format/FormatTestCSharp.cpp
@@ -564,7 +564,7 @@ var myDict = new Dictionary<string, string> {
 
 TEST_F(FormatTestCSharp, CSharpArrayInitializers) {
   FormatStyle Style = getGoogleStyle(FormatStyle::LK_CSharp);
-  
+
   verifyFormat(R"(//
 private MySet<Node>[] setPoints = {
   new Point<Node>(),
@@ -709,6 +709,23 @@ class ItemFactory<T>
               IAnotherInterface<T>,
               IAnotherInterfaceStill<T> {})",
                Style);
+
+  Style.ColumnLimit = 50; // Force lines to be wrapped.
+  verifyFormat(R"(//
+class ItemFactory<T, U>
+    where T : new(),
+              IAnInterface<T>,
+              IAnotherInterface<T, U>,
+              IAnotherInterfaceStill<T, U> {})",
+               Style);
+
+  // In other languages `where` can be used as a normal identifier.
+  // This example is in C++!
+  verifyFormat(R"(//
+class A {
+  int f(int where) {}
+};)",
+               getGoogleStyle(FormatStyle::LK_Cpp));
 }
 
 } // namespace format
diff --git a/clang/unittests/Format/FormatTestJS.cpp b/clang/unittests/Format/FormatTestJS.cpp
index 6efb8662f0f5b..3fd795c526b16 100644
--- a/clang/unittests/Format/FormatTestJS.cpp
+++ b/clang/unittests/Format/FormatTestJS.cpp
@@ -386,13 +386,6 @@ TEST_F(FormatTestJS, ReservedWordsParenthesized) {
                "return (x);\n");
 }
 
-TEST_F(FormatTestJS, CppKeywords) {
-  // Make sure we don't mess stuff up because of C++ keywords.
-  verifyFormat("return operator && (aa);");
-  // .. or QT ones.
-  verifyFormat("slots: Slot[];");
-}
-
 TEST_F(FormatTestJS, ES6DestructuringAssignment) {
   verifyFormat("var [a, b, c] = [1, 2, 3];");
   verifyFormat("const [a, b, c] = [1, 2, 3];");
@@ -2366,6 +2359,61 @@ TEST_F(FormatTestJS, NonNullAssertionOperator) {
   verifyFormat("return !!x;\n");
 }
 
+TEST_F(FormatTestJS, CppKeywords) {
+  // Make sure we don't mess stuff up because of C++ keywords.
+  verifyFormat("return operator && (aa);");
+  // .. or QT ones.
+  verifyFormat("const slots: Slot[];");
+  // use the "!" assertion operator to validate that clang-format understands
+  // these C++ keywords aren't keywords in JS/TS.
+  verifyFormat("auto!;");
+  verifyFormat("char!;");
+  verifyFormat("concept!;");
+  verifyFormat("double!;");
+  verifyFormat("extern!;");
+  verifyFormat("float!;");
+  verifyFormat("inline!;");
+  verifyFormat("int!;");
+  verifyFormat("long!;");
+  verifyFormat("register!;");
+  verifyFormat("restrict!;");
+  verifyFormat("sizeof!;");
+  verifyFormat("struct!;");
+  verifyFormat("typedef!;");
+  verifyFormat("union!;");
+  verifyFormat("unsigned!;");
+  verifyFormat("volatile!;");
+  verifyFormat("_Alignas!;");
+  verifyFormat("_Alignof!;");
+  verifyFormat("_Atomic!;");
+  verifyFormat("_Bool!;");
+  verifyFormat("_Complex!;");
+  verifyFormat("_Generic!;");
+  verifyFormat("_Imaginary!;");
+  verifyFormat("_Noreturn!;");
+  verifyFormat("_Static_assert!;");
+  verifyFormat("_Thread_local!;");
+  verifyFormat("__func__!;");
+  verifyFormat("__objc_yes!;");
+  verifyFormat("__objc_no!;");
+  verifyFormat("asm!;");
+  verifyFormat("bool!;");
+  verifyFormat("const_cast!;");
+  verifyFormat("dynamic_cast!;");
+  verifyFormat("explicit!;");
+  verifyFormat("friend!;");
+  verifyFormat("mutable!;");
+  verifyFormat("operator!;");
+  verifyFormat("reinterpret_cast!;");
+  verifyFormat("static_cast!;");
+  verifyFormat("template!;");
+  verifyFormat("typename!;");
+  verifyFormat("typeid!;");
+  verifyFormat("using!;");
+  verifyFormat("virtual!;");
+  verifyFormat("wchar_t!;");
+}
+
 TEST_F(FormatTestJS, NullPropagatingOperator) {
   verifyFormat("let x = foo?.bar?.baz();\n");
   verifyFormat("let x = foo?.(foo);\n");
diff --git a/clang/unittests/Sema/CodeCompleteTest.cpp b/clang/unittests/Sema/CodeCompleteTest.cpp
index 5c8940cf6ebae..d8b303d77bb96 100644
--- a/clang/unittests/Sema/CodeCompleteTest.cpp
+++ b/clang/unittests/Sema/CodeCompleteTest.cpp
@@ -486,8 +486,10 @@ TEST(PreferredTypeTest, NoCrashOnInvalidTypes) {
   StringRef Code = R"cpp(
     auto x = decltype(&1)(^);
     auto y = new decltype(&1)(^);
+    // GNU decimal type extension is not supported in clang.
+    auto z = new _Decimal128(^);
   )cpp";
-  EXPECT_THAT(collectPreferredTypes(Code),
-              Each("decltype(<recovery-expr>(1))"));
+  EXPECT_THAT(collectPreferredTypes(Code), Each("NULL TYPE"));
 }
+
 } // namespace
diff --git a/clang/unittests/StaticAnalyzer/RegisterCustomCheckersTest.cpp b/clang/unittests/StaticAnalyzer/RegisterCustomCheckersTest.cpp
index 8027d3338b699..51d52c45c1c86 100644
--- a/clang/unittests/StaticAnalyzer/RegisterCustomCheckersTest.cpp
+++ b/clang/unittests/StaticAnalyzer/RegisterCustomCheckersTest.cpp
@@ -99,7 +99,7 @@ void registerPrerequisiteChecker(CheckerManager &mgr) {
   mgr.registerChecker<PrerequisiteChecker>();
 }
 
-bool shouldRegisterPrerequisiteChecker(const LangOptions &LO) {
+bool shouldRegisterPrerequisiteChecker(const CheckerManager &mgr) {
   return false;
 }
 
@@ -117,7 +117,7 @@ void registerDependentChecker(CheckerManager &mgr) {
   mgr.registerChecker<DependentChecker>();
 }
 
-bool shouldRegisterDependentChecker(const LangOptions &LO) {
+bool shouldRegisterDependentChecker(const CheckerManager &mgr) {
   return true;
 }
 
diff --git a/clang/unittests/StaticAnalyzer/Reusables.h b/clang/unittests/StaticAnalyzer/Reusables.h
index 1346adbac90a8..3f2fd61642008 100644
--- a/clang/unittests/StaticAnalyzer/Reusables.h
+++ b/clang/unittests/StaticAnalyzer/Reusables.h
@@ -56,8 +56,9 @@ class ExprEngineConsumer : public ASTConsumer {
 
 public:
   ExprEngineConsumer(CompilerInstance &C)
-      : C(C), ChkMgr(C.getASTContext(), *C.getAnalyzerOpts()), CTU(C),
-        Consumers(),
+      : C(C),
+        ChkMgr(C.getASTContext(), *C.getAnalyzerOpts(), C.getPreprocessor()),
+        CTU(C), Consumers(),
         AMgr(C.getASTContext(), C.getPreprocessor(), Consumers,
              CreateRegionStoreManager, CreateRangeConstraintManager, &ChkMgr,
              *C.getAnalyzerOpts()),
diff --git a/clang/unittests/Tooling/Syntax/TokensTest.cpp b/clang/unittests/Tooling/Syntax/TokensTest.cpp
index d4b015393286b..256096d6a83ee 100644
--- a/clang/unittests/Tooling/Syntax/TokensTest.cpp
+++ b/clang/unittests/Tooling/Syntax/TokensTest.cpp
@@ -627,6 +627,7 @@ TEST_F(TokenBufferTest, SpelledByExpanded) {
 
     A split B
   )cpp");
+  // Ranges going across expansion boundaries.
   EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1 a2 a3 split b1 b2")),
               ValueIs(SameRange(findSpelled("A split B"))));
   EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1 a2 a3")),
@@ -647,22 +648,28 @@ TEST_F(TokenBufferTest, SpelledByExpanded) {
     ID(ID(ID(a1) a2 a3)) split ID(B)
   )cpp");
 
-  EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1 a2 a3")),
-              ValueIs(SameRange(findSpelled("ID ( ID ( ID ( a1 ) a2 a3 ) )"))));
   EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("b1 b2")),
-              ValueIs(SameRange(findSpelled("ID ( B )"))));
+              ValueIs(SameRange(findSpelled("( B").drop_front())));
   EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1 a2 a3 split b1 b2")),
               ValueIs(SameRange(findSpelled(
                   "ID ( ID ( ID ( a1 ) a2 a3 ) ) split ID ( B )"))));
-  // Ranges crossing macro call boundaries.
-  EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("a1 a2 a3 split b1")),
-            llvm::None);
-  EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("a2 a3 split b1")),
-            llvm::None);
-  // FIXME: next two examples should map to macro arguments, but currently they
-  //        fail.
-  EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("a2")), llvm::None);
-  EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("a1 a2")), llvm::None);
+  // Mixed ranges with expanded and spelled tokens.
+  EXPECT_THAT(
+      Buffer.spelledForExpanded(findExpanded("a1 a2 a3 split")),
+      ValueIs(SameRange(findSpelled("ID ( ID ( ID ( a1 ) a2 a3 ) ) split"))));
+  EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("split b1 b2")),
+              ValueIs(SameRange(findSpelled("split ID ( B )"))));
+  // Macro arguments
+  EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1")),
+              ValueIs(SameRange(findSpelled("a1"))));
+  EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a2")),
+              ValueIs(SameRange(findSpelled("a2"))));
+  EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a3")),
+              ValueIs(SameRange(findSpelled("a3"))));
+  EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1 a2")),
+              ValueIs(SameRange(findSpelled("ID ( a1 ) a2"))));
+  EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1 a2 a3")),
+              ValueIs(SameRange(findSpelled("ID ( a1 ) a2 a3"))));
 
   // Empty macro expansions.
   recordTokens(R"cpp(
@@ -674,11 +681,11 @@ TEST_F(TokenBufferTest, SpelledByExpanded) {
     ID(7 8 9) EMPTY EMPTY
   )cpp");
   EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("1 2 3")),
-              ValueIs(SameRange(findSpelled("ID ( 1 2 3 )"))));
+              ValueIs(SameRange(findSpelled("1 2 3"))));
   EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("4 5 6")),
-              ValueIs(SameRange(findSpelled("ID ( 4 5 6 )"))));
+              ValueIs(SameRange(findSpelled("4 5 6"))));
   EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("7 8 9")),
-              ValueIs(SameRange(findSpelled("ID ( 7 8 9 )"))));
+              ValueIs(SameRange(findSpelled("7 8 9"))));
 
   // Empty mappings coming from various directives.
   recordTokens(R"cpp(
@@ -689,6 +696,27 @@ TEST_F(TokenBufferTest, SpelledByExpanded) {
   )cpp");
   EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("not_mapped")),
               ValueIs(SameRange(findSpelled("not_mapped"))));
+
+  // Multiple macro arguments
+  recordTokens(R"cpp(
+    #define ID(X) X
+    #define ID2(X, Y) X Y
+
+    ID2(ID(a1), ID(a2) a3) ID2(a4, a5 a6 a7)
+  )cpp");
+  // Should fail, spans multiple arguments.
+  EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("a1 a2")), llvm::None);
+  EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a2 a3")),
+              ValueIs(SameRange(findSpelled("ID ( a2 ) a3"))));
+  EXPECT_THAT(
+      Buffer.spelledForExpanded(findExpanded("a1 a2 a3")),
+      ValueIs(SameRange(findSpelled("ID2 ( ID ( a1 ) , ID ( a2 ) a3 )"))));
+  EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a5 a6")),
+              ValueIs(SameRange(findSpelled("a5 a6"))));
+  EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a4 a5 a6 a7")),
+              ValueIs(SameRange(findSpelled("ID2 ( a4 , a5 a6 a7 )"))));
+  // Should fail, spans multiple invocations.
+  EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("a1 a2 a3 a4")), llvm::None);
 }
 
 TEST_F(TokenBufferTest, ExpandedTokensForRange) {
diff --git a/clang/unittests/Tooling/Syntax/TreeTest.cpp b/clang/unittests/Tooling/Syntax/TreeTest.cpp
index ddd9092d8a76d..823045748f992 100644
--- a/clang/unittests/Tooling/Syntax/TreeTest.cpp
+++ b/clang/unittests/Tooling/Syntax/TreeTest.cpp
@@ -224,6 +224,59 @@ void foo() {}
 )txt");
 }
 
+TEST_F(SyntaxTreeTest, SimpleVariable) {
+  expectTreeDumpEqual(
+      R"cpp(
+int a;
+int b = 42;
+    )cpp",
+      R"txt(
+*: TranslationUnit
+|-SimpleDeclaration
+| |-int
+| |-SimpleDeclarator
+| | `-a
+| `-;
+`-SimpleDeclaration
+  |-int
+  |-SimpleDeclarator
+  | |-b
+  | |-=
+  | `-UnknownExpression
+  |   `-42
+  `-;
+)txt");
+}
+
+TEST_F(SyntaxTreeTest, SimpleFunction) {
+  expectTreeDumpEqual(
+      R"cpp(
+void foo(int a, int b) {}
+    )cpp",
+      R"txt(
+*: TranslationUnit
+`-SimpleDeclaration
+  |-void
+  |-SimpleDeclarator
+  | |-foo
+  | `-ParametersAndQualifiers
+  |   |-(
+  |   |-SimpleDeclaration
+  |   | |-int
+  |   | `-SimpleDeclarator
+  |   |   `-a
+  |   |-,
+  |   |-SimpleDeclaration
+  |   | |-int
+  |   | `-SimpleDeclarator
+  |   |   `-b
+  |   `-)
+  `-CompoundStatement
+    |-{
+    `-}
+)txt");
+}
+
 TEST_F(SyntaxTreeTest, If) {
   expectTreeDumpEqual(
       R"cpp(
@@ -541,20 +594,32 @@ void test() {
 TEST_F(SyntaxTreeTest, MultipleDeclaratorsGrouping) {
   expectTreeDumpEqual(
       R"cpp(
-      int *a, b;
+      int *a, b; int *c, d;
   )cpp",
       R"txt(
 *: TranslationUnit
+|-SimpleDeclaration
+| |-int
+| |-SimpleDeclarator
+| | |-*
+| | `-a
+| |-,
+| |-SimpleDeclarator
+| | `-b
+| `-;
 `-SimpleDeclaration
   |-int
   |-SimpleDeclarator
   | |-*
-  | `-a
+  | `-c
   |-,
   |-SimpleDeclarator
-  | `-b
+  | `-d
   `-;
   )txt");
+}
+
+TEST_F(SyntaxTreeTest, MultipleDeclaratorsGroupingTypedef) {
   expectTreeDumpEqual(
       R"cpp(
     typedef int *a, b;
diff --git a/clang/utils/TableGen/ClangAttrEmitter.cpp b/clang/utils/TableGen/ClangAttrEmitter.cpp
index da29a1c744284..55480e13aaa11 100644
--- a/clang/utils/TableGen/ClangAttrEmitter.cpp
+++ b/clang/utils/TableGen/ClangAttrEmitter.cpp
@@ -3729,7 +3729,26 @@ void EmitClangAttrParsedAttrImpl(RecordKeeper &Records, raw_ostream &OS) {
     // ParsedAttr.cpp.
     const std::string &AttrName = I->first;
     const Record &Attr = *I->second;
-    OS << "struct ParsedAttrInfo" << I->first << " : public ParsedAttrInfo {\n";
+    auto Spellings = GetFlattenedSpellings(Attr);
+    if (!Spellings.empty()) {
+      OS << "static constexpr ParsedAttrInfo::Spelling " << I->first
+         << "Spellings[] = {\n";
+      for (const auto &S : Spellings) {
+        const std::string &RawSpelling = S.name();
+        std::string Spelling;
+        if (!S.nameSpace().empty())
+          Spelling += S.nameSpace() + "::";
+        if (S.variety() == "GNU")
+          Spelling += NormalizeGNUAttrSpelling(RawSpelling);
+        else
+          Spelling += RawSpelling;
+        OS << "  {AttributeCommonInfo::AS_" << S.variety();
+        OS << ", \"" << Spelling << "\"},\n";
+      }
+      OS << "};\n";
+    }
+    OS << "struct ParsedAttrInfo" << I->first
+       << " final : public ParsedAttrInfo {\n";
     OS << "  ParsedAttrInfo" << I->first << "() {\n";
     OS << "    AttrKind = ParsedAttr::AT_" << AttrName << ";\n";
     emitArgInfo(Attr, OS);
@@ -3746,20 +3765,8 @@ void EmitClangAttrParsedAttrImpl(RecordKeeper &Records, raw_ostream &OS) {
     OS << IsKnownToGCC(Attr) << ";\n";
     OS << "    IsSupportedByPragmaAttribute = ";
     OS << PragmaAttributeSupport.isAttributedSupported(*I->second) << ";\n";
-    for (const auto &S : GetFlattenedSpellings(Attr)) {
-      const std::string &RawSpelling = S.name();
-      std::string Spelling;
-      if (S.variety() == "CXX11" || S.variety() == "C2x") {
-        Spelling += S.nameSpace();
-        Spelling += "::";
-      }
-      if (S.variety() == "GNU")
-        Spelling += NormalizeGNUAttrSpelling(RawSpelling);
-      else
-        Spelling += RawSpelling;
-      OS << "    Spellings.push_back({AttributeCommonInfo::AS_" << S.variety();
-      OS << ",\"" << Spelling << "\"});\n";
-    }
+    if (!Spellings.empty())
+      OS << "    Spellings = " << I->first << "Spellings;\n";
     OS << "  }\n";
     GenerateAppertainsTo(Attr, OS);
     GenerateLangOptRequirements(Attr, OS);
@@ -3825,12 +3832,12 @@ void EmitClangAttrParsedAttrKinds(RecordKeeper &Records, raw_ostream &OS) {
         const std::string &Variety = S.variety();
         if (Variety == "CXX11") {
           Matches = &CXX11;
-          Spelling += S.nameSpace();
-          Spelling += "::";
+          if (!S.nameSpace().empty())
+            Spelling += S.nameSpace() + "::";
         } else if (Variety == "C2x") {
           Matches = &C2x;
-          Spelling += S.nameSpace();
-          Spelling += "::";
+          if (!S.nameSpace().empty())
+            Spelling += S.nameSpace() + "::";
         } else if (Variety == "GNU")
           Matches = &GNU;
         else if (Variety == "Declspec")
diff --git a/clang/utils/TableGen/ClangProgModelBuiltinEmitter.cpp b/clang/utils/TableGen/ClangProgModelBuiltinEmitter.cpp
index d7e4304327e49..57d84eb47de71 100644
--- a/clang/utils/TableGen/ClangProgModelBuiltinEmitter.cpp
+++ b/clang/utils/TableGen/ClangProgModelBuiltinEmitter.cpp
@@ -588,7 +588,7 @@ void BuiltinNameEmitter::GroupBySignature() {
       CurSignatureList->push_back(Signature.second);
     }
     // Sort the list to facilitate future comparisons.
-    std::sort(CurSignatureList->begin(), CurSignatureList->end());
+    llvm::sort(*CurSignatureList);
 
     // Check if we have already seen another function with the same list of
     // signatures.  If so, just add the name of the function.
diff --git a/clang/utils/TableGen/MveEmitter.cpp b/clang/utils/TableGen/MveEmitter.cpp
index f53c6036766ab..36aa7534aafab 100644
--- a/clang/utils/TableGen/MveEmitter.cpp
+++ b/clang/utils/TableGen/MveEmitter.cpp
@@ -739,7 +739,7 @@ class IRIntrinsicResult : public Result {
         "Intrinsic::ID", "Intrinsic::" + IntrinsicID);
     OS << "Builder.CreateCall(CGM.getIntrinsic(" << IntNo;
     if (!ParamTypes.empty()) {
-      OS << ", llvm::SmallVector<llvm::Type *, " << ParamTypes.size() << "> {";
+      OS << ", {";
       const char *Sep = "";
       for (auto T : ParamTypes) {
         OS << Sep << ParamAlloc.allocParam("llvm::Type *", T->llvmName());
@@ -747,7 +747,7 @@ class IRIntrinsicResult : public Result {
       }
       OS << "}";
     }
-    OS << "), llvm::SmallVector<Value *, " << Args.size() << "> {";
+    OS << "), {";
     const char *Sep = "";
     for (auto Arg : Args) {
       OS << Sep << Arg->asValue();
diff --git a/clang/utils/TableGen/SveEmitter.cpp b/clang/utils/TableGen/SveEmitter.cpp
index e02e94dd98ae6..a792530a03af0 100644
--- a/clang/utils/TableGen/SveEmitter.cpp
+++ b/clang/utils/TableGen/SveEmitter.cpp
@@ -604,7 +604,7 @@ void SVEEmitter::createIntrinsic(
   }
 
   // Remove duplicate type specs.
-  std::sort(TypeSpecs.begin(), TypeSpecs.end());
+  llvm::sort(TypeSpecs);
   TypeSpecs.erase(std::unique(TypeSpecs.begin(), TypeSpecs.end()),
                   TypeSpecs.end());
 
diff --git a/compiler-rt/CMakeLists.txt b/compiler-rt/CMakeLists.txt
index bf4d2801b4920..0ada717d7c77c 100644
--- a/compiler-rt/CMakeLists.txt
+++ b/compiler-rt/CMakeLists.txt
@@ -413,7 +413,6 @@ append_list_if(COMPILER_RT_HAS_NODEFAULTLIBS_FLAG -nodefaultlibs SANITIZER_COMMO
 append_list_if(COMPILER_RT_HAS_Z_TEXT -Wl,-z,text SANITIZER_COMMON_LINK_FLAGS)
 
 if (COMPILER_RT_USE_BUILTINS_LIBRARY)
-  list(APPEND SANITIZER_COMMON_LINK_LIBS ${COMPILER_RT_BUILTINS_LIBRARY})
   string(REPLACE "-Wl,-z,defs" "" CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS}")
 else()
   if (ANDROID)
diff --git a/compiler-rt/cmake/Modules/AddCompilerRT.cmake b/compiler-rt/cmake/Modules/AddCompilerRT.cmake
index f0c8893c42bc9..83a972d443b2b 100644
--- a/compiler-rt/cmake/Modules/AddCompilerRT.cmake
+++ b/compiler-rt/cmake/Modules/AddCompilerRT.cmake
@@ -1,5 +1,6 @@
 include(ExternalProject)
 include(CompilerRTUtils)
+include(HandleCompilerRT)
 
 function(set_target_output_directories target output_dir)
   # For RUNTIME_OUTPUT_DIRECTORY variable, Multi-configuration generators
@@ -233,6 +234,10 @@ function(add_compiler_rt_runtime name type)
           set_output_name(output_name_${libname} ${name} ${arch})
         endif()
       endif()
+      if(COMPILER_RT_USE_BUILTINS_LIBRARY AND NOT type STREQUAL "OBJECT")
+        get_compiler_rt_target(${arch} target)
+        find_compiler_rt_library(builtins ${target} builtins_${libname})
+      endif()
       set(sources_${libname} ${LIB_SOURCES})
       format_object_libs(sources_${libname} ${arch} ${LIB_OBJECT_LIBS})
       set(libnames ${libnames} ${libname})
@@ -326,6 +331,9 @@ function(add_compiler_rt_runtime name type)
     if(LIB_LINK_LIBS)
       target_link_libraries(${libname} PRIVATE ${LIB_LINK_LIBS})
     endif()
+    if(builtins_${libname})
+      target_link_libraries(${libname} PRIVATE ${builtins_${libname}})
+    endif()
     if(${type} STREQUAL "SHARED")
       if(COMMAND llvm_setup_rpath)
         llvm_setup_rpath(${libname})
diff --git a/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake b/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake
index 9f501a61c4b69..657d2c97eac8e 100644
--- a/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake
+++ b/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake
@@ -44,8 +44,11 @@ function(find_darwin_sdk_dir var sdk_name)
 endfunction()
 
 function(find_darwin_sdk_version var sdk_name)
-  # We deliberately don't cache the result here because
-  # CMake's caching causes too many problems.
+  if (DARWIN_${sdk_name}_OVERRIDE_SDK_VERSION)
+    message(WARNING "Overriding ${sdk_name} SDK version to ${DARWIN_${sdk_name}_OVERRIDE_SDK_VERSION}")
+    set(${var} "${DARWIN_${sdk_name}_OVERRIDE_SDK_VERSION}" PARENT_SCOPE)
+    return()
+  endif()
   set(result_process 1)
   if(NOT DARWIN_PREFER_PUBLIC_SDK)
     # Let's first try the internal SDK, otherwise use the public SDK.
diff --git a/compiler-rt/cmake/Modules/HandleCompilerRT.cmake b/compiler-rt/cmake/Modules/HandleCompilerRT.cmake
index 61b7792789e7c..ac9e0871489da 100644
--- a/compiler-rt/cmake/Modules/HandleCompilerRT.cmake
+++ b/compiler-rt/cmake/Modules/HandleCompilerRT.cmake
@@ -1,24 +1,65 @@
-function(find_compiler_rt_library name variable)
-  set(CLANG_COMMAND ${CMAKE_CXX_COMPILER} ${SANITIZER_COMMON_CFLAGS}
-      "--rtlib=compiler-rt" "--print-libgcc-file-name")
-  if (CMAKE_CXX_COMPILER_ID MATCHES Clang AND CMAKE_CXX_COMPILER_TARGET)
-    list(APPEND CLANG_COMMAND "--target=${CMAKE_CXX_COMPILER_TARGET}")
+# Check if compile-rt library file path exists.
+# If found, cache the path in:
+#    COMPILER_RT_LIBRARY-<name>-<target>
+# If err_flag is true OR path not found, emit a message and set:
+#    COMPILER_RT_LIBRARY-<name>-<target> to NOTFOUND
+function(cache_compiler_rt_library err_flag name target library_file)
+  if(err_flag OR NOT EXISTS "${library_file}")
+    message(STATUS "Failed to find compiler-rt ${name} library for ${target}")
+    set(COMPILER_RT_LIBRARY-${name}-${target} "NOTFOUND" CACHE INTERNAL
+        "compiler-rt ${name} library for ${target}")
+  else()
+    message(STATUS "Found compiler-rt ${name} library: ${library_file}")
+    set(COMPILER_RT_LIBRARY-${name}-${target} "${library_file}" CACHE INTERNAL
+        "compiler-rt ${name} library for ${target}")
+  endif()
+endfunction()
+
+# Find the path to compiler-rt library `name` (e.g. "builtins") for
+# the specified `target` (e.g. "x86_64-linux") and return it in `variable`.
+# This calls cache_compiler_rt_library that caches the path to speed up
+# repeated invocations with the same `name` and `target`.
+function(find_compiler_rt_library name target variable)
+  if(NOT CMAKE_CXX_COMPILER_ID MATCHES Clang)
+    set(${variable} "NOTFOUND" PARENT_SCOPE)
+    return()
+  endif()
+  if (NOT target AND CMAKE_CXX_COMPILER_TARGET)
+    set(target "${CMAKE_CXX_COMPILER_TARGET}")
   endif()
-  get_property(SANITIZER_CXX_FLAGS CACHE CMAKE_CXX_FLAGS PROPERTY VALUE)
-  string(REPLACE " " ";" SANITIZER_CXX_FLAGS "${SANITIZER_CXX_FLAGS}")
-  list(APPEND CLANG_COMMAND ${SANITIZER_CXX_FLAGS})
-  execute_process(
+  if(NOT DEFINED COMPILER_RT_LIBRARY-builtins-${target})
+    # If the cache variable is not defined, invoke clang and then 
+    # set it with cache_compiler_rt_library.
+    set(CLANG_COMMAND ${CMAKE_CXX_COMPILER} ${SANITIZER_COMMON_FLAGS}
+        "--rtlib=compiler-rt" "-print-libgcc-file-name")
+    if(target)
+      list(APPEND CLANG_COMMAND "--target=${target}")
+    endif()
+    get_property(SANITIZER_CXX_FLAGS CACHE CMAKE_CXX_FLAGS PROPERTY VALUE)
+    string(REPLACE " " ";" SANITIZER_CXX_FLAGS "${SANITIZER_CXX_FLAGS}")
+    list(APPEND CLANG_COMMAND ${SANITIZER_CXX_FLAGS})
+    execute_process(
       COMMAND ${CLANG_COMMAND}
       RESULT_VARIABLE HAD_ERROR
       OUTPUT_VARIABLE LIBRARY_FILE
-  )
-  string(STRIP "${LIBRARY_FILE}" LIBRARY_FILE)
-  file(TO_CMAKE_PATH "${LIBRARY_FILE}" LIBRARY_FILE)
-  string(REPLACE "builtins" "${name}" LIBRARY_FILE "${LIBRARY_FILE}")
-  if (NOT HAD_ERROR AND EXISTS "${LIBRARY_FILE}")
-    message(STATUS "Found compiler-rt ${name} library: ${LIBRARY_FILE}")
-    set(${variable} "${LIBRARY_FILE}" PARENT_SCOPE)
-  else()
-    message(STATUS "Failed to find compiler-rt ${name} library")
+    )
+    string(STRIP "${LIBRARY_FILE}" LIBRARY_FILE)
+    file(TO_CMAKE_PATH "${LIBRARY_FILE}" LIBRARY_FILE)
+    cache_compiler_rt_library(${HAD_ERROR}
+      builtins "${target}" "${LIBRARY_FILE}")
+  endif()
+  if(NOT COMPILER_RT_LIBRARY-builtins-${target})
+    set(${variable} "NOTFOUND" PARENT_SCOPE)
+    return()
+  endif()
+  if(NOT DEFINED COMPILER_RT_LIBRARY-${name}-${target})
+    # clang gives only the builtins library path. Other library paths are
+    # obtained by substituting "builtins" with ${name} in the builtins
+    # path and then checking if the resultant path exists. The result of
+    # this check is also cached by cache_compiler_rt_library.
+    set(LIBRARY_FILE "${COMPILER_RT_LIBRARY-builtins-${target}}")
+    string(REPLACE "builtins" "${name}" LIBRARY_FILE "${LIBRARY_FILE}")
+    cache_compiler_rt_library(FALSE "${name}" "${target}" "${LIBRARY_FILE}")
   endif()
+  set(${variable} "${COMPILER_RT_LIBRARY-${name}-${target}}" PARENT_SCOPE)
 endfunction()
diff --git a/compiler-rt/cmake/config-ix.cmake b/compiler-rt/cmake/config-ix.cmake
index 0157011d1eadd..8261bc9fcdd52 100644
--- a/compiler-rt/cmake/config-ix.cmake
+++ b/compiler-rt/cmake/config-ix.cmake
@@ -16,7 +16,7 @@ endfunction()
 check_library_exists(c fopen "" COMPILER_RT_HAS_LIBC)
 if (COMPILER_RT_USE_BUILTINS_LIBRARY)
   include(HandleCompilerRT)
-  find_compiler_rt_library(builtins COMPILER_RT_BUILTINS_LIBRARY)
+  find_compiler_rt_library(builtins "" COMPILER_RT_BUILTINS_LIBRARY)
 else()
   if (ANDROID)
     check_library_exists(gcc __gcc_personality_v0 "" COMPILER_RT_HAS_GCC_LIB)
diff --git a/compiler-rt/include/fuzzer/FuzzedDataProvider.h b/compiler-rt/include/fuzzer/FuzzedDataProvider.h
index 21d5fac457d62..83bcd0134a7d3 100644
--- a/compiler-rt/include/fuzzer/FuzzedDataProvider.h
+++ b/compiler-rt/include/fuzzer/FuzzedDataProvider.h
@@ -129,7 +129,7 @@ std::vector<T> FuzzedDataProvider::ConsumeRemainingBytes() {
 // |.c_str()| on the resulting string is the best way to get an immutable
 // null-terminated C string. If fewer than |num_bytes| of data remain, returns
 // a shorter std::string containing all of the data that's left.
-std::string FuzzedDataProvider::ConsumeBytesAsString(size_t num_bytes) {
+inline std::string FuzzedDataProvider::ConsumeBytesAsString(size_t num_bytes) {
   static_assert(sizeof(std::string::value_type) == sizeof(uint8_t),
                 "ConsumeBytesAsString cannot convert the data to a string.");
 
@@ -144,7 +144,8 @@ std::string FuzzedDataProvider::ConsumeBytesAsString(size_t num_bytes) {
 // input data, returns what remains of the input. Designed to be more stable
 // with respect to a fuzzer inserting characters than just picking a random
 // length and then consuming that many bytes with |ConsumeBytes|.
-std::string FuzzedDataProvider::ConsumeRandomLengthString(size_t max_length) {
+inline std::string
+FuzzedDataProvider::ConsumeRandomLengthString(size_t max_length) {
   // Reads bytes from the start of |data_ptr_|. Maps "\\" to "\", and maps "\"
   // followed by anything else to the end of the string. As a result of this
   // logic, a fuzzer can insert characters into the string, and the string
@@ -172,14 +173,14 @@ std::string FuzzedDataProvider::ConsumeRandomLengthString(size_t max_length) {
 }
 
 // Returns a std::string of length from 0 to |remaining_bytes_|.
-std::string FuzzedDataProvider::ConsumeRandomLengthString() {
+inline std::string FuzzedDataProvider::ConsumeRandomLengthString() {
   return ConsumeRandomLengthString(remaining_bytes_);
 }
 
 // Returns a std::string containing all remaining bytes of the input data.
 // Prefer using |ConsumeRemainingBytes| unless you actually need a std::string
 // object.
-std::string FuzzedDataProvider::ConsumeRemainingBytesAsString() {
+inline std::string FuzzedDataProvider::ConsumeRemainingBytesAsString() {
   return ConsumeBytesAsString(remaining_bytes_);
 }
 
@@ -280,7 +281,7 @@ template <typename T> T FuzzedDataProvider::ConsumeProbability() {
 }
 
 // Reads one byte and returns a bool, or false when no data remains.
-bool FuzzedDataProvider::ConsumeBool() {
+inline bool FuzzedDataProvider::ConsumeBool() {
   return 1 & ConsumeIntegral<uint8_t>();
 }
 
@@ -315,19 +316,21 @@ T FuzzedDataProvider::PickValueInArray(std::initializer_list<const T> list) {
 // In general, it's better to avoid using this function, but it may be useful
 // in cases when it's necessary to fill a certain buffer or object with
 // fuzzing data.
-size_t FuzzedDataProvider::ConsumeData(void *destination, size_t num_bytes) {
+inline size_t FuzzedDataProvider::ConsumeData(void *destination,
+                                              size_t num_bytes) {
   num_bytes = std::min(num_bytes, remaining_bytes_);
   CopyAndAdvance(destination, num_bytes);
   return num_bytes;
 }
 
 // Private methods.
-void FuzzedDataProvider::CopyAndAdvance(void *destination, size_t num_bytes) {
+inline void FuzzedDataProvider::CopyAndAdvance(void *destination,
+                                               size_t num_bytes) {
   std::memcpy(destination, data_ptr_, num_bytes);
   Advance(num_bytes);
 }
 
-void FuzzedDataProvider::Advance(size_t num_bytes) {
+inline void FuzzedDataProvider::Advance(size_t num_bytes) {
   if (num_bytes > remaining_bytes_)
     abort();
 
diff --git a/compiler-rt/lib/asan/asan_report.cpp b/compiler-rt/lib/asan/asan_report.cpp
index 2e6ce436d0306..99e8678aa7857 100644
--- a/compiler-rt/lib/asan/asan_report.cpp
+++ b/compiler-rt/lib/asan/asan_report.cpp
@@ -160,6 +160,9 @@ class ScopedInErrorReport {
       BlockingMutexLock l(&error_message_buf_mutex);
       internal_memcpy(buffer_copy.data(),
                       error_message_buffer, kErrorMessageBufferSize);
+      // Clear error_message_buffer so that if we find other errors
+      // we don't re-log this error.
+      error_message_buffer_pos = 0;
     }
 
     LogFullErrorReport(buffer_copy.data());
diff --git a/compiler-rt/lib/gwp_asan/tests/backtrace.cpp b/compiler-rt/lib/gwp_asan/tests/backtrace.cpp
index 6c9a9309ed8b4..b3d44270bb2a1 100644
--- a/compiler-rt/lib/gwp_asan/tests/backtrace.cpp
+++ b/compiler-rt/lib/gwp_asan/tests/backtrace.cpp
@@ -11,34 +11,52 @@
 #include "gwp_asan/crash_handler.h"
 #include "gwp_asan/tests/harness.h"
 
-TEST_F(BacktraceGuardedPoolAllocator, DoubleFree) {
-  void *Ptr = GPA.allocate(1);
+// Optnone to ensure that the calls to these functions are not optimized away,
+// as we're looking for them in the backtraces.
+__attribute((optnone)) void *
+AllocateMemory(gwp_asan::GuardedPoolAllocator &GPA) {
+  return GPA.allocate(1);
+}
+__attribute((optnone)) void
+DeallocateMemory(gwp_asan::GuardedPoolAllocator &GPA, void *Ptr) {
   GPA.deallocate(Ptr);
+}
+__attribute((optnone)) void
+DeallocateMemory2(gwp_asan::GuardedPoolAllocator &GPA, void *Ptr) {
+  GPA.deallocate(Ptr);
+}
+__attribute__((optnone)) void TouchMemory(void *Ptr) {
+  *(reinterpret_cast<volatile char *>(Ptr)) = 7;
+}
+
+TEST_F(BacktraceGuardedPoolAllocator, DoubleFree) {
+  void *Ptr = AllocateMemory(GPA);
+  DeallocateMemory(GPA, Ptr);
 
   std::string DeathRegex = "Double Free.*";
-  DeathRegex.append("backtrace\\.cpp:26.*");
+  DeathRegex.append("DeallocateMemory2.*");
 
   DeathRegex.append("was deallocated.*");
-  DeathRegex.append("backtrace\\.cpp:16.*");
+  DeathRegex.append("DeallocateMemory.*");
 
   DeathRegex.append("was allocated.*");
-  DeathRegex.append("backtrace\\.cpp:15.*");
-  ASSERT_DEATH(GPA.deallocate(Ptr), DeathRegex);
+  DeathRegex.append("AllocateMemory.*");
+  ASSERT_DEATH(DeallocateMemory2(GPA, Ptr), DeathRegex);
 }
 
 TEST_F(BacktraceGuardedPoolAllocator, UseAfterFree) {
-  char *Ptr = static_cast<char *>(GPA.allocate(1));
-  GPA.deallocate(Ptr);
+  void *Ptr = AllocateMemory(GPA);
+  DeallocateMemory(GPA, Ptr);
 
   std::string DeathRegex = "Use After Free.*";
-  DeathRegex.append("backtrace\\.cpp:41.*");
+  DeathRegex.append("TouchMemory.*");
 
   DeathRegex.append("was deallocated.*");
-  DeathRegex.append("backtrace\\.cpp:31.*");
+  DeathRegex.append("DeallocateMemory.*");
 
   DeathRegex.append("was allocated.*");
-  DeathRegex.append("backtrace\\.cpp:30.*");
-  ASSERT_DEATH({ *Ptr = 7; }, DeathRegex);
+  DeathRegex.append("AllocateMemory.*");
+  ASSERT_DEATH(TouchMemory(Ptr), DeathRegex);
 }
 
 TEST(Backtrace, Short) {
diff --git a/compiler-rt/lib/msan/msan_interceptors.cpp b/compiler-rt/lib/msan/msan_interceptors.cpp
index 9b1340b0104e2..6459c7a593eb8 100644
--- a/compiler-rt/lib/msan/msan_interceptors.cpp
+++ b/compiler-rt/lib/msan/msan_interceptors.cpp
@@ -1304,6 +1304,8 @@ int OnExit() {
       ForEachMappedRegion(map, __msan_unpoison);                               \
   } while (false)
 
+#define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED (!msan_inited)
+
 #define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end)                           \
   if (MsanThread *t = GetCurrentThread()) {                                    \
     *begin = t->tls_begin();                                                   \
diff --git a/compiler-rt/lib/profile/GCDAProfiling.c b/compiler-rt/lib/profile/GCDAProfiling.c
index 498c05900bf27..5ff1e9cd80703 100644
--- a/compiler-rt/lib/profile/GCDAProfiling.c
+++ b/compiler-rt/lib/profile/GCDAProfiling.c
@@ -348,20 +348,29 @@ void llvm_gcda_start_file(const char *orig_filename, const char version[4],
   fd = open(filename, O_RDWR | O_BINARY);
 
   if (fd == -1) {
-    /* Try opening the file, creating it if necessary. */
-    new_file = 1;
-    mode = "w+b";
-    fd = open(filename, O_RDWR | O_CREAT | O_BINARY, 0644);
-    if (fd == -1) {
+    /* Try creating the file. */
+    fd = open(filename, O_RDWR | O_CREAT | O_EXCL | O_BINARY, 0644);
+    if (fd != -1) {
+      new_file = 1;
+      mode = "w+b";
+    } else {
       /* Try creating the directories first then opening the file. */
       __llvm_profile_recursive_mkdir(filename);
-      fd = open(filename, O_RDWR | O_CREAT | O_BINARY, 0644);
-      if (fd == -1) {
-        /* Bah! It's hopeless. */
-        int errnum = errno;
-        fprintf(stderr, "profiling: %s: cannot open: %s\n", filename,
-                strerror(errnum));
-        return;
+      fd = open(filename, O_RDWR | O_CREAT | O_EXCL | O_BINARY, 0644);
+      if (fd != -1) {
+        new_file = 1;
+        mode = "w+b";
+      } else {
+        /* Another process may have created the file just now.
+         * Try opening it without O_CREAT and O_EXCL. */
+        fd = open(filename, O_RDWR | O_BINARY);
+        if (fd == -1) {
+          /* Bah! It's hopeless. */
+          int errnum = errno;
+          fprintf(stderr, "profiling: %s: cannot open: %s\n", filename,
+                  strerror(errnum));
+          return;
+        }
       }
     }
   }
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp
index 906d4af7f5eed..ec77b9cbfee8c 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp
@@ -25,7 +25,7 @@ const char *PrimaryAllocatorName = "SizeClassAllocator";
 const char *SecondaryAllocatorName = "LargeMmapAllocator";
 
 // ThreadSanitizer for Go uses libc malloc/free.
-#if SANITIZER_GO || defined(SANITIZER_USE_MALLOC)
+#if defined(SANITIZER_USE_MALLOC)
 # if SANITIZER_LINUX && !SANITIZER_ANDROID
 extern "C" void *__libc_malloc(uptr size);
 #  if !SANITIZER_GO
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_common.cpp
index f5f9f49d8cffc..87efda5bd3721 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common.cpp
@@ -274,6 +274,7 @@ uptr ReadBinaryNameCached(/*out*/char *buf, uptr buf_len) {
   return name_len;
 }
 
+#if !SANITIZER_GO
 void PrintCmdline() {
   char **argv = GetArgv();
   if (!argv) return;
@@ -282,6 +283,7 @@ void PrintCmdline() {
     Printf("%s ", argv[i]);
   Printf("\n\n");
 }
+#endif
 
 // Malloc hooks.
 static const int kMaxMallocFreeHooks = 5;
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
index 1671273174de1..0ca86dee69a32 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
@@ -9748,6 +9748,10 @@ INTERCEPTOR(int, sigaltstack, void *ss, void *oss) {
 
 #if SANITIZER_INTERCEPT_UNAME
 INTERCEPTOR(int, uname, struct utsname *utsname) {
+#if SANITIZER_LINUX
+  if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
+    return internal_uname(utsname);
+#endif
   void *ctx;
   COMMON_INTERCEPTOR_ENTER(ctx, uname, utsname);
   int res = REAL(uname)(utsname);
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
index 16a3dc9a279d5..23469953241bf 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
@@ -26,7 +26,7 @@
 #include "sanitizer_placement_new.h"
 #include "sanitizer_procmaps.h"
 
-#if SANITIZER_LINUX
+#if SANITIZER_LINUX && !SANITIZER_GO
 #include <asm/param.h>
 #endif
 
@@ -552,7 +552,8 @@ const char *GetEnv(const char *name) {
 #endif
 }
 
-#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD && !SANITIZER_OPENBSD
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD && !SANITIZER_OPENBSD && \
+    !SANITIZER_GO
 extern "C" {
 SANITIZER_WEAK_ATTRIBUTE extern void *__libc_stack_end;
 }
@@ -584,7 +585,7 @@ static void ReadNullSepFileToArray(const char *path, char ***arr,
 }
 #endif
 
-#if !SANITIZER_OPENBSD
+#if !SANITIZER_OPENBSD && !SANITIZER_GO
 static void GetArgsAndEnv(char ***argv, char ***envp) {
 #if SANITIZER_FREEBSD
   // On FreeBSD, retrieving the argument and environment arrays is done via the
@@ -1014,9 +1015,8 @@ static uptr GetKernelAreaSize() {
   // is modified (e.g. under schroot) so check this as well.
   struct utsname uname_info;
   int pers = personality(0xffffffffUL);
-  if (!(pers & PER_MASK)
-      && uname(&uname_info) == 0
-      && internal_strstr(uname_info.machine, "64"))
+  if (!(pers & PER_MASK) && internal_uname(&uname_info) == 0 &&
+      internal_strstr(uname_info.machine, "64"))
     return 0;
 #endif  // SANITIZER_ANDROID
 
@@ -1071,7 +1071,8 @@ uptr GetMaxUserVirtualAddress() {
 
 #if !SANITIZER_ANDROID
 uptr GetPageSize() {
-#if SANITIZER_LINUX && (defined(__x86_64__) || defined(__i386__))
+#if SANITIZER_LINUX && (defined(__x86_64__) || defined(__i386__)) && \
+    defined(EXEC_PAGESIZE)
   return EXEC_PAGESIZE;
 #elif SANITIZER_FREEBSD || SANITIZER_NETBSD
 // Use sysctl as sysconf can trigger interceptors internally.
@@ -1627,6 +1628,12 @@ uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg,
 }
 #endif  // defined(__x86_64__) && SANITIZER_LINUX
 
+#if SANITIZER_LINUX
+int internal_uname(struct utsname *buf) {
+  return internal_syscall(SYSCALL(uname), buf);
+}
+#endif
+
 #if SANITIZER_ANDROID
 #if __ANDROID_API__ < 21
 extern "C" __attribute__((weak)) int dl_iterate_phdr(
@@ -1854,6 +1861,105 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
 #endif
   u32 instr = *(u32 *)pc;
   return (instr >> 21) & 1 ? WRITE: READ;
+#elif defined(__riscv)
+  unsigned long pc = ucontext->uc_mcontext.__gregs[REG_PC];
+  unsigned faulty_instruction = *(uint16_t *)pc;
+
+#if defined(__riscv_compressed)
+  if ((faulty_instruction & 0x3) != 0x3) {  // it's a compressed instruction
+    // set op_bits to the instruction bits [1, 0, 15, 14, 13]
+    unsigned op_bits =
+        ((faulty_instruction & 0x3) << 3) | (faulty_instruction >> 13);
+    unsigned rd = faulty_instruction & 0xF80;  // bits 7-11, inclusive
+    switch (op_bits) {
+      case 0b10'010:  // c.lwsp (rd != x0)
+#if __riscv_xlen == 64
+      case 0b10'011:  // c.ldsp (rd != x0)
+#endif
+        return rd ? SignalContext::READ : SignalContext::UNKNOWN;
+      case 0b00'010:  // c.lw
+#if __riscv_flen >= 32 && __riscv_xlen == 32
+      case 0b10'011:  // c.flwsp
+#endif
+#if __riscv_flen >= 32 || __riscv_xlen == 64
+      case 0b00'011:  // c.flw / c.ld
+#endif
+#if __riscv_flen == 64
+      case 0b00'001:  // c.fld
+      case 0b10'001:  // c.fldsp
+#endif
+        return SignalContext::READ;
+      case 0b00'110:  // c.sw
+      case 0b10'110:  // c.swsp
+#if __riscv_flen >= 32 || __riscv_xlen == 64
+      case 0b00'111:  // c.fsw / c.sd
+      case 0b10'111:  // c.fswsp / c.sdsp
+#endif
+#if __riscv_flen == 64
+      case 0b00'101:  // c.fsd
+      case 0b10'101:  // c.fsdsp
+#endif
+        return SignalContext::WRITE;
+      default:
+        return SignalContext::UNKNOWN;
+    }
+  }
+#endif
+
+  unsigned opcode = faulty_instruction & 0x7f;         // lower 7 bits
+  unsigned funct3 = (faulty_instruction >> 12) & 0x7;  // bits 12-14, inclusive
+  switch (opcode) {
+    case 0b0000011:  // loads
+      switch (funct3) {
+        case 0b000:  // lb
+        case 0b001:  // lh
+        case 0b010:  // lw
+#if __riscv_xlen == 64
+        case 0b011:  // ld
+#endif
+        case 0b100:  // lbu
+        case 0b101:  // lhu
+          return SignalContext::READ;
+        default:
+          return SignalContext::UNKNOWN;
+      }
+    case 0b0100011:  // stores
+      switch (funct3) {
+        case 0b000:  // sb
+        case 0b001:  // sh
+        case 0b010:  // sw
+#if __riscv_xlen == 64
+        case 0b011:  // sd
+#endif
+          return SignalContext::WRITE;
+        default:
+          return SignalContext::UNKNOWN;
+      }
+#if __riscv_flen >= 32
+    case 0b0000111:  // floating-point loads
+      switch (funct3) {
+        case 0b010:  // flw
+#if __riscv_flen == 64
+        case 0b011:  // fld
+#endif
+          return SignalContext::READ;
+        default:
+          return SignalContext::UNKNOWN;
+      }
+    case 0b0100111:  // floating-point stores
+      switch (funct3) {
+        case 0b010:  // fsw
+#if __riscv_flen == 64
+        case 0b011:  // fsd
+#endif
+          return SignalContext::WRITE;
+        default:
+          return SignalContext::UNKNOWN;
+      }
+#endif
+    default:
+      return SignalContext::UNKNOWN;
+  }
 #else
   (void)ucontext;
   return UNKNOWN;  // FIXME: Implement.
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux.h b/compiler-rt/lib/sanitizer_common/sanitizer_linux.h
index c28347ad963a7..25a7e0b79bab9 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_linux.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux.h
@@ -25,6 +25,7 @@
 #include "sanitizer_posix.h"
 
 struct link_map;  // Opaque type returned by dlopen().
+struct utsname;
 
 namespace __sanitizer {
 // Dirent structure for getdents(). Note that this structure is different from
@@ -64,6 +65,7 @@ void internal_sigdelset(__sanitizer_sigset_t *set, int signum);
   || defined(__arm__)
 uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg,
                     int *parent_tidptr, void *newtls, int *child_tidptr);
+int internal_uname(struct utsname *buf);
 #endif
 #elif SANITIZER_FREEBSD
 void internal_sigdelset(__sanitizer_sigset_t *set, int signum);
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
index 4d17c9686e4ed..71b3b2f4e7e04 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
@@ -807,7 +807,7 @@ u64 MonotonicNanoTime() {
 }
 #endif  // SANITIZER_LINUX && !SANITIZER_GO
 
-#if !SANITIZER_OPENBSD
+#if !SANITIZER_OPENBSD && !SANITIZER_GO
 void ReExec() {
   const char *pathname = "/proc/self/exe";
 
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux_s390.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux_s390.cpp
index 16b4c9b633d8f..bb2f5b5f9f7df 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_linux_s390.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux_s390.cpp
@@ -123,12 +123,8 @@ static bool FixedCVE_2016_2143() {
   // adjust this for their own kernels.
   struct utsname buf;
   unsigned int major, minor, patch = 0;
-  // Depending on the concrete sanitizer being used, uname may or may not
-  // be intercepted. Make sure we use the libc version in either case.
-  using Uname = int (*)(struct utsname *);
-  Uname uname = reinterpret_cast<Uname>(dlsym(RTLD_NEXT, "uname"));
   // This should never fail, but just in case...
-  if (uname == nullptr || uname(&buf))
+  if (internal_uname(&buf))
     return false;
   const char *ptr = buf.release;
   major = internal_simple_strtoll(ptr, &ptr, 10);
diff --git a/compiler-rt/lib/tsan/go/build.bat b/compiler-rt/lib/tsan/go/build.bat
index bf502873b1132..0755688e5bd3e 100644
--- a/compiler-rt/lib/tsan/go/build.bat
+++ b/compiler-rt/lib/tsan/go/build.bat
@@ -59,4 +59,4 @@ gcc ^
   -DSANITIZER_DEBUG=0 ^
   -O3 ^
   -fomit-frame-pointer ^
-  -std=c++11
+  -std=c++14
diff --git a/compiler-rt/lib/tsan/go/buildgo.sh b/compiler-rt/lib/tsan/go/buildgo.sh
index 99a6a9ea19d33..6133421318f2d 100755
--- a/compiler-rt/lib/tsan/go/buildgo.sh
+++ b/compiler-rt/lib/tsan/go/buildgo.sh
@@ -66,6 +66,10 @@ if [ "`uname -a | grep Linux`" != "" ]; then
 		ARCHCFLAGS=""
 	fi
 elif [ "`uname -a | grep FreeBSD`" != "" ]; then
+	# The resulting object still depends on libc.
+	# We removed this dependency for Go runtime for other OSes,
+	# and we should remove it for FreeBSD as well, but there is no pressing need.
+	DEPENDS_ON_LIBC=1
 	SUFFIX="freebsd_amd64"
 	OSCFLAGS="-fno-strict-aliasing -fPIC -Werror"
 	ARCHCFLAGS="-m64"
@@ -149,16 +153,20 @@ for F in $SRCS; do
 	cat $F >> $DIR/gotsan.cpp
 done
 
-FLAGS=" -I../rtl -I../.. -I../../sanitizer_common -I../../../include -std=c++11 -Wall -fno-exceptions -fno-rtti -DSANITIZER_GO=1 -DSANITIZER_DEADLOCK_DETECTOR_VERSION=2 $OSCFLAGS $ARCHCFLAGS"
+FLAGS=" -I../rtl -I../.. -I../../sanitizer_common -I../../../include -std=c++14 -Wall -fno-exceptions -fno-rtti -DSANITIZER_GO=1 -DSANITIZER_DEADLOCK_DETECTOR_VERSION=2 $OSCFLAGS $ARCHCFLAGS"
+DEBUG_FLAGS="$FLAGS -DSANITIZER_DEBUG=1 -g"
+FLAGS="$FLAGS -DSANITIZER_DEBUG=0 -O3 -fomit-frame-pointer"
+if [ "$SUFFIX" = "linux_ppc64le" ]; then
+	FLAGS="$FLAGS -mcpu=power8 -fno-function-sections"
+elif [ "$SUFFIX" = "linux_amd64" ]; then
+	FLAGS="$FLAGS -msse3"
+fi
+
 if [ "$DEBUG" = "" ]; then
-	FLAGS="$FLAGS -DSANITIZER_DEBUG=0 -O3 -fomit-frame-pointer"
-	if [ "$SUFFIX" = "linux_ppc64le" ]; then
-		FLAGS="$FLAGS -mcpu=power8 -fno-function-sections"
-	elif [ "$SUFFIX" = "linux_amd64" ]; then
-		FLAGS="$FLAGS -msse3"
-	fi
+	# Do a build test with debug flags.
+	$CC $DIR/gotsan.cpp -c -o $DIR/race_debug_$SUFFIX.syso $DEBUG_FLAGS $CFLAGS
 else
-	FLAGS="$FLAGS -DSANITIZER_DEBUG=1 -g"
+	FLAGS="$DEBUG_FLAGS"
 fi
 
 if [ "$SILENT" != "1" ]; then
@@ -168,6 +176,14 @@ $CC $DIR/gotsan.cpp -c -o $DIR/race_$SUFFIX.syso $FLAGS $CFLAGS
 
 $CC $OSCFLAGS $ARCHCFLAGS test.c $DIR/race_$SUFFIX.syso -g -o $DIR/test $OSLDFLAGS $LDFLAGS
 
+# Verify that no libc specific code is present.
+if [ "$DEPENDS_ON_LIBC" != "1" ]; then
+	if nm $DIR/race_$SUFFIX.syso | grep -q __libc_; then
+		printf -- '%s seems to link to libc\n' "race_$SUFFIX.syso"
+		exit 1
+	fi
+fi
+
 export GORACE="exitcode=0 atexit_sleep_ms=0"
 if [ "$SILENT" != "1" ]; then
   $DIR/test
diff --git a/compiler-rt/lib/tsan/rtl/tsan_clock.cpp b/compiler-rt/lib/tsan/rtl/tsan_clock.cpp
index 7989a90837cd5..96bbfa1d4cc2f 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_clock.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_clock.cpp
@@ -187,7 +187,7 @@ void ThreadClock::acquire(ClockCache *c, SyncClock *src) {
 
 void ThreadClock::releaseStoreAcquire(ClockCache *c, SyncClock *sc) {
   DCHECK_LE(nclk_, kMaxTid);
-  DCHECK_LE(dst->size_, kMaxTid);
+  DCHECK_LE(sc->size_, kMaxTid);
 
   if (sc->size_ == 0) {
     // ReleaseStore will correctly set release_store_tid_,
diff --git a/compiler-rt/test/asan/TestCases/Darwin/duplicate_os_log_reports.cpp b/compiler-rt/test/asan/TestCases/Darwin/duplicate_os_log_reports.cpp
new file mode 100644
index 0000000000000..5f923d22a9a41
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/Darwin/duplicate_os_log_reports.cpp
@@ -0,0 +1,68 @@
+// UNSUPPORTED: ios
+// REQUIRES: shell
+// REQUIRES: darwin_log_cmd
+// RUN: %clangxx_asan -fsanitize-recover=address %s -o %t
+// RUN: { %env_asan_opts=halt_on_error=0,log_to_syslog=1 %run %t > %t.process_output.txt 2>&1 & } \
+// RUN: ; export TEST_PID=$! ; wait ${TEST_PID}
+
+// Check process output.
+// RUN: FileCheck %s --check-prefixes CHECK,CHECK-PROC -input-file=%t.process_output.txt
+
+// Check syslog output. We filter recent system logs based on PID to avoid
+// getting the logs of previous test runs.
+// RUN: log show --debug --last 2m  --predicate "processID == ${TEST_PID}" --style syslog > %t.process_syslog_output.txt
+// RUN: FileCheck %s -input-file=%t.process_syslog_output.txt
+#include <cassert>
+#include <cstdio>
+#include <sanitizer/asan_interface.h>
+
+const int kBufferSize = 512;
+char *buffer;
+
+// `readZero` and `readOne` exist so that we can distinguish the two
+// error reports based on the symbolized stacktrace.
+void readZero() {
+  assert(__asan_address_is_poisoned(buffer));
+  char c = buffer[0];
+  printf("Read %c\n", c);
+}
+
+void readOne() {
+  assert(__asan_address_is_poisoned(buffer + 1));
+  char c = buffer[1];
+  printf("Read %c\n", c);
+}
+
+int main() {
+  buffer = static_cast<char *>(malloc(kBufferSize));
+  assert(buffer);
+  // Deliberately poison `buffer` so that we have a deterministic way
+  // triggering two ASan reports in a row in the no halt_on_error mode (e.g. Two
+  // heap-use-after free in a row might not be deterministic).
+  __asan_poison_memory_region(buffer, kBufferSize);
+
+  // This sequence of ASan reports are designed to catch an old bug in the way
+  // ASan's internal syslog buffer was handled after reporting an issue.
+  // Previously in the no halt_on_error mode the internal buffer wasn't cleared
+  // after reporting an issue. When another issue was encountered everything
+  // that was already in the buffer would be written to the syslog again
+  // leading to duplicate reports in the syslog.
+
+  // First bad access.
+  // CHECK: use-after-poison
+  // CHECK-NEXT: READ of size 1
+  // CHECK-NEXT: #0 0x{{[0-9a-f]+}} in readZero
+  // CHECK: SUMMARY: {{.*}} use-after-poison {{.*}} in readZero
+  readZero();
+
+  // Second bad access.
+  // CHECK: use-after-poison
+  // CHECK-NEXT: READ of size 1
+  // CHECK-NEXT: #0 0x{{[0-9a-f]+}} in readOne
+  // CHECK: SUMMARY: {{.*}} use-after-poison {{.*}} in readOne
+  readOne();
+
+  // CHECK-PROC: DONE
+  printf("DONE\n");
+  return 0;
+}
diff --git a/compiler-rt/test/asan/TestCases/Linux/read_binary_name_regtest.c b/compiler-rt/test/asan/TestCases/Linux/read_binary_name_regtest.c
index 2cb22fa29cb29..08bf5e125cf40 100644
--- a/compiler-rt/test/asan/TestCases/Linux/read_binary_name_regtest.c
+++ b/compiler-rt/test/asan/TestCases/Linux/read_binary_name_regtest.c
@@ -2,7 +2,11 @@
 // the binary name because of sandbox restrictions.
 // This test uses seccomp-BPF to restrict the readlink() system call and makes
 // sure ASan is still able to
-// RUN: not ls /usr/include/linux/seccomp.h || ( %clang_asan %s -o %t && ( not %run %t 2>&1 ) | FileCheck %s )
+// Disable symbolizing results, since this will invoke llvm-symbolizer, which
+// will be unable to resolve its $ORIGIN due to readlink() restriction and will
+// thus fail to start, causing the test to die with SIGPIPE when attempting to
+// talk to it.
+// RUN: not ls /usr/include/linux/seccomp.h || ( %clang_asan %s -o %t && ( not env ASAN_OPTIONS=symbolize=0 %run %t 2>&1 ) | FileCheck %s )
 // REQUIRES: shell
 // UNSUPPORTED: android
 
diff --git a/compiler-rt/test/lit.common.cfg.py b/compiler-rt/test/lit.common.cfg.py
index 3b1ea53fb8d96..f0618dfc067c2 100644
--- a/compiler-rt/test/lit.common.cfg.py
+++ b/compiler-rt/test/lit.common.cfg.py
@@ -533,6 +533,19 @@ def is_windows_lto_supported():
   # much slower. Let's override this and run lit tests with 'abort_on_error=0'.
   config.default_sanitizer_opts += ['abort_on_error=0']
   config.default_sanitizer_opts += ['log_to_syslog=0']
+  if lit.util.which('log'):
+    # Querying the log can only done by a privileged user so
+    # so check if we can query the log.
+    exit_code = -1
+    with open('/dev/null', 'r') as f:
+      # Run a `log show` command the should finish fairly quickly and produce very little output.
+      exit_code = subprocess.call(['log', 'show', '--last', '1m', '--predicate', '1 == 0'], stdout=f, stderr=f)
+    if exit_code == 0:
+      config.available_features.add('darwin_log_cmd')
+    else:
+      lit_config.warning('log command found but cannot queried')
+  else:
+    lit_config.warning('log command not found. Some tests will be skipped.')
 elif config.android:
   config.default_sanitizer_opts += ['abort_on_error=0']
 
diff --git a/compiler-rt/test/profile/Inputs/instrprof-gcov-parallel.driver.c b/compiler-rt/test/profile/Inputs/instrprof-gcov-parallel.driver.c
new file mode 100644
index 0000000000000..6ce12d35772f1
--- /dev/null
+++ b/compiler-rt/test/profile/Inputs/instrprof-gcov-parallel.driver.c
@@ -0,0 +1,36 @@
+#include <stdatomic.h>
+#include <stdlib.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#define CHILDREN 7
+
+int main(int argc, char *argv[]) {
+  _Atomic int *sync = mmap(NULL, sizeof(int), PROT_READ | PROT_WRITE,
+                           MAP_SHARED | MAP_ANONYMOUS, -1, 0);
+  if (sync == MAP_FAILED)
+    return 1;
+  *sync = 0;
+
+  for (int i = 0; i < CHILDREN; i++) {
+    pid_t pid = fork();
+    if (!pid) {
+      // child
+      while (*sync == 0)
+        ; // wait the parent in order to call execl simultaneously
+      execl(argv[1], argv[1], NULL);
+    } else if (pid == -1) {
+      *sync = 1; // release all children
+      return 1;
+    }
+  }
+
+  // parent
+  *sync = 1; // start the program in all children simultaneously
+  for (int i = 0; i < CHILDREN; i++)
+    wait(NULL);
+
+  return 0;
+}
diff --git a/compiler-rt/test/profile/Inputs/instrprof-gcov-parallel.target.c b/compiler-rt/test/profile/Inputs/instrprof-gcov-parallel.target.c
new file mode 100644
index 0000000000000..ae6e60fb21901
--- /dev/null
+++ b/compiler-rt/test/profile/Inputs/instrprof-gcov-parallel.target.c
@@ -0,0 +1,9 @@
+#define COUNT 101
+
+static volatile int aaa;
+
+int main(int argc, char *argv[]) {
+  for (int i = 0; i < COUNT; i++)
+    aaa++;
+  return 0;
+}
diff --git a/compiler-rt/test/profile/Posix/instrprof-gcov-parallel.test b/compiler-rt/test/profile/Posix/instrprof-gcov-parallel.test
new file mode 100644
index 0000000000000..0c7198e3c4e9e
--- /dev/null
+++ b/compiler-rt/test/profile/Posix/instrprof-gcov-parallel.test
@@ -0,0 +1,16 @@
+RUN: mkdir -p %t.d
+RUN: cd %t.d
+
+RUN: %clang -o %t.driver %S/../Inputs/instrprof-gcov-parallel.driver.c
+RUN: %clang --coverage -o %t.target %S/../Inputs/instrprof-gcov-parallel.target.c
+RUN: test -f instrprof-gcov-parallel.target.gcno
+
+RUN: rm -f instrprof-gcov-parallel.target.gcda
+RUN: %run %t.driver %t.target
+RUN: llvm-cov gcov instrprof-gcov-parallel.target.gcda
+RUN: FileCheck --input-file instrprof-gcov-parallel.target.c.gcov %s
+
+# Test if the .gcda file is correctly created from one of child processes
+# and counters of all processes are recorded correctly.
+# 707 = CHILDREN * COUNT
+CHECK: 707: {{[0-9]+}}: aaa++;
diff --git a/compiler-rt/test/sanitizer_common/ios_commands/iossim_prepare.py b/compiler-rt/test/sanitizer_common/ios_commands/iossim_prepare.py
index 1a350006c33f6..0cfec6b5777b9 100755
--- a/compiler-rt/test/sanitizer_common/ios_commands/iossim_prepare.py
+++ b/compiler-rt/test/sanitizer_common/ios_commands/iossim_prepare.py
@@ -9,8 +9,9 @@
 if not device_id:
     raise EnvironmentError('Specify SANITIZER_IOSSIM_TEST_DEVICE_IDENTIFIER to select which simulator to use.')
 
-boot_cmd = ['xcrun', 'simctl', 'bootstatus', device_id, '-b']
-subprocess.check_call(boot_cmd)
+DEVNULL = open(os.devnull, 'w')
+subprocess.call(['xcrun', 'simctl', 'shutdown', device_id], stderr=DEVNULL)
+subprocess.check_call(['xcrun', 'simctl', 'boot', device_id])
 # TODO(rdar58118442): we start the simulator here, but we never tear it down
 
 
diff --git a/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_program_state.cpp b/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_program_state.cpp
index 476906846793f..46ae81872fa99 100644
--- a/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_program_state.cpp
+++ b/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_program_state.cpp
@@ -2,11 +2,9 @@
 //      Check that \DexExpectProgramState correctly applies a penalty when
 //      an expected program state is never found.
 //
-// REQUIRES: system-linux, lldb
+// UNSUPPORTED: system-darwin
 //
-// RUN: not %dexter_base test --fail-lt 1.0 -w \
-// RUN:     --builder 'clang' --debugger 'lldb' --cflags "-O0 -glldb" -- %s \
-// RUN:     | FileCheck %s
+// RUN: not %dexter_regression_test -- %s | FileCheck %s
 // CHECK: expect_program_state.cpp:
 
 int GCD(int lhs, int rhs)
diff --git a/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_step_kinds.cpp b/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_step_kinds.cpp
index 9612c068ad611..2075cd22fa673 100644
--- a/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_step_kinds.cpp
+++ b/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_step_kinds.cpp
@@ -2,11 +2,9 @@
 //      Check that \DexExpectStepKind correctly applies a penalty when
 //      unexpected step kinds are encountered.
 //
-// REQUIRES: system-linux, lldb
+// UNSUPPORTED: system-darwin
 //
-// RUN: not %dexter_base test --fail-lt 1.0 -w  \
-// RUN:     --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \
-// RUN:     | FileCheck %s
+// RUN: not %dexter_regression_test -- %s | FileCheck %s
 // CHECK: expect_step_kinds.cpp:
 
 int abs(int i){
diff --git a/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_step_order.cpp b/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_step_order.cpp
index e52a7fffe203e..dc5f8c6b0c92b 100644
--- a/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_step_order.cpp
+++ b/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_step_order.cpp
@@ -2,11 +2,9 @@
 //      Check that \DexExpectStepOrder correctly applies a penalty for steps
 //      found out of expected order.
 //
-// REQUIRES: system-linux, lldb
+// UNSUPPORTED: system-darwin
 //
-// RUN: not %dexter_base test --fail-lt 1.0 -w \
-// RUN:     --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \
-// RUN:     | FileCheck %s
+// RUN: not %dexter_regression_test -- %s | FileCheck %s
 // CHECK: expect_step_order.cpp:
 
 int main()
diff --git a/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_watch_type.cpp b/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_watch_type.cpp
index 145eeed7ef861..838e1f87e3039 100644
--- a/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_watch_type.cpp
+++ b/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_watch_type.cpp
@@ -2,11 +2,14 @@
 //      Check that \DexExpectWatchType applies penalties when expected
 //      types are not found and unexpected types are.
 //
-// REQUIRES: system-linux, lldb
+// UNSUPPORTED: system-darwin
 //
-// RUN: not %dexter_base test --fail-lt 1.0 -w \
-// RUN:     --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \
-// RUN:     | FileCheck %s
+//
+// NOTE: This test passes but not in the expected way on Windows.
+// TODO: Reduce this test's coverage and be more specific about
+// expected behaviour.
+//
+// RUN: not %dexter_regression_test -- %s | FileCheck %s
 // CHECK: expect_watch_type.cpp:
 
 template<class T>
diff --git a/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_watch_value.cpp b/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_watch_value.cpp
index b4c60fa321791..fa0ed119980ea 100644
--- a/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_watch_value.cpp
+++ b/debuginfo-tests/dexter/feature_tests/commands/penalty/expect_watch_value.cpp
@@ -2,11 +2,10 @@
 //      Check that \DexExpectWatchValue correctly applies a penalty when
 //      expected values are not found.
 //
-// REQUIRES: system-linux, lldb
+// UNSUPPORTED: system-darwin
 //
-// RUN: not %dexter_base test --fail-lt 1.0 -w \
-// RUN:     --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \
-// RUN:     | FileCheck %s
+//
+// RUN: not %dexter_regression_test -- %s | FileCheck %s
 // CHECK: expect_watch_value.cpp:
 
 int main()
diff --git a/debuginfo-tests/dexter/feature_tests/commands/penalty/unreachable.cpp b/debuginfo-tests/dexter/feature_tests/commands/penalty/unreachable.cpp
index 5cea1c4257d0c..48f56b7d0452f 100644
--- a/debuginfo-tests/dexter/feature_tests/commands/penalty/unreachable.cpp
+++ b/debuginfo-tests/dexter/feature_tests/commands/penalty/unreachable.cpp
@@ -2,11 +2,10 @@
 //      Check that \DexUnreachable correctly applies a penalty if the command
 //      line is stepped on.
 //
-// REQUIRES: system-linux, lldb
+// UNSUPPORTED: system-darwin
 //
-// RUN: not %dexter_base test --fail-lt 1.0 -w \
-// RUN:     --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \
-// RUN:     | FileCheck %s
+//
+// RUN: not %dexter_regression_test -- %s | FileCheck %s
 // CHECK: unreachable.cpp:
 
 int
diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_program_state.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_program_state.cpp
index ebb4766f089a8..3e77938446b07 100644
--- a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_program_state.cpp
+++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_program_state.cpp
@@ -2,11 +2,10 @@
 //      Check that \DexExpectWatchValue applies no penalties when expected
 //      program states are found.
 //
-// REQUIRES: system-linux, lldb
+// UNSUPPORTED: system-darwin
 //
-// RUN: %dexter_base test --fail-lt 1.0 -w \
-// RUN:     --builder 'clang' --debugger 'lldb' --cflags "-O0 -glldb" -- %s \
-// RUN:     | FileCheck %s
+//
+// RUN: %dexter_regression_test -- %s | FileCheck %s
 // CHECK: expect_program_state.cpp:
 
 int GCD(int lhs, int rhs)
diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/direction.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/direction.cpp
index 3ff0e9fecbc67..1f7a96bafffae 100644
--- a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/direction.cpp
+++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/direction.cpp
@@ -5,11 +5,12 @@
 //      'VERTICAL_FORWARD' for every step onto a greater source line number in
 //      the same function.
 //
-// REQUIRES: system-linux, lldb
+// UNSUPPORTED: system-darwin
 //
-// RUN: %dexter_base test --fail-lt 1.0 -w  \
-// RUN:     --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \
-// RUN:     | FileCheck %s
+// TODO: The dbgeng debugger does not support column step reporting at present.
+// XFAIL: system-windows
+//
+// RUN: %dexter_regression_test -- %s | FileCheck %s
 // CHECK: direction.cpp:
 
 int func(int i) {
diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/func.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/func.cpp
index d5ab3ffbd0367..b386dc4aa48c6 100644
--- a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/func.cpp
+++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/func.cpp
@@ -3,11 +3,9 @@
 //      trivial test. Expect one 'FUNC' per call to a function which is defined
 //      in one of the source files in the test directory.
 //
-// REQUIRES: system-linux, lldb
+// UNSUPPORTED: system-darwin
 //
-// RUN: %dexter_base test --fail-lt 1.0 -w  \
-// RUN:     --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \
-// RUN:     | FileCheck %s
+// RUN: %dexter_regression_test -- %s | FileCheck %s
 // CHECK: func.cpp:
 
 int func(int i) {
diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/func_external.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/func_external.cpp
index 6dff5c8029687..cbefcb9690822 100644
--- a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/func_external.cpp
+++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/func_external.cpp
@@ -2,13 +2,13 @@
 //      Check that \DexExpectStepKind correctly counts 'FUNC_EXTERNAL' steps
 //      for a trivial test. Expect one 'FUNC_EXTERNAL' per external call.
 //
-// REQUIRES: system-linux, lldb
-// XFAIL: system-linux
-// This fails right now on my linux machine, needs examining as to why.
+// UNSUPPORTED: system-darwin
 //
-// RUN: %dexter --fail-lt 1.0 -w  \
-// RUN:     --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \
-// RUN:     | FileCheck %s
+// XFAIL:*
+// This fails right now on my linux and windows machine, needs examining as to
+// why.
+//
+// RUN: %dexter_regression_test -- %s | FileCheck %s
 // CHECK: func_external.cpp:
 
 #include <cstdlib>
diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/recursive.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/recursive.cpp
index f2b54e4426790..0264420a46be2 100644
--- a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/recursive.cpp
+++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/recursive.cpp
@@ -3,11 +3,9 @@
 //      Specifically, ensure recursive calls count towards 'FUNC' and not
 //      'VERTICAL_BACKWARD'.
 //
-// REQUIRES: system-linux, lldb
+// UNSUPPORTED: system-darwin
 //
-// RUN: %dexter_base test --fail-lt 1.0 -w  \
-// RUN:     --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \
-// RUN:     | FileCheck %s
+// RUN: %dexter_regression_test -- %s | FileCheck %s
 // CHECK: recursive.cpp:
 
 int func(int i) {
diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/small_loop.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/small_loop.cpp
index 1f9060747ec2c..fb19869f9182d 100644
--- a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/small_loop.cpp
+++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_kind/small_loop.cpp
@@ -3,11 +3,9 @@
 //      where the last source line in the loop is a call. Expect steps out
 //      of a function to a line before the call to count as 'VERTICAL_BACKWARD'.
 //
-// REQUIRES: system-linux, lldb
+// UNSUPPORTED: system-darwin
 //
-// RUN: %dexter_base test --fail-lt 1.0 -w  \
-// RUN:     --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \
-// RUN:     | FileCheck %s
+// RUN: %dexter_regression_test -- %s | FileCheck %s
 // CHECK: small_loop.cpp:
 
 int func(int i){
diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_order.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_order.cpp
index a89a0cee3cc5c..cbcb5063c10f1 100644
--- a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_order.cpp
+++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_step_order.cpp
@@ -2,11 +2,9 @@
 //      Check that \DexExpectStepOrder applies no penalty when the expected
 //      order is found.
 //
-// REQUIRES: system-linux, lldb
+// UNSUPPORTED: system-darwin
 //
-// RUN: %dexter_base test --fail-lt 1.0 -w \
-// RUN:     --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \
-// RUN:     | FileCheck %s
+// RUN: %dexter_regression_test -- %s | FileCheck %s
 // CHECK: expect_step_order.cpp:
 
 int main()
diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_watch_type.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_watch_type.cpp
index 41593837a9c88..67077bebcef8c 100644
--- a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_watch_type.cpp
+++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_watch_type.cpp
@@ -2,11 +2,13 @@
 //      Check that \DexExpectWatchType applies no penalties when expected
 //      types are found.
 //
-// REQUIRES: system-linux, lldb
+// UNSUPPORTED: system-darwin
 //
-// RUN: %dexter_base test --fail-lt 1.0 -w \
-// RUN:     --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \
-// RUN:     | FileCheck %s
+// TODO: On Windows WITH dbgeng, This test takes a long time to run and doesn't evaluate type values
+// in the same manner as LLDB.
+// XFAIL: system-windows
+//
+// RUN: %dexter_regression_test -- %s | FileCheck %s
 // CHECK: expect_watch_type.cpp:
 
 template<class T>
diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_watch_value.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_watch_value.cpp
index 035e411369803..f22431a0900d9 100644
--- a/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_watch_value.cpp
+++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/expect_watch_value.cpp
@@ -2,11 +2,9 @@
 //      Check that \DexExpectWatchValue applies no penalties when expected
 //      values are found.
 //
-// REQUIRES: system-linux, lldb
+// UNSUPPORTED: system-darwin
 //
-// RUN: %dexter_base test --fail-lt 1.0 -w \
-// RUN:     --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \
-// RUN:     | FileCheck %s
+// RUN: %dexter_regression_test -- %s | FileCheck %s
 // CHECK: expect_watch_value.cpp:
 
 unsigned long Factorial(int n) {
diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/unreachable.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/unreachable.cpp
index b158b4851be44..bcc1b835a6487 100644
--- a/debuginfo-tests/dexter/feature_tests/commands/perfect/unreachable.cpp
+++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/unreachable.cpp
@@ -2,11 +2,9 @@
 //    Check that \DexUnreachable has no effect if the command line is never
 //    stepped on.
 //
-// REQUIRES: system-linux, lldb
+// UNSUPPORTED: system-darwin
 //
-// RUN: %dexter_base test --fail-lt 1.0 -w \
-// RUN:     --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \
-// RUN:     | FileCheck %s
+// RUN: %dexter_regression_test -- %s | FileCheck %s
 // CHECK: unreachable.cpp:
 
 int main()
diff --git a/debuginfo-tests/dexter/feature_tests/subtools/view.cpp b/debuginfo-tests/dexter/feature_tests/subtools/view.cpp
index 43598dbdc9b91..03e0b9420fb1d 100644
--- a/debuginfo-tests/dexter/feature_tests/subtools/view.cpp
+++ b/debuginfo-tests/dexter/feature_tests/subtools/view.cpp
@@ -1,11 +1,7 @@
 // Purpose:
 //      Check the `view` subtool works with typical inputs.
 //
-// REQUIRES: system-linux, lldb
-//
-// RUN: %dexter_base test --fail-lt 1.0 -w \
-// RUN:    --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" \
-// RUN:    --results %t -- %s
+// RUN: %dexter_regression_test --results %t -- %s
 //
 // RUN: %dexter_base view %t/view.cpp.dextIR | FileCheck %s
 // CHECK: ## BEGIN
diff --git a/debuginfo-tests/lit.cfg.py b/debuginfo-tests/lit.cfg.py
index 2d93b8da9bf6d..fb2f72357b263 100644
--- a/debuginfo-tests/lit.cfg.py
+++ b/debuginfo-tests/lit.cfg.py
@@ -112,6 +112,35 @@ def get_required_attr(config, attr_name):
 dexter_base_cmd = '"{}" "{}"'.format(config.python3_executable, dexter_path)
 tools.append(ToolSubst('%dexter_base', dexter_base_cmd))
 
+# Set up commands for DexTer regression tests.
+# Builder, debugger, optimisation level and several other flags differ
+# depending on whether we're running a unix like or windows os.
+if platform.system() == 'Windows': 
+  dexter_regression_test_builder = '--builder clang-cl_vs2015'
+  dexter_regression_test_debugger = '--debugger dbgeng'
+  dexter_regression_test_cflags = '--cflags "/Zi /Od"'
+  dexter_regression_test_ldflags = '--ldflags "/Zi"'
+else:
+  dexter_regression_test_builder = '--builder clang'
+  dexter_regression_test_debugger = "--debugger lldb"
+  dexter_regression_test_cflags = '--cflags "-O0 -glldb"'
+  dexter_regression_test_ldflags = ''
+
+# Typical command would take the form:
+# ./path_to_py/python.exe ./path_to_dex/dexter.py test --fail-lt 1.0 -w --builder clang --debugger lldb --cflags '-O0 -g'
+dexter_regression_test_command = ' '.join(
+  # "python3", "dexter.py", test, fail_mode, builder, debugger, cflags, ldflags
+  ["{}".format(config.python3_executable),
+  "{}".format(dexter_path),
+  'test',
+  '--fail-lt 1.0 -w',
+  dexter_regression_test_builder,
+  dexter_regression_test_debugger,
+  dexter_regression_test_cflags,
+  dexter_regression_test_ldflags])
+
+tools.append(ToolSubst('%dexter_regression_test', dexter_regression_test_command))
+
 tool_dirs = [config.llvm_tools_dir]
 
 llvm_config.add_tool_substitutions(tools, tool_dirs)
diff --git a/libc/CMakeLists.txt b/libc/CMakeLists.txt
index 40c7f707872ba..bcac7cbd78dcd 100644
--- a/libc/CMakeLists.txt
+++ b/libc/CMakeLists.txt
@@ -28,6 +28,10 @@ add_subdirectory(config)
 add_subdirectory(include)
 add_subdirectory(utils)
 
+# The loader can potentially depend on the library components so add it
+# after the library implementation directories.
+add_subdirectory(loader)
+
 # The lib and test directories are added at the very end as tests
 # and libraries potentially draw from the components present in all
 # of the other directories.
diff --git a/libc/cmake/modules/LLVMLibCRules.cmake b/libc/cmake/modules/LLVMLibCRules.cmake
index 2391ea50b0db5..d47c16d3e694d 100644
--- a/libc/cmake/modules/LLVMLibCRules.cmake
+++ b/libc/cmake/modules/LLVMLibCRules.cmake
@@ -94,32 +94,35 @@ function(add_gen_header target_name)
   )
 endfunction(add_gen_header)
 
-set(SINGLE_OBJECT_TARGET_TYPE "LIBC_SINGLE_OBJECT")
+set(OBJECT_LIBRARY_TARGET_TYPE "OBJECT_LIBRARY")
 
-# Function to generate single object file.
+# Rule which is essentially a wrapper over add_library to compile a set of
+# sources to object files.
 # Usage:
-#     add_object(
+#     add_object_library(
 #       <target_name>
-#       SRC <source file to compile>
+#       HDRS <list of header files>
+#       SRCS <list of source files>
 #       DEPENDS <list of dependencies>
 #       COMPILE_OPTIONS <optional list of special compile options for this target>
-function(add_object target_name)
+function(add_object_library target_name)
   cmake_parse_arguments(
     "ADD_OBJECT"
     "" # No option arguments
-    "SRC" # Single value arguments
-    "COMPILE_OPTIONS;DEPENDS" # Multivalue arguments
+    "" # Single value arguments
+    "SRCS;HDRS;COMPILE_OPTIONS;DEPENDS" # Multivalue arguments
     ${ARGN}
   )
 
-  if(NOT ADD_OBJECT_SRC)
-    message(FATAL_ERROR "'add_object' rules requires a SRC to be specified.")
+  if(NOT ADD_OBJECT_SRCS)
+    message(FATAL_ERROR "'add_object_library' rule requires SRCS to be specified.")
   endif()
 
   add_library(
     ${target_name}
     OBJECT
-    ${ADD_OBJECT_SRC}
+    ${ADD_OBJECT_SRCS}
+    ${ADD_OBJECT_HDRS}
   )
   target_include_directories(
     ${target_name}
@@ -132,19 +135,33 @@ function(add_object target_name)
       PRIVATE ${ADD_OBJECT_COMPILE_OPTIONS}
     )
   endif()
+
+  set(all_object_files $<TARGET_OBJECTS:${target_name}>)
   if(ADD_OBJECT_DEPENDS)
     add_dependencies(
       ${target_name}
       ${ADD_OBJECT_DEPENDS}
     )
+    foreach(obj_target IN LISTS ADD_ENTRYPOINT_OBJ_SPECIAL_OBJECTS)
+      get_target_property(obj_type ${obj_target} "TARGET_TYPE")
+      if((NOT obj_type) OR (NOT (${obj_type} STREQUAL ${OBJECT_LIBRARY_TARGET_TYPE})))
+        continue()
+      endif()
+      # If a dependency is also a object file library, we will collect the list of
+      # object files from it.
+      get_target_property(obj_files ${obj_target} "OBJECT_FILES")
+      list(APPEND all_object_files ${obj_files})
+    endforeach(obj_target)
   endif()
+  list(REMOVE_DUPLICATES all_object_files)
+
   set_target_properties(
     ${target_name}
     PROPERTIES
-      "TARGET_TYPE" ${SINGLE_OBJECT_TARGET_TYPE}
-      "OBJECT_FILE" $<TARGET_OBJECTS:${target_name}>
+      "TARGET_TYPE" ${OBJECT_LIBRARY_TARGET_TYPE}
+      "OBJECT_FILES" "${all_object_files}"
   )
-endfunction(add_object)
+endfunction(add_object_library)
 
 set(ENTRYPOINT_OBJ_TARGET_TYPE "ENTRYPOINT_OBJ")
 
@@ -165,7 +182,7 @@ function(add_entrypoint_object target_name)
     "ADD_ENTRYPOINT_OBJ"
     "REDIRECTED" # Optional argument
     "NAME" # Single value arguments
-    "SRCS;HDRS;SPECIAL_OBJECTS;DEPENDS;COMPILE_OPTIONS"  # Multi value arguments
+    "SRCS;HDRS;DEPENDS;COMPILE_OPTIONS"  # Multi value arguments
     ${ARGN}
   )
   if(NOT ADD_ENTRYPOINT_OBJ_SRCS)
@@ -203,12 +220,34 @@ function(add_entrypoint_object target_name)
     ${target_name}_objects
     support_common_h
   )
+  set(dep_objects "")
   if(ADD_ENTRYPOINT_OBJ_DEPENDS)
     add_dependencies(
       ${target_name}_objects
       ${ADD_ENTRYPOINT_OBJ_DEPENDS}
     )
+    foreach(dep_target IN LISTS ADD_ENTRYPOINT_OBJ_DEPENDS)
+      if(NOT TARGET ${dep_target})
+        # Not all targets will be visible. So, we will ignore those which aren't
+        # visible yet.
+        continue()
+      endif()
+      get_target_property(obj_type ${dep_target} "TARGET_TYPE")
+      if((NOT obj_type) OR (NOT (${obj_type} STREQUAL ${OBJECT_LIBRARY_TARGET_TYPE})))
+        # Even from among the visible targets, we will collect object files
+        # only from add_object_library targets.
+        continue()
+      endif()
+      # Calling get_target_property requires that the target be visible at this
+      # point. For object library dependencies, this is a reasonable requirement.
+      # We can revisit this in future if we need cases which break under this
+      # requirement.
+      get_target_property(obj_files ${dep_target} "OBJECT_FILES")
+      list(APPEND dep_objects ${obj_files})
+    endforeach(dep_target)
   endif()
+  list(REMOVE_DUPLICATES dep_objects)
+
   if(ADD_ENTRYPOINT_OBJ_COMPILE_OPTIONS)
     target_compile_options(
       ${target_name}_objects
@@ -220,16 +259,6 @@ function(add_entrypoint_object target_name)
   set(object_file "${CMAKE_CURRENT_BINARY_DIR}/${target_name}.o")
 
   set(input_objects $<TARGET_OBJECTS:${target_name}_objects>)
-  if(ADD_ENTRYPOINT_OBJ_SPECIAL_OBJECTS)
-    foreach(obj_target IN LISTS ADD_ENTRYPOINT_OBJ_SPECIAL_OBJECTS)
-      get_target_property(obj_type ${obj_target} "TARGET_TYPE")
-      if((NOT obj_type) OR (NOT (${obj_type} STREQUAL ${SINGLE_OBJECT_TARGET_TYPE})))
-        message(FATAL_ERROR "Unexpected target type for 'SPECIAL_OBJECT' - should be a target introduced by the `add_object` rule.")
-      endif()
-      list(APPEND input_objects $<TARGET_OBJECTS:${obj_target}>)
-    endforeach(obj_target)
-  endif()
-
   add_custom_command(
     OUTPUT ${object_file_raw}
     DEPENDS ${input_objects}
@@ -253,12 +282,16 @@ function(add_entrypoint_object target_name)
     ALL
     DEPENDS ${object_file}
   )
+  set(all_objects ${object_file})
+  list(APPEND all_objects ${dep_objects})
+  set(all_objects_raw ${object_file_raw})
+  list(APPEND all_objects_raw ${dep_objects})
   set_target_properties(
     ${target_name}
     PROPERTIES
       "TARGET_TYPE" ${ENTRYPOINT_OBJ_TARGET_TYPE}
-      "OBJECT_FILE" ${object_file}
-      "OBJECT_FILE_RAW" ${object_file_raw}
+      "OBJECT_FILES" "${all_objects}"
+      "OBJECT_FILES_RAW" "${all_objects_raw}"
   )
 endfunction(add_entrypoint_object)
 
@@ -282,13 +315,13 @@ function(add_entrypoint_library target_name)
   set(obj_list "")
   foreach(dep IN LISTS ENTRYPOINT_LIBRARY_DEPENDS)
     get_target_property(dep_type ${dep} "TARGET_TYPE")
-    string(COMPARE EQUAL ${dep_type} ${ENTRYPOINT_OBJ_TARGET_TYPE} dep_is_entrypoint)
-    if(NOT dep_is_entrypoint)
+    if(NOT (${dep_type} STREQUAL ${ENTRYPOINT_OBJ_TARGET_TYPE}))
       message(FATAL_ERROR "Dependency '${dep}' of 'add_entrypoint_collection' is not an 'add_entrypoint_object' target.")
     endif()
-    get_target_property(target_obj_file ${dep} "OBJECT_FILE")
-    list(APPEND obj_list "${target_obj_file}")
+    get_target_property(target_obj_files ${dep} "OBJECT_FILES")
+    list(APPEND obj_list "${target_obj_files}")
   endforeach(dep)
+  list(REMOVE_DUPLICATES obj_list)
 
   set(library_file "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}${target_name}${CMAKE_STATIC_LIBRARY_SUFFIX}")
   add_custom_command(
@@ -396,17 +429,17 @@ function(add_libc_unittest target_name)
   set(library_deps "")
   foreach(dep IN LISTS LIBC_UNITTEST_DEPENDS)
     get_target_property(dep_type ${dep} "TARGET_TYPE")
-    if (dep_type)
-      string(COMPARE EQUAL ${dep_type} ${ENTRYPOINT_OBJ_TARGET_TYPE} dep_is_entrypoint)
-      if(dep_is_entrypoint)
-        get_target_property(obj_file ${dep} "OBJECT_FILE_RAW")
-        list(APPEND library_deps ${obj_file})
-        continue()
-      endif()
+    if(${dep_type} STREQUAL ${ENTRYPOINT_OBJ_TARGET_TYPE})
+      get_target_property(obj_files ${dep} "OBJECT_FILES_RAW")
+      list(APPEND library_deps ${obj_files})
+    elseif(${dep_type} STREQUAL ${OBJECT_LIBRARY_TARGET_TYPE})
+      get_target_property(obj_files ${dep} "OBJECT_FILES")
+      list(APPEND library_deps ${obj_files})
     endif()
     # TODO: Check if the dep is a normal CMake library target. If yes, then add it
     # to the list of library_deps.
   endforeach(dep)
+  list(REMOVE_DUPLICATES library_deps)
 
   add_executable(
     ${target_name}
@@ -488,7 +521,7 @@ function(add_libc_fuzzer target_name)
     if (dep_type)
       string(COMPARE EQUAL ${dep_type} ${ENTRYPOINT_OBJ_TARGET_TYPE} dep_is_entrypoint)
       if(dep_is_entrypoint)
-        get_target_property(obj_file ${dep} "OBJECT_FILE_RAW")
+        get_target_property(obj_file ${dep} "OBJECT_FILES_RAW")
         list(APPEND library_deps ${obj_file})
         continue()
       endif()
diff --git a/libc/loader/CMakeLists.txt b/libc/loader/CMakeLists.txt
new file mode 100644
index 0000000000000..b4bbe81c92ff2
--- /dev/null
+++ b/libc/loader/CMakeLists.txt
@@ -0,0 +1,3 @@
+if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
+  add_subdirectory(${LIBC_TARGET_OS})
+endif()
diff --git a/libc/loader/linux/CMakeLists.txt b/libc/loader/linux/CMakeLists.txt
new file mode 100644
index 0000000000000..9df1b17f831a0
--- /dev/null
+++ b/libc/loader/linux/CMakeLists.txt
@@ -0,0 +1,34 @@
+function(add_loader_object name)
+  cmake_parse_arguments(
+    "ADD_LOADER_OBJECT"
+    ""    # No option arguments
+    "SRC" # Single value arguments
+    "DEPENDS;COMPILE_OPTIONS" # Multi value arguments
+    ${ARGN}
+  )
+  add_object_library(
+    ${name}_object
+    SRCS ${ADD_LOADER_OBJECT_SRC}
+    DEPENDS ${ADD_LOADER_OBJECT_DEPENDS}
+    COMPILE_OPTIONS ${ADD_LOADER_OBJECT_COMPILE_OPTIONS}
+  )
+
+  set(objfile ${LIBC_BUILD_DIR}/lib/${name}.o)
+  add_custom_command(
+    OUTPUT ${objfile}
+    COMMAND cp $<TARGET_OBJECTS:${name}_object> ${objfile}
+    DEPENDS $<TARGET_OBJECTS:${name}_object>
+  )
+  add_custom_target(
+    ${name}
+    DEPENDS ${objfile}
+  )
+  set_target_properties(
+    ${name}
+    PROPERTIES
+      "TARGET_TYPE" "LOADER_OBJECT"
+      "OBJECT_FILES" ${objfile}
+  )
+endfunction()
+
+add_subdirectory(${LIBC_TARGET_MACHINE})
diff --git a/libc/loader/linux/x86_64/CMakeLists.txt b/libc/loader/linux/x86_64/CMakeLists.txt
new file mode 100644
index 0000000000000..e7d7ee563701b
--- /dev/null
+++ b/libc/loader/linux/x86_64/CMakeLists.txt
@@ -0,0 +1,11 @@
+add_loader_object(
+  crt1
+  SRC
+    start.cpp
+  DEPENDS
+    linux_syscall_h
+    sys_syscall_h
+  COMPILE_OPTIONS
+    -fno-omit-frame-pointer
+    -ffreestanding # To avoid compiler warnings about calling the main function.
+)
diff --git a/libc/loader/linux/x86_64/start.cpp b/libc/loader/linux/x86_64/start.cpp
new file mode 100644
index 0000000000000..57af0b1ed528b
--- /dev/null
+++ b/libc/loader/linux/x86_64/start.cpp
@@ -0,0 +1,65 @@
+//===------------------ Implementation of crt for x86_64 ------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "config/linux/syscall.h"
+#include "include/sys/syscall.h"
+
+#include <linux/auxvec.h>
+#include <stdint.h>
+
+extern "C" int main(int, char **, char **);
+
+struct Args {
+  // At the language level, argc is an int. But we use uint64_t as the x86_64
+  // ABI specifies it as an 8 byte value.
+  uint64_t argc;
+
+  // At the language level, argv is a char** value. However, we use uint64_t as
+  // the x86_64 ABI specifies the argv vector be an |argc| long array of 8-byte
+  // values. Even though a flexible length array would be more suitable here, we
+  // set the array length to 1 to avoid a compiler warning about it being a C99
+  // extension. Length of 1 is not really wrong as |argc| is guaranteed to be
+  // atleast 1, and there is an 8-byte null entry at the end of the argv array.
+  uint64_t argv[1];
+};
+
+// TODO: Would be nice to use the aux entry structure from elf.h when available.
+struct AuxEntry {
+  uint64_t type;
+  uint64_t value;
+};
+
+extern "C" void _start() {
+  uintptr_t *frame_ptr =
+      reinterpret_cast<uintptr_t *>(__builtin_frame_address(0));
+
+  // This TU is compiled with -fno-omit-frame-pointer. Hence, the previous value
+  // of the base pointer is pushed on to the stack. So, we step over it (the
+  // "+ 1" below) to get to the args.
+  Args *args = reinterpret_cast<Args *>(frame_ptr + 1);
+
+  // After the argv array, is a 8-byte long NULL value before the array of env
+  // values. The end of the env values is marked by another 8-byte long NULL
+  // value. We step over it (the "+ 1" below) to get to the env values.
+  uint64_t *env_ptr = args->argv + args->argc + 1;
+  uint64_t *env_end_marker = env_ptr;
+  while (*env_end_marker)
+    ++env_end_marker;
+
+  // After the env array, is the aux-vector. The end of the aux-vector is
+  // denoted by an AT_NULL entry.
+  for (AuxEntry *aux_entry = reinterpret_cast<AuxEntry *>(env_end_marker + 1);
+       aux_entry->type != AT_NULL; ++aux_entry) {
+    // TODO: Read the aux vector and store necessary information in a libc wide
+    // data structure.
+  }
+
+  __llvm_libc::syscall(SYS_exit,
+                       main(args->argc, reinterpret_cast<char **>(args->argv),
+                            reinterpret_cast<char **>(env_ptr)));
+}
diff --git a/libc/spec/gnu_ext.td b/libc/spec/gnu_ext.td
new file mode 100644
index 0000000000000..7ac99783bc470
--- /dev/null
+++ b/libc/spec/gnu_ext.td
@@ -0,0 +1,19 @@
+def GnuExtensions : StandardSpec<"GNUExtensions"> {
+  HeaderSpec Math = HeaderSpec<
+      "math.h",
+      [], // Macros
+      [], // Types
+      [], // Enumerations
+      [
+        FunctionSpec<
+            "sincosf",
+            RetValSpec<VoidType>,
+            [ArgSpec<FloatType>, ArgSpec<FloatPtr>, ArgSpec<FloatPtr>]
+        >,
+      ]
+  >;
+
+  let Headers = [
+    Math,
+  ];
+}
diff --git a/libc/spec/spec.td b/libc/spec/spec.td
index 979efeaf066fc..ee04cef277b41 100644
--- a/libc/spec/spec.td
+++ b/libc/spec/spec.td
@@ -47,6 +47,7 @@ def CharType : NamedType<"char">;
 // Common types
 def VoidPtr : PtrType<VoidType>;
 def SizeTType : NamedType<"size_t">;
+def FloatPtr : PtrType<FloatType>;
 
 // _Noreturn is really not a type, but it is convenient to treat it as a type.
 def NoReturn : NamedType<"_Noreturn void">;
diff --git a/libc/src/signal/linux/CMakeLists.txt b/libc/src/signal/linux/CMakeLists.txt
index 447818e0fd208..7992703a42d26 100644
--- a/libc/src/signal/linux/CMakeLists.txt
+++ b/libc/src/signal/linux/CMakeLists.txt
@@ -12,9 +12,9 @@ add_entrypoint_object(
     signal_h
 )
 
-add_object(
+add_object_library(
   __restore
-  SRC
+  SRCS
     __restore.cpp
   COMPILE_OPTIONS
     -fomit-frame-pointer
@@ -41,8 +41,6 @@ add_entrypoint_object(
     sys_syscall_h
     linux_syscall_h
     signal_h
-  SPECIAL_OBJECTS
-    __restore
 )
 
 add_entrypoint_object(
@@ -96,3 +94,29 @@ add_entrypoint_object(
     sigaction
     signal_h
 )
+
+add_entrypoint_object(
+  sigfillset
+  SRCS
+    sigfillset.cpp
+  HDRS
+    signal.h
+    ../sigfillset.h
+  DEPENDS
+    __errno_location
+    errno_h
+    signal_h
+)
+
+add_entrypoint_object(
+  sigdelset
+  SRCS
+    sigdelset.cpp
+  HDRS
+    signal.h
+    ../sigdelset.h
+  DEPENDS
+    __errno_location
+    errno_h
+    signal_h
+)
diff --git a/libc/src/signal/linux/__restore.cpp b/libc/src/signal/linux/__restore.cpp
index bfdee4bab1c38..8d99d6abb0145 100644
--- a/libc/src/signal/linux/__restore.cpp
+++ b/libc/src/signal/linux/__restore.cpp
@@ -13,8 +13,12 @@
 #include "config/linux/syscall.h"
 #include "include/sys/syscall.h"
 
+namespace __llvm_libc {
+
 extern "C" void __restore_rt()
     __attribute__((no_sanitize("thread", "memory", "undefined", "fuzzer"),
                    hidden));
 
 extern "C" void __restore_rt() { __llvm_libc::syscall(SYS_rt_sigreturn); }
+
+} // namespace __llvm_libc
diff --git a/libc/src/signal/linux/sigdelset.cpp b/libc/src/signal/linux/sigdelset.cpp
new file mode 100644
index 0000000000000..b04ec56cc33e6
--- /dev/null
+++ b/libc/src/signal/linux/sigdelset.cpp
@@ -0,0 +1,28 @@
+//===----------------- Linux implementation of sigdelset ------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/signal/sigdelset.h"
+#include "include/errno.h"
+#include "src/errno/llvmlibc_errno.h"
+#include "src/signal/linux/signal.h"
+
+#include "src/__support/common.h"
+
+namespace __llvm_libc {
+
+int LLVM_LIBC_ENTRYPOINT(sigdelset)(sigset_t *set, int signum) {
+  if (!set || (unsigned)(signum - 1) >= (8 * sizeof(sigset_t))) {
+    llvmlibc_errno = EINVAL;
+    return -1;
+  }
+  auto *sigset = reinterpret_cast<__llvm_libc::Sigset *>(set);
+  sigset->delset(signum);
+  return 0;
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/signal/linux/sigfillset.cpp b/libc/src/signal/linux/sigfillset.cpp
new file mode 100644
index 0000000000000..6c10da3349933
--- /dev/null
+++ b/libc/src/signal/linux/sigfillset.cpp
@@ -0,0 +1,28 @@
+//===----------------- Linux implementation of sigfillset -----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/signal/sigfillset.h"
+#include "include/errno.h"
+#include "src/errno/llvmlibc_errno.h"
+#include "src/signal/linux/signal.h"
+
+#include "src/__support/common.h"
+
+namespace __llvm_libc {
+
+int LLVM_LIBC_ENTRYPOINT(sigfillset)(sigset_t *set) {
+  if (!set) {
+    llvmlibc_errno = EINVAL;
+    return -1;
+  }
+  auto *sigset = reinterpret_cast<__llvm_libc::Sigset *>(set);
+  *sigset = __llvm_libc::Sigset::fullset();
+  return 0;
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/signal/linux/signal.h b/libc/src/signal/linux/signal.h
index 93b33596580f1..9a02ea7d847da 100644
--- a/libc/src/signal/linux/signal.h
+++ b/libc/src/signal/linux/signal.h
@@ -26,9 +26,9 @@ struct Sigset {
   constexpr static Sigset fullset() { return {-1UL}; }
   constexpr static Sigset emptySet() { return {0}; }
 
-  constexpr void addset(int signal) {
-    nativeSigset |= (1L << (signal - 1));
-  }
+  constexpr void addset(int signal) { nativeSigset |= (1L << (signal - 1)); }
+
+  constexpr void delset(int signal) { nativeSigset &= ~(1L << (signal - 1)); }
 
   operator sigset_t() const { return nativeSigset; }
 };
@@ -39,16 +39,15 @@ static inline int block_all_signals(Sigset &set) {
   sigset_t nativeSigset = all;
   sigset_t oldSet = set;
   int ret = __llvm_libc::syscall(SYS_rt_sigprocmask, SIG_BLOCK, &nativeSigset,
-                              &oldSet, sizeof(sigset_t));
+                                 &oldSet, sizeof(sigset_t));
   set = {oldSet};
   return ret;
 }
 
 static inline int restore_signals(const Sigset &set) {
   sigset_t nativeSigset = set;
-  return __llvm_libc::syscall(SYS_rt_sigprocmask, SIG_SETMASK,
-                              &nativeSigset, nullptr,
-                              sizeof(sigset_t));
+  return __llvm_libc::syscall(SYS_rt_sigprocmask, SIG_SETMASK, &nativeSigset,
+                              nullptr, sizeof(sigset_t));
 }
 
 } // namespace __llvm_libc
diff --git a/libc/src/signal/sigdelset.h b/libc/src/signal/sigdelset.h
new file mode 100644
index 0000000000000..05cc47ce4b9cf
--- /dev/null
+++ b/libc/src/signal/sigdelset.h
@@ -0,0 +1,20 @@
+//===------------- Implementation header for sigdelset ---------*- C++ -*--===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_SIGNAL_SIGDELSET_H
+#define LLVM_LIBC_SRC_SIGNAL_SIGDELSET_H
+
+#include "include/signal.h"
+
+namespace __llvm_libc {
+
+int sigdelset(sigset_t *set, int signum);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_SIGNAL_SIGDELSET_H
diff --git a/libc/src/signal/sigfillset.h b/libc/src/signal/sigfillset.h
new file mode 100644
index 0000000000000..facf67919a9ff
--- /dev/null
+++ b/libc/src/signal/sigfillset.h
@@ -0,0 +1,20 @@
+//===------------- Implementation header for sigfillset --------*- C++ -*--===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_SIGNAL_SIGFILLSET_H
+#define LLVM_LIBC_SRC_SIGNAL_SIGFILLSET_H
+
+#include "include/signal.h"
+
+namespace __llvm_libc {
+
+int sigfillset(sigset_t *set);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_SIGNAL_SIGFILLSET_H
diff --git a/libc/src/string/memory_utils/memcpy_utils.h b/libc/src/string/memory_utils/memcpy_utils.h
index c69e557574c05..f6f283e3730d1 100644
--- a/libc/src/string/memory_utils/memcpy_utils.h
+++ b/libc/src/string/memory_utils/memcpy_utils.h
@@ -14,20 +14,22 @@
 
 // __builtin_memcpy_inline guarantees to never call external functions.
 // Unfortunately it is not widely available.
-#if defined(__clang__) && __has_builtin(__builtin_memcpy_inline)
+#ifdef __clang__
+#if __has_builtin(__builtin_memcpy_inline)
 #define USE_BUILTIN_MEMCPY_INLINE
+#endif
 #elif defined(__GNUC__)
 #define USE_BUILTIN_MEMCPY
 #endif
 
+namespace __llvm_libc {
+
 // This is useful for testing.
 #if defined(LLVM_LIBC_MEMCPY_MONITOR)
 extern "C" void LLVM_LIBC_MEMCPY_MONITOR(char *__restrict,
                                          const char *__restrict, size_t);
 #endif
 
-namespace __llvm_libc {
-
 // Copies `kBlockSize` bytes from `src` to `dst`.
 template <size_t kBlockSize>
 static void Copy(char *__restrict dst, const char *__restrict src) {
diff --git a/libc/src/threads/linux/thread_utils.h b/libc/src/threads/linux/thread_utils.h
index e9dfb4c29de9b..37198ad0afb49 100644
--- a/libc/src/threads/linux/thread_utils.h
+++ b/libc/src/threads/linux/thread_utils.h
@@ -14,6 +14,8 @@
 #include <stdatomic.h>
 #include <stdint.h>
 
+namespace __llvm_libc {
+
 // The futex data has to be exactly 4 bytes long. However, we use a uint type
 // here as we do not want to use `_Atomic uint32_t` as the _Atomic keyword which
 // is C only. The header stdatomic.h does not define an atomic type
@@ -35,4 +37,6 @@ struct ThreadParams {
   static constexpr uint32_t ClearTIDValue = 0xABCD1234;
 };
 
+} // namespace __llvm_libc
+
 #endif // LLVM_LIBC_SRC_THREADS_LINUX_THREAD_UTILS_H
diff --git a/libc/test/CMakeLists.txt b/libc/test/CMakeLists.txt
index 8f5f0bb6d7e6a..aa77dcd2727fd 100644
--- a/libc/test/CMakeLists.txt
+++ b/libc/test/CMakeLists.txt
@@ -1,4 +1,5 @@
 add_custom_target(check-libc)
 
 add_subdirectory(config)
+add_subdirectory(loader)
 add_subdirectory(src)
diff --git a/libc/test/loader/CMakeLists.txt b/libc/test/loader/CMakeLists.txt
new file mode 100644
index 0000000000000..2918d22193ff4
--- /dev/null
+++ b/libc/test/loader/CMakeLists.txt
@@ -0,0 +1,74 @@
+add_custom_target(libc_loader_tests)
+
+# A rule to add loader tests. When we have a complete loader, we should
+# be able to use the add_libc_unittest rule or an extension of it. But,
+# while the loader is getting built, we need to use a special rule like
+# this.
+function(add_loader_test target_name)
+  if(NOT CMAKE_HOST_UNIX)
+    message(
+        WARNING
+        "Loader tests currently assume a POSIX/Unix like environment and "
+        "may not work on your platform.")
+  endif()
+
+  cmake_parse_arguments(
+    "ADD_LOADER_TEST"
+    ""    # No option arguments
+    "SRC" # Single value arguments
+    "DEPENDS;ARGS;ENV" # Multivalue arguments.
+    ${ARGN}
+  )
+
+  add_executable(
+    ${target_name}
+    EXCLUDE_FROM_ALL
+    ${ADD_LOADER_TEST_SRC}
+  )
+
+  set_target_properties(${target_name} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+
+  set(dep_objects "")
+  if(ADD_LOADER_TEST_DEPENDS)
+    add_dependencies(${target_name} ${ADD_LOADER_TEST_DEPENDS})
+    foreach(dep IN LISTS ADD_LOADER_TEST_DEPENDS)
+      get_target_property(objfile ${dep} "OBJECT_FILES")
+      if(NOT objfile)
+        message(
+            FATAL_ERROR
+            "Unexpected dependency of an `add_loader_test` target. A dependency "
+            "should be a target of type `add_entrypoint_object, `add_object`, or "
+            "`add_loader_object`.")
+      endif()
+      list(APPEND dep_objects ${objfile})
+    endforeach(dep)
+  endif()
+
+  target_include_directories(
+    ${target_name}
+    PRIVATE
+      ${LIBC_SOURCE_DIR}
+      ${LIBC_BUILD_DIR}
+      ${LIBC_BUILD_DIR}/include
+  )
+
+  target_link_libraries(${target_name} ${dep_objects})
+
+  target_link_options(
+    ${target_name}
+    BEFORE PRIVATE
+    -nostdlib
+  )
+
+  add_custom_command(
+    TARGET ${target_name}
+    POST_BUILD
+    COMMAND ${ADD_LOADER_TEST_ENV} $<TARGET_FILE:${target_name}> ${ADD_LOADER_TEST_ARGS}
+  )
+
+  add_dependencies(libc_loader_tests ${target_name})
+endfunction(add_loader_test)
+
+if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
+  add_subdirectory(${LIBC_TARGET_OS})
+endif()
diff --git a/libc/test/loader/linux/CMakeLists.txt b/libc/test/loader/linux/CMakeLists.txt
new file mode 100644
index 0000000000000..004d1673cff7d
--- /dev/null
+++ b/libc/test/loader/linux/CMakeLists.txt
@@ -0,0 +1,32 @@
+add_loader_test(
+  loader_args_test
+  SRC
+    args_test.cpp
+  DEPENDS
+    __assert_fail
+    _Exit
+    abort
+    crt1
+    raise
+  ARGS
+    1 2 3
+  ENV
+    FRANCE=Paris
+    GERMANY=Berlin
+)
+
+add_loader_test(
+  loader_no_envp_test
+  SRC
+    main_without_envp.cpp
+  DEPENDS
+    crt1
+)
+
+add_loader_test(
+  loader_no_args_test
+  SRC
+    main_without_args.cpp
+  DEPENDS
+    crt1
+)
diff --git a/libc/test/loader/linux/args_test.cpp b/libc/test/loader/linux/args_test.cpp
new file mode 100644
index 0000000000000..a3c58526b143f
--- /dev/null
+++ b/libc/test/loader/linux/args_test.cpp
@@ -0,0 +1,40 @@
+//===----------------- Loader test to check args to main ------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#undef NDEBUG
+#include "src/assert/assert.h"
+
+static bool my_streq(const char *lhs, const char *rhs) {
+  const char *l, *r;
+  for (l = lhs, r = rhs; *l != '\0' && *r != '\0'; ++l, ++r)
+    if (*l != *r)
+      return false;
+
+  return *l == '\0' && *r == '\0';
+}
+
+int main(int argc, char **argv, char **envp) {
+  assert(argc == 4 && "Unexpected argc.");
+  assert(my_streq(argv[1], "1") && "Unexpected argv[1].");
+  assert(my_streq(argv[2], "2") && "Unexpected argv[2].");
+  assert(my_streq(argv[3], "3") && "Unexpected argv[3].");
+
+  bool found_france = false;
+  bool found_germany = false;
+  for (; *envp != nullptr; ++envp) {
+    if (my_streq(*envp, "FRANCE=Paris"))
+      found_france = true;
+    if (my_streq(*envp, "GERMANY=Berlin"))
+      found_germany = true;
+  }
+
+  assert(found_france && found_germany &&
+         "Did not find whats expected in envp.");
+
+  return 0;
+}
diff --git a/libcxx/test/std/experimental/memory/memory.polymorphic.allocator.class/nothing_to_do.pass.cpp b/libc/test/loader/linux/main_without_args.cpp
similarity index 75%
rename from libcxx/test/std/experimental/memory/memory.polymorphic.allocator.class/nothing_to_do.pass.cpp
rename to libc/test/loader/linux/main_without_args.cpp
index 796f3c353ba17..d08ee3b8b7448 100644
--- a/libcxx/test/std/experimental/memory/memory.polymorphic.allocator.class/nothing_to_do.pass.cpp
+++ b/libc/test/loader/linux/main_without_args.cpp
@@ -1,4 +1,4 @@
-//===----------------------------------------------------------------------===//
+//===----------------- Loader test for main without args ------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,6 +6,4 @@
 //
 //===----------------------------------------------------------------------===//
 
-int main(int, char**) {
-  return 0;
-}
+int main() { return 0; }
diff --git a/libc/test/loader/linux/main_without_envp.cpp b/libc/test/loader/linux/main_without_envp.cpp
new file mode 100644
index 0000000000000..ec46a64df5823
--- /dev/null
+++ b/libc/test/loader/linux/main_without_envp.cpp
@@ -0,0 +1,9 @@
+//===----------------- Loader test for main without envp ------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+int main(int argc, char **argv) { return 0; }
diff --git a/libc/test/src/signal/CMakeLists.txt b/libc/test/src/signal/CMakeLists.txt
index 3fc33c3dbecc0..67219c85227a7 100644
--- a/libc/test/src/signal/CMakeLists.txt
+++ b/libc/test/src/signal/CMakeLists.txt
@@ -23,7 +23,6 @@ add_libc_unittest(
     signal_h
     errno_h
     __errno_location
-    __restore
 )
 
 add_libc_unittest(
@@ -67,3 +66,34 @@ add_libc_unittest(
     __errno_location
     errno_h
 )
+
+add_libc_unittest(
+  sigfillset_test
+  SUITE
+    libc_signal_unittests
+  SRCS
+    sigfillset_test.cpp
+  DEPENDS
+    sigfillset
+    sigprocmask
+    signal_h
+    raise
+    errno_h
+    __errno_location
+)
+
+add_libc_unittest(
+  sigdelset_test
+  SUITE
+    libc_signal_unittests
+  SRCS
+    sigdelset_test.cpp
+  DEPENDS
+    sigdelset
+    sigfillset
+    sigprocmask
+    signal_h
+    raise
+    errno_h
+    __errno_location
+)
diff --git a/libc/test/src/signal/sigdelset_test.cpp b/libc/test/src/signal/sigdelset_test.cpp
new file mode 100644
index 0000000000000..d6e259ca4ca0d
--- /dev/null
+++ b/libc/test/src/signal/sigdelset_test.cpp
@@ -0,0 +1,36 @@
+//===--------------------- Unittests for sigdelset ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "include/errno.h"
+#include "include/signal.h"
+#include "src/signal/raise.h"
+#include "src/signal/sigdelset.h"
+#include "src/signal/sigfillset.h"
+#include "src/signal/sigprocmask.h"
+
+#include "utils/UnitTest/ErrnoSetterMatcher.h"
+#include "utils/UnitTest/Test.h"
+
+TEST(Sigdelset, Invalid) {
+  using __llvm_libc::testing::ErrnoSetterMatcher::Fails;
+  // Invalid set.
+  EXPECT_THAT(__llvm_libc::sigdelset(nullptr, SIGUSR1), Fails(EINVAL));
+
+  sigset_t set;
+  // Valid set, invalid signum.
+  EXPECT_THAT(__llvm_libc::sigdelset(&set, -1), Fails(EINVAL));
+}
+
+TEST(Sigdelset, UnblockOne) {
+  using __llvm_libc::testing::ErrnoSetterMatcher::Succeeds;
+  sigset_t set;
+  EXPECT_THAT(__llvm_libc::sigfillset(&set), Succeeds());
+  EXPECT_THAT(__llvm_libc::sigdelset(&set, SIGUSR1), Succeeds());
+  EXPECT_THAT(__llvm_libc::sigprocmask(SIG_SETMASK, &set, nullptr), Succeeds());
+  EXPECT_DEATH([] { __llvm_libc::raise(SIGUSR1); }, SIGUSR1);
+}
diff --git a/libc/test/src/signal/sigfillset_test.cpp b/libc/test/src/signal/sigfillset_test.cpp
new file mode 100644
index 0000000000000..35e6721ab6c6a
--- /dev/null
+++ b/libc/test/src/signal/sigfillset_test.cpp
@@ -0,0 +1,29 @@
+//===-------------------- Unittests for sigfillset ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "include/errno.h"
+#include "include/signal.h"
+#include "src/signal/raise.h"
+#include "src/signal/sigfillset.h"
+#include "src/signal/sigprocmask.h"
+
+#include "utils/UnitTest/ErrnoSetterMatcher.h"
+#include "utils/UnitTest/Test.h"
+
+TEST(Sigfillset, Invalid) {
+  using __llvm_libc::testing::ErrnoSetterMatcher::Fails;
+  EXPECT_THAT(__llvm_libc::sigfillset(nullptr), Fails(EINVAL));
+}
+
+TEST(Sigfillset, BlocksAll) {
+  using __llvm_libc::testing::ErrnoSetterMatcher::Succeeds;
+  sigset_t set;
+  EXPECT_THAT(__llvm_libc::sigfillset(&set), Succeeds());
+  EXPECT_THAT(__llvm_libc::sigprocmask(SIG_SETMASK, &set, nullptr), Succeeds());
+  EXPECT_EXITS([] { __llvm_libc::raise(SIGUSR1); }, 0);
+}
diff --git a/libcxx/.gitignore b/libcxx/.gitignore
index 49657ed8102fb..4b214ca4b3e63 100644
--- a/libcxx/.gitignore
+++ b/libcxx/.gitignore
@@ -22,12 +22,6 @@ var/
 .installed.cfg
 *.egg
 
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
 # Installer logs
 pip-log.txt
 pip-delete-this-directory.txt
diff --git a/libcxx/CMakeLists.txt b/libcxx/CMakeLists.txt
index 8bdb24e5fd2af..a4297002644f9 100644
--- a/libcxx/CMakeLists.txt
+++ b/libcxx/CMakeLists.txt
@@ -1,6 +1,10 @@
 # See https://libcxx.llvm.org/docs/BuildingLibcxx.html for instructions on how
 # to build libcxx with CMake.
 
+if (NOT IS_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}/../libcxxabi")
+  message(FATAL_ERROR "libc++ now requires being built in a monorepo layout with libcxxabi available")
+endif()
+
 #===============================================================================
 # Setup Project
 #===============================================================================
@@ -145,45 +149,19 @@ set(LIBCXX_LIBCPPABI_VERSION "2" CACHE STRING "Version of libc++abi's ABI to re-
                                                Note that this is not related to the version of libc++'s ABI itself!")
 
 # ABI Library options ---------------------------------------------------------
-set(LIBCXX_CXX_ABI "default" CACHE STRING
-    "Specify C++ ABI library to use.")
+set(LIBCXX_CXX_ABI "default" CACHE STRING "Specify C++ ABI library to use.")
 set(CXXABIS none default libcxxabi libcxxrt libstdc++ libsupc++ vcruntime)
 set_property(CACHE LIBCXX_CXX_ABI PROPERTY STRINGS ;${CXXABIS})
 
-# FIXME: This is a temporary hack to get the buildbots working while D63883 is in flight.
-# Without this all the bots fail while building libc++
-if (DEFINED ENV{USER})
-  if (("$ENV{USER}" STREQUAL "buildbot") OR (("$ENV{USER}" STREQUAL "llvmbb") OR ("$ENV{USER}" STREQUAL "buildslave")))
-    if (LIBCXX_CXX_ABI STREQUAL "libcxxabi" AND NOT DEFINED LIBCXX_CXX_ABI_INCLUDE_PATHS)
-      message(WARNING "OVERRIDING BUILDBOT CONFIG")
-      set(LIBCXX_CXX_ABI "default" CACHE STRING "FIXME" FORCE)
-    endif()
-  endif()
-endif()
 # Setup the default options if LIBCXX_CXX_ABI is not specified.
 if (LIBCXX_CXX_ABI STREQUAL "default")
-  find_path(
-    LIBCXX_LIBCXXABI_INCLUDES_INTERNAL cxxabi.h
-    PATHS ${LLVM_MAIN_SRC_DIR}/projects/libcxxabi/include
-          ${LLVM_MAIN_SRC_DIR}/runtimes/libcxxabi/include
-          ${LLVM_MAIN_SRC_DIR}/../libcxxabi/include
-    NO_DEFAULT_PATH
-    NO_CMAKE_FIND_ROOT_PATH
-  )
   if (LIBCXX_TARGETING_MSVC)
     # FIXME: Figure out how to configure the ABI library on Windows.
     set(LIBCXX_CXX_ABI_LIBNAME "vcruntime")
-  elseif ((NOT LIBCXX_STANDALONE_BUILD OR HAVE_LIBCXXABI) AND
-          IS_DIRECTORY "${LIBCXX_LIBCXXABI_INCLUDES_INTERNAL}")
-    set(LIBCXX_CXX_ABI_LIBNAME "libcxxabi")
-    set(LIBCXX_CXX_ABI_INCLUDE_PATHS "${LIBCXX_LIBCXXABI_INCLUDES_INTERNAL}")
-    set(LIBCXX_CXX_ABI_INTREE 1)
-  elseif (APPLE)
-    set(LIBCXX_CXX_ABI_LIBNAME "libcxxabi")
-    set(LIBCXX_CXX_ABI_SYSTEM 1)
   elseif (${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD")
     set(LIBCXX_CXX_ABI_LIBNAME "libcxxrt")
-    set(LIBCXX_CXX_ABI_INCLUDE_PATHS "/usr/include/c++/v1")
+  elseif (NOT LIBCXX_STANDALONE_BUILD OR HAVE_LIBCXXABI)
+    set(LIBCXX_CXX_ABI_LIBNAME "libcxxabi")
   else()
     set(LIBCXX_CXX_ABI_LIBNAME "default")
   endif()
@@ -229,7 +207,6 @@ option(LIBCXX_ENABLE_NEW_DELETE_DEFINITIONS
 # Build libc++abi with libunwind. We need this option to determine whether to
 # link with libunwind or libgcc_s while running the test cases.
 option(LIBCXXABI_USE_LLVM_UNWINDER "Build and use the LLVM unwinder." OFF)
-option(LIBCXXABI_ENABLE_STATIC_UNWINDER "Statically link the LLVM unwinder." OFF)
 
 # Target options --------------------------------------------------------------
 option(LIBCXX_BUILD_32_BITS "Build 32 bit libc++." ${LLVM_BUILD_32_BITS})
diff --git a/libcxx/cmake/Modules/HandleLibCXXABI.cmake b/libcxx/cmake/Modules/HandleLibCXXABI.cmake
index 2228fca1a551b..a62efc0afa952 100644
--- a/libcxx/cmake/Modules/HandleLibCXXABI.cmake
+++ b/libcxx/cmake/Modules/HandleLibCXXABI.cmake
@@ -102,24 +102,21 @@ if ("${LIBCXX_CXX_ABI_LIBNAME}" STREQUAL "libstdc++" OR
     "${_LIBSUPCXX_LIBNAME}" "${_LIBSUPCXX_LIBNAME}" "${_LIBSUPCXX_INCLUDE_FILES}" "bits"
     )
 elseif ("${LIBCXX_CXX_ABI_LIBNAME}" STREQUAL "libcxxabi")
-  if (LIBCXX_CXX_ABI_INTREE)
-    # Link against just-built "cxxabi" target.
-    set(CXXABI_SHARED_LIBNAME cxxabi_shared)
-    set(CXXABI_STATIC_LIBNAME cxxabi_static)
-  else()
-    # Assume c++abi is installed in the system, rely on -lc++abi link flag.
-    set(CXXABI_SHARED_LIBNAME "c++abi")
-    set(CXXABI_STATIC_LIBNAME "c++abi")
-  endif()
-  if (LIBCXX_CXX_ABI_SYSTEM)
-    set(HEADERS "")
+  set(LIBCXX_CXX_ABI_INCLUDE_PATHS "${LIBCXX_SOURCE_DIR}/../libcxxabi/include")
+
+  if(LIBCXX_STANDALONE_BUILD AND NOT (LIBCXX_CXX_ABI_INTREE OR HAVE_LIBCXXABI))
+    set(shared c++abi)
+    set(static c++abi)
   else()
-    set(HEADERS "cxxabi.h;__cxxabi_config.h")
+    set(shared cxxabi_shared)
+    set(static cxxabi_static)
   endif()
+
   setup_abi_lib(
     "-DLIBCXX_BUILDING_LIBCXXABI"
-    "${CXXABI_SHARED_LIBNAME}" "${CXXABI_STATIC_LIBNAME}" "${HEADERS}" "")
+    "${shared}" "${static}" "cxxabi.h;__cxxabi_config.h" "")
 elseif ("${LIBCXX_CXX_ABI_LIBNAME}" STREQUAL "libcxxrt")
+  set(LIBCXX_CXX_ABI_INCLUDE_PATHS "/usr/include/c++/v1")
   setup_abi_lib(
     "-DLIBCXXRT"
     "cxxrt" "cxxrt" "cxxabi.h;unwind.h;unwind-arm.h;unwind-itanium.h" ""
diff --git a/libcxx/docs/BuildingLibcxx.rst b/libcxx/docs/BuildingLibcxx.rst
index f2fff0e1f3049..262558e79add0 100644
--- a/libcxx/docs/BuildingLibcxx.rst
+++ b/libcxx/docs/BuildingLibcxx.rst
@@ -68,7 +68,7 @@ rebuilding LLVM as often. An out-of-tree build would look like this:
           -DLIBCXX_CXX_ABI_INCLUDE_PATHS=path/to/separate/libcxxabi/include \
           path/to/llvm-project/libcxx
   $ make
-  $ make check-libcxx # optional
+  $ make check-cxx # optional
 
 
 Experimental Support for Windows
@@ -166,7 +166,7 @@ libc++ specific options
 
 .. option:: LIBCXX_ENABLE_ASSERTIONS:BOOL
 
-  **Default**: ``ON``
+  **Default**: ``OFF``
 
   Build libc++ with assertions enabled.
 
diff --git a/libcxx/docs/TestingLibcxx.rst b/libcxx/docs/TestingLibcxx.rst
index d295d13d26f32..774003698beae 100644
--- a/libcxx/docs/TestingLibcxx.rst
+++ b/libcxx/docs/TestingLibcxx.rst
@@ -8,9 +8,9 @@ Testing libc++
 Getting Started
 ===============
 
-libc++ uses LIT to configure and run its tests. 
+libc++ uses LIT to configure and run its tests.
 
-The primary way to run the libc++ tests is by using `make check-libcxx`. 
+The primary way to run the libc++ tests is by using `make check-cxx`.
 
 However since libc++ can be used in any number of possible
 configurations it is important to customize the way LIT builds and runs
@@ -209,6 +209,29 @@ Environment Variables
   to use color diagnostic outputs from the compiler.
   Also see `color_diagnostics`.
 
+Writing Tests
+-------------
+
+When writing tests for the libc++ test suite, you should follow a few guidelines.
+This will ensure that your tests can run on a wide variety of hardware and under
+a wide variety of configurations. We have several unusual configurations such as
+building the tests on one host but running them on a different host, which add a
+few requirements to the test suite. Here's some stuff you should know:
+
+- All tests are run in a temporary directory that is unique to that test and
+  cleaned up after the test is done.
+- When a test needs data files as inputs, these data files can be saved in the
+  repository (when reasonable) and referrenced by the test as
+  ``// FILE_DEPENDENCIES: <path-to-dependencies>``. Copies of these files or
+  directories will be made available to the test in the temporary directory
+  where it is run.
+- You should never hardcode a path from the build-host in a test, because that
+  path will not necessarily be available on the host where the tests are run.
+- You should try to reduce the runtime dependencies of each test to the minimum.
+  For example, requiring Python to run a test is bad, since Python is not
+  necessarily available on all devices we may want to run the tests on (even
+  though supporting Python is probably trivial for the build-host).
+
 Benchmarks
 ==========
 
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index ca6c7c5c56754..2f375f1447746 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -270,8 +270,5 @@ if (LIBCXX_INSTALL_HEADERS)
                               -P "${CMAKE_BINARY_DIR}/cmake_install.cmake")
     # Stripping is a no-op for headers
     add_custom_target(install-${CXX_HEADER_TARGET}-stripped DEPENDS install-${CXX_HEADER_TARGET})
-
-    add_custom_target(install-libcxx-headers DEPENDS install-${CXX_HEADER_TARGET})
-    add_custom_target(install-libcxx-headers-stripped DEPENDS install-${CXX_HEADER_TARGET}-stripped)
   endif()
 endif()
diff --git a/libcxx/include/type_traits b/libcxx/include/type_traits
index 8fdf4a4939d1d..029bb4aa2ae9c 100644
--- a/libcxx/include/type_traits
+++ b/libcxx/include/type_traits
@@ -852,18 +852,6 @@ _LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_integral_v
 
 // is_floating_point
 
-#if __has_keyword(__is_floating_point)
-
-template <class _Tp>
-struct _LIBCPP_TEMPLATE_VIS is_floating_point : _BoolConstant<__is_floating_point(_Tp)> { };
-
-#if _LIBCPP_STD_VER > 14
-template <class _Tp>
-_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_floating_point_v = __is_floating_point(_Tp);
-#endif
-
-#else
-
 template <class _Tp> struct __libcpp_is_floating_point              : public false_type {};
 template <>          struct __libcpp_is_floating_point<float>       : public true_type {};
 template <>          struct __libcpp_is_floating_point<double>      : public true_type {};
@@ -878,8 +866,6 @@ _LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_floating_point_v
     = is_floating_point<_Tp>::value;
 #endif
 
-#endif // __has_keyword(__is_floating_point)
-
 // is_array
 
 #if __has_keyword(__is_array)
@@ -922,18 +908,6 @@ template <class _Tp> struct __libcpp_remove_objc_qualifiers<_Tp __autoreleasing>
 template <class _Tp> struct __libcpp_remove_objc_qualifiers<_Tp __unsafe_unretained> { typedef _Tp type; };
 #endif
 
-#if __has_keyword(__is_pointer)
-
-template<class _Tp>
-struct _LIBCPP_TEMPLATE_VIS is_pointer : _BoolConstant<__is_pointer(_Tp)> { };
-
-#if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES)
-template <class _Tp>
-_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_pointer_v = __is_pointer(_Tp);
-#endif
-
-#else // __has_keyword(__is_pointer)
-
 template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_pointer
     : public __libcpp_is_pointer<typename __libcpp_remove_objc_qualifiers<typename remove_cv<_Tp>::type>::type> {};
 
@@ -943,8 +917,6 @@ _LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_pointer_v
     = is_pointer<_Tp>::value;
 #endif
 
-#endif // __has_keyword(__is_pointer)
-
 // is_reference
 
 #if __has_keyword(__is_lvalue_reference) && \
@@ -1192,17 +1164,6 @@ _LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_enum_v
 
 // is_arithmetic
 
-#if __has_keyword(__is_arithmetic)
-
-template<class _Tp>
-struct _LIBCPP_TEMPLATE_VIS is_arithmetic : _BoolConstant<__is_arithmetic(_Tp)> { };
-
-#if _LIBCPP_STD_VER > 14 && !defined(_LIBCPP_HAS_NO_VARIABLE_TEMPLATES)
-template <class _Tp>
-_LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_arithmetic_v = __is_arithmetic(_Tp);
-#endif
-
-#else // __has_keyword(__is_arithmetic)
 
 template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_arithmetic
     : public integral_constant<bool, is_integral<_Tp>::value      ||
@@ -1214,8 +1175,6 @@ _LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_arithmetic_v
     = is_arithmetic<_Tp>::value;
 #endif
 
-#endif // __has_keyword(__is_arithmetic)
-
 // is_fundamental
 
 // In clang 9 and lower, this builtin did not work for nullptr_t. Additionally, in C++03 mode,
diff --git a/libcxx/src/CMakeLists.txt b/libcxx/src/CMakeLists.txt
index 63ca34d1ec470..6b801b4f14923 100644
--- a/libcxx/src/CMakeLists.txt
+++ b/libcxx/src/CMakeLists.txt
@@ -392,5 +392,4 @@ if (NOT CMAKE_CONFIGURATION_TYPES AND (LIBCXX_INSTALL_LIBRARY OR
                       -DCMAKE_INSTALL_COMPONENT=cxx
                       -DCMAKE_INSTALL_DO_STRIP=1
                       -P "${LIBCXX_BINARY_DIR}/cmake_install.cmake")
-    add_custom_target(install-libcxx DEPENDS install-cxx)
 endif()
diff --git a/libcxx/test/CMakeLists.txt b/libcxx/test/CMakeLists.txt
index aa0f5da2839f9..9b579f72605c6 100644
--- a/libcxx/test/CMakeLists.txt
+++ b/libcxx/test/CMakeLists.txt
@@ -46,11 +46,11 @@ if(DEFINED LIBCXXABI_ENABLE_STATIC
   message(FATAL_ERROR "LIBCXX_LINK_TESTS_WITH_SHARED_LIBCXXABI being OFF requires LIBCXXABI_ENABLE_STATIC to be ON")
 endif()
 
-if(LIBCXX_LINK_TESTS_WITH_SHARED_LIBCXX AND NOT LIBCXX_ENABLE_SHARED)
+if(LIBCXX_INCLUDE_TESTS AND LIBCXX_LINK_TESTS_WITH_SHARED_LIBCXX AND NOT LIBCXX_ENABLE_SHARED)
   message(FATAL_ERROR "LIBCXX_LINK_TESTS_WITH_SHARED_LIBCXX being ON requires LIBCXX_ENABLE_SHARED to be ON")
 endif()
 
-if(NOT LIBCXX_LINK_TESTS_WITH_SHARED_LIBCXX AND NOT LIBCXX_ENABLE_STATIC)
+if(LIBCXX_INCLUDE_TESTS AND NOT LIBCXX_LINK_TESTS_WITH_SHARED_LIBCXX AND NOT LIBCXX_ENABLE_STATIC)
   message(FATAL_ERROR "LIBCXX_LINK_TESTS_WITH_SHARED_LIBCXX being OFF requires LIBCXX_ENABLE_STATIC to be ON")
 endif()
 
@@ -113,8 +113,6 @@ if (LIBCXX_INCLUDE_TESTS)
     "Running libcxx tests"
     ${CMAKE_CURRENT_BINARY_DIR}
     DEPENDS cxx ${LIBCXX_TEST_DEPS})
-
-  add_custom_target(check-libcxx DEPENDS check-cxx)
 endif()
 
 if (LIBCXX_GENERATE_COVERAGE)
@@ -141,15 +139,6 @@ if (LIBCXX_CONFIGURE_IDE)
           ${LIBCXX_TESTS} ${LIBCXX_TEST_HEADERS} ${LIBCXX_HEADERS})
   add_dependencies(libcxx_test_objects cxx)
 
-  set(STATIC_ROOT ${LIBCXX_SOURCE_DIR}/test/std/input.output/filesystems/Inputs/static_test_env)
-  add_definitions(-DLIBCXX_FILESYSTEM_STATIC_TEST_ROOT="${STATIC_ROOT}")
-
-  set(DYNAMIC_ROOT ${LIBCXX_BINARY_DIR}/test/filesystem/Output/dynamic_env)
-  add_definitions(-DLIBCXX_FILESYSTEM_DYNAMIC_TEST_ROOT="${DYNAMIC_ROOT}")
-
-  set(DYNAMIC_HELPER "python ${LIBCXX_SOURCE_DIR}/test/support/filesystem_dynamic_test_helper.py ")
-  add_definitions(-DLIBCXX_FILESYSTEM_DYNAMIC_TEST_HELPER="${DYNAMIC_HELPER}")
-
   split_list(LIBCXX_COMPILE_FLAGS)
   split_list(LIBCXX_LINK_FLAGS)
 
diff --git a/libcxx/test/libcxx/algorithms/alg.modifying.operations/alg.random.shuffle/random_shuffle.depr_in_cxx14.fail.cpp b/libcxx/test/libcxx/algorithms/alg.modifying.operations/alg.random.shuffle/random_shuffle.depr_in_cxx14.fail.cpp
index 10f6498716b25..b67f9a8b75fa7 100644
--- a/libcxx/test/libcxx/algorithms/alg.modifying.operations/alg.random.shuffle/random_shuffle.depr_in_cxx14.fail.cpp
+++ b/libcxx/test/libcxx/algorithms/alg.modifying.operations/alg.random.shuffle/random_shuffle.depr_in_cxx14.fail.cpp
@@ -41,9 +41,9 @@ struct gen
 int main(int, char**)
 {
     int v[1] = {1};
-    std::random_shuffle(&v[0], &v[1]); // expected-error{{'random_shuffle<int *>' is deprecated}}
+    std::random_shuffle(&v[0], &v[1]); // expected-warning {{'random_shuffle<int *>' is deprecated}}
     gen r;
-    std::random_shuffle(&v[0], &v[1], r); // expected-error{{'random_shuffle<int *, gen &>' is deprecated}}
+    std::random_shuffle(&v[0], &v[1], r); // expected-warning {{'random_shuffle<int *, gen &>' is deprecated}}
 
   return 0;
 }
diff --git a/libcxx/test/libcxx/atomics/atomics.align/align.pass.sh.cpp b/libcxx/test/libcxx/atomics/atomics.align/align.pass.sh.cpp
index 16badab8490a8..48f79238c2afc 100644
--- a/libcxx/test/libcxx/atomics/atomics.align/align.pass.sh.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.align/align.pass.sh.cpp
@@ -8,8 +8,8 @@
 //
 // UNSUPPORTED: libcpp-has-no-threads, c++98, c++03
 // REQUIRES: libatomic
-// RUN: %build -latomic
-// RUN: %run
+// RUN: %{build} -latomic
+// RUN: %{run}
 //
 // GCC currently fails because it needs -fabi-version=6 to fix mangling of
 // std::atomic when used with __attribute__((vector(X))).
diff --git a/libcxx/test/libcxx/containers/sequences/vector/exception_safety_exceptions_disabled.sh.cpp b/libcxx/test/libcxx/containers/sequences/vector/exception_safety_exceptions_disabled.pass.cpp
similarity index 93%
rename from libcxx/test/libcxx/containers/sequences/vector/exception_safety_exceptions_disabled.sh.cpp
rename to libcxx/test/libcxx/containers/sequences/vector/exception_safety_exceptions_disabled.pass.cpp
index 1e9cbde0114b2..87a4ea5fdbd6a 100644
--- a/libcxx/test/libcxx/containers/sequences/vector/exception_safety_exceptions_disabled.sh.cpp
+++ b/libcxx/test/libcxx/containers/sequences/vector/exception_safety_exceptions_disabled.pass.cpp
@@ -6,8 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// RUN: %build -fno-exceptions
-// RUN: %run
+// ADDITIONAL_COMPILE_FLAGS: -fno-exceptions
 
 // UNSUPPORTED: c++98, c++03
 
@@ -43,7 +42,7 @@ int main(int, char**) {
   // have to be moved when it is resized.
   v.reserve(10);
   size_t old_cap = v.capacity();
-  for (int i = 0; i < v.capacity(); ++i) {
+  for (size_t i = 0; i < v.capacity(); ++i) {
     v.emplace_back(42);
   }
   assert(v.capacity() == old_cap);
diff --git a/libcxx/test/libcxx/depr/depr.auto.ptr/auto.ptr/auto_ptr.depr_in_cxx11.fail.cpp b/libcxx/test/libcxx/depr/depr.auto.ptr/auto.ptr/auto_ptr.depr_in_cxx11.fail.cpp
index b00390f88073e..0b7d908e2bdd5 100644
--- a/libcxx/test/libcxx/depr/depr.auto.ptr/auto.ptr/auto_ptr.depr_in_cxx11.fail.cpp
+++ b/libcxx/test/libcxx/depr/depr.auto.ptr/auto.ptr/auto_ptr.depr_in_cxx11.fail.cpp
@@ -30,9 +30,9 @@
 
 int main(int, char**)
 {
-    typedef std::auto_ptr<int> AP; // expected-error{{'auto_ptr<int>' is deprecated}}
-    typedef std::auto_ptr<void> APV; // expected-error{{'auto_ptr<void>' is deprecated}}
-    typedef std::auto_ptr_ref<int> APR; // expected-error{{'auto_ptr_ref<int>' is deprecated}}
+    typedef std::auto_ptr<int> AP; // expected-warning {{'auto_ptr<int>' is deprecated}}
+    typedef std::auto_ptr<void> APV; // expected-warning {{'auto_ptr<void>' is deprecated}}
+    typedef std::auto_ptr_ref<int> APR; // expected-warning {{'auto_ptr_ref<int>' is deprecated}}
 
   return 0;
 }
diff --git a/libcxx/test/libcxx/depr/depr.c.headers/math_h.sh.cpp b/libcxx/test/libcxx/depr/depr.c.headers/math_h.sh.cpp
index 8e0fad78db6b9..a7ae41aaa2ba9 100644
--- a/libcxx/test/libcxx/depr/depr.c.headers/math_h.sh.cpp
+++ b/libcxx/test/libcxx/depr/depr.c.headers/math_h.sh.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// RUN: %compile -fsyntax-only
+// RUN: %{compile} -fsyntax-only
 
 #ifdef _MSC_VER
 
diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.depr_in_cxx17.fail.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.depr_in_cxx17.fail.cpp
index f659f43b230eb..03cd3b7bd67eb 100644
--- a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.depr_in_cxx17.fail.cpp
+++ b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.depr_in_cxx17.fail.cpp
@@ -28,7 +28,7 @@ int main(int, char**)
   int x = 0;
   std::allocator<int> a;
 
-  int* p = a.address(x); // expected-error{{'address' is deprecated}}
+  int* p = a.address(x); // expected-warning {{'address' is deprecated}}
   (void)p;
 
   return 0;
diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx2a.fail.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx2a.fail.cpp
index db06f99f7f421..9c2e4b1cecd7e 100644
--- a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx2a.fail.cpp
+++ b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx2a.fail.cpp
@@ -8,6 +8,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 // <memory>
 
@@ -31,7 +32,7 @@
 int main(int, char**)
 {
     std::allocator<int> a;
-    a.allocate(3, nullptr); // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    a.allocate(3, nullptr); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.depr_in_cxx17.fail.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.depr_in_cxx17.fail.cpp
index 8a2090a9e8abe..03b1791d16014 100644
--- a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.depr_in_cxx17.fail.cpp
+++ b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.depr_in_cxx17.fail.cpp
@@ -25,7 +25,7 @@
 int main(int, char**)
 {
     std::allocator<int> a;
-    TEST_IGNORE_NODISCARD a.allocate(3, nullptr); // expected-error {{'allocate' is deprecated}}
+    TEST_IGNORE_NODISCARD a.allocate(3, nullptr); // expected-warning {{'allocate' is deprecated}}
 
     return 0;
 }
diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/allocator_types.depr_in_cxx17.fail.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/allocator_types.depr_in_cxx17.fail.cpp
index 907df94d779b4..c1a6333c0b070 100644
--- a/libcxx/test/libcxx/depr/depr.default.allocator/allocator_types.depr_in_cxx17.fail.cpp
+++ b/libcxx/test/libcxx/depr/depr.default.allocator/allocator_types.depr_in_cxx17.fail.cpp
@@ -42,13 +42,13 @@
 
 int main(int, char**)
 {
-    typedef std::allocator<char>::size_type AST;          // expected-error{{'size_type' is deprecated}}
-    typedef std::allocator<char>::difference_type ADT;    // expected-error{{'difference_type' is deprecated}}
-    typedef std::allocator<char>::pointer AP;             // expected-error{{'pointer' is deprecated}}
-    typedef std::allocator<char>::const_pointer ACP;      // expected-error{{'const_pointer' is deprecated}}
-    typedef std::allocator<char>::reference AR;           // expected-error{{'reference' is deprecated}}
-    typedef std::allocator<char>::const_reference ACR;    // expected-error{{'const_reference' is deprecated}}
-    typedef std::allocator<char>::rebind<int>::other ARO; // expected-error{{'rebind<int>' is deprecated}}
+    typedef std::allocator<char>::size_type AST;          // expected-warning {{'size_type' is deprecated}}
+    typedef std::allocator<char>::difference_type ADT;    // expected-warning {{'difference_type' is deprecated}}
+    typedef std::allocator<char>::pointer AP;             // expected-warning {{'pointer' is deprecated}}
+    typedef std::allocator<char>::const_pointer ACP;      // expected-warning {{'const_pointer' is deprecated}}
+    typedef std::allocator<char>::reference AR;           // expected-warning {{'reference' is deprecated}}
+    typedef std::allocator<char>::const_reference ACR;    // expected-warning {{'const_reference' is deprecated}}
+    typedef std::allocator<char>::rebind<int>::other ARO; // expected-warning {{'rebind<int>' is deprecated}}
 
   return 0;
 }
diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/allocator_void.depr_in_cxx17.fail.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/allocator_void.depr_in_cxx17.fail.cpp
index 64b8ecda8d61a..82d1777a5d2b4 100644
--- a/libcxx/test/libcxx/depr/depr.default.allocator/allocator_void.depr_in_cxx17.fail.cpp
+++ b/libcxx/test/libcxx/depr/depr.default.allocator/allocator_void.depr_in_cxx17.fail.cpp
@@ -32,9 +32,9 @@
 
 int main(int, char**)
 {
-    typedef std::allocator<void>::pointer AP;             // expected-error{{'allocator<void>' is deprecated}}
-    typedef std::allocator<void>::const_pointer ACP;      // expected-error{{'allocator<void>' is deprecated}}
-    typedef std::allocator<void>::rebind<int>::other ARO; // expected-error{{'allocator<void>' is deprecated}}
+    typedef std::allocator<void>::pointer AP;             // expected-warning {{'allocator<void>' is deprecated}}
+    typedef std::allocator<void>::const_pointer ACP;      // expected-warning {{'allocator<void>' is deprecated}}
+    typedef std::allocator<void>::rebind<int>::other ARO; // expected-warning {{'allocator<void>' is deprecated}}
 
   return 0;
 }
diff --git a/libcxx/test/libcxx/depr/depr.function.objects/adaptors.depr_in_cxx11.fail.cpp b/libcxx/test/libcxx/depr/depr.function.objects/adaptors.depr_in_cxx11.fail.cpp
index 4e9a5312c22b7..329b3acb23542 100644
--- a/libcxx/test/libcxx/depr/depr.function.objects/adaptors.depr_in_cxx11.fail.cpp
+++ b/libcxx/test/libcxx/depr/depr.function.objects/adaptors.depr_in_cxx11.fail.cpp
@@ -31,28 +31,28 @@ struct Foo {
 
 int main(int, char**)
 {
-    typedef std::pointer_to_unary_function<int, int> PUF; // expected-error{{'pointer_to_unary_function<int, int>' is deprecated}}
-    typedef std::pointer_to_binary_function<int, int, int> PBF; // expected-error{{'pointer_to_binary_function<int, int, int>' is deprecated}}
-    std::ptr_fun<int, int>(identity); // expected-error{{'ptr_fun<int, int>' is deprecated}}
-    std::ptr_fun<int, int, int>(sum); // expected-error{{'ptr_fun<int, int, int>' is deprecated}}
-
-    typedef std::mem_fun_t<int, Foo> MFT0; // expected-error{{'mem_fun_t<int, Foo>' is deprecated}}
-    typedef std::mem_fun1_t<int, Foo, int> MFT1; // expected-error{{'mem_fun1_t<int, Foo, int>' is deprecated}}
-    typedef std::const_mem_fun_t<int, Foo> CMFT0; // expected-error{{'const_mem_fun_t<int, Foo>' is deprecated}}
-    typedef std::const_mem_fun1_t<int, Foo, int> CMFT1; // expected-error{{'const_mem_fun1_t<int, Foo, int>' is deprecated}}
-    std::mem_fun<int, Foo>(&Foo::zero); // expected-error{{'mem_fun<int, Foo>' is deprecated}}
-    std::mem_fun<int, Foo, int>(&Foo::identity); // expected-error{{'mem_fun<int, Foo, int>' is deprecated}}
-    std::mem_fun<int, Foo>(&Foo::const_zero); // expected-error{{'mem_fun<int, Foo>' is deprecated}}
-    std::mem_fun<int, Foo, int>(&Foo::const_identity); // expected-error{{'mem_fun<int, Foo, int>' is deprecated}}
-
-    typedef std::mem_fun_ref_t<int, Foo> MFR0; // expected-error{{'mem_fun_ref_t<int, Foo>' is deprecated}}
-    typedef std::mem_fun1_ref_t<int, Foo, int> MFR1; // expected-error{{'mem_fun1_ref_t<int, Foo, int>' is deprecated}}
-    typedef std::const_mem_fun_ref_t<int, Foo> CMFR0; // expected-error{{'const_mem_fun_ref_t<int, Foo>' is deprecated}}
-    typedef std::const_mem_fun1_ref_t<int, Foo, int> CMFR1; // expected-error{{'const_mem_fun1_ref_t<int, Foo, int>' is deprecated}}
-    std::mem_fun_ref<int, Foo>(&Foo::zero); // expected-error{{'mem_fun_ref<int, Foo>' is deprecated}}
-    std::mem_fun_ref<int, Foo, int>(&Foo::identity); // expected-error{{'mem_fun_ref<int, Foo, int>' is deprecated}}
-    std::mem_fun_ref<int, Foo>(&Foo::const_zero); // expected-error{{'mem_fun_ref<int, Foo>' is deprecated}}
-    std::mem_fun_ref<int, Foo, int>(&Foo::const_identity); // expected-error{{'mem_fun_ref<int, Foo, int>' is deprecated}}
+    typedef std::pointer_to_unary_function<int, int> PUF; // expected-warning {{'pointer_to_unary_function<int, int>' is deprecated}}
+    typedef std::pointer_to_binary_function<int, int, int> PBF; // expected-warning {{'pointer_to_binary_function<int, int, int>' is deprecated}}
+    std::ptr_fun<int, int>(identity); // expected-warning {{'ptr_fun<int, int>' is deprecated}}
+    std::ptr_fun<int, int, int>(sum); // expected-warning {{'ptr_fun<int, int, int>' is deprecated}}
+
+    typedef std::mem_fun_t<int, Foo> MFT0; // expected-warning {{'mem_fun_t<int, Foo>' is deprecated}}
+    typedef std::mem_fun1_t<int, Foo, int> MFT1; // expected-warning {{'mem_fun1_t<int, Foo, int>' is deprecated}}
+    typedef std::const_mem_fun_t<int, Foo> CMFT0; // expected-warning {{'const_mem_fun_t<int, Foo>' is deprecated}}
+    typedef std::const_mem_fun1_t<int, Foo, int> CMFT1; // expected-warning {{'const_mem_fun1_t<int, Foo, int>' is deprecated}}
+    std::mem_fun<int, Foo>(&Foo::zero); // expected-warning {{'mem_fun<int, Foo>' is deprecated}}
+    std::mem_fun<int, Foo, int>(&Foo::identity); // expected-warning {{'mem_fun<int, Foo, int>' is deprecated}}
+    std::mem_fun<int, Foo>(&Foo::const_zero); // expected-warning {{'mem_fun<int, Foo>' is deprecated}}
+    std::mem_fun<int, Foo, int>(&Foo::const_identity); // expected-warning {{'mem_fun<int, Foo, int>' is deprecated}}
+
+    typedef std::mem_fun_ref_t<int, Foo> MFR0; // expected-warning {{'mem_fun_ref_t<int, Foo>' is deprecated}}
+    typedef std::mem_fun1_ref_t<int, Foo, int> MFR1; // expected-warning {{'mem_fun1_ref_t<int, Foo, int>' is deprecated}}
+    typedef std::const_mem_fun_ref_t<int, Foo> CMFR0; // expected-warning {{'const_mem_fun_ref_t<int, Foo>' is deprecated}}
+    typedef std::const_mem_fun1_ref_t<int, Foo, int> CMFR1; // expected-warning {{'const_mem_fun1_ref_t<int, Foo, int>' is deprecated}}
+    std::mem_fun_ref<int, Foo>(&Foo::zero); // expected-warning {{'mem_fun_ref<int, Foo>' is deprecated}}
+    std::mem_fun_ref<int, Foo, int>(&Foo::identity); // expected-warning {{'mem_fun_ref<int, Foo, int>' is deprecated}}
+    std::mem_fun_ref<int, Foo>(&Foo::const_zero); // expected-warning {{'mem_fun_ref<int, Foo>' is deprecated}}
+    std::mem_fun_ref<int, Foo, int>(&Foo::const_identity); // expected-warning {{'mem_fun_ref<int, Foo, int>' is deprecated}}
 
   return 0;
 }
diff --git a/libcxx/test/libcxx/diagnostics/enable_nodiscard.fail.cpp b/libcxx/test/libcxx/diagnostics/enable_nodiscard.fail.cpp
index b42e4c4e962f0..d579a4ccea8c7 100644
--- a/libcxx/test/libcxx/diagnostics/enable_nodiscard.fail.cpp
+++ b/libcxx/test/libcxx/diagnostics/enable_nodiscard.fail.cpp
@@ -16,6 +16,8 @@
 // GCC 7 is the first version to introduce [[nodiscard]]
 // UNSUPPORTED: gcc-5, gcc-6
 
+// REQUIRES: verify-support
+
 // MODULES_DEFINES: _LIBCPP_ENABLE_NODISCARD
 #define _LIBCPP_ENABLE_NODISCARD
 
@@ -25,8 +27,8 @@ _LIBCPP_NODISCARD_EXT int foo() { return 42; }
 _LIBCPP_NODISCARD_AFTER_CXX17 int bar() { return 42; }
 
 int main(int, char**) {
-  foo(); // expected-error-re {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
-  bar(); // expected-error-re {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  foo(); // expected-warning-re {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  bar(); // expected-warning-re {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   (void)foo(); // OK. void casts disable the diagnostic.
   (void)bar();
 
diff --git a/libcxx/test/libcxx/diagnostics/enable_nodiscard_disable_after_cxx17.fail.cpp b/libcxx/test/libcxx/diagnostics/enable_nodiscard_disable_after_cxx17.fail.cpp
index 72fbc01198efa..f4f74de52a106 100644
--- a/libcxx/test/libcxx/diagnostics/enable_nodiscard_disable_after_cxx17.fail.cpp
+++ b/libcxx/test/libcxx/diagnostics/enable_nodiscard_disable_after_cxx17.fail.cpp
@@ -8,6 +8,7 @@
 //===----------------------------------------------------------------------===//
 
 // UNSUPPORTED: c++98, c++03
+// REQUIRES: verify-support
 
 // GCC 7 is the first version to introduce [[nodiscard]]
 // UNSUPPORTED: gcc-5, gcc-6
@@ -26,7 +27,7 @@ _LIBCPP_NODISCARD_EXT int foo() { return 42; }
 _LIBCPP_NODISCARD_AFTER_CXX17 int bar() { return 42; }
 
 int main(int, char**) {
-  foo(); // expected-error-re {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  foo(); // expected-warning-re{{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   bar(); // OK.
   (void)foo(); // OK.
 
diff --git a/libcxx/test/libcxx/diagnostics/enable_nodiscard_disable_nodiscard_ext.fail.cpp b/libcxx/test/libcxx/diagnostics/enable_nodiscard_disable_nodiscard_ext.fail.cpp
index 65e5e1521331f..c2f29387b3bee 100644
--- a/libcxx/test/libcxx/diagnostics/enable_nodiscard_disable_nodiscard_ext.fail.cpp
+++ b/libcxx/test/libcxx/diagnostics/enable_nodiscard_disable_nodiscard_ext.fail.cpp
@@ -8,6 +8,7 @@
 //===----------------------------------------------------------------------===//
 
 // UNSUPPORTED: c++98, c++03
+// REQUIRES: verify-support
 
 // GCC 7 is the first version to introduce [[nodiscard]]
 // UNSUPPORTED: gcc-5, gcc-6
@@ -24,7 +25,7 @@ _LIBCPP_NODISCARD_EXT int foo() { return 42; }
 _LIBCPP_NODISCARD_AFTER_CXX17 int bar() { return 42; }
 
 int main(int, char**) {
-  bar(); // expected-error-re {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  bar(); // expected-warning-re{{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   foo(); // OK.
   (void)bar(); // OK.
 
diff --git a/libcxx/test/libcxx/diagnostics/nodiscard_aftercxx17.fail.cpp b/libcxx/test/libcxx/diagnostics/nodiscard_aftercxx17.fail.cpp
index 8cfbc3117853b..39861eddd85ca 100644
--- a/libcxx/test/libcxx/diagnostics/nodiscard_aftercxx17.fail.cpp
+++ b/libcxx/test/libcxx/diagnostics/nodiscard_aftercxx17.fail.cpp
@@ -11,6 +11,7 @@
 //	#define _LIBCPP_NODISCARD_AFTER_CXX17 [[nodiscard]]
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
+// REQUIRES: verify-support
 
 #include <__config>
 
@@ -18,7 +19,7 @@ _LIBCPP_NODISCARD_AFTER_CXX17 int foo() { return 6; }
 
 int main(int, char**)
 {
-	foo();	// expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    foo(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/libcxx/diagnostics/nodiscard_extensions.fail.cpp b/libcxx/test/libcxx/diagnostics/nodiscard_extensions.fail.cpp
index dd02a3dd25db6..ef6e1fedfb88a 100644
--- a/libcxx/test/libcxx/diagnostics/nodiscard_extensions.fail.cpp
+++ b/libcxx/test/libcxx/diagnostics/nodiscard_extensions.fail.cpp
@@ -8,9 +8,7 @@
 //===----------------------------------------------------------------------===//
 
 // UNSUPPORTED: c++98, c++03
-
-// GCC versions prior to 7.0 don't provide the required [[nodiscard]] attribute.
-// UNSUPPORTED: gcc-5, gcc-6
+// REQUIRES: verify-support
 
 // AppleClang9 doesn't yet support C++17's implicitly synthesized deduction
 // guides from existing ctors, needed by default_searcher() below.
@@ -40,260 +38,260 @@ struct P {
 int main(int, char**) {
   int arr[1] = { 1 };
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::adjacent_find(std::begin(arr), std::end(arr));
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::adjacent_find(std::begin(arr), std::end(arr), std::greater<int>());
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::all_of(std::begin(arr), std::end(arr), P());
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::any_of(std::begin(arr), std::end(arr), P());
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::binary_search(std::begin(arr), std::end(arr), 1);
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::binary_search(std::begin(arr), std::end(arr), 1, std::greater<int>());
 
 #if TEST_STD_VER >= 17
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::clamp(2, 1, 3);
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::clamp(2, 1, 3, std::greater<int>());
 #endif
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::count_if(std::begin(arr), std::end(arr), P());
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::count(std::begin(arr), std::end(arr), 1);
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::equal_range(std::begin(arr), std::end(arr), 1);
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::equal_range(std::begin(arr), std::end(arr), 1, std::greater<int>());
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::equal(std::begin(arr), std::end(arr), std::begin(arr));
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::equal(std::begin(arr), std::end(arr), std::begin(arr),
              std::greater<int>());
 
 #if TEST_STD_VER >= 14
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::equal(std::begin(arr), std::end(arr), std::begin(arr), std::end(arr));
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::equal(std::begin(arr), std::end(arr), std::begin(arr), std::end(arr),
              std::greater<int>());
 #endif
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::find_end(std::begin(arr), std::end(arr), std::begin(arr), std::end(arr));
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::find_end(std::begin(arr), std::end(arr), std::begin(arr), std::end(arr),
                 std::greater<int>());
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::find_first_of(std::begin(arr), std::end(arr), std::begin(arr),
                      std::end(arr));
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::find_first_of(std::begin(arr), std::end(arr), std::begin(arr),
                      std::end(arr), std::greater<int>());
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::find_if_not(std::begin(arr), std::end(arr), P());
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::find_if(std::begin(arr), std::end(arr), P());
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::find(std::begin(arr), std::end(arr), 1);
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::get_temporary_buffer<int>(1);
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::includes(std::begin(arr), std::end(arr), std::begin(arr), std::end(arr));
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::includes(std::begin(arr), std::end(arr), std::begin(arr), std::end(arr),
                 std::greater<int>());
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::is_heap_until(std::begin(arr), std::end(arr));
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::is_heap_until(std::begin(arr), std::end(arr), std::greater<int>());
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::is_heap(std::begin(arr), std::end(arr));
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::is_heap(std::begin(arr), std::end(arr), std::greater<int>());
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::is_partitioned(std::begin(arr), std::end(arr), P());
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::is_permutation(std::begin(arr), std::end(arr), std::begin(arr));
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::is_permutation(std::begin(arr), std::end(arr), std::begin(arr),
                       std::greater<int>());
 
 #if TEST_STD_VER >= 14
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::is_permutation(std::begin(arr), std::end(arr), std::begin(arr),
                       std::end(arr));
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::is_permutation(std::begin(arr), std::end(arr), std::begin(arr),
                       std::end(arr), std::greater<int>());
 #endif
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::is_sorted_until(std::begin(arr), std::end(arr));
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::is_sorted_until(std::begin(arr), std::end(arr), std::greater<int>());
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::is_sorted(std::begin(arr), std::end(arr));
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::is_sorted(std::begin(arr), std::end(arr), std::greater<int>());
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::lexicographical_compare(std::begin(arr), std::end(arr), std::begin(arr),
                                std::end(arr));
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::lexicographical_compare(std::begin(arr), std::end(arr), std::begin(arr),
                                std::end(arr), std::greater<int>());
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::lower_bound(std::begin(arr), std::end(arr), 1);
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::lower_bound(std::begin(arr), std::end(arr), 1, std::greater<int>());
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::max_element(std::begin(arr), std::end(arr));
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::max_element(std::begin(arr), std::end(arr), std::greater<int>());
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::max(1, 2);
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::max(1, 2, std::greater<int>());
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::max({1, 2, 3});
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::max({1, 2, 3}, std::greater<int>());
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::min_element(std::begin(arr), std::end(arr));
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::min_element(std::begin(arr), std::end(arr), std::greater<int>());
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::min(1, 2);
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::min(1, 2, std::greater<int>());
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::min({1, 2, 3});
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::min({1, 2, 3}, std::greater<int>());
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::minmax_element(std::begin(arr), std::end(arr));
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::minmax_element(std::begin(arr), std::end(arr), std::greater<int>());
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::minmax(1, 2);
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::minmax(1, 2, std::greater<int>());
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::minmax({1, 2, 3});
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::minmax({1, 2, 3}, std::greater<int>());
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::mismatch(std::begin(arr), std::end(arr), std::begin(arr));
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::mismatch(std::begin(arr), std::end(arr), std::begin(arr),
                 std::greater<int>());
 
 #if TEST_STD_VER >= 14
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::mismatch(std::begin(arr), std::end(arr), std::begin(arr), std::end(arr));
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::mismatch(std::begin(arr), std::end(arr), std::begin(arr), std::end(arr),
                 std::greater<int>());
 #endif
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::none_of(std::begin(arr), std::end(arr), P());
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::remove_if(std::begin(arr), std::end(arr), P());
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::remove(std::begin(arr), std::end(arr), 1);
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::search_n(std::begin(arr), std::end(arr), 1, 1);
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::search_n(std::begin(arr), std::end(arr), 1, 1, std::greater<int>());
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::search(std::begin(arr), std::end(arr), std::begin(arr), std::end(arr));
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::search(std::begin(arr), std::end(arr), std::begin(arr), std::end(arr),
               std::greater<int>());
 
 #if TEST_STD_VER >= 17
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::search(std::begin(arr), std::end(arr),
               std::default_searcher(std::begin(arr), std::end(arr)));
 #endif
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::unique(std::begin(arr), std::end(arr));
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::unique(std::begin(arr), std::end(arr), std::greater<int>());
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::upper_bound(std::begin(arr), std::end(arr), 1);
 
-  // expected-error-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
+  // expected-warning-re@+1 {{ignoring return value of function declared with {{'nodiscard'|warn_unused_result}} attribute}}
   std::upper_bound(std::begin(arr), std::end(arr), 1, std::greater<int>());
 
   return 0;
diff --git a/libcxx/test/libcxx/double_include.sh.cpp b/libcxx/test/libcxx/double_include.sh.cpp
index 42f6376142f25..17236df3cda78 100644
--- a/libcxx/test/libcxx/double_include.sh.cpp
+++ b/libcxx/test/libcxx/double_include.sh.cpp
@@ -9,10 +9,11 @@
 
 // Test that we can include each header in two TU's and link them together.
 
-// RUN: %cxx -c %s -o %t.first.o %flags %compile_flags
-// RUN: %cxx -c %s -o %t.second.o -DWITH_MAIN %flags %compile_flags
-// RUN: %cxx -o %t.exe %t.first.o %t.second.o %flags %link_flags
-// RUN: %run
+// FILE_DEPENDENCIES: %t.exe
+// RUN: %{cxx} -c %s -o %t.first.o %{flags} %{compile_flags}
+// RUN: %{cxx} -c %s -o %t.second.o -DWITH_MAIN %{flags} %{compile_flags}
+// RUN: %{cxx} -o %t.exe %t.first.o %t.second.o %{flags} %{link_flags}
+// RUN: %{run}
 
 // Prevent <ext/hash_map> from generating deprecated warnings for this test.
 #if defined(__DEPRECATED)
diff --git a/libcxx/test/libcxx/experimental/filesystem/deprecated.fail.cpp b/libcxx/test/libcxx/experimental/filesystem/deprecated.fail.cpp
index 0c583ad90aa9c..4497bc2faf6a3 100644
--- a/libcxx/test/libcxx/experimental/filesystem/deprecated.fail.cpp
+++ b/libcxx/test/libcxx/experimental/filesystem/deprecated.fail.cpp
@@ -13,7 +13,7 @@
 
 #include <experimental/filesystem>
 
-using namespace std::experimental::filesystem; // expected-error {{'filesystem' is deprecated: std::experimental::filesystem has now been deprecated in favor of C++17's std::filesystem. Please stop using it and start using std::filesystem. This experimental version will be removed in LLVM 11. You can remove this warning by defining the _LIBCPP_NO_EXPERIMENTAL_DEPRECATION_WARNING_FILESYSTEM macro.}}
+using namespace std::experimental::filesystem; // expected-warning {{'filesystem' is deprecated: std::experimental::filesystem has now been deprecated in favor of C++17's std::filesystem. Please stop using it and start using std::filesystem. This experimental version will be removed in LLVM 11. You can remove this warning by defining the _LIBCPP_NO_EXPERIMENTAL_DEPRECATION_WARNING_FILESYSTEM macro.}}
 
 int main(int, char**) {
   return 0;
diff --git a/libcxx/test/libcxx/experimental/language.support/support.coroutines/dialect_support.sh.cpp b/libcxx/test/libcxx/experimental/language.support/support.coroutines/dialect_support.pass.cpp
similarity index 97%
rename from libcxx/test/libcxx/experimental/language.support/support.coroutines/dialect_support.sh.cpp
rename to libcxx/test/libcxx/experimental/language.support/support.coroutines/dialect_support.pass.cpp
index 237fbc4bbd615..1699d6714b22b 100644
--- a/libcxx/test/libcxx/experimental/language.support/support.coroutines/dialect_support.sh.cpp
+++ b/libcxx/test/libcxx/experimental/language.support/support.coroutines/dialect_support.pass.cpp
@@ -8,9 +8,7 @@
 //===----------------------------------------------------------------------===//
 
 // REQUIRES: fcoroutines-ts
-
-// RUN: %build -fcoroutines-ts
-// RUN: %run
+// ADDITIONAL_COMPILE_FLAGS: -fcoroutines-ts
 
 // A simple "breathing" test that checks that <experimental/coroutine>
 // can be parsed and used in all dialects, including C++03 in order to match
diff --git a/libcxx/test/libcxx/experimental/language.support/support.coroutines/version.sh.cpp b/libcxx/test/libcxx/experimental/language.support/support.coroutines/version.pass.cpp
similarity index 92%
rename from libcxx/test/libcxx/experimental/language.support/support.coroutines/version.sh.cpp
rename to libcxx/test/libcxx/experimental/language.support/support.coroutines/version.pass.cpp
index b11ea931f9212..7a09406407bf0 100644
--- a/libcxx/test/libcxx/experimental/language.support/support.coroutines/version.sh.cpp
+++ b/libcxx/test/libcxx/experimental/language.support/support.coroutines/version.pass.cpp
@@ -9,9 +9,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11
 // REQUIRES: fcoroutines-ts
-
-// RUN: %build -fcoroutines-ts
-// RUN: %run
+// ADDITIONAL_COMPILE_FLAGS: -fcoroutines-ts
 
 #include <experimental/coroutine>
 
diff --git a/libcxx/test/libcxx/include_as_c.sh.cpp b/libcxx/test/libcxx/include_as_c.sh.cpp
index 67d5e14b3ec18..c001c63e83592 100644
--- a/libcxx/test/libcxx/include_as_c.sh.cpp
+++ b/libcxx/test/libcxx/include_as_c.sh.cpp
@@ -12,7 +12,7 @@
 // NOTE: It's not common or recommended to have libc++ in the header search
 // path when compiling C files, but it does happen often enough.
 
-// RUN: %cxx -c -xc %s -fsyntax-only %flags %compile_flags -std=c99
+// RUN: %{cxx} -c -xc %s -fsyntax-only %{flags} %{compile_flags} -std=c99
 
 #include <complex.h>
 #include <ctype.h>
diff --git a/libcxx/test/libcxx/input.output/filesystems/class.directory_entry/directory_entry.mods/last_write_time.sh.cpp b/libcxx/test/libcxx/input.output/filesystems/class.directory_entry/directory_entry.mods/last_write_time.pass.cpp
similarity index 97%
rename from libcxx/test/libcxx/input.output/filesystems/class.directory_entry/directory_entry.mods/last_write_time.sh.cpp
rename to libcxx/test/libcxx/input.output/filesystems/class.directory_entry/directory_entry.mods/last_write_time.pass.cpp
index 17b654e221f2e..e4c5a1892972b 100644
--- a/libcxx/test/libcxx/input.output/filesystems/class.directory_entry/directory_entry.mods/last_write_time.sh.cpp
+++ b/libcxx/test/libcxx/input.output/filesystems/class.directory_entry/directory_entry.mods/last_write_time.pass.cpp
@@ -7,14 +7,12 @@
 //===----------------------------------------------------------------------===//
 
 // UNSUPPORTED: c++98, c++03
+// ADDITIONAL_COMPILE_FLAGS: -I%{libcxx_src_root}/src/filesystem
 
 // <filesystem>
 
 // class directory_entry
 
-// RUN: %build -I%libcxx_src_root/src/filesystem
-// RUN: %run
-
 #include "filesystem_include.h"
 #include <type_traits>
 #include <cassert>
diff --git a/libcxx/test/libcxx/input.output/filesystems/class.path/path.itr/iterator_db.pass.cpp b/libcxx/test/libcxx/input.output/filesystems/class.path/path.itr/iterator_db.pass.cpp
index d5df253603c1a..6fa9d2c274bde 100644
--- a/libcxx/test/libcxx/input.output/filesystems/class.path/path.itr/iterator_db.pass.cpp
+++ b/libcxx/test/libcxx/input.output/filesystems/class.path/path.itr/iterator_db.pass.cpp
@@ -25,7 +25,6 @@
 #include <cassert>
 
 #include "test_macros.h"
-#include "filesystem_test_helper.h"
 #include "debug_mode_helper.h"
 
 int main(int, char**) {
diff --git a/libcxx/test/libcxx/input.output/filesystems/convert_file_time.sh.cpp b/libcxx/test/libcxx/input.output/filesystems/convert_file_time.pass.cpp
similarity index 99%
rename from libcxx/test/libcxx/input.output/filesystems/convert_file_time.sh.cpp
rename to libcxx/test/libcxx/input.output/filesystems/convert_file_time.pass.cpp
index 55cd659cc4d13..d00f40fa47665 100644
--- a/libcxx/test/libcxx/input.output/filesystems/convert_file_time.sh.cpp
+++ b/libcxx/test/libcxx/input.output/filesystems/convert_file_time.pass.cpp
@@ -12,8 +12,7 @@
 
 // typedef TrivialClock file_time_type;
 
-// RUN: %build -I%libcxx_src_root/src/filesystem
-// RUN: %run
+// ADDITIONAL_COMPILE_FLAGS: -I%{libcxx_src_root}/src/filesystem
 
 #include <filesystem>
 #include <chrono>
diff --git a/libcxx/test/libcxx/input.output/filesystems/lit.local.cfg b/libcxx/test/libcxx/input.output/filesystems/lit.local.cfg
index 132b73ccbde55..4eadb33e04d08 100644
--- a/libcxx/test/libcxx/input.output/filesystems/lit.local.cfg
+++ b/libcxx/test/libcxx/input.output/filesystems/lit.local.cfg
@@ -1,5 +1,4 @@
-# Disable all of the filesystem tests if the dylib under test doesn't support them.
-if 'dylib-has-no-filesystem' in config.available_features:
-  config.unsupported = True
-if 'c++filesystem-disabled' in config.available_features:
-  config.unsupported = True
+# Load the same local configuration as the filesystem tests in libcxx/test/std
+import os
+std_filesystem_tests = os.path.join(config.test_source_root, 'std', 'input.output', 'filesystems')
+config.load_from_path(os.path.join(std_filesystem_tests, 'lit.local.cfg'), lit_config)
diff --git a/libcxx/test/libcxx/language.support/support.dynamic/libcpp_deallocate.sh.cpp b/libcxx/test/libcxx/language.support/support.dynamic/libcpp_deallocate.sh.cpp
index 970b7fcf88068..da5f3b6b12b43 100644
--- a/libcxx/test/libcxx/language.support/support.dynamic/libcpp_deallocate.sh.cpp
+++ b/libcxx/test/libcxx/language.support/support.dynamic/libcpp_deallocate.sh.cpp
@@ -39,14 +39,15 @@
 // GCC doesn't support the aligned-allocation flags.
 // XFAIL: gcc
 
-// RUN: %build -faligned-allocation -fsized-deallocation
-// RUN: %run
-// RUN: %build -faligned-allocation -fno-sized-deallocation -DNO_SIZE
-// RUN: %run
-// RUN: %build -fno-aligned-allocation -fsized-deallocation -DNO_ALIGN
-// RUN: %run
-// RUN: %build -fno-aligned-allocation -fno-sized-deallocation -DNO_ALIGN -DNO_SIZE
-// RUN: %run
+// FILE_DEPENDENCIES: %t.exe
+// RUN: %{build} -faligned-allocation -fsized-deallocation
+// RUN: %{run}
+// RUN: %{build} -faligned-allocation -fno-sized-deallocation -DNO_SIZE
+// RUN: %{run}
+// RUN: %{build} -fno-aligned-allocation -fsized-deallocation -DNO_ALIGN
+// RUN: %{run}
+// RUN: %{build} -fno-aligned-allocation -fno-sized-deallocation -DNO_ALIGN -DNO_SIZE
+// RUN: %{run}
 
 #include <new>
 #include <typeinfo>
diff --git a/libcxx/test/libcxx/language.support/support.dynamic/new_faligned_allocation.sh.cpp b/libcxx/test/libcxx/language.support/support.dynamic/new_faligned_allocation.pass.cpp
similarity index 98%
rename from libcxx/test/libcxx/language.support/support.dynamic/new_faligned_allocation.sh.cpp
rename to libcxx/test/libcxx/language.support/support.dynamic/new_faligned_allocation.pass.cpp
index bfc26df704d97..e79d2e699af60 100644
--- a/libcxx/test/libcxx/language.support/support.dynamic/new_faligned_allocation.sh.cpp
+++ b/libcxx/test/libcxx/language.support/support.dynamic/new_faligned_allocation.pass.cpp
@@ -13,8 +13,6 @@
 // definitions, which does not yet provide aligned allocation
 // XFAIL: LIBCXX-WINDOWS-FIXME
 
-// REQUIRES: -faligned-allocation
-
 // The dylibs shipped before macosx10.13 do not contain the aligned allocation
 // functions, so trying to force using those with -faligned-allocation results
 // in a link error.
@@ -25,8 +23,8 @@
 // XFAIL: with_system_cxx_lib=macosx10.8
 // XFAIL: with_system_cxx_lib=macosx10.7
 
-// RUN: %build -faligned-allocation
-// RUN: %run
+// REQUIRES: -faligned-allocation
+// ADDITIONAL_COMPILE_FLAGS: -faligned-allocation
 
 #include <new>
 #include <typeinfo>
diff --git a/libcxx/test/libcxx/min_max_macros.sh.cpp b/libcxx/test/libcxx/min_max_macros.sh.cpp
index 47c8a09e991b4..4e09e8ee42d1c 100644
--- a/libcxx/test/libcxx/min_max_macros.sh.cpp
+++ b/libcxx/test/libcxx/min_max_macros.sh.cpp
@@ -9,7 +9,7 @@
 
 // Test that we can include each header in two TU's and link them together.
 
-// RUN: %compile -fsyntax-only
+// RUN: %{compile} -fsyntax-only
 
 // Prevent <ext/hash_map> from generating deprecated warnings for this test.
 #if defined(__DEPRECATED)
diff --git a/libcxx/test/libcxx/modules/cinttypes_exports.sh.cpp b/libcxx/test/libcxx/modules/cinttypes_exports.sh.cpp
index 70477d97678ac..135fb834030f3 100644
--- a/libcxx/test/libcxx/modules/cinttypes_exports.sh.cpp
+++ b/libcxx/test/libcxx/modules/cinttypes_exports.sh.cpp
@@ -18,7 +18,7 @@
 
 // Test that <cinttypes> re-exports <cstdint>
 
-// RUN: %build_module
+// RUN: %{build_module}
 
 #include <cinttypes>
 
diff --git a/libcxx/test/libcxx/modules/clocale_exports.sh.cpp b/libcxx/test/libcxx/modules/clocale_exports.sh.cpp
index 6c4bc2675bb9e..29086de4f614d 100644
--- a/libcxx/test/libcxx/modules/clocale_exports.sh.cpp
+++ b/libcxx/test/libcxx/modules/clocale_exports.sh.cpp
@@ -17,7 +17,7 @@
 // REQUIRES: modules-support
 // UNSUPPORTED: c++98, c++03
 
-// RUN: %build_module
+// RUN: %{build_module}
 
 #include <clocale>
 
diff --git a/libcxx/test/libcxx/modules/cstdint_exports.sh.cpp b/libcxx/test/libcxx/modules/cstdint_exports.sh.cpp
index c473245fe64ff..3ac9999bef2ad 100644
--- a/libcxx/test/libcxx/modules/cstdint_exports.sh.cpp
+++ b/libcxx/test/libcxx/modules/cstdint_exports.sh.cpp
@@ -18,7 +18,7 @@
 
 // Test that <cstdint> re-exports <stdint.h>
 
-// RUN: %build_module
+// RUN: %{build_module}
 
 #include <cstdint>
 
diff --git a/libcxx/test/libcxx/modules/inttypes_h_exports.sh.cpp b/libcxx/test/libcxx/modules/inttypes_h_exports.sh.cpp
index a653b05932778..828635a7f1c16 100644
--- a/libcxx/test/libcxx/modules/inttypes_h_exports.sh.cpp
+++ b/libcxx/test/libcxx/modules/inttypes_h_exports.sh.cpp
@@ -18,7 +18,7 @@
 
 // Test that intypes.h re-exports stdint.h
 
-// RUN: %build_module
+// RUN: %{build_module}
 
 #include <inttypes.h>
 
diff --git a/libcxx/test/libcxx/modules/stdint_h_exports.sh.cpp b/libcxx/test/libcxx/modules/stdint_h_exports.sh.cpp
index ec86ad827dd88..f38e5f470faab 100644
--- a/libcxx/test/libcxx/modules/stdint_h_exports.sh.cpp
+++ b/libcxx/test/libcxx/modules/stdint_h_exports.sh.cpp
@@ -14,7 +14,7 @@
 
 // Test that int8_t and the like are exported from stdint.h not inttypes.h
 
-// RUN: %build_module
+// RUN: %{build_module}
 
 #include <stdint.h>
 
diff --git a/libcxx/test/libcxx/modules/stds_include.sh.cpp b/libcxx/test/libcxx/modules/stds_include.sh.cpp
index 35c9eee2be871..c71fb9ab978ec 100644
--- a/libcxx/test/libcxx/modules/stds_include.sh.cpp
+++ b/libcxx/test/libcxx/modules/stds_include.sh.cpp
@@ -17,13 +17,13 @@
 
 // REQUIRES: modules-support
 
-// NOTE: The -std=XXX flag is present in %flags, so we overwrite it by passing it after %flags.
-// RUN: %cxx %flags %compile_flags -fmodules -fcxx-modules -fsyntax-only -std=c++98 %s
-// RUN: %cxx %flags %compile_flags -fmodules -fcxx-modules -fsyntax-only -std=c++03 %s
-// RUN: %cxx %flags %compile_flags -fmodules -fcxx-modules -fsyntax-only -std=c++11 %s
-// RUN: %cxx %flags %compile_flags -fmodules -fcxx-modules -fsyntax-only -std=c++14 %s
-// RUN: %cxx %flags %compile_flags -fmodules -fcxx-modules -fsyntax-only -std=c++17 %s
-// RUN: %cxx %flags %compile_flags -fmodules -fcxx-modules -fsyntax-only -std=c++2a %s
+// NOTE: The -std=XXX flag is present in %{flags}, so we overwrite it by passing it after %{flags}.
+// RUN: %{cxx} %{flags} %{compile_flags} -fmodules -fcxx-modules -fsyntax-only -std=c++98 %s
+// RUN: %{cxx} %{flags} %{compile_flags} -fmodules -fcxx-modules -fsyntax-only -std=c++03 %s
+// RUN: %{cxx} %{flags} %{compile_flags} -fmodules -fcxx-modules -fsyntax-only -std=c++11 %s
+// RUN: %{cxx} %{flags} %{compile_flags} -fmodules -fcxx-modules -fsyntax-only -std=c++14 %s
+// RUN: %{cxx} %{flags} %{compile_flags} -fmodules -fcxx-modules -fsyntax-only -std=c++17 %s
+// RUN: %{cxx} %{flags} %{compile_flags} -fmodules -fcxx-modules -fsyntax-only -std=c++2a %s
 
 #include <vector>
 
diff --git a/libcxx/test/libcxx/no_assert_include.sh.cpp b/libcxx/test/libcxx/no_assert_include.sh.cpp
index 25e3477b03de8..2b5f5f04ebddc 100644
--- a/libcxx/test/libcxx/no_assert_include.sh.cpp
+++ b/libcxx/test/libcxx/no_assert_include.sh.cpp
@@ -10,7 +10,7 @@
 // Ensure that none of the standard C++ headers implicitly include cassert or
 // assert.h (because assert() is implemented as a macro).
 
-// RUN: %compile -fsyntax-only
+// RUN: %{compile} -fsyntax-only
 
 // Prevent <ext/hash_map> from generating deprecated warnings for this test.
 #if defined(__DEPRECATED)
diff --git a/libcxx/test/libcxx/numerics/c.math/fdelayed-template-parsing.sh.cpp b/libcxx/test/libcxx/numerics/c.math/fdelayed-template-parsing.pass.cpp
similarity index 92%
rename from libcxx/test/libcxx/numerics/c.math/fdelayed-template-parsing.sh.cpp
rename to libcxx/test/libcxx/numerics/c.math/fdelayed-template-parsing.pass.cpp
index 37d09a8796f9d..8cebf75e458e4 100644
--- a/libcxx/test/libcxx/numerics/c.math/fdelayed-template-parsing.sh.cpp
+++ b/libcxx/test/libcxx/numerics/c.math/fdelayed-template-parsing.pass.cpp
@@ -9,9 +9,7 @@
 // test that cmath builds with -fdelayed-template-parsing
 
 // REQUIRES: fdelayed-template-parsing
-
-// RUN: %build -fdelayed-template-parsing
-// RUN: %run
+// ADDITIONAL_COMPILE_FLAGS: -fdelayed-template-parsing
 
 #include <cmath>
 #include <cassert>
diff --git a/libcxx/test/std/experimental/func/nothing_to_do.pass.cpp b/libcxx/test/libcxx/selftest/exec.sh.cpp
similarity index 63%
rename from libcxx/test/std/experimental/func/nothing_to_do.pass.cpp
rename to libcxx/test/libcxx/selftest/exec.sh.cpp
index 779762e7e995d..de42de3e7df27 100644
--- a/libcxx/test/std/experimental/func/nothing_to_do.pass.cpp
+++ b/libcxx/test/libcxx/selftest/exec.sh.cpp
@@ -7,8 +7,18 @@
 //
 //===----------------------------------------------------------------------===//
 
-int main(int, char**)
-{
+// FILE_DEPENDENCIES: %t.exe
 
+// RUN: %{build}
+// RUN: %{exec} %t.exe "HELLO"
+
+#include <cassert>
+#include <string>
+
+int main(int argc, char** argv) {
+  assert(argc == 2);
+
+  std::string arg = argv[1];
+  assert(arg == "HELLO");
   return 0;
 }
diff --git a/libcxx/test/libcxx/selftest/newformat/additional_compile_flags/substitutes-in-compile-flags.sh.cpp b/libcxx/test/libcxx/selftest/newformat/additional_compile_flags/substitutes-in-compile-flags.sh.cpp
new file mode 100644
index 0000000000000..6fa65a4175244
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/additional_compile_flags/substitutes-in-compile-flags.sh.cpp
@@ -0,0 +1,13 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Make sure that substitutions are performed inside additional compiler flags.
+
+// ADDITIONAL_COMPILE_FLAGS: -I %t.1
+// ADDITIONAL_COMPILE_FLAGS: -isystem %t.2 , -isysroot %t.3
+// RUN: echo "%{compile_flags}" | grep -e '-I %t.1 -isystem %t.2 -isysroot %t.3'
diff --git a/libcxx/test/libcxx/selftest/newformat/additional_compile_flags/substitutes-in-run.sh.cpp b/libcxx/test/libcxx/selftest/newformat/additional_compile_flags/substitutes-in-run.sh.cpp
new file mode 100644
index 0000000000000..387b1a1c3440c
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/additional_compile_flags/substitutes-in-run.sh.cpp
@@ -0,0 +1,15 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Make sure that additional compiler flags are added to the %{compile_flags}
+// substitution.
+
+// ADDITIONAL_COMPILE_FLAGS: -foo
+// ADDITIONAL_COMPILE_FLAGS: -bar
+// ADDITIONAL_COMPILE_FLAGS: -baz, -foom
+// RUN: echo "%{compile_flags}" | grep -e '-foo -bar -baz -foom'
diff --git a/libcxx/test/libcxx/selftest/newformat/compile.fail.cpp/compile-error.compile.fail.cpp b/libcxx/test/libcxx/selftest/newformat/compile.fail.cpp/compile-error.compile.fail.cpp
new file mode 100644
index 0000000000000..e54b8572a3ff9
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/compile.fail.cpp/compile-error.compile.fail.cpp
@@ -0,0 +1,16 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: verify-support
+
+// Make sure the test passes if it fails at compile-time
+
+struct Foo { };
+typedef Foo::x x; // expected-error{{no type named 'x' in 'Foo'}}
+
+int main() { }
diff --git a/libcxx/test/std/input.output/iostreams.base/ios.base/nothing_to_do.pass.cpp b/libcxx/test/libcxx/selftest/newformat/compile.fail.cpp/compile-success.compile.fail.cpp
similarity index 79%
rename from libcxx/test/std/input.output/iostreams.base/ios.base/nothing_to_do.pass.cpp
rename to libcxx/test/libcxx/selftest/newformat/compile.fail.cpp/compile-success.compile.fail.cpp
index 09b469211a20c..1cec8a3632c1a 100644
--- a/libcxx/test/std/input.output/iostreams.base/ios.base/nothing_to_do.pass.cpp
+++ b/libcxx/test/libcxx/selftest/newformat/compile.fail.cpp/compile-success.compile.fail.cpp
@@ -6,14 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// <ios>
+// XFAIL: *
 
-#include <ios>
+// Make sure the test DOES NOT pass if it succeeds at compile-time
 
-#include "test_macros.h"
-
-int main(int, char**)
-{
-
-  return 0;
-}
+int main() { }
diff --git a/libcxx/test/libcxx/selftest/newformat/compile.fail.cpp/wrong-expected.compile.fail.cpp b/libcxx/test/libcxx/selftest/newformat/compile.fail.cpp/wrong-expected.compile.fail.cpp
new file mode 100644
index 0000000000000..8b02098e4c738
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/compile.fail.cpp/wrong-expected.compile.fail.cpp
@@ -0,0 +1,17 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// XFAIL: *
+
+// Make sure the test DOES NOT pass if it fails at compile-time, but the
+// expected-error is wrong.
+
+struct Foo { };
+typedef Foo::x x; // expected-error{{this is not found in the errors}}
+
+int main() { }
diff --git a/libcxx/test/libcxx/selftest/newformat/compile.pass.cpp/compile-error.compile.pass.cpp b/libcxx/test/libcxx/selftest/newformat/compile.pass.cpp/compile-error.compile.pass.cpp
new file mode 100644
index 0000000000000..9ba8a5edbed91
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/compile.pass.cpp/compile-error.compile.pass.cpp
@@ -0,0 +1,16 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// XFAIL: *
+
+// Make sure the test DOES NOT pass if it fails at compile-time
+
+struct Foo { };
+typedef Foo::x x;
+
+int main() { }
diff --git a/libcxx/test/std/algorithms/alg.sorting/alg.binary.search/nothing_to_do.pass.cpp b/libcxx/test/libcxx/selftest/newformat/compile.pass.cpp/compile-success.compile.pass.cpp
similarity index 84%
rename from libcxx/test/std/algorithms/alg.sorting/alg.binary.search/nothing_to_do.pass.cpp
rename to libcxx/test/libcxx/selftest/newformat/compile.pass.cpp/compile-success.compile.pass.cpp
index 1f764da05d6b5..36d46904c8fc7 100644
--- a/libcxx/test/std/algorithms/alg.sorting/alg.binary.search/nothing_to_do.pass.cpp
+++ b/libcxx/test/libcxx/selftest/newformat/compile.pass.cpp/compile-success.compile.pass.cpp
@@ -6,8 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-int main(int, char**)
-{
+// Make sure the test passes if it succeeds to compile
 
-  return 0;
-}
+int main() { }
diff --git a/libcxx/test/libcxx/selftest/newformat/compile.pass.cpp/link-error.compile.pass.cpp b/libcxx/test/libcxx/selftest/newformat/compile.pass.cpp/link-error.compile.pass.cpp
new file mode 100644
index 0000000000000..8dd6a59688284
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/compile.pass.cpp/link-error.compile.pass.cpp
@@ -0,0 +1,15 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Make sure the test passes even if there's a link error, i.e. it isn't linked.
+
+extern void this_is_an_undefined_symbol();
+
+int main() {
+    this_is_an_undefined_symbol();
+}
diff --git a/libcxx/test/libcxx/selftest/newformat/compile.pass.cpp/run-error.compile.pass.cpp b/libcxx/test/libcxx/selftest/newformat/compile.pass.cpp/run-error.compile.pass.cpp
new file mode 100644
index 0000000000000..f3fbc64a38dd1
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/compile.pass.cpp/run-error.compile.pass.cpp
@@ -0,0 +1,13 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Make sure the test passes even if there's a runtime error, i.e. it isn't run.
+
+int main() {
+    return 1;
+}
diff --git a/libcxx/test/libcxx/selftest/newformat/fail.cpp/compile-error-with-verify.fail.cpp b/libcxx/test/libcxx/selftest/newformat/fail.cpp/compile-error-with-verify.fail.cpp
new file mode 100644
index 0000000000000..4442c784e12b3
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/fail.cpp/compile-error-with-verify.fail.cpp
@@ -0,0 +1,14 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Make sure the test passes if it fails at compile-time, with verify
+
+struct Foo { };
+typedef Foo::x x; // expected-error{{no type named 'x' in 'Foo'}}
+
+int main() { }
diff --git a/libcxx/test/libcxx/selftest/newformat/fail.cpp/compile-error-without-verify.fail.cpp b/libcxx/test/libcxx/selftest/newformat/fail.cpp/compile-error-without-verify.fail.cpp
new file mode 100644
index 0000000000000..ae8bf0b5e9816
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/fail.cpp/compile-error-without-verify.fail.cpp
@@ -0,0 +1,14 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Make sure the test passes if it fails at compile-time, without verify
+
+struct Foo { };
+typedef Foo::x x;
+
+int main() { }
diff --git a/libcxx/test/libcxx/selftest/newformat/fail.cpp/compile-success.fail.cpp b/libcxx/test/libcxx/selftest/newformat/fail.cpp/compile-success.fail.cpp
new file mode 100644
index 0000000000000..1cec8a3632c1a
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/fail.cpp/compile-success.fail.cpp
@@ -0,0 +1,13 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// XFAIL: *
+
+// Make sure the test DOES NOT pass if it succeeds at compile-time
+
+int main() { }
diff --git a/libcxx/test/libcxx/selftest/newformat/fail.cpp/wrong-expected.fail.cpp b/libcxx/test/libcxx/selftest/newformat/fail.cpp/wrong-expected.fail.cpp
new file mode 100644
index 0000000000000..d06759fb8172e
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/fail.cpp/wrong-expected.fail.cpp
@@ -0,0 +1,19 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: verify-support
+
+// XFAIL: *
+
+// Make sure the test DOES NOT pass if it fails at compile-time, but the
+// expected-error is wrong.
+
+struct Foo { };
+typedef Foo::x x; // expected-error{{this is not found in the errors}}
+
+int main() { }
diff --git a/libcxx/test/libcxx/selftest/newformat/fail.mm/compile-error-with-verify.fail.mm b/libcxx/test/libcxx/selftest/newformat/fail.mm/compile-error-with-verify.fail.mm
new file mode 100644
index 0000000000000..a45f05e3a925a
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/fail.mm/compile-error-with-verify.fail.mm
@@ -0,0 +1,16 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: objective-c++
+
+// Make sure the test passes if it fails at compile-time, with verify
+
+struct Foo { };
+typedef Foo::x x; // expected-error{{no type named 'x' in 'Foo'}}
+
+int main() { }
diff --git a/libcxx/test/libcxx/selftest/newformat/fail.mm/compile-error-without-verify.fail.mm b/libcxx/test/libcxx/selftest/newformat/fail.mm/compile-error-without-verify.fail.mm
new file mode 100644
index 0000000000000..57edb18c70736
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/fail.mm/compile-error-without-verify.fail.mm
@@ -0,0 +1,16 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: objective-c++
+
+// Make sure the test passes if it fails at compile-time, without verify
+
+struct Foo { };
+typedef Foo::x x;
+
+int main() { }
diff --git a/libcxx/test/libcxx/selftest/newformat/fail.mm/compile-success.fail.mm b/libcxx/test/libcxx/selftest/newformat/fail.mm/compile-success.fail.mm
new file mode 100644
index 0000000000000..a802d9cb12520
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/fail.mm/compile-success.fail.mm
@@ -0,0 +1,15 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: objective-c++
+
+// XFAIL: *
+
+// Make sure the test DOES NOT pass if it succeeds at compile-time
+
+int main() { }
diff --git a/libcxx/test/libcxx/selftest/newformat/fail.mm/wrong-expected.fail.cpp b/libcxx/test/libcxx/selftest/newformat/fail.mm/wrong-expected.fail.cpp
new file mode 100644
index 0000000000000..d0f0f9344e373
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/fail.mm/wrong-expected.fail.cpp
@@ -0,0 +1,19 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: objective-c++
+
+// XFAIL: *
+
+// Make sure the test DOES NOT pass if it fails at compile-time, but the
+// expected-error is wrong.
+
+struct Foo { };
+typedef Foo::x x; // expected-error{{this is not found in the errors}}
+
+int main() { }
diff --git a/libcxx/test/libcxx/selftest/newformat/file_dependencies/no-dependencies.sh.cpp b/libcxx/test/libcxx/selftest/newformat/file_dependencies/no-dependencies.sh.cpp
new file mode 100644
index 0000000000000..adfb15ad50dd6
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/file_dependencies/no-dependencies.sh.cpp
@@ -0,0 +1,13 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Make sure that %{file_dependencies} is empty when no FILE_DEPENDENCIES
+// line appears. Amongst other things, this makes sure that we don't share
+// file dependencies across unrelated Lit tests.
+
+// RUN: test -z "%{file_dependencies}"
diff --git a/libcxx/test/libcxx/selftest/newformat/file_dependencies/substitute-in-dependencies.sh.cpp b/libcxx/test/libcxx/selftest/newformat/file_dependencies/substitute-in-dependencies.sh.cpp
new file mode 100644
index 0000000000000..291afcb333c00
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/file_dependencies/substitute-in-dependencies.sh.cpp
@@ -0,0 +1,12 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Make sure that lit substitutions are expanded inside FILE_DEPENDENCIES lines.
+
+// FILE_DEPENDENCIES: %s
+// RUN: echo %{file_dependencies} | grep 'substitute-in-dependencies.sh.cpp'
diff --git a/libcxx/test/libcxx/selftest/newformat/file_dependencies/substitute-in-run.sh.cpp b/libcxx/test/libcxx/selftest/newformat/file_dependencies/substitute-in-run.sh.cpp
new file mode 100644
index 0000000000000..5bd0e335cd29b
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/file_dependencies/substitute-in-run.sh.cpp
@@ -0,0 +1,17 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Make sure %{file_dependencies} are substituted properly into RUN commands.
+
+// FILE_DEPENDENCIES: a.txt
+// FILE_DEPENDENCIES: b.txt, c.txt
+// FILE_DEPENDENCIES: /absolute/d.txt
+// RUN: echo %{file_dependencies} | grep 'a.txt'
+// RUN: echo %{file_dependencies} | grep 'b.txt'
+// RUN: echo %{file_dependencies} | grep 'c.txt'
+// RUN: echo %{file_dependencies} | grep '/absolute/d.txt'
diff --git a/libcxx/test/libcxx/selftest/newformat/link.fail.cpp/compile-error.link.fail.cpp b/libcxx/test/libcxx/selftest/newformat/link.fail.cpp/compile-error.link.fail.cpp
new file mode 100644
index 0000000000000..b0d709a082d22
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/link.fail.cpp/compile-error.link.fail.cpp
@@ -0,0 +1,16 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// XFAIL: *
+
+// Make sure the test DOES NOT pass if it fails at compile-time.
+
+struct Foo { };
+typedef Foo::x x;
+
+int main() { }
diff --git a/libcxx/test/libcxx/selftest/newformat/link.fail.cpp/link-error.link.fail.cpp b/libcxx/test/libcxx/selftest/newformat/link.fail.cpp/link-error.link.fail.cpp
new file mode 100644
index 0000000000000..e40b6bf8f0eea
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/link.fail.cpp/link-error.link.fail.cpp
@@ -0,0 +1,15 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Make sure the test passes if it fails at link time.
+
+extern void this_is_an_undefined_symbol();
+
+int main() {
+    this_is_an_undefined_symbol();
+}
diff --git a/libcxx/test/std/strings/string.view/string.view.template/nothing_to_do.pass.cpp b/libcxx/test/libcxx/selftest/newformat/link.fail.cpp/link-success.link.fail.cpp
similarity index 81%
rename from libcxx/test/std/strings/string.view/string.view.template/nothing_to_do.pass.cpp
rename to libcxx/test/libcxx/selftest/newformat/link.fail.cpp/link-success.link.fail.cpp
index 9ab36cbdfe9a5..43d3cc3ee7144 100644
--- a/libcxx/test/std/strings/string.view/string.view.template/nothing_to_do.pass.cpp
+++ b/libcxx/test/libcxx/selftest/newformat/link.fail.cpp/link-success.link.fail.cpp
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <string_view>
+// XFAIL: *
 
-#include "test_macros.h"
+// Make sure the test DOES NOT pass if it succeeds to link.
 
-int main(int, char**) {
-  return 0;
-}
+int main() { }
diff --git a/libcxx/test/libcxx/selftest/newformat/link.pass.cpp/compile-error.link.pass.cpp b/libcxx/test/libcxx/selftest/newformat/link.pass.cpp/compile-error.link.pass.cpp
new file mode 100644
index 0000000000000..b0d709a082d22
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/link.pass.cpp/compile-error.link.pass.cpp
@@ -0,0 +1,16 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// XFAIL: *
+
+// Make sure the test DOES NOT pass if it fails at compile-time.
+
+struct Foo { };
+typedef Foo::x x;
+
+int main() { }
diff --git a/libcxx/test/libcxx/selftest/newformat/link.pass.cpp/link-error.link.pass.cpp b/libcxx/test/libcxx/selftest/newformat/link.pass.cpp/link-error.link.pass.cpp
new file mode 100644
index 0000000000000..ed766675af017
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/link.pass.cpp/link-error.link.pass.cpp
@@ -0,0 +1,17 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// XFAIL: *
+
+// Make sure the test DOES NOT pass if it fails to link.
+
+extern void this_is_an_undefined_symbol();
+
+int main() {
+    this_is_an_undefined_symbol();
+}
diff --git a/libcxx/test/libcxx/extensions/nothing_to_do.pass.cpp b/libcxx/test/libcxx/selftest/newformat/link.pass.cpp/link-success.link.pass.cpp
similarity index 84%
rename from libcxx/test/libcxx/extensions/nothing_to_do.pass.cpp
rename to libcxx/test/libcxx/selftest/newformat/link.pass.cpp/link-success.link.pass.cpp
index 1f764da05d6b5..2187fbdeb98c0 100644
--- a/libcxx/test/libcxx/extensions/nothing_to_do.pass.cpp
+++ b/libcxx/test/libcxx/selftest/newformat/link.pass.cpp/link-success.link.pass.cpp
@@ -6,8 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-int main(int, char**)
-{
+// Make sure the test passes if it succeeds to link.
 
-  return 0;
-}
+int main() { }
diff --git a/libcxx/test/libcxx/selftest/newformat/link.pass.cpp/run-error.link.pass.cpp b/libcxx/test/libcxx/selftest/newformat/link.pass.cpp/run-error.link.pass.cpp
new file mode 100644
index 0000000000000..04460a4994e01
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/link.pass.cpp/run-error.link.pass.cpp
@@ -0,0 +1,14 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Make sure the test passes if it succeeds to link, even though it would have
+// failed at runtime.
+
+int main() {
+    return 1;
+}
diff --git a/libcxx/test/libcxx/selftest/newformat/lit.local.cfg b/libcxx/test/libcxx/selftest/newformat/lit.local.cfg
new file mode 100644
index 0000000000000..8789ca4253afb
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/lit.local.cfg
@@ -0,0 +1,4 @@
+import libcxx.test.newformat
+
+if not isinstance(config.test_format, libcxx.test.newformat.CxxStandardLibraryTest):
+    config.unsupported = True
diff --git a/libcxx/test/libcxx/selftest/newformat/pass.cpp/compile-error.pass.cpp b/libcxx/test/libcxx/selftest/newformat/pass.cpp/compile-error.pass.cpp
new file mode 100644
index 0000000000000..9ba8a5edbed91
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/pass.cpp/compile-error.pass.cpp
@@ -0,0 +1,16 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// XFAIL: *
+
+// Make sure the test DOES NOT pass if it fails at compile-time
+
+struct Foo { };
+typedef Foo::x x;
+
+int main() { }
diff --git a/libcxx/test/libcxx/selftest/newformat/pass.cpp/link-error.pass.cpp b/libcxx/test/libcxx/selftest/newformat/pass.cpp/link-error.pass.cpp
new file mode 100644
index 0000000000000..f62e23493331f
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/pass.cpp/link-error.pass.cpp
@@ -0,0 +1,17 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// XFAIL: *
+
+// Make sure the test DOES NOT pass if it fails at link-time
+
+extern void this_is_an_undefined_symbol();
+
+int main() {
+    this_is_an_undefined_symbol();
+}
diff --git a/libcxx/test/std/experimental/iterator/nothing_to_do.pass.cpp b/libcxx/test/libcxx/selftest/newformat/pass.cpp/run-error.pass.cpp
similarity index 79%
rename from libcxx/test/std/experimental/iterator/nothing_to_do.pass.cpp
rename to libcxx/test/libcxx/selftest/newformat/pass.cpp/run-error.pass.cpp
index 1cd92d047c4c7..999ee65e1e1e2 100644
--- a/libcxx/test/std/experimental/iterator/nothing_to_do.pass.cpp
+++ b/libcxx/test/libcxx/selftest/newformat/pass.cpp/run-error.pass.cpp
@@ -6,10 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <experimental/iterator>
+// XFAIL: *
 
-#include "test_macros.h"
+// Make sure the test DOES NOT pass if it fails at runtime.
 
-int main(int, char**) {
-  return 0;
+int main() {
+    return 1;
 }
diff --git a/libcxx/test/libcxx/selftest/newformat/pass.cpp/run-success.pass.cpp b/libcxx/test/libcxx/selftest/newformat/pass.cpp/run-success.pass.cpp
new file mode 100644
index 0000000000000..85cf334bfaefd
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/pass.cpp/run-success.pass.cpp
@@ -0,0 +1,13 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Make sure the test passes pass if it succeeds at runtime.
+
+int main() {
+    return 0;
+}
diff --git a/libcxx/test/libcxx/selftest/newformat/pass.mm/compile-error.pass.mm b/libcxx/test/libcxx/selftest/newformat/pass.mm/compile-error.pass.mm
new file mode 100644
index 0000000000000..95c661887cad7
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/pass.mm/compile-error.pass.mm
@@ -0,0 +1,18 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: objective-c++
+
+// XFAIL: *
+
+// Make sure the test DOES NOT pass if it fails at compile-time
+
+struct Foo { };
+typedef Foo::x x;
+
+int main() { }
diff --git a/libcxx/test/libcxx/selftest/newformat/pass.mm/link-error.pass.mm b/libcxx/test/libcxx/selftest/newformat/pass.mm/link-error.pass.mm
new file mode 100644
index 0000000000000..5a33a2c5b5d1d
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/pass.mm/link-error.pass.mm
@@ -0,0 +1,19 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: objective-c++
+
+// XFAIL: *
+
+// Make sure the test DOES NOT pass if it fails at link-time
+
+extern void this_is_an_undefined_symbol();
+
+int main() {
+    this_is_an_undefined_symbol();
+}
diff --git a/libcxx/test/libcxx/selftest/newformat/pass.mm/no-arc.pass.mm b/libcxx/test/libcxx/selftest/newformat/pass.mm/no-arc.pass.mm
new file mode 100644
index 0000000000000..cc1bc96469bad
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/pass.mm/no-arc.pass.mm
@@ -0,0 +1,19 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: objective-c++
+
+// Make sure ARC is not enabled by default in these tests.
+
+#if __has_feature(objc_arc)
+#   error "arc should not be enabled by default"
+#endif
+
+int main() {
+    return 0;
+}
diff --git a/libcxx/test/libcxx/selftest/newformat/pass.mm/run-error.pass.mm b/libcxx/test/libcxx/selftest/newformat/pass.mm/run-error.pass.mm
new file mode 100644
index 0000000000000..03e640d332e10
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/pass.mm/run-error.pass.mm
@@ -0,0 +1,17 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: objective-c++
+
+// XFAIL: *
+
+// Make sure the test DOES NOT pass if it fails at runtime.
+
+int main() {
+    return 1;
+}
diff --git a/libcxx/test/libcxx/selftest/newformat/pass.mm/run-success.pass.mm b/libcxx/test/libcxx/selftest/newformat/pass.mm/run-success.pass.mm
new file mode 100644
index 0000000000000..471af21c2dc21
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/pass.mm/run-success.pass.mm
@@ -0,0 +1,15 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: objective-c++
+
+// Make sure the test passes pass if it succeeds at runtime.
+
+int main() {
+    return 0;
+}
diff --git a/libcxx/test/libcxx/selftest/newformat/pass.mm/use-objective-cxx.pass.mm b/libcxx/test/libcxx/selftest/newformat/pass.mm/use-objective-cxx.pass.mm
new file mode 100644
index 0000000000000..f45d3c1186b5a
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/pass.mm/use-objective-cxx.pass.mm
@@ -0,0 +1,18 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: objective-c++
+
+// Make sure we can use Objective C++ features in these tests.
+
+@interface Foo
+@end
+
+int main() {
+    return 0;
+}
diff --git a/libcxx/test/libcxx/selftest/newformat/run.fail.cpp/compile-error.run.fail.cpp b/libcxx/test/libcxx/selftest/newformat/run.fail.cpp/compile-error.run.fail.cpp
new file mode 100644
index 0000000000000..94cff9c9854fd
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/run.fail.cpp/compile-error.run.fail.cpp
@@ -0,0 +1,18 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// XFAIL: *
+
+// Make sure the test DOES NOT pass if it fails at compile-time
+
+struct Foo { };
+typedef Foo::x x;
+
+int main() {
+    return 1;
+}
diff --git a/libcxx/test/libcxx/selftest/newformat/run.fail.cpp/link-error.run.fail.cpp b/libcxx/test/libcxx/selftest/newformat/run.fail.cpp/link-error.run.fail.cpp
new file mode 100644
index 0000000000000..31a054272d154
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/run.fail.cpp/link-error.run.fail.cpp
@@ -0,0 +1,18 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// XFAIL: *
+
+// Make sure that the test DOES NOT pass if it fails at link-time
+
+extern void this_is_an_undefined_symbol();
+
+int main() {
+    this_is_an_undefined_symbol();
+    return 1;
+}
diff --git a/libcxx/test/libcxx/selftest/newformat/run.fail.cpp/run-error.run.fail.cpp b/libcxx/test/libcxx/selftest/newformat/run.fail.cpp/run-error.run.fail.cpp
new file mode 100644
index 0000000000000..29c0ce9ad44b5
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/run.fail.cpp/run-error.run.fail.cpp
@@ -0,0 +1,13 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Make sure the test passes if it fails at runtime.
+
+int main() {
+    return 1;
+}
diff --git a/libcxx/test/libcxx/selftest/newformat/run.fail.cpp/run-success.run.fail.cpp b/libcxx/test/libcxx/selftest/newformat/run.fail.cpp/run-success.run.fail.cpp
new file mode 100644
index 0000000000000..ebc53227e2d8a
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/run.fail.cpp/run-success.run.fail.cpp
@@ -0,0 +1,15 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// XFAIL: *
+
+// Make sure the test DOES NOT pass if it succeeds at runtime.
+
+int main() {
+    return 0;
+}
diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/nothing_to_do.pass.cpp b/libcxx/test/libcxx/selftest/newformat/sh.cpp/empty.sh.cpp
similarity index 83%
rename from libcxx/test/std/algorithms/alg.modifying.operations/nothing_to_do.pass.cpp
rename to libcxx/test/libcxx/selftest/newformat/sh.cpp/empty.sh.cpp
index 1f764da05d6b5..de97543319bc8 100644
--- a/libcxx/test/std/algorithms/alg.modifying.operations/nothing_to_do.pass.cpp
+++ b/libcxx/test/libcxx/selftest/newformat/sh.cpp/empty.sh.cpp
@@ -6,8 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-int main(int, char**)
-{
+// XFAIL: *
 
-  return 0;
-}
+// Make sure the test DOES NOT pass if it has no 'RUN:' steps
diff --git a/libcxx/test/libcxx/selftest/newformat/sh.cpp/remote-substitutions.sh.cpp b/libcxx/test/libcxx/selftest/newformat/sh.cpp/remote-substitutions.sh.cpp
new file mode 100644
index 0000000000000..40e48c6f3ac75
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/sh.cpp/remote-substitutions.sh.cpp
@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Make sure that test-executables can appear in RUN lines and be executed
+// effectively. This somewhat difficult-to-understand test checks that when
+// we run with a remote executor, test-executables are copied to the remote
+// host and their path is fixed up (directly in the command-line) to their
+// path on the remote host instead of the local host.
+//
+// We also check that the path of test-executables is replaced whether they
+// appear first in the command-line or not.
+
+// FILE_DEPENDENCIES: %t.exe
+// RUN: %{cxx} %{compile_flags} %{link_flags} -o %t.exe %s
+// RUN: %{exec} %t.exe 0
+// RUN: %{exec} ! %t.exe 1
+
+#include <cassert>
+#include <cstdlib>
+
+int main(int argc, char** argv) {
+  assert(argc == 2);
+  int ret = std::atoi(argv[1]);
+  return ret;
+}
diff --git a/libcxx/test/libcxx/selftest/newformat/sh.cpp/run-error.sh.cpp b/libcxx/test/libcxx/selftest/newformat/sh.cpp/run-error.sh.cpp
new file mode 100644
index 0000000000000..ce5cd4f03349a
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/sh.cpp/run-error.sh.cpp
@@ -0,0 +1,13 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// XFAIL: *
+
+// Make sure the test DOES NOT pass if it fails to run
+
+// RUN: exit 1
diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/nothing_to_do.pass.cpp b/libcxx/test/libcxx/selftest/newformat/sh.cpp/run-success.sh.cpp
similarity index 84%
rename from libcxx/test/std/algorithms/alg.nonmodifying/nothing_to_do.pass.cpp
rename to libcxx/test/libcxx/selftest/newformat/sh.cpp/run-success.sh.cpp
index 1f764da05d6b5..9f2d58236f1be 100644
--- a/libcxx/test/std/algorithms/alg.nonmodifying/nothing_to_do.pass.cpp
+++ b/libcxx/test/libcxx/selftest/newformat/sh.cpp/run-success.sh.cpp
@@ -6,8 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-int main(int, char**)
-{
+// Make sure the test passes if it succeeds to run
 
-  return 0;
-}
+// RUN: exit 0
diff --git a/libcxx/test/libcxx/selftest/newformat/sh.cpp/substitutions.sh.cpp b/libcxx/test/libcxx/selftest/newformat/sh.cpp/substitutions.sh.cpp
new file mode 100644
index 0000000000000..45ff9643bc66a
--- /dev/null
+++ b/libcxx/test/libcxx/selftest/newformat/sh.cpp/substitutions.sh.cpp
@@ -0,0 +1,28 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Make sure we have access to the following substitutions at all times:
+// - %{cxx}
+// - %{compile_flags}
+// - %{link_flags}
+// - %{exec}
+
+// FILE_DEPENDENCIES: %t.exe
+// RUN: %{cxx} %{compile_flags} %{link_flags} -o %t.exe %s
+// RUN: %{exec} %t.exe "HELLO"
+
+#include <cassert>
+#include <string>
+
+int main(int argc, char** argv) {
+  assert(argc == 2);
+
+  std::string arg = argv[1];
+  assert(arg == "HELLO");
+  return 0;
+}
diff --git a/libcxx/test/libcxx/selftest/not_test.sh.cpp b/libcxx/test/libcxx/selftest/not_test.sh.cpp
index 8dbf708f0cb1a..56da5ed5ea505 100644
--- a/libcxx/test/libcxx/selftest/not_test.sh.cpp
+++ b/libcxx/test/libcxx/selftest/not_test.sh.cpp
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-// RUN: %build
-// RUN: not %run
+// RUN: %{build}
+// RUN: %{not} %{run}
 
 int main(int, char**)
 {
diff --git a/libcxx/test/libcxx/selftest/test.arc.fail.mm b/libcxx/test/libcxx/selftest/test.arc.fail.mm
index 712947bcae50f..0e8b62c09db51 100644
--- a/libcxx/test/libcxx/selftest/test.arc.fail.mm
+++ b/libcxx/test/libcxx/selftest/test.arc.fail.mm
@@ -7,6 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+// ADDITIONAL_COMPILE_FLAGS: -fobjc-arc
+
 #if __has_feature(objc_arc)
 #error This test should not compile.
 #endif
diff --git a/libcxx/test/libcxx/selftest/test.arc.pass.mm b/libcxx/test/libcxx/selftest/test.arc.pass.mm
index eecdbb616185d..b70b24e631836 100644
--- a/libcxx/test/libcxx/selftest/test.arc.pass.mm
+++ b/libcxx/test/libcxx/selftest/test.arc.pass.mm
@@ -7,6 +7,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+// REQUIRES: has-fobjc-arc
+// ADDITIONAL_COMPILE_FLAGS: -fobjc-arc
+
 #if __has_feature(objc_arc) == 0
 #error "arc should be enabled"
 #endif
diff --git a/libcxx/test/libcxx/selftest/test.file_dependencies.sh.cpp b/libcxx/test/libcxx/selftest/test.file_dependencies.sh.cpp
index b850b44b2bfec..5974393b13703 100644
--- a/libcxx/test/libcxx/selftest/test.file_dependencies.sh.cpp
+++ b/libcxx/test/libcxx/selftest/test.file_dependencies.sh.cpp
@@ -8,5 +8,7 @@
 //===----------------------------------------------------------------------===//
 
 // FILE_DEPENDENCIES: test.pass.cpp
+// FILE_DEPENDENCIES: %s
 
-// RUN: echo %file_dependencies | grep 'test.pass.cpp'
+// RUN: echo %{file_dependencies} | grep 'test.pass.cpp'
+// RUN: echo %{file_dependencies} | grep '%s'
diff --git a/libcxx/test/libcxx/selftest/test.pass.mm b/libcxx/test/libcxx/selftest/test.pass.mm
index ba4d36bbe2110..2475c31e18952 100644
--- a/libcxx/test/libcxx/selftest/test.pass.mm
+++ b/libcxx/test/libcxx/selftest/test.pass.mm
@@ -7,6 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+// REQUIRES: objective-c++
+
 #if __has_feature(objc_arc)
 #error "arc should *not* be enabled"
 #endif
diff --git a/libcxx/test/libcxx/selftest/test.sh.cpp b/libcxx/test/libcxx/selftest/test.sh.cpp
index 1fc175413ab7c..ad75f0227e819 100644
--- a/libcxx/test/libcxx/selftest/test.sh.cpp
+++ b/libcxx/test/libcxx/selftest/test.sh.cpp
@@ -7,8 +7,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-// RUN: %build
-// RUN: %run
+// FILE_DEPENDENCIES: %t.exe
+
+// RUN: %{build}
+// RUN: %{run}
 
 int main(int, char**)
 {
diff --git a/libcxx/test/libcxx/strings/basic.string/PR42676.sh.cpp b/libcxx/test/libcxx/strings/basic.string/PR42676.sh.cpp
index 7037482858543..18faaf02871de 100644
--- a/libcxx/test/libcxx/strings/basic.string/PR42676.sh.cpp
+++ b/libcxx/test/libcxx/strings/basic.string/PR42676.sh.cpp
@@ -8,8 +8,9 @@
 
 // Regression test for PR42676.
 
-// RUN: %cxx %flags %s -o %t.exe %compile_flags %link_flags -D_LIBCPP_HIDE_FROM_ABI_PER_TU
-// RUN: %run
+// FILE_DEPENDENCIES: %t.exe
+// RUN: %{cxx} %{flags} %s -o %t.exe %{compile_flags} %{link_flags} -D_LIBCPP_HIDE_FROM_ABI_PER_TU
+// RUN: %{run}
 
 #include <memory>
 #include <string>
diff --git a/libcxx/test/libcxx/thread/thread.lock/thread.lock.guard/nodiscard.fail.cpp b/libcxx/test/libcxx/thread/thread.lock/thread.lock.guard/nodiscard.fail.cpp
index 802892a5eb39c..e14bcd979d901 100644
--- a/libcxx/test/libcxx/thread/thread.lock/thread.lock.guard/nodiscard.fail.cpp
+++ b/libcxx/test/libcxx/thread/thread.lock/thread.lock.guard/nodiscard.fail.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 // UNSUPPORTED: libcpp-has-no-threads
+// REQUIRES: verify-support
 
 // [[nodiscard]] on constructors isn't supported by all compilers
 // UNSUPPORTED: clang-6, clang-7, clang-8, clang-9
@@ -32,7 +33,7 @@
 
 int main(int, char**) {
     std::mutex m;
-    std::lock_guard<std::mutex>{m}; // expected-error{{ignoring temporary created by a constructor declared with 'nodiscard' attribute}}
-    std::lock_guard<std::mutex>{m, std::adopt_lock}; // expected-error{{ignoring temporary created by a constructor declared with 'nodiscard' attribute}}
+    std::lock_guard<std::mutex>{m}; // expected-warning {{ignoring temporary created by a constructor declared with 'nodiscard' attribute}}
+    std::lock_guard<std::mutex>{m, std::adopt_lock}; // expected-warning {{ignoring temporary created by a constructor declared with 'nodiscard' attribute}}
     return 0;
 }
diff --git a/libcxx/test/libcxx/thread/thread.threads/thread.thread.this/sleep_for.pass.cpp b/libcxx/test/libcxx/thread/thread.threads/thread.thread.this/sleep_for.pass.cpp
index 7818b6995ac1e..ad3aa3104e1a6 100644
--- a/libcxx/test/libcxx/thread/thread.threads/thread.thread.this/sleep_for.pass.cpp
+++ b/libcxx/test/libcxx/thread/thread.threads/thread.thread.this/sleep_for.pass.cpp
@@ -8,7 +8,7 @@
 //
 // UNSUPPORTED: libcpp-has-no-threads
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // This test uses the POSIX header <sys/time.h> which Windows doesn't provide
 // UNSUPPORTED: windows
diff --git a/libcxx/test/libcxx/type_traits/is_pointer.arc.pass.mm b/libcxx/test/libcxx/type_traits/is_pointer.arc.pass.mm
index 1f4c5089af943..e8296b3f36987 100644
--- a/libcxx/test/libcxx/type_traits/is_pointer.arc.pass.mm
+++ b/libcxx/test/libcxx/type_traits/is_pointer.arc.pass.mm
@@ -5,8 +5,10 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-//
+
 // UNSUPPORTED: c++98, c++03
+// REQUIRES: has-fobjc-arc
+// ADDITIONAL_COMPILE_FLAGS: -fobjc-arc
 
 // <type_traits>
 
@@ -15,31 +17,40 @@
 // Test that we correctly handle Objective-C++ ARC qualifiers on pointers.
 
 #include <type_traits>
+#include "test_macros.h"
 
 
 template <typename T>
-void test_is_pointer() {
+void assert_is_pointer() {
     static_assert(std::is_pointer<T>::value, "");
+#if TEST_STD_VER > 14
+    static_assert(std::is_pointer_v<T>, "");
+#endif
+}
+
+template <typename T>
+void test_is_pointer() {
+    assert_is_pointer<T>();
+
+    assert_is_pointer<T __weak>();
+    assert_is_pointer<T __strong>();
+    assert_is_pointer<T __autoreleasing>();
+    assert_is_pointer<T __unsafe_unretained>();
+
+    assert_is_pointer<T __weak const>();
+    assert_is_pointer<T __strong const>();
+    assert_is_pointer<T __autoreleasing const>();
+    assert_is_pointer<T __unsafe_unretained const>();
+
+    assert_is_pointer<T __weak volatile>();
+    assert_is_pointer<T __strong volatile>();
+    assert_is_pointer<T __autoreleasing volatile>();
+    assert_is_pointer<T __unsafe_unretained volatile>();
 
-    static_assert(std::is_pointer<T __weak>::value, "");
-    static_assert(std::is_pointer<T __strong>::value, "");
-    static_assert(std::is_pointer<T __autoreleasing>::value, "");
-    static_assert(std::is_pointer<T __unsafe_unretained>::value, "");
-
-    static_assert(std::is_pointer<T __weak const>::value, "");
-    static_assert(std::is_pointer<T __strong const>::value, "");
-    static_assert(std::is_pointer<T __autoreleasing const>::value, "");
-    static_assert(std::is_pointer<T __unsafe_unretained const>::value, "");
-
-    static_assert(std::is_pointer<T __weak volatile>::value, "");
-    static_assert(std::is_pointer<T __strong volatile>::value, "");
-    static_assert(std::is_pointer<T __autoreleasing volatile>::value, "");
-    static_assert(std::is_pointer<T __unsafe_unretained volatile>::value, "");
-
-    static_assert(std::is_pointer<T __weak const volatile>::value, "");
-    static_assert(std::is_pointer<T __strong const volatile>::value, "");
-    static_assert(std::is_pointer<T __autoreleasing const volatile>::value, "");
-    static_assert(std::is_pointer<T __unsafe_unretained const volatile>::value, "");
+    assert_is_pointer<T __weak const volatile>();
+    assert_is_pointer<T __strong const volatile>();
+    assert_is_pointer<T __autoreleasing const volatile>();
+    assert_is_pointer<T __unsafe_unretained const volatile>();
 }
 
 @class Foo;
diff --git a/libcxx/test/libcxx/type_traits/is_scalar.objc.pass.mm b/libcxx/test/libcxx/type_traits/is_scalar.objc.pass.mm
index 1f7ffec8efda1..23693fc064eb9 100644
--- a/libcxx/test/libcxx/type_traits/is_scalar.objc.pass.mm
+++ b/libcxx/test/libcxx/type_traits/is_scalar.objc.pass.mm
@@ -5,8 +5,10 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-//
+
 // UNSUPPORTED: c++98, c++03
+// REQUIRES: has-fblocks
+// ADDITIONAL_COMPILE_FLAGS: -fblocks
 
 // <type_traits>
 
diff --git a/libcxx/test/libcxx/utilities/function.objects/abi_bug_cxx03_cxx11_example.sh.cpp b/libcxx/test/libcxx/utilities/function.objects/abi_bug_cxx03_cxx11_example.sh.cpp
index 2fc2f32d8a959..e367074cd4feb 100644
--- a/libcxx/test/libcxx/utilities/function.objects/abi_bug_cxx03_cxx11_example.sh.cpp
+++ b/libcxx/test/libcxx/utilities/function.objects/abi_bug_cxx03_cxx11_example.sh.cpp
@@ -13,10 +13,10 @@
 // This tests is meant to demonstrate an existing ABI bug between the
 // C++03 and C++11 implementations of std::function. It is not a real test.
 
-// RUN: %cxx -c %s -o %t.first.o %flags %compile_flags -std=c++03 -g
-// RUN: %cxx -c %s -o %t.second.o -DWITH_MAIN %flags %compile_flags -g -std=c++11
-// RUN: %cxx -o %t.exe %t.first.o %t.second.o %flags %link_flags -g
-// RUN: %run
+// RUN: %{cxx} -c %s -o %t.first.o %{flags} %{compile_flags} -std=c++03 -g
+// RUN: %{cxx} -c %s -o %t.second.o -DWITH_MAIN %{flags} %{compile_flags} -g -std=c++11
+// RUN: %{cxx} -o %t.exe %t.first.o %t.second.o %{flags} %{link_flags} -g
+// RUN: %{run}
 
 #include <functional>
 #include <cassert>
diff --git a/libcxx/test/libcxx/utilities/function.objects/func.wrap/depr_in_cxx03.fail.cpp b/libcxx/test/libcxx/utilities/function.objects/func.wrap/depr_in_cxx03.fail.cpp
index 282a7112cced1..f8a162ce1beef 100644
--- a/libcxx/test/libcxx/utilities/function.objects/func.wrap/depr_in_cxx03.fail.cpp
+++ b/libcxx/test/libcxx/utilities/function.objects/func.wrap/depr_in_cxx03.fail.cpp
@@ -20,10 +20,10 @@
 int main() {
     // Note:
     // We use sizeof() to require it to be a complete type. We don't create a
-    // variable because otherwise we get two errors for each variable (the
-    // second error is when the destructor is implicitly called).
-    (void)sizeof(std::function<void ()>); // expected-error{{'function<void ()>' is deprecated}}
-    (void)sizeof(std::function<void (int)>); // expected-error{{'function<void (int)>' is deprecated}}
-    (void)sizeof(std::function<void (int, int)>); // expected-error{{'function<void (int, int)>' is deprecated}}
-    (void)sizeof(std::function<void (int, int, int)>); // expected-error{{'function<void (int, int, int)>' is deprecated}}
+    // variable because otherwise we get two warnings for each variable (the
+    // second warning is when the destructor is implicitly called).
+    (void)sizeof(std::function<void ()>); // expected-warning {{'function<void ()>' is deprecated}}
+    (void)sizeof(std::function<void (int)>); // expected-warning {{'function<void (int)>' is deprecated}}
+    (void)sizeof(std::function<void (int, int)>); // expected-warning {{'function<void (int, int)>' is deprecated}}
+    (void)sizeof(std::function<void (int, int, int)>); // expected-warning {{'function<void (int, int, int)>' is deprecated}}
 }
diff --git a/libcxx/test/libcxx/utilities/meta/stress_tests/stress_test_is_same.sh.cpp b/libcxx/test/libcxx/utilities/meta/stress_tests/stress_test_is_same.sh.cpp
index 72d3efaa71f29..cfd8adb52d854 100644
--- a/libcxx/test/libcxx/utilities/meta/stress_tests/stress_test_is_same.sh.cpp
+++ b/libcxx/test/libcxx/utilities/meta/stress_tests/stress_test_is_same.sh.cpp
@@ -17,8 +17,8 @@
 // std::_IsSame:    689.634 ms     356 K
 // std::is_same:  8,129.180 ms     560 K
 //
-// RUN: %cxx %flags %compile_flags -c %s -o %S/orig.o -ggdb  -ggnu-pubnames -ftemplate-depth=5000 -ftime-trace -std=c++17
-// RUN: %cxx %flags %compile_flags -c %s -o %S/new.o -ggdb  -ggnu-pubnames -ftemplate-depth=5000 -ftime-trace -std=c++17 -DTEST_NEW
+// RUN: %{cxx} %{flags} %{compile_flags} -c %s -o %S/orig.o -ggdb  -ggnu-pubnames -ftemplate-depth=5000 -ftime-trace -std=c++17
+// RUN: %{cxx} %{flags} %{compile_flags} -c %s -o %S/new.o -ggdb  -ggnu-pubnames -ftemplate-depth=5000 -ftime-trace -std=c++17 -DTEST_NEW
 
 #include <type_traits>
 #include <cassert>
diff --git a/libcxx/test/libcxx/utilities/meta/stress_tests/stress_test_metafunctions.sh.cpp b/libcxx/test/libcxx/utilities/meta/stress_tests/stress_test_metafunctions.sh.cpp
index 5096691d8bb67..6222d5c8d2e13 100644
--- a/libcxx/test/libcxx/utilities/meta/stress_tests/stress_test_metafunctions.sh.cpp
+++ b/libcxx/test/libcxx/utilities/meta/stress_tests/stress_test_metafunctions.sh.cpp
@@ -19,9 +19,9 @@
 // __and_:      14,181.851 ms     648 M
 //
 
-// RUN: %cxx %flags %compile_flags -c %s -o %S/new.o -ggdb  -ggnu-pubnames -ftemplate-depth=5000 -ftime-trace -std=c++17
-// RUN: %cxx %flags %compile_flags -c %s -o %S/lazy.o -ggdb  -ggnu-pubnames -ftemplate-depth=5000 -ftime-trace  -std=c++17 -DTEST_LAZY_AND
-// RUN: %cxx %flags %compile_flags -c %s -o %S/std.o -ggdb  -ggnu-pubnames -ftemplate-depth=5000 -ftime-trace   -std=c++17 -DTEST_STD_AND
+// RUN: %{cxx} %{flags} %{compile_flags} -c %s -o %S/new.o -ggdb  -ggnu-pubnames -ftemplate-depth=5000 -ftime-trace -std=c++17
+// RUN: %{cxx} %{flags} %{compile_flags} -c %s -o %S/lazy.o -ggdb  -ggnu-pubnames -ftemplate-depth=5000 -ftime-trace  -std=c++17 -DTEST_LAZY_AND
+// RUN: %{cxx} %{flags} %{compile_flags} -c %s -o %S/std.o -ggdb  -ggnu-pubnames -ftemplate-depth=5000 -ftime-trace   -std=c++17 -DTEST_STD_AND
 
 #include <type_traits>
 #include <cassert>
diff --git a/libcxx/test/libcxx/utilities/meta/stress_tests/stress_test_variant_overloads_impl.sh.cpp b/libcxx/test/libcxx/utilities/meta/stress_tests/stress_test_variant_overloads_impl.sh.cpp
index 013d434f40d7b..212d915c6970e 100644
--- a/libcxx/test/libcxx/utilities/meta/stress_tests/stress_test_variant_overloads_impl.sh.cpp
+++ b/libcxx/test/libcxx/utilities/meta/stress_tests/stress_test_variant_overloads_impl.sh.cpp
@@ -24,13 +24,13 @@
 // variant_new:     1,105 ms        828 KiB
 
 
-// RUN: %cxx %flags %compile_flags -std=c++17 -c %s \
+// RUN: %{cxx} %{flags} %{compile_flags} -std=c++17 -c %s \
 // RUN:    -ggdb  -ggnu-pubnames -ftemplate-depth=5000 -ftime-trace -g \
 // RUN:    -DTEST_NS=flat_impl -o %S/flat.o
-// RUN: %cxx %flags %compile_flags -std=c++17 -c %s \
+// RUN: %{cxx} %{flags} %{compile_flags} -std=c++17 -c %s \
 // RUN:    -ggdb  -ggnu-pubnames -ftemplate-depth=5000 -ftime-trace -g \
 // RUN:    -DTEST_NS=rec_impl -o %S/rec.o
-// RUN: %cxx %flags %compile_flags -std=c++17 -c %s \
+// RUN: %{cxx} %{flags} %{compile_flags} -std=c++17 -c %s \
 // RUN:    -ggdb  -ggnu-pubnames -ftemplate-depth=5000 -ftime-trace -g \
 // RUN:    -DTEST_NS=variant_impl -o %S/variant.o
 
diff --git a/libcxx/test/libcxx/utilities/optional/block.objc.pass.mm b/libcxx/test/libcxx/utilities/optional/block.objc.pass.mm
index 8d2722bf85340..77674602f377e 100644
--- a/libcxx/test/libcxx/utilities/optional/block.objc.pass.mm
+++ b/libcxx/test/libcxx/utilities/optional/block.objc.pass.mm
@@ -7,6 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14
+// REQUIRES: has-fblocks
+// ADDITIONAL_COMPILE_FLAGS: -fblocks
 
 // <optional>
 
diff --git a/libcxx/test/lit.cfg b/libcxx/test/lit.cfg
index ed5aa29d91758..8f42dd7c468b8 100644
--- a/libcxx/test/lit.cfg
+++ b/libcxx/test/lit.cfg
@@ -15,7 +15,7 @@ if 'PYLINT_IMPORT' in os.environ:
 config.name = 'libc++'
 
 # suffixes: A list of file extensions to treat as test files.
-config.suffixes = ['.pass.cpp', '.fail.cpp', '.sh.cpp']
+config.suffixes = ['.pass.cpp', '.fail.cpp', '.sh.cpp', '.pass.mm']
 
 # test_source_root: The root path where tests are located.
 config.test_source_root = os.path.dirname(__file__)
@@ -52,4 +52,9 @@ config_module = __import__(config_module_name, fromlist=['Configuration'])
 configuration = config_module.Configuration(lit_config, config)
 configuration.configure()
 configuration.print_config_info()
-config.test_format = configuration.get_test_format()
+if lit_config.params.get('use_new_format', False):
+    lit_config.note("Using the experimental libc++ testing format")
+    import libcxx.test.newformat
+    config.test_format = libcxx.test.newformat.CxxStandardLibraryTest()
+else:
+    config.test_format = configuration.get_test_format()
diff --git a/libcxx/test/nothing_to_do.pass.cpp b/libcxx/test/nothing_to_do.pass.cpp
deleted file mode 100644
index 6d95c42f6e142..0000000000000
--- a/libcxx/test/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-  return 0;
-}
diff --git a/libcxx/test/pretty_printers/gdb_pretty_printer_test.sh.cpp b/libcxx/test/pretty_printers/gdb_pretty_printer_test.sh.cpp
index 3050cf15f35a0..268501fb9d991 100644
--- a/libcxx/test/pretty_printers/gdb_pretty_printer_test.sh.cpp
+++ b/libcxx/test/pretty_printers/gdb_pretty_printer_test.sh.cpp
@@ -9,9 +9,9 @@
 // UNSUPPORTED: system-windows
 // REQUIRES: libcxx_gdb
 //
-// RUN: %cxx %flags %s -o %t.exe %compile_flags -g %link_flags
+// RUN: %{cxx} %{flags} %s -o %t.exe %{compile_flags} -g %{link_flags}
 // Ensure locale-independence for unicode tests.
-// RUN: %libcxx_gdb -nx -batch -iex "set autoload off" -ex "source %libcxx_src_root/utils/gdb/libcxx/printers.py" -ex "python register_libcxx_printer_loader()" -ex "source %libcxx_src_root/test/pretty_printers/gdb_pretty_printer_test.py" %t.exe
+// RUN: %{libcxx_gdb} -nx -batch -iex "set autoload off" -ex "source %{libcxx_src_root}/utils/gdb/libcxx/printers.py" -ex "python register_libcxx_printer_loader()" -ex "source %{libcxx_src_root}/test/pretty_printers/gdb_pretty_printer_test.py" %t.exe
 
 #include <bitset>
 #include <deque>
@@ -153,6 +153,16 @@ void string_test() {
                             "\"mehmet bizim dostumuz agzi kirik testimiz\"");
 }
 
+namespace a_namespace {
+// To test name-lookup in the presence of using inside a namespace. Inside this
+// namespace, unqualified string_view variables will appear in the debug info as
+// "a_namespace::string_view, rather than "std::string_view".
+//
+// There is nothing special here about string_view; it's just the data structure
+// where lookup with using inside a namespace wasn't always working.
+
+using string_view = std::string_view;
+
 void string_view_test() {
   std::string_view i_am_empty;
   ComparePrettyPrintToChars(i_am_empty, "std::string_view of length 0: \"\"");
@@ -166,6 +176,12 @@ void string_view_test() {
   std::string_view wonderful(&char_arr[7], 9);
   ComparePrettyPrintToChars(
       wonderful, "std::string_view of length 9: \"wonderful\"");
+
+  const char char_arr1[] = "namespace_stringview";
+  string_view namespace_stringview(&char_arr1[10], 10);
+  ComparePrettyPrintToChars(
+      namespace_stringview, "std::string_view of length 10: \"stringview\"");
+}
 }
 
 void u16string_test() {
@@ -628,7 +644,7 @@ int main(int argc, char* argv[]) {
   framework_self_test();
 
   string_test();
-  string_view_test();
+  a_namespace::string_view_test();
 
   u32string_test();
   tuple_test();
diff --git a/libcxx/test/std/algorithms/alg.sorting/alg.heap.operations/nothing_to_do.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.heap.operations/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/algorithms/alg.sorting/alg.heap.operations/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/nothing_to_do.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/algorithms/alg.sorting/alg.sort/nothing_to_do.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.sort/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/algorithms/alg.sorting/alg.sort/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/algorithms/alg.sorting/nothing_to_do.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/algorithms/alg.sorting/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/atomics/atomics.general/nothing_to_do.pass.cpp b/libcxx/test/std/atomics/atomics.general/nothing_to_do.pass.cpp
deleted file mode 100644
index 779762e7e995d..0000000000000
--- a/libcxx/test/std/atomics/atomics.general/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,14 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/atomics/atomics.types.operations/nothing_to_do.pass.cpp b/libcxx/test/std/atomics/atomics.types.operations/nothing_to_do.pass.cpp
deleted file mode 100644
index 779762e7e995d..0000000000000
--- a/libcxx/test/std/atomics/atomics.types.operations/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,14 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/containers/associative/map/PR28469_undefined_behavior_segfault.sh.cpp b/libcxx/test/std/containers/associative/map/PR28469_undefined_behavior_segfault.sh.cpp
index 030fdaf39184a..83aa36dff3ddb 100644
--- a/libcxx/test/std/containers/associative/map/PR28469_undefined_behavior_segfault.sh.cpp
+++ b/libcxx/test/std/containers/associative/map/PR28469_undefined_behavior_segfault.sh.cpp
@@ -6,8 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-// RUN: %build -O2
-// RUN: %run
+// FILE_DEPENDENCIES: %t.exe
+// RUN: %{build} -O2
+// RUN: %{run}
 
 // <map>
 
diff --git a/libcxx/test/std/containers/associative/map/map.access/empty.fail.cpp b/libcxx/test/std/containers/associative/map/map.access/empty.fail.cpp
index 61c4e757b0c36..004309758a5df 100644
--- a/libcxx/test/std/containers/associative/map/map.access/empty.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.access/empty.fail.cpp
@@ -15,6 +15,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 #include <map>
 
@@ -23,7 +24,7 @@
 int main(int, char**)
 {
     std::map<int, int> c;
-    c.empty();  // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/empty.fail.cpp b/libcxx/test/std/containers/associative/multimap/empty.fail.cpp
index bc305b9cc8db4..2492be5820216 100644
--- a/libcxx/test/std/containers/associative/multimap/empty.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/empty.fail.cpp
@@ -15,6 +15,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 #include <map>
 
@@ -23,7 +24,7 @@
 int main(int, char**)
 {
     std::multimap<int, int> c;
-    c.empty();  // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/empty.fail.cpp b/libcxx/test/std/containers/associative/multiset/empty.fail.cpp
index 4467b487089b0..abc67ed1e8d24 100644
--- a/libcxx/test/std/containers/associative/multiset/empty.fail.cpp
+++ b/libcxx/test/std/containers/associative/multiset/empty.fail.cpp
@@ -15,6 +15,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 #include <set>
 
@@ -23,7 +24,7 @@
 int main(int, char**)
 {
     std::multiset<int> c;
-    c.empty();  // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/containers/associative/set/empty.fail.cpp b/libcxx/test/std/containers/associative/set/empty.fail.cpp
index fc5856fbdbf49..ba6aadca3d43e 100644
--- a/libcxx/test/std/containers/associative/set/empty.fail.cpp
+++ b/libcxx/test/std/containers/associative/set/empty.fail.cpp
@@ -15,6 +15,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 #include <set>
 
@@ -23,7 +24,7 @@
 int main(int, char**)
 {
     std::set<int> c;
-    c.empty();  // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/nothing_to_do.pass.cpp b/libcxx/test/std/containers/container.adaptors/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/containers/container.adaptors/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/empty.fail.cpp b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/empty.fail.cpp
index 698553d33e10f..fa3476a0dbf2d 100644
--- a/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/empty.fail.cpp
+++ b/libcxx/test/std/containers/container.adaptors/priority.queue/priqueue.members/empty.fail.cpp
@@ -15,6 +15,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 #include <queue>
 
@@ -23,7 +24,7 @@
 int main(int, char**)
 {
     std::priority_queue<int> c;
-    c.empty();  // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/empty.fail.cpp b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/empty.fail.cpp
index 454bf32bede8e..6cd0a5e745e30 100644
--- a/libcxx/test/std/containers/container.adaptors/queue/queue.defn/empty.fail.cpp
+++ b/libcxx/test/std/containers/container.adaptors/queue/queue.defn/empty.fail.cpp
@@ -15,6 +15,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 #include <queue>
 
@@ -23,7 +24,7 @@
 int main(int, char**)
 {
     std::queue<int> c;
-    c.empty();  // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/stack/stack.defn/empty.fail.cpp b/libcxx/test/std/containers/container.adaptors/stack/stack.defn/empty.fail.cpp
index 54cd986295a68..fb107582780ab 100644
--- a/libcxx/test/std/containers/container.adaptors/stack/stack.defn/empty.fail.cpp
+++ b/libcxx/test/std/containers/container.adaptors/stack/stack.defn/empty.fail.cpp
@@ -15,6 +15,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 #include <stack>
 
@@ -23,7 +24,7 @@
 int main(int, char**)
 {
     std::stack<int> c;
-    c.empty();  // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/containers/container.requirements/associative.reqmts/nothing_to_do.pass.cpp b/libcxx/test/std/containers/container.requirements/associative.reqmts/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/containers/container.requirements/associative.reqmts/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/containers/container.requirements/container.requirements.general/nothing_to_do.pass.cpp b/libcxx/test/std/containers/container.requirements/container.requirements.general/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/containers/container.requirements/container.requirements.general/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/containers/container.requirements/nothing_to_do.pass.cpp b/libcxx/test/std/containers/container.requirements/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/containers/container.requirements/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/containers/container.requirements/unord.req/nothing_to_do.pass.cpp b/libcxx/test/std/containers/container.requirements/unord.req/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/containers/container.requirements/unord.req/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/containers/nothing_to_do.pass.cpp b/libcxx/test/std/containers/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/containers/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/containers/sequences/array/array.creation/to_array.fail.cpp b/libcxx/test/std/containers/sequences/array/array.creation/to_array.fail.cpp
index 9e194cf2254f5..49ad6a61037f2 100644
--- a/libcxx/test/std/containers/sequences/array/array.creation/to_array.fail.cpp
+++ b/libcxx/test/std/containers/sequences/array/array.creation/to_array.fail.cpp
@@ -20,7 +20,6 @@ int main(int, char**) {
     // expected-error@array:* {{to_array does not accept multidimensional arrays}}
     // expected-error@array:* {{to_array requires copy constructible elements}}
     // expected-error@array:* 3 {{cannot initialize}}
-    // expected-error@array:* 0+ {{suggest braces}}
     std::to_array(source); // expected-note {{requested here}}
   }
 
diff --git a/libcxx/test/std/containers/sequences/array/empty.fail.cpp b/libcxx/test/std/containers/sequences/array/empty.fail.cpp
index 3bbb3c8e1c045..4afd901df639b 100644
--- a/libcxx/test/std/containers/sequences/array/empty.fail.cpp
+++ b/libcxx/test/std/containers/sequences/array/empty.fail.cpp
@@ -15,6 +15,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 #include <array>
 
@@ -24,9 +25,9 @@ int main(int, char**)
 {
 
     std::array<int, 1> c;
-    c.empty(); // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
     std::array<int, 0> c0;
-    c0.empty(); // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    c0.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/deque/deque.capacity/empty.fail.cpp b/libcxx/test/std/containers/sequences/deque/deque.capacity/empty.fail.cpp
index 79e4b30b2b27d..1f38354d5a95f 100644
--- a/libcxx/test/std/containers/sequences/deque/deque.capacity/empty.fail.cpp
+++ b/libcxx/test/std/containers/sequences/deque/deque.capacity/empty.fail.cpp
@@ -15,6 +15,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 #include <deque>
 
@@ -23,7 +24,7 @@
 int main(int, char**)
 {
     std::deque<int> c;
-    c.empty();  // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/forwardlist/empty.fail.cpp b/libcxx/test/std/containers/sequences/forwardlist/empty.fail.cpp
index 2ca3e024e661a..da74faa6dca78 100644
--- a/libcxx/test/std/containers/sequences/forwardlist/empty.fail.cpp
+++ b/libcxx/test/std/containers/sequences/forwardlist/empty.fail.cpp
@@ -15,6 +15,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 #include <forward_list>
 
@@ -23,7 +24,7 @@
 int main(int, char**)
 {
     std::forward_list<int> c;
-    c.empty();  // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.capacity/empty.fail.cpp b/libcxx/test/std/containers/sequences/list/list.capacity/empty.fail.cpp
index 0cbaa464d4a1d..d8b6216dd675f 100644
--- a/libcxx/test/std/containers/sequences/list/list.capacity/empty.fail.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.capacity/empty.fail.cpp
@@ -15,6 +15,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 #include <list>
 
@@ -23,7 +24,7 @@
 int main(int, char**)
 {
     std::list<int> c;
-    c.empty();  // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/nothing_to_do.pass.cpp b/libcxx/test/std/containers/sequences/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/containers/sequences/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/containers/sequences/vector.bool/empty.fail.cpp b/libcxx/test/std/containers/sequences/vector.bool/empty.fail.cpp
index 0a84eb79d125d..02d737b96726d 100644
--- a/libcxx/test/std/containers/sequences/vector.bool/empty.fail.cpp
+++ b/libcxx/test/std/containers/sequences/vector.bool/empty.fail.cpp
@@ -15,6 +15,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 #include <vector>
 
@@ -23,7 +24,7 @@
 int main(int, char**)
 {
     std::vector<bool> c;
-    c.empty();  // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/vector/vector.capacity/empty.fail.cpp b/libcxx/test/std/containers/sequences/vector/vector.capacity/empty.fail.cpp
index 0f7dc6c64cc00..82195b24b502e 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.capacity/empty.fail.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.capacity/empty.fail.cpp
@@ -15,6 +15,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 #include <vector>
 
@@ -23,7 +24,7 @@
 int main(int, char**)
 {
     std::vector<int> c;
-    c.empty();  // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/empty.fail.cpp b/libcxx/test/std/containers/unord/unord.map/empty.fail.cpp
index 283d6fa3e243b..1f39c7a6554c4 100644
--- a/libcxx/test/std/containers/unord/unord.map/empty.fail.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/empty.fail.cpp
@@ -15,6 +15,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 #include <unordered_map>
 
@@ -23,7 +24,7 @@
 int main(int, char**)
 {
     std::unordered_map<int, int> c;
-    c.empty();  // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/empty.fail.cpp b/libcxx/test/std/containers/unord/unord.multimap/empty.fail.cpp
index 4eb7bfc9c36c4..735ad197b4fdc 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/empty.fail.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/empty.fail.cpp
@@ -15,6 +15,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 #include <unordered_map>
 
@@ -23,7 +24,7 @@
 int main(int, char**)
 {
     std::unordered_multimap<int, int> c;
-    c.empty();  // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/empty.fail.cpp b/libcxx/test/std/containers/unord/unord.multiset/empty.fail.cpp
index 449b2116a547c..d0eba659c8c6e 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/empty.fail.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/empty.fail.cpp
@@ -15,6 +15,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 #include <unordered_set>
 
@@ -23,7 +24,7 @@
 int main(int, char**)
 {
     std::unordered_multiset<int> c;
-    c.empty();  // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/empty.fail.cpp b/libcxx/test/std/containers/unord/unord.set/empty.fail.cpp
index 11273f44fceb1..2eeeafbd47387 100644
--- a/libcxx/test/std/containers/unord/unord.set/empty.fail.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/empty.fail.cpp
@@ -15,6 +15,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 #include <unordered_set>
 
@@ -23,7 +24,7 @@
 int main(int, char**)
 {
     std::unordered_set<int> c;
-    c.empty();  // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/depr/depr.auto.ptr/nothing_to_do.pass.cpp b/libcxx/test/std/depr/depr.auto.ptr/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/depr/depr.auto.ptr/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/depr/depr.c.headers/stdint_h.sh.cpp b/libcxx/test/std/depr/depr.c.headers/stdint_h.sh.cpp
index bcc8a11343e43..e385669a93a38 100644
--- a/libcxx/test/std/depr/depr.c.headers/stdint_h.sh.cpp
+++ b/libcxx/test/std/depr/depr.c.headers/stdint_h.sh.cpp
@@ -12,8 +12,8 @@
 // _STD_TYPES_T, stdint.h can be entered to get to macros like UINT32_MAX.
 //
 // REQUIRES: aix
-// RUN: %compile -c
-// RUN: %compile -c -D_XOPEN_SOURCE=700
+// RUN: %{compile} -c
+// RUN: %{compile} -c -D_XOPEN_SOURCE=700
 
 // test <stdint.h>
 //
diff --git a/libcxx/test/std/depr/depr.function.objects/depr.adaptors/nothing_to_do.pass.cpp b/libcxx/test/std/depr/depr.function.objects/depr.adaptors/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/depr/depr.function.objects/depr.adaptors/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/depr/depr.function.objects/nothing_to_do.pass.cpp b/libcxx/test/std/depr/depr.function.objects/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/depr/depr.function.objects/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/depr/depr.lib.binders/depr.lib.bind.1st/bind1st.depr_in_cxx11.fail.cpp b/libcxx/test/std/depr/depr.lib.binders/depr.lib.bind.1st/bind1st.depr_in_cxx11.fail.cpp
index 65cde496c1a72..9efea4be04040 100644
--- a/libcxx/test/std/depr/depr.lib.binders/depr.lib.bind.1st/bind1st.depr_in_cxx11.fail.cpp
+++ b/libcxx/test/std/depr/depr.lib.binders/depr.lib.bind.1st/bind1st.depr_in_cxx11.fail.cpp
@@ -24,7 +24,7 @@
 
 int main(int, char**)
 {
-    std::bind1st(test_func(1), 5); // expected-error{{'bind1st<test_func, int>' is deprecated}}
+    std::bind1st(test_func(1), 5); // expected-warning {{'bind1st<test_func, int>' is deprecated}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/depr/depr.lib.binders/depr.lib.bind.2nd/bind2nd.depr_in_cxx11.fail.cpp b/libcxx/test/std/depr/depr.lib.binders/depr.lib.bind.2nd/bind2nd.depr_in_cxx11.fail.cpp
index 7a98e7991446e..a1c8081436bd7 100644
--- a/libcxx/test/std/depr/depr.lib.binders/depr.lib.bind.2nd/bind2nd.depr_in_cxx11.fail.cpp
+++ b/libcxx/test/std/depr/depr.lib.binders/depr.lib.bind.2nd/bind2nd.depr_in_cxx11.fail.cpp
@@ -24,7 +24,7 @@
 
 int main(int, char**)
 {
-    std::bind2nd(test_func(1), 5); // expected-error{{'bind2nd<test_func, int>' is deprecated}}
+    std::bind2nd(test_func(1), 5); // expected-warning {{'bind2nd<test_func, int>' is deprecated}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/depr/depr.lib.binders/depr.lib.binder.1st/binder1st.depr_in_cxx11.fail.cpp b/libcxx/test/std/depr/depr.lib.binders/depr.lib.binder.1st/binder1st.depr_in_cxx11.fail.cpp
index b66ca3be92514..6a5bc321a009e 100644
--- a/libcxx/test/std/depr/depr.lib.binders/depr.lib.binder.1st/binder1st.depr_in_cxx11.fail.cpp
+++ b/libcxx/test/std/depr/depr.lib.binders/depr.lib.binder.1st/binder1st.depr_in_cxx11.fail.cpp
@@ -24,7 +24,7 @@
 
 int main(int, char**)
 {
-    typedef std::binder1st<test_func> B1ST; // expected-error{{'binder1st<test_func>' is deprecated}}
+    typedef std::binder1st<test_func> B1ST; // expected-warning {{'binder1st<test_func>' is deprecated}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/depr/depr.lib.binders/depr.lib.binder.2nd/binder2nd.depr_in_cxx11.fail.cpp b/libcxx/test/std/depr/depr.lib.binders/depr.lib.binder.2nd/binder2nd.depr_in_cxx11.fail.cpp
index 8afc9eed6026a..6ec5b6efd9108 100644
--- a/libcxx/test/std/depr/depr.lib.binders/depr.lib.binder.2nd/binder2nd.depr_in_cxx11.fail.cpp
+++ b/libcxx/test/std/depr/depr.lib.binders/depr.lib.binder.2nd/binder2nd.depr_in_cxx11.fail.cpp
@@ -24,7 +24,7 @@
 
 int main(int, char**)
 {
-    typedef std::binder2nd<test_func> B2ND; // expected-error{{'binder2nd<test_func>' is deprecated}}
+    typedef std::binder2nd<test_func> B2ND; // expected-warning {{'binder2nd<test_func>' is deprecated}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/depr/depr.lib.binders/nothing_to_do.pass.cpp b/libcxx/test/std/depr/depr.lib.binders/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/depr/depr.lib.binders/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/depr/exception.unexpected/nothing_to_do.pass.cpp b/libcxx/test/std/depr/exception.unexpected/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/depr/exception.unexpected/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/depr/nothing_to_do.pass.cpp b/libcxx/test/std/depr/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/depr/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/diagnostics/nothing_to_do.pass.cpp b/libcxx/test/std/diagnostics/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/diagnostics/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/diagnostics/syserr/syserr.errcat/nothing_to_do.pass.cpp b/libcxx/test/std/diagnostics/syserr/syserr.errcat/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/diagnostics/syserr/syserr.errcat/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/diagnostics/syserr/syserr.errcode/nothing_to_do.pass.cpp b/libcxx/test/std/diagnostics/syserr/syserr.errcode/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/diagnostics/syserr/syserr.errcode/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/diagnostics/syserr/syserr.errcondition/nothing_to_do.pass.cpp b/libcxx/test/std/diagnostics/syserr/syserr.errcondition/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/diagnostics/syserr/syserr.errcondition/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/diagnostics/syserr/syserr.syserr/nothing_to_do.pass.cpp b/libcxx/test/std/diagnostics/syserr/syserr.syserr/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/diagnostics/syserr/syserr.syserr/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/experimental/filesystem/lit.local.cfg b/libcxx/test/std/experimental/filesystem/lit.local.cfg
index 1d0cf19256ef6..460a0b8f7a255 100644
--- a/libcxx/test/std/experimental/filesystem/lit.local.cfg
+++ b/libcxx/test/std/experimental/filesystem/lit.local.cfg
@@ -1 +1 @@
-config.test_format.cxx.compile_flags += ['-D_LIBCPP_NO_EXPERIMENTAL_DEPRECATION_WARNING_FILESYSTEM']
+config.test_format.addCompileFlags(config, '-D_LIBCPP_NO_EXPERIMENTAL_DEPRECATION_WARNING_FILESYSTEM')
diff --git a/libcxx/test/std/experimental/func/func.searchers/nothing_to_do.pass.cpp b/libcxx/test/std/experimental/func/func.searchers/nothing_to_do.pass.cpp
deleted file mode 100644
index 779762e7e995d..0000000000000
--- a/libcxx/test/std/experimental/func/func.searchers/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,14 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/experimental/language.support/support.coroutines/lit.local.cfg b/libcxx/test/std/experimental/language.support/support.coroutines/lit.local.cfg
index a0b3de8afaa39..d32e99fc2721a 100644
--- a/libcxx/test/std/experimental/language.support/support.coroutines/lit.local.cfg
+++ b/libcxx/test/std/experimental/language.support/support.coroutines/lit.local.cfg
@@ -4,6 +4,4 @@
 if 'fcoroutines-ts' not in config.available_features:
   config.unsupported = True
 else:
-  import copy
-  config.test_format.cxx = copy.deepcopy(config.test_format.cxx)
-  config.test_format.cxx.compile_flags += ['-fcoroutines-ts']
+  config.test_format.addCompileFlags(config, '-fcoroutines-ts')
diff --git a/libcxx/test/std/experimental/memory/nothing_to_do.pass.cpp b/libcxx/test/std/experimental/memory/nothing_to_do.pass.cpp
deleted file mode 100644
index 779762e7e995d..0000000000000
--- a/libcxx/test/std/experimental/memory/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,14 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/experimental/nothing_to_do.pass.cpp b/libcxx/test/std/experimental/nothing_to_do.pass.cpp
deleted file mode 100644
index 796f3c353ba17..0000000000000
--- a/libcxx/test/std/experimental/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,11 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**) {
-  return 0;
-}
diff --git a/libcxx/test/std/experimental/utilities/nothing_to_do.pass.cpp b/libcxx/test/std/experimental/utilities/nothing_to_do.pass.cpp
deleted file mode 100644
index 779762e7e995d..0000000000000
--- a/libcxx/test/std/experimental/utilities/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,14 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/input.output/file.streams/fstreams/filebuf.virtuals/pbackfail.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/filebuf.virtuals/pbackfail.pass.cpp
index 728eec2938e11..03aff1681227c 100644
--- a/libcxx/test/std/input.output/file.streams/fstreams/filebuf.virtuals/pbackfail.pass.cpp
+++ b/libcxx/test/std/input.output/file.streams/fstreams/filebuf.virtuals/pbackfail.pass.cpp
@@ -10,6 +10,8 @@
 
 // int_type pbackfail(int_type c = traits::eof());
 
+// FILE_DEPENDENCIES: underflow.dat
+
 #include <fstream>
 #include <cassert>
 
diff --git a/libcxx/test/std/input.output/file.streams/nothing_to_do.pass.cpp b/libcxx/test/std/input.output/file.streams/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/input.output/file.streams/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.cons/path.pass.cpp b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.cons/path.pass.cpp
index 5188427b90956..850be12e839c0 100644
--- a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.cons/path.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.cons/path.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.mods/replace_filename.pass.cpp b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.mods/replace_filename.pass.cpp
index 8ceb48ba50daa..2a2146be1b849 100644
--- a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.mods/replace_filename.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.mods/replace_filename.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/file_size.pass.cpp b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/file_size.pass.cpp
index 0ebbb293c99dc..8ff1b46b3b992 100644
--- a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/file_size.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/file_size.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/hard_link_count.pass.cpp b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/hard_link_count.pass.cpp
index c7d30d1919090..d9958a6c5f030 100644
--- a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/hard_link_count.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/hard_link_count.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/last_write_time.pass.cpp b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/last_write_time.pass.cpp
index fd6c81cd7300b..de6c36c218f2b 100644
--- a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/last_write_time.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/last_write_time.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.members/copy.pass.cpp b/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.members/copy.pass.cpp
index ca607ad10358d..d01854c1e4037 100644
--- a/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.members/copy.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.members/copy.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.members/copy_assign.pass.cpp b/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.members/copy_assign.pass.cpp
index 966501b6abd4a..8d33240791052 100644
--- a/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.members/copy_assign.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.members/copy_assign.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.members/ctor.pass.cpp b/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.members/ctor.pass.cpp
index 325228a5e619a..a06c4a64c7ffa 100644
--- a/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.members/ctor.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.members/ctor.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.members/increment.pass.cpp b/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.members/increment.pass.cpp
index ca954e9b2e5e6..c527013237ac6 100644
--- a/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.members/increment.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.members/increment.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.members/move.pass.cpp b/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.members/move.pass.cpp
index 055b157523e95..c4d0e23b46f8d 100644
--- a/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.members/move.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.members/move.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.members/move_assign.pass.cpp b/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.members/move_assign.pass.cpp
index de769546f8eef..bf737146d585c 100644
--- a/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.members/move_assign.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.members/move_assign.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.nonmembers/begin_end.pass.cpp b/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.nonmembers/begin_end.pass.cpp
index 31344524a428c..5b1205edc6dbc 100644
--- a/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.nonmembers/begin_end.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.directory_iterator/directory_iterator.nonmembers/begin_end.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.decompose/empty.fail.cpp b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.decompose/empty.fail.cpp
index dbadf468805d9..1371fa10317b6 100644
--- a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.decompose/empty.fail.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.decompose/empty.fail.cpp
@@ -15,6 +15,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 #include "filesystem_include.h"
 
@@ -23,7 +24,7 @@
 int main(int, char**)
 {
     fs::path c;
-    c.empty();  // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/copy.pass.cpp b/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/copy.pass.cpp
index 695ac374e2de2..bff2a18dd70a5 100644
--- a/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/copy.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/copy.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/copy_assign.pass.cpp b/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/copy_assign.pass.cpp
index 2fcebe96fcdb5..d7468a21c4ed9 100644
--- a/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/copy_assign.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/copy_assign.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/ctor.pass.cpp b/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/ctor.pass.cpp
index 538f547f30849..177edb46e058d 100644
--- a/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/ctor.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/ctor.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/depth.pass.cpp b/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/depth.pass.cpp
index 23adbbe250736..da6c10a88cc99 100644
--- a/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/depth.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/depth.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/disable_recursion_pending.pass.cpp b/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/disable_recursion_pending.pass.cpp
index eb5e63fc1c042..5d29b7e32f60f 100644
--- a/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/disable_recursion_pending.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/disable_recursion_pending.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/increment.pass.cpp b/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/increment.pass.cpp
index 799cedf055c17..1c7650543396b 100644
--- a/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/increment.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/increment.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/move.pass.cpp b/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/move.pass.cpp
index 316f8e6f57ed1..d709012bef8cc 100644
--- a/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/move.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/move.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/move_assign.pass.cpp b/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/move_assign.pass.cpp
index b8dd1168cc5e7..2dee77c71731e 100644
--- a/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/move_assign.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/move_assign.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/pop.pass.cpp b/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/pop.pass.cpp
index 55325e270eccb..c6ecdae56db3c 100644
--- a/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/pop.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/pop.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/recursion_pending.pass.cpp b/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/recursion_pending.pass.cpp
index 120491a5aed57..6bb7b939554a1 100644
--- a/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/recursion_pending.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/recursion_pending.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.nonmembers/begin_end.pass.cpp b/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.nonmembers/begin_end.pass.cpp
index f1869ac988c5b..24eaf84c2931a 100644
--- a/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.nonmembers/begin_end.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.nonmembers/begin_end.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.canonical/canonical.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.canonical/canonical.pass.cpp
index 9009e7d5d1044..de2fa54aca482 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.canonical/canonical.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.canonical/canonical.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.current_path/current_path.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.current_path/current_path.pass.cpp
index cb2d0d69c8a46..cae78c2f56140 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.current_path/current_path.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.current_path/current_path.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.equivalent/equivalent.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.equivalent/equivalent.pass.cpp
index 382e9bbbfcce7..71fce47266612 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.equivalent/equivalent.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.equivalent/equivalent.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.file_size/file_size.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.file_size/file_size.pass.cpp
index 200358df1df4e..6f27a4cb57124 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.file_size/file_size.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.file_size/file_size.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.hard_lk_ct/hard_link_count.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.hard_lk_ct/hard_link_count.pass.cpp
index 7a204ede109ab..96da42be0bc76 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.hard_lk_ct/hard_link_count.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.hard_lk_ct/hard_link_count.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_directory/is_directory.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_directory/is_directory.pass.cpp
index 73dc53807884a..35dd5d40f7b0d 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_directory/is_directory.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_directory/is_directory.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_empty/is_empty.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_empty/is_empty.pass.cpp
index affc77f48896f..e3393240aa406 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_empty/is_empty.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_empty/is_empty.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_symlink/is_symlink.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_symlink/is_symlink.pass.cpp
index 97b61427f39e2..75ab6059f4d89 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_symlink/is_symlink.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_symlink/is_symlink.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.last_write_time/last_write_time.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.last_write_time/last_write_time.pass.cpp
index a66d4bff5d26d..3ef4c53303ad8 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.last_write_time/last_write_time.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.last_write_time/last_write_time.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.relative/relative.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.relative/relative.pass.cpp
index cd3c133e09025..2a8d8296c861e 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.relative/relative.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.relative/relative.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.space/space.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.space/space.pass.cpp
index f92f21acfd831..4a0936b920db4 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.space/space.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.space/space.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.status/status.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.status/status.pass.cpp
index 2c7c673645a0a..60c99ee923d5a 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.status/status.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.status/status.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.symlink_status/symlink_status.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.symlink_status/symlink_status.pass.cpp
index 88bef42f5e2ad..350076fcf3709 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.symlink_status/symlink_status.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.symlink_status/symlink_status.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.weakly_canonical/weakly_canonical.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.weakly_canonical/weakly_canonical.pass.cpp
index ed9105652c9f0..35d97b8703d59 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.weakly_canonical/weakly_canonical.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.weakly_canonical/weakly_canonical.pass.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+// FILE_DEPENDENCIES: ../../Inputs/static_test_env
 // UNSUPPORTED: c++98, c++03
 
 // <filesystem>
diff --git a/libcxx/test/std/input.output/filesystems/lit.local.cfg b/libcxx/test/std/input.output/filesystems/lit.local.cfg
index 132b73ccbde55..38de64e5a0828 100644
--- a/libcxx/test/std/input.output/filesystems/lit.local.cfg
+++ b/libcxx/test/std/input.output/filesystems/lit.local.cfg
@@ -1,3 +1,6 @@
+import os
+import sys
+
 # Disable all of the filesystem tests if the dylib under test doesn't support them.
 if 'dylib-has-no-filesystem' in config.available_features:
   config.unsupported = True
diff --git a/libcxx/test/std/input.output/iostream.format/input.streams/istream.formatted/nothing_to_do.pass.cpp b/libcxx/test/std/input.output/iostream.format/input.streams/istream.formatted/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/input.output/iostream.format/input.streams/istream.formatted/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/input.output/iostream.format/nothing_to_do.pass.cpp b/libcxx/test/std/input.output/iostream.format/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/input.output/iostream.format/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/input.output/iostream.format/output.streams/ostream.formatted/nothing_to_do.pass.cpp b/libcxx/test/std/input.output/iostream.format/output.streams/ostream.formatted/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/input.output/iostream.format/output.streams/ostream.formatted/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/input.output/iostreams.base/fpos/nothing_to_do.pass.cpp b/libcxx/test/std/input.output/iostreams.base/fpos/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/input.output/iostreams.base/fpos/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/input.output/iostreams.base/ios.base/ios.types/nothing_to_do.pass.cpp b/libcxx/test/std/input.output/iostreams.base/ios.base/ios.types/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/input.output/iostreams.base/ios.base/ios.types/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/input.output/iostreams.base/std.ios.manip/nothing_to_do.pass.cpp b/libcxx/test/std/input.output/iostreams.base/std.ios.manip/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/input.output/iostreams.base/std.ios.manip/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/input.output/iostreams.requirements/nothing_to_do.pass.cpp b/libcxx/test/std/input.output/iostreams.requirements/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/input.output/iostreams.requirements/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/input.output/nothing_to_do.pass.cpp b/libcxx/test/std/input.output/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/input.output/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/input.output/stream.buffers/streambuf/streambuf.members/nothing_to_do.pass.cpp b/libcxx/test/std/input.output/stream.buffers/streambuf/streambuf.members/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/input.output/stream.buffers/streambuf/streambuf.members/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/input.output/stream.buffers/streambuf/streambuf.protected/nothing_to_do.pass.cpp b/libcxx/test/std/input.output/stream.buffers/streambuf/streambuf.protected/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/input.output/stream.buffers/streambuf/streambuf.protected/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/input.output/stream.buffers/streambuf/streambuf.virtuals/nothing_to_do.pass.cpp b/libcxx/test/std/input.output/stream.buffers/streambuf/streambuf.virtuals/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/input.output/stream.buffers/streambuf/streambuf.virtuals/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/iterators/iterator.container/empty.array.fail.cpp b/libcxx/test/std/iterators/iterator.container/empty.array.fail.cpp
index 08014776c9df3..98871f1094091 100644
--- a/libcxx/test/std/iterators/iterator.container/empty.array.fail.cpp
+++ b/libcxx/test/std/iterators/iterator.container/empty.array.fail.cpp
@@ -14,6 +14,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 #include <vector>
 #include <iterator>
@@ -23,7 +24,7 @@
 int main(int, char**)
 {
     int c[5];
-    std::empty(c);  // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    std::empty(c); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/iterators/iterator.container/empty.container.fail.cpp b/libcxx/test/std/iterators/iterator.container/empty.container.fail.cpp
index 153ea13b643e1..3f1179f2f10f8 100644
--- a/libcxx/test/std/iterators/iterator.container/empty.container.fail.cpp
+++ b/libcxx/test/std/iterators/iterator.container/empty.container.fail.cpp
@@ -14,6 +14,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 #include <vector>
 #include <iterator>
@@ -23,7 +24,7 @@
 int main(int, char**)
 {
     std::vector<int> c;
-    std::empty(c);  // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    std::empty(c); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/iterators/iterator.container/empty.initializer_list.fail.cpp b/libcxx/test/std/iterators/iterator.container/empty.initializer_list.fail.cpp
index 5dafb5119fe91..cd8ccb8eca70c 100644
--- a/libcxx/test/std/iterators/iterator.container/empty.initializer_list.fail.cpp
+++ b/libcxx/test/std/iterators/iterator.container/empty.initializer_list.fail.cpp
@@ -14,6 +14,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 #include <initializer_list>
 #include <iterator>
@@ -23,7 +24,7 @@
 int main(int, char**)
 {
     std::initializer_list<int> c = { 4 };
-    std::empty(c);  // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    std::empty(c); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/iterators/iterator.primitives/nothing_to_do.pass.cpp b/libcxx/test/std/iterators/iterator.primitives/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/iterators/iterator.primitives/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/iterators/iterator.requirements/nothing_to_do.pass.cpp b/libcxx/test/std/iterators/iterator.requirements/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/iterators/iterator.requirements/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/iterators/iterators.general/nothing_to_do.pass.cpp b/libcxx/test/std/iterators/iterators.general/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/iterators/iterators.general/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iter.ops/nothing_to_do.pass.cpp b/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iter.ops/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/iterators/predef.iterators/insert.iterators/back.insert.iter.ops/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/nothing_to_do.pass.cpp b/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/iterators/predef.iterators/insert.iterators/front.insert.iter.ops/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/iterators/predef.iterators/insert.iterators/insert.iter.ops/nothing_to_do.pass.cpp b/libcxx/test/std/iterators/predef.iterators/insert.iterators/insert.iter.ops/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/iterators/predef.iterators/insert.iterators/insert.iter.ops/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/iterators/predef.iterators/insert.iterators/nothing_to_do.pass.cpp b/libcxx/test/std/iterators/predef.iterators/insert.iterators/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/iterators/predef.iterators/insert.iterators/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/iterators/predef.iterators/move.iterators/move.iter.ops/nothing_to_do.pass.cpp b/libcxx/test/std/iterators/predef.iterators/move.iterators/move.iter.ops/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/iterators/predef.iterators/move.iterators/move.iter.ops/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/iterators/predef.iterators/move.iterators/nothing_to_do.pass.cpp b/libcxx/test/std/iterators/predef.iterators/move.iterators/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/iterators/predef.iterators/move.iterators/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/iterators/predef.iterators/nothing_to_do.pass.cpp b/libcxx/test/std/iterators/predef.iterators/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/iterators/predef.iterators/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/iterators/predef.iterators/reverse.iterators/nothing_to_do.pass.cpp b/libcxx/test/std/iterators/predef.iterators/reverse.iterators/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/iterators/predef.iterators/reverse.iterators/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/iterators/predef.iterators/reverse.iterators/reverse.iter.ops/nothing_to_do.pass.cpp b/libcxx/test/std/iterators/predef.iterators/reverse.iterators/reverse.iter.ops/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/iterators/predef.iterators/reverse.iterators/reverse.iter.ops/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/iterators/stream.iterators/nothing_to_do.pass.cpp b/libcxx/test/std/iterators/stream.iterators/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/iterators/stream.iterators/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/language.support/nothing_to_do.pass.cpp b/libcxx/test/std/language.support/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/language.support/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/language.support/support.dynamic/alloc.errors/nothing_to_do.pass.cpp b/libcxx/test/std/language.support/support.dynamic/alloc.errors/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/language.support/support.dynamic/alloc.errors/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new_size.sh.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new_size.sh.cpp
index a04ceb6ad42ee..9d1423adc44b5 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new_size.sh.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new_size.sh.cpp
@@ -15,7 +15,7 @@
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
 
 // REQUIRES: -faligned-allocation
-// RUN: %compile %verify -faligned-allocation
+// RUN: %{compile} %{verify} -faligned-allocation
 
 #include <new>
 
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new_size_align.sh.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new_size_align.sh.cpp
index e2a61591ad2b5..8bd3294970e80 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new_size_align.sh.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new_size_align.sh.cpp
@@ -15,7 +15,7 @@
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
 
 // REQUIRES: -faligned-allocation
-// RUN: %compile %verify -faligned-allocation
+// RUN: %{compile} %{verify} -faligned-allocation
 
 #include <new>
 
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new_size_align_nothrow.sh.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new_size_align_nothrow.sh.cpp
index 5ad81863d4d82..aa7074054ff33 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new_size_align_nothrow.sh.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new_size_align_nothrow.sh.cpp
@@ -15,7 +15,7 @@
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
 
 // REQUIRES: -faligned-allocation
-// RUN: %compile %verify -faligned-allocation
+// RUN: %{compile} %{verify} -faligned-allocation
 
 #include <new>
 
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new_size_nothrow.sh.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new_size_nothrow.sh.cpp
index 53af2c764c005..11f8d4e0c8b26 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new_size_nothrow.sh.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new_size_nothrow.sh.cpp
@@ -15,7 +15,7 @@
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
 
 // REQUIRES: -faligned-allocation
-// RUN: %compile %verify -faligned-allocation
+// RUN: %{compile} %{verify} -faligned-allocation
 
 #include <new>
 
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/sized_delete_array_fsizeddeallocation.sh.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/sized_delete_array_fsizeddeallocation.pass.cpp
similarity index 98%
rename from libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/sized_delete_array_fsizeddeallocation.sh.cpp
rename to libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/sized_delete_array_fsizeddeallocation.pass.cpp
index 41739c0ccdc7f..c6e55469f19a3 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/sized_delete_array_fsizeddeallocation.sh.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/sized_delete_array_fsizeddeallocation.pass.cpp
@@ -21,9 +21,7 @@
 
 // NOTE: Only clang-3.7 and GCC 5.1 and greater support -fsized-deallocation.
 // REQUIRES: -fsized-deallocation
-
-// RUN: %build -fsized-deallocation
-// RUN: %run
+// ADDITIONAL_COMPILE_FLAGS: -fsized-deallocation
 
 #if !defined(__cpp_sized_deallocation)
 # error __cpp_sized_deallocation should be defined
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.placement/new_array_ptr.fail.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.placement/new_array_ptr.fail.cpp
index 4cba717dba805..2972e85a91369 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.placement/new_array_ptr.fail.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.placement/new_array_ptr.fail.cpp
@@ -13,6 +13,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 #include <new>
 
@@ -21,7 +22,7 @@
 int main(int, char**)
 {
     char buffer[100];
-    ::operator new[](4, buffer);  // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    ::operator new[](4, buffer); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.placement/new_ptr.fail.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.placement/new_ptr.fail.cpp
index 05a9b244c1e45..45853b58a4a64 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.placement/new_ptr.fail.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.placement/new_ptr.fail.cpp
@@ -13,6 +13,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 #include <new>
 
@@ -21,7 +22,7 @@
 int main(int, char**)
 {
     char buffer[100];
-    ::operator new(4, buffer);  // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    ::operator new(4, buffer); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new_size.fail.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new_size.fail.cpp
index 4769933d399a1..3c7ce2ed92131 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new_size.fail.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new_size.fail.cpp
@@ -13,6 +13,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 #include <new>
 
@@ -20,7 +21,7 @@
 
 int main(int, char**)
 {
-    ::operator new(4);  // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    ::operator new(4); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new_size_align.sh.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new_size_align.sh.cpp
index a0d99c76d0723..c062a221507fd 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new_size_align.sh.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new_size_align.sh.cpp
@@ -15,7 +15,7 @@
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
 
 // REQUIRES: -faligned-allocation
-// RUN: %compile %verify -faligned-allocation
+// RUN: %{compile} %{verify} -faligned-allocation
 
 #include <new>
 
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new_size_align_nothrow.sh.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new_size_align_nothrow.sh.cpp
index 54b25ac5589ac..8a51b0366f900 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new_size_align_nothrow.sh.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new_size_align_nothrow.sh.cpp
@@ -15,7 +15,7 @@
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
 
 // REQUIRES: -faligned-allocation
-// RUN: %compile %verify -faligned-allocation
+// RUN: %{compile} %{verify} -faligned-allocation
 
 #include <new>
 
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new_size_nothrow.fail.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new_size_nothrow.fail.cpp
index a0bfa8b7f921f..f620c4e990984 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new_size_nothrow.fail.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new_size_nothrow.fail.cpp
@@ -13,6 +13,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 #include <new>
 
@@ -20,7 +21,7 @@
 
 int main(int, char**)
 {
-    ::operator new(4, std::nothrow);  // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    ::operator new(4, std::nothrow); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/sized_delete_fsizeddeallocation.sh.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/sized_delete_fsizeddeallocation.pass.cpp
similarity index 97%
rename from libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/sized_delete_fsizeddeallocation.sh.cpp
rename to libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/sized_delete_fsizeddeallocation.pass.cpp
index 3d62040cbe81d..06e2c38fcadbb 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/sized_delete_fsizeddeallocation.sh.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/sized_delete_fsizeddeallocation.pass.cpp
@@ -20,9 +20,7 @@
 
 // NOTE: Only clang-3.7 and GCC 5.1 and greater support -fsized-deallocation.
 // REQUIRES: -fsized-deallocation
-
-// RUN: %build -fsized-deallocation -O3
-// RUN: %run
+// ADDITIONAL_COMPILE_FLAGS: -fsized-deallocation, -O3
 
 #if !defined(__cpp_sized_deallocation)
 # error __cpp_sized_deallocation should be defined
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/nothing_to_do.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/language.support/support.dynamic/ptr.launder/launder.nodiscard.fail.cpp b/libcxx/test/std/language.support/support.dynamic/ptr.launder/launder.nodiscard.fail.cpp
index e7252395d3531..bec35f5793925 100644
--- a/libcxx/test/std/language.support/support.dynamic/ptr.launder/launder.nodiscard.fail.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/ptr.launder/launder.nodiscard.fail.cpp
@@ -13,6 +13,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 #include <new>
 #include <cassert>
@@ -22,7 +23,7 @@
 int main(int, char**)
 {
     int *p = nullptr;
-    std::launder(p);  // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    std::launder(p); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/language.support/support.exception/exception.terminate/nothing_to_do.pass.cpp b/libcxx/test/std/language.support/support.exception/exception.terminate/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/language.support/support.exception/exception.terminate/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/language.support/support.limits/nothing_to_do.pass.cpp b/libcxx/test/std/language.support/support.limits/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/language.support/support.limits/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/localization/locale.categories/category.collate/nothing_to_do.pass.cpp b/libcxx/test/std/localization/locale.categories/category.collate/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/localization/locale.categories/category.collate/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/localization/locale.categories/category.messages/nothing_to_do.pass.cpp b/libcxx/test/std/localization/locale.categories/category.messages/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/localization/locale.categories/category.messages/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/localization/locale.categories/category.monetary/nothing_to_do.pass.cpp b/libcxx/test/std/localization/locale.categories/category.monetary/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/localization/locale.categories/category.monetary/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/localization/locale.categories/category.numeric/nothing_to_do.pass.cpp b/libcxx/test/std/localization/locale.categories/category.numeric/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/localization/locale.categories/category.numeric/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/localization/locale.categories/category.time/nothing_to_do.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/localization/locale.categories/category.time/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/localization/locale.categories/facet.numpunct/nothing_to_do.pass.cpp b/libcxx/test/std/localization/locale.categories/facet.numpunct/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/localization/locale.categories/facet.numpunct/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/localization/locales/locale.convenience/conversions/nothing_to_do.pass.cpp b/libcxx/test/std/localization/locales/locale.convenience/conversions/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/localization/locales/locale.convenience/conversions/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/localization/locales/locale.convenience/nothing_to_do.pass.cpp b/libcxx/test/std/localization/locales/locale.convenience/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/localization/locales/locale.convenience/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/localization/locales/locale/locale.types/nothing_to_do.pass.cpp b/libcxx/test/std/localization/locales/locale/locale.types/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/localization/locales/locale/locale.types/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/localization/locales/locale/nothing_to_do.pass.cpp b/libcxx/test/std/localization/locales/locale/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/localization/locales/locale/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/localization/locales/nothing_to_do.pass.cpp b/libcxx/test/std/localization/locales/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/localization/locales/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/nothing_to_do.pass.cpp b/libcxx/test/std/nothing_to_do.pass.cpp
deleted file mode 100644
index 779762e7e995d..0000000000000
--- a/libcxx/test/std/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,14 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/numerics/bit/nothing_to_do.pass.cpp b/libcxx/test/std/numerics/bit/nothing_to_do.pass.cpp
deleted file mode 100644
index b58f5c55b643a..0000000000000
--- a/libcxx/test/std/numerics/bit/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,12 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-int main()
-{
-}
diff --git a/libcxx/test/std/numerics/c.math/abs.fail.cpp b/libcxx/test/std/numerics/c.math/abs.fail.cpp
index 16f112c30fce3..d58cf0d563d18 100644
--- a/libcxx/test/std/numerics/c.math/abs.fail.cpp
+++ b/libcxx/test/std/numerics/c.math/abs.fail.cpp
@@ -16,10 +16,10 @@ int main(int, char**)
     ui = std::abs(ui); // expected-error {{call to 'abs' is ambiguous}}
 
     unsigned char uc = -5;
-    uc = std::abs(uc); // expected-error {{taking the absolute value of unsigned type 'unsigned char' has no effect}}
+    uc = std::abs(uc); // expected-warning {{taking the absolute value of unsigned type 'unsigned char' has no effect}}
 
     unsigned short us = -5;
-    us = std::abs(us); // expected-error {{taking the absolute value of unsigned type 'unsigned short' has no effect}}
+    us = std::abs(us); // expected-warning {{taking the absolute value of unsigned type 'unsigned short' has no effect}}
 
     unsigned long ul = -5;
     ul = std::abs(ul); // expected-error {{call to 'abs' is ambiguous}}
diff --git a/libcxx/test/std/numerics/nothing_to_do.pass.cpp b/libcxx/test/std/numerics/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/numerics/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/numerics/numarray/class.gslice/nothing_to_do.pass.cpp b/libcxx/test/std/numerics/numarray/class.gslice/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/numerics/numarray/class.gslice/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/numerics/numarray/class.slice/nothing_to_do.pass.cpp b/libcxx/test/std/numerics/numarray/class.slice/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/numerics/numarray/class.slice/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/numerics/numarray/valarray.nonmembers/nothing_to_do.pass.cpp b/libcxx/test/std/numerics/numarray/valarray.nonmembers/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/numerics/numarray/valarray.nonmembers/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/numerics/rand/nothing_to_do.pass.cpp b/libcxx/test/std/numerics/rand/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/numerics/rand/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/numerics/rand/rand.adapt/nothing_to_do.pass.cpp b/libcxx/test/std/numerics/rand/rand.adapt/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/numerics/rand/rand.adapt/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/numerics/rand/rand.dis/nothing_to_do.pass.cpp b/libcxx/test/std/numerics/rand/rand.dis/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/numerics/rand/rand.dis/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/numerics/rand/rand.dis/rand.dist.bern/nothing_to_do.pass.cpp b/libcxx/test/std/numerics/rand/rand.dis/rand.dist.bern/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/numerics/rand/rand.dis/rand.dist.bern/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/numerics/rand/rand.dis/rand.dist.norm/nothing_to_do.pass.cpp b/libcxx/test/std/numerics/rand/rand.dis/rand.dist.norm/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/numerics/rand/rand.dis/rand.dist.norm/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/numerics/rand/rand.dis/rand.dist.pois/nothing_to_do.pass.cpp b/libcxx/test/std/numerics/rand/rand.dis/rand.dist.pois/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/numerics/rand/rand.dis/rand.dist.pois/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/numerics/rand/rand.dis/rand.dist.samp/nothing_to_do.pass.cpp b/libcxx/test/std/numerics/rand/rand.dis/rand.dist.samp/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/numerics/rand/rand.dis/rand.dist.samp/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/numerics/rand/rand.dis/rand.dist.uni/nothing_to_do.pass.cpp b/libcxx/test/std/numerics/rand/rand.dis/rand.dist.uni/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/numerics/rand/rand.dis/rand.dist.uni/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/numerics/rand/rand.eng/nothing_to_do.pass.cpp b/libcxx/test/std/numerics/rand/rand.eng/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/numerics/rand/rand.eng/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/numerics/rand/rand.req/nothing_to_do.pass.cpp b/libcxx/test/std/numerics/rand/rand.req/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/numerics/rand/rand.req/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/numerics/rand/rand.util/nothing_to_do.pass.cpp b/libcxx/test/std/numerics/rand/rand.util/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/numerics/rand/rand.util/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/re/nothing_to_do.pass.cpp b/libcxx/test/std/re/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/re/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/re/re.alg/nothing_to_do.pass.cpp b/libcxx/test/std/re/re.alg/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/re/re.alg/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/re/re.const/nothing_to_do.pass.cpp b/libcxx/test/std/re/re.const/nothing_to_do.pass.cpp
deleted file mode 100644
index 779762e7e995d..0000000000000
--- a/libcxx/test/std/re/re.const/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,14 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/re/re.def/nothing_to_do.pass.cpp b/libcxx/test/std/re/re.def/nothing_to_do.pass.cpp
deleted file mode 100644
index 779762e7e995d..0000000000000
--- a/libcxx/test/std/re/re.def/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,14 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/re/re.grammar/nothing_to_do.pass.cpp b/libcxx/test/std/re/re.grammar/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/re/re.grammar/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/re/re.iter/nothing_to_do.pass.cpp b/libcxx/test/std/re/re.iter/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/re/re.iter/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/re/re.regex/re.regex.nonmemb/nothing_to_do.pass.cpp b/libcxx/test/std/re/re.regex/re.regex.nonmemb/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/re/re.regex/re.regex.nonmemb/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/re/re.results/re.results.size/empty.fail.cpp b/libcxx/test/std/re/re.results/re.results.size/empty.fail.cpp
index e17c77424656f..f496e0fb415c6 100644
--- a/libcxx/test/std/re/re.results/re.results.size/empty.fail.cpp
+++ b/libcxx/test/std/re/re.results/re.results.size/empty.fail.cpp
@@ -14,6 +14,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 #include <regex>
 
@@ -22,7 +23,7 @@
 int main(int, char**)
 {
     std::match_results<const char*> c;
-    c.empty();  // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/strings/basic.string/string.capacity/empty.fail.cpp b/libcxx/test/std/strings/basic.string/string.capacity/empty.fail.cpp
index 1bfa388b7b9c6..ff1abd589722c 100644
--- a/libcxx/test/std/strings/basic.string/string.capacity/empty.fail.cpp
+++ b/libcxx/test/std/strings/basic.string/string.capacity/empty.fail.cpp
@@ -15,6 +15,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 #include <string>
 
@@ -23,7 +24,7 @@
 int main(int, char**)
 {
     std::string c;
-    c.empty();  // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    c.empty(); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/nothing_to_do.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/strings/basic.string/string.modifiers/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/strings/basic.string/string.nonmembers/nothing_to_do.pass.cpp b/libcxx/test/std/strings/basic.string/string.nonmembers/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/strings/basic.string/string.nonmembers/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/strings/basic.string/string.ops/nothing_to_do.pass.cpp b/libcxx/test/std/strings/basic.string/string.ops/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/strings/basic.string/string.ops/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/strings/char.traits/char.traits.specializations/nothing_to_do.pass.cpp b/libcxx/test/std/strings/char.traits/char.traits.specializations/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/strings/char.traits/char.traits.specializations/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/strings/char.traits/nothing_to_do.pass.cpp b/libcxx/test/std/strings/char.traits/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/strings/char.traits/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/strings/string.view/string.view.capacity/empty.fail.cpp b/libcxx/test/std/strings/string.view/string.view.capacity/empty.fail.cpp
index 1dd1dcf7cb5b2..cbb7b8f174da1 100644
--- a/libcxx/test/std/strings/string.view/string.view.capacity/empty.fail.cpp
+++ b/libcxx/test/std/strings/string.view/string.view.capacity/empty.fail.cpp
@@ -15,6 +15,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 #include <string_view>
 
@@ -23,7 +24,7 @@
 int main(int, char**)
 {
     std::string_view c;
-    c.empty();  // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    c.empty();  // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/thread/futures/futures.async/async.fail.cpp b/libcxx/test/std/thread/futures/futures.async/async.fail.cpp
index 3e7fb80e0e1fc..19c17d23e8917 100644
--- a/libcxx/test/std/thread/futures/futures.async/async.fail.cpp
+++ b/libcxx/test/std/thread/futures/futures.async/async.fail.cpp
@@ -9,6 +9,7 @@
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 // <future>
 
@@ -32,8 +33,8 @@ int foo (int x) { return x; }
 
 int main(int, char**)
 {
-    std::async(                    foo, 3); // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
-    std::async(std::launch::async, foo, 3); // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    std::async(                    foo, 3); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+    std::async(std::launch::async, foo, 3); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/thread/futures/futures.shared_future/wait_for.pass.cpp b/libcxx/test/std/thread/futures/futures.shared_future/wait_for.pass.cpp
index be53d4613c1c6..7e3cda0fd40be 100644
--- a/libcxx/test/std/thread/futures/futures.shared_future/wait_for.pass.cpp
+++ b/libcxx/test/std/thread/futures/futures.shared_future/wait_for.pass.cpp
@@ -9,7 +9,7 @@
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++98, c++03
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <future>
 
diff --git a/libcxx/test/std/thread/futures/futures.shared_future/wait_until.pass.cpp b/libcxx/test/std/thread/futures/futures.shared_future/wait_until.pass.cpp
index e45125901d934..9d4200e9f2f1e 100644
--- a/libcxx/test/std/thread/futures/futures.shared_future/wait_until.pass.cpp
+++ b/libcxx/test/std/thread/futures/futures.shared_future/wait_until.pass.cpp
@@ -9,7 +9,7 @@
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++98, c++03
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <future>
 
diff --git a/libcxx/test/std/thread/futures/futures.unique_future/wait_for.pass.cpp b/libcxx/test/std/thread/futures/futures.unique_future/wait_for.pass.cpp
index 258cd284eda77..d055f6468467a 100644
--- a/libcxx/test/std/thread/futures/futures.unique_future/wait_for.pass.cpp
+++ b/libcxx/test/std/thread/futures/futures.unique_future/wait_for.pass.cpp
@@ -8,7 +8,7 @@
 //
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++98, c++03
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <future>
 
diff --git a/libcxx/test/std/thread/futures/futures.unique_future/wait_until.pass.cpp b/libcxx/test/std/thread/futures/futures.unique_future/wait_until.pass.cpp
index c5dcc3ddbe6a9..b91261cd27ee3 100644
--- a/libcxx/test/std/thread/futures/futures.unique_future/wait_until.pass.cpp
+++ b/libcxx/test/std/thread/futures/futures.unique_future/wait_until.pass.cpp
@@ -8,7 +8,7 @@
 //
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++98, c++03
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <future>
 
diff --git a/libcxx/test/std/thread/thread.condition/thread.condition.condvar/wait_for.pass.cpp b/libcxx/test/std/thread/thread.condition/thread.condition.condvar/wait_for.pass.cpp
index 1bb74a30d45f4..3bead1b6ff892 100644
--- a/libcxx/test/std/thread/thread.condition/thread.condition.condvar/wait_for.pass.cpp
+++ b/libcxx/test/std/thread/thread.condition/thread.condition.condvar/wait_for.pass.cpp
@@ -8,7 +8,7 @@
 //
 // UNSUPPORTED: libcpp-has-no-threads
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <condition_variable>
 
diff --git a/libcxx/test/std/thread/thread.condition/thread.condition.condvar/wait_until_pred.pass.cpp b/libcxx/test/std/thread/thread.condition/thread.condition.condvar/wait_until_pred.pass.cpp
index fde6f98ab9858..ab43bdb1a02a0 100644
--- a/libcxx/test/std/thread/thread.condition/thread.condition.condvar/wait_until_pred.pass.cpp
+++ b/libcxx/test/std/thread/thread.condition/thread.condition.condvar/wait_until_pred.pass.cpp
@@ -8,7 +8,7 @@
 //
 // UNSUPPORTED: libcpp-has-no-threads
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <condition_variable>
 
diff --git a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/notify_one.pass.cpp b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/notify_one.pass.cpp
index 38b0cba8fc100..60a7dd7f10f7c 100644
--- a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/notify_one.pass.cpp
+++ b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/notify_one.pass.cpp
@@ -8,7 +8,7 @@
 //
 // UNSUPPORTED: libcpp-has-no-threads
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <condition_variable>
 
diff --git a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_for.pass.cpp b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_for.pass.cpp
index a8e036d94da6e..d89d54ace951c 100644
--- a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_for.pass.cpp
+++ b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_for.pass.cpp
@@ -8,7 +8,7 @@
 //
 // UNSUPPORTED: libcpp-has-no-threads
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <condition_variable>
 
diff --git a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_terminates.sh.cpp b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_terminates.sh.cpp
index 8afa0515920c7..0ee02919f2214 100644
--- a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_terminates.sh.cpp
+++ b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_terminates.sh.cpp
@@ -13,13 +13,14 @@
 
 // class condition_variable_any;
 
-// RUN: %build
-// RUN: %run 1
-// RUN: %run 2
-// RUN: %run 3
-// RUN: %run 4
-// RUN: %run 5
-// RUN: %run 6
+// FILE_DEPENDENCIES: %t.exe
+// RUN: %{build}
+// RUN: %{run} 1
+// RUN: %{run} 2
+// RUN: %{run} 3
+// RUN: %{run} 4
+// RUN: %{run} 5
+// RUN: %{run} 6
 
 // -----------------------------------------------------------------------------
 // Overview
diff --git a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_until_pred.pass.cpp b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_until_pred.pass.cpp
index 7cb48a6f68ec0..1c3e5814d0935 100644
--- a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_until_pred.pass.cpp
+++ b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_until_pred.pass.cpp
@@ -8,7 +8,7 @@
 //
 // UNSUPPORTED: libcpp-has-no-threads
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <condition_variable>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.guard/adopt_lock.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.guard/adopt_lock.pass.cpp
index dc0719596c69f..e42c16b31fcba 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.guard/adopt_lock.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.guard/adopt_lock.pass.cpp
@@ -8,7 +8,7 @@
 //
 // UNSUPPORTED: libcpp-has-no-threads
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.guard/mutex.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.guard/mutex.pass.cpp
index fa6aa4615aa83..fc2230591aea7 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.guard/mutex.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.guard/mutex.pass.cpp
@@ -8,7 +8,7 @@
 //
 // UNSUPPORTED: libcpp-has-no-threads
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex.pass.cpp
index d35a2490bdcb4..e008dcf937251 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex.pass.cpp
@@ -10,7 +10,7 @@
 // UNSUPPORTED: c++98, c++03, c++11
 // XFAIL: dylib-has-no-shared_mutex
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <shared_mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_duration.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_duration.pass.cpp
index c58e921882cfd..2df1b68efc1b0 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_duration.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_duration.pass.cpp
@@ -10,7 +10,7 @@
 // UNSUPPORTED: c++98, c++03, c++11
 // XFAIL: dylib-has-no-shared_mutex
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <shared_mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_time_point.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_time_point.pass.cpp
index 086cb484ab2e7..5453a55376f03 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_time_point.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_time_point.pass.cpp
@@ -10,7 +10,7 @@
 // UNSUPPORTED: c++98, c++03, c++11
 // XFAIL: dylib-has-no-shared_mutex
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <shared_mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_try_to_lock.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_try_to_lock.pass.cpp
index 630544b440fe4..b1957a27bb20b 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_try_to_lock.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/mutex_try_to_lock.pass.cpp
@@ -10,7 +10,7 @@
 // UNSUPPORTED: c++98, c++03, c++11
 // XFAIL: dylib-has-no-shared_mutex
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <shared_mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.locking/lock.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.locking/lock.pass.cpp
index 2d6ebe51092b5..3d2a72576189c 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.locking/lock.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.locking/lock.pass.cpp
@@ -10,7 +10,7 @@
 // UNSUPPORTED: c++98, c++03, c++11
 // XFAIL: dylib-has-no-shared_mutex
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <shared_mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.locking/try_lock.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.locking/try_lock.pass.cpp
index 6ce1d92b33710..74a237c162759 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.locking/try_lock.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.locking/try_lock.pass.cpp
@@ -9,7 +9,7 @@
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++98, c++03, c++11
 //
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <shared_mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.cons/mutex.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.cons/mutex.pass.cpp
index 61c0dacf17a47..c9e0c671fb74e 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.cons/mutex.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.cons/mutex.pass.cpp
@@ -8,7 +8,7 @@
 //
 // UNSUPPORTED: libcpp-has-no-threads
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.cons/mutex_duration.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.cons/mutex_duration.pass.cpp
index 110d2b91fcb14..fdc3c8b065b5e 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.cons/mutex_duration.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.cons/mutex_duration.pass.cpp
@@ -8,7 +8,7 @@
 //
 // UNSUPPORTED: libcpp-has-no-threads
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.cons/mutex_time_point.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.cons/mutex_time_point.pass.cpp
index 50d55427a95f2..2bac8784dbfbc 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.cons/mutex_time_point.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.cons/mutex_time_point.pass.cpp
@@ -8,7 +8,7 @@
 //
 // UNSUPPORTED: libcpp-has-no-threads
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.locking/lock.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.locking/lock.pass.cpp
index b8a0c2d343e55..b5425a79de365 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.locking/lock.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.locking/lock.pass.cpp
@@ -8,7 +8,7 @@
 //
 // UNSUPPORTED: libcpp-has-no-threads
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.locking/try_lock.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.locking/try_lock.pass.cpp
index 3ddfd75081c84..88ac9f843565f 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.locking/try_lock.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.locking/try_lock.pass.cpp
@@ -8,7 +8,7 @@
 //
 // UNSUPPORTED: libcpp-has-no-threads
 //
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/nothing_to_do.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.mutex.requirements.mutex/nothing_to_do.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.mutex.requirements.mutex/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.mutex.requirements.mutex/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.mutex.requirements.mutex/thread.mutex.class/lock.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.mutex.requirements.mutex/thread.mutex.class/lock.pass.cpp
index d955ccbe08895..3c2d15dfd5b8a 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.mutex.requirements.mutex/thread.mutex.class/lock.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.mutex.requirements.mutex/thread.mutex.class/lock.pass.cpp
@@ -8,7 +8,7 @@
 //
 // UNSUPPORTED: libcpp-has-no-threads
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.mutex.requirements.mutex/thread.mutex.class/try_lock.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.mutex.requirements.mutex/thread.mutex.class/try_lock.pass.cpp
index 3a6b8a3657198..5a6715c0c5c46 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.mutex.requirements.mutex/thread.mutex.class/try_lock.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.mutex.requirements.mutex/thread.mutex.class/try_lock.pass.cpp
@@ -8,7 +8,7 @@
 //
 // UNSUPPORTED: libcpp-has-no-threads
 //
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.mutex.requirements.mutex/thread.mutex.recursive/try_lock.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.mutex.requirements.mutex/thread.mutex.recursive/try_lock.pass.cpp
index fd8b652b6fabd..c0ab6ff5c5c5a 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.mutex.requirements.mutex/thread.mutex.recursive/try_lock.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.mutex.requirements.mutex/thread.mutex.recursive/try_lock.pass.cpp
@@ -8,7 +8,7 @@
 //
 // UNSUPPORTED: libcpp-has-no-threads
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/nothing_to_do.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/lock.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/lock.pass.cpp
index 74c66d0f672b8..bda2bab1d7709 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/lock.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/lock.pass.cpp
@@ -9,7 +9,7 @@
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++98, c++03, c++11, c++14
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <shared_mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/lock_shared.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/lock_shared.pass.cpp
index 7707af8d8b8fe..d7767bcee578d 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/lock_shared.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/lock_shared.pass.cpp
@@ -9,7 +9,7 @@
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++98, c++03, c++11, c++14
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <shared_mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/try_lock.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/try_lock.pass.cpp
index ad2094b1f65ea..f76b11889a8dc 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/try_lock.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/try_lock.pass.cpp
@@ -9,7 +9,7 @@
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++98, c++03, c++11, c++14
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <shared_mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/try_lock_shared.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/try_lock_shared.pass.cpp
index fb605f6db573c..477bd6f5e45d6 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/try_lock_shared.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.shared_mutex.requirements/thread.shared_mutex.class/try_lock_shared.pass.cpp
@@ -9,7 +9,7 @@
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++98, c++03, c++11, c++14
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <shared_mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/nothing_to_do.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/lock.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/lock.pass.cpp
index 56464b24005d9..9ccc5b6ed6264 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/lock.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/lock.pass.cpp
@@ -9,7 +9,7 @@
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++98, c++03, c++11
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <shared_mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/lock_shared.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/lock_shared.pass.cpp
index 08d35861e4708..17adc5fe4a8ec 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/lock_shared.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/lock_shared.pass.cpp
@@ -9,7 +9,7 @@
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++98, c++03, c++11
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <shared_mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock.pass.cpp
index 820c4a5ffa035..4448d25638434 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock.pass.cpp
@@ -9,7 +9,7 @@
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++98, c++03, c++11
 //
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <shared_mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_for.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_for.pass.cpp
index 45fea3e96fc66..6a6d07ec1f596 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_for.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_for.pass.cpp
@@ -9,7 +9,7 @@
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++98, c++03, c++11
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <shared_mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared.pass.cpp
index 830445af9a01d..c6e54ed7237f1 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared.pass.cpp
@@ -9,7 +9,7 @@
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++98, c++03, c++11
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <shared_mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared_for.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared_for.pass.cpp
index d89e4aa8cfc59..a5165acc7b707 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared_for.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared_for.pass.cpp
@@ -9,7 +9,7 @@
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++98, c++03, c++11
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <shared_mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared_until.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared_until.pass.cpp
index 5898fe41e0541..922ddb897983f 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared_until.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared_until.pass.cpp
@@ -9,7 +9,7 @@
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++98, c++03, c++11
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <shared_mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_until.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_until.pass.cpp
index 9539215c46fc9..edfcf231f9adf 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_until.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_until.pass.cpp
@@ -9,7 +9,7 @@
 // UNSUPPORTED: libcpp-has-no-threads
 // UNSUPPORTED: c++98, c++03, c++11
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <shared_mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.timedmutex.requirements/nothing_to_do.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.timedmutex.requirements/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.timedmutex.requirements/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.timedmutex.requirements/thread.timedmutex.class/try_lock.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.timedmutex.requirements/thread.timedmutex.class/try_lock.pass.cpp
index 058cdd9d244aa..2d92c0875317b 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.timedmutex.requirements/thread.timedmutex.class/try_lock.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.timedmutex.requirements/thread.timedmutex.class/try_lock.pass.cpp
@@ -8,7 +8,7 @@
 //
 // UNSUPPORTED: libcpp-has-no-threads
 //
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.timedmutex.requirements/thread.timedmutex.recursive/lock.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.timedmutex.requirements/thread.timedmutex.recursive/lock.pass.cpp
index 3dabae3b5a95e..c4f0078284aa5 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.timedmutex.requirements/thread.timedmutex.recursive/lock.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.timedmutex.requirements/thread.timedmutex.recursive/lock.pass.cpp
@@ -8,7 +8,7 @@
 //
 // UNSUPPORTED: libcpp-has-no-threads
 
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.timedmutex.requirements/thread.timedmutex.recursive/try_lock.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.timedmutex.requirements/thread.timedmutex.recursive/try_lock.pass.cpp
index 0b0f2a3ccf33b..e915a3f9907fc 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.timedmutex.requirements/thread.timedmutex.recursive/try_lock.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.mutex.requirements/thread.timedmutex.requirements/thread.timedmutex.recursive/try_lock.pass.cpp
@@ -8,7 +8,7 @@
 //
 // UNSUPPORTED: libcpp-has-no-threads
 //
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <mutex>
 
diff --git a/libcxx/test/std/thread/thread.mutex/thread.once/nothing_to_do.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.once/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/thread/thread.mutex/thread.once/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/thread/thread.req/nothing_to_do.pass.cpp b/libcxx/test/std/thread/thread.req/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/thread/thread.req/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/thread/thread.req/thread.req.lockable/nothing_to_do.pass.cpp b/libcxx/test/std/thread/thread.req/thread.req.lockable/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/thread/thread.req/thread.req.lockable/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/thread/thread.threads/thread.thread.this/sleep_until.pass.cpp b/libcxx/test/std/thread/thread.threads/thread.thread.this/sleep_until.pass.cpp
index 16fb12e79e1d2..8777e4aac133c 100644
--- a/libcxx/test/std/thread/thread.threads/thread.thread.this/sleep_until.pass.cpp
+++ b/libcxx/test/std/thread/thread.threads/thread.thread.this/sleep_until.pass.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: libcpp-has-no-threads
-// FLAKY_TEST.
+// ALLOW_RETRIES: 2
 
 // <thread>
 
diff --git a/libcxx/test/std/utilities/allocator.adaptor/allocator.adaptor.members/allocate_size.fail.cpp b/libcxx/test/std/utilities/allocator.adaptor/allocator.adaptor.members/allocate_size.fail.cpp
index 6d644db85e8c5..59fb1b5e2eb40 100644
--- a/libcxx/test/std/utilities/allocator.adaptor/allocator.adaptor.members/allocate_size.fail.cpp
+++ b/libcxx/test/std/utilities/allocator.adaptor/allocator.adaptor.members/allocate_size.fail.cpp
@@ -8,6 +8,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 // <memory>
 
@@ -24,7 +25,7 @@
 int main(int, char**)
 {
     std::scoped_allocator_adaptor<A1<int>> a;
-    a.allocate(10);  // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    a.allocate(10);  // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/utilities/allocator.adaptor/allocator.adaptor.members/allocate_size_hint.fail.cpp b/libcxx/test/std/utilities/allocator.adaptor/allocator.adaptor.members/allocate_size_hint.fail.cpp
index 7cf3d41090964..aa46da8da20fc 100644
--- a/libcxx/test/std/utilities/allocator.adaptor/allocator.adaptor.members/allocate_size_hint.fail.cpp
+++ b/libcxx/test/std/utilities/allocator.adaptor/allocator.adaptor.members/allocate_size_hint.fail.cpp
@@ -8,6 +8,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 // <memory>
 
@@ -24,7 +25,7 @@
 int main(int, char**)
 {
     std::scoped_allocator_adaptor<A1<int>> a;
-    a.allocate(10, (const void*)0); // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    a.allocate(10, (const void*)0); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/utilities/function.objects/bind/func.bind/nothing_to_do.pass.cpp b/libcxx/test/std/utilities/function.objects/bind/func.bind/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/utilities/function.objects/bind/func.bind/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/utilities/function.objects/bind/nothing_to_do.pass.cpp b/libcxx/test/std/utilities/function.objects/bind/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/utilities/function.objects/bind/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/utilities/function.objects/func.search/nothing_to_do.pass.cpp b/libcxx/test/std/utilities/function.objects/func.search/nothing_to_do.pass.cpp
deleted file mode 100644
index 779762e7e995d..0000000000000
--- a/libcxx/test/std/utilities/function.objects/func.search/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,14 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/utilities/function.objects/func.wrap/nothing_to_do.pass.cpp b/libcxx/test/std/utilities/function.objects/func.wrap/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/utilities/function.objects/func.wrap/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/utilities/function.objects/negators/binary_negate.depr_in_cxx17.fail.cpp b/libcxx/test/std/utilities/function.objects/negators/binary_negate.depr_in_cxx17.fail.cpp
index 1270482feec86..e6aa2377345fd 100644
--- a/libcxx/test/std/utilities/function.objects/negators/binary_negate.depr_in_cxx17.fail.cpp
+++ b/libcxx/test/std/utilities/function.objects/negators/binary_negate.depr_in_cxx17.fail.cpp
@@ -26,8 +26,8 @@ struct Predicate {
 };
 
 int main(int, char**) {
-    std::binary_negate<Predicate> f((Predicate())); // expected-error{{'binary_negate<Predicate>' is deprecated}}
+    std::binary_negate<Predicate> f((Predicate())); // expected-warning {{'binary_negate<Predicate>' is deprecated}}
     (void)f;
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/utilities/function.objects/negators/not1.depr_in_cxx17.fail.cpp b/libcxx/test/std/utilities/function.objects/negators/not1.depr_in_cxx17.fail.cpp
index e45605c8b30b4..49be77d7c1015 100644
--- a/libcxx/test/std/utilities/function.objects/negators/not1.depr_in_cxx17.fail.cpp
+++ b/libcxx/test/std/utilities/function.objects/negators/not1.depr_in_cxx17.fail.cpp
@@ -25,7 +25,7 @@ struct Predicate {
 };
 
 int main(int, char**) {
-    std::not1(Predicate()); // expected-error{{'not1<Predicate>' is deprecated}}
+    std::not1(Predicate()); // expected-warning {{'not1<Predicate>' is deprecated}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/utilities/function.objects/negators/not2.depr_in_cxx17.fail.cpp b/libcxx/test/std/utilities/function.objects/negators/not2.depr_in_cxx17.fail.cpp
index e73ee89b2ce3b..6737badf8f00c 100644
--- a/libcxx/test/std/utilities/function.objects/negators/not2.depr_in_cxx17.fail.cpp
+++ b/libcxx/test/std/utilities/function.objects/negators/not2.depr_in_cxx17.fail.cpp
@@ -26,7 +26,7 @@ struct Predicate {
 };
 
 int main(int, char**) {
-    std::not2(Predicate()); // expected-error{{'not2<Predicate>' is deprecated}}
+    std::not2(Predicate()); // expected-warning {{'not2<Predicate>' is deprecated}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/utilities/function.objects/negators/unary_negate.depr_in_cxx17.fail.cpp b/libcxx/test/std/utilities/function.objects/negators/unary_negate.depr_in_cxx17.fail.cpp
index 1dc905510a745..6530c55b4c386 100644
--- a/libcxx/test/std/utilities/function.objects/negators/unary_negate.depr_in_cxx17.fail.cpp
+++ b/libcxx/test/std/utilities/function.objects/negators/unary_negate.depr_in_cxx17.fail.cpp
@@ -25,7 +25,7 @@ struct Predicate {
 };
 
 int main(int, char**) {
-    std::unary_negate<Predicate> f((Predicate())); // expected-error{{'unary_negate<Predicate>' is deprecated}}
+    std::unary_negate<Predicate> f((Predicate())); // expected-warning {{'unary_negate<Predicate>' is deprecated}}
     (void)f;
 
   return 0;
diff --git a/libcxx/test/std/utilities/intseq/nothing_to_do.pass.cpp b/libcxx/test/std/utilities/intseq/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/utilities/intseq/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/utilities/memory/allocator.traits/allocator.traits.members/allocate.fail.cpp b/libcxx/test/std/utilities/memory/allocator.traits/allocator.traits.members/allocate.fail.cpp
index 47cfbb0494b56..0545b6d218502 100644
--- a/libcxx/test/std/utilities/memory/allocator.traits/allocator.traits.members/allocate.fail.cpp
+++ b/libcxx/test/std/utilities/memory/allocator.traits/allocator.traits.members/allocate.fail.cpp
@@ -17,6 +17,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 #include <memory>
 #include <cstdint>
@@ -45,8 +46,8 @@ struct A
 int main(int, char**)
 {
     A<int> a;
-    std::allocator_traits<A<int> >::allocate(a, 10);          // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
-    std::allocator_traits<A<int> >::allocate(a, 10, nullptr); // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    std::allocator_traits<A<int> >::allocate(a, 10);          // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+    std::allocator_traits<A<int> >::allocate(a, 10, nullptr); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/utilities/memory/allocator.uses/nothing_to_do.pass.cpp b/libcxx/test/std/utilities/memory/allocator.uses/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/utilities/memory/allocator.uses/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/utilities/memory/default.allocator/allocator.members/allocate.fail.cpp b/libcxx/test/std/utilities/memory/default.allocator/allocator.members/allocate.fail.cpp
index 4cbd29029331a..b24104d3411c4 100644
--- a/libcxx/test/std/utilities/memory/default.allocator/allocator.members/allocate.fail.cpp
+++ b/libcxx/test/std/utilities/memory/default.allocator/allocator.members/allocate.fail.cpp
@@ -8,6 +8,7 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14, c++17
 // UNSUPPORTED: clang-3.3, clang-3.4, clang-3.5, clang-3.6, clang-3.7, clang-3.8
+// REQUIRES: verify-support
 
 // <memory>
 
@@ -22,7 +23,7 @@
 int main(int, char**)
 {
     std::allocator<int> a;
-    a.allocate(3);          // expected-error {{ignoring return value of function declared with 'nodiscard' attribute}}
+    a.allocate(3); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/utilities/memory/specialized.algorithms/nothing_to_do.pass.cpp b/libcxx/test/std/utilities/memory/specialized.algorithms/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/utilities/memory/specialized.algorithms/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/nothing_to_do.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/utilities/memory/util.smartptr/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/utilities/meta/meta.const.eval/is_constant_evaluated.fail.cpp b/libcxx/test/std/utilities/meta/meta.const.eval/is_constant_evaluated.fail.cpp
index 2d2e071576244..a6ce1141ad682 100644
--- a/libcxx/test/std/utilities/meta/meta.const.eval/is_constant_evaluated.fail.cpp
+++ b/libcxx/test/std/utilities/meta/meta.const.eval/is_constant_evaluated.fail.cpp
@@ -24,7 +24,7 @@ int main(int, char**)
 #else
   // expected-error@+1 {{static_assert failed}}
   static_assert(!std::is_constant_evaluated(), "");
-  // expected-error@-1 0-1 {{'std::is_constant_evaluated' will always evaluate to 'true' in a manifestly constant-evaluated expression}}
+  // expected-warning@-1 0-1 {{'std::is_constant_evaluated' will always evaluate to 'true' in a manifestly constant-evaluated expression}}
 #endif
   return 0;
 }
diff --git a/libcxx/test/std/utilities/meta/meta.trans/nothing_to_do.pass.cpp b/libcxx/test/std/utilities/meta/meta.trans/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/utilities/meta/meta.trans/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/utilities/meta/meta.unary/nothing_to_do.pass.cpp b/libcxx/test/std/utilities/meta/meta.unary/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/utilities/meta/meta.unary/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/utilities/nothing_to_do.pass.cpp b/libcxx/test/std/utilities/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/utilities/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/utilities/smartptr/unique.ptr/nothing_to_do.pass.cpp b/libcxx/test/std/utilities/smartptr/unique.ptr/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/utilities/smartptr/unique.ptr/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.modifiers/reset.runtime.fail.cpp b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.modifiers/reset.runtime.fail.cpp
index 7e3085a5b0fbf..afb19b3c343ba 100644
--- a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.modifiers/reset.runtime.fail.cpp
+++ b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.modifiers/reset.runtime.fail.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 // <memory>
+// UNSUPPORTED: c++98, c++03
 
 // unique_ptr
 
diff --git a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.dltr/nothing_to_do.pass.cpp b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.dltr/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.dltr/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/utilities/time/time.cal/nothing_to_do.pass.cpp b/libcxx/test/std/utilities/time/time.cal/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/utilities/time/time.cal/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/utilities/time/time.clock/nothing_to_do.pass.cpp b/libcxx/test/std/utilities/time/time.clock/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/utilities/time/time.clock/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/utilities/time/time.traits/nothing_to_do.pass.cpp b/libcxx/test/std/utilities/time/time.traits/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/utilities/time/time.traits/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/utilities/utility.requirements/nothing_to_do.pass.cpp b/libcxx/test/std/utilities/utility.requirements/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/utilities/utility.requirements/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/utilities/utility/pairs/nothing_to_do.pass.cpp b/libcxx/test/std/utilities/utility/pairs/nothing_to_do.pass.cpp
deleted file mode 100644
index 1f764da05d6b5..0000000000000
--- a/libcxx/test/std/utilities/utility/pairs/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/support/filesystem_dynamic_test_helper.py b/libcxx/test/support/filesystem_dynamic_test_helper.py
deleted file mode 100644
index 0789582749785..0000000000000
--- a/libcxx/test/support/filesystem_dynamic_test_helper.py
+++ /dev/null
@@ -1,92 +0,0 @@
-import sys
-import os
-import socket
-import stat
-
-# Ensure that this is being run on a specific platform
-assert sys.platform.startswith('linux') or sys.platform.startswith('darwin') \
-    or sys.platform.startswith('cygwin') or sys.platform.startswith('freebsd') \
-    or sys.platform.startswith('netbsd')
-
-def env_path():
-    ep = os.environ.get('LIBCXX_FILESYSTEM_DYNAMIC_TEST_ROOT')
-    assert ep is not None
-    ep = os.path.realpath(ep)
-    assert os.path.isdir(ep)
-    return ep
-
-env_path_global = env_path()
-
-# Make sure we don't try and write outside of env_path.
-# All paths used should be sanitized
-def sanitize(p):
-    p = os.path.realpath(p)
-    if os.path.commonprefix([env_path_global, p]):
-        return p
-    assert False
-
-"""
-Some of the tests restrict permissions to induce failures.
-Before we delete the test environment, we have to walk it and re-raise the
-permissions.
-"""
-def clean_recursive(root_p):
-    if not os.path.islink(root_p):
-        os.chmod(root_p, 0o777)
-    for ent in os.listdir(root_p):
-        p = os.path.join(root_p, ent)
-        if os.path.islink(p) or not os.path.isdir(p):
-            os.remove(p)
-        else:
-            assert os.path.isdir(p)
-            clean_recursive(p)
-            os.rmdir(p)
-
-
-def init_test_directory(root_p):
-    root_p = sanitize(root_p)
-    assert not os.path.exists(root_p)
-    os.makedirs(root_p)
-
-
-def destroy_test_directory(root_p):
-    root_p = sanitize(root_p)
-    clean_recursive(root_p)
-    os.rmdir(root_p)
-
-
-def create_file(fname, size):
-    with open(sanitize(fname), 'w') as f:
-        f.write('c' * size)
-
-
-def create_dir(dname):
-    os.mkdir(sanitize(dname))
-
-
-def create_symlink(source, link):
-    os.symlink(sanitize(source), sanitize(link))
-
-
-def create_hardlink(source, link):
-    os.link(sanitize(source), sanitize(link))
-
-
-def create_fifo(source):
-    os.mkfifo(sanitize(source))
-
-
-def create_socket(source):
-    sock = socket.socket(socket.AF_UNIX)
-    sanitized_source = sanitize(source)
-    # AF_UNIX sockets may have very limited path length, so split it
-    # into chdir call (with technically unlimited length) followed
-    # by bind() relative to the directory
-    os.chdir(os.path.dirname(sanitized_source))
-    sock.bind(os.path.basename(sanitized_source))
-
-
-if __name__ == '__main__':
-    command = " ".join(sys.argv[1:])
-    eval(command)
-    sys.exit(0)
diff --git a/libcxx/test/support/filesystem_test_helper.h b/libcxx/test/support/filesystem_test_helper.h
index 8161820d05da2..8553317de76f6 100644
--- a/libcxx/test/support/filesystem_test_helper.h
+++ b/libcxx/test/support/filesystem_test_helper.h
@@ -18,22 +18,28 @@
 #include "rapid-cxx-test.h"
 #include "format_string.h"
 
-// static test helpers
+// For creating socket files
+#if !defined(__FreeBSD__) && !defined(__APPLE__)
+# include <sys/socket.h>
+# include <sys/un.h>
+#endif
 
-#ifndef LIBCXX_FILESYSTEM_STATIC_TEST_ROOT
-#warning "STATIC TESTS DISABLED"
-#else // LIBCXX_FILESYSTEM_STATIC_TEST_ROOT
+// static test helpers
 
 namespace StaticEnv {
 
+// Tests that use these utilities should add '<...>/Inputs/static_test_env'
+// to their FILE_DEPENDENCIES, to make sure the directory is made available
+// to the test. Assuming that, the 'static_test_env' will be available in the
+// directory where the test is run.
+static const fs::path Root = fs::current_path() / "static_test_env";
+
 inline fs::path makePath(fs::path const& p) {
     // env_path is expected not to contain symlinks.
-    static const fs::path env_path = LIBCXX_FILESYSTEM_STATIC_TEST_ROOT;
+    fs::path const& env_path = Root;
     return env_path / p;
 }
 
-static const fs::path Root = LIBCXX_FILESYSTEM_STATIC_TEST_ROOT;
-
 static const fs::path TestFileList[] = {
         makePath("empty_file"),
         makePath("non_empty_file"),
@@ -101,16 +107,6 @@ static const fs::path RecDirFollowSymlinksIterationList[] = {
 
 } // namespace StaticEnv
 
-#endif // LIBCXX_FILESYSTEM_STATIC_TEST_ROOT
-
-#ifndef LIBCXX_FILESYSTEM_DYNAMIC_TEST_ROOT
-#warning LIBCXX_FILESYSTEM_DYNAMIC_TEST_ROOT must be defined
-#else // LIBCXX_FILESYSTEM_DYNAMIC_TEST_ROOT
-
-#ifndef LIBCXX_FILESYSTEM_DYNAMIC_TEST_HELPER
-#error LIBCXX_FILESYSTEM_DYNAMIC_TEST_HELPER must be defined
-#endif
-
 namespace random_utils {
 inline char to_hex(int ch) {
   return ch < 10 ? static_cast<char>('0' + ch)
@@ -127,11 +123,27 @@ inline char random_hex_char() {
 
 struct scoped_test_env
 {
-    scoped_test_env() : test_root(random_env_path())
-        { fs_helper_run(fs_make_cmd("init_test_directory", test_root)); }
+    scoped_test_env() : test_root(random_path()) {
+        std::string cmd = "mkdir -p " + test_root.native();
+        int ret = std::system(cmd.c_str());
+        assert(ret == 0);
 
-    ~scoped_test_env()
-        { fs_helper_run(fs_make_cmd("destroy_test_directory", test_root)); }
+        // Ensure that the root_path is fully resolved, i.e. it contains no
+        // symlinks. The filesystem tests depend on that. We do this after
+        // creating the root_path, because `fs::canonical` requires the
+        // path to exist.
+        test_root = fs::canonical(test_root);
+    }
+
+    ~scoped_test_env() {
+        std::string cmd = "chmod -R 777 " + test_root.native();
+        int ret = std::system(cmd.c_str());
+        assert(ret == 0);
+
+        cmd = "rm -r " + test_root.native();
+        ret = std::system(cmd.c_str());
+        assert(ret == 0);
+    }
 
     scoped_test_env(scoped_test_env const &) = delete;
     scoped_test_env & operator=(scoped_test_env const &) = delete;
@@ -195,27 +207,35 @@ struct scoped_test_env
 
     std::string create_dir(std::string filename) {
         filename = sanitize_path(std::move(filename));
-        fs_helper_run(fs_make_cmd("create_dir", filename));
+        std::string cmd = "mkdir " + filename;
+        int ret = std::system(cmd.c_str());
+        assert(ret == 0);
         return filename;
     }
 
     std::string create_symlink(std::string source, std::string to) {
         source = sanitize_path(std::move(source));
         to = sanitize_path(std::move(to));
-        fs_helper_run(fs_make_cmd("create_symlink", source, to));
+        std::string cmd = "ln -s " + source + ' ' + to;
+        int ret = std::system(cmd.c_str());
+        assert(ret == 0);
         return to;
     }
 
     std::string create_hardlink(std::string source, std::string to) {
         source = sanitize_path(std::move(source));
         to = sanitize_path(std::move(to));
-        fs_helper_run(fs_make_cmd("create_hardlink", source, to));
+        std::string cmd = "ln " + source + ' ' + to;
+        int ret = std::system(cmd.c_str());
+        assert(ret == 0);
         return to;
     }
 
     std::string create_fifo(std::string file) {
         file = sanitize_path(std::move(file));
-        fs_helper_run(fs_make_cmd("create_fifo", file));
+        std::string cmd = "mkfifo " + file;
+        int ret = std::system(cmd.c_str());
+        assert(ret == 0);
         return file;
     }
 
@@ -224,12 +244,18 @@ struct scoped_test_env
 #if !defined(__FreeBSD__) && !defined(__APPLE__)
     std::string create_socket(std::string file) {
         file = sanitize_path(std::move(file));
-        fs_helper_run(fs_make_cmd("create_socket", file));
+
+        ::sockaddr_un address;
+        address.sun_family = AF_UNIX;
+        assert(file.size() <= sizeof(address.sun_path));
+        ::strncpy(address.sun_path, file.c_str(), sizeof(address.sun_path));
+        int fd = ::socket(AF_UNIX, SOCK_STREAM, 0);
+        ::bind(fd, reinterpret_cast<::sockaddr*>(&address), sizeof(address));
         return file;
     }
 #endif
 
-    fs::path const test_root;
+    fs::path test_root;
 
 private:
     static std::string unique_path_suffix() {
@@ -243,66 +269,13 @@ struct scoped_test_env
 
     // This could potentially introduce a filesystem race with other tests
     // running at the same time, but oh well, it's just test code.
-    static inline fs::path random_env_path() {
-        static const char* env_path = LIBCXX_FILESYSTEM_DYNAMIC_TEST_ROOT;
-        fs::path p = fs::path(env_path) / unique_path_suffix();
-        assert(p.parent_path() == env_path);
+    static inline fs::path random_path() {
+        fs::path tmp = fs::temp_directory_path();
+        fs::path p = fs::path(tmp) / unique_path_suffix();
         return p;
     }
-
-    static inline std::string make_arg(std::string const& arg) {
-        return "'" + arg + "'";
-    }
-
-    static inline std::string make_arg(std::size_t arg) {
-        return std::to_string(arg);
-    }
-
-    template <class T>
-    static inline std::string
-    fs_make_cmd(std::string const& cmd_name, T const& arg) {
-        return cmd_name + "(" + make_arg(arg) + ")";
-    }
-
-    template <class T, class U>
-    static inline std::string
-    fs_make_cmd(std::string const& cmd_name, T const& arg1, U const& arg2) {
-        return cmd_name + "(" + make_arg(arg1) + ", " + make_arg(arg2) + ")";
-    }
-
-    static inline void fs_helper_run(std::string const& raw_cmd) {
-        // check that the fs test root in the environment matches what we were
-        // compiled with.
-        static bool checked = checkDynamicTestRoot();
-        ((void)checked);
-        std::string cmd = LIBCXX_FILESYSTEM_DYNAMIC_TEST_HELPER;
-        cmd += " \"" + raw_cmd + "\"";
-        int ret = std::system(cmd.c_str());
-        assert(ret == 0);
-    }
-
-    static bool checkDynamicTestRoot() {
-        // LIBCXX_FILESYSTEM_DYNAMIC_TEST_ROOT is expected not to contain symlinks.
-        char* fs_root = std::getenv("LIBCXX_FILESYSTEM_DYNAMIC_TEST_ROOT");
-        if (!fs_root) {
-            std::printf("ERROR: LIBCXX_FILESYSTEM_DYNAMIC_TEST_ROOT must be a defined "
-                         "environment variable when running the test.\n");
-            std::abort();
-        }
-        if (std::string(fs_root) != LIBCXX_FILESYSTEM_DYNAMIC_TEST_ROOT) {
-            std::printf("ERROR: LIBCXX_FILESYSTEM_DYNAMIC_TEST_ROOT environment variable"
-                        " must have the same value as when the test was compiled.\n");
-            std::printf("   Current Value:  '%s'\n", fs_root);
-            std::printf("   Expected Value: '%s'\n", LIBCXX_FILESYSTEM_DYNAMIC_TEST_ROOT);
-            std::abort();
-        }
-        return true;
-    }
-
 };
 
-#endif // LIBCXX_FILESYSTEM_DYNAMIC_TEST_ROOT
-
 // Misc test types
 
 #define MKSTR(Str) {Str, TEST_CONCAT(L, Str), TEST_CONCAT(u, Str), TEST_CONCAT(U, Str)}
diff --git a/libcxx/test/support/msvc_stdlib_force_include.h b/libcxx/test/support/msvc_stdlib_force_include.h
index 768558efe76ff..cf8696af5588a 100644
--- a/libcxx/test/support/msvc_stdlib_force_include.h
+++ b/libcxx/test/support/msvc_stdlib_force_include.h
@@ -101,7 +101,8 @@ const AssertionDialogAvoider assertion_dialog_avoider{};
 #else // ^^^ clang / MSVC vvv
 #define _LIBCPP_SUPPRESS_DEPRECATED_PUSH \
     __pragma(warning(push)) \
-    __pragma(warning(disable : 4996))
+    __pragma(warning(disable : 4996)) \
+    __pragma(warning(disable : 5215))
 #define _LIBCPP_SUPPRESS_DEPRECATED_POP \
     __pragma(warning(pop))
 #endif // __clang__
diff --git a/libcxx/test/support/nothing_to_do.pass.cpp b/libcxx/test/support/nothing_to_do.pass.cpp
deleted file mode 100644
index 779762e7e995d..0000000000000
--- a/libcxx/test/support/nothing_to_do.pass.cpp
+++ /dev/null
@@ -1,14 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/support/test_macros.h b/libcxx/test/support/test_macros.h
index 6ab015db667ac..6cbb7f0ccc166 100644
--- a/libcxx/test/support/test_macros.h
+++ b/libcxx/test/support/test_macros.h
@@ -28,6 +28,9 @@
 #pragma GCC diagnostic ignored "-Wvariadic-macros"
 #endif
 
+#define TEST_STRINGIZE_IMPL(x) #x
+#define TEST_STRINGIZE(x) TEST_STRINGIZE_IMPL(x)
+
 #define TEST_CONCAT1(X, Y) X##Y
 #define TEST_CONCAT(X, Y) TEST_CONCAT1(X, Y)
 
diff --git a/libcxx/utils/gdb/libcxx/printers.py b/libcxx/utils/gdb/libcxx/printers.py
index 4b724c94368d0..7cccc07997b3b 100644
--- a/libcxx/utils/gdb/libcxx/printers.py
+++ b/libcxx/utils/gdb/libcxx/printers.py
@@ -982,10 +982,10 @@ def __call__(self, val):
         # Don't attempt types known to be inside libstdcxx.
         typename = val.type.name or val.type.tag or str(val.type)
         match = re.match("^std::(__.*?)::", typename)
-        if match is None or match.group(1) in ["__cxx1998",
-                                               "__debug",
-                                               "__7",
-                                               "__g"]:
+        if match is not None and match.group(1) in ["__cxx1998",
+                                                    "__debug",
+                                                    "__7",
+                                                    "__g"]:
             return None
 
         # Handle any using declarations or other typedefs.
diff --git a/libcxx/utils/libcxx/compiler.py b/libcxx/utils/libcxx/compiler.py
index dd334cdbcff9f..248b5ba8a98ce 100644
--- a/libcxx/utils/libcxx/compiler.py
+++ b/libcxx/utils/libcxx/compiler.py
@@ -29,6 +29,7 @@ def __init__(self, config, path, flags=None, compile_flags=None, link_flags=None
         self.flags = list(flags or [])
         self.compile_flags = list(compile_flags or [])
         self.link_flags = list(link_flags or [])
+        self.link_libcxxabi_flag = '-lc++abi'
         self.warning_flags = list(warning_flags or [])
         self.verify_supported = verify_supported
         self.use_verify = use_verify
diff --git a/libcxx/utils/libcxx/test/config.py b/libcxx/utils/libcxx/test/config.py
index a9a353916a9c6..0a5f62eefe30c 100644
--- a/libcxx/utils/libcxx/test/config.py
+++ b/libcxx/utils/libcxx/test/config.py
@@ -6,6 +6,7 @@
 #
 #===----------------------------------------------------------------------===##
 
+import copy
 import locale
 import os
 import platform
@@ -140,7 +141,6 @@ def configure(self):
         self.configure_execute_external()
         self.configure_ccache()
         self.configure_compile_flags()
-        self.configure_filesystem_compile_flags()
         self.configure_link_flags()
         self.configure_env()
         self.configure_color_diagnostics()
@@ -150,6 +150,8 @@ def configure(self):
         self.configure_coverage()
         self.configure_modules()
         self.configure_coroutines()
+        self.configure_blocks()
+        self.configure_objc_arc()
         self.configure_substitutions()
         self.configure_features()
 
@@ -724,29 +726,6 @@ def configure_compile_flags_abi_version(self):
           self.config.available_features.add('libcpp-abi-unstable')
           self.cxx.compile_flags += ['-D_LIBCPP_ABI_UNSTABLE']
 
-    def configure_filesystem_compile_flags(self):
-        static_env = os.path.join(self.libcxx_src_root, 'test', 'std',
-                                  'input.output', 'filesystems', 'Inputs', 'static_test_env')
-        static_env = os.path.realpath(static_env)
-        assert os.path.isdir(static_env)
-        self.cxx.compile_flags += ['-DLIBCXX_FILESYSTEM_STATIC_TEST_ROOT="%s"' % static_env]
-
-        dynamic_env = os.path.join(self.config.test_exec_root,
-                                   'filesystem', 'Output', 'dynamic_env')
-        dynamic_env = os.path.realpath(dynamic_env)
-        if not os.path.isdir(dynamic_env):
-            os.makedirs(dynamic_env)
-        self.cxx.compile_flags += ['-DLIBCXX_FILESYSTEM_DYNAMIC_TEST_ROOT="%s"' % dynamic_env]
-        self.exec_env['LIBCXX_FILESYSTEM_DYNAMIC_TEST_ROOT'] = ("%s" % dynamic_env)
-
-        dynamic_helper = os.path.join(self.libcxx_src_root, 'test', 'support',
-                                      'filesystem_dynamic_test_helper.py')
-        assert os.path.isfile(dynamic_helper)
-
-        self.cxx.compile_flags += ['-DLIBCXX_FILESYSTEM_DYNAMIC_TEST_HELPER="%s %s"'
-                                   % (sys.executable, dynamic_helper)]
-
-
     def configure_link_flags(self):
         # Configure library path
         self.configure_link_flags_cxx_library_path()
@@ -846,6 +825,7 @@ def configure_link_flags_abi_library(self):
                     if cxxabi_library_root:
                         libname = self.make_static_lib_name('c++abi')
                         abs_path = os.path.join(cxxabi_library_root, libname)
+                        self.cxx.link_libcxxabi_flag = abs_path
                         self.cxx.link_flags += [abs_path]
                     else:
                         self.cxx.link_flags += ['-lc++abi']
@@ -1010,6 +990,16 @@ def configure_coroutines(self):
             if intMacroValue(macros['__cpp_coroutines']) >= 201703:
                 self.config.available_features.add('fcoroutines-ts')
 
+    def configure_blocks(self):
+        if self.cxx.hasCompileFlag('-fblocks'):
+            self.config.available_features.add('has-fblocks')
+
+    def configure_objc_arc(self):
+        cxx = copy.deepcopy(self.cxx)
+        cxx.source_lang = 'objective-c++'
+        if cxx.hasCompileFlag('-fobjc-arc'):
+            self.config.available_features.add('has-fobjc-arc')
+
     def configure_modules(self):
         modules_flags = ['-fmodules']
         if not self.target_info.is_darwin():
@@ -1039,58 +1029,50 @@ def configure_substitutions(self):
         if self.target_info.is_darwin():
             # Do not pass DYLD_LIBRARY_PATH to the compiler, linker, etc. as
             # these tools are not meant to exercise the just-built libraries.
-            tool_env += 'env DYLD_LIBRARY_PATH="" '
+            tool_env += 'env DYLD_LIBRARY_PATH=""'
 
         sub = self.config.substitutions
-        cxx_path = tool_env + pipes.quote(self.cxx.path)
         # Configure compiler substitutions
-        sub.append(('%cxx', cxx_path))
-        sub.append(('%libcxx_src_root', self.libcxx_src_root))
+        sub.append(('%{cxx}', '{} {}'.format(tool_env, pipes.quote(self.cxx.path))))
+        sub.append(('%{libcxx_src_root}', self.libcxx_src_root))
         # Configure flags substitutions
-        flags_str = ' '.join([pipes.quote(f) for f in self.cxx.flags])
-        compile_flags_str = ' '.join([pipes.quote(f) for f in self.cxx.compile_flags])
-        link_flags_str = ' '.join([pipes.quote(f) for f in self.cxx.link_flags])
-        all_flags = '%s %s %s' % (flags_str, compile_flags_str, link_flags_str)
-        sub.append(('%flags', flags_str))
-        sub.append(('%compile_flags', compile_flags_str))
-        sub.append(('%link_flags', link_flags_str))
-        sub.append(('%all_flags', all_flags))
+        sub.append(('%{flags}',         ' '.join(map(pipes.quote, self.cxx.flags))))
+        sub.append(('%{compile_flags}', ' '.join(map(pipes.quote, self.cxx.compile_flags))))
+        sub.append(('%{link_flags}',    ' '.join(map(pipes.quote, self.cxx.link_flags))))
+        sub.append(('%{link_libcxxabi}', pipes.quote(self.cxx.link_libcxxabi_flag)))
         if self.cxx.isVerifySupported():
-            verify_str = ' ' + ' '.join(self.cxx.verify_flags) + ' '
-            sub.append(('%verify', verify_str))
-        # Add compile and link shortcuts
-        compile_str = (cxx_path + ' -o %t.o %s -c ' + flags_str
-                       + ' ' + compile_flags_str)
-        link_str = (cxx_path + ' -o %t.exe %t.o ' + flags_str + ' '
-                    + link_flags_str)
-        assert type(link_str) is str
-        build_str = cxx_path + ' -o %t.exe %s ' + all_flags
+            sub.append(('%{verify}', ' '.join(self.cxx.verify_flags)))
+        # Add compile and build shortcuts
+        sub.append(('%{compile}', '%{cxx} -o %t.o %s -c %{flags} %{compile_flags}'))
+        sub.append(('%{build}',   '%{cxx} -o %t.exe %s %{flags} %{compile_flags} %{link_flags}'))
         if self.cxx.use_modules:
-            sub.append(('%compile_module', compile_str))
-            sub.append(('%build_module', build_str))
+            sub.append(('%{build_module}', '%{build}'))
         elif self.cxx.modules_flags is not None:
-            modules_str = ' '.join(self.cxx.modules_flags) + ' '
-            sub.append(('%compile_module', compile_str + ' ' + modules_str))
-            sub.append(('%build_module', build_str + ' ' + modules_str))
-        sub.append(('%compile', compile_str))
-        sub.append(('%link', link_str))
-        sub.append(('%build', build_str))
+            sub.append(('%{build_module}', '%{{build}} {}'.format(' '.join(self.cxx.modules_flags))))
+
         # Configure exec prefix substitutions.
         # Configure run env substitution.
         codesign_ident = self.get_lit_conf('llvm_codesign_identity', '')
-        run_py = os.path.join(self.libcxx_src_root, 'utils', 'run.py')
         env_vars = ' '.join('%s=%s' % (k, pipes.quote(v)) for (k, v) in self.exec_env.items())
-        run_str = '%s %s --codesign_identity "%s" --working_directory "%%S" ' \
-                  '--dependencies %%file_dependencies --env %s -- %%t.exe' %  \
-            (pipes.quote(sys.executable), pipes.quote(run_py),
-             codesign_ident, env_vars)
-        sub.append(('%run', run_str))
+        exec_args = [
+            '--codesign_identity "{}"'.format(codesign_ident),
+            '--dependencies %{file_dependencies}',
+            '--env {}'.format(env_vars)
+        ]
+        if isinstance(self.executor, SSHExecutor):
+            exec_args.append('--host {}'.format(self.executor.user_prefix + self.executor.host))
+            executor = os.path.join(self.libcxx_src_root, 'utils', 'ssh.py')
+        else:
+            executor = os.path.join(self.libcxx_src_root, 'utils', 'run.py')
+        sub.append(('%{exec}', '{} {} {} -- '.format(pipes.quote(sys.executable),
+                                                     pipes.quote(executor),
+                                                     ' '.join(exec_args))))
+        sub.append(('%{run}', '%{exec} %t.exe'))
         # Configure not program substitutions
         not_py = os.path.join(self.libcxx_src_root, 'utils', 'not.py')
-        not_str = '%s %s ' % (pipes.quote(sys.executable), pipes.quote(not_py))
-        sub.append(('not ', not_str))
+        sub.append(('%{not}', '{} {}'.format(pipes.quote(sys.executable), pipes.quote(not_py))))
         if self.get_lit_conf('libcxx_gdb'):
-            sub.append(('%libcxx_gdb', self.get_lit_conf('libcxx_gdb')))
+            sub.append(('%{libcxx_gdb}', self.get_lit_conf('libcxx_gdb')))
 
     def can_use_deployment(self):
         # Check if the host is on an Apple platform using clang.
diff --git a/libcxx/utils/libcxx/test/executor.py b/libcxx/utils/libcxx/test/executor.py
index b555b1f03df91..c34310cdd2e22 100644
--- a/libcxx/utils/libcxx/test/executor.py
+++ b/libcxx/utils/libcxx/test/executor.py
@@ -10,6 +10,7 @@
 import os
 import posixpath
 import ntpath
+import shutil
 
 from libcxx.test import tracing
 from libcxx.util import executeCommand
@@ -61,6 +62,12 @@ def run(self, exe_path, cmd=None, work_dir='.', file_deps=None, env=None):
         if env:
             env = self.merge_environments(os.environ, env)
 
+        for dep in file_deps:
+            if os.path.isdir(dep):
+                shutil.copytree(dep, os.path.join(work_dir, os.path.basename(dep)), symlinks=True)
+            else:
+                shutil.copy2(dep, work_dir)
+
         out, err, rc = executeCommand(cmd, cwd=work_dir, env=env)
         return (cmd, out, err, rc)
 
diff --git a/libcxx/utils/libcxx/test/format.py b/libcxx/utils/libcxx/test/format.py
index bf526d3618d13..1bc85f24976a2 100644
--- a/libcxx/utils/libcxx/test/format.py
+++ b/libcxx/utils/libcxx/test/format.py
@@ -9,6 +9,8 @@
 import copy
 import errno
 import os
+import shutil
+import tempfile
 import time
 import random
 
@@ -18,6 +20,7 @@
     # pylint: disable=import-error
 
 from libcxx.test.executor import LocalExecutor as LocalExecutor
+from libcxx.test.executor import SSHExecutor as SSHExecutor
 import libcxx.util
 
 
@@ -43,14 +46,19 @@ def __init__(self, cxx, use_verify_for_fail, execute_external,
     @staticmethod
     def _make_custom_parsers(test):
         return [
-            IntegratedTestKeywordParser('FLAKY_TEST.', ParserKind.TAG,
-                                        initial_value=False),
             IntegratedTestKeywordParser('MODULES_DEFINES:', ParserKind.LIST,
                                         initial_value=[]),
             IntegratedTestKeywordParser('FILE_DEPENDENCIES:', ParserKind.LIST,
-                                        initial_value=test.file_dependencies)
+                                        initial_value=test.file_dependencies),
+            IntegratedTestKeywordParser('ADDITIONAL_COMPILE_FLAGS:', ParserKind.LIST,
+                                        initial_value=[])
         ]
 
+    # Utility function to add compile flags in lit.local.cfg files.
+    def addCompileFlags(self, config, *flags):
+        self.cxx = copy.deepcopy(self.cxx)
+        self.cxx.compile_flags += flags
+
     @staticmethod
     def _get_parser(key, parsers):
         for p in parsers:
@@ -91,8 +99,6 @@ def _execute(self, test, lit_config):
         is_pass_test = name.endswith('.pass.cpp') or name.endswith('.pass.mm')
         is_fail_test = name.endswith('.fail.cpp') or name.endswith('.fail.mm')
         is_objcxx_test = name.endswith('.mm')
-        is_objcxx_arc_test = name.endswith('.arc.pass.mm') or \
-                             name.endswith('.arc.fail.mm')
         assert is_sh_test or name_ext == '.cpp' or name_ext == '.mm', \
             'non-cpp file must be sh test'
 
@@ -109,8 +115,6 @@ def _execute(self, test, lit_config):
         script = lit.TestRunner.parseIntegratedTestScript(
             test, additional_parsers=parsers, require_script=is_sh_test)
 
-        local_cwd = os.path.dirname(test.getSourcePath())
-        data_files = [os.path.join(local_cwd, f) for f in test.file_dependencies]
         # Check if a result for the test was returned. If so return that
         # result.
         if isinstance(script, lit.Test.Result):
@@ -123,10 +127,18 @@ def _execute(self, test, lit_config):
             lit_config.fatal('Unsupported RUN line found in test %s' % name)
 
         tmpDir, tmpBase = lit.TestRunner.getTempPaths(test)
-        substitutions = lit.TestRunner.getDefaultSubstitutions(test, tmpDir,
-                                                               tmpBase)
-        substitutions.append(('%file_dependencies', ' '.join(data_files)))
-        script = lit.TestRunner.applySubstitutions(script, substitutions)
+        substitutions = lit.TestRunner.getDefaultSubstitutions(
+            test, tmpDir, tmpBase, normalize_slashes=self.execute_external)
+
+        # Apply substitutions in FILE_DEPENDENCIES markup
+        data_files = lit.TestRunner.applySubstitutions(test.file_dependencies, substitutions,
+                                                       recursion_limit=10)
+        local_cwd = os.path.dirname(test.getSourcePath())
+        data_files = [f if os.path.isabs(f) else os.path.join(local_cwd, f) for f in data_files]
+        substitutions.append(('%{file_dependencies}', ' '.join(data_files)))
+
+        script = lit.TestRunner.applySubstitutions(script, substitutions,
+                                                   recursion_limit=10)
 
         test_cxx = copy.deepcopy(self.cxx)
         if is_fail_test:
@@ -147,18 +159,21 @@ def _execute(self, test, lit_config):
                 if b'#define _LIBCPP_ASSERT' in contents:
                     test_cxx.useModules(False)
 
+        # Handle ADDITIONAL_COMPILE_FLAGS keywords by adding those compilation
+        # flags, but first perform substitutions in those flags.
+        extra_compile_flags = self._get_parser('ADDITIONAL_COMPILE_FLAGS:', parsers).getValue()
+        extra_compile_flags = lit.TestRunner.applySubstitutions(extra_compile_flags, substitutions)
+        test_cxx.compile_flags.extend(extra_compile_flags)
+
         if is_objcxx_test:
             test_cxx.source_lang = 'objective-c++'
-            if is_objcxx_arc_test:
-                test_cxx.compile_flags += ['-fobjc-arc']
-            else:
-                test_cxx.compile_flags += ['-fno-objc-arc']
             test_cxx.link_flags += ['-framework', 'Foundation']
 
         # Dispatch the test based on its suffix.
         if is_sh_test:
-            if not isinstance(self.executor, LocalExecutor):
-                # We can't run ShTest tests with a executor yet.
+            if not isinstance(self.executor, LocalExecutor) and not isinstance(self.executor, SSHExecutor):
+                # We can't run ShTest tests with other executors than
+                # LocalExecutor and SSHExecutor yet.
                 # For now, bail on trying to run them
                 return lit.Test.UNSUPPORTED, 'ShTest format not yet supported'
             test.config.environment = self.executor.merge_environments(os.environ, self.exec_env)
@@ -196,16 +211,21 @@ def _evaluate_pass_test(self, test, tmpBase, lit_config,
                 report += "Compilation failed unexpectedly!"
                 return lit.Test.Result(lit.Test.FAIL, report)
             # Run the test
-            local_cwd = os.path.dirname(source_path)
             env = None
             if self.exec_env:
                 env = self.exec_env
-            is_flaky = self._get_parser('FLAKY_TEST.', parsers).getValue()
-            max_retry = 3 if is_flaky else 1
+
+            max_retry = test.allowed_retries + 1
             for retry_count in range(max_retry):
-                cmd, out, err, rc = self.executor.run(exec_path, [exec_path],
-                                                      local_cwd, data_files,
-                                                      env)
+                # Create a temporary directory just for that test and run the
+                # test in that directory
+                try:
+                    execDirTmp = tempfile.mkdtemp(dir=execDir)
+                    cmd, out, err, rc = self.executor.run(exec_path, [exec_path],
+                                                          execDirTmp, data_files,
+                                                          env)
+                finally:
+                    shutil.rmtree(execDirTmp)
                 report = "Compiled With: '%s'\n" % ' '.join(compile_cmd)
                 report += libcxx.util.makeReport(cmd, out, err, rc)
                 if rc == 0:
@@ -240,21 +260,6 @@ def _evaluate_fail_test(self, test, test_cxx, parsers):
             test_cxx.flags += ['-fsyntax-only']
         if use_verify:
             test_cxx.useVerify()
-            test_cxx.useWarnings()
-            if '-Wuser-defined-warnings' in test_cxx.warning_flags:
-                test_cxx.warning_flags += ['-Wno-error=user-defined-warnings']
-        else:
-            # We still need to enable certain warnings on .fail.cpp test when
-            # -verify isn't enabled. Such as -Werror=unused-result. However,
-            # we don't want it enabled too liberally, which might incorrectly
-            # allow unrelated failure tests to 'pass'.
-            #
-            # Therefore, we check if the test was expected to fail because of
-            # nodiscard before enabling it
-            test_str_list = [b'ignoring return value', b'nodiscard',
-                             b'NODISCARD']
-            if any(test_str in contents for test_str in test_str_list):
-                test_cxx.flags += ['-Werror=unused-result']
         cmd, out, err, rc = test_cxx.compile(source_path, out=os.devnull)
         check_rc = lambda rc: rc == 0 if use_verify else rc != 0
         report = libcxx.util.makeReport(cmd, out, err, rc)
diff --git a/libcxx/utils/libcxx/test/newformat.py b/libcxx/utils/libcxx/test/newformat.py
new file mode 100644
index 0000000000000..649259b2f87ac
--- /dev/null
+++ b/libcxx/utils/libcxx/test/newformat.py
@@ -0,0 +1,243 @@
+#===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===----------------------------------------------------------------------===##
+
+import lit
+import os
+import pipes
+
+class CxxStandardLibraryTest(lit.formats.TestFormat):
+    """
+    Lit test format for the C++ Standard Library conformance test suite.
+
+    This test format is based on top of the ShTest format -- it basically
+    creates a shell script performing the right operations (compile/link/run)
+    based on the extension of the test file it encounters. It supports files
+    with the following extensions:
+
+    FOO.pass.cpp            - Compiles, links and runs successfully
+    FOO.pass.mm             - Same as .pass.cpp, but for Objective-C++
+    FOO.run.fail.cpp        - Compiles and links successfully, but fails at runtime
+    FOO.compile.pass.cpp    - Compiles successfully, link and run not attempted
+    FOO.compile.fail.cpp    - Does not compile successfully. These tests run with
+                              clang-verify.
+    FOO.link.pass.cpp       - Compiles and links successfully, run not attempted
+    FOO.link.fail.cpp       - Compiles successfully, but fails to link
+    FOO.sh.cpp              - A builtin lit Shell test
+
+    FOO.fail.cpp            - Does not compile successfully -- run with clang-verify
+                              if any expected-meow appears in the file, otherwise
+                              just test that compilation fails. This is supported
+                              only for backwards compatibility with the test suite
+    FOO.fail.mm             - Same as .fail.cpp, but for Objective-C++
+
+    The test format operates by assuming that each test's configuration provides
+    the following substitutions, which it will reuse in the shell scripts it
+    constructs:
+        %{cxx}           - A command that can be used to invoke the compiler
+        %{compile_flags} - Flags to use when compiling a test case
+        %{link_flags}    - Flags to use when linking a test case
+        %{exec}          - A command to prefix the execution of executables
+
+    Note that when building an executable (as opposed to only compiling a source
+    file), both %{compile_flags} and %{link_flags} will be used in the same
+    command line. In other words, the test format doesn't perform separate
+    compilation and linking steps in this case.
+
+
+    In addition to everything that's supported in Lit ShTests, this test format
+    also understands the following directives inside test files:
+
+        // FILE_DEPENDENCIES: file, directory, /path/to/file
+
+            This directive expresses that the test requires the provided files
+            or directories in order to run. An example is a test that requires
+            some test input stored in a data file. When a test file contains
+            such a directive, this test format will collect them and make them
+            available in a special %{file_dependencies} substitution. The intent
+            is that if one needs to e.g. execute tests on a remote host, the
+            %{exec} substitution could use %{file_dependencies} to know which
+            files and directories to copy to the remote host.
+
+        // ADDITIONAL_COMPILE_FLAGS: flag1, flag2, flag3
+
+            This directive will cause the provided flags to be added to the
+            %{compile_flags} substitution for the test that contains it. This
+            allows adding special compilation flags without having to use a
+            .sh.cpp test, which would be more powerful but perhaps overkill.
+
+
+    Design note:
+    This test format never implicitly disables a type of test. For example,
+    we could be tempted to automatically mark `.compile.fail.cpp` tests as
+    UNSUPPORTED when clang-verify isn't supported by the compiler. However,
+    this sort of logic has been known to cause tests to be ignored in the
+    past, so we favour having tests mark themselves as unsupported explicitly.
+
+    This test format still needs work in the following areas:
+        - It doesn't support the MODULES_DEFINES from the old test format, but
+          that can be achieved easily with ADDITIONAL_COMPILE_FLAGS.
+        - It is unknown how well it works on Windows yet.
+    """
+    def getTestsInDirectory(self, testSuite, pathInSuite, litConfig, localConfig):
+        SUPPORTED_SUFFIXES = ['.pass.cpp', '.run.fail.cpp',
+                              '.pass.mm',
+                              '.compile.pass.cpp', '.compile.fail.cpp',
+                              '.link.pass.cpp', '.link.fail.cpp',
+                              '.sh.cpp',
+                              '.fail.cpp', '.fail.mm']
+        sourcePath = testSuite.getSourcePath(pathInSuite)
+        for filename in os.listdir(sourcePath):
+            # Ignore dot files and excluded tests.
+            if filename.startswith('.') or filename in localConfig.excludes:
+                continue
+
+            filepath = os.path.join(sourcePath, filename)
+            if not os.path.isdir(filepath):
+                if any([filename.endswith(ext) for ext in SUPPORTED_SUFFIXES]):
+                    yield lit.Test.Test(testSuite, pathInSuite + (filename,), localConfig)
+
+    def _checkSubstitutions(self, substitutions):
+        substitutions = [s for (s, _) in substitutions]
+        for s in ['%{cxx}', '%{compile_flags}', '%{link_flags}', '%{exec}']:
+            assert s in substitutions, "Required substitution {} was not provided".format(s)
+
+    # Determine whether -verify should be used for a given test. We use -verify
+    # if the compiler supports it and there's at least one -verify tag in the
+    # source file.
+    #
+    # This is only supported for backwards compatibility with .fail.cpp tests.
+    def _useVerify(self, test, litConfig):
+        VERIFY_TAGS = (b'expected-note', b'expected-remark',
+                       b'expected-warning', b'expected-error',
+                       b'expected-no-diagnostics')
+        with open(test.getSourcePath(), 'rb') as f:
+            contents = f.read()
+        testContainsTags = any(tag in contents for tag in VERIFY_TAGS)
+
+        command = "echo | %{cxx} -xc++ - -Werror -fsyntax-only -Xclang -verify-ignore-unexpected"
+        result = lit.TestRunner.executeShTest(test, litConfig,
+                                              useExternalSh=True,
+                                              preamble_commands=[command])
+        compilerSupportsVerify = result.code != lit.Test.FAIL
+        return compilerSupportsVerify and testContainsTags
+
+    def execute(self, test, litConfig):
+        self._checkSubstitutions(test.config.substitutions)
+        VERIFY_FLAGS = '-Xclang -verify -Xclang -verify-ignore-unexpected=note -ferror-limit=0'
+        filename = test.path_in_suite[-1]
+        if filename.endswith('.sh.cpp'):
+            steps = [ ] # The steps are already in the script
+            return self._executeShTest(test, litConfig, steps)
+        elif filename.endswith('.compile.pass.cpp'):
+            steps = [
+                "%dbg(COMPILED WITH) %{cxx} %s %{compile_flags} -fsyntax-only"
+            ]
+            return self._executeShTest(test, litConfig, steps)
+        elif filename.endswith('.compile.fail.cpp'):
+            steps = [
+                "%dbg(COMPILED WITH) %{cxx} %s %{compile_flags} -fsyntax-only " + VERIFY_FLAGS
+            ]
+            return self._executeShTest(test, litConfig, steps)
+        elif filename.endswith('.link.pass.cpp'):
+            steps = [
+                "%dbg(COMPILED WITH) %{cxx} %s %{compile_flags} %{link_flags} -o %t.exe"
+            ]
+            return self._executeShTest(test, litConfig, steps)
+        elif filename.endswith('.link.fail.cpp'):
+            steps = [
+                "%dbg(COMPILED WITH) %{cxx} %s %{compile_flags} -c -o %t.o",
+                "%dbg(LINKED WITH) ! %{cxx} %t.o %{link_flags} -o %t.exe"
+            ]
+            return self._executeShTest(test, litConfig, steps)
+        elif filename.endswith('.run.fail.cpp'):
+            steps = [
+                "%dbg(COMPILED WITH) %{cxx} %s %{compile_flags} %{link_flags} -o %t.exe",
+                "%dbg(EXECUTED AS) %{exec} ! %t.exe"
+            ]
+            return self._executeShTest(test, litConfig, steps, fileDependencies=['%t.exe'])
+        # Make sure to check these ones last, since they will match other
+        # suffixes above too.
+        elif filename.endswith('.pass.cpp') or filename.endswith('.pass.mm'):
+            steps = [
+                "%dbg(COMPILED WITH) %{cxx} %s %{compile_flags} %{link_flags} -o %t.exe",
+                "%dbg(EXECUTED AS) %{exec} %t.exe"
+            ]
+            return self._executeShTest(test, litConfig, steps, fileDependencies=['%t.exe'])
+        # This is the same as .compile.fail.cpp, provided for backwards
+        # compatibility with the test suite.
+        elif filename.endswith('.fail.cpp') or filename.endswith('.fail.mm'):
+            if self._useVerify(test, litConfig):
+                steps = [
+                    "%dbg(COMPILED WITH) %{cxx} %s %{compile_flags} -fsyntax-only " + VERIFY_FLAGS
+                ]
+            else:
+                steps = [
+                    "%dbg(COMPILED WITH) ! %{cxx} %s %{compile_flags} -fsyntax-only"
+                ]
+            return self._executeShTest(test, litConfig, steps)
+        else:
+            return lit.Test.Result(lit.Test.FAIL, "Unknown test suffix for '{}'".format(filename))
+
+    # Utility function to add compile flags in lit.local.cfg files.
+    def addCompileFlags(self, config, *flags):
+        string = ' '.join(flags)
+        config.substitutions = [(s, x + ' ' + string) if s == '%{compile_flags}' else (s, x) for (s, x) in config.substitutions]
+
+    # Modified version of lit.TestRunner.executeShTest to handle custom parsers correctly.
+    def _executeShTest(self, test, litConfig, steps, fileDependencies=None):
+        recursiveExpansionLimit = 10 # TODO: Use the value in litConfig once we set it.
+        if test.config.unsupported:
+            return lit.Test.Result(lit.Test.UNSUPPORTED, 'Test is unsupported')
+
+        additionalCompileFlags = []
+        fileDependencies = fileDependencies or []
+        parsers = [
+            lit.TestRunner.IntegratedTestKeywordParser('FILE_DEPENDENCIES:',
+                                                       lit.TestRunner.ParserKind.LIST,
+                                                       initial_value=fileDependencies),
+            lit.TestRunner.IntegratedTestKeywordParser('ADDITIONAL_COMPILE_FLAGS:',
+                                                       lit.TestRunner.ParserKind.LIST,
+                                                       initial_value=additionalCompileFlags)
+        ]
+
+        script = list(steps)
+        parsed = lit.TestRunner.parseIntegratedTestScript(test, additional_parsers=parsers,
+                                                                require_script=not script)
+        if isinstance(parsed, lit.Test.Result):
+            return parsed
+        script += parsed
+
+        if litConfig.noExecute:
+            return lit.Test.Result(lit.Test.PASS)
+
+        # Add compile flags specified with ADDITIONAL_COMPILE_FLAGS.
+        self.addCompileFlags(test.config, *additionalCompileFlags)
+
+        tmpDir, tmpBase = lit.TestRunner.getTempPaths(test)
+        useExternalSh = True
+        substitutions = lit.TestRunner.getDefaultSubstitutions(test, tmpDir, tmpBase,
+                                                               normalize_slashes=useExternalSh)
+
+        # Perform substitutions inside FILE_DEPENDENCIES lines (or injected dependencies).
+        # This allows using variables like %t in file dependencies. Also note that we really
+        # need to resolve %{file_dependencies} now, because otherwise we won't be able to
+        # make all paths absolute below.
+        fileDependencies = lit.TestRunner.applySubstitutions(fileDependencies, substitutions,
+                                                             recursion_limit=recursiveExpansionLimit)
+
+        # Add the %{file_dependencies} substitution before we perform substitutions
+        # inside the script.
+        testDir = os.path.dirname(test.getSourcePath())
+        fileDependencies = [f if os.path.isabs(f) else os.path.join(testDir, f) for f in fileDependencies]
+        substitutions.append(('%{file_dependencies}', ' '.join(map(pipes.quote, fileDependencies))))
+
+        # Perform substitution in the script itself.
+        script = lit.TestRunner.applySubstitutions(script, substitutions,
+                                                   recursion_limit=recursiveExpansionLimit)
+
+        return lit.TestRunner._runShTest(test, litConfig, useExternalSh, script, tmpBase)
diff --git a/libcxx/utils/run.py b/libcxx/utils/run.py
index 3e9a4703c8c7e..7de82c78dbfa7 100644
--- a/libcxx/utils/run.py
+++ b/libcxx/utils/run.py
@@ -14,14 +14,15 @@
 
 import argparse
 import os
+import shutil
 import subprocess
 import sys
+import tempfile
 
 
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('--codesign_identity', type=str, required=False)
-    parser.add_argument('--working_directory', type=str, required=True)
     parser.add_argument('--dependencies', type=str, nargs='*', required=True)
     parser.add_argument('--env', type=str, nargs='*', required=True)
     (args, remaining) = parser.parse_known_args(sys.argv[1:])
@@ -42,14 +43,23 @@ def main():
     # Extract environment variables into a dictionary
     env = {k : v  for (k, v) in map(lambda s: s.split('=', 1), args.env)}
 
-    # Ensure the file dependencies exist
-    for file in args.dependencies:
-        if not os.path.exists(file):
-            sys.stderr.write('Missing file {} marked as a dependency of a test'.format(file))
-            exit(1)
+    try:
+        tmpDir = tempfile.mkdtemp()
 
-    # Run the executable with the given environment in the given working directory
-    return subprocess.call(remaining, cwd=args.working_directory, env=env)
+        # Ensure the file dependencies exist and copy them to a temporary directory.
+        for dep in args.dependencies:
+            if not os.path.exists(dep):
+                sys.stderr.write('Missing file or directory "{}" marked as a dependency of a test'.format(dep))
+                exit(1)
+            if os.path.isdir(dep):
+                shutil.copytree(dep, os.path.join(tmpDir, os.path.basename(dep)), symlinks=True)
+            else:
+                shutil.copy2(dep, tmpDir)
+
+        # Run the executable with the given environment in the temporary directory.
+        return subprocess.call(' '.join(remaining), cwd=tmpDir, env=env, shell=True)
+    finally:
+        shutil.rmtree(tmpDir)
 
 if __name__ == '__main__':
     exit(main())
diff --git a/libcxx/utils/ssh.py b/libcxx/utils/ssh.py
new file mode 100644
index 0000000000000..f6b9574e39b29
--- /dev/null
+++ b/libcxx/utils/ssh.py
@@ -0,0 +1,108 @@
+#===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===----------------------------------------------------------------------===##
+
+"""
+Runs an executable on a remote host.
+
+This is meant to be used as an executor when running the C++ Standard Library
+conformance test suite.
+"""
+
+import argparse
+import os
+import posixpath
+import subprocess
+import sys
+import tarfile
+import tempfile
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--host', type=str, required=True)
+    parser.add_argument('--codesign_identity', type=str, required=False)
+    parser.add_argument('--dependencies', type=str, nargs='*', required=True)
+    parser.add_argument('--env', type=str, nargs='*', required=True)
+    (args, remaining) = parser.parse_known_args(sys.argv[1:])
+
+    if len(remaining) < 2:
+        sys.stderr.write('Missing actual commands to run')
+        return 1
+
+    commandLine = remaining[1:] # Skip the '--'
+
+    ssh = lambda command: ['ssh', '-oBatchMode=yes', args.host, command]
+    scp = lambda src, dst: ['scp', '-oBatchMode=yes', src, '{}:{}'.format(args.host, dst)]
+
+    # Create a temporary directory where the test will be run.
+    tmp = subprocess.check_output(ssh('mktemp -d /tmp/libcxx.XXXXXXXXXX'), universal_newlines=True).strip()
+
+    # HACK:
+    # If an argument is a file that ends in `.tmp.exe`, assume it is the name
+    # of an executable generated by a test file. We call these test-executables
+    # below. This allows us to do custom processing like codesigning test-executables
+    # and changing their path when running on the remote host. It's also possible
+    # for there to be no such executable, for example in the case of a .sh.cpp
+    # test.
+    isTestExe = lambda exe: exe.endswith('.tmp.exe') and os.path.exists(exe)
+    pathOnRemote = lambda file: posixpath.join(tmp, os.path.basename(file))
+
+    try:
+        # Do any necessary codesigning of test-executables found in the command line.
+        if args.codesign_identity:
+            for exe in filter(isTestExe, commandLine):
+                subprocess.check_call(['xcrun', 'codesign', '-f', '-s', args.codesign_identity, exe], env={})
+
+        # Ensure the test dependencies exist, tar them up and copy the tarball
+        # over to the remote host.
+        with tempfile.NamedTemporaryFile(suffix='.tar') as tmpTar:
+            with tarfile.open(fileobj=tmpTar, mode='w') as tarball:
+                for dep in args.dependencies:
+                    if not os.path.exists(dep):
+                        sys.stderr.write('Missing file or directory "{}" marked as a dependency of a test'.format(dep))
+                        return 1
+                    tarball.add(dep, arcname=os.path.basename(dep))
+
+            remoteTarball = pathOnRemote(tmpTar.name)
+            tmpTar.flush()
+            subprocess.check_call(scp(tmpTar.name, remoteTarball))
+
+        # Untar the dependencies in the temporary directory and remove the tarball.
+        remoteCommands = [
+            'tar -xf {} -C {}'.format(remoteTarball, tmp),
+            'rm {}'.format(remoteTarball)
+        ]
+
+        # Make sure all test-executables in the remote command line have 'execute'
+        # permissions on the remote host. The host that compiled the test-executable
+        # might not have a notion of 'executable' permissions.
+        for exe in map(pathOnRemote, filter(isTestExe, commandLine)):
+            remoteCommands.append('chmod +x {}'.format(exe))
+
+        # Execute the command through SSH in the temporary directory, with the
+        # correct environment. We tweak the command line to run it on the remote
+        # host by transforming the path of test-executables to their path in the
+        # temporary directory, where we know they have been copied when we handled
+        # test dependencies above.
+        remoteCommands += [
+            'cd {}'.format(tmp),
+            'export {}'.format(' '.join(args.env)),
+            ' '.join(pathOnRemote(x) if isTestExe(x) else x for x in commandLine)
+        ]
+
+        # Finally, SSH to the remote host and execute all the commands.
+        rc = subprocess.call(ssh(' && '.join(remoteCommands)))
+        return rc
+
+    finally:
+        # Make sure the temporary directory is removed when we're done.
+        subprocess.check_call(ssh('rm -r {}'.format(tmp)))
+
+
+if __name__ == '__main__':
+    exit(main())
diff --git a/libcxxabi/CMakeLists.txt b/libcxxabi/CMakeLists.txt
index 9b96224774fb7..a269806891d94 100644
--- a/libcxxabi/CMakeLists.txt
+++ b/libcxxabi/CMakeLists.txt
@@ -1,5 +1,9 @@
 # See www/CMake.html for instructions on how to build libcxxabi with CMake.
 
+if (NOT IS_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}/../libcxx")
+  message(FATAL_ERROR "libc++abi now requires being built in a monorepo layout with libcxx available")
+endif()
+
 #===============================================================================
 # Setup Project
 #===============================================================================
@@ -65,6 +69,7 @@ option(LIBCXXABI_BUILD_EXTERNAL_THREAD_LIBRARY
 # FIXME: This option should default to off. Unfortunatly GCC 4.9 fails to link
 # programs to due undefined references to new/delete in libc++abi. Once this
 # has been fixed or worked around the default value should be changed.
+# See https://reviews.llvm.org/D68269 for more details.
 option(LIBCXXABI_ENABLE_NEW_DELETE_DEFINITIONS
     "Build libc++abi with definitions for operator new/delete. Normally libc++
     provides these definitions" ON)
diff --git a/libcxxabi/src/CMakeLists.txt b/libcxxabi/src/CMakeLists.txt
index 965b1e2238301..44aa6384e0b71 100644
--- a/libcxxabi/src/CMakeLists.txt
+++ b/libcxxabi/src/CMakeLists.txt
@@ -295,7 +295,4 @@ if (NOT CMAKE_CONFIGURATION_TYPES AND LIBCXXABI_INSTALL_LIBRARY)
             -DCMAKE_INSTALL_COMPONENT=cxxabi
             -DCMAKE_INSTALL_DO_STRIP=1
             -P "${LIBCXXABI_BINARY_DIR}/cmake_install.cmake")
-
-  # TODO: This is a legacy target name and should be removed at some point.
-  add_custom_target(install-libcxxabi DEPENDS install-cxxabi)
 endif()
diff --git a/libcxxabi/test/CMakeLists.txt b/libcxxabi/test/CMakeLists.txt
index b47eef750a744..bedfce8bc397b 100644
--- a/libcxxabi/test/CMakeLists.txt
+++ b/libcxxabi/test/CMakeLists.txt
@@ -81,6 +81,3 @@ add_lit_testsuite(check-cxxabi "Running libcxxabi tests"
   ${CMAKE_CURRENT_BINARY_DIR}
   DEPENDS ${LIBCXXABI_TEST_DEPS}
   )
-
-# TODO: This is a legacy target name and should be removed at some point.
-add_custom_target(check-libcxxabi DEPENDS check-cxxabi)
diff --git a/libcxxabi/test/incomplete_type.sh.cpp b/libcxxabi/test/incomplete_type.sh.cpp
index 5937c83a0d0d6..a762d2f9c60ee 100644
--- a/libcxxabi/test/incomplete_type.sh.cpp
+++ b/libcxxabi/test/incomplete_type.sh.cpp
@@ -15,14 +15,15 @@
 
 // UNSUPPORTED: libcxxabi-no-exceptions
 
-// NOTE: Pass -lc++abi explicitly and before -lc++ so that -lc++ doesn't drag
+// NOTE: Link libc++abi explicitly and before libc++ so that libc++ doesn't drag
 // in the system libc++abi installation on OS X. (DYLD_LIBRARY_PATH is ignored
 // for shell tests because of Apple security features).
 
-// RUN: %cxx %flags %compile_flags -c %s -o %t.one.o
-// RUN: %cxx %flags %compile_flags -c %s -o %t.two.o -DTU_ONE
-// RUN: %cxx %flags %t.one.o %t.two.o -lc++abi %link_flags -o %t.exe
-// RUN: %t.exe
+// FILE_DEPENDENCIES: %t.exe
+// RUN: %{cxx} %{flags} %{compile_flags} -c %s -o %t.one.o
+// RUN: %{cxx} %{flags} %{compile_flags} -c %s -o %t.two.o -DTU_ONE
+// RUN: %{cxx} %{flags} %t.one.o %t.two.o %{link_libcxxabi} %{link_flags} -o %t.exe
+// RUN: %{exec} %t.exe
 
 #include <stdio.h>
 #include <cstring>
diff --git a/libcxxabi/test/libcxxabi/test/config.py b/libcxxabi/test/libcxxabi/test/config.py
index 32b2f74357ba3..5f652c2f00c01 100644
--- a/libcxxabi/test/libcxxabi/test/config.py
+++ b/libcxxabi/test/libcxxabi/test/config.py
@@ -42,9 +42,6 @@ def configure_features(self):
             self.config.available_features.add('libcxxabi-no-exceptions')
         if not self.has_cpp_feature('noexcept_function_type', 201510):
             self.config.available_features.add('libcxxabi-no-noexcept-function-type')
-        # test_exception_storage_nodynmem.pass.cpp fails under this specific configuration
-        if self.get_lit_bool('cxx_ext_threads', False) and self.get_lit_bool('libcxxabi_shared', False):
-            self.config.available_features.add('libcxxabi-shared-externally-threaded')
         if not self.get_lit_bool('llvm_unwinder', False):
             self.config.available_features.add('libcxxabi-has-system-unwinder')
 
@@ -61,7 +58,7 @@ def configure_compile_flags(self):
             self.cxx.compile_flags += ['-D_LIBCXXABI_HAS_NO_THREADS']
             self.config.available_features.add('libcxxabi-no-threads')
         super(Configuration, self).configure_compile_flags()
-    
+
     def configure_compile_flags_header_includes(self):
         self.configure_config_site_header()
         cxx_headers = self.get_lit_conf(
diff --git a/libcxxabi/test/lit.cfg b/libcxxabi/test/lit.cfg
index 2e23c564b7924..5d79f169de784 100644
--- a/libcxxabi/test/lit.cfg
+++ b/libcxxabi/test/lit.cfg
@@ -64,4 +64,9 @@ config_module = __import__(config_module_name, fromlist=['Configuration'])
 configuration = config_module.Configuration(lit_config, config)
 configuration.configure()
 configuration.print_config_info()
-config.test_format = configuration.get_test_format()
+if lit_config.params.get('use_new_format', False):
+    lit_config.note("Using the experimental libc++ testing format")
+    import libcxx.test.newformat
+    config.test_format = libcxx.test.newformat.CxxStandardLibraryTest()
+else:
+    config.test_format = configuration.get_test_format()
diff --git a/libcxxabi/test/native/arm-linux-eabi/ttype-encoding-00.pass.sh.s b/libcxxabi/test/native/arm-linux-eabi/ttype-encoding-00.pass.sh.s
index 0484e9fac2bd4..49166dad846c5 100644
--- a/libcxxabi/test/native/arm-linux-eabi/ttype-encoding-00.pass.sh.s
+++ b/libcxxabi/test/native/arm-linux-eabi/ttype-encoding-00.pass.sh.s
@@ -1,4 +1,4 @@
-@ RUN: %cxx %flags %link_flags %s -o %t.exe
+@ RUN: %{cxx} %{flags} %{link_flags} %s -o %t.exe
 @ RUN: %t.exe
 @ UNSUPPORTED: libcxxabi-no-exceptions
 
diff --git a/libcxxabi/test/native/arm-linux-eabi/ttype-encoding-90.pass.sh.s b/libcxxabi/test/native/arm-linux-eabi/ttype-encoding-90.pass.sh.s
index 837602bceab68..40139bbd87bce 100644
--- a/libcxxabi/test/native/arm-linux-eabi/ttype-encoding-90.pass.sh.s
+++ b/libcxxabi/test/native/arm-linux-eabi/ttype-encoding-90.pass.sh.s
@@ -1,4 +1,4 @@
-@ RUN: %cxx %flags %link_flags %s -o %t.exe
+@ RUN: %{cxx} %{flags} %{link_flags} %s -o %t.exe
 @ RUN: %t.exe
 @ UNSUPPORTED: libcxxabi-no-exceptions
 
diff --git a/libcxxabi/test/testit b/libcxxabi/test/testit
deleted file mode 100755
index eca0b37bae36e..0000000000000
--- a/libcxxabi/test/testit
+++ /dev/null
@@ -1,126 +0,0 @@
-#!/bin/sh
-#===------------------------------ testit ----------------------------------===#
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-#===------------------------------------------------------------------------===#
-
-if [ -z "$CC" ]
-then
-	CC=clang++
-fi
-
-if [ -z "$OPTIONS" ]
-then
-	OPTIONS="-std=c++0x -stdlib=libc++"
-fi
-
-case $TRIPLE in
-  *-*-mingw* | *-*-cygwin* | *-*-win*)
-	TEST_EXE=test.exe
-    ;;
-  *)
-    TEST_EXE=a.out
-    ;;
-esac
-
-FAIL=0
-PASS=0
-UNIMPLEMENTED=0
-IMPLEMENTED_FAIL=0
-IMPLEMENTED_PASS=0
-
-afunc()
-{
-	fail=0
-	pass=0
-	if (ls *.fail.cpp > /dev/null 2>&1)
-	then
-		for FILE in $(ls *.fail.cpp); do
-			if $CC $OPTIONS $HEADER_INCLUDE $SOURCE_LIB $FILE $LIBS -o ./$TEST_EXE > /dev/null 2>&1
-			then
-				rm ./$TEST_EXE
-				echo "$FILE should not compile"
-				fail=$(($fail + 1))
-			else
-				pass=$(($pass + 1))
-			fi
-		done
-	fi
-
-	if (ls *.cpp > /dev/null 2>&1)
-	then
-		for FILE in $(ls *.pass.cpp); do
-			if $CC $OPTIONS $HEADER_INCLUDE $SOURCE_LIB $FILE $LIBS -o ./$TEST_EXE
-			then
-				if ./$TEST_EXE
-				then
-					rm ./$TEST_EXE
-					pass=$(($pass + 1))
-				else
-					echo "$FILE failed at run time"
-					fail=$(($fail + 1))
-					rm ./$TEST_EXE
-				fi
-			else
-				echo "$FILE failed to compile"
-				fail=$(($fail + 1))
-			fi
-		done
-	fi
-
-	if [ $fail -gt 0 ]
-	then
-		echo "failed $fail tests in `pwd`"
-		IMPLEMENTED_FAIL=$(($IMPLEMENTED_FAIL + 1))
-	fi
-	if [ $pass -gt 0 ]
-	then
-		echo "passed $pass tests in `pwd`"
-		if [ $fail -eq 0 ]
-		then
-			IMPLEMENTED_PASS=$(($IMPLEMENTED_PASS + 1))
-		fi
-	fi
-	if [ $fail -eq 0 -a $pass -eq 0 ]
-	then
-		echo "not implemented:  `pwd`"
-		UNIMPLEMENTED=$(($UNIMPLEMENTED + 1))
-	fi
-
-	FAIL=$(($FAIL + $fail))
-	PASS=$(($PASS + $pass))
-
-	for FILE in *
-	do
-		if [ -d "$FILE" ];
-		then
-			cd $FILE
-			afunc
-			cd ..
-		fi
-	done
-}
-
-afunc
-
-echo "****************************************************"
-echo "Results for `pwd`:"
-echo "using `$CC --version`"
-echo "with $OPTIONS $HEADER_INCLUDE $SOURCE_LIB"
-echo "----------------------------------------------------"
-echo "sections without tests   : $UNIMPLEMENTED"
-echo "sections with failures   : $IMPLEMENTED_FAIL"
-echo "sections without failures: $IMPLEMENTED_PASS"
-echo "                       +   ----"
-echo "total number of sections : $(($UNIMPLEMENTED+$IMPLEMENTED_FAIL+$IMPLEMENTED_PASS))"
-echo "----------------------------------------------------"
-echo "number of tests failed   : $FAIL"
-echo "number of tests passed   : $PASS"
-echo "                       +   ----"
-echo "total number of tests    : $(($FAIL+$PASS))"
-echo "****************************************************"
-
-exit $FAIL
diff --git a/libunwind/src/UnwindCursor.hpp b/libunwind/src/UnwindCursor.hpp
index e7fb70cc57189..32d71c2c1ed31 100644
--- a/libunwind/src/UnwindCursor.hpp
+++ b/libunwind/src/UnwindCursor.hpp
@@ -1848,6 +1848,12 @@ void UnwindCursor<A, R>::setInfoBasedOnIPRegister(bool isReturnAddress) {
   pc &= (pint_t)~0x1;
 #endif
 
+  // Exit early if at the top of the stack.
+  if (pc == 0) {
+    _unwindInfoMissing = true;
+    return;
+  }
+
   // If the last line of a function is a "throw" the compiler sometimes
   // emits no instructions after the call to __cxa_throw.  This means
   // the return address is actually the start of the next function.
diff --git a/lld/CMakeLists.txt b/lld/CMakeLists.txt
index a506afa31fd38..3bd9e9556a855 100644
--- a/lld/CMakeLists.txt
+++ b/lld/CMakeLists.txt
@@ -194,5 +194,6 @@ endif()
 add_subdirectory(docs)
 add_subdirectory(COFF)
 add_subdirectory(ELF)
+add_subdirectory(MachO)
 add_subdirectory(MinGW)
 add_subdirectory(wasm)
diff --git a/lld/COFF/Config.h b/lld/COFF/Config.h
index e376ea51f133d..93b732ea68216 100644
--- a/lld/COFF/Config.h
+++ b/lld/COFF/Config.h
@@ -144,7 +144,7 @@ struct Configuration {
   unsigned ltoo = 2;
 
   // Used for /opt:lldltojobs=N
-  unsigned thinLTOJobs = 0;
+  std::string thinLTOJobs;
   // Used for /opt:lldltopartitions=N
   unsigned ltoPartitions = 1;
 
diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp
index 8b3ba1cdf24b5..931b8cf601a75 100644
--- a/lld/COFF/Driver.cpp
+++ b/lld/COFF/Driver.cpp
@@ -1144,7 +1144,17 @@ void LinkerDriver::link(ArrayRef<const char *> argsArr) {
     return;
   }
 
-  lld::threadsEnabled = args.hasFlag(OPT_threads, OPT_threads_no, true);
+  // /threads: takes a positive integer and provides the default value for
+  // /opt:lldltojobs=.
+  if (auto *arg = args.getLastArg(OPT_threads)) {
+    StringRef v(arg->getValue());
+    unsigned threads = 0;
+    if (!llvm::to_integer(v, threads, 0) || threads == 0)
+      error(arg->getSpelling() + ": expected a positive integer, but got '" +
+            arg->getValue() + "'");
+    parallel::strategy = hardware_concurrency(threads);
+    config->thinLTOJobs = v.str();
+  }
 
   if (args.hasArg(OPT_show_timing))
     config->showTiming = true;
@@ -1417,9 +1427,9 @@ void LinkerDriver::link(ArrayRef<const char *> argsArr) {
           error("/opt:lldlto: invalid optimization level: " + optLevel);
       } else if (s.startswith("lldltojobs=")) {
         StringRef jobs = s.substr(11);
-        if (jobs.getAsInteger(10, config->thinLTOJobs) ||
-            config->thinLTOJobs == 0)
+        if (!get_threadpool_strategy(jobs))
           error("/opt:lldltojobs: invalid job count: " + jobs);
+        config->thinLTOJobs = jobs.str();
       } else if (s.startswith("lldltopartitions=")) {
         StringRef n = s.substr(17);
         if (n.getAsInteger(10, config->ltoPartitions) ||
diff --git a/lld/COFF/InputFiles.cpp b/lld/COFF/InputFiles.cpp
index 8cf2a037af548..afe8ef455336f 100644
--- a/lld/COFF/InputFiles.cpp
+++ b/lld/COFF/InputFiles.cpp
@@ -829,7 +829,7 @@ Optional<DILineInfo> ObjFile::getDILineInfo(uint32_t offset,
   return dwarf->getDILineInfo(offset, sectionIndex);
 }
 
-StringRef ltrim1(StringRef s, const char *chars) {
+static StringRef ltrim1(StringRef s, const char *chars) {
   if (!s.empty() && strchr(chars, s[0]))
     return s.substr(1);
   return s;
diff --git a/lld/COFF/LTO.cpp b/lld/COFF/LTO.cpp
index 247331b169286..94906e5005932 100644
--- a/lld/COFF/LTO.cpp
+++ b/lld/COFF/LTO.cpp
@@ -101,8 +101,9 @@ BitcodeCompiler::BitcodeCompiler() {
         std::string(config->thinLTOPrefixReplace.first),
         std::string(config->thinLTOPrefixReplace.second),
         config->thinLTOEmitImportsFiles, indexFile.get(), OnIndexWrite);
-  } else if (config->thinLTOJobs != 0) {
-    backend = lto::createInProcessThinBackend(config->thinLTOJobs);
+  } else {
+    backend = lto::createInProcessThinBackend(
+        llvm::heavyweight_hardware_concurrency(config->thinLTOJobs));
   }
 
   ltoObj = std::make_unique<lto::LTO>(createConfig(), backend,
diff --git a/lld/COFF/MarkLive.cpp b/lld/COFF/MarkLive.cpp
index 6d34cb864e3c3..0afa615a1933e 100644
--- a/lld/COFF/MarkLive.cpp
+++ b/lld/COFF/MarkLive.cpp
@@ -28,10 +28,12 @@ void markLive(ArrayRef<Chunk *> chunks) {
   // as we push, so sections never appear twice in the list.
   SmallVector<SectionChunk *, 256> worklist;
 
-  // COMDAT section chunks are dead by default. Add non-COMDAT chunks.
+  // COMDAT section chunks are dead by default. Add non-COMDAT chunks. Do not
+  // traverse DWARF sections. They are live, but they should not keep other
+  // sections alive.
   for (Chunk *c : chunks)
     if (auto *sc = dyn_cast<SectionChunk>(c))
-      if (sc->live)
+      if (sc->live && !sc->isDWARF())
         worklist.push_back(sc);
 
   auto enqueue = [&](SectionChunk *c) {
diff --git a/lld/COFF/Options.td b/lld/COFF/Options.td
index 72fe9ce8c118c..f8fe6f9efc194 100644
--- a/lld/COFF/Options.td
+++ b/lld/COFF/Options.td
@@ -219,9 +219,9 @@ def lto_obj_path : P<
     "output native object for merged LTO unit to this path">;
 def dash_dash_version : Flag<["--"], "version">,
   HelpText<"Print version information">;
-defm threads: B<"threads",
-    "Run the linker multi-threaded (default)",
-    "Do not run the linker multi-threaded">;
+def threads
+    : P<"threads", "Number of threads. '1' disables multi-threading. By "
+                   "default all available hardware threads are used">;
 
 // Flags for debugging
 def lldmap : F<"lldmap">;
diff --git a/lld/COFF/PDB.cpp b/lld/COFF/PDB.cpp
index 3754bd0d1cf6e..8ecfdca768a97 100644
--- a/lld/COFF/PDB.cpp
+++ b/lld/COFF/PDB.cpp
@@ -1385,6 +1385,8 @@ void PDBLinker::printStats() {
       TypeIndex typeIndex;
       uint64_t totalInputSize() const { return uint64_t(dupCount) * typeSize; }
       bool operator<(const TypeSizeInfo &rhs) const {
+        if (totalInputSize() == rhs.totalInputSize())
+          return typeIndex < rhs.typeIndex;
         return totalInputSize() < rhs.totalInputSize();
       }
     };
diff --git a/lld/COFF/SymbolTable.cpp b/lld/COFF/SymbolTable.cpp
index 7072f4d8d0e30..8c5556a451b97 100644
--- a/lld/COFF/SymbolTable.cpp
+++ b/lld/COFF/SymbolTable.cpp
@@ -204,7 +204,7 @@ static void reportUndefinedSymbol(const UndefinedDiag &undefDiag) {
   llvm::raw_string_ostream os(out);
   os << "undefined symbol: " << toString(*undefDiag.sym);
 
-  const size_t maxUndefReferences = 10;
+  const size_t maxUndefReferences = 3;
   size_t i = 0, numRefs = 0;
   for (const UndefinedDiag::File &ref : undefDiag.files) {
     std::vector<std::string> symbolLocations =
diff --git a/lld/Common/CMakeLists.txt b/lld/Common/CMakeLists.txt
index 7d5ad654925ee..53649032bd987 100644
--- a/lld/Common/CMakeLists.txt
+++ b/lld/Common/CMakeLists.txt
@@ -36,7 +36,6 @@ add_lld_library(lldCommon
   Reproduce.cpp
   Strings.cpp
   TargetOptionsCommandFlags.cpp
-  Threads.cpp
   Timer.cpp
   VCSVersion.inc
   Version.cpp
diff --git a/lld/Common/Filesystem.cpp b/lld/Common/Filesystem.cpp
index 75e88dbce1ab4..d75c462d244d5 100644
--- a/lld/Common/Filesystem.cpp
+++ b/lld/Common/Filesystem.cpp
@@ -43,7 +43,7 @@ void lld::unlinkAsync(StringRef path) {
 #if defined(_WIN32)
   sys::fs::remove(path);
 #else
-  if (!threadsEnabled || !sys::fs::exists(path) ||
+  if (parallel::strategy.ThreadsRequested == 1 || !sys::fs::exists(path) ||
       !sys::fs::is_regular_file(path))
     return;
 
diff --git a/lld/Common/Threads.cpp b/lld/Common/Threads.cpp
deleted file mode 100644
index af04972a37602..0000000000000
--- a/lld/Common/Threads.cpp
+++ /dev/null
@@ -1,11 +0,0 @@
-//===- Threads.cpp --------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "lld/Common/Threads.h"
-
-bool lld::threadsEnabled = true;
diff --git a/lld/ELF/Arch/PPC64.cpp b/lld/ELF/Arch/PPC64.cpp
index d2af7f50cc4c2..fc212f5397316 100644
--- a/lld/ELF/Arch/PPC64.cpp
+++ b/lld/ELF/Arch/PPC64.cpp
@@ -137,7 +137,7 @@ getRelaTocSymAndAddend(InputSectionBase *tocSec, uint64_t offset) {
 
 // When accessing a symbol defined in another translation unit, compilers
 // reserve a .toc entry, allocate a local label and generate toc-indirect
-// instuctions:
+// instructions:
 //
 //   addis 3, 2, .LC0@toc@ha  # R_PPC64_TOC16_HA
 //   ld    3, .LC0@toc@l(3)   # R_PPC64_TOC16_LO_DS, load the address from a .toc entry
diff --git a/lld/ELF/CallGraphSort.cpp b/lld/ELF/CallGraphSort.cpp
index 6dad7c965f1a7..9be27986693cf 100644
--- a/lld/ELF/CallGraphSort.cpp
+++ b/lld/ELF/CallGraphSort.cpp
@@ -263,7 +263,7 @@ DenseMap<const InputSectionBase *, int> CallGraphSort::run() {
 // Sort sections by the profile data provided by -callgraph-profile-file
 //
 // This first builds a call graph based on the profile data then merges sections
-// according to the C³ huristic. All clusters are then sorted by a density
+// according to the C³ heuristic. All clusters are then sorted by a density
 // metric to further improve locality.
 DenseMap<const InputSectionBase *, int> computeCallGraphProfileOrder() {
   return CallGraphSort().run();
diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index 9f9f0d696f632..0263708634cbb 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -246,7 +246,7 @@ struct Configuration {
   unsigned ltoPartitions;
   unsigned ltoo;
   unsigned optimize;
-  unsigned thinLTOJobs;
+  StringRef thinLTOJobs;
   unsigned timeTraceGranularity;
   int32_t splitStackAdjustSize;
 
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index f9a1340e25d3e..c31cbcc8a54d0 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -860,7 +860,6 @@ static void readConfigs(opt::InputArgList &args) {
       args.hasFlag(OPT_fatal_warnings, OPT_no_fatal_warnings, false);
   errorHandler().vsDiagnostics =
       args.hasArg(OPT_visual_studio_diagnostics_format, false);
-  threadsEnabled = args.hasFlag(OPT_threads, OPT_no_threads, true);
 
   config->allowMultipleDefinition =
       args.hasFlag(OPT_allow_multiple_definition,
@@ -974,7 +973,6 @@ static void readConfigs(opt::InputArgList &args) {
   config->thinLTOIndexOnly = args.hasArg(OPT_thinlto_index_only) ||
                              args.hasArg(OPT_thinlto_index_only_eq);
   config->thinLTOIndexOnlyArg = args.getLastArgValue(OPT_thinlto_index_only_eq);
-  config->thinLTOJobs = args::getInteger(args, OPT_thinlto_jobs, -1u);
   config->thinLTOObjectSuffixReplace =
       getOldNewOptions(args, OPT_thinlto_object_suffix_replace_eq);
   config->thinLTOPrefixReplace =
@@ -1036,12 +1034,26 @@ static void readConfigs(opt::InputArgList &args) {
   for (auto *arg : args.filtered(OPT_mllvm))
     parseClangOption(arg->getValue(), arg->getSpelling());
 
+  // --threads= takes a positive integer and provides the default value for
+  // --thinlto-jobs=.
+  if (auto *arg = args.getLastArg(OPT_threads)) {
+    StringRef v(arg->getValue());
+    unsigned threads = 0;
+    if (!llvm::to_integer(v, threads, 0) || threads == 0)
+      error(arg->getSpelling() + ": expected a positive integer, but got '" +
+            arg->getValue() + "'");
+    parallel::strategy = hardware_concurrency(threads);
+    config->thinLTOJobs = v;
+  }
+  if (auto *arg = args.getLastArg(OPT_thinlto_jobs))
+    config->thinLTOJobs = arg->getValue();
+
   if (config->ltoo > 3)
     error("invalid optimization level for LTO: " + Twine(config->ltoo));
   if (config->ltoPartitions == 0)
     error("--lto-partitions: number of threads must be > 0");
-  if (config->thinLTOJobs == 0)
-    error("--thinlto-jobs: number of threads must be > 0");
+  if (!get_threadpool_strategy(config->thinLTOJobs))
+    error("--thinlto-jobs: invalid job count: " + config->thinLTOJobs);
 
   if (config->splitStackAdjustSize < 0)
     error("--split-stack-adjust-size: size must be >= 0");
diff --git a/lld/ELF/ICF.cpp b/lld/ELF/ICF.cpp
index cdc4a15ea691e..4b1ecadde5abc 100644
--- a/lld/ELF/ICF.cpp
+++ b/lld/ELF/ICF.cpp
@@ -400,7 +400,7 @@ template <class ELFT>
 void ICF<ELFT>::forEachClass(llvm::function_ref<void(size_t, size_t)> fn) {
   // If threading is disabled or the number of sections are
   // too small to use threading, call Fn sequentially.
-  if (!threadsEnabled || sections.size() < 1024) {
+  if (parallel::strategy.ThreadsRequested == 1 || sections.size() < 1024) {
     forEachClassRange(0, sections.size(), fn);
     ++cnt;
     return;
diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp
index 8c7ea91e05f0b..f3dd11fde3f66 100644
--- a/lld/ELF/InputFiles.cpp
+++ b/lld/ELF/InputFiles.cpp
@@ -424,6 +424,9 @@ StringRef ObjFile<ELFT>::getShtGroupSignature(ArrayRef<Elf_Shdr> sections,
 
 template <class ELFT>
 bool ObjFile<ELFT>::shouldMerge(const Elf_Shdr &sec, StringRef name) {
+  if (!(sec.sh_flags & SHF_MERGE))
+    return false;
+
   // On a regular link we don't merge sections if -O0 (default is -O1). This
   // sometimes makes the linker significantly faster, although the output will
   // be bigger.
@@ -459,10 +462,7 @@ bool ObjFile<ELFT>::shouldMerge(const Elf_Shdr &sec, StringRef name) {
           Twine(sec.sh_size) + ") must be a multiple of sh_entsize (" +
           Twine(entSize) + ")");
 
-  uint64_t flags = sec.sh_flags;
-  if (!(flags & SHF_MERGE))
-    return false;
-  if (flags & SHF_WRITE)
+  if (sec.sh_flags & SHF_WRITE)
     fatal(toString(this) + ":(" + name +
           "): writable SHF_MERGE section is not supported");
 
@@ -669,7 +669,9 @@ void ObjFile<ELFT>::initializeSections(bool ignoreComdats) {
 // the input objects have been compiled.
 static void updateARMVFPArgs(const ARMAttributeParser &attributes,
                              const InputFile *f) {
-  if (!attributes.hasAttribute(ARMBuildAttrs::ABI_VFP_args))
+  Optional<unsigned> attr =
+      attributes.getAttributeValue(ARMBuildAttrs::ABI_VFP_args);
+  if (!attr.hasValue())
     // If an ABI tag isn't present then it is implicitly given the value of 0
     // which maps to ARMBuildAttrs::BaseAAPCS. However many assembler files,
     // including some in glibc that don't use FP args (and should have value 3)
@@ -677,7 +679,7 @@ static void updateARMVFPArgs(const ARMAttributeParser &attributes,
     // as a clash.
     return;
 
-  unsigned vfpArgs = attributes.getAttributeValue(ARMBuildAttrs::ABI_VFP_args);
+  unsigned vfpArgs = attr.getValue();
   ARMVFPArgKind arg;
   switch (vfpArgs) {
   case ARMBuildAttrs::BaseAAPCS:
@@ -714,9 +716,11 @@ static void updateARMVFPArgs(const ARMAttributeParser &attributes,
 // is compiled with an architecture that supports these features then lld is
 // permitted to use them.
 static void updateSupportedARMFeatures(const ARMAttributeParser &attributes) {
-  if (!attributes.hasAttribute(ARMBuildAttrs::CPU_arch))
+  Optional<unsigned> attr =
+      attributes.getAttributeValue(ARMBuildAttrs::CPU_arch);
+  if (!attr.hasValue())
     return;
-  auto arch = attributes.getAttributeValue(ARMBuildAttrs::CPU_arch);
+  auto arch = attr.getValue();
   switch (arch) {
   case ARMBuildAttrs::Pre_v4:
   case ARMBuildAttrs::v4:
diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp
index b3031a3723c2b..e93dec947d903 100644
--- a/lld/ELF/InputSection.cpp
+++ b/lld/ELF/InputSection.cpp
@@ -708,7 +708,7 @@ static uint64_t getRelocTargetVA(const InputFile *file, RelType type, int64_t a,
     // ftp://www.linux-mips.org/pub/linux/mips/doc/ABI/mipsabi.pdf
     // microMIPS variants of these relocations use slightly different
     // expressions: AHL + GP - P + 3 for %lo() and AHL + GP - P - 1 for %hi()
-    // to correctly handle less-sugnificant bit of the microMIPS symbol.
+    // to correctly handle less-significant bit of the microMIPS symbol.
     uint64_t v = in.mipsGot->getGp(file) + a - p;
     if (type == R_MIPS_LO16 || type == R_MICROMIPS_LO16)
       v += 4;
diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp
index ba943283706a9..f4e04a82f0c2a 100644
--- a/lld/ELF/LTO.cpp
+++ b/lld/ELF/LTO.cpp
@@ -146,8 +146,9 @@ BitcodeCompiler::BitcodeCompiler() {
         std::string(config->thinLTOPrefixReplace.first),
         std::string(config->thinLTOPrefixReplace.second),
         config->thinLTOEmitImportsFiles, indexFile.get(), onIndexWrite);
-  } else if (config->thinLTOJobs != -1U) {
-    backend = lto::createInProcessThinBackend(config->thinLTOJobs);
+  } else {
+    backend = lto::createInProcessThinBackend(
+        llvm::heavyweight_hardware_concurrency(config->thinLTOJobs));
   }
 
   ltoObj = std::make_unique<lto::LTO>(createConfig(), backend,
diff --git a/lld/ELF/LinkerScript.cpp b/lld/ELF/LinkerScript.cpp
index ef767700ccd9b..4d507ee9c2c95 100644
--- a/lld/ELF/LinkerScript.cpp
+++ b/lld/ELF/LinkerScript.cpp
@@ -833,6 +833,7 @@ void LinkerScript::assignOffsets(OutputSection *sec) {
   if (!(sec->flags & SHF_ALLOC))
     dot = 0;
 
+  bool prevLMARegionIsDefault = ctx->lmaRegion == nullptr;
   ctx->memRegion = sec->memRegion;
   ctx->lmaRegion = sec->lmaRegion;
   if (ctx->memRegion)
@@ -851,19 +852,19 @@ void LinkerScript::assignOffsets(OutputSection *sec) {
 
   switchTo(sec);
 
-  ctx->lmaOffset = 0;
-
+  // ctx->lmaOffset is LMA minus VMA. If LMA is explicitly specified via AT() or
+  // AT>, recompute ctx->lmaOffset; otherwise, if both previous/current LMA
+  // region is the default, reuse previous lmaOffset; otherwise, reset lmaOffset
+  // to 0. This emulates heuristics described in
+  // https://sourceware.org/binutils/docs/ld/Output-Section-LMA.html
   if (sec->lmaExpr)
     ctx->lmaOffset = sec->lmaExpr().getValue() - dot;
-  if (MemoryRegion *mr = sec->lmaRegion)
+  else if (MemoryRegion *mr = sec->lmaRegion)
     ctx->lmaOffset = alignTo(mr->curPos, sec->alignment) - dot;
+  else if (!prevLMARegionIsDefault)
+    ctx->lmaOffset = 0;
 
-  // If neither AT nor AT> is specified for an allocatable section, the linker
-  // will set the LMA such that the difference between VMA and LMA for the
-  // section is the same as the preceding output section in the same region
-  // https://sourceware.org/binutils/docs-2.20/ld/Output-Section-LMA.html
-  // This, however, should only be done by the first "non-header" section
-  // in the segment.
+  // Propagate ctx->lmaOffset to the first "non-header" section.
   if (PhdrEntry *l = ctx->outSec->ptLoad)
     if (sec == findFirstSection(l))
       l->lmaOffset = ctx->lmaOffset;
@@ -1223,7 +1224,7 @@ std::vector<size_t> LinkerScript::getPhdrIndices(OutputSection *cmd) {
     if (Optional<size_t> idx = getPhdrIndex(phdrsCommands, s))
       ret.push_back(*idx);
     else if (s != "NONE")
-      error(cmd->location + ": section header '" + s +
+      error(cmd->location + ": program header '" + s +
             "' is not listed in PHDRS");
   }
   return ret;
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index e1c489c4a1e38..f3afb03cba471 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -350,9 +350,10 @@ defm target2:
   Eq<"target2", "Interpret R_ARM_TARGET2 as <type>, where <type> is one of rel, abs, or got-rel">,
   MetaVarName<"<type>">;
 
-defm threads: B<"threads",
-    "Run the linker multi-threaded (default)",
-    "Do not run the linker multi-threaded">;
+defm threads
+    : Eq<"threads",
+         "Number of threads. '1' disables multi-threading. By default all "
+         "available hardware threads are used">;
 
 def time_trace: F<"time-trace">, HelpText<"Record time trace">;
 def time_trace_file_eq: J<"time-trace-file=">, HelpText<"Specify time trace output file">;
@@ -509,47 +510,48 @@ defm thinlto_cache_policy: Eq<"thinlto-cache-policy", "Pruning policy for the Th
 def thinlto_emit_imports_files: F<"thinlto-emit-imports-files">;
 def thinlto_index_only: F<"thinlto-index-only">;
 def thinlto_index_only_eq: J<"thinlto-index-only=">;
-def thinlto_jobs: J<"thinlto-jobs=">, HelpText<"Number of ThinLTO jobs">;
+def thinlto_jobs: J<"thinlto-jobs=">,
+  HelpText<"Number of ThinLTO jobs. Default to --threads=">;
 def thinlto_object_suffix_replace_eq: J<"thinlto-object-suffix-replace=">;
 def thinlto_prefix_replace_eq: J<"thinlto-prefix-replace=">;
 
-def: J<"plugin-opt=O">, Alias<lto_O>, HelpText<"Alias for -lto-O">;
+def: J<"plugin-opt=O">, Alias<lto_O>, HelpText<"Alias for --lto-O">;
 def: F<"plugin-opt=debug-pass-manager">,
-  Alias<lto_debug_pass_manager>, HelpText<"Alias for -lto-debug-pass-manager">;
-def: F<"plugin-opt=disable-verify">, Alias<disable_verify>, HelpText<"Alias for -disable-verify">;
+  Alias<lto_debug_pass_manager>, HelpText<"Alias for --lto-debug-pass-manager">;
+def: F<"plugin-opt=disable-verify">, Alias<disable_verify>, HelpText<"Alias for --disable-verify">;
 def plugin_opt_dwo_dir_eq: J<"plugin-opt=dwo_dir=">,
   HelpText<"Directory to store .dwo files when LTO and debug fission are used">;
 def plugin_opt_emit_llvm: F<"plugin-opt=emit-llvm">;
-def: J<"plugin-opt=jobs=">, Alias<thinlto_jobs>, HelpText<"Alias for -thinlto-jobs">;
-def: J<"plugin-opt=lto-partitions=">, Alias<lto_partitions>, HelpText<"Alias for -lto-partitions">;
+def: J<"plugin-opt=jobs=">, Alias<thinlto_jobs>, HelpText<"Alias for --thinlto-jobs">;
+def: J<"plugin-opt=lto-partitions=">, Alias<lto_partitions>, HelpText<"Alias for --lto-partitions">;
 def plugin_opt_mcpu_eq: J<"plugin-opt=mcpu=">;
 def: F<"plugin-opt=new-pass-manager">,
-  Alias<lto_new_pass_manager>, HelpText<"Alias for -lto-new-pass-manager">;
+  Alias<lto_new_pass_manager>, HelpText<"Alias for --lto-new-pass-manager">;
 def: F<"plugin-opt=cs-profile-generate">,
-  Alias<lto_cs_profile_generate>, HelpText<"Alias for -lto-cs-profile-generate">;
+  Alias<lto_cs_profile_generate>, HelpText<"Alias for --lto-cs-profile-generate">;
 def: J<"plugin-opt=cs-profile-path=">,
-  Alias<lto_cs_profile_file>, HelpText<"Alias for -lto-cs-profile-file">;
+  Alias<lto_cs_profile_file>, HelpText<"Alias for --lto-cs-profile-file">;
 def: J<"plugin-opt=obj-path=">,
   Alias<lto_obj_path_eq>,
-  HelpText<"Alias for -lto-obj-path=">;
+  HelpText<"Alias for --lto-obj-path=">;
 def: J<"plugin-opt=sample-profile=">,
-  Alias<lto_sample_profile>, HelpText<"Alias for -lto-sample-profile">;
-def: F<"plugin-opt=save-temps">, Alias<save_temps>, HelpText<"Alias for -save-temps">;
+  Alias<lto_sample_profile>, HelpText<"Alias for --lto-sample-profile">;
+def: F<"plugin-opt=save-temps">, Alias<save_temps>, HelpText<"Alias for --save-temps">;
 def: F<"plugin-opt=thinlto-emit-imports-files">,
   Alias<thinlto_emit_imports_files>,
-  HelpText<"Alias for -thinlto-emit-imports-files">;
+  HelpText<"Alias for --thinlto-emit-imports-files">;
 def: F<"plugin-opt=thinlto-index-only">,
   Alias<thinlto_index_only>,
-  HelpText<"Alias for -thinlto-index-only">;
+  HelpText<"Alias for --thinlto-index-only">;
 def: J<"plugin-opt=thinlto-index-only=">,
   Alias<thinlto_index_only_eq>,
-  HelpText<"Alias for -thinlto-index-only=">;
+  HelpText<"Alias for --thinlto-index-only=">;
 def: J<"plugin-opt=thinlto-object-suffix-replace=">,
  Alias<thinlto_object_suffix_replace_eq>,
- HelpText<"Alias for -thinlto-object-suffix-replace=">;
+ HelpText<"Alias for --thinlto-object-suffix-replace=">;
 def: J<"plugin-opt=thinlto-prefix-replace=">,
   Alias<thinlto_prefix_replace_eq>,
-  HelpText<"Alias for -thinlto-prefix-replace=">;
+  HelpText<"Alias for --thinlto-prefix-replace=">;
 
 // Ignore LTO plugin-related options.
 // clang -flto passes -plugin and -plugin-opt to the linker. This is required
diff --git a/lld/ELF/OutputSections.cpp b/lld/ELF/OutputSections.cpp
index e5360c1ce23ac..c38d4f0b67508 100644
--- a/lld/ELF/OutputSections.cpp
+++ b/lld/ELF/OutputSections.cpp
@@ -114,8 +114,7 @@ void OutputSection::commitSection(InputSection *isec) {
     flags = isec->flags;
   } else {
     // Otherwise, check if new type or flags are compatible with existing ones.
-    unsigned mask = SHF_TLS | SHF_LINK_ORDER;
-    if ((flags & mask) != (isec->flags & mask))
+    if ((flags ^ isec->flags) & SHF_TLS)
       error("incompatible section flags for " + name + "\n>>> " + toString(isec) +
             ": 0x" + utohexstr(isec->flags) + "\n>>> output section " + name +
             ": 0x" + utohexstr(flags));
@@ -366,8 +365,9 @@ void OutputSection::finalize() {
     // all InputSections in the OutputSection have the same dependency.
     if (auto *ex = dyn_cast<ARMExidxSyntheticSection>(first))
       link = ex->getLinkOrderDep()->getParent()->sectionIndex;
-    else if (auto *d = first->getLinkOrderDep())
-      link = d->getParent()->sectionIndex;
+    else if (first->flags & SHF_LINK_ORDER)
+      if (auto *d = first->getLinkOrderDep())
+        link = d->getParent()->sectionIndex;
   }
 
   if (type == SHT_GROUP) {
diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp
index 59c9ecf0019d0..21fec6e44aa0a 100644
--- a/lld/ELF/Relocations.cpp
+++ b/lld/ELF/Relocations.cpp
@@ -870,7 +870,7 @@ static void reportUndefinedSymbol(const UndefinedDiag &undef,
   if (msg.empty())
     msg = "undefined " + visibility() + "symbol: " + toString(sym);
 
-  const size_t maxUndefReferences = 10;
+  const size_t maxUndefReferences = 3;
   size_t i = 0;
   for (UndefinedDiag::Loc l : undef.locs) {
     if (i >= maxUndefReferences)
@@ -901,8 +901,9 @@ static void reportUndefinedSymbol(const UndefinedDiag &undef,
   }
 
   if (sym.getName().startswith("_ZTV"))
-    msg += "\nthe vtable symbol may be undefined because the class is missing "
-           "its key function (see https://lld.llvm.org/missingkeyfunction)";
+    msg +=
+        "\n>>> the vtable symbol may be undefined because the class is missing "
+        "its key function (see https://lld.llvm.org/missingkeyfunction)";
 
   if (undef.isWarning)
     warn(msg);
@@ -1333,7 +1334,10 @@ static void scanReloc(InputSectionBase &sec, OffsetGetter &getOffset, RelTy *&i,
       // stub type. It should be ignored if optimized to R_PC.
       if (config->emachine == EM_PPC && expr == R_PPC32_PLTREL)
         addend &= ~0x8000;
-      expr = fromPlt(expr);
+      // R_HEX_GD_PLT_B22_PCREL (call a@GDPLT) is transformed into
+      // call __tls_get_addr even if the symbol is non-preemptible.
+      if (!(config->emachine == EM_HEXAGON && type == R_HEX_GD_PLT_B22_PCREL))
+        expr = fromPlt(expr);
     }
   }
 
diff --git a/lld/ELF/ScriptLexer.cpp b/lld/ELF/ScriptLexer.cpp
index 7453acfcf396b..200ade76d306e 100644
--- a/lld/ELF/ScriptLexer.cpp
+++ b/lld/ELF/ScriptLexer.cpp
@@ -187,7 +187,7 @@ static std::vector<StringRef> tokenizeExpr(StringRef s) {
       break;
     }
 
-    // Get a token before the opreator.
+    // Get a token before the operator.
     if (e != 0)
       ret.push_back(s.substr(0, e));
 
diff --git a/lld/ELF/ScriptParser.cpp b/lld/ELF/ScriptParser.cpp
index 89b099d492000..037cb9f31eff7 100644
--- a/lld/ELF/ScriptParser.cpp
+++ b/lld/ELF/ScriptParser.cpp
@@ -746,6 +746,7 @@ bool ScriptParser::readSectionDirective(OutputSection *cmd, StringRef tok1, Stri
   expect("(");
   if (consume("NOLOAD")) {
     cmd->noload = true;
+    cmd->type = SHT_NOBITS;
   } else {
     skip(); // This is "COPY", "INFO" or "OVERLAY".
     cmd->nonAlloc = true;
diff --git a/lld/ELF/Symbols.cpp b/lld/ELF/Symbols.cpp
index cb7ee8829e538..42c8a71c4185e 100644
--- a/lld/ELF/Symbols.cpp
+++ b/lld/ELF/Symbols.cpp
@@ -31,7 +31,18 @@ static std::string demangle(StringRef symName) {
   return std::string(symName);
 }
 
-std::string toString(const elf::Symbol &b) { return demangle(b.getName()); }
+std::string toString(const elf::Symbol &sym) {
+  StringRef name = sym.getName();
+  std::string ret = demangle(name);
+
+  // If sym has a non-default version, its name may have been truncated at '@'
+  // by Symbol::parseSymbolVersion(). Add the trailing part. This check is safe
+  // because every symbol name ends with '\0'.
+  if (name.data()[name.size()] == '@')
+    ret += name.data() + name.size();
+  return ret;
+}
+
 std::string toELFString(const Archive::Symbol &b) {
   return demangle(b.getName());
 }
diff --git a/lld/ELF/Symbols.h b/lld/ELF/Symbols.h
index ac606198afd86..ebee4af1fdad2 100644
--- a/lld/ELF/Symbols.h
+++ b/lld/ELF/Symbols.h
@@ -21,6 +21,7 @@
 #include "llvm/Object/ELF.h"
 
 namespace lld {
+// Returns a string representation for a symbol for diagnostics.
 std::string toString(const elf::Symbol &);
 
 // There are two different ways to convert an Archive::Symbol to a string:
diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp
index ed2cdb70a2d5f..8cf813ceffd0f 100644
--- a/lld/ELF/SyntheticSections.cpp
+++ b/lld/ELF/SyntheticSections.cpp
@@ -2151,7 +2151,7 @@ template <class ELFT> void SymbolTableSection<ELFT>::writeTo(uint8_t *buf) {
       eSym->st_size = sym->getSize();
 
     // st_value is usually an address of a symbol, but that has a
-    // special meaining for uninstantiated common symbols (this can
+    // special meaning for uninstantiated common symbols (this can
     // occur if -r is given).
     if (BssSection *commonSec = getCommonSec(ent.sym))
       eSym->st_value = commonSec->alignment;
@@ -2250,7 +2250,7 @@ size_t SymtabShndxSection::getSize() const {
 // DSOs. That means resolving all dynamic symbols takes O(m)*O(n)
 // where m is the number of DSOs and n is the number of dynamic
 // symbols. For modern large programs, both m and n are large.  So
-// making each step faster by using hash tables substiantially
+// making each step faster by using hash tables substantially
 // improves time to load programs.
 //
 // (Note that this is not the only way to design the shared library.
@@ -2746,11 +2746,11 @@ createSymbols(ArrayRef<std::vector<GdbIndexSection::NameAttrEntry>> nameAttrs,
   // The number of symbols we will handle in this function is of the order
   // of millions for very large executables, so we use multi-threading to
   // speed it up.
-  size_t numShards = 32;
-  size_t concurrency = 1;
-  if (threadsEnabled)
-    concurrency = std::min<size_t>(
-        hardware_concurrency().compute_thread_count(), numShards);
+  constexpr size_t numShards = 32;
+  size_t concurrency = PowerOf2Floor(
+      std::min<size_t>(hardware_concurrency(parallel::strategy.ThreadsRequested)
+                           .compute_thread_count(),
+                       numShards));
 
   // A sharded map to uniquify symbols by name.
   std::vector<DenseMap<CachedHashStringRef, size_t>> map(numShards);
@@ -3193,10 +3193,10 @@ void MergeNoTailSection::finalizeContents() {
 
   // Concurrency level. Must be a power of 2 to avoid expensive modulo
   // operations in the following tight loop.
-  size_t concurrency = 1;
-  if (threadsEnabled)
-    concurrency = std::min<size_t>(
-        hardware_concurrency().compute_thread_count(), numShards);
+  size_t concurrency = PowerOf2Floor(
+      std::min<size_t>(hardware_concurrency(parallel::strategy.ThreadsRequested)
+                           .compute_thread_count(),
+                       numShards));
 
   // Add section pieces to the builders.
   parallelForEachN(0, concurrency, [&](size_t threadId) {
diff --git a/lld/ELF/SyntheticSections.h b/lld/ELF/SyntheticSections.h
index 190a4fd3ac9e2..05eee24ce34fa 100644
--- a/lld/ELF/SyntheticSections.h
+++ b/lld/ELF/SyntheticSections.h
@@ -364,7 +364,7 @@ class MipsGotSection final : public SyntheticSection {
 
   // Try to merge two GOTs. In case of success the `Dst` contains
   // result of merging and the function returns true. In case of
-  // ovwerflow the `Dst` is unchanged and the function returns false.
+  // overflow the `Dst` is unchanged and the function returns false.
   bool tryMergeGots(FileGot & dst, FileGot & src, bool isPrimary);
 };
 
diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp
index 15b04d6fe332b..801ca7a0dc5ef 100644
--- a/lld/ELF/Writer.cpp
+++ b/lld/ELF/Writer.cpp
@@ -819,7 +819,8 @@ static bool isRelroSection(const OutputSection *sec) {
   StringRef s = sec->name;
   return s == ".data.rel.ro" || s == ".bss.rel.ro" || s == ".ctors" ||
          s == ".dtors" || s == ".jcr" || s == ".eh_frame" ||
-         s == ".openbsd.randomdata";
+         s == ".fini_array" || s == ".init_array" ||
+         s == ".openbsd.randomdata" || s == ".preinit_array";
 }
 
 // We compute a rank for each section. The rank indicates where the
@@ -1562,17 +1563,30 @@ template <class ELFT> void Writer<ELFT>::resolveShfLinkOrder() {
     // but sort must consider them all at once.
     std::vector<InputSection **> scriptSections;
     std::vector<InputSection *> sections;
+    bool started = false, stopped = false;
     for (BaseCommand *base : sec->sectionCommands) {
       if (auto *isd = dyn_cast<InputSectionDescription>(base)) {
         for (InputSection *&isec : isd->sections) {
-          scriptSections.push_back(&isec);
-          sections.push_back(isec);
-
-          InputSection *link = isec->getLinkOrderDep();
-          if (!link->getParent())
-            error(toString(isec) + ": sh_link points to discarded section " +
-                  toString(link));
+          if (!(isec->flags & SHF_LINK_ORDER)) {
+            if (started)
+              stopped = true;
+          } else if (stopped) {
+            error(toString(isec) + ": SHF_LINK_ORDER sections in " + sec->name +
+                  " are not contiguous");
+          } else {
+            started = true;
+
+            scriptSections.push_back(&isec);
+            sections.push_back(isec);
+
+            InputSection *link = isec->getLinkOrderDep();
+            if (!link->getParent())
+              error(toString(isec) + ": sh_link points to discarded section " +
+                    toString(link));
+          }
         }
+      } else if (started) {
+        stopped = true;
       }
     }
 
diff --git a/lld/MachO/Arch/X86_64.cpp b/lld/MachO/Arch/X86_64.cpp
new file mode 100644
index 0000000000000..6c0989edfd396
--- /dev/null
+++ b/lld/MachO/Arch/X86_64.cpp
@@ -0,0 +1,60 @@
+//===- X86_64.cpp ---------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Target.h"
+#include "lld/Common/ErrorHandler.h"
+#include "llvm/BinaryFormat/MachO.h"
+#include "llvm/Support/Endian.h"
+
+using namespace llvm::MachO;
+using namespace llvm::support::endian;
+using namespace lld;
+using namespace lld::macho;
+
+namespace {
+
+struct X86_64 : TargetInfo {
+  X86_64();
+  uint64_t getImplicitAddend(const uint8_t *loc, uint8_t type) const override;
+  void relocateOne(uint8_t *loc, uint8_t type, uint64_t val) const override;
+};
+
+X86_64::X86_64() {
+  cpuType = CPU_TYPE_X86_64;
+  cpuSubtype = CPU_SUBTYPE_X86_64_ALL;
+}
+
+uint64_t X86_64::getImplicitAddend(const uint8_t *loc, uint8_t type) const {
+  switch (type) {
+  case X86_64_RELOC_SIGNED:
+    return read32le(loc);
+  default:
+    error("TODO: Unhandled relocation type " + std::to_string(type));
+    return 0;
+  }
+}
+
+void X86_64::relocateOne(uint8_t *loc, uint8_t type, uint64_t val) const {
+  switch (type) {
+  case X86_64_RELOC_SIGNED:
+    // This type is only used for pc-relative relocations, so offset by 4 since
+    // the RIP has advanced by 4 at this point.
+    write32le(loc, val - 4);
+    break;
+  default:
+    llvm_unreachable(
+        "getImplicitAddend should have flagged all unhandled relocation types");
+  }
+}
+
+} // namespace
+
+TargetInfo *macho::createX86_64TargetInfo() {
+  static X86_64 t;
+  return &t;
+}
diff --git a/lld/MachO/CMakeLists.txt b/lld/MachO/CMakeLists.txt
new file mode 100644
index 0000000000000..47e4207d949b3
--- /dev/null
+++ b/lld/MachO/CMakeLists.txt
@@ -0,0 +1,31 @@
+set(LLVM_TARGET_DEFINITIONS Options.td)
+tablegen(LLVM Options.inc -gen-opt-parser-defs)
+add_public_tablegen_target(MachOOptionsTableGen)
+
+add_lld_library(lldMachO2
+  Arch/X86_64.cpp
+  Driver.cpp
+  InputFiles.cpp
+  InputSection.cpp
+  OutputSegment.cpp
+  SymbolTable.cpp
+  Symbols.cpp
+  Target.cpp
+  Writer.cpp
+
+  LINK_COMPONENTS
+  ${LLVM_TARGETS_TO_BUILD}
+  BinaryFormat
+  Core
+  Object
+  Option
+  Support
+
+  LINK_LIBS
+  lldCommon
+  ${LLVM_PTHREAD_LIB}
+
+  DEPENDS
+  MachOOptionsTableGen
+  ${tablegen_deps}
+  )
diff --git a/lld/MachO/Config.h b/lld/MachO/Config.h
new file mode 100644
index 0000000000000..ea8df75b6803b
--- /dev/null
+++ b/lld/MachO/Config.h
@@ -0,0 +1,29 @@
+//===- Config.h -------------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLD_MACHO_CONFIG_H
+#define LLD_MACHO_CONFIG_H
+
+#include "llvm/ADT/StringRef.h"
+
+namespace lld {
+namespace macho {
+
+class Symbol;
+
+struct Configuration {
+  llvm::StringRef outputFile;
+  Symbol *entry;
+};
+
+extern Configuration *config;
+
+} // namespace macho
+} // namespace lld
+
+#endif
diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
new file mode 100644
index 0000000000000..475f81a6471ad
--- /dev/null
+++ b/lld/MachO/Driver.cpp
@@ -0,0 +1,150 @@
+//===- Driver.cpp ---------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Driver.h"
+#include "Config.h"
+#include "InputFiles.h"
+#include "OutputSegment.h"
+#include "SymbolTable.h"
+#include "Symbols.h"
+#include "Target.h"
+#include "Writer.h"
+
+#include "lld/Common/Args.h"
+#include "lld/Common/Driver.h"
+#include "lld/Common/ErrorHandler.h"
+#include "lld/Common/LLVM.h"
+#include "lld/Common/Memory.h"
+#include "lld/Common/Version.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/BinaryFormat/MachO.h"
+#include "llvm/BinaryFormat/Magic.h"
+#include "llvm/Option/ArgList.h"
+#include "llvm/Option/Option.h"
+#include "llvm/Support/MemoryBuffer.h"
+
+using namespace llvm;
+using namespace llvm::MachO;
+using namespace llvm::sys;
+using namespace lld;
+using namespace lld::macho;
+
+Configuration *lld::macho::config;
+
+// Create prefix string literals used in Options.td
+#define PREFIX(NAME, VALUE) const char *NAME[] = VALUE;
+#include "Options.inc"
+#undef PREFIX
+
+// Create table mapping all options defined in Options.td
+static const opt::OptTable::Info optInfo[] = {
+#define OPTION(X1, X2, ID, KIND, GROUP, ALIAS, X7, X8, X9, X10, X11, X12)      \
+  {X1, X2, X10,         X11,         OPT_##ID, opt::Option::KIND##Class,       \
+   X9, X8, OPT_##GROUP, OPT_##ALIAS, X7,       X12},
+#include "Options.inc"
+#undef OPTION
+};
+
+MachOOptTable::MachOOptTable() : OptTable(optInfo) {}
+
+opt::InputArgList MachOOptTable::parse(ArrayRef<const char *> argv) {
+  // Make InputArgList from string vectors.
+  unsigned missingIndex;
+  unsigned missingCount;
+  SmallVector<const char *, 256> vec(argv.data(), argv.data() + argv.size());
+
+  opt::InputArgList args = ParseArgs(vec, missingIndex, missingCount);
+
+  if (missingCount)
+    error(Twine(args.getArgString(missingIndex)) + ": missing argument");
+
+  for (opt::Arg *arg : args.filtered(OPT_UNKNOWN))
+    error("unknown argument: " + arg->getSpelling());
+  return args;
+}
+
+static TargetInfo *createTargetInfo(opt::InputArgList &args) {
+  StringRef s = args.getLastArgValue(OPT_arch, "x86_64");
+  if (s != "x86_64")
+    error("missing or unsupported -arch " + s);
+  return createX86_64TargetInfo();
+}
+
+static void addFile(StringRef path) {
+  Optional<MemoryBufferRef> buffer = readFile(path);
+  if (!buffer)
+    return;
+  MemoryBufferRef mbref = *buffer;
+
+  switch (identify_magic(mbref.getBuffer())) {
+  case file_magic::macho_object:
+    inputFiles.push_back(make<ObjFile>(mbref));
+    break;
+  default:
+    error(path + ": unhandled file type");
+  }
+}
+
+bool macho::link(llvm::ArrayRef<const char *> argsArr, bool canExitEarly,
+                 raw_ostream &stdoutOS, raw_ostream &stderrOS) {
+  lld::stdoutOS = &stdoutOS;
+  lld::stderrOS = &stderrOS;
+
+  MachOOptTable parser;
+  opt::InputArgList args = parser.parse(argsArr.slice(1));
+
+  if (args.hasArg(OPT_v)) {
+    message(getLLDVersion());
+    freeArena();
+    return !errorCount();
+  }
+
+  config = make<Configuration>();
+  symtab = make<SymbolTable>();
+  target = createTargetInfo(args);
+
+  config->entry = symtab->addUndefined(args.getLastArgValue(OPT_e, "_main"));
+  config->outputFile = args.getLastArgValue(OPT_o, "a.out");
+
+  getOrCreateOutputSegment("__TEXT", VM_PROT_READ | VM_PROT_EXECUTE);
+  getOrCreateOutputSegment("__DATA", VM_PROT_READ | VM_PROT_WRITE);
+
+  for (opt::Arg *arg : args) {
+    switch (arg->getOption().getID()) {
+    case OPT_INPUT:
+      addFile(arg->getValue());
+      break;
+    }
+  }
+
+  if (!isa<Defined>(config->entry)) {
+    error("undefined symbol: " + config->entry->getName());
+    return false;
+  }
+
+  // Initialize InputSections.
+  for (InputFile *file : inputFiles)
+    for (InputSection *sec : file->sections)
+      inputSections.push_back(sec);
+
+  // Add input sections to output segments.
+  for (InputSection *isec : inputSections) {
+    OutputSegment *os =
+        getOrCreateOutputSegment(isec->segname, VM_PROT_READ | VM_PROT_WRITE);
+    os->sections[isec->name].push_back(isec);
+  }
+
+  // Write to an output file.
+  writeResult();
+
+  if (canExitEarly)
+    exitLld(errorCount() ? 1 : 0);
+
+  freeArena();
+  return !errorCount();
+}
diff --git a/lld/MachO/Driver.h b/lld/MachO/Driver.h
new file mode 100644
index 0000000000000..142f00d2e80c4
--- /dev/null
+++ b/lld/MachO/Driver.h
@@ -0,0 +1,35 @@
+//===- Driver.h -------------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLD_MACHO_DRIVER_H
+#define LLD_MACHO_DRIVER_H
+
+#include "lld/Common/LLVM.h"
+#include "llvm/Option/OptTable.h"
+
+namespace lld {
+namespace macho {
+
+class MachOOptTable : public llvm::opt::OptTable {
+public:
+  MachOOptTable();
+  llvm::opt::InputArgList parse(ArrayRef<const char *> argv);
+};
+
+// Create enum with OPT_xxx values for each option in Options.td
+enum {
+  OPT_INVALID = 0,
+#define OPTION(_1, _2, ID, _4, _5, _6, _7, _8, _9, _10, _11, _12) OPT_##ID,
+#include "Options.inc"
+#undef OPTION
+};
+
+} // namespace macho
+} // namespace lld
+
+#endif
diff --git a/lld/MachO/InputFiles.cpp b/lld/MachO/InputFiles.cpp
new file mode 100644
index 0000000000000..e2cfd1f0cc5df
--- /dev/null
+++ b/lld/MachO/InputFiles.cpp
@@ -0,0 +1,204 @@
+//===- InputFiles.cpp -----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains functions to parse Mach-O object files. In this comment,
+// we describe the Mach-O file structure and how we parse it.
+//
+// Mach-O is not very different from ELF or COFF. The notion of symbols,
+// sections and relocations exists in Mach-O as it does in ELF and COFF.
+//
+// Perhaps the notion that is new to those who know ELF/COFF is "subsections".
+// In ELF/COFF, sections are an atomic unit of data copied from input files to
+// output files. When we merge or garbage-collect sections, we treat each
+// section as an atomic unit. In Mach-O, that's not the case. Sections can
+// consist of multiple subsections, and subsections are a unit of merging and
+// garbage-collecting. Therefore, Mach-O's subsections are more similar to
+// ELF/COFF's sections than Mach-O's sections are.
+//
+// A section can have multiple symbols. A symbol that does not have the
+// N_ALT_ENTRY attribute indicates a beginning of a subsection. Therefore, by
+// definition, a symbol is always present at the beginning of each subsection. A
+// symbol with N_ALT_ENTRY attribute does not start a new subsection and can
+// point to a middle of a subsection.
+//
+// The notion of subsections also affects how relocations are represented in
+// Mach-O. All references within a section need to be explicitly represented as
+// relocations if they refer to different subsections, because we obviously need
+// to fix up addresses if subsections are laid out in an output file differently
+// than they were in object files. To represent that, Mach-O relocations can
+// refer to an unnamed location via its address. Scattered relocations (those
+// with the R_SCATTERED bit set) always refer to unnamed locations.
+// Non-scattered relocations refer to an unnamed location if r_extern is not set
+// and r_symbolnum is zero.
+//
+// Without the above differences, I think you can use your knowledge about ELF
+// and COFF for Mach-O.
+//
+//===----------------------------------------------------------------------===//
+
+#include "InputFiles.h"
+#include "InputSection.h"
+#include "OutputSegment.h"
+#include "SymbolTable.h"
+#include "Symbols.h"
+#include "Target.h"
+
+#include "lld/Common/ErrorHandler.h"
+#include "lld/Common/Memory.h"
+#include "llvm/BinaryFormat/MachO.h"
+#include "llvm/Support/Endian.h"
+#include "llvm/Support/MemoryBuffer.h"
+
+using namespace llvm;
+using namespace llvm::MachO;
+using namespace llvm::support::endian;
+using namespace lld;
+using namespace lld::macho;
+
+std::vector<InputFile *> macho::inputFiles;
+
+// Open a given file path and return it as a memory-mapped file.
+Optional<MemoryBufferRef> macho::readFile(StringRef path) {
+  // Open a file.
+  auto mbOrErr = MemoryBuffer::getFile(path);
+  if (auto ec = mbOrErr.getError()) {
+    error("cannot open " + path + ": " + ec.message());
+    return None;
+  }
+
+  std::unique_ptr<MemoryBuffer> &mb = *mbOrErr;
+  MemoryBufferRef mbref = mb->getMemBufferRef();
+  make<std::unique_ptr<MemoryBuffer>>(std::move(mb)); // take mb ownership
+  return mbref;
+}
+
+static const load_command *findCommand(const mach_header_64 *hdr,
+                                       uint32_t type) {
+  const uint8_t *p =
+      reinterpret_cast<const uint8_t *>(hdr) + sizeof(mach_header_64);
+
+  for (uint32_t i = 0, n = hdr->ncmds; i < n; ++i) {
+    auto *cmd = reinterpret_cast<const load_command *>(p);
+    if (cmd->cmd == type)
+      return cmd;
+    p += cmd->cmdsize;
+  }
+  return nullptr;
+}
+
+std::vector<InputSection *>
+InputFile::parseSections(ArrayRef<section_64> sections) {
+  std::vector<InputSection *> ret;
+  ret.reserve(sections.size());
+
+  auto *buf = reinterpret_cast<const uint8_t *>(mb.getBufferStart());
+
+  for (const section_64 &sec : sections) {
+    InputSection *isec = make<InputSection>();
+    isec->file = this;
+    isec->name = StringRef(sec.sectname, strnlen(sec.sectname, 16));
+    isec->segname = StringRef(sec.segname, strnlen(sec.segname, 16));
+    isec->data = {buf + sec.offset, static_cast<size_t>(sec.size)};
+    if (sec.align >= 32)
+      error("alignment " + std::to_string(sec.align) + " of section " +
+            isec->name + " is too large");
+    else
+      isec->align = 1 << sec.align;
+    isec->flags = sec.flags;
+    ret.push_back(isec);
+  }
+
+  return ret;
+}
+
+void InputFile::parseRelocations(const section_64 &sec,
+                                 std::vector<Reloc> &relocs) {
+  auto *buf = reinterpret_cast<const uint8_t *>(mb.getBufferStart());
+  ArrayRef<any_relocation_info> relInfos(
+      reinterpret_cast<const any_relocation_info *>(buf + sec.reloff),
+      sec.nreloc);
+
+  for (const any_relocation_info &anyRel : relInfos) {
+    Reloc r;
+    if (anyRel.r_word0 & R_SCATTERED) {
+      error("TODO: Scattered relocations not supported");
+    } else {
+      auto rel = reinterpret_cast<const relocation_info &>(anyRel);
+      r.type = rel.r_type;
+      r.offset = rel.r_address;
+      r.addend = target->getImplicitAddend(buf + sec.offset + r.offset, r.type);
+      if (rel.r_extern)
+        r.target = symbols[rel.r_symbolnum];
+      else {
+        error("TODO: Non-extern relocations are not supported");
+        continue;
+      }
+    }
+    relocs.push_back(r);
+  }
+}
+
+ObjFile::ObjFile(MemoryBufferRef mb) : InputFile(ObjKind, mb) {
+  auto *buf = reinterpret_cast<const uint8_t *>(mb.getBufferStart());
+  auto *hdr = reinterpret_cast<const mach_header_64 *>(mb.getBufferStart());
+  ArrayRef<section_64> objSections;
+
+  if (const load_command *cmd = findCommand(hdr, LC_SEGMENT_64)) {
+    auto *c = reinterpret_cast<const segment_command_64 *>(cmd);
+    objSections = ArrayRef<section_64>{
+        reinterpret_cast<const section_64 *>(c + 1), c->nsects};
+    sections = parseSections(objSections);
+  }
+
+  if (const load_command *cmd = findCommand(hdr, LC_SYMTAB)) {
+    auto *c = reinterpret_cast<const symtab_command *>(cmd);
+    const char *strtab = reinterpret_cast<const char *>(buf) + c->stroff;
+    ArrayRef<const nlist_64> nList(
+        reinterpret_cast<const nlist_64 *>(buf + c->symoff), c->nsyms);
+
+    symbols.reserve(c->nsyms);
+
+    for (const nlist_64 &sym : nList) {
+      StringRef name = strtab + sym.n_strx;
+
+      // Undefined symbol
+      if (!sym.n_sect) {
+        error("TODO: Support undefined symbols");
+        continue;
+      }
+
+      InputSection *isec = sections[sym.n_sect - 1];
+      const section_64 &objSec = objSections[sym.n_sect - 1];
+      uint64_t value = sym.n_value - objSec.addr;
+
+      // Global defined symbol
+      if (sym.n_type & N_EXT) {
+        symbols.push_back(symtab->addDefined(name, isec, value));
+        continue;
+      }
+
+      // Local defined symbol
+      symbols.push_back(make<Defined>(name, isec, value));
+    }
+  }
+
+  // The relocations may refer to the symbols, so we parse them after we have
+  // the symbols loaded.
+  if (!sections.empty()) {
+    auto it = sections.begin();
+    for (const section_64 &sec : objSections) {
+      parseRelocations(sec, (*it)->relocs);
+      ++it;
+    }
+  }
+}
+
+// Returns "<internal>" or "baz.o".
+std::string lld::toString(const InputFile *file) {
+  return file ? std::string(file->getName()) : "<internal>";
+}
diff --git a/lld/MachO/InputFiles.h b/lld/MachO/InputFiles.h
new file mode 100644
index 0000000000000..7b13b1f9f6bd5
--- /dev/null
+++ b/lld/MachO/InputFiles.h
@@ -0,0 +1,70 @@
+//===- InputFiles.h ---------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLD_MACHO_INPUT_FILES_H
+#define LLD_MACHO_INPUT_FILES_H
+
+#include "lld/Common/LLVM.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/BinaryFormat/MachO.h"
+#include "llvm/Object/Archive.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include <vector>
+
+namespace lld {
+namespace macho {
+
+class InputSection;
+class Symbol;
+struct Reloc;
+
+class InputFile {
+public:
+  enum Kind {
+    ObjKind,
+  };
+
+  virtual ~InputFile() = default;
+
+  Kind kind() const { return fileKind; }
+  StringRef getName() const { return mb.getBufferIdentifier(); }
+
+  MemoryBufferRef mb;
+  std::vector<Symbol *> symbols;
+  std::vector<InputSection *> sections;
+  StringRef dylibName;
+
+protected:
+  InputFile(Kind kind, MemoryBufferRef mb) : mb(mb), fileKind(kind) {}
+
+  std::vector<InputSection *> parseSections(ArrayRef<llvm::MachO::section_64>);
+
+  void parseRelocations(const llvm::MachO::section_64 &,
+                        std::vector<Reloc> &relocs);
+
+private:
+  const Kind fileKind;
+};
+
+// .o file
+class ObjFile : public InputFile {
+public:
+  explicit ObjFile(MemoryBufferRef mb);
+  static bool classof(const InputFile *f) { return f->kind() == ObjKind; }
+};
+
+extern std::vector<InputFile *> inputFiles;
+
+llvm::Optional<MemoryBufferRef> readFile(StringRef path);
+
+} // namespace macho
+
+std::string toString(const macho::InputFile *file);
+} // namespace lld
+
+#endif
diff --git a/lld/MachO/InputSection.cpp b/lld/MachO/InputSection.cpp
new file mode 100644
index 0000000000000..49dccf641c4c5
--- /dev/null
+++ b/lld/MachO/InputSection.cpp
@@ -0,0 +1,39 @@
+//===- InputSection.cpp ---------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "InputSection.h"
+#include "Symbols.h"
+#include "Target.h"
+#include "lld/Common/Memory.h"
+#include "llvm/Support/Endian.h"
+
+using namespace llvm::MachO;
+using namespace llvm::support;
+using namespace lld;
+using namespace lld::macho;
+
+std::vector<InputSection *> macho::inputSections;
+
+void InputSection::writeTo(uint8_t *buf) {
+  memcpy(buf, data.data(), data.size());
+
+  for (Reloc &r : relocs) {
+    uint64_t va = 0;
+    if (auto *s = r.target.dyn_cast<Symbol *>())
+      va = s->getVA();
+    else if (auto *isec = r.target.dyn_cast<InputSection *>())
+      va = isec->addr;
+    else
+      llvm_unreachable("Unknown relocation target");
+
+    uint64_t val = va + r.addend;
+    if (1) // TODO: handle non-pcrel relocations
+      val -= addr - ImageBase + r.offset;
+    target->relocateOne(buf + r.offset, r.type, val);
+  }
+}
diff --git a/lld/MachO/InputSection.h b/lld/MachO/InputSection.h
new file mode 100644
index 0000000000000..05da49726bacb
--- /dev/null
+++ b/lld/MachO/InputSection.h
@@ -0,0 +1,52 @@
+//===- InputSection.h -------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLD_MACHO_INPUT_SECTION_H
+#define LLD_MACHO_INPUT_SECTION_H
+
+#include "lld/Common/LLVM.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/PointerUnion.h"
+#include "llvm/BinaryFormat/MachO.h"
+
+namespace lld {
+namespace macho {
+
+class InputFile;
+class InputSection;
+class Symbol;
+
+struct Reloc {
+  uint8_t type;
+  uint32_t addend;
+  uint32_t offset;
+  llvm::PointerUnion<Symbol *, InputSection *> target;
+};
+
+class InputSection {
+public:
+  void writeTo(uint8_t *buf);
+
+  InputFile *file = nullptr;
+  StringRef name;
+  StringRef segname;
+
+  ArrayRef<uint8_t> data;
+  uint64_t addr = 0;
+  uint32_t align = 1;
+  uint32_t flags = 0;
+
+  std::vector<Reloc> relocs;
+};
+
+extern std::vector<InputSection *> inputSections;
+
+} // namespace macho
+} // namespace lld
+
+#endif
diff --git a/lld/MachO/Options.td b/lld/MachO/Options.td
new file mode 100644
index 0000000000000..2d9713c342e66
--- /dev/null
+++ b/lld/MachO/Options.td
@@ -0,0 +1,18 @@
+include "llvm/Option/OptParser.td"
+
+def arch: Separate<["-"], "arch">, MetaVarName<"<arch-name>">,
+  HelpText<"Architecture to link">;
+
+def e: Separate<["-"], "e">, HelpText<"Name of entry point symbol">;
+
+def o: Separate<["-"], "o">, MetaVarName<"<path>">,
+  HelpText<"Path to file to write output">;
+
+def v: Flag<["-"], "v">, HelpText<"Display the version number and exit">;
+
+// Ignored options
+def: Flag<["-"], "demangle">;
+def: Flag<["-"], "dynamic">;
+def: Flag<["-"], "no_deduplicate">;
+def: Separate<["-"], "lto_library">;
+def: Separate<["-"], "macosx_version_min">;
diff --git a/lld/MachO/OutputSegment.cpp b/lld/MachO/OutputSegment.cpp
new file mode 100644
index 0000000000000..75f5c20146b60
--- /dev/null
+++ b/lld/MachO/OutputSegment.cpp
@@ -0,0 +1,30 @@
+//===- OutputSegment.cpp --------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "OutputSegment.h"
+#include "lld/Common/Memory.h"
+
+using namespace llvm;
+using namespace lld;
+using namespace lld::macho;
+
+std::vector<OutputSegment *> macho::outputSegments;
+
+OutputSegment *macho::getOrCreateOutputSegment(StringRef name, uint32_t perms) {
+  for (OutputSegment *os : outputSegments)
+    if (os->name == name)
+      // TODO: assert that os->perms == perms, once we figure out what to do
+      // about default-created segments.
+      return os;
+
+  auto *os = make<OutputSegment>();
+  os->name = name;
+  os->perms = perms;
+  outputSegments.push_back(os);
+  return os;
+}
diff --git a/lld/MachO/OutputSegment.h b/lld/MachO/OutputSegment.h
new file mode 100644
index 0000000000000..108c1804d0313
--- /dev/null
+++ b/lld/MachO/OutputSegment.h
@@ -0,0 +1,34 @@
+//===- OutputSegment.h ------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLD_MACHO_OUTPUT_SEGMENT_H
+#define LLD_MACHO_OUTPUT_SEGMENT_H
+
+#include "lld/Common/LLVM.h"
+#include "llvm/ADT/MapVector.h"
+
+namespace lld {
+namespace macho {
+
+class InputSection;
+
+class OutputSegment {
+public:
+  StringRef name;
+  uint32_t perms;
+  llvm::MapVector<StringRef, std::vector<InputSection *>> sections;
+};
+
+extern std::vector<OutputSegment *> outputSegments;
+
+OutputSegment *getOrCreateOutputSegment(StringRef name, uint32_t perms);
+
+} // namespace macho
+} // namespace lld
+
+#endif
diff --git a/lld/MachO/SymbolTable.cpp b/lld/MachO/SymbolTable.cpp
new file mode 100644
index 0000000000000..ce54416a96e70
--- /dev/null
+++ b/lld/MachO/SymbolTable.cpp
@@ -0,0 +1,62 @@
+//===- SymbolTable.cpp ----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SymbolTable.h"
+#include "InputFiles.h"
+#include "Symbols.h"
+#include "lld/Common/ErrorHandler.h"
+#include "lld/Common/Memory.h"
+
+using namespace llvm;
+using namespace lld;
+using namespace lld::macho;
+
+Symbol *SymbolTable::find(StringRef name) {
+  auto it = symMap.find(llvm::CachedHashStringRef(name));
+  if (it == symMap.end())
+    return nullptr;
+  return symVector[it->second];
+}
+
+std::pair<Symbol *, bool> SymbolTable::insert(StringRef name) {
+  auto p = symMap.insert({CachedHashStringRef(name), (int)symVector.size()});
+
+  // Name already present in the symbol table.
+  if (!p.second)
+    return {symVector[p.first->second], false};
+
+  // Name is a new symbol.
+  Symbol *sym = reinterpret_cast<Symbol *>(make<SymbolUnion>());
+  symVector.push_back(sym);
+  return {sym, true};
+}
+
+Symbol *SymbolTable::addDefined(StringRef name, InputSection *isec,
+                                uint32_t value) {
+  Symbol *s;
+  bool wasInserted;
+  std::tie(s, wasInserted) = insert(name);
+
+  if (!wasInserted && isa<Defined>(s))
+    error("duplicate symbol: " + name);
+
+  replaceSymbol<Defined>(s, name, isec, value);
+  return s;
+}
+
+Symbol *SymbolTable::addUndefined(StringRef name) {
+  Symbol *s;
+  bool wasInserted;
+  std::tie(s, wasInserted) = insert(name);
+
+  if (wasInserted)
+    replaceSymbol<Undefined>(s, name);
+  return s;
+}
+
+SymbolTable *macho::symtab;
diff --git a/lld/MachO/SymbolTable.h b/lld/MachO/SymbolTable.h
new file mode 100644
index 0000000000000..2027d367ae4b4
--- /dev/null
+++ b/lld/MachO/SymbolTable.h
@@ -0,0 +1,44 @@
+//===- SymbolTable.h --------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLD_MACHO_SYMBOL_TABLE_H
+#define LLD_MACHO_SYMBOL_TABLE_H
+
+#include "lld/Common/LLVM.h"
+#include "llvm/ADT/CachedHashString.h"
+#include "llvm/Object/Archive.h"
+
+namespace lld {
+namespace macho {
+
+class InputFile;
+class InputSection;
+class ArchiveFile;
+class Symbol;
+
+class SymbolTable {
+public:
+  Symbol *addDefined(StringRef name, InputSection *isec, uint32_t value);
+
+  Symbol *addUndefined(StringRef name);
+
+  ArrayRef<Symbol *> getSymbols() const { return symVector; }
+  Symbol *find(StringRef name);
+
+private:
+  std::pair<Symbol *, bool> insert(StringRef name);
+  llvm::DenseMap<llvm::CachedHashStringRef, int> symMap;
+  std::vector<Symbol *> symVector;
+};
+
+extern SymbolTable *symtab;
+
+} // namespace macho
+} // namespace lld
+
+#endif
diff --git a/lld/MachO/Symbols.cpp b/lld/MachO/Symbols.cpp
new file mode 100644
index 0000000000000..ec383ec9781c3
--- /dev/null
+++ b/lld/MachO/Symbols.cpp
@@ -0,0 +1,23 @@
+//===- Symbols.cpp --------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Symbols.h"
+#include "InputFiles.h"
+#include "lld/Common/ErrorHandler.h"
+#include "lld/Common/Strings.h"
+
+using namespace llvm;
+using namespace lld;
+using namespace lld::macho;
+
+// Returns a symbol for an error message.
+std::string lld::toString(const Symbol &sym) {
+  if (Optional<std::string> s = demangleItanium(sym.getName()))
+    return *s;
+  return std::string(sym.getName());
+}
diff --git a/lld/MachO/Symbols.h b/lld/MachO/Symbols.h
new file mode 100644
index 0000000000000..f754ec8475077
--- /dev/null
+++ b/lld/MachO/Symbols.h
@@ -0,0 +1,100 @@
+//===- Symbols.h ------------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLD_MACHO_SYMBOLS_H
+#define LLD_MACHO_SYMBOLS_H
+
+#include "InputSection.h"
+#include "Target.h"
+#include "lld/Common/Strings.h"
+#include "llvm/Object/Archive.h"
+
+namespace lld {
+namespace macho {
+
+class InputSection;
+class InputFile;
+class ArchiveFile;
+
+struct StringRefZ {
+  StringRefZ(const char *s) : data(s), size(-1) {}
+  StringRefZ(StringRef s) : data(s.data()), size(s.size()) {}
+
+  const char *data;
+  const uint32_t size;
+};
+
+class Symbol {
+public:
+  enum Kind {
+    DefinedKind,
+    UndefinedKind,
+  };
+
+  Kind kind() const { return static_cast<Kind>(symbolKind); }
+
+  StringRef getName() const { return {name.data, name.size}; }
+
+  uint64_t getVA() const;
+
+  InputFile *file;
+
+protected:
+  Symbol(Kind k, InputFile *file, StringRefZ name)
+      : file(file), symbolKind(k), name(name) {}
+
+  Kind symbolKind;
+  StringRefZ name;
+};
+
+class Defined : public Symbol {
+public:
+  Defined(StringRefZ name, InputSection *isec, uint32_t value)
+      : Symbol(DefinedKind, nullptr, name), isec(isec), value(value) {}
+
+  InputSection *isec;
+  uint32_t value;
+
+  static bool classof(const Symbol *s) { return s->kind() == DefinedKind; }
+};
+
+class Undefined : public Symbol {
+public:
+  Undefined(StringRefZ name) : Symbol(UndefinedKind, nullptr, name) {}
+
+  static bool classof(const Symbol *s) { return s->kind() == UndefinedKind; }
+};
+
+inline uint64_t Symbol::getVA() const {
+  if (auto *d = dyn_cast<Defined>(this))
+    return d->isec->addr + d->value - ImageBase;
+  return 0;
+}
+
+union SymbolUnion {
+  alignas(Defined) char a[sizeof(Defined)];
+  alignas(Undefined) char b[sizeof(Undefined)];
+};
+
+template <typename T, typename... ArgT>
+void replaceSymbol(Symbol *s, ArgT &&... arg) {
+  static_assert(sizeof(T) <= sizeof(SymbolUnion), "SymbolUnion too small");
+  static_assert(alignof(T) <= alignof(SymbolUnion),
+                "SymbolUnion not aligned enough");
+  assert(static_cast<Symbol *>(static_cast<T *>(nullptr)) == nullptr &&
+         "Not a Symbol");
+
+  new (s) T(std::forward<ArgT>(arg)...);
+}
+
+} // namespace macho
+
+std::string toString(const macho::Symbol &);
+} // namespace lld
+
+#endif
diff --git a/lld/MachO/Target.cpp b/lld/MachO/Target.cpp
new file mode 100644
index 0000000000000..0f70776a507ff
--- /dev/null
+++ b/lld/MachO/Target.cpp
@@ -0,0 +1,14 @@
+//===- Target.cpp ---------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Target.h"
+
+using namespace lld;
+using namespace lld::macho;
+
+TargetInfo *macho::target = nullptr;
diff --git a/lld/MachO/Target.h b/lld/MachO/Target.h
new file mode 100644
index 0000000000000..9ebc56c9ffbd5
--- /dev/null
+++ b/lld/MachO/Target.h
@@ -0,0 +1,41 @@
+//===- Target.h -------------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLD_MACHO_TARGET_H
+#define LLD_MACHO_TARGET_H
+
+#include <cstdint>
+
+namespace lld {
+namespace macho {
+
+enum {
+  PageSize = 4096,
+  ImageBase = 4096,
+  MaxAlignmentPowerOf2 = 32,
+};
+
+class TargetInfo {
+public:
+  virtual ~TargetInfo() = default;
+  virtual uint64_t getImplicitAddend(const uint8_t *loc,
+                                     uint8_t type) const = 0;
+  virtual void relocateOne(uint8_t *loc, uint8_t type, uint64_t val) const = 0;
+
+  uint32_t cpuType;
+  uint32_t cpuSubtype;
+};
+
+TargetInfo *createX86_64TargetInfo();
+
+extern TargetInfo *target;
+
+} // namespace macho
+} // namespace lld
+
+#endif
diff --git a/lld/MachO/Writer.cpp b/lld/MachO/Writer.cpp
new file mode 100644
index 0000000000000..ce0c30d5300b9
--- /dev/null
+++ b/lld/MachO/Writer.cpp
@@ -0,0 +1,359 @@
+//===- Writer.cpp ---------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Writer.h"
+#include "Config.h"
+#include "InputFiles.h"
+#include "InputSection.h"
+#include "OutputSegment.h"
+#include "SymbolTable.h"
+#include "Symbols.h"
+#include "Target.h"
+
+#include "lld/Common/ErrorHandler.h"
+#include "lld/Common/Memory.h"
+#include "llvm/BinaryFormat/MachO.h"
+#include "llvm/Support/LEB128.h"
+#include "llvm/Support/MathExtras.h"
+
+using namespace llvm;
+using namespace llvm::MachO;
+using namespace llvm::support;
+using namespace lld;
+using namespace lld::macho;
+
+namespace {
+class LCLinkEdit;
+class LCDyldInfo;
+class LCSymtab;
+
+class LoadCommand {
+public:
+  virtual ~LoadCommand() = default;
+  virtual uint32_t getSize() const = 0;
+  virtual void writeTo(uint8_t *buf) const = 0;
+};
+
+class Writer {
+public:
+  Writer() : buffer(errorHandler().outputBuffer) {}
+
+  void createLoadCommands();
+  void assignAddresses();
+
+  void openFile();
+  void writeHeader();
+  void writeSections();
+
+  void run();
+
+  std::vector<LoadCommand *> loadCommands;
+  std::unique_ptr<FileOutputBuffer> &buffer;
+  uint64_t fileSize = 0;
+  uint64_t sizeofCmds = 0;
+  LCLinkEdit *linkEditSeg = nullptr;
+  LCDyldInfo *dyldInfoSeg = nullptr;
+  LCSymtab *symtabSeg = nullptr;
+};
+
+class LCPagezero : public LoadCommand {
+public:
+  uint32_t getSize() const override { return sizeof(segment_command_64); }
+
+  void writeTo(uint8_t *buf) const override {
+    auto *c = reinterpret_cast<segment_command_64 *>(buf);
+    c->cmd = LC_SEGMENT_64;
+    c->cmdsize = getSize();
+    strcpy(c->segname, "__PAGEZERO");
+    c->vmsize = PageSize;
+  }
+};
+
+class LCLinkEdit : public LoadCommand {
+public:
+  uint32_t getSize() const override { return sizeof(segment_command_64); }
+
+  void writeTo(uint8_t *buf) const override {
+    auto *c = reinterpret_cast<segment_command_64 *>(buf);
+    c->cmd = LC_SEGMENT_64;
+    c->cmdsize = getSize();
+    strcpy(c->segname, "__LINKEDIT");
+    c->fileoff = fileOff;
+    c->filesize = contents.size();
+    c->maxprot = VM_PROT_READ | VM_PROT_WRITE;
+    c->initprot = VM_PROT_READ;
+  }
+
+  uint64_t getOffset() const { return fileOff + contents.size(); }
+
+  uint64_t fileOff = 0;
+  SmallVector<char, 128> contents;
+};
+
+class LCDyldInfo : public LoadCommand {
+public:
+  uint32_t getSize() const override { return sizeof(dyld_info_command); }
+
+  void writeTo(uint8_t *buf) const override {
+    auto *c = reinterpret_cast<dyld_info_command *>(buf);
+    c->cmd = LC_DYLD_INFO_ONLY;
+    c->cmdsize = getSize();
+    c->export_off = exportOff;
+    c->export_size = exportSize;
+  }
+
+  uint64_t exportOff = 0;
+  uint64_t exportSize = 0;
+};
+
+class LCDysymtab : public LoadCommand {
+public:
+  uint32_t getSize() const override { return sizeof(dysymtab_command); }
+
+  void writeTo(uint8_t *buf) const override {
+    auto *c = reinterpret_cast<dysymtab_command *>(buf);
+    c->cmd = LC_DYSYMTAB;
+    c->cmdsize = getSize();
+  }
+};
+
+class LCSegment : public LoadCommand {
+public:
+  LCSegment(StringRef name, OutputSegment *seg) : name(name), seg(seg) {}
+
+  uint32_t getSize() const override {
+    return sizeof(segment_command_64) +
+           seg->sections.size() * sizeof(section_64);
+  }
+
+  void writeTo(uint8_t *buf) const override {
+    auto *c = reinterpret_cast<segment_command_64 *>(buf);
+    buf += sizeof(segment_command_64);
+
+    c->cmd = LC_SEGMENT_64;
+    c->cmdsize = getSize();
+    memcpy(c->segname, name.data(), name.size());
+
+    InputSection *firstSec = seg->sections.front().second[0];
+    InputSection *lastSec = seg->sections.back().second.back();
+
+    // dyld3's MachOLoaded::getSlide() assumes that the __TEXT segment starts
+    // from the beginning of the file (i.e. the header).
+    // TODO: replace this logic by creating a synthetic __TEXT,__mach_header
+    // section instead.
+    c->fileoff = name == "__TEXT" ? 0 : firstSec->addr - ImageBase;
+    c->vmaddr = c->fileoff + ImageBase;
+    c->vmsize = c->filesize = lastSec->addr + lastSec->data.size() - c->vmaddr;
+    c->maxprot = VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE;
+    c->initprot = seg->perms;
+    c->nsects = seg->sections.size();
+
+    for (auto &p : seg->sections) {
+      StringRef s = p.first;
+      std::vector<InputSection *> &sections = p.second;
+
+      auto *sectHdr = reinterpret_cast<section_64 *>(buf);
+      buf += sizeof(section_64);
+
+      memcpy(sectHdr->sectname, s.data(), s.size());
+      memcpy(sectHdr->segname, name.data(), name.size());
+
+      sectHdr->addr = sections[0]->addr;
+      sectHdr->offset = sections[0]->addr - ImageBase;
+      sectHdr->align = sections[0]->align;
+      uint32_t maxAlign = 0;
+      for (const InputSection *section : sections)
+        maxAlign = std::max(maxAlign, section->align);
+      sectHdr->align = Log2_32(maxAlign);
+      sectHdr->flags = sections[0]->flags;
+      sectHdr->size = sections.back()->addr + sections.back()->data.size() -
+                      sections[0]->addr;
+    }
+  }
+
+private:
+  StringRef name;
+  OutputSegment *seg;
+};
+
+class LCMain : public LoadCommand {
+  uint32_t getSize() const override { return sizeof(entry_point_command); }
+
+  void writeTo(uint8_t *buf) const override {
+    auto *c = reinterpret_cast<entry_point_command *>(buf);
+    c->cmd = LC_MAIN;
+    c->cmdsize = getSize();
+    c->entryoff = config->entry->getVA();
+    c->stacksize = 0;
+  }
+};
+
+class LCSymtab : public LoadCommand {
+public:
+  uint32_t getSize() const override { return sizeof(symtab_command); }
+
+  void writeTo(uint8_t *buf) const override {
+    auto *c = reinterpret_cast<symtab_command *>(buf);
+    c->cmd = LC_SYMTAB;
+    c->cmdsize = getSize();
+  }
+};
+
+class LCLoadDylib : public LoadCommand {
+public:
+  LCLoadDylib(StringRef path) : path(path) {}
+
+  uint32_t getSize() const override {
+    return alignTo(sizeof(dylib_command) + path.size() + 1, 8);
+  }
+
+  void writeTo(uint8_t *buf) const override {
+    auto *c = reinterpret_cast<dylib_command *>(buf);
+    buf += sizeof(dylib_command);
+
+    c->cmd = LC_LOAD_DYLIB;
+    c->cmdsize = getSize();
+    c->dylib.name = sizeof(dylib_command);
+
+    memcpy(buf, path.data(), path.size());
+    buf[path.size()] = '\0';
+  }
+
+private:
+  StringRef path;
+};
+
+class LCLoadDylinker : public LoadCommand {
+public:
+  uint32_t getSize() const override {
+    return alignTo(sizeof(dylinker_command) + path.size() + 1, 8);
+  }
+
+  void writeTo(uint8_t *buf) const override {
+    auto *c = reinterpret_cast<dylinker_command *>(buf);
+    buf += sizeof(dylinker_command);
+
+    c->cmd = LC_LOAD_DYLINKER;
+    c->cmdsize = getSize();
+    c->name = sizeof(dylinker_command);
+
+    memcpy(buf, path.data(), path.size());
+    buf[path.size()] = '\0';
+  }
+
+private:
+  // Recent versions of Darwin won't run any binary that has dyld at a
+  // different location.
+  const StringRef path = "/usr/lib/dyld";
+};
+} // namespace
+
+void Writer::createLoadCommands() {
+  linkEditSeg = make<LCLinkEdit>();
+  dyldInfoSeg = make<LCDyldInfo>();
+  symtabSeg = make<LCSymtab>();
+
+  loadCommands.push_back(linkEditSeg);
+  loadCommands.push_back(dyldInfoSeg);
+  loadCommands.push_back(symtabSeg);
+  loadCommands.push_back(make<LCPagezero>());
+  loadCommands.push_back(make<LCLoadDylinker>());
+  loadCommands.push_back(make<LCDysymtab>());
+  loadCommands.push_back(make<LCMain>());
+  // TODO: dyld requires libSystem to be loaded. libSystem is a universal
+  // binary and we don't have support for that yet, so mock it out here.
+  loadCommands.push_back(make<LCLoadDylib>("/usr/lib/libSystem.B.dylib"));
+
+  for (OutputSegment *seg : outputSegments)
+    if (!seg->sections.empty())
+      loadCommands.push_back(make<LCSegment>(seg->name, seg));
+}
+
+void Writer::assignAddresses() {
+  uint64_t addr = ImageBase + sizeof(mach_header_64);
+
+  uint64_t size = 0;
+  for (LoadCommand *lc : loadCommands)
+    size += lc->getSize();
+  sizeofCmds = size;
+  addr += size;
+
+  for (OutputSegment *seg : outputSegments) {
+    addr = alignTo(addr, PageSize);
+
+    for (auto &p : seg->sections) {
+      ArrayRef<InputSection *> sections = p.second;
+      for (InputSection *isec : sections) {
+        addr = alignTo(addr, isec->align);
+        isec->addr = addr;
+        addr += isec->data.size();
+      }
+    }
+  }
+
+  linkEditSeg->fileOff = addr - ImageBase;
+}
+
+void Writer::openFile() {
+  Expected<std::unique_ptr<FileOutputBuffer>> bufferOrErr =
+      FileOutputBuffer::create(config->outputFile, fileSize,
+                               FileOutputBuffer::F_executable);
+
+  if (!bufferOrErr)
+    error("failed to open " + config->outputFile + ": " +
+          llvm::toString(bufferOrErr.takeError()));
+  else
+    buffer = std::move(*bufferOrErr);
+}
+
+void Writer::writeHeader() {
+  auto *hdr = reinterpret_cast<mach_header_64 *>(buffer->getBufferStart());
+  hdr->magic = MH_MAGIC_64;
+  hdr->cputype = CPU_TYPE_X86_64;
+  hdr->cpusubtype = CPU_SUBTYPE_X86_64_ALL | CPU_SUBTYPE_LIB64;
+  hdr->filetype = MH_EXECUTE;
+  hdr->ncmds = loadCommands.size();
+  hdr->sizeofcmds = sizeofCmds;
+  hdr->flags = MH_NOUNDEFS | MH_DYLDLINK | MH_TWOLEVEL;
+
+  uint8_t *p = reinterpret_cast<uint8_t *>(hdr + 1);
+  for (LoadCommand *lc : loadCommands) {
+    lc->writeTo(p);
+    p += lc->getSize();
+  }
+}
+
+void Writer::writeSections() {
+  uint8_t *buf = buffer->getBufferStart();
+
+  for (OutputSegment *seg : outputSegments)
+    for (auto &sect : seg->sections)
+      for (InputSection *isec : sect.second)
+        isec->writeTo(buf + isec->addr - ImageBase);
+
+  memcpy(buf + linkEditSeg->fileOff, linkEditSeg->contents.data(),
+         linkEditSeg->contents.size());
+}
+
+void Writer::run() {
+  createLoadCommands();
+  assignAddresses();
+  fileSize = linkEditSeg->fileOff + linkEditSeg->contents.size();
+
+  openFile();
+  if (errorCount())
+    return;
+
+  writeHeader();
+  writeSections();
+
+  if (auto e = buffer->commit())
+    error("failed to write to the output file: " + toString(std::move(e)));
+}
+
+void macho::writeResult() { Writer().run(); }
diff --git a/lld/MachO/Writer.h b/lld/MachO/Writer.h
new file mode 100644
index 0000000000000..a0b90631f9ef7
--- /dev/null
+++ b/lld/MachO/Writer.h
@@ -0,0 +1,20 @@
+//===- Writer.h -------------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLD_MACHO_WRITER_H
+#define LLD_MACHO_WRITER_H
+
+namespace lld {
+namespace macho {
+
+void writeResult();
+
+} // namespace macho
+} // namespace lld
+
+#endif
diff --git a/lld/docs/ELF/linker_script.rst b/lld/docs/ELF/linker_script.rst
index 5b904bb3a1e14..c5115c1c9d6f3 100644
--- a/lld/docs/ELF/linker_script.rst
+++ b/lld/docs/ELF/linker_script.rst
@@ -51,3 +51,27 @@ sh_addralign of an *OutputSection* *S* is the maximum of
 
 When an *OutputSection* *S* has both ``address`` and ``ALIGN(section_align)``,
 GNU ld will set sh_addralign to ``ALIGN(section_align)``.
+
+Output section LMA
+------------------
+
+A load address (LMA) can be specified by ``AT(lma)`` or ``AT>lma_region``.
+
+- ``AT(lma)`` specifies the exact load address. If the linker script does not
+  have a PHDRS command, then a new loadable segment will be generated.
+- ``AT>lma_region`` specifies the LMA region. The lack of ``AT>lma_region``
+  means the default region is used. Note, GNU ld propagates the previous LMA
+  memory region when ``address`` is not specified. The LMA is set to the
+  current location of the memory region aligned to the section alignment.
+  If the linker script does not have a PHDRS command, then if
+  ``lma_region`` is different from the ``lma_region`` for
+  the previous OutputSection a new loadable segment will be generated.
+
+The two keywords cannot be specified at the same time.
+
+If neither ``AT(lma)`` nor ``AT>lma_region`` is specified:
+
+- If the previous section is also in the default LMA region, the difference
+  between the LMA and the VMA is computed to be the same as the previous
+  difference.
+- Otherwise, the LMA is set to the VMA.
diff --git a/lld/docs/ld.lld.1 b/lld/docs/ld.lld.1
index 238ac28a599ce..7af9a16e83f91 100644
--- a/lld/docs/ld.lld.1
+++ b/lld/docs/ld.lld.1
@@ -302,8 +302,6 @@ Page align sections.
 Do not set the text data sections to be writable, page align sections.
 .It Fl -no-rosegment
 Do not put read-only non-executable sections in their own segment.
-.It Fl -no-threads
-Do not run the linker multi-threaded.
 .It Fl -no-undefined-version
 Report version scripts that refer undefined symbols.
 .It Fl -no-undefined
@@ -525,9 +523,12 @@ Path to ThinLTO cached object file directory.
 Pruning policy for the ThinLTO cache.
 .It Fl -thinlto-jobs Ns = Ns Ar value
 Number of ThinLTO jobs.
-.It Fl -threads
-Run the linker multi-threaded.
-This option is enabled by default.
+.It Fl -threads Ns = Ns Ar N
+Number of threads.
+.Cm all
+(default) means all of concurrent threads supported.
+.Cm 1
+disables multi-threading.
 .It Fl -trace
 Print the names of the input files.
 .It Fl -trace-symbol Ns = Ns Ar symbol , Fl y Ar symbol
diff --git a/lld/include/lld/Common/Driver.h b/lld/include/lld/Common/Driver.h
index 0a358d8aff6bd..6db3d234eb564 100644
--- a/lld/include/lld/Common/Driver.h
+++ b/lld/include/lld/Common/Driver.h
@@ -33,6 +33,11 @@ bool link(llvm::ArrayRef<const char *> args, bool canExitEarly,
           llvm::raw_ostream &stdoutOS, llvm::raw_ostream &stderrOS);
 }
 
+namespace macho {
+bool link(llvm::ArrayRef<const char *> args, bool canExitEarly,
+          llvm::raw_ostream &stdoutOS, llvm::raw_ostream &stderrOS);
+}
+
 namespace wasm {
 bool link(llvm::ArrayRef<const char *> args, bool canExitEarly,
           llvm::raw_ostream &stdoutOS, llvm::raw_ostream &stderrOS);
diff --git a/lld/include/lld/Common/Threads.h b/lld/include/lld/Common/Threads.h
index 7834130fdf726..1090961b18cdd 100644
--- a/lld/include/lld/Common/Threads.h
+++ b/lld/include/lld/Common/Threads.h
@@ -63,10 +63,8 @@
 
 namespace lld {
 
-extern bool threadsEnabled;
-
 template <typename R, class FuncTy> void parallelForEach(R &&range, FuncTy fn) {
-  if (threadsEnabled)
+  if (llvm::parallel::strategy.ThreadsRequested != 1)
     for_each(llvm::parallel::par, std::begin(range), std::end(range), fn);
   else
     for_each(llvm::parallel::seq, std::begin(range), std::end(range), fn);
@@ -74,14 +72,14 @@ template <typename R, class FuncTy> void parallelForEach(R &&range, FuncTy fn) {
 
 inline void parallelForEachN(size_t begin, size_t end,
                              llvm::function_ref<void(size_t)> fn) {
-  if (threadsEnabled)
+  if (llvm::parallel::strategy.ThreadsRequested != 1)
     for_each_n(llvm::parallel::par, begin, end, fn);
   else
     for_each_n(llvm::parallel::seq, begin, end, fn);
 }
 
 template <typename R, class FuncTy> void parallelSort(R &&range, FuncTy fn) {
-  if (threadsEnabled)
+  if (llvm::parallel::strategy.ThreadsRequested != 1)
     sort(llvm::parallel::par, std::begin(range), std::end(range), fn);
   else
     sort(llvm::parallel::seq, std::begin(range), std::end(range), fn);
diff --git a/lld/include/lld/Core/Reference.h b/lld/include/lld/Core/Reference.h
index 4769882cde507..b104f84954747 100644
--- a/lld/include/lld/Core/Reference.h
+++ b/lld/include/lld/Core/Reference.h
@@ -91,7 +91,7 @@ class Reference {
   /// Some relocations require a symbol and a value (e.g. foo + 4).
   virtual Addend addend() const = 0;
 
-  /// During linking, some optimzations may change addend value.
+  /// During linking, some optimizations may change addend value.
   virtual void setAddend(Addend) = 0;
 
   /// Returns target specific attributes of the reference.
diff --git a/lld/lib/ReaderWriter/MachO/ArchHandler_arm64.cpp b/lld/lib/ReaderWriter/MachO/ArchHandler_arm64.cpp
index a424edf4985aa..bee081aec0676 100644
--- a/lld/lib/ReaderWriter/MachO/ArchHandler_arm64.cpp
+++ b/lld/lib/ReaderWriter/MachO/ArchHandler_arm64.cpp
@@ -61,7 +61,7 @@ class ArchHandler_arm64 : public ArchHandler {
 
   /// Used by GOTPass to update GOT References.
   void updateReferenceToGOT(const Reference *ref, bool targetNowGOT) override {
-    // If GOT slot was instanciated, transform:
+    // If GOT slot was instantiated, transform:
     //   gotPage21/gotOffset12 -> page21/offset12scale8
     // If GOT slot optimized away, transform:
     //   gotPage21/gotOffset12 -> page21/addOffset12
diff --git a/lld/lib/ReaderWriter/MachO/CompactUnwindPass.cpp b/lld/lib/ReaderWriter/MachO/CompactUnwindPass.cpp
index 94a105a6f159e..f3636feb217bb 100644
--- a/lld/lib/ReaderWriter/MachO/CompactUnwindPass.cpp
+++ b/lld/lib/ReaderWriter/MachO/CompactUnwindPass.cpp
@@ -576,5 +576,5 @@ void addCompactUnwindPass(PassManager &pm, const MachOLinkingContext &ctx) {
   pm.add(std::make_unique<CompactUnwindPass>(ctx));
 }
 
-} // end namesapce mach_o
-} // end namesapce lld
+} // end namespace mach_o
+} // end namespace lld
diff --git a/lld/lib/ReaderWriter/MachO/GOTPass.cpp b/lld/lib/ReaderWriter/MachO/GOTPass.cpp
index 0f80dfa19d09c..10e611c1bd2b4 100644
--- a/lld/lib/ReaderWriter/MachO/GOTPass.cpp
+++ b/lld/lib/ReaderWriter/MachO/GOTPass.cpp
@@ -179,5 +179,5 @@ void addGOTPass(PassManager &pm, const MachOLinkingContext &ctx) {
   pm.add(std::make_unique<GOTPass>(ctx));
 }
 
-} // end namesapce mach_o
-} // end namesapce lld
+} // end namespace mach_o
+} // end namespace lld
diff --git a/lld/lib/ReaderWriter/MachO/TLVPass.cpp b/lld/lib/ReaderWriter/MachO/TLVPass.cpp
index 5f457b863d904..e0a031cfb07b2 100644
--- a/lld/lib/ReaderWriter/MachO/TLVPass.cpp
+++ b/lld/lib/ReaderWriter/MachO/TLVPass.cpp
@@ -136,5 +136,5 @@ void addTLVPass(PassManager &pm, const MachOLinkingContext &ctx) {
   pm.add(std::make_unique<TLVPass>(ctx));
 }
 
-} // end namesapce mach_o
-} // end namesapce lld
+} // end namespace mach_o
+} // end namespace lld
diff --git a/lld/test/COFF/delayimports32.test b/lld/test/COFF/delayimports32.test
index 41b4bdc97ef10..3c1f892ff837c 100644
--- a/lld/test/COFF/delayimports32.test
+++ b/lld/test/COFF/delayimports32.test
@@ -72,14 +72,14 @@ BASEREL-NEXT:   }
 BASEREL-NEXT: ]
 
 DISASM:      1029:      b8 20 30 40 00  movl    $4206624, %eax
-DISASM-NEXT: 102e:      e9 0a 00 00 00  jmp     10 <.text+0x3d>
+DISASM-NEXT: 102e:      e9 0a 00 00 00  jmp     0x40103d <.text+0x3d>
 DISASM-NEXT: 1033:      b8 24 30 40 00  movl    $4206628, %eax
-DISASM-NEXT: 1038:      e9 00 00 00 00  jmp     0 <.text+0x3d>
+DISASM-NEXT: 1038:      e9 00 00 00 00  jmp     0x40103d <.text+0x3d>
 DISASM-NEXT: 103d:      51      pushl   %ecx
 DISASM-NEXT: 103e:      52      pushl   %edx
 DISASM-NEXT: 103f:      50      pushl   %eax
 DISASM-NEXT: 1040:      68 00 20 40 00  pushl   $4202496
-DISASM-NEXT: 1045:      e8 b6 ff ff ff  calll   -74 <.text>
+DISASM-NEXT: 1045:      e8 b6 ff ff ff  calll   0x401000 <.text>
 DISASM-NEXT: 104a:      5a      popl    %edx
 DISASM-NEXT: 104b:      59      popl    %ecx
 DISASM-NEXT: 104c:      ff e0   jmpl    *%eax
diff --git a/lld/test/COFF/gc-dwarf.s b/lld/test/COFF/gc-dwarf.s
new file mode 100644
index 0000000000000..56b866f4a4ac4
--- /dev/null
+++ b/lld/test/COFF/gc-dwarf.s
@@ -0,0 +1,60 @@
+# REQUIRES: x86
+
+# RUN: llvm-mc -triple=x86_64-windows-msvc %s -filetype=obj -o %t.obj
+# RUN: lld-link -lldmap:%t.map -out:%t.exe -opt:ref -entry:main %t.obj -verbose 2>&1 | FileCheck %s
+# RUN: FileCheck %s --check-prefix=MAP --input-file=%t.map
+
+# CHECK:      Discarded unused1
+# CHECK-NEXT: Discarded unused2
+# CHECK-NOT: Discarded
+
+# MAP: In Symbol
+# MAP: gc-dwarf.s.tmp.obj:(.text)
+# MAP: {{ main$}}
+# MAP: gc-dwarf.s.tmp.obj:(.text)
+# MAP: {{ used$}}
+
+	.def	 @feat.00; .scl	3; .type	0; .endef
+	.globl	@feat.00
+.set @feat.00, 0
+
+	.def	 main; .scl	2; .type	32; .endef
+	.section	.text,"xr",one_only,main
+	.globl	main
+main:
+	callq used
+	xorl	%eax, %eax
+	retq
+
+	.def	 used; .scl	2; .type	32; .endef
+	.section	.text,"xr",one_only,used
+	.globl	used
+used:
+	retq
+
+
+	.def	 unused1; .scl	2; .type	32; .endef
+	.section	.text,"xr",one_only,unused1
+	.globl	unused1
+unused1:
+	retq
+
+	.def	 unused2; .scl	2; .type	32; .endef
+	.section	.text,"xr",one_only,unused2
+	.globl	unused2
+unused2:
+	retq
+
+# This isn't valid DWARF, but LLD doesn't care. Make up some data that
+# references the functions above.
+.section .debug_info,"r"
+.long main@IMGREL
+.long unused1@IMGREL
+.long unused2@IMGREL
+
+# Similarly, .eh_frame unwind info should not keep functions alive. Again, this
+# is not valid unwind info, but it doesn't matter for testing purposes.
+.section .eh_frame,"r"
+.long main@IMGREL
+.long unused1@IMGREL
+.long unused2@IMGREL
diff --git a/lld/test/COFF/imports.test b/lld/test/COFF/imports.test
index c120b82055edb..a5e030b57744b 100644
--- a/lld/test/COFF/imports.test
+++ b/lld/test/COFF/imports.test
@@ -19,10 +19,10 @@ TEXT-NEXT: movq    $0, %rcx
 TEXT-NEXT: leaq    8180(%rip), %rdx
 TEXT-NEXT: leaq    8167(%rip), %r8
 TEXT-NEXT: movl    $0, %r9d
-TEXT-NEXT: callq   60
+TEXT-NEXT: callq   0x140001060
 TEXT-NEXT: movl    $0, %ecx
-TEXT-NEXT: callq   18
-TEXT-NEXT: callq   29
+TEXT-NEXT: callq   0x140001040
+TEXT-NEXT: callq   0x140001050
 TEXT:      jmpq    *4098(%rip)
 TEXT:      jmpq    *4090(%rip)
 TEXT:      jmpq    *4082(%rip)
diff --git a/lld/test/COFF/lto-comdat.ll b/lld/test/COFF/lto-comdat.ll
index 68a82c6d584a5..517dbd66314a7 100644
--- a/lld/test/COFF/lto-comdat.ll
+++ b/lld/test/COFF/lto-comdat.ll
@@ -52,9 +52,9 @@
 ; TEXT-01-EMPTY:
 ; TEXT-01-NEXT: <.text>:
 ; TEXT-01-NEXT: subq	$40, %rsp
-; TEXT-01-NEXT: callq	23
-; TEXT-01-NEXT: callq	18
-; TEXT-01-NEXT: callq	13
+; TEXT-01-NEXT: callq	0x140001020
+; TEXT-01-NEXT: callq	0x140001020
+; TEXT-01-NEXT: callq	0x140001020
 ; TEXT-01-NEXT: xorl	%eax, %eax
 ; TEXT-01-NEXT: addq	$40, %rsp
 ; TEXT-01: retq
@@ -70,20 +70,20 @@
 ; TEXT-10-EMPTY:
 ; TEXT-10-NEXT: <.text>:
 ; TEXT-10-NEXT: subq	$40, %rsp
-; TEXT-10-NEXT: callq	55
+; TEXT-10-NEXT: callq	0x140001040
 ; TEXT-10-NEXT: nop
 ; TEXT-10-NEXT: addq	$40, %rsp
 ; TEXT-10-NEXT: retq
 ; TEXT-10-NEXT: int3
 ; TEXT-10-NEXT: subq	$40, %rsp
-; TEXT-10-NEXT: callq	39
+; TEXT-10-NEXT: callq	0x140001040
 ; TEXT-10-NEXT: nop
 ; TEXT-10-NEXT: addq	$40, %rsp
 ; TEXT-10-NEXT: retq
 ; TEXT-10-NEXT: int3
 ; TEXT-10-NEXT: subq	$40, %rsp
-; TEXT-10-NEXT: callq	-41
-; TEXT-10-NEXT: callq	-30
+; TEXT-10-NEXT: callq	0x140001000
+; TEXT-10-NEXT: callq	0x140001010
 ; TEXT-10-NEXT: xorl	%eax, %eax
 ; TEXT-10-NEXT: addq	$40, %rsp
 ; TEXT-10-NEXT: retq
diff --git a/lld/test/COFF/lto.ll b/lld/test/COFF/lto.ll
index f00d25dda68a3..d8ccf566939eb 100644
--- a/lld/test/COFF/lto.ll
+++ b/lld/test/COFF/lto.ll
@@ -59,7 +59,7 @@
 ; TEXT-01-EMPTY:
 ; TEXT-01-NEXT: <.text>:
 ; TEXT-01-NEXT: subq	$40, %rsp
-; TEXT-01-NEXT: callq	23
+; TEXT-01-NEXT: callq	0x140001020
 ; TEXT-01-NEXT: xorl	%eax, %eax
 ; TEXT-01-NEXT: addq	$40, %rsp
 ; TEXT-01-NEXT: retq
@@ -118,7 +118,7 @@
 ; TEXT-10-NEXT: int3
 ; TEXT-10-NEXT: int3
 ; TEXT-10-NEXT: subq	$40, %rsp
-; TEXT-10-NEXT: callq	-41
+; TEXT-10-NEXT: callq	0x140001000
 ; TEXT-10-NEXT: xorl	%eax, %eax
 ; TEXT-10-NEXT: addq	$40, %rsp
 ; TEXT-10-NEXT: retq
diff --git a/lld/test/COFF/pdb-globals.test b/lld/test/COFF/pdb-globals.test
index 00909fa317bbe..169a169076338 100644
--- a/lld/test/COFF/pdb-globals.test
+++ b/lld/test/COFF/pdb-globals.test
@@ -2,9 +2,9 @@ RUN: yaml2obj %S/Inputs/pdb-globals.yaml > %t.obj
 RUN: lld-link /debug /nodefaultlib /entry:main /out:%t.exe /pdb:%t.pdb %t.obj
 RUN: llvm-pdbutil dump -symbols -globals %t.pdb | FileCheck %s
 
-RUN: lld-link /debug /nodefaultlib /entry:main /out:%t.exe /pdb:%t.pdb %t.obj /threads
+RUN: lld-link /debug /nodefaultlib /entry:main /out:%t.exe /pdb:%t.pdb %t.obj /threads:1
 RUN: llvm-pdbutil dump -symbols -globals %t.pdb | FileCheck %s
-RUN: lld-link /debug /nodefaultlib /entry:main /out:%t.exe /pdb:%t.pdb %t.obj /threads:no
+RUN: lld-link /debug /nodefaultlib /entry:main /out:%t.exe /pdb:%t.pdb %t.obj /threads:2
 RUN: llvm-pdbutil dump -symbols -globals %t.pdb | FileCheck %s
 
 # Test that we correctly distribute symbols between the globals and module
diff --git a/lld/test/COFF/secidx-absolute.s b/lld/test/COFF/secidx-absolute.s
index 0b467bbb09bfa..8befaf2f456c3 100644
--- a/lld/test/COFF/secidx-absolute.s
+++ b/lld/test/COFF/secidx-absolute.s
@@ -3,7 +3,7 @@
 # RUN: lld-link -entry:main -nodefaultlib %t.obj -out:%t.exe
 # RUN: llvm-readobj %t.exe -sections -section-data | FileCheck %s
 
-# Section relocations against absolute symbols resolve to the last real ouput
+# Section relocations against absolute symbols resolve to the last real output
 # section index plus one.
 
 .text
diff --git a/lld/test/COFF/thinlto.ll b/lld/test/COFF/thinlto.ll
index 517be330c7109..a8390721cc428 100644
--- a/lld/test/COFF/thinlto.ll
+++ b/lld/test/COFF/thinlto.ll
@@ -6,6 +6,16 @@
 ; RUN: lld-link /lldsavetemps /out:%T/thinlto/main.exe /entry:main /subsystem:console %T/thinlto/main.obj %T/thinlto/foo.obj
 ; RUN: llvm-nm %T/thinlto/main.exe1.lto.obj | FileCheck %s
 
+; Test various possible options for /opt:lldltojobs
+; RUN: lld-link /lldsavetemps /out:%T/thinlto/main.exe /entry:main /subsystem:console %T/thinlto/main.obj %T/thinlto/foo.obj /opt:lldltojobs=1
+; RUN: llvm-nm %T/thinlto/main.exe1.lto.obj | FileCheck %s
+; RUN: lld-link /lldsavetemps /out:%T/thinlto/main.exe /entry:main /subsystem:console %T/thinlto/main.obj %T/thinlto/foo.obj /opt:lldltojobs=all
+; RUN: llvm-nm %T/thinlto/main.exe1.lto.obj | FileCheck %s
+; RUN: lld-link /lldsavetemps /out:%T/thinlto/main.exe /entry:main /subsystem:console %T/thinlto/main.obj %T/thinlto/foo.obj /opt:lldltojobs=1000
+; RUN: llvm-nm %T/thinlto/main.exe1.lto.obj | FileCheck %s
+; RUN: not lld-link /lldsavetemps /out:%T/thinlto/main.exe /entry:main /subsystem:console %T/thinlto/main.obj %T/thinlto/foo.obj /opt:lldltojobs=foo 2>&1 | FileCheck %s --check-prefix=BAD-JOBS
+; BAD-JOBS: error: /opt:lldltojobs: invalid job count: foo
+
 ; This command will store full path to foo.obj in the archive %t.lib
 ; Check that /lldsavetemps is still usable in such case.
 ; RUN: lld-link /lib %T/thinlto/foo.obj /out:%t.lib
diff --git a/lld/test/COFF/undefined-symbol-multi.s b/lld/test/COFF/undefined-symbol-multi.s
index 5942972a02519..2d3d8285338c5 100644
--- a/lld/test/COFF/undefined-symbol-multi.s
+++ b/lld/test/COFF/undefined-symbol-multi.s
@@ -22,14 +22,7 @@
 # CHECK-NEXT: >>> referenced by {{.*}}tmp.obj:(main)
 # CHECK-NEXT: >>> referenced by {{.*}}tmp.obj:(main)
 # CHECK-NEXT: >>> referenced by {{.*}}tmp2.obj:(bar)
-# CHECK-NEXT: >>> referenced by {{.*}}tmp2.obj:(bar)
-# CHECK-NEXT: >>> referenced by {{.*}}tmp2.obj:(bar)
-# CHECK-NEXT: >>> referenced by {{.*}}tmp2.obj:(bar)
-# CHECK-NEXT: >>> referenced by {{.*}}tmp2.obj:(bar)
-# CHECK-NEXT: >>> referenced by {{.*}}tmp2.obj:(bar)
-# CHECK-NEXT: >>> referenced by {{.*}}tmp2.obj:(bar)
-# CHECK-NEXT: >>> referenced by {{.*}}tmp2.obj:(bar)
-# CHECK-NEXT: >>> referenced 2 more times
+# CHECK-NEXT: >>> referenced 9 more times
 # CHECK-EMPTY:
 # CHECK-NEXT: error: undefined symbol: int __cdecl bar(void)
 # CHECK-NEXT: >>> referenced by {{.*}}.obj:(main)
diff --git a/lld/test/ELF/Inputs/copy-rel.s b/lld/test/ELF/Inputs/copy-rel.s
index e6714943019c4..bcfc7a58a33f4 100644
--- a/lld/test/ELF/Inputs/copy-rel.s
+++ b/lld/test/ELF/Inputs/copy-rel.s
@@ -1,4 +1,4 @@
-.globl foo, bar
+.globl foo
 .type foo, @object
 .size foo, 4
 foo:
diff --git a/lld/test/ELF/Inputs/deplibs-lib_bar.s b/lld/test/ELF/Inputs/deplibs-lib_bar.s
deleted file mode 100644
index b092459eef337..0000000000000
--- a/lld/test/ELF/Inputs/deplibs-lib_bar.s
+++ /dev/null
@@ -1,2 +0,0 @@
-        .global bar
-bar:
diff --git a/lld/test/ELF/Inputs/deplibs-lib_foo.s b/lld/test/ELF/Inputs/deplibs-lib_foo.s
deleted file mode 100644
index 2f0ccf81e395d..0000000000000
--- a/lld/test/ELF/Inputs/deplibs-lib_foo.s
+++ /dev/null
@@ -1,2 +0,0 @@
-        .global foo
-foo:
diff --git a/lld/test/ELF/Inputs/undef-bad-debug.s b/lld/test/ELF/Inputs/undef-bad-debug.s
index bf517f3ea1cd8..d3171f0236167 100644
--- a/lld/test/ELF/Inputs/undef-bad-debug.s
+++ b/lld/test/ELF/Inputs/undef-bad-debug.s
@@ -16,7 +16,7 @@ sym4:
     .long .Lprologue_end - .Lprologue_start # prologue length
 .Lprologue_start:
     .byte 1                         # minimum instruction length
-    .byte 1                         # maximum operatiosn per instruction
+    .byte 1                         # maximum operations per instruction
     .byte 1                         # default is_stmt
     .byte -5                        # line base
     .byte 14                        # line range
@@ -59,7 +59,7 @@ sym4:
     .long .Lprologue2_end - .Lprologue2_start # prologue length
 .Lprologue2_start:
     .byte 1                         # minimum instruction length
-    .byte 1                         # maximum operatiosn per instruction
+    .byte 1                         # maximum operations per instruction
     .byte 1                         # default is_stmt
     .byte -5                        # line base
     .byte 14                        # line range
diff --git a/lld/test/ELF/allow-multiple-definition.s b/lld/test/ELF/allow-multiple-definition.s
index 8a3b97c0102e5..bcf5cc7213c32 100644
--- a/lld/test/ELF/allow-multiple-definition.s
+++ b/lld/test/ELF/allow-multiple-definition.s
@@ -14,7 +14,7 @@
 # RUN: llvm-objdump -d %t3 | FileCheck %s
 # RUN: llvm-objdump -d %t4 | FileCheck --check-prefix=REVERT %s
 
-# inputs contain different constants for instuction movl.
+# inputs contain different constants for instruction movl.
 # Tests below checks that order of files in command line
 # affects on what symbol will be used.
 # If flag allow-multiple-definition is enabled the first
diff --git a/lld/test/ELF/arm-exidx-add-missing.s b/lld/test/ELF/arm-exidx-add-missing.s
index 1beaa299a82e0..da3c271f2355e 100644
--- a/lld/test/ELF/arm-exidx-add-missing.s
+++ b/lld/test/ELF/arm-exidx-add-missing.s
@@ -10,7 +10,7 @@
 // The range of addresses covered by the table entry is terminated by the
 // next table entry. This means that an executable section without a .ARM.exidx
 // section does not terminate the range of addresses. To fix this the linker
-// synthesises an EXIDX_CANTUNWIND entry for each section wihout a .ARM.exidx
+// synthesises an EXIDX_CANTUNWIND entry for each section without a .ARM.exidx
 // section.
 
         .syntax unified
diff --git a/lld/test/ELF/arm-exidx-link.s b/lld/test/ELF/arm-exidx-link.s
deleted file mode 100644
index 071d28546c0e3..0000000000000
--- a/lld/test/ELF/arm-exidx-link.s
+++ /dev/null
@@ -1,25 +0,0 @@
-// REQUIRES: arm
-// RUN: llvm-mc -filetype=obj -triple=armv7a-none-linux-gnueabi %s -o %t.o
-// RUN: ld.lld %t.o -o %t.so -shared
-// RUN: llvm-readobj -S %t.so | FileCheck %s
-
-// CHECK:      Name: .ARM.exidx
-// CHECK-NEXT: Type: SHT_ARM_EXIDX
-// CHECK-NEXT: Flags [
-// CHECK-NEXT:   SHF_ALLOC
-// CHECK-NEXT:   SHF_LINK_ORDER
-// CHECK-NEXT: ]
-// CHECK-NEXT: Address:
-// CHECK-NEXT: Offset:
-// CHECK-NEXT: Size:
-// CHECK-NEXT: Link: [[INDEX:.*]]
-
-// CHECK:      Index: [[INDEX]]
-// CHECK-NEXT: Name: .text
-
-
-        f:
-	.fnstart
-	bx	lr
-	.cantunwind
-	.fnend
diff --git a/lld/test/ELF/arm-exidx-output.s b/lld/test/ELF/arm-exidx-output.s
index 4946bbad7a804..50e7820f2bdc6 100644
--- a/lld/test/ELF/arm-exidx-output.s
+++ b/lld/test/ELF/arm-exidx-output.s
@@ -1,12 +1,29 @@
 // REQUIRES: arm
-// RUN: llvm-mc -filetype=obj -triple=armv7a-none-linux-gnueabi %s -o %t
-// RUN: ld.lld %t -o %t2
-// RUN: llvm-readobj --sections %t2 | FileCheck %s
+// RUN: llvm-mc -filetype=obj -triple=armv7a-none-linux-gnueabi %s -o %t.o
+// RUN: ld.lld %t.o -o %t
+// RUN: llvm-readelf -S %t | FileCheck %s
+
+// RUN: echo 'SECTIONS { .text.f1 : { *(.text.f1) } .text.f2 : { *(.text.f2) } }' > %t.lds
+// RUN: ld.lld -T %t.lds %t.o -o %t1
+// RUN: llvm-readelf -S %t1 | FileCheck --check-prefix=MULTI %s
 
 // Check that only a single .ARM.exidx output section is created when
 // there are input sections of the form .ARM.exidx.<section-name>. The
 // assembler creates the .ARM.exidx input sections with the .cantunwind
 // directive
+
+// CHECK:      [Nr] Name       Type      {{.*}} Flg Lk
+// CHECK-NEXT: [ 0]
+// CHECK-NEXT: [ 1] .ARM.exidx ARM_EXIDX {{.*}}  AL  2
+// CHECK-NEXT: [ 2] .text      PROGBITS  {{.*}}  AX  0
+
+// MULTI:      [Nr] Name       Type      {{.*}} Flg Lk
+// MULTI-NEXT: [ 0]
+// MULTI-NEXT: [ 1] .ARM.exidx ARM_EXIDX {{.*}}  AL  2
+// MULTI-NEXT: [ 2] .text.f1   PROGBITS  {{.*}}  AX  0
+// MULTI-NEXT: [ 3] .text.f2   PROGBITS  {{.*}}  AX  0
+// MULTI-NEXT: [ 4] .text      PROGBITS  {{.*}}  AX  0
+
  .syntax unified
  .section .text, "ax",%progbits
  .globl _start
@@ -31,14 +48,3 @@ f2:
  bx lr
  .cantunwind
  .fnend
-
-// CHECK:         Section {
-// CHECK:         Name: .ARM.exidx
-// CHECK-NEXT:    Type: SHT_ARM_EXIDX (0x70000001)
-// CHECK-NEXT:    Flags [
-// CHECK-NEXT:      SHF_ALLOC
-// CHECK-NEXT:      SHF_LINK_ORDER
-// CHECK-NEXT:    ]
-
-// CHECK-NOT:     Name: .ARM.exidx.text.f1
-// CHECK-NOT:     Name: .ARM.exidx.text.f2
diff --git a/lld/test/ELF/arm-thumb-adr-err.s b/lld/test/ELF/arm-thumb-adr-err.s
index c37ccf94f3a5d..4b3a70f16c7d7 100644
--- a/lld/test/ELF/arm-thumb-adr-err.s
+++ b/lld/test/ELF/arm-thumb-adr-err.s
@@ -1,5 +1,5 @@
 // REQUIRES: arm
-// RUN: llvm-mc -g --triple=thumbv6m-none-eabi --arm-add-build-attributes -filetype=obj -o %t.o %s
+// RUN: llvm-mc --triple=thumbv6m-none-eabi --arm-add-build-attributes -filetype=obj -o %t.o %s
 // RUN: not ld.lld %t.o -o /dev/null 2>&1 | FileCheck %s
 
  .section .text.0, "ax", %progbits
@@ -13,12 +13,18 @@ low:
  .global _start
  .thumb_func
 _start:
-// CHECK: {{.*}}.s:[[# @LINE+1]]:(.text.1+0x0): relocation R_ARM_THM_PC8 out of range: 18446744073709551612 is not in [0, 1023]
- adr r0, low
-// CHECK: {{.*}}.s:[[# @LINE+1]]:(.text.1+0x2): improper alignment for relocation R_ARM_THM_PC8: 0x2 is not aligned to 4 bytes
- adr r1, unaligned
-// CHECK: {{.*}}.s:[[# @LINE+1]]:(.text.1+0x4): relocation R_ARM_THM_PC8 out of range: 1024 is not in [0, 1023]
- adr r2, range
+// CHECK: {{.*}}.s.tmp.o:(.text.1+0x0): relocation R_ARM_THM_PC8 out of range: 18446744073709551612 is not in [0, 1023]
+/// adr r0, low
+ .inst.n 0xa0ff
+ .reloc 0, R_ARM_THM_PC8, low
+// CHECK: {{.*}}.s.tmp.o:(.text.1+0x2): improper alignment for relocation R_ARM_THM_PC8: 0x2 is not aligned to 4 bytes
+/// adr r1, unaligned
+ .inst.n 0xa1ff
+ .reloc 2, R_ARM_THM_PC8, unaligned
+// CHECK: {{.*}}.s.tmp.o:(.text.1+0x4): relocation R_ARM_THM_PC8 out of range: 1024 is not in [0, 1023]
+/// adr r2, range
+ .inst.n 0xa2ff
+ .reloc 4, R_ARM_THM_PC8, range
 
  .section .text.2, "ax", %progbits
  .balign 4
diff --git a/lld/test/ELF/arm-thumb-adr.s b/lld/test/ELF/arm-thumb-adr.s
index 35d3b4ca35018..adeac7d5f21bd 100644
--- a/lld/test/ELF/arm-thumb-adr.s
+++ b/lld/test/ELF/arm-thumb-adr.s
@@ -10,9 +10,12 @@
  .global _start
  .thumb_func
 _start:
- adr r0, target1
- adr r1, target2
-
+/// adr r0, target1
+ .inst.n 0xa0ff
+ .reloc 0, R_ARM_THM_PC8, target1
+/// adr r1, target2
+ .inst.n 0xa1ff
+ .reloc 2, R_ARM_THM_PC8, target2
  .section .text.02, "ax", %progbits
  .balign 4
  .global target1
diff --git a/lld/test/ELF/arm-thumb-ldrlit-err.s b/lld/test/ELF/arm-thumb-ldrlit-err.s
index 5f53eacb2f1f7..14f1f377a99b8 100644
--- a/lld/test/ELF/arm-thumb-ldrlit-err.s
+++ b/lld/test/ELF/arm-thumb-ldrlit-err.s
@@ -1,5 +1,5 @@
 // REQUIRES: arm
-// RUN: llvm-mc -g --triple=thumbv6m-none-eabi --arm-add-build-attributes -filetype=obj -o %t.o %s
+// RUN: llvm-mc --triple=thumbv6m-none-eabi --arm-add-build-attributes -filetype=obj -o %t.o %s
 // RUN: not ld.lld %t.o -o /dev/null 2>&1 | FileCheck %s
 
  .section .text.0, "ax", %progbits
@@ -13,12 +13,18 @@ low:
  .global _start
  .thumb_func
 _start:
-// CHECK: {{.*}}.s:[[# @LINE+1]]:(.text.1+0x0): relocation R_ARM_THM_PC8 out of range: 18446744073709551612 is not in [0, 1023]
- ldr r0, low
-// CHECK: {{.*}}.s:[[# @LINE+1]]:(.text.1+0x2): improper alignment for relocation R_ARM_THM_PC8: 0x2 is not aligned to 4 bytes
- ldr r1, unaligned
-// CHECK: {{.*}}.s:[[# @LINE+1]]:(.text.1+0x4): relocation R_ARM_THM_PC8 out of range: 1024 is not in [0, 1023]
- ldr r2, range
+// CHECK: {{.*}}.s.tmp.o:(.text.1+0x0): relocation R_ARM_THM_PC8 out of range: 18446744073709551612 is not in [0, 1023]
+/// ldr r0, low
+ .inst.n 0x48ff
+ .reloc 0, R_ARM_THM_PC8, low
+// CHECK: {{.*}}.s.tmp.o:(.text.1+0x2): improper alignment for relocation R_ARM_THM_PC8: 0x2 is not aligned to 4 bytes
+/// ldr r1, unaligned
+ .inst.n 0x49ff
+ .reloc 2, R_ARM_THM_PC8, unaligned
+// CHECK: {{.*}}.s.tmp.o:(.text.1+0x4): relocation R_ARM_THM_PC8 out of range: 1024 is not in [0, 1023]
+/// ldr r2, range
+ .inst.n 0x4aff
+ .reloc 4, R_ARM_THM_PC8, range
 
  .section .text.2, "ax", %progbits
  .balign 4
diff --git a/lld/test/ELF/arm-thumb-ldrlit.s b/lld/test/ELF/arm-thumb-ldrlit.s
index 43bcf81b71bad..31eea197e1ada 100644
--- a/lld/test/ELF/arm-thumb-ldrlit.s
+++ b/lld/test/ELF/arm-thumb-ldrlit.s
@@ -10,9 +10,12 @@
  .global _start
  .thumb_func
 _start:
- ldr r0, target1
- ldr r1, target2
-
+/// ldr r0, target1
+ .inst.n 0x48ff
+ .reloc 0, R_ARM_THM_PC8, target1
+/// ldr r1, target2
+ .inst.n 0x49ff
+ .reloc 2, R_ARM_THM_PC8, target2
  .section .text.02, "ax", %progbits
  .balign 4
  .global target1
diff --git a/lld/test/ELF/arm-thumb2-adr-err.s b/lld/test/ELF/arm-thumb2-adr-err.s
index fb3afafe85087..20ded45e329a8 100644
--- a/lld/test/ELF/arm-thumb2-adr-err.s
+++ b/lld/test/ELF/arm-thumb2-adr-err.s
@@ -1,5 +1,5 @@
 // REQUIRES: arm
-// RUN: llvm-mc -g --triple=thumbv7m-none-eabi --arm-add-build-attributes -filetype=obj -o %t.o %s
+// RUN: llvm-mc --triple=thumbv7m-none-eabi --arm-add-build-attributes -filetype=obj -o %t.o %s
 // RUN: not ld.lld %t.o -o /dev/null 2>&1 | FileCheck %s
 
  .section .text.0, "ax", %progbits
@@ -14,11 +14,14 @@ low:
  .global _start
  .thumb_func
 _start:
-// CHECK: {{.*}}.s:[[# @LINE+1]]:(.text.1+0x0): relocation R_ARM_THM_ALU_PREL_11_0 out of range: 4098 is not in [0, 4095]
- adr.w r0, low - 4091
-// CHECK: {{.*}}.s:[[# @LINE+1]]:(.text.1+0x4): relocation R_ARM_THM_ALU_PREL_11_0 out of range: 4096 is not in [0, 4095]
- adr.w r0, high + 4091
-
+// CHECK: {{.*}}.s.tmp.o:(.text.1+0x0): relocation R_ARM_THM_ALU_PREL_11_0 out of range: 4098 is not in [0, 4095]
+/// adr.w r0, low - 4091
+ .inst.w 0xf6af70ff
+ .reloc 0, R_ARM_THM_ALU_PREL_11_0, low
+// CHECK: {{.*}}.s.tmp.o:(.text.1+0x4): relocation R_ARM_THM_ALU_PREL_11_0 out of range: 4096 is not in [0, 4095]
+/// adr.w r0, high + 4091
+ .inst.w 0xf60f70f7
+ .reloc 4, R_ARM_THM_ALU_PREL_11_0, high
  .section .text.2
  .thumb_func
  .balign 4
diff --git a/lld/test/ELF/arm-thumb2-adr.s b/lld/test/ELF/arm-thumb2-adr.s
index 86b29ad9e1113..a6895bc878b6b 100644
--- a/lld/test/ELF/arm-thumb2-adr.s
+++ b/lld/test/ELF/arm-thumb2-adr.s
@@ -10,7 +10,7 @@
 // RUN:               } " > %t.script
 // RUN: ld.lld --script %t.script %t.o -o %t
 // RUN: llvm-readobj --symbols %t | FileCheck %s --check-prefix=SYMS
-// RUN: llvm-objdump -d --no-show-raw-insn --triple=thumbv7m-none-eabi %t
+// RUN: llvm-objdump -d --triple=thumbv7m-none-eabi %t
 // RUN: llvm-objdump -d --no-show-raw-insn --triple=thumbv7m-none-eabi %t | FileCheck %s
 
 /// Test the various legal cases for the R_ARM_THM_ALU_PREL_11_0 relocation
@@ -44,32 +44,58 @@ target2:
  .thumb_func
 _start:
  nop
- adr.w r0, dat1
- adr.w r1, dat2
+///  adr.w r0, dat1
+ .inst.w 0xf2af0004
+ .reloc 2, R_ARM_THM_ALU_PREL_11_0, dat1
+/// adr.w r1, dat2
+ .inst.w 0xf2af0104
+ .reloc 6, R_ARM_THM_ALU_PREL_11_0, dat2
  nop
- adr.w r2, dat3
- adr.w r3, dat4
- .balign 4
- adr.w r0, target1
+/// adr.w r2, dat3
+ .inst.w 0xf2af0204
+ .reloc 0xc, R_ARM_THM_ALU_PREL_11_0, dat3
+/// adr.w r3, dat4
+ .inst.w 0xf2af0304
+ .reloc 0x10, R_ARM_THM_ALU_PREL_11_0, dat4
+/// adr.w r0, target1
+ .inst.w 0xf2af0004
+ .reloc 0x14, R_ARM_THM_ALU_PREL_11_0, target1
  nop
- adr.w r1, target2
-
+/// adr.w r1, target2
+ .inst.w 0xf2af0104
+ .reloc 0x1a, R_ARM_THM_ALU_PREL_11_0, target2
  .section .text.pos, "ax", %progbits
  .balign 4
  .global pos
  .thumb_func
 pos:
- adr.w r2, target3
+/// adr.w r2, target3
+ .inst.w 0xf2af0204
+ .reloc 0, R_ARM_THM_ALU_PREL_11_0, target3
  nop
- adr.w r3, target4
- .balign 4
- adr.w r0, dat5
- adr.w r1, dat6
+/// adr.w r3, target4
+ .inst.w 0xf2af0304
+ .reloc 6, R_ARM_THM_ALU_PREL_11_0, target4
  nop
- adr.w r2, dat7
- adr.w r3, dat8
+/// adr.w r0, dat5
+ .inst.w 0xf2af0004
+ .reloc 0xc, R_ARM_THM_ALU_PREL_11_0, dat5
+/// adr.w r1, dat6
+ .inst.w 0xf2af0104
+ .reloc 0x10, R_ARM_THM_ALU_PREL_11_0, dat6
+
+ nop
+/// adr.w r2, dat7
+ .inst.w 0xf2af0204
+ .reloc 0x16, R_ARM_THM_ALU_PREL_11_0, dat7
+
+/// adr.w r3, dat8
+ .inst.w 0xf2af0304
+ .reloc 0x1a, R_ARM_THM_ALU_PREL_11_0, dat8
 /// positive addend in instruction, all others are -4 (PC bias)
- adr.w r4, dat5 + 8
+/// adr.w r4, dat5 + 8
+ .inst.w 0xf20f0404
+ .reloc 0x1e, R_ARM_THM_ALU_PREL_11_0, dat5 + 8
 
  .section .text.high, "ax", %progbits
  .balign 4
diff --git a/lld/test/ELF/arm-thumb2-ldrlit-err.s b/lld/test/ELF/arm-thumb2-ldrlit-err.s
index 451c3f6a12159..8d13e6dd0fd58 100644
--- a/lld/test/ELF/arm-thumb2-ldrlit-err.s
+++ b/lld/test/ELF/arm-thumb2-ldrlit-err.s
@@ -1,5 +1,5 @@
 // REQUIRES: arm
-// RUN: llvm-mc -g --triple=thumbv7m-none-eabi --arm-add-build-attributes -filetype=obj -o %t.o %s
+// RUN: llvm-mc --triple=thumbv7m-none-eabi --arm-add-build-attributes -filetype=obj -o %t.o %s
 // RUN: not ld.lld %t.o -o /dev/null 2>&1 | FileCheck %s
 
  .section .text.0, "ax", %progbits
@@ -14,10 +14,14 @@ low:
  .global _start
  .thumb_func
 _start:
-// CHECK: {{.*}}.s:[[# @LINE+1]]:(.text.1+0x0): relocation R_ARM_THM_PC12 out of range: 4098 is not in [0, 4095]
- ldr.w r0, low - 4091
-// CHECK: {{.*}}.s:[[# @LINE+1]]:(.text.1+0x4): relocation R_ARM_THM_PC12 out of range: 4096 is not in [0, 4095]
- ldr.w r0, high + 4091
+// CHECK: {{.*}}.s.tmp.o:(.text.1+0x0): relocation R_ARM_THM_PC12 out of range: 4098 is not in [0, 4095]
+/// ldr.w r0, low - 4091
+ .inst.w 0xf85f0fff
+ .reloc 0, R_ARM_THM_PC12, low
+// CHECK: {{.*}}.s.tmp.o:(.text.1+0x4): relocation R_ARM_THM_PC12 out of range: 4096 is not in [0, 4095]
+/// ldr.w r0, high + 4091
+ .inst.w 0xf8df0ff7
+ .reloc 4, R_ARM_THM_PC12, high
 
  .section .text.2
  .thumb_func
diff --git a/lld/test/ELF/arm-thumb2-ldrlit.s b/lld/test/ELF/arm-thumb2-ldrlit.s
index cbf092208740c..a179a19149de5 100644
--- a/lld/test/ELF/arm-thumb2-ldrlit.s
+++ b/lld/test/ELF/arm-thumb2-ldrlit.s
@@ -44,32 +44,59 @@ target2:
  .thumb_func
 _start:
  nop
- ldr r0, dat1
- ldr r1, dat2
+/// ldr r0, dat1
+ .inst.w 0xf85f0004
+ .reloc 2, R_ARM_THM_PC12, dat1
+/// ldr r1, dat2
+ .inst.w 0xf85f1004
+ .reloc 6, R_ARM_THM_PC12, dat2
  nop
- ldr r2, dat3
- ldr r3, dat4
- .balign 4
- ldr r0, target1
+/// ldr r2, dat3
+ .inst.w 0xf85f2004
+ .reloc 0xc, R_ARM_THM_PC12, dat3
+/// ldr r3, dat4
+ .inst.w 0xf85f3004
+ .reloc 0x10, R_ARM_THM_PC12, dat4
+
+/// ldr r0, target1
+ .inst.w 0xf85f0004
+ .reloc 0x14, R_ARM_THM_PC12, target1
+
  nop
- ldr r1, target2
+/// ldr r1, target2
+ .inst.w 0xf85f1004
+ .reloc 0x1a, R_ARM_THM_PC12, target2
 
  .section .text.pos, "ax", %progbits
  .balign 4
  .global pos
  .thumb_func
 pos:
- ldr r2, target3
+/// ldr r2, target3
+ .inst.w 0xf85f2004
+ .reloc 0, R_ARM_THM_PC12, target3
  nop
- ldr r3, target4
- .balign 4
- ldr r0, dat5
- ldr r1, dat6
+/// ldr r3, target4
+ .inst.w 0xf85f3004
+ .reloc 6, R_ARM_THM_PC12, target4
+ nop
+/// ldr r0, dat5
+ .inst.w 0xf85f0004
+ .reloc 0xc, R_ARM_THM_PC12, dat5
+/// ldr r1, dat6
+ .inst.w 0xf85f1004
+ .reloc 0x10, R_ARM_THM_PC12, dat6
  nop
- ldr r2, dat7
- ldr r3, dat8
+/// ldr r2, dat7
+ .inst.w 0xf85f2004
+ .reloc 0x16, R_ARM_THM_PC12, dat7
+/// ldr r3, dat8
+ .inst.w 0xf85f3004
+ .reloc 0x1a, R_ARM_THM_PC12, dat8
 /// positive addend in instruction, all others are -4 (PC bias)
- ldr.w r4, dat5 + 8
+///ldr.w r4, dat5 + 8
+ .inst 0xf8df4004
+ .reloc 0x1e, R_ARM_THM_PC12, dat5
 
  .section .text.high, "ax", %progbits
  .balign 4
diff --git a/lld/test/ELF/basic.s b/lld/test/ELF/basic.s
index b0509a2dc1236..f6954b1434938 100644
--- a/lld/test/ELF/basic.s
+++ b/lld/test/ELF/basic.s
@@ -230,28 +230,8 @@ _start:
 # RUN:  FileCheck --check-prefix=CANNOT_OPEN %s
 # CANNOT_OPEN: cannot open {{.*}}.no.such.file: {{[Nn]}}o such file or directory
 
-# RUN: not ld.lld %t -o 2>&1 | FileCheck --check-prefix=NO_O_VAL %s
-# NO_O_VAL: -o: missing argument
-
 # RUN: not ld.lld --foo 2>&1 | FileCheck --check-prefix=UNKNOWN %s
 # UNKNOWN: unknown argument '--foo'
 
-# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t
-# RUN: not ld.lld %t %t -o /dev/null 2>&1 | FileCheck --check-prefix=DUP %s
-# DUP:      duplicate symbol: _start
-# DUP-NEXT: >>> defined at {{.*}}:(.text+0x0)
-# DUP-NEXT: >>> defined at {{.*}}:(.text+0x0)
-
 # RUN: not ld.lld %t -o /dev/null -m wrong_emul_fbsd 2>&1 | FileCheck --check-prefix=UNKNOWN_EMUL %s
 # UNKNOWN_EMUL: unknown emulation: wrong_emul_fbsd
-
-# RUN: not ld.lld %t --lto-partitions=0 2>&1 | FileCheck --check-prefix=NOTHREADS %s
-# RUN: not ld.lld %t --plugin-opt=lto-partitions=0 2>&1 | FileCheck --check-prefix=NOTHREADS %s
-# NOTHREADS: --lto-partitions: number of threads must be > 0
-
-# RUN: not ld.lld %t --thinlto-jobs=0 2>&1 | FileCheck --check-prefix=NOTHREADSTHIN %s
-# RUN: not ld.lld %t --plugin-opt=jobs=0 2>&1 | FileCheck --check-prefix=NOTHREADSTHIN %s
-# NOTHREADSTHIN: --thinlto-jobs: number of threads must be > 0
-
-# RUN: not ld.lld %t -z ifunc-noplt -z text 2>&1 | FileCheck --check-prefix=NOIFUNCPLTNOTEXTREL %s
-# NOIFUNCPLTNOTEXTREL: -z text and -z ifunc-noplt may not be used together
diff --git a/lld/test/ELF/build-id.s b/lld/test/ELF/build-id.s
index 32966b8a378b8..b693a088a2668 100644
--- a/lld/test/ELF/build-id.s
+++ b/lld/test/ELF/build-id.s
@@ -5,26 +5,26 @@
 # RUN: ld.lld --build-id %t -o %t2
 # RUN: llvm-readobj -S %t2 | FileCheck -check-prefix=ALIGN %s
 
-# RUN: ld.lld --build-id %t -o %t2 -threads
+# RUN: ld.lld --build-id %t -o %t2
 # RUN: llvm-objdump -s %t2 | FileCheck --check-prefix=DEFAULT %s
-# RUN: ld.lld --build-id=fast %t -o %t2 -threads
+# RUN: ld.lld --build-id=fast %t -o %t2
 # RUN: llvm-objdump -s %t2 | FileCheck --check-prefix=DEFAULT %s
-# RUN: ld.lld --build-id %t -o %t2 -no-threads
+# RUN: ld.lld --build-id %t -o %t2 --threads=1
 # RUN: llvm-objdump -s %t2 | FileCheck --check-prefix=DEFAULT %s
 
-# RUN: ld.lld --build-id=md5 %t -o %t2 -threads
+# RUN: ld.lld --build-id=md5 %t -o %t2
 # RUN: llvm-objdump -s %t2 | FileCheck --check-prefix=MD5 %s
-# RUN: ld.lld --build-id=md5 %t -o %t2 -no-threads
+# RUN: ld.lld --build-id=md5 %t -o %t2 --threads=1
 # RUN: llvm-objdump -s %t2 | FileCheck --check-prefix=MD5 %s
 
-# RUN: ld.lld --build-id=sha1 %t -o %t2 -threads
+# RUN: ld.lld --build-id=sha1 %t -o %t2
 # RUN: llvm-objdump -s %t2 | FileCheck --check-prefix=SHA1 %s
-# RUN: ld.lld --build-id=sha1 %t -o %t2 -no-threads
+# RUN: ld.lld --build-id=sha1 %t -o %t2 --threads=1
 # RUN: llvm-objdump -s %t2 | FileCheck --check-prefix=SHA1 %s
 
-# RUN: ld.lld --build-id=tree %t -o %t2 -threads
+# RUN: ld.lld --build-id=tree %t -o %t2
 # RUN: llvm-objdump -s %t2 | FileCheck --check-prefix=SHA1 %s
-# RUN: ld.lld --build-id=tree %t -o %t2 -no-threads
+# RUN: ld.lld --build-id=tree %t -o %t2 --threads=1
 # RUN: llvm-objdump -s %t2 | FileCheck --check-prefix=SHA1 %s
 
 # RUN: ld.lld --build-id=uuid %t -o %t2
diff --git a/lld/test/ELF/comdat.s b/lld/test/ELF/comdat.s
index bf754fbbfaf0a..689b8ca9d50a1 100644
--- a/lld/test/ELF/comdat.s
+++ b/lld/test/ELF/comdat.s
@@ -31,7 +31,7 @@ foo:
 // CHECK-EMPTY:
 // CHECK-NEXT: <bar>:
 // 0x1234 - 0x1235 - 5 = -6
-// CHECK-NEXT:   1235:	{{.*}}  callq  -6
+// CHECK-NEXT:   1235:	{{.*}}  callq  0x1234
 
         .section .text3,"axG",@progbits,zed,comdat,unique,0
 
diff --git a/lld/test/ELF/default-output.s b/lld/test/ELF/default-output.s
index c0766acf6e55e..e18f5da542597 100644
--- a/lld/test/ELF/default-output.s
+++ b/lld/test/ELF/default-output.s
@@ -9,6 +9,9 @@
 # RUN: ld.lld %t
 # RUN: llvm-readobj a.out > /dev/null 2>&1
 
+# RUN: not ld.lld %t -o 2>&1 | FileCheck --check-prefix=NO_O_VAL %s
+# NO_O_VAL: error: -o: missing argument
+
 .globl _start
 _start:
   mov $60, %rax
diff --git a/lld/test/ELF/deplibs-colon-prefix.s b/lld/test/ELF/deplibs-colon-prefix.s
index 6f480717b5b2c..6431f6f460816 100644
--- a/lld/test/ELF/deplibs-colon-prefix.s
+++ b/lld/test/ELF/deplibs-colon-prefix.s
@@ -1,7 +1,8 @@
 # REQUIRES: x86
 
-# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t.o
-# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %p/Inputs/deplibs-lib_foo.s -o %tfoo.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o
+# RUN: echo ".global foo; foo:" | \
+# RUN:   llvm-mc -filetype=obj -triple=x86_64 - -o %tfoo.o
 # RUN: rm -rf %t.dir
 # RUN: mkdir -p %t.dir
 # RUN: llvm-ar rc %t.dir/foo.a %tfoo.o
diff --git a/lld/test/ELF/deplibs.s b/lld/test/ELF/deplibs.s
index a0023163e8211..982ee3724c910 100644
--- a/lld/test/ELF/deplibs.s
+++ b/lld/test/ELF/deplibs.s
@@ -1,56 +1,74 @@
 # REQUIRES: x86
 
-# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t.o
-# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %p/Inputs/deplibs-lib_foo.s -o %tfoo.o
-# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %p/Inputs/deplibs-lib_bar.s -o %tbar.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o
+# RUN: echo ".global foo; foo:" | \
+# RUN:   llvm-mc -filetype=obj -triple=x86_64 - -o %tfoo.o
 # RUN: rm -rf %t.dir %t.cwd
 # RUN: mkdir -p %t.dir
 
-# error if dependent libraries cannot be found
+## Error if dependent libraries cannot be found.
 # RUN: not ld.lld %t.o -o /dev/null 2>&1 | FileCheck %s -DOBJ=%t.o --check-prefix MISSING
-# MISSING: error: [[OBJ]]: unable to find library from dependent library specifier: foo.a
+# MISSING:      error: [[OBJ]]: unable to find library from dependent library specifier: foo.a
 # MISSING-NEXT: error: [[OBJ]]: unable to find library from dependent library specifier: bar
 
-# can ignore dependent libraries
+## Can ignore dependent libraries.
 # RUN: not ld.lld %t.o -o /dev/null --no-dependent-libraries 2>&1 | FileCheck %s --check-prefix IGNORE
 # IGNORE: error: undefined symbol: foo
-# IGNORE: error: undefined symbol: bar
 
-# -r links preserve dependent libraries
+## -r links preserve dependent libraries.
 # RUN: ld.lld %t.o %t.o -r -o %t-r.o
 # RUN: not ld.lld %t-r.o -o /dev/null 2>&1 | sort | FileCheck %s -DOBJ=%t-r.o --check-prefixes MINUSR
-# MINUSR: error: [[OBJ]]: unable to find library from dependent library specifier: bar
+# MINUSR:      error: [[OBJ]]: unable to find library from dependent library specifier: bar
 # MINUSR-NEXT: error: [[OBJ]]: unable to find library from dependent library specifier: foo.a
-# MINUSR-NOT: unable to find library from dependent library specifier
+# MINUSR-NOT:  unable to find library from dependent library specifier
 
-# static archives located relative to library search paths
-# RUN: llvm-ar rc %t.dir/foo.a %tfoo.o
-# RUN: llvm-ar rc %t.dir/libbar.a %tbar.o
-# RUN: ld.lld %t.o -o /dev/null -L %t.dir
-
-# shared objects located relative to library search paths
-# RUN: rm %t.dir/libbar.a
-# RUN: ld.lld -shared -o %t.dir/libbar.so %tbar.o
-# RUN: ld.lld -Bdynamic %t.o -o /dev/null -L %t.dir
-
-# dependent libraries searched for symbols after libraries on the command line
+## Dependent libraries searched for symbols after libraries on the command line.
 # RUN: mkdir -p %t.cwd
 # RUN: cd %t.cwd
+# RUN: llvm-ar rc %t.dir/foo.a %tfoo.o
+# RUN: touch %t.dir/libbar.so
 # RUN: cp %t.dir/foo.a %t.cwd/libcmdline.a
-# RUN: ld.lld %t.o libcmdline.a -o /dev/null -L %t.dir --trace 2>&1 | FileCheck %s -DOBJ=%t.o -DSO=%t.dir --check-prefix CMDLINE --implicit-check-not foo.a
-# CMDLINE: [[OBJ]]
+# RUN: ld.lld %t.o libcmdline.a -o /dev/null -L %t.dir --trace 2>&1 | \
+# RUN:   FileCheck %s -DOBJ=%t.o -DSO=%t.dir --check-prefix CMDLINE \
+# RUN:                --implicit-check-not=foo.a --implicit-check-not=libbar.so
+# CMDLINE:      [[OBJ]]
 # CMDLINE-NEXT: {{^libcmdline\.a}}
-# CMDLINE-NEXT: [[SO]]{{[\\/]}}libbar.so
 
-# libraries can be found from specifiers as if the specifiers were listed on on the command-line.
+## LLD tries to resolve dependent library specifiers in the following order:
+##   1) The literal name in the current working directory.
+##   2) The literal name in a library search path.
+##   3) The name, prefixed with "lib" and suffixed with ".so" or ".a" in a
+##      library search path. This means that a directive of "foo.a" could lead
+##      to a library named "libfoo.a.a" being linked in.
+## When using library search paths for dependent libraries, LLD follows the same
+## rules as for libraries specified on the command line.
 # RUN: cp %t.dir/foo.a %t.cwd/foo.a
-# RUN: ld.lld %t.o -o /dev/null -L %t.dir --trace 2>&1 | FileCheck %s -DOBJ=%t.o -DSO=%t.dir --check-prefix ASIFCMDLINE --implicit-check-not foo.a
-# ASIFCMDLINE: [[OBJ]]
-# ASIFCMDLINE-NEXT: {{^foo\.a}}
-# ASIFCMDLINE-NEXT: [[SO]]{{[\\/]}}libbar.so
+# RUN: cp %t.dir/foo.a %t.dir/libfoo.a.a
+# RUN: ld.lld %t.o -o /dev/null -L %t.dir --trace 2>&1 | \
+# RUN:   FileCheck %s -DOBJ=%t.o -DDIR=%t.dir --check-prefix=CWD \
+# RUN:                --implicit-check-not=foo.a --implicit-check-not=libbar.so
+
+# CWD:      [[OBJ]]
+# CWD-NEXT: {{^foo\.a}}
+
+# RUN: rm %t.cwd/foo.a
+# RUN: ld.lld %t.o -o /dev/null -L %t.dir --trace 2>&1 | \
+# RUN:   FileCheck %s -DOBJ=%t.o -DDIR=%t.dir --check-prefix=PLAIN-DIR \
+# RUN:                --implicit-check-not=foo.a --implicit-check-not=libbar.so
+
+# PLAIN-DIR:      [[OBJ]]
+# PLAIN-DIR-NEXT: [[DIR]]{{[\\/]}}foo.a
+
+# RUN: rm %t.dir/foo.a
+# RUN: ld.lld %t.o -o /dev/null -L %t.dir --trace 2>&1 | \
+# RUN:   FileCheck %s -DOBJ=%t.o -DDIR=%t.dir --check-prefix=LIBA-DIR \
+# RUN:                --implicit-check-not=foo.a --implicit-check-not=libbar.so
+
+# LIBA-DIR:      [[OBJ]]
+# LIBA-DIR-NEXT: [[DIR]]{{[\\/]}}libfoo.a.a
 
     call foo
-    call bar
 .section ".deplibs","MS",@llvm_dependent_libraries,1
-    .asciz  "foo.a"
-    .asciz  "bar"
+    .asciz "foo.a"
+    ## Show that an unneeded archive must be present but may not be linked in.
+    .asciz "bar"
diff --git a/lld/test/ELF/executable-undefined-ignoreall.s b/lld/test/ELF/executable-undefined-ignoreall.s
index a479317c0f71c..cc38e17cdf619 100644
--- a/lld/test/ELF/executable-undefined-ignoreall.s
+++ b/lld/test/ELF/executable-undefined-ignoreall.s
@@ -1,7 +1,7 @@
 # REQUIRES: x86
 
 ## --unresolved-symbols=ignore-all behaves similar to -shared:
-## for PLT relocations to undefined symbols, produce dynamic reloctions if we
+## for PLT relocations to undefined symbols, produce dynamic relocations if we
 ## emit .dynsym.
 
 # RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t.o
diff --git a/lld/test/ELF/gnu-ifunc-i386.s b/lld/test/ELF/gnu-ifunc-i386.s
index 8023872da161d..60c16f54e4f11 100644
--- a/lld/test/ELF/gnu-ifunc-i386.s
+++ b/lld/test/ELF/gnu-ifunc-i386.s
@@ -113,8 +113,8 @@
 // DISASM: <bar_resolver>:
 // DISASM-NEXT:   4010e5:       retl
 // DISASM:      <_start>:
-// DISASM-NEXT:   4010e6:       calll 21 <foo>
-// DISASM-NEXT:                 calll 32 <bar>
+// DISASM-NEXT:   4010e6:       calll 0x401100 <foo>
+// DISASM-NEXT:                 calll 0x401110 <bar>
 // DISASM-NEXT:                 movl $4194516, %edx
 // DISASM-NEXT:                 movl $4194532, %edx
 // DISASM-EMPTY:
@@ -123,11 +123,11 @@
 // DISASM-NEXT: <foo>:
 // DISASM-NEXT:   401100:       jmpl *4202784
 // DISASM-NEXT:                 pushl $0
-// DISASM-NEXT:                 jmp -4198672
+// DISASM-NEXT:                 jmp 0x0
 // DISASM:      <bar>:
 // DISASM-NEXT:   401110:       jmpl *4202788
 // DISASM-NEXT:                 pushl $8
-// DISASM-NEXT:                 jmp -4198688
+// DISASM-NEXT:                 jmp 0x0
 
 .text
 .type foo STT_GNU_IFUNC
diff --git a/lld/test/ELF/gnu-ifunc-noplt-i386.s b/lld/test/ELF/gnu-ifunc-noplt-i386.s
index 0b08912996ff3..324ef95fd8917 100644
--- a/lld/test/ELF/gnu-ifunc-noplt-i386.s
+++ b/lld/test/ELF/gnu-ifunc-noplt-i386.s
@@ -27,10 +27,10 @@
 // DISASM-NEXT:   retl
 // DISASM-EMPTY:
 // DISASM-NEXT: 004011ee <_start>:
-// DISASM-NEXT:   calll	-0x4 <_start+0x1>
-// DISASM-NEXT:   calll	-0x4 <_start+0x6>
-// DISASM-NEXT:   calll	0x23 <bar2@plt>
-// DISASM-NEXT:   calll	0x2e <zed2@plt>
+// DISASM-NEXT:   calll	0x4011ef <_start+0x1>
+// DISASM-NEXT:   calll	0x4011f4 <_start+0x6>
+// DISASM-NEXT:   calll	0x401220 <bar2@plt>
+// DISASM-NEXT:   calll	0x401230 <zed2@plt>
 // DISASM-EMPTY:
 // DISASM-NEXT: Disassembly of section .plt:
 // DISASM-EMPTY:
@@ -45,12 +45,12 @@
 // DISASM-NEXT: 00401220 <bar2@plt>:
 // DISASM-NEXT:   jmpl	*0x4032d4
 // DISASM-NEXT:   pushl	$0x0
-// DISASM-NEXT:   jmp	-0x20 <.plt>
+// DISASM-NEXT:   jmp	0x401210 <.plt>
 // DISASM-EMPTY:
 // DISASM-NEXT: 00401230 <zed2@plt>:
 // DISASM-NEXT:   jmpl	*0x4032d8
 // DISASM-NEXT:   pushl	$0x8
-// DISASM-NEXT:   jmp	-0x30 <.plt>
+// DISASM-NEXT:   jmp	0x401210 <.plt>
 
 .text
 .type foo STT_GNU_IFUNC
diff --git a/lld/test/ELF/gnu-ifunc-noplt.s b/lld/test/ELF/gnu-ifunc-noplt.s
index db9edc20b9915..80fcbce44a044 100644
--- a/lld/test/ELF/gnu-ifunc-noplt.s
+++ b/lld/test/ELF/gnu-ifunc-noplt.s
@@ -5,6 +5,12 @@
 // RUN: llvm-mc -filetype=obj -triple=x86_64-pc-freebsd %S/Inputs/shared2-x86-64.s -o %t1.o
 // RUN: ld.lld %t1.o --shared -soname=so -o %t.so
 // RUN: llvm-mc -filetype=obj -triple=x86_64-pc-freebsd %s -o %t.o
+
+/// The default -z text is not compatible with -z ifunc-noplt.
+// RUN: not ld.lld -z ifunc-noplt %t.o -o /dev/null 2>&1| FileCheck --check-prefix=INCOMPATIBLE %s
+// RUN: not ld.lld -z ifunc-noplt -z text %t.o -o /dev/null 2>&1| FileCheck --check-prefix=INCOMPATIBLE %s
+// INCOMPATIBLE: -z text and -z ifunc-noplt may not be used together
+
 // RUN: ld.lld -z ifunc-noplt -z notext --hash-style=sysv %t.so %t.o -o %tout
 // RUN: llvm-objdump -d --no-show-raw-insn %tout | FileCheck %s --check-prefix=DISASM
 // RUN: llvm-readobj -r --dynamic-table %tout | FileCheck %s
@@ -30,10 +36,10 @@
 // DISASM-NEXT:   201321:      	retq
 // DISASM-EMPTY:
 // DISASM-NEXT: 0000000000201322 <_start>:
-// DISASM-NEXT:   201322:      	callq	0 <_start+0x5>
-// DISASM-NEXT:   201327:      	callq	0 <_start+0xa>
-// DISASM-NEXT:   20132c:      	callq	31 <bar2@plt>
-// DISASM-NEXT:   201331:      	callq	42 <zed2@plt>
+// DISASM-NEXT:   201322:      	callq	0x201327 <_start+0x5>
+// DISASM-NEXT:   201327:      	callq	0x20132c <_start+0xa>
+// DISASM-NEXT:   20132c:      	callq	0x201350 <bar2@plt>
+// DISASM-NEXT:   201331:      	callq	0x201360 <zed2@plt>
 // DISASM-EMPTY:
 // DISASM-NEXT: Disassembly of section .plt:
 // DISASM-EMPTY:
@@ -45,12 +51,12 @@
 // DISASM-NEXT: 0000000000201350 <bar2@plt>:
 // DISASM-NEXT:   201350:      	jmpq	*8514(%rip)
 // DISASM-NEXT:   201356:      	pushq	$0
-// DISASM-NEXT:   20135b:      	jmp	-32 <.plt>
+// DISASM-NEXT:   20135b:      	jmp	0x201340 <.plt>
 // DISASM-EMPTY:
 // DISASM-NEXT: 0000000000201360 <zed2@plt>:
 // DISASM-NEXT:   201360:      	jmpq	*8506(%rip)
 // DISASM-NEXT:   201366:      	pushq	$1
-// DISASM-NEXT:   20136b:      	jmp	-48 <.plt>
+// DISASM-NEXT:   20136b:      	jmp	0x201340 <.plt>
 
 .text
 .type foo STT_GNU_IFUNC
diff --git a/lld/test/ELF/gnu-ifunc-plt-i386.s b/lld/test/ELF/gnu-ifunc-plt-i386.s
index 28b039f145fd6..90a21a212eb76 100644
--- a/lld/test/ELF/gnu-ifunc-plt-i386.s
+++ b/lld/test/ELF/gnu-ifunc-plt-i386.s
@@ -37,10 +37,10 @@
 // DISASM:      <bar>:
 // DISASM-NEXT:    4011b5:       retl
 // DISASM:      <_start>:
-// DISASM-NEXT:    4011b6:       calll   69 <zed2+0x401200>
-// DISASM-NEXT:                  calll   80 <zed2+0x401210>
-// DISASM-NEXT:                  calll   27 <bar2@plt>
-// DISASM-NEXT:                  calll   38 <zed2@plt>
+// DISASM-NEXT:    4011b6:       calll   0x401200 <zed2+0x401200>
+// DISASM-NEXT:                  calll   0x401210 <zed2+0x401210>
+// DISASM-NEXT:                  calll   0x4011e0 <bar2@plt>
+// DISASM-NEXT:                  calll   0x4011f0 <zed2@plt>
 // DISASM-EMPTY:
 // DISASM-NEXT: Disassembly of section .plt:
 // DISASM-EMPTY:
@@ -55,22 +55,22 @@
 // DISASM-NEXT:   <bar2@plt>:
 // DISASM-NEXT:    4011e0:       jmpl    *4207268
 // DISASM-NEXT:                  pushl   $0
-// DISASM-NEXT:                  jmp     -32 <.plt>
+// DISASM-NEXT:                  jmp     0x4011d0 <.plt>
 // DISASM-EMPTY:
 // DISASM-NEXT:   <zed2@plt>:
 // DISASM-NEXT:    4011f0:       jmpl    *4207272
 // DISASM-NEXT:                  pushl   $8
-// DISASM-NEXT:                  jmp     -48 <.plt>
+// DISASM-NEXT:                  jmp     0x4011d0 <.plt>
 // DISASM-EMPTY:
 // DISASM-NEXT: Disassembly of section .iplt:
 // DISASM-EMPTY:
 // DISASM-NEXT: <.iplt>:
 // DISASM-NEXT:                  jmpl    *4207276
 // DISASM-NEXT:                  pushl   $0
-// DISASM-NEXT:                  jmp     -64 <.plt>
+// DISASM-NEXT:                  jmp     0x4011d0 <.plt>
 // DISASM-NEXT:                  jmpl    *4207280
 // DISASM-NEXT:                  pushl   $8
-// DISASM-NEXT:                  jmp     -80 <.plt>
+// DISASM-NEXT:                  jmp     0x4011d0 <.plt>
 
 .text
 .type foo STT_GNU_IFUNC
diff --git a/lld/test/ELF/gnu-ifunc-plt.s b/lld/test/ELF/gnu-ifunc-plt.s
index d23a2b21b4d3f..b8b1f39a18fb4 100644
--- a/lld/test/ELF/gnu-ifunc-plt.s
+++ b/lld/test/ELF/gnu-ifunc-plt.s
@@ -41,8 +41,8 @@
 // DISASM:      <bar>:
 // DISASM-NEXT:   2012d9:       retq
 // DISASM:      <_start>:
-// DISASM-NEXT:   2012da:       callq   65
-// DISASM-NEXT:   2012df:       callq   76
+// DISASM-NEXT:   2012da:       callq   0x201320
+// DISASM-NEXT:   2012df:       callq   0x201330
 // DISASM-NEXT:                 callq   {{.*}} <bar2@plt>
 // DISASM-NEXT:                 callq   {{.*}} <zed2@plt>
 // DISASM-EMPTY:
@@ -56,22 +56,22 @@
 // DISASM-NEXT:   <bar2@plt>:
 // DISASM-NEXT:   201300:       jmpq    *8514(%rip)
 // DISASM-NEXT:   201306:       pushq   $0
-// DISASM-NEXT:   20130b:       jmp     -32 <.plt>
+// DISASM-NEXT:   20130b:       jmp     0x2012f0 <.plt>
 // DISASM-EMPTY:
 // DISASM-NEXT:   <zed2@plt>:
 // DISASM-NEXT:   201310:       jmpq    *8506(%rip)
 // DISASM-NEXT:   201316:       pushq   $1
-// DISASM-NEXT:   20131b:       jmp     -48 <.plt>
+// DISASM-NEXT:   20131b:       jmp     0x2012f0 <.plt>
 // DISASM-EMPTY:
 // DISASM-NEXT: Disassembly of section .iplt:
 // DISASM-EMPTY:
 // DISASM-NEXT: <.iplt>:
 // DISASM-NEXT:   201320:       jmpq    *8498(%rip)
 // DISASM-NEXT:   201326:       pushq   $0
-// DISASM-NEXT:   20132b:       jmp     -64 <.plt>
+// DISASM-NEXT:   20132b:       jmp     0x2012f0 <.plt>
 // DISASM-NEXT:   201330:       jmpq    *8490(%rip)
 // DISASM-NEXT:   201336:       pushq   $1
-// DISASM-NEXT:   20133b:       jmp     -80 <.plt>
+// DISASM-NEXT:   20133b:       jmp     0x2012f0 <.plt>
 
 .text
 .type foo STT_GNU_IFUNC
diff --git a/lld/test/ELF/gnu-ifunc-shared.s b/lld/test/ELF/gnu-ifunc-shared.s
index 9a2e8beb9f6bc..c88b7b11b7fa6 100644
--- a/lld/test/ELF/gnu-ifunc-shared.s
+++ b/lld/test/ELF/gnu-ifunc-shared.s
@@ -16,9 +16,9 @@
 // DISASM:     <fct2>:
 // DISASM-NEXT:     1309:       retq
 // DISASM:     <f1>:
-// DISASM-NEXT:     130a:       callq   65
-// DISASM-NEXT:     130f:       callq   28
-// DISASM-NEXT:     1314:       callq   39
+// DISASM-NEXT:     130a:       callq   0x1350
+// DISASM-NEXT:     130f:       callq   0x1330
+// DISASM-NEXT:     1314:       callq   0x1340
 // DISASM-NEXT:     1319:       retq
 // DISASM:     <f2>:
 // DISASM-NEXT:     131a:       retq
@@ -33,18 +33,18 @@
 // DISASM-NEXT:   <fct2@plt>:
 // DISASM-NEXT:     1330:       jmpq    *8482(%rip)
 // DISASM-NEXT:     1336:       pushq   $0
-// DISASM-NEXT:     133b:       jmp     -32 <.plt>
+// DISASM-NEXT:     133b:       jmp     0x1320 <.plt>
 // DISASM-EMPTY:
 // DISASM-NEXT:   <f2@plt>:
 // DISASM-NEXT:     1340:       jmpq    *8474(%rip)
 // DISASM-NEXT:     1346:       pushq   $1
-// DISASM-NEXT:     134b:       jmp     -48 <.plt>
+// DISASM-NEXT:     134b:       jmp     0x1320 <.plt>
 // DISASM:      Disassembly of section .iplt:
 // DISASM-EMPTY:
 // DISASM:      <.iplt>:
 // DISASM-NEXT:     1350:       jmpq    *8466(%rip)
 // DISASM-NEXT:     1356:       pushq   $0
-// DISASM-NEXT:     135b:       jmp     -64 <.plt>
+// DISASM-NEXT:     135b:       jmp     0x1320 <.plt>
 
 // CHECK: Relocations [
 // CHECK-NEXT:   Section (5) .rela.dyn {
diff --git a/lld/test/ELF/gnu-ifunc.s b/lld/test/ELF/gnu-ifunc.s
index 60955336d1d29..c43bef28b2e24 100644
--- a/lld/test/ELF/gnu-ifunc.s
+++ b/lld/test/ELF/gnu-ifunc.s
@@ -96,8 +96,8 @@
 // DISASM:      <bar>:
 // DISASM-NEXT:  201189: {{.*}} retq
 // DISASM:      <_start>:
-// DISASM-NEXT:  20118a: {{.*}} callq 33
-// DISASM-NEXT:  20118f: {{.*}} callq 44
+// DISASM-NEXT:  20118a: {{.*}} callq 0x2011b0
+// DISASM-NEXT:  20118f: {{.*}} callq 0x2011c0
 // DISASM-NEXT:  201194: {{.*}} movl $2097496, %edx
 // DISASM-NEXT:  201199: {{.*}} movl $2097544, %edx
 // DISASM-NEXT:  20119e: {{.*}} movl $2097545, %edx
@@ -107,10 +107,10 @@
 // DISASM-NEXT: <.iplt>:
 // DISASM-NEXT:  2011b0: {{.*}} jmpq *4122(%rip)
 // DISASM-NEXT:  2011b6: {{.*}} pushq $0
-// DISASM-NEXT:  2011bb: {{.*}} jmp -2101696
+// DISASM-NEXT:  2011bb: {{.*}} jmp 0x0
 // DISASM-NEXT:  2011c0: {{.*}} jmpq *4114(%rip)
 // DISASM-NEXT:  2011c6: {{.*}} pushq $1
-// DISASM-NEXT:  2011cb: {{.*}} jmp -2101712
+// DISASM-NEXT:  2011cb: {{.*}} jmp 0x0
 
 .text
 .type foo STT_GNU_IFUNC
diff --git a/lld/test/ELF/hexagon-tls-gd-nonpreemptible.s b/lld/test/ELF/hexagon-tls-gd-nonpreemptible.s
new file mode 100644
index 0000000000000..ba0eee999c806
--- /dev/null
+++ b/lld/test/ELF/hexagon-tls-gd-nonpreemptible.s
@@ -0,0 +1,39 @@
+
+# REQUIRES: hexagon
+# RUN: llvm-mc -filetype=obj -triple=hexagon-unknown-elf %s -o %t.o
+# RUN: ld.lld -shared %t.o -o %t.so
+# RUN: llvm-readobj -r %t.so | FileCheck --check-prefix=RELOC %s
+# RUN: llvm-objdump -d --no-show-raw-insn --print-imm-hex %t.so | FileCheck %s
+
+## Prior to D77021 lld would error "relocation R_HEX_GD_PLT_B22_PCREL cannot refer to absolute symbol".
+## A PC-relative relocation referencing a non-preemptible absolute symbol (due to STT_TLS) is not representable in -pie/-shared mode.
+## For this case we will actually patch the symbol to the external __tls_get_addr which is preemptible.
+
+.globl _start
+.type _start, @function
+
+# RELOC:      Section ({{.*}}) .rela.plt {
+# RELOC-NEXT:   R_HEX_JMP_SLOT - 0x0
+# RELOC-NEXT:   R_HEX_JMP_SLOT __tls_get_addr 0x0
+# RELOC-NEXT: }
+
+# CHECK:      { immext(#{{.*}})
+# CHECK-NEXT:   r2 = add(pc,##{{.*}}) }
+# CHECK-NEXT: { immext(#{{.*}})
+# CHECK-NEXT:   r0 = add(r2,##-{{.*}}) }
+# CHECK-NEXT: { call {{.*}} }
+# CHECK-NEXT: { r0 = memw(r0+#0x0) }
+
+_start:
+  r2 = add(pc,##_GLOBAL_OFFSET_TABLE_@PCREL)
+  r0 = add(r2,##a@GDGOT)
+  call a@GDPLT
+  r0 = memw(r0+#0)
+
+## a is non-preemptible due to STV_HIDDEN visibility.
+## We can achieve the same effect with -Bsymbolic.
+.section        .tdata,"awT",@progbits
+.globl  a
+.hidden a
+a:
+.word 1
diff --git a/lld/test/ELF/i386-feature-cet.s b/lld/test/ELF/i386-feature-cet.s
index b4073a40232e5..25a44ee6e52da 100644
--- a/lld/test/ELF/i386-feature-cet.s
+++ b/lld/test/ELF/i386-feature-cet.s
@@ -41,8 +41,8 @@
 
 # DISASM:      Disassembly of section .text:
 # DISASM:      00401200 <func1>:
-# DISASM-NEXT: 401200:       calll   0x2b <func2+0x401230>
-# DISASM-NEXT: 401205:       calll   0x36 <ifunc>
+# DISASM-NEXT: 401200:       calll   0x401230 <func2+0x401230>
+# DISASM-NEXT: 401205:       calll   0x401240 <ifunc>
 # DISASM-NEXT:               retl
 
 # DISASM:      Disassembly of section .plt:
@@ -55,7 +55,7 @@
 # DISASM-NEXT:               nop
 # DISASM-NEXT:               endbr32
 # DISASM-NEXT:               pushl   $0x0
-# DISASM-NEXT:               jmp     -0x1e <.plt>
+# DISASM-NEXT:               jmp     0x401210 <.plt>
 # DISASM-NEXT:               nop
 
 # DISASM:      Disassembly of section .plt.sec:
diff --git a/lld/test/ELF/i386-plt.s b/lld/test/ELF/i386-plt.s
index c8f66c69a730d..bad417851f539 100644
--- a/lld/test/ELF/i386-plt.s
+++ b/lld/test/ELF/i386-plt.s
@@ -52,10 +52,10 @@
 // DISASM-NEXT:  4011bc:
 // DISASM-NEXT:  4011be:
 // DISASM:       <_start>:
-// DISASM-NEXT: 4011c0:       jmp 0x2b <bar@plt>
-// DISASM-NEXT: 4011c5:       jmp 0x26 <bar@plt>
-// DISASM-NEXT: 4011ca:       jmp 0x31 <zed@plt>
-// DISASM-NEXT: 4011cf:       jmp -0x18 <local>
+// DISASM-NEXT: 4011c0:       jmp 0x4011f0 <bar@plt>
+// DISASM-NEXT: 4011c5:       jmp 0x4011f0 <bar@plt>
+// DISASM-NEXT: 4011ca:       jmp 0x401200 <zed@plt>
+// DISASM-NEXT: 4011cf:       jmp 0x4011bc <local>
 
 // DISASM:      Disassembly of section .plt:
 // DISASM-EMPTY:
@@ -72,13 +72,13 @@
 /// .got.plt[3] = 0x403278 + 12 = 0x403284
 // DISASM-NEXT: 4011f0:       jmpl *0x403284
 // DISASM-NEXT:               pushl $0x0
-// DISASM-NEXT:               jmp -0x20 <.plt>
+// DISASM-NEXT:               jmp 0x4011e0 <.plt>
 // DISASM-EMPTY:
 // DISASM-NEXT: <zed@plt>:
 /// .got.plt[4] = 0x403278 + 16 = 0x403288
 // DISASM-NEXT: 401200:       jmpl *0x403288
 // DISASM-NEXT:               pushl $0x8
-// DISASM-NEXT:               jmp -0x30 <.plt>
+// DISASM-NEXT:               jmp 0x4011e0 <.plt>
 
 // CHECKSHARED:        Name: .plt
 // CHECKSHARED-NEXT:   Type: SHT_PROGBITS
@@ -122,10 +122,10 @@
 // DISASMSHARED-NEXT: 11e0:
 // DISASMSHARED-NEXT: 11e2:
 // DISASMSHARED:      <_start>:
-// DISASMSHARED-NEXT: 11e4:       jmp 39 <bar@plt>
-// DISASMSHARED-NEXT:             jmp 34 <bar@plt>
-// DISASMSHARED-NEXT:             jmp 45 <zed@plt>
-// DISASMSHARED-NEXT:             jmp -24 <local>
+// DISASMSHARED-NEXT: 11e4:       jmp 0x1210 <bar@plt>
+// DISASMSHARED-NEXT:             jmp 0x1210 <bar@plt>
+// DISASMSHARED-NEXT:             jmp 0x1220 <zed@plt>
+// DISASMSHARED-NEXT:             jmp 0x11e0 <local>
 // DISASMSHARED-EMPTY:
 // DISASMSHARED-NEXT: Disassembly of section .plt:
 // DISASMSHARED-EMPTY:
@@ -139,11 +139,11 @@
 // DISASMSHARED:      <bar@plt>:
 // DISASMSHARED-NEXT: 1210:       jmpl *12(%ebx)
 // DISASMSHARED-NEXT:             pushl $0
-// DISASMSHARED-NEXT:             jmp -32 <.plt>
+// DISASMSHARED-NEXT:             jmp 0x1200 <.plt>
 // DISASMSHARED:      <zed@plt>:
 // DISASMSHARED-NEXT: 1220:       jmpl *16(%ebx)
 // DISASMSHARED-NEXT:             pushl $8
-// DISASMSHARED-NEXT:             jmp -48 <.plt>
+// DISASMSHARED-NEXT:             jmp 0x1200 <.plt>
 
 // DISASMPIE:      Disassembly of section .plt:
 // DISASMPIE-EMPTY:
@@ -158,12 +158,12 @@
 // DISASMPIE-NEXT: <bar@plt>:
 // DISASMPIE-NEXT: 11f0:       jmpl *12(%ebx)
 // DISASMPIE-NEXT:             pushl $0
-// DISASMPIE-NEXT:             jmp -32 <.plt>
+// DISASMPIE-NEXT:             jmp 0x11e0 <.plt>
 // DISASMPIE-EMPTY:
 // DISASMPIE-NEXT: <zed@plt>:
 // DISASMPIE-NEXT: 1200:       jmpl *16(%ebx)
 // DISASMPIE-NEXT:             pushl $8
-// DISASMPIE-NEXT:             jmp -48 <.plt>
+// DISASMPIE-NEXT:             jmp 0x11e0 <.plt>
 
 local:
 .long 0
diff --git a/lld/test/ELF/i386-reloc-large-addend.s b/lld/test/ELF/i386-reloc-large-addend.s
index 7d4816dc0a6f3..bcdc99115b7a7 100644
--- a/lld/test/ELF/i386-reloc-large-addend.s
+++ b/lld/test/ELF/i386-reloc-large-addend.s
@@ -10,7 +10,7 @@
 // CHECK:        Disassembly of section .text:
 // CHECK-EMPTY:
 // CHECK-NEXT: <_start>:
-// CHECK-NEXT:     7000:       e9 fe 1f        jmp     8190
+// CHECK-NEXT:     7000:       e9 fe 1f        jmp     0x9001
 //                            0x1 + 0x9000 - 0x7003 == 8190
         .global _start
 _start:
diff --git a/lld/test/ELF/i386-reloc-range.s b/lld/test/ELF/i386-reloc-range.s
index ffdae86f516ad..0c15e6a9e0554 100644
--- a/lld/test/ELF/i386-reloc-range.s
+++ b/lld/test/ELF/i386-reloc-range.s
@@ -12,7 +12,7 @@
 // CHECK:        Disassembly of section .text:
 // CHECK-EMPTY:
 // CHECK-NEXT: <_start>:
-// CHECK-NEXT:      200:       jmp -1
+// CHECK-NEXT:      200:       jmp 0x202
 //              0x10202 - 0x203 == 0xffff
 
 // RUN: not ld.lld -Ttext 0x200 %t.o %t2.o -o /dev/null 2>&1 | FileCheck --check-prefix=ERR %s
diff --git a/lld/test/ELF/i386-retpoline-nopic-linkerscript.s b/lld/test/ELF/i386-retpoline-nopic-linkerscript.s
index e883a409670cd..88e07c3c1544f 100644
--- a/lld/test/ELF/i386-retpoline-nopic-linkerscript.s
+++ b/lld/test/ELF/i386-retpoline-nopic-linkerscript.s
@@ -18,10 +18,10 @@
 // CHECK-NEXT: 10:       pushl   236
 // CHECK-NEXT: 16:       pushl   %eax
 // CHECK-NEXT: 17:       movl    240, %eax
-// CHECK-NEXT: 1c:       calll   15 <.plt+0x20>
+// CHECK-NEXT: 1c:       calll   0x30 <.plt+0x20>
 // CHECK-NEXT: 21:       pause
 // CHECK-NEXT: 23:       lfence
-// CHECK-NEXT: 26:       jmp     -7 <.plt+0x11>
+// CHECK-NEXT: 26:       jmp     0x21 <.plt+0x11>
 // CHECK-NEXT: 28:       int3
 // CHECK-NEXT: 29:       int3
 // CHECK-NEXT: 2a:       int3
@@ -39,10 +39,10 @@
 // CHECK-NEXT: 3f:       int3
 // CHECK-NEXT: 40:       pushl   %eax
 // CHECK-NEXT: 41:       movl    244, %eax
-// CHECK-NEXT: 46:       calll   -27 <.plt+0x20>
-// CHECK-NEXT: 4b:       jmp     -47 <.plt+0x11>
+// CHECK-NEXT: 46:       calll   0x30 <.plt+0x20>
+// CHECK-NEXT: 4b:       jmp     0x21 <.plt+0x11>
 // CHECK-NEXT: 50:       pushl   $0
-// CHECK-NEXT: 55:       jmp     -74 <.plt>
+// CHECK-NEXT: 55:       jmp     0x10 <.plt>
 // CHECK-NEXT: 5a:       int3
 // CHECK-NEXT: 5b:       int3
 // CHECK-NEXT: 5c:       int3
@@ -51,10 +51,10 @@
 // CHECK-NEXT: 5f:       int3
 // CHECK-NEXT: 60:       pushl   %eax
 // CHECK-NEXT: 61:       movl    248, %eax
-// CHECK-NEXT: 66:       calll   -59 <.plt+0x20>
-// CHECK-NEXT: 6b:       jmp     -79 <.plt+0x11>
+// CHECK-NEXT: 66:       calll   0x30 <.plt+0x20>
+// CHECK-NEXT: 6b:       jmp     0x21 <.plt+0x11>
 // CHECK-NEXT: 70:       pushl   $8
-// CHECK-NEXT: 75:       jmp     -106 <.plt>
+// CHECK-NEXT: 75:       jmp     0x10 <.plt>
 // CHECK-NEXT: 7a:       int3
 // CHECK-NEXT: 7b:       int3
 // CHECK-NEXT: 7c:       int3
diff --git a/lld/test/ELF/i386-retpoline-nopic.s b/lld/test/ELF/i386-retpoline-nopic.s
index e5eba67d3958d..170e1a251c3c2 100644
--- a/lld/test/ELF/i386-retpoline-nopic.s
+++ b/lld/test/ELF/i386-retpoline-nopic.s
@@ -16,10 +16,10 @@
 // CHECK-NEXT: 4011d0:       pushl   4207276
 // CHECK-NEXT: 4011d6:       pushl   %eax
 // CHECK-NEXT: 4011d7:       movl    4207280, %eax
-// CHECK-NEXT: 4011dc:       calll   15 <.plt+0x20>
+// CHECK-NEXT: 4011dc:       calll   0x4011f0 <.plt+0x20>
 // CHECK-NEXT: 4011e1:       pause
 // CHECK-NEXT: 4011e3:       lfence
-// CHECK-NEXT: 4011e6:       jmp     -7 <.plt+0x11>
+// CHECK-NEXT: 4011e6:       jmp     0x4011e1 <.plt+0x11>
 // CHECK-NEXT: 4011e8:       int3
 // CHECK-NEXT: 4011e9:       int3
 // CHECK-NEXT: 4011ea:       int3
@@ -37,10 +37,10 @@
 // CHECK-NEXT: 4011ff:       int3
 // CHECK-NEXT: 401200:       pushl   %eax
 // CHECK-NEXT: 401201:       movl    4207284, %eax
-// CHECK-NEXT: 401206:       calll   -27 <.plt+0x20>
-// CHECK-NEXT: 40120b:       jmp     -47 <.plt+0x11>
+// CHECK-NEXT: 401206:       calll   0x4011f0 <.plt+0x20>
+// CHECK-NEXT: 40120b:       jmp     0x4011e1 <.plt+0x11>
 // CHECK-NEXT: 401210:       pushl   $0
-// CHECK-NEXT: 401215:       jmp     -74 <.plt>
+// CHECK-NEXT: 401215:       jmp     0x4011d0 <.plt>
 // CHECK-NEXT: 40121a:       int3
 // CHECK-NEXT: 40121b:       int3
 // CHECK-NEXT: 40121c:       int3
@@ -49,10 +49,10 @@
 // CHECK-NEXT: 40121f:       int3
 // CHECK-NEXT: 401220:       pushl   %eax
 // CHECK-NEXT: 401221:       movl    4207288, %eax
-// CHECK-NEXT: 401226:       calll   -59 <.plt+0x20>
-// CHECK-NEXT: 40122b:       jmp     -79 <.plt+0x11>
+// CHECK-NEXT: 401226:       calll   0x4011f0 <.plt+0x20>
+// CHECK-NEXT: 40122b:       jmp     0x4011e1 <.plt+0x11>
 // CHECK-NEXT: 401230:       pushl   $8
-// CHECK-NEXT: 401235:       jmp     -106 <.plt>
+// CHECK-NEXT: 401235:       jmp     0x4011d0 <.plt>
 // CHECK-NEXT: 40123a:       int3
 // CHECK-NEXT: 40123b:       int3
 // CHECK-NEXT: 40123c:       int3
diff --git a/lld/test/ELF/i386-retpoline-pic-linkerscript.s b/lld/test/ELF/i386-retpoline-pic-linkerscript.s
index 5db492bd733ee..f14c47b52dd68 100644
--- a/lld/test/ELF/i386-retpoline-pic-linkerscript.s
+++ b/lld/test/ELF/i386-retpoline-pic-linkerscript.s
@@ -18,10 +18,10 @@
 // CHECK-NEXT: 10:       ff b3 04 00 00 00       pushl   4(%ebx)
 // CHECK-NEXT: 16:       50      pushl   %eax
 // CHECK-NEXT: 17:       8b 83 08 00 00 00 movl    8(%ebx), %eax
-// CHECK-NEXT: 1d:       e8 0e 00 00 00  calll   14 <.plt+0x20>
+// CHECK-NEXT: 1d:       e8 0e 00 00 00  calll   0x30 <.plt+0x20>
 // CHECK-NEXT: 22:       f3 90   pause
 // CHECK-NEXT: 24:       0f ae e8        lfence
-// CHECK-NEXT: 27:       eb f9   jmp     -7 <.plt+0x12>
+// CHECK-NEXT: 27:       eb f9   jmp     0x22 <.plt+0x12>
 // CHECK-NEXT: 29:       cc      int3
 // CHECK-NEXT: 2a:       cc      int3
 // CHECK-NEXT: 2b:       cc      int3
@@ -38,10 +38,10 @@
 // CHECK-NEXT: 3f:       cc      int3
 // CHECK-NEXT: 40:       50      pushl   %eax
 // CHECK-NEXT: 41:       8b 83 0c 00 00 00       movl    12(%ebx), %eax
-// CHECK-NEXT: 47:       e8 e4 ff ff ff  calll   -28 <.plt+0x20>
-// CHECK-NEXT: 4c:       e9 d1 ff ff ff  jmp     -47 <.plt+0x12>
+// CHECK-NEXT: 47:       e8 e4 ff ff ff  calll   0x30 <.plt+0x20>
+// CHECK-NEXT: 4c:       e9 d1 ff ff ff  jmp     0x22 <.plt+0x12>
 // CHECK-NEXT: 51:       68 00 00 00 00  pushl   $0
-// CHECK-NEXT: 56:       e9 b5 ff ff ff  jmp     -75 <.plt>
+// CHECK-NEXT: 56:       e9 b5 ff ff ff  jmp     0x10 <.plt>
 // CHECK-NEXT: 5b:       cc      int3
 // CHECK-NEXT: 5c:       cc      int3
 // CHECK-NEXT: 5d:       cc      int3
@@ -49,10 +49,10 @@
 // CHECK-NEXT: 5f:       cc      int3
 // CHECK-NEXT: 60:       50      pushl   %eax
 // CHECK-NEXT: 61:       8b 83 10 00 00 00       movl    16(%ebx), %eax
-// CHECK-NEXT: 67:       e8 c4 ff ff ff  calll   -60 <.plt+0x20>
-// CHECK-NEXT: 6c:       e9 b1 ff ff ff  jmp     -79 <.plt+0x12>
+// CHECK-NEXT: 67:       e8 c4 ff ff ff  calll   0x30 <.plt+0x20>
+// CHECK-NEXT: 6c:       e9 b1 ff ff ff  jmp     0x22 <.plt+0x12>
 // CHECK-NEXT: 71:       68 08 00 00 00  pushl   $8
-// CHECK-NEXT: 76:       e9 95 ff ff ff  jmp     -107 <.plt>
+// CHECK-NEXT: 76:       e9 95 ff ff ff  jmp     0x10 <.plt>
 // CHECK-NEXT: 7b:       cc      int3
 // CHECK-NEXT: 7c:       cc      int3
 // CHECK-NEXT: 7d:       cc      int3
diff --git a/lld/test/ELF/i386-retpoline-pic.s b/lld/test/ELF/i386-retpoline-pic.s
index e5e24da6e3f6b..b6a01243c175e 100644
--- a/lld/test/ELF/i386-retpoline-pic.s
+++ b/lld/test/ELF/i386-retpoline-pic.s
@@ -16,10 +16,10 @@
 // CHECK-NEXT: 11d0:       pushl   4(%ebx)
 // CHECK-NEXT: 11d6:       pushl   %eax
 // CHECK-NEXT: 11d7:       movl    8(%ebx), %eax
-// CHECK-NEXT: 11dd:       calll   14 <.plt+0x20>
+// CHECK-NEXT: 11dd:       calll   0x11f0 <.plt+0x20>
 // CHECK-NEXT: 11e2:       pause
 // CHECK-NEXT: 11e4:       lfence
-// CHECK-NEXT: 11e7:       jmp     -7 <.plt+0x12>
+// CHECK-NEXT: 11e7:       jmp     0x11e2 <.plt+0x12>
 // CHECK-NEXT: 11e9:       int3
 // CHECK-NEXT: 11ea:       int3
 // CHECK-NEXT: 11eb:       int3
@@ -36,10 +36,10 @@
 // CHECK-NEXT: 11ff:       int3
 // CHECK-NEXT: 1200:       pushl   %eax
 // CHECK-NEXT: 1201:       movl    12(%ebx), %eax
-// CHECK-NEXT: 1207:       calll   -28 <.plt+0x20>
-// CHECK-NEXT: 120c:       jmp     -47 <.plt+0x12>
+// CHECK-NEXT: 1207:       calll   0x11f0 <.plt+0x20>
+// CHECK-NEXT: 120c:       jmp     0x11e2 <.plt+0x12>
 // CHECK-NEXT: 1211:       pushl   $0
-// CHECK-NEXT: 1216:       jmp     -75 <.plt>
+// CHECK-NEXT: 1216:       jmp     0x11d0 <.plt>
 // CHECK-NEXT: 121b:       int3
 // CHECK-NEXT: 121c:       int3
 // CHECK-NEXT: 121d:       int3
@@ -47,10 +47,10 @@
 // CHECK-NEXT: 121f:       int3
 // CHECK-NEXT: 1220:       pushl   %eax
 // CHECK-NEXT: 1221:       movl    16(%ebx), %eax
-// CHECK-NEXT: 1227:       calll   -60 <.plt+0x20>
-// CHECK-NEXT: 122c:       jmp     -79 <.plt+0x12>
+// CHECK-NEXT: 1227:       calll   0x11f0 <.plt+0x20>
+// CHECK-NEXT: 122c:       jmp     0x11e2 <.plt+0x12>
 // CHECK-NEXT: 1231:       pushl   $8
-// CHECK-NEXT: 1236:       jmp     -107 <.plt>
+// CHECK-NEXT: 1236:       jmp     0x11d0 <.plt>
 // CHECK-NEXT: 123b:       int3
 // CHECK-NEXT: 123c:       int3
 // CHECK-NEXT: 123d:       int3
diff --git a/lld/test/ELF/i386-tls-dynamic.s b/lld/test/ELF/i386-tls-dynamic.s
index 7b5da4ad0b7c4..34dc99dea4314 100644
--- a/lld/test/ELF/i386-tls-dynamic.s
+++ b/lld/test/ELF/i386-tls-dynamic.s
@@ -82,16 +82,16 @@ addl tls1@gotntpoff(%ebx),%eax
 ## -4128 and -4120 are first and second GOT entries offsets.
 ## Each one is a pair of records.
 # DIS-NEXT: 1260:       leal -4128(,%ebx), %eax
-# DIS-NEXT: 1267:       calll 100
+# DIS-NEXT: 1267:       calll 0x12d0
 # DIS-NEXT: 126c:       leal -4120(,%ebx), %eax
-# DIS-NEXT: 1273:       calll 88
+# DIS-NEXT: 1273:       calll 0x12d0
 ## Local dynamic model:
 ## -16 is a local module tls index offset.
 # DIS-NEXT: 1278:       leal -4112(%ebx), %eax
-# DIS-NEXT: 127e:       calll 77
+# DIS-NEXT: 127e:       calll 0x12d0
 # DIS-NEXT: 1283:       leal 8(%eax), %edx
 # DIS-NEXT: 1289:       leal -4112(%ebx), %eax
-# DIS-NEXT: 128f:       calll 60
+# DIS-NEXT: 128f:       calll 0x12d0
 # DIS-NEXT: 1294:       leal 12(%eax), %edx
 ## Initial exec model:
 # DIS-NEXT: 129a:       movl %gs:0, %eax
diff --git a/lld/test/ELF/icf-absolute2.s b/lld/test/ELF/icf-absolute2.s
index 37e26a9d9c46b..5cf1d69bdecf8 100644
--- a/lld/test/ELF/icf-absolute2.s
+++ b/lld/test/ELF/icf-absolute2.s
@@ -4,7 +4,7 @@
 # RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %S/Inputs/icf-absolute2.s -o %t2
 # RUN: ld.lld %t %t2 -o /dev/null --icf=all --print-icf-sections | FileCheck -allow-empty %s
 
-## Test we do not crash and do not fold sections which relocations reffering to
+## Test we do not crash and do not fold sections which relocations referring to
 ## absolute symbols with a different values.
 # CHECK-NOT: selected
 
diff --git a/lld/test/ELF/invalid/broken-relaxation-x64.test b/lld/test/ELF/invalid/broken-relaxation-x64.test
index 1bd7fe0cc36d9..97a977e2c03ac 100644
--- a/lld/test/ELF/invalid/broken-relaxation-x64.test
+++ b/lld/test/ELF/invalid/broken-relaxation-x64.test
@@ -8,7 +8,7 @@
 ## YAML below contains 2 relocations of type R_X86_64_GOTTPOFF, and a .text
 ## with fake content filled by 0xFF. That means instructions for relaxation are
 ## "broken", so they does not match any known valid relaxations. We also generate
-## .tls section because we need it for correct proccessing of STT_TLS symbol.
+## .tls section because we need it for correct processing of STT_TLS symbol.
 !ELF
 FileHeader:
   Class:           ELFCLASS64
@@ -44,4 +44,4 @@ Symbols:
     Value:    0x12345
     Size:     4
     Binding:  STB_GLOBAL
- 
\ No newline at end of file
+ 
diff --git a/lld/test/ELF/invalid/entsize.yaml b/lld/test/ELF/invalid/entsize.yaml
new file mode 100644
index 0000000000000..a5fb7a90dca8d
--- /dev/null
+++ b/lld/test/ELF/invalid/entsize.yaml
@@ -0,0 +1,17 @@
+## Ignore invalid sh_entsize (sh_size%sh_entsize!=0) for non-SHF_MERGE sections.
+## GNU as before 2.35 had such a bug for .stab
+## https://sourceware.org/bugzilla/show_bug.cgi?id=25768
+# RUN: yaml2obj %s -o %t.o
+# RUN: ld.lld -e 0 --fatal-warnings %t.o -o /dev/null
+
+!ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_X86_64
+Sections:
+  - Name:    .stab
+    Type:    SHT_PROGBITS
+    EntSize: 20
+    Size:    24
diff --git a/lld/test/ELF/lazy-arch-conflict.s b/lld/test/ELF/lazy-arch-conflict.s
index e17c301798f1a..991476c5d6ec9 100644
--- a/lld/test/ELF/lazy-arch-conflict.s
+++ b/lld/test/ELF/lazy-arch-conflict.s
@@ -1,6 +1,6 @@
 # REQUIRES: x86
 
-# RUN: echo '.globl foo; .weak foo; .quad foo;' | llvm-mc -filetype=obj -triple=x86_64-unknown-linux - -o %t64.o
+# RUN: echo '.weak foo; .quad foo;' | llvm-mc -filetype=obj -triple=x86_64 - -o %t64.o
 # RUN: echo '.globl foo; foo:' | llvm-mc -filetype=obj -triple=i686-pc-linux - -o %t32.o
 # RUN: not ld.lld %t64.o --start-lib %t32.o --end-lib -o /dev/null 2>&1 | FileCheck %s
 
diff --git a/lld/test/ELF/linkerscript/align1.test b/lld/test/ELF/linkerscript/align1.test
index e7b65fc75ffe2..62dfdbfc5bd99 100644
--- a/lld/test/ELF/linkerscript/align1.test
+++ b/lld/test/ELF/linkerscript/align1.test
@@ -34,7 +34,7 @@ SECTIONS {
 # RUN: ld.lld -o %t5 --script %t.script %t.o
 # RUN: llvm-objdump --section-headers %t5 | FileCheck %s --check-prefix=ZERO
 
-## Test we fail gracefuly when alignment value is not a power of 2 (#1).
+## Test we fail gracefully when alignment value is not a power of 2 (#1).
 # RUN: echo "SECTIONS { . = 0x123; . = ALIGN(0x123, 3); .aaa : { *(.aaa) } }" > %t.script
 # RUN: not ld.lld -o /dev/null --script %t.script %t.o 2>&1 | FileCheck -check-prefix=ERR %s
 
diff --git a/lld/test/ELF/linkerscript/at4.s b/lld/test/ELF/linkerscript/at4.s
deleted file mode 100644
index a52a33e5cee4e..0000000000000
--- a/lld/test/ELF/linkerscript/at4.s
+++ /dev/null
@@ -1,28 +0,0 @@
-# REQUIRES: x86
-# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o
-# RUN: echo "SECTIONS { \
-# RUN:  . = 0x1000; \
-# RUN:  .aaa : { *(.aaa) } \
-# RUN:  .bbb : AT(0x2008) { *(.bbb) } \
-# RUN:  .ccc : { *(.ccc) } \
-# RUN: }" > %t.script
-# RUN: ld.lld %t.o --script %t.script -o %t
-# RUN: llvm-readelf -l %t | FileCheck %s
-
-# CHECK:      Type  Offset   VirtAddr           PhysAddr           FileSiz  MemSiz   Flg Align
-# CHECK-NEXT: LOAD  0x001000 0x0000000000001000 0x0000000000001000 0x000008 0x000008 R   0x1000
-# CHECK-NEXT: LOAD  0x001008 0x0000000000001008 0x0000000000002008 0x000010 0x000010 R   0x1000
-# CHECK-NEXT: LOAD  0x001018 0x0000000000001018 0x0000000000001018 0x000001 0x000001 R E 0x1000
-
-.global _start
-_start:
- nop
-
-.section .aaa, "a"
-.quad 0
-
-.section .bbb, "a"
-.quad 0
-
-.section .ccc, "a"
-.quad 0
diff --git a/lld/test/ELF/linkerscript/linkorder-linked-to.s b/lld/test/ELF/linkerscript/linkorder-linked-to.s
new file mode 100644
index 0000000000000..c203744bbde44
--- /dev/null
+++ b/lld/test/ELF/linkerscript/linkorder-linked-to.s
@@ -0,0 +1,74 @@
+# REQUIRES: x86
+## Test how we set the sh_link field of a SHF_LINK_ORDER output section.
+## Additionally, test that in a relocatable link, SHF_LINK_ORDER sections are
+## not combined. Combining them will arbitrarily choose a single output
+## section, losing tracking of correct dependencies.
+
+# RUN: llvm-mc -filetype=obj --triple=x86_64 %s -o %t.o
+
+## In the absence of a SECTIONS command.
+# RUN: ld.lld %t.o -o %t
+# RUN: llvm-readelf -S -x foo %t | FileCheck --check-prefixes=EXE,EXE-HEX %s
+# RUN: ld.lld %t.o -o %t.ro -r
+# RUN: llvm-readelf -S %t.ro | FileCheck --check-prefix=REL %s
+
+## A non-relocatable link places readonly sections before read-executable sections.
+# EXE:      [Nr] Name     {{.*}} Flg Lk
+# EXE-NEXT: [ 0]
+# EXE-NEXT: [ 1] foo      {{.*}}  AL  2
+# EXE-NEXT: [ 2] .text    {{.*}}  AX  0
+
+## An implementation detail: foo is moved after its linked-to section (orphan).
+# REL:      [Nr] Name     {{.*}} Flg Lk
+# REL-NEXT: [ 0]
+# REL-NEXT: [ 1] .text    {{.*}}  AX  0
+# REL-NEXT: [ 2] .text.f1 {{.*}}  AX  0
+# REL-NEXT: [ 3] foo      {{.*}}  AL  2
+# REL-NEXT: [ 4] .text.f2 {{.*}}  AX  0
+# REL-NEXT: [ 5] foo      {{.*}}  AL  4
+
+## A SECTIONS command combines .text.*
+# RUN: echo 'SECTIONS { .text : { *(.text.f1) *(.text.f2) } }' > %t1.lds
+# RUN: ld.lld -T %t1.lds %t.o -o %t1
+# RUN: llvm-readelf -S -x foo %t1 | FileCheck --check-prefixes=EXE,EXE-HEX %s
+# RUN: ld.lld -T %t1.lds %t.o -o %t1.ro -r
+# RUN: llvm-readelf -S -x foo %t1.ro | FileCheck --check-prefix=REL1 %s
+
+# REL1:      [Nr] Name    {{.*}} Flg Lk
+# REL1-NEXT: [ 0]
+# REL1-NEXT: [ 1] .text   {{.*}}  AX  0
+# REL1-NEXT: [ 2] foo     {{.*}}  AL  1
+
+## A SECTIONS command separates .text.*
+# RUN: echo 'SECTIONS { .text.f1 : { *(.text.f1) } .text.f2 : { *(.text.f2) } }' > %t2.lds
+# RUN: ld.lld -T %t2.lds %t.o -o %t2
+# RUN: llvm-readelf -S -x foo %t2 | FileCheck --check-prefixes=EXE2,EXE-HEX %s
+# RUN: ld.lld -T %t2.lds %t.o -o %t2.ro -r
+# RUN: llvm-readelf -S %t2.ro | FileCheck --check-prefixes=REL2 %s
+
+# EXE2:      [Nr] Name     {{.*}} Flg Lk
+# EXE2-NEXT: [ 0]
+# EXE2-NEXT: [ 1] foo      {{.*}}  AL  2
+# EXE2-NEXT: [ 2] .text.f1 {{.*}}  AX  0
+# EXE2-NEXT: [ 3] .text.f2 {{.*}}  AX  0
+
+# REL2:      [Nr] Name     {{.*}} Flg Lk
+# REL2-NEXT: [ 0]
+# REL2-NEXT: [ 1] .text.f1 {{.*}}  AX  0
+# REL2-NEXT: [ 2] .text.f2 {{.*}}  AX  0
+# REL2-NEXT: [ 3] .text    {{.*}}  AX  0
+# REL2-NEXT: [ 4] foo      {{.*}}  AL  1
+# REL2-NEXT: [ 5] foo      {{.*}}  AL  2
+
+# EXE-HEX:      Hex dump of section 'foo':
+# EXE-HEX-NEXT: 0102
+
+.section .text.f1,"ax",@progbits
+ret
+.section .text.f2,"ax",@progbits
+ret
+
+.section foo,"ao",@progbits,.text.f2
+.byte 2
+.section foo,"ao",@progbits,.text.f1
+.byte 1
diff --git a/lld/test/ELF/linkerscript/linkorder.s b/lld/test/ELF/linkerscript/linkorder.s
index 042e8b3293feb..1eb93ecd3218a 100644
--- a/lld/test/ELF/linkerscript/linkorder.s
+++ b/lld/test/ELF/linkerscript/linkorder.s
@@ -1,33 +1,67 @@
 # REQUIRES: x86
 # RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t.o
 
-# RUN: echo "SECTIONS { .rodata : {*(.rodata*)} .text : {*(.text.bar) *(.text.foo)} }" > %t.script
-# RUN: ld.lld -o %t --script %t.script %t.o
-# RUN: llvm-objdump -s %t | FileCheck %s
+## Contiguous SHF_LINK_ORDER sections.
+# RUN: echo 'SECTIONS { .rodata : {BYTE(0) *(.rodata*) BYTE(3)} \
+# RUN:   .text : {*(.text.bar) *(.text.foo)} }' > %t.lds
+# RUN: ld.lld -T %t.lds %t.o -o %t
+# RUN: llvm-readelf -S -x .rodata -x .text %t | FileCheck %s
 
-# RUN: echo "SECTIONS { .rodata : {*(.rodata*)} .text : {*(.text.foo) *(.text.bar)} }" > %t.script
-# RUN: ld.lld -o %t --script %t.script %t.o
-# RUN: llvm-objdump -s %t | FileCheck --check-prefix=INV %s
+# CHECK:      [ 1] .rodata   {{.*}} AL 3
+# CHECK:      [ 3] .text     {{.*}} AX 0
+# CHECK:      Hex dump of section '.rodata':
+# CHECK-NEXT: 00020103
+# CHECK:      Hex dump of section '.text':
+# CHECK-NEXT: 0201
 
+# RUN: echo 'SECTIONS { .rodata : {BYTE(0) *(.rodata*) BYTE(3)} \
+# RUN:  .text : {*(.text.foo) *(.text.bar)} }' > %t1.lds
+# RUN: ld.lld -T %t1.lds %t.o -o %t1
+# RUN: llvm-readelf -S -x .rodata -x .text %t1 | FileCheck --check-prefix=CHECK1 %s
 
-# CHECK:      Contents of section .rodata:
-# CHECK-NEXT: 02000000 00000000 01000000 00000000
-# CHECK:      Contents of section .text:
-# CHECK-NEXT: 02000000 00000000 01000000 00000000
+# CHECK1:      [ 1] .rodata   {{.*}} AL 3
+# CHECK1:      [ 3] .text     {{.*}} AX 0
+# CHECK1:      Hex dump of section '.rodata':
+# CHECK1-NEXT: 00010203
+# CHECK1:      Hex dump of section '.text':
+# CHECK1-NEXT: 0102
 
-# INV:      Contents of section .rodata:
-# INV-NEXT: 01000000 00000000 02000000 00000000
-# INV:      Contents of section .text:
-# INV-NEXT: 01000000 00000000 02000000 00000000
+## Adjacent input sections descriptions are contiguous.
+## Orphan section .text.bar precedes .text.foo, so swap the order of .rodata.*
+# RUN: echo 'SECTIONS { .rodata : {*(.rodata.foo) *(.rodata.bar)} }' > %t2.lds
+# RUN: ld.lld -T %t2.lds %t.o -o %t2
+# RUN: llvm-readelf -S -x .rodata %t2 | FileCheck --check-prefix=CHECK2 %s
+
+# CHECK2:      [ 1] .rodata   {{.*}} AL 4
+# CHECK2:      [ 4] .text.bar {{.*}} AX 0
+# CHECK2:      Hex dump of section '.rodata':
+# CHECK2-NEXT: 0201
+
+## Non-contiguous SHF_LINK_ORDER sections, separated by a BYTE.
+# RUN: echo 'SECTIONS { .rodata : {*(.rodata.foo) BYTE(0) *(.rodata.bar)} }' > %terr1.lds
+# RUN: not ld.lld -T %terr1.lds %t.o -o /dev/null 2>&1 | FileCheck --check-prefix=ERR %s
+
+## Non-contiguous SHF_LINK_ORDER sections, separated by a non-SHF_LINK_ORDER section.
+# RUN: echo 'SECTIONS { .rodata : {*(.rodata.foo) *(.text) *(.rodata.bar)} }' > %terr2.lds
+# RUN: not ld.lld -T %terr2.lds %t.o -o /dev/null 2>&1 | FileCheck --check-prefix=ERR %s
+
+## Non-contiguous SHF_LINK_ORDER sections, separated by a symbol assignment.
+# RUN: echo 'SECTIONS { .rodata : {*(.rodata.foo) a = .; *(.rodata.bar)} }' > %terr3.lds
+# RUN: not ld.lld -T %terr3.lds %t.o -o /dev/null 2>&1 | FileCheck --check-prefix=ERR %s
+
+# ERR: error: {{.*}}.o:(.rodata.bar): SHF_LINK_ORDER sections in .rodata are not contiguous
 
 .global _start
 _start:
 
+.section .ro,"a"
+.byte 0
+
 .section .text.bar,"a",@progbits
-.quad 2
+.byte 2
 .section .text.foo,"a",@progbits
-.quad 1
+.byte 1
 .section .rodata.foo,"ao",@progbits,.text.foo
-.quad 1
+.byte 1
 .section .rodata.bar,"ao",@progbits,.text.bar
-.quad 2
+.byte 2
diff --git a/lld/test/ELF/linkerscript/lma-offset.s b/lld/test/ELF/linkerscript/lma-offset.s
new file mode 100644
index 0000000000000..3c739724538e3
--- /dev/null
+++ b/lld/test/ELF/linkerscript/lma-offset.s
@@ -0,0 +1,39 @@
+# REQUIRES: x86
+## Test the difference between the VMA and the LMA for sections with AT().
+
+# RUN: echo '.globl _start; _start: ret; \
+# RUN:   .section .a,"a"; .byte 0; \
+# RUN:   .section .b,"a"; .byte 0; \
+# RUN:   .section .c,"a"; .byte 0; \
+# RUN:   .section .d,"a"; .byte 0; \
+# RUN:   .data; .byte 0' | \
+# RUN:   llvm-mc -filetype=obj -triple=x86_64 - -o %t.o
+# RUN: ld.lld -T %s %t.o -o %t
+# RUN: llvm-readelf -l %t | FileCheck %s
+
+# CHECK:      Type  Offset   VirtAddr           PhysAddr           FileSiz  MemSiz   Flg Align
+# CHECK-NEXT: LOAD  0x001000 0x0000000000001000 0x0000000000001000 0x000001 0x000001 R   0x1000
+
+## .b has AT(). It starts a PT_LOAD segment which also includes .c
+# CHECK-NEXT: LOAD  0x001001 0x0000000000001001 0x0000000000002005 0x000002 0x000002 R   0x1000
+
+## .d has AT(). It starts a PT_LOAD segment, even if the difference between
+## LMA and VMA (0x2007-0x1003) is the same as the previous one.
+# CHECK-NEXT: LOAD  0x001003 0x0000000000001003 0x0000000000002007 0x000001 0x000001 R   0x1000
+
+## The orphan section .text starts a PT_LOAD segment. The difference between
+## LMA and VMA (0x2008-0x1004) remains the same
+# CHECK-NEXT: LOAD  0x001004 0x0000000000001004 0x0000000000002008 0x000001 0x000001 R E 0x1000
+
+## .data starts a PT_LOAD segment. The difference remains the same.
+# CHECK-NEXT: LOAD  0x001005 0x0000000000001005 0x0000000000002009 0x000001 0x000001 RW  0x1000
+
+SECTIONS {
+  . = 0x1000;
+  .a : { *(.a) }
+  .b : AT(0x2005) { *(.b) }
+  .c : { *(.c) }
+  .d : AT(0x2007) { *(.d) }
+  ## Orphan section .text will be inserted here.
+  .data : { *(.data) }
+}
diff --git a/lld/test/ELF/linkerscript/loadaddr.s b/lld/test/ELF/linkerscript/loadaddr.s
index e2c82fc6c8cb5..055b7422baeb5 100644
--- a/lld/test/ELF/linkerscript/loadaddr.s
+++ b/lld/test/ELF/linkerscript/loadaddr.s
@@ -22,7 +22,7 @@
 # CHECK-NEXT: 0000000000002008 g       *ABS*  0000000000000000 bbb_lma
 # CHECK-NEXT: 0000000000003000 g       *ABS*  0000000000000000 ccc_lma
 # CHECK-NEXT: 0000000000004000 g       *ABS*  0000000000000000 ddd_lma
-# CHECK-NEXT: 0000000000001020 g       *ABS*  0000000000000000 txt_lma
+# CHECK-NEXT: 0000000000004008 g       *ABS*  0000000000000000 txt_lma
 # ERROR: {{.*}}.script:1: undefined section .zzz
 
 .global _start
diff --git a/lld/test/ELF/linkerscript/map-file2.test b/lld/test/ELF/linkerscript/map-file2.test
index f527e8ecdf80f..535043282249f 100644
--- a/lld/test/ELF/linkerscript/map-file2.test
+++ b/lld/test/ELF/linkerscript/map-file2.test
@@ -32,10 +32,10 @@ SECTIONS {
 # CHECK-NEXT:       1219             3209        8     1         {{.*}}{{/|\\}}map-file2.test.tmp.o:(.ddd)
 # CHECK-NEXT:       1228             3218       34     8 .eh_frame
 # CHECK-NEXT:       1228             3218       30     1         {{.*}}{{/|\\}}map-file2.test.tmp.o:(.eh_frame+0x0)
-# CHECK-NEXT:       125c             125c        1     4 .text
-# CHECK-NEXT:       125c             125c        1     4         {{.*}}{{/|\\}}map-file2.test.tmp.o:(.text)
-# CHECK-NEXT:       125c             125c        0     1                 f(int)
-# CHECK-NEXT:       125c             125c        0     1                 _start
+# CHECK-NEXT:       125c             324c        1     4 .text
+# CHECK-NEXT:       125c             324c        1     4         {{.*}}{{/|\\}}map-file2.test.tmp.o:(.text)
+# CHECK-NEXT:       125c             324c        0     1                 f(int)
+# CHECK-NEXT:       125c             324c        0     1                 _start
 # CHECK-NEXT:          0                0        8     1 .comment
 # CHECK-NEXT:          0                0        8     1         <internal>:(.comment)
 # CHECK-NEXT:          0                0       48     8 .symtab
diff --git a/lld/test/ELF/linkerscript/noload.s b/lld/test/ELF/linkerscript/noload.s
index eb6ace45adcd7..2f52b465854e2 100644
--- a/lld/test/ELF/linkerscript/noload.s
+++ b/lld/test/ELF/linkerscript/noload.s
@@ -3,13 +3,15 @@
 # RUN: echo "SECTIONS { \
 # RUN:        .data_noload_a (NOLOAD) : { *(.data_noload_a) } \
 # RUN:        .data_noload_b (0x10000) (NOLOAD) : { *(.data_noload_b) } \
+# RUN:        .no_input_sec_noload (NOLOAD) : { . += 1; } \
 # RUN:        .text (0x20000) : { *(.text) } };" > %t.script
 # RUN: ld.lld -o %t --script %t.script %t.o
 # RUN: llvm-readelf -S -l %t | FileCheck %s
 
-# CHECK:      Name           Type   Address          Off               Size
-# CHECK:      .data_noload_a NOBITS 0000000000000000 [[OFF:[0-9a-f]+]] 001000
-# CHECK-NEXT: .data_noload_b NOBITS 0000000000010000 [[OFF]]           001000
+# CHECK:      Name                 Type   Address          Off               Size
+# CHECK:      .data_noload_a       NOBITS 0000000000000000 [[OFF:[0-9a-f]+]] 001000
+# CHECK-NEXT: .data_noload_b       NOBITS 0000000000010000 [[OFF]]           001000
+# CHECK-NEXT: .no_input_sec_noload NOBITS 0000000000011000 [[OFF]]           000001
 
 # CHECK:      Type Offset   VirtAddr           PhysAddr
 # CHECK-NEXT: LOAD 0x001000 0x0000000000020000 0x0000000000020000
diff --git a/lld/test/ELF/linkerscript/overlay.test b/lld/test/ELF/linkerscript/overlay.test
index 85e140d60ab0c..2d3c88759c630 100644
--- a/lld/test/ELF/linkerscript/overlay.test
+++ b/lld/test/ELF/linkerscript/overlay.test
@@ -28,4 +28,4 @@ SECTIONS {
 # CHECK:      Type Offset   VirtAddr           PhysAddr           FileSiz  MemSiz   Flg Align
 # CHECK-NEXT: LOAD 0x001000 0x0000000000001000 0x0000000000004000 0x000008 0x000008 R   0x1000
 # CHECK-NEXT: LOAD 0x002000 0x0000000000001000 0x0000000000004008 0x000004 0x000004 R   0x1000
-# CHECK-NEXT: LOAD 0x002008 0x0000000000001008 0x0000000000001008 0x000001 0x000001 R E 0x1000
+# CHECK-NEXT: LOAD 0x002008 0x0000000000001008 0x0000000000004010 0x000001 0x000001 R E 0x1000
diff --git a/lld/test/ELF/linkerscript/phdrs.s b/lld/test/ELF/linkerscript/phdrs.s
index 8774bc04d4978..2e9fcf23bf710 100644
--- a/lld/test/ELF/linkerscript/phdrs.s
+++ b/lld/test/ELF/linkerscript/phdrs.s
@@ -128,7 +128,7 @@
 # DEFHDR-NEXT:      PF_X (0x1)
 # DEFHDR-NEXT:    ]
 
-# BADHDR:       {{.*}}.script:1: section header 'bar' is not listed in PHDRS
+# BADHDR:       {{.*}}.script:1: program header 'bar' is not listed in PHDRS
 
 # RUN: echo "PHDRS { text PT_LOAD FOOHDR; }" > %t1.script
 # RUN: not ld.lld -o /dev/null --script %t1.script %t 2>&1 | FileCheck --check-prefix=FOOHDR %s
diff --git a/lld/test/ELF/linkerscript/subalign.s b/lld/test/ELF/linkerscript/subalign.s
index bf812d17bb870..a817a69933143 100644
--- a/lld/test/ELF/linkerscript/subalign.s
+++ b/lld/test/ELF/linkerscript/subalign.s
@@ -34,7 +34,7 @@
 # RUN: ld.lld %t1.o --script %t4.script -o %t4
 # RUN: llvm-objdump -s %t4 | FileCheck --check-prefix=SUBALIGN %s
 
-## Test we fail gracefuly when alignment value is not a power of 2.
+## Test we fail gracefully when alignment value is not a power of 2.
 # RUN: echo "SECTIONS { .aaa : SUBALIGN(3) { *(.aaa*) } }" > %t5.script
 # RUN: not ld.lld %t1.o --script %t5.script -o /dev/null 2>&1 | FileCheck --check-prefix=ERR2 %s
 # ERR2: {{.*}}.script:1: alignment must be power of 2
diff --git a/lld/test/ELF/linkorder-err3.s b/lld/test/ELF/linkorder-err3.s
deleted file mode 100644
index 2e36ab457677c..0000000000000
--- a/lld/test/ELF/linkorder-err3.s
+++ /dev/null
@@ -1,17 +0,0 @@
-# REQUIRES: x86
-
-# RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t.o
-# RUN: not ld.lld %t.o -o /dev/null 2>&1 | FileCheck %s
-
-# CHECK:      error: incompatible section flags for .bar
-# CHECK-NEXT: >>> {{.*}}.o:(.bar): 0x2
-# CHECK-NEXT: >>> output section .bar: 0x82
-
-.section .foo,"a",@progbits
-.quad 0
-
-.section .bar,"ao",@progbits,.foo,unique,1
-.quad 0
-
-.section .bar,"a",@progbits,unique,2
-.quad 1
diff --git a/lld/test/ELF/local-got-pie.s b/lld/test/ELF/local-got-pie.s
index cab43e42836ab..41c33c2513b5b 100644
--- a/lld/test/ELF/local-got-pie.s
+++ b/lld/test/ELF/local-got-pie.s
@@ -17,7 +17,7 @@ foo:
 // DISASM:      Disassembly of section .text:
 // DISASM-EMPTY:
 // DISASM-NEXT: <_start>:
-// DISASM-NEXT:   1210: {{.*}} callq 4275
+// DISASM-NEXT:   1210: {{.*}} callq 0x22c8
 // DISASM:      <foo>:
 // DISASM-NEXT:   1215: {{.*}} nop
 
diff --git a/lld/test/ELF/local-got-shared.s b/lld/test/ELF/local-got-shared.s
index 554e2b126eace..083790562d55f 100644
--- a/lld/test/ELF/local-got-shared.s
+++ b/lld/test/ELF/local-got-shared.s
@@ -14,7 +14,7 @@ foo:
 
 // 0x22E0 - 0x1228 - 5 = 4275
 // DISASM:      <bar>:
-// DISASM-NEXT:   1228:       callq 4275
+// DISASM-NEXT:   1228:       callq 0x22e0
 
 // DISASM:      <foo>:
 // DISASM-NEXT:   122d:       nop
diff --git a/lld/test/ELF/local-got.s b/lld/test/ELF/local-got.s
index 17cb2da8a1658..6ae94924761af 100644
--- a/lld/test/ELF/local-got.s
+++ b/lld/test/ELF/local-got.s
@@ -18,8 +18,8 @@ foo:
 // 0x202320 - 0x201250 - 5 =  4299
 // 0x202328 - 0x201255 - 5 =  4302
 // DISASM:      <_start>:
-// DISASM-NEXT:   201250:       callq 4299
-// DISASM-NEXT:   201255:       callq 4302
+// DISASM-NEXT:   201250:       callq 0x202320
+// DISASM-NEXT:   201255:       callq 0x202328
 
 // DISASM:      <foo>:
 // DISASM-NEXT:   20125a:       nop
diff --git a/lld/test/ELF/lto/common4.ll b/lld/test/ELF/lto/common4.ll
index 7a40e4be32fa4..1b041667fb7bf 100644
--- a/lld/test/ELF/lto/common4.ll
+++ b/lld/test/ELF/lto/common4.ll
@@ -2,7 +2,7 @@
 
 ;; Make sure that common symbols are properly internalized.
 ;; In this file, @a does not interpose any symbol in a DSO,
-;; so LTO should be able to internelize it.
+;; so LTO should be able to internalize it.
 
 ; RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux /dev/null -o %t.so.o
 ; RUN: ld.lld -shared -o %t.so %t.so.o
diff --git a/lld/test/ELF/lto/linker-script-symbols-ipo.ll b/lld/test/ELF/lto/linker-script-symbols-ipo.ll
index 179cc833ffec2..967721db74d7c 100644
--- a/lld/test/ELF/lto/linker-script-symbols-ipo.ll
+++ b/lld/test/ELF/lto/linker-script-symbols-ipo.ll
@@ -18,7 +18,7 @@
 ; NOIPO:      <foo>:
 ; NOIPO-NEXT:   movl $2, %eax
 ; NOIPO:      <_start>:
-; NOIPO-NEXT:   jmp -21 <foo>
+; NOIPO-NEXT:   jmp 0x201160 <foo>
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/lld/test/ELF/lto/parallel.ll b/lld/test/ELF/lto/parallel.ll
index f400e4342534c..d9cb4fed7bfae 100644
--- a/lld/test/ELF/lto/parallel.ll
+++ b/lld/test/ELF/lto/parallel.ll
@@ -5,6 +5,9 @@
 ; RUN: llvm-nm %t.lto.o | FileCheck --check-prefix=CHECK0 %s
 ; RUN: llvm-nm %t1.lto.o | FileCheck --check-prefix=CHECK1 %s
 
+; RUN: not ld.lld --lto-partitions=0 %t.bc -o /dev/null 2>&1 | FileCheck --check-prefix=INVALID %s
+; INVALID: --lto-partitions: number of threads must be > 0
+
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/lld/test/ELF/lto/thinlto.ll b/lld/test/ELF/lto/thinlto.ll
index 2a7ffb90f6ff3..50e31d1e62324 100644
--- a/lld/test/ELF/lto/thinlto.ll
+++ b/lld/test/ELF/lto/thinlto.ll
@@ -16,8 +16,43 @@
 ; RUN: llvm-nm %t31.lto.o | FileCheck %s --check-prefix=NM1
 ; RUN: llvm-nm %t32.lto.o | FileCheck %s --check-prefix=NM2
 
-; Then check without --thinlto-jobs (which currently default to hardware_concurrency)
-; RUN: ld.lld -shared %t1.o %t2.o -o %t3
+;; --plugin-opt=jobs= is an alias.
+; RUN: rm -f %t31.lto.o %t32.lto.o
+; RUN: ld.lld -save-temps --plugin-opt=jobs=2 -shared %t1.o %t2.o -o %t3
+; RUN: llvm-nm %t31.lto.o | FileCheck %s --check-prefix=NM1
+; RUN: llvm-nm %t32.lto.o | FileCheck %s --check-prefix=NM2
+
+;; --thinlto-jobs= defaults to --threads=.
+; RUN: rm -f %t31.lto.o %t32.lto.o
+; RUN: ld.lld -save-temps --threads=2 -shared %t1.o %t2.o -o %t3
+; RUN: llvm-nm %t31.lto.o | FileCheck %s --check-prefix=NM1
+; RUN: llvm-nm %t32.lto.o | FileCheck %s --check-prefix=NM2
+
+;; --thinlto-jobs= overrides --threads=.
+; RUN: rm -f %t31.lto.o %t32.lto.o
+; RUN: ld.lld -save-temps --threads=1 --plugin-opt=jobs=2 -shared %t1.o %t2.o -o %t3
+; RUN: llvm-nm %t31.lto.o | FileCheck %s --check-prefix=NM1
+; RUN: llvm-nm %t32.lto.o | FileCheck %s --check-prefix=NM2
+
+; Test with all threads, on all cores, on all CPU sockets
+; RUN: rm -f %t31.lto.o %t32.lto.o
+; RUN: ld.lld -save-temps --thinlto-jobs=all -shared %t1.o %t2.o -o %t3
+; RUN: llvm-nm %t31.lto.o | FileCheck %s --check-prefix=NM1
+; RUN: llvm-nm %t32.lto.o | FileCheck %s --check-prefix=NM2
+
+; Test with many more threads than the system has
+; RUN: rm -f %t31.lto.o %t32.lto.o
+; RUN: ld.lld -save-temps --thinlto-jobs=1000 -shared %t1.o %t2.o -o %t3
+; RUN: llvm-nm %t31.lto.o | FileCheck %s --check-prefix=NM1
+; RUN: llvm-nm %t32.lto.o | FileCheck %s --check-prefix=NM2
+
+; Test with a bad value
+; RUN: rm -f %t31.lto.o %t32.lto.o
+; RUN: not ld.lld -save-temps --thinlto-jobs=foo -shared %t1.o %t2.o -o %t3 2>&1 | FileCheck %s --check-prefix=BAD-JOBS
+; BAD-JOBS: error: --thinlto-jobs: invalid job count: foo
+
+; Then check without --thinlto-jobs (which currently defaults to heavyweight_hardware_concurrency, meanning one thread per hardware core -- not SMT)
+; RUN: ld.lld -shared -save-temps %t1.o %t2.o -o %t3
 ; RUN: llvm-nm %t31.lto.o | FileCheck %s --check-prefix=NM1
 ; RUN: llvm-nm %t32.lto.o | FileCheck %s --check-prefix=NM2
 
diff --git a/lld/test/ELF/many-alloc-sections.s b/lld/test/ELF/many-alloc-sections.s
index 3b42dad87a90c..233f11475989d 100644
--- a/lld/test/ELF/many-alloc-sections.s
+++ b/lld/test/ELF/many-alloc-sections.s
@@ -11,13 +11,64 @@
 // CHECK-NEXT: Binding: Global
 // CHECK-NEXT: Type: None
 // CHECK-NEXT: Other: 0
-// CHECK-NEXT: Section: dm
+// CHECK-NEXT: Section: dn
+
+// Show that --gc-sections works when there are many sections, and that
+// referenced common and absolute symbols in such cases are not removed, nor are
+// they incorrectly attributed to the sections with index 0xFFF1 or 0xFFF2.
+// RUN: ld.lld %t.o -T %t.script -o %t --gc-sections
+// RUN: llvm-readobj --symbols --sections %t | FileCheck %s --check-prefix=GC
+
+// GC:      Sections [
+// GC-NEXT:   Section {
+// GC-NEXT:     Index: 0
+// GC-NEXT:     Name:  (0)
+// GC:        Section {
+// GC-NEXT:     Index: 1
+// GC-NEXT:     Name: dn
+// GC:        Section {
+// GC-NEXT:     Index: 2
+// GC-NEXT:     Name: .bss
+// GC:        Section {
+// GC-NEXT:     Index: 3
+// GC-NEXT:     Name: .comment
+// GC:        Section {
+// GC-NEXT:     Index: 4
+// GC-NEXT:     Name: .symtab
+// GC:        Section {
+// GC-NEXT:     Index: 5
+// GC-NEXT:     Name: .shstrtab
+// GC:        Section {
+// GC-NEXT:     Index: 6
+// GC-NEXT:     Name: .strtab
+// GC-NOT:    Section {
+
+// GC:      Symbols [
+// GC-NEXT:   Symbol {
+// GC-NEXT:     Name:  (0)
+// GC:        Symbol {
+// GC-NEXT:     Name: sdn
+// GC:        Symbol {
+// GC-NEXT:     Name: _start
+// GC:        Symbol {
+// GC-NEXT:     Name: abs
+// GC:        Symbol {
+// GC-NEXT:     Name: common
+// GC-NOT:    Symbol {
 
 .macro gen_sections4 x
         .section a\x,"a"
+        .global sa\x
+        sa\x:
         .section b\x,"a"
+        .global sa\x
+        sb\x:
         .section c\x,"a"
+        .global sa\x
+        sc\x:
         .section d\x,"a"
+        .global sa\x
+        sd\x:
 .endm
 
 .macro gen_sections8 x
@@ -85,9 +136,6 @@
         gen_sections16384 b\x
 .endm
 
-        .bss
-        .section bar
-
 gen_sections32768 a
 gen_sections16384 b
 gen_sections8192 c
@@ -95,12 +143,20 @@ gen_sections4096 d
 gen_sections2048 e
 gen_sections1024 f
 gen_sections512 g
-gen_sections128 h
-gen_sections64 i
-gen_sections32 j
-gen_sections16 k
-gen_sections8 l
-gen_sections4 m
+gen_sections256 h
+gen_sections128 i
+gen_sections64 j
+gen_sections32 k
+gen_sections16 l
+gen_sections8 m
+gen_sections4 n
+
+.global abs
+abs = 0x12345678
+
+.comm common,4,4
 
 .global _start
 _start:
+  .quad abs
+  .quad common
diff --git a/lld/test/ELF/mips-sto-pic-flag.s b/lld/test/ELF/mips-sto-pic-flag.s
index c313a8eb3c7f8..6c9a660efca47 100644
--- a/lld/test/ELF/mips-sto-pic-flag.s
+++ b/lld/test/ELF/mips-sto-pic-flag.s
@@ -1,7 +1,7 @@
 # REQUIRES: mips
 # In case of linking PIC and non-PIC code together and generation
 # of a relocatable object, all PIC symbols should have STO_MIPS_PIC
-# flag in the symbol table of the ouput file.
+# flag in the symbol table of the output file.
 
 # RUN: llvm-mc -filetype=obj -triple=mips-unknown-linux %s -o %t-npic.o
 # RUN: llvm-mc -filetype=obj -triple=mips-unknown-linux \
diff --git a/lld/test/ELF/no-inhibit-exec.s b/lld/test/ELF/no-inhibit-exec.s
index 788a58f365a04..66031eaea912c 100644
--- a/lld/test/ELF/no-inhibit-exec.s
+++ b/lld/test/ELF/no-inhibit-exec.s
@@ -8,7 +8,7 @@
 # CHECK: Disassembly of section .text:
 # CHECK-EMPTY:
 # CHECK-NEXT: _start
-# CHECK-NEXT: 201120: {{.*}} callq -2101541
+# CHECK-NEXT: 201120: {{.*}} callq 0x0
 
 # RELOC:      Relocations [
 # RELOC-NEXT: ]
diff --git a/lld/test/ELF/pack-dyn-relocs.s b/lld/test/ELF/pack-dyn-relocs.s
index 34c9a3cb315cf..6ecf354f68a2c 100644
--- a/lld/test/ELF/pack-dyn-relocs.s
+++ b/lld/test/ELF/pack-dyn-relocs.s
@@ -69,7 +69,7 @@
 // ANDROID32-HEADERS: 0x6000000F ANDROID_REL          [[ADDR]]
 // ANDROID32-HEADERS: 0x60000010 ANDROID_RELSZ        [[SIZE]]
 
-// Packed should have the groups of non-relative reloations first, followed
+// Packed should have the groups of non-relative relocations first, followed
 // by the larger groups of relative relocations (i.e. the 8 and 9 followed
 // by the 7.)
 // ANDROID32:          Section ({{.+}}) .rel.dyn {
diff --git a/lld/test/ELF/ppc32-call-stub-nopic.s b/lld/test/ELF/ppc32-call-stub-nopic.s
index 03ffa5a1e37c9..272a8b2ad3601 100644
--- a/lld/test/ELF/ppc32-call-stub-nopic.s
+++ b/lld/test/ELF/ppc32-call-stub-nopic.s
@@ -20,10 +20,10 @@
 
 ## .got2+0x8000-0x10004 = 0x30000+0x8000-0x10004 = 65536*2+32764
 # CHECK-LABEL: <_start>:
-# CHECK-NEXT:    bl .+16
-# CHECK-NEXT:    bl .+12
-# CHECK-NEXT:    bl .+24
-# CHECK-NEXT:    bl .+20
+# CHECK-NEXT:    bl 0x100101d0
+# CHECK-NEXT:    bl 0x100101d0
+# CHECK-NEXT:    bl 0x100101e0
+# CHECK-NEXT:    bl 0x100101e0
 # CHECK-EMPTY:
 
 ## -fno-PIC call stubs of f and g.
@@ -47,8 +47,8 @@
 
 ## These instructions are referenced by .plt entries.
 # CHECK: 10010200 <.glink>:
-# CHECK-NEXT: b .+8
-# CHECK-NEXT: b .+4
+# CHECK-NEXT: b 0x10010208
+# CHECK-NEXT: b 0x10010208
 
 ## PLTresolve
 ## Operands of lis & lwz: .got+4 = 0x10020070+4 = 65536*4098+700
diff --git a/lld/test/ELF/ppc32-call-stub-pic.s b/lld/test/ELF/ppc32-call-stub-pic.s
index e35367d33686b..b9e19791e4851 100644
--- a/lld/test/ELF/ppc32-call-stub-pic.s
+++ b/lld/test/ELF/ppc32-call-stub-pic.s
@@ -33,29 +33,39 @@
 
 ## .got2+0x8000-0x10004 = 0x30000+0x8000-0x10004 = 65536*2+32764
 # CHECK-LABEL: <_start>:
-# CHECK-NEXT:         bcl 20, 31, .+4
+# PIE-NEXT:           bcl 20, 31, 0x10210
 # PIE-NEXT:    10210: mflr 30
 # PIE-NEXT:           addis 30, 30, 3
 # PIE-NEXT:           addi 30, 30, -32412
-# SHARED-NEXT: 10230: mflr 30
-# SHARED-NEXT:        addis 30, 30, 3
-# SHARED-NEXT:        addi 30, 30, -32420
-
 ## Two bl 00008000.got2.plt_pic32.f
-# CHECK-NEXT:    bl .+40
-# CHECK-NEXT:    bl .+36
+# PIE-NEXT:           bl 0x10244
+# PIE-NEXT:           bl 0x10244
 ## Two bl 00008000.got2.plt_pic32.g
-# CHECK-NEXT:    bl .+48
-# CHECK-NEXT:    bl .+44
+# PIE-NEXT:           bl 0x10254
+# PIE-NEXT:           bl 0x10254
 ## Two bl 00008000.got2.plt_pic32.h
-# CHECK-NEXT:    bl .+56
-# CHECK-NEXT:    bl .+52
-# CHECK-NEXT:    addis 30, 30, {{.*}}
-# CHECK-NEXT:    addi 30, 30, {{.*}}
+# PIE-NEXT:           bl 0x10264
+# PIE-NEXT:           bl 0x10264
+# PIE-NEXT:           addis 30, 30, {{.*}}
+# PIE-NEXT:           addi 30, 30, {{.*}}
 ## bl 00008000.plt_pic32.f
-# CHECK-NEXT:    bl .+56
+# PIE-NEXT:           bl 0x10274
 ## bl 00008000.plt_pic32.f
-# CHECK-NEXT:    bl .+68
+# PIE-NEXT:           bl 0x10284
+# SHARED-NEXT:        bcl 20, 31, 0x10230
+# SHARED-NEXT: 10230: mflr 30
+# SHARED-NEXT:        addis 30, 30, 3
+# SHARED-NEXT:        addi 30, 30, -32420
+# SHARED-NEXT:        bl 0x10264
+# SHARED-NEXT:        bl 0x10264
+# SHARED-NEXT:        bl 0x10274
+# SHARED-NEXT:        bl 0x10274
+# SHARED-NEXT:        bl 0x10284
+# SHARED-NEXT:        bl 0x10284
+# SHARED-NEXT:        addis 30, 30, {{.*}}
+# SHARED-NEXT:        addi 30, 30, {{.*}}
+# SHARED-NEXT:        bl 0x10294
+# SHARED-NEXT:        bl 0x102a4
 # CHECK-EMPTY:
 
 ## -fPIC call stubs of f and g.
@@ -97,19 +107,17 @@
 # HEX: 0x0004036c 00010294 00010298 0001029c
 
 ## These instructions are referenced by .plt entries.
-# PIE:    00010294 <.glink>:
-# SHARED: 000102b4 <.glink>:
-# CHECK-NEXT: b .+12
-# CHECK-NEXT: b .+8
-# CHECK-NEXT: b .+4
+# CHECK:      [[#%x,GLINK:]] <.glink>:
+# CHECK-NEXT: b 0x[[#%x,GLINK+12]]
+# CHECK-NEXT: b 0x[[#%x,GLINK+12]]
+# CHECK-NEXT: b 0x[[#%x,GLINK+12]]
 
 ## PLTresolve
 ## Operand of addi: 0x100a8-.glink = 24
 # CHECK-NEXT:         addis 11, 11, 0
 # CHECK-NEXT:         mflr 0
-# CHECK-NEXT:         bcl 20, 31, .+4
-# PIE-NEXT:    102ac: addi 11, 11, 24
-# SHARED-NEXT: 102cc: addi 11, 11, 24
+# CHECK-NEXT:         bcl 20, 31, 0x[[#%x,NEXT:]]
+# CHECK-NEXT: [[#%x,NEXT]]: addi 11, 11, 24
 
 # CHECK-NEXT: mflr 12
 # CHECK-NEXT: mtlr 0
diff --git a/lld/test/ELF/ppc32-canonical-plt.s b/lld/test/ELF/ppc32-canonical-plt.s
index 027f2aa7e2222..f29e66ea0c28a 100644
--- a/lld/test/ELF/ppc32-canonical-plt.s
+++ b/lld/test/ELF/ppc32-canonical-plt.s
@@ -57,9 +57,9 @@
 # CHECK-NEXT:           bctr
 
 ## The 3 b instructions are referenced by .plt entries.
-# CHECK-NEXT: 1001025c: b .+12
-# CHECK-NEXT:           b .+8
-# CHECK-NEXT:           b .+4
+# CHECK-NEXT: 1001025c: b 0x10010268
+# CHECK-NEXT:           b 0x10010268
+# CHECK-NEXT:           b 0x10010268
 
 ## PLTresolve of 64 bytes is at the end.
 ## Operands of addis & addi: -0x1001025c = 65536*-4097-604
diff --git a/lld/test/ELF/ppc32-ifunc-nonpreemptible-nopic.s b/lld/test/ELF/ppc32-ifunc-nonpreemptible-nopic.s
index 9c2b445584833..179f79efc6ff1 100644
--- a/lld/test/ELF/ppc32-ifunc-nonpreemptible-nopic.s
+++ b/lld/test/ELF/ppc32-ifunc-nonpreemptible-nopic.s
@@ -16,7 +16,7 @@
 # CHECK:      <.text>:
 # CHECK-NEXT: 100100e0: blr
 # CHECK:      <_start>:
-# CHECK-NEXT:   bl .+12
+# CHECK-NEXT:   bl 0x100100f0
 # CHECK-NEXT:   lis 9, 4097
 # CHECK-NEXT:   addi 9, 9, 256
 # CHECK-EMPTY:
diff --git a/lld/test/ELF/ppc32-ifunc-nonpreemptible-pic.s b/lld/test/ELF/ppc32-ifunc-nonpreemptible-pic.s
index 2e52939642ca2..0bb715a191d83 100644
--- a/lld/test/ELF/ppc32-ifunc-nonpreemptible-pic.s
+++ b/lld/test/ELF/ppc32-ifunc-nonpreemptible-pic.s
@@ -21,7 +21,7 @@
 # CHECK:      <.text>:
 # CHECK-NEXT: 10188: blr
 # CHECK:      <_start>:
-# CHECK-NEXT:   bl .+12
+# CHECK-NEXT:   bl 0x10198
 # CHECK-NEXT:   lis 9, 1
 # CHECK-NEXT:   addi 9, 9, 424
 # CHECK-EMPTY:
diff --git a/lld/test/ELF/ppc32-local-branch.s b/lld/test/ELF/ppc32-local-branch.s
index 526bea609122c..95099df43be2f 100644
--- a/lld/test/ELF/ppc32-local-branch.s
+++ b/lld/test/ELF/ppc32-local-branch.s
@@ -7,10 +7,10 @@
 ## R_PPC_REL24 and R_PPC_PLTREL24 are converted to PC relative relocations if the
 ## symbol is non-preemptable. The addend of R_PPC_PLTREL24 should be ignored.
 
-# CHECK:      <_start>:
-# CHECK-NEXT:   b .+12
-# CHECK-NEXT:   b .+8
-# CHECK-NEXT:   b .+4
+# CHECK:      [[#%x,ADDR:]] <_start>:
+# CHECK-NEXT:   b 0x[[#%x,ADDR+12]]
+# CHECK-NEXT:   b 0x[[#%x,ADDR+12]]
+# CHECK-NEXT:   b 0x[[#%x,ADDR+12]]
 # CHECK-EMPTY:
 # CHECK-NEXT: <foo>:
 
diff --git a/lld/test/ELF/ppc32-long-thunk.s b/lld/test/ELF/ppc32-long-thunk.s
index 21b1eaeff2855..90c284ded5fae 100644
--- a/lld/test/ELF/ppc32-long-thunk.s
+++ b/lld/test/ELF/ppc32-long-thunk.s
@@ -16,12 +16,12 @@
 # SEC: There are no relocations in this file.
 
 # CHECK:      <_start>:
-# CHECK-NEXT:     2000: bl .+24
-# CHECK-NEXT:           bl .+20
-# CHECK-NEXT:           bl .+16
-# CHECK-NEXT:           bl .+33554428
-# PD-NEXT:              bl .+24
-# PI-NEXT:              bl .+40
+# CHECK-NEXT:     2000: bl 0x2018
+# CHECK-NEXT:           bl 0x2018
+# CHECK-NEXT:           bl 0x2018
+# CHECK-NEXT:           bl 0x2002008
+# PD-NEXT:              bl 0x2028
+# PI-NEXT:              bl 0x2038
 
 ## high = 0x02002008 = 65536*512+8200
 # PD:         <__LongThunk_high>:
@@ -40,7 +40,7 @@
 ## high-0x2028 = 0x02002008-0x2020 = 65536*512-24
 # PI:         <__LongThunk_high>:
 # PI-NEXT:        2018: mflr 0
-# PI-NEXT:              bcl 20, 31, .+4
+# PI-NEXT:              bcl 20, 31, 0x2020
 # PI-NEXT:        2020: mflr 12
 # PI-NEXT:              addis 12, 12, 512
 # PI-NEXT:              addi 12, 12, -24
@@ -51,7 +51,7 @@
 ## .text_high+16-0x2048 = 0x02002010-0x2048 = 65536*512-48
 # PI:         <__LongThunk_>:
 # PI-NEXT:        2038: mflr 0
-# PI-NEXT:              bcl 20, 31, .+4
+# PI-NEXT:              bcl 20, 31, 0x2040
 # PI-NEXT:        2040: mflr 12
 # PI-NEXT:              addis 12, 12, 512
 # PI-NEXT:              addi 12, 12, -48
@@ -70,8 +70,8 @@ bl .text_high+16  # Need a thunk
 blr
 
 # PD:         02002008 <high>:
-# PD-NEXT:              bl .-33554432
-# PD-NEXT:              bl .+4
+# PD-NEXT:              bl 0x2008
+# PD-NEXT:              bl 0x2002010
 # PD:         <__LongThunk_>:
 # PD-NEXT:     2002010: lis 12, 0
 # PD-NEXT:              addi 12, 12, 8200
diff --git a/lld/test/ELF/ppc32-reloc-rel.s b/lld/test/ELF/ppc32-reloc-rel.s
index 49d149550aae8..fefeeba5e2fee 100644
--- a/lld/test/ELF/ppc32-reloc-rel.s
+++ b/lld/test/ELF/ppc32-reloc-rel.s
@@ -7,13 +7,13 @@
   beq 1f
 1:
 # CHECK-LABEL: section .R_PPC_REL14:
-# CHECK: bt 2, .+4
+# CHECK: 100100b4: bt 2, 0x100100b8
 
 .section .R_PPC_REL24,"ax",@progbits
   b 1f
 1:
 # CHECK-LABEL: section .R_PPC_REL24:
-# CHECK: b .+4
+# CHECK: b 0x100100bc
 
 .section .R_PPC_REL32,"ax",@progbits
   .long 1f - .
@@ -25,10 +25,10 @@
   b 1f@PLT+32768
 1:
 # CHECK-LABEL: section .R_PPC_PLTREL24:
-# CHECK: b .+4
+# CHECK: b 0x100100c4
 
 .section .R_PPC_LOCAL24PC,"ax",@progbits
   b 1f@local
 1:
 # CHECK-LABEL: section .R_PPC_LOCAL24PC:
-# CHECK: b .+4
+# CHECK: b 0x100100c8
diff --git a/lld/test/ELF/ppc32-tls-gd.s b/lld/test/ELF/ppc32-tls-gd.s
index 847ab83655a40..c3a1ffd367049 100644
--- a/lld/test/ELF/ppc32-tls-gd.s
+++ b/lld/test/ELF/ppc32-tls-gd.s
@@ -31,17 +31,17 @@
 
 ## &DTPMOD(a) - _GLOBAL_OFFSET_TABLE_ = 0x20078 - 0x20078 = 0
 # GD:      addi 3, 31, 0
-# GD-NEXT: bl .+32
+# GD-NEXT: bl 0x1028c
 # GD-NEXT: lwz 3, 0(3)
 
 ## &DTPMOD(b) - _GLOBAL_OFFSET_TABLE_ = 0x20080 - 0x20078 = 8
 # GD-NEXT: addi 3, 31, 8
-# GD-NEXT: bl .+20
+# GD-NEXT: bl 0x1028c
 # GD-NEXT: lwz 3, 0(3)
 
 ## &DTPMOD(c) - _GLOBAL_OFFSET_TABLE_ = 0x20088 - 0x20078 = 16
 # GD-NEXT: addi 3, 9, 16
-# GD-NEXT: bl .+8
+# GD-NEXT: bl 0x1028c
 # GD-NEXT: lwz 3, 0(3)
 
 # NOREL: no relocations
diff --git a/lld/test/ELF/ppc32-tls-ld.s b/lld/test/ELF/ppc32-tls-ld.s
index de6700bde597c..0eda8e95ff208 100644
--- a/lld/test/ELF/ppc32-tls-ld.s
+++ b/lld/test/ELF/ppc32-tls-ld.s
@@ -16,7 +16,7 @@
 
 ## .got - _GLOBAL_OFFSET_TABLE_ = 0
 # LD:      addi 3, 30, 0
-# LD-NEXT: bl .+40
+# LD-NEXT: bl 0x101f8
 ## a@dtprel = st_value(a)-0x8000 = 65540-0x8000 = 65536*1-32764
 ## b@dtprel = st_value(a)-0x8000 = 131080-0x8000 = 65536*2-32760
 # LD-NEXT: addis 9, 3, 1
@@ -30,7 +30,7 @@
 ## It shares In.Got->TlsIndexOff allocated when processing a@got@tlsld.
 ## .got - _GLOBAL_OFFSET_TABLE_ = 0
 # LD-NEXT: addi 3, 9, 0
-# LD-NEXT: bl .+12
+# LD-NEXT: bl 0x101f8
 ## b@dtprel = st_value(a)-0x8000 = 131080-0x8000 = 65536*2-32760
 # LD-NEXT: addis 29, 3, 2
 # LD-NEXT: addi 29, 29, -32760
diff --git a/lld/test/ELF/ppc32-weak-undef-call.s b/lld/test/ELF/ppc32-weak-undef-call.s
index 3371e429a0c40..dfb45e5fe18cf 100644
--- a/lld/test/ELF/ppc32-weak-undef-call.s
+++ b/lld/test/ELF/ppc32-weak-undef-call.s
@@ -9,11 +9,12 @@
 
 ## It does not really matter how we fixup it, but we cannot overflow and
 ## should not generate a call stub (this would waste space).
-# PDE: bl .+0
+# PDE: bl 0x100100b4
 
 ## With -pie or -shared, create a call stub. ld.bfd produces bl .+0
-# PIC: bl .+4
-# PIC: <00000000.plt_pic32.foo>:
+# PIC:       bl 0x[[PLT:[0-9a-f]+]]
+# PIC-EMPTY:
+# PIC-NEXT:  000[[PLT]] <00000000.plt_pic32.foo>:
 
 .weak foo
 bl foo
diff --git a/lld/test/ELF/ppc64-bsymbolic-toc-restore.s b/lld/test/ELF/ppc64-bsymbolic-toc-restore.s
index 1e3148e24392f..a8f821ec9d16a 100644
--- a/lld/test/ELF/ppc64-bsymbolic-toc-restore.s
+++ b/lld/test/ELF/ppc64-bsymbolic-toc-restore.s
@@ -3,13 +3,13 @@
 # RUN: llvm-mc -filetype=obj -triple=powerpc64le-unknown-linux %s -o %t1.o
 # RUN: llvm-mc -filetype=obj -triple=powerpc64le-unknown-linux %p/Inputs/ppc64-bsymbolic-local-def.s  -o %t2.o
 # RUN: ld.lld -Bsymbolic -shared %t1.o %t2.o -o %t
-# RUN: llvm-objdump -d -r %t | FileCheck %s
+# RUN: llvm-objdump -d -r --no-show-raw-insn %t | FileCheck %s
 # RUN: not ld.lld -shared %t1.o %t2.o -o /dev/null 2>&1 | FileCheck --check-prefix=FAIL %s
 
 # RUN: llvm-mc -filetype=obj -triple=powerpc64-unknown-linux %s -o %t1.o
 # RUN: llvm-mc -filetype=obj -triple=powerpc64-unknown-linux %p/Inputs/ppc64-bsymbolic-local-def.s  -o %t2.o
 # RUN: ld.lld -Bsymbolic -shared %t1.o %t2.o -o %t
-# RUN: llvm-objdump -d -r %t | FileCheck %s
+# RUN: llvm-objdump -d -r --no-show-raw-insn %t | FileCheck %s
 # RUN: not ld.lld -shared %t1.o %t2.o -o /dev/null 2>&1 | FileCheck --check-prefix=FAIL %s
 
 # FAIL: call to def lacks nop, can't restore toc
@@ -51,9 +51,9 @@ caller:
 # being called.
 
 # CHECK-LABEL: caller
-# CHECK:         bl .+44
+# CHECK:         bl 0x[[DEF:[0-9a-f]+]]
 # CHECK-NEXT:    mr 31, 3
-# CHECK-NEXT:    bl .+44
+# CHECK-NEXT:    bl 0x[[NOT_DEFINED:[0-9a-f]+]]
 # CHECK-NEXT:    ld 2, 24(1)
 # CHECK-NEXT:    add 3, 3, 31
 # CHECK-NEXT:    addi 1, 1, 32
@@ -64,5 +64,7 @@ caller:
 # CHECK-NEXT:  <def>:
 # CHECK-NEXT:    addis 2, 12, 2
 # CHECK-NEXT:    addi 2, 2, -32456
-# CHECK-NEXT:    li 3, 55
+# CHECK-NEXT:    [[DEF]]: li 3, 55
 # CHECK-NEXT:    blr
+# CHECK-EMPTY:
+# CHECK-NEXT:  00000000000[[NOT_DEFINED]] <__plt_not_defined>:
diff --git a/lld/test/ELF/ppc64-call-reach.s b/lld/test/ELF/ppc64-call-reach.s
index c54bb051ff3cf..5f0e2dc47015c 100644
--- a/lld/test/ELF/ppc64-call-reach.s
+++ b/lld/test/ELF/ppc64-call-reach.s
@@ -58,16 +58,16 @@ test:
 # Check that we are branching to the definitions, and not range-extending
 # thunks.
 # CHECK-LABEL: test
-# CHECK:  10010014:       bl .+33554428
-# CHECK:  10010024:       b  .+33554428
+# CHECK:  10010014:       bl 0x12010010
+# CHECK:  10010024:       b  0x12010020
 
 # NEGOFFSET-LABEL: test
-# NEGOFFSET:  10010014:       bl .-33554432
-# NEGOFFSET:  10010024:       b  .+33554432
+# NEGOFFSET:  10010014:       bl 0xe010014
+# NEGOFFSET:  10010024:       b  0xe010024
 
 # THUNK-LABEL: <test>:
-# THUNK: 10010014:       bl .+20
-# THUNK: 10010024:       b .+20
+# THUNK: 10010014:       bl 0x10010028
+# THUNK: 10010024:       b 0x10010038
 
 # .branch_lt[0]
 # THUNK-LABEL: <__long_branch_callee>:
diff --git a/lld/test/ELF/ppc64-func-entry-points.s b/lld/test/ELF/ppc64-func-entry-points.s
index 30336afa6c8f2..4bdea4f359f4e 100644
--- a/lld/test/ELF/ppc64-func-entry-points.s
+++ b/lld/test/ELF/ppc64-func-entry-points.s
@@ -70,8 +70,8 @@ glob:
 # point and we branch to start of foo_external_same.
 
 // CHECK-LABEL: <_start>:
-// CHECK:         100101f0: bl .+144
-// CHECK:         10010204: bl .+84
+// CHECK:         100101f0: bl 0x10010280
+// CHECK:         10010204: bl 0x10010258
 // CHECK-LABEL: <foo_external_diff>:
 // CHECK-NEXT:    10010250: addis 2, 12, 2
 // CHECK-NEXT:    10010254: addi 2, 2, -32696
diff --git a/lld/test/ELF/ppc64-ifunc.s b/lld/test/ELF/ppc64-ifunc.s
index 804b991990dd9..75a3e1072e06e 100644
--- a/lld/test/ELF/ppc64-ifunc.s
+++ b/lld/test/ELF/ppc64-ifunc.s
@@ -27,9 +27,9 @@
 # CHECK: <_start>:
 # CHECK-NEXT:                 addis 2, 12, 2
 # CHECK-NEXT:                 addi 2, 2, -32636
-# CHECK-NEXT: 1001021c:       bl .+32
+# CHECK-NEXT: 1001021c:       bl 0x1001023c
 # CHECK-NEXT:                 ld 2, 24(1)
-# CHECK-NEXT: 10010224:       bl .+44
+# CHECK-NEXT: 10010224:       bl 0x10010250
 # CHECK-NEXT:                 ld 2, 24(1)
 # CHECK-NEXT:                 addis 3, 2, -2
 # CHECK-NEXT:                 addi 3, 3, 32748
diff --git a/lld/test/ELF/ppc64-local-dynamic.s b/lld/test/ELF/ppc64-local-dynamic.s
index 884e9c158393f..f0f3967387dd0 100644
--- a/lld/test/ELF/ppc64-local-dynamic.s
+++ b/lld/test/ELF/ppc64-local-dynamic.s
@@ -95,7 +95,7 @@ k:
 // OutputRelocs-NEXT: Offset Info Type Symbol's Value Symbol's Name + Addend
 // OutputRelocs-NEXT: R_PPC64_DTPMOD64
 
-// Check that the got has 3 entries, 1 for the TOC and 1 stucture of 2 entries
+// Check that the got has 3 entries, 1 for the TOC and 1 structure of 2 entries
 // for the tls variables. Also verify the address so we can check the offsets
 // we calculate for each relocation type.
 // CheckGot: got          00000018 0000000000020100
@@ -113,7 +113,7 @@ k:
 // Dis:     <test>:
 // Dis:        addis 3, 2, 0
 // Dis-NEXT:   addi 3, 3, -32760
-// Dis-NEXT:   bl .+60
+// Dis-NEXT:   bl 0x10058
 // Dis-NEXT:   ld 2, 24(1)
 // Dis-NEXT:   addis 3, 3, 0
 // Dis-NEXT:   lwa 3, -32768(3)
diff --git a/lld/test/ELF/ppc64-long-branch-pi.s b/lld/test/ELF/ppc64-long-branch-pi.s
index f49b09ef8f811..dca63878c0e78 100644
--- a/lld/test/ELF/ppc64-long-branch-pi.s
+++ b/lld/test/ELF/ppc64-long-branch-pi.s
@@ -30,10 +30,10 @@
 # RELOC-NEXT: }
 
 # CHECK:      <_start>:
-# CHECK-NEXT:     2000:       bl .+16
-# CHECK-NEXT:                 bl .+33554428
-# CHECK-NEXT:                 bl .+24
-# CHECK-NEXT:                 bl .+36
+# CHECK-NEXT:     2000:       bl 0x2010
+# CHECK-NEXT:                 bl 0x2002000
+# CHECK-NEXT:                 bl 0x2020
+# CHECK-NEXT:                 bl 0x2030
 
 ## &.branch_lt[0] - .TOC. = .branch_lt - (.got+0x8000) = -32752
 # CHECK:      <__long_branch_>:
@@ -65,9 +65,9 @@ bl .text_high+8   # Need a thunk
 bl .text_high+0xc # Need a thunk
 
 # CHECK:      <high_target>:
-# CHECK-NEXT:  2002000:   bl .-33554428
-# CHECK-NEXT:             bl .-33554432
-# CHECK-NEXT:             bl .+8
+# CHECK-NEXT:  2002000:   bl 0x2004
+# CHECK-NEXT:             bl 0x2004
+# CHECK-NEXT:             bl 0x2002010
 
 ## &.branch_lt[3] - .TOC. = .branch_lt - (.got+0x8000) = -32728
 # CHECK:      <__long_branch_>:
diff --git a/lld/test/ELF/ppc64-long-branch.s b/lld/test/ELF/ppc64-long-branch.s
index c6979e70e1dad..35286e1015874 100644
--- a/lld/test/ELF/ppc64-long-branch.s
+++ b/lld/test/ELF/ppc64-long-branch.s
@@ -29,10 +29,10 @@
 # BRANCH-BE-NEXT: 0x02002040 00000000 00002008
 
 # CHECK:      <_start>:
-# CHECK-NEXT:     2000:       bl .+24
-# CHECK-NEXT:                 bl .+20
-# CHECK-NEXT:                 bl .+16
-# CHECK-NEXT:                 bl .+33554428
+# CHECK-NEXT:     2000:       bl 0x2018
+# CHECK-NEXT:                 bl 0x2018
+# CHECK-NEXT:                 bl 0x2018
+# CHECK-NEXT:                 bl 0x2002008
 
 ## &.branch_lt[0] - .TOC. = .branch_lt - (.got+0x8000) = -32760
 # CHECK:      <__long_branch_high>:
@@ -63,8 +63,8 @@ blr
 # CHECK-NEXT: <high>:
 # CHECK-NEXT:  2002000:       addis 2, 12, 1
 # CHECK-NEXT:                 addi 2, 2, -32728
-# CHECK-NEXT:                 bl .-33554432
-# CHECK-NEXT:                 bl .+8
+# CHECK-NEXT:                 bl 0x2008
+# CHECK-NEXT:                 bl 0x2002014
 # CHECK:      <__long_branch_>:
 # CHECK-NEXT:  2002014:       addis 12, 2, 0
 # CHECK-NEXT:                 ld 12, -32744(12)
diff --git a/lld/test/ELF/ppc64-plt-stub.s b/lld/test/ELF/ppc64-plt-stub.s
index 9f89d731f2fb6..d21ddbfc7566e 100644
--- a/lld/test/ELF/ppc64-plt-stub.s
+++ b/lld/test/ELF/ppc64-plt-stub.s
@@ -24,7 +24,7 @@
 # RELOC: 0x10030010 R_PPC64_JMP_SLOT foo 0x0
 
 # CHECK:      <_start>:
-# CHECK:      10010298: bl .+16
+# CHECK:      10010298: bl 0x100102a8
 
 # CHECK-LABEL: 00000000100102a8 <__plt_foo>:
 # CHECK-NEXT:      std 2, 24(1)
diff --git a/lld/test/ELF/ppc64-rel-calls.s b/lld/test/ELF/ppc64-rel-calls.s
index 9969239d8e651..366db77197cf4 100644
--- a/lld/test/ELF/ppc64-rel-calls.s
+++ b/lld/test/ELF/ppc64-rel-calls.s
@@ -31,8 +31,8 @@ bar:
   nop
   blr
 
-# CHECK:      10010164: bl .-12
+# CHECK:      10010164: bl 0x10010158
 # CHECK-NEXT:           nop
-# CHECK-NEXT: 1001016c: bl .-20
+# CHECK-NEXT: 1001016c: bl 0x10010158
 # CHECK-NEXT:           nop
 # CHECK-NEXT:           blr
diff --git a/lld/test/ELF/ppc64-reloc-rel.s b/lld/test/ELF/ppc64-reloc-rel.s
index be64a4f767ac3..e086cad7d63cc 100644
--- a/lld/test/ELF/ppc64-reloc-rel.s
+++ b/lld/test/ELF/ppc64-reloc-rel.s
@@ -12,7 +12,7 @@
   beq 1f
 1:
 # CHECK-LABEL: Disassembly of section .R_PPC64_REL14:
-# CHECK: bt 2, .+4
+# CHECK: bt 2, 0x10010198
 
 .section .R_PPC64_REL16,"ax",@progbits
 .globl rel16
@@ -34,7 +34,7 @@ rel16:
 .section .R_PPC64_REL24,"ax",@progbits
   b rel16
 # CHECK-LABEL: Disassembly of section .R_PPC64_REL24:
-# CHECK: b .+67108840
+# CHECK: 100101b0: b 0x10010198
 
 .section .REL32_AND_REL64,"ax",@progbits
   .cfi_startproc
diff --git a/lld/test/ELF/ppc64-split-stack-adjust-overflow.s b/lld/test/ELF/ppc64-split-stack-adjust-overflow.s
index b1a104474f108..f9bbf6176af57 100644
--- a/lld/test/ELF/ppc64-split-stack-adjust-overflow.s
+++ b/lld/test/ELF/ppc64-split-stack-adjust-overflow.s
@@ -59,6 +59,6 @@ caller:
 # CHECK-NEXT: addis 12, 1, -32768
 # CHECK-NEXT: nop
 # CHECK-NEXT: cmpld 7, 12, 0
-# CHECK-NEXT: bt- 28, .+36
+# CHECK-NEXT: bt- 28, 0x10010204
 
 .section        .note.GNU-split-stack,"",@progbits
diff --git a/lld/test/ELF/ppc64-split-stack-adjust-size-success.s b/lld/test/ELF/ppc64-split-stack-adjust-size-success.s
index 63e0b414dc09a..27fbb95c01df3 100644
--- a/lld/test/ELF/ppc64-split-stack-adjust-size-success.s
+++ b/lld/test/ELF/ppc64-split-stack-adjust-size-success.s
@@ -58,21 +58,21 @@ caller:
 # CHECK-NEXT: addis 12, 1, -1
 # CHECK-NEXT: addi 12, 12, 32736
 # CHECK-NEXT: cmpld 7, 12, 0
-# CHECK-NEXT: bt- 28, .+36
+# CHECK-NEXT: bt- 28, 0x10010204
 
 # SMALL-LABEL: caller
 # SMALL:      ld 0, -28736(13)
 # SMALL-NEXT: addi 12, 1, -4128
 # SMALL-NEXT: nop
 # SMALL-NEXT: cmpld 7, 12, 0
-# SMALL-NEXT: bt- 28, .+36
+# SMALL-NEXT: bt- 28, 0x10010204
 
 # ZERO-LABEL: caller
 # ZERO:      ld 0, -28736(13)
 # ZERO-NEXT: addi 12, 1, -32
 # ZERO-NEXT: nop
 # ZERO-NEXT: cmpld 7, 12, 0
-# ZERO-NEXT: bt- 28, .+36
+# ZERO-NEXT: bt- 28, 0x10010204
         .p2align    2
         .global main
 	.type  main, @function
diff --git a/lld/test/ELF/ppc64-split-stack-prologue-adjust-success.s b/lld/test/ELF/ppc64-split-stack-prologue-adjust-success.s
index 197df150c4959..002f80d5cb801 100644
--- a/lld/test/ELF/ppc64-split-stack-prologue-adjust-success.s
+++ b/lld/test/ELF/ppc64-split-stack-prologue-adjust-success.s
@@ -16,7 +16,7 @@
 
 # A caller with a stack that is small enough that the addis instruction
 # from the split-stack prologue is unneeded, and after the prologue adjustment
-# the stack size still fits whithin 16 bits.
+# the stack size still fits within 16 bits.
         .p2align    2
         .global caller_small_stack
         .type caller_small_stack, @function
@@ -54,9 +54,9 @@ caller_small_stack:
 # CHECK-NEXT:  addi 12, 1, -16416
 # CHECK-NEXT:  nop
 # CHECK-NEXT:  cmpld 7, 12, 0
-# CHECK-NEXT:  bt-  28, .+36
+# CHECK-NEXT:  bt-  28, 0x10010204
 
-# A caller that has a stack size that fits whithin 16 bits, but the adjusted
+# A caller that has a stack size that fits within 16 bits, but the adjusted
 # stack size after prologue adjustment now overflows 16 bits needing both addis
 # and addi instructions.
         .p2align    2
@@ -132,7 +132,7 @@ caller_large_stack:
 # CHECK-NEXT:  addis 12, 1, -1
 # CHECK-NEXT:  addi 12, 12, -16416
 # CHECK-NEXT:  cmpld 7, 12, 0
-# CHECK-NEXT:  bt-  28, .+44
+# CHECK-NEXT:  bt-  28, 0x100102bc
 
 # A caller with a stack size that is larger then 16 bits, but aligned such that
 # the addi instruction is unneeded.
@@ -174,7 +174,7 @@ caller_large_aligned_stack:
 # CHECK-NEXT:  addis 12, 1, -2
 # CHECK-NEXT:  addi 12, 12, -16384
 # CHECK-NEXT:  cmpld 7, 12, 0
-# CHECK-NEXT:  bt-  28, .+40
+# CHECK-NEXT:  bt-  28, 0x10010318
 
 # main only calls split-stack functions or __morestack so
 # there should be no adjustment of its split-stack prologue.
diff --git a/lld/test/ELF/ppc64-tls-gd.s b/lld/test/ELF/ppc64-tls-gd.s
index bb90a7f3943f5..90047331762fd 100644
--- a/lld/test/ELF/ppc64-tls-gd.s
+++ b/lld/test/ELF/ppc64-tls-gd.s
@@ -27,18 +27,18 @@
 ## &DTPMOD(a) - .TOC. = &.got[0] - (.got+0x8000) = -32768
 # GD:      addis 3, 2, 0
 # GD-NEXT: addi 3, 3, -32768
-# GD-NEXT: bl .+36
+# GD-NEXT: bl 0x103f4
 # GD-NEXT: ld 2, 24(1)
 
 ## &DTPMOD(b) - .TOC. = &.got[2] - (.got+0x8000) = -32752
 # GD-NEXT: addis 3, 2, 0
 # GD-NEXT: addi 3, 3, -32752
-# GD-NEXT: bl .+20
+# GD-NEXT: bl 0x103f4
 # GD-NEXT: ld 2, 24(1)
 
 ## &DTPMOD(b) - .TOC. = &.got[4] - (.got+0x8000) = -32736
 # GD-NEXT: li 3, -32736
-# GD-NEXT: bl .+8
+# GD-NEXT: bl 0x103f4
 # GD-NEXT: ld 2, 24(1)
 
 # NOREL: no relocations
diff --git a/lld/test/ELF/ppc64-toc-restore-recursive-call.s b/lld/test/ELF/ppc64-toc-restore-recursive-call.s
index aae4a75e909cc..a49705a2b654e 100644
--- a/lld/test/ELF/ppc64-toc-restore-recursive-call.s
+++ b/lld/test/ELF/ppc64-toc-restore-recursive-call.s
@@ -15,7 +15,7 @@
 # with non-recursive calls.
 
 # CHECK-LABEL: 0000000000010290 <recursive_func>:
-# CHECK:       102b8:       bl .+32
+# CHECK:       102b8:       bl 0x102d8
 # CHECK-NEXT:               ld 2, 24(1)
 
 # CHECK-LABEL: 00000000000102d8 <__plt_recursive_func>:
diff --git a/lld/test/ELF/ppc64-toc-restore.s b/lld/test/ELF/ppc64-toc-restore.s
index 4db8358e1268d..dec760903632f 100644
--- a/lld/test/ELF/ppc64-toc-restore.s
+++ b/lld/test/ELF/ppc64-toc-restore.s
@@ -29,9 +29,9 @@ _start:
   nop
   bl bar_local
 // CHECK-LABEL: <_start>:
-// CHECK-NEXT:  100102c8:       bl .+60
+// CHECK-NEXT:  100102c8:       bl 0x10010304
 // CHECK-NEXT:  100102cc:       ld 2, 24(1)
-// CHECK-NEXT:  100102d0:       bl .-16
+// CHECK-NEXT:  100102d0:       bl 0x100102c0
 // CHECK-EMPTY:
 
 # Calling a function in another object file which will have same
@@ -43,8 +43,8 @@ _diff_object:
   bl foo_not_shared
   nop
 // CHECK-LABEL: <_diff_object>:
-// CHECK-NEXT:  100102d4:       bl .+28
-// CHECK-NEXT:  100102d8:       bl .+24
+// CHECK-NEXT:  100102d4:       bl 0x100102f0
+// CHECK-NEXT:  100102d8:       bl 0x100102f0
 // CHECK-NEXT:  100102dc:       nop
 
 # Branching to a local function does not need a nop
@@ -52,7 +52,7 @@ _diff_object:
 noretbranch:
   b bar_local
 // CHECK-LABEL: <noretbranch>:
-// CHECK:       100102e0:        b .+67108832
+// CHECK-NEXT:  100102e0:        b 0x100102c0
 // CHECK-EMPTY:
 
 // This should come last to check the end-of-buffer condition.
@@ -61,5 +61,5 @@ last:
   bl foo
   nop
 // CHECK-LABEL: <last>:
-// CHECK-NEXT:  100102e4:       bl .+32
+// CHECK-NEXT:  100102e4:       bl 0x10010304
 // CHECK-NEXT:  100102e8:       ld 2, 24(1)
diff --git a/lld/test/ELF/ppc64-weak-undef-call.s b/lld/test/ELF/ppc64-weak-undef-call.s
index 018bc7e6a25ed..1f6869311708c 100644
--- a/lld/test/ELF/ppc64-weak-undef-call.s
+++ b/lld/test/ELF/ppc64-weak-undef-call.s
@@ -24,6 +24,6 @@ _start:
 # be unreachable. But, we should link successfully. We should not, however,
 # generate a .plt entry (this would be wasted space). For now, we do nothing
 # (leaving the zero relative offset present in the input).
-# CHECK: 10010158:       bl .+0
+# CHECK: 10010158:       bl 0x10010158
 # CHECK: 1001015c:       nop
 # CHECK: 10010160:       blr
diff --git a/lld/test/ELF/pre_init_fini_array.s b/lld/test/ELF/pre_init_fini_array.s
index 95bc4778c1e06..d72001edb812f 100644
--- a/lld/test/ELF/pre_init_fini_array.s
+++ b/lld/test/ELF/pre_init_fini_array.s
@@ -3,17 +3,17 @@
 // RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %p/Inputs/shared.s -o %t2
 // RUN: ld.lld %t2 -o %t2.so -shared -soname=so
 // RUN: ld.lld %t %t2.so -o %t2
-// RUN: llvm-readobj -r --symbols --sections --dynamic-table %t2 | FileCheck %s
-// RUN: llvm-objdump -d %t2 | FileCheck --check-prefix=DISASM %s
+// RUN: llvm-readelf -S -d -r -s %t2 | FileCheck %s
+// RUN: llvm-objdump -d --syms %t2 | FileCheck --check-prefix=DISASM %s
 
 .globl _start
 _start:
-  call __preinit_array_start
-  call __preinit_array_end
-  call __init_array_start
-  call __init_array_end
-  call __fini_array_start
-  call __fini_array_end
+  call *__preinit_array_start
+  call *__preinit_array_end
+  call *__init_array_start
+  call *__init_array_end
+  call *__fini_array_start
+  call *__fini_array_end
 
 
 .section .init_array,"aw",@init_array
@@ -27,119 +27,43 @@ _start:
         .quad 0
         .short 0
 
-// CHECK:      Name: .init_array
-// CHECK-NEXT: Type: SHT_INIT_ARRAY
-// CHECK-NEXT: Flags [
-// CHECK-NEXT:   SHF_ALLOC
-// CHECK-NEXT:   SHF_WRITE
-// CHECK-NEXT: ]
-// CHECK-NEXT: Address: [[INIT_ADDR:.*]]
-// CHECK-NEXT: Offset:
-// CHECK-NEXT: Size: [[INIT_SIZE:.*]]
+// CHECK-LABEL: Section Headers:
+// CHECK:      Name           Type          Address                 Off       Size                    ES Flg
+// CHECK:      .init_array    INIT_ARRAY    [[# %x, INIT_ADDR:]]    [[# %x,]] [[# %x, INIT_SIZE:]]    00 WA
+// CHECK-NEXT: .preinit_array PREINIT_ARRAY [[# %x, PREINIT_ADDR:]] [[# %x,]] [[# %x, PREINIT_SIZE:]] 00 WA
+// CHECK-NEXT: .fini_array    FINI_ARRAY    [[# %x, FINI_ADDR:]]    [[# %x,]] [[# %x, FINI_SIZE:]]    00 WA
 
+// CHECK-LABEL: Dynamic section
+// CHECK: (PREINIT_ARRAY)        0x[[# PREINIT_ADDR]]
+// CHECK: (PREINIT_ARRAYSZ)      [[# %u, PREINIT_SIZE]] (bytes)
+// CHECK: (INIT_ARRAY)           0x[[# INIT_ADDR]]
+// CHECK: (INIT_ARRAYSZ)         [[# %u, INIT_SIZE]] (bytes)
+// CHECK: (FINI_ARRAY)           0x[[# FINI_ADDR]]
+// CHECK: (FINI_ARRAYSZ)         [[# %u, FINI_SIZE]] (bytes)
 
-// CHECK:     Name: .preinit_array
-// CHECK-NEXT: Type: SHT_PREINIT_ARRAY
-// CHECK-NEXT: Flags [
-// CHECK-NEXT:   SHF_ALLOC
-// CHECK-NEXT:   SHF_WRITE
-// CHECK-NEXT:    ]
-// CHECK-NEXT: Address: [[PREINIT_ADDR:.*]]
-// CHECK-NEXT: Offset:
-// CHECK-NEXT: Size: [[PREINIT_SIZE:.*]]
+// CHECK-LABEL:      There are no relocations in this file.
 
+// CHECK-LABEL: Symbol table '.symtab'
+// CHECK:       Value                             Size Type    Bind   Vis       Ndx   Name
+// CHECK:       [[# FINI_ADDR + FINI_SIZE]]       0    NOTYPE  LOCAL  HIDDEN    [[#]] __fini_array_end
+// CHECK-NEXT:  [[# FINI_ADDR]]                   0    NOTYPE  LOCAL  HIDDEN    [[#]] __fini_array_start
+// CHECK-NEXT:  [[# INIT_ADDR + INIT_SIZE]]       0    NOTYPE  LOCAL  HIDDEN    [[#]] __init_array_end
+// CHECK-NEXT:  [[# INIT_ADDR]]                   0    NOTYPE  LOCAL  HIDDEN    [[#]] __init_array_start
+// CHECK-NEXT:  [[# PREINIT_ADDR + PREINIT_SIZE]] 0    NOTYPE  LOCAL  HIDDEN    [[#]] __preinit_array_end
+// CHECK-NEXT:  [[# PREINIT_ADDR]]                0    NOTYPE  LOCAL  HIDDEN    [[#]] __preinit_array_start
 
-// CHECK:      Name: .fini_array
-// CHECK-NEXT: Type: SHT_FINI_ARRAY
-// CHECK-NEXT: Flags [
-// CHECK-NEXT:   SHF_ALLOC
-// CHECK-NEXT:   SHF_WRITE
-// CHECK-NEXT: ]
-// CHECK-NEXT: Address: [[FINI_ADDR:.*]]
-// CHECK-NEXT: Offset:
-// CHECK-NEXT: Size: [[FINI_SIZE:.*]]
-
-// CHECK: DynamicSection
-// CHECK: PREINIT_ARRAY        [[PREINIT_ADDR]]
-// CHECK: PREINIT_ARRAYSZ      [[PREINIT_SIZE]] (bytes)
-// CHECK: INIT_ARRAY           [[INIT_ADDR]]
-// CHECK: INIT_ARRAYSZ         [[INIT_SIZE]] (bytes)
-// CHECK: FINI_ARRAY           [[FINI_ADDR]]
-// CHECK: FINI_ARRAYSZ         [[FINI_SIZE]] (bytes)
-
-// CHECK:      Relocations [
-// CHECK-NEXT: ]
-
-// CHECK:        Name: __fini_array_end
-// CHECK-NEXT:   Value:
-// CHECK-NEXT:   Size: 0
-// CHECK-NEXT:   Binding: Local
-// CHECK-NEXT:   Type: None
-// CHECK-NEXT:   Other [
-// CHECK-NEXT:     STV_HIDDEN
-// CHECK-NEXT:   ]
-// CHECK-NEXT:   Section: .fini_array
-// CHECK-NEXT: }
-// CHECK-NEXT: Symbol {
-// CHECK-NEXT:   Name: __fini_array_start
-// CHECK-NEXT:   Value: [[FINI_ADDR]]
-// CHECK-NEXT:   Size: 0
-// CHECK-NEXT:   Binding: Local
-// CHECK-NEXT:   Type: None
-// CHECK-NEXT:   Other [
-// CHECK-NEXT:     STV_HIDDEN
-// CHECK-NEXT:   ]
-// CHECK-NEXT:   Section: .fini_array
-// CHECK-NEXT: }
-// CHECK-NEXT: Symbol {
-// CHECK-NEXT:   Name: __init_array_end
-// CHECK-NEXT:   Value:
-// CHECK-NEXT:   Size: 0
-// CHECK-NEXT:   Binding: Local
-// CHECK-NEXT:   Type: None
-// CHECK-NEXT:   Other [
-// CHECK-NEXT:     STV_HIDDEN
-// CHECK-NEXT:   ]
-// CHECK-NEXT:   Section: .init_array
-// CHECK-NEXT: }
-// CHECK-NEXT: Symbol {
-// CHECK-NEXT:   Name: __init_array_start
-// CHECK-NEXT:   Value: [[INIT_ADDR]]
-// CHECK-NEXT:   Size: 0
-// CHECK-NEXT:   Binding: Local
-// CHECK-NEXT:   Type: None
-// CHECK-NEXT:   Other [
-// CHECK-NEXT:     STV_HIDDEN
-// CHECK-NEXT:   ]
-// CHECK-NEXT:   Section: .init_array
-// CHECK-NEXT: }
-// CHECK-NEXT: Symbol {
-// CHECK-NEXT:   Name: __preinit_array_end
-// CHECK-NEXT:   Value:
-// CHECK-NEXT:   Size: 0
-// CHECK-NEXT:   Binding: Local
-// CHECK-NEXT:   Type: None
-// CHECK-NEXT:   Other [
-// CHECK-NEXT:     STV_HIDDEN
-// CHECK-NEXT:   ]
-// CHECK-NEXT:   Section: .preinit_array
-// CHECK-NEXT: }
-// CHECK-NEXT: Symbol {
-// CHECK-NEXT:   Name: __preinit_array_start
-// CHECK-NEXT:   Value: [[PREINIT_ADDR]]
-// CHECK-NEXT:   Size: 0
-// CHECK-NEXT:   Binding: Local
-// CHECK-NEXT:   Type: None
-// CHECK-NEXT:   Other [
-// CHECK-NEXT:     STV_HIDDEN
-// CHECK-NEXT:   ]
-// CHECK-NEXT:   Section: .preinit_array
-// CHECK-NEXT: }
+// DISASM:      SYMBOL TABLE:
+// DISASM-DAG: {{0*}}[[# %x, PREINIT_ARRAY_START:]]  l  .preinit_array  {{0+}}  .hidden  __preinit_array_start
+// DISASM-DAG: {{0*}}[[# %x, PREINIT_ARRAY_END:]]    l  .preinit_array  {{0+}}  .hidden  __preinit_array_end
+// DISASM-DAG: {{0*}}[[# %x, INIT_ARRAY_START:]]     l  .init_array  {{0+}}  .hidden  __init_array_start
+// DISASM-DAG: {{0*}}[[# %x, INIT_ARRAY_END:]]       l  .init_array  {{0+}}  .hidden  __init_array_end
+// DISASM-DAG: {{0*}}[[# %x, FINI_ARRAY_START:]]     l  .fini_array  {{0+}}  .hidden  __fini_array_start
+// DISASM-DAG: {{0*}}[[# %x, FINI_ARRAY_END:]]       l  .fini_array  {{0+}}  .hidden  __fini_array_end
 
 // DISASM:      <_start>:
-// DISASM-NEXT:   callq   {{.*}} <__preinit_array_start>
-// DISASM-NEXT:   callq   {{.*}} <__fini_array_start>
-// DISASM-NEXT:   callq   {{.*}} <__init_array_start>
-// DISASM-NEXT:   callq   {{.*}} <__preinit_array_start>
-// DISASM-NEXT:   callq   {{.*}} <__fini_array_start>
-// DISASM-NEXT:   callq   {{.*}} <__fini_array_end>
+// DISASM-NEXT:   callq   *[[# %u, PREINIT_ARRAY_START]]
+// DISASM-NEXT:   callq   *[[# %u, PREINIT_ARRAY_END]]
+// DISASM-NEXT:   callq   *[[# %u, INIT_ARRAY_START]]
+// DISASM-NEXT:   callq   *[[# %u, INIT_ARRAY_END]]
+// DISASM-NEXT:   callq   *[[# %u, FINI_ARRAY_START]]
+// DISASM-NEXT:   callq   *[[# %u, FINI_ARRAY_END]]
diff --git a/lld/test/ELF/pre_init_fini_array_missing.s b/lld/test/ELF/pre_init_fini_array_missing.s
index 5f7c02bfb81d1..22cf5fe9e2bea 100644
--- a/lld/test/ELF/pre_init_fini_array_missing.s
+++ b/lld/test/ELF/pre_init_fini_array_missing.s
@@ -22,21 +22,21 @@ _start:
 // CHECK: Disassembly of section .text:
 // CHECK-EMPTY:
 // CHECK-NEXT:  <_start>:
-// CHECK-NEXT:   201120:       callq    -5
-// CHECK-NEXT:                 callq    -10
-// CHECK-NEXT:                 callq    -15
-// CHECK-NEXT:                 callq    -20
-// CHECK-NEXT:                 callq    -25
-// CHECK-NEXT:                 callq    -30
+// CHECK-NEXT:   201120:       callq    0x201120
+// CHECK-NEXT:                 callq    0x201120
+// CHECK-NEXT:                 callq    0x201120
+// CHECK-NEXT:                 callq    0x201120
+// CHECK-NEXT:                 callq    0x201120
+// CHECK-NEXT:                 callq    0x201120
 
 // In position-independent binaries, they resolve to .text too.
 
 // PIE:      Disassembly of section .text:
 // PIE-EMPTY:
 // PIE-NEXT: <_start>:
-// PIE-NEXT:     1210:       callq   -5
-// PIE-NEXT:                 callq   -10
-// PIE-NEXT:                 callq   -15
-// PIE-NEXT:                 callq   -20
-// PIE-NEXT:                 callq   -25
-// PIE-NEXT:                 callq   -30
+// PIE-NEXT:     1210:       callq   0x1210
+// PIE-NEXT:                 callq   0x1210
+// PIE-NEXT:                 callq   0x1210
+// PIE-NEXT:                 callq   0x1210
+// PIE-NEXT:                 callq   0x1210
+// PIE-NEXT:                 callq   0x1210
diff --git a/lld/test/ELF/relocatable-linkorder.s b/lld/test/ELF/relocatable-linkorder.s
deleted file mode 100644
index 0a7dfc6c12a2c..0000000000000
--- a/lld/test/ELF/relocatable-linkorder.s
+++ /dev/null
@@ -1,36 +0,0 @@
-// REQUIRES: x86
-// RUN: llvm-mc %s -o %t.o -filetype=obj --triple=x86_64-unknown-linux
-// RUN: ld.lld %t.o -o %t -r
-// RUN: llvm-readelf -S %t | FileCheck --check-prefix=DIFFERENT %s
-// RUN: echo 'SECTIONS { .text.f1 : { *(.text.f1) } .text.f2 : { *(.text.f2) } }' > %t.lds
-// RUN: ld.lld %t.o -o %t -r %t.lds
-// RUN: llvm-readelf -S %t | FileCheck --check-prefix=DIFFERENT %s
-// RUN: echo 'SECTIONS { .text : { *(.text.f1) *(.text.f2) } }' > %t.lds
-// RUN: ld.lld %t.o -o %t -r %t.lds
-// RUN: llvm-readelf -S -x foo %t | FileCheck --check-prefix=SAME %s
-
-/// Test that SHF_LINK_ORDER sections with different linked sections
-/// aren't merged.
-
-.section .text.f1,"ax",@progbits
-.globl f1
-f1:
-ret
-
-.section .text.f2,"ax",@progbits
-.globl f2
-f2:
-ret
-
-// SAME: foo
-// DIFFERENT: foo
-.section foo,"ao",@progbits,.text.f2,unique,2
-.quad 2
-
-// SAME-NOT: foo
-// DIFFERENT: foo
-.section foo,"ao",@progbits,.text.f1,unique,1
-.quad 1
-
-// SAME: Hex dump of section 'foo':
-// SAME: 01000000 00000000 02000000 00000000
diff --git a/lld/test/ELF/relocation-i686.s b/lld/test/ELF/relocation-i686.s
index 3ae6f4dec64af..bbaaf1c3434a3 100644
--- a/lld/test/ELF/relocation-i686.s
+++ b/lld/test/ELF/relocation-i686.s
@@ -33,7 +33,7 @@ R_386_PC32_2:
 // CHECK: Disassembly of section .R_386_PC32:
 // CHECK-EMPTY:
 // CHECK-NEXT: <R_386_PC32>:
-// CHECK-NEXT:   calll 4
+// CHECK-NEXT:   calll 0x4011ba
 
 // CHECK:      <R_386_PC32_2>:
 // CHECK-NEXT:   nop
@@ -77,7 +77,7 @@ R_386_GOTPC:
 // CHECK:      Disassembly of section .dynamic_reloc:
 // CHECK-EMPTY:
 // CHECK-NEXT: <.dynamic_reloc>:
-// CHECK-NEXT:   4011c5:       calll 38 <bar@plt>
+// CHECK-NEXT:   4011c5:       calll 0x4011f0 <bar@plt>
 
 .section .R_386_GOT32,"ax",@progbits
 .global R_386_GOT32
diff --git a/lld/test/ELF/relocation-relative-weak.s b/lld/test/ELF/relocation-relative-weak.s
index 5ae5d1df7f747..18dcb0530a9ca 100644
--- a/lld/test/ELF/relocation-relative-weak.s
+++ b/lld/test/ELF/relocation-relative-weak.s
@@ -12,6 +12,5 @@
 .globl _start
 _start:
 
-.globl w
 .weak w
 call w@PLT
diff --git a/lld/test/ELF/relocation.s b/lld/test/ELF/relocation.s
index 7a7eaef06abf6..f2dcbac3db527 100644
--- a/lld/test/ELF/relocation.s
+++ b/lld/test/ELF/relocation.s
@@ -93,7 +93,7 @@ lulz:
 // CHECK: Disassembly of section .text:
 // CHECK-EMPTY:
 // CHECK-NEXT: <_start>:
-// CHECK-NEXT:   201310:  e8 04 00 00 00   callq 4
+// CHECK-NEXT:   201310:  e8 04 00 00 00   callq 0x201319
 // CHECK-NEXT:   201315:
 
 // CHECK:      <lulz>:
@@ -132,7 +132,7 @@ R_X86_64_PC32:
 // CHECK:      Disassembly of section .R_X86_64_PC32:
 // CHECK-EMPTY:
 // CHECK-NEXT: <R_X86_64_PC32>:
-// CHECK-NEXT:  201327:   {{.*}}  callq  36
+// CHECK-NEXT:  201327:   {{.*}}  callq  0x201350
 // CHECK-NEXT:  20132c:   {{.*}}  movl $2102096, %eax
 
 .section .R_X86_64_32S_2,"ax",@progbits
diff --git a/lld/test/ELF/relro-init-fini-script.s b/lld/test/ELF/relro-init-fini-script.s
new file mode 100644
index 0000000000000..ab7c928d49b93
--- /dev/null
+++ b/lld/test/ELF/relro-init-fini-script.s
@@ -0,0 +1,39 @@
+// REQUIRES: x86
+// RUN: llvm-mc -filetype=obj -triple=x86_64-none-elf %s -o %t.o
+// RUN: echo "SECTIONS { \
+// RUN:  .dynamic 0x10000 : { *(.dynamic) } \
+// RUN:  .preinit_array : { PROVIDE_HIDDEN (__preinit_array_start = .); \
+// RUN:                  KEEP (*(.preinit_array)) } \
+// RUN:  .init_array : { PROVIDE_HIDDEN (__init_array_start = .); \
+// RUN:                  KEEP (*(.init_array)) } \
+// RUN:  .fini_array : { PROVIDE_HIDDEN (__fini_array_start = .); \
+// RUN:                  KEEP (*(.fini_array)) } \
+// RUN:  .data.rel.ro : { *(.data.rel.ro) } \
+// RUN:  .data : { *(.data) } } " > %t.script
+// RUN: ld.lld %t.o -o %t.so --shared --script=%t.script
+// RUN: llvm-readelf -S %t.so | FileCheck %s
+// RUN: llvm-readobj --segments %t.so | FileCheck %s --check-prefix=PHDR
+
+/// Check that an empty .init_array, .fini_array or .preinit_array that is
+/// kept due to symbol references, is still counted as RELRO. The _array
+/// sections are zero size. The RELRO extent is [.dynamic, .data.rel.ro)
+
+// CHECK:      .dynamic       DYNAMIC         0000000000010000 002000 000110
+// CHECK-NEXT: .preinit_array PROGBITS        {{0+}}[[# %x,ADDR:]]
+// CHECK-NEXT: .init_array    PROGBITS        {{0+}}[[# ADDR]]
+// CHECK-NEXT: .fini_array    PROGBITS        {{0+}}[[# ADDR]]
+// CHECK-NEXT: .data.rel.ro   PROGBITS        0000000000010110 002110 000008
+
+// PHDR:      Type: PT_GNU_RELRO
+// PHDR-NEXT: Offset: 0x2000
+// PHDR-NEXT: VirtualAddress: 0x10000
+// PHDR-NEXT: PhysicalAddress: 0x10000
+// PHDR-NEXT: FileSize: 280
+ .section .data.rel.ro, "aw", %progbits
+ .global foo
+ .quad foo
+
+ .data
+ .quad __init_array_start
+ .quad __fini_array_start
+ .quad __preinit_array_start
diff --git a/lld/test/ELF/startstop.s b/lld/test/ELF/startstop.s
index 9eddd4cc91f44..86f203b84ff1e 100644
--- a/lld/test/ELF/startstop.s
+++ b/lld/test/ELF/startstop.s
@@ -5,9 +5,9 @@
 // RUN: llvm-readobj --symbols -r %t.so | FileCheck -check-prefix=SYMBOL %s
 
 // DISASM: <_start>:
-// DISASM:    1330:       callq   10 <__start_foo>
-// DISASM:    1335:       callq   8 <__start_bar>
-// DISASM:    133a:       callq   3 <__start_bar>
+// DISASM:    1330:       callq   0x133f <__start_foo>
+// DISASM:    1335:       callq   0x1342 <__start_bar>
+// DISASM:    133a:       callq   0x1342 <__start_bar>
 // DISASM: Disassembly of section foo:
 // DISASM-EMPTY:
 // DISASM: <__start_foo>:
diff --git a/lld/test/ELF/threads.s b/lld/test/ELF/threads.s
new file mode 100644
index 0000000000000..7a9eeec00e08a
--- /dev/null
+++ b/lld/test/ELF/threads.s
@@ -0,0 +1,12 @@
+# REQUIRES: x86
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o
+
+## A positive integer is allowed.
+# RUN: ld.lld --threads=1 %t.o -o /dev/null
+# RUN: ld.lld --threads=2 %t.o -o /dev/null
+
+# RUN: not ld.lld --threads=all %t.o -o /dev/null 2>&1 | FileCheck %s -DN=all
+# RUN: not ld.lld --threads=0 %t.o -o /dev/null 2>&1 | FileCheck %s -DN=0
+# RUN: not ld.lld --threads=-1 %t.o -o /dev/null 2>&1 | FileCheck %s -DN=-1
+
+# CHECK: error: --threads: expected a positive integer, but got '[[N]]'
diff --git a/lld/test/ELF/undef-multi.s b/lld/test/ELF/undef-multi.s
index 627235b37b15d..bc1b0623fa580 100644
--- a/lld/test/ELF/undef-multi.s
+++ b/lld/test/ELF/undef-multi.s
@@ -10,9 +10,7 @@
 # CHECK-NEXT: >>>               {{.*}}:(.text+0x6)
 # CHECK-NEXT: >>> referenced by undef-multi.s
 # CHECK-NEXT: >>>               {{.*}}:(.text+0xB)
-# CHECK-NEXT: >>> referenced by undef-multi.s
-# CHECK-NEXT: >>>               {{.*}}:(.text+0x10)
-# CHECK-NEXT: >>> referenced by {{.*}}tmp2.o:(.text+0x0)
+# CHECK-NEXT: >>> referenced 2 more times
 
 # All references to a single undefined symbol count as a single error -- but
 # at most 10 references are printed.
@@ -36,15 +34,7 @@
 # LIMIT-NEXT: >>>               {{.*}}:(.text+0x6)
 # LIMIT-NEXT: >>> referenced by undef-multi.s
 # LIMIT-NEXT: >>>               {{.*}}:(.text+0xB)
-# LIMIT-NEXT: >>> referenced by undef-multi.s
-# LIMIT-NEXT: >>>               {{.*}}:(.text+0x10)
-# LIMIT-NEXT: >>> referenced by {{.*}}tmp2.o:(.text+0x0)
-# LIMIT-NEXT: >>> referenced by {{.*}}tmp3.o:(.text+0x1)
-# LIMIT-NEXT: >>> referenced by {{.*}}tmp3.o:(.text+0x6)
-# LIMIT-NEXT: >>> referenced by {{.*}}tmp3.o:(.text+0xB)
-# LIMIT-NEXT: >>> referenced by {{.*}}tmp3.o:(.text+0x10)
-# LIMIT-NEXT: >>> referenced by {{.*}}tmp3.o:(.text+0x15)
-# LIMIT-NEXT: >>> referenced 2 more times
+# LIMIT-NEXT: >>> referenced 9 more times
 
 .file "undef-multi.s"
 
diff --git a/lld/test/ELF/undef-suggest-version.s b/lld/test/ELF/undef-suggest-version.s
new file mode 100644
index 0000000000000..790b9fcab788c
--- /dev/null
+++ b/lld/test/ELF/undef-suggest-version.s
@@ -0,0 +1,57 @@
+# REQUIRES: x86
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o
+# RUN: echo 'v1 {bar;};' > %t.ver
+# RUN: ld.lld -shared --version-script %t.ver %t.o -o %t.so
+
+## For an unversioned undefined symbol, check we can suggest the symbol with the
+## default version.
+# RUN: echo 'call bat' | llvm-mc -filetype=obj -triple=x86_64 - -o %tdef1.o
+# RUN: not ld.lld %t.so %tdef1.o -o /dev/null 2>&1 | FileCheck --check-prefix=DEFAULT1 %s
+
+# DEFAULT1:      error: undefined symbol: bat
+# DEFAULT1-NEXT: >>> referenced by {{.*}}.o:(.text+0x1)
+# DEFAULT1-NEXT: >>> did you mean: bar{{$}}
+# DEFAULT1-NEXT: >>> defined in: {{.*}}.so
+
+## For a versioned undefined symbol, check we can suggest the symbol with the
+## default version.
+# RUN: echo '.symver bar.v2,bar@v2; call bar.v2' | llvm-mc -filetype=obj -triple=x86_64 - -o %tdef2.o
+# RUN: not ld.lld %t.so %tdef2.o -o /dev/null 2>&1 | FileCheck --check-prefix=DEFAULT2 %s
+
+# DEFAULT2:      error: undefined symbol: bar@v2
+# DEFAULT2-NEXT: >>> referenced by {{.*}}.o:(.text+0x1)
+# DEFAULT2-NEXT: >>> did you mean: bar{{$}}
+# DEFAULT2-NEXT: >>> defined in: {{.*}}.so
+
+## For an unversioned undefined symbol, check we can suggest a symbol with
+## a non-default version.
+# RUN: echo 'call foo; call _Z3fooi' | llvm-mc -filetype=obj -triple=x86_64 - -o %thidden1.o
+# RUN: not ld.lld %t.so %thidden1.o -o /dev/null 2>&1 | FileCheck --check-prefix=HIDDEN1 %s
+
+# HIDDEN1:      error: undefined symbol: foo
+# HIDDEN1-NEXT: >>> referenced by {{.*}}.o:(.text+0x1)
+# HIDDEN1-NEXT: >>> did you mean: foo@v1
+# HIDDEN1-NEXT: >>> defined in: {{.*}}.so
+# HIDDEN1-EMPTY:
+# HIDDEN1-NEXT: error: undefined symbol: foo(int)
+# HIDDEN1-NEXT: >>> referenced by {{.*}}.o:(.text+0x6)
+# HIDDEN1-NEXT: >>> did you mean: foo(int)@v1
+# HIDDEN1-NEXT: >>> defined in: {{.*}}.so
+
+## For a versioned undefined symbol, check we can suggest a symbol with
+## a different version.
+# RUN: echo '.symver foo.v2,foo@v2; call foo.v2' | llvm-mc -filetype=obj -triple=x86_64 - -o %thidden2.o
+# RUN: not ld.lld %t.so %thidden2.o -o /dev/null 2>&1 | FileCheck --check-prefix=HIDDEN2 %s
+
+# HIDDEN2:      error: undefined symbol: foo@v2
+# HIDDEN2-NEXT: >>> referenced by {{.*}}.o:(.text+0x1)
+# HIDDEN2-NEXT: >>> did you mean: foo@v1
+# HIDDEN2-NEXT: >>> defined in: {{.*}}.so
+
+## %t.so exports bar@@v1 and two VERSYM_HIDDEN symbols: foo@v1 and _Z3fooi@v1.
+.globl foo.v1, _Z3fooi.v1, bar
+.symver foo.v1,foo@v1
+.symver _Z3fooi.v1,_Z3fooi@v1
+foo.v1:
+_Z3fooi.v1:
+bar:
diff --git a/lld/test/ELF/undef.s b/lld/test/ELF/undef.s
index 2e5ed25dec499..db444e3615277 100644
--- a/lld/test/ELF/undef.s
+++ b/lld/test/ELF/undef.s
@@ -25,7 +25,7 @@
 # CHECK:      error: undefined symbol: vtable for Foo 
 # CHECK-NEXT: >>> referenced by undef.s
 # CHECK-NEXT: >>>               {{.*}}:(.text+0x15)
-# CHECK-NEXT: the vtable symbol may be undefined because the class is missing its key function (see https://lld.llvm.org/missingkeyfunction)
+# CHECK-NEXT: >>> the vtable symbol may be undefined because the class is missing its key function (see https://lld.llvm.org/missingkeyfunction)
 
 # Check that this symbol isn't demangled
 
diff --git a/lld/test/ELF/warn-backrefs.s b/lld/test/ELF/warn-backrefs.s
index 629bd3be2fd99..8a9ee2291ecef 100644
--- a/lld/test/ELF/warn-backrefs.s
+++ b/lld/test/ELF/warn-backrefs.s
@@ -39,7 +39,7 @@
 # RUN: echo ".globl foo; foo: call bar" | llvm-mc -filetype=obj -triple=x86_64-unknown-linux - -o %t4.o
 # RUN: ld.lld --fatal-warnings --warn-backrefs %t1.o --start-lib %t3.o %t4.o --end-lib -o /dev/null
 
-# We don't report backward references to weak symbols as they can be overriden later.
+# We don't report backward references to weak symbols as they can be overridden later.
 # RUN: echo ".weak foo; foo:" | llvm-mc -filetype=obj -triple=x86_64-unknown-linux - -o %t5.o
 # RUN: ld.lld --fatal-warnings --warn-backrefs --start-lib %t5.o --end-lib %t1.o %t2.o -o /dev/null
 
diff --git a/lld/test/ELF/x86-64-feature-cet.s b/lld/test/ELF/x86-64-feature-cet.s
index 0361a291c4c8c..7cd26696dc5ae 100644
--- a/lld/test/ELF/x86-64-feature-cet.s
+++ b/lld/test/ELF/x86-64-feature-cet.s
@@ -42,8 +42,8 @@
 
 # DISASM:      Disassembly of section .text:
 # DISASM:      0000000000201330 <func1>:
-# DISASM-NEXT: 201330:       callq   0x2b <func2+0x201360>
-# DISASM-NEXT: 201335:       callq   0x36 <func2+0x201370>
+# DISASM-NEXT: 201330:       callq   0x201360 <func2+0x201360>
+# DISASM-NEXT: 201335:       callq   0x201370 <func2+0x201370>
 # DISASM-NEXT:               retq
 
 # DISASM:      Disassembly of section .plt:
@@ -53,7 +53,7 @@
 # DISASM-NEXT:               nopl    (%rax)
 # DISASM-NEXT:               endbr64
 # DISASM-NEXT:               pushq   $0x0
-# DISASM-NEXT:               jmp     -0x1e <.plt>
+# DISASM-NEXT:               jmp     0x201340 <.plt>
 # DISASM-NEXT:               nop
 
 # DISASM:      Disassembly of section .plt.sec:
diff --git a/lld/test/ELF/x86-64-gotpc-relax.s b/lld/test/ELF/x86-64-gotpc-relax.s
index 0541bb66baa88..66aca36b21367 100644
--- a/lld/test/ELF/x86-64-gotpc-relax.s
+++ b/lld/test/ELF/x86-64-gotpc-relax.s
@@ -33,19 +33,19 @@
 # DISASM-NEXT: leaq -72(%rip), %rax
 # DISASM-NEXT: movq 4192(%rip), %rax
 # DISASM-NEXT: movq 4185(%rip), %rax
-# DISASM-NEXT: callq -93 <foo>
-# DISASM-NEXT: callq -99 <foo>
-# DISASM-NEXT: callq -104 <hid>
-# DISASM-NEXT: callq -110 <hid>
+# DISASM-NEXT: callq 0x201170 <foo>
+# DISASM-NEXT: callq 0x201170 <foo>
+# DISASM-NEXT: callq 0x201171 <hid>
+# DISASM-NEXT: callq 0x201171 <hid>
 # DISASM-NEXT: callq *4155(%rip)
 # DISASM-NEXT: callq *4149(%rip)
-# DISASM-NEXT: jmp   -128 <foo>
+# DISASM-NEXT: jmp   0x201170 <foo>
 # DISASM-NEXT: nop
-# DISASM-NEXT: jmp   -134 <foo>
+# DISASM-NEXT: jmp   0x201170 <foo>
 # DISASM-NEXT: nop
-# DISASM-NEXT: jmp   -139 <hid>
+# DISASM-NEXT: jmp   0x201171 <hid>
 # DISASM-NEXT: nop
-# DISASM-NEXT: jmp   -145 <hid>
+# DISASM-NEXT: jmp   0x201171 <hid>
 # DISASM-NEXT: nop
 # DISASM-NEXT: jmpq  *4119(%rip)
 # DISASM-NEXT: jmpq  *4113(%rip)
diff --git a/lld/test/ELF/x86-64-plt.s b/lld/test/ELF/x86-64-plt.s
index 72cfb8c1f526f..cd0567405b3e7 100644
--- a/lld/test/ELF/x86-64-plt.s
+++ b/lld/test/ELF/x86-64-plt.s
@@ -70,17 +70,17 @@
 // DISASM-NEXT:   <bar@plt>:
 // DISASM-NEXT:   1330:       jmpq *8450(%rip)
 // DISASM-NEXT:               pushq $0
-// DISASM-NEXT:               jmp -32 <.plt>
+// DISASM-NEXT:               jmp 0x1320 <.plt>
 // DISASM-EMPTY:
 // DISASM-NEXT:   <zed@plt>:
 // DISASM-NEXT:   1340:       jmpq *8442(%rip)
 // DISASM-NEXT:               pushq $1
-// DISASM-NEXT:               jmp -48 <.plt>
+// DISASM-NEXT:               jmp 0x1320 <.plt>
 // DISASM-EMPTY:
 // DISASM-NEXT:   <_start@plt>:
 // DISASM-NEXT:   1350:       jmpq *8434(%rip)
 // DISASM-NEXT:               pushq $2
-// DISASM-NEXT:               jmp -64 <.plt>
+// DISASM-NEXT:               jmp 0x1320 <.plt>
 
 // 0x201030 - (0x201000 + 1) - 4 = 43
 // 0x201030 - (0x201005 + 1) - 4 = 38
@@ -88,10 +88,10 @@
 // 0x201000 - (0x20100f + 1) - 4 = -20
 
 // DISASM2:      <_start>:
-// DISASM2-NEXT:   jmp  43 <bar@plt>
-// DISASM2-NEXT:   jmp  38 <bar@plt>
-// DISASM2-NEXT:   jmp  49 <zed@plt>
-// DISASM2-NEXT:   jmp  -20 <_start>
+// DISASM2-NEXT:   jmp  0x2012f0 <bar@plt>
+// DISASM2-NEXT:   jmp  0x2012f0 <bar@plt>
+// DISASM2-NEXT:   jmp  0x201300 <zed@plt>
+// DISASM2-NEXT:   jmp  0x2012c0 <_start>
 
 // 0x202018 - 0x201036  = 4066
 // 0x202020 - 0x201046  = 4058
@@ -106,12 +106,12 @@
 // DISASM2-NEXT: <bar@plt>:
 // DISASM2-NEXT:  2012f0:       jmpq *8450(%rip)
 // DISASM2-NEXT:                pushq $0
-// DISASM2-NEXT:                jmp -32 <.plt>
+// DISASM2-NEXT:                jmp 0x2012e0 <.plt>
 // DISASM2-EMPTY:
 // DISASM2-NEXT: <zed@plt>:
 // DISASM2-NEXT:  201300:       jmpq *8442(%rip)
 // DISASM2-NEXT:                pushq $1
-// DISASM2-NEXT:                jmp -48 <.plt>
+// DISASM2-NEXT:                jmp 0x2012e0 <.plt>
 // DISASM2-NOT:   {{.}}
 
 .global _start
diff --git a/lld/test/ELF/x86-64-retpoline-linkerscript.s b/lld/test/ELF/x86-64-retpoline-linkerscript.s
index 5a14595b4844e..295af0ab608b0 100644
--- a/lld/test/ELF/x86-64-retpoline-linkerscript.s
+++ b/lld/test/ELF/x86-64-retpoline-linkerscript.s
@@ -17,10 +17,10 @@
 // CHECK-NEXT: <.plt>:
 // CHECK-NEXT: 10:       ff 35 32 01 00 00       pushq   306(%rip)
 // CHECK-NEXT: 16:       4c 8b 1d 33 01 00 00    movq    307(%rip), %r11
-// CHECK-NEXT: 1d:       e8 0e 00 00 00  callq   14 <.plt+0x20>
+// CHECK-NEXT: 1d:       e8 0e 00 00 00  callq   0x30 <.plt+0x20>
 // CHECK-NEXT: 22:       f3 90   pause
 // CHECK-NEXT: 24:       0f ae e8        lfence
-// CHECK-NEXT: 27:       eb f9   jmp     -7 <.plt+0x12>
+// CHECK-NEXT: 27:       eb f9   jmp     0x22 <.plt+0x12>
 // CHECK-NEXT: 29:       cc      int3
 // CHECK-NEXT: 2a:       cc      int3
 // CHECK-NEXT: 2b:       cc      int3
@@ -42,20 +42,20 @@
 // CHECK-NEXT: 3e:       cc      int3
 // CHECK-NEXT: 3f:       cc      int3
 // CHECK-NEXT: 40:       4c 8b 1d 11 01 00 00    movq    273(%rip), %r11
-// CHECK-NEXT: 47:       e8 e4 ff ff ff  callq   -28 <.plt+0x20>
-// CHECK-NEXT: 4c:       e9 d1 ff ff ff  jmp     -47 <.plt+0x12>
+// CHECK-NEXT: 47:       e8 e4 ff ff ff  callq   0x30 <.plt+0x20>
+// CHECK-NEXT: 4c:       e9 d1 ff ff ff  jmp     0x22 <.plt+0x12>
 // CHECK-NEXT: 51:       68 00 00 00 00  pushq   $0
-// CHECK-NEXT: 56:       e9 b5 ff ff ff  jmp     -75 <.plt>
+// CHECK-NEXT: 56:       e9 b5 ff ff ff  jmp     0x10 <.plt>
 // CHECK-NEXT: 5b:       cc      int3
 // CHECK-NEXT: 5c:       cc      int3
 // CHECK-NEXT: 5d:       cc      int3
 // CHECK-NEXT: 5e:       cc      int3
 // CHECK-NEXT: 5f:       cc      int3
 // CHECK-NEXT: 60:       4c 8b 1d f9 00 00 00    movq    249(%rip), %r11
-// CHECK-NEXT: 67:       e8 c4 ff ff ff  callq   -60 <.plt+0x20>
-// CHECK-NEXT: 6c:       e9 b1 ff ff ff  jmp     -79 <.plt+0x12>
+// CHECK-NEXT: 67:       e8 c4 ff ff ff  callq   0x30 <.plt+0x20>
+// CHECK-NEXT: 6c:       e9 b1 ff ff ff  jmp     0x22 <.plt+0x12>
 // CHECK-NEXT: 71:       68 01 00 00 00  pushq   $1
-// CHECK-NEXT: 76:       e9 95 ff ff ff  jmp     -107 <.plt>
+// CHECK-NEXT: 76:       e9 95 ff ff ff  jmp     0x10 <.plt>
 // CHECK-NEXT: 7b:       cc      int3
 // CHECK-NEXT: 7c:       cc      int3
 // CHECK-NEXT: 7d:       cc      int3
diff --git a/lld/test/ELF/x86-64-retpoline-znow-linkerscript.s b/lld/test/ELF/x86-64-retpoline-znow-linkerscript.s
index 61e5d28cd6807..f516489b0aa1f 100644
--- a/lld/test/ELF/x86-64-retpoline-znow-linkerscript.s
+++ b/lld/test/ELF/x86-64-retpoline-znow-linkerscript.s
@@ -15,10 +15,10 @@
 // CHECK:      Disassembly of section .plt:
 // CHECK-EMPTY:
 // CHECK-NEXT: <.plt>:
-// CHECK-NEXT: 10:	e8 0b 00 00 00 	callq	11 <.plt+0x10>
+// CHECK-NEXT: 10:	e8 0b 00 00 00 	callq	0x20 <.plt+0x10>
 // CHECK-NEXT: 15:	f3 90 	pause
 // CHECK-NEXT: 17:	0f ae e8 	lfence
-// CHECK-NEXT: 1a:	eb f9 	jmp	-7 <.plt+0x5>
+// CHECK-NEXT: 1a:	eb f9 	jmp	0x15 <.plt+0x5>
 // CHECK-NEXT: 1c:	cc 	int3
 // CHECK-NEXT: 1d:	cc 	int3
 // CHECK-NEXT: 1e:	cc 	int3
@@ -37,13 +37,13 @@
 // CHECK-NEXT: 2e:	cc 	int3
 // CHECK-NEXT: 2f:	cc 	int3
 // CHECK-NEXT: 30:	4c 8b 1d 31 00 00 00 	movq	49(%rip), %r11
-// CHECK-NEXT: 37:	e9 d4 ff ff ff 	jmp	-44 <.plt>
+// CHECK-NEXT: 37:	e9 d4 ff ff ff 	jmp	0x10 <.plt>
 // CHECK-NEXT: 3c:	cc 	int3
 // CHECK-NEXT: 3d:	cc 	int3
 // CHECK-NEXT: 3e:	cc 	int3
 // CHECK-NEXT: 3f:	cc 	int3
 // CHECK-NEXT: 40:      4c 8b 1d 29 00 00 00 	movq	41(%rip), %r11
-// CHECK-NEXT: 47:	e9 c4 ff ff ff 	jmp	-60 <.plt>
+// CHECK-NEXT: 47:	e9 c4 ff ff ff 	jmp	0x10 <.plt>
 // CHECK-NEXT: 4c:	cc 	int3
 // CHECK-NEXT: 4d:	cc 	int3
 // CHECK-NEXT: 4e:	cc 	int3
diff --git a/lld/test/ELF/x86-64-retpoline-znow-static-iplt.s b/lld/test/ELF/x86-64-retpoline-znow-static-iplt.s
index 01cc1ec4579b9..eecd5a692a361 100644
--- a/lld/test/ELF/x86-64-retpoline-znow-static-iplt.s
+++ b/lld/test/ELF/x86-64-retpoline-znow-static-iplt.s
@@ -5,16 +5,16 @@
 
 #0x2011a9+5 + 34 = 0x2011d0 (foo@plt)
 # CHECK:      <_start>:
-# CHECK-NEXT:  2011a9:       callq   34
+# CHECK-NEXT:  2011a9:       callq   0x2011d0
 
 #Static IPLT header due to -z retpolineplt
 # CHECK:       00000000002011b0 <.plt>:
-# CHECK-NEXT:  2011b0:       callq   11 <.plt+0x10>
+# CHECK-NEXT:  2011b0:       callq   0x2011c0 <.plt+0x10>
 # CHECK-NEXT:  2011b5:       pause
 # CHECK-NEXT:  2011b7:       lfence
 #foo@plt
 # CHECK:       2011d0:       movq    4105(%rip), %r11
-# CHECK-NEXT:  2011d7:       jmp     -44 <.plt>
+# CHECK-NEXT:  2011d7:       jmp     0x2011b0 <.plt>
 
 .type foo STT_GNU_IFUNC
 .globl foo
diff --git a/lld/test/ELF/x86-64-retpoline-znow.s b/lld/test/ELF/x86-64-retpoline-znow.s
index 20f13ab290ba4..a8a62c18bb369 100644
--- a/lld/test/ELF/x86-64-retpoline-znow.s
+++ b/lld/test/ELF/x86-64-retpoline-znow.s
@@ -14,10 +14,10 @@
 // CHECK:      Disassembly of section .plt:
 // CHECK-EMPTY:
 // CHECK-NEXT: <.plt>:
-// CHECK-NEXT: 12d0:       callq   11 <.plt+0x10>
+// CHECK-NEXT: 12d0:       callq   0x12e0 <.plt+0x10>
 // CHECK-NEXT:             pause
 // CHECK-NEXT:             lfence
-// CHECK-NEXT:             jmp     -7 <.plt+0x5>
+// CHECK-NEXT:             jmp     0x12d5 <.plt+0x5>
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
@@ -36,13 +36,13 @@
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
 // CHECK-NEXT: 12f0:       movq    4369(%rip), %r11
-// CHECK-NEXT:             jmp     -44 <.plt>
+// CHECK-NEXT:             jmp     0x12d0 <.plt>
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
 // CHECK-NEXT: 1300:       movq    4361(%rip), %r11
-// CHECK-NEXT:             jmp     -60 <.plt>
+// CHECK-NEXT:             jmp     0x12d0 <.plt>
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
diff --git a/lld/test/ELF/x86-64-retpoline.s b/lld/test/ELF/x86-64-retpoline.s
index 874e20423578d..8ef218d49b26d 100644
--- a/lld/test/ELF/x86-64-retpoline.s
+++ b/lld/test/ELF/x86-64-retpoline.s
@@ -16,10 +16,10 @@
 // CHECK-NEXT: <.plt>:
 // CHECK-NEXT: 1300:       pushq   8498(%rip)
 // CHECK-NEXT:             movq    8499(%rip), %r11
-// CHECK-NEXT:             callq   14 <.plt+0x20>
+// CHECK-NEXT:             callq   0x1320 <.plt+0x20>
 // CHECK-NEXT:             pause
 // CHECK-NEXT:             lfence
-// CHECK-NEXT:             jmp     -7 <.plt+0x12>
+// CHECK-NEXT:             jmp     0x1312 <.plt+0x12>
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
@@ -41,20 +41,20 @@
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
 // CHECK-NEXT: 1330:       movq    8465(%rip), %r11
-// CHECK-NEXT:             callq   -28 <.plt+0x20>
-// CHECK-NEXT:             jmp     -47 <.plt+0x12>
+// CHECK-NEXT:             callq   0x1320 <.plt+0x20>
+// CHECK-NEXT:             jmp     0x1312 <.plt+0x12>
 // CHECK-NEXT:             pushq   $0
-// CHECK-NEXT:             jmp     -75 <.plt>
+// CHECK-NEXT:             jmp     0x1300 <.plt>
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
 // CHECK-NEXT: 1350:       movq    8441(%rip), %r11
-// CHECK-NEXT:             callq   -60 <.plt+0x20>
-// CHECK-NEXT:             jmp     -79 <.plt+0x12>
+// CHECK-NEXT:             callq   0x1320 <.plt+0x20>
+// CHECK-NEXT:             jmp     0x1312 <.plt+0x12>
 // CHECK-NEXT:             pushq   $1
-// CHECK-NEXT:             jmp     -107 <.plt>
+// CHECK-NEXT:             jmp     0x1300 <.plt>
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
diff --git a/lld/test/ELF/znotext-weak-undef.s b/lld/test/ELF/znotext-weak-undef.s
index d1e1e209ef0eb..ad9fb9e2b3379 100644
--- a/lld/test/ELF/znotext-weak-undef.s
+++ b/lld/test/ELF/znotext-weak-undef.s
@@ -8,8 +8,6 @@
 # EXE:      Relocations [
 # EXE-NEXT: ]
 
-.text
-.global foo
 .weak foo
 
 _start:
diff --git a/lld/test/MachO/alignment-too-large.yaml b/lld/test/MachO/alignment-too-large.yaml
new file mode 100644
index 0000000000000..18d133f558690
--- /dev/null
+++ b/lld/test/MachO/alignment-too-large.yaml
@@ -0,0 +1,58 @@
+# RUN: yaml2obj %s -o %t.o
+# RUN: not lld -flavor darwinnew -o %t %t.o 2>&1 | FileCheck %s
+#
+# CHECK: alignment 32 of section __text is too large
+--- !mach-o
+FileHeader:
+  magic:           0xFEEDFACF
+  cputype:         0x01000007
+  cpusubtype:      0x00000003
+  filetype:        0x00000001
+  ncmds:           4
+  sizeofcmds:      280
+  flags:           0x00000000
+  reserved:        0x00000000
+LoadCommands:
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         152
+    segname:         ''
+    vmaddr:          0
+    vmsize:          8
+    fileoff:         312
+    filesize:        8
+    maxprot:         7
+    initprot:        7
+    nsects:          1
+    flags:           0
+    Sections:
+      - sectname:        __text
+        segname:         __TEXT
+        addr:            0x0000000000000000
+        size:            8
+        offset:          0x00000138
+        align:           32
+        reloff:          0x00000000
+        nreloc:          0
+        flags:           0x80000400
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+        content:         48C7C000000000C3
+  - cmd:             LC_SYMTAB
+    cmdsize:         24
+    symoff:          320
+    nsyms:           1
+    stroff:          336
+    strsize:         8
+LinkEditData:
+  NameList:
+    - n_strx:          1
+      n_type:          0x0F
+      n_sect:          1
+      n_desc:          0
+      n_value:         0
+  StringTable:
+    - ''
+    - _main
+    - ''
+...
diff --git a/lld/test/MachO/arch.s b/lld/test/MachO/arch.s
new file mode 100644
index 0000000000000..313654937c801
--- /dev/null
+++ b/lld/test/MachO/arch.s
@@ -0,0 +1,11 @@
+# REQUIRES: x86
+# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-darwin %s -o %t.o
+# RUN: lld -flavor darwinnew -arch x86_64 -o /dev/null %t.o
+# RUN: not lld -flavor darwinnew -arch i386 -o /dev/null %t.o 2>&1 | FileCheck %s
+# CHECK: missing or unsupported -arch i386
+
+.text
+.global _main
+_main:
+  mov $0, %rax
+  ret
diff --git a/lld/test/MachO/duplicate-symbol.s b/lld/test/MachO/duplicate-symbol.s
new file mode 100644
index 0000000000000..47f597f3ec024
--- /dev/null
+++ b/lld/test/MachO/duplicate-symbol.s
@@ -0,0 +1,12 @@
+# REQUIRES: x86
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t-dup.o
+# RUN: not lld -flavor darwinnew -o /dev/null %t-dup.o %t.o 2>&1 | FileCheck %s
+
+# CHECK: duplicate symbol: _main
+
+.text
+.global _main
+_main:
+  mov $0, %rax
+  ret
diff --git a/lld/test/MachO/entry-symbol.s b/lld/test/MachO/entry-symbol.s
new file mode 100644
index 0000000000000..1890f407f1adc
--- /dev/null
+++ b/lld/test/MachO/entry-symbol.s
@@ -0,0 +1,13 @@
+# REQUIRES: x86
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t.o
+# RUN: lld -flavor darwinnew -o /dev/null %t.o -e _not_main
+# RUN: not lld -flavor darwinnew -o /dev/null %t.o -e _missing 2>&1 | FileCheck %s
+# RUN: not lld -flavor darwinnew -o /dev/null %t.o 2>&1 | FileCheck %s --check-prefix=DEFAULT_ENTRY
+
+# CHECK: undefined symbol: _missing
+# DEFAULT_ENTRY: undefined symbol: _main
+
+.text
+.global _not_main
+_not_main:
+  ret
diff --git a/lld/test/MachO/invalid-executable.s b/lld/test/MachO/invalid-executable.s
new file mode 100644
index 0000000000000..9ad9f51e41700
--- /dev/null
+++ b/lld/test/MachO/invalid-executable.s
@@ -0,0 +1,11 @@
+# REQUIRES: x86
+# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-darwin %s -o %t.o
+# RUN: lld -flavor darwinnew -o %t %t.o
+# RUN: not lld -flavor darwinnew -o /dev/null %t 2>&1 | FileCheck %s
+# CHECK: unhandled file type
+
+.text
+.global _main
+_main:
+  mov $0, %rax
+  ret
diff --git a/lld/test/MachO/load-commands.s b/lld/test/MachO/load-commands.s
new file mode 100644
index 0000000000000..298cb76cc55a4
--- /dev/null
+++ b/lld/test/MachO/load-commands.s
@@ -0,0 +1,17 @@
+# REQUIRES: x86
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t.o
+# RUN: lld -flavor darwinnew -o %t %t.o
+# RUN: obj2yaml %t | FileCheck %s
+
+# Check for the presence of a couple of load commands that are essential for
+# a working binary.
+
+# CHECK-DAG: cmd:             LC_DYLD_INFO_ONLY
+# CHECK-DAG: cmd:             LC_SYMTAB
+# CHECK-DAG: cmd:             LC_DYSYMTAB
+
+.text
+.global _main
+_main:
+  mov $0, %rax
+  ret
diff --git a/lld/test/MachO/no-such-file.s b/lld/test/MachO/no-such-file.s
new file mode 100644
index 0000000000000..840b8f9e8a470
--- /dev/null
+++ b/lld/test/MachO/no-such-file.s
@@ -0,0 +1,4 @@
+# REQUIRES: x86
+# RUN: not lld -flavor darwinnew -o /dev/null %t-no-such-file.o 2>&1 | FileCheck %s
+
+# CHECK: cannot open {{.*}}no-such-file.o
diff --git a/lld/test/MachO/relocations.s b/lld/test/MachO/relocations.s
new file mode 100644
index 0000000000000..cf38f96d0177b
--- /dev/null
+++ b/lld/test/MachO/relocations.s
@@ -0,0 +1,21 @@
+# REQUIRES: x86
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t.o
+# RUN: lld -flavor darwinnew -o %t %t.o
+# RUN: llvm-objdump -d %t | FileCheck %s
+
+# CHECK: leaq 17(%rip), %rsi
+
+.section __TEXT,__text
+.globl _main
+_main:
+  movl $0x2000004, %eax # write() syscall
+  mov $1, %rdi # stdout
+  leaq str(%rip), %rsi
+  mov $13, %rdx # length of str
+  syscall
+  mov $0, %rax
+  ret
+
+.section __TEXT,__cstring
+str:
+  .asciz "Hello world!\n"
diff --git a/lld/test/MachO/section-headers.s b/lld/test/MachO/section-headers.s
new file mode 100644
index 0000000000000..9fafc5a912b0f
--- /dev/null
+++ b/lld/test/MachO/section-headers.s
@@ -0,0 +1,46 @@
+# REQUIRES: x86
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t.o
+# RUN: lld -flavor darwinnew -o %t %t.o
+# RUN: llvm-readobj --section-headers %t | FileCheck %s
+
+# CHECK:      Name: __text
+# CHECK-NEXT: Segment: __TEXT
+# CHECK-NOT:  }
+# CHECK:      Alignment: 1
+# CHECK-NOT:  }
+# CHECK:      Type: Regular (0x0)
+# CHECK-NEXT: Attributes [ (0x800004)
+# CHECK-NEXT:   PureInstructions (0x800000)
+# CHECK-NEXT:   SomeInstructions (0x4)
+# CHECK-NEXT: ]
+
+# CHECK:      Name: __cstring
+# CHECK-NEXT: Segment: __TEXT
+# CHECK-NOT:  }
+# CHECK:      Alignment: 2
+# CHECK-NOT:  }
+# CHECK:      Type: CStringLiterals (0x2)
+# CHECK-NEXT: Attributes [ (0x0)
+# CHECK-NEXT: ]
+
+# CHECK:      Name: maxlen_16ch_name
+# CHECK-NEXT: Segment: __TEXT
+# CHECK-NOT:  }
+# CHECK:      Alignment: 3
+# CHECK-NOT:  }
+# CHECK:      Type: Regular (0x0)
+
+.text
+.align 1
+.global _main
+_main:
+  mov $0, %rax
+  ret
+
+.section __TEXT,__cstring
+.align 2
+str:
+  .asciz "Hello world!\n"
+
+.section __TEXT,maxlen_16ch_name
+.align 3
diff --git a/lld/test/MachO/segments.s b/lld/test/MachO/segments.s
new file mode 100644
index 0000000000000..69e3d9f030b0a
--- /dev/null
+++ b/lld/test/MachO/segments.s
@@ -0,0 +1,20 @@
+# REQUIRES: x86
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t.o
+# RUN: lld -flavor darwinnew -o %t %t.o
+# RUN: llvm-readobj --macho-segment %t | FileCheck %s
+
+# These segments must always be present.
+# CHECK-DAG: Name: __PAGEZERO
+# CHECK-DAG: Name: __LINKEDIT
+# CHECK-DAG: Name: __TEXT
+
+# Check that we handle max-length names correctly.
+# CHECK-DAG: Name: maxlen_16ch_name
+
+.text
+.global _main
+_main:
+  mov $0, %rax
+  ret
+
+.section maxlen_16ch_name,foo
diff --git a/lld/test/MachO/silent-ignore.test b/lld/test/MachO/silent-ignore.test
new file mode 100644
index 0000000000000..e863aac0b6251
--- /dev/null
+++ b/lld/test/MachO/silent-ignore.test
@@ -0,0 +1,8 @@
+RUN: lld -flavor darwinnew -v \
+RUN:   -demangle \
+RUN:   -dynamic \
+RUN:   -no_deduplicate \
+RUN:   -lto_library /lib/foo \
+RUN:   -macosx_version_min 0
+RUN: not lld -flavor darwinnew -v --not-an-ignored-argument 2>&1 | FileCheck %s
+CHECK: unknown argument: --not-an-ignored-argument
diff --git a/lld/test/MachO/text-segment.s b/lld/test/MachO/text-segment.s
new file mode 100644
index 0000000000000..a3c7edbc61bff
--- /dev/null
+++ b/lld/test/MachO/text-segment.s
@@ -0,0 +1,15 @@
+# REQUIRES: x86
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t.o
+# RUN: lld -flavor darwinnew -o %t %t.o
+# RUN: llvm-readobj --macho-segment %t | FileCheck %s
+
+# CHECK: Name: __TEXT
+# CHECK-NOT: }
+# dyld3 assumes that the __TEXT segment starts from the file header
+# CHECK:       fileoff: 0
+
+.text
+.global _main
+_main:
+  mov $0, %rax
+  ret
diff --git a/lld/test/mach-o/error-simulator-vs-macosx.yaml b/lld/test/mach-o/error-simulator-vs-macosx.yaml
index 609eb3be43abc..fcf1774d760ac 100644
--- a/lld/test/mach-o/error-simulator-vs-macosx.yaml
+++ b/lld/test/mach-o/error-simulator-vs-macosx.yaml
@@ -1,7 +1,7 @@
 # RUN: ld64.lld -arch i386 -macosx_version_min 10.8 %s %p/Inputs/hello-world-x86.yaml -o %t && llvm-nm -m %t | FileCheck %s
 # RUN: not ld64.lld -arch i386 -ios_simulator_version_min 5.0 %s %p/Inputs/hello-world-x86.yaml -o %t 2>&1 | FileCheck %s --check-prefix=ERROR
 #
-# Test that i386 can link with a macos version but gives an error with a simululator version.
+# Test that i386 can link with a macos version but gives an error with a simulator version.
 #
 
 --- !mach-o
diff --git a/lld/test/mach-o/parse-literals-error.yaml b/lld/test/mach-o/parse-literals-error.yaml
index 9dad0cbbf974b..de8b47c530478 100644
--- a/lld/test/mach-o/parse-literals-error.yaml
+++ b/lld/test/mach-o/parse-literals-error.yaml
@@ -1,7 +1,7 @@
 # RUN: not ld64.lld -arch x86_64 -r -print_atoms %s -o %t 2> %t.err
 # RUN: FileCheck %s < %t.err
 #
-# Test for error if literal section is not correct size mulitple.
+# Test for error if literal section is not correct size multiple.
 #
 
 --- !mach-o
diff --git a/lld/test/wasm/debug-removed-fn.ll b/lld/test/wasm/debug-removed-fn.ll
index 6bb23df06a57c..89a55a3d41878 100644
--- a/lld/test/wasm/debug-removed-fn.ll
+++ b/lld/test/wasm/debug-removed-fn.ll
@@ -1,11 +1,16 @@
 ; RUN: llc -filetype=obj < %s -o %t.o
 ; RUN: wasm-ld %t.o --no-entry --export=foo -o %t.wasm
-; RUN: llvm-dwarfdump -debug-line %t.wasm | FileCheck %s
+; RUN: llvm-dwarfdump -debug-line -debug-ranges %t.wasm | FileCheck %s
 
 ; CHECK: Address
 ; CHECK: 0x0000000000000005
 ; CHECK: 0x0000000000000000
 
+; CHECK: .debug_ranges contents:
+; CHECK: 00000000 {{[0-9]+}} {{[0-9]+}}
+; CHECK: 00000000 {{[0-9]+}} {{[0-9]+}}
+; CHECK: 00000000 <End of list>
+
 ; ModuleID = 't.bc'
 target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown-wasm"
diff --git a/lld/test/wasm/early-exit-for-bad-paths.s b/lld/test/wasm/early-exit-for-bad-paths.s
new file mode 100644
index 0000000000000..2866bfa62f865
--- /dev/null
+++ b/lld/test/wasm/early-exit-for-bad-paths.s
@@ -0,0 +1,22 @@
+# RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-unknown %s -o %t.o
+
+# RUN: not wasm-ld %t.o -o does_not_exist/output 2>&1 | \
+# RUN:   FileCheck %s -check-prefixes=NO-DIR-OUTPUT,CHECK
+# RUN: not wasm-ld %t.o -o %s/dir_is_a_file 2>&1 | \
+# RUN:   FileCheck %s -check-prefixes=DIR-IS-OUTPUT,CHECK
+# TODO(sbc): check similar check for -Map file once we add that option
+
+# NO-DIR-OUTPUT: error: cannot open output file does_not_exist/output:
+# DIR-IS-OUTPUT: error: cannot open output file {{.*}}/dir_is_a_file:
+
+# We should exit before doing the actual link. If an undefined symbol error is
+# discovered we haven't bailed out early as expected.
+# CHECK-NOT: undefined_symbol
+
+# RUN: not wasm-ld %t.o -o / 2>&1 | FileCheck %s -check-prefixes=ROOT,CHECK
+# ROOT: error: cannot open output file /
+
+_start:
+    .functype _start () -> ()
+    call undefined_symbol
+    end_function
diff --git a/lld/test/wasm/entry-signature.ll b/lld/test/wasm/entry-signature.ll
index 8e245b14e9640..f7f3d481acfc6 100644
--- a/lld/test/wasm/entry-signature.ll
+++ b/lld/test/wasm/entry-signature.ll
@@ -1,4 +1,4 @@
-; Verify that the entry point signauture can be flexible.
+; Verify that the entry point signature can be flexible.
 ; RUN: llc -filetype=obj %s -o %t.o
 ; RUN: wasm-ld -o %t1.wasm %t.o
 
diff --git a/lld/test/wasm/event-section.ll b/lld/test/wasm/event-section.ll
index 771a8cbdf6c2f..7bd87907e9316 100644
--- a/lld/test/wasm/event-section.ll
+++ b/lld/test/wasm/event-section.ll
@@ -32,3 +32,6 @@ define void @_start() {
 ; CHECK-NEXT:       - Index:           0
 ; CHECK-NEXT:         Attribute:       0
 ; CHECK-NEXT:         SigIndex:        1
+
+; Global section has to come after event section
+; CHECK:        - Type:            GLOBAL
diff --git a/lld/test/wasm/export-name.ll b/lld/test/wasm/export-name.ll
index 83a8bd907c79f..b02f253760d1e 100644
--- a/lld/test/wasm/export-name.ll
+++ b/lld/test/wasm/export-name.ll
@@ -8,6 +8,10 @@ define void @foo() #0 {
     ret void
 }
 
+define void @qux() #1 {
+    ret void
+}
+
 define void @_start() {
     call void @foo()
     ret void
@@ -15,6 +19,8 @@ define void @_start() {
 
 attributes #0 = { "wasm-export-name"="bar" }
 
+attributes #1 = { "wasm-export-name"="" }
+
 ; CHECK:       - Type:            EXPORT
 ; CHECK-NEXT:    Exports:
 ; CHECK-NEXT:      - Name:            memory
@@ -23,6 +29,9 @@ attributes #0 = { "wasm-export-name"="bar" }
 ; CHECK-NEXT:      - Name:            bar
 ; CHECK-NEXT:        Kind:            FUNCTION
 ; CHECK-NEXT:        Index:           0
-; CHECK-NEXT:      - Name:            _start
+; CHECK-NEXT:      - Name:            ''
 ; CHECK-NEXT:        Kind:            FUNCTION
 ; CHECK-NEXT:        Index:           1
+; CHECK-NEXT:      - Name:            _start
+; CHECK-NEXT:        Kind:            FUNCTION
+; CHECK-NEXT:        Index:           2
diff --git a/lld/test/wasm/export-optional-lazy.ll b/lld/test/wasm/export-optional-lazy.ll
index 960e71c6ae6c6..c37a3e5183eba 100644
--- a/lld/test/wasm/export-optional-lazy.ll
+++ b/lld/test/wasm/export-optional-lazy.ll
@@ -1,7 +1,7 @@
 ; Optional linker-synthetic symbols are only created if they are undefined
 ; in the final output.
 ; This test is for a regression where an explicit --export of an lazy archive
-; symbol caused an undefined referece to an optional symbol to occur *after*
+; symbol caused an undefined reference to an optional symbol to occur *after*
 ; the optional symbols were created.
 
 ; RUN: llc -filetype=obj %s -o %t.o
diff --git a/lld/test/wasm/lto/incompatible.ll b/lld/test/wasm/lto/incompatible.ll
index 6f7c154e959c4..335756420639f 100644
--- a/lld/test/wasm/lto/incompatible.ll
+++ b/lld/test/wasm/lto/incompatible.ll
@@ -1,6 +1,6 @@
 ; REQUIRES: x86
 ; RUN: llvm-as %s -o %t.bc
-; RUN: not wasm-ld %t.bc -o out.wasm 2>&1 | FileCheck %s
+; RUN: not wasm-ld %t.bc -o %t.wasm 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/lld/test/wasm/lto/opt-level.ll b/lld/test/wasm/lto/opt-level.ll
index f6156e725c7d1..d2d552728b164 100644
--- a/lld/test/wasm/lto/opt-level.ll
+++ b/lld/test/wasm/lto/opt-level.ll
@@ -11,7 +11,7 @@
 ; RUN:   FileCheck --check-prefix=INVALID %s
 ; INVALID: invalid optimization level for LTO: 6
 
-; RUN: not wasm-ld -o %t3 -m elf_x86_64 -e main --lto-O-1 %t.o 2>&1 | \
+; RUN: not wasm-ld -o %t3 -e main --lto-O-1 %t.o 2>&1 | \
 ; RUN:   FileCheck --check-prefix=INVALIDNEGATIVE %s
 ; INVALIDNEGATIVE: invalid optimization level for LTO: 4294967295
 
diff --git a/lld/test/wasm/lto/signature-mismatch.ll b/lld/test/wasm/lto/signature-mismatch.ll
index e12d91866023a..cf1a998826fc0 100644
--- a/lld/test/wasm/lto/signature-mismatch.ll
+++ b/lld/test/wasm/lto/signature-mismatch.ll
@@ -3,7 +3,7 @@
 ; RUN: not wasm-ld --fatal-warnings %t.o %t1.o -o %t.wasm 2>&1 | FileCheck %s
 
 ; Test that functions defined in bitcode correctly report signature
-; mistmaches with existing undefined sybmols in normal objects.
+; mismatches with existing undefined sybmols in normal objects.
 
 target triple = "wasm32-unknown-unknown"
 
diff --git a/lld/test/wasm/lto/thinlto.ll b/lld/test/wasm/lto/thinlto.ll
index 062da1a33b899..da42469f2deb6 100644
--- a/lld/test/wasm/lto/thinlto.ll
+++ b/lld/test/wasm/lto/thinlto.ll
@@ -14,8 +14,38 @@
 ; RUN: llvm-nm %t31.lto.o | FileCheck %s --check-prefix=NM1
 ; RUN: llvm-nm %t32.lto.o | FileCheck %s --check-prefix=NM2
 
-; Check without --thinlto-jobs (which currently default to hardware_concurrency)
-; RUN: wasm-ld -r %t1.o %t2.o -o %t3
+;; --thinlto-jobs= defaults to --threads=.
+; RUN: rm -f %t31.lto.o %t32.lto.o
+; RUN: wasm-ld -r -save-temps --threads=2 %t1.o %t2.o -o %t3
+; RUN: llvm-nm %t31.lto.o | FileCheck %s --check-prefix=NM1
+; RUN: llvm-nm %t32.lto.o | FileCheck %s --check-prefix=NM2
+
+;; --thinlto-jobs= overrides --threads=.
+; RUN: rm -f %t31.lto.o %t32.lto.o
+; RUN: wasm-ld -r -save-temps --threads=1 --thinlto-jobs=2 %t1.o %t2.o -o %t3
+; RUN: llvm-nm %t31.lto.o | FileCheck %s --check-prefix=NM1
+; RUN: llvm-nm %t32.lto.o | FileCheck %s --check-prefix=NM2
+
+; Test with all threads, on all cores, on all CPU sockets
+; RUN: rm -f %t31.lto.o %t32.lto.o
+; RUN: wasm-ld -r -save-temps --thinlto-jobs=all %t1.o %t2.o -o %t3
+; RUN: llvm-nm %t31.lto.o | FileCheck %s --check-prefix=NM1
+; RUN: llvm-nm %t32.lto.o | FileCheck %s --check-prefix=NM2
+
+; Test with many more threads than the system has
+; RUN: rm -f %t31.lto.o %t32.lto.o
+; RUN: wasm-ld -r -save-temps --thinlto-jobs=1000 %t1.o %t2.o -o %t3
+; RUN: llvm-nm %t31.lto.o | FileCheck %s --check-prefix=NM1
+; RUN: llvm-nm %t32.lto.o | FileCheck %s --check-prefix=NM2
+
+; Test with a bad value
+; RUN: rm -f %t31.lto.o %t32.lto.o
+; RUN: not wasm-ld -r -save-temps --thinlto-jobs=foo %t1.o %t2.o -o %t3 2>&1 | FileCheck %s --check-prefix=BAD-JOBS
+; BAD-JOBS: error: --thinlto-jobs: invalid job count: foo
+
+; Check without --thinlto-jobs (which currently defaults to heavyweight_hardware_concurrency, meanning one thread per hardware core -- not SMT)
+; RUN: rm -f %t31.lto.o %t32.lto.o
+; RUN: wasm-ld -r -save-temps %t1.o %t2.o -o %t3
 ; RUN: llvm-nm %t31.lto.o | FileCheck %s --check-prefix=NM1
 ; RUN: llvm-nm %t32.lto.o | FileCheck %s --check-prefix=NM2
 
diff --git a/lld/test/wasm/lto/undef.ll b/lld/test/wasm/lto/undef.ll
index 65e8e4642d5e0..a5477cb3c3bdf 100644
--- a/lld/test/wasm/lto/undef.ll
+++ b/lld/test/wasm/lto/undef.ll
@@ -8,7 +8,7 @@ target triple = "wasm32-unknown-unknown"
 declare i32 @bar()
 
 ; Symbols such as foo which are only called indirectly are handled slightly
-; differently with resepect to signature checking.
+; differently with respect to signature checking.
 declare i32 @foo()
 
 @ptr = global i8* bitcast (i32 ()* @foo to i8*), align 8
diff --git a/lld/test/wasm/responsefile.test b/lld/test/wasm/responsefile.test
index ba2afd0cc8fdf..85ac41f93f2be 100644
--- a/lld/test/wasm/responsefile.test
+++ b/lld/test/wasm/responsefile.test
@@ -10,11 +10,11 @@ RUN:     FileCheck --check-prefix=INVRSP %s
 INVRSP: invalid response file quoting: foobar
 
 RUN: echo "blah\foo" > %t.rsp
-RUN: not wasm-ld --rsp-quoting=windows @%t.rsp 2>&1 | \
+RUN: not wasm-ld -o a.out --rsp-quoting=windows @%t.rsp 2>&1 | \
 RUN:     FileCheck --check-prefix=WINRSP %s
 WINRSP: error: cannot open blah\foo:
 
 RUN: echo "blah\foo" > %t.rsp
-RUN: not wasm-ld --rsp-quoting=posix @%t.rsp 2>&1 | \
+RUN: not wasm-ld -o a.out --rsp-quoting=posix @%t.rsp 2>&1 | \
 RUN:     FileCheck --check-prefix=POSRSP %s
 POSRSP: error: cannot open blahfoo:
diff --git a/lld/test/wasm/signature-mismatch-unknown.ll b/lld/test/wasm/signature-mismatch-unknown.ll
index b354ed12207da..9bbad0065dcdc 100644
--- a/lld/test/wasm/signature-mismatch-unknown.ll
+++ b/lld/test/wasm/signature-mismatch-unknown.ll
@@ -4,7 +4,7 @@
 ; RUN: wasm-ld --fatal-warnings -o %t.wasm %t.main.o %t.ret32.o
 
 ; Also test the case where there are two different object files that contains
-; referneces ret32:
+; references ret32:
 ; %t.main.o: Does not call ret32 directly; used the wrong signature.
 ; %t.call-ret32.o: Calls ret32 directly; uses the correct signature.
 ; RUN: llc -filetype=obj %p/Inputs/call-ret32.ll -o %t.call-ret32.o
diff --git a/lld/test/wasm/threads.s b/lld/test/wasm/threads.s
new file mode 100644
index 0000000000000..e02f5e620805d
--- /dev/null
+++ b/lld/test/wasm/threads.s
@@ -0,0 +1,12 @@
+# RUN: llvm-mc -filetype=obj -triple=wasm32 %s -o %t.o
+
+## A positive integer is allowed.
+# RUN: wasm-ld --no-entry %t.o -o /dev/null
+# RUN: wasm-ld --no-entry --threads=1 %t.o -o /dev/null
+# RUN: wasm-ld --no-entry --threads=2 %t.o -o /dev/null
+
+# RUN: not wasm-ld --threads=all %t.o -o /dev/null 2>&1 | FileCheck %s -DN=all
+# RUN: not wasm-ld --threads=0 %t.o -o /dev/null 2>&1 | FileCheck %s -DN=0
+# RUN: not wasm-ld --threads=-1 %t.o -o /dev/null 2>&1 | FileCheck %s -DN=-1
+
+# CHECK: error: --threads: expected a positive integer, but got '[[N]]'
diff --git a/lld/test/wasm/undefined-entry.test b/lld/test/wasm/undefined-entry.test
index a36212f2c9e3c..3106a76510f46 100644
--- a/lld/test/wasm/undefined-entry.test
+++ b/lld/test/wasm/undefined-entry.test
@@ -3,7 +3,7 @@ RUN: not wasm-ld -o %t.wasm %t.ret32.o 2>&1 | FileCheck %s
 RUN: not wasm-ld --allow-undefined -o %t.wasm %t.ret32.o 2>&1 | FileCheck %s
 RUN: not wasm-ld -entry=foo -o %t.wasm %t.ret32.o 2>&1 | FileCheck %s -check-prefix=CHECK-CUSTOM
 
-CHECK: error: entry symbol not defined (pass --no-entry to supress): _start
-CHECK-CUSTOM: error: entry symbol not defined (pass --no-entry to supress): foo
+CHECK: error: entry symbol not defined (pass --no-entry to suppress): _start
+CHECK-CUSTOM: error: entry symbol not defined (pass --no-entry to suppress): foo
 
 RUN: wasm-ld --no-entry -o %t.wasm %t.ret32.o
diff --git a/lld/test/wasm/visibility-hidden.ll b/lld/test/wasm/visibility-hidden.ll
index f5731c4e964ad..99acd5651f7ba 100644
--- a/lld/test/wasm/visibility-hidden.ll
+++ b/lld/test/wasm/visibility-hidden.ll
@@ -3,12 +3,12 @@
 ; RUN: rm -f %t2.a
 ; RUN: llvm-ar rcs %t2.a %t2.o
 
-; Test that symbols with hidden visitiblity are not export, even with
+; Test that symbols with hidden visibility are not export, even with
 ; --export-dynamic
 ; RUN: wasm-ld --export-dynamic %t.o %t2.a -o %t.wasm
 ; RUN: obj2yaml %t.wasm | FileCheck %s
 
-; Test that symbols with default visitiblity are not exported without
+; Test that symbols with default visibility are not exported without
 ; --export-dynamic
 ; RUN: wasm-ld %t.o %t2.a -o %t.nodef.wasm
 ; RUN: obj2yaml %t.nodef.wasm | FileCheck %s -check-prefix=NO-DEFAULT
diff --git a/lld/tools/lld/CMakeLists.txt b/lld/tools/lld/CMakeLists.txt
index a37c2c702bd5c..2cdd9b854246d 100644
--- a/lld/tools/lld/CMakeLists.txt
+++ b/lld/tools/lld/CMakeLists.txt
@@ -16,6 +16,7 @@ target_link_libraries(lld
   lldCOFF
   lldDriver
   lldELF
+  lldMachO2
   lldMinGW
   lldWasm
   )
diff --git a/lld/tools/lld/lld.cpp b/lld/tools/lld/lld.cpp
index 646fc3d6468eb..58a291d1e9210 100644
--- a/lld/tools/lld/lld.cpp
+++ b/lld/tools/lld/lld.cpp
@@ -45,10 +45,11 @@ using namespace llvm::sys;
 
 enum Flavor {
   Invalid,
-  Gnu,     // -flavor gnu
-  WinLink, // -flavor link
-  Darwin,  // -flavor darwin
-  Wasm,    // -flavor wasm
+  Gnu,       // -flavor gnu
+  WinLink,   // -flavor link
+  Darwin,    // -flavor darwin
+  DarwinNew, // -flavor darwinnew
+  Wasm,      // -flavor wasm
 };
 
 LLVM_ATTRIBUTE_NORETURN static void die(const Twine &s) {
@@ -62,6 +63,7 @@ static Flavor getFlavor(StringRef s) {
       .CasesLower("wasm", "ld-wasm", Wasm)
       .CaseLower("link", WinLink)
       .CasesLower("ld64", "ld64.lld", "darwin", Darwin)
+      .CaseLower("darwinnew", DarwinNew)
       .Default(Invalid);
 }
 
@@ -157,6 +159,8 @@ int main(int argc, const char **argv) {
     return !coff::link(args, canExitEarly(), llvm::outs(), llvm::errs());
   case Darwin:
     return !mach_o::link(args, canExitEarly(), llvm::outs(), llvm::errs());
+  case DarwinNew:
+    return !macho::link(args, canExitEarly(), llvm::outs(), llvm::errs());
   case Wasm:
     return !wasm::link(args, canExitEarly(), llvm::outs(), llvm::errs());
   default:
diff --git a/lld/wasm/Config.h b/lld/wasm/Config.h
index 3b3f2daf576c7..b396bda93d58b 100644
--- a/lld/wasm/Config.h
+++ b/lld/wasm/Config.h
@@ -53,7 +53,7 @@ struct Configuration {
   unsigned ltoPartitions;
   unsigned ltoo;
   unsigned optimize;
-  unsigned thinLTOJobs;
+  llvm::StringRef thinLTOJobs;
 
   llvm::StringRef entry;
   llvm::StringRef outputFile;
diff --git a/lld/wasm/Driver.cpp b/lld/wasm/Driver.cpp
index 73b330ae14708..b6cd879e89c63 100644
--- a/lld/wasm/Driver.cpp
+++ b/lld/wasm/Driver.cpp
@@ -15,6 +15,7 @@
 #include "Writer.h"
 #include "lld/Common/Args.h"
 #include "lld/Common/ErrorHandler.h"
+#include "lld/Common/Filesystem.h"
 #include "lld/Common/Memory.h"
 #include "lld/Common/Reproduce.h"
 #include "lld/Common/Strings.h"
@@ -304,6 +305,8 @@ void LinkerDriver::createFiles(opt::InputArgList &args) {
       break;
     }
   }
+  if (files.empty() && errorCount() == 0)
+    error("no input files");
 }
 
 static StringRef getEntry(opt::InputArgList &args) {
@@ -362,10 +365,8 @@ static void readConfigs(opt::InputArgList &args) {
   config->thinLTOCachePolicy = CHECK(
       parseCachePruningPolicy(args.getLastArgValue(OPT_thinlto_cache_policy)),
       "--thinlto-cache-policy: invalid cache policy");
-  config->thinLTOJobs = args::getInteger(args, OPT_thinlto_jobs, -1u);
   errorHandler().verbose = args.hasArg(OPT_verbose);
   LLVM_DEBUG(errorHandler().verbose = true);
-  threadsEnabled = args.hasFlag(OPT_threads, OPT_no_threads, true);
 
   config->initialMemory = args::getInteger(args, OPT_initial_memory, 0);
   config->globalBase = args::getInteger(args, OPT_global_base, 1024);
@@ -377,6 +378,20 @@ static void readConfigs(opt::InputArgList &args) {
   config->exportDynamic =
       args.hasFlag(OPT_export_dynamic, OPT_no_export_dynamic, config->shared);
 
+  // --threads= takes a positive integer and provides the default value for
+  // --thinlto-jobs=.
+  if (auto *arg = args.getLastArg(OPT_threads)) {
+    StringRef v(arg->getValue());
+    unsigned threads = 0;
+    if (!llvm::to_integer(v, threads, 0) || threads == 0)
+      error(arg->getSpelling() + ": expected a positive integer, but got '" +
+            arg->getValue() + "'");
+    parallel::strategy = hardware_concurrency(threads);
+    config->thinLTOJobs = v;
+  }
+  if (auto *arg = args.getLastArg(OPT_thinlto_jobs))
+    config->thinLTOJobs = arg->getValue();
+
   if (auto *arg = args.getLastArg(OPT_features)) {
     config->features =
         llvm::Optional<std::vector<std::string>>(std::vector<std::string>());
@@ -415,8 +430,8 @@ static void checkOptions(opt::InputArgList &args) {
     error("invalid optimization level for LTO: " + Twine(config->ltoo));
   if (config->ltoPartitions == 0)
     error("--lto-partitions: number of threads must be > 0");
-  if (config->thinLTOJobs == 0)
-    error("--thinlto-jobs: number of threads must be > 0");
+  if (!get_threadpool_strategy(config->thinLTOJobs))
+    error("--thinlto-jobs: invalid job count: " + config->thinLTOJobs);
 
   if (config->pie && config->shared)
     error("-shared and -pie may not be used together");
@@ -716,16 +731,27 @@ void LinkerDriver::link(ArrayRef<const char *> argsArr) {
   errorHandler().errorLimit = args::getInteger(args, OPT_error_limit, 20);
 
   readConfigs(args);
+
+  createFiles(args);
+  if (errorCount())
+    return;
+
   setConfigs();
   checkOptions(args);
+  if (errorCount())
+    return;
 
   if (auto *arg = args.getLastArg(OPT_allow_undefined_file))
     readImportFile(arg->getValue());
 
-  if (!args.hasArg(OPT_INPUT)) {
-    error("no input files");
+  // Fail early if the output file or map file is not writable. If a user has a
+  // long link, e.g. due to a large LTO link, they do not wish to run it and
+  // find that it failed because there was a mistake in their command-line.
+  if (auto e = tryCreateFile(config->outputFile))
+    error("cannot open output file " + config->outputFile + ": " + e.message());
+  // TODO(sbc): add check for map file too once we add support for that.
+  if (errorCount())
     return;
-  }
 
   // Handle --trace-symbol.
   for (auto *arg : args.filtered(OPT_trace_symbol))
@@ -736,10 +762,6 @@ void LinkerDriver::link(ArrayRef<const char *> argsArr) {
 
   createSyntheticSymbols();
 
-  createFiles(args);
-  if (errorCount())
-    return;
-
   // Add all files to the symbol table. This will add almost all
   // symbols that we need to the symbol table.
   for (InputFile *f : files)
@@ -762,7 +784,7 @@ void LinkerDriver::link(ArrayRef<const char *> argsArr) {
     if (entrySym && entrySym->isDefined())
       entrySym->forceExport = true;
     else
-      error("entry symbol not defined (pass --no-entry to supress): " +
+      error("entry symbol not defined (pass --no-entry to suppress): " +
             config->entry);
   }
 
diff --git a/lld/wasm/InputChunks.h b/lld/wasm/InputChunks.h
index 030c9a9dd5619..cadff6883fa4f 100644
--- a/lld/wasm/InputChunks.h
+++ b/lld/wasm/InputChunks.h
@@ -130,8 +130,8 @@ class InputFunction : public InputChunk {
   void writeTo(uint8_t *sectionStart) const override;
   StringRef getName() const override { return function->SymbolName; }
   StringRef getDebugName() const override { return function->DebugName; }
-  StringRef getExportName() const {
-    return function ? function->ExportName : "";
+  llvm::Optional<StringRef> getExportName() const {
+    return function ? function->ExportName : llvm::Optional<StringRef>();
   }
   uint32_t getComdat() const override { return function->Comdat; }
   uint32_t getFunctionInputOffset() const { return getInputSectionOffset(); }
diff --git a/lld/wasm/InputFiles.cpp b/lld/wasm/InputFiles.cpp
index 2aab5bf10e61d..15a85872eefe1 100644
--- a/lld/wasm/InputFiles.cpp
+++ b/lld/wasm/InputFiles.cpp
@@ -168,9 +168,11 @@ uint32_t ObjFile::calcNewValue(const WasmRelocation &reloc) const {
     sym = symbols[reloc.Index];
 
     // We can end up with relocations against non-live symbols.  For example
-    // in debug sections.
+    // in debug sections. We return reloc.Addend because always returning zero
+    // causes the generation of spurious range-list terminators in the
+    // .debug_ranges section.
     if ((isa<FunctionSymbol>(sym) || isa<DataSymbol>(sym)) && !sym->isLive())
-      return 0;
+      return reloc.Addend;
   }
 
   switch (reloc.Type) {
diff --git a/lld/wasm/LTO.cpp b/lld/wasm/LTO.cpp
index 0b7d8cb12d0ea..28b62022f29fe 100644
--- a/lld/wasm/LTO.cpp
+++ b/lld/wasm/LTO.cpp
@@ -63,10 +63,8 @@ static std::unique_ptr<lto::LTO> createLTO() {
   if (config->saveTemps)
     checkError(c.addSaveTemps(config->outputFile.str() + ".",
                               /*UseInputModulePath*/ true));
-
-  lto::ThinBackend backend;
-  if (config->thinLTOJobs != -1U)
-    backend = lto::createInProcessThinBackend(config->thinLTOJobs);
+  lto::ThinBackend backend = lto::createInProcessThinBackend(
+      llvm::heavyweight_hardware_concurrency(config->thinLTOJobs));
   return std::make_unique<lto::LTO>(std::move(c), backend,
                                      config->ltoPartitions);
 }
diff --git a/lld/wasm/Options.td b/lld/wasm/Options.td
index d0275cf5cd49a..0910a74660f3c 100644
--- a/lld/wasm/Options.td
+++ b/lld/wasm/Options.td
@@ -64,9 +64,6 @@ def L: JoinedOrSeparate<["-"], "L">, MetaVarName<"<dir>">,
 
 def mllvm: S<"mllvm">, HelpText<"Options to pass to LLVM">;
 
-def no_threads: F<"no-threads">,
-  HelpText<"Do not run the linker multi-threaded">;
-
 def no_color_diagnostics: F<"no-color-diagnostics">,
   HelpText<"Do not use colors in diagnostics">;
 
@@ -98,7 +95,9 @@ def strip_all: F<"strip-all">, HelpText<"Strip all symbols">;
 
 def strip_debug: F<"strip-debug">, HelpText<"Strip debugging information">;
 
-def threads: F<"threads">, HelpText<"Run the linker multi-threaded">;
+defm threads
+    : Eq<"threads", "Number of threads. '1' disables multi-threading. By "
+                    "default all available hardware threads are used">;
 
 def trace: F<"trace">, HelpText<"Print the names of the input files">;
 
@@ -198,4 +197,5 @@ def save_temps: F<"save-temps">;
 def thinlto_cache_dir: J<"thinlto-cache-dir=">,
   HelpText<"Path to ThinLTO cached object file directory">;
 defm thinlto_cache_policy: Eq<"thinlto-cache-policy", "Pruning policy for the ThinLTO cache">;
-def thinlto_jobs: J<"thinlto-jobs=">, HelpText<"Number of ThinLTO jobs">;
+def thinlto_jobs: J<"thinlto-jobs=">,
+  HelpText<"Number of ThinLTO jobs. Default to --threads=">;
diff --git a/lld/wasm/SymbolTable.cpp b/lld/wasm/SymbolTable.cpp
index dc095adf6dff0..99d18178eeacf 100644
--- a/lld/wasm/SymbolTable.cpp
+++ b/lld/wasm/SymbolTable.cpp
@@ -138,7 +138,7 @@ static void reportTypeError(const Symbol *existing, const InputFile *file,
 }
 
 // Check the type of new symbol matches that of the symbol is replacing.
-// Returns true if the function types match, false is there is a singature
+// Returns true if the function types match, false is there is a signature
 // mismatch.
 static bool signatureMatches(FunctionSymbol *existing,
                              const WasmSignature *newSig) {
@@ -279,7 +279,7 @@ Symbol *SymbolTable::addDefinedFunction(StringRef name, uint32_t flags,
   std::tie(s, wasInserted) = insert(name, file);
 
   auto replaceSym = [&](Symbol *sym) {
-    // If the new defined function doesn't have signture (i.e. bitcode
+    // If the new defined function doesn't have signature (i.e. bitcode
     // functions) but the old symbol does, then preserve the old signature
     const WasmSignature *oldSig = s->getSignature();
     auto* newSym = replaceSymbol<DefinedFunction>(sym, name, flags, file, function);
diff --git a/lld/wasm/SymbolTable.h b/lld/wasm/SymbolTable.h
index 9803ad439f9c1..522ea46b38bc2 100644
--- a/lld/wasm/SymbolTable.h
+++ b/lld/wasm/SymbolTable.h
@@ -108,7 +108,7 @@ class SymbolTable {
   llvm::DenseMap<llvm::CachedHashStringRef, int> symMap;
   std::vector<Symbol *> symVector;
 
-  // For certain symbols types, e.g. function symbols, we allow for muliple
+  // For certain symbols types, e.g. function symbols, we allow for multiple
   // variants of the same symbol with different signatures.
   llvm::DenseMap<llvm::CachedHashStringRef, std::vector<Symbol *>> symVariants;
 
diff --git a/lld/wasm/Symbols.cpp b/lld/wasm/Symbols.cpp
index 349fc651533e7..7112db6b08260 100644
--- a/lld/wasm/Symbols.cpp
+++ b/lld/wasm/Symbols.cpp
@@ -156,7 +156,7 @@ void Symbol::setGOTIndex(uint32_t index) {
   LLVM_DEBUG(dbgs() << "setGOTIndex " << name << " -> " << index << "\n");
   assert(gotIndex == INVALID_INDEX);
   if (config->isPic) {
-    // Any symbol that is assigned a GOT entry must be exported othewise the
+    // Any symbol that is assigned a GOT entry must be exported otherwise the
     // dynamic linker won't be able create the entry that contains it.
     forceExport = true;
   }
diff --git a/lld/wasm/Symbols.h b/lld/wasm/Symbols.h
index 11836e11f3eca..3400cde1c7e25 100644
--- a/lld/wasm/Symbols.h
+++ b/lld/wasm/Symbols.h
@@ -415,7 +415,7 @@ class LazySymbol : public Symbol {
 
   // Lazy symbols can have a signature because they can replace an
   // UndefinedFunction which which case we need to be able to preserve the
-  // signture.
+  // signature.
   // TODO(sbc): This repetition of the signature field is inelegant.  Revisit
   // the use of class hierarchy to represent symbol taxonomy.
   const WasmSignature *signature = nullptr;
diff --git a/lld/wasm/SyntheticSections.cpp b/lld/wasm/SyntheticSections.cpp
index 1f7f15b1bd911..87e4251bffd4a 100644
--- a/lld/wasm/SyntheticSections.cpp
+++ b/lld/wasm/SyntheticSections.cpp
@@ -240,6 +240,26 @@ void MemorySection::writeBody() {
     writeUleb128(os, maxMemoryPages, "max pages");
 }
 
+void EventSection::writeBody() {
+  raw_ostream &os = bodyOutputStream;
+
+  writeUleb128(os, inputEvents.size(), "event count");
+  for (InputEvent *e : inputEvents) {
+    e->event.Type.SigIndex = out.typeSec->lookupType(e->signature);
+    writeEvent(os, e->event);
+  }
+}
+
+void EventSection::addEvent(InputEvent *event) {
+  if (!event->live)
+    return;
+  uint32_t eventIndex =
+      out.importSec->getNumImportedEvents() + inputEvents.size();
+  LLVM_DEBUG(dbgs() << "addEvent: " << eventIndex << "\n");
+  event->setEventIndex(eventIndex);
+  inputEvents.push_back(event);
+}
+
 void GlobalSection::assignIndexes() {
   uint32_t globalIndex = out.importSec->getNumImportedGlobals();
   for (InputGlobal *g : inputGlobals)
@@ -297,26 +317,6 @@ void GlobalSection::addGlobal(InputGlobal *global) {
   inputGlobals.push_back(global);
 }
 
-void EventSection::writeBody() {
-  raw_ostream &os = bodyOutputStream;
-
-  writeUleb128(os, inputEvents.size(), "event count");
-  for (InputEvent *e : inputEvents) {
-    e->event.Type.SigIndex = out.typeSec->lookupType(e->signature);
-    writeEvent(os, e->event);
-  }
-}
-
-void EventSection::addEvent(InputEvent *event) {
-  if (!event->live)
-    return;
-  uint32_t eventIndex =
-      out.importSec->getNumImportedEvents() + inputEvents.size();
-  LLVM_DEBUG(dbgs() << "addEvent: " << eventIndex << "\n");
-  event->setEventIndex(eventIndex);
-  inputEvents.push_back(event);
-}
-
 void ExportSection::writeBody() {
   raw_ostream &os = bodyOutputStream;
 
diff --git a/lld/wasm/SyntheticSections.h b/lld/wasm/SyntheticSections.h
index c33c9f5946eeb..ceb6d604da877 100644
--- a/lld/wasm/SyntheticSections.h
+++ b/lld/wasm/SyntheticSections.h
@@ -141,17 +141,6 @@ class FunctionSection : public SyntheticSection {
 protected:
 };
 
-class MemorySection : public SyntheticSection {
-public:
-  MemorySection() : SyntheticSection(llvm::wasm::WASM_SEC_MEMORY) {}
-
-  bool isNeeded() const override { return !config->importMemory; }
-  void writeBody() override;
-
-  uint32_t numMemoryPages = 0;
-  uint32_t maxMemoryPages = 0;
-};
-
 class TableSection : public SyntheticSection {
 public:
   TableSection() : SyntheticSection(llvm::wasm::WASM_SEC_TABLE) {}
@@ -171,27 +160,15 @@ class TableSection : public SyntheticSection {
   void writeBody() override;
 };
 
-class GlobalSection : public SyntheticSection {
+class MemorySection : public SyntheticSection {
 public:
-  GlobalSection() : SyntheticSection(llvm::wasm::WASM_SEC_GLOBAL) {}
-  uint32_t numGlobals() const {
-    assert(isSealed);
-    return inputGlobals.size() + dataAddressGlobals.size() +
-           staticGotSymbols.size();
-  }
-  bool isNeeded() const override { return numGlobals() > 0; }
-  void assignIndexes() override;
-  void writeBody() override;
-  void addGlobal(InputGlobal *global);
-  void addDataAddressGlobal(DefinedData *global);
-  void addStaticGOTEntry(Symbol *sym);
+  MemorySection() : SyntheticSection(llvm::wasm::WASM_SEC_MEMORY) {}
 
-  std::vector<const DefinedData *> dataAddressGlobals;
+  bool isNeeded() const override { return !config->importMemory; }
+  void writeBody() override;
 
-protected:
-  bool isSealed = false;
-  std::vector<InputGlobal *> inputGlobals;
-  std::vector<Symbol *> staticGotSymbols;
+  uint32_t numMemoryPages = 0;
+  uint32_t maxMemoryPages = 0;
 };
 
 // The event section contains a list of declared wasm events associated with the
@@ -214,6 +191,29 @@ class EventSection : public SyntheticSection {
   std::vector<InputEvent *> inputEvents;
 };
 
+class GlobalSection : public SyntheticSection {
+public:
+  GlobalSection() : SyntheticSection(llvm::wasm::WASM_SEC_GLOBAL) {}
+  uint32_t numGlobals() const {
+    assert(isSealed);
+    return inputGlobals.size() + dataAddressGlobals.size() +
+           staticGotSymbols.size();
+  }
+  bool isNeeded() const override { return numGlobals() > 0; }
+  void assignIndexes() override;
+  void writeBody() override;
+  void addGlobal(InputGlobal *global);
+  void addDataAddressGlobal(DefinedData *global);
+  void addStaticGOTEntry(Symbol *sym);
+
+  std::vector<const DefinedData *> dataAddressGlobals;
+
+protected:
+  bool isSealed = false;
+  std::vector<InputGlobal *> inputGlobals;
+  std::vector<Symbol *> staticGotSymbols;
+};
+
 class ExportSection : public SyntheticSection {
 public:
   ExportSection() : SyntheticSection(llvm::wasm::WASM_SEC_EXPORT) {}
diff --git a/lld/wasm/Writer.cpp b/lld/wasm/Writer.cpp
index 366ca01502e39..9f4b6adc48bb4 100644
--- a/lld/wasm/Writer.cpp
+++ b/lld/wasm/Writer.cpp
@@ -336,8 +336,8 @@ void Writer::addSections() {
   addSection(out.functionSec);
   addSection(out.tableSec);
   addSection(out.memorySec);
-  addSection(out.globalSec);
   addSection(out.eventSec);
+  addSection(out.globalSec);
   addSection(out.exportSec);
   addSection(out.startSec);
   addSection(out.elemSec);
@@ -520,9 +520,8 @@ void Writer::calculateExports() {
     StringRef name = sym->getName();
     WasmExport export_;
     if (auto *f = dyn_cast<DefinedFunction>(sym)) {
-      StringRef exportName = f->function->getExportName();
-      if (!exportName.empty()) {
-        name = exportName;
+      if (Optional<StringRef> exportName = f->function->getExportName()) {
+        name = *exportName;
       }
       export_ = {name, WASM_EXTERNAL_FUNCTION, f->getFunctionIndex()};
     } else if (auto *g = dyn_cast<DefinedGlobal>(sym)) {
@@ -966,8 +965,8 @@ void Writer::createSyntheticSections() {
   out.functionSec = make<FunctionSection>();
   out.tableSec = make<TableSection>();
   out.memorySec = make<MemorySection>();
-  out.globalSec = make<GlobalSection>();
   out.eventSec = make<EventSection>();
+  out.globalSec = make<GlobalSection>();
   out.exportSec = make<ExportSection>();
   out.startSec = make<StartSection>(segments.size());
   out.elemSec = make<ElemSection>();
diff --git a/lldb/CMakeLists.txt b/lldb/CMakeLists.txt
index 00b06119d64ab..bf748020ea407 100644
--- a/lldb/CMakeLists.txt
+++ b/lldb/CMakeLists.txt
@@ -224,6 +224,10 @@ if (LLDB_ENABLE_PYTHON)
   else()
     set(LLDB_PYTHON_INSTALL_PATH ${LLDB_PYTHON_RELATIVE_PATH})
   endif()
+  if (NOT CMAKE_CFG_INTDIR STREQUAL  ".")
+    string(REPLACE ${CMAKE_CFG_INTDIR} "\$\{CMAKE_INSTALL_CONFIG_NAME\}" LLDB_PYTHON_INSTALL_PATH ${LLDB_PYTHON_INSTALL_PATH})
+    string(REPLACE ${CMAKE_CFG_INTDIR} "\$\{CMAKE_INSTALL_CONFIG_NAME\}" lldb_python_build_path ${lldb_python_build_path})
+  endif()
   add_custom_target(lldb-python-scripts)
   add_dependencies(lldb-python-scripts finish_swig)
   install(DIRECTORY ${lldb_python_build_path}/../
diff --git a/lldb/include/lldb/Core/Communication.h b/lldb/include/lldb/Core/Communication.h
index 048541aa4c7c4..4233d5bf5f812 100644
--- a/lldb/include/lldb/Core/Communication.h
+++ b/lldb/include/lldb/Core/Communication.h
@@ -221,7 +221,7 @@ class Communication : public Broadcaster {
   ///
   /// \see
   ///     class Connection
-  void SetConnection(Connection *connection);
+  void SetConnection(std::unique_ptr<Connection> connection);
 
   /// Starts a read thread whose sole purpose it to read bytes from the
   /// current connection. This function will call connection's read function:
diff --git a/lldb/include/lldb/Symbol/Type.h b/lldb/include/lldb/Symbol/Type.h
index 11b83d7734b90..dfff30029168a 100644
--- a/lldb/include/lldb/Symbol/Type.h
+++ b/lldb/include/lldb/Symbol/Type.h
@@ -97,7 +97,7 @@ class Type : public std::enable_shared_from_this<Type>, public UserID {
        llvm::Optional<uint64_t> byte_size, SymbolContextScope *context,
        lldb::user_id_t encoding_uid, EncodingDataType encoding_uid_type,
        const Declaration &decl, const CompilerType &compiler_qual_type,
-       ResolveState compiler_type_resolve_state);
+       ResolveState compiler_type_resolve_state, uint32_t opaque_payload = 0);
 
   // This makes an invalid type.  Used for functions that return a Type when
   // they get an error.
@@ -196,11 +196,11 @@ class Type : public std::enable_shared_from_this<Type>, public UserID {
 
   uint32_t GetEncodingMask();
 
-  bool IsCompleteObjCClass() { return m_is_complete_objc_class; }
-
-  void SetIsCompleteObjCClass(bool is_complete_objc_class) {
-    m_is_complete_objc_class = is_complete_objc_class;
-  }
+  typedef uint32_t Payload;
+  /// Return the language-specific payload.
+  Payload GetPayload() { return m_payload; }
+  /// Return the language-specific payload.
+  void SetPayload(Payload opaque_payload) { m_payload = opaque_payload; }
 
 protected:
   ConstString m_name;
@@ -215,7 +215,8 @@ class Type : public std::enable_shared_from_this<Type>, public UserID {
   Declaration m_decl;
   CompilerType m_compiler_type;
   ResolveState m_compiler_type_resolve_state;
-  bool m_is_complete_objc_class;
+  /// Language-specific flags.
+  Payload m_payload;
 
   Type *GetEncodingType();
 
diff --git a/lldb/include/lldb/Utility/Reproducer.h b/lldb/include/lldb/Utility/Reproducer.h
index bffb0f7c0647a..b8b897d02ea22 100644
--- a/lldb/include/lldb/Utility/Reproducer.h
+++ b/lldb/include/lldb/Utility/Reproducer.h
@@ -98,6 +98,8 @@ class FileProvider : public Provider<FileProvider> {
     return m_collector;
   }
 
+  void recordInterestingDirectory(const llvm::Twine &dir);
+
   void Keep() override {
     auto mapping = GetRoot().CopyByAppendingPathComponent(Info::file);
     // Temporary files that are removed during execution can cause copy errors.
diff --git a/lldb/packages/Python/lldbsuite/test/lldbinline.py b/lldb/packages/Python/lldbsuite/test/lldbinline.py
index 22668b673fdaf..3df26356d908c 100644
--- a/lldb/packages/Python/lldbsuite/test/lldbinline.py
+++ b/lldb/packages/Python/lldbsuite/test/lldbinline.py
@@ -137,6 +137,8 @@ def do_test(self):
         parser.set_breakpoints(target)
 
         process = target.LaunchSimple(None, None, self.get_process_working_directory())
+        self.assertIsNotNone(process, PROCESS_IS_VALID)
+
         hit_breakpoints = 0
 
         while lldbutil.get_stopped_thread(process, lldb.eStopReasonBreakpoint):
diff --git a/lldb/packages/Python/lldbsuite/test/lldbtest.py b/lldb/packages/Python/lldbsuite/test/lldbtest.py
index 966d460ea13d9..5058594505f5b 100644
--- a/lldb/packages/Python/lldbsuite/test/lldbtest.py
+++ b/lldb/packages/Python/lldbsuite/test/lldbtest.py
@@ -2414,9 +2414,12 @@ def expect_expr(
 
         # Set the usual default options for normal expressions.
         options.SetIgnoreBreakpoints(True)
-        options.SetLanguage(frame.GuessLanguage())
 
-        eval_result = frame.EvaluateExpression(expr, options)
+        if self.frame().IsValid():
+          options.SetLanguage(frame.GuessLanguage())
+          eval_result = self.frame().EvaluateExpression(expr, options)
+        else:
+          eval_result = self.target().EvaluateExpression(expr, options)
 
         if not eval_result.GetError().Success():
             self.assertTrue(eval_result.GetError().Success(),
diff --git a/lldb/packages/Python/lldbsuite/test/test_categories.py b/lldb/packages/Python/lldbsuite/test/test_categories.py
index 05ce2a15d844a..f0d6b9ce17dea 100644
--- a/lldb/packages/Python/lldbsuite/test/test_categories.py
+++ b/lldb/packages/Python/lldbsuite/test/test_categories.py
@@ -37,6 +37,7 @@
     'darwin-log': 'Darwin log tests',
     'watchpoint': 'Watchpoint-related tests',
     'lldb-vscode': 'Visual Studio Code debug adaptor tests',
+    'lldb-server': 'Tests related to lldb-server',
 }
 
 
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/.categories b/lldb/packages/Python/lldbsuite/test/tools/lldb-server/.categories
new file mode 100644
index 0000000000000..1966eb022b491
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-server/.categories
@@ -0,0 +1 @@
+lldb-server
diff --git a/lldb/source/API/SBCommunication.cpp b/lldb/source/API/SBCommunication.cpp
index 7c7ba39257d9c..d55ecd35b5570 100644
--- a/lldb/source/API/SBCommunication.cpp
+++ b/lldb/source/API/SBCommunication.cpp
@@ -63,7 +63,7 @@ ConnectionStatus SBCommunication::Connect(const char *url) {
 
   if (m_opaque) {
     if (!m_opaque->HasConnection())
-      m_opaque->SetConnection(Host::CreateDefaultConnection(url).release());
+      m_opaque->SetConnection(Host::CreateDefaultConnection(url));
     return m_opaque->Connect(url, nullptr);
   }
   return eConnectionStatusNoConnection;
@@ -79,7 +79,8 @@ ConnectionStatus SBCommunication::AdoptFileDesriptor(int fd, bool owns_fd) {
       if (m_opaque->IsConnected())
         m_opaque->Disconnect();
     }
-    m_opaque->SetConnection(new ConnectionFileDescriptor(fd, owns_fd));
+    m_opaque->SetConnection(
+        std::make_unique<ConnectionFileDescriptor>(fd, owns_fd));
     if (m_opaque->IsConnected())
       status = eConnectionStatusSuccess;
     else
diff --git a/lldb/source/API/SBPlatform.cpp b/lldb/source/API/SBPlatform.cpp
index ddb77f4f008a7..7ac852488ffbb 100644
--- a/lldb/source/API/SBPlatform.cpp
+++ b/lldb/source/API/SBPlatform.cpp
@@ -306,9 +306,12 @@ SBPlatform &SBPlatform::operator=(const SBPlatform &rhs) {
 SBPlatform::~SBPlatform() = default;
 
 SBPlatform SBPlatform::GetHostPlatform() {
+  LLDB_RECORD_STATIC_METHOD_NO_ARGS(lldb::SBPlatform, SBPlatform,
+                                    GetHostPlatform);
+
   SBPlatform host_platform;
   host_platform.m_opaque_sp = Platform::GetHostPlatform();
-  return host_platform;
+  return LLDB_RECORD_RESULT(host_platform);
 }
 
 bool SBPlatform::IsValid() const {
@@ -756,6 +759,8 @@ void RegisterMethods<SBPlatform>(Registry &R) {
   LLDB_REGISTER_METHOD(lldb::SBEnvironment, SBPlatform, GetEnvironment, ());
   LLDB_REGISTER_METHOD_CONST(lldb::SBUnixSignals, SBPlatform, GetUnixSignals,
                              ());
+  LLDB_REGISTER_STATIC_METHOD(lldb::SBPlatform, SBPlatform, GetHostPlatform,
+                              ());
 }
 
 }
diff --git a/lldb/source/Breakpoint/BreakpointResolverName.cpp b/lldb/source/Breakpoint/BreakpointResolverName.cpp
index 2eb0d3ab58884..25f5bb3f6eed7 100644
--- a/lldb/source/Breakpoint/BreakpointResolverName.cpp
+++ b/lldb/source/Breakpoint/BreakpointResolverName.cpp
@@ -332,65 +332,66 @@ BreakpointResolverName::SearchCallback(SearchFilter &filter,
 
   // Remove any duplicates between the function list and the symbol list
   SymbolContext sc;
-  if (func_list.GetSize()) {
-    for (uint32_t i = 0; i < func_list.GetSize(); i++) {
-      if (func_list.GetContextAtIndex(i, sc)) {
-        bool is_reexported = false;
-
-        if (sc.block && sc.block->GetInlinedFunctionInfo()) {
-          if (!sc.block->GetStartAddress(break_addr))
-            break_addr.Clear();
-        } else if (sc.function) {
-          break_addr = sc.function->GetAddressRange().GetBaseAddress();
-          if (m_skip_prologue && break_addr.IsValid()) {
-            const uint32_t prologue_byte_size =
-                sc.function->GetPrologueByteSize();
-            if (prologue_byte_size)
-              break_addr.SetOffset(break_addr.GetOffset() + prologue_byte_size);
-          }
-        } else if (sc.symbol) {
-          if (sc.symbol->GetType() == eSymbolTypeReExported) {
-            const Symbol *actual_symbol =
-                sc.symbol->ResolveReExportedSymbol(breakpoint.GetTarget());
-            if (actual_symbol) {
-              is_reexported = true;
-              break_addr = actual_symbol->GetAddress();
-            }
-          } else {
-            break_addr = sc.symbol->GetAddress();
-          }
-
-          if (m_skip_prologue && break_addr.IsValid()) {
-            const uint32_t prologue_byte_size =
-                sc.symbol->GetPrologueByteSize();
-            if (prologue_byte_size)
-              break_addr.SetOffset(break_addr.GetOffset() + prologue_byte_size);
-            else {
-              const Architecture *arch =
-                  breakpoint.GetTarget().GetArchitecturePlugin();
-              if (arch)
-                arch->AdjustBreakpointAddress(*sc.symbol, break_addr);
-            }
-          }
+  if (!func_list.GetSize())
+    return Searcher::eCallbackReturnContinue;
+
+  for (uint32_t i = 0; i < func_list.GetSize(); i++) {
+    if (!func_list.GetContextAtIndex(i, sc))
+      continue;
+
+    bool is_reexported = false;
+
+    if (sc.block && sc.block->GetInlinedFunctionInfo()) {
+      if (!sc.block->GetStartAddress(break_addr))
+        break_addr.Clear();
+    } else if (sc.function) {
+      break_addr = sc.function->GetAddressRange().GetBaseAddress();
+      if (m_skip_prologue && break_addr.IsValid()) {
+        const uint32_t prologue_byte_size = sc.function->GetPrologueByteSize();
+        if (prologue_byte_size)
+          break_addr.SetOffset(break_addr.GetOffset() + prologue_byte_size);
+      }
+    } else if (sc.symbol) {
+      if (sc.symbol->GetType() == eSymbolTypeReExported) {
+        const Symbol *actual_symbol =
+            sc.symbol->ResolveReExportedSymbol(breakpoint.GetTarget());
+        if (actual_symbol) {
+          is_reexported = true;
+          break_addr = actual_symbol->GetAddress();
         }
+      } else {
+        break_addr = sc.symbol->GetAddress();
+      }
 
-        if (break_addr.IsValid()) {
-          if (filter.AddressPasses(break_addr)) {
-            bool new_location;
-            BreakpointLocationSP bp_loc_sp(
-                AddLocation(break_addr, &new_location));
-            bp_loc_sp->SetIsReExported(is_reexported);
-            if (bp_loc_sp && new_location && !breakpoint.IsInternal()) {
-              if (log) {
-                StreamString s;
-                bp_loc_sp->GetDescription(&s, lldb::eDescriptionLevelVerbose);
-                LLDB_LOGF(log, "Added location: %s\n", s.GetData());
-              }
-            }
-          }
+      if (m_skip_prologue && break_addr.IsValid()) {
+        const uint32_t prologue_byte_size = sc.symbol->GetPrologueByteSize();
+        if (prologue_byte_size)
+          break_addr.SetOffset(break_addr.GetOffset() + prologue_byte_size);
+        else {
+          const Architecture *arch =
+              breakpoint.GetTarget().GetArchitecturePlugin();
+          if (arch)
+            arch->AdjustBreakpointAddress(*sc.symbol, break_addr);
         }
       }
     }
+
+    if (!break_addr.IsValid())
+      continue;
+
+    if (!filter.AddressPasses(break_addr))
+      continue;
+
+    bool new_location;
+    BreakpointLocationSP bp_loc_sp(AddLocation(break_addr, &new_location));
+    bp_loc_sp->SetIsReExported(is_reexported);
+    if (bp_loc_sp && new_location && !breakpoint.IsInternal()) {
+      if (log) {
+        StreamString s;
+        bp_loc_sp->GetDescription(&s, lldb::eDescriptionLevelVerbose);
+        LLDB_LOGF(log, "Added location: %s\n", s.GetData());
+      }
+    }
   }
 
   return Searcher::eCallbackReturnContinue;
diff --git a/lldb/source/Commands/CommandObjectCommands.cpp b/lldb/source/Commands/CommandObjectCommands.cpp
index cf5125dc9306e..b12377d71512f 100644
--- a/lldb/source/Commands/CommandObjectCommands.cpp
+++ b/lldb/source/Commands/CommandObjectCommands.cpp
@@ -527,14 +527,13 @@ rather than using a positional placeholder:"
     m_option_group.NotifyOptionParsingStarting(&exe_ctx);
 
     OptionsWithRaw args_with_suffix(raw_command_line);
-    const char *remainder = args_with_suffix.GetRawPart().c_str();
 
     if (args_with_suffix.HasArgs())
       if (!ParseOptionsAndNotify(args_with_suffix.GetArgs(), result,
                                  m_option_group, exe_ctx))
         return false;
 
-    llvm::StringRef raw_command_string(remainder);
+    llvm::StringRef raw_command_string = args_with_suffix.GetRawPart();
     Args args(raw_command_string);
 
     if (args.GetArgumentCount() < 2) {
@@ -1171,14 +1170,9 @@ a number follows 'f':"
       return llvm::makeArrayRef(g_regex_options);
     }
 
-    // TODO: Convert these functions to return StringRefs.
-    const char *GetHelp() {
-      return (m_help.empty() ? nullptr : m_help.c_str());
-    }
+    llvm::StringRef GetHelp() { return m_help; }
 
-    const char *GetSyntax() {
-      return (m_syntax.empty() ? nullptr : m_syntax.c_str());
-    }
+    llvm::StringRef GetSyntax() { return m_syntax; }
 
   protected:
     // Instance variables to hold the values for command options.
diff --git a/lldb/source/Commands/CommandObjectSettings.cpp b/lldb/source/Commands/CommandObjectSettings.cpp
index cc1080c8cc0ce..87e0352636e10 100644
--- a/lldb/source/Commands/CommandObjectSettings.cpp
+++ b/lldb/source/Commands/CommandObjectSettings.cpp
@@ -531,10 +531,8 @@ class CommandObjectSettingsList : public CommandObjectParsed {
     if (argc > 0) {
       const bool dump_qualified_name = true;
 
-      // TODO: Convert to StringRef based enumeration.  Requires converting
-      // GetPropertyAtPath first.
-      for (size_t i = 0; i < argc; ++i) {
-        const char *property_path = args.GetArgumentAtIndex(i);
+      for (const Args::ArgEntry &arg : args) {
+        const char *property_path = arg.c_str();
 
         const Property *property =
             GetDebugger().GetValueProperties()->GetPropertyAtPath(
diff --git a/lldb/source/Commands/CommandObjectTarget.cpp b/lldb/source/Commands/CommandObjectTarget.cpp
index 95f81fc6cd545..decdd9c53de2b 100644
--- a/lldb/source/Commands/CommandObjectTarget.cpp
+++ b/lldb/source/Commands/CommandObjectTarget.cpp
@@ -571,7 +571,7 @@ class CommandObjectTargetSelect : public CommandObjectParsed {
   }
 };
 
-#pragma mark CommandObjectTargetSelect
+#pragma mark CommandObjectTargetDelete
 
 // "target delete"
 
@@ -876,21 +876,18 @@ class CommandObjectTargetVariable : public CommandObjectParsed {
     Stream &s = result.GetOutputStream();
 
     if (argc > 0) {
-
-      // TODO: Convert to entry-based iteration.  Requires converting
-      // DumpValueObject.
-      for (size_t idx = 0; idx < argc; ++idx) {
+      for (const Args::ArgEntry &arg : args) {
         VariableList variable_list;
         ValueObjectList valobj_list;
 
-        const char *arg = args.GetArgumentAtIndex(idx);
         size_t matches = 0;
         bool use_var_name = false;
         if (m_option_variable.use_regex) {
-          RegularExpression regex(llvm::StringRef::withNullAsEmpty(arg));
+          RegularExpression regex(
+              llvm::StringRef::withNullAsEmpty(arg.c_str()));
           if (!regex.IsValid()) {
             result.GetErrorStream().Printf(
-                "error: invalid regular expression: '%s'\n", arg);
+                "error: invalid regular expression: '%s'\n", arg.c_str());
             result.SetStatus(eReturnStatusFailed);
             return false;
           }
@@ -900,14 +897,14 @@ class CommandObjectTargetVariable : public CommandObjectParsed {
           matches = variable_list.GetSize();
         } else {
           Status error(Variable::GetValuesForVariableExpressionPath(
-              arg, m_exe_ctx.GetBestExecutionContextScope(),
+              arg.c_str(), m_exe_ctx.GetBestExecutionContextScope(),
               GetVariableCallback, target, variable_list, valobj_list));
           matches = variable_list.GetSize();
         }
 
         if (matches == 0) {
           result.GetErrorStream().Printf(
-              "error: can't find global variable '%s'\n", arg);
+              "error: can't find global variable '%s'\n", arg.c_str());
           result.SetStatus(eReturnStatusFailed);
           return false;
         } else {
@@ -923,7 +920,7 @@ class CommandObjectTargetVariable : public CommandObjectParsed {
               if (valobj_sp)
                 DumpValueObject(s, var_sp, valobj_sp,
                                 use_var_name ? var_sp->GetName().GetCString()
-                                             : arg);
+                                             : arg.c_str());
             }
           }
         }
@@ -2198,7 +2195,7 @@ class CommandObjectTargetModulesDumpSections
   }
 };
 
-#pragma mark CommandObjectTargetModulesDumpSections
+#pragma mark CommandObjectTargetModulesDumpClangAST
 
 // Clang AST dumping command
 
@@ -3058,17 +3055,14 @@ class CommandObjectTargetModulesList : public CommandObjectParsed {
           module_list_ptr = &target->GetImages();
         }
       } else {
-        // TODO: Convert to entry based iteration.  Requires converting
-        // FindModulesByName.
-        for (size_t i = 0; i < argc; ++i) {
+        for (const Args::ArgEntry &arg : command) {
           // Dump specified images (by basename or fullpath)
-          const char *arg_cstr = command.GetArgumentAtIndex(i);
           const size_t num_matches = FindModulesByName(
-              target, arg_cstr, module_list, use_global_module_list);
+              target, arg.c_str(), module_list, use_global_module_list);
           if (num_matches == 0) {
             if (argc == 1) {
               result.AppendErrorWithFormat("no modules found that match '%s'",
-                                           arg_cstr);
+                                           arg.c_str());
               result.SetStatus(eReturnStatusFailed);
               return false;
             }
diff --git a/lldb/source/Commands/CommandObjectWatchpointCommand.cpp b/lldb/source/Commands/CommandObjectWatchpointCommand.cpp
index d86f4a35c5ebd..11bf88de2fea5 100644
--- a/lldb/source/Commands/CommandObjectWatchpointCommand.cpp
+++ b/lldb/source/Commands/CommandObjectWatchpointCommand.cpp
@@ -39,7 +39,7 @@ static constexpr OptionEnumValueElement g_script_option_enumeration[] = {
     {
         eScriptLanguageLua,
         "lua",
-        "Commands are in the Python language.",
+        "Commands are in the Lua language.",
     },
     {
         eSortOrderByName,
diff --git a/lldb/source/Core/Communication.cpp b/lldb/source/Core/Communication.cpp
index 88f6a21ea4d5c..f4163847e4bbd 100644
--- a/lldb/source/Core/Communication.cpp
+++ b/lldb/source/Core/Communication.cpp
@@ -399,10 +399,10 @@ void Communication::SynchronizeWithReadThread() {
   listener_sp->GetEvent(event_sp, llvm::None);
 }
 
-void Communication::SetConnection(Connection *connection) {
+void Communication::SetConnection(std::unique_ptr<Connection> connection) {
   Disconnect(nullptr);
   StopReadThread(nullptr);
-  m_connection_sp.reset(connection);
+  m_connection_sp = std::move(connection);
 }
 
 const char *
diff --git a/lldb/source/Core/Module.cpp b/lldb/source/Core/Module.cpp
index cff74dc9d518d..fefc23a9b1c5b 100644
--- a/lldb/source/Core/Module.cpp
+++ b/lldb/source/Core/Module.cpp
@@ -1414,7 +1414,7 @@ void Module::SetSymbolFileFileSpec(const FileSpec &file) {
         if (FileSystem::Instance().IsDirectory(file)) {
           std::string new_path(file.GetPath());
           std::string old_path(obj_file->GetFileSpec().GetPath());
-          if (old_path.find(new_path) == 0) {
+          if (llvm::StringRef(old_path).startswith(new_path)) {
             // We specified the same bundle as the symbol file that we already
             // have
             return;
diff --git a/lldb/source/Expression/DWARFExpression.cpp b/lldb/source/Expression/DWARFExpression.cpp
index 7f12d06450c6a..04fbfccd0c2e2 100644
--- a/lldb/source/Expression/DWARFExpression.cpp
+++ b/lldb/source/Expression/DWARFExpression.cpp
@@ -2318,6 +2318,12 @@ bool DWARFExpression::Evaluate(
     // rather is a constant value.  The value from the top of the stack is the
     // value to be used.  This is the actual object value and not the location.
     case DW_OP_stack_value:
+      if (stack.empty()) {
+        if (error_ptr)
+          error_ptr->SetErrorString(
+              "Expression stack needs at least 1 item for DW_OP_stack_value.");
+        return false;
+      }
       stack.back().SetValueType(Value::eValueTypeScalar);
       break;
 
diff --git a/lldb/source/Host/common/FileSystem.cpp b/lldb/source/Host/common/FileSystem.cpp
index dcfa594597a1b..7e33eeb2a812d 100644
--- a/lldb/source/Host/common/FileSystem.cpp
+++ b/lldb/source/Host/common/FileSystem.cpp
@@ -87,6 +87,11 @@ Optional<FileSystem> &FileSystem::InstanceImpl() {
 
 vfs::directory_iterator FileSystem::DirBegin(const FileSpec &file_spec,
                                              std::error_code &ec) {
+  if (!file_spec) {
+    ec = std::error_code(static_cast<int>(errc::no_such_file_or_directory),
+                         std::system_category());
+    return {};
+  }
   return DirBegin(file_spec.GetPath(), ec);
 }
 
@@ -97,6 +102,9 @@ vfs::directory_iterator FileSystem::DirBegin(const Twine &dir,
 
 llvm::ErrorOr<vfs::Status>
 FileSystem::GetStatus(const FileSpec &file_spec) const {
+  if (!file_spec)
+    return std::error_code(static_cast<int>(errc::no_such_file_or_directory),
+                           std::system_category());
   return GetStatus(file_spec.GetPath());
 }
 
@@ -106,6 +114,8 @@ llvm::ErrorOr<vfs::Status> FileSystem::GetStatus(const Twine &path) const {
 
 sys::TimePoint<>
 FileSystem::GetModificationTime(const FileSpec &file_spec) const {
+  if (!file_spec)
+    return sys::TimePoint<>();
   return GetModificationTime(file_spec.GetPath());
 }
 
@@ -117,6 +127,8 @@ sys::TimePoint<> FileSystem::GetModificationTime(const Twine &path) const {
 }
 
 uint64_t FileSystem::GetByteSize(const FileSpec &file_spec) const {
+  if (!file_spec)
+    return 0;
   return GetByteSize(file_spec.GetPath());
 }
 
@@ -133,6 +145,8 @@ uint32_t FileSystem::GetPermissions(const FileSpec &file_spec) const {
 
 uint32_t FileSystem::GetPermissions(const FileSpec &file_spec,
                                     std::error_code &ec) const {
+  if (!file_spec)
+    return sys::fs::perms::perms_not_known;
   return GetPermissions(file_spec.GetPath(), ec);
 }
 
@@ -154,7 +168,7 @@ uint32_t FileSystem::GetPermissions(const Twine &path,
 bool FileSystem::Exists(const Twine &path) const { return m_fs->exists(path); }
 
 bool FileSystem::Exists(const FileSpec &file_spec) const {
-  return Exists(file_spec.GetPath());
+  return file_spec && Exists(file_spec.GetPath());
 }
 
 bool FileSystem::Readable(const Twine &path) const {
@@ -162,7 +176,7 @@ bool FileSystem::Readable(const Twine &path) const {
 }
 
 bool FileSystem::Readable(const FileSpec &file_spec) const {
-  return Readable(file_spec.GetPath());
+  return file_spec && Readable(file_spec.GetPath());
 }
 
 bool FileSystem::IsDirectory(const Twine &path) const {
@@ -173,7 +187,7 @@ bool FileSystem::IsDirectory(const Twine &path) const {
 }
 
 bool FileSystem::IsDirectory(const FileSpec &file_spec) const {
-  return IsDirectory(file_spec.GetPath());
+  return file_spec && IsDirectory(file_spec.GetPath());
 }
 
 bool FileSystem::IsLocal(const Twine &path) const {
@@ -183,7 +197,7 @@ bool FileSystem::IsLocal(const Twine &path) const {
 }
 
 bool FileSystem::IsLocal(const FileSpec &file_spec) const {
-  return IsLocal(file_spec.GetPath());
+  return file_spec && IsLocal(file_spec.GetPath());
 }
 
 void FileSystem::EnumerateDirectory(Twine path, bool find_directories,
@@ -261,6 +275,9 @@ void FileSystem::Resolve(SmallVectorImpl<char> &path) {
 }
 
 void FileSystem::Resolve(FileSpec &file_spec) {
+  if (!file_spec)
+    return;
+
   // Extract path from the FileSpec.
   SmallString<128> path;
   file_spec.GetPath(path);
diff --git a/lldb/source/Host/common/Host.cpp b/lldb/source/Host/common/Host.cpp
index b2485393cd6aa..8a6af3881a0fd 100644
--- a/lldb/source/Host/common/Host.cpp
+++ b/lldb/source/Host/common/Host.cpp
@@ -501,6 +501,8 @@ Status Host::RunShellCommand(const Args &args, const FileSpec &working_dir,
     launch_info.SetArguments(args, first_arg_is_executable);
   }
 
+  launch_info.GetEnvironment() = Host::GetEnvironment();
+
   if (working_dir)
     launch_info.SetWorkingDirectory(working_dir);
   llvm::SmallString<64> output_file_path;
diff --git a/lldb/source/Host/macosx/objcxx/Host.mm b/lldb/source/Host/macosx/objcxx/Host.mm
index eba3060f8ec63..045ba7f3671f0 100644
--- a/lldb/source/Host/macosx/objcxx/Host.mm
+++ b/lldb/source/Host/macosx/objcxx/Host.mm
@@ -9,13 +9,9 @@
 #include "lldb/Host/Host.h"
 
 #include <AvailabilityMacros.h>
+#include <TargetConditionals.h>
 
-// On device doesn't have supporty for XPC.
-#if defined(__APPLE__) && (defined(__arm64__) || defined(__aarch64__))
-#define NO_XPC_SERVICES 1
-#endif
-
-#if !defined(NO_XPC_SERVICES)
+#if TARGET_OS_OSX
 #define __XPC_PRIVATE_H__
 #include <xpc/xpc.h>
 
@@ -135,6 +131,8 @@
   return false;
 }
 
+#if TARGET_OS_OSX
+
 static void *AcceptPIDFromInferior(void *arg) {
   const char *connect_url = (const char *)arg;
   ConnectionFileDescriptor file_conn;
@@ -153,8 +151,6 @@
   return NULL;
 }
 
-#if !defined(__arm__) && !defined(__arm64__) && !defined(__aarch64__)
-
 const char *applscript_in_new_tty = "tell application \"Terminal\"\n"
                                     "   activate\n"
                                     "	do script \"/bin/bash -c '%s';exit\"\n"
@@ -307,13 +303,13 @@ repeat with the_window in (get windows)\n\
   return error;
 }
 
-#endif // #if !defined(__arm__) && !defined(__arm64__) && !defined(__aarch64__)
+#endif // TARGET_OS_OSX
 
 bool Host::OpenFileInExternalEditor(const FileSpec &file_spec,
                                     uint32_t line_no) {
-#if defined(__arm__) || defined(__arm64__) || defined(__aarch64__)
+#if !TARGET_OS_OSX
   return false;
-#else
+#else // !TARGET_OS_OSX
   // We attach this to an 'odoc' event to specify a particular selection
   typedef struct {
     int16_t reserved0; // must be zero
@@ -404,7 +400,7 @@ repeat with the_window in (get windows)\n\
   }
 
   return true;
-#endif // #if !defined(__arm__) && !defined(__arm64__) && !defined(__aarch64__)
+#endif // TARGET_OS_OSX
 }
 
 Environment Host::GetEnvironment() { return Environment(*_NSGetEnviron()); }
@@ -689,7 +685,7 @@ static bool GetMacOSXProcessUserAndGroup(ProcessInstanceInfo &process_info) {
   return false;
 }
 
-#if !NO_XPC_SERVICES
+#if TARGET_OS_OSX
 static void PackageXPCArguments(xpc_object_t message, const char *prefix,
                                 const Args &args) {
   size_t count = args.GetArgumentCount();
@@ -841,7 +837,7 @@ static short GetPosixspawnFlags(const ProcessLaunchInfo &launch_info) {
 static Status LaunchProcessXPC(const char *exe_path,
                                ProcessLaunchInfo &launch_info,
                                lldb::pid_t &pid) {
-#if !NO_XPC_SERVICES
+#if TARGET_OS_OSX
   Status error = getXPCAuthorization(launch_info);
   if (error.Fail())
     return error;
@@ -1194,7 +1190,7 @@ static Status LaunchProcessPosixSpawn(const char *exe_path,
 static bool ShouldLaunchUsingXPC(ProcessLaunchInfo &launch_info) {
   bool result = false;
 
-#if !NO_XPC_SERVICES
+#if TARGET_OS_OSX
   bool launchingAsRoot = launch_info.GetUserID() == 0;
   bool currentUserIsRoot = HostInfo::GetEffectiveUserID() == 0;
 
@@ -1226,7 +1222,7 @@ static bool ShouldLaunchUsingXPC(ProcessLaunchInfo &launch_info) {
   }
 
   if (launch_info.GetFlags().Test(eLaunchFlagLaunchInTTY)) {
-#if !defined(__arm__) && !defined(__arm64__) && !defined(__aarch64__)
+#if TARGET_OS_OSX
     return LaunchInNewTerminalWithAppleScript(exe_spec.GetPath().c_str(),
                                               launch_info);
 #else
diff --git a/lldb/source/Host/windows/ProcessLauncherWindows.cpp b/lldb/source/Host/windows/ProcessLauncherWindows.cpp
index 31101944d75cb..00470f558e94d 100644
--- a/lldb/source/Host/windows/ProcessLauncherWindows.cpp
+++ b/lldb/source/Host/windows/ProcessLauncherWindows.cpp
@@ -23,10 +23,9 @@ using namespace lldb_private;
 namespace {
 void CreateEnvironmentBuffer(const Environment &env,
                              std::vector<char> &buffer) {
-  if (env.size() == 0)
-    return;
-
-  // Environment buffer is a null terminated list of null terminated strings
+  // The buffer is a list of null-terminated UTF-16 strings, followed by an
+  // extra L'\0' (two bytes of 0).  An empty environment must have one
+  // empty string, followed by an extra L'\0'.
   for (const auto &KV : env) {
     std::wstring warg;
     if (llvm::ConvertUTF8toWide(Environment::compose(KV), warg)) {
@@ -38,6 +37,9 @@ void CreateEnvironmentBuffer(const Environment &env,
   // One null wchar_t (to end the block) is two null bytes
   buffer.push_back(0);
   buffer.push_back(0);
+  // Insert extra two bytes, just in case the environment was empty.
+  buffer.push_back(0);
+  buffer.push_back(0);
 }
 
 bool GetFlattenedWindowsCommandString(Args args, std::string &command) {
@@ -94,8 +96,7 @@ ProcessLauncherWindows::LaunchProcess(const ProcessLaunchInfo &launch_info,
 
   LPVOID env_block = nullptr;
   ::CreateEnvironmentBuffer(launch_info.GetEnvironment(), environment);
-  if (!environment.empty())
-    env_block = environment.data();
+  env_block = environment.data();
 
   executable = launch_info.GetExecutableFile().GetPath();
   GetFlattenedWindowsCommandString(launch_info.GetArguments(), commandLine);
diff --git a/lldb/source/Interpreter/CommandObject.cpp b/lldb/source/Interpreter/CommandObject.cpp
index 01fa86750b122..f1f17dbd66ef4 100644
--- a/lldb/source/Interpreter/CommandObject.cpp
+++ b/lldb/source/Interpreter/CommandObject.cpp
@@ -1091,7 +1091,7 @@ CommandObject::ArgumentTableEntry CommandObject::g_arguments_data[] = {
     { eArgTypeRunArgs, "run-args", CommandCompletions::eNoCompletion, { nullptr, false }, "Arguments to be passed to the target program when it starts executing." },
     { eArgTypeRunMode, "run-mode", CommandCompletions::eNoCompletion, { nullptr, false }, "Help text goes here." },
     { eArgTypeScriptedCommandSynchronicity, "script-cmd-synchronicity", CommandCompletions::eNoCompletion, { nullptr, false }, "The synchronicity to use to run scripted commands with regard to LLDB event system." },
-    { eArgTypeScriptLang, "script-language", CommandCompletions::eNoCompletion, { nullptr, false }, "The scripting language to be used for script-based commands.  Currently only Python is valid." },
+    { eArgTypeScriptLang, "script-language", CommandCompletions::eNoCompletion, { nullptr, false }, "The scripting language to be used for script-based commands.  Supported languages are python and lua." },
     { eArgTypeSearchWord, "search-word", CommandCompletions::eNoCompletion, { nullptr, false }, "Any word of interest for search purposes." },
     { eArgTypeSelector, "selector", CommandCompletions::eNoCompletion, { nullptr, false }, "An Objective-C selector name." },
     { eArgTypeSettingIndex, "setting-index", CommandCompletions::eNoCompletion, { nullptr, false }, "An index into a settings variable that is an array (try 'settings list' to see all the possible settings variables and their types)." },
diff --git a/lldb/source/Interpreter/OptionValuePathMappings.cpp b/lldb/source/Interpreter/OptionValuePathMappings.cpp
index ebff5c4dca3ec..2784279579f0b 100644
--- a/lldb/source/Interpreter/OptionValuePathMappings.cpp
+++ b/lldb/source/Interpreter/OptionValuePathMappings.cpp
@@ -61,7 +61,7 @@ Status OptionValuePathMappings::SetValueFromString(llvm::StringRef value,
             count);
       } else {
         bool changed = false;
-        for (size_t i = 1; i < argc; i += 2, ++idx) {
+        for (size_t i = 1; i < argc; i += 2) {
           const char *orginal_path = args.GetArgumentAtIndex(i);
           const char *replace_path = args.GetArgumentAtIndex(i + 1);
           if (VerifyPathExists(replace_path)) {
@@ -70,9 +70,13 @@ Status OptionValuePathMappings::SetValueFromString(llvm::StringRef value,
             if (!m_path_mappings.Replace(a, b, idx, m_notify_changes))
               m_path_mappings.Append(a, b, m_notify_changes);
             changed = true;
+            idx++;
           } else {
+            std::string previousError =
+                error.Fail() ? std::string(error.AsCString()) + "\n" : "";
             error.SetErrorStringWithFormat(
-                "the replacement path doesn't exist: \"%s\"", replace_path);
+                "%sthe replacement path doesn't exist: \"%s\"",
+                previousError.c_str(), replace_path);
             break;
           }
         }
@@ -109,9 +113,11 @@ Status OptionValuePathMappings::SetValueFromString(llvm::StringRef value,
           m_value_was_set = true;
           changed = true;
         } else {
+          std::string previousError =
+              error.Fail() ? std::string(error.AsCString()) + "\n" : "";
           error.SetErrorStringWithFormat(
-              "the replacement path doesn't exist: \"%s\"", replace_path);
-          break;
+              "%sthe replacement path doesn't exist: \"%s\"",
+              previousError.c_str(), replace_path);
         }
       }
       if (changed)
@@ -135,7 +141,7 @@ Status OptionValuePathMappings::SetValueFromString(llvm::StringRef value,
         bool changed = false;
         if (op == eVarSetOperationInsertAfter)
           ++idx;
-        for (size_t i = 1; i < argc; i += 2, ++idx) {
+        for (size_t i = 1; i < argc; i += 2) {
           const char *orginal_path = args.GetArgumentAtIndex(i);
           const char *replace_path = args.GetArgumentAtIndex(i + 1);
           if (VerifyPathExists(replace_path)) {
@@ -143,9 +149,13 @@ Status OptionValuePathMappings::SetValueFromString(llvm::StringRef value,
             ConstString b(replace_path);
             m_path_mappings.Insert(a, b, idx, m_notify_changes);
             changed = true;
+            idx++;
           } else {
+            std::string previousError =
+                error.Fail() ? std::string(error.AsCString()) + "\n" : "";
             error.SetErrorStringWithFormat(
-                "the replacement path doesn't exist: \"%s\"", replace_path);
+                "%sthe replacement path doesn't exist: \"%s\"",
+                previousError.c_str(), replace_path);
             break;
           }
         }
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.cpp
index 479cb64deefd1..4b3e237dc62cd 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.cpp
@@ -997,6 +997,8 @@ static void MaybeCompleteReturnType(ClangASTImporter &importer,
   clang::RecordDecl *rd = return_type->getPointeeType()->getAsRecordDecl();
   if (!rd)
     return;
+  if (rd->getDefinition())
+    return;
 
   importer.CompleteTagDecl(rd);
 }
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.cpp
index 698fea4c2d3cd..e5de4b4651df1 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.cpp
@@ -219,11 +219,10 @@ class ClangDiagnosticManagerAdapter : public clang::DiagnosticConsumer {
       // FIXME: Should we try to filter out FixIts that apply to our generated
       // code, and not the user's expression?
       if (severity == eDiagnosticSeverityError) {
-        size_t num_fixit_hints = Info.getNumFixItHints();
-        for (size_t i = 0; i < num_fixit_hints; i++) {
-          const clang::FixItHint &fixit = Info.getFixItHint(i);
-          if (!fixit.isNull())
-            new_diagnostic->AddFixitHint(fixit);
+        for (const clang::FixItHint &fixit : Info.getFixItHints()) {
+          if (fixit.isNull())
+            continue;
+          new_diagnostic->AddFixitHint(fixit);
         }
       }
 
@@ -1071,6 +1070,28 @@ ClangExpressionParser::GetClangTargetABI(const ArchSpec &target_arch) {
   return abi;
 }
 
+/// Applies the given Fix-It hint to the given commit.
+static void ApplyFixIt(const FixItHint &fixit, clang::edit::Commit &commit) {
+  // This is cobbed from clang::Rewrite::FixItRewriter.
+  if (fixit.CodeToInsert.empty()) {
+    if (fixit.InsertFromRange.isValid()) {
+      commit.insertFromRange(fixit.RemoveRange.getBegin(),
+                             fixit.InsertFromRange, /*afterToken=*/false,
+                             fixit.BeforePreviousInsertions);
+      return;
+    }
+    commit.remove(fixit.RemoveRange);
+    return;
+  }
+  if (fixit.RemoveRange.isTokenRange() ||
+      fixit.RemoveRange.getBegin() != fixit.RemoveRange.getEnd()) {
+    commit.replace(fixit.RemoveRange, fixit.CodeToInsert);
+    return;
+  }
+  commit.insert(fixit.RemoveRange.getBegin(), fixit.CodeToInsert,
+                /*afterToken=*/false, fixit.BeforePreviousInsertions);
+}
+
 bool ClangExpressionParser::RewriteExpression(
     DiagnosticManager &diagnostic_manager) {
   clang::SourceManager &source_manager = m_compiler->getSourceManager();
@@ -1102,26 +1123,12 @@ bool ClangExpressionParser::RewriteExpression(
 
   for (const auto &diag : diagnostic_manager.Diagnostics()) {
     const auto *diagnostic = llvm::dyn_cast<ClangDiagnostic>(diag.get());
-    if (diagnostic && diagnostic->HasFixIts()) {
-      for (const FixItHint &fixit : diagnostic->FixIts()) {
-        // This is cobbed from clang::Rewrite::FixItRewriter.
-        if (fixit.CodeToInsert.empty()) {
-          if (fixit.InsertFromRange.isValid()) {
-            commit.insertFromRange(fixit.RemoveRange.getBegin(),
-                                   fixit.InsertFromRange, /*afterToken=*/false,
-                                   fixit.BeforePreviousInsertions);
-          } else
-            commit.remove(fixit.RemoveRange);
-        } else {
-          if (fixit.RemoveRange.isTokenRange() ||
-              fixit.RemoveRange.getBegin() != fixit.RemoveRange.getEnd())
-            commit.replace(fixit.RemoveRange, fixit.CodeToInsert);
-          else
-            commit.insert(fixit.RemoveRange.getBegin(), fixit.CodeToInsert,
-                          /*afterToken=*/false, fixit.BeforePreviousInsertions);
-        }
-      }
-    }
+    if (!diagnostic)
+      continue;
+    if (!diagnostic->HasFixIts())
+      continue;
+    for (const FixItHint &fixit : diagnostic->FixIts())
+      ApplyFixIt(fixit, commit);
   }
 
   // FIXME - do we want to try to propagate specific errors here?
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.cpp
index b246fc374d1cc..2b75c4f75c632 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.cpp
@@ -616,15 +616,14 @@ bool ClangUserExpression::Parse(DiagnosticManager &diagnostic_manager,
       if (parser.RewriteExpression(diagnostic_manager)) {
         size_t fixed_start;
         size_t fixed_end;
-        const std::string &fixed_expression =
-            diagnostic_manager.GetFixedExpression();
+        m_fixed_text = diagnostic_manager.GetFixedExpression();
         // Retrieve the original expression in case we don't have a top level
         // expression (which has no surrounding source code).
         if (m_source_code &&
-            m_source_code->GetOriginalBodyBounds(fixed_expression, m_expr_lang,
+            m_source_code->GetOriginalBodyBounds(m_fixed_text, m_expr_lang,
                                                  fixed_start, fixed_end))
           m_fixed_text =
-              fixed_expression.substr(fixed_start, fixed_end - fixed_start);
+              m_fixed_text.substr(fixed_start, fixed_end - fixed_start);
       }
     }
     return false;
diff --git a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusNameParser.cpp b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusNameParser.cpp
index fb85609a249dd..eca36fff18f83 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusNameParser.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusNameParser.cpp
@@ -329,6 +329,37 @@ bool CPlusPlusNameParser::ConsumeOperator() {
   }
 
   const auto &token = Peek();
+
+  // When clang generates debug info it adds template parameters to names.
+  // Since clang doesn't add a space between the name and the template parameter
+  // in some cases we are not generating valid C++ names e.g.:
+  //
+  //   operator<<A::B>
+  //
+  // In some of these cases we will not parse them correctly. This fixes the
+  // issue by detecting this case and inserting tok::less in place of
+  // tok::lessless and returning successfully that we consumed the operator.
+  if (token.getKind() == tok::lessless) {
+    // Make sure we have more tokens before attempting to look ahead one more.
+    if (m_next_token_index + 1 < m_tokens.size()) {
+      // Look ahead two tokens.
+      clang::Token n_token = m_tokens[m_next_token_index + 1];
+      // If we find ( or < then this is indeed operator<< no need for fix.
+      if (n_token.getKind() != tok::l_paren && n_token.getKind() != tok::less) {
+        clang::Token tmp_tok;
+        tmp_tok.startToken();
+        tmp_tok.setLength(1);
+        tmp_tok.setLocation(token.getLocation().getLocWithOffset(1));
+        tmp_tok.setKind(tok::less);
+
+        m_tokens[m_next_token_index] = tmp_tok;
+
+        start_position.Remove();
+        return true;
+      }
+    }
+  }
+
   switch (token.getKind()) {
   case tok::kw_new:
   case tok::kw_delete:
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.cpp
index 459e02e557450..5fb03a67fcf93 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.cpp
@@ -129,7 +129,7 @@ ObjCLanguageRuntime::LookupInCompleteClassCache(ConstString &name) {
 
       if (TypeSystemClang::IsObjCObjectOrInterfaceType(
               type_sp->GetForwardCompilerType())) {
-        if (type_sp->IsCompleteObjCClass()) {
+        if (TypePayloadClang(type_sp->GetPayload()).IsCompleteObjCClass()) {
           m_complete_class_cache[name] = type_sp;
           return type_sp;
         }
diff --git a/lldb/source/Plugins/ObjectContainer/BSD-Archive/ObjectContainerBSDArchive.cpp b/lldb/source/Plugins/ObjectContainer/BSD-Archive/ObjectContainerBSDArchive.cpp
index 47c7ae8c8d639..83cf9f8bd2694 100644
--- a/lldb/source/Plugins/ObjectContainer/BSD-Archive/ObjectContainerBSDArchive.cpp
+++ b/lldb/source/Plugins/ObjectContainer/BSD-Archive/ObjectContainerBSDArchive.cpp
@@ -88,7 +88,7 @@ ObjectContainerBSDArchive::Object::Extract(const DataExtractor &data,
     return LLDB_INVALID_OFFSET;
 
   str.assign((const char *)data.GetData(&offset, 16), 16);
-  if (str.find("#1/") == 0) {
+  if (llvm::StringRef(str).startswith("#1/")) {
     // If the name is longer than 16 bytes, or contains an embedded space then
     // it will use this format where the length of the name is here and the
     // name characters are after this header.
diff --git a/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp b/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
index 403aa5909bd6a..2b3779ec7826f 100644
--- a/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
+++ b/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
@@ -1239,7 +1239,7 @@ void ObjectFileELF::ParseARMAttributes(DataExtractor &data, uint64_t length,
   lldb::offset_t Offset = 0;
 
   uint8_t FormatVersion = data.GetU8(&Offset);
-  if (FormatVersion != llvm::ARMBuildAttrs::Format_Version)
+  if (FormatVersion != llvm::ELFAttrs::Format_Version)
     return;
 
   Offset = Offset + sizeof(uint32_t); // Section Length
diff --git a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
index ad12819847cda..5044bed309dca 100644
--- a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
+++ b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
@@ -1907,6 +1907,7 @@ class MachSymtabSectionInfo {
   std::vector<SectionInfo> m_section_infos;
 };
 
+#define TRIE_SYMBOL_IS_THUMB (1ULL << 63)
 struct TrieEntry {
   void Dump() const {
     printf("0x%16.16llx 0x%16.16llx 0x%16.16llx \"%s\"",
@@ -1920,7 +1921,9 @@ struct TrieEntry {
   }
   ConstString name;
   uint64_t address = LLDB_INVALID_ADDRESS;
-  uint64_t flags = 0;
+  uint64_t flags =
+      0; // EXPORT_SYMBOL_FLAGS_REEXPORT, EXPORT_SYMBOL_FLAGS_STUB_AND_RESOLVER,
+         // TRIE_SYMBOL_IS_THUMB
   uint64_t other = 0;
   ConstString import_name;
 };
@@ -1943,13 +1946,16 @@ struct TrieEntryWithOffset {
 };
 
 static bool ParseTrieEntries(DataExtractor &data, lldb::offset_t offset,
-                             const bool is_arm,
+                             const bool is_arm, addr_t text_seg_base_addr,
                              std::vector<llvm::StringRef> &nameSlices,
                              std::set<lldb::addr_t> &resolver_addresses,
-                             std::vector<TrieEntryWithOffset> &output) {
+                             std::vector<TrieEntryWithOffset> &reexports,
+                             std::vector<TrieEntryWithOffset> &ext_symbols) {
   if (!data.ValidOffset(offset))
     return true;
 
+  // Terminal node -- end of a branch, possibly add this to
+  // the symbol table or resolver table.
   const uint64_t terminalSize = data.GetULEB128(&offset);
   lldb::offset_t children_offset = offset + terminalSize;
   if (terminalSize != 0) {
@@ -1962,6 +1968,8 @@ static bool ParseTrieEntries(DataExtractor &data, lldb::offset_t offset,
       import_name = data.GetCStr(&offset);
     } else {
       e.entry.address = data.GetULEB128(&offset);
+      if (text_seg_base_addr != LLDB_INVALID_ADDRESS)
+        e.entry.address += text_seg_base_addr;
       if (e.entry.flags & EXPORT_SYMBOL_FLAGS_STUB_AND_RESOLVER) {
         e.entry.other = data.GetULEB128(&offset);
         uint64_t resolver_addr = e.entry.other;
@@ -1971,9 +1979,18 @@ static bool ParseTrieEntries(DataExtractor &data, lldb::offset_t offset,
       } else
         e.entry.other = 0;
     }
-    // Only add symbols that are reexport symbols with a valid import name
-    if (EXPORT_SYMBOL_FLAGS_REEXPORT & e.entry.flags && import_name &&
-        import_name[0]) {
+    bool add_this_entry = false;
+    if (Flags(e.entry.flags).Test(EXPORT_SYMBOL_FLAGS_REEXPORT) &&
+        import_name && import_name[0]) {
+      // add symbols that are reexport symbols with a valid import name.
+      add_this_entry = true;
+    } else if (e.entry.flags == 0 &&
+               (import_name == nullptr || import_name[0] == '\0')) {
+      // add externally visible symbols, in case the nlist record has
+      // been stripped/omitted.
+      add_this_entry = true;
+    }
+    if (add_this_entry) {
       std::string name;
       if (!nameSlices.empty()) {
         for (auto name_slice : nameSlices)
@@ -1987,7 +2004,15 @@ static bool ParseTrieEntries(DataExtractor &data, lldb::offset_t offset,
         // Skip the leading '_'
         e.entry.import_name.SetCString(import_name + 1);
       }
-      output.push_back(e);
+      if (Flags(e.entry.flags).Test(EXPORT_SYMBOL_FLAGS_REEXPORT)) {
+        reexports.push_back(e);
+      } else {
+        if (is_arm && (e.entry.address & 1)) {
+          e.entry.flags |= TRIE_SYMBOL_IS_THUMB;
+          e.entry.address &= THUMB_ADDRESS_BIT_MASK;
+        }
+        ext_symbols.push_back(e);
+      }
     }
   }
 
@@ -2000,8 +2025,9 @@ static bool ParseTrieEntries(DataExtractor &data, lldb::offset_t offset,
       return false; // Corrupt data
     lldb::offset_t childNodeOffset = data.GetULEB128(&children_offset);
     if (childNodeOffset) {
-      if (!ParseTrieEntries(data, childNodeOffset, is_arm, nameSlices,
-                            resolver_addresses, output)) {
+      if (!ParseTrieEntries(data, childNodeOffset, is_arm, text_seg_base_addr,
+                            nameSlices, resolver_addresses, reexports,
+                            ext_symbols)) {
         return false;
       }
     }
@@ -2066,7 +2092,15 @@ size_t ObjectFileMachO::ParseSymtab() {
   struct symtab_command symtab_load_command = {0, 0, 0, 0, 0, 0};
   struct linkedit_data_command function_starts_load_command = {0, 0, 0, 0};
   struct dyld_info_command dyld_info = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+  // The data element of type bool indicates that this entry is thumb
+  // code.
   typedef AddressDataArray<lldb::addr_t, bool, 100> FunctionStarts;
+
+  // Record the address of every function/data that we add to the symtab.
+  // We add symbols to the table in the order of most information (nlist
+  // records) to least (function starts), and avoid duplicating symbols
+  // via this set.
+  std::set<addr_t> symbols_added;
   FunctionStarts function_starts;
   lldb::offset_t offset = MachHeaderSizeFromMagic(m_header.magic);
   uint32_t i;
@@ -2091,29 +2125,6 @@ size_t ObjectFileMachO::ParseSymtab() {
       if (m_data.GetU32(&offset, &symtab_load_command.symoff, 4) ==
           nullptr) // fill in symoff, nsyms, stroff, strsize fields
         return 0;
-      if (symtab_load_command.symoff == 0) {
-        if (log)
-          module_sp->LogMessage(log, "LC_SYMTAB.symoff == 0");
-        return 0;
-      }
-
-      if (symtab_load_command.stroff == 0) {
-        if (log)
-          module_sp->LogMessage(log, "LC_SYMTAB.stroff == 0");
-        return 0;
-      }
-
-      if (symtab_load_command.nsyms == 0) {
-        if (log)
-          module_sp->LogMessage(log, "LC_SYMTAB.nsyms == 0");
-        return 0;
-      }
-
-      if (symtab_load_command.strsize == 0) {
-        if (log)
-          module_sp->LogMessage(log, "LC_SYMTAB.strsize == 0");
-        return 0;
-      }
       break;
 
     case LC_DYLD_INFO:
@@ -2354,27 +2365,7 @@ size_t ObjectFileMachO::ParseSymtab() {
     }
   }
 
-  if (nlist_data.GetByteSize() == 0 &&
-      memory_module_load_level == eMemoryModuleLoadLevelComplete) {
-    if (log)
-      module_sp->LogMessage(log, "failed to read nlist data");
-    return 0;
-  }
-
   const bool have_strtab_data = strtab_data.GetByteSize() > 0;
-  if (!have_strtab_data) {
-    if (process) {
-      if (strtab_addr == LLDB_INVALID_ADDRESS) {
-        if (log)
-          module_sp->LogMessage(log, "failed to locate the strtab in memory");
-        return 0;
-      }
-    } else {
-      if (log)
-        module_sp->LogMessage(log, "failed to read strtab data");
-      return 0;
-    }
-  }
 
   ConstString g_segment_name_TEXT = GetSegmentNameTEXT();
   ConstString g_segment_name_DATA = GetSegmentNameDATA();
@@ -2401,6 +2392,7 @@ size_t ObjectFileMachO::ParseSymtab() {
         section_list->FindSectionByName(g_section_name_eh_frame);
 
   const bool is_arm = (m_header.cputype == llvm::MachO::CPU_TYPE_ARM);
+  const bool always_thumb = GetArchitecture().IsAlwaysThumbInstructions();
 
   // lldb works best if it knows the start address of all functions in a
   // module. Linker symbols or debug info are normally the best source of
@@ -2426,6 +2418,14 @@ size_t ObjectFileMachO::ParseSymtab() {
            0) {
       // Now append the current entry
       function_start_entry.addr += delta;
+      if (is_arm) {
+        if (function_start_entry.addr & 1) {
+          function_start_entry.addr &= THUMB_ADDRESS_BIT_MASK;
+          function_start_entry.data = true;
+        } else if (always_thumb) {
+          function_start_entry.data = true;
+        }
+      }
       function_starts.Append(function_start_entry);
     }
   } else {
@@ -2448,6 +2448,14 @@ size_t ObjectFileMachO::ParseSymtab() {
         if (func) {
           FunctionStarts::Entry function_start_entry;
           function_start_entry.addr = func->base - text_base_addr;
+          if (is_arm) {
+            if (function_start_entry.addr & 1) {
+              function_start_entry.addr &= THUMB_ADDRESS_BIT_MASK;
+              function_start_entry.data = true;
+            } else if (always_thumb) {
+              function_start_entry.data = true;
+            }
+          }
           function_starts.Append(function_start_entry);
         }
       }
@@ -2508,25 +2516,21 @@ size_t ObjectFileMachO::ParseSymtab() {
   std::string memory_symbol_name;
   uint32_t unmapped_local_symbols_found = 0;
 
-  std::vector<TrieEntryWithOffset> trie_entries;
+  std::vector<TrieEntryWithOffset> reexport_trie_entries;
+  std::vector<TrieEntryWithOffset> external_sym_trie_entries;
   std::set<lldb::addr_t> resolver_addresses;
 
   if (dyld_trie_data.GetByteSize() > 0) {
-    std::vector<llvm::StringRef> nameSlices;
-    ParseTrieEntries(dyld_trie_data, 0, is_arm, nameSlices, resolver_addresses,
-                     trie_entries);
-
     ConstString text_segment_name("__TEXT");
     SectionSP text_segment_sp =
         GetSectionList()->FindSectionByName(text_segment_name);
-    if (text_segment_sp) {
-      const lldb::addr_t text_segment_file_addr =
-          text_segment_sp->GetFileAddress();
-      if (text_segment_file_addr != LLDB_INVALID_ADDRESS) {
-        for (auto &e : trie_entries)
-          e.entry.address += text_segment_file_addr;
-      }
-    }
+    lldb::addr_t text_segment_file_addr = LLDB_INVALID_ADDRESS;
+    if (text_segment_sp)
+      text_segment_file_addr = text_segment_sp->GetFileAddress();
+    std::vector<llvm::StringRef> nameSlices;
+    ParseTrieEntries(dyld_trie_data, 0, is_arm, text_segment_file_addr,
+                     nameSlices, resolver_addresses, reexport_trie_entries,
+                     external_sym_trie_entries);
   }
 
   typedef std::set<ConstString> IndirectSymbols;
@@ -3598,6 +3602,9 @@ size_t ObjectFileMachO::ParseSymtab() {
                                     symbol_section);
                                 sym[GSYM_sym_idx].GetAddressRef().SetOffset(
                                     symbol_value);
+                                symbols_added.insert(sym[GSYM_sym_idx]
+                                                         .GetAddress()
+                                                         .GetFileAddress());
                                 // We just need the flags from the linker
                                 // symbol, so put these flags
                                 // into the N_GSYM flags to avoid duplicate
@@ -3617,6 +3624,8 @@ size_t ObjectFileMachO::ParseSymtab() {
                       if (set_value) {
                         sym[sym_idx].GetAddressRef().SetSection(symbol_section);
                         sym[sym_idx].GetAddressRef().SetOffset(symbol_value);
+                        symbols_added.insert(
+                            sym[sym_idx].GetAddress().GetFileAddress());
                       }
                       sym[sym_idx].SetFlags(nlist.n_type << 16 | nlist.n_desc);
 
@@ -4428,6 +4437,8 @@ size_t ObjectFileMachO::ParseSymtab() {
                 // invalid address of zero when the global is a common symbol.
                 sym[GSYM_sym_idx].GetAddressRef().SetSection(symbol_section);
                 sym[GSYM_sym_idx].GetAddressRef().SetOffset(symbol_value);
+                symbols_added.insert(
+                    sym[GSYM_sym_idx].GetAddress().GetFileAddress());
                 // We just need the flags from the linker symbol, so put these
                 // flags into the N_GSYM flags to avoid duplicate symbols in
                 // the symbol table.
@@ -4445,6 +4456,7 @@ size_t ObjectFileMachO::ParseSymtab() {
       if (set_value) {
         sym[sym_idx].GetAddressRef().SetSection(symbol_section);
         sym[sym_idx].GetAddressRef().SetOffset(symbol_value);
+        symbols_added.insert(sym[sym_idx].GetAddress().GetFileAddress());
       }
       sym[sym_idx].SetFlags(nlist.n_type << 16 | nlist.n_desc);
       if (nlist.n_desc & N_WEAK_REF)
@@ -4501,12 +4513,60 @@ size_t ObjectFileMachO::ParseSymtab() {
     }
   }
 
+  // Count how many trie symbols we'll add to the symbol table
+  int trie_symbol_table_augment_count = 0;
+  for (auto &e : external_sym_trie_entries) {
+    if (symbols_added.find(e.entry.address) == symbols_added.end())
+      trie_symbol_table_augment_count++;
+  }
+
+  if (num_syms < sym_idx + trie_symbol_table_augment_count) {
+    num_syms = sym_idx + trie_symbol_table_augment_count;
+    sym = symtab->Resize(num_syms);
+  }
   uint32_t synthetic_sym_id = symtab_load_command.nsyms;
 
+  // Add symbols from the trie to the symbol table.
+  for (auto &e : external_sym_trie_entries) {
+    if (symbols_added.find(e.entry.address) != symbols_added.end())
+      continue;
+
+    // Find the section that this trie address is in, use that to annotate
+    // symbol type as we add the trie address and name to the symbol table.
+    Address symbol_addr;
+    if (module_sp->ResolveFileAddress(e.entry.address, symbol_addr)) {
+      SectionSP symbol_section(symbol_addr.GetSection());
+      if (symbol_section) {
+        sym[sym_idx].SetID(synthetic_sym_id++);
+        sym[sym_idx].GetMangled().SetMangledName(e.entry.name);
+        switch (symbol_section->GetType()) {
+        case eSectionTypeCode:
+          sym[sym_idx].SetType(eSymbolTypeCode);
+          break;
+        case eSectionTypeOther:
+        case eSectionTypeData:
+        case eSectionTypeZeroFill:
+          sym[sym_idx].SetType(eSymbolTypeData);
+          break;
+        default:
+          break;
+        }
+        sym[sym_idx].SetIsSynthetic(false);
+        sym[sym_idx].SetExternal(true);
+        sym[sym_idx].GetAddressRef() = symbol_addr;
+        symbols_added.insert(symbol_addr.GetFileAddress());
+        if (e.entry.flags & TRIE_SYMBOL_IS_THUMB)
+          sym[sym_idx].SetFlags(MACHO_NLIST_ARM_SYMBOL_IS_THUMB);
+        ++sym_idx;
+      }
+    }
+  }
+
   if (function_starts_count > 0) {
     uint32_t num_synthetic_function_symbols = 0;
     for (i = 0; i < function_starts_count; ++i) {
-      if (!function_starts.GetEntryRef(i).data)
+      if (symbols_added.find(function_starts.GetEntryRef(i).addr) ==
+          symbols_added.end())
         ++num_synthetic_function_symbols;
     }
 
@@ -4518,14 +4578,11 @@ size_t ObjectFileMachO::ParseSymtab() {
       for (i = 0; i < function_starts_count; ++i) {
         const FunctionStarts::Entry *func_start_entry =
             function_starts.GetEntryAtIndex(i);
-        if (!func_start_entry->data) {
+        if (symbols_added.find(func_start_entry->addr) == symbols_added.end()) {
           addr_t symbol_file_addr = func_start_entry->addr;
           uint32_t symbol_flags = 0;
-          if (is_arm) {
-            if (symbol_file_addr & 1)
-              symbol_flags = MACHO_NLIST_ARM_SYMBOL_IS_THUMB;
-            symbol_file_addr &= THUMB_ADDRESS_BIT_MASK;
-          }
+          if (func_start_entry->data)
+            symbol_flags = MACHO_NLIST_ARM_SYMBOL_IS_THUMB;
           Address symbol_addr;
           if (module_sp->ResolveFileAddress(symbol_file_addr, symbol_addr)) {
             SectionSP symbol_section(symbol_addr.GetSection());
@@ -4552,6 +4609,7 @@ size_t ObjectFileMachO::ParseSymtab() {
               sym[sym_idx].SetType(eSymbolTypeCode);
               sym[sym_idx].SetIsSynthetic(true);
               sym[sym_idx].GetAddressRef() = symbol_addr;
+              symbols_added.insert(symbol_addr.GetFileAddress());
               if (symbol_flags)
                 sym[sym_idx].SetFlags(symbol_flags);
               if (symbol_byte_size)
@@ -4652,6 +4710,7 @@ size_t ObjectFileMachO::ParseSymtab() {
                     sym[sym_idx].SetType(eSymbolTypeResolver);
                   sym[sym_idx].SetIsSynthetic(true);
                   sym[sym_idx].GetAddressRef() = so_addr;
+                  symbols_added.insert(so_addr.GetFileAddress());
                   sym[sym_idx].SetByteSize(symbol_stub_byte_size);
                   ++sym_idx;
                 }
@@ -4669,8 +4728,8 @@ size_t ObjectFileMachO::ParseSymtab() {
     }
   }
 
-  if (!trie_entries.empty()) {
-    for (const auto &e : trie_entries) {
+  if (!reexport_trie_entries.empty()) {
+    for (const auto &e : reexport_trie_entries) {
       if (e.entry.import_name) {
         // Only add indirect symbols from the Trie entries if we didn't have
         // a N_INDR nlist entry for this already
@@ -5081,10 +5140,10 @@ uint32_t ObjectFileMachO::GetDependentModules(FileSpecList &files) {
       std::string loader_path("@loader_path");
       std::string executable_path("@executable_path");
       for (auto &rpath : rpath_paths) {
-        if (rpath.find(loader_path) == 0) {
+        if (llvm::StringRef(rpath).startswith(loader_path)) {
           rpath.erase(0, loader_path.size());
           rpath.insert(0, this_file_spec.GetDirectory().GetCString());
-        } else if (rpath.find(executable_path) == 0) {
+        } else if (llvm::StringRef(rpath).startswith(executable_path)) {
           rpath.erase(0, executable_path.size());
           rpath.insert(0, this_file_spec.GetDirectory().GetCString());
         }
diff --git a/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp b/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp
index 38b4472f50a75..385b291df709e 100644
--- a/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp
+++ b/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp
@@ -782,6 +782,9 @@ std::unique_ptr<CallFrameInfo> ObjectFilePECOFF::CreateCallFrameInfo() {
   if (!data_dir_exception.vmaddr)
     return {};
 
+  if (m_coff_header.machine != llvm::COFF::IMAGE_FILE_MACHINE_AMD64)
+    return {};
+
   return std::make_unique<PECallFrameInfo>(*this, data_dir_exception.vmaddr,
                                            data_dir_exception.vmsize);
 }
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformAppleSimulator.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformAppleSimulator.cpp
index 41c6999759090..cb6fbce19e589 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformAppleSimulator.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformAppleSimulator.cpp
@@ -77,9 +77,10 @@ void PlatformAppleSimulator::GetStatus(Stream &strm) {
   // simulator
   PlatformAppleSimulator::LoadCoreSimulator();
 
+  std::string developer_dir = GetXcodeDeveloperDirectory().GetPath();
   CoreSimulatorSupport::DeviceSet devices =
       CoreSimulatorSupport::DeviceSet::GetAvailableDevices(
-          GetDeveloperDirectory());
+          developer_dir.c_str());
   const size_t num_devices = devices.GetNumDevices();
   if (num_devices) {
     strm.Printf("Available devices:\n");
@@ -123,9 +124,10 @@ Status PlatformAppleSimulator::ConnectRemote(Args &args) {
     const char *arg_cstr = args.GetArgumentAtIndex(0);
     if (arg_cstr) {
       std::string arg_str(arg_cstr);
+      std::string developer_dir = GetXcodeDeveloperDirectory().GetPath();
       CoreSimulatorSupport::DeviceSet devices =
           CoreSimulatorSupport::DeviceSet::GetAvailableDevices(
-              GetDeveloperDirectory());
+              developer_dir.c_str());
       devices.ForEach(
           [this, &arg_str](const CoreSimulatorSupport::Device &device) -> bool {
             if (arg_str == device.GetUDID() || arg_str == device.GetName()) {
@@ -212,12 +214,12 @@ FileSpec PlatformAppleSimulator::GetCoreSimulatorPath() {
 #if defined(__APPLE__)
   std::lock_guard<std::mutex> guard(m_core_sim_path_mutex);
   if (!m_core_simulator_framework_path.hasValue()) {
-    const char *developer_dir = GetDeveloperDirectory();
-    if (developer_dir) {
+    if (FileSpec fspec = GetXcodeDeveloperDirectory()) {
+      std::string developer_dir = fspec.GetPath();
       StreamString cs_path;
       cs_path.Printf(
           "%s/Library/PrivateFrameworks/CoreSimulator.framework/CoreSimulator",
-          developer_dir);
+          developer_dir.c_str());
       m_core_simulator_framework_path = FileSpec(cs_path.GetData());
       FileSystem::Instance().Resolve(*m_core_simulator_framework_path);
     }
@@ -245,8 +247,9 @@ CoreSimulatorSupport::Device PlatformAppleSimulator::GetSimulatorDevice() {
   if (!m_device.hasValue()) {
     const CoreSimulatorSupport::DeviceType::ProductFamilyID dev_id =
         CoreSimulatorSupport::DeviceType::ProductFamilyID::iPhone;
+    std::string developer_dir = GetXcodeDeveloperDirectory().GetPath();
     m_device = CoreSimulatorSupport::DeviceSet::GetAvailableDevices(
-                   GetDeveloperDirectory())
+                   developer_dir.c_str())
                    .GetFanciest(dev_id);
   }
 
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformAppleTVSimulator.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformAppleTVSimulator.cpp
index 7e3e7c92f1804..fa5f93ba14887 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformAppleTVSimulator.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformAppleTVSimulator.cpp
@@ -255,14 +255,14 @@ EnumerateDirectoryCallback(void *baton, llvm::sys::fs::file_type ft,
 const char *PlatformAppleTVSimulator::GetSDKDirectoryAsCString() {
   std::lock_guard<std::mutex> guard(m_sdk_dir_mutex);
   if (m_sdk_directory.empty()) {
-    const char *developer_dir = GetDeveloperDirectory();
-    if (developer_dir) {
+    if (FileSpec fspec = GetXcodeDeveloperDirectory()) {
+      std::string developer_dir = fspec.GetPath();
       char sdks_directory[PATH_MAX];
       char sdk_dirname[PATH_MAX];
       sdk_dirname[0] = '\0';
       snprintf(sdks_directory, sizeof(sdks_directory),
                "%s/Platforms/AppleTVSimulator.platform/Developer/SDKs",
-               developer_dir);
+               developer_dir.c_str());
       FileSpec simulator_sdk_spec;
       bool find_directories = true;
       bool find_files = false;
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformAppleWatchSimulator.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformAppleWatchSimulator.cpp
index e24923609d4a9..6cd20164ae78d 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformAppleWatchSimulator.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformAppleWatchSimulator.cpp
@@ -255,14 +255,14 @@ EnumerateDirectoryCallback(void *baton, llvm::sys::fs::file_type ft,
 const char *PlatformAppleWatchSimulator::GetSDKDirectoryAsCString() {
   std::lock_guard<std::mutex> guard(m_sdk_dir_mutex);
   if (m_sdk_directory.empty()) {
-    const char *developer_dir = GetDeveloperDirectory();
-    if (developer_dir) {
+    if (FileSpec fspec = GetXcodeDeveloperDirectory()) {
+      std::string developer_dir = fspec.GetPath();
       char sdks_directory[PATH_MAX];
       char sdk_dirname[PATH_MAX];
       sdk_dirname[0] = '\0';
       snprintf(sdks_directory, sizeof(sdks_directory),
                "%s/Platforms/AppleWatchSimulator.platform/Developer/SDKs",
-               developer_dir);
+               developer_dir.c_str());
       FileSpec simulator_sdk_spec;
       bool find_directories = true;
       bool find_files = false;
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp
index 350043f8d4e9a..e1633007f23f9 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp
@@ -48,9 +48,7 @@ using namespace lldb;
 using namespace lldb_private;
 
 /// Default Constructor
-PlatformDarwin::PlatformDarwin(bool is_host)
-    : PlatformPOSIX(is_host), // This is the local host platform
-      m_developer_directory() {}
+PlatformDarwin::PlatformDarwin(bool is_host) : PlatformPOSIX(is_host) {}
 
 /// Destructor.
 ///
@@ -1135,88 +1133,17 @@ static FileSpec GetXcodeSelectPath() {
   return g_xcode_select_filespec;
 }
 
-// Return a directory path like /Applications/Xcode.app/Contents/Developer
-const char *PlatformDarwin::GetDeveloperDirectory() {
-  std::lock_guard<std::mutex> guard(m_mutex);
-  if (m_developer_directory.empty()) {
-    bool developer_dir_path_valid = false;
-    char developer_dir_path[PATH_MAX];
-
-    // Get the lldb framework's file path, and if it exists, truncate some
-    // components to only the developer directory path.
-    FileSpec temp_file_spec = HostInfo::GetShlibDir();
-    if (temp_file_spec) {
-      if (temp_file_spec.GetPath(developer_dir_path,
-                                 sizeof(developer_dir_path))) {
-        // e.g.
-        // /Applications/Xcode.app/Contents/SharedFrameworks/LLDB.framework
-        char *shared_frameworks =
-            strstr(developer_dir_path, "/SharedFrameworks/LLDB.framework");
-        if (shared_frameworks) {
-          shared_frameworks[0] = '\0';  // truncate developer_dir_path at this point
-          strncat (developer_dir_path, "/Developer", sizeof (developer_dir_path) - 1); // add /Developer on
-          developer_dir_path_valid = true;
-        } else {
-          // e.g.
-          // /Applications/Xcode.app/Contents/Developer/Toolchains/iOS11.2.xctoolchain/System/Library/PrivateFrameworks/LLDB.framework
-          char *developer_toolchains =
-            strstr(developer_dir_path, "/Contents/Developer/Toolchains/");
-          if (developer_toolchains) {
-            developer_toolchains += sizeof ("/Contents/Developer") - 1;
-            developer_toolchains[0] = '\0'; // truncate developer_dir_path at this point
-            developer_dir_path_valid = true;
-          }
-        }
-      }
-    }
-
-    if (!developer_dir_path_valid) {
-      std::string xcode_dir_path;
-      const char *xcode_select_prefix_dir = getenv("XCODE_SELECT_PREFIX_DIR");
-      if (xcode_select_prefix_dir)
-        xcode_dir_path.append(xcode_select_prefix_dir);
-      xcode_dir_path.append("/usr/share/xcode-select/xcode_dir_path");
-      temp_file_spec.SetFile(xcode_dir_path, FileSpec::Style::native);
-      auto dir_buffer =
-          FileSystem::Instance().CreateDataBuffer(temp_file_spec.GetPath());
-      if (dir_buffer && dir_buffer->GetByteSize() > 0) {
-        llvm::StringRef path_ref(dir_buffer->GetChars());
-        // Trim tailing newlines and make sure there is enough room for a null
-        // terminator.
-        path_ref =
-            path_ref.rtrim("\r\n").take_front(sizeof(developer_dir_path) - 1);
-        ::memcpy(developer_dir_path, path_ref.data(), path_ref.size());
-        developer_dir_path[path_ref.size()] = '\0';
-        developer_dir_path_valid = true;
-      }
-    }
-
-    if (!developer_dir_path_valid) {
-      FileSpec devel_dir = GetXcodeSelectPath();
-      if (FileSystem::Instance().IsDirectory(devel_dir)) {
-        devel_dir.GetPath(&developer_dir_path[0], sizeof(developer_dir_path));
-        developer_dir_path_valid = true;
-      }
-    }
-
-    if (developer_dir_path_valid) {
-      temp_file_spec.SetFile(developer_dir_path, FileSpec::Style::native);
-      if (FileSystem::Instance().Exists(temp_file_spec)) {
-        m_developer_directory.assign(developer_dir_path);
-        return m_developer_directory.c_str();
-      }
+lldb_private::FileSpec PlatformDarwin::GetXcodeDeveloperDirectory() {
+  static lldb_private::FileSpec g_developer_directory;
+  static llvm::once_flag g_once_flag;
+  llvm::call_once(g_once_flag, []() {
+    if (FileSpec fspec = GetXcodeContentsDirectory()) {
+      fspec.AppendPathComponent("Developer");
+      if (FileSystem::Instance().Exists(fspec))
+        g_developer_directory = fspec;
     }
-    // Assign a single NULL character so we know we tried to find the device
-    // support directory and we don't keep trying to find it over and over.
-    m_developer_directory.assign(1, '\0');
-  }
-
-  // We should have put a single NULL character into m_developer_directory or
-  // it should have a valid path if the code gets here
-  assert(m_developer_directory.empty() == false);
-  if (m_developer_directory[0])
-    return m_developer_directory.c_str();
-  return nullptr;
+  });
+  return g_developer_directory;
 }
 
 BreakpointSP PlatformDarwin::SetThreadCreationBreakpoint(Target &target) {
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.h b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.h
index f6729c508f009..5bdc61c98ae55 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.h
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.h
@@ -104,6 +104,7 @@ class PlatformDarwin : public PlatformPOSIX {
   static llvm::StringRef GetSDKNameForType(SDKType type);
   static lldb_private::FileSpec GetXcodeSDK(SDKType type);
   static lldb_private::FileSpec GetXcodeContentsDirectory();
+  static lldb_private::FileSpec GetXcodeDeveloperDirectory();
 
   /// Return the toolchain directroy the current LLDB instance is located in.
   static lldb_private::FileSpec GetCurrentToolchainDirectory();
@@ -176,7 +177,6 @@ class PlatformDarwin : public PlatformPOSIX {
                                              std::vector<std::string> &options,
                                              SDKType sdk_type);
 
-  const char *GetDeveloperDirectory();
 
   lldb_private::Status FindBundleBinaryInExecSearchPaths(
       const lldb_private::ModuleSpec &module_spec,
@@ -188,7 +188,6 @@ class PlatformDarwin : public PlatformPOSIX {
                                          llvm::StringRef component);
   static std::string FindXcodeContentsDirectoryInPath(llvm::StringRef path);
 
-  std::string m_developer_directory;
 
 private:
   DISALLOW_COPY_AND_ASSIGN(PlatformDarwin);
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwinKernel.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwinKernel.cpp
index 5859856a4e6d3..6d9f20a773695 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwinKernel.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwinKernel.cpp
@@ -327,7 +327,7 @@ void PlatformDarwinKernel::CollectKextAndKernelDirectories() {
 
   // DeveloperDirectory is something like
   // "/Applications/Xcode.app/Contents/Developer"
-  std::string developer_dir = GetDeveloperDirectory();
+  std::string developer_dir = GetXcodeDeveloperDirectory().GetPath();
   if (developer_dir.empty())
     developer_dir = "/Applications/Xcode.app/Contents/Developer";
 
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteDarwinDevice.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteDarwinDevice.cpp
index bfb3f1e0f6b34..40dd903201519 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteDarwinDevice.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteDarwinDevice.cpp
@@ -342,9 +342,8 @@ PlatformRemoteDarwinDevice::GetSDKDirectoryForLatestOSVersion() {
 const char *PlatformRemoteDarwinDevice::GetDeviceSupportDirectory() {
   std::string platform_dir = "/Platforms/" + GetPlatformName() + "/DeviceSupport";
   if (m_device_support_directory.empty()) {
-    const char *device_support_dir = GetDeveloperDirectory();
-    if (device_support_dir) {
-      m_device_support_directory.assign(device_support_dir);
+    if (FileSpec fspec = GetXcodeDeveloperDirectory()) {
+      m_device_support_directory = fspec.GetPath();
       m_device_support_directory.append(platform_dir.c_str());
     } else {
       // Assign a single NULL character so we know we tried to find the device
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformiOSSimulator.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformiOSSimulator.cpp
index 375ddd0722626..b09c4051e2855 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformiOSSimulator.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformiOSSimulator.cpp
@@ -260,14 +260,14 @@ EnumerateDirectoryCallback(void *baton, llvm::sys::fs::file_type ft,
 const char *PlatformiOSSimulator::GetSDKDirectoryAsCString() {
   std::lock_guard<std::mutex> guard(m_sdk_dir_mutex);
   if (m_sdk_directory.empty()) {
-    const char *developer_dir = GetDeveloperDirectory();
-    if (developer_dir) {
+    if (FileSpec fspec = GetXcodeDeveloperDirectory()) {
+      std::string developer_dir = fspec.GetPath();
       char sdks_directory[PATH_MAX];
       char sdk_dirname[PATH_MAX];
       sdk_dirname[0] = '\0';
       snprintf(sdks_directory, sizeof(sdks_directory),
                "%s/Platforms/iPhoneSimulator.platform/Developer/SDKs",
-               developer_dir);
+               developer_dir.c_str());
       FileSpec simulator_sdk_spec;
       bool find_directories = true;
       bool find_files = false;
diff --git a/lldb/source/Plugins/Platform/gdb-server/PlatformRemoteGDBServer.cpp b/lldb/source/Plugins/Platform/gdb-server/PlatformRemoteGDBServer.cpp
index b0467555665c7..657b8fdc729a8 100644
--- a/lldb/source/Plugins/Platform/gdb-server/PlatformRemoteGDBServer.cpp
+++ b/lldb/source/Plugins/Platform/gdb-server/PlatformRemoteGDBServer.cpp
@@ -290,7 +290,7 @@ Status PlatformRemoteGDBServer::ConnectRemote(Args &args) {
                                    GetHostname());
   } else {
     if (args.GetArgumentCount() == 1) {
-      m_gdb_client.SetConnection(new ConnectionFileDescriptor());
+      m_gdb_client.SetConnection(std::make_unique<ConnectionFileDescriptor>());
       // we're going to reuse the hostname when we connect to the debugserver
       int port;
       std::string path;
diff --git a/lldb/source/Plugins/Process/MacOSX-Kernel/ProcessKDP.cpp b/lldb/source/Plugins/Process/MacOSX-Kernel/ProcessKDP.cpp
index 09d1965f25ed7..5b728a5f2960f 100644
--- a/lldb/source/Plugins/Process/MacOSX-Kernel/ProcessKDP.cpp
+++ b/lldb/source/Plugins/Process/MacOSX-Kernel/ProcessKDP.cpp
@@ -250,7 +250,7 @@ Status ProcessKDP::DoConnectRemote(Stream *strm, llvm::StringRef remote_url) {
     const uint16_t reply_port = socket.GetLocalPortNumber();
 
     if (reply_port != 0) {
-      m_comm.SetConnection(conn_up.release());
+      m_comm.SetConnection(std::move(conn_up));
 
       if (m_comm.SendRequestReattach(reply_port)) {
         if (m_comm.SendRequestConnect(reply_port, reply_port,
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp
index 73bd9fb0bed81..070220e0a574c 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp
@@ -869,7 +869,7 @@ Status GDBRemoteCommunication::StartListenThread(const char *hostname,
   else
     snprintf(listen_url, sizeof(listen_url), "listen://%i", port);
   m_listen_url = listen_url;
-  SetConnection(new ConnectionFileDescriptor());
+  SetConnection(std::make_unique<ConnectionFileDescriptor>());
   llvm::Expected<HostThread> listen_thread = ThreadLauncher::LaunchThread(
       listen_url, GDBRemoteCommunication::ListenThread, this);
   if (!listen_thread)
@@ -1252,11 +1252,12 @@ GDBRemoteCommunication::ConnectLocally(GDBRemoteCommunication &client,
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "Unable to connect: %s", status.AsCString());
 
-  client.SetConnection(conn_up.release());
+  client.SetConnection(std::move(conn_up));
   if (llvm::Error error = accept_status.get().ToError())
     return error;
 
-  server.SetConnection(new ConnectionFileDescriptor(accept_socket));
+  server.SetConnection(
+      std::make_unique<ConnectionFileDescriptor>(accept_socket));
   return llvm::Error::success();
 }
 
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.cpp
index 508bc2a634a4a..7d6cb2a3484b9 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.cpp
@@ -1015,9 +1015,9 @@ void GDBRemoteCommunicationServerLLGS::DataAvailableCallback() {
 }
 
 Status GDBRemoteCommunicationServerLLGS::InitializeConnection(
-    std::unique_ptr<Connection> &&connection) {
+    std::unique_ptr<Connection> connection) {
   IOObjectSP read_object_sp = connection->GetReadObject();
-  GDBRemoteCommunicationServer::SetConnection(connection.release());
+  GDBRemoteCommunicationServer::SetConnection(std::move(connection));
 
   Status error;
   m_network_handle_up = m_mainloop.RegisterReadObject(
@@ -1053,7 +1053,7 @@ Status GDBRemoteCommunicationServerLLGS::SetSTDIOFileDescriptor(int fd) {
   }
 
   m_stdio_communication.SetCloseOnEOF(false);
-  m_stdio_communication.SetConnection(conn_up.release());
+  m_stdio_communication.SetConnection(std::move(conn_up));
   if (!m_stdio_communication.IsConnected()) {
     error.SetErrorString(
         "failed to set connection for inferior I/O communication");
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.h b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.h
index ba2fc37539153..f2cb3a8c7e5dd 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.h
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.h
@@ -67,7 +67,7 @@ class GDBRemoteCommunicationServerLLGS
 
   void DidExec(NativeProcessProtocol *process) override;
 
-  Status InitializeConnection(std::unique_ptr<Connection> &&connection);
+  Status InitializeConnection(std::unique_ptr<Connection> connection);
 
 protected:
   MainLoop &m_mainloop;
diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
index 72907a95f3ab3..618ed7ccc040b 100644
--- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
@@ -960,7 +960,7 @@ Status ProcessGDBRemote::ConnectToDebugserver(llvm::StringRef connect_url) {
       uint32_t retry_count = 0;
       while (!m_gdb_comm.IsConnected()) {
         if (conn_up->Connect(connect_url, &error) == eConnectionStatusSuccess) {
-          m_gdb_comm.SetConnection(conn_up.release());
+          m_gdb_comm.SetConnection(std::move(conn_up));
           break;
         } else if (error.WasInterrupted()) {
           // If we were interrupted, don't keep retrying.
@@ -3482,7 +3482,8 @@ Status ProcessGDBRemote::LaunchAndConnectToDebugserver(
       // Our process spawned correctly, we can now set our connection to use
       // our end of the socket pair
       cleanup_our.release();
-      m_gdb_comm.SetConnection(new ConnectionFileDescriptor(our_socket, true));
+      m_gdb_comm.SetConnection(
+          std::make_unique<ConnectionFileDescriptor>(our_socket, true));
 #endif
       StartAsyncThread();
     }
@@ -4418,7 +4419,7 @@ bool ParseRegisters(XMLNode feature_node, GdbServerTargetInfo &target_info,
         });
 
         if (!gdb_type.empty() && !(encoding_set || format_set)) {
-          if (gdb_type.find("int") == 0) {
+          if (llvm::StringRef(gdb_type).startswith("int")) {
             reg_info.format = eFormatHex;
             reg_info.encoding = eEncodingUint;
           } else if (gdb_type == "data_ptr" || gdb_type == "code_ptr") {
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
index f59b70ac31d27..ee94a183e0dcd 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
@@ -953,7 +953,7 @@ bool ScriptInterpreterPythonImpl::ExecuteOneLine(
                                            true));
 #endif
           if (conn_up->IsConnected()) {
-            output_comm.SetConnection(conn_up.release());
+            output_comm.SetConnection(std::move(conn_up));
             output_comm.SetReadThreadBytesReceivedCallback(
                 ReadThreadBytesReceived, &result->GetOutputStream());
             output_comm.StartReadThread();
diff --git a/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.cpp b/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.cpp
index 9c2cea4d97273..eeec7296747e2 100644
--- a/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.cpp
+++ b/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.cpp
@@ -408,20 +408,25 @@ GetRule(llvm::StringRef &unwind_rules) {
 }
 
 static const RegisterInfo *
-ResolveRegister(const SymbolFile::RegisterInfoResolver &resolver,
+ResolveRegister(const llvm::Triple &triple,
+                const SymbolFile::RegisterInfoResolver &resolver,
                 llvm::StringRef name) {
-  if (name.consume_front("$"))
-    return resolver.ResolveName(name);
-
-  return nullptr;
+  if (triple.isX86() || triple.isMIPS()) {
+    // X86 and MIPS registers have '$' in front of their register names. Arm and
+    // AArch64 don't.
+    if (!name.consume_front("$"))
+      return nullptr;
+  }
+  return resolver.ResolveName(name);
 }
 
 static const RegisterInfo *
-ResolveRegisterOrRA(const SymbolFile::RegisterInfoResolver &resolver,
+ResolveRegisterOrRA(const llvm::Triple &triple,
+                    const SymbolFile::RegisterInfoResolver &resolver,
                     llvm::StringRef name) {
   if (name == ".ra")
     return resolver.ResolveNumber(eRegisterKindGeneric, LLDB_REGNUM_GENERIC_PC);
-  return ResolveRegister(resolver, name);
+  return ResolveRegister(triple, resolver, name);
 }
 
 llvm::ArrayRef<uint8_t> SymbolFileBreakpad::SaveAsDWARF(postfix::Node &node) {
@@ -440,6 +445,7 @@ bool SymbolFileBreakpad::ParseCFIUnwindRow(llvm::StringRef unwind_rules,
   Log *log = GetLogIfAllCategoriesSet(LIBLLDB_LOG_SYMBOLS);
 
   llvm::BumpPtrAllocator node_alloc;
+  llvm::Triple triple = m_objfile_sp->GetArchitecture().GetTriple();
   while (auto rule = GetRule(unwind_rules)) {
     node_alloc.Reset();
     llvm::StringRef lhs = rule->first;
@@ -455,7 +461,8 @@ bool SymbolFileBreakpad::ParseCFIUnwindRow(llvm::StringRef unwind_rules,
           if (name == ".cfa" && lhs != ".cfa")
             return postfix::MakeNode<postfix::InitialValueNode>(node_alloc);
 
-          if (const RegisterInfo *info = ResolveRegister(resolver, name)) {
+          if (const RegisterInfo *info =
+                  ResolveRegister(triple, resolver, name)) {
             return postfix::MakeNode<postfix::RegisterNode>(
                 node_alloc, info->kinds[eRegisterKindLLDB]);
           }
@@ -470,7 +477,8 @@ bool SymbolFileBreakpad::ParseCFIUnwindRow(llvm::StringRef unwind_rules,
     llvm::ArrayRef<uint8_t> saved = SaveAsDWARF(*rhs);
     if (lhs == ".cfa") {
       row.GetCFAValue().SetIsDWARFExpression(saved.data(), saved.size());
-    } else if (const RegisterInfo *info = ResolveRegisterOrRA(resolver, lhs)) {
+    } else if (const RegisterInfo *info =
+                   ResolveRegisterOrRA(triple, resolver, lhs)) {
       UnwindPlan::Row::RegisterLocation loc;
       loc.SetIsDWARFExpression(saved.data(), saved.size());
       row.SetRegisterInfo(info->kinds[eRegisterKindLLDB], loc);
@@ -574,6 +582,7 @@ SymbolFileBreakpad::ParseWinUnwindPlan(const Bookmark &bookmark,
     return nullptr;
   }
   auto it = program.begin();
+  llvm::Triple triple = m_objfile_sp->GetArchitecture().GetTriple();
   const auto &symbol_resolver =
       [&](postfix::SymbolNode &symbol) -> postfix::Node * {
     llvm::StringRef name = symbol.GetName();
@@ -581,7 +590,7 @@ SymbolFileBreakpad::ParseWinUnwindPlan(const Bookmark &bookmark,
       if (rule.first == name)
         return rule.second;
     }
-    if (const RegisterInfo *info = ResolveRegister(resolver, name))
+    if (const RegisterInfo *info = ResolveRegister(triple, resolver, name))
       return postfix::MakeNode<postfix::RegisterNode>(
           node_alloc, info->kinds[eRegisterKindLLDB]);
     return nullptr;
@@ -611,7 +620,7 @@ SymbolFileBreakpad::ParseWinUnwindPlan(const Bookmark &bookmark,
 
   // Now process the rest of the assignments.
   for (++it; it != program.end(); ++it) {
-    const RegisterInfo *info = ResolveRegister(resolver, it->first);
+    const RegisterInfo *info = ResolveRegister(triple, resolver, it->first);
     // It is not an error if the resolution fails because the program may
     // contain temporary variables.
     if (!info)
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/AppleDWARFIndex.cpp b/lldb/source/Plugins/SymbolFile/DWARF/AppleDWARFIndex.cpp
index 50b3a2cba61f5..027eb08e56219 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/AppleDWARFIndex.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/AppleDWARFIndex.cpp
@@ -53,8 +53,9 @@ std::unique_ptr<AppleDWARFIndex> AppleDWARFIndex::Create(
 }
 
 void AppleDWARFIndex::GetGlobalVariables(ConstString basename, DIEArray &offsets) {
-  if (m_apple_names_up)
-    m_apple_names_up->FindByName(basename.GetStringRef(), offsets);
+  if (!m_apple_names_up)
+    return;
+  m_apple_names_up->FindByName(basename.GetStringRef(), offsets);
 }
 
 void AppleDWARFIndex::GetGlobalVariables(const RegularExpression &regex,
@@ -80,22 +81,24 @@ void AppleDWARFIndex::GetGlobalVariables(const DWARFUnit &cu,
 
 void AppleDWARFIndex::GetObjCMethods(ConstString class_name,
                                      DIEArray &offsets) {
-  if (m_apple_objc_up)
-    m_apple_objc_up->FindByName(class_name.GetStringRef(), offsets);
+  if (!m_apple_objc_up)
+    return;
+  m_apple_objc_up->FindByName(class_name.GetStringRef(), offsets);
 }
 
 void AppleDWARFIndex::GetCompleteObjCClass(ConstString class_name,
                                            bool must_be_implementation,
                                            DIEArray &offsets) {
-  if (m_apple_types_up) {
-    m_apple_types_up->FindCompleteObjCClassByName(
-        class_name.GetStringRef(), offsets, must_be_implementation);
-  }
+  if (!m_apple_types_up)
+    return;
+  m_apple_types_up->FindCompleteObjCClassByName(
+      class_name.GetStringRef(), offsets, must_be_implementation);
 }
 
 void AppleDWARFIndex::GetTypes(ConstString name, DIEArray &offsets) {
-  if (m_apple_types_up)
-    m_apple_types_up->FindByName(name.GetStringRef(), offsets);
+  if (!m_apple_types_up)
+    return;
+  m_apple_types_up->FindByName(name.GetStringRef(), offsets);
 }
 
 void AppleDWARFIndex::GetTypes(const DWARFDeclContext &context,
@@ -149,8 +152,9 @@ void AppleDWARFIndex::GetTypes(const DWARFDeclContext &context,
 }
 
 void AppleDWARFIndex::GetNamespaces(ConstString name, DIEArray &offsets) {
-  if (m_apple_namespaces_up)
-    m_apple_namespaces_up->FindByName(name.GetStringRef(), offsets);
+  if (!m_apple_namespaces_up)
+    return;
+  m_apple_namespaces_up->FindByName(name.GetStringRef(), offsets);
 }
 
 void AppleDWARFIndex::GetFunctions(ConstString name, SymbolFileDWARF &dwarf,
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
index 8838888b2e23d..96500da4c5168 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
@@ -1634,12 +1634,11 @@ DWARFASTParserClang::ParseStructureLikeDIE(const SymbolContext &sc,
   // parameters in any class methods need it for the clang types for
   // function prototypes.
   LinkDeclContextToDIE(m_ast.GetDeclContextForType(clang_type), die);
-  type_sp = std::make_shared<Type>(die.GetID(), dwarf, attrs.name,
-                                   attrs.byte_size, nullptr, LLDB_INVALID_UID,
-                                   Type::eEncodingIsUID, &attrs.decl,
-                                   clang_type, Type::ResolveState::Forward);
-
-  type_sp->SetIsCompleteObjCClass(attrs.is_complete_objc_class);
+  type_sp = std::make_shared<Type>(
+      die.GetID(), dwarf, attrs.name, attrs.byte_size, nullptr,
+      LLDB_INVALID_UID, Type::eEncodingIsUID, &attrs.decl, clang_type,
+      Type::ResolveState::Forward,
+      TypePayloadClang(attrs.is_complete_objc_class));
 
   // Add our type to the unique type map so we don't end up creating many
   // copies of the same type over and over in the ASTContext for our
@@ -2668,9 +2667,19 @@ void DWARFASTParserClang::ParseSingleMember(
               }
 
               // If we have a gap between the last_field_end and the current
-              // field we have an unnamed bit-field
+              // field we have an unnamed bit-field.
+              // If we have a base class, we assume there is no unnamed
+              // bit-field if this is the first field since the gap can be
+              // attributed to the members from the base class. This assumption
+              // is not correct if the first field of the derived class is
+              // indeed an unnamed bit-field. We currently do not have the
+              // machinary to track the offset of the last field of classes we
+              // have seen before, so we are not handling this case.
               if (this_field_info.bit_offset != last_field_end &&
-                  !(this_field_info.bit_offset < last_field_end)) {
+                  this_field_info.bit_offset > last_field_end &&
+                  !(last_field_info.bit_offset == 0 &&
+                    last_field_info.bit_size == 0 &&
+                    layout_info.base_offsets.size() != 0)) {
                 unnamed_field_info = FieldInfo{};
                 unnamed_field_info->bit_size =
                     this_field_info.bit_offset - last_field_end;
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp
index 0198a10645c22..8447d4fcea4c5 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp
@@ -263,7 +263,8 @@ void DWARFUnit::SetDwoStrOffsetsBase() {
   lldb::offset_t baseOffset = 0;
 
   if (const llvm::DWARFUnitIndex::Entry *entry = m_header.GetIndexEntry()) {
-    if (const auto *contribution = entry->getOffset(llvm::DW_SECT_STR_OFFSETS))
+    if (const auto *contribution =
+            entry->getContribution(llvm::DW_SECT_STR_OFFSETS))
       baseOffset = contribution->Offset;
     else
       return;
@@ -479,7 +480,7 @@ DWARFDataExtractor DWARFUnit::GetLocationData() const {
   const DWARFDataExtractor &data =
       GetVersion() >= 5 ? Ctx.getOrLoadLocListsData() : Ctx.getOrLoadLocData();
   if (const llvm::DWARFUnitIndex::Entry *entry = m_header.GetIndexEntry()) {
-    if (const auto *contribution = entry->getOffset(llvm::DW_SECT_LOC))
+    if (const auto *contribution = entry->getContribution(llvm::DW_SECT_LOC))
       return DWARFDataExtractor(data, contribution->Offset,
                                 contribution->Length);
     return DWARFDataExtractor();
@@ -815,12 +816,13 @@ DWARFUnitHeader::extract(const DWARFDataExtractor &data,
           llvm::inconvertibleErrorCode(),
           "Package unit with a non-zero abbreviation offset");
     }
-    auto *unit_contrib = header.m_index_entry->getOffset();
+    auto *unit_contrib = header.m_index_entry->getContribution();
     if (!unit_contrib || unit_contrib->Length != header.m_length + 4) {
       return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                      "Inconsistent DWARF package unit index");
     }
-    auto *abbr_entry = header.m_index_entry->getOffset(llvm::DW_SECT_ABBREV);
+    auto *abbr_entry =
+        header.m_index_entry->getContribution(llvm::DW_SECT_ABBREV);
     if (!abbr_entry) {
       return llvm::createStringError(
           llvm::inconvertibleErrorCode(),
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.cpp
index 5a2fa70138dd7..d3a4b92b7e3ea 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.cpp
@@ -165,9 +165,8 @@ void DebugNamesDWARFIndex::GetCompleteObjCClass(ConstString class_name,
       // If we find the complete version we're done.
       offsets.push_back(*ref);
       return;
-    } else {
-      incomplete_types.push_back(*ref);
     }
+    incomplete_types.push_back(*ref);
   }
 
   offsets.insert(offsets.end(), incomplete_types.begin(),
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/HashedNameToDIE.cpp b/lldb/source/Plugins/SymbolFile/DWARF/HashedNameToDIE.cpp
index aa01668ba05c1..5f01b8176b98e 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/HashedNameToDIE.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/HashedNameToDIE.cpp
@@ -21,19 +21,19 @@ void DWARFMappedHash::ExtractDIEArray(const DIEInfoArray &die_info_array,
                                       DIEArray &die_offsets) {
   if (tag == 0) {
     ExtractDIEArray(die_info_array, die_offsets);
-  } else {
-    const size_t count = die_info_array.size();
-    for (size_t i = 0; i < count; ++i) {
-      const dw_tag_t die_tag = die_info_array[i].tag;
-      bool tag_matches = die_tag == 0 || tag == die_tag;
-      if (!tag_matches) {
-        if (die_tag == DW_TAG_class_type || die_tag == DW_TAG_structure_type)
-          tag_matches =
-              tag == DW_TAG_structure_type || tag == DW_TAG_class_type;
-      }
-      if (tag_matches)
-        die_offsets.emplace_back(die_info_array[i]);
+    return;
+  }
+
+  const size_t count = die_info_array.size();
+  for (size_t i = 0; i < count; ++i) {
+    const dw_tag_t die_tag = die_info_array[i].tag;
+    bool tag_matches = die_tag == 0 || tag == die_tag;
+    if (!tag_matches) {
+      if (die_tag == DW_TAG_class_type || die_tag == DW_TAG_structure_type)
+        tag_matches = tag == DW_TAG_structure_type || tag == DW_TAG_class_type;
     }
+    if (tag_matches)
+      die_offsets.emplace_back(die_info_array[i]);
   }
 }
 
@@ -43,21 +43,21 @@ void DWARFMappedHash::ExtractDIEArray(const DIEInfoArray &die_info_array,
                                       DIEArray &die_offsets) {
   if (tag == 0) {
     ExtractDIEArray(die_info_array, die_offsets);
-  } else {
-    const size_t count = die_info_array.size();
-    for (size_t i = 0; i < count; ++i) {
-      if (qualified_name_hash != die_info_array[i].qualified_name_hash)
-        continue;
-      const dw_tag_t die_tag = die_info_array[i].tag;
-      bool tag_matches = die_tag == 0 || tag == die_tag;
-      if (!tag_matches) {
-        if (die_tag == DW_TAG_class_type || die_tag == DW_TAG_structure_type)
-          tag_matches =
-              tag == DW_TAG_structure_type || tag == DW_TAG_class_type;
-      }
-      if (tag_matches)
-        die_offsets.emplace_back(die_info_array[i]);
+    return;
+  }
+
+  const size_t count = die_info_array.size();
+  for (size_t i = 0; i < count; ++i) {
+    if (qualified_name_hash != die_info_array[i].qualified_name_hash)
+      continue;
+    const dw_tag_t die_tag = die_info_array[i].tag;
+    bool tag_matches = die_tag == 0 || tag == die_tag;
+    if (!tag_matches) {
+      if (die_tag == DW_TAG_class_type || die_tag == DW_TAG_structure_type)
+        tag_matches = tag == DW_TAG_structure_type || tag == DW_TAG_class_type;
     }
+    if (tag_matches)
+      die_offsets.emplace_back(die_info_array[i]);
   }
 }
 
@@ -67,22 +67,22 @@ void DWARFMappedHash::ExtractClassOrStructDIEArray(
   const size_t count = die_info_array.size();
   for (size_t i = 0; i < count; ++i) {
     const dw_tag_t die_tag = die_info_array[i].tag;
-    if (die_tag == 0 || die_tag == DW_TAG_class_type ||
-        die_tag == DW_TAG_structure_type) {
-      if (die_info_array[i].type_flags & eTypeFlagClassIsImplementation) {
-        if (return_implementation_only_if_available) {
-          // We found the one true definition for this class, so only return
-          // that
-          die_offsets.clear();
-          die_offsets.emplace_back(die_info_array[i]);
-          return;
-        } else {
-          // Put the one true definition as the first entry so it matches first
-          die_offsets.emplace(die_offsets.begin(), die_info_array[i]);
-        }
-      } else {
+    if (die_tag != 0 && die_tag != DW_TAG_class_type &&
+        die_tag != DW_TAG_structure_type)
+      continue;
+    if (die_info_array[i].type_flags & eTypeFlagClassIsImplementation) {
+      if (return_implementation_only_if_available) {
+        // We found the one true definition for this class, so only return
+        // that
+        die_offsets.clear();
         die_offsets.emplace_back(die_info_array[i]);
+        return;
+      } else {
+        // Put the one true definition as the first entry so it matches first
+        die_offsets.emplace(die_offsets.begin(), die_info_array[i]);
       }
+    } else {
+      die_offsets.emplace_back(die_info_array[i]);
     }
   }
 }
@@ -548,24 +548,24 @@ size_t DWARFMappedHash::MemoryTable::FindByNameAndTagAndQualifiedNameHash(
 size_t DWARFMappedHash::MemoryTable::FindCompleteObjCClassByName(
     llvm::StringRef name, DIEArray &die_offsets, bool must_be_implementation) {
   DIEInfoArray die_info_array;
-  if (FindByName(name, die_info_array)) {
-    if (must_be_implementation &&
-        GetHeader().header_data.ContainsAtom(eAtomTypeTypeFlags)) {
-      // If we have two atoms, then we have the DIE offset and the type flags
-      // so we can find the objective C class efficiently.
-      DWARFMappedHash::ExtractTypesFromDIEArray(die_info_array, UINT32_MAX,
-                                                eTypeFlagClassIsImplementation,
-                                                die_offsets);
-    } else {
-      // We don't only want the one true definition, so try and see what we can
-      // find, and only return class or struct DIEs. If we do have the full
-      // implementation, then return it alone, else return all possible
-      // matches.
-      const bool return_implementation_only_if_available = true;
-      DWARFMappedHash::ExtractClassOrStructDIEArray(
-          die_info_array, return_implementation_only_if_available, die_offsets);
-    }
+  if (!FindByName(name, die_info_array))
+    return 0;
+  if (must_be_implementation &&
+      GetHeader().header_data.ContainsAtom(eAtomTypeTypeFlags)) {
+    // If we have two atoms, then we have the DIE offset and the type flags
+    // so we can find the objective C class efficiently.
+    DWARFMappedHash::ExtractTypesFromDIEArray(die_info_array, UINT32_MAX,
+                                              eTypeFlagClassIsImplementation,
+                                              die_offsets);
+    return die_offsets.size();
   }
+  // We don't only want the one true definition, so try and see what we can
+  // find, and only return class or struct DIEs. If we do have the full
+  // implementation, then return it alone, else return all possible
+  // matches.
+  const bool return_implementation_only_if_available = true;
+  DWARFMappedHash::ExtractClassOrStructDIEArray(
+      die_info_array, return_implementation_only_if_available, die_offsets);
   return die_offsets.size();
 }
 
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/ManualDWARFIndex.cpp b/lldb/source/Plugins/SymbolFile/DWARF/ManualDWARFIndex.cpp
index cdeb309649879..cb783314d3200 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/ManualDWARFIndex.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/ManualDWARFIndex.cpp
@@ -426,8 +426,9 @@ void ManualDWARFIndex::GetFunctions(ConstString name, SymbolFileDWARF &dwarf,
       DWARFDIE die = dwarf.GetDIE(die_ref);
       if (!die)
         continue;
-      if (SymbolFileDWARF::DIEInDeclContext(parent_decl_ctx, die))
-        dies.push_back(die);
+      if (!SymbolFileDWARF::DIEInDeclContext(parent_decl_ctx, die))
+        continue;
+      dies.push_back(die);
     }
   }
   if (name_type_mask & eFunctionNameTypeBase) {
@@ -437,18 +438,20 @@ void ManualDWARFIndex::GetFunctions(ConstString name, SymbolFileDWARF &dwarf,
       DWARFDIE die = dwarf.GetDIE(die_ref);
       if (!die)
         continue;
-      if (SymbolFileDWARF::DIEInDeclContext(parent_decl_ctx, die))
-        dies.push_back(die);
+      if (!SymbolFileDWARF::DIEInDeclContext(parent_decl_ctx, die))
+        continue;
+      dies.push_back(die);
     }
-    offsets.clear();
   }
 
   if (name_type_mask & eFunctionNameTypeMethod && !parent_decl_ctx.IsValid()) {
     DIEArray offsets;
     m_set.function_methods.Find(name, offsets);
     for (const DIERef &die_ref: offsets) {
-      if (DWARFDIE die = dwarf.GetDIE(die_ref))
-        dies.push_back(die);
+      DWARFDIE die = dwarf.GetDIE(die_ref);
+      if (!die)
+        continue;
+      dies.push_back(die);
     }
   }
 
@@ -457,8 +460,10 @@ void ManualDWARFIndex::GetFunctions(ConstString name, SymbolFileDWARF &dwarf,
     DIEArray offsets;
     m_set.function_selectors.Find(name, offsets);
     for (const DIERef &die_ref: offsets) {
-      if (DWARFDIE die = dwarf.GetDIE(die_ref))
-        dies.push_back(die);
+      DWARFDIE die = dwarf.GetDIE(die_ref);
+      if (!die)
+        continue;
+      dies.push_back(die);
     }
   }
 }
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
index c98694fca6b5a..1113ed1195a3b 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
@@ -631,6 +631,22 @@ DWARFDebugRanges *SymbolFileDWARF::GetDebugRanges() {
   return m_ranges.get();
 }
 
+/// Make an absolute path out of \p file_spec and remap it using the
+/// module's source remapping dictionary.
+static void MakeAbsoluteAndRemap(FileSpec &file_spec, DWARFUnit &dwarf_cu,
+                                 const ModuleSP &module_sp) {
+  if (!file_spec)
+    return;
+  // If we have a full path to the compile unit, we don't need to
+  // resolve the file.  This can be expensive e.g. when the source
+  // files are NFS mounted.
+  file_spec.MakeAbsolute(dwarf_cu.GetCompilationDirectory());
+
+  std::string remapped_file;
+  if (module_sp->RemapSourceFile(file_spec.GetPath(), remapped_file))
+    file_spec.SetFile(remapped_file, FileSpec::Style::native);
+}
+
 lldb::CompUnitSP SymbolFileDWARF::ParseCompileUnit(DWARFCompileUnit &dwarf_cu) {
   CompUnitSP cu_sp;
   CompileUnit *comp_unit = (CompileUnit *)dwarf_cu.GetUserData();
@@ -649,17 +665,7 @@ lldb::CompUnitSP SymbolFileDWARF::ParseCompileUnit(DWARFCompileUnit &dwarf_cu) {
             dwarf_cu.GetNonSkeletonUnit().GetUnitDIEOnly();
         if (cu_die) {
           FileSpec cu_file_spec(cu_die.GetName(), dwarf_cu.GetPathStyle());
-          if (cu_file_spec) {
-            // If we have a full path to the compile unit, we don't need to
-            // resolve the file.  This can be expensive e.g. when the source
-            // files are NFS mounted.
-            cu_file_spec.MakeAbsolute(dwarf_cu.GetCompilationDirectory());
-
-            std::string remapped_file;
-            if (module_sp->RemapSourceFile(cu_file_spec.GetPath(),
-                                           remapped_file))
-              cu_file_spec.SetFile(remapped_file, FileSpec::Style::native);
-          }
+          MakeAbsoluteAndRemap(cu_file_spec, dwarf_cu, module_sp);
 
           LanguageType cu_language = SymbolFileDWARF::LanguageTypeFromDWARF(
               cu_die.GetAttributeValueAsUnsigned(DW_AT_language, 0));
@@ -921,8 +927,11 @@ bool SymbolFileDWARF::ParseImportedModules(
       }
       std::reverse(module.path.begin(), module.path.end());
       if (const char *include_path = module_die.GetAttributeValueAsString(
-              DW_AT_LLVM_include_path, nullptr))
-        module.search_path = ConstString(include_path);
+              DW_AT_LLVM_include_path, nullptr)) {
+        FileSpec include_spec(include_path, dwarf_cu->GetPathStyle());
+        MakeAbsoluteAndRemap(include_spec, *dwarf_cu, m_objfile_sp->GetModule());
+        module.search_path = ConstString(include_spec.GetPath());
+      }
       if (const char *sysroot = dwarf_cu->DIE().GetAttributeValueAsString(
               DW_AT_LLVM_sysroot, nullptr))
         module.sysroot = ConstString(sysroot);
@@ -2133,25 +2142,23 @@ void SymbolFileDWARF::FindGlobalVariables(const RegularExpression &regex,
   assert(sc.module_sp);
 
   const size_t num_matches = die_offsets.size();
-  if (num_matches) {
-    for (size_t i = 0; i < num_matches; ++i) {
-      const DIERef &die_ref = die_offsets[i];
-      DWARFDIE die = GetDIE(die_ref);
+  for (size_t i = 0; i < num_matches; ++i) {
+    const DIERef &die_ref = die_offsets[i];
+    DWARFDIE die = GetDIE(die_ref);
+    if (!die) {
+      m_index->ReportInvalidDIERef(die_ref, regex.GetText());
+      continue;
+    }
 
-      if (die) {
-        DWARFCompileUnit *dwarf_cu =
-            llvm::dyn_cast<DWARFCompileUnit>(die.GetCU());
-        if (!dwarf_cu)
-          continue;
-        sc.comp_unit = GetCompUnitForDWARFCompUnit(*dwarf_cu);
+    DWARFCompileUnit *dwarf_cu = llvm::dyn_cast<DWARFCompileUnit>(die.GetCU());
+    if (!dwarf_cu)
+      continue;
+    sc.comp_unit = GetCompUnitForDWARFCompUnit(*dwarf_cu);
 
-        ParseVariables(sc, die, LLDB_INVALID_ADDRESS, false, false, &variables);
+    ParseVariables(sc, die, LLDB_INVALID_ADDRESS, false, false, &variables);
 
-        if (variables.GetSize() - original_size >= max_matches)
-          break;
-      } else
-        m_index->ReportInvalidDIERef(die_ref, regex.GetText());
-    }
+    if (variables.GetSize() - original_size >= max_matches)
+      break;
   }
 }
 
@@ -2265,7 +2272,6 @@ void SymbolFileDWARF::FindFunctions(ConstString name,
   const uint32_t original_size = sc_list.GetSize();
 
   llvm::DenseSet<const DWARFDebugInfoEntry *> resolved_dies;
-  DIEArray offsets;
 
   std::vector<DWARFDIE> dies;
   m_index->GetFunctions(name, *this, parent_decl_ctx, name_type_mask, dies);
@@ -2379,21 +2385,23 @@ void SymbolFileDWARF::FindTypes(
   for (size_t i = 0; i < num_die_matches; ++i) {
     const DIERef &die_ref = die_offsets[i];
     DWARFDIE die = GetDIE(die_ref);
-    if (die) {
-      if (!DIEInDeclContext(parent_decl_ctx, die))
-        continue; // The containing decl contexts don't match
-
-      Type *matching_type = ResolveType(die, true, true);
-      if (matching_type) {
-        // We found a type pointer, now find the shared pointer form our type
-        // list
-        types.InsertUnique(matching_type->shared_from_this());
-        if (types.GetSize() >= max_matches)
-          break;
-      }
-    } else {
+    if (!die) {
       m_index->ReportInvalidDIERef(die_ref, name.GetStringRef());
+      continue;
     }
+
+    if (!DIEInDeclContext(parent_decl_ctx, die))
+      continue; // The containing decl contexts don't match
+
+    Type *matching_type = ResolveType(die, true, true);
+    if (!matching_type)
+      continue;
+
+    // We found a type pointer, now find the shared pointer form our type
+    // list
+    types.InsertUnique(matching_type->shared_from_this());
+    if (types.GetSize() >= max_matches)
+      break;
   }
 
   // Next search through the reachable Clang modules. This only applies for
@@ -2451,11 +2459,11 @@ void SymbolFileDWARF::FindTypes(
   for (size_t i = 0; i < num_die_matches; ++i) {
     const DIERef &die_ref = die_offsets[i];
     DWARFDIE die = GetDIE(die_ref);
-
     if (!die) {
       m_index->ReportInvalidDIERef(die_ref, name.GetStringRef());
       continue;
     }
+
     if (!languages[GetLanguage(*die.GetCU())])
       continue;
 
@@ -2502,24 +2510,24 @@ SymbolFileDWARF::FindNamespace(ConstString name,
   DIEArray die_offsets;
   m_index->GetNamespaces(name, die_offsets);
   const size_t num_matches = die_offsets.size();
-  if (num_matches) {
-    for (size_t i = 0; i < num_matches; ++i) {
-      const DIERef &die_ref = die_offsets[i];
-      DWARFDIE die = GetDIE(die_ref);
+  for (size_t i = 0; i < num_matches; ++i) {
+    const DIERef &die_ref = die_offsets[i];
+    DWARFDIE die = GetDIE(die_ref);
+    if (!die) {
+      m_index->ReportInvalidDIERef(die_ref, name.GetStringRef());
+      continue;
+    }
 
-      if (die) {
-        if (!DIEInDeclContext(parent_decl_ctx, die))
-          continue; // The containing decl contexts don't match
+    if (!DIEInDeclContext(parent_decl_ctx, die))
+      continue; // The containing decl contexts don't match
 
-        if (DWARFASTParser *dwarf_ast = GetDWARFParser(*die.GetCU())) {
-          namespace_decl_ctx = dwarf_ast->GetDeclContextForUIDFromDWARF(die);
-          if (namespace_decl_ctx)
-            break;
-        }
-      } else {
-        m_index->ReportInvalidDIERef(die_ref, name.GetStringRef());
-      }
-    }
+    DWARFASTParser *dwarf_ast = GetDWARFParser(*die.GetCU());
+    if (!dwarf_ast)
+      continue;
+
+    namespace_decl_ctx = dwarf_ast->GetDeclContextForUIDFromDWARF(die);
+    if (namespace_decl_ctx)
+      break;
   }
   if (log && namespace_decl_ctx) {
     GetObjectFile()->GetModule()->LogMessage(
@@ -3093,19 +3101,18 @@ size_t SymbolFileDWARF::ParseVariablesForContext(const SymbolContext &sc) {
         m_index->GetGlobalVariables(dwarf_cu->GetNonSkeletonUnit(),
                                     die_offsets);
         const size_t num_matches = die_offsets.size();
-        if (num_matches) {
-          for (size_t i = 0; i < num_matches; ++i) {
-            const DIERef &die_ref = die_offsets[i];
-            DWARFDIE die = GetDIE(die_ref);
-            if (die) {
-              VariableSP var_sp(
-                  ParseVariableDIE(sc, die, LLDB_INVALID_ADDRESS));
-              if (var_sp) {
-                variables->AddVariableIfUnique(var_sp);
-                ++vars_added;
-              }
-            } else
-              m_index->ReportInvalidDIERef(die_ref, "");
+        for (size_t i = 0; i < num_matches; ++i) {
+          const DIERef &die_ref = die_offsets[i];
+          DWARFDIE die = GetDIE(die_ref);
+          if (!die) {
+            m_index->ReportInvalidDIERef(die_ref, "");
+            continue;
+          }
+
+          VariableSP var_sp(ParseVariableDIE(sc, die, LLDB_INVALID_ADDRESS));
+          if (var_sp) {
+            variables->AddVariableIfUnique(var_sp);
+            ++vars_added;
           }
         }
       }
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.cpp
index da8fec46dba74..9bf3206f1c619 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.cpp
@@ -38,7 +38,7 @@ SymbolFileDWARFDwo::SymbolFileDWARFDwo(SymbolFileDWARF &base_symbol_file,
 DWARFCompileUnit *SymbolFileDWARFDwo::GetDWOCompileUnitForHash(uint64_t hash) {
   if (const llvm::DWARFUnitIndex &index = m_context.GetAsLLVM().getCUIndex()) {
     if (const llvm::DWARFUnitIndex::Entry *entry = index.getFromHash(hash)) {
-      if (auto *unit_contrib = entry->getOffset())
+      if (auto *unit_contrib = entry->getContribution())
         return llvm::dyn_cast_or_null<DWARFCompileUnit>(
             DebugInfo().GetUnitAtOffset(DIERef::Section::DebugInfo,
                                         unit_contrib->Offset));
diff --git a/lldb/source/Plugins/SymbolVendor/MacOSX/SymbolVendorMacOSX.cpp b/lldb/source/Plugins/SymbolVendor/MacOSX/SymbolVendorMacOSX.cpp
index 2b67fee706178..fc426d1c9c376 100644
--- a/lldb/source/Plugins/SymbolVendor/MacOSX/SymbolVendorMacOSX.cpp
+++ b/lldb/source/Plugins/SymbolVendor/MacOSX/SymbolVendorMacOSX.cpp
@@ -20,6 +20,7 @@
 #include "lldb/Symbol/LocateSymbolFile.h"
 #include "lldb/Symbol/ObjectFile.h"
 #include "lldb/Target/Target.h"
+#include "lldb/Utility/Reproducer.h"
 #include "lldb/Utility/StreamString.h"
 #include "lldb/Utility/Timer.h"
 
@@ -145,6 +146,11 @@ SymbolVendorMacOSX::CreateInstance(const lldb::ModuleSP &module_sp,
     }
 
     if (dsym_fspec) {
+      // Compute dSYM root.
+      std::string dsym_root = dsym_fspec.GetPath();
+      const size_t pos = dsym_root.find("/Contents/Resources/");
+      dsym_root = pos != std::string::npos ? dsym_root.substr(0, pos) : "";
+
       DataBufferSP dsym_file_data_sp;
       lldb::offset_t dsym_file_data_offset = 0;
       dsym_objfile_sp =
@@ -154,136 +160,132 @@ SymbolVendorMacOSX::CreateInstance(const lldb::ModuleSP &module_sp,
       if (UUIDsMatch(module_sp.get(), dsym_objfile_sp.get(), feedback_strm)) {
         // We need a XML parser if we hope to parse a plist...
         if (XMLDocument::XMLEnabled()) {
-          char dsym_path[PATH_MAX];
-          if (module_sp->GetSourceMappingList().IsEmpty() &&
-              dsym_fspec.GetPath(dsym_path, sizeof(dsym_path))) {
+          if (module_sp->GetSourceMappingList().IsEmpty()) {
             lldb_private::UUID dsym_uuid = dsym_objfile_sp->GetUUID();
             if (dsym_uuid) {
               std::string uuid_str = dsym_uuid.GetAsString();
-              if (!uuid_str.empty()) {
-                char *resources = strstr(dsym_path, "/Contents/Resources/");
-                if (resources) {
-                  char dsym_uuid_plist_path[PATH_MAX];
-                  resources[strlen("/Contents/Resources/")] = '\0';
-                  snprintf(dsym_uuid_plist_path, sizeof(dsym_uuid_plist_path),
-                           "%s%s.plist", dsym_path, uuid_str.c_str());
-                  FileSpec dsym_uuid_plist_spec(dsym_uuid_plist_path);
-                  if (FileSystem::Instance().Exists(dsym_uuid_plist_spec)) {
-                    ApplePropertyList plist(dsym_uuid_plist_path);
-                    if (plist) {
-                      std::string DBGBuildSourcePath;
-                      std::string DBGSourcePath;
-
-                      // DBGSourcePathRemapping is a dictionary in the plist
-                      // with keys which are DBGBuildSourcePath file paths and
-                      // values which are DBGSourcePath file paths
-
-                      StructuredData::ObjectSP plist_sp =
-                          plist.GetStructuredData();
-                      if (plist_sp.get() && plist_sp->GetAsDictionary() &&
-                          plist_sp->GetAsDictionary()->HasKey(
-                              "DBGSourcePathRemapping") &&
-                          plist_sp->GetAsDictionary()
-                              ->GetValueForKey("DBGSourcePathRemapping")
-                              ->GetAsDictionary()) {
-
-                        // If DBGVersion 1 or DBGVersion missing, ignore DBGSourcePathRemapping.
-                        // If DBGVersion 2, strip last two components of path remappings from
-                        //                  entries to fix an issue with a specific set of
-                        //                  DBGSourcePathRemapping entries that lldb worked
-                        //                  with.
-                        // If DBGVersion 3, trust & use the source path remappings as-is.
-                        //
-
-                        bool new_style_source_remapping_dictionary = false;
-                        bool do_truncate_remapping_names = false;
-                        std::string original_DBGSourcePath_value =
-                            DBGSourcePath;
-                        if (plist_sp->GetAsDictionary()->HasKey("DBGVersion")) {
-                          std::string version_string =
-                              std::string(plist_sp->GetAsDictionary()
-                                              ->GetValueForKey("DBGVersion")
-                                              ->GetStringValue(""));
-                          if (!version_string.empty() &&
-                              isdigit(version_string[0])) {
-                            int version_number = atoi(version_string.c_str());
-                            if (version_number > 1) {
-                              new_style_source_remapping_dictionary = true;
-                            }
-                            if (version_number == 2) {
-                                do_truncate_remapping_names = true;
-                            }
+              if (!uuid_str.empty() && !dsym_root.empty()) {
+                char dsym_uuid_plist_path[PATH_MAX];
+                snprintf(dsym_uuid_plist_path, sizeof(dsym_uuid_plist_path),
+                         "%s/Contents/Resources/%s.plist", dsym_root.c_str(),
+                         uuid_str.c_str());
+                FileSpec dsym_uuid_plist_spec(dsym_uuid_plist_path);
+                if (FileSystem::Instance().Exists(dsym_uuid_plist_spec)) {
+                  ApplePropertyList plist(dsym_uuid_plist_path);
+                  if (plist) {
+                    std::string DBGBuildSourcePath;
+                    std::string DBGSourcePath;
+
+                    // DBGSourcePathRemapping is a dictionary in the plist
+                    // with keys which are DBGBuildSourcePath file paths and
+                    // values which are DBGSourcePath file paths
+
+                    StructuredData::ObjectSP plist_sp =
+                        plist.GetStructuredData();
+                    if (plist_sp.get() && plist_sp->GetAsDictionary() &&
+                        plist_sp->GetAsDictionary()->HasKey(
+                            "DBGSourcePathRemapping") &&
+                        plist_sp->GetAsDictionary()
+                            ->GetValueForKey("DBGSourcePathRemapping")
+                            ->GetAsDictionary()) {
+
+                      // If DBGVersion 1 or DBGVersion missing, ignore
+                      // DBGSourcePathRemapping. If DBGVersion 2, strip last two
+                      // components of path remappings from
+                      //                  entries to fix an issue with a
+                      //                  specific set of DBGSourcePathRemapping
+                      //                  entries that lldb worked with.
+                      // If DBGVersion 3, trust & use the source path remappings
+                      // as-is.
+                      //
+
+                      bool new_style_source_remapping_dictionary = false;
+                      bool do_truncate_remapping_names = false;
+                      std::string original_DBGSourcePath_value = DBGSourcePath;
+                      if (plist_sp->GetAsDictionary()->HasKey("DBGVersion")) {
+                        std::string version_string =
+                            std::string(plist_sp->GetAsDictionary()
+                                            ->GetValueForKey("DBGVersion")
+                                            ->GetStringValue(""));
+                        if (!version_string.empty() &&
+                            isdigit(version_string[0])) {
+                          int version_number = atoi(version_string.c_str());
+                          if (version_number > 1) {
+                            new_style_source_remapping_dictionary = true;
+                          }
+                          if (version_number == 2) {
+                            do_truncate_remapping_names = true;
                           }
                         }
+                      }
 
-                        StructuredData::Dictionary *remappings_dict =
-                            plist_sp->GetAsDictionary()
-                                ->GetValueForKey("DBGSourcePathRemapping")
-                                ->GetAsDictionary();
-                        remappings_dict->ForEach(
-                            [&module_sp, new_style_source_remapping_dictionary,
-                             original_DBGSourcePath_value, do_truncate_remapping_names](
-                                ConstString key,
-                                StructuredData::Object *object) -> bool {
-                              if (object && object->GetAsString()) {
-
-                                // key is DBGBuildSourcePath
-                                // object is DBGSourcePath
-                                std::string DBGSourcePath =
-                                    std::string(object->GetStringValue());
-                                if (!new_style_source_remapping_dictionary &&
-                                    !original_DBGSourcePath_value.empty()) {
-                                  DBGSourcePath = original_DBGSourcePath_value;
-                                }
-                                if (DBGSourcePath[0] == '~') {
-                                  FileSpec resolved_source_path(
-                                      DBGSourcePath.c_str());
-                                  FileSystem::Instance().Resolve(
-                                      resolved_source_path);
-                                  DBGSourcePath =
-                                      resolved_source_path.GetPath();
-                                }
+                      StructuredData::Dictionary *remappings_dict =
+                          plist_sp->GetAsDictionary()
+                              ->GetValueForKey("DBGSourcePathRemapping")
+                              ->GetAsDictionary();
+                      remappings_dict->ForEach(
+                          [&module_sp, new_style_source_remapping_dictionary,
+                           original_DBGSourcePath_value,
+                           do_truncate_remapping_names](
+                              ConstString key,
+                              StructuredData::Object *object) -> bool {
+                            if (object && object->GetAsString()) {
+
+                              // key is DBGBuildSourcePath
+                              // object is DBGSourcePath
+                              std::string DBGSourcePath =
+                                  std::string(object->GetStringValue());
+                              if (!new_style_source_remapping_dictionary &&
+                                  !original_DBGSourcePath_value.empty()) {
+                                DBGSourcePath = original_DBGSourcePath_value;
+                              }
+                              if (DBGSourcePath[0] == '~') {
+                                FileSpec resolved_source_path(
+                                    DBGSourcePath.c_str());
+                                FileSystem::Instance().Resolve(
+                                    resolved_source_path);
+                                DBGSourcePath = resolved_source_path.GetPath();
+                              }
+                              module_sp->GetSourceMappingList().Append(
+                                  key, ConstString(DBGSourcePath), true);
+                              // With version 2 of DBGSourcePathRemapping, we
+                              // can chop off the last two filename parts
+                              // from the source remapping and get a more
+                              // general source remapping that still works.
+                              // Add this as another option in addition to
+                              // the full source path remap.
+                              if (do_truncate_remapping_names) {
+                                FileSpec build_path(key.AsCString());
+                                FileSpec source_path(DBGSourcePath.c_str());
+                                build_path.RemoveLastPathComponent();
+                                build_path.RemoveLastPathComponent();
+                                source_path.RemoveLastPathComponent();
+                                source_path.RemoveLastPathComponent();
                                 module_sp->GetSourceMappingList().Append(
-                                    key, ConstString(DBGSourcePath), true);
-                                // With version 2 of DBGSourcePathRemapping, we
-                                // can chop off the last two filename parts
-                                // from the source remapping and get a more
-                                // general source remapping that still works.
-                                // Add this as another option in addition to
-                                // the full source path remap.
-                                if (do_truncate_remapping_names) {
-                                  FileSpec build_path(key.AsCString());
-                                  FileSpec source_path(DBGSourcePath.c_str());
-                                  build_path.RemoveLastPathComponent();
-                                  build_path.RemoveLastPathComponent();
-                                  source_path.RemoveLastPathComponent();
-                                  source_path.RemoveLastPathComponent();
-                                  module_sp->GetSourceMappingList().Append(
-                                      ConstString(build_path.GetPath().c_str()),
-                                      ConstString(source_path.GetPath().c_str()), true);
-                                }
+                                    ConstString(build_path.GetPath().c_str()),
+                                    ConstString(source_path.GetPath().c_str()),
+                                    true);
                               }
-                              return true;
-                            });
-                      }
+                            }
+                            return true;
+                          });
+                    }
 
-                      // If we have a DBGBuildSourcePath + DBGSourcePath pair,
-                      // append those to the source path remappings.
-
-                      plist.GetValueAsString("DBGBuildSourcePath",
-                                             DBGBuildSourcePath);
-                      plist.GetValueAsString("DBGSourcePath", DBGSourcePath);
-                      if (!DBGBuildSourcePath.empty() &&
-                          !DBGSourcePath.empty()) {
-                        if (DBGSourcePath[0] == '~') {
-                          FileSpec resolved_source_path(DBGSourcePath.c_str());
-                          FileSystem::Instance().Resolve(resolved_source_path);
-                          DBGSourcePath = resolved_source_path.GetPath();
-                        }
-                        module_sp->GetSourceMappingList().Append(
-                            ConstString(DBGBuildSourcePath),
-                            ConstString(DBGSourcePath), true);
+                    // If we have a DBGBuildSourcePath + DBGSourcePath pair,
+                    // append those to the source path remappings.
+
+                    plist.GetValueAsString("DBGBuildSourcePath",
+                                           DBGBuildSourcePath);
+                    plist.GetValueAsString("DBGSourcePath", DBGSourcePath);
+                    if (!DBGBuildSourcePath.empty() && !DBGSourcePath.empty()) {
+                      if (DBGSourcePath[0] == '~') {
+                        FileSpec resolved_source_path(DBGSourcePath.c_str());
+                        FileSystem::Instance().Resolve(resolved_source_path);
+                        DBGSourcePath = resolved_source_path.GetPath();
                       }
+                      module_sp->GetSourceMappingList().Append(
+                          ConstString(DBGBuildSourcePath),
+                          ConstString(DBGSourcePath), true);
                     }
                   }
                 }
@@ -293,6 +295,13 @@ SymbolVendorMacOSX::CreateInstance(const lldb::ModuleSP &module_sp,
         }
 
         symbol_vendor->AddSymbolFileRepresentation(dsym_objfile_sp);
+        if (!dsym_root.empty()) {
+          if (repro::Generator *g =
+                  repro::Reproducer::Instance().GetGenerator()) {
+            repro::FileProvider &fp = g->GetOrCreate<repro::FileProvider>();
+            fp.recordInterestingDirectory(dsym_root);
+          }
+        }
         return symbol_vendor;
       }
     }
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
index c278aef8949e7..9f425ae7c30b4 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
@@ -316,6 +316,10 @@ static ClangASTMap &GetASTMap() {
   return *g_map_ptr;
 }
 
+TypePayloadClang::TypePayloadClang(bool is_complete_objc_class) {
+  SetIsCompleteObjCClass(is_complete_objc_class);
+}
+
 char TypeSystemClang::ID;
 
 bool TypeSystemClang::IsOperator(llvm::StringRef name,
@@ -4661,6 +4665,8 @@ lldb::Encoding TypeSystemClang::GetEncoding(lldb::opaque_compiler_type_t type,
     case clang::BuiltinType::Kind::OCLReserveID:
     case clang::BuiltinType::Kind::OCLSampler:
     case clang::BuiltinType::Kind::OMPArraySection:
+    case clang::BuiltinType::Kind::OMPArrayShaping:
+    case clang::BuiltinType::Kind::OMPIterator:
     case clang::BuiltinType::Kind::Overload:
     case clang::BuiltinType::Kind::PseudoObject:
     case clang::BuiltinType::Kind::UnknownAny:
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
index 49b0bd70f2ae8..f355681f26790 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
@@ -33,6 +33,7 @@
 #include "lldb/Symbol/TypeSystem.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Utility/ConstString.h"
+#include "lldb/Utility/Flags.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/Logging.h"
 #include "lldb/lldb-enumerations.h"
@@ -42,7 +43,9 @@ class PDBASTParser;
 
 namespace clang {
 class FileManager;
-}
+class HeaderSearch;
+class ModuleMap;
+} // namespace clang
 
 namespace lldb_private {
 
@@ -50,6 +53,24 @@ class ClangASTMetadata;
 class ClangASTSource;
 class Declaration;
 
+/// The implementation of lldb::Type's m_payload field for TypeSystemClang.
+class TypePayloadClang {
+  /// Layout: bit 31 ... IsCompleteObjCClass.
+  Type::Payload m_payload = 0;
+public:
+  TypePayloadClang() = default;
+  explicit TypePayloadClang(bool is_complete_objc_class);
+  explicit TypePayloadClang(uint32_t opaque_payload) : m_payload(opaque_payload) {}
+  operator Type::Payload() { return m_payload; }
+
+  static constexpr unsigned ObjCClassBit = 1 << 31;
+  bool IsCompleteObjCClass() { return Flags(m_payload).Test(ObjCClassBit); }
+  void SetIsCompleteObjCClass(bool is_complete_objc_class) {
+    m_payload = is_complete_objc_class ? Flags(m_payload).Set(ObjCClassBit)
+                                       : Flags(m_payload).Clear(ObjCClassBit);
+  }
+};
+  
 /// A TypeSystem implementation based on Clang.
 ///
 /// This class uses a single clang::ASTContext as the backend for storing
diff --git a/lldb/source/Symbol/LocateSymbolFileMacOSX.cpp b/lldb/source/Symbol/LocateSymbolFileMacOSX.cpp
index 2af32584b9d12..251605085c588 100644
--- a/lldb/source/Symbol/LocateSymbolFileMacOSX.cpp
+++ b/lldb/source/Symbol/LocateSymbolFileMacOSX.cpp
@@ -253,48 +253,34 @@ int LocateMacOSXFilesUsingDebugSymbols(const ModuleSpec &module_spec,
 FileSpec Symbols::FindSymbolFileInBundle(const FileSpec &dsym_bundle_fspec,
                                          const lldb_private::UUID *uuid,
                                          const ArchSpec *arch) {
-  char path[PATH_MAX];
-  if (dsym_bundle_fspec.GetPath(path, sizeof(path)) == 0)
-    return {};
-
-  ::strncat(path, "/Contents/Resources/DWARF", sizeof(path) - strlen(path) - 1);
-
-  DIR *dirp = opendir(path);
-  if (!dirp)
-    return {};
-
-  // Make sure we close the directory before exiting this scope.
-  auto cleanup_dir = llvm::make_scope_exit([&]() { closedir(dirp); });
-
-  FileSpec dsym_fspec;
-  dsym_fspec.GetDirectory().SetCString(path);
-  struct dirent *dp;
-  while ((dp = readdir(dirp)) != NULL) {
-    // Only search directories
-    if (dp->d_type == DT_DIR || dp->d_type == DT_UNKNOWN) {
-      if (dp->d_namlen == 1 && dp->d_name[0] == '.')
-        continue;
-
-      if (dp->d_namlen == 2 && dp->d_name[0] == '.' && dp->d_name[1] == '.')
-        continue;
-    }
-
-    if (dp->d_type == DT_REG || dp->d_type == DT_UNKNOWN) {
-      dsym_fspec.GetFilename().SetCString(dp->d_name);
-      ModuleSpecList module_specs;
-      if (ObjectFile::GetModuleSpecifications(dsym_fspec, 0, 0, module_specs)) {
-        ModuleSpec spec;
-        for (size_t i = 0; i < module_specs.GetSize(); ++i) {
-          bool got_spec = module_specs.GetModuleSpecAtIndex(i, spec);
-          UNUSED_IF_ASSERT_DISABLED(got_spec);
-          assert(got_spec);
-          if ((uuid == NULL ||
-               (spec.GetUUIDPtr() && spec.GetUUID() == *uuid)) &&
-              (arch == NULL ||
-               (spec.GetArchitecturePtr() &&
-                spec.GetArchitecture().IsCompatibleMatch(*arch)))) {
-            return dsym_fspec;
-          }
+  std::string dsym_bundle_path = dsym_bundle_fspec.GetPath();
+  llvm::SmallString<128> buffer(dsym_bundle_path);
+  llvm::sys::path::append(buffer, "Contents", "Resources", "DWARF");
+
+  std::error_code EC;
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> vfs =
+      FileSystem::Instance().GetVirtualFileSystem();
+  llvm::vfs::recursive_directory_iterator Iter(*vfs, buffer.str(), EC);
+  llvm::vfs::recursive_directory_iterator End;
+  for (; Iter != End && !EC; Iter.increment(EC)) {
+    llvm::ErrorOr<llvm::vfs::Status> Status = vfs->status(Iter->path());
+    if (Status->isDirectory())
+      continue;
+
+    FileSpec dsym_fspec(Iter->path());
+    ModuleSpecList module_specs;
+    if (ObjectFile::GetModuleSpecifications(dsym_fspec, 0, 0, module_specs)) {
+      ModuleSpec spec;
+      for (size_t i = 0; i < module_specs.GetSize(); ++i) {
+        bool got_spec = module_specs.GetModuleSpecAtIndex(i, spec);
+        assert(got_spec); // The call has side-effects so can't be inlined.
+        UNUSED_IF_ASSERT_DISABLED(got_spec);
+        if ((uuid == nullptr ||
+             (spec.GetUUIDPtr() && spec.GetUUID() == *uuid)) &&
+            (arch == nullptr ||
+             (spec.GetArchitecturePtr() &&
+              spec.GetArchitecture().IsCompatibleMatch(*arch)))) {
+          return dsym_fspec;
         }
       }
     }
diff --git a/lldb/source/Symbol/Type.cpp b/lldb/source/Symbol/Type.cpp
index 09c0e16387937..804f7e02d3c93 100644
--- a/lldb/source/Symbol/Type.cpp
+++ b/lldb/source/Symbol/Type.cpp
@@ -143,15 +143,14 @@ Type::Type(lldb::user_id_t uid, SymbolFile *symbol_file, ConstString name,
            llvm::Optional<uint64_t> byte_size, SymbolContextScope *context,
            user_id_t encoding_uid, EncodingDataType encoding_uid_type,
            const Declaration &decl, const CompilerType &compiler_type,
-           ResolveState compiler_type_resolve_state)
+           ResolveState compiler_type_resolve_state, uint32_t opaque_payload)
     : std::enable_shared_from_this<Type>(), UserID(uid), m_name(name),
       m_symbol_file(symbol_file), m_context(context), m_encoding_type(nullptr),
       m_encoding_uid(encoding_uid), m_encoding_uid_type(encoding_uid_type),
       m_decl(decl), m_compiler_type(compiler_type),
-      m_compiler_type_resolve_state(
-          compiler_type ? compiler_type_resolve_state
-                        : ResolveState::Unresolved),
-      m_is_complete_objc_class(false) {
+      m_compiler_type_resolve_state(compiler_type ? compiler_type_resolve_state
+                                                  : ResolveState::Unresolved),
+      m_payload(opaque_payload) {
   if (byte_size) {
     m_byte_size = *byte_size;
     m_byte_size_has_value = true;
diff --git a/lldb/source/Symbol/Variable.cpp b/lldb/source/Symbol/Variable.cpp
index f1533ef6d4b3b..6c18ef15e1a2d 100644
--- a/lldb/source/Symbol/Variable.cpp
+++ b/lldb/source/Symbol/Variable.cpp
@@ -529,7 +529,7 @@ static void PrivateAutoCompleteMembers(
           i, member_name, nullptr, nullptr, nullptr);
 
       if (partial_member_name.empty() ||
-          member_name.find(partial_member_name) == 0) {
+          llvm::StringRef(member_name).startswith(partial_member_name)) {
         if (member_name == partial_member_name) {
           PrivateAutoComplete(
               frame, partial_path,
diff --git a/lldb/source/Target/Process.cpp b/lldb/source/Target/Process.cpp
index af990c945bec7..7dac2bedd42b0 100644
--- a/lldb/source/Target/Process.cpp
+++ b/lldb/source/Target/Process.cpp
@@ -4428,23 +4428,18 @@ class IOHandlerProcessSTDIO : public IOHandler {
 
 void Process::SetSTDIOFileDescriptor(int fd) {
   // First set up the Read Thread for reading/handling process I/O
-
-  std::unique_ptr<ConnectionFileDescriptor> conn_up(
-      new ConnectionFileDescriptor(fd, true));
-
-  if (conn_up) {
-    m_stdio_communication.SetConnection(conn_up.release());
-    if (m_stdio_communication.IsConnected()) {
-      m_stdio_communication.SetReadThreadBytesReceivedCallback(
-          STDIOReadThreadBytesReceived, this);
-      m_stdio_communication.StartReadThread();
-
-      // Now read thread is set up, set up input reader.
-
-      if (!m_process_input_reader)
-        m_process_input_reader =
-            std::make_shared<IOHandlerProcessSTDIO>(this, fd);
-    }
+  m_stdio_communication.SetConnection(
+      std::make_unique<ConnectionFileDescriptor>(fd, true));
+  if (m_stdio_communication.IsConnected()) {
+    m_stdio_communication.SetReadThreadBytesReceivedCallback(
+        STDIOReadThreadBytesReceived, this);
+    m_stdio_communication.StartReadThread();
+
+    // Now read thread is set up, set up input reader.
+
+    if (!m_process_input_reader)
+      m_process_input_reader =
+          std::make_shared<IOHandlerProcessSTDIO>(this, fd);
   }
 }
 
diff --git a/lldb/source/Utility/Reproducer.cpp b/lldb/source/Utility/Reproducer.cpp
index 8daa3216f7b1d..bbf399e599bfd 100644
--- a/lldb/source/Utility/Reproducer.cpp
+++ b/lldb/source/Utility/Reproducer.cpp
@@ -321,6 +321,11 @@ void WorkingDirectoryProvider::Keep() {
   os << m_cwd << "\n";
 }
 
+void FileProvider::recordInterestingDirectory(const llvm::Twine &dir) {
+  if (m_collector)
+    m_collector->addDirectory(dir);
+}
+
 void ProviderBase::anchor() {}
 char CommandProvider::ID = 0;
 char FileProvider::ID = 0;
diff --git a/lldb/source/Utility/UriParser.cpp b/lldb/source/Utility/UriParser.cpp
index a6172caa8c5cd..8169b0eee121a 100644
--- a/lldb/source/Utility/UriParser.cpp
+++ b/lldb/source/Utility/UriParser.cpp
@@ -42,7 +42,7 @@ bool UriParser::Parse(llvm::StringRef uri, llvm::StringRef &scheme,
   // Extract hostname
   if (!host_port.empty() && host_port[0] == '[') {
     // hostname is enclosed with square brackets.
-    pos = host_port.find(']');
+    pos = host_port.rfind(']');
     if (pos == std::string::npos)
       return false;
 
diff --git a/lldb/test/API/commands/expression/anonymous-struct/TestCallUserAnonTypedef.py b/lldb/test/API/commands/expression/anonymous-struct/TestCallUserAnonTypedef.py
index b6e0357528966..1a4e166100427 100644
--- a/lldb/test/API/commands/expression/anonymous-struct/TestCallUserAnonTypedef.py
+++ b/lldb/test/API/commands/expression/anonymous-struct/TestCallUserAnonTypedef.py
@@ -6,22 +6,15 @@
 Ticket: https://llvm.org/bugs/show_bug.cgi?id=26790
 """
 
-
 import lldb
 
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
 
-
 class TestExprLookupAnonStructTypedef(TestBase):
     mydir = TestBase.compute_mydir(__file__)
 
-    def setUp(self):
-        TestBase.setUp(self)
-        # Find the breakpoint
-        self.line = line_number('main.cpp', '// lldb testsuite break')
-
     @expectedFailureAll(
         oslist=['linux'],
         archs=['arm'],
@@ -29,16 +22,5 @@ def setUp(self):
     def test(self):
         """Test typedeffed untagged struct arguments for function call expressions"""
         self.build()
-
-        self.runCmd("file "+self.getBuildArtifact("a.out"),
-                    CURRENT_EXECUTABLE_SET)
-        lldbutil.run_break_set_by_file_and_line(
-            self,
-            "main.cpp",
-            self.line,
-            num_expected_locations=-1,
-            loc_exact=True
-        )
-
-        self.runCmd("run", RUN_SUCCEEDED)
+        lldbutil.run_to_source_breakpoint(self, "// break here", lldb.SBFileSpec("main.cpp"))
         self.expect_expr("multiply(&s)", result_type="double", result_value="1")
diff --git a/lldb/test/API/commands/expression/anonymous-struct/main.cpp b/lldb/test/API/commands/expression/anonymous-struct/main.cpp
index 5b170c5f943a4..d6366e7871523 100644
--- a/lldb/test/API/commands/expression/anonymous-struct/main.cpp
+++ b/lldb/test/API/commands/expression/anonymous-struct/main.cpp
@@ -1,26 +1,17 @@
-#include <tgmath.h>
-
 typedef struct {
-    float f;
-    int i;
+  float f;
+  int i;
 } my_untagged_struct;
 
-double multiply(my_untagged_struct *s)
-{
-    return s->f * s->i;
-}
+double multiply(my_untagged_struct *s) { return s->f * s->i; }
 
-double multiply(my_untagged_struct *s, int x)
-{
-    return multiply(s) * x;
-}
+double multiply(my_untagged_struct *s, int x) { return multiply(s) * x; }
 
-int main(int argc, char **argv)
-{
-    my_untagged_struct s = {
-        .f = (float)argc,
-        .i = argc,
-    };
-    // lldb testsuite break
-    return !(multiply(&s, argc) == pow(argc, 3));
+int main(int argc, char **argv) {
+  my_untagged_struct s = {
+      .f = (float)argc,
+      .i = argc,
+  };
+  // break here
+  return multiply(&s, argc) > 0;
 }
diff --git a/lldb/test/API/commands/expression/call-function/TestCallBuiltinFunction.py b/lldb/test/API/commands/expression/call-function/TestCallBuiltinFunction.py
index 31478884ad7df..55ba2717c013b 100644
--- a/lldb/test/API/commands/expression/call-function/TestCallBuiltinFunction.py
+++ b/lldb/test/API/commands/expression/call-function/TestCallBuiltinFunction.py
@@ -17,24 +17,10 @@ class ExprCommandCallBuiltinFunction(TestBase):
     # Builtins are expanded by Clang, so debug info shouldn't matter.
     NO_DEBUG_INFO_TESTCASE = True
 
-    def setUp(self):
-        TestBase.setUp(self)
-        # Find the line number to break for main.c.
-        self.line = line_number(
-            'main.cpp',
-            '// Please test these expressions while stopped at this line:')
-
     def test(self):
         self.build()
 
-        # Set breakpoint in main and run exe
-        self.runCmd("file " + self.getBuildArtifact("a.out"), CURRENT_EXECUTABLE_SET)
-        lldbutil.run_break_set_by_file_and_line(
-            self, "main.cpp", self.line, num_expected_locations=-1, loc_exact=True)
-
-        self.runCmd("run", RUN_SUCCEEDED)
-
-        # Test different builtin functions.
+        target = self.dbg.CreateTarget(self.getBuildArtifact("a.out"))
 
         self.expect_expr("__builtin_isinf(0.0f)", result_type="int", result_value="0")
         self.expect_expr("__builtin_isnormal(0.0f)", result_type="int", result_value="0")
diff --git a/lldb/test/API/commands/expression/call-function/TestCallStdStringFunction.py b/lldb/test/API/commands/expression/call-function/TestCallStdStringFunction.py
index 261e702fa59a2..f94bcae34cf93 100644
--- a/lldb/test/API/commands/expression/call-function/TestCallStdStringFunction.py
+++ b/lldb/test/API/commands/expression/call-function/TestCallStdStringFunction.py
@@ -2,26 +2,15 @@
 Test calling std::String member functions.
 """
 
-
-
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
 
-
 class ExprCommandCallFunctionTestCase(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    def setUp(self):
-        # Call super's setUp().
-        TestBase.setUp(self)
-        # Find the line number to break for main.c.
-        self.line = line_number(
-            'main.cpp',
-            '// Please test these expressions while stopped at this line:')
-
     @expectedFailureAll(
         compiler="icc",
         bugnumber="llvm.org/pr14437, fails with ICC 13.1")
@@ -29,15 +18,7 @@ def setUp(self):
     def test_with(self):
         """Test calling std::String member function."""
         self.build()
-        self.runCmd("file " + self.getBuildArtifact("a.out"),
-                    CURRENT_EXECUTABLE_SET)
-
-        # Some versions of GCC encode two locations for the 'return' statement
-        # in main.cpp
-        lldbutil.run_break_set_by_file_and_line(
-            self, "main.cpp", self.line, num_expected_locations=-1, loc_exact=True)
-
-        self.runCmd("run", RUN_SUCCEEDED)
+        lldbutil.run_to_source_breakpoint(self, "// break here", lldb.SBFileSpec("main.cpp"))
 
         self.expect("print str",
                     substrs=['Hello world'])
diff --git a/lldb/test/API/commands/expression/call-function/TestCallStopAndContinue.py b/lldb/test/API/commands/expression/call-function/TestCallStopAndContinue.py
index 0f0f1a54e31c1..1191176aa706b 100644
--- a/lldb/test/API/commands/expression/call-function/TestCallStopAndContinue.py
+++ b/lldb/test/API/commands/expression/call-function/TestCallStopAndContinue.py
@@ -2,13 +2,10 @@
 Test calling a function, stopping in the call, continue and gather the result on stop.
 """
 
-
-
 import lldb
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
 
-
 class ExprCommandCallStopContinueTestCase(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
@@ -17,27 +14,16 @@ def setUp(self):
         # Call super's setUp().
         TestBase.setUp(self)
         # Find the line number to break for main.c.
-        self.line = line_number(
-            'main.cpp',
-            '// Please test these expressions while stopped at this line:')
-        self.func_line = line_number('main.cpp', '{5, "five"}')
 
     def test(self):
         """Test gathering result from interrupted function call."""
         self.build()
-        self.runCmd("file " + self.getBuildArtifact("a.out"), CURRENT_EXECUTABLE_SET)
-
-        # Some versions of GCC encode two locations for the 'return' statement
-        # in main.cpp
-        lldbutil.run_break_set_by_file_and_line(
-            self, "main.cpp", self.line, num_expected_locations=-1, loc_exact=True)
-
-        self.runCmd("run", RUN_SUCCEEDED)
+        lldbutil.run_to_source_breakpoint(self, "// break here", lldb.SBFileSpec("main.cpp"))
 
         lldbutil.run_break_set_by_file_and_line(
             self,
             "main.cpp",
-            self.func_line,
+            line_number('main.cpp', '{5, "five"}'),
             num_expected_locations=-1,
             loc_exact=True)
 
diff --git a/lldb/test/API/commands/expression/call-function/TestCallUserDefinedFunction.py b/lldb/test/API/commands/expression/call-function/TestCallUserDefinedFunction.py
index 98cd0f24f36c4..edaa76174b47a 100644
--- a/lldb/test/API/commands/expression/call-function/TestCallUserDefinedFunction.py
+++ b/lldb/test/API/commands/expression/call-function/TestCallUserDefinedFunction.py
@@ -7,36 +7,19 @@
 
 """
 
-
-
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
 
-
 class ExprCommandCallUserDefinedFunction(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    def setUp(self):
-        # Call super's setUp().
-        TestBase.setUp(self)
-        # Find the line number to break for main.c.
-        self.line = line_number(
-            'main.cpp',
-            '// Please test these expressions while stopped at this line:')
-
     def test(self):
         """Test return values of user defined function calls."""
         self.build()
-
-        # Set breakpoint in main and run exe
-        self.runCmd("file " + self.getBuildArtifact("a.out"), CURRENT_EXECUTABLE_SET)
-        lldbutil.run_break_set_by_file_and_line(
-            self, "main.cpp", self.line, num_expected_locations=-1, loc_exact=True)
-
-        self.runCmd("run", RUN_SUCCEEDED)
+        lldbutil.run_to_source_breakpoint(self, "// break here", lldb.SBFileSpec("main.cpp"))
 
         # Test recursive function call.
         self.expect_expr("fib(5)", result_type="unsigned int", result_value="5")
diff --git a/lldb/test/API/commands/expression/call-function/main.cpp b/lldb/test/API/commands/expression/call-function/main.cpp
index cc5f52dbf5670..a383ce5c22a0e 100644
--- a/lldb/test/API/commands/expression/call-function/main.cpp
+++ b/lldb/test/API/commands/expression/call-function/main.cpp
@@ -1,53 +1,34 @@
-#include <iostream>
-#include <string>
 #include <cstring>
+#include <string>
 
-struct Five
-{
-    int number;
-    const char *name;
+struct Five {
+  int number;
+  const char *name;
 };
 
-Five
-returnsFive()
-{
-    Five my_five = {5, "five"};
-    return my_five;
+Five returnsFive() {
+  Five my_five = {5, "five"};
+  return my_five;
 }
 
-unsigned int
-fib(unsigned int n)
-{
-    if (n < 2)
-        return n;
-    else
-        return fib(n - 1) + fib(n - 2);
+unsigned int fib(unsigned int n) {
+  if (n < 2)
+    return n;
+  else
+    return fib(n - 1) + fib(n - 2);
 }
 
-int
-add(int a, int b)
-{
-    return a + b;
-}
+int add(int a, int b) { return a + b; }
 
-bool
-stringCompare(const char *str)
-{
-    if (strcmp( str, "Hello world" ) == 0)
-        return true;
-    else
-        return false;
+bool stringCompare(const char *str) {
+  if (strcmp(str, "Hello world") == 0)
+    return true;
+  else
+    return false;
 }
 
-int main (int argc, char const *argv[])
-{
-    std::string str = "Hello world";
-    std::cout << str << std::endl;
-    std::cout << str.c_str() << std::endl;
-    Five main_five = returnsFive();
-#if 0
-    print str
-    print str.c_str()
-#endif
-    return 0; // Please test these expressions while stopped at this line:
+int main(int argc, char const *argv[]) {
+  std::string str = "Hello world";
+  Five main_five = returnsFive();
+  return strlen(str.c_str()); // break here
 }
diff --git a/lldb/test/API/commands/expression/char/TestExprsChar.py b/lldb/test/API/commands/expression/char/TestExprsChar.py
index f1fa780538466..a1a4568aa92ea 100644
--- a/lldb/test/API/commands/expression/char/TestExprsChar.py
+++ b/lldb/test/API/commands/expression/char/TestExprsChar.py
@@ -1,44 +1,21 @@
-
-
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
 
-
 class ExprCharTestCase(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    def setUp(self):
-        # Call super's setUp().
-        TestBase.setUp(self)
-
-        self.main_source = "main.cpp"
-        self.main_source_spec = lldb.SBFileSpec(self.main_source)
-
     def do_test(self, dictionary=None):
         """These basic expression commands should work as expected."""
         self.build(dictionary=dictionary)
 
-        (target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(self,
-                                          '// Break here', self.main_source_spec)
-        frame = thread.GetFrameAtIndex(0)
-
-        value = frame.EvaluateExpression("foo(c)")
-        self.assertTrue(value.IsValid())
-        self.assertTrue(value.GetError().Success())
-        self.assertEqual(value.GetValueAsSigned(0), 1)
-
-        value = frame.EvaluateExpression("foo(sc)")
-        self.assertTrue(value.IsValid())
-        self.assertTrue(value.GetError().Success())
-        self.assertEqual(value.GetValueAsSigned(0), 2)
+        lldbutil.run_to_source_breakpoint(self, '// Break here', lldb.SBFileSpec("main.cpp"))
 
-        value = frame.EvaluateExpression("foo(uc)")
-        self.assertTrue(value.IsValid())
-        self.assertTrue(value.GetError().Success())
-        self.assertEqual(value.GetValueAsSigned(0), 3)
+        self.expect_expr("foo(c)", result_value="1")
+        self.expect_expr("foo(sc)", result_value="2")
+        self.expect_expr("foo(uc)", result_value="3")
 
     def test_default_char(self):
         self.do_test()
diff --git a/lldb/test/API/commands/expression/diagnostics/TestExprDiagnostics.py b/lldb/test/API/commands/expression/diagnostics/TestExprDiagnostics.py
index da29d7b2c1af5..b5eb552badb57 100644
--- a/lldb/test/API/commands/expression/diagnostics/TestExprDiagnostics.py
+++ b/lldb/test/API/commands/expression/diagnostics/TestExprDiagnostics.py
@@ -60,10 +60,10 @@ def test_source_and_caret_printing(self):
         self.assertIn("<user expression 3>:1:10", value.GetError().GetCString())
 
         # Multiline top-level expressions.
-        value = frame.EvaluateExpression("void x() {}\nvoid foo(unknown_type x) {}", top_level_opts)
+        value = frame.EvaluateExpression("void x() {}\nvoid foo;", top_level_opts)
         self.assertFalse(value.GetError().Success())
-        self.assertIn("\nvoid foo(unknown_type x) {}\n         ^\n", value.GetError().GetCString())
-        self.assertIn("<user expression 4>:2:10", value.GetError().GetCString())
+        self.assertIn("\nvoid foo;\n     ^", value.GetError().GetCString())
+        self.assertIn("<user expression 4>:2:6", value.GetError().GetCString())
 
         # Test that we render Clang's 'notes' correctly.
         value = frame.EvaluateExpression("struct SFoo{}; struct SFoo { int x; };", top_level_opts)
diff --git a/lldb/test/API/commands/expression/fixits/TestFixIts.py b/lldb/test/API/commands/expression/fixits/TestFixIts.py
index 7aca1c5aa8631..eb1dd97aa9a9f 100644
--- a/lldb/test/API/commands/expression/fixits/TestFixIts.py
+++ b/lldb/test/API/commands/expression/fixits/TestFixIts.py
@@ -2,8 +2,6 @@
 Test calling an expression with errors that a FixIt can fix.
 """
 
-
-
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
@@ -14,19 +12,6 @@ class ExprCommandWithFixits(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    def setUp(self):
-        # Call super's setUp().
-        TestBase.setUp(self)
-
-        self.main_source = "main.cpp"
-        self.main_source_spec = lldb.SBFileSpec(self.main_source)
-
-    @skipUnlessDarwin
-    def test_with_target(self):
-        """Test calling expressions with errors that can be fixed by the FixIts."""
-        self.build()
-        self.try_expressions()
-
     def test_with_dummy_target(self):
         """Test calling expressions in the dummy target with errors that can be fixed by the FixIts."""
 
@@ -38,14 +23,20 @@ def test_with_dummy_target(self):
         self.assertEqual(result, lldb.eReturnStatusSuccessFinishResult, "The expression was successful.")
         self.assertTrue("Fix-it applied" in ret_val.GetError(), "Found the applied FixIt.")
 
-    def try_expressions(self):
+    def test_with_target(self):
         """Test calling expressions with errors that can be fixed by the FixIts."""
+        self.build()
         (target, process, self.thread, bkpt) = lldbutil.run_to_source_breakpoint(self,
-                                        'Stop here to evaluate expressions', self.main_source_spec)
+                                        'Stop here to evaluate expressions',
+                                         lldb.SBFileSpec("main.cpp"))
 
         options = lldb.SBExpressionOptions()
         options.SetAutoApplyFixIts(True)
 
+        top_level_options = lldb.SBExpressionOptions()
+        top_level_options.SetAutoApplyFixIts(True)
+        top_level_options.SetTopLevel(True)
+
         frame = self.thread.GetFrameAtIndex(0)
 
         # Try with one error:
@@ -54,6 +45,15 @@ def try_expressions(self):
         self.assertTrue(value.GetError().Success())
         self.assertEquals(value.GetValueAsUnsigned(), 10)
 
+        # Try with one error in a top-level expression.
+        # The Fix-It changes "ptr.m" to "ptr->m".
+        expr = "struct X { int m; }; X x; X *ptr = &x; int m = ptr.m;"
+        value = frame.EvaluateExpression(expr, top_level_options)
+        # A successfully parsed top-level expression will yield an error
+        # that there is 'no value'. If a parsing error would have happened we
+        # would get a different error kind, so let's check the error kind here.
+        self.assertEquals(value.GetError().GetCString(), "error: No value")
+
         # Try with two errors:
         two_error_expression = "my_pointer.second->a"
         value = frame.EvaluateExpression(two_error_expression, options)
diff --git a/lldb/test/API/commands/expression/xvalue/TestXValuePrinting.py b/lldb/test/API/commands/expression/xvalue/TestXValuePrinting.py
index 3a394d781f0a3..f5122b84839a9 100644
--- a/lldb/test/API/commands/expression/xvalue/TestXValuePrinting.py
+++ b/lldb/test/API/commands/expression/xvalue/TestXValuePrinting.py
@@ -1,36 +1,15 @@
-
-
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
 
-
 class ExprXValuePrintingTestCase(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    def setUp(self):
-        # Call super's setUp().
-        TestBase.setUp(self)
-
-        self.main_source = "main.cpp"
-        self.main_source_spec = lldb.SBFileSpec(self.main_source)
-
-    def do_test(self, dictionary=None):
-        """Printing an xvalue should work."""
-        self.build(dictionary=dictionary)
-
-        (target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(self,
-                                          '// Break here', self.main_source_spec)
-        frame = thread.GetFrameAtIndex(0)
-
-        value = frame.EvaluateExpression("foo().data")
-        self.assertTrue(value.IsValid())
-        self.assertTrue(value.GetError().Success())
-        self.assertEqual(value.GetValueAsSigned(), 1234)
-
     @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr21765")
     def test(self):
-        self.do_test()
-
+        """Printing an xvalue should work."""
+        self.build()
+        lldbutil.run_to_source_breakpoint(self, '// Break here', lldb.SBFileSpec("main.cpp"))
+        self.expect_expr("foo().data", result_value="1234")
diff --git a/lldb/test/API/commands/log/invalid-args/TestInvalidArgsLog.py b/lldb/test/API/commands/log/invalid-args/TestInvalidArgsLog.py
index 4d3c5733598f2..44ddae028ba17 100644
--- a/lldb/test/API/commands/log/invalid-args/TestInvalidArgsLog.py
+++ b/lldb/test/API/commands/log/invalid-args/TestInvalidArgsLog.py
@@ -18,5 +18,5 @@ def test_disable_empty(self):
 
     @no_debug_info_test
     def test_timer_empty(self):
-        self.expect("log timer", error=True,
+        self.expect("log timers", error=True,
                     substrs=["error: Missing subcommand"])
diff --git a/lldb/test/API/commands/settings/TestSettings.py b/lldb/test/API/commands/settings/TestSettings.py
index c0cdc085f1297..29360856a735b 100644
--- a/lldb/test/API/commands/settings/TestSettings.py
+++ b/lldb/test/API/commands/settings/TestSettings.py
@@ -286,7 +286,6 @@ def do_test_run_args_and_env_vars(self, use_launchsimple):
                 "Environment variable 'MY_ENV_VAR' successfully passed."])
 
     @skipIfRemote  # it doesn't make sense to send host env to remote target
-    @skipIf(oslist=["windows"])
     def test_pass_host_env_vars(self):
         """Test that the host env vars are passed to the launched process."""
         self.build()
diff --git a/lldb/test/API/commands/watchpoints/watchpoint_count/Makefile b/lldb/test/API/commands/watchpoints/watchpoint_count/Makefile
new file mode 100644
index 0000000000000..10495940055b6
--- /dev/null
+++ b/lldb/test/API/commands/watchpoints/watchpoint_count/Makefile
@@ -0,0 +1,3 @@
+C_SOURCES := main.c
+
+include Makefile.rules
diff --git a/lldb/test/API/commands/watchpoints/watchpoint_count/TestWatchpointCount.py b/lldb/test/API/commands/watchpoints/watchpoint_count/TestWatchpointCount.py
new file mode 100644
index 0000000000000..9ad21522b4aa2
--- /dev/null
+++ b/lldb/test/API/commands/watchpoints/watchpoint_count/TestWatchpointCount.py
@@ -0,0 +1,44 @@
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+class TestWatchpointCount(TestBase):
+    mydir = TestBase.compute_mydir(__file__)
+    NO_DEBUG_INFO_TESTCASE = True
+
+    def setUp(self):
+        TestBase.setUp(self)
+
+    @skipIf(oslist=["linux"], archs=["aarch64"])
+    def test_watchpoint_count(self):
+        self.build()
+        (_, process, thread, _) = lldbutil.run_to_source_breakpoint(self, "patatino", lldb.SBFileSpec("main.c"))
+        frame = thread.GetFrameAtIndex(0)
+        first_var = frame.FindVariable("x1")
+        second_var = frame.FindVariable("x2")
+
+        error = lldb.SBError()
+        first_watch = first_var.Watch(True, False, True, error)
+        if not error.Success():
+            self.fail(
+                "Failed to make watchpoint for x1: %s" %
+                (error.GetCString()))
+
+        second_watch = second_var.Watch(True, False, True, error)
+        if not error.Success():
+            self.fail(
+                "Failed to make watchpoint for x2: %s" %
+                (error.GetCString()))
+        process.Continue()
+
+        stop_reason = thread.GetStopReason()
+        self.assertEqual(stop_reason, lldb.eStopReasonWatchpoint, "watchpoint for x1 not hit")
+        stop_reason_descr = thread.GetStopDescription(256)
+        self.assertEqual(stop_reason_descr, "watchpoint 1")
+
+        process.Continue()
+        stop_reason = thread.GetStopReason()
+        self.assertEqual(stop_reason, lldb.eStopReasonWatchpoint, "watchpoint for x2 not hit")
+        stop_reason_descr = thread.GetStopDescription(256)
+        self.assertEqual(stop_reason_descr, "watchpoint 2")
diff --git a/lldb/test/API/commands/watchpoints/watchpoint_count/main.c b/lldb/test/API/commands/watchpoints/watchpoint_count/main.c
new file mode 100644
index 0000000000000..fc9a370e41f31
--- /dev/null
+++ b/lldb/test/API/commands/watchpoints/watchpoint_count/main.c
@@ -0,0 +1,13 @@
+#include <stdint.h>
+#include <stdio.h>
+
+int main() {
+  uint8_t x1 = 0;
+  uint16_t x2 = 0;
+
+  printf("patatino\n");
+
+  x1 += 1;
+  x2 += 2;
+  return 0;
+}
diff --git a/lldb/test/API/functionalities/source-map/TestTargetSourceMap.py b/lldb/test/API/functionalities/source-map/TestTargetSourceMap.py
index ac03d80023e40..c9800e6f199e9 100644
--- a/lldb/test/API/functionalities/source-map/TestTargetSourceMap.py
+++ b/lldb/test/API/functionalities/source-map/TestTargetSourceMap.py
@@ -1,6 +1,7 @@
 import lldb
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test.decorators import *
+import os
 
 
 class TestTargetSourceMap(TestBase):
@@ -10,6 +11,21 @@ class TestTargetSourceMap(TestBase):
     @no_debug_info_test
     def test_source_map(self):
         """Test target.source-map' functionality."""
+
+        def assertBreakpointWithSourceMap(src_path):
+            # Set a breakpoint after we remap source and verify that it succeeds
+            bp = target.BreakpointCreateByLocation(src_path, 2)
+            self.assertEquals(bp.GetNumLocations(), 1,
+                            "make sure breakpoint was resolved with map")
+
+            # Now make sure that we can actually FIND the source file using this
+            # remapping:
+            retval = lldb.SBCommandReturnObject()
+            self.dbg.GetCommandInterpreter().HandleCommand("source list -f main.c -l 2", retval)
+            self.assertTrue(retval.Succeeded(), "source list didn't succeed.")
+            self.assertNotEqual(retval.GetOutput(), None, "We got no ouput from source list")
+            self.assertTrue("return" in retval.GetOutput(), "We didn't find the source file...")
+
         # Set the target soure map to map "./" to the current test directory
         src_dir = self.getSourceDir()
         src_path = os.path.join(src_dir, "main.c")
@@ -25,19 +41,68 @@ def test_source_map(self):
         bp = target.BreakpointCreateByLocation(src_path, 2)
         self.assertEquals(bp.GetNumLocations(), 0,
                         "make sure no breakpoints were resolved without map")
-        src_map_cmd = 'settings set target.source-map . "%s"' % (src_dir)
-        self.dbg.HandleCommand(src_map_cmd)
 
-        # Set a breakpoint after we remap source and verify that it succeeds
-        bp = target.BreakpointCreateByLocation(src_path, 2)
-        self.assertEquals(bp.GetNumLocations(), 1,
-                        "make sure breakpoint was resolved with map")
-
-        # Now make sure that we can actually FIND the source file using this
-        # remapping:
-        retval = lldb.SBCommandReturnObject()
-        self.dbg.GetCommandInterpreter().HandleCommand("source list -f main.c -l 2", retval)
-        self.assertTrue(retval.Succeeded(), "source list didn't succeed.")
-        self.assertNotEqual(retval.GetOutput(), None, "We got no ouput from source list")
-        self.assertTrue("return" in retval.GetOutput(), "We didn't find the source file...")
+        invalid_path = src_dir + "invalid_path"
+        invalid_path2 = src_dir + "invalid_path2"
+
+        # We make sure the error message contains all the invalid paths
+        self.expect(
+            'settings set target.source-map . "%s" . "%s" . "%s"' % (invalid_path, src_dir, invalid_path2),
+            substrs=[
+                'the replacement path doesn\'t exist: "%s"' % (invalid_path),
+                'the replacement path doesn\'t exist: "%s"' % (invalid_path2),
+            ],
+            error=True,
+        )
+        self.expect(
+            'settings show target.source-map',
+            substrs=['[0] "." -> "%s"' % (src_dir)],
+        )
+        assertBreakpointWithSourceMap(src_path)
+
+        # Index 0 is the valid mapping, and modifying it to an invalid one should have no effect
+        self.expect(
+            'settings replace target.source-map 0 . "%s"' % (invalid_path),
+            substrs=['error: the replacement path doesn\'t exist: "%s"' % (invalid_path)],
+            error=True,
+        )
+        self.expect(
+            'settings show target.source-map',
+            substrs=['[0] "." -> "%s"' % (src_dir)]
+        )
+        assertBreakpointWithSourceMap(src_path)
+
+        # Let's clear and add the mapping in with insert-after
+        self.runCmd('settings remove target.source-map 0')
+        self.expect(
+            'settings show target.source-map',
+            endstr="target.source-map (path-map) =\n",
+        )
         
+        # We add a valid but useless mapping so that we can use insert-after
+        another_valid_path = os.path.dirname(src_dir)
+        self.runCmd('settings set target.source-map . "%s"' % (another_valid_path))
+
+        self.expect(
+            'settings replace target.source-map 0 . "%s"' % (invalid_path),
+            substrs=['error: the replacement path doesn\'t exist: "%s"' % (invalid_path)],
+            error=True,
+        )
+        self.expect(
+            'settings show target.source-map',
+            substrs=['[0] "." -> "%s"' % (another_valid_path)]
+        )
+
+        # Let's clear and add the mapping in with append
+        self.expect('settings remove target.source-map 0')
+        self.expect(
+            'settings show target.source-map',
+            endstr="target.source-map (path-map) =\n",
+        )
+
+        self.expect(
+            'settings append target.source-map . "%s" . "%s"' % (invalid_path, src_dir),
+            substrs=['error: the replacement path doesn\'t exist: "%s"' % (invalid_path)],
+            error=True,
+        )
+        assertBreakpointWithSourceMap(src_path)
diff --git a/lldb/test/API/lang/cpp/bitfields/TestCppBitfields.py b/lldb/test/API/lang/cpp/bitfields/TestCppBitfields.py
index 1b362e6b04f93..7320ce2b816ec 100644
--- a/lldb/test/API/lang/cpp/bitfields/TestCppBitfields.py
+++ b/lldb/test/API/lang/cpp/bitfields/TestCppBitfields.py
@@ -103,3 +103,10 @@ def test_and_run_command(self):
                    '(uint64_t:1) k = 1',
                 ])
 
+        self.expect(
+            "frame variable --show-types derived",
+            VARIABLES_DISPLAYED_CORRECTLY,
+            substrs=[
+                '(uint32_t) b_a = 2',
+                '(uint32_t:1) d_a = 1',
+                ])
diff --git a/lldb/test/API/lang/cpp/bitfields/main.cpp b/lldb/test/API/lang/cpp/bitfields/main.cpp
index e43bf8c138e9b..986b7cb947ed0 100644
--- a/lldb/test/API/lang/cpp/bitfields/main.cpp
+++ b/lldb/test/API/lang/cpp/bitfields/main.cpp
@@ -60,6 +60,16 @@ int main(int argc, char const *argv[]) {
     } 
   } clang_example;
 
+  class B {
+  public:
+    uint32_t b_a;
+  };
+
+  class D : public B {
+  public:
+    uint32_t d_a : 1;
+  } derived;
+
   lba.a = 2;
 
   lbb.a = 1;
@@ -76,6 +86,8 @@ int main(int argc, char const *argv[]) {
   lbd.arr[2] = '\0';
   lbd.a = 5;
 
+  derived.b_a = 2;
+  derived.d_a = 1;
 
   return 0; // Set break point at this line.
 }
diff --git a/lldb/test/API/lang/cpp/covariant-return-types/TestCovariantReturnTypes.py b/lldb/test/API/lang/cpp/covariant-return-types/TestCovariantReturnTypes.py
index 86a6ddd6e47b6..6f2b3eafd2e9c 100644
--- a/lldb/test/API/lang/cpp/covariant-return-types/TestCovariantReturnTypes.py
+++ b/lldb/test/API/lang/cpp/covariant-return-types/TestCovariantReturnTypes.py
@@ -38,3 +38,5 @@ def test(self):
         self.expect_expr("derived.getOtherRef().value()", result_summary='"derived"')
         self.expect_expr("base_ptr_to_derived->getOtherRef().value()", result_summary='"derived"')
         self.expect_expr("base.getOtherRef().value()", result_summary='"base"')
+
+        self.expect_expr("referencing_derived.getOther()->get()->a", result_value='42')
diff --git a/lldb/test/API/lang/cpp/covariant-return-types/main.cpp b/lldb/test/API/lang/cpp/covariant-return-types/main.cpp
index 2a6fc682c39a4..ea05043d6cf9e 100644
--- a/lldb/test/API/lang/cpp/covariant-return-types/main.cpp
+++ b/lldb/test/API/lang/cpp/covariant-return-types/main.cpp
@@ -28,6 +28,23 @@ struct Derived : public Base {
   OtherDerived& getOtherRef() override { return other_derived; }
 };
 
+// A regression test for a class with at least two members containing a
+// covariant function, which is referenced through another covariant function.
+struct BaseWithMembers {
+  int a = 42;
+  int b = 47;
+  virtual BaseWithMembers *get() { return this; }
+};
+struct DerivedWithMembers: BaseWithMembers {
+  DerivedWithMembers *get() override { return this; }
+};
+struct ReferencingBase {
+  virtual BaseWithMembers *getOther() { return new BaseWithMembers(); }
+};
+struct ReferencingDerived: ReferencingBase {
+  DerivedWithMembers *getOther() { return new DerivedWithMembers(); }
+};
+
 int main() {
   Derived derived;
   Base base;
@@ -36,5 +53,7 @@ int main() {
   (void)base_ptr_to_derived->getRef();
   (void)base_ptr_to_derived->getOtherPtr();
   (void)base_ptr_to_derived->getOtherRef();
+
+  ReferencingDerived referencing_derived;
   return 0; // break here
 }
diff --git a/lldb/test/API/macosx/dyld-trie-symbols/Makefile b/lldb/test/API/macosx/dyld-trie-symbols/Makefile
new file mode 100644
index 0000000000000..230000eb72ca5
--- /dev/null
+++ b/lldb/test/API/macosx/dyld-trie-symbols/Makefile
@@ -0,0 +1,13 @@
+CXX_SOURCES := main.cpp
+EXE := a.out
+MAKE_DSYM := NO
+LD_EXTRAS = -dynamiclib -image_base 0x8000
+CFLAGS = $(CFLAGS_NO_DEBUG)
+
+include Makefile.rules
+
+all: a.out a.out-stripped
+
+a.out-stripped:
+	cp a.out a.out-stripped
+	strip a.out-stripped
diff --git a/lldb/test/API/macosx/dyld-trie-symbols/TestDyldTrieSymbols.py b/lldb/test/API/macosx/dyld-trie-symbols/TestDyldTrieSymbols.py
new file mode 100644
index 0000000000000..51f4fb1dc5ab7
--- /dev/null
+++ b/lldb/test/API/macosx/dyld-trie-symbols/TestDyldTrieSymbols.py
@@ -0,0 +1,87 @@
+"""
+Test that we read the exported symbols from the dyld trie
+"""
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+class DyldTrieSymbolsTestCase(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    NO_DEBUG_INFO_TESTCASE = True
+
+    @skipIfRemote
+    @skipUnlessDarwin
+    
+    def test_dyld_trie_symbols(self):
+        """Test that we make create symbol table entries from the dyld trie data structure."""
+        self.build()
+        unstripped_exe = self.getBuildArtifact("a.out")
+        stripped_exe = self.getBuildArtifact("a.out-stripped")
+
+        unstripped_target = self.dbg.CreateTarget(unstripped_exe)
+        self.assertTrue(unstripped_target.IsValid(), "Got a vaid stripped target.")
+
+        # Verify that the expected symbols are present in an unstripped
+        # binary, and that we didn't duplicate the entries in the symbol
+        # table.
+        unstripped_bazval_symbols = unstripped_target.FindSymbols("bazval")
+        self.assertEqual(unstripped_bazval_symbols.GetSize(), 1)
+        unstripped_patval_symbols = unstripped_target.FindSymbols("patval")
+        self.assertEqual(unstripped_patval_symbols.GetSize(), 1)
+        unstripped_Z3foo_symbols = unstripped_target.FindSymbols("_Z3foov")
+        self.assertEqual(unstripped_Z3foo_symbols.GetSize(), 1)
+        unstripped_foo_symbols = unstripped_target.FindSymbols("foo")
+        self.assertEqual(unstripped_foo_symbols.GetSize(), 1)
+
+        # make sure we can look up the mangled name, demangled base name,
+        # demangled name with argument.
+        unstripped_Z3pat_symbols = unstripped_target.FindSymbols("_Z3pati")
+        self.assertEqual(unstripped_Z3pat_symbols.GetSize(), 1)
+        unstripped_pat_symbols = unstripped_target.FindSymbols("pat")
+        self.assertEqual(unstripped_pat_symbols.GetSize(), 1)
+        unstripped_patint_symbols = unstripped_target.FindSymbols("pat(int)")
+        self.assertEqual(unstripped_patint_symbols.GetSize(), 1)
+
+        unstripped_bar_symbols = unstripped_target.FindSymbols("bar")
+        self.assertEqual(unstripped_bar_symbols.GetSize(), 1)
+
+
+
+        # Verify that we can retrieve all the symbols with external
+        # linkage after the binary has been stripped; they should not
+        # exist in the nlist records at this point and can only be
+        # retrieved from the dyld trie structure.
+
+        stripped_target = self.dbg.CreateTarget(stripped_exe)
+        self.assertTrue(stripped_target.IsValid(), "Got a vaid stripped target.")
+
+        # Check that we're able to still retrieve all the symbols after
+        # the binary has been stripped. Check that one we know will be
+        # removed is absent.
+        stripped_bazval_symbols = stripped_target.FindSymbols("bazval")
+        self.assertEqual(stripped_bazval_symbols.GetSize(), 1)
+        stripped_patval_symbols = stripped_target.FindSymbols("patval")
+        self.assertEqual(stripped_patval_symbols.GetSize(), 1)
+        stripped_Z3foo_symbols = stripped_target.FindSymbols("_Z3foov")
+        self.assertEqual(stripped_Z3foo_symbols.GetSize(), 1)
+        stripped_foo_symbols = stripped_target.FindSymbols("foo")
+        self.assertEqual(stripped_foo_symbols.GetSize(), 1)
+
+        # make sure we can look up the mangled name, demangled base name,
+        # demangled name with argument.
+        stripped_Z3pat_symbols = stripped_target.FindSymbols("_Z3pati")
+        self.assertEqual(stripped_Z3pat_symbols.GetSize(), 1)
+        stripped_pat_symbols = stripped_target.FindSymbols("pat")
+        self.assertEqual(stripped_pat_symbols.GetSize(), 1)
+        stripped_patint_symbols = stripped_target.FindSymbols("pat(int)")
+        self.assertEqual(stripped_patint_symbols.GetSize(), 1)
+
+        # bar should have been strippped.  We should not find it, or the
+        # stripping went wrong.
+        stripped_bar_symbols = stripped_target.FindSymbols("bar")
+        self.assertEqual(stripped_bar_symbols.GetSize(), 0)
+
diff --git a/lldb/test/API/macosx/dyld-trie-symbols/main.cpp b/lldb/test/API/macosx/dyld-trie-symbols/main.cpp
new file mode 100644
index 0000000000000..dc4ab5f6dcd65
--- /dev/null
+++ b/lldb/test/API/macosx/dyld-trie-symbols/main.cpp
@@ -0,0 +1,29 @@
+int patval;          // external symbol, will not be completely stripped
+int pat(int in) {    // external symbol, will not be completely stripped
+  if (patval == 0)
+    patval = in;
+  return patval;
+}
+
+static int fooval;  // static symbol, stripped
+int foo() {         // external symbol, will not be completely stripped
+  if (fooval == 0)
+    fooval = 5;
+  return fooval;
+}
+
+int bazval = 10;   // external symbol, will not be completely stripped
+int baz () {       // external symbol, will not be completely stripped
+  return foo() + bazval;
+}
+
+static int barval = 15; // static symbol, stripped
+static int bar () {     // static symbol, stripped; __lldb_unnamed_symbol from func starts
+  return baz() + barval;
+}
+
+int calculate ()   // external symbol, will not be completely stripped
+{
+  return bar();
+}
+
diff --git a/lldb/test/API/python_api/sbplatform/Makefile b/lldb/test/API/python_api/sbplatform/Makefile
new file mode 100644
index 0000000000000..99998b20bcb05
--- /dev/null
+++ b/lldb/test/API/python_api/sbplatform/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/python_api/sbplatform/TestSBPlatform.py b/lldb/test/API/python_api/sbplatform/TestSBPlatform.py
new file mode 100644
index 0000000000000..4735f6ea3b492
--- /dev/null
+++ b/lldb/test/API/python_api/sbplatform/TestSBPlatform.py
@@ -0,0 +1,22 @@
+"""Test the SBPlatform APIs."""
+
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+
+class SBPlatformAPICase(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+    NO_DEBUG_INFO_TESTCASE = True
+
+    @add_test_categories(['pyapi'])
+    def test_run(self):
+        self.build()
+        plat = lldb.SBPlatform.GetHostPlatform()
+
+        os.environ["MY_TEST_ENV_VAR"]="SBPlatformAPICase.test_run"
+        def cleanup():
+            del os.environ["MY_TEST_ENV_VAR"]
+        self.addTearDownHook(cleanup)
+        cmd = lldb.SBPlatformShellCommand(self.getBuildArtifact("a.out"))
+        self.assertTrue(plat.Run(cmd).Success())
+        self.assertIn("MY_TEST_ENV_VAR=SBPlatformAPICase.test_run", cmd.GetOutput())
diff --git a/lldb/test/API/python_api/sbplatform/main.cpp b/lldb/test/API/python_api/sbplatform/main.cpp
new file mode 100644
index 0000000000000..9f2aca26ab8df
--- /dev/null
+++ b/lldb/test/API/python_api/sbplatform/main.cpp
@@ -0,0 +1,8 @@
+#include <cstdlib>
+#include <cstdio>
+
+int main() {
+  printf("MY_TEST_ENV_VAR=%s\n", getenv("MY_TEST_ENV_VAR"));
+
+  return 0;
+}
diff --git a/lldb/test/API/tools/intel-features/intel-pt/test/Makefile b/lldb/test/API/tools/intel-features/intel-pt/test/Makefile
new file mode 100644
index 0000000000000..99998b20bcb05
--- /dev/null
+++ b/lldb/test/API/tools/intel-features/intel-pt/test/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/tools/intel-features/intel-pt/test/TestIntelPTSimpleBinary.py b/lldb/test/API/tools/intel-features/intel-pt/test/TestIntelPTSimpleBinary.py
new file mode 100644
index 0000000000000..ea7f6a469f8fc
--- /dev/null
+++ b/lldb/test/API/tools/intel-features/intel-pt/test/TestIntelPTSimpleBinary.py
@@ -0,0 +1,60 @@
+from __future__ import print_function
+
+import os
+import lldb
+import time
+
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class TestIntelPTSimpleBinary(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+    NO_DEBUG_INFO_TESTCASE = True
+
+    @skipIf(oslist=no_match(['linux']))
+    @skipIf(archs=no_match(['i386', 'x86_64']))
+    @skipIfRemote
+    def test_basic_flow(self):
+        """Test collection, decoding, and dumping instructions"""
+        if os.environ.get('TEST_INTEL_PT') != '1':
+            self.skipTest("The environment variable TEST_INTEL_PT=1 is needed to run this test.")
+
+        lldb_exec_dir = os.environ["LLDB_IMPLIB_DIR"]
+        lldb_lib_dir = os.path.join(lldb_exec_dir, os.pardir, "lib")
+        plugin_file = os.path.join(lldb_lib_dir, "liblldbIntelFeatures.so")
+
+        self.build()
+
+        self.runCmd("plugin load " + plugin_file)
+
+        exe = self.getBuildArtifact("a.out")
+        lldbutil.run_to_name_breakpoint(self, "main", exe_name=exe)
+        # We start tracing from main
+        self.runCmd("processor-trace start all")
+
+        # We check the trace after the for loop
+        self.runCmd("b " + str(line_number('main.cpp', '// Break 1')))
+        self.runCmd("c")
+
+        # We wait a little bit to ensure the processor has send the PT packets to
+        # the memory
+        time.sleep(.1)
+
+        # We find the start address of the 'fun' function for a later check
+        target = self.dbg.GetSelectedTarget()
+        fun_start_adddress = target.FindFunctions("fun")[0].GetSymbol() \
+            .GetStartAddress().GetLoadAddress(target)
+
+        # We print the last instructions
+        self.expect("processor-trace show-instr-log -c 100",
+            patterns=[
+                # We expect to have seen the first instruction of 'fun'
+                hex(fun_start_adddress),  
+                # We expect to see the exit condition of the for loop
+                "at main.cpp:" + str(line_number('main.cpp', '// Break for loop')) 
+            ])
+
+        self.runCmd("processor-trace stop")
diff --git a/lldb/test/API/tools/intel-features/intel-pt/test/main.cpp b/lldb/test/API/tools/intel-features/intel-pt/test/main.cpp
new file mode 100644
index 0000000000000..d2750fff56b58
--- /dev/null
+++ b/lldb/test/API/tools/intel-features/intel-pt/test/main.cpp
@@ -0,0 +1,10 @@
+int fun(int a) { return a * a + 1; }
+
+int main() {
+  int z = 0;
+  for (int i = 0; i < 10000; i++) { // Break for loop
+    z += fun(z);
+  }
+
+  return 0; // Break 1
+}
diff --git a/lldb/test/API/tools/lldb-vscode/breakpoint/TestVSCode_setBreakpoints.py b/lldb/test/API/tools/lldb-vscode/breakpoint/TestVSCode_setBreakpoints.py
index aff210d9f8acc..dde9989703fb2 100644
--- a/lldb/test/API/tools/lldb-vscode/breakpoint/TestVSCode_setBreakpoints.py
+++ b/lldb/test/API/tools/lldb-vscode/breakpoint/TestVSCode_setBreakpoints.py
@@ -36,7 +36,7 @@ def test_set_and_clear(self):
         first_line = line_number('main.cpp', 'break 12')
         second_line = line_number('main.cpp', 'break 13')
         third_line = line_number('main.cpp', 'break 14')
-        lines = [first_line, second_line, third_line]
+        lines = [first_line, third_line, second_line]
 
         # Visual Studio Code Debug Adaptors have no way to specify the file
         # without launching or attaching to a process, so we must start a
@@ -51,8 +51,9 @@ def test_set_and_clear(self):
             breakpoints = response['body']['breakpoints']
             self.assertEquals(len(breakpoints), len(lines),
                             "expect %u source breakpoints" % (len(lines)))
-            for breakpoint in breakpoints:
+            for (breakpoint, index) in zip(breakpoints, range(len(lines))):
                 line = breakpoint['line']
+                self.assertTrue(line, lines[index])
                 # Store the "id" of the breakpoint that was set for later
                 line_to_id[line] = breakpoint['id']
                 self.assertTrue(line in lines, "line expected in lines array")
@@ -74,8 +75,9 @@ def test_set_and_clear(self):
             breakpoints = response['body']['breakpoints']
             self.assertEquals(len(breakpoints), len(lines),
                             "expect %u source breakpoints" % (len(lines)))
-            for breakpoint in breakpoints:
+            for (breakpoint, index) in zip(breakpoints, range(len(lines))):
                 line = breakpoint['line']
+                self.assertTrue(line, lines[index])
                 # Verify the same breakpoints are still set within LLDB by
                 # making sure the breakpoint ID didn't change
                 self.assertEquals(line_to_id[line], breakpoint['id'],
diff --git a/lldb/test/API/tools/lldb-vscode/completions/TestVSCode_completions.py b/lldb/test/API/tools/lldb-vscode/completions/TestVSCode_completions.py
index 155d476bb58aa..06ab8f4145491 100644
--- a/lldb/test/API/tools/lldb-vscode/completions/TestVSCode_completions.py
+++ b/lldb/test/API/tools/lldb-vscode/completions/TestVSCode_completions.py
@@ -17,10 +17,10 @@ class TestVSCode_variables(lldbvscode_testcase.VSCodeTestCaseBase):
 
     def verify_completions(self, actual_list, expected_list, not_expected_list=[]):
         for expected_item in expected_list:
-            self.assertTrue(expected_item in actual_list)
+            self.assertIn(expected_item, actual_list)
 
         for not_expected_item in not_expected_list:
-            self.assertFalse(not_expected_item in actual_list)
+            self.assertNotIn(not_expected_item, actual_list)
 
     @skipIfWindows
     @skipIfDarwin # Skip this test for now until we can figure out why tings aren't working on build bots
@@ -44,7 +44,7 @@ def test_completions(self):
             [
                 {
                     "text": "var",
-                    "label": "var -- vector<basic_string<char, char_traits<char>, allocator<char> >, allocator<basic_string<char, char_traits<char>, allocator<char> > > > &",
+                    "label": "var -- vector<basic_string<char, char_traits<char>, allocator<char>>, allocator<basic_string<char, char_traits<char>, allocator<char>>>> &",
                 }
             ],
             [{"text": "var1", "label": "var1 -- int &"}],
diff --git a/lldb/test/API/tools/lldb-vscode/console/Makefile b/lldb/test/API/tools/lldb-vscode/console/Makefile
new file mode 100644
index 0000000000000..99998b20bcb05
--- /dev/null
+++ b/lldb/test/API/tools/lldb-vscode/console/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/tools/lldb-vscode/console/TestVSCode_console.py b/lldb/test/API/tools/lldb-vscode/console/TestVSCode_console.py
new file mode 100644
index 0000000000000..fd42b66ccac25
--- /dev/null
+++ b/lldb/test/API/tools/lldb-vscode/console/TestVSCode_console.py
@@ -0,0 +1,70 @@
+"""
+Test lldb-vscode setBreakpoints request
+"""
+
+from __future__ import print_function
+
+import unittest2
+import vscode
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+import lldbvscode_testcase
+
+
+class TestVSCode_console(lldbvscode_testcase.VSCodeTestCaseBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    def check_lldb_command(self, lldb_command, contains_string, assert_msg):
+        response = self.vscode.request_evaluate('`%s' % (lldb_command))
+        output = response['body']['result']
+        self.assertTrue(contains_string in output,
+                        ("""Verify %s by checking the command output:\n"""
+                         """'''\n%s'''\nfor the string: "%s" """ % (
+                         assert_msg, output, contains_string)))
+
+    @skipIfWindows
+    @skipIfRemote
+    def test_scopes_variables_setVariable_evaluate(self):
+        '''
+            Tests that the "scopes" request causes the currently selected
+            thread and frame to be updated. There are no DAP packets that tell
+            lldb-vscode which thread and frame are selected other than the
+            "scopes" request. lldb-vscode will now select the thread and frame
+            for the latest "scopes" request that it receives.
+
+            The LLDB command interpreter needs to have the right thread and
+            frame selected so that commands executed in the debug console act
+            on the right scope. This applies both to the expressions that are
+            evaluated and the lldb commands that start with the backtick
+            character.
+        '''
+        program = self.getBuildArtifact("a.out")
+        self.build_and_launch(program)
+        source = 'main.cpp'
+        breakpoint1_line = line_number(source, '// breakpoint 1')
+        lines = [breakpoint1_line]
+        # Set breakpoint in the thread function so we can step the threads
+        breakpoint_ids = self.set_source_breakpoints(source, lines)
+        self.assertTrue(len(breakpoint_ids) == len(lines),
+                        "expect correct number of breakpoints")
+        self.continue_to_breakpoints(breakpoint_ids)
+        # Cause a "scopes" to be sent for frame zero which should update the
+        # selected thread and frame to frame 0.
+        self.vscode.get_local_variables(frameIndex=0)
+        # Verify frame #0 is selected in the command interpreter by running
+        # the "frame select" command with no frame index which will print the
+        # currently selected frame.
+        self.check_lldb_command("frame select", "frame #0",
+                                "frame 0 is selected")
+
+        # Cause a "scopes" to be sent for frame one which should update the
+        # selected thread and frame to frame 1.
+        self.vscode.get_local_variables(frameIndex=1)
+        # Verify frame #1 is selected in the command interpreter by running
+        # the "frame select" command with no frame index which will print the
+        # currently selected frame.
+
+        self.check_lldb_command("frame select", "frame #1",
+                                "frame 1 is selected")
diff --git a/lldb/test/API/tools/lldb-vscode/console/main.cpp b/lldb/test/API/tools/lldb-vscode/console/main.cpp
new file mode 100644
index 0000000000000..f4f8c6695b060
--- /dev/null
+++ b/lldb/test/API/tools/lldb-vscode/console/main.cpp
@@ -0,0 +1,9 @@
+
+int multiply(int x, int y) {
+  return x * y; // breakpoint 1
+}
+
+int main(int argc, char const *argv[]) {
+  int result = multiply(argc, 20);
+  return result < 0;
+}
diff --git a/lldb/test/CMakeLists.txt b/lldb/test/CMakeLists.txt
index d5b9fab1cd145..e86471609275d 100644
--- a/lldb/test/CMakeLists.txt
+++ b/lldb/test/CMakeLists.txt
@@ -1,6 +1,16 @@
 # Test runner infrastructure for LLDB. This configures the LLDB test trees
 # for use by Lit, and delegates to LLVM's lit test handlers.
 
+if(LLDB_BUILT_STANDALONE)
+  # In order to run check-lldb-* we need the correct map_config directives in
+  # llvm-lit. Because this is a standalone build, LLVM doesn't know about LLDB,
+  # and the lldb mappings are missing. We build our own llvm-lit, and tell LLVM
+  # to use the llvm-lit in the lldb build directory.
+  if (EXISTS ${LLVM_MAIN_SRC_DIR}/utils/llvm-lit)
+    set(LLVM_EXTERNAL_LIT ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/llvm-lit)
+  endif()
+endif()
+
 # Configure the build directory.
 set(LLDB_TEST_BUILD_DIRECTORY "${PROJECT_BINARY_DIR}/lldb-test-build.noindex" CACHE PATH "The build root for building tests.")
 
@@ -180,3 +190,10 @@ add_custom_target(check-lldb)
 add_dependencies(check-lldb lldb-test-deps)
 set_target_properties(check-lldb PROPERTIES FOLDER "lldb misc")
 add_dependencies(check-lldb check-lldb-lit)
+
+if(LLDB_BUILT_STANDALONE)
+  # This has to happen *AFTER* add_lit_testsuite.
+  if (EXISTS ${LLVM_MAIN_SRC_DIR}/utils/llvm-lit)
+    add_subdirectory(${LLVM_MAIN_SRC_DIR}/utils/llvm-lit ${CMAKE_CURRENT_BINARY_DIR}/llvm-lit)
+  endif()
+endif()
diff --git a/lldb/test/Shell/Reproducer/TestDSYM.test b/lldb/test/Shell/Reproducer/TestDSYM.test
new file mode 100644
index 0000000000000..bddacda3242f9
--- /dev/null
+++ b/lldb/test/Shell/Reproducer/TestDSYM.test
@@ -0,0 +1,11 @@
+# REQUIRES: system-darwin
+# Ensure that the reproducers captures the whole dSYM bundle.
+
+# RUN: rm -rf %t.repro
+# RUN: %clang_host %S/Inputs/simple.c -g -o %t.out
+# RUN: touch %t.out.dSYM/foo.bar
+
+# RUN: %lldb -x -b --capture --capture-path %t.repro %t.out -o 'b main' -o 'run' -o 'reproducer generate'
+
+# RUN: %lldb -b -o 'reproducer dump -p files -f %t.repro' | FileCheck %s --check-prefix FILES
+# FILES: foo.bar
diff --git a/lldb/test/Shell/SymbolFile/Breakpad/Inputs/stack-cfi-arm.syms b/lldb/test/Shell/SymbolFile/Breakpad/Inputs/stack-cfi-arm.syms
new file mode 100644
index 0000000000000..7a4e7d04f99eb
--- /dev/null
+++ b/lldb/test/Shell/SymbolFile/Breakpad/Inputs/stack-cfi-arm.syms
@@ -0,0 +1,5 @@
+MODULE Linux arm E5894855C35DCCCCCCCCCCCCCCCCCCCC0 linux.out
+INFO CODE_ID E35C283BC327C28762DB788BF5A4078BE2351448
+FUNC 0 2 0 func0
+STACK CFI INIT 0 2 .cfa: sp 0 + .ra: lr
+STACK CFI 2 .cfa: sp 8 + .ra: .cfa -4 + ^ r7: .cfa -8 + ^
diff --git a/lldb/test/Shell/SymbolFile/Breakpad/Inputs/unwind-via-raSearch.syms b/lldb/test/Shell/SymbolFile/Breakpad/Inputs/unwind-via-raSearch.syms
index 329a280bde858..eb4d615da0446 100644
--- a/lldb/test/Shell/SymbolFile/Breakpad/Inputs/unwind-via-raSearch.syms
+++ b/lldb/test/Shell/SymbolFile/Breakpad/Inputs/unwind-via-raSearch.syms
@@ -1,15 +1,15 @@
 MODULE windows x86 897DD83EA8C8411897F3A925EE4BF7411 unwind-via-stack-win.pdb
 INFO CODE_ID 5D499B5C5000 unwind-via-stack-win.exe
-PUBLIC  0 0 dummy
-PUBLIC 10 0 call_many
-PUBLIC 80 0 main
-PUBLIC 90 0 many_pointer_args
-PUBLIC 100 0 complex_rasearch
-PUBLIC 110 0 esp_rasearch
-PUBLIC 120 0 nonzero_frame_size
-STACK WIN 4 10 6d 0 0 0 0 0 0 1 $T0 .raSearch = $eip $T0 ^ = $esp $T0 4 + =
-STACK WIN 4 80 8 0 0 0 0 0 0 1 $T0 .raSearch = $eip $T0 ^ = $esp $T0 4 + =
-STACK WIN 4 90 5 0 0 50 0 0 0 1 $T0 .raSearch = $eip $T0 ^ = $esp $T0 4 + =
-STACK WIN 4 100 4 0 0 0 0 0 0 1 $T0 .raSearch 80 + = $eip $T0 ^ = $esp $T0 4 + =
-STACK WIN 4 110 4 0 0 0 0 0 0 1 $T0 .raSearch = $eip $T0 ^ = $esp .raSearch 4 + =
-STACK WIN 4 120 4 0 0 0 4 8 0 1 $T0 .raSearch = $eip $T0 ^ = $esp $T0 4 + =
+PUBLIC 1000  0 dummy
+PUBLIC 1010 0 call_many
+PUBLIC 1080 0 main
+PUBLIC 1090 0 many_pointer_args
+PUBLIC 1100 0 complex_rasearch
+PUBLIC 1110 0 esp_rasearch
+PUBLIC 1120 0 nonzero_frame_size
+STACK WIN 4 1010 6d 0 0 0 0 0 0 1 $T0 .raSearch = $eip $T0 ^ = $esp $T0 4 + =
+STACK WIN 4 1080 8 0 0 0 0 0 0 1 $T0 .raSearch = $eip $T0 ^ = $esp $T0 4 + =
+STACK WIN 4 1090 5 0 0 50 0 0 0 1 $T0 .raSearch = $eip $T0 ^ = $esp $T0 4 + =
+STACK WIN 4 1100 4 0 0 0 0 0 0 1 $T0 .raSearch 80 + = $eip $T0 ^ = $esp $T0 4 + =
+STACK WIN 4 1110 4 0 0 0 0 0 0 1 $T0 .raSearch = $eip $T0 ^ = $esp .raSearch 4 + =
+STACK WIN 4 1120 4 0 0 0 4 8 0 1 $T0 .raSearch = $eip $T0 ^ = $esp $T0 4 + =
diff --git a/lldb/test/Shell/SymbolFile/Breakpad/Inputs/unwind-via-stack-win.syms b/lldb/test/Shell/SymbolFile/Breakpad/Inputs/unwind-via-stack-win.syms
index b9b8a67095f01..24b6a21c9232f 100644
--- a/lldb/test/Shell/SymbolFile/Breakpad/Inputs/unwind-via-stack-win.syms
+++ b/lldb/test/Shell/SymbolFile/Breakpad/Inputs/unwind-via-stack-win.syms
@@ -1,17 +1,17 @@
 MODULE windows x86 897DD83EA8C8411897F3A925EE4BF7411 unwind-via-stack-win.pdb
 INFO CODE_ID 5D499B5C5000 unwind-via-stack-win.exe
-PUBLIC  0 0 dummy
-PUBLIC 10 0 call_many
-PUBLIC 80 0 main
-PUBLIC 90 0 many_pointer_args
-PUBLIC 100 0 bogus_rule
-PUBLIC 110 0 bogus_cfa_rhs
-PUBLIC 120 0 bogus_esp_rhs
-PUBLIC 130 0 temporary_var
-STACK WIN 4 10 6d 0 0 0 0 0 0 1 $T0 $esp 80 + = $eip $T0 ^ = $esp $T0 4 + =
-STACK WIN 4 80 8 0 0 0 0 0 0 1 $T0 $esp = $eip $T0 ^ = $esp $T0 4 + =
-STACK WIN 4 90 5 0 0 50 0 0 0 1 $T0 $esp = $eip $T0 ^ = $esp $T0 4 + =
-STACK WIN 4 100 4 0 0 0 0 0 0 1 bogus
-STACK WIN 4 110 4 0 0 0 0 0 0 1 $T0 $bogus = $eip $T0 ^ = $esp $T0 4 + =
-STACK WIN 4 120 4 0 0 0 0 0 0 1 $T0 $esp = $eip $T0 ^ = $esp $bogus 4 + =
-STACK WIN 4 130 4 0 0 0 0 0 0 1 $T0 $esp = $bogus $T0 = $eip $bogus ^ = $esp $T0 4 + =
+PUBLIC 1000 0 dummy
+PUBLIC 1010 0 call_many
+PUBLIC 1080 0 main
+PUBLIC 1090 0 many_pointer_args
+PUBLIC 1100 0 bogus_rule
+PUBLIC 1110 0 bogus_cfa_rhs
+PUBLIC 1120 0 bogus_esp_rhs
+PUBLIC 1130 0 temporary_var
+STACK WIN 4 1010 6d 0 0 0 0 0 0 1 $T0 $esp 80 + = $eip $T0 ^ = $esp $T0 4 + =
+STACK WIN 4 1080 8 0 0 0 0 0 0 1 $T0 $esp = $eip $T0 ^ = $esp $T0 4 + =
+STACK WIN 4 1090 5 0 0 50 0 0 0 1 $T0 $esp = $eip $T0 ^ = $esp $T0 4 + =
+STACK WIN 4 1100 4 0 0 0 0 0 0 1 bogus
+STACK WIN 4 1110 4 0 0 0 0 0 0 1 $T0 $bogus = $eip $T0 ^ = $esp $T0 4 + =
+STACK WIN 4 1120 4 0 0 0 0 0 0 1 $T0 $esp = $eip $T0 ^ = $esp $bogus 4 + =
+STACK WIN 4 1130 4 0 0 0 0 0 0 1 $T0 $esp = $bogus $T0 = $eip $bogus ^ = $esp $T0 4 + =
diff --git a/lldb/test/Shell/SymbolFile/Breakpad/Inputs/unwind-via-stack-win.yaml b/lldb/test/Shell/SymbolFile/Breakpad/Inputs/unwind-via-stack-win.yaml
index e0140061d6ead..5b1e8031d9b9a 100644
--- a/lldb/test/Shell/SymbolFile/Breakpad/Inputs/unwind-via-stack-win.yaml
+++ b/lldb/test/Shell/SymbolFile/Breakpad/Inputs/unwind-via-stack-win.yaml
@@ -11,7 +11,7 @@ Streams:
           Content:         0000000079100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0085100B0094842777
   - Type:            ModuleList
     Modules:
-      - Base of Image:   0x00000000000B1000
+      - Base of Image:   0x00000000000B0000
         Size of Image:   0x00004000
         Module Name:     'unwind-via-stack-win.exe'
         CodeView Record: 525344533ED87D89C8A8184197F3A925EE4BF74101000000433A5C70726F6A656374735C746573745F6170705C436F6E736F6C654170706C69636174696F6E315C44656275675C436F6E736F6C654170706C69636174696F6E312E70646200
diff --git a/lldb/test/Shell/SymbolFile/Breakpad/lit.local.cfg b/lldb/test/Shell/SymbolFile/Breakpad/lit.local.cfg
new file mode 100644
index 0000000000000..db82cc231003a
--- /dev/null
+++ b/lldb/test/Shell/SymbolFile/Breakpad/lit.local.cfg
@@ -0,0 +1 @@
+config.suffixes = ['.test', '.yaml']
diff --git a/lldb/test/Shell/SymbolFile/Breakpad/stack-cfi-arm.yaml b/lldb/test/Shell/SymbolFile/Breakpad/stack-cfi-arm.yaml
new file mode 100644
index 0000000000000..f7f46b7e2bff1
--- /dev/null
+++ b/lldb/test/Shell/SymbolFile/Breakpad/stack-cfi-arm.yaml
@@ -0,0 +1,32 @@
+# REQUIRES: arm
+
+# RUN: yaml2obj %s >%t
+# RUN: %lldb -c %t -o "target symbols add %S/Inputs/stack-cfi-arm.syms" \
+# RUN:   -o "image show-unwind -n func0" -b | FileCheck %s
+
+# CHECK:      Symbol file UnwindPlan:
+# CHECK:      row[0]:    0: CFA=DW_OP_breg13 +0, DW_OP_consts +0, DW_OP_plus  => pc=DW_OP_breg14 +0 
+# CHECK-NEXT: row[1]:    2: CFA=DW_OP_breg13 +0, DW_OP_consts +8, DW_OP_plus  => r7=DW_OP_pick 0x00, DW_OP_consts -8, DW_OP_plus , DW_OP_deref  pc=DW_OP_pick 0x00, DW_OP_consts -4, DW_OP_plus , DW_OP_deref  
+# 
+
+--- !minidump
+Streams:         
+  - Type:            ThreadList
+    Threads:         
+      - Thread Id:       0x00003E81
+        Context:         DEAD
+        Stack:           
+          Start of Memory Range: 0x00007FFCEB34A000
+          Content:         DEAD
+  - Type:            ModuleList
+    Modules:         
+      - Base of Image:   0x0000000000400000
+        Size of Image:   0x00001000
+        Module Name:     '/tmp/stack-cfi-arm.out'
+        CodeView Record: 4C457042E35C283BC327C28762DB788BF5A4078BE2351448
+  - Type:            SystemInfo
+    Processor Arch:  ARM
+    Platform ID:     Linux
+    CPU:             
+      CPUID:           0
+...
diff --git a/lldb/test/Shell/SymbolFile/Breakpad/unwind-via-raSearch.test b/lldb/test/Shell/SymbolFile/Breakpad/unwind-via-raSearch.test
index ea259a9de7e47..02404a86fa94e 100644
--- a/lldb/test/Shell/SymbolFile/Breakpad/unwind-via-raSearch.test
+++ b/lldb/test/Shell/SymbolFile/Breakpad/unwind-via-raSearch.test
@@ -12,7 +12,7 @@ image show-unwind -n call_many
 # CHECK: This UnwindPlan originally sourced from breakpad STACK WIN
 # CHECK: This UnwindPlan is sourced from the compiler: yes.
 # CHECK: This UnwindPlan is valid at all instruction locations: no.
-# CHECK: Address range of this UnwindPlan: [unwind-via-stack-win.exe..module_image + 16-0x0000007d)
+# CHECK: Address range of this UnwindPlan: [unwind-via-stack-win.exe..module_image + 4112-0x0000107d)
 # CHECK: row[0]:    0: CFA=RaSearch@SP+0 => esp=DW_OP_pick 0x00, DW_OP_consts +4, DW_OP_plus  eip=DW_OP_pick 0x00, DW_OP_deref
 
 image show-unwind -n nonzero_frame_size
diff --git a/lldb/test/Shell/SymbolFile/Breakpad/unwind-via-stack-win-no-memory-info.yaml b/lldb/test/Shell/SymbolFile/Breakpad/unwind-via-stack-win-no-memory-info.yaml
index 5fb75533ccd3a..e57fcdf715338 100644
--- a/lldb/test/Shell/SymbolFile/Breakpad/unwind-via-stack-win-no-memory-info.yaml
+++ b/lldb/test/Shell/SymbolFile/Breakpad/unwind-via-stack-win-no-memory-info.yaml
@@ -1,13 +1,13 @@
 # RUN: yaml2obj --docnum=1 %s > %t.dmp
-# RUN: yaml2obj --docnum=2 %s > %t.exe
-# RUN: %lldb -c %t.dmp %t.exe \
+# RUN: yaml2obj --docnum=2 %s > %T/unwind-via-stack-win-no-memory-info.exe
+# RUN: %lldb -c %t.dmp %T/unwind-via-stack-win-no-memory-info.exe \
 # RUN:   -o "target symbols add %S/Inputs/unwind-via-raSearch.syms" \
 # RUN:   -o "thread backtrace" -b | FileCheck %s
 
 # CHECK-LABEL: thread backtrace
-# CHECK: frame #0: 0x000b1092 unwind-via-stack-win.exe`many_pointer_args
-# CHECK: frame #1: 0x000b1079 unwind-via-stack-win.exe`call_many + 105
-# CHECK: frame #2: 0x000b1085 unwind-via-stack-win.exe`main + 5
+# CHECK: frame #0: 0x000b1092 unwind-via-stack-win-no-memory-info.exe`many_pointer_args
+# CHECK: frame #1: 0x000b1079 unwind-via-stack-win-no-memory-info.exe`call_many + 105
+# CHECK: frame #2: 0x000b1085 unwind-via-stack-win-no-memory-info.exe`main + 5
 # CHECK: frame #3: 0x77278494 kernel32.dll
 # CHECK-NOT: frame
 
@@ -24,7 +24,7 @@ Streams:
           Content:         0000000079100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0085100B0094842777
   - Type:            ModuleList
     Modules:
-      - Base of Image:   0x00000000000B1000
+      - Base of Image:   0x00000000000B0000
         Size of Image:   0x00004000
         Module Name:     'unwind-via-stack-win.exe'
         CodeView Record: 525344533ED87D89C8A8184197F3A925EE4BF74101000000433A5C70726F6A656374735C746573745F6170705C436F6E736F6C654170706C69636174696F6E315C44656275675C436F6E736F6C654170706C69636174696F6E312E70646200
@@ -65,51 +65,9 @@ OptionalHeader:
   SizeOfStackCommit: 4096
   SizeOfHeapReserve: 1048576
   SizeOfHeapCommit: 4096
-  ExportTable:     
-    RelativeVirtualAddress: 8327
-    Size:            90
-  ImportTable:     
-    RelativeVirtualAddress: 0
-    Size:            0
-  ResourceTable:   
-    RelativeVirtualAddress: 0
-    Size:            0
-  ExceptionTable:  
-    RelativeVirtualAddress: 12288
-    Size:            12
-  CertificateTable: 
-    RelativeVirtualAddress: 0
-    Size:            0
-  BaseRelocationTable: 
-    RelativeVirtualAddress: 0
-    Size:            0
   Debug:           
     RelativeVirtualAddress: 8192
     Size:            28
-  Architecture:    
-    RelativeVirtualAddress: 0
-    Size:            0
-  GlobalPtr:       
-    RelativeVirtualAddress: 0
-    Size:            0
-  TlsTable:        
-    RelativeVirtualAddress: 0
-    Size:            0
-  LoadConfigTable: 
-    RelativeVirtualAddress: 0
-    Size:            0
-  BoundImport:     
-    RelativeVirtualAddress: 0
-    Size:            0
-  IAT:             
-    RelativeVirtualAddress: 0
-    Size:            0
-  DelayImportDescriptor: 
-    RelativeVirtualAddress: 0
-    Size:            0
-  ClrRuntimeHeader: 
-    RelativeVirtualAddress: 0
-    Size:            0
 header:          
   Machine:         IMAGE_FILE_MACHINE_AMD64
   Characteristics: [ IMAGE_FILE_EXECUTABLE_IMAGE, IMAGE_FILE_LARGE_ADDRESS_AWARE, IMAGE_FILE_DLL ]
@@ -117,17 +75,11 @@ sections:
   - Name:            .text
     Characteristics: [ IMAGE_SCN_CNT_CODE, IMAGE_SCN_MEM_EXECUTE, IMAGE_SCN_MEM_READ ]
     VirtualAddress:  4096
-    VirtualSize:     22
-    SectionData:     50894C24048B4C24040FAF4C2404890C248B042459C3
+    VirtualSize:     512
   - Name:            .rdata
     Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_READ ]
     VirtualAddress:  8192
     VirtualSize:     236
-    SectionData:     00000000A565B65C00000000020000006B0000001C2000001C060000525344533ED87D89C8A8184197F3A925EE4BF74101000000443A5C757073747265616D5C6275696C645C746F6F6C735C6C6C64625C6C69745C4D6F64756C65735C5045434F46465C4F75747075745C6578706F72742D646C6C66756E632E79616D6C2E746D702E70646200000000000000000000000000AF200000000000000200000001000000CB200000D3200000D72000006578706F72742D646C6C66756E632E79616D6C2E746D702E646C6C000000000000100000D92000000100446C6C46756E63000000000101010001020000
-  - Name:            .pdata
-    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_READ ]
-    VirtualAddress:  12288
-    VirtualSize:     12
-    SectionData:     '0010000016100000E4200000'
+    SectionData:     00000000A565B65C00000000020000006B0000001C2000001C060000525344533ED87D89C8A8184197F3A925EE4BF74101
 symbols:         []
 ...
diff --git a/lldb/test/Shell/SymbolFile/Breakpad/unwind-via-stack-win.test b/lldb/test/Shell/SymbolFile/Breakpad/unwind-via-stack-win.test
index e462d8e988395..522637bf952fc 100644
--- a/lldb/test/Shell/SymbolFile/Breakpad/unwind-via-stack-win.test
+++ b/lldb/test/Shell/SymbolFile/Breakpad/unwind-via-stack-win.test
@@ -14,7 +14,7 @@ image show-unwind -n call_many
 # CHECK: This UnwindPlan originally sourced from breakpad STACK WIN
 # CHECK: This UnwindPlan is sourced from the compiler: yes.
 # CHECK: This UnwindPlan is valid at all instruction locations: no.
-# CHECK: Address range of this UnwindPlan: [unwind-via-stack-win.exe..module_image + 16-0x0000007d)
+# CHECK: Address range of this UnwindPlan: [unwind-via-stack-win.exe..module_image + 4112-0x0000107d)
 # CHECK: row[0]:    0: CFA=DW_OP_breg7 +0, DW_OP_consts +80, DW_OP_plus  => esp=DW_OP_pick 0x00, DW_OP_consts +4, DW_OP_plus  eip=DW_OP_pick 0x00, DW_OP_deref
 
 # Then, some invalid rules.
@@ -42,7 +42,7 @@ image show-unwind -n temporary_var
 # CHECK: This UnwindPlan originally sourced from breakpad STACK WIN
 # CHECK: This UnwindPlan is sourced from the compiler: yes.
 # CHECK: This UnwindPlan is valid at all instruction locations: no.
-# CHECK: Address range of this UnwindPlan: [unwind-via-stack-win.exe..module_image + 304-0x00000134)
+# CHECK: Address range of this UnwindPlan: [unwind-via-stack-win.exe..module_image + 4400-0x00001134)
 # CHECK: row[0]:    0: CFA=DW_OP_breg7 +0 => esp=DW_OP_pick 0x00, DW_OP_consts +4, DW_OP_plus  eip=DW_OP_pick 0x00, DW_OP_deref
 
 # And finally, check that backtracing works as a whole by unwinding a simple
diff --git a/lldb/tools/debugserver/source/MacOSX/arm64/DNBArchImplARM64.cpp b/lldb/tools/debugserver/source/MacOSX/arm64/DNBArchImplARM64.cpp
index e5d4b05d987c1..3e7bda88e6af3 100644
--- a/lldb/tools/debugserver/source/MacOSX/arm64/DNBArchImplARM64.cpp
+++ b/lldb/tools/debugserver/source/MacOSX/arm64/DNBArchImplARM64.cpp
@@ -1067,31 +1067,34 @@ uint32_t DNBArchMachARM64::GetHardwareWatchpointHit(nub_addr_t &addr) {
                    "DNBArchMachARM64::GetHardwareWatchpointHit() addr = 0x%llx",
                    (uint64_t)addr);
 
-  // This is the watchpoint value to match against, i.e., word address.
-  nub_addr_t wp_val = addr & ~((nub_addr_t)3);
   if (kret == KERN_SUCCESS) {
     DBG &debug_state = m_state.dbg;
     uint32_t i, num = NumSupportedHardwareWatchpoints();
     for (i = 0; i < num; ++i) {
       nub_addr_t wp_addr = GetWatchAddress(debug_state, i);
-      DNBLogThreadedIf(LOG_WATCHPOINTS, "DNBArchMachARM64::"
-                                        "GetHardwareWatchpointHit() slot: %u "
-                                        "(addr = 0x%llx).",
-                       i, (uint64_t)wp_addr);
-      if (wp_val == wp_addr) {
-        uint32_t byte_mask = bits(debug_state.__wcr[i], 12, 5);
-
-        // Sanity check the byte_mask, first.
-        if (LowestBitSet(byte_mask) < 0)
-          continue;
-
-        // Check that the watchpoint is enabled.
-        if (!IsWatchpointEnabled(debug_state, i))
-          continue;
-
-        // Compute the starting address (from the point of view of the
-        // debugger).
-        addr = wp_addr + LowestBitSet(byte_mask);
+      uint32_t byte_mask = bits(debug_state.__wcr[i], 12, 5);
+
+      DNBLogThreadedIf(LOG_WATCHPOINTS, "DNBArchImplX86_64::"
+                       "GetHardwareWatchpointHit() slot: %u "
+                       "(addr = 0x%llx; byte_mask = 0x%x)",
+                       i, static_cast<uint64_t>(wp_addr),
+                       byte_mask);
+
+      if (!IsWatchpointEnabled(debug_state, i))
+        continue;
+
+      if (bits(wp_addr, 48, 3) != bits(addr, 48, 3))
+        continue;
+
+      // Sanity check the byte_mask
+      uint32_t lsb = LowestBitSet(byte_mask);
+      if (lsb < 0)
+        continue;
+
+      uint64_t byte_to_match = bits(addr, 2, 0);
+
+      if (byte_mask & (1 << byte_to_match)) {
+        addr = wp_addr + lsb;
         return i;
       }
     }
diff --git a/lldb/tools/intel-features/CMakeLists.txt b/lldb/tools/intel-features/CMakeLists.txt
index aff75d7db334e..efba2f74904f7 100644
--- a/lldb/tools/intel-features/CMakeLists.txt
+++ b/lldb/tools/intel-features/CMakeLists.txt
@@ -56,6 +56,7 @@ add_lldb_library(lldbIntelFeatures SHARED
 
   LINK_LIBS
     ${FEATURE_LIBS}
+    ${PYTHON_LIBRARY}
   )
 
 # Add link dependencies for python wrapper
diff --git a/lldb/tools/intel-features/intel-pt/Decoder.cpp b/lldb/tools/intel-features/intel-pt/Decoder.cpp
index 2ce5c985c5460..21642c75327a9 100644
--- a/lldb/tools/intel-features/intel-pt/Decoder.cpp
+++ b/lldb/tools/intel-features/intel-pt/Decoder.cpp
@@ -564,6 +564,61 @@ void Decoder::InitializePTInstDecoder(
   }
 }
 
+void Decoder::AppendErrorWithOffsetToInstructionList(
+    int errcode, uint64_t decoder_offset, Instructions &instruction_list,
+    lldb::SBError &sberror) {
+  sberror.SetErrorStringWithFormat(
+      "processor trace decoding library: \"%s\"  [decoder_offset] => "
+      "[0x%" PRIu64 "]",
+      pt_errstr(pt_errcode(errcode)), decoder_offset);
+  instruction_list.emplace_back(sberror.GetCString());
+}
+
+void Decoder::AppendErrorWithoutOffsetToInstructionList(
+    int errcode, Instructions &instruction_list, lldb::SBError &sberror) {
+  sberror.SetErrorStringWithFormat("processor trace decoding library: \"%s\"",
+                                   pt_errstr(pt_errcode(errcode)));
+  instruction_list.emplace_back(sberror.GetCString());
+}
+
+int Decoder::AppendErrorToInstructionList(int errcode, pt_insn_decoder *decoder,
+                                          Instructions &instruction_list,
+                                          lldb::SBError &sberror) {
+  uint64_t decoder_offset = 0;
+  int errcode_off = pt_insn_get_offset(decoder, &decoder_offset);
+  if (errcode_off < 0) {
+    AppendErrorWithoutOffsetToInstructionList(errcode, instruction_list,
+                                              sberror);
+    return errcode_off;
+  }
+  AppendErrorWithOffsetToInstructionList(errcode, decoder_offset,
+                                         instruction_list, sberror);
+  return 0;
+}
+
+int Decoder::HandlePTInstructionEvents(pt_insn_decoder *decoder, int errcode,
+                                       Instructions &instruction_list,
+                                       lldb::SBError &sberror) {
+  while (errcode & pts_event_pending) {
+    pt_event event;
+    errcode = pt_insn_event(decoder, &event, sizeof(event));
+    if (errcode < 0)
+      return errcode;
+
+    // The list of events are in
+    // https://github.com/intel/libipt/blob/master/doc/man/pt_qry_event.3.md
+    if (event.type == ptev_overflow) {
+      int append_errcode = AppendErrorToInstructionList(
+          errcode, decoder, instruction_list, sberror);
+      if (append_errcode < 0)
+        return append_errcode;
+    }
+    // Other events don't signal stream errors
+  }
+
+  return 0;
+}
+
 // Start actual decoding of raw trace
 void Decoder::DecodeTrace(struct pt_insn_decoder *decoder,
                           Instructions &instruction_list,
@@ -585,10 +640,8 @@ void Decoder::DecodeTrace(struct pt_insn_decoder *decoder,
 
       int errcode_off = pt_insn_get_offset(decoder, &decoder_offset);
       if (errcode_off < 0) {
-        sberror.SetErrorStringWithFormat(
-            "processor trace decoding library: \"%s\"",
-            pt_errstr(pt_errcode(errcode)));
-        instruction_list.emplace_back(sberror.GetCString());
+        AppendErrorWithoutOffsetToInstructionList(errcode, instruction_list,
+                                                  sberror);
         return;
       }
 
@@ -619,16 +672,22 @@ void Decoder::DecodeTrace(struct pt_insn_decoder *decoder,
           // progress further. Hence, returning in this situation.
           return;
         }
-        sberror.SetErrorStringWithFormat(
-            "processor trace decoding library: \"%s\"  [decoder_offset] => "
-            "[0x%" PRIu64 "]",
-            pt_errstr(pt_errcode(errcode)), new_decoder_offset);
-        instruction_list.emplace_back(sberror.GetCString());
+        AppendErrorWithOffsetToInstructionList(errcode, new_decoder_offset,
+                                               instruction_list, sberror);
         decoder_offset = new_decoder_offset;
       }
     }
 
     while (1) {
+      errcode = HandlePTInstructionEvents(decoder, errcode, instruction_list,
+                                          sberror);
+      if (errcode < 0) {
+        int append_errcode = AppendErrorToInstructionList(
+            errcode, decoder, instruction_list, sberror);
+        if (append_errcode < 0)
+          return;
+        break;
+      }
       errcode = pt_insn_next(decoder, &insn, sizeof(insn));
       if (errcode < 0) {
         if (insn.iclass == ptic_error)
diff --git a/lldb/tools/intel-features/intel-pt/Decoder.h b/lldb/tools/intel-features/intel-pt/Decoder.h
index 2ae89653df50a..ff3e6f34b6a83 100644
--- a/lldb/tools/intel-features/intel-pt/Decoder.h
+++ b/lldb/tools/intel-features/intel-pt/Decoder.h
@@ -245,6 +245,22 @@ class Decoder {
       lldb::SBError &sberror) const;
   void DecodeTrace(struct pt_insn_decoder *decoder,
                    Instructions &instruction_list, lldb::SBError &sberror);
+  int HandlePTInstructionEvents(pt_insn_decoder *decoder, int errcode,
+                                Instructions &instruction_list,
+                                lldb::SBError &sberror);
+
+  int AppendErrorToInstructionList(int errcode, pt_insn_decoder *decoder,
+                                   Instructions &instruction_list,
+                                   lldb::SBError &sberror);
+
+  void AppendErrorWithOffsetToInstructionList(int errcode,
+                                              uint64_t decoder_offset,
+                                              Instructions &instruction_list,
+                                              lldb::SBError &sberror);
+
+  void AppendErrorWithoutOffsetToInstructionList(int errcode,
+                                                 Instructions &instruction_list,
+                                                 lldb::SBError &sberror);
 
   // Function to diagnose and indicate errors during raw trace decoding
   void Diagnose(struct pt_insn_decoder *decoder, int errcode,
diff --git a/lldb/tools/intel-features/intel-pt/cli-wrapper-pt.cpp b/lldb/tools/intel-features/intel-pt/cli-wrapper-pt.cpp
index 8db1c0f82d667..5e409a269fa46 100644
--- a/lldb/tools/intel-features/intel-pt/cli-wrapper-pt.cpp
+++ b/lldb/tools/intel-features/intel-pt/cli-wrapper-pt.cpp
@@ -191,6 +191,7 @@ class ProcessorTraceStart : public lldb::SBCommandPluginInterface {
       result.SetStatus(lldb::eReturnStatusFailed);
       return false;
     }
+    result.SetStatus(lldb::eReturnStatusSuccessFinishResult);
     return true;
   }
 
@@ -290,6 +291,7 @@ class ProcessorTraceInfo : public lldb::SBCommandPluginInterface {
                  s.GetData());
       result.AppendMessage(res.GetOutput());
     }
+    result.SetStatus(lldb::eReturnStatusSuccessFinishResult);
     return true;
   }
 
@@ -428,6 +430,7 @@ class ProcessorTraceShowInstrLog : public lldb::SBCommandPluginInterface {
       }
       result.AppendMessage(res.GetOutput());
     }
+    result.SetStatus(lldb::eReturnStatusSuccessFinishResult);
     return true;
   }
 
@@ -480,6 +483,7 @@ class ProcessorTraceStop : public lldb::SBCommandPluginInterface {
       result.SetStatus(lldb::eReturnStatusFailed);
       return false;
     }
+    result.SetStatus(lldb::eReturnStatusSuccessFinishResult);
     return true;
   }
 
diff --git a/lldb/tools/lldb-server/lldb-platform.cpp b/lldb/tools/lldb-server/lldb-platform.cpp
index d045660ce7af8..33f918ffc2a15 100644
--- a/lldb/tools/lldb-server/lldb-platform.cpp
+++ b/lldb/tools/lldb-server/lldb-platform.cpp
@@ -343,7 +343,7 @@ int main_platform(int argc, char *argv[]) {
       // connections while a connection is active.
       acceptor_up.reset();
     }
-    platform.SetConnection(conn);
+    platform.SetConnection(std::unique_ptr<Connection>(conn));
 
     if (platform.IsConnected()) {
       if (inferior_arguments.GetArgumentCount() > 0) {
diff --git a/lldb/tools/lldb-vscode/VSCode.cpp b/lldb/tools/lldb-vscode/VSCode.cpp
index 8112f4c8d0d97..36bc8ec8ebfd1 100644
--- a/lldb/tools/lldb-vscode/VSCode.cpp
+++ b/lldb/tools/lldb-vscode/VSCode.cpp
@@ -28,8 +28,8 @@ namespace lldb_vscode {
 VSCode g_vsc;
 
 VSCode::VSCode()
-    : launch_info(nullptr), variables(), broadcaster("lldb-vscode"),
-      num_regs(0), num_locals(0), num_globals(0), log(),
+    : variables(), broadcaster("lldb-vscode"), num_regs(0), num_locals(0),
+      num_globals(0), log(),
       exception_breakpoints(
           {{"cpp_catch", "C++ Catch", lldb::eLanguageTypeC_plus_plus},
            {"cpp_throw", "C++ Throw", lldb::eLanguageTypeC_plus_plus},
diff --git a/lldb/tools/lldb-vscode/VSCode.h b/lldb/tools/lldb-vscode/VSCode.h
index f23b24d0114ea..5298d7554f4de 100644
--- a/lldb/tools/lldb-vscode/VSCode.h
+++ b/lldb/tools/lldb-vscode/VSCode.h
@@ -70,8 +70,6 @@ struct VSCode {
   OutputStream output;
   lldb::SBDebugger debugger;
   lldb::SBTarget target;
-  lldb::SBAttachInfo attach_info;
-  lldb::SBLaunchInfo launch_info;
   lldb::SBValueList variables;
   lldb::SBBroadcaster broadcaster;
   int64_t num_regs;
diff --git a/lldb/tools/lldb-vscode/lldb-vscode.cpp b/lldb/tools/lldb-vscode/lldb-vscode.cpp
index 8c68dd0e70554..6aff01bcbc204 100644
--- a/lldb/tools/lldb-vscode/lldb-vscode.cpp
+++ b/lldb/tools/lldb-vscode/lldb-vscode.cpp
@@ -508,13 +508,14 @@ void request_attach(const llvm::json::Object &request) {
   llvm::json::Object response;
   lldb::SBError error;
   FillResponse(request, response);
+  lldb::SBAttachInfo attach_info;
   auto arguments = request.getObject("arguments");
   const lldb::pid_t pid =
       GetUnsigned(arguments, "pid", LLDB_INVALID_PROCESS_ID);
   if (pid != LLDB_INVALID_PROCESS_ID)
-    g_vsc.attach_info.SetProcessID(pid);
+    attach_info.SetProcessID(pid);
   const auto wait_for = GetBoolean(arguments, "waitFor", false);
-  g_vsc.attach_info.SetWaitForLaunch(wait_for, false /*async*/);
+  attach_info.SetWaitForLaunch(wait_for, false /*async*/);
   g_vsc.init_commands = GetStrings(arguments, "initCommands");
   g_vsc.pre_run_commands = GetStrings(arguments, "preRunCommands");
   g_vsc.stop_commands = GetStrings(arguments, "stopCommands");
@@ -547,20 +548,19 @@ void request_attach(const llvm::json::Object &request) {
   g_vsc.RunPreRunCommands();
 
   if (pid == LLDB_INVALID_PROCESS_ID && wait_for) {
-    char attach_info[256];
-    auto attach_info_len =
-        snprintf(attach_info, sizeof(attach_info),
-                 "Waiting to attach to \"%s\"...",
-                 g_vsc.target.GetExecutable().GetFilename());
-    g_vsc.SendOutput(OutputType::Console, llvm::StringRef(attach_info,
-                                                          attach_info_len));
+    char attach_msg[256];
+    auto attach_msg_len = snprintf(attach_msg, sizeof(attach_msg),
+                                   "Waiting to attach to \"%s\"...",
+                                   g_vsc.target.GetExecutable().GetFilename());
+    g_vsc.SendOutput(OutputType::Console,
+                     llvm::StringRef(attach_msg, attach_msg_len));
   }
   if (attachCommands.empty()) {
     // No "attachCommands", just attach normally.
     // Disable async events so the attach will be successful when we return from
     // the launch call and the launch will happen synchronously
     g_vsc.debugger.SetAsync(false);
-    g_vsc.target.Attach(g_vsc.attach_info, error);
+    g_vsc.target.Attach(attach_info, error);
     // Reenable async events
     g_vsc.debugger.SetAsync(true);
   } else {
@@ -959,7 +959,7 @@ void request_completions(const llvm::json::Object &request) {
   for (size_t i = 0; i < count; i++) {
     std::string match = matches.GetStringAtIndex(i);
     std::string description = descriptions.GetStringAtIndex(i);
-    
+
     llvm::json::Object item;
 
     llvm::StringRef match_ref = match;
@@ -1262,7 +1262,7 @@ void request_initialize(const llvm::json::Object &request) {
   // The debug adapter supports the stepInTargetsRequest.
   body.try_emplace("supportsStepInTargetsRequest", false);
   // We need to improve the current implementation of completions in order to
-  // enable it again. For some context, this is how VSCode works: 
+  // enable it again. For some context, this is how VSCode works:
   // - VSCode sends a completion request whenever chars are added, the user
   //   triggers completion manually via CTRL-space or similar mechanisms, but
   //   not when there's a deletion. Besides, VSCode doesn't let us know which
@@ -1381,26 +1381,26 @@ void request_launch(const llvm::json::Object &request) {
   }
 
   // Instantiate a launch info instance for the target.
-  g_vsc.launch_info = g_vsc.target.GetLaunchInfo();
+  auto launch_info = g_vsc.target.GetLaunchInfo();
 
   // Grab the current working directory if there is one and set it in the
   // launch info.
   const auto cwd = GetString(arguments, "cwd");
   if (!cwd.empty())
-    g_vsc.launch_info.SetWorkingDirectory(cwd.data());
+    launch_info.SetWorkingDirectory(cwd.data());
 
   // Extract any extra arguments and append them to our program arguments for
   // when we launch
   auto args = GetStrings(arguments, "args");
   if (!args.empty())
-    g_vsc.launch_info.SetArguments(MakeArgv(args).data(), true);
+    launch_info.SetArguments(MakeArgv(args).data(), true);
 
   // Pass any environment variables along that the user specified.
   auto envs = GetStrings(arguments, "env");
   if (!envs.empty())
-    g_vsc.launch_info.SetEnvironmentEntries(MakeArgv(envs).data(), true);
+    launch_info.SetEnvironmentEntries(MakeArgv(envs).data(), true);
 
-  auto flags = g_vsc.launch_info.GetLaunchFlags();
+  auto flags = launch_info.GetLaunchFlags();
 
   if (GetBoolean(arguments, "disableASLR", true))
     flags |= lldb::eLaunchFlagDisableASLR;
@@ -1409,9 +1409,9 @@ void request_launch(const llvm::json::Object &request) {
   if (GetBoolean(arguments, "shellExpandArguments", false))
     flags |= lldb::eLaunchFlagShellExpandArguments;
   const bool detatchOnError = GetBoolean(arguments, "detachOnError", false);
-  g_vsc.launch_info.SetDetachOnError(detatchOnError);
-  g_vsc.launch_info.SetLaunchFlags(flags | lldb::eLaunchFlagDebug |
-                                   lldb::eLaunchFlagStopAtEntry);
+  launch_info.SetDetachOnError(detatchOnError);
+  launch_info.SetLaunchFlags(flags | lldb::eLaunchFlagDebug |
+                             lldb::eLaunchFlagStopAtEntry);
 
   // Run any pre run LLDB commands the user specified in the launch.json
   g_vsc.RunPreRunCommands();
@@ -1419,7 +1419,7 @@ void request_launch(const llvm::json::Object &request) {
     // Disable async events so the launch will be successful when we return from
     // the launch call and the launch will happen synchronously
     g_vsc.debugger.SetAsync(false);
-    g_vsc.target.Launch(g_vsc.launch_info, error);
+    g_vsc.target.Launch(launch_info, error);
     g_vsc.debugger.SetAsync(true);
   } else {
     g_vsc.RunLLDBCommands("Running launchCommands:", launchCommands);
@@ -1595,6 +1595,24 @@ void request_scopes(const llvm::json::Object &request) {
   llvm::json::Object body;
   auto arguments = request.getObject("arguments");
   lldb::SBFrame frame = g_vsc.GetLLDBFrame(*arguments);
+  // As the user selects different stack frames in the GUI, a "scopes" request
+  // will be sent to the DAP. This is the only way we know that the user has
+  // selected a frame in a thread. There are no other notifications that are
+  // sent and VS code doesn't allow multiple frames to show variables
+  // concurrently. If we select the thread and frame as the "scopes" requests
+  // are sent, this allows users to type commands in the debugger console
+  // with a backtick character to run lldb commands and these lldb commands
+  // will now have the right context selected as they are run. If the user
+  // types "`bt" into the debugger console and we had another thread selected
+  // in the LLDB library, we would show the wrong thing to the user. If the
+  // users switches threads with a lldb command like "`thread select 14", the
+  // GUI will not update as there are no "event" notification packets that
+  // allow us to change the currently selected thread or frame in the GUI that
+  // I am aware of.
+  if (frame.IsValid()) {
+    frame.GetThread().GetProcess().SetSelectedThread(frame.GetThread());
+    frame.GetThread().SetSelectedFrame(frame.GetFrameID());
+  }
   g_vsc.variables.Clear();
   g_vsc.variables.Append(frame.GetVariables(true,   // arguments
                                             true,   // locals
@@ -1733,70 +1751,45 @@ void request_setBreakpoints(const llvm::json::Object &request) {
   const auto path = GetString(source, "path");
   auto breakpoints = arguments->getArray("breakpoints");
   llvm::json::Array response_breakpoints;
+
   // Decode the source breakpoint infos for this "setBreakpoints" request
   SourceBreakpointMap request_bps;
   for (const auto &bp : *breakpoints) {
     auto bp_obj = bp.getAsObject();
     if (bp_obj) {
       SourceBreakpoint src_bp(*bp_obj);
-      request_bps[src_bp.line] = std::move(src_bp);
+      request_bps[src_bp.line] = src_bp;
+
+      // We check if this breakpoint already exists to update it
+      auto existing_source_bps = g_vsc.source_breakpoints.find(path);
+      if (existing_source_bps != g_vsc.source_breakpoints.end()) {
+        const auto &existing_bp = existing_source_bps->second.find(src_bp.line);
+        if (existing_bp != existing_source_bps->second.end()) {
+          existing_bp->second.UpdateBreakpoint(src_bp);
+          AppendBreakpoint(existing_bp->second.bp, response_breakpoints);
+          continue;
+        }
+      }
+      // At this point the breakpoint is new
+      src_bp.SetBreakpoint(path.data());
+      AppendBreakpoint(src_bp.bp, response_breakpoints);
+      g_vsc.source_breakpoints[path][src_bp.line] = std::move(src_bp);
     }
   }
 
-  // See if we already have breakpoints set for this source file from a
-  // previous "setBreakpoints" request
+  // Delete any breakpoints in this source file that aren't in the
+  // request_bps set. There is no call to remove breakpoints other than
+  // calling this function with a smaller or empty "breakpoints" list.
   auto old_src_bp_pos = g_vsc.source_breakpoints.find(path);
   if (old_src_bp_pos != g_vsc.source_breakpoints.end()) {
-
-    // We have already set breakpoints in this source file and they are giving
-    // use a new list of lines to set breakpoints on. Some breakpoints might
-    // already be set, and some might not. We need to remove any breakpoints
-    // whose lines are not contained in the any breakpoints lines in in the
-    // "breakpoints" array.
-
-    // Delete any breakpoints in this source file that aren't in the
-    // request_bps set. There is no call to remove breakpoints other than
-    // calling this function with a smaller or empty "breakpoints" list.
-    std::vector<uint32_t> remove_lines;
-    for (auto &pair: old_src_bp_pos->second) {
-      auto request_pos = request_bps.find(pair.first);
+    for (auto &old_bp : old_src_bp_pos->second) {
+      auto request_pos = request_bps.find(old_bp.first);
       if (request_pos == request_bps.end()) {
         // This breakpoint no longer exists in this source file, delete it
-        g_vsc.target.BreakpointDelete(pair.second.bp.GetID());
-        remove_lines.push_back(pair.first);
-      } else {
-        pair.second.UpdateBreakpoint(request_pos->second);
-        // Remove this breakpoint from the request breakpoints since we have
-        // handled it here and we don't need to set a new breakpoint below.
-        request_bps.erase(request_pos);
-        // Add this breakpoint info to the response
-        AppendBreakpoint(pair.second.bp, response_breakpoints);
+        g_vsc.target.BreakpointDelete(old_bp.second.bp.GetID());
+        old_src_bp_pos->second.erase(old_bp.first);
       }
     }
-    // Remove any lines from this existing source breakpoint map
-    for (auto line: remove_lines)
-     old_src_bp_pos->second.erase(line);
-
-    // Now add any breakpoint infos left over in request_bps are the
-    // breakpoints that weren't set in this source file yet. We need to update
-    // thread source breakpoint info for the source file in the variable
-    // "old_src_bp_pos->second" so the info for this source file is up to date.
-    for (auto &pair : request_bps) {
-      pair.second.SetBreakpoint(path.data());
-      // Add this breakpoint info to the response
-      AppendBreakpoint(pair.second.bp, response_breakpoints);
-      old_src_bp_pos->second[pair.first] = std::move(pair.second);
-    }
-  } else {
-    // No breakpoints were set for this source file yet. Set all breakpoints
-    // for each line and add them to the response and create an entry in
-    // g_vsc.source_breakpoints for this source file.
-    for (auto &pair : request_bps) {
-      pair.second.SetBreakpoint(path.data());
-      // Add this breakpoint info to the response
-      AppendBreakpoint(pair.second.bp, response_breakpoints);
-    }
-    g_vsc.source_breakpoints[path] = std::move(request_bps);
   }
 
   llvm::json::Object body;
diff --git a/lldb/unittests/Expression/DWARFExpressionTest.cpp b/lldb/unittests/Expression/DWARFExpressionTest.cpp
index 64755a9066de2..4c42819830787 100644
--- a/lldb/unittests/Expression/DWARFExpressionTest.cpp
+++ b/lldb/unittests/Expression/DWARFExpressionTest.cpp
@@ -234,6 +234,10 @@ TEST(DWARFExpression, DW_OP_convert) {
       llvm::Failed());
 }
 
+TEST(DWARFExpression, DW_OP_stack_value) {
+  EXPECT_THAT_EXPECTED(Evaluate({DW_OP_stack_value}), llvm::Failed());
+}
+
 TEST(DWARFExpression, DW_OP_piece) {
   EXPECT_THAT_EXPECTED(Evaluate({DW_OP_const2u, 0x11, 0x22, DW_OP_piece, 2,
                                  DW_OP_const2u, 0x33, 0x44, DW_OP_piece, 2}),
diff --git a/lldb/unittests/Host/FileSystemTest.cpp b/lldb/unittests/Host/FileSystemTest.cpp
index d8db2c12b0eea..f31c5c42a1e12 100644
--- a/lldb/unittests/Host/FileSystemTest.cpp
+++ b/lldb/unittests/Host/FileSystemTest.cpp
@@ -303,3 +303,29 @@ TEST(FileSystemTest, OpenErrno) {
   EXPECT_EQ(code.value(), ENOENT);
 }
 
+TEST(FileSystemTest, EmptyTest) {
+  FileSpec spec;
+  FileSystem fs;
+
+  {
+    std::error_code ec;
+    fs.DirBegin(spec, ec);
+    EXPECT_EQ(ec.category(), std::system_category());
+    EXPECT_EQ(ec.value(), ENOENT);
+  }
+
+  {
+    llvm::ErrorOr<vfs::Status> status = fs.GetStatus(spec);
+    ASSERT_FALSE(status);
+    EXPECT_EQ(status.getError().category(), std::system_category());
+    EXPECT_EQ(status.getError().value(), ENOENT);
+  }
+
+  EXPECT_EQ(sys::TimePoint<>(), fs.GetModificationTime(spec));
+  EXPECT_EQ(static_cast<uint64_t>(0), fs.GetByteSize(spec));
+  EXPECT_EQ(llvm::sys::fs::perms::perms_not_known, fs.GetPermissions(spec));
+  EXPECT_FALSE(fs.Exists(spec));
+  EXPECT_FALSE(fs.Readable(spec));
+  EXPECT_FALSE(fs.IsDirectory(spec));
+  EXPECT_FALSE(fs.IsLocal(spec));
+}
diff --git a/lldb/unittests/Language/CPlusPlus/CPlusPlusLanguageTest.cpp b/lldb/unittests/Language/CPlusPlus/CPlusPlusLanguageTest.cpp
index 4928534a40d1a..e8e6a4c0da018 100644
--- a/lldb/unittests/Language/CPlusPlus/CPlusPlusLanguageTest.cpp
+++ b/lldb/unittests/Language/CPlusPlus/CPlusPlusLanguageTest.cpp
@@ -140,12 +140,20 @@ TEST(CPlusPlusLanguage, ExtractContextAndIdentifier) {
        "std::vector<Class, std::allocator<Class>>",
        "_M_emplace_back_aux<Class const&>"},
       {"`anonymous namespace'::foo", "`anonymous namespace'", "foo"},
-      {"`operator<<A>'::`2'::B<0>::operator>",
-       "`operator<<A>'::`2'::B<0>",
+      {"`operator<<A>'::`2'::B<0>::operator>", "`operator<<A>'::`2'::B<0>",
        "operator>"},
       {"`anonymous namespace'::S::<<::__l2::Foo",
-       "`anonymous namespace'::S::<<::__l2",
-       "Foo"}};
+       "`anonymous namespace'::S::<<::__l2", "Foo"},
+      // These cases are idiosyncratic in how clang generates debug info for
+      // names when we have template parameters. They are not valid C++ names
+      // but if we fix this we need to support them for older compilers.
+      {"A::operator><A::B>", "A", "operator><A::B>"},
+      {"operator><A::B>", "", "operator><A::B>"},
+      {"A::operator<<A::B>", "A", "operator<<A::B>"},
+      {"operator<<A::B>", "", "operator<<A::B>"},
+      {"A::operator<<<A::B>", "A", "operator<<<A::B>"},
+      {"operator<<<A::B>", "", "operator<<<A::B>"},
+  };
 
   llvm::StringRef context, basename;
   for (const auto &test : test_cases) {
@@ -169,6 +177,12 @@ TEST(CPlusPlusLanguage, ExtractContextAndIdentifier) {
       "abc::", context, basename));
   EXPECT_FALSE(CPlusPlusLanguage::ExtractContextAndIdentifier(
       "f<A<B><C>>", context, basename));
+
+  // We expect these cases to fail until we turn on C++2a
+  EXPECT_FALSE(CPlusPlusLanguage::ExtractContextAndIdentifier(
+      "A::operator<=><A::B>", context, basename));
+  EXPECT_FALSE(CPlusPlusLanguage::ExtractContextAndIdentifier(
+      "operator<=><A::B>", context, basename));
 }
 
 static std::set<std::string> FindAlternate(llvm::StringRef Name) {
diff --git a/lldb/unittests/Process/gdb-remote/GDBRemoteTestUtils.h b/lldb/unittests/Process/gdb-remote/GDBRemoteTestUtils.h
index 0a48f34c9dac9..27ce6b9b26f20 100644
--- a/lldb/unittests/Process/gdb-remote/GDBRemoteTestUtils.h
+++ b/lldb/unittests/Process/gdb-remote/GDBRemoteTestUtils.h
@@ -77,7 +77,7 @@ class MockServer : public GDBRemoteCommunicationServer {
 class MockServerWithMockConnection : public MockServer {
 public:
   MockServerWithMockConnection() : MockServer() {
-    SetConnection(new MockConnection(m_packets));
+    SetConnection(std::make_unique<MockConnection>(m_packets));
   }
 
   llvm::ArrayRef<std::string> GetPackets() { return m_packets; };
diff --git a/lldb/unittests/Utility/UriParserTest.cpp b/lldb/unittests/Utility/UriParserTest.cpp
index c07d59a55e019..c88a647ef9378 100644
--- a/lldb/unittests/Utility/UriParserTest.cpp
+++ b/lldb/unittests/Utility/UriParserTest.cpp
@@ -74,12 +74,19 @@ TEST(UriParserTest, LongPath) {
   VALIDATE
 }
 
-TEST(UriParserTest, TypicalPortPath) {
+TEST(UriParserTest, TypicalPortPathIPv4) {
   const UriTestCase testCase("connect://192.168.100.132:5432/", "connect",
                              "192.168.100.132", 5432, "/");
   VALIDATE;
 }
 
+TEST(UriParserTest, TypicalPortPathIPv6) {
+  const UriTestCase testCase(
+      "connect://[2601:600:107f:db64:a42b:4faa:284:3082]:5432/", "connect",
+      "2601:600:107f:db64:a42b:4faa:284:3082", 5432, "/");
+  VALIDATE;
+}
+
 TEST(UriParserTest, BracketedHostnamePort) {
   const UriTestCase testCase("connect://[192.168.100.132]:5432/", "connect",
                              "192.168.100.132", 5432, "/");
@@ -102,6 +109,21 @@ TEST(UriParserTest, BracketedHostname) {
   VALIDATE
 }
 
+TEST(UriParserTest, BracketedHostnameWithPortIPv4) {
+  // Android device over IPv4: port is a part of the hostname.
+  const UriTestCase testCase("connect://[192.168.100.132:1234]", "connect",
+                             "192.168.100.132:1234", -1, "/");
+  VALIDATE
+}
+
+TEST(UriParserTest, BracketedHostnameWithPortIPv6) {
+  // Android device over IPv6: port is a part of the hostname.
+  const UriTestCase testCase(
+      "connect://[[2601:600:107f:db64:a42b:4faa:284]:1234]", "connect",
+      "[2601:600:107f:db64:a42b:4faa:284]:1234", -1, "/");
+  VALIDATE
+}
+
 TEST(UriParserTest, BracketedHostnameWithColon) {
   const UriTestCase testCase("connect://[192.168.100.132:5555]:1234", "connect",
                              "192.168.100.132:5555", 1234, "/");
diff --git a/lldb/unittests/tools/lldb-server/tests/TestClient.cpp b/lldb/unittests/tools/lldb-server/tests/TestClient.cpp
index b5a5ab3b27b24..52428e4afc8ad 100644
--- a/lldb/unittests/tools/lldb-server/tests/TestClient.cpp
+++ b/lldb/unittests/tools/lldb-server/tests/TestClient.cpp
@@ -30,7 +30,7 @@ using namespace llgs_tests;
 #endif
 
 TestClient::TestClient(std::unique_ptr<Connection> Conn) {
-  SetConnection(Conn.release());
+  SetConnection(std::move(Conn));
   SetPacketTimeout(std::chrono::seconds(10));
 }
 
diff --git a/llvm-spirv/include/LLVMSPIRVExtensions.inc b/llvm-spirv/include/LLVMSPIRVExtensions.inc
index 6071892c8b400..6acdc9486bfa6 100644
--- a/llvm-spirv/include/LLVMSPIRVExtensions.inc
+++ b/llvm-spirv/include/LLVMSPIRVExtensions.inc
@@ -16,3 +16,4 @@ EXT(SPV_INTEL_function_pointers)
 EXT(SPV_INTEL_kernel_attributes)
 EXT(SPV_INTEL_io_pipes)
 EXT(SPV_INTEL_inline_assembly)
+EXT(SPV_INTEL_arbitrary_precision_integers)
diff --git a/llvm-spirv/lib/SPIRV/SPIRVReader.cpp b/llvm-spirv/lib/SPIRV/SPIRVReader.cpp
index 30594a095aeaf..642077c36cf8c 100644
--- a/llvm-spirv/lib/SPIRV/SPIRVReader.cpp
+++ b/llvm-spirv/lib/SPIRV/SPIRVReader.cpp
@@ -1974,20 +1974,23 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *BV, Function *F,
       // Fastpath
       switch (ColNum) {
       case 2: {
-        Value *V1 = Builder.CreateShuffleVector(MCache[0], MCache[1], {0, 2});
+        Value *V1 = Builder.CreateShuffleVector(MCache[0], MCache[1],
+                                                ArrayRef<int>({0, 2}));
         V = Builder.CreateInsertValue(V, V1, 0);
-        Value *V2 = Builder.CreateShuffleVector(MCache[0], MCache[1], {1, 3});
+        Value *V2 = Builder.CreateShuffleVector(MCache[0], MCache[1],
+                                                ArrayRef<int>({1, 3}));
         V = Builder.CreateInsertValue(V, V2, 1);
         return mapValue(BV, V);
       }
 
       case 4: {
-        for (unsigned Idx = 0; Idx < 4; ++Idx) {
-          Value *V1 =
-              Builder.CreateShuffleVector(MCache[0], MCache[1], {Idx, Idx + 4});
-          Value *V2 =
-              Builder.CreateShuffleVector(MCache[2], MCache[3], {Idx, Idx + 4});
-          Value *V3 = Builder.CreateShuffleVector(V1, V2, {0, 1, 2, 3});
+        for (int Idx = 0; Idx < 4; ++Idx) {
+          Value *V1 = Builder.CreateShuffleVector(
+              MCache[0], MCache[1], ArrayRef<int>({Idx, Idx + 4}));
+          Value *V2 = Builder.CreateShuffleVector(
+              MCache[2], MCache[3], ArrayRef<int>({Idx, Idx + 4}));
+          Value *V3 =
+              Builder.CreateShuffleVector(V1, V2, ArrayRef<int>({0, 1, 2, 3}));
           V = Builder.CreateInsertValue(V, V3, Idx);
         }
         return mapValue(BV, V);
diff --git a/llvm-spirv/lib/SPIRV/SPIRVToOCL20.cpp b/llvm-spirv/lib/SPIRV/SPIRVToOCL20.cpp
index 833313d469e28..c87124f72897f 100644
--- a/llvm-spirv/lib/SPIRV/SPIRVToOCL20.cpp
+++ b/llvm-spirv/lib/SPIRV/SPIRVToOCL20.cpp
@@ -135,11 +135,23 @@ void SPIRVToOCL20::visitCallSPIRVControlBarrier(CallInst *CI) {
           return cast<ConstantInt>(Args[I])->getZExtValue();
         };
         auto ExecScope = static_cast<Scope>(GetArg(0));
-        auto MScope = static_cast<Scope>(GetArg(1));
-        auto Sema = mapSPIRVMemSemanticToOCL(GetArg(2));
+        auto MemScope = static_cast<Scope>(GetArg(1));
+
+        if (auto Arg = dyn_cast<ConstantInt>(Args[2])) {
+          auto Sema = mapSPIRVMemSemanticToOCL(Arg->getZExtValue());
+          Args[0] = getInt32(M, Sema.first);
+        } else {
+          int ClMemFenceMask = MemorySemanticsWorkgroupMemoryMask |
+                               MemorySemanticsCrossWorkgroupMemoryMask |
+                               MemorySemanticsImageMemoryMask;
+          Args[0] = getOrCreateSwitchFunc(
+              kSPIRVName::TranslateSPIRVMemFence, Args[2],
+              OCLMemFenceExtendedMap::getRMap(), true /*IsReverse*/, None, CI,
+              M, ClMemFenceMask);
+        }
+        Args[1] = getInt32(M, rmap<OCLScopeKind>(MemScope));
         Args.resize(2);
-        Args[0] = getInt32(M, Sema.first);
-        Args[1] = getInt32(M, rmap<OCLScopeKind>(MScope));
+
         return (ExecScope == ScopeWorkgroup) ? kOCLBuiltinName::WorkGroupBarrier
                                              : kOCLBuiltinName::SubGroupBarrier;
       },
diff --git a/llvm-spirv/lib/SPIRV/SPIRVWriter.cpp b/llvm-spirv/lib/SPIRV/SPIRVWriter.cpp
index 8b355974259eb..95b7df9d11a11 100644
--- a/llvm-spirv/lib/SPIRV/SPIRVWriter.cpp
+++ b/llvm-spirv/lib/SPIRV/SPIRVWriter.cpp
@@ -269,7 +269,9 @@ SPIRVType *LLVMToSPIRV::transType(Type *T) {
     // SPIR-V 2.16.1. Universal Validation Rules: Scalar integer types can be
     // parameterized only as 32 bit, plus any additional sizes enabled by
     // capabilities.
-    if (BM->getErrorLog().checkError(
+    if (BM->isAllowedToUseExtension(
+            ExtensionID::SPV_INTEL_arbitrary_precision_integers) ||
+        BM->getErrorLog().checkError(
             BitWidth == 8 || BitWidth == 16 || BitWidth == 32 || BitWidth == 64,
             SPIRVEC_InvalidBitWidth, std::to_string(BitWidth))) {
       return mapType(T, BM->addIntegerType(T->getIntegerBitWidth()));
@@ -1351,6 +1353,30 @@ bool LLVMToSPIRV::transDecoration(Value *V, SPIRVValue *BV) {
     }
   }
 
+  if (auto BVF = dyn_cast_or_null<FPMathOperator>(V)) {
+    auto Opcode = BVF->getOpcode();
+    if (Opcode == Instruction::FAdd || Opcode == Instruction::FSub ||
+        Opcode == Instruction::FMul || Opcode == Instruction::FDiv ||
+        Opcode == Instruction::FRem) {
+      FastMathFlags FMF = BVF->getFastMathFlags();
+      SPIRVWord M{0};
+      if (FMF.isFast())
+        M |= FPFastMathModeFastMask;
+      else {
+        if (FMF.noNaNs())
+          M |= FPFastMathModeNotNaNMask;
+        if (FMF.noInfs())
+          M |= FPFastMathModeNotInfMask;
+        if (FMF.noSignedZeros())
+          M |= FPFastMathModeNSZMask;
+        if (FMF.allowReciprocal())
+          M |= FPFastMathModeAllowRecipMask;
+      }
+      if (M != 0)
+        BV->setFPFastMathMode(M);
+    }
+  }
+
   return true;
 }
 
diff --git a/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVIsValidEnum.h b/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVIsValidEnum.h
index f460ecea25fb0..393471b874711 100644
--- a/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVIsValidEnum.h
+++ b/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVIsValidEnum.h
@@ -564,6 +564,7 @@ inline bool isValid(spv::Capability V) {
   case CapabilityNamedBarrier:
   case CapabilityPipeStorage:
   case CapabilityFPGAMemoryAttributesINTEL:
+  case CapabilityArbitraryPrecisionIntegersINTEL:
   case CapabilityFPGALoopControlsINTEL:
   case CapabilityBlockingPipesINTEL:
   case CapabilityUnstructuredLoopControlsINTEL:
diff --git a/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h b/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h
index 3d43fe1e10f9b..d52105861fbcb 100644
--- a/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h
+++ b/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h
@@ -498,6 +498,8 @@ template <> inline void SPIRVMap<Capability, std::string>::init() {
   add(CapabilitySubgroupAvcMotionEstimationChromaINTEL,
       "SubgroupAvcMotionEstimationChromaINTEL");
   add(CapabilityFPGAMemoryAttributesINTEL, "FPGAMemoryAttributesINTEL");
+  add(CapabilityArbitraryPrecisionIntegersINTEL,
+      "ArbitraryPrecisionIntegersINTEL");
   add(CapabilityFPGALoopControlsINTEL, "FPGALoopControlsINTEL");
   add(CapabilityFPGARegINTEL, "FPGARegINTEL");
   add(CapabilityBlockingPipesINTEL, "BlockingPipesINTEL");
diff --git a/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVType.h b/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVType.h
index cdbca79c11fd8..c717bc005f1bf 100644
--- a/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVType.h
+++ b/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVType.h
@@ -155,14 +155,29 @@ class SPIRVTypeInt : public SPIRVType {
     case 16:
       CV.push_back(CapabilityInt16);
       break;
+    case 32:
+      break;
     case 64:
       CV.push_back(CapabilityInt64);
       break;
     default:
-      break;
+      if (Module->isAllowedToUseExtension(
+              ExtensionID::SPV_INTEL_arbitrary_precision_integers))
+        CV.push_back(CapabilityArbitraryPrecisionIntegersINTEL);
     }
     return CV;
   }
+  SPIRVExtSet getRequiredExtensions() const override {
+    switch (BitWidth) {
+    case 8:
+    case 16:
+    case 32:
+    case 64:
+      return SPIRVExtSet();
+    default:
+      return getSet(ExtensionID::SPV_INTEL_arbitrary_precision_integers);
+    }
+  }
 
 protected:
   _SPIRV_DEF_ENCDEC3(Id, BitWidth, IsSigned)
diff --git a/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVValue.cpp b/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVValue.cpp
index 2b52c309a0c0c..7a8c68799f916 100644
--- a/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVValue.cpp
+++ b/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVValue.cpp
@@ -111,4 +111,14 @@ void SPIRVValue::setNoUnsignedWrap(bool HasNoUnsignedWrap) {
   }
 }
 
+void SPIRVValue::setFPFastMathMode(SPIRVWord M) {
+  if (M == 0) {
+    eraseDecorate(DecorationFPFastMathMode);
+    return;
+  }
+  addDecorate(new SPIRVDecorate(DecorationFPFastMathMode, this, M));
+  SPIRVDBG(spvdbgs() << "Set fast math mode to " << M << " for obj " << Id
+                     << "\n")
+}
+
 } // namespace SPIRV
diff --git a/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVValue.h b/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVValue.h
index 557a124abf70d..0843f4dc0fe58 100644
--- a/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVValue.h
+++ b/llvm-spirv/lib/SPIRV/libSPIRV/SPIRVValue.h
@@ -98,6 +98,7 @@ class SPIRVValue : public SPIRVEntry {
   void setVolatile(bool IsVolatile);
   void setNoSignedWrap(bool HasNoSignedWrap);
   void setNoUnsignedWrap(bool HasNoUnsignedWrap);
+  void setFPFastMathMode(SPIRVWord FPFastMathMode);
 
   void validate() const override {
     SPIRVEntry::validate();
diff --git a/llvm-spirv/lib/SPIRV/libSPIRV/spirv.hpp b/llvm-spirv/lib/SPIRV/libSPIRV/spirv.hpp
index 6f46dd7c975ee..05e3cca1a1b7a 100644
--- a/llvm-spirv/lib/SPIRV/libSPIRV/spirv.hpp
+++ b/llvm-spirv/lib/SPIRV/libSPIRV/spirv.hpp
@@ -692,6 +692,7 @@ enum Capability {
   CapabilitySubgroupAvcMotionEstimationIntraINTEL = 5697,
   CapabilitySubgroupAvcMotionEstimationChromaINTEL = 5698,
   CapabilityFPGAMemoryAttributesINTEL = 5824,
+  CapabilityArbitraryPrecisionIntegersINTEL = 5844,
   CapabilityUnstructuredLoopControlsINTEL = 5886,
   CapabilityFPGALoopControlsINTEL = 5888,
   CapabilityBlockingPipesINTEL = 5945,
diff --git a/llvm-spirv/test/FPFastMathModeNotNaNFast.spvasm b/llvm-spirv/test/FPFastMathModeNotNaNFast.spvasm
new file mode 100644
index 0000000000000..a7956d51f2446
--- /dev/null
+++ b/llvm-spirv/test/FPFastMathModeNotNaNFast.spvasm
@@ -0,0 +1,38 @@
+; REQUIRES: spirv-as
+; RUN: spirv-as --target-env spv1.0 -o %t.spv %s
+; RUN: spirv-val %t.spv
+; RUN: llvm-spirv -r -o - %t.spv | llvm-dis | FileCheck %s
+               OpCapability Addresses
+               OpCapability Kernel
+               OpMemoryModel Physical32 OpenCL
+               OpEntryPoint Kernel %1 "testFPFastMathModeNotNaNFast"
+               OpName %a "a"
+               OpName %b "b"
+               OpName %r1 "r1"
+               OpName %r2 "r2"
+               OpName %r3 "r3"
+               OpName %r4 "r4"
+               OpName %r5 "r5"
+               OpDecorate %dec FPFastMathMode NotNaN|Fast
+        %dec = OpDecorationGroup
+               OpGroupDecorate %dec %r1 %r2 %r3 %r4 %r5
+       %void = OpTypeVoid
+      %float = OpTypeFloat 32
+          %5 = OpTypeFunction %void %float %float
+          %1 = OpFunction %void None %5
+          %a = OpFunctionParameter %float
+          %b = OpFunctionParameter %float
+          %6 = OpLabel
+         %r1 = OpFAdd %float %a %b
+         %r2 = OpFSub %float %a %b
+         %r3 = OpFMul %float %a %b
+         %r4 = OpFDiv %float %a %b
+         %r5 = OpFRem %float %a %b
+               OpReturn
+               OpFunctionEnd
+
+; CHECK: %r1 = fadd fast float %a, %b
+; CHECK: %r2 = fsub fast float %a, %b
+; CHECK: %r3 = fmul fast float %a, %b
+; CHECK: %r4 = fdiv fast float %a, %b
+; CHECK: %r5 = frem fast float %a, %b
diff --git a/llvm-spirv/test/OpFAdd.spvasm b/llvm-spirv/test/OpFAdd.spvasm
deleted file mode 100644
index e4629057e1707..0000000000000
--- a/llvm-spirv/test/OpFAdd.spvasm
+++ /dev/null
@@ -1,55 +0,0 @@
-; REQUIRES: spirv-as
-; RUN: spirv-as --target-env spv1.0 -o %t.spv %s
-; RUN: spirv-val %t.spv
-; RUN: llvm-spirv -r -o - %t.spv | llvm-dis | FileCheck %s
-               OpCapability Addresses
-               OpCapability Kernel
-               OpMemoryModel Physical32 OpenCL
-               OpEntryPoint Kernel %1 "testFAdd"
-               OpName %a "a"
-               OpName %b "b"
-               OpName %r1 "r1"
-               OpName %r2 "r2"
-               OpName %r3 "r3"
-               OpName %r4 "r4"
-               OpName %r5 "r5"
-               OpName %r6 "r6"
-               OpName %r7 "r7"
-               OpName %r8 "r8"
-               OpName %r9 "r9"
-               OpDecorate %r2 FPFastMathMode None
-               OpDecorate %r3 FPFastMathMode NotNaN
-               OpDecorate %r4 FPFastMathMode NotInf
-               OpDecorate %r5 FPFastMathMode NSZ
-               OpDecorate %r6 FPFastMathMode AllowRecip
-               OpDecorate %r7 FPFastMathMode Fast
-               OpDecorate %r8 FPFastMathMode NotNaN|NotInf
-               OpDecorate %r9 FPFastMathMode NotNaN|Fast
-       %void = OpTypeVoid
-      %float = OpTypeFloat 32
-          %5 = OpTypeFunction %void %float %float
-          %1 = OpFunction %void None %5
-          %a = OpFunctionParameter %float
-          %b = OpFunctionParameter %float
-          %6 = OpLabel
-         %r1 = OpFAdd %float %a %b
-         %r2 = OpFAdd %float %a %b
-         %r3 = OpFAdd %float %a %b
-         %r4 = OpFAdd %float %a %b
-         %r5 = OpFAdd %float %a %b
-         %r6 = OpFAdd %float %a %b
-         %r7 = OpFAdd %float %a %b
-         %r8 = OpFAdd %float %a %b
-         %r9 = OpFAdd %float %a %b
-               OpReturn
-               OpFunctionEnd
-
-; CHECK: %r1 = fadd float %a, %b
-; CHECK: %r2 = fadd float %a, %b
-; CHECK: %r3 = fadd nnan float %a, %b
-; CHECK: %r4 = fadd ninf float %a, %b
-; CHECK: %r5 = fadd nsz float %a, %b
-; CHECK: %r6 = fadd arcp float %a, %b
-; CHECK: %r7 = fadd fast float %a, %b
-; CHECK: %r8 = fadd nnan ninf float %a, %b
-; CHECK: %r9 = fadd fast float %a, %b
diff --git a/llvm-spirv/test/OpFDiv.spvasm b/llvm-spirv/test/OpFDiv.spvasm
deleted file mode 100644
index 1386f3012f623..0000000000000
--- a/llvm-spirv/test/OpFDiv.spvasm
+++ /dev/null
@@ -1,55 +0,0 @@
-; REQUIRES: spirv-as
-; RUN: spirv-as --target-env spv1.0 -o %t.spv %s
-; RUN: spirv-val %t.spv
-; RUN: llvm-spirv -r -o - %t.spv | llvm-dis | FileCheck %s
-               OpCapability Addresses
-               OpCapability Kernel
-               OpMemoryModel Physical32 OpenCL
-               OpEntryPoint Kernel %1 "testFDiv"
-               OpName %a "a"
-               OpName %b "b"
-               OpName %r1 "r1"
-               OpName %r2 "r2"
-               OpName %r3 "r3"
-               OpName %r4 "r4"
-               OpName %r5 "r5"
-               OpName %r6 "r6"
-               OpName %r7 "r7"
-               OpName %r8 "r8"
-               OpName %r9 "r9"
-               OpDecorate %r2 FPFastMathMode None
-               OpDecorate %r3 FPFastMathMode NotNaN
-               OpDecorate %r4 FPFastMathMode NotInf
-               OpDecorate %r5 FPFastMathMode NSZ
-               OpDecorate %r6 FPFastMathMode AllowRecip
-               OpDecorate %r7 FPFastMathMode Fast
-               OpDecorate %r8 FPFastMathMode NotNaN|NotInf
-               OpDecorate %r9 FPFastMathMode NotNaN|Fast
-       %void = OpTypeVoid
-      %float = OpTypeFloat 32
-          %5 = OpTypeFunction %void %float %float
-          %1 = OpFunction %void None %5
-          %a = OpFunctionParameter %float
-          %b = OpFunctionParameter %float
-          %6 = OpLabel
-         %r1 = OpFDiv %float %a %b
-         %r2 = OpFDiv %float %a %b
-         %r3 = OpFDiv %float %a %b
-         %r4 = OpFDiv %float %a %b
-         %r5 = OpFDiv %float %a %b
-         %r6 = OpFDiv %float %a %b
-         %r7 = OpFDiv %float %a %b
-         %r8 = OpFDiv %float %a %b
-         %r9 = OpFDiv %float %a %b
-               OpReturn
-               OpFunctionEnd
-
-; CHECK: %r1 = fdiv float %a, %b
-; CHECK: %r2 = fdiv float %a, %b
-; CHECK: %r3 = fdiv nnan float %a, %b
-; CHECK: %r4 = fdiv ninf float %a, %b
-; CHECK: %r5 = fdiv nsz float %a, %b
-; CHECK: %r6 = fdiv arcp float %a, %b
-; CHECK: %r7 = fdiv fast float %a, %b
-; CHECK: %r8 = fdiv nnan ninf float %a, %b
-; CHECK: %r9 = fdiv fast float %a, %b
diff --git a/llvm-spirv/test/OpFMul.spvasm b/llvm-spirv/test/OpFMul.spvasm
deleted file mode 100644
index 036315e283548..0000000000000
--- a/llvm-spirv/test/OpFMul.spvasm
+++ /dev/null
@@ -1,55 +0,0 @@
-; REQUIRES: spirv-as
-; RUN: spirv-as --target-env spv1.0 -o %t.spv %s
-; RUN: spirv-val %t.spv
-; RUN: llvm-spirv -r -o - %t.spv | llvm-dis | FileCheck %s
-               OpCapability Addresses
-               OpCapability Kernel
-               OpMemoryModel Physical32 OpenCL
-               OpEntryPoint Kernel %1 "testFMul"
-               OpName %a "a"
-               OpName %b "b"
-               OpName %r1 "r1"
-               OpName %r2 "r2"
-               OpName %r3 "r3"
-               OpName %r4 "r4"
-               OpName %r5 "r5"
-               OpName %r6 "r6"
-               OpName %r7 "r7"
-               OpName %r8 "r8"
-               OpName %r9 "r9"
-               OpDecorate %r2 FPFastMathMode None
-               OpDecorate %r3 FPFastMathMode NotNaN
-               OpDecorate %r4 FPFastMathMode NotInf
-               OpDecorate %r5 FPFastMathMode NSZ
-               OpDecorate %r6 FPFastMathMode AllowRecip
-               OpDecorate %r7 FPFastMathMode Fast
-               OpDecorate %r8 FPFastMathMode NotNaN|NotInf
-               OpDecorate %r9 FPFastMathMode NotNaN|Fast
-       %void = OpTypeVoid
-      %float = OpTypeFloat 32
-          %5 = OpTypeFunction %void %float %float
-          %1 = OpFunction %void None %5
-          %a = OpFunctionParameter %float
-          %b = OpFunctionParameter %float
-          %6 = OpLabel
-         %r1 = OpFMul %float %a %b
-         %r2 = OpFMul %float %a %b
-         %r3 = OpFMul %float %a %b
-         %r4 = OpFMul %float %a %b
-         %r5 = OpFMul %float %a %b
-         %r6 = OpFMul %float %a %b
-         %r7 = OpFMul %float %a %b
-         %r8 = OpFMul %float %a %b
-         %r9 = OpFMul %float %a %b
-               OpReturn
-               OpFunctionEnd
-
-; CHECK: %r1 = fmul float %a, %b
-; CHECK: %r2 = fmul float %a, %b
-; CHECK: %r3 = fmul nnan float %a, %b
-; CHECK: %r4 = fmul ninf float %a, %b
-; CHECK: %r5 = fmul nsz float %a, %b
-; CHECK: %r6 = fmul arcp float %a, %b
-; CHECK: %r7 = fmul fast float %a, %b
-; CHECK: %r8 = fmul nnan ninf float %a, %b
-; CHECK: %r9 = fmul fast float %a, %b
diff --git a/llvm-spirv/test/OpFRem.spvasm b/llvm-spirv/test/OpFRem.spvasm
deleted file mode 100644
index 47b060c902d42..0000000000000
--- a/llvm-spirv/test/OpFRem.spvasm
+++ /dev/null
@@ -1,55 +0,0 @@
-; REQUIRES: spirv-as
-; RUN: spirv-as --target-env spv1.0 -o %t.spv %s
-; RUN: spirv-val %t.spv
-; RUN: llvm-spirv -r -o - %t.spv | llvm-dis | FileCheck %s
-               OpCapability Addresses
-               OpCapability Kernel
-               OpMemoryModel Physical32 OpenCL
-               OpEntryPoint Kernel %1 "testFRem"
-               OpName %a "a"
-               OpName %b "b"
-               OpName %r1 "r1"
-               OpName %r2 "r2"
-               OpName %r3 "r3"
-               OpName %r4 "r4"
-               OpName %r5 "r5"
-               OpName %r6 "r6"
-               OpName %r7 "r7"
-               OpName %r8 "r8"
-               OpName %r9 "r9"
-               OpDecorate %r2 FPFastMathMode None
-               OpDecorate %r3 FPFastMathMode NotNaN
-               OpDecorate %r4 FPFastMathMode NotInf
-               OpDecorate %r5 FPFastMathMode NSZ
-               OpDecorate %r6 FPFastMathMode AllowRecip
-               OpDecorate %r7 FPFastMathMode Fast
-               OpDecorate %r8 FPFastMathMode NotNaN|NotInf
-               OpDecorate %r9 FPFastMathMode NotNaN|Fast
-       %void = OpTypeVoid
-      %float = OpTypeFloat 32
-          %5 = OpTypeFunction %void %float %float
-          %1 = OpFunction %void None %5
-          %a = OpFunctionParameter %float
-          %b = OpFunctionParameter %float
-          %6 = OpLabel
-         %r1 = OpFRem %float %a %b
-         %r2 = OpFRem %float %a %b
-         %r3 = OpFRem %float %a %b
-         %r4 = OpFRem %float %a %b
-         %r5 = OpFRem %float %a %b
-         %r6 = OpFRem %float %a %b
-         %r7 = OpFRem %float %a %b
-         %r8 = OpFRem %float %a %b
-         %r9 = OpFRem %float %a %b
-               OpReturn
-               OpFunctionEnd
-
-; CHECK: %r1 = frem float %a, %b
-; CHECK: %r2 = frem float %a, %b
-; CHECK: %r3 = frem nnan float %a, %b
-; CHECK: %r4 = frem ninf float %a, %b
-; CHECK: %r5 = frem nsz float %a, %b
-; CHECK: %r6 = frem arcp float %a, %b
-; CHECK: %r7 = frem fast float %a, %b
-; CHECK: %r8 = frem nnan ninf float %a, %b
-; CHECK: %r9 = frem fast float %a, %b
diff --git a/llvm-spirv/test/OpFSub.spvasm b/llvm-spirv/test/OpFSub.spvasm
deleted file mode 100644
index c978a69eb5229..0000000000000
--- a/llvm-spirv/test/OpFSub.spvasm
+++ /dev/null
@@ -1,55 +0,0 @@
-; REQUIRES: spirv-as
-; RUN: spirv-as --target-env spv1.0 -o %t.spv %s
-; RUN: spirv-val %t.spv
-; RUN: llvm-spirv -r -o - %t.spv | llvm-dis | FileCheck %s
-               OpCapability Addresses
-               OpCapability Kernel
-               OpMemoryModel Physical32 OpenCL
-               OpEntryPoint Kernel %1 "testFSub"
-               OpName %a "a"
-               OpName %b "b"
-               OpName %r1 "r1"
-               OpName %r2 "r2"
-               OpName %r3 "r3"
-               OpName %r4 "r4"
-               OpName %r5 "r5"
-               OpName %r6 "r6"
-               OpName %r7 "r7"
-               OpName %r8 "r8"
-               OpName %r9 "r9"
-               OpDecorate %r2 FPFastMathMode None
-               OpDecorate %r3 FPFastMathMode NotNaN
-               OpDecorate %r4 FPFastMathMode NotInf
-               OpDecorate %r5 FPFastMathMode NSZ
-               OpDecorate %r6 FPFastMathMode AllowRecip
-               OpDecorate %r7 FPFastMathMode Fast
-               OpDecorate %r8 FPFastMathMode NotNaN|NotInf
-               OpDecorate %r9 FPFastMathMode NotNaN|Fast
-       %void = OpTypeVoid
-      %float = OpTypeFloat 32
-          %5 = OpTypeFunction %void %float %float
-          %1 = OpFunction %void None %5
-          %a = OpFunctionParameter %float
-          %b = OpFunctionParameter %float
-          %6 = OpLabel
-         %r1 = OpFSub %float %a %b
-         %r2 = OpFSub %float %a %b
-         %r3 = OpFSub %float %a %b
-         %r4 = OpFSub %float %a %b
-         %r5 = OpFSub %float %a %b
-         %r6 = OpFSub %float %a %b
-         %r7 = OpFSub %float %a %b
-         %r8 = OpFSub %float %a %b
-         %r9 = OpFSub %float %a %b
-               OpReturn
-               OpFunctionEnd
-
-; CHECK: %r1 = fsub float %a, %b
-; CHECK: %r2 = fsub float %a, %b
-; CHECK: %r3 = fsub nnan float %a, %b
-; CHECK: %r4 = fsub ninf float %a, %b
-; CHECK: %r5 = fsub nsz float %a, %b
-; CHECK: %r6 = fsub arcp float %a, %b
-; CHECK: %r7 = fsub fast float %a, %b
-; CHECK: %r8 = fsub nnan ninf float %a, %b
-; CHECK: %r9 = fsub fast float %a, %b
diff --git a/llvm-spirv/test/barrier_explicit_arguments.spt b/llvm-spirv/test/barrier_explicit_arguments.spt
index df5328d0061ad..627cd1a999add 100644
--- a/llvm-spirv/test/barrier_explicit_arguments.spt
+++ b/llvm-spirv/test/barrier_explicit_arguments.spt
@@ -4,7 +4,6 @@
 2 Capability Kernel
 5 ExtInstImport 1 "OpenCL.std"
 3 MemoryModel 1 2
-3 Source 3 102000
 4 Name 5 "test"
 3 Name 6 "val"
 4 Name 7 "entry"
@@ -12,6 +11,7 @@
 4 TypeInt 3 32 0
 4 Constant 3 8 2
 2 TypeVoid 2
+4 Constant 3 9 3
 4 TypeFunction 4 2 3
 
 5 Function 2 5 0 4
@@ -19,18 +19,26 @@
 
 2 Label 7
 4 ControlBarrier 8 8 6
+4 ControlBarrier 9 9 6
 1 Return
 
 1 FunctionEnd
 
 ; RUN: llvm-spirv %s -to-binary -o %t.spv
 ; RUN: spirv-val %t.spv
-; RUN: llvm-spirv -r %t.spv -o %t.bc
-; RUN: llvm-dis < %t.bc | FileCheck %s
+; RUN: llvm-spirv -r -spirv-ocl-builtins-version=CL2.0 %t.spv -o %t.bc
+; RUN: llvm-dis < %t.bc | FileCheck %s --check-prefixes=CHECK,CHECK-20
+; RUN: llvm-spirv -r -spirv-ocl-builtins-version=CL1.2 %t.spv -o %t.bc
+; RUN: llvm-dis < %t.bc | FileCheck %s --check-prefixes=CHECK,CHECK-12
 
 ; CHECK: define spir_func void @test(i32 %val)
 ; CHECK: %call = call spir_func i32 @__translate_spirv_memory_fence(i32 %val)
-; CHECK: call spir_func void @_Z7barrierj(i32 %call)
+; CHECK-12: call spir_func void @_Z7barrierj(i32 %call)
+; There is no support for sub-groups in OpenCL 1.2, so sub-group barrier is
+; equal to regular barrier in there
+; CHECK-12: call spir_func void @_Z7barrierj(i32 %call1)
+; CHECK-20: call spir_func void @_Z18work_group_barrierj12memory_scope(i32 %call
+; CHECK-20: call spir_func void @_Z17sub_group_barrierj12memory_scope(i32 %call
 ; CHECK: define private spir_func i32 @__translate_spirv_memory_fence(i32 %key)
 ; CHECK: entry:
 ; CHECK:   %key.masked = and i32 2816, %key
@@ -60,4 +68,3 @@
 ; CHECK: case.2816:
 ; CHECK:   ret i32 7
 ; CHECK: }
-; CHECK: declare spir_func void @_Z7barrierj(i32)
diff --git a/llvm-spirv/test/capability-arbitrary-precision-integers.ll b/llvm-spirv/test/capability-arbitrary-precision-integers.ll
new file mode 100644
index 0000000000000..9de37f7a33dde
--- /dev/null
+++ b/llvm-spirv/test/capability-arbitrary-precision-integers.ll
@@ -0,0 +1,38 @@
+; RUN: llvm-as %s -o %t.bc
+; RUN: llvm-spirv %t.bc --spirv-ext=+SPV_INTEL_arbitrary_precision_integers -o %t.spv
+; RUN: llvm-spirv %t.spv -to-text -o %t.spt
+; RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV
+
+; RUN: llvm-spirv -r %t.spv -o %t.rev.bc
+; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM
+
+; RUN: llvm-spirv -spirv-text -r %t.spt -o %t.rev.bc
+; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM
+
+; CHECK-SPIRV: Capability ArbitraryPrecisionIntegersINTEL
+; CHECK-SPIRV: Extension "SPV_INTEL_arbitrary_precision_integers"
+
+; CHECK-SPIRV-DAG: TypeInt {{[0-9]+}} 13 0
+; CHECK-SPIRV-DAG: TypeInt {{[0-9]+}} 58 0
+; CHECK-SPIRV-DAG: TypeInt {{[0-9]+}} 30 0
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024"
+target triple = "spir64-unknown-unknown"
+
+; CHECK-LLVM: @a = addrspace(1) global i13 0, align 2
+; CHECK-LLVM: @b = addrspace(1) global i58 0, align 8
+@a = addrspace(1) global i13 0, align 2
+@b = addrspace(1) global i58 0, align 8
+
+; Function Attrs: noinline nounwind optnone
+; CHECK-LLVM: void @_Z4funci(i30 %a)
+define spir_func void @_Z4funci(i30 %a) {
+entry:
+; CHECK-LLVM: %a.addr = alloca i30
+  %a.addr = alloca i30, align 4
+; CHECK-LLVM: store i30 %a, i30* %a.addr
+  store i30 %a, i30* %a.addr, align 4
+; CHECK-LLVM: store i30 1, i30* %a.addr
+  store i30 1, i30* %a.addr, align 4
+  ret void
+}
diff --git a/llvm-spirv/test/AllowIntrinsics.ll b/llvm-spirv/test/transcoding/AllowIntrinsics.ll
similarity index 100%
rename from llvm-spirv/test/AllowIntrinsics.ll
rename to llvm-spirv/test/transcoding/AllowIntrinsics.ll
diff --git a/llvm-spirv/test/CreatePipeFromPipeStorage.ll b/llvm-spirv/test/transcoding/CreatePipeFromPipeStorage.ll
similarity index 100%
rename from llvm-spirv/test/CreatePipeFromPipeStorage.ll
rename to llvm-spirv/test/transcoding/CreatePipeFromPipeStorage.ll
diff --git a/llvm-spirv/test/ExecutionMode_SPIR_to_SPIRV.ll b/llvm-spirv/test/transcoding/ExecutionMode_SPIR_to_SPIRV.ll
similarity index 100%
rename from llvm-spirv/test/ExecutionMode_SPIR_to_SPIRV.ll
rename to llvm-spirv/test/transcoding/ExecutionMode_SPIR_to_SPIRV.ll
diff --git a/llvm-spirv/test/InfiniteLoopMetadataPlacement.ll b/llvm-spirv/test/transcoding/InfiniteLoopMetadataPlacement.ll
similarity index 100%
rename from llvm-spirv/test/InfiniteLoopMetadataPlacement.ll
rename to llvm-spirv/test/transcoding/InfiniteLoopMetadataPlacement.ll
diff --git a/llvm-spirv/test/IntelFPGAFunctionAttributes.ll b/llvm-spirv/test/transcoding/IntelFPGAFunctionAttributes.ll
similarity index 100%
rename from llvm-spirv/test/IntelFPGAFunctionAttributes.ll
rename to llvm-spirv/test/transcoding/IntelFPGAFunctionAttributes.ll
diff --git a/llvm-spirv/test/IntelFPGAMemoryAttributes.ll b/llvm-spirv/test/transcoding/IntelFPGAMemoryAttributes.ll
similarity index 100%
rename from llvm-spirv/test/IntelFPGAMemoryAttributes.ll
rename to llvm-spirv/test/transcoding/IntelFPGAMemoryAttributes.ll
diff --git a/llvm-spirv/test/IntelFPGAMemoryAttributesForStaticVar.ll b/llvm-spirv/test/transcoding/IntelFPGAMemoryAttributesForStaticVar.ll
similarity index 100%
rename from llvm-spirv/test/IntelFPGAMemoryAttributesForStaticVar.ll
rename to llvm-spirv/test/transcoding/IntelFPGAMemoryAttributesForStaticVar.ll
diff --git a/llvm-spirv/test/IntelFPGAMemoryAttributesForStruct.ll b/llvm-spirv/test/transcoding/IntelFPGAMemoryAttributesForStruct.ll
similarity index 100%
rename from llvm-spirv/test/IntelFPGAMemoryAttributesForStruct.ll
rename to llvm-spirv/test/transcoding/IntelFPGAMemoryAttributesForStruct.ll
diff --git a/llvm-spirv/test/IntelFPGAReg.ll b/llvm-spirv/test/transcoding/IntelFPGAReg.ll
similarity index 100%
rename from llvm-spirv/test/IntelFPGAReg.ll
rename to llvm-spirv/test/transcoding/IntelFPGAReg.ll
diff --git a/llvm-spirv/test/KernelArgTypeInOpString.ll b/llvm-spirv/test/transcoding/KernelArgTypeInOpString.ll
similarity index 100%
rename from llvm-spirv/test/KernelArgTypeInOpString.ll
rename to llvm-spirv/test/transcoding/KernelArgTypeInOpString.ll
diff --git a/llvm-spirv/test/KernelArgTypeInOpString2.ll b/llvm-spirv/test/transcoding/KernelArgTypeInOpString2.ll
similarity index 100%
rename from llvm-spirv/test/KernelArgTypeInOpString2.ll
rename to llvm-spirv/test/transcoding/KernelArgTypeInOpString2.ll
diff --git a/llvm-spirv/test/NoSignedUnsignedWrap.ll b/llvm-spirv/test/transcoding/NoSignedUnsignedWrap.ll
similarity index 100%
rename from llvm-spirv/test/NoSignedUnsignedWrap.ll
rename to llvm-spirv/test/transcoding/NoSignedUnsignedWrap.ll
diff --git a/llvm-spirv/test/OpBitReverse_i32.ll b/llvm-spirv/test/transcoding/OpBitReverse_i32.ll
similarity index 100%
rename from llvm-spirv/test/OpBitReverse_i32.ll
rename to llvm-spirv/test/transcoding/OpBitReverse_i32.ll
diff --git a/llvm-spirv/test/OpBitReverse_v2i16.ll b/llvm-spirv/test/transcoding/OpBitReverse_v2i16.ll
similarity index 100%
rename from llvm-spirv/test/OpBitReverse_v2i16.ll
rename to llvm-spirv/test/transcoding/OpBitReverse_v2i16.ll
diff --git a/llvm-spirv/test/OpLine.ll b/llvm-spirv/test/transcoding/OpLine.ll
similarity index 100%
rename from llvm-spirv/test/OpLine.ll
rename to llvm-spirv/test/transcoding/OpLine.ll
diff --git a/llvm-spirv/test/OpSwitch32.ll b/llvm-spirv/test/transcoding/OpSwitch32.ll
similarity index 100%
rename from llvm-spirv/test/OpSwitch32.ll
rename to llvm-spirv/test/transcoding/OpSwitch32.ll
diff --git a/llvm-spirv/test/OpSwitch64.ll b/llvm-spirv/test/transcoding/OpSwitch64.ll
similarity index 100%
rename from llvm-spirv/test/OpSwitch64.ll
rename to llvm-spirv/test/transcoding/OpSwitch64.ll
diff --git a/llvm-spirv/test/OpSwitchChar.ll b/llvm-spirv/test/transcoding/OpSwitchChar.ll
similarity index 100%
rename from llvm-spirv/test/OpSwitchChar.ll
rename to llvm-spirv/test/transcoding/OpSwitchChar.ll
diff --git a/llvm-spirv/test/PipeBlocking.ll b/llvm-spirv/test/transcoding/PipeBlocking.ll
similarity index 100%
rename from llvm-spirv/test/PipeBlocking.ll
rename to llvm-spirv/test/transcoding/PipeBlocking.ll
diff --git a/llvm-spirv/test/PipeStorage.ll b/llvm-spirv/test/transcoding/PipeStorage.ll
similarity index 100%
rename from llvm-spirv/test/PipeStorage.ll
rename to llvm-spirv/test/transcoding/PipeStorage.ll
diff --git a/llvm-spirv/test/PipeStorageIOINTEL.ll b/llvm-spirv/test/transcoding/PipeStorageIOINTEL.ll
similarity index 100%
rename from llvm-spirv/test/PipeStorageIOINTEL.ll
rename to llvm-spirv/test/transcoding/PipeStorageIOINTEL.ll
diff --git a/llvm-spirv/test/SampledImage.cl b/llvm-spirv/test/transcoding/SampledImage.cl
similarity index 100%
rename from llvm-spirv/test/SampledImage.cl
rename to llvm-spirv/test/transcoding/SampledImage.cl
diff --git a/llvm-spirv/test/annotate_attribute.ll b/llvm-spirv/test/transcoding/annotate_attribute.ll
similarity index 100%
rename from llvm-spirv/test/annotate_attribute.ll
rename to llvm-spirv/test/transcoding/annotate_attribute.ll
diff --git a/llvm-spirv/test/annotation_dbg_info_drop.ll b/llvm-spirv/test/transcoding/annotation_dbg_info_drop.ll
similarity index 100%
rename from llvm-spirv/test/annotation_dbg_info_drop.ll
rename to llvm-spirv/test/transcoding/annotation_dbg_info_drop.ll
diff --git a/llvm-spirv/test/builtin_vars.ll b/llvm-spirv/test/transcoding/builtin_vars.ll
similarity index 100%
rename from llvm-spirv/test/builtin_vars.ll
rename to llvm-spirv/test/transcoding/builtin_vars.ll
diff --git a/llvm-spirv/test/transcoding/fadd.ll b/llvm-spirv/test/transcoding/fadd.ll
new file mode 100644
index 0000000000000..f1bb0a7f8b26c
--- /dev/null
+++ b/llvm-spirv/test/transcoding/fadd.ll
@@ -0,0 +1,65 @@
+; RUN: llvm-as %s -o %t.bc
+; RUN: llvm-spirv -spirv-text %t.bc -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: llvm-spirv %t.bc -o %t.spv
+; RUN: spirv-val %t.spv
+; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o - | FileCheck %s --check-prefix=CHECK-LLVM
+
+; CHECK-SPIRV: 3 Name [[#r1:]] "r1"
+; CHECK-SPIRV: 3 Name [[#r2:]] "r2"
+; CHECK-SPIRV: 3 Name [[#r3:]] "r3"
+; CHECK-SPIRV: 3 Name [[#r4:]] "r4"
+; CHECK-SPIRV: 3 Name [[#r5:]] "r5"
+; CHECK-SPIRV: 3 Name [[#r6:]] "r6"
+; CHECK-SPIRV: 3 Name [[#r7:]] "r7"
+; CHECK-SPIRV-NOT: 4 Decorate [[#r1]] FPFastMathMode
+; CHECK-SPIRV-DAG: 4 Decorate [[#r2]] FPFastMathMode 1
+; CHECK-SPIRV-DAG: 4 Decorate [[#r3]] FPFastMathMode 2
+; CHECK-SPIRV-DAG: 4 Decorate [[#r4]] FPFastMathMode 4
+; CHECK-SPIRV-DAG: 4 Decorate [[#r5]] FPFastMathMode 8
+; CHECK-SPIRV-DAG: 4 Decorate [[#r6]] FPFastMathMode 16
+; CHECK-SPIRV-DAG: 4 Decorate [[#r7]] FPFastMathMode 3
+; CHECK-SPIRV: 3 TypeFloat [[float:[0-9]+]] 32
+; CHECK-SPIRV: 5 FAdd [[float]] [[#r1]]
+; CHECK-SPIRV: 5 FAdd [[float]] [[#r2]]
+; CHECK-SPIRV: 5 FAdd [[float]] [[#r3]]
+; CHECK-SPIRV: 5 FAdd [[float]] [[#r4]]
+; CHECK-SPIRV: 5 FAdd [[float]] [[#r5]]
+; CHECK-SPIRV: 5 FAdd [[float]] [[#r6]]
+; CHECK-SPIRV: 5 FAdd [[float]] [[#r7]]
+
+; CHECK-LLVM: %r1 = fadd float %a, %b
+; CHECK-LLVM: %r2 = fadd nnan float %a, %b
+; CHECK-LLVM: %r3 = fadd ninf float %a, %b
+; CHECK-LLVM: %r4 = fadd nsz float %a, %b
+; CHECK-LLVM: %r5 = fadd arcp float %a, %b
+; CHECK-LLVM: %r6 = fadd fast float %a, %b
+; CHECK-LLVM: %r7 = fadd nnan ninf float %a, %b
+
+target datalayout = "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024"
+target triple = "spir-unknown-unknown"
+
+; Function Attrs: nounwind
+define spir_kernel void @testFAdd(float %a, float %b) local_unnamed_addr #0 !kernel_arg_addr_space !2 !kernel_arg_access_qual !3 !kernel_arg_type !4 !kernel_arg_base_type !4 !kernel_arg_type_qual !5 {
+entry:
+  %r1 = fadd float %a, %b
+  %r2 = fadd nnan float %a, %b
+  %r3 = fadd ninf float %a, %b
+  %r4 = fadd nsz float %a, %b
+  %r5 = fadd arcp float %a, %b
+  %r6 = fadd fast float %a, %b
+  %r7 = fadd nnan ninf float %a, %b
+  ret void
+}
+
+attributes #0 = { convergent nounwind writeonly "correctly-rounded-divide-sqrt-fp-math"="false" "denorms-are-zero"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "uniform-work-group-size"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.module.flags = !{!0}
+!opencl.ocl.version = !{!1}
+!opencl.spir.version = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 2, i32 0}
+!2 = !{i32 0, i32 0}
+!3 = !{!"none", !"none"}
+!4 = !{!"float", !"float"}
+!5 = !{!"", !""}
diff --git a/llvm-spirv/test/transcoding/fdiv.ll b/llvm-spirv/test/transcoding/fdiv.ll
new file mode 100644
index 0000000000000..1830efe16cdec
--- /dev/null
+++ b/llvm-spirv/test/transcoding/fdiv.ll
@@ -0,0 +1,65 @@
+; RUN: llvm-as %s -o %t.bc
+; RUN: llvm-spirv -spirv-text %t.bc -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: llvm-spirv %t.bc -o %t.spv
+; RUN: spirv-val %t.spv
+; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o - | FileCheck %s --check-prefix=CHECK-LLVM
+
+; CHECK-SPIRV: 3 Name [[#r1:]] "r1"
+; CHECK-SPIRV: 3 Name [[#r2:]] "r2"
+; CHECK-SPIRV: 3 Name [[#r3:]] "r3"
+; CHECK-SPIRV: 3 Name [[#r4:]] "r4"
+; CHECK-SPIRV: 3 Name [[#r5:]] "r5"
+; CHECK-SPIRV: 3 Name [[#r6:]] "r6"
+; CHECK-SPIRV: 3 Name [[#r7:]] "r7"
+; CHECK-SPIRV-NOT: 4 Decorate [[#r1]] FPFastMathMode
+; CHECK-SPIRV-DAG: 4 Decorate [[#r2]] FPFastMathMode 1
+; CHECK-SPIRV-DAG: 4 Decorate [[#r3]] FPFastMathMode 2
+; CHECK-SPIRV-DAG: 4 Decorate [[#r4]] FPFastMathMode 4
+; CHECK-SPIRV-DAG: 4 Decorate [[#r5]] FPFastMathMode 8
+; CHECK-SPIRV-DAG: 4 Decorate [[#r6]] FPFastMathMode 16
+; CHECK-SPIRV-DAG: 4 Decorate [[#r7]] FPFastMathMode 3
+; CHECK-SPIRV: 3 TypeFloat [[float:[0-9]+]] 32
+; CHECK-SPIRV: 5 FDiv [[float]] [[#r1]]
+; CHECK-SPIRV: 5 FDiv [[float]] [[#r2]]
+; CHECK-SPIRV: 5 FDiv [[float]] [[#r3]]
+; CHECK-SPIRV: 5 FDiv [[float]] [[#r4]]
+; CHECK-SPIRV: 5 FDiv [[float]] [[#r5]]
+; CHECK-SPIRV: 5 FDiv [[float]] [[#r6]]
+; CHECK-SPIRV: 5 FDiv [[float]] [[#r7]]
+
+; CHECK-LLVM: %r1 = fdiv float %a, %b
+; CHECK-LLVM: %r2 = fdiv nnan float %a, %b
+; CHECK-LLVM: %r3 = fdiv ninf float %a, %b
+; CHECK-LLVM: %r4 = fdiv nsz float %a, %b
+; CHECK-LLVM: %r5 = fdiv arcp float %a, %b
+; CHECK-LLVM: %r6 = fdiv fast float %a, %b
+; CHECK-LLVM: %r7 = fdiv nnan ninf float %a, %b
+
+target datalayout = "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024"
+target triple = "spir-unknown-unknown"
+
+; Function Attrs: nounwind
+define spir_kernel void @testFDiv(float %a, float %b) local_unnamed_addr #0 !kernel_arg_addr_space !2 !kernel_arg_access_qual !3 !kernel_arg_type !4 !kernel_arg_base_type !4 !kernel_arg_type_qual !5 {
+entry:
+  %r1 = fdiv float %a, %b
+  %r2 = fdiv nnan float %a, %b
+  %r3 = fdiv ninf float %a, %b
+  %r4 = fdiv nsz float %a, %b
+  %r5 = fdiv arcp float %a, %b
+  %r6 = fdiv fast float %a, %b
+  %r7 = fdiv nnan ninf float %a, %b
+  ret void
+}
+
+attributes #0 = { convergent nounwind writeonly "correctly-rounded-divide-sqrt-fp-math"="false" "denorms-are-zero"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "uniform-work-group-size"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.module.flags = !{!0}
+!opencl.ocl.version = !{!1}
+!opencl.spir.version = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 2, i32 0}
+!2 = !{i32 0, i32 0}
+!3 = !{!"none", !"none"}
+!4 = !{!"float", !"float"}
+!5 = !{!"", !""}
diff --git a/llvm-spirv/test/fmod.ll b/llvm-spirv/test/transcoding/fmod.ll
similarity index 100%
rename from llvm-spirv/test/fmod.ll
rename to llvm-spirv/test/transcoding/fmod.ll
diff --git a/llvm-spirv/test/transcoding/fmul.ll b/llvm-spirv/test/transcoding/fmul.ll
new file mode 100644
index 0000000000000..04203a3aacf2d
--- /dev/null
+++ b/llvm-spirv/test/transcoding/fmul.ll
@@ -0,0 +1,65 @@
+; RUN: llvm-as %s -o %t.bc
+; RUN: llvm-spirv -spirv-text %t.bc -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: llvm-spirv %t.bc -o %t.spv
+; RUN: spirv-val %t.spv
+; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o - | FileCheck %s --check-prefix=CHECK-LLVM
+
+; CHECK-SPIRV: 3 Name [[#r1:]] "r1"
+; CHECK-SPIRV: 3 Name [[#r2:]] "r2"
+; CHECK-SPIRV: 3 Name [[#r3:]] "r3"
+; CHECK-SPIRV: 3 Name [[#r4:]] "r4"
+; CHECK-SPIRV: 3 Name [[#r5:]] "r5"
+; CHECK-SPIRV: 3 Name [[#r6:]] "r6"
+; CHECK-SPIRV: 3 Name [[#r7:]] "r7"
+; CHECK-SPIRV-NOT: 4 Decorate [[#r1]] FPFastMathMode
+; CHECK-SPIRV-DAG: 4 Decorate [[#r2]] FPFastMathMode 1
+; CHECK-SPIRV-DAG: 4 Decorate [[#r3]] FPFastMathMode 2
+; CHECK-SPIRV-DAG: 4 Decorate [[#r4]] FPFastMathMode 4
+; CHECK-SPIRV-DAG: 4 Decorate [[#r5]] FPFastMathMode 8
+; CHECK-SPIRV-DAG: 4 Decorate [[#r6]] FPFastMathMode 16
+; CHECK-SPIRV-DAG: 4 Decorate [[#r7]] FPFastMathMode 3
+; CHECK-SPIRV: 3 TypeFloat [[float:[0-9]+]] 32
+; CHECK-SPIRV: 5 FMul [[float]] [[#r1]]
+; CHECK-SPIRV: 5 FMul [[float]] [[#r2]]
+; CHECK-SPIRV: 5 FMul [[float]] [[#r3]]
+; CHECK-SPIRV: 5 FMul [[float]] [[#r4]]
+; CHECK-SPIRV: 5 FMul [[float]] [[#r5]]
+; CHECK-SPIRV: 5 FMul [[float]] [[#r6]]
+; CHECK-SPIRV: 5 FMul [[float]] [[#r7]]
+
+; CHECK-LLVM: %r1 = fmul float %a, %b
+; CHECK-LLVM: %r2 = fmul nnan float %a, %b
+; CHECK-LLVM: %r3 = fmul ninf float %a, %b
+; CHECK-LLVM: %r4 = fmul nsz float %a, %b
+; CHECK-LLVM: %r5 = fmul arcp float %a, %b
+; CHECK-LLVM: %r6 = fmul fast float %a, %b
+; CHECK-LLVM: %r7 = fmul nnan ninf float %a, %b
+
+target datalayout = "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024"
+target triple = "spir-unknown-unknown"
+
+; Function Attrs: nounwind
+define spir_kernel void @testFMul(float %a, float %b) local_unnamed_addr #0 !kernel_arg_addr_space !2 !kernel_arg_access_qual !3 !kernel_arg_type !4 !kernel_arg_base_type !4 !kernel_arg_type_qual !5 {
+entry:
+  %r1 = fmul float %a, %b
+  %r2 = fmul nnan float %a, %b
+  %r3 = fmul ninf float %a, %b
+  %r4 = fmul nsz float %a, %b
+  %r5 = fmul arcp float %a, %b
+  %r6 = fmul fast float %a, %b
+  %r7 = fmul nnan ninf float %a, %b
+  ret void
+}
+
+attributes #0 = { convergent nounwind writeonly "correctly-rounded-divide-sqrt-fp-math"="false" "denorms-are-zero"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "uniform-work-group-size"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.module.flags = !{!0}
+!opencl.ocl.version = !{!1}
+!opencl.spir.version = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 2, i32 0}
+!2 = !{i32 0, i32 0}
+!3 = !{!"none", !"none"}
+!4 = !{!"float", !"float"}
+!5 = !{!"", !""}
diff --git a/llvm-spirv/test/transcoding/frem.ll b/llvm-spirv/test/transcoding/frem.ll
new file mode 100644
index 0000000000000..100d8bc8984bd
--- /dev/null
+++ b/llvm-spirv/test/transcoding/frem.ll
@@ -0,0 +1,65 @@
+; RUN: llvm-as %s -o %t.bc
+; RUN: llvm-spirv -spirv-text %t.bc -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: llvm-spirv %t.bc -o %t.spv
+; RUN: spirv-val %t.spv
+; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o - | FileCheck %s --check-prefix=CHECK-LLVM
+
+; CHECK-SPIRV: 3 Name [[#r1:]] "r1"
+; CHECK-SPIRV: 3 Name [[#r2:]] "r2"
+; CHECK-SPIRV: 3 Name [[#r3:]] "r3"
+; CHECK-SPIRV: 3 Name [[#r4:]] "r4"
+; CHECK-SPIRV: 3 Name [[#r5:]] "r5"
+; CHECK-SPIRV: 3 Name [[#r6:]] "r6"
+; CHECK-SPIRV: 3 Name [[#r7:]] "r7"
+; CHECK-SPIRV-NOT: 4 Decorate [[#r1]] FPFastMathMode
+; CHECK-SPIRV-DAG: 4 Decorate [[#r2]] FPFastMathMode 1
+; CHECK-SPIRV-DAG: 4 Decorate [[#r3]] FPFastMathMode 2
+; CHECK-SPIRV-DAG: 4 Decorate [[#r4]] FPFastMathMode 4
+; CHECK-SPIRV-DAG: 4 Decorate [[#r5]] FPFastMathMode 8
+; CHECK-SPIRV-DAG: 4 Decorate [[#r6]] FPFastMathMode 16
+; CHECK-SPIRV-DAG: 4 Decorate [[#r7]] FPFastMathMode 3
+; CHECK-SPIRV: 3 TypeFloat [[float:[0-9]+]] 32
+; CHECK-SPIRV: 5 FRem [[float]] [[#r1]]
+; CHECK-SPIRV: 5 FRem [[float]] [[#r2]]
+; CHECK-SPIRV: 5 FRem [[float]] [[#r3]]
+; CHECK-SPIRV: 5 FRem [[float]] [[#r4]]
+; CHECK-SPIRV: 5 FRem [[float]] [[#r5]]
+; CHECK-SPIRV: 5 FRem [[float]] [[#r6]]
+; CHECK-SPIRV: 5 FRem [[float]] [[#r7]]
+
+; CHECK-LLVM: %r1 = frem float %a, %b
+; CHECK-LLVM: %r2 = frem nnan float %a, %b
+; CHECK-LLVM: %r3 = frem ninf float %a, %b
+; CHECK-LLVM: %r4 = frem nsz float %a, %b
+; CHECK-LLVM: %r5 = frem arcp float %a, %b
+; CHECK-LLVM: %r6 = frem fast float %a, %b
+; CHECK-LLVM: %r7 = frem nnan ninf float %a, %b
+
+target datalayout = "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024"
+target triple = "spir-unknown-unknown"
+
+; Function Attrs: nounwind
+define spir_kernel void @testFRem(float %a, float %b) local_unnamed_addr #0 !kernel_arg_addr_space !2 !kernel_arg_access_qual !3 !kernel_arg_type !4 !kernel_arg_base_type !4 !kernel_arg_type_qual !5 {
+entry:
+  %r1 = frem float %a, %b
+  %r2 = frem nnan float %a, %b
+  %r3 = frem ninf float %a, %b
+  %r4 = frem nsz float %a, %b
+  %r5 = frem arcp float %a, %b
+  %r6 = frem fast float %a, %b
+  %r7 = frem nnan ninf float %a, %b
+  ret void
+}
+
+attributes #0 = { convergent nounwind writeonly "correctly-rounded-divide-sqrt-fp-math"="false" "denorms-are-zero"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "uniform-work-group-size"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.module.flags = !{!0}
+!opencl.ocl.version = !{!1}
+!opencl.spir.version = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 2, i32 0}
+!2 = !{i32 0, i32 0}
+!3 = !{!"none", !"none"}
+!4 = !{!"float", !"float"}
+!5 = !{!"", !""}
diff --git a/llvm-spirv/test/transcoding/fsub.ll b/llvm-spirv/test/transcoding/fsub.ll
new file mode 100644
index 0000000000000..8e500785d145f
--- /dev/null
+++ b/llvm-spirv/test/transcoding/fsub.ll
@@ -0,0 +1,65 @@
+; RUN: llvm-as %s -o %t.bc
+; RUN: llvm-spirv -spirv-text %t.bc -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: llvm-spirv %t.bc -o %t.spv
+; RUN: spirv-val %t.spv
+; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o - | FileCheck %s --check-prefix=CHECK-LLVM
+
+; CHECK-SPIRV: 3 Name [[#r1:]] "r1"
+; CHECK-SPIRV: 3 Name [[#r2:]] "r2"
+; CHECK-SPIRV: 3 Name [[#r3:]] "r3"
+; CHECK-SPIRV: 3 Name [[#r4:]] "r4"
+; CHECK-SPIRV: 3 Name [[#r5:]] "r5"
+; CHECK-SPIRV: 3 Name [[#r6:]] "r6"
+; CHECK-SPIRV: 3 Name [[#r7:]] "r7"
+; CHECK-SPIRV-NOT: 4 Decorate [[#r1]] FPFastMathMode
+; CHECK-SPIRV-DAG: 4 Decorate [[#r2]] FPFastMathMode 1
+; CHECK-SPIRV-DAG: 4 Decorate [[#r3]] FPFastMathMode 2
+; CHECK-SPIRV-DAG: 4 Decorate [[#r4]] FPFastMathMode 4
+; CHECK-SPIRV-DAG: 4 Decorate [[#r5]] FPFastMathMode 8
+; CHECK-SPIRV-DAG: 4 Decorate [[#r6]] FPFastMathMode 16
+; CHECK-SPIRV-DAG: 4 Decorate [[#r7]] FPFastMathMode 3
+; CHECK-SPIRV: 3 TypeFloat [[float:[0-9]+]] 32
+; CHECK-SPIRV: 5 FSub [[float]] [[#r1]]
+; CHECK-SPIRV: 5 FSub [[float]] [[#r2]]
+; CHECK-SPIRV: 5 FSub [[float]] [[#r3]]
+; CHECK-SPIRV: 5 FSub [[float]] [[#r4]]
+; CHECK-SPIRV: 5 FSub [[float]] [[#r5]]
+; CHECK-SPIRV: 5 FSub [[float]] [[#r6]]
+; CHECK-SPIRV: 5 FSub [[float]] [[#r7]]
+
+; CHECK-LLVM: %r1 = fsub float %a, %b
+; CHECK-LLVM: %r2 = fsub nnan float %a, %b
+; CHECK-LLVM: %r3 = fsub ninf float %a, %b
+; CHECK-LLVM: %r4 = fsub nsz float %a, %b
+; CHECK-LLVM: %r5 = fsub arcp float %a, %b
+; CHECK-LLVM: %r6 = fsub fast float %a, %b
+; CHECK-LLVM: %r7 = fsub nnan ninf float %a, %b
+
+target datalayout = "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024"
+target triple = "spir-unknown-unknown"
+
+; Function Attrs: nounwind
+define spir_kernel void @testFSub(float %a, float %b) local_unnamed_addr #0 !kernel_arg_addr_space !2 !kernel_arg_access_qual !3 !kernel_arg_type !4 !kernel_arg_base_type !4 !kernel_arg_type_qual !5 {
+entry:
+  %r1 = fsub float %a, %b
+  %r2 = fsub nnan float %a, %b
+  %r3 = fsub ninf float %a, %b
+  %r4 = fsub nsz float %a, %b
+  %r5 = fsub arcp float %a, %b
+  %r6 = fsub fast float %a, %b
+  %r7 = fsub nnan ninf float %a, %b
+  ret void
+}
+
+attributes #0 = { convergent nounwind writeonly "correctly-rounded-divide-sqrt-fp-math"="false" "denorms-are-zero"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "uniform-work-group-size"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.module.flags = !{!0}
+!opencl.ocl.version = !{!1}
+!opencl.spir.version = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 2, i32 0}
+!2 = !{i32 0, i32 0}
+!3 = !{!"none", !"none"}
+!4 = !{!"float", !"float"}
+!5 = !{!"", !""}
diff --git a/llvm-spirv/test/global_block.cl b/llvm-spirv/test/transcoding/global_block.cl
similarity index 100%
rename from llvm-spirv/test/global_block.cl
rename to llvm-spirv/test/transcoding/global_block.cl
diff --git a/llvm-spirv/test/inline_asm_basic.cl b/llvm-spirv/test/transcoding/inline_asm_basic.cl
similarity index 100%
rename from llvm-spirv/test/inline_asm_basic.cl
rename to llvm-spirv/test/transcoding/inline_asm_basic.cl
diff --git a/llvm-spirv/test/inline_asm_clobbers.cl b/llvm-spirv/test/transcoding/inline_asm_clobbers.cl
similarity index 100%
rename from llvm-spirv/test/inline_asm_clobbers.cl
rename to llvm-spirv/test/transcoding/inline_asm_clobbers.cl
diff --git a/llvm-spirv/test/inline_asm_constraints.cl b/llvm-spirv/test/transcoding/inline_asm_constraints.cl
similarity index 100%
rename from llvm-spirv/test/inline_asm_constraints.cl
rename to llvm-spirv/test/transcoding/inline_asm_constraints.cl
diff --git a/llvm-spirv/test/lifetime.ll b/llvm-spirv/test/transcoding/lifetime.ll
similarity index 100%
rename from llvm-spirv/test/lifetime.ll
rename to llvm-spirv/test/transcoding/lifetime.ll
diff --git a/llvm-spirv/test/llvm.memcpy.align.ll b/llvm-spirv/test/transcoding/llvm.memcpy.align.ll
similarity index 100%
rename from llvm-spirv/test/llvm.memcpy.align.ll
rename to llvm-spirv/test/transcoding/llvm.memcpy.align.ll
diff --git a/llvm-spirv/test/memory_access.ll b/llvm-spirv/test/transcoding/memory_access.ll
similarity index 100%
rename from llvm-spirv/test/memory_access.ll
rename to llvm-spirv/test/transcoding/memory_access.ll
diff --git a/llvm-spirv/test/sycl_array_zero_init.ll b/llvm-spirv/test/transcoding/sycl_array_zero_init.ll
similarity index 100%
rename from llvm-spirv/test/sycl_array_zero_init.ll
rename to llvm-spirv/test/transcoding/sycl_array_zero_init.ll
diff --git a/llvm-spirv/test/undef_initializer.ll b/llvm-spirv/test/transcoding/undef_initializer.ll
similarity index 100%
rename from llvm-spirv/test/undef_initializer.ll
rename to llvm-spirv/test/transcoding/undef_initializer.ll
diff --git a/llvm-spirv/test/unreachable.ll b/llvm-spirv/test/transcoding/unreachable.ll
similarity index 100%
rename from llvm-spirv/test/unreachable.ll
rename to llvm-spirv/test/transcoding/unreachable.ll
diff --git a/llvm-spirv/test/vec_type_hint.cl b/llvm-spirv/test/transcoding/vec_type_hint.cl
similarity index 100%
rename from llvm-spirv/test/vec_type_hint.cl
rename to llvm-spirv/test/transcoding/vec_type_hint.cl
diff --git a/llvm/cmake/modules/AddLLVM.cmake b/llvm/cmake/modules/AddLLVM.cmake
index 8cd71eef2332e..8be19b12ed4fa 100644
--- a/llvm/cmake/modules/AddLLVM.cmake
+++ b/llvm/cmake/modules/AddLLVM.cmake
@@ -1031,7 +1031,7 @@ endfunction()
 
 # Export symbols if LLVM plugins are enabled.
 function(export_executable_symbols_for_plugins target)
-  if(LLVM_ENABLE_PLUGINS)
+  if(LLVM_ENABLE_PLUGINS OR LLVM_EXPORT_SYMBOLS_FOR_PLUGINS)
     export_executable_symbols(${target})
   endif()
 endfunction()
@@ -1396,8 +1396,14 @@ endmacro()
 # variables needed for the 'lit.site.cfg' files. This function bundles the
 # common variables that any Lit instance is likely to need, and custom
 # variables can be passed in.
+# The keyword PATHS is followed by a list of cmake variable names that are
+# mentioned as `path("@varname@")` in the lit.cfg.py.in file. Variables in that
+# list are treated as paths that are relative to the directory the generated
+# lit.cfg.py file is in, and the `path()` function converts the relative
+# path back to absolute form. This makes it possible to move a build directory
+# containing lit.cfg.py files from one machine to another.
 function(configure_lit_site_cfg site_in site_out)
-  cmake_parse_arguments(ARG "" "" "MAIN_CONFIG;OUTPUT_MAPPING" ${ARGN})
+  cmake_parse_arguments(ARG "" "" "MAIN_CONFIG;OUTPUT_MAPPING;PATHS" ${ARGN})
 
   if ("${ARG_MAIN_CONFIG}" STREQUAL "")
     get_filename_component(INPUT_DIR ${site_in} DIRECTORY)
@@ -1447,12 +1453,26 @@ function(configure_lit_site_cfg site_in site_out)
   set(HOST_CXX "${CMAKE_CXX_COMPILER} ${CMAKE_CXX_COMPILER_ARG1}")
   set(HOST_LDFLAGS "${CMAKE_EXE_LINKER_FLAGS}")
 
-  set(LIT_SITE_CFG_IN_HEADER  "## Autogenerated from ${site_in}\n## Do not edit!")
+  set(LIT_SITE_CFG_IN_HEADER  "# Autogenerated from ${site_in}\n# Do not edit!")
+
+  # Lit converts config paths to lower case in discovery.py, before
+  # loading the config. This causes __file__ to be all lower-case (including
+  # the drive letter), but several clang tests pass -include %s and a
+  # clang warning checks that passed case matches on-disk cache. So it's
+  # important that this restores the on-disk case of the prefix.
+  string(CONCAT LIT_SITE_CFG_IN_HEADER "${LIT_SITE_CFG_IN_HEADER}\n\n"
+    "# Allow generated lit.site.cfg.py to be relocatable.\n"
+    "def path(p):\n"
+    "  if not p: return ''\n"
+    "  p = os.path.normpath(os.path.join(os.path.dirname(__file__), p)).replace(os.sep, '/')\n"
+    "  if os.name == 'nt' and os.path.isabs(p): return p[0].upper() + p[1:]\n"
+    "  return p\n"
+    )
 
   # Override config_target_triple (and the env)
   if(LLVM_TARGET_TRIPLE_ENV)
     # This is expanded into the heading.
-    string(CONCAT LIT_SITE_CFG_IN_HEADER "${LIT_SITE_CFG_IN_HEADER}\n\n"
+    string(CONCAT LIT_SITE_CFG_IN_HEADER "${LIT_SITE_CFG_IN_HEADER}"
       "import os\n"
       "target_env = \"${LLVM_TARGET_TRIPLE_ENV}\"\n"
       "config.target_triple = config.environment[target_env] = os.environ.get(target_env, \"${TARGET_TRIPLE}\")\n"
@@ -1462,7 +1482,51 @@ function(configure_lit_site_cfg site_in site_out)
     set(TARGET_TRIPLE "\"+config.target_triple+\"")
   endif()
 
+  if (ARG_PATHS)
+    # Walk ARG_PATHS and collect the current value of the variables in there.
+    foreach(path ${ARG_PATHS})
+      list(APPEND ARG_PATH_VALUES "${${path}}")
+    endforeach()
+
+    # Compute paths relative to the directory containing output lit.site.cfg.py.
+    # Passing ARG_PATH_VALUES as-is to execute_process() makes cmake strip
+    # empty list entries. So escape the ;s in the list and do the splitting
+    # outselves. cmake has no relpath function, so use Python for that.
+    string(REPLACE ";" "\\;" ARG_PATH_VALUES_ESCAPED "${ARG_PATH_VALUES}")
+    get_filename_component(OUTPUT_DIR ${site_out} DIRECTORY)
+    execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c" "\n
+import os, sys\n
+drive = os.path.splitdrive(sys.argv[1])[0]\n
+def relpath(p):\n
+    if not p: return ''\n
+    if os.path.splitdrive(p)[0] != drive: return p\n
+    return os.path.relpath(p, sys.argv[1]).replace(os.sep, '/')\n
+sys.stdout.write(';'.join(relpath(p) for p in sys.argv[2].split(';')))"
+      ${OUTPUT_DIR}
+      ${ARG_PATH_VALUES_ESCAPED}
+      OUTPUT_VARIABLE ARG_PATH_VALUES_RELATIVE)
+
+    list(LENGTH ARG_PATHS len_paths)
+    list(LENGTH ARG_PATH_VALUES len_path_values)
+    list(LENGTH ARG_PATH_VALUES_RELATIVE len_path_value_rels)
+    if ((NOT ${len_paths} EQUAL ${len_path_values}) OR
+        (NOT ${len_paths} EQUAL ${len_path_value_rels}))
+      message(SEND_ERROR "PATHS lengths got confused")
+    endif()
+
+    # Transform variables mentioned in ARG_PATHS to relative paths for
+    # the configure_file() call. Variables are copied to subscopeds by cmake,
+    # so this only modifies the local copy of the variables.
+    math(EXPR arg_path_limit "${len_paths} - 1")
+    foreach(i RANGE ${arg_path_limit})
+      list(GET ARG_PATHS ${i} val1)
+      list(GET ARG_PATH_VALUES_RELATIVE ${i} val2)
+      set(${val1} ${val2})
+    endforeach()
+  endif()
+
   configure_file(${site_in} ${site_out} @ONLY)
+
   if (EXISTS "${ARG_MAIN_CONFIG}")
     set(PYTHON_STATEMENT "map_config('${ARG_MAIN_CONFIG}', '${site_out}')")
     get_property(LLVM_LIT_CONFIG_MAP GLOBAL PROPERTY LLVM_LIT_CONFIG_MAP)
diff --git a/llvm/cmake/modules/LLVMExternalProjectUtils.cmake b/llvm/cmake/modules/LLVMExternalProjectUtils.cmake
index 29f4fcbf8aba6..4eb5fad85634b 100644
--- a/llvm/cmake/modules/LLVMExternalProjectUtils.cmake
+++ b/llvm/cmake/modules/LLVMExternalProjectUtils.cmake
@@ -62,7 +62,17 @@ function(llvm_ExternalProject_Add name source_dir)
   foreach(tool ${ARG_TOOLCHAIN_TOOLS})
     if(TARGET ${tool})
       list(APPEND TOOLCHAIN_TOOLS ${tool})
-      list(APPEND TOOLCHAIN_BINS $<TARGET_FILE:${tool}>)
+
+      # $<TARGET_FILE:tgt> only works on add_executable or add_library targets
+      # The below logic mirrors cmake's own implementation
+      get_target_property(target_type "${tool}" TYPE)
+      if(NOT target_type STREQUAL "OBJECT_LIBRARY" AND
+         NOT target_type STREQUAL "UTILITY" AND
+         NOT target_type STREQUAL "GLOBAL_TARGET" AND
+         NOT target_type STREQUAL "INTERFACE_LIBRARY")
+        list(APPEND TOOLCHAIN_BINS $<TARGET_FILE:${tool}>)
+      endif()
+
     endif()
   endforeach()
 
diff --git a/llvm/cmake/modules/TableGen.cmake b/llvm/cmake/modules/TableGen.cmake
index 632f69aa33865..65e31d0624f08 100644
--- a/llvm/cmake/modules/TableGen.cmake
+++ b/llvm/cmake/modules/TableGen.cmake
@@ -2,10 +2,6 @@
 # Extra parameters for `tblgen' may come after `ofn' parameter.
 # Adds the name of the generated file to TABLEGEN_OUTPUT.
 
-if(LLVM_MAIN_INCLUDE_DIR)
-  set(LLVM_TABLEGEN_FLAGS -I ${LLVM_MAIN_INCLUDE_DIR})
-endif()
-
 function(tablegen project ofn)
   # Validate calling context.
   if(NOT ${project}_TABLEGEN_EXE)
@@ -75,6 +71,14 @@ function(tablegen project ofn)
     set(tblgen_change_flag "--write-if-changed")
   endif()
 
+  # With CMake 3.12 this can be reduced to:
+  # get_directory_property(tblgen_includes "INCLUDE_DIRECTORIES")
+  # list(TRANSFORM tblgen_includes PREPEND -I)
+  set(tblgen_includes)
+  get_directory_property(includes "INCLUDE_DIRECTORIES")
+  foreach(include ${includes})
+    list(APPEND tblgen_includes -I ${include})
+  endforeach()
   # We need both _TABLEGEN_TARGET and _TABLEGEN_EXE in the  DEPENDS list
   # (both the target and the file) to have .inc files rebuilt on
   # a tablegen change, as cmake does not propagate file-level dependencies
@@ -86,6 +90,7 @@ function(tablegen project ofn)
   # but lets us having smaller and cleaner code here.
   add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${ofn}
     COMMAND ${${project}_TABLEGEN_EXE} ${ARGN} -I ${CMAKE_CURRENT_SOURCE_DIR}
+    ${tblgen_includes}
     ${LLVM_TABLEGEN_FLAGS}
     ${LLVM_TARGET_DEFINITIONS_ABSOLUTE}
     ${tblgen_change_flag}
diff --git a/llvm/docs/CodingStandards.rst b/llvm/docs/CodingStandards.rst
index 0e9bd8535b76c..15483e5d27988 100644
--- a/llvm/docs/CodingStandards.rst
+++ b/llvm/docs/CodingStandards.rst
@@ -320,6 +320,46 @@ Preferred:
   /// Builds a B-tree in order to do foo.  See paper by...
   void example() { ... }
 
+Error and Warning Messages
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Clear diagnostic messages are important to help users identify and fix issues in
+their inputs. Use succinct but correct English prose that gives the user the
+context needed to understand what went wrong. Also, to match error message
+styles commonly produced by other tools, start the first sentence with a
+lower-case letter, and finish the last sentence without a period, if it would
+end in one otherwise. Sentences which end with different punctuation, such as
+"did you forget ';'?", should still do so.
+
+For example this is a good error message:
+
+.. code-block:: none
+
+  error: file.o: section header 3 is corrupt. Size is 10 when it should be 20
+
+This is a bad message, since it does not provide useful information and uses the
+wrong style:
+
+.. code-block:: none
+
+  error: file.o: Corrupt section header.
+
+As with other coding standards, individual projects, such as the Clang Static
+Analyzer, may have preexisting styles that do not conform to this. If a
+different formatting scheme is used consistently throughout the project, use
+that style instead. Otherwise, this standard applies to all LLVM tools,
+including clang, clang-tidy, and so on.
+
+If the tool or project does not have existing functions to emit warnings or
+errors, use the error and warning handlers provided in ``Support/WithColor.h``
+to ensure they are printed in the appropriate style, rather than printing to
+stderr directly.
+
+When using ``report_fatal_error``, follow the same standards for the message as
+regular error messages. Assertion messages and ``llvm_unreachable`` calls do not
+necessarily need to follow these same styles as they are automatically
+formatted, and thus these guidelines may not be suitable.
+
 ``#include`` Style
 ^^^^^^^^^^^^^^^^^^
 
@@ -1153,6 +1193,13 @@ builds), ``llvm_unreachable`` becomes a hint to compilers to skip generating
 code for this branch. If the compiler does not support this, it will fall back
 to the "abort" implementation.
 
+Use ``llvm_unreachable`` to mark a specific point in code that should never be
+reached. This is especially desirable for addressing warnings about unreachable
+branches, etc., but can be used whenever reaching a particular code path is
+unconditionally a bug (not originating from user input; see below) of some kind.
+Use of ``assert`` should always include a testable predicate (as opposed to
+``assert(false)``).
+
 Neither assertions or ``llvm_unreachable`` will abort the program on a release
 build. If the error condition can be triggered by user input then the
 recoverable error mechanism described in :doc:`ProgrammersManual` should be
diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index 63518fb20adcb..ebb89d2f853dd 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -410,11 +410,12 @@ be used to define subdirectories of optional tests, or to change other
 configuration parameters --- for example, to change the test format, or the
 suffixes which identify test files.
 
-PRE-DEFINED SUBSTITUTIONS
-~~~~~~~~~~~~~~~~~~~~~~~~~~
+SUBSTITUTIONS
+~~~~~~~~~~~~~
 
-:program:`lit` provides various patterns that can be used with the RUN command.
-These are defined in TestRunner.py. The base set of substitutions are:
+:program:`lit` allows patterns to be substituted inside RUN commands. It also
+provides the following base set of substitutions, which are defined in
+TestRunner.py:
 
  ======================= ==============
   Macro                   Substitution
@@ -453,6 +454,14 @@ Other substitutions are provided that are variations on this base set and
 further substitution patterns can be defined by each test module. See the
 modules :ref:`local-configuration-files`.
 
+By default, substitutions are expanded exactly once, so that if e.g. a
+substitution ``%build`` is defined in top of another substitution ``%cxx``,
+``%build`` will expand to ``%cxx`` textually, not to what ``%cxx`` expands to.
+However, if the ``recursiveExpansionLimit`` property of the ``LitConfig`` is
+set to a non-negative integer, substitutions will be expanded recursively until
+that limit is reached. It is an error if the limit is reached and expanding
+substitutions again would yield a different result.
+
 More detailed information on substitutions can be found in the
 :doc:`../TestingGuide`.
 
diff --git a/llvm/docs/CommandGuide/llvm-addr2line.rst b/llvm/docs/CommandGuide/llvm-addr2line.rst
index 4f5881dc8e897..27b0fd9a947bc 100644
--- a/llvm/docs/CommandGuide/llvm-addr2line.rst
+++ b/llvm/docs/CommandGuide/llvm-addr2line.rst
@@ -17,17 +17,19 @@ GNU's :program:`addr2line`.
 
 Here are some of those differences:
 
--  Defaults not to print function names. Use `-f`_ to enable that.
+-  ``llvm-addr2line`` defaults not to print function names. Use `-f`_ to enable
+   that.
 
--  Defaults not to demangle function names. Use `-C`_ to switch the
-   demangling on.
+-  ``llvm-addr2line`` defaults not to demangle function names. Use `-C`_ to
+   switch the demangling on.
 
--  Defaults not to print inlined frames. Use `-i`_ to show inlined
-   frames for a source code location in an inlined function.
+-  ``llvm-addr2line`` defaults not to print inlined frames. Use `-i`_ to show
+   inlined frames for a source code location in an inlined function.
 
--  Uses `--output-style=GNU`_ by default.
+-  ``llvm-addr2line`` uses `--output-style=GNU`_ by default.
 
--  Parses options from the environment variable ``LLVM_ADDR2LINE_OPTS``.
+-  ``llvm-addr2line`` parses options from the environment variable
+   ``LLVM_ADDR2LINE_OPTS`` instead of from ``LLVM_SYMBOLIZER_OPTS``.
 
 SEE ALSO
 --------
diff --git a/llvm/docs/CommandGuide/llvm-dwarfdump.rst b/llvm/docs/CommandGuide/llvm-dwarfdump.rst
index b8de0847aaaed..840c93345d296 100644
--- a/llvm/docs/CommandGuide/llvm-dwarfdump.rst
+++ b/llvm/docs/CommandGuide/llvm-dwarfdump.rst
@@ -105,6 +105,10 @@ OPTIONS
             When displaying debug info entries, only show children to a maximum
             depth of <N>.
 
+.. option:: --show-section-sizes
+
+            Show the sizes of all debug sections, expressed in bytes.
+
 .. option:: --statistics
 
             Collect debug info quality metrics and print the results
diff --git a/llvm/docs/CommandGuide/llvm-exegesis.rst b/llvm/docs/CommandGuide/llvm-exegesis.rst
index 31be33cc861f6..321cdf5a6dab1 100644
--- a/llvm/docs/CommandGuide/llvm-exegesis.rst
+++ b/llvm/docs/CommandGuide/llvm-exegesis.rst
@@ -196,14 +196,16 @@ OPTIONS
  to specify at least one of the `-analysis-clusters-output-file=` and
  `-analysis-inconsistencies-output-file=`.
 
-.. option:: -repetition-mode=[duplicate|loop]
+.. option:: -repetition-mode=[duplicate|loop|min]
 
  Specify the repetition mode. `duplicate` will create a large, straight line
  basic block with `num-repetitions` copies of the snippet. `loop` will wrap
  the snippet in a loop which will be run `num-repetitions` times. The `loop`
  mode tends to better hide the effects of the CPU frontend on architectures
  that cache decoded instructions, but consumes a register for counting
- iterations.
+ iterations. If performing an analysis over many opcodes, it may be best
+ to instead use the `min` mode, which will run each other mode, and produce
+ the minimal measured result.
 
 .. option:: -num-repetitions=<Number of repetitions>
 
diff --git a/llvm/docs/CommandGuide/llvm-symbolizer.rst b/llvm/docs/CommandGuide/llvm-symbolizer.rst
index dd93efd03cfcd..820b15061715f 100644
--- a/llvm/docs/CommandGuide/llvm-symbolizer.rst
+++ b/llvm/docs/CommandGuide/llvm-symbolizer.rst
@@ -147,6 +147,27 @@ Example 4 - CODE and DATA prefixes:
   bar
   6295592 4
 
+Example 5 - path-style options:
+
+This example uses the same source file as above, but the source file's
+full path is /tmp/foo/test.cpp and is compiled as follows. The first case
+shows the default absolute path, the second --basenames, and the third
+shows --relativenames.
+
+.. code-block:: console
+  $ pwd
+  /tmp
+  $ clang -g foo/test.cpp -o test.elf
+  $ llvm-symbolizer --obj=test.elf 0x4004a0 
+  main
+  /tmp/foo/test.cpp:15:0
+  $ llvm-symbolizer --obj=test.elf 0x4004a0 --basenames
+  main
+  test.cpp:15:0
+  $ llvm-symbolizer --obj=test.elf 0x4004a0 --relativenames
+  main
+  foo/test.cpp:15:0
+   
 OPTIONS
 -------
 
@@ -158,8 +179,15 @@ OPTIONS
 
 .. option:: --basenames, -s
 
-  Strip directories when printing the file path.
+  Print just the file's name without any directories, instead of the
+  absolute path.
 
+.. option:: --relativenames
+
+  Print the file's path relative to the compilation directory, instead
+  of the absolute path. If the command-line to the compiler included
+  the full path, this will be the same as the default.
+  
 .. _llvm-symbolizer-opt-C:
 
 .. option:: --demangle, -C
diff --git a/llvm/docs/DeveloperPolicy.rst b/llvm/docs/DeveloperPolicy.rst
index 0b3051a4393b1..22f29a06b2893 100644
--- a/llvm/docs/DeveloperPolicy.rst
+++ b/llvm/docs/DeveloperPolicy.rst
@@ -89,7 +89,9 @@ to read it as possible.  As such, we recommend that you:
    patches may not apply correctly if the underlying code changes between the
    time the patch was created and the time it is applied.
 
-#. Patches should be made with ``git format-patch``, or similar. If you use a
+#. Patches should be made with ``git format-patch``, or similar (see special
+   commands for `Requesting Phabricator review via the web interface
+   <Phabricator.html#phabricator-request-review-web>`_ ). If you use a
    different tool, make sure it uses the ``diff -u`` format and that it
    doesn't contain clutter which makes it hard to read.
 
diff --git a/llvm/docs/GettingStarted.rst b/llvm/docs/GettingStarted.rst
index 0ede510edb3a2..b21c125b871c7 100644
--- a/llvm/docs/GettingStarted.rst
+++ b/llvm/docs/GettingStarted.rst
@@ -447,8 +447,8 @@ either via emailing to llvm-commits, or, preferably, via :ref:`Phabricator
 You'll generally want to make sure your branch has a single commit,
 corresponding to the review you wish to send, up-to-date with the upstream
 ``origin/master`` branch, and doesn't contain merges. Once you have that, you
-can use ``git show`` or ``git format-patch`` to output the diff, and attach it
-to a Phabricator review (or to an email message).
+can start `a Phabricator review <Phabricator.html>`_ (or use ``git show`` or
+``git format-patch`` to output the diff, and attach it to an email message).
 
 However, using the "Arcanist" tool is often easier. After `installing
 arcanist`_, you can upload the latest commit using:
@@ -1095,6 +1095,61 @@ If you are having problems building or using LLVM, or if you have any other
 general questions about LLVM, please consult the `Frequently Asked
 Questions <FAQ.html>`_ page.
 
+If you are having problems with limited memory and build time, please try
+building with ninja instead of make. Please consider configuring the
+following options with cmake:
+
+ * -G Ninja
+   Setting this option will allow you to build with ninja instead of make.
+   Building with ninja significantly improves your build time, especially with
+   incremental builds, and improves your memory usage.
+
+ * -DLLVM_USE_LINKER
+   Setting this option to either gold or lld will significantly improve
+   performance. In the case that you are compiling lld, you may wish to use the
+   gold linker as a faster alternative.
+
+ * -DCMAKE_BUILD_TYPE
+   This option defaults to Debug; however, this may consume more memory during
+   the linking phase. So, you may wish to use the build type Release. Another
+   build type you may wish to consider is release-with-asserts which compiles at
+   nearly the same rate as the Release build; however, it may not be as easy
+   to debug. MinSizeRel is another build type you may wish to consider, if you
+   are still having problems with slow build time.
+
+ * -DLLVM_PARALLEL_LINK_JOBS
+   Set this equal to number of jobs you wish to run simultaneously. This is
+   similar to the -j option used with make, but only for link jobs. This option
+   is of course only meaningful if you plan to build with ninja. You may wish to
+   use a very low number of jobs, as this will greatly reduce the amount memory
+   used during the build process. If you have limited memory, you may wish to
+   set this to 1.
+
+ * -DLLVM_TARGETS_TO_BUILD
+   Set this equal to the target you wish to build. You may wish to set this to
+   X86; however, you will find a full list of targets within the
+   llvm-project/llvm/lib/Target directory.
+
+ * -DLLVM_OPTIMIZED_TABLEGEN
+   Set this to ON to generate a fully optimized tablegen during build. This will
+   significantly improve your build time.
+
+ * -DLLVM_ENABLE_PROJECTS
+   Set this equal to the projects you wish to compile (e.g. clang, lld, etc.) If
+   compiling more than one project, deliniate the list with a semicolon. Should
+   you run into issues with the semicolon, try surrounding it with single quotes.
+
+ * -DCLANG_ENABLE_STATIC_ANALYZER
+   Set this option to OFF if you do not require the clang static analyzer. This
+   should improve your build time significantly.
+
+ * -DLLVM_USE_SPLIT_DWARF
+   Consider setting this to ON if you require a debug build, as this will ease
+   memory pressure on the linker. This will make linking much faster, as the
+   binaries will not contain any of the debug information; however, this will
+   generate the debug information in the form of a DWARF object file (with the
+   extension .dwo).
+
 .. _links:
 
 Links
diff --git a/llvm/docs/GlobalISel/GenericOpcode.rst b/llvm/docs/GlobalISel/GenericOpcode.rst
index 3110e1fa8398c..7c418a0c05c17 100644
--- a/llvm/docs/GlobalISel/GenericOpcode.rst
+++ b/llvm/docs/GlobalISel/GenericOpcode.rst
@@ -672,12 +672,9 @@ Other Operations
 G_DYN_STACKALLOC
 ^^^^^^^^^^^^^^^^
 
-Dynamically realign the stack pointer to the specified alignment
+Dynamically realigns the stack pointer to the specified size and alignment.
+An alignment value of `0` or `1` mean no specific alignment.
 
 .. code-block:: none
 
   %8:_(p0) = G_DYN_STACKALLOC %7(s64), 32
-
-.. caution::
-
-  What does it mean for the immediate to be 0? It happens in the tests
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 4f3640443c4ba..f6c0a66f4407d 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -1107,14 +1107,16 @@ Currently, only the following parameter attributes are defined:
 .. _noalias:
 
 ``noalias``
-    This indicates that objects accessed via pointer values
+    This indicates that memory locations accessed via pointer values
     :ref:`based <pointeraliasing>` on the argument or return value are not also
     accessed, during the execution of the function, via pointer values not
-    *based* on the argument or return value. The attribute on a return value
-    also has additional semantics described below. The caller shares the
-    responsibility with the callee for ensuring that these requirements are met.
-    For further details, please see the discussion of the NoAlias response in
-    :ref:`alias analysis <Must, May, or No>`.
+    *based* on the argument or return value. This guarantee only holds for
+    memory locations that are *modified*, by any means, during the execution of
+    the function. The attribute on a return value also has additional semantics
+    described below. The caller shares the responsibility with the callee for
+    ensuring that these requirements are met.  For further details, please see
+    the discussion of the NoAlias response in :ref:`alias analysis <Must, May,
+    or No>`.
 
     Note that this definition of ``noalias`` is intentionally similar
     to the definition of ``restrict`` in C99 for function arguments.
@@ -3416,6 +3418,7 @@ uses with" concept would not hold.
 
 To ensure all uses of a given register observe the same value (even if
 '``undef``'), the :ref:`freeze instruction <i_freeze>` can be used.
+A value is frozen if its uses see the same value.
 
 .. code-block:: llvm
 
@@ -3452,7 +3455,34 @@ match what was already there. However, a store *to* an undefined
 location could clobber arbitrary memory, therefore, it has undefined
 behavior.
 
-**MemorySanitizer**, a detector of uses of uninitialized memory,
+Branching on an undefined value is undefined behavior.
+This explains optimizations that depend on branch conditions to construct
+predicates, such as Correlated Value Propagation and Global Value Numbering.
+In case of switch instruction, the branch condition should be frozen, otherwise
+it is undefined behavior.
+
+.. code-block:: text
+
+    Unsafe:
+      br undef, BB1, BB2 ; UB
+
+      %X = and i32 undef, 255
+      switch %X, label %ret [ .. ] ; UB
+
+      store undef, i8* %ptr
+      %X = load i8* %ptr ; %X is undef
+      switch i8 %X, label %ret [ .. ] ; UB
+
+    Safe:
+      %X = or i8 undef, 255 ; always 255
+      switch i8 %X, label %ret [ .. ] ; Well-defined
+
+      %X = freeze i1 undef
+      br %X, BB1, BB2 ; Well-defined (non-deterministic jump)
+
+
+This is also consistent with the behavior of MemorySanitizer.
+MemorySanitizer, detector of uses of uninitialized memory,
 defines a branch with condition that depends on an undef value (or
 certain other values, like e.g. a result of a load from heap-allocated
 memory that has never been stored to) to have an externally visible
@@ -7035,7 +7065,8 @@ Upon execution of a conditional '``br``' instruction, the '``i1``'
 argument is evaluated. If the value is ``true``, control flows to the
 '``iftrue``' ``label`` argument. If "cond" is ``false``, control flows
 to the '``iffalse``' ``label`` argument.
-If '``cond``' is ``poison``, this instruction has undefined behavior.
+If '``cond``' is ``poison`` or ``undef``, this instruction has undefined
+behavior.
 
 Example:
 """"""""
@@ -7086,7 +7117,8 @@ When the '``switch``' instruction is executed, this table is searched
 for the given value. If the value is found, control flow is transferred
 to the corresponding destination; otherwise, control flow is transferred
 to the default destination.
-If '``value``' is ``poison``, this instruction has undefined behavior.
+If '``value``' is ``poison`` or ``undef``, this instruction has undefined
+behavior.
 
 Implementation:
 """""""""""""""
@@ -7151,7 +7183,8 @@ Control transfers to the block specified in the address argument. All
 possible destination blocks must be listed in the label list, otherwise
 this instruction has undefined behavior. This implies that jumps to
 labels defined in other functions have undefined behavior as well.
-If '``address``' is ``poison``, this instruction has undefined behavior.
+If '``address``' is ``poison`` or ``undef``, this instruction has undefined
+behavior.
 
 Implementation:
 """""""""""""""
diff --git a/llvm/docs/LoopTerminology.rst b/llvm/docs/LoopTerminology.rst
index bb2b40a098eff..72ff024321a02 100644
--- a/llvm/docs/LoopTerminology.rst
+++ b/llvm/docs/LoopTerminology.rst
@@ -55,46 +55,46 @@ is branched to).  They may have multiple predecessors, phis, etc...
 Key Terminology
 ===============
 
-Header Block - The basic block which dominates all other blocks
+**Header Block** - The basic block which dominates all other blocks
 contained within the loop.  As such, it is the first one executed if
 the loop executes at all.  Note that a block can be the header of
 two separate loops at the same time, but only if one is a sub-loop
 of the other.
 
-Exiting Block - A basic block contained within a given loop which has
+**Exiting Block** - A basic block contained within a given loop which has
 at least one successor outside of the loop and one successor inside the
 loop.  (The latter is a consequence of the block being contained within
 an SCC which is part of the loop.)  That is, it has a successor which
 is an Exit Block.  
 
-Exit Block - A basic block outside of the associated loop which has a
+**Exit Block** - A basic block outside of the associated loop which has a
 predecessor inside the loop.  That is, it has a predecessor which is
 an Exiting Block.
 
-Latch Block - A basic block within the loop whose successors include
+**Latch Block** - A basic block within the loop whose successors include
 the header block of the loop.  Thus, a latch is a source of backedge.
 A loop may have multiple latch blocks.  A latch block may be either
 conditional or unconditional.
 
-Backedge(s) - The edge(s) in the CFG from latch blocks to the header
+**Backedge(s)** - The edge(s) in the CFG from latch blocks to the header
 block.  Note that there can be multiple such edges, and even multiple
 such edges leaving a single latch block.  
 
-Loop Predecessor -  The predecessor blocks of the loop header which
+**Loop Predecessor** -  The predecessor blocks of the loop header which
 are not contained by the loop itself.  These are the only blocks
 through which execution can enter the loop.  When used in the
 singular form implies that there is only one such unique block. 
 
-Preheader Block - A preheader is a (singular) loop predecessor which
+**Preheader Block** - A preheader is a (singular) loop predecessor which
 ends in an unconditional transfer of control to the loop header.  Note
 that not all loops have such blocks.
 
-Backedge Taken Count - The number of times the backedge will execute
+**Backedge Taken Count** - The number of times the backedge will execute
 before some interesting event happens.  Commonly used without
 qualification of the event as a shorthand for when some exiting block
 branches to some exit block. May be zero, or not statically computable.
 
-Iteration Count - The number of times the header will execute before
+**Iteration Count** - The number of times the header will execute before
 some interesting event happens.  Commonly used without qualification to
 refer to the iteration count at which the loop exits.  Will always be
 one greater than the backedge taken count.  *Warning*: Preceding
@@ -161,11 +161,164 @@ When it is successful, the loop has:
   has a predecessor that is outside the loop. This implies
   that all exit blocks are dominated by the loop header.
 
+.. _loop-terminology-lcssa:
 
 Loop Closed SSA (LCSSA)
 =======================
 
-TBD
+A program is in Loop Closed SSA Form if it is in SSA form
+and all values that are defined in a loop are used only inside
+this loop.
+Programs written in LLVM IR are always in SSA form but not necessarily
+in LCSSA. To achieve the latter, single entry PHI nodes are inserted
+at the end of the loops for all values that are live
+across the loop boundary [#lcssa-construction]_.
+In particular, consider the following loop:
+
+.. code-block:: C
+
+    c = ...;
+    for (...) {
+      if (c)
+        X1 = ...
+      else
+        X2 = ...
+      X3 = phi(X1, X2);  // X3 defined
+    }
+
+    ... = X3 + 4;  // X3 used, i.e. live
+                   // outside the loop
+
+In the inner loop, the X3 is defined inside the loop, but used
+outside of it. In Loop Closed SSA form, this would be represented as follows:
+
+.. code-block:: C
+
+    c = ...;
+    for (...) {
+      if (c)
+        X1 = ...
+      else
+        X2 = ...
+      X3 = phi(X1, X2);
+    }
+    X4 = phi(X3);
+
+    ... = X4 + 4;
+
+This is still valid LLVM; the extra phi nodes are purely redundant,
+but all LoopPass'es are required to preserve them.
+This form is ensured by the LCSSA (:ref:`-lcssa <passes-lcssa>`)
+pass and is added automatically by the LoopPassManager when
+scheduling a LoopPass.
+After the loop optimizations are done, these extra phi nodes
+will be deleted by :ref:`-instcombine <passes-instcombine>`.
+
+The major benefit of this transformation is that it makes many other
+loop optimizations simpler.
+
+First of all, a simple observation is that if one needs to see all
+the outside users, they can just iterate over all the (loop closing)
+PHI nodes in the exit blocks (the alternative would be to
+scan the def-use chain [#def-use-chain]_ of all instructions in the loop).
+
+Then, consider for example
+:ref:`-loop-unswitch <passes-loop-unswitch>` ing the loop above.
+Because it is in LCSSA form, we know that any value defined inside of
+the loop will be used either only inside the loop or in a loop closing
+PHI node. In this case, the only loop closing PHI node is X4.
+This means that we can just copy the loop and change the X4
+accordingly, like so:
+
+.. code-block:: C
+
+  for (...) {
+    c = ...;
+    if (c) {
+      for (...) {
+        if (true)
+          X1 = ...
+        else
+          X2 = ...
+        X3 = phi(X1, X2);
+      }
+    } else {
+      for (...) {
+        if (false)
+          X1' = ...
+        else
+          X2' = ...
+        X3' = phi(X1', X2');
+      }
+    }
+    X4 = phi(X3, X3')
+
+Now, all uses of X4 will get the updated value (in general,
+if a loop is in LCSSA form, in any loop transformation,
+we only need to update the loop closing PHI nodes for the changes
+to take effect).  If we did not have Loop Closed SSA form, it means that X3 could
+possibly be used outside the loop. So, we would have to introduce the
+X4 (which is the new X3) and replace all uses of X3 with that.
+However, we should note that because LLVM keeps a def-use chain
+[#def-use-chain]_ for each Value, we wouldn't need
+to perform data-flow analysis to find and replace all the uses
+(there is even a utility function, replaceAllUsesWith(),
+that performs this transformation by iterating the def-use chain).
+
+Another important advantage is that the behavior of all uses
+of an induction variable is the same.  Without this, you need to
+distinguish the case when the variable is used outside of
+the loop it is defined in, for example:
+
+.. code-block:: C
+
+  for (i = 0; i < 100; i++) {
+    for (j = 0; j < 100; j++) {
+      k = i + j;
+      use(k);    // use 1
+    }
+    use(k);      // use 2
+  }
+
+Looking from the outer loop with the normal SSA form, the first use of k
+is not well-behaved, while the second one is an induction variable with
+base 100 and step 1.  Although, in practice, and in the LLVM context,
+such cases can be handled effectively by SCEV. Scalar Evolution
+(:ref:`scalar-evolution <passes-scalar-evolution>`) or SCEV, is a
+(analysis) pass that analyzes and categorizes the evolution of scalar
+expressions in loops.
+
+In general, it's easier to use SCEV in loops that are in LCSSA form.
+The evolution of a scalar (loop-variant) expression that
+SCEV can analyze is, by definition, relative to a loop.
+An expression is represented in LLVM by an
+`llvm::Instruction <https://llvm.org/doxygen/classllvm_1_1Instruction.html>`.
+If the expression is inside two (or more) loops (which can only
+happen if the loops are nested, like in the example above) and you want
+to get an analysis of its evolution (from SCEV),
+you have to also specify relative to what Loop you want it.
+Specifically, you have to use
+`getSCEVAtScope() <https://llvm.org/doxygen/classllvm_1_1ScalarEvolution.html#a21d6ee82eed29080d911dbb548a8bb68>`_.
+
+However, if all loops are in LCSSA form, each expression is actually
+represented by two different llvm::Instructions.  One inside the loop
+and one outside, which is the loop-closing PHI node and represents
+the value of the expression after the last iteration (effectively,
+we break each loop-variant expression into two expressions and so, every
+expression is at most in one loop).  You can now just use
+`getSCEV() <https://llvm.org/doxygen/classllvm_1_1ScalarEvolution.html#a30bd18ac905eacf3601bc6a553a9ff49>`_.
+and which of these two llvm::Instructions you pass to it disambiguates
+the context / scope / relative loop.
+
+.. rubric:: Footnotes
+
+.. [#lcssa-construction] To insert these loop-closing PHI nodes, one has to
+  (re-)compute dominance frontiers (if the loop has multiple exits).
+
+.. [#def-use-chain] A property of SSA is that there exists a def-use chain
+  for each definition, which is a list of all the uses of this definition.
+  LLVM implements this property by keeping a list of all the uses of a Value
+  in an internal data structure.
 
 "More Canonical" Loops
 ======================
diff --git a/llvm/docs/Passes.rst b/llvm/docs/Passes.rst
index 39451a915ee4a..abb7b9d820e0e 100644
--- a/llvm/docs/Passes.rst
+++ b/llvm/docs/Passes.rst
@@ -331,6 +331,8 @@ The ``RegionInfo`` pass detects single entry single exit regions in a function,
 where a region is defined as any subgraph that is connected to the remaining
 graph at only two spots.  Furthermore, a hierarchical region tree is built.
 
+.. _passes-scalar-evolution:
+
 ``-scalar-evolution``: Scalar Evolution Analysis
 ------------------------------------------------
 
@@ -711,6 +713,8 @@ An example of when this can occur is code like this:
 In this case, the unconditional branch at the end of the first if can be
 revectored to the false side of the second if.
 
+.. _passes-lcssa:
+
 ``-lcssa``: Loop-Closed SSA Form Pass
 -------------------------------------
 
@@ -732,7 +736,8 @@ into the right code:
 This is still valid LLVM; the extra phi nodes are purely redundant, and will be
 trivially eliminated by ``InstCombine``.  The major benefit of this
 transformation is that it makes many other loop optimizations, such as
-``LoopUnswitch``\ ing, simpler.
+``LoopUnswitch``\ ing, simpler. You can read more in the
+:ref:`loop terminology section for the LCSSA form <loop-terminology-lcssa>`.
 
 .. _passes-licm:
 
@@ -864,6 +869,8 @@ loops into one. When variables or loads can be shared in the new inner loop, thi
 can lead to significant performance improvements. It uses
 :ref:`Dependence Analysis <passes-da>` for proving the transformations are safe.
 
+.. _passes-loop-unswitch:
+
 ``-loop-unswitch``: Unswitch loops
 ----------------------------------
 
diff --git a/llvm/docs/Phabricator.rst b/llvm/docs/Phabricator.rst
index 9f35af9068904..f6b7fa50d79e1 100644
--- a/llvm/docs/Phabricator.rst
+++ b/llvm/docs/Phabricator.rst
@@ -59,7 +59,8 @@ To get a full diff, use one of the following commands (or just use Arcanist
 to upload your patch):
 
 * ``git show HEAD -U999999 > mypatch.patch``
-* ``git format-patch -U999999 @{u}``
+* ``git diff -U999999 @{u} > mypatch.patch``
+* ``git diff HEAD~1 -U999999 > mypatch.patch``
 
 Before uploading your patch, please make sure it is formatted properly, as
 described in :ref:`How to Submit a Patch <format patches>`.
@@ -169,7 +170,7 @@ yourself.
 Using the Arcanist tool can simplify the process of committing reviewed code as
 it will retrieve reviewers, the ``Differential Revision``, etc from the review
 and place it in the commit message. You may also commit an accepted change
-directly using ``git llvm push``, per the section in the :ref:`getting started
+directly using ``git push``, per the section in the :ref:`getting started
 guide <commit_from_git>`.
 
 Note that if you commit the change without using Arcanist and forget to add the
@@ -194,8 +195,15 @@ This will create a new branch called ``arcpatch-D<Revision>`` based on the
 current ``master`` and will create a commit corresponding to ``D<Revision>`` with a
 commit message derived from information in the Phabricator review.
 
-Check you are happy with the commit message and amend it if necessary. Then,
-make sure the commit is up-to-date, and commit it. This can be done by running
+Check you are happy with the commit message and amend it if necessary.
+For example, ensure the 'Author' property of the commit is set to the original author.
+You can use a command to correct the author property if it is incorrect:
+
+::
+
+  git commit --amend --author="John Doe <jdoe@llvm.org>"
+
+Then, make sure the commit is up-to-date, and commit it. This can be done by running
 the following:
 
 ::
@@ -203,7 +211,7 @@ the following:
   git pull --rebase origin master
   git show # Ensure the patch looks correct.
   ninja check-$whatever # Rerun the appropriate tests if needed.
-  git llvm push
+  git push
 
 Or
 
diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst
index 4f6e759bbeb39..f23756b1ab97e 100644
--- a/llvm/docs/ReleaseNotes.rst
+++ b/llvm/docs/ReleaseNotes.rst
@@ -100,6 +100,11 @@ During this release ...
 Changes to the AMDGPU Target
 -----------------------------
 
+* The backend default denormal handling mode has been switched to on
+  for all targets for all compute function types. Frontends wishing to
+  retain the old behavior should explicitly request f32 denormal
+  flushing.
+
 Changes to the AVR Target
 -----------------------------
 
diff --git a/llvm/docs/Statepoints.rst b/llvm/docs/Statepoints.rst
index 3832a43b287ef..5d6033bc50a70 100644
--- a/llvm/docs/Statepoints.rst
+++ b/llvm/docs/Statepoints.rst
@@ -522,11 +522,8 @@ of the callee. The '# transition args' field indicates how many operands
 are to be interpreted as 'transition parameters'.
 
 The 'deopt parameters' arguments contain an arbitrary list of Values
-which is meaningful to the runtime.  The runtime may read any of these
-values, but is assumed not to modify them.  If the garbage collector
-might need to modify one of these values, it must also be listed in
-the 'gc pointer' argument list.  The '# deopt args' field indicates
-how many operands are to be interpreted as 'deopt parameters'.
+which is meaningful to the runtime.  The '# deopt args' field
+indicates how many operands are to be interpreted as 'deopt parameters'.
 
 The 'gc parameters' arguments contain every pointer to a garbage
 collector object which potentially needs to be updated by the garbage
diff --git a/llvm/include/llvm-c/Orc.h b/llvm/include/llvm-c/Orc.h
index 2a060a6037748..935e5d123f68f 100644
--- a/llvm/include/llvm-c/Orc.h
+++ b/llvm/include/llvm-c/Orc.h
@@ -36,6 +36,11 @@ typedef uint64_t LLVMOrcJITTargetAddress;
 
 /**
  * Create a ThreadSafeContext containing a new LLVMContext.
+ *
+ * Ownership of the underlying ThreadSafeContext data is shared: Clients
+ * can and should dispose of their ThreadSafeContext as soon as they no longer
+ * need to refer to it directly. Other references (e.g. from ThreadSafeModules
+ * will keep the data alive as long as it is needed.
  */
 LLVMOrcThreadSafeContextRef LLVMOrcCreateNewThreadSafeContext(void);
 
@@ -54,6 +59,11 @@ void LLVMOrcDisposeThreadSafeContext(LLVMOrcThreadSafeContextRef TSCtx);
  * Create a ThreadSafeModule wrapper around the given LLVM module. This takes
  * ownership of the M argument which should not be disposed of or referenced
  * after this function returns.
+ *
+ * Ownership of the ThreadSafeModule is unique: If it is transferred to the JIT
+ * (e.g. by LLVMOrcLLJITAddLLVMIRModule), in which case the client is no longer
+ * responsible for it. If it is not transferred to the JIT then the client
+ * should call LLVMOrcDisposeThreadSafeModule to dispose of it.
  */
 LLVMOrcThreadSafeModuleRef
 LLVMOrcCreateNewThreadSafeModule(LLVMModuleRef M,
@@ -66,6 +76,12 @@ void LLVMOrcDisposeThreadSafeModule(LLVMOrcThreadSafeModuleRef TSM);
 
 /**
  * Create an LLJIT instance using all default values.
+ *
+ * The LLJIT instance is uniquely owned by the client and automatically manages
+ * the memory of all JIT'd code and all modules that are transferred to it
+ * (e.g. via LLVMOrcLLJITAddLLVMIRModule). Disposing of the LLJIT instance will
+ * free all memory managed by the JIT, including JIT'd code and not-yet
+ * compiled modules.
  */
 LLVMErrorRef LLVMOrcCreateDefaultLLJIT(LLVMOrcLLJITRef *Result);
 
diff --git a/llvm/include/llvm/ADT/ArrayRef.h b/llvm/include/llvm/ADT/ArrayRef.h
index 5cba73e67f042..d11f8b5b44e93 100644
--- a/llvm/include/llvm/ADT/ArrayRef.h
+++ b/llvm/include/llvm/ADT/ArrayRef.h
@@ -532,11 +532,21 @@ namespace llvm {
     return LHS.equals(RHS);
   }
 
-  template<typename T>
+  template <typename T>
+  inline bool operator==(SmallVectorImpl<T> &LHS, ArrayRef<T> RHS) {
+    return ArrayRef<T>(LHS).equals(RHS);
+  }
+
+  template <typename T>
   inline bool operator!=(ArrayRef<T> LHS, ArrayRef<T> RHS) {
     return !(LHS == RHS);
   }
 
+  template <typename T>
+  inline bool operator!=(SmallVectorImpl<T> &LHS, ArrayRef<T> RHS) {
+    return !(LHS == RHS);
+  }
+
   /// @}
 
   template <typename T> hash_code hash_value(ArrayRef<T> S) {
diff --git a/llvm/include/llvm/ADT/BitmaskEnum.h b/llvm/include/llvm/ADT/BitmaskEnum.h
index d5854d0de2006..89e5508e08e16 100644
--- a/llvm/include/llvm/ADT/BitmaskEnum.h
+++ b/llvm/include/llvm/ADT/BitmaskEnum.h
@@ -94,6 +94,10 @@ template <typename E> std::underlying_type_t<E> Underlying(E Val) {
   return U;
 }
 
+constexpr unsigned bitWidth(uint64_t Value) {
+  return Value ? 1 + bitWidth(Value >> 1) : 0;
+}
+
 template <typename E, typename = std::enable_if_t<is_bitmask_enum<E>::value>>
 E operator~(E Val) {
   return static_cast<E>(~Underlying(Val) & Mask<E>());
@@ -139,6 +143,10 @@ E &operator^=(E &LHS, E RHS) {
 
 // Enable bitmask enums in namespace ::llvm and all nested namespaces.
 LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
+template <typename E, typename = std::enable_if_t<is_bitmask_enum<E>::value>>
+constexpr unsigned BitWidth = BitmaskEnumDetail::bitWidth(uint64_t{
+    static_cast<std::underlying_type_t<E>>(
+        E::LLVM_BITMASK_LARGEST_ENUMERATOR)});
 
 } // namespace llvm
 
diff --git a/llvm/include/llvm/ADT/PostOrderIterator.h b/llvm/include/llvm/ADT/PostOrderIterator.h
index 2fe7447a8e778..bb413a956d9f7 100644
--- a/llvm/include/llvm/ADT/PostOrderIterator.h
+++ b/llvm/include/llvm/ADT/PostOrderIterator.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/GraphTraits.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/iterator_range.h"
 #include <iterator>
 #include <set>
@@ -101,7 +102,7 @@ class po_iterator
 
   // VisitStack - Used to maintain the ordering.  Top = current block
   // First element is basic block pointer, second is the 'next child' to visit
-  std::vector<std::pair<NodeRef, ChildItTy>> VisitStack;
+  SmallVector<std::pair<NodeRef, ChildItTy>, 8> VisitStack;
 
   po_iterator(NodeRef BB) {
     this->insertEdge(Optional<NodeRef>(), BB);
diff --git a/llvm/include/llvm/ADT/STLExtras.h b/llvm/include/llvm/ADT/STLExtras.h
index 638895ee4dc67..a6619adc44b33 100644
--- a/llvm/include/llvm/ADT/STLExtras.h
+++ b/llvm/include/llvm/ADT/STLExtras.h
@@ -114,9 +114,11 @@ class function_ref<Ret(Params...)> {
   function_ref(std::nullptr_t) {}
 
   template <typename Callable>
-  function_ref(Callable &&callable,
-               std::enable_if_t<!std::is_same<std::remove_reference_t<Callable>,
-                                              function_ref>::value> * = nullptr)
+  function_ref(
+      Callable &&callable,
+      std::enable_if_t<
+          !std::is_same<std::remove_cv_t<std::remove_reference_t<Callable>>,
+                        function_ref>::value> * = nullptr)
       : callback(callback_fn<typename std::remove_reference<Callable>::type>),
         callable(reinterpret_cast<intptr_t>(&callable)) {}
 
@@ -124,7 +126,7 @@ class function_ref<Ret(Params...)> {
     return callback(callable, std::forward<Params>(params)...);
   }
 
-  operator bool() const { return callback; }
+  explicit operator bool() const { return callback; }
 };
 
 // deleter - Very very very simple method that is used to invoke operator
@@ -1103,9 +1105,20 @@ inline void array_pod_sort(
         reinterpret_cast<int (*)(const void *, const void *)>(Compare));
 }
 
+namespace detail {
+template <typename T>
+// We can use qsort if the iterator type is a pointer and the underlying value
+// is trivially copyable.
+using sort_trivially_copyable = conjunction<
+    std::is_pointer<T>,
+    is_trivially_copyable<typename std::iterator_traits<T>::value_type>>;
+} // namespace detail
+
 // Provide wrappers to std::sort which shuffle the elements before sorting
 // to help uncover non-deterministic behavior (PR35135).
-template <typename IteratorTy>
+template <typename IteratorTy,
+          std::enable_if_t<!detail::sort_trivially_copyable<IteratorTy>::value,
+                           int> = 0>
 inline void sort(IteratorTy Start, IteratorTy End) {
 #ifdef EXPENSIVE_CHECKS
   detail::presortShuffle<IteratorTy>(Start, End);
@@ -1113,6 +1126,15 @@ inline void sort(IteratorTy Start, IteratorTy End) {
   std::sort(Start, End);
 }
 
+// Forward trivially copyable types to array_pod_sort. This avoids a large
+// amount of code bloat for a minor performance hit.
+template <typename IteratorTy,
+          std::enable_if_t<detail::sort_trivially_copyable<IteratorTy>::value,
+                           int> = 0>
+inline void sort(IteratorTy Start, IteratorTy End) {
+  array_pod_sort(Start, End);
+}
+
 template <typename Container> inline void sort(Container &&C) {
   llvm::sort(adl_begin(C), adl_end(C));
 }
diff --git a/llvm/include/llvm/ADT/StringSet.h b/llvm/include/llvm/ADT/StringSet.h
index c740aee41a31d..7f3bbd4901713 100644
--- a/llvm/include/llvm/ADT/StringSet.h
+++ b/llvm/include/llvm/ADT/StringSet.h
@@ -36,7 +36,6 @@ namespace llvm {
     explicit StringSet(AllocatorTy A) : base(A) {}
 
     std::pair<typename base::iterator, bool> insert(StringRef Key) {
-      assert(!Key.empty());
       return base::insert(std::make_pair(Key, None));
     }
 
diff --git a/llvm/include/llvm/ADT/Triple.h b/llvm/include/llvm/ADT/Triple.h
index 7c9e9ff8c72a7..9c4bce6e1a8bd 100644
--- a/llvm/include/llvm/ADT/Triple.h
+++ b/llvm/include/llvm/ADT/Triple.h
@@ -105,6 +105,7 @@ class Triple {
   enum SubArchType {
     NoSubArch,
 
+    ARMSubArch_v8_6a,
     ARMSubArch_v8_5a,
     ARMSubArch_v8_4a,
     ARMSubArch_v8_3a,
diff --git a/llvm/include/llvm/ADT/Twine.h b/llvm/include/llvm/ADT/Twine.h
index 2dc7486c924f7..4140c22aad3d9 100644
--- a/llvm/include/llvm/ADT/Twine.h
+++ b/llvm/include/llvm/ADT/Twine.h
@@ -153,11 +153,11 @@ namespace llvm {
 
     /// LHS - The prefix in the concatenation, which may be uninitialized for
     /// Null or Empty kinds.
-    Child LHS = {0};
+    Child LHS;
 
     /// RHS - The suffix in the concatenation, which may be uninitialized for
     /// Null or Empty kinds.
-    Child RHS = {0};
+    Child RHS;
 
     /// LHSKind - The NodeKind of the left hand side, \see getLHSKind().
     NodeKind LHSKind = EmptyKind;
diff --git a/llvm/include/llvm/ADT/Waymarking.h b/llvm/include/llvm/ADT/Waymarking.h
new file mode 100644
index 0000000000000..547538c75f14b
--- /dev/null
+++ b/llvm/include/llvm/ADT/Waymarking.h
@@ -0,0 +1,325 @@
+//===- Waymarking.h - Array waymarking algorithm ----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Utility to backtrace an array's head, from a pointer into it. For the
+// backtrace to work, we use "Waymarks", which are special tags embedded into
+// the array's elements.
+//
+// A Tag of n-bits (in size) is composed as follows:
+//
+// bits: |   n-1   |             n-2 ... 0              |
+//       .---------.------------------------------------.
+//       |Stop Mask|(2^(n-1))-ary numeric system - digit|
+//       '---------'------------------------------------'
+//
+// Backtracing is done as follows:
+// Walk back (starting from a given pointer to an element into the array), until
+// a tag with a "Stop Mask" is reached. Then start calculating the "Offset" from
+// the array's head, by picking up digits along the way, until another stop is
+// reached. The "Offset" is then subtracted from the current pointer, and the
+// result is the array's head.
+// A special case - if we first encounter a Tag with a Stop and a zero digit,
+// then this is already the head.
+//
+// For example:
+// In case of 2 bits:
+//
+// Tags:
+// x0 - binary digit 0
+// x1 - binary digit 1
+// 1x - stop and calculate (s)
+//
+// Array:
+//         .---.---.---.---.---.---.---.---.---.---.---.---.---.---.---.---.
+// head -> |s0 |s1 | 0 |s1 | 0 | 0 |s1 | 1 | 1 |s1 | 0 | 1 | 0 |s1 | 0 | 1 |
+//         '---'---'---'---'---'---'---'---'---'---'---'---'---'---'---'---'
+//             |-1 |-2     |-4         |-7         |-10            |-14
+//          <_ |   |       |           |           |               |
+//          <_____ |       |           |           |               |
+//          <_____________ |           |           |               |
+//          <_________________________ |           |               |
+//          <_____________________________________ |               |
+//          <_____________________________________________________ |
+//
+//
+// In case of 3 bits:
+//
+// Tags:
+// x00 - quaternary digit 0
+// x01 - quaternary digit 1
+// x10 - quaternary digit 2
+// x11 - quaternary digit 3
+// 1xy - stop and calculate (s)
+//
+// Array:
+//         .---.---.---.---.---.---.---.---.---.---.---.---.---.---.---.---.
+// head -> |s0 |s1 |s2 |s3 | 0 |s1 | 2 |s1 | 0 |s2 | 2 |s2 | 0 |s3 | 2 |s3 |
+//         '---'---'---'---'---'---'---'---'---'---'---'---'---'---'---'---'
+//             |-1 |-2 |-3 |-4     |-6     |-8     |-10    |-12    |-14    |-16
+//          <_ |   |   |   |       |       |       |       |       |       |
+//          <_____ |   |   |       |       |       |       |       |       |
+//          <_________ |   |       |       |       |       |       |       |
+//          <_____________ |       |       |       |       |       |       |
+//          <_____________________ |       |       |       |       |       |
+//          <_____________________________ |       |       |       |       |
+//          <_____________________________________ |       |       |       |
+//          <_____________________________________________ |       |       |
+//          <_____________________________________________________ |       |
+//          <_____________________________________________________________ |
+//
+//
+// The API introduce 2 functions:
+// 1. fillWaymarks
+// 2. followWaymarks
+//
+// Example:
+//   int N = 10;
+//   int M = 5;
+//   int **A = new int *[N + M];   // Define the array.
+//   for (int I = 0; I < N + M; ++I)
+//     A[I] = new int(I);
+//
+//   fillWaymarks(A, A + N);       // Set the waymarks for the first N elements
+//                                 // of the array.
+//                                 // Note that it must be done AFTER we fill
+//                                 // the array's elements.
+//
+//   ...                           // Elements which are not in the range
+//                                 // [A, A+N) will not be marked, and we won't
+//                                 // be able to call followWaymarks on them.
+//
+//   ...                           // Elements which will be changed after the
+//                                 // call to fillWaymarks, will have to be
+//                                 // retagged.
+//
+//   fillWaymarks(A + N, A + N + M, N); // Set the waymarks of the remaining M
+//                                      // elements.
+//   ...
+//   int **It = A + N + 1;
+//   int **B = followWaymarks(It); // Find the head of the array containing It.
+//   assert(B == A);
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ADT_WAYMARKING_H
+#define LLVM_ADT_WAYMARKING_H
+
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/PointerLikeTypeTraits.h"
+
+namespace llvm {
+
+namespace detail {
+
+template <unsigned NumBits> struct WaymarkingTraits {
+  enum : unsigned {
+    // The number of bits of a Waymarking Tag.
+    NUM_BITS = NumBits,
+
+    // A Tag is composed from a Mark and a Stop mask.
+    MARK_SIZE = NUM_BITS - 1,
+    STOP_MASK = (1 << MARK_SIZE),
+    MARK_MASK = (STOP_MASK - 1),
+    TAG_MASK = (MARK_MASK | STOP_MASK),
+
+    // The number of pre-calculated tags (for fast fill).
+    NUM_STATIC_TAGS = 32
+  };
+
+private:
+  // Add a new tag, calculated from Count and Stop, to the Vals pack, while
+  // continuing recursively to decrease Len down to 0.
+  template <unsigned Len, bool Stop, unsigned Count, uint8_t... Vals>
+  struct AddTag;
+
+  // Delegate to the specialized AddTag according to the need of a Stop mask.
+  template <unsigned Len, unsigned Count, uint8_t... Vals> struct GenTag {
+    typedef
+        typename AddTag<Len, (Count <= MARK_MASK), Count, Vals...>::Xdata Xdata;
+  };
+
+  // Start adding tags while calculating the next Count, which is actually the
+  // number of already calculated tags (equivalent to the position in the
+  // array).
+  template <unsigned Len, uint8_t... Vals> struct GenOffset {
+    typedef typename GenTag<Len, sizeof...(Vals), Vals...>::Xdata Xdata;
+  };
+
+  // Add the tag and remove it from Count.
+  template <unsigned Len, unsigned Count, uint8_t... Vals>
+  struct AddTag<Len, false, Count, Vals...> {
+    typedef typename GenTag<Len - 1, (Count >> MARK_SIZE), Vals...,
+                            Count & MARK_MASK>::Xdata Xdata;
+  };
+
+  // We have reached the end of this Count, so start with a new Count.
+  template <unsigned Len, unsigned Count, uint8_t... Vals>
+  struct AddTag<Len, true, Count, Vals...> {
+    typedef typename GenOffset<Len - 1, Vals...,
+                               (Count & MARK_MASK) | STOP_MASK>::Xdata Xdata;
+  };
+
+  template <unsigned Count, uint8_t... Vals> struct TagsData {
+    // The remaining number for calculating the next tag, following the last one
+    // in Values.
+    static const unsigned Remain = Count;
+
+    // The array of ordered pre-calculated Tags.
+    static const uint8_t Values[sizeof...(Vals)];
+  };
+
+  // Specialize the case when Len equals 0, as the recursion stop condition.
+  template <unsigned Count, uint8_t... Vals>
+  struct AddTag<0, false, Count, Vals...> {
+    typedef TagsData<Count, Vals...> Xdata;
+  };
+
+  template <unsigned Count, uint8_t... Vals>
+  struct AddTag<0, true, Count, Vals...> {
+    typedef TagsData<Count, Vals...> Xdata;
+  };
+
+public:
+  typedef typename GenOffset<NUM_STATIC_TAGS>::Xdata Tags;
+};
+
+template <unsigned NumBits>
+template <unsigned Count, uint8_t... Vals>
+const uint8_t WaymarkingTraits<NumBits>::TagsData<
+    Count, Vals...>::Values[sizeof...(Vals)] = {Vals...};
+
+} // end namespace detail
+
+/// This class is responsible for tagging (and retrieving the tag of) a given
+/// element of type T.
+template <class T, class WTraits = detail::WaymarkingTraits<
+                       PointerLikeTypeTraits<T>::NumLowBitsAvailable>>
+struct Waymarker {
+  using Traits = WTraits;
+  static void setWaymark(T &N, unsigned Tag) { N.setWaymark(Tag); }
+  static unsigned getWaymark(const T &N) { return N.getWaymark(); }
+};
+
+template <class T, class WTraits> struct Waymarker<T *, WTraits> {
+  using Traits = WTraits;
+  static void setWaymark(T *&N, unsigned Tag) {
+    reinterpret_cast<uintptr_t &>(N) |= static_cast<uintptr_t>(Tag);
+  }
+  static unsigned getWaymark(const T *N) {
+    return static_cast<unsigned>(reinterpret_cast<uintptr_t>(N)) &
+           Traits::TAG_MASK;
+  }
+};
+
+/// Sets up the waymarking algorithm's tags for a given range [Begin, End).
+///
+/// \param Begin The beginning of the range to mark with tags (inclusive).
+/// \param End The ending of the range to mark with tags (exclusive).
+/// \param Offset The position in the supposed tags array from which to start
+/// marking the given range.
+template <class TIter, class Marker = Waymarker<
+                           typename std::iterator_traits<TIter>::value_type>>
+void fillWaymarks(TIter Begin, TIter End, size_t Offset = 0) {
+  if (Begin == End)
+    return;
+
+  size_t Count = Marker::Traits::Tags::Remain;
+  if (Offset <= Marker::Traits::NUM_STATIC_TAGS) {
+    // Start by filling the pre-calculated tags, starting from the given offset.
+    while (Offset != Marker::Traits::NUM_STATIC_TAGS) {
+      Marker::setWaymark(*Begin, Marker::Traits::Tags::Values[Offset]);
+
+      ++Offset;
+      ++Begin;
+
+      if (Begin == End)
+        return;
+    }
+  } else {
+    // The given offset is larger than the number of pre-calculated tags, so we
+    // must do it the hard way.
+    // Calculate the next remaining Count, as if we have filled the tags up to
+    // the given offset.
+    size_t Off = Marker::Traits::NUM_STATIC_TAGS;
+    do {
+      ++Off;
+
+      unsigned Tag = Count & Marker::Traits::MARK_MASK;
+
+      // If the count can fit into the tag, then the counting must stop.
+      if (Count <= Marker::Traits::MARK_MASK) {
+        Tag |= Marker::Traits::STOP_MASK;
+        Count = Off;
+      } else
+        Count >>= Marker::Traits::MARK_SIZE;
+    } while (Off != Offset);
+  }
+
+  // By now, we have the matching remaining Count for the current offset.
+  do {
+    ++Offset;
+
+    unsigned Tag = Count & Marker::Traits::MARK_MASK;
+
+    // If the count can fit into the tag, then the counting must stop.
+    if (Count <= Marker::Traits::MARK_MASK) {
+      Tag |= Marker::Traits::STOP_MASK;
+      Count = Offset;
+    } else
+      Count >>= Marker::Traits::MARK_SIZE;
+
+    Marker::setWaymark(*Begin, Tag);
+    ++Begin;
+  } while (Begin != End);
+}
+
+/// Sets up the waymarking algorithm's tags for a given range.
+///
+/// \param Range The range to mark with tags.
+/// \param Offset The position in the supposed tags array from which to start
+/// marking the given range.
+template <typename R, class Marker = Waymarker<typename std::remove_reference<
+                          decltype(*std::begin(std::declval<R &>()))>::type>>
+void fillWaymarks(R &&Range, size_t Offset = 0) {
+  return fillWaymarks<decltype(std::begin(std::declval<R &>())), Marker>(
+      adl_begin(Range), adl_end(Range), Offset);
+}
+
+/// Retrieves the element marked with tag of only STOP_MASK, by following the
+/// waymarks. This is the first element in a range passed to a previous call to
+/// \c fillWaymarks with \c Offset 0.
+///
+/// For the trivial usage of calling \c fillWaymarks(Array), and \I is an
+/// iterator inside \c Array, this function retrieves the head of \c Array, by
+/// following the waymarks.
+///
+/// \param I The iterator into an array which was marked by the waymarking tags
+/// (by a previous call to \c fillWaymarks).
+template <class TIter, class Marker = Waymarker<
+                           typename std::iterator_traits<TIter>::value_type>>
+TIter followWaymarks(TIter I) {
+  unsigned Tag;
+  do
+    Tag = Marker::getWaymark(*I--);
+  while (!(Tag & Marker::Traits::STOP_MASK));
+
+  // Special case for the first Use.
+  if (Tag != Marker::Traits::STOP_MASK) {
+    ptrdiff_t Offset = Tag & Marker::Traits::MARK_MASK;
+    while (!((Tag = Marker::getWaymark(*I)) & Marker::Traits::STOP_MASK)) {
+      Offset = (Offset << Marker::Traits::MARK_SIZE) + Tag;
+      --I;
+    }
+    I -= Offset;
+  }
+  return ++I;
+}
+
+} // end namespace llvm
+
+#endif // LLVM_ADT_WAYMARKING_H
diff --git a/llvm/include/llvm/IR/KnowledgeRetention.h b/llvm/include/llvm/Analysis/AssumeBundleQueries.h
similarity index 78%
rename from llvm/include/llvm/IR/KnowledgeRetention.h
rename to llvm/include/llvm/Analysis/AssumeBundleQueries.h
index a7de7cd16b660..ba82842556453 100644
--- a/llvm/include/llvm/IR/KnowledgeRetention.h
+++ b/llvm/include/llvm/Analysis/AssumeBundleQueries.h
@@ -1,4 +1,4 @@
-//===- KnowledgeRetention.h - utilities to preserve informations *- C++ -*-===//
+//===- AssumeBundleQueries.h - utilities to query assume bundles *- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,32 +6,28 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file contain tools to preserve informations. They should be used before
-// performing a transformation that may move and delete instructions as those
-// transformation may destroy or worsen information that can be derived from the
-// IR.
+// This file contain tools to query into assume bundles. assume bundles can be
+// built using utilities from Transform/Utils/AssumeBundleBuilder.h
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_TRANSFORMS_UTILS_ASSUMEBUILDER_H
-#define LLVM_TRANSFORMS_UTILS_ASSUMEBUILDER_H
+#ifndef LLVM_TRANSFORMS_UTILS_ASSUMEBUNDLEQUERIES_H
+#define LLVM_TRANSFORMS_UTILS_ASSUMEBUNDLEQUERIES_H
 
 #include "llvm/IR/Attributes.h"
-#include "llvm/IR/Instruction.h"
-#include "llvm/IR/PassManager.h"
+#include "llvm/IR/Instructions.h"
 #include "llvm/ADT/DenseMap.h"
 
 namespace llvm {
 class IntrinsicInst;
 
-/// Build a call to llvm.assume to preserve informations that can be derived
-/// from the given instruction.
-/// If no information derived from \p I, this call returns null.
-/// The returned instruction is not inserted anywhere.
-CallInst *BuildAssumeFromInst(const Instruction *I, Module *M);
-inline CallInst *BuildAssumeFromInst(Instruction *I) {
-  return BuildAssumeFromInst(I, I->getModule());
-}
+/// Index of elements in the operand bundle.
+/// If the element exist it is guaranteed to be what is specified in this enum
+/// but it may not exist.
+enum AssumeBundleArg {
+  ABA_WasOn = 0,
+  ABA_Argument = 1,
+};
 
 /// It is possible to have multiple Value for the argument of an attribute in
 /// the same llvm.assume on the same llvm::Value. This is rare but need to be
@@ -133,16 +129,6 @@ inline RetainedKnowledge getKnowledgeFromUseInAssume(const Use *U) {
 /// function returned true.
 bool isAssumeWithEmptyBundle(CallInst &Assume);
 
-//===----------------------------------------------------------------------===//
-// Utilities for testing
-//===----------------------------------------------------------------------===//
-
-/// This pass will try to build an llvm.assume for every instruction in the
-/// function. Its main purpose is testing.
-struct AssumeBuilderPass : public PassInfoMixin<AssumeBuilderPass> {
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
-};
-
 } // namespace llvm
 
 #endif
diff --git a/llvm/include/llvm/Analysis/BranchProbabilityInfo.h b/llvm/include/llvm/Analysis/BranchProbabilityInfo.h
index fdf3a2c75fa43..8a3cfad13cb6c 100644
--- a/llvm/include/llvm/Analysis/BranchProbabilityInfo.h
+++ b/llvm/include/llvm/Analysis/BranchProbabilityInfo.h
@@ -98,7 +98,7 @@ class BranchProbabilityInfo {
                                        const BasicBlock *Dst) const;
 
   BranchProbability getEdgeProbability(const BasicBlock *Src,
-                                       succ_const_iterator Dst) const;
+                                       const_succ_iterator Dst) const;
 
   /// Test if an edge is hot relative to other out-edges of the Src.
   ///
diff --git a/llvm/include/llvm/Analysis/CFGPrinter.h b/llvm/include/llvm/Analysis/CFGPrinter.h
index 634be322a4a9f..b0c88f2962508 100644
--- a/llvm/include/llvm/Analysis/CFGPrinter.h
+++ b/llvm/include/llvm/Analysis/CFGPrinter.h
@@ -128,7 +128,7 @@ struct DOTGraphTraits<const Function*> : public DefaultDOTGraphTraits {
   }
 
   static std::string getEdgeSourceLabel(const BasicBlock *Node,
-                                        succ_const_iterator I) {
+                                        const_succ_iterator I) {
     // Label source of conditional branches with "T" or "F"
     if (const BranchInst *BI = dyn_cast<BranchInst>(Node->getTerminator()))
       if (BI->isConditional())
@@ -150,7 +150,7 @@ struct DOTGraphTraits<const Function*> : public DefaultDOTGraphTraits {
   }
 
   /// Display the raw branch weights from PGO.
-  std::string getEdgeAttributes(const BasicBlock *Node, succ_const_iterator I,
+  std::string getEdgeAttributes(const BasicBlock *Node, const_succ_iterator I,
                                 const Function *F) {
     const Instruction *TI = Node->getTerminator();
     if (TI->getNumSuccessors() == 1)
diff --git a/llvm/include/llvm/Analysis/ConstantFolding.h b/llvm/include/llvm/Analysis/ConstantFolding.h
index 90dc14b0ed9e6..68aa1948ab88e 100644
--- a/llvm/include/llvm/Analysis/ConstantFolding.h
+++ b/llvm/include/llvm/Analysis/ConstantFolding.h
@@ -119,10 +119,11 @@ Constant *ConstantFoldInsertElementInstruction(Constant *Val,
 Constant *ConstantFoldExtractElementInstruction(Constant *Val, Constant *Idx);
 
 /// Attempt to constant fold a shufflevector instruction with the
-/// specified operands and indices.  The constant result is returned if
-/// successful; if not, null is returned.
+/// specified operands and mask.  See class ShuffleVectorInst for a description
+/// of the mask representation. The constant result is returned if successful;
+/// if not, null is returned.
 Constant *ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2,
-                                               Constant *Mask);
+                                               ArrayRef<int> Mask);
 
 /// ConstantFoldLoadFromConstPtr - Return the value that a load from C would
 /// produce if it is constant and determinable.  If this is not determinable,
diff --git a/llvm/include/llvm/Analysis/InstructionSimplify.h b/llvm/include/llvm/Analysis/InstructionSimplify.h
index b661caee6848f..7a9a1a81555b4 100644
--- a/llvm/include/llvm/Analysis/InstructionSimplify.h
+++ b/llvm/include/llvm/Analysis/InstructionSimplify.h
@@ -230,7 +230,8 @@ Value *SimplifyCastInst(unsigned CastOpc, Value *Op, Type *Ty,
                         const SimplifyQuery &Q);
 
 /// Given operands for a ShuffleVectorInst, fold the result or return null.
-Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask,
+/// See class ShuffleVectorInst for a description of the mask representation.
+Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, ArrayRef<int> Mask,
                                  Type *RetTy, const SimplifyQuery &Q);
 
 //=== Helper functions for higher up the class hierarchy.
diff --git a/llvm/include/llvm/Analysis/LazyValueInfo.h b/llvm/include/llvm/Analysis/LazyValueInfo.h
index 74e8f50720376..53970e3792b9c 100644
--- a/llvm/include/llvm/Analysis/LazyValueInfo.h
+++ b/llvm/include/llvm/Analysis/LazyValueInfo.h
@@ -85,7 +85,9 @@ class LazyValueInfo {
   /// Return the ConstantRange constraint that is known to hold for the
   /// specified value at the end of the specified block. This may only be called
   /// on integer-typed Values.
-  ConstantRange getConstantRange(Value *V, BasicBlock *BB, Instruction *CxtI = nullptr);
+  ConstantRange getConstantRange(Value *V, BasicBlock *BB,
+                                 Instruction *CxtI = nullptr,
+                                 bool UndefAllowed = true);
 
   /// Determine whether the specified value is known to be a
   /// constant on the specified edge.  Return null if not.
diff --git a/llvm/include/llvm/Analysis/MemoryBuiltins.h b/llvm/include/llvm/Analysis/MemoryBuiltins.h
index a89d76b9e5bd9..a674a3a65f478 100644
--- a/llvm/include/llvm/Analysis/MemoryBuiltins.h
+++ b/llvm/include/llvm/Analysis/MemoryBuiltins.h
@@ -75,6 +75,14 @@ bool isMallocLikeFn(const Value *V,
                     function_ref<const TargetLibraryInfo &(Function &)> GetTLI,
                     bool LookThroughBitCast = false);
 
+/// Tests if a value is a call or invoke to a library function that
+/// allocates uninitialized memory with alignment (such as aligned_alloc).
+bool isAlignedAllocLikeFn(const Value *V, const TargetLibraryInfo *TLI,
+                          bool LookThroughBitCast = false);
+bool isAlignedAllocLikeFn(
+    const Value *V, function_ref<const TargetLibraryInfo &(Function &)> GetTLI,
+    bool LookThroughBitCast = false);
+
 /// Tests if a value is a call or invoke to a library function that
 /// allocates zero-filled memory (such as calloc).
 bool isCallocLikeFn(const Value *V, const TargetLibraryInfo *TLI,
diff --git a/llvm/include/llvm/Analysis/TargetFolder.h b/llvm/include/llvm/Analysis/TargetFolder.h
index 7277a1230e282..b23316ac3d9b6 100644
--- a/llvm/include/llvm/Analysis/TargetFolder.h
+++ b/llvm/include/llvm/Analysis/TargetFolder.h
@@ -259,7 +259,7 @@ class TargetFolder final : public IRBuilderFolder {
   }
 
   Constant *CreateShuffleVector(Constant *V1, Constant *V2,
-                                Constant *Mask) const override {
+                                ArrayRef<int> Mask) const override {
     return Fold(ConstantExpr::getShuffleVector(V1, V2, Mask));
   }
 
diff --git a/llvm/include/llvm/Analysis/TargetLibraryInfo.def b/llvm/include/llvm/Analysis/TargetLibraryInfo.def
index ed0e125282288..b5688cecc4adc 100644
--- a/llvm/include/llvm/Analysis/TargetLibraryInfo.def
+++ b/llvm/include/llvm/Analysis/TargetLibraryInfo.def
@@ -481,6 +481,9 @@ TLI_DEFINE_STRING_INTERNAL("acoshl")
 /// long double acosl(long double x);
 TLI_DEFINE_ENUM_INTERNAL(acosl)
 TLI_DEFINE_STRING_INTERNAL("acosl")
+/// void *aligned_alloc(size_t alignment, size_t size);
+TLI_DEFINE_ENUM_INTERNAL(aligned_alloc)
+TLI_DEFINE_STRING_INTERNAL("aligned_alloc")
 /// double asin(double x);
 TLI_DEFINE_ENUM_INTERNAL(asin)
 TLI_DEFINE_STRING_INTERNAL("asin")
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index ce04592bf53ea..d70396a61bc4a 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -233,31 +233,6 @@ class TargetTransformInfo {
   /// the EXT operation.
   int getExtCost(const Instruction *I, const Value *Src) const;
 
-  /// Estimate the cost of a function call when lowered.
-  ///
-  /// The contract for this is the same as \c getOperationCost except that it
-  /// supports an interface that provides extra information specific to call
-  /// instructions.
-  ///
-  /// This is the most basic query for estimating call cost: it only knows the
-  /// function type and (potentially) the number of arguments at the call site.
-  /// The latter is only interesting for varargs function types.
-  int getCallCost(FunctionType *FTy, int NumArgs = -1,
-                  const User *U = nullptr) const;
-
-  /// Estimate the cost of calling a specific function when lowered.
-  ///
-  /// This overload adds the ability to reason about the particular function
-  /// being called in the event it is a library call with special lowering.
-  int getCallCost(const Function *F, int NumArgs = -1,
-                  const User *U = nullptr) const;
-
-  /// Estimate the cost of calling a specific function when lowered.
-  ///
-  /// This overload allows specifying a set of candidate argument values.
-  int getCallCost(const Function *F, ArrayRef<const Value *> Arguments,
-                  const User *U = nullptr) const;
-
   /// \returns A value by which our inlining threshold should be multiplied.
   /// This is primarily used to bump up the inlining threshold wholesale on
   /// targets where calls are unusually expensive.
@@ -279,15 +254,11 @@ class TargetTransformInfo {
   int getInlinerVectorBonusPercent() const;
 
   /// Estimate the cost of an intrinsic when lowered.
-  ///
-  /// Mirrors the \c getCallCost method but uses an intrinsic identifier.
   int getIntrinsicCost(Intrinsic::ID IID, Type *RetTy,
                        ArrayRef<Type *> ParamTys,
                        const User *U = nullptr) const;
 
   /// Estimate the cost of an intrinsic when lowered.
-  ///
-  /// Mirrors the \c getCallCost method but uses an intrinsic identifier.
   int getIntrinsicCost(Intrinsic::ID IID, Type *RetTy,
                        ArrayRef<const Value *> Arguments,
                        const User *U = nullptr) const;
@@ -873,17 +844,36 @@ class TargetTransformInfo {
   /// instructions.
   unsigned getPrefetchDistance() const;
 
-  /// \return Some HW prefetchers can handle accesses up to a certain
-  /// constant stride.  This is the minimum stride in bytes where it
-  /// makes sense to start adding SW prefetches.  The default is 1,
-  /// i.e. prefetch with any stride.
-  unsigned getMinPrefetchStride() const;
+  /// Some HW prefetchers can handle accesses up to a certain constant stride.
+  /// Sometimes prefetching is beneficial even below the HW prefetcher limit,
+  /// and the arguments provided are meant to serve as a basis for deciding this
+  /// for a particular loop.
+  ///
+  /// \param NumMemAccesses        Number of memory accesses in the loop.
+  /// \param NumStridedMemAccesses Number of the memory accesses that
+  ///                              ScalarEvolution could find a known stride
+  ///                              for.
+  /// \param NumPrefetches         Number of software prefetches that will be
+  ///                              emitted as determined by the addresses
+  ///                              involved and the cache line size.
+  /// \param HasCall               True if the loop contains a call.
+  ///
+  /// \return This is the minimum stride in bytes where it makes sense to start
+  ///         adding SW prefetches. The default is 1, i.e. prefetch with any
+  ///         stride.
+  unsigned getMinPrefetchStride(unsigned NumMemAccesses,
+                                unsigned NumStridedMemAccesses,
+                                unsigned NumPrefetches,
+                                bool HasCall) const;
 
   /// \return The maximum number of iterations to prefetch ahead.  If
   /// the required number of iterations is more than this number, no
   /// prefetching is performed.
   unsigned getMaxPrefetchIterationsAhead() const;
 
+  /// \return True if prefetching should also be done for writes.
+  bool enableWritePrefetching() const;
+
   /// \return The maximum interleave factor that any transform should try to
   /// perform for this target. This number depends on the level of parallelism
   /// and the number of execution units in the CPU.
@@ -1206,10 +1196,6 @@ class TargetTransformInfo::Concept {
   virtual int getGEPCost(Type *PointeeType, const Value *Ptr,
                          ArrayRef<const Value *> Operands) = 0;
   virtual int getExtCost(const Instruction *I, const Value *Src) = 0;
-  virtual int getCallCost(FunctionType *FTy, int NumArgs, const User *U) = 0;
-  virtual int getCallCost(const Function *F, int NumArgs, const User *U) = 0;
-  virtual int getCallCost(const Function *F,
-                          ArrayRef<const Value *> Arguments, const User *U) = 0;
   virtual unsigned getInliningThresholdMultiplier() = 0;
   virtual int getInlinerVectorBonusPercent() = 0;
   virtual int getIntrinsicCost(Intrinsic::ID IID, Type *RetTy,
@@ -1331,14 +1317,22 @@ class TargetTransformInfo::Concept {
   /// \return Some HW prefetchers can handle accesses up to a certain
   /// constant stride.  This is the minimum stride in bytes where it
   /// makes sense to start adding SW prefetches.  The default is 1,
-  /// i.e. prefetch with any stride.
-  virtual unsigned getMinPrefetchStride() const = 0;
+  /// i.e. prefetch with any stride.  Sometimes prefetching is beneficial
+  /// even below the HW prefetcher limit, and the arguments provided are
+  /// meant to serve as a basis for deciding this for a particular loop.
+  virtual unsigned getMinPrefetchStride(unsigned NumMemAccesses,
+                                        unsigned NumStridedMemAccesses,
+                                        unsigned NumPrefetches,
+                                        bool HasCall) const = 0;
 
   /// \return The maximum number of iterations to prefetch ahead.  If
   /// the required number of iterations is more than this number, no
   /// prefetching is performed.
   virtual unsigned getMaxPrefetchIterationsAhead() const = 0;
 
+  /// \return True if prefetching should also be done for writes.
+  virtual bool enableWritePrefetching() const = 0;
+
   virtual unsigned getMaxInterleaveFactor(unsigned VF) = 0;
   virtual unsigned getArithmeticInstrCost(
       unsigned Opcode, Type *Ty, OperandValueKind Opd1Info,
@@ -1455,16 +1449,6 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
   int getExtCost(const Instruction *I, const Value *Src) override {
     return Impl.getExtCost(I, Src);
   }
-  int getCallCost(FunctionType *FTy, int NumArgs, const User *U) override {
-    return Impl.getCallCost(FTy, NumArgs, U);
-  }
-  int getCallCost(const Function *F, int NumArgs, const User *U) override {
-    return Impl.getCallCost(F, NumArgs, U);
-  }
-  int getCallCost(const Function *F,
-                  ArrayRef<const Value *> Arguments, const User *U) override {
-    return Impl.getCallCost(F, Arguments, U);
-  }
   unsigned getInliningThresholdMultiplier() override {
     return Impl.getInliningThresholdMultiplier();
   }
@@ -1727,8 +1711,12 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
   /// Return the minimum stride necessary to trigger software
   /// prefetching.
   ///
-  unsigned getMinPrefetchStride() const override {
-    return Impl.getMinPrefetchStride();
+  unsigned getMinPrefetchStride(unsigned NumMemAccesses,
+                                unsigned NumStridedMemAccesses,
+                                unsigned NumPrefetches,
+                                bool HasCall) const override {
+    return Impl.getMinPrefetchStride(NumMemAccesses, NumStridedMemAccesses,
+                                     NumPrefetches, HasCall);
   }
 
   /// Return the maximum prefetch distance in terms of loop
@@ -1738,6 +1726,11 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
     return Impl.getMaxPrefetchIterationsAhead();
   }
 
+  /// \return True if prefetching should also be done for writes.
+  bool enableWritePrefetching() const override {
+    return Impl.enableWritePrefetching();
+  }
+
   unsigned getMaxInterleaveFactor(unsigned VF) override {
     return Impl.getMaxInterleaveFactor(VF);
   }
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 765d35a05a46c..0cd3dba6c9950 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -132,21 +132,6 @@ class TargetTransformInfoImplBase {
     return TTI::TCC_Basic;
   }
 
-  unsigned getCallCost(FunctionType *FTy, int NumArgs, const User *U) {
-    assert(FTy && "FunctionType must be provided to this routine.");
-
-    // The target-independent implementation just measures the size of the
-    // function by approximating that each argument will take on average one
-    // instruction to prepare.
-
-    if (NumArgs < 0)
-      // Set the argument number to the number of explicit arguments in the
-      // function.
-      NumArgs = FTy->getNumParams();
-
-    return TTI::TCC_Basic * (NumArgs + 1);
-  }
-
   unsigned getInliningThresholdMultiplier() { return 1; }
 
   int getInlinerVectorBonusPercent() { return 150; }
@@ -431,8 +416,12 @@ class TargetTransformInfoImplBase {
   }
 
   unsigned getPrefetchDistance() const { return 0; }
-  unsigned getMinPrefetchStride() const { return 1; }
+  unsigned getMinPrefetchStride(unsigned NumMemAccesses,
+                                unsigned NumStridedMemAccesses,
+                                unsigned NumPrefetches,
+                                bool HasCall) const { return 1; }
   unsigned getMaxPrefetchIterationsAhead() const { return UINT_MAX; }
+  bool enableWritePrefetching() const { return false; }
 
   unsigned getMaxInterleaveFactor(unsigned VF) { return 1; }
 
@@ -726,37 +715,6 @@ class TargetTransformInfoImplCRTPBase : public TargetTransformInfoImplBase {
   explicit TargetTransformInfoImplCRTPBase(const DataLayout &DL) : BaseT(DL) {}
 
 public:
-  using BaseT::getCallCost;
-
-  unsigned getCallCost(const Function *F, int NumArgs, const User *U) {
-    assert(F && "A concrete function must be provided to this routine.");
-
-    if (NumArgs < 0)
-      // Set the argument number to the number of explicit arguments in the
-      // function.
-      NumArgs = F->arg_size();
-
-    if (Intrinsic::ID IID = F->getIntrinsicID()) {
-      FunctionType *FTy = F->getFunctionType();
-      SmallVector<Type *, 8> ParamTys(FTy->param_begin(), FTy->param_end());
-      return static_cast<T *>(this)
-          ->getIntrinsicCost(IID, FTy->getReturnType(), ParamTys, U);
-    }
-
-    if (!static_cast<T *>(this)->isLoweredToCall(F))
-      return TTI::TCC_Basic; // Give a basic cost if it will be lowered
-                             // directly.
-
-    return static_cast<T *>(this)->getCallCost(F->getFunctionType(), NumArgs, U);
-  }
-
-  unsigned getCallCost(const Function *F, ArrayRef<const Value *> Arguments,
-                       const User *U) {
-    // Simply delegate to generic handling of the call.
-    // FIXME: We should use instsimplify or something else to catch calls which
-    // will constant fold with these arguments.
-    return static_cast<T *>(this)->getCallCost(F, Arguments.size(), U);
-  }
 
   using BaseT::getGEPCost;
 
@@ -898,15 +856,19 @@ class TargetTransformInfoImplCRTPBase : public TargetTransformInfoImplBase {
 
     if (auto CS = ImmutableCallSite(U)) {
       const Function *F = CS.getCalledFunction();
-      if (!F) {
-        // Just use the called value type.
-        Type *FTy = CS.getCalledValue()->getType()->getPointerElementType();
-        return TargetTTI->getCallCost(cast<FunctionType>(FTy),
-                                      CS.arg_size(), U);
-      }
+      if (F) {
+        FunctionType *FTy = F->getFunctionType();
+        if (Intrinsic::ID IID = F->getIntrinsicID()) {
+          SmallVector<Type *, 8> ParamTys(FTy->param_begin(), FTy->param_end());
+          return TargetTTI->getIntrinsicCost(IID, FTy->getReturnType(), ParamTys, U);
+        }
 
-      SmallVector<const Value *, 8> Arguments(CS.arg_begin(), CS.arg_end());
-      return TargetTTI->getCallCost(F, Arguments, U);
+        if (!TargetTTI->isLoweredToCall(F))
+          return TTI::TCC_Basic; // Give a basic cost if it will be lowered
+
+        return TTI::TCC_Basic * (FTy->getNumParams() + 1);
+      }
+      return TTI::TCC_Basic * (CS.arg_size() + 1);
     }
 
     if (isa<SExtInst>(U) || isa<ZExtInst>(U) || isa<FPExtInst>(U))
diff --git a/llvm/include/llvm/Analysis/ValueLattice.h b/llvm/include/llvm/Analysis/ValueLattice.h
index 2810ba5a3fc3a..b9ee103eb606c 100644
--- a/llvm/include/llvm/Analysis/ValueLattice.h
+++ b/llvm/include/llvm/Analysis/ValueLattice.h
@@ -37,7 +37,7 @@ class ValueLatticeElement {
     /// assuming all uses of the result will be replaced.
     /// Transition allowed to the following states:
     ///  constant
-    ///  singlecrfromundef
+    ///  constantrange_including_undef
     ///  overdefined
     undef,
 
@@ -59,20 +59,20 @@ class ValueLatticeElement {
     /// The Value falls within this range. (Used only for integer typed values.)
     /// Transition allowed to the following states:
     ///  constantrange (new range must be a superset of the existing range)
-    ///  singlecrfromundef (range must stay a single element range)
+    ///  constantrange_including_undef
     ///  overdefined
     constantrange,
 
-    /// This Value contains a single element constant range that was merged with
-    /// an Undef value. Merging it with other constant ranges results in
-    /// overdefined, unless they match the single element constant range.
+    /// This Value falls within this range, but also may be undef.
+    /// Merging it with other constant ranges results in
+    /// constantrange_including_undef.
     /// Transition allowed to the following states:
     ///  overdefined
-    singlecrfromundef,
+    constantrange_including_undef,
 
     /// We can not precisely model the dynamic values this value might take.
     /// No transitions are allowed after reaching overdefined.
-    overdefined
+    overdefined,
   };
 
   ValueLatticeElementTy Tag;
@@ -97,9 +97,9 @@ class ValueLatticeElement {
     case unknown:
     case undef:
     case constant:
-    case singlecrfromundef:
     case notconstant:
       break;
+    case constantrange_including_undef:
     case constantrange:
       Range.~ConstantRange();
       break;
@@ -128,7 +128,7 @@ class ValueLatticeElement {
 
     switch (Other.Tag) {
     case constantrange:
-    case singlecrfromundef:
+    case constantrange_including_undef:
       if (!isConstantRange())
         new (&Range) ConstantRange(Other.Range);
       else
@@ -161,12 +161,13 @@ class ValueLatticeElement {
     Res.markNotConstant(C);
     return Res;
   }
-  static ValueLatticeElement getRange(ConstantRange CR) {
+  static ValueLatticeElement getRange(ConstantRange CR,
+                                      bool MayIncludeUndef = false) {
     if (CR.isFullSet())
       return getOverdefined();
 
     ValueLatticeElement Res;
-    Res.markConstantRange(std::move(CR));
+    Res.markConstantRange(std::move(CR), MayIncludeUndef);
     return Res;
   }
   static ValueLatticeElement getOverdefined() {
@@ -179,10 +180,17 @@ class ValueLatticeElement {
   bool isUnknown() const { return Tag == unknown; }
   bool isUnknownOrUndef() const { return Tag == unknown || Tag == undef; }
   bool isConstant() const { return Tag == constant; }
-  bool isSingleCRFromUndef() const { return Tag == singlecrfromundef; }
   bool isNotConstant() const { return Tag == notconstant; }
-  bool isConstantRange() const {
-    return Tag == constantrange || Tag == singlecrfromundef;
+  bool isConstantRangeIncludingUndef() const {
+    return Tag == constantrange_including_undef;
+  }
+  /// Returns true if this value is a constant range. Use \p UndefAllowed to
+  /// exclude non-singleton constant ranges that may also be undef. Note that
+  /// this function also returns true if the range may include undef, but only
+  /// contains a single element. In that case, it can be replaced by a constant.
+  bool isConstantRange(bool UndefAllowed = true) const {
+    return Tag == constantrange || (Tag == constantrange_including_undef &&
+                                    (UndefAllowed || Range.isSingleElement()));
   }
   bool isOverdefined() const { return Tag == overdefined; }
 
@@ -196,8 +204,12 @@ class ValueLatticeElement {
     return ConstVal;
   }
 
-  const ConstantRange &getConstantRange() const {
-    assert(isConstantRange() &&
+  /// Returns the constant range for this value. Use \p UndefAllowed to exclude
+  /// non-singleton constant ranges that may also be undef. Note that this
+  /// function also returns a range if the range may include undef, but only
+  /// contains a single element. In that case, it can be replaced by a constant.
+  const ConstantRange &getConstantRange(bool UndefAllowed = true) const {
+    assert(isConstantRange(UndefAllowed) &&
            "Cannot get the constant-range of a non-constant-range!");
     return Range;
   }
@@ -231,7 +243,7 @@ class ValueLatticeElement {
     return true;
   }
 
-  bool markConstant(Constant *V) {
+  bool markConstant(Constant *V, bool MayIncludeUndef = false) {
     if (isa<UndefValue>(V))
       return markUndef();
 
@@ -241,7 +253,7 @@ class ValueLatticeElement {
     }
 
     if (ConstantInt *CI = dyn_cast<ConstantInt>(V))
-      return markConstantRange(ConstantRange(CI->getValue()));
+      return markConstantRange(ConstantRange(CI->getValue()), MayIncludeUndef);
 
     assert(isUnknown() || isUndef());
     Tag = constant;
@@ -271,29 +283,38 @@ class ValueLatticeElement {
 
   /// Mark the object as constant range with \p NewR. If the object is already a
   /// constant range, nothing changes if the existing range is equal to \p
-  /// NewR. Otherwise \p NewR must be a superset of the existing range or the
-  /// object must be undef.
-  bool markConstantRange(ConstantRange NewR) {
-    if (isConstantRange()) {
-      if (getConstantRange() == NewR)
-        return false;
-
-      assert(!isSingleCRFromUndef());
+  /// NewR and the tag. Otherwise \p NewR must be a superset of the existing
+  /// range or the object must be undef. The tag is set to
+  /// constant_range_including_undef if either the existing value or the new
+  /// range may include undef.
+  bool markConstantRange(ConstantRange NewR, bool MayIncludeUndef = false) {
+    if (NewR.isFullSet())
+      return markOverdefined();
 
+    ValueLatticeElementTy OldTag = Tag;
+    ValueLatticeElementTy NewTag =
+        (isUndef() || isConstantRangeIncludingUndef() || MayIncludeUndef)
+            ? constantrange_including_undef
+            : constantrange;
+    if (isConstantRange()) {
       if (NewR.isEmptySet())
         return markOverdefined();
 
+      Tag = NewTag;
+      if (getConstantRange() == NewR)
+        return Tag != OldTag;
+
       assert(NewR.contains(getConstantRange()) &&
              "Existing range must be a subset of NewR");
       Range = std::move(NewR);
       return true;
     }
 
-    assert(isUnknown() || (isUndef() && NewR.isSingleElement()));
+    assert(isUnknown() || isUndef());
     if (NewR.isEmptySet())
       return markOverdefined();
 
-    Tag = isUnknown() ? constantrange : singlecrfromundef;
+    Tag = NewTag;
     new (&Range) ConstantRange(std::move(NewR));
     return true;
   }
@@ -313,9 +334,10 @@ class ValueLatticeElement {
       if (RHS.isUndef())
         return false;
       if (RHS.isConstant())
-        return markConstant(RHS.getConstant());
-      if (RHS.isConstantRange() && RHS.getConstantRange().isSingleElement())
-        return markConstantRange(RHS.getConstantRange());
+        return markConstant(RHS.getConstant(), /*MayIncludeUndef=*/true);
+      if (RHS.isConstantRange())
+        return markConstantRange(RHS.getConstantRange(true),
+                                 /*MayIncludeUndef=*/true);
       return markOverdefined();
     }
 
@@ -341,9 +363,12 @@ class ValueLatticeElement {
       return true;
     }
 
+    auto OldTag = Tag;
     assert(isConstantRange() && "New ValueLattice type?");
-    if (RHS.isUndef() && getConstantRange().isSingleElement())
-      return false;
+    if (RHS.isUndef()) {
+      Tag = constantrange_including_undef;
+      return OldTag != Tag;
+    }
 
     if (!RHS.isConstantRange()) {
       // We can get here if we've encountered a constantexpr of integer type
@@ -353,21 +378,9 @@ class ValueLatticeElement {
     }
 
     ConstantRange NewR = getConstantRange().unionWith(RHS.getConstantRange());
-
-    if (isSingleCRFromUndef() || RHS.isSingleCRFromUndef()) {
-      if (NewR.isSingleElement()) {
-        assert(getConstantRange() == NewR);
-        return false;
-      }
-      markOverdefined();
-      return true;
-    }
-    if (NewR.isFullSet())
-      return markOverdefined();
-    else if (NewR == getConstantRange())
-      return false;
-    else
-      return markConstantRange(std::move(NewR));
+    return markConstantRange(
+        std::move(NewR),
+        /*MayIncludeUndef=*/RHS.isConstantRangeIncludingUndef());
   }
 
   // Compares this symbolic value with Other using Pred and returns either
diff --git a/llvm/include/llvm/Analysis/VecFuncs.def b/llvm/include/llvm/Analysis/VecFuncs.def
index 86bec0be7546d..2f64b0fedc7aa 100644
--- a/llvm/include/llvm/Analysis/VecFuncs.def
+++ b/llvm/include/llvm/Analysis/VecFuncs.def
@@ -245,6 +245,29 @@ TLI_DEFINE_VECFUNC("llvm.log.f32", "__svml_logf4", 4)
 TLI_DEFINE_VECFUNC("llvm.log.f32", "__svml_logf8", 8)
 TLI_DEFINE_VECFUNC("llvm.log.f32", "__svml_logf16", 16)
 
+TLI_DEFINE_VECFUNC("exp2", "__svml_exp22", 2)
+TLI_DEFINE_VECFUNC("exp2", "__svml_exp24", 4)
+TLI_DEFINE_VECFUNC("exp2", "__svml_exp28", 8)
+
+TLI_DEFINE_VECFUNC("exp2f", "__svml_exp2f4", 4)
+TLI_DEFINE_VECFUNC("exp2f", "__svml_exp2f8", 8)
+TLI_DEFINE_VECFUNC("exp2f", "__svml_exp2f16", 16)
+
+TLI_DEFINE_VECFUNC("llvm.exp2.f64", "__svml_exp22", 2)
+TLI_DEFINE_VECFUNC("llvm.exp2.f64", "__svml_exp24", 4)
+TLI_DEFINE_VECFUNC("llvm.exp2.f64", "__svml_exp28", 8)
+
+TLI_DEFINE_VECFUNC("llvm.exp2.f32", "__svml_exp2f4", 4)
+TLI_DEFINE_VECFUNC("llvm.exp2.f32", "__svml_exp2f8", 8)
+TLI_DEFINE_VECFUNC("llvm.exp2.f32", "__svml_exp2f16", 16)
+
+TLI_DEFINE_VECFUNC("__exp2_finite", "__svml_exp22", 2)
+TLI_DEFINE_VECFUNC("__exp2_finite", "__svml_exp24", 4)
+TLI_DEFINE_VECFUNC("__exp2_finite", "__svml_exp28", 8)
+
+TLI_DEFINE_VECFUNC("__exp2f_finite", "__svml_exp2f4", 4)
+TLI_DEFINE_VECFUNC("__exp2f_finite", "__svml_exp2f8", 8)
+TLI_DEFINE_VECFUNC("__exp2f_finite", "__svml_exp2f16", 16)
 
 #else
 #error "Must choose which vector library functions are to be defined."
diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h
index 6797ed2369d84..36aea31365c27 100644
--- a/llvm/include/llvm/Analysis/VectorUtils.h
+++ b/llvm/include/llvm/Analysis/VectorUtils.h
@@ -339,22 +339,8 @@ bool isSplatValue(const Value *V, int Index = -1, unsigned Depth = 0);
 ///
 /// This is the reverse process of "canWidenShuffleElements", but can always
 /// succeed.
-template <typename T>
-void scaleShuffleMask(size_t Scale, ArrayRef<T> Mask,
-                      SmallVectorImpl<T> &ScaledMask) {
-  assert(Scale > 0 && "Unexpected scaling factor");
-
-  // Fast-path: if no scaling, then it is just a copy.
-  if (Scale == 1) {
-    ScaledMask.assign(Mask.begin(), Mask.end());
-    return;
-  }
-
-  ScaledMask.clear();
-  for (int MaskElt : Mask)
-    for (int ScaleElt = 0; ScaleElt != (int)Scale; ++ScaleElt)
-      ScaledMask.push_back(MaskElt < 0 ? MaskElt : Scale * MaskElt + ScaleElt);
-}
+void scaleShuffleMask(size_t Scale, ArrayRef<int> Mask,
+                      SmallVectorImpl<int> &ScaledMask);
 
 /// Compute a map of integer instructions to their minimum legal type
 /// size.
diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h
index a943f553e18cd..9bf5a3c99a554 100644
--- a/llvm/include/llvm/BinaryFormat/ELF.h
+++ b/llvm/include/llvm/BinaryFormat/ELF.h
@@ -816,50 +816,51 @@ enum {
 
 // Section types.
 enum : unsigned {
-  SHT_NULL = 0,                         // No associated section (inactive entry).
-  SHT_PROGBITS = 1,                     // Program-defined contents.
-  SHT_SYMTAB = 2,                       // Symbol table.
-  SHT_STRTAB = 3,                       // String table.
-  SHT_RELA = 4,                         // Relocation entries; explicit addends.
-  SHT_HASH = 5,                         // Symbol hash table.
-  SHT_DYNAMIC = 6,                      // Information for dynamic linking.
-  SHT_NOTE = 7,                         // Information about the file.
-  SHT_NOBITS = 8,                       // Data occupies no space in the file.
-  SHT_REL = 9,                          // Relocation entries; no explicit addends.
-  SHT_SHLIB = 10,                       // Reserved.
-  SHT_DYNSYM = 11,                      // Symbol table.
-  SHT_INIT_ARRAY = 14,                  // Pointers to initialization functions.
-  SHT_FINI_ARRAY = 15,                  // Pointers to termination functions.
-  SHT_PREINIT_ARRAY = 16,               // Pointers to pre-init functions.
-  SHT_GROUP = 17,                       // Section group.
-  SHT_SYMTAB_SHNDX = 18,                // Indices for SHN_XINDEX entries.
+  SHT_NULL = 0,           // No associated section (inactive entry).
+  SHT_PROGBITS = 1,       // Program-defined contents.
+  SHT_SYMTAB = 2,         // Symbol table.
+  SHT_STRTAB = 3,         // String table.
+  SHT_RELA = 4,           // Relocation entries; explicit addends.
+  SHT_HASH = 5,           // Symbol hash table.
+  SHT_DYNAMIC = 6,        // Information for dynamic linking.
+  SHT_NOTE = 7,           // Information about the file.
+  SHT_NOBITS = 8,         // Data occupies no space in the file.
+  SHT_REL = 9,            // Relocation entries; no explicit addends.
+  SHT_SHLIB = 10,         // Reserved.
+  SHT_DYNSYM = 11,        // Symbol table.
+  SHT_INIT_ARRAY = 14,    // Pointers to initialization functions.
+  SHT_FINI_ARRAY = 15,    // Pointers to termination functions.
+  SHT_PREINIT_ARRAY = 16, // Pointers to pre-init functions.
+  SHT_GROUP = 17,         // Section group.
+  SHT_SYMTAB_SHNDX = 18,  // Indices for SHN_XINDEX entries.
   // Experimental support for SHT_RELR sections. For details, see proposal
   // at https://groups.google.com/forum/#!topic/generic-abi/bX460iggiKg
-  SHT_RELR = 19,                        // Relocation entries; only offsets.
-  SHT_LOOS = 0x60000000,                // Lowest operating system-specific type.
+  SHT_RELR = 19,         // Relocation entries; only offsets.
+  SHT_LOOS = 0x60000000, // Lowest operating system-specific type.
   // Android packed relocation section types.
   // https://android.googlesource.com/platform/bionic/+/6f12bfece5dcc01325e0abba56a46b1bcf991c69/tools/relocation_packer/src/elf_file.cc#37
   SHT_ANDROID_REL = 0x60000001,
   SHT_ANDROID_RELA = 0x60000002,
-  SHT_LLVM_ODRTAB = 0x6fff4c00,         // LLVM ODR table.
-  SHT_LLVM_LINKER_OPTIONS = 0x6fff4c01, // LLVM Linker Options.
+  SHT_LLVM_ODRTAB = 0x6fff4c00,             // LLVM ODR table.
+  SHT_LLVM_LINKER_OPTIONS = 0x6fff4c01,     // LLVM Linker Options.
   SHT_LLVM_CALL_GRAPH_PROFILE = 0x6fff4c02, // LLVM Call Graph Profile.
-  SHT_LLVM_ADDRSIG = 0x6fff4c03,        // List of address-significant symbols
-                                        // for safe ICF.
-  SHT_LLVM_DEPENDENT_LIBRARIES = 0x6fff4c04, // LLVM Dependent Library Specifiers.
-  SHT_LLVM_SYMPART = 0x6fff4c05,        // Symbol partition specification.
-  SHT_LLVM_PART_EHDR = 0x6fff4c06,      // ELF header for loadable partition.
-  SHT_LLVM_PART_PHDR = 0x6fff4c07,      // Phdrs for loadable partition.
+  SHT_LLVM_ADDRSIG = 0x6fff4c03, // List of address-significant symbols
+                                 // for safe ICF.
+  SHT_LLVM_DEPENDENT_LIBRARIES =
+      0x6fff4c04,                  // LLVM Dependent Library Specifiers.
+  SHT_LLVM_SYMPART = 0x6fff4c05,   // Symbol partition specification.
+  SHT_LLVM_PART_EHDR = 0x6fff4c06, // ELF header for loadable partition.
+  SHT_LLVM_PART_PHDR = 0x6fff4c07, // Phdrs for loadable partition.
   // Android's experimental support for SHT_RELR sections.
   // https://android.googlesource.com/platform/bionic/+/b7feec74547f84559a1467aca02708ff61346d2a/libc/include/elf.h#512
-  SHT_ANDROID_RELR = 0x6fffff00,        // Relocation entries; only offsets.
-  SHT_GNU_ATTRIBUTES = 0x6ffffff5,      // Object attributes.
-  SHT_GNU_HASH = 0x6ffffff6,            // GNU-style hash table.
-  SHT_GNU_verdef = 0x6ffffffd,          // GNU version definitions.
-  SHT_GNU_verneed = 0x6ffffffe,         // GNU version references.
-  SHT_GNU_versym = 0x6fffffff,          // GNU symbol versions table.
-  SHT_HIOS = 0x6fffffff,                // Highest operating system-specific type.
-  SHT_LOPROC = 0x70000000,              // Lowest processor arch-specific type.
+  SHT_ANDROID_RELR = 0x6fffff00,   // Relocation entries; only offsets.
+  SHT_GNU_ATTRIBUTES = 0x6ffffff5, // Object attributes.
+  SHT_GNU_HASH = 0x6ffffff6,       // GNU-style hash table.
+  SHT_GNU_verdef = 0x6ffffffd,     // GNU version definitions.
+  SHT_GNU_verneed = 0x6ffffffe,    // GNU version references.
+  SHT_GNU_versym = 0x6fffffff,     // GNU symbol versions table.
+  SHT_HIOS = 0x6fffffff,           // Highest operating system-specific type.
+  SHT_LOPROC = 0x70000000,         // Lowest processor arch-specific type.
   // Fixme: All this is duplicated in MCSectionELF. Why??
   // Exception Index table
   SHT_ARM_EXIDX = 0x70000001U,
@@ -869,20 +870,22 @@ enum : unsigned {
   SHT_ARM_ATTRIBUTES = 0x70000003U,
   SHT_ARM_DEBUGOVERLAY = 0x70000004U,
   SHT_ARM_OVERLAYSECTION = 0x70000005U,
-  SHT_HEX_ORDERED = 0x70000000,         // Link editor is to sort the entries in
-                                        // this section based on their sizes
-  SHT_X86_64_UNWIND = 0x70000001,       // Unwind information
+  SHT_HEX_ORDERED = 0x70000000,   // Link editor is to sort the entries in
+                                  // this section based on their sizes
+  SHT_X86_64_UNWIND = 0x70000001, // Unwind information
 
-  SHT_MIPS_REGINFO = 0x70000006,        // Register usage information
-  SHT_MIPS_OPTIONS = 0x7000000d,        // General options
-  SHT_MIPS_DWARF = 0x7000001e,          // DWARF debugging section.
-  SHT_MIPS_ABIFLAGS = 0x7000002a,       // ABI information.
+  SHT_MIPS_REGINFO = 0x70000006,  // Register usage information
+  SHT_MIPS_OPTIONS = 0x7000000d,  // General options
+  SHT_MIPS_DWARF = 0x7000001e,    // DWARF debugging section.
+  SHT_MIPS_ABIFLAGS = 0x7000002a, // ABI information.
 
   SHT_MSP430_ATTRIBUTES = 0x70000003U,
 
-  SHT_HIPROC = 0x7fffffff,              // Highest processor arch-specific type.
-  SHT_LOUSER = 0x80000000,              // Lowest type reserved for applications.
-  SHT_HIUSER = 0xffffffff               // Highest type reserved for applications.
+  SHT_RISCV_ATTRIBUTES = 0x70000003U,
+
+  SHT_HIPROC = 0x7fffffff, // Highest processor arch-specific type.
+  SHT_LOUSER = 0x80000000, // Lowest type reserved for applications.
+  SHT_HIUSER = 0xffffffff  // Highest type reserved for applications.
 };
 
 // Section flags.
diff --git a/llvm/include/llvm/BinaryFormat/Wasm.h b/llvm/include/llvm/BinaryFormat/Wasm.h
index e39760e59ec25..dd743dbbbefa6 100644
--- a/llvm/include/llvm/BinaryFormat/Wasm.h
+++ b/llvm/include/llvm/BinaryFormat/Wasm.h
@@ -132,7 +132,7 @@ struct WasmFunction {
   uint32_t CodeSectionOffset;
   uint32_t Size;
   uint32_t CodeOffset;  // start of Locals and Body
-  StringRef ExportName; // from the "export" section
+  Optional<StringRef> ExportName; // from the "export" section
   StringRef SymbolName; // from the "linking" section
   StringRef DebugName;  // from the "name" section
   uint32_t Comdat;      // from the "comdat info" section
diff --git a/llvm/include/llvm/BinaryFormat/XCOFF.h b/llvm/include/llvm/BinaryFormat/XCOFF.h
index b58bcdc488223..f10fcb7b2a62f 100644
--- a/llvm/include/llvm/BinaryFormat/XCOFF.h
+++ b/llvm/include/llvm/BinaryFormat/XCOFF.h
@@ -258,6 +258,7 @@ enum CFileCpuId : uint8_t {
 };
 
 StringRef getMappingClassString(XCOFF::StorageMappingClass SMC);
+StringRef getRelocationTypeString(XCOFF::RelocationType Type);
 
 } // end namespace XCOFF
 } // end namespace llvm
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 3b1658929c34f..8a13fd8419b8a 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -551,14 +551,22 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     return getST()->getPrefetchDistance();
   }
 
-  virtual unsigned getMinPrefetchStride() const {
-    return getST()->getMinPrefetchStride();
+  virtual unsigned getMinPrefetchStride(unsigned NumMemAccesses,
+                                        unsigned NumStridedMemAccesses,
+                                        unsigned NumPrefetches,
+                                        bool HasCall) const {
+    return getST()->getMinPrefetchStride(NumMemAccesses, NumStridedMemAccesses,
+                                         NumPrefetches, HasCall);
   }
 
   virtual unsigned getMaxPrefetchIterationsAhead() const {
     return getST()->getMaxPrefetchIterationsAhead();
   }
 
+  virtual bool enableWritePrefetching() const {
+    return getST()->enableWritePrefetching();
+  }
+
   /// @}
 
   /// \name Vector TTI Implementations
@@ -1340,12 +1348,12 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     case Intrinsic::experimental_vector_reduce_fmin:
       return ConcreteTTI->getMinMaxReductionCost(
           Tys[0], CmpInst::makeCmpResultType(Tys[0]), /*IsPairwiseForm=*/false,
-          /*IsUnsigned=*/true);
+          /*IsUnsigned=*/false);
     case Intrinsic::experimental_vector_reduce_umax:
     case Intrinsic::experimental_vector_reduce_umin:
       return ConcreteTTI->getMinMaxReductionCost(
           Tys[0], CmpInst::makeCmpResultType(Tys[0]), /*IsPairwiseForm=*/false,
-          /*IsUnsigned=*/false);
+          /*IsUnsigned=*/true);
     case Intrinsic::sadd_sat:
     case Intrinsic::ssub_sat: {
       Type *CondTy = RetTy->getWithNewBitWidth(1);
diff --git a/llvm/include/llvm/CodeGen/CommandFlags.h b/llvm/include/llvm/CodeGen/CommandFlags.h
index cdec26879f732..c017652b469bb 100644
--- a/llvm/include/llvm/CodeGen/CommandFlags.h
+++ b/llvm/include/llvm/CodeGen/CommandFlags.h
@@ -12,6 +12,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/ADT/FloatingPointMode.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/Intrinsics.h"
@@ -62,7 +63,8 @@ bool getEnableNoSignedZerosFPMath();
 
 bool getEnableNoTrappingFPMath();
 
-llvm::FPDenormal::DenormalMode getDenormalFPMath();
+DenormalMode::DenormalModeKind getDenormalFPMath();
+DenormalMode::DenormalModeKind getDenormalFP32Math();
 
 bool getEnableHonorSignDependentRoundingFPMath();
 
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index f97c22ac420ad..f01cd36751afe 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -62,6 +62,10 @@ class CombinerHelper {
                  GISelKnownBits *KB = nullptr,
                  MachineDominatorTree *MDT = nullptr);
 
+  GISelKnownBits *getKnownBits() const {
+    return KB;
+  }
+
   /// MachineRegisterInfo::replaceRegWith() and inform the observer of the changes
   void replaceRegWith(MachineRegisterInfo &MRI, Register FromReg, Register ToReg) const;
 
@@ -217,9 +221,16 @@ class CombinerHelper {
   /// equivalent instructions.
   bool matchEqualDefs(const MachineOperand &MOP1, const MachineOperand &MOP2);
 
+  /// Return true if \p MOP is defined by a G_CONSTANT with a value equal to
+  /// \p C.
+  bool matchConstantOp(const MachineOperand &MOP, int64_t C);
+
   /// Optimize (cond ? x : x) -> x
   bool matchSelectSameVal(MachineInstr &MI);
 
+  /// Optimize (x op x) -> x
+  bool matchBinOpSameVal(MachineInstr &MI);
+
   /// Try to transform \p MI by using all of the above
   /// combine functions. Returns true if changed.
   bool tryCombine(MachineInstr &MI);
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
index 6a2ea05f1b088..a4aa4a7cbd691 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
@@ -582,7 +582,16 @@ class IRTranslator : public MachineFunctionPass {
   /// Get the alignment of the given memory operation instruction. This will
   /// either be the explicitly specified value or the ABI-required alignment for
   /// the type being accessed (according to the Module's DataLayout).
-  unsigned getMemOpAlignment(const Instruction &I);
+  LLVM_ATTRIBUTE_DEPRECATED(
+      inline unsigned getMemOpAlignment(const Instruction &I),
+      "Use getMemOpAlign instead") {
+    return getMemOpAlign(I).value();
+  }
+
+  /// Get the alignment of the given memory operation instruction. This will
+  /// either be the explicitly specified value or the ABI-required alignment for
+  /// the type being accessed (according to the Module's DataLayout).
+  Align getMemOpAlign(const Instruction &I);
 
   /// Get the MachineBasicBlock that represents \p BB. Specifically, the block
   /// returned will be the head of the translated block (suitable for branch
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h
index eea39fc0fe627..9e9e1806bc6fe 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h
@@ -429,7 +429,7 @@ bool InstructionSelector::executeMatchTable(
                       dbgs() << CurrentIdx << ": GIM_CheckMemoryAlignment"
                       << "(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
                       << ")->getAlignment() >= " << MinAlign << ")\n");
-      if (MMO->getAlignment() < MinAlign && handleReject() == RejectAndGiveUp)
+      if (MMO->getAlign() < MinAlign && handleReject() == RejectAndGiveUp)
         return false;
 
       break;
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
index 60c0a71762e51..823d1eb329420 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
@@ -304,6 +304,13 @@ class LegalizerHelper {
   GISelChangeObserver &Observer;
 };
 
+/// Helper function that creates a libcall to the given \p Name using the given
+/// calling convention \p CC.
+LegalizerHelper::LegalizeResult
+createLibcall(MachineIRBuilder &MIRBuilder, const char *Name,
+              const CallLowering::ArgInfo &Result,
+              ArrayRef<CallLowering::ArgInfo> Args, CallingConv::ID CC);
+
 /// Helper function that creates the given libcall.
 LegalizerHelper::LegalizeResult
 createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h b/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h
index 65b1948a19679..1fae635f48fd3 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h
@@ -241,6 +241,12 @@ inline BinaryOp_match<LHS, RHS, TargetOpcode::G_OR, true> m_GOr(const LHS &L,
   return BinaryOp_match<LHS, RHS, TargetOpcode::G_OR, true>(L, R);
 }
 
+template <typename LHS, typename RHS>
+inline BinaryOp_match<LHS, RHS, TargetOpcode::G_SHL, false>
+m_GShl(const LHS &L, const RHS &R) {
+  return BinaryOp_match<LHS, RHS, TargetOpcode::G_SHL, false>(L, R);
+}
+
 template <typename LHS, typename RHS>
 inline BinaryOp_match<LHS, RHS, TargetOpcode::G_LSHR, false>
 m_GLShr(const LHS &L, const RHS &R) {
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index 59c2aa898d4fd..c163c310672bc 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -379,7 +379,14 @@ class MachineIRBuilder {
   ///
   /// \return a MachineInstrBuilder for the newly created instruction.
   MachineInstrBuilder buildDynStackAlloc(const DstOp &Res, const SrcOp &Size,
-                                         unsigned Align);
+                                         Align Alignment);
+
+  LLVM_ATTRIBUTE_DEPRECATED(inline MachineInstrBuilder buildDynStackAlloc(
+                                const DstOp &Res, const SrcOp &Size,
+                                unsigned Align),
+                            "Use the version that takes MaybeAlign instead") {
+    return buildDynStackAlloc(Res, Size, assumeAligned(Align));
+  }
 
   /// Build and insert \p Res = G_FRAME_INDEX \p Idx
   ///
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
index c30f4db1cc8c0..956f3b444f607 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
@@ -16,6 +16,7 @@
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/CodeGen/Register.h"
+#include "llvm/Support/Alignment.h"
 #include "llvm/Support/LowLevelTypeImpl.h"
 #include "llvm/Support/MachineValueType.h"
 
@@ -181,8 +182,7 @@ inline bool isKnownNeverSNaN(Register Val, const MachineRegisterInfo &MRI) {
   return isKnownNeverNaN(Val, MRI, true);
 }
 
-unsigned inferAlignmentFromPtrInfo(MachineFunction &MF,
-                                   const MachinePointerInfo &MPO);
+Align inferAlignFromPtrInfo(MachineFunction &MF, const MachinePointerInfo &MPO);
 
 /// Return the least common multiple type of \p Ty0 and \p Ty1, by changing
 /// the number of vector elements or scalar bitwidth. The intent is a
diff --git a/llvm/include/llvm/CodeGen/MIRYamlMapping.h b/llvm/include/llvm/CodeGen/MIRYamlMapping.h
index 069d0aa45095b..c68b073ebb8c5 100644
--- a/llvm/include/llvm/CodeGen/MIRYamlMapping.h
+++ b/llvm/include/llvm/CodeGen/MIRYamlMapping.h
@@ -142,6 +142,23 @@ template <> struct ScalarEnumerationTraits<MachineJumpTableInfo::JTEntryKind> {
   }
 };
 
+template <> struct ScalarTraits<MaybeAlign> {
+  static void output(const MaybeAlign &Alignment, void *,
+                     llvm::raw_ostream &out) {
+    out << uint64_t(Alignment ? Alignment->value() : 0U);
+  }
+  static StringRef input(StringRef Scalar, void *, MaybeAlign &Alignment) {
+    unsigned long long n;
+    if (getAsUnsignedInteger(Scalar, 10, n))
+      return "invalid number";
+    if (n > 0 && !isPowerOf2_64(n))
+      return "must be 0 or a power of two";
+    Alignment = MaybeAlign(n);
+    return StringRef();
+  }
+  static QuotingType mustQuote(StringRef) { return QuotingType::None; }
+};
+
 } // end namespace yaml
 } // end namespace llvm
 
@@ -212,7 +229,7 @@ struct MachineStackObject {
   ObjectType Type = DefaultType;
   int64_t Offset = 0;
   uint64_t Size = 0;
-  unsigned Alignment = 0;
+  MaybeAlign Alignment = None;
   TargetStackID::Value StackID;
   StringValue CalleeSavedRegister;
   bool CalleeSavedRestored = true;
@@ -252,7 +269,7 @@ template <> struct MappingTraits<MachineStackObject> {
     YamlIO.mapOptional("offset", Object.Offset, (int64_t)0);
     if (Object.Type != MachineStackObject::VariableSized)
       YamlIO.mapRequired("size", Object.Size);
-    YamlIO.mapOptional("alignment", Object.Alignment, (unsigned)0);
+    YamlIO.mapOptional("alignment", Object.Alignment, None);
     YamlIO.mapOptional("stack-id", Object.StackID, TargetStackID::Default);
     YamlIO.mapOptional("callee-saved-register", Object.CalleeSavedRegister,
                        StringValue()); // Don't print it out when it's empty.
@@ -278,7 +295,7 @@ struct FixedMachineStackObject {
   ObjectType Type = DefaultType;
   int64_t Offset = 0;
   uint64_t Size = 0;
-  unsigned Alignment = 0;
+  MaybeAlign Alignment = None;
   TargetStackID::Value StackID;
   bool IsImmutable = false;
   bool IsAliased = false;
@@ -327,7 +344,7 @@ template <> struct MappingTraits<FixedMachineStackObject> {
         FixedMachineStackObject::DefaultType); // Don't print the default type.
     YamlIO.mapOptional("offset", Object.Offset, (int64_t)0);
     YamlIO.mapOptional("size", Object.Size, (uint64_t)0);
-    YamlIO.mapOptional("alignment", Object.Alignment, (unsigned)0);
+    YamlIO.mapOptional("alignment", Object.Alignment, None);
     YamlIO.mapOptional("stack-id", Object.StackID, TargetStackID::Default);
     if (Object.Type != FixedMachineStackObject::SpillSlot) {
       YamlIO.mapOptional("isImmutable", Object.IsImmutable, false);
@@ -411,7 +428,7 @@ template <> struct MappingTraits<CallSiteInfo> {
 struct MachineConstantPoolValue {
   UnsignedValue ID;
   StringValue Value;
-  unsigned Alignment = 0;
+  MaybeAlign Alignment = None;
   bool IsTargetSpecific = false;
 
   bool operator==(const MachineConstantPoolValue &Other) const {
@@ -425,7 +442,7 @@ template <> struct MappingTraits<MachineConstantPoolValue> {
   static void mapping(IO &YamlIO, MachineConstantPoolValue &Constant) {
     YamlIO.mapRequired("id", Constant.ID);
     YamlIO.mapOptional("value", Constant.Value, StringValue());
-    YamlIO.mapOptional("alignment", Constant.Alignment, (unsigned)0);
+    YamlIO.mapOptional("alignment", Constant.Alignment, None);
     YamlIO.mapOptional("isTargetSpecific", Constant.IsTargetSpecific, false);
   }
 };
@@ -571,7 +588,7 @@ template <> struct MappingTraits<std::unique_ptr<MachineFunctionInfo>> {
 
 struct MachineFunction {
   StringRef Name;
-  unsigned Alignment = 0;
+  MaybeAlign Alignment = None;
   bool ExposesReturnsTwice = false;
   // GISel MachineFunctionProperties.
   bool Legalized = false;
@@ -599,7 +616,7 @@ struct MachineFunction {
 template <> struct MappingTraits<MachineFunction> {
   static void mapping(IO &YamlIO, MachineFunction &MF) {
     YamlIO.mapRequired("name", MF.Name);
-    YamlIO.mapOptional("alignment", MF.Alignment, (unsigned)0);
+    YamlIO.mapOptional("alignment", MF.Alignment, None);
     YamlIO.mapOptional("exposesReturnsTwice", MF.ExposesReturnsTwice, false);
     YamlIO.mapOptional("legalized", MF.Legalized, false);
     YamlIO.mapOptional("regBankSelected", MF.RegBankSelected, false);
diff --git a/llvm/include/llvm/CodeGen/MachineFrameInfo.h b/llvm/include/llvm/CodeGen/MachineFrameInfo.h
index 03811bc5145c6..c5e57e6da5492 100644
--- a/llvm/include/llvm/CodeGen/MachineFrameInfo.h
+++ b/llvm/include/llvm/CodeGen/MachineFrameInfo.h
@@ -460,12 +460,10 @@ class MachineFrameInfo {
     Objects[ObjectIdx+NumFixedObjects].Size = Size;
   }
 
-  /// Return the alignment of the specified stack object.
-  /// FIXME: Remove this function once transition to Align is over.
-  unsigned getObjectAlignment(int ObjectIdx) const {
-    assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() &&
-           "Invalid Object Idx!");
-    return Objects[ObjectIdx + NumFixedObjects].Alignment.value();
+  LLVM_ATTRIBUTE_DEPRECATED(inline unsigned getObjectAlignment(int ObjectIdx)
+                                const,
+                            "Use getObjectAlign instead") {
+    return getObjectAlign(ObjectIdx).value();
   }
 
   /// Return the alignment of the specified stack object.
@@ -475,18 +473,6 @@ class MachineFrameInfo {
     return Objects[ObjectIdx + NumFixedObjects].Alignment;
   }
 
-  /// setObjectAlignment - Change the alignment of the specified stack object.
-  /// FIXME: Remove this function once transition to Align is over.
-  void setObjectAlignment(int ObjectIdx, unsigned Align) {
-    assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() &&
-           "Invalid Object Idx!");
-    Objects[ObjectIdx + NumFixedObjects].Alignment = assumeAligned(Align);
-
-    // Only ensure max alignment for the default stack.
-    if (getStackID(ObjectIdx) == 0)
-      ensureMaxAlignment(assumeAligned(Align));
-  }
-
   /// setObjectAlignment - Change the alignment of the specified stack object.
   void setObjectAlignment(int ObjectIdx, Align Alignment) {
     assert(unsigned(ObjectIdx + NumFixedObjects) < Objects.size() &&
@@ -498,6 +484,12 @@ class MachineFrameInfo {
       ensureMaxAlignment(Alignment);
   }
 
+  LLVM_ATTRIBUTE_DEPRECATED(inline void setObjectAlignment(int ObjectIdx,
+                                                           unsigned Align),
+                            "Use the version that takes Align instead") {
+    setObjectAlignment(ObjectIdx, assumeAligned(Align));
+  }
+
   /// Return the underlying Alloca of the specified
   /// stack object if it exists. Returns 0 if none exists.
   const AllocaInst* getObjectAllocation(int ObjectIdx) const {
@@ -593,7 +585,7 @@ class MachineFrameInfo {
 
   /// Make sure the function is at least Align bytes aligned.
   void ensureMaxAlignment(Align Alignment);
-  /// FIXME: Remove this once transition to Align is over.
+
   LLVM_ATTRIBUTE_DEPRECATED(inline void ensureMaxAlignment(unsigned Align),
                             "Use the version that uses Align instead") {
     ensureMaxAlignment(assumeAligned(Align));
diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h
index 7b767ff7c7a53..b4a7e99b5521f 100644
--- a/llvm/include/llvm/CodeGen/MachineFunction.h
+++ b/llvm/include/llvm/CodeGen/MachineFunction.h
@@ -813,12 +813,24 @@ class MachineFunction {
   /// explicitly deallocated.
   MachineMemOperand *getMachineMemOperand(
       MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s,
-      unsigned base_alignment, const AAMDNodes &AAInfo = AAMDNodes(),
-      const MDNode *Ranges = nullptr,
-      SyncScope::ID SSID = SyncScope::System,
+      Align base_alignment, const AAMDNodes &AAInfo = AAMDNodes(),
+      const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System,
       AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
       AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
 
+  LLVM_ATTRIBUTE_DEPRECATED(
+      inline MachineMemOperand *getMachineMemOperand(
+          MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s,
+          unsigned base_alignment, const AAMDNodes &AAInfo = AAMDNodes(),
+          const MDNode *Ranges = nullptr,
+          SyncScope::ID SSID = SyncScope::System,
+          AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
+          AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic),
+      "Use the version that takes Align instead") {
+    return getMachineMemOperand(PtrInfo, f, s, Align(base_alignment), AAInfo,
+                                Ranges, SSID, Ordering, FailureOrdering);
+  }
+
   /// getMachineMemOperand - Allocate a new MachineMemOperand by copying
   /// an existing one, adjusting by an offset and using the given size.
   /// MachineMemOperands are owned by the MachineFunction and need not be
diff --git a/llvm/include/llvm/CodeGen/MachineMemOperand.h b/llvm/include/llvm/CodeGen/MachineMemOperand.h
index 7ee700c62a25c..4f0c4aeb94f4a 100644
--- a/llvm/include/llvm/CodeGen/MachineMemOperand.h
+++ b/llvm/include/llvm/CodeGen/MachineMemOperand.h
@@ -169,7 +169,7 @@ class MachineMemOperand {
   MachinePointerInfo PtrInfo;
   uint64_t Size;
   Flags FlagVals;
-  uint16_t BaseAlignLog2; // log_2(base_alignment) + 1
+  Align BaseAlign;
   MachineAtomicInfo AtomicInfo;
   AAMDNodes AAInfo;
   const MDNode *Ranges;
@@ -181,8 +181,7 @@ class MachineMemOperand {
   /// atomic operations the atomic ordering requirements when store does not
   /// occur must also be specified.
   MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, uint64_t s,
-                    uint64_t a,
-                    const AAMDNodes &AAInfo = AAMDNodes(),
+                    Align a, const AAMDNodes &AAInfo = AAMDNodes(),
                     const MDNode *Ranges = nullptr,
                     SyncScope::ID SSID = SyncScope::System,
                     AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
@@ -223,13 +222,21 @@ class MachineMemOperand {
   /// Return the size in bits of the memory reference.
   uint64_t getSizeInBits() const { return Size * 8; }
 
+  LLVM_ATTRIBUTE_DEPRECATED(uint64_t getAlignment() const,
+                            "Use getAlign instead");
+
   /// Return the minimum known alignment in bytes of the actual memory
   /// reference.
-  uint64_t getAlignment() const;
+  Align getAlign() const;
+
+  LLVM_ATTRIBUTE_DEPRECATED(uint64_t getBaseAlignment() const,
+                            "Use getBaseAlign instead") {
+    return BaseAlign.value();
+  }
 
   /// Return the minimum known alignment in bytes of the base address, without
   /// the offset.
-  uint64_t getBaseAlignment() const { return (1ull << BaseAlignLog2) >> 1; }
+  Align getBaseAlign() const { return BaseAlign; }
 
   /// Return the AA tags for the memory reference.
   AAMDNodes getAAInfo() const { return AAInfo; }
@@ -307,7 +314,7 @@ class MachineMemOperand {
            LHS.getFlags() == RHS.getFlags() &&
            LHS.getAAInfo() == RHS.getAAInfo() &&
            LHS.getRanges() == RHS.getRanges() &&
-           LHS.getAlignment() == RHS.getAlignment() &&
+           LHS.getAlign() == RHS.getAlign() &&
            LHS.getAddrSpace() == RHS.getAddrSpace();
   }
 
diff --git a/llvm/include/llvm/CodeGen/MachineOperand.h b/llvm/include/llvm/CodeGen/MachineOperand.h
index 9ba2b01cb4bdf..ac1a4d1b2bc5b 100644
--- a/llvm/include/llvm/CodeGen/MachineOperand.h
+++ b/llvm/include/llvm/CodeGen/MachineOperand.h
@@ -698,6 +698,11 @@ class MachineOperand {
     Contents.RegMask = RegMaskPtr;
   }
 
+  void setIntrinsicID(Intrinsic::ID IID) {
+    assert(isIntrinsicID() && "Wrong MachineOperand mutator");
+    Contents.IntrinsicID = IID;
+  }
+
   void setPredicate(unsigned Predicate) {
     assert(isPredicate() && "Wrong MachineOperand mutator");
     Contents.Pred = Predicate;
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index 3cf7b6a346805..62ad8242ef714 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -1114,14 +1114,36 @@ class SelectionDAG {
   /// INTRINSIC_W_CHAIN, or a target-specific opcode with a value not
   /// less than FIRST_TARGET_MEMORY_OPCODE.
   SDValue getMemIntrinsicNode(
-    unsigned Opcode, const SDLoc &dl, SDVTList VTList,
-    ArrayRef<SDValue> Ops, EVT MemVT,
-    MachinePointerInfo PtrInfo,
-    unsigned Align = 0,
-    MachineMemOperand::Flags Flags
-    = MachineMemOperand::MOLoad | MachineMemOperand::MOStore,
-    uint64_t Size = 0,
-    const AAMDNodes &AAInfo = AAMDNodes());
+      unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
+      EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
+      MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad |
+                                       MachineMemOperand::MOStore,
+      uint64_t Size = 0, const AAMDNodes &AAInfo = AAMDNodes());
+
+  inline SDValue getMemIntrinsicNode(
+      unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
+      EVT MemVT, MachinePointerInfo PtrInfo, MaybeAlign Alignment = None,
+      MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad |
+                                       MachineMemOperand::MOStore,
+      uint64_t Size = 0, const AAMDNodes &AAInfo = AAMDNodes()) {
+    // Ensure that codegen never sees alignment 0
+    return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, PtrInfo,
+                               Alignment.getValueOr(getEVTAlign(MemVT)), Flags,
+                               Size, AAInfo);
+  }
+
+  LLVM_ATTRIBUTE_DEPRECATED(
+      inline SDValue getMemIntrinsicNode(
+          unsigned Opcode, const SDLoc &dl, SDVTList VTList,
+          ArrayRef<SDValue> Ops, EVT MemVT, MachinePointerInfo PtrInfo,
+          unsigned Alignment,
+          MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad |
+                                           MachineMemOperand::MOStore,
+          uint64_t Size = 0, const AAMDNodes &AAInfo = AAMDNodes()),
+      "") {
+    return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, PtrInfo,
+                               MaybeAlign(Alignment), Flags, Size, AAInfo);
+  }
 
   SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList,
                               ArrayRef<SDValue> Ops, EVT MemVT,
@@ -1142,18 +1164,38 @@ class SelectionDAG {
   /// This function will set the MOLoad flag on MMOFlags, but you can set it if
   /// you want.  The MOStore flag must not be set.
   SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
-                  MachinePointerInfo PtrInfo, unsigned Alignment = 0,
+                  MachinePointerInfo PtrInfo, MaybeAlign Alignment,
                   MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
                   const AAMDNodes &AAInfo = AAMDNodes(),
                   const MDNode *Ranges = nullptr);
+  /// FIXME: Remove once transition to Align is over.
+  inline SDValue
+  getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
+          MachinePointerInfo PtrInfo, unsigned Alignment = 0,
+          MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
+          const AAMDNodes &AAInfo = AAMDNodes(),
+          const MDNode *Ranges = nullptr) {
+    return getLoad(VT, dl, Chain, Ptr, PtrInfo, MaybeAlign(Alignment), MMOFlags,
+                   AAInfo, Ranges);
+  }
   SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
                   MachineMemOperand *MMO);
   SDValue
   getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain,
              SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT,
-             unsigned Alignment = 0,
+             MaybeAlign Alignment,
              MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
              const AAMDNodes &AAInfo = AAMDNodes());
+  /// FIXME: Remove once transition to Align is over.
+  inline SDValue
+  getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain,
+             SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT,
+             unsigned Alignment = 0,
+             MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
+             const AAMDNodes &AAInfo = AAMDNodes()) {
+    return getExtLoad(ExtType, dl, VT, Chain, Ptr, PtrInfo, MemVT,
+                      MaybeAlign(Alignment), MMOFlags, AAInfo);
+  }
   SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT,
                      SDValue Chain, SDValue Ptr, EVT MemVT,
                      MachineMemOperand *MMO);
@@ -1161,10 +1203,33 @@ class SelectionDAG {
                          SDValue Offset, ISD::MemIndexedMode AM);
   SDValue getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT,
                   const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset,
-                  MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment = 0,
+                  MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment,
                   MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
                   const AAMDNodes &AAInfo = AAMDNodes(),
                   const MDNode *Ranges = nullptr);
+  inline SDValue
+  getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT,
+          const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset,
+          MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment,
+          MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
+          const AAMDNodes &AAInfo = AAMDNodes(),
+          const MDNode *Ranges = nullptr) {
+    // Ensures that codegen never sees a None Alignment.
+    return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, PtrInfo, MemVT,
+                   Alignment.getValueOr(getEVTAlign(MemVT)), MMOFlags, AAInfo,
+                   Ranges);
+  }
+  /// FIXME: Remove once transition to Align is over.
+  inline SDValue
+  getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT,
+          const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset,
+          MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment = 0,
+          MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
+          const AAMDNodes &AAInfo = AAMDNodes(),
+          const MDNode *Ranges = nullptr) {
+    return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, PtrInfo, MemVT,
+                   MaybeAlign(Alignment), MMOFlags, AAInfo, Ranges);
+  }
   SDValue getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT,
                   const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset,
                   EVT MemVT, MachineMemOperand *MMO);
@@ -1173,18 +1238,55 @@ class SelectionDAG {
   ///
   /// This function will set the MOStore flag on MMOFlags, but you can set it if
   /// you want.  The MOLoad and MOInvariant flags must not be set.
+
   SDValue
   getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
-           MachinePointerInfo PtrInfo, unsigned Alignment = 0,
+           MachinePointerInfo PtrInfo, Align Alignment,
            MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
            const AAMDNodes &AAInfo = AAMDNodes());
+  inline SDValue
+  getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
+           MachinePointerInfo PtrInfo, MaybeAlign Alignment,
+           MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
+           const AAMDNodes &AAInfo = AAMDNodes()) {
+    return getStore(Chain, dl, Val, Ptr, PtrInfo,
+                    Alignment.getValueOr(getEVTAlign(Val.getValueType())),
+                    MMOFlags, AAInfo);
+  }
+  /// FIXME: Remove once transition to Align is over.
+  inline SDValue
+  getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
+           MachinePointerInfo PtrInfo, unsigned Alignment = 0,
+           MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
+           const AAMDNodes &AAInfo = AAMDNodes()) {
+    return getStore(Chain, dl, Val, Ptr, PtrInfo, MaybeAlign(Alignment),
+                    MMOFlags, AAInfo);
+  }
   SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
                    MachineMemOperand *MMO);
   SDValue
   getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
-                MachinePointerInfo PtrInfo, EVT SVT, unsigned Alignment = 0,
+                MachinePointerInfo PtrInfo, EVT SVT, Align Alignment,
                 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
                 const AAMDNodes &AAInfo = AAMDNodes());
+  inline SDValue
+  getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
+                MachinePointerInfo PtrInfo, EVT SVT, MaybeAlign Alignment,
+                MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
+                const AAMDNodes &AAInfo = AAMDNodes()) {
+    return getTruncStore(Chain, dl, Val, Ptr, PtrInfo, SVT,
+                         Alignment.getValueOr(getEVTAlign(SVT)), MMOFlags,
+                         AAInfo);
+  }
+  /// FIXME: Remove once transition to Align is over.
+  inline SDValue
+  getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
+                MachinePointerInfo PtrInfo, EVT SVT, unsigned Alignment = 0,
+                MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
+                const AAMDNodes &AAInfo = AAMDNodes()) {
+    return getTruncStore(Chain, dl, Val, Ptr, PtrInfo, SVT,
+                         MaybeAlign(Alignment), MMOFlags, AAInfo);
+  }
   SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
                         SDValue Ptr, EVT SVT, MachineMemOperand *MMO);
   SDValue getIndexedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base,
@@ -1224,6 +1326,9 @@ class SelectionDAG {
   SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS,
                            unsigned DestAS);
 
+  /// Return a freeze using the SDLoc of the value operand.
+  SDValue getFreeze(SDValue V);
+
   /// Return the specified value casted to
   /// the target's desired shift amount type.
   SDValue getShiftAmountOperand(EVT LHSTy, SDValue Op);
@@ -1710,9 +1815,17 @@ class SelectionDAG {
   bool areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base,
                                       unsigned Bytes, int Dist) const;
 
-  /// Infer alignment of a load / store address. Return 0 if
-  /// it cannot be inferred.
-  unsigned InferPtrAlignment(SDValue Ptr) const;
+  /// Infer alignment of a load / store address. Return None if it cannot be
+  /// inferred.
+  MaybeAlign InferPtrAlign(SDValue Ptr) const;
+
+  LLVM_ATTRIBUTE_DEPRECATED(inline unsigned InferPtrAlignment(SDValue Ptr)
+                                const,
+                            "Use InferPtrAlign instead") {
+    if (auto A = InferPtrAlign(Ptr))
+      return A->value();
+    return 0;
+  }
 
   /// Compute the VTs needed for the low/hi parts of a type
   /// which is split (or expanded) into two not necessarily identical pieces.
@@ -1749,7 +1862,12 @@ class SelectionDAG {
                              EVT EltVT = EVT());
 
   /// Compute the default alignment value for the given type.
-  unsigned getEVTAlignment(EVT MemoryVT) const;
+  Align getEVTAlign(EVT MemoryVT) const;
+  /// Compute the default alignment value for the given type.
+  /// FIXME: Remove once transition to Align is over.
+  inline unsigned getEVTAlignment(EVT MemoryVT) const {
+    return getEVTAlign(MemoryVT).value();
+  }
 
   /// Test whether the given value is a constant int or similar node.
   SDNode *isConstantIntBuildVectorOrConstantInt(SDValue N);
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
index 084badcbe0291..b42bf7a7c36a3 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGISel.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
@@ -318,7 +318,7 @@ class SelectionDAGISel : public MachineFunctionPass {
 private:
 
   // Calls to these functions are generated by tblgen.
-  void Select_INLINEASM(SDNode *N, bool Branch);
+  void Select_INLINEASM(SDNode *N);
   void Select_READ_REGISTER(SDNode *Op);
   void Select_WRITE_REGISTER(SDNode *Op);
   void Select_UNDEF(SDNode *N);
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index faee8e8ffa171..487d658f0dd9e 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -368,7 +368,6 @@ struct SDNodeFlags {
   bool NoInfs : 1;
   bool NoSignedZeros : 1;
   bool AllowReciprocal : 1;
-  bool VectorReduction : 1;
   bool AllowContract : 1;
   bool ApproximateFuncs : 1;
   bool AllowReassociation : 1;
@@ -385,7 +384,7 @@ struct SDNodeFlags {
   SDNodeFlags()
       : AnyDefined(false), NoUnsignedWrap(false), NoSignedWrap(false),
         Exact(false), NoNaNs(false), NoInfs(false),
-        NoSignedZeros(false), AllowReciprocal(false), VectorReduction(false),
+        NoSignedZeros(false), AllowReciprocal(false),
         AllowContract(false), ApproximateFuncs(false),
         AllowReassociation(false), NoFPExcept(false) {}
 
@@ -434,10 +433,6 @@ struct SDNodeFlags {
     setDefined();
     AllowReciprocal = b;
   }
-  void setVectorReduction(bool b) {
-    setDefined();
-    VectorReduction = b;
-  }
   void setAllowContract(bool b) {
     setDefined();
     AllowContract = b;
@@ -463,7 +458,6 @@ struct SDNodeFlags {
   bool hasNoInfs() const { return NoInfs; }
   bool hasNoSignedZeros() const { return NoSignedZeros; }
   bool hasAllowReciprocal() const { return AllowReciprocal; }
-  bool hasVectorReduction() const { return VectorReduction; }
   bool hasAllowContract() const { return AllowContract; }
   bool hasApproximateFuncs() const { return ApproximateFuncs; }
   bool hasAllowReassociation() const { return AllowReassociation; }
@@ -481,7 +475,6 @@ struct SDNodeFlags {
     NoInfs &= Flags.NoInfs;
     NoSignedZeros &= Flags.NoSignedZeros;
     AllowReciprocal &= Flags.AllowReciprocal;
-    VectorReduction &= Flags.VectorReduction;
     AllowContract &= Flags.AllowContract;
     ApproximateFuncs &= Flags.ApproximateFuncs;
     AllowReassociation &= Flags.AllowReassociation;
@@ -1299,12 +1292,14 @@ class MemSDNode : public SDNode {
   bool writeMem() const { return MMO->isStore(); }
 
   /// Returns alignment and volatility of the memory access
-  unsigned getOriginalAlignment() const {
-    return MMO->getBaseAlignment();
-  }
-  unsigned getAlignment() const {
-    return MMO->getAlignment();
-  }
+  Align getOriginalAlign() const { return MMO->getBaseAlign(); }
+  Align getAlign() const { return MMO->getAlign(); }
+  LLVM_ATTRIBUTE_DEPRECATED(unsigned getOriginalAlignment() const,
+                            "Use getOriginalAlign() instead") {
+    return MMO->getBaseAlign().value();
+  }
+  // FIXME: Remove once transition to getAlign is over.
+  unsigned getAlignment() const { return MMO->getAlign().value(); }
 
   /// Return the SubclassData value, without HasDebugValue. This contains an
   /// encoding of the volatile flag, as well as bits used by subclasses. This
diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
index fade3b6600ed9..58ea804d27479 100644
--- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
@@ -235,8 +235,8 @@ class TargetInstrInfo : public MCInstrInfo {
   /// destination. e.g. X86::MOVSX64rr32. If this returns true, then it's
   /// expected the pre-extension value is available as a subreg of the result
   /// register. This also returns the sub-register index in SubIdx.
-  virtual bool isCoalescableExtInstr(const MachineInstr &MI, unsigned &SrcReg,
-                                     unsigned &DstReg, unsigned &SubIdx) const {
+  virtual bool isCoalescableExtInstr(const MachineInstr &MI, Register &SrcReg,
+                                     Register &DstReg, unsigned &SubIdx) const {
     return false;
   }
 
@@ -368,7 +368,7 @@ class TargetInstrInfo : public MCInstrInfo {
   /// DestReg:SubIdx. Any existing subreg index is preserved or composed with
   /// SubIdx.
   virtual void reMaterialize(MachineBasicBlock &MBB,
-                             MachineBasicBlock::iterator MI, unsigned DestReg,
+                             MachineBasicBlock::iterator MI, Register DestReg,
                              unsigned SubIdx, const MachineInstr &Orig,
                              const TargetRegisterInfo &TRI) const;
 
@@ -448,10 +448,10 @@ class TargetInstrInfo : public MCInstrInfo {
   /// A pair composed of a register and a sub-register index.
   /// Used to give some type checking when modeling Reg:SubReg.
   struct RegSubRegPair {
-    unsigned Reg;
+    Register Reg;
     unsigned SubReg;
 
-    RegSubRegPair(unsigned Reg = 0, unsigned SubReg = 0)
+    RegSubRegPair(Register Reg = Register(), unsigned SubReg = 0)
         : Reg(Reg), SubReg(SubReg) {}
 
     bool operator==(const RegSubRegPair& P) const {
@@ -468,7 +468,7 @@ class TargetInstrInfo : public MCInstrInfo {
   struct RegSubRegPairAndIdx : RegSubRegPair {
     unsigned SubIdx;
 
-    RegSubRegPairAndIdx(unsigned Reg = 0, unsigned SubReg = 0,
+    RegSubRegPairAndIdx(Register Reg = Register(), unsigned SubReg = 0,
                         unsigned SubIdx = 0)
         : RegSubRegPair(Reg, SubReg), SubIdx(SubIdx) {}
   };
@@ -845,8 +845,8 @@ class TargetInstrInfo : public MCInstrInfo {
   /// @param TrueCycles  Latency from TrueReg to select output.
   /// @param FalseCycles Latency from FalseReg to select output.
   virtual bool canInsertSelect(const MachineBasicBlock &MBB,
-                               ArrayRef<MachineOperand> Cond, unsigned DstReg,
-                               unsigned TrueReg, unsigned FalseReg,
+                               ArrayRef<MachineOperand> Cond, Register DstReg,
+                               Register TrueReg, Register FalseReg,
                                int &CondCycles, int &TrueCycles,
                                int &FalseCycles) const {
     return false;
@@ -869,8 +869,8 @@ class TargetInstrInfo : public MCInstrInfo {
   /// @param FalseReg Virtual register to copy when Cons is false.
   virtual void insertSelect(MachineBasicBlock &MBB,
                             MachineBasicBlock::iterator I, const DebugLoc &DL,
-                            unsigned DstReg, ArrayRef<MachineOperand> Cond,
-                            unsigned TrueReg, unsigned FalseReg) const {
+                            Register DstReg, ArrayRef<MachineOperand> Cond,
+                            Register TrueReg, Register FalseReg) const {
     llvm_unreachable("Target didn't implement TargetInstrInfo::insertSelect!");
   }
 
@@ -1416,16 +1416,16 @@ class TargetInstrInfo : public MCInstrInfo {
   /// in SrcReg and SrcReg2 if having two register operands, and the value it
   /// compares against in CmpValue. Return true if the comparison instruction
   /// can be analyzed.
-  virtual bool analyzeCompare(const MachineInstr &MI, unsigned &SrcReg,
-                              unsigned &SrcReg2, int &Mask, int &Value) const {
+  virtual bool analyzeCompare(const MachineInstr &MI, Register &SrcReg,
+                              Register &SrcReg2, int &Mask, int &Value) const {
     return false;
   }
 
   /// See if the comparison instruction can be converted
   /// into something more efficient. E.g., on ARM most instructions can set the
   /// flags register, obviating the need for a separate CMP.
-  virtual bool optimizeCompareInstr(MachineInstr &CmpInstr, unsigned SrcReg,
-                                    unsigned SrcReg2, int Mask, int Value,
+  virtual bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
+                                    Register SrcReg2, int Mask, int Value,
                                     const MachineRegisterInfo *MRI) const {
     return false;
   }
@@ -1452,7 +1452,7 @@ class TargetInstrInfo : public MCInstrInfo {
   /// block. The caller may assume that it will not be erased by this
   /// function otherwise.
   virtual bool FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
-                             unsigned Reg, MachineRegisterInfo *MRI) const {
+                             Register Reg, MachineRegisterInfo *MRI) const {
     return false;
   }
 
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index fefa8daa60a16..3a54bcbefb61a 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -278,7 +278,7 @@ class TargetLoweringBase {
     bool IsSwiftSelf : 1;
     bool IsSwiftError : 1;
     bool IsCFGuardTarget : 1;
-    uint16_t Alignment = 0;
+    MaybeAlign Alignment = None;
     Type *ByValType = nullptr;
 
     ArgListEntry()
@@ -1962,6 +1962,18 @@ class TargetLoweringBase {
     return ISD::ZERO_EXTEND;
   }
 
+  /// Returns how the platform's atomic compare and swap expects its comparison
+  /// value to be extended (ZERO_EXTEND, SIGN_EXTEND, or ANY_EXTEND). This is
+  /// separate from getExtendForAtomicOps, which is concerned with the
+  /// sign-extension of the instruction's output, whereas here we are concerned
+  /// with the sign-extension of the input. For targets with compare-and-swap
+  /// instructions (or sub-word comparisons in their LL/SC loop expansions),
+  /// the input can be ANY_EXTEND, but the output will still have a specific
+  /// extension.
+  virtual ISD::NodeType getExtendForAtomicCmpSwapArg() const {
+    return ISD::ANY_EXTEND;
+  }
+
   /// @}
 
   /// Returns true if we should normalize
diff --git a/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h b/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
index 2a1123e27dada..6d47f51dba8b2 100644
--- a/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
+++ b/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
@@ -253,7 +253,9 @@ class TargetLoweringObjectFileXCOFF : public TargetLoweringObjectFile {
 
   static XCOFF::StorageClass getStorageClassForGlobal(const GlobalObject *GO);
 
-  MCSection *getSectionForFunctionDescriptor(const MCSymbol *) const override;
+  MCSection *
+  getSectionForFunctionDescriptor(const Function *F,
+                                  const TargetMachine &TM) const override;
   MCSection *getSectionForTOCEntry(const MCSymbol *Sym) const override;
 
   /// For external functions, this will always return a function descriptor
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h
index 4f2130cd93991..c448c5f690a30 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h
@@ -61,6 +61,7 @@ class DWARFContext : public DIContext {
   std::unique_ptr<DWARFDebugLine> Line;
   std::unique_ptr<DWARFDebugFrame> DebugFrame;
   std::unique_ptr<DWARFDebugFrame> EHFrame;
+  std::unique_ptr<DWARFDebugMacro> Macro;
   std::unique_ptr<DWARFDebugMacro> Macinfo;
   std::unique_ptr<DWARFDebugNames> Names;
   std::unique_ptr<AppleAcceleratorTable> AppleNames;
@@ -104,6 +105,15 @@ class DWARFContext : public DIContext {
 
   std::unique_ptr<const DWARFObject> DObj;
 
+  /// Helper enum to distinguish between macro[.dwo] and macinfo[.dwo]
+  /// section.
+  enum MacroSecType {
+    MacinfoSection,
+    MacinfoDwoSection,
+    MacroSection
+    // FIXME: Add support for.debug_macro.dwo section.
+  };
+
 public:
   DWARFContext(std::unique_ptr<const DWARFObject> DObj,
                std::string DWPName = "",
@@ -272,12 +282,15 @@ class DWARFContext : public DIContext {
   /// Get a pointer to the parsed eh frame information object.
   const DWARFDebugFrame *getEHFrame();
 
-  /// Get a pointer to the parsed DebugMacro object.
+  /// Get a pointer to the parsed DebugMacinfo information object.
   const DWARFDebugMacro *getDebugMacinfo();
 
-  /// Get a pointer to the parsed dwo DebugMacro object.
+  /// Get a pointer to the parsed DebugMacinfoDWO information object.
   const DWARFDebugMacro *getDebugMacinfoDWO();
 
+  /// Get a pointer to the parsed DebugMacro information object.
+  const DWARFDebugMacro *getDebugMacro();
+
   /// Get a reference to the parsed accelerator table object.
   const DWARFDebugNames &getDebugNames();
 
@@ -382,6 +395,10 @@ class DWARFContext : public DIContext {
   }
 
 private:
+  /// Parse a macro[.dwo] or macinfo[.dwo] section.
+  std::unique_ptr<DWARFDebugMacro>
+  parseMacroOrMacinfo(MacroSecType SectionType);
+
   /// Return the compile unit which contains instruction with provided
   /// address.
   /// TODO: change input parameter from "uint64_t Address"
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h
index e666c82bca080..9d60cdc621527 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h
@@ -10,7 +10,10 @@
 #define LLVM_DEBUGINFO_DWARF_DWARFDEBUGMACRO_H
 
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/Support/DataExtractor.h"
+#include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h"
+#include "llvm/Support/Errc.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/WithColor.h"
 #include <cstdint>
 
 namespace llvm {
@@ -18,6 +21,45 @@ namespace llvm {
 class raw_ostream;
 
 class DWARFDebugMacro {
+  /// DWARFv5 section 6.3.1 Macro Information Header.
+  enum HeaderFlagMask {
+#define HANDLE_MACRO_FLAG(ID, NAME) MACRO_##NAME = ID,
+#include "llvm/BinaryFormat/Dwarf.def"
+  };
+  struct MacroHeader {
+    /// Macro version information number.
+    uint16_t Version = 0;
+
+    /// The bits of the flags field are interpreted as a set of flags, some of
+    /// which may indicate that additional fields follow. The following flags,
+    /// beginning with the least significant bit, are defined:
+    /// offset_size_flag:
+    ///   If the offset_size_flag is zero, the header is for a 32-bit DWARF
+    ///   format macro section and all offsets are 4 bytes long; if it is one,
+    ///   the header is for a 64-bit DWARF format macro section and all offsets
+    ///   are 8 bytes long.
+    /// debug_line_offset_flag:
+    ///   If the debug_line_offset_flag is one, the debug_line_offset field (see
+    ///   below) is present. If zero, that field is omitted.
+    /// opcode_operands_table_flag:
+    ///   If the opcode_operands_table_flag is one, the opcode_operands_table
+    ///   field (see below) is present. If zero, that field is omitted.
+    uint8_t Flags;
+
+    /// debug_line_offset
+    ///   An offset in the .debug_line section of the beginning of the line
+    ///   number information in the containing compilation unit, encoded as a
+    ///   4-byte offset for a 32-bit DWARF format macro section and an 8-byte
+    ///   offset for a 64-bit DWARF format macro section.
+    uint64_t DebugLineOffset;
+
+    /// Print the macro header from the debug_macro section.
+    void dumpMacroHeader(raw_ostream &OS) const;
+
+    /// Parse the debug_macro header.
+    Error parseMacroHeader(DWARFDataExtractor Data, uint64_t *Offset);
+  };
+
   /// A single macro entry within a macro list.
   struct Entry {
     /// The type of the macro entry.
@@ -40,6 +82,10 @@ class DWARFDebugMacro {
   };
 
   struct MacroList {
+    // A value 0 in the `Header.Version` field indicates that we're parsing
+    // a macinfo[.dwo] section which doesn't have header itself, hence
+    // for that case other fields in the `Header` are uninitialized.
+    MacroHeader Header;
     SmallVector<Entry, 4> Macros;
     uint64_t Offset;
   };
@@ -50,11 +96,13 @@ class DWARFDebugMacro {
 public:
   DWARFDebugMacro() = default;
 
-  /// Print the macro list found within the debug_macinfo section.
+  /// Print the macro list found within the debug_macinfo/debug_macro section.
   void dump(raw_ostream &OS) const;
 
-  /// Parse the debug_macinfo section accessible via the 'data' parameter.
-  void parse(DataExtractor data);
+  /// Parse the debug_macinfo/debug_macro section accessible via the 'Data'
+  /// parameter.
+  Error parse(DataExtractor StringExtractor, DWARFDataExtractor Data,
+              bool IsMacro);
 
   /// Return whether the section has any entries.
   bool empty() const { return MacroLists.empty(); }
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFObject.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFObject.h
index fbcde7d7cd788..89fccf04a01df 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFObject.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFObject.h
@@ -47,6 +47,7 @@ class DWARFObject {
   virtual StringRef getStrSection() const { return ""; }
   virtual const DWARFSection &getRangesSection() const { return Dummy; }
   virtual const DWARFSection &getRnglistsSection() const { return Dummy; }
+  virtual const DWARFSection &getMacroSection() const { return Dummy; }
   virtual StringRef getMacinfoSection() const { return ""; }
   virtual StringRef getMacinfoDWOSection() const { return ""; }
   virtual const DWARFSection &getPubnamesSection() const { return Dummy; }
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h
index 1d313a3cca24c..635c437fff109 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h
@@ -487,7 +487,7 @@ class DWARFUnit {
 
   uint32_t getLineTableOffset() const {
     if (auto IndexEntry = Header.getIndexEntry())
-      if (const auto *Contrib = IndexEntry->getOffset(DW_SECT_LINE))
+      if (const auto *Contrib = IndexEntry->getContribution(DW_SECT_LINE))
         return Contrib->Offset;
     return 0;
   }
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h
index 684103aac2fce..d6b5e72cf16c1 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h
@@ -56,10 +56,10 @@ class DWARFUnitIndex {
     friend class DWARFUnitIndex;
 
   public:
-    const SectionContribution *getOffset(DWARFSectionKind Sec) const;
-    const SectionContribution *getOffset() const;
+    const SectionContribution *getContribution(DWARFSectionKind Sec) const;
+    const SectionContribution *getContribution() const;
 
-    const SectionContribution *getOffsets() const {
+    const SectionContribution *getContributions() const {
       return Contributions.get();
     }
 
diff --git a/llvm/include/llvm/ExecutionEngine/JITSymbol.h b/llvm/include/llvm/ExecutionEngine/JITSymbol.h
index a7e8b3aef7c19..976f62306735a 100644
--- a/llvm/include/llvm/ExecutionEngine/JITSymbol.h
+++ b/llvm/include/llvm/ExecutionEngine/JITSymbol.h
@@ -84,7 +84,9 @@ class JITSymbolFlags {
     Absolute = 1U << 3,
     Exported = 1U << 4,
     Callable = 1U << 5,
-    LLVM_MARK_AS_BITMASK_ENUM(/* LargestValue = */ Callable)
+    MaterializationSideEffectsOnly = 1U << 6,
+    LLVM_MARK_AS_BITMASK_ENUM( // LargestValue =
+        MaterializationSideEffectsOnly)
   };
 
   /// Default-construct a JITSymbolFlags instance.
@@ -146,6 +148,21 @@ class JITSymbolFlags {
   /// Returns true if the given symbol is known to be callable.
   bool isCallable() const { return (Flags & Callable) == Callable; }
 
+  /// Returns true if this symbol is a materialization-side-effects-only
+  /// symbol. Such symbols do not have a real address. They exist to trigger
+  /// and support synchronization of materialization side effects, e.g. for
+  /// collecting initialization information. These symbols will vanish from
+  /// the symbol table immediately upon reaching the ready state, and will
+  /// appear to queries as if they were never defined (except that query
+  /// callback execution will be delayed until they reach the ready state).
+  /// MaterializationSideEffectOnly symbols should only be queried using the
+  /// SymbolLookupFlags::WeaklyReferencedSymbol flag (see
+  /// llvm/include/llvm/ExecutionEngine/Orc/Core.h).
+  bool hasMaterializationSideEffectsOnly() const {
+    return (Flags & MaterializationSideEffectsOnly) ==
+           MaterializationSideEffectsOnly;
+  }
+
   /// Get the underlying flags value as an integer.
   UnderlyingType getRawFlagsValue() const {
     return static_cast<UnderlyingType>(Flags);
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Core.h b/llvm/include/llvm/ExecutionEngine/Orc/Core.h
index f56e819114652..be5152d09ad29 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Core.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Core.h
@@ -483,6 +483,20 @@ class MaterializationResponsibility {
   /// callbacks, metadata).
   Error defineMaterializing(SymbolFlagsMap SymbolFlags);
 
+  /// Define the given symbols as non-existent, removing it from the symbol
+  /// table and notifying any pending queries. Queries that lookup up the
+  /// symbol using the SymbolLookupFlags::WeaklyReferencedSymbol flag will
+  /// behave as if the symbol had not been matched in the first place. Queries
+  /// that required this symbol will fail with a missing symbol definition
+  /// error.
+  ///
+  /// This method is intended to support cleanup of special symbols like
+  /// initializer symbols: Queries using
+  /// SymbolLookupFlags::WeaklyReferencedSymbol can be used to trigger their
+  /// emission, and this method can be used to remove them from the JITDylib
+  /// once materialization is complete.
+  void defineNonExistent(ArrayRef<SymbolStringPtr> Symbols);
+
   /// Notify all not-yet-emitted covered by this MaterializationResponsibility
   /// instance that an error has occurred.
   /// This will remove all symbols covered by this MaterializationResponsibilty
@@ -721,15 +735,6 @@ class AsynchronousSymbolQuery {
   void notifySymbolMetRequiredState(const SymbolStringPtr &Name,
                                     JITEvaluatedSymbol Sym);
 
-  /// Remove a symbol from the query. This is used to drop weakly referenced
-  /// symbols that are not found.
-  void dropSymbol(const SymbolStringPtr &Name) {
-    assert(ResolvedSymbols.count(Name) &&
-           "Redundant removal of weakly-referenced symbol");
-    ResolvedSymbols.erase(Name);
-    --OutstandingSymbolsCount;
-  }
-
   /// Returns true if all symbols covered by this query have been
   ///        resolved.
   bool isComplete() const { return OutstandingSymbolsCount == 0; }
@@ -747,6 +752,8 @@ class AsynchronousSymbolQuery {
 
   void removeQueryDependence(JITDylib &JD, const SymbolStringPtr &Name);
 
+  void dropSymbol(const SymbolStringPtr &Name);
+
   bool canStillFail();
 
   void handleFailed(Error Err);
@@ -1298,6 +1305,19 @@ auto JITDylib::withSearchOrderDo(Func &&F)
 template <typename MaterializationUnitType>
 Error JITDylib::define(std::unique_ptr<MaterializationUnitType> &&MU) {
   assert(MU && "Can not define with a null MU");
+
+  if (MU->getSymbols().empty()) {
+    // Empty MUs are allowable but pathological, so issue a warning.
+    DEBUG_WITH_TYPE("orc", {
+      dbgs() << "Warning: Discarding empty MU " << MU->getName() << " for "
+             << getName() << "\n";
+    });
+    return Error::success();
+  } else
+    DEBUG_WITH_TYPE("orc", {
+      dbgs() << "Defining MU " << MU->getName() << " for " << getName() << "\n";
+    });
+
   return ES.runSessionLocked([&, this]() -> Error {
     if (auto Err = defineImpl(*MU))
       return Err;
@@ -1320,6 +1340,18 @@ template <typename MaterializationUnitType>
 Error JITDylib::define(std::unique_ptr<MaterializationUnitType> &MU) {
   assert(MU && "Can not define with a null MU");
 
+  if (MU->getSymbols().empty()) {
+    // Empty MUs are allowable but pathological, so issue a warning.
+    DEBUG_WITH_TYPE("orc", {
+      dbgs() << "Warning: Discarding empty MU " << MU->getName() << getName()
+             << "\n";
+    });
+    return Error::success();
+  } else
+    DEBUG_WITH_TYPE("orc", {
+      dbgs() << "Defining MU " << MU->getName() << " for " << getName() << "\n";
+    });
+
   return ES.runSessionLocked([&, this]() -> Error {
     if (auto Err = defineImpl(*MU))
       return Err;
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/DebugUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/DebugUtils.h
index ac6cc30c948a0..4b4472e0ac4db 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/DebugUtils.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/DebugUtils.h
@@ -38,6 +38,9 @@ raw_ostream &operator<<(raw_ostream &OS, const SymbolNameSet &Symbols);
 /// Render a SymbolNameVector.
 raw_ostream &operator<<(raw_ostream &OS, const SymbolNameVector &Symbols);
 
+/// Render an array of SymbolStringPtrs.
+raw_ostream &operator<<(raw_ostream &OS, ArrayRef<SymbolStringPtr> Symbols);
+
 /// Render JITSymbolFlags.
 raw_ostream &operator<<(raw_ostream &OS, const JITSymbolFlags &Flags);
 
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
index 5d26f07a7f5ac..c80ecc7cbf778 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
@@ -93,6 +93,8 @@ __OMP_DIRECTIVE_EXT(parallel_master_taskloop_simd,
                     "parallel master taskloop simd")
 __OMP_DIRECTIVE(depobj)
 __OMP_DIRECTIVE(scan)
+__OMP_DIRECTIVE_EXT(begin_declare_variant, "begin declare variant")
+__OMP_DIRECTIVE_EXT(end_declare_variant, "end declare variant")
 
 // Has to be the last because Clang implicitly expects it to be.
 __OMP_DIRECTIVE(unknown)
@@ -251,17 +253,17 @@ __OMP_RTL(__last, false, Void, )
 __OMP_ATTRS_SET(GetterAttrs,
                 OptimisticAttributes
                     ? AttributeSet(EnumAttr(NoUnwind), EnumAttr(ReadOnly),
-                                   EnumAttr(NoSync), EnumAttr(NoFree))
+                                   EnumAttr(NoSync), EnumAttr(NoFree), EnumAttr(InaccessibleMemOnly))
                     : AttributeSet(EnumAttr(NoUnwind)))
 __OMP_ATTRS_SET(GetterArgWriteAttrs,
                 OptimisticAttributes
                     ? AttributeSet(EnumAttr(NoUnwind), EnumAttr(NoSync),
-                                   EnumAttr(NoFree))
+                                   EnumAttr(NoFree), EnumAttr(InaccessibleMemOrArgMemOnly))
                     : AttributeSet(EnumAttr(NoUnwind)))
 __OMP_ATTRS_SET(SetterAttrs,
                 OptimisticAttributes
                     ? AttributeSet(EnumAttr(NoUnwind), EnumAttr(WriteOnly),
-                                   EnumAttr(NoSync), EnumAttr(NoFree))
+                                   EnumAttr(NoSync), EnumAttr(NoFree), EnumAttr(InaccessibleMemOnly))
                     : AttributeSet(EnumAttr(NoUnwind)))
 
 #undef __OMP_ATTRS_SET
@@ -274,10 +276,40 @@ __OMP_ATTRS_SET(SetterAttrs,
 #define __OMP_RTL_ATTRS(Name, FnAttrSet, RetAttrSet, ArgAttrSets)              \
   OMP_RTL_ATTRS(OMPRTL_##Name, FnAttrSet, RetAttrSet, ArgAttrSets)
 
+__OMP_RTL_ATTRS(__kmpc_barrier,
+                AttributeSet(EnumAttr(InaccessibleMemOrArgMemOnly)),
+                AttributeSet(), {})
+__OMP_RTL_ATTRS(__kmpc_cancel,
+                AttributeSet(EnumAttr(InaccessibleMemOrArgMemOnly)),
+                AttributeSet(), {})
+__OMP_RTL_ATTRS(__kmpc_cancel_barrier,
+                AttributeSet(EnumAttr(InaccessibleMemOrArgMemOnly)),
+                AttributeSet(), {})
+__OMP_RTL_ATTRS(__kmpc_flush,
+                AttributeSet(EnumAttr(InaccessibleMemOrArgMemOnly)),
+                AttributeSet(), {})
+__OMP_RTL_ATTRS(__kmpc_global_thread_num, GetterAttrs, AttributeSet(), {})
 __OMP_RTL_ATTRS(__kmpc_fork_call, AttributeSet(EnumAttr(NoUnwind)),
                 AttributeSet(), {})
+__OMP_RTL_ATTRS(__kmpc_omp_taskwait,
+                AttributeSet(EnumAttr(InaccessibleMemOrArgMemOnly)),
+                AttributeSet(), {})
+__OMP_RTL_ATTRS(__kmpc_omp_taskyield,
+                AttributeSet(EnumAttr(InaccessibleMemOrArgMemOnly)),
+                AttributeSet(), {})
+__OMP_RTL_ATTRS(__kmpc_push_num_threads,
+                AttributeSet(EnumAttr(InaccessibleMemOrArgMemOnly)),
+                AttributeSet(), {})
+__OMP_RTL_ATTRS(__kmpc_push_proc_bind,
+                AttributeSet(EnumAttr(InaccessibleMemOrArgMemOnly)),
+                AttributeSet(), {})
+__OMP_RTL_ATTRS(__kmpc_serialized_parallel,
+                AttributeSet(EnumAttr(InaccessibleMemOrArgMemOnly)),
+                AttributeSet(), {})
+__OMP_RTL_ATTRS(__kmpc_end_serialized_parallel,
+                AttributeSet(EnumAttr(InaccessibleMemOrArgMemOnly)),
+                AttributeSet(), {})
 
-__OMP_RTL_ATTRS(__kmpc_global_thread_num, GetterAttrs, AttributeSet(), {})
 __OMP_RTL_ATTRS(omp_get_thread_num, GetterAttrs, AttributeSet(), {})
 __OMP_RTL_ATTRS(omp_get_num_threads, GetterAttrs, AttributeSet(), {})
 __OMP_RTL_ATTRS(omp_get_max_threads, GetterAttrs, AttributeSet(), {})
@@ -315,6 +347,22 @@ __OMP_RTL_ATTRS(omp_set_nested, SetterAttrs, AttributeSet(), {})
 __OMP_RTL_ATTRS(omp_set_schedule, SetterAttrs, AttributeSet(), {})
 __OMP_RTL_ATTRS(omp_set_max_active_levels, SetterAttrs, AttributeSet(), {})
 
+__OMP_RTL_ATTRS(__kmpc_master,
+                AttributeSet(EnumAttr(InaccessibleMemOrArgMemOnly)),
+                AttributeSet(), {})
+__OMP_RTL_ATTRS(__kmpc_end_master,
+                AttributeSet(EnumAttr(InaccessibleMemOrArgMemOnly)),
+                AttributeSet(), {})
+__OMP_RTL_ATTRS(__kmpc_critical,
+                AttributeSet(EnumAttr(InaccessibleMemOrArgMemOnly)),
+                AttributeSet(), {})
+__OMP_RTL_ATTRS(__kmpc_critical_with_hint,
+                AttributeSet(EnumAttr(InaccessibleMemOrArgMemOnly)),
+                AttributeSet(), {})
+__OMP_RTL_ATTRS(__kmpc_end_critical,
+                AttributeSet(EnumAttr(InaccessibleMemOrArgMemOnly)),
+                AttributeSet(), {})
+
 #undef __OMP_RTL_ATTRS
 #undef OMP_RTL_ATTRS
 #undef AttributeSet
diff --git a/llvm/include/llvm/IR/CFG.h b/llvm/include/llvm/IR/CFG.h
index 55aff7137e86f..b166df954cd70 100644
--- a/llvm/include/llvm/IR/CFG.h
+++ b/llvm/include/llvm/IR/CFG.h
@@ -103,7 +103,7 @@ using pred_iterator = PredIterator<BasicBlock, Value::user_iterator>;
 using const_pred_iterator =
     PredIterator<const BasicBlock, Value::const_user_iterator>;
 using pred_range = iterator_range<pred_iterator>;
-using pred_const_range = iterator_range<const_pred_iterator>;
+using const_pred_range = iterator_range<const_pred_iterator>;
 
 inline pred_iterator pred_begin(BasicBlock *BB) { return pred_iterator(BB); }
 inline const_pred_iterator pred_begin(const BasicBlock *BB) {
@@ -124,8 +124,8 @@ inline unsigned pred_size(const BasicBlock *BB) {
 inline pred_range predecessors(BasicBlock *BB) {
   return pred_range(pred_begin(BB), pred_end(BB));
 }
-inline pred_const_range predecessors(const BasicBlock *BB) {
-  return pred_const_range(pred_begin(BB), pred_end(BB));
+inline const_pred_range predecessors(const BasicBlock *BB) {
+  return const_pred_range(pred_begin(BB), pred_end(BB));
 }
 
 //===----------------------------------------------------------------------===//
@@ -238,17 +238,17 @@ class SuccIterator
 };
 
 using succ_iterator = SuccIterator<Instruction, BasicBlock>;
-using succ_const_iterator = SuccIterator<const Instruction, const BasicBlock>;
+using const_succ_iterator = SuccIterator<const Instruction, const BasicBlock>;
 using succ_range = iterator_range<succ_iterator>;
-using succ_const_range = iterator_range<succ_const_iterator>;
+using const_succ_range = iterator_range<const_succ_iterator>;
 
 inline succ_iterator succ_begin(Instruction *I) { return succ_iterator(I); }
-inline succ_const_iterator succ_begin(const Instruction *I) {
-  return succ_const_iterator(I);
+inline const_succ_iterator succ_begin(const Instruction *I) {
+  return const_succ_iterator(I);
 }
 inline succ_iterator succ_end(Instruction *I) { return succ_iterator(I, true); }
-inline succ_const_iterator succ_end(const Instruction *I) {
-  return succ_const_iterator(I, true);
+inline const_succ_iterator succ_end(const Instruction *I) {
+  return const_succ_iterator(I, true);
 }
 inline bool succ_empty(const Instruction *I) {
   return succ_begin(I) == succ_end(I);
@@ -259,21 +259,21 @@ inline unsigned succ_size(const Instruction *I) {
 inline succ_range successors(Instruction *I) {
   return succ_range(succ_begin(I), succ_end(I));
 }
-inline succ_const_range successors(const Instruction *I) {
-  return succ_const_range(succ_begin(I), succ_end(I));
+inline const_succ_range successors(const Instruction *I) {
+  return const_succ_range(succ_begin(I), succ_end(I));
 }
 
 inline succ_iterator succ_begin(BasicBlock *BB) {
   return succ_iterator(BB->getTerminator());
 }
-inline succ_const_iterator succ_begin(const BasicBlock *BB) {
-  return succ_const_iterator(BB->getTerminator());
+inline const_succ_iterator succ_begin(const BasicBlock *BB) {
+  return const_succ_iterator(BB->getTerminator());
 }
 inline succ_iterator succ_end(BasicBlock *BB) {
   return succ_iterator(BB->getTerminator(), true);
 }
-inline succ_const_iterator succ_end(const BasicBlock *BB) {
-  return succ_const_iterator(BB->getTerminator(), true);
+inline const_succ_iterator succ_end(const BasicBlock *BB) {
+  return const_succ_iterator(BB->getTerminator(), true);
 }
 inline bool succ_empty(const BasicBlock *BB) {
   return succ_begin(BB) == succ_end(BB);
@@ -284,8 +284,8 @@ inline unsigned succ_size(const BasicBlock *BB) {
 inline succ_range successors(BasicBlock *BB) {
   return succ_range(succ_begin(BB), succ_end(BB));
 }
-inline succ_const_range successors(const BasicBlock *BB) {
-  return succ_const_range(succ_begin(BB), succ_end(BB));
+inline const_succ_range successors(const BasicBlock *BB) {
+  return const_succ_range(succ_begin(BB), succ_end(BB));
 }
 
 //===--------------------------------------------------------------------===//
@@ -306,7 +306,7 @@ template <> struct GraphTraits<BasicBlock*> {
 
 template <> struct GraphTraits<const BasicBlock*> {
   using NodeRef = const BasicBlock *;
-  using ChildIteratorType = succ_const_iterator;
+  using ChildIteratorType = const_succ_iterator;
 
   static NodeRef getEntryNode(const BasicBlock *BB) { return BB; }
 
diff --git a/llvm/include/llvm/IR/CFGDiff.h b/llvm/include/llvm/IR/CFGDiff.h
index cc38addd058e8..c50db0de79a30 100644
--- a/llvm/include/llvm/IR/CFGDiff.h
+++ b/llvm/include/llvm/IR/CFGDiff.h
@@ -78,13 +78,25 @@ namespace llvm {
 // remove all existing edges between two blocks.
 template <typename NodePtr, bool InverseGraph = false> class GraphDiff {
   using UpdateMapType = SmallDenseMap<NodePtr, SmallVector<NodePtr, 2>>;
-  UpdateMapType SuccInsert;
-  UpdateMapType SuccDelete;
-  UpdateMapType PredInsert;
-  UpdateMapType PredDelete;
+  struct EdgesInsertedDeleted {
+    UpdateMapType Succ;
+    UpdateMapType Pred;
+  };
+  // Store Deleted edges on position 0, and Inserted edges on position 1.
+  EdgesInsertedDeleted Edges[2];
+  // By default, it is assumed that, given a CFG and a set of updates, we wish
+  // to apply these updates as given. If UpdatedAreReverseApplied is set, the
+  // updates will be applied in reverse: deleted edges are considered re-added
+  // and inserted edges are considered deleted when returning children.
+  bool UpdatedAreReverseApplied;
   // Using a singleton empty vector for all node requests with no
   // children.
-  SmallVector<NodePtr, 1> Empty;
+  SmallVector<NodePtr, 0> Empty;
+
+  // Keep the list of legalized updates for a deterministic order of updates
+  // when using a GraphDiff for incremental updates in the DominatorTree.
+  // The list is kept in reverse to allow popping from end.
+  SmallVector<cfg::Update<NodePtr>, 4> LegalizedUpdates;
 
   void printMap(raw_ostream &OS, const UpdateMapType &M) const {
     for (auto Pair : M)
@@ -99,24 +111,53 @@ template <typename NodePtr, bool InverseGraph = false> class GraphDiff {
   }
 
 public:
-  GraphDiff() {}
-  GraphDiff(ArrayRef<cfg::Update<NodePtr>> Updates) {
-    SmallVector<cfg::Update<NodePtr>, 4> LegalizedUpdates;
-    cfg::LegalizeUpdates<NodePtr>(Updates, LegalizedUpdates, InverseGraph);
+  GraphDiff() : UpdatedAreReverseApplied(false) {}
+  GraphDiff(ArrayRef<cfg::Update<NodePtr>> Updates,
+            bool ReverseApplyUpdates = false) {
+    cfg::LegalizeUpdates<NodePtr>(Updates, LegalizedUpdates, InverseGraph,
+                                  /*ReverseResultOrder=*/true);
+    // The legalized updates are stored in reverse so we can pop_back when doing
+    // incremental updates.
     for (auto U : LegalizedUpdates) {
-      if (U.getKind() == cfg::UpdateKind::Insert) {
-        SuccInsert[U.getFrom()].push_back(U.getTo());
-        PredInsert[U.getTo()].push_back(U.getFrom());
-      } else {
-        SuccDelete[U.getFrom()].push_back(U.getTo());
-        PredDelete[U.getTo()].push_back(U.getFrom());
-      }
+      unsigned IsInsert =
+          (U.getKind() == cfg::UpdateKind::Insert) == !ReverseApplyUpdates;
+      Edges[IsInsert].Succ[U.getFrom()].push_back(U.getTo());
+      Edges[IsInsert].Pred[U.getTo()].push_back(U.getFrom());
     }
+    UpdatedAreReverseApplied = ReverseApplyUpdates;
+  }
+
+  auto getLegalizedUpdates() const {
+    return make_range(LegalizedUpdates.begin(), LegalizedUpdates.end());
+  }
+
+  unsigned getNumLegalizedUpdates() const { return LegalizedUpdates.size(); }
+
+  cfg::Update<NodePtr> popUpdateForIncrementalUpdates() {
+    assert(!LegalizedUpdates.empty() && "No updates to apply!");
+    auto U = LegalizedUpdates.pop_back_val();
+    unsigned IsInsert =
+        (U.getKind() == cfg::UpdateKind::Insert) == !UpdatedAreReverseApplied;
+    auto &SuccList = Edges[IsInsert].Succ[U.getFrom()];
+    assert(SuccList.back() == U.getTo());
+    SuccList.pop_back();
+    if (SuccList.empty())
+      Edges[IsInsert].Succ.erase(U.getFrom());
+
+    auto &PredList = Edges[IsInsert].Pred[U.getTo()];
+    assert(PredList.back() == U.getFrom());
+    PredList.pop_back();
+    if (PredList.empty())
+      Edges[IsInsert].Pred.erase(U.getTo());
+    return U;
   }
 
   bool ignoreChild(const NodePtr BB, NodePtr EdgeEnd, bool InverseEdge) const {
+    // Used to filter nullptr in clang.
+    if (EdgeEnd == nullptr)
+      return true;
     auto &DeleteChildren =
-        (InverseEdge != InverseGraph) ? PredDelete : SuccDelete;
+        (InverseEdge != InverseGraph) ? Edges[0].Pred : Edges[0].Succ;
     auto It = DeleteChildren.find(BB);
     if (It == DeleteChildren.end())
       return false;
@@ -127,7 +168,7 @@ template <typename NodePtr, bool InverseGraph = false> class GraphDiff {
   iterator_range<typename SmallVectorImpl<NodePtr>::const_iterator>
   getAddedChildren(const NodePtr BB, bool InverseEdge) const {
     auto &InsertChildren =
-        (InverseEdge != InverseGraph) ? PredInsert : SuccInsert;
+        (InverseEdge != InverseGraph) ? Edges[1].Pred : Edges[1].Succ;
     auto It = InsertChildren.find(BB);
     if (It == InsertChildren.end())
       return make_range(Empty.begin(), Empty.end());
@@ -139,13 +180,13 @@ template <typename NodePtr, bool InverseGraph = false> class GraphDiff {
           "===== (Note: notion of children/inverse_children depends on "
           "the direction of edges and the graph.)\n";
     OS << "Children to insert:\n\t";
-    printMap(OS, SuccInsert);
+    printMap(OS, Edges[1].Succ);
     OS << "Children to delete:\n\t";
-    printMap(OS, SuccDelete);
+    printMap(OS, Edges[0].Succ);
     OS << "Inverse_children to insert:\n\t";
-    printMap(OS, PredInsert);
+    printMap(OS, Edges[1].Pred);
     OS << "Inverse_children to delete:\n\t";
-    printMap(OS, PredDelete);
+    printMap(OS, Edges[0].Pred);
     OS << "\n";
   }
 
@@ -181,6 +222,7 @@ struct CFGViewChildren {
     auto Second = makeChildRange(InsertVec, N.first);
 
     auto CR = concat<NodeRef>(First, Second);
+
     // concat_range contains references to other ranges, returning it would
     // leave those references dangling - the iterators contain
     // other iterators by value so they're safe to return.
diff --git a/llvm/include/llvm/IR/CallSite.h b/llvm/include/llvm/IR/CallSite.h
index 6a82e73537cf5..41833d118c951 100644
--- a/llvm/include/llvm/IR/CallSite.h
+++ b/llvm/include/llvm/IR/CallSite.h
@@ -413,13 +413,27 @@ class CallSiteBase {
   }
 
   /// Extract the alignment of the return value.
+  /// FIXME: Remove when the transition to Align is over.
   unsigned getRetAlignment() const {
-    CALLSITE_DELEGATE_GETTER(getRetAlignment());
+    if (auto MA = getRetAlign())
+      return MA->value();
+    return 0;
   }
 
+  /// Extract the alignment of the return value.
+  MaybeAlign getRetAlign() const { CALLSITE_DELEGATE_GETTER(getRetAlign()); }
+
   /// Extract the alignment for a call or parameter (0=unknown).
+  /// FIXME: Remove when the transition to Align is over.
   unsigned getParamAlignment(unsigned ArgNo) const {
-    CALLSITE_DELEGATE_GETTER(getParamAlignment(ArgNo));
+    if (auto MA = getParamAlign(ArgNo))
+      return MA->value();
+    return 0;
+  }
+
+  /// Extract the alignment for a call or parameter (0=unknown).
+  MaybeAlign getParamAlign(unsigned ArgNo) const {
+    CALLSITE_DELEGATE_GETTER(getParamAlign(ArgNo));
   }
 
   /// Extract the byval type for a call or parameter (nullptr=unknown).
diff --git a/llvm/include/llvm/IR/ConstantFolder.h b/llvm/include/llvm/IR/ConstantFolder.h
index 32939e79cd1df..da4a18e3c181a 100644
--- a/llvm/include/llvm/IR/ConstantFolder.h
+++ b/llvm/include/llvm/IR/ConstantFolder.h
@@ -265,7 +265,7 @@ class ConstantFolder final : public IRBuilderFolder {
   }
 
   Constant *CreateShuffleVector(Constant *V1, Constant *V2,
-                                Constant *Mask) const override {
+                                ArrayRef<int> Mask) const override {
     return ConstantExpr::getShuffleVector(V1, V2, Mask);
   }
 
diff --git a/llvm/include/llvm/IR/Constants.h b/llvm/include/llvm/IR/Constants.h
index d0ea1adbbf183..868b038b055d2 100644
--- a/llvm/include/llvm/IR/Constants.h
+++ b/llvm/include/llvm/IR/Constants.h
@@ -766,7 +766,12 @@ class ConstantDataVector final : public ConstantDataSequential {
   friend class ConstantDataSequential;
 
   explicit ConstantDataVector(Type *ty, const char *Data)
-      : ConstantDataSequential(ty, ConstantDataVectorVal, Data) {}
+      : ConstantDataSequential(ty, ConstantDataVectorVal, Data),
+        IsSplatSet(false) {}
+  // Cache whether or not the constant is a splat.
+  mutable bool IsSplatSet : 1;
+  mutable bool IsSplat : 1;
+  bool isSplatData() const;
 
 public:
   ConstantDataVector(const ConstantDataVector &) = delete;
@@ -1201,7 +1206,8 @@ class ConstantExpr : public Constant {
                                      Type *OnlyIfReducedTy = nullptr);
   static Constant *getInsertElement(Constant *Vec, Constant *Elt, Constant *Idx,
                                     Type *OnlyIfReducedTy = nullptr);
-  static Constant *getShuffleVector(Constant *V1, Constant *V2, Constant *Mask,
+  static Constant *getShuffleVector(Constant *V1, Constant *V2,
+                                    ArrayRef<int> Mask,
                                     Type *OnlyIfReducedTy = nullptr);
   static Constant *getExtractValue(Constant *Agg, ArrayRef<unsigned> Idxs,
                                    Type *OnlyIfReducedTy = nullptr);
@@ -1220,6 +1226,16 @@ class ConstantExpr : public Constant {
   /// expression and return the list of indices.
   ArrayRef<unsigned> getIndices() const;
 
+  /// Assert that this is a shufflevector and return the mask. See class
+  /// ShuffleVectorInst for a description of the mask representation.
+  ArrayRef<int> getShuffleMask() const;
+
+  /// Assert that this is a shufflevector and return the mask.
+  ///
+  /// TODO: This is a temporary hack until we update the bitcode format for
+  /// shufflevector.
+  Constant *getShuffleMaskForBitcode() const;
+
   /// Return a string representation for an opcode.
   const char *getOpcodeName() const;
 
diff --git a/llvm/include/llvm/IR/FPEnv.h b/llvm/include/llvm/IR/FPEnv.h
index a1e0665d41128..0e7e96ff700a1 100644
--- a/llvm/include/llvm/IR/FPEnv.h
+++ b/llvm/include/llvm/IR/FPEnv.h
@@ -15,6 +15,7 @@
 #ifndef LLVM_IR_FLOATINGPOINT_H
 #define LLVM_IR_FLOATINGPOINT_H
 
+#include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/StringRef.h"
 #include <stdint.h>
@@ -66,5 +67,8 @@ Optional<fp::ExceptionBehavior> StrToExceptionBehavior(StringRef);
 /// input in constrained intrinsic exception behavior metadata.
 Optional<StringRef> ExceptionBehaviorToStr(fp::ExceptionBehavior);
 
+/// Converts rounding mode represented by fp::RoundingMode to the rounding mode
+/// index used by APFloat. For fp::rmDynamic it returns None.
+Optional<APFloatBase::roundingMode> getAPFloatRoundingMode(fp::RoundingMode);
 }
 #endif
diff --git a/llvm/include/llvm/IR/GlobalObject.h b/llvm/include/llvm/IR/GlobalObject.h
index faa9e296da721..cd0b2dccdcac3 100644
--- a/llvm/include/llvm/IR/GlobalObject.h
+++ b/llvm/include/llvm/IR/GlobalObject.h
@@ -82,7 +82,6 @@ class GlobalObject : public GlobalValue {
     return decodeMaybeAlign(AlignmentData);
   }
 
-  /// FIXME: Remove this setter once the migration to MaybeAlign is over.
   LLVM_ATTRIBUTE_DEPRECATED(void setAlignment(unsigned Align),
                             "Please use `void setAlignment(MaybeAlign Align)`");
   void setAlignment(MaybeAlign Align);
diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index 3b0ca0c1ae5c6..92c282aec629a 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -520,8 +520,6 @@ class IRBuilderBase {
   /// If the pointer isn't an i8*, it will be converted. If a TBAA tag is
   /// specified, it will be added to the instruction. Likewise with alias.scope
   /// and noalias tags.
-  /// FIXME: Remove this function once transition to Align is over.
-  /// Use the version that takes Align instead of this one.
   LLVM_ATTRIBUTE_DEPRECATED(
       CallInst *CreateElementUnorderedAtomicMemSet(
           Value *Ptr, Value *Val, uint64_t Size, unsigned Alignment,
@@ -544,8 +542,6 @@ class IRBuilderBase {
                                               TBAATag, ScopeTag, NoAliasTag);
   }
 
-  /// FIXME: Remove this function once transition to Align is over.
-  /// Use the version that takes Align instead of this one.
   LLVM_ATTRIBUTE_DEPRECATED(
       CallInst *CreateElementUnorderedAtomicMemSet(
           Value *Ptr, Value *Val, Value *Size, unsigned Alignment,
@@ -569,8 +565,6 @@ class IRBuilderBase {
   /// If the pointers aren't i8*, they will be converted.  If a TBAA tag is
   /// specified, it will be added to the instruction. Likewise with alias.scope
   /// and noalias tags.
-  /// FIXME: Remove this function once transition to Align is over.
-  /// Use the version that takes MaybeAlign instead of this one.
   LLVM_ATTRIBUTE_DEPRECATED(
       CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value *Src,
                              unsigned SrcAlign, uint64_t Size,
@@ -595,8 +589,6 @@ class IRBuilderBase {
                         NoAliasTag);
   }
 
-  /// FIXME: Remove this function once transition to Align is over.
-  /// Use the version that takes MaybeAlign instead of this one.
   LLVM_ATTRIBUTE_DEPRECATED(
       CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value *Src,
                              unsigned SrcAlign, Value *Size,
@@ -629,8 +621,6 @@ class IRBuilderBase {
       MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr,
       MDNode *NoAliasTag = nullptr);
 
-  /// FIXME: Remove this function once transition to Align is over.
-  /// Use the version that takes Align instead of this one.
   LLVM_ATTRIBUTE_DEPRECATED(CallInst *CreateElementUnorderedAtomicMemCpy(
                                 Value *Dst, unsigned DstAlign, Value *Src,
                                 unsigned SrcAlign, uint64_t Size,
@@ -644,8 +634,6 @@ class IRBuilderBase {
         TBAATag, TBAAStructTag, ScopeTag, NoAliasTag);
   }
 
-  /// FIXME: Remove this function once transition to Align is over.
-  /// Use the version that takes Align instead of this one.
   LLVM_ATTRIBUTE_DEPRECATED(CallInst *CreateElementUnorderedAtomicMemCpy(
                                 Value *Dst, unsigned DstAlign, Value *Src,
                                 unsigned SrcAlign, Value *Size,
@@ -659,14 +647,6 @@ class IRBuilderBase {
         TBAAStructTag, ScopeTag, NoAliasTag);
   }
 
-  /// Create and insert a memmove between the specified
-  /// pointers.
-  ///
-  /// If the pointers aren't i8*, they will be converted.  If a TBAA tag is
-  /// specified, it will be added to the instruction. Likewise with alias.scope
-  /// and noalias tags.
-  /// FIXME: Remove this function once transition to Align is over.
-  /// Use the version that takes MaybeAlign instead of this one.
   LLVM_ATTRIBUTE_DEPRECATED(
       CallInst *CreateMemMove(
           Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign,
@@ -685,8 +665,7 @@ class IRBuilderBase {
     return CreateMemMove(Dst, DstAlign, Src, SrcAlign, getInt64(Size),
                          isVolatile, TBAATag, ScopeTag, NoAliasTag);
   }
-  /// FIXME: Remove this function once transition to Align is over.
-  /// Use the version that takes MaybeAlign instead of this one.
+
   LLVM_ATTRIBUTE_DEPRECATED(
       CallInst *CreateMemMove(
           Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign,
@@ -717,8 +696,6 @@ class IRBuilderBase {
       MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr,
       MDNode *NoAliasTag = nullptr);
 
-  /// FIXME: Remove this function once transition to Align is over.
-  /// Use the version that takes Align instead of this one.
   LLVM_ATTRIBUTE_DEPRECATED(CallInst *CreateElementUnorderedAtomicMemMove(
                                 Value *Dst, unsigned DstAlign, Value *Src,
                                 unsigned SrcAlign, uint64_t Size,
@@ -732,8 +709,6 @@ class IRBuilderBase {
         TBAATag, TBAAStructTag, ScopeTag, NoAliasTag);
   }
 
-  /// FIXME: Remove this function once transition to Align is over.
-  /// Use the version that takes Align instead of this one.
   LLVM_ATTRIBUTE_DEPRECATED(CallInst *CreateElementUnorderedAtomicMemMove(
                                 Value *Dst, unsigned DstAlign, Value *Src,
                                 unsigned SrcAlign, Value *Size,
@@ -1747,11 +1722,6 @@ class IRBuilderBase {
     return Insert(new StoreInst(Val, Ptr, isVolatile));
   }
 
-  /// Provided to resolve 'CreateAlignedLoad(Ptr, Align, "...")'
-  /// correctly, instead of converting the string to 'bool' for the isVolatile
-  /// parameter.
-  /// FIXME: Remove this function once transition to Align is over.
-  /// Use the version that takes MaybeAlign instead of this one.
   LLVM_ATTRIBUTE_DEPRECATED(LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr,
                                                         unsigned Align,
                                                         const char *Name),
@@ -1764,8 +1734,7 @@ class IRBuilderBase {
     LI->setAlignment(Align);
     return LI;
   }
-  /// FIXME: Remove this function once transition to Align is over.
-  /// Use the version that takes MaybeAlign instead of this one.
+
   LLVM_ATTRIBUTE_DEPRECATED(LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr,
                                                         unsigned Align,
                                                         const Twine &Name = ""),
@@ -1778,8 +1747,7 @@ class IRBuilderBase {
     LI->setAlignment(Align);
     return LI;
   }
-  /// FIXME: Remove this function once transition to Align is over.
-  /// Use the version that takes MaybeAlign instead of this one.
+
   LLVM_ATTRIBUTE_DEPRECATED(LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr,
                                                         unsigned Align,
                                                         bool isVolatile,
@@ -1837,8 +1805,6 @@ class IRBuilderBase {
                              Align, isVolatile, Name);
   }
 
-  /// FIXME: Remove this function once transition to Align is over.
-  /// Use the version that takes MaybeAlign instead of this one.
   LLVM_ATTRIBUTE_DEPRECATED(
       StoreInst *CreateAlignedStore(Value *Val, Value *Ptr, unsigned Align,
                                     bool isVolatile = false),
@@ -2617,17 +2583,25 @@ class IRBuilderBase {
 
   Value *CreateShuffleVector(Value *V1, Value *V2, Value *Mask,
                              const Twine &Name = "") {
-    if (auto *V1C = dyn_cast<Constant>(V1))
-      if (auto *V2C = dyn_cast<Constant>(V2))
-        if (auto *MC = dyn_cast<Constant>(Mask))
-          return Insert(Folder.CreateShuffleVector(V1C, V2C, MC), Name);
-    return Insert(new ShuffleVectorInst(V1, V2, Mask), Name);
+    SmallVector<int, 16> IntMask;
+    ShuffleVectorInst::getShuffleMask(cast<Constant>(Mask), IntMask);
+    return CreateShuffleVector(V1, V2, IntMask, Name);
+  }
+
+  Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef<uint32_t> Mask,
+                             const Twine &Name = "") {
+    SmallVector<int, 16> IntMask;
+    IntMask.assign(Mask.begin(), Mask.end());
+    return CreateShuffleVector(V1, V2, IntMask, Name);
   }
 
-  Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef<uint32_t> IntMask,
+  /// See class ShuffleVectorInst for a description of the mask representation.
+  Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef<int> Mask,
                              const Twine &Name = "") {
-    Value *Mask = ConstantDataVector::get(Context, IntMask);
-    return CreateShuffleVector(V1, V2, Mask, Name);
+    if (auto *V1C = dyn_cast<Constant>(V1))
+      if (auto *V2C = dyn_cast<Constant>(V2))
+        return Insert(Folder.CreateShuffleVector(V1C, V2C, Mask), Name);
+    return Insert(new ShuffleVectorInst(V1, V2, Mask), Name);
   }
 
   Value *CreateExtractValue(Value *Agg,
diff --git a/llvm/include/llvm/IR/IRBuilderFolder.h b/llvm/include/llvm/IR/IRBuilderFolder.h
index a77a5213dceac..e781e8e094af9 100644
--- a/llvm/include/llvm/IR/IRBuilderFolder.h
+++ b/llvm/include/llvm/IR/IRBuilderFolder.h
@@ -129,7 +129,7 @@ class IRBuilderFolder {
   virtual Value *CreateInsertElement(Constant *Vec, Constant *NewElt,
                                      Constant *Idx) const = 0;
   virtual Value *CreateShuffleVector(Constant *V1, Constant *V2,
-                                     Constant *Mask) const = 0;
+                                     ArrayRef<int> Mask) const = 0;
   virtual Value *CreateExtractValue(Constant *Agg,
                                     ArrayRef<unsigned> IdxList) const = 0;
   virtual Value *CreateInsertValue(Constant *Agg, Constant *Val,
diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h
index cad3f45ed5d22..d28b3301fa6a1 100644
--- a/llvm/include/llvm/IR/InstrTypes.h
+++ b/llvm/include/llvm/IR/InstrTypes.h
@@ -1577,10 +1577,8 @@ class CallBase : public Instruction {
            dataOperandHasImpliedAttr(OpNo + 1, Attribute::ReadNone);
   }
 
-  /// Extract the alignment of the return value.
-  /// FIXME: Remove this function once transition to Align is over.
-  /// Use getRetAlign() instead.
-  unsigned getRetAlignment() const {
+  LLVM_ATTRIBUTE_DEPRECATED(unsigned getRetAlignment() const,
+                            "Use getRetAlign() instead") {
     if (const auto MA = Attrs.getRetAlignment())
       return MA->value();
     return 0;
@@ -1590,9 +1588,8 @@ class CallBase : public Instruction {
   MaybeAlign getRetAlign() const { return Attrs.getRetAlignment(); }
 
   /// Extract the alignment for a call or parameter (0=unknown).
-  /// FIXME: Remove this function once transition to Align is over.
-  /// Use getParamAlign() instead.
-  unsigned getParamAlignment(unsigned ArgNo) const {
+  LLVM_ATTRIBUTE_DEPRECATED(unsigned getParamAlignment(unsigned ArgNo) const,
+                            "Use getParamAlign() instead") {
     if (const auto MA = Attrs.getParamAlignment(ArgNo))
       return MA->value();
     return 0;
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index 713624d13bef0..857b7374ac470 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -1988,10 +1988,22 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InsertElementInst, Value)
 //                           ShuffleVectorInst Class
 //===----------------------------------------------------------------------===//
 
+constexpr int UndefMaskElem = -1;
+
 /// This instruction constructs a fixed permutation of two
 /// input vectors.
 ///
+/// For each element of the result vector, the shuffle mask selects an element
+/// from one of the input vectors to copy to the result. Non-negative elements
+/// in the mask represent an index into the concatenated pair of input vectors.
+/// UndefMaskElem (-1) specifies that the result element is undefined.
+///
+/// For scalable vectors, all the elements of the mask must be 0 or -1. This
+/// requirement may be relaxed in the future.
 class ShuffleVectorInst : public Instruction {
+  SmallVector<int, 4> ShuffleMask;
+  Constant *ShuffleMaskForBitcode;
+
 protected:
   // Note: Instruction needs to be a friend here to call cloneImpl.
   friend class Instruction;
@@ -2004,13 +2016,15 @@ class ShuffleVectorInst : public Instruction {
                     Instruction *InsertBefor = nullptr);
   ShuffleVectorInst(Value *V1, Value *V2, Value *Mask,
                     const Twine &NameStr, BasicBlock *InsertAtEnd);
+  ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
+                    const Twine &NameStr = "",
+                    Instruction *InsertBefor = nullptr);
+  ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
+                    const Twine &NameStr, BasicBlock *InsertAtEnd);
 
-  // allocate space for exactly three operands
-  void *operator new(size_t s) {
-    return User::operator new(s, 3);
-  }
+  void *operator new(size_t s) { return User::operator new(s, 2); }
 
-  /// Swap the first 2 operands and adjust the mask to preserve the semantics
+  /// Swap the operands and adjust the mask to preserve the semantics
   /// of the instruction.
   void commute();
 
@@ -2018,6 +2032,8 @@ class ShuffleVectorInst : public Instruction {
   /// formed with the specified operands.
   static bool isValidOperands(const Value *V1, const Value *V2,
                               const Value *Mask);
+  static bool isValidOperands(const Value *V1, const Value *V2,
+                              ArrayRef<int> Mask);
 
   /// Overload to return most specific vector type.
   ///
@@ -2028,36 +2044,33 @@ class ShuffleVectorInst : public Instruction {
   /// Transparently provide more efficient getOperand methods.
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
 
-  Constant *getMask() const {
-    return cast<Constant>(getOperand(2));
-  }
-
-  /// Return the shuffle mask value for the specified element of the mask.
-  /// Return -1 if the element is undef.
-  static int getMaskValue(const Constant *Mask, unsigned Elt);
-
   /// Return the shuffle mask value of this instruction for the given element
-  /// index. Return -1 if the element is undef.
-  int getMaskValue(unsigned Elt) const {
-    return getMaskValue(getMask(), Elt);
-  }
+  /// index. Return UndefMaskElem if the element is undef.
+  int getMaskValue(unsigned Elt) const { return ShuffleMask[Elt]; }
 
   /// Convert the input shuffle mask operand to a vector of integers. Undefined
-  /// elements of the mask are returned as -1.
+  /// elements of the mask are returned as UndefMaskElem.
   static void getShuffleMask(const Constant *Mask,
                              SmallVectorImpl<int> &Result);
 
   /// Return the mask for this instruction as a vector of integers. Undefined
-  /// elements of the mask are returned as -1.
+  /// elements of the mask are returned as UndefMaskElem.
   void getShuffleMask(SmallVectorImpl<int> &Result) const {
-    return getShuffleMask(getMask(), Result);
+    Result.assign(ShuffleMask.begin(), ShuffleMask.end());
   }
 
-  SmallVector<int, 16> getShuffleMask() const {
-    SmallVector<int, 16> Mask;
-    getShuffleMask(Mask);
-    return Mask;
-  }
+  /// Return the mask for this instruction, for use in bitcode.
+  ///
+  /// TODO: This is temporary until we decide a new bitcode encoding for
+  /// shufflevector.
+  Constant *getShuffleMaskForBitcode() const { return ShuffleMaskForBitcode; }
+
+  static Constant *convertShuffleMaskForBitcode(ArrayRef<int> Mask,
+                                                Type *ResultTy);
+
+  void setShuffleMask(ArrayRef<int> Mask);
+
+  ArrayRef<int> getShuffleMask() const { return ShuffleMask; }
 
   /// Return true if this shuffle returns a vector with a different number of
   /// elements than its source vectors.
@@ -2065,7 +2078,7 @@ class ShuffleVectorInst : public Instruction {
   ///           shufflevector <4 x n> A, <4 x n> B, <1,2,3,4,5>
   bool changesLength() const {
     unsigned NumSourceElts = Op<0>()->getType()->getVectorNumElements();
-    unsigned NumMaskElts = getMask()->getType()->getVectorNumElements();
+    unsigned NumMaskElts = ShuffleMask.size();
     return NumSourceElts != NumMaskElts;
   }
 
@@ -2074,7 +2087,7 @@ class ShuffleVectorInst : public Instruction {
   /// Example: shufflevector <2 x n> A, <2 x n> B, <1,2,3>
   bool increasesLength() const {
     unsigned NumSourceElts = Op<0>()->getType()->getVectorNumElements();
-    unsigned NumMaskElts = getMask()->getType()->getVectorNumElements();
+    unsigned NumMaskElts = ShuffleMask.size();
     return NumSourceElts < NumMaskElts;
   }
 
@@ -2095,7 +2108,7 @@ class ShuffleVectorInst : public Instruction {
   /// Example: shufflevector <4 x n> A, <4 x n> B, <3,0,undef,3>
   /// TODO: Optionally allow length-changing shuffles.
   bool isSingleSource() const {
-    return !changesLength() && isSingleSourceMask(getMask());
+    return !changesLength() && isSingleSourceMask(ShuffleMask);
   }
 
   /// Return true if this shuffle mask chooses elements from exactly one source
@@ -2116,7 +2129,7 @@ class ShuffleVectorInst : public Instruction {
   /// from its input vectors.
   /// Example: shufflevector <4 x n> A, <4 x n> B, <4,undef,6,undef>
   bool isIdentity() const {
-    return !changesLength() && isIdentityMask(getShuffleMask());
+    return !changesLength() && isIdentityMask(ShuffleMask);
   }
 
   /// Return true if this shuffle lengthens exactly one source vector with
@@ -2157,7 +2170,7 @@ class ShuffleVectorInst : public Instruction {
   /// In that case, the shuffle is better classified as an identity shuffle.
   /// TODO: Optionally allow length-changing shuffles.
   bool isSelect() const {
-    return !changesLength() && isSelectMask(getMask());
+    return !changesLength() && isSelectMask(ShuffleMask);
   }
 
   /// Return true if this shuffle mask swaps the order of elements from exactly
@@ -2177,7 +2190,7 @@ class ShuffleVectorInst : public Instruction {
   /// Example: shufflevector <4 x n> A, <4 x n> B, <3,undef,1,undef>
   /// TODO: Optionally allow length-changing shuffles.
   bool isReverse() const {
-    return !changesLength() && isReverseMask(getMask());
+    return !changesLength() && isReverseMask(ShuffleMask);
   }
 
   /// Return true if this shuffle mask chooses all elements with the same value
@@ -2199,7 +2212,7 @@ class ShuffleVectorInst : public Instruction {
   /// TODO: Optionally allow length-changing shuffles.
   /// TODO: Optionally allow splats from other elements.
   bool isZeroEltSplat() const {
-    return !changesLength() && isZeroEltSplatMask(getMask());
+    return !changesLength() && isZeroEltSplatMask(ShuffleMask);
   }
 
   /// Return true if this shuffle mask is a transpose mask.
@@ -2248,7 +2261,7 @@ class ShuffleVectorInst : public Instruction {
   /// exact specification.
   /// Example: shufflevector <4 x n> A, <4 x n> B, <0,4,2,6>
   bool isTranspose() const {
-    return !changesLength() && isTransposeMask(getMask());
+    return !changesLength() && isTransposeMask(ShuffleMask);
   }
 
   /// Return true if this shuffle mask is an extract subvector mask.
@@ -2267,7 +2280,7 @@ class ShuffleVectorInst : public Instruction {
   /// Return true if this shuffle mask is an extract subvector mask.
   bool isExtractSubvectorMask(int &Index) const {
     int NumSrcElts = Op<0>()->getType()->getVectorNumElements();
-    return isExtractSubvectorMask(getMask(), NumSrcElts, Index);
+    return isExtractSubvectorMask(ShuffleMask, NumSrcElts, Index);
   }
 
   /// Change values in a shuffle permute mask assuming the two vector operands
@@ -2293,9 +2306,8 @@ class ShuffleVectorInst : public Instruction {
 };
 
 template <>
-struct OperandTraits<ShuffleVectorInst> :
-  public FixedNumOperandTraits<ShuffleVectorInst, 3> {
-};
+struct OperandTraits<ShuffleVectorInst>
+    : public FixedNumOperandTraits<ShuffleVectorInst, 2> {};
 
 DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ShuffleVectorInst, Value)
 
diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h
index 0ca1688a7c91f..81c9cab190ba4 100644
--- a/llvm/include/llvm/IR/IntrinsicInst.h
+++ b/llvm/include/llvm/IR/IntrinsicInst.h
@@ -392,7 +392,11 @@ namespace llvm {
 
     /// FIXME: Remove this function once transition to Align is over.
     /// Use getDestAlign() instead.
-    unsigned getDestAlignment() const { return getParamAlignment(ARG_DEST); }
+    unsigned getDestAlignment() const {
+      if (auto MA = getParamAlign(ARG_DEST))
+        return MA->value();
+      return 0;
+    }
     MaybeAlign getDestAlign() const { return getParamAlign(ARG_DEST); }
 
     /// Set the specified arguments of the instruction.
@@ -454,7 +458,9 @@ namespace llvm {
     /// FIXME: Remove this function once transition to Align is over.
     /// Use getSourceAlign() instead.
     unsigned getSourceAlignment() const {
-      return BaseCL::getParamAlignment(ARG_SOURCE);
+      if (auto MA = BaseCL::getParamAlign(ARG_SOURCE))
+        return MA->value();
+      return 0;
     }
 
     MaybeAlign getSourceAlign() const {
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 0812d707e4fed..eda69b31fdb23 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -1151,7 +1151,9 @@ def int_clear_cache : Intrinsic<[], [llvm_ptr_ty, llvm_ptr_ty],
                                 [], "llvm.clear_cache">;
 
 // Intrinsic to detect whether its argument is a constant.
-def int_is_constant : Intrinsic<[llvm_i1_ty], [llvm_any_ty], [IntrNoMem, IntrWillReturn], "llvm.is.constant">;
+def int_is_constant : Intrinsic<[llvm_i1_ty], [llvm_any_ty],
+                                [IntrNoMem, IntrWillReturn, IntrConvergent],
+                                "llvm.is.constant">;
 
 // Intrinsic to mask out bits of a pointer.
 def int_ptrmask: Intrinsic<[llvm_anyptr_ty], [llvm_anyptr_ty, llvm_anyint_ty],
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index cad4f7cb4cd13..e3277a2107962 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -790,6 +790,25 @@ let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
                  LLVMPointerToElt<0>],
                 [IntrArgMemOnly, NoCapture<2>]>;
 
+  class AdvSIMD_2Vec_PredStore_Intrinsic
+      : Intrinsic<[],
+                  [llvm_anyvector_ty, LLVMMatchType<0>,
+                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerTo<0>],
+                  [IntrArgMemOnly, NoCapture<3>]>;
+
+  class AdvSIMD_3Vec_PredStore_Intrinsic
+      : Intrinsic<[],
+                  [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>,
+                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerTo<0>],
+                  [IntrArgMemOnly, NoCapture<4>]>;
+
+  class AdvSIMD_4Vec_PredStore_Intrinsic
+      : Intrinsic<[],
+                  [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>,
+                   LLVMMatchType<0>,
+                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerTo<0>],
+                  [IntrArgMemOnly, NoCapture<5>]>;
+
   class AdvSIMD_SVE_Index_Intrinsic
     : Intrinsic<[llvm_anyvector_ty],
                 [LLVMVectorElementType<0>,
@@ -1292,7 +1311,10 @@ def int_aarch64_sve_ldff1 : AdvSIMD_1Vec_PredLoad_Intrinsic;
 // Stores
 //
 
-def int_aarch64_sve_st1   : AdvSIMD_1Vec_PredStore_Intrinsic;
+def int_aarch64_sve_st1  : AdvSIMD_1Vec_PredStore_Intrinsic;
+def int_aarch64_sve_st2  : AdvSIMD_2Vec_PredStore_Intrinsic;
+def int_aarch64_sve_st3  : AdvSIMD_3Vec_PredStore_Intrinsic;
+def int_aarch64_sve_st4  : AdvSIMD_4Vec_PredStore_Intrinsic;
 
 def int_aarch64_sve_stnt1 : AdvSIMD_1Vec_PredStore_Intrinsic;
 
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index c01db52b1622e..d1eae65beb3d4 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -1348,6 +1348,9 @@ def int_amdgcn_fcmp :
   Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, LLVMMatchType<1>, llvm_i32_ty],
             [IntrNoMem, IntrConvergent, ImmArg<2>]>;
 
+def int_amdgcn_ballot :
+  Intrinsic<[llvm_anyint_ty], [llvm_i1_ty], [IntrNoMem, IntrConvergent]>;
+
 def int_amdgcn_readfirstlane :
   GCCBuiltin<"__builtin_amdgcn_readfirstlane">,
   Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem, IntrConvergent]>;
@@ -1875,4 +1878,10 @@ def int_amdgcn_fdiv_fast : Intrinsic<
   [llvm_float_ty], [llvm_float_ty, llvm_float_ty],
   [IntrNoMem, IntrSpeculatable]
 >;
+
+// Represent a relocation constant.
+def int_amdgcn_reloc_constant : Intrinsic<
+  [llvm_i32_ty], [llvm_metadata_ty],
+  [IntrNoMem, IntrSpeculatable]
+>;
 }
diff --git a/llvm/include/llvm/IR/IntrinsicsARM.td b/llvm/include/llvm/IR/IntrinsicsARM.td
index b41831ed1f61e..4b74c627372c4 100644
--- a/llvm/include/llvm/IR/IntrinsicsARM.td
+++ b/llvm/include/llvm/IR/IntrinsicsARM.td
@@ -1332,6 +1332,17 @@ multiclass CDEVCXVecIntrinsics<list<LLVMType> args> {
     !listconcat([llvm_i32_ty /* coproc */, llvm_v16i8_ty /* acc */],
                 args, [llvm_i32_ty /* imm */]),
     [IntrNoMem, ImmArg<0>, ImmArg<!add(!size(args), 2)>]>;
+
+  def _predicated : Intrinsic<
+    [llvm_anyvector_ty],
+    !listconcat([llvm_i32_ty /* coproc */, LLVMMatchType<0> /* inactive */],
+                args, [llvm_i32_ty /* imm */, llvm_anyvector_ty /* mask */]),
+    [IntrNoMem, ImmArg<0>, ImmArg<!add(!size(args), 2)>]>;
+  def a_predicated : Intrinsic<
+    [llvm_anyvector_ty],
+    !listconcat([llvm_i32_ty /* coproc */, LLVMMatchType<0> /* acc */],
+                args, [llvm_i32_ty /* imm */, llvm_anyvector_ty /* mask */]),
+    [IntrNoMem, ImmArg<0>, ImmArg<!add(!size(args), 2)>]>;
 }
 
 defm int_arm_cde_vcx1q : CDEVCXVecIntrinsics<[]>;
diff --git a/llvm/include/llvm/IR/IntrinsicsX86.td b/llvm/include/llvm/IR/IntrinsicsX86.td
index 4d353c7e93c24..8623be78478af 100644
--- a/llvm/include/llvm/IR/IntrinsicsX86.td
+++ b/llvm/include/llvm/IR/IntrinsicsX86.td
@@ -4930,3 +4930,11 @@ let TargetPrefix = "x86" in {
   def int_x86_enqcmds : GCCBuiltin<"__builtin_ia32_enqcmds">,
               Intrinsic<[llvm_i8_ty], [llvm_ptr_ty, llvm_ptr_ty], []>;
 }
+
+//===----------------------------------------------------------------------===//
+// SERIALIZE - Serialize instruction fetch and execution
+
+let TargetPrefix = "x86" in {
+  def int_x86_serialize : GCCBuiltin<"__builtin_ia32_serialize">,
+              Intrinsic<[], [], []>;
+}
diff --git a/llvm/include/llvm/IR/NoFolder.h b/llvm/include/llvm/IR/NoFolder.h
index 1e35cfe096eb8..dcffa6b2f9dad 100644
--- a/llvm/include/llvm/IR/NoFolder.h
+++ b/llvm/include/llvm/IR/NoFolder.h
@@ -300,7 +300,7 @@ class NoFolder final : public IRBuilderFolder {
   }
 
   Instruction *CreateShuffleVector(Constant *V1, Constant *V2,
-                                   Constant *Mask) const override {
+                                   ArrayRef<int> Mask) const override {
     return new ShuffleVectorInst(V1, V2, Mask);
   }
 
diff --git a/llvm/include/llvm/IR/Operator.h b/llvm/include/llvm/IR/Operator.h
index 35e08d9215e2b..ec594666f0a43 100644
--- a/llvm/include/llvm/IR/Operator.h
+++ b/llvm/include/llvm/IR/Operator.h
@@ -599,6 +599,25 @@ class BitCastOperator
   }
 };
 
+class AddrSpaceCastOperator
+    : public ConcreteOperator<Operator, Instruction::AddrSpaceCast> {
+  friend class AddrSpaceCastInst;
+  friend class ConstantExpr;
+
+public:
+  Value *getPointerOperand() { return getOperand(0); }
+
+  const Value *getPointerOperand() const { return getOperand(0); }
+
+  unsigned getSrcAddressSpace() const {
+    return getPointerOperand()->getType()->getPointerAddressSpace();
+  }
+
+  unsigned getDestAddressSpace() const {
+    return getType()->getPointerAddressSpace();
+  }
+};
+
 } // end namespace llvm
 
 #endif // LLVM_IR_OPERATOR_H
diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h
index 03b92e524ad50..30c203dedf67c 100644
--- a/llvm/include/llvm/IR/PatternMatch.h
+++ b/llvm/include/llvm/IR/PatternMatch.h
@@ -50,6 +50,10 @@ template <typename Val, typename Pattern> bool match(Val *V, const Pattern &P) {
   return const_cast<Pattern &>(P).match(V);
 }
 
+template <typename Pattern> bool match(ArrayRef<int> Mask, const Pattern &P) {
+  return const_cast<Pattern &>(P).match(Mask);
+}
+
 template <typename SubPattern_t> struct OneUse_match {
   SubPattern_t SubPattern;
 
@@ -1350,12 +1354,69 @@ m_ExtractElement(const Val_t &Val, const Idx_t &Idx) {
   return TwoOps_match<Val_t, Idx_t, Instruction::ExtractElement>(Val, Idx);
 }
 
-/// Matches ShuffleVectorInst.
+/// Matches shuffle.
+template <typename T0, typename T1, typename T2> struct Shuffle_match {
+  T0 Op1;
+  T1 Op2;
+  T2 Mask;
+
+  Shuffle_match(const T0 &Op1, const T1 &Op2, const T2 &Mask)
+      : Op1(Op1), Op2(Op2), Mask(Mask) {}
+
+  template <typename OpTy> bool match(OpTy *V) {
+    if (auto *I = dyn_cast<ShuffleVectorInst>(V)) {
+      return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1)) &&
+             Mask.match(I->getShuffleMask());
+    }
+    return false;
+  }
+};
+
+struct m_Mask {
+  ArrayRef<int> &MaskRef;
+  m_Mask(ArrayRef<int> &MaskRef) : MaskRef(MaskRef) {}
+  bool match(ArrayRef<int> Mask) {
+    MaskRef = Mask;
+    return true;
+  }
+};
+
+struct m_ZeroMask {
+  bool match(ArrayRef<int> Mask) {
+    return all_of(Mask, [](int Elem) { return Elem == 0 || Elem == -1; });
+  }
+};
+
+struct m_SpecificMask {
+  ArrayRef<int> &MaskRef;
+  m_SpecificMask(ArrayRef<int> &MaskRef) : MaskRef(MaskRef) {}
+  bool match(ArrayRef<int> Mask) { return MaskRef == Mask; }
+};
+
+struct m_SplatOrUndefMask {
+  int &SplatIndex;
+  m_SplatOrUndefMask(int &SplatIndex) : SplatIndex(SplatIndex) {}
+  bool match(ArrayRef<int> Mask) {
+    auto First = find_if(Mask, [](int Elem) { return Elem != -1; });
+    if (First == Mask.end())
+      return false;
+    SplatIndex = *First;
+    return all_of(Mask,
+                  [First](int Elem) { return Elem == *First || Elem == -1; });
+  }
+};
+
+/// Matches ShuffleVectorInst independently of mask value.
+template <typename V1_t, typename V2_t>
+inline TwoOps_match<V1_t, V2_t, Instruction::ShuffleVector>
+m_ShuffleVector(const V1_t &v1, const V2_t &v2) {
+  return TwoOps_match<V1_t, V2_t, Instruction::ShuffleVector>(v1, v2);
+}
+
 template <typename V1_t, typename V2_t, typename Mask_t>
-inline ThreeOps_match<V1_t, V2_t, Mask_t, Instruction::ShuffleVector>
-m_ShuffleVector(const V1_t &v1, const V2_t &v2, const Mask_t &m) {
-  return ThreeOps_match<V1_t, V2_t, Mask_t, Instruction::ShuffleVector>(v1, v2,
-                                                                        m);
+inline Shuffle_match<V1_t, V2_t, Mask_t>
+m_ShuffleVector(const V1_t &v1, const V2_t &v2, const Mask_t &mask) {
+  return Shuffle_match<V1_t, V2_t, Mask_t>(v1, v2, mask);
 }
 
 /// Matches LoadInst.
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index fece1583da057..91da2129f904c 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -416,6 +416,7 @@ void initializeTwoAddressInstructionPassPass(PassRegistry&);
 void initializeTypeBasedAAWrapperPassPass(PassRegistry&);
 void initializeTypePromotionPass(PassRegistry&);
 void initializeUnifyFunctionExitNodesPass(PassRegistry&);
+void initializeUnifyLoopExitsPass(PassRegistry &);
 void initializeUnpackMachineBundlesPass(PassRegistry&);
 void initializeUnreachableBlockElimLegacyPassPass(PassRegistry&);
 void initializeUnreachableMachineBlockElimPass(PassRegistry&);
diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h
index df28533456827..e934398194155 100644
--- a/llvm/include/llvm/LTO/LTO.h
+++ b/llvm/include/llvm/LTO/LTO.h
@@ -228,7 +228,7 @@ using ThinBackend = std::function<std::unique_ptr<ThinBackendProc>(
 
 /// This ThinBackend runs the individual backend jobs in-process.
 /// The default value means to use one job per hardware core (not hyper-thread).
-ThinBackend createInProcessThinBackend(unsigned ParallelismLevel = 0);
+ThinBackend createInProcessThinBackend(ThreadPoolStrategy Parallelism);
 
 /// This ThinBackend writes individual module indexes to files, instead of
 /// running the individual backend jobs. This backend is for distributed builds
diff --git a/llvm/include/llvm/LinkAllPasses.h b/llvm/include/llvm/LinkAllPasses.h
index 616dd7825f3bb..17441486a18e0 100644
--- a/llvm/include/llvm/LinkAllPasses.h
+++ b/llvm/include/llvm/LinkAllPasses.h
@@ -231,7 +231,8 @@ namespace {
       (void) llvm::createScalarizeMaskedMemIntrinPass();
       (void) llvm::createWarnMissedTransformationsPass();
       (void) llvm::createHardwareLoopsPass();
-      (void)llvm::createInjectTLIMappingsLegacyPass();
+      (void) llvm::createInjectTLIMappingsLegacyPass();
+      (void) llvm::createUnifyLoopExitsPass();
 
       (void)new llvm::IntervalPartition();
       (void)new llvm::ScalarEvolutionWrapperPass();
diff --git a/llvm/include/llvm/MC/LaneBitmask.h b/llvm/include/llvm/MC/LaneBitmask.h
index b070bea3201cf..a467407f17064 100644
--- a/llvm/include/llvm/MC/LaneBitmask.h
+++ b/llvm/include/llvm/MC/LaneBitmask.h
@@ -40,7 +40,7 @@ namespace llvm {
     // When changing the underlying type, change the format string as well.
     using Type = uint64_t;
     enum : unsigned { BitWidth = 8*sizeof(Type) };
-    constexpr static const char *const FormatStr = "%016lX";
+    constexpr static const char *const FormatStr = "%016llX";
 
     constexpr LaneBitmask() = default;
     explicit constexpr LaneBitmask(Type V) : Mask(V) {}
diff --git a/llvm/include/llvm/MC/MCAsmInfo.h b/llvm/include/llvm/MC/MCAsmInfo.h
index b4f7c350739a5..18f1637deb502 100644
--- a/llvm/include/llvm/MC/MCAsmInfo.h
+++ b/llvm/include/llvm/MC/MCAsmInfo.h
@@ -322,9 +322,10 @@ class MCAsmInfo {
   /// symbol that can be hidden (unexported).  Defaults to false.
   bool HasWeakDefCanBeHiddenDirective = false;
 
-  /// True if we have a .linkonce directive.  This is used on cygwin/mingw.
+  /// True if we should mark symbols as global instead of weak, for
+  /// weak*/linkonce*, if the symbol has a comdat.
   /// Defaults to false.
-  bool HasLinkOnceDirective = false;
+  bool AvoidWeakIfComdat = false;
 
   /// True if we have a .lglobl directive, which is used to emit the information
   /// of a static symbol into the symbol table. Defaults to false.
@@ -590,7 +591,7 @@ class MCAsmInfo {
     return HasWeakDefCanBeHiddenDirective;
   }
 
-  bool hasLinkOnceDirective() const { return HasLinkOnceDirective; }
+  bool avoidWeakIfComdat() const { return AvoidWeakIfComdat; }
 
   bool hasDotLGloblDirective() const { return HasDotLGloblDirective; }
 
diff --git a/llvm/include/llvm/MC/MCDirectives.h b/llvm/include/llvm/MC/MCDirectives.h
index ea79e68674e53..cad08c8574d23 100644
--- a/llvm/include/llvm/MC/MCDirectives.h
+++ b/llvm/include/llvm/MC/MCDirectives.h
@@ -16,34 +16,34 @@
 namespace llvm {
 
 enum MCSymbolAttr {
-  MCSA_Invalid = 0,    ///< Not a valid directive.
+  MCSA_Invalid = 0, ///< Not a valid directive.
 
   // Various directives in alphabetical order.
-  MCSA_Cold,                ///< .cold (MachO)
-  MCSA_ELF_TypeFunction,    ///< .type _foo, STT_FUNC  # aka @function
-  MCSA_ELF_TypeIndFunction, ///< .type _foo, STT_GNU_IFUNC
-  MCSA_ELF_TypeObject,      ///< .type _foo, STT_OBJECT  # aka @object
-  MCSA_ELF_TypeTLS,         ///< .type _foo, STT_TLS     # aka @tls_object
-  MCSA_ELF_TypeCommon,      ///< .type _foo, STT_COMMON  # aka @common
-  MCSA_ELF_TypeNoType,      ///< .type _foo, STT_NOTYPE  # aka @notype
+  MCSA_Cold,                    ///< .cold (MachO)
+  MCSA_ELF_TypeFunction,        ///< .type _foo, STT_FUNC  # aka @function
+  MCSA_ELF_TypeIndFunction,     ///< .type _foo, STT_GNU_IFUNC
+  MCSA_ELF_TypeObject,          ///< .type _foo, STT_OBJECT  # aka @object
+  MCSA_ELF_TypeTLS,             ///< .type _foo, STT_TLS     # aka @tls_object
+  MCSA_ELF_TypeCommon,          ///< .type _foo, STT_COMMON  # aka @common
+  MCSA_ELF_TypeNoType,          ///< .type _foo, STT_NOTYPE  # aka @notype
   MCSA_ELF_TypeGnuUniqueObject, /// .type _foo, @gnu_unique_object
-  MCSA_Global,              ///< .globl
-  MCSA_LGlobal,             ///< .lglobl (XCOFF)
-  MCSA_Hidden,              ///< .hidden (ELF)
-  MCSA_IndirectSymbol,      ///< .indirect_symbol (MachO)
-  MCSA_Internal,            ///< .internal (ELF)
-  MCSA_LazyReference,       ///< .lazy_reference (MachO)
-  MCSA_Local,               ///< .local (ELF)
-  MCSA_NoDeadStrip,         ///< .no_dead_strip (MachO)
-  MCSA_SymbolResolver,      ///< .symbol_resolver (MachO)
-  MCSA_AltEntry,            ///< .alt_entry (MachO)
-  MCSA_PrivateExtern,       ///< .private_extern (MachO)
-  MCSA_Protected,           ///< .protected (ELF)
-  MCSA_Reference,           ///< .reference (MachO)
-  MCSA_Weak,                ///< .weak
-  MCSA_WeakDefinition,      ///< .weak_definition (MachO)
-  MCSA_WeakReference,       ///< .weak_reference (MachO)
-  MCSA_WeakDefAutoPrivate   ///< .weak_def_can_be_hidden (MachO)
+  MCSA_Global,                  ///< .globl
+  MCSA_LGlobal,                 ///< .lglobl (XCOFF)
+  MCSA_Hidden,                  ///< .hidden (ELF)
+  MCSA_IndirectSymbol,          ///< .indirect_symbol (MachO)
+  MCSA_Internal,                ///< .internal (ELF)
+  MCSA_LazyReference,           ///< .lazy_reference (MachO)
+  MCSA_Local,                   ///< .local (ELF)
+  MCSA_NoDeadStrip,             ///< .no_dead_strip (MachO)
+  MCSA_SymbolResolver,          ///< .symbol_resolver (MachO)
+  MCSA_AltEntry,                ///< .alt_entry (MachO)
+  MCSA_PrivateExtern,           ///< .private_extern (MachO)
+  MCSA_Protected,               ///< .protected (ELF)
+  MCSA_Reference,               ///< .reference (MachO)
+  MCSA_Weak,                    ///< .weak
+  MCSA_WeakDefinition,          ///< .weak_definition (MachO)
+  MCSA_WeakReference,           ///< .weak_reference (MachO)
+  MCSA_WeakDefAutoPrivate       ///< .weak_def_can_be_hidden (MachO)
 };
 
 enum MCAssemblerFlag {
diff --git a/llvm/include/llvm/MC/MCFixup.h b/llvm/include/llvm/MC/MCFixup.h
index 29e321e2354c3..affc846cbdd44 100644
--- a/llvm/include/llvm/MC/MCFixup.h
+++ b/llvm/include/llvm/MC/MCFixup.h
@@ -55,9 +55,14 @@ enum MCFixupKind {
 
   FirstTargetFixupKind = 128,
 
-  // Limit range of target fixups, in case we want to pack more efficiently
-  // later.
-  MaxTargetFixupKind = (1 << 8)
+  /// The range [FirstLiteralRelocationKind, MaxTargetFixupKind) is used for
+  /// relocations coming from .reloc directive. Fixup kind
+  /// FirstLiteralRelocationKind+V represents the relocation type with number V.
+  FirstLiteralRelocationKind = 256,
+
+  /// Set limit to accommodate the highest reloc type in use for all Targets,
+  /// currently R_AARCH64_IRELATIVE at 1032, including room for expansion.
+  MaxFixupKind = FirstLiteralRelocationKind + 1032 + 32,
 };
 
 /// Encode information on a single operation to perform on a byte
@@ -92,7 +97,7 @@ class MCFixup {
 public:
   static MCFixup create(uint32_t Offset, const MCExpr *Value,
                         MCFixupKind Kind, SMLoc Loc = SMLoc()) {
-    assert(Kind < MaxTargetFixupKind && "Kind out of range!");
+    assert(Kind <= MaxFixupKind && "Kind out of range!");
     MCFixup FI;
     FI.Value = Value;
     FI.Offset = Offset;
diff --git a/llvm/include/llvm/MC/MCFragment.h b/llvm/include/llvm/MC/MCFragment.h
index bde0835b6a553..4c8a895592ef2 100644
--- a/llvm/include/llvm/MC/MCFragment.h
+++ b/llvm/include/llvm/MC/MCFragment.h
@@ -259,6 +259,8 @@ class MCRelaxableFragment : public MCEncodedFragmentWithFixups<8, 1> {
 
   /// The instruction this is a fragment for.
   MCInst Inst;
+  /// Can we auto pad the instruction?
+  bool AllowAutoPadding = false;
 
 public:
   MCRelaxableFragment(const MCInst &Inst, const MCSubtargetInfo &STI,
@@ -269,6 +271,9 @@ class MCRelaxableFragment : public MCEncodedFragmentWithFixups<8, 1> {
   const MCInst &getInst() const { return Inst; }
   void setInst(const MCInst &Value) { Inst = Value; }
 
+  bool getAllowAutoPadding() const { return AllowAutoPadding; }
+  void setAllowAutoPadding(bool V) { AllowAutoPadding = V; }
+
   static bool classof(const MCFragment *F) {
     return F->getKind() == MCFragment::FT_Relaxable;
   }
diff --git a/llvm/include/llvm/MC/MCInstPrinter.h b/llvm/include/llvm/MC/MCInstPrinter.h
index 0df658ab29fd8..71e049b92455a 100644
--- a/llvm/include/llvm/MC/MCInstPrinter.h
+++ b/llvm/include/llvm/MC/MCInstPrinter.h
@@ -58,6 +58,11 @@ class MCInstPrinter {
   /// Which style to use for printing hexadecimal values.
   HexStyle::Style PrintHexStyle = HexStyle::C;
 
+  /// If true, a branch immediate (e.g. bl 4) will be printed as a hexadecimal
+  /// address (e.g. bl 0x20004). This is useful for a stream disassembler
+  /// (llvm-objdump -d).
+  bool PrintBranchImmAsAddress = false;
+
   /// Utility function for printing annotations.
   void printAnnotation(raw_ostream &OS, StringRef Annot);
 
@@ -79,6 +84,12 @@ class MCInstPrinter {
   void setCommentStream(raw_ostream &OS) { CommentStream = &OS; }
 
   /// Print the specified MCInst to the specified raw_ostream.
+  ///
+  /// \p Address the address of current instruction on most targets, used to
+  /// print a PC relative immediate as the target address. On targets where a PC
+  /// relative immediate is relative to the next instruction and the length of a
+  /// MCInst is difficult to measure (e.g. x86), this is the address of the next
+  /// instruction. If Address is 0, the immediate will be printed.
   virtual void printInst(const MCInst *MI, uint64_t Address, StringRef Annot,
                          const MCSubtargetInfo &STI, raw_ostream &OS) = 0;
 
@@ -100,6 +111,10 @@ class MCInstPrinter {
 
   void setPrintHexStyle(HexStyle::Style Value) { PrintHexStyle = Value; }
 
+  void setPrintBranchImmAsAddress(bool Value) {
+    PrintBranchImmAsAddress = Value;
+  }
+
   /// Utility function to print immediates in decimal or hex.
   format_object<int64_t> formatImm(int64_t Value) const {
     return PrintImmHex ? formatHex(Value) : formatDec(Value);
diff --git a/llvm/include/llvm/MC/MCInstrDesc.h b/llvm/include/llvm/MC/MCInstrDesc.h
index 29175af7eb093..fa620d21a1201 100644
--- a/llvm/include/llvm/MC/MCInstrDesc.h
+++ b/llvm/include/llvm/MC/MCInstrDesc.h
@@ -197,15 +197,6 @@ class MCInstrDesc {
   const MCPhysReg *ImplicitUses; // Registers implicitly read by this instr
   const MCPhysReg *ImplicitDefs; // Registers implicitly defined by this instr
   const MCOperandInfo *OpInfo;   // 'NumOperands' entries about operands
-  // Subtarget feature that this is deprecated on, if any
-  // -1 implies this is not deprecated by any single feature. It may still be
-  // deprecated due to a "complex" reason, below.
-  int64_t DeprecatedFeature;
-
-  // A complex method to determine if a certain instruction is deprecated or
-  // not, and return the reason for deprecation.
-  bool (*ComplexDeprecationInfo)(MCInst &, const MCSubtargetInfo &,
-                                 std::string &);
 
   /// Returns the value of the specific constraint if
   /// it is set. Returns -1 if it is not set.
@@ -219,11 +210,6 @@ class MCInstrDesc {
     return -1;
   }
 
-  /// Returns true if a certain instruction is deprecated and if so
-  /// returns the reason in \p Info.
-  bool getDeprecatedInfo(MCInst &MI, const MCSubtargetInfo &STI,
-                         std::string &Info) const;
-
   /// Return the opcode number for this descriptor.
   unsigned getOpcode() const { return Opcode; }
 
diff --git a/llvm/include/llvm/MC/MCInstrInfo.h b/llvm/include/llvm/MC/MCInstrInfo.h
index 874b1e46795b1..f5c944eed6827 100644
--- a/llvm/include/llvm/MC/MCInstrInfo.h
+++ b/llvm/include/llvm/MC/MCInstrInfo.h
@@ -21,19 +21,35 @@ namespace llvm {
 //---------------------------------------------------------------------------
 /// Interface to description of machine instruction set.
 class MCInstrInfo {
+public:
+  using ComplexDeprecationPredicate = bool (*)(MCInst &,
+                                               const MCSubtargetInfo &,
+                                               std::string &);
+
+private:
   const MCInstrDesc *Desc;          // Raw array to allow static init'n
   const unsigned *InstrNameIndices; // Array for name indices in InstrNameData
   const char *InstrNameData;        // Instruction name string pool
+  // Subtarget feature that an instruction is deprecated on, if any
+  // -1 implies this is not deprecated by any single feature. It may still be
+  // deprecated due to a "complex" reason, below.
+  const uint8_t *DeprecatedFeatures;
+  // A complex method to determine if a certain instruction is deprecated or
+  // not, and return the reason for deprecation.
+  const ComplexDeprecationPredicate *ComplexDeprecationInfos;
   unsigned NumOpcodes;              // Number of entries in the desc array
 
 public:
   /// Initialize MCInstrInfo, called by TableGen auto-generated routines.
   /// *DO NOT USE*.
   void InitMCInstrInfo(const MCInstrDesc *D, const unsigned *NI, const char *ND,
-                       unsigned NO) {
+                       const uint8_t *DF,
+                       const ComplexDeprecationPredicate *CDI, unsigned NO) {
     Desc = D;
     InstrNameIndices = NI;
     InstrNameData = ND;
+    DeprecatedFeatures = DF;
+    ComplexDeprecationInfos = CDI;
     NumOpcodes = NO;
   }
 
@@ -51,6 +67,11 @@ class MCInstrInfo {
     assert(Opcode < NumOpcodes && "Invalid opcode!");
     return StringRef(&InstrNameData[InstrNameIndices[Opcode]]);
   }
+
+  /// Returns true if a certain instruction is deprecated and if so
+  /// returns the reason in \p Info.
+  bool getDeprecatedInfo(MCInst &MI, const MCSubtargetInfo &STI,
+                         std::string &Info) const;
 };
 
 } // End llvm namespace
diff --git a/llvm/include/llvm/MC/MCObjectStreamer.h b/llvm/include/llvm/MC/MCObjectStreamer.h
index b0a3bd31a3cb6..362b6e923e5db 100644
--- a/llvm/include/llvm/MC/MCObjectStreamer.h
+++ b/llvm/include/llvm/MC/MCObjectStreamer.h
@@ -87,11 +87,6 @@ class MCObjectStreamer : public MCStreamer {
   /// if the Subtarget differs from the current fragment.
   MCDataFragment *getOrCreateDataFragment(const MCSubtargetInfo* STI = nullptr);
 
-  /// Get a boundary-align fragment to write into, creating a new one if the
-  /// current fragment is not a boundary-align fragment or has been used to emit
-  /// something.
-  MCBoundaryAlignFragment *getOrCreateBoundaryAlignFragment();
-
 protected:
   bool changeSectionImpl(MCSection *Section, const MCExpr *Subsection);
 
diff --git a/llvm/include/llvm/MC/MCSectionXCOFF.h b/llvm/include/llvm/MC/MCSectionXCOFF.h
index 611eb69c14935..5cc6921e058f4 100644
--- a/llvm/include/llvm/MC/MCSectionXCOFF.h
+++ b/llvm/include/llvm/MC/MCSectionXCOFF.h
@@ -48,7 +48,7 @@ class MCSectionXCOFF final : public MCSection {
            "Invalid or unhandled type for csect.");
     assert(QualName != nullptr && "QualName is needed.");
     QualName->setStorageClass(SC);
-    QualName->setContainingCsect(this);
+    QualName->setRepresentedCsect(this);
   }
 
 public:
diff --git a/llvm/include/llvm/MC/MCSubtargetInfo.h b/llvm/include/llvm/MC/MCSubtargetInfo.h
index 09130c4641ef1..61cbb842502e4 100644
--- a/llvm/include/llvm/MC/MCSubtargetInfo.h
+++ b/llvm/include/llvm/MC/MCSubtargetInfo.h
@@ -263,10 +263,17 @@ class MCSubtargetInfo {
   ///
   virtual unsigned getMaxPrefetchIterationsAhead() const;
 
+  /// \return True if prefetching should also be done for writes.
+  ///
+  virtual bool enableWritePrefetching() const;
+
   /// Return the minimum stride necessary to trigger software
   /// prefetching.
   ///
-  virtual unsigned getMinPrefetchStride() const;
+  virtual unsigned getMinPrefetchStride(unsigned NumMemAccesses,
+                                        unsigned NumStridedMemAccesses,
+                                        unsigned NumPrefetches,
+                                        bool HasCall) const;
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/MC/MCSymbolXCOFF.h b/llvm/include/llvm/MC/MCSymbolXCOFF.h
index 07dfb5d299776..6815f421d598a 100644
--- a/llvm/include/llvm/MC/MCSymbolXCOFF.h
+++ b/llvm/include/llvm/MC/MCSymbolXCOFF.h
@@ -36,21 +36,6 @@ class MCSymbolXCOFF : public MCSymbol {
     return StorageClass.getValue();
   }
 
-  void setContainingCsect(MCSectionXCOFF *C) {
-    assert((!ContainingCsect || ContainingCsect == C) &&
-           "Trying to set a containing csect that doesn't match the one that"
-           "this symbol is already mapped to.");
-    ContainingCsect = C;
-  }
-
-  MCSectionXCOFF *getContainingCsect() const {
-    assert(ContainingCsect &&
-           "Trying to get containing csect but none was set.");
-    return ContainingCsect;
-  }
-
-  bool hasContainingCsect() const { return ContainingCsect != nullptr; }
-
   StringRef getUnqualifiedName() const {
     const StringRef name = getName();
     if (name.back() == ']') {
@@ -62,9 +47,15 @@ class MCSymbolXCOFF : public MCSymbol {
     return name;
   }
 
+  bool hasRepresentedCsectSet() const { return RepresentedCsect != nullptr; }
+
+  MCSectionXCOFF *getRepresentedCsect() const;
+
+  void setRepresentedCsect(MCSectionXCOFF *C);
+
 private:
   Optional<XCOFF::StorageClass> StorageClass;
-  MCSectionXCOFF *ContainingCsect = nullptr;
+  MCSectionXCOFF *RepresentedCsect = nullptr;
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/MCA/CodeEmitter.h b/llvm/include/llvm/MCA/CodeEmitter.h
index c8d222bd8c2fd..edbadcc8fafa4 100644
--- a/llvm/include/llvm/MCA/CodeEmitter.h
+++ b/llvm/include/llvm/MCA/CodeEmitter.h
@@ -20,11 +20,8 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCCodeEmitter.h"
-#include "llvm/MC/MCFixup.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCSubtargetInfo.h"
-#include "llvm/MCA/Instruction.h"
-#include "llvm/MCA/Support.h"
 #include "llvm/Support/raw_ostream.h"
 
 #include <string>
diff --git a/llvm/include/llvm/MCA/HardwareUnits/LSUnit.h b/llvm/include/llvm/MCA/HardwareUnits/LSUnit.h
index 34903794db4a6..9143adf0e97bd 100644
--- a/llvm/include/llvm/MCA/HardwareUnits/LSUnit.h
+++ b/llvm/include/llvm/MCA/HardwareUnits/LSUnit.h
@@ -24,8 +24,6 @@
 namespace llvm {
 namespace mca {
 
-class Scheduler;
-
 /// A node of a memory dependency graph. A MemoryGroup describes a set of
 /// instructions with same memory dependencies.
 ///
diff --git a/llvm/include/llvm/MCA/HardwareUnits/RegisterFile.h b/llvm/include/llvm/MCA/HardwareUnits/RegisterFile.h
index cd7718d987449..344df7dbf7fa0 100644
--- a/llvm/include/llvm/MCA/HardwareUnits/RegisterFile.h
+++ b/llvm/include/llvm/MCA/HardwareUnits/RegisterFile.h
@@ -22,7 +22,6 @@
 #include "llvm/MC/MCSchedule.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MCA/HardwareUnits/HardwareUnit.h"
-#include "llvm/Support/Error.h"
 
 namespace llvm {
 namespace mca {
diff --git a/llvm/include/llvm/MCA/HardwareUnits/ResourceManager.h b/llvm/include/llvm/MCA/HardwareUnits/ResourceManager.h
index 917af3750044e..b6d4e345da2cc 100644
--- a/llvm/include/llvm/MCA/HardwareUnits/ResourceManager.h
+++ b/llvm/include/llvm/MCA/HardwareUnits/ResourceManager.h
@@ -15,7 +15,6 @@
 #ifndef LLVM_MCA_RESOURCE_MANAGER_H
 #define LLVM_MCA_RESOURCE_MANAGER_H
 
-#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/MC/MCSchedule.h"
diff --git a/llvm/include/llvm/MCA/Pipeline.h b/llvm/include/llvm/MCA/Pipeline.h
index 935033f67f8bf..0ac988c52dc13 100644
--- a/llvm/include/llvm/MCA/Pipeline.h
+++ b/llvm/include/llvm/MCA/Pipeline.h
@@ -15,8 +15,6 @@
 #ifndef LLVM_MCA_PIPELINE_H
 #define LLVM_MCA_PIPELINE_H
 
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/MCA/HardwareUnits/Scheduler.h"
 #include "llvm/MCA/Stages/Stage.h"
 #include "llvm/Support/Error.h"
 
diff --git a/llvm/include/llvm/MCA/Stages/DispatchStage.h b/llvm/include/llvm/MCA/Stages/DispatchStage.h
index d80ededeaca14..597f7312448d2 100644
--- a/llvm/include/llvm/MCA/Stages/DispatchStage.h
+++ b/llvm/include/llvm/MCA/Stages/DispatchStage.h
@@ -20,7 +20,6 @@
 
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
-#include "llvm/MCA/HWEventListener.h"
 #include "llvm/MCA/HardwareUnits/RegisterFile.h"
 #include "llvm/MCA/HardwareUnits/RetireControlUnit.h"
 #include "llvm/MCA/Instruction.h"
diff --git a/llvm/include/llvm/Object/COFF.h b/llvm/include/llvm/Object/COFF.h
index b91ee5887fec7..f1386924eccaa 100644
--- a/llvm/include/llvm/Object/COFF.h
+++ b/llvm/include/llvm/Object/COFF.h
@@ -912,6 +912,7 @@ class COFFObjectFile : public ObjectFile {
   bool isSectionData(DataRefImpl Sec) const override;
   bool isSectionBSS(DataRefImpl Sec) const override;
   bool isSectionVirtual(DataRefImpl Sec) const override;
+  bool isDebugSection(StringRef SectionName) const override;
   relocation_iterator section_rel_begin(DataRefImpl Sec) const override;
   relocation_iterator section_rel_end(DataRefImpl Sec) const override;
 
diff --git a/llvm/include/llvm/Object/ELFObjectFile.h b/llvm/include/llvm/Object/ELFObjectFile.h
index e2c9b42e04b42..5c10badda394d 100644
--- a/llvm/include/llvm/Object/ELFObjectFile.h
+++ b/llvm/include/llvm/Object/ELFObjectFile.h
@@ -28,8 +28,8 @@
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Object/SymbolicFile.h"
 #include "llvm/Support/ARMAttributeParser.h"
-#include "llvm/Support/ARMBuildAttributes.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/ELFAttributes.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -64,7 +64,7 @@ class ELFObjectFileBase : public ObjectFile {
   virtual uint64_t getSectionOffset(DataRefImpl Sec) const = 0;
 
   virtual Expected<int64_t> getRelocationAddend(DataRefImpl Rel) const = 0;
-  virtual Error getBuildAttributes(ARMAttributeParser &Attributes) const = 0;
+  virtual Error getBuildAttributes(ELFAttributeParser &Attributes) const = 0;
 
 public:
   using elf_symbol_iterator_range = iterator_range<elf_symbol_iterator>;
@@ -285,6 +285,7 @@ template <class ELFT> class ELFObjectFile : public ELFObjectFileBase {
   bool isSectionVirtual(DataRefImpl Sec) const override;
   bool isBerkeleyText(DataRefImpl Sec) const override;
   bool isBerkeleyData(DataRefImpl Sec) const override;
+  bool isDebugSection(StringRef SectionName) const override;
   relocation_iterator section_rel_begin(DataRefImpl Sec) const override;
   relocation_iterator section_rel_end(DataRefImpl Sec) const override;
   std::vector<SectionRef> dynamic_relocation_sections() const override;
@@ -365,19 +366,20 @@ template <class ELFT> class ELFObjectFile : public ELFObjectFileBase {
         (Visibility == ELF::STV_DEFAULT || Visibility == ELF::STV_PROTECTED));
   }
 
-  Error getBuildAttributes(ARMAttributeParser &Attributes) const override {
+  Error getBuildAttributes(ELFAttributeParser &Attributes) const override {
     auto SectionsOrErr = EF.sections();
     if (!SectionsOrErr)
       return SectionsOrErr.takeError();
 
     for (const Elf_Shdr &Sec : *SectionsOrErr) {
-      if (Sec.sh_type == ELF::SHT_ARM_ATTRIBUTES) {
+      if (Sec.sh_type == ELF::SHT_ARM_ATTRIBUTES ||
+          Sec.sh_type == ELF::SHT_RISCV_ATTRIBUTES) {
         auto ErrorOrContents = EF.getSectionContents(&Sec);
         if (!ErrorOrContents)
           return ErrorOrContents.takeError();
 
         auto Contents = ErrorOrContents.get();
-        if (Contents[0] != ARMBuildAttrs::Format_Version || Contents.size() == 1)
+        if (Contents[0] != ELFAttrs::Format_Version || Contents.size() == 1)
           return Error::success();
 
         if (Error E = Attributes.parse(Contents, ELFT::TargetEndianness))
@@ -812,6 +814,12 @@ bool ELFObjectFile<ELFT>::isBerkeleyData(DataRefImpl Sec) const {
          EShdr->sh_flags & ELF::SHF_ALLOC;
 }
 
+template <class ELFT>
+bool ELFObjectFile<ELFT>::isDebugSection(StringRef SectionName) const {
+  return SectionName.startswith(".debug") ||
+         SectionName.startswith(".zdebug") || SectionName == ".gdb_index";
+}
+
 template <class ELFT>
 relocation_iterator
 ELFObjectFile<ELFT>::section_rel_begin(DataRefImpl Sec) const {
diff --git a/llvm/include/llvm/Object/MachO.h b/llvm/include/llvm/Object/MachO.h
index 380e3908d9da2..351ee419d6aa5 100644
--- a/llvm/include/llvm/Object/MachO.h
+++ b/llvm/include/llvm/Object/MachO.h
@@ -309,6 +309,7 @@ class MachOObjectFile : public ObjectFile {
   bool isSectionBSS(DataRefImpl Sec) const override;
   bool isSectionVirtual(DataRefImpl Sec) const override;
   bool isSectionBitcode(DataRefImpl Sec) const override;
+  bool isDebugSection(StringRef SectionName) const override;
 
   /// When dsymutil generates the companion file, it strips all unnecessary
   /// sections (e.g. everything in the _TEXT segment) by omitting their body
diff --git a/llvm/include/llvm/Object/ObjectFile.h b/llvm/include/llvm/Object/ObjectFile.h
index 2f14934576059..ef82c057012f5 100644
--- a/llvm/include/llvm/Object/ObjectFile.h
+++ b/llvm/include/llvm/Object/ObjectFile.h
@@ -123,6 +123,9 @@ class SectionRef {
   /// contains data (e.g. PROGBITS), but is not text.
   bool isBerkeleyData() const;
 
+  /// Whether this section is a debug section.
+  bool isDebugSection(StringRef SectionName) const;
+
   bool containsSymbol(SymbolRef S) const;
 
   relocation_iterator relocation_begin() const;
@@ -272,6 +275,7 @@ class ObjectFile : public SymbolicFile {
   virtual bool isSectionStripped(DataRefImpl Sec) const;
   virtual bool isBerkeleyText(DataRefImpl Sec) const;
   virtual bool isBerkeleyData(DataRefImpl Sec) const;
+  virtual bool isDebugSection(StringRef SectionName) const;
   virtual relocation_iterator section_rel_begin(DataRefImpl Sec) const = 0;
   virtual relocation_iterator section_rel_end(DataRefImpl Sec) const = 0;
   virtual Expected<section_iterator> getRelocatedSection(DataRefImpl Sec) const;
@@ -495,6 +499,10 @@ inline bool SectionRef::isBerkeleyData() const {
   return OwningObject->isBerkeleyData(SectionPimpl);
 }
 
+inline bool SectionRef::isDebugSection(StringRef SectionName) const {
+  return OwningObject->isDebugSection(SectionName);
+}
+
 inline relocation_iterator SectionRef::relocation_begin() const {
   return OwningObject->section_rel_begin(SectionPimpl);
 }
diff --git a/llvm/include/llvm/Object/Wasm.h b/llvm/include/llvm/Object/Wasm.h
index b3cbb5703605c..e5b21d02744db 100644
--- a/llvm/include/llvm/Object/Wasm.h
+++ b/llvm/include/llvm/Object/Wasm.h
@@ -242,8 +242,8 @@ class WasmObjectFile : public ObjectFile {
   Error parseFunctionSection(ReadContext &Ctx);
   Error parseTableSection(ReadContext &Ctx);
   Error parseMemorySection(ReadContext &Ctx);
-  Error parseGlobalSection(ReadContext &Ctx);
   Error parseEventSection(ReadContext &Ctx);
+  Error parseGlobalSection(ReadContext &Ctx);
   Error parseExportSection(ReadContext &Ctx);
   Error parseStartSection(ReadContext &Ctx);
   Error parseElemSection(ReadContext &Ctx);
@@ -290,8 +290,8 @@ class WasmObjectFile : public ObjectFile {
   uint32_t NumImportedEvents = 0;
   uint32_t CodeSection = 0;
   uint32_t DataSection = 0;
-  uint32_t GlobalSection = 0;
   uint32_t EventSection = 0;
+  uint32_t GlobalSection = 0;
 };
 
 class WasmSectionOrderChecker {
@@ -307,8 +307,8 @@ class WasmSectionOrderChecker {
     WASM_SEC_ORDER_FUNCTION,
     WASM_SEC_ORDER_TABLE,
     WASM_SEC_ORDER_MEMORY,
-    WASM_SEC_ORDER_GLOBAL,
     WASM_SEC_ORDER_EVENT,
+    WASM_SEC_ORDER_GLOBAL,
     WASM_SEC_ORDER_EXPORT,
     WASM_SEC_ORDER_START,
     WASM_SEC_ORDER_ELEM,
diff --git a/llvm/include/llvm/Object/XCOFFObjectFile.h b/llvm/include/llvm/Object/XCOFFObjectFile.h
index fcdbf7a8095ce..02550c6bb86c6 100644
--- a/llvm/include/llvm/Object/XCOFFObjectFile.h
+++ b/llvm/include/llvm/Object/XCOFFObjectFile.h
@@ -15,6 +15,7 @@
 
 #include "llvm/BinaryFormat/XCOFF.h"
 #include "llvm/Object/ObjectFile.h"
+#include <limits>
 
 namespace llvm {
 namespace object {
@@ -247,6 +248,9 @@ class XCOFFObjectFile : public ObjectFile {
   void checkSectionAddress(uintptr_t Addr, uintptr_t TableAddr) const;
 
 public:
+  static constexpr uint64_t InvalidRelocOffset =
+      std::numeric_limits<uint64_t>::max();
+
   // Interface inherited from base classes.
   void moveSymbolNext(DataRefImpl &Symb) const override;
   uint32_t getSymbolFlags(DataRefImpl Symb) const override;
@@ -278,6 +282,10 @@ class XCOFFObjectFile : public ObjectFile {
   relocation_iterator section_rel_end(DataRefImpl Sec) const override;
 
   void moveRelocationNext(DataRefImpl &Rel) const override;
+
+  /// \returns the relocation offset with the base address of the containing
+  /// section as zero, or InvalidRelocOffset on errors (such as a relocation
+  /// that does not refer to an address in any section).
   uint64_t getRelocationOffset(DataRefImpl Rel) const override;
   symbol_iterator getRelocationSymbol(DataRefImpl Rel) const override;
   uint64_t getRelocationType(DataRefImpl Rel) const override;
diff --git a/llvm/include/llvm/ObjectYAML/ELFYAML.h b/llvm/include/llvm/ObjectYAML/ELFYAML.h
index 44f5d7cd069ed..1b12a9b23130d 100644
--- a/llvm/include/llvm/ObjectYAML/ELFYAML.h
+++ b/llvm/include/llvm/ObjectYAML/ELFYAML.h
@@ -177,6 +177,9 @@ struct Section : public Chunk {
   // When they are, this flag is used to signal about that.
   bool IsImplicit;
 
+  // Holds the original section index.
+  unsigned OriginalSecNdx;
+
   Section(ChunkKind Kind, bool IsImplicit = false)
       : Chunk(Kind), IsImplicit(IsImplicit) {}
 
@@ -279,6 +282,11 @@ struct HashSection : Section {
   Optional<std::vector<uint32_t>> Bucket;
   Optional<std::vector<uint32_t>> Chain;
 
+  // The following members are used to override section fields.
+  // This is useful for creating invalid objects.
+  Optional<llvm::yaml::Hex64> NBucket;
+  Optional<llvm::yaml::Hex64> NChain;
+
   HashSection() : Section(ChunkKind::Hash) {}
 
   static bool classof(const Chunk *S) { return S->Kind == ChunkKind::Hash; }
diff --git a/llvm/include/llvm/ObjectYAML/WasmYAML.h b/llvm/include/llvm/ObjectYAML/WasmYAML.h
index 15a8cc215020b..3f9d6bf41ea6d 100644
--- a/llvm/include/llvm/ObjectYAML/WasmYAML.h
+++ b/llvm/include/llvm/ObjectYAML/WasmYAML.h
@@ -309,24 +309,24 @@ struct MemorySection : Section {
   std::vector<Limits> Memories;
 };
 
-struct GlobalSection : Section {
-  GlobalSection() : Section(wasm::WASM_SEC_GLOBAL) {}
+struct EventSection : Section {
+  EventSection() : Section(wasm::WASM_SEC_EVENT) {}
 
   static bool classof(const Section *S) {
-    return S->Type == wasm::WASM_SEC_GLOBAL;
+    return S->Type == wasm::WASM_SEC_EVENT;
   }
 
-  std::vector<Global> Globals;
+  std::vector<Event> Events;
 };
 
-struct EventSection : Section {
-  EventSection() : Section(wasm::WASM_SEC_EVENT) {}
+struct GlobalSection : Section {
+  GlobalSection() : Section(wasm::WASM_SEC_GLOBAL) {}
 
   static bool classof(const Section *S) {
-    return S->Type == wasm::WASM_SEC_EVENT;
+    return S->Type == wasm::WASM_SEC_GLOBAL;
   }
 
-  std::vector<Event> Events;
+  std::vector<Global> Globals;
 };
 
 struct ExportSection : Section {
diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h
index 38af896229776..12e914d44ba3d 100644
--- a/llvm/include/llvm/Passes/PassBuilder.h
+++ b/llvm/include/llvm/Passes/PassBuilder.h
@@ -105,6 +105,10 @@ class PipelineTuningOptions {
   /// Tuning option to disable promotion to scalars in LICM with MemorySSA, if
   /// the number of access is too large.
   unsigned LicmMssaNoAccForPromotionCap;
+
+  /// Tuning option to enable/disable call graph profile. Its default value is
+  /// that of the flag: `-enable-npm-call-graph-profile`.
+  bool CallGraphProfile;
 };
 
 /// This class provides access to building LLVM's passes.
diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h
index 4019fe2549be7..7b1086c74919e 100644
--- a/llvm/include/llvm/ProfileData/SampleProf.h
+++ b/llvm/include/llvm/ProfileData/SampleProf.h
@@ -101,14 +101,14 @@ static inline uint64_t SPMagic(SampleProfileFormat Format = SPF_Binary) {
          uint64_t('2') << (64 - 56) | uint64_t(Format);
 }
 
-// Get the proper representation of a string in the input Format.
-static inline StringRef getRepInFormat(StringRef Name,
-                                       SampleProfileFormat Format,
+/// Get the proper representation of a string according to whether the
+/// current Format uses MD5 to represent the string.
+static inline StringRef getRepInFormat(StringRef Name, bool UseMD5,
                                        std::string &GUIDBuf) {
   if (Name.empty())
     return Name;
   GUIDBuf = std::to_string(Function::getGUID(Name));
-  return (Format == SPF_Compact_Binary) ? StringRef(GUIDBuf) : Name;
+  return UseMD5 ? StringRef(GUIDBuf) : Name;
 }
 
 static inline uint64_t SPVersion() { return 103; }
@@ -154,18 +154,64 @@ struct SecHdrTableEntry {
   uint64_t Size;
 };
 
-enum SecFlags { SecFlagInValid = 0, SecFlagCompress = (1 << 0) };
+// Flags common for all sections are defined here. In SecHdrTableEntry::Flags,
+// common flags will be saved in the lower 32bits and section specific flags
+// will be saved in the higher 32 bits.
+enum class SecCommonFlags : uint32_t {
+  SecFlagInValid = 0,
+  SecFlagCompress = (1 << 0)
+};
+
+// Section specific flags are defined here.
+// !!!Note: Everytime a new enum class is created here, please add
+// a new check in verifySecFlag.
+enum class SecNameTableFlags : uint32_t {
+  SecFlagInValid = 0,
+  SecFlagMD5Name = (1 << 0)
+};
+
+// Verify section specific flag is used for the correct section.
+template <class SecFlagType>
+static inline void verifySecFlag(SecType Type, SecFlagType Flag) {
+  // No verification is needed for common flags.
+  if (std::is_same<SecCommonFlags, SecFlagType>())
+    return;
+
+  // Verification starts here for section specific flag.
+  bool IsFlagLegal = false;
+  switch (Type) {
+  case SecNameTable:
+    IsFlagLegal = std::is_same<SecNameTableFlags, SecFlagType>();
+    break;
+  default:
+    break;
+  }
+  if (!IsFlagLegal)
+    llvm_unreachable("Misuse of a flag in an incompatible section");
+}
 
-static inline void addSecFlags(SecHdrTableEntry &Entry, uint64_t Flags) {
-  Entry.Flags |= Flags;
+template <class SecFlagType>
+static inline void addSecFlag(SecHdrTableEntry &Entry, SecFlagType Flag) {
+  verifySecFlag(Entry.Type, Flag);
+  auto FVal = static_cast<uint64_t>(Flag);
+  bool IsCommon = std::is_same<SecCommonFlags, SecFlagType>();
+  Entry.Flags |= IsCommon ? FVal : (FVal << 32);
 }
 
-static inline void removeSecFlags(SecHdrTableEntry &Entry, uint64_t Flags) {
-  Entry.Flags &= ~Flags;
+template <class SecFlagType>
+static inline void removeSecFlag(SecHdrTableEntry &Entry, SecFlagType Flag) {
+  verifySecFlag(Entry.Type, Flag);
+  auto FVal = static_cast<uint64_t>(Flag);
+  bool IsCommon = std::is_same<SecCommonFlags, SecFlagType>();
+  Entry.Flags &= ~(IsCommon ? FVal : (FVal << 32));
 }
 
-static inline bool hasSecFlag(SecHdrTableEntry &Entry, SecFlags Flag) {
-  return Entry.Flags & Flag;
+template <class SecFlagType>
+static inline bool hasSecFlag(const SecHdrTableEntry &Entry, SecFlagType Flag) {
+  verifySecFlag(Entry.Type, Flag);
+  auto FVal = static_cast<uint64_t>(Flag);
+  bool IsCommon = std::is_same<SecCommonFlags, SecFlagType>();
+  return Entry.Flags & (IsCommon ? FVal : (FVal << 32));
 }
 
 /// Represents the relative location of an instruction.
@@ -379,7 +425,7 @@ class FunctionSamples {
   const FunctionSamples *findFunctionSamplesAt(const LineLocation &Loc,
                                                StringRef CalleeName) const {
     std::string CalleeGUID;
-    CalleeName = getRepInFormat(CalleeName, Format, CalleeGUID);
+    CalleeName = getRepInFormat(CalleeName, UseMD5, CalleeGUID);
 
     auto iter = CallsiteSamples.find(Loc);
     if (iter == CallsiteSamples.end())
@@ -527,13 +573,13 @@ class FunctionSamples {
   }
 
   /// Translate \p Name into its original name in Module.
-  /// When the Format is not SPF_Compact_Binary, \p Name needs no translation.
-  /// When the Format is SPF_Compact_Binary, \p Name in current FunctionSamples
+  /// When profile doesn't use MD5, \p Name needs no translation.
+  /// When profile uses MD5, \p Name in current FunctionSamples
   /// is actually GUID of the original function name. getNameInModule will
   /// translate \p Name in current FunctionSamples into its original name.
   /// If the original name doesn't exist in \p M, return empty StringRef.
   StringRef getNameInModule(StringRef Name, const Module *M) const {
-    if (Format != SPF_Compact_Binary)
+    if (!UseMD5)
       return Name;
 
     assert(GUIDToFuncNameMap && "GUIDToFuncNameMap needs to be popluated first");
@@ -560,16 +606,18 @@ class FunctionSamples {
 
   static SampleProfileFormat Format;
 
+  /// Whether the profile uses MD5 to represent string.
+  static bool UseMD5;
+
   /// GUIDToFuncNameMap saves the mapping from GUID to the symbol name, for
   /// all the function symbols defined or declared in current module.
   DenseMap<uint64_t, StringRef> *GUIDToFuncNameMap = nullptr;
 
   // Assume the input \p Name is a name coming from FunctionSamples itself.
-  // If the format is SPF_Compact_Binary, the name is already a GUID and we
+  // If UseMD5 is true, the name is already a GUID and we
   // don't want to return the GUID of GUID.
   static uint64_t getGUID(StringRef Name) {
-    return (Format == SPF_Compact_Binary) ? std::stoull(Name.data())
-                                          : Function::getGUID(Name);
+    return UseMD5 ? std::stoull(Name.data()) : Function::getGUID(Name);
   }
 
 private:
diff --git a/llvm/include/llvm/ProfileData/SampleProfReader.h b/llvm/include/llvm/ProfileData/SampleProfReader.h
index 72b178edc260d..0e8ee7696c543 100644
--- a/llvm/include/llvm/ProfileData/SampleProfReader.h
+++ b/llvm/include/llvm/ProfileData/SampleProfReader.h
@@ -335,6 +335,7 @@ class SampleProfileReader {
       return EC;
     if (Remapper)
       Remapper->applyRemapping(Ctx);
+    FunctionSamples::UseMD5 = useMD5();
     return sampleprof_error::success;
   }
 
@@ -363,7 +364,7 @@ class SampleProfileReader {
   FunctionSamples *getOrCreateSamplesFor(const Function &F) {
     std::string FGUID;
     StringRef CanonName = FunctionSamples::getCanonicalFnName(F);
-    CanonName = getRepInFormat(CanonName, getFormat(), FGUID);
+    CanonName = getRepInFormat(CanonName, useMD5(), FGUID);
     return &Profiles[CanonName];
   }
 
@@ -374,7 +375,7 @@ class SampleProfileReader {
         return FS;
     }
     std::string FGUID;
-    Fname = getRepInFormat(Fname, getFormat(), FGUID);
+    Fname = getRepInFormat(Fname, useMD5(), FGUID);
     auto It = Profiles.find(Fname);
     if (It != Profiles.end())
       return &It->second;
@@ -419,6 +420,9 @@ class SampleProfileReader {
   virtual std::vector<StringRef> *getNameTable() { return nullptr; }
   virtual bool dumpSectionInfo(raw_ostream &OS = dbgs()) { return false; };
 
+  /// Return whether names in the profile are all MD5 numbers.
+  virtual bool useMD5() { return false; }
+
 protected:
   /// Map every function to its associated profile.
   ///
@@ -590,7 +594,7 @@ class SampleProfileReaderExtBinaryBase : public SampleProfileReaderBinary {
   virtual std::error_code readHeader() override;
   virtual std::error_code verifySPMagic(uint64_t Magic) override = 0;
   virtual std::error_code readOneSection(const uint8_t *Start, uint64_t Size,
-                                         SecType Type) = 0;
+                                         const SecHdrTableEntry &Entry) = 0;
 
 public:
   SampleProfileReaderExtBinaryBase(std::unique_ptr<MemoryBuffer> B,
@@ -610,11 +614,14 @@ class SampleProfileReaderExtBinaryBase : public SampleProfileReaderBinary {
 class SampleProfileReaderExtBinary : public SampleProfileReaderExtBinaryBase {
 private:
   virtual std::error_code verifySPMagic(uint64_t Magic) override;
-  virtual std::error_code readOneSection(const uint8_t *Start, uint64_t Size,
-                                         SecType Type) override;
+  virtual std::error_code
+  readOneSection(const uint8_t *Start, uint64_t Size,
+                 const SecHdrTableEntry &Entry) override;
   std::error_code readProfileSymbolList();
   std::error_code readFuncOffsetTable();
   std::error_code readFuncProfiles();
+  std::error_code readMD5NameTable();
+  std::error_code readNameTableSec(bool IsMD5);
 
   /// The table mapping from function name to the offset of its FunctionSample
   /// towards file start.
@@ -624,6 +631,15 @@ class SampleProfileReaderExtBinary : public SampleProfileReaderExtBinaryBase {
   /// Use all functions from the input profile.
   bool UseAllFuncs = true;
 
+  /// If MD5 is used in NameTable section, the section saves uint64_t data.
+  /// The uint64_t data has to be converted to a string and then the string
+  /// will be used to initialize StringRef in NameTable.
+  /// Note NameTable contains StringRef so it needs another buffer to own
+  /// the string data. MD5StringBuf serves as the string buffer that is
+  /// referenced by NameTable (vector of StringRef). We make sure
+  /// the lifetime of MD5StringBuf is not shorter than that of NameTable.
+  std::unique_ptr<std::vector<std::string>> MD5StringBuf;
+
 public:
   SampleProfileReaderExtBinary(std::unique_ptr<MemoryBuffer> B, LLVMContext &C,
                                SampleProfileFormat Format = SPF_Ext_Binary)
@@ -638,6 +654,12 @@ class SampleProfileReaderExtBinary : public SampleProfileReaderExtBinaryBase {
 
   /// Collect functions with definitions in Module \p M.
   void collectFuncsFrom(const Module &M) override;
+
+  /// Return whether names in the profile are all MD5 numbers.
+  virtual bool useMD5() override {
+    assert(!NameTable.empty() && "NameTable should have been initialized");
+    return MD5StringBuf && !MD5StringBuf->empty();
+  }
 };
 
 class SampleProfileReaderCompactBinary : public SampleProfileReaderBinary {
@@ -671,6 +693,9 @@ class SampleProfileReaderCompactBinary : public SampleProfileReaderBinary {
 
   /// Collect functions to be used when compiling Module \p M.
   void collectFuncsFrom(const Module &M) override;
+
+  /// Return whether names in the profile are all MD5 numbers.
+  virtual bool useMD5() override { return true; }
 };
 
 using InlineCallStack = SmallVector<FunctionSamples *, 10>;
diff --git a/llvm/include/llvm/ProfileData/SampleProfWriter.h b/llvm/include/llvm/ProfileData/SampleProfWriter.h
index 5814f69fdcabf..54500f513ad1f 100644
--- a/llvm/include/llvm/ProfileData/SampleProfWriter.h
+++ b/llvm/include/llvm/ProfileData/SampleProfWriter.h
@@ -57,6 +57,8 @@ class SampleProfileWriter {
   create(std::unique_ptr<raw_ostream> &OS, SampleProfileFormat Format);
 
   virtual void setProfileSymbolList(ProfileSymbolList *PSL) {}
+  virtual void setToCompressAllSections() {}
+  virtual void setUseMD5() {}
 
 protected:
   SampleProfileWriter(std::unique_ptr<raw_ostream> &OS)
@@ -147,12 +149,20 @@ class SampleProfileWriterExtBinaryBase : public SampleProfileWriterBinary {
   virtual std::error_code
   write(const StringMap<FunctionSamples> &ProfileMap) override;
 
-  void setToCompressAllSections();
+  virtual void setToCompressAllSections() override;
   void setToCompressSection(SecType Type);
 
 protected:
   uint64_t markSectionStart(SecType Type);
   std::error_code addNewSection(SecType Sec, uint64_t SectionStart);
+  template <class SecFlagType>
+  void addSectionFlag(SecType Type, SecFlagType Flag) {
+    for (auto &Entry : SectionHdrLayout) {
+      if (Entry.Type == Type)
+        addSecFlag(Entry, Flag);
+    }
+  }
+
   virtual void initSectionHdrLayout() = 0;
   virtual std::error_code
   writeSections(const StringMap<FunctionSamples> &ProfileMap) = 0;
@@ -168,7 +178,6 @@ class SampleProfileWriterExtBinaryBase : public SampleProfileWriterBinary {
   std::error_code writeSecHdrTable();
   virtual std::error_code
   writeHeader(const StringMap<FunctionSamples> &ProfileMap) override;
-  void addSectionFlags(SecType Type, SecFlags Flags);
   SecHdrTableEntry &getEntryInLayout(SecType Type);
   std::error_code compressAndOutput();
 
@@ -202,6 +211,12 @@ class SampleProfileWriterExtBinary : public SampleProfileWriterExtBinaryBase {
     ProfSymList = PSL;
   };
 
+  // Set to use MD5 to represent string in NameTable.
+  virtual void setUseMD5() override {
+    UseMD5 = true;
+    addSectionFlag(SecNameTable, SecNameTableFlags::SecFlagMD5Name);
+  }
+
 private:
   virtual void initSectionHdrLayout() override {
     // Note that SecFuncOffsetTable section is written after SecLBRProfile
@@ -222,6 +237,10 @@ class SampleProfileWriterExtBinary : public SampleProfileWriterExtBinaryBase {
   };
   virtual std::error_code
   writeSections(const StringMap<FunctionSamples> &ProfileMap) override;
+
+  std::error_code writeFuncOffsetTable();
+  virtual std::error_code writeNameTable() override;
+
   ProfileSymbolList *ProfSymList = nullptr;
 
   // Save the start of SecLBRProfile so we can compute the offset to the
@@ -231,7 +250,8 @@ class SampleProfileWriterExtBinary : public SampleProfileWriterExtBinaryBase {
   // FuncOffsetTable maps function name to its profile offset in SecLBRProfile
   // section. It is used to load function profile on demand.
   MapVector<StringRef, uint64_t> FuncOffsetTable;
-  std::error_code writeFuncOffsetTable();
+  // Whether to use MD5 to represent string.
+  bool UseMD5 = false;
 };
 
 // CompactBinary is a compact format of binary profile which both reduces
diff --git a/llvm/include/llvm/Support/AArch64TargetParser.def b/llvm/include/llvm/Support/AArch64TargetParser.def
index 0fd4117d93a17..39b24453b93de 100644
--- a/llvm/include/llvm/Support/AArch64TargetParser.def
+++ b/llvm/include/llvm/Support/AArch64TargetParser.def
@@ -44,6 +44,13 @@ AARCH64_ARCH("armv8.5-a", ARMV8_5A, "8.5-A", "v8.5a",
              (AArch64::AEK_CRC | AArch64::AEK_CRYPTO | AArch64::AEK_FP |
               AArch64::AEK_SIMD | AArch64::AEK_RAS | AArch64::AEK_LSE |
               AArch64::AEK_RDM | AArch64::AEK_RCPC | AArch64::AEK_DOTPROD))
+AARCH64_ARCH("armv8.6-a", ARMV8_6A, "8.6-A", "v8.6a",
+             ARMBuildAttrs::CPUArch::v8_A, FK_CRYPTO_NEON_FP_ARMV8,
+             (AArch64::AEK_CRC  | AArch64::AEK_FP   |
+              AArch64::AEK_SIMD | AArch64::AEK_RAS  | AArch64::AEK_LSE     |
+              AArch64::AEK_RDM  | AArch64::AEK_RCPC | AArch64::AEK_DOTPROD |
+              AArch64::AEK_SM4  | AArch64::AEK_SHA3 | AArch64::AEK_BF16    |
+              AArch64::AEK_SHA2 | AArch64::AEK_AES  | AArch64::AEK_I8MM))
 #undef AARCH64_ARCH
 
 #ifndef AARCH64_ARCH_EXT_NAME
@@ -79,6 +86,8 @@ AARCH64_ARCH_EXT_NAME("memtag",       AArch64::AEK_MTE,         "+mte",   "-mte"
 AARCH64_ARCH_EXT_NAME("ssbs",         AArch64::AEK_SSBS,        "+ssbs",  "-ssbs")
 AARCH64_ARCH_EXT_NAME("sb",           AArch64::AEK_SB,          "+sb",    "-sb")
 AARCH64_ARCH_EXT_NAME("predres",      AArch64::AEK_PREDRES,     "+predres", "-predres")
+AARCH64_ARCH_EXT_NAME("bf16",         AArch64::AEK_BF16,        "+bf16",  "-bf16")
+AARCH64_ARCH_EXT_NAME("i8mm",         AArch64::AEK_I8MM,        "+i8mm",  "-i8mm")
 AARCH64_ARCH_EXT_NAME("tme",          AArch64::AEK_TME,         "+tme",   "-tme")
 #undef AARCH64_ARCH_EXT_NAME
 
diff --git a/llvm/include/llvm/Support/AArch64TargetParser.h b/llvm/include/llvm/Support/AArch64TargetParser.h
index fbe08945a038e..1946392180fc7 100644
--- a/llvm/include/llvm/Support/AArch64TargetParser.h
+++ b/llvm/include/llvm/Support/AArch64TargetParser.h
@@ -55,6 +55,8 @@ enum ArchExtKind : unsigned {
   AEK_SVE2SHA3 =    1 << 26,
   AEK_SVE2BITPERM = 1 << 27,
   AEK_TME =         1 << 28,
+  AEK_BF16 =        1 << 29,
+  AEK_I8MM =        1 << 30,
 };
 
 enum class ArchKind {
diff --git a/llvm/include/llvm/Support/ARMAttributeParser.h b/llvm/include/llvm/Support/ARMAttributeParser.h
index c0f86ecd8b6a9..bf85ea14cfe33 100644
--- a/llvm/include/llvm/Support/ARMAttributeParser.h
+++ b/llvm/include/llvm/Support/ARMAttributeParser.h
@@ -1,4 +1,4 @@
-//===--- ARMAttributeParser.h - ARM Attribute Information Printer ---------===//
+//===- ARMAttributeParser.h - ARM Attribute Information Printer -*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -10,36 +10,24 @@
 #define LLVM_SUPPORT_ARMATTRIBUTEPARSER_H
 
 #include "ARMBuildAttributes.h"
+#include "ELFAttributeParser.h"
 #include "ScopedPrinter.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/Support/DataExtractor.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 
-#include <map>
-
 namespace llvm {
 class StringRef;
 
-class ARMAttributeParser {
-  ScopedPrinter *sw;
-
-  std::map<unsigned, unsigned> attributes;
-  DataExtractor de{ArrayRef<uint8_t>{}, true, 0};
-  DataExtractor::Cursor cursor{0};
-
+class ARMAttributeParser : public ELFAttributeParser {
   struct DisplayHandler {
     ARMBuildAttrs::AttrType attribute;
     Error (ARMAttributeParser::*routine)(ARMBuildAttrs::AttrType);
   };
   static const DisplayHandler displayRoutines[];
 
-  Error parseAttributeList(uint32_t length);
-  void parseIndexList(SmallVectorImpl<uint8_t> &indexList);
-  Error parseSubsection(uint32_t length);
-  Error parseStringAttribute(const char *name, ARMBuildAttrs::AttrType tag,
-                             const ArrayRef<const char *> array);
-  void printAttribute(unsigned tag, unsigned value, StringRef valueDesc);
+  Error handler(uint64_t tag, bool &handled) override;
 
   Error stringAttribute(ARMBuildAttrs::AttrType tag);
 
@@ -82,20 +70,11 @@ class ARMAttributeParser {
   Error nodefaults(ARMBuildAttrs::AttrType tag);
 
 public:
-  ARMAttributeParser(ScopedPrinter *sw) : sw(sw) {}
-  ARMAttributeParser() : sw(nullptr) {}
-  ~ARMAttributeParser() { static_cast<void>(!cursor.takeError()); }
-
-  Error parse(ArrayRef<uint8_t> section, support::endianness endian);
-
-  bool hasAttribute(unsigned tag) const { return attributes.count(tag); }
-
-  unsigned getAttributeValue(unsigned tag) const {
-    return attributes.find(tag)->second;
-  }
+  ARMAttributeParser(ScopedPrinter *sw)
+      : ELFAttributeParser(sw, ARMBuildAttrs::ARMAttributeTags, "aeabi") {}
+  ARMAttributeParser()
+      : ELFAttributeParser(ARMBuildAttrs::ARMAttributeTags, "aeabi") {}
 };
-
 }
 
 #endif
-
diff --git a/llvm/include/llvm/Support/ARMBuildAttributes.h b/llvm/include/llvm/Support/ARMBuildAttributes.h
index 90481eaa1677c..5a06fd6ca7be7 100644
--- a/llvm/include/llvm/Support/ARMBuildAttributes.h
+++ b/llvm/include/llvm/Support/ARMBuildAttributes.h
@@ -18,77 +18,70 @@
 #ifndef LLVM_SUPPORT_ARMBUILDATTRIBUTES_H
 #define LLVM_SUPPORT_ARMBUILDATTRIBUTES_H
 
-namespace llvm {
-class StringRef;
+#include "llvm/Support/ELFAttributes.h"
 
+namespace llvm {
 namespace ARMBuildAttrs {
 
+extern const TagNameMap ARMAttributeTags;
+
 enum SpecialAttr {
   // This is for the .cpu asm attr. It translates into one or more
   // AttrType (below) entries in the .ARM.attributes section in the ELF.
   SEL_CPU
 };
 
-enum AttrType {
+enum AttrType : unsigned {
   // Rest correspond to ELF/.ARM.attributes
-  File                      = 1,
-  CPU_raw_name              = 4,
-  CPU_name                  = 5,
-  CPU_arch                  = 6,
-  CPU_arch_profile          = 7,
-  ARM_ISA_use               = 8,
-  THUMB_ISA_use             = 9,
-  FP_arch                   = 10,
-  WMMX_arch                 = 11,
-  Advanced_SIMD_arch        = 12,
-  PCS_config                = 13,
-  ABI_PCS_R9_use            = 14,
-  ABI_PCS_RW_data           = 15,
-  ABI_PCS_RO_data           = 16,
-  ABI_PCS_GOT_use           = 17,
-  ABI_PCS_wchar_t           = 18,
-  ABI_FP_rounding           = 19,
-  ABI_FP_denormal           = 20,
-  ABI_FP_exceptions         = 21,
-  ABI_FP_user_exceptions    = 22,
-  ABI_FP_number_model       = 23,
-  ABI_align_needed          = 24,
-  ABI_align_preserved       = 25,
-  ABI_enum_size             = 26,
-  ABI_HardFP_use            = 27,
-  ABI_VFP_args              = 28,
-  ABI_WMMX_args             = 29,
-  ABI_optimization_goals    = 30,
+  File = 1,
+  CPU_raw_name = 4,
+  CPU_name = 5,
+  CPU_arch = 6,
+  CPU_arch_profile = 7,
+  ARM_ISA_use = 8,
+  THUMB_ISA_use = 9,
+  FP_arch = 10,
+  WMMX_arch = 11,
+  Advanced_SIMD_arch = 12,
+  PCS_config = 13,
+  ABI_PCS_R9_use = 14,
+  ABI_PCS_RW_data = 15,
+  ABI_PCS_RO_data = 16,
+  ABI_PCS_GOT_use = 17,
+  ABI_PCS_wchar_t = 18,
+  ABI_FP_rounding = 19,
+  ABI_FP_denormal = 20,
+  ABI_FP_exceptions = 21,
+  ABI_FP_user_exceptions = 22,
+  ABI_FP_number_model = 23,
+  ABI_align_needed = 24,
+  ABI_align_preserved = 25,
+  ABI_enum_size = 26,
+  ABI_HardFP_use = 27,
+  ABI_VFP_args = 28,
+  ABI_WMMX_args = 29,
+  ABI_optimization_goals = 30,
   ABI_FP_optimization_goals = 31,
-  compatibility             = 32,
-  CPU_unaligned_access      = 34,
-  FP_HP_extension           = 36,
-  ABI_FP_16bit_format       = 38,
-  MPextension_use           = 42, // recoded from 70 (ABI r2.08)
-  DIV_use                   = 44,
-  DSP_extension             = 46,
-  MVE_arch                  = 48,
-  also_compatible_with      = 65,
-  conformance               = 67,
-  Virtualization_use        = 68,
+  compatibility = 32,
+  CPU_unaligned_access = 34,
+  FP_HP_extension = 36,
+  ABI_FP_16bit_format = 38,
+  MPextension_use = 42, // recoded from 70 (ABI r2.08)
+  DIV_use = 44,
+  DSP_extension = 46,
+  MVE_arch = 48,
+  also_compatible_with = 65,
+  conformance = 67,
+  Virtualization_use = 68,
 
   /// Legacy Tags
-  Section                   = 2,  // deprecated (ABI r2.09)
-  Symbol                    = 3,  // deprecated (ABI r2.09)
-  ABI_align8_needed         = 24, // renamed to ABI_align_needed (ABI r2.09)
-  ABI_align8_preserved      = 25, // renamed to ABI_align_preserved (ABI r2.09)
-  nodefaults                = 64, // deprecated (ABI r2.09)
-  T2EE_use                  = 66, // deprecated (ABI r2.09)
-  MPextension_use_old       = 70  // recoded to MPextension_use (ABI r2.08)
-};
-
-StringRef AttrTypeAsString(unsigned Attr, bool HasTagPrefix = true);
-StringRef AttrTypeAsString(AttrType Attr, bool HasTagPrefix = true);
-int AttrTypeFromString(StringRef Tag);
-
-// Magic numbers for .ARM.attributes
-enum AttrMagic {
-  Format_Version  = 0x41
+  Section = 2,               // deprecated (ABI r2.09)
+  Symbol = 3,                // deprecated (ABI r2.09)
+  ABI_align8_needed = 24,    // renamed to ABI_align_needed (ABI r2.09)
+  ABI_align8_preserved = 25, // renamed to ABI_align_preserved (ABI r2.09)
+  nodefaults = 64,           // deprecated (ABI r2.09)
+  T2EE_use = 66,             // deprecated (ABI r2.09)
+  MPextension_use_old = 70   // recoded to MPextension_use (ABI r2.08)
 };
 
 // Legal Values for CPU_arch, (=6), uleb128
diff --git a/llvm/include/llvm/Support/ARMTargetParser.def b/llvm/include/llvm/Support/ARMTargetParser.def
index af0c58982ea70..87bd4997914d2 100644
--- a/llvm/include/llvm/Support/ARMTargetParser.def
+++ b/llvm/include/llvm/Support/ARMTargetParser.def
@@ -112,6 +112,11 @@ ARM_ARCH("armv8.5-a", ARMV8_5A, "8.5-A", "v8.5a",
          (ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM |
           ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP | ARM::AEK_CRC | ARM::AEK_RAS |
           ARM::AEK_DOTPROD))
+ARM_ARCH("armv8.6-a", ARMV8_6A, "8.6-A", "v8.6a",
+         ARMBuildAttrs::CPUArch::v8_A, FK_CRYPTO_NEON_FP_ARMV8,
+         (ARM::AEK_SEC        | ARM::AEK_MP   | ARM::AEK_VIRT | ARM::AEK_HWDIVARM |
+          ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP  | ARM::AEK_CRC  | ARM::AEK_RAS |
+          ARM::AEK_DOTPROD    | ARM::AEK_BF16 | ARM::AEK_SHA2 | ARM::AEK_AES))
 ARM_ARCH("armv8-r", ARMV8R, "8-R", "v8r", ARMBuildAttrs::CPUArch::v8_R,
           FK_NEON_FP_ARMV8,
           (ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB |
@@ -164,6 +169,7 @@ ARM_ARCH_EXT_NAME("iwmmxt2",  ARM::AEK_IWMMXT2,  nullptr,  nullptr)
 ARM_ARCH_EXT_NAME("maverick", ARM::AEK_MAVERICK, nullptr,  nullptr)
 ARM_ARCH_EXT_NAME("xscale",   ARM::AEK_XSCALE,   nullptr,  nullptr)
 ARM_ARCH_EXT_NAME("fp16fml",  ARM::AEK_FP16FML,  "+fp16fml", "-fp16fml")
+ARM_ARCH_EXT_NAME("bf16",     ARM::AEK_BF16,     "+bf16",    "-bf16")
 ARM_ARCH_EXT_NAME("sb",       ARM::AEK_SB,       "+sb",      "-sb")
 ARM_ARCH_EXT_NAME("lob",      ARM::AEK_LOB,      "+lob",   "-lob")
 ARM_ARCH_EXT_NAME("cdecp0",   ARM::AEK_CDECP0,   "+cdecp0",  "-cdecp0")
diff --git a/llvm/include/llvm/Support/ARMTargetParser.h b/llvm/include/llvm/Support/ARMTargetParser.h
index 6f7a11f2eaba9..0de333031e6d1 100644
--- a/llvm/include/llvm/Support/ARMTargetParser.h
+++ b/llvm/include/llvm/Support/ARMTargetParser.h
@@ -46,14 +46,15 @@ enum ArchExtKind : uint64_t {
   AEK_SB      =     1 << 17,
   AEK_FP_DP   =     1 << 18,
   AEK_LOB     =     1 << 19,
-  AEK_CDECP0 =      1 << 20,
-  AEK_CDECP1 =      1 << 21,
-  AEK_CDECP2 =      1 << 22,
-  AEK_CDECP3 =      1 << 23,
-  AEK_CDECP4 =      1 << 24,
-  AEK_CDECP5 =      1 << 25,
-  AEK_CDECP6 =      1 << 26,
-  AEK_CDECP7 =      1 << 27,
+  AEK_BF16    =     1 << 20,
+  AEK_CDECP0 =      1 << 21,
+  AEK_CDECP1 =      1 << 22,
+  AEK_CDECP2 =      1 << 23,
+  AEK_CDECP3 =      1 << 24,
+  AEK_CDECP4 =      1 << 25,
+  AEK_CDECP5 =      1 << 26,
+  AEK_CDECP6 =      1 << 27,
+  AEK_CDECP7 =      1 << 28,
 
   // Unsupported extensions.
   AEK_OS       =    1ULL << 59,
diff --git a/llvm/include/llvm/Support/CFGUpdate.h b/llvm/include/llvm/Support/CFGUpdate.h
index eeaf5d0a21acd..af4cd6ed1f1df 100644
--- a/llvm/include/llvm/Support/CFGUpdate.h
+++ b/llvm/include/llvm/Support/CFGUpdate.h
@@ -62,7 +62,7 @@ template <typename NodePtr> class Update {
 template <typename NodePtr>
 void LegalizeUpdates(ArrayRef<Update<NodePtr>> AllUpdates,
                      SmallVectorImpl<Update<NodePtr>> &Result,
-                     bool InverseGraph) {
+                     bool InverseGraph, bool ReverseResultOrder = false) {
   // Count the total number of inserions of each edge.
   // Each insertion adds 1 and deletion subtracts 1. The end number should be
   // one of {-1 (deletion), 0 (NOP), +1 (insertion)}. Otherwise, the sequence
@@ -104,11 +104,11 @@ void LegalizeUpdates(ArrayRef<Update<NodePtr>> AllUpdates,
       Operations[{U.getTo(), U.getFrom()}] = int(i);
   }
 
-  llvm::sort(Result,
-             [&Operations](const Update<NodePtr> &A, const Update<NodePtr> &B) {
-               return Operations[{A.getFrom(), A.getTo()}] >
-                      Operations[{B.getFrom(), B.getTo()}];
-             });
+  llvm::sort(Result, [&](const Update<NodePtr> &A, const Update<NodePtr> &B) {
+    const auto &OpA = Operations[{A.getFrom(), A.getTo()}];
+    const auto &OpB = Operations[{B.getFrom(), B.getTo()}];
+    return ReverseResultOrder ? OpA < OpB : OpA > OpB;
+  });
 }
 
 } // end namespace cfg
diff --git a/llvm/include/llvm/Support/ELFAttributeParser.h b/llvm/include/llvm/Support/ELFAttributeParser.h
new file mode 100644
index 0000000000000..8bf87b2d84f05
--- /dev/null
+++ b/llvm/include/llvm/Support/ELFAttributeParser.h
@@ -0,0 +1,72 @@
+//===- ELF AttributeParser.h - ELF Attribute Parser -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_ELFATTRIBUTEPARSER_H
+#define LLVM_SUPPORT_ELFATTRIBUTEPARSER_H
+
+#include "ELFAttributes.h"
+#include "ScopedPrinter.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/Support/DataExtractor.h"
+#include "llvm/Support/Error.h"
+
+#include <unordered_map>
+
+namespace llvm {
+class StringRef;
+
+class ELFAttributeParser {
+  StringRef vendor;
+  std::unordered_map<unsigned, unsigned> attributes;
+  std::unordered_map<unsigned, StringRef> attributesStr;
+
+  virtual Error handler(uint64_t tag, bool &handled) = 0;
+
+protected:
+  ScopedPrinter *sw;
+  TagNameMap tagToStringMap;
+  DataExtractor de{ArrayRef<uint8_t>{}, true, 0};
+  DataExtractor::Cursor cursor{0};
+
+  void printAttribute(unsigned tag, unsigned value, StringRef valueDesc);
+
+  Error parseStringAttribute(const char *name, unsigned tag,
+                             ArrayRef<const char *> strings);
+  Error parseAttributeList(uint32_t length);
+  void parseIndexList(SmallVectorImpl<uint8_t> &indexList);
+  Error parseSubsection(uint32_t length);
+
+public:
+  virtual ~ELFAttributeParser() { static_cast<void>(!cursor.takeError()); }
+  Error integerAttribute(unsigned tag);
+  Error stringAttribute(unsigned tag);
+
+  ELFAttributeParser(ScopedPrinter *sw, TagNameMap tagNameMap, StringRef vendor)
+      : vendor(vendor), sw(sw), tagToStringMap(tagNameMap) {}
+
+  ELFAttributeParser(TagNameMap tagNameMap, StringRef vendor)
+      : vendor(vendor), sw(nullptr), tagToStringMap(tagNameMap) {}
+
+  Error parse(ArrayRef<uint8_t> section, support::endianness endian);
+
+  Optional<unsigned> getAttributeValue(unsigned tag) const {
+    auto I = attributes.find(tag);
+    if (I == attributes.end())
+      return None;
+    return I->second;
+  }
+  Optional<StringRef> getAttributeString(unsigned tag) const {
+    auto I = attributesStr.find(tag);
+    if (I == attributesStr.end())
+      return None;
+    return I->second;
+  }
+};
+
+} // namespace llvm
+#endif
diff --git a/llvm/include/llvm/Support/ELFAttributes.h b/llvm/include/llvm/Support/ELFAttributes.h
new file mode 100644
index 0000000000000..c8a7ae142b9af
--- /dev/null
+++ b/llvm/include/llvm/Support/ELFAttributes.h
@@ -0,0 +1,37 @@
+//===-- ELFAttributes.h - ELF Attributes ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_ELFATTRIBUTES_H
+#define LLVM_SUPPORT_ELFATTRIBUTES_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringRef.h"
+
+namespace llvm {
+
+struct TagNameItem {
+  unsigned attr;
+  StringRef tagName;
+};
+
+using TagNameMap = ArrayRef<TagNameItem>;
+
+namespace ELFAttrs {
+
+enum AttrType : unsigned { File = 1, Section = 2, Symbol = 3 };
+
+StringRef attrTypeAsString(unsigned attr, TagNameMap tagNameMap,
+                           bool hasTagPrefix = true);
+Optional<unsigned> attrTypeFromString(StringRef tag, TagNameMap tagNameMap);
+
+// Magic numbers for ELF attributes.
+enum AttrMagic { Format_Version = 0x41 };
+
+} // namespace ELFAttrs
+} // namespace llvm
+#endif
diff --git a/llvm/include/llvm/Support/FileCollector.h b/llvm/include/llvm/Support/FileCollector.h
index 079fe3efab9d3..1b6383ef8cc2d 100644
--- a/llvm/include/llvm/Support/FileCollector.h
+++ b/llvm/include/llvm/Support/FileCollector.h
@@ -18,7 +18,7 @@
 #include <mutex>
 
 namespace llvm {
-
+class FileCollectorFileSystem;
 /// Collects files into a directory and generates a mapping that can be used by
 /// the VFS.
 class FileCollector {
@@ -26,9 +26,10 @@ class FileCollector {
   FileCollector(std::string Root, std::string OverlayRoot);
 
   void addFile(const Twine &file);
+  void addDirectory(const Twine &Dir);
 
   /// Write the yaml mapping (for the VFS) to the given file.
-  std::error_code writeMapping(StringRef mapping_file);
+  std::error_code writeMapping(StringRef MappingFile);
 
   /// Copy the files into the root directory.
   ///
@@ -44,7 +45,7 @@ class FileCollector {
                      std::shared_ptr<FileCollector> Collector);
 
 private:
-  void addFileImpl(StringRef SrcPath);
+  friend FileCollectorFileSystem;
 
   bool markAsSeen(StringRef Path) {
     if (Path.empty())
@@ -55,10 +56,19 @@ class FileCollector {
   bool getRealPath(StringRef SrcPath, SmallVectorImpl<char> &Result);
 
   void addFileToMapping(StringRef VirtualPath, StringRef RealPath) {
-    VFSWriter.addFileMapping(VirtualPath, RealPath);
+    if (sys::fs::is_directory(VirtualPath))
+      VFSWriter.addDirectoryMapping(VirtualPath, RealPath);
+    else
+      VFSWriter.addFileMapping(VirtualPath, RealPath);
   }
 
 protected:
+  void addFileImpl(StringRef SrcPath);
+
+  llvm::vfs::directory_iterator
+  addDirectoryImpl(const llvm::Twine &Dir,
+                   IntrusiveRefCntPtr<vfs::FileSystem> FS, std::error_code &EC);
+
   /// Synchronizes adding files.
   std::mutex Mutex;
 
diff --git a/llvm/include/llvm/Support/GenericDomTreeConstruction.h b/llvm/include/llvm/Support/GenericDomTreeConstruction.h
index 571550012374c..a344f66c548e2 100644
--- a/llvm/include/llvm/Support/GenericDomTreeConstruction.h
+++ b/llvm/include/llvm/Support/GenericDomTreeConstruction.h
@@ -599,8 +599,6 @@ struct SemiNCAInfo {
     // Loop over all of the discovered blocks in the function...
     for (size_t i = 1, e = NumToNode.size(); i != e; ++i) {
       NodePtr W = NumToNode[i];
-      LLVM_DEBUG(dbgs() << "\tdiscovered a new reachable node "
-                        << BlockNamePrinter(W) << "\n");
 
       // Don't replace this with 'count', the insertion side effect is important
       if (DT.DomTreeNodes[W]) continue;  // Haven't calculated this node yet?
diff --git a/llvm/include/llvm/Support/MathExtras.h b/llvm/include/llvm/Support/MathExtras.h
index 78afb3bc11096..16da3046c8ced 100644
--- a/llvm/include/llvm/Support/MathExtras.h
+++ b/llvm/include/llvm/Support/MathExtras.h
@@ -313,6 +313,34 @@ T reverseBits(T Val) {
   return Val;
 }
 
+#if __has_builtin(__builtin_bitreverse8)
+template<>
+inline uint8_t reverseBits<uint8_t>(uint8_t Val) {
+  return __builtin_bitreverse8(Val);
+}
+#endif
+
+#if __has_builtin(__builtin_bitreverse16)
+template<>
+inline uint16_t reverseBits<uint16_t>(uint16_t Val) {
+  return __builtin_bitreverse16(Val);
+}
+#endif
+
+#if __has_builtin(__builtin_bitreverse32)
+template<>
+inline uint32_t reverseBits<uint32_t>(uint32_t Val) {
+  return __builtin_bitreverse32(Val);
+}
+#endif
+
+#if __has_builtin(__builtin_bitreverse64)
+template<>
+inline uint64_t reverseBits<uint64_t>(uint64_t Val) {
+  return __builtin_bitreverse64(Val);
+}
+#endif
+
 // NOTE: The following support functions use the _32/_64 extensions instead of
 // type overloading so that signed and unsigned integers can be used without
 // ambiguity.
diff --git a/llvm/include/llvm/Support/Parallel.h b/llvm/include/llvm/Support/Parallel.h
index 3c0ed2c111275..e9d04bfb279f6 100644
--- a/llvm/include/llvm/Support/Parallel.h
+++ b/llvm/include/llvm/Support/Parallel.h
@@ -12,6 +12,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/Support/MathExtras.h"
+#include "llvm/Support/Threading.h"
 
 #include <algorithm>
 #include <condition_variable>
@@ -33,6 +34,11 @@ struct is_execution_policy
 constexpr sequential_execution_policy seq{};
 constexpr parallel_execution_policy par{};
 
+// Strategy for the default executor used by the parallel routines provided by
+// this file. It defaults to using all hardware threads and should be
+// initialized before the first use of parallel routines.
+extern ThreadPoolStrategy strategy;
+
 namespace detail {
 
 #if LLVM_ENABLE_THREADS
diff --git a/llvm/include/llvm/Support/Path.h b/llvm/include/llvm/Support/Path.h
index f0b2810cd7a9b..728b63c54c050 100644
--- a/llvm/include/llvm/Support/Path.h
+++ b/llvm/include/llvm/Support/Path.h
@@ -153,32 +153,23 @@ void replace_extension(SmallVectorImpl<char> &path, const Twine &extension,
 /// @code
 ///   /foo, /old, /new => /foo
 ///   /old, /old, /new => /new
-///   /old, /old/, /new, false => /old
-///   /old, /old/, /new, true => /new
+///   /old, /old/, /new => /old
 ///   /old/foo, /old, /new => /new/foo
 ///   /old/foo, /old/, /new => /new/foo
 ///   /old/foo, /old/, /new/ => /new/foo
 ///   /oldfoo, /old, /new => /oldfoo
 ///   /foo, <empty>, /new => /new/foo
 ///   /foo, <empty>, new => new/foo
-///   /old/foo, /old, <empty>, false => /foo
-///   /old/foo, /old, <empty>, true => foo
+///   /old/foo, /old, <empty> => /foo
 /// @endcode
 ///
 /// @param Path If \a Path starts with \a OldPrefix modify to instead
 ///        start with \a NewPrefix.
-/// @param OldPrefix The path prefix to strip from \a Path. Any trailing
-///        path separator is ignored if strict is true.
+/// @param OldPrefix The path prefix to strip from \a Path.
 /// @param NewPrefix The path prefix to replace \a NewPrefix with.
-/// @param style The path separator style
-/// @param strict If strict is true, a directory separator following
-///        \a OldPrefix will also be stripped. Otherwise, directory
-///        separators will only be matched and stripped when present
-///        in \a OldPrefix.
 /// @result true if \a Path begins with OldPrefix
-bool replace_path_prefix(SmallVectorImpl<char> &Path,
-                         const StringRef &OldPrefix, const StringRef &NewPrefix,
-                         Style style = Style::native, bool strict = false);
+bool replace_path_prefix(SmallVectorImpl<char> &Path, StringRef OldPrefix,
+                         StringRef NewPrefix);
 
 /// Append to path.
 ///
diff --git a/llvm/include/llvm/Support/PrettyStackTrace.h b/llvm/include/llvm/Support/PrettyStackTrace.h
index 6eb070b2297ec..32975f3c5a353 100644
--- a/llvm/include/llvm/Support/PrettyStackTrace.h
+++ b/llvm/include/llvm/Support/PrettyStackTrace.h
@@ -37,6 +37,10 @@ namespace llvm {
   /// \see PrettyStackTraceEntry
   void EnablePrettyStackTraceOnSigInfoForThisThread(bool ShouldEnable = true);
 
+  /// Replaces the generic bug report message that is output upon
+  /// a crash.
+  void setBugReportMsg(const char *Msg);
+
   /// PrettyStackTraceEntry - This class is used to represent a frame of the
   /// "pretty" stack trace that is dumped when a program crashes. You can define
   /// subclasses of this and declare them on the program stack: when they are
diff --git a/llvm/include/llvm/Support/RISCVAttributeParser.h b/llvm/include/llvm/Support/RISCVAttributeParser.h
new file mode 100644
index 0000000000000..89f660edcc777
--- /dev/null
+++ b/llvm/include/llvm/Support/RISCVAttributeParser.h
@@ -0,0 +1,38 @@
+//===-- RISCVAttributeParser.h - RISCV Attribute Parser ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_RISCVATTRIBUTEPARSER_H
+#define LLVM_SUPPORT_RISCVATTRIBUTEPARSER_H
+
+#include "ScopedPrinter.h"
+#include "llvm/Support/ELFAttributeParser.h"
+#include "llvm/Support/RISCVAttributes.h"
+
+namespace llvm {
+class RISCVAttributeParser : public ELFAttributeParser {
+  struct DisplayHandler {
+    RISCVAttrs::AttrType attribute;
+    Error (RISCVAttributeParser::*routine)(unsigned);
+  };
+  static const DisplayHandler displayRoutines[];
+
+  Error handler(uint64_t tag, bool &handled) override;
+
+  Error unalignedAccess(unsigned tag);
+  Error stackAlign(unsigned tag);
+
+public:
+  RISCVAttributeParser(ScopedPrinter *sw)
+      : ELFAttributeParser(sw, RISCVAttrs::RISCVAttributeTags, "riscv") {}
+  RISCVAttributeParser()
+      : ELFAttributeParser(RISCVAttrs::RISCVAttributeTags, "riscv") {}
+};
+
+} // namespace llvm
+
+#endif
diff --git a/llvm/include/llvm/Support/RISCVAttributes.h b/llvm/include/llvm/Support/RISCVAttributes.h
new file mode 100644
index 0000000000000..caded9519b668
--- /dev/null
+++ b/llvm/include/llvm/Support/RISCVAttributes.h
@@ -0,0 +1,44 @@
+//===-- RISCVAttributes.h - RISCV Attributes --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains enumerations for RISCV attributes as defined in RISC-V
+// ELF psABI specification.
+//
+// RISC-V ELF psABI specification
+//
+// https://github.com/riscv/riscv-elf-psabi-doc/blob/master/riscv-elf.md
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_SUPPORT_RISCVATTRIBUTES_H
+#define LLVM_SUPPORT_RISCVATTRIBUTES_H
+
+#include "llvm/Support/ELFAttributes.h"
+
+namespace llvm {
+namespace RISCVAttrs {
+
+extern const TagNameMap RISCVAttributeTags;
+
+enum AttrType : unsigned {
+  // Attribute types in ELF/.riscv.attributes.
+  STACK_ALIGN = 4,
+  ARCH = 5,
+  UNALIGNED_ACCESS = 6,
+  PRIV_SPEC = 8,
+  PRIV_SPEC_MINOR = 10,
+  PRIV_SPEC_REVISION = 12,
+};
+
+enum StackAlign { ALIGN_4 = 4, ALIGN_16 = 16 };
+
+enum { NOT_ALLOWED = 0, ALLOWED = 1 };
+
+} // namespace RISCVAttrs
+} // namespace llvm
+
+#endif
diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def
index 12bd1a77b18da..6d5bef112cdbb 100644
--- a/llvm/include/llvm/Support/TargetOpcodes.def
+++ b/llvm/include/llvm/Support/TargetOpcodes.def
@@ -385,6 +385,12 @@ HANDLE_TARGET_OPCODE(G_LSHR)
 // Generic arithmetic right-shift
 HANDLE_TARGET_OPCODE(G_ASHR)
 
+// Generic funnel left shift
+HANDLE_TARGET_OPCODE(G_FSHL)
+
+// Generic funnel right shift
+HANDLE_TARGET_OPCODE(G_FSHR)
+
 /// Generic integer-base comparison, also applicable to vectors of integers.
 HANDLE_TARGET_OPCODE(G_ICMP)
 
diff --git a/llvm/include/llvm/Support/Threading.h b/llvm/include/llvm/Support/Threading.h
index 24a0789e09649..f133c790ad963 100644
--- a/llvm/include/llvm/Support/Threading.h
+++ b/llvm/include/llvm/Support/Threading.h
@@ -166,8 +166,20 @@ void llvm_execute_on_thread_async(
     /// sockets. \p ThreadPoolNum represents a number bounded by [0,
     /// compute_thread_count()).
     void apply_thread_strategy(unsigned ThreadPoolNum) const;
+
+    /// Finds the CPU socket where a thread should go. Returns 'None' if the
+    /// thread shall remain on the actual CPU socket.
+    Optional<unsigned> compute_cpu_socket(unsigned ThreadPoolNum) const;
   };
 
+  /// Build a strategy from a number of threads as a string provided in \p Num.
+  /// When Num is above the max number of threads specified by the \p Default
+  /// strategy, we attempt to equally allocate the threads on all CPU sockets.
+  /// "0" or an empty string will return the \p Default strategy.
+  /// "all" for using all hardware threads.
+  Optional<ThreadPoolStrategy>
+  get_threadpool_strategy(StringRef Num, ThreadPoolStrategy Default = {});
+
   /// Returns a thread strategy for tasks requiring significant memory or other
   /// resources. To be used for workloads where hardware_concurrency() proves to
   /// be less efficient. Avoid this strategy if doing lots of I/O. Currently
@@ -182,6 +194,18 @@ void llvm_execute_on_thread_async(
     return S;
   }
 
+  /// Like heavyweight_hardware_concurrency() above, but builds a strategy
+  /// based on the rules described for get_threadpool_strategy().
+  /// If \p Num is invalid, returns a default strategy where one thread per
+  /// hardware core is used.
+  inline ThreadPoolStrategy heavyweight_hardware_concurrency(StringRef Num) {
+    Optional<ThreadPoolStrategy> S =
+        get_threadpool_strategy(Num, heavyweight_hardware_concurrency());
+    if (S)
+      return *S;
+    return heavyweight_hardware_concurrency();
+  }
+
   /// Returns a default thread strategy where all available hardware ressources
   /// are to be used, except for those initially excluded by an affinity mask.
   /// This function takes affinity into consideration. Returns 1 when LLVM is
diff --git a/llvm/include/llvm/Support/TypeSize.h b/llvm/include/llvm/Support/TypeSize.h
index f66c043b29fe7..35db42336a7a1 100644
--- a/llvm/include/llvm/Support/TypeSize.h
+++ b/llvm/include/llvm/Support/TypeSize.h
@@ -149,6 +149,9 @@ class TypeSize {
   // Returns true if the type size is non-zero.
   bool isNonZero() const { return MinSize != 0; }
 
+  // Returns true if the type size is zero.
+  bool isZero() const { return MinSize == 0; }
+
   // Casts to a uint64_t if this is a fixed-width size.
   //
   // This interface is deprecated and will be removed in a future version
diff --git a/llvm/include/llvm/Support/VirtualFileSystem.h b/llvm/include/llvm/Support/VirtualFileSystem.h
index 9b8d0574f008b..f13140cd34481 100644
--- a/llvm/include/llvm/Support/VirtualFileSystem.h
+++ b/llvm/include/llvm/Support/VirtualFileSystem.h
@@ -506,10 +506,12 @@ getVFSFromYAML(std::unique_ptr<llvm::MemoryBuffer> Buffer,
 
 struct YAMLVFSEntry {
   template <typename T1, typename T2>
-  YAMLVFSEntry(T1 &&VPath, T2 &&RPath)
-      : VPath(std::forward<T1>(VPath)), RPath(std::forward<T2>(RPath)) {}
+  YAMLVFSEntry(T1 &&VPath, T2 &&RPath, bool IsDirectory = false)
+      : VPath(std::forward<T1>(VPath)), RPath(std::forward<T2>(RPath)),
+        IsDirectory(IsDirectory) {}
   std::string VPath;
   std::string RPath;
+  bool IsDirectory = false;
 };
 
 class VFSFromYamlDirIterImpl;
@@ -771,10 +773,13 @@ class YAMLVFSWriter {
   Optional<bool> UseExternalNames;
   std::string OverlayDir;
 
+  void addEntry(StringRef VirtualPath, StringRef RealPath, bool IsDirectory);
+
 public:
   YAMLVFSWriter() = default;
 
   void addFileMapping(StringRef VirtualPath, StringRef RealPath);
+  void addDirectoryMapping(StringRef VirtualPath, StringRef RealPath);
 
   void setCaseSensitivity(bool CaseSensitive) {
     IsCaseSensitive = CaseSensitive;
diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td
index 901c3d17457d6..d314d7d2f862f 100644
--- a/llvm/include/llvm/Target/GenericOpcodes.td
+++ b/llvm/include/llvm/Target/GenericOpcodes.td
@@ -308,6 +308,22 @@ def G_ASHR : GenericInstruction {
   let hasSideEffects = 0;
 }
 
+/// Funnel 'double' shifts take 3 operands, 2 inputs and the shift amount.
+/// fshl(X,Y,Z): (X << (Z % bitwidth)) | (Y >> (bitwidth - (Z % bitwidth)))
+def G_FSHL : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src1, type0:$src2, type1:$src3);
+  let hasSideEffects = 0;
+}
+
+/// Funnel 'double' shifts take 3 operands, 2 inputs and the shift amount.
+/// fshr(X,Y,Z): (X << (bitwidth - (Z % bitwidth))) | (Y >> (Z % bitwidth))
+def G_FSHR : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src1, type0:$src2, type1:$src3);
+  let hasSideEffects = 0;
+}
+
 // Generic integer comparison.
 def G_ICMP : GenericInstruction {
   let OutOperandList = (outs type0:$dst);
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index b0f189a3cd085..e9fe821a6f22a 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -192,6 +192,22 @@ def select_same_val: GICombineRule<
   (apply [{ return Helper.replaceSingleDefInstWithOperand(*${root}, 2); }])
 >;
 
+// Fold x op 0 -> x
+def right_identity_zero: GICombineRule<
+  (defs root:$root),
+  (match (wip_match_opcode G_SUB, G_ADD, G_OR, G_XOR, G_SHL, G_ASHR, G_LSHR):$root,
+    [{ return Helper.matchConstantOp(${root}->getOperand(2), 0); }]),
+  (apply [{ return Helper.replaceSingleDefInstWithOperand(*${root}, 1); }])
+>;
+
+// Fold (x op x) - > x
+def binop_same_val: GICombineRule<
+  (defs root:$root),
+  (match (wip_match_opcode G_AND, G_OR):$root,
+    [{ return Helper.matchBinOpSameVal(*${root}); }]),
+  (apply [{ return Helper.replaceSingleDefInstWithOperand(*${root}, 1); }])
+>;
+
 // FIXME: These should use the custom predicate feature once it lands.
 def undef_combines : GICombineGroup<[undef_to_fp_zero, undef_to_int_zero,
                                      undef_to_negative_one,
@@ -199,7 +215,8 @@ def undef_combines : GICombineGroup<[undef_to_fp_zero, undef_to_int_zero,
                                      propagate_undef_all_ops,
                                      propagate_undef_shuffle_mask]>;
 
-def identity_combines : GICombineGroup<[select_same_val]>;
+def identity_combines : GICombineGroup<[select_same_val, right_identity_zero,
+                                        binop_same_val]>;
 
 def trivial_combines : GICombineGroup<[copy_prop, mul_to_shl]>;
 def all_combines : GICombineGroup<[trivial_combines, ptr_add_immed_chain,
diff --git a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
index 2129588d4aedc..dec8797b7d33c 100644
--- a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
+++ b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
@@ -93,6 +93,8 @@ def : GINodeEquiv<G_INTRINSIC_W_SIDE_EFFECTS, intrinsic_w_chain>;
 def : GINodeEquiv<G_BR, br>;
 def : GINodeEquiv<G_BSWAP, bswap>;
 def : GINodeEquiv<G_BITREVERSE, bitreverse>;
+def : GINodeEquiv<G_FSHL, fshl>;
+def : GINodeEquiv<G_FSHR, fshr>;
 def : GINodeEquiv<G_CTLZ, ctlz>;
 def : GINodeEquiv<G_CTTZ, cttz>;
 def : GINodeEquiv<G_CTLZ_ZERO_UNDEF, ctlz_zero_undef>;
diff --git a/llvm/include/llvm/Target/TargetLoweringObjectFile.h b/llvm/include/llvm/Target/TargetLoweringObjectFile.h
index 6d571c0ba1e25..4c5b1cd1e458b 100644
--- a/llvm/include/llvm/Target/TargetLoweringObjectFile.h
+++ b/llvm/include/llvm/Target/TargetLoweringObjectFile.h
@@ -221,7 +221,9 @@ class TargetLoweringObjectFile : public MCObjectFileInfo {
 
   /// On targets that use separate function descriptor symbols, return a section
   /// for the descriptor given its symbol. Use only with defined functions.
-  virtual MCSection *getSectionForFunctionDescriptor(const MCSymbol *S) const {
+  virtual MCSection *
+  getSectionForFunctionDescriptor(const Function *F,
+                                  const TargetMachine &TM) const {
     return nullptr;
   }
 
diff --git a/llvm/include/llvm/Target/TargetOptions.h b/llvm/include/llvm/Target/TargetOptions.h
index 7282040a352fa..56ead308da076 100644
--- a/llvm/include/llvm/Target/TargetOptions.h
+++ b/llvm/include/llvm/Target/TargetOptions.h
@@ -14,11 +14,13 @@
 #ifndef LLVM_TARGET_TARGETOPTIONS_H
 #define LLVM_TARGET_TARGETOPTIONS_H
 
+#include "llvm/ADT/FloatingPointMode.h"
 #include "llvm/MC/MCTargetOptions.h"
 
 #include <memory>
 
 namespace llvm {
+  struct fltSemantics;
   class MachineFunction;
   class MemoryBuffer;
   class Module;
@@ -57,15 +59,6 @@ namespace llvm {
     };
   }
 
-  namespace FPDenormal {
-    enum DenormalMode {
-      IEEE,           // IEEE 754 denormal numbers
-      PreserveSign,   // the sign of a flushed-to-zero number is preserved in
-                      // the sign of 0
-      PositiveZero    // denormals are flushed to positive zero
-    };
-  }
-
   enum class BasicBlockSection {
     All,    // Use Basic Block Sections for all basic blocks.  A section
             // for every basic block can significantly bloat object file sizes.
@@ -135,7 +128,8 @@ namespace llvm {
           EmitStackSizeSection(false), EnableMachineOutliner(false),
           SupportsDefaultOutlining(false), EmitAddrsig(false),
           EmitCallSiteInfo(false), SupportsDebugEntryValues(false),
-          EnableDebugEntryValues(false), ForceDwarfFrameSection(false) {}
+          EnableDebugEntryValues(false), ForceDwarfFrameSection(false),
+          FPDenormalMode(DenormalMode::IEEE, DenormalMode::IEEE) {}
 
     /// PrintMachineCode - This flag is enabled when the -print-machineinstrs
     /// option is specified on the command line, and should enable debugging
@@ -336,9 +330,32 @@ namespace llvm {
     /// Which debugger to tune for.
     DebuggerKind DebuggerTuning = DebuggerKind::Default;
 
-    /// FPDenormalMode - This flags specificies which denormal numbers the code
-    /// is permitted to require.
-    FPDenormal::DenormalMode FPDenormalMode = FPDenormal::IEEE;
+  private:
+    /// Flushing mode to assume in default FP environment.
+    DenormalMode FPDenormalMode;
+
+    /// Flushing mode to assume in default FP environment, for float/vector of
+    /// float.
+    DenormalMode FP32DenormalMode;
+
+  public:
+    void setFPDenormalMode(DenormalMode Mode) {
+      FPDenormalMode = Mode;
+    }
+
+    void setFP32DenormalMode(DenormalMode Mode) {
+      FP32DenormalMode = Mode;
+    }
+
+    DenormalMode getRawFPDenormalMode() const {
+      return FPDenormalMode;
+    }
+
+    DenormalMode getRawFP32DenormalMode() const {
+      return FP32DenormalMode;
+    }
+
+    DenormalMode getDenormalMode(const fltSemantics &FPType) const;
 
     /// What exception model to use
     ExceptionHandling ExceptionModel = ExceptionHandling::None;
diff --git a/llvm/include/llvm/Testing/Support/Annotations.h b/llvm/include/llvm/Testing/Support/Annotations.h
index aad1a44f4ec9c..cc99d10615202 100644
--- a/llvm/include/llvm/Testing/Support/Annotations.h
+++ b/llvm/include/llvm/Testing/Support/Annotations.h
@@ -68,12 +68,14 @@ class Annotations {
   /// Crashes if there isn't exactly one.
   size_t point(llvm::StringRef Name = "") const;
   /// Returns the position of all points marked by ^ (or $name^) in the text.
+  /// Order matches the order within the text.
   std::vector<size_t> points(llvm::StringRef Name = "") const;
 
   /// Returns the location of the range marked by [[ ]] (or $name[[ ]]).
   /// Crashes if there isn't exactly one.
   Range range(llvm::StringRef Name = "") const;
   /// Returns the location of all ranges marked by [[ ]] (or $name[[ ]]).
+  /// They are ordered by start position within the text.
   std::vector<Range> ranges(llvm::StringRef Name = "") const;
 
 private:
diff --git a/llvm/include/llvm/TextAPI/MachO/InterfaceFile.h b/llvm/include/llvm/TextAPI/MachO/InterfaceFile.h
index f86851143b8d6..09d2b8c3cdf0a 100644
--- a/llvm/include/llvm/TextAPI/MachO/InterfaceFile.h
+++ b/llvm/include/llvm/TextAPI/MachO/InterfaceFile.h
@@ -274,6 +274,11 @@ class InterfaceFile {
   /// \param Target_ The target applicable to Parent
   /// \param Parent  The name of Parent
   void addParentUmbrella(const Target &Target_, StringRef Parent);
+
+  /// Get the list of Parent Umbrella frameworks.
+  ///
+  /// \return Returns a list of target information and install name of parent
+  /// umbrellas.
   const std::vector<std::pair<Target, std::string>> &umbrellas() const {
     return ParentUmbrellas;
   }
@@ -328,6 +333,20 @@ class InterfaceFile {
     return UUIDs;
   }
 
+  /// Add a library for inlining to top level library.
+  ///
+  ///\param Document The library to inline with top level library.
+  void addDocument(std::shared_ptr<InterfaceFile> &&Document) {
+    Documents.emplace_back(std::move(Document));
+  }
+
+  /// Get the list of inlined libraries.
+  ///
+  /// \return Returns a list of the inlined frameworks.
+  const std::vector<std::shared_ptr<InterfaceFile>> &documents() const {
+    return Documents;
+  }
+
   /// Add a symbol to the symbols list or extend an existing one.
   void addSymbol(SymbolKind Kind, StringRef Name, const TargetList &Targets,
                  SymbolFlags Flags = SymbolFlags::None);
@@ -403,6 +422,7 @@ class InterfaceFile {
   std::vector<std::pair<Target, std::string>> ParentUmbrellas;
   std::vector<InterfaceFileRef> AllowableClients;
   std::vector<InterfaceFileRef> ReexportedLibraries;
+  std::vector<std::shared_ptr<InterfaceFile>> Documents;
   std::vector<std::pair<Target, std::string>> UUIDs;
   SymbolMapType Symbols;
 };
diff --git a/llvm/include/llvm/Transforms/IPO/Attributor.h b/llvm/include/llvm/Transforms/IPO/Attributor.h
index 2113197c067db..b0c7db2822c97 100644
--- a/llvm/include/llvm/Transforms/IPO/Attributor.h
+++ b/llvm/include/llvm/Transforms/IPO/Attributor.h
@@ -101,6 +101,7 @@
 #include "llvm/ADT/SCCIterator.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/AssumeBundleQueries.h"
 #include "llvm/Analysis/CGSCCPassManager.h"
 #include "llvm/Analysis/CallGraph.h"
 #include "llvm/Analysis/InlineCost.h"
@@ -110,8 +111,8 @@
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/IR/CallSite.h"
 #include "llvm/IR/ConstantRange.h"
-#include "llvm/IR/KnowledgeRetention.h"
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/Allocator.h"
 #include "llvm/Transforms/Utils/CallGraphUpdater.h"
 
 namespace llvm {
@@ -601,6 +602,13 @@ struct InformationCache {
     return AG.getAnalysis<AAManager>(F);
   }
 
+  /// Return true if \p Arg is involved in a must-tail call, thus the argument
+  /// of the caller or callee.
+  bool isInvolvedInMustTailCall(const Argument &Arg) const {
+    return FunctionsCalledViaMustTail.count(Arg.getParent()) ||
+           FunctionsWithMustTailCall.count(Arg.getParent());
+  }
+
   /// Return the analysis result from a pass \p AP for function \p F.
   template <typename AP>
   typename AP::Result *getAnalysisResultForFunction(const Function &F) {
@@ -634,6 +642,12 @@ struct InformationCache {
   /// A map from functions to their instructions that may read or write memory.
   FuncRWInstsMapTy FuncRWInstsMap;
 
+  /// Functions called by a `musttail` call.
+  SmallPtrSet<Function *, 8> FunctionsCalledViaMustTail;
+
+  /// Functions containing a `musttail` call.
+  SmallPtrSet<Function *, 8> FunctionsWithMustTailCall;
+
   /// The datalayout used in the module.
   const DataLayout &DL;
 
@@ -700,11 +714,7 @@ struct Attributor {
       : Functions(Functions), InfoCache(InfoCache), CGUpdater(CGUpdater),
         DepRecomputeInterval(DepRecomputeInterval), Whitelist(Whitelist) {}
 
-  ~Attributor() {
-    DeleteContainerPointers(AllAbstractAttributes);
-    for (auto &It : ArgumentReplacementMap)
-      DeleteContainerPointers(It.second);
-  }
+  ~Attributor();
 
   /// Run the analyses until a fixpoint is reached or enforced (timeout).
   ///
@@ -1070,6 +1080,9 @@ struct Attributor {
   /// Return the data layout associated with the anchor scope.
   const DataLayout &getDataLayout() const { return InfoCache.DL; }
 
+  /// The allocator used to allocate memory, e.g. for `AbstractAttribute`s.
+  BumpPtrAllocator Allocator;
+
 private:
   /// Check \p Pred on all call sites of \p Fn.
   ///
diff --git a/llvm/include/llvm/Transforms/Utils.h b/llvm/include/llvm/Transforms/Utils.h
index bb31646ce4622..402caf7a7d3d2 100644
--- a/llvm/include/llvm/Transforms/Utils.h
+++ b/llvm/include/llvm/Transforms/Utils.h
@@ -126,6 +126,14 @@ FunctionPass *createControlHeightReductionLegacyPass();
 // scalar-to-vector mappings from the TargetLibraryInfo.
 //
 FunctionPass *createInjectTLIMappingsLegacyPass();
+
+//===----------------------------------------------------------------------===//
+//
+// UnifyLoopExits - For each loop, creates a new block N such that all exiting
+// blocks branch to N, and then N distributes control flow to all the original
+// exit blocks.
+//
+FunctionPass *createUnifyLoopExitsPass();
 }
 
 #endif
diff --git a/llvm/include/llvm/Transforms/Utils/AssumeBundleBuilder.h b/llvm/include/llvm/Transforms/Utils/AssumeBundleBuilder.h
new file mode 100644
index 0000000000000..50f89194411ab
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Utils/AssumeBundleBuilder.h
@@ -0,0 +1,45 @@
+//===- AssumeBundleBuilder.h - utils to build assume bundles ----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contain tools to preserve informations. They should be used before
+// performing a transformation that may move and delete instructions as those
+// transformation may destroy or worsen information that can be derived from the
+// IR.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_UTILS_ASSUMEBUNDLEBUILDER_H
+#define LLVM_TRANSFORMS_UTILS_ASSUMEBUNDLEBUILDER_H
+
+#include "llvm/IR/Attributes.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+class IntrinsicInst;
+
+/// Build a call to llvm.assume to preserve informations that can be derived
+/// from the given instruction.
+/// If no information derived from \p I, this call returns null.
+/// The returned instruction is not inserted anywhere.
+IntrinsicInst *buildAssumeFromInst(Instruction *I);
+
+/// Calls BuildAssumeFromInst and if the resulting llvm.assume is valid insert
+/// if before I. This is usually what need to be done to salvage the knowledge
+/// contained in the instruction I.
+void salvageKnowledge(Instruction *I);
+
+/// This pass will try to build an llvm.assume for every instruction in the
+/// function. Its main purpose is testing.
+struct AssumeBuilderPass : public PassInfoMixin<AssumeBuilderPass> {
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+};
+
+} // namespace llvm
+
+#endif
diff --git a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
index f87bee5a6c833..905233b54c9cb 100644
--- a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
+++ b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
@@ -17,6 +17,7 @@
 // FIXME: Move to this file: BasicBlock::removePredecessor, BB::splitBasicBlock
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SetVector.h"
 #include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/CFG.h"
@@ -363,6 +364,81 @@ bool SplitIndirectBrCriticalEdges(Function &F,
                                   BranchProbabilityInfo *BPI = nullptr,
                                   BlockFrequencyInfo *BFI = nullptr);
 
+/// Given a set of incoming and outgoing blocks, create a "hub" such that every
+/// edge from an incoming block InBB to an outgoing block OutBB is now split
+/// into two edges, one from InBB to the hub and another from the hub to
+/// OutBB. The hub consists of a series of guard blocks, one for each outgoing
+/// block. Each guard block conditionally branches to the corresponding outgoing
+/// block, or the next guard block in the chain. These guard blocks are returned
+/// in the argument vector.
+///
+/// Since the control flow edges from InBB to OutBB have now been replaced, the
+/// function also updates any PHINodes in OutBB. For each such PHINode, the
+/// operands corresponding to incoming blocks are moved to a new PHINode in the
+/// hub, and the hub is made an operand of the original PHINode.
+///
+/// Input CFG:
+/// ----------
+///
+///                    Def
+///                     |
+///                     v
+///           In1      In2
+///            |        |
+///            |        |
+///            v        v
+///  Foo ---> Out1     Out2
+///                     |
+///                     v
+///                    Use
+///
+///
+/// Create hub: Incoming = {In1, In2}, Outgoing = {Out1, Out2}
+/// ----------------------------------------------------------
+///
+///             Def
+///              |
+///              v
+///  In1        In2          Foo
+///   |    Hub   |            |
+///   |    + - - | - - +      |
+///   |    '     v     '      V
+///   +------> Guard1 -----> Out1
+///        '     |     '
+///        '     v     '
+///        '   Guard2 -----> Out2
+///        '           '      |
+///        + - - - - - +      |
+///                           v
+///                          Use
+///
+/// Limitations:
+/// -----------
+/// 1. This assumes that all terminators in the CFG are direct branches (the
+///    "br" instruction). The presence of any other control flow such as
+///    indirectbr, switch or callbr will cause an assert.
+///
+/// 2. The updates to the PHINodes are not sufficient to restore SSA
+///    form. Consider a definition Def, its use Use, incoming block In2 and
+///    outgoing block Out2, such that:
+///    a. In2 is reachable from D or contains D.
+///    b. U is reachable from Out2 or is contained in Out2.
+///    c. U is not a PHINode if U is contained in Out2.
+///
+///    Clearly, Def dominates Out2 since the program is valid SSA. But when the
+///    hub is introduced, there is a new path through the hub along which Use is
+///    reachable from entry without passing through Def, and SSA is no longer
+///    valid. To fix this, we need to look at all the blocks post-dominated by
+///    the hub on the one hand, and dominated by Out2 on the other. This is left
+///    for the caller to accomplish, since each specific use of this function
+///    may have additional information which simplifies this fixup. For example,
+///    see restoreSSA() in the UnifyLoopExits pass.
+BasicBlock *CreateControlFlowHub(DomTreeUpdater *DTU,
+                                 SmallVectorImpl<BasicBlock *> &GuardBlocks,
+                                 const SetVector<BasicBlock *> &Predecessors,
+                                 const SetVector<BasicBlock *> &Successors,
+                                 const StringRef Prefix);
+
 } // end namespace llvm
 
 #endif // LLVM_TRANSFORMS_UTILS_BASICBLOCKUTILS_H
diff --git a/llvm/include/llvm/Transforms/Utils/Local.h b/llvm/include/llvm/Transforms/Utils/Local.h
index 4c717100a4027..c8afb5c3f304b 100644
--- a/llvm/include/llvm/Transforms/Utils/Local.h
+++ b/llvm/include/llvm/Transforms/Utils/Local.h
@@ -533,6 +533,13 @@ void maybeMarkSanitizerLibraryCallNoBuiltin(CallInst *CI,
 /// value?
 bool canReplaceOperandWithVariable(const Instruction *I, unsigned OpIdx);
 
+//===----------------------------------------------------------------------===//
+//  Value helper functions
+//
+
+/// Invert the given true/false value, possibly reusing an existing copy.
+Value *invertCondition(Value *Condition);
+
 } // end namespace llvm
 
 #endif // LLVM_TRANSFORMS_UTILS_LOCAL_H
diff --git a/llvm/include/llvm/module.modulemap b/llvm/include/llvm/module.modulemap
index ff05f6251fb05..1709ee6e047cb 100644
--- a/llvm/include/llvm/module.modulemap
+++ b/llvm/include/llvm/module.modulemap
@@ -272,7 +272,6 @@ module LLVM_intrinsic_gen {
   module IR_PatternMatch { header "IR/PatternMatch.h" export * }
   module IR_SafepointIRVerifier { header "IR/SafepointIRVerifier.h" export * }
   module IR_Statepoint { header "IR/Statepoint.h" export * }
-  module IR_KnowledgeRetention { header "IR/KnowledgeRetention.h" export * }
 
   export *
 }
diff --git a/llvm/lib/Analysis/AssumeBundleQueries.cpp b/llvm/lib/Analysis/AssumeBundleQueries.cpp
new file mode 100644
index 0000000000000..5521ac9f556c7
--- /dev/null
+++ b/llvm/lib/Analysis/AssumeBundleQueries.cpp
@@ -0,0 +1,120 @@
+//===- AssumeBundleQueries.cpp - tool to query assume bundles ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/AssumeBundleQueries.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/InstIterator.h"
+#include "llvm/IR/IntrinsicInst.h"
+
+using namespace llvm;
+
+static bool bundleHasArgument(const CallBase::BundleOpInfo &BOI, unsigned Idx) {
+  return BOI.End - BOI.Begin > Idx;
+}
+
+static Value *getValueFromBundleOpInfo(IntrinsicInst &Assume,
+                                       const CallBase::BundleOpInfo &BOI,
+                                       unsigned Idx) {
+  assert(bundleHasArgument(BOI, Idx) && "index out of range");
+  return (Assume.op_begin() + BOI.Begin + Idx)->get();
+}
+
+bool llvm::hasAttributeInAssume(CallInst &AssumeCI, Value *IsOn,
+                                StringRef AttrName, uint64_t *ArgVal,
+                                AssumeQuery AQR) {
+  assert(isa<IntrinsicInst>(AssumeCI) &&
+         "this function is intended to be used on llvm.assume");
+  IntrinsicInst &Assume = cast<IntrinsicInst>(AssumeCI);
+  assert(Assume.getIntrinsicID() == Intrinsic::assume &&
+         "this function is intended to be used on llvm.assume");
+  assert(Attribute::isExistingAttribute(AttrName) &&
+         "this attribute doesn't exist");
+  assert((ArgVal == nullptr || Attribute::doesAttrKindHaveArgument(
+                                   Attribute::getAttrKindFromName(AttrName))) &&
+         "requested value for an attribute that has no argument");
+  if (Assume.bundle_op_infos().empty())
+    return false;
+
+  auto Loop = [&](auto &&Range) {
+    for (auto &BOI : Range) {
+      if (BOI.Tag->getKey() != AttrName)
+        continue;
+      if (IsOn && (BOI.End - BOI.Begin <= ABA_WasOn ||
+                   IsOn != getValueFromBundleOpInfo(Assume, BOI, ABA_WasOn)))
+        continue;
+      if (ArgVal) {
+        assert(BOI.End - BOI.Begin > ABA_Argument);
+        *ArgVal = cast<ConstantInt>(
+                      getValueFromBundleOpInfo(Assume, BOI, ABA_Argument))
+                      ->getZExtValue();
+      }
+      return true;
+    }
+    return false;
+  };
+
+  if (AQR == AssumeQuery::Lowest)
+    return Loop(Assume.bundle_op_infos());
+  return Loop(reverse(Assume.bundle_op_infos()));
+}
+
+void llvm::fillMapFromAssume(CallInst &AssumeCI, RetainedKnowledgeMap &Result) {
+  IntrinsicInst &Assume = cast<IntrinsicInst>(AssumeCI);
+  assert(Assume.getIntrinsicID() == Intrinsic::assume &&
+         "this function is intended to be used on llvm.assume");
+  for (auto &Bundles : Assume.bundle_op_infos()) {
+    std::pair<Value *, Attribute::AttrKind> Key{
+        nullptr, Attribute::getAttrKindFromName(Bundles.Tag->getKey())};
+    if (bundleHasArgument(Bundles, ABA_WasOn))
+      Key.first = getValueFromBundleOpInfo(Assume, Bundles, ABA_WasOn);
+
+    if (Key.first == nullptr && Key.second == Attribute::None)
+      continue;
+    if (!bundleHasArgument(Bundles, ABA_Argument)) {
+      Result[Key][&Assume] = {0, 0};
+      continue;
+    }
+    unsigned Val = cast<ConstantInt>(
+                       getValueFromBundleOpInfo(Assume, Bundles, ABA_Argument))
+                       ->getZExtValue();
+    auto Lookup = Result.find(Key);
+    if (Lookup == Result.end() || !Lookup->second.count(&Assume)) {
+      Result[Key][&Assume] = {Val, Val};
+      continue;
+    }
+    Lookup->second[&Assume].Min = std::min(Val, Lookup->second[&Assume].Min);
+    Lookup->second[&Assume].Max = std::max(Val, Lookup->second[&Assume].Max);
+  }
+}
+
+RetainedKnowledge llvm::getKnowledgeFromOperandInAssume(CallInst &AssumeCI,
+                                                        unsigned Idx) {
+  IntrinsicInst &Assume = cast<IntrinsicInst>(AssumeCI);
+  assert(Assume.getIntrinsicID() == Intrinsic::assume &&
+         "this function is intended to be used on llvm.assume");
+  CallBase::BundleOpInfo BOI = Assume.getBundleOpInfoForOperand(Idx);
+  RetainedKnowledge Result;
+  Result.AttrKind = Attribute::getAttrKindFromName(BOI.Tag->getKey());
+  Result.WasOn = getValueFromBundleOpInfo(Assume, BOI, ABA_WasOn);
+  if (BOI.End - BOI.Begin > ABA_Argument)
+    Result.ArgValue =
+        cast<ConstantInt>(getValueFromBundleOpInfo(Assume, BOI, ABA_Argument))
+            ->getZExtValue();
+
+  return Result;
+}
+
+bool llvm::isAssumeWithEmptyBundle(CallInst &CI) {
+  IntrinsicInst &Assume = cast<IntrinsicInst>(CI);
+  assert(Assume.getIntrinsicID() == Intrinsic::assume &&
+         "this function is intended to be used on llvm.assume");
+  return none_of(Assume.bundle_op_infos(),
+                 [](const CallBase::BundleOpInfo &BOI) {
+                   return BOI.Tag->getKey() != "ignore";
+                 });
+}
diff --git a/llvm/lib/Analysis/BasicAliasAnalysis.cpp b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
index 0cc28efd44898..9ab056f4bb3eb 100644
--- a/llvm/lib/Analysis/BasicAliasAnalysis.cpp
+++ b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
@@ -960,7 +960,7 @@ ModRefInfo BasicAAResult::getModRefInfo(const CallBase *Call,
     }
   }
 
-  // If the call is to malloc or calloc, we can assume that it doesn't
+  // If the call is malloc/calloc like, we can assume that it doesn't
   // modify any IR visible value.  This is only valid because we assume these
   // routines do not read values visible in the IR.  TODO: Consider special
   // casing realloc and strdup routines which access only their arguments as
diff --git a/llvm/lib/Analysis/BlockFrequencyInfo.cpp b/llvm/lib/Analysis/BlockFrequencyInfo.cpp
index 544bd7757ae4a..b02b009318f40 100644
--- a/llvm/lib/Analysis/BlockFrequencyInfo.cpp
+++ b/llvm/lib/Analysis/BlockFrequencyInfo.cpp
@@ -98,7 +98,7 @@ static GVDAGType getGVDT() {
 template <>
 struct GraphTraits<BlockFrequencyInfo *> {
   using NodeRef = const BasicBlock *;
-  using ChildIteratorType = succ_const_iterator;
+  using ChildIteratorType = const_succ_iterator;
   using nodes_iterator = pointer_iterator<Function::const_iterator>;
 
   static NodeRef getEntryNode(const BlockFrequencyInfo *G) {
diff --git a/llvm/lib/Analysis/BranchProbabilityInfo.cpp b/llvm/lib/Analysis/BranchProbabilityInfo.cpp
index c69993877a108..38fb90f7915c2 100644
--- a/llvm/lib/Analysis/BranchProbabilityInfo.cpp
+++ b/llvm/lib/Analysis/BranchProbabilityInfo.cpp
@@ -240,7 +240,7 @@ bool BranchProbabilityInfo::calcUnreachableHeuristics(const BasicBlock *BB) {
   SmallVector<unsigned, 4> UnreachableEdges;
   SmallVector<unsigned, 4> ReachableEdges;
 
-  for (succ_const_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I)
+  for (const_succ_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I)
     if (PostDominatedByUnreachable.count(*I))
       UnreachableEdges.push_back(I.getSuccessorIndex());
     else
@@ -386,7 +386,7 @@ bool BranchProbabilityInfo::calcColdCallHeuristics(const BasicBlock *BB) {
   // Determine which successors are post-dominated by a cold block.
   SmallVector<unsigned, 4> ColdEdges;
   SmallVector<unsigned, 4> NormalEdges;
-  for (succ_const_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I)
+  for (const_succ_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I)
     if (PostDominatedByColdCall.count(*I))
       ColdEdges.push_back(I.getSuccessorIndex());
     else
@@ -614,7 +614,7 @@ bool BranchProbabilityInfo::calcLoopBranchHeuristics(const BasicBlock *BB,
   SmallVector<unsigned, 8> InEdges; // Edges from header to the loop.
   SmallVector<unsigned, 8> UnlikelyEdges;
 
-  for (succ_const_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I) {
+  for (const_succ_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I) {
     // Use LoopInfo if we have it, otherwise fall-back to SCC info to catch
     // irreducible loops.
     if (L) {
@@ -871,7 +871,7 @@ void BranchProbabilityInfo::print(raw_ostream &OS) const {
   // or the function it is currently running over.
   assert(LastF && "Cannot print prior to running over a function");
   for (const auto &BI : *LastF) {
-    for (succ_const_iterator SI = succ_begin(&BI), SE = succ_end(&BI); SI != SE;
+    for (const_succ_iterator SI = succ_begin(&BI), SE = succ_end(&BI); SI != SE;
          ++SI) {
       printEdgeProbability(OS << "  ", &BI, *SI);
     }
@@ -890,7 +890,7 @@ BranchProbabilityInfo::getHotSucc(const BasicBlock *BB) const {
   auto MaxProb = BranchProbability::getZero();
   const BasicBlock *MaxSucc = nullptr;
 
-  for (succ_const_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I) {
+  for (const_succ_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I) {
     const BasicBlock *Succ = *I;
     auto Prob = getEdgeProbability(BB, Succ);
     if (Prob > MaxProb) {
@@ -923,7 +923,7 @@ BranchProbabilityInfo::getEdgeProbability(const BasicBlock *Src,
 
 BranchProbability
 BranchProbabilityInfo::getEdgeProbability(const BasicBlock *Src,
-                                          succ_const_iterator Dst) const {
+                                          const_succ_iterator Dst) const {
   return getEdgeProbability(Src, Dst.getSuccessorIndex());
 }
 
@@ -934,7 +934,7 @@ BranchProbabilityInfo::getEdgeProbability(const BasicBlock *Src,
                                           const BasicBlock *Dst) const {
   auto Prob = BranchProbability::getZero();
   bool FoundProb = false;
-  for (succ_const_iterator I = succ_begin(Src), E = succ_end(Src); I != E; ++I)
+  for (const_succ_iterator I = succ_begin(Src), E = succ_end(Src); I != E; ++I)
     if (*I == Dst) {
       auto MapI = Probs.find(std::make_pair(Src, I.getSuccessorIndex()));
       if (MapI != Probs.end()) {
diff --git a/llvm/lib/Analysis/CFG.cpp b/llvm/lib/Analysis/CFG.cpp
index 8215b4ecbb03d..c80429c972450 100644
--- a/llvm/lib/Analysis/CFG.cpp
+++ b/llvm/lib/Analysis/CFG.cpp
@@ -31,16 +31,16 @@ void llvm::FindFunctionBackedges(const Function &F,
     return;
 
   SmallPtrSet<const BasicBlock*, 8> Visited;
-  SmallVector<std::pair<const BasicBlock*, succ_const_iterator>, 8> VisitStack;
+  SmallVector<std::pair<const BasicBlock *, const_succ_iterator>, 8> VisitStack;
   SmallPtrSet<const BasicBlock*, 8> InStack;
 
   Visited.insert(BB);
   VisitStack.push_back(std::make_pair(BB, succ_begin(BB)));
   InStack.insert(BB);
   do {
-    std::pair<const BasicBlock*, succ_const_iterator> &Top = VisitStack.back();
+    std::pair<const BasicBlock *, const_succ_iterator> &Top = VisitStack.back();
     const BasicBlock *ParentBB = Top.first;
-    succ_const_iterator &I = Top.second;
+    const_succ_iterator &I = Top.second;
 
     bool FoundNew = false;
     while (I != succ_end(ParentBB)) {
diff --git a/llvm/lib/Analysis/CMakeLists.txt b/llvm/lib/Analysis/CMakeLists.txt
index 969049f907821..8a658aa6adf7b 100644
--- a/llvm/lib/Analysis/CMakeLists.txt
+++ b/llvm/lib/Analysis/CMakeLists.txt
@@ -4,6 +4,7 @@ add_llvm_component_library(LLVMAnalysis
   AliasAnalysisSummary.cpp
   AliasSetTracker.cpp
   Analysis.cpp
+  AssumeBundleQueries.cpp
   AssumptionCache.cpp
   BasicAliasAnalysis.cpp
   BlockFrequencyInfo.cpp
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 87c22b3609a15..721ba735e9db7 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -38,6 +38,7 @@
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/IR/IntrinsicsX86.h"
@@ -363,7 +364,7 @@ Constant *llvm::ConstantFoldLoadThroughBitcast(Constant *C, Type *DestTy,
       Constant *ElemC;
       do {
         ElemC = C->getAggregateElement(Elem++);
-      } while (ElemC && DL.getTypeSizeInBits(ElemC->getType()) == 0);
+      } while (ElemC && DL.getTypeSizeInBits(ElemC->getType()).isZero());
       C = ElemC;
     } else {
       C = C->getAggregateElement(0u);
@@ -1065,7 +1066,8 @@ Constant *ConstantFoldInstOperandsImpl(const Value *InstOrCE, unsigned Opcode,
   case Instruction::InsertElement:
     return ConstantExpr::getInsertElement(Ops[0], Ops[1], Ops[2]);
   case Instruction::ShuffleVector:
-    return ConstantExpr::getShuffleVector(Ops[0], Ops[1], Ops[2]);
+    return ConstantExpr::getShuffleVector(
+        Ops[0], Ops[1], cast<ShuffleVectorInst>(InstOrCE)->getShuffleMask());
   }
 }
 
@@ -1396,41 +1398,19 @@ llvm::ConstantFoldLoadThroughGEPIndices(Constant *C,
 //
 
 bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) {
-  if (Call->isNoBuiltin() || Call->isStrictFP())
+  if (Call->isNoBuiltin())
     return false;
   switch (F->getIntrinsicID()) {
-  case Intrinsic::fabs:
-  case Intrinsic::minnum:
-  case Intrinsic::maxnum:
-  case Intrinsic::minimum:
-  case Intrinsic::maximum:
-  case Intrinsic::log:
-  case Intrinsic::log2:
-  case Intrinsic::log10:
-  case Intrinsic::exp:
-  case Intrinsic::exp2:
-  case Intrinsic::floor:
-  case Intrinsic::ceil:
-  case Intrinsic::sqrt:
-  case Intrinsic::sin:
-  case Intrinsic::cos:
-  case Intrinsic::trunc:
-  case Intrinsic::rint:
-  case Intrinsic::nearbyint:
-  case Intrinsic::pow:
-  case Intrinsic::powi:
+  // Operations that do not operate floating-point numbers and do not depend on
+  // FP environment can be folded even in strictfp functions.
   case Intrinsic::bswap:
   case Intrinsic::ctpop:
   case Intrinsic::ctlz:
   case Intrinsic::cttz:
   case Intrinsic::fshl:
   case Intrinsic::fshr:
-  case Intrinsic::fma:
-  case Intrinsic::fmuladd:
-  case Intrinsic::copysign:
   case Intrinsic::launder_invariant_group:
   case Intrinsic::strip_invariant_group:
-  case Intrinsic::round:
   case Intrinsic::masked_load:
   case Intrinsic::sadd_with_overflow:
   case Intrinsic::uadd_with_overflow:
@@ -1444,9 +1424,31 @@ bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) {
   case Intrinsic::usub_sat:
   case Intrinsic::smul_fix:
   case Intrinsic::smul_fix_sat:
+  case Intrinsic::bitreverse:
+  case Intrinsic::is_constant:
+    return true;
+
+  // Floating point operations cannot be folded in strictfp functions in
+  // general case. They can be folded if FP environment is known to compiler.
+  case Intrinsic::minnum:
+  case Intrinsic::maxnum:
+  case Intrinsic::minimum:
+  case Intrinsic::maximum:
+  case Intrinsic::log:
+  case Intrinsic::log2:
+  case Intrinsic::log10:
+  case Intrinsic::exp:
+  case Intrinsic::exp2:
+  case Intrinsic::sqrt:
+  case Intrinsic::sin:
+  case Intrinsic::cos:
+  case Intrinsic::pow:
+  case Intrinsic::powi:
+  case Intrinsic::fma:
+  case Intrinsic::fmuladd:
   case Intrinsic::convert_from_fp16:
   case Intrinsic::convert_to_fp16:
-  case Intrinsic::bitreverse:
+  // The intrinsics below depend on rounding mode in MXCSR.
   case Intrinsic::amdgcn_cubeid:
   case Intrinsic::amdgcn_cubema:
   case Intrinsic::amdgcn_cubesc:
@@ -1477,14 +1479,35 @@ bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) {
   case Intrinsic::x86_avx512_vcvtsd2usi64:
   case Intrinsic::x86_avx512_cvttsd2usi:
   case Intrinsic::x86_avx512_cvttsd2usi64:
-  case Intrinsic::is_constant:
+    return !Call->isStrictFP();
+
+  // Sign operations are actually bitwise operations, they do not raise
+  // exceptions even for SNANs.
+  case Intrinsic::fabs:
+  case Intrinsic::copysign:
+  // Non-constrained variants of rounding operations means default FP
+  // environment, they can be folded in any case.
+  case Intrinsic::ceil:
+  case Intrinsic::floor:
+  case Intrinsic::round:
+  case Intrinsic::trunc:
+  case Intrinsic::nearbyint:
+  case Intrinsic::rint:
+  // Constrained intrinsics can be folded if FP environment is known
+  // to compiler.
+  case Intrinsic::experimental_constrained_ceil:
+  case Intrinsic::experimental_constrained_floor:
+  case Intrinsic::experimental_constrained_round:
+  case Intrinsic::experimental_constrained_trunc:
+  case Intrinsic::experimental_constrained_nearbyint:
+  case Intrinsic::experimental_constrained_rint:
     return true;
   default:
     return false;
   case Intrinsic::not_intrinsic: break;
   }
 
-  if (!F->hasName())
+  if (!F->hasName() || Call->isStrictFP())
     return false;
 
   // In these cases, the check of the length is required.  We don't want to
@@ -1792,6 +1815,55 @@ static Constant *ConstantFoldScalarCall1(StringRef Name,
       return ConstantFP::get(Ty->getContext(), minimum(FractU, AlmostOne));
     }
 
+    // Rounding operations (floor, trunc, ceil, round and nearbyint) do not
+    // raise FP exceptions, unless the argument is signaling NaN.
+
+    Optional<APFloat::roundingMode> RM;
+    switch (IntrinsicID) {
+    default:
+      break;
+    case Intrinsic::experimental_constrained_nearbyint:
+    case Intrinsic::experimental_constrained_rint: {
+      auto CI = cast<ConstrainedFPIntrinsic>(Call);
+      Optional<fp::RoundingMode> RMOp = CI->getRoundingMode();
+      if (RMOp)
+        RM = getAPFloatRoundingMode(*RMOp);
+      if (!RM)
+        return nullptr;
+      break;
+    }
+    case Intrinsic::experimental_constrained_round:
+      RM = APFloat::rmNearestTiesToAway;
+      break;
+    case Intrinsic::experimental_constrained_ceil:
+      RM = APFloat::rmTowardPositive;
+      break;
+    case Intrinsic::experimental_constrained_floor:
+      RM = APFloat::rmTowardNegative;
+      break;
+    case Intrinsic::experimental_constrained_trunc:
+      RM = APFloat::rmTowardZero;
+      break;
+    }
+    if (RM) {
+      auto CI = cast<ConstrainedFPIntrinsic>(Call);
+      if (U.isFinite()) {
+        APFloat::opStatus St = U.roundToIntegral(*RM);
+        if (IntrinsicID == Intrinsic::experimental_constrained_rint &&
+            St == APFloat::opInexact) {
+          Optional<fp::ExceptionBehavior> EB = CI->getExceptionBehavior();
+          if (EB && *EB == fp::ebStrict)
+            return nullptr;
+        }
+      } else if (U.isSignaling()) {
+        Optional<fp::ExceptionBehavior> EB = CI->getExceptionBehavior();
+        if (EB && *EB != fp::ebIgnore)
+          return nullptr;
+        U = APFloat::getQNaN(U.getSemantics());
+      }
+      return ConstantFP::get(Ty->getContext(), U);
+    }
+
     /// We only fold functions with finite arguments. Folding NaN and inf is
     /// likely to be aborted with an exception anyway, and some host libms
     /// have known errors raising exceptions.
@@ -2573,7 +2645,7 @@ static Constant *ConstantFoldVectorCall(StringRef Name,
 Constant *llvm::ConstantFoldCall(const CallBase *Call, Function *F,
                                  ArrayRef<Constant *> Operands,
                                  const TargetLibraryInfo *TLI) {
-  if (Call->isNoBuiltin() || Call->isStrictFP())
+  if (Call->isNoBuiltin())
     return nullptr;
   if (!F->hasName())
     return nullptr;
diff --git a/llvm/lib/Analysis/InstructionPrecedenceTracking.cpp b/llvm/lib/Analysis/InstructionPrecedenceTracking.cpp
index 3f5a161c8659f..a6e4455c2e6ed 100644
--- a/llvm/lib/Analysis/InstructionPrecedenceTracking.cpp
+++ b/llvm/lib/Analysis/InstructionPrecedenceTracking.cpp
@@ -126,26 +126,7 @@ bool ImplicitControlFlowTracking::isSpecialInstruction(
   // to avoid wrong assumptions of sort "if A is executed and B post-dominates
   // A, then B is also executed". This is not true is there is an implicit
   // control flow instruction (e.g. a guard) between them.
-  //
-  // TODO: Currently, isGuaranteedToTransferExecutionToSuccessor returns false
-  // for volatile stores and loads because they can trap. The discussion on
-  // whether or not it is correct is still ongoing. We might want to get rid
-  // of this logic in the future. Anyways, trapping instructions shouldn't
-  // introduce implicit control flow, so we explicitly allow them here. This
-  // must be removed once isGuaranteedToTransferExecutionToSuccessor is fixed.
-  if (isGuaranteedToTransferExecutionToSuccessor(Insn))
-    return false;
-  if (isa<LoadInst>(Insn)) {
-    assert(cast<LoadInst>(Insn)->isVolatile() &&
-           "Non-volatile load should transfer execution to successor!");
-    return false;
-  }
-  if (isa<StoreInst>(Insn)) {
-    assert(cast<StoreInst>(Insn)->isVolatile() &&
-           "Non-volatile store should transfer execution to successor!");
-    return false;
-  }
-  return true;
+  return !isGuaranteedToTransferExecutionToSuccessor(Insn);
 }
 
 bool MemoryWriteTracking::isSpecialInstruction(
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index ba1f550487006..086ff74335449 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -4439,36 +4439,31 @@ static Value *foldIdentityShuffles(int DestElt, Value *Op0, Value *Op1,
   return RootVec;
 }
 
-static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask,
-                                        Type *RetTy, const SimplifyQuery &Q,
+static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1,
+                                        ArrayRef<int> Mask, Type *RetTy,
+                                        const SimplifyQuery &Q,
                                         unsigned MaxRecurse) {
-  if (isa<UndefValue>(Mask))
+  if (all_of(Mask, [](int Elem) { return Elem == UndefMaskElem; }))
     return UndefValue::get(RetTy);
 
   Type *InVecTy = Op0->getType();
-  ElementCount MaskEltCount = Mask->getType()->getVectorElementCount();
+  unsigned MaskNumElts = Mask.size();
   ElementCount InVecEltCount = InVecTy->getVectorElementCount();
 
-  assert(MaskEltCount.Scalable == InVecEltCount.Scalable &&
-         "vscale mismatch between input vector and mask");
-
-  bool Scalable = MaskEltCount.Scalable;
+  bool Scalable = InVecEltCount.Scalable;
 
   SmallVector<int, 32> Indices;
-  if (!Scalable) {
-    ShuffleVectorInst::getShuffleMask(Mask, Indices);
-    assert(MaskEltCount.Min == Indices.size() &&
-           "Size of Indices not same as number of mask elements?");
-  }
+  Indices.assign(Mask.begin(), Mask.end());
 
+  // Canonicalization: If mask does not select elements from an input vector,
+  // replace that input vector with undef.
   if (!Scalable) {
-    // Canonicalization: If mask does not select elements from an input vector,
-    // replace that input vector with undef.
     bool MaskSelects0 = false, MaskSelects1 = false;
-    for (unsigned i = 0; i != MaskEltCount.Min; ++i) {
+    unsigned InVecNumElts = InVecEltCount.Min;
+    for (unsigned i = 0; i != MaskNumElts; ++i) {
       if (Indices[i] == -1)
         continue;
-      if ((unsigned)Indices[i] < InVecEltCount.Min)
+      if ((unsigned)Indices[i] < InVecNumElts)
         MaskSelects0 = true;
       else
         MaskSelects1 = true;
@@ -4514,8 +4509,8 @@ static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask,
       assert(isa<UndefValue>(Op1) && "Expected undef operand 1 for splat");
 
       // Shuffle mask undefs become undefined constant result elements.
-      SmallVector<Constant *, 16> VecC(MaskEltCount.Min, C);
-      for (unsigned i = 0; i != MaskEltCount.Min; ++i)
+      SmallVector<Constant *, 16> VecC(MaskNumElts, C);
+      for (unsigned i = 0; i != MaskNumElts; ++i)
         if (Indices[i] == -1)
           VecC[i] = UndefValue::get(C->getType());
       return ConstantVector::get(VecC);
@@ -4526,7 +4521,7 @@ static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask,
   // value type is same as the input vectors' type.
   if (auto *OpShuf = dyn_cast<ShuffleVectorInst>(Op0))
     if (isa<UndefValue>(Op1) && RetTy == InVecTy &&
-        OpShuf->getMask()->getSplatValue())
+        is_splat(OpShuf->getShuffleMask()))
       return Op0;
 
   // All remaining transformation depend on the value of the mask, which is
@@ -4545,7 +4540,7 @@ static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask,
   // shuffle. This handles simple identity shuffles as well as chains of
   // shuffles that may widen/narrow and/or move elements across lanes and back.
   Value *RootVec = nullptr;
-  for (unsigned i = 0; i != MaskEltCount.Min; ++i) {
+  for (unsigned i = 0; i != MaskNumElts; ++i) {
     // Note that recursion is limited for each vector element, so if any element
     // exceeds the limit, this will fail to simplify.
     RootVec =
@@ -4559,8 +4554,9 @@ static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask,
 }
 
 /// Given operands for a ShuffleVectorInst, fold the result or return null.
-Value *llvm::SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask,
-                                       Type *RetTy, const SimplifyQuery &Q) {
+Value *llvm::SimplifyShuffleVectorInst(Value *Op0, Value *Op1,
+                                       ArrayRef<int> Mask, Type *RetTy,
+                                       const SimplifyQuery &Q) {
   return ::SimplifyShuffleVectorInst(Op0, Op1, Mask, RetTy, Q, RecursionLimit);
 }
 
@@ -5368,8 +5364,11 @@ Value *llvm::SimplifyCall(CallBase *Call, const SimplifyQuery &Q) {
   ConstantArgs.reserve(NumArgs);
   for (auto &Arg : Call->args()) {
     Constant *C = dyn_cast<Constant>(&Arg);
-    if (!C)
+    if (!C) {
+      if (isa<MetadataAsValue>(Arg.get()))
+        continue;
       return nullptr;
+    }
     ConstantArgs.push_back(C);
   }
 
@@ -5520,8 +5519,9 @@ Value *llvm::SimplifyInstruction(Instruction *I, const SimplifyQuery &SQ,
   }
   case Instruction::ShuffleVector: {
     auto *SVI = cast<ShuffleVectorInst>(I);
-    Result = SimplifyShuffleVectorInst(SVI->getOperand(0), SVI->getOperand(1),
-                                       SVI->getMask(), SVI->getType(), Q);
+    Result =
+        SimplifyShuffleVectorInst(SVI->getOperand(0), SVI->getOperand(1),
+                                  SVI->getShuffleMask(), SVI->getType(), Q);
     break;
   }
   case Instruction::PHI:
diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp
index 296f27799b945..4ad1795f613dd 100644
--- a/llvm/lib/Analysis/LazyValueInfo.cpp
+++ b/llvm/lib/Analysis/LazyValueInfo.cpp
@@ -121,11 +121,13 @@ static ValueLatticeElement intersect(const ValueLatticeElement &A,
 
   // Intersect two constant ranges
   ConstantRange Range =
-    A.getConstantRange().intersectWith(B.getConstantRange());
+      A.getConstantRange().intersectWith(B.getConstantRange());
   // Note: An empty range is implicitly converted to overdefined internally.
   // TODO: We could instead use Undefined here since we've proven a conflict
   // and thus know this path must be unreachable.
-  return ValueLatticeElement::getRange(std::move(Range));
+  return ValueLatticeElement::getRange(
+      std::move(Range), /*MayIncludeUndef=*/A.isConstantRangeIncludingUndef() |
+                            B.isConstantRangeIncludingUndef());
 }
 
 //===----------------------------------------------------------------------===//
@@ -901,17 +903,21 @@ bool LazyValueInfoImpl::solveBlockValueSelect(ValueLatticeElement &BBLV,
           return TrueCR.umax(FalseCR);
         };
       }();
-      BBLV = ValueLatticeElement::getRange(ResultCR);
+      BBLV = ValueLatticeElement::getRange(
+          ResultCR, TrueVal.isConstantRangeIncludingUndef() |
+                        FalseVal.isConstantRangeIncludingUndef());
       return true;
     }
 
     if (SPR.Flavor == SPF_ABS) {
       if (LHS == SI->getTrueValue()) {
-        BBLV = ValueLatticeElement::getRange(TrueCR.abs());
+        BBLV = ValueLatticeElement::getRange(
+            TrueCR.abs(), TrueVal.isConstantRangeIncludingUndef());
         return true;
       }
       if (LHS == SI->getFalseValue()) {
-        BBLV = ValueLatticeElement::getRange(FalseCR.abs());
+        BBLV = ValueLatticeElement::getRange(
+            FalseCR.abs(), FalseVal.isConstantRangeIncludingUndef());
         return true;
       }
     }
@@ -919,11 +925,13 @@ bool LazyValueInfoImpl::solveBlockValueSelect(ValueLatticeElement &BBLV,
     if (SPR.Flavor == SPF_NABS) {
       ConstantRange Zero(APInt::getNullValue(TrueCR.getBitWidth()));
       if (LHS == SI->getTrueValue()) {
-        BBLV = ValueLatticeElement::getRange(Zero.sub(TrueCR.abs()));
+        BBLV = ValueLatticeElement::getRange(
+            Zero.sub(TrueCR.abs()), FalseVal.isConstantRangeIncludingUndef());
         return true;
       }
       if (LHS == SI->getFalseValue()) {
-        BBLV = ValueLatticeElement::getRange(Zero.sub(FalseCR.abs()));
+        BBLV = ValueLatticeElement::getRange(
+            Zero.sub(FalseCR.abs()), FalseVal.isConstantRangeIncludingUndef());
         return true;
       }
     }
@@ -1706,7 +1714,8 @@ Constant *LazyValueInfo::getConstant(Value *V, BasicBlock *BB,
 }
 
 ConstantRange LazyValueInfo::getConstantRange(Value *V, BasicBlock *BB,
-                                              Instruction *CxtI) {
+                                              Instruction *CxtI,
+                                              bool UndefAllowed) {
   assert(V->getType()->isIntegerTy());
   unsigned Width = V->getType()->getIntegerBitWidth();
   const DataLayout &DL = BB->getModule()->getDataLayout();
@@ -1714,8 +1723,8 @@ ConstantRange LazyValueInfo::getConstantRange(Value *V, BasicBlock *BB,
       getImpl(PImpl, AC, &DL, DT).getValueInBlock(V, BB, CxtI);
   if (Result.isUnknown())
     return ConstantRange::getEmpty(Width);
-  if (Result.isConstantRange())
-    return Result.getConstantRange();
+  if (Result.isConstantRange(UndefAllowed))
+    return Result.getConstantRange(UndefAllowed);
   // We represent ConstantInt constants as constant ranges but other kinds
   // of integer constants, i.e. ConstantExpr will be tagged as constants
   assert(!(Result.isConstant() && isa<ConstantInt>(Result.getConstant())) &&
diff --git a/llvm/lib/Analysis/MemoryBuiltins.cpp b/llvm/lib/Analysis/MemoryBuiltins.cpp
index be0feeb9237e9..ac72bda4eb633 100644
--- a/llvm/lib/Analysis/MemoryBuiltins.cpp
+++ b/llvm/lib/Analysis/MemoryBuiltins.cpp
@@ -52,11 +52,12 @@ using namespace llvm;
 enum AllocType : uint8_t {
   OpNewLike          = 1<<0, // allocates; never returns null
   MallocLike         = 1<<1 | OpNewLike, // allocates; may return null
-  CallocLike         = 1<<2, // allocates + bzero
-  ReallocLike        = 1<<3, // reallocates
-  StrDupLike         = 1<<4,
-  MallocOrCallocLike = MallocLike | CallocLike,
-  AllocLike          = MallocLike | CallocLike | StrDupLike,
+  AlignedAllocLike   = 1<<2, // allocates with alignment; may return null
+  CallocLike         = 1<<3, // allocates + bzero
+  ReallocLike        = 1<<4, // reallocates
+  StrDupLike         = 1<<5,
+  MallocOrCallocLike = MallocLike | CallocLike | AlignedAllocLike,
+  AllocLike          = MallocOrCallocLike | StrDupLike,
   AnyAlloc           = AllocLike | ReallocLike
 };
 
@@ -100,6 +101,7 @@ static const std::pair<LibFunc, AllocFnsTy> AllocationFnData[] = {
   {LibFunc_msvc_new_array_int_nothrow, {MallocLike,  2, 0,  -1}}, // new[](unsigned int, nothrow)
   {LibFunc_msvc_new_array_longlong,         {OpNewLike,   1, 0,  -1}}, // new[](unsigned long long)
   {LibFunc_msvc_new_array_longlong_nothrow, {MallocLike,  2, 0,  -1}}, // new[](unsigned long long, nothrow)
+  {LibFunc_aligned_alloc,       {AlignedAllocLike, 2, 1,  -1}},
   {LibFunc_calloc,              {CallocLike,  2, 0,   1}},
   {LibFunc_realloc,             {ReallocLike, 2, 1,  -1}},
   {LibFunc_reallocf,            {ReallocLike, 2, 1,  -1}},
@@ -265,6 +267,20 @@ bool llvm::isMallocLikeFn(
       .hasValue();
 }
 
+/// Tests if a value is a call or invoke to a library function that
+/// allocates uninitialized memory with alignment (such as aligned_alloc).
+bool llvm::isAlignedAllocLikeFn(const Value *V, const TargetLibraryInfo *TLI,
+                                bool LookThroughBitCast) {
+  return getAllocationData(V, AlignedAllocLike, TLI, LookThroughBitCast)
+      .hasValue();
+}
+bool llvm::isAlignedAllocLikeFn(
+    const Value *V, function_ref<const TargetLibraryInfo &(Function &)> GetTLI,
+    bool LookThroughBitCast) {
+  return getAllocationData(V, AlignedAllocLike, GetTLI, LookThroughBitCast)
+      .hasValue();
+}
+
 /// Tests if a value is a call or invoke to a library function that
 /// allocates zero-filled memory (such as calloc).
 bool llvm::isCallocLikeFn(const Value *V, const TargetLibraryInfo *TLI,
diff --git a/llvm/lib/Analysis/MemorySSAUpdater.cpp b/llvm/lib/Analysis/MemorySSAUpdater.cpp
index 473268982f2df..4fcc8affcd90f 100644
--- a/llvm/lib/Analysis/MemorySSAUpdater.cpp
+++ b/llvm/lib/Analysis/MemorySSAUpdater.cpp
@@ -781,24 +781,24 @@ void MemorySSAUpdater::updateExitBlocksForClonedLoop(
 
 void MemorySSAUpdater::applyUpdates(ArrayRef<CFGUpdate> Updates,
                                     DominatorTree &DT) {
-  SmallVector<CFGUpdate, 4> RevDeleteUpdates;
+  SmallVector<CFGUpdate, 4> DeleteUpdates;
   SmallVector<CFGUpdate, 4> InsertUpdates;
   for (auto &Update : Updates) {
     if (Update.getKind() == DT.Insert)
       InsertUpdates.push_back({DT.Insert, Update.getFrom(), Update.getTo()});
     else
-      RevDeleteUpdates.push_back({DT.Insert, Update.getFrom(), Update.getTo()});
+      DeleteUpdates.push_back({DT.Delete, Update.getFrom(), Update.getTo()});
   }
 
-  if (!RevDeleteUpdates.empty()) {
+  if (!DeleteUpdates.empty()) {
     // Update for inserted edges: use newDT and snapshot CFG as if deletes had
     // not occurred.
     // FIXME: This creates a new DT, so it's more expensive to do mix
     // delete/inserts vs just inserts. We can do an incremental update on the DT
     // to revert deletes, than re-delete the edges. Teaching DT to do this, is
     // part of a pending cleanup.
-    DominatorTree NewDT(DT, RevDeleteUpdates);
-    GraphDiff<BasicBlock *> GD(RevDeleteUpdates);
+    DominatorTree NewDT(DT, DeleteUpdates);
+    GraphDiff<BasicBlock *> GD(DeleteUpdates, /*ReverseApplyUpdates=*/true);
     applyInsertUpdates(InsertUpdates, NewDT, &GD);
   } else {
     GraphDiff<BasicBlock *> GD;
@@ -806,7 +806,7 @@ void MemorySSAUpdater::applyUpdates(ArrayRef<CFGUpdate> Updates,
   }
 
   // Update for deleted edges
-  for (auto &Update : RevDeleteUpdates)
+  for (auto &Update : DeleteUpdates)
     removeEdge(Update.getFrom(), Update.getTo());
 }
 
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index dffbaf139c4c3..b9fd2b422ace2 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -8285,10 +8285,11 @@ const SCEV *ScalarEvolution::computeSCEVAtScope(const SCEV *V, const Loop *L) {
           if (!isa<SCEVCouldNotCompute>(BackedgeTakenCount) &&
               isKnownPositive(BackedgeTakenCount) &&
               PN->getNumIncomingValues() == 2) {
+
             unsigned InLoopPred = LI->contains(PN->getIncomingBlock(0)) ? 0 : 1;
-            const SCEV *OnBackedge = getSCEV(PN->getIncomingValue(InLoopPred));
-            if (IsAvailableOnEntry(LI, DT, OnBackedge, PN->getParent()))
-              return OnBackedge;
+            Value *BackedgeVal = PN->getIncomingValue(InLoopPred);
+            if (LI->isLoopInvariant(BackedgeVal))
+              return getSCEV(BackedgeVal);
           }
           if (auto *BTCC = dyn_cast<SCEVConstant>(BackedgeTakenCount)) {
             // Okay, we know how many times the containing loop executes.  If
diff --git a/llvm/lib/Analysis/SyncDependenceAnalysis.cpp b/llvm/lib/Analysis/SyncDependenceAnalysis.cpp
index c16daa91a7758..ccf520dcea66e 100644
--- a/llvm/lib/Analysis/SyncDependenceAnalysis.cpp
+++ b/llvm/lib/Analysis/SyncDependenceAnalysis.cpp
@@ -369,7 +369,7 @@ SyncDependenceAnalysis::join_blocks(const Instruction &Term) {
   // compute all join points
   DivergencePropagator Propagator{FuncRPOT, DT, PDT, LI};
   const auto &TermBlock = *Term.getParent();
-  auto JoinBlocks = Propagator.computeJoinPoints<succ_const_range>(
+  auto JoinBlocks = Propagator.computeJoinPoints<const_succ_range>(
       TermBlock, successors(Term.getParent()), LI.getLoopFor(&TermBlock));
 
   auto ItInserted = CachedBranchJoins.emplace(&Term, std::move(JoinBlocks));
diff --git a/llvm/lib/Analysis/TargetLibraryInfo.cpp b/llvm/lib/Analysis/TargetLibraryInfo.cpp
index fa4deea632d8f..ffa6921c51bef 100644
--- a/llvm/lib/Analysis/TargetLibraryInfo.cpp
+++ b/llvm/lib/Analysis/TargetLibraryInfo.cpp
@@ -901,6 +901,8 @@ bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy,
             FTy.getParamType(1)->isPointerTy());
   case LibFunc_write:
     return (NumParams == 3 && FTy.getParamType(1)->isPointerTy());
+  case LibFunc_aligned_alloc:
+    return (NumParams == 2 && FTy.getReturnType()->isPointerTy());
   case LibFunc_bcopy:
   case LibFunc_bcmp:
     return (NumParams == 3 && FTy.getParamType(0)->isPointerTy() &&
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 89b1783a9a007..150a395ef8c52 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -153,21 +153,6 @@ int TargetTransformInfo::getOperationCost(unsigned Opcode, Type *Ty,
   return Cost;
 }
 
-int TargetTransformInfo::getCallCost(FunctionType *FTy, int NumArgs,
-                                     const User *U) const {
-  int Cost = TTIImpl->getCallCost(FTy, NumArgs, U);
-  assert(Cost >= 0 && "TTI should not produce negative costs!");
-  return Cost;
-}
-
-int TargetTransformInfo::getCallCost(const Function *F,
-                                     ArrayRef<const Value *> Arguments,
-                                     const User *U) const {
-  int Cost = TTIImpl->getCallCost(F, Arguments, U);
-  assert(Cost >= 0 && "TTI should not produce negative costs!");
-  return Cost;
-}
-
 unsigned TargetTransformInfo::getInliningThresholdMultiplier() const {
   return TTIImpl->getInliningThresholdMultiplier();
 }
@@ -534,14 +519,22 @@ unsigned TargetTransformInfo::getPrefetchDistance() const {
   return TTIImpl->getPrefetchDistance();
 }
 
-unsigned TargetTransformInfo::getMinPrefetchStride() const {
-  return TTIImpl->getMinPrefetchStride();
+unsigned TargetTransformInfo::getMinPrefetchStride(unsigned NumMemAccesses,
+                                                  unsigned NumStridedMemAccesses,
+                                                   unsigned NumPrefetches,
+                                                   bool HasCall) const {
+  return TTIImpl->getMinPrefetchStride(NumMemAccesses, NumStridedMemAccesses,
+                                       NumPrefetches, HasCall);
 }
 
 unsigned TargetTransformInfo::getMaxPrefetchIterationsAhead() const {
   return TTIImpl->getMaxPrefetchIterationsAhead();
 }
 
+bool TargetTransformInfo::enableWritePrefetching() const {
+  return TTIImpl->enableWritePrefetching();
+}
+
 unsigned TargetTransformInfo::getMaxInterleaveFactor(unsigned VF) const {
   return TTIImpl->getMaxInterleaveFactor(VF);
 }
@@ -888,7 +881,7 @@ static bool matchPairwiseShuffleMask(ShuffleVectorInst *SI, bool IsLeft,
   for (unsigned i = 0, e = (1 << Level), val = !IsLeft; i != e; ++i, val += 2)
     Mask[i] = val;
 
-  SmallVector<int, 16> ActualMask = SI->getShuffleMask();
+  ArrayRef<int> ActualMask = SI->getShuffleMask();
   return Mask == ActualMask;
 }
 
@@ -1153,7 +1146,7 @@ matchVectorSplittingReduction(const ExtractElementInst *ReduxRoot,
     // Fill the rest of the mask with -1 for undef.
     std::fill(&ShuffleMask[MaskStart], ShuffleMask.end(), -1);
 
-    SmallVector<int, 16> Mask = Shuffle->getShuffleMask();
+    ArrayRef<int> Mask = Shuffle->getShuffleMask();
     if (ShuffleMask != Mask)
       return RK_None;
 
diff --git a/llvm/lib/Analysis/ValueLattice.cpp b/llvm/lib/Analysis/ValueLattice.cpp
index 1a15605ce1e13..627166e2409d3 100644
--- a/llvm/lib/Analysis/ValueLattice.cpp
+++ b/llvm/lib/Analysis/ValueLattice.cpp
@@ -20,10 +20,10 @@ raw_ostream &operator<<(raw_ostream &OS, const ValueLatticeElement &Val) {
   if (Val.isNotConstant())
     return OS << "notconstant<" << *Val.getNotConstant() << ">";
 
-  if (Val.isSingleCRFromUndef())
-    return OS << "constantrange (from undef)<"
-              << Val.getConstantRange().getLower() << ", "
-              << Val.getConstantRange().getUpper() << ">";
+  if (Val.isConstantRangeIncludingUndef())
+    return OS << "constantrange incl. undef <"
+              << Val.getConstantRange(true).getLower() << ", "
+              << Val.getConstantRange(true).getUpper() << ">";
 
   if (Val.isConstantRange())
     return OS << "constantrange<" << Val.getConstantRange().getLower() << ", "
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index ab6afa1a81dcc..6a4d596a3b767 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -168,13 +168,19 @@ static bool getShuffleDemandedElts(const ShuffleVectorInst *Shuf,
                                    APInt &DemandedLHS, APInt &DemandedRHS) {
   // The length of scalable vectors is unknown at compile time, thus we
   // cannot check their values
-  if (Shuf->getMask()->getType()->getVectorElementCount().Scalable)
+  if (Shuf->getType()->getVectorElementCount().Scalable)
     return false;
 
   int NumElts = Shuf->getOperand(0)->getType()->getVectorNumElements();
-  int NumMaskElts = Shuf->getMask()->getType()->getVectorNumElements();
+  int NumMaskElts = Shuf->getType()->getVectorNumElements();
   DemandedLHS = DemandedRHS = APInt::getNullValue(NumElts);
-
+  if (DemandedElts.isNullValue())
+    return true;
+  // Simple case of a shuffle with zeroinitializer.
+  if (all_of(Shuf->getShuffleMask(), [](int Elt) { return Elt == 0; })) {
+    DemandedLHS.setBit(0);
+    return true;
+  }
   for (int i = 0; i != NumMaskElts; ++i) {
     if (!DemandedElts[i])
       continue;
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index 532518acc6ad9..8b98e05a68847 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -339,9 +339,9 @@ const llvm::Value *llvm::getSplatValue(const Value *V) {
 
   // shuf (inselt ?, Splat, 0), ?, <0, undef, 0, ...>
   Value *Splat;
-  if (match(V, m_ShuffleVector(m_InsertElement(m_Value(), m_Value(Splat),
-                                               m_ZeroInt()),
-                               m_Value(), m_ZeroInt())))
+  if (match(V, m_ShuffleVector(
+                   m_InsertElement(m_Value(), m_Value(Splat), m_ZeroInt()),
+                   m_Value(), m_ZeroMask())))
     return Splat;
 
   return nullptr;
@@ -366,7 +366,7 @@ bool llvm::isSplatValue(const Value *V, int Index, unsigned Depth) {
   if (auto *Shuf = dyn_cast<ShuffleVectorInst>(V)) {
     // FIXME: We can safely allow undefs here. If Index was specified, we will
     //        check that the mask elt is defined at the required index.
-    if (!Shuf->getMask()->getSplatValue())
+    if (!is_splat(Shuf->getShuffleMask()))
       return false;
 
     // Match any index.
@@ -397,6 +397,22 @@ bool llvm::isSplatValue(const Value *V, int Index, unsigned Depth) {
   return false;
 }
 
+void llvm::scaleShuffleMask(size_t Scale, ArrayRef<int> Mask,
+                            SmallVectorImpl<int> &ScaledMask) {
+  assert(Scale > 0 && "Unexpected scaling factor");
+
+  // Fast-path: if no scaling, then it is just a copy.
+  if (Scale == 1) {
+    ScaledMask.assign(Mask.begin(), Mask.end());
+    return;
+  }
+
+  ScaledMask.clear();
+  for (int MaskElt : Mask)
+    for (int ScaleElt = 0; ScaleElt != (int)Scale; ++ScaleElt)
+      ScaledMask.push_back(MaskElt < 0 ? MaskElt : Scale * MaskElt + ScaleElt);
+}
+
 MapVector<Instruction *, uint64_t>
 llvm::computeMinimumValueSizes(ArrayRef<BasicBlock *> Blocks, DemandedBits &DB,
                                const TargetTransformInfo *TTI) {
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index d219f607410b9..ebb1720486462 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -3636,8 +3636,9 @@ bool LLParser::ParseValID(ValID &ID, PerFunctionState *PFS) {
         return Error(ID.Loc, "expected three operands to shufflevector");
       if (!ShuffleVectorInst::isValidOperands(Elts[0], Elts[1], Elts[2]))
         return Error(ID.Loc, "invalid operands to shufflevector");
-      ID.ConstantVal =
-                 ConstantExpr::getShuffleVector(Elts[0], Elts[1],Elts[2]);
+      SmallVector<int, 16> Mask;
+      ShuffleVectorInst::getShuffleMask(cast<Constant>(Elts[2]), Mask);
+      ID.ConstantVal = ConstantExpr::getShuffleVector(Elts[0], Elts[1], Mask);
     } else if (Opc == Instruction::ExtractElement) {
       if (Elts.size() != 2)
         return Error(ID.Loc, "expected two operands to extractelement");
diff --git a/llvm/lib/BinaryFormat/XCOFF.cpp b/llvm/lib/BinaryFormat/XCOFF.cpp
index 29ccbaea3584d..b5391fc82660c 100644
--- a/llvm/lib/BinaryFormat/XCOFF.cpp
+++ b/llvm/lib/BinaryFormat/XCOFF.cpp
@@ -32,3 +32,36 @@ StringRef XCOFF::getMappingClassString(XCOFF::StorageMappingClass SMC) {
     report_fatal_error("Unhandled storage-mapping class.");
   }
 }
+
+#define RELOC_CASE(A)                                                          \
+  case XCOFF::A:                                                               \
+    return #A;
+StringRef XCOFF::getRelocationTypeString(XCOFF::RelocationType Type) {
+  switch (Type) {
+    RELOC_CASE(R_POS)
+    RELOC_CASE(R_RL)
+    RELOC_CASE(R_RLA)
+    RELOC_CASE(R_NEG)
+    RELOC_CASE(R_REL)
+    RELOC_CASE(R_TOC)
+    RELOC_CASE(R_TRL)
+    RELOC_CASE(R_TRLA)
+    RELOC_CASE(R_GL)
+    RELOC_CASE(R_TCL)
+    RELOC_CASE(R_REF)
+    RELOC_CASE(R_BA)
+    RELOC_CASE(R_BR)
+    RELOC_CASE(R_RBA)
+    RELOC_CASE(R_RBR)
+    RELOC_CASE(R_TLS)
+    RELOC_CASE(R_TLS_IE)
+    RELOC_CASE(R_TLS_LD)
+    RELOC_CASE(R_TLS_LE)
+    RELOC_CASE(R_TLSM)
+    RELOC_CASE(R_TLSML)
+    RELOC_CASE(R_TOCU)
+    RELOC_CASE(R_TOCL)
+  }
+  return "Unknown";
+}
+#undef RELOC_CASE
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index de2a6b47e6e29..16d3b79598f1f 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -2337,6 +2337,15 @@ Error BitcodeReader::parseConstants() {
   Type *CurFullTy = Type::getInt32Ty(Context);
   unsigned NextCstNo = ValueList.size();
 
+  struct DelayedShufTy {
+    VectorType *OpTy;
+    VectorType *RTy;
+    Type *CurFullTy;
+    uint64_t Op0Idx;
+    uint64_t Op1Idx;
+    uint64_t Op2Idx;
+  };
+  std::vector<DelayedShufTy> DelayedShuffles;
   while (true) {
     Expected<BitstreamEntry> MaybeEntry = Stream.advanceSkippingSubblocks();
     if (!MaybeEntry)
@@ -2353,6 +2362,29 @@ Error BitcodeReader::parseConstants() {
 
       // Once all the constants have been read, go through and resolve forward
       // references.
+      //
+      // We have to treat shuffles specially because they don't have three
+      // operands anymore.  We need to convert the shuffle mask into an array,
+      // and we can't convert a forward reference.
+      for (auto &DelayedShuffle : DelayedShuffles) {
+        VectorType *OpTy = DelayedShuffle.OpTy;
+        VectorType *RTy = DelayedShuffle.RTy;
+        uint64_t Op0Idx = DelayedShuffle.Op0Idx;
+        uint64_t Op1Idx = DelayedShuffle.Op1Idx;
+        uint64_t Op2Idx = DelayedShuffle.Op2Idx;
+        Constant *Op0 = ValueList.getConstantFwdRef(Op0Idx, OpTy);
+        Constant *Op1 = ValueList.getConstantFwdRef(Op1Idx, OpTy);
+        Type *ShufTy =
+            VectorType::get(Type::getInt32Ty(Context), RTy->getElementCount());
+        Constant *Op2 = ValueList.getConstantFwdRef(Op2Idx, ShufTy);
+        if (!ShuffleVectorInst::isValidOperands(Op0, Op1, Op2))
+          return error("Invalid shufflevector operands");
+        SmallVector<int, 16> Mask;
+        ShuffleVectorInst::getShuffleMask(Op2, Mask);
+        Value *V = ConstantExpr::getShuffleVector(Op0, Op1, Mask);
+        ValueList.assignValue(V, NextCstNo, DelayedShuffle.CurFullTy);
+        ++NextCstNo;
+      }
       ValueList.resolveConstantForwardRefs();
       return Error::success();
     case BitstreamEntry::Record:
@@ -2694,13 +2726,9 @@ Error BitcodeReader::parseConstants() {
       VectorType *OpTy = dyn_cast<VectorType>(CurTy);
       if (Record.size() < 3 || !OpTy)
         return error("Invalid record");
-      Constant *Op0 = ValueList.getConstantFwdRef(Record[0], OpTy);
-      Constant *Op1 = ValueList.getConstantFwdRef(Record[1], OpTy);
-      Type *ShufTy = VectorType::get(Type::getInt32Ty(Context),
-                                     OpTy->getElementCount());
-      Constant *Op2 = ValueList.getConstantFwdRef(Record[2], ShufTy);
-      V = ConstantExpr::getShuffleVector(Op0, Op1, Op2);
-      break;
+      DelayedShuffles.push_back(
+          {OpTy, OpTy, CurFullTy, Record[0], Record[1], Record[2]});
+      continue;
     }
     case bitc::CST_CODE_CE_SHUFVEC_EX: { // [opty, opval, opval, opval]
       VectorType *RTy = dyn_cast<VectorType>(CurTy);
@@ -2708,13 +2736,9 @@ Error BitcodeReader::parseConstants() {
         dyn_cast_or_null<VectorType>(getTypeByID(Record[0]));
       if (Record.size() < 4 || !RTy || !OpTy)
         return error("Invalid record");
-      Constant *Op0 = ValueList.getConstantFwdRef(Record[1], OpTy);
-      Constant *Op1 = ValueList.getConstantFwdRef(Record[2], OpTy);
-      Type *ShufTy = VectorType::get(Type::getInt32Ty(Context),
-                                     RTy->getElementCount());
-      Constant *Op2 = ValueList.getConstantFwdRef(Record[3], ShufTy);
-      V = ConstantExpr::getShuffleVector(Op0, Op1, Op2);
-      break;
+      DelayedShuffles.push_back(
+          {OpTy, RTy, CurFullTy, Record[1], Record[2], Record[3]});
+      continue;
     }
     case bitc::CST_CODE_CE_CMP: {     // CE_CMP: [opty, opval, opval, pred]
       if (Record.size() < 4)
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index a27eada55d7e3..e9ad00d6d4bad 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -2515,7 +2515,7 @@ void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal,
         }
         Record.push_back(VE.getValueID(C->getOperand(0)));
         Record.push_back(VE.getValueID(C->getOperand(1)));
-        Record.push_back(VE.getValueID(C->getOperand(2)));
+        Record.push_back(VE.getValueID(CE->getShuffleMaskForBitcode()));
         break;
       case Instruction::ICmp:
       case Instruction::FCmp:
@@ -2702,7 +2702,8 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
     Code = bitc::FUNC_CODE_INST_SHUFFLEVEC;
     pushValueAndType(I.getOperand(0), InstID, Vals);
     pushValue(I.getOperand(1), InstID, Vals);
-    pushValue(I.getOperand(2), InstID, Vals);
+    pushValue(cast<ShuffleVectorInst>(I).getShuffleMaskForBitcode(), InstID,
+              Vals);
     break;
   case Instruction::ICmp:
   case Instruction::FCmp: {
diff --git a/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp b/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp
index f59c906c7b757..7419c4687facf 100644
--- a/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp
+++ b/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp
@@ -88,11 +88,16 @@ static void orderValue(const Value *V, OrderMap &OM) {
   if (OM.lookup(V).first)
     return;
 
-  if (const Constant *C = dyn_cast<Constant>(V))
-    if (C->getNumOperands() && !isa<GlobalValue>(C))
+  if (const Constant *C = dyn_cast<Constant>(V)) {
+    if (C->getNumOperands() && !isa<GlobalValue>(C)) {
       for (const Value *Op : C->operands())
         if (!isa<BasicBlock>(Op) && !isa<GlobalValue>(Op))
           orderValue(Op, OM);
+      if (auto *CE = dyn_cast<ConstantExpr>(C))
+        if (CE->getOpcode() == Instruction::ShuffleVector)
+          orderValue(CE->getShuffleMaskForBitcode(), OM);
+    }
+  }
 
   // Note: we cannot cache this lookup above, since inserting into the map
   // changes the map's size, and thus affects the other IDs.
@@ -155,11 +160,14 @@ static OrderMap orderModule(const Module &M) {
     for (const Argument &A : F.args())
       orderValue(&A, OM);
     for (const BasicBlock &BB : F)
-      for (const Instruction &I : BB)
+      for (const Instruction &I : BB) {
         for (const Value *Op : I.operands())
           if ((isa<Constant>(*Op) && !isa<GlobalValue>(*Op)) ||
               isa<InlineAsm>(*Op))
             orderValue(Op, OM);
+        if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
+          orderValue(SVI->getShuffleMaskForBitcode(), OM);
+      }
     for (const BasicBlock &BB : F)
       for (const Instruction &I : BB)
         orderValue(&I, OM);
@@ -250,11 +258,17 @@ static void predictValueUseListOrder(const Value *V, const Function *F,
     predictValueUseListOrderImpl(V, F, IDPair.first, OM, Stack);
 
   // Recursive descent into constants.
-  if (const Constant *C = dyn_cast<Constant>(V))
-    if (C->getNumOperands()) // Visit GlobalValues.
+  if (const Constant *C = dyn_cast<Constant>(V)) {
+    if (C->getNumOperands()) { // Visit GlobalValues.
       for (const Value *Op : C->operands())
         if (isa<Constant>(Op)) // Visit GlobalValues.
           predictValueUseListOrder(Op, F, OM, Stack);
+      if (auto *CE = dyn_cast<ConstantExpr>(C))
+        if (CE->getOpcode() == Instruction::ShuffleVector)
+          predictValueUseListOrder(CE->getShuffleMaskForBitcode(), F, OM,
+                                   Stack);
+    }
+  }
 }
 
 static UseListOrderStack predictUseListOrder(const Module &M) {
@@ -279,10 +293,14 @@ static UseListOrderStack predictUseListOrder(const Module &M) {
     for (const Argument &A : F.args())
       predictValueUseListOrder(&A, &F, OM, Stack);
     for (const BasicBlock &BB : F)
-      for (const Instruction &I : BB)
+      for (const Instruction &I : BB) {
         for (const Value *Op : I.operands())
           if (isa<Constant>(*Op) || isa<InlineAsm>(*Op)) // Visit GlobalValues.
             predictValueUseListOrder(Op, &F, OM, Stack);
+        if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
+          predictValueUseListOrder(SVI->getShuffleMaskForBitcode(), &F, OM,
+                                   Stack);
+      }
     for (const BasicBlock &BB : F)
       for (const Instruction &I : BB)
         predictValueUseListOrder(&I, &F, OM, Stack);
@@ -413,6 +431,8 @@ ValueEnumerator::ValueEnumerator(const Module &M,
 
           EnumerateMetadata(&F, MD->getMetadata());
         }
+        if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
+          EnumerateType(SVI->getShuffleMaskForBitcode()->getType());
         EnumerateType(I.getType());
         if (const auto *Call = dyn_cast<CallBase>(&I))
           EnumerateAttributes(Call->getAttributes());
@@ -836,6 +856,9 @@ void ValueEnumerator::EnumerateValue(const Value *V) {
            I != E; ++I)
         if (!isa<BasicBlock>(*I)) // Don't enumerate BB operand to BlockAddress.
           EnumerateValue(*I);
+      if (auto *CE = dyn_cast<ConstantExpr>(C))
+        if (CE->getOpcode() == Instruction::ShuffleVector)
+          EnumerateValue(CE->getShuffleMaskForBitcode());
 
       // Finally, add the value.  Doing this could make the ValueID reference be
       // dangling, don't reuse it.
@@ -913,6 +936,9 @@ void ValueEnumerator::EnumerateOperandType(const Value *V) {
 
     EnumerateOperandType(Op);
   }
+  if (auto *CE = dyn_cast<ConstantExpr>(C))
+    if (CE->getOpcode() == Instruction::ShuffleVector)
+      EnumerateOperandType(CE->getShuffleMaskForBitcode());
 }
 
 void ValueEnumerator::EnumerateAttributes(AttributeList PAL) {
@@ -958,11 +984,14 @@ void ValueEnumerator::incorporateFunction(const Function &F) {
 
   // Add all function-level constants to the value table.
   for (const BasicBlock &BB : F) {
-    for (const Instruction &I : BB)
+    for (const Instruction &I : BB) {
       for (const Use &OI : I.operands()) {
         if ((isa<Constant>(OI) && !isa<GlobalValue>(OI)) || isa<InlineAsm>(OI))
           EnumerateValue(OI);
       }
+      if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
+        EnumerateValue(SVI->getShuffleMaskForBitcode());
+    }
     BasicBlocks.push_back(&BB);
     ValueMap[&BB] = BasicBlocks.size();
   }
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index f44457f3b054b..8c51203a517b2 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -414,7 +414,7 @@ void AsmPrinter::emitLinkage(const GlobalValue *GV, MCSymbol *GVSym) const {
         OutStreamer->emitSymbolAttribute(GVSym, MCSA_WeakDefinition);
       else
         OutStreamer->emitSymbolAttribute(GVSym, MCSA_WeakDefAutoPrivate);
-    } else if (MAI->hasLinkOnceDirective()) {
+    } else if (MAI->avoidWeakIfComdat() && GV->hasComdat()) {
       // .globl _foo
       OutStreamer->emitSymbolAttribute(GVSym, MCSA_Global);
       //NOTE: linkonce is handled by the section the symbol was assigned to.
@@ -1747,13 +1747,8 @@ void AsmPrinter::SetupMachineFunction(MachineFunction &MF) {
                                " initalized first.");
 
     // Get the function entry point symbol.
-    CurrentFnSym =
-        OutContext.getOrCreateSymbol("." + CurrentFnDescSym->getName());
-
-    // Set the containing csect.
-    MCSectionXCOFF *FnEntryPointSec =
-        cast<MCSectionXCOFF>(getObjFileLowering().SectionForGlobal(&F, TM));
-    cast<MCSymbolXCOFF>(CurrentFnSym)->setContainingCsect(FnEntryPointSec);
+    CurrentFnSym = OutContext.getOrCreateSymbol(
+        "." + cast<MCSymbolXCOFF>(CurrentFnDescSym)->getUnqualifiedName());
   }
 
   CurrentFnSymForSize = CurrentFnSym;
@@ -1846,11 +1841,6 @@ void AsmPrinter::emitConstantPool() {
       if (!Sym->isUndefined())
         continue;
 
-      if (TM.getTargetTriple().isOSBinFormatXCOFF()) {
-        cast<MCSymbolXCOFF>(Sym)->setContainingCsect(
-            cast<MCSectionXCOFF>(CPSections[i].S));
-      }
-
       if (CurSection != CPSections[i].S) {
         OutStreamer->SwitchSection(CPSections[i].S);
         emitAlignment(Align(CPSections[i].Alignment));
@@ -1945,10 +1935,6 @@ void AsmPrinter::emitJumpTableInfo() {
       OutStreamer->emitLabel(GetJTISymbol(JTI, true));
 
     MCSymbol* JTISymbol = GetJTISymbol(JTI);
-    if (TM.getTargetTriple().isOSBinFormatXCOFF()) {
-      cast<MCSymbolXCOFF>(JTISymbol)->setContainingCsect(
-          cast<MCSectionXCOFF>(TLOF.getSectionForJumpTable(F, TM)));
-    }
     OutStreamer->emitLabel(JTISymbol);
 
     for (unsigned ii = 0, ee = JTBBs.size(); ii != ee; ++ii)
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 1fd5154cbf5be..1654eba6bc572 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -495,6 +495,9 @@ bool CodeGenPrepare::runOnFunction(Function &F) {
     if (!LargeOffsetGEPMap.empty())
       MadeChange |= splitLargeGEPOffsets();
 
+    if (MadeChange)
+      eliminateFallThrough(F);
+
     // Really free removed instructions during promotion.
     for (Instruction *I : RemovedInsts)
       I->deleteValue();
@@ -1964,6 +1967,11 @@ bool CodeGenPrepare::optimizeCallInst(CallInst *CI, bool &ModifiedDT) {
   if (II) {
     switch (II->getIntrinsicID()) {
     default: break;
+    case Intrinsic::assume: {
+      II->eraseFromParent();
+      return true;
+    }
+
     case Intrinsic::experimental_widenable_condition: {
       // Give up on future widening oppurtunties so that we can fold away dead
       // paths and merge blocks before going into block-local instruction
@@ -6249,7 +6257,7 @@ bool CodeGenPrepare::optimizeSelectInst(SelectInst *SI) {
 }
 
 static bool isBroadcastShuffle(ShuffleVectorInst *SVI) {
-  SmallVector<int, 16> Mask(SVI->getShuffleMask());
+  ArrayRef<int> Mask(SVI->getShuffleMask());
   int SplatElem = -1;
   for (unsigned i = 0; i < Mask.size(); ++i) {
     if (SplatElem != -1 && Mask[i] != -1 && Mask[i] != SplatElem)
@@ -6299,7 +6307,7 @@ bool CodeGenPrepare::optimizeShuffleVectorInst(ShuffleVectorInst *SVI) {
       assert(InsertPt != UserBB->end());
       InsertedShuffle =
           new ShuffleVectorInst(SVI->getOperand(0), SVI->getOperand(1),
-                                SVI->getOperand(2), "", &*InsertPt);
+                                SVI->getShuffleMask(), "", &*InsertPt);
       InsertedShuffle->setDebugLoc(SVI->getDebugLoc());
     }
 
diff --git a/llvm/lib/CodeGen/CommandFlags.cpp b/llvm/lib/CodeGen/CommandFlags.cpp
index d5dc49a91177b..12fc1df8d9b78 100644
--- a/llvm/lib/CodeGen/CommandFlags.cpp
+++ b/llvm/lib/CodeGen/CommandFlags.cpp
@@ -54,7 +54,8 @@ CGOPT(bool, EnableNoInfsFPMath)
 CGOPT(bool, EnableNoNaNsFPMath)
 CGOPT(bool, EnableNoSignedZerosFPMath)
 CGOPT(bool, EnableNoTrappingFPMath)
-CGOPT(FPDenormal::DenormalMode, DenormalFPMath)
+CGOPT(DenormalMode::DenormalModeKind, DenormalFPMath)
+CGOPT(DenormalMode::DenormalModeKind, DenormalFP32Math)
 CGOPT(bool, EnableHonorSignDependentRoundingFPMath)
 CGOPT(FloatABI::ABIType, FloatABIForCalls)
 CGOPT(FPOpFusion::FPOpFusionMode, FuseFPOps)
@@ -212,20 +213,30 @@ codegen::RegisterCodeGenFlags::RegisterCodeGenFlags() {
       cl::init(false));
   CGBINDOPT(EnableNoTrappingFPMath);
 
-  static cl::opt<FPDenormal::DenormalMode> DenormalFPMath(
-      "denormal-fp-math",
-      cl::desc(
-          "Select which denormal numbers the code is permitted to require"),
-      cl::init(FPDenormal::IEEE),
-      cl::values(
-          clEnumValN(FPDenormal::IEEE, "ieee", "IEEE 754 denormal numbers"),
-          clEnumValN(FPDenormal::PreserveSign, "preserve-sign",
-                     "the sign of a  flushed-to-zero number is preserved "
-                     "in the sign of 0"),
-          clEnumValN(FPDenormal::PositiveZero, "positive-zero",
-                     "denormals are flushed to positive zero")));
+  static const auto DenormFlagEnumOptions =
+  cl::values(clEnumValN(DenormalMode::IEEE, "ieee",
+                        "IEEE 754 denormal numbers"),
+             clEnumValN(DenormalMode::PreserveSign, "preserve-sign",
+                        "the sign of a  flushed-to-zero number is preserved "
+                        "in the sign of 0"),
+             clEnumValN(DenormalMode::PositiveZero, "positive-zero",
+                        "denormals are flushed to positive zero"));
+
+  // FIXME: Doesn't have way to specify separate input and output modes.
+  static cl::opt<DenormalMode::DenormalModeKind> DenormalFPMath(
+    "denormal-fp-math",
+    cl::desc("Select which denormal numbers the code is permitted to require"),
+    cl::init(DenormalMode::IEEE),
+    DenormFlagEnumOptions);
   CGBINDOPT(DenormalFPMath);
 
+  static cl::opt<DenormalMode::DenormalModeKind> DenormalFP32Math(
+    "denormal-fp-math-f32",
+    cl::desc("Select which denormal numbers the code is permitted to require for float"),
+    cl::init(DenormalMode::Invalid),
+    DenormFlagEnumOptions);
+  CGBINDOPT(DenormalFP32Math);
+
   static cl::opt<bool> EnableHonorSignDependentRoundingFPMath(
       "enable-sign-dependent-rounding-fp-math", cl::Hidden,
       cl::desc("Force codegen to assume rounding mode can change dynamically"),
@@ -425,7 +436,12 @@ TargetOptions codegen::InitTargetOptionsFromCodeGenFlags() {
   Options.NoNaNsFPMath = getEnableNoNaNsFPMath();
   Options.NoSignedZerosFPMath = getEnableNoSignedZerosFPMath();
   Options.NoTrappingFPMath = getEnableNoTrappingFPMath();
-  Options.FPDenormalMode = getDenormalFPMath();
+
+  DenormalMode::DenormalModeKind DenormKind = getDenormalFPMath();
+
+  // FIXME: Should have separate input and output flags
+  Options.setFPDenormalMode(DenormalMode(DenormKind, DenormKind));
+
   Options.HonorSignDependentRoundingFPMathOption =
       getEnableHonorSignDependentRoundingFPMath();
   if (getFloatABIForCalls() != FloatABI::Default)
@@ -563,6 +579,25 @@ void codegen::setFunctionAttributes(StringRef CPU, StringRef Features,
   HANDLE_BOOL_ATTR(EnableNoNaNsFPMathView, "no-nans-fp-math");
   HANDLE_BOOL_ATTR(EnableNoSignedZerosFPMathView, "no-signed-zeros-fp-math");
 
+  if (DenormalFPMathView->getNumOccurrences() > 0 &&
+      !F.hasFnAttribute("denormal-fp-math")) {
+    DenormalMode::DenormalModeKind DenormKind = getDenormalFPMath();
+
+    // FIXME: Command line flag should expose separate input/output modes.
+    NewAttrs.addAttribute("denormal-fp-math",
+                          DenormalMode(DenormKind, DenormKind).str());
+  }
+
+  if (DenormalFP32MathView->getNumOccurrences() > 0 &&
+      !F.hasFnAttribute("denormal-fp-math-f32")) {
+    // FIXME: Command line flag should expose separate input/output modes.
+    DenormalMode::DenormalModeKind DenormKind = getDenormalFP32Math();
+
+    NewAttrs.addAttribute(
+      "denormal-fp-math-f32",
+      DenormalMode(DenormKind, DenormKind).str());
+  }
+
   if (TrapFuncNameView->getNumOccurrences() > 0)
     for (auto &B : F)
       for (auto &I : B)
diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
index 2aea679e82433..7dcc7bfcfe026 100644
--- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
@@ -35,7 +35,7 @@ bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, ImmutableCallSite CS,
                              Register SwiftErrorVReg,
                              std::function<unsigned()> GetCalleeReg) const {
   CallLoweringInfo Info;
-  auto &DL = CS.getParent()->getParent()->getParent()->getDataLayout();
+  const DataLayout &DL = MIRBuilder.getDataLayout();
 
   // First step is to marshall all the function's parameters into the correct
   // physregs and memory locations. Gather the sequence of argument types that
@@ -62,17 +62,17 @@ bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, ImmutableCallSite CS,
   if (!Info.OrigRet.Ty->isVoidTy())
     setArgFlags(Info.OrigRet, AttributeList::ReturnIndex, DL, CS);
 
+  MachineFunction &MF = MIRBuilder.getMF();
   Info.KnownCallees =
       CS.getInstruction()->getMetadata(LLVMContext::MD_callees);
   Info.CallConv = CS.getCallingConv();
   Info.SwiftErrorVReg = SwiftErrorVReg;
   Info.IsMustTailCall = CS.isMustTailCall();
   Info.IsTailCall = CS.isTailCall() &&
-                    isInTailCallPosition(CS, MIRBuilder.getMF().getTarget()) &&
-                    (MIRBuilder.getMF()
-                         .getFunction()
-                         .getFnAttribute("disable-tail-calls")
-                         .getValueAsString() != "true");
+                    isInTailCallPosition(CS, MF.getTarget()) &&
+                    (MF.getFunction()
+                       .getFnAttribute("disable-tail-calls")
+                       .getValueAsString() != "true");
   Info.IsVarArg = CS.getFunctionType()->isVarArg();
   return lowerCall(MIRBuilder, Info);
 }
@@ -108,12 +108,12 @@ void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx,
 
     // For ByVal, alignment should be passed from FE.  BE will guess if
     // this info is not there but there are cases it cannot get right.
-    unsigned FrameAlign;
-    if (FuncInfo.getParamAlignment(OpIdx - 2))
-      FrameAlign = FuncInfo.getParamAlignment(OpIdx - 2);
+    Align FrameAlign;
+    if (auto ParamAlign = FuncInfo.getParamAlign(OpIdx - 2))
+      FrameAlign = *ParamAlign;
     else
-      FrameAlign = getTLI()->getByValTypeAlignment(ElementTy, DL);
-    Flags.setByValAlign(Align(FrameAlign));
+      FrameAlign = Align(getTLI()->getByValTypeAlignment(ElementTy, DL));
+    Flags.setByValAlign(FrameAlign);
   }
   if (Attrs.hasAttribute(OpIdx, Attribute::Nest))
     Flags.setNest();
@@ -160,7 +160,7 @@ void CallLowering::unpackRegs(ArrayRef<Register> DstRegs, Register SrcReg,
                               MachineIRBuilder &MIRBuilder) const {
   assert(DstRegs.size() > 1 && "Nothing to unpack");
 
-  const DataLayout &DL = MIRBuilder.getMF().getDataLayout();
+  const DataLayout &DL = MIRBuilder.getDataLayout();
 
   SmallVector<LLT, 8> LLTs;
   SmallVector<uint64_t, 8> Offsets;
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index e9e4e2b30698a..1e4824e857572 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -9,6 +9,7 @@
 #include "llvm/CodeGen/GlobalISel/Combiner.h"
 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
 #include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
+#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
 #include "llvm/CodeGen/GlobalISel/Utils.h"
 #include "llvm/CodeGen/MachineDominators.h"
@@ -23,6 +24,7 @@
 #define DEBUG_TYPE "gi-combiner"
 
 using namespace llvm;
+using namespace MIPatternMatch;
 
 // Option to allow testing of the combiner while no targets know about indexed
 // addressing.
@@ -1268,7 +1270,7 @@ bool CombinerHelper::tryCombineMemCpyFamily(MachineInstr &MI, unsigned MaxLen) {
   if (IsVolatile)
     return false;
 
-  Align DstAlign(MemOp->getBaseAlignment());
+  Align DstAlign = MemOp->getBaseAlign();
   Align SrcAlign;
   Register Dst = MI.getOperand(1).getReg();
   Register Src = MI.getOperand(2).getReg();
@@ -1277,7 +1279,7 @@ bool CombinerHelper::tryCombineMemCpyFamily(MachineInstr &MI, unsigned MaxLen) {
   if (ID != Intrinsic::memset) {
     assert(MMOIt != MI.memoperands_end() && "Expected a second MMO on MI");
     MemOp = *(++MMOIt);
-    SrcAlign = Align(MemOp->getBaseAlignment());
+    SrcAlign = MemOp->getBaseAlign();
   }
 
   // See if this is a constant length copy
@@ -1514,11 +1516,45 @@ bool CombinerHelper::matchEqualDefs(const MachineOperand &MOP1,
   if (!I2)
     return false;
 
+  // Check for physical registers on the instructions first to avoid cases like
+  // this:
+  //
+  // %a = COPY $physreg
+  // ...
+  // SOMETHING implicit-def $physreg
+  // ...
+  // %b = COPY $physreg
+  //
+  // These copies are not equivalent.
+  if (any_of(I1->uses(), [](const MachineOperand &MO) {
+        return MO.isReg() && MO.getReg().isPhysical();
+      })) {
+    // Check if we have a case like this:
+    //
+    // %a = COPY $physreg
+    // %b = COPY %a
+    //
+    // In this case, I1 and I2 will both be equal to %a = COPY $physreg.
+    // From that, we know that they must have the same value, since they must
+    // have come from the same COPY.
+    return I1->isIdenticalTo(*I2);
+  }
+
+  // We don't have any physical registers, so we don't necessarily need the
+  // same vreg defs.
+  //
   // On the off-chance that there's some target instruction feeding into the
-  // select, let's use produceSameValue instead of isIdenticalTo.
+  // instruction, let's use produceSameValue instead of isIdenticalTo.
   return Builder.getTII().produceSameValue(*I1, *I2, &MRI);
 }
 
+bool CombinerHelper::matchConstantOp(const MachineOperand &MOP, int64_t C) {
+  if (!MOP.isReg())
+    return false;
+  int64_t Cst;
+  return mi_match(MOP.getReg(), MRI, m_ICst(Cst)) && Cst == C;
+}
+
 bool CombinerHelper::replaceSingleDefInstWithOperand(MachineInstr &MI,
                                                      unsigned OpIdx) {
   assert(MI.getNumExplicitDefs() == 1 && "Expected one explicit def?");
@@ -1538,6 +1574,12 @@ bool CombinerHelper::matchSelectSameVal(MachineInstr &MI) {
                        MRI);
 }
 
+bool CombinerHelper::matchBinOpSameVal(MachineInstr &MI) {
+  return matchEqualDefs(MI.getOperand(1), MI.getOperand(2)) &&
+         canReplaceReg(MI.getOperand(0).getReg(), MI.getOperand(1).getReg(),
+                       MRI);
+}
+
 bool CombinerHelper::replaceInstWithFConstant(MachineInstr &MI, double C) {
   assert(MI.getNumDefs() == 1 && "Expected only one def?");
   Builder.setInstr(MI);
diff --git a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
index 213af320531c8..75487075cb7a9 100644
--- a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
@@ -34,7 +34,7 @@ GISelKnownBits::GISelKnownBits(MachineFunction &MF, unsigned MaxDepth)
 Align GISelKnownBits::inferAlignmentForFrameIdx(int FrameIdx, int Offset,
                                                 const MachineFunction &MF) {
   const MachineFrameInfo &MFI = MF.getFrameInfo();
-  return commonAlignment(Align(MFI.getObjectAlignment(FrameIdx)), Offset);
+  return commonAlignment(MFI.getObjectAlign(FrameIdx), Offset);
   // TODO: How to handle cases with Base + Offset?
 }
 
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 567b2f9c50629..0ab9e7eaf47cd 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -242,37 +242,33 @@ int IRTranslator::getOrCreateFrameIndex(const AllocaInst &AI) {
   return FI;
 }
 
-unsigned IRTranslator::getMemOpAlignment(const Instruction &I) {
-  unsigned Alignment = 0;
-  Type *ValTy = nullptr;
+Align IRTranslator::getMemOpAlign(const Instruction &I) {
   if (const StoreInst *SI = dyn_cast<StoreInst>(&I)) {
-    Alignment = SI->getAlignment();
-    ValTy = SI->getValueOperand()->getType();
-  } else if (const LoadInst *LI = dyn_cast<LoadInst>(&I)) {
-    Alignment = LI->getAlignment();
-    ValTy = LI->getType();
-  } else if (const AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(&I)) {
+    Type *ValTy = SI->getValueOperand()->getType();
+    return SI->getAlign().getValueOr(DL->getABITypeAlign(ValTy));
+  }
+  if (const LoadInst *LI = dyn_cast<LoadInst>(&I)) {
+    Type *ValTy = LI->getType();
+    return LI->getAlign().getValueOr(DL->getABITypeAlign(ValTy));
+  }
+  if (const AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(&I)) {
     // TODO(PR27168): This instruction has no alignment attribute, but unlike
     // the default alignment for load/store, the default here is to assume
     // it has NATURAL alignment, not DataLayout-specified alignment.
     const DataLayout &DL = AI->getModule()->getDataLayout();
-    Alignment = DL.getTypeStoreSize(AI->getCompareOperand()->getType());
-    ValTy = AI->getCompareOperand()->getType();
-  } else if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(&I)) {
+    return Align(DL.getTypeStoreSize(AI->getCompareOperand()->getType()));
+  }
+  if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(&I)) {
     // TODO(PR27168): This instruction has no alignment attribute, but unlike
     // the default alignment for load/store, the default here is to assume
     // it has NATURAL alignment, not DataLayout-specified alignment.
     const DataLayout &DL = AI->getModule()->getDataLayout();
-    Alignment = DL.getTypeStoreSize(AI->getValOperand()->getType());
-    ValTy = AI->getType();
-  } else {
-    OptimizationRemarkMissed R("gisel-irtranslator", "", &I);
-    R << "unable to translate memop: " << ore::NV("Opcode", &I);
-    reportTranslationError(*MF, *TPC, *ORE, R);
-    return 1;
+    return Align(DL.getTypeStoreSize(AI->getValOperand()->getType()));
   }
-
-  return Alignment ? Alignment : DL->getABITypeAlignment(ValTy);
+  OptimizationRemarkMissed R("gisel-irtranslator", "", &I);
+  R << "unable to translate memop: " << ore::NV("Opcode", &I);
+  reportTranslationError(*MF, *TPC, *ORE, R);
+  return Align(1);
 }
 
 MachineBasicBlock &IRTranslator::getMBB(const BasicBlock &BB) {
@@ -887,12 +883,12 @@ bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
     MIRBuilder.materializePtrAdd(Addr, Base, OffsetTy, Offsets[i] / 8);
 
     MachinePointerInfo Ptr(LI.getPointerOperand(), Offsets[i] / 8);
-    unsigned BaseAlign = getMemOpAlignment(LI);
+    Align BaseAlign = getMemOpAlign(LI);
     AAMDNodes AAMetadata;
     LI.getAAMetadata(AAMetadata);
     auto MMO = MF->getMachineMemOperand(
-        Ptr, Flags, (MRI->getType(Regs[i]).getSizeInBits() + 7) / 8,
-        MinAlign(BaseAlign, Offsets[i] / 8), AAMetadata, Ranges,
+        Ptr, Flags, MRI->getType(Regs[i]).getSizeInBytes(),
+        commonAlignment(BaseAlign, Offsets[i] / 8), AAMetadata, Ranges,
         LI.getSyncScopeID(), LI.getOrdering());
     MIRBuilder.buildLoad(Regs[i], Addr, *MMO);
   }
@@ -929,12 +925,12 @@ bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
     MIRBuilder.materializePtrAdd(Addr, Base, OffsetTy, Offsets[i] / 8);
 
     MachinePointerInfo Ptr(SI.getPointerOperand(), Offsets[i] / 8);
-    unsigned BaseAlign = getMemOpAlignment(SI);
+    Align BaseAlign = getMemOpAlign(SI);
     AAMDNodes AAMetadata;
     SI.getAAMetadata(AAMetadata);
     auto MMO = MF->getMachineMemOperand(
-        Ptr, Flags, (MRI->getType(Vals[i]).getSizeInBits() + 7) / 8,
-        MinAlign(BaseAlign, Offsets[i] / 8), AAMetadata, nullptr,
+        Ptr, Flags, MRI->getType(Vals[i]).getSizeInBytes(),
+        commonAlignment(BaseAlign, Offsets[i] / 8), AAMetadata, nullptr,
         SI.getSyncScopeID(), SI.getOrdering());
     MIRBuilder.buildStore(Vals[i], Addr, *MMO);
   }
@@ -1149,20 +1145,21 @@ bool IRTranslator::translateMemFunc(const CallInst &CI,
   for (auto AI = CI.arg_begin(), AE = CI.arg_end(); std::next(AI) != AE; ++AI)
     ICall.addUse(getOrCreateVReg(**AI));
 
-  unsigned DstAlign = 0, SrcAlign = 0;
+  Align DstAlign;
+  Align SrcAlign;
   unsigned IsVol =
       cast<ConstantInt>(CI.getArgOperand(CI.getNumArgOperands() - 1))
           ->getZExtValue();
 
   if (auto *MCI = dyn_cast<MemCpyInst>(&CI)) {
-    DstAlign = std::max<unsigned>(MCI->getDestAlignment(), 1);
-    SrcAlign = std::max<unsigned>(MCI->getSourceAlignment(), 1);
+    DstAlign = MCI->getDestAlign().valueOrOne();
+    SrcAlign = MCI->getSourceAlign().valueOrOne();
   } else if (auto *MMI = dyn_cast<MemMoveInst>(&CI)) {
-    DstAlign = std::max<unsigned>(MMI->getDestAlignment(), 1);
-    SrcAlign = std::max<unsigned>(MMI->getSourceAlignment(), 1);
+    DstAlign = MMI->getDestAlign().valueOrOne();
+    SrcAlign = MMI->getSourceAlign().valueOrOne();
   } else {
     auto *MSI = cast<MemSetInst>(&CI);
-    DstAlign = std::max<unsigned>(MSI->getDestAlignment(), 1);
+    DstAlign = MSI->getDestAlign().valueOrOne();
   }
 
   // We need to propagate the tail call flag from the IR inst as an argument.
@@ -1200,7 +1197,7 @@ void IRTranslator::getStackGuard(Register DstReg,
                MachineMemOperand::MODereferenceable;
   MachineMemOperand *MemRef =
       MF->getMachineMemOperand(MPInfo, Flags, DL->getPointerSizeInBits() / 8,
-                               DL->getPointerABIAlignment(0).value());
+                               DL->getPointerABIAlignment(0));
   MIB.setMemRefs({MemRef});
 }
 
@@ -1220,8 +1217,12 @@ unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
       break;
     case Intrinsic::bswap:
       return TargetOpcode::G_BSWAP;
-  case Intrinsic::bitreverse:
+    case Intrinsic::bitreverse:
       return TargetOpcode::G_BITREVERSE;
+    case Intrinsic::fshl:
+      return TargetOpcode::G_FSHL;
+    case Intrinsic::fshr:
+      return TargetOpcode::G_FSHR;
     case Intrinsic::ceil:
       return TargetOpcode::G_FCEIL;
     case Intrinsic::cos:
@@ -1384,8 +1385,9 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
 
     // FIXME: Get alignment
     MIRBuilder.buildInstr(TargetOpcode::G_VASTART, {}, {getOrCreateVReg(*Ptr)})
-        .addMemOperand(MF->getMachineMemOperand(
-            MachinePointerInfo(Ptr), MachineMemOperand::MOStore, ListSize, 1));
+        .addMemOperand(MF->getMachineMemOperand(MachinePointerInfo(Ptr),
+                                                MachineMemOperand::MOStore,
+                                                ListSize, Align(1)));
     return true;
   }
   case Intrinsic::dbg_value: {
@@ -1489,7 +1491,7 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
         *MF->getMachineMemOperand(MachinePointerInfo::getFixedStack(*MF, FI),
                                   MachineMemOperand::MOStore |
                                       MachineMemOperand::MOVolatile,
-                                  PtrTy.getSizeInBits() / 8, 8));
+                                  PtrTy.getSizeInBits() / 8, Align(8)));
     return true;
   }
   case Intrinsic::stacksave: {
@@ -1711,14 +1713,12 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
   TargetLowering::IntrinsicInfo Info;
   // TODO: Add a GlobalISel version of getTgtMemIntrinsic.
   if (TLI.getTgtMemIntrinsic(Info, CI, *MF, ID)) {
-    MaybeAlign Align = Info.align;
-    if (!Align)
-      Align = MaybeAlign(
-          DL->getABITypeAlignment(Info.memVT.getTypeForEVT(F->getContext())));
+    Align Alignment = Info.align.getValueOr(
+        DL->getABITypeAlign(Info.memVT.getTypeForEVT(F->getContext())));
 
     uint64_t Size = Info.memVT.getStoreSize();
-    MIB.addMemOperand(MF->getMachineMemOperand(
-        MachinePointerInfo(Info.ptrVal), Info.flags, Size, Align->value()));
+    MIB.addMemOperand(MF->getMachineMemOperand(MachinePointerInfo(Info.ptrVal),
+                                               Info.flags, Size, Alignment));
   }
 
   return true;
@@ -1858,12 +1858,7 @@ bool IRTranslator::translateAlloca(const User &U,
     return false;
 
   // Now we're in the harder dynamic case.
-  Type *Ty = AI.getAllocatedType();
-  unsigned Align =
-      std::max((unsigned)DL->getPrefTypeAlignment(Ty), AI.getAlignment());
-
   Register NumElts = getOrCreateVReg(*AI.getArraySize());
-
   Type *IntPtrIRTy = DL->getIntPtrType(AI.getType());
   LLT IntPtrTy = getLLTForType(*IntPtrIRTy, *DL);
   if (MRI->getType(NumElts) != IntPtrTy) {
@@ -1872,29 +1867,30 @@ bool IRTranslator::translateAlloca(const User &U,
     NumElts = ExtElts;
   }
 
+  Type *Ty = AI.getAllocatedType();
+
   Register AllocSize = MRI->createGenericVirtualRegister(IntPtrTy);
   Register TySize =
       getOrCreateVReg(*ConstantInt::get(IntPtrIRTy, DL->getTypeAllocSize(Ty)));
   MIRBuilder.buildMul(AllocSize, NumElts, TySize);
 
-  unsigned StackAlign =
-      MF->getSubtarget().getFrameLowering()->getStackAlignment();
-  if (Align <= StackAlign)
-    Align = 0;
-
   // Round the size of the allocation up to the stack alignment size
   // by add SA-1 to the size. This doesn't overflow because we're computing
   // an address inside an alloca.
-  auto SAMinusOne = MIRBuilder.buildConstant(IntPtrTy, StackAlign - 1);
+  Align StackAlign = MF->getSubtarget().getFrameLowering()->getStackAlign();
+  auto SAMinusOne = MIRBuilder.buildConstant(IntPtrTy, StackAlign.value() - 1);
   auto AllocAdd = MIRBuilder.buildAdd(IntPtrTy, AllocSize, SAMinusOne,
                                       MachineInstr::NoUWrap);
   auto AlignCst =
-      MIRBuilder.buildConstant(IntPtrTy, ~(uint64_t)(StackAlign - 1));
+      MIRBuilder.buildConstant(IntPtrTy, ~(uint64_t)(StackAlign.value() - 1));
   auto AlignedAlloc = MIRBuilder.buildAnd(IntPtrTy, AllocAdd, AlignCst);
 
-  MIRBuilder.buildDynStackAlloc(getOrCreateVReg(AI), AlignedAlloc, Align);
+  Align Alignment = max(AI.getAlign(), DL->getPrefTypeAlign(Ty));
+  if (Alignment <= StackAlign)
+    Alignment = Align(1);
+  MIRBuilder.buildDynStackAlloc(getOrCreateVReg(AI), AlignedAlloc, Alignment);
 
-  MF->getFrameInfo().CreateVariableSizedObject(Align ? Align : 1, &AI);
+  MF->getFrameInfo().CreateVariableSizedObject(Alignment, &AI);
   assert(MF->getFrameInfo().hasVarSizedObjects());
   return true;
 }
@@ -1973,8 +1969,11 @@ bool IRTranslator::translateExtractElement(const User &U,
 
 bool IRTranslator::translateShuffleVector(const User &U,
                                           MachineIRBuilder &MIRBuilder) {
-  SmallVector<int, 8> Mask;
-  ShuffleVectorInst::getShuffleMask(cast<Constant>(U.getOperand(2)), Mask);
+  ArrayRef<int> Mask;
+  if (auto *SVI = dyn_cast<ShuffleVectorInst>(&U))
+    Mask = SVI->getShuffleMask();
+  else
+    Mask = cast<ConstantExpr>(U).getShuffleMask();
   ArrayRef<int> MaskAlloc = MF->allocateShuffleMask(Mask);
   MIRBuilder
       .buildInstr(TargetOpcode::G_SHUFFLE_VECTOR, {getOrCreateVReg(U)},
@@ -2022,11 +2021,10 @@ bool IRTranslator::translateAtomicCmpXchg(const User &U,
 
   MIRBuilder.buildAtomicCmpXchgWithSuccess(
       OldValRes, SuccessRes, Addr, Cmp, NewVal,
-      *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
-                                Flags, DL->getTypeStoreSize(ValType),
-                                getMemOpAlignment(I), AAMetadata, nullptr,
-                                I.getSyncScopeID(), I.getSuccessOrdering(),
-                                I.getFailureOrdering()));
+      *MF->getMachineMemOperand(
+          MachinePointerInfo(I.getPointerOperand()), Flags,
+          DL->getTypeStoreSize(ValType), getMemOpAlign(I), AAMetadata, nullptr,
+          I.getSyncScopeID(), I.getSuccessOrdering(), I.getFailureOrdering()));
   return true;
 }
 
@@ -2094,8 +2092,8 @@ bool IRTranslator::translateAtomicRMW(const User &U,
       Opcode, Res, Addr, Val,
       *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
                                 Flags, DL->getTypeStoreSize(ResType),
-                                getMemOpAlignment(I), AAMetadata,
-                                nullptr, I.getSyncScopeID(), I.getOrdering()));
+                                getMemOpAlign(I), AAMetadata, nullptr,
+                                I.getSyncScopeID(), I.getOrdering()));
   return true;
 }
 
@@ -2176,15 +2174,9 @@ bool IRTranslator::translate(const Constant &C, Register Reg) {
     EntryBuilder->buildFConstant(Reg, *CF);
   else if (isa<UndefValue>(C))
     EntryBuilder->buildUndef(Reg);
-  else if (isa<ConstantPointerNull>(C)) {
-    // As we are trying to build a constant val of 0 into a pointer,
-    // insert a cast to make them correct with respect to types.
-    unsigned NullSize = DL->getTypeSizeInBits(C.getType());
-    auto *ZeroTy = Type::getIntNTy(C.getContext(), NullSize);
-    auto *ZeroVal = ConstantInt::get(ZeroTy, 0);
-    Register ZeroReg = getOrCreateVReg(*ZeroVal);
-    EntryBuilder->buildCast(Reg, ZeroReg);
-  } else if (auto GV = dyn_cast<GlobalValue>(&C))
+  else if (isa<ConstantPointerNull>(C))
+    EntryBuilder->buildConstant(Reg, 0);
+  else if (auto GV = dyn_cast<GlobalValue>(&C))
     EntryBuilder->buildGlobalValue(Reg, GV);
   else if (auto CAZ = dyn_cast<ConstantAggregateZero>(&C)) {
     if (!CAZ->getType()->isVectorTy())
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index ac9b35aeb18c6..3c24933921b29 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -489,15 +489,14 @@ static bool isLibCallInTailPosition(MachineInstr &MI) {
 }
 
 LegalizerHelper::LegalizeResult
-llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
+llvm::createLibcall(MachineIRBuilder &MIRBuilder, const char *Name,
                     const CallLowering::ArgInfo &Result,
-                    ArrayRef<CallLowering::ArgInfo> Args) {
+                    ArrayRef<CallLowering::ArgInfo> Args,
+                    const CallingConv::ID CC) {
   auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
-  auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
-  const char *Name = TLI.getLibcallName(Libcall);
 
   CallLowering::CallLoweringInfo Info;
-  Info.CallConv = TLI.getLibcallCallingConv(Libcall);
+  Info.CallConv = CC;
   Info.Callee = MachineOperand::CreateES(Name);
   Info.OrigRet = Result;
   std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
@@ -507,6 +506,16 @@ llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
   return LegalizerHelper::Legalized;
 }
 
+LegalizerHelper::LegalizeResult
+llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
+                    const CallLowering::ArgInfo &Result,
+                    ArrayRef<CallLowering::ArgInfo> Args) {
+  auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
+  const char *Name = TLI.getLibcallName(Libcall);
+  const CallingConv::ID CC = TLI.getLibcallCallingConv(Libcall);
+  return createLibcall(MIRBuilder, Name, Result, Args, CC);
+}
+
 // Useful for libcalls where all operands have the same type.
 static LegalizerHelper::LegalizeResult
 simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size,
@@ -625,6 +634,7 @@ LegalizerHelper::libcall(MachineInstr &MI) {
   auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
 
   MIRBuilder.setInstr(MI);
+  MIRBuilder.setDebugLoc(MI.getDebugLoc());
 
   switch (MI.getOpcode()) {
   default:
@@ -722,6 +732,7 @@ LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
                                                               unsigned TypeIdx,
                                                               LLT NarrowTy) {
   MIRBuilder.setInstr(MI);
+  MIRBuilder.setDebugLoc(MI.getDebugLoc());
 
   uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
   uint64_t NarrowSize = NarrowTy.getSizeInBits();
@@ -1587,6 +1598,7 @@ LegalizerHelper::widenScalarInsert(MachineInstr &MI, unsigned TypeIdx,
 LegalizerHelper::LegalizeResult
 LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
   MIRBuilder.setInstr(MI);
+  MIRBuilder.setDebugLoc(MI.getDebugLoc());
 
   switch (MI.getOpcode()) {
   default:
@@ -2179,6 +2191,7 @@ LegalizerHelper::LegalizeResult
 LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
   using namespace TargetOpcode;
   MIRBuilder.setInstr(MI);
+  MIRBuilder.setDebugLoc(MI.getDebugLoc());
 
   switch(MI.getOpcode()) {
   default:
@@ -3214,6 +3227,7 @@ LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
   using namespace TargetOpcode;
 
   MIRBuilder.setInstr(MI);
+  MIRBuilder.setDebugLoc(MI.getDebugLoc());
   switch (MI.getOpcode()) {
   case G_IMPLICIT_DEF:
     return fewerElementsVectorImplicitDef(MI, TypeIdx, NarrowTy);
@@ -3267,6 +3281,8 @@ LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
   case G_FMAXNUM_IEEE:
   case G_FMINIMUM:
   case G_FMAXIMUM:
+  case G_FSHL:
+  case G_FSHR:
     return fewerElementsVectorBasic(MI, TypeIdx, NarrowTy);
   case G_SHL:
   case G_LSHR:
@@ -4875,7 +4891,7 @@ LegalizerHelper::LegalizeResult
 LegalizerHelper::lowerDynStackAlloc(MachineInstr &MI) {
   Register Dst = MI.getOperand(0).getReg();
   Register AllocSize = MI.getOperand(1).getReg();
-  unsigned Align = MI.getOperand(2).getImm();
+  Align Alignment = assumeAligned(MI.getOperand(2).getImm());
 
   const auto &MF = *MI.getMF();
   const auto &TLI = *MF.getSubtarget().getTargetLowering();
@@ -4891,8 +4907,8 @@ LegalizerHelper::lowerDynStackAlloc(MachineInstr &MI) {
   // have to generate an extra instruction to negate the alloc and then use
   // G_PTR_ADD to add the negative offset.
   auto Alloc = MIRBuilder.buildSub(IntPtrTy, SPTmp, AllocSize);
-  if (Align) {
-    APInt AlignMask(IntPtrTy.getSizeInBits(), Align, true);
+  if (Alignment > Align(1)) {
+    APInt AlignMask(IntPtrTy.getSizeInBits(), Alignment.value(), true);
     AlignMask.negate();
     auto AlignCst = MIRBuilder.buildConstant(IntPtrTy, AlignMask);
     Alloc = MIRBuilder.buildAnd(IntPtrTy, Alloc, AlignCst);
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp
index 2658434fe9491..5a8f8fd6a698a 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp
@@ -506,8 +506,7 @@ LegalizerInfo::getAction(const MachineInstr &MI,
   SmallVector<LegalityQuery::MemDesc, 2> MemDescrs;
   for (const auto &MMO : MI.memoperands())
     MemDescrs.push_back({8 * MMO->getSize() /* in bits */,
-                         8 * MMO->getAlignment(),
-                         MMO->getOrdering()});
+                         8 * MMO->getAlign().value(), MMO->getOrdering()});
 
   return getAction({MI.getOpcode(), Types, MemDescrs});
 }
diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
index e97be39ea70fe..834a7ada78932 100644
--- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
@@ -162,12 +162,12 @@ MachineInstrBuilder MachineIRBuilder::buildDbgLabel(const MDNode *Label) {
 
 MachineInstrBuilder MachineIRBuilder::buildDynStackAlloc(const DstOp &Res,
                                                          const SrcOp &Size,
-                                                         unsigned Align) {
+                                                         Align Alignment) {
   assert(Res.getLLTTy(*getMRI()).isPointer() && "expected ptr dst type");
   auto MIB = buildInstr(TargetOpcode::G_DYN_STACKALLOC);
   Res.addDefToMIB(*getMRI(), MIB);
   Size.addSrcToMIB(MIB);
-  MIB.addImm(Align);
+  MIB.addImm(Alignment.value());
   return MIB;
 }
 
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index 27b346d05dbd4..d248a2f1d9eba 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -457,15 +457,16 @@ bool llvm::isKnownNeverNaN(Register Val, const MachineRegisterInfo &MRI,
   return false;
 }
 
-unsigned llvm::inferAlignmentFromPtrInfo(MachineFunction &MF,
-                                         const MachinePointerInfo &MPO) {
+Align llvm::inferAlignFromPtrInfo(MachineFunction &MF,
+                                  const MachinePointerInfo &MPO) {
   auto PSV = MPO.V.dyn_cast<const PseudoSourceValue *>();
   if (auto FSPV = dyn_cast_or_null<FixedStackPseudoSourceValue>(PSV)) {
     MachineFrameInfo &MFI = MF.getFrameInfo();
-    return MinAlign(MFI.getObjectAlignment(FSPV->getFrameIndex()), MPO.Offset);
+    return commonAlignment(MFI.getObjectAlign(FSPV->getFrameIndex()),
+                           MPO.Offset);
   }
 
-  return 1;
+  return Align(1);
 }
 
 Optional<APInt> llvm::ConstantFoldExtOp(unsigned Opcode, const unsigned Op1,
diff --git a/llvm/lib/CodeGen/LiveDebugVariables.cpp b/llvm/lib/CodeGen/LiveDebugVariables.cpp
index 2b3e98f933284..a686d12c45a9b 100644
--- a/llvm/lib/CodeGen/LiveDebugVariables.cpp
+++ b/llvm/lib/CodeGen/LiveDebugVariables.cpp
@@ -154,9 +154,8 @@ class LDVImpl;
 /// holds part of a user variable. The part is identified by a byte offset.
 ///
 /// UserValues are grouped into equivalence classes for easier searching. Two
-/// user values are related if they refer to the same variable, or if they are
-/// held by the same virtual register. The equivalence class is the transitive
-/// closure of that relation.
+/// user values are related if they are held by the same virtual register. The
+/// equivalence class is the transitive closure of that relation.
 class UserValue {
   const DILocalVariable *Variable; ///< The debug info variable we are part of.
   /// The part of the variable we describe.
@@ -207,24 +206,6 @@ class UserValue {
   /// Return the next UserValue in the equivalence class.
   UserValue *getNext() const { return next; }
 
-  /// Does this UserValue match the parameters?
-  bool matches(const DILocalVariable *Var,
-             Optional<DIExpression::FragmentInfo> OtherFragment,
-             const DILocation *IA) const {
-    // FIXME: Handle partially overlapping fragments.
-    // A DBG_VALUE with a fragment which overlaps a previous DBG_VALUE fragment
-    // for the same variable terminates the interval opened by the first.
-    // getUserValue() uses matches() to filter DBG_VALUEs into interval maps to
-    // represent these intervals.
-    // Given two _partially_ overlapping fragments matches() will always return
-    // false. The DBG_VALUEs will be filtered into separate interval maps and
-    // therefore we do not faithfully represent the original intervals.
-    // See D70121#1849741 for a more detailed explanation and further
-    // discussion.
-    return Var == Variable && OtherFragment == Fragment &&
-           dl->getInlinedAt() == IA;
-  }
-
   /// Merge equivalence classes.
   static UserValue *merge(UserValue *L1, UserValue *L2) {
     L2 = L2->getLeader();
@@ -429,8 +410,8 @@ class LDVImpl {
   using VRMap = DenseMap<unsigned, UserValue *>;
   VRMap virtRegToEqClass;
 
-  /// Map user variable to eq class leader.
-  using UVMap = DenseMap<const DILocalVariable *, UserValue *>;
+  /// Map to find existing UserValue instances.
+  using UVMap = DenseMap<DebugVariable, UserValue *>;
   UVMap userVarMap;
 
   /// Find or create a UserValue.
@@ -600,19 +581,15 @@ void UserValue::mapVirtRegs(LDVImpl *LDV) {
 UserValue *LDVImpl::getUserValue(const DILocalVariable *Var,
                                  Optional<DIExpression::FragmentInfo> Fragment,
                                  const DebugLoc &DL) {
-  UserValue *&Leader = userVarMap[Var];
-  if (Leader) {
-    UserValue *UV = Leader->getLeader();
-    Leader = UV;
-    for (; UV; UV = UV->getNext())
-      if (UV->matches(Var, Fragment, DL->getInlinedAt()))
-        return UV;
+  // FIXME: Handle partially overlapping fragments. See
+  // https://reviews.llvm.org/D70121#1849741.
+  DebugVariable ID(Var, Fragment, DL->getInlinedAt());
+  UserValue *&UV = userVarMap[ID];
+  if (!UV) {
+    userValues.push_back(
+        std::make_unique<UserValue>(Var, Fragment, DL, allocator));
+    UV = userValues.back().get();
   }
-
-  userValues.push_back(
-      std::make_unique<UserValue>(Var, Fragment, DL, allocator));
-  UserValue *UV = userValues.back().get();
-  Leader = UserValue::merge(Leader, UV);
   return UV;
 }
 
diff --git a/llvm/lib/CodeGen/LocalStackSlotAllocation.cpp b/llvm/lib/CodeGen/LocalStackSlotAllocation.cpp
index 5022726dc70aa..6c5ef0255a082 100644
--- a/llvm/lib/CodeGen/LocalStackSlotAllocation.cpp
+++ b/llvm/lib/CodeGen/LocalStackSlotAllocation.cpp
@@ -79,11 +79,11 @@ namespace {
     using StackObjSet = SmallSetVector<int, 8>;
 
     void AdjustStackOffset(MachineFrameInfo &MFI, int FrameIdx, int64_t &Offset,
-                           bool StackGrowsDown, unsigned &MaxAlign);
+                           bool StackGrowsDown, Align &MaxAlign);
     void AssignProtectedObjSet(const StackObjSet &UnassignedObjs,
                                SmallSet<int, 16> &ProtectedObjs,
                                MachineFrameInfo &MFI, bool StackGrowsDown,
-                               int64_t &Offset, unsigned &MaxAlign);
+                               int64_t &Offset, Align &MaxAlign);
     void calculateFrameObjectOffsets(MachineFunction &Fn);
     bool insertFrameReferenceRegisters(MachineFunction &Fn);
 
@@ -140,22 +140,21 @@ bool LocalStackSlotPass::runOnMachineFunction(MachineFunction &MF) {
 }
 
 /// AdjustStackOffset - Helper function used to adjust the stack frame offset.
-void LocalStackSlotPass::AdjustStackOffset(MachineFrameInfo &MFI,
-                                           int FrameIdx, int64_t &Offset,
-                                           bool StackGrowsDown,
-                                           unsigned &MaxAlign) {
+void LocalStackSlotPass::AdjustStackOffset(MachineFrameInfo &MFI, int FrameIdx,
+                                           int64_t &Offset, bool StackGrowsDown,
+                                           Align &MaxAlign) {
   // If the stack grows down, add the object size to find the lowest address.
   if (StackGrowsDown)
     Offset += MFI.getObjectSize(FrameIdx);
 
-  unsigned Align = MFI.getObjectAlignment(FrameIdx);
+  Align Alignment = MFI.getObjectAlign(FrameIdx);
 
   // If the alignment of this object is greater than that of the stack, then
   // increase the stack alignment to match.
-  MaxAlign = std::max(MaxAlign, Align);
+  MaxAlign = std::max(MaxAlign, Alignment);
 
   // Adjust to alignment boundary.
-  Offset = (Offset + Align - 1) / Align * Align;
+  Offset = alignTo(Offset, Alignment);
 
   int64_t LocalOffset = StackGrowsDown ? -Offset : Offset;
   LLVM_DEBUG(dbgs() << "Allocate FI(" << FrameIdx << ") to local offset "
@@ -173,11 +172,10 @@ void LocalStackSlotPass::AdjustStackOffset(MachineFrameInfo &MFI,
 
 /// AssignProtectedObjSet - Helper function to assign large stack objects (i.e.,
 /// those required to be close to the Stack Protector) to stack offsets.
-void LocalStackSlotPass::AssignProtectedObjSet(const StackObjSet &UnassignedObjs,
-                                           SmallSet<int, 16> &ProtectedObjs,
-                                           MachineFrameInfo &MFI,
-                                           bool StackGrowsDown, int64_t &Offset,
-                                           unsigned &MaxAlign) {
+void LocalStackSlotPass::AssignProtectedObjSet(
+    const StackObjSet &UnassignedObjs, SmallSet<int, 16> &ProtectedObjs,
+    MachineFrameInfo &MFI, bool StackGrowsDown, int64_t &Offset,
+    Align &MaxAlign) {
   for (StackObjSet::const_iterator I = UnassignedObjs.begin(),
         E = UnassignedObjs.end(); I != E; ++I) {
     int i = *I;
@@ -195,7 +193,7 @@ void LocalStackSlotPass::calculateFrameObjectOffsets(MachineFunction &Fn) {
   bool StackGrowsDown =
     TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown;
   int64_t Offset = 0;
-  unsigned MaxAlign = 0;
+  Align MaxAlign;
 
   // Make sure that the stack protector comes before the local variables on the
   // stack.
@@ -262,7 +260,7 @@ void LocalStackSlotPass::calculateFrameObjectOffsets(MachineFunction &Fn) {
 
   // Remember how big this blob of stack space is
   MFI.setLocalFrameSize(Offset);
-  MFI.setLocalFrameMaxAlign(assumeAligned(MaxAlign));
+  MFI.setLocalFrameMaxAlign(MaxAlign);
 }
 
 static inline bool
diff --git a/llvm/lib/CodeGen/MIRParser/MIParser.cpp b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
index 689e5afcf08e8..c948fbaac6038 100644
--- a/llvm/lib/CodeGen/MIRParser/MIParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
@@ -3095,8 +3095,8 @@ bool MIParser::parseMachineMemoryOperand(MachineMemOperand *&Dest) {
   }
   if (expectAndConsume(MIToken::rparen))
     return true;
-  Dest = MF.getMachineMemOperand(Ptr, Flags, Size, BaseAlignment, AAInfo, Range,
-                                 SSID, Order, FailureOrder);
+  Dest = MF.getMachineMemOperand(Ptr, Flags, Size, Align(BaseAlignment), AAInfo,
+                                 Range, SSID, Order, FailureOrder);
   return false;
 }
 
diff --git a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
index cad0a8d0899a4..135b2f2234afc 100644
--- a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
@@ -401,8 +401,7 @@ MIRParserImpl::initializeMachineFunction(const yaml::MachineFunction &YamlMF,
     Target.reset(new PerTargetMIParsingState(MF.getSubtarget()));
   }
 
-  if (YamlMF.Alignment)
-    MF.setAlignment(Align(YamlMF.Alignment));
+  MF.setAlignment(YamlMF.Alignment.valueOrOne());
   MF.setExposesReturnsTwice(YamlMF.ExposesReturnsTwice);
   MF.setHasWinCFI(YamlMF.HasWinCFI);
 
@@ -691,7 +690,7 @@ bool MIRParserImpl::initializeFrameInfo(PerFunctionMIParsingState &PFS,
       return error(Object.ID.SourceRange.Start,
                    Twine("StackID is not supported by target"));
     MFI.setStackID(ObjectIdx, Object.StackID);
-    MFI.setObjectAlignment(ObjectIdx, Object.Alignment);
+    MFI.setObjectAlignment(ObjectIdx, Object.Alignment.valueOrOne());
     if (!PFS.FixedStackObjectSlots.insert(std::make_pair(Object.ID.Value,
                                                          ObjectIdx))
              .second)
@@ -723,10 +722,11 @@ bool MIRParserImpl::initializeFrameInfo(PerFunctionMIParsingState &PFS,
       return error(Object.ID.SourceRange.Start,
                    Twine("StackID is not supported by target"));
     if (Object.Type == yaml::MachineStackObject::VariableSized)
-      ObjectIdx = MFI.CreateVariableSizedObject(Object.Alignment, Alloca);
+      ObjectIdx =
+          MFI.CreateVariableSizedObject(Object.Alignment.valueOrOne(), Alloca);
     else
       ObjectIdx = MFI.CreateStackObject(
-          Object.Size, Object.Alignment,
+          Object.Size, Object.Alignment.valueOrOne(),
           Object.Type == yaml::MachineStackObject::SpillSlot, Alloca,
           Object.StackID);
     MFI.setObjectOffset(ObjectIdx, Object.Offset);
@@ -838,11 +838,11 @@ bool MIRParserImpl::initializeConstantPool(PerFunctionMIParsingState &PFS,
         parseConstantValue(YamlConstant.Value.Value, Error, M));
     if (!Value)
       return error(Error, YamlConstant.Value.SourceRange);
-    unsigned Alignment =
-        YamlConstant.Alignment
-            ? YamlConstant.Alignment
-            : M.getDataLayout().getPrefTypeAlignment(Value->getType());
-    unsigned Index = ConstantPool.getConstantPoolIndex(Value, Alignment);
+    const Align PrefTypeAlign =
+        M.getDataLayout().getPrefTypeAlign(Value->getType());
+    const Align Alignment = YamlConstant.Alignment.getValueOr(PrefTypeAlign);
+    unsigned Index =
+        ConstantPool.getConstantPoolIndex(Value, Alignment.value());
     if (!ConstantPoolSlots.insert(std::make_pair(YamlConstant.ID.Value, Index))
              .second)
       return error(YamlConstant.ID.SourceRange.Start,
diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp
index 22f7e1644a48e..58eb720ca7999 100644
--- a/llvm/lib/CodeGen/MIRPrinter.cpp
+++ b/llvm/lib/CodeGen/MIRPrinter.cpp
@@ -198,7 +198,7 @@ void MIRPrinter::print(const MachineFunction &MF) {
 
   yaml::MachineFunction YamlMF;
   YamlMF.Name = MF.getName();
-  YamlMF.Alignment = MF.getAlignment().value();
+  YamlMF.Alignment = MF.getAlignment();
   YamlMF.ExposesReturnsTwice = MF.exposesReturnsTwice();
   YamlMF.HasWinCFI = MF.hasWinCFI();
 
@@ -373,7 +373,7 @@ void MIRPrinter::convertStackObjects(yaml::MachineFunction &YMF,
                           : yaml::FixedMachineStackObject::DefaultType;
     YamlObject.Offset = MFI.getObjectOffset(I);
     YamlObject.Size = MFI.getObjectSize(I);
-    YamlObject.Alignment = MFI.getObjectAlignment(I);
+    YamlObject.Alignment = MFI.getObjectAlign(I);
     YamlObject.StackID = (TargetStackID::Value)MFI.getStackID(I);
     YamlObject.IsImmutable = MFI.isImmutableObjectIndex(I);
     YamlObject.IsAliased = MFI.isAliasedObjectIndex(I);
@@ -400,7 +400,7 @@ void MIRPrinter::convertStackObjects(yaml::MachineFunction &YMF,
                                 : yaml::MachineStackObject::DefaultType;
     YamlObject.Offset = MFI.getObjectOffset(I);
     YamlObject.Size = MFI.getObjectSize(I);
-    YamlObject.Alignment = MFI.getObjectAlignment(I);
+    YamlObject.Alignment = MFI.getObjectAlign(I);
     YamlObject.StackID = (TargetStackID::Value)MFI.getStackID(I);
 
     YMF.StackObjects.push_back(YamlObject);
@@ -514,7 +514,7 @@ void MIRPrinter::convert(yaml::MachineFunction &MF,
     yaml::MachineConstantPoolValue YamlConstant;
     YamlConstant.ID = ID++;
     YamlConstant.Value = StrOS.str();
-    YamlConstant.Alignment = Constant.getAlignment();
+    YamlConstant.Alignment = MaybeAlign(Constant.getAlignment());
     YamlConstant.IsTargetSpecific = Constant.isMachineConstantPoolEntry();
 
     MF.Constants.push_back(YamlConstant);
diff --git a/llvm/lib/CodeGen/MIRVRegNamerUtils.cpp b/llvm/lib/CodeGen/MIRVRegNamerUtils.cpp
index 47e712aaccf38..004abc8a6ab07 100644
--- a/llvm/lib/CodeGen/MIRVRegNamerUtils.cpp
+++ b/llvm/lib/CodeGen/MIRVRegNamerUtils.cpp
@@ -112,7 +112,7 @@ std::string VRegRenamer::getInstructionOpcodeHash(MachineInstr &MI) {
     MIOperands.push_back((unsigned)Op->getOrdering());
     MIOperands.push_back((unsigned)Op->getAddrSpace());
     MIOperands.push_back((unsigned)Op->getSyncScopeID());
-    MIOperands.push_back((unsigned)Op->getBaseAlignment());
+    MIOperands.push_back((unsigned)Op->getBaseAlign().value());
     MIOperands.push_back((unsigned)Op->getFailureOrdering());
   }
 
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index 281d1bc15bfac..22dbc7f77d68b 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -471,7 +471,7 @@ MachineFunction::DeleteMachineBasicBlock(MachineBasicBlock *MBB) {
 
 MachineMemOperand *MachineFunction::getMachineMemOperand(
     MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s,
-    unsigned base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
+    Align base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
     SyncScope::ID SSID, AtomicOrdering Ordering,
     AtomicOrdering FailureOrdering) {
   return new (Allocator)
@@ -486,13 +486,13 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
 
   // If there is no pointer value, the offset isn't tracked so we need to adjust
   // the base alignment.
-  unsigned Align = PtrInfo.V.isNull()
-                       ? MinAlign(MMO->getBaseAlignment(), Offset)
-                       : MMO->getBaseAlignment();
+  Align Alignment = PtrInfo.V.isNull()
+                        ? commonAlignment(MMO->getBaseAlign(), Offset)
+                        : MMO->getBaseAlign();
 
   return new (Allocator)
       MachineMemOperand(PtrInfo.getWithOffset(Offset), MMO->getFlags(), Size,
-                        Align, AAMDNodes(), nullptr, MMO->getSyncScopeID(),
+                        Alignment, AAMDNodes(), nullptr, MMO->getSyncScopeID(),
                         MMO->getOrdering(), MMO->getFailureOrdering());
 }
 
@@ -503,18 +503,17 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
              MachinePointerInfo(MMO->getValue(), MMO->getOffset()) :
              MachinePointerInfo(MMO->getPseudoValue(), MMO->getOffset());
 
-  return new (Allocator)
-             MachineMemOperand(MPI, MMO->getFlags(), MMO->getSize(),
-                               MMO->getBaseAlignment(), AAInfo,
-                               MMO->getRanges(), MMO->getSyncScopeID(),
-                               MMO->getOrdering(), MMO->getFailureOrdering());
+  return new (Allocator) MachineMemOperand(
+      MPI, MMO->getFlags(), MMO->getSize(), MMO->getBaseAlign(), AAInfo,
+      MMO->getRanges(), MMO->getSyncScopeID(), MMO->getOrdering(),
+      MMO->getFailureOrdering());
 }
 
 MachineMemOperand *
 MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
                                       MachineMemOperand::Flags Flags) {
   return new (Allocator) MachineMemOperand(
-      MMO->getPointerInfo(), Flags, MMO->getSize(), MMO->getBaseAlignment(),
+      MMO->getPointerInfo(), Flags, MMO->getSize(), MMO->getBaseAlign(),
       MMO->getAAInfo(), MMO->getRanges(), MMO->getSyncScopeID(),
       MMO->getOrdering(), MMO->getFailureOrdering());
 }
diff --git a/llvm/lib/CodeGen/MachineModuleInfo.cpp b/llvm/lib/CodeGen/MachineModuleInfo.cpp
index ce5961aba9a72..a55943aff14be 100644
--- a/llvm/lib/CodeGen/MachineModuleInfo.cpp
+++ b/llvm/lib/CodeGen/MachineModuleInfo.cpp
@@ -105,15 +105,6 @@ ArrayRef<MCSymbol *> MMIAddrLabelMap::getAddrLabelSymbolToEmit(BasicBlock *BB) {
   Entry.Index = BBCallbacks.size() - 1;
   Entry.Fn = BB->getParent();
   MCSymbol *Sym = Context.createTempSymbol(!BB->hasAddressTaken());
-  if (Context.getObjectFileInfo()->getTargetTriple().isOSBinFormatXCOFF()) {
-    MCSymbol *FnEntryPointSym =
-        Context.lookupSymbol("." + Entry.Fn->getName());
-    assert(FnEntryPointSym && "The function entry pointer symbol should have"
-		              " already been initialized.");
-    MCSectionXCOFF *Csect =
-        cast<MCSymbolXCOFF>(FnEntryPointSym)->getContainingCsect();
-    cast<MCSymbolXCOFF>(Sym)->setContainingCsect(Csect);
-  }
   Entry.Symbols.push_back(Sym);
   return Entry.Symbols;
 }
diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp
index c7cbd9e51aa5c..5e4d5edb9ce68 100644
--- a/llvm/lib/CodeGen/MachineOperand.cpp
+++ b/llvm/lib/CodeGen/MachineOperand.cpp
@@ -1004,17 +1004,16 @@ MachinePointerInfo MachinePointerInfo::getUnknownStack(MachineFunction &MF) {
 }
 
 MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
-                                     uint64_t s, uint64_t a,
+                                     uint64_t s, Align a,
                                      const AAMDNodes &AAInfo,
                                      const MDNode *Ranges, SyncScope::ID SSID,
                                      AtomicOrdering Ordering,
                                      AtomicOrdering FailureOrdering)
-    : PtrInfo(ptrinfo), Size(s), FlagVals(f), BaseAlignLog2(Log2_32(a) + 1),
-      AAInfo(AAInfo), Ranges(Ranges) {
+    : PtrInfo(ptrinfo), Size(s), FlagVals(f), BaseAlign(a), AAInfo(AAInfo),
+      Ranges(Ranges) {
   assert((PtrInfo.V.isNull() || PtrInfo.V.is<const PseudoSourceValue *>() ||
           isa<PointerType>(PtrInfo.V.get<const Value *>()->getType())) &&
          "invalid pointer value");
-  assert(getBaseAlignment() == a && a != 0 && "Alignment is not a power of 2!");
   assert((isLoad() || isStore()) && "Not a load/store!");
 
   AtomicInfo.SSID = static_cast<unsigned>(SSID);
@@ -1032,7 +1031,7 @@ void MachineMemOperand::Profile(FoldingSetNodeID &ID) const {
   ID.AddInteger(Size);
   ID.AddPointer(getOpaqueValue());
   ID.AddInteger(getFlags());
-  ID.AddInteger(getBaseAlignment());
+  ID.AddInteger(getBaseAlign().value());
 }
 
 void MachineMemOperand::refineAlignment(const MachineMemOperand *MMO) {
@@ -1041,9 +1040,9 @@ void MachineMemOperand::refineAlignment(const MachineMemOperand *MMO) {
   assert(MMO->getFlags() == getFlags() && "Flags mismatch!");
   assert(MMO->getSize() == getSize() && "Size mismatch!");
 
-  if (MMO->getBaseAlignment() >= getBaseAlignment()) {
+  if (MMO->getBaseAlign() >= getBaseAlign()) {
     // Update the alignment value.
-    BaseAlignLog2 = Log2_32(MMO->getBaseAlignment()) + 1;
+    BaseAlign = MMO->getBaseAlign();
     // Also update the base and offset, because the new alignment may
     // not be applicable with the old ones.
     PtrInfo = MMO->PtrInfo;
@@ -1052,8 +1051,12 @@ void MachineMemOperand::refineAlignment(const MachineMemOperand *MMO) {
 
 /// getAlignment - Return the minimum known alignment in bytes of the
 /// actual memory reference.
-uint64_t MachineMemOperand::getAlignment() const {
-  return MinAlign(getBaseAlignment(), getOffset());
+uint64_t MachineMemOperand::getAlignment() const { return getAlign().value(); }
+
+/// getAlign - Return the minimum known alignment in bytes of the
+/// actual memory reference.
+Align MachineMemOperand::getAlign() const {
+  return commonAlignment(getBaseAlign(), getOffset());
 }
 
 void MachineMemOperand::print(raw_ostream &OS, ModuleSlotTracker &MST,
@@ -1148,8 +1151,8 @@ void MachineMemOperand::print(raw_ostream &OS, ModuleSlotTracker &MST,
     }
   }
   MachineOperand::printOperandOffset(OS, getOffset());
-  if (getBaseAlignment() != getSize())
-    OS << ", align " << getBaseAlignment();
+  if (getBaseAlign() != getSize())
+    OS << ", align " << getBaseAlign().value();
   auto AAInfo = getAAInfo();
   if (AAInfo.TBAA) {
     OS << ", !tbaa ";
diff --git a/llvm/lib/CodeGen/PeepholeOptimizer.cpp b/llvm/lib/CodeGen/PeepholeOptimizer.cpp
index 919ad890a6ce1..ae309a30e154a 100644
--- a/llvm/lib/CodeGen/PeepholeOptimizer.cpp
+++ b/llvm/lib/CodeGen/PeepholeOptimizer.cpp
@@ -457,12 +457,12 @@ INITIALIZE_PASS_END(PeepholeOptimizer, DEBUG_TYPE,
 bool PeepholeOptimizer::
 optimizeExtInstr(MachineInstr &MI, MachineBasicBlock &MBB,
                  SmallPtrSetImpl<MachineInstr*> &LocalMIs) {
-  unsigned SrcReg, DstReg, SubIdx;
+  Register SrcReg, DstReg;
+  unsigned SubIdx;
   if (!TII->isCoalescableExtInstr(MI, SrcReg, DstReg, SubIdx))
     return false;
 
-  if (Register::isPhysicalRegister(DstReg) ||
-      Register::isPhysicalRegister(SrcReg))
+  if (DstReg.isPhysical() || SrcReg.isPhysical())
     return false;
 
   if (MRI->hasOneNonDBGUse(SrcReg))
@@ -607,11 +607,10 @@ optimizeExtInstr(MachineInstr &MI, MachineBasicBlock &MBB,
 bool PeepholeOptimizer::optimizeCmpInstr(MachineInstr &MI) {
   // If this instruction is a comparison against zero and isn't comparing a
   // physical register, we can try to optimize it.
-  unsigned SrcReg, SrcReg2;
+  Register SrcReg, SrcReg2;
   int CmpMask, CmpValue;
   if (!TII->analyzeCompare(MI, SrcReg, SrcReg2, CmpMask, CmpValue) ||
-      Register::isPhysicalRegister(SrcReg) ||
-      (SrcReg2 != 0 && Register::isPhysicalRegister(SrcReg2)))
+      SrcReg.isPhysical() || SrcReg2.isPhysical())
     return false;
 
   // Attempt to optimize the comparison instruction.
@@ -663,8 +662,8 @@ bool PeepholeOptimizer::findNextSource(RegSubRegPair RegSubReg,
   // So far we do not have any motivating example for doing that.
   // Thus, instead of maintaining untested code, we will revisit that if
   // that changes at some point.
-  unsigned Reg = RegSubReg.Reg;
-  if (Register::isPhysicalRegister(Reg))
+  Register Reg = RegSubReg.Reg;
+  if (Reg.isPhysical())
     return false;
   const TargetRegisterClass *DefRC = MRI->getRegClass(Reg);
 
diff --git a/llvm/lib/CodeGen/RegAllocFast.cpp b/llvm/lib/CodeGen/RegAllocFast.cpp
index 89b5bcebd61ce..46a1311a206c0 100644
--- a/llvm/lib/CodeGen/RegAllocFast.cpp
+++ b/llvm/lib/CodeGen/RegAllocFast.cpp
@@ -255,8 +255,8 @@ int RegAllocFast::getStackSpaceFor(Register VirtReg) {
   // Allocate a new stack object for this spill location...
   const TargetRegisterClass &RC = *MRI->getRegClass(VirtReg);
   unsigned Size = TRI->getSpillSize(RC);
-  unsigned Align = TRI->getSpillAlignment(RC);
-  int FrameIdx = MFI->CreateSpillStackObject(Size, Align);
+  Align Alignment = TRI->getSpillAlign(RC);
+  int FrameIdx = MFI->CreateSpillStackObject(Size, Alignment);
 
   // Assign the slot.
   StackSlotForVirtReg[VirtReg] = FrameIdx;
diff --git a/llvm/lib/CodeGen/RegisterScavenging.cpp b/llvm/lib/CodeGen/RegisterScavenging.cpp
index a5bea1463468a..41b6de1441d75 100644
--- a/llvm/lib/CodeGen/RegisterScavenging.cpp
+++ b/llvm/lib/CodeGen/RegisterScavenging.cpp
@@ -466,7 +466,7 @@ RegScavenger::spill(Register Reg, const TargetRegisterClass &RC, int SPAdj,
   const MachineFunction &MF = *Before->getMF();
   const MachineFrameInfo &MFI = MF.getFrameInfo();
   unsigned NeedSize = TRI->getSpillSize(RC);
-  unsigned NeedAlign = TRI->getSpillAlignment(RC);
+  Align NeedAlign = TRI->getSpillAlign(RC);
 
   unsigned SI = Scavenged.size(), Diff = std::numeric_limits<unsigned>::max();
   int FIB = MFI.getObjectIndexBegin(), FIE = MFI.getObjectIndexEnd();
@@ -478,7 +478,7 @@ RegScavenger::spill(Register Reg, const TargetRegisterClass &RC, int SPAdj,
     if (FI < FIB || FI >= FIE)
       continue;
     unsigned S = MFI.getObjectSize(FI);
-    unsigned A = MFI.getObjectAlignment(FI);
+    Align A = MFI.getObjectAlign(FI);
     if (NeedSize > S || NeedAlign > A)
       continue;
     // Avoid wasting slots with large size and/or large alignment. Pick one
@@ -487,7 +487,7 @@ RegScavenger::spill(Register Reg, const TargetRegisterClass &RC, int SPAdj,
     // larger register is reserved before a slot for a smaller one. When
     // trying to spill a smaller register, the large slot would be found
     // first, thus making it impossible to spill the larger register later.
-    unsigned D = (S-NeedSize) + (A-NeedAlign);
+    unsigned D = (S - NeedSize) + (A.value() - NeedAlign.value());
     if (D < Diff) {
       SI = I;
       Diff = D;
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 574a80fcc3468..77fa5c793c8eb 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -436,6 +436,7 @@ namespace {
     SDValue visitZERO_EXTEND_VECTOR_INREG(SDNode *N);
     SDValue visitTRUNCATE(SDNode *N);
     SDValue visitBITCAST(SDNode *N);
+    SDValue visitFREEZE(SDNode *N);
     SDValue visitBUILD_PAIR(SDNode *N);
     SDValue visitFADD(SDNode *N);
     SDValue visitFSUB(SDNode *N);
@@ -902,6 +903,13 @@ static bool isAnyConstantBuildVector(SDValue V, bool NoOpaques = false) {
          ISD::isBuildVectorOfConstantFPSDNodes(V.getNode());
 }
 
+// Determine if this an indexed load with an opaque target constant index.
+static bool canSplitIdx(LoadSDNode *LD) {
+  return MaySplitLoadIndex &&
+         (LD->getOperand(2).getOpcode() != ISD::TargetConstant ||
+          !cast<ConstantSDNode>(LD->getOperand(2))->isOpaque());
+}
+
 bool DAGCombiner::reassociationCanBreakAddressingModePattern(unsigned Opc,
                                                              const SDLoc &DL,
                                                              SDValue N0,
@@ -967,10 +975,6 @@ SDValue DAGCombiner::reassociateOpsCommutative(unsigned Opc, const SDLoc &DL,
   if (N0.getOpcode() != Opc)
     return SDValue();
 
-  // Don't reassociate reductions.
-  if (N0->getFlags().hasVectorReduction())
-    return SDValue();
-
   if (DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1))) {
     if (DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
       // Reassociate: (op (op x, c1), c2) -> (op x, (op c1, c2))
@@ -995,9 +999,6 @@ SDValue DAGCombiner::reassociateOpsCommutative(unsigned Opc, const SDLoc &DL,
 SDValue DAGCombiner::reassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
                                     SDValue N1, SDNodeFlags Flags) {
   assert(TLI.isCommutativeBinOp(Opc) && "Operation not commutative.");
-  // Don't reassociate reductions.
-  if (Flags.hasVectorReduction())
-    return SDValue();
 
   // Floating-point reassociation is not allowed without loose FP math.
   if (N0.getValueType().isFloatingPoint() ||
@@ -1629,6 +1630,7 @@ SDValue DAGCombiner::visit(SDNode *N) {
   case ISD::LIFETIME_END:       return visitLIFETIME_END(N);
   case ISD::FP_TO_FP16:         return visitFP_TO_FP16(N);
   case ISD::FP16_TO_FP:         return visitFP16_TO_FP(N);
+  case ISD::FREEZE:             return visitFREEZE(N);
   case ISD::VECREDUCE_FADD:
   case ISD::VECREDUCE_FMUL:
   case ISD::VECREDUCE_ADD:
@@ -11583,6 +11585,20 @@ SDValue DAGCombiner::visitBUILD_PAIR(SDNode *N) {
   return CombineConsecutiveLoads(N, VT);
 }
 
+SDValue DAGCombiner::visitFREEZE(SDNode *N) {
+  SDValue N0 = N->getOperand(0);
+
+  // (freeze (freeze x)) -> (freeze x)
+  if (N0.getOpcode() == ISD::FREEZE)
+    return N0;
+
+  // If the input is a constant, return it.
+  if (isa<ConstantSDNode>(N0) || isa<ConstantFPSDNode>(N0))
+    return N0;
+
+  return SDValue();
+}
+
 /// We know that BV is a build_vector node with Constant, ConstantFP or Undef
 /// operands. DstEltVT indicates the destination element value type.
 SDValue DAGCombiner::
@@ -13093,8 +13109,12 @@ SDValue DAGCombiner::visitFREM(SDNode *N) {
 
 SDValue DAGCombiner::visitFSQRT(SDNode *N) {
   SDNodeFlags Flags = N->getFlags();
-  if (!DAG.getTarget().Options.UnsafeFPMath &&
-      !Flags.hasApproximateFuncs())
+  const TargetOptions &Options = DAG.getTarget().Options;
+
+  // Require 'ninf' flag since sqrt(+Inf) = +Inf, but the estimation goes as:
+  // sqrt(+Inf) == rsqrt(+Inf) * +Inf = 0 * +Inf = NaN
+  if ((!Options.UnsafeFPMath && !Flags.hasApproximateFuncs()) ||
+      (!Options.NoInfsFPMath && !Flags.hasNoInfs()))
     return SDValue();
 
   SDValue N0 = N->getOperand(0);
@@ -14481,11 +14501,11 @@ SDValue DAGCombiner::ForwardStoreValueToDirectLoad(LoadSDNode *LD) {
 
   auto ReplaceLd = [&](LoadSDNode *LD, SDValue Val, SDValue Chain) -> SDValue {
     if (LD->isIndexed()) {
-      bool IsSub = (LD->getAddressingMode() == ISD::PRE_DEC ||
-                    LD->getAddressingMode() == ISD::POST_DEC);
-      unsigned Opc = IsSub ? ISD::SUB : ISD::ADD;
-      SDValue Idx = DAG.getNode(Opc, SDLoc(LD), LD->getOperand(1).getValueType(),
-                             LD->getOperand(1), LD->getOperand(2));
+      // Cannot handle opaque target constants and we must respect the user's
+      // request not to split indexes from loads.
+      if (!canSplitIdx(LD))
+        return SDValue();
+      SDValue Idx = SplitIndexingFromLoad(LD);
       SDValue Ops[] = {Val, Idx, Chain};
       return CombineTo(LD, Ops, 3);
     }
@@ -14581,14 +14601,12 @@ SDValue DAGCombiner::visitLOAD(SDNode *N) {
       // the indexing into an add/sub directly (that TargetConstant may not be
       // valid for a different type of node, and we cannot convert an opaque
       // target constant into a regular constant).
-      bool HasOTCInc = LD->getOperand(2).getOpcode() == ISD::TargetConstant &&
-                       cast<ConstantSDNode>(LD->getOperand(2))->isOpaque();
+      bool CanSplitIdx = canSplitIdx(LD);
 
-      if (!N->hasAnyUseOfValue(0) &&
-          ((MaySplitLoadIndex && !HasOTCInc) || !N->hasAnyUseOfValue(1))) {
+      if (!N->hasAnyUseOfValue(0) && (CanSplitIdx || !N->hasAnyUseOfValue(1))) {
         SDValue Undef = DAG.getUNDEF(N->getValueType(0));
         SDValue Index;
-        if (N->hasAnyUseOfValue(1) && MaySplitLoadIndex && !HasOTCInc) {
+        if (N->hasAnyUseOfValue(1) && CanSplitIdx) {
           Index = SplitIndexingFromLoad(LD);
           // Try to fold the base pointer arithmetic into subsequent loads and
           // stores.
@@ -14615,11 +14633,12 @@ SDValue DAGCombiner::visitLOAD(SDNode *N) {
 
   // Try to infer better alignment information than the load already has.
   if (OptLevel != CodeGenOpt::None && LD->isUnindexed() && !LD->isAtomic()) {
-    if (unsigned Align = DAG.InferPtrAlignment(Ptr)) {
-      if (Align > LD->getAlignment() && LD->getSrcValueOffset() % Align == 0) {
+    if (MaybeAlign Alignment = DAG.InferPtrAlign(Ptr)) {
+      if (*Alignment > LD->getAlign() &&
+          isAligned(*Alignment, LD->getSrcValueOffset())) {
         SDValue NewLoad = DAG.getExtLoad(
             LD->getExtensionType(), SDLoc(N), LD->getValueType(0), Chain, Ptr,
-            LD->getPointerInfo(), LD->getMemoryVT(), Align,
+            LD->getPointerInfo(), LD->getMemoryVT(), *Alignment,
             LD->getMemOperand()->getFlags(), LD->getAAInfo());
         // NewLoad will always be N as we are only refining the alignment
         assert(NewLoad.getNode() == N);
@@ -16681,11 +16700,12 @@ SDValue DAGCombiner::visitSTORE(SDNode *N) {
 
   // Try to infer better alignment information than the store already has.
   if (OptLevel != CodeGenOpt::None && ST->isUnindexed() && !ST->isAtomic()) {
-    if (unsigned Align = DAG.InferPtrAlignment(Ptr)) {
-      if (Align > ST->getAlignment() && ST->getSrcValueOffset() % Align == 0) {
+    if (MaybeAlign Alignment = DAG.InferPtrAlign(Ptr)) {
+      if (*Alignment > ST->getAlign() &&
+          isAligned(*Alignment, ST->getSrcValueOffset())) {
         SDValue NewStore =
             DAG.getTruncStore(Chain, SDLoc(N), Value, Ptr, ST->getPointerInfo(),
-                              ST->getMemoryVT(), Align,
+                              ST->getMemoryVT(), *Alignment,
                               ST->getMemOperand()->getFlags(), ST->getAAInfo());
         // NewStore will always be N as we are only refining the alignment
         assert(NewStore.getNode() == N);
@@ -19801,8 +19821,8 @@ SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) {
         ShuffleVectorSDNode *InnerSVN = cast<ShuffleVectorSDNode>(BC0);
         SmallVector<int, 8> InnerMask;
         SmallVector<int, 8> OuterMask;
-        scaleShuffleMask<int>(InnerScale, InnerSVN->getMask(), InnerMask);
-        scaleShuffleMask<int>(OuterScale, SVN->getMask(), OuterMask);
+        scaleShuffleMask(InnerScale, InnerSVN->getMask(), InnerMask);
+        scaleShuffleMask(OuterScale, SVN->getMask(), OuterMask);
 
         // Merge the shuffle masks.
         SmallVector<int, 8> NewMask;
@@ -21333,16 +21353,16 @@ bool DAGCombiner::isAlias(SDNode *Op0, SDNode *Op1) const {
   // multiples of the size of the data.
   int64_t SrcValOffset0 = MUC0.MMO->getOffset();
   int64_t SrcValOffset1 = MUC1.MMO->getOffset();
-  unsigned OrigAlignment0 = MUC0.MMO->getBaseAlignment();
-  unsigned OrigAlignment1 = MUC1.MMO->getBaseAlignment();
+  Align OrigAlignment0 = MUC0.MMO->getBaseAlign();
+  Align OrigAlignment1 = MUC1.MMO->getBaseAlign();
   auto &Size0 = MUC0.NumBytes;
   auto &Size1 = MUC1.NumBytes;
   if (OrigAlignment0 == OrigAlignment1 && SrcValOffset0 != SrcValOffset1 &&
       Size0.hasValue() && Size1.hasValue() && *Size0 == *Size1 &&
       OrigAlignment0 > *Size0 && SrcValOffset0 % *Size0 == 0 &&
       SrcValOffset1 % *Size1 == 0) {
-    int64_t OffAlign0 = SrcValOffset0 % OrigAlignment0;
-    int64_t OffAlign1 = SrcValOffset1 % OrigAlignment1;
+    int64_t OffAlign0 = SrcValOffset0 % OrigAlignment0.value();
+    int64_t OffAlign1 = SrcValOffset1 % OrigAlignment1.value();
 
     // There is no overlap between these relatively aligned accesses of
     // similar size. Return no alias.
diff --git a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
index 461d481c822f5..03f9950bb3fd6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
@@ -225,6 +225,21 @@ static bool isRegUsedByPhiNodes(unsigned DefReg,
   return false;
 }
 
+static bool isTerminatingEHLabel(MachineBasicBlock *MBB, MachineInstr &MI) {
+  // Ignore non-EH labels.
+  if (!MI.isEHLabel())
+    return false;
+
+  // Any EH label outside a landing pad must be for an invoke. Consider it a
+  // terminator.
+  if (!MBB->isEHPad())
+    return true;
+
+  // If this is a landingpad, the first non-phi instruction will be an EH_LABEL.
+  // Don't consider that label to be a terminator.
+  return MI.getIterator() != MBB->getFirstNonPHI();
+}
+
 /// Build a map of instruction orders. Return the first terminator and its
 /// order. Consider EH_LABEL instructions to be terminators as well, since local
 /// values for phis after invokes must be materialized before the call.
@@ -233,7 +248,7 @@ void FastISel::InstOrderMap::initialize(
   unsigned Order = 0;
   for (MachineInstr &I : *MBB) {
     if (!FirstTerminator &&
-        (I.isTerminator() || (I.isEHLabel() && &I != &MBB->front()))) {
+        (I.isTerminator() || isTerminatingEHLabel(MBB, I))) {
       FirstTerminator = &I;
       FirstTerminatorOrder = Order;
     }
@@ -1211,17 +1226,17 @@ bool FastISel::lowerCallTo(CallLoweringInfo &CLI) {
 
       // For ByVal, alignment should come from FE. BE will guess if this info
       // is not there, but there are cases it cannot get right.
-      unsigned FrameAlign = Arg.Alignment;
+      MaybeAlign FrameAlign = Arg.Alignment;
       if (!FrameAlign)
-        FrameAlign = TLI.getByValTypeAlignment(ElementTy, DL);
+        FrameAlign = Align(TLI.getByValTypeAlignment(ElementTy, DL));
       Flags.setByValSize(FrameSize);
-      Flags.setByValAlign(Align(FrameAlign));
+      Flags.setByValAlign(*FrameAlign);
     }
     if (Arg.IsNest)
       Flags.setNest();
     if (NeedsRegBlock)
       Flags.setInConsecutiveRegs();
-    Flags.setOrigAlign(Align(DL.getABITypeAlignment(Arg.Ty)));
+    Flags.setOrigAlign(DL.getABITypeAlign(Arg.Ty));
 
     CLI.OutVals.push_back(Arg.Val);
     CLI.OutFlags.push_back(Flags);
@@ -2418,18 +2433,18 @@ MachineMemOperand *
 FastISel::createMachineMemOperandFor(const Instruction *I) const {
   const Value *Ptr;
   Type *ValTy;
-  unsigned Alignment;
+  MaybeAlign Alignment;
   MachineMemOperand::Flags Flags;
   bool IsVolatile;
 
   if (const auto *LI = dyn_cast<LoadInst>(I)) {
-    Alignment = LI->getAlignment();
+    Alignment = LI->getAlign();
     IsVolatile = LI->isVolatile();
     Flags = MachineMemOperand::MOLoad;
     Ptr = LI->getPointerOperand();
     ValTy = LI->getType();
   } else if (const auto *SI = dyn_cast<StoreInst>(I)) {
-    Alignment = SI->getAlignment();
+    Alignment = SI->getAlign();
     IsVolatile = SI->isVolatile();
     Flags = MachineMemOperand::MOStore;
     Ptr = SI->getPointerOperand();
@@ -2445,8 +2460,8 @@ FastISel::createMachineMemOperandFor(const Instruction *I) const {
   AAMDNodes AAInfo;
   I->getAAMetadata(AAInfo);
 
-  if (Alignment == 0) // Ensure that codegen never sees alignment 0.
-    Alignment = DL.getABITypeAlignment(ValTy);
+  if (!Alignment) // Ensure that codegen never sees alignment 0.
+    Alignment = DL.getABITypeAlign(ValTy);
 
   unsigned Size = DL.getTypeStoreSize(ValTy);
 
@@ -2460,7 +2475,7 @@ FastISel::createMachineMemOperandFor(const Instruction *I) const {
     Flags |= MachineMemOperand::MOInvariant;
 
   return FuncInfo.MF->getMachineMemOperand(MachinePointerInfo(Ptr), Flags, Size,
-                                           Alignment, AAInfo, Ranges);
+                                           *Alignment, AAInfo, Ranges);
 }
 
 CmpInst::Predicate FastISel::optimizeCmpPredicate(const CmpInst *CI) const {
diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
index 95b97872398c2..ff7ad7186e238 100644
--- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
@@ -486,8 +486,8 @@ void InstrEmitter::EmitSubregNode(SDNode *Node,
   for (SDNode *User : Node->uses()) {
     if (User->getOpcode() == ISD::CopyToReg &&
         User->getOperand(2).getNode() == Node) {
-      unsigned DestReg = cast<RegisterSDNode>(User->getOperand(1))->getReg();
-      if (Register::isVirtualRegister(DestReg)) {
+      Register DestReg = cast<RegisterSDNode>(User->getOperand(1))->getReg();
+      if (DestReg.isVirtual()) {
         VRBase = DestReg;
         break;
       }
@@ -502,7 +502,7 @@ void InstrEmitter::EmitSubregNode(SDNode *Node,
     const TargetRegisterClass *TRC =
       TLI->getRegClassFor(Node->getSimpleValueType(0), Node->isDivergent());
 
-    unsigned Reg;
+    Register Reg;
     MachineInstr *DefMI;
     RegisterSDNode *R = dyn_cast<RegisterSDNode>(Node->getOperand(0));
     if (R && Register::isPhysicalRegister(R->getReg())) {
@@ -513,7 +513,8 @@ void InstrEmitter::EmitSubregNode(SDNode *Node,
       DefMI = MRI->getVRegDef(Reg);
     }
 
-    unsigned SrcReg, DstReg, DefSubIdx;
+    Register SrcReg, DstReg;
+    unsigned DefSubIdx;
     if (DefMI &&
         TII->isCoalescableExtInstr(*DefMI, SrcReg, DstReg, DefSubIdx) &&
         SubIdx == DefSubIdx &&
@@ -531,7 +532,7 @@ void InstrEmitter::EmitSubregNode(SDNode *Node,
       // Reg may not support a SubIdx sub-register, and we may need to
       // constrain its register class or issue a COPY to a compatible register
       // class.
-      if (Register::isVirtualRegister(Reg))
+      if (Reg.isVirtual())
         Reg = ConstrainForSubReg(Reg, SubIdx,
                                  Node->getOperand(0).getSimpleValueType(),
                                  Node->isDivergent(), Node->getDebugLoc());
@@ -543,7 +544,7 @@ void InstrEmitter::EmitSubregNode(SDNode *Node,
       MachineInstrBuilder CopyMI =
           BuildMI(*MBB, InsertPos, Node->getDebugLoc(),
                   TII->get(TargetOpcode::COPY), VRBase);
-      if (Register::isVirtualRegister(Reg))
+      if (Reg.isVirtual())
         CopyMI.addReg(Reg, 0, SubIdx);
       else
         CopyMI.addReg(TRI->getSubReg(Reg, SubIdx));
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index 60174fc9da8f8..994c491da6915 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -125,6 +125,7 @@ void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
     case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
     case ISD::SELECT:      R = SoftenFloatRes_SELECT(N); break;
     case ISD::SELECT_CC:   R = SoftenFloatRes_SELECT_CC(N); break;
+    case ISD::FREEZE:      R = SoftenFloatRes_FREEZE(N); break;
     case ISD::STRICT_SINT_TO_FP:
     case ISD::STRICT_UINT_TO_FP:
     case ISD::SINT_TO_FP:
@@ -184,6 +185,12 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
   return BitConvertToInteger(N->getOperand(0));
 }
 
+SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
+  EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
+  return DAG.getNode(ISD::FREEZE, SDLoc(N), Ty,
+                     GetSoftenedFloat(N->getOperand(0)));
+}
+
 SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
                                                       unsigned ResNo) {
   SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 0248b5121e3f2..ed67f7dc8ea3f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -278,8 +278,24 @@ SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
     return Res.getValue(1);
   }
 
-  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
+  // Op2 is used for the comparison and thus must be extended according to the
+  // target's atomic operations. Op3 is merely stored and so can be left alone.
+  SDValue Op2 = N->getOperand(2);
   SDValue Op3 = GetPromotedInteger(N->getOperand(3));
+  switch (TLI.getExtendForAtomicCmpSwapArg()) {
+  case ISD::SIGN_EXTEND:
+    Op2 = SExtPromotedInteger(Op2);
+    break;
+  case ISD::ZERO_EXTEND:
+    Op2 = ZExtPromotedInteger(Op2);
+    break;
+  case ISD::ANY_EXTEND:
+    Op2 = GetPromotedInteger(Op2);
+    break;
+  default:
+    llvm_unreachable("Invalid atomic op extension");
+  }
+
   SDVTList VTs =
       DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
   SDValue Res = DAG.getAtomicCmpSwap(
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index aee4ab1fcd619..e27f1e5b32311 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -519,6 +519,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   SDValue SoftenFloatRes_FP_ROUND(SDNode *N);
   SDValue SoftenFloatRes_FPOW(SDNode *N);
   SDValue SoftenFloatRes_FPOWI(SDNode *N);
+  SDValue SoftenFloatRes_FREEZE(SDNode *N);
   SDValue SoftenFloatRes_FREM(SDNode *N);
   SDValue SoftenFloatRes_FRINT(SDNode *N);
   SDValue SoftenFloatRes_FROUND(SDNode *N);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index afed415af5ed9..c761d4b8b526e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -191,8 +191,8 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
   SDValue Op0 = GetScalarizedVector(N->getOperand(0));
   SDValue Op1 = GetScalarizedVector(N->getOperand(1));
   SDValue Op2 = GetScalarizedVector(N->getOperand(2));
-  return DAG.getNode(N->getOpcode(), SDLoc(N),
-                     Op0.getValueType(), Op0, Op1, Op2);
+  return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
+                     Op2, N->getFlags());
 }
 
 SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
@@ -200,7 +200,7 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
   SDValue Op1 = GetScalarizedVector(N->getOperand(1));
   SDValue Op2 = N->getOperand(2);
   return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
-                     Op2);
+                     Op2, N->getFlags());
 }
 
 SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
@@ -225,7 +225,8 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
     Opers[i] = Oper;
   }
 
-  SDValue Result = DAG.getNode(N->getOpcode(), dl, ValueVTs, Opers);
+  SDValue Result = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(ValueVTs),
+                               Opers, N->getFlags());
 
   // Legalize the chain result - switch anything that used the old chain to
   // use the new one.
@@ -255,6 +256,7 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
       ResVT.getVectorElementType(), OvVT.getVectorElementType());
   SDNode *ScalarNode = DAG.getNode(
       N->getOpcode(), DL, ScalarVTs, ScalarLHS, ScalarRHS).getNode();
+  ScalarNode->setFlags(N->getFlags());
 
   // Replace the other vector result not being explicitly scalarized here.
   unsigned OtherNo = 1 - ResNo;
@@ -335,8 +337,7 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
       N->getValueType(0).getVectorElementType(), SDLoc(N), N->getChain(),
       N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()),
       N->getPointerInfo(), N->getMemoryVT().getVectorElementType(),
-      N->getOriginalAlignment(), N->getMemOperand()->getFlags(),
-      N->getAAInfo());
+      N->getOriginalAlign(), N->getMemOperand()->getFlags(), N->getAAInfo());
 
   // Legalize the chain result - switch anything that used the old chain to
   // use the new one.
@@ -364,7 +365,7 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
     Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
                      DAG.getVectorIdxConstant(0, DL));
   }
-  return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op);
+  return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op, N->getFlags());
 }
 
 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
@@ -753,7 +754,7 @@ SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
 
   return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
                       N->getBasePtr(), N->getPointerInfo(),
-                      N->getOriginalAlignment(), N->getMemOperand()->getFlags(),
+                      N->getOriginalAlign(), N->getMemOperand()->getFlags(),
                       N->getAAInfo());
 }
 
@@ -997,10 +998,10 @@ void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
   GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
   SDLoc dl(N);
 
-  Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(),
-                   Op0Lo, Op1Lo, Op2Lo);
-  Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(),
-                   Op0Hi, Op1Hi, Op2Hi);
+  Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(), Op0Lo, Op1Lo,
+                   Op2Lo, N->getFlags());
+  Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(), Op0Hi, Op1Hi,
+                   Op2Hi, N->getFlags());
 }
 
 void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
@@ -1012,8 +1013,10 @@ void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
   SDValue Op2 = N->getOperand(2);
 
   unsigned Opcode = N->getOpcode();
-  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Op2);
-  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Op2);
+  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Op2,
+                   N->getFlags());
+  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Op2,
+                   N->getFlags());
 }
 
 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
@@ -1294,8 +1297,10 @@ void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
 
   EVT LoValueVTs[] = {LoVT, MVT::Other};
   EVT HiValueVTs[] = {HiVT, MVT::Other};
-  Lo = DAG.getNode(N->getOpcode(), dl, LoValueVTs, OpsLo);
-  Hi = DAG.getNode(N->getOpcode(), dl, HiValueVTs, OpsHi);
+  Lo = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(LoValueVTs), OpsLo,
+                   N->getFlags());
+  Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(HiValueVTs), OpsHi,
+                   N->getFlags());
 
   // Build a factor node to remember that this Op is independent of the
   // other one.
@@ -1385,6 +1390,8 @@ void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
   SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT);
   SDNode *LoNode = DAG.getNode(Opcode, dl, LoVTs, LoLHS, LoRHS).getNode();
   SDNode *HiNode = DAG.getNode(Opcode, dl, HiVTs, HiLHS, HiRHS).getNode();
+  LoNode->setFlags(N->getFlags());
+  HiNode->setFlags(N->getFlags());
 
   Lo = SDValue(LoNode, ResNo);
   Hi = SDValue(HiNode, ResNo);
@@ -1501,7 +1508,7 @@ void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
   SDValue Ptr = LD->getBasePtr();
   SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
   EVT MemoryVT = LD->getMemoryVT();
-  unsigned Alignment = LD->getOriginalAlignment();
+  Align Alignment = LD->getOriginalAlign();
   MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
   AAMDNodes AAInfo = LD->getAAInfo();
 
@@ -1548,7 +1555,7 @@ void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
   assert(Offset.isUndef() && "Unexpected indexed masked load offset");
   SDValue Mask = MLD->getMask();
   SDValue PassThru = MLD->getPassThru();
-  unsigned Alignment = MLD->getOriginalAlignment();
+  Align Alignment = MLD->getOriginalAlign();
   ISD::LoadExtType ExtType = MLD->getExtensionType();
 
   // Split Mask operand
@@ -1572,10 +1579,9 @@ void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
   else
     std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
 
-  MachineMemOperand *MMO = DAG.getMachineFunction().
-    getMachineMemOperand(MLD->getPointerInfo(),
-                         MachineMemOperand::MOLoad,  LoMemVT.getStoreSize(),
-                         Alignment, MLD->getAAInfo(), MLD->getRanges());
+  MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
+      MLD->getPointerInfo(), MachineMemOperand::MOLoad, LoMemVT.getStoreSize(),
+      Alignment, MLD->getAAInfo(), MLD->getRanges());
 
   Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, Offset, MaskLo, PassThruLo, LoMemVT,
                          MMO, MLD->getAddressingMode(), ExtType,
@@ -1587,8 +1593,7 @@ void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
 
   MMO = DAG.getMachineFunction().getMachineMemOperand(
       MLD->getPointerInfo().getWithOffset(HiOffset), MachineMemOperand::MOLoad,
-      HiMemVT.getStoreSize(), Alignment, MLD->getAAInfo(),
-      MLD->getRanges());
+      HiMemVT.getStoreSize(), Alignment, MLD->getAAInfo(), MLD->getRanges());
 
   Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, Offset, MaskHi, PassThruHi, HiMemVT,
                          MMO, MLD->getAddressingMode(), ExtType,
@@ -1617,7 +1622,7 @@ void DAGTypeLegalizer::SplitVecRes_MGATHER(MaskedGatherSDNode *MGT,
   SDValue PassThru = MGT->getPassThru();
   SDValue Index = MGT->getIndex();
   SDValue Scale = MGT->getScale();
-  unsigned Alignment = MGT->getOriginalAlignment();
+  Align Alignment = MGT->getOriginalAlign();
 
   // Split Mask operand
   SDValue MaskLo, MaskHi;
@@ -1710,11 +1715,13 @@ void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
     std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, OpNo);
 
   if (N->getOpcode() == ISD::FP_ROUND) {
-    Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1));
-    Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1));
+    Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1),
+                     N->getFlags());
+    Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1),
+                     N->getFlags());
   } else {
-    Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
-    Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
+    Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getFlags());
+    Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getFlags());
   }
 }
 
@@ -2245,7 +2252,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT,
   SDValue Scale = MGT->getScale();
   SDValue Mask = MGT->getMask();
   SDValue PassThru = MGT->getPassThru();
-  unsigned Alignment = MGT->getOriginalAlignment();
+  Align Alignment = MGT->getOriginalAlign();
 
   SDValue MaskLo, MaskHi;
   if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
@@ -2270,20 +2277,17 @@ SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT,
   else
     std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
 
-  MachineMemOperand *MMO = DAG.getMachineFunction().
-    getMachineMemOperand(MGT->getPointerInfo(),
-                         MachineMemOperand::MOLoad,  LoMemVT.getStoreSize(),
-                         Alignment, MGT->getAAInfo(), MGT->getRanges());
+  MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
+      MGT->getPointerInfo(), MachineMemOperand::MOLoad, LoMemVT.getStoreSize(),
+      Alignment, MGT->getAAInfo(), MGT->getRanges());
 
   SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
   SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl,
                                    OpsLo, MMO, MGT->getIndexType());
 
-  MMO = DAG.getMachineFunction().
-    getMachineMemOperand(MGT->getPointerInfo(),
-                         MachineMemOperand::MOLoad,  HiMemVT.getStoreSize(),
-                         Alignment, MGT->getAAInfo(),
-                         MGT->getRanges());
+  MMO = DAG.getMachineFunction().getMachineMemOperand(
+      MGT->getPointerInfo(), MachineMemOperand::MOLoad, HiMemVT.getStoreSize(),
+      Alignment, MGT->getAAInfo(), MGT->getRanges());
 
   SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
   SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl,
@@ -2314,7 +2318,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
   SDValue Mask = N->getMask();
   SDValue Data = N->getValue();
   EVT MemoryVT = N->getMemoryVT();
-  unsigned Alignment = N->getOriginalAlignment();
+  Align Alignment = N->getOriginalAlign();
   SDLoc DL(N);
 
   EVT LoMemVT, HiMemVT;
@@ -2339,10 +2343,9 @@ SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
   }
 
   SDValue Lo, Hi;
-  MachineMemOperand *MMO = DAG.getMachineFunction().
-    getMachineMemOperand(N->getPointerInfo(),
-                         MachineMemOperand::MOStore, LoMemVT.getStoreSize(),
-                         Alignment, N->getAAInfo(), N->getRanges());
+  MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
+      N->getPointerInfo(), MachineMemOperand::MOStore, LoMemVT.getStoreSize(),
+      Alignment, N->getAAInfo(), N->getRanges());
 
   Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, Offset, MaskLo, LoMemVT, MMO,
                           N->getAddressingMode(), N->isTruncatingStore(),
@@ -2354,8 +2357,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
 
   MMO = DAG.getMachineFunction().getMachineMemOperand(
       N->getPointerInfo().getWithOffset(HiOffset), MachineMemOperand::MOStore,
-      HiMemVT.getStoreSize(), Alignment, N->getAAInfo(),
-      N->getRanges());
+      HiMemVT.getStoreSize(), Alignment, N->getAAInfo(), N->getRanges());
 
   Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, Offset, MaskHi, HiMemVT, MMO,
                           N->getAddressingMode(), N->isTruncatingStore(),
@@ -2374,7 +2376,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_MSCATTER(MaskedScatterSDNode *N,
   SDValue Index = N->getIndex();
   SDValue Scale = N->getScale();
   SDValue Data = N->getValue();
-  unsigned Alignment = N->getOriginalAlignment();
+  Align Alignment = N->getOriginalAlign();
   SDLoc DL(N);
 
   // Split all operands
@@ -2429,7 +2431,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
   SDValue Ch  = N->getChain();
   SDValue Ptr = N->getBasePtr();
   EVT MemoryVT = N->getMemoryVT();
-  unsigned Alignment = N->getOriginalAlignment();
+  Align Alignment = N->getOriginalAlign();
   MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
   AAMDNodes AAInfo = N->getAAInfo();
   SDValue Lo, Hi;
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 58311069c09ae..5f6d8bf95517b 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -1000,12 +1000,12 @@ SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
   return Node;
 }
 
-unsigned SelectionDAG::getEVTAlignment(EVT VT) const {
+Align SelectionDAG::getEVTAlign(EVT VT) const {
   Type *Ty = VT == MVT::iPTR ?
                    PointerType::get(Type::getInt8Ty(*getContext()), 0) :
                    VT.getTypeForEVT(*getContext());
 
-  return getDataLayout().getABITypeAlignment(Ty);
+  return getDataLayout().getABITypeAlign(Ty);
 }
 
 // EntryNode could meaningfully have debug info if we can find it...
@@ -1927,6 +1927,10 @@ SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,
   return SDValue(N, 0);
 }
 
+SDValue SelectionDAG::getFreeze(SDValue V) {
+  return getNode(ISD::FREEZE, SDLoc(V), V.getValueType(), V);
+}
+
 /// getShiftAmountOperand - Return the specified value casted to
 /// the target's desired shift amount type.
 SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
@@ -4546,6 +4550,9 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
 
   unsigned OpOpcode = Operand.getNode()->getOpcode();
   switch (Opcode) {
+  case ISD::FREEZE:
+    assert(VT == Operand.getValueType() && "Unexpected VT!");
+    break;
   case ISD::TokenFactor:
   case ISD::MERGE_VALUES:
   case ISD::CONCAT_VECTORS:
@@ -5909,7 +5916,7 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
   if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
     DstAlignCanChange = true;
-  MaybeAlign SrcAlign(DAG.InferPtrAlignment(Src));
+  MaybeAlign SrcAlign = DAG.InferPtrAlign(Src);
   if (!SrcAlign || Alignment > *SrcAlign)
     SrcAlign = Alignment;
   assert(SrcAlign && "SrcAlign must be set");
@@ -6094,7 +6101,7 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
   if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
     DstAlignCanChange = true;
-  MaybeAlign SrcAlign(DAG.InferPtrAlignment(Src));
+  MaybeAlign SrcAlign = DAG.InferPtrAlign(Src);
   if (!SrcAlign || Alignment > *SrcAlign)
     SrcAlign = Alignment;
   assert(SrcAlign && "SrcAlign must be set");
@@ -6672,11 +6679,8 @@ SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {
 
 SDValue SelectionDAG::getMemIntrinsicNode(
     unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
-    EVT MemVT, MachinePointerInfo PtrInfo, unsigned Align,
+    EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
     MachineMemOperand::Flags Flags, uint64_t Size, const AAMDNodes &AAInfo) {
-  if (Align == 0)  // Ensure that codegen never sees alignment 0
-    Align = getEVTAlignment(MemVT);
-
   if (!Size && MemVT.isScalableVector())
     Size = MemoryLocation::UnknownSize;
   else if (!Size)
@@ -6684,7 +6688,7 @@ SDValue SelectionDAG::getMemIntrinsicNode(
 
   MachineFunction &MF = getMachineFunction();
   MachineMemOperand *MMO =
-      MF.getMachineMemOperand(PtrInfo, Flags, Size, Align, AAInfo);
+      MF.getMachineMemOperand(PtrInfo, Flags, Size, Alignment, AAInfo);
 
   return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
 }
@@ -6803,13 +6807,11 @@ SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
                               EVT VT, const SDLoc &dl, SDValue Chain,
                               SDValue Ptr, SDValue Offset,
                               MachinePointerInfo PtrInfo, EVT MemVT,
-                              unsigned Alignment,
+                              Align Alignment,
                               MachineMemOperand::Flags MMOFlags,
                               const AAMDNodes &AAInfo, const MDNode *Ranges) {
   assert(Chain.getValueType() == MVT::Other &&
         "Invalid chain type");
-  if (Alignment == 0)  // Ensure that codegen never sees alignment 0
-    Alignment = getEVTAlignment(MemVT);
 
   MMOFlags |= MachineMemOperand::MOLoad;
   assert((MMOFlags & MachineMemOperand::MOStore) == 0);
@@ -6820,8 +6822,8 @@ SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
 
   uint64_t Size = MemoryLocation::getSizeOrUnknown(MemVT.getStoreSize());
   MachineFunction &MF = getMachineFunction();
-  MachineMemOperand *MMO = MF.getMachineMemOperand(
-      PtrInfo, MMOFlags, Size, Alignment, AAInfo, Ranges);
+  MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
+                                                   Alignment, AAInfo, Ranges);
   return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
 }
 
@@ -6876,7 +6878,7 @@ SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
 
 SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
                               SDValue Ptr, MachinePointerInfo PtrInfo,
-                              unsigned Alignment,
+                              MaybeAlign Alignment,
                               MachineMemOperand::Flags MMOFlags,
                               const AAMDNodes &AAInfo, const MDNode *Ranges) {
   SDValue Undef = getUNDEF(Ptr.getValueType());
@@ -6894,7 +6896,7 @@ SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
                                  EVT VT, SDValue Chain, SDValue Ptr,
                                  MachinePointerInfo PtrInfo, EVT MemVT,
-                                 unsigned Alignment,
+                                 MaybeAlign Alignment,
                                  MachineMemOperand::Flags MMOFlags,
                                  const AAMDNodes &AAInfo) {
   SDValue Undef = getUNDEF(Ptr.getValueType());
@@ -6927,12 +6929,10 @@ SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
 
 SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
                                SDValue Ptr, MachinePointerInfo PtrInfo,
-                               unsigned Alignment,
+                               Align Alignment,
                                MachineMemOperand::Flags MMOFlags,
                                const AAMDNodes &AAInfo) {
   assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
-  if (Alignment == 0)  // Ensure that codegen never sees alignment 0
-    Alignment = getEVTAlignment(Val.getValueType());
 
   MMOFlags |= MachineMemOperand::MOStore;
   assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
@@ -6980,13 +6980,11 @@ SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
 
 SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
                                     SDValue Ptr, MachinePointerInfo PtrInfo,
-                                    EVT SVT, unsigned Alignment,
+                                    EVT SVT, Align Alignment,
                                     MachineMemOperand::Flags MMOFlags,
                                     const AAMDNodes &AAInfo) {
   assert(Chain.getValueType() == MVT::Other &&
         "Invalid chain type");
-  if (Alignment == 0)  // Ensure that codegen never sees alignment 0
-    Alignment = getEVTAlignment(SVT);
 
   MMOFlags |= MachineMemOperand::MOStore;
   assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
@@ -9420,9 +9418,9 @@ bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,
   return false;
 }
 
-/// InferPtrAlignment - Infer alignment of a load / store address. Return 0 if
-/// it cannot be inferred.
-unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const {
+/// InferPtrAlignment - Infer alignment of a load / store address. Return None
+/// if it cannot be inferred.
+MaybeAlign SelectionDAG::InferPtrAlign(SDValue Ptr) const {
   // If this is a GlobalAddress + cst, return the alignment.
   const GlobalValue *GV = nullptr;
   int64_t GVOffset = 0;
@@ -9431,9 +9429,8 @@ unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const {
     KnownBits Known(PtrWidth);
     llvm::computeKnownBits(GV, Known, getDataLayout());
     unsigned AlignBits = Known.countMinTrailingZeros();
-    unsigned Align = AlignBits ? 1 << std::min(31U, AlignBits) : 0;
-    if (Align)
-      return MinAlign(Align, GVOffset);
+    if (AlignBits)
+      return commonAlignment(Align(1ull << std::min(31U, AlignBits)), GVOffset);
   }
 
   // If this is a direct reference to a stack slot, use information about the
@@ -9451,12 +9448,10 @@ unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const {
 
   if (FrameIdx != INT_MIN) {
     const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
-    unsigned FIInfoAlign = MinAlign(MFI.getObjectAlignment(FrameIdx),
-                                    FrameOffset);
-    return FIInfoAlign;
+    return commonAlignment(MFI.getObjectAlign(FrameIdx), FrameOffset);
   }
 
-  return 0;
+  return None;
 }
 
 /// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 1ad86208ed4ea..823e2a83c9658 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -1604,6 +1604,9 @@ SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
     return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
   }
 
+  if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V)) {
+    return DAG.getMDNode(cast<MDNode>(MD->getMetadata()));
+  }
   llvm_unreachable("Can't get register for value!");
 }
 
@@ -2512,7 +2515,7 @@ static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL,
     auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
                  MachineMemOperand::MODereferenceable;
     MachineMemOperand *MemRef = MF.getMachineMemOperand(
-        MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlignment(PtrTy));
+        MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlign(PtrTy));
     DAG.setNodeMemRefs(Node, {MemRef});
   }
   if (PtrTy != PtrMemTy)
@@ -2998,133 +3001,6 @@ void SelectionDAGBuilder::visitFSub(const User &I) {
   visitBinary(I, ISD::FSUB);
 }
 
-/// Checks if the given instruction performs a vector reduction, in which case
-/// we have the freedom to alter the elements in the result as long as the
-/// reduction of them stays unchanged.
-static bool isVectorReductionOp(const User *I) {
-  const Instruction *Inst = dyn_cast<Instruction>(I);
-  if (!Inst || !Inst->getType()->isVectorTy())
-    return false;
-
-  auto OpCode = Inst->getOpcode();
-  switch (OpCode) {
-  case Instruction::Add:
-  case Instruction::Mul:
-  case Instruction::And:
-  case Instruction::Or:
-  case Instruction::Xor:
-    break;
-  case Instruction::FAdd:
-  case Instruction::FMul:
-    if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
-      if (FPOp->getFastMathFlags().isFast())
-        break;
-    LLVM_FALLTHROUGH;
-  default:
-    return false;
-  }
-
-  unsigned ElemNum = Inst->getType()->getVectorNumElements();
-  // Ensure the reduction size is a power of 2.
-  if (!isPowerOf2_32(ElemNum))
-    return false;
-
-  unsigned ElemNumToReduce = ElemNum;
-
-  // Do DFS search on the def-use chain from the given instruction. We only
-  // allow four kinds of operations during the search until we reach the
-  // instruction that extracts the first element from the vector:
-  //
-  //   1. The reduction operation of the same opcode as the given instruction.
-  //
-  //   2. PHI node.
-  //
-  //   3. ShuffleVector instruction together with a reduction operation that
-  //      does a partial reduction.
-  //
-  //   4. ExtractElement that extracts the first element from the vector, and we
-  //      stop searching the def-use chain here.
-  //
-  // 3 & 4 above perform a reduction on all elements of the vector. We push defs
-  // from 1-3 to the stack to continue the DFS. The given instruction is not
-  // a reduction operation if we meet any other instructions other than those
-  // listed above.
-
-  SmallVector<const User *, 16> UsersToVisit{Inst};
-  SmallPtrSet<const User *, 16> Visited;
-  bool ReduxExtracted = false;
-
-  while (!UsersToVisit.empty()) {
-    auto User = UsersToVisit.back();
-    UsersToVisit.pop_back();
-    if (!Visited.insert(User).second)
-      continue;
-
-    for (const auto *U : User->users()) {
-      auto Inst = dyn_cast<Instruction>(U);
-      if (!Inst)
-        return false;
-
-      if (Inst->getOpcode() == OpCode || isa<PHINode>(U)) {
-        if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
-          if (!isa<PHINode>(FPOp) && !FPOp->getFastMathFlags().isFast())
-            return false;
-        UsersToVisit.push_back(U);
-      } else if (const ShuffleVectorInst *ShufInst =
-                     dyn_cast<ShuffleVectorInst>(U)) {
-        // Detect the following pattern: A ShuffleVector instruction together
-        // with a reduction that do partial reduction on the first and second
-        // ElemNumToReduce / 2 elements, and store the result in
-        // ElemNumToReduce / 2 elements in another vector.
-
-        unsigned ResultElements = ShufInst->getType()->getVectorNumElements();
-        if (ResultElements < ElemNum)
-          return false;
-
-        if (ElemNumToReduce == 1)
-          return false;
-        if (!isa<UndefValue>(U->getOperand(1)))
-          return false;
-        for (unsigned i = 0; i < ElemNumToReduce / 2; ++i)
-          if (ShufInst->getMaskValue(i) != int(i + ElemNumToReduce / 2))
-            return false;
-        for (unsigned i = ElemNumToReduce / 2; i < ElemNum; ++i)
-          if (ShufInst->getMaskValue(i) != -1)
-            return false;
-
-        // There is only one user of this ShuffleVector instruction, which
-        // must be a reduction operation.
-        if (!U->hasOneUse())
-          return false;
-
-        auto U2 = dyn_cast<Instruction>(*U->user_begin());
-        if (!U2 || U2->getOpcode() != OpCode)
-          return false;
-
-        // Check operands of the reduction operation.
-        if ((U2->getOperand(0) == U->getOperand(0) && U2->getOperand(1) == U) ||
-            (U2->getOperand(1) == U->getOperand(0) && U2->getOperand(0) == U)) {
-          UsersToVisit.push_back(U2);
-          ElemNumToReduce /= 2;
-        } else
-          return false;
-      } else if (isa<ExtractElementInst>(U)) {
-        // At this moment we should have reduced all elements in the vector.
-        if (ElemNumToReduce != 1)
-          return false;
-
-        const ConstantInt *Val = dyn_cast<ConstantInt>(U->getOperand(1));
-        if (!Val || !Val->isZero())
-          return false;
-
-        ReduxExtracted = true;
-      } else
-        return false;
-    }
-  }
-  return ReduxExtracted;
-}
-
 void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
   SDNodeFlags Flags;
 
@@ -3143,17 +3019,6 @@ void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
   if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I)) {
     Flags.setExact(ExactOp->isExact());
   }
-  if (isVectorReductionOp(&I)) {
-    Flags.setVectorReduction(true);
-    LLVM_DEBUG(dbgs() << "Detected a reduction operation:" << I << "\n");
-
-    // If no flags are set we will propagate the incoming flags, if any flags
-    // are set, we will intersect them with the incoming flag and so we need to
-    // copy the FMF flags here.
-    if (auto *FPOp = dyn_cast<FPMathOperator>(&I)) {
-      Flags.copyFMF(*FPOp);
-    }
-  }
 
   SDValue Op1 = getValue(I.getOperand(0));
   SDValue Op2 = getValue(I.getOperand(1));
@@ -3572,14 +3437,19 @@ void SelectionDAGBuilder::visitExtractElement(const User &I) {
 void SelectionDAGBuilder::visitShuffleVector(const User &I) {
   SDValue Src1 = getValue(I.getOperand(0));
   SDValue Src2 = getValue(I.getOperand(1));
-  Constant *MaskV = cast<Constant>(I.getOperand(2));
+  ArrayRef<int> Mask;
+  if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
+    Mask = SVI->getShuffleMask();
+  else
+    Mask = cast<ConstantExpr>(I).getShuffleMask();
   SDLoc DL = getCurSDLoc();
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
   EVT SrcVT = Src1.getValueType();
   unsigned SrcNumElts = SrcVT.getVectorNumElements();
 
-  if (MaskV->isNullValue() && VT.isScalableVector()) {
+  if (all_of(Mask, [](int Elem) { return Elem == 0; }) &&
+      VT.isScalableVector()) {
     // Canonical splat form of first element of first input vector.
     SDValue FirstElt =
         DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT.getScalarType(), Src1,
@@ -3593,8 +3463,6 @@ void SelectionDAGBuilder::visitShuffleVector(const User &I) {
   // for targets that support a SPLAT_VECTOR for non-scalable vector types.
   assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");
 
-  SmallVector<int, 8> Mask;
-  ShuffleVectorInst::getShuffleMask(MaskV, Mask);
   unsigned MaskNumElts = Mask.size();
 
   if (SrcNumElts == MaskNumElts) {
@@ -4308,25 +4176,26 @@ void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
                                            bool IsCompressing) {
   SDLoc sdl = getCurSDLoc();
 
-  auto getMaskedStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
-                           unsigned& Alignment) {
+  auto getMaskedStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
+                               MaybeAlign &Alignment) {
     // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
     Src0 = I.getArgOperand(0);
     Ptr = I.getArgOperand(1);
-    Alignment = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
+    Alignment =
+        MaybeAlign(cast<ConstantInt>(I.getArgOperand(2))->getZExtValue());
     Mask = I.getArgOperand(3);
   };
-  auto getCompressingStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
-                           unsigned& Alignment) {
+  auto getCompressingStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
+                                    MaybeAlign &Alignment) {
     // llvm.masked.compressstore.*(Src0, Ptr, Mask)
     Src0 = I.getArgOperand(0);
     Ptr = I.getArgOperand(1);
     Mask = I.getArgOperand(2);
-    Alignment = 0;
+    Alignment = None;
   };
 
   Value  *PtrOperand, *MaskOperand, *Src0Operand;
-  unsigned Alignment;
+  MaybeAlign Alignment;
   if (IsCompressing)
     getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
   else
@@ -4339,19 +4208,16 @@ void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
 
   EVT VT = Src0.getValueType();
   if (!Alignment)
-    Alignment = DAG.getEVTAlignment(VT);
+    Alignment = DAG.getEVTAlign(VT);
 
   AAMDNodes AAInfo;
   I.getAAMetadata(AAInfo);
 
-  MachineMemOperand *MMO =
-    DAG.getMachineFunction().
-    getMachineMemOperand(MachinePointerInfo(PtrOperand),
-                          MachineMemOperand::MOStore,
-                          // TODO: Make MachineMemOperands aware of scalable
-                          // vectors.
-                          VT.getStoreSize().getKnownMinSize(),
-                          Alignment, AAInfo);
+  MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
+      MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,
+      // TODO: Make MachineMemOperands aware of scalable
+      // vectors.
+      VT.getStoreSize().getKnownMinSize(), *Alignment, AAInfo);
   SDValue StoreNode =
       DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask, VT, MMO,
                          ISD::UNINDEXED, false /* Truncating */, IsCompressing);
@@ -4451,9 +4317,9 @@ void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
   SDValue Src0 = getValue(I.getArgOperand(0));
   SDValue Mask = getValue(I.getArgOperand(3));
   EVT VT = Src0.getValueType();
-  unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
+  MaybeAlign Alignment(cast<ConstantInt>(I.getArgOperand(2))->getZExtValue());
   if (!Alignment)
-    Alignment = DAG.getEVTAlignment(VT);
+    Alignment = DAG.getEVTAlign(VT);
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
 
   AAMDNodes AAInfo;
@@ -4466,13 +4332,11 @@ void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
   bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this);
 
   unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
-  MachineMemOperand *MMO = DAG.getMachineFunction().
-    getMachineMemOperand(MachinePointerInfo(AS),
-                         MachineMemOperand::MOStore,
-                         // TODO: Make MachineMemOperands aware of scalable
-                         // vectors.
-                         MemoryLocation::UnknownSize,
-                         Alignment, AAInfo);
+  MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
+      MachinePointerInfo(AS), MachineMemOperand::MOStore,
+      // TODO: Make MachineMemOperands aware of scalable
+      // vectors.
+      MemoryLocation::UnknownSize, *Alignment, AAInfo);
   if (!UniformBase) {
     Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
     Index = getValue(Ptr);
@@ -4489,25 +4353,26 @@ void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
 void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
   SDLoc sdl = getCurSDLoc();
 
-  auto getMaskedLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
-                           unsigned& Alignment) {
+  auto getMaskedLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
+                              MaybeAlign &Alignment) {
     // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
     Ptr = I.getArgOperand(0);
-    Alignment = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
+    Alignment =
+        MaybeAlign(cast<ConstantInt>(I.getArgOperand(1))->getZExtValue());
     Mask = I.getArgOperand(2);
     Src0 = I.getArgOperand(3);
   };
-  auto getExpandingLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
-                           unsigned& Alignment) {
+  auto getExpandingLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
+                                 MaybeAlign &Alignment) {
     // @llvm.masked.expandload.*(Ptr, Mask, Src0)
     Ptr = I.getArgOperand(0);
-    Alignment = 0;
+    Alignment = None;
     Mask = I.getArgOperand(1);
     Src0 = I.getArgOperand(2);
   };
 
   Value  *PtrOperand, *MaskOperand, *Src0Operand;
-  unsigned Alignment;
+  MaybeAlign Alignment;
   if (IsExpanding)
     getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
   else
@@ -4520,7 +4385,7 @@ void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
 
   EVT VT = Src0.getValueType();
   if (!Alignment)
-    Alignment = DAG.getEVTAlignment(VT);
+    Alignment = DAG.getEVTAlign(VT);
 
   AAMDNodes AAInfo;
   I.getAAMetadata(AAInfo);
@@ -4538,14 +4403,11 @@ void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
 
   SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
 
-  MachineMemOperand *MMO =
-    DAG.getMachineFunction().
-    getMachineMemOperand(MachinePointerInfo(PtrOperand),
-                          MachineMemOperand::MOLoad,
-                          // TODO: Make MachineMemOperands aware of scalable
-                          // vectors.
-                          VT.getStoreSize().getKnownMinSize(),
-                          Alignment, AAInfo, Ranges);
+  MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
+      MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
+      // TODO: Make MachineMemOperands aware of scalable
+      // vectors.
+      VT.getStoreSize().getKnownMinSize(), *Alignment, AAInfo, Ranges);
 
   SDValue Load =
       DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO,
@@ -4565,9 +4427,9 @@ void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
 
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
-  unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(1)))->getZExtValue();
+  MaybeAlign Alignment(cast<ConstantInt>(I.getArgOperand(1))->getZExtValue());
   if (!Alignment)
-    Alignment = DAG.getEVTAlignment(VT);
+    Alignment = DAG.getEVTAlign(VT);
 
   AAMDNodes AAInfo;
   I.getAAMetadata(AAInfo);
@@ -4580,14 +4442,11 @@ void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
   SDValue Scale;
   bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this);
   unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
-  MachineMemOperand *MMO =
-    DAG.getMachineFunction().
-    getMachineMemOperand(MachinePointerInfo(AS),
-                         MachineMemOperand::MOLoad,
-                         // TODO: Make MachineMemOperands aware of scalable
-                         // vectors.
-                         MemoryLocation::UnknownSize,
-                         Alignment, AAInfo, Ranges);
+  MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
+      MachinePointerInfo(AS), MachineMemOperand::MOLoad,
+      // TODO: Make MachineMemOperands aware of scalable
+      // vectors.
+      MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
 
   if (!UniformBase) {
     Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
@@ -4614,16 +4473,14 @@ void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
   MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
   SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
 
-  auto Alignment = DAG.getEVTAlignment(MemVT);
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());
 
   MachineFunction &MF = DAG.getMachineFunction();
-  MachineMemOperand *MMO =
-    MF.getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
-                            Flags, MemVT.getStoreSize(), Alignment,
-                            AAMDNodes(), nullptr, SSID, SuccessOrdering,
-                            FailureOrdering);
+  MachineMemOperand *MMO = MF.getMachineMemOperand(
+      MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
+      DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, SuccessOrdering,
+      FailureOrdering);
 
   SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
                                    dl, MemVT, VTs, InChain,
@@ -4662,15 +4519,13 @@ void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
   SDValue InChain = getRoot();
 
   auto MemVT = getValue(I.getValOperand()).getSimpleValueType();
-  auto Alignment = DAG.getEVTAlignment(MemVT);
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());
 
   MachineFunction &MF = DAG.getMachineFunction();
-  MachineMemOperand *MMO =
-    MF.getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()), Flags,
-                            MemVT.getStoreSize(), Alignment, AAMDNodes(),
-                            nullptr, SSID, Ordering);
+  MachineMemOperand *MMO = MF.getMachineMemOperand(
+      MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
+      DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, Ordering);
 
   SDValue L =
     DAG.getAtomic(NT, dl, MemVT, InChain,
@@ -4712,13 +4567,10 @@ void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
 
   auto Flags = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout());
 
-  MachineMemOperand *MMO =
-      DAG.getMachineFunction().
-      getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
-                           Flags, MemVT.getStoreSize(),
-                           I.getAlignment() ? I.getAlignment() :
-                                              DAG.getEVTAlignment(MemVT),
-                           AAMDNodes(), nullptr, SSID, Order);
+  MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
+      MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
+      I.getAlign().getValueOr(DAG.getEVTAlign(MemVT)), AAMDNodes(), nullptr,
+      SSID, Order);
 
   InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
 
@@ -4769,10 +4621,9 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
   auto Flags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
 
   MachineFunction &MF = DAG.getMachineFunction();
-  MachineMemOperand *MMO =
-    MF.getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()), Flags,
-                            MemVT.getStoreSize(), I.getAlignment(), AAMDNodes(),
-                            nullptr, SSID, Ordering);
+  MachineMemOperand *MMO = MF.getMachineMemOperand(
+      MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
+      *I.getAlign(), AAMDNodes(), nullptr, SSID, Ordering);
 
   SDValue Val = getValue(I.getValueOperand());
   if (Val.getValueType() != MemVT)
@@ -4862,10 +4713,10 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
     // This is target intrinsic that touches memory
     AAMDNodes AAInfo;
     I.getAAMetadata(AAInfo);
-    Result = DAG.getMemIntrinsicNode(
-        Info.opc, getCurSDLoc(), VTs, Ops, Info.memVT,
-        MachinePointerInfo(Info.ptrVal, Info.offset),
-        Info.align ? Info.align->value() : 0, Info.flags, Info.size, AAInfo);
+    Result =
+        DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, Ops, Info.memVT,
+                                MachinePointerInfo(Info.ptrVal, Info.offset),
+                                Info.align, Info.flags, Info.size, AAInfo);
   } else if (!HasChain) {
     Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
   } else if (!I.getType()->isVoidTy()) {
@@ -6678,12 +6529,10 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
     Ops[2] = getValue(I.getArgOperand(1));
     Ops[3] = getValue(I.getArgOperand(2));
     Ops[4] = getValue(I.getArgOperand(3));
-    SDValue Result = DAG.getMemIntrinsicNode(ISD::PREFETCH, sdl,
-                                             DAG.getVTList(MVT::Other), Ops,
-                                             EVT::getIntegerVT(*Context, 8),
-                                             MachinePointerInfo(I.getArgOperand(0)),
-                                             0, /* align */
-                                             Flags);
+    SDValue Result = DAG.getMemIntrinsicNode(
+        ISD::PREFETCH, sdl, DAG.getVTList(MVT::Other), Ops,
+        EVT::getIntegerVT(*Context, 8), MachinePointerInfo(I.getArgOperand(0)),
+        /* align */ None, Flags);
 
     // Chain the prefetch in parallell with any pending loads, to stay out of
     // the way of later optimizations.
@@ -7484,10 +7333,10 @@ bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
   SDValue Src = getValue(I.getArgOperand(1));
   SDValue Size = getValue(I.getArgOperand(2));
 
-  unsigned DstAlign = DAG.InferPtrAlignment(Dst);
-  unsigned SrcAlign = DAG.InferPtrAlignment(Src);
+  Align DstAlign = DAG.InferPtrAlign(Dst).valueOrOne();
+  Align SrcAlign = DAG.InferPtrAlign(Src).valueOrOne();
   // DAG::getMemcpy needs Alignment to be defined.
-  Align Alignment = assumeAligned(std::min(DstAlign, SrcAlign));
+  Align Alignment = std::min(DstAlign, SrcAlign);
 
   bool isVol = false;
   SDLoc sdl = getCurSDLoc();
@@ -9130,9 +8979,10 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
     // assert(!CS.hasInAllocaArgument() &&
     //        "sret demotion is incompatible with inalloca");
     uint64_t TySize = DL.getTypeAllocSize(CLI.RetTy);
-    unsigned Align = DL.getPrefTypeAlignment(CLI.RetTy);
+    Align Alignment = DL.getPrefTypeAlign(CLI.RetTy);
     MachineFunction &MF = CLI.DAG.getMachineFunction();
-    DemoteStackIdx = MF.getFrameInfo().CreateStackObject(TySize, Align, false);
+    DemoteStackIdx =
+        MF.getFrameInfo().CreateStackObject(TySize, Alignment, false);
     Type *StackSlotPtrType = PointerType::get(CLI.RetTy,
                                               DL.getAllocaAddrSpace());
 
@@ -9150,7 +9000,7 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
     Entry.IsSwiftSelf = false;
     Entry.IsSwiftError = false;
     Entry.IsCFGuardTarget = false;
-    Entry.Alignment = Align;
+    Entry.Alignment = Alignment;
     CLI.getArgs().insert(CLI.getArgs().begin(), Entry);
     CLI.NumFixedArgs += 1;
     CLI.RetTy = Type::getVoidTy(CLI.RetTy->getContext());
@@ -9284,12 +9134,12 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
         Flags.setByValSize(FrameSize);
 
         // info is not there but there are cases it cannot get right.
-        unsigned FrameAlign;
-        if (Args[i].Alignment)
-          FrameAlign = Args[i].Alignment;
+        Align FrameAlign;
+        if (auto MA = Args[i].Alignment)
+          FrameAlign = *MA;
         else
-          FrameAlign = getByValTypeAlignment(ElementTy, DL);
-        Flags.setByValAlign(Align(FrameAlign));
+          FrameAlign = Align(getByValTypeAlignment(ElementTy, DL));
+        Flags.setByValAlign(FrameAlign);
       }
       if (Args[i].IsNest)
         Flags.setNest();
@@ -9643,16 +9493,13 @@ static void tryToElideArgumentCopy(
                   "object size\n");
     return;
   }
-  unsigned RequiredAlignment = AI->getAlignment();
-  if (!RequiredAlignment) {
-    RequiredAlignment = FuncInfo.MF->getDataLayout().getABITypeAlignment(
-        AI->getAllocatedType());
-  }
-  if (MFI.getObjectAlignment(FixedIndex) < RequiredAlignment) {
+  Align RequiredAlignment = AI->getAlign().getValueOr(
+      FuncInfo.MF->getDataLayout().getABITypeAlign(AI->getAllocatedType()));
+  if (MFI.getObjectAlign(FixedIndex) < RequiredAlignment) {
     LLVM_DEBUG(dbgs() << "  argument copy elision failed: alignment of alloca "
                          "greater than stack argument alignment ("
-                      << RequiredAlignment << " vs "
-                      << MFI.getObjectAlignment(FixedIndex) << ")\n");
+                      << RequiredAlignment.value() << " vs "
+                      << MFI.getObjectAlign(FixedIndex).value() << ")\n");
     return;
   }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index aca462f566746..f81d18c6ad521 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -553,9 +553,6 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const {
   if (getFlags().hasAllowReassociation())
     OS << " reassoc";
 
-  if (getFlags().hasVectorReduction())
-    OS << " vector-reduction";
-
   if (getFlags().hasNoFPExcept())
     OS << " nofpexcept";
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 6fa6bde047c54..6626210e9185b 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -2239,14 +2239,14 @@ bool SelectionDAGISel::IsLegalToFold(SDValue N, SDNode *U, SDNode *Root,
   return !findNonImmUse(Root, N.getNode(), U, IgnoreChains);
 }
 
-void SelectionDAGISel::Select_INLINEASM(SDNode *N, bool Branch) {
+void SelectionDAGISel::Select_INLINEASM(SDNode *N) {
   SDLoc DL(N);
 
   std::vector<SDValue> Ops(N->op_begin(), N->op_end());
   SelectInlineAsmMemoryOperands(Ops, DL);
 
   const EVT VTs[] = {MVT::Other, MVT::Glue};
-  SDValue New = CurDAG->getNode(Branch ? ISD::INLINEASM_BR : ISD::INLINEASM, DL, VTs, Ops);
+  SDValue New = CurDAG->getNode(N->getOpcode(), DL, VTs, Ops);
   New->setNodeId(-1);
   ReplaceUses(N, New.getNode());
   CurDAG->RemoveDeadNode(N);
@@ -2822,8 +2822,7 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
     return;
   case ISD::INLINEASM:
   case ISD::INLINEASM_BR:
-    Select_INLINEASM(NodeToMatch,
-                     NodeToMatch->getOpcode() == ISD::INLINEASM_BR);
+    Select_INLINEASM(NodeToMatch);
     return;
   case ISD::READ_REGISTER:
     Select_READ_REGISTER(NodeToMatch);
@@ -3709,12 +3708,11 @@ bool SelectionDAGISel::isOrEquivalentToAdd(const SDNode *N) const {
   // Detect when "or" is used to add an offset to a stack object.
   if (auto *FN = dyn_cast<FrameIndexSDNode>(N->getOperand(0))) {
     MachineFrameInfo &MFI = MF->getFrameInfo();
-    unsigned A = MFI.getObjectAlignment(FN->getIndex());
-    assert(isPowerOf2_32(A) && "Unexpected alignment");
+    Align A = MFI.getObjectAlign(FN->getIndex());
     int32_t Off = C->getSExtValue();
     // If the alleged offset fits in the zero bits guaranteed by
     // the alignment, then this or is really an add.
-    return (Off >= 0) && (((A - 1) & Off) == unsigned(Off));
+    return (Off >= 0) && (((A.value() - 1) & Off) == unsigned(Off));
   }
   return false;
 }
diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
index 012cc23ca7e48..f292d510f65d2 100644
--- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
@@ -314,9 +314,9 @@ static MachineMemOperand* getMachineMemOperand(MachineFunction &MF,
   auto MMOFlags = MachineMemOperand::MOStore |
     MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile;
   auto &MFI = MF.getFrameInfo();
-  return MF.getMachineMemOperand(PtrInfo, MMOFlags, 
+  return MF.getMachineMemOperand(PtrInfo, MMOFlags,
                                  MFI.getObjectSize(FI.getIndex()),
-                                 MFI.getObjectAlignment(FI.getIndex()));
+                                 MFI.getObjectAlign(FI.getIndex()));
 }
 
 /// Spill a value incoming to the statepoint. It might be either part of
@@ -354,10 +354,9 @@ spillIncomingStatepointValue(SDValue Incoming, SDValue Chain,
     // slots with preferred alignments larger than frame alignment..
     auto &MF = Builder.DAG.getMachineFunction();
     auto PtrInfo = MachinePointerInfo::getFixedStack(MF, Index);
-    auto *StoreMMO =
-      MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore, 
-                              MFI.getObjectSize(Index),
-                              MFI.getObjectAlignment(Index));
+    auto *StoreMMO = MF.getMachineMemOperand(
+        PtrInfo, MachineMemOperand::MOStore, MFI.getObjectSize(Index),
+        MFI.getObjectAlign(Index));
     Chain = Builder.DAG.getStore(Chain, Builder.getCurSDLoc(), Incoming, Loc,
                                  StoreMMO);
 
@@ -483,8 +482,14 @@ lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
   const bool LiveInDeopt =
     SI.StatepointFlags & (uint64_t)StatepointFlags::DeoptLiveIn;
 
-  auto isGCValue =[&](const Value *V) {
-    return is_contained(SI.Ptrs, V) || is_contained(SI.Bases, V);
+  auto isGCValue = [&](const Value *V) {
+    auto *Ty = V->getType();
+    if (!Ty->isPtrOrPtrVectorTy())
+      return false;
+    if (auto *GFI = Builder.GFI)
+      if (auto IsManaged = GFI->getStrategy().isGCManagedPointer(Ty))
+        return *IsManaged;
+    return true; // conservative
   };
 
   // Before we actually start lowering (and allocating spill slots for values),
@@ -849,8 +854,8 @@ SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP,
 
   SI.GCArgs = ArrayRef<const Use>(ISP.gc_args_begin(), ISP.gc_args_end());
   SI.StatepointInstr = ISP.getInstruction();
-  SI.GCTransitionArgs =
-      ArrayRef<const Use>(ISP.gc_args_begin(), ISP.gc_args_end());
+  SI.GCTransitionArgs = ArrayRef<const Use>(ISP.gc_transition_args_begin(),
+                                            ISP.gc_transition_args_end());
   SI.ID = ISP.getID();
   SI.DeoptState = ArrayRef<const Use>(ISP.deopt_begin(), ISP.deopt_end());
   SI.StatepointFlags = ISP.getFlags();
@@ -1002,10 +1007,9 @@ void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
   auto &MF = DAG.getMachineFunction();
   auto &MFI = MF.getFrameInfo();
   auto PtrInfo = MachinePointerInfo::getFixedStack(MF, Index);
-  auto *LoadMMO =
-    MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad, 
-                            MFI.getObjectSize(Index),
-                            MFI.getObjectAlignment(Index));
+  auto *LoadMMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad,
+                                          MFI.getObjectSize(Index),
+                                          MFI.getObjectAlign(Index));
 
   auto LoadVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
                                                          Relocate.getType());
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 3f0c6443211ee..7e613570eaf92 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -114,7 +114,7 @@ void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call,
   IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned);
   IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf);
   IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError);
-  Alignment = Call->getParamAlignment(ArgIdx);
+  Alignment = Call->getParamAlign(ArgIdx);
   ByValType = nullptr;
   if (Call->paramHasAttr(ArgIdx, Attribute::ByVal))
     ByValType = Call->getParamByValType(ArgIdx);
@@ -2762,9 +2762,9 @@ void TargetLowering::computeKnownBitsForFrameIndex(const SDValue Op,
                                                    unsigned Depth) const {
   assert(isa<FrameIndexSDNode>(Op) && "expected FrameIndex");
 
-  if (unsigned Align = DAG.InferPtrAlignment(Op)) {
+  if (MaybeAlign Alignment = DAG.InferPtrAlign(Op)) {
     // The low bits are known zero if the pointer is aligned.
-    Known.Zero.setLowBits(Log2_32(Align));
+    Known.Zero.setLowBits(Log2(*Alignment));
   }
 }
 
diff --git a/llvm/lib/CodeGen/StackColoring.cpp b/llvm/lib/CodeGen/StackColoring.cpp
index 9d4fdc6b624cf..5ed5e8a46b08b 100644
--- a/llvm/lib/CodeGen/StackColoring.cpp
+++ b/llvm/lib/CodeGen/StackColoring.cpp
@@ -1290,8 +1290,8 @@ bool StackColoring::runOnMachineFunction(MachineFunction &Func) {
           SortedSlots[J] = -1;
           LLVM_DEBUG(dbgs() << "Merging #" << FirstSlot << " and slots #"
                             << SecondSlot << " together.\n");
-          unsigned MaxAlignment = std::max(MFI->getObjectAlignment(FirstSlot),
-                                           MFI->getObjectAlignment(SecondSlot));
+          Align MaxAlignment = std::max(MFI->getObjectAlign(FirstSlot),
+                                        MFI->getObjectAlign(SecondSlot));
 
           assert(MFI->getObjectSize(FirstSlot) >=
                  MFI->getObjectSize(SecondSlot) &&
diff --git a/llvm/lib/CodeGen/StackSlotColoring.cpp b/llvm/lib/CodeGen/StackSlotColoring.cpp
index 7ae7583232800..3cc5d30ebad7d 100644
--- a/llvm/lib/CodeGen/StackSlotColoring.cpp
+++ b/llvm/lib/CodeGen/StackSlotColoring.cpp
@@ -74,7 +74,7 @@ namespace {
     SmallVector<SmallVector<MachineMemOperand *, 8>, 16> SSRefs;
 
     // OrigAlignments - Alignments of stack objects before coloring.
-    SmallVector<unsigned, 16> OrigAlignments;
+    SmallVector<Align, 16> OrigAlignments;
 
     // OrigSizes - Sizess of stack objects before coloring.
     SmallVector<unsigned, 16> OrigSizes;
@@ -227,7 +227,7 @@ void StackSlotColoring::InitializeSlots() {
       continue;
 
     SSIntervals.push_back(&li);
-    OrigAlignments[FI] = MFI->getObjectAlignment(FI);
+    OrigAlignments[FI] = MFI->getObjectAlign(FI);
     OrigSizes[FI]      = MFI->getObjectSize(FI);
 
     auto StackID = MFI->getStackID(FI);
@@ -309,9 +309,9 @@ int StackSlotColoring::ColorSlot(LiveInterval *li) {
   // Change size and alignment of the allocated slot. If there are multiple
   // objects sharing the same slot, then make sure the size and alignment
   // are large enough for all.
-  unsigned Align = OrigAlignments[FI];
-  if (!Share || Align > MFI->getObjectAlignment(Color))
-    MFI->setObjectAlignment(Color, Align);
+  Align Alignment = OrigAlignments[FI];
+  if (!Share || Alignment > MFI->getObjectAlign(Color))
+    MFI->setObjectAlignment(Color, Alignment);
   int64_t Size = OrigSizes[FI];
   if (!Share || Size > MFI->getObjectSize(Color))
     MFI->setObjectSize(Color, Size);
diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp
index cc390cfbec198..2e515094cf6cb 100644
--- a/llvm/lib/CodeGen/TargetInstrInfo.cpp
+++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp
@@ -408,7 +408,7 @@ bool TargetInstrInfo::getStackSlotRange(const TargetRegisterClass *RC,
 
 void TargetInstrInfo::reMaterialize(MachineBasicBlock &MBB,
                                     MachineBasicBlock::iterator I,
-                                    unsigned DestReg, unsigned SubIdx,
+                                    Register DestReg, unsigned SubIdx,
                                     const MachineInstr &Orig,
                                     const TargetRegisterInfo &TRI) const {
   MachineInstr *MI = MBB.getParent()->CloneMachineInstr(&Orig);
@@ -591,9 +591,9 @@ MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineInstr &MI,
             NewMI->mayLoad()) &&
            "Folded a use to a non-load!");
     assert(MFI.getObjectOffset(FI) != -1);
-    MachineMemOperand *MMO = MF.getMachineMemOperand(
-        MachinePointerInfo::getFixedStack(MF, FI), Flags, MemSize,
-        MFI.getObjectAlignment(FI));
+    MachineMemOperand *MMO =
+        MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(MF, FI),
+                                Flags, MemSize, MFI.getObjectAlign(FI));
     NewMI->addMemOperand(MF, MMO);
 
     return NewMI;
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 27440b2dd02c3..e5caec4181583 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -1066,7 +1066,7 @@ TargetLoweringBase::emitPatchPoint(MachineInstr &InitialMI,
       auto Flags = MachineMemOperand::MOLoad;
       MachineMemOperand *MMO = MF.getMachineMemOperand(
           MachinePointerInfo::getFixedStack(MF, FI), Flags,
-          MF.getDataLayout().getPointerSize(), MFI.getObjectAlignment(FI));
+          MF.getDataLayout().getPointerSize(), MFI.getObjectAlign(FI));
       MIB->addMemOperand(MF, MMO);
     }
 
@@ -1530,7 +1530,7 @@ void llvm::GetReturnInfo(CallingConv::ID CC, Type *ReturnType,
 /// alignment, not its logarithm.
 unsigned TargetLoweringBase::getByValTypeAlignment(Type *Ty,
                                                    const DataLayout &DL) const {
-  return DL.getABITypeAlignment(Ty);
+  return DL.getABITypeAlign(Ty).value();
 }
 
 bool TargetLoweringBase::allowsMemoryAccessForAlignment(
@@ -1542,7 +1542,7 @@ bool TargetLoweringBase::allowsMemoryAccessForAlignment(
   // For example, the ABI alignment may change based on software platform while
   // this function should only be affected by hardware implementation.
   Type *Ty = VT.getTypeForEVT(Context);
-  if (Alignment >= DL.getABITypeAlignment(Ty)) {
+  if (Alignment >= DL.getABITypeAlign(Ty).value()) {
     // Assume that an access that meets the ABI-specified alignment is fast.
     if (Fast != nullptr)
       *Fast = true;
@@ -1557,7 +1557,7 @@ bool TargetLoweringBase::allowsMemoryAccessForAlignment(
     LLVMContext &Context, const DataLayout &DL, EVT VT,
     const MachineMemOperand &MMO, bool *Fast) const {
   return allowsMemoryAccessForAlignment(Context, DL, VT, MMO.getAddrSpace(),
-                                        MMO.getAlignment(), MMO.getFlags(),
+                                        MMO.getAlign().value(), MMO.getFlags(),
                                         Fast);
 }
 
@@ -1573,7 +1573,7 @@ bool TargetLoweringBase::allowsMemoryAccess(LLVMContext &Context,
                                             const MachineMemOperand &MMO,
                                             bool *Fast) const {
   return allowsMemoryAccess(Context, DL, VT, MMO.getAddrSpace(),
-                            MMO.getAlignment(), MMO.getFlags(), Fast);
+                            MMO.getAlign().value(), MMO.getFlags(), Fast);
 }
 
 BranchProbability TargetLoweringBase::getPredictableBranchThreshold() const {
diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
index dedb6735b4708..cc54b41171d45 100644
--- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
@@ -2022,9 +2022,11 @@ XCOFF::StorageClass TargetLoweringObjectFileXCOFF::getStorageClassForGlobal(
 }
 
 MCSection *TargetLoweringObjectFileXCOFF::getSectionForFunctionDescriptor(
-    const MCSymbol *FuncSym) const {
-  return getContext().getXCOFFSection(FuncSym->getName(), XCOFF::XMC_DS,
-                                      XCOFF::XTY_SD, XCOFF::C_HIDEXT,
+    const Function *F, const TargetMachine &TM) const {
+  SmallString<128> NameStr;
+  getNameWithPrefix(NameStr, F, TM);
+  return getContext().getXCOFFSection(NameStr, XCOFF::XMC_DS, XCOFF::XTY_SD,
+                                      getStorageClassForGlobal(F),
                                       SectionKind::getData());
 }
 
diff --git a/llvm/lib/CodeGen/ValueTypes.cpp b/llvm/lib/CodeGen/ValueTypes.cpp
index f93c4b87729b6..eac07be08a818 100644
--- a/llvm/lib/CodeGen/ValueTypes.cpp
+++ b/llvm/lib/CodeGen/ValueTypes.cpp
@@ -139,7 +139,8 @@ std::string EVT::getEVTString() const {
   switch (V.SimpleTy) {
   default:
     if (isVector())
-      return (isScalableVector() ? "nxv" : "v") + utostr(getVectorNumElements())
+      return (isScalableVector() ? "nxv" : "v")
+             + utostr(getVectorElementCount().Min)
              + getVectorElementType().getEVTString();
     if (isInteger())
       return "i" + utostr(getSizeInBits());
diff --git a/llvm/lib/CodeGen/VirtRegMap.cpp b/llvm/lib/CodeGen/VirtRegMap.cpp
index 5312e2eea96b4..2c83f13b651b7 100644
--- a/llvm/lib/CodeGen/VirtRegMap.cpp
+++ b/llvm/lib/CodeGen/VirtRegMap.cpp
@@ -92,8 +92,8 @@ void VirtRegMap::assignVirt2Phys(Register virtReg, MCPhysReg physReg) {
 
 unsigned VirtRegMap::createSpillSlot(const TargetRegisterClass *RC) {
   unsigned Size = TRI->getSpillSize(*RC);
-  unsigned Align = TRI->getSpillAlignment(*RC);
-  int SS = MF->getFrameInfo().CreateSpillStackObject(Size, Align);
+  Align Alignment = TRI->getSpillAlign(*RC);
+  int SS = MF->getFrameInfo().CreateSpillStackObject(Size, Alignment);
   ++NumSpillSlots;
   return SS;
 }
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp b/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp
index 7541bb794c73b..2150df411f421 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp
@@ -292,6 +292,36 @@ static void dumpRnglistsSection(
   }
 }
 
+std::unique_ptr<DWARFDebugMacro>
+DWARFContext::parseMacroOrMacinfo(MacroSecType SectionType) {
+  auto Macro = std::make_unique<DWARFDebugMacro>();
+  auto ParseAndDump = [&](DWARFDataExtractor &Data, bool IsMacro) {
+    if (Error Err = Macro->parse(getStringExtractor(), Data, IsMacro)) {
+      RecoverableErrorHandler(std::move(Err));
+      Macro = nullptr;
+    }
+  };
+  switch (SectionType) {
+  case MacinfoSection: {
+    DWARFDataExtractor Data(DObj->getMacinfoSection(), isLittleEndian(), 0);
+    ParseAndDump(Data, /*IsMacro=*/false);
+    break;
+  }
+  case MacinfoDwoSection: {
+    DWARFDataExtractor Data(DObj->getMacinfoDWOSection(), isLittleEndian(), 0);
+    ParseAndDump(Data, /*IsMacro=*/false);
+    break;
+  }
+  case MacroSection: {
+    DWARFDataExtractor Data(*DObj, DObj->getMacroSection(), isLittleEndian(),
+                            0);
+    ParseAndDump(Data, /*IsMacro=*/true);
+    break;
+  }
+  }
+  return Macro;
+}
+
 static void dumpLoclistsSection(raw_ostream &OS, DIDumpOptions DumpOpts,
                                 DWARFDataExtractor Data,
                                 const MCRegisterInfo *MRI,
@@ -444,14 +474,22 @@ void DWARFContext::dump(
                                    DObj->getEHFrameSection().Data))
     getEHFrame()->dump(OS, getRegisterInfo(), *Off);
 
+  if (shouldDump(Explicit, ".debug_macro", DIDT_ID_DebugMacro,
+                 DObj->getMacroSection().Data)) {
+    if (auto Macro = getDebugMacro())
+      Macro->dump(OS);
+  }
+
   if (shouldDump(Explicit, ".debug_macinfo", DIDT_ID_DebugMacro,
                  DObj->getMacinfoSection())) {
-    getDebugMacinfo()->dump(OS);
+    if (auto Macinfo = getDebugMacinfo())
+      Macinfo->dump(OS);
   }
 
   if (shouldDump(Explicit, ".debug_macinfo.dwo", DIDT_ID_DebugMacro,
                  DObj->getMacinfoDWOSection())) {
-    getDebugMacinfoDWO()->dump(OS);
+    if (auto MacinfoDWO = getDebugMacinfoDWO())
+      MacinfoDWO->dump(OS);
   }
 
   if (shouldDump(Explicit, ".debug_aranges", DIDT_ID_DebugAranges,
@@ -815,27 +853,24 @@ const DWARFDebugFrame *DWARFContext::getEHFrame() {
   return DebugFrame.get();
 }
 
-const DWARFDebugMacro *DWARFContext::getDebugMacinfoDWO() {
-  if (MacinfoDWO)
-    return MacinfoDWO.get();
-
-  DataExtractor MacinfoDWOData(DObj->getMacinfoDWOSection(), isLittleEndian(),
-                               0);
-  MacinfoDWO.reset(new DWARFDebugMacro());
-  MacinfoDWO->parse(MacinfoDWOData);
-  return MacinfoDWO.get();
+const DWARFDebugMacro *DWARFContext::getDebugMacro() {
+  if (!Macro)
+    Macro = parseMacroOrMacinfo(MacroSection);
+  return Macro.get();
 }
 
 const DWARFDebugMacro *DWARFContext::getDebugMacinfo() {
-  if (Macinfo)
-    return Macinfo.get();
-
-  DataExtractor MacinfoData(DObj->getMacinfoSection(), isLittleEndian(), 0);
-  Macinfo.reset(new DWARFDebugMacro());
-  Macinfo->parse(MacinfoData);
+  if (!Macinfo)
+    Macinfo = parseMacroOrMacinfo(MacinfoSection);
   return Macinfo.get();
 }
 
+const DWARFDebugMacro *DWARFContext::getDebugMacinfoDWO() {
+  if (!MacinfoDWO)
+    MacinfoDWO = parseMacroOrMacinfo(MacinfoDwoSection);
+  return MacinfoDWO.get();
+}
+
 template <typename T>
 static T &getAccelTable(std::unique_ptr<T> &Cache, const DWARFObject &Obj,
                         const DWARFSection &Section, StringRef StringSection,
@@ -1476,6 +1511,7 @@ class DWARFObjInMemory final : public DWARFObject {
   DWARFSectionMap PubtypesSection;
   DWARFSectionMap GnuPubnamesSection;
   DWARFSectionMap GnuPubtypesSection;
+  DWARFSectionMap MacroSection;
 
   DWARFSectionMap *mapNameToDWARFSection(StringRef Name) {
     return StringSwitch<DWARFSectionMap *>(Name)
@@ -1503,6 +1539,7 @@ class DWARFObjInMemory final : public DWARFObject {
         .Case("apple_namespaces", &AppleNamespacesSection)
         .Case("apple_namespac", &AppleNamespacesSection)
         .Case("apple_objc", &AppleObjCSection)
+        .Case("debug_macro", &MacroSection)
         .Default(nullptr);
   }
 
@@ -1847,6 +1884,7 @@ class DWARFObjInMemory final : public DWARFObject {
   const DWARFSection &getRnglistsSection() const override {
     return RnglistsSection;
   }
+  const DWARFSection &getMacroSection() const override { return MacroSection; }
   StringRef getMacinfoSection() const override { return MacinfoSection; }
   StringRef getMacinfoDWOSection() const override { return MacinfoDWOSection; }
   const DWARFSection &getPubnamesSection() const override { return PubnamesSection; }
@@ -1931,14 +1969,9 @@ Error DWARFContext::loadRegisterInfo(const object::ObjectFile &Obj) {
 uint8_t DWARFContext::getCUAddrSize() {
   // In theory, different compile units may have different address byte
   // sizes, but for simplicity we just use the address byte size of the
-  // last compile unit. In practice the address size field is repeated across
+  // first compile unit. In practice the address size field is repeated across
   // various DWARF headers (at least in version 5) to make it easier to dump
   // them independently, not to enable varying the address size.
-  uint8_t Addr = 0;
-  for (const auto &CU : compile_units()) {
-    Addr = CU->getAddressByteSize();
-    break;
-  }
-  return Addr;
+  unit_iterator_range CUs = compile_units();
+  return CUs.empty() ? 0 : (*CUs.begin())->getAddressByteSize();
 }
-
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFDebugMacro.cpp b/llvm/lib/DebugInfo/DWARF/DWARFDebugMacro.cpp
index 2e06e14ee3be3..09238e316173d 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFDebugMacro.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFDebugMacro.cpp
@@ -8,6 +8,8 @@
 
 #include "llvm/DebugInfo/DWARF/DWARFDebugMacro.h"
 #include "llvm/BinaryFormat/Dwarf.h"
+#include "llvm/DebugInfo/DWARF/DWARFContext.h"
+#include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h"
 #include "llvm/Support/WithColor.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cstdint>
@@ -15,10 +17,22 @@
 using namespace llvm;
 using namespace dwarf;
 
+void DWARFDebugMacro::MacroHeader::dumpMacroHeader(raw_ostream &OS) const {
+  // FIXME: Add support for dumping opcode_operands_table
+  OS << format("macro header: version = 0x%04" PRIx16 ", flags = 0x%02" PRIx8,
+               Version, Flags);
+  if (Flags & MACRO_DEBUG_LINE_OFFSET)
+    OS << format(", debug_line_offset = 0x%04" PRIx64 "\n", DebugLineOffset);
+  else
+    OS << "\n";
+}
+
 void DWARFDebugMacro::dump(raw_ostream &OS) const {
   unsigned IndLevel = 0;
   for (const auto &Macros : MacroLists) {
     OS << format("0x%08" PRIx64 ":\n", Macros.Offset);
+    if (Macros.Header.Version >= 5)
+      Macros.Header.dumpMacroHeader(OS);
     for (const Entry &E : Macros.Macros) {
       // There should not be DW_MACINFO_end_file when IndLevel is Zero. However,
       // this check handles the case of corrupted ".debug_macinfo" section.
@@ -28,22 +42,34 @@ void DWARFDebugMacro::dump(raw_ostream &OS) const {
       for (unsigned I = 0; I < IndLevel; I++)
         OS << "  ";
       IndLevel += (E.Type == DW_MACINFO_start_file);
-
-      WithColor(OS, HighlightColor::Macro).get() << MacinfoString(E.Type);
+      // Based on which version we are handling choose appropriate macro forms.
+      if (Macros.Header.Version >= 5)
+        WithColor(OS, HighlightColor::Macro).get() << MacroString(E.Type);
+      else
+        WithColor(OS, HighlightColor::Macro).get() << MacinfoString(E.Type);
       switch (E.Type) {
       default:
-        // Got a corrupted ".debug_macinfo" section (invalid macinfo type).
+        // Got a corrupted ".debug_macinfo/.debug_macro" section (invalid
+        // macinfo type).
         break;
-      case DW_MACINFO_define:
-      case DW_MACINFO_undef:
+        // debug_macro and debug_macinfo share some common encodings.
+        // DW_MACRO_define     == DW_MACINFO_define
+        // DW_MACRO_undef      == DW_MACINFO_undef
+        // DW_MACRO_start_file == DW_MACINFO_start_file
+        // DW_MACRO_end_file   == DW_MACINFO_end_file
+        // For readability/uniformity we are using DW_MACRO_*.
+      case DW_MACRO_define:
+      case DW_MACRO_undef:
+      case DW_MACRO_define_strp:
+      case DW_MACRO_undef_strp:
         OS << " - lineno: " << E.Line;
         OS << " macro: " << E.MacroStr;
         break;
-      case DW_MACINFO_start_file:
+      case DW_MACRO_start_file:
         OS << " - lineno: " << E.Line;
         OS << " filenum: " << E.File;
         break;
-      case DW_MACINFO_end_file:
+      case DW_MACRO_end_file:
         break;
       case DW_MACINFO_vendor_ext:
         OS << " - constant: " << E.ExtConstant;
@@ -55,23 +81,29 @@ void DWARFDebugMacro::dump(raw_ostream &OS) const {
   }
 }
 
-void DWARFDebugMacro::parse(DataExtractor data) {
+Error DWARFDebugMacro::parse(DataExtractor StringExtractor,
+                             DWARFDataExtractor Data, bool IsMacro) {
   uint64_t Offset = 0;
   MacroList *M = nullptr;
-  while (data.isValidOffset(Offset)) {
+  while (Data.isValidOffset(Offset)) {
     if (!M) {
       MacroLists.emplace_back();
       M = &MacroLists.back();
       M->Offset = Offset;
+      if (IsMacro) {
+        auto Err = M->Header.parseMacroHeader(Data, &Offset);
+        if (Err)
+          return Err;
+      }
     }
     // A macro list entry consists of:
     M->Macros.emplace_back();
     Entry &E = M->Macros.back();
     // 1. Macinfo type
-    E.Type = data.getULEB128(&Offset);
+    E.Type = Data.getULEB128(&Offset);
 
     if (E.Type == 0) {
-      // Reached end of a ".debug_macinfo" section contribution.
+      // Reached end of a ".debug_macinfo/debug_macro" section contribution.
       M = nullptr;
       continue;
     }
@@ -81,28 +113,64 @@ void DWARFDebugMacro::parse(DataExtractor data) {
       // Got a corrupted ".debug_macinfo" section (invalid macinfo type).
       // Push the corrupted entry to the list and halt parsing.
       E.Type = DW_MACINFO_invalid;
-      return;
-    case DW_MACINFO_define:
-    case DW_MACINFO_undef:
+      return Error::success();
+    // debug_macro and debug_macinfo share some common encodings.
+    // DW_MACRO_define     == DW_MACINFO_define
+    // DW_MACRO_undef      == DW_MACINFO_undef
+    // DW_MACRO_start_file == DW_MACINFO_start_file
+    // DW_MACRO_end_file   == DW_MACINFO_end_file
+    // For readibility/uniformity we are using DW_MACRO_*.
+    case DW_MACRO_define:
+    case DW_MACRO_undef:
       // 2. Source line
-      E.Line = data.getULEB128(&Offset);
+      E.Line = Data.getULEB128(&Offset);
       // 3. Macro string
-      E.MacroStr = data.getCStr(&Offset);
+      E.MacroStr = Data.getCStr(&Offset);
       break;
-    case DW_MACINFO_start_file:
+    case DW_MACRO_define_strp:
+    case DW_MACRO_undef_strp: {
+      uint64_t StrOffset = 0;
       // 2. Source line
-      E.Line = data.getULEB128(&Offset);
+      E.Line = Data.getULEB128(&Offset);
+      // 3. Macro string
+      // FIXME: Add support for DWARF64
+      StrOffset = Data.getRelocatedValue(/*OffsetSize=*/4, &Offset);
+      E.MacroStr = StringExtractor.getCStr(&StrOffset);
+      break;
+    }
+    case DW_MACRO_start_file:
+      // 2. Source line
+      E.Line = Data.getULEB128(&Offset);
       // 3. Source file id
-      E.File = data.getULEB128(&Offset);
+      E.File = Data.getULEB128(&Offset);
       break;
-    case DW_MACINFO_end_file:
+    case DW_MACRO_end_file:
       break;
     case DW_MACINFO_vendor_ext:
       // 2. Vendor extension constant
-      E.ExtConstant = data.getULEB128(&Offset);
+      E.ExtConstant = Data.getULEB128(&Offset);
       // 3. Vendor extension string
-      E.ExtStr = data.getCStr(&Offset);
+      E.ExtStr = Data.getCStr(&Offset);
       break;
     }
   }
+  return Error::success();
+}
+
+Error DWARFDebugMacro::MacroHeader::parseMacroHeader(DWARFDataExtractor Data,
+                                                     uint64_t *Offset) {
+  Version = Data.getU16(Offset);
+  uint8_t FlagData = Data.getU8(Offset);
+  // FIXME: Add support for DWARF64
+  if (FlagData & MACRO_OFFSET_SIZE)
+    return createStringError(errc::not_supported, "DWARF64 is not supported");
+
+  // FIXME: Add support for parsing opcode_operands_table
+  if (FlagData & MACRO_OPCODE_OPERANDS_TABLE)
+    return createStringError(errc::not_supported,
+                             "opcode_operands_table is not supported");
+  Flags = FlagData;
+  if (Flags & MACRO_DEBUG_LINE_OFFSET)
+    DebugLineOffset = Data.getU32(Offset);
+  return Error::success();
 }
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp b/llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp
index 9e71ef5f43f54..80f09dbd4c18c 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp
@@ -139,7 +139,7 @@ DWARFUnit *DWARFUnitVector::getUnitForOffset(uint64_t Offset) const {
 
 DWARFUnit *
 DWARFUnitVector::getUnitForIndexEntry(const DWARFUnitIndex::Entry &E) {
-  const auto *CUOff = E.getOffset(DW_SECT_INFO);
+  const auto *CUOff = E.getContribution(DW_SECT_INFO);
   if (!CUOff)
     return nullptr;
 
@@ -183,7 +183,7 @@ DWARFUnit::DWARFUnit(DWARFContext &DC, const DWARFSection &Section,
     // data based on the index entries.
     StringRef Data = LocSection->Data;
     if (auto *IndexEntry = Header.getIndexEntry())
-      if (const auto *C = IndexEntry->getOffset(DW_SECT_LOC))
+      if (const auto *C = IndexEntry->getContribution(DW_SECT_LOC))
         Data = Data.substr(C->Offset, C->Length);
 
     DWARFDataExtractor DWARFData =
@@ -294,11 +294,11 @@ bool DWARFUnitHeader::extract(DWARFContext &Context,
   if (IndexEntry) {
     if (AbbrOffset)
       return false;
-    auto *UnitContrib = IndexEntry->getOffset();
+    auto *UnitContrib = IndexEntry->getContribution();
     if (!UnitContrib ||
         UnitContrib->Length != (Length + getUnitLengthFieldByteSize()))
       return false;
-    auto *AbbrEntry = IndexEntry->getOffset(DW_SECT_ABBREV);
+    auto *AbbrEntry = IndexEntry->getContribution(DW_SECT_ABBREV);
     if (!AbbrEntry)
       return false;
     AbbrOffset = AbbrEntry->Offset;
@@ -966,7 +966,7 @@ DWARFUnit::determineStringOffsetsTableContributionDWO(DWARFDataExtractor & DA) {
   uint64_t Offset = 0;
   auto IndexEntry = Header.getIndexEntry();
   const auto *C =
-      IndexEntry ? IndexEntry->getOffset(DW_SECT_STR_OFFSETS) : nullptr;
+      IndexEntry ? IndexEntry->getContribution(DW_SECT_STR_OFFSETS) : nullptr;
   if (C)
     Offset = C->Offset;
   if (getVersion() >= 5) {
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFUnitIndex.cpp b/llvm/lib/DebugInfo/DWARF/DWARFUnitIndex.cpp
index 2780abfcf03fe..bdfdb1889c025 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFUnitIndex.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFUnitIndex.cpp
@@ -154,7 +154,7 @@ void DWARFUnitIndex::dump(raw_ostream &OS) const {
 }
 
 const DWARFUnitIndex::Entry::SectionContribution *
-DWARFUnitIndex::Entry::getOffset(DWARFSectionKind Sec) const {
+DWARFUnitIndex::Entry::getContribution(DWARFSectionKind Sec) const {
   uint32_t i = 0;
   for (; i != Index->Header.NumColumns; ++i)
     if (Index->ColumnKinds[i] == Sec)
@@ -163,7 +163,7 @@ DWARFUnitIndex::Entry::getOffset(DWARFSectionKind Sec) const {
 }
 
 const DWARFUnitIndex::Entry::SectionContribution *
-DWARFUnitIndex::Entry::getOffset() const {
+DWARFUnitIndex::Entry::getContribution() const {
   return &Contributions[Index->InfoColumn];
 }
 
diff --git a/llvm/lib/ExecutionEngine/Interpreter/Execution.cpp b/llvm/lib/ExecutionEngine/Interpreter/Execution.cpp
index 51f31d3d5d8f6..4a5fb1feb18f5 100644
--- a/llvm/lib/ExecutionEngine/Interpreter/Execution.cpp
+++ b/llvm/lib/ExecutionEngine/Interpreter/Execution.cpp
@@ -1868,7 +1868,6 @@ void Interpreter::visitShuffleVectorInst(ShuffleVectorInst &I){
 
   GenericValue Src1 = getOperandValue(I.getOperand(0), SF);
   GenericValue Src2 = getOperandValue(I.getOperand(1), SF);
-  GenericValue Src3 = getOperandValue(I.getOperand(2), SF);
   GenericValue Dest;
 
   // There is no need to check types of src1 and src2, because the compiled
@@ -1878,7 +1877,7 @@ void Interpreter::visitShuffleVectorInst(ShuffleVectorInst &I){
   Type *TyContained = Ty->getElementType();
   unsigned src1Size = (unsigned)Src1.AggregateVal.size();
   unsigned src2Size = (unsigned)Src2.AggregateVal.size();
-  unsigned src3Size = (unsigned)Src3.AggregateVal.size();
+  unsigned src3Size = I.getShuffleMask().size();
 
   Dest.AggregateVal.resize(src3Size);
 
@@ -1888,7 +1887,7 @@ void Interpreter::visitShuffleVectorInst(ShuffleVectorInst &I){
       break;
     case Type::IntegerTyID:
       for( unsigned i=0; i<src3Size; i++) {
-        unsigned j = Src3.AggregateVal[i].IntVal.getZExtValue();
+        unsigned j = std::max(0, I.getMaskValue(i));
         if(j < src1Size)
           Dest.AggregateVal[i].IntVal = Src1.AggregateVal[j].IntVal;
         else if(j < src1Size + src2Size)
@@ -1904,7 +1903,7 @@ void Interpreter::visitShuffleVectorInst(ShuffleVectorInst &I){
       break;
     case Type::FloatTyID:
       for( unsigned i=0; i<src3Size; i++) {
-        unsigned j = Src3.AggregateVal[i].IntVal.getZExtValue();
+        unsigned j = std::max(0, I.getMaskValue(i));
         if(j < src1Size)
           Dest.AggregateVal[i].FloatVal = Src1.AggregateVal[j].FloatVal;
         else if(j < src1Size + src2Size)
@@ -1915,7 +1914,7 @@ void Interpreter::visitShuffleVectorInst(ShuffleVectorInst &I){
       break;
     case Type::DoubleTyID:
       for( unsigned i=0; i<src3Size; i++) {
-        unsigned j = Src3.AggregateVal[i].IntVal.getZExtValue();
+        unsigned j = std::max(0, I.getMaskValue(i));
         if(j < src1Size)
           Dest.AggregateVal[i].DoubleVal = Src1.AggregateVal[j].DoubleVal;
         else if(j < src1Size + src2Size)
diff --git a/llvm/lib/ExecutionEngine/Orc/Core.cpp b/llvm/lib/ExecutionEngine/Orc/Core.cpp
index 552a7f2ab4f69..97755802bb24b 100644
--- a/llvm/lib/ExecutionEngine/Orc/Core.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Core.cpp
@@ -118,7 +118,13 @@ void AsynchronousSymbolQuery::notifySymbolMetRequiredState(
   assert(I != ResolvedSymbols.end() &&
          "Resolving symbol outside the requested set");
   assert(I->second.getAddress() == 0 && "Redundantly resolving symbol Name");
-  I->second = std::move(Sym);
+
+  // If this is a materialization-side-effects-only symbol then drop it,
+  // otherwise update its map entry with its resolved address.
+  if (Sym.getFlags().hasMaterializationSideEffectsOnly())
+    ResolvedSymbols.erase(I);
+  else
+    I->second = std::move(Sym);
   --OutstandingSymbolsCount;
 }
 
@@ -159,6 +165,14 @@ void AsynchronousSymbolQuery::removeQueryDependence(
     QueryRegistrations.erase(QRI);
 }
 
+void AsynchronousSymbolQuery::dropSymbol(const SymbolStringPtr &Name) {
+  auto I = ResolvedSymbols.find(Name);
+  assert(I != ResolvedSymbols.end() &&
+         "Redundant removal of weakly-referenced symbol");
+  ResolvedSymbols.erase(I);
+  --OutstandingSymbolsCount;
+}
+
 void AsynchronousSymbolQuery::detach() {
   ResolvedSymbols.clear();
   OutstandingSymbolsCount = 0;
@@ -186,6 +200,8 @@ Error MaterializationResponsibility::notifyResolved(const SymbolMap &Symbols) {
     auto I = SymbolFlags.find(KV.first);
     assert(I != SymbolFlags.end() &&
            "Resolving symbol outside this responsibility set");
+    assert(!I->second.hasMaterializationSideEffectsOnly() &&
+           "Can't resolve materialization-side-effects-only symbol");
     assert((KV.second.getFlags() & ~WeakFlags) == (I->second & ~WeakFlags) &&
            "Resolving symbol with incorrect flags");
   }
@@ -398,11 +414,13 @@ void ReExportsMaterializationUnit::materialize(
     SymbolAliasMap Aliases;
   };
 
-  // Build a list of queries to issue. In each round we build the largest set of
-  // aliases that we can resolve without encountering a chain definition of the
-  // form Foo -> Bar, Bar -> Baz. Such a form would deadlock as the query would
-  // be waitin on a symbol that it itself had to resolve. Usually this will just
-  // involve one round and a single query.
+  // Build a list of queries to issue. In each round we build a query for the
+  // largest set of aliases that we can resolve without encountering a chain of
+  // aliases (e.g. Foo -> Bar, Bar -> Baz). Such a chain would deadlock as the
+  // query would be waiting on a symbol that it itself had to resolve. Creating
+  // a new query for each link in such a chain eliminates the possibility of
+  // deadlock. In practice chains are likely to be rare, and this algorithm will
+  // usually result in a single query to issue.
 
   std::vector<std::pair<SymbolLookupSet, std::shared_ptr<OnResolveInfo>>>
       QueryInfos;
@@ -419,7 +437,10 @@ void ReExportsMaterializationUnit::materialize(
         continue;
 
       ResponsibilitySymbols.insert(KV.first);
-      QuerySymbols.add(KV.second.Aliasee);
+      QuerySymbols.add(KV.second.Aliasee,
+                       KV.second.AliasFlags.hasMaterializationSideEffectsOnly()
+                           ? SymbolLookupFlags::WeaklyReferencedSymbol
+                           : SymbolLookupFlags::RequiredSymbol);
       QueryAliases[KV.first] = std::move(KV.second);
     }
 
@@ -468,8 +489,13 @@ void ReExportsMaterializationUnit::materialize(
       if (Result) {
         SymbolMap ResolutionMap;
         for (auto &KV : QueryInfo->Aliases) {
-          assert(Result->count(KV.second.Aliasee) &&
+          assert((KV.second.AliasFlags.hasMaterializationSideEffectsOnly() ||
+                  Result->count(KV.second.Aliasee)) &&
                  "Result map missing entry?");
+          // Don't try to resolve materialization-side-effects-only symbols.
+          if (KV.second.AliasFlags.hasMaterializationSideEffectsOnly())
+            continue;
+
           ResolutionMap[KV.first] = JITEvaluatedSymbol(
               (*Result)[KV.second.Aliasee].getAddress(), KV.second.AliasFlags);
         }
@@ -759,8 +785,6 @@ void JITDylib::addDependencies(const SymbolStringPtr &Name,
 
       // If the dependency was not in the error state then add it to
       // our list of dependencies.
-      assert(OtherJITDylib.MaterializingInfos.count(OtherSymbol) &&
-             "No MaterializingInfo for dependency");
       auto &OtherMI = OtherJITDylib.MaterializingInfos[OtherSymbol];
 
       if (OtherSymEntry.getState() == SymbolState::Emitted)
@@ -841,7 +865,11 @@ Error JITDylib::resolve(const SymbolMap &Resolved) {
       SymI->second.setFlags(ResolvedFlags);
       SymI->second.setState(SymbolState::Resolved);
 
-      auto &MI = MaterializingInfos[Name];
+      auto MII = MaterializingInfos.find(Name);
+      if (MII == MaterializingInfos.end())
+        continue;
+
+      auto &MI = MII->second;
       for (auto &Q : MI.takeQueriesMeeting(SymbolState::Resolved)) {
         Q->notifySymbolMetRequiredState(Name, ResolvedSym);
         Q->removeQueryDependence(*this, Name);
@@ -904,13 +932,21 @@ Error JITDylib::emit(const SymbolFlagsMap &Emitted) {
       auto &SymEntry = SymI->second;
 
       // Move symbol to the emitted state.
-      assert(SymEntry.getState() == SymbolState::Resolved &&
+      assert(((SymEntry.getFlags().hasMaterializationSideEffectsOnly() &&
+               SymEntry.getState() == SymbolState::Materializing) ||
+              SymEntry.getState() == SymbolState::Resolved) &&
              "Emitting from state other than Resolved");
       SymEntry.setState(SymbolState::Emitted);
 
       auto MII = MaterializingInfos.find(Name);
-      assert(MII != MaterializingInfos.end() &&
-             "Missing MaterializingInfo entry");
+
+      // If this symbol has no MaterializingInfo then it's trivially ready.
+      // Update its state and continue.
+      if (MII == MaterializingInfos.end()) {
+        SymEntry.setState(SymbolState::Ready);
+        continue;
+      }
+
       auto &MI = MII->second;
 
       // For each dependant, transfer this node's emitted dependencies to
@@ -1303,6 +1339,12 @@ Error JITDylib::lodgeQueryImpl(MaterializationUnitList &MUs,
         if (SymI == Symbols.end())
           return false;
 
+        // If we match against a materialization-side-effects only symbol then
+        // make sure it is weakly-referenced. Otherwise bail out with an error.
+        if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() &&
+            SymLookupFlags != SymbolLookupFlags::WeaklyReferencedSymbol)
+          return make_error<SymbolsNotFound>(SymbolNameVector({Name}));
+
         // If this is a non exported symbol and we're matching exported symbols
         // only then skip this symbol without removal.
         if (!SymI->second.getFlags().isExported() &&
@@ -1572,6 +1614,9 @@ JITDylib::JITDylib(ExecutionSession &ES, std::string Name)
 }
 
 Error JITDylib::defineImpl(MaterializationUnit &MU) {
+
+  LLVM_DEBUG({ dbgs() << "  " << MU.getSymbols() << "\n"; });
+
   SymbolNameSet Duplicates;
   std::vector<SymbolStringPtr> ExistingDefsOverridden;
   std::vector<SymbolStringPtr> MUDefsOverridden;
@@ -1596,14 +1641,26 @@ Error JITDylib::defineImpl(MaterializationUnit &MU) {
   }
 
   // If there were any duplicate definitions then bail out.
-  if (!Duplicates.empty())
+  if (!Duplicates.empty()) {
+    LLVM_DEBUG(
+        { dbgs() << "  Error: Duplicate symbols " << Duplicates << "\n"; });
     return make_error<DuplicateDefinition>(std::string(**Duplicates.begin()));
+  }
 
   // Discard any overridden defs in this MU.
+  LLVM_DEBUG({
+    if (!MUDefsOverridden.empty())
+      dbgs() << "  Defs in this MU overridden: " << MUDefsOverridden << "\n";
+  });
   for (auto &S : MUDefsOverridden)
     MU.doDiscard(*this, S);
 
   // Discard existing overridden defs.
+  LLVM_DEBUG({
+    if (!ExistingDefsOverridden.empty())
+      dbgs() << "  Existing defs overridden by this MU: " << MUDefsOverridden
+             << "\n";
+  });
   for (auto &S : ExistingDefsOverridden) {
 
     auto UMII = UnmaterializedInfos.find(S);
diff --git a/llvm/lib/ExecutionEngine/Orc/DebugUtils.cpp b/llvm/lib/ExecutionEngine/Orc/DebugUtils.cpp
index b816363cc547e..6247158919fa1 100644
--- a/llvm/lib/ExecutionEngine/Orc/DebugUtils.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/DebugUtils.cpp
@@ -150,6 +150,10 @@ raw_ostream &operator<<(raw_ostream &OS, const SymbolNameVector &Symbols) {
   return OS << printSequence(Symbols, '[', ']', PrintAll<SymbolStringPtr>());
 }
 
+raw_ostream &operator<<(raw_ostream &OS, ArrayRef<SymbolStringPtr> Symbols) {
+  return OS << printSequence(Symbols, '[', ']', PrintAll<SymbolStringPtr>());
+}
+
 raw_ostream &operator<<(raw_ostream &OS, const JITSymbolFlags &Flags) {
   if (Flags.hasError())
     OS << "[*ERROR*]";
diff --git a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
index a98445a2295e4..50f1ca3fe3df3 100644
--- a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
@@ -349,8 +349,8 @@ Error StaticLibraryDefinitionGenerator::tryToGenerate(
     MemoryBufferRef ChildBufferRef(ChildBufferInfo.first,
                                    ChildBufferInfo.second);
 
-    if (auto Err =
-            L.add(JD, MemoryBuffer::getMemBuffer(ChildBufferRef), VModuleKey()))
+    if (auto Err = L.add(JD, MemoryBuffer::getMemBuffer(ChildBufferRef, false),
+                         VModuleKey()))
       return Err;
   }
 
diff --git a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
index 9451a57254931..9a868a6fbac30 100644
--- a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
@@ -29,14 +29,6 @@ using namespace llvm::orc;
 
 namespace {
 
-/// Add a reference to the __dso_handle global to the given module.
-/// Returns a reference to the __dso_handle IR decl.
-GlobalVariable *addDSOHandleDecl(Module &M) {
-  auto DSOHandleTy = StructType::create(M.getContext(), "lljit.dso_handle");
-  return new GlobalVariable(M, DSOHandleTy, true, GlobalValue::ExternalLinkage,
-                            nullptr, "__dso_handle");
-}
-
 /// Adds helper function decls and wrapper functions that call the helper with
 /// some additional prefix arguments.
 ///
@@ -143,11 +135,10 @@ class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport {
     SymbolMap StdInterposes;
 
     StdInterposes[Mangle("__lljit.platform_support_instance")] =
-        JITEvaluatedSymbol(pointerToJITTargetAddress(this), JITSymbolFlags());
+        JITEvaluatedSymbol(pointerToJITTargetAddress(this),
+                           JITSymbolFlags::Exported);
     StdInterposes[Mangle("__lljit.cxa_atexit_helper")] = JITEvaluatedSymbol(
         pointerToJITTargetAddress(registerAtExitHelper), JITSymbolFlags());
-    StdInterposes[Mangle("__lljit.run_atexits_helper")] = JITEvaluatedSymbol(
-        pointerToJITTargetAddress(runAtExitsHelper), JITSymbolFlags());
 
     cantFail(
         J.getMainJITDylib().define(absoluteSymbols(std::move(StdInterposes))));
@@ -159,6 +150,14 @@ class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport {
 
   /// Adds a module that defines the __dso_handle global.
   Error setupJITDylib(JITDylib &JD) {
+
+    // Add per-jitdylib standard interposes.
+    MangleAndInterner Mangle(getExecutionSession(), J.getDataLayout());
+    SymbolMap PerJDInterposes;
+    PerJDInterposes[Mangle("__lljit.run_atexits_helper")] = JITEvaluatedSymbol(
+        pointerToJITTargetAddress(runAtExitsHelper), JITSymbolFlags());
+    cantFail(JD.define(absoluteSymbols(std::move(PerJDInterposes))));
+
     auto Ctx = std::make_unique<LLVMContext>();
     auto M = std::make_unique<Module>("__standard_lib", *Ctx);
     M->setDataLayout(J.getDataLayout());
@@ -168,15 +167,29 @@ class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport {
         *M, Int64Ty, true, GlobalValue::ExternalLinkage,
         ConstantInt::get(Int64Ty, reinterpret_cast<uintptr_t>(&JD)),
         "__dso_handle");
-    DSOHandle->setVisibility(GlobalValue::HiddenVisibility);
+    DSOHandle->setVisibility(GlobalValue::DefaultVisibility);
     DSOHandle->setInitializer(
         ConstantInt::get(Int64Ty, pointerToJITTargetAddress(&JD)));
+
+    auto *GenericIRPlatformSupportTy =
+        StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport");
+
+    auto *PlatformInstanceDecl = new GlobalVariable(
+        *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage,
+        nullptr, "__lljit.platform_support_instance");
+
+    auto *VoidTy = Type::getVoidTy(*Ctx);
+    addHelperAndWrapper(
+        *M, "__lljit_run_atexits", FunctionType::get(VoidTy, {}, false),
+        GlobalValue::HiddenVisibility, "__lljit.run_atexits_helper",
+        {PlatformInstanceDecl, DSOHandle});
+
     return J.addIRModule(JD, ThreadSafeModule(std::move(M), std::move(Ctx)));
   }
 
   Error notifyAdding(JITDylib &JD, const MaterializationUnit &MU) {
     if (auto &InitSym = MU.getInitializerSymbol())
-      InitSymbols[&JD].add(InitSym);
+      InitSymbols[&JD].add(InitSym, SymbolLookupFlags::WeaklyReferencedSymbol);
     else {
       // If there's no identified init symbol attached, but there is a symbol
       // with the GenericIRPlatform::InitFunctionPrefix, then treat that as
@@ -185,7 +198,8 @@ class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport {
       // map (which holds the names of the symbols to execute).
       for (auto &KV : MU.getSymbols())
         if ((*KV.first).startswith(*InitFunctionPrefix)) {
-          InitSymbols[&JD].add(KV.first);
+          InitSymbols[&JD].add(KV.first,
+                               SymbolLookupFlags::WeaklyReferencedSymbol);
           InitFunctions[&JD].add(KV.first);
         }
     }
@@ -315,6 +329,16 @@ class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport {
       }
     });
 
+    LLVM_DEBUG({
+      dbgs() << "JITDylib deinit order is [ ";
+      for (auto *JD : DFSLinkOrder)
+        dbgs() << "\"" << JD->getName() << "\" ";
+      dbgs() << "]\n";
+      dbgs() << "Looking up deinit functions:\n";
+      for (auto &KV : LookupSymbols)
+        dbgs() << "  \"" << KV.first->getName() << "\": " << KV.second << "\n";
+    });
+
     auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols);
 
     if (!LookupResult)
@@ -386,11 +410,19 @@ class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport {
 
   static void registerAtExitHelper(void *Self, void (*F)(void *), void *Ctx,
                                    void *DSOHandle) {
+    LLVM_DEBUG({
+      dbgs() << "Registering atexit function " << (void *)F << " for JD "
+             << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
+    });
     static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.registerAtExit(
         F, Ctx, DSOHandle);
   }
 
   static void runAtExitsHelper(void *Self, void *DSOHandle) {
+    LLVM_DEBUG({
+      dbgs() << "Running atexit functions for JD "
+             << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
+    });
     static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.runAtExits(
         DSOHandle);
   }
@@ -409,8 +441,6 @@ class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport {
         *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage,
         nullptr, "__lljit.platform_support_instance");
 
-    auto *DSOHandleDecl = addDSOHandleDecl(*M);
-
     auto *Int8Ty = Type::getInt8Ty(*Ctx);
     auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT);
     auto *VoidTy = Type::getVoidTy(*Ctx);
@@ -422,14 +452,9 @@ class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport {
         *M, "__cxa_atexit",
         FunctionType::get(IntTy, {AtExitCallbackPtrTy, BytePtrTy, BytePtrTy},
                           false),
-        GlobalValue::HiddenVisibility, "__lljit.cxa_atexit_helper",
+        GlobalValue::DefaultVisibility, "__lljit.cxa_atexit_helper",
         {PlatformInstanceDecl});
 
-    addHelperAndWrapper(
-        *M, "__lljit_run_atexits", FunctionType::get(VoidTy, {}, false),
-        GlobalValue::HiddenVisibility, "__lljit.run_atexits_helper",
-        {PlatformInstanceDecl, DSOHandleDecl});
-
     return ThreadSafeModule(std::move(M), std::move(Ctx));
   }
 
@@ -675,7 +700,7 @@ class MachOPlatformSupport : public LLJIT::PlatformSupport {
     auto *DSOHandle =
         new GlobalVariable(M, Int64Ty, true, GlobalValue::ExternalLinkage,
                            ConstantInt::get(Int64Ty, 0), "__dso_handle");
-    DSOHandle->setVisibility(GlobalValue::HiddenVisibility);
+    DSOHandle->setVisibility(GlobalValue::DefaultVisibility);
 
     return cantFail(J.getIRCompileLayer().getCompiler()(M));
   }
diff --git a/llvm/lib/ExecutionEngine/Orc/Layer.cpp b/llvm/lib/ExecutionEngine/Orc/Layer.cpp
index 5930c6800c652..61e7ab5ae68b7 100644
--- a/llvm/lib/ExecutionEngine/Orc/Layer.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Layer.cpp
@@ -81,11 +81,19 @@ IRMaterializationUnit::IRMaterializationUnit(
 
     // If we need an init symbol for this module then create one.
     if (!llvm::empty(getStaticInitGVs(M))) {
-      std::string InitSymbolName;
-      raw_string_ostream(InitSymbolName)
-          << "$." << M.getModuleIdentifier() << ".__inits";
-      InitSymbol = ES.intern(InitSymbolName);
-      SymbolFlags[InitSymbol] = JITSymbolFlags();
+      size_t Counter = 0;
+
+      while (true) {
+        std::string InitSymbolName;
+        raw_string_ostream(InitSymbolName)
+            << "$." << M.getModuleIdentifier() << ".__inits." << Counter++;
+        InitSymbol = ES.intern(InitSymbolName);
+        if (SymbolFlags.count(InitSymbol))
+          continue;
+        SymbolFlags[InitSymbol] =
+            JITSymbolFlags::MaterializationSideEffectsOnly;
+        break;
+      }
     }
   });
 }
diff --git a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
index d69398663aff5..c547c8ef7f150 100644
--- a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
@@ -164,7 +164,8 @@ Error MachOPlatform::notifyAdding(JITDylib &JD, const MaterializationUnit &MU) {
   if (!InitSym)
     return Error::success();
 
-  RegisteredInitSymbols[&JD].add(InitSym);
+  RegisteredInitSymbols[&JD].add(InitSym,
+                                 SymbolLookupFlags::WeaklyReferencedSymbol);
   LLVM_DEBUG({
     dbgs() << "MachOPlatform: Registered init symbol " << *InitSym << " for MU "
            << MU.getName() << "\n";
diff --git a/llvm/lib/ExecutionEngine/Orc/Mangling.cpp b/llvm/lib/ExecutionEngine/Orc/Mangling.cpp
index 6baf18669573d..b85f7290fd161 100644
--- a/llvm/lib/ExecutionEngine/Orc/Mangling.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Mangling.cpp
@@ -130,11 +130,19 @@ getObjectSymbolInfo(ExecutionSession &ES, MemoryBufferRef ObjBuffer) {
     for (auto &Sec : MachOObj.sections()) {
       auto SecType = MachOObj.getSectionType(Sec);
       if ((SecType & MachO::SECTION_TYPE) == MachO::S_MOD_INIT_FUNC_POINTERS) {
-        std::string InitSymString;
-        raw_string_ostream(InitSymString)
-            << "$." << ObjBuffer.getBufferIdentifier() << ".__inits";
-        InitSymbol = ES.intern(InitSymString);
-        SymbolFlags[InitSymbol] = JITSymbolFlags();
+        size_t Counter = 0;
+        while (true) {
+          std::string InitSymString;
+          raw_string_ostream(InitSymString)
+              << "$." << ObjBuffer.getBufferIdentifier() << ".__inits."
+              << Counter++;
+          InitSymbol = ES.intern(InitSymString);
+          if (SymbolFlags.count(InitSymbol))
+            continue;
+          SymbolFlags[InitSymbol] =
+              JITSymbolFlags::MaterializationSideEffectsOnly;
+          break;
+        }
         break;
       }
     }
diff --git a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
index f0704bb83e2d3..83d3f676dc495 100644
--- a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
@@ -145,34 +145,50 @@ class ObjectLinkingLayerJITLinkContext final : public JITLinkContext {
       if (auto Err = MR.defineMaterializing(ExtraSymbolsToClaim))
         return notifyFailed(std::move(Err));
 
-    if (const auto &InitSym = MR.getInitializerSymbol())
-      InternedResult[InitSym] = JITEvaluatedSymbol();
-
     {
+
       // Check that InternedResult matches up with MR.getSymbols().
       // This guards against faulty transformations / compilers / object caches.
 
-      if (InternedResult.size() > MR.getSymbols().size()) {
-        SymbolNameVector ExtraSymbols;
-        for (auto &KV : InternedResult)
-          if (!MR.getSymbols().count(KV.first))
+      // First check that there aren't any missing symbols.
+      size_t NumMaterializationSideEffectsOnlySymbols = 0;
+      SymbolNameVector ExtraSymbols;
+      SymbolNameVector MissingSymbols;
+      for (auto &KV : MR.getSymbols()) {
+
+        // If this is a materialization-side-effects only symbol then bump
+        // the counter and make sure it's *not* defined, otherwise make
+        // sure that it is defined.
+        if (KV.second.hasMaterializationSideEffectsOnly()) {
+          ++NumMaterializationSideEffectsOnlySymbols;
+          if (InternedResult.count(KV.first))
             ExtraSymbols.push_back(KV.first);
-        ES.reportError(
-          make_error<UnexpectedSymbolDefinitions>(G.getName(),
-                                                  std::move(ExtraSymbols)));
+          continue;
+        } else if (!InternedResult.count(KV.first))
+          MissingSymbols.push_back(KV.first);
+      }
+
+      // If there were missing symbols then report the error.
+      if (!MissingSymbols.empty()) {
+        ES.reportError(make_error<MissingSymbolDefinitions>(
+            G.getName(), std::move(MissingSymbols)));
         MR.failMaterialization();
         return;
       }
 
-      SymbolNameVector MissingSymbols;
-      for (auto &KV : MR.getSymbols())
-        if (!InternedResult.count(KV.first))
-          MissingSymbols.push_back(KV.first);
+      // If there are more definitions than expected, add them to the
+      // ExtraSymbols vector.
+      if (InternedResult.size() >
+          MR.getSymbols().size() - NumMaterializationSideEffectsOnlySymbols) {
+        for (auto &KV : InternedResult)
+          if (!MR.getSymbols().count(KV.first))
+            ExtraSymbols.push_back(KV.first);
+      }
 
-      if (!MissingSymbols.empty()) {
-        ES.reportError(
-          make_error<MissingSymbolDefinitions>(G.getName(),
-                                               std::move(MissingSymbols)));
+      // If there were extra definitions then report the error.
+      if (!ExtraSymbols.empty()) {
+        ES.reportError(make_error<UnexpectedSymbolDefinitions>(
+            G.getName(), std::move(ExtraSymbols)));
         MR.failMaterialization();
         return;
       }
diff --git a/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp
index f660c4290e4ca..e0582f73b2fcb 100644
--- a/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp
@@ -258,9 +258,6 @@ Error RTDyldObjectLinkingLayer::onObjLoad(
         Symbols.erase(KV.first);
   }
 
-  if (const auto &InitSym = R.getInitializerSymbol())
-    Symbols[InitSym] = JITEvaluatedSymbol();
-
   if (auto Err = R.notifyResolved(Symbols)) {
     R.failMaterialization();
     return Err;
diff --git a/llvm/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp b/llvm/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp
index f22acf50419d4..7240c1ed0ce98 100644
--- a/llvm/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp
@@ -209,7 +209,7 @@ void SequenceBBQuery::traverseToExitBlock(const BasicBlock *AtBB,
     VisitedBlocks.insert(std::make_pair(AtBB, BlockHint));
   }
 
-  succ_const_iterator PIt = succ_begin(AtBB), EIt = succ_end(AtBB);
+  const_succ_iterator PIt = succ_begin(AtBB), EIt = succ_end(AtBB);
   if (PIt == EIt) // No succs.
     return;
 
diff --git a/llvm/lib/Frontend/OpenMP/OMPContext.cpp b/llvm/lib/Frontend/OpenMP/OMPContext.cpp
index 748caa645a37e..dd3292ab776e0 100644
--- a/llvm/lib/Frontend/OpenMP/OMPContext.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPContext.cpp
@@ -79,8 +79,8 @@ OMPContext::OMPContext(bool IsDeviceCompilation, Triple TargetTriple) {
   LLVM_DEBUG({
     dbgs() << "[" << DEBUG_TYPE
            << "] New OpenMP context with the following properties:\n";
-    for (const auto &SetBitsIt : ActiveTraits.set_bits()) {
-      TraitProperty Property = TraitProperty(SetBitsIt);
+    for (unsigned Bit : ActiveTraits.set_bits()) {
+      TraitProperty Property = TraitProperty(Bit);
       dbgs() << "\t " << getOpenMPContextTraitPropertyFullName(Property)
              << "\n";
     }
@@ -127,8 +127,8 @@ static bool isStrictSubset(const VariantMatchInfo &VMI0,
   // relation is not required to be strict.
   if (VMI0.RequiredTraits.count() >= VMI1.RequiredTraits.count())
     return false;
-  for (const auto &SetBitsIt : VMI0.RequiredTraits.set_bits())
-    if (!VMI1.RequiredTraits.test(SetBitsIt))
+  for (unsigned Bit : VMI0.RequiredTraits.set_bits())
+    if (!VMI1.RequiredTraits.test(Bit))
       return false;
   if (!isSubset<TraitProperty>(VMI0.ConstructTraits, VMI1.ConstructTraits))
     return false;
@@ -139,8 +139,8 @@ static int isVariantApplicableInContextHelper(
     const VariantMatchInfo &VMI, const OMPContext &Ctx,
     SmallVectorImpl<unsigned> *ConstructMatches) {
 
-  for (const auto &SetBitsIt : VMI.RequiredTraits.set_bits()) {
-    TraitProperty Property = TraitProperty(SetBitsIt);
+  for (unsigned Bit : VMI.RequiredTraits.set_bits()) {
+    TraitProperty Property = TraitProperty(Bit);
 
     bool IsActiveTrait = Ctx.ActiveTraits.test(unsigned(Property));
     if (!IsActiveTrait) {
@@ -191,8 +191,8 @@ static APInt getVariantMatchScore(const VariantMatchInfo &VMI,
   APInt Score(64, 1);
 
   unsigned NoConstructTraits = VMI.ConstructTraits.size();
-  for (const auto &SetBitsIt : VMI.RequiredTraits.set_bits()) {
-    TraitProperty Property = TraitProperty(SetBitsIt);
+  for (unsigned Bit : VMI.RequiredTraits.set_bits()) {
+    TraitProperty Property = TraitProperty(Bit);
     // If there is a user score attached, use it.
     if (VMI.ScoreMap.count(Property)) {
       const APInt &UserScore = VMI.ScoreMap.lookup(Property);
@@ -450,6 +450,8 @@ llvm::omp::listOpenMPContextTraitProperties(TraitSet Set,
       StringRef(Str) != "invalid")                                             \
     S.append("'").append(Str).append("'").append(" ");
 #include "llvm/Frontend/OpenMP/OMPKinds.def"
+  if (S.empty())
+    return "<none>";
   S.pop_back();
   return S;
 }
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index 733fc10c2ee33..45f393f8b8e70 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -462,6 +462,33 @@ static void PrintLLVMName(raw_ostream &OS, const Value *V) {
                 isa<GlobalValue>(V) ? GlobalPrefix : LocalPrefix);
 }
 
+static void PrintShuffleMask(raw_ostream &Out, Type *Ty, ArrayRef<int> Mask) {
+  Out << ", <";
+  if (Ty->getVectorIsScalable())
+    Out << "vscale x ";
+  Out << Mask.size() << " x i32> ";
+  bool FirstElt = true;
+  if (all_of(Mask, [](int Elt) { return Elt == 0; })) {
+    Out << "zeroinitializer";
+  } else if (all_of(Mask, [](int Elt) { return Elt == UndefMaskElem; })) {
+    Out << "undef";
+  } else {
+    Out << "<";
+    for (int Elt : Mask) {
+      if (FirstElt)
+        FirstElt = false;
+      else
+        Out << ", ";
+      Out << "i32 ";
+      if (Elt == UndefMaskElem)
+        Out << "undef";
+      else
+        Out << Elt;
+    }
+    Out << ">";
+  }
+}
+
 namespace {
 
 class TypePrinting {
@@ -1547,6 +1574,9 @@ static void WriteConstantInternal(raw_ostream &Out, const Constant *CV,
       TypePrinter.print(CE->getType(), Out);
     }
 
+    if (CE->getOpcode() == Instruction::ShuffleVector)
+      PrintShuffleMask(Out, CE->getType(), CE->getShuffleMask());
+
     Out << ')';
     return;
   }
@@ -4093,6 +4123,8 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
                 RMWI->getSyncScopeID());
   } else if (const FenceInst *FI = dyn_cast<FenceInst>(&I)) {
     writeAtomic(FI->getContext(), FI->getOrdering(), FI->getSyncScopeID());
+  } else if (const ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(&I)) {
+    PrintShuffleMask(Out, SVI->getType(), SVI->getShuffleMask());
   }
 
   // Print Metadata info.
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index de5b94403bd77..96021dc1fbd55 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -2335,11 +2335,12 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
                                             PointerType::getUnqual(VT));
       Value *Load = Builder.CreateAlignedLoad(VT, Op, Align(1));
       if (NumSrcElts == 2)
-        Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),
-                                          { 0, 1, 0, 1 });
+        Rep = Builder.CreateShuffleVector(
+            Load, UndefValue::get(Load->getType()), ArrayRef<int>{0, 1, 0, 1});
       else
-        Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),
-                                          { 0, 1, 2, 3, 0, 1, 2, 3 });
+        Rep =
+            Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),
+                                        ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
     } else if (IsX86 && (Name.startswith("avx512.mask.shuf.i") ||
                          Name.startswith("avx512.mask.shuf.f"))) {
       unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
diff --git a/llvm/lib/IR/BasicBlock.cpp b/llvm/lib/IR/BasicBlock.cpp
index 675deea1d755b..358362cf07e8e 100644
--- a/llvm/lib/IR/BasicBlock.cpp
+++ b/llvm/lib/IR/BasicBlock.cpp
@@ -291,7 +291,7 @@ bool BasicBlock::hasNPredecessorsOrMore(unsigned N) const {
 }
 
 const BasicBlock *BasicBlock::getSingleSuccessor() const {
-  succ_const_iterator SI = succ_begin(this), E = succ_end(this);
+  const_succ_iterator SI = succ_begin(this), E = succ_end(this);
   if (SI == E) return nullptr; // no successors
   const BasicBlock *TheSucc = *SI;
   ++SI;
@@ -299,7 +299,7 @@ const BasicBlock *BasicBlock::getSingleSuccessor() const {
 }
 
 const BasicBlock *BasicBlock::getUniqueSuccessor() const {
-  succ_const_iterator SI = succ_begin(this), E = succ_end(this);
+  const_succ_iterator SI = succ_begin(this), E = succ_end(this);
   if (SI == E) return nullptr; // No successors
   const BasicBlock *SuccBB = *SI;
   ++SI;
diff --git a/llvm/lib/IR/CMakeLists.txt b/llvm/lib/IR/CMakeLists.txt
index 14207bce57694..8fcc10fa38af3 100644
--- a/llvm/lib/IR/CMakeLists.txt
+++ b/llvm/lib/IR/CMakeLists.txt
@@ -28,7 +28,6 @@ add_llvm_component_library(LLVMCore
   Instruction.cpp
   Instructions.cpp
   IntrinsicInst.cpp
-  KnowledgeRetention.cpp
   LLVMContext.cpp
   LLVMContextImpl.cpp
   LLVMRemarkStreamer.cpp
diff --git a/llvm/lib/IR/ConstantFold.cpp b/llvm/lib/IR/ConstantFold.cpp
index 3e2e74c31fc0a..c4465d9bfa02f 100644
--- a/llvm/lib/IR/ConstantFold.cpp
+++ b/llvm/lib/IR/ConstantFold.cpp
@@ -60,6 +60,11 @@ static Constant *BitCastConstantVector(Constant *CV, VectorType *DstTy) {
     return nullptr;
 
   Type *DstEltTy = DstTy->getElementType();
+  // Fast path for splatted constants.
+  if (Constant *Splat = CV->getSplatValue()) {
+    return ConstantVector::getSplat(DstTy->getVectorElementCount(),
+                                    ConstantExpr::getBitCast(Splat, DstEltTy));
+  }
 
   SmallVector<Constant*, 16> Result;
   Type *Ty = IntegerType::get(CV->getContext(), 32);
@@ -577,9 +582,15 @@ Constant *llvm::ConstantFoldCastInstruction(unsigned opc, Constant *V,
   if ((isa<ConstantVector>(V) || isa<ConstantDataVector>(V)) &&
       DestTy->isVectorTy() &&
       DestTy->getVectorNumElements() == V->getType()->getVectorNumElements()) {
-    SmallVector<Constant*, 16> res;
     VectorType *DestVecTy = cast<VectorType>(DestTy);
     Type *DstEltTy = DestVecTy->getElementType();
+    // Fast path for splatted constants.
+    if (Constant *Splat = V->getSplatValue()) {
+      return ConstantVector::getSplat(
+          DestTy->getVectorElementCount(),
+          ConstantExpr::getCast(opc, Splat, DstEltTy));
+    }
+    SmallVector<Constant *, 16> res;
     Type *Ty = IntegerType::get(V->getContext(), 32);
     for (unsigned i = 0, e = V->getType()->getVectorNumElements(); i != e; ++i) {
       Constant *C =
@@ -865,32 +876,39 @@ Constant *llvm::ConstantFoldInsertElementInstruction(Constant *Val,
   return ConstantVector::get(Result);
 }
 
-Constant *llvm::ConstantFoldShuffleVectorInstruction(Constant *V1,
-                                                     Constant *V2,
-                                                     Constant *Mask) {
-  ElementCount MaskEltCount = Mask->getType()->getVectorElementCount();
+Constant *llvm::ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2,
+                                                     ArrayRef<int> Mask) {
+  unsigned MaskNumElts = Mask.size();
+  ElementCount MaskEltCount = {MaskNumElts,
+                               V1->getType()->getVectorIsScalable()};
   Type *EltTy = V1->getType()->getVectorElementType();
 
   // Undefined shuffle mask -> undefined value.
-  if (isa<UndefValue>(Mask))
-    return UndefValue::get(VectorType::get(EltTy, MaskEltCount));
-
-  // Don't break the bitcode reader hack.
-  if (isa<ConstantExpr>(Mask)) return nullptr;
+  if (all_of(Mask, [](int Elt) { return Elt == UndefMaskElem; })) {
+    return UndefValue::get(VectorType::get(EltTy, MaskNumElts));
+  }
 
+  // If the mask is all zeros this is a splat, no need to go through all
+  // elements.
+  if (all_of(Mask, [](int Elt) { return Elt == 0; }) &&
+      !MaskEltCount.Scalable) {
+    Type *Ty = IntegerType::get(V1->getContext(), 32);
+    Constant *Elt =
+        ConstantExpr::getExtractElement(V1, ConstantInt::get(Ty, 0));
+    return ConstantVector::getSplat(MaskEltCount, Elt);
+  }
   // Do not iterate on scalable vector. The num of elements is unknown at
   // compile-time.
   VectorType *ValTy = cast<VectorType>(V1->getType());
   if (ValTy->isScalable())
     return nullptr;
 
-  unsigned MaskNumElts = MaskEltCount.Min;
   unsigned SrcNumElts = V1->getType()->getVectorNumElements();
 
   // Loop over the shuffle mask, evaluating each element.
   SmallVector<Constant*, 32> Result;
   for (unsigned i = 0; i != MaskNumElts; ++i) {
-    int Elt = ShuffleVectorInst::getMaskValue(Mask, i);
+    int Elt = Mask[i];
     if (Elt == -1) {
       Result.push_back(UndefValue::get(EltTy));
       continue;
@@ -993,10 +1011,15 @@ Constant *llvm::ConstantFoldUnaryInstruction(unsigned Opcode, Constant *C) {
     // compile-time.
     if (IsScalableVector)
       return nullptr;
+    Type *Ty = IntegerType::get(VTy->getContext(), 32);
+    // Fast path for splatted constants.
+    if (Constant *Splat = C->getSplatValue()) {
+      Constant *Elt = ConstantExpr::get(Opcode, Splat);
+      return ConstantVector::getSplat(VTy->getElementCount(), Elt);
+    }
 
     // Fold each element and create a vector constant from those constants.
     SmallVector<Constant*, 16> Result;
-    Type *Ty = IntegerType::get(VTy->getContext(), 32);
     for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) {
       Constant *ExtractIdx = ConstantInt::get(Ty, i);
       Constant *Elt = ConstantExpr::getExtractElement(C, ExtractIdx);
@@ -1357,6 +1380,16 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1,
     // compile-time.
     if (IsScalableVector)
       return nullptr;
+    // Fast path for splatted constants.
+    if (Constant *C2Splat = C2->getSplatValue()) {
+      if (Instruction::isIntDivRem(Opcode) && C2Splat->isNullValue())
+        return UndefValue::get(VTy);
+      if (Constant *C1Splat = C1->getSplatValue()) {
+        return ConstantVector::getSplat(
+            VTy->getVectorElementCount(),
+            ConstantExpr::get(Opcode, C1Splat, C2Splat));
+      }
+    }
 
     // Fold each element and create a vector constant from those constants.
     SmallVector<Constant*, 16> Result;
@@ -1975,6 +2008,12 @@ Constant *llvm::ConstantFoldCompareInstruction(unsigned short pred,
     // compile-time.
     if (C1->getType()->getVectorIsScalable())
       return nullptr;
+    // Fast path for splatted constants.
+    if (Constant *C1Splat = C1->getSplatValue())
+      if (Constant *C2Splat = C2->getSplatValue())
+        return ConstantVector::getSplat(
+            C1->getType()->getVectorElementCount(),
+            ConstantExpr::getCompare(pred, C1Splat, C2Splat));
 
     // If we can constant fold the comparison of each element, constant fold
     // the whole vector comparison.
diff --git a/llvm/lib/IR/ConstantFold.h b/llvm/lib/IR/ConstantFold.h
index 9ad6e14e9e40e..0cdd5cf3cbce6 100644
--- a/llvm/lib/IR/ConstantFold.h
+++ b/llvm/lib/IR/ConstantFold.h
@@ -38,7 +38,7 @@ template <typename T> class ArrayRef;
   Constant *ConstantFoldInsertElementInstruction(Constant *Val, Constant *Elt,
                                                  Constant *Idx);
   Constant *ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2,
-                                                 Constant *Mask);
+                                                 ArrayRef<int> Mask);
   Constant *ConstantFoldExtractValueInstruction(Constant *Agg,
                                                 ArrayRef<unsigned> Idxs);
   Constant *ConstantFoldInsertValueInstruction(Constant *Agg, Constant *Val,
diff --git a/llvm/lib/IR/ConstantRange.cpp b/llvm/lib/IR/ConstantRange.cpp
index b8c57533568be..eabaaa203927f 100644
--- a/llvm/lib/IR/ConstantRange.cpp
+++ b/llvm/lib/IR/ConstantRange.cpp
@@ -1196,6 +1196,10 @@ ConstantRange::binaryAnd(const ConstantRange &Other) const {
   if (isEmptySet() || Other.isEmptySet())
     return getEmpty();
 
+  // Use APInt's implementation of AND for single element ranges.
+  if (isSingleElement() && Other.isSingleElement())
+    return {*getSingleElement() & *Other.getSingleElement()};
+
   // TODO: replace this with something less conservative
 
   APInt umin = APIntOps::umin(Other.getUnsignedMax(), getUnsignedMax());
@@ -1207,6 +1211,10 @@ ConstantRange::binaryOr(const ConstantRange &Other) const {
   if (isEmptySet() || Other.isEmptySet())
     return getEmpty();
 
+  // Use APInt's implementation of OR for single element ranges.
+  if (isSingleElement() && Other.isSingleElement())
+    return {*getSingleElement() | *Other.getSingleElement()};
+
   // TODO: replace this with something less conservative
 
   APInt umax = APIntOps::umax(getUnsignedMin(), Other.getUnsignedMin());
diff --git a/llvm/lib/IR/Constants.cpp b/llvm/lib/IR/Constants.cpp
index bde4c07e15a31..60ab1a595c349 100644
--- a/llvm/lib/IR/Constants.cpp
+++ b/llvm/lib/IR/Constants.cpp
@@ -1230,8 +1230,7 @@ Constant *ConstantVector::getSplat(ElementCount EC, Constant *V) {
   Constant *UndefV = UndefValue::get(VTy);
   V = ConstantExpr::getInsertElement(UndefV, V, ConstantInt::get(I32Ty, 0));
   // Build shuffle mask to perform the splat.
-  Type *MaskTy = VectorType::get(I32Ty, EC);
-  Constant *Zeros = ConstantAggregateZero::get(MaskTy);
+  SmallVector<int, 8> Zeros(EC.Min, 0);
   // Splat.
   return ConstantExpr::getShuffleVector(V, UndefV, Zeros);
 }
@@ -1298,6 +1297,14 @@ unsigned ConstantExpr::getPredicate() const {
   return cast<CompareConstantExpr>(this)->predicate;
 }
 
+ArrayRef<int> ConstantExpr::getShuffleMask() const {
+  return cast<ShuffleVectorConstantExpr>(this)->ShuffleMask;
+}
+
+Constant *ConstantExpr::getShuffleMaskForBitcode() const {
+  return cast<ShuffleVectorConstantExpr>(this)->ShuffleMaskForBitcode;
+}
+
 Constant *
 ConstantExpr::getWithOperandReplaced(unsigned OpNo, Constant *Op) const {
   assert(Op->getType() == getOperand(OpNo)->getType() &&
@@ -1349,7 +1356,7 @@ Constant *ConstantExpr::getWithOperands(ArrayRef<Constant *> Ops, Type *Ty,
   case Instruction::ExtractValue:
     return ConstantExpr::getExtractValue(Ops[0], getIndices(), OnlyIfReducedTy);
   case Instruction::ShuffleVector:
-    return ConstantExpr::getShuffleVector(Ops[0], Ops[1], Ops[2],
+    return ConstantExpr::getShuffleVector(Ops[0], Ops[1], getShuffleMask(),
                                           OnlyIfReducedTy);
   case Instruction::GetElementPtr: {
     auto *GEPO = cast<GEPOperator>(this);
@@ -2152,7 +2159,7 @@ Constant *ConstantExpr::getGetElementPtr(Type *Ty, Constant *C,
   if (InRangeIndex && *InRangeIndex < 63)
     SubClassOptionalData |= (*InRangeIndex + 1) << 1;
   const ConstantExprKeyType Key(Instruction::GetElementPtr, ArgVec, 0,
-                                SubClassOptionalData, None, Ty);
+                                SubClassOptionalData, None, None, Ty);
 
   LLVMContextImpl *pImpl = C->getContext().pImpl;
   return pImpl->ExprConstants.getOrCreate(ReqTy, Key);
@@ -2254,23 +2261,25 @@ Constant *ConstantExpr::getInsertElement(Constant *Val, Constant *Elt,
 }
 
 Constant *ConstantExpr::getShuffleVector(Constant *V1, Constant *V2,
-                                         Constant *Mask, Type *OnlyIfReducedTy) {
+                                         ArrayRef<int> Mask,
+                                         Type *OnlyIfReducedTy) {
   assert(ShuffleVectorInst::isValidOperands(V1, V2, Mask) &&
          "Invalid shuffle vector constant expr operands!");
 
   if (Constant *FC = ConstantFoldShuffleVectorInstruction(V1, V2, Mask))
     return FC;          // Fold a few common cases.
 
-  ElementCount NElts = Mask->getType()->getVectorElementCount();
+  unsigned NElts = Mask.size();
   Type *EltTy = V1->getType()->getVectorElementType();
-  Type *ShufTy = VectorType::get(EltTy, NElts);
+  bool TypeIsScalable = V1->getType()->getVectorIsScalable();
+  Type *ShufTy = VectorType::get(EltTy, NElts, TypeIsScalable);
 
   if (OnlyIfReducedTy == ShufTy)
     return nullptr;
 
   // Look up the constant in the table first to ensure uniqueness
-  Constant *ArgVec[] = { V1, V2, Mask };
-  const ConstantExprKeyType Key(Instruction::ShuffleVector, ArgVec);
+  Constant *ArgVec[] = {V1, V2};
+  ConstantExprKeyType Key(Instruction::ShuffleVector, ArgVec, 0, 0, None, Mask);
 
   LLVMContextImpl *pImpl = ShufTy->getContext().pImpl;
   return pImpl->ExprConstants.getOrCreate(ShufTy, Key);
@@ -2891,7 +2900,7 @@ bool ConstantDataSequential::isCString() const {
   return Str.drop_back().find(0) == StringRef::npos;
 }
 
-bool ConstantDataVector::isSplat() const {
+bool ConstantDataVector::isSplatData() const {
   const char *Base = getRawDataValues().data();
 
   // Compare elements 1+ to the 0'th element.
@@ -2903,6 +2912,14 @@ bool ConstantDataVector::isSplat() const {
   return true;
 }
 
+bool ConstantDataVector::isSplat() const {
+  if (!IsSplatSet) {
+    IsSplatSet = true;
+    IsSplat = isSplatData();
+  }
+  return IsSplat;
+}
+
 Constant *ConstantDataVector::getSplatValue() const {
   // If they're all the same, return the 0th one as a representative.
   return isSplat() ? getElementAsConstant(0) : nullptr;
@@ -3109,7 +3126,7 @@ Instruction *ConstantExpr::getAsInstruction() const {
   case Instruction::ExtractValue:
     return ExtractValueInst::Create(Ops[0], getIndices());
   case Instruction::ShuffleVector:
-    return new ShuffleVectorInst(Ops[0], Ops[1], Ops[2]);
+    return new ShuffleVectorInst(Ops[0], Ops[1], getShuffleMask());
 
   case Instruction::GetElementPtr: {
     const auto *GO = cast<GEPOperator>(this);
diff --git a/llvm/lib/IR/ConstantsContext.h b/llvm/lib/IR/ConstantsContext.h
index 9ecbb61378360..ad521c8a1bdf7 100644
--- a/llvm/lib/IR/ConstantsContext.h
+++ b/llvm/lib/IR/ConstantsContext.h
@@ -26,6 +26,7 @@
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/InlineAsm.h"
 #include "llvm/IR/Instruction.h"
+#include "llvm/IR/Instructions.h"
 #include "llvm/IR/OperandTraits.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Debug.h"
@@ -146,21 +147,24 @@ class InsertElementConstantExpr : public ConstantExpr {
 /// shufflevector constant exprs.
 class ShuffleVectorConstantExpr : public ConstantExpr {
 public:
-  ShuffleVectorConstantExpr(Constant *C1, Constant *C2, Constant *C3)
-  : ConstantExpr(VectorType::get(
-                   cast<VectorType>(C1->getType())->getElementType(),
-                   cast<VectorType>(C3->getType())->getElementCount()),
-                 Instruction::ShuffleVector,
-                 &Op<0>(), 3) {
+  ShuffleVectorConstantExpr(Constant *C1, Constant *C2, ArrayRef<int> Mask)
+      : ConstantExpr(
+            VectorType::get(cast<VectorType>(C1->getType())->getElementType(),
+                            Mask.size(), C1->getType()->getVectorIsScalable()),
+            Instruction::ShuffleVector, &Op<0>(), 2) {
+    assert(ShuffleVectorInst::isValidOperands(C1, C2, Mask) &&
+           "Invalid shuffle vector instruction operands!");
     Op<0>() = C1;
     Op<1>() = C2;
-    Op<2>() = C3;
+    ShuffleMask.assign(Mask.begin(), Mask.end());
+    ShuffleMaskForBitcode =
+        ShuffleVectorInst::convertShuffleMaskForBitcode(Mask, getType());
   }
 
-  // allocate space for exactly three operands
-  void *operator new(size_t s) {
-    return User::operator new(s, 3);
-  }
+  SmallVector<int, 4> ShuffleMask;
+  Constant *ShuffleMaskForBitcode;
+
+  void *operator new(size_t s) { return User::operator new(s, 2); }
 
   /// Transparently provide more efficient getOperand methods.
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
@@ -319,7 +323,7 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InsertElementConstantExpr, Value)
 
 template <>
 struct OperandTraits<ShuffleVectorConstantExpr>
-    : public FixedNumOperandTraits<ShuffleVectorConstantExpr, 3> {};
+    : public FixedNumOperandTraits<ShuffleVectorConstantExpr, 2> {};
 DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ShuffleVectorConstantExpr, Value)
 
 template <>
@@ -460,36 +464,58 @@ struct InlineAsmKeyType {
 };
 
 struct ConstantExprKeyType {
+private:
   uint8_t Opcode;
   uint8_t SubclassOptionalData;
   uint16_t SubclassData;
   ArrayRef<Constant *> Ops;
   ArrayRef<unsigned> Indexes;
+  ArrayRef<int> ShuffleMask;
   Type *ExplicitTy;
 
+  static ArrayRef<int> getShuffleMaskIfValid(const ConstantExpr *CE) {
+    if (CE->getOpcode() == Instruction::ShuffleVector)
+      return CE->getShuffleMask();
+    return None;
+  }
+
+  static ArrayRef<unsigned> getIndicesIfValid(const ConstantExpr *CE) {
+    if (CE->hasIndices())
+      return CE->getIndices();
+    return None;
+  }
+
+  static Type *getSourceElementTypeIfValid(const ConstantExpr *CE) {
+    if (auto *GEPCE = dyn_cast<GetElementPtrConstantExpr>(CE))
+      return GEPCE->getSourceElementType();
+    return nullptr;
+  }
+
+public:
   ConstantExprKeyType(unsigned Opcode, ArrayRef<Constant *> Ops,
                       unsigned short SubclassData = 0,
                       unsigned short SubclassOptionalData = 0,
                       ArrayRef<unsigned> Indexes = None,
+                      ArrayRef<int> ShuffleMask = None,
                       Type *ExplicitTy = nullptr)
       : Opcode(Opcode), SubclassOptionalData(SubclassOptionalData),
         SubclassData(SubclassData), Ops(Ops), Indexes(Indexes),
-        ExplicitTy(ExplicitTy) {}
+        ShuffleMask(ShuffleMask), ExplicitTy(ExplicitTy) {}
 
   ConstantExprKeyType(ArrayRef<Constant *> Operands, const ConstantExpr *CE)
       : Opcode(CE->getOpcode()),
         SubclassOptionalData(CE->getRawSubclassOptionalData()),
         SubclassData(CE->isCompare() ? CE->getPredicate() : 0), Ops(Operands),
-        Indexes(CE->hasIndices() ? CE->getIndices() : ArrayRef<unsigned>()),
-        ExplicitTy(nullptr) {}
+        Indexes(getIndicesIfValid(CE)), ShuffleMask(getShuffleMaskIfValid(CE)),
+        ExplicitTy(getSourceElementTypeIfValid(CE)) {}
 
   ConstantExprKeyType(const ConstantExpr *CE,
                       SmallVectorImpl<Constant *> &Storage)
       : Opcode(CE->getOpcode()),
         SubclassOptionalData(CE->getRawSubclassOptionalData()),
         SubclassData(CE->isCompare() ? CE->getPredicate() : 0),
-        Indexes(CE->hasIndices() ? CE->getIndices() : ArrayRef<unsigned>()),
-        ExplicitTy(nullptr) {
+        Indexes(getIndicesIfValid(CE)), ShuffleMask(getShuffleMaskIfValid(CE)),
+        ExplicitTy(getSourceElementTypeIfValid(CE)) {
     assert(Storage.empty() && "Expected empty storage");
     for (unsigned I = 0, E = CE->getNumOperands(); I != E; ++I)
       Storage.push_back(CE->getOperand(I));
@@ -499,7 +525,8 @@ struct ConstantExprKeyType {
   bool operator==(const ConstantExprKeyType &X) const {
     return Opcode == X.Opcode && SubclassData == X.SubclassData &&
            SubclassOptionalData == X.SubclassOptionalData && Ops == X.Ops &&
-           Indexes == X.Indexes;
+           Indexes == X.Indexes && ShuffleMask == X.ShuffleMask &&
+           ExplicitTy == X.ExplicitTy;
   }
 
   bool operator==(const ConstantExpr *CE) const {
@@ -514,15 +541,21 @@ struct ConstantExprKeyType {
     for (unsigned I = 0, E = Ops.size(); I != E; ++I)
       if (Ops[I] != CE->getOperand(I))
         return false;
-    if (Indexes != (CE->hasIndices() ? CE->getIndices() : ArrayRef<unsigned>()))
+    if (Indexes != getIndicesIfValid(CE))
+      return false;
+    if (ShuffleMask != getShuffleMaskIfValid(CE))
+      return false;
+    if (ExplicitTy != getSourceElementTypeIfValid(CE))
       return false;
     return true;
   }
 
   unsigned getHash() const {
-    return hash_combine(Opcode, SubclassOptionalData, SubclassData,
-                        hash_combine_range(Ops.begin(), Ops.end()),
-                        hash_combine_range(Indexes.begin(), Indexes.end()));
+    return hash_combine(
+        Opcode, SubclassOptionalData, SubclassData,
+        hash_combine_range(Ops.begin(), Ops.end()),
+        hash_combine_range(Indexes.begin(), Indexes.end()),
+        hash_combine_range(ShuffleMask.begin(), ShuffleMask.end()), ExplicitTy);
   }
 
   using TypeClass = ConstantInfo<ConstantExpr>::TypeClass;
@@ -546,17 +579,14 @@ struct ConstantExprKeyType {
     case Instruction::InsertElement:
       return new InsertElementConstantExpr(Ops[0], Ops[1], Ops[2]);
     case Instruction::ShuffleVector:
-      return new ShuffleVectorConstantExpr(Ops[0], Ops[1], Ops[2]);
+      return new ShuffleVectorConstantExpr(Ops[0], Ops[1], ShuffleMask);
     case Instruction::InsertValue:
       return new InsertValueConstantExpr(Ops[0], Ops[1], Indexes, Ty);
     case Instruction::ExtractValue:
       return new ExtractValueConstantExpr(Ops[0], Indexes, Ty);
     case Instruction::GetElementPtr:
-      return GetElementPtrConstantExpr::Create(
-          ExplicitTy ? ExplicitTy
-                     : cast<PointerType>(Ops[0]->getType()->getScalarType())
-                           ->getElementType(),
-          Ops[0], Ops.slice(1), Ty, SubclassOptionalData);
+      return GetElementPtrConstantExpr::Create(ExplicitTy, Ops[0], Ops.slice(1),
+                                               Ty, SubclassOptionalData);
     case Instruction::ICmp:
       return new CompareConstantExpr(Ty, Instruction::ICmp, SubclassData,
                                      Ops[0], Ops[1]);
diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp
index 313bc65e775c9..da2e25e779785 100644
--- a/llvm/lib/IR/Core.cpp
+++ b/llvm/lib/IR/Core.cpp
@@ -1781,9 +1781,11 @@ LLVMValueRef LLVMConstInsertElement(LLVMValueRef VectorConstant,
 LLVMValueRef LLVMConstShuffleVector(LLVMValueRef VectorAConstant,
                                     LLVMValueRef VectorBConstant,
                                     LLVMValueRef MaskConstant) {
+  SmallVector<int, 16> IntMask;
+  ShuffleVectorInst::getShuffleMask(unwrap<Constant>(MaskConstant), IntMask);
   return wrap(ConstantExpr::getShuffleVector(unwrap<Constant>(VectorAConstant),
                                              unwrap<Constant>(VectorBConstant),
-                                             unwrap<Constant>(MaskConstant)));
+                                             IntMask));
 }
 
 LLVMValueRef LLVMConstExtractValue(LLVMValueRef AggConstant, unsigned *IdxList,
diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp
index aaa8443769af0..1c63476785f44 100644
--- a/llvm/lib/IR/DebugInfo.cpp
+++ b/llvm/lib/IR/DebugInfo.cpp
@@ -606,7 +606,9 @@ bool llvm::stripNonLineTableDebugInfo(Module &M) {
       Changed = true;
     }
   };
+  RemoveUses("llvm.dbg.addr");
   RemoveUses("llvm.dbg.declare");
+  RemoveUses("llvm.dbg.label");
   RemoveUses("llvm.dbg.value");
 
   // Delete non-CU debug info named metadata nodes.
diff --git a/llvm/lib/IR/FPEnv.cpp b/llvm/lib/IR/FPEnv.cpp
index 49b6d9a8dac34..ab68f55f834d8 100644
--- a/llvm/lib/IR/FPEnv.cpp
+++ b/llvm/lib/IR/FPEnv.cpp
@@ -75,4 +75,20 @@ Optional<StringRef> ExceptionBehaviorToStr(fp::ExceptionBehavior UseExcept) {
   return ExceptStr;
 }
 
+Optional<APFloatBase::roundingMode>
+getAPFloatRoundingMode(fp::RoundingMode RM) {
+  switch (RM) {
+  case fp::rmDynamic:
+    return None;
+  case fp::rmToNearest:
+    return APFloat::rmNearestTiesToEven;
+  case fp::rmDownward:
+    return APFloat::rmTowardNegative;
+  case fp::rmUpward:
+    return APFloat::rmTowardPositive;
+  case fp::rmTowardZero:
+    return APFloat::rmTowardZero;
+  }
+  llvm_unreachable("Unexpected rounding mode");
+}
 }
diff --git a/llvm/lib/IR/Instruction.cpp b/llvm/lib/IR/Instruction.cpp
index 2ef109067dd69..cd476a3601550 100644
--- a/llvm/lib/IR/Instruction.cpp
+++ b/llvm/lib/IR/Instruction.cpp
@@ -443,6 +443,9 @@ static bool haveSameSpecialState(const Instruction *I1, const Instruction *I2,
            RMWI->isVolatile() == cast<AtomicRMWInst>(I2)->isVolatile() &&
            RMWI->getOrdering() == cast<AtomicRMWInst>(I2)->getOrdering() &&
            RMWI->getSyncScopeID() == cast<AtomicRMWInst>(I2)->getSyncScopeID();
+  if (const ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(I1))
+    return SVI->getShuffleMask() ==
+           cast<ShuffleVectorInst>(I2)->getShuffleMask();
 
   return true;
 }
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index 0884a24a709e5..37e194a0feabf 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -1843,57 +1843,109 @@ bool InsertElementInst::isValidOperands(const Value *Vec, const Value *Elt,
 ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, Value *Mask,
                                      const Twine &Name,
                                      Instruction *InsertBefore)
-: Instruction(VectorType::get(cast<VectorType>(V1->getType())->getElementType(),
-                cast<VectorType>(Mask->getType())->getElementCount()),
-              ShuffleVector,
-              OperandTraits<ShuffleVectorInst>::op_begin(this),
-              OperandTraits<ShuffleVectorInst>::operands(this),
-              InsertBefore) {
+    : Instruction(
+          VectorType::get(cast<VectorType>(V1->getType())->getElementType(),
+                          cast<VectorType>(Mask->getType())->getElementCount()),
+          ShuffleVector, OperandTraits<ShuffleVectorInst>::op_begin(this),
+          OperandTraits<ShuffleVectorInst>::operands(this), InsertBefore) {
   assert(isValidOperands(V1, V2, Mask) &&
          "Invalid shuffle vector instruction operands!");
+
   Op<0>() = V1;
   Op<1>() = V2;
-  Op<2>() = Mask;
+  SmallVector<int, 16> MaskArr;
+  getShuffleMask(cast<Constant>(Mask), MaskArr);
+  setShuffleMask(MaskArr);
   setName(Name);
 }
 
 ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, Value *Mask,
+                                     const Twine &Name, BasicBlock *InsertAtEnd)
+    : Instruction(
+          VectorType::get(cast<VectorType>(V1->getType())->getElementType(),
+                          cast<VectorType>(Mask->getType())->getElementCount()),
+          ShuffleVector, OperandTraits<ShuffleVectorInst>::op_begin(this),
+          OperandTraits<ShuffleVectorInst>::operands(this), InsertAtEnd) {
+  assert(isValidOperands(V1, V2, Mask) &&
+         "Invalid shuffle vector instruction operands!");
+
+  Op<0>() = V1;
+  Op<1>() = V2;
+  SmallVector<int, 16> MaskArr;
+  getShuffleMask(cast<Constant>(Mask), MaskArr);
+  setShuffleMask(MaskArr);
+  setName(Name);
+}
+
+ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
                                      const Twine &Name,
-                                     BasicBlock *InsertAtEnd)
-: Instruction(VectorType::get(cast<VectorType>(V1->getType())->getElementType(),
-                cast<VectorType>(Mask->getType())->getElementCount()),
-              ShuffleVector,
-              OperandTraits<ShuffleVectorInst>::op_begin(this),
-              OperandTraits<ShuffleVectorInst>::operands(this),
-              InsertAtEnd) {
+                                     Instruction *InsertBefore)
+    : Instruction(
+          VectorType::get(cast<VectorType>(V1->getType())->getElementType(),
+                          Mask.size(), V1->getType()->getVectorIsScalable()),
+          ShuffleVector, OperandTraits<ShuffleVectorInst>::op_begin(this),
+          OperandTraits<ShuffleVectorInst>::operands(this), InsertBefore) {
+  assert(isValidOperands(V1, V2, Mask) &&
+         "Invalid shuffle vector instruction operands!");
+  Op<0>() = V1;
+  Op<1>() = V2;
+  setShuffleMask(Mask);
+  setName(Name);
+}
+
+ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
+                                     const Twine &Name, BasicBlock *InsertAtEnd)
+    : Instruction(
+          VectorType::get(cast<VectorType>(V1->getType())->getElementType(),
+                          Mask.size(), V1->getType()->getVectorIsScalable()),
+          ShuffleVector, OperandTraits<ShuffleVectorInst>::op_begin(this),
+          OperandTraits<ShuffleVectorInst>::operands(this), InsertAtEnd) {
   assert(isValidOperands(V1, V2, Mask) &&
          "Invalid shuffle vector instruction operands!");
 
   Op<0>() = V1;
   Op<1>() = V2;
-  Op<2>() = Mask;
+  setShuffleMask(Mask);
   setName(Name);
 }
 
 void ShuffleVectorInst::commute() {
   int NumOpElts = Op<0>()->getType()->getVectorNumElements();
-  int NumMaskElts = getMask()->getType()->getVectorNumElements();
-  SmallVector<Constant*, 16> NewMask(NumMaskElts);
-  Type *Int32Ty = Type::getInt32Ty(getContext());
+  int NumMaskElts = ShuffleMask.size();
+  SmallVector<int, 16> NewMask(NumMaskElts);
   for (int i = 0; i != NumMaskElts; ++i) {
     int MaskElt = getMaskValue(i);
-    if (MaskElt == -1) {
-      NewMask[i] = UndefValue::get(Int32Ty);
+    if (MaskElt == UndefMaskElem) {
+      NewMask[i] = UndefMaskElem;
       continue;
     }
     assert(MaskElt >= 0 && MaskElt < 2 * NumOpElts && "Out-of-range mask");
     MaskElt = (MaskElt < NumOpElts) ? MaskElt + NumOpElts : MaskElt - NumOpElts;
-    NewMask[i] = ConstantInt::get(Int32Ty, MaskElt);
+    NewMask[i] = MaskElt;
   }
-  Op<2>() = ConstantVector::get(NewMask);
+  setShuffleMask(NewMask);
   Op<0>().swap(Op<1>());
 }
 
+bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2,
+                                        ArrayRef<int> Mask) {
+  // V1 and V2 must be vectors of the same type.
+  if (!V1->getType()->isVectorTy() || V1->getType() != V2->getType())
+    return false;
+
+  // Make sure the mask elements make sense.
+  int V1Size = cast<VectorType>(V1->getType())->getNumElements();
+  for (int Elem : Mask)
+    if (Elem != UndefMaskElem && Elem >= V1Size * 2)
+      return false;
+
+  if (V1->getType()->getVectorIsScalable())
+    if ((Mask[0] != 0 && Mask[0] != UndefMaskElem) || !is_splat(Mask))
+      return false;
+
+  return true;
+}
+
 bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2,
                                         const Value *Mask) {
   // V1 and V2 must be vectors of the same type.
@@ -1902,7 +1954,8 @@ bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2,
 
   // Mask must be vector of i32.
   auto *MaskTy = dyn_cast<VectorType>(Mask->getType());
-  if (!MaskTy || !MaskTy->getElementType()->isIntegerTy(32))
+  if (!MaskTy || !MaskTy->getElementType()->isIntegerTy(32) ||
+      MaskTy->isScalable() != V1->getType()->getVectorIsScalable())
     return false;
 
   // Check to see if Mask is valid.
@@ -1930,35 +1983,17 @@ bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2,
     return true;
   }
 
-  // The bitcode reader can create a place holder for a forward reference
-  // used as the shuffle mask. When this occurs, the shuffle mask will
-  // fall into this case and fail. To avoid this error, do this bit of
-  // ugliness to allow such a mask pass.
-  if (const auto *CE = dyn_cast<ConstantExpr>(Mask))
-    if (CE->getOpcode() == Instruction::UserOp1)
-      return true;
-
   return false;
 }
 
-int ShuffleVectorInst::getMaskValue(const Constant *Mask, unsigned i) {
-  assert(i < Mask->getType()->getVectorNumElements() && "Index out of range");
-  assert(!Mask->getType()->getVectorElementCount().Scalable &&
-    "Length of scalable vectors unknown at compile time");
-  if (auto *CDS = dyn_cast<ConstantDataSequential>(Mask))
-    return CDS->getElementAsInteger(i);
-  Constant *C = Mask->getAggregateElement(i);
-  if (isa<UndefValue>(C))
-    return -1;
-  return cast<ConstantInt>(C)->getZExtValue();
-}
-
 void ShuffleVectorInst::getShuffleMask(const Constant *Mask,
                                        SmallVectorImpl<int> &Result) {
-  assert(!Mask->getType()->getVectorElementCount().Scalable &&
-    "Length of scalable vectors unknown at compile time");
-  unsigned NumElts = Mask->getType()->getVectorNumElements();
-
+  unsigned NumElts = Mask->getType()->getVectorElementCount().Min;
+  if (isa<ConstantAggregateZero>(Mask)) {
+    Result.resize(NumElts, 0);
+    return;
+  }
+  Result.reserve(NumElts);
   if (auto *CDS = dyn_cast<ConstantDataSequential>(Mask)) {
     for (unsigned i = 0; i != NumElts; ++i)
       Result.push_back(CDS->getElementAsInteger(i));
@@ -1971,6 +2006,30 @@ void ShuffleVectorInst::getShuffleMask(const Constant *Mask,
   }
 }
 
+void ShuffleVectorInst::setShuffleMask(ArrayRef<int> Mask) {
+  ShuffleMask.assign(Mask.begin(), Mask.end());
+  ShuffleMaskForBitcode = convertShuffleMaskForBitcode(Mask, getType());
+}
+Constant *ShuffleVectorInst::convertShuffleMaskForBitcode(ArrayRef<int> Mask,
+                                                          Type *ResultTy) {
+  Type *Int32Ty = Type::getInt32Ty(ResultTy->getContext());
+  if (ResultTy->getVectorIsScalable()) {
+    assert(is_splat(Mask) && "Unexpected shuffle");
+    Type *VecTy = VectorType::get(Int32Ty, Mask.size(), true);
+    if (Mask[0] == 0)
+      return Constant::getNullValue(VecTy);
+    return UndefValue::get(VecTy);
+  }
+  SmallVector<Constant *, 16> MaskConst;
+  for (int Elem : Mask) {
+    if (Elem == UndefMaskElem)
+      MaskConst.push_back(UndefValue::get(Int32Ty));
+    else
+      MaskConst.push_back(ConstantInt::get(Int32Ty, Elem));
+  }
+  return ConstantVector::get(MaskConst);
+}
+
 static bool isSingleSourceMaskImpl(ArrayRef<int> Mask, int NumOpElts) {
   assert(!Mask.empty() && "Shuffle mask must contain elements");
   bool UsesLHS = false;
@@ -2120,7 +2179,7 @@ bool ShuffleVectorInst::isIdentityWithPadding() const {
     return false;
 
   // The first part of the mask must choose elements from exactly 1 source op.
-  SmallVector<int, 16> Mask = getShuffleMask();
+  ArrayRef<int> Mask = getShuffleMask();
   if (!isIdentityMaskImpl(Mask, NumOpElts))
     return false;
 
@@ -3237,57 +3296,54 @@ CastInst::castIsValid(Instruction::CastOps op, Value *S, Type *DstTy) {
       SrcTy->isAggregateType() || DstTy->isAggregateType())
     return false;
 
-  // Get the size of the types in bits, we'll need this later
-  unsigned SrcBitSize = SrcTy->getScalarSizeInBits();
-  unsigned DstBitSize = DstTy->getScalarSizeInBits();
+  // Get the size of the types in bits, and whether we are dealing
+  // with vector types, we'll need this later.
+  bool SrcIsVec = isa<VectorType>(SrcTy);
+  bool DstIsVec = isa<VectorType>(DstTy);
+  unsigned SrcScalarBitSize = SrcTy->getScalarSizeInBits();
+  unsigned DstScalarBitSize = DstTy->getScalarSizeInBits();
 
   // If these are vector types, get the lengths of the vectors (using zero for
   // scalar types means that checking that vector lengths match also checks that
   // scalars are not being converted to vectors or vectors to scalars).
-  unsigned SrcLength = SrcTy->isVectorTy() ?
-    cast<VectorType>(SrcTy)->getNumElements() : 0;
-  unsigned DstLength = DstTy->isVectorTy() ?
-    cast<VectorType>(DstTy)->getNumElements() : 0;
+  ElementCount SrcEC = SrcIsVec ? cast<VectorType>(SrcTy)->getElementCount()
+                                : ElementCount(0, false);
+  ElementCount DstEC = DstIsVec ? cast<VectorType>(DstTy)->getElementCount()
+                                : ElementCount(0, false);
 
   // Switch on the opcode provided
   switch (op) {
   default: return false; // This is an input error
   case Instruction::Trunc:
     return SrcTy->isIntOrIntVectorTy() && DstTy->isIntOrIntVectorTy() &&
-      SrcLength == DstLength && SrcBitSize > DstBitSize;
+           SrcEC == DstEC && SrcScalarBitSize > DstScalarBitSize;
   case Instruction::ZExt:
     return SrcTy->isIntOrIntVectorTy() && DstTy->isIntOrIntVectorTy() &&
-      SrcLength == DstLength && SrcBitSize < DstBitSize;
+           SrcEC == DstEC && SrcScalarBitSize < DstScalarBitSize;
   case Instruction::SExt:
     return SrcTy->isIntOrIntVectorTy() && DstTy->isIntOrIntVectorTy() &&
-      SrcLength == DstLength && SrcBitSize < DstBitSize;
+           SrcEC == DstEC && SrcScalarBitSize < DstScalarBitSize;
   case Instruction::FPTrunc:
     return SrcTy->isFPOrFPVectorTy() && DstTy->isFPOrFPVectorTy() &&
-      SrcLength == DstLength && SrcBitSize > DstBitSize;
+           SrcEC == DstEC && SrcScalarBitSize > DstScalarBitSize;
   case Instruction::FPExt:
     return SrcTy->isFPOrFPVectorTy() && DstTy->isFPOrFPVectorTy() &&
-      SrcLength == DstLength && SrcBitSize < DstBitSize;
+           SrcEC == DstEC && SrcScalarBitSize < DstScalarBitSize;
   case Instruction::UIToFP:
   case Instruction::SIToFP:
     return SrcTy->isIntOrIntVectorTy() && DstTy->isFPOrFPVectorTy() &&
-      SrcLength == DstLength;
+           SrcEC == DstEC;
   case Instruction::FPToUI:
   case Instruction::FPToSI:
     return SrcTy->isFPOrFPVectorTy() && DstTy->isIntOrIntVectorTy() &&
-      SrcLength == DstLength;
+           SrcEC == DstEC;
   case Instruction::PtrToInt:
-    if (isa<VectorType>(SrcTy) != isa<VectorType>(DstTy))
+    if (SrcEC != DstEC)
       return false;
-    if (VectorType *VT = dyn_cast<VectorType>(SrcTy))
-      if (VT->getNumElements() != cast<VectorType>(DstTy)->getNumElements())
-        return false;
     return SrcTy->isPtrOrPtrVectorTy() && DstTy->isIntOrIntVectorTy();
   case Instruction::IntToPtr:
-    if (isa<VectorType>(SrcTy) != isa<VectorType>(DstTy))
+    if (SrcEC != DstEC)
       return false;
-    if (VectorType *VT = dyn_cast<VectorType>(SrcTy))
-      if (VT->getNumElements() != cast<VectorType>(DstTy)->getNumElements())
-        return false;
     return SrcTy->isIntOrIntVectorTy() && DstTy->isPtrOrPtrVectorTy();
   case Instruction::BitCast: {
     PointerType *SrcPtrTy = dyn_cast<PointerType>(SrcTy->getScalarType());
@@ -3308,14 +3364,12 @@ CastInst::castIsValid(Instruction::CastOps op, Value *S, Type *DstTy) {
       return false;
 
     // A vector of pointers must have the same number of elements.
-    VectorType *SrcVecTy = dyn_cast<VectorType>(SrcTy);
-    VectorType *DstVecTy = dyn_cast<VectorType>(DstTy);
-    if (SrcVecTy && DstVecTy)
-      return (SrcVecTy->getNumElements() == DstVecTy->getNumElements());
-    if (SrcVecTy)
-      return SrcVecTy->getNumElements() == 1;
-    if (DstVecTy)
-      return DstVecTy->getNumElements() == 1;
+    if (SrcIsVec && DstIsVec)
+      return SrcEC == DstEC;
+    if (SrcIsVec)
+      return SrcEC == ElementCount(1, false);
+    if (DstIsVec)
+      return DstEC == ElementCount(1, false);
 
     return true;
   }
@@ -3331,14 +3385,7 @@ CastInst::castIsValid(Instruction::CastOps op, Value *S, Type *DstTy) {
     if (SrcPtrTy->getAddressSpace() == DstPtrTy->getAddressSpace())
       return false;
 
-    if (VectorType *SrcVecTy = dyn_cast<VectorType>(SrcTy)) {
-      if (VectorType *DstVecTy = dyn_cast<VectorType>(DstTy))
-        return (SrcVecTy->getNumElements() == DstVecTy->getNumElements());
-
-      return false;
-    }
-
-    return true;
+    return SrcEC == DstEC;
   }
   }
 }
@@ -4306,7 +4353,7 @@ InsertElementInst *InsertElementInst::cloneImpl() const {
 }
 
 ShuffleVectorInst *ShuffleVectorInst::cloneImpl() const {
-  return new ShuffleVectorInst(getOperand(0), getOperand(1), getOperand(2));
+  return new ShuffleVectorInst(getOperand(0), getOperand(1), getShuffleMask());
 }
 
 PHINode *PHINode::cloneImpl() const { return new PHINode(*this); }
diff --git a/llvm/lib/IR/MDBuilder.cpp b/llvm/lib/IR/MDBuilder.cpp
index 7bdb85ace5227..40d70f43132d6 100644
--- a/llvm/lib/IR/MDBuilder.cpp
+++ b/llvm/lib/IR/MDBuilder.cpp
@@ -68,7 +68,7 @@ MDNode *MDBuilder::createFunctionEntryCount(
   Ops.push_back(createConstant(ConstantInt::get(Int64Ty, Count)));
   if (Imports) {
     SmallVector<GlobalValue::GUID, 2> OrderID(Imports->begin(), Imports->end());
-    llvm::stable_sort(OrderID);
+    llvm::sort(OrderID);
     for (auto ID : OrderID)
       Ops.push_back(createConstant(ConstantInt::get(Int64Ty, ID)));
   }
diff --git a/llvm/lib/IR/Type.cpp b/llvm/lib/IR/Type.cpp
index e91bc8aa7e708..b288ab00fde3d 100644
--- a/llvm/lib/IR/Type.cpp
+++ b/llvm/lib/IR/Type.cpp
@@ -132,7 +132,8 @@ TypeSize Type::getPrimitiveSizeInBits() const {
 }
 
 unsigned Type::getScalarSizeInBits() const {
-  return getScalarType()->getPrimitiveSizeInBits();
+  // It is safe to assume that the scalar types have a fixed size.
+  return getScalarType()->getPrimitiveSizeInBits().getFixedSize();
 }
 
 int Type::getFPMantissaWidth() const {
diff --git a/llvm/lib/IR/Value.cpp b/llvm/lib/IR/Value.cpp
index 962e67437280e..afb509f601bcd 100644
--- a/llvm/lib/IR/Value.cpp
+++ b/llvm/lib/IR/Value.cpp
@@ -746,22 +746,22 @@ MaybeAlign Value::getPointerAlignment(const DataLayout &DL) const {
           if (GVar->isStrongDefinitionForLinker())
             return MaybeAlign(DL.getPreferredAlignment(GVar));
           else
-            return Align(DL.getABITypeAlignment(ObjectType));
+            return DL.getABITypeAlign(ObjectType);
         }
       }
     }
     return Alignment;
   } else if (const Argument *A = dyn_cast<Argument>(this)) {
-    const MaybeAlign Alignment(A->getParamAlignment());
+    const MaybeAlign Alignment = A->getParamAlign();
     if (!Alignment && A->hasStructRetAttr()) {
       // An sret parameter has at least the ABI alignment of the return type.
       Type *EltTy = cast<PointerType>(A->getType())->getElementType();
       if (EltTy->isSized())
-        return Align(DL.getABITypeAlignment(EltTy));
+        return DL.getABITypeAlign(EltTy);
     }
     return Alignment;
   } else if (const AllocaInst *AI = dyn_cast<AllocaInst>(this)) {
-    const MaybeAlign Alignment(AI->getAlignment());
+    const MaybeAlign Alignment = AI->getAlign();
     if (!Alignment) {
       Type *AllocatedType = AI->getAllocatedType();
       if (AllocatedType->isSized())
@@ -769,7 +769,7 @@ MaybeAlign Value::getPointerAlignment(const DataLayout &DL) const {
     }
     return Alignment;
   } else if (const auto *Call = dyn_cast<CallBase>(this)) {
-    const MaybeAlign Alignment(Call->getRetAlignment());
+    const MaybeAlign Alignment = Call->getRetAlign();
     if (!Alignment && Call->getCalledFunction())
       return MaybeAlign(
           Call->getCalledFunction()->getAttributes().getRetAlignment());
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index c8871a1f3e641..5a9a926fcaabb 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -3076,15 +3076,16 @@ static bool isTypeCongruent(Type *L, Type *R) {
 
 static AttrBuilder getParameterABIAttributes(int I, AttributeList Attrs) {
   static const Attribute::AttrKind ABIAttrs[] = {
-      Attribute::StructRet, Attribute::ByVal, Attribute::InAlloca,
-      Attribute::InReg, Attribute::Returned, Attribute::SwiftSelf,
-      Attribute::SwiftError};
+      Attribute::StructRet, Attribute::ByVal,     Attribute::InAlloca,
+      Attribute::InReg,     Attribute::SwiftSelf, Attribute::SwiftError};
   AttrBuilder Copy;
   for (auto AK : ABIAttrs) {
     if (Attrs.hasParamAttribute(I, AK))
       Copy.addAttribute(AK);
   }
-  if (Attrs.hasParamAttribute(I, Attribute::Alignment))
+  // `align` is ABI-affecting only in combination with `byval`.
+  if (Attrs.hasParamAttribute(I, Attribute::Alignment) &&
+      Attrs.hasParamAttribute(I, Attribute::ByVal))
     Copy.addAlignmentAttr(Attrs.getParamAlignment(I));
   return Copy;
 }
@@ -3308,7 +3309,7 @@ void Verifier::visitInsertElementInst(InsertElementInst &IE) {
 
 void Verifier::visitShuffleVectorInst(ShuffleVectorInst &SV) {
   Assert(ShuffleVectorInst::isValidOperands(SV.getOperand(0), SV.getOperand(1),
-                                            SV.getOperand(2)),
+                                            SV.getShuffleMask()),
          "Invalid shufflevector operands!", &SV);
   visitInstruction(SV);
 }
@@ -4789,6 +4790,42 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
     Assert(Size % 16 == 0, "bswap must be an even number of bytes", &Call);
     break;
   }
+  case Intrinsic::matrix_multiply:
+  case Intrinsic::matrix_transpose:
+  case Intrinsic::matrix_columnwise_load:
+  case Intrinsic::matrix_columnwise_store: {
+    ConstantInt *NumRows;
+    ConstantInt *NumColumns;
+    VectorType *TypeToCheck;
+    switch (ID) {
+    case Intrinsic::matrix_multiply:
+      NumRows = cast<ConstantInt>(Call.getArgOperand(2));
+      NumColumns = cast<ConstantInt>(Call.getArgOperand(4));
+      TypeToCheck = cast<VectorType>(Call.getType());
+      break;
+    case Intrinsic::matrix_transpose:
+      NumRows = cast<ConstantInt>(Call.getArgOperand(1));
+      NumColumns = cast<ConstantInt>(Call.getArgOperand(2));
+      TypeToCheck = cast<VectorType>(Call.getType());
+      break;
+    case Intrinsic::matrix_columnwise_load:
+      NumRows = cast<ConstantInt>(Call.getArgOperand(2));
+      NumColumns = cast<ConstantInt>(Call.getArgOperand(3));
+      TypeToCheck = cast<VectorType>(Call.getType());
+      break;
+    case Intrinsic::matrix_columnwise_store:
+      NumRows = cast<ConstantInt>(Call.getArgOperand(3));
+      NumColumns = cast<ConstantInt>(Call.getArgOperand(4));
+      TypeToCheck = cast<VectorType>(Call.getArgOperand(0)->getType());
+      break;
+    default:
+      llvm_unreachable("unexpected intrinsic");
+    }
+    Assert(TypeToCheck->getNumElements() ==
+               NumRows->getZExtValue() * NumColumns->getZExtValue(),
+           "result of a matrix operation does not fit in the returned vector");
+    break;
+  }
   };
 }
 
diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp
index f8affcb20ceff..f652bd6287700 100644
--- a/llvm/lib/LTO/LTO.cpp
+++ b/llvm/lib/LTO/LTO.cpp
@@ -477,7 +477,8 @@ LTO::RegularLTOState::RegularLTOState(unsigned ParallelCodeGenParallelismLevel,
 LTO::ThinLTOState::ThinLTOState(ThinBackend Backend)
     : Backend(Backend), CombinedIndex(/*HaveGVs*/ false) {
   if (!Backend)
-    this->Backend = createInProcessThinBackend();
+    this->Backend =
+        createInProcessThinBackend(llvm::heavyweight_hardware_concurrency());
 }
 
 LTO::LTO(Config Conf, ThinBackend Backend,
@@ -1090,13 +1091,12 @@ class InProcessThinBackend : public ThinBackendProc {
 public:
   InProcessThinBackend(
       const Config &Conf, ModuleSummaryIndex &CombinedIndex,
-      unsigned ThinLTOParallelismLevel,
+      ThreadPoolStrategy ThinLTOParallelism,
       const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
       AddStreamFn AddStream, NativeObjectCache Cache)
       : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries),
-        BackendThreadPool(
-            heavyweight_hardware_concurrency(ThinLTOParallelismLevel)),
-        AddStream(std::move(AddStream)), Cache(std::move(Cache)) {
+        BackendThreadPool(ThinLTOParallelism), AddStream(std::move(AddStream)),
+        Cache(std::move(Cache)) {
     for (auto &Name : CombinedIndex.cfiFunctionDefs())
       CfiFunctionDefs.insert(
           GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
@@ -1192,13 +1192,13 @@ class InProcessThinBackend : public ThinBackendProc {
 };
 } // end anonymous namespace
 
-ThinBackend lto::createInProcessThinBackend(unsigned ParallelismLevel) {
+ThinBackend lto::createInProcessThinBackend(ThreadPoolStrategy Parallelism) {
   return [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex,
              const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
              AddStreamFn AddStream, NativeObjectCache Cache) {
     return std::make_unique<InProcessThinBackend>(
-        Conf, CombinedIndex, ParallelismLevel, ModuleToDefinedGVSummaries,
-        AddStream, Cache);
+        Conf, CombinedIndex, Parallelism, ModuleToDefinedGVSummaries, AddStream,
+        Cache);
   };
 }
 
diff --git a/llvm/lib/MC/CMakeLists.txt b/llvm/lib/MC/CMakeLists.txt
index ab809daf5273d..9dca8d793395c 100644
--- a/llvm/lib/MC/CMakeLists.txt
+++ b/llvm/lib/MC/CMakeLists.txt
@@ -23,6 +23,7 @@ add_llvm_component_library(LLVMMC
   MCInstPrinter.cpp
   MCInstrAnalysis.cpp
   MCInstrDesc.cpp
+  MCInstrInfo.cpp
   MCLabel.cpp
   MCLinkerOptimizationHint.cpp
   MCMachOStreamer.cpp
@@ -43,6 +44,7 @@ add_llvm_component_library(LLVMMC
   MCSubtargetInfo.cpp
   MCSymbol.cpp
   MCSymbolELF.cpp
+  MCSymbolXCOFF.cpp
   MCTargetOptions.cpp
   MCTargetOptionsCommandFlags.cpp
   MCValue.cpp
diff --git a/llvm/lib/MC/MCAsmInfoCOFF.cpp b/llvm/lib/MC/MCAsmInfoCOFF.cpp
index 9f19d163f57bc..d9971028649c9 100644
--- a/llvm/lib/MC/MCAsmInfoCOFF.cpp
+++ b/llvm/lib/MC/MCAsmInfoCOFF.cpp
@@ -26,7 +26,7 @@ MCAsmInfoCOFF::MCAsmInfoCOFF() {
   HasDotTypeDotSizeDirective = false;
   HasSingleParameterDotFile = true;
   WeakRefDirective = "\t.weak\t";
-  HasLinkOnceDirective = true;
+  AvoidWeakIfComdat = true;
 
   // Doesn't support visibility:
   HiddenVisibilityAttr = HiddenDeclarationVisibilityAttr = MCSA_Invalid;
diff --git a/llvm/lib/MC/MCInstrDesc.cpp b/llvm/lib/MC/MCInstrDesc.cpp
index d54aeba89edc5..b5c43f5edc0d0 100644
--- a/llvm/lib/MC/MCInstrDesc.cpp
+++ b/llvm/lib/MC/MCInstrDesc.cpp
@@ -18,17 +18,6 @@
 
 using namespace llvm;
 
-bool MCInstrDesc::getDeprecatedInfo(MCInst &MI, const MCSubtargetInfo &STI,
-                                    std::string &Info) const {
-  if (ComplexDeprecationInfo)
-    return ComplexDeprecationInfo(MI, STI, Info);
-  if (DeprecatedFeature != -1 && STI.getFeatureBits()[DeprecatedFeature]) {
-    // FIXME: it would be nice to include the subtarget feature here.
-    Info = "deprecated";
-    return true;
-  }
-  return false;
-}
 bool MCInstrDesc::mayAffectControlFlow(const MCInst &MI,
                                        const MCRegisterInfo &RI) const {
   if (isBranch() || isCall() || isReturn() || isIndirectBranch())
diff --git a/llvm/lib/MC/MCInstrInfo.cpp b/llvm/lib/MC/MCInstrInfo.cpp
new file mode 100644
index 0000000000000..ab63db0405324
--- /dev/null
+++ b/llvm/lib/MC/MCInstrInfo.cpp
@@ -0,0 +1,27 @@
+//===- lib/MC/MCInstrInfo.cpp - Target Instruction Info -------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCSubtargetInfo.h"
+
+using namespace llvm;
+
+bool MCInstrInfo::getDeprecatedInfo(MCInst &MI, const MCSubtargetInfo &STI,
+                                    std::string &Info) const {
+  unsigned Opcode = MI.getOpcode();
+  if (ComplexDeprecationInfos && ComplexDeprecationInfos[Opcode])
+    return ComplexDeprecationInfos[Opcode](MI, STI, Info);
+  if (DeprecatedFeatures && DeprecatedFeatures[Opcode] != uint8_t(-1U) &&
+      STI.getFeatureBits()[DeprecatedFeatures[Opcode]]) {
+    // FIXME: it would be nice to include the subtarget feature here.
+    Info = "deprecated";
+    return true;
+  }
+  return false;
+}
diff --git a/llvm/lib/MC/MCSubtargetInfo.cpp b/llvm/lib/MC/MCSubtargetInfo.cpp
index 7d38cd0f729bc..efe1e95b73627 100644
--- a/llvm/lib/MC/MCSubtargetInfo.cpp
+++ b/llvm/lib/MC/MCSubtargetInfo.cpp
@@ -185,7 +185,7 @@ static FeatureBitset getFeatures(StringRef CPU, StringRef FS,
     // Check for help
     if (Feature == "+help")
       Help(ProcDesc, ProcFeatures);
-    else if (Feature == "+cpuHelp")
+    else if (Feature == "+cpuhelp")
       cpuHelp(ProcDesc);
     else
       ApplyFeatureFlag(Bits, Feature, ProcFeatures);
@@ -339,6 +339,13 @@ unsigned MCSubtargetInfo::getMaxPrefetchIterationsAhead() const {
   return UINT_MAX;
 }
 
-unsigned MCSubtargetInfo::getMinPrefetchStride() const {
+bool MCSubtargetInfo::enableWritePrefetching() const {
+  return false;
+}
+
+unsigned MCSubtargetInfo::getMinPrefetchStride(unsigned NumMemAccesses,
+                                               unsigned NumStridedMemAccesses,
+                                               unsigned NumPrefetches,
+                                               bool HasCall) const {
   return 1;
 }
diff --git a/llvm/lib/MC/MCSymbolXCOFF.cpp b/llvm/lib/MC/MCSymbolXCOFF.cpp
new file mode 100644
index 0000000000000..53aec88c89780
--- /dev/null
+++ b/llvm/lib/MC/MCSymbolXCOFF.cpp
@@ -0,0 +1,33 @@
+//===- lib/MC/MCSymbolXCOFF.cpp - XCOFF Code Symbol Representation --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/MC/MCSectionXCOFF.h"
+
+using namespace llvm;
+
+MCSectionXCOFF *MCSymbolXCOFF::getRepresentedCsect() const {
+  assert(RepresentedCsect &&
+         "Trying to get csect representation of this symbol but none was set.");
+  assert((!getName().equals(getUnqualifiedName()) ||
+          RepresentedCsect->getCSectType() == XCOFF::XTY_ER) &&
+         "Symbol does not represent a csect; MCSectionXCOFF that represents "
+         "the symbol should not be (but is) set.");
+  return RepresentedCsect;
+}
+
+void MCSymbolXCOFF::setRepresentedCsect(MCSectionXCOFF *C) {
+  assert(C && "Assigned csect should not be null.");
+  assert((!RepresentedCsect || RepresentedCsect == C) &&
+         "Trying to set a csect that doesn't match the one that"
+         "this symbol is already mapped to.");
+  assert((!getName().equals(getUnqualifiedName()) ||
+          C->getCSectType() == XCOFF::XTY_ER) &&
+         "Symbol does not represent a csect; can only set a MCSectionXCOFF "
+         "representation for a csect.");
+  RepresentedCsect = C;
+}
diff --git a/llvm/lib/MC/XCOFFObjectWriter.cpp b/llvm/lib/MC/XCOFFObjectWriter.cpp
index 2c054426fa2a0..fc0f067fd7fe8 100644
--- a/llvm/lib/MC/XCOFFObjectWriter.cpp
+++ b/llvm/lib/MC/XCOFFObjectWriter.cpp
@@ -313,6 +313,12 @@ CsectGroup &XCOFFObjectWriter::getCsectGroup(const MCSectionXCOFF *MCSec) {
   }
 }
 
+static MCSectionXCOFF *getContainingCsect(const MCSymbolXCOFF *XSym) {
+  if (XSym->isDefined())
+    return cast<MCSectionXCOFF>(XSym->getFragment()->getParent());
+  return XSym->getRepresentedCsect();
+}
+
 void XCOFFObjectWriter::executePostLayoutBinding(MCAssembler &Asm,
                                                  const MCAsmLayout &Layout) {
   if (TargetObjectWriter->is64Bit())
@@ -341,7 +347,7 @@ void XCOFFObjectWriter::executePostLayoutBinding(MCAssembler &Asm,
       continue;
 
     const MCSymbolXCOFF *XSym = cast<MCSymbolXCOFF>(&S);
-    const MCSectionXCOFF *ContainingCsect = XSym->getContainingCsect();
+    const MCSectionXCOFF *ContainingCsect = getContainingCsect(XSym);
 
     if (ContainingCsect->getCSectType() == XCOFF::XTY_ER) {
       // Handle undefined symbol.
@@ -394,7 +400,7 @@ void XCOFFObjectWriter::recordRelocation(MCAssembler &Asm,
       TargetObjectWriter->getRelocTypeAndSignSize(Target, Fixup, IsPCRel);
 
   const MCSectionXCOFF *SymASec =
-      cast<MCSymbolXCOFF>(SymA).getContainingCsect();
+      getContainingCsect(cast<MCSymbolXCOFF>(&SymA));
   assert(SectionMap.find(SymASec) != SectionMap.end() &&
          "Expected containing csect to exist in map.");
 
diff --git a/llvm/lib/Object/COFFObjectFile.cpp b/llvm/lib/Object/COFFObjectFile.cpp
index 384c19d4fb1a0..1fd95451e72d7 100644
--- a/llvm/lib/Object/COFFObjectFile.cpp
+++ b/llvm/lib/Object/COFFObjectFile.cpp
@@ -328,6 +328,12 @@ bool COFFObjectFile::isSectionBSS(DataRefImpl Ref) const {
   return (Sec->Characteristics & BssFlags) == BssFlags;
 }
 
+// The .debug sections are the only debug sections for COFF
+// (\see MCObjectFileInfo.cpp).
+bool COFFObjectFile::isDebugSection(StringRef SectionName) const {
+  return SectionName.startswith(".debug");
+}
+
 unsigned COFFObjectFile::getSectionID(SectionRef Sec) const {
   uintptr_t Offset =
       uintptr_t(Sec.getRawDataRefImpl().p) - uintptr_t(SectionTable);
diff --git a/llvm/lib/Object/ELF.cpp b/llvm/lib/Object/ELF.cpp
index f17a6da23d7d3..e58d62382a554 100644
--- a/llvm/lib/Object/ELF.cpp
+++ b/llvm/lib/Object/ELF.cpp
@@ -223,6 +223,9 @@ StringRef llvm::object::getELFSectionTypeName(uint32_t Machine, unsigned Type) {
       STRINGIFY_ENUM_CASE(ELF, SHT_MIPS_ABIFLAGS);
     }
     break;
+  case ELF::EM_RISCV:
+    switch (Type) { STRINGIFY_ENUM_CASE(ELF, SHT_RISCV_ATTRIBUTES); }
+    break;
   default:
     break;
   }
@@ -580,7 +583,18 @@ Expected<const uint8_t *> ELFFile<ELFT>::toMappedAddr(uint64_t VAddr) const {
   if (Delta >= Phdr.p_filesz)
     return createError("virtual address is not in any segment: 0x" +
                        Twine::utohexstr(VAddr));
-  return base() + Phdr.p_offset + Delta;
+
+  uint64_t Offset = Phdr.p_offset + Delta;
+  if (Offset >= getBufSize())
+    return createError("can't map virtual address 0x" +
+                       Twine::utohexstr(VAddr) + " to the segment with index " +
+                       Twine(&Phdr - (*ProgramHeadersOrError).data() + 1) +
+                       ": the segment ends at 0x" +
+                       Twine::utohexstr(Phdr.p_offset + Phdr.p_filesz) +
+                       ", which is greater than the file size (0x" +
+                       Twine::utohexstr(getBufSize()) + ")");
+
+  return base() + Offset;
 }
 
 template class llvm::object::ELFFile<ELF32LE>;
diff --git a/llvm/lib/Object/ELFObjectFile.cpp b/llvm/lib/Object/ELFObjectFile.cpp
index 855e032417fde..8627d1b3751ba 100644
--- a/llvm/lib/Object/ELFObjectFile.cpp
+++ b/llvm/lib/Object/ELFObjectFile.cpp
@@ -23,6 +23,8 @@
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MathExtras.h"
+#include "llvm/Support/RISCVAttributeParser.h"
+#include "llvm/Support/RISCVAttributes.h"
 #include "llvm/Support/TargetRegistry.h"
 #include <algorithm>
 #include <cstddef>
@@ -164,12 +166,14 @@ SubtargetFeatures ELFObjectFileBase::getARMFeatures() const {
 
   // both ARMv7-M and R have to support thumb hardware div
   bool isV7 = false;
-  if (Attributes.hasAttribute(ARMBuildAttrs::CPU_arch))
-    isV7 = Attributes.getAttributeValue(ARMBuildAttrs::CPU_arch)
-      == ARMBuildAttrs::v7;
-
-  if (Attributes.hasAttribute(ARMBuildAttrs::CPU_arch_profile)) {
-    switch(Attributes.getAttributeValue(ARMBuildAttrs::CPU_arch_profile)) {
+  Optional<unsigned> Attr =
+      Attributes.getAttributeValue(ARMBuildAttrs::CPU_arch);
+  if (Attr.hasValue())
+    isV7 = Attr.getValue() == ARMBuildAttrs::v7;
+
+  Attr = Attributes.getAttributeValue(ARMBuildAttrs::CPU_arch_profile);
+  if (Attr.hasValue()) {
+    switch (Attr.getValue()) {
     case ARMBuildAttrs::ApplicationProfile:
       Features.AddFeature("aclass");
       break;
@@ -186,8 +190,9 @@ SubtargetFeatures ELFObjectFileBase::getARMFeatures() const {
     }
   }
 
-  if (Attributes.hasAttribute(ARMBuildAttrs::THUMB_ISA_use)) {
-    switch(Attributes.getAttributeValue(ARMBuildAttrs::THUMB_ISA_use)) {
+  Attr = Attributes.getAttributeValue(ARMBuildAttrs::THUMB_ISA_use);
+  if (Attr.hasValue()) {
+    switch (Attr.getValue()) {
     default:
       break;
     case ARMBuildAttrs::Not_Allowed:
@@ -200,8 +205,9 @@ SubtargetFeatures ELFObjectFileBase::getARMFeatures() const {
     }
   }
 
-  if (Attributes.hasAttribute(ARMBuildAttrs::FP_arch)) {
-    switch(Attributes.getAttributeValue(ARMBuildAttrs::FP_arch)) {
+  Attr = Attributes.getAttributeValue(ARMBuildAttrs::FP_arch);
+  if (Attr.hasValue()) {
+    switch (Attr.getValue()) {
     default:
       break;
     case ARMBuildAttrs::Not_Allowed:
@@ -223,8 +229,9 @@ SubtargetFeatures ELFObjectFileBase::getARMFeatures() const {
     }
   }
 
-  if (Attributes.hasAttribute(ARMBuildAttrs::Advanced_SIMD_arch)) {
-    switch(Attributes.getAttributeValue(ARMBuildAttrs::Advanced_SIMD_arch)) {
+  Attr = Attributes.getAttributeValue(ARMBuildAttrs::Advanced_SIMD_arch);
+  if (Attr.hasValue()) {
+    switch (Attr.getValue()) {
     default:
       break;
     case ARMBuildAttrs::Not_Allowed:
@@ -241,8 +248,9 @@ SubtargetFeatures ELFObjectFileBase::getARMFeatures() const {
     }
   }
 
-  if (Attributes.hasAttribute(ARMBuildAttrs::MVE_arch)) {
-    switch(Attributes.getAttributeValue(ARMBuildAttrs::MVE_arch)) {
+  Attr = Attributes.getAttributeValue(ARMBuildAttrs::MVE_arch);
+  if (Attr.hasValue()) {
+    switch (Attr.getValue()) {
     default:
       break;
     case ARMBuildAttrs::Not_Allowed:
@@ -259,8 +267,9 @@ SubtargetFeatures ELFObjectFileBase::getARMFeatures() const {
     }
   }
 
-  if (Attributes.hasAttribute(ARMBuildAttrs::DIV_use)) {
-    switch(Attributes.getAttributeValue(ARMBuildAttrs::DIV_use)) {
+  Attr = Attributes.getAttributeValue(ARMBuildAttrs::DIV_use);
+  if (Attr.hasValue()) {
+    switch (Attr.getValue()) {
     default:
       break;
     case ARMBuildAttrs::DisallowDIV:
@@ -285,6 +294,48 @@ SubtargetFeatures ELFObjectFileBase::getRISCVFeatures() const {
     Features.AddFeature("c");
   }
 
+  // Add features according to the ELF attribute section.
+  // If there are any unrecognized features, ignore them.
+  RISCVAttributeParser Attributes;
+  if (Error E = getBuildAttributes(Attributes))
+    return Features; // Keep "c" feature if there is one in PlatformFlags.
+
+  Optional<StringRef> Attr = Attributes.getAttributeString(RISCVAttrs::ARCH);
+  if (Attr.hasValue()) {
+    // The Arch pattern is [rv32|rv64][i|e]version(_[m|a|f|d|c]version)*
+    // Version string pattern is (major)p(minor). Major and minor are optional.
+    // For example, a version number could be 2p0, 2, or p92.
+    StringRef Arch = Attr.getValue();
+    if (Arch.consume_front("rv32"))
+      Features.AddFeature("64bit", false);
+    else if (Arch.consume_front("rv64"))
+      Features.AddFeature("64bit");
+
+    while (!Arch.empty()) {
+      switch (Arch[0]) {
+      default:
+        break; // Ignore unexpected features.
+      case 'i':
+        Features.AddFeature("e", false);
+        break;
+      case 'd':
+        Features.AddFeature("f"); // D-ext will imply F-ext.
+        LLVM_FALLTHROUGH;
+      case 'e':
+      case 'm':
+      case 'a':
+      case 'f':
+      case 'c':
+        Features.AddFeature(Arch.take_front());
+        break;
+      }
+
+      // FIXME: Handle version numbers.
+      Arch = Arch.drop_until([](char c) { return c == '_' || c == '\0'; });
+      Arch = Arch.drop_while([](char c) { return c == '_'; });
+    }
+  }
+
   return Features;
 }
 
@@ -320,8 +371,10 @@ void ELFObjectFileBase::setARMSubArch(Triple &TheTriple) const {
   else
     Triple = "arm";
 
-  if (Attributes.hasAttribute(ARMBuildAttrs::CPU_arch)) {
-    switch(Attributes.getAttributeValue(ARMBuildAttrs::CPU_arch)) {
+  Optional<unsigned> Attr =
+      Attributes.getAttributeValue(ARMBuildAttrs::CPU_arch);
+  if (Attr.hasValue()) {
+    switch (Attr.getValue()) {
     case ARMBuildAttrs::v4:
       Triple += "v4";
       break;
diff --git a/llvm/lib/Object/MachOObjectFile.cpp b/llvm/lib/Object/MachOObjectFile.cpp
index feea11cafbaca..c0aa0b8b3ee31 100644
--- a/llvm/lib/Object/MachOObjectFile.cpp
+++ b/llvm/lib/Object/MachOObjectFile.cpp
@@ -2030,6 +2030,11 @@ bool MachOObjectFile::isSectionBSS(DataRefImpl Sec) const {
           SectionType == MachO::S_GB_ZEROFILL);
 }
 
+bool MachOObjectFile::isDebugSection(StringRef SectionName) const {
+  return SectionName.startswith("__debug") ||
+         SectionName.startswith("__zdebug") || SectionName == "__gdb_index";
+}
+
 unsigned MachOObjectFile::getSectionID(SectionRef Sec) const {
   return Sec.getRawDataRefImpl().d.a;
 }
diff --git a/llvm/lib/Object/ObjectFile.cpp b/llvm/lib/Object/ObjectFile.cpp
index 098b3d8f8dd05..1453f9d88130f 100644
--- a/llvm/lib/Object/ObjectFile.cpp
+++ b/llvm/lib/Object/ObjectFile.cpp
@@ -91,6 +91,10 @@ bool ObjectFile::isBerkeleyData(DataRefImpl Sec) const {
   return isSectionData(Sec);
 }
 
+bool ObjectFile::isDebugSection(StringRef SectionName) const {
+  return false;
+}
+
 Expected<section_iterator>
 ObjectFile::getRelocatedSection(DataRefImpl Sec) const {
   return section_iterator(SectionRef(Sec, this));
@@ -108,14 +112,17 @@ Triple ObjectFile::makeTriple() const {
     setARMSubArch(TheTriple);
 
   // TheTriple defaults to ELF, and COFF doesn't have an environment:
-  // the best we can do here is indicate that it is mach-o.
-  if (isMachO())
+  // something we can do here is indicate that it is mach-o.
+  if (isMachO()) {
     TheTriple.setObjectFormat(Triple::MachO);
-
-  if (isCOFF()) {
+  } else if (isCOFF()) {
     const auto COFFObj = cast<COFFObjectFile>(this);
     if (COFFObj->getArch() == Triple::thumb)
       TheTriple.setTriple("thumbv7-windows");
+  } else if (isXCOFF()) {
+    // XCOFF implies AIX.
+    TheTriple.setOS(Triple::AIX);
+    TheTriple.setObjectFormat(Triple::XCOFF);
   }
 
   return TheTriple;
diff --git a/llvm/lib/Object/WasmObjectFile.cpp b/llvm/lib/Object/WasmObjectFile.cpp
index ca11ec3ad7bbd..e9a8e08144729 100644
--- a/llvm/lib/Object/WasmObjectFile.cpp
+++ b/llvm/lib/Object/WasmObjectFile.cpp
@@ -302,10 +302,10 @@ Error WasmObjectFile::parseSection(WasmSection &Sec) {
     return parseTableSection(Ctx);
   case wasm::WASM_SEC_MEMORY:
     return parseMemorySection(Ctx);
-  case wasm::WASM_SEC_GLOBAL:
-    return parseGlobalSection(Ctx);
   case wasm::WASM_SEC_EVENT:
     return parseEventSection(Ctx);
+  case wasm::WASM_SEC_GLOBAL:
+    return parseGlobalSection(Ctx);
   case wasm::WASM_SEC_EXPORT:
     return parseExportSection(Ctx);
   case wasm::WASM_SEC_START:
@@ -993,6 +993,24 @@ Error WasmObjectFile::parseMemorySection(ReadContext &Ctx) {
   return Error::success();
 }
 
+Error WasmObjectFile::parseEventSection(ReadContext &Ctx) {
+  EventSection = Sections.size();
+  uint32_t Count = readVarint32(Ctx);
+  Events.reserve(Count);
+  while (Count--) {
+    wasm::WasmEvent Event;
+    Event.Index = NumImportedEvents + Events.size();
+    Event.Type.Attribute = readVaruint32(Ctx);
+    Event.Type.SigIndex = readVarint32(Ctx);
+    Events.push_back(Event);
+  }
+
+  if (Ctx.Ptr != Ctx.End)
+    return make_error<GenericBinaryError>("Event section ended prematurely",
+                                          object_error::parse_failed);
+  return Error::success();
+}
+
 Error WasmObjectFile::parseGlobalSection(ReadContext &Ctx) {
   GlobalSection = Sections.size();
   uint32_t Count = readVaruint32(Ctx);
@@ -1012,24 +1030,6 @@ Error WasmObjectFile::parseGlobalSection(ReadContext &Ctx) {
   return Error::success();
 }
 
-Error WasmObjectFile::parseEventSection(ReadContext &Ctx) {
-  EventSection = Sections.size();
-  uint32_t Count = readVarint32(Ctx);
-  Events.reserve(Count);
-  while (Count--) {
-    wasm::WasmEvent Event;
-    Event.Index = NumImportedEvents + Events.size();
-    Event.Type.Attribute = readVaruint32(Ctx);
-    Event.Type.SigIndex = readVarint32(Ctx);
-    Events.push_back(Event);
-  }
-
-  if (Ctx.Ptr != Ctx.End)
-    return make_error<GenericBinaryError>("Event section ended prematurely",
-                                          object_error::parse_failed);
-  return Error::success();
-}
-
 Error WasmObjectFile::parseExportSection(ReadContext &Ctx) {
   uint32_t Count = readVaruint32(Ctx);
   Exports.reserve(Count);
@@ -1622,30 +1622,50 @@ int WasmSectionOrderChecker::getSectionOrder(unsigned ID,
 // Represents the edges in a directed graph where any node B reachable from node
 // A is not allowed to appear before A in the section ordering, but may appear
 // afterward.
-int WasmSectionOrderChecker::DisallowedPredecessors[WASM_NUM_SEC_ORDERS][WASM_NUM_SEC_ORDERS] = {
-  {}, // WASM_SEC_ORDER_NONE
-  {WASM_SEC_ORDER_TYPE, WASM_SEC_ORDER_IMPORT}, // WASM_SEC_ORDER_TYPE,
-  {WASM_SEC_ORDER_IMPORT, WASM_SEC_ORDER_FUNCTION}, // WASM_SEC_ORDER_IMPORT,
-  {WASM_SEC_ORDER_FUNCTION, WASM_SEC_ORDER_TABLE}, // WASM_SEC_ORDER_FUNCTION,
-  {WASM_SEC_ORDER_TABLE, WASM_SEC_ORDER_MEMORY}, // WASM_SEC_ORDER_TABLE,
-  {WASM_SEC_ORDER_MEMORY, WASM_SEC_ORDER_GLOBAL}, // WASM_SEC_ORDER_MEMORY,
-  {WASM_SEC_ORDER_GLOBAL, WASM_SEC_ORDER_EVENT}, // WASM_SEC_ORDER_GLOBAL,
-  {WASM_SEC_ORDER_EVENT, WASM_SEC_ORDER_EXPORT}, // WASM_SEC_ORDER_EVENT,
-  {WASM_SEC_ORDER_EXPORT, WASM_SEC_ORDER_START}, // WASM_SEC_ORDER_EXPORT,
-  {WASM_SEC_ORDER_START, WASM_SEC_ORDER_ELEM}, // WASM_SEC_ORDER_START,
-  {WASM_SEC_ORDER_ELEM, WASM_SEC_ORDER_DATACOUNT}, // WASM_SEC_ORDER_ELEM,
-  {WASM_SEC_ORDER_DATACOUNT, WASM_SEC_ORDER_CODE}, // WASM_SEC_ORDER_DATACOUNT,
-  {WASM_SEC_ORDER_CODE, WASM_SEC_ORDER_DATA}, // WASM_SEC_ORDER_CODE,
-  {WASM_SEC_ORDER_DATA, WASM_SEC_ORDER_LINKING}, // WASM_SEC_ORDER_DATA,
-
-  // Custom Sections
-  {WASM_SEC_ORDER_DYLINK, WASM_SEC_ORDER_TYPE}, // WASM_SEC_ORDER_DYLINK,
-  {WASM_SEC_ORDER_LINKING, WASM_SEC_ORDER_RELOC, WASM_SEC_ORDER_NAME}, // WASM_SEC_ORDER_LINKING,
-  {}, // WASM_SEC_ORDER_RELOC (can be repeated),
-  {WASM_SEC_ORDER_NAME, WASM_SEC_ORDER_PRODUCERS}, // WASM_SEC_ORDER_NAME,
-  {WASM_SEC_ORDER_PRODUCERS, WASM_SEC_ORDER_TARGET_FEATURES}, // WASM_SEC_ORDER_PRODUCERS,
-  {WASM_SEC_ORDER_TARGET_FEATURES}  // WASM_SEC_ORDER_TARGET_FEATURES
-};
+int WasmSectionOrderChecker::DisallowedPredecessors
+    [WASM_NUM_SEC_ORDERS][WASM_NUM_SEC_ORDERS] = {
+        // WASM_SEC_ORDER_NONE
+        {},
+        // WASM_SEC_ORDER_TYPE
+        {WASM_SEC_ORDER_TYPE, WASM_SEC_ORDER_IMPORT},
+        // WASM_SEC_ORDER_IMPORT
+        {WASM_SEC_ORDER_IMPORT, WASM_SEC_ORDER_FUNCTION},
+        // WASM_SEC_ORDER_FUNCTION
+        {WASM_SEC_ORDER_FUNCTION, WASM_SEC_ORDER_TABLE},
+        // WASM_SEC_ORDER_TABLE
+        {WASM_SEC_ORDER_TABLE, WASM_SEC_ORDER_MEMORY},
+        // WASM_SEC_ORDER_MEMORY
+        {WASM_SEC_ORDER_MEMORY, WASM_SEC_ORDER_EVENT},
+        // WASM_SEC_ORDER_EVENT
+        {WASM_SEC_ORDER_EVENT, WASM_SEC_ORDER_GLOBAL},
+        // WASM_SEC_ORDER_GLOBAL
+        {WASM_SEC_ORDER_GLOBAL, WASM_SEC_ORDER_EXPORT},
+        // WASM_SEC_ORDER_EXPORT
+        {WASM_SEC_ORDER_EXPORT, WASM_SEC_ORDER_START},
+        // WASM_SEC_ORDER_START
+        {WASM_SEC_ORDER_START, WASM_SEC_ORDER_ELEM},
+        // WASM_SEC_ORDER_ELEM
+        {WASM_SEC_ORDER_ELEM, WASM_SEC_ORDER_DATACOUNT},
+        // WASM_SEC_ORDER_DATACOUNT
+        {WASM_SEC_ORDER_DATACOUNT, WASM_SEC_ORDER_CODE},
+        // WASM_SEC_ORDER_CODE
+        {WASM_SEC_ORDER_CODE, WASM_SEC_ORDER_DATA},
+        // WASM_SEC_ORDER_DATA
+        {WASM_SEC_ORDER_DATA, WASM_SEC_ORDER_LINKING},
+
+        // Custom Sections
+        // WASM_SEC_ORDER_DYLINK
+        {WASM_SEC_ORDER_DYLINK, WASM_SEC_ORDER_TYPE},
+        // WASM_SEC_ORDER_LINKING
+        {WASM_SEC_ORDER_LINKING, WASM_SEC_ORDER_RELOC, WASM_SEC_ORDER_NAME},
+        // WASM_SEC_ORDER_RELOC (can be repeated)
+        {},
+        // WASM_SEC_ORDER_NAME
+        {WASM_SEC_ORDER_NAME, WASM_SEC_ORDER_PRODUCERS},
+        // WASM_SEC_ORDER_PRODUCERS
+        {WASM_SEC_ORDER_PRODUCERS, WASM_SEC_ORDER_TARGET_FEATURES},
+        // WASM_SEC_ORDER_TARGET_FEATURES
+        {WASM_SEC_ORDER_TARGET_FEATURES}};
 
 bool WasmSectionOrderChecker::isValidSectionOrder(unsigned ID,
                                                   StringRef CustomSectionName) {
diff --git a/llvm/lib/Object/XCOFFObjectFile.cpp b/llvm/lib/Object/XCOFFObjectFile.cpp
index 51fc661016469..d56c1689aaae3 100644
--- a/llvm/lib/Object/XCOFFObjectFile.cpp
+++ b/llvm/lib/Object/XCOFFObjectFile.cpp
@@ -17,7 +17,10 @@
 namespace llvm {
 namespace object {
 
-enum { FUNCTION_SYM = 0x20, SYM_TYPE_MASK = 0x07, RELOC_OVERFLOW = 65535 };
+static const uint8_t FunctionSym = 0x20;
+static const uint8_t SymTypeMask = 0x07;
+static const uint16_t RelocOverflow = 65535;
+static const uint16_t NoRelMask = 0x0001;
 
 // Checks that [Ptr, Ptr + Size) bytes fall inside the memory buffer
 // 'M'. Returns a pointer to the underlying object on success.
@@ -314,41 +317,79 @@ bool XCOFFObjectFile::isSectionVirtual(DataRefImpl Sec) const {
 }
 
 relocation_iterator XCOFFObjectFile::section_rel_begin(DataRefImpl Sec) const {
-  llvm_unreachable("Not yet implemented!");
-  return relocation_iterator(RelocationRef());
+  if (is64Bit())
+    report_fatal_error("64-bit support not implemented yet");
+  const XCOFFSectionHeader32 *SectionEntPtr = toSection32(Sec);
+  auto RelocationsOrErr = relocations(*SectionEntPtr);
+  if (Error E = RelocationsOrErr.takeError())
+    return relocation_iterator(RelocationRef());
+  DataRefImpl Ret;
+  Ret.p = reinterpret_cast<uintptr_t>(&*RelocationsOrErr.get().begin());
+  return relocation_iterator(RelocationRef(Ret, this));
 }
 
 relocation_iterator XCOFFObjectFile::section_rel_end(DataRefImpl Sec) const {
-  llvm_unreachable("Not yet implemented!");
-  return relocation_iterator(RelocationRef());
+  if (is64Bit())
+    report_fatal_error("64-bit support not implemented yet");
+  const XCOFFSectionHeader32 *SectionEntPtr = toSection32(Sec);
+  auto RelocationsOrErr = relocations(*SectionEntPtr);
+  if (Error E = RelocationsOrErr.takeError())
+    return relocation_iterator(RelocationRef());
+  DataRefImpl Ret;
+  Ret.p = reinterpret_cast<uintptr_t>(&*RelocationsOrErr.get().end());
+  return relocation_iterator(RelocationRef(Ret, this));
 }
 
 void XCOFFObjectFile::moveRelocationNext(DataRefImpl &Rel) const {
-  llvm_unreachable("Not yet implemented!");
-  return;
+  Rel.p = reinterpret_cast<uintptr_t>(viewAs<XCOFFRelocation32>(Rel.p) + 1);
 }
 
 uint64_t XCOFFObjectFile::getRelocationOffset(DataRefImpl Rel) const {
-  llvm_unreachable("Not yet implemented!");
-  uint64_t Result = 0;
-  return Result;
+  if (is64Bit())
+    report_fatal_error("64-bit support not implemented yet");
+  const XCOFFRelocation32 *Reloc = viewAs<XCOFFRelocation32>(Rel.p);
+  const XCOFFSectionHeader32 *Sec32 = sectionHeaderTable32();
+  const uint32_t RelocAddress = Reloc->VirtualAddress;
+  const uint16_t NumberOfSections = getNumberOfSections();
+  for (uint16_t i = 0; i < NumberOfSections; ++i) {
+    // Find which section this relocation is belonging to, and get the
+    // relocation offset relative to the start of the section.
+    if (Sec32->VirtualAddress <= RelocAddress &&
+        RelocAddress < Sec32->VirtualAddress + Sec32->SectionSize) {
+      return RelocAddress - Sec32->VirtualAddress;
+    }
+    ++Sec32;
+  }
+  return InvalidRelocOffset;
 }
 
 symbol_iterator XCOFFObjectFile::getRelocationSymbol(DataRefImpl Rel) const {
-  llvm_unreachable("Not yet implemented!");
-  return symbol_iterator(SymbolRef());
+  if (is64Bit())
+    report_fatal_error("64-bit support not implemented yet");
+  const XCOFFRelocation32 *Reloc = viewAs<XCOFFRelocation32>(Rel.p);
+  const uint32_t Index = Reloc->SymbolIndex;
+
+  if (Index >= getLogicalNumberOfSymbolTableEntries32())
+    return symbol_end();
+
+  DataRefImpl SymDRI;
+  SymDRI.p = reinterpret_cast<uintptr_t>(getPointerToSymbolTable() + Index);
+  return symbol_iterator(SymbolRef(SymDRI, this));
 }
 
 uint64_t XCOFFObjectFile::getRelocationType(DataRefImpl Rel) const {
-  llvm_unreachable("Not yet implemented!");
-  uint64_t Result = 0;
-  return Result;
+  if (is64Bit())
+    report_fatal_error("64-bit support not implemented yet");
+  return viewAs<XCOFFRelocation32>(Rel.p)->Type;
 }
 
 void XCOFFObjectFile::getRelocationTypeName(
     DataRefImpl Rel, SmallVectorImpl<char> &Result) const {
-  llvm_unreachable("Not yet implemented!");
-  return;
+  if (is64Bit())
+    report_fatal_error("64-bit support not implemented yet");
+  const XCOFFRelocation32 *Reloc = viewAs<XCOFFRelocation32>(Rel.p);
+  StringRef Res = XCOFF::getRelocationTypeString(Reloc->Type);
+  Result.append(Res.begin(), Res.end());
 }
 
 uint32_t XCOFFObjectFile::getSymbolFlags(DataRefImpl Symb) const {
@@ -358,14 +399,16 @@ uint32_t XCOFFObjectFile::getSymbolFlags(DataRefImpl Symb) const {
 }
 
 basic_symbol_iterator XCOFFObjectFile::symbol_begin() const {
-  assert(!is64Bit() && "64-bit support not implemented yet.");
+  if (is64Bit())
+    report_fatal_error("64-bit support not implemented yet");
   DataRefImpl SymDRI;
   SymDRI.p = reinterpret_cast<uintptr_t>(SymbolTblPtr);
   return basic_symbol_iterator(SymbolRef(SymDRI, this));
 }
 
 basic_symbol_iterator XCOFFObjectFile::symbol_end() const {
-  assert(!is64Bit() && "64-bit support not implemented yet.");
+  if (is64Bit())
+    report_fatal_error("64-bit support not implemented yet");
   DataRefImpl SymDRI;
   SymDRI.p = reinterpret_cast<uintptr_t>(
       SymbolTblPtr + getLogicalNumberOfSymbolTableEntries32());
@@ -400,9 +443,9 @@ SubtargetFeatures XCOFFObjectFile::getFeatures() const {
 }
 
 bool XCOFFObjectFile::isRelocatableObject() const {
-  bool Result = false;
-  llvm_unreachable("Not yet implemented!");
-  return Result;
+  if (is64Bit())
+    report_fatal_error("64-bit support not implemented yet");
+  return !(fileHeader32()->Flags & NoRelMask);
 }
 
 Expected<uint64_t> XCOFFObjectFile::getStartAddress() const {
@@ -588,7 +631,7 @@ Expected<uint32_t> XCOFFObjectFile::getLogicalNumberOfRelocationEntries(
 
   uint16_t SectionIndex = &Sec - sectionHeaderTable32() + 1;
 
-  if (Sec.NumberOfRelocations < RELOC_OVERFLOW)
+  if (Sec.NumberOfRelocations < RelocOverflow)
     return Sec.NumberOfRelocations;
   for (const auto &Sec : sections32()) {
     if (Sec.Flags == XCOFF::STYP_OVRFLO &&
@@ -758,7 +801,7 @@ bool XCOFFSymbolRef::isFunction() const {
   if (OwningObjectPtr->is64Bit())
     report_fatal_error("64-bit support is unimplemented yet.");
 
-  if (getType() & FUNCTION_SYM)
+  if (getType() & FunctionSym)
     return true;
 
   if (!hasCsectAuxEnt())
@@ -767,7 +810,7 @@ bool XCOFFSymbolRef::isFunction() const {
   const XCOFFCsectAuxEnt32 *CsectAuxEnt = getXCOFFCsectAuxEnt32();
 
   // A function definition should be a label definition.
-  if ((CsectAuxEnt->SymbolAlignmentAndType & SYM_TYPE_MASK) != XCOFF::XTY_LD)
+  if ((CsectAuxEnt->SymbolAlignmentAndType & SymTypeMask) != XCOFF::XTY_LD)
     return false;
 
   if (CsectAuxEnt->StorageMappingClass != XCOFF::XMC_PR)
diff --git a/llvm/lib/ObjectYAML/ELFEmitter.cpp b/llvm/lib/ObjectYAML/ELFEmitter.cpp
index b48c377debda3..71e21b90344db 100644
--- a/llvm/lib/ObjectYAML/ELFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/ELFEmitter.cpp
@@ -1065,10 +1065,13 @@ void ELFState<ELFT>::writeSectionContent(Elf_Shdr &SHeader,
     return;
   }
 
-  support::endian::write<uint32_t>(OS, Section.Bucket->size(),
-                                   ELFT::TargetEndianness);
-  support::endian::write<uint32_t>(OS, Section.Chain->size(),
-                                   ELFT::TargetEndianness);
+  support::endian::write<uint32_t>(
+      OS, Section.NBucket.getValueOr(llvm::yaml::Hex64(Section.Bucket->size())),
+      ELFT::TargetEndianness);
+  support::endian::write<uint32_t>(
+      OS, Section.NChain.getValueOr(llvm::yaml::Hex64(Section.Chain->size())),
+      ELFT::TargetEndianness);
+
   for (uint32_t Val : *Section.Bucket)
     support::endian::write<uint32_t>(OS, Val, ELFT::TargetEndianness);
   for (uint32_t Val : *Section.Chain)
diff --git a/llvm/lib/ObjectYAML/ELFYAML.cpp b/llvm/lib/ObjectYAML/ELFYAML.cpp
index e9977b4899eeb..5adcb25dcec42 100644
--- a/llvm/lib/ObjectYAML/ELFYAML.cpp
+++ b/llvm/lib/ObjectYAML/ELFYAML.cpp
@@ -500,6 +500,9 @@ void ScalarEnumerationTraits<ELFYAML::ELF_SHT>::enumeration(
     ECase(SHT_MIPS_DWARF);
     ECase(SHT_MIPS_ABIFLAGS);
     break;
+  case ELF::EM_RISCV:
+    ECase(SHT_RISCV_ATTRIBUTES);
+    break;
   default:
     // Nothing to do.
     break;
@@ -1088,6 +1091,13 @@ static void sectionMapping(IO &IO, ELFYAML::HashSection &Section) {
   IO.mapOptional("Bucket", Section.Bucket);
   IO.mapOptional("Chain", Section.Chain);
   IO.mapOptional("Size", Section.Size);
+
+  // obj2yaml does not dump these fields. They can be used to override nchain
+  // and nbucket values for creating broken sections.
+  assert(!IO.outputting() ||
+         (!Section.NBucket.hasValue() && !Section.NChain.hasValue()));
+  IO.mapOptional("NChain", Section.NChain);
+  IO.mapOptional("NBucket", Section.NBucket);
 }
 
 static void sectionMapping(IO &IO, ELFYAML::NoteSection &Section) {
diff --git a/llvm/lib/ObjectYAML/WasmEmitter.cpp b/llvm/lib/ObjectYAML/WasmEmitter.cpp
index debc040587a87..ef54eaedfc0b8 100644
--- a/llvm/lib/ObjectYAML/WasmEmitter.cpp
+++ b/llvm/lib/ObjectYAML/WasmEmitter.cpp
@@ -41,8 +41,8 @@ class WasmWriter {
   void writeSectionContent(raw_ostream &OS, WasmYAML::FunctionSection &Section);
   void writeSectionContent(raw_ostream &OS, WasmYAML::TableSection &Section);
   void writeSectionContent(raw_ostream &OS, WasmYAML::MemorySection &Section);
-  void writeSectionContent(raw_ostream &OS, WasmYAML::GlobalSection &Section);
   void writeSectionContent(raw_ostream &OS, WasmYAML::EventSection &Section);
+  void writeSectionContent(raw_ostream &OS, WasmYAML::GlobalSection &Section);
   void writeSectionContent(raw_ostream &OS, WasmYAML::ExportSection &Section);
   void writeSectionContent(raw_ostream &OS, WasmYAML::StartSection &Section);
   void writeSectionContent(raw_ostream &OS, WasmYAML::ElemSection &Section);
@@ -414,6 +414,21 @@ void WasmWriter::writeSectionContent(raw_ostream &OS,
     writeLimits(Mem, OS);
 }
 
+void WasmWriter::writeSectionContent(raw_ostream &OS,
+                                     WasmYAML::EventSection &Section) {
+  encodeULEB128(Section.Events.size(), OS);
+  uint32_t ExpectedIndex = NumImportedEvents;
+  for (auto &Event : Section.Events) {
+    if (Event.Index != ExpectedIndex) {
+      reportError("unexpected event index: " + Twine(Event.Index));
+      return;
+    }
+    ++ExpectedIndex;
+    encodeULEB128(Event.Attribute, OS);
+    encodeULEB128(Event.SigIndex, OS);
+  }
+}
+
 void WasmWriter::writeSectionContent(raw_ostream &OS,
                                      WasmYAML::GlobalSection &Section) {
   encodeULEB128(Section.Globals.size(), OS);
@@ -430,21 +445,6 @@ void WasmWriter::writeSectionContent(raw_ostream &OS,
   }
 }
 
-void WasmWriter::writeSectionContent(raw_ostream &OS,
-                                    WasmYAML::EventSection &Section) {
-  encodeULEB128(Section.Events.size(), OS);
-  uint32_t ExpectedIndex = NumImportedEvents;
-  for (auto &Event : Section.Events) {
-    if (Event.Index != ExpectedIndex) {
-      reportError("unexpected event index: " + Twine(Event.Index));
-      return;
-    }
-    ++ExpectedIndex;
-    encodeULEB128(Event.Attribute, OS);
-    encodeULEB128(Event.SigIndex, OS);
-  }
-}
-
 void WasmWriter::writeSectionContent(raw_ostream &OS,
                                      WasmYAML::ElemSection &Section) {
   encodeULEB128(Section.Segments.size(), OS);
@@ -571,10 +571,10 @@ bool WasmWriter::writeWasm(raw_ostream &OS) {
       writeSectionContent(StringStream, *S);
     else if (auto S = dyn_cast<WasmYAML::MemorySection>(Sec.get()))
       writeSectionContent(StringStream, *S);
-    else if (auto S = dyn_cast<WasmYAML::GlobalSection>(Sec.get()))
-      writeSectionContent(StringStream, *S);
     else if (auto S = dyn_cast<WasmYAML::EventSection>(Sec.get()))
       writeSectionContent(StringStream, *S);
+    else if (auto S = dyn_cast<WasmYAML::GlobalSection>(Sec.get()))
+      writeSectionContent(StringStream, *S);
     else if (auto S = dyn_cast<WasmYAML::ExportSection>(Sec.get()))
       writeSectionContent(StringStream, *S);
     else if (auto S = dyn_cast<WasmYAML::StartSection>(Sec.get()))
diff --git a/llvm/lib/ObjectYAML/WasmYAML.cpp b/llvm/lib/ObjectYAML/WasmYAML.cpp
index 232d5122004a7..3e9ba9122428f 100644
--- a/llvm/lib/ObjectYAML/WasmYAML.cpp
+++ b/llvm/lib/ObjectYAML/WasmYAML.cpp
@@ -118,14 +118,14 @@ static void sectionMapping(IO &IO, WasmYAML::MemorySection &Section) {
   IO.mapOptional("Memories", Section.Memories);
 }
 
-static void sectionMapping(IO &IO, WasmYAML::GlobalSection &Section) {
+static void sectionMapping(IO &IO, WasmYAML::EventSection &Section) {
   commonSectionMapping(IO, Section);
-  IO.mapOptional("Globals", Section.Globals);
+  IO.mapOptional("Events", Section.Events);
 }
 
-static void sectionMapping(IO &IO, WasmYAML::EventSection &Section) {
+static void sectionMapping(IO &IO, WasmYAML::GlobalSection &Section) {
   commonSectionMapping(IO, Section);
-  IO.mapOptional("Events", Section.Events);
+  IO.mapOptional("Globals", Section.Globals);
 }
 
 static void sectionMapping(IO &IO, WasmYAML::ExportSection &Section) {
@@ -227,16 +227,16 @@ void MappingTraits<std::unique_ptr<WasmYAML::Section>>::mapping(
       Section.reset(new WasmYAML::MemorySection());
     sectionMapping(IO, *cast<WasmYAML::MemorySection>(Section.get()));
     break;
-  case wasm::WASM_SEC_GLOBAL:
-    if (!IO.outputting())
-      Section.reset(new WasmYAML::GlobalSection());
-    sectionMapping(IO, *cast<WasmYAML::GlobalSection>(Section.get()));
-    break;
   case wasm::WASM_SEC_EVENT:
     if (!IO.outputting())
       Section.reset(new WasmYAML::EventSection());
     sectionMapping(IO, *cast<WasmYAML::EventSection>(Section.get()));
     break;
+  case wasm::WASM_SEC_GLOBAL:
+    if (!IO.outputting())
+      Section.reset(new WasmYAML::GlobalSection());
+    sectionMapping(IO, *cast<WasmYAML::GlobalSection>(Section.get()));
+    break;
   case wasm::WASM_SEC_EXPORT:
     if (!IO.outputting())
       Section.reset(new WasmYAML::ExportSection());
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 32d0a380ae3ab..c56c35866ddaf 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -59,7 +59,6 @@
 #include "llvm/CodeGen/UnreachableBlockElim.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/IRPrintingPasses.h"
-#include "llvm/IR/KnowledgeRetention.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/SafepointIRVerifier.h"
 #include "llvm/IR/Verifier.h"
@@ -176,6 +175,7 @@
 #include "llvm/Transforms/Scalar/TailRecursionElimination.h"
 #include "llvm/Transforms/Scalar/WarnMissedTransforms.h"
 #include "llvm/Transforms/Utils/AddDiscriminators.h"
+#include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
 #include "llvm/Transforms/Utils/BreakCriticalEdges.h"
 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
 #include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
@@ -237,6 +237,10 @@ static cl::opt<bool>
     EnableCHR("enable-chr-npm", cl::init(true), cl::Hidden,
               cl::desc("Enable control height reduction optimization (CHR)"));
 
+static cl::opt<bool> EnableCallGraphProfile(
+    "enable-npm-call-graph-profile", cl::init(true), cl::Hidden,
+    cl::desc("Enable call graph profile pass for the new PM (default = on)"));
+
 PipelineTuningOptions::PipelineTuningOptions() {
   LoopInterleaving = EnableLoopInterleaving;
   LoopVectorization = EnableLoopVectorization;
@@ -246,6 +250,7 @@ PipelineTuningOptions::PipelineTuningOptions() {
   Coroutines = false;
   LicmMssaOptCap = SetLicmMssaOptCap;
   LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap;
+  CallGraphProfile = EnableCallGraphProfile;
 }
 
 extern cl::opt<bool> EnableHotColdSplit;
@@ -1060,7 +1065,8 @@ ModulePassManager PassBuilder::buildModuleOptimizationPipeline(
   // Add the core optimizing pipeline.
   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM)));
 
-  MPM.addPass(CGProfilePass());
+  if (PTO.CallGraphProfile)
+    MPM.addPass(CGProfilePass());
 
   // Now we need to do some global optimization transforms.
   // FIXME: It would seem like these should come first in the optimization
diff --git a/llvm/lib/ProfileData/SampleProf.cpp b/llvm/lib/ProfileData/SampleProf.cpp
index 003e8d4d42969..e5d0fdba5fc4a 100644
--- a/llvm/lib/ProfileData/SampleProf.cpp
+++ b/llvm/lib/ProfileData/SampleProf.cpp
@@ -30,6 +30,7 @@ using namespace sampleprof;
 namespace llvm {
 namespace sampleprof {
 SampleProfileFormat FunctionSamples::Format;
+bool FunctionSamples::UseMD5;
 } // namespace sampleprof
 } // namespace llvm
 
diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp
index 6f74cec07b96a..33efb51dca07d 100644
--- a/llvm/lib/ProfileData/SampleProfReader.cpp
+++ b/llvm/lib/ProfileData/SampleProfReader.cpp
@@ -470,18 +470,18 @@ std::error_code SampleProfileReaderBinary::readImpl() {
   return sampleprof_error::success;
 }
 
-std::error_code
-SampleProfileReaderExtBinary::readOneSection(const uint8_t *Start,
-                                             uint64_t Size, SecType Type) {
+std::error_code SampleProfileReaderExtBinary::readOneSection(
+    const uint8_t *Start, uint64_t Size, const SecHdrTableEntry &Entry) {
   Data = Start;
   End = Start + Size;
-  switch (Type) {
+  switch (Entry.Type) {
   case SecProfSummary:
     if (std::error_code EC = readSummary())
       return EC;
     break;
   case SecNameTable:
-    if (std::error_code EC = readNameTable())
+    if (std::error_code EC = readNameTableSec(
+            hasSecFlag(Entry, SecNameTableFlags::SecFlagMD5Name)))
       return EC;
     break;
   case SecLBRProfile:
@@ -546,15 +546,28 @@ std::error_code SampleProfileReaderExtBinary::readFuncProfiles() {
     }
   }
 
-  for (auto NameOffset : FuncOffsetTable) {
-    auto FuncName = NameOffset.first;
-    if (!FuncsToUse.count(FuncName) &&
-        (!Remapper || !Remapper->exist(FuncName)))
-      continue;
-    const uint8_t *FuncProfileAddr = Start + NameOffset.second;
-    assert(FuncProfileAddr < End && "out of LBRProfile section");
-    if (std::error_code EC = readFuncProfile(FuncProfileAddr))
-      return EC;
+  if (useMD5()) {
+    for (auto Name : FuncsToUse) {
+      auto GUID = std::to_string(MD5Hash(Name));
+      auto iter = FuncOffsetTable.find(StringRef(GUID));
+      if (iter == FuncOffsetTable.end())
+        continue;
+      const uint8_t *FuncProfileAddr = Start + iter->second;
+      assert(FuncProfileAddr < End && "out of LBRProfile section");
+      if (std::error_code EC = readFuncProfile(FuncProfileAddr))
+        return EC;
+    }
+  } else {
+    for (auto NameOffset : FuncOffsetTable) {
+      auto FuncName = NameOffset.first;
+      if (!FuncsToUse.count(FuncName) &&
+          (!Remapper || !Remapper->exist(FuncName)))
+        continue;
+      const uint8_t *FuncProfileAddr = Start + NameOffset.second;
+      assert(FuncProfileAddr < End && "out of LBRProfile section");
+      if (std::error_code EC = readFuncProfile(FuncProfileAddr))
+        return EC;
+    }
   }
 
   Data = End;
@@ -617,7 +630,7 @@ std::error_code SampleProfileReaderExtBinaryBase::readImpl() {
     // DecompressBuf before reading the actual data. The pointee of
     // 'Data' will be changed to buffer hold by DecompressBuf
     // temporarily when reading the actual data.
-    bool isCompressed = hasSecFlag(Entry, SecFlagCompress);
+    bool isCompressed = hasSecFlag(Entry, SecCommonFlags::SecFlagCompress);
     if (isCompressed) {
       const uint8_t *DecompressBuf;
       uint64_t DecompressBufSize;
@@ -628,7 +641,7 @@ std::error_code SampleProfileReaderExtBinaryBase::readImpl() {
       SecSize = DecompressBufSize;
     }
 
-    if (std::error_code EC = readOneSection(SecStart, SecSize, Entry.Type))
+    if (std::error_code EC = readOneSection(SecStart, SecSize, Entry))
       return EC;
     if (Data != SecStart + SecSize)
       return sampleprof_error::malformed;
@@ -705,6 +718,31 @@ std::error_code SampleProfileReaderBinary::readNameTable() {
   return sampleprof_error::success;
 }
 
+std::error_code SampleProfileReaderExtBinary::readMD5NameTable() {
+  auto Size = readNumber<uint64_t>();
+  if (std::error_code EC = Size.getError())
+    return EC;
+  NameTable.reserve(*Size);
+  MD5StringBuf = std::make_unique<std::vector<std::string>>();
+  MD5StringBuf->reserve(*Size);
+  for (uint32_t I = 0; I < *Size; ++I) {
+    auto FID = readNumber<uint64_t>();
+    if (std::error_code EC = FID.getError())
+      return EC;
+    MD5StringBuf->push_back(std::to_string(*FID));
+    // NameTable is a vector of StringRef. Here it is pushing back a
+    // StringRef initialized with the last string in MD5stringBuf.
+    NameTable.push_back(MD5StringBuf->back());
+  }
+  return sampleprof_error::success;
+}
+
+std::error_code SampleProfileReaderExtBinary::readNameTableSec(bool IsMD5) {
+  if (IsMD5)
+    return readMD5NameTable();
+  return SampleProfileReaderBinary::readNameTable();
+}
+
 std::error_code SampleProfileReaderCompactBinary::readNameTable() {
   auto Size = readNumber<uint64_t>();
   if (std::error_code EC = Size.getError())
@@ -1210,9 +1248,9 @@ bool SampleProfileReaderGCC::hasFormat(const MemoryBuffer &Buffer) {
 }
 
 void SampleProfileReaderItaniumRemapper::applyRemapping(LLVMContext &Ctx) {
-  // If the reader is in compact format, we can't remap it because
+  // If the reader uses MD5 to represent string, we can't remap it because
   // we don't know what the original function names were.
-  if (Reader.getFormat() == SPF_Compact_Binary) {
+  if (Reader.useMD5()) {
     Ctx.diagnose(DiagnosticInfoSampleProfile(
         Reader.getBuffer()->getBufferIdentifier(),
         "Profile data remapping cannot be applied to profile data "
diff --git a/llvm/lib/ProfileData/SampleProfWriter.cpp b/llvm/lib/ProfileData/SampleProfWriter.cpp
index 8d09af31f94bf..48d3faa6cd2f6 100644
--- a/llvm/lib/ProfileData/SampleProfWriter.cpp
+++ b/llvm/lib/ProfileData/SampleProfWriter.cpp
@@ -87,7 +87,7 @@ uint64_t SampleProfileWriterExtBinaryBase::markSectionStart(SecType Type) {
   uint64_t SectionStart = OutputStream->tell();
   auto &Entry = getEntryInLayout(Type);
   // Use LocalBuf as a temporary output for writting data.
-  if (hasSecFlag(Entry, SecFlagCompress))
+  if (hasSecFlag(Entry, SecCommonFlags::SecFlagCompress))
     LocalBufStream.swap(OutputStream);
   return SectionStart;
 }
@@ -117,7 +117,7 @@ std::error_code
 SampleProfileWriterExtBinaryBase::addNewSection(SecType Type,
                                                 uint64_t SectionStart) {
   auto Entry = getEntryInLayout(Type);
-  if (hasSecFlag(Entry, SecFlagCompress)) {
+  if (hasSecFlag(Entry, SecCommonFlags::SecFlagCompress)) {
     LocalBufStream.swap(OutputStream);
     if (std::error_code EC = compressAndOutput())
       return EC;
@@ -166,6 +166,22 @@ std::error_code SampleProfileWriterExtBinary::writeFuncOffsetTable() {
   return sampleprof_error::success;
 }
 
+std::error_code SampleProfileWriterExtBinary::writeNameTable() {
+  if (!UseMD5)
+    return SampleProfileWriterBinary::writeNameTable();
+
+  auto &OS = *OutputStream;
+  std::set<StringRef> V;
+  stablizeNameTable(V);
+
+  // Write out the name table.
+  encodeULEB128(NameTable.size(), OS);
+  for (auto N : V) {
+    encodeULEB128(MD5Hash(N), OS);
+  }
+  return sampleprof_error::success;
+}
+
 std::error_code SampleProfileWriterExtBinary::writeSections(
     const StringMap<FunctionSamples> &ProfileMap) {
   uint64_t SectionStart = markSectionStart(SecProfSummary);
@@ -390,19 +406,11 @@ std::error_code SampleProfileWriterBinary::writeHeader(
 
 void SampleProfileWriterExtBinaryBase::setToCompressAllSections() {
   for (auto &Entry : SectionHdrLayout)
-    addSecFlags(Entry, SecFlagCompress);
+    addSecFlag(Entry, SecCommonFlags::SecFlagCompress);
 }
 
 void SampleProfileWriterExtBinaryBase::setToCompressSection(SecType Type) {
-  addSectionFlags(Type, SecFlagCompress);
-}
-
-void SampleProfileWriterExtBinaryBase::addSectionFlags(SecType Type,
-                                                       SecFlags Flags) {
-  for (auto &Entry : SectionHdrLayout) {
-    if (Entry.Type == Type)
-      addSecFlags(Entry, Flags);
-  }
+  addSectionFlag(Type, SecCommonFlags::SecFlagCompress);
 }
 
 void SampleProfileWriterExtBinaryBase::allocSecHdrTable() {
diff --git a/llvm/lib/Support/AArch64TargetParser.cpp b/llvm/lib/Support/AArch64TargetParser.cpp
index 32b9402fb1586..cce2d82082962 100644
--- a/llvm/lib/Support/AArch64TargetParser.cpp
+++ b/llvm/lib/Support/AArch64TargetParser.cpp
@@ -116,6 +116,8 @@ bool AArch64::getArchFeatures(AArch64::ArchKind AK,
     Features.push_back("+v8.4a");
   if (AK == ArchKind::ARMV8_5A)
     Features.push_back("+v8.5a");
+  if (AK == AArch64::ArchKind::ARMV8_6A)
+    Features.push_back("+v8.6a");
 
   return AK != ArchKind::INVALID;
 }
diff --git a/llvm/lib/Support/ARMAttributeParser.cpp b/llvm/lib/Support/ARMAttributeParser.cpp
index c7b8abe7ad2d0..17ad38d226143 100644
--- a/llvm/lib/Support/ARMAttributeParser.cpp
+++ b/llvm/lib/Support/ARMAttributeParser.cpp
@@ -1,4 +1,4 @@
-//===--- ARMAttributeParser.cpp - ARM Attribute Information Printer -------===//
+//===- ARMAttributeParser.cpp - ARM Attribute Information Printer ---------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -16,12 +16,6 @@
 using namespace llvm;
 using namespace llvm::ARMBuildAttrs;
 
-static const EnumEntry<unsigned> tagNames[] = {
-    {"Tag_File", ARMBuildAttrs::File},
-    {"Tag_Section", ARMBuildAttrs::Section},
-    {"Tag_Symbol", ARMBuildAttrs::Symbol},
-};
-
 #define ATTRIBUTE_HANDLER(attr)                                                \
   { ARMBuildAttrs::attr, &ARMAttributeParser::attr }
 
@@ -71,7 +65,8 @@ const ARMAttributeParser::DisplayHandler ARMAttributeParser::displayRoutines[] =
 #undef ATTRIBUTE_HANDLER
 
 Error ARMAttributeParser::stringAttribute(AttrType tag) {
-  StringRef tagName = ARMBuildAttrs::AttrTypeAsString(tag, /*TagPrefix=*/false);
+  StringRef tagName =
+      ELFAttrs::attrTypeAsString(tag, tagToStringMap, /*TagPrefix=*/false);
   StringRef desc = de.getCStrRef(cursor);
 
   if (sw) {
@@ -84,36 +79,6 @@ Error ARMAttributeParser::stringAttribute(AttrType tag) {
   return Error::success();
 }
 
-void ARMAttributeParser::printAttribute(unsigned tag, unsigned value,
-                                        StringRef valueDesc) {
-  attributes.insert(std::make_pair(tag, value));
-
-  if (sw) {
-    StringRef tagName =
-        ARMBuildAttrs::AttrTypeAsString(tag, /*TagPrefix=*/false);
-    DictScope as(*sw, "Attribute");
-    sw->printNumber("Tag", tag);
-    sw->printNumber("Value", value);
-    if (!tagName.empty())
-      sw->printString("TagName", tagName);
-    if (!valueDesc.empty())
-      sw->printString("Description", valueDesc);
-  }
-}
-
-Error ARMAttributeParser::parseStringAttribute(const char *name, AttrType tag,
-                                               ArrayRef<const char *> strings) {
-  uint64_t value = de.getULEB128(cursor);
-  if (value >= strings.size()) {
-    printAttribute(tag, value, "");
-    return createStringError(errc::invalid_argument,
-                             "unknown " + Twine(name) +
-                                 " value: " + Twine(value));
-  }
-  printAttribute(tag, value, strings[value]);
-  return Error::success();
-}
-
 Error ARMAttributeParser::CPU_arch(AttrType tag) {
   static const char *strings[] = {
     "Pre-v4", "ARM v4", "ARM v4T", "ARM v5T", "ARM v5TE", "ARM v5TEJ", "ARM v6",
@@ -323,7 +288,9 @@ Error ARMAttributeParser::compatibility(AttrType tag) {
     DictScope scope(*sw, "Attribute");
     sw->printNumber("Tag", tag);
     sw->startLine() << "Value: " << integer << ", " << string << '\n';
-    sw->printString("TagName", AttrTypeAsString(tag, /*TagPrefix*/ false));
+    sw->printString("TagName",
+                    ELFAttrs::attrTypeAsString(tag, tagToStringMap,
+                                               /*hasTagPrefix=*/false));
     switch (integer) {
     case 0:
       sw->printString("Description", StringRef("No Specific Requirements"));
@@ -389,167 +356,18 @@ Error ARMAttributeParser::nodefaults(AttrType tag) {
   return Error::success();
 }
 
-void ARMAttributeParser::parseIndexList(SmallVectorImpl<uint8_t> &indexList) {
-  for (;;) {
-    uint64_t value = de.getULEB128(cursor);
-    if (!cursor || !value)
+Error ARMAttributeParser::handler(uint64_t tag, bool &handled) {
+  handled = false;
+  for (unsigned AHI = 0, AHE = array_lengthof(displayRoutines); AHI != AHE;
+       ++AHI) {
+    if (uint64_t(displayRoutines[AHI].attribute) == tag) {
+      if (Error e =
+              (this->*displayRoutines[AHI].routine)(static_cast<AttrType>(tag)))
+        return e;
+      handled = true;
       break;
-    indexList.push_back(value);
-  }
-}
-
-Error ARMAttributeParser::parseAttributeList(uint32_t length) {
-  uint64_t pos;
-  uint64_t end = cursor.tell() + length;
-  while ((pos = cursor.tell()) < end) {
-    uint64_t tag = de.getULEB128(cursor);
-    bool handled = false;
-    for (unsigned AHI = 0, AHE = array_lengthof(displayRoutines);
-         AHI != AHE && !handled; ++AHI) {
-      if (uint64_t(displayRoutines[AHI].attribute) == tag) {
-        if (Error e = (this->*displayRoutines[AHI].routine)(
-                ARMBuildAttrs::AttrType(tag)))
-          return e;
-        handled = true;
-        break;
-      }
-    }
-    if (!handled) {
-      if (tag < 32)
-        return createStringError(errc::invalid_argument,
-                                 "invalid AEABI tag 0x" +
-                                     Twine::utohexstr(tag) + " at offset 0x" +
-                                     Twine::utohexstr(pos));
-
-      if (tag % 2 == 0) {
-        uint64_t value = de.getULEB128(cursor);
-        attributes.insert(std::make_pair(tag, value));
-        if (sw)
-          sw->printNumber(ARMBuildAttrs::AttrTypeAsString(tag), value);
-      } else {
-        StringRef tagName =
-            ARMBuildAttrs::AttrTypeAsString(tag, /*TagPrefix=*/false);
-        StringRef desc = de.getCStrRef(cursor);
-
-        if (sw) {
-          DictScope scope(*sw, "Attribute");
-          sw->printNumber("Tag", tag);
-          if (!tagName.empty())
-            sw->printString("TagName", tagName);
-          sw->printString("Value", desc);
-        }
-      }
     }
   }
-  return Error::success();
-}
-
-Error ARMAttributeParser::parseSubsection(uint32_t length) {
-  uint64_t end = cursor.tell() - sizeof(length) + length;
-  StringRef vendorName = de.getCStrRef(cursor);
-  if (sw) {
-    sw->printNumber("SectionLength", length);
-    sw->printString("Vendor", vendorName);
-  }
-
-  // Ignore unrecognized vendor-name.
-  if (vendorName.lower() != "aeabi")
-    return createStringError(errc::invalid_argument,
-                             "unrecognized vendor-name: " + vendorName);
-
-  while (cursor.tell() < end) {
-    /// Tag_File | Tag_Section | Tag_Symbol   uleb128:byte-size
-    uint8_t tag = de.getU8(cursor);
-    uint32_t size = de.getU32(cursor);
-    if (!cursor)
-      return cursor.takeError();
-
-    if (sw) {
-      sw->printEnum("Tag", tag, makeArrayRef(tagNames));
-      sw->printNumber("Size", size);
-    }
-    if (size < 5)
-      return createStringError(errc::invalid_argument,
-                               "invalid attribute size " + Twine(size) +
-                                   " at offset 0x" +
-                                   Twine::utohexstr(cursor.tell() - 5));
-
-    StringRef scopeName, indexName;
-    SmallVector<uint8_t, 8> indicies;
-    switch (tag) {
-    case ARMBuildAttrs::File:
-      scopeName = "FileAttributes";
-      break;
-    case ARMBuildAttrs::Section:
-      scopeName = "SectionAttributes";
-      indexName = "Sections";
-      parseIndexList(indicies);
-      break;
-    case ARMBuildAttrs::Symbol:
-      scopeName = "SymbolAttributes";
-      indexName = "Symbols";
-      parseIndexList(indicies);
-      break;
-    default:
-      return createStringError(errc::invalid_argument,
-                               "unrecognized tag 0x" + Twine::utohexstr(tag) +
-                                   " at offset 0x" +
-                                   Twine::utohexstr(cursor.tell() - 5));
-    }
 
-    if (sw) {
-      DictScope scope(*sw, scopeName);
-      if (!indicies.empty())
-        sw->printList(indexName, indicies);
-      if (Error e = parseAttributeList(size - 5))
-        return e;
-    } else if (Error e = parseAttributeList(size - 5))
-      return e;
-  }
   return Error::success();
 }
-
-Error ARMAttributeParser::parse(ArrayRef<uint8_t> section,
-                                support::endianness endian) {
-  unsigned sectionNumber = 0;
-  de = DataExtractor(section, endian == support::little, 0);
-
-  // For early returns, we have more specific errors, consume the Error in
-  // cursor.
-  struct ClearCursorError {
-    DataExtractor::Cursor &cursor;
-    ~ClearCursorError() { consumeError(cursor.takeError()); }
-  } clear{cursor};
-
-  // Unrecognized format-version.
-  uint8_t formatVersion = de.getU8(cursor);
-  if (formatVersion != 'A')
-    return createStringError(errc::invalid_argument,
-                             "unrecognized format-version: 0x" +
-                                 utohexstr(formatVersion));
-
-  while (!de.eof(cursor)) {
-    uint32_t sectionLength = de.getU32(cursor);
-    if (!cursor)
-      return cursor.takeError();
-
-    if (sw) {
-      sw->startLine() << "Section " << ++sectionNumber << " {\n";
-      sw->indent();
-    }
-
-    if (sectionLength < 4 || cursor.tell() - 4 + sectionLength > section.size())
-      return createStringError(errc::invalid_argument,
-                               "invalid subsection length " +
-                                   Twine(sectionLength) + " at offset 0x" +
-                                   utohexstr(cursor.tell() - 4));
-    if (Error e = parseSubsection(sectionLength))
-      return e;
-    if (sw) {
-      sw->unindent();
-      sw->startLine() << "}\n";
-    }
-  }
-
-  return cursor.takeError();
-}
diff --git a/llvm/lib/Support/ARMBuildAttrs.cpp b/llvm/lib/Support/ARMBuildAttrs.cpp
index d0c4fb792cb8c..5aaf0a4e7c62e 100644
--- a/llvm/lib/Support/ARMBuildAttrs.cpp
+++ b/llvm/lib/Support/ARMBuildAttrs.cpp
@@ -6,97 +6,63 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/ADT/StringRef.h"
 #include "llvm/Support/ARMBuildAttributes.h"
 
 using namespace llvm;
 
-namespace {
-const struct {
-  ARMBuildAttrs::AttrType Attr;
-  StringRef TagName;
-} ARMAttributeTags[] = {
-  { ARMBuildAttrs::File, "Tag_File" },
-  { ARMBuildAttrs::Section, "Tag_Section" },
-  { ARMBuildAttrs::Symbol, "Tag_Symbol" },
-  { ARMBuildAttrs::CPU_raw_name, "Tag_CPU_raw_name" },
-  { ARMBuildAttrs::CPU_name, "Tag_CPU_name" },
-  { ARMBuildAttrs::CPU_arch, "Tag_CPU_arch" },
-  { ARMBuildAttrs::CPU_arch_profile, "Tag_CPU_arch_profile" },
-  { ARMBuildAttrs::ARM_ISA_use, "Tag_ARM_ISA_use" },
-  { ARMBuildAttrs::THUMB_ISA_use, "Tag_THUMB_ISA_use" },
-  { ARMBuildAttrs::FP_arch, "Tag_FP_arch" },
-  { ARMBuildAttrs::WMMX_arch, "Tag_WMMX_arch" },
-  { ARMBuildAttrs::Advanced_SIMD_arch, "Tag_Advanced_SIMD_arch" },
-  { ARMBuildAttrs::MVE_arch, "Tag_MVE_arch" },
-  { ARMBuildAttrs::PCS_config, "Tag_PCS_config" },
-  { ARMBuildAttrs::ABI_PCS_R9_use, "Tag_ABI_PCS_R9_use" },
-  { ARMBuildAttrs::ABI_PCS_RW_data, "Tag_ABI_PCS_RW_data" },
-  { ARMBuildAttrs::ABI_PCS_RO_data, "Tag_ABI_PCS_RO_data" },
-  { ARMBuildAttrs::ABI_PCS_GOT_use, "Tag_ABI_PCS_GOT_use" },
-  { ARMBuildAttrs::ABI_PCS_wchar_t, "Tag_ABI_PCS_wchar_t" },
-  { ARMBuildAttrs::ABI_FP_rounding, "Tag_ABI_FP_rounding" },
-  { ARMBuildAttrs::ABI_FP_denormal, "Tag_ABI_FP_denormal" },
-  { ARMBuildAttrs::ABI_FP_exceptions, "Tag_ABI_FP_exceptions" },
-  { ARMBuildAttrs::ABI_FP_user_exceptions, "Tag_ABI_FP_user_exceptions" },
-  { ARMBuildAttrs::ABI_FP_number_model, "Tag_ABI_FP_number_model" },
-  { ARMBuildAttrs::ABI_align_needed, "Tag_ABI_align_needed" },
-  { ARMBuildAttrs::ABI_align_preserved, "Tag_ABI_align_preserved" },
-  { ARMBuildAttrs::ABI_enum_size, "Tag_ABI_enum_size" },
-  { ARMBuildAttrs::ABI_HardFP_use, "Tag_ABI_HardFP_use" },
-  { ARMBuildAttrs::ABI_VFP_args, "Tag_ABI_VFP_args" },
-  { ARMBuildAttrs::ABI_WMMX_args, "Tag_ABI_WMMX_args" },
-  { ARMBuildAttrs::ABI_optimization_goals, "Tag_ABI_optimization_goals" },
-  { ARMBuildAttrs::ABI_FP_optimization_goals, "Tag_ABI_FP_optimization_goals" },
-  { ARMBuildAttrs::compatibility, "Tag_compatibility" },
-  { ARMBuildAttrs::CPU_unaligned_access, "Tag_CPU_unaligned_access" },
-  { ARMBuildAttrs::FP_HP_extension, "Tag_FP_HP_extension" },
-  { ARMBuildAttrs::ABI_FP_16bit_format, "Tag_ABI_FP_16bit_format" },
-  { ARMBuildAttrs::MPextension_use, "Tag_MPextension_use" },
-  { ARMBuildAttrs::DIV_use, "Tag_DIV_use" },
-  { ARMBuildAttrs::DSP_extension, "Tag_DSP_extension" },
-  { ARMBuildAttrs::nodefaults, "Tag_nodefaults" },
-  { ARMBuildAttrs::also_compatible_with, "Tag_also_compatible_with" },
-  { ARMBuildAttrs::T2EE_use, "Tag_T2EE_use" },
-  { ARMBuildAttrs::conformance, "Tag_conformance" },
-  { ARMBuildAttrs::Virtualization_use, "Tag_Virtualization_use" },
+static const TagNameItem tagData[] = {
+    {ARMBuildAttrs::File, "Tag_File"},
+    {ARMBuildAttrs::Section, "Tag_Section"},
+    {ARMBuildAttrs::Symbol, "Tag_Symbol"},
+    {ARMBuildAttrs::CPU_raw_name, "Tag_CPU_raw_name"},
+    {ARMBuildAttrs::CPU_name, "Tag_CPU_name"},
+    {ARMBuildAttrs::CPU_arch, "Tag_CPU_arch"},
+    {ARMBuildAttrs::CPU_arch_profile, "Tag_CPU_arch_profile"},
+    {ARMBuildAttrs::ARM_ISA_use, "Tag_ARM_ISA_use"},
+    {ARMBuildAttrs::THUMB_ISA_use, "Tag_THUMB_ISA_use"},
+    {ARMBuildAttrs::FP_arch, "Tag_FP_arch"},
+    {ARMBuildAttrs::WMMX_arch, "Tag_WMMX_arch"},
+    {ARMBuildAttrs::Advanced_SIMD_arch, "Tag_Advanced_SIMD_arch"},
+    {ARMBuildAttrs::MVE_arch, "Tag_MVE_arch"},
+    {ARMBuildAttrs::PCS_config, "Tag_PCS_config"},
+    {ARMBuildAttrs::ABI_PCS_R9_use, "Tag_ABI_PCS_R9_use"},
+    {ARMBuildAttrs::ABI_PCS_RW_data, "Tag_ABI_PCS_RW_data"},
+    {ARMBuildAttrs::ABI_PCS_RO_data, "Tag_ABI_PCS_RO_data"},
+    {ARMBuildAttrs::ABI_PCS_GOT_use, "Tag_ABI_PCS_GOT_use"},
+    {ARMBuildAttrs::ABI_PCS_wchar_t, "Tag_ABI_PCS_wchar_t"},
+    {ARMBuildAttrs::ABI_FP_rounding, "Tag_ABI_FP_rounding"},
+    {ARMBuildAttrs::ABI_FP_denormal, "Tag_ABI_FP_denormal"},
+    {ARMBuildAttrs::ABI_FP_exceptions, "Tag_ABI_FP_exceptions"},
+    {ARMBuildAttrs::ABI_FP_user_exceptions, "Tag_ABI_FP_user_exceptions"},
+    {ARMBuildAttrs::ABI_FP_number_model, "Tag_ABI_FP_number_model"},
+    {ARMBuildAttrs::ABI_align_needed, "Tag_ABI_align_needed"},
+    {ARMBuildAttrs::ABI_align_preserved, "Tag_ABI_align_preserved"},
+    {ARMBuildAttrs::ABI_enum_size, "Tag_ABI_enum_size"},
+    {ARMBuildAttrs::ABI_HardFP_use, "Tag_ABI_HardFP_use"},
+    {ARMBuildAttrs::ABI_VFP_args, "Tag_ABI_VFP_args"},
+    {ARMBuildAttrs::ABI_WMMX_args, "Tag_ABI_WMMX_args"},
+    {ARMBuildAttrs::ABI_optimization_goals, "Tag_ABI_optimization_goals"},
+    {ARMBuildAttrs::ABI_FP_optimization_goals, "Tag_ABI_FP_optimization_goals"},
+    {ARMBuildAttrs::compatibility, "Tag_compatibility"},
+    {ARMBuildAttrs::CPU_unaligned_access, "Tag_CPU_unaligned_access"},
+    {ARMBuildAttrs::FP_HP_extension, "Tag_FP_HP_extension"},
+    {ARMBuildAttrs::ABI_FP_16bit_format, "Tag_ABI_FP_16bit_format"},
+    {ARMBuildAttrs::MPextension_use, "Tag_MPextension_use"},
+    {ARMBuildAttrs::DIV_use, "Tag_DIV_use"},
+    {ARMBuildAttrs::DSP_extension, "Tag_DSP_extension"},
+    {ARMBuildAttrs::nodefaults, "Tag_nodefaults"},
+    {ARMBuildAttrs::also_compatible_with, "Tag_also_compatible_with"},
+    {ARMBuildAttrs::T2EE_use, "Tag_T2EE_use"},
+    {ARMBuildAttrs::conformance, "Tag_conformance"},
+    {ARMBuildAttrs::Virtualization_use, "Tag_Virtualization_use"},
 
-  // Legacy Names
-  { ARMBuildAttrs::FP_arch, "Tag_VFP_arch" },
-  { ARMBuildAttrs::FP_HP_extension, "Tag_VFP_HP_extension" },
-  { ARMBuildAttrs::ABI_align_needed, "Tag_ABI_align8_needed" },
-  { ARMBuildAttrs::ABI_align_preserved, "Tag_ABI_align8_preserved" },
+    // Legacy Names
+    {ARMBuildAttrs::FP_arch, "Tag_VFP_arch"},
+    {ARMBuildAttrs::FP_HP_extension, "Tag_VFP_HP_extension"},
+    {ARMBuildAttrs::ABI_align_needed, "Tag_ABI_align8_needed"},
+    {ARMBuildAttrs::ABI_align_preserved, "Tag_ABI_align8_preserved"},
 };
-}
-
-namespace llvm {
-namespace ARMBuildAttrs {
-StringRef AttrTypeAsString(unsigned Attr, bool HasTagPrefix) {
-  return AttrTypeAsString(static_cast<AttrType>(Attr), HasTagPrefix);
-}
-
-StringRef AttrTypeAsString(AttrType Attr, bool HasTagPrefix) {
-  for (unsigned TI = 0, TE = sizeof(ARMAttributeTags) / sizeof(*ARMAttributeTags);
-       TI != TE; ++TI)
-    if (ARMAttributeTags[TI].Attr == Attr) {
-      auto TagName = ARMAttributeTags[TI].TagName;
-      return HasTagPrefix ? TagName : TagName.drop_front(4);
-    }
-  return "";
-}
-
-int AttrTypeFromString(StringRef Tag) {
-  bool HasTagPrefix = Tag.startswith("Tag_");
-  for (unsigned TI = 0,
-                TE = sizeof(ARMAttributeTags) / sizeof(*ARMAttributeTags);
-       TI != TE; ++TI) {
-    auto TagName = ARMAttributeTags[TI].TagName;
-    if (TagName.drop_front(HasTagPrefix ? 0 : 4) == Tag) {
-      return ARMAttributeTags[TI].Attr;
-    }
-  }
-  return -1;
-}
-}
-}
 
+const TagNameMap llvm::ARMBuildAttrs::ARMAttributeTags(tagData,
+                                                       sizeof(tagData) /
+                                                           sizeof(TagNameItem));
diff --git a/llvm/lib/Support/ARMTargetParser.cpp b/llvm/lib/Support/ARMTargetParser.cpp
index 3c3c2103a5c13..2c581248e1838 100644
--- a/llvm/lib/Support/ARMTargetParser.cpp
+++ b/llvm/lib/Support/ARMTargetParser.cpp
@@ -74,6 +74,7 @@ unsigned ARM::parseArchVersion(StringRef Arch) {
   case ArchKind::ARMV8_3A:
   case ArchKind::ARMV8_4A:
   case ArchKind::ARMV8_5A:
+  case ArchKind::ARMV8_6A:
   case ArchKind::ARMV8R:
   case ArchKind::ARMV8MBaseline:
   case ArchKind::ARMV8MMainline:
@@ -108,6 +109,7 @@ ARM::ProfileKind ARM::parseArchProfile(StringRef Arch) {
   case ArchKind::ARMV8_3A:
   case ArchKind::ARMV8_4A:
   case ArchKind::ARMV8_5A:
+  case ArchKind::ARMV8_6A:
     return ProfileKind::A;
   case ArchKind::ARMV2:
   case ArchKind::ARMV2A:
@@ -150,6 +152,7 @@ StringRef ARM::getArchSynonym(StringRef Arch) {
       .Case("v8.3a", "v8.3-a")
       .Case("v8.4a", "v8.4-a")
       .Case("v8.5a", "v8.5-a")
+      .Case("v8.6a", "v8.6-a")
       .Case("v8r", "v8-r")
       .Case("v8m.base", "v8-m.base")
       .Case("v8m.main", "v8-m.main")
diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt
index 06ee9b1c023b9..5a0413ac487aa 100644
--- a/llvm/lib/Support/CMakeLists.txt
+++ b/llvm/lib/Support/CMakeLists.txt
@@ -87,6 +87,8 @@ add_llvm_component_library(LLVMSupport
   DeltaAlgorithm.cpp
   DAGDeltaAlgorithm.cpp
   DJB.cpp
+  ELFAttributeParser.cpp
+  ELFAttributes.cpp
   Error.cpp
   ErrorHandling.cpp
   FileCheck.cpp
@@ -123,6 +125,8 @@ add_llvm_component_library(LLVMSupport
   PropertySetIO.cpp
   RandomNumberGenerator.cpp
   Regex.cpp
+  RISCVAttributes.cpp
+  RISCVAttributeParser.cpp
   ScaledNumber.cpp
   ScopedPrinter.cpp
   SHA1.cpp
diff --git a/llvm/lib/Support/ELFAttributeParser.cpp b/llvm/lib/Support/ELFAttributeParser.cpp
new file mode 100644
index 0000000000000..df955cdf5d30a
--- /dev/null
+++ b/llvm/lib/Support/ELFAttributeParser.cpp
@@ -0,0 +1,233 @@
+//===--- ELFAttributeParser.cpp - ELF Attribute Parser --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/ELFAttributeParser.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/Errc.h"
+#include "llvm/Support/LEB128.h"
+#include "llvm/Support/ScopedPrinter.h"
+
+using namespace llvm;
+using namespace llvm::ELFAttrs;
+
+static const EnumEntry<unsigned> tagNames[] = {
+    {"Tag_File", ELFAttrs::File},
+    {"Tag_Section", ELFAttrs::Section},
+    {"Tag_Symbol", ELFAttrs::Symbol},
+};
+
+Error ELFAttributeParser::parseStringAttribute(const char *name, unsigned tag,
+                                               ArrayRef<const char *> strings) {
+  uint64_t value = de.getULEB128(cursor);
+  if (value >= strings.size()) {
+    printAttribute(tag, value, "");
+    return createStringError(errc::invalid_argument,
+                             "unknown " + Twine(name) +
+                                 " value: " + Twine(value));
+  }
+  printAttribute(tag, value, strings[value]);
+  return Error::success();
+}
+
+Error ELFAttributeParser::integerAttribute(unsigned tag) {
+  StringRef tagName =
+      ELFAttrs::attrTypeAsString(tag, tagToStringMap, /*hasTagPrefix=*/false);
+  uint64_t value = de.getULEB128(cursor);
+  attributes.insert(std::make_pair(tag, value));
+
+  if (sw) {
+    DictScope scope(*sw, "Attribute");
+    sw->printNumber("Tag", tag);
+    if (!tagName.empty())
+      sw->printString("TagName", tagName);
+    sw->printNumber("Value", value);
+  }
+  return Error::success();
+}
+
+Error ELFAttributeParser::stringAttribute(unsigned tag) {
+  StringRef tagName =
+      ELFAttrs::attrTypeAsString(tag, tagToStringMap, /*hasTagPrefix=*/false);
+  StringRef desc = de.getCStrRef(cursor);
+  attributesStr.insert(std::make_pair(tag, desc));
+
+  if (sw) {
+    DictScope scope(*sw, "Attribute");
+    sw->printNumber("Tag", tag);
+    if (!tagName.empty())
+      sw->printString("TagName", tagName);
+    sw->printString("Value", desc);
+  }
+  return Error::success();
+}
+
+void ELFAttributeParser::printAttribute(unsigned tag, unsigned value,
+                                        StringRef valueDesc) {
+  attributes.insert(std::make_pair(tag, value));
+
+  if (sw) {
+    StringRef tagName = ELFAttrs::attrTypeAsString(tag, tagToStringMap,
+                                                   /*hasTagPrefix=*/false);
+    DictScope as(*sw, "Attribute");
+    sw->printNumber("Tag", tag);
+    sw->printNumber("Value", value);
+    if (!tagName.empty())
+      sw->printString("TagName", tagName);
+    if (!valueDesc.empty())
+      sw->printString("Description", valueDesc);
+  }
+}
+
+void ELFAttributeParser::parseIndexList(SmallVectorImpl<uint8_t> &indexList) {
+  for (;;) {
+    uint64_t value = de.getULEB128(cursor);
+    if (!cursor || !value)
+      break;
+    indexList.push_back(value);
+  }
+}
+
+Error ELFAttributeParser::parseAttributeList(uint32_t length) {
+  uint64_t pos;
+  uint64_t end = cursor.tell() + length;
+  while ((pos = cursor.tell()) < end) {
+    uint64_t tag = de.getULEB128(cursor);
+    bool handled;
+    if (Error e = handler(tag, handled))
+      return e;
+
+    if (!handled) {
+      if (tag < 32) {
+        return createStringError(errc::invalid_argument,
+                                 "invalid tag 0x" + Twine::utohexstr(tag) +
+                                     " at offset 0x" + Twine::utohexstr(pos));
+      }
+
+      if (tag % 2 == 0) {
+        if (Error e = integerAttribute(tag))
+          return e;
+      } else {
+        if (Error e = stringAttribute(tag))
+          return e;
+      }
+    }
+  }
+  return Error::success();
+}
+
+Error ELFAttributeParser::parseSubsection(uint32_t length) {
+  uint64_t end = cursor.tell() - sizeof(length) + length;
+  StringRef vendorName = de.getCStrRef(cursor);
+  if (sw) {
+    sw->printNumber("SectionLength", length);
+    sw->printString("Vendor", vendorName);
+  }
+
+  // Ignore unrecognized vendor-name.
+  if (vendorName.lower() != vendor)
+    return createStringError(errc::invalid_argument,
+                             "unrecognized vendor-name: " + vendorName);
+
+  while (cursor.tell() < end) {
+    /// Tag_File | Tag_Section | Tag_Symbol   uleb128:byte-size
+    uint8_t tag = de.getU8(cursor);
+    uint32_t size = de.getU32(cursor);
+    if (!cursor)
+      return cursor.takeError();
+
+    if (sw) {
+      sw->printEnum("Tag", tag, makeArrayRef(tagNames));
+      sw->printNumber("Size", size);
+    }
+    if (size < 5)
+      return createStringError(errc::invalid_argument,
+                               "invalid attribute size " + Twine(size) +
+                                   " at offset 0x" +
+                                   Twine::utohexstr(cursor.tell() - 5));
+
+    StringRef scopeName, indexName;
+    SmallVector<uint8_t, 8> indicies;
+    switch (tag) {
+    case ELFAttrs::File:
+      scopeName = "FileAttributes";
+      break;
+    case ELFAttrs::Section:
+      scopeName = "SectionAttributes";
+      indexName = "Sections";
+      parseIndexList(indicies);
+      break;
+    case ELFAttrs::Symbol:
+      scopeName = "SymbolAttributes";
+      indexName = "Symbols";
+      parseIndexList(indicies);
+      break;
+    default:
+      return createStringError(errc::invalid_argument,
+                               "unrecognized tag 0x" + Twine::utohexstr(tag) +
+                                   " at offset 0x" +
+                                   Twine::utohexstr(cursor.tell() - 5));
+    }
+
+    if (sw) {
+      DictScope scope(*sw, scopeName);
+      if (!indicies.empty())
+        sw->printList(indexName, indicies);
+      if (Error e = parseAttributeList(size - 5))
+        return e;
+    } else if (Error e = parseAttributeList(size - 5))
+      return e;
+  }
+  return Error::success();
+}
+
+Error ELFAttributeParser::parse(ArrayRef<uint8_t> section,
+                                support::endianness endian) {
+  unsigned sectionNumber = 0;
+  de = DataExtractor(section, endian == support::little, 0);
+
+  // For early returns, we have more specific errors, consume the Error in
+  // cursor.
+  struct ClearCursorError {
+    DataExtractor::Cursor &cursor;
+    ~ClearCursorError() { consumeError(cursor.takeError()); }
+  } clear{cursor};
+
+  // Unrecognized format-version.
+  uint8_t formatVersion = de.getU8(cursor);
+  if (formatVersion != 'A')
+    return createStringError(errc::invalid_argument,
+                             "unrecognized format-version: 0x" +
+                                 utohexstr(formatVersion));
+
+  while (!de.eof(cursor)) {
+    uint32_t sectionLength = de.getU32(cursor);
+    if (!cursor)
+      return cursor.takeError();
+
+    if (sw) {
+      sw->startLine() << "Section " << ++sectionNumber << " {\n";
+      sw->indent();
+    }
+
+    if (sectionLength < 4 || cursor.tell() - 4 + sectionLength > section.size())
+      return createStringError(errc::invalid_argument,
+                               "invalid section length " +
+                                   Twine(sectionLength) + " at offset 0x" +
+                                   utohexstr(cursor.tell() - 4));
+
+    if (Error e = parseSubsection(sectionLength))
+      return e;
+    if (sw) {
+      sw->unindent();
+      sw->startLine() << "}\n";
+    }
+  }
+
+  return cursor.takeError();
+}
diff --git a/llvm/lib/Support/ELFAttributes.cpp b/llvm/lib/Support/ELFAttributes.cpp
new file mode 100644
index 0000000000000..5be38825d6c6f
--- /dev/null
+++ b/llvm/lib/Support/ELFAttributes.cpp
@@ -0,0 +1,34 @@
+//===-- ELFAttributes.cpp - ELF Attributes --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/ELFAttributes.h"
+#include "llvm/ADT/StringRef.h"
+
+using namespace llvm;
+
+StringRef ELFAttrs::attrTypeAsString(unsigned attr, TagNameMap tagNameMap,
+                                     bool hasTagPrefix) {
+  auto tagNameIt = find_if(
+      tagNameMap, [attr](const TagNameItem item) { return item.attr == attr; });
+  if (tagNameIt == tagNameMap.end())
+    return "";
+  StringRef tagName = tagNameIt->tagName;
+  return hasTagPrefix ? tagName : tagName.drop_front(4);
+}
+
+Optional<unsigned> ELFAttrs::attrTypeFromString(StringRef tag,
+                                                TagNameMap tagNameMap) {
+  bool hasTagPrefix = tag.startswith("Tag_");
+  auto tagNameIt =
+      find_if(tagNameMap, [tag, hasTagPrefix](const TagNameItem item) {
+        return item.tagName.drop_front(hasTagPrefix ? 0 : 4) == tag;
+      });
+  if (tagNameIt == tagNameMap.end())
+    return None;
+  return tagNameIt->attr;
+}
diff --git a/llvm/lib/Support/FileCollector.cpp b/llvm/lib/Support/FileCollector.cpp
index 3a8bc99327bc4..f6d72685a8c3b 100644
--- a/llvm/lib/Support/FileCollector.cpp
+++ b/llvm/lib/Support/FileCollector.cpp
@@ -61,13 +61,19 @@ bool FileCollector::getRealPath(StringRef SrcPath,
   return true;
 }
 
-void FileCollector::addFile(const Twine &file) {
+void FileCollector::addFile(const Twine &File) {
   std::lock_guard<std::mutex> lock(Mutex);
-  std::string FileStr = file.str();
+  std::string FileStr = File.str();
   if (markAsSeen(FileStr))
     addFileImpl(FileStr);
 }
 
+void FileCollector::addDirectory(const Twine &Dir) {
+  assert(sys::fs::is_directory(Dir));
+  std::error_code EC;
+  addDirectoryImpl(Dir, vfs::getRealFileSystem(), EC);
+}
+
 void FileCollector::addFileImpl(StringRef SrcPath) {
   // We need an absolute src path to append to the root.
   SmallString<256> AbsoluteSrc = SrcPath;
@@ -101,6 +107,27 @@ void FileCollector::addFileImpl(StringRef SrcPath) {
   addFileToMapping(VirtualPath, DstPath);
 }
 
+llvm::vfs::directory_iterator
+FileCollector::addDirectoryImpl(const llvm::Twine &Dir,
+                                IntrusiveRefCntPtr<vfs::FileSystem> FS,
+                                std::error_code &EC) {
+  auto It = FS->dir_begin(Dir, EC);
+  if (EC)
+    return It;
+  addFile(Dir);
+  for (; !EC && It != llvm::vfs::directory_iterator(); It.increment(EC)) {
+    if (It->type() == sys::fs::file_type::regular_file ||
+        It->type() == sys::fs::file_type::directory_file ||
+        It->type() == sys::fs::file_type::symlink_file) {
+      addFile(It->path());
+    }
+  }
+  if (EC)
+    return It;
+  // Return a new iterator.
+  return FS->dir_begin(Dir, EC);
+}
+
 /// Set the access and modification time for the given file from the given
 /// status object.
 static std::error_code
@@ -171,7 +198,7 @@ std::error_code FileCollector::copyFiles(bool StopOnError) {
   return {};
 }
 
-std::error_code FileCollector::writeMapping(StringRef mapping_file) {
+std::error_code FileCollector::writeMapping(StringRef MappingFile) {
   std::lock_guard<std::mutex> lock(Mutex);
 
   VFSWriter.setOverlayDir(OverlayRoot);
@@ -179,7 +206,7 @@ std::error_code FileCollector::writeMapping(StringRef mapping_file) {
   VFSWriter.setUseExternalNames(false);
 
   std::error_code EC;
-  raw_fd_ostream os(mapping_file, EC, sys::fs::OF_Text);
+  raw_fd_ostream os(MappingFile, EC, sys::fs::OF_Text);
   if (EC)
     return EC;
 
@@ -188,7 +215,7 @@ std::error_code FileCollector::writeMapping(StringRef mapping_file) {
   return {};
 }
 
-namespace {
+namespace llvm {
 
 class FileCollectorFileSystem : public vfs::FileSystem {
 public:
@@ -213,22 +240,7 @@ class FileCollectorFileSystem : public vfs::FileSystem {
 
   llvm::vfs::directory_iterator dir_begin(const llvm::Twine &Dir,
                                           std::error_code &EC) override {
-    auto It = FS->dir_begin(Dir, EC);
-    if (EC)
-      return It;
-    // Collect everything that's listed in case the user needs it.
-    Collector->addFile(Dir);
-    for (; !EC && It != llvm::vfs::directory_iterator(); It.increment(EC)) {
-      if (It->type() == sys::fs::file_type::regular_file ||
-          It->type() == sys::fs::file_type::directory_file ||
-          It->type() == sys::fs::file_type::symlink_file) {
-        Collector->addFile(It->path());
-      }
-    }
-    if (EC)
-      return It;
-    // Return a new iterator.
-    return FS->dir_begin(Dir, EC);
+    return Collector->addDirectoryImpl(Dir, FS, EC);
   }
 
   std::error_code getRealPath(const Twine &Path,
@@ -259,7 +271,7 @@ class FileCollectorFileSystem : public vfs::FileSystem {
   std::shared_ptr<FileCollector> Collector;
 };
 
-} // end anonymous namespace
+} // namespace llvm
 
 IntrusiveRefCntPtr<vfs::FileSystem>
 FileCollector::createCollectorVFS(IntrusiveRefCntPtr<vfs::FileSystem> BaseFS,
diff --git a/llvm/lib/Support/Host.cpp b/llvm/lib/Support/Host.cpp
index d7558faed7846..6b8f7ca2c45bb 100644
--- a/llvm/lib/Support/Host.cpp
+++ b/llvm/lib/Support/Host.cpp
@@ -1477,6 +1477,7 @@ bool sys::getHostCPUFeatures(StringMap<bool> &Features) {
   Features["movdir64b"]       = HasLeaf7 && ((ECX >> 28) & 1);
   Features["enqcmd"]          = HasLeaf7 && ((ECX >> 29) & 1);
 
+  Features["serialize"]       = HasLeaf7 && ((EDX >> 14) & 1);
   // There are two CPUID leafs which information associated with the pconfig
   // instruction:
   // EAX=0x7, ECX=0x0 indicates the availability of the instruction (via the 18th
diff --git a/llvm/lib/Support/Parallel.cpp b/llvm/lib/Support/Parallel.cpp
index 0272a53beb393..7f6ce82763d6e 100644
--- a/llvm/lib/Support/Parallel.cpp
+++ b/llvm/lib/Support/Parallel.cpp
@@ -20,6 +20,8 @@
 #include <thread>
 #include <vector>
 
+llvm::ThreadPoolStrategy llvm::parallel::strategy;
+
 namespace llvm {
 namespace parallel {
 namespace detail {
@@ -78,6 +80,9 @@ class ThreadPoolExecutor : public Executor {
         T.join();
   }
 
+  struct Creator {
+    static void *call() { return new ThreadPoolExecutor(strategy); }
+  };
   struct Deleter {
     static void call(void *Ptr) { ((ThreadPoolExecutor *)Ptr)->stop(); }
   };
@@ -131,7 +136,8 @@ Executor *Executor::getDefaultExecutor() {
   // are more frequent with the debug static runtime.
   //
   // This also prevents intermittent deadlocks on exit with the MinGW runtime.
-  static ManagedStatic<ThreadPoolExecutor, object_creator<ThreadPoolExecutor>,
+
+  static ManagedStatic<ThreadPoolExecutor, ThreadPoolExecutor::Creator,
                        ThreadPoolExecutor::Deleter>
       ManagedExec;
   static std::unique_ptr<ThreadPoolExecutor> Exec(&(*ManagedExec));
diff --git a/llvm/lib/Support/Path.cpp b/llvm/lib/Support/Path.cpp
index fa3bf47569e83..6f065b9c5a3c8 100644
--- a/llvm/lib/Support/Path.cpp
+++ b/llvm/lib/Support/Path.cpp
@@ -496,49 +496,25 @@ void replace_extension(SmallVectorImpl<char> &path, const Twine &extension,
   path.append(ext.begin(), ext.end());
 }
 
-bool replace_path_prefix(SmallVectorImpl<char> &Path,
-                         const StringRef &OldPrefix, const StringRef &NewPrefix,
-                         Style style, bool strict) {
+bool replace_path_prefix(SmallVectorImpl<char> &Path, StringRef OldPrefix,
+                         StringRef NewPrefix) {
   if (OldPrefix.empty() && NewPrefix.empty())
     return false;
 
   StringRef OrigPath(Path.begin(), Path.size());
-  StringRef OldPrefixDir;
-
-  if (!strict && OldPrefix.size() > OrigPath.size())
-    return false;
-
-  // Ensure OldPrefixDir does not have a trailing separator.
-  if (!OldPrefix.empty() && is_separator(OldPrefix.back()))
-    OldPrefixDir = parent_path(OldPrefix, style);
-  else
-    OldPrefixDir = OldPrefix;
-
-  if (!OrigPath.startswith(OldPrefixDir))
+  if (!OrigPath.startswith(OldPrefix))
     return false;
 
-  if (OrigPath.size() > OldPrefixDir.size())
-    if (!is_separator(OrigPath[OldPrefixDir.size()], style) && strict)
-      return false;
-
   // If prefixes have the same size we can simply copy the new one over.
-  if (OldPrefixDir.size() == NewPrefix.size() && !strict) {
+  if (OldPrefix.size() == NewPrefix.size()) {
     llvm::copy(NewPrefix, Path.begin());
     return true;
   }
 
-  StringRef RelPath = OrigPath.substr(OldPrefixDir.size());
+  StringRef RelPath = OrigPath.substr(OldPrefix.size());
   SmallString<256> NewPath;
-  path::append(NewPath, style, NewPrefix);
-  if (!RelPath.empty()) {
-    if (!is_separator(RelPath[0], style) || !strict)
-      path::append(NewPath, style, RelPath);
-    else
-      path::append(NewPath, style, relative_path(RelPath, style));
-  }
-
+  (Twine(NewPrefix) + RelPath).toVector(NewPath);
   Path.swap(NewPath);
-
   return true;
 }
 
diff --git a/llvm/lib/Support/PrettyStackTrace.cpp b/llvm/lib/Support/PrettyStackTrace.cpp
index 30a12f65966ae..3d310ed8916ea 100644
--- a/llvm/lib/Support/PrettyStackTrace.cpp
+++ b/llvm/lib/Support/PrettyStackTrace.cpp
@@ -142,9 +142,15 @@ using CrashHandlerStringStorage =
 static CrashHandlerStringStorage crashHandlerStringStorage;
 #endif
 
+static const char *BugReportMsg =
+    "PLEASE submit a bug report to " BUG_REPORT_URL
+    " and include the crash backtrace.\n";
+
 /// This callback is run if a fatal signal is delivered to the process, it
 /// prints the pretty stack trace.
 static void CrashHandler(void *) {
+  errs() << BugReportMsg ;
+
 #ifndef __APPLE__
   // On non-apple systems, just emit the crash stack trace to stderr.
   PrintCurStackTrace(errs());
@@ -196,6 +202,12 @@ static void printForSigInfoIfNeeded() {
 
 #endif // ENABLE_BACKTRACES
 
+void llvm::setBugReportMsg(const char *Msg) {
+#if ENABLE_BACKTRACES
+  BugReportMsg = Msg;
+#endif
+}
+
 PrettyStackTraceEntry::PrettyStackTraceEntry() {
 #if ENABLE_BACKTRACES
   // Handle SIGINFO first, because we haven't finished constructing yet.
diff --git a/llvm/lib/Support/RISCVAttributeParser.cpp b/llvm/lib/Support/RISCVAttributeParser.cpp
new file mode 100644
index 0000000000000..393861c73a4a9
--- /dev/null
+++ b/llvm/lib/Support/RISCVAttributeParser.cpp
@@ -0,0 +1,67 @@
+//===-- RISCVAttributeParser.cpp - RISCV Attribute Parser -----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/RISCVAttributeParser.h"
+#include "llvm/ADT/StringExtras.h"
+
+using namespace llvm;
+
+const RISCVAttributeParser::DisplayHandler
+    RISCVAttributeParser::displayRoutines[] = {
+        {
+            RISCVAttrs::ARCH,
+            &ELFAttributeParser::stringAttribute,
+        },
+        {
+            RISCVAttrs::PRIV_SPEC,
+            &ELFAttributeParser::integerAttribute,
+        },
+        {
+            RISCVAttrs::PRIV_SPEC_MINOR,
+            &ELFAttributeParser::integerAttribute,
+        },
+        {
+            RISCVAttrs::PRIV_SPEC_REVISION,
+            &ELFAttributeParser::integerAttribute,
+        },
+        {
+            RISCVAttrs::STACK_ALIGN,
+            &RISCVAttributeParser::stackAlign,
+        },
+        {
+            RISCVAttrs::UNALIGNED_ACCESS,
+            &RISCVAttributeParser::unalignedAccess,
+        }};
+
+Error RISCVAttributeParser::unalignedAccess(unsigned tag) {
+  static const char *strings[] = {"No unaligned access", "Unaligned access"};
+  return parseStringAttribute("Unaligned_access", tag, makeArrayRef(strings));
+}
+
+Error RISCVAttributeParser::stackAlign(unsigned tag) {
+  uint64_t value = de.getULEB128(cursor);
+  std::string description =
+      "Stack alignment is " + utostr(value) + std::string("-bytes");
+  printAttribute(tag, value, description);
+  return Error::success();
+}
+
+Error RISCVAttributeParser::handler(uint64_t tag, bool &handled) {
+  handled = false;
+  for (unsigned AHI = 0, AHE = array_lengthof(displayRoutines); AHI != AHE;
+       ++AHI) {
+    if (uint64_t(displayRoutines[AHI].attribute) == tag) {
+      if (Error e = (this->*displayRoutines[AHI].routine)(tag))
+        return e;
+      handled = true;
+      break;
+    }
+  }
+
+  return Error::success();
+}
diff --git a/llvm/lib/Support/RISCVAttributes.cpp b/llvm/lib/Support/RISCVAttributes.cpp
new file mode 100644
index 0000000000000..201048e03009b
--- /dev/null
+++ b/llvm/lib/Support/RISCVAttributes.cpp
@@ -0,0 +1,25 @@
+//===-- RISCVAttributes.cpp - RISCV Attributes ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/RISCVAttributes.h"
+
+using namespace llvm;
+using namespace llvm::RISCVAttrs;
+
+static const TagNameItem tagData[] = {
+    {STACK_ALIGN, "Tag_stack_align"},
+    {ARCH, "Tag_arch"},
+    {UNALIGNED_ACCESS, "Tag_unaligned_access"},
+    {PRIV_SPEC, "Tag_priv_spec"},
+    {PRIV_SPEC_MINOR, "Tag_priv_spec_minor"},
+    {PRIV_SPEC_REVISION, "Tag_priv_spec_revision"},
+};
+
+const TagNameMap llvm::RISCVAttrs::RISCVAttributeTags(tagData,
+                                                      sizeof(tagData) /
+                                                          sizeof(TagNameItem));
diff --git a/llvm/lib/Support/Threading.cpp b/llvm/lib/Support/Threading.cpp
index de5adaddd9d38..4bf373ddbd72a 100644
--- a/llvm/lib/Support/Threading.cpp
+++ b/llvm/lib/Support/Threading.cpp
@@ -84,16 +84,14 @@ void llvm::llvm_execute_on_thread_async(
 int computeHostNumHardwareThreads();
 
 unsigned llvm::ThreadPoolStrategy::compute_thread_count() const {
+  if (ThreadsRequested > 0)
+    return ThreadsRequested;
+
   int MaxThreadCount = UseHyperThreads ? computeHostNumHardwareThreads()
                                        : sys::getHostNumPhysicalCores();
   if (MaxThreadCount <= 0)
     MaxThreadCount = 1;
-
-  // No need to create more threads than there are hardware threads, it would
-  // uselessly induce more context-switching and cache eviction.
-  if (!ThreadsRequested || ThreadsRequested > (unsigned)MaxThreadCount)
-    return MaxThreadCount;
-  return ThreadsRequested;
+  return MaxThreadCount;
 }
 
 namespace {
@@ -132,3 +130,23 @@ void llvm::llvm_execute_on_thread_async(
 }
 
 #endif
+
+Optional<ThreadPoolStrategy>
+llvm::get_threadpool_strategy(StringRef Num, ThreadPoolStrategy Default) {
+  if (Num == "all")
+    return llvm::hardware_concurrency();
+  if (Num.empty())
+    return Default;
+  unsigned V;
+  if (Num.getAsInteger(10, V))
+    return None; // malformed 'Num' value
+  if (V == 0)
+    return Default;
+
+  // Do not take the Default into account. This effectively disables
+  // heavyweight_hardware_concurrency() if the user asks for any number of
+  // threads on the cmd-line.
+  ThreadPoolStrategy S = llvm::hardware_concurrency();
+  S.ThreadsRequested = V;
+  return S;
+}
diff --git a/llvm/lib/Support/Triple.cpp b/llvm/lib/Support/Triple.cpp
index 03a17f4dd18ba..11da0e3924a50 100644
--- a/llvm/lib/Support/Triple.cpp
+++ b/llvm/lib/Support/Triple.cpp
@@ -662,6 +662,8 @@ static Triple::SubArchType parseSubArch(StringRef SubArchName) {
     return Triple::ARMSubArch_v8_4a;
   case ARM::ArchKind::ARMV8_5A:
     return Triple::ARMSubArch_v8_5a;
+  case ARM::ArchKind::ARMV8_6A:
+    return Triple::ARMSubArch_v8_6a;
   case ARM::ArchKind::ARMV8R:
     return Triple::ARMSubArch_v8r;
   case ARM::ArchKind::ARMV8MBaseline:
diff --git a/llvm/lib/Support/Unix/Process.inc b/llvm/lib/Support/Unix/Process.inc
index dfe81d7e28337..a68b30a546c8e 100644
--- a/llvm/lib/Support/Unix/Process.inc
+++ b/llvm/lib/Support/Unix/Process.inc
@@ -280,7 +280,7 @@ bool Process::FileDescriptorIsDisplayed(int fd) {
 #endif
 }
 
-static unsigned getColumns(int FileID) {
+static unsigned getColumns() {
   // If COLUMNS is defined in the environment, wrap to that many columns.
   if (const char *ColumnsStr = std::getenv("COLUMNS")) {
     int Columns = std::atoi(ColumnsStr);
@@ -288,31 +288,23 @@ static unsigned getColumns(int FileID) {
       return Columns;
   }
 
-  unsigned Columns = 0;
-
-#if defined(HAVE_SYS_IOCTL_H) && defined(HAVE_TERMIOS_H) \
-  && !(defined(_XOPEN_SOURCE) || defined(_POSIX_C_SOURCE))
-  // Try to determine the width of the terminal.
-  struct winsize ws;
-  if (ioctl(FileID, TIOCGWINSZ, &ws) == 0)
-    Columns = ws.ws_col;
-#endif
-
-  return Columns;
+  // We used to call ioctl TIOCGWINSZ to determine the width. It is considered
+  // unuseful.
+  return 0;
 }
 
 unsigned Process::StandardOutColumns() {
   if (!StandardOutIsDisplayed())
     return 0;
 
-  return getColumns(1);
+  return getColumns();
 }
 
 unsigned Process::StandardErrColumns() {
   if (!StandardErrIsDisplayed())
     return 0;
 
-  return getColumns(2);
+  return getColumns();
 }
 
 #ifdef HAVE_TERMINFO
diff --git a/llvm/lib/Support/Unix/Threading.inc b/llvm/lib/Support/Unix/Threading.inc
index 8cacaa83e961a..307a244691447 100644
--- a/llvm/lib/Support/Unix/Threading.inc
+++ b/llvm/lib/Support/Unix/Threading.inc
@@ -273,7 +273,7 @@ SetThreadPriorityResult llvm::set_thread_priority(ThreadPriority Priority) {
 int computeHostNumHardwareThreads() {
 #if defined(HAVE_SCHED_GETAFFINITY) && defined(HAVE_CPU_COUNT)
   cpu_set_t Set;
-  if (sched_getaffinity(0, sizeof(Set), &Set))
+  if (sched_getaffinity(0, sizeof(Set), &Set) == 0)
     return CPU_COUNT(&Set);
 #endif
   // Guard against std::thread::hardware_concurrency() returning 0.
diff --git a/llvm/lib/Support/VirtualFileSystem.cpp b/llvm/lib/Support/VirtualFileSystem.cpp
index 737116ce5ad2f..22b4a08db9f74 100644
--- a/llvm/lib/Support/VirtualFileSystem.cpp
+++ b/llvm/lib/Support/VirtualFileSystem.cpp
@@ -1912,11 +1912,21 @@ UniqueID vfs::getNextVirtualUniqueID() {
   return UniqueID(std::numeric_limits<uint64_t>::max(), ID);
 }
 
-void YAMLVFSWriter::addFileMapping(StringRef VirtualPath, StringRef RealPath) {
+void YAMLVFSWriter::addEntry(StringRef VirtualPath, StringRef RealPath,
+                             bool IsDirectory) {
   assert(sys::path::is_absolute(VirtualPath) && "virtual path not absolute");
   assert(sys::path::is_absolute(RealPath) && "real path not absolute");
   assert(!pathHasTraversal(VirtualPath) && "path traversal is not supported");
-  Mappings.emplace_back(VirtualPath, RealPath);
+  Mappings.emplace_back(VirtualPath, RealPath, IsDirectory);
+}
+
+void YAMLVFSWriter::addFileMapping(StringRef VirtualPath, StringRef RealPath) {
+  addEntry(VirtualPath, RealPath, /*IsDirectory=*/false);
+}
+
+void YAMLVFSWriter::addDirectoryMapping(StringRef VirtualPath,
+                                        StringRef RealPath) {
+  addEntry(VirtualPath, RealPath, /*IsDirectory=*/true);
 }
 
 namespace {
@@ -2017,7 +2027,10 @@ void JSONWriter::write(ArrayRef<YAMLVFSEntry> Entries,
 
   if (!Entries.empty()) {
     const YAMLVFSEntry &Entry = Entries.front();
-    startDirectory(path::parent_path(Entry.VPath));
+    bool first_entry_is_directory = Entry.IsDirectory;
+    StringRef Dir =
+        first_entry_is_directory ? Entry.VPath : path::parent_path(Entry.VPath);
+    startDirectory(Dir);
 
     StringRef RPath = Entry.RPath;
     if (UseOverlayRelative) {
@@ -2027,13 +2040,18 @@ void JSONWriter::write(ArrayRef<YAMLVFSEntry> Entries,
       RPath = RPath.slice(OverlayDirLen, RPath.size());
     }
 
-    writeEntry(path::filename(Entry.VPath), RPath);
+    if (!first_entry_is_directory)
+      writeEntry(path::filename(Entry.VPath), RPath);
 
     for (const auto &Entry : Entries.slice(1)) {
-      StringRef Dir = path::parent_path(Entry.VPath);
-      if (Dir == DirStack.back())
-        OS << ",\n";
-      else {
+      StringRef Dir =
+          Entry.IsDirectory ? Entry.VPath : path::parent_path(Entry.VPath);
+      if (Dir == DirStack.back()) {
+        if (!first_entry_is_directory) {
+          OS << ",\n";
+          first_entry_is_directory = false;
+        }
+      } else {
         while (!DirStack.empty() && !containedIn(DirStack.back(), Dir)) {
           OS << "\n";
           endDirectory();
diff --git a/llvm/lib/Support/Windows/Threading.inc b/llvm/lib/Support/Windows/Threading.inc
index 50c94068b395b..296e87b776959 100644
--- a/llvm/lib/Support/Windows/Threading.inc
+++ b/llvm/lib/Support/Windows/Threading.inc
@@ -131,6 +131,10 @@ struct ProcessorGroup {
   unsigned UsableThreads;
   unsigned ThreadsPerCore;
   uint64_t Affinity;
+
+  unsigned useableCores() const {
+    return std::max(1U, UsableThreads / ThreadsPerCore);
+  }
 };
 
 template <typename F>
@@ -232,33 +236,41 @@ int computeHostNumHardwareThreads() {
   return Threads;
 }
 
-// Assign the current thread to a more appropriate CPU socket or CPU group
-void llvm::ThreadPoolStrategy::apply_thread_strategy(
-    unsigned ThreadPoolNum) const {
+// Finds the proper CPU socket where a thread number should go. Returns 'None'
+// if the thread shall remain on the actual CPU socket.
+Optional<unsigned>
+llvm::ThreadPoolStrategy::compute_cpu_socket(unsigned ThreadPoolNum) const {
   ArrayRef<ProcessorGroup> Groups = getProcessorGroups();
+  // Only one CPU socket in the system or process affinity was set, no need to
+  // move the thread(s) to another CPU socket.
+  if (Groups.size() <= 1)
+    return None;
+
+  // We ask for less threads than there are hardware threads per CPU socket, no
+  // need to dispatch threads to other CPU sockets.
+  unsigned MaxThreadsPerSocket =
+      UseHyperThreads ? Groups[0].UsableThreads : Groups[0].useableCores();
+  if (compute_thread_count() <= MaxThreadsPerSocket)
+    return None;
 
   assert(ThreadPoolNum < compute_thread_count() &&
          "The thread index is not within thread strategy's range!");
 
-  // In this mode, the ThreadNumber represents the core number, not the
-  // hyper-thread number. Assumes all NUMA groups have the same amount of
-  // hyper-threads.
-  if (!UseHyperThreads)
-    ThreadPoolNum *= Groups[0].ThreadsPerCore;
-
-  unsigned ThreadRangeStart = 0;
-  for (unsigned I = 0; I < Groups.size(); ++I) {
-    const ProcessorGroup &G = Groups[I];
-    if (ThreadPoolNum >= ThreadRangeStart &&
-        ThreadPoolNum < ThreadRangeStart + G.UsableThreads) {
-
-      GROUP_AFFINITY Affinity{};
-      Affinity.Group = G.ID;
-      Affinity.Mask = G.Affinity;
-      SetThreadGroupAffinity(GetCurrentThread(), &Affinity, nullptr);
-    }
-    ThreadRangeStart += G.UsableThreads;
-  }
+  // Assumes the same number of hardware threads per CPU socket.
+  return (ThreadPoolNum * Groups.size()) / compute_thread_count();
+}
+
+// Assign the current thread to a more appropriate CPU socket or CPU group
+void llvm::ThreadPoolStrategy::apply_thread_strategy(
+    unsigned ThreadPoolNum) const {
+  Optional<unsigned> Socket = compute_cpu_socket(ThreadPoolNum);
+  if (!Socket)
+    return;
+  ArrayRef<ProcessorGroup> Groups = getProcessorGroups();
+  GROUP_AFFINITY Affinity{};
+  Affinity.Group = Groups[*Socket].ID;
+  Affinity.Mask = Groups[*Socket].Affinity;
+  SetThreadGroupAffinity(GetCurrentThread(), &Affinity, nullptr);
 }
 
 llvm::BitVector llvm::get_thread_affinity_mask() {
diff --git a/llvm/lib/Support/YAMLParser.cpp b/llvm/lib/Support/YAMLParser.cpp
index 4f9feee4a8a0d..ca8ffdc47afa6 100644
--- a/llvm/lib/Support/YAMLParser.cpp
+++ b/llvm/lib/Support/YAMLParser.cpp
@@ -268,8 +268,8 @@ class Scanner {
   }
 
   void setError(const Twine &Message, StringRef::iterator Position) {
-    if (Current >= End)
-      Current = End - 1;
+    if (Position >= End)
+      Position = End - 1;
 
     // propagate the error if possible
     if (EC)
@@ -278,14 +278,10 @@ class Scanner {
     // Don't print out more errors after the first one we encounter. The rest
     // are just the result of the first, and have no meaning.
     if (!Failed)
-      printError(SMLoc::getFromPointer(Current), SourceMgr::DK_Error, Message);
+      printError(SMLoc::getFromPointer(Position), SourceMgr::DK_Error, Message);
     Failed = true;
   }
 
-  void setError(const Twine &Message) {
-    setError(Message, Current);
-  }
-
   /// Returns true if an error occurred while parsing.
   bool failed() {
     return Failed;
@@ -934,13 +930,13 @@ void Scanner::scan_ns_uri_char() {
 
 bool Scanner::consume(uint32_t Expected) {
   if (Expected >= 0x80) {
-    setError("Cannot consume non-ascii characters");
+    setError("Cannot consume non-ascii characters", Current);
     return false;
   }
   if (Current == End)
     return false;
   if (uint8_t(*Current) >= 0x80) {
-    setError("Cannot consume non-ascii characters");
+    setError("Cannot consume non-ascii characters", Current);
     return false;
   }
   if (uint8_t(*Current) == Expected) {
@@ -1763,7 +1759,7 @@ bool Scanner::fetchMoreTokens() {
                       && !isBlankOrBreak(Current + 2)))
     return scanPlainScalar();
 
-  setError("Unrecognized character while tokenizing.");
+  setError("Unrecognized character while tokenizing.", Current);
   return false;
 }
 
diff --git a/llvm/lib/Target/AArch64/AArch64.td b/llvm/lib/Target/AArch64/AArch64.td
index a6af2a1538eca..a5d4e597120b8 100644
--- a/llvm/lib/Target/AArch64/AArch64.td
+++ b/llvm/lib/Target/AArch64/AArch64.td
@@ -365,6 +365,9 @@ def FeatureTaggedGlobals : SubtargetFeature<"tagged-globals",
     "true", "Use an instruction sequence for taking the address of a global "
     "that allows a memory tag in the upper address bits">;
 
+def FeatureBF16 : SubtargetFeature<"bf16", "HasBF16",
+    "true", "Enable BFloat16 Extension" >;
+
 //===----------------------------------------------------------------------===//
 // Architectures.
 //
@@ -391,8 +394,11 @@ def HasV8_5aOps : SubtargetFeature<
   "v8.5a", "HasV8_5aOps", "true", "Support ARM v8.5a instructions",
   [HasV8_4aOps, FeatureAltFPCmp, FeatureFRInt3264, FeatureSpecRestrict,
    FeatureSSBS, FeatureSB, FeaturePredRes, FeatureCacheDeepPersist,
-   FeatureBranchTargetId]
->;
+   FeatureBranchTargetId]>;
+
+def HasV8_6aOps : SubtargetFeature<
+  "v8.6a", "HasV8_6aOps", "true", "Support ARM v8.6a instructions",
+  [HasV8_5aOps, FeatureBF16]>;
 
 //===----------------------------------------------------------------------===//
 // Register File Description
diff --git a/llvm/lib/Target/AArch64/AArch64CallLowering.cpp b/llvm/lib/Target/AArch64/AArch64CallLowering.cpp
index 080c1f1364b7d..a6d71959777a8 100644
--- a/llvm/lib/Target/AArch64/AArch64CallLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64CallLowering.cpp
@@ -87,10 +87,9 @@ struct IncomingArgHandler : public CallLowering::ValueHandler {
   void assignValueToAddress(Register ValVReg, Register Addr, uint64_t Size,
                             MachinePointerInfo &MPO, CCValAssign &VA) override {
     MachineFunction &MF = MIRBuilder.getMF();
-    unsigned Align = inferAlignmentFromPtrInfo(MF, MPO);
     auto MMO = MF.getMachineMemOperand(
         MPO, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, Size,
-        Align);
+        inferAlignFromPtrInfo(MF, MPO));
     MIRBuilder.buildLoad(ValVReg, Addr, *MMO);
   }
 
@@ -177,9 +176,8 @@ struct OutgoingArgHandler : public CallLowering::ValueHandler {
                     .getReg(0);
     }
     MachineFunction &MF = MIRBuilder.getMF();
-    unsigned Align = inferAlignmentFromPtrInfo(MF, MPO);
-    auto MMO = MF.getMachineMemOperand(
-        MPO, MachineMemOperand::MOStore, Size, Align);
+    auto MMO = MF.getMachineMemOperand(MPO, MachineMemOperand::MOStore, Size,
+                                       inferAlignFromPtrInfo(MF, MPO));
     MIRBuilder.buildStore(ValVReg, Addr, *MMO);
   }
 
@@ -420,7 +418,7 @@ bool AArch64CallLowering::lowerFormalArguments(
   SmallVector<ArgInfo, 8> SplitArgs;
   unsigned i = 0;
   for (auto &Arg : F.args()) {
-    if (DL.getTypeStoreSize(Arg.getType()) == 0)
+    if (DL.getTypeStoreSize(Arg.getType()).isZero())
       continue;
 
     ArgInfo OrigArg{VRegs[i], Arg.getType()};
diff --git a/llvm/lib/Target/AArch64/AArch64FastISel.cpp b/llvm/lib/Target/AArch64/AArch64FastISel.cpp
index 7e9c68f2bb305..c7cfccd693bf0 100644
--- a/llvm/lib/Target/AArch64/AArch64FastISel.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FastISel.cpp
@@ -1130,7 +1130,7 @@ void AArch64FastISel::addLoadStoreOperands(Address &Addr,
     // and alignment should be based on the VT.
     MMO = FuncInfo.MF->getMachineMemOperand(
         MachinePointerInfo::getFixedStack(*FuncInfo.MF, FI, Offset), Flags,
-        MFI.getObjectSize(FI), MFI.getObjectAlignment(FI));
+        MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
     // Now add the rest of the operands.
     MIB.addFrameIndex(FI).addImm(Offset);
   } else {
@@ -3137,7 +3137,7 @@ bool AArch64FastISel::processCallArgs(CallLoweringInfo &CLI,
       Addr.setReg(AArch64::SP);
       Addr.setOffset(VA.getLocMemOffset() + BEAlign);
 
-      unsigned Alignment = DL.getABITypeAlignment(ArgVal->getType());
+      Align Alignment = DL.getABITypeAlign(ArgVal->getType());
       MachineMemOperand *MMO = FuncInfo.MF->getMachineMemOperand(
           MachinePointerInfo::getStack(*FuncInfo.MF, Addr.getOffset()),
           MachineMemOperand::MOStore, ArgVT.getStoreSize(), Alignment);
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index 9f17d7d0762bc..17b13f6f96fb2 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -216,6 +216,24 @@ AArch64FrameLowering::getStackIDForScalableVectors() const {
   return TargetStackID::SVEVector;
 }
 
+/// Returns the size of the fixed object area (allocated next to sp on entry)
+/// On Win64 this may include a var args area and an UnwindHelp object for EH.
+static unsigned getFixedObjectSize(const MachineFunction &MF,
+                                   const AArch64FunctionInfo *AFI, bool IsWin64,
+                                   bool IsFunclet) {
+  if (!IsWin64 || IsFunclet) {
+    // Only Win64 uses fixed objects, and then only for the function (not
+    // funclets)
+    return 0;
+  } else {
+    // Var args are stored here in the primary function.
+    const unsigned VarArgsArea = AFI->getVarArgsGPRSize();
+    // To support EH funclets we allocate an UnwindHelp object
+    const unsigned UnwindHelpObject = (MF.hasEHFunclets() ? 8 : 0);
+    return alignTo(VarArgsArea + UnwindHelpObject, 16);
+  }
+}
+
 /// Returns the size of the entire SVE stackframe (calleesaves + spills).
 static StackOffset getSVEStackSize(const MachineFunction &MF) {
   const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
@@ -291,10 +309,8 @@ MachineBasicBlock::iterator AArch64FrameLowering::eliminateCallFramePseudoInstr(
   uint64_t CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0;
 
   if (!hasReservedCallFrame(MF)) {
-    unsigned Align = getStackAlignment();
-
     int64_t Amount = I->getOperand(0).getImm();
-    Amount = alignTo(Amount, Align);
+    Amount = alignTo(Amount, getStackAlign());
     if (!IsDestroy)
       Amount = -Amount;
 
@@ -997,10 +1013,7 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
 
   bool IsWin64 =
       Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
-  // Var args are accounted for in the containing function, so don't
-  // include them for funclets.
-  unsigned FixedObject = (IsWin64 && !IsFunclet) ?
-                         alignTo(AFI->getVarArgsGPRSize(), 16) : 0;
+  unsigned FixedObject = getFixedObjectSize(MF, AFI, IsWin64, IsFunclet);
 
   auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
   // All of the remaining stack allocations are for locals.
@@ -1031,32 +1044,8 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
     ++MBBI;
   }
 
-  // The code below is not applicable to funclets. We have emitted all the SEH
-  // opcodes that we needed to emit.  The FP and BP belong to the containing
-  // function.
-  if (IsFunclet) {
-    if (NeedsWinCFI) {
-      HasWinCFI = true;
-      BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
-          .setMIFlag(MachineInstr::FrameSetup);
-    }
-
-    // SEH funclets are passed the frame pointer in X1.  If the parent
-    // function uses the base register, then the base register is used
-    // directly, and is not retrieved from X1.
-    if (F.hasPersonalityFn()) {
-      EHPersonality Per = classifyEHPersonality(F.getPersonalityFn());
-      if (isAsynchronousEHPersonality(Per)) {
-        BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), AArch64::FP)
-            .addReg(AArch64::X1).setMIFlag(MachineInstr::FrameSetup);
-        MBB.addLiveIn(AArch64::X1);
-      }
-    }
-
-    return;
-  }
-
-  if (HasFP) {
+  // For funclets the FP belongs to the containing function.
+  if (!IsFunclet && HasFP) {
     // Only set up FP if we actually need to.
     int64_t FPOffset = isTargetDarwin(MF) ? (AFI->getCalleeSavedStackSize() - 16) : 0;
 
@@ -1199,7 +1188,9 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
 
   // Allocate space for the rest of the frame.
   if (NumBytes) {
-    const bool NeedsRealignment = RegInfo->needsStackRealignment(MF);
+    // Alignment is required for the parent frame, not the funclet
+    const bool NeedsRealignment =
+        !IsFunclet && RegInfo->needsStackRealignment(MF);
     unsigned scratchSPReg = AArch64::SP;
 
     if (NeedsRealignment) {
@@ -1252,7 +1243,8 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
   // FIXME: Clarify FrameSetup flags here.
   // Note: Use emitFrameOffset() like above for FP if the FrameSetup flag is
   // needed.
-  if (RegInfo->hasBasePointer(MF)) {
+  // For funclets the BP belongs to the containing function.
+  if (!IsFunclet && RegInfo->hasBasePointer(MF)) {
     TII->copyPhysReg(MBB, MBBI, DL, RegInfo->getBaseRegister(), AArch64::SP,
                      false);
     if (NeedsWinCFI) {
@@ -1269,6 +1261,19 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
         .setMIFlag(MachineInstr::FrameSetup);
   }
 
+  // SEH funclets are passed the frame pointer in X1.  If the parent
+  // function uses the base register, then the base register is used
+  // directly, and is not retrieved from X1.
+  if (IsFunclet && F.hasPersonalityFn()) {
+    EHPersonality Per = classifyEHPersonality(F.getPersonalityFn());
+    if (isAsynchronousEHPersonality(Per)) {
+      BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), AArch64::FP)
+          .addReg(AArch64::X1)
+          .setMIFlag(MachineInstr::FrameSetup);
+      MBB.addLiveIn(AArch64::X1);
+    }
+  }
+
   if (needsFrameMoves) {
     const DataLayout &TD = MF.getDataLayout();
     const int StackGrowth = isTargetDarwin(MF)
@@ -1487,10 +1492,7 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
 
   bool IsWin64 =
       Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
-  // Var args are accounted for in the containing function, so don't
-  // include them for funclets.
-  unsigned FixedObject =
-      (IsWin64 && !IsFunclet) ? alignTo(AFI->getVarArgsGPRSize(), 16) : 0;
+  unsigned FixedObject = getFixedObjectSize(MF, AFI, IsWin64, IsFunclet);
 
   uint64_t AfterCSRPopSize = ArgumentPopSize;
   auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
@@ -1716,7 +1718,9 @@ static StackOffset getFPOffset(const MachineFunction &MF, int64_t ObjectOffset)
   const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
   bool IsWin64 =
       Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
-  unsigned FixedObject = IsWin64 ? alignTo(AFI->getVarArgsGPRSize(), 16) : 0;
+
+  unsigned FixedObject =
+      getFixedObjectSize(MF, AFI, IsWin64, /*IsFunclet=*/false);
   unsigned FPAdjust = isTargetDarwin(MF)
                         ? 16 : AFI->getCalleeSavedStackSize(MF.getFrameInfo());
   return {ObjectOffset + FixedObject + FPAdjust, MVT::i8};
@@ -2095,8 +2099,8 @@ static void computeCalleeSaveRegisterPairs(
       FixupDone = true;
       ByteOffset -= 8;
       assert(ByteOffset % 16 == 0);
-      assert(MFI.getObjectAlignment(RPI.FrameIdx) <= 16);
-      MFI.setObjectAlignment(RPI.FrameIdx, 16);
+      assert(MFI.getObjectAlign(RPI.FrameIdx) <= Align(16));
+      MFI.setObjectAlignment(RPI.FrameIdx, Align(16));
     }
 
     int Offset = RPI.isScalable() ? ScalableByteOffset : ByteOffset;
@@ -2178,32 +2182,33 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
     // Rationale: This sequence saves uop updates compared to a sequence of
     // pre-increment spills like stp xi,xj,[sp,#-16]!
     // Note: Similar rationale and sequence for restores in epilog.
-    unsigned Size, Align;
+    unsigned Size;
+    Align Alignment;
     switch (RPI.Type) {
     case RegPairInfo::GPR:
        StrOpc = RPI.isPaired() ? AArch64::STPXi : AArch64::STRXui;
        Size = 8;
-       Align = 8;
+       Alignment = Align(8);
        break;
     case RegPairInfo::FPR64:
        StrOpc = RPI.isPaired() ? AArch64::STPDi : AArch64::STRDui;
        Size = 8;
-       Align = 8;
+       Alignment = Align(8);
        break;
     case RegPairInfo::FPR128:
        StrOpc = RPI.isPaired() ? AArch64::STPQi : AArch64::STRQui;
        Size = 16;
-       Align = 16;
+       Alignment = Align(16);
        break;
     case RegPairInfo::ZPR:
        StrOpc = AArch64::STR_ZXI;
        Size = 16;
-       Align = 16;
+       Alignment = Align(16);
        break;
     case RegPairInfo::PPR:
        StrOpc = AArch64::STR_PXI;
        Size = 2;
-       Align = 2;
+       Alignment = Align(2);
        break;
     }
     LLVM_DEBUG(dbgs() << "CSR spill: (" << printReg(Reg1, TRI);
@@ -2232,7 +2237,7 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
       MIB.addReg(Reg2, getPrologueDeath(MF, Reg2));
       MIB.addMemOperand(MF.getMachineMemOperand(
           MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
-          MachineMemOperand::MOStore, Size, Align));
+          MachineMemOperand::MOStore, Size, Alignment));
     }
     MIB.addReg(Reg1, getPrologueDeath(MF, Reg1))
         .addReg(AArch64::SP)
@@ -2240,8 +2245,8 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
                             // where factor*scale is implicit
         .setMIFlag(MachineInstr::FrameSetup);
     MIB.addMemOperand(MF.getMachineMemOperand(
-        MachinePointerInfo::getFixedStack(MF,FrameIdxReg1),
-        MachineMemOperand::MOStore, Size, Align));
+        MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
+        MachineMemOperand::MOStore, Size, Alignment));
     if (NeedsWinCFI)
       InsertSEH(MIB, TII, MachineInstr::FrameSetup);
 
@@ -2283,32 +2288,33 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
     //    ldp     x22, x21, [sp, #0]      // addImm(+0)
     // Note: see comment in spillCalleeSavedRegisters()
     unsigned LdrOpc;
-    unsigned Size, Align;
+    unsigned Size;
+    Align Alignment;
     switch (RPI.Type) {
     case RegPairInfo::GPR:
        LdrOpc = RPI.isPaired() ? AArch64::LDPXi : AArch64::LDRXui;
        Size = 8;
-       Align = 8;
+       Alignment = Align(8);
        break;
     case RegPairInfo::FPR64:
        LdrOpc = RPI.isPaired() ? AArch64::LDPDi : AArch64::LDRDui;
        Size = 8;
-       Align = 8;
+       Alignment = Align(8);
        break;
     case RegPairInfo::FPR128:
        LdrOpc = RPI.isPaired() ? AArch64::LDPQi : AArch64::LDRQui;
        Size = 16;
-       Align = 16;
+       Alignment = Align(16);
        break;
     case RegPairInfo::ZPR:
        LdrOpc = AArch64::LDR_ZXI;
        Size = 16;
-       Align = 16;
+       Alignment = Align(16);
        break;
     case RegPairInfo::PPR:
        LdrOpc = AArch64::LDR_PXI;
        Size = 2;
-       Align = 2;
+       Alignment = Align(2);
        break;
     }
     LLVM_DEBUG(dbgs() << "CSR restore: (" << printReg(Reg1, TRI);
@@ -2331,7 +2337,7 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
       MIB.addReg(Reg2, getDefRegState(true));
       MIB.addMemOperand(MF.getMachineMemOperand(
           MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
-          MachineMemOperand::MOLoad, Size, Align));
+          MachineMemOperand::MOLoad, Size, Alignment));
     }
     MIB.addReg(Reg1, getDefRegState(true))
         .addReg(AArch64::SP)
@@ -2340,7 +2346,7 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
         .setMIFlag(MachineInstr::FrameDestroy);
     MIB.addMemOperand(MF.getMachineMemOperand(
         MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
-        MachineMemOperand::MOLoad, Size, Align));
+        MachineMemOperand::MOLoad, Size, Alignment));
     if (NeedsWinCFI)
       InsertSEH(MIB, TII, MachineInstr::FrameDestroy);
   };
@@ -2584,12 +2590,12 @@ static int64_t determineSVEStackObjectOffsets(MachineFrameInfo &MFI,
   // Then process all callee saved slots.
   if (getSVECalleeSaveSlotRange(MFI, MinCSFrameIndex, MaxCSFrameIndex)) {
     // Make sure to align the last callee save slot.
-    MFI.setObjectAlignment(MaxCSFrameIndex, 16U);
+    MFI.setObjectAlignment(MaxCSFrameIndex, Align(16));
 
     // Assign offsets to the callee save slots.
     for (int I = MinCSFrameIndex; I <= MaxCSFrameIndex; ++I) {
       Offset += MFI.getObjectSize(I);
-      Offset = alignTo(Offset, MFI.getObjectAlignment(I));
+      Offset = alignTo(Offset, MFI.getObjectAlign(I));
       if (AssignOffsets)
         Assign(I, -Offset);
     }
@@ -2611,15 +2617,15 @@ static int64_t determineSVEStackObjectOffsets(MachineFrameInfo &MFI,
 
   // Allocate all SVE locals and spills
   for (unsigned FI : ObjectsToAllocate) {
-    unsigned Align = MFI.getObjectAlignment(FI);
+    Align Alignment = MFI.getObjectAlign(FI);
     // FIXME: Given that the length of SVE vectors is not necessarily a power of
     // two, we'd need to align every object dynamically at runtime if the
     // alignment is larger than 16. This is not yet supported.
-    if (Align > 16)
+    if (Alignment > Align(16))
       report_fatal_error(
           "Alignment of scalable vectors > 16 bytes is not yet supported");
 
-    Offset = alignTo(Offset + MFI.getObjectSize(FI), Align);
+    Offset = alignTo(Offset + MFI.getObjectSize(FI), Alignment);
     if (AssignOffsets)
       Assign(FI, -Offset);
   }
@@ -2667,9 +2673,14 @@ void AArch64FrameLowering::processFunctionBeforeFrameFinalized(
     ++MBBI;
 
   // Create an UnwindHelp object.
-  int UnwindHelpFI =
-      MFI.CreateStackObject(/*size*/8, /*alignment*/16, false);
+  // The UnwindHelp object is allocated at the start of the fixed object area
+  int64_t FixedObject =
+      getFixedObjectSize(MF, AFI, /*IsWin64*/ true, /*IsFunclet*/ false);
+  int UnwindHelpFI = MFI.CreateFixedObject(/*Size*/ 8,
+                                           /*SPOffset*/ -FixedObject,
+                                           /*IsImmutable=*/false);
   EHInfo.UnwindHelpFrameIdx = UnwindHelpFI;
+
   // We need to store -2 into the UnwindHelp object at the start of the
   // function.
   DebugLoc DL;
@@ -3081,10 +3092,14 @@ int AArch64FrameLowering::getFrameIndexReferencePreferSP(
     const MachineFunction &MF, int FI, unsigned &FrameReg,
     bool IgnoreSPUpdates) const {
   const MachineFrameInfo &MFI = MF.getFrameInfo();
-  LLVM_DEBUG(dbgs() << "Offset from the SP for " << FI << " is "
-                    << MFI.getObjectOffset(FI) << "\n");
-  FrameReg = AArch64::SP;
-  return MFI.getObjectOffset(FI);
+  if (IgnoreSPUpdates) {
+    LLVM_DEBUG(dbgs() << "Offset from the SP for " << FI << " is "
+                      << MFI.getObjectOffset(FI) << "\n");
+    FrameReg = AArch64::SP;
+    return MFI.getObjectOffset(FI);
+  }
+
+  return getFrameIndexReference(MF, FI, FrameReg);
 }
 
 /// The parent frame offset (aka dispFrame) is only used on X86_64 to retrieve
@@ -3103,5 +3118,5 @@ unsigned AArch64FrameLowering::getWinEHFuncletFrameSize(
       MF.getInfo<AArch64FunctionInfo>()->getCalleeSavedStackSize();
   // This is the amount of stack a funclet needs to allocate.
   return alignTo(CSSize + MF.getFrameInfo().getMaxCallFrameSize(),
-                 getStackAlignment());
+                 getStackAlign());
 }
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index bae5c7f4eef8d..5bc22abde3fa7 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -223,6 +223,9 @@ class AArch64DAGToDAGISel : public SelectionDAGISel {
   /// unchanged; otherwise a REG_SEQUENCE value is returned.
   SDValue createDTuple(ArrayRef<SDValue> Vecs);
   SDValue createQTuple(ArrayRef<SDValue> Vecs);
+  // Form a sequence of SVE registers for instructions using list of vectors,
+  // e.g. structured loads and stores (ldN, stN).
+  SDValue createZTuple(ArrayRef<SDValue> Vecs);
 
   /// Generic helper for the createDTuple/createQTuple
   /// functions. Those should almost always be called instead.
@@ -258,6 +261,7 @@ class AArch64DAGToDAGISel : public SelectionDAGISel {
   void SelectPostStore(SDNode *N, unsigned NumVecs, unsigned Opc);
   void SelectStoreLane(SDNode *N, unsigned NumVecs, unsigned Opc);
   void SelectPostStoreLane(SDNode *N, unsigned NumVecs, unsigned Opc);
+  void SelectPredicatedStore(SDNode *N, unsigned NumVecs, const unsigned Opc);
 
   bool tryBitfieldExtractOp(SDNode *N);
   bool tryBitfieldExtractOpFromSExt(SDNode *N);
@@ -1192,6 +1196,16 @@ SDValue AArch64DAGToDAGISel::createQTuple(ArrayRef<SDValue> Regs) {
   return createTuple(Regs, RegClassIDs, SubRegs);
 }
 
+SDValue AArch64DAGToDAGISel::createZTuple(ArrayRef<SDValue> Regs) {
+  static const unsigned RegClassIDs[] = {AArch64::ZPR2RegClassID,
+                                         AArch64::ZPR3RegClassID,
+                                         AArch64::ZPR4RegClassID};
+  static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
+                                     AArch64::zsub2, AArch64::zsub3};
+
+  return createTuple(Regs, RegClassIDs, SubRegs);
+}
+
 SDValue AArch64DAGToDAGISel::createTuple(ArrayRef<SDValue> Regs,
                                          const unsigned RegClassIDs[],
                                          const unsigned SubRegs[]) {
@@ -1414,6 +1428,23 @@ void AArch64DAGToDAGISel::SelectStore(SDNode *N, unsigned NumVecs,
   ReplaceNode(N, St);
 }
 
+void AArch64DAGToDAGISel::SelectPredicatedStore(SDNode *N, unsigned NumVecs,
+                                                const unsigned Opc) {
+  SDLoc dl(N);
+
+  // Form a REG_SEQUENCE to force register allocation.
+  SmallVector<SDValue, 4> Regs(N->op_begin() + 2, N->op_begin() + 2 + NumVecs);
+  SDValue RegSeq = createZTuple(Regs);
+
+  SDValue Ops[] = {RegSeq, N->getOperand(NumVecs + 2),         // predicate
+                   N->getOperand(NumVecs + 3),                 // address
+                   CurDAG->getTargetConstant(0, dl, MVT::i64), // offset
+                   N->getOperand(0)};                          // chain
+  SDNode *St = CurDAG->getMachineNode(Opc, dl, N->getValueType(0), Ops);
+
+  ReplaceNode(N, St);
+}
+
 bool AArch64DAGToDAGISel::SelectAddrModeFrameIndexSVE(SDValue N, SDValue &Base,
                                                       SDValue &OffImm) {
   SDLoc dl(N);
@@ -3877,6 +3908,54 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
       }
       break;
     }
+    case Intrinsic::aarch64_sve_st2: {
+      if (VT == MVT::nxv16i8) {
+        SelectPredicatedStore(Node, 2, AArch64::ST2B_IMM);
+        return;
+      } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16) {
+        SelectPredicatedStore(Node, 2, AArch64::ST2H_IMM);
+        return;
+      } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
+        SelectPredicatedStore(Node, 2, AArch64::ST2W_IMM);
+        return;
+      } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
+        SelectPredicatedStore(Node, 2, AArch64::ST2D_IMM);
+        return;
+      }
+      break;
+    }
+    case Intrinsic::aarch64_sve_st3: {
+      if (VT == MVT::nxv16i8) {
+        SelectPredicatedStore(Node, 3, AArch64::ST3B_IMM);
+        return;
+      } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16) {
+        SelectPredicatedStore(Node, 3, AArch64::ST3H_IMM);
+        return;
+      } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
+        SelectPredicatedStore(Node, 3, AArch64::ST3W_IMM);
+        return;
+      } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
+        SelectPredicatedStore(Node, 3, AArch64::ST3D_IMM);
+        return;
+      }
+      break;
+    }
+    case Intrinsic::aarch64_sve_st4: {
+      if (VT == MVT::nxv16i8) {
+        SelectPredicatedStore(Node, 4, AArch64::ST4B_IMM);
+        return;
+      } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16) {
+        SelectPredicatedStore(Node, 4, AArch64::ST4H_IMM);
+        return;
+      } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
+        SelectPredicatedStore(Node, 4, AArch64::ST4W_IMM);
+        return;
+      } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
+        SelectPredicatedStore(Node, 4, AArch64::ST4D_IMM);
+        return;
+      }
+      break;
+    }
     }
     break;
   }
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index fdbdc42a96894..523ffe993efc5 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -198,6 +198,20 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
          { MVT::nxv2i8, MVT::nxv2i16, MVT::nxv2i32, MVT::nxv2i64, MVT::nxv4i8,
            MVT::nxv4i16, MVT::nxv4i32, MVT::nxv8i8, MVT::nxv8i16 })
       setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Legal);
+
+    for (auto VT :
+         { MVT::nxv2f16, MVT::nxv4f16, MVT::nxv8f16, MVT::nxv2f32, MVT::nxv4f32,
+           MVT::nxv2f64 }) {
+      setCondCodeAction(ISD::SETO, VT, Expand);
+      setCondCodeAction(ISD::SETOLT, VT, Expand);
+      setCondCodeAction(ISD::SETOLE, VT, Expand);
+      setCondCodeAction(ISD::SETULT, VT, Expand);
+      setCondCodeAction(ISD::SETULE, VT, Expand);
+      setCondCodeAction(ISD::SETUGE, VT, Expand);
+      setCondCodeAction(ISD::SETUGT, VT, Expand);
+      setCondCodeAction(ISD::SETUEQ, VT, Expand);
+      setCondCodeAction(ISD::SETUNE, VT, Expand);
+    }
   }
 
   // Compute derived properties from the register classes
@@ -7544,9 +7558,15 @@ SDValue AArch64TargetLowering::LowerSPLAT_VECTOR(SDValue Op,
   // FPRs don't have this restriction.
   switch (ElemVT.getSimpleVT().SimpleTy) {
   case MVT::i1: {
+    // The only legal i1 vectors are SVE vectors, so we can use SVE-specific
+    // lowering code.
+    if (auto *ConstVal = dyn_cast<ConstantSDNode>(SplatVal)) {
+      if (ConstVal->isOne())
+        return getPTrue(DAG, dl, VT, AArch64SVEPredPattern::all);
+      // TODO: Add special case for constant false
+    }
     // The general case of i1.  There isn't any natural way to do this,
     // so we use some trickery with whilelo.
-    // TODO: Add special cases for splat of constant true/false.
     SplatVal = DAG.getAnyExtOrTrunc(SplatVal, dl, MVT::i64);
     SplatVal = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::i64, SplatVal,
                            DAG.getValueType(MVT::i1));
@@ -9187,10 +9207,10 @@ static bool areExtractShuffleVectors(Value *Op1, Value *Op2) {
     return FullVT->getNumElements() == 2 * HalfVT->getNumElements();
   };
 
-  Constant *M1, *M2;
+  ArrayRef<int> M1, M2;
   Value *S1Op1, *S2Op1;
-  if (!match(Op1, m_ShuffleVector(m_Value(S1Op1), m_Undef(), m_Constant(M1))) ||
-      !match(Op2, m_ShuffleVector(m_Value(S2Op1), m_Undef(), m_Constant(M2))))
+  if (!match(Op1, m_ShuffleVector(m_Value(S1Op1), m_Undef(), m_Mask(M1))) ||
+      !match(Op2, m_ShuffleVector(m_Value(S2Op1), m_Undef(), m_Mask(M2))))
     return false;
 
   // Check that the operands are half as wide as the result and we extract
diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
index 20a7cf6b1d57b..6b23c7cfd17a1 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@@ -7786,6 +7786,110 @@ class BaseSIMDIndexedTied<bit Q, bit U, bit Scalar, bits<2> size, bits<4> opc,
   let Inst{4-0}   = Rd;
 }
 
+
+//----------------------------------------------------------------------------
+// Armv8.6 BFloat16 Extension
+//----------------------------------------------------------------------------
+let mayStore = 0, mayLoad = 0, hasSideEffects = 0 in {
+
+class BaseSIMDThreeSameVectorBFDot<bit Q, bit U, string asm, string kind1,
+                                   string kind2, RegisterOperand RegType,
+                                   ValueType AccumType, ValueType InputType>
+  : BaseSIMDThreeSameVectorTied<Q, U, 0b010, 0b11111, RegType, asm, kind1, []> {
+  let AsmString = !strconcat(asm,
+                             "{\t$Rd" # kind1 # ", $Rn" # kind2 #
+                               ", $Rm" # kind2 # "}");
+}
+
+multiclass SIMDThreeSameVectorBFDot<bit U, string asm> {
+  def v4f16 : BaseSIMDThreeSameVectorBFDot<0, U, asm, ".2s", ".4h", V64,
+                                           v2f32, v8i8>;
+  def v8f16 : BaseSIMDThreeSameVectorBFDot<1, U, asm, ".4s", ".8h", V128,
+                                           v4f32, v16i8>;
+}
+
+class BaseSIMDThreeSameVectorBF16DotI<bit Q, bit U, string asm,
+                                      string dst_kind, string lhs_kind,
+                                      string rhs_kind,
+                                      RegisterOperand RegType,
+                                      ValueType AccumType,
+                                      ValueType InputType>
+  : BaseSIMDIndexedTied<Q, U, 0b0, 0b01, 0b1111,
+                        RegType, RegType, V128, VectorIndexS,
+                        asm, "", dst_kind, lhs_kind, rhs_kind,
+        []> {
+
+  bits<2> idx;
+  let Inst{21}    = idx{0};  // L
+  let Inst{11}    = idx{1};  // H
+}
+
+multiclass SIMDThreeSameVectorBF16DotI<bit U, string asm> {
+
+  def v4f16  : BaseSIMDThreeSameVectorBF16DotI<0, U, asm, ".2s", ".4h",
+                                               ".2h", V64, v2f32, v8i8>;
+  def v8f16 : BaseSIMDThreeSameVectorBF16DotI<1, U, asm, ".4s", ".8h",
+                                              ".2h", V128, v4f32, v16i8>;
+}
+
+class SIMDBF16MLAL<bit Q, string asm>
+  : BaseSIMDThreeSameVectorTied<Q, 0b1, 0b110, 0b11111, V128, asm, ".4s",
+              []> { // TODO: Add intrinsics
+  let AsmString = !strconcat(asm, "{\t$Rd.4s, $Rn.8h, $Rm.8h}");
+}
+
+class SIMDBF16MLALIndex<bit Q, string asm>
+  : I<(outs V128:$dst),
+      (ins V128:$Rd, V128:$Rn, V128_lo:$Rm, VectorIndexH:$idx), asm,
+      "{\t$Rd.4s, $Rn.8h, $Rm.h$idx}", "$Rd = $dst",
+          []>, // TODO: Add intrinsics
+    Sched<[WriteV]> {
+  bits<5> Rd;
+  bits<5> Rn;
+  bits<4> Rm;
+  bits<3> idx;
+
+  let Inst{31}    = 0;
+  let Inst{30}    = Q;
+  let Inst{29-22} = 0b00111111;
+  let Inst{21-20} = idx{1-0};
+  let Inst{19-16} = Rm;
+  let Inst{15-12} = 0b1111;
+  let Inst{11}    = idx{2};   // H
+  let Inst{10}    = 0;
+  let Inst{9-5}   = Rn;
+  let Inst{4-0}   = Rd;
+}
+
+class SIMDThreeSameVectorBF16MatrixMul<string asm>
+  : BaseSIMDThreeSameVectorTied<1, 1, 0b010, 0b11101,
+                                V128, asm, ".4s",
+                          []> {
+  let AsmString = !strconcat(asm, "{\t$Rd", ".4s", ", $Rn", ".8h",
+                                    ", $Rm", ".8h", "}");
+}
+
+class SIMD_BFCVTN
+  : BaseSIMDMixedTwoVector<0, 0, 0b10, 0b10110, V128, V128,
+                           "bfcvtn", ".4h", ".4s",
+    []>;
+
+class SIMD_BFCVTN2
+  : BaseSIMDMixedTwoVectorTied<1, 0, 0b10, 0b10110, V128, V128,
+                           "bfcvtn2", ".8h", ".4s",
+    []>;
+
+class BF16ToSinglePrecision<string asm>
+  : I<(outs FPR16:$Rd), (ins FPR32:$Rn), asm, "\t$Rd, $Rn", "", []>,
+    Sched<[WriteFCvt]> {
+  bits<5> Rd;
+  bits<5> Rn;
+  let Inst{31-10} = 0b0001111001100011010000;
+  let Inst{9-5}   = Rn;
+  let Inst{4-0}   = Rd;
+}
+} // End of let mayStore = 0, mayLoad = 0, hasSideEffects = 0
+
 // ARMv8.2-A Dot Product Instructions (Indexed)
 class BaseSIMDThreeSameVectorDotIndex<bit Q, bit U, string asm, string dst_kind,
                                       string lhs_kind, string rhs_kind,
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 436b26c36d2d9..6ae21d5ef343f 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -510,8 +510,8 @@ static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg,
 
 bool AArch64InstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
                                        ArrayRef<MachineOperand> Cond,
-                                       unsigned DstReg, unsigned TrueReg,
-                                       unsigned FalseReg, int &CondCycles,
+                                       Register DstReg, Register TrueReg,
+                                       Register FalseReg, int &CondCycles,
                                        int &TrueCycles,
                                        int &FalseCycles) const {
   // Check register classes.
@@ -559,9 +559,9 @@ bool AArch64InstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
 
 void AArch64InstrInfo::insertSelect(MachineBasicBlock &MBB,
                                     MachineBasicBlock::iterator I,
-                                    const DebugLoc &DL, unsigned DstReg,
+                                    const DebugLoc &DL, Register DstReg,
                                     ArrayRef<MachineOperand> Cond,
-                                    unsigned TrueReg, unsigned FalseReg) const {
+                                    Register TrueReg, Register FalseReg) const {
   MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
 
   // Parse the condition code, see parseCondBranch() above.
@@ -931,7 +931,7 @@ bool AArch64InstrInfo::isSEHInstruction(const MachineInstr &MI) {
 }
 
 bool AArch64InstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
-                                             unsigned &SrcReg, unsigned &DstReg,
+                                             Register &SrcReg, Register &DstReg,
                                              unsigned &SubIdx) const {
   switch (MI.getOpcode()) {
   default:
@@ -1011,8 +1011,8 @@ bool AArch64InstrInfo::isSchedulingBoundary(const MachineInstr &MI,
 /// analyzeCompare - For a comparison instruction, return the source registers
 /// in SrcReg and SrcReg2, and the value it compares against in CmpValue.
 /// Return true if the comparison instruction can be analyzed.
-bool AArch64InstrInfo::analyzeCompare(const MachineInstr &MI, unsigned &SrcReg,
-                                      unsigned &SrcReg2, int &CmpMask,
+bool AArch64InstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
+                                      Register &SrcReg2, int &CmpMask,
                                       int &CmpValue) const {
   // The first operand can be a frame index where we'd normally expect a
   // register.
@@ -1207,7 +1207,7 @@ static bool areCFlagsAccessedBetweenInstrs(
 ///    instruction.
 ///    Only comparison with zero is supported.
 bool AArch64InstrInfo::optimizeCompareInstr(
-    MachineInstr &CmpInstr, unsigned SrcReg, unsigned SrcReg2, int CmpMask,
+    MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int CmpMask,
     int CmpValue, const MachineRegisterInfo *MRI) const {
   assert(CmpInstr.getParent());
   assert(MRI);
@@ -2922,11 +2922,11 @@ void AArch64InstrInfo::storeRegToStackSlot(
     const TargetRegisterInfo *TRI) const {
   MachineFunction &MF = *MBB.getParent();
   MachineFrameInfo &MFI = MF.getFrameInfo();
-  unsigned Align = MFI.getObjectAlignment(FI);
 
   MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
-  MachineMemOperand *MMO = MF.getMachineMemOperand(
-      PtrInfo, MachineMemOperand::MOStore, MFI.getObjectSize(FI), Align);
+  MachineMemOperand *MMO =
+      MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore,
+                              MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
   unsigned Opc = 0;
   bool Offset = true;
   switch (TRI->getSpillSize(*RC)) {
@@ -3064,10 +3064,10 @@ void AArch64InstrInfo::loadRegFromStackSlot(
     const TargetRegisterInfo *TRI) const {
   MachineFunction &MF = *MBB.getParent();
   MachineFrameInfo &MFI = MF.getFrameInfo();
-  unsigned Align = MFI.getObjectAlignment(FI);
   MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
-  MachineMemOperand *MMO = MF.getMachineMemOperand(
-      PtrInfo, MachineMemOperand::MOLoad, MFI.getObjectSize(FI), Align);
+  MachineMemOperand *MMO =
+      MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad,
+                              MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
 
   unsigned Opc = 0;
   bool Offset = true;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index ee425f47a5ee8..c67a4dadca205 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -52,8 +52,8 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
 
   bool isAsCheapAsAMove(const MachineInstr &MI) const override;
 
-  bool isCoalescableExtInstr(const MachineInstr &MI, unsigned &SrcReg,
-                             unsigned &DstReg, unsigned &SubIdx) const override;
+  bool isCoalescableExtInstr(const MachineInstr &MI, Register &SrcReg,
+                             Register &DstReg, unsigned &SubIdx) const override;
 
   bool
   areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
@@ -197,12 +197,12 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
   bool
   reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const override;
   bool canInsertSelect(const MachineBasicBlock &, ArrayRef<MachineOperand> Cond,
-                       unsigned, unsigned, unsigned, int &, int &,
+                       Register, Register, Register, int &, int &,
                        int &) const override;
   void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
-                    const DebugLoc &DL, unsigned DstReg,
-                    ArrayRef<MachineOperand> Cond, unsigned TrueReg,
-                    unsigned FalseReg) const override;
+                    const DebugLoc &DL, Register DstReg,
+                    ArrayRef<MachineOperand> Cond, Register TrueReg,
+                    Register FalseReg) const override;
   void getNoop(MCInst &NopInst) const override;
 
   bool isSchedulingBoundary(const MachineInstr &MI,
@@ -212,13 +212,13 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
   /// analyzeCompare - For a comparison instruction, return the source registers
   /// in SrcReg and SrcReg2, and the value it compares against in CmpValue.
   /// Return true if the comparison instruction can be analyzed.
-  bool analyzeCompare(const MachineInstr &MI, unsigned &SrcReg,
-                      unsigned &SrcReg2, int &CmpMask,
+  bool analyzeCompare(const MachineInstr &MI, Register &SrcReg,
+                      Register &SrcReg2, int &CmpMask,
                       int &CmpValue) const override;
   /// optimizeCompareInstr - Convert the instruction supplying the argument to
   /// the comparison into one that sets the zero bit in the flags register.
-  bool optimizeCompareInstr(MachineInstr &CmpInstr, unsigned SrcReg,
-                            unsigned SrcReg2, int CmpMask, int CmpValue,
+  bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
+                            Register SrcReg2, int CmpMask, int CmpValue,
                             const MachineRegisterInfo *MRI) const override;
   bool optimizeCondBranch(MachineInstr &MI) const override;
 
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 1537adc5c13ad..edff05a514385 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -23,6 +23,8 @@ def HasV8_4a         : Predicate<"Subtarget->hasV8_4aOps()">,
                                  AssemblerPredicate<(all_of HasV8_4aOps), "armv8.4a">;
 def HasV8_5a         : Predicate<"Subtarget->hasV8_5aOps()">,
                                  AssemblerPredicate<(all_of HasV8_5aOps), "armv8.5a">;
+def HasV8_6a         : Predicate<"Subtarget->hasV8_6aOps()">,
+                                 AssemblerPredicate<(all_of HasV8_6aOps), "armv8.6a">;
 def HasVH            : Predicate<"Subtarget->hasVH()">,
                        AssemblerPredicate<(all_of FeatureVH), "vh">;
 
@@ -142,6 +144,8 @@ def HasETE           : Predicate<"Subtarget->hasETE()">,
                        AssemblerPredicate<(all_of FeatureETE), "ete">;
 def HasTRBE          : Predicate<"Subtarget->hasTRBE()">,
                        AssemblerPredicate<(all_of FeatureTRBE), "trbe">;
+def HasBF16          : Predicate<"Subtarget->hasBF16()">,
+                       AssemblerPredicate<(all_of FeatureBF16), "bf16">;
 def IsLE             : Predicate<"Subtarget->isLittleEndian()">;
 def IsBE             : Predicate<"!Subtarget->isLittleEndian()">;
 def IsWindows        : Predicate<"Subtarget->isTargetWindows()">;
@@ -746,6 +750,20 @@ defm SDOTlane : SIMDThreeSameVectorDotIndex<0, "sdot", int_aarch64_neon_sdot>;
 defm UDOTlane : SIMDThreeSameVectorDotIndex<1, "udot", int_aarch64_neon_udot>;
 }
 
+// ARMv8.6-A BFloat
+let Predicates = [HasBF16] in {
+defm BFDOT       : SIMDThreeSameVectorBFDot<1, "bfdot">;
+defm BF16DOTlane : SIMDThreeSameVectorBF16DotI<0, "bfdot">;
+def BFMMLA       : SIMDThreeSameVectorBF16MatrixMul<"bfmmla">;
+def BFMLALB      : SIMDBF16MLAL<0, "bfmlalb">;
+def BFMLALT      : SIMDBF16MLAL<1, "bfmlalt">;
+def BFMLALBIdx   : SIMDBF16MLALIndex<0, "bfmlalb">;
+def BFMLALTIdx   : SIMDBF16MLALIndex<1, "bfmlalt">;
+def BFCVTN       : SIMD_BFCVTN;
+def BFCVTN2      : SIMD_BFCVTN2;
+def BFCVT        : BF16ToSinglePrecision<"bfcvt">;
+}
+
 // ARMv8.2-A FP16 Fused Multiply-Add Long
 let Predicates = [HasNEON, HasFP16FML] in {
 defm FMLAL      : SIMDThreeSameVectorFML<0, 1, 0b001, "fmlal", int_aarch64_neon_fmlal>;
diff --git a/llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp
index ea94aca3e2f5c..f4b139b4a796f 100644
--- a/llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp
@@ -1716,16 +1716,15 @@ bool AArch64InstructionSelector::earlySelect(MachineInstr &I) const {
 
     Register DefReg = I.getOperand(0).getReg();
     LLT Ty = MRI.getType(DefReg);
-    if (Ty != LLT::scalar(64) && Ty != LLT::scalar(32))
-      return false;
-
-    if (Ty == LLT::scalar(64)) {
+    if (Ty.getSizeInBits() == 64) {
       I.getOperand(1).ChangeToRegister(AArch64::XZR, false);
       RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass, MRI);
-    } else {
+    } else if (Ty.getSizeInBits() == 32) {
       I.getOperand(1).ChangeToRegister(AArch64::WZR, false);
       RBI.constrainGenericRegister(DefReg, AArch64::GPR32RegClass, MRI);
-    }
+    } else
+      return false;
+
     I.setDesc(TII.get(TargetOpcode::COPY));
     return true;
   }
@@ -2767,14 +2766,13 @@ bool AArch64InstructionSelector::selectBrJT(MachineInstr &I,
 
   Register TargetReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
   Register ScratchReg = MRI.createVirtualRegister(&AArch64::GPR64spRegClass);
-  MIB.buildInstr(AArch64::JumpTableDest32, {TargetReg, ScratchReg},
-                 {JTAddr, Index})
-      .addJumpTableIndex(JTI);
-
+  auto JumpTableInst = MIB.buildInstr(AArch64::JumpTableDest32,
+                                      {TargetReg, ScratchReg}, {JTAddr, Index})
+                           .addJumpTableIndex(JTI);
   // Build the indirect branch.
   MIB.buildInstr(AArch64::BR, {}, {TargetReg});
   I.eraseFromParent();
-  return true;
+  return constrainSelectedInstRegOperands(*JumpTableInst, TII, TRI, RBI);
 }
 
 bool AArch64InstructionSelector::selectJumpTable(
diff --git a/llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp
index d3d43ad5c2ea5..60ccb3621a2e8 100644
--- a/llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp
@@ -724,7 +724,7 @@ bool AArch64LegalizerInfo::legalizeVaArg(MachineInstr &MI,
                                          MachineIRBuilder &MIRBuilder) const {
   MIRBuilder.setInstr(MI);
   MachineFunction &MF = MIRBuilder.getMF();
-  unsigned Align = MI.getOperand(2).getImm();
+  Align Alignment(MI.getOperand(2).getImm());
   Register Dst = MI.getOperand(0).getReg();
   Register ListPtr = MI.getOperand(1).getReg();
 
@@ -732,19 +732,19 @@ bool AArch64LegalizerInfo::legalizeVaArg(MachineInstr &MI,
   LLT IntPtrTy = LLT::scalar(PtrTy.getSizeInBits());
 
   const unsigned PtrSize = PtrTy.getSizeInBits() / 8;
+  const Align PtrAlign = Align(PtrSize);
   auto List = MIRBuilder.buildLoad(
       PtrTy, ListPtr,
       *MF.getMachineMemOperand(MachinePointerInfo(), MachineMemOperand::MOLoad,
-                               PtrSize, /* Align = */ PtrSize));
+                               PtrSize, PtrAlign));
 
   MachineInstrBuilder DstPtr;
-  if (Align > PtrSize) {
+  if (Alignment > PtrAlign) {
     // Realign the list to the actual required alignment.
-    auto AlignMinus1 = MIRBuilder.buildConstant(IntPtrTy, Align - 1);
-
+    auto AlignMinus1 =
+        MIRBuilder.buildConstant(IntPtrTy, Alignment.value() - 1);
     auto ListTmp = MIRBuilder.buildPtrAdd(PtrTy, List, AlignMinus1.getReg(0));
-
-    DstPtr = MIRBuilder.buildPtrMask(PtrTy, ListTmp, Log2_64(Align));
+    DstPtr = MIRBuilder.buildPtrMask(PtrTy, ListTmp, Log2(Alignment));
   } else
     DstPtr = List;
 
@@ -752,16 +752,16 @@ bool AArch64LegalizerInfo::legalizeVaArg(MachineInstr &MI,
   MIRBuilder.buildLoad(
       Dst, DstPtr,
       *MF.getMachineMemOperand(MachinePointerInfo(), MachineMemOperand::MOLoad,
-                               ValSize, std::max(Align, PtrSize)));
+                               ValSize, std::max(Alignment, PtrAlign)));
 
-  auto Size = MIRBuilder.buildConstant(IntPtrTy, alignTo(ValSize, PtrSize));
+  auto Size = MIRBuilder.buildConstant(IntPtrTy, alignTo(ValSize, PtrAlign));
 
   auto NewList = MIRBuilder.buildPtrAdd(PtrTy, DstPtr, Size.getReg(0));
 
-  MIRBuilder.buildStore(
-      NewList, ListPtr,
-      *MF.getMachineMemOperand(MachinePointerInfo(), MachineMemOperand::MOStore,
-                               PtrSize, /* Align = */ PtrSize));
+  MIRBuilder.buildStore(NewList, ListPtr,
+                        *MF.getMachineMemOperand(MachinePointerInfo(),
+                                                 MachineMemOperand::MOStore,
+                                                 PtrSize, PtrAlign));
 
   MI.eraseFromParent();
   return true;
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 7395f24f2118d..e19aef9955e4e 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -443,14 +443,14 @@ let Predicates = [HasSVE] in {
   defm PFIRST  : sve_int_pfirst<0b00000, "pfirst", int_aarch64_sve_pfirst>;
   defm PNEXT   : sve_int_pnext<0b00110, "pnext", int_aarch64_sve_pnext>;
 
-  defm AND_PPzPP   : sve_int_pred_log<0b0000, "and", int_aarch64_sve_and_z>;
+  defm AND_PPzPP   : sve_int_pred_log<0b0000, "and", int_aarch64_sve_and_z, and>;
   defm BIC_PPzPP   : sve_int_pred_log<0b0001, "bic", int_aarch64_sve_bic_z>;
-  defm EOR_PPzPP   : sve_int_pred_log<0b0010, "eor", int_aarch64_sve_eor_z>;
+  defm EOR_PPzPP   : sve_int_pred_log<0b0010, "eor", int_aarch64_sve_eor_z, xor>;
   defm SEL_PPPP    : sve_int_pred_log<0b0011, "sel", vselect>;
   defm ANDS_PPzPP  : sve_int_pred_log<0b0100, "ands", null_frag>;
   defm BICS_PPzPP  : sve_int_pred_log<0b0101, "bics", null_frag>;
   defm EORS_PPzPP  : sve_int_pred_log<0b0110, "eors", null_frag>;
-  defm ORR_PPzPP   : sve_int_pred_log<0b1000, "orr", int_aarch64_sve_orr_z>;
+  defm ORR_PPzPP   : sve_int_pred_log<0b1000, "orr", int_aarch64_sve_orr_z, or>;
   defm ORN_PPzPP   : sve_int_pred_log<0b1001, "orn", int_aarch64_sve_orn_z>;
   defm NOR_PPzPP   : sve_int_pred_log<0b1010, "nor", int_aarch64_sve_nor_z>;
   defm NAND_PPzPP  : sve_int_pred_log<0b1011, "nand", int_aarch64_sve_nand_z>;
@@ -980,11 +980,11 @@ multiclass sve_prefetch<SDPatternOperator prefetch, ValueType PredTy, Instructio
   defm CMPLO_PPzZI : sve_int_ucmp_vi<0b10, "cmplo", SETULT, null_frag, int_aarch64_sve_cmphi>;
   defm CMPLS_PPzZI : sve_int_ucmp_vi<0b11, "cmpls", SETULE, null_frag, int_aarch64_sve_cmphs>;
 
-  defm FCMGE_PPzZZ : sve_fp_3op_p_pd<0b000, "fcmge", int_aarch64_sve_fcmpge>;
-  defm FCMGT_PPzZZ : sve_fp_3op_p_pd<0b001, "fcmgt", int_aarch64_sve_fcmpgt>;
-  defm FCMEQ_PPzZZ : sve_fp_3op_p_pd<0b010, "fcmeq", int_aarch64_sve_fcmpeq>;
-  defm FCMNE_PPzZZ : sve_fp_3op_p_pd<0b011, "fcmne", int_aarch64_sve_fcmpne>;
-  defm FCMUO_PPzZZ : sve_fp_3op_p_pd<0b100, "fcmuo", int_aarch64_sve_fcmpuo>;
+  defm FCMGE_PPzZZ : sve_fp_3op_p_pd_cc<0b000, "fcmge", int_aarch64_sve_fcmpge, setoge>;
+  defm FCMGT_PPzZZ : sve_fp_3op_p_pd_cc<0b001, "fcmgt", int_aarch64_sve_fcmpgt, setogt>;
+  defm FCMEQ_PPzZZ : sve_fp_3op_p_pd_cc<0b010, "fcmeq", int_aarch64_sve_fcmpeq, setoeq>;
+  defm FCMNE_PPzZZ : sve_fp_3op_p_pd_cc<0b011, "fcmne", int_aarch64_sve_fcmpne, setone>;
+  defm FCMUO_PPzZZ : sve_fp_3op_p_pd_cc<0b100, "fcmuo", int_aarch64_sve_fcmpuo, setuo>;
   defm FACGE_PPzZZ : sve_fp_3op_p_pd<0b101, "facge", int_aarch64_sve_facge>;
   defm FACGT_PPzZZ : sve_fp_3op_p_pd<0b111, "facgt", int_aarch64_sve_facgt>;
 
@@ -1197,6 +1197,18 @@ multiclass sve_prefetch<SDPatternOperator prefetch, ValueType PredTy, Instructio
   defm FRECPX_ZPmZ : sve_fp_2op_p_zd_HSD<0b01100, "frecpx", int_aarch64_sve_frecpx>;
   defm FSQRT_ZPmZ  : sve_fp_2op_p_zd_HSD<0b01101, "fsqrt",  int_aarch64_sve_fsqrt>;
 
+  let Predicates = [HasBF16, HasSVE] in {
+    def BFDOT_ZZZ    : sve_bfloat_dot<"bfdot">;
+    def BFDOT_ZZI    : sve_bfloat_dot_indexed<"bfdot">;
+    def BFMMLA_ZZZ  : sve_bfloat_matmul<"bfmmla">;
+    def BFMMLA_B_ZZZ  : sve_bfloat_matmul_longvecl<0b0, "bfmlalb">;
+    def BFMMLA_T_ZZZ  : sve_bfloat_matmul_longvecl<0b1, "bfmlalt">;
+    def BFMMLA_B_ZZI  : sve_bfloat_matmul_longvecl_idx<0b0, "bfmlalb">;
+    def BFMMLA_T_ZZI  : sve_bfloat_matmul_longvecl_idx<0b1, "bfmlalt">;
+    def BFCVT_ZPmZ   : sve_bfloat_convert<0b1, "bfcvt">;
+    def BFCVTNT_ZPmZ : sve_bfloat_convert<0b0, "bfcvtnt">;
+  }
+
   // InstAliases
   def : InstAlias<"mov $Zd, $Zn",
                   (ORR_ZZZ ZPR64:$Zd, ZPR64:$Zn, ZPR64:$Zn), 1>;
diff --git a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
index 65c089a1d37fc..ea7b87f0c7663 100644
--- a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
@@ -117,7 +117,7 @@ SDValue AArch64SelectionDAGInfo::EmitTargetCodeForSetTag(
 
   MachineFunction &MF = DAG.getMachineFunction();
   MachineMemOperand *BaseMemOperand = MF.getMachineMemOperand(
-      DstPtrInfo, MachineMemOperand::MOStore, ObjSize, 16);
+      DstPtrInfo, MachineMemOperand::MOStore, ObjSize, Align(16));
 
   bool UseSetTagRangeLoop =
       kSetTagLoopThreshold >= 0 && (int)ObjSize >= kSetTagLoopThreshold;
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h
index dca2b8b11bad3..e69404e6921a0 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.h
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h
@@ -76,6 +76,7 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
   bool HasV8_3aOps = false;
   bool HasV8_4aOps = false;
   bool HasV8_5aOps = false;
+  bool HasV8_6aOps = false;
 
   bool HasFPARMv8 = false;
   bool HasNEON = false;
@@ -144,6 +145,9 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
   bool HasMTE = false;
   bool HasTME = false;
 
+  // Armv8.6-A Extensions
+  bool HasBF16 = false;
+
   // Arm SVE2 extensions
   bool HasSVE2AES = false;
   bool HasSVE2SM4 = false;
@@ -360,7 +364,12 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
   }
   unsigned getCacheLineSize() const override { return CacheLineSize; }
   unsigned getPrefetchDistance() const override { return PrefetchDistance; }
-  unsigned getMinPrefetchStride() const override { return MinPrefetchStride; }
+  unsigned getMinPrefetchStride(unsigned NumMemAccesses,
+                                unsigned NumStridedMemAccesses,
+                                unsigned NumPrefetches,
+                                bool HasCall) const override {
+    return MinPrefetchStride;
+  }
   unsigned getMaxPrefetchIterationsAhead() const override {
     return MaxPrefetchIterationsAhead;
   }
@@ -403,6 +412,9 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
   bool hasSVE2SHA3() const { return HasSVE2SHA3; }
   bool hasSVE2BitPerm() const { return HasSVE2BitPerm; }
 
+  // Armv8.6-A Extensions
+  bool hasBF16() const { return HasBF16; }
+
   bool isLittleEndian() const { return IsLittle; }
 
   bool isTargetDarwin() const { return TargetTriple.isOSDarwin(); }
diff --git a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
index 578ffb4e42ba9..f50c6823ad107 100644
--- a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
+++ b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
@@ -2859,6 +2859,8 @@ static void setRequiredFeatureString(FeatureBitset FBS, std::string &Str) {
     Str += "ARMv8.4a";
   else if (FBS[AArch64::HasV8_5aOps])
     Str += "ARMv8.5a";
+  else if (FBS[AArch64::HasV8_6aOps])
+    Str += "ARMv8.6a";
   else {
     auto ext = std::find_if(std::begin(ExtensionMap),
       std::end(ExtensionMap),
@@ -5094,6 +5096,7 @@ static void ExpandCryptoAEK(AArch64::ArchKind ArchKind,
       break;
     case AArch64::ArchKind::ARMV8_4A:
     case AArch64::ArchKind::ARMV8_5A:
+    case AArch64::ArchKind::ARMV8_6A:
       RequestedExtensions.push_back("sm4");
       RequestedExtensions.push_back("sha3");
       RequestedExtensions.push_back("sha2");
@@ -5113,6 +5116,7 @@ static void ExpandCryptoAEK(AArch64::ArchKind ArchKind,
       break;
     case AArch64::ArchKind::ARMV8_4A:
     case AArch64::ArchKind::ARMV8_5A:
+    case AArch64::ArchKind::ARMV8_6A:
       RequestedExtensions.push_back("nosm4");
       RequestedExtensions.push_back("nosha3");
       RequestedExtensions.push_back("nosha2");
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
index 67a9e6d2cf1aa..47d115a867a0e 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
@@ -69,6 +69,11 @@ class AArch64AsmBackend : public MCAsmBackend {
         {"fixup_aarch64_pcrel_call26", 0, 26, PCRelFlagVal},
         {"fixup_aarch64_tlsdesc_call", 0, 0, 0}};
 
+    // Fixup kinds from .reloc directive are like R_AARCH64_NONE. They do not
+    // require any extra processing.
+    if (Kind >= FirstLiteralRelocationKind)
+      return MCAsmBackend::getFixupKindInfo(FK_NONE);
+
     if (Kind < FirstTargetFixupKind)
       return MCAsmBackend::getFixupKindInfo(Kind);
 
@@ -109,7 +114,6 @@ static unsigned getFixupKindNumBytes(unsigned Kind) {
   default:
     llvm_unreachable("Unknown fixup kind!");
 
-  case FK_NONE:
   case AArch64::fixup_aarch64_tlsdesc_call:
     return 0;
 
@@ -330,7 +334,6 @@ static uint64_t adjustFixupValue(const MCFixup &Fixup, const MCValue &Target,
     if (!valueFitsIntoFixupKind(Fixup.getTargetKind(), Value))
       Ctx.reportError(Fixup.getLoc(), "fixup value too large for data type!");
     LLVM_FALLTHROUGH;
-  case FK_NONE:
   case FK_SecRel_2:
   case FK_SecRel_4:
     return Value;
@@ -338,9 +341,17 @@ static uint64_t adjustFixupValue(const MCFixup &Fixup, const MCValue &Target,
 }
 
 Optional<MCFixupKind> AArch64AsmBackend::getFixupKind(StringRef Name) const {
-  if (TheTriple.isOSBinFormatELF() && Name == "R_AARCH64_NONE")
-    return FK_NONE;
-  return MCAsmBackend::getFixupKind(Name);
+  if (!TheTriple.isOSBinFormatELF())
+    return None;
+
+  unsigned Type = llvm::StringSwitch<unsigned>(Name)
+#define ELF_RELOC(X, Y)  .Case(#X, Y)
+#include "llvm/BinaryFormat/ELFRelocs/AArch64.def"
+#undef ELF_RELOC
+                      .Default(-1u);
+  if (Type == -1u)
+    return None;
+  return static_cast<MCFixupKind>(FirstLiteralRelocationKind + Type);
 }
 
 /// getFixupKindContainereSizeInBytes - The number of bytes of the
@@ -387,9 +398,12 @@ void AArch64AsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
                                    MutableArrayRef<char> Data, uint64_t Value,
                                    bool IsResolved,
                                    const MCSubtargetInfo *STI) const {
-  unsigned NumBytes = getFixupKindNumBytes(Fixup.getKind());
   if (!Value)
     return; // Doesn't change encoding.
+  unsigned Kind = Fixup.getKind();
+  if (Kind >= FirstLiteralRelocationKind)
+    return;
+  unsigned NumBytes = getFixupKindNumBytes(Kind);
   MCFixupKindInfo Info = getFixupKindInfo(Fixup.getKind());
   MCContext &Ctx = Asm.getContext();
   int64_t SignedValue = static_cast<int64_t>(Value);
@@ -475,7 +489,7 @@ bool AArch64AsmBackend::shouldForceRelocation(const MCAssembler &Asm,
                                               const MCFixup &Fixup,
                                               const MCValue &Target) {
   unsigned Kind = Fixup.getKind();
-  if (Kind == FK_NONE)
+  if (Kind >= FirstLiteralRelocationKind)
     return true;
 
   // The ADRP instruction adds some multiple of 0x1000 to the current PC &
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp
index 0fd1ca187be7f..b7422ce76f935 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp
@@ -106,6 +106,9 @@ unsigned AArch64ELFObjectWriter::getRelocType(MCContext &Ctx,
                                               const MCValue &Target,
                                               const MCFixup &Fixup,
                                               bool IsPCRel) const {
+  unsigned Kind = Fixup.getTargetKind();
+  if (Kind >= FirstLiteralRelocationKind)
+    return Kind - FirstLiteralRelocationKind;
   AArch64MCExpr::VariantKind RefKind =
       static_cast<AArch64MCExpr::VariantKind>(Target.getRefKind());
   AArch64MCExpr::VariantKind SymLoc = AArch64MCExpr::getSymbolLoc(RefKind);
@@ -120,7 +123,7 @@ unsigned AArch64ELFObjectWriter::getRelocType(MCContext &Ctx,
          "Should only be expression-level modifiers here");
 
   if (IsPCRel) {
-    switch (Fixup.getTargetKind()) {
+    switch (Kind) {
     case FK_Data_1:
       Ctx.reportError(Fixup.getLoc(), "1-byte data relocations not supported");
       return ELF::R_AARCH64_NONE;
@@ -185,8 +188,6 @@ unsigned AArch64ELFObjectWriter::getRelocType(MCContext &Ctx,
     if (IsILP32 && isNonILP32reloc(Fixup, RefKind, Ctx))
       return ELF::R_AARCH64_NONE;
     switch (Fixup.getTargetKind()) {
-    case FK_NONE:
-      return ELF::R_AARCH64_NONE;
     case FK_Data_1:
       Ctx.reportError(Fixup.getLoc(), "1-byte data relocations not supported");
       return ELF::R_AARCH64_NONE;
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp
index 06dfbaa302877..c64a40abb768a 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp
@@ -295,7 +295,7 @@ void AArch64InstPrinter::printInst(const MCInst *MI, uint64_t Address,
     return;
   }
 
-  if (!printAliasInstr(MI, STI, O))
+  if (!printAliasInstr(MI, Address, STI, O))
     printInstruction(MI, Address, STI, O);
 
   printAnnotation(O, Annot);
@@ -1347,7 +1347,8 @@ void AArch64InstPrinter::printVectorIndex(const MCInst *MI, unsigned OpNum,
   O << "[" << MI->getOperand(OpNum).getImm() << "]";
 }
 
-void AArch64InstPrinter::printAlignedLabel(const MCInst *MI, unsigned OpNum,
+void AArch64InstPrinter::printAlignedLabel(const MCInst *MI, uint64_t Address,
+                                           unsigned OpNum,
                                            const MCSubtargetInfo &STI,
                                            raw_ostream &O) {
   const MCOperand &Op = MI->getOperand(OpNum);
@@ -1362,10 +1363,9 @@ void AArch64InstPrinter::printAlignedLabel(const MCInst *MI, unsigned OpNum,
   // If the branch target is simply an address then print it in hex.
   const MCConstantExpr *BranchTarget =
       dyn_cast<MCConstantExpr>(MI->getOperand(OpNum).getExpr());
-  int64_t Address;
-  if (BranchTarget && BranchTarget->evaluateAsAbsolute(Address)) {
-    O << "0x";
-    O.write_hex(Address);
+  int64_t TargetAddress;
+  if (BranchTarget && BranchTarget->evaluateAsAbsolute(TargetAddress)) {
+    O << formatHex(TargetAddress);
   } else {
     // Otherwise, just print the expression.
     MI->getOperand(OpNum).getExpr()->print(O, &MAI);
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.h
index 62613e957778c..6da5f0e81c803 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.h
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.h
@@ -32,10 +32,10 @@ class AArch64InstPrinter : public MCInstPrinter {
   // Autogenerated by tblgen.
   virtual void printInstruction(const MCInst *MI, uint64_t Address,
                                 const MCSubtargetInfo &STI, raw_ostream &O);
-  virtual bool printAliasInstr(const MCInst *MI, const MCSubtargetInfo &STI,
-                               raw_ostream &O);
-  virtual void printCustomAliasOperand(const MCInst *MI, unsigned OpIdx,
-                                       unsigned PrintMethodIdx,
+  virtual bool printAliasInstr(const MCInst *MI, uint64_t Address,
+                               const MCSubtargetInfo &STI, raw_ostream &O);
+  virtual void printCustomAliasOperand(const MCInst *MI, uint64_t Address,
+                                       unsigned OpIdx, unsigned PrintMethodIdx,
                                        const MCSubtargetInfo &STI,
                                        raw_ostream &O);
 
@@ -100,7 +100,7 @@ class AArch64InstPrinter : public MCInstPrinter {
                      const MCSubtargetInfo &STI, raw_ostream &O);
   void printInverseCondCode(const MCInst *MI, unsigned OpNum,
                             const MCSubtargetInfo &STI, raw_ostream &O);
-  void printAlignedLabel(const MCInst *MI, unsigned OpNum,
+  void printAlignedLabel(const MCInst *MI, uint64_t Address, unsigned OpNum,
                          const MCSubtargetInfo &STI, raw_ostream &O);
   void printUImm12Offset(const MCInst *MI, unsigned OpNum, unsigned Scale,
                          raw_ostream &O);
@@ -205,10 +205,10 @@ class AArch64AppleInstPrinter : public AArch64InstPrinter {
 
   void printInstruction(const MCInst *MI, uint64_t Address,
                         const MCSubtargetInfo &STI, raw_ostream &O) override;
-  bool printAliasInstr(const MCInst *MI, const MCSubtargetInfo &STI,
-                       raw_ostream &O) override;
-  void printCustomAliasOperand(const MCInst *MI, unsigned OpIdx,
-                               unsigned PrintMethodIdx,
+  bool printAliasInstr(const MCInst *MI, uint64_t Address,
+                       const MCSubtargetInfo &STI, raw_ostream &O) override;
+  void printCustomAliasOperand(const MCInst *MI, uint64_t Address,
+                               unsigned OpIdx, unsigned PrintMethodIdx,
                                const MCSubtargetInfo &STI,
                                raw_ostream &O) override;
 
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 6b4924b8f2259..677d52a3a7f48 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -392,6 +392,11 @@ class SVE_1_Op_AllActive_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
 : Pat<(vtd (op vt1:$Op1)),
       (inst (IMPLICIT_DEF), (ptrue 31), $Op1)>;
 
+class SVE_2_Op_AllActive_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
+                             ValueType vt2, Instruction inst, Instruction ptrue>
+: Pat<(vtd (op vt1:$Op1, vt2:$Op2)),
+      (inst (ptrue 31), $Op1, $Op2)>;
+
 //
 // Pseudo -> Instruction mappings
 //
@@ -1358,13 +1363,22 @@ class sve_int_pred_log<bits<4> opc, string asm>
 
 }
 
-multiclass sve_int_pred_log<bits<4> opc, string asm, SDPatternOperator op> {
+multiclass sve_int_pred_log<bits<4> opc, string asm, SDPatternOperator op,
+                            SDPatternOperator op_nopred = null_frag> {
   def NAME : sve_int_pred_log<opc, asm>;
 
   def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>;
   def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, nxv8i1, !cast<Instruction>(NAME)>;
   def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, nxv4i1, !cast<Instruction>(NAME)>;
   def : SVE_3_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, nxv2i1, !cast<Instruction>(NAME)>;
+  def : SVE_2_Op_AllActive_Pat<nxv16i1, op_nopred, nxv16i1, nxv16i1,
+                               !cast<Instruction>(NAME), PTRUE_B>;
+  def : SVE_2_Op_AllActive_Pat<nxv8i1, op_nopred, nxv8i1, nxv8i1,
+                               !cast<Instruction>(NAME), PTRUE_H>;
+  def : SVE_2_Op_AllActive_Pat<nxv4i1, op_nopred, nxv4i1, nxv4i1,
+                               !cast<Instruction>(NAME), PTRUE_S>;
+  def : SVE_2_Op_AllActive_Pat<nxv2i1, op_nopred, nxv2i1, nxv2i1,
+                               !cast<Instruction>(NAME), PTRUE_D>;
 }
 
 
@@ -4518,6 +4532,22 @@ multiclass sve_fp_3op_p_pd<bits<3> opc, string asm, SDPatternOperator op> {
   def : SVE_3_Op_Pat<nxv2i1, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;
 }
 
+multiclass sve_fp_3op_p_pd_cc<bits<3> opc, string asm, SDPatternOperator op,
+                              SDPatternOperator op_nopred>
+: sve_fp_3op_p_pd<opc, asm, op> {
+  def : SVE_2_Op_AllActive_Pat<nxv8i1, op_nopred, nxv8f16, nxv8f16,
+                               !cast<Instruction>(NAME # _H), PTRUE_H>;
+  def : SVE_2_Op_AllActive_Pat<nxv4i1, op_nopred, nxv4f16, nxv4f16,
+                               !cast<Instruction>(NAME # _H), PTRUE_S>;
+  def : SVE_2_Op_AllActive_Pat<nxv2i1, op_nopred, nxv2f16, nxv2f16,
+                               !cast<Instruction>(NAME # _H), PTRUE_D>;
+  def : SVE_2_Op_AllActive_Pat<nxv4i1, op_nopred, nxv4f32, nxv4f32,
+                               !cast<Instruction>(NAME # _S), PTRUE_S>;
+  def : SVE_2_Op_AllActive_Pat<nxv2i1, op_nopred, nxv2f32, nxv2f32,
+                               !cast<Instruction>(NAME # _S), PTRUE_D>;
+  def : SVE_2_Op_AllActive_Pat<nxv2i1, op_nopred, nxv2f64, nxv2f64,
+                               !cast<Instruction>(NAME # _D), PTRUE_D>;
+}
 
 //===----------------------------------------------------------------------===//
 // SVE Floating Point Compare - with Zero Group
@@ -7394,6 +7424,96 @@ multiclass sve2_crypto_unary_op<bit opc, string asm, SDPatternOperator op> {
   def : SVE_1_Op_Pat<nxv16i8, op, nxv16i8, !cast<Instruction>(NAME)>;
 }
 
+//===----------------------------------------------------------------------===//
+// SVE BFloat16 Group
+//===----------------------------------------------------------------------===//
+
+class sve_bfloat_dot_base<bits<2> opc, string asm, string ops, dag iops>
+: I<(outs ZPR32:$Zda), iops, asm, ops, "", []>, Sched<[]> {
+  bits<5> Zda;
+  bits<5> Zn;
+  let Inst{31-21} = 0b01100100011;
+  let Inst{15-14} = opc;
+  let Inst{13-10} = 0b0000;
+  let Inst{9-5}   = Zn;
+  let Inst{4-0}   = Zda;
+
+  let Constraints = "$Zda = $_Zda";
+  let DestructiveInstType = DestructiveOther;
+  let ElementSize = ElementSizeH;
+}
+
+class sve_bfloat_dot<string asm>
+: sve_bfloat_dot_base<0b10, asm, "\t$Zda, $Zn, $Zm",
+  (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm)> {
+  bits<5> Zm;
+  let Inst{20-16} = Zm;
+}
+
+class sve_bfloat_dot_indexed<string asm>
+: sve_bfloat_dot_base<0b01, asm, "\t$Zda, $Zn, $Zm$iop",
+  (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexS:$iop)> {
+  bits<2> iop;
+  bits<3> Zm;
+  let Inst{20-19} = iop;
+  let Inst{18-16} = Zm;
+}
+
+class sve_bfloat_matmul<string asm>
+: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm),
+  asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> {
+  bits<5> Zm;
+  bits<5> Zda;
+  bits<5> Zn;
+  let Inst{31-21} = 0b01100100011;
+  let Inst{20-16} = Zm;
+  let Inst{15-10} = 0b111001;
+  let Inst{9-5}   = Zn;
+  let Inst{4-0}   = Zda;
+
+  let Constraints = "$Zda = $_Zda";
+  let DestructiveInstType = DestructiveOther;
+  let ElementSize = ElementSizeH;
+}
+
+class sve_bfloat_matmul_longvecl<bit BT, string asm>
+: sve_bfloat_matmul<asm> {
+  let Inst{23}    = 0b1;
+  let Inst{14-13} = 0b00;
+  let Inst{10}    = BT;
+}
+
+class sve_bfloat_matmul_longvecl_idx<bit BT, string asm>
+: sve_bfloat_dot_base<0b01, asm, "\t$Zda, $Zn, $Zm$iop",
+  (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexH:$iop)> {
+  bits<3> iop;
+  bits<3> Zm;
+  let Inst{23}    = 0b1;
+  let Inst{20-19} = iop{2-1};
+  let Inst{18-16} = Zm;
+  let Inst{11}    = iop{0};
+  let Inst{10}    = BT;
+}
+
+class sve_bfloat_convert<bit N, string asm>
+: I<(outs ZPR16:$Zd), (ins ZPR16:$_Zd, PPR3bAny:$Pg, ZPR32:$Zn),
+  asm, "\t$Zd, $Pg/m, $Zn", "", []>, Sched<[]> {
+  bits<5> Zd;
+  bits<3> Pg;
+  bits<5> Zn;
+  let Inst{31-25} = 0b0110010;
+  let Inst{24}    = N;
+  let Inst{23-13} = 0b10001010101;
+  let Inst{12-10} = Pg;
+  let Inst{9-5}   = Zn;
+  let Inst{4-0}   = Zd;
+
+  let Constraints = "$Zd = $_Zd";
+  let DestructiveInstType = DestructiveOther;
+  let hasSideEffects = 1;
+  let ElementSize = ElementSizeS;
+}
+
 /// Addressing modes
 def am_sve_indexed_s4 :ComplexPattern<i64, 2, "SelectAddrModeIndexedSVE<-8,7>", [], [SDNPWantRoot]>;
 def am_sve_indexed_s6 :ComplexPattern<i64, 2, "SelectAddrModeIndexedSVE<-32,31>", [], [SDNPWantRoot]>;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index bae5e4059c37e..ded1107423277 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -61,9 +61,9 @@ FunctionPass *createSIMemoryLegalizerPass();
 FunctionPass *createSIInsertWaitcntsPass();
 FunctionPass *createSIPreAllocateWWMRegsPass();
 FunctionPass *createSIFormMemoryClausesPass();
+
 FunctionPass *createSIPostRABundlerPass();
-FunctionPass *createAMDGPUSimplifyLibCallsPass(const TargetOptions &,
-                                               const TargetMachine *);
+FunctionPass *createAMDGPUSimplifyLibCallsPass(const TargetMachine *);
 FunctionPass *createAMDGPUUseNativeCallsPass();
 FunctionPass *createAMDGPUCodeGenPreparePass();
 FunctionPass *createAMDGPUMachineCFGStructurizerPass();
@@ -166,6 +166,9 @@ extern char &SILowerControlFlowID;
 void initializeSIRemoveShortExecBranchesPass(PassRegistry &);
 extern char &SIRemoveShortExecBranchesID;
 
+void initializeSIPreEmitPeepholePass(PassRegistry &);
+extern char &SIPreEmitPeepholeID;
+
 void initializeSIInsertSkipsPass(PassRegistry &);
 extern char &SIInsertSkipsPassID;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index c9db0a6032231..1f1065749c35e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -494,30 +494,6 @@ def FeatureNoDataDepHazard : SubtargetFeature<"no-data-dep-hazard",
 // Subtarget Features (options and debugging)
 //===------------------------------------------------------------===//
 
-// Denormal handling for fp64 and fp16 is controlled by the same
-// config register when fp16 supported.
-// TODO: Do we need a separate f16 setting when not legal?
-def FeatureFP64FP16Denormals : SubtargetFeature<"fp64-fp16-denormals",
-  "FP64FP16Denormals",
-  "true",
-  "Enable double and half precision denormal handling",
-  [FeatureFP64]
->;
-
-def FeatureFP64Denormals : SubtargetFeature<"fp64-denormals",
-  "FP64FP16Denormals",
-  "true",
-  "Enable double and half precision denormal handling",
-  [FeatureFP64, FeatureFP64FP16Denormals]
->;
-
-def FeatureFP16Denormals : SubtargetFeature<"fp16-denormals",
-  "FP64FP16Denormals",
-  "true",
-  "Enable half precision denormal handling",
-  [FeatureFP64FP16Denormals]
->;
-
 def FeatureFPExceptions : SubtargetFeature<"fp-exceptions",
   "FPExceptions",
   "true",
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp
index ff2bda6bed533..22947544ac07f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp
@@ -71,6 +71,13 @@ void AMDGPUAlwaysInline::recursivelyVisitUsers(
     if (Instruction *I = dyn_cast<Instruction>(U)) {
       Function *F = I->getParent()->getParent();
       if (!AMDGPU::isEntryFunctionCC(F->getCallingConv())) {
+        // FIXME: This is a horrible hack. We should always respect noinline,
+        // and just let us hit the error when we can't handle this.
+        //
+        // Unfortunately, clang adds noinline to all functions at -O0. We have
+        // to override this here. until that's fixed.
+        F->removeFnAttr(Attribute::NoInline);
+
         FuncsToAlwaysInline.insert(F);
         Stack.push_back(F);
       }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
index 59aa0ea98aa79..46ffc77d8edd4 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
@@ -447,9 +447,8 @@ void AMDGPUAtomicOptimizer::optimizeAtomic(Instruction &I,
   // We need to know how many lanes are active within the wavefront, and we do
   // this by doing a ballot of active lanes.
   Type *const WaveTy = B.getIntNTy(ST->getWavefrontSize());
-  CallInst *const Ballot = B.CreateIntrinsic(
-      Intrinsic::amdgcn_icmp, {WaveTy, B.getInt32Ty()},
-      {B.getInt32(1), B.getInt32(0), B.getInt32(CmpInst::ICMP_NE)});
+  CallInst *const Ballot =
+      B.CreateIntrinsic(Intrinsic::amdgcn_ballot, WaveTy, B.getTrue());
 
   // We need to know how many lanes are active within the wavefront that are
   // below us. If we counted each lane linearly starting from 0, a lane is
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
index 86586b3009ad0..671110ec96c47 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
@@ -131,12 +131,11 @@ struct IncomingArgHandler : public CallLowering::ValueHandler {
   void assignValueToAddress(Register ValVReg, Register Addr, uint64_t Size,
                             MachinePointerInfo &MPO, CCValAssign &VA) override {
     MachineFunction &MF = MIRBuilder.getMF();
-    unsigned Align = inferAlignmentFromPtrInfo(MF, MPO);
 
     // FIXME: Get alignment
     auto MMO = MF.getMachineMemOperand(
         MPO, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, Size,
-        Align);
+        inferAlignFromPtrInfo(MF, MPO));
     MIRBuilder.buildLoad(ValVReg, Addr, *MMO);
   }
 
@@ -226,6 +225,19 @@ void AMDGPUCallLowering::splitToValueTypes(
     MVT RegVT = TLI.getRegisterTypeForCallingConv(Ctx, CallConv, VT);
 
     if (NumParts == 1) {
+      // Fixup EVTs to an MVT.
+      //
+      // FIXME: This is pretty hacky. Why do we have to split the type
+      // legalization logic between here and handleAssignments?
+      if (OrigArgIdx != AttributeList::ReturnIndex && VT != RegVT) {
+        assert(VT.getSizeInBits() < 32 &&
+               "unexpected illegal type");
+        Ty = Type::getInt32Ty(Ctx);
+        Register OrigReg = Reg;
+        Reg = B.getMRI()->createGenericVirtualRegister(LLT::scalar(32));
+        B.buildTrunc(OrigReg, Reg);
+      }
+
       // No splitting to do, but we want to replace the original type (e.g. [1 x
       // double] -> double).
       SplitArgs.emplace_back(Reg, Ty, OrigArg.Flags, OrigArg.IsFixed);
@@ -405,9 +417,8 @@ Register AMDGPUCallLowering::lowerParameterPtr(MachineIRBuilder &B,
   return B.buildPtrAdd(PtrType, KernArgSegmentVReg, OffsetReg).getReg(0);
 }
 
-void AMDGPUCallLowering::lowerParameter(MachineIRBuilder &B,
-                                        Type *ParamTy, uint64_t Offset,
-                                        unsigned Align,
+void AMDGPUCallLowering::lowerParameter(MachineIRBuilder &B, Type *ParamTy,
+                                        uint64_t Offset, Align Alignment,
                                         Register DstReg) const {
   MachineFunction &MF = B.getMF();
   const Function &F = MF.getFunction();
@@ -416,11 +427,11 @@ void AMDGPUCallLowering::lowerParameter(MachineIRBuilder &B,
   unsigned TypeSize = DL.getTypeStoreSize(ParamTy);
   Register PtrReg = lowerParameterPtr(B, ParamTy, Offset);
 
-  MachineMemOperand *MMO =
-      MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad |
-                                       MachineMemOperand::MODereferenceable |
-                                       MachineMemOperand::MOInvariant,
-                                       TypeSize, Align);
+  MachineMemOperand *MMO = MF.getMachineMemOperand(
+      PtrInfo,
+      MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
+          MachineMemOperand::MOInvariant,
+      TypeSize, Alignment);
 
   B.buildLoad(DstReg, PtrReg, *MMO);
 }
@@ -495,7 +506,7 @@ bool AMDGPUCallLowering::lowerFormalArgumentsKernel(
   allocateHSAUserSGPRs(CCInfo, B, MF, *TRI, *Info);
 
   unsigned i = 0;
-  const unsigned KernArgBaseAlign = 16;
+  const Align KernArgBaseAlign(16);
   const unsigned BaseOffset = Subtarget->getExplicitKernelArgOffset(F);
   uint64_t ExplicitArgOffset = 0;
 
@@ -516,9 +527,9 @@ bool AMDGPUCallLowering::lowerFormalArgumentsKernel(
       OrigArgRegs.size() == 1
       ? OrigArgRegs[0]
       : MRI.createGenericVirtualRegister(getLLTForType(*ArgTy, DL));
-    unsigned Align = MinAlign(KernArgBaseAlign, ArgOffset);
+    Align Alignment = commonAlignment(KernArgBaseAlign, ArgOffset);
     ArgOffset = alignTo(ArgOffset, DL.getABITypeAlignment(ArgTy));
-    lowerParameter(B, ArgTy, ArgOffset, Align, ArgReg);
+    lowerParameter(B, ArgTy, ArgOffset, Alignment, ArgReg);
     if (OrigArgRegs.size() > 1)
       unpackRegs(OrigArgRegs, ArgReg, ArgTy, B);
     ++i;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.h
index 3651dd40bc9f5..446619d1502ee 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.h
@@ -27,7 +27,7 @@ class AMDGPUCallLowering: public CallLowering {
                              uint64_t Offset) const;
 
   void lowerParameter(MachineIRBuilder &B, Type *ParamTy, uint64_t Offset,
-                      unsigned Align, Register DstReg) const;
+                      Align Alignment, Register DstReg) const;
 
   /// A function of this type is used to perform value split action.
   using SplitArgTy = std::function<void(ArrayRef<Register>, Register, LLT, LLT, int)>;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
index 60fab532c88a1..da17d94f3265b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
@@ -15,6 +15,7 @@
 #include "AMDGPU.h"
 #include "AMDGPUSubtarget.h"
 #include "AMDGPUTargetMachine.h"
+#include "llvm/ADT/FloatingPointMode.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/ConstantFolding.h"
@@ -1387,7 +1388,9 @@ bool AMDGPUCodeGenPrepare::runOnFunction(Function &F) {
   DT = DTWP ? &DTWP->getDomTree() : nullptr;
 
   HasUnsafeFPMath = hasUnsafeFPMath(F);
-  HasFP32Denormals = ST->hasFP32Denormals(F);
+
+  AMDGPU::SIModeRegisterDefaults Mode(F);
+  HasFP32Denormals = Mode.allFP32Denormals();
 
   bool MadeChange = false;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCombine.td b/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
index 4063bb1fca2c2..ff8a31dee3370 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
@@ -20,6 +20,13 @@ def fcmp_select_to_fmin_fmax_legacy : GICombineRule<
   (apply [{ applySelectFCmpToFMinToFMaxLegacy(*${select}, ${matchinfo}); }])>;
 
 
+def uchar_to_float : GICombineRule<
+  (defs root:$itofp),
+  (match (wip_match_opcode G_UITOFP, G_SITOFP):$itofp,
+         [{ return matchUCharToFloat(*${itofp}, MRI, *MF, Helper); }]),
+  (apply [{ applyUCharToFloat(*${itofp}); }])>;
+
+
 // Combines which should only apply on SI/VI
 def gfx6gfx7_combines : GICombineGroup<[fcmp_select_to_fmin_fmax_legacy]>;
 
@@ -32,6 +39,6 @@ def AMDGPUPreLegalizerCombinerHelper: GICombinerHelper<
 
 def AMDGPUPostLegalizerCombinerHelper: GICombinerHelper<
   "AMDGPUGenPostLegalizerCombinerHelper", [all_combines,
-                                           gfx6gfx7_combines]> {
+                                           gfx6gfx7_combines, uchar_to_float]> {
   let DisableRuleOption = "amdgpupostlegalizercombiner-disable-rule";
 }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUFeatures.td b/llvm/lib/Target/AMDGPU/AMDGPUFeatures.td
index ea3952c316e4d..6ca896c6de08e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUFeatures.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUFeatures.td
@@ -18,15 +18,6 @@ def FeatureFMA : SubtargetFeature<"fmaf",
   "Enable single precision FMA (not as fast as mul+add, but fused)"
 >;
 
-// Some instructions do not support denormals despite this flag. Using
-// fp32 denormals also causes instructions to run at the double
-// precision rate for the device.
-def FeatureFP32Denormals : SubtargetFeature<"fp32-denormals",
-  "FP32Denormals",
-  "true",
-  "Enable single precision denormal handling"
->;
-
 class SubtargetFeatureLocalMemorySize <int Value> : SubtargetFeature<
   "localmemorysize"#Value,
   "LocalMemorySize",
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
index faf007f7c639d..3d6a2c707765a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
@@ -153,6 +153,11 @@ def : GINodeEquiv<G_AMDGPU_FMIN_LEGACY, AMDGPUfmin_legacy>;
 def : GINodeEquiv<G_AMDGPU_FMAX_LEGACY, AMDGPUfmax_legacy>;
 def : GINodeEquiv<G_AMDGPU_RCP_IFLAG, AMDGPUrcp_iflag>;
 
+def : GINodeEquiv<G_AMDGPU_CVT_F32_UBYTE0, AMDGPUcvt_f32_ubyte0>;
+def : GINodeEquiv<G_AMDGPU_CVT_F32_UBYTE1, AMDGPUcvt_f32_ubyte1>;
+def : GINodeEquiv<G_AMDGPU_CVT_F32_UBYTE2, AMDGPUcvt_f32_ubyte2>;
+def : GINodeEquiv<G_AMDGPU_CVT_F32_UBYTE3, AMDGPUcvt_f32_ubyte3>;
+
 def : GINodeEquiv<G_AMDGPU_ATOMIC_CMPXCHG, AMDGPUatomic_cmp_swap>;
 def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD, SIbuffer_load>;
 def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD_USHORT, SIbuffer_load_ushort>;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h
index e585608599590..766750758efc2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUGLOBALISELUTILS_H
 #define LLVM_LIB_TARGET_AMDGPU_AMDGPUGLOBALISELUTILS_H
 
+#include "AMDGPUInstrInfo.h"
 #include "llvm/CodeGen/Register.h"
 #include <tuple>
 
@@ -24,6 +25,37 @@ std::tuple<Register, unsigned, MachineInstr *>
 getBaseWithConstantOffset(MachineRegisterInfo &MRI, Register Reg);
 
 bool isLegalVOP3PShuffleMask(ArrayRef<int> Mask);
+
+/// Return number of address arguments, and the number of gradients for an image
+/// intrinsic.
+inline std::pair<int, int>
+getImageNumVAddr(const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr,
+                 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode) {
+  const AMDGPU::MIMGDimInfo *DimInfo
+    = AMDGPU::getMIMGDimInfo(ImageDimIntr->Dim);
+
+  int NumGradients = BaseOpcode->Gradients ? DimInfo->NumGradients : 0;
+  int NumCoords = BaseOpcode->Coordinates ? DimInfo->NumCoords : 0;
+  int NumLCM = BaseOpcode->LodOrClampOrMip ? 1 : 0;
+  int NumVAddr = BaseOpcode->NumExtraArgs + NumGradients + NumCoords + NumLCM;
+  return {NumVAddr, NumGradients};
+}
+
+/// Return index of dmask in an gMIR image intrinsic
+inline int getDMaskIdx(const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode,
+                       int NumDefs) {
+  assert(!BaseOpcode->Atomic);
+  return NumDefs + 1 + (BaseOpcode->Store ? 1 : 0);
+}
+
+/// Return first address operand index in a gMIR image intrinsic.
+inline int getImageVAddrIdxBegin(const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode,
+                                 int NumDefs) {
+  if (BaseOpcode->Atomic)
+    return NumDefs + 1 + (BaseOpcode->AtomicX2 ? 2 : 1);
+  return getDMaskIdx(BaseOpcode, NumDefs) + 1;
+}
+
 }
 }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 269434d31e21d..eadd4bcaab170 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -402,7 +402,7 @@ bool AMDGPUDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
   }
 #endif
   Subtarget = &MF.getSubtarget<GCNSubtarget>();
-  Mode = AMDGPU::SIModeRegisterDefaults(MF.getFunction(), *Subtarget);
+  Mode = AMDGPU::SIModeRegisterDefaults(MF.getFunction());
   return SelectionDAGISel::runOnMachineFunction(MF);
 }
 
@@ -1766,30 +1766,43 @@ bool AMDGPUDAGToDAGISel::SelectFlatAtomicSigned(SDNode *N,
 
 bool AMDGPUDAGToDAGISel::SelectSMRDOffset(SDValue ByteOffsetNode,
                                           SDValue &Offset, bool &Imm) const {
-
-  // FIXME: Handle non-constant offsets.
   ConstantSDNode *C = dyn_cast<ConstantSDNode>(ByteOffsetNode);
-  if (!C)
+  if (!C) {
+    if (ByteOffsetNode.getValueType().isScalarInteger() &&
+        ByteOffsetNode.getValueType().getSizeInBits() == 32) {
+      Offset = ByteOffsetNode;
+      Imm = false;
+      return true;
+    }
+    if (ByteOffsetNode.getOpcode() == ISD::ZERO_EXTEND) {
+      if (ByteOffsetNode.getOperand(0).getValueType().getSizeInBits() == 32) {
+        Offset = ByteOffsetNode.getOperand(0);
+        Imm = false;
+        return true;
+      }
+    }
     return false;
+  }
 
   SDLoc SL(ByteOffsetNode);
-  GCNSubtarget::Generation Gen = Subtarget->getGeneration();
-  uint64_t ByteOffset = C->getZExtValue();
+  // GFX9 and GFX10 have signed byte immediate offsets.
+  int64_t ByteOffset = C->getSExtValue();
   Optional<int64_t> EncodedOffset =
-      AMDGPU::getSMRDEncodedOffset(*Subtarget, ByteOffset);
+      AMDGPU::getSMRDEncodedOffset(*Subtarget, ByteOffset, false);
   if (EncodedOffset) {
     Offset = CurDAG->getTargetConstant(*EncodedOffset, SL, MVT::i32);
     Imm = true;
     return true;
   }
 
-  if (Gen == AMDGPUSubtarget::SEA_ISLANDS) {
-    EncodedOffset =
-        AMDGPU::getSMRDEncodedLiteralOffset32(*Subtarget, ByteOffset);
-    if (EncodedOffset) {
-      Offset = CurDAG->getTargetConstant(*EncodedOffset, SL, MVT::i32);
-      return true;
-    }
+  // SGPR and literal offsets are unsigned.
+  if (ByteOffset < 0)
+    return false;
+
+  EncodedOffset = AMDGPU::getSMRDEncodedLiteralOffset32(*Subtarget, ByteOffset);
+  if (EncodedOffset) {
+    Offset = CurDAG->getTargetConstant(*EncodedOffset, SL, MVT::i32);
+    return true;
   }
 
   if (!isUInt<32>(ByteOffset) && !isInt<32>(ByteOffset))
@@ -1835,7 +1848,8 @@ bool AMDGPUDAGToDAGISel::SelectSMRD(SDValue Addr, SDValue &SBase,
   // wraparound, because s_load instructions perform the addition in 64 bits.
   if ((Addr.getValueType() != MVT::i32 ||
        Addr->getFlags().hasNoUnsignedWrap()) &&
-      CurDAG->isBaseWithConstantOffset(Addr)) {
+      (CurDAG->isBaseWithConstantOffset(Addr) ||
+       Addr.getOpcode() == ISD::ADD)) {
     SDValue N0 = Addr.getOperand(0);
     SDValue N1 = Addr.getOperand(1);
 
@@ -1878,8 +1892,9 @@ bool AMDGPUDAGToDAGISel::SelectSMRDSgpr(SDValue Addr, SDValue &SBase,
 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm(SDValue Addr,
                                              SDValue &Offset) const {
   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr)) {
-    if (auto Imm = AMDGPU::getSMRDEncodedOffset(*Subtarget,
-                                                C->getZExtValue())) {
+    // The immediate offset for S_BUFFER instructions is unsigned.
+    if (auto Imm =
+            AMDGPU::getSMRDEncodedOffset(*Subtarget, C->getZExtValue(), true)) {
       Offset = CurDAG->getTargetConstant(*Imm, SDLoc(Addr), MVT::i32);
       return true;
     }
@@ -2084,7 +2099,7 @@ void AMDGPUDAGToDAGISel::SelectBRCOND(SDNode *N) {
 
   bool UseSCCBr = isCBranchSCC(N) && isUniformBr(N);
   unsigned BrOp = UseSCCBr ? AMDGPU::S_CBRANCH_SCC1 : AMDGPU::S_CBRANCH_VCCNZ;
-  unsigned CondReg = UseSCCBr ? (unsigned)AMDGPU::SCC : TRI->getVCC();
+  Register CondReg = UseSCCBr ? AMDGPU::SCC : TRI->getVCC();
   SDLoc SL(N);
 
   if (!UseSCCBr) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index bdf5c3b036635..51f4132b10962 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -1644,9 +1644,10 @@ SDValue AMDGPUTargetLowering::LowerDIVREM24(SDValue Op, SelectionDAG &DAG,
   const AMDGPUMachineFunction *MFI = MF.getInfo<AMDGPUMachineFunction>();
 
   // float fr = mad(fqneg, fb, fa);
-  unsigned OpCode = MFI->getMode().allFP32Denormals() ?
-                    (unsigned)AMDGPUISD::FMAD_FTZ :
-                    (unsigned)ISD::FMAD;
+  unsigned OpCode = !MFI->getMode().allFP32Denormals() ?
+                    (unsigned)ISD::FMAD :
+                    (unsigned)AMDGPUISD::FMAD_FTZ;
+
   SDValue fr = DAG.getNode(OpCode, DL, FltVT, fqneg, fb, fa);
 
   // int iq = (int)fq;
@@ -1729,9 +1730,10 @@ void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op,
     const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
 
     // Compute denominator reciprocal.
-    unsigned FMAD = MFI->getMode().allFP32Denormals() ?
-                    (unsigned)AMDGPUISD::FMAD_FTZ :
-                    (unsigned)ISD::FMAD;
+    unsigned FMAD = !MFI->getMode().allFP32Denormals() ?
+                    (unsigned)ISD::FMAD :
+                    (unsigned)AMDGPUISD::FMAD_FTZ;
+
 
     SDValue Cvt_Lo = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Lo);
     SDValue Cvt_Hi = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Hi);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.h
index 698189e14c21e..61b78acad3f4b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.h
@@ -55,6 +55,9 @@ struct ImageDimIntrinsicInfo {
 };
 const ImageDimIntrinsicInfo *getImageDimIntrinsicInfo(unsigned Intr);
 
+const ImageDimIntrinsicInfo *getImageDimInstrinsicByBaseOpcode(unsigned BaseOpcode,
+                                                               unsigned Dim);
+
 } // end AMDGPU namespace
 } // End llvm namespace
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index f3cfbc1061ff7..088ce885a7042 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -737,7 +737,7 @@ bool AMDGPUInstructionSelector::selectG_INSERT(MachineInstr &I) const {
   // Deal with weird cases where the class only partially supports the subreg
   // index.
   Src0RC = TRI.getSubClassWithSubReg(Src0RC, SubReg);
-  if (!Src0RC)
+  if (!Src0RC || !Src1RC)
     return false;
 
   if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI) ||
@@ -1185,11 +1185,282 @@ bool AMDGPUInstructionSelector::selectDSAppendConsume(MachineInstr &MI,
     .addImm(Offset)
     .addImm(IsGDS ? -1 : 0)
     .cloneMemRefs(MI);
-
   MI.eraseFromParent();
   return true;
 }
 
+static bool parseTexFail(uint64_t TexFailCtrl, bool &TFE, bool &LWE,
+                         bool &IsTexFail) {
+  if (TexFailCtrl)
+    IsTexFail = true;
+
+  TFE = (TexFailCtrl & 0x1) ? 1 : 0;
+  TexFailCtrl &= ~(uint64_t)0x1;
+  LWE = (TexFailCtrl & 0x2) ? 1 : 0;
+  TexFailCtrl &= ~(uint64_t)0x2;
+
+  return TexFailCtrl == 0;
+}
+
+static bool parseCachePolicy(uint64_t Value,
+                             bool *GLC, bool *SLC, bool *DLC) {
+  if (GLC) {
+    *GLC = (Value & 0x1) ? 1 : 0;
+    Value &= ~(uint64_t)0x1;
+  }
+  if (SLC) {
+    *SLC = (Value & 0x2) ? 1 : 0;
+    Value &= ~(uint64_t)0x2;
+  }
+  if (DLC) {
+    *DLC = (Value & 0x4) ? 1 : 0;
+    Value &= ~(uint64_t)0x4;
+  }
+
+  return Value == 0;
+}
+
+bool AMDGPUInstructionSelector::selectImageIntrinsic(
+  MachineInstr &MI, const AMDGPU::ImageDimIntrinsicInfo *Intr) const {
+  MachineBasicBlock *MBB = MI.getParent();
+  const DebugLoc &DL = MI.getDebugLoc();
+
+  const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
+    AMDGPU::getMIMGBaseOpcodeInfo(Intr->BaseOpcode);
+
+  const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfo(Intr->Dim);
+  const AMDGPU::MIMGLZMappingInfo *LZMappingInfo =
+      AMDGPU::getMIMGLZMappingInfo(Intr->BaseOpcode);
+  const AMDGPU::MIMGMIPMappingInfo *MIPMappingInfo =
+      AMDGPU::getMIMGMIPMappingInfo(Intr->BaseOpcode);
+  unsigned IntrOpcode = Intr->BaseOpcode;
+  const bool IsGFX10 = STI.getGeneration() >= AMDGPUSubtarget::GFX10;
+
+  const LLT S16 = LLT::scalar(16);
+  const int VAddrIdx = getImageVAddrIdxBegin(BaseOpcode,
+                                             MI.getNumExplicitDefs());
+  int NumVAddr, NumGradients;
+  std::tie(NumVAddr, NumGradients) = getImageNumVAddr(Intr, BaseOpcode);
+
+  const LLT AddrTy = MRI->getType(MI.getOperand(VAddrIdx).getReg());
+  const bool IsA16 = AddrTy.getScalarType() == S16;
+
+  Register VDataIn, VDataOut;
+  LLT VDataTy;
+  int NumVDataDwords = -1;
+  bool IsD16 = false;
+
+  // XXX - Can we just get the second to last argument for ctrl?
+  unsigned CtrlIdx; // Index of texfailctrl argument
+  bool Unorm;
+  if (!BaseOpcode->Sampler) {
+    Unorm = true;
+    CtrlIdx = VAddrIdx + NumVAddr + 1;
+  } else {
+    Unorm = MI.getOperand(VAddrIdx + NumVAddr + 2).getImm() != 0;
+    CtrlIdx = VAddrIdx + NumVAddr + 3;
+  }
+
+  bool TFE;
+  bool LWE;
+  bool IsTexFail = false;
+  if (!parseTexFail(MI.getOperand(CtrlIdx).getImm(), TFE, LWE, IsTexFail))
+    return false;
+
+  unsigned DMask = 0;
+  unsigned DMaskLanes = 0;
+
+  if (BaseOpcode->Atomic) {
+    VDataOut = MI.getOperand(0).getReg();
+    VDataIn = MI.getOperand(2).getReg();
+    LLT Ty = MRI->getType(VDataIn);
+
+    // Be careful to allow atomic swap on 16-bit element vectors.
+    const bool Is64Bit = BaseOpcode->AtomicX2 ?
+      Ty.getSizeInBits() == 128 :
+      Ty.getSizeInBits() == 64;
+
+    if (BaseOpcode->AtomicX2) {
+      assert(MI.getOperand(3).getReg() == AMDGPU::NoRegister);
+
+      DMask = Is64Bit ? 0xf : 0x3;
+      NumVDataDwords = Is64Bit ? 4 : 2;
+    } else {
+      DMask = Is64Bit ? 0x3 : 0x1;
+      NumVDataDwords = Is64Bit ? 2 : 1;
+    }
+  } else {
+    const int DMaskIdx = 2; // Input/output + intrinsic ID.
+
+    DMask = MI.getOperand(DMaskIdx).getImm();
+    DMaskLanes = BaseOpcode->Gather4 ? 4 : countPopulation(DMask);
+
+    if (BaseOpcode->Store) {
+      VDataIn = MI.getOperand(1).getReg();
+      VDataTy = MRI->getType(VDataIn);
+      NumVDataDwords = (VDataTy.getSizeInBits() + 31) / 32;
+    } else {
+      VDataOut = MI.getOperand(0).getReg();
+      VDataTy = MRI->getType(VDataOut);
+      NumVDataDwords = DMaskLanes;
+
+      // One memoperand is mandatory, except for getresinfo.
+      // FIXME: Check this in verifier.
+      if (!MI.memoperands_empty()) {
+        const MachineMemOperand *MMO = *MI.memoperands_begin();
+
+        // Infer d16 from the memory size, as the register type will be mangled by
+        // unpacked subtargets, or by TFE.
+        IsD16 = ((8 * MMO->getSize()) / DMaskLanes) < 32;
+
+        if (IsD16 && !STI.hasUnpackedD16VMem())
+          NumVDataDwords = (DMaskLanes + 1) / 2;
+      }
+    }
+  }
+
+  // Optimize _L to _LZ when _L is zero
+  if (LZMappingInfo) {
+    // The legalizer replaced the register with an immediate 0 if we need to
+    // change the opcode.
+    const MachineOperand &Lod = MI.getOperand(VAddrIdx + NumVAddr - 1);
+    if (Lod.isImm()) {
+      assert(Lod.getImm() == 0);
+      IntrOpcode = LZMappingInfo->LZ;  // set new opcode to _lz variant of _l
+    }
+  }
+
+  // Optimize _mip away, when 'lod' is zero
+  if (MIPMappingInfo) {
+    const MachineOperand &Lod = MI.getOperand(VAddrIdx + NumVAddr - 1);
+    if (Lod.isImm()) {
+      assert(Lod.getImm() == 0);
+      IntrOpcode = MIPMappingInfo->NONMIP;  // set new opcode to variant without _mip
+    }
+  }
+
+  // TODO: Check this in verifier.
+  assert((!IsTexFail || DMaskLanes >= 1) && "should have legalized this");
+
+  bool GLC = false;
+  bool SLC = false;
+  bool DLC = false;
+  if (BaseOpcode->Atomic) {
+    GLC = true; // TODO no-return optimization
+    if (!parseCachePolicy(MI.getOperand(CtrlIdx + 1).getImm(), nullptr, &SLC,
+                          IsGFX10 ? &DLC : nullptr))
+      return false;
+  } else {
+    if (!parseCachePolicy(MI.getOperand(CtrlIdx + 1).getImm(), &GLC, &SLC,
+                          IsGFX10 ? &DLC : nullptr))
+      return false;
+  }
+
+  int NumVAddrRegs = 0;
+  int NumVAddrDwords = 0;
+  for (int I = 0; I < NumVAddr; ++I) {
+    // Skip the $noregs and 0s inserted during legalization.
+    MachineOperand &AddrOp = MI.getOperand(VAddrIdx + I);
+    if (!AddrOp.isReg())
+      continue; // XXX - Break?
+
+    Register Addr = AddrOp.getReg();
+    if (!Addr)
+      break;
+
+    ++NumVAddrRegs;
+    NumVAddrDwords += (MRI->getType(Addr).getSizeInBits() + 31) / 32;
+  }
+
+  // The legalizer preprocessed the intrinsic arguments. If we aren't using
+  // NSA, these should have beeen packed into a single value in the first
+  // address register
+  const bool UseNSA = NumVAddrRegs != 1 && NumVAddrDwords == NumVAddrRegs;
+  if (UseNSA && !STI.hasFeature(AMDGPU::FeatureNSAEncoding)) {
+    LLVM_DEBUG(dbgs() << "Trying to use NSA on non-NSA target\n");
+    return false;
+  }
+
+  if (IsTexFail)
+    ++NumVDataDwords;
+
+  int Opcode = -1;
+  if (IsGFX10) {
+    Opcode = AMDGPU::getMIMGOpcode(IntrOpcode,
+                                   UseNSA ? AMDGPU::MIMGEncGfx10NSA
+                                          : AMDGPU::MIMGEncGfx10Default,
+                                   NumVDataDwords, NumVAddrDwords);
+  } else {
+    if (STI.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
+      Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx8,
+                                     NumVDataDwords, NumVAddrDwords);
+    if (Opcode == -1)
+      Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx6,
+                                     NumVDataDwords, NumVAddrDwords);
+  }
+  assert(Opcode != -1);
+
+  auto MIB = BuildMI(*MBB, &MI, DL, TII.get(Opcode))
+    .cloneMemRefs(MI);
+
+  if (VDataOut) {
+    if (BaseOpcode->AtomicX2) {
+      const bool Is64 = MRI->getType(VDataOut).getSizeInBits() == 64;
+
+      Register TmpReg = MRI->createVirtualRegister(
+        Is64 ? &AMDGPU::VReg_128RegClass : &AMDGPU::VReg_64RegClass);
+      unsigned SubReg = Is64 ? AMDGPU::sub0_sub1 : AMDGPU::sub0;
+
+      MIB.addDef(TmpReg);
+      BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), VDataOut)
+        .addReg(TmpReg, RegState::Kill, SubReg);
+
+    } else {
+      MIB.addDef(VDataOut); // vdata output
+    }
+  }
+
+  if (VDataIn)
+    MIB.addReg(VDataIn); // vdata input
+
+  for (int i = 0; i != NumVAddrRegs; ++i) {
+    MachineOperand &SrcOp = MI.getOperand(VAddrIdx + i);
+    if (SrcOp.isReg()) {
+      assert(SrcOp.getReg() != 0);
+      MIB.addReg(SrcOp.getReg());
+    }
+  }
+
+  MIB.addReg(MI.getOperand(VAddrIdx + NumVAddr).getReg()); // rsrc
+  if (BaseOpcode->Sampler)
+    MIB.addReg(MI.getOperand(VAddrIdx + NumVAddr + 1).getReg()); // sampler
+
+  MIB.addImm(DMask); // dmask
+
+  if (IsGFX10)
+    MIB.addImm(DimInfo->Encoding);
+  MIB.addImm(Unorm);
+  if (IsGFX10)
+    MIB.addImm(DLC);
+
+  MIB.addImm(GLC);
+  MIB.addImm(SLC);
+  MIB.addImm(IsA16 &&  // a16 or r128
+             STI.hasFeature(AMDGPU::FeatureR128A16) ? -1 : 0);
+  if (IsGFX10)
+    MIB.addImm(IsA16 ? -1 : 0);
+
+  MIB.addImm(TFE); // tfe
+  MIB.addImm(LWE); // lwe
+  if (!IsGFX10)
+    MIB.addImm(DimInfo->DA ? -1 : 0);
+  if (BaseOpcode->HasD16)
+    MIB.addImm(IsD16 ? -1 : 0);
+
+  MI.eraseFromParent();
+  return constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
+}
+
 bool AMDGPUInstructionSelector::selectG_INTRINSIC_W_SIDE_EFFECTS(
     MachineInstr &I) const {
   unsigned IntrinsicID = I.getIntrinsicID();
@@ -1210,9 +1481,10 @@ bool AMDGPUInstructionSelector::selectG_INTRINSIC_W_SIDE_EFFECTS(
     return selectDSAppendConsume(I, true);
   case Intrinsic::amdgcn_ds_consume:
     return selectDSAppendConsume(I, false);
-  default:
+  default: {
     return selectImpl(I, *CoverageInfo);
   }
+  }
 }
 
 bool AMDGPUInstructionSelector::selectG_SELECT(MachineInstr &I) const {
@@ -2371,6 +2643,13 @@ bool AMDGPUInstructionSelector::select(MachineInstr &I) {
   case AMDGPU::G_AMDGPU_ATOMIC_DEC:
     initM0(I);
     return selectImpl(I, *CoverageInfo);
+  case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD:
+  case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE: {
+    const AMDGPU::ImageDimIntrinsicInfo *Intr
+      = AMDGPU::getImageDimIntrinsicInfo(I.getIntrinsicID());
+    assert(Intr && "not an image intrinsic with image pseudo");
+    return selectImageIntrinsic(I, Intr);
+  }
   default:
     return selectImpl(I, *CoverageInfo);
   }
@@ -2547,7 +2826,8 @@ AMDGPUInstructionSelector::selectSmrdImm(MachineOperand &Root) const {
     return None;
 
   const GEPInfo &GEPInfo = AddrInfo[0];
-  Optional<int64_t> EncodedImm = AMDGPU::getSMRDEncodedOffset(STI, GEPInfo.Imm);
+  Optional<int64_t> EncodedImm =
+      AMDGPU::getSMRDEncodedOffset(STI, GEPInfo.Imm, false);
   if (!EncodedImm)
     return None;
 
@@ -2593,7 +2873,8 @@ AMDGPUInstructionSelector::selectSmrdSgpr(MachineOperand &Root) const {
     return None;
 
   const GEPInfo &GEPInfo = AddrInfo[0];
-  if (!GEPInfo.Imm || !isUInt<32>(GEPInfo.Imm))
+  // SGPR offset is unsigned.
+  if (!GEPInfo.Imm || GEPInfo.Imm < 0 || !isUInt<32>(GEPInfo.Imm))
     return None;
 
   // If we make it this far we have a load with an 32-bit immediate offset.
@@ -3233,7 +3514,8 @@ AMDGPUInstructionSelector::selectSMRDBufferImm(MachineOperand &Root) const {
   if (!OffsetVal)
     return {};
 
-  Optional<int64_t> EncodedImm = AMDGPU::getSMRDEncodedOffset(STI, *OffsetVal);
+  Optional<int64_t> EncodedImm =
+      AMDGPU::getSMRDEncodedOffset(STI, *OffsetVal, true);
   if (!EncodedImm)
     return {};
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
index dd29fb40563e7..85577563c1275 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
@@ -31,6 +31,10 @@ namespace {
 
 namespace llvm {
 
+namespace AMDGPU {
+struct ImageDimIntrinsicInfo;
+}
+
 class AMDGPUInstrInfo;
 class AMDGPURegisterBankInfo;
 class GCNSubtarget;
@@ -107,6 +111,8 @@ class AMDGPUInstructionSelector : public InstructionSelector {
   bool selectDSGWSIntrinsic(MachineInstr &MI, Intrinsic::ID IID) const;
   bool selectDSAppendConsume(MachineInstr &MI, bool IsAppend) const;
 
+  bool selectImageIntrinsic(MachineInstr &MI,
+                            const AMDGPU::ImageDimIntrinsicInfo *Intr) const;
   bool selectG_INTRINSIC_W_SIDE_EFFECTS(MachineInstr &I) const;
   int getS_CMPOpcode(CmpInst::Predicate P, unsigned Size) const;
   bool selectG_ICMP(MachineInstr &I) const;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
index dedbe15ab029d..483b1568e532e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
@@ -807,3 +807,8 @@ def fmaxnum_like_oneuse : PatFrags<(ops node:$src0, node:$src1),
   [(fmaxnum_ieee_oneuse node:$src0, node:$src1),
    (fmaxnum_oneuse node:$src0, node:$src1)]
 >;
+
+def any_fmad : PatFrags<(ops node:$src0, node:$src1, node:$src2),
+  [(fmad node:$src0, node:$src1, node:$src2),
+   (AMDGPUfmad_ftz node:$src0, node:$src1, node:$src2)]
+>;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 01fd8483e5c14..5ee3267822b79 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -1313,6 +1313,11 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
     .clampScalar(0, S32, S64)
     .lower();
 
+  getActionDefinitionsBuilder(G_FSHR)
+    .legalFor({{S32, S32}})
+    .scalarize(0)
+    .lower();
+
   getActionDefinitionsBuilder(G_READCYCLECOUNTER)
     .legalFor({S64});
 
@@ -1327,7 +1332,8 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
       G_SADDO, G_SSUBO,
 
        // TODO: Implement
-      G_FMINIMUM, G_FMAXIMUM
+      G_FMINIMUM, G_FMAXIMUM,
+      G_FSHL
     }).lower();
 
   getActionDefinitionsBuilder({G_VASTART, G_VAARG, G_BRJT, G_JUMP_TABLE,
@@ -1458,12 +1464,10 @@ Register AMDGPULegalizerInfo::getSegmentAperture(
   // TODO: can we be smarter about machine pointer info?
   MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
   MachineMemOperand *MMO = MF.getMachineMemOperand(
-    PtrInfo,
-    MachineMemOperand::MOLoad |
-    MachineMemOperand::MODereferenceable |
-    MachineMemOperand::MOInvariant,
-    4,
-    MinAlign(64, StructOffset));
+      PtrInfo,
+      MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
+          MachineMemOperand::MOInvariant,
+      4, commonAlignment(Align(64), StructOffset));
 
   Register LoadAddr;
 
@@ -2022,10 +2026,10 @@ bool AMDGPULegalizerInfo::legalizeGlobalValue(
   Register GOTAddr = MRI.createGenericVirtualRegister(PtrTy);
 
   MachineMemOperand *GOTMMO = MF.getMachineMemOperand(
-    MachinePointerInfo::getGOT(MF),
-    MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
-    MachineMemOperand::MOInvariant,
-    8 /*Size*/, 8 /*Align*/);
+      MachinePointerInfo::getGOT(MF),
+      MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
+          MachineMemOperand::MOInvariant,
+      8 /*Size*/, Align(8));
 
   buildPCRelGlobalAddress(GOTAddr, PtrTy, B, GV, 0, SIInstrInfo::MO_GOTPCREL32);
 
@@ -2071,7 +2075,7 @@ bool AMDGPULegalizerInfo::legalizeFMad(
   MachineIRBuilder HelperBuilder(MI);
   GISelObserverWrapper DummyObserver;
   LegalizerHelper Helper(MF, DummyObserver, HelperBuilder);
-  HelperBuilder.setMBB(*MI.getParent());
+  HelperBuilder.setInstr(MI);
   return Helper.lowerFMad(MI) == LegalizerHelper::Legalized;
 }
 
@@ -2243,12 +2247,8 @@ bool AMDGPULegalizerInfo::legalizeFFloor(MachineInstr &MI,
 bool AMDGPULegalizerInfo::legalizeBuildVector(
   MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
   Register Dst = MI.getOperand(0).getReg();
-  LLT DstTy = MRI.getType(Dst);
   const LLT S32 = LLT::scalar(32);
-  const LLT V2S16 = LLT::vector(2, 16);
-  (void)DstTy;
-  (void)V2S16;
-  assert(DstTy == V2S16);
+  assert(MRI.getType(Dst) == LLT::vector(2, 16));
 
   Register Src0 = MI.getOperand(1).getReg();
   Register Src1 = MI.getOperand(2).getReg();
@@ -2533,11 +2533,176 @@ bool AMDGPULegalizerInfo::legalizeUDIV_UREM32(MachineInstr &MI,
   return true;
 }
 
+// Build integer reciprocal sequence arounud V_RCP_IFLAG_F32
+//
+// Return lo, hi of result
+//
+// %cvt.lo = G_UITOFP Val.lo
+// %cvt.hi = G_UITOFP Val.hi
+// %mad = G_FMAD %cvt.hi, 2**32, %cvt.lo
+// %rcp = G_AMDGPU_RCP_IFLAG %mad
+// %mul1 = G_FMUL %rcp, 0x5f7ffffc
+// %mul2 = G_FMUL %mul1, 2**(-32)
+// %trunc = G_INTRINSIC_TRUNC %mul2
+// %mad2 = G_FMAD %trunc, -(2**32), %mul1
+// return {G_FPTOUI %mad2, G_FPTOUI %trunc}
+static std::pair<Register, Register> emitReciprocalU64(MachineIRBuilder &B,
+                                                       Register Val) {
+  const LLT S32 = LLT::scalar(32);
+  auto Unmerge = B.buildUnmerge(S32, Val);
+
+  auto CvtLo = B.buildUITOFP(S32, Unmerge.getReg(0));
+  auto CvtHi = B.buildUITOFP(S32, Unmerge.getReg(1));
+
+  auto Mad = B.buildFMAD(S32, CvtHi, // 2**32
+                         B.buildFConstant(S32, BitsToFloat(0x4f800000)), CvtLo);
+
+  auto Rcp = B.buildInstr(AMDGPU::G_AMDGPU_RCP_IFLAG, {S32}, {Mad});
+  auto Mul1 =
+      B.buildFMul(S32, Rcp, B.buildFConstant(S32, BitsToFloat(0x5f7ffffc)));
+
+  // 2**(-32)
+  auto Mul2 =
+      B.buildFMul(S32, Mul1, B.buildFConstant(S32, BitsToFloat(0x2f800000)));
+  auto Trunc = B.buildIntrinsicTrunc(S32, Mul2);
+
+  // -(2**32)
+  auto Mad2 = B.buildFMAD(S32, Trunc,
+                          B.buildFConstant(S32, BitsToFloat(0xcf800000)), Mul1);
+
+  auto ResultLo = B.buildFPTOUI(S32, Mad2);
+  auto ResultHi = B.buildFPTOUI(S32, Trunc);
+
+  return {ResultLo.getReg(0), ResultHi.getReg(0)};
+}
+
+bool AMDGPULegalizerInfo::legalizeUDIV_UREM64(MachineInstr &MI,
+                                              MachineRegisterInfo &MRI,
+                                              MachineIRBuilder &B) const {
+  B.setInstr(MI);
+
+  const bool IsDiv = MI.getOpcode() == TargetOpcode::G_UDIV;
+  const LLT S32 = LLT::scalar(32);
+  const LLT S64 = LLT::scalar(64);
+  const LLT S1 = LLT::scalar(1);
+  Register Numer = MI.getOperand(1).getReg();
+  Register Denom = MI.getOperand(2).getReg();
+  Register RcpLo, RcpHi;
+
+  std::tie(RcpLo, RcpHi) = emitReciprocalU64(B, Denom);
+
+  auto Rcp = B.buildMerge(S64, {RcpLo, RcpHi});
+
+  auto Zero64 = B.buildConstant(S64, 0);
+  auto NegDenom = B.buildSub(S64, Zero64, Denom);
+
+  auto MulLo1 = B.buildMul(S64, NegDenom, Rcp);
+  auto MulHi1 = B.buildUMulH(S64, Rcp, MulLo1);
+
+  auto UnmergeMulHi1 = B.buildUnmerge(S32, MulHi1);
+  Register MulHi1_Lo = UnmergeMulHi1.getReg(0);
+  Register MulHi1_Hi = UnmergeMulHi1.getReg(1);
+
+  auto Add1_Lo = B.buildUAddo(S32, S1, RcpLo, MulHi1_Lo);
+  auto Add1_Hi = B.buildUAdde(S32, S1, RcpHi, MulHi1_Hi, Add1_Lo.getReg(1));
+  auto Add1_HiNc = B.buildAdd(S32, RcpHi, MulHi1_Hi);
+  auto Add1 = B.buildMerge(S64, {Add1_Lo, Add1_Hi});
+
+  auto MulLo2 = B.buildMul(S64, NegDenom, Add1);
+  auto MulHi2 = B.buildUMulH(S64, Add1, MulLo2);
+  auto UnmergeMulHi2 = B.buildUnmerge(S32, MulHi2);
+  Register MulHi2_Lo = UnmergeMulHi2.getReg(0);
+  Register MulHi2_Hi = UnmergeMulHi2.getReg(1);
+
+  auto Zero32 = B.buildConstant(S32, 0);
+  auto Add2_Lo = B.buildUAddo(S32, S1, Add1_Lo, MulHi2_Lo);
+  auto Add2_HiC =
+      B.buildUAdde(S32, S1, Add1_HiNc, MulHi2_Hi, Add1_Lo.getReg(1));
+  auto Add2_Hi = B.buildUAdde(S32, S1, Add2_HiC, Zero32, Add2_Lo.getReg(1));
+  auto Add2 = B.buildMerge(S64, {Add2_Lo, Add2_Hi});
+
+  auto UnmergeNumer = B.buildUnmerge(S32, Numer);
+  Register NumerLo = UnmergeNumer.getReg(0);
+  Register NumerHi = UnmergeNumer.getReg(1);
+
+  auto MulHi3 = B.buildUMulH(S64, Numer, Add2);
+  auto Mul3 = B.buildMul(S64, Denom, MulHi3);
+  auto UnmergeMul3 = B.buildUnmerge(S32, Mul3);
+  Register Mul3_Lo = UnmergeMul3.getReg(0);
+  Register Mul3_Hi = UnmergeMul3.getReg(1);
+  auto Sub1_Lo = B.buildUSubo(S32, S1, NumerLo, Mul3_Lo);
+  auto Sub1_Hi = B.buildUSube(S32, S1, NumerHi, Mul3_Hi, Sub1_Lo.getReg(1));
+  auto Sub1_Mi = B.buildSub(S32, NumerHi, Mul3_Hi);
+  auto Sub1 = B.buildMerge(S64, {Sub1_Lo, Sub1_Hi});
+
+  auto UnmergeDenom = B.buildUnmerge(S32, Denom);
+  Register DenomLo = UnmergeDenom.getReg(0);
+  Register DenomHi = UnmergeDenom.getReg(1);
+
+  auto CmpHi = B.buildICmp(CmpInst::ICMP_UGE, S1, Sub1_Hi, DenomHi);
+  auto C1 = B.buildSExt(S32, CmpHi);
+
+  auto CmpLo = B.buildICmp(CmpInst::ICMP_UGE, S1, Sub1_Lo, DenomLo);
+  auto C2 = B.buildSExt(S32, CmpLo);
+
+  auto CmpEq = B.buildICmp(CmpInst::ICMP_EQ, S1, Sub1_Hi, DenomHi);
+  auto C3 = B.buildSelect(S32, CmpEq, C2, C1);
+
+  // TODO: Here and below portions of the code can be enclosed into if/endif.
+  // Currently control flow is unconditional and we have 4 selects after
+  // potential endif to substitute PHIs.
+
+  // if C3 != 0 ...
+  auto Sub2_Lo = B.buildUSubo(S32, S1, Sub1_Lo, DenomLo);
+  auto Sub2_Mi = B.buildUSube(S32, S1, Sub1_Mi, DenomHi, Sub1_Lo.getReg(1));
+  auto Sub2_Hi = B.buildUSube(S32, S1, Sub2_Mi, Zero32, Sub2_Lo.getReg(1));
+  auto Sub2 = B.buildMerge(S64, {Sub2_Lo, Sub2_Hi});
+
+  auto One64 = B.buildConstant(S64, 1);
+  auto Add3 = B.buildAdd(S64, MulHi3, One64);
+
+  auto C4 =
+      B.buildSExt(S32, B.buildICmp(CmpInst::ICMP_UGE, S1, Sub2_Hi, DenomHi));
+  auto C5 =
+      B.buildSExt(S32, B.buildICmp(CmpInst::ICMP_UGE, S1, Sub2_Lo, DenomLo));
+  auto C6 = B.buildSelect(
+      S32, B.buildICmp(CmpInst::ICMP_EQ, S1, Sub2_Hi, DenomHi), C5, C4);
+
+  // if (C6 != 0)
+  auto Add4 = B.buildAdd(S64, Add3, One64);
+  auto Sub3_Lo = B.buildUSubo(S32, S1, Sub2_Lo, DenomLo);
+
+  auto Sub3_Mi = B.buildUSube(S32, S1, Sub2_Mi, DenomHi, Sub2_Lo.getReg(1));
+  auto Sub3_Hi = B.buildUSube(S32, S1, Sub3_Mi, Zero32, Sub3_Lo.getReg(1));
+  auto Sub3 = B.buildMerge(S64, {Sub3_Lo, Sub3_Hi});
+
+  // endif C6
+  // endif C3
+
+  if (IsDiv) {
+    auto Sel1 = B.buildSelect(
+        S64, B.buildICmp(CmpInst::ICMP_NE, S1, C6, Zero32), Add4, Add3);
+    B.buildSelect(MI.getOperand(0),
+                  B.buildICmp(CmpInst::ICMP_NE, S1, C3, Zero32), Sel1, MulHi3);
+  } else {
+    auto Sel2 = B.buildSelect(
+        S64, B.buildICmp(CmpInst::ICMP_NE, S1, C6, Zero32), Sub3, Sub2);
+    B.buildSelect(MI.getOperand(0),
+                  B.buildICmp(CmpInst::ICMP_NE, S1, C3, Zero32), Sel2, Sub1);
+  }
+
+  MI.eraseFromParent();
+  return true;
+}
+
 bool AMDGPULegalizerInfo::legalizeUDIV_UREM(MachineInstr &MI,
                                             MachineRegisterInfo &MRI,
                                             MachineIRBuilder &B) const {
-  if (MRI.getType(MI.getOperand(0).getReg()) == LLT::scalar(32))
+  LLT Ty = MRI.getType(MI.getOperand(0).getReg());
+  if (Ty == LLT::scalar(32))
     return legalizeUDIV_UREM32(MI, MRI, B);
+  if (Ty == LLT::scalar(64))
+    return legalizeUDIV_UREM64(MI, MRI, B);
   return false;
 }
 
@@ -3384,7 +3549,11 @@ static void packImageA16AddressToDwords(MachineIRBuilder &B, MachineInstr &MI,
   const LLT V2S16 = LLT::vector(2, 16);
 
   for (int I = AddrIdx; I < AddrIdx + NumVAddrs; ++I) {
-    Register AddrReg = MI.getOperand(I).getReg();
+    MachineOperand &SrcOp = MI.getOperand(I);
+    if (!SrcOp.isReg())
+      continue; // _L to _LZ may have eliminated this.
+
+    Register AddrReg = SrcOp.getReg();
 
     if (I < DimIdx) {
       AddrReg = B.buildBitcast(V2S16, AddrReg).getReg(0);
@@ -3395,7 +3564,9 @@ static void packImageA16AddressToDwords(MachineIRBuilder &B, MachineInstr &MI,
       if (((I + 1) >= (AddrIdx + NumVAddrs)) ||
           ((NumGradients / 2) % 2 == 1 &&
            (I == DimIdx + (NumGradients / 2) - 1 ||
-            I == DimIdx + NumGradients - 1))) {
+            I == DimIdx + NumGradients - 1)) ||
+          // Check for _L to _LZ optimization
+          !MI.getOperand(I + 1).isReg()) {
         PackedAddrs.push_back(
             B.buildBuildVector(V2S16, {AddrReg, B.buildUndef(S16).getReg(0)})
                 .getReg(0));
@@ -3413,44 +3584,37 @@ static void packImageA16AddressToDwords(MachineIRBuilder &B, MachineInstr &MI,
 /// and replace the remaining operands with $noreg.
 static void convertImageAddrToPacked(MachineIRBuilder &B, MachineInstr &MI,
                                      int DimIdx, int NumVAddrs) {
-  SmallVector<Register, 8> AddrRegs(NumVAddrs);
+  const LLT S32 = LLT::scalar(32);
+
+  SmallVector<Register, 8> AddrRegs;
   for (int I = 0; I != NumVAddrs; ++I) {
-    AddrRegs[I] = MI.getOperand(DimIdx + I).getReg();
-    assert(B.getMRI()->getType(AddrRegs[I]) == LLT::scalar(32));
+    MachineOperand &SrcOp = MI.getOperand(DimIdx + I);
+    if (SrcOp.isReg()) {
+      AddrRegs.push_back(SrcOp.getReg());
+      assert(B.getMRI()->getType(SrcOp.getReg()) == S32);
+    }
   }
 
-  auto VAddr = B.buildBuildVector(LLT::vector(NumVAddrs, 32), AddrRegs);
-  MI.getOperand(DimIdx).setReg(VAddr.getReg(0));
-  for (int I = 1; I != NumVAddrs; ++I)
-    MI.getOperand(DimIdx + I).setReg(AMDGPU::NoRegister);
-}
-
-/// Return number of address arguments, and the number of gradients
-static std::pair<int, int>
-getImageNumVAddr(const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr,
-                 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode) {
-  const AMDGPU::MIMGDimInfo *DimInfo
-    = AMDGPU::getMIMGDimInfo(ImageDimIntr->Dim);
-
-  int NumGradients = BaseOpcode->Gradients ? DimInfo->NumGradients : 0;
-  int NumCoords = BaseOpcode->Coordinates ? DimInfo->NumCoords : 0;
-  int NumLCM = BaseOpcode->LodOrClampOrMip ? 1 : 0;
-  int NumVAddr = BaseOpcode->NumExtraArgs + NumGradients + NumCoords + NumLCM;
-  return {NumVAddr, NumGradients};
-}
+  int NumAddrRegs = AddrRegs.size();
+  if (NumAddrRegs != 1) {
+    // Round up to 8 elements for v5-v7
+    // FIXME: Missing intermediate sized register classes and instructions.
+    if (NumAddrRegs > 4 && !isPowerOf2_32(NumAddrRegs)) {
+      const int RoundedNumRegs = NextPowerOf2(NumAddrRegs);
+      auto Undef = B.buildUndef(S32);
+      AddrRegs.append(RoundedNumRegs - NumAddrRegs, Undef.getReg(0));
+      NumAddrRegs = RoundedNumRegs;
+    }
 
-static int getDMaskIdx(const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode,
-                       int NumDefs) {
-  assert(!BaseOpcode->Atomic);
-  return NumDefs + 1 + (BaseOpcode->Store ? 1 : 0);
-}
+    auto VAddr = B.buildBuildVector(LLT::vector(NumAddrRegs, 32), AddrRegs);
+    MI.getOperand(DimIdx).setReg(VAddr.getReg(0));
+  }
 
-/// Return first address operand index in an image intrinsic.
-static int getImageVAddrIdxBegin(const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode,
-                                 int NumDefs) {
-  if (BaseOpcode->Atomic)
-    return NumDefs + 1 + (BaseOpcode->AtomicX2 ? 2 : 1);
-  return getDMaskIdx(BaseOpcode, NumDefs) + 1;
+  for (int I = 1; I != NumVAddrs; ++I) {
+    MachineOperand &SrcOp = MI.getOperand(DimIdx + I);
+    if (SrcOp.isReg())
+      MI.getOperand(DimIdx + I).setReg(AMDGPU::NoRegister);
+  }
 }
 
 /// Rewrite image intrinsics to use register layouts expected by the subtarget.
@@ -3532,6 +3696,64 @@ bool AMDGPULegalizerInfo::legalizeImageIntrinsic(
     MI.getOperand(DMaskIdx).setImm(DMask);
   }
 
+  if (BaseOpcode->Atomic) {
+    Register VData0 = MI.getOperand(2).getReg();
+    LLT Ty = MRI->getType(VData0);
+
+    // TODO: Allow atomic swap and bit ops for v2s16/v4s16
+    if (Ty.isVector())
+      return false;
+
+    if (BaseOpcode->AtomicX2) {
+      Register VData1 = MI.getOperand(3).getReg();
+      // The two values are packed in one register.
+      LLT PackedTy = LLT::vector(2, Ty);
+      auto Concat = B.buildBuildVector(PackedTy, {VData0, VData1});
+      MI.getOperand(2).setReg(Concat.getReg(0));
+      MI.getOperand(3).setReg(AMDGPU::NoRegister);
+    }
+  }
+
+  int CorrectedNumVAddrs = NumVAddrs;
+
+  // Optimize _L to _LZ when _L is zero
+  if (const AMDGPU::MIMGLZMappingInfo *LZMappingInfo =
+        AMDGPU::getMIMGLZMappingInfo(ImageDimIntr->BaseOpcode)) {
+    const ConstantFP *ConstantLod;
+    const int LodIdx = AddrIdx + NumVAddrs - 1;
+
+    if (mi_match(MI.getOperand(LodIdx).getReg(), *MRI, m_GFCst(ConstantLod))) {
+      if (ConstantLod->isZero() || ConstantLod->isNegative()) {
+        // Set new opcode to _lz variant of _l, and change the intrinsic ID.
+        ImageDimIntr = AMDGPU::getImageDimInstrinsicByBaseOpcode(
+          LZMappingInfo->LZ, ImageDimIntr->Dim);
+
+        // The starting indexes should remain in the same place.
+        --NumVAddrs;
+        --CorrectedNumVAddrs;
+
+        MI.getOperand(MI.getNumExplicitDefs()).setIntrinsicID(
+          static_cast<Intrinsic::ID>(ImageDimIntr->Intr));
+        MI.RemoveOperand(LodIdx);
+      }
+    }
+  }
+
+  // Optimize _mip away, when 'lod' is zero
+  if (AMDGPU::getMIMGMIPMappingInfo(ImageDimIntr->BaseOpcode)) {
+    int64_t ConstantLod;
+    const int LodIdx = AddrIdx + NumVAddrs - 1;
+
+    if (mi_match(MI.getOperand(LodIdx).getReg(), *MRI, m_ICst(ConstantLod))) {
+      if (ConstantLod == 0) {
+        // TODO: Change intrinsic opcode and remove operand instead or replacing
+        // it with 0, as the _L to _LZ handling is done above.
+        MI.getOperand(LodIdx).ChangeToImmediate(0);
+        --CorrectedNumVAddrs;
+      }
+    }
+  }
+
   // If the register allocator cannot place the address registers contiguously
   // without introducing moves, then using the non-sequential address encoding
   // is always preferable, since it saves VALU instructions and is usually a
@@ -3543,8 +3765,7 @@ bool AMDGPULegalizerInfo::legalizeImageIntrinsic(
   //
   // SIShrinkInstructions will convert NSA encodings to non-NSA after register
   // allocation when possible.
-  const bool UseNSA = NumVAddrs >= 3 &&
-                      ST.hasFeature(AMDGPU::FeatureNSAEncoding);
+  const bool UseNSA = CorrectedNumVAddrs >= 3 && ST.hasNSAEncoding();
 
   // Rewrite the addressing register layout before doing anything else.
   if (IsA16) {
@@ -3567,18 +3788,25 @@ bool AMDGPULegalizerInfo::legalizeImageIntrinsic(
 
       const int NumPacked = PackedRegs.size();
       for (int I = 0; I != NumVAddrs; ++I) {
-        assert(MI.getOperand(AddrIdx + I).getReg() != AMDGPU::NoRegister);
+        MachineOperand &SrcOp = MI.getOperand(AddrIdx + I);
+        if (!SrcOp.isReg()) {
+          assert(SrcOp.isImm() && SrcOp.getImm() == 0);
+          continue;
+        }
+
+        assert(SrcOp.getReg() != AMDGPU::NoRegister);
 
         if (I < NumPacked)
-          MI.getOperand(AddrIdx + I).setReg(PackedRegs[I]);
+          SrcOp.setReg(PackedRegs[I]);
         else
-          MI.getOperand(AddrIdx + I).setReg(AMDGPU::NoRegister);
+          SrcOp.setReg(AMDGPU::NoRegister);
       }
     }
   } else if (!UseNSA && NumVAddrs > 1) {
     convertImageAddrToPacked(B, MI, AddrIdx, NumVAddrs);
   }
 
+
   if (BaseOpcode->Store) { // No TFE for stores?
     // TODO: Handle dmask trim
     Register VData = MI.getOperand(1).getReg();
@@ -3778,11 +4006,12 @@ bool AMDGPULegalizerInfo::legalizeSBufferLoad(
   // FIXME: When intrinsic definition is fixed, this should have an MMO already.
   // TODO: Should this use datalayout alignment?
   const unsigned MemSize = (Size + 7) / 8;
-  const unsigned MemAlign = 4;
+  const Align MemAlign(4);
   MachineMemOperand *MMO = MF.getMachineMemOperand(
-    MachinePointerInfo(),
-    MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
-    MachineMemOperand::MOInvariant, MemSize, MemAlign);
+      MachinePointerInfo(),
+      MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
+          MachineMemOperand::MOInvariant,
+      MemSize, MemAlign);
   MI.addMemOperand(MF, MMO);
 
   // There are no 96-bit result scalar loads, but widening to 128-bit should
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
index 270195f8eddd4..48012d7a1013d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
@@ -111,9 +111,11 @@ class AMDGPULegalizerInfo : public LegalizerInfo {
                                bool IsRem) const;
   bool legalizeUDIV_UREM32(MachineInstr &MI, MachineRegisterInfo &MRI,
                            MachineIRBuilder &B) const;
-
   bool legalizeSDIV_SREM32(MachineInstr &MI, MachineRegisterInfo &MRI,
                            MachineIRBuilder &B) const;
+
+  bool legalizeUDIV_UREM64(MachineInstr &MI, MachineRegisterInfo &MRI,
+                           MachineIRBuilder &B) const;
   bool legalizeSDIV_SREM(MachineInstr &MI, MachineRegisterInfo &MRI,
                          MachineIRBuilder &B) const;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp b/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp
index 98931c72bc67b..4fb85832412f8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp
@@ -32,7 +32,6 @@
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetMachine.h"
-#include "llvm/Target/TargetOptions.h"
 #include <cmath>
 #include <vector>
 
@@ -170,16 +169,13 @@ namespace {
 
   class AMDGPUSimplifyLibCalls : public FunctionPass {
 
-  const TargetOptions Options;
-
   AMDGPULibCalls Simplifier;
 
   public:
     static char ID; // Pass identification
 
-    AMDGPUSimplifyLibCalls(const TargetOptions &Opt = TargetOptions(),
-                           const TargetMachine *TM = nullptr)
-      : FunctionPass(ID), Options(Opt), Simplifier(TM) {
+    AMDGPUSimplifyLibCalls(const TargetMachine *TM = nullptr)
+      : FunctionPass(ID), Simplifier(TM) {
       initializeAMDGPUSimplifyLibCallsPass(*PassRegistry::getPassRegistry());
     }
 
@@ -1711,35 +1707,14 @@ bool AMDGPULibCalls::evaluateCall(CallInst *aCI, FuncInfo &FInfo) {
 }
 
 // Public interface to the Simplify LibCalls pass.
-FunctionPass *llvm::createAMDGPUSimplifyLibCallsPass(const TargetOptions &Opt,
-                                                     const TargetMachine *TM) {
-  return new AMDGPUSimplifyLibCalls(Opt, TM);
+FunctionPass *llvm::createAMDGPUSimplifyLibCallsPass(const TargetMachine *TM) {
+  return new AMDGPUSimplifyLibCalls(TM);
 }
 
 FunctionPass *llvm::createAMDGPUUseNativeCallsPass() {
   return new AMDGPUUseNativeCalls();
 }
 
-static bool setFastFlags(Function &F, const TargetOptions &Options) {
-  AttrBuilder B;
-
-  if (Options.UnsafeFPMath || Options.NoInfsFPMath)
-    B.addAttribute("no-infs-fp-math", "true");
-  if (Options.UnsafeFPMath || Options.NoNaNsFPMath)
-    B.addAttribute("no-nans-fp-math", "true");
-  if (Options.UnsafeFPMath) {
-    B.addAttribute("less-precise-fpmad", "true");
-    B.addAttribute("unsafe-fp-math", "true");
-  }
-
-  if (!B.hasAttributes())
-    return false;
-
-  F.addAttributes(AttributeList::FunctionIndex, B);
-
-  return true;
-}
-
 bool AMDGPUSimplifyLibCalls::runOnFunction(Function &F) {
   if (skipFunction(F))
     return false;
@@ -1750,9 +1725,6 @@ bool AMDGPUSimplifyLibCalls::runOnFunction(Function &F) {
   LLVM_DEBUG(dbgs() << "AMDIC: process function ";
              F.printAsOperand(dbgs(), false, F.getParent()); dbgs() << '\n';);
 
-  if (!EnablePreLink)
-    Changed |= setFastFlags(F, Options);
-
   for (auto &BB : F) {
     for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E; ) {
       // Ignore non-calls.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
index 395c04161ea7b..b7baecdec6b6e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
@@ -210,7 +210,7 @@ bool AMDGPULowerKernelArguments::runOnFunction(Function &F) {
       Arg.replaceAllUsesWith(NewVal);
     } else if (IsV3) {
       Value *Shuf = Builder.CreateShuffleVector(Load, UndefValue::get(V4Ty),
-                                                {0, 1, 2},
+                                                ArrayRef<int>{0, 1, 2},
                                                 Arg.getName() + ".load");
       Arg.replaceAllUsesWith(Shuf);
     } else {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp
index 940ddff85d73f..6ce22a3f78bda 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp
@@ -18,7 +18,7 @@ AMDGPUMachineFunction::AMDGPUMachineFunction(const MachineFunction &MF) :
   LocalMemoryObjects(),
   ExplicitKernArgSize(0),
   LDSSize(0),
-  Mode(MF.getFunction(), MF.getSubtarget<GCNSubtarget>()),
+  Mode(MF.getFunction()),
   IsEntryFunction(AMDGPU::isEntryFunctionCC(MF.getFunction().getCallingConv())),
   NoSignedZerosFPMath(MF.getTarget().Options.NoSignedZerosFPMath),
   MemoryBound(false),
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp
index 8f43e6e9f0e40..7fe5e484db0b9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp
@@ -127,6 +127,43 @@ static void applySelectFCmpToFMinToFMaxLegacy(MachineInstr &MI,
   MI.eraseFromParent();
 }
 
+static bool matchUCharToFloat(MachineInstr &MI, MachineRegisterInfo &MRI,
+                              MachineFunction &MF, CombinerHelper &Helper) {
+  Register DstReg = MI.getOperand(0).getReg();
+
+  // TODO: We could try to match extracting the higher bytes, which would be
+  // easier if i8 vectors weren't promoted to i32 vectors, particularly after
+  // types are legalized. v4i8 -> v4f32 is probably the only case to worry
+  // about in practice.
+  LLT Ty = MRI.getType(DstReg);
+  if (Ty == LLT::scalar(32) || Ty == LLT::scalar(16)) {
+    const APInt Mask = APInt::getHighBitsSet(32, 24);
+    return Helper.getKnownBits()->maskedValueIsZero(MI.getOperand(1).getReg(),
+                                                    Mask);
+  }
+
+  return false;
+}
+
+static void applyUCharToFloat(MachineInstr &MI) {
+  MachineIRBuilder B(MI);
+
+  const LLT S32 = LLT::scalar(32);
+
+  Register DstReg = MI.getOperand(0).getReg();
+  LLT Ty = B.getMRI()->getType(DstReg);
+
+  if (Ty == S32) {
+    B.buildInstr(AMDGPU::G_AMDGPU_CVT_F32_UBYTE0, {DstReg},
+                   {MI.getOperand(1)}, MI.getFlags());
+  } else {
+    auto Cvt0 = B.buildInstr(AMDGPU::G_AMDGPU_CVT_F32_UBYTE0, {S32},
+                             {MI.getOperand(1)}, MI.getFlags());
+    B.buildFPTrunc(DstReg, Cvt0, MI.getFlags());
+  }
+
+  MI.eraseFromParent();
+}
 
 #define AMDGPUPOSTLEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_DEPS
 #include "AMDGPUGenPostLegalizeGICombiner.inc"
@@ -154,8 +191,8 @@ class AMDGPUPostLegalizerCombinerInfo : public CombinerInfo {
       report_fatal_error("Invalid rule identifier");
   }
 
-  virtual bool combine(GISelChangeObserver &Observer, MachineInstr &MI,
-                       MachineIRBuilder &B) const override;
+  bool combine(GISelChangeObserver &Observer, MachineInstr &MI,
+               MachineIRBuilder &B) const override;
 };
 
 bool AMDGPUPostLegalizerCombinerInfo::combine(GISelChangeObserver &Observer,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPropagateAttributes.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPropagateAttributes.cpp
index 68fa5bafa97bd..982aae3748849 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPropagateAttributes.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPropagateAttributes.cpp
@@ -48,19 +48,62 @@ extern const SubtargetFeatureKV AMDGPUFeatureKV[AMDGPU::NumSubtargetFeatures-1];
 
 namespace {
 
+// Target features to propagate.
+static constexpr const FeatureBitset TargetFeatures = {
+  AMDGPU::FeatureWavefrontSize16,
+  AMDGPU::FeatureWavefrontSize32,
+  AMDGPU::FeatureWavefrontSize64
+};
+
+// Attributes to propagate.
+static constexpr const char* AttributeNames[] = {
+  "amdgpu-waves-per-eu"
+};
+
+static constexpr unsigned NumAttr =
+  sizeof(AttributeNames) / sizeof(AttributeNames[0]);
+
 class AMDGPUPropagateAttributes {
-  const FeatureBitset TargetFeatures = {
-    AMDGPU::FeatureWavefrontSize16,
-    AMDGPU::FeatureWavefrontSize32,
-    AMDGPU::FeatureWavefrontSize64
+
+  class FnProperties {
+  private:
+    explicit FnProperties(const FeatureBitset &&FB) : Features(FB) {}
+
+  public:
+    explicit FnProperties(const TargetMachine &TM, const Function &F) {
+      Features = TM.getSubtargetImpl(F)->getFeatureBits();
+
+      for (unsigned I = 0; I < NumAttr; ++I)
+        if (F.hasFnAttribute(AttributeNames[I]))
+          Attributes[I] = F.getFnAttribute(AttributeNames[I]);
+    }
+
+    bool operator == (const FnProperties &Other) const {
+      if ((Features & TargetFeatures) != (Other.Features & TargetFeatures))
+        return false;
+      for (unsigned I = 0; I < NumAttr; ++I)
+        if (Attributes[I] != Other.Attributes[I])
+          return false;
+      return true;
+    }
+
+    FnProperties adjustToCaller(const FnProperties &CallerProps) const {
+      FnProperties New((Features & ~TargetFeatures) | CallerProps.Features);
+      for (unsigned I = 0; I < NumAttr; ++I)
+        New.Attributes[I] = CallerProps.Attributes[I];
+      return New;
+    }
+
+    FeatureBitset Features;
+    Optional<Attribute> Attributes[NumAttr];
   };
 
-  class Clone{
+  class Clone {
   public:
-    Clone(FeatureBitset FeatureMask, Function *OrigF, Function *NewF) :
-      FeatureMask(FeatureMask), OrigF(OrigF), NewF(NewF) {}
+    Clone(const FnProperties &Props, Function *OrigF, Function *NewF) :
+      Properties(Props), OrigF(OrigF), NewF(NewF) {}
 
-    FeatureBitset FeatureMask;
+    FnProperties Properties;
     Function *OrigF;
     Function *NewF;
   };
@@ -77,17 +120,19 @@ class AMDGPUPropagateAttributes {
   SmallVector<Clone, 32> Clones;
 
   // Find a clone with required features.
-  Function *findFunction(const FeatureBitset &FeaturesNeeded,
+  Function *findFunction(const FnProperties &PropsNeeded,
                          Function *OrigF);
 
-  // Clone function F and set NewFeatures on the clone.
+  // Clone function \p F and set \p NewProps on the clone.
   // Cole takes the name of original function.
-  Function *cloneWithFeatures(Function &F,
-                              const FeatureBitset &NewFeatures);
+  Function *cloneWithProperties(Function &F, const FnProperties &NewProps);
 
   // Set new function's features in place.
   void setFeatures(Function &F, const FeatureBitset &NewFeatures);
 
+  // Set new function's attributes in place.
+  void setAttributes(Function &F, const ArrayRef<Optional<Attribute>> NewAttrs);
+
   std::string getFeatureString(const FeatureBitset &Features) const;
 
   // Propagate attributes from Roots.
@@ -155,11 +200,11 @@ INITIALIZE_PASS(AMDGPUPropagateAttributesLate,
                 false, false)
 
 Function *
-AMDGPUPropagateAttributes::findFunction(const FeatureBitset &FeaturesNeeded,
+AMDGPUPropagateAttributes::findFunction(const FnProperties &PropsNeeded,
                                         Function *OrigF) {
   // TODO: search for clone's clones.
   for (Clone &C : Clones)
-    if (C.OrigF == OrigF && FeaturesNeeded == C.FeatureMask)
+    if (C.OrigF == OrigF && PropsNeeded == C.Properties)
       return C.NewF;
 
   return nullptr;
@@ -192,12 +237,12 @@ bool AMDGPUPropagateAttributes::process() {
     NewRoots.clear();
 
     for (auto &F : M.functions()) {
-      if (F.isDeclaration() || Roots.count(&F) || Roots.count(&F))
+      if (F.isDeclaration())
         continue;
 
-      const FeatureBitset &CalleeBits =
-        TM->getSubtargetImpl(F)->getFeatureBits();
+      const FnProperties CalleeProps(*TM, F);
       SmallVector<std::pair<CallBase *, Function *>, 32> ToReplace;
+      SmallSet<CallBase *, 32> Visited;
 
       for (User *U : F.users()) {
         Instruction *I = dyn_cast<Instruction>(U);
@@ -207,36 +252,36 @@ bool AMDGPUPropagateAttributes::process() {
         if (!CI)
           continue;
         Function *Caller = CI->getCaller();
-        if (!Caller)
+        if (!Caller || !Visited.insert(CI).second)
           continue;
-        if (!Roots.count(Caller))
+        if (!Roots.count(Caller) && !NewRoots.count(Caller))
           continue;
 
-        const FeatureBitset &CallerBits =
-          TM->getSubtargetImpl(*Caller)->getFeatureBits() & TargetFeatures;
+        const FnProperties CallerProps(*TM, *Caller);
 
-        if (CallerBits == (CalleeBits  & TargetFeatures)) {
-          NewRoots.insert(&F);
+        if (CalleeProps == CallerProps) {
+          if (!Roots.count(&F))
+            NewRoots.insert(&F);
           continue;
         }
 
-        Function *NewF = findFunction(CallerBits, &F);
+        Function *NewF = findFunction(CallerProps, &F);
         if (!NewF) {
-          FeatureBitset NewFeatures((CalleeBits & ~TargetFeatures) |
-                                    CallerBits);
+          const FnProperties NewProps = CalleeProps.adjustToCaller(CallerProps);
           if (!AllowClone) {
             // This may set different features on different iteartions if
             // there is a contradiction in callers' attributes. In this case
             // we rely on a second pass running on Module, which is allowed
             // to clone.
-            setFeatures(F, NewFeatures);
+            setFeatures(F, NewProps.Features);
+            setAttributes(F, NewProps.Attributes);
             NewRoots.insert(&F);
             Changed = true;
             break;
           }
 
-          NewF = cloneWithFeatures(F, NewFeatures);
-          Clones.push_back(Clone(CallerBits, &F, NewF));
+          NewF = cloneWithProperties(F, NewProps);
+          Clones.push_back(Clone(CallerProps, &F, NewF));
           NewRoots.insert(NewF);
         }
 
@@ -258,28 +303,30 @@ bool AMDGPUPropagateAttributes::process() {
       F->eraseFromParent();
   }
 
+  Roots.clear();
+  Clones.clear();
+
   return Changed;
 }
 
 Function *
-AMDGPUPropagateAttributes::cloneWithFeatures(Function &F,
-                                             const FeatureBitset &NewFeatures) {
+AMDGPUPropagateAttributes::cloneWithProperties(Function &F,
+                                               const FnProperties &NewProps) {
   LLVM_DEBUG(dbgs() << "Cloning " << F.getName() << '\n');
 
   ValueToValueMapTy dummy;
   Function *NewF = CloneFunction(&F, dummy);
-  setFeatures(*NewF, NewFeatures);
+  setFeatures(*NewF, NewProps.Features);
+  setAttributes(*NewF, NewProps.Attributes);
+  NewF->setVisibility(GlobalValue::DefaultVisibility);
+  NewF->setLinkage(GlobalValue::InternalLinkage);
 
   // Swap names. If that is the only clone it will retain the name of now
-  // dead value.
-  if (F.hasName()) {
+  // dead value. Preserve original name for externally visible functions.
+  if (F.hasName() && F.hasLocalLinkage()) {
     std::string NewName = std::string(NewF->getName());
     NewF->takeName(&F);
     F.setName(NewName);
-
-    // Name has changed, it does not need an external symbol.
-    F.setVisibility(GlobalValue::DefaultVisibility);
-    F.setLinkage(GlobalValue::InternalLinkage);
   }
 
   return NewF;
@@ -297,6 +344,18 @@ void AMDGPUPropagateAttributes::setFeatures(Function &F,
   F.addFnAttr("target-features", NewFeatureStr);
 }
 
+void AMDGPUPropagateAttributes::setAttributes(Function &F,
+    const ArrayRef<Optional<Attribute>> NewAttrs) {
+  LLVM_DEBUG(dbgs() << "Set attributes on " << F.getName() << ":\n");
+  for (unsigned I = 0; I < NumAttr; ++I) {
+    F.removeFnAttr(AttributeNames[I]);
+    if (NewAttrs[I]) {
+      LLVM_DEBUG(dbgs() << '\t' << NewAttrs[I]->getAsString() << '\n');
+      F.addFnAttr(*NewAttrs[I]);
+    }
+  }
+}
+
 std::string
 AMDGPUPropagateAttributes::getFeatureString(const FeatureBitset &Features) const
 {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index 7f498e606a70c..adfcecd2329b5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -444,15 +444,15 @@ static bool isScalarLoadLegal(const MachineInstr &MI) {
                        AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT;
 
   // There are no extending SMRD/SMEM loads, and they require 4-byte alignment.
-  return MMO->getSize() >= 4 && MMO->getAlignment() >= 4 &&
-    // Can't do a scalar atomic load.
-    !MMO->isAtomic() &&
-    // Don't use scalar loads for volatile accesses to non-constant address
-    // spaces.
-    (IsConst || !MMO->isVolatile()) &&
-    // Memory must be known constant, or not written before this load.
-    (IsConst || MMO->isInvariant() || memOpHasNoClobbered(MMO)) &&
-    AMDGPUInstrInfo::isUniformMMO(MMO);
+  return MMO->getSize() >= 4 && MMO->getAlign() >= Align(4) &&
+         // Can't do a scalar atomic load.
+         !MMO->isAtomic() &&
+         // Don't use scalar loads for volatile accesses to non-constant address
+         // spaces.
+         (IsConst || !MMO->isVolatile()) &&
+         // Memory must be known constant, or not written before this load.
+         (IsConst || MMO->isInvariant() || memOpHasNoClobbered(MMO)) &&
+         AMDGPUInstrInfo::isUniformMMO(MMO);
 }
 
 RegisterBankInfo::InstructionMappings
@@ -1352,7 +1352,7 @@ bool AMDGPURegisterBankInfo::applyMappingSBufferLoad(
 
   // Use the alignment to ensure that the required offsets will fit into the
   // immediate offsets.
-  const unsigned Align = NumLoads > 1 ? 16 * NumLoads : 1;
+  const unsigned Alignment = NumLoads > 1 ? 16 * NumLoads : 1;
 
   MachineIRBuilder B(MI);
   MachineFunction &MF = B.getMF();
@@ -1362,12 +1362,12 @@ bool AMDGPURegisterBankInfo::applyMappingSBufferLoad(
   int64_t ImmOffset = 0;
 
   unsigned MMOOffset = setBufferOffsets(B, *this, MI.getOperand(2).getReg(),
-                                        VOffset, SOffset, ImmOffset, Align);
+                                        VOffset, SOffset, ImmOffset, Alignment);
 
   // TODO: 96-bit loads were widened to 128-bit results. Shrink the result if we
   // can, but we neeed to track an MMO for that.
   const unsigned MemSize = (Ty.getSizeInBits() + 7) / 8;
-  const unsigned MemAlign = 4; // FIXME: ABI type alignment?
+  const Align MemAlign(4); // FIXME: ABI type alignment?
   MachineMemOperand *BaseMMO = MF.getMachineMemOperand(
     MachinePointerInfo(),
     MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
@@ -2920,6 +2920,10 @@ AMDGPURegisterBankInfo::getImageMapping(const MachineRegisterInfo &MRI,
       continue;
 
     Register OpReg = MI.getOperand(I).getReg();
+    // We replace some dead address operands with $noreg
+    if (!OpReg)
+      continue;
+
     unsigned Size = getSizeInBits(OpReg, MRI, *TRI);
 
     // FIXME: Probably need a new intrinsic register bank searchable table to
@@ -3284,10 +3288,15 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   case AMDGPU::G_FCANONICALIZE:
   case AMDGPU::G_INTRINSIC_TRUNC:
   case AMDGPU::G_BSWAP: // TODO: Somehow expand for scalar?
+  case AMDGPU::G_FSHR: // TODO: Expand for scalar
   case AMDGPU::G_AMDGPU_FFBH_U32:
   case AMDGPU::G_AMDGPU_FMIN_LEGACY:
   case AMDGPU::G_AMDGPU_FMAX_LEGACY:
   case AMDGPU::G_AMDGPU_RCP_IFLAG:
+  case AMDGPU::G_AMDGPU_CVT_F32_UBYTE0:
+  case AMDGPU::G_AMDGPU_CVT_F32_UBYTE1:
+  case AMDGPU::G_AMDGPU_CVT_F32_UBYTE2:
+  case AMDGPU::G_AMDGPU_CVT_F32_UBYTE3:
     return getDefaultMappingVOP(MI);
   case AMDGPU::G_UMULH:
   case AMDGPU::G_SMULH: {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBanks.td b/llvm/lib/Target/AMDGPU/AMDGPURegisterBanks.td
index c495316c5bce0..cb831f3a53b96 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBanks.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBanks.td
@@ -11,7 +11,7 @@ def SGPRRegBank : RegisterBank<"SGPR",
 >;
 
 def VGPRRegBank : RegisterBank<"VGPR",
-  [VGPR_32, VReg_64, VReg_96, VReg_128, VReg_256, VReg_512, VReg_1024]
+  [VGPR_LO16, VGPR_HI16, VGPR_32, VReg_64, VReg_96, VReg_128, VReg_256, VReg_512, VReg_1024]
 >;
 
 // It is helpful to distinguish conditions from ordinary SGPRs.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp b/llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
index cf641e3948acc..5a2cd48a81e92 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
@@ -409,7 +409,7 @@ bool AMDGPURewriteOutArguments::runOnFunction(Function &F) {
             DL->getTypeSizeInBits(Val->getType())) {
           assert(isVec3ToVec4Shuffle(EffectiveEltTy, Val->getType()));
           Val = B.CreateShuffleVector(Val, UndefValue::get(Val->getType()),
-                                      { 0, 1, 2 });
+                                      ArrayRef<int>{0, 1, 2});
         }
 
         Val = B.CreateBitCast(Val, EffectiveEltTy);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
index 248c3cbceb328..df4c6308fee34 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
@@ -59,13 +59,6 @@ R600Subtarget::initializeSubtargetDependencies(const Triple &TT,
   FullFS += FS;
   ParseSubtargetFeatures(GPU, FullFS);
 
-  // FIXME: I don't think think Evergreen has any useful support for
-  // denormals, but should be checked. Should we issue a warning somewhere
-  // if someone tries to enable these?
-  if (getGeneration() <= AMDGPUSubtarget::NORTHERN_ISLANDS) {
-    FP32Denormals = false;
-  }
-
   HasMulU24 = getGeneration() >= EVERGREEN;
   HasMulI24 = hasCaymanISA();
 
@@ -76,9 +69,6 @@ GCNSubtarget &
 GCNSubtarget::initializeSubtargetDependencies(const Triple &TT,
                                               StringRef GPU, StringRef FS) {
   // Determine default and user-specified characteristics
-  // On SI+, we want FP64 denormals to be on by default. FP32 denormals can be
-  // enabled, but some instructions do not respect them and they run at the
-  // double precision rate, so don't enable by default.
   //
   // We want to be able to turn these off, but making this a subtarget feature
   // for SI has the unhelpful behavior that it unsets everything else if you
@@ -88,20 +78,11 @@ GCNSubtarget::initializeSubtargetDependencies(const Triple &TT,
   // unset everything else if it is disabled
 
   // Assuming ECC is enabled is the conservative default.
-  SmallString<256> FullFS("+promote-alloca,+load-store-opt,+sram-ecc,+xnack,");
+  SmallString<256> FullFS("+promote-alloca,+load-store-opt,+enable-ds128,+sram-ecc,+xnack,");
 
   if (isAmdHsaOS()) // Turn on FlatForGlobal for HSA.
     FullFS += "+flat-for-global,+unaligned-buffer-access,+trap-handler,";
 
-  // FIXME: I don't think think Evergreen has any useful support for
-  // denormals, but should be checked. Should we issue a warning somewhere
-  // if someone tries to enable these?
-  if (getGeneration() >= AMDGPUSubtarget::SOUTHERN_ISLANDS) {
-    FullFS += "+fp64-fp16-denormals,";
-  } else {
-    FullFS += "-fp32-denormals,";
-  }
-
   FullFS += "+enable-prt-strict-null,"; // This is overridden by a disable in FS
 
   // Disable mutually exclusive bits.
@@ -172,7 +153,6 @@ AMDGPUSubtarget::AMDGPUSubtarget(const Triple &TT) :
   TargetTriple(TT),
   Has16BitInsts(false),
   HasMadMixInsts(false),
-  FP32Denormals(false),
   FPExceptions(false),
   HasSDWA(false),
   HasVOP3PInsts(false),
@@ -200,7 +180,6 @@ GCNSubtarget::GCNSubtarget(const Triple &TT, StringRef GPU, StringRef FS,
     FastFMAF32(false),
     HalfRate64Ops(false),
 
-    FP64FP16Denormals(false),
     FlatForGlobal(false),
     AutoWaitcntBeforeBarrier(false),
     CodeObjectV3(false),
@@ -422,7 +401,7 @@ std::pair<unsigned, unsigned> AMDGPUSubtarget::getWavesPerEU(
   // number of waves per execution unit to values implied by requested
   // minimum/maximum flat work group sizes.
   unsigned MinImpliedByFlatWorkGroupSize =
-    getMaxWavesPerEU(FlatWorkGroupSizes.second);
+    getWavesPerEUForWorkGroup(FlatWorkGroupSizes.second);
   bool RequestedFlatWorkGroupSize = false;
 
   if (F.hasFnAttribute("amdgpu-flat-work-group-size")) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
index 5fdcd71537caf..c565c17ff03d7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
@@ -66,7 +66,6 @@ class AMDGPUSubtarget {
 protected:
   bool Has16BitInsts;
   bool HasMadMixInsts;
-  bool FP32Denormals;
   bool FPExceptions;
   bool HasSDWA;
   bool HasVOP3PInsts;
@@ -149,15 +148,6 @@ class AMDGPUSubtarget {
     return HasMadMixInsts;
   }
 
-  bool hasFP32Denormals(const Function &F) const {
-    // FIXME: This should not be a property of the subtarget. This should be a
-    // property with a default set by the calling convention which can be
-    // overridden by attributes. For now, use the subtarget feature as a
-    // placeholder attribute. The function arguments only purpose is to
-    // discourage use without a function context until this is removed.
-    return FP32Denormals;
-  }
-
   bool hasFPExceptions() const {
     return FPExceptions;
   }
@@ -222,9 +212,10 @@ class AMDGPUSubtarget {
   /// \returns Maximum flat work group size supported by the subtarget.
   virtual unsigned getMaxFlatWorkGroupSize() const = 0;
 
-  /// \returns Maximum number of waves per execution unit supported by the
-  /// subtarget and limited by given \p FlatWorkGroupSize.
-  virtual unsigned getMaxWavesPerEU(unsigned FlatWorkGroupSize) const  = 0;
+  /// \returns Number of waves per execution unit required to support the given
+  /// \p FlatWorkGroupSize.
+  virtual unsigned
+  getWavesPerEUForWorkGroup(unsigned FlatWorkGroupSize) const = 0;
 
   /// \returns Minimum number of waves per execution unit supported by the
   /// subtarget.
@@ -303,7 +294,6 @@ class GCNSubtarget : public AMDGPUGenSubtargetInfo,
   bool HalfRate64Ops;
 
   // Dynamially set bits that enable features.
-  bool FP64FP16Denormals;
   bool FlatForGlobal;
   bool AutoWaitcntBeforeBarrier;
   bool CodeObjectV3;
@@ -635,20 +625,6 @@ class GCNSubtarget : public AMDGPUGenSubtargetInfo,
   unsigned getMaxLocalMemSizeWithWaveCount(unsigned WaveCount,
                                            const Function &) const;
 
-  /// Alias for hasFP64FP16Denormals
-  bool hasFP16Denormals(const Function &F) const {
-    return FP64FP16Denormals;
-  }
-
-  /// Alias for hasFP64FP16Denormals
-  bool hasFP64Denormals(const Function &F) const {
-    return FP64FP16Denormals;
-  }
-
-  bool hasFP64FP16Denormals(const Function &F) const {
-    return FP64FP16Denormals;
-  }
-
   bool supportsMinMaxDenormModes() const {
     return getGeneration() >= AMDGPUSubtarget::GFX9;
   }
@@ -912,30 +888,6 @@ class GCNSubtarget : public AMDGPUGenSubtargetInfo,
   void setScalarizeGlobalBehavior(bool b) { ScalarizeGlobal = b; }
   bool getScalarizeGlobalBehavior() const { return ScalarizeGlobal; }
 
-  /// \returns Number of execution units per compute unit supported by the
-  /// subtarget.
-  unsigned getEUsPerCU() const {
-    return AMDGPU::IsaInfo::getEUsPerCU(this);
-  }
-
-  /// \returns Maximum number of waves per compute unit supported by the
-  /// subtarget without any kind of limitation.
-  unsigned getMaxWavesPerCU() const {
-    return AMDGPU::IsaInfo::getMaxWavesPerCU(this);
-  }
-
-  /// \returns Maximum number of waves per compute unit supported by the
-  /// subtarget and limited by given \p FlatWorkGroupSize.
-  unsigned getMaxWavesPerCU(unsigned FlatWorkGroupSize) const {
-    return AMDGPU::IsaInfo::getMaxWavesPerCU(this, FlatWorkGroupSize);
-  }
-
-  /// \returns Number of waves per work group supported by the subtarget and
-  /// limited by given \p FlatWorkGroupSize.
-  unsigned getWavesPerWorkGroup(unsigned FlatWorkGroupSize) const {
-    return AMDGPU::IsaInfo::getWavesPerWorkGroup(this, FlatWorkGroupSize);
-  }
-
   // static wrappers
   static bool hasHalfRate64Ops(const TargetSubtargetInfo &STI);
 
@@ -1227,10 +1179,11 @@ class GCNSubtarget : public AMDGPUGenSubtargetInfo,
     return AMDGPU::IsaInfo::getMaxFlatWorkGroupSize(this);
   }
 
-  /// \returns Maximum number of waves per execution unit supported by the
-  /// subtarget and limited by given \p FlatWorkGroupSize.
-  unsigned getMaxWavesPerEU(unsigned FlatWorkGroupSize) const override {
-    return AMDGPU::IsaInfo::getMaxWavesPerEU(this, FlatWorkGroupSize);
+  /// \returns Number of waves per execution unit required to support the given
+  /// \p FlatWorkGroupSize.
+  unsigned
+  getWavesPerEUForWorkGroup(unsigned FlatWorkGroupSize) const override {
+    return AMDGPU::IsaInfo::getWavesPerEUForWorkGroup(this, FlatWorkGroupSize);
   }
 
   /// \returns Minimum number of waves per execution unit supported by the
@@ -1364,10 +1317,11 @@ class R600Subtarget final : public R600GenSubtargetInfo,
     return AMDGPU::IsaInfo::getMaxFlatWorkGroupSize(this);
   }
 
-  /// \returns Maximum number of waves per execution unit supported by the
-  /// subtarget and limited by given \p FlatWorkGroupSize.
-  unsigned getMaxWavesPerEU(unsigned FlatWorkGroupSize) const override {
-    return AMDGPU::IsaInfo::getMaxWavesPerEU(this, FlatWorkGroupSize);
+  /// \returns Number of waves per execution unit required to support the given
+  /// \p FlatWorkGroupSize.
+  unsigned
+  getWavesPerEUForWorkGroup(unsigned FlatWorkGroupSize) const override {
+    return AMDGPU::IsaInfo::getWavesPerEUForWorkGroup(this, FlatWorkGroupSize);
   }
 
   /// \returns Minimum number of waves per execution unit supported by the
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index a1c20220ae264..17c48a96cd5ea 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -192,6 +192,11 @@ static cl::opt<bool> EnableScalarIRPasses(
   cl::init(true),
   cl::Hidden);
 
+static cl::opt<bool> EnableStructurizerWorkarounds(
+    "amdgpu-enable-structurizer-workarounds",
+    cl::desc("Enable workarounds for the StructurizeCFG pass"), cl::init(false),
+    cl::Hidden);
+
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   // Register the target
   RegisterTargetMachine<R600TargetMachine> X(getTheAMDGPUTarget());
@@ -240,6 +245,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   initializeSIWholeQuadModePass(*PR);
   initializeSILowerControlFlowPass(*PR);
   initializeSIRemoveShortExecBranchesPass(*PR);
+  initializeSIPreEmitPeepholePass(*PR);
   initializeSIInsertSkipsPass(*PR);
   initializeSIMemoryLegalizerPass(*PR);
   initializeSIOptimizeExecMaskingPass(*PR);
@@ -436,20 +442,19 @@ void AMDGPUTargetMachine::adjustPassManager(PassManagerBuilder &Builder) {
       }
       PM.add(createAMDGPUUnifyMetadataPass());
       PM.add(createAMDGPUPrintfRuntimeBinding());
-      PM.add(createAMDGPUPropagateAttributesLatePass(this));
-      if (Internalize) {
+      if (Internalize)
         PM.add(createInternalizePass(mustPreserveGV));
+      PM.add(createAMDGPUPropagateAttributesLatePass(this));
+      if (Internalize)
         PM.add(createGlobalDCEPass());
-      }
       if (EarlyInline)
         PM.add(createAMDGPUAlwaysInlinePass(false));
   });
 
-  const auto &Opt = Options;
   Builder.addExtension(
     PassManagerBuilder::EP_EarlyAsPossible,
-    [AMDGPUAA, LibCallSimplify, &Opt, this](const PassManagerBuilder &,
-                                            legacy::PassManagerBase &PM) {
+    [AMDGPUAA, LibCallSimplify, this](const PassManagerBuilder &,
+                                      legacy::PassManagerBase &PM) {
       if (AMDGPUAA) {
         PM.add(createAMDGPUAAWrapperPass());
         PM.add(createAMDGPUExternalAAWrapperPass());
@@ -457,7 +462,7 @@ void AMDGPUTargetMachine::adjustPassManager(PassManagerBuilder &Builder) {
       PM.add(llvm::createAMDGPUPropagateAttributesEarlyPass(this));
       PM.add(llvm::createAMDGPUUseNativeCallsPass());
       if (LibCallSimplify)
-        PM.add(llvm::createAMDGPUSimplifyLibCallsPass(Opt, this));
+        PM.add(llvm::createAMDGPUSimplifyLibCallsPass(this));
   });
 
   Builder.addExtension(
@@ -858,6 +863,9 @@ bool GCNPassConfig::addPreISel() {
   // regions formed by them.
   addPass(&AMDGPUUnifyDivergentExitNodesID);
   if (!LateCFGStructurize) {
+    if (EnableStructurizerWorkarounds) {
+      addPass(createUnifyLoopExitsPass());
+    }
     addPass(createStructurizeCFGPass(true)); // true -> SkipUniformRegions
   }
   addPass(createSinkingPass());
@@ -1029,6 +1037,7 @@ void GCNPassConfig::addPreEmitPass() {
   addPass(&PostRAHazardRecognizerID);
 
   addPass(&SIRemoveShortExecBranchesID);
+  addPass(&SIPreEmitPeepholeID);
   addPass(&SIInsertSkipsPassID);
   addPass(&BranchRelaxationPassID);
 }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 318e536da6aa4..dd8a08baa8b4f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -311,6 +311,76 @@ bool GCNTTIImpl::isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes,
   return isLegalToVectorizeMemChain(ChainSizeInBytes, Alignment, AddrSpace);
 }
 
+// FIXME: Really we would like to issue multiple 128-bit loads and stores per
+// iteration. Should we report a larger size and let it legalize?
+//
+// FIXME: Should we use narrower types for local/region, or account for when
+// unaligned access is legal?
+//
+// FIXME: This could use fine tuning and microbenchmarks.
+Type *GCNTTIImpl::getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length,
+                                            unsigned SrcAddrSpace,
+                                            unsigned DestAddrSpace,
+                                            unsigned SrcAlign,
+                                            unsigned DestAlign) const {
+  unsigned MinAlign = std::min(SrcAlign, DestAlign);
+
+  // A (multi-)dword access at an address == 2 (mod 4) will be decomposed by the
+  // hardware into byte accesses. If you assume all alignments are equally
+  // probable, it's more efficient on average to use short accesses for this
+  // case.
+  if (MinAlign == 2)
+    return Type::getInt16Ty(Context);
+
+  // Not all subtargets have 128-bit DS instructions, and we currently don't
+  // form them by default.
+  if (SrcAddrSpace == AMDGPUAS::LOCAL_ADDRESS ||
+      SrcAddrSpace == AMDGPUAS::REGION_ADDRESS ||
+      DestAddrSpace == AMDGPUAS::LOCAL_ADDRESS ||
+      DestAddrSpace == AMDGPUAS::REGION_ADDRESS) {
+    return VectorType::get(Type::getInt32Ty(Context), 2);
+  }
+
+  // Global memory works best with 16-byte accesses. Private memory will also
+  // hit this, although they'll be decomposed.
+  return VectorType::get(Type::getInt32Ty(Context), 4);
+}
+
+void GCNTTIImpl::getMemcpyLoopResidualLoweringType(
+  SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
+  unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
+  unsigned SrcAlign, unsigned DestAlign) const {
+  assert(RemainingBytes < 16);
+
+  unsigned MinAlign = std::min(SrcAlign, DestAlign);
+
+  if (MinAlign != 2) {
+    Type *I64Ty = Type::getInt64Ty(Context);
+    while (RemainingBytes >= 8) {
+      OpsOut.push_back(I64Ty);
+      RemainingBytes -= 8;
+    }
+
+    Type *I32Ty = Type::getInt32Ty(Context);
+    while (RemainingBytes >= 4) {
+      OpsOut.push_back(I32Ty);
+      RemainingBytes -= 4;
+    }
+  }
+
+  Type *I16Ty = Type::getInt16Ty(Context);
+  while (RemainingBytes >= 2) {
+    OpsOut.push_back(I16Ty);
+    RemainingBytes -= 2;
+  }
+
+  Type *I8Ty = Type::getInt8Ty(Context);
+  while (RemainingBytes) {
+    OpsOut.push_back(I8Ty);
+    --RemainingBytes;
+  }
+}
+
 unsigned GCNTTIImpl::getMaxInterleaveFactor(unsigned VF) {
   // Disable unrolling if the loop is not vectorized.
   // TODO: Enable this again.
@@ -734,6 +804,7 @@ bool GCNTTIImpl::isAlwaysUniform(const Value *V) const {
     case Intrinsic::amdgcn_readlane:
     case Intrinsic::amdgcn_icmp:
     case Intrinsic::amdgcn_fcmp:
+    case Intrinsic::amdgcn_ballot:
     case Intrinsic::amdgcn_if_break:
       return true;
     }
@@ -870,8 +941,8 @@ bool GCNTTIImpl::areInlineCompatible(const Function *Caller,
 
   // FIXME: dx10_clamp can just take the caller setting, but there seems to be
   // no way to support merge for backend defined attributes.
-  AMDGPU::SIModeRegisterDefaults CallerMode(*Caller, *CallerST);
-  AMDGPU::SIModeRegisterDefaults CalleeMode(*Callee, *CalleeST);
+  AMDGPU::SIModeRegisterDefaults CallerMode(*Caller);
+  AMDGPU::SIModeRegisterDefaults CalleeMode(*Callee);
   return CallerMode.isInlineCompatible(CalleeMode);
 }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index d2a25bf2fb904..1313e2f8f5765 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -133,7 +133,7 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
       TLI(ST->getTargetLowering()),
       CommonTTI(TM, F),
       IsGraphicsShader(AMDGPU::isShader(F.getCallingConv())),
-      HasFP32Denormals(ST->hasFP32Denormals(F)) { }
+      HasFP32Denormals(AMDGPU::SIModeRegisterDefaults(F).allFP32Denormals()) {}
 
   bool hasBranchDivergence() { return true; }
   bool useGPUDivergenceAnalysis() const;
@@ -167,7 +167,17 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
   bool isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes,
                                     unsigned Alignment,
                                     unsigned AddrSpace) const;
-
+  Type *getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length,
+                                  unsigned SrcAddrSpace, unsigned DestAddrSpace,
+                                  unsigned SrcAlign, unsigned DestAlign) const;
+
+  void getMemcpyLoopResidualLoweringType(SmallVectorImpl<Type *> &OpsOut,
+                                         LLVMContext &Context,
+                                         unsigned RemainingBytes,
+                                         unsigned SrcAddrSpace,
+                                         unsigned DestAddrSpace,
+                                         unsigned SrcAlign,
+                                         unsigned DestAlign) const;
   unsigned getMaxInterleaveFactor(unsigned VF);
 
   bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info) const;
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index 972d90db026a4..d9c900bb7446e 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -121,6 +121,7 @@ add_llvm_target(AMDGPUCodeGen
   SIOptimizeExecMaskingPreRA.cpp
   SIPeepholeSDWA.cpp
   SIPostRABundler.cpp
+  SIPreEmitPeephole.cpp
   SIRegisterInfo.cpp
   SIRemoveShortExecBranches.cpp
   SIShrinkInstructions.cpp
diff --git a/llvm/lib/Target/AMDGPU/GCNRegBankReassign.cpp b/llvm/lib/Target/AMDGPU/GCNRegBankReassign.cpp
index d1b9e9b648ee4..b1c64ed2e896b 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegBankReassign.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegBankReassign.cpp
@@ -636,7 +636,11 @@ unsigned GCNRegBankReassign::tryReassign(Candidate &C) {
 
   struct BankStall {
     BankStall(unsigned b, unsigned s) : Bank(b), Stalls(s) {};
-    bool operator< (const BankStall &RHS) const { return Stalls > RHS.Stalls; }
+    bool operator<(const BankStall &RHS) const {
+      if (Stalls == RHS.Stalls)
+        return Bank < RHS.Bank;
+      return Stalls > RHS.Stalls;
+    }
     unsigned Bank;
     unsigned Stalls;
   };
@@ -653,7 +657,7 @@ unsigned GCNRegBankReassign::tryReassign(Candidate &C) {
       }
     }
   }
-  std::sort(BankStalls.begin(), BankStalls.end());
+  llvm::sort(BankStalls);
 
   Register OrigReg = VRM->getPhys(C.Reg);
   LRM->unassign(LI);
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.h b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
index 909266e6e0265..2ef79410719f6 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.h
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
@@ -216,7 +216,7 @@ getLiveRegMap(Range &&R, bool After, LiveIntervals &LIS) {
     auto SI = SII.getInstructionIndex(*I);
     Indexes.push_back(After ? SI.getDeadSlot() : SI.getBaseIndex());
   }
-  std::sort(Indexes.begin(), Indexes.end());
+  llvm::sort(Indexes);
 
   auto &MRI = (*R.begin())->getParent()->getParent()->getRegInfo();
   DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet> LiveRegMap;
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h
index 90a78a9549dab..be61e3b60f714 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h
@@ -115,6 +115,10 @@ class AMDGPUInstPrinter : public MCInstPrinter {
                         raw_ostream &O);
   void printOperand(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
                     raw_ostream &O);
+  void printOperand(const MCInst *MI, uint64_t /*Address*/, unsigned OpNum,
+                    const MCSubtargetInfo &STI, raw_ostream &O) {
+    printOperand(MI, OpNum, STI, O);
+  }
   void printOperandAndFPInputMods(const MCInst *MI, unsigned OpNo,
                                   const MCSubtargetInfo &STI, raw_ostream &O);
   void printOperandAndIntInputMods(const MCInst *MI, unsigned OpNo,
diff --git a/llvm/lib/Target/AMDGPU/MIMGInstructions.td b/llvm/lib/Target/AMDGPU/MIMGInstructions.td
index 8a439425062e7..c4dbb65ee5b82 100644
--- a/llvm/lib/Target/AMDGPU/MIMGInstructions.td
+++ b/llvm/lib/Target/AMDGPU/MIMGInstructions.td
@@ -817,6 +817,11 @@ def ImageDimIntrinsicTable : GenericTable {
   let PrimaryKeyEarlyOut = 1;
 }
 
+def getImageDimInstrinsicByBaseOpcode : SearchIndex {
+  let Table = ImageDimIntrinsicTable;
+  let Key = ["BaseOpcode", "Dim"];
+}
+
 foreach intr = !listconcat(AMDGPUImageDimIntrinsics,
                            AMDGPUImageDimAtomicIntrinsics) in {
   def : ImageDimIntrinsicInfo<intr>;
diff --git a/llvm/lib/Target/AMDGPU/R600ControlFlowFinalizer.cpp b/llvm/lib/Target/AMDGPU/R600ControlFlowFinalizer.cpp
index 660893af92a0f..8124df68f6886 100644
--- a/llvm/lib/Target/AMDGPU/R600ControlFlowFinalizer.cpp
+++ b/llvm/lib/Target/AMDGPU/R600ControlFlowFinalizer.cpp
@@ -159,8 +159,7 @@ unsigned CFStack::getSubEntrySize(CFStack::StackItem Item) {
 }
 
 void CFStack::updateMaxStackSize() {
-  unsigned CurrentStackSize =
-      CurrentEntries + (alignTo(CurrentSubEntries, 4) / 4);
+  unsigned CurrentStackSize = CurrentEntries + divideCeil(CurrentSubEntries, 4);
   MaxStackSize = std::max(CurrentStackSize, MaxStackSize);
 }
 
diff --git a/llvm/lib/Target/AMDGPU/R600FrameLowering.cpp b/llvm/lib/Target/AMDGPU/R600FrameLowering.cpp
index d9aa9ebe878d8..d2fe3c9f93c66 100644
--- a/llvm/lib/Target/AMDGPU/R600FrameLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/R600FrameLowering.cpp
@@ -35,15 +35,15 @@ int R600FrameLowering::getFrameIndexReference(const MachineFunction &MF,
   int UpperBound = FI == -1 ? MFI.getNumObjects() : FI;
 
   for (int i = MFI.getObjectIndexBegin(); i < UpperBound; ++i) {
-    OffsetBytes = alignTo(OffsetBytes, MFI.getObjectAlignment(i));
+    OffsetBytes = alignTo(OffsetBytes, MFI.getObjectAlign(i));
     OffsetBytes += MFI.getObjectSize(i);
     // Each register holds 4 bytes, so we must always align the offset to at
     // least 4 bytes, so that 2 frame objects won't share the same register.
-    OffsetBytes = alignTo(OffsetBytes, 4);
+    OffsetBytes = alignTo(OffsetBytes, Align(4));
   }
 
   if (FI != -1)
-    OffsetBytes = alignTo(OffsetBytes, MFI.getObjectAlignment(FI));
+    OffsetBytes = alignTo(OffsetBytes, MFI.getObjectAlign(FI));
 
   return OffsetBytes / (getStackWidth(MF) * 4);
 }
diff --git a/llvm/lib/Target/AMDGPU/R600InstrInfo.cpp b/llvm/lib/Target/AMDGPU/R600InstrInfo.cpp
index c6cc8e8dfb526..e1da6e48eb5ed 100644
--- a/llvm/lib/Target/AMDGPU/R600InstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/R600InstrInfo.cpp
@@ -541,7 +541,7 @@ R600InstrInfo::fitsReadPortLimitations(const std::vector<MachineInstr *> &IG,
 
   std::vector<std::vector<std::pair<int, unsigned>>> IGSrcs;
   ValidSwizzle.clear();
-  unsigned ConstCount = 0;
+  unsigned ConstCount;
   BankSwizzle TransBS = ALU_VEC_012_SCL_210;
   for (unsigned i = 0, e = IG.size(); i < e; ++i) {
     IGSrcs.push_back(ExtractSrcs(*IG[i], PV, ConstCount));
diff --git a/llvm/lib/Target/AMDGPU/R600Instructions.td b/llvm/lib/Target/AMDGPU/R600Instructions.td
index 869c183e2245b..2cc21364c4397 100644
--- a/llvm/lib/Target/AMDGPU/R600Instructions.td
+++ b/llvm/lib/Target/AMDGPU/R600Instructions.td
@@ -1006,7 +1006,7 @@ class MULADD_Common <bits<5> inst> : R600_3OP <
 
 class MULADD_IEEE_Common <bits<5> inst> : R600_3OP <
   inst, "MULADD_IEEE",
-  [(set f32:$dst, (fmad f32:$src0, f32:$src1, f32:$src2))]
+  [(set f32:$dst, (any_fmad f32:$src0, f32:$src1, f32:$src2))]
 >;
 
 class FMA_Common <bits<5> inst> : R600_3OP <
diff --git a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
index f067e96b8aca2..7283c33fe9851 100644
--- a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
@@ -766,6 +766,7 @@ void SIFixSGPRCopies::processPHINode(MachineInstr &MI) {
   bool AllAGPRUses = true;
   SetVector<const MachineInstr *> worklist;
   SmallSet<const MachineInstr *, 4> Visited;
+  SetVector<MachineInstr *> PHIOperands;
   worklist.insert(&MI);
   Visited.insert(&MI);
   while (!worklist.empty()) {
@@ -810,6 +811,11 @@ void SIFixSGPRCopies::processPHINode(MachineInstr &MI) {
   if (AllAGPRUses && numVGPRUses && !TRI->hasAGPRs(RC0)) {
     LLVM_DEBUG(dbgs() << "Moving PHI to AGPR: " << MI);
     MRI->setRegClass(PHIRes, TRI->getEquivalentAGPRClass(RC0));
+    for (unsigned I = 1, N = MI.getNumOperands(); I != N; I += 2) {
+      MachineInstr *DefMI = MRI->getVRegDef(MI.getOperand(I).getReg());
+      if (DefMI && DefMI->isPHI())
+        PHIOperands.insert(DefMI);
+    }
   }
 
   bool hasVGPRInput = false;
@@ -845,4 +851,8 @@ void SIFixSGPRCopies::processPHINode(MachineInstr &MI) {
     TII->legalizeOperands(MI, MDT);
   }
 
+  // Propagate register class back to PHI operands which are PHI themselves.
+  while (!PHIOperands.empty()) {
+    processPHINode(*PHIOperands.pop_back_val());
+  }
 }
diff --git a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
index 1f2517bec6db8..46515aea708d9 100644
--- a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
@@ -94,7 +94,7 @@ static void buildPrologSpill(LivePhysRegs &LiveRegs, MachineBasicBlock &MBB,
 
   MachineMemOperand *MMO = MF->getMachineMemOperand(
       MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOStore, 4,
-      MFI.getObjectAlignment(FI));
+      MFI.getObjectAlign(FI));
 
   if (isUInt<12>(Offset)) {
     BuildMI(MBB, I, DebugLoc(), TII->get(AMDGPU::BUFFER_STORE_DWORD_OFFSET))
@@ -141,7 +141,7 @@ static void buildEpilogReload(LivePhysRegs &LiveRegs, MachineBasicBlock &MBB,
 
   MachineMemOperand *MMO = MF->getMachineMemOperand(
       MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOLoad, 4,
-      MFI.getObjectAlignment(FI));
+      MFI.getObjectAlign(FI));
 
   if (isUInt<12>(Offset)) {
     BuildMI(MBB, I, DebugLoc(),
@@ -462,9 +462,9 @@ void SIFrameLowering::emitEntryFunctionScratchRsrcRegSetup(
     const MCInstrDesc &LoadDwordX4 = TII->get(AMDGPU::S_LOAD_DWORDX4_IMM);
     auto MMO = MF.getMachineMemOperand(PtrInfo,
                                        MachineMemOperand::MOLoad |
-                                       MachineMemOperand::MOInvariant |
-                                       MachineMemOperand::MODereferenceable,
-                                       16, 4);
+                                           MachineMemOperand::MOInvariant |
+                                           MachineMemOperand::MODereferenceable,
+                                       16, Align(4));
     unsigned Offset = Fn.getCallingConv() == CallingConv::AMDGPU_CS ? 16 : 0;
     const GCNSubtarget &Subtarget = MF.getSubtarget<GCNSubtarget>();
     unsigned EncodedOffset = AMDGPU::convertSMRDOffsetUnits(Subtarget, Offset);
@@ -499,11 +499,11 @@ void SIFrameLowering::emitEntryFunctionScratchRsrcRegSetup(
         const MCInstrDesc &LoadDwordX2 = TII->get(AMDGPU::S_LOAD_DWORDX2_IMM);
 
         MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
-        auto MMO = MF.getMachineMemOperand(PtrInfo,
-                                           MachineMemOperand::MOLoad |
-                                           MachineMemOperand::MOInvariant |
-                                           MachineMemOperand::MODereferenceable,
-                                           8, 4);
+        auto MMO = MF.getMachineMemOperand(
+            PtrInfo,
+            MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
+                MachineMemOperand::MODereferenceable,
+            8, Align(4));
         BuildMI(MBB, I, DL, LoadDwordX2, Rsrc01)
           .addReg(MFI->getImplicitBufferPtrUserSGPR())
           .addImm(0) // offset
@@ -1013,9 +1013,7 @@ MachineBasicBlock::iterator SIFrameLowering::eliminateCallFramePseudoInstr(
   uint64_t CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0;
 
   if (!hasReservedCallFrame(MF)) {
-    unsigned Align = getStackAlignment();
-
-    Amount = alignTo(Amount, Align);
+    Amount = alignTo(Amount, getStackAlign());
     assert(isUInt<32>(Amount) && "exceeded stack address space size");
     const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
     unsigned SPReg = MFI->getStackPtrOffsetReg();
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index f551c69ca840b..a6846ec1ddd8d 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -887,11 +887,25 @@ unsigned SITargetLowering::getVectorTypeBreakdownForCallingConv(
     Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
 }
 
+static EVT memVTFromImageData(Type *Ty, unsigned DMaskLanes) {
+  assert(DMaskLanes != 0);
+
+  if (auto *VT = dyn_cast<VectorType>(Ty)) {
+    unsigned NumElts = std::min(DMaskLanes,
+                                static_cast<unsigned>(VT->getNumElements()));
+    return EVT::getVectorVT(Ty->getContext(),
+                            EVT::getEVT(VT->getElementType()),
+                            NumElts);
+  }
+
+  return EVT::getEVT(Ty);
+}
+
 // Peek through TFE struct returns to only use the data size.
-static EVT memVTFromImageReturn(Type *Ty) {
+static EVT memVTFromImageReturn(Type *Ty, unsigned DMaskLanes) {
   auto *ST = dyn_cast<StructType>(Ty);
   if (!ST)
-    return EVT::getEVT(Ty, true);
+    return memVTFromImageData(Ty, DMaskLanes);
 
   // Some intrinsics return an aggregate type - special case to work out the
   // correct memVT.
@@ -900,7 +914,7 @@ static EVT memVTFromImageReturn(Type *Ty) {
   if (ST->getNumContainedTypes() != 2 ||
       !ST->getContainedType(1)->isIntegerTy(32))
     return EVT();
-  return EVT::getEVT(ST->getContainedType(0));
+  return memVTFromImageData(ST->getContainedType(0), DMaskLanes);
 }
 
 bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
@@ -929,13 +943,40 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
 
     Info.flags = MachineMemOperand::MODereferenceable;
     if (Attr.hasFnAttribute(Attribute::ReadOnly)) {
+      unsigned DMaskLanes = 4;
+
+      if (RsrcIntr->IsImage) {
+        const AMDGPU::ImageDimIntrinsicInfo *Intr
+          = AMDGPU::getImageDimIntrinsicInfo(IntrID);
+        const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
+          AMDGPU::getMIMGBaseOpcodeInfo(Intr->BaseOpcode);
+
+        if (!BaseOpcode->Gather4) {
+          // If this isn't a gather, we may have excess loaded elements in the
+          // IR type. Check the dmask for the real number of elements loaded.
+          unsigned DMask
+            = cast<ConstantInt>(CI.getArgOperand(0))->getZExtValue();
+          DMaskLanes = DMask == 0 ? 1 : countPopulation(DMask);
+        }
+
+        Info.memVT = memVTFromImageReturn(CI.getType(), DMaskLanes);
+      } else
+        Info.memVT = EVT::getEVT(CI.getType());
+
+      // FIXME: What does alignment mean for an image?
       Info.opc = ISD::INTRINSIC_W_CHAIN;
-      // TODO: Account for dmask reducing loaded size.
-      Info.memVT = memVTFromImageReturn(CI.getType());
       Info.flags |= MachineMemOperand::MOLoad;
     } else if (Attr.hasFnAttribute(Attribute::WriteOnly)) {
       Info.opc = ISD::INTRINSIC_VOID;
-      Info.memVT = MVT::getVT(CI.getArgOperand(0)->getType());
+
+      Type *DataTy = CI.getArgOperand(0)->getType();
+      if (RsrcIntr->IsImage) {
+        unsigned DMask = cast<ConstantInt>(CI.getArgOperand(1))->getZExtValue();
+        unsigned DMaskLanes = DMask == 0 ? 1 : countPopulation(DMask);
+        Info.memVT = memVTFromImageData(DataTy, DMaskLanes);
+      } else
+        Info.memVT = EVT::getEVT(DataTy);
+
       Info.flags |= MachineMemOperand::MOStore;
     } else {
       // Atomic
@@ -1439,7 +1480,7 @@ SDValue SITargetLowering::convertArgType(SelectionDAG &DAG, EVT VT, EVT MemVT,
   }
 
   if (MemVT.isFloatingPoint())
-    Val = getFPExtOrFPTrunc(DAG, Val, SL, VT);
+    Val = getFPExtOrFPRound(DAG, Val, SL, VT);
   else if (Signed)
     Val = DAG.getSExtOrTrunc(Val, SL, VT);
   else
@@ -4223,6 +4264,43 @@ static SDValue lowerFCMPIntrinsic(const SITargetLowering &TLI,
   return DAG.getZExtOrTrunc(SetCC, SL, VT);
 }
 
+static SDValue lowerBALLOTIntrinsic(const SITargetLowering &TLI, SDNode *N,
+                                    SelectionDAG &DAG) {
+  EVT VT = N->getValueType(0);
+  SDValue Src = N->getOperand(1);
+  SDLoc SL(N);
+
+  if (Src.getOpcode() == ISD::SETCC) {
+    // (ballot (ISD::SETCC ...)) -> (AMDGPUISD::SETCC ...)
+    return DAG.getNode(AMDGPUISD::SETCC, SL, VT, Src.getOperand(0),
+                       Src.getOperand(1), Src.getOperand(2));
+  }
+  if (const ConstantSDNode *Arg = dyn_cast<ConstantSDNode>(Src)) {
+    // (ballot 0) -> 0
+    if (Arg->isNullValue())
+      return DAG.getConstant(0, SL, VT);
+
+    // (ballot 1) -> EXEC/EXEC_LO
+    if (Arg->isOne()) {
+      Register Exec;
+      if (VT.getScalarSizeInBits() == 32)
+        Exec = AMDGPU::EXEC_LO;
+      else if (VT.getScalarSizeInBits() == 64)
+        Exec = AMDGPU::EXEC;
+      else
+        return SDValue();
+
+      return DAG.getCopyFromReg(DAG.getEntryNode(), SL, Exec, VT);
+    }
+  }
+
+  // (ballot (i1 $src)) -> (AMDGPUISD::SETCC (i32 (zext $src)) (i32 0)
+  // ISD::SETNE)
+  return DAG.getNode(
+      AMDGPUISD::SETCC, SL, VT, DAG.getZExtOrTrunc(Src, SL, MVT::i32),
+      DAG.getConstant(0, SL, MVT::i32), DAG.getCondCode(ISD::SETNE));
+}
+
 void SITargetLowering::ReplaceNodeResults(SDNode *N,
                                           SmallVectorImpl<SDValue> &Results,
                                           SelectionDAG &DAG) const {
@@ -4538,13 +4616,14 @@ SDValue SITargetLowering::LowerRETURNADDR(SDValue Op,
   return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, VT);
 }
 
-SDValue SITargetLowering::getFPExtOrFPTrunc(SelectionDAG &DAG,
+SDValue SITargetLowering::getFPExtOrFPRound(SelectionDAG &DAG,
                                             SDValue Op,
                                             const SDLoc &DL,
                                             EVT VT) const {
   return Op.getValueType().bitsLE(VT) ?
       DAG.getNode(ISD::FP_EXTEND, DL, VT, Op) :
-      DAG.getNode(ISD::FTRUNC, DL, VT, Op);
+    DAG.getNode(ISD::FP_ROUND, DL, VT, Op,
+                DAG.getTargetConstant(0, DL, MVT::i32));
 }
 
 SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
@@ -4570,7 +4649,7 @@ SDValue SITargetLowering::lowerFMINNUM_FMAXNUM(SDValue Op,
   const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
   bool IsIEEEMode = Info->getMode().IEEE;
 
-  // FIXME: Assert during eslection that this is only selected for
+  // FIXME: Assert during selection that this is only selected for
   // ieee_mode. Currently a combine can produce the ieee version for non-ieee
   // mode functions, but this happens to be OK since it's only done in cases
   // where there is known no sNaN.
@@ -5626,14 +5705,14 @@ SDValue SITargetLowering::lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc,
   MachineFunction &MF = DAG.getMachineFunction();
 
   const DataLayout &DataLayout = DAG.getDataLayout();
-  unsigned Align =
-      DataLayout.getABITypeAlignment(VT.getTypeForEVT(*DAG.getContext()));
+  Align Alignment =
+      DataLayout.getABITypeAlign(VT.getTypeForEVT(*DAG.getContext()));
 
   MachineMemOperand *MMO = MF.getMachineMemOperand(
       MachinePointerInfo(),
       MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
           MachineMemOperand::MOInvariant,
-      VT.getStoreSize(), Align);
+      VT.getStoreSize(), Alignment);
 
   if (!Offset->isDivergent()) {
     SDValue Ops[] = {
@@ -5940,6 +6019,8 @@ SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
   case Intrinsic::amdgcn_fcmp: {
     return lowerFCMPIntrinsic(*this, Op.getNode(), DAG);
   }
+  case Intrinsic::amdgcn_ballot:
+    return lowerBALLOTIntrinsic(*this, Op.getNode(), DAG);
   case Intrinsic::amdgcn_fmed3:
     return DAG.getNode(AMDGPUISD::FMED3, DL, VT,
                        Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
@@ -6021,6 +6102,16 @@ SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
   case Intrinsic::amdgcn_alignbit:
     return DAG.getNode(ISD::FSHR, DL, VT,
                        Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
+  case Intrinsic::amdgcn_reloc_constant: {
+    Module *M = const_cast<Module *>(MF.getFunction().getParent());
+    const MDNode *Metadata = cast<MDNodeSDNode>(Op.getOperand(1))->getMD();
+    auto SymbolName = cast<MDString>(Metadata->getOperand(0))->getString();
+    auto RelocSymbol = cast<GlobalVariable>(
+        M->getOrInsertGlobal(SymbolName, Type::getInt32Ty(M->getContext())));
+    SDValue GA = DAG.getTargetGlobalAddress(RelocSymbol, DL, MVT::i32, 0,
+                                            SIInstrInfo::MO_ABS32_LO);
+    return {DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, GA), 0};
+  }
   default:
     if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
             AMDGPU::getImageDimIntrinsicInfo(IntrinsicID))
@@ -7926,7 +8017,7 @@ SDValue SITargetLowering::performUCharToFloatCombine(SDNode *N,
                                                      DAGCombinerInfo &DCI) const {
   EVT VT = N->getValueType(0);
   EVT ScalarVT = VT.getScalarType();
-  if (ScalarVT != MVT::f32)
+  if (ScalarVT != MVT::f32 && ScalarVT != MVT::f16)
     return SDValue();
 
   SelectionDAG &DAG = DCI.DAG;
@@ -7941,8 +8032,14 @@ SDValue SITargetLowering::performUCharToFloatCombine(SDNode *N,
   // about in practice.
   if (DCI.isAfterLegalizeDAG() && SrcVT == MVT::i32) {
     if (DAG.MaskedValueIsZero(Src, APInt::getHighBitsSet(32, 24))) {
-      SDValue Cvt = DAG.getNode(AMDGPUISD::CVT_F32_UBYTE0, DL, VT, Src);
+      SDValue Cvt = DAG.getNode(AMDGPUISD::CVT_F32_UBYTE0, DL, MVT::f32, Src);
       DCI.AddToWorklist(Cvt.getNode());
+
+      // For the f16 case, fold to a cast to f32 and then cast back to f16.
+      if (ScalarVT != MVT::f32) {
+        Cvt = DAG.getNode(ISD::FP_ROUND, DL, VT, Cvt,
+                          DAG.getTargetConstant(0, DL, MVT::i32));
+      }
       return Cvt;
     }
   }
@@ -10765,8 +10862,8 @@ bool SITargetLowering::isSDNodeSourceOfDivergence(const SDNode * N,
       const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
       const MachineRegisterInfo &MRI = MF->getRegInfo();
       const SIRegisterInfo &TRI = ST.getInstrInfo()->getRegisterInfo();
-      unsigned Reg = R->getReg();
-      if (Register::isPhysicalRegister(Reg))
+      Register Reg = R->getReg();
+      if (Reg.isPhysical())
         return !TRI.isSGPRReg(MRI, Reg);
 
       if (MRI.isLiveIn(Reg)) {
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index a5d18f170e66e..1e02fcdcedbf6 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -107,7 +107,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
 
   /// Converts \p Op, which must be of floating point type, to the
   /// floating point type \p VT, by either extending or truncating it.
-  SDValue getFPExtOrFPTrunc(SelectionDAG &DAG,
+  SDValue getFPExtOrFPRound(SelectionDAG &DAG,
                             SDValue Op,
                             const SDLoc &DL,
                             EVT VT) const;
diff --git a/llvm/lib/Target/AMDGPU/SIInsertSkips.cpp b/llvm/lib/Target/AMDGPU/SIInsertSkips.cpp
index 557da1e1cd3b4..de4d7904caa40 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertSkips.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertSkips.cpp
@@ -68,8 +68,6 @@ class SIInsertSkips : public MachineFunctionPass {
 
   bool skipMaskBranch(MachineInstr &MI, MachineBasicBlock &MBB);
 
-  bool optimizeVccBranch(MachineInstr &MI) const;
-
 public:
   static char ID;
 
@@ -361,98 +359,6 @@ bool SIInsertSkips::skipMaskBranch(MachineInstr &MI,
   return true;
 }
 
-bool SIInsertSkips::optimizeVccBranch(MachineInstr &MI) const {
-  // Match:
-  // sreg = -1
-  // vcc = S_AND_B64 exec, sreg
-  // S_CBRANCH_VCC[N]Z
-  // =>
-  // S_CBRANCH_EXEC[N]Z
-  bool Changed = false;
-  MachineBasicBlock &MBB = *MI.getParent();
-  const GCNSubtarget &ST = MBB.getParent()->getSubtarget<GCNSubtarget>();
-  const bool IsWave32 = ST.isWave32();
-  const unsigned CondReg = TRI->getVCC();
-  const unsigned ExecReg = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
-  const unsigned And = IsWave32 ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
-
-  MachineBasicBlock::reverse_iterator A = MI.getReverseIterator(),
-                                      E = MBB.rend();
-  bool ReadsCond = false;
-  unsigned Threshold = 5;
-  for (++A ; A != E ; ++A) {
-    if (!--Threshold)
-      return false;
-    if (A->modifiesRegister(ExecReg, TRI))
-      return false;
-    if (A->modifiesRegister(CondReg, TRI)) {
-      if (!A->definesRegister(CondReg, TRI) || A->getOpcode() != And)
-        return false;
-      break;
-    }
-    ReadsCond |= A->readsRegister(CondReg, TRI);
-  }
-  if (A == E)
-    return false;
-
-  MachineOperand &Op1 = A->getOperand(1);
-  MachineOperand &Op2 = A->getOperand(2);
-  if (Op1.getReg() != ExecReg && Op2.isReg() && Op2.getReg() == ExecReg) {
-    TII->commuteInstruction(*A);
-    Changed = true;
-  }
-  if (Op1.getReg() != ExecReg)
-    return Changed;
-  if (Op2.isImm() && Op2.getImm() != -1)
-    return Changed;
-
-  unsigned SReg = AMDGPU::NoRegister;
-  if (Op2.isReg()) {
-    SReg = Op2.getReg();
-    auto M = std::next(A);
-    bool ReadsSreg = false;
-    for ( ; M != E ; ++M) {
-      if (M->definesRegister(SReg, TRI))
-        break;
-      if (M->modifiesRegister(SReg, TRI))
-        return Changed;
-      ReadsSreg |= M->readsRegister(SReg, TRI);
-    }
-    if (M == E ||
-        !M->isMoveImmediate() ||
-        !M->getOperand(1).isImm() ||
-        M->getOperand(1).getImm() != -1)
-      return Changed;
-    // First if sreg is only used in and instruction fold the immediate
-    // into that and.
-    if (!ReadsSreg && Op2.isKill()) {
-      A->getOperand(2).ChangeToImmediate(-1);
-      M->eraseFromParent();
-    }
-  }
-
-  if (!ReadsCond && A->registerDefIsDead(AMDGPU::SCC) &&
-      MI.killsRegister(CondReg, TRI))
-    A->eraseFromParent();
-
-  bool IsVCCZ = MI.getOpcode() == AMDGPU::S_CBRANCH_VCCZ;
-  if (SReg == ExecReg) {
-    if (IsVCCZ) {
-      MI.eraseFromParent();
-      return true;
-    }
-    MI.setDesc(TII->get(AMDGPU::S_BRANCH));
-  } else {
-    MI.setDesc(TII->get(IsVCCZ ? AMDGPU::S_CBRANCH_EXECZ
-                               : AMDGPU::S_CBRANCH_EXECNZ));
-  }
-
-  MI.RemoveOperand(MI.findRegisterUseOperandIdx(CondReg, false /*Kill*/, TRI));
-  MI.addImplicitDefUseOperands(*MBB.getParent());
-
-  return true;
-}
-
 bool SIInsertSkips::runOnMachineFunction(MachineFunction &MF) {
   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
   TII = ST.getInstrInfo();
@@ -534,11 +440,6 @@ bool SIInsertSkips::runOnMachineFunction(MachineFunction &MF) {
         }
         break;
 
-      case AMDGPU::S_CBRANCH_VCCZ:
-      case AMDGPU::S_CBRANCH_VCCNZ:
-        MadeChange |= optimizeVccBranch(MI);
-        break;
-
       default:
         break;
       }
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 2e0b78bc1007f..bd3741645c1d3 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -654,7 +654,7 @@ void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
       // Registers in the sequence are allocated contiguously so we can just
       // use register number to pick one of three round-robin temps.
       unsigned RegNo = DestReg % 3;
-      unsigned Tmp = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0);
+      Register Tmp = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0);
       if (!Tmp)
         report_fatal_error("Cannot scavenge VGPR to copy to AGPR");
       RS.setRegUsed(Tmp);
@@ -812,10 +812,10 @@ SIInstrInfo::getPreferredSelectRegClass(unsigned Size) const {
 
 void SIInstrInfo::insertVectorSelect(MachineBasicBlock &MBB,
                                      MachineBasicBlock::iterator I,
-                                     const DebugLoc &DL, unsigned DstReg,
+                                     const DebugLoc &DL, Register DstReg,
                                      ArrayRef<MachineOperand> Cond,
-                                     unsigned TrueReg,
-                                     unsigned FalseReg) const {
+                                     Register TrueReg,
+                                     Register FalseReg) const {
   MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
   MachineFunction *MF = MBB.getParent();
   const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
@@ -938,10 +938,10 @@ void SIInstrInfo::insertVectorSelect(MachineBasicBlock &MBB,
   }
 }
 
-unsigned SIInstrInfo::insertEQ(MachineBasicBlock *MBB,
+Register SIInstrInfo::insertEQ(MachineBasicBlock *MBB,
                                MachineBasicBlock::iterator I,
                                const DebugLoc &DL,
-                               unsigned SrcReg, int Value) const {
+                               Register SrcReg, int Value) const {
   MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
   Register Reg = MRI.createVirtualRegister(RI.getBoolRC());
   BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg)
@@ -951,10 +951,10 @@ unsigned SIInstrInfo::insertEQ(MachineBasicBlock *MBB,
   return Reg;
 }
 
-unsigned SIInstrInfo::insertNE(MachineBasicBlock *MBB,
+Register SIInstrInfo::insertNE(MachineBasicBlock *MBB,
                                MachineBasicBlock::iterator I,
                                const DebugLoc &DL,
-                               unsigned SrcReg, int Value) const {
+                               Register SrcReg, int Value) const {
   MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
   Register Reg = MRI.createVirtualRegister(RI.getBoolRC());
   BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg)
@@ -1132,13 +1132,11 @@ void SIInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
   MachineFrameInfo &FrameInfo = MF->getFrameInfo();
   const DebugLoc &DL = MBB.findDebugLoc(MI);
 
-  unsigned Size = FrameInfo.getObjectSize(FrameIndex);
-  unsigned Align = FrameInfo.getObjectAlignment(FrameIndex);
   MachinePointerInfo PtrInfo
     = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
-  MachineMemOperand *MMO
-    = MF->getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore,
-                               Size, Align);
+  MachineMemOperand *MMO = MF->getMachineMemOperand(
+      PtrInfo, MachineMemOperand::MOStore, FrameInfo.getObjectSize(FrameIndex),
+      FrameInfo.getObjectAlign(FrameIndex));
   unsigned SpillSize = TRI->getSpillSize(*RC);
 
   if (RI.isSGPRClass(RC)) {
@@ -1260,15 +1258,14 @@ void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
   SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
   MachineFrameInfo &FrameInfo = MF->getFrameInfo();
   const DebugLoc &DL = MBB.findDebugLoc(MI);
-  unsigned Align = FrameInfo.getObjectAlignment(FrameIndex);
-  unsigned Size = FrameInfo.getObjectSize(FrameIndex);
   unsigned SpillSize = TRI->getSpillSize(*RC);
 
   MachinePointerInfo PtrInfo
     = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
 
   MachineMemOperand *MMO = MF->getMachineMemOperand(
-    PtrInfo, MachineMemOperand::MOLoad, Size, Align);
+      PtrInfo, MachineMemOperand::MOLoad, FrameInfo.getObjectSize(FrameIndex),
+      FrameInfo.getObjectAlign(FrameIndex));
 
   if (RI.isSGPRClass(RC)) {
     MFI->setHasSpilledSGPRs();
@@ -1277,7 +1274,7 @@ void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
     // FIXME: Maybe this should not include a memoperand because it will be
     // lowered to non-memory instructions.
     const MCInstrDesc &OpDesc = get(getSGPRSpillRestoreOpcode(SpillSize));
-    if (Register::isVirtualRegister(DestReg) && SpillSize == 4) {
+    if (DestReg.isVirtual() && SpillSize == 4) {
       MachineRegisterInfo &MRI = MF->getRegInfo();
       MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0RegClass);
     }
@@ -1318,7 +1315,7 @@ unsigned SIInstrInfo::calculateLDSSpillAddress(
   unsigned WorkGroupSize = MFI->getMaxFlatWorkGroupSize();
   unsigned WavefrontSize = ST.getWavefrontSize();
 
-  unsigned TIDReg = MFI->getTIDReg();
+  Register TIDReg = MFI->getTIDReg();
   if (!MFI->hasCalculatedTID()) {
     MachineBasicBlock &Entry = MBB.getParent()->front();
     MachineBasicBlock::iterator Insert = Entry.front();
@@ -1346,8 +1343,8 @@ unsigned SIInstrInfo::calculateLDSSpillAddress(
 
       RS->enterBasicBlock(Entry);
       // FIXME: Can we scavenge an SReg_64 and access the subregs?
-      unsigned STmp0 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
-      unsigned STmp1 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
+      Register STmp0 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
+      Register STmp1 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
       BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp0)
               .addReg(InputPtrReg)
               .addImm(SI::KernelInputOffsets::NGROUPS_Z);
@@ -2208,8 +2205,8 @@ bool SIInstrInfo::reverseBranchCondition(
 
 bool SIInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
                                   ArrayRef<MachineOperand> Cond,
-                                  unsigned DstReg, unsigned TrueReg,
-                                  unsigned FalseReg, int &CondCycles,
+                                  Register DstReg, Register TrueReg,
+                                  Register FalseReg, int &CondCycles,
                                   int &TrueCycles, int &FalseCycles) const {
   switch (Cond[0].getImm()) {
   case VCCNZ:
@@ -2248,8 +2245,8 @@ bool SIInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
 
 void SIInstrInfo::insertSelect(MachineBasicBlock &MBB,
                                MachineBasicBlock::iterator I, const DebugLoc &DL,
-                               unsigned DstReg, ArrayRef<MachineOperand> Cond,
-                               unsigned TrueReg, unsigned FalseReg) const {
+                               Register DstReg, ArrayRef<MachineOperand> Cond,
+                               Register TrueReg, Register FalseReg) const {
   BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());
   if (Pred == VCCZ || Pred == SCC_FALSE) {
     Pred = static_cast<BranchPredicate>(-Pred);
@@ -2322,7 +2319,7 @@ void SIInstrInfo::insertSelect(MachineBasicBlock &MBB,
 
   I = MIB->getIterator();
 
-  SmallVector<unsigned, 8> Regs;
+  SmallVector<Register, 8> Regs;
   for (int Idx = 0; Idx != NElts; ++Idx) {
     Register DstElt = MRI.createVirtualRegister(EltRC);
     Regs.push_back(DstElt);
@@ -2396,7 +2393,7 @@ static void removeModOperands(MachineInstr &MI) {
 }
 
 bool SIInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
-                                unsigned Reg, MachineRegisterInfo *MRI) const {
+                                Register Reg, MachineRegisterInfo *MRI) const {
   if (!MRI->hasOneNonDBGUse(Reg))
     return false;
 
@@ -3218,7 +3215,7 @@ bool SIInstrInfo::usesConstantBus(const MachineRegisterInfo &MRI,
   }
 }
 
-static unsigned findImplicitSGPRRead(const MachineInstr &MI) {
+static Register findImplicitSGPRRead(const MachineInstr &MI) {
   for (const MachineOperand &MO : MI.implicit_operands()) {
     // We only care about reads.
     if (MO.isDef())
@@ -3526,8 +3523,8 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
     if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1)
       ++ConstantBusCount;
 
-    SmallVector<unsigned, 2> SGPRsUsed;
-    unsigned SGPRUsed = findImplicitSGPRRead(MI);
+    SmallVector<Register, 2> SGPRsUsed;
+    Register SGPRUsed = findImplicitSGPRRead(MI);
     if (SGPRUsed != AMDGPU::NoRegister) {
       ++ConstantBusCount;
       SGPRsUsed.push_back(SGPRUsed);
@@ -4319,7 +4316,7 @@ void SIInstrInfo::legalizeOperandsVOP3(MachineRegisterInfo &MRI,
   }
 }
 
-unsigned SIInstrInfo::readlaneVGPRToSGPR(unsigned SrcReg, MachineInstr &UseMI,
+Register SIInstrInfo::readlaneVGPRToSGPR(Register SrcReg, MachineInstr &UseMI,
                                          MachineRegisterInfo &MRI) const {
   const TargetRegisterClass *VRC = MRI.getRegClass(SrcReg);
   const TargetRegisterClass *SRC = RI.getEquivalentSGPRClass(VRC);
@@ -5725,7 +5722,7 @@ void SIInstrInfo::splitScalar64BitBFE(SetVectorType &Worklist,
 }
 
 void SIInstrInfo::addUsersToMoveToVALUWorklist(
-  unsigned DstReg,
+  Register DstReg,
   MachineRegisterInfo &MRI,
   SetVectorType &Worklist) const {
   for (MachineRegisterInfo::use_iterator I = MRI.use_begin(DstReg),
@@ -5891,7 +5888,7 @@ const TargetRegisterClass *SIInstrInfo::getDestEquivalentVGPRClass(
 }
 
 // Find the one SGPR operand we are allowed to use.
-unsigned SIInstrInfo::findUsedSGPR(const MachineInstr &MI,
+Register SIInstrInfo::findUsedSGPR(const MachineInstr &MI,
                                    int OpIndices[3]) const {
   const MCInstrDesc &Desc = MI.getDesc();
 
@@ -5904,11 +5901,11 @@ unsigned SIInstrInfo::findUsedSGPR(const MachineInstr &MI,
   //
   // If the operand's class is an SGPR, we can never move it.
 
-  unsigned SGPRReg = findImplicitSGPRRead(MI);
+  Register SGPRReg = findImplicitSGPRRead(MI);
   if (SGPRReg != AMDGPU::NoRegister)
     return SGPRReg;
 
-  unsigned UsedSGPRs[3] = { AMDGPU::NoRegister };
+  Register UsedSGPRs[3] = { AMDGPU::NoRegister };
   const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
 
   for (unsigned i = 0; i < 3; ++i) {
@@ -6299,7 +6296,7 @@ MachineInstrBuilder
 SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB,
                            MachineBasicBlock::iterator I,
                            const DebugLoc &DL,
-                           unsigned DestReg) const {
+                           Register DestReg) const {
   if (ST.hasAddNoCarry())
     return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e64), DestReg);
 
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index d23283e67b644..9fcc5caf7dfdd 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -119,7 +119,7 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
                       MachineRegisterInfo &MRI,
                       MachineInstr &Inst) const;
 
-  void addUsersToMoveToVALUWorklist(unsigned Reg, MachineRegisterInfo &MRI,
+  void addUsersToMoveToVALUWorklist(Register Reg, MachineRegisterInfo &MRI,
                                     SetVectorType &Worklist) const;
 
   void addSCCDefUsersToVALUWorklist(MachineOperand &Op,
@@ -132,7 +132,7 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
   bool checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
                                     const MachineInstr &MIb) const;
 
-  unsigned findUsedSGPR(const MachineInstr &MI, int OpIndices[3]) const;
+  Register findUsedSGPR(const MachineInstr &MI, int OpIndices[3]) const;
 
 protected:
   bool swapSourceModifiers(MachineInstr &MI,
@@ -211,13 +211,13 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
   const TargetRegisterClass *getPreferredSelectRegClass(
                                unsigned Size) const;
 
-  unsigned insertNE(MachineBasicBlock *MBB,
+  Register insertNE(MachineBasicBlock *MBB,
                     MachineBasicBlock::iterator I, const DebugLoc &DL,
-                    unsigned SrcReg, int Value) const;
+                    Register SrcReg, int Value) const;
 
-  unsigned insertEQ(MachineBasicBlock *MBB,
+  Register insertEQ(MachineBasicBlock *MBB,
                     MachineBasicBlock::iterator I, const DebugLoc &DL,
-                    unsigned SrcReg, int Value)  const;
+                    Register SrcReg, int Value)  const;
 
   void storeRegToStackSlot(MachineBasicBlock &MBB,
                            MachineBasicBlock::iterator MI, Register SrcReg,
@@ -297,19 +297,19 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
     SmallVectorImpl<MachineOperand> &Cond) const override;
 
   bool canInsertSelect(const MachineBasicBlock &MBB,
-                       ArrayRef<MachineOperand> Cond, unsigned DstReg,
-                       unsigned TrueReg, unsigned FalseReg, int &CondCycles,
+                       ArrayRef<MachineOperand> Cond, Register DstReg,
+                       Register TrueReg, Register FalseReg, int &CondCycles,
                        int &TrueCycles, int &FalseCycles) const override;
 
   void insertSelect(MachineBasicBlock &MBB,
                     MachineBasicBlock::iterator I, const DebugLoc &DL,
-                    unsigned DstReg, ArrayRef<MachineOperand> Cond,
-                    unsigned TrueReg, unsigned FalseReg) const override;
+                    Register DstReg, ArrayRef<MachineOperand> Cond,
+                    Register TrueReg, Register FalseReg) const override;
 
   void insertVectorSelect(MachineBasicBlock &MBB,
                           MachineBasicBlock::iterator I, const DebugLoc &DL,
-                          unsigned DstReg, ArrayRef<MachineOperand> Cond,
-                          unsigned TrueReg, unsigned FalseReg) const;
+                          Register DstReg, ArrayRef<MachineOperand> Cond,
+                          Register TrueReg, Register FalseReg) const;
 
   unsigned getAddressSpaceForPseudoSourceKind(
              unsigned Kind) const override;
@@ -320,7 +320,7 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
 
   bool isFoldableCopy(const MachineInstr &MI) const;
 
-  bool FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, unsigned Reg,
+  bool FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg,
                      MachineRegisterInfo *MRI) const final;
 
   unsigned getMachineCSELookAheadLimit() const override { return 500; }
@@ -877,7 +877,7 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
   /// be used when it is know that the value in SrcReg is same across all
   /// threads in the wave.
   /// \returns The SGPR register that \p SrcReg was copied to.
-  unsigned readlaneVGPRToSGPR(unsigned SrcReg, MachineInstr &UseMI,
+  Register readlaneVGPRToSGPR(Register SrcReg, MachineInstr &UseMI,
                               MachineRegisterInfo &MRI) const;
 
   void legalizeOperandsSMRD(MachineRegisterInfo &MRI, MachineInstr &MI) const;
@@ -998,7 +998,7 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
   MachineInstrBuilder getAddNoCarry(MachineBasicBlock &MBB,
                                     MachineBasicBlock::iterator I,
                                     const DebugLoc &DL,
-                                    unsigned DestReg) const;
+                                    Register DestReg) const;
 
   MachineInstrBuilder getAddNoCarry(MachineBasicBlock &MBB,
                                     MachineBasicBlock::iterator I,
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index e80061bb40cd3..6e6b2e77b0fe2 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -779,14 +779,14 @@ def : RsqPat<V_RSQ_F64_e32, f64>;
 def : GCNPat <
   (f32 (fsub (f32 (VOP3Mods f32:$x, i32:$mods)),
              (f32 (ffloor (f32 (VOP3Mods f32:$x, i32:$mods)))))),
-  (V_FRACT_F32_e64 $mods, $x, DSTCLAMP.NONE, DSTOMOD.NONE)
+  (V_FRACT_F32_e64 $mods, $x)
 >;
 
 // Convert (x + (-floor(x))) to fract(x)
 def : GCNPat <
   (f64 (fadd (f64 (VOP3Mods f64:$x, i32:$mods)),
              (f64 (fneg (f64 (ffloor (f64 (VOP3Mods f64:$x, i32:$mods)))))))),
-  (V_FRACT_F64_e64 $mods, $x, DSTCLAMP.NONE, DSTOMOD.NONE)
+  (V_FRACT_F64_e64 $mods, $x)
 >;
 
 } // End OtherPredicates = [UnsafeFPMath]
@@ -795,27 +795,27 @@ def : GCNPat <
 // f16_to_fp patterns
 def : GCNPat <
   (f32 (f16_to_fp i32:$src0)),
-  (V_CVT_F32_F16_e64 SRCMODS.NONE, $src0, DSTCLAMP.NONE, DSTOMOD.NONE)
+  (V_CVT_F32_F16_e64 SRCMODS.NONE, $src0)
 >;
 
 def : GCNPat <
   (f32 (f16_to_fp (and_oneuse i32:$src0, 0x7fff))),
-  (V_CVT_F32_F16_e64 SRCMODS.ABS, $src0, DSTCLAMP.NONE, DSTOMOD.NONE)
+  (V_CVT_F32_F16_e64 SRCMODS.ABS, $src0)
 >;
 
 def : GCNPat <
   (f32 (f16_to_fp (i32 (srl_oneuse (and_oneuse i32:$src0, 0x7fff0000), (i32 16))))),
-  (V_CVT_F32_F16_e64 SRCMODS.ABS, (i32 (V_LSHRREV_B32_e64 (i32 16), i32:$src0)), DSTCLAMP.NONE, DSTOMOD.NONE)
+  (V_CVT_F32_F16_e64 SRCMODS.ABS, (i32 (V_LSHRREV_B32_e64 (i32 16), i32:$src0)))
 >;
 
 def : GCNPat <
   (f32 (f16_to_fp (or_oneuse i32:$src0, 0x8000))),
-  (V_CVT_F32_F16_e64 SRCMODS.NEG_ABS, $src0, DSTCLAMP.NONE, DSTOMOD.NONE)
+  (V_CVT_F32_F16_e64 SRCMODS.NEG_ABS, $src0)
 >;
 
 def : GCNPat <
   (f32 (f16_to_fp (xor_oneuse i32:$src0, 0x8000))),
-  (V_CVT_F32_F16_e64 SRCMODS.NEG, $src0, DSTCLAMP.NONE, DSTOMOD.NONE)
+  (V_CVT_F32_F16_e64 SRCMODS.NEG, $src0)
 >;
 
 def : GCNPat <
@@ -826,7 +826,7 @@ def : GCNPat <
 // fp_to_fp16 patterns
 def : GCNPat <
   (i32 (AMDGPUfp_to_f16 (f32 (VOP3Mods f32:$src0, i32:$src0_modifiers)))),
-  (V_CVT_F16_F32_e64 $src0_modifiers, f32:$src0, DSTCLAMP.NONE, DSTOMOD.NONE)
+  (V_CVT_F16_F32_e64 $src0_modifiers, f32:$src0)
 >;
 
 def : GCNPat <
@@ -1870,12 +1870,12 @@ def : GCNPat <
 let OtherPredicates = [NoFP16Denormals] in {
 def : GCNPat<
   (fcanonicalize (f16 (VOP3Mods f16:$src, i32:$src_mods))),
-  (V_MUL_F16_e64 0, (i32 CONST.FP16_ONE), $src_mods, $src, 0, 0)
+  (V_MUL_F16_e64 0, (i32 CONST.FP16_ONE), $src_mods, $src)
 >;
 
 def : GCNPat<
   (fcanonicalize (f16 (fneg (VOP3Mods f16:$src, i32:$src_mods)))),
-  (V_MUL_F16_e64 0, (i32 CONST.FP16_NEG_ONE), $src_mods, $src, 0, 0)
+  (V_MUL_F16_e64 0, (i32 CONST.FP16_NEG_ONE), $src_mods, $src)
 >;
 
 def : GCNPat<
@@ -1901,53 +1901,53 @@ def : GCNPat<
 let OtherPredicates = [NoFP32Denormals] in {
 def : GCNPat<
   (fcanonicalize (f32 (VOP3Mods f32:$src, i32:$src_mods))),
-  (V_MUL_F32_e64 0, (i32 CONST.FP32_ONE), $src_mods, $src, 0, 0)
+  (V_MUL_F32_e64 0, (i32 CONST.FP32_ONE), $src_mods, $src)
 >;
 
 def : GCNPat<
   (fcanonicalize (f32 (fneg (VOP3Mods f32:$src, i32:$src_mods)))),
-  (V_MUL_F32_e64 0, (i32 CONST.FP32_NEG_ONE), $src_mods, $src, 0, 0)
+  (V_MUL_F32_e64 0, (i32 CONST.FP32_NEG_ONE), $src_mods, $src)
 >;
 }
 
 let OtherPredicates = [FP32Denormals] in {
 def : GCNPat<
   (fcanonicalize (f32 (VOP3Mods f32:$src, i32:$src_mods))),
-  (V_MAX_F32_e64 $src_mods, $src, $src_mods, $src, 0, 0)
+  (V_MAX_F32_e64 $src_mods, $src, $src_mods, $src)
 >;
 }
 
 let OtherPredicates = [NoFP64Denormals] in {
 def : GCNPat<
   (fcanonicalize (f64 (VOP3Mods f64:$src, i32:$src_mods))),
-  (V_MUL_F64 0, CONST.FP64_ONE, $src_mods, $src, 0, 0)
+  (V_MUL_F64 0, CONST.FP64_ONE, $src_mods, $src)
 >;
 }
 
 let OtherPredicates = [FP64Denormals] in {
 def : GCNPat<
   (fcanonicalize (f64 (VOP3Mods f64:$src, i32:$src_mods))),
-  (V_MAX_F64 $src_mods, $src, $src_mods, $src, 0, 0)
+  (V_MAX_F64 $src_mods, $src, $src_mods, $src)
 >;
 }
 
 let OtherPredicates = [HasDLInsts] in {
 def : GCNPat <
-  (fma (f32 (VOP3Mods0 f32:$src0, i32:$src0_modifiers, i1:$clamp, i32:$omod)),
+  (fma (f32 (VOP3Mods f32:$src0, i32:$src0_modifiers)),
        (f32 (VOP3Mods f32:$src1, i32:$src1_modifiers)),
        (f32 (VOP3NoMods f32:$src2))),
   (V_FMAC_F32_e64 $src0_modifiers, $src0, $src1_modifiers, $src1,
-                  SRCMODS.NONE, $src2, $clamp, $omod)
+                  SRCMODS.NONE, $src2)
 >;
 } // End OtherPredicates = [HasDLInsts]
 
 let SubtargetPredicate = isGFX10Plus in
 def : GCNPat <
-  (fma (f16 (VOP3Mods0 f32:$src0, i32:$src0_modifiers, i1:$clamp, i32:$omod)),
+  (fma (f16 (VOP3Mods f32:$src0, i32:$src0_modifiers)),
        (f16 (VOP3Mods f32:$src1, i32:$src1_modifiers)),
        (f16 (VOP3NoMods f32:$src2))),
   (V_FMAC_F16_e64 $src0_modifiers, $src0, $src1_modifiers, $src1,
-                  SRCMODS.NONE, $src2, $clamp, $omod)
+                  SRCMODS.NONE, $src2)
 >;
 
 // COPY is workaround tablegen bug from multiple outputs
@@ -2075,13 +2075,11 @@ def : GCNPat <
       (V_CNDMASK_B64_PSEUDO
          (V_MIN_F64
              SRCMODS.NONE,
-             (V_FRACT_F64_e64 $mods, $x, DSTCLAMP.NONE, DSTOMOD.NONE),
+             (V_FRACT_F64_e64 $mods, $x),
              SRCMODS.NONE,
-             (V_MOV_B64_PSEUDO 0x3fefffffffffffff),
-             DSTCLAMP.NONE, DSTOMOD.NONE),
+             (V_MOV_B64_PSEUDO 0x3fefffffffffffff)),
          $x,
-         (V_CMP_CLASS_F64_e64 SRCMODS.NONE, $x, (i32 3 /*NaN*/))),
-      DSTCLAMP.NONE, DSTOMOD.NONE)
+         (V_CMP_CLASS_F64_e64 SRCMODS.NONE, $x, (i32 3 /*NaN*/))))
 >;
 
 } // End SubtargetPredicates = isGFX6
@@ -2299,6 +2297,14 @@ def G_AMDGPU_FMAX_LEGACY : AMDGPUGenericInstruction {
   let hasSideEffects = 0;
 }
 
+foreach N = 0-3 in {
+def G_AMDGPU_CVT_F32_UBYTE#N : AMDGPUGenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src0);
+  let hasSideEffects = 0;
+}
+}
+
 // Atomic cmpxchg. $cmpval ad $newval are packed in a single vector
 // operand Expects a MachineMemOperand in addition to explicit
 // operands.
diff --git a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
index dd94a29a9e501..a2b42fcf70210 100644
--- a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
@@ -884,8 +884,19 @@ bool SILoadStoreOptimizer::checkAndPrepareMerge(
 
   MachineBasicBlock::iterator E = std::next(Paired.I);
   MachineBasicBlock::iterator MBBI = std::next(CI.I);
+  MachineBasicBlock::iterator MBBE = CI.I->getParent()->end();
   for (; MBBI != E; ++MBBI) {
 
+    if (MBBI == MBBE) {
+      // CombineInfo::Order is a hint on the instruction ordering within the
+      // basic block. This hint suggests that CI precedes Paired, which is
+      // true most of the time. However, moveInstsAfter() processing a
+      // previous list may have changed this order in a situation when it
+      // moves an instruction which exists in some other merge list.
+      // In this case it must be dependent.
+      return false;
+    }
+
     if ((getInstClass(MBBI->getOpcode(), *TII) != InstClass) ||
         (getInstSubclass(MBBI->getOpcode(), *TII) != InstSubclass)) {
       // This is not a matching instruction, but we can keep looking as
diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
index f17608ad972a7..58f66d3a394c0 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
@@ -323,7 +323,7 @@ bool SIMachineFunctionInfo::allocateSGPRSpillToVGPR(MachineFunction &MF,
       Optional<int> CSRSpillFI;
       if ((FrameInfo.hasCalls() || !isEntryFunction()) && CSRegs &&
           isCalleeSavedReg(CSRegs, LaneVGPR)) {
-        CSRSpillFI = FrameInfo.CreateSpillStackObject(4, 4);
+        CSRSpillFI = FrameInfo.CreateSpillStackObject(4, Align(4));
       }
 
       SpillVGPRs.push_back(SGPRSpillVGPRCSR(LaneVGPR, CSRSpillFI));
diff --git a/llvm/lib/Target/AMDGPU/SIPostRABundler.cpp b/llvm/lib/Target/AMDGPU/SIPostRABundler.cpp
index acb0fbfe812d6..3a64d519d105d 100644
--- a/llvm/lib/Target/AMDGPU/SIPostRABundler.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPostRABundler.cpp
@@ -76,7 +76,7 @@ bool SIPostRABundler::isDependentLoad(const MachineInstr &MI) const {
     if (!Op.isReg())
       continue;
     Register Reg = Op.getReg();
-    for (const Register Def : Defs)
+    for (Register Def : Defs)
       if (TRI->regsOverlap(Reg, Def))
         return true;
   }
diff --git a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
new file mode 100644
index 0000000000000..2c3619811c348
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
@@ -0,0 +1,169 @@
+//===-- SIPreEmitPeephole.cpp ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// This pass performs the peephole optimizations before code emission.
+///
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPU.h"
+#include "AMDGPUSubtarget.h"
+#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
+#include "SIInstrInfo.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/Support/CommandLine.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "si-pre-emit-peephole"
+
+namespace {
+
+class SIPreEmitPeephole : public MachineFunctionPass {
+private:
+  const SIInstrInfo *TII = nullptr;
+  const SIRegisterInfo *TRI = nullptr;
+
+  bool optimizeVccBranch(MachineInstr &MI) const;
+
+public:
+  static char ID;
+
+  SIPreEmitPeephole() : MachineFunctionPass(ID) {
+    initializeSIPreEmitPeepholePass(*PassRegistry::getPassRegistry());
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+};
+
+} // End anonymous namespace.
+
+INITIALIZE_PASS(SIPreEmitPeephole, DEBUG_TYPE,
+                "SI peephole optimizations", false, false)
+
+char SIPreEmitPeephole::ID = 0;
+
+char &llvm::SIPreEmitPeepholeID = SIPreEmitPeephole::ID;
+
+bool SIPreEmitPeephole::optimizeVccBranch(MachineInstr &MI) const {
+  // Match:
+  // sreg = -1
+  // vcc = S_AND_B64 exec, sreg
+  // S_CBRANCH_VCC[N]Z
+  // =>
+  // S_CBRANCH_EXEC[N]Z
+  // We end up with this pattern sometimes after basic block placement.
+  // It happens while combining a block which assigns -1 to a saved mask and
+  // another block which consumes that saved mask and then a branch.
+  bool Changed = false;
+  MachineBasicBlock &MBB = *MI.getParent();
+  const GCNSubtarget &ST = MBB.getParent()->getSubtarget<GCNSubtarget>();
+  const bool IsWave32 = ST.isWave32();
+  const unsigned CondReg = TRI->getVCC();
+  const unsigned ExecReg = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
+  const unsigned And = IsWave32 ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
+
+  MachineBasicBlock::reverse_iterator A = MI.getReverseIterator(),
+                                      E = MBB.rend();
+  bool ReadsCond = false;
+  unsigned Threshold = 5;
+  for (++A; A != E; ++A) {
+    if (!--Threshold)
+      return false;
+    if (A->modifiesRegister(ExecReg, TRI))
+      return false;
+    if (A->modifiesRegister(CondReg, TRI)) {
+      if (!A->definesRegister(CondReg, TRI) || A->getOpcode() != And)
+        return false;
+      break;
+    }
+    ReadsCond |= A->readsRegister(CondReg, TRI);
+  }
+  if (A == E)
+    return false;
+
+  MachineOperand &Op1 = A->getOperand(1);
+  MachineOperand &Op2 = A->getOperand(2);
+  if (Op1.getReg() != ExecReg && Op2.isReg() && Op2.getReg() == ExecReg) {
+    TII->commuteInstruction(*A);
+    Changed = true;
+  }
+  if (Op1.getReg() != ExecReg)
+    return Changed;
+  if (Op2.isImm() && Op2.getImm() != -1)
+    return Changed;
+
+  Register SReg;
+  if (Op2.isReg()) {
+    SReg = Op2.getReg();
+    auto M = std::next(A);
+    bool ReadsSreg = false;
+    for (; M != E; ++M) {
+      if (M->definesRegister(SReg, TRI))
+        break;
+      if (M->modifiesRegister(SReg, TRI))
+        return Changed;
+      ReadsSreg |= M->readsRegister(SReg, TRI);
+    }
+    if (M == E || !M->isMoveImmediate() || !M->getOperand(1).isImm() ||
+        M->getOperand(1).getImm() != -1)
+      return Changed;
+    // First if sreg is only used in and instruction fold the immediate
+    // into that and.
+    if (!ReadsSreg && Op2.isKill()) {
+      A->getOperand(2).ChangeToImmediate(-1);
+      M->eraseFromParent();
+    }
+  }
+
+  if (!ReadsCond && A->registerDefIsDead(AMDGPU::SCC) &&
+      MI.killsRegister(CondReg, TRI))
+    A->eraseFromParent();
+
+  bool IsVCCZ = MI.getOpcode() == AMDGPU::S_CBRANCH_VCCZ;
+  if (SReg == ExecReg) {
+    if (IsVCCZ) {
+      MI.eraseFromParent();
+      return true;
+    }
+    MI.setDesc(TII->get(AMDGPU::S_BRANCH));
+  } else {
+    MI.setDesc(
+        TII->get(IsVCCZ ? AMDGPU::S_CBRANCH_EXECZ : AMDGPU::S_CBRANCH_EXECNZ));
+  }
+
+  MI.RemoveOperand(MI.findRegisterUseOperandIdx(CondReg, false /*Kill*/, TRI));
+  MI.addImplicitDefUseOperands(*MBB.getParent());
+
+  return true;
+}
+
+bool SIPreEmitPeephole::runOnMachineFunction(MachineFunction &MF) {
+  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+  TII = ST.getInstrInfo();
+  TRI = &TII->getRegisterInfo();
+  bool Changed = false;
+
+  for (MachineBasicBlock &MBB : MF) {
+    MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
+    if (MBBI == MBB.end())
+      continue;
+
+    MachineInstr &MI = *MBBI;
+    switch (MI.getOpcode()) {
+    case AMDGPU::S_CBRANCH_VCCZ:
+    case AMDGPU::S_CBRANCH_VCCNZ:
+      Changed |= optimizeVccBranch(MI);
+      break;
+    default:
+      break;
+    }
+  }
+
+  return Changed;
+}
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 93fde905beee9..06d05d349f366 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -40,10 +40,23 @@ static cl::opt<bool> EnableSpillSGPRToVGPR(
 
 SIRegisterInfo::SIRegisterInfo(const GCNSubtarget &ST)
     : AMDGPUGenRegisterInfo(AMDGPU::PC_REG, ST.getAMDGPUDwarfFlavour()), ST(ST),
-      SpillSGPRToVGPR(EnableSpillSGPRToVGPR), isWave32(ST.isWave32()) {}
+      SpillSGPRToVGPR(EnableSpillSGPRToVGPR), isWave32(ST.isWave32()) {
+
+  assert(getSubRegIndexLaneMask(AMDGPU::sub0).getAsInteger() == 3 &&
+         getSubRegIndexLaneMask(AMDGPU::sub31).getAsInteger() == (3ULL << 62) &&
+         (getSubRegIndexLaneMask(AMDGPU::lo16) |
+          getSubRegIndexLaneMask(AMDGPU::hi16)).getAsInteger() ==
+           getSubRegIndexLaneMask(AMDGPU::sub0).getAsInteger() &&
+         "getNumCoveredRegs() will not work with generated subreg masks!");
+
+  RegPressureIgnoredUnits.resize(getNumRegUnits());
+  RegPressureIgnoredUnits.set(*MCRegUnitIterator(AMDGPU::M0, this));
+  for (auto Reg : AMDGPU::VGPR_HI16RegClass)
+    RegPressureIgnoredUnits.set(*MCRegUnitIterator(Reg, this));
+}
 
 void SIRegisterInfo::reserveRegisterTuples(BitVector &Reserved,
-                                           unsigned Reg) const {
+                                           MCRegister Reg) const {
   MCRegAliasIterator R(Reg, this, true);
 
   for (; R.isValid(); ++R)
@@ -174,10 +187,10 @@ unsigned SIRegisterInfo::getSubRegFromChannel(unsigned Channel,
   return SubRegFromChannelTable[NumRegIndex][Channel];
 }
 
-unsigned SIRegisterInfo::reservedPrivateSegmentBufferReg(
+MCRegister SIRegisterInfo::reservedPrivateSegmentBufferReg(
   const MachineFunction &MF) const {
   unsigned BaseIdx = alignDown(ST.getMaxNumSGPRs(MF), 4) - 4;
-  unsigned BaseReg(AMDGPU::SGPR_32RegClass.getRegister(BaseIdx));
+  MCRegister BaseReg(AMDGPU::SGPR_32RegClass.getRegister(BaseIdx));
   return getMatchingSuperReg(BaseReg, AMDGPU::sub0, &AMDGPU::SGPR_128RegClass);
 }
 
@@ -271,20 +284,20 @@ BitVector SIRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
   // We have to assume the SP is needed in case there are calls in the function,
   // which is detected after the function is lowered. If we aren't really going
   // to need SP, don't bother reserving it.
-  unsigned StackPtrReg = MFI->getStackPtrOffsetReg();
+  MCRegister StackPtrReg = MFI->getStackPtrOffsetReg();
 
-  if (StackPtrReg != AMDGPU::NoRegister) {
+  if (StackPtrReg) {
     reserveRegisterTuples(Reserved, StackPtrReg);
     assert(!isSubRegister(ScratchRSrcReg, StackPtrReg));
   }
 
-  unsigned FrameReg = MFI->getFrameOffsetReg();
-  if (FrameReg != AMDGPU::NoRegister) {
+  MCRegister FrameReg = MFI->getFrameOffsetReg();
+  if (FrameReg) {
     reserveRegisterTuples(Reserved, FrameReg);
     assert(!isSubRegister(ScratchRSrcReg, FrameReg));
   }
 
-  for (unsigned Reg : MFI->WWMReservedRegs) {
+  for (MCRegister Reg : MFI->WWMReservedRegs) {
     reserveRegisterTuples(Reserved, Reg);
   }
 
@@ -647,10 +660,10 @@ static bool buildMUBUFOffsetLoadStore(const GCNSubtarget &ST,
 void SIRegisterInfo::buildSpillLoadStore(MachineBasicBlock::iterator MI,
                                          unsigned LoadStoreOp,
                                          int Index,
-                                         unsigned ValueReg,
+                                         Register ValueReg,
                                          bool IsKill,
-                                         unsigned ScratchRsrcReg,
-                                         unsigned ScratchOffsetReg,
+                                         MCRegister ScratchRsrcReg,
+                                         MCRegister ScratchOffsetReg,
                                          int64_t InstOffset,
                                          MachineMemOperand *MMO,
                                          RegScavenger *RS) const {
@@ -664,7 +677,7 @@ void SIRegisterInfo::buildSpillLoadStore(MachineBasicBlock::iterator MI,
   bool IsStore = Desc.mayStore();
 
   bool Scavenged = false;
-  unsigned SOffset = ScratchOffsetReg;
+  MCRegister SOffset = ScratchOffsetReg;
 
   const unsigned EltSize = 4;
   const TargetRegisterClass *RC = getRegClassForReg(MF->getRegInfo(), ValueReg);
@@ -673,7 +686,7 @@ void SIRegisterInfo::buildSpillLoadStore(MachineBasicBlock::iterator MI,
   int64_t Offset = InstOffset + MFI.getObjectOffset(Index);
   int64_t ScratchOffsetRegDelta = 0;
 
-  unsigned Align = MFI.getObjectAlignment(Index);
+  Align Alignment = MFI.getObjectAlign(Index);
   const MachinePointerInfo &BasePtrInfo = MMO->getPointerInfo();
 
   Register TmpReg =
@@ -683,7 +696,7 @@ void SIRegisterInfo::buildSpillLoadStore(MachineBasicBlock::iterator MI,
   assert((Offset % EltSize) == 0 && "unexpected VGPR spill offset");
 
   if (!isUInt<12>(Offset + Size - EltSize)) {
-    SOffset = AMDGPU::NoRegister;
+    SOffset = MCRegister();
 
     // We currently only support spilling VGPRs to EltSize boundaries, meaning
     // we can simplify the adjustment of Offset here to just scale with
@@ -695,8 +708,8 @@ void SIRegisterInfo::buildSpillLoadStore(MachineBasicBlock::iterator MI,
     if (RS)
       SOffset = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, MI, 0, false);
 
-    if (SOffset == AMDGPU::NoRegister) {
-      if (ScratchOffsetReg == AMDGPU::NoRegister) {
+    if (!SOffset) {
+      if (!ScratchOffsetReg) {
         report_fatal_error("could not scavenge SGPR to spill in entry function");
       }
       // There are no free SGPRs, and since we are in the process of spilling
@@ -749,9 +762,9 @@ void SIRegisterInfo::buildSpillLoadStore(MachineBasicBlock::iterator MI,
       }
 
       MachinePointerInfo PInfo = BasePtrInfo.getWithOffset(EltSize * i);
-      MachineMemOperand *NewMMO
-        = MF->getMachineMemOperand(PInfo, MMO->getFlags(),
-                                   EltSize, MinAlign(Align, EltSize * i));
+      MachineMemOperand *NewMMO =
+          MF->getMachineMemOperand(PInfo, MMO->getFlags(), EltSize,
+                                   commonAlignment(Alignment, EltSize * i));
 
       MIB = BuildMI(*MBB, MI, DL, Desc)
                 .addReg(SubReg,
@@ -877,12 +890,12 @@ bool SIRegisterInfo::spillSGPR(MachineBasicBlock::iterator MI,
         Mov.addReg(SuperReg, RegState::Implicit | SuperKillState);
       }
 
-      unsigned Align = FrameInfo.getObjectAlignment(Index);
+      Align Alignment = FrameInfo.getObjectAlign(Index);
       MachinePointerInfo PtrInfo
         = MachinePointerInfo::getFixedStack(*MF, Index, EltSize * i);
-      MachineMemOperand *MMO
-        = MF->getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore,
-                                   EltSize, MinAlign(Align, EltSize * i));
+      MachineMemOperand *MMO =
+          MF->getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore, EltSize,
+                                   commonAlignment(Alignment, EltSize * i));
       BuildMI(*MBB, MI, DL, TII->get(AMDGPU::SI_SPILL_V32_SAVE))
         .addReg(TmpVGPR, RegState::Kill)      // src
         .addFrameIndex(Index)                 // vaddr
@@ -951,14 +964,14 @@ bool SIRegisterInfo::restoreSGPR(MachineBasicBlock::iterator MI,
       // FIXME: We should use S_LOAD_DWORD here for VI.
       if (!TmpVGPR.isValid())
         TmpVGPR = RS->scavengeRegister(&AMDGPU::VGPR_32RegClass, MI, 0);
-      unsigned Align = FrameInfo.getObjectAlignment(Index);
+      Align Alignment = FrameInfo.getObjectAlign(Index);
 
       MachinePointerInfo PtrInfo
         = MachinePointerInfo::getFixedStack(*MF, Index, EltSize * i);
 
-      MachineMemOperand *MMO = MF->getMachineMemOperand(PtrInfo,
-        MachineMemOperand::MOLoad, EltSize,
-        MinAlign(Align, EltSize * i));
+      MachineMemOperand *MMO =
+          MF->getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad, EltSize,
+                                   commonAlignment(Alignment, EltSize * i));
 
       BuildMI(*MBB, MI, DL, TII->get(AMDGPU::SI_SPILL_V32_RESTORE), TmpVGPR)
         .addFrameIndex(Index)                 // vaddr
@@ -1263,9 +1276,8 @@ StringRef SIRegisterInfo::getRegAsmName(unsigned Reg) const {
 
 // FIXME: This is very slow. It might be worth creating a map from physreg to
 // register class.
-const TargetRegisterClass *SIRegisterInfo::getPhysRegClass(unsigned Reg) const {
-  assert(!Register::isVirtualRegister(Reg));
-
+const TargetRegisterClass *
+SIRegisterInfo::getPhysRegClass(MCRegister Reg) const {
   static const TargetRegisterClass *const BaseClasses[] = {
     &AMDGPU::VGPR_32RegClass,
     &AMDGPU::SReg_32RegClass,
@@ -1523,15 +1535,15 @@ bool SIRegisterInfo::shouldRewriteCopySrc(
 /// Returns a register that is not used at any point in the function.
 ///        If all registers are used, then this function will return
 //         AMDGPU::NoRegister.
-unsigned
+MCRegister
 SIRegisterInfo::findUnusedRegister(const MachineRegisterInfo &MRI,
                                    const TargetRegisterClass *RC,
                                    const MachineFunction &MF) const {
 
-  for (unsigned Reg : *RC)
+  for (MCRegister Reg : *RC)
     if (MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg))
       return Reg;
-  return AMDGPU::NoRegister;
+  return MCRegister();
 }
 
 ArrayRef<int16_t> SIRegisterInfo::getRegSplitParts(const TargetRegisterClass *RC,
@@ -1725,23 +1737,20 @@ ArrayRef<int16_t> SIRegisterInfo::getRegSplitParts(const TargetRegisterClass *RC
 
 const TargetRegisterClass*
 SIRegisterInfo::getRegClassForReg(const MachineRegisterInfo &MRI,
-                                  unsigned Reg) const {
-  if (Register::isVirtualRegister(Reg))
-    return  MRI.getRegClass(Reg);
-
-  return getPhysRegClass(Reg);
+                                  Register Reg) const {
+  return Reg.isVirtual() ? MRI.getRegClass(Reg) : getPhysRegClass(Reg);
 }
 
 bool SIRegisterInfo::isVGPR(const MachineRegisterInfo &MRI,
-                            unsigned Reg) const {
-  const TargetRegisterClass * RC = getRegClassForReg(MRI, Reg);
+                            Register Reg) const {
+  const TargetRegisterClass *RC = getRegClassForReg(MRI, Reg);
   assert(RC && "Register class for the reg not found");
   return hasVGPRs(RC);
 }
 
 bool SIRegisterInfo::isAGPR(const MachineRegisterInfo &MRI,
-                            unsigned Reg) const {
-  const TargetRegisterClass * RC = getRegClassForReg(MRI, Reg);
+                            Register Reg) const {
+  const TargetRegisterClass *RC = getRegClassForReg(MRI, Reg);
   assert(RC && "Register class for the reg not found");
   return hasAGPRs(RC);
 }
@@ -1777,6 +1786,8 @@ unsigned SIRegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC,
   default:
     return AMDGPUGenRegisterInfo::getRegPressureLimit(RC, MF);
   case AMDGPU::VGPR_32RegClassID:
+  case AMDGPU::VGPR_LO16RegClassID:
+  case AMDGPU::VGPR_HI16RegClassID:
     return std::min(ST.getMaxNumVGPRs(Occupancy), ST.getMaxNumVGPRs(MF));
   case AMDGPU::SGPR_32RegClassID:
     return std::min(ST.getMaxNumSGPRs(Occupancy, true), ST.getMaxNumSGPRs(MF));
@@ -1800,12 +1811,13 @@ unsigned SIRegisterInfo::getRegPressureSetLimit(const MachineFunction &MF,
 const int *SIRegisterInfo::getRegUnitPressureSets(unsigned RegUnit) const {
   static const int Empty[] = { -1 };
 
-  if (hasRegUnit(AMDGPU::M0, RegUnit))
+  if (RegPressureIgnoredUnits[RegUnit])
     return Empty;
+
   return AMDGPUGenRegisterInfo::getRegUnitPressureSets(RegUnit);
 }
 
-unsigned SIRegisterInfo::getReturnAddressReg(const MachineFunction &MF) const {
+MCRegister SIRegisterInfo::getReturnAddressReg(const MachineFunction &MF) const {
   // Not a callee saved register.
   return AMDGPU::SGPR30_SGPR31;
 }
@@ -1871,7 +1883,7 @@ SIRegisterInfo::getConstrainedRegClassForOperand(const MachineOperand &MO,
   return getAllocatableClass(RC);
 }
 
-unsigned SIRegisterInfo::getVCC() const {
+MCRegister SIRegisterInfo::getVCC() const {
   return isWave32 ? AMDGPU::VCC_LO : AMDGPU::VCC;
 }
 
@@ -1891,7 +1903,7 @@ SIRegisterInfo::getRegClass(unsigned RCID) const {
 }
 
 // Find reaching register definition
-MachineInstr *SIRegisterInfo::findReachingDef(unsigned Reg, unsigned SubReg,
+MachineInstr *SIRegisterInfo::findReachingDef(Register Reg, unsigned SubReg,
                                               MachineInstr &Use,
                                               MachineRegisterInfo &MRI,
                                               LiveIntervals *LIS) const {
@@ -1899,7 +1911,7 @@ MachineInstr *SIRegisterInfo::findReachingDef(unsigned Reg, unsigned SubReg,
   SlotIndex UseIdx = LIS->getInstructionIndex(Use);
   SlotIndex DefIdx;
 
-  if (Register::isVirtualRegister(Reg)) {
+  if (Reg.isVirtual()) {
     if (!LIS->hasInterval(Reg))
       return nullptr;
     LiveInterval &LI = LIS->getInterval(Reg);
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
index a13f6dc4c0e7f..8e50c26742c49 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
@@ -32,8 +32,9 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
   const GCNSubtarget &ST;
   bool SpillSGPRToVGPR;
   bool isWave32;
+  BitVector RegPressureIgnoredUnits;
 
-  void reserveRegisterTuples(BitVector &, unsigned Reg) const;
+  void reserveRegisterTuples(BitVector &, MCRegister Reg) const;
 
 public:
   SIRegisterInfo(const GCNSubtarget &ST);
@@ -48,7 +49,7 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
 
   /// Return the end register initially reserved for the scratch buffer in case
   /// spilling is needed.
-  unsigned reservedPrivateSegmentBufferReg(const MachineFunction &MF) const;
+  MCRegister reservedPrivateSegmentBufferReg(const MachineFunction &MF) const;
 
   BitVector getReservedRegs(const MachineFunction &MF) const override;
 
@@ -111,13 +112,13 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
 
   StringRef getRegAsmName(unsigned Reg) const override;
 
-  unsigned getHWRegIndex(unsigned Reg) const {
+  unsigned getHWRegIndex(MCRegister Reg) const {
     return getEncodingValue(Reg) & 0xff;
   }
 
   /// Return the 'base' register class for this register.
   /// e.g. SGPR0 => SReg_32, VGPR => VGPR_32 SGPR0_SGPR1 -> SReg_32, etc.
-  const TargetRegisterClass *getPhysRegClass(unsigned Reg) const;
+  const TargetRegisterClass *getPhysRegClass(MCRegister Reg) const;
 
   /// \returns true if this class contains only SGPR registers
   bool isSGPRClass(const TargetRegisterClass *RC) const {
@@ -190,20 +191,19 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
   /// -4.0f, -2.0f, -1.0f, -0.5f, 0.0f, 0.5f, 1.0f, 2.0f, 4.0f.
   bool opCanUseInlineConstant(unsigned OpType) const;
 
-  unsigned findUnusedRegister(const MachineRegisterInfo &MRI,
-                              const TargetRegisterClass *RC,
-                              const MachineFunction &MF) const;
+  MCRegister findUnusedRegister(const MachineRegisterInfo &MRI,
+                                const TargetRegisterClass *RC,
+                                const MachineFunction &MF) const;
 
   const TargetRegisterClass *getRegClassForReg(const MachineRegisterInfo &MRI,
-                                               unsigned Reg) const;
-  bool isVGPR(const MachineRegisterInfo &MRI, unsigned Reg) const;
-  bool isAGPR(const MachineRegisterInfo &MRI, unsigned Reg) const;
-  bool isVectorRegister(const MachineRegisterInfo &MRI, unsigned Reg) const {
+                                               Register Reg) const;
+  bool isVGPR(const MachineRegisterInfo &MRI, Register Reg) const;
+  bool isAGPR(const MachineRegisterInfo &MRI, Register Reg) const;
+  bool isVectorRegister(const MachineRegisterInfo &MRI, Register Reg) const {
     return isVGPR(MRI, Reg) || isAGPR(MRI, Reg);
   }
 
-  virtual bool
-  isDivergentRegClass(const TargetRegisterClass *RC) const override {
+  bool isDivergentRegClass(const TargetRegisterClass *RC) const override {
     return !isSGPRClass(RC);
   }
 
@@ -226,7 +226,7 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
 
   const int *getRegUnitPressureSets(unsigned RegUnit) const override;
 
-  unsigned getReturnAddressReg(const MachineFunction &MF) const;
+  MCRegister getReturnAddressReg(const MachineFunction &MF) const;
 
   const TargetRegisterClass *
   getRegClassForSizeOnBank(unsigned Size,
@@ -254,12 +254,12 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
                     : &AMDGPU::SReg_64_XEXECRegClass;
   }
 
-  unsigned getVCC() const;
+  MCRegister getVCC() const;
 
   const TargetRegisterClass *getRegClass(unsigned RCID) const;
 
   // Find reaching register definition
-  MachineInstr *findReachingDef(unsigned Reg, unsigned SubReg,
+  MachineInstr *findReachingDef(Register Reg, unsigned SubReg,
                                 MachineInstr &Use,
                                 MachineRegisterInfo &MRI,
                                 LiveIntervals *LIS) const;
@@ -269,12 +269,18 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
 
   // \returns number of 32 bit registers covered by a \p LM
   static unsigned getNumCoveredRegs(LaneBitmask LM) {
-    return LM.getNumLanes();
+    // The assumption is that every lo16 subreg is an even bit and every hi16
+    // is an adjacent odd bit or vice versa.
+    uint64_t Mask = LM.getAsInteger();
+    uint64_t Even = Mask & 0xAAAAAAAAAAAAAAAAULL;
+    Mask = (Even >> 1) | Mask;
+    uint64_t Odd = Mask & 0x5555555555555555ULL;
+    return countPopulation(Odd);
   }
 
   // \returns a DWORD offset of a \p SubReg
   unsigned getChannelFromSubReg(unsigned SubReg) const {
-    return SubReg ? alignTo(getSubRegIdxOffset(SubReg), 32) / 32 : 0;
+    return SubReg ? divideCeil(getSubRegIdxOffset(SubReg), 32) : 0;
   }
 
   // \returns a DWORD size of a \p SubReg
@@ -286,10 +292,10 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
   void buildSpillLoadStore(MachineBasicBlock::iterator MI,
                            unsigned LoadStoreOp,
                            int Index,
-                           unsigned ValueReg,
+                           Register ValueReg,
                            bool ValueIsKill,
-                           unsigned ScratchRsrcReg,
-                           unsigned ScratchOffsetReg,
+                           MCRegister ScratchRsrcReg,
+                           MCRegister ScratchOffsetReg,
                            int64_t InstrOffset,
                            MachineMemOperand *MMO,
                            RegScavenger *RS) const;
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.td b/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
index 987a93040ec32..1431ed74fe65c 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
@@ -10,10 +10,44 @@
 //  Subregister declarations
 //===----------------------------------------------------------------------===//
 
+class Indexes<int N> {
+  list<int> all = [0,   1,  2,  3,  4,  5,  6 , 7,
+                   8,   9, 10, 11, 12, 13, 14, 15,
+                   16, 17, 18, 19, 20, 21, 22, 23,
+                   24, 25, 26, 27, 28, 29, 30, 31];
+
+  // Returns list of indexes [0..N)
+  list<int> slice =
+    !foldl([]<int>, all, acc, cur,
+           !listconcat(acc, !if(!lt(cur, N), [cur], [])));
+}
+
 let Namespace = "AMDGPU" in {
+
+def lo16 : SubRegIndex<16, 0>;
+def hi16 : SubRegIndex<16, 16>;
+
 foreach Index = 0-31 in {
   def sub#Index : SubRegIndex<32, !shl(Index, 5)>;
 }
+
+foreach Index = 1-31 in {
+  def sub#Index#_lo16 : ComposedSubRegIndex<!cast<SubRegIndex>(sub#Index), lo16>;
+  def sub#Index#_hi16 : ComposedSubRegIndex<!cast<SubRegIndex>(sub#Index), hi16>;
+}
+
+foreach Size = {2-5,8,16} in {
+  foreach Index = Indexes<!add(33, !mul(Size, -1))>.slice in {
+    def !foldl("", Indexes<Size>.slice, acc, cur,
+               !strconcat(acc#!if(!eq(acc,""),"","_"), "sub"#!add(cur, Index))) :
+      SubRegIndex<!mul(Size, 32), !shl(Index, 5)> {
+      let CoveringSubRegIndices =
+        !foldl([]<SubRegIndex>, Indexes<Size>.slice, acc, cur,
+               !listconcat(acc, [!cast<SubRegIndex>(sub#!add(cur, Index))]));
+    }
+  }
+}
+
 }
 
 //===----------------------------------------------------------------------===//
@@ -92,6 +126,16 @@ class SIReg <string n, bits<16> regIdx = 0> :
   let HWEncoding = regIdx;
 }
 
+class SIRegWithSubRegs <string n, list<Register> subregs, bits<16> regIdx = 0> :
+  RegisterWithSubRegs<n, subregs> {
+  let Namespace = "AMDGPU";
+
+  // This is the not yet the complete register encoding. An additional
+  // bit is set for VGPRs.
+  let HWEncoding = regIdx;
+  let CoveredBySubRegs = 1;
+}
+
 // Special Registers
 def VCC_LO : SIReg<"vcc_lo", 106>;
 def VCC_HI : SIReg<"vcc_hi", 107>;
@@ -103,7 +147,7 @@ def FP_REG : SIReg<"fp", 0>;
 def SP_REG : SIReg<"sp", 0>;
 
 // Pseudo-register to represent the program-counter DWARF register.
-def PC_REG : SIReg<"pc", 0>, DwarfRegNum<[16, 0]> {
+def PC_REG : SIReg<"pc", 0>, DwarfRegNum<[16, 16]> {
   // There is no physical register corresponding to a "program counter", but
   // we need to encode the concept in debug information in order to represent
   // things like the return value in unwind information.
@@ -221,10 +265,29 @@ foreach Index = 0-255 in {
   // The ratio of index/allocation_granularity is taken as the cost value.
   // Considered the allocation granularity as 4 here.
   let CostPerUse=!if(!gt(Index, 31), !srl(Index, 2), 0) in {
-  def VGPR#Index :
-      SIReg <"v"#Index, Index>,
-      DwarfRegNum<[!add(Index, 2560), !add(Index, 1536)]> {
+
+  // There is no special encoding for low 16 bit subreg, this not a real
+  // register but rather an operand for instructions preserving high 16 bits
+  // of the result or reading just low 16 bits of a 32 bit VGPR.
+  // It is encoded as a corresponding 32 bit register.
+  def VGPR#Index#_LO16 : SIReg <"v"#Index#".l", Index>,
+    DwarfRegNum<[!add(Index, 2560), !add(Index, 1536)]> {
+    let HWEncoding{8} = 1;
+  }
+  // There is no special encoding for low 16 bit subreg, this not a real
+  // register but rather an operand for instructions preserving low 16 bits
+  // of the result or reading just high 16 bits of a 32 bit VGPR.
+  // It is encoded as a corresponding 32 bit register.
+  def VGPR#Index#_HI16 : SIReg <"v"#Index#".h", Index>,
+    DwarfRegNum<[!add(Index, 2560), !add(Index, 1536)]> {
+    let HWEncoding{8} = 1;
+  }
+  def VGPR#Index : SIRegWithSubRegs <"v"#Index,
+    [!cast<Register>("VGPR"#Index#"_LO16"), !cast<Register>("VGPR"#Index#"_HI16")],
+    Index>,
+    DwarfRegNum<[!add(Index, 2560), !add(Index, 1536)]> {
     let HWEncoding{8} = 1;
+    let SubRegIndices = [lo16, hi16];
   }
   }
 }
@@ -386,6 +449,19 @@ class RegisterTypes<list<ValueType> reg_types> {
 def Reg16Types : RegisterTypes<[i16, f16]>;
 def Reg32Types : RegisterTypes<[i32, f32, v2i16, v2f16, p2, p3, p5, p6]>;
 
+def VGPR_LO16 : RegisterClass<"AMDGPU", Reg16Types.types, 16,
+                              (add (sequence "VGPR%u_LO16", 0, 255))> {
+  let AllocationPriority = 1;
+  let Size = 16;
+  let GeneratePressureSet = 0;
+}
+
+def VGPR_HI16 : RegisterClass<"AMDGPU", Reg16Types.types, 16,
+                              (add (sequence "VGPR%u_HI16", 0, 255))> {
+  let AllocationPriority = 1;
+  let Size = 16;
+  let GeneratePressureSet = 0;
+}
 
 // VGPR 32-bit registers
 // i16/f16 only on VI+
@@ -393,6 +469,7 @@ def VGPR_32 : RegisterClass<"AMDGPU", !listconcat(Reg32Types.types, Reg16Types.t
                             (add (sequence "VGPR%u", 0, 255))> {
   let AllocationPriority = 1;
   let Size = 32;
+  let Weight = 1;
 }
 
 // VGPR 64-bit registers
@@ -634,6 +711,7 @@ def VReg_64 : RegisterClass<"AMDGPU", [i64, f64, v2i32, v2f32, v4f16, v4i16, p0,
   // Requires 2 v_mov_b32 to copy
   let CopyCost = 2;
   let AllocationPriority = 2;
+  let Weight = 2;
 }
 
 def VReg_96 : RegisterClass<"AMDGPU", [v3i32, v3f32], 32, (add VGPR_96)> {
@@ -642,6 +720,7 @@ def VReg_96 : RegisterClass<"AMDGPU", [v3i32, v3f32], 32, (add VGPR_96)> {
   // Requires 3 v_mov_b32 to copy
   let CopyCost = 3;
   let AllocationPriority = 3;
+  let Weight = 3;
 }
 
 def VReg_128 : RegisterClass<"AMDGPU", [v4i32, v4f32, v2i64, v2f64, i128], 32,
@@ -651,6 +730,7 @@ def VReg_128 : RegisterClass<"AMDGPU", [v4i32, v4f32, v2i64, v2f64, i128], 32,
   // Requires 4 v_mov_b32 to copy
   let CopyCost = 4;
   let AllocationPriority = 4;
+  let Weight = 4;
 }
 
 def VReg_160 : RegisterClass<"AMDGPU", [v5i32, v5f32], 32,
@@ -660,6 +740,7 @@ def VReg_160 : RegisterClass<"AMDGPU", [v5i32, v5f32], 32,
   // Requires 5 v_mov_b32 to copy
   let CopyCost = 5;
   let AllocationPriority = 5;
+  let Weight = 5;
 }
 
 def VReg_256 : RegisterClass<"AMDGPU", [v8i32, v8f32], 32,
@@ -667,6 +748,7 @@ def VReg_256 : RegisterClass<"AMDGPU", [v8i32, v8f32], 32,
   let Size = 256;
   let CopyCost = 8;
   let AllocationPriority = 6;
+  let Weight = 8;
 }
 
 def VReg_512 : RegisterClass<"AMDGPU", [v16i32, v16f32], 32,
@@ -674,6 +756,7 @@ def VReg_512 : RegisterClass<"AMDGPU", [v16i32, v16f32], 32,
   let Size = 512;
   let CopyCost = 16;
   let AllocationPriority = 7;
+  let Weight = 16;
 }
 
 def VReg_1024 : RegisterClass<"AMDGPU", [v32i32, v32f32], 32,
@@ -681,6 +764,7 @@ def VReg_1024 : RegisterClass<"AMDGPU", [v32i32, v32f32], 32,
   let Size = 1024;
   let CopyCost = 32;
   let AllocationPriority = 8;
+  let Weight = 32;
 }
 
 def AReg_64 : RegisterClass<"AMDGPU", [i64, f64, v2i32, v2f32, v4f16, v4i16], 32,
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index a5ed44198992c..4f8d9e925df18 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -268,6 +268,13 @@ unsigned getLocalMemorySize(const MCSubtargetInfo *STI) {
 }
 
 unsigned getEUsPerCU(const MCSubtargetInfo *STI) {
+  // "Per CU" really means "per whatever functional block the waves of a
+  // workgroup must share". For gfx10 in CU mode this is the CU, which contains
+  // two SIMDs.
+  if (isGFX10(*STI) && STI->getFeatureBits().test(FeatureCuMode))
+    return 2;
+  // Pre-gfx10 a CU contains four SIMDs. For gfx10 in WGP mode the WGP contains
+  // two CUs, so a total of four SIMDs.
   return 4;
 }
 
@@ -283,15 +290,6 @@ unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo *STI,
   return std::min(N, 16u);
 }
 
-unsigned getMaxWavesPerCU(const MCSubtargetInfo *STI) {
-  return getMaxWavesPerEU(STI) * getEUsPerCU(STI);
-}
-
-unsigned getMaxWavesPerCU(const MCSubtargetInfo *STI,
-                          unsigned FlatWorkGroupSize) {
-  return getWavesPerWorkGroup(STI, FlatWorkGroupSize);
-}
-
 unsigned getMinWavesPerEU(const MCSubtargetInfo *STI) {
   return 1;
 }
@@ -303,10 +301,10 @@ unsigned getMaxWavesPerEU(const MCSubtargetInfo *STI) {
   return 20;
 }
 
-unsigned getMaxWavesPerEU(const MCSubtargetInfo *STI,
-                          unsigned FlatWorkGroupSize) {
-  return alignTo(getMaxWavesPerCU(STI, FlatWorkGroupSize),
-                 getEUsPerCU(STI)) / getEUsPerCU(STI);
+unsigned getWavesPerEUForWorkGroup(const MCSubtargetInfo *STI,
+                                   unsigned FlatWorkGroupSize) {
+  return divideCeil(getWavesPerWorkGroup(STI, FlatWorkGroupSize),
+                    getEUsPerCU(STI));
 }
 
 unsigned getMinFlatWorkGroupSize(const MCSubtargetInfo *STI) {
@@ -320,8 +318,7 @@ unsigned getMaxFlatWorkGroupSize(const MCSubtargetInfo *STI) {
 
 unsigned getWavesPerWorkGroup(const MCSubtargetInfo *STI,
                               unsigned FlatWorkGroupSize) {
-  return alignTo(FlatWorkGroupSize, getWavefrontSize(STI)) /
-                 getWavefrontSize(STI);
+  return divideCeil(FlatWorkGroupSize, getWavefrontSize(STI));
 }
 
 unsigned getSGPRAllocGranule(const MCSubtargetInfo *STI) {
@@ -1251,8 +1248,12 @@ static bool hasSMEMByteOffset(const MCSubtargetInfo &ST) {
   return isGCN3Encoding(ST) || isGFX10(ST);
 }
 
-static bool isLegalSMRDEncodedImmOffset(const MCSubtargetInfo &ST,
-                                        int64_t EncodedOffset) {
+static bool hasSMRDSignedImmOffset(const MCSubtargetInfo &ST) {
+  return isGFX9(ST) || isGFX10(ST);
+}
+
+static bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST,
+                                             int64_t EncodedOffset) {
   return hasSMEMByteOffset(ST) ? isUInt<20>(EncodedOffset)
                                : isUInt<8>(EncodedOffset);
 }
@@ -1271,21 +1272,27 @@ uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST,
 }
 
 Optional<int64_t> getSMRDEncodedOffset(const MCSubtargetInfo &ST,
-                                       int64_t ByteOffset) {
+                                       int64_t ByteOffset, bool IsBuffer) {
+  // The signed version is always a byte offset.
+  if (!IsBuffer && hasSMRDSignedImmOffset(ST)) {
+    assert(hasSMEMByteOffset(ST));
+    return isInt<20>(ByteOffset) ? Optional<int64_t>(ByteOffset) : None;
+  }
+
   if (!isDwordAligned(ByteOffset) && !hasSMEMByteOffset(ST))
     return None;
 
   int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset);
-  return isLegalSMRDEncodedImmOffset(ST, EncodedOffset) ?
-    Optional<int64_t>(EncodedOffset) : None;
+  return isLegalSMRDEncodedUnsignedOffset(ST, EncodedOffset)
+             ? Optional<int64_t>(EncodedOffset)
+             : None;
 }
 
 Optional<int64_t> getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST,
                                                 int64_t ByteOffset) {
-  if (!isDwordAligned(ByteOffset) && !hasSMEMByteOffset(ST))
+  if (!isCI(ST) || !isDwordAligned(ByteOffset))
     return None;
 
-  assert(isCI(ST));
   int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset);
   return isUInt<32>(EncodedOffset) ? Optional<int64_t>(EncodedOffset) : None;
 }
@@ -1336,8 +1343,7 @@ bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset,
   return true;
 }
 
-SIModeRegisterDefaults::SIModeRegisterDefaults(const Function &F,
-                                               const GCNSubtarget &ST) {
+SIModeRegisterDefaults::SIModeRegisterDefaults(const Function &F) {
   *this = getDefaultForCallingConv(F.getCallingConv());
 
   StringRef IEEEAttr = F.getFnAttribute("amdgpu-ieee").getValueAsString();
@@ -1349,11 +1355,25 @@ SIModeRegisterDefaults::SIModeRegisterDefaults(const Function &F,
   if (!DX10ClampAttr.empty())
     DX10Clamp = DX10ClampAttr == "true";
 
-  // FIXME: Split this when denormal-fp-math is used
-  FP32InputDenormals = ST.hasFP32Denormals(F);
-  FP32OutputDenormals = FP32InputDenormals;
-  FP64FP16InputDenormals = ST.hasFP64FP16Denormals(F);
-  FP64FP16OutputDenormals = FP64FP16InputDenormals;
+  StringRef DenormF32Attr = F.getFnAttribute("denormal-fp-math-f32").getValueAsString();
+  if (!DenormF32Attr.empty()) {
+    DenormalMode DenormMode = parseDenormalFPAttribute(DenormF32Attr);
+    FP32InputDenormals = DenormMode.Input == DenormalMode::IEEE;
+    FP32OutputDenormals = DenormMode.Output == DenormalMode::IEEE;
+  }
+
+  StringRef DenormAttr = F.getFnAttribute("denormal-fp-math").getValueAsString();
+  if (!DenormAttr.empty()) {
+    DenormalMode DenormMode = parseDenormalFPAttribute(DenormAttr);
+
+    if (DenormF32Attr.empty()) {
+      FP32InputDenormals = DenormMode.Input == DenormalMode::IEEE;
+      FP32OutputDenormals = DenormMode.Output == DenormalMode::IEEE;
+    }
+
+    FP64FP16InputDenormals = DenormMode.Input == DenormalMode::IEEE;
+    FP64FP16OutputDenormals = DenormMode.Output == DenormalMode::IEEE;
+  }
 }
 
 namespace {
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index 4cfb2adee96eb..9f1670c52e69f 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -87,15 +87,6 @@ unsigned getEUsPerCU(const MCSubtargetInfo *STI);
 unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo *STI,
                                unsigned FlatWorkGroupSize);
 
-/// \returns Maximum number of waves per compute unit for given subtarget \p
-/// STI without any kind of limitation.
-unsigned getMaxWavesPerCU(const MCSubtargetInfo *STI);
-
-/// \returns Maximum number of waves per compute unit for given subtarget \p
-/// STI and limited by given \p FlatWorkGroupSize.
-unsigned getMaxWavesPerCU(const MCSubtargetInfo *STI,
-                          unsigned FlatWorkGroupSize);
-
 /// \returns Minimum number of waves per execution unit for given subtarget \p
 /// STI.
 unsigned getMinWavesPerEU(const MCSubtargetInfo *STI);
@@ -104,10 +95,10 @@ unsigned getMinWavesPerEU(const MCSubtargetInfo *STI);
 /// STI without any kind of limitation.
 unsigned getMaxWavesPerEU(const MCSubtargetInfo *STI);
 
-/// \returns Maximum number of waves per execution unit for given subtarget \p
-/// STI and limited by given \p FlatWorkGroupSize.
-unsigned getMaxWavesPerEU(const MCSubtargetInfo *STI,
-                          unsigned FlatWorkGroupSize);
+/// \returns Number of waves per execution unit required to support the given \p
+/// FlatWorkGroupSize.
+unsigned getWavesPerEUForWorkGroup(const MCSubtargetInfo *STI,
+                                   unsigned FlatWorkGroupSize);
 
 /// \returns Minimum flat work group size for given subtarget \p STI.
 unsigned getMinFlatWorkGroupSize(const MCSubtargetInfo *STI);
@@ -116,7 +107,7 @@ unsigned getMinFlatWorkGroupSize(const MCSubtargetInfo *STI);
 unsigned getMaxFlatWorkGroupSize(const MCSubtargetInfo *STI);
 
 /// \returns Number of waves per work group for given subtarget \p STI and
-/// limited by given \p FlatWorkGroupSize.
+/// \p FlatWorkGroupSize.
 unsigned getWavesPerWorkGroup(const MCSubtargetInfo *STI,
                               unsigned FlatWorkGroupSize);
 
@@ -653,10 +644,12 @@ bool isArgPassedInSGPR(const Argument *Arg);
 /// offsets.
 uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST, uint64_t ByteOffset);
 
-/// \returns The encoding that will be used for \p ByteOffset in the SMRD offset
-/// field, or None if it won't fit. This is useful on all subtargets.
+/// \returns The encoding that will be used for \p ByteOffset in the
+/// SMRD offset field, or None if it won't fit. On GFX9 and GFX10
+/// S_LOAD instructions have a signed offset, on other subtargets it is
+/// unsigned. S_BUFFER has an unsigned offset for all subtargets.
 Optional<int64_t> getSMRDEncodedOffset(const MCSubtargetInfo &ST,
-                                       int64_t ByteOffset);
+                                       int64_t ByteOffset, bool IsBuffer);
 
 /// \return The encoding that can be used for a 32-bit literal offset in an SMRD
 /// instruction. This is only useful on CI.s
@@ -704,19 +697,13 @@ struct SIModeRegisterDefaults {
     FP64FP16InputDenormals(true),
     FP64FP16OutputDenormals(true) {}
 
-  // FIXME: Should not depend on the subtarget
-  SIModeRegisterDefaults(const Function &F, const GCNSubtarget &ST);
+  SIModeRegisterDefaults(const Function &F);
 
   static SIModeRegisterDefaults getDefaultForCallingConv(CallingConv::ID CC) {
     const bool IsCompute = AMDGPU::isCompute(CC);
 
     SIModeRegisterDefaults Mode;
-    Mode.DX10Clamp = true;
     Mode.IEEE = IsCompute;
-    Mode.FP32InputDenormals = false; // FIXME: Should be on by default.
-    Mode.FP32OutputDenormals = false; // FIXME: Should be on by default.
-    Mode.FP64FP16InputDenormals = true;
-    Mode.FP64FP16OutputDenormals = true;
     return Mode;
   }
 
diff --git a/llvm/lib/Target/AMDGPU/VOP1Instructions.td b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
index d39e698d98095..60567c77b5905 100644
--- a/llvm/lib/Target/AMDGPU/VOP1Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
@@ -89,9 +89,7 @@ class VOP1_DPP_Pseudo <string OpName, VOPProfile P, list<dag> pattern=[]> :
 class getVOP1Pat64 <SDPatternOperator node, VOPProfile P> : LetDummies {
   list<dag> ret =
     !if(P.HasModifiers,
-        [(set P.DstVT:$vdst, (node (P.Src0VT (VOP3Mods0 P.Src0VT:$src0,
-                                              i32:$src0_modifiers,
-                                              i1:$clamp, i32:$omod))))],
+        [(set P.DstVT:$vdst, (node (P.Src0VT (VOP3Mods P.Src0VT:$src0, i32:$src0_modifiers))))],
         !if(P.HasOMod,
             [(set P.DstVT:$vdst, (node (P.Src0VT (VOP3OMods P.Src0VT:$src0,
                                                   i1:$clamp, i32:$omod))))],
diff --git a/llvm/lib/Target/ARC/ARCInstrInfo.cpp b/llvm/lib/Target/ARC/ARCInstrInfo.cpp
index 8261655c965f5..527f239c2643c 100644
--- a/llvm/lib/Target/ARC/ARCInstrInfo.cpp
+++ b/llvm/lib/Target/ARC/ARCInstrInfo.cpp
@@ -299,11 +299,11 @@ void ARCInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
   DebugLoc dl = MBB.findDebugLoc(I);
   MachineFunction &MF = *MBB.getParent();
   MachineFrameInfo &MFI = MF.getFrameInfo();
-  unsigned Align = MFI.getObjectAlignment(FrameIndex);
 
   MachineMemOperand *MMO = MF.getMachineMemOperand(
       MachinePointerInfo::getFixedStack(MF, FrameIndex),
-      MachineMemOperand::MOStore, MFI.getObjectSize(FrameIndex), Align);
+      MachineMemOperand::MOStore, MFI.getObjectSize(FrameIndex),
+      MFI.getObjectAlign(FrameIndex));
 
   assert(MMO && "Couldn't get MachineMemOperand for store to stack.");
   assert(TRI->getSpillSize(*RC) == 4 &&
@@ -327,10 +327,10 @@ void ARCInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
   DebugLoc dl = MBB.findDebugLoc(I);
   MachineFunction &MF = *MBB.getParent();
   MachineFrameInfo &MFI = MF.getFrameInfo();
-  unsigned Align = MFI.getObjectAlignment(FrameIndex);
   MachineMemOperand *MMO = MF.getMachineMemOperand(
       MachinePointerInfo::getFixedStack(MF, FrameIndex),
-      MachineMemOperand::MOLoad, MFI.getObjectSize(FrameIndex), Align);
+      MachineMemOperand::MOLoad, MFI.getObjectSize(FrameIndex),
+      MFI.getObjectAlign(FrameIndex));
 
   assert(MMO && "Couldn't get MachineMemOperand for store to stack.");
   assert(TRI->getSpillSize(*RC) == 4 &&
diff --git a/llvm/lib/Target/ARC/MCTargetDesc/ARCInstPrinter.h b/llvm/lib/Target/ARC/MCTargetDesc/ARCInstPrinter.h
index 53ca4066c02d5..266f2de087727 100644
--- a/llvm/lib/Target/ARC/MCTargetDesc/ARCInstPrinter.h
+++ b/llvm/lib/Target/ARC/MCTargetDesc/ARCInstPrinter.h
@@ -36,6 +36,10 @@ class ARCInstPrinter : public MCInstPrinter {
 private:
   void printMemOperandRI(const MCInst *MI, unsigned OpNum, raw_ostream &O);
   void printOperand(const MCInst *MI, unsigned OpNum, raw_ostream &O);
+  void printOperand(const MCInst *MI, uint64_t /*Address*/, unsigned OpNum,
+                    raw_ostream &O) {
+    printOperand(MI, OpNum, O);
+  }
   void printPredicateOperand(const MCInst *MI, unsigned OpNum, raw_ostream &O);
   void printBRCCPredicateOperand(const MCInst *MI, unsigned OpNum,
                                  raw_ostream &O);
diff --git a/llvm/lib/Target/ARM/ARM.td b/llvm/lib/Target/ARM/ARM.td
index 0dbb2d6b71006..72cfd64ac566b 100644
--- a/llvm/lib/Target/ARM/ARM.td
+++ b/llvm/lib/Target/ARM/ARM.td
@@ -424,6 +424,10 @@ def FeatureNoPostRASched : SubtargetFeature<"disable-postra-scheduler",
 def FeatureSB       : SubtargetFeature<"sb", "HasSB", "true",
   "Enable v8.5a Speculation Barrier" >;
 
+// Armv8.6-A extensions
+def FeatureBF16     : SubtargetFeature<"bf16", "HasBF16", "true",
+  "Enable support for BFloat16 instructions",  [FeatureNEON]>;
+
 // Armv8.1-M extensions
 
 def FeatureLOB            : SubtargetFeature<"lob", "HasLOB", "true",
@@ -523,6 +527,10 @@ def HasV8_5aOps   : SubtargetFeature<"v8.5a", "HasV8_5aOps", "true",
                                    "Support ARM v8.5a instructions",
                                    [HasV8_4aOps, FeatureSB]>;
 
+def HasV8_6aOps   : SubtargetFeature<"v8.6a", "HasV8_6aOps", "true",
+                                   "Support ARM v8.6a instructions",
+                                   [HasV8_5aOps, FeatureBF16]>;
+
 def HasV8_1MMainlineOps : SubtargetFeature<
                "v8.1m.main", "HasV8_1MMainlineOps", "true",
                "Support ARM v8-1M Mainline instructions",
@@ -797,6 +805,19 @@ def ARMv85a   : Architecture<"armv8.5-a", "ARMv85a",  [HasV8_5aOps,
                                                        FeatureCRC,
                                                        FeatureRAS,
                                                        FeatureDotProd]>;
+def ARMv86a   : Architecture<"armv8.6-a", "ARMv86a",  [HasV8_6aOps,
+                                                       FeatureAClass,
+                                                       FeatureDB,
+                                                       FeatureFPARMv8,
+                                                       FeatureNEON,
+                                                       FeatureDSP,
+                                                       FeatureTrustZone,
+                                                       FeatureMP,
+                                                       FeatureVirtualization,
+                                                       FeatureCrypto,
+                                                       FeatureCRC,
+                                                       FeatureRAS,
+                                                       FeatureDotProd]>;
 
 def ARMv8r    : Architecture<"armv8-r",   "ARMv8r",   [HasV8Ops,
                                                        FeatureRClass,
diff --git a/llvm/lib/Target/ARM/ARMAsmPrinter.cpp b/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
index f6ccdf1856952..1342f2bf5ad42 100644
--- a/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
+++ b/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
@@ -652,16 +652,13 @@ void ARMAsmPrinter::emitAttributes() {
   }
 
   // Set FP Denormals.
-  if (checkDenormalAttributeConsistency(*MMI->getModule(),
-                                        "denormal-fp-math",
-                                        DenormalMode::getPreserveSign()) ||
-      TM.Options.FPDenormalMode == FPDenormal::PreserveSign)
+  if (checkDenormalAttributeConsistency(*MMI->getModule(), "denormal-fp-math",
+                                        DenormalMode::getPreserveSign()))
     ATS.emitAttribute(ARMBuildAttrs::ABI_FP_denormal,
                       ARMBuildAttrs::PreserveFPSign);
   else if (checkDenormalAttributeConsistency(*MMI->getModule(),
                                              "denormal-fp-math",
-                                             DenormalMode::getPositiveZero()) ||
-           TM.Options.FPDenormalMode == FPDenormal::PositiveZero)
+                                             DenormalMode::getPositiveZero()))
     ATS.emitAttribute(ARMBuildAttrs::ABI_FP_denormal,
                       ARMBuildAttrs::PositiveZero);
   else if (!TM.Options.UnsafeFPMath)
diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
index 2660c83e8abc5..c17dfd029109d 100644
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
@@ -1076,11 +1076,11 @@ storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
                     const TargetRegisterInfo *TRI) const {
   MachineFunction &MF = *MBB.getParent();
   MachineFrameInfo &MFI = MF.getFrameInfo();
-  unsigned Align = MFI.getObjectAlignment(FI);
+  Align Alignment = MFI.getObjectAlign(FI);
 
   MachineMemOperand *MMO = MF.getMachineMemOperand(
       MachinePointerInfo::getFixedStack(MF, FI), MachineMemOperand::MOStore,
-      MFI.getObjectSize(FI), Align);
+      MFI.getObjectSize(FI), Alignment);
 
   switch (TRI->getSpillSize(*RC)) {
     case 2:
@@ -1150,7 +1150,7 @@ storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
     case 16:
       if (ARM::DPairRegClass.hasSubClassEq(RC) && Subtarget.hasNEON()) {
         // Use aligned spills if the stack can be realigned.
-        if (Align >= 16 && getRegisterInfo().canRealignStack(MF)) {
+        if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF)) {
           BuildMI(MBB, I, DebugLoc(), get(ARM::VST1q64))
               .addFrameIndex(FI)
               .addImm(16)
@@ -1178,7 +1178,7 @@ storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
     case 24:
       if (ARM::DTripleRegClass.hasSubClassEq(RC)) {
         // Use aligned spills if the stack can be realigned.
-        if (Align >= 16 && getRegisterInfo().canRealignStack(MF) &&
+        if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) &&
             Subtarget.hasNEON()) {
           BuildMI(MBB, I, DebugLoc(), get(ARM::VST1d64TPseudo))
               .addFrameIndex(FI)
@@ -1201,7 +1201,7 @@ storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
       break;
     case 32:
       if (ARM::QQPRRegClass.hasSubClassEq(RC) || ARM::DQuadRegClass.hasSubClassEq(RC)) {
-        if (Align >= 16 && getRegisterInfo().canRealignStack(MF) &&
+        if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) &&
             Subtarget.hasNEON()) {
           // FIXME: It's possible to only store part of the QQ register if the
           // spilled def has a sub-register index.
@@ -1319,10 +1319,10 @@ loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
   if (I != MBB.end()) DL = I->getDebugLoc();
   MachineFunction &MF = *MBB.getParent();
   MachineFrameInfo &MFI = MF.getFrameInfo();
-  unsigned Align = MFI.getObjectAlignment(FI);
+  const Align Alignment = MFI.getObjectAlign(FI);
   MachineMemOperand *MMO = MF.getMachineMemOperand(
       MachinePointerInfo::getFixedStack(MF, FI), MachineMemOperand::MOLoad,
-      MFI.getObjectSize(FI), Align);
+      MFI.getObjectSize(FI), Alignment);
 
   switch (TRI->getSpillSize(*RC)) {
   case 2:
@@ -1391,7 +1391,7 @@ loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
     break;
   case 16:
     if (ARM::DPairRegClass.hasSubClassEq(RC) && Subtarget.hasNEON()) {
-      if (Align >= 16 && getRegisterInfo().canRealignStack(MF)) {
+      if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF)) {
         BuildMI(MBB, I, DL, get(ARM::VLD1q64), DestReg)
             .addFrameIndex(FI)
             .addImm(16)
@@ -1415,7 +1415,7 @@ loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
     break;
   case 24:
     if (ARM::DTripleRegClass.hasSubClassEq(RC)) {
-      if (Align >= 16 && getRegisterInfo().canRealignStack(MF) &&
+      if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) &&
           Subtarget.hasNEON()) {
         BuildMI(MBB, I, DL, get(ARM::VLD1d64TPseudo), DestReg)
             .addFrameIndex(FI)
@@ -1438,7 +1438,7 @@ loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
     break;
    case 32:
     if (ARM::QQPRRegClass.hasSubClassEq(RC) || ARM::DQuadRegClass.hasSubClassEq(RC)) {
-      if (Align >= 16 && getRegisterInfo().canRealignStack(MF) &&
+      if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) &&
           Subtarget.hasNEON()) {
         BuildMI(MBB, I, DL, get(ARM::VLD1d64QPseudo), DestReg)
             .addFrameIndex(FI)
@@ -1736,7 +1736,7 @@ static unsigned duplicateCPV(MachineFunction &MF, unsigned &CPI) {
 
 void ARMBaseInstrInfo::reMaterialize(MachineBasicBlock &MBB,
                                      MachineBasicBlock::iterator I,
-                                     unsigned DestReg, unsigned SubIdx,
+                                     Register DestReg, unsigned SubIdx,
                                      const MachineInstr &Orig,
                                      const TargetRegisterInfo &TRI) const {
   unsigned Opcode = Orig.getOpcode();
@@ -2719,8 +2719,8 @@ bool llvm::rewriteARMFrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
 /// in SrcReg and SrcReg2 if having two register operands, and the value it
 /// compares against in CmpValue. Return true if the comparison instruction
 /// can be analyzed.
-bool ARMBaseInstrInfo::analyzeCompare(const MachineInstr &MI, unsigned &SrcReg,
-                                      unsigned &SrcReg2, int &CmpMask,
+bool ARMBaseInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
+                                      Register &SrcReg2, int &CmpMask,
                                       int &CmpValue) const {
   switch (MI.getOpcode()) {
   default: break;
@@ -2756,7 +2756,7 @@ bool ARMBaseInstrInfo::analyzeCompare(const MachineInstr &MI, unsigned &SrcReg,
 /// operates on the given source register and applies the same mask
 /// as a 'tst' instruction. Provide a limited look-through for copies.
 /// When successful, MI will hold the found instruction.
-static bool isSuitableForMask(MachineInstr *&MI, unsigned SrcReg,
+static bool isSuitableForMask(MachineInstr *&MI, Register SrcReg,
                               int CmpMask, bool CommonUse) {
   switch (MI->getOpcode()) {
     case ARM::ANDri:
@@ -2791,7 +2791,7 @@ inline static ARMCC::CondCodes getCmpToAddCondition(ARMCC::CondCodes CC) {
 /// CMPrr(r0, r1) can be made redundant by ADDr[ri](r0, r1, X).
 /// This function can be extended later on.
 inline static bool isRedundantFlagInstr(const MachineInstr *CmpI,
-                                        unsigned SrcReg, unsigned SrcReg2,
+                                        Register SrcReg, Register SrcReg2,
                                         int ImmValue, const MachineInstr *OI,
                                         bool &IsThumb1) {
   if ((CmpI->getOpcode() == ARM::CMPrr || CmpI->getOpcode() == ARM::t2CMPrr) &&
@@ -2927,7 +2927,7 @@ static bool isOptimizeCompareCandidate(MachineInstr *MI, bool &IsThumb1) {
 /// operands are swapped: SUBrr(r1,r2) and CMPrr(r2,r1), by updating the
 /// condition code of instructions which use the flags.
 bool ARMBaseInstrInfo::optimizeCompareInstr(
-    MachineInstr &CmpInstr, unsigned SrcReg, unsigned SrcReg2, int CmpMask,
+    MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int CmpMask,
     int CmpValue, const MachineRegisterInfo *MRI) const {
   // Get the unique definition of SrcReg.
   MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg);
@@ -3214,7 +3214,7 @@ bool ARMBaseInstrInfo::shouldSink(const MachineInstr &MI) const {
     return true;
   MachineBasicBlock::const_iterator Next = &MI;
   ++Next;
-  unsigned SrcReg, SrcReg2;
+  Register SrcReg, SrcReg2;
   int CmpMask, CmpValue;
   bool IsThumb1;
   if (Next != MI.getParent()->end() &&
@@ -3225,7 +3225,7 @@ bool ARMBaseInstrInfo::shouldSink(const MachineInstr &MI) const {
 }
 
 bool ARMBaseInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
-                                     unsigned Reg,
+                                     Register Reg,
                                      MachineRegisterInfo *MRI) const {
   // Fold large immediates into add, sub, or, xor.
   unsigned DefOpc = DefMI.getOpcode();
@@ -3777,7 +3777,7 @@ unsigned ARMBaseInstrInfo::getNumMicroOps(const InstrItineraryData *ItinData,
       // If there are odd number of registers or if it's not 64-bit aligned,
       // then it takes an extra AGU (Address Generation Unit) cycle.
       if ((NumRegs % 2) || !MI.hasOneMemOperand() ||
-          (*MI.memoperands_begin())->getAlignment() < 8)
+          (*MI.memoperands_begin())->getAlign() < Align(8))
         ++UOps;
       return UOps;
       }
@@ -4364,10 +4364,10 @@ int ARMBaseInstrInfo::getOperandLatencyImpl(
     return -1;
 
   unsigned DefAlign = DefMI.hasOneMemOperand()
-                          ? (*DefMI.memoperands_begin())->getAlignment()
+                          ? (*DefMI.memoperands_begin())->getAlign().value()
                           : 0;
   unsigned UseAlign = UseMI.hasOneMemOperand()
-                          ? (*UseMI.memoperands_begin())->getAlignment()
+                          ? (*UseMI.memoperands_begin())->getAlign().value()
                           : 0;
 
   // Get the itinerary's latency if possible, and handle variable_ops.
@@ -4414,10 +4414,12 @@ ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
   const MCInstrDesc &UseMCID = get(UseNode->getMachineOpcode());
   auto *DefMN = cast<MachineSDNode>(DefNode);
   unsigned DefAlign = !DefMN->memoperands_empty()
-    ? (*DefMN->memoperands_begin())->getAlignment() : 0;
+                          ? (*DefMN->memoperands_begin())->getAlign().value()
+                          : 0;
   auto *UseMN = cast<MachineSDNode>(UseNode);
   unsigned UseAlign = !UseMN->memoperands_empty()
-    ? (*UseMN->memoperands_begin())->getAlignment() : 0;
+                          ? (*UseMN->memoperands_begin())->getAlign().value()
+                          : 0;
   int Latency = getOperandLatency(ItinData, DefMCID, DefIdx, DefAlign,
                                   UseMCID, UseIdx, UseAlign);
 
@@ -4708,7 +4710,7 @@ unsigned ARMBaseInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
 
   // Adjust for dynamic def-side opcode variants not captured by the itinerary.
   unsigned DefAlign =
-      MI.hasOneMemOperand() ? (*MI.memoperands_begin())->getAlignment() : 0;
+      MI.hasOneMemOperand() ? (*MI.memoperands_begin())->getAlign().value() : 0;
   int Adj = adjustDefLatency(Subtarget, MI, MCID, DefAlign);
   if (Adj >= 0 || (int)Latency > -Adj) {
     return Latency + Adj;
@@ -4830,7 +4832,7 @@ void ARMBaseInstrInfo::expandLoadStackGuardBase(MachineBasicBlock::iterator MI,
                  MachineMemOperand::MODereferenceable |
                  MachineMemOperand::MOInvariant;
     MachineMemOperand *MMO = MBB.getParent()->getMachineMemOperand(
-        MachinePointerInfo::getGOT(*MBB.getParent()), Flags, 4, 4);
+        MachinePointerInfo::getGOT(*MBB.getParent()), Flags, 4, Align(4));
     MIB.addMemOperand(MMO).add(predOps(ARMCC::AL));
   }
 
diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.h b/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
index 8a3dea8152f19..fc3c57915287b 100644
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
@@ -232,7 +232,7 @@ class ARMBaseInstrInfo : public ARMGenInstrInfo {
   bool shouldSink(const MachineInstr &MI) const override;
 
   void reMaterialize(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
-                     unsigned DestReg, unsigned SubIdx,
+                     Register DestReg, unsigned SubIdx,
                      const MachineInstr &Orig,
                      const TargetRegisterInfo &TRI) const override;
 
@@ -296,16 +296,16 @@ class ARMBaseInstrInfo : public ARMGenInstrInfo {
   /// in SrcReg and SrcReg2 if having two register operands, and the value it
   /// compares against in CmpValue. Return true if the comparison instruction
   /// can be analyzed.
-  bool analyzeCompare(const MachineInstr &MI, unsigned &SrcReg,
-                      unsigned &SrcReg2, int &CmpMask,
+  bool analyzeCompare(const MachineInstr &MI, Register &SrcReg,
+                      Register &SrcReg2, int &CmpMask,
                       int &CmpValue) const override;
 
   /// optimizeCompareInstr - Convert the instruction to set the zero flag so
   /// that we can remove a "comparison with zero"; Remove a redundant CMP
   /// instruction if the flags can be updated in the same way by an earlier
   /// instruction such as SUB.
-  bool optimizeCompareInstr(MachineInstr &CmpInstr, unsigned SrcReg,
-                            unsigned SrcReg2, int CmpMask, int CmpValue,
+  bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
+                            Register SrcReg2, int CmpMask, int CmpValue,
                             const MachineRegisterInfo *MRI) const override;
 
   bool analyzeSelect(const MachineInstr &MI,
@@ -318,7 +318,7 @@ class ARMBaseInstrInfo : public ARMGenInstrInfo {
 
   /// FoldImmediate - 'Reg' is known to be defined by a move immediate
   /// instruction, try to fold the immediate into the use instruction.
-  bool FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, unsigned Reg,
+  bool FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg,
                      MachineRegisterInfo *MRI) const override;
 
   unsigned getNumMicroOps(const InstrItineraryData *ItinData,
@@ -503,16 +503,16 @@ bool isUncondBranchOpcode(int Opc) {
 // the ArmARM.
 
 
-inline static unsigned getARMVPTBlockMask(unsigned NumInsts) {
+inline static ARM::PredBlockMask getARMVPTBlockMask(unsigned NumInsts) {
   switch (NumInsts) {
   case 1:
-    return ARMVCC::T;
+    return ARM::PredBlockMask::T;
   case 2:
-    return ARMVCC::TT;
+    return ARM::PredBlockMask::TT;
   case 3:
-    return ARMVCC::TTT;
+    return ARM::PredBlockMask::TTT;
   case 4:
-    return ARMVCC::TTTT;
+    return ARM::PredBlockMask::TTTT;
   default:
     break;
   };
diff --git a/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp b/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp
index 7d6fe0c49692b..9e057a82a43fc 100644
--- a/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp
@@ -601,9 +601,9 @@ needsFrameBaseReg(MachineInstr *MI, int64_t Offset) const {
   // The FP is only available if there is no dynamic realignment. We
   // don't know for sure yet whether we'll need that, so we guess based
   // on whether there are any local variables that would trigger it.
-  unsigned StackAlign = TFI->getStackAlignment();
   if (TFI->hasFP(MF) &&
-      !((MFI.getLocalFrameMaxAlign() > StackAlign) && canRealignStack(MF))) {
+      !((MFI.getLocalFrameMaxAlign() > TFI->getStackAlign()) &&
+        canRealignStack(MF))) {
     if (isFrameOffsetLegal(MI, getFrameRegister(MF), FPOffset))
       return false;
   }
diff --git a/llvm/lib/Target/ARM/ARMCallLowering.cpp b/llvm/lib/Target/ARM/ARMCallLowering.cpp
index c2bd24d9cd0ea..6a8ad3fdff266 100644
--- a/llvm/lib/Target/ARM/ARMCallLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMCallLowering.cpp
@@ -130,7 +130,7 @@ struct OutgoingValueHandler : public CallLowering::ValueHandler {
     Register ExtReg = extendRegister(ValVReg, VA);
     auto MMO = MIRBuilder.getMF().getMachineMemOperand(
         MPO, MachineMemOperand::MOStore, VA.getLocVT().getStoreSize(),
-        /* Alignment */ 1);
+        Align(1));
     MIRBuilder.buildStore(ExtReg, Addr, *MMO);
   }
 
@@ -323,10 +323,9 @@ struct IncomingValueHandler : public CallLowering::ValueHandler {
   MachineInstrBuilder buildLoad(const DstOp &Res, Register Addr, uint64_t Size,
                                 MachinePointerInfo &MPO) {
     MachineFunction &MF = MIRBuilder.getMF();
-    unsigned Alignment = inferAlignmentFromPtrInfo(MF, MPO);
 
-    auto MMO = MF.getMachineMemOperand(
-        MPO, MachineMemOperand::MOLoad, Size, Alignment);
+    auto MMO = MF.getMachineMemOperand(MPO, MachineMemOperand::MOLoad, Size,
+                                       inferAlignFromPtrInfo(MF, MPO));
     return MIRBuilder.buildLoad(Res, Addr, *MMO);
   }
 
diff --git a/llvm/lib/Target/ARM/ARMFastISel.cpp b/llvm/lib/Target/ARM/ARMFastISel.cpp
index 6e19db3c7e224..11bff747837c5 100644
--- a/llvm/lib/Target/ARM/ARMFastISel.cpp
+++ b/llvm/lib/Target/ARM/ARMFastISel.cpp
@@ -209,7 +209,7 @@ class ARMFastISel final : public FastISel {
     unsigned ARMMoveToFPReg(MVT VT, unsigned SrcReg);
     unsigned ARMMoveToIntReg(MVT VT, unsigned SrcReg);
     unsigned ARMSelectCallOp(bool UseReg);
-    unsigned ARMLowerPICELF(const GlobalValue *GV, unsigned Align, MVT VT);
+    unsigned ARMLowerPICELF(const GlobalValue *GV, MVT VT);
 
     const TargetLowering *getTargetLowering() { return &TLI; }
 
@@ -570,14 +570,10 @@ unsigned ARMFastISel::ARMMaterializeGV(const GlobalValue *GV, MVT VT) {
                             TII.get(Opc), DestReg).addGlobalAddress(GV, 0, TF));
   } else {
     // MachineConstantPool wants an explicit alignment.
-    unsigned Align = DL.getPrefTypeAlignment(GV->getType());
-    if (Align == 0) {
-      // TODO: Figure out if this is correct.
-      Align = DL.getTypeAllocSize(GV->getType());
-    }
+    Align Alignment = DL.getPrefTypeAlign(GV->getType());
 
     if (Subtarget->isTargetELF() && IsPositionIndependent)
-      return ARMLowerPICELF(GV, Align, VT);
+      return ARMLowerPICELF(GV, VT);
 
     // Grab index.
     unsigned PCAdj = IsPositionIndependent ? (Subtarget->isThumb() ? 4 : 8) : 0;
@@ -585,7 +581,7 @@ unsigned ARMFastISel::ARMMaterializeGV(const GlobalValue *GV, MVT VT) {
     ARMConstantPoolValue *CPV = ARMConstantPoolConstant::Create(GV, Id,
                                                                 ARMCP::CPValue,
                                                                 PCAdj);
-    unsigned Idx = MCP.getConstantPoolIndex(CPV, Align);
+    unsigned Idx = MCP.getConstantPoolIndex(CPV, Alignment.value());
 
     // Load value.
     MachineInstrBuilder MIB;
@@ -882,7 +878,7 @@ void ARMFastISel::AddLoadStoreOperands(MVT VT, Address &Addr,
     int Offset = Addr.Offset;
     MachineMemOperand *MMO = FuncInfo.MF->getMachineMemOperand(
         MachinePointerInfo::getFixedStack(*FuncInfo.MF, FI, Offset), Flags,
-        MFI.getObjectSize(FI), MFI.getObjectAlignment(FI));
+        MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
     // Now add the rest of the operands.
     MIB.addFrameIndex(FI);
 
@@ -2949,8 +2945,7 @@ bool ARMFastISel::tryToFoldLoadIntoMI(MachineInstr *MI, unsigned OpNo,
   return true;
 }
 
-unsigned ARMFastISel::ARMLowerPICELF(const GlobalValue *GV,
-                                     unsigned Align, MVT VT) {
+unsigned ARMFastISel::ARMLowerPICELF(const GlobalValue *GV, MVT VT) {
   bool UseGOT_PREL = !TM.shouldAssumeDSOLocal(*GV->getParent(), GV);
 
   LLVMContext *Context = &MF->getFunction().getContext();
@@ -2966,7 +2961,7 @@ unsigned ARMFastISel::ARMLowerPICELF(const GlobalValue *GV,
   unsigned Idx = MF->getConstantPool()->getConstantPoolIndex(CPV, ConstAlign);
   MachineMemOperand *CPMMO =
       MF->getMachineMemOperand(MachinePointerInfo::getConstantPool(*MF),
-                               MachineMemOperand::MOLoad, 4, 4);
+                               MachineMemOperand::MOLoad, 4, Align(4));
 
   Register TempReg = MF->getRegInfo().createVirtualRegister(&ARM::rGPRRegClass);
   unsigned Opc = isThumb2 ? ARM::t2LDRpci : ARM::LDRcp;
diff --git a/llvm/lib/Target/ARM/ARMFrameLowering.cpp b/llvm/lib/Target/ARM/ARMFrameLowering.cpp
index 0822a1c062439..f9203f0e453ee 100644
--- a/llvm/lib/Target/ARM/ARMFrameLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMFrameLowering.cpp
@@ -1596,7 +1596,7 @@ checkNumAlignedDPRCS2Regs(MachineFunction &MF, BitVector &SavedRegs) {
     return;
 
   // Don't bother if the default stack alignment is sufficiently high.
-  if (MF.getSubtarget().getFrameLowering()->getStackAlignment() >= 8)
+  if (MF.getSubtarget().getFrameLowering()->getStackAlign() >= Align(8))
     return;
 
   // Aligned spills require stack realignment.
@@ -2292,8 +2292,8 @@ void ARMFrameLowering::determineCalleeSaves(MachineFunction &MF,
     // of GPRs, spill one extra callee save GPR so we won't have to pad between
     // the integer and double callee save areas.
     LLVM_DEBUG(dbgs() << "NumGPRSpills = " << NumGPRSpills << "\n");
-    unsigned TargetAlign = getStackAlignment();
-    if (TargetAlign >= 8 && (NumGPRSpills & 1)) {
+    const Align TargetAlign = getStackAlign();
+    if (TargetAlign >= Align(8) && (NumGPRSpills & 1)) {
       if (CS1Spilled && !UnspilledCS1GPRs.empty()) {
         for (unsigned i = 0, e = UnspilledCS1GPRs.size(); i != e; ++i) {
           unsigned Reg = UnspilledCS1GPRs[i];
@@ -2330,7 +2330,7 @@ void ARMFrameLowering::determineCalleeSaves(MachineFunction &MF,
     if (BigFrameOffsets && !ExtraCSSpill) {
       // If any non-reserved CS register isn't spilled, just spill one or two
       // extra. That should take care of it!
-      unsigned NumExtras = TargetAlign / 4;
+      unsigned NumExtras = TargetAlign.value() / 4;
       SmallVector<unsigned, 2> Extras;
       while (NumExtras && !UnspilledCS1GPRs.empty()) {
         unsigned Reg = UnspilledCS1GPRs.back();
diff --git a/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp
index b334f4156559a..3c6f446580bbe 100644
--- a/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp
+++ b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp
@@ -1191,8 +1191,8 @@ bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N,
     // Only multiples of 4 are allowed for the offset, so the frame object
     // alignment must be at least 4.
     MachineFrameInfo &MFI = MF->getFrameInfo();
-    if (MFI.getObjectAlignment(FI) < 4)
-      MFI.setObjectAlignment(FI, 4);
+    if (MFI.getObjectAlign(FI) < Align(4))
+      MFI.setObjectAlignment(FI, Align(4));
     Base = CurDAG->getTargetFrameIndex(
         FI, TLI->getPointerTy(CurDAG->getDataLayout()));
     OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
@@ -1215,9 +1215,9 @@ bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N,
       if (RHSC * 4 < MFI.getObjectSize(FI)) {
         // For LHS+RHS to result in an offset that's a multiple of 4 the object
         // indexed by the LHS must be 4-byte aligned.
-        if (!MFI.isFixedObjectIndex(FI) && MFI.getObjectAlignment(FI) < 4)
-          MFI.setObjectAlignment(FI, 4);
-        if (MFI.getObjectAlignment(FI) >= 4) {
+        if (!MFI.isFixedObjectIndex(FI) && MFI.getObjectAlign(FI) < Align(4))
+          MFI.setObjectAlignment(FI, Align(4));
+        if (MFI.getObjectAlign(FI) >= Align(4)) {
           Base = CurDAG->getTargetFrameIndex(
               FI, TLI->getPointerTy(CurDAG->getDataLayout()));
           OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
@@ -3400,7 +3400,7 @@ void ARMDAGToDAGISel::Select(SDNode *N) {
       MachineFunction& MF = CurDAG->getMachineFunction();
       MachineMemOperand *MemOp =
           MF.getMachineMemOperand(MachinePointerInfo::getConstantPool(MF),
-                                  MachineMemOperand::MOLoad, 4, 4);
+                                  MachineMemOperand::MOLoad, 4, Align(4));
 
       CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {MemOp});
 
@@ -3420,8 +3420,8 @@ void ARMDAGToDAGISel::Select(SDNode *N) {
       // Set the alignment of the frame object to 4, to avoid having to generate
       // more than one ADD
       MachineFrameInfo &MFI = MF->getFrameInfo();
-      if (MFI.getObjectAlignment(FI) < 4)
-        MFI.setObjectAlignment(FI, 4);
+      if (MFI.getObjectAlign(FI) < Align(4))
+        MFI.setObjectAlignment(FI, Align(4));
       CurDAG->SelectNodeTo(N, ARM::tADDframe, MVT::i32, TFI,
                            CurDAG->getTargetConstant(0, dl, MVT::i32));
       return;
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 03cff21104791..e4d1caa0b1e0d 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -943,6 +943,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
     setTargetDAGCombine(ISD::INTRINSIC_VOID);
     setTargetDAGCombine(ISD::VECREDUCE_ADD);
     setTargetDAGCombine(ISD::ADD);
+    setTargetDAGCombine(ISD::BITCAST);
   }
 
   if (!Subtarget->hasFP64()) {
@@ -2272,9 +2273,9 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
         for (i = 0, j = RegBegin; j < RegEnd; i++, j++) {
           SDValue Const = DAG.getConstant(4*i, dl, MVT::i32);
           SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const);
-          SDValue Load = DAG.getLoad(PtrVT, dl, Chain, AddArg,
-                                     MachinePointerInfo(),
-                                     DAG.InferPtrAlignment(AddArg));
+          SDValue Load =
+              DAG.getLoad(PtrVT, dl, Chain, AddArg, MachinePointerInfo(),
+                          DAG.InferPtrAlign(AddArg));
           MemOpChains.push_back(Load.getValue(1));
           RegsToPass.push_back(std::make_pair(j, Load));
         }
@@ -6442,9 +6443,10 @@ static SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) {
 /// immediate" operand (e.g., VMOV).  If so, return the encoded value.
 static SDValue isVMOVModifiedImm(uint64_t SplatBits, uint64_t SplatUndef,
                                  unsigned SplatBitSize, SelectionDAG &DAG,
-                                 const SDLoc &dl, EVT &VT, bool is128Bits,
+                                 const SDLoc &dl, EVT &VT, EVT VectorVT,
                                  VMOVModImmType type) {
   unsigned OpCmode, Imm;
+  bool is128Bits = VectorVT.is128BitVector();
 
   // SplatBitSize is set to the smallest size that splats the vector, so a
   // zero vector will always have SplatBitSize == 8.  However, NEON modified
@@ -6562,9 +6564,18 @@ static SDValue isVMOVModifiedImm(uint64_t SplatBits, uint64_t SplatUndef,
       ImmMask <<= 1;
     }
 
-    if (DAG.getDataLayout().isBigEndian())
-      // swap higher and lower 32 bit word
-      Imm = ((Imm & 0xf) << 4) | ((Imm & 0xf0) >> 4);
+    if (DAG.getDataLayout().isBigEndian()) {
+      // Reverse the order of elements within the vector.
+      unsigned BytesPerElem = VectorVT.getScalarSizeInBits() / 8;
+      unsigned Mask = (1 << BytesPerElem) - 1;
+      unsigned NumElems = 8 / BytesPerElem;
+      unsigned NewImm = 0;
+      for (unsigned ElemNum = 0; ElemNum < NumElems; ++ElemNum) {
+        unsigned Elem = ((Imm >> ElemNum * BytesPerElem) & Mask);
+        NewImm |= Elem << (NumElems - ElemNum - 1) * BytesPerElem;
+      }
+      Imm = NewImm;
+    }
 
     // Op=1, Cmode=1110.
     OpCmode = 0x1e;
@@ -6657,7 +6668,7 @@ SDValue ARMTargetLowering::LowerConstantFP(SDValue Op, SelectionDAG &DAG,
 
   // Try a VMOV.i32 (FIXME: i8, i16, or i64 could work too).
   SDValue NewVal = isVMOVModifiedImm(iVal & 0xffffffffU, 0, 32, DAG, SDLoc(Op),
-                                     VMovVT, false, VMOVModImm);
+                                     VMovVT, VT, VMOVModImm);
   if (NewVal != SDValue()) {
     SDLoc DL(Op);
     SDValue VecConstant = DAG.getNode(ARMISD::VMOVIMM, DL, VMovVT,
@@ -6674,7 +6685,7 @@ SDValue ARMTargetLowering::LowerConstantFP(SDValue Op, SelectionDAG &DAG,
 
   // Finally, try a VMVN.i32
   NewVal = isVMOVModifiedImm(~iVal & 0xffffffffU, 0, 32, DAG, SDLoc(Op), VMovVT,
-                             false, VMVNModImm);
+                             VT, VMVNModImm);
   if (NewVal != SDValue()) {
     SDLoc DL(Op);
     SDValue VecConstant = DAG.getNode(ARMISD::VMVNIMM, DL, VMovVT, NewVal);
@@ -7181,13 +7192,12 @@ SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
       return DAG.getUNDEF(VT);
 
     if ((ST->hasNEON() && SplatBitSize <= 64) ||
-        (ST->hasMVEIntegerOps() && SplatBitSize <= 32)) {
+        (ST->hasMVEIntegerOps() && SplatBitSize <= 64)) {
       // Check if an immediate VMOV works.
       EVT VmovVT;
-      SDValue Val = isVMOVModifiedImm(SplatBits.getZExtValue(),
-                                      SplatUndef.getZExtValue(), SplatBitSize,
-                                      DAG, dl, VmovVT, VT.is128BitVector(),
-                                      VMOVModImm);
+      SDValue Val =
+          isVMOVModifiedImm(SplatBits.getZExtValue(), SplatUndef.getZExtValue(),
+                            SplatBitSize, DAG, dl, VmovVT, VT, VMOVModImm);
 
       if (Val.getNode()) {
         SDValue Vmov = DAG.getNode(ARMISD::VMOVIMM, dl, VmovVT, Val);
@@ -7197,9 +7207,8 @@ SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
       // Try an immediate VMVN.
       uint64_t NegatedImm = (~SplatBits).getZExtValue();
       Val = isVMOVModifiedImm(
-          NegatedImm, SplatUndef.getZExtValue(), SplatBitSize,
-          DAG, dl, VmovVT, VT.is128BitVector(),
-          ST->hasMVEIntegerOps() ? MVEVMVNModImm : VMVNModImm);
+          NegatedImm, SplatUndef.getZExtValue(), SplatBitSize, DAG, dl, VmovVT,
+          VT, ST->hasMVEIntegerOps() ? MVEVMVNModImm : VMVNModImm);
       if (Val.getNode()) {
         SDValue Vmov = DAG.getNode(ARMISD::VMVNIMM, dl, VmovVT, Val);
         return DAG.getNode(ISD::BITCAST, dl, VT, Vmov);
@@ -9223,9 +9232,10 @@ static SDValue LowerMLOAD(SDValue Op, SelectionDAG &DAG) {
       N->getMemoryVT(), N->getMemOperand(), N->getAddressingMode(),
       N->getExtensionType(), N->isExpandingLoad());
   SDValue Combo = NewLoad;
-  if (!PassThru.isUndef() &&
-      (PassThru.getOpcode() != ISD::BITCAST ||
-       !isZeroVector(PassThru->getOperand(0))))
+  bool PassThruIsCastZero = (PassThru.getOpcode() == ISD::BITCAST ||
+                             PassThru.getOpcode() == ARMISD::VECTOR_REG_CAST) &&
+                            isZeroVector(PassThru->getOperand(0));
+  if (!PassThru.isUndef() && !PassThruIsCastZero)
     Combo = DAG.getNode(ISD::VSELECT, dl, VT, Mask, NewLoad, PassThru);
   return DAG.getMergeValues({Combo, NewLoad.getValue(1)}, dl);
 }
@@ -9578,11 +9588,11 @@ void ARMTargetLowering::SetupEntryBlockForSjLj(MachineInstr &MI,
   // Grab constant pool and fixed stack memory operands.
   MachineMemOperand *CPMMO =
       MF->getMachineMemOperand(MachinePointerInfo::getConstantPool(*MF),
-                               MachineMemOperand::MOLoad, 4, 4);
+                               MachineMemOperand::MOLoad, 4, Align(4));
 
   MachineMemOperand *FIMMOSt =
       MF->getMachineMemOperand(MachinePointerInfo::getFixedStack(*MF, FI),
-                               MachineMemOperand::MOStore, 4, 4);
+                               MachineMemOperand::MOStore, 4, Align(4));
 
   // Load the address of the dispatch MBB into the jump buffer.
   if (isThumb2) {
@@ -9768,7 +9778,7 @@ void ARMTargetLowering::EmitSjLjDispatchBlock(MachineInstr &MI,
 
   MachineMemOperand *FIMMOLd = MF->getMachineMemOperand(
       MachinePointerInfo::getFixedStack(*MF, FI),
-      MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile, 4, 4);
+      MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile, 4, Align(4));
 
   MachineInstrBuilder MIB;
   MIB = BuildMI(DispatchBB, dl, TII->get(ARM::Int_eh_sjlj_dispatchsetup));
@@ -9899,8 +9909,9 @@ void ARMTargetLowering::EmitSjLjDispatchBlock(MachineInstr &MI,
         .addReg(NewVReg3)
         .add(predOps(ARMCC::AL));
 
-    MachineMemOperand *JTMMOLd = MF->getMachineMemOperand(
-        MachinePointerInfo::getJumpTable(*MF), MachineMemOperand::MOLoad, 4, 4);
+    MachineMemOperand *JTMMOLd =
+        MF->getMachineMemOperand(MachinePointerInfo::getJumpTable(*MF),
+                                 MachineMemOperand::MOLoad, 4, Align(4));
 
     Register NewVReg5 = MRI->createVirtualRegister(TRC);
     BuildMI(DispContBB, dl, TII->get(ARM::tLDRi), NewVReg5)
@@ -9993,8 +10004,9 @@ void ARMTargetLowering::EmitSjLjDispatchBlock(MachineInstr &MI,
         .addJumpTableIndex(MJTI)
         .add(predOps(ARMCC::AL));
 
-    MachineMemOperand *JTMMOLd = MF->getMachineMemOperand(
-        MachinePointerInfo::getJumpTable(*MF), MachineMemOperand::MOLoad, 4, 4);
+    MachineMemOperand *JTMMOLd =
+        MF->getMachineMemOperand(MachinePointerInfo::getJumpTable(*MF),
+                                 MachineMemOperand::MOLoad, 4, Align(4));
     Register NewVReg5 = MRI->createVirtualRegister(TRC);
     BuildMI(DispContBB, dl, TII->get(ARM::LDRrs), NewVReg5)
         .addReg(NewVReg3, RegState::Kill)
@@ -10233,7 +10245,7 @@ ARMTargetLowering::EmitStructByval(MachineInstr &MI,
   Register dest = MI.getOperand(0).getReg();
   Register src = MI.getOperand(1).getReg();
   unsigned SizeVal = MI.getOperand(2).getImm();
-  unsigned Align = MI.getOperand(3).getImm();
+  unsigned Alignment = MI.getOperand(3).getImm();
   DebugLoc dl = MI.getDebugLoc();
 
   MachineFunction *MF = BB->getParent();
@@ -10246,17 +10258,17 @@ ARMTargetLowering::EmitStructByval(MachineInstr &MI,
   bool IsThumb2 = Subtarget->isThumb2();
   bool IsThumb = Subtarget->isThumb();
 
-  if (Align & 1) {
+  if (Alignment & 1) {
     UnitSize = 1;
-  } else if (Align & 2) {
+  } else if (Alignment & 2) {
     UnitSize = 2;
   } else {
     // Check whether we can use NEON instructions.
     if (!MF->getFunction().hasFnAttribute(Attribute::NoImplicitFloat) &&
         Subtarget->hasNEON()) {
-      if ((Align % 16 == 0) && SizeVal >= 16)
+      if ((Alignment % 16 == 0) && SizeVal >= 16)
         UnitSize = 16;
-      else if ((Align % 8 == 0) && SizeVal >= 8)
+      else if ((Alignment % 8 == 0) && SizeVal >= 8)
         UnitSize = 8;
     }
     // Can't use NEON instructions.
@@ -10362,13 +10374,11 @@ ARMTargetLowering::EmitStructByval(MachineInstr &MI,
     const Constant *C = ConstantInt::get(Int32Ty, LoopSize);
 
     // MachineConstantPool wants an explicit alignment.
-    unsigned Align = MF->getDataLayout().getPrefTypeAlignment(Int32Ty);
-    if (Align == 0)
-      Align = MF->getDataLayout().getTypeAllocSize(C->getType());
-    unsigned Idx = ConstantPool->getConstantPoolIndex(C, Align);
+    Align Alignment = MF->getDataLayout().getPrefTypeAlign(Int32Ty);
+    unsigned Idx = ConstantPool->getConstantPoolIndex(C, Alignment.value());
     MachineMemOperand *CPMMO =
         MF->getMachineMemOperand(MachinePointerInfo::getConstantPool(*MF),
-                                 MachineMemOperand::MOLoad, 4, 4);
+                                 MachineMemOperand::MOLoad, 4, Align(4));
 
     if (IsThumb)
       BuildMI(*BB, MI, dl, TII->get(ARM::tLDRpci))
@@ -12125,18 +12135,86 @@ static SDValue PerformVMULCombine(SDNode *N,
                      DAG.getNode(ISD::MUL, DL, VT, N01, N1));
 }
 
+static SDValue PerformMVEVMULLCombine(SDNode *N, SelectionDAG &DAG,
+                                      const ARMSubtarget *Subtarget) {
+  EVT VT = N->getValueType(0);
+  if (VT != MVT::v2i64)
+    return SDValue();
+
+  SDValue N0 = N->getOperand(0);
+  SDValue N1 = N->getOperand(1);
+
+  auto IsSignExt = [&](SDValue Op) {
+    if (Op->getOpcode() != ISD::SIGN_EXTEND_INREG)
+      return SDValue();
+    EVT VT = cast<VTSDNode>(Op->getOperand(1))->getVT();
+    if (VT.getScalarSizeInBits() == 32)
+      return Op->getOperand(0);
+    return SDValue();
+  };
+  auto IsZeroExt = [&](SDValue Op) {
+    // Zero extends are a little more awkward. At the point we are matching
+    // this, we are looking for an AND with a (-1, 0, -1, 0) buildvector mask.
+    // That might be before of after a bitcast depending on how the and is
+    // placed. Because this has to look through bitcasts, it is currently only
+    // supported on LE.
+    if (!Subtarget->isLittle())
+      return SDValue();
+
+    SDValue And = Op;
+    if (And->getOpcode() == ISD::BITCAST)
+      And = And->getOperand(0);
+    if (And->getOpcode() != ISD::AND)
+      return SDValue();
+    SDValue Mask = And->getOperand(1);
+    if (Mask->getOpcode() == ISD::BITCAST)
+      Mask = Mask->getOperand(0);
+
+    if (Mask->getOpcode() != ISD::BUILD_VECTOR ||
+        Mask.getValueType() != MVT::v4i32)
+      return SDValue();
+    if (isAllOnesConstant(Mask->getOperand(0)) &&
+        isNullConstant(Mask->getOperand(1)) &&
+        isAllOnesConstant(Mask->getOperand(2)) &&
+        isNullConstant(Mask->getOperand(3)))
+      return And->getOperand(0);
+    return SDValue();
+  };
+
+  SDLoc dl(N);
+  if (SDValue Op0 = IsSignExt(N0)) {
+    if (SDValue Op1 = IsSignExt(N1)) {
+      SDValue New0a = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Op0);
+      SDValue New1a = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Op1);
+      return DAG.getNode(ARMISD::VMULLs, dl, VT, New0a, New1a);
+    }
+  }
+  if (SDValue Op0 = IsZeroExt(N0)) {
+    if (SDValue Op1 = IsZeroExt(N1)) {
+      SDValue New0a = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Op0);
+      SDValue New1a = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Op1);
+      return DAG.getNode(ARMISD::VMULLu, dl, VT, New0a, New1a);
+    }
+  }
+
+  return SDValue();
+}
+
 static SDValue PerformMULCombine(SDNode *N,
                                  TargetLowering::DAGCombinerInfo &DCI,
                                  const ARMSubtarget *Subtarget) {
   SelectionDAG &DAG = DCI.DAG;
 
+  EVT VT = N->getValueType(0);
+  if (Subtarget->hasMVEIntegerOps() && VT == MVT::v2i64)
+    return PerformMVEVMULLCombine(N, DAG, Subtarget);
+
   if (Subtarget->isThumb1Only())
     return SDValue();
 
   if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
     return SDValue();
 
-  EVT VT = N->getValueType(0);
   if (VT.is64BitVector() || VT.is128BitVector())
     return PerformVMULCombine(N, DCI, Subtarget);
   if (VT != MVT::i32)
@@ -12333,8 +12411,7 @@ static SDValue PerformANDCombine(SDNode *N,
       EVT VbicVT;
       SDValue Val = isVMOVModifiedImm((~SplatBits).getZExtValue(),
                                       SplatUndef.getZExtValue(), SplatBitSize,
-                                      DAG, dl, VbicVT, VT.is128BitVector(),
-                                      OtherModImm);
+                                      DAG, dl, VbicVT, VT, OtherModImm);
       if (Val.getNode()) {
         SDValue Input =
           DAG.getNode(ISD::BITCAST, dl, VbicVT, N->getOperand(0));
@@ -12638,10 +12715,9 @@ static SDValue PerformORCombine(SDNode *N,
       BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
     if (SplatBitSize <= 64) {
       EVT VorrVT;
-      SDValue Val = isVMOVModifiedImm(SplatBits.getZExtValue(),
-                                      SplatUndef.getZExtValue(), SplatBitSize,
-                                      DAG, dl, VorrVT, VT.is128BitVector(),
-                                      OtherModImm);
+      SDValue Val =
+          isVMOVModifiedImm(SplatBits.getZExtValue(), SplatUndef.getZExtValue(),
+                            SplatBitSize, DAG, dl, VorrVT, VT, OtherModImm);
       if (Val.getNode()) {
         SDValue Input =
           DAG.getNode(ISD::BITCAST, dl, VorrVT, N->getOperand(0));
@@ -13924,7 +14000,7 @@ static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St,
   // Details about the old store
   SDValue Ch = St->getChain();
   SDValue BasePtr = St->getBasePtr();
-  unsigned Alignment = St->getOriginalAlignment();
+  Align Alignment = St->getOriginalAlign();
   MachineMemOperand::Flags MMOFlags = St->getMemOperand()->getFlags();
   AAMDNodes AAInfo = St->getAAInfo();
 
@@ -13941,7 +14017,7 @@ static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St,
                     DAG.getConstant(i * NumElements, DL, MVT::i32));
     SDValue Store = DAG.getTruncStore(
         Ch, DL, Extract, NewPtr, St->getPointerInfo().getWithOffset(NewOffset),
-        NewToVT, Alignment, MMOFlags, AAInfo);
+        NewToVT, Alignment.value(), MMOFlags, AAInfo);
     Stores.push_back(Store);
   }
   return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Stores);
@@ -14586,7 +14662,7 @@ static SDValue PerformSplittingToWideningLoad(SDNode *N, SelectionDAG &DAG) {
   // Details about the old load
   SDValue Ch = LD->getChain();
   SDValue BasePtr = LD->getBasePtr();
-  unsigned Alignment = LD->getOriginalAlignment();
+  Align Alignment = LD->getOriginalAlign();
   MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
   AAMDNodes AAInfo = LD->getAAInfo();
 
@@ -14602,13 +14678,13 @@ static SDValue PerformSplittingToWideningLoad(SDNode *N, SelectionDAG &DAG) {
   // is good enough, as legalisation will take it from there. They are either
   // already legal or they will be split further into something that is
   // legal.
-  SDValue NewLoad1 =
-      DAG.getLoad(ISD::UNINDEXED, NewExtType, NewToVT, DL, Ch, BasePtr, Offset,
-                  LD->getPointerInfo(), NewFromVT, Alignment, MMOFlags, AAInfo);
+  SDValue NewLoad1 = DAG.getLoad(
+      ISD::UNINDEXED, NewExtType, NewToVT, DL, Ch, BasePtr, Offset,
+      LD->getPointerInfo(), NewFromVT, Alignment.value(), MMOFlags, AAInfo);
   SDValue NewLoad2 =
       DAG.getLoad(ISD::UNINDEXED, NewExtType, NewToVT, DL, Ch, NewPtr, Offset,
                   LD->getPointerInfo().getWithOffset(NewOffset), NewFromVT,
-                  Alignment, MMOFlags, AAInfo);
+                  Alignment.value(), MMOFlags, AAInfo);
 
   SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
                                  SDValue(NewLoad1.getNode(), 1),
@@ -15143,6 +15219,28 @@ ARMTargetLowering::PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const {
   return Res;
 }
 
+static SDValue PerformBITCASTCombine(SDNode *N, SelectionDAG &DAG) {
+  SDValue Src = N->getOperand(0);
+
+  // We may have a bitcast of something that has already had this bitcast
+  // combine performed on it, so skip past any VECTOR_REG_CASTs.
+  while (Src.getOpcode() == ARMISD::VECTOR_REG_CAST)
+    Src = Src.getOperand(0);
+
+  // Bitcast from element-wise VMOV or VMVN doesn't need VREV if the VREV that
+  // would be generated is at least the width of the element type.
+  EVT SrcVT = Src.getValueType();
+  EVT DstVT = N->getValueType(0);
+  if ((Src.getOpcode() == ARMISD::VMOVIMM ||
+       Src.getOpcode() == ARMISD::VMVNIMM ||
+       Src.getOpcode() == ARMISD::VMOVFPIMM) &&
+      SrcVT.getScalarSizeInBits() <= DstVT.getScalarSizeInBits() &&
+      DAG.getDataLayout().isBigEndian())
+    return DAG.getNode(ARMISD::VECTOR_REG_CAST, SDLoc(N), DstVT, Src);
+
+  return SDValue();
+}
+
 SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N,
                                              DAGCombinerInfo &DCI) const {
   switch (N->getOpcode()) {
@@ -15196,6 +15294,8 @@ SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N,
     return PerformVLDCombine(N, DCI);
   case ARMISD::BUILD_VECTOR:
     return PerformARMBUILD_VECTORCombine(N, DCI);
+  case ISD::BITCAST:
+    return PerformBITCASTCombine(N, DCI.DAG);
   case ARMISD::PREDICATE_CAST:
     return PerformPREDICATE_CASTCombine(N, DCI);
   case ARMISD::VECTOR_REG_CAST:
@@ -15512,13 +15612,25 @@ bool ARMTargetLowering::shouldSinkOperands(Instruction *I,
   if (!Subtarget->hasMVEIntegerOps())
     return false;
 
-  auto IsSinker = [](Instruction *I, int Operand) {
+  auto IsFMSMul = [&](Instruction *I) {
+    if (!I->hasOneUse())
+      return false;
+    auto *Sub = cast<Instruction>(*I->users().begin());
+    return Sub->getOpcode() == Instruction::FSub && Sub->getOperand(1) == I;
+  };
+
+  auto IsSinker = [&](Instruction *I, int Operand) {
     switch (I->getOpcode()) {
     case Instruction::Add:
     case Instruction::Mul:
+    case Instruction::FAdd:
     case Instruction::ICmp:
+    case Instruction::FCmp:
       return true;
+    case Instruction::FMul:
+      return !IsFMSMul(I);
     case Instruction::Sub:
+    case Instruction::FSub:
     case Instruction::Shl:
     case Instruction::LShr:
     case Instruction::AShr:
@@ -15535,7 +15647,7 @@ bool ARMTargetLowering::shouldSinkOperands(Instruction *I,
     return false;
   if (!match(I->getOperand(Op),
              m_ShuffleVector(m_InsertElement(m_Undef(), m_Value(), m_ZeroInt()),
-                             m_Undef(), m_Zero()))) {
+                             m_Undef(), m_ZeroMask()))) {
     return false;
   }
   Instruction *Shuffle = cast<Instruction>(I->getOperand(Op));
@@ -17225,7 +17337,7 @@ bool ARMTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     Info.memVT = MVT::getVT(PtrTy->getElementType());
     Info.ptrVal = I.getArgOperand(0);
     Info.offset = 0;
-    Info.align = MaybeAlign(DL.getABITypeAlignment(PtrTy->getElementType()));
+    Info.align = DL.getABITypeAlign(PtrTy->getElementType());
     Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile;
     return true;
   }
@@ -17237,7 +17349,7 @@ bool ARMTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     Info.memVT = MVT::getVT(PtrTy->getElementType());
     Info.ptrVal = I.getArgOperand(1);
     Info.offset = 0;
-    Info.align = MaybeAlign(DL.getABITypeAlignment(PtrTy->getElementType()));
+    Info.align = DL.getABITypeAlign(PtrTy->getElementType());
     Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
     return true;
   }
diff --git a/llvm/lib/Target/ARM/ARMInstrCDE.td b/llvm/lib/Target/ARM/ARMInstrCDE.td
index 93f3d9be24a40..4c4a65a519c46 100644
--- a/llvm/lib/Target/ARM/ARMInstrCDE.td
+++ b/llvm/lib/Target/ARM/ARMInstrCDE.td
@@ -606,3 +606,61 @@ let Predicates = [HasCDE, HasMVEInt] in {
             (v16i8 (CDE_VCX3A_vec p_imm:$coproc, MQPR:$acc, MQPR:$n, MQPR:$m,
                                   imm_4b:$imm))>;
 }
+
+multiclass VCXPredicatedPat_m<MVEVectorVTInfo VTI> {
+  def : Pat<(VTI.Vec (int_arm_cde_vcx1q_predicated timm:$coproc,
+                                    (VTI.Vec MQPR:$inactive), timm:$imm,
+                                    (VTI.Pred VCCR:$pred))),
+            (VTI.Vec (CDE_VCX1_vec p_imm:$coproc, imm_12b:$imm, ARMVCCThen,
+                                    (VTI.Pred VCCR:$pred),
+                                    (VTI.Vec MQPR:$inactive)))>;
+  def : Pat<(VTI.Vec (int_arm_cde_vcx1qa_predicated timm:$coproc,
+                                    (VTI.Vec MQPR:$acc), timm:$imm,
+                                    (VTI.Pred VCCR:$pred))),
+            (VTI.Vec (CDE_VCX1A_vec p_imm:$coproc, (VTI.Vec MQPR:$acc),
+                                    imm_12b:$imm, ARMVCCThen,
+                                    (VTI.Pred VCCR:$pred)))>;
+
+  def : Pat<(VTI.Vec (int_arm_cde_vcx2q_predicated timm:$coproc,
+                                    (VTI.Vec MQPR:$inactive),
+                                    (v16i8 MQPR:$n), timm:$imm,
+                                    (VTI.Pred VCCR:$pred))),
+            (VTI.Vec (CDE_VCX2_vec p_imm:$coproc, (v16i8 MQPR:$n),
+                                    imm_7b:$imm, ARMVCCThen,
+                                    (VTI.Pred VCCR:$pred),
+                                    (VTI.Vec MQPR:$inactive)))>;
+  def : Pat<(VTI.Vec (int_arm_cde_vcx2qa_predicated timm:$coproc,
+                                    (VTI.Vec MQPR:$acc),
+                                    (v16i8 MQPR:$n), timm:$imm,
+                                    (VTI.Pred VCCR:$pred))),
+            (VTI.Vec (CDE_VCX2A_vec p_imm:$coproc, (VTI.Vec MQPR:$acc),
+                                    (v16i8 MQPR:$n), timm:$imm, ARMVCCThen,
+                                    (VTI.Pred VCCR:$pred)))>;
+
+  def : Pat<(VTI.Vec (int_arm_cde_vcx3q_predicated timm:$coproc,
+                                    (VTI.Vec MQPR:$inactive),
+                                    (v16i8 MQPR:$n), (v16i8 MQPR:$m),
+                                    timm:$imm,
+                                    (VTI.Pred VCCR:$pred))),
+            (VTI.Vec (CDE_VCX3_vec p_imm:$coproc, (v16i8 MQPR:$n),
+                                    (v16i8 MQPR:$m),
+                                    imm_4b:$imm, ARMVCCThen,
+                                    (VTI.Pred VCCR:$pred),
+                                    (VTI.Vec MQPR:$inactive)))>;
+  def : Pat<(VTI.Vec (int_arm_cde_vcx3qa_predicated timm:$coproc,
+                                    (VTI.Vec MQPR:$acc),
+                                    (v16i8 MQPR:$n), (v16i8 MQPR:$m), timm:$imm,
+                                    (VTI.Pred VCCR:$pred))),
+            (VTI.Vec (CDE_VCX3A_vec p_imm:$coproc, (VTI.Vec MQPR:$acc),
+                                    (v16i8 MQPR:$n), (v16i8 MQPR:$m),
+                                    imm_4b:$imm, ARMVCCThen,
+                                    (VTI.Pred VCCR:$pred)))>;
+}
+
+let Predicates = [HasCDE, HasMVEInt] in
+  foreach VTI = [ MVE_v16i8, MVE_v8i16, MVE_v4i32, MVE_v2i64 ] in
+    defm : VCXPredicatedPat_m<VTI>;
+
+let Predicates = [HasCDE, HasMVEFloat] in
+  foreach VTI = [ MVE_v8f16, MVE_v4f32 ] in
+    defm : VCXPredicatedPat_m<VTI>;
diff --git a/llvm/lib/Target/ARM/ARMInstrFormats.td b/llvm/lib/Target/ARM/ARMInstrFormats.td
index 39646e3d3557b..2cd4e052a4407 100644
--- a/llvm/lib/Target/ARM/ARMInstrFormats.td
+++ b/llvm/lib/Target/ARM/ARMInstrFormats.td
@@ -410,6 +410,7 @@ class InstTemplate<AddrMode am, int sz, IndexMode im,
   bit validForTailPredication = 0;
   bit retainsPreviousHalfElement = 0;
   bit horizontalReduction = 0;
+  bit doubleWidthResult = 0;
 
   // If this is a pseudo instruction, mark it isCodeGenOnly.
   let isCodeGenOnly = !eq(!cast<string>(f), "Pseudo");
@@ -425,6 +426,7 @@ class InstTemplate<AddrMode am, int sz, IndexMode im,
   let TSFlags{20}    = validForTailPredication;
   let TSFlags{21}    = retainsPreviousHalfElement;
   let TSFlags{22}    = horizontalReduction;
+  let TSFlags{23}    = doubleWidthResult;
 
   let Constraints = cstr;
   let Itinerary = itin;
diff --git a/llvm/lib/Target/ARM/ARMInstrInfo.cpp b/llvm/lib/Target/ARM/ARMInstrInfo.cpp
index a802d5a06f074..2790ac215f862 100644
--- a/llvm/lib/Target/ARM/ARMInstrInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMInstrInfo.cpp
@@ -126,7 +126,7 @@ void ARMInstrInfo::expandLoadStackGuard(MachineBasicBlock::iterator MI) const {
                MachineMemOperand::MODereferenceable |
                MachineMemOperand::MOInvariant;
   MachineMemOperand *MMO = MBB.getParent()->getMachineMemOperand(
-      MachinePointerInfo::getGOT(*MBB.getParent()), Flags, 4, 4);
+      MachinePointerInfo::getGOT(*MBB.getParent()), Flags, 4, Align(4));
   MIB.addMemOperand(MMO);
   BuildMI(MBB, MI, DL, get(ARM::LDRi12), Reg)
       .addReg(Reg, RegState::Kill)
diff --git a/llvm/lib/Target/ARM/ARMInstrInfo.td b/llvm/lib/Target/ARM/ARMInstrInfo.td
index de87b323f79cc..228ba4f93c371 100644
--- a/llvm/lib/Target/ARM/ARMInstrInfo.td
+++ b/llvm/lib/Target/ARM/ARMInstrInfo.td
@@ -289,6 +289,11 @@ def ARMvshruImm  : SDNode<"ARMISD::VSHRuIMM", SDTARMVSHIMM>;
 def ARMvshls     : SDNode<"ARMISD::VSHLs", SDTARMVSH>;
 def ARMvshlu     : SDNode<"ARMISD::VSHLu", SDTARMVSH>;
 
+def SDTARMVMULL   : SDTypeProfile<1, 2, [SDTCisInt<0>, SDTCisInt<1>,
+                                         SDTCisSameAs<1, 2>]>;
+def ARMvmulls    : SDNode<"ARMISD::VMULLs", SDTARMVMULL>;
+def ARMvmullu    : SDNode<"ARMISD::VMULLu", SDTARMVMULL>;
+
 def SDTARMVCMP    : SDTypeProfile<1, 3, [SDTCisInt<0>, SDTCisSameAs<1, 2>,
                                          SDTCisInt<3>]>;
 def SDTARMVCMPZ   : SDTypeProfile<1, 2, [SDTCisInt<2>]>;
@@ -3829,9 +3834,8 @@ def QSUB16  : AAIIntrinsic<0b01100010, 0b11110111, "qsub16", int_arm_qsub16>;
 def QSUB8   : AAIIntrinsic<0b01100010, 0b11111111, "qsub8", int_arm_qsub8>;
 
 def QDADD   : AAIRevOpr<0b00010100, 0b00000101, "qdadd",
-              [(set GPRnopc:$Rd, (int_arm_qadd (int_arm_qadd GPRnopc:$Rm,
-                                                             GPRnopc:$Rm),
-                                  GPRnopc:$Rn))]>;
+              [(set GPRnopc:$Rd, (int_arm_qadd GPRnopc:$Rm,
+                                  (int_arm_qadd GPRnopc:$Rn, GPRnopc:$Rn)))]>;
 def QDSUB   : AAIRevOpr<0b00010110, 0b00000101, "qdsub",
               [(set GPRnopc:$Rd, (int_arm_qsub GPRnopc:$Rm,
                                   (int_arm_qadd GPRnopc:$Rn, GPRnopc:$Rn)))]>;
@@ -3846,7 +3850,7 @@ def : ARMV5TEPat<(saddsat GPR:$a, GPR:$b),
                  (QADD GPR:$a, GPR:$b)>;
 def : ARMV5TEPat<(ssubsat GPR:$a, GPR:$b),
                  (QSUB GPR:$a, GPR:$b)>;
-def : ARMV5TEPat<(saddsat(saddsat rGPR:$Rm, rGPR:$Rm), rGPR:$Rn),
+def : ARMV5TEPat<(saddsat rGPR:$Rm, (saddsat rGPR:$Rn, rGPR:$Rn)),
                  (QDADD rGPR:$Rm, rGPR:$Rn)>;
 def : ARMV5TEPat<(ssubsat rGPR:$Rm, (saddsat rGPR:$Rn, rGPR:$Rn)),
                  (QDSUB rGPR:$Rm, rGPR:$Rn)>;
diff --git a/llvm/lib/Target/ARM/ARMInstrMVE.td b/llvm/lib/Target/ARM/ARMInstrMVE.td
index fcd3b5775869e..74993ac4b75f7 100644
--- a/llvm/lib/Target/ARM/ARMInstrMVE.td
+++ b/llvm/lib/Target/ARM/ARMInstrMVE.td
@@ -439,6 +439,8 @@ class MVE_ScalarShiftDoubleReg<string iname, dag iops, string asm,
 
   let Inst{19-17} = RdaLo{3-1};
   let Inst{11-9} = RdaHi{3-1};
+
+  let hasSideEffects = 0;
 }
 
 class MVE_ScalarShiftDRegImm<string iname, bits<2> op5_4, bit op16,
@@ -607,6 +609,7 @@ class MVE_VADDV<string iname, string suffix, dag iops, string cstr,
   let Inst{3-1} = Qm{2-0};
   let Inst{0} = 0b0;
   let horizontalReduction = 1;
+  let validForTailPredication = 1;
 }
 
 def ARMVADDVs       : SDNode<"ARMISD::VADDVs", SDTVecReduce>;
@@ -755,6 +758,7 @@ class MVE_VMINMAXNMV<string iname, string suffix, bit sz,
   let horizontalReduction = 1;
 
   let Predicates = [HasMVEFloat];
+  let hasSideEffects = 0;
 }
 
 multiclass MVE_VMINMAXNMV_p<string iname, bit notAbs, bit isMin,
@@ -1064,6 +1068,8 @@ class MVE_VMLALDAVBase<string iname, string suffix, dag iops, string cstr,
   let Inst{3-1} = Qm{2-0};
   let Inst{0} = bit_0;
   let horizontalReduction = 1;
+
+  let hasSideEffects = 0;
 }
 
 multiclass MVE_VMLALDAVBase_A<string iname, string x, string suffix,
@@ -1609,6 +1615,8 @@ class MVE_VMOV_lane<string suffix, bit U, dag indexop,
   let Inst{11-8} = 0b1011;
   let Inst{7} = Qd{3};
   let Inst{4-0} = 0b10000;
+
+  let hasSideEffects = 0;
 }
 
 class MVE_VMOV_lane_32<MVE_VMOV_lane_direction dir>
@@ -2389,6 +2397,8 @@ let Predicates = [HasMVEInt] in {
             (v8i16 (MVE_VMOVimmi16 nImmSplatI16:$simm))>;
   def : Pat<(v4i32 (ARMvmovImm timm:$simm)),
             (v4i32 (MVE_VMOVimmi32 nImmVMOVI32:$simm))>;
+  def : Pat<(v2i64 (ARMvmovImm timm:$simm)),
+            (v2i64 (MVE_VMOVimmi64 nImmSplatI64:$simm))>;
 
   def : Pat<(v8i16 (ARMvmvnImm timm:$simm)),
             (v8i16 (MVE_VMVNimmi16 nImmSplatI16:$simm))>;
@@ -2513,6 +2523,7 @@ class MVE_VMOVL<string iname, string suffix, bits<2> sz, bit U, bit top,
   let Inst{11-6} = 0b111101;
   let Inst{4} = 0b0;
   let Inst{0} = 0b0;
+  let doubleWidthResult = 1;
 }
 
 multiclass MVE_VMOVL_m<bit top, string chr, MVEVectorVTInfo OutVTI,
@@ -2547,22 +2558,23 @@ let Predicates = [HasMVEInt] in {
   def : Pat<(sext_inreg (v4i32 MQPR:$src), v4i8),
             (MVE_VMOVLs16bh (MVE_VMOVLs8bh MQPR:$src))>;
 
-  def : Pat<(sext_inreg (v8i16 (ARMVectorRegCast (ARMvrev16 (v16i8 MQPR:$src)))),
-                        v8i8), (MVE_VMOVLs8th MQPR:$src)>;
-  def : Pat<(sext_inreg (v4i32 (ARMVectorRegCast (ARMvrev32 (v8i16 MQPR:$src)))),
-                        v4i16), (MVE_VMOVLs16th MQPR:$src)>;
-  def : Pat<(ARMvbicImm (v8i16 (ARMVectorRegCast (ARMvrev16 (v16i8 MQPR:$src)))),
-                        (i32 0xAFF)), (MVE_VMOVLu8th MQPR:$src)>;
-  def : Pat<(and (v4i32 (ARMVectorRegCast (ARMvrev32 (v8i16 MQPR:$src)))),
-                 (v4i32 (ARMvmovImm (i32 0xCFF)))),
-                 (MVE_VMOVLu16th MQPR:$src)>;
+  def : Pat<(sext_inreg (v8i16 (ARMVectorRegCast (ARMvrev16 (v16i8 MQPR:$src)))), v8i8),
+            (MVE_VMOVLs8th MQPR:$src)>;
+  def : Pat<(sext_inreg (v4i32 (ARMVectorRegCast (ARMvrev32 (v8i16 MQPR:$src)))), v4i16),
+            (MVE_VMOVLs16th MQPR:$src)>;
 
-  // zext_inreg 16 -> 32
-  def : Pat<(and (v4i32 MQPR:$src), (v4i32 (ARMvmovImm (i32 0xCFF)))),
-            (MVE_VMOVLu16bh MQPR:$src)>;
   // zext_inreg 8 -> 16
   def : Pat<(ARMvbicImm (v8i16 MQPR:$src), (i32 0xAFF)),
             (MVE_VMOVLu8bh MQPR:$src)>;
+  // zext_inreg 16 -> 32
+  def : Pat<(and (v4i32 MQPR:$src), (v4i32 (ARMvmovImm (i32 0xCFF)))),
+            (MVE_VMOVLu16bh MQPR:$src)>;
+  // Same zext_inreg with vrevs, picking the top half
+  def : Pat<(ARMvbicImm (v8i16 (ARMVectorRegCast (ARMvrev16 (v16i8 MQPR:$src)))), (i32 0xAFF)),
+            (MVE_VMOVLu8th MQPR:$src)>;
+  def : Pat<(and (v4i32 (ARMVectorRegCast (ARMvrev32 (v8i16 MQPR:$src)))),
+                 (v4i32 (ARMvmovImm (i32 0xCFF)))),
+            (MVE_VMOVLu16th MQPR:$src)>;
 }
 
 
@@ -2580,6 +2592,8 @@ class MVE_VSHLL_imm<string iname, string suffix, bit U, bit th,
 
   // For the MVE_VSHLL_patterns multiclass to refer to
   Operand immediateType = immtype;
+
+  let doubleWidthResult = 1;
 }
 
 // The immediate VSHLL instructions accept shift counts from 1 up to
@@ -2623,6 +2637,7 @@ class MVE_VSHLL_by_lane_width<string iname, string suffix, bits<2> size,
   let Inst{11-6} = 0b111000;
   let Inst{4} = 0b0;
   let Inst{0} = 0b1;
+  let doubleWidthResult = 1;
 }
 
 multiclass MVE_VSHLL_lw<string iname, string suffix, bits<2> sz, bit U,
@@ -4101,7 +4116,7 @@ multiclass unpred_vcmpf_z<PatLeaf fc> {
                 (v4i1 (MVE_VCMPf32r (v4f32 MQPR:$v1), ZR, fc))>;
 
   def : Pat<(v8i1 (and (v8i1 VCCR:$p1), (v8i1 (ARMvcmpz (v8f16 MQPR:$v1), fc)))),
-            (v8i1 (MVE_VCMPf32r (v8f16 MQPR:$v1), ZR, fc, ARMVCCThen, VCCR:$p1))>;
+            (v8i1 (MVE_VCMPf16r (v8f16 MQPR:$v1), ZR, fc, ARMVCCThen, VCCR:$p1))>;
   def : Pat<(v4i1 (and (v4i1 VCCR:$p1), (v4i1 (ARMvcmpz (v4f32 MQPR:$v1), fc)))),
             (v4i1 (MVE_VCMPf32r (v4f32 MQPR:$v1), ZR, fc, ARMVCCThen, VCCR:$p1))>;
 }
@@ -4357,6 +4372,7 @@ class MVE_VMULL<string iname, string suffix, bit bit_28, bits<2> bits_21_20,
   let Inst{7} = Qn{3};
   let Inst{0} = 0b0;
   let validForTailPredication = 1;
+  let doubleWidthResult = 1;
 }
 
 multiclass MVE_VMULL_m<MVEVectorVTInfo VTI,
@@ -4428,6 +4444,50 @@ defm MVE_VMULLBp16 : MVE_VMULL_m<MVE_v8p16, int_arm_mve_vmull_poly,
 defm MVE_VMULLTp16 : MVE_VMULL_m<MVE_v8p16, int_arm_mve_vmull_poly,
                                  int_arm_mve_mull_poly_predicated, 0b1>;
 
+let Predicates = [HasMVEInt] in {
+  def : Pat<(v2i64 (ARMvmulls (v4i32 MQPR:$src1), (v4i32 MQPR:$src2))),
+            (MVE_VMULLBs32 MQPR:$src1, MQPR:$src2)>;
+  def : Pat<(v2i64 (ARMvmulls (v4i32 (ARMvrev64 (v4i32 MQPR:$src1))),
+                              (v4i32 (ARMvrev64 (v4i32 MQPR:$src2))))),
+            (MVE_VMULLTs32 MQPR:$src1, MQPR:$src2)>;
+
+  def : Pat<(mul (sext_inreg (v4i32 MQPR:$src1), v4i16),
+                 (sext_inreg (v4i32 MQPR:$src2), v4i16)),
+            (MVE_VMULLBs16 MQPR:$src1, MQPR:$src2)>;
+  def : Pat<(mul (sext_inreg (v4i32 (ARMVectorRegCast (ARMvrev32 (v8i16 MQPR:$src1)))), v4i16),
+                 (sext_inreg (v4i32 (ARMVectorRegCast (ARMvrev32 (v8i16 MQPR:$src2)))), v4i16)),
+            (MVE_VMULLTs16 MQPR:$src1, MQPR:$src2)>;
+
+  def : Pat<(mul (sext_inreg (v8i16 MQPR:$src1), v8i8),
+                 (sext_inreg (v8i16 MQPR:$src2), v8i8)),
+            (MVE_VMULLBs8 MQPR:$src1, MQPR:$src2)>;
+  def : Pat<(mul (sext_inreg (v8i16 (ARMVectorRegCast (ARMvrev16 (v16i8 MQPR:$src1)))), v8i8),
+                 (sext_inreg (v8i16 (ARMVectorRegCast (ARMvrev16 (v16i8 MQPR:$src2)))), v8i8)),
+            (MVE_VMULLTs8 MQPR:$src1, MQPR:$src2)>;
+
+  def : Pat<(v2i64 (ARMvmullu (v4i32 MQPR:$src1), (v4i32 MQPR:$src2))),
+            (MVE_VMULLBu32 MQPR:$src1, MQPR:$src2)>;
+  def : Pat<(v2i64 (ARMvmullu (v4i32 (ARMvrev64 (v4i32 MQPR:$src1))),
+                              (v4i32 (ARMvrev64 (v4i32 MQPR:$src2))))),
+            (MVE_VMULLTu32 MQPR:$src1, MQPR:$src2)>;
+
+  def : Pat<(mul (and (v4i32 MQPR:$src1), (v4i32 (ARMvmovImm (i32 0xCFF)))),
+                 (and (v4i32 MQPR:$src2), (v4i32 (ARMvmovImm (i32 0xCFF))))),
+            (MVE_VMULLBu16 MQPR:$src1, MQPR:$src2)>;
+  def : Pat<(mul (and (v4i32 (ARMVectorRegCast (ARMvrev32 (v8i16 MQPR:$src1)))),
+                      (v4i32 (ARMvmovImm (i32 0xCFF)))),
+                 (and (v4i32 (ARMVectorRegCast (ARMvrev32 (v8i16 MQPR:$src2)))),
+                      (v4i32 (ARMvmovImm (i32 0xCFF))))),
+            (MVE_VMULLTu16 MQPR:$src1, MQPR:$src2)>;
+
+  def : Pat<(mul (ARMvbicImm (v8i16 MQPR:$src1), (i32 0xAFF)),
+                 (ARMvbicImm (v8i16 MQPR:$src2), (i32 0xAFF))),
+            (MVE_VMULLBu8 MQPR:$src1, MQPR:$src2)>;
+  def : Pat<(mul (ARMvbicImm (v8i16 (ARMVectorRegCast (ARMvrev16 (v16i8 MQPR:$src1)))), (i32 0xAFF)),
+                 (ARMvbicImm (v8i16 (ARMVectorRegCast (ARMvrev16 (v16i8 MQPR:$src2)))), (i32 0xAFF))),
+            (MVE_VMULLTu8 MQPR:$src1, MQPR:$src2)>;
+}
+
 class MVE_VxMULH<string iname, string suffix, bit U, bits<2> size, bit round,
                  list<dag> pattern=[]>
   : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),
@@ -4731,6 +4791,7 @@ class MVE_VQDMULL<string iname, string suffix, bit size, bit T,
   let Inst{7} = Qn{3};
   let Inst{0} = 0b1;
   let validForTailPredication = 1;
+  let doubleWidthResult = 1;
 }
 
 multiclass MVE_VQDMULL_m<string iname, MVEVectorVTInfo VTI, bit size, bit T,
@@ -4937,6 +4998,7 @@ class MVE_VQDMULL_qr<string iname, string suffix, bit size,
   let Inst{8} = 0b1;
   let Inst{5} = 0b1;
   let validForTailPredication = 1;
+  let doubleWidthResult = 1;
 }
 
 multiclass MVE_VQDMULL_qr_m<string iname, MVEVectorVTInfo VTI, bit size,
@@ -5217,6 +5279,7 @@ class MVE_VFMAMLA_qr<string iname, string suffix,
   let Inst{8} = 0b0;
   let Inst{5} = 0b0;
   let validForTailPredication = 1;
+  let hasSideEffects = 0;
 }
 
 multiclass MVE_VMLA_qr_multi<string iname, MVEVectorVTInfo VTI,
@@ -5365,6 +5428,7 @@ class MVE_VxDUP<string iname, string suffix, bits<2> size, bit bit_12,
   let Inst{6-1} = 0b110111;
   let Inst{0} = imm{0};
   let validForTailPredication = 1;
+  let hasSideEffects = 0;
 }
 
 def MVE_VIDUPu8  : MVE_VxDUP<"vidup", "u8",  0b00, 0b0>;
@@ -5400,6 +5464,7 @@ class MVE_VxWDUP<string iname, string suffix, bits<2> size, bit bit_12,
   let Inst{3-1} = Rm{3-1};
   let Inst{0} = imm{0};
   let validForTailPredication = 1;
+  let hasSideEffects = 0;
 }
 
 def MVE_VIWDUPu8  : MVE_VxWDUP<"viwdup", "u8",  0b00, 0b0>;
@@ -5468,6 +5533,8 @@ class MVE_VMOV_64bit<dag oops, dag iops, bit to_qreg, string ops, string cstr>
   let Inst{12-5} = 0b01111000;
   let Inst{4} = idx2;
   let Inst{3-0} = Rt{3-0};
+
+  let hasSideEffects = 0;
 }
 
 // The assembly syntax for these instructions mentions the vector
diff --git a/llvm/lib/Target/ARM/ARMInstrNEON.td b/llvm/lib/Target/ARM/ARMInstrNEON.td
index 19a901bfc811e..d41a4907987f0 100644
--- a/llvm/lib/Target/ARM/ARMInstrNEON.td
+++ b/llvm/lib/Target/ARM/ARMInstrNEON.td
@@ -526,11 +526,6 @@ def NEONzip       : SDNode<"ARMISD::VZIP", SDTARMVSHUF2>;
 def NEONuzp       : SDNode<"ARMISD::VUZP", SDTARMVSHUF2>;
 def NEONtrn       : SDNode<"ARMISD::VTRN", SDTARMVSHUF2>;
 
-def SDTARMVMULL   : SDTypeProfile<1, 2, [SDTCisInt<0>, SDTCisInt<1>,
-                                         SDTCisSameAs<1, 2>]>;
-def NEONvmulls    : SDNode<"ARMISD::VMULLs", SDTARMVMULL>;
-def NEONvmullu    : SDNode<"ARMISD::VMULLu", SDTARMVMULL>;
-
 def SDTARMVTBL1   : SDTypeProfile<1, 2, [SDTCisVT<0, v8i8>, SDTCisVT<1, v8i8>,
                                          SDTCisVT<2, v8i8>]>;
 def SDTARMVTBL2   : SDTypeProfile<1, 3, [SDTCisVT<0, v8i8>, SDTCisVT<1, v8i8>,
@@ -4428,17 +4423,17 @@ def : Pat<(v4i32 (int_arm_neon_vqrdmulh (v4i32 QPR:$src1),
 let PostEncoderMethod = "NEONThumb2DataIPostEncoder",
     DecoderNamespace = "NEONData" in {
   defm VMULLs   : N3VL_QHS<0,1,0b1100,0, IIC_VMULi16D, IIC_VMULi32D,
-                           "vmull", "s", NEONvmulls, 1>;
+                           "vmull", "s", ARMvmulls, 1>;
   defm VMULLu   : N3VL_QHS<1,1,0b1100,0, IIC_VMULi16D, IIC_VMULi32D,
-                           "vmull", "u", NEONvmullu, 1>;
+                           "vmull", "u", ARMvmullu, 1>;
   def  VMULLp8   :  N3VLInt<0, 1, 0b00, 0b1110, 0, IIC_VMULi16D, "vmull", "p8",
                             v8i16, v8i8, int_arm_neon_vmullp, 1>;
   def  VMULLp64  : N3VLIntnp<0b00101, 0b10, 0b1110, 0, 0, NoItinerary,
                           "vmull", "p64", v2i64, v1i64, int_arm_neon_vmullp, 1>,
                     Requires<[HasV8, HasCrypto]>;
 }
-defm VMULLsls : N3VLSL_HS<0, 0b1010, IIC_VMULi16D, "vmull", "s", NEONvmulls>;
-defm VMULLslu : N3VLSL_HS<1, 0b1010, IIC_VMULi16D, "vmull", "u", NEONvmullu>;
+defm VMULLsls : N3VLSL_HS<0, 0b1010, IIC_VMULi16D, "vmull", "s", ARMvmulls>;
+defm VMULLslu : N3VLSL_HS<1, 0b1010, IIC_VMULi16D, "vmull", "u", ARMvmullu>;
 
 //   VQDMULL  : Vector Saturating Doubling Multiply Long (Q = D * D)
 defm VQDMULL  : N3VLInt_HS<0,1,0b1101,0, IIC_VMULi16D, IIC_VMULi32D,
@@ -4508,12 +4503,12 @@ def : Pat<(v4f32 (fadd_mlx (v4f32 QPR:$src1),
 
 //   VMLAL    : Vector Multiply Accumulate Long (Q += D * D)
 defm VMLALs   : N3VLMulOp_QHS<0,1,0b1000,0, IIC_VMACi16D, IIC_VMACi32D,
-                              "vmlal", "s", NEONvmulls, add>;
+                              "vmlal", "s", ARMvmulls, add>;
 defm VMLALu   : N3VLMulOp_QHS<1,1,0b1000,0, IIC_VMACi16D, IIC_VMACi32D,
-                              "vmlal", "u", NEONvmullu, add>;
+                              "vmlal", "u", ARMvmullu, add>;
 
-defm VMLALsls : N3VLMulOpSL_HS<0, 0b0010, "vmlal", "s", NEONvmulls, add>;
-defm VMLALslu : N3VLMulOpSL_HS<1, 0b0010, "vmlal", "u", NEONvmullu, add>;
+defm VMLALsls : N3VLMulOpSL_HS<0, 0b0010, "vmlal", "s", ARMvmulls, add>;
+defm VMLALslu : N3VLMulOpSL_HS<1, 0b0010, "vmlal", "u", ARMvmullu, add>;
 
 let Predicates = [HasNEON, HasV8_1a] in {
   // v8.1a Neon Rounding Double Multiply-Op vector operations,
@@ -4741,12 +4736,12 @@ def : Pat<(v4f32 (fsub_mlx (v4f32 QPR:$src1),
 
 //   VMLSL    : Vector Multiply Subtract Long (Q -= D * D)
 defm VMLSLs   : N3VLMulOp_QHS<0,1,0b1010,0, IIC_VMACi16D, IIC_VMACi32D,
-                              "vmlsl", "s", NEONvmulls, sub>;
+                              "vmlsl", "s", ARMvmulls, sub>;
 defm VMLSLu   : N3VLMulOp_QHS<1,1,0b1010,0, IIC_VMACi16D, IIC_VMACi32D,
-                              "vmlsl", "u", NEONvmullu, sub>;
+                              "vmlsl", "u", ARMvmullu, sub>;
 
-defm VMLSLsls : N3VLMulOpSL_HS<0, 0b0110, "vmlsl", "s", NEONvmulls, sub>;
-defm VMLSLslu : N3VLMulOpSL_HS<1, 0b0110, "vmlsl", "u", NEONvmullu, sub>;
+defm VMLSLsls : N3VLMulOpSL_HS<0, 0b0110, "vmlsl", "s", ARMvmulls, sub>;
+defm VMLSLslu : N3VLMulOpSL_HS<1, 0b0110, "vmlsl", "u", ARMvmullu, sub>;
 
 //   VQDMLSL  : Vector Saturating Doubling Multiply Subtract Long (Q -= D * D)
 defm VQDMLSL  : N3VLInt3_HS<0, 1, 0b1011, 0, IIC_VMACi16D, IIC_VMACi32D,
@@ -8926,3 +8921,93 @@ def : NEONInstAlias<"vmov${p}.f32 $Vd, $imm",
                      (VMOVv4i32 QPR:$Vd, nImmVMOVI32:$imm, pred:$p)>;
 def : NEONInstAlias<"vmov${p}.f32 $Vd, $imm",
                      (VMOVv2i32 DPR:$Vd, nImmVMOVI32:$imm, pred:$p)>;
+
+// ARMv8.6a BFloat16 instructions.
+let Predicates = [HasBF16, HasNEON] in {
+class BF16VDOT<bits<5> op27_23, bits<2> op21_20, bit op6,
+               dag oops, dag iops>
+   : N3Vnp<op27_23, op21_20, 0b1101, op6, 0, oops, iops,
+           N3RegFrm, IIC_VDOTPROD, "", "", []> {
+    let hasNoSchedulingInfo = 1;
+    let DecoderNamespace = "VFPV8";
+}
+
+class BF16VDOTS<bit Q, RegisterClass RegTy, string opc, ValueType AccumTy, ValueType InputTy>
+   : BF16VDOT<0b11000, 0b00,  Q, (outs RegTy:$dst),
+              (ins RegTy:$Vd, RegTy:$Vn, RegTy:$Vm)> {
+  let Constraints = "$dst = $Vd";
+  let AsmString = !strconcat(opc, ".bf16", "\t$Vd, $Vn, $Vm");
+    let DecoderNamespace = "VFPV8";
+}
+
+multiclass BF16VDOTI<bit Q, RegisterClass RegTy, string opc, ValueType AccumTy,
+                     ValueType InputTy, dag RHS> {
+
+  def "" : BF16VDOT<0b11100, 0b00, Q, (outs RegTy:$dst),
+                    (ins RegTy:$Vd, RegTy:$Vn,
+                    DPR_VFP2:$Vm, VectorIndex32:$lane)> {
+    bit lane;
+    let Inst{5} = lane;
+    let Constraints = "$dst = $Vd";
+    let AsmString = !strconcat(opc, ".bf16", "\t$Vd, $Vn, $Vm$lane");
+    let DecoderNamespace = "VFPV8";
+  }
+
+}
+
+def BF16VDOTS_VDOTD : BF16VDOTS<0, DPR, "vdot", v2f32, v8i8>;
+def BF16VDOTS_VDOTQ : BF16VDOTS<1, QPR, "vdot", v4f32, v16i8>;
+
+defm BF16VDOTI_VDOTD : BF16VDOTI<0, DPR, "vdot", v2f32, v8i8, (v2f32 DPR_VFP2:$Vm)>;
+defm BF16VDOTI_VDOTQ : BF16VDOTI<1, QPR, "vdot", v4f32, v16i8, (EXTRACT_SUBREG QPR:$Vm, dsub_0)>;
+
+class BF16MM<bit Q, RegisterClass RegTy,
+             string opc>
+   : N3Vnp<0b11000, 0b00, 0b1100, Q, 0,
+           (outs RegTy:$dst), (ins RegTy:$Vd, RegTy:$Vn, RegTy:$Vm),
+           N3RegFrm, IIC_VDOTPROD, "", "", []> {
+   let Constraints = "$dst = $Vd";
+   let AsmString = !strconcat(opc, ".bf16", "\t$Vd, $Vn, $Vm");
+   let DecoderNamespace = "VFPV8";
+   let hasNoSchedulingInfo = 1;
+}
+
+def VMMLA : BF16MM<1, QPR, "vmmla">;
+
+class VBF16MALQ<bit T, string suffix>
+  : N3VCP8<0b00, 0b11, T, 1,
+           (outs QPR:$dst), (ins QPR:$Vd, QPR:$Vn, QPR:$Vm),
+           NoItinerary, "vfma" # suffix, "bf16", "$Vd, $Vn, $Vm", "",
+              []> { // TODO: Add intrinsics
+  let Constraints = "$dst = $Vd";
+  let DecoderNamespace = "VFPV8";
+  let hasNoSchedulingInfo = 1;
+}
+
+def VBF16MALTQ: VBF16MALQ<1, "t">;
+def VBF16MALBQ: VBF16MALQ<0, "b">;
+
+multiclass VBF16MALQI<bit T, string suffix> {
+  def "" : N3VLaneCP8<0, 0b11, T, 1, (outs QPR:$dst),
+              (ins QPR:$Vd, QPR:$Vn, DPR_8:$Vm, VectorIndex16:$idx),
+               IIC_VMACD, "vfma" # suffix, "bf16", "$Vd, $Vn, $Vm$idx", "", []> {
+  bits<2> idx;
+  let Inst{5} = idx{1};
+  let Inst{3} = idx{0};
+  let Constraints = "$dst = $Vd";
+  let DecoderNamespace = "VFPV8";
+  let hasNoSchedulingInfo = 1;
+  }
+
+}
+
+defm VBF16MALTQI: VBF16MALQI<1, "t">;
+defm VBF16MALBQI: VBF16MALQI<0, "b">;
+
+let hasNoSchedulingInfo = 1 in {
+def BF16_VCVT :  N2V<0b11, 0b11, 0b01, 0b10, 0b01100, 1, 0,
+                    (outs DPR:$Vd), (ins QPR:$Vm),
+                    NoItinerary, "vcvt", "bf16.f32", "$Vd, $Vm", "", []>;
+}
+}
+// End of BFloat16 instructions
diff --git a/llvm/lib/Target/ARM/ARMInstrThumb2.td b/llvm/lib/Target/ARM/ARMInstrThumb2.td
index d69526b32c7ca..de168da6479bc 100644
--- a/llvm/lib/Target/ARM/ARMInstrThumb2.td
+++ b/llvm/lib/Target/ARM/ARMInstrThumb2.td
@@ -2485,7 +2485,7 @@ def : Thumb2DSPPat<(int_arm_qadd rGPR:$Rm, rGPR:$Rn),
                    (t2QADD rGPR:$Rm, rGPR:$Rn)>;
 def : Thumb2DSPPat<(int_arm_qsub rGPR:$Rm, rGPR:$Rn),
                    (t2QSUB rGPR:$Rm, rGPR:$Rn)>;
-def : Thumb2DSPPat<(int_arm_qadd(int_arm_qadd rGPR:$Rm, rGPR:$Rm), rGPR:$Rn),
+def : Thumb2DSPPat<(int_arm_qadd rGPR:$Rm, (int_arm_qadd rGPR:$Rn, rGPR:$Rn)),
                    (t2QDADD rGPR:$Rm, rGPR:$Rn)>;
 def : Thumb2DSPPat<(int_arm_qsub rGPR:$Rm, (int_arm_qadd rGPR:$Rn, rGPR:$Rn)),
                    (t2QDSUB rGPR:$Rm, rGPR:$Rn)>;
@@ -2494,7 +2494,7 @@ def : Thumb2DSPPat<(saddsat rGPR:$Rm, rGPR:$Rn),
                    (t2QADD rGPR:$Rm, rGPR:$Rn)>;
 def : Thumb2DSPPat<(ssubsat rGPR:$Rm, rGPR:$Rn),
                    (t2QSUB rGPR:$Rm, rGPR:$Rn)>;
-def : Thumb2DSPPat<(saddsat(saddsat rGPR:$Rm, rGPR:$Rm), rGPR:$Rn),
+def : Thumb2DSPPat<(saddsat rGPR:$Rm, (saddsat rGPR:$Rn, rGPR:$Rn)),
                    (t2QDADD rGPR:$Rm, rGPR:$Rn)>;
 def : Thumb2DSPPat<(ssubsat rGPR:$Rm, (saddsat rGPR:$Rn, rGPR:$Rn)),
                    (t2QDSUB rGPR:$Rm, rGPR:$Rn)>;
diff --git a/llvm/lib/Target/ARM/ARMInstrVFP.td b/llvm/lib/Target/ARM/ARMInstrVFP.td
index f1d1d8a891649..b7e1300174336 100644
--- a/llvm/lib/Target/ARM/ARMInstrVFP.td
+++ b/llvm/lib/Target/ARM/ARMInstrVFP.td
@@ -1867,6 +1867,35 @@ def VULTOD : AVConv1XInsD_Encode<0b11101, 0b11, 0b1011, 0b1011, 1,
 
 } // End of 'let Constraints = "$a = $dst" in'
 
+// BFloat16  - Single precision, unary, predicated
+class BF16_VCVT<string opc, bits<2> op7_6>
+   : VFPAI<(outs SPR:$Sd), (ins SPR:$dst, SPR:$Sm),
+           VFPUnaryFrm, NoItinerary,
+           opc, ".bf16.f32\t$Sd, $Sm", []>,
+      RegConstraint<"$dst = $Sd">,
+      Requires<[HasBF16]>,
+     Sched<[]> {
+  bits<5> Sd;
+  bits<5> Sm;
+
+  // Encode instruction operands.
+  let Inst{3-0}   = Sm{4-1};
+  let Inst{5}     = Sm{0};
+  let Inst{15-12} = Sd{4-1};
+  let Inst{22}    = Sd{0};
+
+  let Inst{27-23} = 0b11101; // opcode1
+  let Inst{21-20} = 0b11;    // opcode2
+  let Inst{19-16} = 0b0011;  // opcode3
+  let Inst{11-8}  = 0b1001;
+  let Inst{7-6}   = op7_6;
+  let Inst{4}     = 0;
+  let DecoderNamespace = "VFPV8";
+}
+
+def BF16_VCVTB : BF16_VCVT<"vcvtb", 0b01>;
+def BF16_VCVTT : BF16_VCVT<"vcvtt", 0b11>;
+
 //===----------------------------------------------------------------------===//
 // FP Multiply-Accumulate Operations.
 //
diff --git a/llvm/lib/Target/ARM/ARMInstructionSelector.cpp b/llvm/lib/Target/ARM/ARMInstructionSelector.cpp
index 39b16a4065c58..e8d49cbbea32b 100644
--- a/llvm/lib/Target/ARM/ARMInstructionSelector.cpp
+++ b/llvm/lib/Target/ARM/ARMInstructionSelector.cpp
@@ -627,7 +627,7 @@ bool ARMInstructionSelector::selectGlobal(MachineInstrBuilder &MIB,
   bool UseMovt = STI.useMovt();
 
   unsigned Size = TM.getPointerSize(0);
-  unsigned Alignment = 4;
+  const Align Alignment(4);
 
   auto addOpsForConstantPoolLoad = [&MF, Alignment,
                                     Size](MachineInstrBuilder &MIB,
@@ -639,10 +639,10 @@ bool ARMInstructionSelector::selectGlobal(MachineInstrBuilder &MIB,
     auto CPIndex =
         // For SB relative entries we need a target-specific constant pool.
         // Otherwise, just use a regular constant pool entry.
-        IsSBREL
-            ? ConstPool->getConstantPoolIndex(
-                  ARMConstantPoolConstant::Create(GV, ARMCP::SBREL), Alignment)
-            : ConstPool->getConstantPoolIndex(GV, Alignment);
+        IsSBREL ? ConstPool->getConstantPoolIndex(
+                      ARMConstantPoolConstant::Create(GV, ARMCP::SBREL),
+                      Alignment.value())
+                : ConstPool->getConstantPoolIndex(GV, Alignment.value());
     MIB.addConstantPoolIndex(CPIndex, /*Offset*/ 0, /*TargetFlags*/ 0)
         .addMemOperand(MF.getMachineMemOperand(
             MachinePointerInfo::getConstantPool(MF), MachineMemOperand::MOLoad,
@@ -990,14 +990,14 @@ bool ARMInstructionSelector::select(MachineInstr &I) {
   case G_FCONSTANT: {
     // Load from constant pool
     unsigned Size = MRI.getType(I.getOperand(0).getReg()).getSizeInBits() / 8;
-    unsigned Alignment = Size;
+    Align Alignment(Size);
 
     assert((Size == 4 || Size == 8) && "Unsupported FP constant type");
     auto LoadOpcode = Size == 4 ? ARM::VLDRS : ARM::VLDRD;
 
     auto ConstPool = MF.getConstantPool();
-    auto CPIndex =
-        ConstPool->getConstantPoolIndex(I.getOperand(1).getFPImm(), Alignment);
+    auto CPIndex = ConstPool->getConstantPoolIndex(I.getOperand(1).getFPImm(),
+                                                   Alignment.value());
     MIB->setDesc(TII.get(LoadOpcode));
     MIB->RemoveOperand(1);
     MIB.addConstantPoolIndex(CPIndex, /*Offset*/ 0, /*TargetFlags*/ 0)
diff --git a/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp b/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
index a0e7a6f2b6ff4..381425f30b361 100644
--- a/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
@@ -1608,7 +1608,7 @@ static bool isMemoryOp(const MachineInstr &MI) {
 
   // Unaligned ldr/str is emulated by some kernels, but unaligned ldm/stm is
   // not.
-  if (MMO.getAlignment() < 4)
+  if (MMO.getAlign() < Align(4))
     return false;
 
   // str <undef> could probably be eliminated entirely, but for now we just want
@@ -2183,12 +2183,12 @@ ARMPreAllocLoadStoreOpt::CanFormLdStDWord(MachineInstr *Op0, MachineInstr *Op1,
       (*Op0->memoperands_begin())->isAtomic())
     return false;
 
-  unsigned Align = (*Op0->memoperands_begin())->getAlignment();
+  Align Alignment = (*Op0->memoperands_begin())->getAlign();
   const Function &Func = MF->getFunction();
-  unsigned ReqAlign = STI->hasV6Ops()
-    ? TD->getABITypeAlignment(Type::getInt64Ty(Func.getContext()))
-    : 8;  // Pre-v6 need 8-byte align
-  if (Align < ReqAlign)
+  Align ReqAlign =
+      STI->hasV6Ops() ? TD->getABITypeAlign(Type::getInt64Ty(Func.getContext()))
+                      : Align(8); // Pre-v6 need 8-byte align
+  if (Alignment < ReqAlign)
     return false;
 
   // Then make sure the immediate offset fits.
diff --git a/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp b/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp
index fd7ce212d5912..4ccf62759f5ab 100644
--- a/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp
+++ b/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp
@@ -520,19 +520,50 @@ static bool isRegInClass(const MachineOperand &MO,
   return MO.isReg() && MO.getReg() && Class->contains(MO.getReg());
 }
 
+// MVE 'narrowing' operate on half a lane, reading from half and writing
+// to half, which are referred to has the top and bottom half. The other
+// half retains its previous value.
+static bool retainsPreviousHalfElement(const MachineInstr &MI) {
+  const MCInstrDesc &MCID = MI.getDesc();
+  uint64_t Flags = MCID.TSFlags;
+  return (Flags & ARMII::RetainsPreviousHalfElement) != 0;
+}
+
+// Some MVE instructions read from the top/bottom halves of their operand(s)
+// and generate a vector result with result elements that are double the
+// width of the input.
+static bool producesDoubleWidthResult(const MachineInstr &MI) {
+  const MCInstrDesc &MCID = MI.getDesc();
+  uint64_t Flags = MCID.TSFlags;
+  return (Flags & ARMII::DoubleWidthResult) != 0;
+}
+
+static bool isHorizontalReduction(const MachineInstr &MI) {
+  const MCInstrDesc &MCID = MI.getDesc();
+  uint64_t Flags = MCID.TSFlags;
+  return (Flags & ARMII::HorizontalReduction) != 0;
+}
+
 // Can this instruction generate a non-zero result when given only zeroed
 // operands? This allows us to know that, given operands with false bytes
 // zeroed by masked loads, that the result will also contain zeros in those
 // bytes.
 static bool canGenerateNonZeros(const MachineInstr &MI) {
+
+  // Check for instructions which can write into a larger element size,
+  // possibly writing into a previous zero'd lane.
+  if (producesDoubleWidthResult(MI))
+    return true;
+
   switch (MI.getOpcode()) {
   default:
     break;
-  // FIXME: FP minus 0?
-  //case ARM::MVE_VNEGf16:
-  //case ARM::MVE_VNEGf32:
+  // FIXME: VNEG FP and -0? I think we'll need to handle this once we allow
+  // fp16 -> fp32 vector conversions.
+  // Instructions that perform a NOT will generate 1s from 0s.
   case ARM::MVE_VMVN:
   case ARM::MVE_VORN:
+  // Count leading zeros will do just that!
   case ARM::MVE_VCLZs8:
   case ARM::MVE_VCLZs16:
   case ARM::MVE_VCLZs32:
@@ -541,31 +572,27 @@ static bool canGenerateNonZeros(const MachineInstr &MI) {
   return false;
 }
 
-// MVE 'narrowing' operate on half a lane, reading from half and writing
-// to half, which are referred to has the top and bottom half. The other
-// half retains its previous value.
-static bool retainsPreviousHalfElement(const MachineInstr &MI) {
-  const MCInstrDesc &MCID = MI.getDesc();
-  uint64_t Flags = MCID.TSFlags;
-  return (Flags & ARMII::RetainsPreviousHalfElement) != 0;
-}
 
 // Look at its register uses to see if it only can only receive zeros
 // into its false lanes which would then produce zeros. Also check that
-// the output register is also defined by an FalseLaneZeros instruction
+// the output register is also defined by an FalseLanesZero instruction
 // so that if tail-predication happens, the lanes that aren't updated will
 // still be zeros.
-static bool producesFalseLaneZeros(MachineInstr &MI,
+static bool producesFalseLanesZero(MachineInstr &MI,
                                    const TargetRegisterClass *QPRs,
                                    const ReachingDefAnalysis &RDA,
-                                   InstSet &FalseLaneZeros) {
+                                   InstSet &FalseLanesZero) {
   if (canGenerateNonZeros(MI))
     return false;
+
+  bool AllowScalars = isHorizontalReduction(MI);
   for (auto &MO : MI.operands()) {
     if (!MO.isReg() || !MO.getReg())
       continue;
+    if (!isRegInClass(MO, QPRs) && AllowScalars)
+      continue;
     if (auto *OpDef = RDA.getMIOperand(&MI, MO))
-      if (FalseLaneZeros.count(OpDef))
+      if (FalseLanesZero.count(OpDef))
        continue;
     return false;
   }
@@ -596,8 +623,8 @@ bool LowOverheadLoop::ValidateLiveOuts() const {
   // loads, stores and other predicated instructions into our Predicated
   // set and build from there.
   const TargetRegisterClass *QPRs = TRI.getRegClass(ARM::MQPRRegClassID);
-  SetVector<MachineInstr *> Unknown;
-  SmallPtrSet<MachineInstr *, 4> FalseLaneZeros;
+  SetVector<MachineInstr *> FalseLanesUnknown;
+  SmallPtrSet<MachineInstr *, 4> FalseLanesZero;
   SmallPtrSet<MachineInstr *, 4> Predicated;
   MachineBasicBlock *MBB = ML.getHeader();
 
@@ -607,9 +634,14 @@ bool LowOverheadLoop::ValidateLiveOuts() const {
     if ((Flags & ARMII::DomainMask) != ARMII::DomainMVE)
       continue;
 
+    if (isVCTP(&MI) || MI.getOpcode() == ARM::MVE_VPST)
+      continue;
+
+    // Predicated loads will write zeros to the falsely predicated bytes of the
+    // destination register.
     if (isVectorPredicated(&MI)) {
       if (MI.mayLoad())
-        FalseLaneZeros.insert(&MI);
+        FalseLanesZero.insert(&MI);
       Predicated.insert(&MI);
       continue;
     }
@@ -617,12 +649,16 @@ bool LowOverheadLoop::ValidateLiveOuts() const {
     if (MI.getNumDefs() == 0)
       continue;
 
-    if (producesFalseLaneZeros(MI, QPRs, RDA, FalseLaneZeros))
-      FalseLaneZeros.insert(&MI);
-    else if (retainsPreviousHalfElement(MI))
-      return false;
-    else
-      Unknown.insert(&MI);
+    if (!producesFalseLanesZero(MI, QPRs, RDA, FalseLanesZero)) {
+      // We require retaining and horizontal operations to operate upon zero'd
+      // false lanes to ensure the conversion doesn't change the output.
+      if (retainsPreviousHalfElement(MI) || isHorizontalReduction(MI))
+        return false;
+      // Otherwise we need to evaluate this instruction later to see whether
+      // unknown false lanes will get masked away by their user(s).
+      FalseLanesUnknown.insert(&MI);
+    } else if (!isHorizontalReduction(MI))
+      FalseLanesZero.insert(&MI);
   }
 
   auto HasPredicatedUsers = [this](MachineInstr *MI, const MachineOperand &MO,
@@ -638,8 +674,9 @@ bool LowOverheadLoop::ValidateLiveOuts() const {
 
   // Visit the unknowns in reverse so that we can start at the values being
   // stored and then we can work towards the leaves, hopefully adding more
-  // instructions to Predicated.
-  for (auto *MI : reverse(Unknown)) {
+  // instructions to Predicated. Successfully terminating the loop means that
+  // all the unknown values have to found to be masked by predicated user(s).
+  for (auto *MI : reverse(FalseLanesUnknown)) {
     for (auto &MO : MI->operands()) {
       if (!isRegInClass(MO, QPRs) || !MO.isDef())
         continue;
@@ -1161,7 +1198,7 @@ void ARMLowOverheadLoops::ConvertVPTBlocks(LowOverheadLoop &LoLoop) {
       if (isVCTP(Divergent->MI)) {
         // The vctp will be removed, so the size of the vpt block needs to be
         // modified.
-        uint64_t Size = getARMVPTBlockMask(Block.size() - 1);
+        uint64_t Size = (uint64_t)getARMVPTBlockMask(Block.size() - 1);
         Block.getVPST()->getOperand(0).setImm(Size);
         LLVM_DEBUG(dbgs() << "ARM Loops: Modified VPT block mask.\n");
       } else if (Block.IsOnlyPredicatedOn(LoLoop.VCTP)) {
@@ -1190,7 +1227,7 @@ void ARMLowOverheadLoops::ConvertVPTBlocks(LowOverheadLoop &LoLoop) {
         MachineInstrBuilder MIB = BuildMI(*InsertAt->getParent(), InsertAt,
                                           InsertAt->getDebugLoc(),
                                           TII->get(ARM::MVE_VPST));
-        MIB.addImm(getARMVPTBlockMask(Size));
+        MIB.addImm((uint64_t)getARMVPTBlockMask(Size));
         LLVM_DEBUG(dbgs() << "ARM Loops: Removing VPST: " << *Block.getVPST());
         LLVM_DEBUG(dbgs() << "ARM Loops: Created VPST: " << *MIB);
         LoLoop.ToRemove.insert(Block.getVPST());
diff --git a/llvm/lib/Target/ARM/ARMPredicates.td b/llvm/lib/Target/ARM/ARMPredicates.td
index 6a70f68aaf496..f2cb2cc7e3099 100644
--- a/llvm/lib/Target/ARM/ARMPredicates.td
+++ b/llvm/lib/Target/ARM/ARMPredicates.td
@@ -72,6 +72,8 @@ def HasV8_4a         : Predicate<"Subtarget->hasV8_4aOps()">,
                                  AssemblerPredicate<(all_of HasV8_4aOps), "armv8.4a">;
 def HasV8_5a         : Predicate<"Subtarget->hasV8_5aOps()">,
                                  AssemblerPredicate<(all_of HasV8_5aOps), "armv8.5a">;
+def HasV8_6a         : Predicate<"Subtarget->hasV8_6aOps()">,
+                                 AssemblerPredicate<(all_of HasV8_6aOps), "armv8.6a">;
 def NoVFP            : Predicate<"!Subtarget->hasVFP2Base()">;
 def HasVFP2          : Predicate<"Subtarget->hasVFP2Base()">,
                                  AssemblerPredicate<(all_of FeatureVFP2_SP), "VFP2">;
@@ -106,6 +108,8 @@ def HasFullFP16      : Predicate<"Subtarget->hasFullFP16()">,
                                  AssemblerPredicate<(all_of FeatureFullFP16),"full half-float">;
 def HasFP16FML       : Predicate<"Subtarget->hasFP16FML()">,
                                  AssemblerPredicate<(all_of FeatureFP16FML),"full half-float fml">;
+def HasBF16          : Predicate<"Subtarget->hasBF16()">,
+                                 AssemblerPredicate<(all_of FeatureBF16),"BFloat16 floating point extension">;
 def HasDivideInThumb : Predicate<"Subtarget->hasDivideInThumbMode()">,
                                  AssemblerPredicate<(all_of FeatureHWDivThumb), "divide in THUMB">;
 def HasDivideInARM   : Predicate<"Subtarget->hasDivideInARMMode()">,
diff --git a/llvm/lib/Target/ARM/ARMSubtarget.h b/llvm/lib/Target/ARM/ARMSubtarget.h
index 27f67a9acbd3c..66c50fb590d9a 100644
--- a/llvm/lib/Target/ARM/ARMSubtarget.h
+++ b/llvm/lib/Target/ARM/ARMSubtarget.h
@@ -108,6 +108,7 @@ class ARMSubtarget : public ARMGenSubtargetInfo {
     ARMv83a,
     ARMv84a,
     ARMv85a,
+    ARMv86a,
     ARMv8a,
     ARMv8mBaseline,
     ARMv8mMainline,
@@ -157,6 +158,7 @@ class ARMSubtarget : public ARMGenSubtargetInfo {
   bool HasV8_3aOps = false;
   bool HasV8_4aOps = false;
   bool HasV8_5aOps = false;
+  bool HasV8_6aOps = false;
   bool HasV8MBaselineOps = false;
   bool HasV8MMainlineOps = false;
   bool HasV8_1MMainlineOps = false;
@@ -255,6 +257,9 @@ class ARMSubtarget : public ARMGenSubtargetInfo {
   /// HasFP16FML - True if subtarget supports half-precision FP fml operations
   bool HasFP16FML = false;
 
+  /// HasBF16 - True if subtarget supports BFloat16 floating point operations
+  bool HasBF16 = false;
+
   /// HasD32 - True if subtarget has the full 32 double precision
   /// FP registers for VFPv3.
   bool HasD32 = false;
@@ -581,6 +586,7 @@ class ARMSubtarget : public ARMGenSubtargetInfo {
   bool hasV8_3aOps() const { return HasV8_3aOps; }
   bool hasV8_4aOps() const { return HasV8_4aOps; }
   bool hasV8_5aOps() const { return HasV8_5aOps; }
+  bool hasV8_6aOps() const { return HasV8_6aOps; }
   bool hasV8MBaselineOps() const { return HasV8MBaselineOps; }
   bool hasV8MMainlineOps() const { return HasV8MMainlineOps; }
   bool hasV8_1MMainlineOps() const { return HasV8_1MMainlineOps; }
diff --git a/llvm/lib/Target/ARM/ARMTargetMachine.cpp b/llvm/lib/Target/ARM/ARMTargetMachine.cpp
index 63aa65267ef26..947bbc3ad3eef 100644
--- a/llvm/lib/Target/ARM/ARMTargetMachine.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetMachine.cpp
@@ -362,6 +362,7 @@ class ARMPassConfig : public TargetPassConfig {
   void addPreRegAlloc() override;
   void addPreSched2() override;
   void addPreEmitPass() override;
+  void addPreEmitPass2() override;
 
   std::unique_ptr<CSEConfigBase> getCSEConfig() const override;
 };
@@ -541,7 +542,9 @@ void ARMPassConfig::addPreEmitPass() {
   // Don't optimize barriers at -O0.
   if (getOptLevel() != CodeGenOpt::None)
     addPass(createARMOptimizeBarriersPass());
+}
 
+void ARMPassConfig::addPreEmitPass2() {
   addPass(createARMConstantIslandPass());
   addPass(createARMLowOverheadLoopsPass());
 
diff --git a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
index 8ae7525ddea61..9304d83391938 100644
--- a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
+++ b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
@@ -6322,6 +6322,7 @@ StringRef ARMAsmParser::splitMnemonic(StringRef Mnemonic,
       Mnemonic == "vrintp" || Mnemonic == "vrintm" || Mnemonic == "hvc" ||
       Mnemonic.startswith("vsel") || Mnemonic == "vins" || Mnemonic == "vmovx" ||
       Mnemonic == "bxns"  || Mnemonic == "blxns" ||
+      Mnemonic == "vdot"  || Mnemonic == "vmmla"  ||
       Mnemonic == "vudot" || Mnemonic == "vsdot" ||
       Mnemonic == "vcmla" || Mnemonic == "vcadd" ||
       Mnemonic == "vfmal" || Mnemonic == "vfmsl" ||
@@ -6462,6 +6463,8 @@ void ARMAsmParser::getMnemonicAcceptInfo(StringRef Mnemonic,
       Mnemonic == "vudot" || Mnemonic == "vsdot" ||
       Mnemonic == "vcmla" || Mnemonic == "vcadd" ||
       Mnemonic == "vfmal" || Mnemonic == "vfmsl" ||
+      Mnemonic == "vfmat" || Mnemonic == "vfmab" ||
+      Mnemonic == "vdot"  || Mnemonic == "vmmla" ||
       Mnemonic == "sb"    || Mnemonic == "ssbb"  ||
       Mnemonic == "pssbb" ||
       Mnemonic == "bfcsel" || Mnemonic == "wls" ||
@@ -6977,6 +6980,8 @@ bool ARMAsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
   //    ITx   -> x100    (ITT -> 0100, ITE -> 1100)
   //    ITxy  -> xy10    (e.g. ITET -> 1010)
   //    ITxyz -> xyz1    (e.g. ITEET -> 1101)
+  // Note: See the ARM::PredBlockMask enum in
+  //   /lib/Target/ARM/Utils/ARMBaseInfo.h
   if (Mnemonic == "it" || Mnemonic.startswith("vpt") ||
       Mnemonic.startswith("vpst")) {
     SMLoc Loc = Mnemonic == "it"  ? SMLoc::getFromPointer(NameLoc.getPointer() + 2) :
@@ -11140,11 +11145,13 @@ bool ARMAsmParser::parseDirectiveEabiAttr(SMLoc L) {
   TagLoc = Parser.getTok().getLoc();
   if (Parser.getTok().is(AsmToken::Identifier)) {
     StringRef Name = Parser.getTok().getIdentifier();
-    Tag = ARMBuildAttrs::AttrTypeFromString(Name);
-    if (Tag == -1) {
+    Optional<unsigned> Ret =
+        ELFAttrs::attrTypeFromString(Name, ARMBuildAttrs::ARMAttributeTags);
+    if (!Ret.hasValue()) {
       Error(TagLoc, "attribute name not recognised: " + Name);
       return false;
     }
+    Tag = Ret.getValue();
     Parser.Lex();
   } else {
     const MCExpr *AttrExpr;
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
index d118b4f9431ab..813406c0ecb11 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
@@ -48,10 +48,17 @@ class ARMELFObjectWriter : public MCELFObjectTargetWriter {
 } // end anonymous namespace
 
 Optional<MCFixupKind> ARMAsmBackend::getFixupKind(StringRef Name) const {
-  if (STI.getTargetTriple().isOSBinFormatELF() && Name == "R_ARM_NONE")
-    return FK_NONE;
-
-  return MCAsmBackend::getFixupKind(Name);
+  if (!STI.getTargetTriple().isOSBinFormatELF())
+    return None;
+
+  unsigned Type = llvm::StringSwitch<unsigned>(Name)
+#define ELF_RELOC(X, Y) .Case(#X, Y)
+#include "llvm/BinaryFormat/ELFRelocs/ARM.def"
+#undef ELF_RELOC
+                      .Default(-1u);
+  if (Type == -1u)
+    return None;
+  return static_cast<MCFixupKind>(FirstLiteralRelocationKind + Type);
 }
 
 const MCFixupKindInfo &ARMAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
@@ -172,6 +179,11 @@ const MCFixupKindInfo &ARMAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
       {"fixup_le", 0, 32, MCFixupKindInfo::FKF_IsPCRel}
   };
 
+  // Fixup kinds from .reloc directive are like R_ARM_NONE. They do not require
+  // any extra processing.
+  if (Kind >= FirstLiteralRelocationKind)
+    return MCAsmBackend::getFixupKindInfo(FK_NONE);
+
   if (Kind < FirstTargetFixupKind)
     return MCAsmBackend::getFixupKindInfo(Kind);
 
@@ -438,7 +450,6 @@ unsigned ARMAsmBackend::adjustFixupValue(const MCAssembler &Asm,
   default:
     Ctx.reportError(Fixup.getLoc(), "bad relocation fixup type");
     return 0;
-  case FK_NONE:
   case FK_Data_1:
   case FK_Data_2:
   case FK_Data_4:
@@ -871,7 +882,7 @@ bool ARMAsmBackend::shouldForceRelocation(const MCAssembler &Asm,
   const MCSymbolRefExpr *A = Target.getSymA();
   const MCSymbol *Sym = A ? &A->getSymbol() : nullptr;
   const unsigned FixupKind = Fixup.getKind();
-  if (FixupKind == FK_NONE)
+  if (FixupKind >= FirstLiteralRelocationKind)
     return true;
   if (FixupKind == ARM::fixup_arm_thumb_bl) {
     assert(Sym && "How did we resolve this?");
@@ -915,9 +926,6 @@ static unsigned getFixupKindNumBytes(unsigned Kind) {
   default:
     llvm_unreachable("Unknown fixup kind!");
 
-  case FK_NONE:
-    return 0;
-
   case FK_Data_1:
   case ARM::fixup_arm_thumb_bcc:
   case ARM::fixup_arm_thumb_cp:
@@ -979,9 +987,6 @@ static unsigned getFixupKindContainerSizeBytes(unsigned Kind) {
   default:
     llvm_unreachable("Unknown fixup kind!");
 
-  case FK_NONE:
-    return 0;
-
   case FK_Data_1:
     return 1;
   case FK_Data_2:
@@ -1037,7 +1042,10 @@ void ARMAsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
                                MutableArrayRef<char> Data, uint64_t Value,
                                bool IsResolved,
                                const MCSubtargetInfo* STI) const {
-  unsigned NumBytes = getFixupKindNumBytes(Fixup.getKind());
+  unsigned Kind = Fixup.getKind();
+  if (Kind >= FirstLiteralRelocationKind)
+    return;
+  unsigned NumBytes = getFixupKindNumBytes(Kind);
   MCContext &Ctx = Asm.getContext();
   Value = adjustFixupValue(Asm, Fixup, Target, Value, IsResolved, Ctx, STI);
   if (!Value)
@@ -1049,7 +1057,7 @@ void ARMAsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
   // Used to point to big endian bytes.
   unsigned FullSizeBytes;
   if (Endian == support::big) {
-    FullSizeBytes = getFixupKindContainerSizeBytes(Fixup.getKind());
+    FullSizeBytes = getFixupKindContainerSizeBytes(Kind);
     assert((Offset + FullSizeBytes) <= Data.size() && "Invalid fixup size!");
     assert(NumBytes <= FullSizeBytes && "Invalid fixup size!");
   }
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMBaseInfo.h b/llvm/lib/Target/ARM/MCTargetDesc/ARMBaseInfo.h
index 7d7d0af238938..a6e19b67095ea 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMBaseInfo.h
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMBaseInfo.h
@@ -403,6 +403,10 @@ namespace ARMII {
     // Whether the instruction produces a scalar result from vector operands.
     HorizontalReduction = 1 << 22,
 
+    // Whether this instruction produces a vector result that is larger than
+    // its input, typically reading from the top/bottom halves of the input(s).
+    DoubleWidthResult = 1 << 23,
+
     //===------------------------------------------------------------------===//
     // Code domain.
     DomainShift   = 15,
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMELFObjectWriter.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMELFObjectWriter.cpp
index 1e2399c4be994..84cbd4a42686a 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMELFObjectWriter.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMELFObjectWriter.cpp
@@ -79,6 +79,9 @@ unsigned ARMELFObjectWriter::GetRelocTypeInner(const MCValue &Target,
                                                const MCFixup &Fixup,
                                                bool IsPCRel,
                                                MCContext &Ctx) const {
+  unsigned Kind = Fixup.getTargetKind();
+  if (Kind >= FirstLiteralRelocationKind)
+    return Kind - FirstLiteralRelocationKind;
   MCSymbolRefExpr::VariantKind Modifier = Target.getAccessVariant();
 
   if (IsPCRel) {
@@ -152,12 +155,10 @@ unsigned ARMELFObjectWriter::GetRelocTypeInner(const MCValue &Target,
       return ELF::R_ARM_THM_BF18;
     }
   }
-  switch (Fixup.getTargetKind()) {
+  switch (Kind) {
   default:
     Ctx.reportFatalError(Fixup.getLoc(), "unsupported relocation on symbol");
     return ELF::R_ARM_NONE;
-  case FK_NONE:
-    return ELF::R_ARM_NONE;
   case FK_Data_1:
     switch (Modifier) {
     default:
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
index 6aeb57dbb5dce..e75e7b7ee221e 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
@@ -177,7 +177,8 @@ void ARMTargetAsmStreamer::switchVendor(StringRef Vendor) {}
 void ARMTargetAsmStreamer::emitAttribute(unsigned Attribute, unsigned Value) {
   OS << "\t.eabi_attribute\t" << Attribute << ", " << Twine(Value);
   if (IsVerboseAsm) {
-    StringRef Name = ARMBuildAttrs::AttrTypeAsString(Attribute);
+    StringRef Name =
+        ELFAttrs::attrTypeAsString(Attribute, ARMBuildAttrs::ARMAttributeTags);
     if (!Name.empty())
       OS << "\t@ " << Name;
   }
@@ -193,7 +194,8 @@ void ARMTargetAsmStreamer::emitTextAttribute(unsigned Attribute,
   default:
     OS << "\t.eabi_attribute\t" << Attribute << ", \"" << String << "\"";
     if (IsVerboseAsm) {
-      StringRef Name = ARMBuildAttrs::AttrTypeAsString(Attribute);
+      StringRef Name = ELFAttrs::attrTypeAsString(
+          Attribute, ARMBuildAttrs::ARMAttributeTags);
       if (!Name.empty())
         OS << "\t@ " << Name;
     }
@@ -212,7 +214,9 @@ void ARMTargetAsmStreamer::emitIntTextAttribute(unsigned Attribute,
     if (!StringValue.empty())
       OS << ", \"" << StringValue << "\"";
     if (IsVerboseAsm)
-      OS << "\t@ " << ARMBuildAttrs::AttrTypeAsString(Attribute);
+      OS << "\t@ "
+         << ELFAttrs::attrTypeAsString(Attribute,
+                                       ARMBuildAttrs::ARMAttributeTags);
     break;
   }
   OS << "\n";
@@ -856,6 +860,7 @@ void ARMTargetELFStreamer::emitArchDefaultAttributes() {
   case ARM::ArchKind::ARMV8_3A:
   case ARM::ArchKind::ARMV8_4A:
   case ARM::ArchKind::ARMV8_5A:
+  case ARM::ArchKind::ARMV8_6A:
     setAttributeItem(CPU_arch_profile, ApplicationProfile, false);
     setAttributeItem(ARM_ISA_use, Allowed, false);
     setAttributeItem(THUMB_ISA_use, AllowThumb32, false);
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp
index c33e568269699..744d919f2fd49 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp
@@ -288,7 +288,7 @@ void ARMInstPrinter::printInst(const MCInst *MI, uint64_t Address,
   case ARM::t2DSB:
     switch (MI->getOperand(0).getImm()) {
     default:
-      if (!printAliasInstr(MI, STI, O))
+      if (!printAliasInstr(MI, Address, STI, O))
         printInstruction(MI, Address, STI, O);
       break;
     case 0:
@@ -302,7 +302,7 @@ void ARMInstPrinter::printInst(const MCInst *MI, uint64_t Address,
     return;
   }
 
-  if (!printAliasInstr(MI, STI, O))
+  if (!printAliasInstr(MI, Address, STI, O))
     printInstruction(MI, Address, STI, O);
 
   printAnnotation(O, Annot);
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.h b/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.h
index eba57a89e544f..37cb731ff0014 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.h
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMInstPrinter.h
@@ -32,10 +32,10 @@ class ARMInstPrinter : public MCInstPrinter {
   // Autogenerated by tblgen.
   void printInstruction(const MCInst *MI, uint64_t Address,
                         const MCSubtargetInfo &STI, raw_ostream &O);
-  virtual bool printAliasInstr(const MCInst *MI, const MCSubtargetInfo &STI,
-                               raw_ostream &O);
-  virtual void printCustomAliasOperand(const MCInst *MI, unsigned OpIdx,
-                                       unsigned PrintMethodIdx,
+  virtual bool printAliasInstr(const MCInst *MI, uint64_t Address,
+                               const MCSubtargetInfo &STI, raw_ostream &O);
+  virtual void printCustomAliasOperand(const MCInst *MI, uint64_t Address,
+                                       unsigned OpIdx, unsigned PrintMethodIdx,
                                        const MCSubtargetInfo &STI,
                                        raw_ostream &O);
   static const char *getRegisterName(unsigned RegNo,
@@ -43,6 +43,10 @@ class ARMInstPrinter : public MCInstPrinter {
 
   void printOperand(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
                     raw_ostream &O);
+  void printOperand(const MCInst *MI, uint64_t /*Address*/, unsigned OpNum,
+                    const MCSubtargetInfo &STI, raw_ostream &O) {
+    printOperand(MI, OpNum, STI, O);
+  }
 
   void printSORegRegOperand(const MCInst *MI, unsigned OpNum,
                             const MCSubtargetInfo &STI, raw_ostream &O);
@@ -109,6 +113,12 @@ class ARMInstPrinter : public MCInstPrinter {
   template <unsigned scale>
   void printAdrLabelOperand(const MCInst *MI, unsigned OpNum,
                             const MCSubtargetInfo &STI, raw_ostream &O);
+  template <unsigned scale>
+  void printAdrLabelOperand(const MCInst *MI, uint64_t /*Address*/,
+                            unsigned OpNum, const MCSubtargetInfo &STI,
+                            raw_ostream &O) {
+    printAdrLabelOperand<scale>(MI, OpNum, STI, O);
+  }
   void printThumbS4ImmOperand(const MCInst *MI, unsigned OpNum,
                               const MCSubtargetInfo &STI, raw_ostream &O);
   void printThumbSRImm(const MCInst *MI, unsigned OpNum,
@@ -206,6 +216,11 @@ class ARMInstPrinter : public MCInstPrinter {
                     const MCSubtargetInfo &STI, raw_ostream &O);
   void printThumbLdrLabelOperand(const MCInst *MI, unsigned OpNum,
                                  const MCSubtargetInfo &STI, raw_ostream &O);
+  void printThumbLdrLabelOperand(const MCInst *MI, uint64_t /*Address*/,
+                                 unsigned OpNum, const MCSubtargetInfo &STI,
+                                 raw_ostream &O) {
+    printThumbLdrLabelOperand(MI, OpNum, STI, O);
+  }
   void printFBits16(const MCInst *MI, unsigned OpNum,
                     const MCSubtargetInfo &STI, raw_ostream &O);
   void printFBits32(const MCInst *MI, unsigned OpNum,
diff --git a/llvm/lib/Target/ARM/MVETailPredication.cpp b/llvm/lib/Target/ARM/MVETailPredication.cpp
index dd7867397b453..26b71ba5b6003 100644
--- a/llvm/lib/Target/ARM/MVETailPredication.cpp
+++ b/llvm/lib/Target/ARM/MVETailPredication.cpp
@@ -286,7 +286,7 @@ bool MVETailPredication::isTailPredicate(TripCountPattern &TCP) {
   Instruction *Insert = nullptr;
   // The shuffle which broadcasts the index iv into a vector.
   if (!match(BroadcastSplat,
-             m_ShuffleVector(m_Instruction(Insert), m_Undef(), m_Zero())))
+             m_ShuffleVector(m_Instruction(Insert), m_Undef(), m_ZeroMask())))
     return false;
 
   // The insert element which initialises a vector with the index iv.
@@ -409,7 +409,7 @@ static bool MatchElemCountLoopSetup(Loop *L, Instruction *Shuffle,
   Instruction *Insert = nullptr;
 
   if (!match(Shuffle,
-             m_ShuffleVector(m_Instruction(Insert), m_Undef(), m_Zero())))
+             m_ShuffleVector(m_Instruction(Insert), m_Undef(), m_ZeroMask())))
     return false;
 
   // Insert the limit into a vector.
diff --git a/llvm/lib/Target/ARM/MVEVPTBlockPass.cpp b/llvm/lib/Target/ARM/MVEVPTBlockPass.cpp
index c8b725f339e26..2de9829480db4 100644
--- a/llvm/lib/Target/ARM/MVEVPTBlockPass.cpp
+++ b/llvm/lib/Target/ARM/MVEVPTBlockPass.cpp
@@ -34,30 +34,30 @@ using namespace llvm;
 #define DEBUG_TYPE "arm-mve-vpt"
 
 namespace {
-  class MVEVPTBlock : public MachineFunctionPass {
-  public:
-    static char ID;
-    const Thumb2InstrInfo *TII;
-    const TargetRegisterInfo *TRI;
+class MVEVPTBlock : public MachineFunctionPass {
+public:
+  static char ID;
+  const Thumb2InstrInfo *TII;
+  const TargetRegisterInfo *TRI;
 
-    MVEVPTBlock() : MachineFunctionPass(ID) {}
+  MVEVPTBlock() : MachineFunctionPass(ID) {}
 
-    bool runOnMachineFunction(MachineFunction &Fn) override;
+  bool runOnMachineFunction(MachineFunction &Fn) override;
 
-    MachineFunctionProperties getRequiredProperties() const override {
-      return MachineFunctionProperties().set(
-          MachineFunctionProperties::Property::NoVRegs);
-    }
+  MachineFunctionProperties getRequiredProperties() const override {
+    return MachineFunctionProperties().set(
+        MachineFunctionProperties::Property::NoVRegs);
+  }
 
-    StringRef getPassName() const override {
-      return "MVE VPT block insertion pass";
-    }
+  StringRef getPassName() const override {
+    return "MVE VPT block insertion pass";
+  }
 
-  private:
-    bool InsertVPTBlocks(MachineBasicBlock &MBB);
-  };
+private:
+  bool InsertVPTBlocks(MachineBasicBlock &MBB);
+};
 
-  char MVEVPTBlock::ID = 0;
+char MVEVPTBlock::ID = 0;
 
 } // end anonymous namespace
 
@@ -94,24 +94,184 @@ static MachineInstr *findVCMPToFoldIntoVPST(MachineBasicBlock::iterator MI,
   return &*CmpMI;
 }
 
+static ARM::PredBlockMask ExpandBlockMask(ARM::PredBlockMask BlockMask,
+                                          ARMVCC::VPTCodes Kind) {
+  using PredBlockMask = ARM::PredBlockMask;
+  assert(Kind != ARMVCC::None && "Cannot expand mask with 'None'");
+  assert(countTrailingZeros((unsigned)BlockMask) != 0 &&
+         "Mask is already full");
+
+  auto ChooseMask = [&](PredBlockMask AddedThen, PredBlockMask AddedElse) {
+    return (Kind == ARMVCC::Then) ? AddedThen : AddedElse;
+  };
+
+  switch (BlockMask) {
+  case PredBlockMask::T:
+    return ChooseMask(PredBlockMask::TT, PredBlockMask::TE);
+  case PredBlockMask::TT:
+    return ChooseMask(PredBlockMask::TTT, PredBlockMask::TTE);
+  case PredBlockMask::TE:
+    return ChooseMask(PredBlockMask::TET, PredBlockMask::TEE);
+  case PredBlockMask::TTT:
+    return ChooseMask(PredBlockMask::TTTT, PredBlockMask::TTTE);
+  case PredBlockMask::TTE:
+    return ChooseMask(PredBlockMask::TTET, PredBlockMask::TTEE);
+  case PredBlockMask::TET:
+    return ChooseMask(PredBlockMask::TETT, PredBlockMask::TETE);
+  case PredBlockMask::TEE:
+    return ChooseMask(PredBlockMask::TEET, PredBlockMask::TEEE);
+  default:
+    llvm_unreachable("Unknown Mask");
+  }
+}
+
+// Advances Iter past a block of predicated instructions.
+// Returns true if it successfully skipped the whole block of predicated
+// instructions. Returns false when it stopped early (due to MaxSteps), or if
+// Iter didn't point to a predicated instruction.
+static bool StepOverPredicatedInstrs(MachineBasicBlock::instr_iterator &Iter,
+                                     MachineBasicBlock::instr_iterator EndIter,
+                                     unsigned MaxSteps,
+                                     unsigned &NumInstrsSteppedOver) {
+  ARMVCC::VPTCodes NextPred = ARMVCC::None;
+  unsigned PredReg;
+  NumInstrsSteppedOver = 0;
+
+  while (Iter != EndIter) {
+    NextPred = getVPTInstrPredicate(*Iter, PredReg);
+    assert(NextPred != ARMVCC::Else &&
+           "VPT block pass does not expect Else preds");
+    if (NextPred == ARMVCC::None || MaxSteps == 0)
+      break;
+    --MaxSteps;
+    ++Iter;
+    ++NumInstrsSteppedOver;
+  };
+
+  return NumInstrsSteppedOver != 0 &&
+         (NextPred == ARMVCC::None || Iter == EndIter);
+}
+
+// Returns true if at least one instruction in the range [Iter, End) defines
+// or kills VPR.
+static bool IsVPRDefinedOrKilledByBlock(MachineBasicBlock::iterator Iter,
+                                        MachineBasicBlock::iterator End) {
+  for (; Iter != End; ++Iter)
+    if (Iter->definesRegister(ARM::VPR) || Iter->killsRegister(ARM::VPR))
+      return true;
+  return false;
+}
+
+// Given an iterator (Iter) that points at an instruction with a "Then"
+// predicate, tries to create the largest block of continuous predicated
+// instructions possible, and returns the VPT Block Mask of that block.
+//
+// This will try to perform some minor optimization in order to maximize the
+// size of the block.
+static ARM::PredBlockMask
+CreateVPTBlock(MachineBasicBlock::instr_iterator &Iter,
+               MachineBasicBlock::instr_iterator EndIter,
+               SmallVectorImpl<MachineInstr *> &DeadInstructions) {
+  MachineBasicBlock::instr_iterator BlockBeg = Iter;
+  (void)BlockBeg;
+  assert(getVPTInstrPredicate(*Iter) == ARMVCC::Then &&
+         "Expected a Predicated Instruction");
+
+  LLVM_DEBUG(dbgs() << "VPT block created for: "; Iter->dump());
+
+  unsigned BlockSize;
+  StepOverPredicatedInstrs(Iter, EndIter, 4, BlockSize);
+
+  LLVM_DEBUG(for (MachineBasicBlock::instr_iterator AddedInstIter =
+                      std::next(BlockBeg);
+                  AddedInstIter != Iter; ++AddedInstIter) {
+    dbgs() << "  adding: ";
+    AddedInstIter->dump();
+  });
+
+  // Generate the initial BlockMask
+  ARM::PredBlockMask BlockMask = getARMVPTBlockMask(BlockSize);
+
+  // Remove VPNOTs while there's still room in the block, so we can make the
+  // largest block possible.
+  ARMVCC::VPTCodes CurrentPredicate = ARMVCC::Then;
+  while (BlockSize < 4 && Iter != EndIter &&
+         Iter->getOpcode() == ARM::MVE_VPNOT) {
+
+    // Try to skip all of the predicated instructions after the VPNOT, stopping
+    // after (4 - BlockSize). If we can't skip them all, stop.
+    unsigned ElseInstCnt = 0;
+    MachineBasicBlock::instr_iterator VPNOTBlockEndIter = std::next(Iter);
+    if (!StepOverPredicatedInstrs(VPNOTBlockEndIter, EndIter, (4 - BlockSize),
+                                  ElseInstCnt))
+      break;
+
+    // Check if this VPNOT can be removed or not: It can only be removed if at
+    // least one of the predicated instruction that follows it kills or sets
+    // VPR.
+    if (!IsVPRDefinedOrKilledByBlock(Iter, VPNOTBlockEndIter))
+      break;
+
+    LLVM_DEBUG(dbgs() << "  removing VPNOT: "; Iter->dump(););
+
+    // Record the new size of the block
+    BlockSize += ElseInstCnt;
+    assert(BlockSize <= 4 && "Block is too large!");
+
+    // Record the VPNot to remove it later.
+    DeadInstructions.push_back(&*Iter);
+    ++Iter;
+
+    // Replace "then" by "elses" in the block until we find an instruction that
+    // defines VPR, then after that leave everything to "t".
+    // Note that we are using "Iter" to iterate over the block so we can update
+    // it at the same time.
+    bool ChangeToElse = (CurrentPredicate == ARMVCC::Then);
+    for (; Iter != VPNOTBlockEndIter; ++Iter) {
+      // Find the register in which the predicate is
+      int OpIdx = findFirstVPTPredOperandIdx(*Iter);
+      assert(OpIdx != -1);
+
+      // Update the mask + change the predicate to an else if needed.
+      if (ChangeToElse) {
+        // Change the predicate and update the mask
+        Iter->getOperand(OpIdx).setImm(ARMVCC::Else);
+        BlockMask = ExpandBlockMask(BlockMask, ARMVCC::Else);
+        // Reset back to a "then" predicate if this instruction defines VPR.
+        if (Iter->definesRegister(ARM::VPR))
+          ChangeToElse = false;
+      } else
+        BlockMask = ExpandBlockMask(BlockMask, ARMVCC::Then);
+
+      LLVM_DEBUG(dbgs() << "  adding: "; Iter->dump());
+    }
+
+    CurrentPredicate =
+        (CurrentPredicate == ARMVCC::Then ? ARMVCC::Else : ARMVCC::Then);
+  }
+  return BlockMask;
+}
+
 bool MVEVPTBlock::InsertVPTBlocks(MachineBasicBlock &Block) {
   bool Modified = false;
   MachineBasicBlock::instr_iterator MBIter = Block.instr_begin();
   MachineBasicBlock::instr_iterator EndIter = Block.instr_end();
 
+  SmallVector<MachineInstr *, 4> DeadInstructions;
+
   while (MBIter != EndIter) {
     MachineInstr *MI = &*MBIter;
     unsigned PredReg = 0;
-    DebugLoc dl = MI->getDebugLoc();
+    DebugLoc DL = MI->getDebugLoc();
 
     ARMVCC::VPTCodes Pred = getVPTInstrPredicate(*MI, PredReg);
 
     // The idea of the predicate is that None, Then and Else are for use when
     // handling assembly language: they correspond to the three possible
     // suffixes "", "t" and "e" on the mnemonic. So when instructions are read
-    // from assembly source or disassembled from object code, you expect to see
-    // a mixture whenever there's a long VPT block. But in code generation, we
-    // hope we'll never generate an Else as input to this pass.
+    // from assembly source or disassembled from object code, you expect to
+    // see a mixture whenever there's a long VPT block. But in code
+    // generation, we hope we'll never generate an Else as input to this pass.
     assert(Pred != ARMVCC::Else && "VPT block pass does not expect Else preds");
 
     if (Pred == ARMVCC::None) {
@@ -119,42 +279,25 @@ bool MVEVPTBlock::InsertVPTBlocks(MachineBasicBlock &Block) {
       continue;
     }
 
-    LLVM_DEBUG(dbgs() << "VPT block created for: "; MI->dump());
-    int VPTInstCnt = 1;
-    ARMVCC::VPTCodes NextPred;
-
-    // Look at subsequent instructions, checking if they can be in the same VPT
-    // block.
-    ++MBIter;
-    while (MBIter != EndIter && VPTInstCnt < 4) {
-      NextPred = getVPTInstrPredicate(*MBIter, PredReg);
-      assert(NextPred != ARMVCC::Else &&
-             "VPT block pass does not expect Else preds");
-      if (NextPred != Pred)
-        break;
-      LLVM_DEBUG(dbgs() << "  adding : "; MBIter->dump());
-      ++VPTInstCnt;
-      ++MBIter;
-    };
-
-    unsigned BlockMask = getARMVPTBlockMask(VPTInstCnt);
+    ARM::PredBlockMask BlockMask =
+        CreateVPTBlock(MBIter, EndIter, DeadInstructions);
 
-    // Search back for a VCMP that can be folded to create a VPT, or else create
-    // a VPST directly
+    // Search back for a VCMP that can be folded to create a VPT, or else
+    // create a VPST directly
     MachineInstrBuilder MIBuilder;
     unsigned NewOpcode;
-    MachineInstr *VCMP = findVCMPToFoldIntoVPST(MI, TRI, NewOpcode);
-    if (VCMP) {
+    LLVM_DEBUG(dbgs() << "  final block mask: " << (unsigned)BlockMask << "\n");
+    if (MachineInstr *VCMP = findVCMPToFoldIntoVPST(MI, TRI, NewOpcode)) {
       LLVM_DEBUG(dbgs() << "  folding VCMP into VPST: "; VCMP->dump());
-      MIBuilder = BuildMI(Block, MI, dl, TII->get(NewOpcode));
-      MIBuilder.addImm(BlockMask);
+      MIBuilder = BuildMI(Block, MI, DL, TII->get(NewOpcode));
+      MIBuilder.addImm((uint64_t)BlockMask);
       MIBuilder.add(VCMP->getOperand(1));
       MIBuilder.add(VCMP->getOperand(2));
       MIBuilder.add(VCMP->getOperand(3));
       VCMP->eraseFromParent();
     } else {
-      MIBuilder = BuildMI(Block, MI, dl, TII->get(ARM::MVE_VPST));
-      MIBuilder.addImm(BlockMask);
+      MIBuilder = BuildMI(Block, MI, DL, TII->get(ARM::MVE_VPST));
+      MIBuilder.addImm((uint64_t)BlockMask);
     }
 
     finalizeBundle(
@@ -162,6 +305,15 @@ bool MVEVPTBlock::InsertVPTBlocks(MachineBasicBlock &Block) {
 
     Modified = true;
   }
+
+  // Erase all dead instructions
+  for (MachineInstr *DeadMI : DeadInstructions) {
+    if (DeadMI->isInsideBundle())
+      DeadMI->eraseFromBundle();
+    else
+      DeadMI->eraseFromParent();
+  }
+
   return Modified;
 }
 
diff --git a/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp b/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp
index 0e024437667e8..df7a3fed6acd2 100644
--- a/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp
+++ b/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp
@@ -127,7 +127,7 @@ eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB,
       // We need to keep the stack aligned properly.  To do this, we round the
       // amount of space needed for the outgoing arguments up to the next
       // alignment boundary.
-      Amount = alignTo(Amount, getStackAlignment());
+      Amount = alignTo(Amount, getStackAlign());
 
       // Replace the pseudo instruction with a new instruction...
       unsigned Opc = Old.getOpcode();
diff --git a/llvm/lib/Target/ARM/Thumb1InstrInfo.cpp b/llvm/lib/Target/ARM/Thumb1InstrInfo.cpp
index 1a054a416e167..79afa378cb624 100644
--- a/llvm/lib/Target/ARM/Thumb1InstrInfo.cpp
+++ b/llvm/lib/Target/ARM/Thumb1InstrInfo.cpp
@@ -92,7 +92,7 @@ storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
     MachineFrameInfo &MFI = MF.getFrameInfo();
     MachineMemOperand *MMO = MF.getMachineMemOperand(
         MachinePointerInfo::getFixedStack(MF, FI), MachineMemOperand::MOStore,
-        MFI.getObjectSize(FI), MFI.getObjectAlignment(FI));
+        MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
     BuildMI(MBB, I, DL, get(ARM::tSTRspi))
         .addReg(SrcReg, getKillRegState(isKill))
         .addFrameIndex(FI)
@@ -121,7 +121,7 @@ loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
     MachineFrameInfo &MFI = MF.getFrameInfo();
     MachineMemOperand *MMO = MF.getMachineMemOperand(
         MachinePointerInfo::getFixedStack(MF, FI), MachineMemOperand::MOLoad,
-        MFI.getObjectSize(FI), MFI.getObjectAlignment(FI));
+        MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
     BuildMI(MBB, I, DL, get(ARM::tLDRspi), DestReg)
         .addFrameIndex(FI)
         .addImm(0)
diff --git a/llvm/lib/Target/ARM/Thumb2InstrInfo.cpp b/llvm/lib/Target/ARM/Thumb2InstrInfo.cpp
index f4d1107fe7a9d..0719b85db116d 100644
--- a/llvm/lib/Target/ARM/Thumb2InstrInfo.cpp
+++ b/llvm/lib/Target/ARM/Thumb2InstrInfo.cpp
@@ -143,7 +143,7 @@ storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
   MachineFrameInfo &MFI = MF.getFrameInfo();
   MachineMemOperand *MMO = MF.getMachineMemOperand(
       MachinePointerInfo::getFixedStack(MF, FI), MachineMemOperand::MOStore,
-      MFI.getObjectSize(FI), MFI.getObjectAlignment(FI));
+      MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
 
   if (ARM::GPRRegClass.hasSubClassEq(RC)) {
     BuildMI(MBB, I, DL, get(ARM::t2STRi12))
@@ -183,7 +183,7 @@ loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
   MachineFrameInfo &MFI = MF.getFrameInfo();
   MachineMemOperand *MMO = MF.getMachineMemOperand(
       MachinePointerInfo::getFixedStack(MF, FI), MachineMemOperand::MOLoad,
-      MFI.getObjectSize(FI), MFI.getObjectAlignment(FI));
+      MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
   DebugLoc DL;
   if (I != MBB.end()) DL = I->getDebugLoc();
 
diff --git a/llvm/lib/Target/ARM/Thumb2InstrInfo.h b/llvm/lib/Target/ARM/Thumb2InstrInfo.h
index 3e8e77a9db1f4..583a09163f4e3 100644
--- a/llvm/lib/Target/ARM/Thumb2InstrInfo.h
+++ b/llvm/lib/Target/ARM/Thumb2InstrInfo.h
@@ -74,6 +74,10 @@ ARMCC::CondCodes getITInstrPredicate(const MachineInstr &MI, unsigned &PredReg);
 int findFirstVPTPredOperandIdx(const MachineInstr &MI);
 ARMVCC::VPTCodes getVPTInstrPredicate(const MachineInstr &MI,
                                       unsigned &PredReg);
+inline ARMVCC::VPTCodes getVPTInstrPredicate(const MachineInstr &MI) {
+  unsigned PredReg;
+  return getVPTInstrPredicate(MI, PredReg);
 }
+} // namespace llvm
 
 #endif
diff --git a/llvm/lib/Target/ARM/Thumb2SizeReduction.cpp b/llvm/lib/Target/ARM/Thumb2SizeReduction.cpp
index c5a62aa339907..c732f03053392 100644
--- a/llvm/lib/Target/ARM/Thumb2SizeReduction.cpp
+++ b/llvm/lib/Target/ARM/Thumb2SizeReduction.cpp
@@ -457,7 +457,7 @@ Thumb2SizeReduce::ReduceLoadStore(MachineBasicBlock &MBB, MachineInstr *MI,
       return false;
 
     if (!MI->hasOneMemOperand() ||
-        (*MI->memoperands_begin())->getAlignment() < 4)
+        (*MI->memoperands_begin())->getAlign() < Align(4))
       return false;
 
     // We're creating a completely different type of load/store - LDM from LDR.
diff --git a/llvm/lib/Target/ARM/Utils/ARMBaseInfo.h b/llvm/lib/Target/ARM/Utils/ARMBaseInfo.h
index a7578ee172ca9..b2e434fbd78ae 100644
--- a/llvm/lib/Target/ARM/Utils/ARMBaseInfo.h
+++ b/llvm/lib/Target/ARM/Utils/ARMBaseInfo.h
@@ -91,25 +91,35 @@ namespace ARMVCC {
     Then,
     Else
   };
-
-  enum VPTMaskValue {
-    T     =  8, // 0b1000
-    TT    =  4, // 0b0100
-    TE    = 12, // 0b1100
-    TTT   =  2, // 0b0010
-    TTE   =  6, // 0b0110
-    TEE   = 10, // 0b1010
-    TET   = 14, // 0b1110
-    TTTT  =  1, // 0b0001
-    TTTE  =  3, // 0b0011
-    TTEE  =  5, // 0b0101
-    TTET  =  7, // 0b0111
-    TEEE  =  9, // 0b1001
-    TEET  = 11, // 0b1011
-    TETT  = 13, // 0b1101
-    TETE  = 15  // 0b1111
+} // namespace ARMVCC
+
+namespace ARM {
+  /// Mask values for IT and VPT Blocks, to be used by MCOperands.
+  /// Note that this is different from the "real" encoding used by the
+  /// instructions. In this encoding, the lowest set bit indicates the end of
+  /// the encoding, and above that, "1" indicates an else, while "0" indicates
+  /// a then.
+  ///   Tx = x100
+  ///   Txy = xy10
+  ///   Txyz = xyz1
+  enum class PredBlockMask {
+    T = 0b1000,
+    TT = 0b0100,
+    TE = 0b1100,
+    TTT = 0b0010,
+    TTE = 0b0110,
+    TEE = 0b1110,
+    TET = 0b1010,
+    TTTT = 0b0001,
+    TTTE = 0b0011,
+    TTEE = 0b0111,
+    TTET = 0b0101,
+    TEEE = 0b1111,
+    TEET = 0b1101,
+    TETT = 0b1001,
+    TETE = 0b1011
   };
-}
+} // namespace ARM
 
 inline static const char *ARMVPTPredToString(ARMVCC::VPTCodes CC) {
   switch (CC) {
diff --git a/llvm/lib/Target/AVR/AVRFrameLowering.cpp b/llvm/lib/Target/AVR/AVRFrameLowering.cpp
index 119bd21248bf3..7fc15f0af77d6 100644
--- a/llvm/lib/Target/AVR/AVRFrameLowering.cpp
+++ b/llvm/lib/Target/AVR/AVRFrameLowering.cpp
@@ -52,14 +52,14 @@ bool AVRFrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
 void AVRFrameLowering::emitPrologue(MachineFunction &MF,
                                     MachineBasicBlock &MBB) const {
   MachineBasicBlock::iterator MBBI = MBB.begin();
-  CallingConv::ID CallConv = MF.getFunction().getCallingConv();
   DebugLoc DL = (MBBI != MBB.end()) ? MBBI->getDebugLoc() : DebugLoc();
   const AVRSubtarget &STI = MF.getSubtarget<AVRSubtarget>();
   const AVRInstrInfo &TII = *STI.getInstrInfo();
+  const AVRMachineFunctionInfo *AFI = MF.getInfo<AVRMachineFunctionInfo>();
   bool HasFP = hasFP(MF);
 
   // Interrupt handlers re-enable interrupts in function entry.
-  if (CallConv == CallingConv::AVR_INTR) {
+  if (AFI->isInterruptHandler()) {
     BuildMI(MBB, MBBI, DL, TII.get(AVR::BSETs))
         .addImm(0x07)
         .setMIFlag(MachineInstr::FrameSetup);
@@ -74,8 +74,7 @@ void AVRFrameLowering::emitPrologue(MachineFunction &MF,
 
   // Emit special prologue code to save R1, R0 and SREG in interrupt/signal
   // handlers before saving any other registers.
-  if (CallConv == CallingConv::AVR_INTR ||
-      CallConv == CallingConv::AVR_SIGNAL) {
+  if (AFI->isInterruptOrSignalHandler()) {
     BuildMI(MBB, MBBI, DL, TII.get(AVR::PUSHWRr))
         .addReg(AVR::R1R0, RegState::Kill)
         .setMIFlag(MachineInstr::FrameSetup);
@@ -99,7 +98,6 @@ void AVRFrameLowering::emitPrologue(MachineFunction &MF,
   }
 
   const MachineFrameInfo &MFI = MF.getFrameInfo();
-  const AVRMachineFunctionInfo *AFI = MF.getInfo<AVRMachineFunctionInfo>();
   unsigned FrameSize = MFI.getStackSize() - AFI->getCalleeSavedFrameSize();
 
   // Skip the callee-saved push instructions.
@@ -142,13 +140,11 @@ void AVRFrameLowering::emitPrologue(MachineFunction &MF,
 
 void AVRFrameLowering::emitEpilogue(MachineFunction &MF,
                                     MachineBasicBlock &MBB) const {
-  CallingConv::ID CallConv = MF.getFunction().getCallingConv();
-  bool isHandler = (CallConv == CallingConv::AVR_INTR ||
-                    CallConv == CallingConv::AVR_SIGNAL);
+  const AVRMachineFunctionInfo *AFI = MF.getInfo<AVRMachineFunctionInfo>();
 
   // Early exit if the frame pointer is not needed in this function except for
   // signal/interrupt handlers where special code generation is required.
-  if (!hasFP(MF) && !isHandler) {
+  if (!hasFP(MF) && !AFI->isInterruptOrSignalHandler()) {
     return;
   }
 
@@ -158,14 +154,13 @@ void AVRFrameLowering::emitEpilogue(MachineFunction &MF,
 
   DebugLoc DL = MBBI->getDebugLoc();
   const MachineFrameInfo &MFI = MF.getFrameInfo();
-  const AVRMachineFunctionInfo *AFI = MF.getInfo<AVRMachineFunctionInfo>();
   unsigned FrameSize = MFI.getStackSize() - AFI->getCalleeSavedFrameSize();
   const AVRSubtarget &STI = MF.getSubtarget<AVRSubtarget>();
   const AVRInstrInfo &TII = *STI.getInstrInfo();
 
   // Emit special epilogue code to restore R1, R0 and SREG in interrupt/signal
   // handlers at the very end of the function, just before reti.
-  if (isHandler) {
+  if (AFI->isInterruptOrSignalHandler()) {
     BuildMI(MBB, MBBI, DL, TII.get(AVR::POPRd), AVR::R0);
     BuildMI(MBB, MBBI, DL, TII.get(AVR::OUTARr))
         .addImm(0x3f)
diff --git a/llvm/lib/Target/AVR/AVRISelLowering.cpp b/llvm/lib/Target/AVR/AVRISelLowering.cpp
index 1c77b68a9012f..be8f8f75a21e1 100644
--- a/llvm/lib/Target/AVR/AVRISelLowering.cpp
+++ b/llvm/lib/Target/AVR/AVRISelLowering.cpp
@@ -1429,10 +1429,12 @@ AVRTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
     return Chain;
   }
 
+  const AVRMachineFunctionInfo *AFI = MF.getInfo<AVRMachineFunctionInfo>();
+
   unsigned RetOpc =
-      (CallConv == CallingConv::AVR_INTR || CallConv == CallingConv::AVR_SIGNAL)
-          ? AVRISD::RETI_FLAG
-          : AVRISD::RET_FLAG;
+    AFI->isInterruptOrSignalHandler()
+        ? AVRISD::RETI_FLAG
+        : AVRISD::RET_FLAG;
 
   RetOps[0] = Chain; // Update chain.
 
diff --git a/llvm/lib/Target/AVR/AVRInstrInfo.cpp b/llvm/lib/Target/AVR/AVRInstrInfo.cpp
index ba12df991ffc8..ae268b1ddf083 100644
--- a/llvm/lib/Target/AVR/AVRInstrInfo.cpp
+++ b/llvm/lib/Target/AVR/AVRInstrInfo.cpp
@@ -138,7 +138,7 @@ void AVRInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
   MachineMemOperand *MMO = MF.getMachineMemOperand(
       MachinePointerInfo::getFixedStack(MF, FrameIndex),
       MachineMemOperand::MOStore, MFI.getObjectSize(FrameIndex),
-      MFI.getObjectAlignment(FrameIndex));
+      MFI.getObjectAlign(FrameIndex));
 
   unsigned Opcode = 0;
   if (TRI->isTypeLegalForClass(*RC, MVT::i8)) {
@@ -172,7 +172,7 @@ void AVRInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
   MachineMemOperand *MMO = MF.getMachineMemOperand(
       MachinePointerInfo::getFixedStack(MF, FrameIndex),
       MachineMemOperand::MOLoad, MFI.getObjectSize(FrameIndex),
-      MFI.getObjectAlignment(FrameIndex));
+      MFI.getObjectAlign(FrameIndex));
 
   unsigned Opcode = 0;
   if (TRI->isTypeLegalForClass(*RC, MVT::i8)) {
diff --git a/llvm/lib/Target/AVR/AVRMachineFunctionInfo.h b/llvm/lib/Target/AVR/AVRMachineFunctionInfo.h
index 5226e30491c35..5432fac122efb 100644
--- a/llvm/lib/Target/AVR/AVRMachineFunctionInfo.h
+++ b/llvm/lib/Target/AVR/AVRMachineFunctionInfo.h
@@ -31,6 +31,12 @@ class AVRMachineFunctionInfo : public MachineFunctionInfo {
   /// used inside the function.
   bool HasStackArgs;
 
+  /// Whether or not the function is an interrupt handler.
+  bool IsInterruptHandler;
+
+  /// Whether or not the function is an non-blocking interrupt handler.
+  bool IsSignalHandler;
+
   /// Size of the callee-saved register portion of the
   /// stack frame in bytes.
   unsigned CalleeSavedFrameSize;
@@ -41,11 +47,17 @@ class AVRMachineFunctionInfo : public MachineFunctionInfo {
 public:
   AVRMachineFunctionInfo()
       : HasSpills(false), HasAllocas(false), HasStackArgs(false),
+        IsInterruptHandler(false), IsSignalHandler(false),
         CalleeSavedFrameSize(0), VarArgsFrameIndex(0) {}
 
   explicit AVRMachineFunctionInfo(MachineFunction &MF)
       : HasSpills(false), HasAllocas(false), HasStackArgs(false),
-        CalleeSavedFrameSize(0), VarArgsFrameIndex(0) {}
+        CalleeSavedFrameSize(0), VarArgsFrameIndex(0) {
+    unsigned CallConv = MF.getFunction().getCallingConv();
+
+    this->IsInterruptHandler = CallConv == CallingConv::AVR_INTR || MF.getFunction().hasFnAttribute("interrupt");
+    this->IsSignalHandler = CallConv == CallingConv::AVR_SIGNAL || MF.getFunction().hasFnAttribute("signal");
+  }
 
   bool getHasSpills() const { return HasSpills; }
   void setHasSpills(bool B) { HasSpills = B; }
@@ -56,6 +68,12 @@ class AVRMachineFunctionInfo : public MachineFunctionInfo {
   bool getHasStackArgs() const { return HasStackArgs; }
   void setHasStackArgs(bool B) { HasStackArgs = B; }
 
+  /// Checks if the function is some form of interrupt service routine.
+  bool isInterruptOrSignalHandler() const { return isInterruptHandler() || isSignalHandler(); }
+
+  bool isInterruptHandler() const { return IsInterruptHandler; }
+  bool isSignalHandler() const { return IsSignalHandler; }
+
   unsigned getCalleeSavedFrameSize() const { return CalleeSavedFrameSize; }
   void setCalleeSavedFrameSize(unsigned Bytes) { CalleeSavedFrameSize = Bytes; }
 
diff --git a/llvm/lib/Target/AVR/AVRRegisterInfo.cpp b/llvm/lib/Target/AVR/AVRRegisterInfo.cpp
index 601510da56e83..2a4905ce2461e 100644
--- a/llvm/lib/Target/AVR/AVRRegisterInfo.cpp
+++ b/llvm/lib/Target/AVR/AVRRegisterInfo.cpp
@@ -22,6 +22,7 @@
 
 #include "AVR.h"
 #include "AVRInstrInfo.h"
+#include "AVRMachineFunctionInfo.h"
 #include "AVRTargetMachine.h"
 #include "MCTargetDesc/AVRMCTargetDesc.h"
 
@@ -34,19 +35,21 @@ AVRRegisterInfo::AVRRegisterInfo() : AVRGenRegisterInfo(0) {}
 
 const uint16_t *
 AVRRegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
-  CallingConv::ID CC = MF->getFunction().getCallingConv();
+  const AVRMachineFunctionInfo *AFI = MF->getInfo<AVRMachineFunctionInfo>();
 
-  return ((CC == CallingConv::AVR_INTR || CC == CallingConv::AVR_SIGNAL)
+  return AFI->isInterruptOrSignalHandler()
               ? CSR_Interrupts_SaveList
-              : CSR_Normal_SaveList);
+              : CSR_Normal_SaveList;
 }
 
 const uint32_t *
 AVRRegisterInfo::getCallPreservedMask(const MachineFunction &MF,
                                       CallingConv::ID CC) const {
-  return ((CC == CallingConv::AVR_INTR || CC == CallingConv::AVR_SIGNAL)
+  const AVRMachineFunctionInfo *AFI = MF.getInfo<AVRMachineFunctionInfo>();
+
+  return AFI->isInterruptOrSignalHandler()
               ? CSR_Interrupts_RegMask
-              : CSR_Normal_RegMask);
+              : CSR_Normal_RegMask;
 }
 
 BitVector AVRRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRInstPrinter.cpp b/llvm/lib/Target/AVR/MCTargetDesc/AVRInstPrinter.cpp
index 79e98425c0c90..3283f74e8a204 100644
--- a/llvm/lib/Target/AVR/MCTargetDesc/AVRInstPrinter.cpp
+++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRInstPrinter.cpp
@@ -78,7 +78,7 @@ void AVRInstPrinter::printInst(const MCInst *MI, uint64_t Address,
     printOperand(MI, 2, O);
     break;
   default:
-    if (!printAliasInstr(MI, O))
+    if (!printAliasInstr(MI, Address, O))
       printInstruction(MI, Address, O);
 
     printAnnotation(O, Annot);
diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRInstPrinter.h b/llvm/lib/Target/AVR/MCTargetDesc/AVRInstPrinter.h
index 247e9fc839897..910fd3455dee3 100644
--- a/llvm/lib/Target/AVR/MCTargetDesc/AVRInstPrinter.h
+++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRInstPrinter.h
@@ -38,13 +38,18 @@ class AVRInstPrinter : public MCInstPrinter {
 
   void printOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O);
   void printPCRelImm(const MCInst *MI, unsigned OpNo, raw_ostream &O);
+  void printPCRelImm(const MCInst *MI, uint64_t /*Address*/, unsigned OpNo,
+                     raw_ostream &O) {
+    printPCRelImm(MI, OpNo, O);
+  }
   void printMemri(const MCInst *MI, unsigned OpNo, raw_ostream &O);
 
   // Autogenerated by TableGen.
   void printInstruction(const MCInst *MI, uint64_t Address, raw_ostream &O);
-  bool printAliasInstr(const MCInst *MI, raw_ostream &O);
-  void printCustomAliasOperand(const MCInst *MI, unsigned OpIdx,
-                               unsigned PrintMethodIdx, raw_ostream &O);
+  bool printAliasInstr(const MCInst *MI, uint64_t Address, raw_ostream &O);
+  void printCustomAliasOperand(const MCInst *MI, uint64_t Address,
+                               unsigned OpIdx, unsigned PrintMethodIdx,
+                               raw_ostream &O);
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/Target/BPF/BPFTargetMachine.cpp b/llvm/lib/Target/BPF/BPFTargetMachine.cpp
index d2a332c78aa66..4d42e74140751 100644
--- a/llvm/lib/Target/BPF/BPFTargetMachine.cpp
+++ b/llvm/lib/Target/BPF/BPFTargetMachine.cpp
@@ -42,9 +42,9 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeBPFTarget() {
 // DataLayout: little or big endian
 static std::string computeDataLayout(const Triple &TT) {
   if (TT.getArch() == Triple::bpfeb)
-    return "E-m:e-p:64:64-i64:64-n32:64-S128";
+    return "E-m:e-p:64:64-i64:64-i128:128-n32:64-S128";
   else
-    return "e-m:e-p:64:64-i64:64-n32:64-S128";
+    return "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128";
 }
 
 static Reloc::Model getEffectiveRelocModel(Optional<Reloc::Model> RM) {
diff --git a/llvm/lib/Target/Hexagon/HexagonBitTracker.cpp b/llvm/lib/Target/Hexagon/HexagonBitTracker.cpp
index ebd060ce503ef..1e4030b84bc18 100644
--- a/llvm/lib/Target/Hexagon/HexagonBitTracker.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonBitTracker.cpp
@@ -330,7 +330,7 @@ bool HexagonEvaluator::evaluate(const MachineInstr &MI,
     case PS_fi: {
       int FI = op(1).getIndex();
       int Off = op(2).getImm();
-      unsigned A = MFI.getObjectAlignment(FI) + std::abs(Off);
+      unsigned A = MFI.getObjectAlign(FI).value() + std::abs(Off);
       unsigned L = countTrailingZeros(A);
       RegisterCell RC = RegisterCell::self(Reg[0].Reg, W0);
       RC.fill(0, L, BT::BitValue::Zero);
diff --git a/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp b/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp
index 1601d5600cc29..5979b635d5361 100644
--- a/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp
@@ -645,15 +645,15 @@ void HexagonFrameLowering::insertPrologueInBlock(MachineBasicBlock &MBB,
       auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>();
       for (int i = HMFI.getFirstNamedArgFrameIndex(),
                e = HMFI.getLastNamedArgFrameIndex(); i >= e; --i) {
-        int ObjSize = MFI.getObjectSize(i);
-        int ObjAlign = MFI.getObjectAlignment(i);
+        uint64_t ObjSize = MFI.getObjectSize(i);
+        Align ObjAlign = MFI.getObjectAlign(i);
 
         // Determine the kind of load/store that should be used.
         unsigned LDOpc, STOpc;
-        int OpcodeChecker = ObjAlign;
+        uint64_t OpcodeChecker = ObjAlign.value();
 
         // Handle cases where alignment of an object is > its size.
-        if (ObjSize < ObjAlign) {
+        if (ObjAlign > ObjSize) {
           if (ObjSize <= 1)
             OpcodeChecker = 1;
           else if (ObjSize <= 2)
@@ -702,17 +702,17 @@ void HexagonFrameLowering::insertPrologueInBlock(MachineBasicBlock &MBB,
         while (Count < LoadStoreCount) {
           // Load the value of the named argument on stack.
           BuildMI(MBB, InsertPt, dl, HII.get(LDOpc), RegUsed)
-            .addReg(SP)
-            .addImm(RegisterSavedAreaSizePlusPadding +
-                    ObjAlign * Count + NumBytes)
-            .setMIFlag(MachineInstr::FrameSetup);
+              .addReg(SP)
+              .addImm(RegisterSavedAreaSizePlusPadding +
+                      ObjAlign.value() * Count + NumBytes)
+              .setMIFlag(MachineInstr::FrameSetup);
 
           // Store it below the register saved area plus padding.
           BuildMI(MBB, InsertPt, dl, HII.get(STOpc))
-            .addReg(SP)
-            .addImm(ObjAlign * Count + NumBytes)
-            .addReg(RegUsed)
-            .setMIFlag(MachineInstr::FrameSetup);
+              .addReg(SP)
+              .addImm(ObjAlign.value() * Count + NumBytes)
+              .addReg(RegUsed)
+              .setMIFlag(MachineInstr::FrameSetup);
 
           Count++;
         }
@@ -893,7 +893,7 @@ void HexagonFrameLowering::insertAllocframe(MachineBasicBlock &MBB,
   // Create a dummy memory operand to avoid allocframe from being treated as
   // a volatile memory reference.
   auto *MMO = MF.getMachineMemOperand(MachinePointerInfo::getStack(MF, 0),
-                                      MachineMemOperand::MOStore, 4, 4);
+                                      MachineMemOperand::MOStore, 4, Align(4));
 
   DebugLoc dl = MBB.findDebugLoc(InsertPt);
   unsigned SP = HRI.getStackRegister();
@@ -1520,8 +1520,8 @@ void HexagonFrameLowering::processFunctionBeforeFrameFinalized(
     unsigned S = MFI.getObjectSize(i);
     // Reduce the alignment to at most 8. This will require unaligned vector
     // stores if they happen here.
-    unsigned A = std::max(MFI.getObjectAlignment(i), 8U);
-    MFI.setObjectAlignment(i, 8);
+    Align A = std::max(MFI.getObjectAlign(i), Align(8));
+    MFI.setObjectAlignment(i, Align(8));
     LFS = alignTo(LFS+S, A);
     MFI.mapLocalFrameObject(i, -static_cast<int64_t>(LFS));
     DealignSlots.insert(i);
@@ -1547,12 +1547,11 @@ void HexagonFrameLowering::processFunctionBeforeFrameFinalized(
           if (auto *FS = dyn_cast_or_null<FixedStackPseudoSourceValue>(PV)) {
             int FI = FS->getFrameIndex();
             if (DealignSlots.count(FI)) {
-              unsigned A = MFI.getObjectAlignment(FI);
-              auto *NewMMO = MF.getMachineMemOperand(MMO->getPointerInfo(),
-                                MMO->getFlags(), MMO->getSize(), A,
-                                MMO->getAAInfo(), MMO->getRanges(),
-                                MMO->getSyncScopeID(), MMO->getOrdering(),
-                                MMO->getFailureOrdering());
+              auto *NewMMO = MF.getMachineMemOperand(
+                  MMO->getPointerInfo(), MMO->getFlags(), MMO->getSize(),
+                  MFI.getObjectAlign(FI), MMO->getAAInfo(), MMO->getRanges(),
+                  MMO->getSyncScopeID(), MMO->getOrdering(),
+                  MMO->getFailureOrdering());
               new_memops.push_back(NewMMO);
               KeepOld = false;
               continue;
@@ -1711,9 +1710,8 @@ bool HexagonFrameLowering::assignCalleeSavedSpillSlots(MachineFunction &MF,
     const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(R);
     unsigned Size = TRI->getSpillSize(*RC);
     int Off = MinOffset - Size;
-    unsigned Align = std::min(TRI->getSpillAlignment(*RC), getStackAlignment());
-    assert(isPowerOf2_32(Align));
-    Off &= -Align;
+    Align Alignment = std::min(TRI->getSpillAlign(*RC), getStackAlign());
+    Off &= -Alignment.value();
     int FI = MFI.CreateFixedSpillStackObject(Size, Off);
     MinOffset = std::min(MinOffset, Off);
     CSI.push_back(CalleeSavedInfo(R, FI));
@@ -1936,11 +1934,11 @@ bool HexagonFrameLowering::expandStoreVec2(MachineBasicBlock &B,
   bool NeedsAligna = needsAligna(MF);
 
   unsigned Size = HRI.getSpillSize(Hexagon::HvxVRRegClass);
-  unsigned NeedAlign = HRI.getSpillAlignment(Hexagon::HvxVRRegClass);
-  unsigned HasAlign = MFI.getObjectAlignment(FI);
+  Align NeedAlign = HRI.getSpillAlign(Hexagon::HvxVRRegClass);
+  Align HasAlign = MFI.getObjectAlign(FI);
   unsigned StoreOpc;
 
-  auto UseAligned = [&] (unsigned NeedAlign, unsigned HasAlign) {
+  auto UseAligned = [&](Align NeedAlign, Align HasAlign) {
     return !NeedsAligna && (NeedAlign <= HasAlign);
   };
 
@@ -1988,11 +1986,11 @@ bool HexagonFrameLowering::expandLoadVec2(MachineBasicBlock &B,
   bool NeedsAligna = needsAligna(MF);
 
   unsigned Size = HRI.getSpillSize(Hexagon::HvxVRRegClass);
-  unsigned NeedAlign = HRI.getSpillAlignment(Hexagon::HvxVRRegClass);
-  unsigned HasAlign = MFI.getObjectAlignment(FI);
+  Align NeedAlign = HRI.getSpillAlign(Hexagon::HvxVRRegClass);
+  Align HasAlign = MFI.getObjectAlign(FI);
   unsigned LoadOpc;
 
-  auto UseAligned = [&] (unsigned NeedAlign, unsigned HasAlign) {
+  auto UseAligned = [&](Align NeedAlign, Align HasAlign) {
     return !NeedsAligna && (NeedAlign <= HasAlign);
   };
 
@@ -2032,8 +2030,8 @@ bool HexagonFrameLowering::expandStoreVec(MachineBasicBlock &B,
   bool IsKill = MI->getOperand(2).isKill();
   int FI = MI->getOperand(0).getIndex();
 
-  unsigned NeedAlign = HRI.getSpillAlignment(Hexagon::HvxVRRegClass);
-  unsigned HasAlign = MFI.getObjectAlignment(FI);
+  Align NeedAlign = HRI.getSpillAlign(Hexagon::HvxVRRegClass);
+  Align HasAlign = MFI.getObjectAlign(FI);
   bool UseAligned = !NeedsAligna && (NeedAlign <= HasAlign);
   unsigned StoreOpc = UseAligned ? Hexagon::V6_vS32b_ai
                                  : Hexagon::V6_vS32Ub_ai;
@@ -2062,8 +2060,8 @@ bool HexagonFrameLowering::expandLoadVec(MachineBasicBlock &B,
   Register DstR = MI->getOperand(0).getReg();
   int FI = MI->getOperand(1).getIndex();
 
-  unsigned NeedAlign = HRI.getSpillAlignment(Hexagon::HvxVRRegClass);
-  unsigned HasAlign = MFI.getObjectAlignment(FI);
+  Align NeedAlign = HRI.getSpillAlign(Hexagon::HvxVRRegClass);
+  Align HasAlign = MFI.getObjectAlign(FI);
   bool UseAligned = !NeedsAligna && (NeedAlign <= HasAlign);
   unsigned LoadOpc = UseAligned ? Hexagon::V6_vL32b_ai
                                 : Hexagon::V6_vL32Ub_ai;
@@ -2165,7 +2163,8 @@ void HexagonFrameLowering::determineCalleeSaves(MachineFunction &MF,
           Num = 2; // Vector predicate spills also need a vector register.
           break;
       }
-      unsigned S = HRI.getSpillSize(*RC), A = HRI.getSpillAlignment(*RC);
+      unsigned S = HRI.getSpillSize(*RC);
+      Align A = HRI.getSpillAlign(*RC);
       for (unsigned i = 0; i < Num; i++) {
         int NewFI = MFI.CreateSpillStackObject(S, A);
         RS->addScavengingFrameIndex(NewFI);
diff --git a/llvm/lib/Target/Hexagon/HexagonHardwareLoops.cpp b/llvm/lib/Target/Hexagon/HexagonHardwareLoops.cpp
index 2da2511648e87..4833935f8d24b 100644
--- a/llvm/lib/Target/Hexagon/HexagonHardwareLoops.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonHardwareLoops.cpp
@@ -467,7 +467,7 @@ bool HexagonHardwareLoops::findInductionRegister(MachineLoop *L,
   if (!PredI->isCompare())
     return false;
 
-  unsigned CmpReg1 = 0, CmpReg2 = 0;
+  Register CmpReg1, CmpReg2;
   int CmpImm = 0, CmpMask = 0;
   bool CmpAnalyzed =
       TII->analyzeCompare(*PredI, CmpReg1, CmpReg2, CmpMask, CmpImm);
@@ -651,7 +651,7 @@ CountValue *HexagonHardwareLoops::getLoopTripCount(MachineLoop *L,
   MachineInstr *CondI = MRI->getVRegDef(PredReg);
   unsigned CondOpc = CondI->getOpcode();
 
-  unsigned CmpReg1 = 0, CmpReg2 = 0;
+  Register CmpReg1, CmpReg2;
   int Mask = 0, ImmValue = 0;
   bool AnalyzedCmp =
       TII->analyzeCompare(*CondI, CmpReg1, CmpReg2, Mask, ImmValue);
@@ -1455,7 +1455,7 @@ bool HexagonHardwareLoops::loopCountMayWrapOrUnderFlow(
   for (MachineRegisterInfo::use_instr_nodbg_iterator I = MRI->use_instr_nodbg_begin(Reg),
          E = MRI->use_instr_nodbg_end(); I != E; ++I) {
     MachineInstr *MI = &*I;
-    unsigned CmpReg1 = 0, CmpReg2 = 0;
+    Register CmpReg1, CmpReg2;
     int CmpMask = 0, CmpValue = 0;
 
     if (!TII->analyzeCompare(*MI, CmpReg1, CmpReg2, CmpMask, CmpValue))
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index 5248dc807368a..270e59281f971 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -729,7 +729,7 @@ HexagonTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
   auto &HFI = *Subtarget.getFrameLowering();
   // "Zero" means natural stack alignment.
   if (A == 0)
-    A = HFI.getStackAlignment();
+    A = HFI.getStackAlign().value();
 
   LLVM_DEBUG({
     dbgs () << __func__ << " Align: " << A << " Size: ";
@@ -1087,7 +1087,7 @@ HexagonTargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const {
   Constant *CVal = nullptr;
   bool isVTi1Type = false;
   if (auto *CV = dyn_cast<ConstantVector>(CPN->getConstVal())) {
-    if (CV->getType()->getVectorElementType()->isIntegerTy(1)) {
+    if (cast<VectorType>(CV->getType())->getElementType()->isIntegerTy(1)) {
       IRBuilder<> IRB(CV->getContext());
       SmallVector<Constant*, 128> NewConst;
       unsigned VecLen = CV->getNumOperands();
@@ -2909,10 +2909,10 @@ HexagonTargetLowering::LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG)
   MachineMemOperand *WideMMO = nullptr;
   if (MachineMemOperand *MMO = LN->getMemOperand()) {
     MachineFunction &MF = DAG.getMachineFunction();
-    WideMMO = MF.getMachineMemOperand(MMO->getPointerInfo(), MMO->getFlags(),
-                    2*LoadLen, LoadLen, MMO->getAAInfo(), MMO->getRanges(),
-                    MMO->getSyncScopeID(), MMO->getOrdering(),
-                    MMO->getFailureOrdering());
+    WideMMO = MF.getMachineMemOperand(
+        MMO->getPointerInfo(), MMO->getFlags(), 2 * LoadLen, Align(LoadLen),
+        MMO->getAAInfo(), MMO->getRanges(), MMO->getSyncScopeID(),
+        MMO->getOrdering(), MMO->getFailureOrdering());
   }
 
   SDValue Load0 = DAG.getLoad(LoadTy, dl, Chain, Base0, WideMMO);
diff --git a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
index 17834ba241940..20e3a48d8c862 100644
--- a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
@@ -916,12 +916,11 @@ void HexagonInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
   DebugLoc DL = MBB.findDebugLoc(I);
   MachineFunction &MF = *MBB.getParent();
   MachineFrameInfo &MFI = MF.getFrameInfo();
-  unsigned SlotAlign = MFI.getObjectAlignment(FI);
   unsigned KillFlag = getKillRegState(isKill);
 
   MachineMemOperand *MMO = MF.getMachineMemOperand(
       MachinePointerInfo::getFixedStack(MF, FI), MachineMemOperand::MOStore,
-      MFI.getObjectSize(FI), SlotAlign);
+      MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
 
   if (Hexagon::IntRegsRegClass.hasSubClassEq(RC)) {
     BuildMI(MBB, I, DL, get(Hexagon::S2_storeri_io))
@@ -963,11 +962,10 @@ void HexagonInstrInfo::loadRegFromStackSlot(
   DebugLoc DL = MBB.findDebugLoc(I);
   MachineFunction &MF = *MBB.getParent();
   MachineFrameInfo &MFI = MF.getFrameInfo();
-  unsigned SlotAlign = MFI.getObjectAlignment(FI);
 
   MachineMemOperand *MMO = MF.getMachineMemOperand(
       MachinePointerInfo::getFixedStack(MF, FI), MachineMemOperand::MOLoad,
-      MFI.getObjectSize(FI), SlotAlign);
+      MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
 
   if (Hexagon::IntRegsRegClass.hasSubClassEq(RC)) {
     BuildMI(MBB, I, DL, get(Hexagon::L2_loadri_io), DestReg)
@@ -1027,10 +1025,9 @@ bool HexagonInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
   auto UseAligned = [&] (const MachineInstr &MI, unsigned NeedAlign) {
     if (MI.memoperands().empty())
       return false;
-    return all_of(MI.memoperands(),
-                  [NeedAlign] (const MachineMemOperand *MMO) {
-                    return NeedAlign <= MMO->getAlignment();
-                  });
+    return all_of(MI.memoperands(), [NeedAlign](const MachineMemOperand *MMO) {
+      return MMO->getAlign() >= NeedAlign;
+    });
   };
 
   switch (Opc) {
@@ -1374,7 +1371,8 @@ bool HexagonInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
       static const CrashPseudoSourceValue CrashPSV(*this);
       MachineMemOperand *MMO = MF.getMachineMemOperand(
           MachinePointerInfo(&CrashPSV),
-          MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile, 8, 1);
+          MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile, 8,
+          Align(1));
       BuildMI(MBB, MI, DL, get(Hexagon::PS_loadrdabs), Hexagon::D13)
         .addImm(0xBADC0FEE)  // Misaligned load.
         .addMemOperand(MMO);
@@ -1787,8 +1785,8 @@ HexagonInstrInfo::CreateTargetPostRAHazardRecognizer(
 /// \p SrcReg and \p SrcReg2 if having two register operands, and the value it
 /// compares against in CmpValue. Return true if the comparison instruction
 /// can be analyzed.
-bool HexagonInstrInfo::analyzeCompare(const MachineInstr &MI, unsigned &SrcReg,
-                                      unsigned &SrcReg2, int &Mask,
+bool HexagonInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
+                                      Register &SrcReg2, int &Mask,
                                       int &Value) const {
   unsigned Opc = MI.getOpcode();
 
diff --git a/llvm/lib/Target/Hexagon/HexagonInstrInfo.h b/llvm/lib/Target/Hexagon/HexagonInstrInfo.h
index 07a73cddf377a..d8998d265477d 100644
--- a/llvm/lib/Target/Hexagon/HexagonInstrInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonInstrInfo.h
@@ -269,8 +269,8 @@ class HexagonInstrInfo : public HexagonGenInstrInfo {
   /// in SrcReg and SrcReg2 if having two register operands, and the value it
   /// compares against in CmpValue. Return true if the comparison instruction
   /// can be analyzed.
-  bool analyzeCompare(const MachineInstr &MI, unsigned &SrcReg,
-                      unsigned &SrcReg2, int &Mask, int &Value) const override;
+  bool analyzeCompare(const MachineInstr &MI, Register &SrcReg,
+                      Register &SrcReg2, int &Mask, int &Value) const override;
 
   /// Compute the instruction latency of a given instruction.
   /// If the instruction has higher cost when predicated, it's returned via
diff --git a/llvm/lib/Target/Hexagon/HexagonSplitDouble.cpp b/llvm/lib/Target/Hexagon/HexagonSplitDouble.cpp
index d80e0ed50c931..b45d871e04d6e 100644
--- a/llvm/lib/Target/Hexagon/HexagonSplitDouble.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonSplitDouble.cpp
@@ -504,7 +504,7 @@ void HexagonSplitDoubleRegs::collectIndRegsForLoop(const MachineLoop *L,
 
   // Get the registers on which the loop controlling compare instruction
   // depends.
-  unsigned CmpR1 = 0, CmpR2 = 0;
+  Register CmpR1, CmpR2;
   const MachineInstr *CmpI = MRI->getVRegDef(PR);
   while (CmpI->getOpcode() == Hexagon::C2_not)
     CmpI = MRI->getVRegDef(CmpI->getOperand(1).getReg());
@@ -688,11 +688,12 @@ void HexagonSplitDoubleRegs::splitMemRef(MachineInstr *MI,
   for (auto &MO : MI->memoperands()) {
     const MachinePointerInfo &Ptr = MO->getPointerInfo();
     MachineMemOperand::Flags F = MO->getFlags();
-    int A = MO->getAlignment();
+    Align A = MO->getAlign();
 
-    auto *Tmp1 = MF.getMachineMemOperand(Ptr, F, 4/*size*/, A);
+    auto *Tmp1 = MF.getMachineMemOperand(Ptr, F, 4 /*size*/, A);
     LowI->addMemOperand(MF, Tmp1);
-    auto *Tmp2 = MF.getMachineMemOperand(Ptr, F, 4/*size*/, std::min(A, 4));
+    auto *Tmp2 =
+        MF.getMachineMemOperand(Ptr, F, 4 /*size*/, std::min(A, Align(4)));
     HighI->addMemOperand(MF, Tmp2);
   }
 }
diff --git a/llvm/lib/Target/Hexagon/HexagonStoreWidening.cpp b/llvm/lib/Target/Hexagon/HexagonStoreWidening.cpp
index aab37393ed365..2c4007145bd0d 100644
--- a/llvm/lib/Target/Hexagon/HexagonStoreWidening.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonStoreWidening.cpp
@@ -314,7 +314,7 @@ bool HexagonStoreWidening::selectStores(InstrGroup::iterator Begin,
   MachineInstr *FirstMI = *Begin;
   assert(!FirstMI->memoperands_empty() && "Expecting some memory operands");
   const MachineMemOperand &FirstMMO = getStoreTarget(FirstMI);
-  unsigned Alignment = FirstMMO.getAlignment();
+  unsigned Alignment = FirstMMO.getAlign().value();
   unsigned SizeAccum = FirstMMO.getSize();
   unsigned FirstOffset = getStoreOffset(FirstMI);
 
@@ -417,9 +417,8 @@ bool HexagonStoreWidening::createWideStores(InstrGroup &OG, InstrGroup &NG,
   DebugLoc DL = OG.back()->getDebugLoc();
   const MachineMemOperand &OldM = getStoreTarget(FirstSt);
   MachineMemOperand *NewM =
-    MF->getMachineMemOperand(OldM.getPointerInfo(), OldM.getFlags(),
-                             TotalSize, OldM.getAlignment(),
-                             OldM.getAAInfo());
+      MF->getMachineMemOperand(OldM.getPointerInfo(), OldM.getFlags(),
+                               TotalSize, OldM.getAlign(), OldM.getAAInfo());
 
   if (Acc < 0x10000) {
     // Create mem[hw] = #Acc
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
index 7f6058a78b613..4485b70c6eb5e 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
@@ -60,8 +60,8 @@ bool HexagonTTIImpl::isTypeForHVX(Type *VecTy) const {
 }
 
 unsigned HexagonTTIImpl::getTypeNumElements(Type *Ty) const {
-  if (Ty->isVectorTy())
-    return Ty->getVectorNumElements();
+  if (auto *VTy = dyn_cast<VectorType>(Ty))
+    return VTy->getNumElements();
   assert((Ty->isIntegerTy() || Ty->isFloatingPointTy()) &&
          "Expecting scalar type");
   return 1;
diff --git a/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp b/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp
index aee05eac6a64b..8afd27d53469c 100644
--- a/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp
+++ b/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp
@@ -174,8 +174,8 @@ LanaiInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
   return makeArrayRef(TargetFlags);
 }
 
-bool LanaiInstrInfo::analyzeCompare(const MachineInstr &MI, unsigned &SrcReg,
-                                    unsigned &SrcReg2, int &CmpMask,
+bool LanaiInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
+                                    Register &SrcReg2, int &CmpMask,
                                     int &CmpValue) const {
   switch (MI.getOpcode()) {
   default:
@@ -183,7 +183,7 @@ bool LanaiInstrInfo::analyzeCompare(const MachineInstr &MI, unsigned &SrcReg,
   case Lanai::SFSUB_F_RI_LO:
   case Lanai::SFSUB_F_RI_HI:
     SrcReg = MI.getOperand(0).getReg();
-    SrcReg2 = 0;
+    SrcReg2 = Register();
     CmpMask = ~0;
     CmpValue = MI.getOperand(1).getImm();
     return true;
@@ -281,7 +281,7 @@ inline static unsigned flagSettingOpcodeVariant(unsigned OldOpcode) {
 }
 
 bool LanaiInstrInfo::optimizeCompareInstr(
-    MachineInstr &CmpInstr, unsigned SrcReg, unsigned SrcReg2, int /*CmpMask*/,
+    MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int /*CmpMask*/,
     int CmpValue, const MachineRegisterInfo *MRI) const {
   // Get the unique definition of SrcReg.
   MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg);
@@ -454,9 +454,9 @@ bool LanaiInstrInfo::analyzeSelect(const MachineInstr &MI,
 
 // Identify instructions that can be folded into a SELECT instruction, and
 // return the defining instruction.
-static MachineInstr *canFoldIntoSelect(unsigned Reg,
+static MachineInstr *canFoldIntoSelect(Register Reg,
                                        const MachineRegisterInfo &MRI) {
-  if (!Register::isVirtualRegister(Reg))
+  if (!Reg.isVirtual())
     return nullptr;
   if (!MRI.hasOneNonDBGUse(Reg))
     return nullptr;
diff --git a/llvm/lib/Target/Lanai/LanaiInstrInfo.h b/llvm/lib/Target/Lanai/LanaiInstrInfo.h
index b0b4ad642b40e..216dbfb63accc 100644
--- a/llvm/lib/Target/Lanai/LanaiInstrInfo.h
+++ b/llvm/lib/Target/Lanai/LanaiInstrInfo.h
@@ -95,15 +95,15 @@ class LanaiInstrInfo : public LanaiGenInstrInfo {
   // For a comparison instruction, return the source registers in SrcReg and
   // SrcReg2 if having two register operands, and the value it compares against
   // in CmpValue. Return true if the comparison instruction can be analyzed.
-  bool analyzeCompare(const MachineInstr &MI, unsigned &SrcReg,
-                      unsigned &SrcReg2, int &CmpMask,
+  bool analyzeCompare(const MachineInstr &MI, Register &SrcReg,
+                      Register &SrcReg2, int &CmpMask,
                       int &CmpValue) const override;
 
   // See if the comparison instruction can be converted into something more
   // efficient. E.g., on Lanai register-register instructions can set the flag
   // register, obviating the need for a separate compare.
-  bool optimizeCompareInstr(MachineInstr &CmpInstr, unsigned SrcReg,
-                            unsigned SrcReg2, int CmpMask, int CmpValue,
+  bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
+                            Register SrcReg2, int CmpMask, int CmpValue,
                             const MachineRegisterInfo *MRI) const override;
 
   // Analyze the given select instruction, returning true if it cannot be
diff --git a/llvm/lib/Target/Lanai/MCTargetDesc/LanaiInstPrinter.cpp b/llvm/lib/Target/Lanai/MCTargetDesc/LanaiInstPrinter.cpp
index ccc4139959175..7027d18126bb1 100644
--- a/llvm/lib/Target/Lanai/MCTargetDesc/LanaiInstPrinter.cpp
+++ b/llvm/lib/Target/Lanai/MCTargetDesc/LanaiInstPrinter.cpp
@@ -141,7 +141,7 @@ void LanaiInstPrinter::printInst(const MCInst *MI, uint64_t Address,
                                  StringRef Annotation,
                                  const MCSubtargetInfo & /*STI*/,
                                  raw_ostream &OS) {
-  if (!printAlias(MI, OS) && !printAliasInstr(MI, OS))
+  if (!printAlias(MI, OS) && !printAliasInstr(MI, Address, OS))
     printInstruction(MI, Address, OS);
   printAnnotation(OS, Annotation);
 }
diff --git a/llvm/lib/Target/Lanai/MCTargetDesc/LanaiInstPrinter.h b/llvm/lib/Target/Lanai/MCTargetDesc/LanaiInstPrinter.h
index a71a9497c691d..ce6df2969d734 100644
--- a/llvm/lib/Target/Lanai/MCTargetDesc/LanaiInstPrinter.h
+++ b/llvm/lib/Target/Lanai/MCTargetDesc/LanaiInstPrinter.h
@@ -44,9 +44,10 @@ class LanaiInstPrinter : public MCInstPrinter {
 
   // Autogenerated by tblgen.
   void printInstruction(const MCInst *MI, uint64_t Address, raw_ostream &O);
-  bool printAliasInstr(const MCInst *MI, raw_ostream &OS);
-  void printCustomAliasOperand(const MCInst *MI, unsigned OpIdx,
-                               unsigned PrintMethodIdx, raw_ostream &O);
+  bool printAliasInstr(const MCInst *MI, uint64_t Address, raw_ostream &OS);
+  void printCustomAliasOperand(const MCInst *MI, uint64_t Address,
+                               unsigned OpIdx, unsigned PrintMethodIdx,
+                               raw_ostream &O);
   static const char *getRegisterName(unsigned RegNo);
   void printRegName(raw_ostream &OS, unsigned RegNo) const override;
 
diff --git a/llvm/lib/Target/MSP430/MCTargetDesc/MSP430InstPrinter.cpp b/llvm/lib/Target/MSP430/MCTargetDesc/MSP430InstPrinter.cpp
index 0c6da5a35c685..420893f65d5b9 100644
--- a/llvm/lib/Target/MSP430/MCTargetDesc/MSP430InstPrinter.cpp
+++ b/llvm/lib/Target/MSP430/MCTargetDesc/MSP430InstPrinter.cpp
@@ -29,7 +29,7 @@ using namespace llvm;
 void MSP430InstPrinter::printInst(const MCInst *MI, uint64_t Address,
                                   StringRef Annot, const MCSubtargetInfo &STI,
                                   raw_ostream &O) {
-  if (!printAliasInstr(MI, O))
+  if (!printAliasInstr(MI, Address, O))
     printInstruction(MI, Address, O);
   printAnnotation(O, Annot);
 }
diff --git a/llvm/lib/Target/MSP430/MCTargetDesc/MSP430InstPrinter.h b/llvm/lib/Target/MSP430/MCTargetDesc/MSP430InstPrinter.h
index 200dc0e6db601..6a6b07f2eba01 100644
--- a/llvm/lib/Target/MSP430/MCTargetDesc/MSP430InstPrinter.h
+++ b/llvm/lib/Target/MSP430/MCTargetDesc/MSP430InstPrinter.h
@@ -27,9 +27,10 @@ namespace llvm {
 
     // Autogenerated by tblgen.
     void printInstruction(const MCInst *MI, uint64_t Address, raw_ostream &O);
-    bool printAliasInstr(const MCInst *MI, raw_ostream &O);
-    void printCustomAliasOperand(const MCInst *MI, unsigned OpIdx,
-                                 unsigned PrintMethodIdx, raw_ostream &O);
+    bool printAliasInstr(const MCInst *MI, uint64_t Address, raw_ostream &O);
+    void printCustomAliasOperand(const MCInst *MI, uint64_t Address,
+                                 unsigned OpIdx, unsigned PrintMethodIdx,
+                                 raw_ostream &O);
     static const char *getRegisterName(unsigned RegNo);
 
 private:
diff --git a/llvm/lib/Target/MSP430/MSP430InstrInfo.cpp b/llvm/lib/Target/MSP430/MSP430InstrInfo.cpp
index e8e573a2dfe2c..a07ec4eb2e2db 100644
--- a/llvm/lib/Target/MSP430/MSP430InstrInfo.cpp
+++ b/llvm/lib/Target/MSP430/MSP430InstrInfo.cpp
@@ -46,7 +46,7 @@ void MSP430InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
   MachineMemOperand *MMO = MF.getMachineMemOperand(
       MachinePointerInfo::getFixedStack(MF, FrameIdx),
       MachineMemOperand::MOStore, MFI.getObjectSize(FrameIdx),
-      MFI.getObjectAlignment(FrameIdx));
+      MFI.getObjectAlign(FrameIdx));
 
   if (RC == &MSP430::GR16RegClass)
     BuildMI(MBB, MI, DL, get(MSP430::MOV16mr))
@@ -73,7 +73,7 @@ void MSP430InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
   MachineMemOperand *MMO = MF.getMachineMemOperand(
       MachinePointerInfo::getFixedStack(MF, FrameIdx),
       MachineMemOperand::MOLoad, MFI.getObjectSize(FrameIdx),
-      MFI.getObjectAlignment(FrameIdx));
+      MFI.getObjectAlign(FrameIdx));
 
   if (RC == &MSP430::GR16RegClass)
     BuildMI(MBB, MI, DL, get(MSP430::MOV16rm))
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsInstPrinter.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsInstPrinter.cpp
index 649ba20324bf0..3700d6309e1a0 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsInstPrinter.cpp
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsInstPrinter.cpp
@@ -109,7 +109,7 @@ void MipsInstPrinter::printInst(const MCInst *MI, uint64_t Address,
   }
 
   // Try to print any aliases first.
-  if (!printAliasInstr(MI, O) && !printAlias(*MI, O))
+  if (!printAliasInstr(MI, Address, O) && !printAlias(*MI, O))
     printInstruction(MI, Address, O);
   printAnnotation(O, Annot);
 
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsInstPrinter.h b/llvm/lib/Target/Mips/MCTargetDesc/MipsInstPrinter.h
index 0b1ee800e4401..3f534a2f18435 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsInstPrinter.h
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsInstPrinter.h
@@ -86,12 +86,17 @@ class MipsInstPrinter : public MCInstPrinter {
   void printInst(const MCInst *MI, uint64_t Address, StringRef Annot,
                  const MCSubtargetInfo &STI, raw_ostream &O) override;
 
-  bool printAliasInstr(const MCInst *MI, raw_ostream &OS);
-  void printCustomAliasOperand(const MCInst *MI, unsigned OpIdx,
-                               unsigned PrintMethodIdx, raw_ostream &O);
+  bool printAliasInstr(const MCInst *MI, uint64_t Address, raw_ostream &OS);
+  void printCustomAliasOperand(const MCInst *MI, uint64_t Address,
+                               unsigned OpIdx, unsigned PrintMethodIdx,
+                               raw_ostream &O);
 
 private:
   void printOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O);
+  void printOperand(const MCInst *MI, uint64_t /*Address*/, unsigned OpNum,
+                    raw_ostream &O) {
+    printOperand(MI, OpNum, O);
+  }
   template <unsigned Bits, unsigned Offset = 0>
   void printUImm(const MCInst *MI, int opNum, raw_ostream &O);
   void printMemOperand(const MCInst *MI, int opNum, raw_ostream &O);
diff --git a/llvm/lib/Target/Mips/MipsBranchExpansion.cpp b/llvm/lib/Target/Mips/MipsBranchExpansion.cpp
index 1523a6c020aad..aa8e298fa7593 100644
--- a/llvm/lib/Target/Mips/MipsBranchExpansion.cpp
+++ b/llvm/lib/Target/Mips/MipsBranchExpansion.cpp
@@ -342,16 +342,25 @@ void MipsBranchExpansion::replaceBranch(MachineBasicBlock &MBB, Iter Br,
   for (unsigned I = 0, E = Br->getDesc().getNumOperands(); I < E; ++I) {
     MachineOperand &MO = Br->getOperand(I);
 
-    if (!MO.isReg()) {
-      assert(MO.isMBB() && "MBB operand expected.");
+    switch (MO.getType()) {
+    case MachineOperand::MO_Register:
+      MIB.addReg(MO.getReg());
       break;
+    case MachineOperand::MO_Immediate:
+      // Octeon BBIT family of branch has an immediate operand
+      // (e.g. BBIT0 $v0, 3, %bb.1).
+      if (!TII->isBranchWithImm(Br->getOpcode()))
+        llvm_unreachable("Unexpected immediate in branch instruction");
+      MIB.addImm(MO.getImm());
+      break;
+    case MachineOperand::MO_MachineBasicBlock:
+      MIB.addMBB(MBBOpnd);
+      break;
+    default:
+      llvm_unreachable("Unexpected operand type in branch instruction");
     }
-
-    MIB.addReg(MO.getReg());
   }
 
-  MIB.addMBB(MBBOpnd);
-
   if (Br->hasDelaySlot()) {
     // Bundle the instruction in the delay slot to the newly created branch
     // and erase the original branch.
diff --git a/llvm/lib/Target/Mips/MipsCallLowering.cpp b/llvm/lib/Target/Mips/MipsCallLowering.cpp
index 41aceceb0ea37..b81ba5439b9b3 100644
--- a/llvm/lib/Target/Mips/MipsCallLowering.cpp
+++ b/llvm/lib/Target/Mips/MipsCallLowering.cpp
@@ -180,8 +180,8 @@ Register IncomingValueHandler::getStackAddress(const CCValAssign &VA,
 
   const TargetFrameLowering *TFL = MF.getSubtarget().getFrameLowering();
   Align Alignment = commonAlignment(TFL->getStackAlign(), Offset);
-  MMO = MF.getMachineMemOperand(MPO, MachineMemOperand::MOLoad, Size,
-                                Alignment.value());
+  MMO =
+      MF.getMachineMemOperand(MPO, MachineMemOperand::MOLoad, Size, Alignment);
 
   return MIRBuilder.buildFrameIndex(LLT::pointer(0, 32), FI).getReg(0);
 }
@@ -271,8 +271,9 @@ Register OutgoingValueHandler::getStackAddress(const CCValAssign &VA,
   MachinePointerInfo MPO =
       MachinePointerInfo::getStack(MIRBuilder.getMF(), Offset);
   unsigned Size = alignTo(VA.getValVT().getSizeInBits(), 8) / 8;
-  unsigned Align = MinAlign(TFL->getStackAlignment(), Offset);
-  MMO = MF.getMachineMemOperand(MPO, MachineMemOperand::MOStore, Size, Align);
+  Align Alignment = commonAlignment(TFL->getStackAlign(), Offset);
+  MMO =
+      MF.getMachineMemOperand(MPO, MachineMemOperand::MOStore, Size, Alignment);
 
   return AddrReg.getReg(0);
 }
@@ -485,9 +486,8 @@ bool MipsCallLowering::lowerFormalArguments(
       MachinePointerInfo MPO = MachinePointerInfo::getFixedStack(MF, FI);
       MachineInstrBuilder FrameIndex =
           MIRBuilder.buildFrameIndex(LLT::pointer(MPO.getAddrSpace(), 32), FI);
-      MachineMemOperand *MMO =
-          MF.getMachineMemOperand(MPO, MachineMemOperand::MOStore, RegSize,
-                                  /* Alignment */ RegSize);
+      MachineMemOperand *MMO = MF.getMachineMemOperand(
+          MPO, MachineMemOperand::MOStore, RegSize, Align(RegSize));
       MIRBuilder.buildStore(Copy, FrameIndex, *MMO);
     }
   }
diff --git a/llvm/lib/Target/Mips/MipsFastISel.cpp b/llvm/lib/Target/Mips/MipsFastISel.cpp
index 80f288ac500cf..1b6a75e7c383e 100644
--- a/llvm/lib/Target/Mips/MipsFastISel.cpp
+++ b/llvm/lib/Target/Mips/MipsFastISel.cpp
@@ -795,12 +795,11 @@ bool MipsFastISel::emitLoad(MVT VT, unsigned &ResultReg, Address &Addr,
   }
   if (Addr.isFIBase()) {
     unsigned FI = Addr.getFI();
-    unsigned Align = 4;
     int64_t Offset = Addr.getOffset();
     MachineFrameInfo &MFI = MF->getFrameInfo();
     MachineMemOperand *MMO = MF->getMachineMemOperand(
         MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOLoad,
-        MFI.getObjectSize(FI), Align);
+        MFI.getObjectSize(FI), Align(4));
     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg)
         .addFrameIndex(FI)
         .addImm(Offset)
@@ -846,12 +845,11 @@ bool MipsFastISel::emitStore(MVT VT, unsigned SrcReg, Address &Addr,
   }
   if (Addr.isFIBase()) {
     unsigned FI = Addr.getFI();
-    unsigned Align = 4;
     int64_t Offset = Addr.getOffset();
     MachineFrameInfo &MFI = MF->getFrameInfo();
     MachineMemOperand *MMO = MF->getMachineMemOperand(
         MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOStore,
-        MFI.getObjectSize(FI), Align);
+        MFI.getObjectSize(FI), Align(4));
     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc))
         .addReg(SrcReg)
         .addFrameIndex(FI)
@@ -1263,7 +1261,7 @@ bool MipsFastISel::processCallArgs(CallLoweringInfo &CLI,
       Addr.setReg(Mips::SP);
       Addr.setOffset(VA.getLocMemOffset() + BEAlign);
 
-      unsigned Alignment = DL.getABITypeAlignment(ArgVal->getType());
+      Align Alignment = DL.getABITypeAlign(ArgVal->getType());
       MachineMemOperand *MMO = FuncInfo.MF->getMachineMemOperand(
           MachinePointerInfo::getStack(*FuncInfo.MF, Addr.getOffset()),
           MachineMemOperand::MOStore, ArgVT.getStoreSize(), Alignment);
diff --git a/llvm/lib/Target/Mips/MipsISelLowering.h b/llvm/lib/Target/Mips/MipsISelLowering.h
index 07e3996c73836..4873bcd3d1659 100644
--- a/llvm/lib/Target/Mips/MipsISelLowering.h
+++ b/llvm/lib/Target/Mips/MipsISelLowering.h
@@ -306,7 +306,7 @@ class TargetRegisterClass;
     /// Return the correct alignment for the current calling convention.
     Align getABIAlignmentForCallingConv(Type *ArgTy,
                                         DataLayout DL) const override {
-      const Align ABIAlign(DL.getABITypeAlignment(ArgTy));
+      const Align ABIAlign = DL.getABITypeAlign(ArgTy);
       if (ArgTy->isVectorTy())
         return std::min(ABIAlign, Align(8));
       return ABIAlign;
diff --git a/llvm/lib/Target/Mips/MipsInstrInfo.cpp b/llvm/lib/Target/Mips/MipsInstrInfo.cpp
index 25bbe5990827b..3eab57cb36252 100644
--- a/llvm/lib/Target/Mips/MipsInstrInfo.cpp
+++ b/llvm/lib/Target/Mips/MipsInstrInfo.cpp
@@ -66,10 +66,10 @@ MipsInstrInfo::GetMemOperand(MachineBasicBlock &MBB, int FI,
                              MachineMemOperand::Flags Flags) const {
   MachineFunction &MF = *MBB.getParent();
   MachineFrameInfo &MFI = MF.getFrameInfo();
-  unsigned Align = MFI.getObjectAlignment(FI);
 
   return MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(MF, FI),
-                                 Flags, MFI.getObjectSize(FI), Align);
+                                 Flags, MFI.getObjectSize(FI),
+                                 MFI.getObjectAlign(FI));
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/Mips/MipsInstrInfo.h b/llvm/lib/Target/Mips/MipsInstrInfo.h
index 728f5b653ef4d..e5a3e9b462a8c 100644
--- a/llvm/lib/Target/Mips/MipsInstrInfo.h
+++ b/llvm/lib/Target/Mips/MipsInstrInfo.h
@@ -106,6 +106,10 @@ class MipsInstrInfo : public MipsGenInstrInfo {
 
   virtual unsigned getOppositeBranchOpc(unsigned Opc) const = 0;
 
+  virtual bool isBranchWithImm(unsigned Opc) const {
+    return false;
+  }
+
   /// Return the number of bytes of code the specified instruction may be.
   unsigned getInstSizeInBytes(const MachineInstr &MI) const override;
 
diff --git a/llvm/lib/Target/Mips/MipsInstructionSelector.cpp b/llvm/lib/Target/Mips/MipsInstructionSelector.cpp
index 1a25f84abb727..3f63f40b90176 100644
--- a/llvm/lib/Target/Mips/MipsInstructionSelector.cpp
+++ b/llvm/lib/Target/Mips/MipsInstructionSelector.cpp
@@ -393,7 +393,7 @@ bool MipsInstructionSelector::select(MachineInstr &I) {
             .addUse(DestAddress)
             .addJumpTableIndex(I.getOperand(1).getIndex(), MipsII::MO_ABS_LO)
             .addMemOperand(MF.getMachineMemOperand(
-                MachinePointerInfo(), MachineMemOperand::MOLoad, 4, 4));
+                MachinePointerInfo(), MachineMemOperand::MOLoad, 4, Align(4)));
     if (!constrainSelectedInstRegOperands(*LW, TII, TRI, RBI))
       return false;
 
@@ -462,7 +462,7 @@ bool MipsInstructionSelector::select(MachineInstr &I) {
     }
 
     // Unaligned memory access
-    if (MMO->getSize() > MMO->getAlignment() &&
+    if (MMO->getAlign() < MMO->getSize() &&
         !STI.systemSupportsUnalignedAccess()) {
       if (MMO->getSize() != 4 || !isRegInGprb(I.getOperand(0).getReg(), MRI))
         return false;
@@ -681,7 +681,7 @@ bool MipsInstructionSelector::select(MachineInstr &I) {
         LWGOT->getOperand(2).setTargetFlags(MipsII::MO_GOT);
       LWGOT->addMemOperand(
           MF, MF.getMachineMemOperand(MachinePointerInfo::getGOT(MF),
-                                      MachineMemOperand::MOLoad, 4, 4));
+                                      MachineMemOperand::MOLoad, 4, Align(4)));
       if (!constrainSelectedInstRegOperands(*LWGOT, TII, TRI, RBI))
         return false;
 
@@ -727,9 +727,9 @@ bool MipsInstructionSelector::select(MachineInstr &I) {
                .addReg(MF.getInfo<MipsFunctionInfo>()
                            ->getGlobalBaseRegForGlobalISel())
                .addJumpTableIndex(I.getOperand(1).getIndex(), MipsII::MO_GOT)
-               .addMemOperand(
-                   MF.getMachineMemOperand(MachinePointerInfo::getGOT(MF),
-                                           MachineMemOperand::MOLoad, 4, 4));
+               .addMemOperand(MF.getMachineMemOperand(
+                   MachinePointerInfo::getGOT(MF), MachineMemOperand::MOLoad, 4,
+                   Align(4)));
     } else {
       MI =
           BuildMI(MBB, I, I.getDebugLoc(), TII.get(Mips::LUi))
diff --git a/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp b/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp
index e71ae23b38d5a..6388e8d2d6577 100644
--- a/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp
+++ b/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp
@@ -525,12 +525,13 @@ bool MipsLegalizerInfo::legalizeIntrinsic(MachineInstr &MI,
   }
   case Intrinsic::vacopy: {
     MachinePointerInfo MPO;
-    auto Tmp = MIRBuilder.buildLoad(LLT::pointer(0, 32), MI.getOperand(2),
-                                    *MI.getMF()->getMachineMemOperand(
-                                        MPO, MachineMemOperand::MOLoad, 4, 4));
+    auto Tmp =
+        MIRBuilder.buildLoad(LLT::pointer(0, 32), MI.getOperand(2),
+                             *MI.getMF()->getMachineMemOperand(
+                                 MPO, MachineMemOperand::MOLoad, 4, Align(4)));
     MIRBuilder.buildStore(Tmp, MI.getOperand(1),
                           *MI.getMF()->getMachineMemOperand(
-                              MPO, MachineMemOperand::MOStore, 4, 4));
+                              MPO, MachineMemOperand::MOStore, 4, Align(4)));
     MI.eraseFromParent();
     return true;
   }
diff --git a/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp b/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp
index f288e734d9ff7..310e54b0ea8d5 100644
--- a/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/Mips/MipsPreLegalizerCombiner.cpp
@@ -52,7 +52,7 @@ bool MipsPreLegalizerCombinerInfo::combine(GISelChangeObserver &Observer,
         static_cast<const MipsSubtarget &>(MI.getMF()->getSubtarget());
     if (!isPowerOf2_64(MMO->getSize()))
       return false;
-    bool isUnaligned = MMO->getSize() > MMO->getAlignment();
+    bool isUnaligned = MMO->getAlign() < MMO->getSize();
     if (!STI.systemSupportsUnalignedAccess() && isUnaligned)
       return false;
 
diff --git a/llvm/lib/Target/Mips/MipsRegisterBankInfo.cpp b/llvm/lib/Target/Mips/MipsRegisterBankInfo.cpp
index 404612621a725..6325e513f9f8f 100644
--- a/llvm/lib/Target/Mips/MipsRegisterBankInfo.cpp
+++ b/llvm/lib/Target/Mips/MipsRegisterBankInfo.cpp
@@ -157,7 +157,7 @@ static bool isGprbTwoInstrUnalignedLoadOrStore(const MachineInstr *MI) {
     const MipsSubtarget &STI =
         static_cast<const MipsSubtarget &>(MI->getMF()->getSubtarget());
     if (MMO->getSize() == 4 && (!STI.systemSupportsUnalignedAccess() &&
-                                MMO->getSize() > MMO->getAlignment()))
+                                MMO->getAlign() < MMO->getSize()))
       return true;
   }
   return false;
diff --git a/llvm/lib/Target/Mips/MipsRegisterInfo.cpp b/llvm/lib/Target/Mips/MipsRegisterInfo.cpp
index 1d1744a49b3a2..08967a534bf91 100644
--- a/llvm/lib/Target/Mips/MipsRegisterInfo.cpp
+++ b/llvm/lib/Target/Mips/MipsRegisterInfo.cpp
@@ -266,7 +266,7 @@ eliminateFrameIndex(MachineBasicBlock::iterator II, int SPAdj,
                     << "spOffset   : " << spOffset << "\n"
                     << "stackSize  : " << stackSize << "\n"
                     << "alignment  : "
-                    << MF.getFrameInfo().getObjectAlignment(FrameIndex)
+                    << MF.getFrameInfo().getObjectAlign(FrameIndex).value()
                     << "\n");
 
   eliminateFI(MI, FIOperandNum, FrameIndex, stackSize, spOffset);
diff --git a/llvm/lib/Target/Mips/MipsSEInstrInfo.cpp b/llvm/lib/Target/Mips/MipsSEInstrInfo.cpp
index 440764582dde3..901a4fe4e2acb 100644
--- a/llvm/lib/Target/Mips/MipsSEInstrInfo.cpp
+++ b/llvm/lib/Target/Mips/MipsSEInstrInfo.cpp
@@ -483,6 +483,20 @@ bool MipsSEInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
   return true;
 }
 
+/// isBranchWithImm - Return true if the branch contains an immediate
+/// operand (\see lib/Target/Mips/MipsBranchExpansion.cpp).
+bool MipsSEInstrInfo::isBranchWithImm(unsigned Opc) const {
+  switch (Opc) {
+  default:
+    return false;
+  case Mips::BBIT0:
+  case Mips::BBIT1:
+  case Mips::BBIT032:
+  case Mips::BBIT132:
+    return true;
+  }
+}
+
 /// getOppositeBranchOpc - Return the inverse of the specified
 /// opcode, e.g. turning BEQ to BNE.
 unsigned MipsSEInstrInfo::getOppositeBranchOpc(unsigned Opc) const {
diff --git a/llvm/lib/Target/Mips/MipsSEInstrInfo.h b/llvm/lib/Target/Mips/MipsSEInstrInfo.h
index 104ab4d24d0c8..44a6dac2ccbd7 100644
--- a/llvm/lib/Target/Mips/MipsSEInstrInfo.h
+++ b/llvm/lib/Target/Mips/MipsSEInstrInfo.h
@@ -62,6 +62,8 @@ class MipsSEInstrInfo : public MipsInstrInfo {
 
   bool expandPostRAPseudo(MachineInstr &MI) const override;
 
+  bool isBranchWithImm(unsigned Opc) const override;
+
   unsigned getOppositeBranchOpc(unsigned Opc) const override;
 
   /// Adjust SP by Amount bytes.
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index f1b66094ff764..2ef583c0a799e 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -88,11 +88,6 @@ static cl::opt<bool> UsePrecSqrtF32(
     cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."),
     cl::init(true));
 
-static cl::opt<bool> FtzEnabled(
-    "nvptx-f32ftz", cl::ZeroOrMore, cl::Hidden,
-    cl::desc("NVPTX Specific: Flush f32 subnormals to sign-preserving zero."),
-    cl::init(false));
-
 int NVPTXTargetLowering::getDivF32Level() const {
   if (UsePrecDivF32.getNumOccurrences() > 0) {
     // If nvptx-prec-div32=N is used on the command-line, always honor it
@@ -117,12 +112,6 @@ bool NVPTXTargetLowering::usePrecSqrtF32() const {
 }
 
 bool NVPTXTargetLowering::useF32FTZ(const MachineFunction &MF) const {
-  // TODO: Get rid of this flag; there can be only one way to do this.
-  if (FtzEnabled.getNumOccurrences() > 0) {
-    // If nvptx-f32ftz is used on the command-line, always honor it
-    return FtzEnabled;
-  }
-
   return MF.getDenormalMode(APFloat::IEEEsingle()).Output ==
          DenormalMode::PreserveSign;
 }
@@ -229,11 +218,10 @@ static void ComputePTXValueVTs(const TargetLowering &TLI, const DataLayout &DL,
 // covered by the vector op. Otherwise, it returns 1.
 static unsigned CanMergeParamLoadStoresStartingAt(
     unsigned Idx, uint32_t AccessSize, const SmallVectorImpl<EVT> &ValueVTs,
-    const SmallVectorImpl<uint64_t> &Offsets, unsigned ParamAlignment) {
-  assert(isPowerOf2_32(AccessSize) && "must be a power of 2!");
+    const SmallVectorImpl<uint64_t> &Offsets, Align ParamAlignment) {
 
   // Can't vectorize if param alignment is not sufficient.
-  if (AccessSize > ParamAlignment)
+  if (ParamAlignment < AccessSize)
     return 1;
   // Can't vectorize if offset is not aligned.
   if (Offsets[Idx] & (AccessSize - 1))
@@ -293,7 +281,7 @@ enum ParamVectorizationFlags {
 static SmallVector<ParamVectorizationFlags, 16>
 VectorizePTXValueVTs(const SmallVectorImpl<EVT> &ValueVTs,
                      const SmallVectorImpl<uint64_t> &Offsets,
-                     unsigned ParamAlignment) {
+                     Align ParamAlignment) {
   // Set vector size to match ValueVTs and mark all elements as
   // scalars by default.
   SmallVector<ParamVectorizationFlags, 16> VectorInfo;
@@ -1254,7 +1242,7 @@ NVPTXTargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const {
 
 std::string NVPTXTargetLowering::getPrototype(
     const DataLayout &DL, Type *retTy, const ArgListTy &Args,
-    const SmallVectorImpl<ISD::OutputArg> &Outs, unsigned retAlignment,
+    const SmallVectorImpl<ISD::OutputArg> &Outs, MaybeAlign retAlignment,
     ImmutableCallSite CS) const {
   auto PtrVT = getPointerTy(DL);
 
@@ -1290,8 +1278,8 @@ std::string NVPTXTargetLowering::getPrototype(
       O << ".param .b" << PtrVT.getSizeInBits() << " _";
     } else if (retTy->isAggregateType() || retTy->isVectorTy() ||
                retTy->isIntegerTy(128)) {
-      O << ".param .align " << retAlignment << " .b8 _["
-        << DL.getTypeAllocSize(retTy) << "]";
+      O << ".param .align " << (retAlignment ? retAlignment->value() : 0)
+        << " .b8 _[" << DL.getTypeAllocSize(retTy) << "]";
     } else {
       llvm_unreachable("Unknown return type");
     }
@@ -1364,16 +1352,16 @@ std::string NVPTXTargetLowering::getPrototype(
   return O.str();
 }
 
-unsigned NVPTXTargetLowering::getArgumentAlignment(SDValue Callee,
-                                                   ImmutableCallSite CS,
-                                                   Type *Ty, unsigned Idx,
-                                                   const DataLayout &DL) const {
+Align NVPTXTargetLowering::getArgumentAlignment(SDValue Callee,
+                                                ImmutableCallSite CS, Type *Ty,
+                                                unsigned Idx,
+                                                const DataLayout &DL) const {
   if (!CS) {
     // CallSite is zero, fallback to ABI type alignment
-    return DL.getABITypeAlignment(Ty);
+    return DL.getABITypeAlign(Ty);
   }
 
-  unsigned Align = 0;
+  unsigned Alignment = 0;
   const Value *DirectCallee = CS.getCalledFunction();
 
   if (!DirectCallee) {
@@ -1385,8 +1373,8 @@ unsigned NVPTXTargetLowering::getArgumentAlignment(SDValue Callee,
     // With bitcast'd call targets, the instruction will be the call
     if (isa<CallInst>(CalleeI)) {
       // Check if we have call alignment metadata
-      if (getAlign(*cast<CallInst>(CalleeI), Idx, Align))
-        return Align;
+      if (getAlign(*cast<CallInst>(CalleeI), Idx, Alignment))
+        return Align(Alignment);
 
       const Value *CalleeV = cast<CallInst>(CalleeI)->getCalledValue();
       // Ignore any bitcast instructions
@@ -1408,12 +1396,12 @@ unsigned NVPTXTargetLowering::getArgumentAlignment(SDValue Callee,
   // Check for function alignment information if we found that the
   // ultimate target is a Function
   if (DirectCallee)
-    if (getAlign(*cast<Function>(DirectCallee), Idx, Align))
-      return Align;
+    if (getAlign(*cast<Function>(DirectCallee), Idx, Alignment))
+      return Align(Alignment);
 
   // Call is indirect or alignment information is not available, fall back to
   // the ABI type alignment
-  return DL.getABITypeAlignment(Ty);
+  return DL.getABITypeAlign(Ty);
 }
 
 SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
@@ -1461,15 +1449,14 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
       SmallVector<EVT, 16> VTs;
       SmallVector<uint64_t, 16> Offsets;
       ComputePTXValueVTs(*this, DL, Ty, VTs, &Offsets);
-      unsigned ArgAlign =
-          getArgumentAlignment(Callee, CS, Ty, paramCount + 1, DL);
+      Align ArgAlign = getArgumentAlignment(Callee, CS, Ty, paramCount + 1, DL);
       unsigned AllocSize = DL.getTypeAllocSize(Ty);
       SDVTList DeclareParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
       bool NeedAlign; // Does argument declaration specify alignment?
       if (Ty->isAggregateType() || Ty->isVectorTy() || Ty->isIntegerTy(128)) {
         // declare .param .align <align> .b8 .param<n>[<size>];
         SDValue DeclareParamOps[] = {
-            Chain, DAG.getConstant(ArgAlign, dl, MVT::i32),
+            Chain, DAG.getConstant(ArgAlign.value(), dl, MVT::i32),
             DAG.getConstant(paramCount, dl, MVT::i32),
             DAG.getConstant(AllocSize, dl, MVT::i32), InFlag};
         Chain = DAG.getNode(NVPTXISD::DeclareParam, dl, DeclareParamVTs,
@@ -1550,8 +1537,9 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
           // Adjust type of the store op if we've extended the scalar
           // return value.
           EVT TheStoreType = ExtendIntegerParam ? MVT::i32 : VTs[j];
-          unsigned EltAlign =
-              NeedAlign ? GreatestCommonDivisor64(ArgAlign, Offsets[j]) : 0;
+          MaybeAlign EltAlign;
+          if (NeedAlign)
+            EltAlign = commonAlignment(ArgAlign, Offsets[j]);
 
           Chain = DAG.getMemIntrinsicNode(
               Op, dl, DAG.getVTList(MVT::Other, MVT::Glue), StoreOperands,
@@ -1615,10 +1603,9 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
                                  DAG.getConstant(paramCount, dl, MVT::i32),
                                  DAG.getConstant(curOffset, dl, MVT::i32),
                                  theVal, InFlag };
-      Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreParam, dl, CopyParamVTs,
-                                      CopyParamOps, elemtype,
-                                      MachinePointerInfo(), /* Align */ 0,
-                                      MachineMemOperand::MOStore);
+      Chain = DAG.getMemIntrinsicNode(
+          NVPTXISD::StoreParam, dl, CopyParamVTs, CopyParamOps, elemtype,
+          MachinePointerInfo(), /* Align */ None, MachineMemOperand::MOStore);
 
       InFlag = Chain.getValue(1);
     }
@@ -1626,7 +1613,7 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   }
 
   GlobalAddressSDNode *Func = dyn_cast<GlobalAddressSDNode>(Callee.getNode());
-  unsigned retAlignment = 0;
+  MaybeAlign retAlignment = None;
 
   // Handle Result
   if (Ins.size() > 0) {
@@ -1655,11 +1642,12 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
       InFlag = Chain.getValue(1);
     } else {
       retAlignment = getArgumentAlignment(Callee, CS, RetTy, 0, DL);
+      assert(retAlignment && "retAlignment is guaranteed to be set");
       SDVTList DeclareRetVTs = DAG.getVTList(MVT::Other, MVT::Glue);
-      SDValue DeclareRetOps[] = { Chain,
-                                  DAG.getConstant(retAlignment, dl, MVT::i32),
-                                  DAG.getConstant(resultsz / 8, dl, MVT::i32),
-                                  DAG.getConstant(0, dl, MVT::i32), InFlag };
+      SDValue DeclareRetOps[] = {
+          Chain, DAG.getConstant(retAlignment->value(), dl, MVT::i32),
+          DAG.getConstant(resultsz / 8, dl, MVT::i32),
+          DAG.getConstant(0, dl, MVT::i32), InFlag};
       Chain = DAG.getNode(NVPTXISD::DeclareRetParam, dl, DeclareRetVTs,
                           DeclareRetOps);
       InFlag = Chain.getValue(1);
@@ -1765,7 +1753,7 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     ComputePTXValueVTs(*this, DL, RetTy, VTs, &Offsets, 0);
     assert(VTs.size() == Ins.size() && "Bad value decomposition");
 
-    unsigned RetAlign = getArgumentAlignment(Callee, CS, RetTy, 0, DL);
+    Align RetAlign = getArgumentAlignment(Callee, CS, RetTy, 0, DL);
     auto VectorInfo = VectorizePTXValueVTs(VTs, Offsets, RetAlign);
 
     SmallVector<EVT, 6> LoadVTs;
@@ -1781,7 +1769,7 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
       bool needTruncate = false;
       EVT TheLoadType = VTs[i];
       EVT EltType = Ins[i].VT;
-      unsigned EltAlign = GreatestCommonDivisor64(RetAlign, Offsets[i]);
+      Align EltAlign = commonAlignment(RetAlign, Offsets[i]);
       if (ExtendIntegerRetVal) {
         TheLoadType = MVT::i32;
         EltType = MVT::i32;
@@ -2556,7 +2544,7 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
       ComputePTXValueVTs(*this, DL, Ty, VTs, &Offsets, 0);
       assert(VTs.size() > 0 && "Unexpected empty type.");
       auto VectorInfo =
-          VectorizePTXValueVTs(VTs, Offsets, DL.getABITypeAlignment(Ty));
+          VectorizePTXValueVTs(VTs, Offsets, DL.getABITypeAlign(Ty));
 
       SDValue Arg = getParamSymbol(DAG, idx, PtrVT);
       int VecIdx = -1; // Index of the first element of the current vector.
@@ -2675,7 +2663,7 @@ NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
   assert(VTs.size() == OutVals.size() && "Bad return value decomposition");
 
   auto VectorInfo = VectorizePTXValueVTs(
-      VTs, Offsets, RetTy->isSized() ? DL.getABITypeAlignment(RetTy) : 1);
+      VTs, Offsets, RetTy->isSized() ? DL.getABITypeAlign(RetTy) : Align(1));
 
   // PTX Interoperability Guide 3.3(A): [Integer] Values shorter than
   // 32-bits are sign extended or zero extended, depending on whether
@@ -2727,10 +2715,9 @@ NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
       // Adjust type of load/store op if we've extended the scalar
       // return value.
       EVT TheStoreType = ExtendIntegerRetVal ? MVT::i32 : VTs[i];
-      Chain = DAG.getMemIntrinsicNode(Op, dl, DAG.getVTList(MVT::Other),
-                                      StoreOperands, TheStoreType,
-                                      MachinePointerInfo(), /* Align */ 1,
-                                      MachineMemOperand::MOStore);
+      Chain = DAG.getMemIntrinsicNode(
+          Op, dl, DAG.getVTList(MVT::Other), StoreOperands, TheStoreType,
+          MachinePointerInfo(), Align(1), MachineMemOperand::MOStore);
       // Cleanup vector state.
       StoreOperands.clear();
     }
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
index 546fe49808e2d..986ad70ed80c9 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
@@ -491,8 +491,7 @@ class NVPTXTargetLowering : public TargetLowering {
 
   std::string getPrototype(const DataLayout &DL, Type *, const ArgListTy &,
                            const SmallVectorImpl<ISD::OutputArg> &,
-                           unsigned retAlignment,
-                           ImmutableCallSite CS) const;
+                           MaybeAlign retAlignment, ImmutableCallSite CS) const;
 
   SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
                       const SmallVectorImpl<ISD::OutputArg> &Outs,
@@ -579,8 +578,8 @@ class NVPTXTargetLowering : public TargetLowering {
                           SelectionDAG &DAG) const override;
   SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
 
-  unsigned getArgumentAlignment(SDValue Callee, ImmutableCallSite CS, Type *Ty,
-                                unsigned Idx, const DataLayout &DL) const;
+  Align getArgumentAlignment(SDValue Callee, ImmutableCallSite CS, Type *Ty,
+                             unsigned Idx, const DataLayout &DL) const;
 };
 } // namespace llvm
 
diff --git a/llvm/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp b/llvm/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp
index 49c2790d7caa8..74c6fd3733f03 100644
--- a/llvm/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp
+++ b/llvm/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp
@@ -60,9 +60,16 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializePowerPCDisassembler() {
                                          createPPCLEDisassembler);
 }
 
-static DecodeStatus DecodePCRel24BranchTarget(MCInst &Inst, unsigned Imm,
-                                              uint64_t Addr,
-                                              const void *Decoder) {
+static DecodeStatus decodeCondBrTarget(MCInst &Inst, unsigned Imm,
+                                       uint64_t /*Address*/,
+                                       const void * /*Decoder*/) {
+  Inst.addOperand(MCOperand::createImm(SignExtend32<14>(Imm)));
+  return MCDisassembler::Success;
+}
+
+static DecodeStatus decodeDirectBrTarget(MCInst &Inst, unsigned Imm,
+                                         uint64_t /*Address*/,
+                                         const void * /*Decoder*/) {
   int32_t Offset = SignExtend32<24>(Imm);
   Inst.addOperand(MCOperand::createImm(Offset));
   return MCDisassembler::Success;
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
index 93418b9fcd748..7320c1e6f51d7 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
@@ -28,7 +28,6 @@ static uint64_t adjustFixupValue(unsigned Kind, uint64_t Value) {
   switch (Kind) {
   default:
     llvm_unreachable("Unknown fixup kind!");
-  case FK_NONE:
   case FK_Data_1:
   case FK_Data_2:
   case FK_Data_4:
@@ -52,8 +51,6 @@ static unsigned getFixupKindNumBytes(unsigned Kind) {
   switch (Kind) {
   default:
     llvm_unreachable("Unknown fixup kind!");
-  case FK_NONE:
-    return 0;
   case FK_Data_1:
     return 1;
   case FK_Data_2:
@@ -109,6 +106,11 @@ class PPCAsmBackend : public MCAsmBackend {
       { "fixup_ppc_nofixup",     0,       0,   0 }
     };
 
+    // Fixup kinds from .reloc directive are like R_PPC_NONE/R_PPC64_NONE. They
+    // do not require any extra processing.
+    if (Kind >= FirstLiteralRelocationKind)
+      return MCAsmBackend::getFixupKindInfo(FK_NONE);
+
     if (Kind < FirstTargetFixupKind)
       return MCAsmBackend::getFixupKindInfo(Kind);
 
@@ -123,11 +125,14 @@ class PPCAsmBackend : public MCAsmBackend {
                   const MCValue &Target, MutableArrayRef<char> Data,
                   uint64_t Value, bool IsResolved,
                   const MCSubtargetInfo *STI) const override {
-    Value = adjustFixupValue(Fixup.getKind(), Value);
+    MCFixupKind Kind = Fixup.getKind();
+    if (Kind >= FirstLiteralRelocationKind)
+      return;
+    Value = adjustFixupValue(Kind, Value);
     if (!Value) return;           // Doesn't change encoding.
 
     unsigned Offset = Fixup.getOffset();
-    unsigned NumBytes = getFixupKindNumBytes(Fixup.getKind());
+    unsigned NumBytes = getFixupKindNumBytes(Kind);
 
     // For each byte of the fragment that the fixup touches, mask in the bits
     // from the fixup value. The Value has been "split up" into the appropriate
@@ -140,11 +145,10 @@ class PPCAsmBackend : public MCAsmBackend {
 
   bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup,
                              const MCValue &Target) override {
-    switch ((unsigned)Fixup.getKind()) {
+    MCFixupKind Kind = Fixup.getKind();
+    switch ((unsigned)Kind) {
     default:
-      return false;
-    case FK_NONE:
-      return true;
+      return Kind >= FirstLiteralRelocationKind;
     case PPC::fixup_ppc_br24:
     case PPC::fixup_ppc_br24abs:
       // If the target symbol has a local entry point we must not attempt
@@ -228,14 +232,25 @@ class XCOFFPPCAsmBackend : public PPCAsmBackend {
 } // end anonymous namespace
 
 Optional<MCFixupKind> ELFPPCAsmBackend::getFixupKind(StringRef Name) const {
-  if (TT.isPPC64()) {
-    if (Name == "R_PPC64_NONE")
-      return FK_NONE;
-  } else {
-    if (Name == "R_PPC_NONE")
-      return FK_NONE;
+  if (TT.isOSBinFormatELF()) {
+    unsigned Type;
+    if (TT.isPPC64()) {
+      Type = llvm::StringSwitch<unsigned>(Name)
+#define ELF_RELOC(X, Y) .Case(#X, Y)
+#include "llvm/BinaryFormat/ELFRelocs/PowerPC64.def"
+#undef ELF_RELOC
+                 .Default(-1u);
+    } else {
+      Type = llvm::StringSwitch<unsigned>(Name)
+#define ELF_RELOC(X, Y) .Case(#X, Y)
+#include "llvm/BinaryFormat/ELFRelocs/PowerPC.def"
+#undef ELF_RELOC
+                 .Default(-1u);
+    }
+    if (Type != -1u)
+      return static_cast<MCFixupKind>(FirstLiteralRelocationKind + Type);
   }
-  return MCAsmBackend::getFixupKind(Name);
+  return None;
 }
 
 MCAsmBackend *llvm::createPPCAsmBackend(const Target &T,
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp
index 20f752c3041ad..c0379620a197d 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp
@@ -73,6 +73,9 @@ static MCSymbolRefExpr::VariantKind getAccessVariant(const MCValue &Target,
 unsigned PPCELFObjectWriter::getRelocType(MCContext &Ctx, const MCValue &Target,
                                           const MCFixup &Fixup,
                                           bool IsPCRel) const {
+  MCFixupKind Kind = Fixup.getKind();
+  if (Kind >= FirstLiteralRelocationKind)
+    return Kind - FirstLiteralRelocationKind;
   MCSymbolRefExpr::VariantKind Modifier = getAccessVariant(Target, Fixup);
 
   // determine the type of the relocation
@@ -133,9 +136,6 @@ unsigned PPCELFObjectWriter::getRelocType(MCContext &Ctx, const MCValue &Target,
   } else {
     switch (Fixup.getTargetKind()) {
       default: llvm_unreachable("invalid fixup kind!");
-    case FK_NONE:
-      Type = ELF::R_PPC_NONE;
-      break;
     case PPC::fixup_ppc_br24abs:
       Type = ELF::R_PPC_ADDR24;
       break;
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
index 4dc2b3579d4b3..60558a67c52bc 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
@@ -193,7 +193,7 @@ void PPCInstPrinter::printInst(const MCInst *MI, uint64_t Address,
     }
   }
 
-  if (!printAliasInstr(MI, O))
+  if (!printAliasInstr(MI, Address, O))
     printInstruction(MI, Address, O);
   printAnnotation(O, Annot);
 }
@@ -413,23 +413,29 @@ void PPCInstPrinter::printU16ImmOperand(const MCInst *MI, unsigned OpNo,
     printOperand(MI, OpNo, O);
 }
 
-void PPCInstPrinter::printBranchOperand(const MCInst *MI, unsigned OpNo,
-                                        raw_ostream &O) {
+void PPCInstPrinter::printBranchOperand(const MCInst *MI, uint64_t Address,
+                                        unsigned OpNo, raw_ostream &O) {
   if (!MI->getOperand(OpNo).isImm())
     return printOperand(MI, OpNo, O);
-
-  // Branches can take an immediate operand. This is used by the branch
-  // selection pass to print, for example `.+8` (for ELF) or `$+8` (for AIX) to
-  // express an eight byte displacement from the program counter.
-  if (!TT.isOSAIX())
-    O << ".";
-  else
-    O << "$";
-
   int32_t Imm = SignExtend32<32>((unsigned)MI->getOperand(OpNo).getImm() << 2);
-  if (Imm >= 0)
-    O << "+";
-  O << Imm;
+  if (PrintBranchImmAsAddress) {
+    uint64_t Target = Address + Imm;
+    if (!TT.isPPC64())
+      Target &= 0xffffffff;
+    O << formatHex(Target);
+  } else {
+    // Branches can take an immediate operand. This is used by the branch
+    // selection pass to print, for example `.+8` (for ELF) or `$+8` (for AIX)
+    // to express an eight byte displacement from the program counter.
+    if (!TT.isOSAIX())
+      O << ".";
+    else
+      O << "$";
+
+    if (Imm >= 0)
+      O << "+";
+    O << Imm;
+  }
 }
 
 void PPCInstPrinter::printAbsBranchOperand(const MCInst *MI, unsigned OpNo,
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.h b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.h
index 15d9c70db5f7e..9763aeceef941 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.h
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.h
@@ -39,9 +39,9 @@ class PPCInstPrinter : public MCInstPrinter {
   void printInstruction(const MCInst *MI, uint64_t Address, raw_ostream &O);
   static const char *getRegisterName(unsigned RegNo);
 
-  bool printAliasInstr(const MCInst *MI, raw_ostream &OS);
-  void printCustomAliasOperand(const MCInst *MI, unsigned OpIdx,
-                               unsigned PrintMethodIdx,
+  bool printAliasInstr(const MCInst *MI, uint64_t Address, raw_ostream &OS);
+  void printCustomAliasOperand(const MCInst *MI, uint64_t Address,
+                               unsigned OpIdx, unsigned PrintMethodIdx,
                                raw_ostream &OS);
 
   void printOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O);
@@ -64,7 +64,8 @@ class PPCInstPrinter : public MCInstPrinter {
   void printS34ImmOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O);
   void printU16ImmOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O);
   void printImmZeroOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O);
-  void printBranchOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O);
+  void printBranchOperand(const MCInst *MI, uint64_t Address, unsigned OpNo,
+                          raw_ostream &O);
   void printAbsBranchOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O);
   void printTLSCall(const MCInst *MI, unsigned OpNo, raw_ostream &O);
 
diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
index c1d88f57e24a6..0a256cb5876bc 100644
--- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
+++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
@@ -1540,13 +1540,13 @@ void PPCLinuxAsmPrinter::emitFunctionBodyEnd() {
 }
 
 void PPCAIXAsmPrinter::SetupMachineFunction(MachineFunction &MF) {
-  // Get the function descriptor symbol.
-  CurrentFnDescSym = getSymbol(&MF.getFunction());
-  // Set the alignment and the containing csect.
-  MCSectionXCOFF *FnDescSec = cast<MCSectionXCOFF>(
-      getObjFileLowering().getSectionForFunctionDescriptor(CurrentFnDescSym));
+  // Setup CurrentFnDescSym and its containing csect.
+  MCSectionXCOFF *FnDescSec =
+      cast<MCSectionXCOFF>(getObjFileLowering().getSectionForFunctionDescriptor(
+          &MF.getFunction(), TM));
   FnDescSec->setAlignment(Align(Subtarget->isPPC64() ? 8 : 4));
-  cast<MCSymbolXCOFF>(CurrentFnDescSym)->setContainingCsect(FnDescSec);
+
+  CurrentFnDescSym = FnDescSec->getQualNameSymbol();
 
   return AsmPrinter::SetupMachineFunction(MF);
 }
@@ -1565,16 +1565,12 @@ void PPCAIXAsmPrinter::ValidateGV(const GlobalVariable *GV) {
 
 const MCExpr *PPCAIXAsmPrinter::lowerConstant(const Constant *CV) {
   if (const Function *F = dyn_cast<Function>(CV)) {
-    MCSymbolXCOFF *FSym = cast<MCSymbolXCOFF>(getSymbol(F));
-    if (!FSym->hasContainingCsect()) {
-      MCSectionXCOFF *Csect = cast<MCSectionXCOFF>(
-          F->isDeclaration()
-              ? getObjFileLowering().getSectionForExternalReference(F, TM)
-              : getObjFileLowering().getSectionForFunctionDescriptor(FSym));
-      FSym->setContainingCsect(Csect);
-    }
-    return MCSymbolRefExpr::create(
-        FSym->getContainingCsect()->getQualNameSymbol(), OutContext);
+    MCSectionXCOFF *Csect = cast<MCSectionXCOFF>(
+        F->isDeclaration()
+            ? getObjFileLowering().getSectionForExternalReference(F, TM)
+            : getObjFileLowering().getSectionForFunctionDescriptor(F, TM));
+
+    return MCSymbolRefExpr::create(Csect->getQualNameSymbol(), OutContext);
   }
   return PPCAsmPrinter::lowerConstant(CV);
 }
@@ -1610,7 +1606,6 @@ void PPCAIXAsmPrinter::emitGlobalVariable(const GlobalVariable *GV) {
           : getObjFileLowering().SectionForGlobal(
                 GV, GVKind = getObjFileLowering().getKindForGlobal(GV, TM),
                 TM));
-  GVSym->setContainingCsect(Csect);
 
   // External global variables are already handled.
   if (GV->isDeclaration())
@@ -1653,8 +1648,7 @@ void PPCAIXAsmPrinter::emitFunctionDescriptor() {
   MCSectionSubPair Current = OutStreamer->getCurrentSection();
   // Emit function descriptor.
   OutStreamer->SwitchSection(
-      cast<MCSymbolXCOFF>(CurrentFnDescSym)->getContainingCsect());
-  OutStreamer->emitLabel(CurrentFnDescSym);
+      cast<MCSymbolXCOFF>(CurrentFnDescSym)->getRepresentedCsect());
   // Emit function entry point address.
   OutStreamer->emitValue(MCSymbolRefExpr::create(CurrentFnSym, OutContext),
                          PointerSize);
@@ -1696,7 +1690,6 @@ void PPCAIXAsmPrinter::emitEndOfAsmFile(Module &M) {
     // Setup the csect for the current TC entry.
     MCSectionXCOFF *TCEntry = cast<MCSectionXCOFF>(
         getObjFileLowering().getSectionForTOCEntry(I.first));
-    cast<MCSymbolXCOFF>(I.second)->setContainingCsect(TCEntry);
     OutStreamer->SwitchSection(TCEntry);
 
     OutStreamer->emitLabel(I.second);
@@ -1721,8 +1714,6 @@ PPCAIXAsmPrinter::getMCSymbolForTOCPseudoMO(const MachineOperand &MO) {
     ValidateGV(GV);
   }
 
-  MCSymbolXCOFF *XSym = cast<MCSymbolXCOFF>(getSymbol(GO));
-
   // If the global object is a global variable without initializer or is a
   // declaration of a function, then XSym is an external referenced symbol.
   // Hence we may need to explictly create a MCSectionXCOFF for it so that we
@@ -1740,7 +1731,8 @@ PPCAIXAsmPrinter::getMCSymbolForTOCPseudoMO(const MachineOperand &MO) {
     // If the MO is a function, we want to make sure to refer to the function
     // descriptor csect.
     return cast<MCSectionXCOFF>(
-               getObjFileLowering().getSectionForFunctionDescriptor(XSym))
+               getObjFileLowering().getSectionForFunctionDescriptor(
+                   cast<const Function>(GO), TM))
         ->getQualNameSymbol();
   } else if (GOKind.isCommon() || GOKind.isBSSLocal()) {
     // If the operand is a common then we should refer to the csect symbol.
diff --git a/llvm/lib/Target/PowerPC/PPCFastISel.cpp b/llvm/lib/Target/PowerPC/PPCFastISel.cpp
index d8425d89da92e..6215ddb0c51bb 100644
--- a/llvm/lib/Target/PowerPC/PPCFastISel.cpp
+++ b/llvm/lib/Target/PowerPC/PPCFastISel.cpp
@@ -536,7 +536,7 @@ bool PPCFastISel::PPCEmitLoad(MVT VT, Register &ResultReg, Address &Addr,
         MachinePointerInfo::getFixedStack(*FuncInfo.MF, Addr.Base.FI,
                                           Addr.Offset),
         MachineMemOperand::MOLoad, MFI.getObjectSize(Addr.Base.FI),
-        MFI.getObjectAlignment(Addr.Base.FI));
+        MFI.getObjectAlign(Addr.Base.FI));
 
     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg)
       .addImm(Addr.Offset).addFrameIndex(Addr.Base.FI).addMemOperand(MMO);
@@ -682,7 +682,7 @@ bool PPCFastISel::PPCEmitStore(MVT VT, unsigned SrcReg, Address &Addr) {
         MachinePointerInfo::getFixedStack(*FuncInfo.MF, Addr.Base.FI,
                                           Addr.Offset),
         MachineMemOperand::MOStore, MFI.getObjectSize(Addr.Base.FI),
-        MFI.getObjectAlignment(Addr.Base.FI));
+        MFI.getObjectAlign(Addr.Base.FI));
 
     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc))
         .addReg(SrcReg)
@@ -1996,9 +1996,9 @@ unsigned PPCFastISel::PPCMaterializeFP(const ConstantFP *CFP, MVT VT) {
     return 0;
 
   // All FP constants are loaded from the constant pool.
-  unsigned Align = DL.getPrefTypeAlignment(CFP->getType());
-  assert(Align > 0 && "Unexpectedly missing alignment information!");
-  unsigned Idx = MCP.getConstantPoolIndex(cast<Constant>(CFP), Align);
+  Align Alignment = DL.getPrefTypeAlign(CFP->getType());
+  unsigned Idx =
+      MCP.getConstantPoolIndex(cast<Constant>(CFP), Alignment.value());
   const bool HasSPE = PPCSubTarget->hasSPE();
   const TargetRegisterClass *RC;
   if (HasSPE)
@@ -2011,7 +2011,7 @@ unsigned PPCFastISel::PPCMaterializeFP(const ConstantFP *CFP, MVT VT) {
 
   MachineMemOperand *MMO = FuncInfo.MF->getMachineMemOperand(
       MachinePointerInfo::getConstantPool(*FuncInfo.MF),
-      MachineMemOperand::MOLoad, (VT == MVT::f32) ? 4 : 8, Align);
+      MachineMemOperand::MOLoad, (VT == MVT::f32) ? 4 : 8, Alignment);
 
   unsigned Opc;
 
diff --git a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
index 523bbe7dd367b..a233dc9b34746 100644
--- a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
@@ -4213,7 +4213,7 @@ bool PPCDAGToDAGISel::isOffsetMultipleOf(SDNode *N, unsigned Val) const {
     // because it is translated to r31 or r1 + slot + offset. We won't know the
     // slot number until the stack frame is finalized.
     const MachineFrameInfo &MFI = CurDAG->getMachineFunction().getFrameInfo();
-    unsigned SlotAlign = MFI.getObjectAlignment(FI->getIndex());
+    unsigned SlotAlign = MFI.getObjectAlign(FI->getIndex()).value();
     if ((SlotAlign % Val) != 0)
       return false;
 
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 9aee6cb6655ac..16b818ed1bf81 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -764,7 +764,7 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
     else
       setOperationAction(ISD::MUL, MVT::v4i32, Custom);
 
-    setOperationAction(ISD::MUL, MVT::v8i16, Custom);
+    setOperationAction(ISD::MUL, MVT::v8i16, Legal);
     setOperationAction(ISD::MUL, MVT::v16i8, Custom);
 
     setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Custom);
@@ -2417,8 +2417,7 @@ static void fixupFuncForFI(SelectionDAG &DAG, int FrameIdx, EVT VT) {
   MachineFunction &MF = DAG.getMachineFunction();
   MachineFrameInfo &MFI = MF.getFrameInfo();
 
-  unsigned Align = MFI.getObjectAlignment(FrameIdx);
-  if (Align >= 4)
+  if (MFI.getObjectAlign(FrameIdx) >= Align(4))
     return;
 
   PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
@@ -2750,7 +2749,7 @@ SDValue PPCTargetLowering::getTOCEntry(SelectionDAG &DAG, const SDLoc &dl,
   SDValue Ops[] = { GA, Reg };
   return DAG.getMemIntrinsicNode(
       PPCISD::TOC_ENTRY, dl, DAG.getVTList(VT, MVT::Other), Ops, VT,
-      MachinePointerInfo::getGOT(DAG.getMachineFunction()), 0,
+      MachinePointerInfo::getGOT(DAG.getMachineFunction()), None,
       MachineMemOperand::MOLoad);
 }
 
@@ -3447,10 +3446,7 @@ static bool CalculateStackSlotUsed(EVT ArgVT, EVT OrigVT,
 /// ensure minimum alignment required for target.
 static unsigned EnsureStackAlignment(const PPCFrameLowering *Lowering,
                                      unsigned NumBytes) {
-  unsigned TargetAlign = Lowering->getStackAlignment();
-  unsigned AlignMask = TargetAlign - 1;
-  NumBytes = (NumBytes + AlignMask) & ~AlignMask;
-  return NumBytes;
+  return alignTo(NumBytes, Lowering->getStackAlign());
 }
 
 SDValue PPCTargetLowering::LowerFormalArguments(
@@ -5139,14 +5135,14 @@ static SDValue transformCallee(const SDValue &Callee, SelectionDAG &DAG,
         MCSymbolXCOFF *S = cast<MCSymbolXCOFF>(
             Context.getOrCreateSymbol(Twine(".") + Twine(FuncName)));
 
-        if (IsDeclaration && !S->hasContainingCsect()) {
+        if (IsDeclaration && !S->hasRepresentedCsectSet()) {
           // On AIX, an undefined symbol needs to be associated with a
           // MCSectionXCOFF to get the correct storage mapping class.
           // In this case, XCOFF::XMC_PR.
           MCSectionXCOFF *Sec = Context.getXCOFFSection(
               S->getName(), XCOFF::XMC_PR, XCOFF::XTY_ER, SC,
               SectionKind::getMetadata());
-          S->setContainingCsect(Sec);
+          S->setRepresentedCsect(Sec);
         }
 
         MVT PtrVT =
@@ -6861,9 +6857,14 @@ static bool CC_AIX(unsigned ValNo, MVT ValVT, MVT LocVT,
 
     const unsigned ByValSize = ArgFlags.getByValSize();
 
-    // An empty aggregate parameter takes up no storage and no registers.
-    if (ByValSize == 0)
+    // An empty aggregate parameter takes up no storage and no registers,
+    // but needs a MemLoc for a stack slot for the formal arguments side.
+    if (ByValSize == 0) {
+      State.addLoc(CCValAssign::getMem(ValNo, MVT::INVALID_SIMPLE_VALUE_TYPE,
+                                       State.getNextStackOffset(), RegVT,
+                                       LocInfo));
       return false;
+    }
 
     if (ByValSize <= PtrByteSize) {
       State.AllocateStack(PtrByteSize, PtrByteSize);
@@ -6978,6 +6979,24 @@ static SDValue truncateScalarIntegerArg(ISD::ArgFlagsTy Flags, EVT ValVT,
   return DAG.getNode(ISD::TRUNCATE, dl, ValVT, ArgValue);
 }
 
+static unsigned mapArgRegToOffsetAIX(unsigned Reg, const PPCFrameLowering *FL) {
+  const unsigned LASize = FL->getLinkageSize();
+
+  if (PPC::GPRCRegClass.contains(Reg)) {
+    assert(Reg >= PPC::R3 && Reg <= PPC::R10 &&
+           "Reg must be a valid argument register!");
+    return LASize + 4 * (Reg - PPC::R3);
+  }
+
+  if (PPC::G8RCRegClass.contains(Reg)) {
+    assert(Reg >= PPC::X3 && Reg <= PPC::X10 &&
+           "Reg must be a valid argument register!");
+    return LASize + 8 * (Reg - PPC::X3);
+  }
+
+  llvm_unreachable("Only general purpose registers expected.");
+}
+
 SDValue PPCTargetLowering::LowerFormalArguments_AIX(
     SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
     const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
@@ -7015,12 +7034,12 @@ SDValue PPCTargetLowering::LowerFormalArguments_AIX(
   CCInfo.AllocateStack(LinkageSize, PtrByteSize);
   CCInfo.AnalyzeFormalArguments(Ins, CC_AIX);
 
+  SmallVector<SDValue, 8> MemOps;
+
   for (CCValAssign &VA : ArgLocs) {
     EVT ValVT = VA.getValVT();
     MVT LocVT = VA.getLocVT();
     ISD::ArgFlagsTy Flags = Ins[VA.getValNo()].Flags;
-    assert(!Flags.isByVal() &&
-           "Passing structure by value is unimplemented for formal arguments.");
     assert((VA.isRegLoc() || VA.isMemLoc()) &&
            "Unexpected location for function call argument.");
 
@@ -7033,6 +7052,59 @@ SDValue PPCTargetLowering::LowerFormalArguments_AIX(
     if (VA.isMemLoc() && VA.needsCustom())
       continue;
 
+    if (Flags.isByVal() && VA.isMemLoc()) {
+      if (Flags.getByValSize() != 0)
+        report_fatal_error(
+            "ByVal arguments passed on stack not implemented yet");
+
+      const int FI = MF.getFrameInfo().CreateFixedObject(
+          PtrByteSize, VA.getLocMemOffset(), /* IsImmutable */ false,
+          /* IsAliased */ true);
+      SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
+      InVals.push_back(FIN);
+
+      continue;
+    }
+
+    if (Flags.isByVal()) {
+      assert(VA.isRegLoc() && "MemLocs should already be handled.");
+
+      const unsigned ByValSize = Flags.getByValSize();
+      if (ByValSize > PtrByteSize)
+        report_fatal_error("Formal arguments greater then register size not "
+                           "implemented yet.");
+
+      const MCPhysReg ArgReg = VA.getLocReg();
+      const PPCFrameLowering *FL = Subtarget.getFrameLowering();
+      const unsigned Offset = mapArgRegToOffsetAIX(ArgReg, FL);
+
+      const unsigned StackSize = alignTo(ByValSize, PtrByteSize);
+      const int FI = MF.getFrameInfo().CreateFixedObject(
+          StackSize, Offset, /* IsImmutable */ false, /* IsAliased */ true);
+      SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
+
+      InVals.push_back(FIN);
+
+      const unsigned VReg = MF.addLiveIn(ArgReg, IsPPC64 ? &PPC::G8RCRegClass
+                                                         : &PPC::GPRCRegClass);
+
+      // Since the callers side has left justified the aggregate in the
+      // register, we can simply store the entire register into the stack
+      // slot.
+      // The store to the fixedstack object is needed becuase accessing a
+      // field of the ByVal will use a gep and load. Ideally we will optimize
+      // to extracting the value from the register directly, and elide the
+      // stores when the arguments address is not taken, but that will need to
+      // be future work.
+      SDValue CopyFrom = DAG.getCopyFromReg(Chain, dl, VReg, LocVT);
+      SDValue Store =
+          DAG.getStore(CopyFrom.getValue(1), dl, CopyFrom, FIN,
+                       MachinePointerInfo::getFixedStack(MF, FI, 0));
+
+      MemOps.push_back(Store);
+      continue;
+    }
+
     if (VA.isRegLoc()) {
       MVT::SimpleValueType SVT = ValVT.getSimpleVT().SimpleTy;
       unsigned VReg =
@@ -7080,6 +7152,9 @@ SDValue PPCTargetLowering::LowerFormalArguments_AIX(
   PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
   FuncInfo->setMinReservedArea(CallerReservedArea);
 
+  if (!MemOps.empty())
+    Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps);
+
   return Chain;
 }
 
@@ -7156,8 +7231,13 @@ SDValue PPCTargetLowering::LowerCall_AIX(
 
     if (Flags.isByVal()) {
       const unsigned ByValSize = Flags.getByValSize();
+
+      // Nothing to do for zero-sized ByVals on the caller side.
+      if (!ByValSize)
+        continue;
+
       assert(
-          VA.isRegLoc() && ByValSize > 0 && ByValSize <= PtrByteSize &&
+          VA.isRegLoc() && ByValSize <= PtrByteSize &&
           "Pass-by-value arguments are only supported in a single register.");
 
       // Loads must be a power-of-2 size and cannot be larger than the
@@ -7819,7 +7899,7 @@ void PPCTargetLowering::LowerFP_TO_INTForReuse(SDValue Op, ReuseLoadInfo &RLI,
   if (i32Stack) {
     MachineFunction &MF = DAG.getMachineFunction();
     MachineMemOperand *MMO =
-      MF.getMachineMemOperand(MPI, MachineMemOperand::MOStore, 4, 4);
+        MF.getMachineMemOperand(MPI, MachineMemOperand::MOStore, 4, Align(4));
     SDValue Ops[] = { DAG.getEntryNode(), Tmp, FIPtr };
     Chain = DAG.getMemIntrinsicNode(PPCISD::STFIWX, dl,
               DAG.getVTList(MVT::Other), Ops, MVT::i32, MMO);
@@ -7973,7 +8053,7 @@ bool PPCTargetLowering::canReuseLoadAddress(SDValue Op, EVT MemVT,
   RLI.MPI = LD->getPointerInfo();
   RLI.IsDereferenceable = LD->isDereferenceable();
   RLI.IsInvariant = LD->isInvariant();
-  RLI.Alignment = LD->getAlignment();
+  RLI.Alignment = LD->getAlign();
   RLI.AAInfo = LD->getAAInfo();
   RLI.Ranges = LD->getRanges();
 
@@ -8299,7 +8379,7 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
       RLI.Chain = Store;
       RLI.MPI =
           MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
-      RLI.Alignment = 4;
+      RLI.Alignment = Align(4);
 
       MachineMemOperand *MMO =
         MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4,
@@ -8351,7 +8431,7 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
       RLI.Chain = Store;
       RLI.MPI =
           MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
-      RLI.Alignment = 4;
+      RLI.Alignment = Align(4);
     }
 
     MachineMemOperand *MMO =
@@ -10370,13 +10450,6 @@ SDValue PPCTargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const {
     HiProd = BuildIntrinsicOp(Intrinsic::ppc_altivec_vslw, HiProd,
                               Neg16, DAG, dl);
     return DAG.getNode(ISD::ADD, dl, MVT::v4i32, LoProd, HiProd);
-  } else if (Op.getValueType() == MVT::v8i16) {
-    SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1);
-
-    SDValue Zero = BuildSplatI(0, 1, MVT::v8i16, DAG, dl);
-
-    return BuildIntrinsicOp(Intrinsic::ppc_altivec_vmladduhm,
-                            LHS, RHS, Zero, DAG, dl);
   } else if (Op.getValueType() == MVT::v16i8) {
     SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1);
     bool isLittleEndian = Subtarget.isLittleEndian();
@@ -11962,9 +12035,9 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
         int FrameIdx = MFI.CreateStackObject(8, 8, false);
 
         MachineMemOperand *MMOStore = F->getMachineMemOperand(
-          MachinePointerInfo::getFixedStack(*F, FrameIdx, 0),
-          MachineMemOperand::MOStore, MFI.getObjectSize(FrameIdx),
-          MFI.getObjectAlignment(FrameIdx));
+            MachinePointerInfo::getFixedStack(*F, FrameIdx, 0),
+            MachineMemOperand::MOStore, MFI.getObjectSize(FrameIdx),
+            MFI.getObjectAlign(FrameIdx));
 
         // Store the SrcReg into the stack.
         BuildMI(*BB, MI, dl, TII->get(StoreOp))
@@ -11974,9 +12047,9 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
           .addMemOperand(MMOStore);
 
         MachineMemOperand *MMOLoad = F->getMachineMemOperand(
-          MachinePointerInfo::getFixedStack(*F, FrameIdx, 0),
-          MachineMemOperand::MOLoad, MFI.getObjectSize(FrameIdx),
-          MFI.getObjectAlignment(FrameIdx));
+            MachinePointerInfo::getFixedStack(*F, FrameIdx, 0),
+            MachineMemOperand::MOLoad, MFI.getObjectSize(FrameIdx),
+            MFI.getObjectAlign(FrameIdx));
 
         // Load from the stack where SrcReg is stored, and save to DestReg,
         // so we have done the RegClass conversion from RegClass::SrcReg to
@@ -13576,8 +13649,8 @@ SDValue PPCTargetLowering::expandVSXLoadForLE(SDNode *N,
 
   // Do not expand to PPCISD::LXVD2X + PPCISD::XXSWAPD when the load is
   // aligned and the type is a vector with elements up to 4 bytes
-  if (Subtarget.needsSwapsForVSXMemOps() && !(MMO->getAlignment()%16)
-      && VecTy.getScalarSizeInBits() <= 32 ) {
+  if (Subtarget.needsSwapsForVSXMemOps() && MMO->getAlign() >= Align(16) &&
+      VecTy.getScalarSizeInBits() <= 32) {
     return SDValue();
   }
 
@@ -13647,8 +13720,8 @@ SDValue PPCTargetLowering::expandVSXStoreForLE(SDNode *N,
 
   // Do not expand to PPCISD::XXSWAPD and PPCISD::STXVD2X when the load is
   // aligned and the type is a vector with elements up to 4 bytes
-  if (Subtarget.needsSwapsForVSXMemOps() && !(MMO->getAlignment()%16)
-      && VecTy.getScalarSizeInBits() <= 32 ) {
+  if (Subtarget.needsSwapsForVSXMemOps() && MMO->getAlign() >= Align(16) &&
+      VecTy.getScalarSizeInBits() <= 32) {
     return SDValue();
   }
 
@@ -13694,7 +13767,7 @@ SDValue PPCTargetLowering::combineStoreFPToInt(SDNode *N,
         (Op1VT == MVT::i32 || Op1VT == MVT::i64 ||
          (Subtarget.hasP9Vector() && (Op1VT == MVT::i16 || Op1VT == MVT::i8)));
 
-  if (ResVT == MVT::ppcf128 || !Subtarget.hasP8Altivec() ||
+  if (ResVT == MVT::ppcf128 || !Subtarget.hasP8Vector() ||
       cast<StoreSDNode>(N)->isTruncatingStore() || !ValidTypeForStoreFltAsInt)
     return SDValue();
 
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.h b/llvm/lib/Target/PowerPC/PPCISelLowering.h
index 70bf4fbfce1d1..4107d2bc188e3 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.h
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.h
@@ -991,7 +991,7 @@ namespace llvm {
       MachinePointerInfo MPI;
       bool IsDereferenceable = false;
       bool IsInvariant = false;
-      unsigned Alignment = 0;
+      Align Alignment;
       AAMDNodes AAInfo;
       const MDNode *Ranges = nullptr;
 
diff --git a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
index c4142af11a91b..542225ca0d10c 100644
--- a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
+++ b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
@@ -1127,7 +1127,7 @@ def ADDISgotTprelHA: PPCEmitTimePseudo<(outs g8rc:$rD), (ins g8rc_nox0:$reg, s16
                            (PPCaddisGotTprelHA i64:$reg,
                                                tglobaltlsaddr:$disp))]>,
                   isPPC64;
-def LDgotTprelL: PPCEmitTimePseudo<(outs g8rc:$rD), (ins s16imm64:$disp, g8rc_nox0:$reg),
+def LDgotTprelL: PPCEmitTimePseudo<(outs g8rc_nox0:$rD), (ins s16imm64:$disp, g8rc_nox0:$reg),
                         "#LDgotTprelL",
                         [(set i64:$rD,
                           (PPCldGotTprelL tglobaltlsaddr:$disp, i64:$reg))]>,
diff --git a/llvm/lib/Target/PowerPC/PPCInstrAltivec.td b/llvm/lib/Target/PowerPC/PPCInstrAltivec.td
index ee4ea4487aab7..1dd8142f46ebe 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrAltivec.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrAltivec.td
@@ -869,6 +869,12 @@ def : Pat<(v8i16 (rotl v8i16:$vA, v8i16:$vB)),
 def : Pat<(v4i32 (rotl v4i32:$vA, v4i32:$vB)),
           (v4i32 (VRLW v4i32:$vA, v4i32:$vB))>;
 
+// Multiply
+def : Pat<(mul v8i16:$vA, v8i16:$vB), (VMLADDUHM $vA, $vB, (v8i16(V_SET0H)))>;
+
+// Add
+def : Pat<(add (mul v8i16:$vA, v8i16:$vB), v8i16:$vC), (VMLADDUHM $vA, $vB, $vC)>;
+
 // Saturating adds/subtracts.
 def : Pat<(v16i8 (saddsat v16i8:$vA, v16i8:$vB)), (v16i8 (VADDSBS $vA, $vB))>;
 def : Pat<(v16i8 (uaddsat v16i8:$vA, v16i8:$vB)), (v16i8 (VADDUBS $vA, $vB))>;
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
index f7a68dd162297..50097171b7f62 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
@@ -281,7 +281,7 @@ bool PPCInstrInfo::getMachineCombinerPatterns(
 
 // Detect 32 -> 64-bit extensions where we may reuse the low sub-register.
 bool PPCInstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
-                                         unsigned &SrcReg, unsigned &DstReg,
+                                         Register &SrcReg, Register &DstReg,
                                          unsigned &SubIdx) const {
   switch (MI.getOpcode()) {
   default: return false;
@@ -754,8 +754,8 @@ unsigned PPCInstrInfo::insertBranch(MachineBasicBlock &MBB,
 // Select analysis.
 bool PPCInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
                                    ArrayRef<MachineOperand> Cond,
-                                   unsigned DstReg, unsigned TrueReg,
-                                   unsigned FalseReg, int &CondCycles,
+                                   Register DstReg, Register TrueReg,
+                                   Register FalseReg, int &CondCycles,
                                    int &TrueCycles, int &FalseCycles) const {
   if (Cond.size() != 2)
     return false;
@@ -792,9 +792,9 @@ bool PPCInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
 
 void PPCInstrInfo::insertSelect(MachineBasicBlock &MBB,
                                 MachineBasicBlock::iterator MI,
-                                const DebugLoc &dl, unsigned DestReg,
-                                ArrayRef<MachineOperand> Cond, unsigned TrueReg,
-                                unsigned FalseReg) const {
+                                const DebugLoc &dl, Register DestReg,
+                                ArrayRef<MachineOperand> Cond, Register TrueReg,
+                                Register FalseReg) const {
   assert(Cond.size() == 2 &&
          "PPC branch conditions have two components!");
 
@@ -853,7 +853,7 @@ void PPCInstrInfo::insertSelect(MachineBasicBlock &MBB,
   case PPC::PRED_BIT_UNSET: SubIdx = 0; SwapOps = true; break;
   }
 
-  unsigned FirstReg =  SwapOps ? FalseReg : TrueReg,
+  Register FirstReg =  SwapOps ? FalseReg : TrueReg,
            SecondReg = SwapOps ? TrueReg  : FalseReg;
 
   // The first input register of isel cannot be r0. If it is a member
@@ -864,7 +864,7 @@ void PPCInstrInfo::insertSelect(MachineBasicBlock &MBB,
     const TargetRegisterClass *FirstRC =
       MRI.getRegClass(FirstReg)->contains(PPC::X0) ?
         &PPC::G8RC_NOX0RegClass : &PPC::GPRC_NOR0RegClass;
-    unsigned OldFirstReg = FirstReg;
+    Register OldFirstReg = FirstReg;
     FirstReg = MRI.createVirtualRegister(FirstRC);
     BuildMI(MBB, MI, dl, get(TargetOpcode::COPY), FirstReg)
       .addReg(OldFirstReg);
@@ -1238,7 +1238,7 @@ void PPCInstrInfo::storeRegToStackSlotNoUpd(
   MachineMemOperand *MMO = MF.getMachineMemOperand(
       MachinePointerInfo::getFixedStack(MF, FrameIdx),
       MachineMemOperand::MOStore, MFI.getObjectSize(FrameIdx),
-      MFI.getObjectAlignment(FrameIdx));
+      MFI.getObjectAlign(FrameIdx));
   NewMIs.back()->addMemOperand(MF, MMO);
 }
 
@@ -1301,7 +1301,7 @@ void PPCInstrInfo::loadRegFromStackSlotNoUpd(
   MachineMemOperand *MMO = MF.getMachineMemOperand(
       MachinePointerInfo::getFixedStack(MF, FrameIdx),
       MachineMemOperand::MOLoad, MFI.getObjectSize(FrameIdx),
-      MFI.getObjectAlignment(FrameIdx));
+      MFI.getObjectAlign(FrameIdx));
   NewMIs.back()->addMemOperand(MF, MMO);
 }
 
@@ -1334,7 +1334,7 @@ reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const {
 }
 
 bool PPCInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
-                                 unsigned Reg, MachineRegisterInfo *MRI) const {
+                                 Register Reg, MachineRegisterInfo *MRI) const {
   // For some instructions, it is legal to fold ZERO into the RA register field.
   // A zero immediate should always be loaded with a single li.
   unsigned DefOpc = DefMI.getOpcode();
@@ -1383,7 +1383,7 @@ bool PPCInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
   if (UseInfo->Constraints != 0)
     return false;
 
-  unsigned ZeroReg;
+  MCRegister ZeroReg;
   if (UseInfo->isLookupPtrRegClass()) {
     bool isPPC64 = Subtarget.isPPC64();
     ZeroReg = isPPC64 ? PPC::ZERO8 : PPC::ZERO;
@@ -1599,8 +1599,8 @@ bool PPCInstrInfo::DefinesPredicate(MachineInstr &MI,
   return Found;
 }
 
-bool PPCInstrInfo::analyzeCompare(const MachineInstr &MI, unsigned &SrcReg,
-                                  unsigned &SrcReg2, int &Mask,
+bool PPCInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
+                                  Register &SrcReg2, int &Mask,
                                   int &Value) const {
   unsigned Opc = MI.getOpcode();
 
@@ -1629,8 +1629,8 @@ bool PPCInstrInfo::analyzeCompare(const MachineInstr &MI, unsigned &SrcReg,
   }
 }
 
-bool PPCInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, unsigned SrcReg,
-                                        unsigned SrcReg2, int Mask, int Value,
+bool PPCInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
+                                        Register SrcReg2, int Mask, int Value,
                                         const MachineRegisterInfo *MRI) const {
   if (DisableCmpOpt)
     return false;
@@ -1658,8 +1658,8 @@ bool PPCInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, unsigned SrcReg,
   bool is64BitUnsignedCompare = OpC == PPC::CMPLDI || OpC == PPC::CMPLD;
 
   // Look through copies unless that gets us to a physical register.
-  unsigned ActualSrc = TRI->lookThruCopyLike(SrcReg, MRI);
-  if (Register::isVirtualRegister(ActualSrc))
+  Register ActualSrc = TRI->lookThruCopyLike(SrcReg, MRI);
+  if (ActualSrc.isVirtual())
     SrcReg = ActualSrc;
 
   // Get the unique definition of SrcReg.
@@ -2597,6 +2597,13 @@ bool PPCInstrInfo::foldFrameOffset(MachineInstr &MI) const {
         return true;
     return false;
   };
+
+  // We are trying to replace the ImmOpNo with ScaleReg. Give up if it is
+  // treated as special zero when ScaleReg is R0/X0 register.
+  if (III.ZeroIsSpecialOrig == III.ImmOpNo &&
+      (ScaleReg == PPC::R0 || ScaleReg == PPC::X0))
+    return false;
+
   // Make sure no other def for ToBeChangedReg and ScaleReg between ADD Instr
   // and Imm Instr.
   if (NewDefFor(ToBeChangedReg, *ADDMI, MI) || NewDefFor(ScaleReg, *ADDMI, MI))
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.h b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
index 12d7e27326a04..5459fa3ee216a 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
@@ -249,7 +249,7 @@ class PPCInstrInfo : public PPCGenInstrInfo {
   bool isAssociativeAndCommutative(const MachineInstr &Inst) const override;
 
   bool isCoalescableExtInstr(const MachineInstr &MI,
-                             unsigned &SrcReg, unsigned &DstReg,
+                             Register &SrcReg, Register &DstReg,
                              unsigned &SubIdx) const override;
   unsigned isLoadFromStackSlot(const MachineInstr &MI,
                                int &FrameIndex) const override;
@@ -279,12 +279,12 @@ class PPCInstrInfo : public PPCGenInstrInfo {
 
   // Select analysis.
   bool canInsertSelect(const MachineBasicBlock &, ArrayRef<MachineOperand> Cond,
-                       unsigned, unsigned, unsigned, int &, int &,
+                       Register, Register, Register, int &, int &,
                        int &) const override;
   void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
-                    const DebugLoc &DL, unsigned DstReg,
-                    ArrayRef<MachineOperand> Cond, unsigned TrueReg,
-                    unsigned FalseReg) const override;
+                    const DebugLoc &DL, Register DstReg,
+                    ArrayRef<MachineOperand> Cond, Register TrueReg,
+                    Register FalseReg) const override;
 
   void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
                    const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg,
@@ -329,7 +329,7 @@ class PPCInstrInfo : public PPCGenInstrInfo {
   bool
   reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const override;
 
-  bool FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, unsigned Reg,
+  bool FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg,
                      MachineRegisterInfo *MRI) const override;
 
   // If conversion by predication (only supported by some branch instructions).
@@ -373,11 +373,11 @@ class PPCInstrInfo : public PPCGenInstrInfo {
 
   // Comparison optimization.
 
-  bool analyzeCompare(const MachineInstr &MI, unsigned &SrcReg,
-                      unsigned &SrcReg2, int &Mask, int &Value) const override;
+  bool analyzeCompare(const MachineInstr &MI, Register &SrcReg,
+                      Register &SrcReg2, int &Mask, int &Value) const override;
 
-  bool optimizeCompareInstr(MachineInstr &CmpInstr, unsigned SrcReg,
-                            unsigned SrcReg2, int Mask, int Value,
+  bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
+                            Register SrcReg2, int Mask, int Value,
                             const MachineRegisterInfo *MRI) const override;
 
 
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
index fc5b26d214812..189dd77ab7ede 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
@@ -750,7 +750,9 @@ def PPCDirectBrAsmOperand : AsmOperandClass {
 def directbrtarget : Operand<OtherVT> {
   let PrintMethod = "printBranchOperand";
   let EncoderMethod = "getDirectBrEncoding";
+  let DecoderMethod = "decodeDirectBrTarget";
   let ParserMatchClass = PPCDirectBrAsmOperand;
+  let OperandType = "OPERAND_PCREL";
 }
 def absdirectbrtarget : Operand<OtherVT> {
   let PrintMethod = "printAbsBranchOperand";
@@ -764,7 +766,9 @@ def PPCCondBrAsmOperand : AsmOperandClass {
 def condbrtarget : Operand<OtherVT> {
   let PrintMethod = "printBranchOperand";
   let EncoderMethod = "getCondBrEncoding";
+  let DecoderMethod = "decodeCondBrTarget";
   let ParserMatchClass = PPCCondBrAsmOperand;
+  let OperandType = "OPERAND_PCREL";
 }
 def abscondbrtarget : Operand<OtherVT> {
   let PrintMethod = "printAbsBranchOperand";
@@ -774,7 +778,7 @@ def abscondbrtarget : Operand<OtherVT> {
 def calltarget : Operand<iPTR> {
   let PrintMethod = "printBranchOperand";
   let EncoderMethod = "getDirectBrEncoding";
-  let DecoderMethod = "DecodePCRel24BranchTarget";
+  let DecoderMethod = "decodeDirectBrTarget";
   let ParserMatchClass = PPCDirectBrAsmOperand;
   let OperandType = "OPERAND_PCREL";
 }
@@ -3213,7 +3217,7 @@ def PPC32GOT: PPCEmitTimePseudo<(outs gprc:$rD), (ins), "#PPC32GOT",
 def PPC32PICGOT: PPCEmitTimePseudo<(outs gprc:$rD, gprc:$rT), (ins), "#PPC32PICGOT", 
                 []>, NoEncode<"$rT">;
 
-def LDgotTprelL32: PPCEmitTimePseudo<(outs gprc:$rD), (ins s16imm:$disp, gprc_nor0:$reg),
+def LDgotTprelL32: PPCEmitTimePseudo<(outs gprc_nor0:$rD), (ins s16imm:$disp, gprc_nor0:$reg),
                            "#LDgotTprelL32",
                            [(set i32:$rD,
                              (PPCldGotTprelL tglobaltlsaddr:$disp, i32:$reg))]>;
@@ -4614,19 +4618,6 @@ def : InstAlias<"mfdccr $Rx", (MFSPR gprc:$Rx, 1018)>, Requires<[IsPPC4xx]>;
 def : InstAlias<"mticcr $Rx", (MTSPR 1019, gprc:$Rx)>, Requires<[IsPPC4xx]>;
 def : InstAlias<"mficcr $Rx", (MFSPR gprc:$Rx, 1019)>, Requires<[IsPPC4xx]>;
 
-def : InstAlias<"mtasr $RS", (MTSPR 280, gprc:$RS)>;
-
-def : InstAlias<"mfdec $RT", (MFSPR gprc:$RT, 22)>;
-def : InstAlias<"mtdec $RT", (MTSPR 22, gprc:$RT)>;
-
-def : InstAlias<"mfsdr1 $RT", (MFSPR gprc:$RT, 25)>;
-def : InstAlias<"mtsdr1 $RT", (MTSPR 25, gprc:$RT)>;
-
-def : InstAlias<"mfsrr0 $RT", (MFSPR gprc:$RT, 26)>;
-def : InstAlias<"mfsrr1 $RT", (MFSPR gprc:$RT, 27)>;
-def : InstAlias<"mtsrr0 $RT", (MTSPR 26, gprc:$RT)>;
-def : InstAlias<"mtsrr1 $RT", (MTSPR 27, gprc:$RT)>;
-
 
 def : InstAlias<"tlbie $RB", (TLBIE R0, gprc:$RB)>;
 
diff --git a/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp b/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp
index 74192cb20cd05..70d18edafe198 100644
--- a/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp
+++ b/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp
@@ -541,10 +541,12 @@ bool PPCMIPeephole::simplifyCode(void) {
           if (!P1 || !P2)
             break;
 
-          // Remove the passed FRSP instruction if it only feeds this MI and
-          // set any uses of that FRSP (in this MI) to the source of the FRSP.
+          // Remove the passed FRSP/XSRSP instruction if it only feeds this MI
+          // and set any uses of that FRSP/XSRSP (in this MI) to the source of
+          // the FRSP/XSRSP.
           auto removeFRSPIfPossible = [&](MachineInstr *RoundInstr) {
-            if (RoundInstr->getOpcode() == PPC::FRSP &&
+            unsigned Opc = RoundInstr->getOpcode();
+            if ((Opc == PPC::FRSP || Opc == PPC::XSRSP) &&
                 MRI->hasOneNonDBGUse(RoundInstr->getOperand(0).getReg())) {
               Simplified = true;
               Register ConvReg1 = RoundInstr->getOperand(1).getReg();
@@ -554,7 +556,7 @@ bool PPCMIPeephole::simplifyCode(void) {
                 if (Use.getOperand(i).isReg() &&
                     Use.getOperand(i).getReg() == FRSPDefines)
                   Use.getOperand(i).setReg(ConvReg1);
-              LLVM_DEBUG(dbgs() << "Removing redundant FRSP:\n");
+              LLVM_DEBUG(dbgs() << "Removing redundant FRSP/XSRSP:\n");
               LLVM_DEBUG(RoundInstr->dump());
               LLVM_DEBUG(dbgs() << "As it feeds instruction:\n");
               LLVM_DEBUG(MI.dump());
diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index 62b6d9b12ee6a..e5a98f4f71cc6 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -32,6 +32,7 @@
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/MathExtras.h"
+#include "llvm/Support/RISCVAttributes.h"
 #include "llvm/Support/TargetRegistry.h"
 
 #include <limits>
@@ -146,6 +147,7 @@ class RISCVAsmParser : public MCTargetAsmParser {
   bool parseOperand(OperandVector &Operands, StringRef Mnemonic);
 
   bool parseDirectiveOption();
+  bool parseDirectiveAttribute();
 
   void setFeatureBits(uint64_t Feature, StringRef FeatureString) {
     if (!(getSTI().getFeatureBits()[Feature])) {
@@ -155,6 +157,10 @@ class RISCVAsmParser : public MCTargetAsmParser {
     }
   }
 
+  bool getFeatureBits(uint64_t Feature) {
+    return getSTI().getFeatureBits()[Feature];
+  }
+
   void clearFeatureBits(uint64_t Feature, StringRef FeatureString) {
     if (getSTI().getFeatureBits()[Feature]) {
       MCSubtargetInfo &STI = copySTI();
@@ -1579,6 +1585,8 @@ bool RISCVAsmParser::ParseDirective(AsmToken DirectiveID) {
 
   if (IDVal == ".option")
     return parseDirectiveOption();
+  else if (IDVal == ".attribute")
+    return parseDirectiveAttribute();
 
   return true;
 }
@@ -1677,6 +1685,151 @@ bool RISCVAsmParser::parseDirectiveOption() {
   return false;
 }
 
+/// parseDirectiveAttribute
+///  ::= .attribute expression ',' ( expression | "string" )
+///  ::= .attribute identifier ',' ( expression | "string" )
+bool RISCVAsmParser::parseDirectiveAttribute() {
+  MCAsmParser &Parser = getParser();
+  int64_t Tag;
+  SMLoc TagLoc;
+  TagLoc = Parser.getTok().getLoc();
+  if (Parser.getTok().is(AsmToken::Identifier)) {
+    StringRef Name = Parser.getTok().getIdentifier();
+    Optional<unsigned> Ret =
+        ELFAttrs::attrTypeFromString(Name, RISCVAttrs::RISCVAttributeTags);
+    if (!Ret.hasValue()) {
+      Error(TagLoc, "attribute name not recognised: " + Name);
+      return false;
+    }
+    Tag = Ret.getValue();
+    Parser.Lex();
+  } else {
+    const MCExpr *AttrExpr;
+
+    TagLoc = Parser.getTok().getLoc();
+    if (Parser.parseExpression(AttrExpr))
+      return true;
+
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(AttrExpr);
+    if (check(!CE, TagLoc, "expected numeric constant"))
+      return true;
+
+    Tag = CE->getValue();
+  }
+
+  if (Parser.parseToken(AsmToken::Comma, "comma expected"))
+    return true;
+
+  StringRef StringValue;
+  int64_t IntegerValue = 0;
+  bool IsIntegerValue = true;
+
+  // RISC-V attributes have a string value if the tag number is odd
+  // and an integer value if the tag number is even.
+  if (Tag % 2)
+    IsIntegerValue = false;
+
+  SMLoc ValueExprLoc = Parser.getTok().getLoc();
+  if (IsIntegerValue) {
+    const MCExpr *ValueExpr;
+    if (Parser.parseExpression(ValueExpr))
+      return true;
+
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(ValueExpr);
+    if (!CE)
+      return Error(ValueExprLoc, "expected numeric constant");
+    IntegerValue = CE->getValue();
+  } else {
+    if (Parser.getTok().isNot(AsmToken::String))
+      return Error(Parser.getTok().getLoc(), "expected string constant");
+
+    StringValue = Parser.getTok().getStringContents();
+    Parser.Lex();
+  }
+
+  if (Parser.parseToken(AsmToken::EndOfStatement,
+                        "unexpected token in '.attribute' directive"))
+    return true;
+
+  if (Tag == RISCVAttrs::ARCH) {
+    StringRef Arch = StringValue;
+    if (Arch.consume_front("rv32"))
+      clearFeatureBits(RISCV::Feature64Bit, "64bit");
+    else if (Arch.consume_front("rv64"))
+      setFeatureBits(RISCV::Feature64Bit, "64bit");
+    else
+      return Error(ValueExprLoc, "bad arch string " + Arch);
+
+    while (!Arch.empty()) {
+      if (Arch[0] == 'i')
+        clearFeatureBits(RISCV::FeatureRV32E, "e");
+      else if (Arch[0] == 'e')
+        setFeatureBits(RISCV::FeatureRV32E, "e");
+      else if (Arch[0] == 'g') {
+        clearFeatureBits(RISCV::FeatureRV32E, "e");
+        setFeatureBits(RISCV::FeatureStdExtM, "m");
+        setFeatureBits(RISCV::FeatureStdExtA, "a");
+        setFeatureBits(RISCV::FeatureStdExtF, "f");
+        setFeatureBits(RISCV::FeatureStdExtD, "d");
+      } else if (Arch[0] == 'm')
+        setFeatureBits(RISCV::FeatureStdExtM, "m");
+      else if (Arch[0] == 'a')
+        setFeatureBits(RISCV::FeatureStdExtA, "a");
+      else if (Arch[0] == 'f')
+        setFeatureBits(RISCV::FeatureStdExtF, "f");
+      else if (Arch[0] == 'd') {
+        setFeatureBits(RISCV::FeatureStdExtF, "f");
+        setFeatureBits(RISCV::FeatureStdExtD, "d");
+      } else if (Arch[0] == 'c') {
+        setFeatureBits(RISCV::FeatureStdExtC, "c");
+      } else
+        return Error(ValueExprLoc, "bad arch string " + Arch);
+
+      Arch = Arch.drop_front(1);
+      int major = 0;
+      int minor = 0;
+      Arch.consumeInteger(10, major);
+      Arch.consume_front("p");
+      Arch.consumeInteger(10, minor);
+      if (major != 0 || minor != 0) {
+        Arch = Arch.drop_until([](char c) { return c == '_' || c == '"'; });
+        Arch = Arch.drop_while([](char c) { return c == '_'; });
+      }
+    }
+  }
+
+  if (IsIntegerValue)
+    getTargetStreamer().emitAttribute(Tag, IntegerValue);
+  else {
+    if (Tag != RISCVAttrs::ARCH) {
+      getTargetStreamer().emitTextAttribute(Tag, StringValue);
+    } else {
+      std::string formalArchStr = "rv32";
+      if (getFeatureBits(RISCV::Feature64Bit))
+        formalArchStr = "rv64";
+      if (getFeatureBits(RISCV::FeatureRV32E))
+        formalArchStr = (Twine(formalArchStr) + "e1p9").str();
+      else
+        formalArchStr = (Twine(formalArchStr) + "i2p0").str();
+
+      if (getFeatureBits(RISCV::FeatureStdExtM))
+        formalArchStr = (Twine(formalArchStr) + "_m2p0").str();
+      if (getFeatureBits(RISCV::FeatureStdExtA))
+        formalArchStr = (Twine(formalArchStr) + "_a2p0").str();
+      if (getFeatureBits(RISCV::FeatureStdExtF))
+        formalArchStr = (Twine(formalArchStr) + "_f2p0").str();
+      if (getFeatureBits(RISCV::FeatureStdExtD))
+        formalArchStr = (Twine(formalArchStr) + "_d2p0").str();
+      if (getFeatureBits(RISCV::FeatureStdExtC))
+        formalArchStr = (Twine(formalArchStr) + "_c2p0").str();
+
+      getTargetStreamer().emitTextAttribute(Tag, formalArchStr);
+    }
+  }
+
+  return false;
+}
+
 void RISCVAsmParser::emitToStreamer(MCStreamer &S, const MCInst &Inst) {
   MCInst CInst;
   bool Res = compressInst(CInst, Inst, getSTI(), S.getContext());
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFStreamer.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFStreamer.cpp
index 40fa195f37909..8382edf09231a 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFStreamer.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFStreamer.cpp
@@ -15,14 +15,18 @@
 #include "RISCVMCTargetDesc.h"
 #include "Utils/RISCVBaseInfo.h"
 #include "llvm/BinaryFormat/ELF.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCSectionELF.h"
 #include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/Support/LEB128.h"
+#include "llvm/Support/RISCVAttributes.h"
 
 using namespace llvm;
 
 // This part is for ELF object output.
 RISCVTargetELFStreamer::RISCVTargetELFStreamer(MCStreamer &S,
                                                const MCSubtargetInfo &STI)
-    : RISCVTargetStreamer(S) {
+    : RISCVTargetStreamer(S), CurrentVendor("riscv") {
   MCAssembler &MCA = getStreamer().getAssembler();
   const FeatureBitset &Features = STI.getFeatureBits();
   auto &MAB = static_cast<RISCVAsmBackend &>(MCA.getBackend());
@@ -66,3 +70,98 @@ void RISCVTargetELFStreamer::emitDirectiveOptionRVC() {}
 void RISCVTargetELFStreamer::emitDirectiveOptionNoRVC() {}
 void RISCVTargetELFStreamer::emitDirectiveOptionRelax() {}
 void RISCVTargetELFStreamer::emitDirectiveOptionNoRelax() {}
+
+void RISCVTargetELFStreamer::emitAttribute(unsigned Attribute, unsigned Value) {
+  setAttributeItem(Attribute, Value, /*OverwriteExisting=*/true);
+}
+
+void RISCVTargetELFStreamer::emitTextAttribute(unsigned Attribute,
+                                               StringRef String) {
+  setAttributeItem(Attribute, String, /*OverwriteExisting=*/true);
+}
+
+void RISCVTargetELFStreamer::emitIntTextAttribute(unsigned Attribute,
+                                                  unsigned IntValue,
+                                                  StringRef StringValue) {
+  setAttributeItems(Attribute, IntValue, StringValue,
+                    /*OverwriteExisting=*/true);
+}
+
+void RISCVTargetELFStreamer::finishAttributeSection() {
+  if (Contents.empty())
+    return;
+
+  if (AttributeSection) {
+    Streamer.SwitchSection(AttributeSection);
+  } else {
+    MCAssembler &MCA = getStreamer().getAssembler();
+    AttributeSection = MCA.getContext().getELFSection(
+        ".riscv.attributes", ELF::SHT_RISCV_ATTRIBUTES, 0);
+    Streamer.SwitchSection(AttributeSection);
+
+    Streamer.emitInt8(ELFAttrs::Format_Version);
+  }
+
+  // Vendor size + Vendor name + '\0'
+  const size_t VendorHeaderSize = 4 + CurrentVendor.size() + 1;
+
+  // Tag + Tag Size
+  const size_t TagHeaderSize = 1 + 4;
+
+  const size_t ContentsSize = calculateContentSize();
+
+  Streamer.emitInt32(VendorHeaderSize + TagHeaderSize + ContentsSize);
+  Streamer.emitBytes(CurrentVendor);
+  Streamer.emitInt8(0); // '\0'
+
+  Streamer.emitInt8(ELFAttrs::File);
+  Streamer.emitInt32(TagHeaderSize + ContentsSize);
+
+  // Size should have been accounted for already, now
+  // emit each field as its type (ULEB or String).
+  for (AttributeItem item : Contents) {
+    Streamer.emitULEB128IntValue(item.Tag);
+    switch (item.Type) {
+    default:
+      llvm_unreachable("Invalid attribute type");
+    case AttributeType::Numeric:
+      Streamer.emitULEB128IntValue(item.IntValue);
+      break;
+    case AttributeType::Text:
+      Streamer.emitBytes(item.StringValue);
+      Streamer.emitInt8(0); // '\0'
+      break;
+    case AttributeType::NumericAndText:
+      Streamer.emitULEB128IntValue(item.IntValue);
+      Streamer.emitBytes(item.StringValue);
+      Streamer.emitInt8(0); // '\0'
+      break;
+    }
+  }
+
+  Contents.clear();
+}
+
+size_t RISCVTargetELFStreamer::calculateContentSize() const {
+  size_t Result = 0;
+  for (AttributeItem item : Contents) {
+    switch (item.Type) {
+    case AttributeType::Hidden:
+      break;
+    case AttributeType::Numeric:
+      Result += getULEB128Size(item.Tag);
+      Result += getULEB128Size(item.IntValue);
+      break;
+    case AttributeType::Text:
+      Result += getULEB128Size(item.Tag);
+      Result += item.StringValue.size() + 1; // string + '\0'
+      break;
+    case AttributeType::NumericAndText:
+      Result += getULEB128Size(item.Tag);
+      Result += getULEB128Size(item.IntValue);
+      Result += item.StringValue.size() + 1; // string + '\0';
+      break;
+    }
+  }
+  return Result;
+}
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFStreamer.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFStreamer.h
index 138df786eaf3f..0221392054cee 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFStreamer.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFStreamer.h
@@ -15,16 +15,92 @@
 namespace llvm {
 
 class RISCVTargetELFStreamer : public RISCVTargetStreamer {
+private:
+  enum class AttributeType { Hidden, Numeric, Text, NumericAndText };
+
+  struct AttributeItem {
+    AttributeType Type;
+    unsigned Tag;
+    unsigned IntValue;
+    std::string StringValue;
+  };
+
+  StringRef CurrentVendor;
+  SmallVector<AttributeItem, 64> Contents;
+
+  MCSection *AttributeSection = nullptr;
+
+  AttributeItem *getAttributeItem(unsigned Attribute) {
+    for (size_t i = 0; i < Contents.size(); ++i)
+      if (Contents[i].Tag == Attribute)
+        return &Contents[i];
+    return nullptr;
+  }
+
+  void setAttributeItem(unsigned Attribute, unsigned Value,
+                        bool OverwriteExisting) {
+    // Look for existing attribute item.
+    if (AttributeItem *Item = getAttributeItem(Attribute)) {
+      if (!OverwriteExisting)
+        return;
+      Item->Type = AttributeType::Numeric;
+      Item->IntValue = Value;
+      return;
+    }
+
+    // Create new attribute item.
+    Contents.push_back({AttributeType::Numeric, Attribute, Value, ""});
+  }
+
+  void setAttributeItem(unsigned Attribute, StringRef Value,
+                        bool OverwriteExisting) {
+    // Look for existing attribute item.
+    if (AttributeItem *Item = getAttributeItem(Attribute)) {
+      if (!OverwriteExisting)
+        return;
+      Item->Type = AttributeType::Text;
+      Item->StringValue = std::string(Value);
+      return;
+    }
+
+    // Create new attribute item.
+    Contents.push_back({AttributeType::Text, Attribute, 0, std::string(Value)});
+  }
+
+  void setAttributeItems(unsigned Attribute, unsigned IntValue,
+                         StringRef StringValue, bool OverwriteExisting) {
+    // Look for existing attribute item.
+    if (AttributeItem *Item = getAttributeItem(Attribute)) {
+      if (!OverwriteExisting)
+        return;
+      Item->Type = AttributeType::NumericAndText;
+      Item->IntValue = IntValue;
+      Item->StringValue = std::string(StringValue);
+      return;
+    }
+
+    // Create new attribute item.
+    Contents.push_back({AttributeType::NumericAndText, Attribute, IntValue,
+                        std::string(StringValue)});
+  }
+
+  void emitAttribute(unsigned Attribute, unsigned Value) override;
+  void emitTextAttribute(unsigned Attribute, StringRef String) override;
+  void emitIntTextAttribute(unsigned Attribute, unsigned IntValue,
+                            StringRef StringValue) override;
+  void finishAttributeSection() override;
+  size_t calculateContentSize() const;
+
 public:
   MCELFStreamer &getStreamer();
   RISCVTargetELFStreamer(MCStreamer &S, const MCSubtargetInfo &STI);
 
-  virtual void emitDirectiveOptionPush();
-  virtual void emitDirectiveOptionPop();
-  virtual void emitDirectiveOptionRVC();
-  virtual void emitDirectiveOptionNoRVC();
-  virtual void emitDirectiveOptionRelax();
-  virtual void emitDirectiveOptionNoRelax();
+  void emitDirectiveOptionPush() override;
+  void emitDirectiveOptionPop() override;
+  void emitDirectiveOptionRVC() override;
+  void emitDirectiveOptionNoRVC() override;
+  void emitDirectiveOptionRelax() override;
+  void emitDirectiveOptionNoRelax() override;
 };
 }
 #endif
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
index 22bb80ae34e24..6cf97dd51baa2 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
@@ -73,7 +73,7 @@ void RISCVInstPrinter::printInst(const MCInst *MI, uint64_t Address,
     Res = uncompressInst(UncompressedMI, *MI, MRI, STI);
   if (Res)
     NewMI = const_cast<MCInst *>(&UncompressedMI);
-  if (NoAliases || !printAliasInstr(NewMI, STI, O))
+  if (NoAliases || !printAliasInstr(NewMI, Address, STI, O))
     printInstruction(NewMI, Address, STI, O);
   printAnnotation(O, Annot);
 }
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h
index aeb2ea6360605..80b64faad210f 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h
@@ -45,10 +45,10 @@ class RISCVInstPrinter : public MCInstPrinter {
   // Autogenerated by tblgen.
   void printInstruction(const MCInst *MI, uint64_t Address,
                         const MCSubtargetInfo &STI, raw_ostream &O);
-  bool printAliasInstr(const MCInst *MI, const MCSubtargetInfo &STI,
-                       raw_ostream &O);
-  void printCustomAliasOperand(const MCInst *MI, unsigned OpIdx,
-                               unsigned PrintMethodIdx,
+  bool printAliasInstr(const MCInst *MI, uint64_t Address,
+                       const MCSubtargetInfo &STI, raw_ostream &O);
+  void printCustomAliasOperand(const MCInst *MI, uint64_t Address,
+                               unsigned OpIdx, unsigned PrintMethodIdx,
                                const MCSubtargetInfo &STI, raw_ostream &O);
   static const char *getRegisterName(unsigned RegNo);
   static const char *getRegisterName(unsigned RegNo, unsigned AltIdx);
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVTargetStreamer.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVTargetStreamer.cpp
index 913e1f744192c..b5b59c1227f60 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVTargetStreamer.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVTargetStreamer.cpp
@@ -11,12 +11,43 @@
 //===----------------------------------------------------------------------===//
 
 #include "RISCVTargetStreamer.h"
+#include "RISCVSubtarget.h"
 #include "llvm/Support/FormattedStream.h"
+#include "llvm/Support/RISCVAttributes.h"
 
 using namespace llvm;
 
 RISCVTargetStreamer::RISCVTargetStreamer(MCStreamer &S) : MCTargetStreamer(S) {}
 
+void RISCVTargetStreamer::finish() { finishAttributeSection(); }
+
+void RISCVTargetStreamer::emitTargetAttributes(const MCSubtargetInfo &STI) {
+  if (STI.hasFeature(RISCV::FeatureRV32E))
+    emitAttribute(RISCVAttrs::STACK_ALIGN, RISCVAttrs::ALIGN_4);
+  else
+    emitAttribute(RISCVAttrs::STACK_ALIGN, RISCVAttrs::ALIGN_16);
+
+  std::string Arch = "rv32";
+  if (STI.hasFeature(RISCV::Feature64Bit))
+    Arch = "rv64";
+  if (STI.hasFeature(RISCV::FeatureRV32E))
+    Arch += "e1p9";
+  else
+    Arch += "i2p0";
+  if (STI.hasFeature(RISCV::FeatureStdExtM))
+    Arch += "_m2p0";
+  if (STI.hasFeature(RISCV::FeatureStdExtA))
+    Arch += "_a2p0";
+  if (STI.hasFeature(RISCV::FeatureStdExtF))
+    Arch += "_f2p0";
+  if (STI.hasFeature(RISCV::FeatureStdExtD))
+    Arch += "_d2p0";
+  if (STI.hasFeature(RISCV::FeatureStdExtC))
+    Arch += "_c2p0";
+
+  emitTextAttribute(RISCVAttrs::ARCH, Arch);
+}
+
 // This part is for ascii assembly output
 RISCVTargetAsmStreamer::RISCVTargetAsmStreamer(MCStreamer &S,
                                                formatted_raw_ostream &OS)
@@ -45,3 +76,18 @@ void RISCVTargetAsmStreamer::emitDirectiveOptionRelax() {
 void RISCVTargetAsmStreamer::emitDirectiveOptionNoRelax() {
   OS << "\t.option\tnorelax\n";
 }
+
+void RISCVTargetAsmStreamer::emitAttribute(unsigned Attribute, unsigned Value) {
+  OS << "\t.attribute\t" << Attribute << ", " << Twine(Value) << "\n";
+}
+
+void RISCVTargetAsmStreamer::emitTextAttribute(unsigned Attribute,
+                                               StringRef String) {
+  OS << "\t.attribute\t" << Attribute << ", \"" << String << "\"\n";
+}
+
+void RISCVTargetAsmStreamer::emitIntTextAttribute(unsigned Attribute,
+                                                  unsigned IntValue,
+                                                  StringRef StringValue) {}
+
+void RISCVTargetAsmStreamer::finishAttributeSection() {}
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVTargetStreamer.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVTargetStreamer.h
index 1becc134b2a23..f7c8db2edb01c 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVTargetStreamer.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVTargetStreamer.h
@@ -10,12 +10,14 @@
 #define LLVM_LIB_TARGET_RISCV_RISCVTARGETSTREAMER_H
 
 #include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/MCSubtargetInfo.h"
 
 namespace llvm {
 
 class RISCVTargetStreamer : public MCTargetStreamer {
 public:
   RISCVTargetStreamer(MCStreamer &S);
+  void finish() override;
 
   virtual void emitDirectiveOptionPush() = 0;
   virtual void emitDirectiveOptionPop() = 0;
@@ -23,12 +25,25 @@ class RISCVTargetStreamer : public MCTargetStreamer {
   virtual void emitDirectiveOptionNoRVC() = 0;
   virtual void emitDirectiveOptionRelax() = 0;
   virtual void emitDirectiveOptionNoRelax() = 0;
+  virtual void emitAttribute(unsigned Attribute, unsigned Value) = 0;
+  virtual void finishAttributeSection() = 0;
+  virtual void emitTextAttribute(unsigned Attribute, StringRef String) = 0;
+  virtual void emitIntTextAttribute(unsigned Attribute, unsigned IntValue,
+                                    StringRef StringValue) = 0;
+
+  void emitTargetAttributes(const MCSubtargetInfo &STI);
 };
 
 // This part is for ascii assembly output
 class RISCVTargetAsmStreamer : public RISCVTargetStreamer {
   formatted_raw_ostream &OS;
 
+  void finishAttributeSection() override;
+  void emitAttribute(unsigned Attribute, unsigned Value) override;
+  void emitTextAttribute(unsigned Attribute, StringRef String) override;
+  void emitIntTextAttribute(unsigned Attribute, unsigned IntValue,
+                            StringRef StringValue) override;
+
 public:
   RISCVTargetAsmStreamer(MCStreamer &S, formatted_raw_ostream &OS);
 
diff --git a/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp b/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
index fc7e7c1efe1a8..8955994b1c2e5 100644
--- a/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
+++ b/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
@@ -11,9 +11,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "RISCV.h"
 #include "MCTargetDesc/RISCVInstPrinter.h"
 #include "MCTargetDesc/RISCVMCExpr.h"
+#include "MCTargetDesc/RISCVTargetStreamer.h"
+#include "RISCV.h"
 #include "RISCVTargetMachine.h"
 #include "TargetInfo/RISCVTargetInfo.h"
 #include "llvm/ADT/Statistic.h"
@@ -63,6 +64,12 @@ class RISCVAsmPrinter : public AsmPrinter {
   bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
     return LowerRISCVMachineOperandToMCOperand(MO, MCOp, *this);
   }
+
+  void emitStartOfAsmFile(Module &M) override;
+  void emitEndOfAsmFile(Module &M) override;
+
+private:
+  void emitAttributes();
 };
 }
 
@@ -170,6 +177,32 @@ bool RISCVAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
   return false;
 }
 
+void RISCVAsmPrinter::emitStartOfAsmFile(Module &M) {
+  if (TM.getTargetTriple().isOSBinFormatELF())
+    emitAttributes();
+}
+
+void RISCVAsmPrinter::emitEndOfAsmFile(Module &M) {
+  RISCVTargetStreamer &RTS =
+      static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
+
+  if (TM.getTargetTriple().isOSBinFormatELF())
+    RTS.finishAttributeSection();
+}
+
+void RISCVAsmPrinter::emitAttributes() {
+  RISCVTargetStreamer &RTS =
+      static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
+
+  const Triple &TT = TM.getTargetTriple();
+  StringRef CPU = TM.getTargetCPU();
+  StringRef FS = TM.getTargetFeatureString();
+  const RISCVTargetMachine &RTM = static_cast<const RISCVTargetMachine &>(TM);
+  const RISCVSubtarget STI(TT, CPU, FS, /*ABIName=*/"", RTM);
+
+  RTS.emitTargetAttributes(STI);
+}
+
 // Force static initialization.
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVAsmPrinter() {
   RegisterAsmPrinter<RISCVAsmPrinter> X(getTheRISCV32Target());
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index f66d06c20e377..66f943c634fe5 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -10,55 +10,18 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "RISCVISelDAGToDAG.h"
 #include "MCTargetDesc/RISCVMCTargetDesc.h"
-#include "RISCV.h"
-#include "RISCVTargetMachine.h"
 #include "Utils/RISCVMatInt.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
-#include "llvm/CodeGen/SelectionDAGISel.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
+
 using namespace llvm;
 
 #define DEBUG_TYPE "riscv-isel"
 
-// RISCV-specific code to select RISCV machine instructions for
-// SelectionDAG operations.
-namespace {
-class RISCVDAGToDAGISel final : public SelectionDAGISel {
-  const RISCVSubtarget *Subtarget = nullptr;
-
-public:
-  explicit RISCVDAGToDAGISel(RISCVTargetMachine &TargetMachine)
-      : SelectionDAGISel(TargetMachine) {}
-
-  StringRef getPassName() const override {
-    return "RISCV DAG->DAG Pattern Instruction Selection";
-  }
-
-  bool runOnMachineFunction(MachineFunction &MF) override {
-    Subtarget = &MF.getSubtarget<RISCVSubtarget>();
-    return SelectionDAGISel::runOnMachineFunction(MF);
-  }
-
-  void PostprocessISelDAG() override;
-
-  void Select(SDNode *Node) override;
-
-  bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
-                                    std::vector<SDValue> &OutOps) override;
-
-  bool SelectAddrFI(SDValue Addr, SDValue &Base);
-
-// Include the pieces autogenerated from the target description.
-#include "RISCVGenDAGISel.inc"
-
-private:
-  void doPeepholeLoadStoreADDI();
-};
-}
-
 void RISCVDAGToDAGISel::PostprocessISelDAG() {
   doPeepholeLoadStoreADDI();
 }
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
new file mode 100644
index 0000000000000..dcf733ec36751
--- /dev/null
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
@@ -0,0 +1,56 @@
+//===---- RISCVISelDAGToDAG.h - A dag to dag inst selector for RISCV ------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines an instruction selector for the RISCV target.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_RISCV_RISCVISELDAGTODAG_H
+#define LLVM_LIB_TARGET_RISCV_RISCVISELDAGTODAG_H
+
+#include "RISCV.h"
+#include "RISCVTargetMachine.h"
+#include "llvm/CodeGen/SelectionDAGISel.h"
+
+// RISCV-specific code to select RISCV machine instructions for
+// SelectionDAG operations.
+namespace llvm {
+class RISCVDAGToDAGISel : public SelectionDAGISel {
+  const RISCVSubtarget *Subtarget = nullptr;
+
+public:
+  explicit RISCVDAGToDAGISel(RISCVTargetMachine &TargetMachine)
+      : SelectionDAGISel(TargetMachine) {}
+
+  StringRef getPassName() const override {
+    return "RISCV DAG->DAG Pattern Instruction Selection";
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override {
+    Subtarget = &MF.getSubtarget<RISCVSubtarget>();
+    return SelectionDAGISel::runOnMachineFunction(MF);
+  }
+
+  void PostprocessISelDAG() override;
+
+  void Select(SDNode *Node) override;
+
+  bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
+                                    std::vector<SDValue> &OutOps) override;
+
+  bool SelectAddrFI(SDValue Addr, SDValue &Base);
+
+// Include the pieces autogenerated from the target description.
+#include "RISCVGenDAGISel.inc"
+
+private:
+  void doPeepholeLoadStoreADDI();
+};
+}
+
+#endif
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 4e3fde556068d..2112ad8138866 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -210,6 +210,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
 
   setOperationAction(ISD::TRAP, MVT::Other, Legal);
   setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
+  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
 
   if (Subtarget.hasStdExtA()) {
     setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
@@ -429,6 +430,8 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
     SDValue FPConv = DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, NewOp0);
     return FPConv;
   }
+  case ISD::INTRINSIC_WO_CHAIN:
+    return LowerINTRINSIC_WO_CHAIN(Op, DAG);
   }
 }
 
@@ -832,6 +835,20 @@ SDValue RISCVTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG,
   return DAG.getMergeValues(Parts, DL);
 }
 
+SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
+                                                     SelectionDAG &DAG) const {
+  unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
+  SDLoc DL(Op);
+  switch (IntNo) {
+  default:
+    return SDValue();    // Don't custom lower most intrinsics.
+  case Intrinsic::thread_pointer: {
+    EVT PtrVT = getPointerTy(DAG.getDataLayout());
+    return DAG.getRegister(RISCV::X4, PtrVT);
+  }
+  }
+}
+
 // Returns the opcode of the target-specific SDNode that implements the 32-bit
 // form of the given Opcode.
 static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode) {
@@ -1189,7 +1206,7 @@ static MachineBasicBlock *emitSplitF64Pseudo(MachineInstr &MI,
                           RI);
   MachineMemOperand *MMO =
       MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(MF, FI),
-                              MachineMemOperand::MOLoad, 8, 8);
+                              MachineMemOperand::MOLoad, 8, Align(8));
   BuildMI(*BB, MI, DL, TII.get(RISCV::LW), LoReg)
       .addFrameIndex(FI)
       .addImm(0)
@@ -1219,7 +1236,7 @@ static MachineBasicBlock *emitBuildPairF64Pseudo(MachineInstr &MI,
 
   MachineMemOperand *MMO =
       MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(MF, FI),
-                              MachineMemOperand::MOStore, 8, 8);
+                              MachineMemOperand::MOStore, 8, Align(8));
   BuildMI(*BB, MI, DL, TII.get(RISCV::SW))
       .addReg(LoReg, getKillRegState(MI.getOperand(1).isKill()))
       .addFrameIndex(FI)
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index b56d6dce27574..f76abf22e4db1 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -129,6 +129,10 @@ class RISCVTargetLowering : public TargetLowering {
     return ISD::SIGN_EXTEND;
   }
 
+  ISD::NodeType getExtendForAtomicCmpSwapArg() const override {
+    return ISD::SIGN_EXTEND;
+  }
+
   bool shouldExpandShift(SelectionDAG &DAG, SDNode *N) const override {
     if (DAG.getMachineFunction().getFunction().hasMinSize())
       return false;
@@ -203,6 +207,7 @@ class RISCVTargetLowering : public TargetLowering {
   SDValue lowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerShiftLeftParts(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerShiftRightParts(SDValue Op, SelectionDAG &DAG, bool IsSRA) const;
+  SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
 
   bool isEligibleForTailCallOptimization(
       CCState &CCInfo, CallLoweringInfo &CLI, MachineFunction &MF,
diff --git a/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.cpp b/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.cpp
index 8a673de699113..f6728a070736d 100644
--- a/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.cpp
+++ b/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.cpp
@@ -46,7 +46,8 @@ void SparcInstPrinter::printRegName(raw_ostream &OS, unsigned RegNo) const
 void SparcInstPrinter::printInst(const MCInst *MI, uint64_t Address,
                                  StringRef Annot, const MCSubtargetInfo &STI,
                                  raw_ostream &O) {
-  if (!printAliasInstr(MI, STI, O) && !printSparcAliasInstr(MI, STI, O))
+  if (!printAliasInstr(MI, Address, STI, O) &&
+      !printSparcAliasInstr(MI, STI, O))
     printInstruction(MI, Address, STI, O);
   printAnnotation(O, Annot);
 }
diff --git a/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.h b/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.h
index cb85fe98ed42b..11587f165ef29 100644
--- a/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.h
+++ b/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.h
@@ -33,10 +33,10 @@ class SparcInstPrinter : public MCInstPrinter {
   // Autogenerated by tblgen.
   void printInstruction(const MCInst *MI, uint64_t Address,
                         const MCSubtargetInfo &STI, raw_ostream &O);
-  bool printAliasInstr(const MCInst *MI, const MCSubtargetInfo &STI,
-                       raw_ostream &O);
-  void printCustomAliasOperand(const MCInst *MI, unsigned OpIdx,
-                               unsigned PrintMethodIdx,
+  bool printAliasInstr(const MCInst *MI, uint64_t Address,
+                       const MCSubtargetInfo &STI, raw_ostream &O);
+  void printCustomAliasOperand(const MCInst *MI, uint64_t Address,
+                               unsigned OpIdx, unsigned PrintMethodIdx,
                                const MCSubtargetInfo &STI, raw_ostream &O);
   static const char *getRegisterName(unsigned RegNo);
 
diff --git a/llvm/lib/Target/Sparc/SparcISelLowering.cpp b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
index f30d62daa7eed..cf6f2183fc01e 100644
--- a/llvm/lib/Target/Sparc/SparcISelLowering.cpp
+++ b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
@@ -2544,15 +2544,15 @@ static SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG,
                                        const SparcSubtarget *Subtarget) {
   SDValue Chain = Op.getOperand(0);  // Legalize the chain.
   SDValue Size  = Op.getOperand(1);  // Legalize the size.
-  unsigned Align = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
-  unsigned StackAlign = Subtarget->getFrameLowering()->getStackAlignment();
+  MaybeAlign Alignment(cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue());
+  Align StackAlign = Subtarget->getFrameLowering()->getStackAlign();
   EVT VT = Size->getValueType(0);
   SDLoc dl(Op);
 
   // TODO: implement over-aligned alloca. (Note: also implies
   // supporting support for overaligned function frames + dynamic
   // allocations, at all, which currently isn't supported)
-  if (Align > StackAlign) {
+  if (Alignment && *Alignment > StackAlign) {
     const MachineFunction &MF = DAG.getMachineFunction();
     report_fatal_error("Function \"" + Twine(MF.getName()) + "\": "
                        "over-aligned dynamic alloca not supported.");
diff --git a/llvm/lib/Target/Sparc/SparcInstrInfo.cpp b/llvm/lib/Target/Sparc/SparcInstrInfo.cpp
index b614e0bc926e3..b31830de517f1 100644
--- a/llvm/lib/Target/Sparc/SparcInstrInfo.cpp
+++ b/llvm/lib/Target/Sparc/SparcInstrInfo.cpp
@@ -403,7 +403,7 @@ storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
   const MachineFrameInfo &MFI = MF->getFrameInfo();
   MachineMemOperand *MMO = MF->getMachineMemOperand(
       MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOStore,
-      MFI.getObjectSize(FI), MFI.getObjectAlignment(FI));
+      MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
 
   // On the order of operands here: think "[FrameIdx + 0] = SrcReg".
   if (RC == &SP::I64RegsRegClass)
@@ -442,7 +442,7 @@ loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
   const MachineFrameInfo &MFI = MF->getFrameInfo();
   MachineMemOperand *MMO = MF->getMachineMemOperand(
       MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOLoad,
-      MFI.getObjectSize(FI), MFI.getObjectAlignment(FI));
+      MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
 
   if (RC == &SP::I64RegsRegClass)
     BuildMI(MBB, I, DL, get(SP::LDXri), DestReg).addFrameIndex(FI).addImm(0)
diff --git a/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp b/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
index 8ad61dd0a1e83..3bf54edab3b1b 100644
--- a/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
+++ b/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
@@ -53,8 +53,6 @@ enum RegisterKind {
   GRH32Reg,
   GR64Reg,
   GR128Reg,
-  ADDR32Reg,
-  ADDR64Reg,
   FP32Reg,
   FP64Reg,
   FP128Reg,
@@ -109,7 +107,7 @@ class SystemZOperand : public MCParsedAsmOperand {
 
   // Base + Disp + Index, where Base and Index are LLVM registers or 0.
   // MemKind says what type of memory this is and RegKind says what type
-  // the base register has (ADDR32Reg or ADDR64Reg).  Length is the operand
+  // the base register has (GR32Reg or GR64Reg).  Length is the operand
   // length for D(L,B)-style operands, otherwise it is null.
   struct MemOp {
     unsigned Base : 12;
@@ -348,8 +346,8 @@ class SystemZOperand : public MCParsedAsmOperand {
   bool isGRX32() const { return false; }
   bool isGR64() const { return isReg(GR64Reg); }
   bool isGR128() const { return isReg(GR128Reg); }
-  bool isADDR32() const { return isReg(ADDR32Reg); }
-  bool isADDR64() const { return isReg(ADDR64Reg); }
+  bool isADDR32() const { return isReg(GR32Reg); }
+  bool isADDR64() const { return isReg(GR64Reg); }
   bool isADDR128() const { return false; }
   bool isFP32() const { return isReg(FP32Reg); }
   bool isFP64() const { return isReg(FP64Reg); }
@@ -361,16 +359,16 @@ class SystemZOperand : public MCParsedAsmOperand {
   bool isAR32() const { return isReg(AR32Reg); }
   bool isCR64() const { return isReg(CR64Reg); }
   bool isAnyReg() const { return (isReg() || isImm(0, 15)); }
-  bool isBDAddr32Disp12() const { return isMemDisp12(BDMem, ADDR32Reg); }
-  bool isBDAddr32Disp20() const { return isMemDisp20(BDMem, ADDR32Reg); }
-  bool isBDAddr64Disp12() const { return isMemDisp12(BDMem, ADDR64Reg); }
-  bool isBDAddr64Disp20() const { return isMemDisp20(BDMem, ADDR64Reg); }
-  bool isBDXAddr64Disp12() const { return isMemDisp12(BDXMem, ADDR64Reg); }
-  bool isBDXAddr64Disp20() const { return isMemDisp20(BDXMem, ADDR64Reg); }
-  bool isBDLAddr64Disp12Len4() const { return isMemDisp12Len4(ADDR64Reg); }
-  bool isBDLAddr64Disp12Len8() const { return isMemDisp12Len8(ADDR64Reg); }
-  bool isBDRAddr64Disp12() const { return isMemDisp12(BDRMem, ADDR64Reg); }
-  bool isBDVAddr64Disp12() const { return isMemDisp12(BDVMem, ADDR64Reg); }
+  bool isBDAddr32Disp12() const { return isMemDisp12(BDMem, GR32Reg); }
+  bool isBDAddr32Disp20() const { return isMemDisp20(BDMem, GR32Reg); }
+  bool isBDAddr64Disp12() const { return isMemDisp12(BDMem, GR64Reg); }
+  bool isBDAddr64Disp20() const { return isMemDisp20(BDMem, GR64Reg); }
+  bool isBDXAddr64Disp12() const { return isMemDisp12(BDXMem, GR64Reg); }
+  bool isBDXAddr64Disp20() const { return isMemDisp20(BDXMem, GR64Reg); }
+  bool isBDLAddr64Disp12Len4() const { return isMemDisp12Len4(GR64Reg); }
+  bool isBDLAddr64Disp12Len8() const { return isMemDisp12Len8(GR64Reg); }
+  bool isBDRAddr64Disp12() const { return isMemDisp12(BDRMem, GR64Reg); }
+  bool isBDVAddr64Disp12() const { return isMemDisp12(BDVMem, GR64Reg); }
   bool isU1Imm() const { return isImm(0, 1); }
   bool isU2Imm() const { return isImm(0, 3); }
   bool isU3Imm() const { return isImm(0, 7); }
@@ -407,8 +405,7 @@ class SystemZAsmParser : public MCTargetAsmParser {
 
   bool parseRegister(Register &Reg, bool RestoreOnFailure = false);
 
-  bool parseRegister(Register &Reg, RegisterGroup Group, const unsigned *Regs,
-                     bool IsAddress = false);
+  bool parseRegister(Register &Reg, RegisterGroup Group, const unsigned *Regs);
 
   OperandMatchResultTy parseRegister(OperandVector &Operands,
                                      RegisterGroup Group, const unsigned *Regs,
@@ -477,10 +474,12 @@ class SystemZAsmParser : public MCTargetAsmParser {
     return parseRegister(Operands, RegGR, SystemZMC::GR128Regs, GR128Reg);
   }
   OperandMatchResultTy parseADDR32(OperandVector &Operands) {
-    return parseRegister(Operands, RegGR, SystemZMC::GR32Regs, ADDR32Reg);
+    // For the AsmParser, we will accept %r0 for ADDR32 as well.
+    return parseRegister(Operands, RegGR, SystemZMC::GR32Regs, GR32Reg);
   }
   OperandMatchResultTy parseADDR64(OperandVector &Operands) {
-    return parseRegister(Operands, RegGR, SystemZMC::GR64Regs, ADDR64Reg);
+    // For the AsmParser, we will accept %r0 for ADDR64 as well.
+    return parseRegister(Operands, RegGR, SystemZMC::GR64Regs, GR64Reg);
   }
   OperandMatchResultTy parseADDR128(OperandVector &Operands) {
     llvm_unreachable("Shouldn't be used as an operand");
@@ -516,22 +515,22 @@ class SystemZAsmParser : public MCTargetAsmParser {
     return parseAnyRegister(Operands);
   }
   OperandMatchResultTy parseBDAddr32(OperandVector &Operands) {
-    return parseAddress(Operands, BDMem, SystemZMC::GR32Regs, ADDR32Reg);
+    return parseAddress(Operands, BDMem, SystemZMC::GR32Regs, GR32Reg);
   }
   OperandMatchResultTy parseBDAddr64(OperandVector &Operands) {
-    return parseAddress(Operands, BDMem, SystemZMC::GR64Regs, ADDR64Reg);
+    return parseAddress(Operands, BDMem, SystemZMC::GR64Regs, GR64Reg);
   }
   OperandMatchResultTy parseBDXAddr64(OperandVector &Operands) {
-    return parseAddress(Operands, BDXMem, SystemZMC::GR64Regs, ADDR64Reg);
+    return parseAddress(Operands, BDXMem, SystemZMC::GR64Regs, GR64Reg);
   }
   OperandMatchResultTy parseBDLAddr64(OperandVector &Operands) {
-    return parseAddress(Operands, BDLMem, SystemZMC::GR64Regs, ADDR64Reg);
+    return parseAddress(Operands, BDLMem, SystemZMC::GR64Regs, GR64Reg);
   }
   OperandMatchResultTy parseBDRAddr64(OperandVector &Operands) {
-    return parseAddress(Operands, BDRMem, SystemZMC::GR64Regs, ADDR64Reg);
+    return parseAddress(Operands, BDRMem, SystemZMC::GR64Regs, GR64Reg);
   }
   OperandMatchResultTy parseBDVAddr64(OperandVector &Operands) {
-    return parseAddress(Operands, BDVMem, SystemZMC::GR64Regs, ADDR64Reg);
+    return parseAddress(Operands, BDVMem, SystemZMC::GR64Regs, GR64Reg);
   }
   OperandMatchResultTy parsePCRel12(OperandVector &Operands) {
     return parsePCRel(Operands, -(1LL << 12), (1LL << 12) - 1, false);
@@ -751,20 +750,17 @@ bool SystemZAsmParser::parseRegister(Register &Reg, bool RestoreOnFailure) {
 
 // Parse a register of group Group.  If Regs is nonnull, use it to map
 // the raw register number to LLVM numbering, with zero entries
-// indicating an invalid register.  IsAddress says whether the
-// register appears in an address context. Allow FP Group if expecting
+// indicating an invalid register.  Allow FP Group if expecting
 // RegV Group, since the f-prefix yields the FP group even while used
 // with vector instructions.
 bool SystemZAsmParser::parseRegister(Register &Reg, RegisterGroup Group,
-                                     const unsigned *Regs, bool IsAddress) {
+                                     const unsigned *Regs) {
   if (parseRegister(Reg))
     return true;
   if (Reg.Group != Group && !(Reg.Group == RegFP && Group == RegV))
     return Error(Reg.StartLoc, "invalid operand for instruction");
   if (Regs && Regs[Reg.Num] == 0)
     return Error(Reg.StartLoc, "invalid register pair");
-  if (Reg.Num == 0 && IsAddress)
-    return Error(Reg.StartLoc, "%r0 used in an address");
   if (Regs)
     Reg.Num = Regs[Reg.Num];
   return false;
@@ -778,8 +774,7 @@ SystemZAsmParser::parseRegister(OperandVector &Operands, RegisterGroup Group,
     return MatchOperand_NoMatch;
 
   Register Reg;
-  bool IsAddress = (Kind == ADDR32Reg || Kind == ADDR64Reg);
-  if (parseRegister(Reg, Group, Regs, IsAddress))
+  if (parseRegister(Reg, Group, Regs))
     return MatchOperand_ParseFail;
 
   Operands.push_back(SystemZOperand::createReg(Kind, Reg.Num,
@@ -900,9 +895,6 @@ SystemZAsmParser::parseAddressRegister(Register &Reg) {
   } else if (Reg.Group != RegGR) {
     Error(Reg.StartLoc, "invalid address register");
     return true;
-  } else if (Reg.Num == 0) {
-    Error(Reg.StartLoc, "%r0 used in an address");
-    return true;
   }
   return false;
 }
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinter.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinter.cpp
index 5893b227c08c3..fac363cae713b 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinter.cpp
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinter.cpp
@@ -155,7 +155,8 @@ void SystemZInstPrinter::printPCRelOperand(const MCInst *MI, int OpNum,
     MO.getExpr()->print(O, &MAI);
 }
 
-void SystemZInstPrinter::printPCRelTLSOperand(const MCInst *MI, int OpNum,
+void SystemZInstPrinter::printPCRelTLSOperand(const MCInst *MI,
+                                              uint64_t Address, int OpNum,
                                               raw_ostream &O) {
   // Output the PC-relative operand.
   printPCRelOperand(MI, OpNum, O);
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinter.h b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinter.h
index 5628e9252f030..cfe1bd89c3eb2 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinter.h
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinter.h
@@ -46,6 +46,10 @@ class SystemZInstPrinter : public MCInstPrinter {
 private:
   // Print various types of operand.
   void printOperand(const MCInst *MI, int OpNum, raw_ostream &O);
+  void printOperand(const MCInst *MI, uint64_t /*Address*/, unsigned OpNum,
+                    raw_ostream &O) {
+    printOperand(MI, OpNum, O);
+  }
   void printBDAddrOperand(const MCInst *MI, int OpNum, raw_ostream &O);
   void printBDXAddrOperand(const MCInst *MI, int OpNum, raw_ostream &O);
   void printBDLAddrOperand(const MCInst *MI, int OpNum, raw_ostream &O);
@@ -65,7 +69,12 @@ class SystemZInstPrinter : public MCInstPrinter {
   void printU32ImmOperand(const MCInst *MI, int OpNum, raw_ostream &O);
   void printU48ImmOperand(const MCInst *MI, int OpNum, raw_ostream &O);
   void printPCRelOperand(const MCInst *MI, int OpNum, raw_ostream &O);
-  void printPCRelTLSOperand(const MCInst *MI, int OpNum, raw_ostream &O);
+  void printPCRelOperand(const MCInst *MI, uint64_t /*Address*/, int OpNum,
+                         raw_ostream &O) {
+    printPCRelOperand(MI, OpNum, O);
+  }
+  void printPCRelTLSOperand(const MCInst *MI, uint64_t Address, int OpNum,
+                            raw_ostream &O);
 
   // Print the mnemonic for a condition-code mask ("ne", "lh", etc.)
   // This forms part of the instruction name rather than the operand list.
diff --git a/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp b/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp
index 3c08fade3b4cb..4109bfc11337e 100644
--- a/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp
@@ -92,9 +92,9 @@ static void lowerAlignmentHint(const MachineInstr *MI, MCInst &LoweredMI,
     return;
   const MachineMemOperand *MMO = *MI->memoperands_begin();
   unsigned AlignmentHint = 0;
-  if (MMO->getAlignment() >= 16)
+  if (MMO->getAlign() >= Align(16))
     AlignmentHint = 4;
-  else if (MMO->getAlignment() >= 8)
+  else if (MMO->getAlign() >= Align(8))
     AlignmentHint = 3;
   if (AlignmentHint == 0)
     return;
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 66da118dd8250..9e5d7f23cdb61 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -4389,7 +4389,7 @@ static bool getShuffleInput(const SmallVectorImpl<int> &Bytes, unsigned Start,
 }
 
 // Bytes is a VPERM-like permute vector, except that -1 is used for
-// undefined bytes.  Return true if it can be performed using VSLDI.
+// undefined bytes.  Return true if it can be performed using VSLDB.
 // When returning true, set StartIndex to the shift amount and OpNo0
 // and OpNo1 to the VPERM operands that should be used as the first
 // and second shift operand respectively.
@@ -4449,14 +4449,14 @@ static SDValue getPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
 
 // Bytes is a VPERM-like permute vector, except that -1 is used for
 // undefined bytes.  Implement it on operands Ops[0] and Ops[1] using
-// VSLDI or VPERM.
+// VSLDB or VPERM.
 static SDValue getGeneralPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
                                      SDValue *Ops,
                                      const SmallVectorImpl<int> &Bytes) {
   for (unsigned I = 0; I < 2; ++I)
     Ops[I] = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Ops[I]);
 
-  // First see whether VSLDI can be used.
+  // First see whether VSLDB can be used.
   unsigned StartIndex, OpNo0, OpNo1;
   if (isShlDoublePermute(Bytes, StartIndex, OpNo0, OpNo1))
     return DAG.getNode(SystemZISD::SHL_DOUBLE, DL, MVT::v16i8, Ops[OpNo0],
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrBuilder.h b/llvm/lib/Target/SystemZ/SystemZInstrBuilder.h
index ec7639e71f819..57c10648612dc 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrBuilder.h
+++ b/llvm/lib/Target/SystemZ/SystemZInstrBuilder.h
@@ -36,7 +36,7 @@ addFrameReference(const MachineInstrBuilder &MIB, int FI) {
   int64_t Offset = 0;
   MachineMemOperand *MMO = MF.getMachineMemOperand(
       MachinePointerInfo::getFixedStack(MF, FI, Offset), Flags,
-      MFFrame.getObjectSize(FI), MFFrame.getObjectAlignment(FI));
+      MFFrame.getObjectSize(FI), MFFrame.getObjectAlign(FI));
   return MIB.addFrameIndex(FI).addImm(Offset).addReg(0).addMemOperand(MMO);
 }
 
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrFormats.td b/llvm/lib/Target/SystemZ/SystemZInstrFormats.td
index f0c117f6d1f5f..94559b0d29820 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrFormats.td
+++ b/llvm/lib/Target/SystemZ/SystemZInstrFormats.td
@@ -4773,7 +4773,9 @@ multiclass BinaryRIAndKPseudo<string key, SDPatternOperator operator,
 class MemFoldPseudo<string mnemonic, RegisterOperand cls, bits<5> bytes,
                     AddressingMode mode>
   : Pseudo<(outs cls:$R1), (ins cls:$R2, mode:$XBD2), []> {
-    let OpKey = mnemonic#"rk"#cls;
+    let OpKey = !subst("mscrk", "msrkc",
+                !subst("msgcrk", "msgrkc",
+                mnemonic#"rk"#cls));
     let OpType = "mem";
     let MemKey = mnemonic#cls;
     let MemType = "pseudo";
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
index ee347ce65a6b3..5f42415cd62e7 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
@@ -513,8 +513,8 @@ unsigned SystemZInstrInfo::insertBranch(MachineBasicBlock &MBB,
   return Count;
 }
 
-bool SystemZInstrInfo::analyzeCompare(const MachineInstr &MI, unsigned &SrcReg,
-                                      unsigned &SrcReg2, int &Mask,
+bool SystemZInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
+                                      Register &SrcReg2, int &Mask,
                                       int &Value) const {
   assert(MI.isCompare() && "Caller should have checked for a comparison");
 
@@ -532,8 +532,8 @@ bool SystemZInstrInfo::analyzeCompare(const MachineInstr &MI, unsigned &SrcReg,
 
 bool SystemZInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
                                        ArrayRef<MachineOperand> Pred,
-                                       unsigned DstReg, unsigned TrueReg,
-                                       unsigned FalseReg, int &CondCycles,
+                                       Register DstReg, Register TrueReg,
+                                       Register FalseReg, int &CondCycles,
                                        int &TrueCycles,
                                        int &FalseCycles) const {
   // Not all subtargets have LOCR instructions.
@@ -566,10 +566,10 @@ bool SystemZInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
 
 void SystemZInstrInfo::insertSelect(MachineBasicBlock &MBB,
                                     MachineBasicBlock::iterator I,
-                                    const DebugLoc &DL, unsigned DstReg,
+                                    const DebugLoc &DL, Register DstReg,
                                     ArrayRef<MachineOperand> Pred,
-                                    unsigned TrueReg,
-                                    unsigned FalseReg) const {
+                                    Register TrueReg,
+                                    Register FalseReg) const {
   MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
   const TargetRegisterClass *RC = MRI.getRegClass(DstReg);
 
@@ -607,7 +607,7 @@ void SystemZInstrInfo::insertSelect(MachineBasicBlock &MBB,
 }
 
 bool SystemZInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
-                                     unsigned Reg,
+                                     Register Reg,
                                      MachineRegisterInfo *MRI) const {
   unsigned DefOpc = DefMI.getOpcode();
   if (DefOpc != SystemZ::LHIMux && DefOpc != SystemZ::LHI &&
@@ -1078,6 +1078,32 @@ MachineInstr *SystemZInstrInfo::foldMemoryOperandImpl(
     return BuiltMI;
   }
 
+  unsigned MemImmOpc = 0;
+  switch (Opcode) {
+  case SystemZ::LHIMux:
+  case SystemZ::LHI:    MemImmOpc = SystemZ::MVHI;  break;
+  case SystemZ::LGHI:   MemImmOpc = SystemZ::MVGHI; break;
+  case SystemZ::CHIMux:
+  case SystemZ::CHI:    MemImmOpc = SystemZ::CHSI;  break;
+  case SystemZ::CGHI:   MemImmOpc = SystemZ::CGHSI; break;
+  case SystemZ::CLFIMux:
+  case SystemZ::CLFI:
+    if (isUInt<16>(MI.getOperand(1).getImm()))
+      MemImmOpc = SystemZ::CLFHSI;
+    break;
+  case SystemZ::CLGFI:
+    if (isUInt<16>(MI.getOperand(1).getImm()))
+      MemImmOpc = SystemZ::CLGHSI;
+    break;
+  default: break;
+  }
+  if (MemImmOpc)
+    return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(),
+                   get(MemImmOpc))
+               .addFrameIndex(FrameIndex)
+               .addImm(0)
+               .addImm(MI.getOperand(1).getImm());
+
   if (Opcode == SystemZ::LGDR || Opcode == SystemZ::LDGR) {
     bool Op0IsGPR = (Opcode == SystemZ::LGDR);
     bool Op1IsGPR = (Opcode == SystemZ::LDGR);
@@ -1172,18 +1198,17 @@ MachineInstr *SystemZInstrInfo::foldMemoryOperandImpl(
   // See if this is a 3-address instruction that is convertible to 2-address
   // and suitable for folding below.  Only try this with virtual registers
   // and a provided VRM (during regalloc).
-  if (SystemZ::getTwoOperandOpcode(Opcode) != -1) {
+  if (NumOps == 3 && SystemZ::getTargetMemOpcode(MemOpcode) != -1) {
     if (VRM == nullptr)
       return nullptr;
     else {
-      assert(NumOps == 3 && "Expected two source registers.");
       Register DstReg = MI.getOperand(0).getReg();
       Register DstPhys =
           (Register::isVirtualRegister(DstReg) ? VRM->getPhys(DstReg) : DstReg);
       Register SrcReg = (OpNum == 2 ? MI.getOperand(1).getReg()
                                     : ((OpNum == 1 && MI.isCommutable())
                                            ? MI.getOperand(2).getReg()
-                                         : Register()));
+                                           : Register()));
       if (DstPhys && !SystemZ::GRH32BitRegClass.contains(DstPhys) && SrcReg &&
           Register::isVirtualRegister(SrcReg) &&
           DstPhys == VRM->getPhys(SrcReg))
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.h b/llvm/lib/Target/SystemZ/SystemZInstrInfo.h
index 56541903ea70d..2247f9f3fdbf2 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.h
@@ -223,16 +223,16 @@ class SystemZInstrInfo : public SystemZGenInstrInfo {
                         MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
                         const DebugLoc &DL,
                         int *BytesAdded = nullptr) const override;
-  bool analyzeCompare(const MachineInstr &MI, unsigned &SrcReg,
-                      unsigned &SrcReg2, int &Mask, int &Value) const override;
+  bool analyzeCompare(const MachineInstr &MI, Register &SrcReg,
+                      Register &SrcReg2, int &Mask, int &Value) const override;
   bool canInsertSelect(const MachineBasicBlock &, ArrayRef<MachineOperand> Cond,
-                       unsigned, unsigned, unsigned, int &, int &,
+                       Register, Register, Register, int &, int &,
                        int &) const override;
   void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
-                    const DebugLoc &DL, unsigned DstReg,
-                    ArrayRef<MachineOperand> Cond, unsigned TrueReg,
-                    unsigned FalseReg) const override;
-  bool FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, unsigned Reg,
+                    const DebugLoc &DL, Register DstReg,
+                    ArrayRef<MachineOperand> Cond, Register TrueReg,
+                    Register FalseReg) const override;
+  bool FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg,
                      MachineRegisterInfo *MRI) const override;
   bool isPredicable(const MachineInstr &MI) const override;
   bool isProfitableToIfCvt(MachineBasicBlock &MBB, unsigned NumCycles,
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.td b/llvm/lib/Target/SystemZ/SystemZInstrInfo.td
index fab6d267af87a..5e153f0d12791 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.td
+++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.td
@@ -1348,8 +1348,8 @@ def  MSG  : BinaryRXY<"msg",  0xE30C, mul, GR64, load, 8>;
 
 // Multiplication of memory, setting the condition code.
 let Predicates = [FeatureMiscellaneousExtensions2], Defs = [CC] in {
-  def MSC  : BinaryRXY<"msc",  0xE353, null_frag, GR32, load, 4>;
-  def MSGC : BinaryRXY<"msgc", 0xE383, null_frag, GR64, load, 8>;
+  defm MSC  : BinaryRXYAndPseudo<"msc",  0xE353, null_frag, GR32, load, 4>;
+  defm MSGC : BinaryRXYAndPseudo<"msgc", 0xE383, null_frag, GR64, load, 8>;
 }
 
 // Multiplication of a register, producing two results.
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrVector.td b/llvm/lib/Target/SystemZ/SystemZInstrVector.td
index c945122ee577a..f7e85f330b241 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrVector.td
+++ b/llvm/lib/Target/SystemZ/SystemZInstrVector.td
@@ -463,49 +463,56 @@ defm : GenericVectorOps<v2f64, v2i64>;
 //===----------------------------------------------------------------------===//
 
 let Predicates = [FeatureVector] in {
-  // Add.
-  def VA  : BinaryVRRcGeneric<"va", 0xE7F3>;
-  def VAB : BinaryVRRc<"vab", 0xE7F3, add, v128b, v128b, 0>;
-  def VAH : BinaryVRRc<"vah", 0xE7F3, add, v128h, v128h, 1>;
-  def VAF : BinaryVRRc<"vaf", 0xE7F3, add, v128f, v128f, 2>;
-  def VAG : BinaryVRRc<"vag", 0xE7F3, add, v128g, v128g, 3>;
-  def VAQ : BinaryVRRc<"vaq", 0xE7F3, int_s390_vaq, v128q, v128q, 4>;
-
-  // Add compute carry.
-  def VACC  : BinaryVRRcGeneric<"vacc", 0xE7F1>;
-  def VACCB : BinaryVRRc<"vaccb", 0xE7F1, int_s390_vaccb, v128b, v128b, 0>;
-  def VACCH : BinaryVRRc<"vacch", 0xE7F1, int_s390_vacch, v128h, v128h, 1>;
-  def VACCF : BinaryVRRc<"vaccf", 0xE7F1, int_s390_vaccf, v128f, v128f, 2>;
-  def VACCG : BinaryVRRc<"vaccg", 0xE7F1, int_s390_vaccg, v128g, v128g, 3>;
-  def VACCQ : BinaryVRRc<"vaccq", 0xE7F1, int_s390_vaccq, v128q, v128q, 4>;
-
-  // Add with carry.
-  def VAC  : TernaryVRRdGeneric<"vac", 0xE7BB>;
-  def VACQ : TernaryVRRd<"vacq", 0xE7BB, int_s390_vacq, v128q, v128q, 4>;
-
-  // Add with carry compute carry.
-  def VACCC  : TernaryVRRdGeneric<"vaccc", 0xE7B9>;
-  def VACCCQ : TernaryVRRd<"vacccq", 0xE7B9, int_s390_vacccq, v128q, v128q, 4>;
+  let isCommutable = 1 in {
+    // Add.
+    def VA  : BinaryVRRcGeneric<"va", 0xE7F3>;
+    def VAB : BinaryVRRc<"vab", 0xE7F3, add, v128b, v128b, 0>;
+    def VAH : BinaryVRRc<"vah", 0xE7F3, add, v128h, v128h, 1>;
+    def VAF : BinaryVRRc<"vaf", 0xE7F3, add, v128f, v128f, 2>;
+    def VAG : BinaryVRRc<"vag", 0xE7F3, add, v128g, v128g, 3>;
+    def VAQ : BinaryVRRc<"vaq", 0xE7F3, int_s390_vaq, v128q, v128q, 4>;
+  }
+
+  let isCommutable = 1 in {
+    // Add compute carry.
+    def VACC  : BinaryVRRcGeneric<"vacc", 0xE7F1>;
+    def VACCB : BinaryVRRc<"vaccb", 0xE7F1, int_s390_vaccb, v128b, v128b, 0>;
+    def VACCH : BinaryVRRc<"vacch", 0xE7F1, int_s390_vacch, v128h, v128h, 1>;
+    def VACCF : BinaryVRRc<"vaccf", 0xE7F1, int_s390_vaccf, v128f, v128f, 2>;
+    def VACCG : BinaryVRRc<"vaccg", 0xE7F1, int_s390_vaccg, v128g, v128g, 3>;
+    def VACCQ : BinaryVRRc<"vaccq", 0xE7F1, int_s390_vaccq, v128q, v128q, 4>;
+
+    // Add with carry.
+    def VAC  : TernaryVRRdGeneric<"vac", 0xE7BB>;
+    def VACQ : TernaryVRRd<"vacq", 0xE7BB, int_s390_vacq, v128q, v128q, 4>;
+
+    // Add with carry compute carry.
+    def VACCC  : TernaryVRRdGeneric<"vaccc", 0xE7B9>;
+    def VACCCQ : TernaryVRRd<"vacccq", 0xE7B9, int_s390_vacccq, v128q, v128q, 4>;
+ }
 
   // And.
-  def VN : BinaryVRRc<"vn", 0xE768, null_frag, v128any, v128any>;
+  let isCommutable = 1 in
+    def VN : BinaryVRRc<"vn", 0xE768, null_frag, v128any, v128any>;
 
   // And with complement.
   def VNC : BinaryVRRc<"vnc", 0xE769, null_frag, v128any, v128any>;
 
-  // Average.
-  def VAVG  : BinaryVRRcGeneric<"vavg", 0xE7F2>;
-  def VAVGB : BinaryVRRc<"vavgb", 0xE7F2, int_s390_vavgb, v128b, v128b, 0>;
-  def VAVGH : BinaryVRRc<"vavgh", 0xE7F2, int_s390_vavgh, v128h, v128h, 1>;
-  def VAVGF : BinaryVRRc<"vavgf", 0xE7F2, int_s390_vavgf, v128f, v128f, 2>;
-  def VAVGG : BinaryVRRc<"vavgg", 0xE7F2, int_s390_vavgg, v128g, v128g, 3>;
-
-  // Average logical.
-  def VAVGL  : BinaryVRRcGeneric<"vavgl", 0xE7F0>;
-  def VAVGLB : BinaryVRRc<"vavglb", 0xE7F0, int_s390_vavglb, v128b, v128b, 0>;
-  def VAVGLH : BinaryVRRc<"vavglh", 0xE7F0, int_s390_vavglh, v128h, v128h, 1>;
-  def VAVGLF : BinaryVRRc<"vavglf", 0xE7F0, int_s390_vavglf, v128f, v128f, 2>;
-  def VAVGLG : BinaryVRRc<"vavglg", 0xE7F0, int_s390_vavglg, v128g, v128g, 3>;
+  let isCommutable = 1 in {
+    // Average.
+    def VAVG  : BinaryVRRcGeneric<"vavg", 0xE7F2>;
+    def VAVGB : BinaryVRRc<"vavgb", 0xE7F2, int_s390_vavgb, v128b, v128b, 0>;
+    def VAVGH : BinaryVRRc<"vavgh", 0xE7F2, int_s390_vavgh, v128h, v128h, 1>;
+    def VAVGF : BinaryVRRc<"vavgf", 0xE7F2, int_s390_vavgf, v128f, v128f, 2>;
+    def VAVGG : BinaryVRRc<"vavgg", 0xE7F2, int_s390_vavgg, v128g, v128g, 3>;
+
+    // Average logical.
+    def VAVGL  : BinaryVRRcGeneric<"vavgl", 0xE7F0>;
+    def VAVGLB : BinaryVRRc<"vavglb", 0xE7F0, int_s390_vavglb, v128b, v128b, 0>;
+    def VAVGLH : BinaryVRRc<"vavglh", 0xE7F0, int_s390_vavglh, v128h, v128h, 1>;
+    def VAVGLF : BinaryVRRc<"vavglf", 0xE7F0, int_s390_vavglf, v128f, v128f, 2>;
+    def VAVGLG : BinaryVRRc<"vavglg", 0xE7F0, int_s390_vavglg, v128g, v128g, 3>;
+  }
 
   // Checksum.
   def VCKSM : BinaryVRRc<"vcksm", 0xE766, int_s390_vcksm, v128f, v128f>;
@@ -524,12 +531,14 @@ let Predicates = [FeatureVector] in {
   def VCTZF : UnaryVRRa<"vctzf", 0xE752, cttz, v128f, v128f, 2>;
   def VCTZG : UnaryVRRa<"vctzg", 0xE752, cttz, v128g, v128g, 3>;
 
-  // Not exclusive or.
-  let Predicates = [FeatureVectorEnhancements1] in
-    def VNX : BinaryVRRc<"vnx", 0xE76C, null_frag, v128any, v128any>;
+  let isCommutable = 1 in {
+    // Not exclusive or.
+    let Predicates = [FeatureVectorEnhancements1] in
+      def VNX : BinaryVRRc<"vnx", 0xE76C, null_frag, v128any, v128any>;
 
-  // Exclusive or.
-  def VX : BinaryVRRc<"vx", 0xE76D, null_frag, v128any, v128any>;
+    // Exclusive or.
+    def VX : BinaryVRRc<"vx", 0xE76D, null_frag, v128any, v128any>;
+  }
 
   // Galois field multiply sum.
   def VGFM  : BinaryVRRcGeneric<"vgfm", 0xE7B4>;
@@ -559,135 +568,145 @@ let Predicates = [FeatureVector] in {
   def VLPF : UnaryVRRa<"vlpf", 0xE7DF, z_viabs32, v128f, v128f, 2>;
   def VLPG : UnaryVRRa<"vlpg", 0xE7DF, z_viabs64, v128g, v128g, 3>;
 
-  // Maximum.
-  def VMX  : BinaryVRRcGeneric<"vmx", 0xE7FF>;
-  def VMXB : BinaryVRRc<"vmxb", 0xE7FF, null_frag, v128b, v128b, 0>;
-  def VMXH : BinaryVRRc<"vmxh", 0xE7FF, null_frag, v128h, v128h, 1>;
-  def VMXF : BinaryVRRc<"vmxf", 0xE7FF, null_frag, v128f, v128f, 2>;
-  def VMXG : BinaryVRRc<"vmxg", 0xE7FF, null_frag, v128g, v128g, 3>;
-
-  // Maximum logical.
-  def VMXL  : BinaryVRRcGeneric<"vmxl", 0xE7FD>;
-  def VMXLB : BinaryVRRc<"vmxlb", 0xE7FD, null_frag, v128b, v128b, 0>;
-  def VMXLH : BinaryVRRc<"vmxlh", 0xE7FD, null_frag, v128h, v128h, 1>;
-  def VMXLF : BinaryVRRc<"vmxlf", 0xE7FD, null_frag, v128f, v128f, 2>;
-  def VMXLG : BinaryVRRc<"vmxlg", 0xE7FD, null_frag, v128g, v128g, 3>;
+  let isCommutable = 1 in {
+    // Maximum.
+    def VMX  : BinaryVRRcGeneric<"vmx", 0xE7FF>;
+    def VMXB : BinaryVRRc<"vmxb", 0xE7FF, null_frag, v128b, v128b, 0>;
+    def VMXH : BinaryVRRc<"vmxh", 0xE7FF, null_frag, v128h, v128h, 1>;
+    def VMXF : BinaryVRRc<"vmxf", 0xE7FF, null_frag, v128f, v128f, 2>;
+    def VMXG : BinaryVRRc<"vmxg", 0xE7FF, null_frag, v128g, v128g, 3>;
+
+    // Maximum logical.
+    def VMXL  : BinaryVRRcGeneric<"vmxl", 0xE7FD>;
+    def VMXLB : BinaryVRRc<"vmxlb", 0xE7FD, null_frag, v128b, v128b, 0>;
+    def VMXLH : BinaryVRRc<"vmxlh", 0xE7FD, null_frag, v128h, v128h, 1>;
+    def VMXLF : BinaryVRRc<"vmxlf", 0xE7FD, null_frag, v128f, v128f, 2>;
+    def VMXLG : BinaryVRRc<"vmxlg", 0xE7FD, null_frag, v128g, v128g, 3>;
+  }
 
-  // Minimum.
-  def VMN  : BinaryVRRcGeneric<"vmn", 0xE7FE>;
-  def VMNB : BinaryVRRc<"vmnb", 0xE7FE, null_frag, v128b, v128b, 0>;
-  def VMNH : BinaryVRRc<"vmnh", 0xE7FE, null_frag, v128h, v128h, 1>;
-  def VMNF : BinaryVRRc<"vmnf", 0xE7FE, null_frag, v128f, v128f, 2>;
-  def VMNG : BinaryVRRc<"vmng", 0xE7FE, null_frag, v128g, v128g, 3>;
-
-  // Minimum logical.
-  def VMNL  : BinaryVRRcGeneric<"vmnl", 0xE7FC>;
-  def VMNLB : BinaryVRRc<"vmnlb", 0xE7FC, null_frag, v128b, v128b, 0>;
-  def VMNLH : BinaryVRRc<"vmnlh", 0xE7FC, null_frag, v128h, v128h, 1>;
-  def VMNLF : BinaryVRRc<"vmnlf", 0xE7FC, null_frag, v128f, v128f, 2>;
-  def VMNLG : BinaryVRRc<"vmnlg", 0xE7FC, null_frag, v128g, v128g, 3>;
-
-  // Multiply and add low.
-  def VMAL   : TernaryVRRdGeneric<"vmal", 0xE7AA>;
-  def VMALB  : TernaryVRRd<"vmalb",  0xE7AA, z_muladd, v128b, v128b, 0>;
-  def VMALHW : TernaryVRRd<"vmalhw", 0xE7AA, z_muladd, v128h, v128h, 1>;
-  def VMALF  : TernaryVRRd<"vmalf",  0xE7AA, z_muladd, v128f, v128f, 2>;
-
-  // Multiply and add high.
-  def VMAH  : TernaryVRRdGeneric<"vmah", 0xE7AB>;
-  def VMAHB : TernaryVRRd<"vmahb", 0xE7AB, int_s390_vmahb, v128b, v128b, 0>;
-  def VMAHH : TernaryVRRd<"vmahh", 0xE7AB, int_s390_vmahh, v128h, v128h, 1>;
-  def VMAHF : TernaryVRRd<"vmahf", 0xE7AB, int_s390_vmahf, v128f, v128f, 2>;
-
-  // Multiply and add logical high.
-  def VMALH  : TernaryVRRdGeneric<"vmalh", 0xE7A9>;
-  def VMALHB : TernaryVRRd<"vmalhb", 0xE7A9, int_s390_vmalhb, v128b, v128b, 0>;
-  def VMALHH : TernaryVRRd<"vmalhh", 0xE7A9, int_s390_vmalhh, v128h, v128h, 1>;
-  def VMALHF : TernaryVRRd<"vmalhf", 0xE7A9, int_s390_vmalhf, v128f, v128f, 2>;
-
-  // Multiply and add even.
-  def VMAE  : TernaryVRRdGeneric<"vmae", 0xE7AE>;
-  def VMAEB : TernaryVRRd<"vmaeb", 0xE7AE, int_s390_vmaeb, v128h, v128b, 0>;
-  def VMAEH : TernaryVRRd<"vmaeh", 0xE7AE, int_s390_vmaeh, v128f, v128h, 1>;
-  def VMAEF : TernaryVRRd<"vmaef", 0xE7AE, int_s390_vmaef, v128g, v128f, 2>;
-
-  // Multiply and add logical even.
-  def VMALE  : TernaryVRRdGeneric<"vmale", 0xE7AC>;
-  def VMALEB : TernaryVRRd<"vmaleb", 0xE7AC, int_s390_vmaleb, v128h, v128b, 0>;
-  def VMALEH : TernaryVRRd<"vmaleh", 0xE7AC, int_s390_vmaleh, v128f, v128h, 1>;
-  def VMALEF : TernaryVRRd<"vmalef", 0xE7AC, int_s390_vmalef, v128g, v128f, 2>;
-
-  // Multiply and add odd.
-  def VMAO  : TernaryVRRdGeneric<"vmao", 0xE7AF>;
-  def VMAOB : TernaryVRRd<"vmaob", 0xE7AF, int_s390_vmaob, v128h, v128b, 0>;
-  def VMAOH : TernaryVRRd<"vmaoh", 0xE7AF, int_s390_vmaoh, v128f, v128h, 1>;
-  def VMAOF : TernaryVRRd<"vmaof", 0xE7AF, int_s390_vmaof, v128g, v128f, 2>;
-
-  // Multiply and add logical odd.
-  def VMALO  : TernaryVRRdGeneric<"vmalo", 0xE7AD>;
-  def VMALOB : TernaryVRRd<"vmalob", 0xE7AD, int_s390_vmalob, v128h, v128b, 0>;
-  def VMALOH : TernaryVRRd<"vmaloh", 0xE7AD, int_s390_vmaloh, v128f, v128h, 1>;
-  def VMALOF : TernaryVRRd<"vmalof", 0xE7AD, int_s390_vmalof, v128g, v128f, 2>;
-
-  // Multiply high.
-  def VMH  : BinaryVRRcGeneric<"vmh", 0xE7A3>;
-  def VMHB : BinaryVRRc<"vmhb", 0xE7A3, int_s390_vmhb, v128b, v128b, 0>;
-  def VMHH : BinaryVRRc<"vmhh", 0xE7A3, int_s390_vmhh, v128h, v128h, 1>;
-  def VMHF : BinaryVRRc<"vmhf", 0xE7A3, int_s390_vmhf, v128f, v128f, 2>;
-
-  // Multiply logical high.
-  def VMLH  : BinaryVRRcGeneric<"vmlh", 0xE7A1>;
-  def VMLHB : BinaryVRRc<"vmlhb", 0xE7A1, int_s390_vmlhb, v128b, v128b, 0>;
-  def VMLHH : BinaryVRRc<"vmlhh", 0xE7A1, int_s390_vmlhh, v128h, v128h, 1>;
-  def VMLHF : BinaryVRRc<"vmlhf", 0xE7A1, int_s390_vmlhf, v128f, v128f, 2>;
-
-  // Multiply low.
-  def VML   : BinaryVRRcGeneric<"vml", 0xE7A2>;
-  def VMLB  : BinaryVRRc<"vmlb",  0xE7A2, mul, v128b, v128b, 0>;
-  def VMLHW : BinaryVRRc<"vmlhw", 0xE7A2, mul, v128h, v128h, 1>;
-  def VMLF  : BinaryVRRc<"vmlf",  0xE7A2, mul, v128f, v128f, 2>;
-
-  // Multiply even.
-  def VME  : BinaryVRRcGeneric<"vme", 0xE7A6>;
-  def VMEB : BinaryVRRc<"vmeb", 0xE7A6, int_s390_vmeb, v128h, v128b, 0>;
-  def VMEH : BinaryVRRc<"vmeh", 0xE7A6, int_s390_vmeh, v128f, v128h, 1>;
-  def VMEF : BinaryVRRc<"vmef", 0xE7A6, int_s390_vmef, v128g, v128f, 2>;
-
-  // Multiply logical even.
-  def VMLE  : BinaryVRRcGeneric<"vmle", 0xE7A4>;
-  def VMLEB : BinaryVRRc<"vmleb", 0xE7A4, int_s390_vmleb, v128h, v128b, 0>;
-  def VMLEH : BinaryVRRc<"vmleh", 0xE7A4, int_s390_vmleh, v128f, v128h, 1>;
-  def VMLEF : BinaryVRRc<"vmlef", 0xE7A4, int_s390_vmlef, v128g, v128f, 2>;
-
-  // Multiply odd.
-  def VMO  : BinaryVRRcGeneric<"vmo", 0xE7A7>;
-  def VMOB : BinaryVRRc<"vmob", 0xE7A7, int_s390_vmob, v128h, v128b, 0>;
-  def VMOH : BinaryVRRc<"vmoh", 0xE7A7, int_s390_vmoh, v128f, v128h, 1>;
-  def VMOF : BinaryVRRc<"vmof", 0xE7A7, int_s390_vmof, v128g, v128f, 2>;
-
-  // Multiply logical odd.
-  def VMLO  : BinaryVRRcGeneric<"vmlo", 0xE7A5>;
-  def VMLOB : BinaryVRRc<"vmlob", 0xE7A5, int_s390_vmlob, v128h, v128b, 0>;
-  def VMLOH : BinaryVRRc<"vmloh", 0xE7A5, int_s390_vmloh, v128f, v128h, 1>;
-  def VMLOF : BinaryVRRc<"vmlof", 0xE7A5, int_s390_vmlof, v128g, v128f, 2>;
+  let isCommutable = 1 in {
+    // Minimum.
+    def VMN  : BinaryVRRcGeneric<"vmn", 0xE7FE>;
+    def VMNB : BinaryVRRc<"vmnb", 0xE7FE, null_frag, v128b, v128b, 0>;
+    def VMNH : BinaryVRRc<"vmnh", 0xE7FE, null_frag, v128h, v128h, 1>;
+    def VMNF : BinaryVRRc<"vmnf", 0xE7FE, null_frag, v128f, v128f, 2>;
+    def VMNG : BinaryVRRc<"vmng", 0xE7FE, null_frag, v128g, v128g, 3>;
+
+    // Minimum logical.
+    def VMNL  : BinaryVRRcGeneric<"vmnl", 0xE7FC>;
+    def VMNLB : BinaryVRRc<"vmnlb", 0xE7FC, null_frag, v128b, v128b, 0>;
+    def VMNLH : BinaryVRRc<"vmnlh", 0xE7FC, null_frag, v128h, v128h, 1>;
+    def VMNLF : BinaryVRRc<"vmnlf", 0xE7FC, null_frag, v128f, v128f, 2>;
+    def VMNLG : BinaryVRRc<"vmnlg", 0xE7FC, null_frag, v128g, v128g, 3>;
+  }
+
+  let isCommutable = 1 in {
+    // Multiply and add low.
+    def VMAL   : TernaryVRRdGeneric<"vmal", 0xE7AA>;
+    def VMALB  : TernaryVRRd<"vmalb",  0xE7AA, z_muladd, v128b, v128b, 0>;
+    def VMALHW : TernaryVRRd<"vmalhw", 0xE7AA, z_muladd, v128h, v128h, 1>;
+    def VMALF  : TernaryVRRd<"vmalf",  0xE7AA, z_muladd, v128f, v128f, 2>;
+
+    // Multiply and add high.
+    def VMAH  : TernaryVRRdGeneric<"vmah", 0xE7AB>;
+    def VMAHB : TernaryVRRd<"vmahb", 0xE7AB, int_s390_vmahb, v128b, v128b, 0>;
+    def VMAHH : TernaryVRRd<"vmahh", 0xE7AB, int_s390_vmahh, v128h, v128h, 1>;
+    def VMAHF : TernaryVRRd<"vmahf", 0xE7AB, int_s390_vmahf, v128f, v128f, 2>;
+
+    // Multiply and add logical high.
+    def VMALH  : TernaryVRRdGeneric<"vmalh", 0xE7A9>;
+    def VMALHB : TernaryVRRd<"vmalhb", 0xE7A9, int_s390_vmalhb, v128b, v128b, 0>;
+    def VMALHH : TernaryVRRd<"vmalhh", 0xE7A9, int_s390_vmalhh, v128h, v128h, 1>;
+    def VMALHF : TernaryVRRd<"vmalhf", 0xE7A9, int_s390_vmalhf, v128f, v128f, 2>;
+
+    // Multiply and add even.
+    def VMAE  : TernaryVRRdGeneric<"vmae", 0xE7AE>;
+    def VMAEB : TernaryVRRd<"vmaeb", 0xE7AE, int_s390_vmaeb, v128h, v128b, 0>;
+    def VMAEH : TernaryVRRd<"vmaeh", 0xE7AE, int_s390_vmaeh, v128f, v128h, 1>;
+    def VMAEF : TernaryVRRd<"vmaef", 0xE7AE, int_s390_vmaef, v128g, v128f, 2>;
+
+    // Multiply and add logical even.
+    def VMALE  : TernaryVRRdGeneric<"vmale", 0xE7AC>;
+    def VMALEB : TernaryVRRd<"vmaleb", 0xE7AC, int_s390_vmaleb, v128h, v128b, 0>;
+    def VMALEH : TernaryVRRd<"vmaleh", 0xE7AC, int_s390_vmaleh, v128f, v128h, 1>;
+    def VMALEF : TernaryVRRd<"vmalef", 0xE7AC, int_s390_vmalef, v128g, v128f, 2>;
+
+    // Multiply and add odd.
+    def VMAO  : TernaryVRRdGeneric<"vmao", 0xE7AF>;
+    def VMAOB : TernaryVRRd<"vmaob", 0xE7AF, int_s390_vmaob, v128h, v128b, 0>;
+    def VMAOH : TernaryVRRd<"vmaoh", 0xE7AF, int_s390_vmaoh, v128f, v128h, 1>;
+    def VMAOF : TernaryVRRd<"vmaof", 0xE7AF, int_s390_vmaof, v128g, v128f, 2>;
+
+    // Multiply and add logical odd.
+    def VMALO  : TernaryVRRdGeneric<"vmalo", 0xE7AD>;
+    def VMALOB : TernaryVRRd<"vmalob", 0xE7AD, int_s390_vmalob, v128h, v128b, 0>;
+    def VMALOH : TernaryVRRd<"vmaloh", 0xE7AD, int_s390_vmaloh, v128f, v128h, 1>;
+    def VMALOF : TernaryVRRd<"vmalof", 0xE7AD, int_s390_vmalof, v128g, v128f, 2>;
+  }
+
+  let isCommutable = 1 in {
+    // Multiply high.
+    def VMH  : BinaryVRRcGeneric<"vmh", 0xE7A3>;
+    def VMHB : BinaryVRRc<"vmhb", 0xE7A3, int_s390_vmhb, v128b, v128b, 0>;
+    def VMHH : BinaryVRRc<"vmhh", 0xE7A3, int_s390_vmhh, v128h, v128h, 1>;
+    def VMHF : BinaryVRRc<"vmhf", 0xE7A3, int_s390_vmhf, v128f, v128f, 2>;
+
+    // Multiply logical high.
+    def VMLH  : BinaryVRRcGeneric<"vmlh", 0xE7A1>;
+    def VMLHB : BinaryVRRc<"vmlhb", 0xE7A1, int_s390_vmlhb, v128b, v128b, 0>;
+    def VMLHH : BinaryVRRc<"vmlhh", 0xE7A1, int_s390_vmlhh, v128h, v128h, 1>;
+    def VMLHF : BinaryVRRc<"vmlhf", 0xE7A1, int_s390_vmlhf, v128f, v128f, 2>;
+
+    // Multiply low.
+    def VML   : BinaryVRRcGeneric<"vml", 0xE7A2>;
+    def VMLB  : BinaryVRRc<"vmlb",  0xE7A2, mul, v128b, v128b, 0>;
+    def VMLHW : BinaryVRRc<"vmlhw", 0xE7A2, mul, v128h, v128h, 1>;
+    def VMLF  : BinaryVRRc<"vmlf",  0xE7A2, mul, v128f, v128f, 2>;
+
+    // Multiply even.
+    def VME  : BinaryVRRcGeneric<"vme", 0xE7A6>;
+    def VMEB : BinaryVRRc<"vmeb", 0xE7A6, int_s390_vmeb, v128h, v128b, 0>;
+    def VMEH : BinaryVRRc<"vmeh", 0xE7A6, int_s390_vmeh, v128f, v128h, 1>;
+    def VMEF : BinaryVRRc<"vmef", 0xE7A6, int_s390_vmef, v128g, v128f, 2>;
+
+    // Multiply logical even.
+    def VMLE  : BinaryVRRcGeneric<"vmle", 0xE7A4>;
+    def VMLEB : BinaryVRRc<"vmleb", 0xE7A4, int_s390_vmleb, v128h, v128b, 0>;
+    def VMLEH : BinaryVRRc<"vmleh", 0xE7A4, int_s390_vmleh, v128f, v128h, 1>;
+    def VMLEF : BinaryVRRc<"vmlef", 0xE7A4, int_s390_vmlef, v128g, v128f, 2>;
+
+    // Multiply odd.
+    def VMO  : BinaryVRRcGeneric<"vmo", 0xE7A7>;
+    def VMOB : BinaryVRRc<"vmob", 0xE7A7, int_s390_vmob, v128h, v128b, 0>;
+    def VMOH : BinaryVRRc<"vmoh", 0xE7A7, int_s390_vmoh, v128f, v128h, 1>;
+    def VMOF : BinaryVRRc<"vmof", 0xE7A7, int_s390_vmof, v128g, v128f, 2>;
+
+    // Multiply logical odd.
+    def VMLO  : BinaryVRRcGeneric<"vmlo", 0xE7A5>;
+    def VMLOB : BinaryVRRc<"vmlob", 0xE7A5, int_s390_vmlob, v128h, v128b, 0>;
+    def VMLOH : BinaryVRRc<"vmloh", 0xE7A5, int_s390_vmloh, v128f, v128h, 1>;
+    def VMLOF : BinaryVRRc<"vmlof", 0xE7A5, int_s390_vmlof, v128g, v128f, 2>;
+  }
 
   // Multiply sum logical.
-  let Predicates = [FeatureVectorEnhancements1] in {
+  let Predicates = [FeatureVectorEnhancements1], isCommutable = 1 in {
     def VMSL  : QuaternaryVRRdGeneric<"vmsl", 0xE7B8>;
     def VMSLG : QuaternaryVRRd<"vmslg", 0xE7B8, int_s390_vmslg,
                                v128q, v128g, v128g, v128q, 3>;
   }
 
   // Nand.
-  let Predicates = [FeatureVectorEnhancements1] in
+  let Predicates = [FeatureVectorEnhancements1], isCommutable = 1 in
     def VNN : BinaryVRRc<"vnn", 0xE76E, null_frag, v128any, v128any>;
 
   // Nor.
-  def VNO : BinaryVRRc<"vno", 0xE76B, null_frag, v128any, v128any>;
+  let isCommutable = 1 in
+    def VNO : BinaryVRRc<"vno", 0xE76B, null_frag, v128any, v128any>;
   def : InstAlias<"vnot\t$V1, $V2", (VNO VR128:$V1, VR128:$V2, VR128:$V2), 0>;
 
   // Or.
-  def VO : BinaryVRRc<"vo", 0xE76A, null_frag, v128any, v128any>;
+  let isCommutable = 1 in
+    def VO : BinaryVRRc<"vo", 0xE76A, null_frag, v128any, v128any>;
 
   // Or with complement.
   let Predicates = [FeatureVectorEnhancements1] in
@@ -1017,7 +1036,7 @@ multiclass VectorRounding<Instruction insn, TypedReg tr> {
 
 let Predicates = [FeatureVector] in {
   // Add.
-  let Uses = [FPC], mayRaiseFPException = 1 in {
+  let Uses = [FPC], mayRaiseFPException = 1, isCommutable = 1 in {
     def VFA   : BinaryVRRcFloatGeneric<"vfa", 0xE7E3>;
     def VFADB : BinaryVRRc<"vfadb", 0xE7E3, any_fadd, v128db, v128db, 3, 0>;
     def WFADB : BinaryVRRc<"wfadb", 0xE7E3, any_fadd, v64db, v64db, 3, 8>;
@@ -1178,7 +1197,7 @@ let Predicates = [FeatureVector] in {
     def : FPMinMax<insn, any_fmaximum, tr, 1>;
   }
   let Predicates = [FeatureVectorEnhancements1] in {
-    let Uses = [FPC], mayRaiseFPException = 1 in {
+    let Uses = [FPC], mayRaiseFPException = 1, isCommutable = 1 in {
       def VFMAX   : TernaryVRRcFloatGeneric<"vfmax", 0xE7EF>;
       def VFMAXDB : TernaryVRRcFloat<"vfmaxdb", 0xE7EF, int_s390_vfmaxdb,
                                      v128db, v128db, 3, 0>;
@@ -1204,7 +1223,7 @@ let Predicates = [FeatureVector] in {
     def : FPMinMax<insn, any_fminimum, tr, 1>;
   }
   let Predicates = [FeatureVectorEnhancements1] in {
-    let Uses = [FPC], mayRaiseFPException = 1 in {
+    let Uses = [FPC], mayRaiseFPException = 1, isCommutable = 1 in {
       def VFMIN   : TernaryVRRcFloatGeneric<"vfmin", 0xE7EE>;
       def VFMINDB : TernaryVRRcFloat<"vfmindb", 0xE7EE, int_s390_vfmindb,
                                      v128db, v128db, 3, 0>;
@@ -1225,7 +1244,7 @@ let Predicates = [FeatureVector] in {
   }
 
   // Multiply.
-  let Uses = [FPC], mayRaiseFPException = 1 in {
+  let Uses = [FPC], mayRaiseFPException = 1, isCommutable = 1 in {
     def VFM   : BinaryVRRcFloatGeneric<"vfm", 0xE7E7>;
     def VFMDB : BinaryVRRc<"vfmdb", 0xE7E7, any_fmul, v128db, v128db, 3, 0>;
     def WFMDB : BinaryVRRc<"wfmdb", 0xE7E7, any_fmul, v64db, v64db, 3, 8>;
@@ -1237,7 +1256,7 @@ let Predicates = [FeatureVector] in {
   }
 
   // Multiply and add.
-  let Uses = [FPC], mayRaiseFPException = 1 in {
+  let Uses = [FPC], mayRaiseFPException = 1, isCommutable = 1 in {
     def VFMA   : TernaryVRReFloatGeneric<"vfma", 0xE78F>;
     def VFMADB : TernaryVRRe<"vfmadb", 0xE78F, any_fma, v128db, v128db, 0, 3>;
     def WFMADB : TernaryVRRe<"wfmadb", 0xE78F, any_fma, v64db, v64db, 8, 3>;
@@ -1249,7 +1268,7 @@ let Predicates = [FeatureVector] in {
   }
 
   // Multiply and subtract.
-  let Uses = [FPC], mayRaiseFPException = 1 in {
+  let Uses = [FPC], mayRaiseFPException = 1, isCommutable = 1 in {
     def VFMS   : TernaryVRReFloatGeneric<"vfms", 0xE78E>;
     def VFMSDB : TernaryVRRe<"vfmsdb", 0xE78E, any_fms, v128db, v128db, 0, 3>;
     def WFMSDB : TernaryVRRe<"wfmsdb", 0xE78E, any_fms, v64db, v64db, 8, 3>;
@@ -1261,7 +1280,7 @@ let Predicates = [FeatureVector] in {
   }
 
   // Negative multiply and add.
-  let Uses = [FPC], mayRaiseFPException = 1,
+  let Uses = [FPC], mayRaiseFPException = 1, isCommutable = 1,
       Predicates = [FeatureVectorEnhancements1] in {
     def VFNMA   : TernaryVRReFloatGeneric<"vfnma", 0xE79F>;
     def VFNMADB : TernaryVRRe<"vfnmadb", 0xE79F, any_fnma, v128db, v128db, 0, 3>;
@@ -1272,7 +1291,7 @@ let Predicates = [FeatureVector] in {
   }
 
   // Negative multiply and subtract.
-  let Uses = [FPC], mayRaiseFPException = 1,
+  let Uses = [FPC], mayRaiseFPException = 1, isCommutable = 1,
       Predicates = [FeatureVectorEnhancements1] in {
     def VFNMS   : TernaryVRReFloatGeneric<"vfnms", 0xE79E>;
     def VFNMSDB : TernaryVRRe<"vfnmsdb", 0xE79E, any_fnms, v128db, v128db, 0, 3>;
diff --git a/llvm/lib/Target/SystemZ/SystemZShortenInst.cpp b/llvm/lib/Target/SystemZ/SystemZShortenInst.cpp
index 092d3a2bcc30f..3d27b70d6ef9d 100644
--- a/llvm/lib/Target/SystemZ/SystemZShortenInst.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZShortenInst.cpp
@@ -65,7 +65,7 @@ SystemZShortenInst::SystemZShortenInst(const SystemZTargetMachine &tm)
 
 // Tie operands if MI has become a two-address instruction.
 static void tieOpsIfNeeded(MachineInstr &MI) {
-  if (MI.getDesc().getOperandConstraint(0, MCOI::TIED_TO) &&
+  if (MI.getDesc().getOperandConstraint(1, MCOI::TIED_TO) == 0 &&
       !MI.getOperand(0).isTied())
     MI.tieOperands(0, 1);
 }
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
index d088682cf7d30..84ab66d87c3ba 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
@@ -323,6 +323,23 @@ unsigned SystemZTTIImpl::getRegisterBitWidth(bool Vector) const {
   return 0;
 }
 
+unsigned SystemZTTIImpl::getMinPrefetchStride(unsigned NumMemAccesses,
+                                              unsigned NumStridedMemAccesses,
+                                              unsigned NumPrefetches,
+                                              bool HasCall) const {
+  // Don't prefetch a loop with many far apart accesses.
+  if (NumPrefetches > 16)
+    return UINT_MAX;
+
+  // Emit prefetch instructions for smaller strides in cases where we think
+  // the hardware prefetcher might not be able to keep up.
+  if (NumStridedMemAccesses > 32 &&
+      NumStridedMemAccesses == NumMemAccesses && !HasCall)
+    return 1;
+
+  return ST->hasMiscellaneousExtensions3() ? 8192 : 2048;
+}
+
 bool SystemZTTIImpl::hasDivRemOp(Type *DataType, bool IsSigned) {
   EVT VT = TLI->getValueType(DL, DataType);
   return (VT.isScalarInteger() && TLI->isTypeLegal(VT));
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
index 590505769c705..c6e3b36bd98ed 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
@@ -60,8 +60,12 @@ class SystemZTTIImpl : public BasicTTIImplBase<SystemZTTIImpl> {
   unsigned getRegisterBitWidth(bool Vector) const;
 
   unsigned getCacheLineSize() const override { return 256; }
-  unsigned getPrefetchDistance() const override { return 2000; }
-  unsigned getMinPrefetchStride() const override { return 2048; }
+  unsigned getPrefetchDistance() const override { return 4500; }
+  unsigned getMinPrefetchStride(unsigned NumMemAccesses,
+                                unsigned NumStridedMemAccesses,
+                                unsigned NumPrefetches,
+                                bool HasCall) const override;
+  bool enableWritePrefetching() const override { return true; }
 
   bool hasDivRemOp(Type *DataType, bool IsSigned);
   bool prefersVectorizedAddressing() { return false; }
diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEInstPrinter.cpp b/llvm/lib/Target/VE/MCTargetDesc/VEInstPrinter.cpp
index 1229a8c6dc711..27dc3a0a000a1 100644
--- a/llvm/lib/Target/VE/MCTargetDesc/VEInstPrinter.cpp
+++ b/llvm/lib/Target/VE/MCTargetDesc/VEInstPrinter.cpp
@@ -42,7 +42,7 @@ void VEInstPrinter::printRegName(raw_ostream &OS, unsigned RegNo) const {
 void VEInstPrinter::printInst(const MCInst *MI, uint64_t Address,
                               StringRef Annot, const MCSubtargetInfo &STI,
                               raw_ostream &OS) {
-  if (!printAliasInstr(MI, STI, OS))
+  if (!printAliasInstr(MI, Address, STI, OS))
     printInstruction(MI, Address, STI, OS);
   printAnnotation(OS, Annot);
 }
diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEInstPrinter.h b/llvm/lib/Target/VE/MCTargetDesc/VEInstPrinter.h
index 05a53d59e8782..90c9f7b717502 100644
--- a/llvm/lib/Target/VE/MCTargetDesc/VEInstPrinter.h
+++ b/llvm/lib/Target/VE/MCTargetDesc/VEInstPrinter.h
@@ -28,7 +28,8 @@ class VEInstPrinter : public MCInstPrinter {
                  const MCSubtargetInfo &STI, raw_ostream &OS) override;
 
   // Autogenerated by tblgen.
-  bool printAliasInstr(const MCInst *, const MCSubtargetInfo &, raw_ostream &);
+  bool printAliasInstr(const MCInst *, uint64_t Address,
+                       const MCSubtargetInfo &, raw_ostream &);
   void printInstruction(const MCInst *, uint64_t, const MCSubtargetInfo &,
                         raw_ostream &);
   static const char *getRegisterName(unsigned RegNo);
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyCFGSort.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyCFGSort.cpp
index b4354e852194b..b186e32e788de 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyCFGSort.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyCFGSort.cpp
@@ -79,7 +79,6 @@ template <> bool ConcreteRegion<MachineLoop>::isLoop() const { return true; }
 class RegionInfo {
   const MachineLoopInfo &MLI;
   const WebAssemblyExceptionInfo &WEI;
-  std::vector<const Region *> Regions;
   DenseMap<const MachineLoop *, std::unique_ptr<Region>> LoopMap;
   DenseMap<const WebAssemblyException *, std::unique_ptr<Region>> ExceptionMap;
 
@@ -93,7 +92,14 @@ class RegionInfo {
     const auto *WE = WEI.getExceptionFor(MBB);
     if (!ML && !WE)
       return nullptr;
-    if ((ML && !WE) || (ML && WE && ML->getNumBlocks() < WE->getNumBlocks())) {
+    // We determine subregion relationship by domination of their headers, i.e.,
+    // if region A's header dominates region B's header, B is a subregion of A.
+    // WebAssemblyException contains BBs in all its subregions (loops or
+    // exceptions), but MachineLoop may not, because MachineLoop does not contain
+    // BBs that don't have a path to its header even if they are dominated by
+    // its header. So here we should use WE->contains(ML->getHeader()), but not
+    // ML->contains(WE->getHeader()).
+    if ((ML && !WE) || (ML && WE && WE->contains(ML->getHeader()))) {
       // If the smallest region containing MBB is a loop
       if (LoopMap.count(ML))
         return LoopMap[ML].get();
@@ -368,6 +374,7 @@ static void sortBlocks(MachineFunction &MF, const MachineLoopInfo &MLI,
     const Region *Region = RI.getRegionFor(&MBB);
 
     if (Region && &MBB == Region->getHeader()) {
+      // Region header.
       if (Region->isLoop()) {
         // Loop header. The loop predecessor should be sorted above, and the
         // other predecessors should be backedges below.
@@ -377,7 +384,7 @@ static void sortBlocks(MachineFunction &MF, const MachineLoopInfo &MLI,
               "Loop header predecessors must be loop predecessors or "
               "backedges");
       } else {
-        // Not a loop header. All predecessors should be sorted above.
+        // Exception header. All predecessors should be sorted above.
         for (auto Pred : MBB.predecessors())
           assert(Pred->getNumber() < MBB.getNumber() &&
                  "Non-loop-header predecessors should be topologically sorted");
@@ -386,7 +393,7 @@ static void sortBlocks(MachineFunction &MF, const MachineLoopInfo &MLI,
              "Regions should be declared at most once.");
 
     } else {
-      // Not a loop header. All predecessors should be sorted above.
+      // Not a region header. All predecessors should be sorted above.
       for (auto Pred : MBB.predecessors())
         assert(Pred->getNumber() < MBB.getNumber() &&
                "Non-loop-header predecessors should be topologically sorted");
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp
index 56da3ec119025..03b70f3a6a24a 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp
@@ -313,6 +313,8 @@ bool WebAssemblyExplicitLocals::runOnMachineFunction(MachineFunction &MF) {
                     .addReg(NewReg);
             // After the drop instruction, this reg operand will not be used
             Drop->getOperand(0).setIsKill();
+            if (MFI.isFrameBaseVirtual() && OldReg == MFI.getFrameBaseVreg())
+              MFI.clearFrameBaseVreg();
           } else {
             unsigned LocalId = getLocalId(Reg2Local, MFI, CurLocal, OldReg);
             unsigned Opc = getLocalSetOpcode(RC);
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp
index 89265a3a95204..c8878a48b243a 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp
@@ -344,6 +344,21 @@ static std::string getSignature(FunctionType *FTy) {
   return Sig;
 }
 
+static void markAsImported(Function *F) {
+  // Tell the linker that this function is expected to be imported from the
+  // 'env' module.
+  if (!F->hasFnAttribute("wasm-import-module")) {
+    llvm::AttrBuilder B;
+    B.addAttribute("wasm-import-module", "env");
+    F->addAttributes(llvm::AttributeList::FunctionIndex, B);
+  }
+  if (!F->hasFnAttribute("wasm-import-name")) {
+    llvm::AttrBuilder B;
+    B.addAttribute("wasm-import-name", F->getName());
+    F->addAttributes(llvm::AttributeList::FunctionIndex, B);
+  }
+}
+
 // Returns __cxa_find_matching_catch_N function, where N = NumClauses + 2.
 // This is because a landingpad instruction contains two more arguments, a
 // personality function and a cleanup bit, and __cxa_find_matching_catch_N
@@ -360,6 +375,7 @@ WebAssemblyLowerEmscriptenEHSjLj::getFindMatchingCatch(Module &M,
   Function *F = Function::Create(
       FTy, GlobalValue::ExternalLinkage,
       "__cxa_find_matching_catch_" + Twine(NumClauses + 2), &M);
+  markAsImported(F);
   FindMatchingCatches[NumClauses] = F;
   return F;
 }
@@ -469,6 +485,7 @@ Function *WebAssemblyLowerEmscriptenEHSjLj::getInvokeWrapper(CallOrInvoke *CI) {
                                         CalleeFTy->isVarArg());
   Function *F =
       Function::Create(FTy, GlobalValue::ExternalLinkage, "__invoke_" + Sig, M);
+  markAsImported(F);
   InvokeWrappers[Sig] = F;
   return F;
 }
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyLowerGlobalDtors.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyLowerGlobalDtors.cpp
index 750b2233e67ab..9ccbee819c357 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyLowerGlobalDtors.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyLowerGlobalDtors.cpp
@@ -76,9 +76,13 @@ bool LowerGlobalDtors::runOnModule(Module &M) {
       !ETy->getTypeAtIndex(2U)->isPointerTy())
     return false; // Not (int, ptr, ptr).
 
-  // Collect the contents of @llvm.global_dtors, collated by priority and
-  // associated symbol.
-  std::map<uint16_t, MapVector<Constant *, std::vector<Constant *>>> DtorFuncs;
+  // Collect the contents of @llvm.global_dtors, ordered by priority. Within a
+  // priority, sequences of destructors with the same associated object are
+  // recorded so that we can register them as a group.
+  std::map<
+      uint16_t,
+      std::vector<std::pair<Constant *, std::vector<Constant *>>>
+  > DtorFuncs;
   for (Value *O : InitList->operands()) {
     auto *CS = dyn_cast<ConstantStruct>(O);
     if (!CS)
@@ -96,7 +100,14 @@ bool LowerGlobalDtors::runOnModule(Module &M) {
     Constant *Associated = CS->getOperand(2);
     Associated = cast<Constant>(Associated->stripPointerCasts());
 
-    DtorFuncs[PriorityValue][Associated].push_back(DtorFunc);
+    auto &AtThisPriority = DtorFuncs[PriorityValue];
+    if (AtThisPriority.empty() || AtThisPriority.back().first != Associated) {
+        std::vector<Constant *> NewList;
+        NewList.push_back(DtorFunc);
+        AtThisPriority.push_back(std::make_pair(Associated, NewList));
+    } else {
+        AtThisPriority.back().second.push_back(DtorFunc);
+    }
   }
   if (DtorFuncs.empty())
     return false;
@@ -131,14 +142,19 @@ bool LowerGlobalDtors::runOnModule(Module &M) {
   // first function with __cxa_atexit.
   for (auto &PriorityAndMore : DtorFuncs) {
     uint16_t Priority = PriorityAndMore.first;
-    for (auto &AssociatedAndMore : PriorityAndMore.second) {
+    uint64_t Id = 0;
+    auto &AtThisPriority = PriorityAndMore.second;
+    for (auto &AssociatedAndMore : AtThisPriority) {
       Constant *Associated = AssociatedAndMore.first;
+      auto ThisId = Id++;
 
       Function *CallDtors = Function::Create(
           AtExitFuncTy, Function::PrivateLinkage,
           "call_dtors" +
               (Priority != UINT16_MAX ? (Twine(".") + Twine(Priority))
                                       : Twine()) +
+              (AtThisPriority.size() > 1 ? Twine("$") + Twine(ThisId)
+                                         : Twine()) +
               (!Associated->isNullValue() ? (Twine(".") + Associated->getName())
                                           : Twine()),
           &M);
@@ -146,7 +162,7 @@ bool LowerGlobalDtors::runOnModule(Module &M) {
       FunctionType *VoidVoid = FunctionType::get(Type::getVoidTy(C),
                                                  /*isVarArg=*/false);
 
-      for (auto Dtor : AssociatedAndMore.second)
+      for (auto Dtor : reverse(AssociatedAndMore.second))
         CallInst::Create(VoidVoid, Dtor, "", BB);
       ReturnInst::Create(C, BB);
 
@@ -155,6 +171,8 @@ bool LowerGlobalDtors::runOnModule(Module &M) {
           "register_call_dtors" +
               (Priority != UINT16_MAX ? (Twine(".") + Twine(Priority))
                                       : Twine()) +
+              (AtThisPriority.size() > 1 ? Twine("$") + Twine(ThisId)
+                                         : Twine()) +
               (!Associated->isNullValue() ? (Twine(".") + Associated->getName())
                                           : Twine()),
           &M);
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblySetP2AlignOperands.cpp b/llvm/lib/Target/WebAssembly/WebAssemblySetP2AlignOperands.cpp
index a249ccf176386..89ae45722e429 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblySetP2AlignOperands.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblySetP2AlignOperands.cpp
@@ -65,7 +65,7 @@ static void rewriteP2Align(MachineInstr &MI, unsigned OperandNo) {
   assert(MI.getDesc().OpInfo[OperandNo].OperandType ==
              WebAssembly::OPERAND_P2ALIGN &&
          "Load and store instructions should have a p2align operand");
-  uint64_t P2Align = Log2_64((*MI.memoperands_begin())->getAlignment());
+  uint64_t P2Align = Log2((*MI.memoperands_begin())->getAlign());
 
   // WebAssembly does not currently support supernatural alignment.
   P2Align = std::min(P2Align,
diff --git a/llvm/lib/Target/X86/CMakeLists.txt b/llvm/lib/Target/X86/CMakeLists.txt
index 3f0d68c0c7889..e94a82aebca46 100644
--- a/llvm/lib/Target/X86/CMakeLists.txt
+++ b/llvm/lib/Target/X86/CMakeLists.txt
@@ -44,6 +44,7 @@ set(sources
   X86ISelDAGToDAG.cpp
   X86ISelLowering.cpp
   X86IndirectBranchTracking.cpp
+  X86IndirectThunks.cpp
   X86InterleavedAccess.cpp
   X86InsertPrefetch.cpp
   X86InstrFMA3Info.cpp
@@ -51,14 +52,16 @@ set(sources
   X86InstrInfo.cpp
   X86EvexToVex.cpp
   X86LegalizerInfo.cpp
+  X86LoadValueInjectionLoadHardening.cpp
+  X86LoadValueInjectionRetHardening.cpp
   X86MCInstLower.cpp
   X86MachineFunctionInfo.cpp
   X86MacroFusion.cpp
   X86OptimizeLEAs.cpp
   X86PadShortFunction.cpp
+  X86PartialReduction.cpp
   X86RegisterBankInfo.cpp
   X86RegisterInfo.cpp
-  X86RetpolineThunks.cpp
   X86SelectionDAGInfo.cpp
   X86ShuffleDecodeConstantPool.cpp
   X86SpeculativeLoadHardening.cpp
diff --git a/llvm/lib/Target/X86/ImmutableGraph.h b/llvm/lib/Target/X86/ImmutableGraph.h
new file mode 100644
index 0000000000000..80c9cf489dedd
--- /dev/null
+++ b/llvm/lib/Target/X86/ImmutableGraph.h
@@ -0,0 +1,432 @@
+//==========-- ImmutableGraph.h - A fast DAG implementation ---------=========//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// Description: ImmutableGraph is a fast DAG implementation that cannot be
+/// modified, except by creating a new ImmutableGraph. ImmutableGraph is
+/// implemented as two arrays: one containing nodes, and one containing edges.
+/// The advantages to this implementation are two-fold:
+/// 1. Iteration and traversal operations should experience terrific caching
+///    performance.
+/// 2. Set representations and operations on nodes and edges become
+///    extraordinarily efficient. For instance, a set of edges is implemented as
+///    a bit vector, wherein each bit corresponds to one edge in the edge
+///    array. This implies a lower bound of 64x spacial improvement over, e.g.,
+///    an llvm::DenseSet or llvm::SmallSet. It also means that
+///    insert/erase/contains operations complete in negligible constant time:
+///    insert and erase require one load and one store, and contains requires
+///    just one load.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef IMMUTABLEGRAPH_H
+#define IMMUTABLEGRAPH_H
+
+#include "llvm/ADT/BitVector.h"
+#include "llvm/ADT/GraphTraits.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/raw_ostream.h"
+#include <algorithm>
+#include <iterator>
+#include <utility>
+#include <vector>
+
+namespace llvm {
+
+template <typename _NodeValueT, typename _EdgeValueT, typename _SizeT = int>
+class ImmutableGraph {
+  using Traits = GraphTraits<ImmutableGraph<_NodeValueT, _EdgeValueT> *>;
+  template <typename> friend class ImmutableGraphBuilder;
+
+public:
+  using NodeValueT = _NodeValueT;
+  using EdgeValueT = _EdgeValueT;
+  using size_type = _SizeT;
+  class Node;
+  class Edge {
+    friend class ImmutableGraph;
+    template <typename> friend class ImmutableGraphBuilder;
+    friend Traits;
+
+    Node *__dest;
+    EdgeValueT __value;
+
+  public:
+    EdgeValueT &value() { return __value; }
+  };
+  class Node {
+    friend class ImmutableGraph;
+    template <typename> friend class ImmutableGraphBuilder;
+    friend Traits;
+
+    Edge *__edges;
+    NodeValueT __value;
+
+  public:
+    NodeValueT &value() { return __value; }
+  };
+
+protected:
+  ImmutableGraph(Node *Nodes, size_type NodesSize, Edge *Edges,
+                 size_type EdgesSize)
+      : __nodes{Nodes}, __nodes_size{NodesSize}, __edges{Edges},
+        __edges_size{EdgesSize} {}
+  ImmutableGraph(const ImmutableGraph &) = delete;
+  ImmutableGraph(ImmutableGraph &&) = delete;
+  ImmutableGraph &operator=(const ImmutableGraph &) = delete;
+  ImmutableGraph &operator=(ImmutableGraph &&) = delete;
+
+public:
+  ~ImmutableGraph() {
+    delete[] __edges;
+    delete[] __nodes;
+  }
+
+  Node *nodes_begin() const { return __nodes; }
+  Node *nodes_end() const { return __nodes + __nodes_size; }
+  Edge *edges_begin() const { return __edges; }
+  Edge *edges_end() const { return __edges + __edges_size; }
+  size_type nodes_size() const { return __nodes_size; }
+  size_type edges_size() const { return __edges_size; }
+  bool empty() const { return __nodes_size == 0; }
+
+  class NodeSet {
+    friend class iterator;
+
+    const ImmutableGraph &__g;
+    BitVector __v;
+
+  public:
+    NodeSet(const ImmutableGraph &G, bool ContainsAll = false)
+        : __g{G}, __v{static_cast<unsigned>(__g.nodes_size()), ContainsAll} {}
+    bool insert(Node *N) {
+      size_type Idx = std::distance(__g.nodes_begin(), N);
+      bool AlreadyExists = __v.test(Idx);
+      __v.set(Idx);
+      return !AlreadyExists;
+    }
+    void erase(Node *N) {
+      size_type Idx = std::distance(__g.nodes_begin(), N);
+      __v.reset(Idx);
+    }
+    bool contains(Node *N) const {
+      size_type Idx = std::distance(__g.nodes_begin(), N);
+      return __v.test(Idx);
+    }
+    void clear() { __v.reset(); }
+    size_type empty() const { return __v.none(); }
+    /// Return the number of elements in the set
+    size_type count() const { return __v.count(); }
+    /// Return the size of the set's domain
+    size_type size() const { return __v.size(); }
+    /// Set union
+    NodeSet &operator|=(const NodeSet &RHS) {
+      assert(&this->__g == &RHS.__g);
+      __v |= RHS.__v;
+      return *this;
+    }
+    /// Set intersection
+    NodeSet &operator&=(const NodeSet &RHS) {
+      assert(&this->__g == &RHS.__g);
+      __v &= RHS.__v;
+      return *this;
+    }
+    /// Set disjoint union
+    NodeSet &operator^=(const NodeSet &RHS) {
+      assert(&this->__g == &RHS.__g);
+      __v ^= RHS.__v;
+      return *this;
+    }
+
+    using index_iterator = typename BitVector::const_set_bits_iterator;
+    index_iterator index_begin() const { return __v.set_bits_begin(); }
+    index_iterator index_end() const { return __v.set_bits_end(); }
+    void set(size_type Idx) { __v.set(Idx); }
+    void reset(size_type Idx) { __v.reset(Idx); }
+
+    class iterator {
+      const NodeSet &__set;
+      size_type __current;
+
+      void advance() {
+        assert(__current != -1);
+        __current = __set.__v.find_next(__current);
+      }
+
+    public:
+      iterator(const NodeSet &Set, size_type Begin)
+          : __set{Set}, __current{Begin} {}
+      iterator operator++(int) {
+        iterator Tmp = *this;
+        advance();
+        return Tmp;
+      }
+      iterator &operator++() {
+        advance();
+        return *this;
+      }
+      Node *operator*() const {
+        assert(__current != -1);
+        return __set.__g.nodes_begin() + __current;
+      }
+      bool operator==(const iterator &other) const {
+        assert(&this->__set == &other.__set);
+        return this->__current == other.__current;
+      }
+      bool operator!=(const iterator &other) const { return !(*this == other); }
+    };
+
+    iterator begin() const { return iterator{*this, __v.find_first()}; }
+    iterator end() const { return iterator{*this, -1}; }
+  };
+
+  class EdgeSet {
+    const ImmutableGraph &__g;
+    BitVector __v;
+
+  public:
+    EdgeSet(const ImmutableGraph &G, bool ContainsAll = false)
+        : __g{G}, __v{static_cast<unsigned>(__g.edges_size()), ContainsAll} {}
+    bool insert(Edge *E) {
+      size_type Idx = std::distance(__g.edges_begin(), E);
+      bool AlreadyExists = __v.test(Idx);
+      __v.set(Idx);
+      return !AlreadyExists;
+    }
+    void erase(Edge *E) {
+      size_type Idx = std::distance(__g.edges_begin(), E);
+      __v.reset(Idx);
+    }
+    bool contains(Edge *E) const {
+      size_type Idx = std::distance(__g.edges_begin(), E);
+      return __v.test(Idx);
+    }
+    void clear() { __v.reset(); }
+    bool empty() const { return __v.none(); }
+    /// Return the number of elements in the set
+    size_type count() const { return __v.count(); }
+    /// Return the size of the set's domain
+    size_type size() const { return __v.size(); }
+    /// Set union
+    EdgeSet &operator|=(const EdgeSet &RHS) {
+      assert(&this->__g == &RHS.__g);
+      __v |= RHS.__v;
+      return *this;
+    }
+    /// Set intersection
+    EdgeSet &operator&=(const EdgeSet &RHS) {
+      assert(&this->__g == &RHS.__g);
+      __v &= RHS.__v;
+      return *this;
+    }
+    /// Set disjoint union
+    EdgeSet &operator^=(const EdgeSet &RHS) {
+      assert(&this->__g == &RHS.__g);
+      __v ^= RHS.__v;
+      return *this;
+    }
+
+    using index_iterator = typename BitVector::const_set_bits_iterator;
+    index_iterator index_begin() const { return __v.set_bits_begin(); }
+    index_iterator index_end() const { return __v.set_bits_end(); }
+    void set(size_type Idx) { __v.set(Idx); }
+    void reset(size_type Idx) { __v.reset(Idx); }
+
+    class iterator {
+      const EdgeSet &__set;
+      size_type __current;
+
+      void advance() {
+        assert(__current != -1);
+        __current = __set.__v.find_next(__current);
+      }
+
+    public:
+      iterator(const EdgeSet &Set, size_type Begin)
+          : __set{Set}, __current{Begin} {}
+      iterator operator++(int) {
+        iterator Tmp = *this;
+        advance();
+        return Tmp;
+      }
+      iterator &operator++() {
+        advance();
+        return *this;
+      }
+      Edge *operator*() const {
+        assert(__current != -1);
+        return __set.__g.edges_begin() + __current;
+      }
+      bool operator==(const iterator &other) const {
+        assert(&this->__set == &other.__set);
+        return this->__current == other.__current;
+      }
+      bool operator!=(const iterator &other) const { return !(*this == other); }
+    };
+
+    iterator begin() const { return iterator{*this, __v.find_first()}; }
+    iterator end() const { return iterator{*this, -1}; }
+  };
+
+private:
+  Node *__nodes;
+  size_type __nodes_size;
+  Edge *__edges;
+  size_type __edges_size;
+};
+
+template <typename GraphT> class ImmutableGraphBuilder {
+  using NodeValueT = typename GraphT::NodeValueT;
+  using EdgeValueT = typename GraphT::EdgeValueT;
+  static_assert(
+      std::is_base_of<ImmutableGraph<NodeValueT, EdgeValueT>, GraphT>::value,
+      "Template argument to ImmutableGraphBuilder must derive from "
+      "ImmutableGraph<>");
+  using size_type = typename GraphT::size_type;
+  using NodeSet = typename GraphT::NodeSet;
+  using Node = typename GraphT::Node;
+  using EdgeSet = typename GraphT::EdgeSet;
+  using Edge = typename GraphT::Edge;
+  using BuilderEdge = std::pair<EdgeValueT, size_type>;
+  using EdgeList = std::vector<BuilderEdge>;
+  using BuilderVertex = std::pair<NodeValueT, EdgeList>;
+  using VertexVec = std::vector<BuilderVertex>;
+
+public:
+  using NodeRef = size_type;
+
+  NodeRef addVertex(const NodeValueT &V) {
+    auto I = __adj_list.emplace(__adj_list.end(), V, EdgeList{});
+    return std::distance(__adj_list.begin(), I);
+  }
+
+  void addEdge(const EdgeValueT &E, NodeRef From, NodeRef To) {
+    __adj_list[From].second.emplace_back(E, To);
+  }
+
+  bool empty() const { return __adj_list.empty(); }
+
+  template <typename... ArgT> GraphT *get(ArgT &&... Args) {
+    size_type VertexSize = __adj_list.size(), EdgeSize = 0;
+    for (const auto &V : __adj_list) {
+      EdgeSize += V.second.size();
+    }
+    auto *VertexArray = new Node[VertexSize + 1 /* terminator node */];
+    auto *EdgeArray = new Edge[EdgeSize];
+    size_type VI = 0, EI = 0;
+    for (; VI < static_cast<size_type>(__adj_list.size()); ++VI) {
+      VertexArray[VI].__value = std::move(__adj_list[VI].first);
+      VertexArray[VI].__edges = &EdgeArray[EI];
+      auto NumEdges = static_cast<size_type>(__adj_list[VI].second.size());
+      if (NumEdges > 0) {
+        for (size_type VEI = 0; VEI < NumEdges; ++VEI, ++EI) {
+          auto &E = __adj_list[VI].second[VEI];
+          EdgeArray[EI].__value = std::move(E.first);
+          EdgeArray[EI].__dest = VertexArray + E.second;
+        }
+      }
+    }
+    assert(VI == VertexSize && EI == EdgeSize && "Gadget graph malformed");
+    VertexArray[VI].__edges = EdgeArray + EdgeSize; // terminator node
+    return new GraphT{VertexArray, VertexSize, EdgeArray, EdgeSize,
+                      std::forward<ArgT>(Args)...};
+  }
+
+  template <typename... ArgT>
+  static GraphT *trim(const GraphT &G, const NodeSet &TrimNodes,
+                      const EdgeSet &TrimEdges, ArgT &&... Args) {
+    size_type NewVertexSize = TrimNodes.size() - TrimNodes.count();
+    size_type NewEdgeSize = TrimEdges.size() - TrimEdges.count();
+    auto *NewVertexArray = new Node[NewVertexSize + 1 /* terminator node */];
+    auto *NewEdgeArray = new Edge[NewEdgeSize];
+    size_type TrimmedNodesSoFar = 0,
+              *TrimmedNodes = new size_type[TrimNodes.size()];
+    for (size_type I = 0; I < TrimNodes.size(); ++I) {
+      TrimmedNodes[I] = TrimmedNodesSoFar;
+      if (TrimNodes.contains(G.nodes_begin() + I))
+        ++TrimmedNodesSoFar;
+    }
+    size_type VertexI = 0, EdgeI = 0;
+    for (Node *NI = G.nodes_begin(), *NE = G.nodes_end(); NI != NE; ++NI) {
+      if (TrimNodes.contains(NI))
+        continue;
+      size_type NewNumEdges =
+          static_cast<int>((NI + 1)->__edges - NI->__edges) > 0
+              ? std::count_if(
+                    NI->__edges, (NI + 1)->__edges,
+                    [&TrimEdges](Edge &E) { return !TrimEdges.contains(&E); })
+              : 0;
+      NewVertexArray[VertexI].__value = NI->__value;
+      NewVertexArray[VertexI].__edges = &NewEdgeArray[EdgeI];
+      if (NewNumEdges > 0) {
+        for (Edge *EI = NI->__edges, *EE = (NI + 1)->__edges; EI != EE; ++EI) {
+          if (TrimEdges.contains(EI))
+            continue;
+          NewEdgeArray[EdgeI].__value = EI->__value;
+          size_type DestIdx = std::distance(G.nodes_begin(), EI->__dest);
+          size_type NewIdx = DestIdx - TrimmedNodes[DestIdx];
+          assert(NewIdx < NewVertexSize);
+          NewEdgeArray[EdgeI].__dest = NewVertexArray + NewIdx;
+          ++EdgeI;
+        }
+      }
+      ++VertexI;
+    }
+    delete[] TrimmedNodes;
+    assert(VertexI == NewVertexSize && EdgeI == NewEdgeSize &&
+           "Gadget graph malformed");
+    NewVertexArray[VertexI].__edges = NewEdgeArray + NewEdgeSize;
+    return new GraphT{NewVertexArray, NewVertexSize, NewEdgeArray, NewEdgeSize,
+                      std::forward<ArgT>(Args)...};
+  }
+
+private:
+  VertexVec __adj_list;
+};
+
+template <typename NodeValueT, typename EdgeValueT>
+struct GraphTraits<ImmutableGraph<NodeValueT, EdgeValueT> *> {
+  using GraphT = ImmutableGraph<NodeValueT, EdgeValueT>;
+  using NodeRef = typename GraphT::Node *;
+  using EdgeRef = typename GraphT::Edge &;
+
+  static NodeRef edge_dest(EdgeRef E) { return E.__dest; }
+  using ChildIteratorType =
+      mapped_iterator<typename GraphT::Edge *, decltype(&edge_dest)>;
+
+  static NodeRef getEntryNode(GraphT *G) { return G->nodes_begin(); }
+  static ChildIteratorType child_begin(NodeRef N) {
+    return {N->__edges, &edge_dest};
+  }
+  static ChildIteratorType child_end(NodeRef N) {
+    return {(N + 1)->__edges, &edge_dest};
+  }
+
+  static NodeRef getNode(typename GraphT::Node &N) { return NodeRef{&N}; }
+  using nodes_iterator =
+      mapped_iterator<typename GraphT::Node *, decltype(&getNode)>;
+  static nodes_iterator nodes_begin(GraphT *G) {
+    return {G->nodes_begin(), &getNode};
+  }
+  static nodes_iterator nodes_end(GraphT *G) {
+    return {G->nodes_end(), &getNode};
+  }
+
+  using ChildEdgeIteratorType = typename GraphT::Edge *;
+
+  static ChildEdgeIteratorType child_edge_begin(NodeRef N) {
+    return N->__edges;
+  }
+  static ChildEdgeIteratorType child_edge_end(NodeRef N) {
+    return (N + 1)->__edges;
+  }
+  static typename GraphT::size_type size(GraphT *G) { return G->nodes_size(); }
+};
+
+} // end namespace llvm
+
+#endif // IMMUTABLEGRAPH_H
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp
index 675a9c377b12d..0134b4efce727 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp
@@ -56,7 +56,7 @@ void X86ATTInstPrinter::printInst(const MCInst *MI, uint64_t Address,
   if (MI->getOpcode() == X86::CALLpcrel32 &&
       (STI.getFeatureBits()[X86::Mode64Bit])) {
     OS << "\tcallq\t";
-    printPCRelImm(MI, 0, OS);
+    printPCRelImm(MI, Address, 0, OS);
   }
   // data16 and data32 both have the same encoding of 0x66. While data32 is
   // valid only in 16 bit systems, data16 is valid in the rest.
@@ -68,8 +68,7 @@ void X86ATTInstPrinter::printInst(const MCInst *MI, uint64_t Address,
    OS << "\tdata32";
   }
   // Try to print any aliases first.
-  else if (!printAliasInstr(MI, OS) &&
-           !printVecCompareInstr(MI, OS))
+  else if (!printAliasInstr(MI, Address, OS) && !printVecCompareInstr(MI, OS))
     printInstruction(MI, Address, OS);
 
   // Next always print the annotation.
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.h b/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.h
index 3d5d384dc4a01..31ea15d2767f5 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.h
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.h
@@ -30,9 +30,10 @@ class X86ATTInstPrinter final : public X86InstPrinterCommon {
 
   // Autogenerated by tblgen, returns true if we successfully printed an
   // alias.
-  bool printAliasInstr(const MCInst *MI, raw_ostream &OS);
-  void printCustomAliasOperand(const MCInst *MI, unsigned OpIdx,
-                               unsigned PrintMethodIdx, raw_ostream &O);
+  bool printAliasInstr(const MCInst *MI, uint64_t Address, raw_ostream &OS);
+  void printCustomAliasOperand(const MCInst *MI, uint64_t Address,
+                               unsigned OpIdx, unsigned PrintMethodIdx,
+                               raw_ostream &O);
 
   // Autogenerated by tblgen.
   void printInstruction(const MCInst *MI, uint64_t Address, raw_ostream &OS);
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
index e3d8b7715f5a4..c2eb78bd056d9 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -136,9 +136,11 @@ class X86AsmBackend : public MCAsmBackend {
 
   bool needAlign(MCObjectStreamer &OS) const;
   bool needAlignInst(const MCInst &Inst) const;
+  bool allowAutoPaddingForInst(const MCInst &Inst, MCObjectStreamer &OS) const;
   MCInst PrevInst;
   MCBoundaryAlignFragment *PendingBoundaryAlign = nullptr;
   std::pair<MCFragment *, size_t> PrevInstPosition;
+  bool AllowAutoPaddingForInst;
 
 public:
   X86AsmBackend(const Target &T, const MCSubtargetInfo &STI)
@@ -538,13 +540,8 @@ static size_t getSizeForInstFragment(const MCFragment *F) {
   }
 }
 
-/// Check if the instruction operand needs to be aligned. Padding is disabled
-/// before intruction which may be rewritten by linker(e.g. TLSCALL).
+/// Check if the instruction operand needs to be aligned.
 bool X86AsmBackend::needAlignInst(const MCInst &Inst) const {
-  // Linker may rewrite the instruction with variant symbol operand.
-  if (hasVariantSymbol(Inst))
-    return false;
-
   const MCInstrDesc &InstDesc = MCII->get(Inst.getOpcode());
   return (InstDesc.isConditionalBranch() &&
           (AlignBranchType & X86::AlignBranchJcc)) ||
@@ -558,31 +555,53 @@ bool X86AsmBackend::needAlignInst(const MCInst &Inst) const {
           (AlignBranchType & X86::AlignBranchIndirect));
 }
 
-/// Insert BoundaryAlignFragment before instructions to align branches.
-void X86AsmBackend::emitInstructionBegin(MCObjectStreamer &OS,
-                                       const MCInst &Inst) {
-  if (!needAlign(OS))
-    return;
+/// Return true if we can insert NOP or prefixes automatically before the
+/// the instruction to be emitted.
+bool X86AsmBackend::allowAutoPaddingForInst(const MCInst &Inst,
+                                            MCObjectStreamer &OS) const {
+  if (hasVariantSymbol(Inst))
+    // Linker may rewrite the instruction with variant symbol operand(e.g.
+    // TLSCALL).
+    return false;
 
   if (hasInterruptDelaySlot(PrevInst))
     // If this instruction follows an interrupt enabling instruction with a one
     // instruction delay, inserting a nop would change behavior.
-    return;
+    return false;
 
   if (isPrefix(PrevInst, *MCII))
-    // If this instruction follows a prefix, inserting a nop would change
+    // If this instruction follows a prefix, inserting a nop/prefix would change
     // semantic.
-    return;
+    return false;
+
+  if (isPrefix(Inst, *MCII))
+    // If this instruction is a prefix, inserting a prefix would change
+    // semantic.
+    return false;
 
   if (isRightAfterData(OS.getCurrentFragment(), PrevInstPosition))
     // If this instruction follows any data, there is no clear
-    // instruction boundary, inserting a nop would change semantic.
+    // instruction boundary, inserting a nop/prefix would change semantic.
+    return false;
+
+  return true;
+}
+
+/// Insert BoundaryAlignFragment before instructions to align branches.
+void X86AsmBackend::emitInstructionBegin(MCObjectStreamer &OS,
+                                         const MCInst &Inst) {
+  AllowAutoPaddingForInst = allowAutoPaddingForInst(Inst, OS);
+
+  if (!needAlign(OS))
     return;
 
   if (!isMacroFused(PrevInst, Inst))
     // Macro fusion doesn't happen indeed, clear the pending.
     PendingBoundaryAlign = nullptr;
 
+  if (!AllowAutoPaddingForInst)
+    return;
+
   if (PendingBoundaryAlign &&
       OS.getCurrentFragment()->getPrevNode() == PendingBoundaryAlign) {
     // Macro fusion actually happens and there is no other fragment inserted
@@ -617,12 +636,14 @@ void X86AsmBackend::emitInstructionBegin(MCObjectStreamer &OS,
 
 /// Set the last fragment to be aligned for the BoundaryAlignFragment.
 void X86AsmBackend::emitInstructionEnd(MCObjectStreamer &OS, const MCInst &Inst) {
-  if (!needAlign(OS))
-    return;
-
   PrevInst = Inst;
   MCFragment *CF = OS.getCurrentFragment();
   PrevInstPosition = std::make_pair(CF, getSizeForInstFragment(CF));
+  if (auto *F = dyn_cast_or_null<MCRelaxableFragment>(CF))
+    F->setAllowAutoPadding(AllowAutoPaddingForInst);
+
+  if (!needAlign(OS))
+    return;
 
   if (!needAlignInst(Inst) || !PendingBoundaryAlign)
     return;
@@ -646,13 +667,23 @@ void X86AsmBackend::emitInstructionEnd(MCObjectStreamer &OS, const MCInst &Inst)
 
 Optional<MCFixupKind> X86AsmBackend::getFixupKind(StringRef Name) const {
   if (STI.getTargetTriple().isOSBinFormatELF()) {
+    unsigned Type;
     if (STI.getTargetTriple().getArch() == Triple::x86_64) {
-      if (Name == "R_X86_64_NONE")
-        return FK_NONE;
+      Type = llvm::StringSwitch<unsigned>(Name)
+#define ELF_RELOC(X, Y) .Case(#X, Y)
+#include "llvm/BinaryFormat/ELFRelocs/x86_64.def"
+#undef ELF_RELOC
+                 .Default(-1u);
     } else {
-      if (Name == "R_386_NONE")
-        return FK_NONE;
+      Type = llvm::StringSwitch<unsigned>(Name)
+#define ELF_RELOC(X, Y) .Case(#X, Y)
+#include "llvm/BinaryFormat/ELFRelocs/i386.def"
+#undef ELF_RELOC
+                 .Default(-1u);
     }
+    if (Type == -1u)
+      return None;
+    return static_cast<MCFixupKind>(FirstLiteralRelocationKind + Type);
   }
   return MCAsmBackend::getFixupKind(Name);
 }
@@ -670,6 +701,11 @@ const MCFixupKindInfo &X86AsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
       {"reloc_branch_4byte_pcrel", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
   };
 
+  // Fixup kinds from .reloc directive are like R_386_NONE/R_X86_64_NONE. They
+  // do not require any extra processing.
+  if (Kind >= FirstLiteralRelocationKind)
+    return MCAsmBackend::getFixupKindInfo(FK_NONE);
+
   if (Kind < FirstTargetFixupKind)
     return MCAsmBackend::getFixupKindInfo(Kind);
 
@@ -682,7 +718,7 @@ const MCFixupKindInfo &X86AsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
 bool X86AsmBackend::shouldForceRelocation(const MCAssembler &,
                                           const MCFixup &Fixup,
                                           const MCValue &) {
-  return Fixup.getKind() == FK_NONE;
+  return Fixup.getKind() >= FirstLiteralRelocationKind;
 }
 
 static unsigned getFixupKindSize(unsigned Kind) {
@@ -724,7 +760,10 @@ void X86AsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
                                MutableArrayRef<char> Data,
                                uint64_t Value, bool IsResolved,
                                const MCSubtargetInfo *STI) const {
-  unsigned Size = getFixupKindSize(Fixup.getKind());
+  unsigned Kind = Fixup.getKind();
+  if (Kind >= FirstLiteralRelocationKind)
+    return;
+  unsigned Size = getFixupKindSize(Kind);
 
   assert(Fixup.getOffset() + Size <= Data.size() && "Invalid fixup offset!");
 
@@ -809,12 +848,6 @@ static bool isFullyRelaxed(const MCRelaxableFragment &RF) {
   return getRelaxedOpcode(Inst, Is16BitMode) == Inst.getOpcode();
 }
 
-
-static bool shouldAddPrefix(const MCInst &Inst, const MCInstrInfo &MCII) {
-  // Linker may rewrite the instruction with variant symbol operand.
-  return !hasVariantSymbol(Inst);
-}
-
 static unsigned getRemainingPrefixSize(const MCInst &Inst,
                                        const MCSubtargetInfo &STI,
                                        MCCodeEmitter &Emitter) {
@@ -838,7 +871,7 @@ static unsigned getRemainingPrefixSize(const MCInst &Inst,
 bool X86AsmBackend::padInstructionViaPrefix(MCRelaxableFragment &RF,
                                             MCCodeEmitter &Emitter,
                                             unsigned &RemainingSize) const {
-  if (!shouldAddPrefix(RF.getInst(), *MCII))
+  if (!RF.getAllowAutoPadding())
     return false;
   // If the instruction isn't fully relaxed, shifting it around might require a
   // larger value for one of the fixups then can be encoded.  The outer loop
@@ -970,10 +1003,6 @@ void X86AsmBackend::finishLayout(MCAssembler const &Asm,
       const uint64_t OrigOffset = Layout.getFragmentOffset(&F);
 #endif
       const uint64_t OrigSize = Asm.computeFragmentSize(Layout, F);
-      if (OrigSize == 0 || Relaxable.empty()) {
-        Relaxable.clear();
-        continue;
-      }
 
       // To keep the effects local, prefer to relax instructions closest to
       // the align directive.  This is purely about human understandability
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h b/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h
index 3e4a5130cc10f..f8e893d589ef6 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h
@@ -346,33 +346,6 @@ namespace X86 {
     llvm_unreachable("unknown fusion type");
   }
 
-  /// \returns true if the instruction with given opcode is a prefix.
-  inline bool isPrefix(unsigned Opcode) {
-    switch (Opcode) {
-    default:
-      return false;
-      // segment override prefix
-    case X86::CS_PREFIX:
-    case X86::DS_PREFIX:
-    case X86::ES_PREFIX:
-    case X86::FS_PREFIX:
-    case X86::GS_PREFIX:
-    case X86::SS_PREFIX:
-      // operand-size override prefix
-    case X86::DATA16_PREFIX:
-      // lock and repeat prefix
-    case X86::LOCK_PREFIX:
-    case X86::REPNE_PREFIX:
-    case X86::REP_PREFIX:
-      // rex64 prefix
-    case X86::REX64_PREFIX:
-      // acquire and release prefix
-    case X86::XACQUIRE_PREFIX:
-    case X86::XRELEASE_PREFIX:
-      return true;
-    }
-  }
-
   /// Defines the possible values of the branch boundary alignment mask.
   enum AlignBranchBoundaryKind : uint8_t {
     AlignBranchNone = 0,
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp
index bd009da60851e..292dd17e2f51c 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp
@@ -317,8 +317,10 @@ static unsigned getRelocType32(MCContext &Ctx,
 unsigned X86ELFObjectWriter::getRelocType(MCContext &Ctx, const MCValue &Target,
                                           const MCFixup &Fixup,
                                           bool IsPCRel) const {
-  MCSymbolRefExpr::VariantKind Modifier = Target.getAccessVariant();
   MCFixupKind Kind = Fixup.getKind();
+  if (Kind >= FirstLiteralRelocationKind)
+    return Kind - FirstLiteralRelocationKind;
+  MCSymbolRefExpr::VariantKind Modifier = Target.getAccessVariant();
   X86_64RelType Type = getType64(Kind, Modifier, IsPCRel);
   if (getEMachine() == ELF::EM_X86_64)
     return getRelocType64(Ctx, Fixup.getLoc(), Modifier, Type, IsPCRel, Kind);
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp
index a215550769765..33d70fdb12142 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp
@@ -13,6 +13,7 @@
 
 #include "X86InstPrinterCommon.h"
 #include "X86BaseInfo.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstrDesc.h"
@@ -287,16 +288,23 @@ void X86InstPrinterCommon::printRoundingControl(const MCInst *MI, unsigned Op,
   }
 }
 
-/// printPCRelImm - This is used to print an immediate value that ends up
-/// being encoded as a pc-relative value (e.g. for jumps and calls).  In
-/// Intel-style these print slightly differently than normal immediates.
-/// for example, a $ is not emitted.
-void X86InstPrinterCommon::printPCRelImm(const MCInst *MI, unsigned OpNo,
-                                         raw_ostream &O) {
+/// value (e.g. for jumps and calls). In Intel-style these print slightly
+/// differently than normal immediates. For example, a $ is not emitted.
+///
+/// \p Address The address of the next instruction.
+/// \see MCInstPrinter::printInst
+void X86InstPrinterCommon::printPCRelImm(const MCInst *MI, uint64_t Address,
+                                         unsigned OpNo, raw_ostream &O) {
   const MCOperand &Op = MI->getOperand(OpNo);
-  if (Op.isImm())
-    O << formatImm(Op.getImm());
-  else {
+  if (Op.isImm()) {
+    if (PrintBranchImmAsAddress) {
+      uint64_t Target = Address + Op.getImm();
+      if (MAI.getCodePointerSize() == 4)
+        Target &= 0xffffffff;
+      O << formatHex(Target);
+    } else
+      O << formatImm(Op.getImm());
+  } else {
     assert(Op.isExpr() && "unknown pcrel immediate operand");
     // If a symbolic branch target was added as a constant expression then print
     // that address in hex.
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.h b/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.h
index 8e28f24b619a9..bb12ede3b7292 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.h
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.h
@@ -29,7 +29,9 @@ class X86InstPrinterCommon : public MCInstPrinter {
   void printVPCMPMnemonic(const MCInst *MI, raw_ostream &OS);
   void printCMPMnemonic(const MCInst *MI, bool IsVCmp, raw_ostream &OS);
   void printRoundingControl(const MCInst *MI, unsigned Op, raw_ostream &O);
-  void printPCRelImm(const MCInst *MI, unsigned OpNo, raw_ostream &O);
+  void printPCRelImm(const MCInst *MI, uint64_t Address, unsigned OpNo,
+                     raw_ostream &O);
+
 protected:
   void printInstFlags(const MCInst *MI, raw_ostream &O);
   void printOptionalSegReg(const MCInst *MI, unsigned OpNo, raw_ostream &O);
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp
index f4bb0fbf62cd7..d1eb4d09851dd 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp
@@ -45,8 +45,7 @@ void X86IntelInstPrinter::printInst(const MCInst *MI, uint64_t Address,
   if (MI->getOpcode() == X86::DATA16_PREFIX &&
       STI.getFeatureBits()[X86::Mode16Bit]) {
     OS << "\tdata32";
-  } else if (!printAliasInstr(MI, OS) &&
-             !printVecCompareInstr(MI, OS))
+  } else if (!printAliasInstr(MI, Address, OS) && !printVecCompareInstr(MI, OS))
     printInstruction(MI, Address, OS);
 
   // Next always print the annotation.
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.h b/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.h
index b409b20cbea88..2a7abc16e4977 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.h
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.h
@@ -31,9 +31,10 @@ class X86IntelInstPrinter final : public X86InstPrinterCommon {
 
   // Autogenerated by tblgen, returns true if we successfully printed an
   // alias.
-  bool printAliasInstr(const MCInst *MI, raw_ostream &OS);
-  void printCustomAliasOperand(const MCInst *MI, unsigned OpIdx,
-                               unsigned PrintMethodIdx, raw_ostream &O);
+  bool printAliasInstr(const MCInst *MI, uint64_t Address, raw_ostream &OS);
+  void printCustomAliasOperand(const MCInst *MI, uint64_t Address,
+                               unsigned OpIdx, unsigned PrintMethodIdx,
+                               raw_ostream &O);
 
   // Autogenerated by tblgen.
   void printInstruction(const MCInst *MI, uint64_t Address, raw_ostream &O);
diff --git a/llvm/lib/Target/X86/X86.h b/llvm/lib/Target/X86/X86.h
index c81b349ecedd3..0c88a6b746efb 100644
--- a/llvm/lib/Target/X86/X86.h
+++ b/llvm/lib/Target/X86/X86.h
@@ -120,7 +120,7 @@ FunctionPass *createX86DomainReassignmentPass();
 FunctionPass *createX86EvexToVexInsts();
 
 /// This pass creates the thunks for the retpoline feature.
-FunctionPass *createX86RetpolineThunksPass();
+FunctionPass *createX86IndirectThunksPass();
 
 /// This pass ensures instructions featuring a memory operand
 /// have distinctive <LineNumber, Discriminator> (with respect to eachother)
@@ -133,10 +133,17 @@ FunctionPass *createX86InsertPrefetchPass();
 /// fp exceptions when strict-fp enabled.
 FunctionPass *createX86InsertX87waitPass();
 
+/// This pass optimizes arithmetic based on knowledge that is only used by
+/// a reduction sequence and is therefore safe to reassociate in interesting
+/// ways.
+FunctionPass *createX86PartialReductionPass();
+
 InstructionSelector *createX86InstructionSelector(const X86TargetMachine &TM,
                                                   X86Subtarget &,
                                                   X86RegisterBankInfo &);
 
+FunctionPass *createX86LoadValueInjectionLoadHardeningPass();
+FunctionPass *createX86LoadValueInjectionRetHardeningPass();
 FunctionPass *createX86SpeculativeLoadHardeningPass();
 
 void initializeEvexToVexInstPassPass(PassRegistry &);
@@ -153,7 +160,10 @@ void initializeX86DomainReassignmentPass(PassRegistry &);
 void initializeX86ExecutionDomainFixPass(PassRegistry &);
 void initializeX86ExpandPseudoPass(PassRegistry &);
 void initializeX86FlagsCopyLoweringPassPass(PassRegistry &);
+void initializeX86LoadValueInjectionLoadHardeningPassPass(PassRegistry &);
+void initializeX86LoadValueInjectionRetHardeningPassPass(PassRegistry &);
 void initializeX86OptimizeLEAPassPass(PassRegistry &);
+void initializeX86PartialReductionPass(PassRegistry &);
 void initializeX86SpeculativeLoadHardeningPassPass(PassRegistry &);
 
 namespace X86AS {
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index 6c4ab8f5a6f3a..7bee814232b75 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -273,6 +273,8 @@ def FeatureWAITPKG  : SubtargetFeature<"waitpkg", "HasWAITPKG", "true",
                                       "Wait and pause enhancements">;
 def FeatureENQCMD : SubtargetFeature<"enqcmd", "HasENQCMD", "true",
                                      "Has ENQCMD instructions">;
+def FeatureSERIALIZE : SubtargetFeature<"serialize", "HasSERIALIZE", "true",
+                                        "Has serialize instruction">;
 // On some processors, instructions that implicitly take two memory operands are
 // slow. In practice, this means that CALL, PUSH, and POP with memory operands
 // should be avoided in favor of a MOV + register CALL/PUSH/POP.
@@ -431,6 +433,22 @@ def FeatureRetpolineExternalThunk
           "ourselves. Only has effect when combined with some other retpoline "
           "feature", [FeatureRetpolineIndirectCalls]>;
 
+// Mitigate LVI attacks against indirect calls/branches and call returns
+def FeatureLVIControlFlowIntegrity
+    : SubtargetFeature<
+          "lvi-cfi", "UseLVIControlFlowIntegrity", "true",
+          "Prevent indirect calls/branches from using a memory operand, and "
+          "precede all indirect calls/branches from a register with an "
+          "LFENCE instruction to serialize control flow. Also decompose RET "
+          "instructions into a POP+LFENCE+JMP sequence.">;
+
+// Mitigate LVI attacks against data loads
+def FeatureLVILoadHardening
+    : SubtargetFeature<
+          "lvi-load-hardening", "UseLVILoadHardening", "true",
+          "Insert LFENCE instructions to prevent data speculatively injected "
+          "into loads from being used maliciously.">;
+
 // Direct Move instructions.
 def FeatureMOVDIRI  : SubtargetFeature<"movdiri", "HasMOVDIRI", "true",
                                        "Support movdiri instruction">;
diff --git a/llvm/lib/Target/X86/X86CallFrameOptimization.cpp b/llvm/lib/Target/X86/X86CallFrameOptimization.cpp
index a5831bc8ef0bc..1d42dd77016ec 100644
--- a/llvm/lib/Target/X86/X86CallFrameOptimization.cpp
+++ b/llvm/lib/Target/X86/X86CallFrameOptimization.cpp
@@ -549,7 +549,7 @@ void X86CallFrameOptimization::adjustCallSequence(MachineFunction &MF,
 
       // If PUSHrmm is not slow on this target, try to fold the source of the
       // push into the instruction.
-      bool SlowPUSHrmm = STI->isAtom() || STI->isSLM();
+      bool SlowPUSHrmm = STI->slowTwoMemOps();
 
       // Check that this is legal to fold. Right now, we're extremely
       // conservative about that.
diff --git a/llvm/lib/Target/X86/X86CallLowering.cpp b/llvm/lib/Target/X86/X86CallLowering.cpp
index 0e986a60d86bc..319dc94706048 100644
--- a/llvm/lib/Target/X86/X86CallLowering.cpp
+++ b/llvm/lib/Target/X86/X86CallLowering.cpp
@@ -148,11 +148,10 @@ struct OutgoingValueHandler : public CallLowering::ValueHandler {
                             MachinePointerInfo &MPO, CCValAssign &VA) override {
     MachineFunction &MF = MIRBuilder.getMF();
     Register ExtReg = extendRegister(ValVReg, VA);
-    unsigned Align = inferAlignmentFromPtrInfo(MF, MPO);
 
-    auto MMO = MF.getMachineMemOperand(
-        MPO, MachineMemOperand::MOStore, VA.getLocVT().getStoreSize(),
-        Align);
+    auto MMO = MF.getMachineMemOperand(MPO, MachineMemOperand::MOStore,
+                                       VA.getLocVT().getStoreSize(),
+                                       inferAlignFromPtrInfo(MF, MPO));
     MIRBuilder.buildStore(ExtReg, Addr, *MMO);
   }
 
@@ -249,10 +248,9 @@ struct IncomingValueHandler : public CallLowering::ValueHandler {
   void assignValueToAddress(Register ValVReg, Register Addr, uint64_t Size,
                             MachinePointerInfo &MPO, CCValAssign &VA) override {
     MachineFunction &MF = MIRBuilder.getMF();
-    unsigned Align = inferAlignmentFromPtrInfo(MF, MPO);
     auto MMO = MF.getMachineMemOperand(
         MPO, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, Size,
-        Align);
+        inferAlignFromPtrInfo(MF, MPO));
     MIRBuilder.buildLoad(ValVReg, Addr, *MMO);
   }
 
diff --git a/llvm/lib/Target/X86/X86CallingConv.td b/llvm/lib/Target/X86/X86CallingConv.td
index db1aef2fd09da..3ec947dfdb8c8 100644
--- a/llvm/lib/Target/X86/X86CallingConv.td
+++ b/llvm/lib/Target/X86/X86CallingConv.td
@@ -1145,7 +1145,7 @@ def CSR_64_Intel_OCL_BI       : CalleeSavedRegs<(add CSR_64,
 def CSR_64_Intel_OCL_BI_AVX    : CalleeSavedRegs<(add CSR_64,
                                                   (sequence "YMM%u", 8, 15))>;
 
-def CSR_64_Intel_OCL_BI_AVX512 : CalleeSavedRegs<(add RBX, RDI, RSI, R14, R15,
+def CSR_64_Intel_OCL_BI_AVX512 : CalleeSavedRegs<(add RBX, RSI, R14, R15,
                                                   (sequence "ZMM%u", 16, 31),
                                                   K4, K5, K6, K7)>;
 
diff --git a/llvm/lib/Target/X86/X86FastISel.cpp b/llvm/lib/Target/X86/X86FastISel.cpp
index c9f3186367ee1..906eef76752a7 100644
--- a/llvm/lib/Target/X86/X86FastISel.cpp
+++ b/llvm/lib/Target/X86/X86FastISel.cpp
@@ -3207,8 +3207,8 @@ bool X86FastISel::fastLowerCall(CallLoweringInfo &CLI) {
       (CalledFn && CalledFn->hasFnAttribute("no_caller_saved_registers")))
     return false;
 
-  // Functions using retpoline for indirect calls need to use SDISel.
-  if (Subtarget->useRetpolineIndirectCalls())
+  // Functions using thunks for indirect calls need to use SDISel.
+  if (Subtarget->useIndirectThunkCalls())
     return false;
 
   // Handle only C, fastcc, and webkit_js calling conventions for now.
@@ -3423,7 +3423,7 @@ bool X86FastISel::fastLowerCall(CallLoweringInfo &CLI) {
       AM.Base.Reg = RegInfo->getStackRegister();
       AM.Disp = LocMemOffset;
       ISD::ArgFlagsTy Flags = OutFlags[VA.getValNo()];
-      unsigned Alignment = DL.getABITypeAlignment(ArgVal->getType());
+      Align Alignment = DL.getABITypeAlign(ArgVal->getType());
       MachineMemOperand *MMO = FuncInfo.MF->getMachineMemOperand(
           MachinePointerInfo::getStack(*FuncInfo.MF, LocMemOffset),
           MachineMemOperand::MOStore, ArgVT.getStoreSize(), Alignment);
@@ -3775,11 +3775,7 @@ unsigned X86FastISel::X86MaterializeFP(const ConstantFP *CFP, MVT VT) {
   }
 
   // MachineConstantPool wants an explicit alignment.
-  unsigned Align = DL.getPrefTypeAlignment(CFP->getType());
-  if (Align == 0) {
-    // Alignment of vector types. FIXME!
-    Align = DL.getTypeAllocSize(CFP->getType());
-  }
+  Align Alignment = DL.getPrefTypeAlign(CFP->getType());
 
   // x86-32 PIC requires a PIC base register for constant pools.
   unsigned PICBase = 0;
@@ -3792,7 +3788,7 @@ unsigned X86FastISel::X86MaterializeFP(const ConstantFP *CFP, MVT VT) {
     PICBase = X86::RIP;
 
   // Create the load from the constant pool.
-  unsigned CPI = MCP.getConstantPoolIndex(CFP, Align);
+  unsigned CPI = MCP.getConstantPoolIndex(CFP, Alignment.value());
   unsigned ResultReg = createResultReg(TLI.getRegClassFor(VT.SimpleTy));
 
   if (CM == CodeModel::Large) {
@@ -3805,7 +3801,7 @@ unsigned X86FastISel::X86MaterializeFP(const ConstantFP *CFP, MVT VT) {
     addDirectMem(MIB, AddrReg);
     MachineMemOperand *MMO = FuncInfo.MF->getMachineMemOperand(
         MachinePointerInfo::getConstantPool(*FuncInfo.MF),
-        MachineMemOperand::MOLoad, DL.getPointerSize(), Align);
+        MachineMemOperand::MOLoad, DL.getPointerSize(), Alignment);
     MIB->addMemOperand(*FuncInfo.MF, MMO);
     return ResultReg;
   }
diff --git a/llvm/lib/Target/X86/X86FrameLowering.cpp b/llvm/lib/Target/X86/X86FrameLowering.cpp
index 96bdb155075d0..6ada3214263ae 100644
--- a/llvm/lib/Target/X86/X86FrameLowering.cpp
+++ b/llvm/lib/Target/X86/X86FrameLowering.cpp
@@ -956,10 +956,10 @@ void X86FrameLowering::emitStackProbeCall(MachineFunction &MF,
                                           bool InProlog) const {
   bool IsLargeCodeModel = MF.getTarget().getCodeModel() == CodeModel::Large;
 
-  // FIXME: Add retpoline support and remove this.
-  if (Is64Bit && IsLargeCodeModel && STI.useRetpolineIndirectCalls())
+  // FIXME: Add indirect thunk support and remove this.
+  if (Is64Bit && IsLargeCodeModel && STI.useIndirectThunkCalls())
     report_fatal_error("Emitting stack probe calls on 64-bit with the large "
-                       "code model and retpoline not yet implemented.");
+                       "code model and indirect thunks not yet implemented.");
 
   unsigned CallOp;
   if (Is64Bit)
@@ -1514,17 +1514,17 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF,
       addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64rm), Establisher),
                    Establisher, false, PSPSlotOffset)
           .addMemOperand(MF.getMachineMemOperand(
-              NoInfo, MachineMemOperand::MOLoad, SlotSize, SlotSize));
+              NoInfo, MachineMemOperand::MOLoad, SlotSize, Align(SlotSize)));
       ;
       // Save the root establisher back into the current funclet's (mostly
       // empty) frame, in case a sub-funclet or the GC needs it.
       addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64mr)), StackPtr,
                    false, PSPSlotOffset)
           .addReg(Establisher)
-          .addMemOperand(
-              MF.getMachineMemOperand(NoInfo, MachineMemOperand::MOStore |
-                                                  MachineMemOperand::MOVolatile,
-                                      SlotSize, SlotSize));
+          .addMemOperand(MF.getMachineMemOperand(
+              NoInfo,
+              MachineMemOperand::MOStore | MachineMemOperand::MOVolatile,
+              SlotSize, Align(SlotSize)));
     }
     SPOrEstablisher = Establisher;
   } else {
@@ -1614,7 +1614,7 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF,
         .addReg(StackPtr)
         .addMemOperand(MF.getMachineMemOperand(
             PSPInfo, MachineMemOperand::MOStore | MachineMemOperand::MOVolatile,
-            SlotSize, SlotSize));
+            SlotSize, Align(SlotSize)));
   }
 
   // Realign stack after we spilled callee-saved registers (so that we'll be
@@ -2012,7 +2012,7 @@ int X86FrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
       // Skip the saved EBP.
       return Offset + SlotSize + FPDelta;
     } else {
-      assert((-(Offset + StackSize)) % MFI.getObjectAlignment(FI) == 0);
+      assert(isAligned(MFI.getObjectAlign(FI), -(Offset + StackSize)));
       return Offset + StackSize;
     }
   } else if (TRI->needsStackRealignment(MF)) {
@@ -2020,7 +2020,7 @@ int X86FrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
       // Skip the saved EBP.
       return Offset + SlotSize + FPDelta;
     } else {
-      assert((-(Offset + StackSize)) % MFI.getObjectAlignment(FI) == 0);
+      assert(isAligned(MFI.getObjectAlign(FI), -(Offset + StackSize)));
       return Offset + StackSize;
     }
     // FIXME: Support tail calls
@@ -2177,7 +2177,7 @@ bool X86FrameLowering::assignCalleeSavedSpillSlots(
   if (this->TRI->hasBasePointer(MF)) {
     // Allocate a spill slot for EBP if we have a base pointer and EH funclets.
     if (MF.hasEHFunclets()) {
-      int FI = MFI.CreateSpillStackObject(SlotSize, SlotSize);
+      int FI = MFI.CreateSpillStackObject(SlotSize, Align(SlotSize));
       X86FI->setHasSEHFramePtrSave(true);
       X86FI->setSEHFramePtrSaveIndex(FI);
     }
@@ -2683,9 +2683,9 @@ void X86FrameLowering::adjustForSegmentedStacks(
     // is laid out within 2^31 bytes of each function body, but this seems
     // to be sufficient for JIT.
     // FIXME: Add retpoline support and remove the error here..
-    if (STI.useRetpolineIndirectCalls())
+    if (STI.useIndirectThunkCalls())
       report_fatal_error("Emitting morestack calls on 64-bit with the large "
-                         "code model and retpoline not yet implemented.");
+                         "code model and thunks not yet implemented.");
     BuildMI(allocMBB, DL, TII.get(X86::CALL64m))
         .addReg(X86::RIP)
         .addImm(0)
@@ -3203,7 +3203,7 @@ struct X86FrameSortingObject {
   bool IsValid = false;         // true if we care about this Object.
   unsigned ObjectIndex = 0;     // Index of Object into MFI list.
   unsigned ObjectSize = 0;      // Size of Object in bytes.
-  unsigned ObjectAlignment = 1; // Alignment of Object in bytes.
+  Align ObjectAlignment = Align(1); // Alignment of Object in bytes.
   unsigned ObjectNumUses = 0;   // Object static number of uses.
 };
 
@@ -3288,7 +3288,7 @@ void X86FrameLowering::orderFrameObjects(
   for (auto &Obj : ObjectsToAllocate) {
     SortingObjects[Obj].IsValid = true;
     SortingObjects[Obj].ObjectIndex = Obj;
-    SortingObjects[Obj].ObjectAlignment = MFI.getObjectAlignment(Obj);
+    SortingObjects[Obj].ObjectAlignment = MFI.getObjectAlign(Obj);
     // Set the size.
     int ObjectSize = MFI.getObjectSize(Obj);
     if (ObjectSize == 0)
@@ -3381,7 +3381,7 @@ void X86FrameLowering::processFunctionBeforeFrameFinalized(
       int FrameIndex = H.CatchObj.FrameIndex;
       if (FrameIndex != INT_MAX) {
         // Ensure alignment.
-        unsigned Align = MFI.getObjectAlignment(FrameIndex);
+        unsigned Align = MFI.getObjectAlign(FrameIndex).value();
         MinFixedObjOffset -= std::abs(MinFixedObjOffset) % Align;
         MinFixedObjOffset -= MFI.getObjectSize(FrameIndex);
         MFI.setObjectOffset(FrameIndex, MinFixedObjOffset);
diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index f705f59b4d8f8..d75913a4fe0e8 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -259,6 +259,8 @@ namespace {
                           SDValue &Index, SDValue &Disp,
                           SDValue &Segment);
 
+    bool isProfitableToFormMaskedOp(SDNode *N) const;
+
     /// Implement addressing mode selection for inline asm expressions.
     bool SelectInlineAsmMemoryOperand(const SDValue &Op,
                                       unsigned ConstraintID,
@@ -722,6 +724,20 @@ X86DAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const {
   return true;
 }
 
+// Indicates it is profitable to form an AVX512 masked operation. Returning
+// false will favor a masked register-register masked move or vblendm and the
+// operation will be selected separately.
+bool X86DAGToDAGISel::isProfitableToFormMaskedOp(SDNode *N) const {
+  assert(
+      (N->getOpcode() == ISD::VSELECT || N->getOpcode() == X86ISD::SELECTS) &&
+      "Unexpected opcode!");
+
+  // If the operation has additional users, the operation will be duplicated.
+  // Check the use count to prevent that.
+  // FIXME: Are there cheap opcodes we might want to duplicate?
+  return N->getOperand(1).hasOneUse();
+}
+
 /// Replace the original chain operand of the call with
 /// load's chain operand and move load below the call's chain operand.
 static void moveBelowOrigChain(SelectionDAG *CurDAG, SDValue Load,
@@ -1023,7 +1039,7 @@ void X86DAGToDAGISel::PreprocessISelDAG() {
     if (OptLevel != CodeGenOpt::None &&
         // Only do this when the target can fold the load into the call or
         // jmp.
-        !Subtarget->useRetpolineIndirectCalls() &&
+        !Subtarget->useIndirectThunkCalls() &&
         ((N->getOpcode() == X86ISD::CALL && !Subtarget->slowTwoMemOps()) ||
          (N->getOpcode() == X86ISD::TC_RETURN &&
           (Subtarget->is64Bit() ||
@@ -1169,7 +1185,7 @@ void X86DAGToDAGISel::PreprocessISelDAG() {
         SDVTList VTs = CurDAG->getVTList(MVT::Other);
         SDValue Ops[] = {N->getOperand(0), N->getOperand(1), MemTmp};
         Store = CurDAG->getMemIntrinsicNode(X86ISD::FST, dl, VTs, Ops, MemVT,
-                                            MPI, /*Align*/ 0,
+                                            MPI, /*Align*/ None,
                                             MachineMemOperand::MOStore);
         if (N->getFlags().hasNoFPExcept()) {
           SDNodeFlags Flags = Store->getFlags();
@@ -1185,9 +1201,9 @@ void X86DAGToDAGISel::PreprocessISelDAG() {
       if (!DstIsSSE) {
         SDVTList VTs = CurDAG->getVTList(DstVT, MVT::Other);
         SDValue Ops[] = {Store, MemTmp};
-        Result =
-            CurDAG->getMemIntrinsicNode(X86ISD::FLD, dl, VTs, Ops, MemVT, MPI,
-                                        /*Align*/ 0, MachineMemOperand::MOLoad);
+        Result = CurDAG->getMemIntrinsicNode(
+            X86ISD::FLD, dl, VTs, Ops, MemVT, MPI,
+            /*Align*/ None, MachineMemOperand::MOLoad);
         if (N->getFlags().hasNoFPExcept()) {
           SDNodeFlags Flags = Result->getFlags();
           Flags.setNoFPExcept(true);
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 1a9e5b4c5d359..57d27eddbc17c 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -4752,6 +4752,7 @@ static bool isTargetShuffle(unsigned Opcode) {
   case X86ISD::INSERTPS:
   case X86ISD::EXTRQI:
   case X86ISD::INSERTQI:
+  case X86ISD::VALIGN:
   case X86ISD::PALIGNR:
   case X86ISD::VSHLDQ:
   case X86ISD::VSRLDQ:
@@ -5434,6 +5435,11 @@ static bool isAnyInRange(ArrayRef<int> Mask, int Low, int Hi) {
   return llvm::any_of(Mask, [Low, Hi](int M) { return isInRange(M, Low, Hi); });
 }
 
+/// Return true if the value of any element in Mask is the zero sentinel value.
+static bool isAnyZero(ArrayRef<int> Mask) {
+  return llvm::any_of(Mask, [](int M) { return M == SM_SentinelZero; });
+}
+
 /// Return true if Val is undef or if its value falls within the
 /// specified range (L, H].
 static bool isUndefOrInRange(int Val, int Low, int Hi) {
@@ -6127,6 +6133,35 @@ static SDValue IsNOT(SDValue V, SelectionDAG &DAG) {
   return SDValue();
 }
 
+void llvm::createUnpackShuffleMask(MVT VT, SmallVectorImpl<int> &Mask,
+                                   bool Lo, bool Unary) {
+  assert(Mask.empty() && "Expected an empty shuffle mask vector");
+  int NumElts = VT.getVectorNumElements();
+  int NumEltsInLane = 128 / VT.getScalarSizeInBits();
+  for (int i = 0; i < NumElts; ++i) {
+    unsigned LaneStart = (i / NumEltsInLane) * NumEltsInLane;
+    int Pos = (i % NumEltsInLane) / 2 + LaneStart;
+    Pos += (Unary ? 0 : NumElts * (i % 2));
+    Pos += (Lo ? 0 : NumEltsInLane / 2);
+    Mask.push_back(Pos);
+  }
+}
+
+/// Similar to unpacklo/unpackhi, but without the 128-bit lane limitation
+/// imposed by AVX and specific to the unary pattern. Example:
+/// v8iX Lo --> <0, 0, 1, 1, 2, 2, 3, 3>
+/// v8iX Hi --> <4, 4, 5, 5, 6, 6, 7, 7>
+void llvm::createSplat2ShuffleMask(MVT VT, SmallVectorImpl<int> &Mask,
+                                   bool Lo) {
+  assert(Mask.empty() && "Expected an empty shuffle mask vector");
+  int NumElts = VT.getVectorNumElements();
+  for (int i = 0; i < NumElts; ++i) {
+    int Pos = i / 2;
+    Pos += (Lo ? 0 : NumElts / 2);
+    Mask.push_back(Pos);
+  }
+}
+
 /// Returns a vector_shuffle node for an unpackl operation.
 static SDValue getUnpackl(SelectionDAG &DAG, const SDLoc &dl, MVT VT,
                           SDValue V1, SDValue V2) {
@@ -6574,20 +6609,26 @@ static bool getTargetShuffleMaskIndices(SDValue MaskNode,
 }
 
 /// Create a shuffle mask that matches the PACKSS/PACKUS truncation.
+/// A multi-stage pack shuffle mask is created by specifying NumStages > 1.
 /// Note: This ignores saturation, so inputs must be checked first.
 static void createPackShuffleMask(MVT VT, SmallVectorImpl<int> &Mask,
-                                  bool Unary) {
+                                  bool Unary, unsigned NumStages = 1) {
   assert(Mask.empty() && "Expected an empty shuffle mask vector");
   unsigned NumElts = VT.getVectorNumElements();
   unsigned NumLanes = VT.getSizeInBits() / 128;
   unsigned NumEltsPerLane = 128 / VT.getScalarSizeInBits();
   unsigned Offset = Unary ? 0 : NumElts;
+  unsigned Repetitions = 1u << (NumStages - 1);
+  unsigned Increment = 1u << NumStages;
+  assert((NumEltsPerLane >> NumStages) > 0 && "Illegal packing compaction");
 
   for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
-    for (unsigned Elt = 0; Elt != NumEltsPerLane; Elt += 2)
-      Mask.push_back(Elt + (Lane * NumEltsPerLane));
-    for (unsigned Elt = 0; Elt != NumEltsPerLane; Elt += 2)
-      Mask.push_back(Elt + (Lane * NumEltsPerLane) + Offset);
+    for (unsigned Stage = 0; Stage != Repetitions; ++Stage) {
+      for (unsigned Elt = 0; Elt != NumEltsPerLane; Elt += Increment)
+        Mask.push_back(Elt + (Lane * NumEltsPerLane));
+      for (unsigned Elt = 0; Elt != NumEltsPerLane; Elt += Increment)
+        Mask.push_back(Elt + (Lane * NumEltsPerLane) + Offset);
+    }
   }
 }
 
@@ -6732,6 +6773,17 @@ static bool getTargetShuffleMask(SDNode *N, MVT VT, bool AllowSentinelZero,
     DecodeMOVLHPSMask(NumElems, Mask);
     IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
     break;
+  case X86ISD::VALIGN:
+    assert((VT.getScalarType() == MVT::i32 || VT.getScalarType() == MVT::i64) &&
+           "Only 32-bit and 64-bit elements are supported!");
+    assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
+    assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
+    ImmN = N->getConstantOperandVal(N->getNumOperands() - 1);
+    DecodeVALIGNMask(NumElems, ImmN, Mask);
+    IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
+    Ops.push_back(N->getOperand(1));
+    Ops.push_back(N->getOperand(0));
+    break;
   case X86ISD::PALIGNR:
     assert(VT.getScalarType() == MVT::i8 && "Byte vector expected");
     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
@@ -7303,8 +7355,8 @@ static bool getFauxShuffleMask(SDValue N, const APInt &DemandedElts,
 
     size_t MaskSize = std::max(SrcMask0.size(), SrcMask1.size());
     SmallVector<int, 64> Mask0, Mask1;
-    scaleShuffleMask<int>(MaskSize / SrcMask0.size(), SrcMask0, Mask0);
-    scaleShuffleMask<int>(MaskSize / SrcMask1.size(), SrcMask1, Mask1);
+    scaleShuffleMask(MaskSize / SrcMask0.size(), SrcMask0, Mask0);
+    scaleShuffleMask(MaskSize / SrcMask1.size(), SrcMask1, Mask1);
     for (size_t i = 0; i != MaskSize; ++i) {
       if (Mask0[i] == SM_SentinelUndef && Mask1[i] == SM_SentinelUndef)
         Mask.push_back(SM_SentinelUndef);
@@ -7362,7 +7414,7 @@ static bool getFauxShuffleMask(SDValue N, const APInt &DemandedElts,
       if ((NumSubElts % SubMask.size()) == 0) {
         int Scale = NumSubElts / SubMask.size();
         SmallVector<int,64> ScaledSubMask;
-        scaleShuffleMask<int>(Scale, SubMask, ScaledSubMask);
+        scaleShuffleMask(Scale, SubMask, ScaledSubMask);
         SubMask = ScaledSubMask;
       } else {
         int Scale = SubMask.size() / NumSubElts;
@@ -7417,15 +7469,20 @@ static bool getFauxShuffleMask(SDValue N, const APInt &DemandedElts,
       }
     }
 
+    // Peek through trunc/aext/zext.
+    // TODO: aext shouldn't require SM_SentinelZero padding.
+    // TODO: handle shift of scalars.
+    while (Scl.getOpcode() == ISD::TRUNCATE ||
+           Scl.getOpcode() == ISD::ANY_EXTEND ||
+           Scl.getOpcode() == ISD::ZERO_EXTEND)
+      Scl = Scl.getOperand(0);
+
     // Attempt to find the source vector the scalar was extracted from.
-    // TODO: Handle truncate/zext/shift of scalars.
     SDValue SrcExtract;
-    if ((Scl.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
-         Scl.getOperand(0).getValueType() == VT) ||
-        (Scl.getOpcode() == X86ISD::PEXTRW &&
-         Scl.getOperand(0).getValueType() == MVT::v8i16) ||
-        (Scl.getOpcode() == X86ISD::PEXTRB &&
-         Scl.getOperand(0).getValueType() == MVT::v16i8)) {
+    if ((Scl.getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
+         Scl.getOpcode() == X86ISD::PEXTRW ||
+         Scl.getOpcode() == X86ISD::PEXTRB) &&
+        Scl.getOperand(0).getValueSizeInBits() == NumSizeInBits) {
       SrcExtract = Scl;
     }
     if (!SrcExtract || !isa<ConstantSDNode>(SrcExtract.getOperand(1)))
@@ -7437,8 +7494,7 @@ static bool getFauxShuffleMask(SDValue N, const APInt &DemandedElts,
     unsigned NumZeros =
         std::max<int>((NumBitsPerElt / SrcVT.getScalarSizeInBits()) - 1, 0);
 
-    if (SrcVT.getSizeInBits() != VT.getSizeInBits() ||
-        (NumSrcElts % NumElts) != 0)
+    if ((NumSrcElts % NumElts) != 0)
       return false;
 
     unsigned SrcIdx = SrcExtract.getConstantOperandVal(1);
@@ -7498,6 +7554,24 @@ static bool getFauxShuffleMask(SDValue N, const APInt &DemandedElts,
     createPackShuffleMask(VT, Mask, IsUnary);
     return true;
   }
+  case ISD::TRUNCATE:
+  case X86ISD::VTRUNC: {
+    SDValue Src = N.getOperand(0);
+    EVT SrcVT = Src.getValueType();
+    // Truncated source must be a simple vector.
+    if (!SrcVT.isSimple() || (SrcVT.getSizeInBits() % 128) != 0 ||
+        (SrcVT.getScalarSizeInBits() % 8) != 0)
+      return false;
+    unsigned NumSrcElts = SrcVT.getVectorNumElements();
+    unsigned NumBitsPerSrcElt = SrcVT.getScalarSizeInBits();
+    unsigned Scale = NumBitsPerSrcElt / NumBitsPerElt;
+    assert((NumBitsPerSrcElt % NumBitsPerElt) == 0 && "Illegal truncation");
+    for (unsigned i = 0; i != NumSrcElts; ++i)
+      Mask.push_back(i * Scale);
+    Mask.append(NumElts - NumSrcElts, SM_SentinelZero);
+    Ops.push_back(Src);
+    return true;
+  }
   case X86ISD::VSHLI:
   case X86ISD::VSRLI: {
     uint64_t ShiftVal = N.getConstantOperandVal(1);
@@ -8065,11 +8139,11 @@ static SDValue LowerAsSplatVectorLoad(SDValue SrcOp, MVT VT, const SDLoc &dl,
 
     // FIXME: 256-bit vector instructions don't require a strict alignment,
     // improve this code to support it better.
-    unsigned RequiredAlign = VT.getSizeInBits()/8;
+    Align RequiredAlign(VT.getSizeInBits() / 8);
     SDValue Chain = LD->getChain();
     // Make sure the stack object alignment is at least 16 or 32.
     MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
-    if (DAG.InferPtrAlignment(Ptr) < RequiredAlign) {
+    if (DAG.InferPtrAlign(Ptr) < RequiredAlign) {
       if (MFI.isFixedObjectIndex(FI)) {
         // Can't change the alignment. FIXME: It's possible to compute
         // the exact stack offset and reference FI + adjust offset instead.
@@ -8084,9 +8158,9 @@ static SDValue LowerAsSplatVectorLoad(SDValue SrcOp, MVT VT, const SDLoc &dl,
     // Ptr + (Offset & ~15).
     if (Offset < 0)
       return SDValue();
-    if ((Offset % RequiredAlign) & 3)
+    if ((Offset % RequiredAlign.value()) & 3)
       return SDValue();
-    int64_t StartOffset = Offset & ~int64_t(RequiredAlign - 1);
+    int64_t StartOffset = Offset & ~int64_t(RequiredAlign.value() - 1);
     if (StartOffset) {
       SDLoc DL(Ptr);
       Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
@@ -8355,11 +8429,9 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
     if (TLI.isTypeLegal(VecVT)) {
       SDVTList Tys = DAG.getVTList(VecVT, MVT::Other);
       SDValue Ops[] = { LDBase->getChain(), LDBase->getBasePtr() };
-      SDValue ResNode =
-          DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, DL, Tys, Ops, VecSVT,
-                                  LDBase->getPointerInfo(),
-                                  LDBase->getAlignment(),
-                                  MachineMemOperand::MOLoad);
+      SDValue ResNode = DAG.getMemIntrinsicNode(
+          X86ISD::VZEXT_LOAD, DL, Tys, Ops, VecSVT, LDBase->getPointerInfo(),
+          LDBase->getAlign(), MachineMemOperand::MOLoad);
       for (auto *LD : Loads)
         if (LD)
           DAG.makeEquivalentMemoryOrdering(LD, ResNode);
@@ -8601,7 +8673,7 @@ static SDValue lowerBuildVectorAsBroadcast(BuildVectorSDNode *BVOp,
           SDValue CP = DAG.getConstantPool(C, PVT);
           unsigned Repeat = VT.getSizeInBits() / SplatBitSize;
 
-          unsigned Alignment = cast<ConstantPoolSDNode>(CP)->getAlignment();
+          MaybeAlign Alignment(cast<ConstantPoolSDNode>(CP)->getAlignment());
           SDVTList Tys =
               DAG.getVTList(MVT::getVectorVT(CVT, Repeat), MVT::Other);
           SDValue Ops[] = {DAG.getEntryNode(), CP};
@@ -8689,7 +8761,7 @@ static SDValue lowerBuildVectorAsBroadcast(BuildVectorSDNode *BVOp,
       const TargetLowering &TLI = DAG.getTargetLoweringInfo();
       SDValue CP =
           DAG.getConstantPool(C, TLI.getPointerTy(DAG.getDataLayout()));
-      unsigned Alignment = cast<ConstantPoolSDNode>(CP)->getAlignment();
+      MaybeAlign Alignment(cast<ConstantPoolSDNode>(CP)->getAlignment());
 
       SDVTList Tys = DAG.getVTList(VT, MVT::Other);
       SDValue Ops[] = {DAG.getEntryNode(), CP};
@@ -11062,6 +11134,45 @@ static SDValue lowerShuffleWithUNPCK256(const SDLoc &DL, MVT VT,
   return DAG.getNode(UnpackOpcode, DL, VT, V1, V1);
 }
 
+// Check if the mask can be mapped to a TRUNCATE or VTRUNC, truncating the
+// source into the lower elements and zeroing the upper elements.
+// TODO: Merge with matchShuffleAsVPMOV.
+static bool matchShuffleAsVTRUNC(MVT &SrcVT, MVT &DstVT, MVT VT,
+                                 ArrayRef<int> Mask, const APInt &Zeroable,
+                                 const X86Subtarget &Subtarget) {
+  if (!VT.is512BitVector() && !Subtarget.hasVLX())
+    return false;
+
+  unsigned NumElts = Mask.size();
+  unsigned EltSizeInBits = VT.getScalarSizeInBits();
+  unsigned MaxScale = 64 / EltSizeInBits;
+
+  for (unsigned Scale = 2; Scale <= MaxScale; Scale += Scale) {
+    unsigned SrcEltBits = EltSizeInBits * Scale;
+    if (SrcEltBits < 32 && !Subtarget.hasBWI())
+      continue;
+    unsigned NumSrcElts = NumElts / Scale;
+    if (!isSequentialOrUndefInRange(Mask, 0, NumSrcElts, 0, Scale))
+      continue;
+    unsigned UpperElts = NumElts - NumSrcElts;
+    if (!Zeroable.extractBits(UpperElts, NumSrcElts).isAllOnesValue())
+      continue;
+    SrcVT = MVT::getIntegerVT(EltSizeInBits * Scale);
+    SrcVT = MVT::getVectorVT(SrcVT, NumSrcElts);
+    DstVT = MVT::getIntegerVT(EltSizeInBits);
+    if ((NumSrcElts * EltSizeInBits) >= 128) {
+      // ISD::TRUNCATE
+      DstVT = MVT::getVectorVT(DstVT, NumSrcElts);
+    } else {
+      // X86ISD::VTRUNC
+      DstVT = MVT::getVectorVT(DstVT, 128 / EltSizeInBits);
+    }
+    return true;
+  }
+
+  return false;
+}
+
 static bool matchShuffleAsVPMOV(ArrayRef<int> Mask, bool SwappedOps,
                                 int Delta) {
   int Size = (int)Mask.size();
@@ -11155,22 +11266,93 @@ static SDValue lowerShuffleWithVPMOV(const SDLoc &DL, ArrayRef<int> Mask,
   return DAG.getNode(X86ISD::VTRUNC, DL, VT, Src);
 }
 
+/// Check whether a compaction lowering can be done by dropping even
+/// elements and compute how many times even elements must be dropped.
+///
+/// This handles shuffles which take every Nth element where N is a power of
+/// two. Example shuffle masks:
+///
+///  N = 1:  0,  2,  4,  6,  8, 10, 12, 14,  0,  2,  4,  6,  8, 10, 12, 14
+///  N = 1:  0,  2,  4,  6,  8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
+///  N = 2:  0,  4,  8, 12,  0,  4,  8, 12,  0,  4,  8, 12,  0,  4,  8, 12
+///  N = 2:  0,  4,  8, 12, 16, 20, 24, 28,  0,  4,  8, 12, 16, 20, 24, 28
+///  N = 3:  0,  8,  0,  8,  0,  8,  0,  8,  0,  8,  0,  8,  0,  8,  0,  8
+///  N = 3:  0,  8, 16, 24,  0,  8, 16, 24,  0,  8, 16, 24,  0,  8, 16, 24
+///
+/// Any of these lanes can of course be undef.
+///
+/// This routine only supports N <= 3.
+/// FIXME: Evaluate whether either AVX or AVX-512 have any opportunities here
+/// for larger N.
+///
+/// \returns N above, or the number of times even elements must be dropped if
+/// there is such a number. Otherwise returns zero.
+static int canLowerByDroppingEvenElements(ArrayRef<int> Mask,
+                                          bool IsSingleInput) {
+  // The modulus for the shuffle vector entries is based on whether this is
+  // a single input or not.
+  int ShuffleModulus = Mask.size() * (IsSingleInput ? 1 : 2);
+  assert(isPowerOf2_32((uint32_t)ShuffleModulus) &&
+         "We should only be called with masks with a power-of-2 size!");
+
+  uint64_t ModMask = (uint64_t)ShuffleModulus - 1;
+
+  // We track whether the input is viable for all power-of-2 strides 2^1, 2^2,
+  // and 2^3 simultaneously. This is because we may have ambiguity with
+  // partially undef inputs.
+  bool ViableForN[3] = {true, true, true};
+
+  for (int i = 0, e = Mask.size(); i < e; ++i) {
+    // Ignore undef lanes, we'll optimistically collapse them to the pattern we
+    // want.
+    if (Mask[i] < 0)
+      continue;
+
+    bool IsAnyViable = false;
+    for (unsigned j = 0; j != array_lengthof(ViableForN); ++j)
+      if (ViableForN[j]) {
+        uint64_t N = j + 1;
+
+        // The shuffle mask must be equal to (i * 2^N) % M.
+        if ((uint64_t)Mask[i] == (((uint64_t)i << N) & ModMask))
+          IsAnyViable = true;
+        else
+          ViableForN[j] = false;
+      }
+    // Early exit if we exhaust the possible powers of two.
+    if (!IsAnyViable)
+      break;
+  }
+
+  for (unsigned j = 0; j != array_lengthof(ViableForN); ++j)
+    if (ViableForN[j])
+      return j + 1;
+
+  // Return 0 as there is no viable power of two.
+  return 0;
+}
+
 // X86 has dedicated pack instructions that can handle specific truncation
 // operations: PACKSS and PACKUS.
+// Checks for compaction shuffle masks if MaxStages > 1.
+// TODO: Add support for matching multiple PACKSS/PACKUS stages.
 static bool matchShuffleWithPACK(MVT VT, MVT &SrcVT, SDValue &V1, SDValue &V2,
                                  unsigned &PackOpcode, ArrayRef<int> TargetMask,
                                  SelectionDAG &DAG,
-                                 const X86Subtarget &Subtarget) {
+                                 const X86Subtarget &Subtarget,
+                                 unsigned MaxStages = 1) {
   unsigned NumElts = VT.getVectorNumElements();
   unsigned BitSize = VT.getScalarSizeInBits();
-  MVT PackSVT = MVT::getIntegerVT(BitSize * 2);
-  MVT PackVT = MVT::getVectorVT(PackSVT, NumElts / 2);
+  assert(0 < MaxStages && MaxStages <= 3 && (BitSize << MaxStages) <= 64 &&
+         "Illegal maximum compaction");
 
-  auto MatchPACK = [&](SDValue N1, SDValue N2) {
+  auto MatchPACK = [&](SDValue N1, SDValue N2, MVT PackVT) {
+    unsigned NumSrcBits = PackVT.getScalarSizeInBits();
+    unsigned NumPackedBits = NumSrcBits - BitSize;
     SDValue VV1 = DAG.getBitcast(PackVT, N1);
     SDValue VV2 = DAG.getBitcast(PackVT, N2);
-    if (Subtarget.hasSSE41() || PackSVT == MVT::i16) {
-      APInt ZeroMask = APInt::getHighBitsSet(BitSize * 2, BitSize);
+    if (Subtarget.hasSSE41() || BitSize == 8) {
+      APInt ZeroMask = APInt::getHighBitsSet(NumSrcBits, NumPackedBits);
       if ((N1.isUndef() || DAG.MaskedValueIsZero(VV1, ZeroMask)) &&
           (N2.isUndef() || DAG.MaskedValueIsZero(VV2, ZeroMask))) {
         V1 = VV1;
@@ -11180,8 +11362,8 @@ static bool matchShuffleWithPACK(MVT VT, MVT &SrcVT, SDValue &V1, SDValue &V2,
         return true;
       }
     }
-    if ((N1.isUndef() || DAG.ComputeNumSignBits(VV1) > BitSize) &&
-        (N2.isUndef() || DAG.ComputeNumSignBits(VV2) > BitSize)) {
+    if ((N1.isUndef() || DAG.ComputeNumSignBits(VV1) > NumPackedBits) &&
+        (N2.isUndef() || DAG.ComputeNumSignBits(VV2) > NumPackedBits)) {
       V1 = VV1;
       V2 = VV2;
       SrcVT = PackVT;
@@ -11191,19 +11373,25 @@ static bool matchShuffleWithPACK(MVT VT, MVT &SrcVT, SDValue &V1, SDValue &V2,
     return false;
   };
 
-  // Try binary shuffle.
-  SmallVector<int, 32> BinaryMask;
-  createPackShuffleMask(VT, BinaryMask, false);
-  if (isTargetShuffleEquivalent(TargetMask, BinaryMask, V1, V2))
-    if (MatchPACK(V1, V2))
-      return true;
+  // Attempt to match against wider and wider compaction patterns.
+  for (unsigned NumStages = 1; NumStages <= MaxStages; ++NumStages) {
+    MVT PackSVT = MVT::getIntegerVT(BitSize << NumStages);
+    MVT PackVT = MVT::getVectorVT(PackSVT, NumElts >> NumStages);
 
-  // Try unary shuffle.
-  SmallVector<int, 32> UnaryMask;
-  createPackShuffleMask(VT, UnaryMask, true);
-  if (isTargetShuffleEquivalent(TargetMask, UnaryMask, V1))
-    if (MatchPACK(V1, V1))
-      return true;
+    // Try binary shuffle.
+    SmallVector<int, 32> BinaryMask;
+    createPackShuffleMask(VT, BinaryMask, false, NumStages);
+    if (isTargetShuffleEquivalent(TargetMask, BinaryMask, V1, V2))
+      if (MatchPACK(V1, V2, PackVT))
+        return true;
+
+    // Try unary shuffle.
+    SmallVector<int, 32> UnaryMask;
+    createPackShuffleMask(VT, UnaryMask, true, NumStages);
+    if (isTargetShuffleEquivalent(TargetMask, UnaryMask, V1))
+      if (MatchPACK(V1, V1, PackVT))
+        return true;
+  }
 
   return false;
 }
@@ -11213,12 +11401,44 @@ static SDValue lowerShuffleWithPACK(const SDLoc &DL, MVT VT, ArrayRef<int> Mask,
                                     const X86Subtarget &Subtarget) {
   MVT PackVT;
   unsigned PackOpcode;
-  if (matchShuffleWithPACK(VT, PackVT, V1, V2, PackOpcode, Mask, DAG,
-                           Subtarget))
-    return DAG.getNode(PackOpcode, DL, VT, DAG.getBitcast(PackVT, V1),
-                       DAG.getBitcast(PackVT, V2));
+  unsigned SizeBits = VT.getSizeInBits();
+  unsigned EltBits = VT.getScalarSizeInBits();
+  unsigned MaxStages = Log2_32(64 / EltBits);
+  if (!matchShuffleWithPACK(VT, PackVT, V1, V2, PackOpcode, Mask, DAG,
+                            Subtarget, MaxStages))
+    return SDValue();
 
-  return SDValue();
+  unsigned CurrentEltBits = PackVT.getScalarSizeInBits();
+  unsigned NumStages = Log2_32(CurrentEltBits / EltBits);
+
+  // Don't lower multi-stage packs on AVX512, truncation is better.
+  if (NumStages != 1 && SizeBits == 128 && Subtarget.hasVLX())
+    return SDValue();
+
+  // Pack to the largest type possible:
+  // vXi64/vXi32 -> PACK*SDW and vXi16 -> PACK*SWB.
+  unsigned MaxPackBits = 16;
+  if (CurrentEltBits > 16 &&
+      (PackOpcode == X86ISD::PACKSS || Subtarget.hasSSE41()))
+    MaxPackBits = 32;
+
+  // Repeatedly pack down to the target size.
+  SDValue Res;
+  for (unsigned i = 0; i != NumStages; ++i) {
+    unsigned SrcEltBits = std::min(MaxPackBits, CurrentEltBits);
+    unsigned NumSrcElts = SizeBits / SrcEltBits;
+    MVT SrcSVT = MVT::getIntegerVT(SrcEltBits);
+    MVT DstSVT = MVT::getIntegerVT(SrcEltBits / 2);
+    MVT SrcVT = MVT::getVectorVT(SrcSVT, NumSrcElts);
+    MVT DstVT = MVT::getVectorVT(DstSVT, NumSrcElts * 2);
+    Res = DAG.getNode(PackOpcode, DL, DstVT, DAG.getBitcast(SrcVT, V1),
+                      DAG.getBitcast(SrcVT, V2));
+    V1 = V2 = Res;
+    CurrentEltBits /= 2;
+  }
+  assert(Res && Res.getValueType() == VT &&
+         "Failed to lower compaction shuffle");
+  return Res;
 }
 
 /// Try to emit a bitmask instruction for a shuffle.
@@ -11845,11 +12065,11 @@ static SDValue lowerShuffleAsBitRotate(const SDLoc &DL, MVT VT, SDValue V1,
   return DAG.getBitcast(VT, Rot);
 }
 
-/// Try to lower a vector shuffle as a byte rotation.
+/// Try to match a vector shuffle as an element rotation.
 ///
 /// This is used for support PALIGNR for SSSE3 or VALIGND/Q for AVX512.
-static int matchShuffleAsByteRotate(SDValue &V1, SDValue &V2,
-                                    ArrayRef<int> Mask) {
+static int matchShuffleAsElementRotate(SDValue &V1, SDValue &V2,
+                                       ArrayRef<int> Mask) {
   int NumElts = Mask.size();
 
   // We need to detect various ways of spelling a rotation:
@@ -11944,7 +12164,7 @@ static int matchShuffleAsByteRotate(MVT VT, SDValue &V1, SDValue &V2,
   if (!is128BitLaneRepeatedShuffleMask(VT, Mask, RepeatedMask))
     return -1;
 
-  int Rotation = matchShuffleAsByteRotate(V1, V2, RepeatedMask);
+  int Rotation = matchShuffleAsElementRotate(V1, V2, RepeatedMask);
   if (Rotation <= 0)
     return -1;
 
@@ -12024,7 +12244,7 @@ static SDValue lowerShuffleAsVALIGN(const SDLoc &DL, MVT VT, SDValue V1,
          && "VLX required for 128/256-bit vectors");
 
   SDValue Lo = V1, Hi = V2;
-  int Rotation = matchShuffleAsByteRotate(Lo, Hi, Mask);
+  int Rotation = matchShuffleAsElementRotate(Lo, Hi, Mask);
   if (Rotation <= 0)
     return SDValue();
 
@@ -14512,72 +14732,6 @@ static SDValue lowerV8I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
                                               Mask, Subtarget, DAG);
 }
 
-/// Check whether a compaction lowering can be done by dropping even
-/// elements and compute how many times even elements must be dropped.
-///
-/// This handles shuffles which take every Nth element where N is a power of
-/// two. Example shuffle masks:
-///
-///  N = 1:  0,  2,  4,  6,  8, 10, 12, 14,  0,  2,  4,  6,  8, 10, 12, 14
-///  N = 1:  0,  2,  4,  6,  8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
-///  N = 2:  0,  4,  8, 12,  0,  4,  8, 12,  0,  4,  8, 12,  0,  4,  8, 12
-///  N = 2:  0,  4,  8, 12, 16, 20, 24, 28,  0,  4,  8, 12, 16, 20, 24, 28
-///  N = 3:  0,  8,  0,  8,  0,  8,  0,  8,  0,  8,  0,  8,  0,  8,  0,  8
-///  N = 3:  0,  8, 16, 24,  0,  8, 16, 24,  0,  8, 16, 24,  0,  8, 16, 24
-///
-/// Any of these lanes can of course be undef.
-///
-/// This routine only supports N <= 3.
-/// FIXME: Evaluate whether either AVX or AVX-512 have any opportunities here
-/// for larger N.
-///
-/// \returns N above, or the number of times even elements must be dropped if
-/// there is such a number. Otherwise returns zero.
-static int canLowerByDroppingEvenElements(ArrayRef<int> Mask,
-                                          bool IsSingleInput) {
-  // The modulus for the shuffle vector entries is based on whether this is
-  // a single input or not.
-  int ShuffleModulus = Mask.size() * (IsSingleInput ? 1 : 2);
-  assert(isPowerOf2_32((uint32_t)ShuffleModulus) &&
-         "We should only be called with masks with a power-of-2 size!");
-
-  uint64_t ModMask = (uint64_t)ShuffleModulus - 1;
-
-  // We track whether the input is viable for all power-of-2 strides 2^1, 2^2,
-  // and 2^3 simultaneously. This is because we may have ambiguity with
-  // partially undef inputs.
-  bool ViableForN[3] = {true, true, true};
-
-  for (int i = 0, e = Mask.size(); i < e; ++i) {
-    // Ignore undef lanes, we'll optimistically collapse them to the pattern we
-    // want.
-    if (Mask[i] < 0)
-      continue;
-
-    bool IsAnyViable = false;
-    for (unsigned j = 0; j != array_lengthof(ViableForN); ++j)
-      if (ViableForN[j]) {
-        uint64_t N = j + 1;
-
-        // The shuffle mask must be equal to (i * 2^N) % M.
-        if ((uint64_t)Mask[i] == (((uint64_t)i << N) & ModMask))
-          IsAnyViable = true;
-        else
-          ViableForN[j] = false;
-      }
-    // Early exit if we exhaust the possible powers of two.
-    if (!IsAnyViable)
-      break;
-  }
-
-  for (unsigned j = 0; j != array_lengthof(ViableForN); ++j)
-    if (ViableForN[j])
-      return j + 1;
-
-  // Return 0 as there is no viable power of two.
-  return 0;
-}
-
 static SDValue lowerShuffleWithPERMV(const SDLoc &DL, MVT VT,
                                      ArrayRef<int> Mask, SDValue V1,
                                      SDValue V2, SelectionDAG &DAG) {
@@ -14760,6 +14914,10 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
                                               Zeroable, Subtarget, DAG))
     return V;
 
+  // Check for compaction patterns.
+  bool IsSingleInput = V2.isUndef();
+  int NumEvenDrops = canLowerByDroppingEvenElements(Mask, IsSingleInput);
+
   // Check for SSSE3 which lets us lower all v16i8 shuffles much more directly
   // with PSHUFB. It is important to do this before we attempt to generate any
   // blends but after all of the single-input lowerings. If the single input
@@ -14770,10 +14928,13 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // and there are *very* few patterns that would actually be faster than the
   // PSHUFB approach because of its ability to zero lanes.
   //
+  // If the mask is a binary compaction, we can more efficiently perform this
+  // as a PACKUS(AND(),AND()) - which is quicker than UNPACK(PSHUFB(),PSHUFB()).
+  //
   // FIXME: The only exceptions to the above are blends which are exact
   // interleavings with direct instructions supporting them. We currently don't
   // handle those well here.
-  if (Subtarget.hasSSSE3()) {
+  if (Subtarget.hasSSSE3() && (IsSingleInput || NumEvenDrops != 1)) {
     bool V1InUse = false;
     bool V2InUse = false;
 
@@ -14831,8 +14992,7 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // We special case these as they can be particularly efficiently handled with
   // the PACKUSB instruction on x86 and they show up in common patterns of
   // rearranging bytes to truncate wide elements.
-  bool IsSingleInput = V2.isUndef();
-  if (int NumEvenDrops = canLowerByDroppingEvenElements(Mask, IsSingleInput)) {
+  if (NumEvenDrops) {
     // NumEvenDrops is the power of two stride of the elements. Another way of
     // thinking about it is that we need to drop the even elements this many
     // times to get the original input.
@@ -14840,23 +15000,23 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
     // First we need to zero all the dropped bytes.
     assert(NumEvenDrops <= 3 &&
            "No support for dropping even elements more than 3 times.");
-    SmallVector<SDValue, 16> ByteClearOps(16, DAG.getConstant(0, DL, MVT::i8));
-    for (unsigned i = 0; i != 16; i += 1 << NumEvenDrops)
-      ByteClearOps[i] = DAG.getConstant(0xFF, DL, MVT::i8);
-    SDValue ByteClearMask = DAG.getBuildVector(MVT::v16i8, DL, ByteClearOps);
-    V1 = DAG.getNode(ISD::AND, DL, MVT::v16i8, V1, ByteClearMask);
+    SmallVector<SDValue, 8> WordClearOps(8, DAG.getConstant(0, DL, MVT::i16));
+    for (unsigned i = 0; i != 8; i += 1 << (NumEvenDrops - 1))
+      WordClearOps[i] = DAG.getConstant(0xFF, DL, MVT::i16);
+    SDValue WordClearMask = DAG.getBuildVector(MVT::v8i16, DL, WordClearOps);
+    V1 = DAG.getNode(ISD::AND, DL, MVT::v8i16, DAG.getBitcast(MVT::v8i16, V1),
+                     WordClearMask);
     if (!IsSingleInput)
-      V2 = DAG.getNode(ISD::AND, DL, MVT::v16i8, V2, ByteClearMask);
+      V2 = DAG.getNode(ISD::AND, DL, MVT::v8i16, DAG.getBitcast(MVT::v8i16, V2),
+                       WordClearMask);
 
     // Now pack things back together.
-    V1 = DAG.getBitcast(MVT::v8i16, V1);
-    V2 = IsSingleInput ? V1 : DAG.getBitcast(MVT::v8i16, V2);
-    SDValue Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1, V2);
+    SDValue Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1,
+                                 IsSingleInput ? V1 : V2);
     for (int i = 1; i < NumEvenDrops; ++i) {
       Result = DAG.getBitcast(MVT::v8i16, Result);
       Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, Result, Result);
     }
-
     return Result;
   }
 
@@ -16199,7 +16359,7 @@ static SDValue lowerV4I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
     SmallVector<int, 2> RepeatedMask;
     if (is128BitLaneRepeatedShuffleMask(MVT::v4i64, Mask, RepeatedMask)) {
       SmallVector<int, 4> PSHUFDMask;
-      scaleShuffleMask<int>(2, RepeatedMask, PSHUFDMask);
+      scaleShuffleMask(2, RepeatedMask, PSHUFDMask);
       return DAG.getBitcast(
           MVT::v4i64,
           DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32,
@@ -16781,13 +16941,14 @@ static SDValue lowerV4X128Shuffle(const SDLoc &DL, MVT VT, ArrayRef<int> Mask,
   assert(VT.is512BitVector() && "Unexpected vector size for 512bit shuffle.");
 
   // TODO - use Zeroable like we do for lowerV2X128VectorShuffle?
-  SmallVector<int, 4> WidenedMask;
-  if (!canWidenShuffleElements(Mask, WidenedMask))
+  SmallVector<int, 4> Widened128Mask;
+  if (!canWidenShuffleElements(Mask, Widened128Mask))
     return SDValue();
+  assert(Widened128Mask.size() == 4 && "Shuffle widening mismatch");
 
   // Try to use an insert into a zero vector.
-  if (WidenedMask[0] == 0 && (Zeroable & 0xf0) == 0xf0 &&
-      (WidenedMask[1] == 1 || (Zeroable & 0x0c) == 0x0c)) {
+  if (Widened128Mask[0] == 0 && (Zeroable & 0xf0) == 0xf0 &&
+      (Widened128Mask[1] == 1 || (Zeroable & 0x0c) == 0x0c)) {
     unsigned NumElts = ((Zeroable & 0x0c) == 0x0c) ? 2 : 4;
     MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), NumElts);
     SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1,
@@ -16799,37 +16960,34 @@ static SDValue lowerV4X128Shuffle(const SDLoc &DL, MVT VT, ArrayRef<int> Mask,
 
   // Check for patterns which can be matched with a single insert of a 256-bit
   // subvector.
-  bool OnlyUsesV1 = isShuffleEquivalent(V1, V2, Mask,
-                                        {0, 1, 2, 3, 0, 1, 2, 3});
-  if (OnlyUsesV1 || isShuffleEquivalent(V1, V2, Mask,
-                                        {0, 1, 2, 3, 8, 9, 10, 11})) {
+  bool OnlyUsesV1 = isShuffleEquivalent(V1, V2, Mask, {0, 1, 2, 3, 0, 1, 2, 3});
+  if (OnlyUsesV1 ||
+      isShuffleEquivalent(V1, V2, Mask, {0, 1, 2, 3, 8, 9, 10, 11})) {
     MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 4);
-    SDValue SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT,
-                                 OnlyUsesV1 ? V1 : V2,
-                              DAG.getIntPtrConstant(0, DL));
+    SDValue SubVec =
+        DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, OnlyUsesV1 ? V1 : V2,
+                    DAG.getIntPtrConstant(0, DL));
     return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, V1, SubVec,
                        DAG.getIntPtrConstant(4, DL));
   }
 
-  assert(WidenedMask.size() == 4);
-
   // See if this is an insertion of the lower 128-bits of V2 into V1.
   bool IsInsert = true;
   int V2Index = -1;
   for (int i = 0; i < 4; ++i) {
-    assert(WidenedMask[i] >= -1);
-    if (WidenedMask[i] < 0)
+    assert(Widened128Mask[i] >= -1 && "Illegal shuffle sentinel value");
+    if (Widened128Mask[i] < 0)
       continue;
 
     // Make sure all V1 subvectors are in place.
-    if (WidenedMask[i] < 4) {
-      if (WidenedMask[i] != i) {
+    if (Widened128Mask[i] < 4) {
+      if (Widened128Mask[i] != i) {
         IsInsert = false;
         break;
       }
     } else {
       // Make sure we only have a single V2 index and its the lowest 128-bits.
-      if (V2Index >= 0 || WidenedMask[i] != 4) {
+      if (V2Index >= 0 || Widened128Mask[i] != 4) {
         IsInsert = false;
         break;
       }
@@ -16843,16 +17001,26 @@ static SDValue lowerV4X128Shuffle(const SDLoc &DL, MVT VT, ArrayRef<int> Mask,
     return insert128BitVector(V1, Subvec, V2Index * 2, DAG, DL);
   }
 
+  // See if we can widen to a 256-bit lane shuffle, we're going to lose 128-lane
+  // UNDEF info by lowering to X86ISD::SHUF128 anyway, so by widening where
+  // possible we at least ensure the lanes stay sequential to help later
+  // combines.
+  SmallVector<int, 2> Widened256Mask;
+  if (canWidenShuffleElements(Widened128Mask, Widened256Mask)) {
+    Widened128Mask.clear();
+    llvm::scaleShuffleMask(2, Widened256Mask, Widened128Mask);
+  }
+
   // Try to lower to vshuf64x2/vshuf32x4.
   SDValue Ops[2] = {DAG.getUNDEF(VT), DAG.getUNDEF(VT)};
   unsigned PermMask = 0;
   // Insure elements came from the same Op.
   for (int i = 0; i < 4; ++i) {
-    assert(WidenedMask[i] >= -1);
-    if (WidenedMask[i] < 0)
+    assert(Widened128Mask[i] >= -1 && "Illegal shuffle sentinel value");
+    if (Widened128Mask[i] < 0)
       continue;
 
-    SDValue Op = WidenedMask[i] >= 4 ? V2 : V1;
+    SDValue Op = Widened128Mask[i] >= 4 ? V2 : V1;
     unsigned OpIndex = i / 2;
     if (Ops[OpIndex].isUndef())
       Ops[OpIndex] = Op;
@@ -16861,7 +17029,7 @@ static SDValue lowerV4X128Shuffle(const SDLoc &DL, MVT VT, ArrayRef<int> Mask,
 
     // Convert the 128-bit shuffle mask selection values into 128-bit selection
     // bits defined by a vshuf64x2 instruction's immediate control byte.
-    PermMask |= (WidenedMask[i] % 4) << (i * 2);
+    PermMask |= (Widened128Mask[i] % 4) << (i * 2);
   }
 
   return DAG.getNode(X86ISD::SHUF128, DL, VT, Ops[0], Ops[1],
@@ -16991,7 +17159,7 @@ static SDValue lowerV8I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
     SmallVector<int, 2> Repeated128Mask;
     if (is128BitLaneRepeatedShuffleMask(MVT::v8i64, Mask, Repeated128Mask)) {
       SmallVector<int, 4> PSHUFDMask;
-      scaleShuffleMask<int>(2, Repeated128Mask, PSHUFDMask);
+      scaleShuffleMask(2, Repeated128Mask, PSHUFDMask);
       return DAG.getBitcast(
           MVT::v8i64,
           DAG.getNode(X86ISD::PSHUFD, DL, MVT::v16i32,
@@ -17128,6 +17296,11 @@ static SDValue lowerV32I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i16, Mask, V1, V2, DAG))
     return V;
 
+  // Use dedicated pack instructions for masks that match their pattern.
+  if (SDValue V =
+          lowerShuffleWithPACK(DL, MVT::v32i16, Mask, V1, V2, DAG, Subtarget))
+    return V;
+
   // Try to use shift instructions.
   if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v32i16, V1, V2, Mask,
                                           Zeroable, Subtarget, DAG))
@@ -17149,13 +17322,13 @@ static SDValue lowerV32I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
       // As this is a single-input shuffle, the repeated mask should be
       // a strictly valid v8i16 mask that we can pass through to the v8i16
       // lowering to handle even the v32 case.
-      return lowerV8I16GeneralSingleInputShuffle(
-          DL, MVT::v32i16, V1, RepeatedMask, Subtarget, DAG);
+      return lowerV8I16GeneralSingleInputShuffle(DL, MVT::v32i16, V1,
+                                                 RepeatedMask, Subtarget, DAG);
     }
   }
 
   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v32i16, V1, V2, Mask,
-                                                Zeroable, Subtarget, DAG))
+                                          Zeroable, Subtarget, DAG))
     return Blend;
 
   if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v32i16, Mask, V1, V2,
@@ -19126,15 +19299,16 @@ SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op,
     ValueToStore = DAG.getBitcast(MVT::f64, ValueToStore);
 
   unsigned Size = SrcVT.getStoreSize();
+  Align Alignment(Size);
   MachineFunction &MF = DAG.getMachineFunction();
   auto PtrVT = getPointerTy(MF.getDataLayout());
-  int SSFI = MF.getFrameInfo().CreateStackObject(Size, Size, false);
+  int SSFI = MF.getFrameInfo().CreateStackObject(Size, Alignment, false);
   MachinePointerInfo MPI =
       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SSFI);
   SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
-  Chain = DAG.getStore(Chain, dl, ValueToStore, StackSlot, MPI, Size);
+  Chain = DAG.getStore(Chain, dl, ValueToStore, StackSlot, MPI, Alignment);
   std::pair<SDValue, SDValue> Tmp =
-      BuildFILD(VT, SrcVT, dl, Chain, StackSlot, MPI, Size, DAG);
+      BuildFILD(VT, SrcVT, dl, Chain, StackSlot, MPI, Alignment, DAG);
 
   if (IsStrict)
     return DAG.getMergeValues({Tmp.first, Tmp.second}, dl);
@@ -19144,7 +19318,7 @@ SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op,
 
 std::pair<SDValue, SDValue> X86TargetLowering::BuildFILD(
     EVT DstVT, EVT SrcVT, const SDLoc &DL, SDValue Chain, SDValue Pointer,
-    MachinePointerInfo PtrInfo, unsigned Align, SelectionDAG &DAG) const {
+    MachinePointerInfo PtrInfo, Align Alignment, SelectionDAG &DAG) const {
   // Build the FILD
   SDVTList Tys;
   bool useSSE = isScalarFPTypeInSSEReg(DstVT);
@@ -19156,7 +19330,7 @@ std::pair<SDValue, SDValue> X86TargetLowering::BuildFILD(
   SDValue FILDOps[] = {Chain, Pointer};
   SDValue Result =
       DAG.getMemIntrinsicNode(X86ISD::FILD, DL, Tys, FILDOps, SrcVT, PtrInfo,
-                              Align, MachineMemOperand::MOLoad);
+                              Alignment, MachineMemOperand::MOLoad);
   Chain = Result.getValue(1);
 
   if (useSSE) {
@@ -19169,7 +19343,7 @@ std::pair<SDValue, SDValue> X86TargetLowering::BuildFILD(
     SDValue FSTOps[] = {Chain, Result, StackSlot};
     MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
         MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SSFI),
-        MachineMemOperand::MOStore, SSFISize, SSFISize);
+        MachineMemOperand::MOStore, SSFISize, Align(SSFISize));
 
     Chain =
         DAG.getMemIntrinsicNode(X86ISD::FST, DL, Tys, FSTOps, DstVT, StoreMMO);
@@ -19437,8 +19611,8 @@ static SDValue lowerUINT_TO_FP_vXi32(SDValue Op, SelectionDAG &DAG,
     SDValue Ops[] = {DAG.getEntryNode(), CPIdx};
     SDValue VBias = DAG.getMemIntrinsicNode(
         X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MVT::f64,
-        MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
-        /*Alignment*/ 8, MachineMemOperand::MOLoad);
+        MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Align(8),
+        MachineMemOperand::MOLoad);
 
     SDValue Or = DAG.getNode(ISD::OR, DL, MVT::v4i64, ZExtIn,
                              DAG.getBitcast(MVT::v4i64, VBias));
@@ -19617,7 +19791,7 @@ SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op,
     SDValue Store2 = DAG.getStore(Store1, dl, DAG.getConstant(0, dl, MVT::i32),
                                   OffsetSlot, MPI.getWithOffset(4), 4);
     std::pair<SDValue, SDValue> Tmp =
-        BuildFILD(DstVT, MVT::i64, dl, Store2, StackSlot, MPI, 8, DAG);
+        BuildFILD(DstVT, MVT::i64, dl, Store2, StackSlot, MPI, Align(8), DAG);
     if (IsStrict)
       return DAG.getMergeValues({Tmp.first, Tmp.second}, dl);
 
@@ -19633,7 +19807,7 @@ SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op,
     ValueToStore = DAG.getBitcast(MVT::f64, ValueToStore);
   }
   SDValue Store =
-      DAG.getStore(Chain, dl, ValueToStore, StackSlot, MPI, 8 /*Align*/);
+      DAG.getStore(Chain, dl, ValueToStore, StackSlot, MPI, Align(8));
   // For i64 source, we need to add the appropriate power of 2 if the input
   // was negative.  This is the same as the optimization in
   // DAGTypeLegalizer::ExpandIntOp_UNIT_TO_FP, and for it to be safe here,
@@ -19641,9 +19815,9 @@ SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op,
   // in SSE. (The generic code can't know it's OK to do this, or how to.)
   SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
   SDValue Ops[] = { Store, StackSlot };
-  SDValue Fild = DAG.getMemIntrinsicNode(X86ISD::FILD, dl, Tys, Ops,
-                                         MVT::i64, MPI, 8 /*Align*/,
-                                         MachineMemOperand::MOLoad);
+  SDValue Fild =
+      DAG.getMemIntrinsicNode(X86ISD::FILD, dl, Tys, Ops, MVT::i64, MPI,
+                              Align(8), MachineMemOperand::MOLoad);
   Chain = Fild.getValue(1);
 
 
@@ -19812,14 +19986,14 @@ X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
     unsigned FLDSize = TheVT.getStoreSize();
     assert(FLDSize <= MemSize && "Stack slot not big enough");
     MachineMemOperand *MMO = MF.getMachineMemOperand(
-        MPI, MachineMemOperand::MOLoad, FLDSize, FLDSize);
+        MPI, MachineMemOperand::MOLoad, FLDSize, Align(FLDSize));
     Value = DAG.getMemIntrinsicNode(X86ISD::FLD, DL, Tys, Ops, TheVT, MMO);
     Chain = Value.getValue(1);
   }
 
   // Build the FP_TO_INT*_IN_MEM
   MachineMemOperand *MMO = MF.getMachineMemOperand(
-      MPI, MachineMemOperand::MOStore, MemSize, MemSize);
+      MPI, MachineMemOperand::MOStore, MemSize, Align(MemSize));
   SDValue Ops[] = { Chain, Value, StackSlot };
   SDValue FIST = DAG.getMemIntrinsicNode(X86ISD::FP_TO_INT_IN_MEM, DL,
                                          DAG.getVTList(MVT::Other),
@@ -20073,7 +20247,7 @@ static SDValue truncateVectorWithPACK(unsigned Opcode, EVT DstVT, SDValue In,
     // Scale shuffle mask to avoid bitcasts and help ComputeNumSignBits.
     SmallVector<int, 64> Mask;
     int Scale = 64 / OutVT.getScalarSizeInBits();
-    scaleShuffleMask<int>(Scale, ArrayRef<int>({ 0, 2, 1, 3 }), Mask);
+    scaleShuffleMask(Scale, { 0, 2, 1, 3 }, Mask);
     Res = DAG.getVectorShuffle(OutVT, DL, Res, Res, Mask);
 
     if (DstVT.is256BitVector())
@@ -20594,14 +20768,13 @@ SDValue X86TargetLowering::LRINT_LLRINTHelper(SDNode *N,
     SDValue Ops[] = { Chain, StackPtr };
 
     Src = DAG.getMemIntrinsicNode(X86ISD::FLD, DL, Tys, Ops, SrcVT, MPI,
-                                  /*Align*/0, MachineMemOperand::MOLoad);
+                                  /*Align*/ None, MachineMemOperand::MOLoad);
     Chain = Src.getValue(1);
   }
 
   SDValue StoreOps[] = { Chain, Src, StackPtr };
-  Chain = DAG.getMemIntrinsicNode(X86ISD::FIST, DL,
-                                  DAG.getVTList(MVT::Other), StoreOps,
-                                  DstVT, MPI, /*Align*/0,
+  Chain = DAG.getMemIntrinsicNode(X86ISD::FIST, DL, DAG.getVTList(MVT::Other),
+                                  StoreOps, DstVT, MPI, /*Align*/ None,
                                   MachineMemOperand::MOStore);
 
   return DAG.getLoad(DstVT, DL, Chain, StackPtr, MPI);
@@ -23501,11 +23674,8 @@ SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
                        DAG.getConstant(Align, dl, MVT::i32)};
   SDVTList VTs = DAG.getVTList(getPointerTy(DAG.getDataLayout()), MVT::Other);
   SDValue VAARG = DAG.getMemIntrinsicNode(
-    X86ISD::VAARG_64, dl,
-    VTs, InstOps, MVT::i64,
-    MachinePointerInfo(SV),
-    /*Align=*/0,
-    MachineMemOperand::MOLoad | MachineMemOperand::MOStore);
+      X86ISD::VAARG_64, dl, VTs, InstOps, MVT::i64, MachinePointerInfo(SV),
+      /*Align=*/None, MachineMemOperand::MOLoad | MachineMemOperand::MOStore);
   Chain = VAARG.getValue(1);
 
   // Load the next argument and return it
@@ -25687,10 +25857,10 @@ SDValue X86TargetLowering::LowerFLT_ROUNDS_(SDValue Op,
   SDValue Ops[] = {Chain, StackSlot};
   Chain = DAG.getMemIntrinsicNode(X86ISD::FNSTCW16m, DL,
                                   DAG.getVTList(MVT::Other), Ops, MVT::i16, MPI,
-                                  2 /*Align*/, MachineMemOperand::MOStore);
+                                  Align(2), MachineMemOperand::MOStore);
 
   // Load FP Control Word from stack slot
-  SDValue CWD = DAG.getLoad(MVT::i16, DL, Chain, StackSlot, MPI, 2 /*Align*/);
+  SDValue CWD = DAG.getLoad(MVT::i16, DL, Chain, StackSlot, MPI, Align(2));
   Chain = CWD.getValue(1);
 
   // Mask and turn the control bits into a shift for the lookup table.
@@ -28365,7 +28535,7 @@ static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG,
         SDValue LdOps[] = {Chain, StackPtr};
         SDValue Value =
             DAG.getMemIntrinsicNode(X86ISD::FILD, dl, Tys, LdOps, MVT::i64, MPI,
-                                    /*Align*/ 0, MachineMemOperand::MOLoad);
+                                    /*Align*/ None, MachineMemOperand::MOLoad);
         Chain = Value.getValue(1);
 
         // Now use an FIST to do the atomic store.
@@ -29776,10 +29946,9 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
         MachinePointerInfo MPI =
             MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
         SDValue StoreOps[] = { Chain, Result, StackPtr };
-        Chain = DAG.getMemIntrinsicNode(X86ISD::FIST, dl,
-                                        DAG.getVTList(MVT::Other), StoreOps,
-                                        MVT::i64, MPI, 0 /*Align*/,
-                                        MachineMemOperand::MOStore);
+        Chain = DAG.getMemIntrinsicNode(
+            X86ISD::FIST, dl, DAG.getVTList(MVT::Other), StoreOps, MVT::i64,
+            MPI, None /*Align*/, MachineMemOperand::MOStore);
 
         // Finally load the value back from the stack temporary and return it.
         // This load is not atomic and doesn't need to be.
@@ -30572,8 +30741,8 @@ bool X86TargetLowering::isVectorClearMaskLegal(ArrayRef<int> Mask,
 }
 
 bool X86TargetLowering::areJTsAllowed(const Function *Fn) const {
-  // If the subtarget is using retpolines, we need to not generate jump tables.
-  if (Subtarget.useRetpolineIndirectBranches())
+  // If the subtarget is using thunks, we need to not generate jump tables.
+  if (Subtarget.useIndirectThunkBranches())
     return false;
 
   // Otherwise, fallback on the generic logic.
@@ -30981,7 +31150,7 @@ MachineBasicBlock *X86TargetLowering::EmitVAStartSaveXMMRegsWithCustomInserter(
     MachineMemOperand *MMO = F->getMachineMemOperand(
         MachinePointerInfo::getFixedStack(*F, RegSaveFrameIndex, Offset),
         MachineMemOperand::MOStore,
-        /*Size=*/16, /*Align=*/16);
+        /*Size=*/16, Align(16));
     BuildMI(XMMSaveMBB, DL, TII->get(MOVOpc))
         .addFrameIndex(RegSaveFrameIndex)
         .addImm(/*Scale=*/1)
@@ -31776,22 +31945,22 @@ X86TargetLowering::EmitLoweredTLSCall(MachineInstr &MI,
   return BB;
 }
 
-static unsigned getOpcodeForRetpoline(unsigned RPOpc) {
+static unsigned getOpcodeForIndirectThunk(unsigned RPOpc) {
   switch (RPOpc) {
-  case X86::RETPOLINE_CALL32:
+  case X86::INDIRECT_THUNK_CALL32:
     return X86::CALLpcrel32;
-  case X86::RETPOLINE_CALL64:
+  case X86::INDIRECT_THUNK_CALL64:
     return X86::CALL64pcrel32;
-  case X86::RETPOLINE_TCRETURN32:
+  case X86::INDIRECT_THUNK_TCRETURN32:
     return X86::TCRETURNdi;
-  case X86::RETPOLINE_TCRETURN64:
+  case X86::INDIRECT_THUNK_TCRETURN64:
     return X86::TCRETURNdi64;
   }
-  llvm_unreachable("not retpoline opcode");
+  llvm_unreachable("not indirect thunk opcode");
 }
 
-static const char *getRetpolineSymbol(const X86Subtarget &Subtarget,
-                                      unsigned Reg) {
+static const char *getIndirectThunkSymbol(const X86Subtarget &Subtarget,
+                                          unsigned Reg) {
   if (Subtarget.useRetpolineExternalThunk()) {
     // When using an external thunk for retpolines, we pick names that match the
     // names GCC happens to use as well. This helps simplify the implementation
@@ -31823,39 +31992,48 @@ static const char *getRetpolineSymbol(const X86Subtarget &Subtarget,
       assert(Subtarget.is64Bit() && "Should not be using a 64-bit thunk!");
       return "__x86_indirect_thunk_r11";
     }
+    llvm_unreachable("unexpected reg for external indirect thunk");
+  }
+
+  if (Subtarget.useRetpolineIndirectCalls() ||
+      Subtarget.useRetpolineIndirectBranches()) {
+    // When targeting an internal COMDAT thunk use an LLVM-specific name.
+    switch (Reg) {
+    case X86::EAX:
+      assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
+      return "__llvm_retpoline_eax";
+    case X86::ECX:
+      assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
+      return "__llvm_retpoline_ecx";
+    case X86::EDX:
+      assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
+      return "__llvm_retpoline_edx";
+    case X86::EDI:
+      assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
+      return "__llvm_retpoline_edi";
+    case X86::R11:
+      assert(Subtarget.is64Bit() && "Should not be using a 64-bit thunk!");
+      return "__llvm_retpoline_r11";
+    }
     llvm_unreachable("unexpected reg for retpoline");
   }
 
-  // When targeting an internal COMDAT thunk use an LLVM-specific name.
-  switch (Reg) {
-  case X86::EAX:
-    assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
-    return "__llvm_retpoline_eax";
-  case X86::ECX:
-    assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
-    return "__llvm_retpoline_ecx";
-  case X86::EDX:
-    assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
-    return "__llvm_retpoline_edx";
-  case X86::EDI:
-    assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
-    return "__llvm_retpoline_edi";
-  case X86::R11:
+  if (Subtarget.useLVIControlFlowIntegrity()) {
     assert(Subtarget.is64Bit() && "Should not be using a 64-bit thunk!");
-    return "__llvm_retpoline_r11";
+    return "__llvm_lvi_thunk_r11";
   }
-  llvm_unreachable("unexpected reg for retpoline");
+  llvm_unreachable("getIndirectThunkSymbol() invoked without thunk feature");
 }
 
 MachineBasicBlock *
-X86TargetLowering::EmitLoweredRetpoline(MachineInstr &MI,
-                                        MachineBasicBlock *BB) const {
+X86TargetLowering::EmitLoweredIndirectThunk(MachineInstr &MI,
+                                            MachineBasicBlock *BB) const {
   // Copy the virtual register into the R11 physical register and
   // call the retpoline thunk.
   DebugLoc DL = MI.getDebugLoc();
   const X86InstrInfo *TII = Subtarget.getInstrInfo();
   Register CalleeVReg = MI.getOperand(0).getReg();
-  unsigned Opc = getOpcodeForRetpoline(MI.getOpcode());
+  unsigned Opc = getOpcodeForIndirectThunk(MI.getOpcode());
 
   // Find an available scratch register to hold the callee. On 64-bit, we can
   // just use R11, but we scan for uses anyway to ensure we don't generate
@@ -31889,7 +32067,7 @@ X86TargetLowering::EmitLoweredRetpoline(MachineInstr &MI,
     report_fatal_error("calling convention incompatible with retpoline, no "
                        "available registers");
 
-  const char *Symbol = getRetpolineSymbol(Subtarget, AvailableReg);
+  const char *Symbol = getIndirectThunkSymbol(Subtarget, AvailableReg);
 
   BuildMI(*BB, MI, DL, TII->get(TargetOpcode::COPY), AvailableReg)
       .addReg(CalleeVReg);
@@ -32665,11 +32843,11 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
   case X86::TLS_base_addr32:
   case X86::TLS_base_addr64:
     return EmitLoweredTLSAddr(MI, BB);
-  case X86::RETPOLINE_CALL32:
-  case X86::RETPOLINE_CALL64:
-  case X86::RETPOLINE_TCRETURN32:
-  case X86::RETPOLINE_TCRETURN64:
-    return EmitLoweredRetpoline(MI, BB);
+  case X86::INDIRECT_THUNK_CALL32:
+  case X86::INDIRECT_THUNK_CALL64:
+  case X86::INDIRECT_THUNK_TCRETURN32:
+  case X86::INDIRECT_THUNK_TCRETURN64:
+    return EmitLoweredIndirectThunk(MI, BB);
   case X86::CATCHRET:
     return EmitLoweredCatchRet(MI, BB);
   case X86::SEG_ALLOCA_32:
@@ -33192,11 +33370,12 @@ unsigned X86TargetLowering::ComputeNumSignBitsForTargetNode(
     return VTBits;
 
   case X86ISD::VTRUNC: {
-    // TODO: Add DemandedElts support.
     SDValue Src = Op.getOperand(0);
-    unsigned NumSrcBits = Src.getScalarValueSizeInBits();
+    MVT SrcVT = Src.getSimpleValueType();
+    unsigned NumSrcBits = SrcVT.getScalarSizeInBits();
     assert(VTBits < NumSrcBits && "Illegal truncation input type");
-    unsigned Tmp = DAG.ComputeNumSignBits(Src, Depth + 1);
+    APInt DemandedSrc = DemandedElts.zextOrTrunc(SrcVT.getVectorNumElements());
+    unsigned Tmp = DAG.ComputeNumSignBits(Src, DemandedSrc, Depth + 1);
     if (Tmp > (NumSrcBits - VTBits))
       return Tmp - (NumSrcBits - VTBits);
     return 1;
@@ -33518,7 +33697,7 @@ static bool matchUnaryPermuteShuffle(MVT MaskVT, ArrayRef<int> Mask,
       // Narrow the repeated mask to create 32-bit element permutes.
       SmallVector<int, 4> WordMask = RepeatedMask;
       if (MaskScalarSizeInBits == 64)
-        scaleShuffleMask<int>(2, RepeatedMask, WordMask);
+        scaleShuffleMask(2, RepeatedMask, WordMask);
 
       Shuffle = (AllowIntDomain ? X86ISD::PSHUFD : X86ISD::VPERMILPI);
       ShuffleVT = (AllowIntDomain ? MVT::i32 : MVT::f32);
@@ -33883,13 +34062,50 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
     }
   }
 
-  // TODO - handle 128/256-bit lane shuffles of 512-bit vectors.
+  // Handle 128/256-bit lane shuffles of 512-bit vectors.
+  if (RootVT.is512BitVector() &&
+      (NumBaseMaskElts == 2 || NumBaseMaskElts == 4)) {
+    MVT ShuffleVT = (FloatDomain ? MVT::v8f64 : MVT::v8i64);
+
+    // If the upper subvectors are zeroable, then an extract+insert is more
+    // optimal than using X86ISD::SHUF128. The insertion is free, even if it has
+    // to zero the upper subvectors.
+    if (isUndefOrZeroInRange(BaseMask, 1, NumBaseMaskElts - 1)) {
+      if (Depth == 0 && Root.getOpcode() == ISD::INSERT_SUBVECTOR)
+        return SDValue(); // Nothing to do!
+      assert(isInRange(BaseMask[0], 0, NumBaseMaskElts) &&
+             "Unexpected lane shuffle");
+      Res = DAG.getBitcast(ShuffleVT, V1);
+      unsigned SubIdx = BaseMask[0] * (8 / NumBaseMaskElts);
+      bool UseZero = isAnyZero(BaseMask);
+      Res = extractSubVector(Res, SubIdx, DAG, DL, BaseMaskEltSizeInBits);
+      Res = widenSubVector(Res, UseZero, Subtarget, DAG, DL, RootSizeInBits);
+      return DAG.getBitcast(RootVT, Res);
+    }
+
+    // TODO - handle AVX512 cases with X86ISD::SHUF128.
+  }
 
   // Handle 128-bit lane shuffles of 256-bit vectors.
   if (RootVT.is256BitVector() && NumBaseMaskElts == 2) {
+    MVT ShuffleVT = (FloatDomain ? MVT::v4f64 : MVT::v4i64);
+
+    // If the upper half is zeroable, then an extract+insert is more optimal
+    // than using X86ISD::VPERM2X128. The insertion is free, even if it has to
+    // zero the upper half.
+    if (isUndefOrZero(BaseMask[1])) {
+      if (Depth == 0 && Root.getOpcode() == ISD::INSERT_SUBVECTOR)
+        return SDValue(); // Nothing to do!
+      assert(isInRange(BaseMask[0], 0, 2) && "Unexpected lane shuffle");
+      Res = DAG.getBitcast(ShuffleVT, V1);
+      Res = extract128BitVector(Res, BaseMask[0] * 2, DAG, DL);
+      Res = widenSubVector(Res, BaseMask[1] == SM_SentinelZero, Subtarget, DAG,
+                           DL, 256);
+      return DAG.getBitcast(RootVT, Res);
+    }
+
     if (Depth == 0 && Root.getOpcode() == X86ISD::VPERM2X128)
       return SDValue(); // Nothing to do!
-    MVT ShuffleVT = (FloatDomain ? MVT::v4f64 : MVT::v4i64);
 
     // If we have AVX2, prefer to use VPERMQ/VPERMPD for unary shuffles unless
     // we need to use the zeroing feature.
@@ -33934,7 +34150,7 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
   if (BaseMaskEltSizeInBits > 64) {
     assert((BaseMaskEltSizeInBits % 64) == 0 && "Illegal mask size");
     int MaskScale = BaseMaskEltSizeInBits / 64;
-    scaleShuffleMask<int>(MaskScale, BaseMask, Mask);
+    scaleShuffleMask(MaskScale, BaseMask, Mask);
   } else {
     Mask = SmallVector<int, 64>(BaseMask.begin(), BaseMask.end());
   }
@@ -34094,6 +34310,44 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
     }
   }
 
+  // Match shuffle against TRUNCATE patterns.
+  if (AllowIntDomain && MaskEltSizeInBits < 64 && Subtarget.hasAVX512()) {
+    // Match against a VTRUNC instruction, accounting for src/dst sizes.
+    if (matchShuffleAsVTRUNC(ShuffleSrcVT, ShuffleVT, IntMaskVT, Mask, Zeroable,
+                             Subtarget)) {
+      bool IsTRUNCATE = ShuffleVT.getVectorNumElements() ==
+                        ShuffleSrcVT.getVectorNumElements();
+      unsigned Opc =
+          IsTRUNCATE ? (unsigned)ISD::TRUNCATE : (unsigned)X86ISD::VTRUNC;
+      if (Depth == 0 && Root.getOpcode() == Opc)
+        return SDValue(); // Nothing to do!
+      V1 = DAG.getBitcast(ShuffleSrcVT, V1);
+      Res = DAG.getNode(Opc, DL, ShuffleVT, V1);
+      if (ShuffleVT.getSizeInBits() < RootSizeInBits)
+        Res = widenSubVector(Res, true, Subtarget, DAG, DL, RootSizeInBits);
+      return DAG.getBitcast(RootVT, Res);
+    }
+
+    // Do we need a more general binary truncation pattern?
+    if (RootSizeInBits < 512 &&
+        ((RootVT.is256BitVector() && Subtarget.useAVX512Regs()) ||
+         (RootVT.is128BitVector() && Subtarget.hasVLX())) &&
+        (MaskEltSizeInBits > 8 || Subtarget.hasBWI()) &&
+        isSequentialOrUndefInRange(Mask, 0, NumMaskElts, 0, 2)) {
+      if (Depth == 0 && Root.getOpcode() == ISD::TRUNCATE)
+        return SDValue(); // Nothing to do!
+      ShuffleSrcVT = MVT::getIntegerVT(MaskEltSizeInBits * 2);
+      ShuffleSrcVT = MVT::getVectorVT(ShuffleSrcVT, NumMaskElts / 2);
+      V1 = DAG.getBitcast(ShuffleSrcVT, V1);
+      V2 = DAG.getBitcast(ShuffleSrcVT, V2);
+      ShuffleSrcVT = MVT::getIntegerVT(MaskEltSizeInBits * 2);
+      ShuffleSrcVT = MVT::getVectorVT(ShuffleSrcVT, NumMaskElts);
+      Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, ShuffleSrcVT, V1, V2);
+      Res = DAG.getNode(ISD::TRUNCATE, DL, IntMaskVT, Res);
+      return DAG.getBitcast(RootVT, Res);
+    }
+  }
+
   // Don't try to re-form single instruction chains under any circumstances now
   // that we've done encoding canonicalization for them.
   if (Depth < 1)
@@ -34103,8 +34357,7 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
   int VariableShuffleDepth = Subtarget.hasFastVariableShuffle() ? 1 : 2;
   AllowVariableMask &= (Depth >= VariableShuffleDepth) || HasVariableMask;
 
-  bool MaskContainsZeros =
-      any_of(Mask, [](int M) { return M == SM_SentinelZero; });
+  bool MaskContainsZeros = isAnyZero(Mask);
 
   if (is128BitLaneCrossingShuffleMask(MaskVT, Mask)) {
     // If we have a single input lane-crossing shuffle then lower to VPERMV.
@@ -35124,11 +35377,9 @@ static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG,
       if (LN->isSimple()) {
         SDVTList Tys = DAG.getVTList(MVT::v2f64, MVT::Other);
         SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
-        SDValue VZLoad =
-            DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, DL, Tys, Ops, MVT::f64,
-                                    LN->getPointerInfo(),
-                                    LN->getAlignment(),
-                                    LN->getMemOperand()->getFlags());
+        SDValue VZLoad = DAG.getMemIntrinsicNode(
+            X86ISD::VZEXT_LOAD, DL, Tys, Ops, MVT::f64, LN->getPointerInfo(),
+            LN->getAlign(), LN->getMemOperand()->getFlags());
         SDValue Movddup = DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, VZLoad);
         DCI.CombineTo(N.getNode(), Movddup);
         DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
@@ -35226,11 +35477,10 @@ static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG,
         if (LN->isSimple()) {
           SDVTList Tys = DAG.getVTList(VT, MVT::Other);
           SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
-          SDValue BcastLd =
-              DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, DL, Tys, Ops,
-                                      MVT::i16, LN->getPointerInfo(),
-                                      LN->getAlignment(),
-                                      LN->getMemOperand()->getFlags());
+          SDValue BcastLd = DAG.getMemIntrinsicNode(
+              X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MVT::i16,
+              LN->getPointerInfo(), LN->getAlign(),
+              LN->getMemOperand()->getFlags());
           DCI.CombineTo(N.getNode(), BcastLd);
           DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
           DCI.recursivelyDeleteUnusedNodes(LN);
@@ -35271,12 +35521,11 @@ static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG,
           SDVTList Tys = DAG.getVTList(VT, MVT::Other);
           SDValue Ptr = DAG.getMemBasePlusOffset(LN->getBasePtr(), Offset, DL);
           SDValue Ops[] = { LN->getChain(), Ptr };
-          SDValue BcastLd =
-              DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, DL, Tys, Ops,
-                                      MVT::i16,
-                                      LN->getPointerInfo().getWithOffset(Offset),
-                                      MinAlign(LN->getAlignment(), Offset),
-                                      LN->getMemOperand()->getFlags());
+          SDValue BcastLd = DAG.getMemIntrinsicNode(
+              X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MVT::i16,
+              LN->getPointerInfo().getWithOffset(Offset),
+              commonAlignment(LN->getAlign(), Offset),
+              LN->getMemOperand()->getFlags());
           DCI.CombineTo(N.getNode(), BcastLd);
           DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
           DCI.recursivelyDeleteUnusedNodes(LN);
@@ -35309,11 +35558,10 @@ static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG,
       if (LN->isSimple()) {
         SDVTList Tys = DAG.getVTList(VT, MVT::Other);
         SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
-        SDValue BcastLd =
-            DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, DL, Tys, Ops,
-                                    MVT::f64, LN->getPointerInfo(),
-                                    LN->getAlignment(),
-                                    LN->getMemOperand()->getFlags());
+        SDValue BcastLd = DAG.getMemIntrinsicNode(
+            X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MVT::f64,
+            LN->getPointerInfo(), LN->getAlign(),
+            LN->getMemOperand()->getFlags());
         DCI.CombineTo(N.getNode(), BcastLd);
         DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
         DCI.recursivelyDeleteUnusedNodes(LN);
@@ -36074,12 +36322,10 @@ static SDValue combineShuffle(SDNode *N, SelectionDAG &DAG,
     if (LN->isSimple()) {
       SDVTList Tys = DAG.getVTList(VT, MVT::Other);
       SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
-      SDValue VZLoad =
-          DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops,
-                                  VT.getVectorElementType(),
-                                  LN->getPointerInfo(),
-                                  LN->getAlignment(),
-                                  LN->getMemOperand()->getFlags());
+      SDValue VZLoad = DAG.getMemIntrinsicNode(
+          X86ISD::VZEXT_LOAD, dl, Tys, Ops, VT.getVectorElementType(),
+          LN->getPointerInfo(), LN->getAlign(),
+          LN->getMemOperand()->getFlags());
       DCI.CombineTo(N, VZLoad);
       DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
       DCI.recursivelyDeleteUnusedNodes(LN);
@@ -38021,7 +38267,7 @@ static SDValue combineExtractWithShuffle(SDNode *N, SelectionDAG &DAG,
     if ((NumSrcElts % Mask.size()) == 0) {
       SmallVector<int, 16> ScaledMask;
       int Scale = NumSrcElts / Mask.size();
-      scaleShuffleMask<int>(Scale, Mask, ScaledMask);
+      scaleShuffleMask(Scale, Mask, ScaledMask);
       Mask = std::move(ScaledMask);
     } else if ((Mask.size() % NumSrcElts) == 0) {
       // Simplify Mask based on demanded element.
@@ -39589,6 +39835,81 @@ static SDValue combineCarryThroughADD(SDValue EFLAGS, SelectionDAG &DAG) {
   return SDValue();
 }
 
+/// If we are inverting an PTEST/TESTP operand, attempt to adjust the CC
+/// to avoid the inversion.
+static SDValue combinePTESTCC(SDValue EFLAGS, X86::CondCode &CC,
+                              SelectionDAG &DAG) {
+  // TODO: Handle X86ISD::KTEST/X86ISD::KORTEST.
+  if (EFLAGS.getOpcode() != X86ISD::PTEST &&
+      EFLAGS.getOpcode() != X86ISD::TESTP)
+    return SDValue();
+
+  // PTEST/TESTP sets EFLAGS as:
+  // TESTZ: ZF = (Op0 & Op1) == 0
+  // TESTC: CF = (~Op0 & Op1) == 0
+  // TESTNZC: ZF == 0 && CF == 0
+  EVT VT = EFLAGS.getValueType();
+  SDValue Op0 = EFLAGS.getOperand(0);
+  SDValue Op1 = EFLAGS.getOperand(1);
+  EVT OpVT = Op0.getValueType();
+
+  // TEST*(~X,Y) == TEST*(X,Y)
+  if (SDValue NotOp0 = IsNOT(Op0, DAG)) {
+    X86::CondCode InvCC;
+    switch (CC) {
+    case X86::COND_B:
+      // testc -> testz.
+      InvCC = X86::COND_E;
+      break;
+    case X86::COND_AE:
+      // !testc -> !testz.
+      InvCC = X86::COND_NE;
+      break;
+    case X86::COND_E:
+      // testz -> testc.
+      InvCC = X86::COND_B;
+      break;
+    case X86::COND_NE:
+      // !testz -> !testc.
+      InvCC = X86::COND_AE;
+      break;
+    case X86::COND_A:
+    case X86::COND_BE:
+      // testnzc -> testnzc (no change).
+      InvCC = CC;
+      break;
+    default:
+      InvCC = X86::COND_INVALID;
+      break;
+    }
+
+    if (InvCC != X86::COND_INVALID) {
+      CC = InvCC;
+      return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT,
+                         DAG.getBitcast(OpVT, NotOp0), Op1);
+    }
+  }
+
+  if (CC == X86::COND_E || CC == X86::COND_NE) {
+    // TESTZ(X,~Y) == TESTC(Y,X)
+    if (SDValue NotOp1 = IsNOT(Op1, DAG)) {
+      CC = (CC == X86::COND_E ? X86::COND_B : X86::COND_AE);
+      return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT,
+                         DAG.getBitcast(OpVT, NotOp1), Op0);
+    }
+
+    // TESTZ(-1,X) == TESTZ(X,X)
+    if (ISD::isBuildVectorAllOnes(Op0.getNode()))
+      return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, Op1, Op1);
+
+    // TESTZ(X,-1) == TESTZ(X,X)
+    if (ISD::isBuildVectorAllOnes(Op1.getNode()))
+      return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, Op0, Op0);
+  }
+
+  return SDValue();
+}
+
 /// Optimize an EFLAGS definition used according to the condition code \p CC
 /// into a simpler EFLAGS value, potentially returning a new \p CC and replacing
 /// uses of chain values.
@@ -39601,6 +39922,10 @@ static SDValue combineSetCCEFLAGS(SDValue EFLAGS, X86::CondCode &CC,
 
   if (SDValue R = checkBoolTestSetCCCombine(EFLAGS, CC))
     return R;
+
+  if (SDValue R = combinePTESTCC(EFLAGS, CC, DAG))
+    return R;
+
   return combineSetCCAtomicArith(EFLAGS, CC, DAG, Subtarget);
 }
 
@@ -44010,11 +44335,9 @@ static SDValue combineX86INT_TO_FP(SDNode *N, SelectionDAG &DAG,
       MVT LoadVT = MVT::getVectorVT(MemVT, 128 / NumBits);
       SDVTList Tys = DAG.getVTList(LoadVT, MVT::Other);
       SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
-      SDValue VZLoad =
-          DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops, MemVT,
-                                  LN->getPointerInfo(),
-                                  LN->getAlignment(),
-                                  LN->getMemOperand()->getFlags());
+      SDValue VZLoad = DAG.getMemIntrinsicNode(
+          X86ISD::VZEXT_LOAD, dl, Tys, Ops, MemVT, LN->getPointerInfo(),
+          LN->getAlign(), LN->getMemOperand()->getFlags());
       SDValue Convert = DAG.getNode(N->getOpcode(), dl, VT,
                                     DAG.getBitcast(InVT, VZLoad));
       DCI.CombineTo(N, Convert);
@@ -44046,11 +44369,9 @@ static SDValue combineCVTP2I_CVTTP2I(SDNode *N, SelectionDAG &DAG,
       MVT LoadVT = MVT::getVectorVT(MemVT, 128 / NumBits);
       SDVTList Tys = DAG.getVTList(LoadVT, MVT::Other);
       SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
-      SDValue VZLoad =
-          DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops, MemVT,
-                                  LN->getPointerInfo(),
-                                  LN->getAlignment(),
-                                  LN->getMemOperand()->getFlags());
+      SDValue VZLoad = DAG.getMemIntrinsicNode(
+          X86ISD::VZEXT_LOAD, dl, Tys, Ops, MemVT, LN->getPointerInfo(),
+          LN->getAlign(), LN->getMemOperand()->getFlags());
       SDValue Convert = DAG.getNode(N->getOpcode(), dl, VT,
                                     DAG.getBitcast(InVT, VZLoad));
       DCI.CombineTo(N, Convert);
@@ -44135,11 +44456,9 @@ static SDValue combineCVTPH2PS(SDNode *N, SelectionDAG &DAG,
         SDLoc dl(N);
         SDVTList Tys = DAG.getVTList(MVT::v2i64, MVT::Other);
         SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
-        SDValue VZLoad =
-            DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops, MVT::i64,
-                                    LN->getPointerInfo(),
-                                    LN->getAlignment(),
-                                    LN->getMemOperand()->getFlags());
+        SDValue VZLoad = DAG.getMemIntrinsicNode(
+            X86ISD::VZEXT_LOAD, dl, Tys, Ops, MVT::i64, LN->getPointerInfo(),
+            LN->getAlign(), LN->getMemOperand()->getFlags());
         SDValue Convert = DAG.getNode(N->getOpcode(), dl, MVT::v4f32,
                                       DAG.getBitcast(MVT::v8i16, VZLoad));
         DCI.CombineTo(N, Convert);
@@ -44497,7 +44816,7 @@ static SDValue combineExtSetcc(SDNode *N, SelectionDAG &DAG,
 
   // We can only do this if the vector size in 256 bits or less.
   unsigned Size = VT.getSizeInBits();
-  if (Size > 256)
+  if (Size > 256 && Subtarget.useAVX512Regs())
     return SDValue();
 
   // Don't fold if the condition code can't be handled by PCMPEQ/PCMPGT since
@@ -45383,7 +45702,7 @@ static SDValue combineSIntToFP(SDNode *N, SelectionDAG &DAG,
       std::pair<SDValue, SDValue> Tmp =
           Subtarget.getTargetLowering()->BuildFILD(
               VT, InVT, SDLoc(N), Ld->getChain(), Ld->getBasePtr(),
-              Ld->getPointerInfo(), Ld->getAlignment(), DAG);
+              Ld->getPointerInfo(), Ld->getAlign(), DAG);
       DAG.ReplaceAllUsesOfValueWith(Op0.getValue(1), Tmp.second);
       return Tmp.first;
     }
@@ -45801,131 +46120,6 @@ static SDValue combineAddOrSubToADCOrSBB(SDNode *N, SelectionDAG &DAG) {
                      DAG.getConstant(0, DL, VT), Cmp1.getValue(1));
 }
 
-static SDValue combineLoopMAddPattern(SDNode *N, SelectionDAG &DAG,
-                                      const X86Subtarget &Subtarget) {
-  if (!Subtarget.hasSSE2())
-    return SDValue();
-
-  EVT VT = N->getValueType(0);
-
-  // If the vector size is less than 128, or greater than the supported RegSize,
-  // do not use PMADD.
-  if (!VT.isVector() || VT.getVectorNumElements() < 8)
-    return SDValue();
-
-  SDValue Op0 = N->getOperand(0);
-  SDValue Op1 = N->getOperand(1);
-
-  auto UsePMADDWD = [&](SDValue Op) {
-    ShrinkMode Mode;
-    return Op.getOpcode() == ISD::MUL &&
-           canReduceVMulWidth(Op.getNode(), DAG, Mode) &&
-           Mode != ShrinkMode::MULU16 &&
-           (!Subtarget.hasSSE41() ||
-            (Op->isOnlyUserOf(Op.getOperand(0).getNode()) &&
-             Op->isOnlyUserOf(Op.getOperand(1).getNode())));
-  };
-
-  SDValue MulOp, OtherOp;
-  if (UsePMADDWD(Op0)) {
-    MulOp = Op0;
-    OtherOp = Op1;
-  } else if (UsePMADDWD(Op1)) {
-    MulOp = Op1;
-    OtherOp = Op0;
-  } else
-   return SDValue();
-
-  SDLoc DL(N);
-  EVT ReducedVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16,
-                                   VT.getVectorNumElements());
-  EVT MAddVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32,
-                                VT.getVectorNumElements() / 2);
-
-  // Shrink the operands of mul.
-  SDValue N0 = DAG.getNode(ISD::TRUNCATE, DL, ReducedVT, MulOp->getOperand(0));
-  SDValue N1 = DAG.getNode(ISD::TRUNCATE, DL, ReducedVT, MulOp->getOperand(1));
-
-  // Madd vector size is half of the original vector size
-  auto PMADDWDBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
-                           ArrayRef<SDValue> Ops) {
-    MVT OpVT = MVT::getVectorVT(MVT::i32, Ops[0].getValueSizeInBits() / 32);
-    return DAG.getNode(X86ISD::VPMADDWD, DL, OpVT, Ops);
-  };
-  SDValue Madd = SplitOpsAndApply(DAG, Subtarget, DL, MAddVT, { N0, N1 },
-                                  PMADDWDBuilder);
-  // Fill the rest of the output with 0
-  SDValue Zero = DAG.getConstant(0, DL, Madd.getSimpleValueType());
-  SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Madd, Zero);
-
-  // Preserve the reduction flag on the ADD. We may need to revisit for the
-  // other operand.
-  SDNodeFlags Flags;
-  Flags.setVectorReduction(true);
-  return DAG.getNode(ISD::ADD, DL, VT, Concat, OtherOp, Flags);
-}
-
-static SDValue combineLoopSADPattern(SDNode *N, SelectionDAG &DAG,
-                                     const X86Subtarget &Subtarget) {
-  if (!Subtarget.hasSSE2())
-    return SDValue();
-
-  SDLoc DL(N);
-  EVT VT = N->getValueType(0);
-
-  // TODO: There's nothing special about i32, any integer type above i16 should
-  // work just as well.
-  if (!VT.isVector() || !isPowerOf2_32(VT.getVectorNumElements()) ||
-      VT.getVectorElementType() != MVT::i32)
-    return SDValue();
-
-  // We know N is a reduction add. To match SAD, we need one of the operands to
-  // be an ABS.
-  SDValue AbsOp = N->getOperand(0);
-  SDValue OtherOp = N->getOperand(1);
-  if (AbsOp.getOpcode() != ISD::ABS)
-    std::swap(AbsOp, OtherOp);
-  if (AbsOp.getOpcode() != ISD::ABS)
-    return SDValue();
-
-  // Check whether we have an abs-diff pattern feeding into the select.
-  SDValue SadOp0, SadOp1;
-  if(!detectZextAbsDiff(AbsOp, SadOp0, SadOp1))
-    return SDValue();
-
-  // SAD pattern detected. Now build a SAD instruction and an addition for
-  // reduction. Note that the number of elements of the result of SAD is less
-  // than the number of elements of its input. Therefore, we could only update
-  // part of elements in the reduction vector.
-  SDValue Sad = createPSADBW(DAG, SadOp0, SadOp1, DL, Subtarget);
-
-  // The output of PSADBW is a vector of i64.
-  // We need to turn the vector of i64 into a vector of i32.
-  // If the reduction vector is at least as wide as the psadbw result, just
-  // bitcast. If it's narrower which can only occur for v2i32, bits 127:16 of
-  // the PSADBW will be zero.
-  MVT ResVT = MVT::getVectorVT(MVT::i32, Sad.getValueSizeInBits() / 32);
-  Sad = DAG.getNode(ISD::BITCAST, DL, ResVT, Sad);
-
-  if (VT.getSizeInBits() > ResVT.getSizeInBits()) {
-    // Fill the upper elements with zero to match the add width.
-    assert(VT.getSizeInBits() % ResVT.getSizeInBits() == 0 && "Unexpected VTs");
-    unsigned NumConcats = VT.getSizeInBits() / ResVT.getSizeInBits();
-    SmallVector<SDValue, 4> Ops(NumConcats, DAG.getConstant(0, DL, ResVT));
-    Ops[0] = Sad;
-    Sad = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Ops);
-  } else if (VT.getSizeInBits() < ResVT.getSizeInBits()) {
-    Sad = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Sad,
-                      DAG.getIntPtrConstant(0, DL));
-  }
-
-  // Preserve the reduction flag on the ADD. We may need to revisit for the
-  // other operand.
-  SDNodeFlags Flags;
-  Flags.setVectorReduction(true);
-  return DAG.getNode(ISD::ADD, DL, VT, Sad, OtherOp, Flags);
-}
-
 static SDValue matchPMADDWD(SelectionDAG &DAG, SDValue Op0, SDValue Op1,
                             const SDLoc &DL, EVT VT,
                             const X86Subtarget &Subtarget) {
@@ -46015,30 +46209,25 @@ static SDValue matchPMADDWD(SelectionDAG &DAG, SDValue Op0, SDValue Op1,
       Mode == ShrinkMode::MULU16)
     return SDValue();
 
+  EVT TruncVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16,
+                                 VT.getVectorNumElements() * 2);
+  SDValue N0 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Mul.getOperand(0));
+  SDValue N1 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Mul.getOperand(1));
+
   auto PMADDBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
                          ArrayRef<SDValue> Ops) {
-    // Shrink by adding truncate nodes and let DAGCombine fold with the
-    // sources.
     EVT InVT = Ops[0].getValueType();
-    assert(InVT.getScalarType() == MVT::i32 &&
-           "Unexpected scalar element type");
     assert(InVT == Ops[1].getValueType() && "Operands' types mismatch");
     EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32,
                                  InVT.getVectorNumElements() / 2);
-    EVT TruncVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16,
-                                   InVT.getVectorNumElements());
-    return DAG.getNode(X86ISD::VPMADDWD, DL, ResVT,
-                       DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Ops[0]),
-                       DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Ops[1]));
+    return DAG.getNode(X86ISD::VPMADDWD, DL, ResVT, Ops[0], Ops[1]);
   };
-  return SplitOpsAndApply(DAG, Subtarget, DL, VT,
-                          { Mul.getOperand(0), Mul.getOperand(1) },
-                          PMADDBuilder);
+  return SplitOpsAndApply(DAG, Subtarget, DL, VT, { N0, N1 }, PMADDBuilder);
 }
 
 // Attempt to turn this pattern into PMADDWD.
-// (mul (add (sext (build_vector)), (sext (build_vector))),
-//      (add (sext (build_vector)), (sext (build_vector)))
+// (add (mul (sext (build_vector)), (sext (build_vector))),
+//      (mul (sext (build_vector)), (sext (build_vector)))
 static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDValue N0, SDValue N1,
                               const SDLoc &DL, EVT VT,
                               const X86Subtarget &Subtarget) {
@@ -46160,13 +46349,6 @@ static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDValue N0, SDValue N1,
 static SDValue combineAdd(SDNode *N, SelectionDAG &DAG,
                           TargetLowering::DAGCombinerInfo &DCI,
                           const X86Subtarget &Subtarget) {
-  const SDNodeFlags Flags = N->getFlags();
-  if (Flags.hasVectorReduction()) {
-    if (SDValue Sad = combineLoopSADPattern(N, DAG, Subtarget))
-      return Sad;
-    if (SDValue MAdd = combineLoopMAddPattern(N, DAG, Subtarget))
-      return MAdd;
-  }
   EVT VT = N->getValueType(0);
   SDValue Op0 = N->getOperand(0);
   SDValue Op1 = N->getOperand(1);
@@ -46523,6 +46705,24 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
                            DAG.getNode(ISD::CONCAT_VECTORS, DL, SrcVT, RHS));
       }
       break;
+    case X86ISD::PALIGNR:
+      if (!IsSplat &&
+          ((VT.is256BitVector() && Subtarget.hasInt256()) ||
+           (VT.is512BitVector() && Subtarget.useBWIRegs())) &&
+          llvm::all_of(Ops, [Op0](SDValue Op) {
+            return Op0.getOperand(2) == Op.getOperand(2);
+          })) {
+        SmallVector<SDValue, 2> LHS, RHS;
+        for (unsigned i = 0; i != NumOps; ++i) {
+          LHS.push_back(Ops[i].getOperand(0));
+          RHS.push_back(Ops[i].getOperand(1));
+        }
+        return DAG.getNode(Op0.getOpcode(), DL, VT,
+                           DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LHS),
+                           DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, RHS),
+                           Op0.getOperand(2));
+      }
+      break;
     }
   }
 
@@ -46612,7 +46812,8 @@ static SDValue combineInsertSubvector(SDNode *N, SelectionDAG &DAG,
   // if the insert or extract can be represented with a subregister operation.
   if (SubVec.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
       SubVec.getOperand(0).getSimpleValueType() == OpVT &&
-      (IdxVal != 0 || !Vec.isUndef())) {
+      (IdxVal != 0 ||
+       !(Vec.isUndef() || ISD::isBuildVectorAllZeros(Vec.getNode())))) {
     int ExtIdxVal = SubVec.getConstantOperandVal(1);
     if (ExtIdxVal != 0) {
       int VecNumElts = OpVT.getVectorNumElements();
@@ -47287,6 +47488,7 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
   case X86ISD::INSERTPS:
   case X86ISD::EXTRQI:
   case X86ISD::INSERTQI:
+  case X86ISD::VALIGN:
   case X86ISD::PALIGNR:
   case X86ISD::VSHLDQ:
   case X86ISD::VSRLDQ:
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index 8ebe36977ba2c..1e008b9598cce 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -1215,7 +1215,7 @@ namespace llvm {
     std::pair<SDValue, SDValue> BuildFILD(EVT DstVT, EVT SrcVT, const SDLoc &DL,
                                           SDValue Chain, SDValue Pointer,
                                           MachinePointerInfo PtrInfo,
-                                          unsigned Align,
+                                          Align Alignment,
                                           SelectionDAG &DAG) const;
 
     bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override;
@@ -1359,7 +1359,6 @@ namespace llvm {
     SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerLRINT_LLRINT(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
-    SDValue LowerSTRICT_FSETCC(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerBRCOND(SDValue Op, SelectionDAG &DAG) const;
@@ -1475,8 +1474,8 @@ namespace llvm {
     MachineBasicBlock *EmitLoweredTLSCall(MachineInstr &MI,
                                           MachineBasicBlock *BB) const;
 
-    MachineBasicBlock *EmitLoweredRetpoline(MachineInstr &MI,
-                                            MachineBasicBlock *BB) const;
+    MachineBasicBlock *EmitLoweredIndirectThunk(MachineInstr &MI,
+                                                MachineBasicBlock *BB) const;
 
     MachineBasicBlock *emitEHSjLjSetJmp(MachineInstr &MI,
                                         MachineBasicBlock *MBB) const;
@@ -1565,35 +1564,14 @@ namespace llvm {
   };
 
   /// Generate unpacklo/unpackhi shuffle mask.
-  template <typename T = int>
-  void createUnpackShuffleMask(MVT VT, SmallVectorImpl<T> &Mask, bool Lo,
-                               bool Unary) {
-    assert(Mask.empty() && "Expected an empty shuffle mask vector");
-    int NumElts = VT.getVectorNumElements();
-    int NumEltsInLane = 128 / VT.getScalarSizeInBits();
-    for (int i = 0; i < NumElts; ++i) {
-      unsigned LaneStart = (i / NumEltsInLane) * NumEltsInLane;
-      int Pos = (i % NumEltsInLane) / 2 + LaneStart;
-      Pos += (Unary ? 0 : NumElts * (i % 2));
-      Pos += (Lo ? 0 : NumEltsInLane / 2);
-      Mask.push_back(Pos);
-    }
-  }
+  void createUnpackShuffleMask(MVT VT, SmallVectorImpl<int> &Mask, bool Lo,
+                               bool Unary);
 
   /// Similar to unpacklo/unpackhi, but without the 128-bit lane limitation
   /// imposed by AVX and specific to the unary pattern. Example:
   /// v8iX Lo --> <0, 0, 1, 1, 2, 2, 3, 3>
   /// v8iX Hi --> <4, 4, 5, 5, 6, 6, 7, 7>
-  template <typename T = int>
-  void createSplat2ShuffleMask(MVT VT, SmallVectorImpl<T> &Mask, bool Lo) {
-    assert(Mask.empty() && "Expected an empty shuffle mask vector");
-    int NumElts = VT.getVectorNumElements();
-    for (int i = 0; i < NumElts; ++i) {
-      int Pos = i / 2;
-      Pos += (Lo ? 0 : NumElts / 2);
-      Mask.push_back(Pos);
-    }
-  }
+  void createSplat2ShuffleMask(MVT VT, SmallVectorImpl<int> &Mask, bool Lo);
 
 } // end namespace llvm
 
diff --git a/llvm/lib/Target/X86/X86IndirectBranchTracking.cpp b/llvm/lib/Target/X86/X86IndirectBranchTracking.cpp
index 743df41a00ffb..0a79b793a980f 100644
--- a/llvm/lib/Target/X86/X86IndirectBranchTracking.cpp
+++ b/llvm/lib/Target/X86/X86IndirectBranchTracking.cpp
@@ -18,6 +18,7 @@
 #include "X86.h"
 #include "X86InstrInfo.h"
 #include "X86Subtarget.h"
+#include "X86TargetMachine.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
@@ -102,7 +103,16 @@ bool X86IndirectBranchTrackingPass::runOnMachineFunction(MachineFunction &MF) {
   // Check that the cf-protection-branch is enabled.
   Metadata *isCFProtectionSupported =
       MF.getMMI().getModule()->getModuleFlag("cf-protection-branch");
-  if (!isCFProtectionSupported && !IndirectBranchTracking)
+  // NB: We need to enable IBT in jitted code if JIT compiler is CET
+  // enabled.
+  const X86TargetMachine *TM =
+      static_cast<const X86TargetMachine *>(&MF.getTarget());
+#ifdef __CET__
+  bool isJITwithCET = TM->isJIT();
+#else
+  bool isJITwithCET = false;
+#endif
+  if (!isCFProtectionSupported && !IndirectBranchTracking && !isJITwithCET)
     return false;
 
   // True if the current MF was changed and false otherwise.
@@ -111,10 +121,11 @@ bool X86IndirectBranchTrackingPass::runOnMachineFunction(MachineFunction &MF) {
   TII = SubTarget.getInstrInfo();
   EndbrOpcode = SubTarget.is64Bit() ? X86::ENDBR64 : X86::ENDBR32;
 
-  // Non-internal function or function whose address was taken, can be
-  // accessed through indirect calls. Mark the first BB with ENDBR instruction
-  // unless nocf_check attribute is used.
-  if ((MF.getFunction().hasAddressTaken() ||
+  // Large code model, non-internal function or function whose address
+  // was taken, can be accessed through indirect calls. Mark the first
+  // BB with ENDBR instruction unless nocf_check attribute is used.
+  if ((TM->getCodeModel() == CodeModel::Large ||
+       MF.getFunction().hasAddressTaken() ||
        !MF.getFunction().hasLocalLinkage()) &&
       !MF.getFunction().doesNoCfCheck()) {
     auto MBB = MF.begin();
@@ -136,8 +147,8 @@ bool X86IndirectBranchTrackingPass::runOnMachineFunction(MachineFunction &MF) {
         Changed |= addENDBR(MBB, std::next(I));
 
       if (EHPadIBTNeeded && I->isEHLabel()) {
-          Changed |= addENDBR(MBB, std::next(I));
-          EHPadIBTNeeded = false;
+        Changed |= addENDBR(MBB, std::next(I));
+        EHPadIBTNeeded = false;
       }
     }
   }
diff --git a/llvm/lib/Target/X86/X86IndirectThunks.cpp b/llvm/lib/Target/X86/X86IndirectThunks.cpp
new file mode 100644
index 0000000000000..36b9c3ccc959a
--- /dev/null
+++ b/llvm/lib/Target/X86/X86IndirectThunks.cpp
@@ -0,0 +1,364 @@
+//==- X86IndirectThunks.cpp - Construct indirect call/jump thunks for x86  --=//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+///
+/// Pass that injects an MI thunk that is used to lower indirect calls in a way
+/// that prevents speculation on some x86 processors and can be used to mitigate
+/// security vulnerabilities due to targeted speculative execution and side
+/// channels such as CVE-2017-5715.
+///
+/// Currently supported thunks include:
+/// - Retpoline -- A RET-implemented trampoline that lowers indirect calls
+/// - LVI Thunk -- A CALL/JMP-implemented thunk that forces load serialization
+///   before making an indirect call/jump
+///
+/// Note that the reason that this is implemented as a MachineFunctionPass and
+/// not a ModulePass is that ModulePasses at this point in the LLVM X86 pipeline
+/// serialize all transformations, which can consume lots of memory.
+///
+/// TODO(chandlerc): All of this code could use better comments and
+/// documentation.
+///
+//===----------------------------------------------------------------------===//
+
+#include "X86.h"
+#include "X86InstrBuilder.h"
+#include "X86Subtarget.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "x86-retpoline-thunks"
+
+static const char RetpolineNamePrefix[] = "__llvm_retpoline_";
+static const char R11RetpolineName[] = "__llvm_retpoline_r11";
+static const char EAXRetpolineName[] = "__llvm_retpoline_eax";
+static const char ECXRetpolineName[] = "__llvm_retpoline_ecx";
+static const char EDXRetpolineName[] = "__llvm_retpoline_edx";
+static const char EDIRetpolineName[] = "__llvm_retpoline_edi";
+
+static const char LVIThunkNamePrefix[] = "__llvm_lvi_thunk_";
+static const char R11LVIThunkName[] = "__llvm_lvi_thunk_r11";
+
+namespace {
+template <typename Derived> class ThunkInserter {
+  Derived &getDerived() { return *static_cast<Derived *>(this); }
+
+protected:
+  bool InsertedThunks;
+  void doInitialization(Module &M) {}
+  void createThunkFunction(MachineModuleInfo &MMI, StringRef Name);
+
+public:
+  void init(Module &M) {
+    InsertedThunks = false;
+    getDerived().doInitialization(M);
+  }
+  // return `true` if `MMI` or `MF` was modified
+  bool run(MachineModuleInfo &MMI, MachineFunction &MF);
+};
+
+struct RetpolineThunkInserter : ThunkInserter<RetpolineThunkInserter> {
+  const char *getThunkPrefix() { return RetpolineNamePrefix; }
+  bool mayUseThunk(const MachineFunction &MF) {
+    const auto &STI = MF.getSubtarget<X86Subtarget>();
+    return (STI.useRetpolineIndirectCalls() ||
+            STI.useRetpolineIndirectBranches()) &&
+           !STI.useRetpolineExternalThunk();
+  }
+  void insertThunks(MachineModuleInfo &MMI);
+  void populateThunk(MachineFunction &MF);
+};
+
+struct LVIThunkInserter : ThunkInserter<LVIThunkInserter> {
+  const char *getThunkPrefix() { return LVIThunkNamePrefix; }
+  bool mayUseThunk(const MachineFunction &MF) {
+    return MF.getSubtarget<X86Subtarget>().useLVIControlFlowIntegrity();
+  }
+  void insertThunks(MachineModuleInfo &MMI) {
+    createThunkFunction(MMI, R11LVIThunkName);
+  }
+  void populateThunk(MachineFunction &MF) {
+    // Grab the entry MBB and erase any other blocks. O0 codegen appears to
+    // generate two bbs for the entry block.
+    MachineBasicBlock *Entry = &MF.front();
+    Entry->clear();
+    while (MF.size() > 1)
+      MF.erase(std::next(MF.begin()));
+
+    // This code mitigates LVI by replacing each indirect call/jump with a
+    // direct call/jump to a thunk that looks like:
+    // ```
+    // lfence
+    // jmpq *%r11
+    // ```
+    // This ensures that if the value in register %r11 was loaded from memory,
+    // then the value in %r11 is (architecturally) correct prior to the jump.
+    const TargetInstrInfo *TII = MF.getSubtarget<X86Subtarget>().getInstrInfo();
+    BuildMI(&MF.front(), DebugLoc(), TII->get(X86::LFENCE));
+    BuildMI(&MF.front(), DebugLoc(), TII->get(X86::JMP64r)).addReg(X86::R11);
+    MF.front().addLiveIn(X86::R11);
+    return;
+  }
+};
+
+class X86IndirectThunks : public MachineFunctionPass {
+public:
+  static char ID;
+
+  X86IndirectThunks() : MachineFunctionPass(ID) {}
+
+  StringRef getPassName() const override { return "X86 Indirect Thunks"; }
+
+  bool doInitialization(Module &M) override;
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    MachineFunctionPass::getAnalysisUsage(AU);
+    AU.addRequired<MachineModuleInfoWrapperPass>();
+    AU.addPreserved<MachineModuleInfoWrapperPass>();
+  }
+
+private:
+  std::tuple<RetpolineThunkInserter, LVIThunkInserter> TIs;
+
+  // FIXME: When LLVM moves to C++17, these can become folds
+  template <typename... ThunkInserterT>
+  static void initTIs(Module &M,
+                      std::tuple<ThunkInserterT...> &ThunkInserters) {
+    (void)std::initializer_list<int>{
+        (std::get<ThunkInserterT>(ThunkInserters).init(M), 0)...};
+  }
+  template <typename... ThunkInserterT>
+  static bool runTIs(MachineModuleInfo &MMI, MachineFunction &MF,
+                     std::tuple<ThunkInserterT...> &ThunkInserters) {
+    bool Modified = false;
+    (void)std::initializer_list<int>{
+        Modified |= std::get<ThunkInserterT>(ThunkInserters).run(MMI, MF)...};
+    return Modified;
+  }
+};
+
+} // end anonymous namespace
+
+void RetpolineThunkInserter::insertThunks(MachineModuleInfo &MMI) {
+  if (MMI.getTarget().getTargetTriple().getArch() == Triple::x86_64)
+    createThunkFunction(MMI, R11RetpolineName);
+  else
+    for (StringRef Name : {EAXRetpolineName, ECXRetpolineName, EDXRetpolineName,
+                           EDIRetpolineName})
+      createThunkFunction(MMI, Name);
+}
+
+void RetpolineThunkInserter::populateThunk(MachineFunction &MF) {
+  bool Is64Bit = MF.getTarget().getTargetTriple().getArch() == Triple::x86_64;
+  Register ThunkReg;
+  if (Is64Bit) {
+    assert(MF.getName() == "__llvm_retpoline_r11" &&
+           "Should only have an r11 thunk on 64-bit targets");
+
+    // __llvm_retpoline_r11:
+    //   callq .Lr11_call_target
+    // .Lr11_capture_spec:
+    //   pause
+    //   lfence
+    //   jmp .Lr11_capture_spec
+    // .align 16
+    // .Lr11_call_target:
+    //   movq %r11, (%rsp)
+    //   retq
+    ThunkReg = X86::R11;
+  } else {
+    // For 32-bit targets we need to emit a collection of thunks for various
+    // possible scratch registers as well as a fallback that uses EDI, which is
+    // normally callee saved.
+    //   __llvm_retpoline_eax:
+    //         calll .Leax_call_target
+    //   .Leax_capture_spec:
+    //         pause
+    //         jmp .Leax_capture_spec
+    //   .align 16
+    //   .Leax_call_target:
+    //         movl %eax, (%esp)  # Clobber return addr
+    //         retl
+    //
+    //   __llvm_retpoline_ecx:
+    //   ... # Same setup
+    //         movl %ecx, (%esp)
+    //         retl
+    //
+    //   __llvm_retpoline_edx:
+    //   ... # Same setup
+    //         movl %edx, (%esp)
+    //         retl
+    //
+    //   __llvm_retpoline_edi:
+    //   ... # Same setup
+    //         movl %edi, (%esp)
+    //         retl
+    if (MF.getName() == EAXRetpolineName)
+      ThunkReg = X86::EAX;
+    else if (MF.getName() == ECXRetpolineName)
+      ThunkReg = X86::ECX;
+    else if (MF.getName() == EDXRetpolineName)
+      ThunkReg = X86::EDX;
+    else if (MF.getName() == EDIRetpolineName)
+      ThunkReg = X86::EDI;
+    else
+      llvm_unreachable("Invalid thunk name on x86-32!");
+  }
+
+  const TargetInstrInfo *TII = MF.getSubtarget<X86Subtarget>().getInstrInfo();
+  // Grab the entry MBB and erase any other blocks. O0 codegen appears to
+  // generate two bbs for the entry block.
+  MachineBasicBlock *Entry = &MF.front();
+  Entry->clear();
+  while (MF.size() > 1)
+    MF.erase(std::next(MF.begin()));
+
+  MachineBasicBlock *CaptureSpec =
+      MF.CreateMachineBasicBlock(Entry->getBasicBlock());
+  MachineBasicBlock *CallTarget =
+      MF.CreateMachineBasicBlock(Entry->getBasicBlock());
+  MCSymbol *TargetSym = MF.getContext().createTempSymbol();
+  MF.push_back(CaptureSpec);
+  MF.push_back(CallTarget);
+
+  const unsigned CallOpc = Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32;
+  const unsigned RetOpc = Is64Bit ? X86::RETQ : X86::RETL;
+
+  Entry->addLiveIn(ThunkReg);
+  BuildMI(Entry, DebugLoc(), TII->get(CallOpc)).addSym(TargetSym);
+
+  // The MIR verifier thinks that the CALL in the entry block will fall through
+  // to CaptureSpec, so mark it as the successor. Technically, CaptureTarget is
+  // the successor, but the MIR verifier doesn't know how to cope with that.
+  Entry->addSuccessor(CaptureSpec);
+
+  // In the capture loop for speculation, we want to stop the processor from
+  // speculating as fast as possible. On Intel processors, the PAUSE instruction
+  // will block speculation without consuming any execution resources. On AMD
+  // processors, the PAUSE instruction is (essentially) a nop, so we also use an
+  // LFENCE instruction which they have advised will stop speculation as well
+  // with minimal resource utilization. We still end the capture with a jump to
+  // form an infinite loop to fully guarantee that no matter what implementation
+  // of the x86 ISA, speculating this code path never escapes.
+  BuildMI(CaptureSpec, DebugLoc(), TII->get(X86::PAUSE));
+  BuildMI(CaptureSpec, DebugLoc(), TII->get(X86::LFENCE));
+  BuildMI(CaptureSpec, DebugLoc(), TII->get(X86::JMP_1)).addMBB(CaptureSpec);
+  CaptureSpec->setHasAddressTaken();
+  CaptureSpec->addSuccessor(CaptureSpec);
+
+  CallTarget->addLiveIn(ThunkReg);
+  CallTarget->setHasAddressTaken();
+  CallTarget->setAlignment(Align(16));
+
+  // Insert return address clobber
+  const unsigned MovOpc = Is64Bit ? X86::MOV64mr : X86::MOV32mr;
+  const Register SPReg = Is64Bit ? X86::RSP : X86::ESP;
+  addRegOffset(BuildMI(CallTarget, DebugLoc(), TII->get(MovOpc)), SPReg, false,
+               0)
+      .addReg(ThunkReg);
+
+  CallTarget->back().setPreInstrSymbol(MF, TargetSym);
+  BuildMI(CallTarget, DebugLoc(), TII->get(RetOpc));
+}
+
+template <typename Derived>
+void ThunkInserter<Derived>::createThunkFunction(MachineModuleInfo &MMI,
+                                                 StringRef Name) {
+  assert(Name.startswith(getDerived().getThunkPrefix()) &&
+         "Created a thunk with an unexpected prefix!");
+
+  Module &M = const_cast<Module &>(*MMI.getModule());
+  LLVMContext &Ctx = M.getContext();
+  auto Type = FunctionType::get(Type::getVoidTy(Ctx), false);
+  Function *F =
+      Function::Create(Type, GlobalValue::LinkOnceODRLinkage, Name, &M);
+  F->setVisibility(GlobalValue::HiddenVisibility);
+  F->setComdat(M.getOrInsertComdat(Name));
+
+  // Add Attributes so that we don't create a frame, unwind information, or
+  // inline.
+  AttrBuilder B;
+  B.addAttribute(llvm::Attribute::NoUnwind);
+  B.addAttribute(llvm::Attribute::Naked);
+  F->addAttributes(llvm::AttributeList::FunctionIndex, B);
+
+  // Populate our function a bit so that we can verify.
+  BasicBlock *Entry = BasicBlock::Create(Ctx, "entry", F);
+  IRBuilder<> Builder(Entry);
+
+  Builder.CreateRetVoid();
+
+  // MachineFunctions/MachineBasicBlocks aren't created automatically for the
+  // IR-level constructs we already made. Create them and insert them into the
+  // module.
+  MachineFunction &MF = MMI.getOrCreateMachineFunction(*F);
+  MachineBasicBlock *EntryMBB = MF.CreateMachineBasicBlock(Entry);
+
+  // Insert EntryMBB into MF. It's not in the module until we do this.
+  MF.insert(MF.end(), EntryMBB);
+  // Set MF properties. We never use vregs...
+  MF.getProperties().set(MachineFunctionProperties::Property::NoVRegs);
+}
+
+template <typename Derived>
+bool ThunkInserter<Derived>::run(MachineModuleInfo &MMI, MachineFunction &MF) {
+  // If MF is not a thunk, check to see if we need to insert a thunk.
+  if (!MF.getName().startswith(getDerived().getThunkPrefix())) {
+    // If we've already inserted a thunk, nothing else to do.
+    if (InsertedThunks)
+      return false;
+
+    // Only add a thunk if one of the functions has the corresponding feature
+    // enabled in its subtarget, and doesn't enable external thunks.
+    // FIXME: Conditionalize on indirect calls so we don't emit a thunk when
+    // nothing will end up calling it.
+    // FIXME: It's a little silly to look at every function just to enumerate
+    // the subtargets, but eventually we'll want to look at them for indirect
+    // calls, so maybe this is OK.
+    if (!getDerived().mayUseThunk(MF))
+      return false;
+
+    getDerived().insertThunks(MMI);
+    InsertedThunks = true;
+    return true;
+  }
+
+  // If this *is* a thunk function, we need to populate it with the correct MI.
+  getDerived().populateThunk(MF);
+  return true;
+}
+
+FunctionPass *llvm::createX86IndirectThunksPass() {
+  return new X86IndirectThunks();
+}
+
+char X86IndirectThunks::ID = 0;
+
+bool X86IndirectThunks::doInitialization(Module &M) {
+  initTIs(M, TIs);
+  return false;
+}
+
+bool X86IndirectThunks::runOnMachineFunction(MachineFunction &MF) {
+  LLVM_DEBUG(dbgs() << getPassName() << '\n');
+  auto &MMI = getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
+  return runTIs(MMI, MF, TIs);
+}
diff --git a/llvm/lib/Target/X86/X86InstrAVX512.td b/llvm/lib/Target/X86/X86InstrAVX512.td
index b23050ef61aca..955a247985ce0 100644
--- a/llvm/lib/Target/X86/X86InstrAVX512.td
+++ b/llvm/lib/Target/X86/X86InstrAVX512.td
@@ -169,6 +169,18 @@ def v16i1_info : X86KVectorVTInfo<VK16, VK16WM, v16i1>;
 def v32i1_info : X86KVectorVTInfo<VK32, VK32WM, v32i1>;
 def v64i1_info : X86KVectorVTInfo<VK64, VK64WM, v64i1>;
 
+// Used for matching masked operations. Ensures the operation part only has a
+// single use.
+def vselect_mask : PatFrag<(ops node:$mask, node:$src1, node:$src2),
+                           (vselect node:$mask, node:$src1, node:$src2), [{
+  return isProfitableToFormMaskedOp(N);
+}]>;
+
+def X86selects_mask : PatFrag<(ops node:$mask, node:$src1, node:$src2),
+                              (X86selects node:$mask, node:$src1, node:$src2), [{
+  return isProfitableToFormMaskedOp(N);
+}]>;
+
 // This multiclass generates the masking variants from the non-masking
 // variant.  It only provides the assembly pieces for the masking variants.
 // It assumes custom ISel patterns for masking which can be provided as
@@ -220,7 +232,7 @@ multiclass AVX512_maskable_common<bits<8> O, Format F, X86VectorVTInfo _,
                                   string OpcodeStr,
                                   string AttSrcAsm, string IntelSrcAsm,
                                   dag RHS, dag MaskingRHS,
-                                  SDNode Select = vselect,
+                                  SDPatternOperator Select = vselect_mask,
                                   string MaskingConstraint = "",
                                   bit IsCommutable = 0,
                                   bit IsKCommutable = 0,
@@ -250,9 +262,9 @@ multiclass AVX512_maskable_split<bits<8> O, Format F, X86VectorVTInfo _,
                           OpcodeStr, AttSrcAsm, IntelSrcAsm,
                           [(set _.RC:$dst, RHS)],
                           [(set _.RC:$dst,
-                              (vselect _.KRCWM:$mask, MaskRHS, _.RC:$src0))],
+                              (vselect_mask _.KRCWM:$mask, MaskRHS, _.RC:$src0))],
                           [(set _.RC:$dst,
-                              (vselect _.KRCWM:$mask, MaskRHS, _.ImmAllZerosV))],
+                              (vselect_mask _.KRCWM:$mask, MaskRHS, _.ImmAllZerosV))],
                           "$src0 = $dst", IsCommutable, IsKCommutable,
                           IsKZCommutable>;
 
@@ -265,7 +277,7 @@ multiclass AVX512_maskable<bits<8> O, Format F, X86VectorVTInfo _,
                            dag RHS,
                            bit IsCommutable = 0, bit IsKCommutable = 0,
                            bit IsKZCommutable = IsCommutable,
-                           SDNode Select = vselect> :
+                           SDPatternOperator Select = vselect_mask> :
    AVX512_maskable_common<O, F, _, Outs, Ins,
                           !con((ins _.RC:$src0, _.KRCWM:$mask), Ins),
                           !con((ins _.KRCWM:$mask), Ins),
@@ -281,7 +293,7 @@ multiclass AVX512_maskable_scalar<bits<8> O, Format F, X86VectorVTInfo _,
                            string AttSrcAsm, string IntelSrcAsm,
                            dag RHS> :
    AVX512_maskable<O, F, _, Outs, Ins, OpcodeStr, AttSrcAsm, IntelSrcAsm,
-                   RHS, 0, 0, 0, X86selects>;
+                   RHS, 0, 0, 0, X86selects_mask>;
 
 // Similar to AVX512_maskable but in this case one of the source operands
 // ($src1) is already tied to $dst so we just use that for the preserved
@@ -293,7 +305,7 @@ multiclass AVX512_maskable_3src<bits<8> O, Format F, X86VectorVTInfo _,
                                 dag RHS,
                                 bit IsCommutable = 0,
                                 bit IsKCommutable = 0,
-                                SDNode Select = vselect,
+                                SDPatternOperator Select = vselect_mask,
                                 bit MaskOnly = 0> :
    AVX512_maskable_common<O, F, _, Outs,
                           !con((ins _.RC:$src1), NonTiedIns),
@@ -318,9 +330,9 @@ multiclass AVX512_maskable_3src_cast<bits<8> O, Format F, X86VectorVTInfo OutVT,
                           !con((ins InVT.RC:$src1, InVT.KRCWM:$mask), NonTiedIns),
                           !con((ins InVT.RC:$src1, InVT.KRCWM:$mask), NonTiedIns),
                           OpcodeStr, AttSrcAsm, IntelSrcAsm, (null_frag),
-                          (vselect InVT.KRCWM:$mask, RHS,
+                          (vselect_mask InVT.KRCWM:$mask, RHS,
                            (bitconvert InVT.RC:$src1)),
-                           vselect, "", IsCommutable>;
+                           vselect_mask, "", IsCommutable>;
 
 multiclass AVX512_maskable_3src_scalar<bits<8> O, Format F, X86VectorVTInfo _,
                                      dag Outs, dag NonTiedIns, string OpcodeStr,
@@ -331,7 +343,7 @@ multiclass AVX512_maskable_3src_scalar<bits<8> O, Format F, X86VectorVTInfo _,
                                      bit MaskOnly = 0> :
    AVX512_maskable_3src<O, F, _, Outs, NonTiedIns, OpcodeStr, AttSrcAsm,
                         IntelSrcAsm, RHS, IsCommutable, IsKCommutable,
-                        X86selects, MaskOnly>;
+                        X86selects_mask, MaskOnly>;
 
 multiclass AVX512_maskable_in_asm<bits<8> O, Format F, X86VectorVTInfo _,
                                   dag Outs, dag Ins,
@@ -426,9 +438,9 @@ multiclass AVX512_maskable_fma<bits<8> O, Format F, X86VectorVTInfo _,
                           OpcodeStr, AttSrcAsm, IntelSrcAsm,
                           [(set _.RC:$dst, RHS)],
                           [(set _.RC:$dst,
-                            (vselect _.KRCWM:$mask, MaskingRHS, _.RC:$src1))],
+                            (vselect_mask _.KRCWM:$mask, MaskingRHS, _.RC:$src1))],
                           [(set _.RC:$dst,
-                            (vselect _.KRCWM:$mask, MaskingRHS, _.ImmAllZerosV))],
+                            (vselect_mask _.KRCWM:$mask, MaskingRHS, _.ImmAllZerosV))],
                           "", IsCommutable, IsKCommutable>;
 
 // Alias instruction that maps zero vector to pxor / xorp* for AVX-512.
@@ -656,45 +668,45 @@ multiclass vinsert_for_mask_cast<string InstrStr, X86VectorVTInfo From,
                                  list<Predicate> p> {
 let Predicates = p in {
   def : Pat<(Cast.VT
-             (vselect Cast.KRCWM:$mask,
-                      (bitconvert
-                       (vinsert_insert:$ins (To.VT To.RC:$src1),
-                                            (From.VT From.RC:$src2),
-                                            (iPTR imm))),
-                      Cast.RC:$src0)),
+             (vselect_mask Cast.KRCWM:$mask,
+                           (bitconvert
+                            (vinsert_insert:$ins (To.VT To.RC:$src1),
+                                                 (From.VT From.RC:$src2),
+                                                 (iPTR imm))),
+                           Cast.RC:$src0)),
             (!cast<Instruction>(InstrStr#"rrk")
              Cast.RC:$src0, Cast.KRCWM:$mask, To.RC:$src1, From.RC:$src2,
              (INSERT_get_vinsert_imm To.RC:$ins))>;
   def : Pat<(Cast.VT
-             (vselect Cast.KRCWM:$mask,
-                      (bitconvert
-                       (vinsert_insert:$ins (To.VT To.RC:$src1),
-                                            (From.VT
-                                             (bitconvert
-                                              (From.LdFrag addr:$src2))),
-                                            (iPTR imm))),
-                      Cast.RC:$src0)),
+             (vselect_mask Cast.KRCWM:$mask,
+                           (bitconvert
+                            (vinsert_insert:$ins (To.VT To.RC:$src1),
+                                                 (From.VT
+                                                  (bitconvert
+                                                   (From.LdFrag addr:$src2))),
+                                                 (iPTR imm))),
+                           Cast.RC:$src0)),
             (!cast<Instruction>(InstrStr#"rmk")
              Cast.RC:$src0, Cast.KRCWM:$mask, To.RC:$src1, addr:$src2,
              (INSERT_get_vinsert_imm To.RC:$ins))>;
 
   def : Pat<(Cast.VT
-             (vselect Cast.KRCWM:$mask,
-                      (bitconvert
-                       (vinsert_insert:$ins (To.VT To.RC:$src1),
-                                            (From.VT From.RC:$src2),
-                                            (iPTR imm))),
-                      Cast.ImmAllZerosV)),
+             (vselect_mask Cast.KRCWM:$mask,
+                           (bitconvert
+                            (vinsert_insert:$ins (To.VT To.RC:$src1),
+                                                 (From.VT From.RC:$src2),
+                                                 (iPTR imm))),
+                           Cast.ImmAllZerosV)),
             (!cast<Instruction>(InstrStr#"rrkz")
              Cast.KRCWM:$mask, To.RC:$src1, From.RC:$src2,
              (INSERT_get_vinsert_imm To.RC:$ins))>;
   def : Pat<(Cast.VT
-             (vselect Cast.KRCWM:$mask,
-                      (bitconvert
-                       (vinsert_insert:$ins (To.VT To.RC:$src1),
-                                            (From.VT (From.LdFrag addr:$src2)),
-                                            (iPTR imm))),
-                      Cast.ImmAllZerosV)),
+             (vselect_mask Cast.KRCWM:$mask,
+                           (bitconvert
+                            (vinsert_insert:$ins (To.VT To.RC:$src1),
+                                                 (From.VT (From.LdFrag addr:$src2)),
+                                                 (iPTR imm))),
+                           Cast.ImmAllZerosV)),
             (!cast<Instruction>(InstrStr#"rmkz")
              Cast.KRCWM:$mask, To.RC:$src1, addr:$src2,
              (INSERT_get_vinsert_imm To.RC:$ins))>;
@@ -1012,20 +1024,20 @@ multiclass vextract_for_mask_cast<string InstrStr, X86VectorVTInfo From,
                                   SDNodeXForm EXTRACT_get_vextract_imm,
                                   list<Predicate> p> {
 let Predicates = p in {
-  def : Pat<(Cast.VT (vselect Cast.KRCWM:$mask,
-                              (bitconvert
-                               (To.VT (vextract_extract:$ext
-                                       (From.VT From.RC:$src), (iPTR imm)))),
-                              To.RC:$src0)),
+  def : Pat<(Cast.VT (vselect_mask Cast.KRCWM:$mask,
+                                   (bitconvert
+                                    (To.VT (vextract_extract:$ext
+                                            (From.VT From.RC:$src), (iPTR imm)))),
+                                   To.RC:$src0)),
             (Cast.VT (!cast<Instruction>(InstrStr#"rrk")
                       Cast.RC:$src0, Cast.KRCWM:$mask, From.RC:$src,
                       (EXTRACT_get_vextract_imm To.RC:$ext)))>;
 
-  def : Pat<(Cast.VT (vselect Cast.KRCWM:$mask,
-                              (bitconvert
-                               (To.VT (vextract_extract:$ext
-                                       (From.VT From.RC:$src), (iPTR imm)))),
-                              Cast.ImmAllZerosV)),
+  def : Pat<(Cast.VT (vselect_mask Cast.KRCWM:$mask,
+                                   (bitconvert
+                                    (To.VT (vextract_extract:$ext
+                                            (From.VT From.RC:$src), (iPTR imm)))),
+                                   Cast.ImmAllZerosV)),
             (Cast.VT (!cast<Instruction>(InstrStr#"rrkz")
                       Cast.KRCWM:$mask, From.RC:$src,
                       (EXTRACT_get_vextract_imm To.RC:$ext)))>;
@@ -1134,15 +1146,15 @@ multiclass avx512_broadcast_scalar<bits<8> opc, string OpcodeStr,
   def : Pat<(DestInfo.VT (X86VBroadcast SrcInfo.FRC:$src)),
             (!cast<Instruction>(Name#DestInfo.ZSuffix#rr)
              (SrcInfo.VT (COPY_TO_REGCLASS SrcInfo.FRC:$src, SrcInfo.RC)))>;
-  def : Pat<(DestInfo.VT (vselect DestInfo.KRCWM:$mask,
-                                  (X86VBroadcast SrcInfo.FRC:$src),
-                                  DestInfo.RC:$src0)),
+  def : Pat<(DestInfo.VT (vselect_mask DestInfo.KRCWM:$mask,
+                                       (X86VBroadcast SrcInfo.FRC:$src),
+                                       DestInfo.RC:$src0)),
             (!cast<Instruction>(Name#DestInfo.ZSuffix#rrk)
              DestInfo.RC:$src0, DestInfo.KRCWM:$mask,
              (SrcInfo.VT (COPY_TO_REGCLASS SrcInfo.FRC:$src, SrcInfo.RC)))>;
-  def : Pat<(DestInfo.VT (vselect DestInfo.KRCWM:$mask,
-                                  (X86VBroadcast SrcInfo.FRC:$src),
-                                  DestInfo.ImmAllZerosV)),
+  def : Pat<(DestInfo.VT (vselect_mask DestInfo.KRCWM:$mask,
+                                       (X86VBroadcast SrcInfo.FRC:$src),
+                                       DestInfo.ImmAllZerosV)),
             (!cast<Instruction>(Name#DestInfo.ZSuffix#rrkz)
              DestInfo.KRCWM:$mask, (SrcInfo.VT (COPY_TO_REGCLASS SrcInfo.FRC:$src, SrcInfo.RC)))>;
 }
@@ -1172,7 +1184,7 @@ multiclass avx512_broadcast_rm_split<bits<8> opc, string OpcodeStr,
                       !strconcat(OpcodeStr, "\t{$src, ${dst} {${mask}} {z}|",
                        "${dst} {${mask}} {z}, $src}"),
                        [(set MaskInfo.RC:$dst,
-                         (vselect MaskInfo.KRCWM:$mask,
+                         (vselect_mask MaskInfo.KRCWM:$mask,
                           (MaskInfo.VT
                            (bitconvert
                             (DestInfo.VT
@@ -1186,7 +1198,7 @@ multiclass avx512_broadcast_rm_split<bits<8> opc, string OpcodeStr,
                      !strconcat(OpcodeStr, "\t{$src, ${dst} {${mask}}|",
                      "${dst} {${mask}}, $src}"),
                      [(set MaskInfo.RC:$dst,
-                       (vselect MaskInfo.KRCWM:$mask,
+                       (vselect_mask MaskInfo.KRCWM:$mask,
                         (MaskInfo.VT
                          (bitconvert
                           (DestInfo.VT
@@ -1211,7 +1223,7 @@ multiclass avx512_broadcast_rm_split<bits<8> opc, string OpcodeStr,
                       !strconcat(OpcodeStr, "\t{$src, ${dst} {${mask}} {z}|",
                        "${dst} {${mask}} {z}, $src}"),
                        [(set MaskInfo.RC:$dst,
-                         (vselect MaskInfo.KRCWM:$mask,
+                         (vselect_mask MaskInfo.KRCWM:$mask,
                           (MaskInfo.VT
                            (bitconvert
                             (DestInfo.VT
@@ -1228,7 +1240,7 @@ multiclass avx512_broadcast_rm_split<bits<8> opc, string OpcodeStr,
                      !strconcat(OpcodeStr, "\t{$src, ${dst} {${mask}}|",
                      "${dst} {${mask}}, $src}"),
                      [(set MaskInfo.RC:$dst,
-                       (vselect MaskInfo.KRCWM:$mask,
+                       (vselect_mask MaskInfo.KRCWM:$mask,
                         (MaskInfo.VT
                          (bitconvert
                           (DestInfo.VT
@@ -1321,11 +1333,11 @@ multiclass avx512_int_broadcastbw_reg<bits<8> opc, string Name, SchedWrite Sched
              (!cast<Instruction>(Name#rr)
               (i32 (INSERT_SUBREG (i32 (IMPLICIT_DEF)), SrcRC:$src, Subreg)))>;
 
-  def : Pat <(vselect _.KRCWM:$mask, (_.VT (OpNode SrcRC:$src)), _.RC:$src0),
+  def : Pat <(vselect_mask _.KRCWM:$mask, (_.VT (OpNode SrcRC:$src)), _.RC:$src0),
              (!cast<Instruction>(Name#rrk) _.RC:$src0, _.KRCWM:$mask,
               (i32 (INSERT_SUBREG (i32 (IMPLICIT_DEF)), SrcRC:$src, Subreg)))>;
 
-  def : Pat <(vselect _.KRCWM:$mask, (_.VT (OpNode SrcRC:$src)), _.ImmAllZerosV),
+  def : Pat <(vselect_mask _.KRCWM:$mask, (_.VT (OpNode SrcRC:$src)), _.ImmAllZerosV),
              (!cast<Instruction>(Name#rrkz) _.KRCWM:$mask,
               (i32 (INSERT_SUBREG (i32 (IMPLICIT_DEF)), SrcRC:$src, Subreg)))>;
 }
@@ -1481,38 +1493,38 @@ def : Pat<(v64i8 (X86SubVBroadcast (loadv16i8 addr:$src))),
           (VBROADCASTI32X4rm addr:$src)>;
 
 // Patterns for selects of bitcasted operations.
-def : Pat<(vselect VK16WM:$mask,
-                   (bc_v16f32 (v8f64 (X86SubVBroadcast (loadv2f64 addr:$src)))),
-                   (v16f32 immAllZerosV)),
+def : Pat<(vselect_mask VK16WM:$mask,
+                        (bc_v16f32 (v8f64 (X86SubVBroadcast (loadv2f64 addr:$src)))),
+                        (v16f32 immAllZerosV)),
           (VBROADCASTF32X4rmkz VK16WM:$mask, addr:$src)>;
-def : Pat<(vselect VK16WM:$mask,
-                   (bc_v16f32 (v8f64 (X86SubVBroadcast (loadv2f64 addr:$src)))),
-                   VR512:$src0),
+def : Pat<(vselect_mask VK16WM:$mask,
+                        (bc_v16f32 (v8f64 (X86SubVBroadcast (loadv2f64 addr:$src)))),
+                        VR512:$src0),
           (VBROADCASTF32X4rmk VR512:$src0, VK16WM:$mask, addr:$src)>;
-def : Pat<(vselect VK16WM:$mask,
-                   (bc_v16i32 (v8i64 (X86SubVBroadcast (loadv2i64 addr:$src)))),
-                   (v16i32 immAllZerosV)),
+def : Pat<(vselect_mask VK16WM:$mask,
+                        (bc_v16i32 (v8i64 (X86SubVBroadcast (loadv2i64 addr:$src)))),
+                        (v16i32 immAllZerosV)),
           (VBROADCASTI32X4rmkz VK16WM:$mask, addr:$src)>;
-def : Pat<(vselect VK16WM:$mask,
-                   (bc_v16i32 (v8i64 (X86SubVBroadcast (loadv2i64 addr:$src)))),
-                   VR512:$src0),
+def : Pat<(vselect_mask VK16WM:$mask,
+                        (bc_v16i32 (v8i64 (X86SubVBroadcast (loadv2i64 addr:$src)))),
+                        VR512:$src0),
           (VBROADCASTI32X4rmk VR512:$src0, VK16WM:$mask, addr:$src)>;
 
-def : Pat<(vselect VK8WM:$mask,
-                   (bc_v8f64 (v16f32 (X86SubVBroadcast (loadv8f32 addr:$src)))),
-                   (v8f64 immAllZerosV)),
+def : Pat<(vselect_mask VK8WM:$mask,
+                        (bc_v8f64 (v16f32 (X86SubVBroadcast (loadv8f32 addr:$src)))),
+                        (v8f64 immAllZerosV)),
           (VBROADCASTF64X4rmkz VK8WM:$mask, addr:$src)>;
-def : Pat<(vselect VK8WM:$mask,
-                   (bc_v8f64 (v16f32 (X86SubVBroadcast (loadv8f32 addr:$src)))),
-                   VR512:$src0),
+def : Pat<(vselect_mask VK8WM:$mask,
+                        (bc_v8f64 (v16f32 (X86SubVBroadcast (loadv8f32 addr:$src)))),
+                        VR512:$src0),
           (VBROADCASTF64X4rmk VR512:$src0, VK8WM:$mask, addr:$src)>;
-def : Pat<(vselect VK8WM:$mask,
-                   (bc_v8i64 (v16i32 (X86SubVBroadcast (loadv8i32 addr:$src)))),
-                   (v8i64 immAllZerosV)),
+def : Pat<(vselect_mask VK8WM:$mask,
+                        (bc_v8i64 (v16i32 (X86SubVBroadcast (loadv8i32 addr:$src)))),
+                        (v8i64 immAllZerosV)),
           (VBROADCASTI64X4rmkz VK8WM:$mask, addr:$src)>;
-def : Pat<(vselect VK8WM:$mask,
-                   (bc_v8i64 (v16i32 (X86SubVBroadcast (loadv8i32 addr:$src)))),
-                   VR512:$src0),
+def : Pat<(vselect_mask VK8WM:$mask,
+                        (bc_v8i64 (v16i32 (X86SubVBroadcast (loadv8i32 addr:$src)))),
+                        VR512:$src0),
           (VBROADCASTI64X4rmk VR512:$src0, VK8WM:$mask, addr:$src)>;
 }
 
@@ -1534,21 +1546,21 @@ def : Pat<(v32i8 (X86SubVBroadcast (loadv16i8 addr:$src))),
           (VBROADCASTI32X4Z256rm addr:$src)>;
 
 // Patterns for selects of bitcasted operations.
-def : Pat<(vselect VK8WM:$mask,
-                   (bc_v8f32 (v4f64 (X86SubVBroadcast (loadv2f64 addr:$src)))),
-                   (v8f32 immAllZerosV)),
+def : Pat<(vselect_mask VK8WM:$mask,
+                        (bc_v8f32 (v4f64 (X86SubVBroadcast (loadv2f64 addr:$src)))),
+                        (v8f32 immAllZerosV)),
           (VBROADCASTF32X4Z256rmkz VK8WM:$mask, addr:$src)>;
-def : Pat<(vselect VK8WM:$mask,
-                   (bc_v8f32 (v4f64 (X86SubVBroadcast (loadv2f64 addr:$src)))),
-                   VR256X:$src0),
+def : Pat<(vselect_mask VK8WM:$mask,
+                        (bc_v8f32 (v4f64 (X86SubVBroadcast (loadv2f64 addr:$src)))),
+                        VR256X:$src0),
           (VBROADCASTF32X4Z256rmk VR256X:$src0, VK8WM:$mask, addr:$src)>;
-def : Pat<(vselect VK8WM:$mask,
-                   (bc_v8i32 (v4i64 (X86SubVBroadcast (loadv2i64 addr:$src)))),
-                   (v8i32 immAllZerosV)),
+def : Pat<(vselect_mask VK8WM:$mask,
+                        (bc_v8i32 (v4i64 (X86SubVBroadcast (loadv2i64 addr:$src)))),
+                        (v8i32 immAllZerosV)),
           (VBROADCASTI32X4Z256rmkz VK8WM:$mask, addr:$src)>;
-def : Pat<(vselect VK8WM:$mask,
-                   (bc_v8i32 (v4i64 (X86SubVBroadcast (loadv2i64 addr:$src)))),
-                   VR256X:$src0),
+def : Pat<(vselect_mask VK8WM:$mask,
+                        (bc_v8i32 (v4i64 (X86SubVBroadcast (loadv2i64 addr:$src)))),
+                        VR256X:$src0),
           (VBROADCASTI32X4Z256rmk VR256X:$src0, VK8WM:$mask, addr:$src)>;
 
 
@@ -1583,21 +1595,21 @@ defm VBROADCASTF64X2Z128 : avx512_subvec_broadcast_rm_dq<0x1a, "vbroadcastf64x2"
                            EVEX_V256, EVEX_CD8<64, CD8VT2>;
 
 // Patterns for selects of bitcasted operations.
-def : Pat<(vselect VK4WM:$mask,
-                   (bc_v4f64 (v8f32 (X86SubVBroadcast (loadv4f32 addr:$src)))),
-                   (v4f64 immAllZerosV)),
+def : Pat<(vselect_mask VK4WM:$mask,
+                        (bc_v4f64 (v8f32 (X86SubVBroadcast (loadv4f32 addr:$src)))),
+                        (v4f64 immAllZerosV)),
           (VBROADCASTF64X2Z128rmkz VK4WM:$mask, addr:$src)>;
-def : Pat<(vselect VK4WM:$mask,
-                   (bc_v4f64 (v8f32 (X86SubVBroadcast (loadv4f32 addr:$src)))),
-                   VR256X:$src0),
+def : Pat<(vselect_mask VK4WM:$mask,
+                        (bc_v4f64 (v8f32 (X86SubVBroadcast (loadv4f32 addr:$src)))),
+                        VR256X:$src0),
           (VBROADCASTF64X2Z128rmk VR256X:$src0, VK4WM:$mask, addr:$src)>;
-def : Pat<(vselect VK4WM:$mask,
-                   (bc_v4i64 (v8i32 (X86SubVBroadcast (loadv4i32 addr:$src)))),
-                   (v4i64 immAllZerosV)),
+def : Pat<(vselect_mask VK4WM:$mask,
+                        (bc_v4i64 (v8i32 (X86SubVBroadcast (loadv4i32 addr:$src)))),
+                        (v4i64 immAllZerosV)),
           (VBROADCASTI64X2Z128rmkz VK4WM:$mask, addr:$src)>;
-def : Pat<(vselect VK4WM:$mask,
-                   (bc_v4i64 (v8i32 (X86SubVBroadcast (loadv4i32 addr:$src)))),
-                   VR256X:$src0),
+def : Pat<(vselect_mask VK4WM:$mask,
+                        (bc_v4i64 (v8i32 (X86SubVBroadcast (loadv4i32 addr:$src)))),
+                        VR256X:$src0),
           (VBROADCASTI64X2Z128rmk VR256X:$src0, VK4WM:$mask, addr:$src)>;
 }
 
@@ -1616,38 +1628,38 @@ defm VBROADCASTF32X8 : avx512_subvec_broadcast_rm_dq<0x1b, "vbroadcastf32x8",
                        EVEX_V512, EVEX_CD8<32, CD8VT8>;
 
 // Patterns for selects of bitcasted operations.
-def : Pat<(vselect VK16WM:$mask,
-                   (bc_v16f32 (v8f64 (X86SubVBroadcast (loadv4f64 addr:$src)))),
-                   (v16f32 immAllZerosV)),
+def : Pat<(vselect_mask VK16WM:$mask,
+                        (bc_v16f32 (v8f64 (X86SubVBroadcast (loadv4f64 addr:$src)))),
+                        (v16f32 immAllZerosV)),
           (VBROADCASTF32X8rmkz VK16WM:$mask, addr:$src)>;
-def : Pat<(vselect VK16WM:$mask,
-                   (bc_v16f32 (v8f64 (X86SubVBroadcast (loadv4f64 addr:$src)))),
-                   VR512:$src0),
+def : Pat<(vselect_mask VK16WM:$mask,
+                        (bc_v16f32 (v8f64 (X86SubVBroadcast (loadv4f64 addr:$src)))),
+                        VR512:$src0),
           (VBROADCASTF32X8rmk VR512:$src0, VK16WM:$mask, addr:$src)>;
-def : Pat<(vselect VK16WM:$mask,
-                   (bc_v16i32 (v8i64 (X86SubVBroadcast (loadv4i64 addr:$src)))),
-                   (v16i32 immAllZerosV)),
+def : Pat<(vselect_mask VK16WM:$mask,
+                        (bc_v16i32 (v8i64 (X86SubVBroadcast (loadv4i64 addr:$src)))),
+                        (v16i32 immAllZerosV)),
           (VBROADCASTI32X8rmkz VK16WM:$mask, addr:$src)>;
-def : Pat<(vselect VK16WM:$mask,
-                   (bc_v16i32 (v8i64 (X86SubVBroadcast (loadv4i64 addr:$src)))),
-                   VR512:$src0),
+def : Pat<(vselect_mask VK16WM:$mask,
+                        (bc_v16i32 (v8i64 (X86SubVBroadcast (loadv4i64 addr:$src)))),
+                        VR512:$src0),
           (VBROADCASTI32X8rmk VR512:$src0, VK16WM:$mask, addr:$src)>;
 
-def : Pat<(vselect VK8WM:$mask,
-                   (bc_v8f64 (v16f32 (X86SubVBroadcast (loadv4f32 addr:$src)))),
-                   (v8f64 immAllZerosV)),
+def : Pat<(vselect_mask VK8WM:$mask,
+                        (bc_v8f64 (v16f32 (X86SubVBroadcast (loadv4f32 addr:$src)))),
+                        (v8f64 immAllZerosV)),
           (VBROADCASTF64X2rmkz VK8WM:$mask, addr:$src)>;
-def : Pat<(vselect VK8WM:$mask,
-                   (bc_v8f64 (v16f32 (X86SubVBroadcast (loadv4f32 addr:$src)))),
-                   VR512:$src0),
+def : Pat<(vselect_mask VK8WM:$mask,
+                        (bc_v8f64 (v16f32 (X86SubVBroadcast (loadv4f32 addr:$src)))),
+                        VR512:$src0),
           (VBROADCASTF64X2rmk VR512:$src0, VK8WM:$mask, addr:$src)>;
-def : Pat<(vselect VK8WM:$mask,
-                   (bc_v8i64 (v16i32 (X86SubVBroadcast (loadv4i32 addr:$src)))),
-                   (v8i64 immAllZerosV)),
+def : Pat<(vselect_mask VK8WM:$mask,
+                        (bc_v8i64 (v16i32 (X86SubVBroadcast (loadv4i32 addr:$src)))),
+                        (v8i64 immAllZerosV)),
           (VBROADCASTI64X2rmkz VK8WM:$mask, addr:$src)>;
-def : Pat<(vselect VK8WM:$mask,
-                   (bc_v8i64 (v16i32 (X86SubVBroadcast (loadv4i32 addr:$src)))),
-                   VR512:$src0),
+def : Pat<(vselect_mask VK8WM:$mask,
+                        (bc_v8i64 (v16i32 (X86SubVBroadcast (loadv4i32 addr:$src)))),
+                        VR512:$src0),
           (VBROADCASTI64X2rmk VR512:$src0, VK8WM:$mask, addr:$src)>;
 }
 
@@ -1801,24 +1813,27 @@ defm VPERMI2PD : avx512_perm_i_sizes<0x77, "vpermi2pd", WriteFVarShuffle256,
 multiclass avx512_perm_i_lowering<string InstrStr, X86VectorVTInfo _,
                                   X86VectorVTInfo IdxVT,
                                   X86VectorVTInfo CastVT> {
-  def : Pat<(_.VT (vselect _.KRCWM:$mask,
-                             (X86VPermt2 (_.VT _.RC:$src2),
-                                         (IdxVT.VT (bitconvert (CastVT.VT _.RC:$src1))), _.RC:$src3),
-                             (_.VT (bitconvert (CastVT.VT _.RC:$src1))))),
+  def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,
+                                (X86VPermt2 (_.VT _.RC:$src2),
+                                            (IdxVT.VT (bitconvert
+                                                       (CastVT.VT _.RC:$src1))),
+                                            _.RC:$src3),
+                                (_.VT (bitconvert (CastVT.VT _.RC:$src1))))),
             (!cast<Instruction>(InstrStr#"rrk") _.RC:$src1, _.KRCWM:$mask,
                                                 _.RC:$src2, _.RC:$src3)>;
-  def : Pat<(_.VT (vselect _.KRCWM:$mask,
-                             (X86VPermt2 _.RC:$src2,
-                                         (IdxVT.VT (bitconvert  (CastVT.VT _.RC:$src1))),
-                                         (_.LdFrag addr:$src3)),
-                             (_.VT (bitconvert  (CastVT.VT _.RC:$src1))))),
+  def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,
+                                (X86VPermt2 _.RC:$src2,
+                                            (IdxVT.VT (bitconvert
+                                                       (CastVT.VT _.RC:$src1))),
+                                            (_.LdFrag addr:$src3)),
+                                (_.VT (bitconvert  (CastVT.VT _.RC:$src1))))),
             (!cast<Instruction>(InstrStr#"rmk") _.RC:$src1, _.KRCWM:$mask,
                                                 _.RC:$src2, addr:$src3)>;
-  def : Pat<(_.VT (vselect _.KRCWM:$mask,
-                             (X86VPermt2 _.RC:$src2,
-                                         (IdxVT.VT (bitconvert  (CastVT.VT _.RC:$src1))),
-                                         (_.BroadcastLdFrag addr:$src3)),
-                             (_.VT (bitconvert  (CastVT.VT _.RC:$src1))))),
+  def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,
+                                 (X86VPermt2 _.RC:$src2,
+                                             (IdxVT.VT (bitconvert  (CastVT.VT _.RC:$src1))),
+                                             (_.BroadcastLdFrag addr:$src3)),
+                                 (_.VT (bitconvert  (CastVT.VT _.RC:$src1))))),
             (!cast<Instruction>(InstrStr#"rmbk") _.RC:$src1, _.KRCWM:$mask,
                                                  _.RC:$src2, addr:$src3)>;
 }
@@ -3374,7 +3389,7 @@ multiclass avx512_load<bits<8> opc, string OpcodeStr, string Name,
                      !strconcat(OpcodeStr, "\t{$src1, ${dst} {${mask}}|",
                       "${dst} {${mask}}, $src1}"),
                      [(set _.RC:$dst, (_.VT
-                         (vselect _.KRCWM:$mask,
+                         (vselect_mask _.KRCWM:$mask,
                           (_.VT (ld_frag addr:$src1)),
                            (_.VT _.RC:$src0))))], _.ExeDomain>,
                      EVEX, EVEX_K, Sched<[Sched.RM]>;
@@ -3383,7 +3398,7 @@ multiclass avx512_load<bits<8> opc, string OpcodeStr, string Name,
                   (ins _.KRCWM:$mask, _.MemOp:$src),
                   OpcodeStr #"\t{$src, ${dst} {${mask}} {z}|"#
                                 "${dst} {${mask}} {z}, $src}",
-                  [(set _.RC:$dst, (_.VT (vselect _.KRCWM:$mask,
+                  [(set _.RC:$dst, (_.VT (vselect_mask _.KRCWM:$mask,
                     (_.VT (ld_frag addr:$src)), _.ImmAllZerosV)))],
                   _.ExeDomain>, EVEX, EVEX_KZ, Sched<[Sched.RM]>;
   }
@@ -4251,6 +4266,17 @@ def : Pat<(f64 (X86selects VK1WM:$mask, (loadf64 addr:$src), (f64 FR64X:$src0)))
 def : Pat<(f64 (X86selects VK1WM:$mask, (loadf64 addr:$src), fp64imm0)),
           (COPY_TO_REGCLASS (v2f64 (VMOVSDZrmkz VK1WM:$mask, addr:$src)), FR64X)>;
 
+
+def : Pat<(v4f32 (X86selects VK1WM:$mask, (v4f32 VR128X:$src1), (v4f32 VR128X:$src2))),
+          (VMOVSSZrrk VR128X:$src2, VK1WM:$mask, VR128X:$src1, VR128X:$src1)>;
+def : Pat<(v2f64 (X86selects VK1WM:$mask, (v2f64 VR128X:$src1), (v2f64 VR128X:$src2))),
+          (VMOVSDZrrk VR128X:$src2, VK1WM:$mask, VR128X:$src1, VR128X:$src1)>;
+
+def : Pat<(v4f32 (X86selects VK1WM:$mask, (v4f32 VR128X:$src1), (v4f32 immAllZerosV))),
+          (VMOVSSZrrkz VK1WM:$mask, VR128X:$src1, VR128X:$src1)>;
+def : Pat<(v2f64 (X86selects VK1WM:$mask, (v2f64 VR128X:$src1), (v2f64 immAllZerosV))),
+          (VMOVSDZrrkz VK1WM:$mask, VR128X:$src1, VR128X:$src1)>;
+
 let hasSideEffects = 0, isCodeGenOnly = 1, ForceDisassemble = 1 in {
   def VMOVSSZrr_REV: AVX512<0x11, MRMDestReg, (outs VR128X:$dst),
                            (ins VR128X:$src1, VR128X:$src2),
@@ -5122,26 +5148,26 @@ multiclass avx512_logical_lowering<string InstrStr, SDNode OpNode,
                                     X86VectorVTInfo _,
                                     X86VectorVTInfo IntInfo> {
   // Masked register-register logical operations.
-  def : Pat<(_.VT (vselect _.KRCWM:$mask,
+  def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,
                    (bitconvert (IntInfo.VT (OpNode _.RC:$src1, _.RC:$src2))),
                    _.RC:$src0)),
             (!cast<Instruction>(InstrStr#rrk) _.RC:$src0, _.KRCWM:$mask,
              _.RC:$src1, _.RC:$src2)>;
 
-  def : Pat<(_.VT (vselect _.KRCWM:$mask,
+  def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,
                    (bitconvert (IntInfo.VT (OpNode _.RC:$src1, _.RC:$src2))),
                    _.ImmAllZerosV)),
             (!cast<Instruction>(InstrStr#rrkz) _.KRCWM:$mask, _.RC:$src1,
              _.RC:$src2)>;
 
   // Masked register-memory logical operations.
-  def : Pat<(_.VT (vselect _.KRCWM:$mask,
+  def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,
                    (bitconvert (IntInfo.VT (OpNode _.RC:$src1,
                                             (load addr:$src2)))),
                    _.RC:$src0)),
             (!cast<Instruction>(InstrStr#rmk) _.RC:$src0, _.KRCWM:$mask,
              _.RC:$src1, addr:$src2)>;
-  def : Pat<(_.VT (vselect _.KRCWM:$mask,
+  def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,
                    (bitconvert (IntInfo.VT (OpNode _.RC:$src1,
                                             (load addr:$src2)))),
                    _.ImmAllZerosV)),
@@ -5153,14 +5179,14 @@ multiclass avx512_logical_lowering_bcast<string InstrStr, SDNode OpNode,
                                          X86VectorVTInfo _,
                                          X86VectorVTInfo IntInfo> {
   // Register-broadcast logical operations.
-  def : Pat<(_.VT (vselect _.KRCWM:$mask,
+  def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,
                    (bitconvert
                     (IntInfo.VT (OpNode _.RC:$src1,
                                  (IntInfo.VT (IntInfo.BroadcastLdFrag addr:$src2))))),
                    _.RC:$src0)),
             (!cast<Instruction>(InstrStr#rmbk) _.RC:$src0, _.KRCWM:$mask,
              _.RC:$src1, addr:$src2)>;
-  def : Pat<(_.VT (vselect _.KRCWM:$mask,
+  def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,
                    (bitconvert
                     (IntInfo.VT (OpNode _.RC:$src1,
                                  (IntInfo.VT (IntInfo.BroadcastLdFrag addr:$src2))))),
@@ -6795,7 +6821,7 @@ multiclass avx512_scalar_fma_patterns<SDNode Op, SDNode MaskedOp,
                addr:$src3)>;
 
     def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector
-               (X86selects VK1WM:$mask,
+               (X86selects_mask VK1WM:$mask,
                 (MaskedOp _.FRC:$src2,
                     (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))),
                     _.FRC:$src3),
@@ -6806,7 +6832,7 @@ multiclass avx512_scalar_fma_patterns<SDNode Op, SDNode MaskedOp,
                (_.VT (COPY_TO_REGCLASS _.FRC:$src3, VR128X)))>;
 
     def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector
-               (X86selects VK1WM:$mask,
+               (X86selects_mask VK1WM:$mask,
                 (MaskedOp _.FRC:$src2,
                     (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))),
                     (_.ScalarLdFrag addr:$src3)),
@@ -6816,7 +6842,7 @@ multiclass avx512_scalar_fma_patterns<SDNode Op, SDNode MaskedOp,
                (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)), addr:$src3)>;
 
     def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector
-               (X86selects VK1WM:$mask,
+               (X86selects_mask VK1WM:$mask,
                 (MaskedOp (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))),
                           (_.ScalarLdFrag addr:$src3), _.FRC:$src2),
                 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))))))),
@@ -6825,7 +6851,7 @@ multiclass avx512_scalar_fma_patterns<SDNode Op, SDNode MaskedOp,
                (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)), addr:$src3)>;
 
     def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector
-               (X86selects VK1WM:$mask,
+               (X86selects_mask VK1WM:$mask,
                 (MaskedOp _.FRC:$src2, _.FRC:$src3,
                           (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))),
                 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))))))),
@@ -6835,7 +6861,7 @@ multiclass avx512_scalar_fma_patterns<SDNode Op, SDNode MaskedOp,
                (_.VT (COPY_TO_REGCLASS _.FRC:$src3, VR128X)))>;
 
     def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector
-               (X86selects VK1WM:$mask,
+               (X86selects_mask VK1WM:$mask,
                 (MaskedOp _.FRC:$src2, (_.ScalarLdFrag addr:$src3),
                           (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))),
                 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))))))),
@@ -6844,7 +6870,7 @@ multiclass avx512_scalar_fma_patterns<SDNode Op, SDNode MaskedOp,
                (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)), addr:$src3)>;
 
     def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector
-               (X86selects VK1WM:$mask,
+               (X86selects_mask VK1WM:$mask,
                 (MaskedOp _.FRC:$src2,
                           (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))),
                           _.FRC:$src3),
@@ -6855,7 +6881,7 @@ multiclass avx512_scalar_fma_patterns<SDNode Op, SDNode MaskedOp,
                (_.VT (COPY_TO_REGCLASS _.FRC:$src3, VR128X)))>;
 
     def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector
-               (X86selects VK1WM:$mask,
+               (X86selects_mask VK1WM:$mask,
                 (MaskedOp _.FRC:$src2, _.FRC:$src3,
                           (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))),
                 (_.EltVT ZeroFP)))))),
@@ -6865,7 +6891,7 @@ multiclass avx512_scalar_fma_patterns<SDNode Op, SDNode MaskedOp,
                (_.VT (COPY_TO_REGCLASS _.FRC:$src3, VR128X)))>;
 
     def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector
-               (X86selects VK1WM:$mask,
+               (X86selects_mask VK1WM:$mask,
                 (MaskedOp _.FRC:$src2,
                           (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))),
                           (_.ScalarLdFrag addr:$src3)),
@@ -6875,7 +6901,7 @@ multiclass avx512_scalar_fma_patterns<SDNode Op, SDNode MaskedOp,
                (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)), addr:$src3)>;
 
     def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector
-               (X86selects VK1WM:$mask,
+               (X86selects_mask VK1WM:$mask,
                 (MaskedOp (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))),
                           _.FRC:$src2, (_.ScalarLdFrag addr:$src3)),
                 (_.EltVT ZeroFP)))))),
@@ -6884,7 +6910,7 @@ multiclass avx512_scalar_fma_patterns<SDNode Op, SDNode MaskedOp,
                (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)), addr:$src3)>;
 
     def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector
-               (X86selects VK1WM:$mask,
+               (X86selects_mask VK1WM:$mask,
                 (MaskedOp _.FRC:$src2, (_.ScalarLdFrag addr:$src3),
                           (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))),
                 (_.EltVT ZeroFP)))))),
@@ -6910,7 +6936,7 @@ multiclass avx512_scalar_fma_patterns<SDNode Op, SDNode MaskedOp,
                (_.VT (COPY_TO_REGCLASS _.FRC:$src3, VR128X)), AVX512RC:$rc)>;
 
     def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector
-               (X86selects VK1WM:$mask,
+               (X86selects_mask VK1WM:$mask,
                 (RndOp _.FRC:$src2,
                        (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))),
                        _.FRC:$src3, (i32 timm:$rc)),
@@ -6921,7 +6947,7 @@ multiclass avx512_scalar_fma_patterns<SDNode Op, SDNode MaskedOp,
                (_.VT (COPY_TO_REGCLASS _.FRC:$src3, VR128X)), AVX512RC:$rc)>;
 
     def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector
-               (X86selects VK1WM:$mask,
+               (X86selects_mask VK1WM:$mask,
                 (RndOp _.FRC:$src2, _.FRC:$src3,
                        (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))),
                        (i32 timm:$rc)),
@@ -6932,7 +6958,7 @@ multiclass avx512_scalar_fma_patterns<SDNode Op, SDNode MaskedOp,
                (_.VT (COPY_TO_REGCLASS _.FRC:$src3, VR128X)), AVX512RC:$rc)>;
 
     def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector
-               (X86selects VK1WM:$mask,
+               (X86selects_mask VK1WM:$mask,
                 (RndOp _.FRC:$src2,
                        (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))),
                        _.FRC:$src3, (i32 timm:$rc)),
@@ -6943,7 +6969,7 @@ multiclass avx512_scalar_fma_patterns<SDNode Op, SDNode MaskedOp,
                (_.VT (COPY_TO_REGCLASS _.FRC:$src3, VR128X)), AVX512RC:$rc)>;
 
     def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector
-               (X86selects VK1WM:$mask,
+               (X86selects_mask VK1WM:$mask,
                 (RndOp _.FRC:$src2, _.FRC:$src3,
                        (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))),
                        (i32 timm:$rc)),
@@ -7566,12 +7592,12 @@ let Uses = [MXCSR], mayRaiseFPException = 1 in {
                          (ins MaskRC:$mask, _Src.RC:$src),
                           OpcodeStr, "$src", "$src",
                          (_.VT (OpNode (_Src.VT _Src.RC:$src))),
-                         (vselect MaskRC:$mask,
-                                  (_.VT (MaskOpNode (_Src.VT _Src.RC:$src))),
-                                  _.RC:$src0),
-                         (vselect MaskRC:$mask,
-                                  (_.VT (MaskOpNode (_Src.VT _Src.RC:$src))),
-                                  _.ImmAllZerosV)>,
+                         (vselect_mask MaskRC:$mask,
+                                       (_.VT (MaskOpNode (_Src.VT _Src.RC:$src))),
+                                       _.RC:$src0),
+                         (vselect_mask MaskRC:$mask,
+                                       (_.VT (MaskOpNode (_Src.VT _Src.RC:$src))),
+                                       _.ImmAllZerosV)>,
                          EVEX, Sched<[sched]>;
 
   defm rm : AVX512_maskable_cvt<opc, MRMSrcMem, _, (outs _.RC:$dst),
@@ -7580,8 +7606,8 @@ let Uses = [MXCSR], mayRaiseFPException = 1 in {
                          (ins MaskRC:$mask, MemOp:$src),
                          OpcodeStr#Alias, "$src", "$src",
                          LdDAG,
-                         (vselect MaskRC:$mask, MaskLdDAG, _.RC:$src0),
-                         (vselect MaskRC:$mask, MaskLdDAG, _.ImmAllZerosV)>,
+                         (vselect_mask MaskRC:$mask, MaskLdDAG, _.RC:$src0),
+                         (vselect_mask MaskRC:$mask, MaskLdDAG, _.ImmAllZerosV)>,
                          EVEX, Sched<[sched.Folded]>;
 
   defm rmb : AVX512_maskable_cvt<opc, MRMSrcMem, _, (outs _.RC:$dst),
@@ -7593,18 +7619,18 @@ let Uses = [MXCSR], mayRaiseFPException = 1 in {
                          (_.VT (OpNode (_Src.VT
                                   (_Src.BroadcastLdFrag addr:$src))
                             )),
-                         (vselect MaskRC:$mask,
-                                  (_.VT
-                                   (MaskOpNode
-                                    (_Src.VT
-                                     (_Src.BroadcastLdFrag addr:$src)))),
-                                  _.RC:$src0),
-                         (vselect MaskRC:$mask,
-                                  (_.VT
-                                   (MaskOpNode
-                                    (_Src.VT
-                                     (_Src.BroadcastLdFrag addr:$src)))),
-                                  _.ImmAllZerosV)>,
+                         (vselect_mask MaskRC:$mask,
+                                       (_.VT
+                                        (MaskOpNode
+                                         (_Src.VT
+                                          (_Src.BroadcastLdFrag addr:$src)))),
+                                       _.RC:$src0),
+                         (vselect_mask MaskRC:$mask,
+                                       (_.VT
+                                        (MaskOpNode
+                                         (_Src.VT
+                                          (_Src.BroadcastLdFrag addr:$src)))),
+                                       _.ImmAllZerosV)>,
                          EVEX, EVEX_B, Sched<[sched.Folded]>;
   }
 }
@@ -8365,70 +8391,70 @@ let Predicates = [HasVLX] in {
 let Predicates = [HasDQI, HasVLX] in {
   def : Pat<(v2i64 (X86cvtp2Int (bc_v4f32 (v2f64 (X86vzload64 addr:$src))))),
             (VCVTPS2QQZ128rm addr:$src)>;
-  def : Pat<(v2i64 (vselect VK2WM:$mask,
-                            (X86cvtp2Int (bc_v4f32 (v2f64 (X86vzload64 addr:$src)))),
-                            VR128X:$src0)),
+  def : Pat<(v2i64 (vselect_mask VK2WM:$mask,
+                                 (X86cvtp2Int (bc_v4f32 (v2f64 (X86vzload64 addr:$src)))),
+                                 VR128X:$src0)),
             (VCVTPS2QQZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;
-  def : Pat<(v2i64 (vselect VK2WM:$mask,
-                            (X86cvtp2Int (bc_v4f32 (v2f64 (X86vzload64 addr:$src)))),
-                            v2i64x_info.ImmAllZerosV)),
+  def : Pat<(v2i64 (vselect_mask VK2WM:$mask,
+                                 (X86cvtp2Int (bc_v4f32 (v2f64 (X86vzload64 addr:$src)))),
+                                 v2i64x_info.ImmAllZerosV)),
             (VCVTPS2QQZ128rmkz VK2WM:$mask, addr:$src)>;
 
   def : Pat<(v2i64 (X86cvtp2UInt (bc_v4f32 (v2f64 (X86vzload64 addr:$src))))),
             (VCVTPS2UQQZ128rm addr:$src)>;
-  def : Pat<(v2i64 (vselect VK2WM:$mask,
-                            (X86cvtp2UInt (bc_v4f32 (v2f64 (X86vzload64 addr:$src)))),
-                            VR128X:$src0)),
+  def : Pat<(v2i64 (vselect_mask VK2WM:$mask,
+                                 (X86cvtp2UInt (bc_v4f32 (v2f64 (X86vzload64 addr:$src)))),
+                                 VR128X:$src0)),
             (VCVTPS2UQQZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;
-  def : Pat<(v2i64 (vselect VK2WM:$mask,
-                            (X86cvtp2UInt (bc_v4f32 (v2f64 (X86vzload64 addr:$src)))),
-                            v2i64x_info.ImmAllZerosV)),
+  def : Pat<(v2i64 (vselect_mask VK2WM:$mask,
+                                 (X86cvtp2UInt (bc_v4f32 (v2f64 (X86vzload64 addr:$src)))),
+                                 v2i64x_info.ImmAllZerosV)),
             (VCVTPS2UQQZ128rmkz VK2WM:$mask, addr:$src)>;
 
   def : Pat<(v2i64 (X86any_cvttp2si (bc_v4f32 (v2f64 (X86vzload64 addr:$src))))),
             (VCVTTPS2QQZ128rm addr:$src)>;
-  def : Pat<(v2i64 (vselect VK2WM:$mask,
-                            (X86cvttp2si (bc_v4f32 (v2f64 (X86vzload64 addr:$src)))),
-                            VR128X:$src0)),
+  def : Pat<(v2i64 (vselect_mask VK2WM:$mask,
+                                 (X86cvttp2si (bc_v4f32 (v2f64 (X86vzload64 addr:$src)))),
+                                 VR128X:$src0)),
             (VCVTTPS2QQZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;
-  def : Pat<(v2i64 (vselect VK2WM:$mask,
-                            (X86cvttp2si (bc_v4f32 (v2f64 (X86vzload64 addr:$src)))),
-                            v2i64x_info.ImmAllZerosV)),
+  def : Pat<(v2i64 (vselect_mask VK2WM:$mask,
+                                 (X86cvttp2si (bc_v4f32 (v2f64 (X86vzload64 addr:$src)))),
+                                 v2i64x_info.ImmAllZerosV)),
             (VCVTTPS2QQZ128rmkz VK2WM:$mask, addr:$src)>;
 
   def : Pat<(v2i64 (X86any_cvttp2ui (bc_v4f32 (v2f64 (X86vzload64 addr:$src))))),
             (VCVTTPS2UQQZ128rm addr:$src)>;
-  def : Pat<(v2i64 (vselect VK2WM:$mask,
-                            (X86cvttp2ui (bc_v4f32 (v2f64 (X86vzload64 addr:$src)))),
-                            VR128X:$src0)),
+  def : Pat<(v2i64 (vselect_mask VK2WM:$mask,
+                                 (X86cvttp2ui (bc_v4f32 (v2f64 (X86vzload64 addr:$src)))),
+                                 VR128X:$src0)),
             (VCVTTPS2UQQZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;
-  def : Pat<(v2i64 (vselect VK2WM:$mask,
-                            (X86cvttp2ui (bc_v4f32 (v2f64 (X86vzload64 addr:$src)))),
-                            v2i64x_info.ImmAllZerosV)),
+  def : Pat<(v2i64 (vselect_mask VK2WM:$mask,
+                                 (X86cvttp2ui (bc_v4f32 (v2f64 (X86vzload64 addr:$src)))),
+                                 v2i64x_info.ImmAllZerosV)),
             (VCVTTPS2UQQZ128rmkz VK2WM:$mask, addr:$src)>;
 }
 
 let Predicates = [HasVLX] in {
   def : Pat<(v2f64 (X86any_VSintToFP (bc_v4i32 (v2i64 (X86vzload64 addr:$src))))),
             (VCVTDQ2PDZ128rm addr:$src)>;
-  def : Pat<(v2f64 (vselect VK2WM:$mask,
-                            (X86VSintToFP (bc_v4i32 (v2i64 (X86vzload64 addr:$src)))),
-                            VR128X:$src0)),
+  def : Pat<(v2f64 (vselect_mask VK2WM:$mask,
+                                 (X86VSintToFP (bc_v4i32 (v2i64 (X86vzload64 addr:$src)))),
+                                 VR128X:$src0)),
             (VCVTDQ2PDZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;
-  def : Pat<(v2f64 (vselect VK2WM:$mask,
-                            (X86VSintToFP (bc_v4i32 (v2i64 (X86vzload64 addr:$src)))),
-                            v2f64x_info.ImmAllZerosV)),
+  def : Pat<(v2f64 (vselect_mask VK2WM:$mask,
+                                 (X86VSintToFP (bc_v4i32 (v2i64 (X86vzload64 addr:$src)))),
+                                 v2f64x_info.ImmAllZerosV)),
             (VCVTDQ2PDZ128rmkz VK2WM:$mask, addr:$src)>;
 
   def : Pat<(v2f64 (X86any_VUintToFP (bc_v4i32 (v2i64 (X86vzload64 addr:$src))))),
             (VCVTUDQ2PDZ128rm addr:$src)>;
-  def : Pat<(v2f64 (vselect VK2WM:$mask,
-                            (X86VUintToFP (bc_v4i32 (v2i64 (X86vzload64 addr:$src)))),
-                            VR128X:$src0)),
+  def : Pat<(v2f64 (vselect_mask VK2WM:$mask,
+                                 (X86VUintToFP (bc_v4i32 (v2i64 (X86vzload64 addr:$src)))),
+                                 VR128X:$src0)),
             (VCVTUDQ2PDZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;
-  def : Pat<(v2f64 (vselect VK2WM:$mask,
-                            (X86VUintToFP (bc_v4i32 (v2i64 (X86vzload64 addr:$src)))),
-                            v2f64x_info.ImmAllZerosV)),
+  def : Pat<(v2f64 (vselect_mask VK2WM:$mask,
+                                 (X86VUintToFP (bc_v4i32 (v2i64 (X86vzload64 addr:$src)))),
+                                 v2f64x_info.ImmAllZerosV)),
             (VCVTUDQ2PDZ128rmkz VK2WM:$mask, addr:$src)>;
 }
 
@@ -9068,13 +9094,13 @@ multiclass avx512_masked_scalar<SDNode OpNode, string OpcPrefix, SDNode Move,
                                 dag Mask, X86VectorVTInfo _, PatLeaf ZeroFP,
                                 dag OutMask, Predicate BasePredicate> {
   let Predicates = [BasePredicate] in {
-    def : Pat<(Move _.VT:$src1, (scalar_to_vector (X86selects Mask,
+    def : Pat<(Move _.VT:$src1, (scalar_to_vector (X86selects_mask Mask,
                (OpNode (extractelt _.VT:$src2, (iPTR 0))),
                (extractelt _.VT:$dst, (iPTR 0))))),
               (!cast<Instruction>("V"#OpcPrefix#r_Intk)
                _.VT:$dst, OutMask, _.VT:$src2, _.VT:$src1)>;
 
-    def : Pat<(Move _.VT:$src1, (scalar_to_vector (X86selects Mask,
+    def : Pat<(Move _.VT:$src1, (scalar_to_vector (X86selects_mask Mask,
                (OpNode (extractelt _.VT:$src2, (iPTR 0))),
                ZeroFP))),
               (!cast<Instruction>("V"#OpcPrefix#r_Intkz)
@@ -9098,14 +9124,14 @@ defm : avx512_masked_scalar<fsqrt, "SQRTSDZ", X86Movsd,
 // same order as X86vmtrunc, X86vmtruncs, X86vmtruncus. This allows us to pass
 // either to the multiclasses.
 def select_trunc : PatFrag<(ops node:$src, node:$src0, node:$mask),
-                           (vselect node:$mask,
-                                    (trunc node:$src), node:$src0)>;
+                           (vselect_mask node:$mask,
+                                         (trunc node:$src), node:$src0)>;
 def select_truncs : PatFrag<(ops node:$src, node:$src0, node:$mask),
-                            (vselect node:$mask,
-                                     (X86vtruncs node:$src), node:$src0)>;
+                            (vselect_mask node:$mask,
+                                          (X86vtruncs node:$src), node:$src0)>;
 def select_truncus : PatFrag<(ops node:$src, node:$src0, node:$mask),
-                             (vselect node:$mask,
-                                      (X86vtruncus node:$src), node:$src0)>;
+                             (vselect_mask node:$mask,
+                                           (X86vtruncus node:$src), node:$src0)>;
 
 multiclass avx512_trunc_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
                               SDPatternOperator MaskNode,
@@ -10503,40 +10529,40 @@ def ValigndImm8XForm : SDNodeXForm<timm, [{
 multiclass avx512_vpalign_mask_lowering<string OpcodeStr, SDNode OpNode,
                                         X86VectorVTInfo From, X86VectorVTInfo To,
                                         SDNodeXForm ImmXForm> {
-  def : Pat<(To.VT (vselect To.KRCWM:$mask,
-                            (bitconvert
-                             (From.VT (OpNode From.RC:$src1, From.RC:$src2,
-                                              timm:$src3))),
-                            To.RC:$src0)),
+  def : Pat<(To.VT (vselect_mask To.KRCWM:$mask,
+                                 (bitconvert
+                                  (From.VT (OpNode From.RC:$src1, From.RC:$src2,
+                                                   timm:$src3))),
+                                 To.RC:$src0)),
             (!cast<Instruction>(OpcodeStr#"rrik") To.RC:$src0, To.KRCWM:$mask,
                                                   To.RC:$src1, To.RC:$src2,
                                                   (ImmXForm timm:$src3))>;
 
-  def : Pat<(To.VT (vselect To.KRCWM:$mask,
-                            (bitconvert
-                             (From.VT (OpNode From.RC:$src1, From.RC:$src2,
-                                              timm:$src3))),
-                            To.ImmAllZerosV)),
+  def : Pat<(To.VT (vselect_mask To.KRCWM:$mask,
+                                 (bitconvert
+                                  (From.VT (OpNode From.RC:$src1, From.RC:$src2,
+                                                   timm:$src3))),
+                                 To.ImmAllZerosV)),
             (!cast<Instruction>(OpcodeStr#"rrikz") To.KRCWM:$mask,
                                                    To.RC:$src1, To.RC:$src2,
                                                    (ImmXForm timm:$src3))>;
 
-  def : Pat<(To.VT (vselect To.KRCWM:$mask,
-                            (bitconvert
-                             (From.VT (OpNode From.RC:$src1,
-                                              (From.LdFrag addr:$src2),
-                                      timm:$src3))),
-                            To.RC:$src0)),
+  def : Pat<(To.VT (vselect_mask To.KRCWM:$mask,
+                                 (bitconvert
+                                  (From.VT (OpNode From.RC:$src1,
+                                                   (From.LdFrag addr:$src2),
+                                           timm:$src3))),
+                                 To.RC:$src0)),
             (!cast<Instruction>(OpcodeStr#"rmik") To.RC:$src0, To.KRCWM:$mask,
                                                   To.RC:$src1, addr:$src2,
                                                   (ImmXForm timm:$src3))>;
 
-  def : Pat<(To.VT (vselect To.KRCWM:$mask,
-                            (bitconvert
-                             (From.VT (OpNode From.RC:$src1,
-                                              (From.LdFrag addr:$src2),
-                                      timm:$src3))),
-                            To.ImmAllZerosV)),
+  def : Pat<(To.VT (vselect_mask To.KRCWM:$mask,
+                                 (bitconvert
+                                  (From.VT (OpNode From.RC:$src1,
+                                                   (From.LdFrag addr:$src2),
+                                           timm:$src3))),
+                                 To.ImmAllZerosV)),
             (!cast<Instruction>(OpcodeStr#"rmikz") To.KRCWM:$mask,
                                                    To.RC:$src1, addr:$src2,
                                                    (ImmXForm timm:$src3))>;
@@ -10553,24 +10579,24 @@ multiclass avx512_vpalign_mask_lowering_mb<string OpcodeStr, SDNode OpNode,
             (!cast<Instruction>(OpcodeStr#"rmbi") To.RC:$src1, addr:$src2,
                                                   (ImmXForm timm:$src3))>;
 
-  def : Pat<(To.VT (vselect To.KRCWM:$mask,
-                            (bitconvert
-                             (From.VT (OpNode From.RC:$src1,
-                                      (bitconvert
-                                       (To.VT (To.BroadcastLdFrag addr:$src2))),
-                                      timm:$src3))),
-                            To.RC:$src0)),
+  def : Pat<(To.VT (vselect_mask To.KRCWM:$mask,
+                                 (bitconvert
+                                  (From.VT (OpNode From.RC:$src1,
+                                           (bitconvert
+                                            (To.VT (To.BroadcastLdFrag addr:$src2))),
+                                           timm:$src3))),
+                                 To.RC:$src0)),
             (!cast<Instruction>(OpcodeStr#"rmbik") To.RC:$src0, To.KRCWM:$mask,
                                                    To.RC:$src1, addr:$src2,
                                                    (ImmXForm timm:$src3))>;
 
-  def : Pat<(To.VT (vselect To.KRCWM:$mask,
-                            (bitconvert
-                             (From.VT (OpNode From.RC:$src1,
-                                      (bitconvert
-                                       (To.VT (To.BroadcastLdFrag addr:$src2))),
-                                      timm:$src3))),
-                            To.ImmAllZerosV)),
+  def : Pat<(To.VT (vselect_mask To.KRCWM:$mask,
+                                 (bitconvert
+                                  (From.VT (OpNode From.RC:$src1,
+                                           (bitconvert
+                                            (To.VT (To.BroadcastLdFrag addr:$src2))),
+                                           timm:$src3))),
+                                 To.ImmAllZerosV)),
             (!cast<Instruction>(OpcodeStr#"rmbikz") To.KRCWM:$mask,
                                                     To.RC:$src1, addr:$src2,
                                                     (ImmXForm timm:$src3))>;
@@ -10814,19 +10840,19 @@ let Predicates = [HasVLX] in {
 def : Pat<(v2f64 (X86VBroadcast f64:$src)),
           (VMOVDDUPZ128rr (v2f64 (COPY_TO_REGCLASS FR64X:$src, VR128X)))>;
 
-def : Pat<(vselect (v2i1 VK2WM:$mask), (v2f64 (X86VBroadcast f64:$src)),
-                   (v2f64 VR128X:$src0)),
+def : Pat<(vselect_mask (v2i1 VK2WM:$mask), (v2f64 (X86VBroadcast f64:$src)),
+                        (v2f64 VR128X:$src0)),
           (VMOVDDUPZ128rrk VR128X:$src0, VK2WM:$mask,
                            (v2f64 (COPY_TO_REGCLASS FR64X:$src, VR128X)))>;
-def : Pat<(vselect (v2i1 VK2WM:$mask), (v2f64 (X86VBroadcast f64:$src)),
-                   immAllZerosV),
+def : Pat<(vselect_mask (v2i1 VK2WM:$mask), (v2f64 (X86VBroadcast f64:$src)),
+                        immAllZerosV),
           (VMOVDDUPZ128rrkz VK2WM:$mask, (v2f64 (COPY_TO_REGCLASS FR64X:$src, VR128X)))>;
 
-def : Pat<(vselect (v2i1 VK2WM:$mask), (v2f64 (X86VBroadcastld64 addr:$src)),
-                   (v2f64 VR128X:$src0)),
+def : Pat<(vselect_mask (v2i1 VK2WM:$mask), (v2f64 (X86VBroadcastld64 addr:$src)),
+                        (v2f64 VR128X:$src0)),
           (VMOVDDUPZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;
-def : Pat<(vselect (v2i1 VK2WM:$mask), (v2f64 (X86VBroadcastld64 addr:$src)),
-                   immAllZerosV),
+def : Pat<(vselect_mask (v2i1 VK2WM:$mask), (v2f64 (X86VBroadcastld64 addr:$src)),
+                        immAllZerosV),
           (VMOVDDUPZ128rmkz VK2WM:$mask, addr:$src)>;
 }
 
@@ -11167,12 +11193,12 @@ multiclass avx512_ternlog<bits<8> opc, string OpcodeStr, SDNode OpNode,
   }// Constraints = "$src1 = $dst"
 
   // Additional patterns for matching passthru operand in other positions.
-  def : Pat<(_.VT (vselect _.KRCWM:$mask,
+  def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,
                    (OpNode _.RC:$src3, _.RC:$src2, _.RC:$src1, (i8 timm:$src4)),
                    _.RC:$src1)),
             (!cast<Instruction>(Name#_.ZSuffix#rrik) _.RC:$src1, _.KRCWM:$mask,
              _.RC:$src2, _.RC:$src3, (VPTERNLOG321_imm8 timm:$src4))>;
-  def : Pat<(_.VT (vselect _.KRCWM:$mask,
+  def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,
                    (OpNode _.RC:$src2, _.RC:$src1, _.RC:$src3, (i8 timm:$src4)),
                    _.RC:$src1)),
             (!cast<Instruction>(Name#_.ZSuffix#rrik) _.RC:$src1, _.KRCWM:$mask,
@@ -11191,13 +11217,13 @@ multiclass avx512_ternlog<bits<8> opc, string OpcodeStr, SDNode OpNode,
 
   // Additional patterns for matching zero masking with loads in other
   // positions.
-  def : Pat<(_.VT (vselect _.KRCWM:$mask,
+  def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,
                    (OpNode (bitconvert (_.LdFrag addr:$src3)),
                     _.RC:$src2, _.RC:$src1, (i8 timm:$src4)),
                    _.ImmAllZerosV)),
             (!cast<Instruction>(Name#_.ZSuffix#rmikz) _.RC:$src1, _.KRCWM:$mask,
              _.RC:$src2, addr:$src3, (VPTERNLOG321_imm8 timm:$src4))>;
-  def : Pat<(_.VT (vselect _.KRCWM:$mask,
+  def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,
                    (OpNode _.RC:$src1, (bitconvert (_.LdFrag addr:$src3)),
                     _.RC:$src2, (i8 timm:$src4)),
                    _.ImmAllZerosV)),
@@ -11206,31 +11232,31 @@ multiclass avx512_ternlog<bits<8> opc, string OpcodeStr, SDNode OpNode,
 
   // Additional patterns for matching masked loads with different
   // operand orders.
-  def : Pat<(_.VT (vselect _.KRCWM:$mask,
+  def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,
                    (OpNode _.RC:$src1, (bitconvert (_.LdFrag addr:$src3)),
                     _.RC:$src2, (i8 timm:$src4)),
                    _.RC:$src1)),
             (!cast<Instruction>(Name#_.ZSuffix#rmik) _.RC:$src1, _.KRCWM:$mask,
              _.RC:$src2, addr:$src3, (VPTERNLOG132_imm8 timm:$src4))>;
-  def : Pat<(_.VT (vselect _.KRCWM:$mask,
+  def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,
                    (OpNode (bitconvert (_.LdFrag addr:$src3)),
                     _.RC:$src2, _.RC:$src1, (i8 timm:$src4)),
                    _.RC:$src1)),
             (!cast<Instruction>(Name#_.ZSuffix#rmik) _.RC:$src1, _.KRCWM:$mask,
              _.RC:$src2, addr:$src3, (VPTERNLOG321_imm8 timm:$src4))>;
-  def : Pat<(_.VT (vselect _.KRCWM:$mask,
+  def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,
                    (OpNode _.RC:$src2, _.RC:$src1,
                     (bitconvert (_.LdFrag addr:$src3)), (i8 timm:$src4)),
                    _.RC:$src1)),
             (!cast<Instruction>(Name#_.ZSuffix#rmik) _.RC:$src1, _.KRCWM:$mask,
              _.RC:$src2, addr:$src3, (VPTERNLOG213_imm8 timm:$src4))>;
-  def : Pat<(_.VT (vselect _.KRCWM:$mask,
+  def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,
                    (OpNode _.RC:$src2, (bitconvert (_.LdFrag addr:$src3)),
                     _.RC:$src1, (i8 timm:$src4)),
                    _.RC:$src1)),
             (!cast<Instruction>(Name#_.ZSuffix#rmik) _.RC:$src1, _.KRCWM:$mask,
              _.RC:$src2, addr:$src3, (VPTERNLOG231_imm8 timm:$src4))>;
-  def : Pat<(_.VT (vselect _.KRCWM:$mask,
+  def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,
                    (OpNode (bitconvert (_.LdFrag addr:$src3)),
                     _.RC:$src1, _.RC:$src2, (i8 timm:$src4)),
                    _.RC:$src1)),
@@ -11250,14 +11276,14 @@ multiclass avx512_ternlog<bits<8> opc, string OpcodeStr, SDNode OpNode,
 
   // Additional patterns for matching zero masking with broadcasts in other
   // positions.
-  def : Pat<(_.VT (vselect _.KRCWM:$mask,
+  def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,
                    (OpNode (_.BroadcastLdFrag addr:$src3),
                     _.RC:$src2, _.RC:$src1, (i8 timm:$src4)),
                    _.ImmAllZerosV)),
             (!cast<Instruction>(Name#_.ZSuffix#rmbikz) _.RC:$src1,
              _.KRCWM:$mask, _.RC:$src2, addr:$src3,
              (VPTERNLOG321_imm8 timm:$src4))>;
-  def : Pat<(_.VT (vselect _.KRCWM:$mask,
+  def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,
                    (OpNode _.RC:$src1,
                     (_.BroadcastLdFrag addr:$src3),
                     _.RC:$src2, (i8 timm:$src4)),
@@ -11268,32 +11294,32 @@ multiclass avx512_ternlog<bits<8> opc, string OpcodeStr, SDNode OpNode,
 
   // Additional patterns for matching masked broadcasts with different
   // operand orders.
-  def : Pat<(_.VT (vselect _.KRCWM:$mask,
+  def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,
                    (OpNode _.RC:$src1, (_.BroadcastLdFrag addr:$src3),
                     _.RC:$src2, (i8 timm:$src4)),
                    _.RC:$src1)),
             (!cast<Instruction>(Name#_.ZSuffix#rmbik) _.RC:$src1, _.KRCWM:$mask,
              _.RC:$src2, addr:$src3, (VPTERNLOG132_imm8 timm:$src4))>;
-  def : Pat<(_.VT (vselect _.KRCWM:$mask,
+  def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,
                    (OpNode (_.BroadcastLdFrag addr:$src3),
                     _.RC:$src2, _.RC:$src1, (i8 timm:$src4)),
                    _.RC:$src1)),
             (!cast<Instruction>(Name#_.ZSuffix#rmbik) _.RC:$src1, _.KRCWM:$mask,
              _.RC:$src2, addr:$src3, (VPTERNLOG321_imm8 timm:$src4))>;
-  def : Pat<(_.VT (vselect _.KRCWM:$mask,
+  def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,
                    (OpNode _.RC:$src2, _.RC:$src1,
                     (_.BroadcastLdFrag addr:$src3),
                     (i8 timm:$src4)), _.RC:$src1)),
             (!cast<Instruction>(Name#_.ZSuffix#rmbik) _.RC:$src1, _.KRCWM:$mask,
              _.RC:$src2, addr:$src3, (VPTERNLOG213_imm8 timm:$src4))>;
-  def : Pat<(_.VT (vselect _.KRCWM:$mask,
+  def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,
                    (OpNode _.RC:$src2,
                     (_.BroadcastLdFrag addr:$src3),
                     _.RC:$src1, (i8 timm:$src4)),
                    _.RC:$src1)),
             (!cast<Instruction>(Name#_.ZSuffix#rmbik) _.RC:$src1, _.KRCWM:$mask,
              _.RC:$src2, addr:$src3, (VPTERNLOG231_imm8 timm:$src4))>;
-  def : Pat<(_.VT (vselect _.KRCWM:$mask,
+  def : Pat<(_.VT (vselect_mask _.KRCWM:$mask,
                    (OpNode (_.BroadcastLdFrag addr:$src3),
                     _.RC:$src1, _.RC:$src2, (i8 timm:$src4)),
                    _.RC:$src1)),
@@ -11702,7 +11728,7 @@ multiclass AVX512_scalar_math_fp_patterns<SDNode Op, SDNode MaskedOp,
     // extracted masked scalar math op with insert via movss
     def : Pat<(MoveNode (_.VT VR128X:$src1),
                (scalar_to_vector
-                (X86selects VK1WM:$mask,
+                (X86selects_mask VK1WM:$mask,
                             (MaskedOp (_.EltVT
                                        (extractelt (_.VT VR128X:$src1), (iPTR 0))),
                                       _.FRC:$src2),
@@ -11713,7 +11739,7 @@ multiclass AVX512_scalar_math_fp_patterns<SDNode Op, SDNode MaskedOp,
                (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)))>;
     def : Pat<(MoveNode (_.VT VR128X:$src1),
                (scalar_to_vector
-                (X86selects VK1WM:$mask,
+                (X86selects_mask VK1WM:$mask,
                             (MaskedOp (_.EltVT
                                        (extractelt (_.VT VR128X:$src1), (iPTR 0))),
                                       (_.ScalarLdFrag addr:$src2)),
@@ -11725,7 +11751,7 @@ multiclass AVX512_scalar_math_fp_patterns<SDNode Op, SDNode MaskedOp,
     // extracted masked scalar math op with insert via movss
     def : Pat<(MoveNode (_.VT VR128X:$src1),
                (scalar_to_vector
-                (X86selects VK1WM:$mask,
+                (X86selects_mask VK1WM:$mask,
                             (MaskedOp (_.EltVT
                                        (extractelt (_.VT VR128X:$src1), (iPTR 0))),
                                       _.FRC:$src2), (_.EltVT ZeroFP)))),
@@ -11734,7 +11760,7 @@ multiclass AVX512_scalar_math_fp_patterns<SDNode Op, SDNode MaskedOp,
           (_.VT (COPY_TO_REGCLASS _.FRC:$src2, VR128X)))>;
     def : Pat<(MoveNode (_.VT VR128X:$src1),
                (scalar_to_vector
-                (X86selects VK1WM:$mask,
+                (X86selects_mask VK1WM:$mask,
                             (MaskedOp (_.EltVT
                                        (extractelt (_.VT VR128X:$src1), (iPTR 0))),
                                       (_.ScalarLdFrag addr:$src2)), (_.EltVT ZeroFP)))),
diff --git a/llvm/lib/Target/X86/X86InstrBuilder.h b/llvm/lib/Target/X86/X86InstrBuilder.h
index aa45e9b191c1c..07079ef87fd46 100644
--- a/llvm/lib/Target/X86/X86InstrBuilder.h
+++ b/llvm/lib/Target/X86/X86InstrBuilder.h
@@ -207,7 +207,7 @@ addFrameReference(const MachineInstrBuilder &MIB, int FI, int Offset = 0) {
     Flags |= MachineMemOperand::MOStore;
   MachineMemOperand *MMO = MF.getMachineMemOperand(
       MachinePointerInfo::getFixedStack(MF, FI, Offset), Flags,
-      MFI.getObjectSize(FI), MFI.getObjectAlignment(FI));
+      MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
   return addOffset(MIB.addFrameIndex(FI), Offset)
             .addMemOperand(MMO);
 }
diff --git a/llvm/lib/Target/X86/X86InstrCompiler.td b/llvm/lib/Target/X86/X86InstrCompiler.td
index 0ad53c42550ef..0b1e66285dde0 100644
--- a/llvm/lib/Target/X86/X86InstrCompiler.td
+++ b/llvm/lib/Target/X86/X86InstrCompiler.td
@@ -1180,14 +1180,14 @@ def X86tcret_6regs : PatFrag<(ops node:$ptr, node:$off),
 
 def : Pat<(X86tcret ptr_rc_tailcall:$dst, imm:$off),
           (TCRETURNri ptr_rc_tailcall:$dst, imm:$off)>,
-          Requires<[Not64BitMode, NotUseRetpolineIndirectCalls]>;
+          Requires<[Not64BitMode, NotUseIndirectThunkCalls]>;
 
 // FIXME: This is disabled for 32-bit PIC mode because the global base
 // register which is part of the address mode may be assigned a
 // callee-saved register.
 def : Pat<(X86tcret (load addr:$dst), imm:$off),
           (TCRETURNmi addr:$dst, imm:$off)>,
-          Requires<[Not64BitMode, IsNotPIC, NotUseRetpolineIndirectCalls]>;
+          Requires<[Not64BitMode, IsNotPIC, NotUseIndirectThunkCalls]>;
 
 def : Pat<(X86tcret (i32 tglobaladdr:$dst), imm:$off),
           (TCRETURNdi tglobaladdr:$dst, imm:$off)>,
@@ -1199,21 +1199,21 @@ def : Pat<(X86tcret (i32 texternalsym:$dst), imm:$off),
 
 def : Pat<(X86tcret ptr_rc_tailcall:$dst, imm:$off),
           (TCRETURNri64 ptr_rc_tailcall:$dst, imm:$off)>,
-          Requires<[In64BitMode, NotUseRetpolineIndirectCalls]>;
+          Requires<[In64BitMode, NotUseIndirectThunkCalls]>;
 
 // Don't fold loads into X86tcret requiring more than 6 regs.
 // There wouldn't be enough scratch registers for base+index.
 def : Pat<(X86tcret_6regs (load addr:$dst), imm:$off),
           (TCRETURNmi64 addr:$dst, imm:$off)>,
-          Requires<[In64BitMode, NotUseRetpolineIndirectCalls]>;
+          Requires<[In64BitMode, NotUseIndirectThunkCalls]>;
 
 def : Pat<(X86tcret ptr_rc_tailcall:$dst, imm:$off),
-          (RETPOLINE_TCRETURN64 ptr_rc_tailcall:$dst, imm:$off)>,
-          Requires<[In64BitMode, UseRetpolineIndirectCalls]>;
+          (INDIRECT_THUNK_TCRETURN64 ptr_rc_tailcall:$dst, imm:$off)>,
+          Requires<[In64BitMode, UseIndirectThunkCalls]>;
 
 def : Pat<(X86tcret ptr_rc_tailcall:$dst, imm:$off),
-          (RETPOLINE_TCRETURN32 ptr_rc_tailcall:$dst, imm:$off)>,
-          Requires<[Not64BitMode, UseRetpolineIndirectCalls]>;
+          (INDIRECT_THUNK_TCRETURN32 ptr_rc_tailcall:$dst, imm:$off)>,
+          Requires<[Not64BitMode, UseIndirectThunkCalls]>;
 
 def : Pat<(X86tcret (i64 tglobaladdr:$dst), imm:$off),
           (TCRETURNdi64 tglobaladdr:$dst, imm:$off)>,
diff --git a/llvm/lib/Target/X86/X86InstrControl.td b/llvm/lib/Target/X86/X86InstrControl.td
index 32faeb1a86f2b..1842dc19ec2e1 100644
--- a/llvm/lib/Target/X86/X86InstrControl.td
+++ b/llvm/lib/Target/X86/X86InstrControl.td
@@ -237,13 +237,13 @@ let isCall = 1 in
                         Sched<[WriteJumpLd]>;
     def CALL32r     : I<0xFF, MRM2r, (outs), (ins GR32:$dst),
                         "call{l}\t{*}$dst", [(X86call GR32:$dst)]>, OpSize32,
-                        Requires<[Not64BitMode,NotUseRetpolineIndirectCalls]>,
+                        Requires<[Not64BitMode,NotUseIndirectThunkCalls]>,
                         Sched<[WriteJump]>;
     def CALL32m     : I<0xFF, MRM2m, (outs), (ins i32mem:$dst),
                         "call{l}\t{*}$dst", [(X86call (loadi32 addr:$dst))]>,
                         OpSize32,
                         Requires<[Not64BitMode,FavorMemIndirectCall,
-                                  NotUseRetpolineIndirectCalls]>,
+                                  NotUseIndirectThunkCalls]>,
                         Sched<[WriteJumpLd]>;
 
     // Non-tracking calls for IBT, use with caution.
@@ -334,11 +334,11 @@ let isCall = 1, Uses = [RSP, SSP], SchedRW = [WriteJump] in {
                       Requires<[In64BitMode]>;
   def CALL64r       : I<0xFF, MRM2r, (outs), (ins GR64:$dst),
                         "call{q}\t{*}$dst", [(X86call GR64:$dst)]>,
-                      Requires<[In64BitMode,NotUseRetpolineIndirectCalls]>;
+                      Requires<[In64BitMode,NotUseIndirectThunkCalls]>;
   def CALL64m       : I<0xFF, MRM2m, (outs), (ins i64mem:$dst),
                         "call{q}\t{*}$dst", [(X86call (loadi64 addr:$dst))]>,
                       Requires<[In64BitMode,FavorMemIndirectCall,
-                                NotUseRetpolineIndirectCalls]>;
+                                NotUseIndirectThunkCalls]>;
 
   // Non-tracking calls for IBT, use with caution.
   let isCodeGenOnly = 1 in {
@@ -393,19 +393,19 @@ let isPseudo = 1, isCall = 1, isCodeGenOnly = 1,
     Uses = [RSP, SSP],
     usesCustomInserter = 1,
     SchedRW = [WriteJump] in {
-  def RETPOLINE_CALL32 :
+  def INDIRECT_THUNK_CALL32 :
     PseudoI<(outs), (ins GR32:$dst), [(X86call GR32:$dst)]>,
-            Requires<[Not64BitMode,UseRetpolineIndirectCalls]>;
+            Requires<[Not64BitMode,UseIndirectThunkCalls]>;
 
-  def RETPOLINE_CALL64 :
+  def INDIRECT_THUNK_CALL64 :
     PseudoI<(outs), (ins GR64:$dst), [(X86call GR64:$dst)]>,
-            Requires<[In64BitMode,UseRetpolineIndirectCalls]>;
+            Requires<[In64BitMode,UseIndirectThunkCalls]>;
 
-  // Retpoline variant of indirect tail calls.
+  // Indirect thunk variant of indirect tail calls.
   let isTerminator = 1, isReturn = 1, isBarrier = 1 in {
-    def RETPOLINE_TCRETURN64 :
+    def INDIRECT_THUNK_TCRETURN64 :
       PseudoI<(outs), (ins GR64:$dst, i32imm:$offset), []>;
-    def RETPOLINE_TCRETURN32 :
+    def INDIRECT_THUNK_TCRETURN32 :
       PseudoI<(outs), (ins GR32:$dst, i32imm:$offset), []>;
   }
 }
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 3bfad3d190f58..0197630a2670e 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -88,7 +88,7 @@ X86InstrInfo::X86InstrInfo(X86Subtarget &STI)
 
 bool
 X86InstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
-                                    unsigned &SrcReg, unsigned &DstReg,
+                                    Register &SrcReg, Register &DstReg,
                                     unsigned &SubIdx) const {
   switch (MI.getOpcode()) {
   default: break;
@@ -1123,7 +1123,7 @@ bool X86InstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI,
 
 void X86InstrInfo::reMaterialize(MachineBasicBlock &MBB,
                                  MachineBasicBlock::iterator I,
-                                 unsigned DestReg, unsigned SubIdx,
+                                 Register DestReg, unsigned SubIdx,
                                  const MachineInstr &Orig,
                                  const TargetRegisterInfo &TRI) const {
   bool ClobbersEFLAGS = Orig.modifiesRegister(X86::EFLAGS, &TRI);
@@ -3312,8 +3312,8 @@ unsigned X86InstrInfo::insertBranch(MachineBasicBlock &MBB,
 
 bool X86InstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
                                    ArrayRef<MachineOperand> Cond,
-                                   unsigned DstReg, unsigned TrueReg,
-                                   unsigned FalseReg, int &CondCycles,
+                                   Register DstReg, Register TrueReg,
+                                   Register FalseReg, int &CondCycles,
                                    int &TrueCycles, int &FalseCycles) const {
   // Not all subtargets have cmov instructions.
   if (!Subtarget.hasCMov())
@@ -3349,9 +3349,9 @@ bool X86InstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
 
 void X86InstrInfo::insertSelect(MachineBasicBlock &MBB,
                                 MachineBasicBlock::iterator I,
-                                const DebugLoc &DL, unsigned DstReg,
-                                ArrayRef<MachineOperand> Cond, unsigned TrueReg,
-                                unsigned FalseReg) const {
+                                const DebugLoc &DL, Register DstReg,
+                                ArrayRef<MachineOperand> Cond, Register TrueReg,
+                                Register FalseReg) const {
   MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
   const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
   const TargetRegisterClass &RC = *MRI.getRegClass(DstReg);
@@ -3757,8 +3757,8 @@ void X86InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
   addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc), DestReg), FrameIdx);
 }
 
-bool X86InstrInfo::analyzeCompare(const MachineInstr &MI, unsigned &SrcReg,
-                                  unsigned &SrcReg2, int &CmpMask,
+bool X86InstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
+                                  Register &SrcReg2, int &CmpMask,
                                   int &CmpValue) const {
   switch (MI.getOpcode()) {
   default: break;
@@ -3845,7 +3845,7 @@ bool X86InstrInfo::analyzeCompare(const MachineInstr &MI, unsigned &SrcReg,
 /// SrcReg, SrcRegs: register operands for FlagI.
 /// ImmValue: immediate for FlagI if it takes an immediate.
 inline static bool isRedundantFlagInstr(const MachineInstr &FlagI,
-                                        unsigned SrcReg, unsigned SrcReg2,
+                                        Register SrcReg, Register SrcReg2,
                                         int ImmMask, int ImmValue,
                                         const MachineInstr &OI) {
   if (((FlagI.getOpcode() == X86::CMP64rr && OI.getOpcode() == X86::SUB64rr) ||
@@ -4034,8 +4034,8 @@ static X86::CondCode isUseDefConvertible(const MachineInstr &MI) {
 /// Check if there exists an earlier instruction that
 /// operates on the same source operands and sets flags in the same way as
 /// Compare; remove Compare if possible.
-bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, unsigned SrcReg,
-                                        unsigned SrcReg2, int CmpMask,
+bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
+                                        Register SrcReg2, int CmpMask,
                                         int CmpValue,
                                         const MachineRegisterInfo *MRI) const {
   // Check whether we can replace SUB with CMP.
@@ -4443,6 +4443,8 @@ static bool ExpandMOVImmSExti8(MachineInstrBuilder &MIB,
     BuildMI(MBB, I, DL, TII.get(X86::PUSH32i8)).addImm(Imm);
     MIB->setDesc(TII.get(X86::POP32r));
   }
+  MIB->RemoveOperand(1);
+  MIB->addImplicitDefUseOperands(*MBB.getParent());
 
   // Build CFI if necessary.
   MachineFunction &MF = *MBB.getParent();
@@ -4473,7 +4475,7 @@ static void expandLoadStackGuard(MachineInstrBuilder &MIB,
                MachineMemOperand::MODereferenceable |
                MachineMemOperand::MOInvariant;
   MachineMemOperand *MMO = MBB.getParent()->getMachineMemOperand(
-      MachinePointerInfo::getGOT(*MBB.getParent()), Flags, 8, 8);
+      MachinePointerInfo::getGOT(*MBB.getParent()), Flags, 8, Align(8));
   MachineBasicBlock::iterator I = MIB.getInstr();
 
   BuildMI(MBB, I, DL, TII.get(X86::MOV64rm), Reg).addReg(X86::RIP).addImm(1)
@@ -5811,7 +5813,7 @@ MachineInstr *X86InstrInfo::foldMemoryOperandImpl(
   // Determine the alignment of the load.
   Align Alignment;
   if (LoadMI.hasOneMemOperand())
-    Alignment = Align((*LoadMI.memoperands_begin())->getAlignment());
+    Alignment = (*LoadMI.memoperands_begin())->getAlign();
   else
     switch (LoadMI.getOpcode()) {
     case X86::AVX512_512_SET0:
@@ -6092,7 +6094,7 @@ bool X86InstrInfo::unfoldMemoryOperand(
       Opc = getBroadcastOpcode(I, RC, Subtarget);
     } else {
       unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*RC), 16);
-      bool isAligned = !MMOs.empty() && MMOs.front()->getAlignment() >= Alignment;
+      bool isAligned = !MMOs.empty() && MMOs.front()->getAlign() >= Alignment;
       Opc = getLoadRegOpcode(Reg, RC, isAligned, Subtarget);
     }
 
@@ -6169,7 +6171,7 @@ bool X86InstrInfo::unfoldMemoryOperand(
     const TargetRegisterClass *DstRC = getRegClass(MCID, 0, &RI, MF);
     auto MMOs = extractStoreMMOs(MI.memoperands(), MF);
     unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*DstRC), 16);
-    bool isAligned = !MMOs.empty() && MMOs.front()->getAlignment() >= Alignment;
+    bool isAligned = !MMOs.empty() && MMOs.front()->getAlign() >= Alignment;
     unsigned Opc = getStoreRegOpcode(Reg, DstRC, isAligned, Subtarget);
     DebugLoc DL;
     MachineInstrBuilder MIB = BuildMI(MF, DL, get(Opc));
@@ -6236,7 +6238,7 @@ X86InstrInfo::unfoldMemoryOperand(SelectionDAG &DAG, SDNode *N,
       Opc = getBroadcastOpcode(I, RC, Subtarget);
     } else {
       unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*RC), 16);
-      bool isAligned = !MMOs.empty() && MMOs.front()->getAlignment() >= Alignment;
+      bool isAligned = !MMOs.empty() && MMOs.front()->getAlign() >= Alignment;
       Opc = getLoadRegOpcode(0, RC, isAligned, Subtarget);
     }
 
@@ -6302,7 +6304,7 @@ X86InstrInfo::unfoldMemoryOperand(SelectionDAG &DAG, SDNode *N,
     // FIXME: If a VR128 can have size 32, we should be checking if a 32-byte
     // memory access is slow above.
     unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*RC), 16);
-    bool isAligned = !MMOs.empty() && MMOs.front()->getAlignment() >= Alignment;
+    bool isAligned = !MMOs.empty() && MMOs.front()->getAlign() >= Alignment;
     SDNode *Store =
         DAG.getMachineNode(getStoreRegOpcode(0, DstRC, isAligned, Subtarget),
                            dl, MVT::Other, AddrOps);
diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h
index d303ea8c7bd69..7622484b0c204 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.h
+++ b/llvm/lib/Target/X86/X86InstrInfo.h
@@ -180,8 +180,8 @@ class X86InstrInfo final : public X86GenInstrInfo {
   /// true, then it's expected the pre-extension value is available as a subreg
   /// of the result register. This also returns the sub-register index in
   /// SubIdx.
-  bool isCoalescableExtInstr(const MachineInstr &MI, unsigned &SrcReg,
-                             unsigned &DstReg, unsigned &SubIdx) const override;
+  bool isCoalescableExtInstr(const MachineInstr &MI, Register &SrcReg,
+                             Register &DstReg, unsigned &SubIdx) const override;
 
   /// Returns true if the instruction has no behavior (specified or otherwise)
   /// that is based on the value of any of its register operands
@@ -237,7 +237,7 @@ class X86InstrInfo final : public X86GenInstrInfo {
   bool isReallyTriviallyReMaterializable(const MachineInstr &MI,
                                          AAResults *AA) const override;
   void reMaterialize(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
-                     unsigned DestReg, unsigned SubIdx,
+                     Register DestReg, unsigned SubIdx,
                      const MachineInstr &Orig,
                      const TargetRegisterInfo &TRI) const override;
 
@@ -336,12 +336,12 @@ class X86InstrInfo final : public X86GenInstrInfo {
                         const DebugLoc &DL,
                         int *BytesAdded = nullptr) const override;
   bool canInsertSelect(const MachineBasicBlock &, ArrayRef<MachineOperand> Cond,
-                       unsigned, unsigned, unsigned, int &, int &,
+                       Register, Register, Register, int &, int &,
                        int &) const override;
   void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
-                    const DebugLoc &DL, unsigned DstReg,
-                    ArrayRef<MachineOperand> Cond, unsigned TrueReg,
-                    unsigned FalseReg) const override;
+                    const DebugLoc &DL, Register DstReg,
+                    ArrayRef<MachineOperand> Cond, Register TrueReg,
+                    Register FalseReg) const override;
   void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
                    const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg,
                    bool KillSrc) const override;
@@ -500,15 +500,15 @@ class X86InstrInfo final : public X86GenInstrInfo {
   /// in SrcReg and SrcReg2 if having two register operands, and the value it
   /// compares against in CmpValue. Return true if the comparison instruction
   /// can be analyzed.
-  bool analyzeCompare(const MachineInstr &MI, unsigned &SrcReg,
-                      unsigned &SrcReg2, int &CmpMask,
+  bool analyzeCompare(const MachineInstr &MI, Register &SrcReg,
+                      Register &SrcReg2, int &CmpMask,
                       int &CmpValue) const override;
 
   /// optimizeCompareInstr - Check if there exists an earlier instruction that
   /// operates on the same source operands and sets flags in the same way as
   /// Compare; remove Compare if possible.
-  bool optimizeCompareInstr(MachineInstr &CmpInstr, unsigned SrcReg,
-                            unsigned SrcReg2, int CmpMask, int CmpValue,
+  bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
+                            Register SrcReg2, int CmpMask, int CmpValue,
                             const MachineRegisterInfo *MRI) const override;
 
   /// optimizeLoadInstr - Try to remove the load by folding it to a register
diff --git a/llvm/lib/Target/X86/X86InstrInfo.td b/llvm/lib/Target/X86/X86InstrInfo.td
index 6940510609a87..4e2bc02dcdd34 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.td
+++ b/llvm/lib/Target/X86/X86InstrInfo.td
@@ -955,6 +955,7 @@ def HasCmpxchg8b : Predicate<"Subtarget->hasCmpxchg8b()">;
 def HasCmpxchg16b: Predicate<"Subtarget->hasCmpxchg16b()">;
 def HasPCONFIG   : Predicate<"Subtarget->hasPCONFIG()">;
 def HasENQCMD    : Predicate<"Subtarget->hasENQCMD()">;
+def HasSERIALIZE : Predicate<"Subtarget->hasSERIALIZE()">;
 def Not64BitMode : Predicate<"!Subtarget->is64Bit()">,
                              AssemblerPredicate<(all_of (not Mode64Bit)), "Not 64-bit mode">;
 def In64BitMode  : Predicate<"Subtarget->is64Bit()">,
@@ -1003,8 +1004,8 @@ def HasFastLZCNT : Predicate<"Subtarget->hasFastLZCNT()">;
 def HasFastSHLDRotate : Predicate<"Subtarget->hasFastSHLDRotate()">;
 def HasERMSB : Predicate<"Subtarget->hasERMSB()">;
 def HasMFence    : Predicate<"Subtarget->hasMFence()">;
-def UseRetpolineIndirectCalls : Predicate<"Subtarget->useRetpolineIndirectCalls()">;
-def NotUseRetpolineIndirectCalls : Predicate<"!Subtarget->useRetpolineIndirectCalls()">;
+def UseIndirectThunkCalls : Predicate<"Subtarget->useIndirectThunkCalls()">;
+def NotUseIndirectThunkCalls : Predicate<"!Subtarget->useIndirectThunkCalls()">;
 
 //===----------------------------------------------------------------------===//
 // X86 Instruction Format Definitions.
@@ -2861,6 +2862,13 @@ let SchedRW = [WriteLoad] in {
 def : InstAlias<"clzero\t{%eax|eax}", (CLZERO32r)>, Requires<[Not64BitMode]>;
 def : InstAlias<"clzero\t{%rax|rax}", (CLZERO64r)>, Requires<[In64BitMode]>;
 
+//===----------------------------------------------------------------------===//
+// SERIALIZE Instruction
+//
+def SERIALIZE : I<0x01, MRM_E8, (outs), (ins), "serialize",
+                  [(int_x86_serialize)]>, PS,
+                  Requires<[HasSERIALIZE]>;
+
 //===----------------------------------------------------------------------===//
 // Pattern fragments to auto generate TBM instructions.
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/X86/X86InstructionSelector.cpp b/llvm/lib/Target/X86/X86InstructionSelector.cpp
index 3f9d626ff9123..23643bb46b660 100644
--- a/llvm/lib/Target/X86/X86InstructionSelector.cpp
+++ b/llvm/lib/Target/X86/X86InstructionSelector.cpp
@@ -71,7 +71,7 @@ class X86InstructionSelector : public InstructionSelector {
 
   // TODO: remove after supported by Tablegen-erated instruction selection.
   unsigned getLoadStoreOp(const LLT &Ty, const RegisterBank &RB, unsigned Opc,
-                          uint64_t Alignment) const;
+                          Align Alignment) const;
 
   bool selectLoadStoreOp(MachineInstr &I, MachineRegisterInfo &MRI,
                          MachineFunction &MF) const;
@@ -394,7 +394,7 @@ bool X86InstructionSelector::select(MachineInstr &I) {
 unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty,
                                                 const RegisterBank &RB,
                                                 unsigned Opc,
-                                                uint64_t Alignment) const {
+                                                Align Alignment) const {
   bool Isload = (Opc == TargetOpcode::G_LOAD);
   bool HasAVX = STI.hasAVX();
   bool HasAVX512 = STI.hasAVX512();
@@ -427,7 +427,7 @@ unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty,
                        HasAVX    ? X86::VMOVSDmr :
                                    X86::MOVSDmr);
   } else if (Ty.isVector() && Ty.getSizeInBits() == 128) {
-    if (Alignment >= 16)
+    if (Alignment >= Align(16))
       return Isload ? (HasVLX ? X86::VMOVAPSZ128rm
                               : HasAVX512
                                     ? X86::VMOVAPSZ128rm_NOVLX
@@ -446,7 +446,7 @@ unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty,
                                     ? X86::VMOVUPSZ128mr_NOVLX
                                     : HasAVX ? X86::VMOVUPSmr : X86::MOVUPSmr);
   } else if (Ty.isVector() && Ty.getSizeInBits() == 256) {
-    if (Alignment >= 32)
+    if (Alignment >= Align(32))
       return Isload ? (HasVLX ? X86::VMOVAPSZ256rm
                               : HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX
                                           : X86::VMOVAPSYrm)
@@ -461,7 +461,7 @@ unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty,
                               : HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX
                                           : X86::VMOVUPSYmr);
   } else if (Ty.isVector() && Ty.getSizeInBits() == 512) {
-    if (Alignment >= 64)
+    if (Alignment >= Align(64))
       return Isload ? X86::VMOVAPSZrm : X86::VMOVAPSZmr;
     else
       return Isload ? X86::VMOVUPSZrm : X86::VMOVUPSZmr;
@@ -520,13 +520,13 @@ bool X86InstructionSelector::selectLoadStoreOp(MachineInstr &I,
       LLVM_DEBUG(dbgs() << "Atomic ordering not supported yet\n");
       return false;
     }
-    if (MemOp.getAlignment() < Ty.getSizeInBits()/8) {
+    if (MemOp.getAlign() < Ty.getSizeInBits() / 8) {
       LLVM_DEBUG(dbgs() << "Unaligned atomics not supported yet\n");
       return false;
     }
   }
 
-  unsigned NewOpc = getLoadStoreOp(Ty, RB, Opc, MemOp.getAlignment());
+  unsigned NewOpc = getLoadStoreOp(Ty, RB, Opc, MemOp.getAlign());
   if (NewOpc == Opc)
     return false;
 
@@ -1435,14 +1435,16 @@ bool X86InstructionSelector::materializeFP(MachineInstr &I,
   const Register DstReg = I.getOperand(0).getReg();
   const LLT DstTy = MRI.getType(DstReg);
   const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI);
-  unsigned Align = DstTy.getSizeInBits();
+  Align Alignment = Align(DstTy.getSizeInBytes());
   const DebugLoc &DbgLoc = I.getDebugLoc();
 
-  unsigned Opc = getLoadStoreOp(DstTy, RegBank, TargetOpcode::G_LOAD, Align);
+  unsigned Opc =
+      getLoadStoreOp(DstTy, RegBank, TargetOpcode::G_LOAD, Alignment);
 
   // Create the load from the constant pool.
   const ConstantFP *CFP = I.getOperand(1).getFPImm();
-  unsigned CPI = MF.getConstantPool()->getConstantPoolIndex(CFP, Align);
+  unsigned CPI =
+      MF.getConstantPool()->getConstantPoolIndex(CFP, Alignment.value());
   MachineInstr *LoadInst = nullptr;
   unsigned char OpFlag = STI.classifyLocalReference(nullptr);
 
@@ -1456,7 +1458,7 @@ bool X86InstructionSelector::materializeFP(MachineInstr &I,
 
     MachineMemOperand *MMO = MF.getMachineMemOperand(
         MachinePointerInfo::getConstantPool(MF), MachineMemOperand::MOLoad,
-        MF.getDataLayout().getPointerSize(), Align);
+        MF.getDataLayout().getPointerSize(), Alignment);
 
     LoadInst =
         addDirectMem(BuildMI(*I.getParent(), I, DbgLoc, TII.get(Opc), DstReg),
diff --git a/llvm/lib/Target/X86/X86InterleavedAccess.cpp b/llvm/lib/Target/X86/X86InterleavedAccess.cpp
index f0288adf52ce2..8c3b185051575 100644
--- a/llvm/lib/Target/X86/X86InterleavedAccess.cpp
+++ b/llvm/lib/Target/X86/X86InterleavedAccess.cpp
@@ -325,19 +325,19 @@ void X86InterleavedAccessGroup::interleave8bitStride4VF8(
 
   MVT VT = MVT::v8i16;
   TransposedMatrix.resize(2);
-  SmallVector<uint32_t, 16> MaskLow;
-  SmallVector<uint32_t, 32> MaskLowTemp1, MaskLowWord;
-  SmallVector<uint32_t, 32> MaskHighTemp1, MaskHighWord;
+  SmallVector<int, 16> MaskLow;
+  SmallVector<int, 32> MaskLowTemp1, MaskLowWord;
+  SmallVector<int, 32> MaskHighTemp1, MaskHighWord;
 
   for (unsigned i = 0; i < 8; ++i) {
     MaskLow.push_back(i);
     MaskLow.push_back(i + 8);
   }
 
-  createUnpackShuffleMask<uint32_t>(VT, MaskLowTemp1, true, false);
-  createUnpackShuffleMask<uint32_t>(VT, MaskHighTemp1, false, false);
-  scaleShuffleMask<uint32_t>(2, MaskHighTemp1, MaskHighWord);
-  scaleShuffleMask<uint32_t>(2, MaskLowTemp1, MaskLowWord);
+  createUnpackShuffleMask(VT, MaskLowTemp1, true, false);
+  createUnpackShuffleMask(VT, MaskHighTemp1, false, false);
+  scaleShuffleMask(2, MaskHighTemp1, MaskHighWord);
+  scaleShuffleMask(2, MaskLowTemp1, MaskLowWord);
   // IntrVec1Low = c0 m0 c1 m1 c2 m2 c3 m3 c4 m4 c5 m5 c6 m6 c7 m7
   // IntrVec2Low = y0 k0 y1 k1 y2 k2 y3 k3 y4 k4 y5 k5 y6 k6 y7 k7
   Value *IntrVec1Low =
@@ -367,25 +367,25 @@ void X86InterleavedAccessGroup::interleave8bitStride4(
   MVT HalfVT = scaleVectorType(VT);
 
   TransposedMatrix.resize(4);
-  SmallVector<uint32_t, 32> MaskHigh;
-  SmallVector<uint32_t, 32> MaskLow;
-  SmallVector<uint32_t, 32> LowHighMask[2];
-  SmallVector<uint32_t, 32> MaskHighTemp;
-  SmallVector<uint32_t, 32> MaskLowTemp;
+  SmallVector<int, 32> MaskHigh;
+  SmallVector<int, 32> MaskLow;
+  SmallVector<int, 32> LowHighMask[2];
+  SmallVector<int, 32> MaskHighTemp;
+  SmallVector<int, 32> MaskLowTemp;
 
   // MaskHighTemp and MaskLowTemp built in the vpunpckhbw and vpunpcklbw X86
   // shuffle pattern.
 
-  createUnpackShuffleMask<uint32_t>(VT, MaskLow, true, false);
-  createUnpackShuffleMask<uint32_t>(VT, MaskHigh, false, false);
+  createUnpackShuffleMask(VT, MaskLow, true, false);
+  createUnpackShuffleMask(VT, MaskHigh, false, false);
 
   // MaskHighTemp1 and MaskLowTemp1 built in the vpunpckhdw and vpunpckldw X86
   // shuffle pattern.
 
-  createUnpackShuffleMask<uint32_t>(HalfVT, MaskLowTemp, true, false);
-  createUnpackShuffleMask<uint32_t>(HalfVT, MaskHighTemp, false, false);
-  scaleShuffleMask<uint32_t>(2, MaskLowTemp, LowHighMask[0]);
-  scaleShuffleMask<uint32_t>(2, MaskHighTemp, LowHighMask[1]);
+  createUnpackShuffleMask(HalfVT, MaskLowTemp, true, false);
+  createUnpackShuffleMask(HalfVT, MaskHighTemp, false, false);
+  scaleShuffleMask(2, MaskLowTemp, LowHighMask[0]);
+  scaleShuffleMask(2, MaskHighTemp, LowHighMask[1]);
 
   // IntrVec1Low  = c0  m0  c1  m1 ... c7  m7  | c16 m16 c17 m17 ... c23 m23
   // IntrVec1High = c8  m8  c9  m9 ... c15 m15 | c24 m24 c25 m25 ... c31 m31
diff --git a/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp b/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp
new file mode 100644
index 0000000000000..7c027e5fca67e
--- /dev/null
+++ b/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp
@@ -0,0 +1,586 @@
+//==-- X86LoadValueInjectionLoadHardening.cpp - LVI load hardening for x86 --=//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// Description: This pass finds Load Value Injection (LVI) gadgets consisting
+/// of a load from memory (i.e., SOURCE), and any operation that may transmit
+/// the value loaded from memory over a covert channel, or use the value loaded
+/// from memory to determine a branch/call target (i.e., SINK).
+///
+//===----------------------------------------------------------------------===//
+
+#include "ImmutableGraph.h"
+#include "X86.h"
+#include "X86Subtarget.h"
+#include "X86TargetMachine.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/SmallSet.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineDominanceFrontier.h"
+#include "llvm/CodeGen/MachineDominators.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineLoopInfo.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/RDFGraph.h"
+#include "llvm/CodeGen/RDFLiveness.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/DOTGraphTraits.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/GraphWriter.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+#define PASS_KEY "x86-lvi-load"
+#define DEBUG_TYPE PASS_KEY
+
+STATISTIC(NumFunctionsConsidered, "Number of functions analyzed");
+STATISTIC(NumFunctionsMitigated, "Number of functions for which mitigations "
+                                 "were deployed");
+STATISTIC(NumGadgets, "Number of LVI gadgets detected during analysis");
+
+static cl::opt<bool> NoConditionalBranches(
+    PASS_KEY "-no-cbranch",
+    cl::desc("Don't treat conditional branches as disclosure gadgets. This "
+             "may improve performance, at the cost of security."),
+    cl::init(false), cl::Hidden);
+
+static cl::opt<bool> EmitDot(
+    PASS_KEY "-dot",
+    cl::desc(
+        "For each function, emit a dot graph depicting potential LVI gadgets"),
+    cl::init(false), cl::Hidden);
+
+static cl::opt<bool> EmitDotOnly(
+    PASS_KEY "-dot-only",
+    cl::desc("For each function, emit a dot graph depicting potential LVI "
+             "gadgets, and do not insert any fences"),
+    cl::init(false), cl::Hidden);
+
+static cl::opt<bool> EmitDotVerify(
+    PASS_KEY "-dot-verify",
+    cl::desc("For each function, emit a dot graph to stdout depicting "
+             "potential LVI gadgets, used for testing purposes only"),
+    cl::init(false), cl::Hidden);
+
+static cl::opt<bool> NoFixedLoads(
+    PASS_KEY "-no-fixed",
+    cl::desc("Don't mitigate RIP-relative or RSP-relative loads. This "
+             "may improve performance, at the cost of security."),
+    cl::init(false), cl::Hidden);
+
+#define ARG_NODE nullptr
+#define GADGET_EDGE ((int)(-1))
+#define WEIGHT(EdgeValue) ((double)(2 * (EdgeValue) + 1))
+
+namespace {
+
+class X86LoadValueInjectionLoadHardeningPass : public MachineFunctionPass {
+public:
+  X86LoadValueInjectionLoadHardeningPass() : MachineFunctionPass(ID) {}
+
+  StringRef getPassName() const override {
+    return "X86 Load Value Injection (LVI) Load Hardening";
+  }
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  static char ID;
+
+private:
+  struct MachineGadgetGraph : ImmutableGraph<MachineInstr *, int> {
+    using GraphT = ImmutableGraph<MachineInstr *, int>;
+    using Node = typename GraphT::Node;
+    using Edge = typename GraphT::Edge;
+    using size_type = typename GraphT::size_type;
+    MachineGadgetGraph(Node *Nodes, size_type NodesSize, Edge *Edges,
+                       size_type EdgesSize, int NumFences = 0,
+                       int NumGadgets = 0)
+        : GraphT{Nodes, NodesSize, Edges, EdgesSize}, NumFences{NumFences},
+          NumGadgets{NumGadgets} {}
+    MachineFunction &getMF() { // FIXME: This function should be cleaner
+      for (Node *NI = nodes_begin(), *const NE = nodes_end(); NI != NE; ++NI) {
+        if (NI->value()) {
+          return *NI->value()->getMF();
+        }
+      }
+      llvm_unreachable("Could not find a valid node");
+    }
+    static inline bool isCFGEdge(Edge &E) { return E.value() != GADGET_EDGE; }
+    static inline bool isGadgetEdge(Edge &E) {
+      return E.value() == GADGET_EDGE;
+    }
+    int NumFences;
+    int NumGadgets;
+  };
+  friend struct llvm::DOTGraphTraits<MachineGadgetGraph *>;
+  using GTraits = llvm::GraphTraits<MachineGadgetGraph *>;
+  using GraphBuilder = ImmutableGraphBuilder<MachineGadgetGraph>;
+  using EdgeSet = MachineGadgetGraph::EdgeSet;
+  using Gadget = std::pair<MachineInstr *, MachineInstr *>;
+
+  const X86Subtarget *STI;
+  const TargetInstrInfo *TII;
+  const TargetRegisterInfo *TRI;
+
+  int hardenLoads(MachineFunction &MF, bool Fixed) const;
+  std::unique_ptr<MachineGadgetGraph>
+  getGadgetGraph(MachineFunction &MF, const MachineLoopInfo &MLI,
+                 const MachineDominatorTree &MDT,
+                 const MachineDominanceFrontier &MDF, bool FixedLoads) const;
+
+  bool instrUsesRegToAccessMemory(const MachineInstr &I, unsigned Reg) const;
+  bool instrUsesRegToBranch(const MachineInstr &I, unsigned Reg) const;
+  template <unsigned K> bool hasLoadFrom(const MachineInstr &MI) const;
+  bool instrAccessesStackSlot(const MachineInstr &MI) const;
+  bool instrAccessesConstantPool(const MachineInstr &MI) const;
+  bool instrAccessesGOT(const MachineInstr &MI) const;
+  inline bool instrIsFixedAccess(const MachineInstr &MI) const {
+    return instrAccessesConstantPool(MI) || instrAccessesStackSlot(MI) ||
+           instrAccessesGOT(MI);
+  }
+  inline bool isFence(const MachineInstr *MI) const {
+    return MI && (MI->getOpcode() == X86::LFENCE ||
+                  (STI->useLVIControlFlowIntegrity() && MI->isCall()));
+  }
+};
+
+} // end anonymous namespace
+
+namespace llvm {
+
+template <>
+struct GraphTraits<X86LoadValueInjectionLoadHardeningPass::MachineGadgetGraph *>
+    : GraphTraits<ImmutableGraph<MachineInstr *, int> *> {};
+
+template <>
+struct DOTGraphTraits<
+    X86LoadValueInjectionLoadHardeningPass::MachineGadgetGraph *>
+    : DefaultDOTGraphTraits {
+  using GraphType = X86LoadValueInjectionLoadHardeningPass::MachineGadgetGraph;
+  using Traits = X86LoadValueInjectionLoadHardeningPass::GTraits;
+  using NodeRef = typename Traits::NodeRef;
+  using EdgeRef = typename Traits::EdgeRef;
+  using ChildIteratorType = typename Traits::ChildIteratorType;
+  using ChildEdgeIteratorType = typename Traits::ChildEdgeIteratorType;
+
+  DOTGraphTraits(bool isSimple = false) : DefaultDOTGraphTraits(isSimple) {}
+
+  static std::string getGraphName(GraphType *G) {
+    std::string GraphName{"Speculative gadgets for \""};
+    GraphName += G->getMF().getName();
+    GraphName += "\" function";
+    return GraphName;
+  }
+
+  std::string getNodeLabel(NodeRef Node, GraphType *) {
+    std::string str;
+    raw_string_ostream str_stream{str};
+    if (Node->value() == ARG_NODE)
+      return "ARGS";
+    str_stream << *Node->value();
+    return str_stream.str();
+  }
+
+  static std::string getNodeAttributes(NodeRef Node, GraphType *) {
+    MachineInstr *MI = Node->value();
+    if (MI == ARG_NODE)
+      return "color = blue";
+    else if (MI->getOpcode() == X86::LFENCE)
+      return "color = green";
+    else
+      return "";
+  }
+
+  static std::string getEdgeAttributes(NodeRef, ChildIteratorType E,
+                                       GraphType *) {
+    int EdgeVal = (*E.getCurrent()).value();
+    return EdgeVal >= 0 ? "label = " + std::to_string(EdgeVal)
+                        : "color = red, style = \"dashed\"";
+  }
+};
+
+} // end namespace llvm
+
+char X86LoadValueInjectionLoadHardeningPass::ID = 0;
+
+void X86LoadValueInjectionLoadHardeningPass::getAnalysisUsage(
+    AnalysisUsage &AU) const {
+  MachineFunctionPass::getAnalysisUsage(AU);
+  AU.addRequired<MachineLoopInfo>();
+  AU.addRequired<MachineDominatorTree>();
+  AU.addRequired<MachineDominanceFrontier>();
+  AU.setPreservesCFG();
+}
+
+bool X86LoadValueInjectionLoadHardeningPass::runOnMachineFunction(
+    MachineFunction &MF) {
+  LLVM_DEBUG(dbgs() << "***** " << getPassName() << " : " << MF.getName()
+                    << " *****\n");
+  STI = &MF.getSubtarget<X86Subtarget>();
+  if (!STI->useLVILoadHardening() || !STI->is64Bit())
+    return false; // FIXME: support 32-bit
+
+  // Don't skip functions with the "optnone" attr but participate in opt-bisect.
+  const Function &F = MF.getFunction();
+  if (!F.hasOptNone() && skipFunction(F))
+    return false;
+
+  ++NumFunctionsConsidered;
+  TII = STI->getInstrInfo();
+  TRI = STI->getRegisterInfo();
+  LLVM_DEBUG(dbgs() << "Hardening data-dependent loads...\n");
+  hardenLoads(MF, false);
+  LLVM_DEBUG(dbgs() << "Hardening data-dependent loads... Done\n");
+  if (!NoFixedLoads) {
+    LLVM_DEBUG(dbgs() << "Hardening fixed loads...\n");
+    hardenLoads(MF, true);
+    LLVM_DEBUG(dbgs() << "Hardening fixed loads... Done\n");
+  }
+  return false;
+}
+
+// Apply the mitigation to `MF`, return the number of fences inserted.
+// If `FixedLoads` is `true`, then the mitigation will be applied to fixed
+// loads; otherwise, mitigation will be applied to non-fixed loads.
+int X86LoadValueInjectionLoadHardeningPass::hardenLoads(MachineFunction &MF,
+                                                        bool FixedLoads) const {
+  LLVM_DEBUG(dbgs() << "Building gadget graph...\n");
+  const auto &MLI = getAnalysis<MachineLoopInfo>();
+  const auto &MDT = getAnalysis<MachineDominatorTree>();
+  const auto &MDF = getAnalysis<MachineDominanceFrontier>();
+  std::unique_ptr<MachineGadgetGraph> Graph =
+      getGadgetGraph(MF, MLI, MDT, MDF, FixedLoads);
+  LLVM_DEBUG(dbgs() << "Building gadget graph... Done\n");
+  if (Graph == nullptr)
+    return 0; // didn't find any gadgets
+
+  if (EmitDotVerify) {
+    WriteGraph(outs(), Graph.get());
+    return 0;
+  }
+
+  if (EmitDot || EmitDotOnly) {
+    LLVM_DEBUG(dbgs() << "Emitting gadget graph...\n");
+    std::error_code FileError;
+    std::string FileName = "lvi.";
+    if (FixedLoads)
+      FileName += "fixed.";
+    FileName += Graph->getMF().getName();
+    FileName += ".dot";
+    raw_fd_ostream FileOut(FileName, FileError);
+    if (FileError)
+      errs() << FileError.message();
+    WriteGraph(FileOut, Graph.get());
+    FileOut.close();
+    LLVM_DEBUG(dbgs() << "Emitting gadget graph... Done\n");
+    if (EmitDotOnly)
+      return 0;
+  }
+
+  return 0;
+}
+
+std::unique_ptr<X86LoadValueInjectionLoadHardeningPass::MachineGadgetGraph>
+X86LoadValueInjectionLoadHardeningPass::getGadgetGraph(
+    MachineFunction &MF, const MachineLoopInfo &MLI,
+    const MachineDominatorTree &MDT, const MachineDominanceFrontier &MDF,
+    bool FixedLoads) const {
+  using namespace rdf;
+
+  // Build the Register Dataflow Graph using the RDF framework
+  TargetOperandInfo TOI{*TII};
+  DataFlowGraph DFG{MF, *TII, *TRI, MDT, MDF, TOI};
+  DFG.build();
+  Liveness L{MF.getRegInfo(), DFG};
+  L.computePhiInfo();
+
+  GraphBuilder Builder;
+  using GraphIter = typename GraphBuilder::NodeRef;
+  DenseMap<MachineInstr *, GraphIter> NodeMap;
+  int FenceCount = 0;
+  auto MaybeAddNode = [&NodeMap, &Builder](MachineInstr *MI) {
+    auto Ref = NodeMap.find(MI);
+    if (Ref == NodeMap.end()) {
+      auto I = Builder.addVertex(MI);
+      NodeMap[MI] = I;
+      return std::pair<GraphIter, bool>{I, true};
+    } else {
+      return std::pair<GraphIter, bool>{Ref->getSecond(), false};
+    }
+  };
+
+  // Analyze all machine instructions to find gadgets and LFENCEs, adding
+  // each interesting value to `Nodes`
+  DenseSet<std::pair<GraphIter, GraphIter>> GadgetEdgeSet;
+  auto AnalyzeDef = [&](NodeAddr<DefNode *> Def) {
+    MachineInstr *MI = Def.Addr->getFlags() & NodeAttrs::PhiRef
+                           ? ARG_NODE
+                           : Def.Addr->getOp().getParent();
+    auto AnalyzeUse = [&](NodeAddr<UseNode *> Use) {
+      assert(!(Use.Addr->getFlags() & NodeAttrs::PhiRef));
+      MachineOperand &UseMO = Use.Addr->getOp();
+      MachineInstr &UseMI = *UseMO.getParent();
+      assert(UseMO.isReg());
+      // We naively assume that an instruction propagates any loaded Uses
+      // to all Defs, unless the instruction is a call
+      if (UseMI.isCall())
+        return false;
+      if (instrUsesRegToAccessMemory(UseMI, UseMO.getReg()) ||
+          (!NoConditionalBranches &&
+           instrUsesRegToBranch(UseMI, UseMO.getReg()))) { // found a gadget!
+        // add the root of this chain
+        auto GadgetBegin = MaybeAddNode(MI);
+        // and the instruction that (transitively) discloses the root
+        auto GadgetEnd = MaybeAddNode(&UseMI);
+        if (GadgetEdgeSet.insert({GadgetBegin.first, GadgetEnd.first}).second)
+          Builder.addEdge(GADGET_EDGE, GadgetBegin.first, GadgetEnd.first);
+        if (UseMI.mayLoad()) // FIXME: This should be more precise
+          return false;      // stop traversing further uses of `Reg`
+      }
+      return true;
+    };
+    SmallSet<NodeId, 8> NodesVisited;
+    std::function<void(NodeAddr<DefNode *>)> AnalyzeDefUseChain =
+        [&](NodeAddr<DefNode *> Def) {
+          if (Def.Addr->getAttrs() & NodeAttrs::Dead)
+            return;
+          RegisterRef DefReg = DFG.getPRI().normalize(Def.Addr->getRegRef(DFG));
+          NodeList Uses;
+          for (auto UseID : L.getAllReachedUses(DefReg, Def)) {
+            auto Use = DFG.addr<UseNode *>(UseID);
+            if (Use.Addr->getFlags() & NodeAttrs::PhiRef) { // phi node
+              NodeAddr<PhiNode *> Phi = Use.Addr->getOwner(DFG);
+              for (auto I : L.getRealUses(Phi.Id)) {
+                if (DFG.getPRI().alias(RegisterRef(I.first), DefReg)) {
+                  for (auto UA : I.second) {
+                    auto PhiUse = DFG.addr<UseNode *>(UA.first);
+                    Uses.push_back(PhiUse);
+                  }
+                }
+              }
+            } else { // not a phi node
+              Uses.push_back(Use);
+            }
+          }
+          for (auto N : Uses) {
+            NodeAddr<UseNode *> Use{N};
+            if (NodesVisited.insert(Use.Id).second && AnalyzeUse(Use)) {
+              NodeAddr<InstrNode *> Owner{Use.Addr->getOwner(DFG)};
+              NodeList Defs = Owner.Addr->members_if(DataFlowGraph::IsDef, DFG);
+              std::for_each(Defs.begin(), Defs.end(), AnalyzeDefUseChain);
+            }
+          }
+        };
+    AnalyzeDefUseChain(Def);
+  };
+
+  LLVM_DEBUG(dbgs() << "Analyzing def-use chains to find gadgets\n");
+  // Analyze function arguments
+  if (!FixedLoads) { // only need to analyze function args once
+    NodeAddr<BlockNode *> EntryBlock = DFG.getFunc().Addr->getEntryBlock(DFG);
+    for (NodeAddr<PhiNode *> ArgPhi :
+         EntryBlock.Addr->members_if(DataFlowGraph::IsPhi, DFG)) {
+      NodeList Defs = ArgPhi.Addr->members_if(DataFlowGraph::IsDef, DFG);
+      std::for_each(Defs.begin(), Defs.end(), AnalyzeDef);
+    }
+  }
+  // Analyze every instruction in MF
+  for (NodeAddr<BlockNode *> BA : DFG.getFunc().Addr->members(DFG)) {
+    for (NodeAddr<StmtNode *> SA :
+         BA.Addr->members_if(DataFlowGraph::IsCode<NodeAttrs::Stmt>, DFG)) {
+      MachineInstr *MI = SA.Addr->getCode();
+      if (isFence(MI)) {
+        MaybeAddNode(MI);
+        ++FenceCount;
+      } else if (MI->mayLoad() && ((FixedLoads && instrIsFixedAccess(*MI)) ||
+                                   (!FixedLoads && !instrIsFixedAccess(*MI)))) {
+        NodeList Defs = SA.Addr->members_if(DataFlowGraph::IsDef, DFG);
+        std::for_each(Defs.begin(), Defs.end(), AnalyzeDef);
+      }
+    }
+  }
+  int GadgetCount = static_cast<int>(GadgetEdgeSet.size());
+  LLVM_DEBUG(dbgs() << "Found " << FenceCount << " fences\n");
+  LLVM_DEBUG(dbgs() << "Found " << GadgetCount << " gadgets\n");
+  if (GadgetCount == 0)
+    return nullptr;
+  NumGadgets += GadgetCount;
+
+  // Traverse CFG to build the rest of the graph
+  SmallSet<MachineBasicBlock *, 8> BlocksVisited;
+  std::function<void(MachineBasicBlock *, GraphIter, unsigned)> TraverseCFG =
+      [&](MachineBasicBlock *MBB, GraphIter GI, unsigned ParentDepth) {
+        unsigned LoopDepth = MLI.getLoopDepth(MBB);
+        if (!MBB->empty()) {
+          // Always add the first instruction in each block
+          auto NI = MBB->begin();
+          auto BeginBB = MaybeAddNode(&*NI);
+          Builder.addEdge(ParentDepth, GI, BeginBB.first);
+          if (!BlocksVisited.insert(MBB).second)
+            return;
+
+          // Add any instructions within the block that are gadget components
+          GI = BeginBB.first;
+          while (++NI != MBB->end()) {
+            auto Ref = NodeMap.find(&*NI);
+            if (Ref != NodeMap.end()) {
+              Builder.addEdge(LoopDepth, GI, Ref->getSecond());
+              GI = Ref->getSecond();
+            }
+          }
+
+          // Always add the terminator instruction, if one exists
+          auto T = MBB->getFirstTerminator();
+          if (T != MBB->end()) {
+            auto EndBB = MaybeAddNode(&*T);
+            if (EndBB.second)
+              Builder.addEdge(LoopDepth, GI, EndBB.first);
+            GI = EndBB.first;
+          }
+        }
+        for (MachineBasicBlock *Succ : MBB->successors())
+          TraverseCFG(Succ, GI, LoopDepth);
+      };
+  // ARG_NODE is a pseudo-instruction that represents MF args in the GadgetGraph
+  GraphIter ArgNode = MaybeAddNode(ARG_NODE).first;
+  TraverseCFG(&MF.front(), ArgNode, 0);
+  std::unique_ptr<MachineGadgetGraph> G{Builder.get(FenceCount, GadgetCount)};
+  LLVM_DEBUG(dbgs() << "Found " << GTraits::size(G.get()) << " nodes\n");
+  return G;
+}
+
+bool X86LoadValueInjectionLoadHardeningPass::instrUsesRegToAccessMemory(
+    const MachineInstr &MI, unsigned Reg) const {
+  if (!MI.mayLoadOrStore() || MI.getOpcode() == X86::MFENCE ||
+      MI.getOpcode() == X86::SFENCE || MI.getOpcode() == X86::LFENCE)
+    return false;
+
+  // FIXME: This does not handle pseudo loading instruction like TCRETURN*
+  const MCInstrDesc &Desc = MI.getDesc();
+  int MemRefBeginIdx = X86II::getMemoryOperandNo(Desc.TSFlags);
+  if (MemRefBeginIdx < 0) {
+    LLVM_DEBUG(dbgs() << "Warning: unable to obtain memory operand for loading "
+                         "instruction:\n";
+               MI.print(dbgs()); dbgs() << '\n';);
+    return false;
+  }
+  MemRefBeginIdx += X86II::getOperandBias(Desc);
+
+  const MachineOperand &BaseMO =
+      MI.getOperand(MemRefBeginIdx + X86::AddrBaseReg);
+  const MachineOperand &IndexMO =
+      MI.getOperand(MemRefBeginIdx + X86::AddrIndexReg);
+  return (BaseMO.isReg() && BaseMO.getReg() != X86::NoRegister &&
+          TRI->regsOverlap(BaseMO.getReg(), Reg)) ||
+         (IndexMO.isReg() && IndexMO.getReg() != X86::NoRegister &&
+          TRI->regsOverlap(IndexMO.getReg(), Reg));
+}
+
+bool X86LoadValueInjectionLoadHardeningPass::instrUsesRegToBranch(
+    const MachineInstr &MI, unsigned Reg) const {
+  if (!MI.isConditionalBranch())
+    return false;
+  for (const MachineOperand &Use : MI.uses())
+    if (Use.isReg() && Use.getReg() == Reg)
+      return true;
+  return false;
+}
+
+template <unsigned K>
+bool X86LoadValueInjectionLoadHardeningPass::hasLoadFrom(
+    const MachineInstr &MI) const {
+  for (auto &MMO : MI.memoperands()) {
+    const PseudoSourceValue *PSV = MMO->getPseudoValue();
+    if (PSV && PSV->kind() == K && MMO->isLoad())
+      return true;
+  }
+  return false;
+}
+
+bool X86LoadValueInjectionLoadHardeningPass::instrAccessesStackSlot(
+    const MachineInstr &MI) const {
+  // Check the PSV first
+  if (hasLoadFrom<PseudoSourceValue::PSVKind::FixedStack>(MI))
+    return true;
+  // Some loads are not marked with a PSV, so we always need to double check
+  const MCInstrDesc &Desc = MI.getDesc();
+  int MemRefBeginIdx = X86II::getMemoryOperandNo(Desc.TSFlags);
+  if (MemRefBeginIdx < 0)
+    return false;
+  MemRefBeginIdx += X86II::getOperandBias(Desc);
+  return MI.getOperand(MemRefBeginIdx + X86::AddrBaseReg).isFI() &&
+         MI.getOperand(MemRefBeginIdx + X86::AddrScaleAmt).isImm() &&
+         MI.getOperand(MemRefBeginIdx + X86::AddrIndexReg).isReg() &&
+         MI.getOperand(MemRefBeginIdx + X86::AddrDisp).isImm() &&
+         MI.getOperand(MemRefBeginIdx + X86::AddrScaleAmt).getImm() == 1 &&
+         MI.getOperand(MemRefBeginIdx + X86::AddrIndexReg).getReg() ==
+             X86::NoRegister &&
+         MI.getOperand(MemRefBeginIdx + X86::AddrDisp).getImm() == 0;
+}
+
+bool X86LoadValueInjectionLoadHardeningPass::instrAccessesConstantPool(
+    const MachineInstr &MI) const {
+  if (hasLoadFrom<PseudoSourceValue::PSVKind::ConstantPool>(MI))
+    return true;
+  const MCInstrDesc &Desc = MI.getDesc();
+  int MemRefBeginIdx = X86II::getMemoryOperandNo(Desc.TSFlags);
+  if (MemRefBeginIdx < 0)
+    return false;
+  MemRefBeginIdx += X86II::getOperandBias(Desc);
+  return MI.getOperand(MemRefBeginIdx + X86::AddrBaseReg).isReg() &&
+         MI.getOperand(MemRefBeginIdx + X86::AddrScaleAmt).isImm() &&
+         MI.getOperand(MemRefBeginIdx + X86::AddrIndexReg).isReg() &&
+         MI.getOperand(MemRefBeginIdx + X86::AddrDisp).isCPI() &&
+         (MI.getOperand(MemRefBeginIdx + X86::AddrBaseReg).getReg() ==
+              X86::RIP ||
+          MI.getOperand(MemRefBeginIdx + X86::AddrBaseReg).getReg() ==
+              X86::NoRegister) &&
+         MI.getOperand(MemRefBeginIdx + X86::AddrScaleAmt).getImm() == 1 &&
+         MI.getOperand(MemRefBeginIdx + X86::AddrIndexReg).getReg() ==
+             X86::NoRegister;
+}
+
+bool X86LoadValueInjectionLoadHardeningPass::instrAccessesGOT(
+    const MachineInstr &MI) const {
+  if (hasLoadFrom<PseudoSourceValue::PSVKind::GOT>(MI))
+    return true;
+  const MCInstrDesc &Desc = MI.getDesc();
+  int MemRefBeginIdx = X86II::getMemoryOperandNo(Desc.TSFlags);
+  if (MemRefBeginIdx < 0)
+    return false;
+  MemRefBeginIdx += X86II::getOperandBias(Desc);
+  return MI.getOperand(MemRefBeginIdx + X86::AddrBaseReg).isReg() &&
+         MI.getOperand(MemRefBeginIdx + X86::AddrScaleAmt).isImm() &&
+         MI.getOperand(MemRefBeginIdx + X86::AddrIndexReg).isReg() &&
+         MI.getOperand(MemRefBeginIdx + X86::AddrDisp).getTargetFlags() ==
+             X86II::MO_GOTPCREL &&
+         MI.getOperand(MemRefBeginIdx + X86::AddrBaseReg).getReg() ==
+             X86::RIP &&
+         MI.getOperand(MemRefBeginIdx + X86::AddrScaleAmt).getImm() == 1 &&
+         MI.getOperand(MemRefBeginIdx + X86::AddrIndexReg).getReg() ==
+             X86::NoRegister;
+}
+
+INITIALIZE_PASS_BEGIN(X86LoadValueInjectionLoadHardeningPass, PASS_KEY,
+                      "X86 LVI load hardening", false, false)
+INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo)
+INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
+INITIALIZE_PASS_DEPENDENCY(MachineDominanceFrontier)
+INITIALIZE_PASS_END(X86LoadValueInjectionLoadHardeningPass, PASS_KEY,
+                    "X86 LVI load hardening", false, false)
+
+FunctionPass *llvm::createX86LoadValueInjectionLoadHardeningPass() {
+  return new X86LoadValueInjectionLoadHardeningPass();
+}
diff --git a/llvm/lib/Target/X86/X86LoadValueInjectionRetHardening.cpp b/llvm/lib/Target/X86/X86LoadValueInjectionRetHardening.cpp
new file mode 100644
index 0000000000000..9c36e86099f99
--- /dev/null
+++ b/llvm/lib/Target/X86/X86LoadValueInjectionRetHardening.cpp
@@ -0,0 +1,140 @@
+//===-- X86LoadValueInjectionRetHardening.cpp - LVI RET hardening for x86 --==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// Description: Replaces every `ret` instruction with the sequence:
+/// ```
+/// pop <scratch-reg>
+/// lfence
+/// jmp *<scratch-reg>
+/// ```
+/// where `<scratch-reg>` is some available scratch register, according to the
+/// calling convention of the function being mitigated.
+///
+//===----------------------------------------------------------------------===//
+
+#include "X86.h"
+#include "X86InstrBuilder.h"
+#include "X86Subtarget.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/IR/Function.h"
+#include "llvm/Support/Debug.h"
+#include <bitset>
+
+using namespace llvm;
+
+#define PASS_KEY "x86-lvi-ret"
+#define DEBUG_TYPE PASS_KEY
+
+STATISTIC(NumFences, "Number of LFENCEs inserted for LVI mitigation");
+STATISTIC(NumFunctionsConsidered, "Number of functions analyzed");
+STATISTIC(NumFunctionsMitigated, "Number of functions for which mitigations "
+                                 "were deployed");
+
+namespace {
+
+class X86LoadValueInjectionRetHardeningPass : public MachineFunctionPass {
+public:
+  X86LoadValueInjectionRetHardeningPass() : MachineFunctionPass(ID) {}
+  StringRef getPassName() const override {
+    return "X86 Load Value Injection (LVI) Ret-Hardening";
+  }
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  static char ID;
+};
+
+} // end anonymous namespace
+
+char X86LoadValueInjectionRetHardeningPass::ID = 0;
+
+bool X86LoadValueInjectionRetHardeningPass::runOnMachineFunction(
+    MachineFunction &MF) {
+  LLVM_DEBUG(dbgs() << "***** " << getPassName() << " : " << MF.getName()
+                    << " *****\n");
+  const X86Subtarget *Subtarget = &MF.getSubtarget<X86Subtarget>();
+  if (!Subtarget->useLVIControlFlowIntegrity() || !Subtarget->is64Bit())
+    return false; // FIXME: support 32-bit
+
+  // Don't skip functions with the "optnone" attr but participate in opt-bisect.
+  const Function &F = MF.getFunction();
+  if (!F.hasOptNone() && skipFunction(F))
+    return false;
+
+  ++NumFunctionsConsidered;
+  const X86RegisterInfo *TRI = Subtarget->getRegisterInfo();
+  const X86InstrInfo *TII = Subtarget->getInstrInfo();
+  unsigned ClobberReg = X86::NoRegister;
+  std::bitset<X86::NUM_TARGET_REGS> UnclobberableGR64s;
+  UnclobberableGR64s.set(X86::RSP); // can't clobber stack pointer
+  UnclobberableGR64s.set(X86::RIP); // can't clobber instruction pointer
+  UnclobberableGR64s.set(X86::RAX); // used for function return
+  UnclobberableGR64s.set(X86::RDX); // used for function return
+
+  // We can clobber any register allowed by the function's calling convention.
+  for (const MCPhysReg *PR = TRI->getCalleeSavedRegs(&MF); auto Reg = *PR; ++PR)
+    UnclobberableGR64s.set(Reg);
+  for (auto &Reg : X86::GR64RegClass) {
+    if (!UnclobberableGR64s.test(Reg)) {
+      ClobberReg = Reg;
+      break;
+    }
+  }
+
+  if (ClobberReg != X86::NoRegister) {
+    LLVM_DEBUG(dbgs() << "Selected register "
+                      << Subtarget->getRegisterInfo()->getRegAsmName(ClobberReg)
+                      << " to clobber\n");
+  } else {
+    LLVM_DEBUG(dbgs() << "Could not find a register to clobber\n");
+  }
+
+  bool Modified = false;
+  for (auto &MBB : MF) {
+    MachineInstr &MI = MBB.back();
+    if (MI.getOpcode() != X86::RETQ)
+      continue;
+
+    if (ClobberReg != X86::NoRegister) {
+      MBB.erase_instr(&MI);
+      BuildMI(MBB, MBB.end(), DebugLoc(), TII->get(X86::POP64r))
+          .addReg(ClobberReg, RegState::Define)
+          .setMIFlag(MachineInstr::FrameDestroy);
+      BuildMI(MBB, MBB.end(), DebugLoc(), TII->get(X86::LFENCE));
+      BuildMI(MBB, MBB.end(), DebugLoc(), TII->get(X86::JMP64r))
+          .addReg(ClobberReg);
+    } else {
+      // In case there is no available scratch register, we can still read from
+      // RSP to assert that RSP points to a valid page. The write to RSP is
+      // also helpful because it verifies that the stack's write permissions
+      // are intact.
+      MachineInstr *Fence = BuildMI(MBB, MI, DebugLoc(), TII->get(X86::LFENCE));
+      addRegOffset(BuildMI(MBB, Fence, DebugLoc(), TII->get(X86::SHL64mi)),
+                   X86::RSP, false, 0)
+          .addImm(0)
+          ->addRegisterDead(X86::EFLAGS, TRI);
+    }
+
+    ++NumFences;
+    Modified = true;
+  }
+
+  if (Modified)
+    ++NumFunctionsMitigated;
+  return Modified;
+}
+
+INITIALIZE_PASS(X86LoadValueInjectionRetHardeningPass, PASS_KEY,
+                "X86 LVI ret hardener", false, false)
+
+FunctionPass *llvm::createX86LoadValueInjectionRetHardeningPass() {
+  return new X86LoadValueInjectionRetHardeningPass();
+}
diff --git a/llvm/lib/Target/X86/X86MCInstLower.cpp b/llvm/lib/Target/X86/X86MCInstLower.cpp
index 22333d0e4175e..06f059a1a852a 100644
--- a/llvm/lib/Target/X86/X86MCInstLower.cpp
+++ b/llvm/lib/Target/X86/X86MCInstLower.cpp
@@ -1223,8 +1223,8 @@ void X86AsmPrinter::LowerSTATEPOINT(const MachineInstr &MI,
       break;
     case MachineOperand::MO_Register:
       // FIXME: Add retpoline support and remove this.
-      if (Subtarget->useRetpolineIndirectCalls())
-        report_fatal_error("Lowering register statepoints with retpoline not "
+      if (Subtarget->useIndirectThunkCalls())
+        report_fatal_error("Lowering register statepoints with thunks not "
                            "yet implemented.");
       CallTargetMCOp = MCOperand::createReg(CallTarget.getReg());
       CallOpcode = X86::CALL64r;
@@ -1402,9 +1402,9 @@ void X86AsmPrinter::LowerPATCHPOINT(const MachineInstr &MI,
     EmitAndCountInstruction(
         MCInstBuilder(X86::MOV64ri).addReg(ScratchReg).addOperand(CalleeMCOp));
     // FIXME: Add retpoline support and remove this.
-    if (Subtarget->useRetpolineIndirectCalls())
+    if (Subtarget->useIndirectThunkCalls())
       report_fatal_error(
-          "Lowering patchpoint with retpoline not yet implemented.");
+          "Lowering patchpoint with thunks not yet implemented.");
     EmitAndCountInstruction(MCInstBuilder(X86::CALL64r).addReg(ScratchReg));
   }
 
diff --git a/llvm/lib/Target/X86/X86PartialReduction.cpp b/llvm/lib/Target/X86/X86PartialReduction.cpp
new file mode 100644
index 0000000000000..26c05eab45b27
--- /dev/null
+++ b/llvm/lib/Target/X86/X86PartialReduction.cpp
@@ -0,0 +1,460 @@
+//===-- X86PartialReduction.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass looks for add instructions used by a horizontal reduction to see
+// if we might be able to use pmaddwd or psadbw. Some cases of this require
+// cross basic block knowledge and can't be done in SelectionDAG.
+//
+//===----------------------------------------------------------------------===//
+
+#include "X86.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicsX86.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Operator.h"
+#include "llvm/Pass.h"
+#include "X86TargetMachine.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "x86-partial-reduction"
+
+namespace {
+
+class X86PartialReduction : public FunctionPass {
+  const DataLayout *DL;
+  const X86Subtarget *ST;
+
+public:
+  static char ID; // Pass identification, replacement for typeid.
+
+  X86PartialReduction() : FunctionPass(ID) { }
+
+  bool runOnFunction(Function &Fn) override;
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+  }
+
+  StringRef getPassName() const override {
+    return "X86 Partial Reduction";
+  }
+
+private:
+  bool tryMAddPattern(BinaryOperator *BO);
+  bool tryMAddReplacement(Value *Op, BinaryOperator *Add);
+
+  bool trySADPattern(BinaryOperator *BO);
+  bool trySADReplacement(Value *Op, BinaryOperator *Add);
+};
+}
+
+FunctionPass *llvm::createX86PartialReductionPass() {
+  return new X86PartialReduction();
+}
+
+char X86PartialReduction::ID = 0;
+
+INITIALIZE_PASS(X86PartialReduction, DEBUG_TYPE,
+                "X86 Partial Reduction", false, false)
+
+static bool isVectorReductionOp(const BinaryOperator &BO) {
+  if (!BO.getType()->isVectorTy())
+    return false;
+
+  unsigned Opcode = BO.getOpcode();
+
+  switch (Opcode) {
+  case Instruction::Add:
+  case Instruction::Mul:
+  case Instruction::And:
+  case Instruction::Or:
+  case Instruction::Xor:
+    break;
+  case Instruction::FAdd:
+  case Instruction::FMul:
+    if (auto *FPOp = dyn_cast<FPMathOperator>(&BO))
+      if (FPOp->getFastMathFlags().isFast())
+        break;
+    LLVM_FALLTHROUGH;
+  default:
+    return false;
+  }
+
+  unsigned ElemNum = BO.getType()->getVectorNumElements();
+  // Ensure the reduction size is a power of 2.
+  if (!isPowerOf2_32(ElemNum))
+    return false;
+
+  unsigned ElemNumToReduce = ElemNum;
+
+  // Do DFS search on the def-use chain from the given instruction. We only
+  // allow four kinds of operations during the search until we reach the
+  // instruction that extracts the first element from the vector:
+  //
+  //   1. The reduction operation of the same opcode as the given instruction.
+  //
+  //   2. PHI node.
+  //
+  //   3. ShuffleVector instruction together with a reduction operation that
+  //      does a partial reduction.
+  //
+  //   4. ExtractElement that extracts the first element from the vector, and we
+  //      stop searching the def-use chain here.
+  //
+  // 3 & 4 above perform a reduction on all elements of the vector. We push defs
+  // from 1-3 to the stack to continue the DFS. The given instruction is not
+  // a reduction operation if we meet any other instructions other than those
+  // listed above.
+
+  SmallVector<const User *, 16> UsersToVisit{&BO};
+  SmallPtrSet<const User *, 16> Visited;
+  bool ReduxExtracted = false;
+
+  while (!UsersToVisit.empty()) {
+    auto User = UsersToVisit.back();
+    UsersToVisit.pop_back();
+    if (!Visited.insert(User).second)
+      continue;
+
+    for (const auto *U : User->users()) {
+      auto *Inst = dyn_cast<Instruction>(U);
+      if (!Inst)
+        return false;
+
+      if (Inst->getOpcode() == Opcode || isa<PHINode>(U)) {
+        if (auto *FPOp = dyn_cast<FPMathOperator>(Inst))
+          if (!isa<PHINode>(FPOp) && !FPOp->getFastMathFlags().isFast())
+            return false;
+        UsersToVisit.push_back(U);
+      } else if (auto *ShufInst = dyn_cast<ShuffleVectorInst>(U)) {
+        // Detect the following pattern: A ShuffleVector instruction together
+        // with a reduction that do partial reduction on the first and second
+        // ElemNumToReduce / 2 elements, and store the result in
+        // ElemNumToReduce / 2 elements in another vector.
+
+        unsigned ResultElements = ShufInst->getType()->getVectorNumElements();
+        if (ResultElements < ElemNum)
+          return false;
+
+        if (ElemNumToReduce == 1)
+          return false;
+        if (!isa<UndefValue>(U->getOperand(1)))
+          return false;
+        for (unsigned i = 0; i < ElemNumToReduce / 2; ++i)
+          if (ShufInst->getMaskValue(i) != int(i + ElemNumToReduce / 2))
+            return false;
+        for (unsigned i = ElemNumToReduce / 2; i < ElemNum; ++i)
+          if (ShufInst->getMaskValue(i) != -1)
+            return false;
+
+        // There is only one user of this ShuffleVector instruction, which
+        // must be a reduction operation.
+        if (!U->hasOneUse())
+          return false;
+
+        auto *U2 = dyn_cast<BinaryOperator>(*U->user_begin());
+        if (!U2 || U2->getOpcode() != Opcode)
+          return false;
+
+        // Check operands of the reduction operation.
+        if ((U2->getOperand(0) == U->getOperand(0) && U2->getOperand(1) == U) ||
+            (U2->getOperand(1) == U->getOperand(0) && U2->getOperand(0) == U)) {
+          UsersToVisit.push_back(U2);
+          ElemNumToReduce /= 2;
+        } else
+          return false;
+      } else if (isa<ExtractElementInst>(U)) {
+        // At this moment we should have reduced all elements in the vector.
+        if (ElemNumToReduce != 1)
+          return false;
+
+        auto *Val = dyn_cast<ConstantInt>(U->getOperand(1));
+        if (!Val || !Val->isZero())
+          return false;
+
+        ReduxExtracted = true;
+      } else
+        return false;
+    }
+  }
+  return ReduxExtracted;
+}
+
+bool X86PartialReduction::tryMAddReplacement(Value *Op, BinaryOperator *Add) {
+  BasicBlock *BB = Add->getParent();
+
+  auto *BO = dyn_cast<BinaryOperator>(Op);
+  if (!BO || BO->getOpcode() != Instruction::Mul || !BO->hasOneUse() ||
+      BO->getParent() != BB)
+    return false;
+
+  Value *LHS = BO->getOperand(0);
+  Value *RHS = BO->getOperand(1);
+
+  // LHS and RHS should be only used once or if they are the same then only
+  // used twice. Only check this when SSE4.1 is enabled and we have zext/sext
+  // instructions, otherwise we use punpck to emulate zero extend in stages. The
+  // trunc/ we need to do likely won't introduce new instructions in that case.
+  if (ST->hasSSE41()) {
+    if (LHS == RHS) {
+      if (!isa<Constant>(LHS) && !LHS->hasNUses(2))
+        return false;
+    } else {
+      if (!isa<Constant>(LHS) && !LHS->hasOneUse())
+        return false;
+      if (!isa<Constant>(RHS) && !RHS->hasOneUse())
+        return false;
+    }
+  }
+
+  auto canShrinkOp = [&](Value *Op) {
+    if (isa<Constant>(Op) && ComputeNumSignBits(Op, *DL, 0, nullptr, BO) > 16)
+      return true;
+    if (auto *Cast = dyn_cast<CastInst>(Op)) {
+      if (Cast->getParent() == BB &&
+          (Cast->getOpcode() == Instruction::SExt ||
+           Cast->getOpcode() == Instruction::ZExt) &&
+          ComputeNumSignBits(Op, *DL, 0, nullptr, BO) > 16)
+        return true;
+    }
+
+    return false;
+  };
+
+  // Both Ops need to be shrinkable.
+  if (!canShrinkOp(LHS) && !canShrinkOp(RHS))
+    return false;
+
+  IRBuilder<> Builder(Add);
+
+  Type *MulTy = Op->getType();
+  unsigned NumElts = MulTy->getVectorNumElements();
+
+  // Extract even elements and odd elements and add them together. This will
+  // be pattern matched by SelectionDAG to pmaddwd. This instruction will be
+  // half the original width.
+  SmallVector<uint32_t, 16> EvenMask(NumElts / 2);
+  SmallVector<uint32_t, 16> OddMask(NumElts / 2);
+  for (int i = 0, e = NumElts / 2; i != e; ++i) {
+    EvenMask[i] = i * 2;
+    OddMask[i] = i * 2 + 1;
+  }
+  Value *EvenElts = Builder.CreateShuffleVector(BO, BO, EvenMask);
+  Value *OddElts = Builder.CreateShuffleVector(BO, BO, OddMask);
+  Value *MAdd = Builder.CreateAdd(EvenElts, OddElts);
+
+  // Concatenate zeroes to extend back to the original type.
+  SmallVector<uint32_t, 32> ConcatMask(NumElts);
+  std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
+  Value *Zero = Constant::getNullValue(MAdd->getType());
+  Value *Concat = Builder.CreateShuffleVector(MAdd, Zero, ConcatMask);
+
+  // Replaces the use of mul in the original Add with the pmaddwd and zeroes.
+  Add->replaceUsesOfWith(BO, Concat);
+  Add->setHasNoSignedWrap(false);
+  Add->setHasNoUnsignedWrap(false);
+
+  return true;
+}
+
+// Try to replace operans of this add with pmaddwd patterns.
+bool X86PartialReduction::tryMAddPattern(BinaryOperator *BO) {
+  if (!ST->hasSSE2())
+    return false;
+
+  // Need at least 8 elements.
+  if (BO->getType()->getVectorNumElements() < 8)
+    return false;
+
+  // Element type should be i32.
+  if (!BO->getType()->getVectorElementType()->isIntegerTy(32))
+    return false;
+
+  bool Changed = false;
+  Changed |= tryMAddReplacement(BO->getOperand(0), BO);
+  Changed |= tryMAddReplacement(BO->getOperand(1), BO);
+  return Changed;
+}
+
+bool X86PartialReduction::trySADReplacement(Value *Op, BinaryOperator *Add) {
+  // Operand should be a select.
+  auto *SI = dyn_cast<SelectInst>(Op);
+  if (!SI)
+    return false;
+
+  // Select needs to implement absolute value.
+  Value *LHS, *RHS;
+  auto SPR = matchSelectPattern(SI, LHS, RHS);
+  if (SPR.Flavor != SPF_ABS)
+    return false;
+
+  // Need a subtract of two values.
+  auto *Sub = dyn_cast<BinaryOperator>(LHS);
+  if (!Sub || Sub->getOpcode() != Instruction::Sub)
+    return false;
+
+  // Look for zero extend from i8.
+  auto getZeroExtendedVal = [](Value *Op) -> Value * {
+    if (auto *ZExt = dyn_cast<ZExtInst>(Op))
+      if (ZExt->getOperand(0)->getType()->getVectorElementType()->isIntegerTy(8))
+        return ZExt->getOperand(0);
+
+    return nullptr;
+  };
+
+  // Both operands of the subtract should be extends from vXi8.
+  Value *Op0 = getZeroExtendedVal(Sub->getOperand(0));
+  Value *Op1 = getZeroExtendedVal(Sub->getOperand(1));
+  if (!Op0 || !Op1)
+    return false;
+
+  IRBuilder<> Builder(Add);
+
+  Type *OpTy = Op->getType();
+  unsigned NumElts = OpTy->getVectorNumElements();
+
+  unsigned IntrinsicNumElts;
+  Intrinsic::ID IID;
+  if (ST->hasBWI() && NumElts >= 64) {
+    IID = Intrinsic::x86_avx512_psad_bw_512;
+    IntrinsicNumElts = 64;
+  } else if (ST->hasAVX2() && NumElts >= 32) {
+    IID = Intrinsic::x86_avx2_psad_bw;
+    IntrinsicNumElts = 32;
+  } else {
+    IID = Intrinsic::x86_sse2_psad_bw;
+    IntrinsicNumElts = 16;
+  }
+
+  Function *PSADBWFn = Intrinsic::getDeclaration(Add->getModule(), IID);
+
+  if (NumElts < 16) {
+    // Pad input with zeroes.
+    SmallVector<uint32_t, 32> ConcatMask(16);
+    for (unsigned i = 0; i != NumElts; ++i)
+      ConcatMask[i] = i;
+    for (unsigned i = NumElts; i != 16; ++i)
+      ConcatMask[i] = (i % NumElts) + NumElts;
+
+    Value *Zero = Constant::getNullValue(Op0->getType());
+    Op0 = Builder.CreateShuffleVector(Op0, Zero, ConcatMask);
+    Op1 = Builder.CreateShuffleVector(Op1, Zero, ConcatMask);
+    NumElts = 16;
+  }
+
+  // Intrinsics produce vXi64 and need to be casted to vXi32.
+  Type *I32Ty = VectorType::get(Builder.getInt32Ty(), IntrinsicNumElts / 4);
+
+  assert(NumElts % IntrinsicNumElts == 0 && "Unexpected number of elements!");
+  unsigned NumSplits = NumElts / IntrinsicNumElts;
+
+  // First collect the pieces we need.
+  SmallVector<Value *, 4> Ops(NumSplits);
+  for (unsigned i = 0; i != NumSplits; ++i) {
+    SmallVector<uint32_t, 64> ExtractMask(IntrinsicNumElts);
+    std::iota(ExtractMask.begin(), ExtractMask.end(), i * IntrinsicNumElts);
+    Value *ExtractOp0 = Builder.CreateShuffleVector(Op0, Op0, ExtractMask);
+    Value *ExtractOp1 = Builder.CreateShuffleVector(Op1, Op0, ExtractMask);
+    Ops[i] = Builder.CreateCall(PSADBWFn, {ExtractOp0, ExtractOp1});
+    Ops[i] = Builder.CreateBitCast(Ops[i], I32Ty);
+  }
+
+  assert(isPowerOf2_32(NumSplits) && "Expected power of 2 splits");
+  unsigned Stages = Log2_32(NumSplits);
+  for (unsigned s = Stages; s > 0; --s) {
+    unsigned NumConcatElts = Ops[0]->getType()->getVectorNumElements() * 2;
+    for (unsigned i = 0; i != 1U << (s - 1); ++i) {
+      SmallVector<uint32_t, 64> ConcatMask(NumConcatElts);
+      std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
+      Ops[i] = Builder.CreateShuffleVector(Ops[i*2], Ops[i*2+1], ConcatMask);
+    }
+  }
+
+  // At this point the final value should be in Ops[0]. Now we need to adjust
+  // it to the final original type.
+  NumElts = OpTy->getVectorNumElements();
+  if (NumElts == 2) {
+    // Extract down to 2 elements.
+    Ops[0] = Builder.CreateShuffleVector(Ops[0], Ops[0], ArrayRef<int>{0, 1});
+  } else if (NumElts >= 8) {
+    SmallVector<uint32_t, 32> ConcatMask(NumElts);
+    unsigned SubElts = Ops[0]->getType()->getVectorNumElements();
+    for (unsigned i = 0; i != SubElts; ++i)
+      ConcatMask[i] = i;
+    for (unsigned i = SubElts; i != NumElts; ++i)
+      ConcatMask[i] = (i % SubElts) + SubElts;
+
+    Value *Zero = Constant::getNullValue(Ops[0]->getType());
+    Ops[0] = Builder.CreateShuffleVector(Ops[0], Zero, ConcatMask);
+  }
+
+  // Replaces the uses of Op in Add with the new sequence.
+  Add->replaceUsesOfWith(Op, Ops[0]);
+  Add->setHasNoSignedWrap(false);
+  Add->setHasNoUnsignedWrap(false);
+
+  return false;
+}
+
+bool X86PartialReduction::trySADPattern(BinaryOperator *BO) {
+  if (!ST->hasSSE2())
+    return false;
+
+  // TODO: There's nothing special about i32, any integer type above i16 should
+  // work just as well.
+  if (!BO->getType()->getVectorElementType()->isIntegerTy(32))
+    return false;
+
+  bool Changed = false;
+  Changed |= trySADReplacement(BO->getOperand(0), BO);
+  Changed |= trySADReplacement(BO->getOperand(1), BO);
+  return Changed;
+}
+
+bool X86PartialReduction::runOnFunction(Function &F) {
+  if (skipFunction(F))
+    return false;
+
+  auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
+  if (!TPC)
+    return false;
+
+  auto &TM = TPC->getTM<X86TargetMachine>();
+  ST = TM.getSubtargetImpl(F);
+
+  DL = &F.getParent()->getDataLayout();
+
+  bool MadeChange = false;
+  for (auto &BB : F) {
+    for (auto &I : BB) {
+      auto *BO = dyn_cast<BinaryOperator>(&I);
+      if (!BO)
+        continue;
+
+      if (!isVectorReductionOp(*BO))
+        continue;
+
+      if (BO->getOpcode() == Instruction::Add) {
+        if (tryMAddPattern(BO)) {
+          MadeChange = true;
+          continue;
+        }
+        if (trySADPattern(BO)) {
+          MadeChange = true;
+          continue;
+        }
+      }
+    }
+  }
+
+  return MadeChange;
+}
diff --git a/llvm/lib/Target/X86/X86RetpolineThunks.cpp b/llvm/lib/Target/X86/X86RetpolineThunks.cpp
deleted file mode 100644
index 9085d7f068ac4..0000000000000
--- a/llvm/lib/Target/X86/X86RetpolineThunks.cpp
+++ /dev/null
@@ -1,286 +0,0 @@
-//======- X86RetpolineThunks.cpp - Construct retpoline thunks for x86  --=====//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-/// \file
-///
-/// Pass that injects an MI thunk implementing a "retpoline". This is
-/// a RET-implemented trampoline that is used to lower indirect calls in a way
-/// that prevents speculation on some x86 processors and can be used to mitigate
-/// security vulnerabilities due to targeted speculative execution and side
-/// channels such as CVE-2017-5715.
-///
-/// TODO(chandlerc): All of this code could use better comments and
-/// documentation.
-///
-//===----------------------------------------------------------------------===//
-
-#include "X86.h"
-#include "X86InstrBuilder.h"
-#include "X86Subtarget.h"
-#include "llvm/CodeGen/MachineFunction.h"
-#include "llvm/CodeGen/MachineInstrBuilder.h"
-#include "llvm/CodeGen/MachineModuleInfo.h"
-#include "llvm/CodeGen/Passes.h"
-#include "llvm/CodeGen/TargetPassConfig.h"
-#include "llvm/IR/IRBuilder.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/IR/Module.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/raw_ostream.h"
-
-using namespace llvm;
-
-#define DEBUG_TYPE "x86-retpoline-thunks"
-
-static const char ThunkNamePrefix[] = "__llvm_retpoline_";
-static const char R11ThunkName[]    = "__llvm_retpoline_r11";
-static const char EAXThunkName[]    = "__llvm_retpoline_eax";
-static const char ECXThunkName[]    = "__llvm_retpoline_ecx";
-static const char EDXThunkName[]    = "__llvm_retpoline_edx";
-static const char EDIThunkName[]    = "__llvm_retpoline_edi";
-
-namespace {
-class X86RetpolineThunks : public MachineFunctionPass {
-public:
-  static char ID;
-
-  X86RetpolineThunks() : MachineFunctionPass(ID) {}
-
-  StringRef getPassName() const override { return "X86 Retpoline Thunks"; }
-
-  bool doInitialization(Module &M) override;
-  bool runOnMachineFunction(MachineFunction &F) override;
-
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    MachineFunctionPass::getAnalysisUsage(AU);
-    AU.addRequired<MachineModuleInfoWrapperPass>();
-    AU.addPreserved<MachineModuleInfoWrapperPass>();
-  }
-
-private:
-  MachineModuleInfo *MMI = nullptr;
-  const TargetMachine *TM = nullptr;
-  bool Is64Bit = false;
-  const X86Subtarget *STI = nullptr;
-  const X86InstrInfo *TII = nullptr;
-
-  bool InsertedThunks = false;
-
-  void createThunkFunction(Module &M, StringRef Name);
-  void insertRegReturnAddrClobber(MachineBasicBlock &MBB, unsigned Reg);
-  void populateThunk(MachineFunction &MF, unsigned Reg);
-};
-
-} // end anonymous namespace
-
-FunctionPass *llvm::createX86RetpolineThunksPass() {
-  return new X86RetpolineThunks();
-}
-
-char X86RetpolineThunks::ID = 0;
-
-bool X86RetpolineThunks::doInitialization(Module &M) {
-  InsertedThunks = false;
-  return false;
-}
-
-bool X86RetpolineThunks::runOnMachineFunction(MachineFunction &MF) {
-  LLVM_DEBUG(dbgs() << getPassName() << '\n');
-
-  TM = &MF.getTarget();;
-  STI = &MF.getSubtarget<X86Subtarget>();
-  TII = STI->getInstrInfo();
-  Is64Bit = TM->getTargetTriple().getArch() == Triple::x86_64;
-
-  MMI = &getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
-  Module &M = const_cast<Module &>(*MMI->getModule());
-
-  // If this function is not a thunk, check to see if we need to insert
-  // a thunk.
-  if (!MF.getName().startswith(ThunkNamePrefix)) {
-    // If we've already inserted a thunk, nothing else to do.
-    if (InsertedThunks)
-      return false;
-
-    // Only add a thunk if one of the functions has the retpoline feature
-    // enabled in its subtarget, and doesn't enable external thunks.
-    // FIXME: Conditionalize on indirect calls so we don't emit a thunk when
-    // nothing will end up calling it.
-    // FIXME: It's a little silly to look at every function just to enumerate
-    // the subtargets, but eventually we'll want to look at them for indirect
-    // calls, so maybe this is OK.
-    if ((!STI->useRetpolineIndirectCalls() &&
-         !STI->useRetpolineIndirectBranches()) ||
-        STI->useRetpolineExternalThunk())
-      return false;
-
-    // Otherwise, we need to insert the thunk.
-    // WARNING: This is not really a well behaving thing to do in a function
-    // pass. We extract the module and insert a new function (and machine
-    // function) directly into the module.
-    if (Is64Bit)
-      createThunkFunction(M, R11ThunkName);
-    else
-      for (StringRef Name :
-           {EAXThunkName, ECXThunkName, EDXThunkName, EDIThunkName})
-        createThunkFunction(M, Name);
-    InsertedThunks = true;
-    return true;
-  }
-
-  // If this *is* a thunk function, we need to populate it with the correct MI.
-  if (Is64Bit) {
-    assert(MF.getName() == "__llvm_retpoline_r11" &&
-           "Should only have an r11 thunk on 64-bit targets");
-
-    // __llvm_retpoline_r11:
-    //   callq .Lr11_call_target
-    // .Lr11_capture_spec:
-    //   pause
-    //   lfence
-    //   jmp .Lr11_capture_spec
-    // .align 16
-    // .Lr11_call_target:
-    //   movq %r11, (%rsp)
-    //   retq
-    populateThunk(MF, X86::R11);
-  } else {
-    // For 32-bit targets we need to emit a collection of thunks for various
-    // possible scratch registers as well as a fallback that uses EDI, which is
-    // normally callee saved.
-    //   __llvm_retpoline_eax:
-    //         calll .Leax_call_target
-    //   .Leax_capture_spec:
-    //         pause
-    //         jmp .Leax_capture_spec
-    //   .align 16
-    //   .Leax_call_target:
-    //         movl %eax, (%esp)  # Clobber return addr
-    //         retl
-    //
-    //   __llvm_retpoline_ecx:
-    //   ... # Same setup
-    //         movl %ecx, (%esp)
-    //         retl
-    //
-    //   __llvm_retpoline_edx:
-    //   ... # Same setup
-    //         movl %edx, (%esp)
-    //         retl
-    //
-    //   __llvm_retpoline_edi:
-    //   ... # Same setup
-    //         movl %edi, (%esp)
-    //         retl
-    if (MF.getName() == EAXThunkName)
-      populateThunk(MF, X86::EAX);
-    else if (MF.getName() == ECXThunkName)
-      populateThunk(MF, X86::ECX);
-    else if (MF.getName() == EDXThunkName)
-      populateThunk(MF, X86::EDX);
-    else if (MF.getName() == EDIThunkName)
-      populateThunk(MF, X86::EDI);
-    else
-      llvm_unreachable("Invalid thunk name on x86-32!");
-  }
-
-  return true;
-}
-
-void X86RetpolineThunks::createThunkFunction(Module &M, StringRef Name) {
-  assert(Name.startswith(ThunkNamePrefix) &&
-         "Created a thunk with an unexpected prefix!");
-
-  LLVMContext &Ctx = M.getContext();
-  auto Type = FunctionType::get(Type::getVoidTy(Ctx), false);
-  Function *F =
-      Function::Create(Type, GlobalValue::LinkOnceODRLinkage, Name, &M);
-  F->setVisibility(GlobalValue::HiddenVisibility);
-  F->setComdat(M.getOrInsertComdat(Name));
-
-  // Add Attributes so that we don't create a frame, unwind information, or
-  // inline.
-  AttrBuilder B;
-  B.addAttribute(llvm::Attribute::NoUnwind);
-  B.addAttribute(llvm::Attribute::Naked);
-  F->addAttributes(llvm::AttributeList::FunctionIndex, B);
-
-  // Populate our function a bit so that we can verify.
-  BasicBlock *Entry = BasicBlock::Create(Ctx, "entry", F);
-  IRBuilder<> Builder(Entry);
-
-  Builder.CreateRetVoid();
-
-  // MachineFunctions/MachineBasicBlocks aren't created automatically for the
-  // IR-level constructs we already made. Create them and insert them into the
-  // module.
-  MachineFunction &MF = MMI->getOrCreateMachineFunction(*F);
-  MachineBasicBlock *EntryMBB = MF.CreateMachineBasicBlock(Entry);
-
-  // Insert EntryMBB into MF. It's not in the module until we do this.
-  MF.insert(MF.end(), EntryMBB);
-}
-
-void X86RetpolineThunks::insertRegReturnAddrClobber(MachineBasicBlock &MBB,
-                                                    unsigned Reg) {
-  const unsigned MovOpc = Is64Bit ? X86::MOV64mr : X86::MOV32mr;
-  const unsigned SPReg = Is64Bit ? X86::RSP : X86::ESP;
-  addRegOffset(BuildMI(&MBB, DebugLoc(), TII->get(MovOpc)), SPReg, false, 0)
-      .addReg(Reg);
-}
-
-void X86RetpolineThunks::populateThunk(MachineFunction &MF,
-                                       unsigned Reg) {
-  // Set MF properties. We never use vregs...
-  MF.getProperties().set(MachineFunctionProperties::Property::NoVRegs);
-
-  // Grab the entry MBB and erase any other blocks. O0 codegen appears to
-  // generate two bbs for the entry block.
-  MachineBasicBlock *Entry = &MF.front();
-  Entry->clear();
-  while (MF.size() > 1)
-    MF.erase(std::next(MF.begin()));
-
-  MachineBasicBlock *CaptureSpec = MF.CreateMachineBasicBlock(Entry->getBasicBlock());
-  MachineBasicBlock *CallTarget = MF.CreateMachineBasicBlock(Entry->getBasicBlock());
-  MCSymbol *TargetSym = MF.getContext().createTempSymbol();
-  MF.push_back(CaptureSpec);
-  MF.push_back(CallTarget);
-
-  const unsigned CallOpc = Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32;
-  const unsigned RetOpc = Is64Bit ? X86::RETQ : X86::RETL;
-
-  Entry->addLiveIn(Reg);
-  BuildMI(Entry, DebugLoc(), TII->get(CallOpc)).addSym(TargetSym);
-
-  // The MIR verifier thinks that the CALL in the entry block will fall through
-  // to CaptureSpec, so mark it as the successor. Technically, CaptureTarget is
-  // the successor, but the MIR verifier doesn't know how to cope with that.
-  Entry->addSuccessor(CaptureSpec);
-
-  // In the capture loop for speculation, we want to stop the processor from
-  // speculating as fast as possible. On Intel processors, the PAUSE instruction
-  // will block speculation without consuming any execution resources. On AMD
-  // processors, the PAUSE instruction is (essentially) a nop, so we also use an
-  // LFENCE instruction which they have advised will stop speculation as well
-  // with minimal resource utilization. We still end the capture with a jump to
-  // form an infinite loop to fully guarantee that no matter what implementation
-  // of the x86 ISA, speculating this code path never escapes.
-  BuildMI(CaptureSpec, DebugLoc(), TII->get(X86::PAUSE));
-  BuildMI(CaptureSpec, DebugLoc(), TII->get(X86::LFENCE));
-  BuildMI(CaptureSpec, DebugLoc(), TII->get(X86::JMP_1)).addMBB(CaptureSpec);
-  CaptureSpec->setHasAddressTaken();
-  CaptureSpec->addSuccessor(CaptureSpec);
-
-  CallTarget->addLiveIn(Reg);
-  CallTarget->setHasAddressTaken();
-  CallTarget->setAlignment(Align(16));
-  insertRegReturnAddrClobber(*CallTarget, Reg);
-  CallTarget->back().setPreInstrSymbol(MF, TargetSym);
-  BuildMI(CallTarget, DebugLoc(), TII->get(RetOpc));
-}
diff --git a/llvm/lib/Target/X86/X86Subtarget.h b/llvm/lib/Target/X86/X86Subtarget.h
index 93801455254c4..b4ad50d88062f 100644
--- a/llvm/lib/Target/X86/X86Subtarget.h
+++ b/llvm/lib/Target/X86/X86Subtarget.h
@@ -397,6 +397,9 @@ class X86Subtarget final : public X86GenSubtargetInfo {
   /// Processor supports PCONFIG instruction
   bool HasPCONFIG = false;
 
+  /// Processor supports SERIALIZE instruction
+  bool HasSERIALIZE = false;
+
   /// Processor has a single uop BEXTR implementation.
   bool HasFastBEXTR = false;
 
@@ -425,6 +428,16 @@ class X86Subtarget final : public X86GenSubtargetInfo {
   /// than emitting one inside the compiler.
   bool UseRetpolineExternalThunk = false;
 
+  /// Prevent generation of indirect call/branch instructions from memory,
+  /// and force all indirect call/branch instructions from a register to be
+  /// preceded by an LFENCE. Also decompose RET instructions into a
+  /// POP+LFENCE+JMP sequence.
+  bool UseLVIControlFlowIntegrity = false;
+
+  /// Insert LFENCE instructions to prevent data speculatively injected into
+  /// loads from being used maliciously.
+  bool UseLVILoadHardening = false;
+
   /// Use software floating point for code generation.
   bool UseSoftFloat = false;
 
@@ -706,13 +719,27 @@ class X86Subtarget final : public X86GenSubtargetInfo {
   bool threewayBranchProfitable() const { return ThreewayBranchProfitable; }
   bool hasINVPCID() const { return HasINVPCID; }
   bool hasENQCMD() const { return HasENQCMD; }
+  bool hasSERIALIZE() const { return HasSERIALIZE; }
   bool useRetpolineIndirectCalls() const { return UseRetpolineIndirectCalls; }
   bool useRetpolineIndirectBranches() const {
     return UseRetpolineIndirectBranches;
   }
   bool useRetpolineExternalThunk() const { return UseRetpolineExternalThunk; }
+
+  // These are generic getters that OR together all of the thunk types
+  // supported by the subtarget. Therefore useIndirectThunk*() will return true
+  // if any respective thunk feature is enabled.
+  bool useIndirectThunkCalls() const {
+    return useRetpolineIndirectCalls() || useLVIControlFlowIntegrity();
+  }
+  bool useIndirectThunkBranches() const {
+    return useRetpolineIndirectBranches() || useLVIControlFlowIntegrity();
+  }
+
   bool preferMaskRegisters() const { return PreferMaskRegisters; }
   bool useGLMDivSqrtCosts() const { return UseGLMDivSqrtCosts; }
+  bool useLVIControlFlowIntegrity() const { return UseLVIControlFlowIntegrity; }
+  bool useLVILoadHardening() const { return UseLVILoadHardening; }
 
   unsigned getPreferVectorWidth() const { return PreferVectorWidth; }
   unsigned getRequiredVectorWidth() const { return RequiredVectorWidth; }
@@ -857,10 +884,10 @@ class X86Subtarget final : public X86GenSubtargetInfo {
   /// Return true if the subtarget allows calls to immediate address.
   bool isLegalToCallImmediateAddr() const;
 
-  /// If we are using retpolines, we need to expand indirectbr to avoid it
+  /// If we are using indirect thunks, we need to expand indirectbr to avoid it
   /// lowering to an actual indirect jump.
   bool enableIndirectBrExpand() const override {
-    return useRetpolineIndirectBranches();
+    return useIndirectThunkBranches();
   }
 
   /// Enable the MachineScheduler pass for all X86 subtargets.
diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp
index dd6b67865ac0d..8446ee9f4e03a 100644
--- a/llvm/lib/Target/X86/X86TargetMachine.cpp
+++ b/llvm/lib/Target/X86/X86TargetMachine.cpp
@@ -83,7 +83,10 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeX86Target() {
   initializeX86SpeculativeLoadHardeningPassPass(PR);
   initializeX86FlagsCopyLoweringPassPass(PR);
   initializeX86CondBrFoldingPassPass(PR);
+  initializeX86LoadValueInjectionLoadHardeningPassPass(PR);
+  initializeX86LoadValueInjectionRetHardeningPassPass(PR);
   initializeX86OptimizeLEAPassPass(PR);
+  initializeX86PartialReductionPass(PR);
 }
 
 static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
@@ -213,7 +216,7 @@ X86TargetMachine::X86TargetMachine(const Target &T, const Triple &TT,
           getEffectiveRelocModel(TT, JIT, RM),
           getEffectiveX86CodeModel(CM, JIT, TT.getArch() == Triple::x86_64),
           OL),
-      TLOF(createTLOF(getTargetTriple())) {
+      TLOF(createTLOF(getTargetTriple())), IsJIT(JIT) {
   // On PS4, the "return address" of a 'noreturn' call must still be within
   // the calling function, and TrapUnreachable is an easy way to get that.
   if (TT.isPS4() || TT.isOSBinFormatMachO()) {
@@ -400,8 +403,10 @@ void X86PassConfig::addIRPasses() {
 
   TargetPassConfig::addIRPasses();
 
-  if (TM->getOptLevel() != CodeGenOpt::None)
+  if (TM->getOptLevel() != CodeGenOpt::None) {
     addPass(createInterleavedAccessPass());
+    addPass(createX86PartialReductionPass());
+  }
 
   // Add passes that handle indirect branch removal and insertion of a retpoline
   // thunk. These will be a no-op unless a function subtarget has the retpoline
@@ -490,6 +495,7 @@ void X86PassConfig::addMachineSSAOptimization() {
 
 void X86PassConfig::addPostRegAlloc() {
   addPass(createX86FloatingPointStackifierPass());
+  addPass(createX86LoadValueInjectionLoadHardeningPass());
 }
 
 void X86PassConfig::addPreSched2() { addPass(createX86ExpandPseudoPass()); }
@@ -520,7 +526,7 @@ void X86PassConfig::addPreEmitPass2() {
   const Triple &TT = TM->getTargetTriple();
   const MCAsmInfo *MAI = TM->getMCAsmInfo();
 
-  addPass(createX86RetpolineThunksPass());
+  addPass(createX86IndirectThunksPass());
 
   // Insert extra int3 instructions after trailing call instructions to avoid
   // issues in the unwinder.
@@ -537,6 +543,7 @@ void X86PassConfig::addPreEmitPass2() {
   // Identify valid longjmp targets for Windows Control Flow Guard.
   if (TT.isOSWindows())
     addPass(createCFGuardLongjmpPass());
+  addPass(createX86LoadValueInjectionRetHardeningPass());
 }
 
 std::unique_ptr<CSEConfigBase> X86PassConfig::getCSEConfig() const {
diff --git a/llvm/lib/Target/X86/X86TargetMachine.h b/llvm/lib/Target/X86/X86TargetMachine.h
index ec3db7b1e9e87..757ce8bc5c727 100644
--- a/llvm/lib/Target/X86/X86TargetMachine.h
+++ b/llvm/lib/Target/X86/X86TargetMachine.h
@@ -30,6 +30,8 @@ class TargetTransformInfo;
 class X86TargetMachine final : public LLVMTargetMachine {
   std::unique_ptr<TargetLoweringObjectFile> TLOF;
   mutable StringMap<std::unique_ptr<X86Subtarget>> SubtargetMap;
+  // True if this is used in JIT.
+  bool IsJIT;
 
 public:
   X86TargetMachine(const Target &T, const Triple &TT, StringRef CPU,
@@ -52,6 +54,8 @@ class X86TargetMachine final : public LLVMTargetMachine {
   TargetLoweringObjectFile *getObjFileLowering() const override {
     return TLOF.get();
   }
+
+  bool isJIT() const { return IsJIT; }
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 3fc61541d5116..cca9cdc250fc5 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -3639,6 +3639,8 @@ X86TTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
   TTI::MemCmpExpansionOptions Options;
   Options.MaxNumLoads = TLI->getMaxExpandSizeMemcmp(OptSize);
   Options.NumLoadsPerBlock = 2;
+  // All GPR and vector loads can be unaligned.
+  Options.AllowOverlappingLoads = true;
   if (IsZeroCmp) {
     // Only enable vector loads for equality comparison. Right now the vector
     // version is not as fast for three way compare (see #33329).
@@ -3646,8 +3648,6 @@ X86TTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
     if (PreferredWidth >= 512 && ST->hasAVX512()) Options.LoadSizes.push_back(64);
     if (PreferredWidth >= 256 && ST->hasAVX()) Options.LoadSizes.push_back(32);
     if (PreferredWidth >= 128 && ST->hasSSE2()) Options.LoadSizes.push_back(16);
-    // All GPR and vector loads can be unaligned.
-    Options.AllowOverlappingLoads = true;
   }
   if (ST->is64Bit()) {
     Options.LoadSizes.push_back(8);
diff --git a/llvm/lib/Target/XCore/XCoreFrameLowering.cpp b/llvm/lib/Target/XCore/XCoreFrameLowering.cpp
index 753ada0b8000b..1a3790d64cd0a 100644
--- a/llvm/lib/Target/XCore/XCoreFrameLowering.cpp
+++ b/llvm/lib/Target/XCore/XCoreFrameLowering.cpp
@@ -179,7 +179,7 @@ static MachineMemOperand *getFrameIndexMMO(MachineBasicBlock &MBB,
   const MachineFrameInfo &MFI = MF->getFrameInfo();
   MachineMemOperand *MMO = MF->getMachineMemOperand(
       MachinePointerInfo::getFixedStack(*MF, FrameIndex), flags,
-      MFI.getObjectSize(FrameIndex), MFI.getObjectAlignment(FrameIndex));
+      MFI.getObjectSize(FrameIndex), MFI.getObjectAlign(FrameIndex));
   return MMO;
 }
 
diff --git a/llvm/lib/Target/XCore/XCoreISelDAGToDAG.cpp b/llvm/lib/Target/XCore/XCoreISelDAGToDAG.cpp
index b1f9717fbddc0..b300697cc5ae5 100644
--- a/llvm/lib/Target/XCore/XCoreISelDAGToDAG.cpp
+++ b/llvm/lib/Target/XCore/XCoreISelDAGToDAG.cpp
@@ -152,7 +152,7 @@ void XCoreDAGToDAGISel::Select(SDNode *N) {
                                             CurDAG->getEntryNode());
       MachineMemOperand *MemOp =
           MF->getMachineMemOperand(MachinePointerInfo::getConstantPool(*MF),
-                                   MachineMemOperand::MOLoad, 4, 4);
+                                   MachineMemOperand::MOLoad, 4, Align(4));
       CurDAG->setNodeMemRefs(cast<MachineSDNode>(node), {MemOp});
       ReplaceNode(N, node);
       return;
diff --git a/llvm/lib/Target/XCore/XCoreInstrInfo.cpp b/llvm/lib/Target/XCore/XCoreInstrInfo.cpp
index 5b0d24f2317f1..c6b2111335858 100644
--- a/llvm/lib/Target/XCore/XCoreInstrInfo.cpp
+++ b/llvm/lib/Target/XCore/XCoreInstrInfo.cpp
@@ -370,7 +370,7 @@ void XCoreInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
   MachineMemOperand *MMO = MF->getMachineMemOperand(
       MachinePointerInfo::getFixedStack(*MF, FrameIndex),
       MachineMemOperand::MOStore, MFI.getObjectSize(FrameIndex),
-      MFI.getObjectAlignment(FrameIndex));
+      MFI.getObjectAlign(FrameIndex));
   BuildMI(MBB, I, DL, get(XCore::STWFI))
     .addReg(SrcReg, getKillRegState(isKill))
     .addFrameIndex(FrameIndex)
@@ -392,7 +392,7 @@ void XCoreInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
   MachineMemOperand *MMO = MF->getMachineMemOperand(
       MachinePointerInfo::getFixedStack(*MF, FrameIndex),
       MachineMemOperand::MOLoad, MFI.getObjectSize(FrameIndex),
-      MFI.getObjectAlignment(FrameIndex));
+      MFI.getObjectAlign(FrameIndex));
   BuildMI(MBB, I, DL, get(XCore::LDWFI), DestReg)
     .addFrameIndex(FrameIndex)
     .addImm(0)
diff --git a/llvm/lib/TextAPI/MachO/TextStub.cpp b/llvm/lib/TextAPI/MachO/TextStub.cpp
index 5637639b8ef83..57e3318f99d85 100644
--- a/llvm/lib/TextAPI/MachO/TextStub.cpp
+++ b/llvm/lib/TextAPI/MachO/TextStub.cpp
@@ -450,10 +450,8 @@ template <> struct MappingTraits<const InterfaceFile *> {
       if (File->isInstallAPI())
         Flags |= TBDFlags::InstallAPI;
 
-      for (const auto &Iter : File->umbrellas()) {
-        ParentUmbrella = Iter.second;
-        break;
-      }
+      if (!File->umbrellas().empty())
+        ParentUmbrella = File->umbrellas().begin()->second;
 
       std::set<ArchitectureSet> ArchSet;
       for (const auto &Library : File->allowableClients())
@@ -1120,6 +1118,10 @@ TextAPIReader::get(MemoryBufferRef InputBuffer) {
   auto File = std::unique_ptr<InterfaceFile>(
       const_cast<InterfaceFile *>(Files.front()));
 
+  for (auto Iter = std::next(Files.begin()); Iter != Files.end(); ++Iter)
+    File->addDocument(
+        std::shared_ptr<InterfaceFile>(const_cast<InterfaceFile *>(*Iter)));
+
   if (YAMLIn.error())
     return make_error<StringError>(Ctx.ErrorMessage, YAMLIn.error());
 
@@ -1135,6 +1137,9 @@ Error TextAPIWriter::writeToStream(raw_ostream &OS, const InterfaceFile &File) {
   std::vector<const InterfaceFile *> Files;
   Files.emplace_back(&File);
 
+  for (auto Document : File.documents())
+    Files.emplace_back(Document.get());
+
   // Stream out yaml.
   YAMLOut << Files;
 
diff --git a/llvm/lib/Transforms/Coroutines/CoroEarly.cpp b/llvm/lib/Transforms/Coroutines/CoroEarly.cpp
index da8eb36e7245e..55cca12acd0cb 100644
--- a/llvm/lib/Transforms/Coroutines/CoroEarly.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroEarly.cpp
@@ -61,14 +61,14 @@ void Lowerer::lowerResumeOrDestroy(CallSite CS,
 // TODO: Handle the case when coroutine promise alloca has align override.
 void Lowerer::lowerCoroPromise(CoroPromiseInst *Intrin) {
   Value *Operand = Intrin->getArgOperand(0);
-  unsigned Alignement = Intrin->getAlignment();
+  Align Alignment = Intrin->getAlignment();
   Type *Int8Ty = Builder.getInt8Ty();
 
   auto *SampleStruct =
       StructType::get(Context, {AnyResumeFnPtrTy, AnyResumeFnPtrTy, Int8Ty});
   const DataLayout &DL = TheModule.getDataLayout();
   int64_t Offset = alignTo(
-      DL.getStructLayout(SampleStruct)->getElementOffset(2), Alignement);
+      DL.getStructLayout(SampleStruct)->getElementOffset(2), Alignment);
   if (Intrin->isFromPromise())
     Offset = -Offset;
 
diff --git a/llvm/lib/Transforms/Coroutines/CoroElide.cpp b/llvm/lib/Transforms/Coroutines/CoroElide.cpp
index ecf5450a1861f..e89c890386b50 100644
--- a/llvm/lib/Transforms/Coroutines/CoroElide.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroElide.cpp
@@ -34,7 +34,8 @@ struct Lowerer : coro::LowererBase {
 
   Lowerer(Module &M) : LowererBase(M) {}
 
-  void elideHeapAllocations(Function *F, Type *FrameTy, AAResults &AA);
+  void elideHeapAllocations(Function *F, uint64_t FrameSize,
+                            MaybeAlign FrameAlign, AAResults &AA);
   bool shouldElide(Function *F, DominatorTree &DT) const;
   void collectPostSplitCoroIds(Function *F);
   bool processCoroId(CoroIdInst *, AAResults &AA, DominatorTree &DT);
@@ -92,10 +93,23 @@ static void removeTailCallAttribute(AllocaInst *Frame, AAResults &AA) {
       }
 }
 
-// Given a resume function @f.resume(%f.frame* %frame), returns %f.frame type.
-static Type *getFrameType(Function *Resume) {
-  auto *ArgType = Resume->arg_begin()->getType();
-  return cast<PointerType>(ArgType)->getElementType();
+// Given a resume function @f.resume(%f.frame* %frame), returns the size
+// and expected alignment of %f.frame type.
+static std::pair<uint64_t, MaybeAlign> getFrameLayout(Function *Resume) {
+  // Prefer to pull information from the function attributes.
+  auto Size = Resume->getParamDereferenceableBytes(0);
+  auto Align = Resume->getParamAlign(0);
+
+  // If those aren't given, extract them from the type.
+  if (Size == 0 || !Align) {
+    auto *FrameTy = Resume->arg_begin()->getType()->getPointerElementType();
+
+    const DataLayout &DL = Resume->getParent()->getDataLayout();
+    if (!Size) Size = DL.getTypeAllocSize(FrameTy);
+    if (!Align) Align = DL.getABITypeAlign(FrameTy);
+  }
+
+  return std::make_pair(Size, Align);
 }
 
 // Finds first non alloca instruction in the entry block of a function.
@@ -108,8 +122,9 @@ static Instruction *getFirstNonAllocaInTheEntryBlock(Function *F) {
 
 // To elide heap allocations we need to suppress code blocks guarded by
 // llvm.coro.alloc and llvm.coro.free instructions.
-void Lowerer::elideHeapAllocations(Function *F, Type *FrameTy, AAResults &AA) {
-  LLVMContext &C = FrameTy->getContext();
+void Lowerer::elideHeapAllocations(Function *F, uint64_t FrameSize,
+                                   MaybeAlign FrameAlign, AAResults &AA) {
+  LLVMContext &C = F->getContext();
   auto *InsertPt =
       getFirstNonAllocaInTheEntryBlock(CoroIds.front()->getFunction());
 
@@ -130,7 +145,9 @@ void Lowerer::elideHeapAllocations(Function *F, Type *FrameTy, AAResults &AA) {
   // here. Possibly we will need to do a mini SROA here and break the coroutine
   // frame into individual AllocaInst recreating the original alignment.
   const DataLayout &DL = F->getParent()->getDataLayout();
+  auto FrameTy = ArrayType::get(Type::getInt8Ty(C), FrameSize);
   auto *Frame = new AllocaInst(FrameTy, DL.getAllocaAddrSpace(), "", InsertPt);
+  Frame->setAlignment(FrameAlign);
   auto *FrameVoidPtr =
       new BitCastInst(Frame, Type::getInt8PtrTy(C), "vFrame", InsertPt);
 
@@ -319,8 +336,9 @@ bool Lowerer::processCoroId(CoroIdInst *CoroId, AAResults &AA,
     replaceWithConstant(DestroyAddrConstant, It.second);
 
   if (ShouldElide) {
-    auto *FrameTy = getFrameType(cast<Function>(ResumeAddrConstant));
-    elideHeapAllocations(CoroId->getFunction(), FrameTy, AA);
+    auto FrameSizeAndAlign = getFrameLayout(cast<Function>(ResumeAddrConstant));
+    elideHeapAllocations(CoroId->getFunction(), FrameSizeAndAlign.first,
+                         FrameSizeAndAlign.second, AA);
     coro::replaceCoroFree(CoroId, /*Elide=*/true);
   }
 
diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
index c85f480775808..e166e28cbcdf5 100644
--- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -29,6 +29,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/circular_raw_ostream.h"
+#include "llvm/Support/OptimalLayout.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Transforms/Utils/PromoteMemToReg.h"
@@ -340,52 +341,182 @@ namespace {
 // coroutine frame and if the alignment specified on the Alloca instruction
 // differs from the natural alignment of the alloca type we will need to insert
 // padding.
-struct PaddingCalculator {
+class FrameTypeBuilder {
+  struct Field {
+    uint64_t Size;
+    uint64_t Offset;
+    Spill *ForSpill;
+    Type *Ty;
+    unsigned FieldIndex;
+    Align Alignment;
+    Align TyAlignment;
+  };
+
   const DataLayout &DL;
   LLVMContext &Context;
-  unsigned StructSize = 0;
+  uint64_t StructSize = 0;
+  Align StructAlign;
+  bool IsFinished = false;
 
-  PaddingCalculator(LLVMContext &Context, DataLayout const &DL)
-      : DL(DL), Context(Context) {}
+  SmallVector<Field, 8> Fields;
+  DenseMap<Value*, unsigned> FieldIndexByKey;
 
-  // Replicate the logic from IR/DataLayout.cpp to match field offset
-  // computation for LLVM structs.
-  void addType(Type *Ty) {
-    unsigned TyAlign = DL.getABITypeAlignment(Ty);
-    if ((StructSize & (TyAlign - 1)) != 0)
-      StructSize = alignTo(StructSize, TyAlign);
+public:
+  FrameTypeBuilder(LLVMContext &Context, DataLayout const &DL)
+    : DL(DL), Context(Context) {}
 
-    StructSize += DL.getTypeAllocSize(Ty); // Consume space for this data item.
-  }
+  class FieldId {
+    size_t Value;
+    explicit FieldId(size_t Value) : Value(Value) {}
 
-  void addTypes(SmallVectorImpl<Type *> const &Types) {
-    for (auto *Ty : Types)
-      addType(Ty);
+    friend class FrameTypeBuilder;
+  };
+
+  /// Add a field to this structure for the storage of an `alloca`
+  /// instruction.
+  FieldId addFieldForAlloca(AllocaInst *AI, Spill *ForSpill = nullptr,
+                            bool IsHeader = false) {
+    Type *Ty = AI->getAllocatedType();
+
+    // Make an array type if this is a static array allocation.
+    if (AI->isArrayAllocation()) {
+      if (auto *CI = dyn_cast<ConstantInt>(AI->getArraySize()))
+        Ty = ArrayType::get(Ty, CI->getValue().getZExtValue());
+      else
+        report_fatal_error("Coroutines cannot handle non static allocas yet");
+    }
+
+    return addField(Ty, AI->getAlign(), ForSpill, IsHeader);
   }
 
-  unsigned computePadding(Type *Ty, unsigned ForcedAlignment) {
-    unsigned TyAlign = DL.getABITypeAlignment(Ty);
-    auto Natural = alignTo(StructSize, TyAlign);
-    auto Forced = alignTo(StructSize, ForcedAlignment);
+  /// Add a field to this structure.
+  FieldId addField(Type *Ty, MaybeAlign FieldAlignment,
+                   Spill *ForSpill = nullptr,
+                   bool IsHeader = false) {
+    assert(!IsFinished && "adding fields to a finished builder");
+    assert(Ty && "must provide a type for a field");
 
-    // Return how many bytes of padding we need to insert.
-    if (Natural != Forced)
-      return std::max(Natural, Forced) - StructSize;
+    // The field size is always the alloc size of the type.
+    uint64_t FieldSize = DL.getTypeAllocSize(Ty);
 
-    // Rely on natural alignment.
-    return 0;
+    // The field alignment might not be the type alignment, but we need
+    // to remember the type alignment anyway to build the type.
+    Align TyAlignment = DL.getABITypeAlign(Ty);
+    if (!FieldAlignment) FieldAlignment = TyAlignment;
+
+    // Lay out header fields immediately.
+    uint64_t Offset;
+    if (IsHeader) {
+      Offset = alignTo(StructSize, FieldAlignment);
+      StructSize = Offset + FieldSize;
+
+    // Everything else has a flexible offset.
+    } else {
+      Offset = OptimalLayoutField::FlexibleOffset;
+    }
+
+    Fields.push_back({FieldSize, Offset, ForSpill, Ty, 0,
+                      *FieldAlignment, TyAlignment});
+    return FieldId(Fields.size() - 1);
   }
 
-  // If padding required, return the padding field type to insert.
-  ArrayType *getPaddingType(Type *Ty, unsigned ForcedAlignment) {
-    if (auto Padding = computePadding(Ty, ForcedAlignment))
-      return ArrayType::get(Type::getInt8Ty(Context), Padding);
+  /// Finish the layout and set the body on the given type.
+  void finish(StructType *Ty);
+
+  uint64_t getStructSize() const {
+    assert(IsFinished && "not yet finished!");
+    return StructSize;
+  }
 
-    return nullptr;
+  Align getStructAlign() const {
+    assert(IsFinished && "not yet finished!");
+    return StructAlign;
+  }
+
+  unsigned getFieldIndex(FieldId Id) const {
+    assert(IsFinished && "not yet finished!");
+    return Fields[Id.Value].FieldIndex;
   }
 };
 } // namespace
 
+void FrameTypeBuilder::finish(StructType *Ty) {
+  assert(!IsFinished && "already finished!");
+
+  // Prepare the optimal-layout field array.
+  // The Id in the layout field is a pointer to our Field for it.
+  SmallVector<OptimalLayoutField, 8> LayoutFields;
+  LayoutFields.reserve(Fields.size());
+  for (auto &Field : Fields) {
+    LayoutFields.emplace_back(&Field, Field.Size, Field.Alignment,
+                              Field.Offset);
+  }
+
+  // Perform layout.
+  auto SizeAndAlign = performOptimalLayout(LayoutFields);
+  StructSize = SizeAndAlign.first;
+  StructAlign = SizeAndAlign.second;
+
+  auto getField = [](const OptimalLayoutField &LayoutField) -> Field & {
+    return *static_cast<Field *>(const_cast<void*>(LayoutField.Id));
+  };
+
+  // We need to produce a packed struct type if there's a field whose
+  // assigned offset isn't a multiple of its natural type alignment.
+  bool Packed = [&] {
+    for (auto &LayoutField : LayoutFields) {
+      auto &F = getField(LayoutField);
+      if (!isAligned(F.TyAlignment, LayoutField.Offset))
+        return true;
+    }
+    return false;
+  }();
+
+  // Build the struct body.
+  SmallVector<Type*, 16> FieldTypes;
+  FieldTypes.reserve(LayoutFields.size() * 3 / 2);
+  uint64_t LastOffset = 0;
+  for (auto &LayoutField : LayoutFields) {
+    auto &F = getField(LayoutField);
+
+    auto Offset = LayoutField.Offset;
+
+    // Add a padding field if there's a padding gap and we're either
+    // building a packed struct or the padding gap is more than we'd
+    // get from aligning to the field type's natural alignment.
+    assert(Offset >= LastOffset);
+    if (Offset != LastOffset) {
+      if (Packed || alignTo(LastOffset, F.TyAlignment) != Offset)
+        FieldTypes.push_back(ArrayType::get(Type::getInt8Ty(Context),
+                                            Offset - LastOffset));
+    }
+
+    // Record the layout information into both the Field and the
+    // original Spill, if there is one.
+    F.Offset = Offset;
+    F.FieldIndex = FieldTypes.size();
+    if (F.ForSpill) {
+      F.ForSpill->setFieldIndex(F.FieldIndex);
+    }
+
+    FieldTypes.push_back(F.Ty);
+    LastOffset = Offset + F.Size;
+  }
+
+  Ty->setBody(FieldTypes, Packed);
+
+#ifndef NDEBUG
+  // Check that the IR layout matches the offsets we expect.
+  auto Layout = DL.getStructLayout(Ty);
+  for (auto &F : Fields) {
+    assert(Ty->getElementType(F.FieldIndex) == F.Ty);
+    assert(Layout->getElementOffset(F.FieldIndex) == F.Offset);
+  }
+#endif
+
+  IsFinished = true;
+}
+
 // Build a struct that will keep state for an active coroutine.
 //   struct f.frame {
 //     ResumeFnTy ResumeFnAddr;
@@ -398,13 +529,17 @@ static StructType *buildFrameType(Function &F, coro::Shape &Shape,
                                   SpillInfo &Spills) {
   LLVMContext &C = F.getContext();
   const DataLayout &DL = F.getParent()->getDataLayout();
-  PaddingCalculator Padder(C, DL);
-  SmallString<32> Name(F.getName());
-  Name.append(".Frame");
-  StructType *FrameTy = StructType::create(C, Name);
-  SmallVector<Type *, 8> Types;
+  StructType *FrameTy = [&] {
+    SmallString<32> Name(F.getName());
+    Name.append(".Frame");
+    return StructType::create(C, Name);
+  }();
+
+  FrameTypeBuilder B(C, DL);
 
   AllocaInst *PromiseAlloca = Shape.getPromiseAlloca();
+  Optional<FrameTypeBuilder::FieldId> PromiseFieldId;
+  Optional<FrameTypeBuilder::FieldId> SwitchIndexFieldId;
 
   if (Shape.ABI == coro::ABI::Switch) {
     auto *FramePtrTy = FrameTy->getPointerTo();
@@ -412,74 +547,74 @@ static StructType *buildFrameType(Function &F, coro::Shape &Shape,
                                    /*IsVarArg=*/false);
     auto *FnPtrTy = FnTy->getPointerTo();
 
-    // Figure out how wide should be an integer type storing the suspend index.
+    // Add header fields for the resume and destroy functions.
+    // We can rely on these being perfectly packed.
+    B.addField(FnPtrTy, None, nullptr, /*header*/ true);
+    B.addField(FnPtrTy, None, nullptr, /*header*/ true);
+
+    // Add a header field for the promise if there is one.
+    if (PromiseAlloca) {
+      PromiseFieldId =
+        B.addFieldForAlloca(PromiseAlloca, nullptr, /*header*/ true);
+    }
+
+    // Add a field to store the suspend index.  This doesn't need to
+    // be in the header.
     unsigned IndexBits = std::max(1U, Log2_64_Ceil(Shape.CoroSuspends.size()));
-    Type *PromiseType = PromiseAlloca
-                            ? PromiseAlloca->getType()->getElementType()
-                            : Type::getInt1Ty(C);
     Type *IndexType = Type::getIntNTy(C, IndexBits);
-    Types.push_back(FnPtrTy);
-    Types.push_back(FnPtrTy);
-    Types.push_back(PromiseType);
-    Types.push_back(IndexType);
+
+    SwitchIndexFieldId = B.addField(IndexType, None);
   } else {
     assert(PromiseAlloca == nullptr && "lowering doesn't support promises");
   }
 
   Value *CurrentDef = nullptr;
 
-  Padder.addTypes(Types);
-
   // Create an entry for every spilled value.
   for (auto &S : Spills) {
+    // We can have multiple entries in Spills for a single value, but
+    // they should form a contiguous run.  Ignore all but the first.
     if (CurrentDef == S.def())
       continue;
 
     CurrentDef = S.def();
-    // PromiseAlloca was already added to Types array earlier.
-    if (CurrentDef == PromiseAlloca)
-      continue;
 
-    uint64_t Count = 1;
-    Type *Ty = nullptr;
+    assert(CurrentDef != PromiseAlloca &&
+           "recorded spill use of promise alloca?");
+
     if (auto *AI = dyn_cast<AllocaInst>(CurrentDef)) {
-      Ty = AI->getAllocatedType();
-      if (unsigned AllocaAlignment = AI->getAlignment()) {
-        // If alignment is specified in alloca, see if we need to insert extra
-        // padding.
-        if (auto PaddingTy = Padder.getPaddingType(Ty, AllocaAlignment)) {
-          Types.push_back(PaddingTy);
-          Padder.addType(PaddingTy);
-        }
-      }
-      if (auto *CI = dyn_cast<ConstantInt>(AI->getArraySize()))
-        Count = CI->getValue().getZExtValue();
-      else
-        report_fatal_error("Coroutines cannot handle non static allocas yet");
+      B.addFieldForAlloca(AI, &S);
     } else {
-      Ty = CurrentDef->getType();
+      Type *Ty = CurrentDef->getType();
+      B.addField(Ty, None, &S);
     }
-    S.setFieldIndex(Types.size());
-    if (Count == 1)
-      Types.push_back(Ty);
-    else
-      Types.push_back(ArrayType::get(Ty, Count));
-    Padder.addType(Ty);
   }
-  FrameTy->setBody(Types);
+
+  B.finish(FrameTy);
+  Shape.FrameAlign = B.getStructAlign();
+  Shape.FrameSize = B.getStructSize();
 
   switch (Shape.ABI) {
+  // In the switch ABI, remember the field indices for the promise and
+  // switch-index fields.
   case coro::ABI::Switch:
+    Shape.SwitchLowering.IndexField =
+      B.getFieldIndex(*SwitchIndexFieldId);
+    Shape.SwitchLowering.PromiseField =
+      (PromiseAlloca ? B.getFieldIndex(*PromiseFieldId) : 0);
+
+    // Also round the frame size up to a multiple of its alignment, as is
+    // generally expected in C/C++.
+    Shape.FrameSize = alignTo(Shape.FrameSize, Shape.FrameAlign);
     break;
 
-  // Remember whether the frame is inline in the storage.
+  // In the retcon ABI, remember whether the frame is inline in the storage.
   case coro::ABI::Retcon:
   case coro::ABI::RetconOnce: {
-    auto &Layout = F.getParent()->getDataLayout();
     auto Id = Shape.getRetconCoroId();
     Shape.RetconLowering.IsFrameInlineInStorage
-      = (Layout.getTypeAllocSize(FrameTy) <= Id->getStorageSize() &&
-         Layout.getABITypeAlignment(FrameTy) <= Id->getStorageAlignment());
+      = (B.getStructSize() <= Id->getStorageSize() &&
+         B.getStructAlign() <= Id->getStorageAlignment());
     break;
   }
   }
@@ -608,10 +743,12 @@ static Instruction *insertSpills(const SpillInfo &Spills, coro::Shape &Shape) {
   // we remember allocas and their indices to be handled once we processed
   // all the spills.
   SmallVector<std::pair<AllocaInst *, unsigned>, 4> Allocas;
-  // Promise alloca (if present) has a fixed field number.
+
+  // Promise alloca (if present) doesn't show in the spills and has a
+  // special field number.
   if (auto *PromiseAlloca = Shape.getPromiseAlloca()) {
     assert(Shape.ABI == coro::ABI::Switch);
-    Allocas.emplace_back(PromiseAlloca, coro::Shape::SwitchFieldIndex::Promise);
+    Allocas.emplace_back(PromiseAlloca, Shape.getPromiseField());
   }
 
   // Create a GEP with the given index into the coroutine frame for the original
diff --git a/llvm/lib/Transforms/Coroutines/CoroInstr.h b/llvm/lib/Transforms/Coroutines/CoroInstr.h
index de2d2920cb15a..384e20cd0a9b2 100644
--- a/llvm/lib/Transforms/Coroutines/CoroInstr.h
+++ b/llvm/lib/Transforms/Coroutines/CoroInstr.h
@@ -211,8 +211,8 @@ class LLVM_LIBRARY_VISIBILITY AnyCoroIdRetconInst : public AnyCoroIdInst {
     return cast<ConstantInt>(getArgOperand(SizeArg))->getZExtValue();
   }
 
-  uint64_t getStorageAlignment() const {
-    return cast<ConstantInt>(getArgOperand(AlignArg))->getZExtValue();
+  Align getStorageAlignment() const {
+    return Align(cast<ConstantInt>(getArgOperand(AlignArg))->getZExtValue());
   }
 
   Value *getStorage() const {
@@ -338,11 +338,16 @@ class LLVM_LIBRARY_VISIBILITY CoroPromiseInst : public IntrinsicInst {
   enum { FrameArg, AlignArg, FromArg };
 
 public:
+  /// Are we translating from the frame to the promise (false) or from
+  /// the promise to the frame (true)?
   bool isFromPromise() const {
     return cast<Constant>(getArgOperand(FromArg))->isOneValue();
   }
-  unsigned getAlignment() const {
-    return cast<ConstantInt>(getArgOperand(AlignArg))->getZExtValue();
+
+  /// The required alignment of the promise.  This must match the
+  /// alignment of the promise alloca in the coroutine.
+  Align getAlignment() const {
+    return Align(cast<ConstantInt>(getArgOperand(AlignArg))->getZExtValue());
   }
 
   // Methods to support type inquiry through isa, cast, and dyn_cast:
diff --git a/llvm/lib/Transforms/Coroutines/CoroInternal.h b/llvm/lib/Transforms/Coroutines/CoroInternal.h
index 7eb35400c0d51..bd76e93c91241 100644
--- a/llvm/lib/Transforms/Coroutines/CoroInternal.h
+++ b/llvm/lib/Transforms/Coroutines/CoroInternal.h
@@ -96,17 +96,22 @@ struct LLVM_LIBRARY_VISIBILITY Shape {
   struct SwitchFieldIndex {
     enum {
       Resume,
-      Destroy,
-      Promise,
-      Index,
-      /// The index of the first spill field.
-      FirstSpill
+      Destroy
+
+      // The promise field is always at a fixed offset from the start of
+      // frame given its type, but the index isn't a constant for all
+      // possible frames.
+
+      // The switch-index field isn't at a fixed offset or index, either;
+      // we just work it in where it fits best.
     };
   };
 
   coro::ABI ABI;
 
   StructType *FrameTy;
+  Align FrameAlign;
+  uint64_t FrameSize;
   Instruction *FramePtr;
   BasicBlock *AllocaSpillBlock;
 
@@ -114,6 +119,8 @@ struct LLVM_LIBRARY_VISIBILITY Shape {
     SwitchInst *ResumeSwitch;
     AllocaInst *PromiseAlloca;
     BasicBlock *ResumeEntryBlock;
+    unsigned IndexField;
+    unsigned PromiseField;
     bool HasFinalSuspend;
   };
 
@@ -141,10 +148,15 @@ struct LLVM_LIBRARY_VISIBILITY Shape {
     return cast<AnyCoroIdRetconInst>(CoroBegin->getId());
   }
 
+  unsigned getSwitchIndexField() const {
+    assert(ABI == coro::ABI::Switch);
+    assert(FrameTy && "frame type not assigned");
+    return SwitchLowering.IndexField;
+  }
   IntegerType *getIndexType() const {
     assert(ABI == coro::ABI::Switch);
     assert(FrameTy && "frame type not assigned");
-    return cast<IntegerType>(FrameTy->getElementType(SwitchFieldIndex::Index));
+    return cast<IntegerType>(FrameTy->getElementType(getSwitchIndexField()));
   }
   ConstantInt *getIndex(uint64_t Value) const {
     return ConstantInt::get(getIndexType(), Value);
@@ -203,23 +215,17 @@ struct LLVM_LIBRARY_VISIBILITY Shape {
     llvm_unreachable("Unknown coro::ABI enum");
   }
 
-  unsigned getFirstSpillFieldIndex() const {
-    switch (ABI) {
-    case coro::ABI::Switch:
-      return SwitchFieldIndex::FirstSpill;
-
-    case coro::ABI::Retcon:
-    case coro::ABI::RetconOnce:
-      return 0;
-    }
-    llvm_unreachable("Unknown coro::ABI enum");
-  }
-
   AllocaInst *getPromiseAlloca() const {
     if (ABI == coro::ABI::Switch)
       return SwitchLowering.PromiseAlloca;
     return nullptr;
   }
+  unsigned getPromiseField() const {
+    assert(ABI == coro::ABI::Switch);
+    assert(FrameTy && "frame type not assigned");
+    assert(SwitchLowering.PromiseAlloca && "no promise alloca");
+    return SwitchLowering.PromiseField;
+  }
 
   /// Allocate memory according to the rules of the active lowering.
   ///
diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
index 465b6598da650..96c888d10d627 100644
--- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
@@ -285,7 +285,7 @@ static void createResumeEntryBlock(Function &F, coro::Shape &Shape) {
   auto *FramePtr = Shape.FramePtr;
   auto *FrameTy = Shape.FrameTy;
   auto *GepIndex = Builder.CreateStructGEP(
-      FrameTy, FramePtr, coro::Shape::SwitchFieldIndex::Index, "index.addr");
+      FrameTy, FramePtr, Shape.getSwitchIndexField(), "index.addr");
   auto *Index = Builder.CreateLoad(Shape.getIndexType(), GepIndex, "index");
   auto *Switch =
       Builder.CreateSwitch(Index, UnreachBB, Shape.CoroSuspends.size());
@@ -311,7 +311,7 @@ static void createResumeEntryBlock(Function &F, coro::Shape &Shape) {
       Builder.CreateStore(NullPtr, GepIndex);
     } else {
       auto *GepIndex = Builder.CreateStructGEP(
-          FrameTy, FramePtr, coro::Shape::SwitchFieldIndex::Index, "index.addr");
+          FrameTy, FramePtr, Shape.getSwitchIndexField(), "index.addr");
       Builder.CreateStore(IndexVal, GepIndex);
     }
     Save->replaceAllUsesWith(ConstantTokenNone::get(C));
@@ -645,6 +645,17 @@ Value *CoroCloner::deriveNewFramePointer() {
   llvm_unreachable("bad ABI");
 }
 
+static void addFramePointerAttrs(AttributeList &Attrs, LLVMContext &Context,
+                                 unsigned ParamIndex,
+                                 uint64_t Size, Align Alignment) {
+  AttrBuilder ParamAttrs;
+  ParamAttrs.addAttribute(Attribute::NonNull);
+  ParamAttrs.addAttribute(Attribute::NoAlias);
+  ParamAttrs.addAlignmentAttr(Alignment);
+  ParamAttrs.addDereferenceableAttr(Size);
+  Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);
+}
+
 /// Clone the body of the original function into a resume function of
 /// some sort.
 void CoroCloner::create() {
@@ -693,6 +704,9 @@ void CoroCloner::create() {
     // original function.  This should include optimization settings and so on.
     NewAttrs = NewAttrs.addAttributes(Context, AttributeList::FunctionIndex,
                                       OrigAttrs.getFnAttributes());
+
+    addFramePointerAttrs(NewAttrs, Context, 0,
+                         Shape.FrameSize, Shape.FrameAlign);
     break;
 
   case coro::ABI::Retcon:
@@ -700,13 +714,13 @@ void CoroCloner::create() {
     // If we have a continuation prototype, just use its attributes,
     // full-stop.
     NewAttrs = Shape.RetconLowering.ResumePrototype->getAttributes();
+
+    addFramePointerAttrs(NewAttrs, Context, 0,
+                         Shape.getRetconCoroId()->getStorageSize(),
+                         Shape.getRetconCoroId()->getStorageAlignment());
     break;
   }
 
-  // Make the frame parameter nonnull and noalias.
-  NewAttrs = NewAttrs.addParamAttribute(Context, 0, Attribute::NonNull);
-  NewAttrs = NewAttrs.addParamAttribute(Context, 0, Attribute::NoAlias);
-
   switch (Shape.ABI) {
   // In these ABIs, the cloned functions always return 'void', and the
   // existing return sites are meaningless.  Note that for unique
@@ -920,17 +934,14 @@ static bool simplifyTerminatorLeadingToRet(Instruction *InitialInst) {
   BasicBlock *UnconditionalSucc = nullptr;
 
   Instruction *I = InitialInst;
-  while (I->isTerminator()) {
+  while (I->isTerminator() ||
+         (isa<CmpInst>(I) && I->getNextNode()->isTerminator())) {
     if (isa<ReturnInst>(I)) {
       if (I != InitialInst) {
         // If InitialInst is an unconditional branch,
         // remove PHI values that come from basic block of InitialInst
         if (UnconditionalSucc)
-          for (PHINode &PN : UnconditionalSucc->phis()) {
-            int idx = PN.getBasicBlockIndex(InitialInst->getParent());
-            if (idx != -1)
-              PN.removeIncomingValue(idx);
-          }
+          UnconditionalSucc->removePredecessor(InitialInst->getParent(), true);
         ReplaceInstWithInst(InitialInst, I->clone());
       }
       return true;
@@ -944,6 +955,29 @@ static bool simplifyTerminatorLeadingToRet(Instruction *InitialInst) {
         I = BB->getFirstNonPHIOrDbgOrLifetime();
         continue;
       }
+    } else if (auto *CondCmp = dyn_cast<CmpInst>(I)) {
+      auto *BR = dyn_cast<BranchInst>(I->getNextNode());
+      if (BR && BR->isConditional() && CondCmp == BR->getCondition()) {
+        // If the case number of suspended switch instruction is reduced to
+        // 1, then it is simplified to CmpInst in llvm::ConstantFoldTerminator.
+        // And the comparsion looks like : %cond = icmp eq i8 %V, constant.
+        ConstantInt *CondConst = dyn_cast<ConstantInt>(CondCmp->getOperand(1));
+        if (CondConst && CondCmp->getPredicate() == CmpInst::ICMP_EQ) {
+          Value *V = CondCmp->getOperand(0);
+          auto it = ResolvedValues.find(V);
+          if (it != ResolvedValues.end())
+            V = it->second;
+
+          if (ConstantInt *Cond0 = dyn_cast<ConstantInt>(V)) {
+            BasicBlock *BB = Cond0->equalsInt(CondConst->getZExtValue())
+                                 ? BR->getSuccessor(0)
+                                 : BR->getSuccessor(1);
+            scanPHIsAndUpdateValueMap(I, BB, ResolvedValues);
+            I = BB->getFirstNonPHIOrDbgOrLifetime();
+            continue;
+          }
+        }
+      }
     } else if (auto *SI = dyn_cast<SwitchInst>(I)) {
       Value *V = SI->getCondition();
       auto it = ResolvedValues.find(V);
@@ -961,6 +995,37 @@ static bool simplifyTerminatorLeadingToRet(Instruction *InitialInst) {
   return false;
 }
 
+// Check whether CI obeys the rules of musttail attribute.
+static bool shouldBeMustTail(const CallInst &CI, const Function &F) {
+  if (CI.isInlineAsm())
+    return false;
+
+  // Match prototypes and calling conventions of resume function.
+  FunctionType *CalleeTy = CI.getFunctionType();
+  if (!CalleeTy->getReturnType()->isVoidTy() || (CalleeTy->getNumParams() != 1))
+    return false;
+
+  Type *CalleeParmTy = CalleeTy->getParamType(0);
+  if (!CalleeParmTy->isPointerTy() ||
+      (CalleeParmTy->getPointerAddressSpace() != 0))
+    return false;
+
+  if (CI.getCallingConv() != F.getCallingConv())
+    return false;
+
+  // CI should not has any ABI-impacting function attributes.
+  static const Attribute::AttrKind ABIAttrs[] = {
+      Attribute::StructRet,  Attribute::ByVal,    Attribute::InAlloca,
+      Attribute::InReg,      Attribute::Returned, Attribute::SwiftSelf,
+      Attribute::SwiftError, Attribute::Alignment};
+  AttributeList Attrs = CI.getAttributes();
+  for (auto AK : ABIAttrs)
+    if (Attrs.hasParamAttribute(0, AK))
+      return false;
+
+  return true;
+}
+
 // Add musttail to any resume instructions that is immediately followed by a
 // suspend (i.e. ret). We do this even in -O0 to support guaranteed tail call
 // for symmetrical coroutine control transfer (C++ Coroutines TS extension).
@@ -973,11 +1038,8 @@ static void addMustTailToCoroResumes(Function &F) {
   SmallVector<CallInst *, 4> Resumes;
   for (auto &I : instructions(F))
     if (auto *Call = dyn_cast<CallInst>(&I))
-      if (auto *CalledValue = Call->getCalledValue())
-        // CoroEarly pass replaced coro resumes with indirect calls to an
-        // address return by CoroSubFnInst intrinsic. See if it is one of those.
-        if (isa<CoroSubFnInst>(CalledValue->stripPointerCasts()))
-          Resumes.push_back(Call);
+      if (shouldBeMustTail(*Call, F))
+        Resumes.push_back(Call);
 
   // Set musttail on those that are followed by a ret instruction.
   for (CallInst *Call : Resumes)
@@ -1002,8 +1064,8 @@ static void handleNoSuspendCoroutine(coro::Shape &Shape) {
     coro::replaceCoroFree(SwitchId, /*Elide=*/AllocInst != nullptr);
     if (AllocInst) {
       IRBuilder<> Builder(AllocInst);
-      // FIXME: Need to handle overaligned members.
       auto *Frame = Builder.CreateAlloca(Shape.FrameTy);
+      Frame->setAlignment(Shape.FrameAlign);
       auto *VFrame = Builder.CreateBitCast(Frame, Builder.getInt8PtrTy());
       AllocInst->replaceAllUsesWith(Builder.getFalse());
       AllocInst->eraseFromParent();
@@ -1237,6 +1299,7 @@ static void splitRetconCoroutine(Function &F, coro::Shape &Shape,
 
     // Allocate.  We don't need to update the call graph node because we're
     // going to recompute it from scratch after splitting.
+    // FIXME: pass the required alignment
     RawFramePtr = Shape.emitAlloc(Builder, Builder.getInt64(Size), nullptr);
     RawFramePtr =
       Builder.CreateBitCast(RawFramePtr, Shape.CoroBegin->getType());
diff --git a/llvm/lib/Transforms/IPO/Attributor.cpp b/llvm/lib/Transforms/IPO/Attributor.cpp
index 053255ae46023..6a148f991b4ed 100644
--- a/llvm/lib/Transforms/IPO/Attributor.cpp
+++ b/llvm/lib/Transforms/IPO/Attributor.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file implements an inter procedural pass that deduces and/or propagating
+// This file implements an interprocedural pass that deduces and/or propagates
 // attributes. This is done in an abstract interpretation style fixpoint
 // iteration. See the Attributor.h file comment and the class descriptions in
 // that file for more information.
@@ -54,9 +54,9 @@ using namespace llvm;
 #define DEBUG_TYPE "attributor"
 
 STATISTIC(NumFnWithExactDefinition,
-          "Number of function with exact definitions");
+          "Number of functions with exact definitions");
 STATISTIC(NumFnWithoutExactDefinition,
-          "Number of function without exact definitions");
+          "Number of functions without exact definitions");
 STATISTIC(NumAttributesTimedOut,
           "Number of abstract attributes timed out before fixpoint");
 STATISTIC(NumAttributesValidFixpoint,
@@ -398,8 +398,10 @@ static Value *constructPointer(Type *ResTy, Value *Ptr, int64_t Offset,
 template <typename AAType, typename StateTy>
 static bool genericValueTraversal(
     Attributor &A, IRPosition IRP, const AAType &QueryingAA, StateTy &State,
-    function_ref<bool(Value &, StateTy &, bool)> VisitValueCB,
-    int MaxValues = 8, function_ref<Value *(Value *)> StripCB = nullptr) {
+    function_ref<bool(Value &, const Instruction *, StateTy &, bool)>
+        VisitValueCB,
+    const Instruction *CtxI, int MaxValues = 16,
+    function_ref<Value *(Value *)> StripCB = nullptr) {
 
   const AAIsDead *LivenessAA = nullptr;
   if (IRP.getAnchorScope())
@@ -408,20 +410,22 @@ static bool genericValueTraversal(
         /* TrackDependence */ false);
   bool AnyDead = false;
 
-  // TODO: Use Positions here to allow context sensitivity in VisitValueCB
-  SmallPtrSet<Value *, 16> Visited;
-  SmallVector<Value *, 16> Worklist;
-  Worklist.push_back(&IRP.getAssociatedValue());
+  using Item = std::pair<Value *, const Instruction *>;
+  SmallSet<Item, 16> Visited;
+  SmallVector<Item, 16> Worklist;
+  Worklist.push_back({&IRP.getAssociatedValue(), CtxI});
 
   int Iteration = 0;
   do {
-    Value *V = Worklist.pop_back_val();
+    Item I = Worklist.pop_back_val();
+    Value *V = I.first;
+    CtxI = I.second;
     if (StripCB)
       V = StripCB(V);
 
     // Check if we should process the current value. To prevent endless
     // recursion keep a record of the values we followed!
-    if (!Visited.insert(V).second)
+    if (!Visited.insert(I).second)
       continue;
 
     // Make sure we limit the compile time for complex expressions.
@@ -444,14 +448,14 @@ static bool genericValueTraversal(
       }
     }
     if (NewV && NewV != V) {
-      Worklist.push_back(NewV);
+      Worklist.push_back({NewV, CtxI});
       continue;
     }
 
     // Look through select instructions, visit both potential values.
     if (auto *SI = dyn_cast<SelectInst>(V)) {
-      Worklist.push_back(SI->getTrueValue());
-      Worklist.push_back(SI->getFalseValue());
+      Worklist.push_back({SI->getTrueValue(), CtxI});
+      Worklist.push_back({SI->getFalseValue(), CtxI});
       continue;
     }
 
@@ -460,20 +464,21 @@ static bool genericValueTraversal(
       assert(LivenessAA &&
              "Expected liveness in the presence of instructions!");
       for (unsigned u = 0, e = PHI->getNumIncomingValues(); u < e; u++) {
-        const BasicBlock *IncomingBB = PHI->getIncomingBlock(u);
+        BasicBlock *IncomingBB = PHI->getIncomingBlock(u);
         if (A.isAssumedDead(*IncomingBB->getTerminator(), &QueryingAA,
                             LivenessAA,
                             /* CheckBBLivenessOnly */ true)) {
           AnyDead = true;
           continue;
         }
-        Worklist.push_back(PHI->getIncomingValue(u));
+        Worklist.push_back(
+            {PHI->getIncomingValue(u), IncomingBB->getTerminator()});
       }
       continue;
     }
 
     // Once a leaf is reached we inform the user through the callback.
-    if (!VisitValueCB(*V, State, Iteration > 1))
+    if (!VisitValueCB(*V, CtxI, State, Iteration > 1))
       return false;
   } while (!Worklist.empty());
 
@@ -710,7 +715,7 @@ void IRPosition::getAttrs(ArrayRef<Attribute::AttrKind> AKs,
   }
   if (A)
     for (Attribute::AttrKind AK : AKs)
-     getAttrsFromAssumes(AK, Attrs, *A);
+      getAttrsFromAssumes(AK, Attrs, *A);
 }
 
 bool IRPosition::getAttrsFromIRAttr(Attribute::AttrKind AK,
@@ -1334,7 +1339,7 @@ ChangeStatus AAReturnedValuesImpl::manifest(Attributor &A) {
   // Bookkeeping.
   assert(isValidState());
   STATS_DECLTRACK(KnownReturnValues, FunctionReturn,
-                  "Number of function with known return values");
+                  "Number of functions with known return values");
 
   // Check if we have an assumed unique return value that we could manifest.
   Optional<Value *> UniqueRV = getAssumedUniqueReturnValue(A);
@@ -1344,7 +1349,7 @@ ChangeStatus AAReturnedValuesImpl::manifest(Attributor &A) {
 
   // Bookkeeping.
   STATS_DECLTRACK(UniqueReturnValue, FunctionReturn,
-                  "Number of function with unique return");
+                  "Number of functions with a unique return");
 
   // Callback to replace the uses of CB with the constant C.
   auto ReplaceCallSiteUsersWith = [&A](CallBase &CB, Constant &C) {
@@ -1466,7 +1471,8 @@ ChangeStatus AAReturnedValuesImpl::updateImpl(Attributor &A) {
   };
 
   // Callback for a leaf value returned by the associated function.
-  auto VisitValueCB = [](Value &Val, RVState &RVS, bool) -> bool {
+  auto VisitValueCB = [](Value &Val, const Instruction *, RVState &RVS,
+                         bool) -> bool {
     auto Size = RVS.RetValsMap[&Val].size();
     RVS.RetValsMap[&Val].insert(RVS.RetInsts.begin(), RVS.RetInsts.end());
     bool Inserted = RVS.RetValsMap[&Val].size() != Size;
@@ -1480,10 +1486,11 @@ ChangeStatus AAReturnedValuesImpl::updateImpl(Attributor &A) {
   };
 
   // Helper method to invoke the generic value traversal.
-  auto VisitReturnedValue = [&](Value &RV, RVState &RVS) {
+  auto VisitReturnedValue = [&](Value &RV, RVState &RVS,
+                                const Instruction *CtxI) {
     IRPosition RetValPos = IRPosition::value(RV);
-    return genericValueTraversal<AAReturnedValues, RVState>(A, RetValPos, *this,
-                                                            RVS, VisitValueCB);
+    return genericValueTraversal<AAReturnedValues, RVState>(
+        A, RetValPos, *this, RVS, VisitValueCB, CtxI);
   };
 
   // Callback for all "return intructions" live in the associated function.
@@ -1491,7 +1498,7 @@ ChangeStatus AAReturnedValuesImpl::updateImpl(Attributor &A) {
     ReturnInst &Ret = cast<ReturnInst>(I);
     RVState RVS({ReturnedValues, Changed, {}});
     RVS.RetInsts.insert(&Ret);
-    return VisitReturnedValue(*Ret.getReturnValue(), RVS);
+    return VisitReturnedValue(*Ret.getReturnValue(), RVS, &I);
   };
 
   // Start by discovering returned values from all live returned instructions in
@@ -1576,7 +1583,7 @@ ChangeStatus AAReturnedValuesImpl::updateImpl(Attributor &A) {
         // again.
         bool Unused = false;
         RVState RVS({NewRVsMap, Unused, RetValAAIt.second});
-        VisitReturnedValue(*CB->getArgOperand(Arg->getArgNo()), RVS);
+        VisitReturnedValue(*CB->getArgOperand(Arg->getArgNo()), RVS, CB);
         continue;
       } else if (isa<CallBase>(RetVal)) {
         // Call sites are resolved by the callee attribute over time, no need to
@@ -2148,11 +2155,11 @@ struct AANonNullFloating
       AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(*Fn);
     }
 
-    auto VisitValueCB = [&](Value &V, AANonNull::StateType &T,
-                            bool Stripped) -> bool {
+    auto VisitValueCB = [&](Value &V, const Instruction *CtxI,
+                            AANonNull::StateType &T, bool Stripped) -> bool {
       const auto &AA = A.getAAFor<AANonNull>(*this, IRPosition::value(V));
       if (!Stripped && this == &AA) {
-        if (!isKnownNonZero(&V, DL, 0, AC, getCtxI(), DT))
+        if (!isKnownNonZero(&V, DL, 0, AC, CtxI, DT))
           T.indicatePessimisticFixpoint();
       } else {
         // Use abstract attribute information.
@@ -2164,8 +2171,8 @@ struct AANonNullFloating
     };
 
     StateType T;
-    if (!genericValueTraversal<AANonNull, StateType>(A, getIRPosition(), *this,
-                                                     T, VisitValueCB))
+    if (!genericValueTraversal<AANonNull, StateType>(
+            A, getIRPosition(), *this, T, VisitValueCB, getCtxI()))
       return indicatePessimisticFixpoint();
 
     return clampStateAndIndicateChange(getState(), T);
@@ -3776,7 +3783,8 @@ struct AADereferenceableFloating
 
     const DataLayout &DL = A.getDataLayout();
 
-    auto VisitValueCB = [&](Value &V, DerefState &T, bool Stripped) -> bool {
+    auto VisitValueCB = [&](Value &V, const Instruction *, DerefState &T,
+                            bool Stripped) -> bool {
       unsigned IdxWidth =
           DL.getIndexSizeInBits(V.getType()->getPointerAddressSpace());
       APInt Offset(IdxWidth, 0);
@@ -3831,7 +3839,7 @@ struct AADereferenceableFloating
 
     DerefState T;
     if (!genericValueTraversal<AADereferenceable, DerefState>(
-            A, getIRPosition(), *this, T, VisitValueCB))
+            A, getIRPosition(), *this, T, VisitValueCB, getCtxI()))
       return indicatePessimisticFixpoint();
 
     return Change | clampStateAndIndicateChange(getState(), T);
@@ -3897,10 +3905,21 @@ struct AADereferenceableCallSiteReturned final
 
 // ------------------------ Align Argument Attribute ------------------------
 
-static unsigned int getKnownAlignForUse(Attributor &A,
-                                        AbstractAttribute &QueryingAA,
-                                        Value &AssociatedValue, const Use *U,
-                                        const Instruction *I, bool &TrackUse) {
+/// \p Ptr is accessed so we can get alignment information if the ABI requires
+/// the element type to be aligned.
+static MaybeAlign getKnownAlignmentFromAccessedPtr(const Value *Ptr,
+                                                   const DataLayout &DL) {
+  MaybeAlign KnownAlignment = Ptr->getPointerAlignment(DL);
+  Type *ElementTy = Ptr->getType()->getPointerElementType();
+  if (ElementTy->isSized())
+    KnownAlignment = max(KnownAlignment, DL.getABITypeAlign(ElementTy));
+  return KnownAlignment;
+}
+
+static unsigned getKnownAlignForUse(Attributor &A,
+                                    AbstractAttribute &QueryingAA,
+                                    Value &AssociatedValue, const Use *U,
+                                    const Instruction *I, bool &TrackUse) {
   // We need to follow common pointer manipulation uses to the accesses they
   // feed into.
   if (isa<CastInst>(I)) {
@@ -3915,7 +3934,7 @@ static unsigned int getKnownAlignForUse(Attributor &A,
     }
   }
 
-  unsigned Alignment = 0;
+  MaybeAlign MA;
   if (ImmutableCallSite ICS = ImmutableCallSite(I)) {
     if (ICS.isBundleOperand(U) || ICS.isCallee(U))
       return 0;
@@ -3926,20 +3945,31 @@ static unsigned int getKnownAlignForUse(Attributor &A,
     // dependences here.
     auto &AlignAA = A.getAAFor<AAAlign>(QueryingAA, IRP,
                                         /* TrackDependence */ false);
-    Alignment = AlignAA.getKnownAlign();
+    MA = MaybeAlign(AlignAA.getKnownAlign());
   }
 
+  const DataLayout &DL = A.getDataLayout();
   const Value *UseV = U->get();
   if (auto *SI = dyn_cast<StoreInst>(I)) {
-    if (SI->getPointerOperand() == UseV)
-      Alignment = SI->getAlignment();
-  } else if (auto *LI = dyn_cast<LoadInst>(I))
-    Alignment = LI->getAlignment();
+    if (SI->getPointerOperand() == UseV) {
+      if (unsigned SIAlign = SI->getAlignment())
+        MA = MaybeAlign(SIAlign);
+      else
+        MA = getKnownAlignmentFromAccessedPtr(UseV, DL);
+    }
+  } else if (auto *LI = dyn_cast<LoadInst>(I)) {
+    if (LI->getPointerOperand() == UseV) {
+      if (unsigned LIAlign = LI->getAlignment())
+        MA = MaybeAlign(LIAlign);
+      else
+        MA = getKnownAlignmentFromAccessedPtr(UseV, DL);
+    }
+  }
 
-  if (Alignment <= 1)
+  if (!MA.hasValue() || MA <= 1)
     return 0;
 
-  auto &DL = A.getDataLayout();
+  unsigned Alignment = MA->value();
   int64_t Offset;
 
   if (const Value *Base = GetPointerBaseWithConstantOffset(UseV, Offset, DL)) {
@@ -3956,6 +3986,7 @@ static unsigned int getKnownAlignForUse(Attributor &A,
 
   return Alignment;
 }
+
 struct AAAlignImpl : AAAlign {
   AAAlignImpl(const IRPosition &IRP) : AAAlign(IRP) {}
 
@@ -4050,8 +4081,8 @@ struct AAAlignFloating : AAFromMustBeExecutedContext<AAAlign, AAAlignImpl> {
 
     const DataLayout &DL = A.getDataLayout();
 
-    auto VisitValueCB = [&](Value &V, AAAlign::StateType &T,
-                            bool Stripped) -> bool {
+    auto VisitValueCB = [&](Value &V, const Instruction *,
+                            AAAlign::StateType &T, bool Stripped) -> bool {
       const auto &AA = A.getAAFor<AAAlign>(*this, IRPosition::value(V));
       if (!Stripped && this == &AA) {
         // Use only IR information if we did not strip anything.
@@ -4069,7 +4100,7 @@ struct AAAlignFloating : AAFromMustBeExecutedContext<AAAlign, AAAlignImpl> {
 
     StateType T;
     if (!genericValueTraversal<AAAlign, StateType>(A, getIRPosition(), *this, T,
-                                                   VisitValueCB))
+                                                   VisitValueCB, getCtxI()))
       return indicatePessimisticFixpoint();
 
     // TODO: If we know we visited all incoming values, thus no are assumed
@@ -4095,10 +4126,20 @@ struct AAAlignReturned final
 struct AAAlignArgument final
     : AAArgumentFromCallSiteArgumentsAndMustBeExecutedContext<AAAlign,
                                                               AAAlignImpl> {
-  AAAlignArgument(const IRPosition &IRP)
-      : AAArgumentFromCallSiteArgumentsAndMustBeExecutedContext<AAAlign,
-                                                                AAAlignImpl>(
-            IRP) {}
+  using Base =
+      AAArgumentFromCallSiteArgumentsAndMustBeExecutedContext<AAAlign,
+                                                              AAAlignImpl>;
+  AAAlignArgument(const IRPosition &IRP) : Base(IRP) {}
+
+  /// See AbstractAttribute::manifest(...).
+  ChangeStatus manifest(Attributor &A) override {
+    // If the associated argument is involved in a must-tail call we give up
+    // because we would need to keep the argument alignments of caller and
+    // callee in-sync. Just does not seem worth the trouble right now.
+    if (A.getInfoCache().isInvolvedInMustTailCall(*getAssociatedArgument()))
+      return ChangeStatus::UNCHANGED;
+    return Base::manifest(A);
+  }
 
   /// See AbstractAttribute::trackStatistics()
   void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(aligned) }
@@ -4109,6 +4150,12 @@ struct AAAlignCallSiteArgument final : AAAlignFloating {
 
   /// See AbstractAttribute::manifest(...).
   ChangeStatus manifest(Attributor &A) override {
+    // If the associated argument is involved in a must-tail call we give up
+    // because we would need to keep the argument alignments of caller and
+    // callee in-sync. Just does not seem worth the trouble right now.
+    if (Argument *Arg = getAssociatedArgument())
+      if (A.getInfoCache().isInvolvedInMustTailCall(*Arg))
+        return ChangeStatus::UNCHANGED;
     ChangeStatus Changed = AAAlignImpl::manifest(A);
     MaybeAlign InheritAlign =
         getAssociatedValue().getPointerAlignment(A.getDataLayout());
@@ -4919,7 +4966,8 @@ struct AAValueSimplifyFloating : AAValueSimplifyImpl {
   ChangeStatus updateImpl(Attributor &A) override {
     bool HasValueBefore = SimplifiedAssociatedValue.hasValue();
 
-    auto VisitValueCB = [&](Value &V, bool &, bool Stripped) -> bool {
+    auto VisitValueCB = [&](Value &V, const Instruction *CtxI, bool &,
+                            bool Stripped) -> bool {
       auto &AA = A.getAAFor<AAValueSimplify>(*this, IRPosition::value(V));
       if (!Stripped && this == &AA) {
         // TODO: Look the instruction and check recursively.
@@ -4932,8 +4980,8 @@ struct AAValueSimplifyFloating : AAValueSimplifyImpl {
     };
 
     bool Dummy = false;
-    if (!genericValueTraversal<AAValueSimplify, bool>(A, getIRPosition(), *this,
-                                                      Dummy, VisitValueCB))
+    if (!genericValueTraversal<AAValueSimplify, bool>(
+            A, getIRPosition(), *this, Dummy, VisitValueCB, getCtxI()))
       if (!askSimplifiedValueForAAValueConstantRange(A))
         return indicatePessimisticFixpoint();
 
@@ -5030,20 +5078,27 @@ struct AAHeapToStackImpl : public AAHeapToStack {
       LLVM_DEBUG(dbgs() << "H2S: Removing malloc call: " << *MallocCall
                         << "\n");
 
+      MaybeAlign Alignment;
       Constant *Size;
       if (isCallocLikeFn(MallocCall, TLI)) {
         auto *Num = cast<ConstantInt>(MallocCall->getOperand(0));
-        auto *SizeT = dyn_cast<ConstantInt>(MallocCall->getOperand(1));
+        auto *SizeT = cast<ConstantInt>(MallocCall->getOperand(1));
         APInt TotalSize = SizeT->getValue() * Num->getValue();
         Size =
             ConstantInt::get(MallocCall->getOperand(0)->getType(), TotalSize);
+      } else if (isAlignedAllocLikeFn(MallocCall, TLI)) {
+        Size = cast<ConstantInt>(MallocCall->getOperand(1));
+        Alignment = MaybeAlign(cast<ConstantInt>(MallocCall->getOperand(0))
+                                   ->getValue()
+                                   .getZExtValue());
       } else {
         Size = cast<ConstantInt>(MallocCall->getOperand(0));
       }
 
       unsigned AS = cast<PointerType>(MallocCall->getType())->getAddressSpace();
-      Instruction *AI = new AllocaInst(Type::getInt8Ty(F->getContext()), AS,
-                                       Size, "", MallocCall->getNextNode());
+      Instruction *AI =
+          new AllocaInst(Type::getInt8Ty(F->getContext()), AS, Size, Alignment,
+                         "", MallocCall->getNextNode());
 
       if (AI->getType() != MallocCall->getType())
         AI = new BitCastInst(AI, MallocCall->getType(), "malloc_bc",
@@ -5059,6 +5114,7 @@ struct AAHeapToStackImpl : public AAHeapToStack {
         A.deleteAfterManifest(*MallocCall);
       }
 
+      // Zero out the allocated memory if it was a calloc.
       if (isCallocLikeFn(MallocCall, TLI)) {
         auto *BI = new BitCastInst(AI, MallocCall->getType(), "calloc_bc",
                                    AI->getNextNode());
@@ -5174,8 +5230,9 @@ ChangeStatus AAHeapToStackImpl::updateImpl(Attributor &A) {
       return true;
 
     bool IsMalloc = isMallocLikeFn(&I, TLI);
+    bool IsAlignedAllocLike = isAlignedAllocLikeFn(&I, TLI);
     bool IsCalloc = !IsMalloc && isCallocLikeFn(&I, TLI);
-    if (!IsMalloc && !IsCalloc) {
+    if (!IsMalloc && !IsAlignedAllocLike && !IsCalloc) {
       BadMallocCalls.insert(&I);
       return true;
     }
@@ -5187,6 +5244,14 @@ ChangeStatus AAHeapToStackImpl::updateImpl(Attributor &A) {
             MallocCalls.insert(&I);
             return true;
           }
+    } else if (IsAlignedAllocLike && isa<ConstantInt>(I.getOperand(0))) {
+      // Only if the alignment and sizes are constant.
+      if (auto *Size = dyn_cast<ConstantInt>(I.getOperand(1)))
+        if (Size->getValue().ule(MaxHeapToStackSize))
+          if (UsesCheck(I) || FreeCheck(I)) {
+            MallocCalls.insert(&I);
+            return true;
+          }
     } else if (IsCalloc) {
       bool Overflow = false;
       if (auto *Num = dyn_cast<ConstantInt>(I.getOperand(0)))
@@ -5216,10 +5281,11 @@ ChangeStatus AAHeapToStackImpl::updateImpl(Attributor &A) {
 struct AAHeapToStackFunction final : public AAHeapToStackImpl {
   AAHeapToStackFunction(const IRPosition &IRP) : AAHeapToStackImpl(IRP) {}
 
-  /// See AbstractAttribute::trackStatistics()
+  /// See AbstractAttribute::trackStatistics().
   void trackStatistics() const override {
-    STATS_DECL(MallocCalls, Function,
-               "Number of malloc calls converted to allocas");
+    STATS_DECL(
+        MallocCalls, Function,
+        "Number of malloc/calloc/aligned_alloc calls converted to allocas");
     for (auto *C : MallocCalls)
       if (!BadMallocCalls.count(C))
         ++BUILD_STAT_NAME(MallocCalls, Function);
@@ -6548,7 +6614,8 @@ void AAMemoryLocationImpl::categorizePtrValue(
     return V;
   };
 
-  auto VisitValueCB = [&](Value &V, AAMemoryLocation::StateType &T,
+  auto VisitValueCB = [&](Value &V, const Instruction *,
+                          AAMemoryLocation::StateType &T,
                           bool Stripped) -> bool {
     assert(!isa<GEPOperator>(V) && "GEPs should have been stripped.");
     if (isa<UndefValue>(V))
@@ -6595,7 +6662,7 @@ void AAMemoryLocationImpl::categorizePtrValue(
   };
 
   if (!genericValueTraversal<AAMemoryLocation, AAMemoryLocation::StateType>(
-          A, IRPosition::value(Ptr), *this, State, VisitValueCB,
+          A, IRPosition::value(Ptr), *this, State, VisitValueCB, getCtxI(),
           /* MaxValues */ 32, StripGEPCB)) {
     LLVM_DEBUG(
         dbgs() << "[AAMemoryLocation] Pointer locations not categorized\n");
@@ -6995,10 +7062,18 @@ struct AAValueConstantRangeImpl : AAValueConstantRange {
 struct AAValueConstantRangeArgument final
     : AAArgumentFromCallSiteArguments<
           AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState> {
-  AAValueConstantRangeArgument(const IRPosition &IRP)
-      : AAArgumentFromCallSiteArguments<
-            AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState>(
-            IRP) {}
+  using Base = AAArgumentFromCallSiteArguments<
+      AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState>;
+  AAValueConstantRangeArgument(const IRPosition &IRP) : Base(IRP) {}
+
+  /// See AbstractAttribute::initialize(..).
+  void initialize(Attributor &A) override {
+    if (!getAnchorScope() || getAnchorScope()->isDeclaration()) {
+      indicatePessimisticFixpoint();
+    } else {
+      Base::initialize(A);
+    }
+  }
 
   /// See AbstractAttribute::trackStatistics()
   void trackStatistics() const override {
@@ -7067,7 +7142,7 @@ struct AAValueConstantRangeFloating : AAValueConstantRangeImpl {
 
   bool calculateBinaryOperator(
       Attributor &A, BinaryOperator *BinOp, IntegerRangeState &T,
-      Instruction *CtxI,
+      const Instruction *CtxI,
       SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
     Value *LHS = BinOp->getOperand(0);
     Value *RHS = BinOp->getOperand(1);
@@ -7095,7 +7170,8 @@ struct AAValueConstantRangeFloating : AAValueConstantRangeImpl {
   }
 
   bool calculateCastInst(
-      Attributor &A, CastInst *CastI, IntegerRangeState &T, Instruction *CtxI,
+      Attributor &A, CastInst *CastI, IntegerRangeState &T,
+      const Instruction *CtxI,
       SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
     assert(CastI->getNumOperands() == 1 && "Expected cast to be unary!");
     // TODO: Allow non integers as well.
@@ -7113,7 +7189,7 @@ struct AAValueConstantRangeFloating : AAValueConstantRangeImpl {
 
   bool
   calculateCmpInst(Attributor &A, CmpInst *CmpI, IntegerRangeState &T,
-                   Instruction *CtxI,
+                   const Instruction *CtxI,
                    SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
     Value *LHS = CmpI->getOperand(0);
     Value *RHS = CmpI->getOperand(1);
@@ -7168,9 +7244,8 @@ struct AAValueConstantRangeFloating : AAValueConstantRangeImpl {
 
   /// See AbstractAttribute::updateImpl(...).
   ChangeStatus updateImpl(Attributor &A) override {
-    Instruction *CtxI = getCtxI();
-    auto VisitValueCB = [&](Value &V, IntegerRangeState &T,
-                            bool Stripped) -> bool {
+    auto VisitValueCB = [&](Value &V, const Instruction *CtxI,
+                            IntegerRangeState &T, bool Stripped) -> bool {
       Instruction *I = dyn_cast<Instruction>(&V);
       if (!I || isa<CallBase>(I)) {
 
@@ -7220,7 +7295,7 @@ struct AAValueConstantRangeFloating : AAValueConstantRangeImpl {
     IntegerRangeState T(getBitWidth());
 
     if (!genericValueTraversal<AAValueConstantRange, IntegerRangeState>(
-            A, getIRPosition(), *this, T, VisitValueCB))
+            A, getIRPosition(), *this, T, VisitValueCB, getCtxI()))
       return indicatePessimisticFixpoint();
 
     return clampStateAndIndicateChange(getState(), T);
@@ -7291,6 +7366,16 @@ struct AAValueConstantRangeCallSiteArgument : AAValueConstantRangeFloating {
 ///                               Attributor
 /// ----------------------------------------------------------------------------
 
+Attributor::~Attributor() {
+  // The abstract attributes are allocated via the BumpPtrAllocator Allocator,
+  // thus we cannot delete them. We can, and want to, destruct them though.
+  for (AbstractAttribute *AA : AllAbstractAttributes)
+    AA->~AbstractAttribute();
+
+  for (auto &It : ArgumentReplacementMap)
+    DeleteContainerPointers(It.second);
+}
+
 bool Attributor::isAssumedDead(const AbstractAttribute &AA,
                                const AAIsDead *FnLivenessAA,
                                bool CheckBBLivenessOnly, DepClassTy DepClass) {
@@ -8351,17 +8436,18 @@ void Attributor::initializeInformationCache(Function &F) {
              "Attributor.");
       break;
     case Instruction::Call:
-      // Calls are interesting but for `llvm.assume` calls we also fill the
-      // KnowledgeMap as we find them.
+      // Calls are interesting on their own, additionally:
+      // For `llvm.assume` calls we also fill the KnowledgeMap as we find them.
+      // For `must-tail` calls we remember the caller and callee.
       if (IntrinsicInst *Assume = dyn_cast<IntrinsicInst>(&I)) {
         if (Assume->getIntrinsicID() == Intrinsic::assume)
           fillMapFromAssume(*Assume, InfoCache.KnowledgeMap);
+      } else if (cast<CallInst>(I).isMustTailCall()) {
+        InfoCache.FunctionsWithMustTailCall.insert(&F);
+        InfoCache.FunctionsCalledViaMustTail.insert(
+            cast<CallInst>(I).getCalledFunction());
       }
       LLVM_FALLTHROUGH;
-    case Instruction::Load:
-      // The alignment of a pointer is interesting for loads.
-    case Instruction::Store:
-      // The alignment of a pointer is interesting for stores.
     case Instruction::CallBr:
     case Instruction::Invoke:
     case Instruction::CleanupRet:
@@ -8371,6 +8457,10 @@ void Attributor::initializeInformationCache(Function &F) {
     case Instruction::Br:
     case Instruction::Resume:
     case Instruction::Ret:
+    case Instruction::Load:
+      // The alignment of a pointer is interesting for loads.
+    case Instruction::Store:
+      // The alignment of a pointer is interesting for stores.
       IsInterestingOpcode = true;
     }
     if (IsInterestingOpcode)
@@ -8405,6 +8495,15 @@ void Attributor::identifyDefaultAbstractAttributes(Function &F) {
   if (F.isDeclaration())
     return;
 
+  // In non-module runs we need to look at the call sites of a function to
+  // determine if it is part of a must-tail call edge. This will influence what
+  // attributes we can derive.
+  if (!isModulePass() && !InfoCache.FunctionsCalledViaMustTail.count(&F))
+    for (const Use &U : F.uses())
+      if (ImmutableCallSite ICS = ImmutableCallSite(U.getUser()))
+        if (ICS.isCallee(&U) && ICS.isMustTailCall())
+          InfoCache.FunctionsCalledViaMustTail.insert(&F);
+
   IRPosition FPos = IRPosition::function(F);
 
   // Check for dead BasicBlocks in every function.
@@ -8713,9 +8812,10 @@ static bool runAttributorOnFunctions(InformationCache &InfoCache,
     A.identifyDefaultAbstractAttributes(*F);
   }
 
+  Module &M = *Functions.front()->getParent();
+  (void)M;
   ChangeStatus Changed = A.run();
-  assert(!verifyModule(*Functions.front()->getParent(), &errs()) &&
-         "Module verification failed!");
+  assert(!verifyModule(M, &errs()) && "Module verification failed!");
   LLVM_DEBUG(dbgs() << "[Attributor] Done with " << Functions.size()
                     << " functions, result: " << Changed << ".\n");
   return Changed == ChangeStatus::CHANGED;
@@ -8873,7 +8973,7 @@ const char AAValueConstantRange::ID = 0;
 
 #define SWITCH_PK_CREATE(CLASS, IRP, PK, SUFFIX)                               \
   case IRPosition::PK:                                                         \
-    AA = new CLASS##SUFFIX(IRP);                                               \
+    AA = new (A.Allocator) CLASS##SUFFIX(IRP);                                 \
     break;
 
 #define CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS)                 \
diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp
index 5ce3da340ade2..1b10c5433b469 100644
--- a/llvm/lib/Transforms/IPO/Inliner.cpp
+++ b/llvm/lib/Transforms/IPO/Inliner.cpp
@@ -712,8 +712,22 @@ inlineCallsImpl(CallGraphSCC &SCC, CallGraph &CG,
           int NewHistoryID = InlineHistory.size();
           InlineHistory.push_back(std::make_pair(Callee, InlineHistoryID));
 
-          for (Value *Ptr : InlineInfo.InlinedCalls)
+#ifndef NDEBUG
+          // Make sure no dupplicates in the inline candidates. This could
+          // happen when a callsite is simpilfied to reusing the return value
+          // of another callsite during function cloning, thus the other
+          // callsite will be reconsidered here.
+          DenseSet<CallSite> DbgCallSites;
+          for (auto &II : CallSites)
+            DbgCallSites.insert(II.first);
+#endif
+
+          for (Value *Ptr : InlineInfo.InlinedCalls) {
+#ifndef NDEBUG
+            assert(DbgCallSites.count(CallSite(Ptr)) == 0);
+#endif
             CallSites.push_back(std::make_pair(CallSite(Ptr), NewHistoryID));
+          }
         }
       }
 
diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp
index b5b834030a38f..9c1369fdcf998 100644
--- a/llvm/lib/Transforms/IPO/SampleProfile.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp
@@ -236,7 +236,7 @@ class GUIDToFuncNameMapper {
                         DenseMap<uint64_t, StringRef> &GUIDToFuncNameMap)
       : CurrentReader(Reader), CurrentModule(M),
       CurrentGUIDToFuncNameMap(GUIDToFuncNameMap) {
-    if (CurrentReader.getFormat() != SPF_Compact_Binary)
+    if (!CurrentReader.useMD5())
       return;
 
     for (const auto &F : CurrentModule) {
@@ -262,7 +262,7 @@ class GUIDToFuncNameMapper {
   }
 
   ~GUIDToFuncNameMapper() {
-    if (CurrentReader.getFormat() != SPF_Compact_Binary)
+    if (!CurrentReader.useMD5())
       return;
 
     CurrentGUIDToFuncNameMap.clear();
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index 06ec02789750a..01ba1031fb726 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -2889,6 +2889,7 @@ Value *InstCombiner::foldXorOfICmps(ICmpInst *LHS, ICmpInst *RHS,
           Builder.SetInsertPoint(Y->getParent(), ++(Y->getIterator()));
           Value *NotY = Builder.CreateNot(Y, Y->getName() + ".not");
           // Replace all uses of Y (excluding the one in NotY!) with NotY.
+          Worklist.pushUsersToWorkList(*Y);
           Y->replaceUsesWithIf(NotY,
                                [NotY](Use &U) { return U.getUser() != NotY; });
         }
@@ -3069,8 +3070,12 @@ Instruction *InstCombiner::visitXor(BinaryOperator &I) {
     // ~(C >>s Y) --> ~C >>u Y (when inverting the replicated sign bits)
     Constant *C;
     if (match(NotVal, m_AShr(m_Constant(C), m_Value(Y))) &&
-        match(C, m_Negative()))
-      return BinaryOperator::CreateLShr(ConstantExpr::getNot(C), Y);
+        match(C, m_Negative())) {
+      Constant *NewC = ConstantExpr::getNot(C);
+      if (C->getType()->isVectorTy())
+        NewC = getSafeVectorConstantForBinop(Instruction::LShr, NewC, false);
+      return BinaryOperator::CreateLShr(NewC, Y);
+    }
 
     // ~(C >>u Y) --> ~C >>s Y (when inverting the replicated sign bits)
     if (match(NotVal, m_LShr(m_Constant(C), m_Value(Y))) &&
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index b28da94649334..cd0b5e87e5f11 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -22,6 +22,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/Analysis/AssumeBundleQueries.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/Loads.h"
@@ -48,7 +49,6 @@
 #include "llvm/IR/IntrinsicsNVPTX.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/IR/IntrinsicsPowerPC.h"
-#include "llvm/IR/KnowledgeRetention.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/PatternMatch.h"
@@ -1164,10 +1164,8 @@ Instruction *InstCombiner::simplifyMaskedStore(IntrinsicInst &II) {
   APInt DemandedElts = possiblyDemandedEltsInMask(ConstMask);
   APInt UndefElts(DemandedElts.getBitWidth(), 0);
   if (Value *V = SimplifyDemandedVectorElts(II.getOperand(0),
-                                            DemandedElts, UndefElts)) {
-    II.setOperand(0, V);
-    return &II;
-  }
+                                            DemandedElts, UndefElts))
+    return replaceOperand(II, 0, V);
 
   return nullptr;
 }
@@ -1202,15 +1200,11 @@ Instruction *InstCombiner::simplifyMaskedScatter(IntrinsicInst &II) {
   APInt DemandedElts = possiblyDemandedEltsInMask(ConstMask);
   APInt UndefElts(DemandedElts.getBitWidth(), 0);
   if (Value *V = SimplifyDemandedVectorElts(II.getOperand(0),
-                                            DemandedElts, UndefElts)) {
-    II.setOperand(0, V);
-    return &II;
-  }
+                                            DemandedElts, UndefElts))
+    return replaceOperand(II, 0, V);
   if (Value *V = SimplifyDemandedVectorElts(II.getOperand(1),
-                                            DemandedElts, UndefElts)) {
-    II.setOperand(1, V);
-    return &II;
-  }
+                                            DemandedElts, UndefElts))
+    return replaceOperand(II, 1, V);
 
   return nullptr;
 }
@@ -2322,6 +2316,22 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
       }
     }
 
+    Value *ExtSrc0;
+    Value *ExtSrc1;
+
+    // minnum (fpext x), (fpext y) -> minnum x, y
+    // maxnum (fpext x), (fpext y) -> maxnum x, y
+    if (match(II->getArgOperand(0), m_OneUse(m_FPExt(m_Value(ExtSrc0)))) &&
+        match(II->getArgOperand(1), m_OneUse(m_FPExt(m_Value(ExtSrc1)))) &&
+        ExtSrc0->getType() == ExtSrc1->getType()) {
+      Value *F = Intrinsic::getDeclaration(II->getModule(), II->getIntrinsicID(),
+                                           { ExtSrc0->getType() });
+      CallInst *NewCall = Builder.CreateCall(F, { ExtSrc0, ExtSrc1 });
+      NewCall->copyFastMathFlags(II);
+      NewCall->takeName(II);
+      return new FPExtInst(NewCall, II->getType());
+    }
+
     break;
   }
   case Intrinsic::fmuladd: {
@@ -2667,10 +2677,8 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
     // we can simplify the input based on that, do so now.
     Value *Arg = II->getArgOperand(0);
     unsigned VWidth = Arg->getType()->getVectorNumElements();
-    if (Value *V = SimplifyDemandedVectorEltsLow(Arg, VWidth, 1)) {
-      II->setArgOperand(0, V);
-      return II;
-    }
+    if (Value *V = SimplifyDemandedVectorEltsLow(Arg, VWidth, 1))
+      return replaceOperand(*II, 0, V);
     break;
   }
 
@@ -2720,11 +2728,11 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
     Value *Arg1 = II->getArgOperand(1);
     unsigned VWidth = Arg0->getType()->getVectorNumElements();
     if (Value *V = SimplifyDemandedVectorEltsLow(Arg0, VWidth, 1)) {
-      II->setArgOperand(0, V);
+      replaceOperand(*II, 0, V);
       MadeChange = true;
     }
     if (Value *V = SimplifyDemandedVectorEltsLow(Arg1, VWidth, 1)) {
-      II->setArgOperand(1, V);
+      replaceOperand(*II, 1, V);
       MadeChange = true;
     }
     if (MadeChange)
@@ -2938,10 +2946,8 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
            "Unexpected packed shift size");
     unsigned VWidth = Arg1->getType()->getVectorNumElements();
 
-    if (Value *V = SimplifyDemandedVectorEltsLow(Arg1, VWidth, VWidth / 2)) {
-      II->setArgOperand(1, V);
-      return II;
-    }
+    if (Value *V = SimplifyDemandedVectorEltsLow(Arg1, VWidth, VWidth / 2))
+      return replaceOperand(*II, 1, V);
     break;
   }
 
@@ -3012,7 +3018,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
                                             APInt(2, (Imm & 0x01) ? 2 : 1));
       if (Value *V = SimplifyDemandedVectorElts(Arg0, DemandedElts1,
                                                 UndefElts1)) {
-        II->setArgOperand(0, V);
+        replaceOperand(*II, 0, V);
         MadeChange = true;
       }
 
@@ -3021,7 +3027,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
                                             APInt(2, (Imm & 0x10) ? 2 : 1));
       if (Value *V = SimplifyDemandedVectorElts(Arg1, DemandedElts2,
                                                 UndefElts2)) {
-        II->setArgOperand(1, V);
+        replaceOperand(*II, 1, V);
         MadeChange = true;
       }
 
@@ -3068,11 +3074,11 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
     // operands and the lowest 16-bits of the second.
     bool MadeChange = false;
     if (Value *V = SimplifyDemandedVectorEltsLow(Op0, VWidth0, 1)) {
-      II->setArgOperand(0, V);
+      replaceOperand(*II, 0, V);
       MadeChange = true;
     }
     if (Value *V = SimplifyDemandedVectorEltsLow(Op1, VWidth1, 2)) {
-      II->setArgOperand(1, V);
+      replaceOperand(*II, 1, V);
       MadeChange = true;
     }
     if (MadeChange)
@@ -3098,10 +3104,8 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
 
     // EXTRQI only uses the lowest 64-bits of the first 128-bit vector
     // operand.
-    if (Value *V = SimplifyDemandedVectorEltsLow(Op0, VWidth, 1)) {
-      II->setArgOperand(0, V);
-      return II;
-    }
+    if (Value *V = SimplifyDemandedVectorEltsLow(Op0, VWidth, 1))
+      return replaceOperand(*II, 0, V);
     break;
   }
 
@@ -3131,10 +3135,8 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
 
     // INSERTQ only uses the lowest 64-bits of the first 128-bit vector
     // operand.
-    if (Value *V = SimplifyDemandedVectorEltsLow(Op0, VWidth, 1)) {
-      II->setArgOperand(0, V);
-      return II;
-    }
+    if (Value *V = SimplifyDemandedVectorEltsLow(Op0, VWidth, 1))
+      return replaceOperand(*II, 0, V);
     break;
   }
 
@@ -3166,11 +3168,11 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
     // operands.
     bool MadeChange = false;
     if (Value *V = SimplifyDemandedVectorEltsLow(Op0, VWidth0, 1)) {
-      II->setArgOperand(0, V);
+      replaceOperand(*II, 0, V);
       MadeChange = true;
     }
     if (Value *V = SimplifyDemandedVectorEltsLow(Op1, VWidth1, 1)) {
-      II->setArgOperand(1, V);
+      replaceOperand(*II, 1, V);
       MadeChange = true;
     }
     if (MadeChange)
@@ -3969,6 +3971,35 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
 
     break;
   }
+  case Intrinsic::amdgcn_ballot: {
+    if (auto *Src = dyn_cast<ConstantInt>(II->getArgOperand(0))) {
+      if (Src->isZero()) {
+        // amdgcn.ballot(i1 0) is zero.
+        return replaceInstUsesWith(*II, Constant::getNullValue(II->getType()));
+      }
+
+      if (Src->isOne()) {
+        // amdgcn.ballot(i1 1) is exec.
+        const char *RegName = "exec";
+        if (II->getType()->isIntegerTy(32))
+          RegName = "exec_lo";
+        else if (!II->getType()->isIntegerTy(64))
+          break;
+
+        Function *NewF = Intrinsic::getDeclaration(
+            II->getModule(), Intrinsic::read_register, II->getType());
+        Metadata *MDArgs[] = {MDString::get(II->getContext(), RegName)};
+        MDNode *MD = MDNode::get(II->getContext(), MDArgs);
+        Value *Args[] = {MetadataAsValue::get(II->getContext(), MD)};
+        CallInst *NewCall = Builder.CreateCall(NewF, Args);
+        NewCall->addAttribute(AttributeList::FunctionIndex,
+                              Attribute::Convergent);
+        NewCall->takeName(II);
+        return replaceInstUsesWith(*II, NewCall);
+      }
+    }
+    break;
+  }
   case Intrinsic::amdgcn_wqm_vote: {
     // wqm_vote is identity when the argument is constant.
     if (!isa<Constant>(II->getArgOperand(0)))
@@ -4193,7 +4224,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
       return replaceOperand(*II, 2,
           ConstantInt::get(OpIntTy, GCR.getBasePtrIndex()));
     }
-    
+
     // Translate facts known about a pointer before relocating into
     // facts about the relocate value, while being careful to
     // preserve relocation semantics.
@@ -4426,7 +4457,8 @@ static void annotateAnyAllocSite(CallBase &Call, const TargetLibraryInfo *TLI) {
   ConstantInt *Op0C = dyn_cast<ConstantInt>(Call.getOperand(0));
   ConstantInt *Op1C =
       (NumArgs == 1) ? nullptr : dyn_cast<ConstantInt>(Call.getOperand(1));
-  // Bail out if the allocation size is zero.
+  // Bail out if the allocation size is zero (or an invalid alignment of zero
+  // with aligned_alloc).
   if ((Op0C && Op0C->isNullValue()) || (Op1C && Op1C->isNullValue()))
     return;
 
@@ -4439,6 +4471,18 @@ static void annotateAnyAllocSite(CallBase &Call, const TargetLibraryInfo *TLI) {
       Call.addAttribute(AttributeList::ReturnIndex,
                         Attribute::getWithDereferenceableOrNullBytes(
                             Call.getContext(), Op0C->getZExtValue()));
+  } else if (isAlignedAllocLikeFn(&Call, TLI) && Op1C) {
+    Call.addAttribute(AttributeList::ReturnIndex,
+                      Attribute::getWithDereferenceableOrNullBytes(
+                          Call.getContext(), Op1C->getZExtValue()));
+    // Add alignment attribute if alignment is a power of two constant.
+    if (Op0C) {
+      uint64_t AlignmentVal = Op0C->getZExtValue();
+      if (llvm::isPowerOf2_64(AlignmentVal))
+        Call.addAttribute(AttributeList::ReturnIndex,
+                          Attribute::getWithAlignment(Call.getContext(),
+                                                      Align(AlignmentVal)));
+    }
   } else if (isReallocLikeFn(&Call, TLI) && Op1C) {
     Call.addAttribute(AttributeList::ReturnIndex,
                       Attribute::getWithDereferenceableOrNullBytes(
@@ -4573,7 +4617,7 @@ Instruction *InstCombiner::visitCallBase(CallBase &Call) {
          I != E; ++I, ++ix) {
       CastInst *CI = dyn_cast<CastInst>(*I);
       if (CI && isSafeToEliminateVarargsCast(Call, DL, CI, ix)) {
-        *I = CI->getOperand(0);
+        replaceUse(*I, CI->getOperand(0));
 
         // Update the byval type to match the argument type.
         if (Call.isByValArgument(ix)) {
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
index afdddad10cea2..6b1824048f755 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
@@ -153,6 +153,7 @@ Instruction *InstCombiner::PromoteCastOfAllocation(BitCastInst &CI,
     // allocation instruction, also pointer typed. Thus, cast to use is BitCast.
     Value *NewCast = Builder.CreateBitCast(New, AI.getType(), "tmpcast");
     replaceInstUsesWith(AI, NewCast);
+    eraseInstFromFunction(AI);
   }
   return replaceInstUsesWith(CI, New);
 }
@@ -641,12 +642,12 @@ static Instruction *shrinkSplatShuffle(TruncInst &Trunc,
                                        InstCombiner::BuilderTy &Builder) {
   auto *Shuf = dyn_cast<ShuffleVectorInst>(Trunc.getOperand(0));
   if (Shuf && Shuf->hasOneUse() && isa<UndefValue>(Shuf->getOperand(1)) &&
-      Shuf->getMask()->getSplatValue() &&
+      is_splat(Shuf->getShuffleMask()) &&
       Shuf->getType() == Shuf->getOperand(0)->getType()) {
     // trunc (shuf X, Undef, SplatMask) --> shuf (trunc X), Undef, SplatMask
     Constant *NarrowUndef = UndefValue::get(Trunc.getType());
     Value *NarrowOp = Builder.CreateTrunc(Shuf->getOperand(0), Trunc.getType());
-    return new ShuffleVectorInst(NarrowOp, NarrowUndef, Shuf->getMask());
+    return new ShuffleVectorInst(NarrowOp, NarrowUndef, Shuf->getShuffleMask());
   }
 
   return nullptr;
@@ -690,6 +691,7 @@ Instruction *InstCombiner::visitTrunc(TruncInst &CI) {
 
   Value *Src = CI.getOperand(0);
   Type *DestTy = CI.getType(), *SrcTy = Src->getType();
+  ConstantInt *Cst;
 
   // Attempt to truncate the entire input expression tree to the destination
   // type.   Only do this if the dest type is a simple type, don't convert the
@@ -758,7 +760,7 @@ Instruction *InstCombiner::visitTrunc(TruncInst &CI) {
   // more efficiently. Support vector types. Cleanup code by using m_OneUse.
 
   // Transform trunc(lshr (zext A), Cst) to eliminate one type conversion.
-  Value *A = nullptr; ConstantInt *Cst = nullptr;
+  Value *A = nullptr;
   if (Src->hasOneUse() &&
       match(Src, m_LShr(m_ZExt(m_Value(A)), m_ConstantInt(Cst)))) {
     // We have three types to worry about here, the type of A, the source of
@@ -843,6 +845,38 @@ Instruction *InstCombiner::visitTrunc(TruncInst &CI) {
   if (Instruction *I = foldVecTruncToExtElt(CI, *this))
     return I;
 
+  // Whenever an element is extracted from a vector, and then truncated,
+  // canonicalize by converting it to a bitcast followed by an
+  // extractelement.
+  //
+  // Example (little endian):
+  //   trunc (extractelement <4 x i64> %X, 0) to i32
+  //   --->
+  //   extractelement <8 x i32> (bitcast <4 x i64> %X to <8 x i32>), i32 0
+  Value *VecOp;
+  if (match(Src,
+            m_OneUse(m_ExtractElement(m_Value(VecOp), m_ConstantInt(Cst))))) {
+    Type *VecOpTy = VecOp->getType();
+    unsigned DestScalarSize = DestTy->getScalarSizeInBits();
+    unsigned VecOpScalarSize = VecOpTy->getScalarSizeInBits();
+    unsigned VecNumElts = VecOpTy->getVectorNumElements();
+
+    // A badly fit destination size would result in an invalid cast.
+    if (VecOpScalarSize % DestScalarSize == 0) {
+      uint64_t TruncRatio = VecOpScalarSize / DestScalarSize;
+      uint64_t BitCastNumElts = VecNumElts * TruncRatio;
+      uint64_t VecOpIdx = Cst->getZExtValue();
+      uint64_t NewIdx = DL.isBigEndian() ? (VecOpIdx + 1) * TruncRatio - 1
+                                         : VecOpIdx * TruncRatio;
+      assert(BitCastNumElts <= std::numeric_limits<uint32_t>::max() &&
+             "overflow 32-bits");
+
+      Type *BitCastTo = VectorType::get(DestTy, BitCastNumElts);
+      Value *BitCast = Builder.CreateBitCast(VecOp, BitCastTo);
+      return ExtractElementInst::Create(BitCast, Builder.getInt32(NewIdx));
+    }
+  }
+
   return nullptr;
 }
 
@@ -2519,7 +2553,7 @@ Instruction *InstCombiner::visitBitCast(BitCastInst &CI) {
         Value *RHS = Builder.CreateBitCast(ShufOp1, DestTy);
         // Return a new shuffle vector.  Use the same element ID's, as we
         // know the vector types match #elts.
-        return new ShuffleVectorInst(LHS, RHS, Shuf->getOperand(2));
+        return new ShuffleVectorInst(LHS, RHS, Shuf->getShuffleMask());
       }
     }
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index 63bf3462faac5..bb7d0c9307555 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -1330,6 +1330,7 @@ static Instruction *processUGT_ADDCST_ADD(ICmpInst &I, Value *A, Value *B,
   // The inner add was the result of the narrow add, zero extended to the
   // wider type.  Replace it with the result computed by the intrinsic.
   IC.replaceInstUsesWith(*OrigAdd, ZExt);
+  IC.eraseInstFromFunction(*OrigAdd);
 
   // The original icmp gets replaced with the overflow value.
   return ExtractValueInst::Create(Call, 1, "sadd.overflow");
@@ -2760,6 +2761,30 @@ static Instruction *foldICmpBitCast(ICmpInst &Cmp,
     if (match(BCSrcOp, m_UIToFP(m_Value(X))))
       if (Cmp.isEquality() && match(Op1, m_Zero()))
         return new ICmpInst(Pred, X, ConstantInt::getNullValue(X->getType()));
+
+    // If this is a sign-bit test of a bitcast of a casted FP value, eliminate
+    // the FP cast because the FP cast does not change the sign-bit.
+    const APInt *C;
+    bool TrueIfSigned;
+    if (match(Op1, m_APInt(C)) && Bitcast->hasOneUse() &&
+        isSignBitCheck(Pred, *C, TrueIfSigned)) {
+      if (match(BCSrcOp, m_FPExt(m_Value(X))) ||
+          match(BCSrcOp, m_FPTrunc(m_Value(X)))) {
+        // (bitcast (fpext/fptrunc X)) to iX) < 0 --> (bitcast X to iY) < 0
+        // (bitcast (fpext/fptrunc X)) to iX) > -1 --> (bitcast X to iY) > -1
+        Type *XType = X->getType();
+        Type *NewType = Builder.getIntNTy(XType->getScalarSizeInBits());
+        if (XType->isVectorTy())
+          NewType = VectorType::get(NewType, XType->getVectorNumElements());
+        Value *NewBitcast = Builder.CreateBitCast(X, NewType);
+        if (TrueIfSigned)
+          return new ICmpInst(ICmpInst::ICMP_SLT, NewBitcast,
+                              ConstantInt::getNullValue(NewType));
+        else
+          return new ICmpInst(ICmpInst::ICMP_SGT, NewBitcast,
+                              ConstantInt::getAllOnesValue(NewType));
+      }
+    }
   }
 
   // Test to see if the operands of the icmp are casted versions of other
@@ -2789,11 +2814,10 @@ static Instruction *foldICmpBitCast(ICmpInst &Cmp,
     return nullptr;
 
   Value *Vec;
-  Constant *Mask;
-  if (match(BCSrcOp,
-            m_ShuffleVector(m_Value(Vec), m_Undef(), m_Constant(Mask)))) {
+  ArrayRef<int> Mask;
+  if (match(BCSrcOp, m_ShuffleVector(m_Value(Vec), m_Undef(), m_Mask(Mask)))) {
     // Check whether every element of Mask is the same constant
-    if (auto *Elem = dyn_cast_or_null<ConstantInt>(Mask->getSplatValue())) {
+    if (is_splat(Mask)) {
       auto *VecTy = cast<VectorType>(BCSrcOp->getType());
       auto *EltTy = cast<IntegerType>(VecTy->getElementType());
       if (C->isSplat(EltTy->getBitWidth())) {
@@ -2802,6 +2826,7 @@ static Instruction *foldICmpBitCast(ICmpInst &Cmp,
         // then:
         //   =>  %E = extractelement <N x iK> %vec, i32 Elem
         //       icmp <pred> iK %SplatVal, <pattern>
+        Value *Elem = Builder.getInt32(Mask[0]);
         Value *Extract = Builder.CreateExtractElement(Vec, Elem);
         Value *NewC = ConstantInt::get(EltTy, C->trunc(EltTy->getBitWidth()));
         return new ICmpInst(Pred, Extract, NewC);
@@ -3646,6 +3671,11 @@ Value *InstCombiner::foldUnsignedMultiplicationOverflowCheck(ICmpInst &I) {
   if (NeedNegation) // This technically increases instruction count.
     Res = Builder.CreateNot(Res, "umul.not.ov");
 
+  // If we replaced the mul, erase it. Do this after all uses of Builder,
+  // as the mul is used as insertion point.
+  if (MulHadOtherUses)
+    eraseInstFromFunction(*Mul);
+
   return Res;
 }
 
@@ -5350,15 +5380,15 @@ static Instruction *foldVectorCmp(CmpInst &Cmp,
   bool IsFP = isa<FCmpInst>(Cmp);
 
   Value *V1, *V2;
-  Constant *M;
-  if (!match(LHS, m_ShuffleVector(m_Value(V1), m_Undef(), m_Constant(M))))
+  ArrayRef<int> M;
+  if (!match(LHS, m_ShuffleVector(m_Value(V1), m_Undef(), m_Mask(M))))
     return nullptr;
 
   // If both arguments of the cmp are shuffles that use the same mask and
   // shuffle within a single vector, move the shuffle after the cmp:
   // cmp (shuffle V1, M), (shuffle V2, M) --> shuffle (cmp V1, V2), M
   Type *V1Ty = V1->getType();
-  if (match(RHS, m_ShuffleVector(m_Value(V2), m_Undef(), m_Specific(M))) &&
+  if (match(RHS, m_ShuffleVector(m_Value(V2), m_Undef(), m_SpecificMask(M))) &&
       V1Ty == V2->getType() && (LHS->hasOneUse() || RHS->hasOneUse())) {
     Value *NewCmp = IsFP ? Builder.CreateFCmp(Pred, V1, V2)
                          : Builder.CreateICmp(Pred, V1, V2);
@@ -5375,16 +5405,16 @@ static Instruction *foldVectorCmp(CmpInst &Cmp,
   // Length-changing splats are ok, so adjust the constants as needed:
   // cmp (shuffle V1, M), C --> shuffle (cmp V1, C'), M
   Constant *ScalarC = C->getSplatValue(/* AllowUndefs */ true);
-  Constant *ScalarM = M->getSplatValue(/* AllowUndefs */ true);
-  if (ScalarC && ScalarM) {
+  int MaskSplatIndex;
+  if (ScalarC && match(M, m_SplatOrUndefMask(MaskSplatIndex))) {
     // We allow undefs in matching, but this transform removes those for safety.
     // Demanded elements analysis should be able to recover some/all of that.
     C = ConstantVector::getSplat(V1Ty->getVectorElementCount(), ScalarC);
-    M = ConstantVector::getSplat(M->getType()->getVectorElementCount(),
-                                 ScalarM);
+    SmallVector<int, 8> NewM(M.size(), MaskSplatIndex);
     Value *NewCmp = IsFP ? Builder.CreateFCmp(Pred, V1, C)
                          : Builder.CreateICmp(Pred, V1, C);
-    return new ShuffleVectorInst(NewCmp, UndefValue::get(NewCmp->getType()), M);
+    return new ShuffleVectorInst(NewCmp, UndefValue::get(NewCmp->getType()),
+                                 NewM);
   }
 
   return nullptr;
@@ -5552,6 +5582,7 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
   if (Instruction *Res = foldICmpBitCast(I, Builder))
     return Res;
 
+  // TODO: Hoist this above the min/max bailout.
   if (Instruction *R = foldICmpWithCastOp(I))
     return R;
 
@@ -5593,6 +5624,7 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
           OptimizeOverflowCheck(Instruction::Add, /*Signed*/ false, A, B, *AddI,
                                 Result, Overflow)) {
         replaceInstUsesWith(*AddI, Result);
+        eraseInstFromFunction(*AddI);
         return replaceInstUsesWith(I, Overflow);
       }
     }
@@ -5939,16 +5971,15 @@ static Instruction *foldFCmpReciprocalAndZero(FCmpInst &I, Instruction *LHSI,
 }
 
 /// Optimize fabs(X) compared with zero.
-static Instruction *foldFabsWithFcmpZero(FCmpInst &I) {
+static Instruction *foldFabsWithFcmpZero(FCmpInst &I, InstCombiner &IC) {
   Value *X;
   if (!match(I.getOperand(0), m_Intrinsic<Intrinsic::fabs>(m_Value(X))) ||
       !match(I.getOperand(1), m_PosZeroFP()))
     return nullptr;
 
-  auto replacePredAndOp0 = [](FCmpInst *I, FCmpInst::Predicate P, Value *X) {
+  auto replacePredAndOp0 = [&IC](FCmpInst *I, FCmpInst::Predicate P, Value *X) {
     I->setPredicate(P);
-    I->setOperand(0, X);
-    return I;
+    return IC.replaceOperand(*I, 0, X);
   };
 
   switch (I.getPredicate()) {
@@ -6113,7 +6144,7 @@ Instruction *InstCombiner::visitFCmpInst(FCmpInst &I) {
   }
   }
 
-  if (Instruction *R = foldFabsWithFcmpZero(I))
+  if (Instruction *R = foldFabsWithFcmpZero(I, *this))
     return R;
 
   if (match(Op0, m_FNeg(m_Value(X)))) {
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
index d6053c2617838..f21c7d14bdea2 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
+++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
@@ -686,6 +686,12 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner
     return &I;
   }
 
+  /// Replace use and add the previously used value to the worklist.
+  void replaceUse(Use &U, Value *NewValue) {
+    Worklist.addValue(U);
+    U = NewValue;
+  }
+
   /// Creates a result tuple for an overflow intrinsic \p II with a given
   /// \p Result and a constant \p Overflow value.
   Instruction *CreateOverflowTuple(IntrinsicInst *II, Value *Result,
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
index 183ab1acbd3f4..195916ead66a7 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
@@ -72,7 +72,7 @@ static Value *simplifyValueKnownNonZero(Value *V, InstCombiner &IC,
     // We know that this is an exact/nuw shift and that the input is a
     // non-zero context as well.
     if (Value *V2 = simplifyValueKnownNonZero(I->getOperand(0), IC, CxtI)) {
-      I->setOperand(0, V2);
+      IC.replaceOperand(*I, 0, V2);
       MadeChange = true;
     }
 
@@ -591,7 +591,7 @@ bool InstCombiner::simplifyDivRemOfSelectWithZeroOp(BinaryOperator &I) {
     return false;
 
   // Change the div/rem to use 'Y' instead of the select.
-  I.setOperand(1, SI->getOperand(NonNullOperand));
+  replaceOperand(I, 1, SI->getOperand(NonNullOperand));
 
   // Okay, we know we replace the operand of the div/rem with 'Y' with no
   // problem.  However, the select, or the condition of the select may have
@@ -619,11 +619,11 @@ bool InstCombiner::simplifyDivRemOfSelectWithZeroOp(BinaryOperator &I) {
     for (Instruction::op_iterator I = BBI->op_begin(), E = BBI->op_end();
          I != E; ++I) {
       if (*I == SI) {
-        *I = SI->getOperand(NonNullOperand);
+        replaceUse(*I, SI->getOperand(NonNullOperand));
         Worklist.push(&*BBI);
       } else if (*I == SelectCond) {
-        *I = NonNullOperand == 1 ? ConstantInt::getTrue(CondTy)
-                                 : ConstantInt::getFalse(CondTy);
+        replaceUse(*I, NonNullOperand == 1 ? ConstantInt::getTrue(CondTy)
+                                           : ConstantInt::getFalse(CondTy));
         Worklist.push(&*BBI);
       }
     }
@@ -682,10 +682,8 @@ Instruction *InstCombiner::commonIDivTransforms(BinaryOperator &I) {
   Type *Ty = I.getType();
 
   // The RHS is known non-zero.
-  if (Value *V = simplifyValueKnownNonZero(I.getOperand(1), *this, I)) {
-    I.setOperand(1, V);
-    return &I;
-  }
+  if (Value *V = simplifyValueKnownNonZero(I.getOperand(1), *this, I))
+    return replaceOperand(I, 1, V);
 
   // Handle cases involving: [su]div X, (select Cond, Y, Z)
   // This does not apply for fdiv.
@@ -799,8 +797,8 @@ Instruction *InstCombiner::commonIDivTransforms(BinaryOperator &I) {
     bool HasNSW = cast<OverflowingBinaryOperator>(Op1)->hasNoSignedWrap();
     bool HasNUW = cast<OverflowingBinaryOperator>(Op1)->hasNoUnsignedWrap();
     if ((IsSigned && HasNSW) || (!IsSigned && HasNUW)) {
-      I.setOperand(0, ConstantInt::get(Ty, 1));
-      I.setOperand(1, Y);
+      replaceOperand(I, 0, ConstantInt::get(Ty, 1));
+      replaceOperand(I, 1, Y);
       return &I;
     }
   }
@@ -1276,8 +1274,8 @@ Instruction *InstCombiner::visitFDiv(BinaryOperator &I) {
   // -X / -Y -> X / Y
   Value *X, *Y;
   if (match(Op0, m_FNeg(m_Value(X))) && match(Op1, m_FNeg(m_Value(Y)))) {
-    I.setOperand(0, X);
-    I.setOperand(1, Y);
+    replaceOperand(I, 0, X);
+    replaceOperand(I, 1, Y);
     return &I;
   }
 
@@ -1286,8 +1284,8 @@ Instruction *InstCombiner::visitFDiv(BinaryOperator &I) {
   // We can ignore the possibility that X is infinity because INF/INF is NaN.
   if (I.hasNoNaNs() && I.hasAllowReassoc() &&
       match(Op1, m_c_FMul(m_Specific(Op0), m_Value(Y)))) {
-    I.setOperand(0, ConstantFP::get(I.getType(), 1.0));
-    I.setOperand(1, Y);
+    replaceOperand(I, 0, ConstantFP::get(I.getType(), 1.0));
+    replaceOperand(I, 1, Y);
     return &I;
   }
 
@@ -1313,10 +1311,8 @@ Instruction *InstCombiner::commonIRemTransforms(BinaryOperator &I) {
   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
 
   // The RHS is known non-zero.
-  if (Value *V = simplifyValueKnownNonZero(I.getOperand(1), *this, I)) {
-    I.setOperand(1, V);
-    return &I;
-  }
+  if (Value *V = simplifyValueKnownNonZero(I.getOperand(1), *this, I))
+    return replaceOperand(I, 1, V);
 
   // Handle cases involving: rem X, (select Cond, Y, Z)
   if (simplifyDivRemOfSelectWithZeroOp(I))
diff --git a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp
index 6c2aead5a7542..4f6a6443b00ff 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp
@@ -1194,15 +1194,22 @@ Instruction *InstCombiner::visitPHINode(PHINode &PN) {
     if (CmpInst && isa<IntegerType>(PN.getType()) && CmpInst->isEquality() &&
         match(CmpInst->getOperand(1), m_Zero())) {
       ConstantInt *NonZeroConst = nullptr;
+      bool MadeChange = false;
       for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i) {
         Instruction *CtxI = PN.getIncomingBlock(i)->getTerminator();
         Value *VA = PN.getIncomingValue(i);
         if (isKnownNonZero(VA, DL, 0, &AC, CtxI, &DT)) {
           if (!NonZeroConst)
             NonZeroConst = GetAnyNonZeroConstInt(PN);
-          PN.setIncomingValue(i, NonZeroConst);
+
+          if (NonZeroConst != VA) {
+            replaceOperand(PN, i, NonZeroConst);
+            MadeChange = true;
+          }
         }
       }
+      if (MadeChange)
+        return &PN;
     }
   }
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index 0e4d1134c80df..15ecfe96c48d6 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -1036,7 +1036,7 @@ canonicalizeMinMaxWithConstant(SelectInst &Sel, ICmpInst &Cmp,
 /// Canonicalize all these variants to 1 pattern.
 /// This makes CSE more likely.
 static Instruction *canonicalizeAbsNabs(SelectInst &Sel, ICmpInst &Cmp,
-                                        InstCombiner::BuilderTy &Builder) {
+                                        InstCombiner &IC) {
   if (!Cmp.hasOneUse() || !isa<Constant>(Cmp.getOperand(1)))
     return nullptr;
 
@@ -1085,12 +1085,14 @@ static Instruction *canonicalizeAbsNabs(SelectInst &Sel, ICmpInst &Cmp,
   // Create the canonical RHS: RHS = sub (0, LHS).
   if (!RHSCanonicalized) {
     assert(RHS->hasOneUse() && "RHS use number is not right");
-    RHS = Builder.CreateNeg(LHS);
+    RHS = IC.Builder.CreateNeg(LHS);
     if (TVal == LHS) {
-      Sel.setFalseValue(RHS);
+      // Replace false value.
+      IC.replaceOperand(Sel, 2, RHS);
       FVal = RHS;
     } else {
-      Sel.setTrueValue(RHS);
+      // Replace true value.
+      IC.replaceOperand(Sel, 1, RHS);
       TVal = RHS;
     }
   }
@@ -1398,7 +1400,7 @@ Instruction *InstCombiner::foldSelectInstWithICmp(SelectInst &SI,
   if (Instruction *NewSel = canonicalizeMinMaxWithConstant(SI, *ICI, *this))
     return NewSel;
 
-  if (Instruction *NewAbs = canonicalizeAbsNabs(SI, *ICI, Builder))
+  if (Instruction *NewAbs = canonicalizeAbsNabs(SI, *ICI, *this))
     return NewAbs;
 
   if (Instruction *NewAbs = canonicalizeClampLike(SI, *ICI, Builder))
@@ -2055,7 +2057,7 @@ static Instruction *foldSelectCmpBitcasts(SelectInst &Sel,
 ///   %1 = extractvalue { i64, i1 } %0, 0
 ///   ret i64 %1
 ///
-static Instruction *foldSelectCmpXchg(SelectInst &SI) {
+static Value *foldSelectCmpXchg(SelectInst &SI) {
   // A helper that determines if V is an extractvalue instruction whose
   // aggregate operand is a cmpxchg instruction and whose single index is equal
   // to I. If such conditions are true, the helper returns the cmpxchg
@@ -2087,19 +2089,15 @@ static Instruction *foldSelectCmpXchg(SelectInst &SI) {
   // value of the same cmpxchg used by the condition, and the false value is the
   // cmpxchg instruction's compare operand.
   if (auto *X = isExtractFromCmpXchg(SI.getTrueValue(), 0))
-    if (X == CmpXchg && X->getCompareOperand() == SI.getFalseValue()) {
-      SI.setTrueValue(SI.getFalseValue());
-      return &SI;
-    }
+    if (X == CmpXchg && X->getCompareOperand() == SI.getFalseValue())
+      return SI.getFalseValue();
 
   // Check the false value case: The false value of the select is the returned
   // value of the same cmpxchg used by the condition, and the true value is the
   // cmpxchg instruction's compare operand.
   if (auto *X = isExtractFromCmpXchg(SI.getFalseValue(), 0))
-    if (X == CmpXchg && X->getCompareOperand() == SI.getTrueValue()) {
-      SI.setTrueValue(SI.getFalseValue());
-      return &SI;
-    }
+    if (X == CmpXchg && X->getCompareOperand() == SI.getTrueValue())
+      return SI.getFalseValue();
 
   return nullptr;
 }
@@ -2698,8 +2696,8 @@ Instruction *InstCombiner::visitSelectInst(SelectInst &SI) {
       // paths for the values (this helps GetUnderlyingObjects() for example).
       if (TrueSI->getFalseValue() == FalseVal && TrueSI->hasOneUse()) {
         Value *And = Builder.CreateAnd(CondVal, TrueSI->getCondition());
-        SI.setOperand(0, And);
-        SI.setOperand(1, TrueSI->getTrueValue());
+        replaceOperand(SI, 0, And);
+        replaceOperand(SI, 1, TrueSI->getTrueValue());
         return &SI;
       }
     }
@@ -2715,8 +2713,8 @@ Instruction *InstCombiner::visitSelectInst(SelectInst &SI) {
       // select(C0, a, select(C1, a, b)) -> select(C0|C1, a, b)
       if (FalseSI->getTrueValue() == TrueVal && FalseSI->hasOneUse()) {
         Value *Or = Builder.CreateOr(CondVal, FalseSI->getCondition());
-        SI.setOperand(0, Or);
-        SI.setOperand(2, FalseSI->getFalseValue());
+        replaceOperand(SI, 0, Or);
+        replaceOperand(SI, 2, FalseSI->getFalseValue());
         return &SI;
       }
     }
@@ -2743,14 +2741,14 @@ Instruction *InstCombiner::visitSelectInst(SelectInst &SI) {
       canMergeSelectThroughBinop(TrueBO)) {
     if (auto *TrueBOSI = dyn_cast<SelectInst>(TrueBO->getOperand(0))) {
       if (TrueBOSI->getCondition() == CondVal) {
-        TrueBO->setOperand(0, TrueBOSI->getTrueValue());
+        replaceOperand(*TrueBO, 0, TrueBOSI->getTrueValue());
         Worklist.push(TrueBO);
         return &SI;
       }
     }
     if (auto *TrueBOSI = dyn_cast<SelectInst>(TrueBO->getOperand(1))) {
       if (TrueBOSI->getCondition() == CondVal) {
-        TrueBO->setOperand(1, TrueBOSI->getTrueValue());
+        replaceOperand(*TrueBO, 1, TrueBOSI->getTrueValue());
         Worklist.push(TrueBO);
         return &SI;
       }
@@ -2763,14 +2761,14 @@ Instruction *InstCombiner::visitSelectInst(SelectInst &SI) {
       canMergeSelectThroughBinop(FalseBO)) {
     if (auto *FalseBOSI = dyn_cast<SelectInst>(FalseBO->getOperand(0))) {
       if (FalseBOSI->getCondition() == CondVal) {
-        FalseBO->setOperand(0, FalseBOSI->getFalseValue());
+        replaceOperand(*FalseBO, 0, FalseBOSI->getFalseValue());
         Worklist.push(FalseBO);
         return &SI;
       }
     }
     if (auto *FalseBOSI = dyn_cast<SelectInst>(FalseBO->getOperand(1))) {
       if (FalseBOSI->getCondition() == CondVal) {
-        FalseBO->setOperand(1, FalseBOSI->getFalseValue());
+        replaceOperand(*FalseBO, 1, FalseBOSI->getFalseValue());
         Worklist.push(FalseBO);
         return &SI;
       }
@@ -2814,8 +2812,8 @@ Instruction *InstCombiner::visitSelectInst(SelectInst &SI) {
     return BitCastSel;
 
   // Simplify selects that test the returned flag of cmpxchg instructions.
-  if (Instruction *Select = foldSelectCmpXchg(SI))
-    return Select;
+  if (Value *V = foldSelectCmpXchg(SI))
+    return replaceInstUsesWith(SI, V);
 
   if (Instruction *Select = foldSelectBinOpIdentity(SI, TLI, *this))
     return Select;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
index e697b99d6cf82..6edf11ae6655d 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
@@ -87,9 +87,10 @@ bool InstCombiner::SimplifyDemandedBits(Instruction *I, unsigned OpNo,
   Value *NewVal = SimplifyDemandedUseBits(U.get(), DemandedMask, Known,
                                           Depth, I);
   if (!NewVal) return false;
-  // Add the old operand back to the worklist.
-  Worklist.addValue(U.get());
-  U = NewVal;
+  if (Instruction* OpInst = dyn_cast<Instruction>(U))
+    salvageDebugInfoOrMarkUndef(*OpInst);
+    
+  replaceUse(U, NewVal);
   return true;
 }
 
@@ -1305,10 +1306,7 @@ Value *InstCombiner::SimplifyDemandedVectorElts(Value *V, APInt DemandedElts,
     auto *II = dyn_cast<IntrinsicInst>(Inst);
     Value *Op = II ? II->getArgOperand(OpNum) : Inst->getOperand(OpNum);
     if (Value *V = SimplifyDemandedVectorElts(Op, Demanded, Undef, Depth + 1)) {
-      if (II)
-        II->setArgOperand(OpNum, V);
-      else
-        Inst->setOperand(OpNum, V);
+      replaceOperand(*Inst, OpNum, V);
       MadeChange = true;
     }
   };
@@ -1389,6 +1387,24 @@ Value *InstCombiner::SimplifyDemandedVectorElts(Value *V, APInt DemandedElts,
            "Expected shuffle operands to have same type");
     unsigned OpWidth =
         Shuffle->getOperand(0)->getType()->getVectorNumElements();
+    // Handle trivial case of a splat. Only check the first element of LHS
+    // operand.
+    if (all_of(Shuffle->getShuffleMask(), [](int Elt) { return Elt == 0; }) &&
+        DemandedElts.isAllOnesValue()) {
+      if (!isa<UndefValue>(I->getOperand(1))) {
+        I->setOperand(1, UndefValue::get(I->getOperand(1)->getType()));
+        MadeChange = true;
+      }
+      APInt LeftDemanded(OpWidth, 1);
+      APInt LHSUndefElts(OpWidth, 0);
+      simplifyAndSetOp(I, 0, LeftDemanded, LHSUndefElts);
+      if (LHSUndefElts[0])
+        UndefElts = EltMask;
+      else
+        UndefElts.clearAllBits();
+      break;
+    }
+
     APInt LeftDemanded(OpWidth, 0), RightDemanded(OpWidth, 0);
     for (unsigned i = 0; i < VWidth; i++) {
       if (DemandedElts[i]) {
@@ -1502,15 +1518,14 @@ Value *InstCombiner::SimplifyDemandedVectorElts(Value *V, APInt DemandedElts,
     }
     if (NewUndefElts) {
       // Add additional discovered undefs.
-      SmallVector<Constant*, 16> Elts;
+      SmallVector<int, 16> Elts;
       for (unsigned i = 0; i < VWidth; ++i) {
         if (UndefElts[i])
-          Elts.push_back(UndefValue::get(Type::getInt32Ty(I->getContext())));
+          Elts.push_back(UndefMaskElem);
         else
-          Elts.push_back(ConstantInt::get(Type::getInt32Ty(I->getContext()),
-                                          Shuffle->getMaskValue(i)));
+          Elts.push_back(Shuffle->getMaskValue(i));
       }
-      I->setOperand(2, ConstantVector::get(Elts));
+      Shuffle->setShuffleMask(Elts);
       MadeChange = true;
     }
     break;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
index bc6696a2a325a..1e95f235b6533 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
@@ -345,10 +345,8 @@ Instruction *InstCombiner::visitExtractElementInst(ExtractElementInst &EI) {
         APInt DemandedElts(NumElts, 0);
         DemandedElts.setBit(IndexC->getZExtValue());
         if (Value *V =
-                SimplifyDemandedVectorElts(SrcVec, DemandedElts, UndefElts)) {
-          EI.setOperand(0, V);
-          return &EI;
-        }
+                SimplifyDemandedVectorElts(SrcVec, DemandedElts, UndefElts))
+          return replaceOperand(EI, 0, V);
       } else {
         // If the input vector has multiple uses, simplify it based on a union
         // of all elements used.
@@ -732,7 +730,7 @@ Instruction *InstCombiner::visitInsertValueInst(InsertValueInst &I) {
 }
 
 static bool isShuffleEquivalentToSelect(ShuffleVectorInst &Shuf) {
-  int MaskSize = Shuf.getMask()->getType()->getVectorNumElements();
+  int MaskSize = Shuf.getShuffleMask().size();
   int VecSize = Shuf.getOperand(0)->getType()->getVectorNumElements();
 
   // A vector select does not change the size of the operands.
@@ -844,13 +842,10 @@ static Instruction *foldInsEltIntoSplat(InsertElementInst &InsElt) {
   // inselt (shuf (inselt undef, X, 0), undef, <0,undef,0,undef>), X, 1
   //   --> shuf (inselt undef, X, 0), undef, <0,0,0,undef>
   unsigned NumMaskElts = Shuf->getType()->getVectorNumElements();
-  SmallVector<Constant *, 16> NewMaskVec(NumMaskElts);
-  Type *I32Ty = IntegerType::getInt32Ty(Shuf->getContext());
-  Constant *Zero = ConstantInt::getNullValue(I32Ty);
+  SmallVector<int, 16> NewMask(NumMaskElts);
   for (unsigned i = 0; i != NumMaskElts; ++i)
-    NewMaskVec[i] = i == IdxC ? Zero : Shuf->getMask()->getAggregateElement(i);
+    NewMask[i] = i == IdxC ? 0 : Shuf->getMaskValue(i);
 
-  Constant *NewMask = ConstantVector::get(NewMaskVec);
   return new ShuffleVectorInst(Op0, UndefValue::get(Op0->getType()), NewMask);
 }
 
@@ -880,26 +875,23 @@ static Instruction *foldInsEltIntoIdentityShuffle(InsertElementInst &InsElt) {
   // For example:
   // inselt (shuf X, IdMask), (extelt X, IdxC), IdxC --> shuf X, IdMask'
   unsigned NumMaskElts = Shuf->getType()->getVectorNumElements();
-  SmallVector<Constant *, 16> NewMaskVec(NumMaskElts);
-  Type *I32Ty = IntegerType::getInt32Ty(Shuf->getContext());
-  Constant *NewMaskEltC = ConstantInt::get(I32Ty, IdxC);
-  Constant *OldMask = Shuf->getMask();
+  SmallVector<int, 16> NewMask(NumMaskElts);
+  ArrayRef<int> OldMask = Shuf->getShuffleMask();
   for (unsigned i = 0; i != NumMaskElts; ++i) {
     if (i != IdxC) {
       // All mask elements besides the inserted element remain the same.
-      NewMaskVec[i] = OldMask->getAggregateElement(i);
-    } else if (OldMask->getAggregateElement(i) == NewMaskEltC) {
+      NewMask[i] = OldMask[i];
+    } else if (OldMask[i] == (int)IdxC) {
       // If the mask element was already set, there's nothing to do
       // (demanded elements analysis may unset it later).
       return nullptr;
     } else {
-      assert(isa<UndefValue>(OldMask->getAggregateElement(i)) &&
+      assert(OldMask[i] == UndefMaskElem &&
              "Unexpected shuffle mask element for identity shuffle");
-      NewMaskVec[i] = NewMaskEltC;
+      NewMask[i] = IdxC;
     }
   }
 
-  Constant *NewMask = ConstantVector::get(NewMaskVec);
   return new ShuffleVectorInst(X, Shuf->getOperand(1), NewMask);
 }
 
@@ -967,27 +959,25 @@ static Instruction *foldConstantInsEltIntoShuffle(InsertElementInst &InsElt) {
     // mask vector with the insertelt index plus the length of the vector
     // (because the constant vector operand of a shuffle is always the 2nd
     // operand).
-    Constant *Mask = Shuf->getMask();
-    unsigned NumElts = Mask->getType()->getVectorNumElements();
+    ArrayRef<int> Mask = Shuf->getShuffleMask();
+    unsigned NumElts = Mask.size();
     SmallVector<Constant *, 16> NewShufElts(NumElts);
-    SmallVector<Constant *, 16> NewMaskElts(NumElts);
+    SmallVector<int, 16> NewMaskElts(NumElts);
     for (unsigned I = 0; I != NumElts; ++I) {
       if (I == InsEltIndex) {
         NewShufElts[I] = InsEltScalar;
-        Type *Int32Ty = Type::getInt32Ty(Shuf->getContext());
-        NewMaskElts[I] = ConstantInt::get(Int32Ty, InsEltIndex + NumElts);
+        NewMaskElts[I] = InsEltIndex + NumElts;
       } else {
         // Copy over the existing values.
         NewShufElts[I] = ShufConstVec->getAggregateElement(I);
-        NewMaskElts[I] = Mask->getAggregateElement(I);
+        NewMaskElts[I] = Mask[I];
       }
     }
 
     // Create new operands for a shuffle that includes the constant of the
     // original insertelt. The old shuffle will be dead now.
     return new ShuffleVectorInst(Shuf->getOperand(0),
-                                 ConstantVector::get(NewShufElts),
-                                 ConstantVector::get(NewMaskElts));
+                                 ConstantVector::get(NewShufElts), NewMaskElts);
   } else if (auto *IEI = dyn_cast<InsertElementInst>(Inst)) {
     // Transform sequences of insertelements ops with constant data/indexes into
     // a single shuffle op.
@@ -1307,17 +1297,9 @@ static Value *evaluateInDifferentElementOrder(Value *V, ArrayRef<int> Mask) {
   if (isa<ConstantAggregateZero>(V))
     return ConstantAggregateZero::get(VectorType::get(EltTy, Mask.size()));
 
-  if (Constant *C = dyn_cast<Constant>(V)) {
-    SmallVector<Constant *, 16> MaskValues;
-    for (int i = 0, e = Mask.size(); i != e; ++i) {
-      if (Mask[i] == -1)
-        MaskValues.push_back(UndefValue::get(I32Ty));
-      else
-        MaskValues.push_back(ConstantInt::get(I32Ty, Mask[i]));
-    }
+  if (Constant *C = dyn_cast<Constant>(V))
     return ConstantExpr::getShuffleVector(C, UndefValue::get(C->getType()),
-                                          ConstantVector::get(MaskValues));
-  }
+                                          Mask);
 
   Instruction *I = cast<Instruction>(V);
   switch (I->getOpcode()) {
@@ -1406,7 +1388,7 @@ static Value *evaluateInDifferentElementOrder(Value *V, ArrayRef<int> Mask) {
 //   Shuffles to:  |EE|FF|GG|HH|
 //                 +--+--+--+--+
 static bool isShuffleExtractingFromLHS(ShuffleVectorInst &SVI,
-                                       SmallVector<int, 16> &Mask) {
+                                       ArrayRef<int> Mask) {
   unsigned LHSElems = SVI.getOperand(0)->getType()->getVectorNumElements();
   unsigned MaskElems = Mask.size();
   unsigned BegIdx = Mask.front();
@@ -1489,12 +1471,12 @@ static Instruction *foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf) {
   // Example: shuf (mul X, {-1,-2,-3,-4}), X, {0,5,6,3} --> mul X, {-1,1,1,-4}
   // Example: shuf X, (add X, {-1,-2,-3,-4}), {0,1,6,7} --> add X, {0,0,-3,-4}
   // The existing binop constant vector remains in the same operand position.
-  Constant *Mask = Shuf.getMask();
+  ArrayRef<int> Mask = Shuf.getShuffleMask();
   Constant *NewC = Op0IsBinop ? ConstantExpr::getShuffleVector(C, IdC, Mask) :
                                 ConstantExpr::getShuffleVector(IdC, C, Mask);
 
   bool MightCreatePoisonOrUB =
-      Mask->containsUndefElement() &&
+      is_contained(Mask, UndefMaskElem) &&
       (Instruction::isIntDivRem(BOpcode) || Instruction::isShift(BOpcode));
   if (MightCreatePoisonOrUB)
     NewC = getSafeVectorConstantForBinop(BOpcode, NewC, true);
@@ -1508,7 +1490,7 @@ static Instruction *foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf) {
   // An undef shuffle mask element may propagate as an undef constant element in
   // the new binop. That would produce poison where the original code might not.
   // If we already made a safe constant, then there's no danger.
-  if (Mask->containsUndefElement() && !MightCreatePoisonOrUB)
+  if (is_contained(Mask, UndefMaskElem) && !MightCreatePoisonOrUB)
     NewBO->dropPoisonGeneratingFlags();
   return NewBO;
 }
@@ -1520,14 +1502,14 @@ static Instruction *foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf) {
 static Instruction *canonicalizeInsertSplat(ShuffleVectorInst &Shuf,
                                             InstCombiner::BuilderTy &Builder) {
   Value *Op0 = Shuf.getOperand(0), *Op1 = Shuf.getOperand(1);
-  Constant *Mask = Shuf.getMask();
+  ArrayRef<int> Mask = Shuf.getShuffleMask();
   Value *X;
   uint64_t IndexC;
 
   // Match a shuffle that is a splat to a non-zero element.
   if (!match(Op0, m_OneUse(m_InsertElement(m_Undef(), m_Value(X),
                                            m_ConstantInt(IndexC)))) ||
-      !match(Op1, m_Undef()) || match(Mask, m_ZeroInt()) || IndexC == 0)
+      !match(Op1, m_Undef()) || match(Mask, m_ZeroMask()) || IndexC == 0)
     return nullptr;
 
   // Insert into element 0 of an undef vector.
@@ -1540,12 +1522,12 @@ static Instruction *canonicalizeInsertSplat(ShuffleVectorInst &Shuf,
   // shuf (inselt undef, X, 2), undef, <2,2,undef>
   //   --> shuf (inselt undef, X, 0), undef, <0,0,undef>
   unsigned NumMaskElts = Shuf.getType()->getVectorNumElements();
-  SmallVector<Constant *, 16> NewMask(NumMaskElts, Zero);
+  SmallVector<int, 16> NewMask(NumMaskElts, 0);
   for (unsigned i = 0; i != NumMaskElts; ++i)
-    if (isa<UndefValue>(Mask->getAggregateElement(i)))
-      NewMask[i] = Mask->getAggregateElement(i);
+    if (Mask[i] == UndefMaskElem)
+      NewMask[i] = Mask[i];
 
-  return new ShuffleVectorInst(NewIns, UndefVec, ConstantVector::get(NewMask));
+  return new ShuffleVectorInst(NewIns, UndefVec, NewMask);
 }
 
 /// Try to fold shuffles that are the equivalent of a vector select.
@@ -1614,14 +1596,14 @@ static Instruction *foldSelectShuffle(ShuffleVectorInst &Shuf,
   BinaryOperator::BinaryOps BOpc = Opc0;
 
   // Select the constant elements needed for the single binop.
-  Constant *Mask = Shuf.getMask();
+  ArrayRef<int> Mask = Shuf.getShuffleMask();
   Constant *NewC = ConstantExpr::getShuffleVector(C0, C1, Mask);
 
   // We are moving a binop after a shuffle. When a shuffle has an undefined
   // mask element, the result is undefined, but it is not poison or undefined
   // behavior. That is not necessarily true for div/rem/shift.
   bool MightCreatePoisonOrUB =
-      Mask->containsUndefElement() &&
+      is_contained(Mask, UndefMaskElem) &&
       (Instruction::isIntDivRem(BOpc) || Instruction::isShift(BOpc));
   if (MightCreatePoisonOrUB)
     NewC = getSafeVectorConstantForBinop(BOpc, NewC, ConstantsAreOp1);
@@ -1670,7 +1652,7 @@ static Instruction *foldSelectShuffle(ShuffleVectorInst &Shuf,
   NewBO->andIRFlags(B1);
   if (DropNSW)
     NewBO->setHasNoSignedWrap(false);
-  if (Mask->containsUndefElement() && !MightCreatePoisonOrUB)
+  if (is_contained(Mask, UndefMaskElem) && !MightCreatePoisonOrUB)
     NewBO->dropPoisonGeneratingFlags();
   return NewBO;
 }
@@ -1696,8 +1678,7 @@ static Instruction *narrowVectorSelect(ShuffleVectorInst &Shuf,
   // and have the same number of elements as this shuffle.
   unsigned NarrowNumElts = Shuf.getType()->getVectorNumElements();
   Value *NarrowCond;
-  if (!match(Cond, m_OneUse(m_ShuffleVector(m_Value(NarrowCond), m_Undef(),
-                                            m_Constant()))) ||
+  if (!match(Cond, m_OneUse(m_ShuffleVector(m_Value(NarrowCond), m_Undef()))) ||
       NarrowCond->getType()->getVectorNumElements() != NarrowNumElts ||
       !cast<ShuffleVectorInst>(Cond)->isIdentityWithPadding())
     return nullptr;
@@ -1705,8 +1686,8 @@ static Instruction *narrowVectorSelect(ShuffleVectorInst &Shuf,
   // shuf (sel (shuf NarrowCond, undef, WideMask), X, Y), undef, NarrowMask) -->
   // sel NarrowCond, (shuf X, undef, NarrowMask), (shuf Y, undef, NarrowMask)
   Value *Undef = UndefValue::get(X->getType());
-  Value *NarrowX = Builder.CreateShuffleVector(X, Undef, Shuf.getMask());
-  Value *NarrowY = Builder.CreateShuffleVector(Y, Undef, Shuf.getMask());
+  Value *NarrowX = Builder.CreateShuffleVector(X, Undef, Shuf.getShuffleMask());
+  Value *NarrowY = Builder.CreateShuffleVector(Y, Undef, Shuf.getShuffleMask());
   return SelectInst::Create(NarrowCond, NarrowX, NarrowY);
 }
 
@@ -1717,8 +1698,8 @@ static Instruction *foldIdentityExtractShuffle(ShuffleVectorInst &Shuf) {
     return nullptr;
 
   Value *X, *Y;
-  Constant *Mask;
-  if (!match(Op0, m_ShuffleVector(m_Value(X), m_Value(Y), m_Constant(Mask))))
+  ArrayRef<int> Mask;
+  if (!match(Op0, m_ShuffleVector(m_Value(X), m_Value(Y), m_Mask(Mask))))
     return nullptr;
 
   // Be conservative with shuffle transforms. If we can't kill the 1st shuffle,
@@ -1738,16 +1719,16 @@ static Instruction *foldIdentityExtractShuffle(ShuffleVectorInst &Shuf) {
   //   shuf (shuf X, Y, <C0, C1, C2, undef, C4>), undef, <0, undef, 2, 3> -->
   //   shuf X, Y, <C0, undef, C2, undef>
   unsigned NumElts = Shuf.getType()->getVectorNumElements();
-  SmallVector<Constant *, 16> NewMask(NumElts);
-  assert(NumElts < Mask->getType()->getVectorNumElements() &&
+  SmallVector<int, 16> NewMask(NumElts);
+  assert(NumElts < Mask.size() &&
          "Identity with extract must have less elements than its inputs");
 
   for (unsigned i = 0; i != NumElts; ++i) {
-    Constant *ExtractMaskElt = Shuf.getMask()->getAggregateElement(i);
-    Constant *MaskElt = Mask->getAggregateElement(i);
-    NewMask[i] = isa<UndefValue>(ExtractMaskElt) ? ExtractMaskElt : MaskElt;
+    int ExtractMaskElt = Shuf.getMaskValue(i);
+    int MaskElt = Mask[i];
+    NewMask[i] = ExtractMaskElt == UndefMaskElem ? ExtractMaskElt : MaskElt;
   }
-  return new ShuffleVectorInst(X, Y, ConstantVector::get(NewMask));
+  return new ShuffleVectorInst(X, Y, NewMask);
 }
 
 /// Try to replace a shuffle with an insertelement or try to replace a shuffle
@@ -1755,7 +1736,8 @@ static Instruction *foldIdentityExtractShuffle(ShuffleVectorInst &Shuf) {
 static Instruction *foldShuffleWithInsert(ShuffleVectorInst &Shuf,
                                           InstCombiner &IC) {
   Value *V0 = Shuf.getOperand(0), *V1 = Shuf.getOperand(1);
-  SmallVector<int, 16> Mask = Shuf.getShuffleMask();
+  SmallVector<int, 16> Mask;
+  Shuf.getShuffleMask(Mask);
 
   // The shuffle must not change vector sizes.
   // TODO: This restriction could be removed if the insert has only one use
@@ -1874,7 +1856,7 @@ static Instruction *foldIdentityPaddedShuffles(ShuffleVectorInst &Shuf) {
   assert(WideElts > NarrowElts && "Unexpected types for identity with padding");
 
   Type *I32Ty = IntegerType::getInt32Ty(Shuf.getContext());
-  SmallVector<int, 16> Mask = Shuf.getShuffleMask();
+  ArrayRef<int> Mask = Shuf.getShuffleMask();
   SmallVector<Constant *, 16> NewMask(Mask.size(), UndefValue::get(I32Ty));
   for (int i = 0, e = Mask.size(); i != e; ++i) {
     if (Mask[i] == -1)
@@ -1907,28 +1889,54 @@ static Instruction *foldIdentityPaddedShuffles(ShuffleVectorInst &Shuf) {
 Instruction *InstCombiner::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
   Value *LHS = SVI.getOperand(0);
   Value *RHS = SVI.getOperand(1);
-  if (auto *V = SimplifyShuffleVectorInst(
-          LHS, RHS, SVI.getMask(), SVI.getType(), SQ.getWithInstruction(&SVI)))
+  SimplifyQuery ShufQuery = SQ.getWithInstruction(&SVI);
+  if (auto *V = SimplifyShuffleVectorInst(LHS, RHS, SVI.getShuffleMask(),
+                                          SVI.getType(), ShufQuery))
     return replaceInstUsesWith(SVI, V);
 
   // shuffle x, x, mask --> shuffle x, undef, mask'
   unsigned VWidth = SVI.getType()->getVectorNumElements();
   unsigned LHSWidth = LHS->getType()->getVectorNumElements();
-  SmallVector<int, 16> Mask = SVI.getShuffleMask();
+  ArrayRef<int> Mask = SVI.getShuffleMask();
   Type *Int32Ty = Type::getInt32Ty(SVI.getContext());
+
+  // Peek through a bitcasted shuffle operand by scaling the mask. If the
+  // simulated shuffle can simplify, then this shuffle is unnecessary:
+  // shuf (bitcast X), undef, Mask --> bitcast X'
+  // TODO: This could be extended to allow length-changing shuffles and/or casts
+  //       to narrower elements. The transform might also be obsoleted if we
+  //       allowed canonicalization of bitcasted shuffles.
+  Value *X;
+  if (match(LHS, m_BitCast(m_Value(X))) && match(RHS, m_Undef()) &&
+      X->getType()->isVectorTy() && VWidth == LHSWidth &&
+      X->getType()->getVectorNumElements() >= VWidth) {
+    // Create the scaled mask constant.
+    Type *XType = X->getType();
+    unsigned XNumElts = XType->getVectorNumElements();
+    assert(XNumElts % VWidth == 0 && "Unexpected vector bitcast");
+    unsigned ScaleFactor = XNumElts / VWidth;
+    SmallVector<int, 16> ScaledMask;
+    scaleShuffleMask(ScaleFactor, Mask, ScaledMask);
+
+    // If the shuffled source vector simplifies, cast that value to this
+    // shuffle's type.
+    if (auto *V = SimplifyShuffleVectorInst(X, UndefValue::get(XType),
+                                            ScaledMask, XType, ShufQuery))
+      return BitCastInst::Create(Instruction::BitCast, V, SVI.getType());
+  }
+
   if (LHS == RHS) {
     assert(!isa<UndefValue>(RHS) && "Shuffle with 2 undef ops not simplified?");
     // Remap any references to RHS to use LHS.
-    SmallVector<Constant*, 16> Elts;
+    SmallVector<int, 16> Elts;
     for (unsigned i = 0; i != VWidth; ++i) {
       // Propagate undef elements or force mask to LHS.
       if (Mask[i] < 0)
-        Elts.push_back(UndefValue::get(Int32Ty));
+        Elts.push_back(UndefMaskElem);
       else
-        Elts.push_back(ConstantInt::get(Int32Ty, Mask[i] % LHSWidth));
+        Elts.push_back(Mask[i] % LHSWidth);
     }
-    return new ShuffleVectorInst(LHS, UndefValue::get(RHS->getType()),
-                                 ConstantVector::get(Elts));
+    return new ShuffleVectorInst(LHS, UndefValue::get(RHS->getType()), Elts);
   }
 
   // shuffle undef, x, mask --> shuffle x, undef, mask'
@@ -2153,8 +2161,8 @@ Instruction *InstCombiner::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
   if (newLHS == LHS && newRHS == RHS)
     return MadeChange ? &SVI : nullptr;
 
-  SmallVector<int, 16> LHSMask;
-  SmallVector<int, 16> RHSMask;
+  ArrayRef<int> LHSMask;
+  ArrayRef<int> RHSMask;
   if (newLHS != LHS)
     LHSMask = LHSShuffle->getShuffleMask();
   if (RHSShuffle && newRHS != RHS)
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index da43f510d32ea..c3890ed694212 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -264,7 +264,7 @@ static void ClearSubclassDataAfterReassociation(BinaryOperator &I) {
 /// cast to eliminate one of the associative operations:
 /// (op (cast (op X, C2)), C1) --> (cast (op X, op (C1, C2)))
 /// (op (cast (op X, C2)), C1) --> (op (cast X), op (C1, C2))
-static bool simplifyAssocCastAssoc(BinaryOperator *BinOp1) {
+static bool simplifyAssocCastAssoc(BinaryOperator *BinOp1, InstCombiner &IC) {
   auto *Cast = dyn_cast<CastInst>(BinOp1->getOperand(0));
   if (!Cast || !Cast->hasOneUse())
     return false;
@@ -297,8 +297,8 @@ static bool simplifyAssocCastAssoc(BinaryOperator *BinOp1) {
   Type *DestTy = C1->getType();
   Constant *CastC2 = ConstantExpr::getCast(CastOpcode, C2, DestTy);
   Constant *FoldedC = ConstantExpr::get(AssocOpcode, C1, CastC2);
-  Cast->setOperand(0, BinOp2->getOperand(0));
-  BinOp1->setOperand(1, FoldedC);
+  IC.replaceOperand(*Cast, 0, BinOp2->getOperand(0));
+  IC.replaceOperand(*BinOp1, 1, FoldedC);
   return true;
 }
 
@@ -393,7 +393,7 @@ bool InstCombiner::SimplifyAssociativeOrCommutative(BinaryOperator &I) {
     }
 
     if (I.isAssociative() && I.isCommutative()) {
-      if (simplifyAssocCastAssoc(&I)) {
+      if (simplifyAssocCastAssoc(&I, *this)) {
         Changed = true;
         ++NumReassoc;
         continue;
@@ -910,6 +910,18 @@ Value *InstCombiner::freelyNegateValue(Value *V) {
     return Builder.CreateSub(
         I->getOperand(1), I->getOperand(0), I->getName() + ".neg");
 
+  // Negation is equivalent to bitwise-not + 1:
+  // 0 - (A ^ C)  =>  ((A ^ C) ^ -1) + 1  =>  A ^ ~C + 1
+  case Instruction::Xor: {
+    Constant *C;
+    if (match(I->getOperand(1), m_Constant(C))) {
+      Value *Xor = Builder.CreateXor(I->getOperand(0), ConstantExpr::getNot(C));
+      return Builder.CreateAdd(Xor, ConstantInt::get(Xor->getType(), 1),
+                               I->getName() + ".neg");
+    }
+    return nullptr;
+  }
+
   // 0-(A sdiv C)  =>  A sdiv (0-C)  provided the negation doesn't overflow.
   case Instruction::SDiv: {
     Constant *C = dyn_cast<Constant>(I->getOperand(1));
@@ -1481,7 +1493,7 @@ Value *InstCombiner::Descale(Value *Val, APInt Scale, bool &NoSignedWrap) {
   assert(Parent.first->hasOneUse() && "Drilled down when more than one use!");
   assert(Op != Parent.first->getOperand(Parent.second) &&
          "Descaling was a no-op?");
-  Parent.first->setOperand(Parent.second, Op);
+  replaceOperand(*Parent.first, Parent.second, Op);
   Worklist.push(Parent.first);
 
   // Now work back up the expression correcting nsw flags.  The logic is based
@@ -1534,9 +1546,10 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) {
   // narrow binop on each pair of the source operands followed by concatenation
   // of the results.
   Value *L0, *L1, *R0, *R1;
-  Constant *Mask;
-  if (match(LHS, m_ShuffleVector(m_Value(L0), m_Value(L1), m_Constant(Mask))) &&
-      match(RHS, m_ShuffleVector(m_Value(R0), m_Value(R1), m_Specific(Mask))) &&
+  ArrayRef<int> Mask;
+  if (match(LHS, m_ShuffleVector(m_Value(L0), m_Value(L1), m_Mask(Mask))) &&
+      match(RHS,
+            m_ShuffleVector(m_Value(R0), m_Value(R1), m_SpecificMask(Mask))) &&
       LHS->hasOneUse() && RHS->hasOneUse() &&
       cast<ShuffleVectorInst>(LHS)->isConcat() &&
       cast<ShuffleVectorInst>(RHS)->isConcat()) {
@@ -1560,7 +1573,7 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) {
   if (!isSafeToSpeculativelyExecute(&Inst))
     return nullptr;
 
-  auto createBinOpShuffle = [&](Value *X, Value *Y, Constant *M) {
+  auto createBinOpShuffle = [&](Value *X, Value *Y, ArrayRef<int> M) {
     Value *XY = Builder.CreateBinOp(Opcode, X, Y);
     if (auto *BO = dyn_cast<BinaryOperator>(XY))
       BO->copyIRFlags(&Inst);
@@ -1570,8 +1583,9 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) {
   // If both arguments of the binary operation are shuffles that use the same
   // mask and shuffle within a single vector, move the shuffle after the binop.
   Value *V1, *V2;
-  if (match(LHS, m_ShuffleVector(m_Value(V1), m_Undef(), m_Constant(Mask))) &&
-      match(RHS, m_ShuffleVector(m_Value(V2), m_Undef(), m_Specific(Mask))) &&
+  if (match(LHS, m_ShuffleVector(m_Value(V1), m_Undef(), m_Mask(Mask))) &&
+      match(RHS,
+            m_ShuffleVector(m_Value(V2), m_Undef(), m_SpecificMask(Mask))) &&
       V1->getType() == V2->getType() &&
       (LHS->hasOneUse() || RHS->hasOneUse() || LHS == RHS)) {
     // Op(shuffle(V1, Mask), shuffle(V2, Mask)) -> shuffle(Op(V1, V2), Mask)
@@ -1581,17 +1595,19 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) {
   // If both arguments of a commutative binop are select-shuffles that use the
   // same mask with commuted operands, the shuffles are unnecessary.
   if (Inst.isCommutative() &&
-      match(LHS, m_ShuffleVector(m_Value(V1), m_Value(V2), m_Constant(Mask))) &&
+      match(LHS, m_ShuffleVector(m_Value(V1), m_Value(V2), m_Mask(Mask))) &&
       match(RHS, m_ShuffleVector(m_Specific(V2), m_Specific(V1),
-                                 m_Specific(Mask)))) {
+                                 m_SpecificMask(Mask)))) {
     auto *LShuf = cast<ShuffleVectorInst>(LHS);
     auto *RShuf = cast<ShuffleVectorInst>(RHS);
     // TODO: Allow shuffles that contain undefs in the mask?
     //       That is legal, but it reduces undef knowledge.
     // TODO: Allow arbitrary shuffles by shuffling after binop?
     //       That might be legal, but we have to deal with poison.
-    if (LShuf->isSelect() && !LShuf->getMask()->containsUndefElement() &&
-        RShuf->isSelect() && !RShuf->getMask()->containsUndefElement()) {
+    if (LShuf->isSelect() &&
+        !is_contained(LShuf->getShuffleMask(), UndefMaskElem) &&
+        RShuf->isSelect() &&
+        !is_contained(RShuf->getShuffleMask(), UndefMaskElem)) {
       // Example:
       // LHS = shuffle V1, V2, <0, 5, 6, 3>
       // RHS = shuffle V2, V1, <0, 5, 6, 3>
@@ -1608,9 +1624,9 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) {
   // other binops, so they can be folded. It may also enable demanded elements
   // transforms.
   Constant *C;
-  if (match(&Inst, m_c_BinOp(
-          m_OneUse(m_ShuffleVector(m_Value(V1), m_Undef(), m_Constant(Mask))),
-          m_Constant(C))) &&
+  if (match(&Inst, m_c_BinOp(m_OneUse(m_ShuffleVector(m_Value(V1), m_Undef(),
+                                                      m_Mask(Mask))),
+                             m_Constant(C))) &&
       V1->getType()->getVectorNumElements() <= NumElts) {
     assert(Inst.getType()->getScalarType() == V1->getType()->getScalarType() &&
            "Shuffle should not change scalar type");
@@ -1621,8 +1637,7 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) {
     // reorder is not possible. A 1-to-1 mapping is not required. Example:
     // ShMask = <1,1,2,2> and C = <5,5,6,6> --> NewC = <undef,5,6,undef>
     bool ConstOp1 = isa<Constant>(RHS);
-    SmallVector<int, 16> ShMask;
-    ShuffleVectorInst::getShuffleMask(Mask, ShMask);
+    ArrayRef<int> ShMask = Mask;
     unsigned SrcVecNumElts = V1->getType()->getVectorNumElements();
     UndefValue *UndefScalar = UndefValue::get(C->getType()->getScalarType());
     SmallVector<Constant *, 16> NewVecC(SrcVecNumElts, UndefScalar);
@@ -1687,12 +1702,12 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) {
       std::swap(LHS, RHS);
 
     Value *X;
-    Constant *MaskC;
-    const APInt *SplatIndex;
+    ArrayRef<int> MaskC;
+    int SplatIndex;
     BinaryOperator *BO;
     if (!match(LHS, m_OneUse(m_ShuffleVector(m_Value(X), m_Undef(),
-                                             m_Constant(MaskC)))) ||
-        !match(MaskC, m_APIntAllowUndef(SplatIndex)) ||
+                                             m_Mask(MaskC)))) ||
+        !match(MaskC, m_SplatOrUndefMask(SplatIndex)) ||
         X->getType() != Inst.getType() || !match(RHS, m_OneUse(m_BinOp(BO))) ||
         BO->getOpcode() != Opcode)
       return nullptr;
@@ -1701,10 +1716,10 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) {
     //        moving 'Y' before the splat shuffle, we are implicitly assuming
     //        that it is not undef/poison at the splat index.
     Value *Y, *OtherOp;
-    if (isSplatValue(BO->getOperand(0), SplatIndex->getZExtValue())) {
+    if (isSplatValue(BO->getOperand(0), SplatIndex)) {
       Y = BO->getOperand(0);
       OtherOp = BO->getOperand(1);
-    } else if (isSplatValue(BO->getOperand(1), SplatIndex->getZExtValue())) {
+    } else if (isSplatValue(BO->getOperand(1), SplatIndex)) {
       Y = BO->getOperand(1);
       OtherOp = BO->getOperand(0);
     } else {
@@ -1716,7 +1731,7 @@ Instruction *InstCombiner::foldVectorBinop(BinaryOperator &Inst) {
     // bo (splat X), (bo Y, OtherOp) --> bo (splat (bo X, Y)), OtherOp
     Value *NewBO = Builder.CreateBinOp(Opcode, X, Y);
     UndefValue *Undef = UndefValue::get(Inst.getType());
-    Constant *NewMask = ConstantInt::get(MaskC->getType(), *SplatIndex);
+    SmallVector<int, 8> NewMask(MaskC.size(), SplatIndex);
     Value *NewSplat = Builder.CreateShuffleVector(NewBO, Undef, NewMask);
     Instruction *R = BinaryOperator::Create(Opcode, NewSplat, OtherOp);
 
@@ -1975,8 +1990,6 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
     if (DI == -1) {
       // All the GEPs feeding the PHI are identical. Clone one down into our
       // BB so that it can be merged with the current GEP.
-      GEP.getParent()->getInstList().insert(
-          GEP.getParent()->getFirstInsertionPt(), NewGEP);
     } else {
       // All the GEPs feeding the PHI differ at a single offset. Clone a GEP
       // into the current block so it can be merged, and create a new PHI to
@@ -1994,12 +2007,11 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
                            PN->getIncomingBlock(I));
 
       NewGEP->setOperand(DI, NewPN);
-      GEP.getParent()->getInstList().insert(
-          GEP.getParent()->getFirstInsertionPt(), NewGEP);
-      NewGEP->setOperand(DI, NewPN);
     }
 
-    GEP.setOperand(0, NewGEP);
+    GEP.getParent()->getInstList().insert(
+        GEP.getParent()->getFirstInsertionPt(), NewGEP);
+    replaceOperand(GEP, 0, NewGEP);
     PtrOp = NewGEP;
   }
 
@@ -2099,8 +2111,8 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
       // Update the GEP in place if possible.
       if (Src->getNumOperands() == 2) {
         GEP.setIsInBounds(isMergedGEPInBounds(*Src, *cast<GEPOperator>(&GEP)));
-        GEP.setOperand(0, Src->getOperand(0));
-        GEP.setOperand(1, Sum);
+        replaceOperand(GEP, 0, Src->getOperand(0));
+        replaceOperand(GEP, 1, Sum);
         return &GEP;
       }
       Indices.append(Src->op_begin()+1, Src->op_end()-1);
@@ -2218,9 +2230,8 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
             // array.  Because the array type is never stepped over (there
             // is a leading zero) we can fold the cast into this GEP.
             if (StrippedPtrTy->getAddressSpace() == GEP.getAddressSpace()) {
-              GEP.setOperand(0, StrippedPtr);
               GEP.setSourceElementType(XATy);
-              return &GEP;
+              return replaceOperand(GEP, 0, StrippedPtr);
             }
             // Cannot replace the base pointer directly because StrippedPtr's
             // address space is different. Instead, create a new GEP followed by
@@ -3352,7 +3363,7 @@ Instruction *InstCombiner::visitFreeze(FreezeInst &I) {
 /// instruction past all of the instructions between it and the end of its
 /// block.
 static bool TryToSinkInstruction(Instruction *I, BasicBlock *DestBlock) {
-  assert(I->hasOneUse() && "Invariants didn't hold!");
+  assert(I->getSingleUndroppableUse() && "Invariants didn't hold!");
   BasicBlock *SrcBlock = I->getParent();
 
   // Cannot move control-flow-involving, volatile loads, vaarg, etc.
@@ -3385,6 +3396,15 @@ static bool TryToSinkInstruction(Instruction *I, BasicBlock *DestBlock) {
       if (Scan->mayWriteToMemory())
         return false;
   }
+
+  I->dropDroppableUses([DestBlock](const Use *U) {
+    if (auto *I = dyn_cast<Instruction>(U->getUser()))
+      return I->getParent() != DestBlock;
+    return true;
+  });
+  /// FIXME: We could remove droppable uses that are not dominated by
+  /// the new position.
+
   BasicBlock::iterator InsertPos = DestBlock->getFirstInsertionPt();
   I->moveBefore(&*InsertPos);
   ++NumSunkInst;
@@ -3488,44 +3508,46 @@ bool InstCombiner::run() {
     }
 
     // See if we can trivially sink this instruction to a successor basic block.
-    if (EnableCodeSinking && I->hasOneUse()) {
-      BasicBlock *BB = I->getParent();
-      Instruction *UserInst = cast<Instruction>(*I->user_begin());
-      BasicBlock *UserParent;
-
-      // Get the block the use occurs in.
-      if (PHINode *PN = dyn_cast<PHINode>(UserInst))
-        UserParent = PN->getIncomingBlock(*I->use_begin());
-      else
-        UserParent = UserInst->getParent();
-
-      if (UserParent != BB) {
-        bool UserIsSuccessor = false;
-        // See if the user is one of our successors.
-        for (succ_iterator SI = succ_begin(BB), E = succ_end(BB); SI != E; ++SI)
-          if (*SI == UserParent) {
-            UserIsSuccessor = true;
-            break;
-          }
+    if (EnableCodeSinking)
+      if (Use *SingleUse = I->getSingleUndroppableUse()) {
+        BasicBlock *BB = I->getParent();
+        Instruction *UserInst = cast<Instruction>(SingleUse->getUser());
+        BasicBlock *UserParent;
+
+        // Get the block the use occurs in.
+        if (PHINode *PN = dyn_cast<PHINode>(UserInst))
+          UserParent = PN->getIncomingBlock(*SingleUse);
+        else
+          UserParent = UserInst->getParent();
+
+        if (UserParent != BB) {
+          bool UserIsSuccessor = false;
+          // See if the user is one of our successors.
+          for (succ_iterator SI = succ_begin(BB), E = succ_end(BB); SI != E;
+               ++SI)
+            if (*SI == UserParent) {
+              UserIsSuccessor = true;
+              break;
+            }
 
-        // If the user is one of our immediate successors, and if that successor
-        // only has us as a predecessors (we'd have to split the critical edge
-        // otherwise), we can keep going.
-        if (UserIsSuccessor && UserParent->getUniquePredecessor()) {
-          // Okay, the CFG is simple enough, try to sink this instruction.
-          if (TryToSinkInstruction(I, UserParent)) {
-            LLVM_DEBUG(dbgs() << "IC: Sink: " << *I << '\n');
-            MadeIRChange = true;
-            // We'll add uses of the sunk instruction below, but since sinking
-            // can expose opportunities for it's *operands* add them to the
-            // worklist
-            for (Use &U : I->operands())
-              if (Instruction *OpI = dyn_cast<Instruction>(U.get()))
-                Worklist.push(OpI);
+          // If the user is one of our immediate successors, and if that
+          // successor only has us as a predecessors (we'd have to split the
+          // critical edge otherwise), we can keep going.
+          if (UserIsSuccessor && UserParent->getUniquePredecessor()) {
+            // Okay, the CFG is simple enough, try to sink this instruction.
+            if (TryToSinkInstruction(I, UserParent)) {
+              LLVM_DEBUG(dbgs() << "IC: Sink: " << *I << '\n');
+              MadeIRChange = true;
+              // We'll add uses of the sunk instruction below, but since sinking
+              // can expose opportunities for it's *operands* add them to the
+              // worklist
+              for (Use &U : I->operands())
+                if (Instruction *OpI = dyn_cast<Instruction>(U.get()))
+                  Worklist.push(OpI);
+            }
           }
         }
       }
-    }
 
     // Now that we have an instruction, try combining it to simplify it.
     Builder.SetInsertPoint(I);
@@ -3589,27 +3611,27 @@ bool InstCombiner::run() {
   return MadeIRChange;
 }
 
-/// Walk the function in depth-first order, adding all reachable code to the
-/// worklist.
+/// Populate the IC worklist from a function, by walking it in depth-first
+/// order and adding all reachable code to the worklist.
 ///
 /// This has a couple of tricks to make the code faster and more powerful.  In
 /// particular, we constant fold and DCE instructions as we go, to avoid adding
 /// them to the worklist (this significantly speeds up instcombine on code where
 /// many instructions are dead or constant).  Additionally, if we find a branch
 /// whose condition is a known constant, we only visit the reachable successors.
-static bool AddReachableCodeToWorklist(BasicBlock *BB, const DataLayout &DL,
-                                       SmallPtrSetImpl<BasicBlock *> &Visited,
-                                       InstCombineWorklist &ICWorklist,
-                                       const TargetLibraryInfo *TLI) {
+static bool prepareICWorklistFromFunction(Function &F, const DataLayout &DL,
+                                          const TargetLibraryInfo *TLI,
+                                          InstCombineWorklist &ICWorklist) {
   bool MadeIRChange = false;
+  SmallPtrSet<BasicBlock *, 32> Visited;
   SmallVector<BasicBlock*, 256> Worklist;
-  Worklist.push_back(BB);
+  Worklist.push_back(&F.front());
 
   SmallVector<Instruction*, 128> InstrsForInstCombineWorklist;
   DenseMap<Constant *, Constant *> FoldedConstants;
 
   do {
-    BB = Worklist.pop_back_val();
+    BasicBlock *BB = Worklist.pop_back_val();
 
     // We have now visited this block!  If we've already been here, ignore it.
     if (!Visited.insert(BB).second)
@@ -3678,6 +3700,18 @@ static bool AddReachableCodeToWorklist(BasicBlock *BB, const DataLayout &DL,
       Worklist.push_back(SuccBB);
   } while (!Worklist.empty());
 
+  // Remove instructions inside unreachable blocks. This prevents the
+  // instcombine code from having to deal with some bad special cases, and
+  // reduces use counts of instructions.
+  for (BasicBlock &BB : F) {
+    if (Visited.count(&BB))
+      continue;
+
+    unsigned NumDeadInstInBB = removeAllNonTerminatorAndEHPadInstructions(&BB);
+    MadeIRChange |= NumDeadInstInBB > 0;
+    NumDeadInst += NumDeadInstInBB;
+  }
+
   // Once we've found all of the instructions to add to instcombine's worklist,
   // add them in reverse order.  This way instcombine will visit from the top
   // of the function down.  This jives well with the way that it adds all uses
@@ -3702,38 +3736,6 @@ static bool AddReachableCodeToWorklist(BasicBlock *BB, const DataLayout &DL,
   return MadeIRChange;
 }
 
-/// Populate the IC worklist from a function, and prune any dead basic
-/// blocks discovered in the process.
-///
-/// This also does basic constant propagation and other forward fixing to make
-/// the combiner itself run much faster.
-static bool prepareICWorklistFromFunction(Function &F, const DataLayout &DL,
-                                          TargetLibraryInfo *TLI,
-                                          InstCombineWorklist &ICWorklist) {
-  bool MadeIRChange = false;
-
-  // Do a depth-first traversal of the function, populate the worklist with
-  // the reachable instructions.  Ignore blocks that are not reachable.  Keep
-  // track of which blocks we visit.
-  SmallPtrSet<BasicBlock *, 32> Visited;
-  MadeIRChange |=
-      AddReachableCodeToWorklist(&F.front(), DL, Visited, ICWorklist, TLI);
-
-  // Do a quick scan over the function.  If we find any blocks that are
-  // unreachable, remove any instructions inside of them.  This prevents
-  // the instcombine code from having to deal with some bad special cases.
-  for (BasicBlock &BB : F) {
-    if (Visited.count(&BB))
-      continue;
-
-    unsigned NumDeadInstInBB = removeAllNonTerminatorAndEHPadInstructions(&BB);
-    MadeIRChange |= NumDeadInstInBB > 0;
-    NumDeadInst += NumDeadInstInBB;
-  }
-
-  return MadeIRChange;
-}
-
 static bool combineInstructionsOverFunction(
     Function &F, InstCombineWorklist &Worklist, AliasAnalysis *AA,
     AssumptionCache &AC, TargetLibraryInfo &TLI, DominatorTree &DT,
diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index dd892921b62b6..ea6156f4c56d8 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -1867,10 +1867,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
   }
 
   void visitShuffleVectorInst(ShuffleVectorInst &I) {
-    insertShadowCheck(I.getOperand(2), &I);
     IRBuilder<> IRB(&I);
     setShadow(&I, IRB.CreateShuffleVector(getShadow(&I, 0), getShadow(&I, 1),
-              I.getOperand(2), "_msprop"));
+                                          I.getShuffleMask(), "_msprop"));
     setOriginForNaryOp(I);
   }
 
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index b5f0a366d7f7c..f1f5da8a10f16 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -1795,7 +1795,7 @@ void setIrrLoopHeaderMetadata(Module *M, Instruction *TI, uint64_t Count) {
 
 template <> struct GraphTraits<PGOUseFunc *> {
   using NodeRef = const BasicBlock *;
-  using ChildIteratorType = succ_const_iterator;
+  using ChildIteratorType = const_succ_iterator;
   using nodes_iterator = pointer_iterator<Function::const_iterator>;
 
   static NodeRef getEntryNode(const PGOUseFunc *G) {
diff --git a/llvm/lib/Transforms/Scalar/AlignmentFromAssumptions.cpp b/llvm/lib/Transforms/Scalar/AlignmentFromAssumptions.cpp
index 06deaf3c4f9a9..18715d8e7fd66 100644
--- a/llvm/lib/Transforms/Scalar/AlignmentFromAssumptions.cpp
+++ b/llvm/lib/Transforms/Scalar/AlignmentFromAssumptions.cpp
@@ -127,6 +127,11 @@ static unsigned getNewAlignment(const SCEV *AASCEV, const SCEV *AlignSCEV,
                                 const SCEV *OffSCEV, Value *Ptr,
                                 ScalarEvolution *SE) {
   const SCEV *PtrSCEV = SE->getSCEV(Ptr);
+  // On a platform with 32-bit allocas, but 64-bit flat/global pointer sizes
+  // (*cough* AMDGPU), the effective SCEV type of AASCEV and PtrSCEV
+  // may disagree. Trunc/extend so they agree.
+  PtrSCEV = SE->getTruncateOrZeroExtend(
+      PtrSCEV, SE->getEffectiveSCEVType(AASCEV->getType()));
   const SCEV *DiffSCEV = SE->getMinusSCEV(PtrSCEV, AASCEV);
 
   // On 32-bit platforms, DiffSCEV might now have type i32 -- we've always
diff --git a/llvm/lib/Transforms/Scalar/CallSiteSplitting.cpp b/llvm/lib/Transforms/Scalar/CallSiteSplitting.cpp
index e34c011b1c871..286e237de1ead 100644
--- a/llvm/lib/Transforms/Scalar/CallSiteSplitting.cpp
+++ b/llvm/lib/Transforms/Scalar/CallSiteSplitting.cpp
@@ -424,22 +424,20 @@ static bool isPredicatedOnPHI(CallSite CS) {
   if (Instr != Parent->getFirstNonPHIOrDbg())
     return false;
 
-  for (auto &BI : *Parent) {
-    if (PHINode *PN = dyn_cast<PHINode>(&BI)) {
-      for (auto &I : CS.args())
-        if (&*I == PN) {
-          assert(PN->getNumIncomingValues() == 2 &&
-                 "Unexpected number of incoming values");
-          if (PN->getIncomingBlock(0) == PN->getIncomingBlock(1))
-            return false;
-          if (PN->getIncomingValue(0) == PN->getIncomingValue(1))
-            continue;
-          if (isa<Constant>(PN->getIncomingValue(0)) &&
-              isa<Constant>(PN->getIncomingValue(1)))
-            return true;
-        }
+  for (auto &PN : Parent->phis()) {
+    for (auto &Arg : CS.args()) {
+      if (&*Arg != &PN)
+        continue;
+      assert(PN.getNumIncomingValues() == 2 &&
+             "Unexpected number of incoming values");
+      if (PN.getIncomingBlock(0) == PN.getIncomingBlock(1))
+        return false;
+      if (PN.getIncomingValue(0) == PN.getIncomingValue(1))
+        continue;
+      if (isa<Constant>(PN.getIncomingValue(0)) &&
+          isa<Constant>(PN.getIncomingValue(1)))
+        return true;
     }
-    break;
   }
   return false;
 }
diff --git a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
index 4d9883dd662ec..4c40155c15149 100644
--- a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
+++ b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
@@ -125,7 +125,7 @@ Pass *llvm::createCorrelatedValuePropagationPass() {
 
 static bool processSelect(SelectInst *S, LazyValueInfo *LVI) {
   if (S->getType()->isVectorTy()) return false;
-  if (isa<Constant>(S->getOperand(0))) return false;
+  if (isa<Constant>(S->getCondition())) return false;
 
   Constant *C = LVI->getConstant(S->getCondition(), S->getParent(), S);
   if (!C) return false;
@@ -133,11 +133,7 @@ static bool processSelect(SelectInst *S, LazyValueInfo *LVI) {
   ConstantInt *CI = dyn_cast<ConstantInt>(C);
   if (!CI) return false;
 
-  Value *ReplaceWith = S->getTrueValue();
-  Value *Other = S->getFalseValue();
-  if (!CI->isOne()) std::swap(ReplaceWith, Other);
-  if (ReplaceWith == S) ReplaceWith = UndefValue::get(S->getType());
-
+  Value *ReplaceWith = CI->isOne() ? S->getTrueValue() : S->getFalseValue();
   S->replaceAllUsesWith(ReplaceWith);
   S->eraseFromParent();
 
@@ -794,7 +790,10 @@ static bool processAnd(BinaryOperator *BinOp, LazyValueInfo *LVI) {
   if (!RHS || !RHS->getValue().isMask())
     return false;
 
-  ConstantRange LRange = LVI->getConstantRange(LHS, BB, BinOp);
+  // We can only replace the AND with LHS based on range info if the range does
+  // not include undef.
+  ConstantRange LRange =
+      LVI->getConstantRange(LHS, BB, BinOp, /*UndefAllowed=*/false);
   if (!LRange.getUnsignedMax().ule(RHS->getValue()))
     return false;
 
diff --git a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
index eb6fcae0a5481..14f07b309b44c 100644
--- a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
@@ -143,7 +143,7 @@ deleteDeadInstruction(Instruction *I, BasicBlock::iterator *BBI,
     ++NumFastOther;
 
     // Try to preserve debug information attached to the dead instruction.
-    salvageDebugInfo(*DeadInst);
+    salvageDebugInfoOrMarkUndef(*DeadInst);
 
     // This instruction is dead, zap it, in stages.  Start by removing it from
     // MemDep, which needs to know the operands and needs it to be in the
diff --git a/llvm/lib/Transforms/Scalar/EarlyCSE.cpp b/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
index 39c86c7fc0b63..fb9db5657db29 100644
--- a/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
+++ b/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
@@ -54,6 +54,7 @@
 #include "llvm/Support/RecyclingAllocator.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
 #include "llvm/Transforms/Utils/GuardUtils.h"
 #include "llvm/Transforms/Utils/Local.h"
 #include <cassert>
@@ -947,6 +948,7 @@ bool EarlyCSE::processNode(DomTreeNode *Node) {
         continue;
       }
 
+      salvageKnowledge(&Inst);
       salvageDebugInfoOrMarkUndef(Inst);
       removeMSSA(Inst);
       Inst.eraseFromParent();
@@ -1013,6 +1015,7 @@ bool EarlyCSE::processNode(DomTreeNode *Node) {
                 cast<ConstantInt>(KnownCond)->isOne()) {
               LLVM_DEBUG(dbgs()
                          << "EarlyCSE removing guard: " << Inst << '\n');
+              salvageKnowledge(&Inst);
               removeMSSA(Inst);
               Inst.eraseFromParent();
               Changed = true;
@@ -1048,6 +1051,7 @@ bool EarlyCSE::processNode(DomTreeNode *Node) {
           Changed = true;
         }
         if (isInstructionTriviallyDead(&Inst, &TLI)) {
+          salvageKnowledge(&Inst);
           removeMSSA(Inst);
           Inst.eraseFromParent();
           Changed = true;
@@ -1073,6 +1077,7 @@ bool EarlyCSE::processNode(DomTreeNode *Node) {
         if (auto *I = dyn_cast<Instruction>(V))
           I->andIRFlags(&Inst);
         Inst.replaceAllUsesWith(V);
+        salvageKnowledge(&Inst);
         removeMSSA(Inst);
         Inst.eraseFromParent();
         Changed = true;
@@ -1133,6 +1138,7 @@ bool EarlyCSE::processNode(DomTreeNode *Node) {
           }
           if (!Inst.use_empty())
             Inst.replaceAllUsesWith(Op);
+          salvageKnowledge(&Inst);
           removeMSSA(Inst);
           Inst.eraseFromParent();
           Changed = true;
@@ -1176,6 +1182,7 @@ bool EarlyCSE::processNode(DomTreeNode *Node) {
         }
         if (!Inst.use_empty())
           Inst.replaceAllUsesWith(InVal.first);
+        salvageKnowledge(&Inst);
         removeMSSA(Inst);
         Inst.eraseFromParent();
         Changed = true;
@@ -1228,6 +1235,7 @@ bool EarlyCSE::processNode(DomTreeNode *Node) {
           LLVM_DEBUG(dbgs() << "Skipping due to debug counter\n");
           continue;
         }
+        salvageKnowledge(&Inst);
         removeMSSA(Inst);
         Inst.eraseFromParent();
         Changed = true;
@@ -1263,6 +1271,7 @@ bool EarlyCSE::processNode(DomTreeNode *Node) {
             if (!DebugCounter::shouldExecute(CSECounter)) {
               LLVM_DEBUG(dbgs() << "Skipping due to debug counter\n");
             } else {
+              salvageKnowledge(&Inst);
               removeMSSA(*LastStore);
               LastStore->eraseFromParent();
               Changed = true;
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index 6b9792aa77b08..2a6ecfcd957a5 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -289,7 +289,7 @@ GVN::Expression GVN::ValueTable::createExpr(Instruction *I) {
     e.commutative = true;
   }
 
-  if (CmpInst *C = dyn_cast<CmpInst>(I)) {
+  if (auto *C = dyn_cast<CmpInst>(I)) {
     // Sort the operand value numbers so x<y and y>x get the same value number.
     CmpInst::Predicate Predicate = C->getPredicate();
     if (e.varargs[0] > e.varargs[1]) {
@@ -298,10 +298,11 @@ GVN::Expression GVN::ValueTable::createExpr(Instruction *I) {
     }
     e.opcode = (C->getOpcode() << 8) | Predicate;
     e.commutative = true;
-  } else if (InsertValueInst *E = dyn_cast<InsertValueInst>(I)) {
-    for (InsertValueInst::idx_iterator II = E->idx_begin(), IE = E->idx_end();
-         II != IE; ++II)
-      e.varargs.push_back(*II);
+  } else if (auto *E = dyn_cast<InsertValueInst>(I)) {
+    e.varargs.append(E->idx_begin(), E->idx_end());
+  } else if (auto *SVI = dyn_cast<ShuffleVectorInst>(I)) {
+    ArrayRef<int> ShuffleMask = SVI->getShuffleMask();
+    e.varargs.append(ShuffleMask.begin(), ShuffleMask.end());
   }
 
   return e;
@@ -946,6 +947,7 @@ bool GVN::AnalyzeLoadAvailability(LoadInst *LI, MemDepResult DepInfo,
 
   // Loading the allocation -> undef.
   if (isa<AllocaInst>(DepInst) || isMallocLikeFn(DepInst, TLI) ||
+      isAlignedAllocLikeFn(DepInst, TLI) ||
       // Loading immediately after lifetime begin -> undef.
       isLifetimeStart(DepInst)) {
     Res = AvailableValue::get(UndefValue::get(LI->getType()));
@@ -1450,7 +1452,7 @@ static bool impliesEquivalanceIfFalse(CmpInst* Cmp) {
       Value *LHS = Cmp->getOperand(0);
       Value *RHS = Cmp->getOperand(1);
       // If we can prove either side non-zero, then equality must imply
-      // equivalence. 
+      // equivalence.
       // FIXME: We should do this optimization if 'no signed zeros' is
       // applicable via an instruction-level fast-math-flag or some other
       // indicator that relaxed FP semantics are being used.
@@ -1515,10 +1517,10 @@ bool GVN::processAssumeIntrinsic(IntrinsicInst *IntrinsicI) {
   // If we find an equality fact, canonicalize all dominated uses in this block
   // to one of the two values.  We heuristically choice the "oldest" of the
   // two where age is determined by value number. (Note that propagateEquality
-  // above handles the cross block case.) 
-  // 
+  // above handles the cross block case.)
+  //
   // Key case to cover are:
-  // 1) 
+  // 1)
   // %cmp = fcmp oeq float 3.000000e+00, %0 ; const on lhs could happen
   // call void @llvm.assume(i1 %cmp)
   // ret float %0 ; will change it to ret float 3.000000e+00
@@ -1559,7 +1561,7 @@ bool GVN::processAssumeIntrinsic(IntrinsicInst *IntrinsicI) {
                  << *CmpLHS << " with "
                  << *CmpRHS << " in block "
                  << IntrinsicI->getParent()->getName() << "\n");
-      
+
 
       // Setup the replacement map - this handles uses within the same block
       if (hasUsersIn(CmpLHS, IntrinsicI->getParent()))
@@ -1732,7 +1734,8 @@ uint32_t GVN::ValueTable::phiTranslateImpl(const BasicBlock *Pred,
     // instead of value numbers. Those index numbers should not be
     // translated.
     if ((i > 1 && Exp.opcode == Instruction::InsertValue) ||
-        (i > 0 && Exp.opcode == Instruction::ExtractValue))
+        (i > 0 && Exp.opcode == Instruction::ExtractValue) ||
+        (i > 1 && Exp.opcode == Instruction::ShuffleVector))
       continue;
     Exp.varargs[i] = phiTranslate(Pred, PhiBlock, Exp.varargs[i], Gvn);
   }
@@ -1824,7 +1827,7 @@ void GVN::assignBlockRPONumber(Function &F) {
 bool GVN::replaceOperandsForInBlockEquality(Instruction *Instr) const {
   bool Changed = false;
   for (unsigned OpNum = 0; OpNum < Instr->getNumOperands(); ++OpNum) {
-    Value *Operand = Instr->getOperand(OpNum); 
+    Value *Operand = Instr->getOperand(OpNum);
     auto it = ReplaceOperandsWithMap.find(Operand);
     if (it != ReplaceOperandsWithMap.end()) {
       LLVM_DEBUG(dbgs() << "GVN replacing: " << *Operand << " with "
@@ -1944,7 +1947,7 @@ bool GVN::propagateEquality(Value *LHS, Value *RHS, const BasicBlockEdge &Root,
 
       // If "A == B" is known true, or "A != B" is known false, then replace
       // A with B everywhere in the scope.  For floating point operations, we
-      // have to be careful since equality does not always imply equivalance.  
+      // have to be careful since equality does not always imply equivalance.
       if ((isKnownTrue && impliesEquivalanceIfTrue(Cmp)) ||
           (isKnownFalse && impliesEquivalanceIfFalse(Cmp)))
         Worklist.push_back(std::make_pair(Op0, Op1));
diff --git a/llvm/lib/Transforms/Scalar/GVNSink.cpp b/llvm/lib/Transforms/Scalar/GVNSink.cpp
index 6b9a88d04eda9..5fee60fc6bf9c 100644
--- a/llvm/lib/Transforms/Scalar/GVNSink.cpp
+++ b/llvm/lib/Transforms/Scalar/GVNSink.cpp
@@ -350,6 +350,7 @@ using ModelledPHISet = DenseSet<ModelledPHI, DenseMapInfo<ModelledPHI>>;
 class InstructionUseExpr : public GVNExpression::BasicExpression {
   unsigned MemoryUseOrder = -1;
   bool Volatile = false;
+  ArrayRef<int> ShuffleMask;
 
 public:
   InstructionUseExpr(Instruction *I, ArrayRecycler<Value *> &R,
@@ -359,6 +360,9 @@ class InstructionUseExpr : public GVNExpression::BasicExpression {
     setOpcode(I->getOpcode());
     setType(I->getType());
 
+    if (ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(I))
+      ShuffleMask = SVI->getShuffleMask().copy(A);
+
     for (auto &U : I->uses())
       op_push_back(U.getUser());
     llvm::sort(op_begin(), op_end());
@@ -369,12 +373,12 @@ class InstructionUseExpr : public GVNExpression::BasicExpression {
 
   hash_code getHashValue() const override {
     return hash_combine(GVNExpression::BasicExpression::getHashValue(),
-                        MemoryUseOrder, Volatile);
+                        MemoryUseOrder, Volatile, ShuffleMask);
   }
 
   template <typename Function> hash_code getHashValue(Function MapFn) {
-    hash_code H =
-        hash_combine(getOpcode(), getType(), MemoryUseOrder, Volatile);
+    hash_code H = hash_combine(getOpcode(), getType(), MemoryUseOrder, Volatile,
+                               ShuffleMask);
     for (auto *V : operands())
       H = hash_combine(H, MapFn(V));
     return H;
diff --git a/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
index 327a1a6f2e7b6..a9a0070c1d57e 100644
--- a/llvm/lib/Transforms/Scalar/JumpThreading.cpp
+++ b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
@@ -2833,6 +2833,16 @@ bool JumpThreadingPass::TryToUnfoldSelect(CmpInst *CondCmp, BasicBlock *BB) {
 /// select is not jump-threaded, it will be folded again in the later
 /// optimizations.
 bool JumpThreadingPass::TryToUnfoldSelectInCurrBB(BasicBlock *BB) {
+  // This transform can introduce a UB (a conditional branch that depends on a
+  // poison value) that was not present in the original program. See
+  // @TryToUnfoldSelectInCurrBB test in test/Transforms/JumpThreading/select.ll.
+  // Disable this transform under MemorySanitizer.
+  // FIXME: either delete it or replace with a valid transform. This issue is
+  // not limited to MemorySanitizer (but has only been observed as an MSan false
+  // positive in practice so far).
+  if (BB->getParent()->hasFnAttribute(Attribute::SanitizeMemory))
+    return false;
+
   // If threading this would thread across a loop header, don't thread the edge.
   // See the comments above FindLoopHeaders for justifications and caveats.
   if (LoopHeaders.count(BB))
diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp
index fbb01fe1563eb..731089f929ffe 100644
--- a/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -501,6 +501,7 @@ bool llvm::sinkRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI,
         if (sink(I, LI, DT, CurLoop, SafetyInfo, MSSAU, ORE)) {
           if (!FreeInLoop) {
             ++II;
+            salvageDebugInfoOrMarkUndef(I);
             eraseInstruction(I, *SafetyInfo, CurAST, MSSAU);
           }
           Changed = true;
diff --git a/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp b/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp
index ab65f56d088f7..7e958ad751102 100644
--- a/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp
@@ -25,6 +25,7 @@
 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/IR/CFG.h"
+#include "llvm/IR/CallSite.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Module.h"
@@ -61,10 +62,10 @@ namespace {
 /// Loop prefetch implementation class.
 class LoopDataPrefetch {
 public:
-  LoopDataPrefetch(AssumptionCache *AC, LoopInfo *LI, ScalarEvolution *SE,
-                   const TargetTransformInfo *TTI,
+  LoopDataPrefetch(AssumptionCache *AC, DominatorTree *DT, LoopInfo *LI,
+                   ScalarEvolution *SE, const TargetTransformInfo *TTI,
                    OptimizationRemarkEmitter *ORE)
-      : AC(AC), LI(LI), SE(SE), TTI(TTI), ORE(ORE) {}
+      : AC(AC), DT(DT), LI(LI), SE(SE), TTI(TTI), ORE(ORE) {}
 
   bool run();
 
@@ -73,12 +74,16 @@ class LoopDataPrefetch {
 
   /// Check if the stride of the accesses is large enough to
   /// warrant a prefetch.
-  bool isStrideLargeEnough(const SCEVAddRecExpr *AR);
+  bool isStrideLargeEnough(const SCEVAddRecExpr *AR, unsigned TargetMinStride);
 
-  unsigned getMinPrefetchStride() {
+  unsigned getMinPrefetchStride(unsigned NumMemAccesses,
+                                unsigned NumStridedMemAccesses,
+                                unsigned NumPrefetches,
+                                bool HasCall) {
     if (MinPrefetchStride.getNumOccurrences() > 0)
       return MinPrefetchStride;
-    return TTI->getMinPrefetchStride();
+    return TTI->getMinPrefetchStride(NumMemAccesses, NumStridedMemAccesses,
+                                     NumPrefetches, HasCall);
   }
 
   unsigned getPrefetchDistance() {
@@ -93,7 +98,14 @@ class LoopDataPrefetch {
     return TTI->getMaxPrefetchIterationsAhead();
   }
 
+  bool doPrefetchWrites() {
+    if (PrefetchWrites.getNumOccurrences() > 0)
+      return PrefetchWrites;
+    return TTI->enableWritePrefetching();
+  }
+
   AssumptionCache *AC;
+  DominatorTree *DT;
   LoopInfo *LI;
   ScalarEvolution *SE;
   const TargetTransformInfo *TTI;
@@ -110,6 +122,7 @@ class LoopDataPrefetchLegacyPass : public FunctionPass {
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.addRequired<AssumptionCacheTracker>();
+    AU.addRequired<DominatorTreeWrapperPass>();
     AU.addPreserved<DominatorTreeWrapperPass>();
     AU.addRequired<LoopInfoWrapperPass>();
     AU.addPreserved<LoopInfoWrapperPass>();
@@ -138,8 +151,8 @@ FunctionPass *llvm::createLoopDataPrefetchPass() {
   return new LoopDataPrefetchLegacyPass();
 }
 
-bool LoopDataPrefetch::isStrideLargeEnough(const SCEVAddRecExpr *AR) {
-  unsigned TargetMinStride = getMinPrefetchStride();
+bool LoopDataPrefetch::isStrideLargeEnough(const SCEVAddRecExpr *AR,
+                                           unsigned TargetMinStride) {
   // No need to check if any stride goes.
   if (TargetMinStride <= 1)
     return true;
@@ -156,6 +169,7 @@ bool LoopDataPrefetch::isStrideLargeEnough(const SCEVAddRecExpr *AR) {
 
 PreservedAnalyses LoopDataPrefetchPass::run(Function &F,
                                             FunctionAnalysisManager &AM) {
+  DominatorTree *DT = &AM.getResult<DominatorTreeAnalysis>(F);
   LoopInfo *LI = &AM.getResult<LoopAnalysis>(F);
   ScalarEvolution *SE = &AM.getResult<ScalarEvolutionAnalysis>(F);
   AssumptionCache *AC = &AM.getResult<AssumptionAnalysis>(F);
@@ -163,7 +177,7 @@ PreservedAnalyses LoopDataPrefetchPass::run(Function &F,
       &AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
   const TargetTransformInfo *TTI = &AM.getResult<TargetIRAnalysis>(F);
 
-  LoopDataPrefetch LDP(AC, LI, SE, TTI, ORE);
+  LoopDataPrefetch LDP(AC, DT, LI, SE, TTI, ORE);
   bool Changed = LDP.run();
 
   if (Changed) {
@@ -180,6 +194,7 @@ bool LoopDataPrefetchLegacyPass::runOnFunction(Function &F) {
   if (skipFunction(F))
     return false;
 
+  DominatorTree *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
   LoopInfo *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
   ScalarEvolution *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
   AssumptionCache *AC =
@@ -189,7 +204,7 @@ bool LoopDataPrefetchLegacyPass::runOnFunction(Function &F) {
   const TargetTransformInfo *TTI =
       &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
 
-  LoopDataPrefetch LDP(AC, LI, SE, TTI, ORE);
+  LoopDataPrefetch LDP(AC, DT, LI, SE, TTI, ORE);
   return LDP.run();
 }
 
@@ -210,6 +225,49 @@ bool LoopDataPrefetch::run() {
   return MadeChange;
 }
 
+/// A record for a potential prefetch made during the initial scan of the
+/// loop. This is used to let a single prefetch target multiple memory accesses.
+struct Prefetch {
+  /// The address formula for this prefetch as returned by ScalarEvolution.
+  const SCEVAddRecExpr *LSCEVAddRec;
+  /// The point of insertion for the prefetch instruction.
+  Instruction *InsertPt;
+  /// True if targeting a write memory access.
+  bool Writes;
+  /// The (first seen) prefetched instruction.
+  Instruction *MemI;
+
+  /// Constructor to create a new Prefetch for \param I.
+  Prefetch(const SCEVAddRecExpr *L, Instruction *I)
+      : LSCEVAddRec(L), InsertPt(nullptr), Writes(false), MemI(nullptr) {
+    addInstruction(I);
+  };
+
+  /// Add the instruction \param I to this prefetch. If it's not the first
+  /// one, 'InsertPt' and 'Writes' will be updated as required.
+  /// \param PtrDiff the known constant address difference to the first added
+  /// instruction.
+  void addInstruction(Instruction *I, DominatorTree *DT = nullptr,
+                      int64_t PtrDiff = 0) {
+    if (!InsertPt) {
+      MemI = I;
+      InsertPt = I;
+      Writes = isa<StoreInst>(I);
+    } else {
+      BasicBlock *PrefBB = InsertPt->getParent();
+      BasicBlock *InsBB = I->getParent();
+      if (PrefBB != InsBB) {
+        BasicBlock *DomBB = DT->findNearestCommonDominator(PrefBB, InsBB);
+        if (DomBB != PrefBB)
+          InsertPt = DomBB->getTerminator();
+      }
+
+      if (isa<StoreInst>(I) && PtrDiff == 0)
+        Writes = true;
+    }
+  }
+};
+
 bool LoopDataPrefetch::runOnLoop(Loop *L) {
   bool MadeChange = false;
 
@@ -222,15 +280,23 @@ bool LoopDataPrefetch::runOnLoop(Loop *L) {
 
   // Calculate the number of iterations ahead to prefetch
   CodeMetrics Metrics;
+  bool HasCall = false;
   for (const auto BB : L->blocks()) {
     // If the loop already has prefetches, then assume that the user knows
     // what they are doing and don't add any more.
-    for (auto &I : *BB)
-      if (CallInst *CI = dyn_cast<CallInst>(&I))
-        if (Function *F = CI->getCalledFunction())
+    for (auto &I : *BB) {
+      if (isa<CallInst>(&I) || isa<InvokeInst>(&I)) {
+        ImmutableCallSite CS(&I);
+        if (const Function *F = CS.getCalledFunction()) {
           if (F->getIntrinsicID() == Intrinsic::prefetch)
             return MadeChange;
-
+          if (TTI->isLoweredToCall(F))
+            HasCall = true;
+        } else { // indirect call.
+          HasCall = true;
+        }
+      }
+    }
     Metrics.analyzeBasicBlock(BB, *TTI, EphValues);
   }
   unsigned LoopSize = Metrics.NumInsts;
@@ -244,12 +310,14 @@ bool LoopDataPrefetch::runOnLoop(Loop *L) {
   if (ItersAhead > getMaxPrefetchIterationsAhead())
     return MadeChange;
 
-  LLVM_DEBUG(dbgs() << "Prefetching " << ItersAhead
-                    << " iterations ahead (loop size: " << LoopSize << ") in "
-                    << L->getHeader()->getParent()->getName() << ": " << *L);
+  unsigned ConstantMaxTripCount = SE->getSmallConstantMaxTripCount(L);
+  if (ConstantMaxTripCount && ConstantMaxTripCount < ItersAhead + 1)
+    return MadeChange;
 
-  SmallVector<std::pair<Instruction *, const SCEVAddRecExpr *>, 16> PrefLoads;
-  for (const auto BB : L->blocks()) {
+  unsigned NumMemAccesses = 0;
+  unsigned NumStridedMemAccesses = 0;
+  SmallVector<Prefetch, 16> Prefetches;
+  for (const auto BB : L->blocks())
     for (auto &I : *BB) {
       Value *PtrValue;
       Instruction *MemI;
@@ -258,7 +326,7 @@ bool LoopDataPrefetch::runOnLoop(Loop *L) {
         MemI = LMemI;
         PtrValue = LMemI->getPointerOperand();
       } else if (StoreInst *SMemI = dyn_cast<StoreInst>(&I)) {
-        if (!PrefetchWrites) continue;
+        if (!doPrefetchWrites()) continue;
         MemI = SMemI;
         PtrValue = SMemI->getPointerOperand();
       } else continue;
@@ -266,7 +334,7 @@ bool LoopDataPrefetch::runOnLoop(Loop *L) {
       unsigned PtrAddrSpace = PtrValue->getType()->getPointerAddressSpace();
       if (PtrAddrSpace)
         continue;
-
+      NumMemAccesses++;
       if (L->isLoopInvariant(PtrValue))
         continue;
 
@@ -274,62 +342,79 @@ bool LoopDataPrefetch::runOnLoop(Loop *L) {
       const SCEVAddRecExpr *LSCEVAddRec = dyn_cast<SCEVAddRecExpr>(LSCEV);
       if (!LSCEVAddRec)
         continue;
+      NumStridedMemAccesses++;
 
-      // Check if the stride of the accesses is large enough to warrant a
-      // prefetch.
-      if (!isStrideLargeEnough(LSCEVAddRec))
-        continue;
-
-      // We don't want to double prefetch individual cache lines. If this load
-      // is known to be within one cache line of some other load that has
-      // already been prefetched, then don't prefetch this one as well.
+      // We don't want to double prefetch individual cache lines. If this
+      // access is known to be within one cache line of some other one that
+      // has already been prefetched, then don't prefetch this one as well.
       bool DupPref = false;
-      for (const auto &PrefLoad : PrefLoads) {
-        const SCEV *PtrDiff = SE->getMinusSCEV(LSCEVAddRec, PrefLoad.second);
+      for (auto &Pref : Prefetches) {
+        const SCEV *PtrDiff = SE->getMinusSCEV(LSCEVAddRec, Pref.LSCEVAddRec);
         if (const SCEVConstant *ConstPtrDiff =
             dyn_cast<SCEVConstant>(PtrDiff)) {
           int64_t PD = std::abs(ConstPtrDiff->getValue()->getSExtValue());
           if (PD < (int64_t) TTI->getCacheLineSize()) {
+            Pref.addInstruction(MemI, DT, PD);
             DupPref = true;
             break;
           }
         }
       }
-      if (DupPref)
-        continue;
+      if (!DupPref)
+        Prefetches.push_back(Prefetch(LSCEVAddRec, MemI));
+    }
 
-      const SCEV *NextLSCEV = SE->getAddExpr(LSCEVAddRec, SE->getMulExpr(
-        SE->getConstant(LSCEVAddRec->getType(), ItersAhead),
-        LSCEVAddRec->getStepRecurrence(*SE)));
-      if (!isSafeToExpand(NextLSCEV, *SE))
-        continue;
+  unsigned TargetMinStride =
+    getMinPrefetchStride(NumMemAccesses, NumStridedMemAccesses,
+                         Prefetches.size(), HasCall);
 
-      PrefLoads.push_back(std::make_pair(MemI, LSCEVAddRec));
-
-      Type *I8Ptr = Type::getInt8PtrTy(BB->getContext(), PtrAddrSpace);
-      SCEVExpander SCEVE(*SE, I.getModule()->getDataLayout(), "prefaddr");
-      Value *PrefPtrValue = SCEVE.expandCodeFor(NextLSCEV, I8Ptr, MemI);
-
-      IRBuilder<> Builder(MemI);
-      Module *M = BB->getParent()->getParent();
-      Type *I32 = Type::getInt32Ty(BB->getContext());
-      Function *PrefetchFunc = Intrinsic::getDeclaration(
-          M, Intrinsic::prefetch, PrefPtrValue->getType());
-      Builder.CreateCall(
-          PrefetchFunc,
-          {PrefPtrValue,
-           ConstantInt::get(I32, MemI->mayReadFromMemory() ? 0 : 1),
-           ConstantInt::get(I32, 3), ConstantInt::get(I32, 1)});
-      ++NumPrefetches;
-      LLVM_DEBUG(dbgs() << "  Access: " << *PtrValue << ", SCEV: " << *LSCEV
-                        << "\n");
-      ORE->emit([&]() {
-        return OptimizationRemark(DEBUG_TYPE, "Prefetched", MemI)
-               << "prefetched memory access";
+  LLVM_DEBUG(dbgs() << "Prefetching " << ItersAhead
+             << " iterations ahead (loop size: " << LoopSize << ") in "
+             << L->getHeader()->getParent()->getName() << ": " << *L);
+  LLVM_DEBUG(dbgs() << "Loop has: "
+             << NumMemAccesses << " memory accesses, "
+             << NumStridedMemAccesses << " strided memory accesses, "
+             << Prefetches.size() << " potential prefetch(es), "
+             << "a minimum stride of " << TargetMinStride << ", "
+             << (HasCall ? "calls" : "no calls") << ".\n");
+
+  for (auto &P : Prefetches) {
+    // Check if the stride of the accesses is large enough to warrant a
+    // prefetch.
+    if (!isStrideLargeEnough(P.LSCEVAddRec, TargetMinStride))
+      continue;
+
+    const SCEV *NextLSCEV = SE->getAddExpr(P.LSCEVAddRec, SE->getMulExpr(
+      SE->getConstant(P.LSCEVAddRec->getType(), ItersAhead),
+      P.LSCEVAddRec->getStepRecurrence(*SE)));
+    if (!isSafeToExpand(NextLSCEV, *SE))
+      continue;
+
+    BasicBlock *BB = P.InsertPt->getParent();
+    Type *I8Ptr = Type::getInt8PtrTy(BB->getContext(), 0/*PtrAddrSpace*/);
+    SCEVExpander SCEVE(*SE, BB->getModule()->getDataLayout(), "prefaddr");
+    Value *PrefPtrValue = SCEVE.expandCodeFor(NextLSCEV, I8Ptr, P.InsertPt);
+
+    IRBuilder<> Builder(P.InsertPt);
+    Module *M = BB->getParent()->getParent();
+    Type *I32 = Type::getInt32Ty(BB->getContext());
+    Function *PrefetchFunc = Intrinsic::getDeclaration(
+        M, Intrinsic::prefetch, PrefPtrValue->getType());
+    Builder.CreateCall(
+        PrefetchFunc,
+        {PrefPtrValue,
+         ConstantInt::get(I32, P.Writes),
+         ConstantInt::get(I32, 3), ConstantInt::get(I32, 1)});
+    ++NumPrefetches;
+    LLVM_DEBUG(dbgs() << "  Access: "
+               << *P.MemI->getOperand(isa<LoadInst>(P.MemI) ? 0 : 1)
+               << ", SCEV: " << *P.LSCEVAddRec << "\n");
+    ORE->emit([&]() {
+        return OptimizationRemark(DEBUG_TYPE, "Prefetched", P.MemI)
+          << "prefetched memory access";
       });
 
-      MadeChange = true;
-    }
+    MadeChange = true;
   }
 
   return MadeChange;
diff --git a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
index 671da3f539931..6b66310b1e3d8 100644
--- a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
@@ -873,11 +873,11 @@ class LowerMatrixIntrinsics {
     }
   }
 
-  /// Compute Res += A * B for tile-sized matrices with left-associating
+  /// Compute \p Result += \p A * \p B for input matrices with left-associating
   /// addition.
-  void emitChainedMatrixMultiply(MatrixTy &Result, const MatrixTy &A,
-                                 const MatrixTy &B, bool AllowContraction,
-                                 IRBuilder<> &Builder, bool isTiled) {
+  void emitMatrixMultiply(MatrixTy &Result, const MatrixTy &A,
+                          const MatrixTy &B, bool AllowContraction,
+                          IRBuilder<> &Builder, bool isTiled) {
     const unsigned VF = std::max<unsigned>(
         TTI.getRegisterBitWidth(true) /
             Result.getElementType()->getPrimitiveSizeInBits().getFixedSize(),
@@ -936,7 +936,7 @@ class LowerMatrixIntrinsics {
 
     bool AllowContract = AllowContractEnabled || (isa<FPMathOperator>(MatMul) &&
                                                   MatMul->hasAllowContract());
-    emitChainedMatrixMultiply(Result, Lhs, Rhs, AllowContract, Builder, false);
+    emitMatrixMultiply(Result, Lhs, Rhs, AllowContract, Builder, false);
     finalizeLowering(MatMul, Result, Builder);
   }
 
diff --git a/llvm/lib/Transforms/Scalar/NewGVN.cpp b/llvm/lib/Transforms/Scalar/NewGVN.cpp
index 6a643480f3128..b93a8bfeaa462 100644
--- a/llvm/lib/Transforms/Scalar/NewGVN.cpp
+++ b/llvm/lib/Transforms/Scalar/NewGVN.cpp
@@ -1470,7 +1470,8 @@ NewGVN::performSymbolicLoadCoercion(Type *LoadType, Value *LoadPtr,
   // undef value.  This can happen when loading for a fresh allocation with no
   // intervening stores, for example.  Note that this is only true in the case
   // that the result of the allocation is pointer equal to the load ptr.
-  if (isa<AllocaInst>(DepInst) || isMallocLikeFn(DepInst, TLI)) {
+  if (isa<AllocaInst>(DepInst) || isMallocLikeFn(DepInst, TLI) ||
+      isAlignedAllocLikeFn(DepInst, TLI)) {
     return createConstantExpression(UndefValue::get(LoadType));
   }
   // If this load occurs either right after a lifetime begin,
@@ -2030,10 +2031,12 @@ NewGVN::performSymbolicEvaluation(Value *V,
     case Instruction::Select:
     case Instruction::ExtractElement:
     case Instruction::InsertElement:
-    case Instruction::ShuffleVector:
     case Instruction::GetElementPtr:
       E = createExpression(I);
       break;
+    case Instruction::ShuffleVector:
+      // FIXME: Add support for shufflevector to createExpression.
+      return nullptr;
     default:
       return nullptr;
     }
diff --git a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
index fe344ce5f8db4..b8b98d673334c 100644
--- a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
+++ b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
@@ -1009,7 +1009,7 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) {
         auto *SV = cast<ShuffleVectorInst>(I);
         UndefValue *VecUndef = UndefValue::get(SV->getOperand(0)->getType());
         std::string Name = suffixed_name_or(I, ".base", "base_sv");
-        return new ShuffleVectorInst(VecUndef, VecUndef, SV->getOperand(2),
+        return new ShuffleVectorInst(VecUndef, VecUndef, SV->getShuffleMask(),
                                      Name, SV);
       }
     };
diff --git a/llvm/lib/Transforms/Scalar/SCCP.cpp b/llvm/lib/Transforms/Scalar/SCCP.cpp
index fcd7ed195291a..eba3a11bff195 100644
--- a/llvm/lib/Transforms/Scalar/SCCP.cpp
+++ b/llvm/lib/Transforms/Scalar/SCCP.cpp
@@ -74,23 +74,20 @@ STATISTIC(IPNumGlobalConst, "Number of globals found to be constant by IPSCCP");
 
 namespace {
 
-// Use ValueLatticeElement as LatticeVal.
-using LatticeVal = ValueLatticeElement;
-
 // Helper to check if \p LV is either a constant or a constant
 // range with a single element. This should cover exactly the same cases as the
-// old LatticeVal::isConstant() and is intended to be used in the transition
-// from LatticeVal to LatticeValueElement.
-bool isConstant(const LatticeVal &LV) {
+// old ValueLatticeElement::isConstant() and is intended to be used in the
+// transition to ValueLatticeElement.
+bool isConstant(const ValueLatticeElement &LV) {
   return LV.isConstant() ||
          (LV.isConstantRange() && LV.getConstantRange().isSingleElement());
 }
 
 // Helper to check if \p LV is either overdefined or a constant range with more
 // than a single element. This should cover exactly the same cases as the old
-// LatticeVal::isOverdefined() and is intended to be used in the transition from
-// LatticeVal to LatticeValueElement.
-bool isOverdefined(const LatticeVal &LV) {
+// ValueLatticeElement::isOverdefined() and is intended to be used in the
+// transition to ValueLatticeElement.
+bool isOverdefined(const ValueLatticeElement &LV) {
   return LV.isOverdefined() ||
          (LV.isConstantRange() && !LV.getConstantRange().isSingleElement());
 }
@@ -104,26 +101,28 @@ class SCCPSolver : public InstVisitor<SCCPSolver> {
   const DataLayout &DL;
   std::function<const TargetLibraryInfo &(Function &)> GetTLI;
   SmallPtrSet<BasicBlock *, 8> BBExecutable; // The BBs that are executable.
-  DenseMap<Value *, LatticeVal> ValueState;  // The state each value is in.
+  DenseMap<Value *, ValueLatticeElement>
+      ValueState; // The state each value is in.
 
   /// StructValueState - This maintains ValueState for values that have
   /// StructType, for example for formal arguments, calls, insertelement, etc.
-  DenseMap<std::pair<Value *, unsigned>, LatticeVal> StructValueState;
+  DenseMap<std::pair<Value *, unsigned>, ValueLatticeElement> StructValueState;
 
   /// GlobalValue - If we are tracking any values for the contents of a global
   /// variable, we keep a mapping from the constant accessor to the element of
   /// the global, to the currently known value.  If the value becomes
   /// overdefined, it's entry is simply removed from this map.
-  DenseMap<GlobalVariable *, LatticeVal> TrackedGlobals;
+  DenseMap<GlobalVariable *, ValueLatticeElement> TrackedGlobals;
 
   /// TrackedRetVals - If we are tracking arguments into and the return
   /// value out of a function, it will have an entry in this map, indicating
   /// what the known return value for the function is.
-  MapVector<Function *, LatticeVal> TrackedRetVals;
+  MapVector<Function *, ValueLatticeElement> TrackedRetVals;
 
   /// TrackedMultipleRetVals - Same as TrackedRetVals, but used for functions
   /// that return multiple values.
-  MapVector<std::pair<Function *, unsigned>, LatticeVal> TrackedMultipleRetVals;
+  MapVector<std::pair<Function *, unsigned>, ValueLatticeElement>
+      TrackedMultipleRetVals;
 
   /// MRVFunctionsTracked - Each function in TrackedMultipleRetVals is
   /// represented here for efficient lookup.
@@ -203,7 +202,7 @@ class SCCPSolver : public InstVisitor<SCCPSolver> {
   void TrackValueOfGlobalVariable(GlobalVariable *GV) {
     // We only track the contents of scalar globals.
     if (GV->getValueType()->isSingleValueType()) {
-      LatticeVal &IV = TrackedGlobals[GV];
+      ValueLatticeElement &IV = TrackedGlobals[GV];
       if (!isa<UndefValue>(GV->getInitializer()))
         IV.markConstant(GV->getInitializer());
     }
@@ -217,10 +216,10 @@ class SCCPSolver : public InstVisitor<SCCPSolver> {
     if (auto *STy = dyn_cast<StructType>(F->getReturnType())) {
       MRVFunctionsTracked.insert(F);
       for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i)
-        TrackedMultipleRetVals.insert(std::make_pair(std::make_pair(F, i),
-                                                     LatticeVal()));
+        TrackedMultipleRetVals.insert(
+            std::make_pair(std::make_pair(F, i), ValueLatticeElement()));
     } else
-      TrackedRetVals.insert(std::make_pair(F, LatticeVal()));
+      TrackedRetVals.insert(std::make_pair(F, ValueLatticeElement()));
   }
 
   /// AddMustTailCallee - If the SCCP solver finds that this function is called
@@ -263,8 +262,8 @@ class SCCPSolver : public InstVisitor<SCCPSolver> {
   // block to the 'To' basic block is currently feasible.
   bool isEdgeFeasible(BasicBlock *From, BasicBlock *To);
 
-  std::vector<LatticeVal> getStructLatticeValueFor(Value *V) const {
-    std::vector<LatticeVal> StructValues;
+  std::vector<ValueLatticeElement> getStructLatticeValueFor(Value *V) const {
+    std::vector<ValueLatticeElement> StructValues;
     auto *STy = dyn_cast<StructType>(V->getType());
     assert(STy && "getStructLatticeValueFor() can be called only on structs");
     for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
@@ -275,23 +274,24 @@ class SCCPSolver : public InstVisitor<SCCPSolver> {
     return StructValues;
   }
 
-  const LatticeVal &getLatticeValueFor(Value *V) const {
+  const ValueLatticeElement &getLatticeValueFor(Value *V) const {
     assert(!V->getType()->isStructTy() &&
            "Should use getStructLatticeValueFor");
-    DenseMap<Value *, LatticeVal>::const_iterator I = ValueState.find(V);
+    DenseMap<Value *, ValueLatticeElement>::const_iterator I =
+        ValueState.find(V);
     assert(I != ValueState.end() &&
            "V not found in ValueState nor Paramstate map!");
     return I->second;
   }
 
   /// getTrackedRetVals - Get the inferred return value map.
-  const MapVector<Function*, LatticeVal> &getTrackedRetVals() {
+  const MapVector<Function *, ValueLatticeElement> &getTrackedRetVals() {
     return TrackedRetVals;
   }
 
   /// getTrackedGlobals - Get and return the set of inferred initializers for
   /// global variables.
-  const DenseMap<GlobalVariable*, LatticeVal> &getTrackedGlobals() {
+  const DenseMap<GlobalVariable *, ValueLatticeElement> &getTrackedGlobals() {
     return TrackedGlobals;
   }
 
@@ -324,7 +324,7 @@ class SCCPSolver : public InstVisitor<SCCPSolver> {
     for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
       const auto &It = TrackedMultipleRetVals.find(std::make_pair(F, i));
       assert(It != TrackedMultipleRetVals.end());
-      LatticeVal LV = It->second;
+      ValueLatticeElement LV = It->second;
       if (!isConstant(LV))
         return false;
     }
@@ -333,7 +333,7 @@ class SCCPSolver : public InstVisitor<SCCPSolver> {
 
   /// Helper to return a Constant if \p LV is either a constant or a constant
   /// range with a single element.
-  Constant *getConstant(const LatticeVal &LV) const {
+  Constant *getConstant(const ValueLatticeElement &LV) const {
     if (LV.isConstant())
       return LV.getConstant();
 
@@ -346,12 +346,12 @@ class SCCPSolver : public InstVisitor<SCCPSolver> {
   }
 
 private:
-  ConstantInt *getConstantInt(const LatticeVal &IV) const {
+  ConstantInt *getConstantInt(const ValueLatticeElement &IV) const {
     return dyn_cast_or_null<ConstantInt>(getConstant(IV));
   }
 
   // pushToWorkList - Helper for markConstant/markOverdefined
-  void pushToWorkList(LatticeVal &IV, Value *V) {
+  void pushToWorkList(ValueLatticeElement &IV, Value *V) {
     if (IV.isOverdefined())
       return OverdefinedInstWorkList.push_back(V);
     InstWorkList.push_back(V);
@@ -359,7 +359,7 @@ class SCCPSolver : public InstVisitor<SCCPSolver> {
 
   // Helper to push \p V to the worklist, after updating it to \p IV. Also
   // prints a debug message with the updated value.
-  void pushToWorkListMsg(LatticeVal &IV, Value *V) {
+  void pushToWorkListMsg(ValueLatticeElement &IV, Value *V) {
     LLVM_DEBUG(dbgs() << "updated " << IV << ": " << *V << '\n');
     if (IV.isOverdefined())
       return OverdefinedInstWorkList.push_back(V);
@@ -369,7 +369,7 @@ class SCCPSolver : public InstVisitor<SCCPSolver> {
   // markConstant - Make a value be marked as "constant".  If the value
   // is not already a constant, add it to the instruction work list so that
   // the users of the instruction are updated later.
-  bool markConstant(LatticeVal &IV, Value *V, Constant *C) {
+  bool markConstant(ValueLatticeElement &IV, Value *V, Constant *C) {
     if (!IV.markConstant(C)) return false;
     LLVM_DEBUG(dbgs() << "markConstant: " << *C << ": " << *V << '\n');
     pushToWorkList(IV, V);
@@ -384,7 +384,7 @@ class SCCPSolver : public InstVisitor<SCCPSolver> {
   // markOverdefined - Make a value be marked as "overdefined". If the
   // value is not already overdefined, add it to the overdefined instruction
   // work list so that the users of the instruction are updated later.
-  bool markOverdefined(LatticeVal &IV, Value *V) {
+  bool markOverdefined(ValueLatticeElement &IV, Value *V) {
     if (!IV.markOverdefined()) return false;
 
     LLVM_DEBUG(dbgs() << "markOverdefined: ";
@@ -396,8 +396,8 @@ class SCCPSolver : public InstVisitor<SCCPSolver> {
     return true;
   }
 
-  bool mergeInValue(LatticeVal &IV, Value *V, LatticeVal MergeWithV,
-                    bool Widen = true) {
+  bool mergeInValue(ValueLatticeElement &IV, Value *V,
+                    ValueLatticeElement MergeWithV, bool Widen = true) {
     // Do a simple form of widening, to avoid extending a range repeatedly in a
     // loop. If IV is a constant range, it means we already set it once. If
     // MergeWithV would extend IV, mark V as overdefined.
@@ -415,21 +415,21 @@ class SCCPSolver : public InstVisitor<SCCPSolver> {
     return false;
   }
 
-  bool mergeInValue(Value *V, LatticeVal MergeWithV, bool Widen = true) {
+  bool mergeInValue(Value *V, ValueLatticeElement MergeWithV,
+                    bool Widen = true) {
     assert(!V->getType()->isStructTy() &&
            "non-structs should use markConstant");
     return mergeInValue(ValueState[V], V, MergeWithV, Widen);
   }
 
-  /// getValueState - Return the LatticeVal object that corresponds to the
-  /// value.  This function handles the case when the value hasn't been seen yet
-  /// by properly seeding constants etc.
-  LatticeVal &getValueState(Value *V) {
+  /// getValueState - Return the ValueLatticeElement object that corresponds to
+  /// the value.  This function handles the case when the value hasn't been seen
+  /// yet by properly seeding constants etc.
+  ValueLatticeElement &getValueState(Value *V) {
     assert(!V->getType()->isStructTy() && "Should use getStructValueState");
 
-    std::pair<DenseMap<Value*, LatticeVal>::iterator, bool> I =
-      ValueState.insert(std::make_pair(V, LatticeVal()));
-    LatticeVal &LV = I.first->second;
+    auto I = ValueState.insert(std::make_pair(V, ValueLatticeElement()));
+    ValueLatticeElement &LV = I.first->second;
 
     if (!I.second)
       return LV;  // Common case, already in the map.
@@ -441,36 +441,17 @@ class SCCPSolver : public InstVisitor<SCCPSolver> {
     return LV;
   }
 
-  LatticeVal toLatticeVal(const ValueLatticeElement &V, Type *T) {
-    LatticeVal Res;
-    if (V.isUnknownOrUndef())
-      return Res;
-
-    if (V.isConstant()) {
-      Res.markConstant(V.getConstant());
-      return Res;
-    }
-    if (V.isConstantRange() && V.getConstantRange().isSingleElement()) {
-      Res.markConstant(
-          ConstantInt::get(T, *V.getConstantRange().getSingleElement()));
-      return Res;
-    }
-    Res.markOverdefined();
-    return Res;
-  }
-
-  /// getStructValueState - Return the LatticeVal object that corresponds to the
-  /// value/field pair.  This function handles the case when the value hasn't
-  /// been seen yet by properly seeding constants etc.
-  LatticeVal &getStructValueState(Value *V, unsigned i) {
+  /// getStructValueState - Return the ValueLatticeElement object that
+  /// corresponds to the value/field pair.  This function handles the case when
+  /// the value hasn't been seen yet by properly seeding constants etc.
+  ValueLatticeElement &getStructValueState(Value *V, unsigned i) {
     assert(V->getType()->isStructTy() && "Should use getValueState");
     assert(i < cast<StructType>(V->getType())->getNumElements() &&
            "Invalid element #");
 
-    std::pair<DenseMap<std::pair<Value*, unsigned>, LatticeVal>::iterator,
-              bool> I = StructValueState.insert(
-                        std::make_pair(std::make_pair(V, i), LatticeVal()));
-    LatticeVal &LV = I.first->second;
+    auto I = StructValueState.insert(
+        std::make_pair(std::make_pair(V, i), ValueLatticeElement()));
+    ValueLatticeElement &LV = I.first->second;
 
     if (!I.second)
       return LV;  // Common case, already in the map.
@@ -627,7 +608,7 @@ void SCCPSolver::getFeasibleSuccessors(Instruction &TI,
       return;
     }
 
-    LatticeVal BCValue = getValueState(BI->getCondition());
+    ValueLatticeElement BCValue = getValueState(BI->getCondition());
     ConstantInt *CI = getConstantInt(BCValue);
     if (!CI) {
       // Overdefined condition variables, and branches on unfoldable constant
@@ -653,7 +634,7 @@ void SCCPSolver::getFeasibleSuccessors(Instruction &TI,
       Succs[0] = true;
       return;
     }
-    LatticeVal SCValue = getValueState(SI->getCondition());
+    ValueLatticeElement SCValue = getValueState(SI->getCondition());
     ConstantInt *CI = getConstantInt(SCValue);
 
     if (!CI) {   // Overdefined or unknown condition?
@@ -671,7 +652,7 @@ void SCCPSolver::getFeasibleSuccessors(Instruction &TI,
   // the target as executable.
   if (auto *IBR = dyn_cast<IndirectBrInst>(&TI)) {
     // Casts are folded by visitCastInst.
-    LatticeVal IBRValue = getValueState(IBR->getAddress());
+    ValueLatticeElement IBRValue = getValueState(IBR->getAddress());
     BlockAddress *Addr = dyn_cast_or_null<BlockAddress>(getConstant(IBRValue));
     if (!Addr) {   // Overdefined or unknown condition?
       // All destinations are executable!
@@ -754,11 +735,11 @@ void SCCPSolver::visitPHINode(PHINode &PN) {
   // If there are no executable operands, the PHI remains unknown.
   bool Changed = false;
   for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i) {
-    LatticeVal IV = getValueState(PN.getIncomingValue(i));
+    ValueLatticeElement IV = getValueState(PN.getIncomingValue(i));
     if (!isEdgeFeasible(PN.getIncomingBlock(i), PN.getParent()))
       continue;
 
-    LatticeVal &Res = getValueState(&PN);
+    ValueLatticeElement &Res = getValueState(&PN);
     Changed |= Res.mergeIn(IV, DL);
     if (Res.isOverdefined())
       break;
@@ -780,8 +761,7 @@ void SCCPSolver::visitReturnInst(ReturnInst &I) {
 
   // If we are tracking the return value of this function, merge it in.
   if (!TrackedRetVals.empty() && !ResultOp->getType()->isStructTy()) {
-    MapVector<Function*, LatticeVal>::iterator TFRVI =
-      TrackedRetVals.find(F);
+    auto TFRVI = TrackedRetVals.find(F);
     if (TFRVI != TrackedRetVals.end()) {
       mergeInValue(TFRVI->second, F, getValueState(ResultOp));
       return;
@@ -813,10 +793,10 @@ void SCCPSolver::visitTerminator(Instruction &TI) {
 void SCCPSolver::visitCastInst(CastInst &I) {
   // ResolvedUndefsIn might mark I as overdefined. Bail out, even if we would
   // discover a concrete value later.
-  if (isOverdefined(ValueState[&I]))
-    return (void)markOverdefined(&I);
+  if (ValueState[&I].isOverdefined())
+    return;
 
-  LatticeVal OpSt = getValueState(I.getOperand(0));
+  ValueLatticeElement OpSt = getValueState(I.getOperand(0));
   if (Constant *OpC = getConstant(OpSt)) {
     // Fold the constant as we build.
     Constant *C = ConstantFoldCastOperand(I.getOpcode(), OpC, I.getType(), DL);
@@ -824,6 +804,13 @@ void SCCPSolver::visitCastInst(CastInst &I) {
       return;
     // Propagate constant value
     markConstant(&I, C);
+  } else if (OpSt.isConstantRange() && I.getDestTy()->isIntegerTy()) {
+    auto &LV = getValueState(&I);
+    ConstantRange OpRange = OpSt.getConstantRange();
+    Type *DestTy = I.getDestTy();
+    ConstantRange Res =
+        OpRange.castOp(I.getOpcode(), DL.getTypeSizeInBits(DestTy));
+    mergeInValue(LV, &I, ValueLatticeElement::getRange(Res));
   } else if (!OpSt.isUnknownOrUndef())
     markOverdefined(&I);
 }
@@ -846,7 +833,7 @@ void SCCPSolver::visitExtractValueInst(ExtractValueInst &EVI) {
   Value *AggVal = EVI.getAggregateOperand();
   if (AggVal->getType()->isStructTy()) {
     unsigned i = *EVI.idx_begin();
-    LatticeVal EltVal = getStructValueState(AggVal, i);
+    ValueLatticeElement EltVal = getStructValueState(AggVal, i);
     mergeInValue(getValueState(&EVI), &EVI, EltVal);
   } else {
     // Otherwise, must be extracting from an array.
@@ -876,7 +863,7 @@ void SCCPSolver::visitInsertValueInst(InsertValueInst &IVI) {
   for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
     // This passes through all values that aren't the inserted element.
     if (i != Idx) {
-      LatticeVal EltVal = getStructValueState(Aggr, i);
+      ValueLatticeElement EltVal = getStructValueState(Aggr, i);
       mergeInValue(getStructValueState(&IVI, i), &IVI, EltVal);
       continue;
     }
@@ -886,7 +873,7 @@ void SCCPSolver::visitInsertValueInst(InsertValueInst &IVI) {
       // We don't track structs in structs.
       markOverdefined(getStructValueState(&IVI, i), &IVI);
     else {
-      LatticeVal InVal = getValueState(Val);
+      ValueLatticeElement InVal = getValueState(Val);
       mergeInValue(getStructValueState(&IVI, i), &IVI, InVal);
     }
   }
@@ -903,7 +890,7 @@ void SCCPSolver::visitSelectInst(SelectInst &I) {
   if (ValueState[&I].isOverdefined())
     return (void)markOverdefined(&I);
 
-  LatticeVal CondValue = getValueState(I.getCondition());
+  ValueLatticeElement CondValue = getValueState(I.getCondition());
   if (CondValue.isUnknownOrUndef())
     return;
 
@@ -916,8 +903,8 @@ void SCCPSolver::visitSelectInst(SelectInst &I) {
   // Otherwise, the condition is overdefined or a constant we can't evaluate.
   // See if we can produce something better than overdefined based on the T/F
   // value.
-  LatticeVal TVal = getValueState(I.getTrueValue());
-  LatticeVal FVal = getValueState(I.getFalseValue());
+  ValueLatticeElement TVal = getValueState(I.getTrueValue());
+  ValueLatticeElement FVal = getValueState(I.getFalseValue());
 
   bool Changed = ValueState[&I].mergeIn(TVal, DL);
   Changed |= ValueState[&I].mergeIn(FVal, DL);
@@ -927,9 +914,9 @@ void SCCPSolver::visitSelectInst(SelectInst &I) {
 
 // Handle Unary Operators.
 void SCCPSolver::visitUnaryOperator(Instruction &I) {
-  LatticeVal V0State = getValueState(I.getOperand(0));
+  ValueLatticeElement V0State = getValueState(I.getOperand(0));
 
-  LatticeVal &IV = ValueState[&I];
+  ValueLatticeElement &IV = ValueState[&I];
   // ResolvedUndefsIn might mark I as overdefined. Bail out, even if we would
   // discover a concrete value later.
   if (isOverdefined(IV))
@@ -953,10 +940,10 @@ void SCCPSolver::visitUnaryOperator(Instruction &I) {
 
 // Handle Binary Operators.
 void SCCPSolver::visitBinaryOperator(Instruction &I) {
-  LatticeVal V1State = getValueState(I.getOperand(0));
-  LatticeVal V2State = getValueState(I.getOperand(1));
+  ValueLatticeElement V1State = getValueState(I.getOperand(0));
+  ValueLatticeElement V2State = getValueState(I.getOperand(1));
 
-  LatticeVal &IV = ValueState[&I];
+  ValueLatticeElement &IV = ValueState[&I];
   if (IV.isOverdefined())
     return;
 
@@ -978,6 +965,10 @@ void SCCPSolver::visitBinaryOperator(Instruction &I) {
     return (void)markConstant(IV, &I, C);
   }
 
+  // Only use ranges for binary operators on integers.
+  if (!I.getType()->isIntegerTy())
+    return markOverdefined(&I);
+
   // Operands are either constant ranges, notconstant, overdefined or one of the
   // operands is a constant.
   ConstantRange A = ConstantRange::getFull(I.getType()->getScalarSizeInBits());
@@ -988,7 +979,7 @@ void SCCPSolver::visitBinaryOperator(Instruction &I) {
     B = V2State.getConstantRange();
 
   ConstantRange R = A.binaryOp(cast<BinaryOperator>(&I)->getOpcode(), B);
-  mergeInValue(&I, LatticeVal::getRange(R));
+  mergeInValue(&I, ValueLatticeElement::getRange(R));
 
   // TODO: Currently we do not exploit special values that produce something
   // better than overdefined with an overdefined operand for vector or floating
@@ -1014,7 +1005,7 @@ void SCCPSolver::visitCmpInst(CmpInst &I) {
   if (C) {
     if (isa<UndefValue>(C))
       return;
-    LatticeVal CV;
+    ValueLatticeElement CV;
     CV.markConstant(C);
     mergeInValue(&I, CV);
     return;
@@ -1038,7 +1029,7 @@ void SCCPSolver::visitGetElementPtrInst(GetElementPtrInst &I) {
   Operands.reserve(I.getNumOperands());
 
   for (unsigned i = 0, e = I.getNumOperands(); i != e; ++i) {
-    LatticeVal State = getValueState(I.getOperand(i));
+    ValueLatticeElement State = getValueState(I.getOperand(i));
     if (State.isUnknownOrUndef())
       return;  // Operands are not resolved yet.
 
@@ -1076,7 +1067,7 @@ void SCCPSolver::visitStoreInst(StoreInst &SI) {
     return (void)markOverdefined(&SI);
 
   GlobalVariable *GV = cast<GlobalVariable>(SI.getOperand(1));
-  DenseMap<GlobalVariable*, LatticeVal>::iterator I = TrackedGlobals.find(GV);
+  auto I = TrackedGlobals.find(GV);
   if (I == TrackedGlobals.end())
     return;
 
@@ -1098,11 +1089,11 @@ void SCCPSolver::visitLoadInst(LoadInst &I) {
   if (isOverdefined(ValueState[&I]))
     return (void)markOverdefined(&I);
 
-  LatticeVal PtrVal = getValueState(I.getOperand(0));
+  ValueLatticeElement PtrVal = getValueState(I.getOperand(0));
   if (PtrVal.isUnknownOrUndef())
     return; // The pointer is not resolved yet!
 
-  LatticeVal &IV = ValueState[&I];
+  ValueLatticeElement &IV = ValueState[&I];
 
   if (!isConstant(PtrVal) || I.isVolatile())
     return (void)markOverdefined(IV, &I);
@@ -1121,8 +1112,7 @@ void SCCPSolver::visitLoadInst(LoadInst &I) {
   if (auto *GV = dyn_cast<GlobalVariable>(Ptr)) {
     if (!TrackedGlobals.empty()) {
       // If we are tracking this global, merge in the known value for it.
-      DenseMap<GlobalVariable*, LatticeVal>::iterator It =
-        TrackedGlobals.find(GV);
+      auto It = TrackedGlobals.find(GV);
       if (It != TrackedGlobals.end()) {
         mergeInValue(IV, &I, It->second);
         return;
@@ -1164,7 +1154,7 @@ void SCCPSolver::handleCallOverdefined(CallSite CS) {
          ++AI) {
       if (AI->get()->getType()->isStructTy())
         return markOverdefined(I); // Can't handle struct args.
-      LatticeVal State = getValueState(*AI);
+      ValueLatticeElement State = getValueState(*AI);
 
       if (State.isUnknownOrUndef())
         return; // Operands are not resolved yet.
@@ -1214,7 +1204,7 @@ void SCCPSolver::handleCallArguments(CallSite CS) {
 
       if (auto *STy = dyn_cast<StructType>(AI->getType())) {
         for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
-          LatticeVal CallArg = getStructValueState(*CAI, i);
+          ValueLatticeElement CallArg = getStructValueState(*CAI, i);
           mergeInValue(getStructValueState(&*AI, i), &*AI, CallArg);
         }
       } else
@@ -1262,9 +1252,9 @@ void SCCPSolver::handleCallResult(CallSite CS) {
       if (CmpOp0 != CopyOf)
         std::swap(CmpOp0, CmpOp1);
 
-      LatticeVal OriginalVal = getValueState(CopyOf);
-      LatticeVal EqVal = getValueState(CmpOp1);
-      LatticeVal &IV = ValueState[I];
+      ValueLatticeElement OriginalVal = getValueState(CopyOf);
+      ValueLatticeElement EqVal = getValueState(CmpOp1);
+      ValueLatticeElement &IV = ValueState[I];
       if (PBranch->TrueEdge && Cmp->getPredicate() == CmpInst::ICMP_EQ) {
         addAdditionalUser(CmpOp1, I);
         if (isConstant(OriginalVal))
@@ -1303,7 +1293,7 @@ void SCCPSolver::handleCallResult(CallSite CS) {
       mergeInValue(getStructValueState(I, i), I,
                    TrackedMultipleRetVals[std::make_pair(F, i)]);
   } else {
-    MapVector<Function*, LatticeVal>::iterator TFRVI = TrackedRetVals.find(F);
+    auto TFRVI = TrackedRetVals.find(F);
     if (TFRVI == TrackedRetVals.end())
       return handleCallOverdefined(CS); // Not tracking this callee.
 
@@ -1405,14 +1395,14 @@ bool SCCPSolver::ResolvedUndefsIn(Function &F) {
         // Send the results of everything else to overdefined.  We could be
         // more precise than this but it isn't worth bothering.
         for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
-          LatticeVal &LV = getStructValueState(&I, i);
+          ValueLatticeElement &LV = getStructValueState(&I, i);
           if (LV.isUnknownOrUndef())
             markOverdefined(LV, &I);
         }
         continue;
       }
 
-      LatticeVal &LV = getValueState(&I);
+      ValueLatticeElement &LV = getValueState(&I);
       if (!LV.isUnknownOrUndef())
         continue;
 
@@ -1525,20 +1515,21 @@ bool SCCPSolver::ResolvedUndefsIn(Function &F) {
 static bool tryToReplaceWithConstant(SCCPSolver &Solver, Value *V) {
   Constant *Const = nullptr;
   if (V->getType()->isStructTy()) {
-    std::vector<LatticeVal> IVs = Solver.getStructLatticeValueFor(V);
-    if (any_of(IVs, [](const LatticeVal &LV) { return isOverdefined(LV); }))
+    std::vector<ValueLatticeElement> IVs = Solver.getStructLatticeValueFor(V);
+    if (any_of(IVs,
+               [](const ValueLatticeElement &LV) { return isOverdefined(LV); }))
       return false;
     std::vector<Constant *> ConstVals;
     auto *ST = cast<StructType>(V->getType());
     for (unsigned i = 0, e = ST->getNumElements(); i != e; ++i) {
-      LatticeVal V = IVs[i];
+      ValueLatticeElement V = IVs[i];
       ConstVals.push_back(isConstant(V)
                               ? Solver.getConstant(V)
                               : UndefValue::get(ST->getElementType(i)));
     }
     Const = ConstantStruct::get(ST, ConstVals);
   } else {
-    const LatticeVal &IV = Solver.getLatticeValueFor(V);
+    const ValueLatticeElement &IV = Solver.getLatticeValueFor(V);
     if (isOverdefined(IV))
       return false;
 
@@ -1718,9 +1709,10 @@ static void findReturnsToZap(Function &F,
                if (!CallSite(U))
                  return true;
                if (U->getType()->isStructTy()) {
-                 return all_of(
-                     Solver.getStructLatticeValueFor(U),
-                     [](const LatticeVal &LV) { return !isOverdefined(LV); });
+                 return all_of(Solver.getStructLatticeValueFor(U),
+                               [](const ValueLatticeElement &LV) {
+                                 return !isOverdefined(LV);
+                               });
                }
                return !isOverdefined(Solver.getLatticeValueFor(U));
              }) &&
@@ -1962,8 +1954,7 @@ bool llvm::runIPSCCP(
   // whether other functions are optimizable.
   SmallVector<ReturnInst*, 8> ReturnsToZap;
 
-  const MapVector<Function*, LatticeVal> &RV = Solver.getTrackedRetVals();
-  for (const auto &I : RV) {
+  for (const auto &I : Solver.getTrackedRetVals()) {
     Function *F = I.first;
     if (isOverdefined(I.second) || F->getReturnType()->isVoidTy())
       continue;
@@ -1986,11 +1977,9 @@ bool llvm::runIPSCCP(
 
   // If we inferred constant or undef values for globals variables, we can
   // delete the global and any stores that remain to it.
-  const DenseMap<GlobalVariable*, LatticeVal> &TG = Solver.getTrackedGlobals();
-  for (DenseMap<GlobalVariable*, LatticeVal>::const_iterator I = TG.begin(),
-         E = TG.end(); I != E; ++I) {
-    GlobalVariable *GV = I->first;
-    if (isOverdefined(I->second))
+  for (auto &I : make_early_inc_range(Solver.getTrackedGlobals())) {
+    GlobalVariable *GV = I.first;
+    if (isOverdefined(I.second))
       continue;
     LLVM_DEBUG(dbgs() << "Found that GV '" << GV->getName()
                       << "' is constant!\n");
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index 0c51fd5ff4238..05025747db8ef 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -662,7 +662,8 @@ class AllocaSlices::SliceBuilder : public PtrUseVisitor<SliceBuilder> {
 public:
   SliceBuilder(const DataLayout &DL, AllocaInst &AI, AllocaSlices &AS)
       : PtrUseVisitor<SliceBuilder>(DL),
-        AllocSize(DL.getTypeAllocSize(AI.getAllocatedType())), AS(AS) {}
+        AllocSize(DL.getTypeAllocSize(AI.getAllocatedType()).getFixedSize()),
+        AS(AS) {}
 
 private:
   void markAsDead(Instruction &I) {
@@ -751,8 +752,10 @@ class AllocaSlices::SliceBuilder : public PtrUseVisitor<SliceBuilder> {
           // For array or vector indices, scale the index by the size of the
           // type.
           APInt Index = OpC->getValue().sextOrTrunc(Offset.getBitWidth());
-          GEPOffset += Index * APInt(Offset.getBitWidth(),
-                                     DL.getTypeAllocSize(GTI.getIndexedType()));
+          GEPOffset +=
+              Index *
+              APInt(Offset.getBitWidth(),
+                    DL.getTypeAllocSize(GTI.getIndexedType()).getFixedSize());
         }
 
         // If this index has computed an intermediate pointer which is not
@@ -787,7 +790,7 @@ class AllocaSlices::SliceBuilder : public PtrUseVisitor<SliceBuilder> {
         LI.getPointerAddressSpace() != DL.getAllocaAddrSpace())
       return PI.setAborted(&LI);
 
-    uint64_t Size = DL.getTypeStoreSize(LI.getType());
+    uint64_t Size = DL.getTypeStoreSize(LI.getType()).getFixedSize();
     return handleLoadOrStore(LI.getType(), LI, Offset, Size, LI.isVolatile());
   }
 
@@ -802,7 +805,7 @@ class AllocaSlices::SliceBuilder : public PtrUseVisitor<SliceBuilder> {
         SI.getPointerAddressSpace() != DL.getAllocaAddrSpace())
       return PI.setAborted(&SI);
 
-    uint64_t Size = DL.getTypeStoreSize(ValOp->getType());
+    uint64_t Size = DL.getTypeStoreSize(ValOp->getType()).getFixedSize();
 
     // If this memory access can be shown to *statically* extend outside the
     // bounds of the allocation, it's behavior is undefined, so simply
@@ -1220,7 +1223,7 @@ static bool isSafePHIToSpeculate(PHINode &PN) {
       if (BBI->mayWriteToMemory())
         return false;
 
-    uint64_t Size = DL.getTypeStoreSize(LI->getType());
+    uint64_t Size = DL.getTypeStoreSize(LI->getType()).getFixedSize();
     MaxAlign = std::max(MaxAlign, MaybeAlign(LI->getAlignment()));
     MaxSize = MaxSize.ult(Size) ? APInt(APWidth, Size) : MaxSize;
     HaveLoad = true;
@@ -1478,7 +1481,8 @@ static Value *getNaturalGEPRecursively(IRBuilderTy &IRB, const DataLayout &DL,
   // extremely poorly defined currently. The long-term goal is to remove GEPing
   // over a vector from the IR completely.
   if (VectorType *VecTy = dyn_cast<VectorType>(Ty)) {
-    unsigned ElementSizeInBits = DL.getTypeSizeInBits(VecTy->getScalarType());
+    unsigned ElementSizeInBits =
+        DL.getTypeSizeInBits(VecTy->getScalarType()).getFixedSize();
     if (ElementSizeInBits % 8 != 0) {
       // GEPs over non-multiple of 8 size vector elements are invalid.
       return nullptr;
@@ -1495,7 +1499,8 @@ static Value *getNaturalGEPRecursively(IRBuilderTy &IRB, const DataLayout &DL,
 
   if (ArrayType *ArrTy = dyn_cast<ArrayType>(Ty)) {
     Type *ElementTy = ArrTy->getElementType();
-    APInt ElementSize(Offset.getBitWidth(), DL.getTypeAllocSize(ElementTy));
+    APInt ElementSize(Offset.getBitWidth(),
+                      DL.getTypeAllocSize(ElementTy).getFixedSize());
     APInt NumSkippedElements = Offset.sdiv(ElementSize);
     if (NumSkippedElements.ugt(ArrTy->getNumElements()))
       return nullptr;
@@ -1517,7 +1522,7 @@ static Value *getNaturalGEPRecursively(IRBuilderTy &IRB, const DataLayout &DL,
   unsigned Index = SL->getElementContainingOffset(StructOffset);
   Offset -= APInt(Offset.getBitWidth(), SL->getElementOffset(Index));
   Type *ElementTy = STy->getElementType(Index);
-  if (Offset.uge(DL.getTypeAllocSize(ElementTy)))
+  if (Offset.uge(DL.getTypeAllocSize(ElementTy).getFixedSize()))
     return nullptr; // The offset points into alignment padding.
 
   Indices.push_back(IRB.getInt32(Index));
@@ -1549,7 +1554,8 @@ static Value *getNaturalGEPWithOffset(IRBuilderTy &IRB, const DataLayout &DL,
   Type *ElementTy = Ty->getElementType();
   if (!ElementTy->isSized())
     return nullptr; // We can't GEP through an unsized element.
-  APInt ElementSize(Offset.getBitWidth(), DL.getTypeAllocSize(ElementTy));
+  APInt ElementSize(Offset.getBitWidth(),
+                    DL.getTypeAllocSize(ElementTy).getFixedSize());
   if (ElementSize == 0)
     return nullptr; // Zero-length arrays can't help us build a natural GEP.
   APInt NumSkippedElements = Offset.sdiv(ElementSize);
@@ -1716,7 +1722,8 @@ static bool canConvertValue(const DataLayout &DL, Type *OldTy, Type *NewTy) {
     return false;
   }
 
-  if (DL.getTypeSizeInBits(NewTy) != DL.getTypeSizeInBits(OldTy))
+  if (DL.getTypeSizeInBits(NewTy).getFixedSize() !=
+      DL.getTypeSizeInBits(OldTy).getFixedSize())
     return false;
   if (!NewTy->isSingleValueType() || !OldTy->isSingleValueType())
     return false;
@@ -1889,7 +1896,8 @@ static VectorType *isVectorPromotionViable(Partition &P, const DataLayout &DL) {
       // Return if bitcast to vectors is different for total size in bits.
       if (!CandidateTys.empty()) {
         VectorType *V = CandidateTys[0];
-        if (DL.getTypeSizeInBits(VTy) != DL.getTypeSizeInBits(V)) {
+        if (DL.getTypeSizeInBits(VTy).getFixedSize() !=
+            DL.getTypeSizeInBits(V).getFixedSize()) {
           CandidateTys.clear();
           return;
         }
@@ -1935,7 +1943,8 @@ static VectorType *isVectorPromotionViable(Partition &P, const DataLayout &DL) {
     // they're all integer vectors. We sort by ascending number of elements.
     auto RankVectorTypes = [&DL](VectorType *RHSTy, VectorType *LHSTy) {
       (void)DL;
-      assert(DL.getTypeSizeInBits(RHSTy) == DL.getTypeSizeInBits(LHSTy) &&
+      assert(DL.getTypeSizeInBits(RHSTy).getFixedSize() ==
+                 DL.getTypeSizeInBits(LHSTy).getFixedSize() &&
              "Cannot have vector types of different sizes!");
       assert(RHSTy->getElementType()->isIntegerTy() &&
              "All non-integer types eliminated!");
@@ -1963,13 +1972,14 @@ static VectorType *isVectorPromotionViable(Partition &P, const DataLayout &DL) {
 
   // Try each vector type, and return the one which works.
   auto CheckVectorTypeForPromotion = [&](VectorType *VTy) {
-    uint64_t ElementSize = DL.getTypeSizeInBits(VTy->getElementType());
+    uint64_t ElementSize =
+        DL.getTypeSizeInBits(VTy->getElementType()).getFixedSize();
 
     // While the definition of LLVM vectors is bitpacked, we don't support sizes
     // that aren't byte sized.
     if (ElementSize % 8)
       return false;
-    assert((DL.getTypeSizeInBits(VTy) % 8) == 0 &&
+    assert((DL.getTypeSizeInBits(VTy).getFixedSize() % 8) == 0 &&
            "vector size not a multiple of element size?");
     ElementSize /= 8;
 
@@ -1999,7 +2009,7 @@ static bool isIntegerWideningViableForSlice(const Slice &S,
                                             Type *AllocaTy,
                                             const DataLayout &DL,
                                             bool &WholeAllocaOp) {
-  uint64_t Size = DL.getTypeStoreSize(AllocaTy);
+  uint64_t Size = DL.getTypeStoreSize(AllocaTy).getFixedSize();
 
   uint64_t RelBegin = S.beginOffset() - AllocBeginOffset;
   uint64_t RelEnd = S.endOffset() - AllocBeginOffset;
@@ -2015,7 +2025,7 @@ static bool isIntegerWideningViableForSlice(const Slice &S,
     if (LI->isVolatile())
       return false;
     // We can't handle loads that extend past the allocated memory.
-    if (DL.getTypeStoreSize(LI->getType()) > Size)
+    if (DL.getTypeStoreSize(LI->getType()).getFixedSize() > Size)
       return false;
     // So far, AllocaSliceRewriter does not support widening split slice tails
     // in rewriteIntegerLoad.
@@ -2027,7 +2037,7 @@ static bool isIntegerWideningViableForSlice(const Slice &S,
     if (!isa<VectorType>(LI->getType()) && RelBegin == 0 && RelEnd == Size)
       WholeAllocaOp = true;
     if (IntegerType *ITy = dyn_cast<IntegerType>(LI->getType())) {
-      if (ITy->getBitWidth() < DL.getTypeStoreSizeInBits(ITy))
+      if (ITy->getBitWidth() < DL.getTypeStoreSizeInBits(ITy).getFixedSize())
         return false;
     } else if (RelBegin != 0 || RelEnd != Size ||
                !canConvertValue(DL, AllocaTy, LI->getType())) {
@@ -2040,7 +2050,7 @@ static bool isIntegerWideningViableForSlice(const Slice &S,
     if (SI->isVolatile())
       return false;
     // We can't handle stores that extend past the allocated memory.
-    if (DL.getTypeStoreSize(ValueTy) > Size)
+    if (DL.getTypeStoreSize(ValueTy).getFixedSize() > Size)
       return false;
     // So far, AllocaSliceRewriter does not support widening split slice tails
     // in rewriteIntegerStore.
@@ -2052,7 +2062,7 @@ static bool isIntegerWideningViableForSlice(const Slice &S,
     if (!isa<VectorType>(ValueTy) && RelBegin == 0 && RelEnd == Size)
       WholeAllocaOp = true;
     if (IntegerType *ITy = dyn_cast<IntegerType>(ValueTy)) {
-      if (ITy->getBitWidth() < DL.getTypeStoreSizeInBits(ITy))
+      if (ITy->getBitWidth() < DL.getTypeStoreSizeInBits(ITy).getFixedSize())
         return false;
     } else if (RelBegin != 0 || RelEnd != Size ||
                !canConvertValue(DL, ValueTy, AllocaTy)) {
@@ -2083,13 +2093,13 @@ static bool isIntegerWideningViableForSlice(const Slice &S,
 /// promote the resulting alloca.
 static bool isIntegerWideningViable(Partition &P, Type *AllocaTy,
                                     const DataLayout &DL) {
-  uint64_t SizeInBits = DL.getTypeSizeInBits(AllocaTy);
+  uint64_t SizeInBits = DL.getTypeSizeInBits(AllocaTy).getFixedSize();
   // Don't create integer types larger than the maximum bitwidth.
   if (SizeInBits > IntegerType::MAX_INT_BITS)
     return false;
 
   // Don't try to handle allocas with bit-padding.
-  if (SizeInBits != DL.getTypeStoreSizeInBits(AllocaTy))
+  if (SizeInBits != DL.getTypeStoreSizeInBits(AllocaTy).getFixedSize())
     return false;
 
   // We need to ensure that an integer type with the appropriate bitwidth can
@@ -2128,11 +2138,13 @@ static Value *extractInteger(const DataLayout &DL, IRBuilderTy &IRB, Value *V,
                              const Twine &Name) {
   LLVM_DEBUG(dbgs() << "       start: " << *V << "\n");
   IntegerType *IntTy = cast<IntegerType>(V->getType());
-  assert(DL.getTypeStoreSize(Ty) + Offset <= DL.getTypeStoreSize(IntTy) &&
+  assert(DL.getTypeStoreSize(Ty).getFixedSize() + Offset <=
+             DL.getTypeStoreSize(IntTy).getFixedSize() &&
          "Element extends past full value");
   uint64_t ShAmt = 8 * Offset;
   if (DL.isBigEndian())
-    ShAmt = 8 * (DL.getTypeStoreSize(IntTy) - DL.getTypeStoreSize(Ty) - Offset);
+    ShAmt = 8 * (DL.getTypeStoreSize(IntTy).getFixedSize() -
+                 DL.getTypeStoreSize(Ty).getFixedSize() - Offset);
   if (ShAmt) {
     V = IRB.CreateLShr(V, ShAmt, Name + ".shift");
     LLVM_DEBUG(dbgs() << "     shifted: " << *V << "\n");
@@ -2157,11 +2169,13 @@ static Value *insertInteger(const DataLayout &DL, IRBuilderTy &IRB, Value *Old,
     V = IRB.CreateZExt(V, IntTy, Name + ".ext");
     LLVM_DEBUG(dbgs() << "    extended: " << *V << "\n");
   }
-  assert(DL.getTypeStoreSize(Ty) + Offset <= DL.getTypeStoreSize(IntTy) &&
+  assert(DL.getTypeStoreSize(Ty).getFixedSize() + Offset <=
+             DL.getTypeStoreSize(IntTy).getFixedSize() &&
          "Element store outside of alloca store");
   uint64_t ShAmt = 8 * Offset;
   if (DL.isBigEndian())
-    ShAmt = 8 * (DL.getTypeStoreSize(IntTy) - DL.getTypeStoreSize(Ty) - Offset);
+    ShAmt = 8 * (DL.getTypeStoreSize(IntTy).getFixedSize() -
+                 DL.getTypeStoreSize(Ty).getFixedSize() - Offset);
   if (ShAmt) {
     V = IRB.CreateShl(V, ShAmt, Name + ".shift");
     LLVM_DEBUG(dbgs() << "     shifted: " << *V << "\n");
@@ -2324,18 +2338,20 @@ class llvm::sroa::AllocaSliceRewriter
         NewAllocaBeginOffset(NewAllocaBeginOffset),
         NewAllocaEndOffset(NewAllocaEndOffset),
         NewAllocaTy(NewAI.getAllocatedType()),
-        IntTy(IsIntegerPromotable
-                  ? Type::getIntNTy(
-                        NewAI.getContext(),
-                        DL.getTypeSizeInBits(NewAI.getAllocatedType()))
-                  : nullptr),
+        IntTy(
+            IsIntegerPromotable
+                ? Type::getIntNTy(NewAI.getContext(),
+                                  DL.getTypeSizeInBits(NewAI.getAllocatedType())
+                                      .getFixedSize())
+                : nullptr),
         VecTy(PromotableVecTy),
         ElementTy(VecTy ? VecTy->getElementType() : nullptr),
-        ElementSize(VecTy ? DL.getTypeSizeInBits(ElementTy) / 8 : 0),
+        ElementSize(VecTy ? DL.getTypeSizeInBits(ElementTy).getFixedSize() / 8
+                          : 0),
         PHIUsers(PHIUsers), SelectUsers(SelectUsers),
         IRB(NewAI.getContext(), ConstantFolder()) {
     if (VecTy) {
-      assert((DL.getTypeSizeInBits(ElementTy) % 8) == 0 &&
+      assert((DL.getTypeSizeInBits(ElementTy).getFixedSize() % 8) == 0 &&
              "Only multiple-of-8 sized vector elements are viable");
       ++NumVectorized;
     }
@@ -2500,7 +2516,8 @@ class llvm::sroa::AllocaSliceRewriter
 
     Type *TargetTy = IsSplit ? Type::getIntNTy(LI.getContext(), SliceSize * 8)
                              : LI.getType();
-    const bool IsLoadPastEnd = DL.getTypeStoreSize(TargetTy) > SliceSize;
+    const bool IsLoadPastEnd =
+        DL.getTypeStoreSize(TargetTy).getFixedSize() > SliceSize;
     bool IsPtrAdjusted = false;
     Value *V;
     if (VecTy) {
@@ -2568,7 +2585,7 @@ class llvm::sroa::AllocaSliceRewriter
       assert(!LI.isVolatile());
       assert(LI.getType()->isIntegerTy() &&
              "Only integer type loads and stores are split");
-      assert(SliceSize < DL.getTypeStoreSize(LI.getType()) &&
+      assert(SliceSize < DL.getTypeStoreSize(LI.getType()).getFixedSize() &&
              "Split load isn't smaller than original load");
       assert(DL.typeSizeEqualsStoreSize(LI.getType()) &&
              "Non-byte-multiple bit width");
@@ -2626,7 +2643,8 @@ class llvm::sroa::AllocaSliceRewriter
   bool rewriteIntegerStore(Value *V, StoreInst &SI, AAMDNodes AATags) {
     assert(IntTy && "We cannot extract an integer from the alloca");
     assert(!SI.isVolatile());
-    if (DL.getTypeSizeInBits(V->getType()) != IntTy->getBitWidth()) {
+    if (DL.getTypeSizeInBits(V->getType()).getFixedSize() !=
+        IntTy->getBitWidth()) {
       Value *Old = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI,
                                          NewAI.getAlign(), "oldload");
       Old = convertValue(DL, IRB, Old, IntTy);
@@ -2661,7 +2679,7 @@ class llvm::sroa::AllocaSliceRewriter
       if (AllocaInst *AI = dyn_cast<AllocaInst>(V->stripInBoundsOffsets()))
         Pass.PostPromotionWorklist.insert(AI);
 
-    if (SliceSize < DL.getTypeStoreSize(V->getType())) {
+    if (SliceSize < DL.getTypeStoreSize(V->getType()).getFixedSize()) {
       assert(!SI.isVolatile());
       assert(V->getType()->isIntegerTy() &&
              "Only integer type loads and stores are split");
@@ -2677,7 +2695,8 @@ class llvm::sroa::AllocaSliceRewriter
     if (IntTy && V->getType()->isIntegerTy())
       return rewriteIntegerStore(V, SI, AATags);
 
-    const bool IsStorePastEnd = DL.getTypeStoreSize(V->getType()) > SliceSize;
+    const bool IsStorePastEnd =
+        DL.getTypeStoreSize(V->getType()).getFixedSize() > SliceSize;
     StoreInst *NewSI;
     if (NewBeginOffset == NewAllocaBeginOffset &&
         NewEndOffset == NewAllocaEndOffset &&
@@ -2792,7 +2811,7 @@ class llvm::sroa::AllocaSliceRewriter
       auto *Int8Ty = IntegerType::getInt8Ty(NewAI.getContext());
       auto *SrcTy = VectorType::get(Int8Ty, Len);
       return canConvertValue(DL, SrcTy, AllocaTy) &&
-        DL.isLegalInteger(DL.getTypeSizeInBits(ScalarTy));
+             DL.isLegalInteger(DL.getTypeSizeInBits(ScalarTy).getFixedSize());
     }();
 
     // If this doesn't map cleanly onto the alloca type, and that type isn't
@@ -2826,8 +2845,8 @@ class llvm::sroa::AllocaSliceRewriter
       unsigned NumElements = EndIndex - BeginIndex;
       assert(NumElements <= VecTy->getNumElements() && "Too many elements!");
 
-      Value *Splat =
-          getIntegerSplat(II.getValue(), DL.getTypeSizeInBits(ElementTy) / 8);
+      Value *Splat = getIntegerSplat(
+          II.getValue(), DL.getTypeSizeInBits(ElementTy).getFixedSize() / 8);
       Splat = convertValue(DL, IRB, Splat, ElementTy);
       if (NumElements > 1)
         Splat = getVectorSplat(Splat, NumElements);
@@ -2860,7 +2879,8 @@ class llvm::sroa::AllocaSliceRewriter
       assert(NewBeginOffset == NewAllocaBeginOffset);
       assert(NewEndOffset == NewAllocaEndOffset);
 
-      V = getIntegerSplat(II.getValue(), DL.getTypeSizeInBits(ScalarTy) / 8);
+      V = getIntegerSplat(II.getValue(),
+                          DL.getTypeSizeInBits(ScalarTy).getFixedSize() / 8);
       if (VectorType *AllocaVecTy = dyn_cast<VectorType>(AllocaTy))
         V = getVectorSplat(V, AllocaVecTy->getNumElements());
 
@@ -2923,7 +2943,8 @@ class llvm::sroa::AllocaSliceRewriter
     bool EmitMemCpy =
         !VecTy && !IntTy &&
         (BeginOffset > NewAllocaBeginOffset || EndOffset < NewAllocaEndOffset ||
-         SliceSize != DL.getTypeStoreSize(NewAI.getAllocatedType()) ||
+         SliceSize !=
+             DL.getTypeStoreSize(NewAI.getAllocatedType()).getFixedSize() ||
          !NewAI.getAllocatedType()->isSingleValueType());
 
     // If we're just going to emit a memcpy, the alloca hasn't changed, and the
@@ -3469,8 +3490,8 @@ static Type *stripAggregateTypeWrapping(const DataLayout &DL, Type *Ty) {
   if (Ty->isSingleValueType())
     return Ty;
 
-  uint64_t AllocSize = DL.getTypeAllocSize(Ty);
-  uint64_t TypeSize = DL.getTypeSizeInBits(Ty);
+  uint64_t AllocSize = DL.getTypeAllocSize(Ty).getFixedSize();
+  uint64_t TypeSize = DL.getTypeSizeInBits(Ty).getFixedSize();
 
   Type *InnerTy;
   if (ArrayType *ArrTy = dyn_cast<ArrayType>(Ty)) {
@@ -3483,8 +3504,8 @@ static Type *stripAggregateTypeWrapping(const DataLayout &DL, Type *Ty) {
     return Ty;
   }
 
-  if (AllocSize > DL.getTypeAllocSize(InnerTy) ||
-      TypeSize > DL.getTypeSizeInBits(InnerTy))
+  if (AllocSize > DL.getTypeAllocSize(InnerTy).getFixedSize() ||
+      TypeSize > DL.getTypeSizeInBits(InnerTy).getFixedSize())
     return Ty;
 
   return stripAggregateTypeWrapping(DL, InnerTy);
@@ -3505,15 +3526,15 @@ static Type *stripAggregateTypeWrapping(const DataLayout &DL, Type *Ty) {
 /// return a type if necessary.
 static Type *getTypePartition(const DataLayout &DL, Type *Ty, uint64_t Offset,
                               uint64_t Size) {
-  if (Offset == 0 && DL.getTypeAllocSize(Ty) == Size)
+  if (Offset == 0 && DL.getTypeAllocSize(Ty).getFixedSize() == Size)
     return stripAggregateTypeWrapping(DL, Ty);
-  if (Offset > DL.getTypeAllocSize(Ty) ||
-      (DL.getTypeAllocSize(Ty) - Offset) < Size)
+  if (Offset > DL.getTypeAllocSize(Ty).getFixedSize() ||
+      (DL.getTypeAllocSize(Ty).getFixedSize() - Offset) < Size)
     return nullptr;
 
   if (SequentialType *SeqTy = dyn_cast<SequentialType>(Ty)) {
     Type *ElementTy = SeqTy->getElementType();
-    uint64_t ElementSize = DL.getTypeAllocSize(ElementTy);
+    uint64_t ElementSize = DL.getTypeAllocSize(ElementTy).getFixedSize();
     uint64_t NumSkippedElements = Offset / ElementSize;
     if (NumSkippedElements >= SeqTy->getNumElements())
       return nullptr;
@@ -3553,7 +3574,7 @@ static Type *getTypePartition(const DataLayout &DL, Type *Ty, uint64_t Offset,
   Offset -= SL->getElementOffset(Index);
 
   Type *ElementTy = STy->getElementType(Index);
-  uint64_t ElementSize = DL.getTypeAllocSize(ElementTy);
+  uint64_t ElementSize = DL.getTypeAllocSize(ElementTy).getFixedSize();
   if (Offset >= ElementSize)
     return nullptr; // The offset points into alignment padding.
 
@@ -4121,7 +4142,7 @@ AllocaInst *SROA::rewritePartition(AllocaInst &AI, AllocaSlices &AS,
   Type *SliceTy = nullptr;
   const DataLayout &DL = AI.getModule()->getDataLayout();
   if (Type *CommonUseTy = findCommonType(P.begin(), P.end(), P.endOffset()))
-    if (DL.getTypeAllocSize(CommonUseTy) >= P.size())
+    if (DL.getTypeAllocSize(CommonUseTy).getFixedSize() >= P.size())
       SliceTy = CommonUseTy;
   if (!SliceTy)
     if (Type *TypePartitionTy = getTypePartition(DL, AI.getAllocatedType(),
@@ -4133,7 +4154,7 @@ AllocaInst *SROA::rewritePartition(AllocaInst &AI, AllocaSlices &AS,
     SliceTy = Type::getIntNTy(*C, P.size() * 8);
   if (!SliceTy)
     SliceTy = ArrayType::get(Type::getInt8Ty(*C), P.size());
-  assert(DL.getTypeAllocSize(SliceTy) >= P.size());
+  assert(DL.getTypeAllocSize(SliceTy).getFixedSize() >= P.size());
 
   bool IsIntegerPromotable = isIntegerWideningViable(P, SliceTy, DL);
 
@@ -4274,7 +4295,8 @@ bool SROA::splitAlloca(AllocaInst &AI, AllocaSlices &AS) {
   // to be rewritten into a partition.
   bool IsSorted = true;
 
-  uint64_t AllocaSize = DL.getTypeAllocSize(AI.getAllocatedType());
+  uint64_t AllocaSize =
+      DL.getTypeAllocSize(AI.getAllocatedType()).getFixedSize();
   const uint64_t MaxBitVectorSize = 1024;
   if (AllocaSize <= MaxBitVectorSize) {
     // If a byte boundary is included in any load or store, a slice starting or
@@ -4338,7 +4360,8 @@ bool SROA::splitAlloca(AllocaInst &AI, AllocaSlices &AS) {
       Changed = true;
       if (NewAI != &AI) {
         uint64_t SizeOfByte = 8;
-        uint64_t AllocaSize = DL.getTypeSizeInBits(NewAI->getAllocatedType());
+        uint64_t AllocaSize =
+            DL.getTypeSizeInBits(NewAI->getAllocatedType()).getFixedSize();
         // Don't include any padding.
         uint64_t Size = std::min(AllocaSize, P.size() * SizeOfByte);
         Fragments.push_back(Fragment(NewAI, P.beginOffset() * SizeOfByte, Size));
@@ -4358,7 +4381,8 @@ bool SROA::splitAlloca(AllocaInst &AI, AllocaSlices &AS) {
     auto *Expr = DbgDeclares.front()->getExpression();
     auto VarSize = Var->getSizeInBits();
     DIBuilder DIB(*AI.getModule(), /*AllowUnresolved*/ false);
-    uint64_t AllocaSize = DL.getTypeSizeInBits(AI.getAllocatedType());
+    uint64_t AllocaSize =
+        DL.getTypeSizeInBits(AI.getAllocatedType()).getFixedSize();
     for (auto Fragment : Fragments) {
       // Create a fragment expression describing the new partition or reuse AI's
       // expression if there is only one partition.
@@ -4446,8 +4470,10 @@ bool SROA::runOnAlloca(AllocaInst &AI) {
   const DataLayout &DL = AI.getModule()->getDataLayout();
 
   // Skip alloca forms that this analysis can't handle.
-  if (AI.isArrayAllocation() || !AI.getAllocatedType()->isSized() ||
-      DL.getTypeAllocSize(AI.getAllocatedType()) == 0)
+  auto *AT = AI.getAllocatedType();
+  if (AI.isArrayAllocation() || !AT->isSized() ||
+      (isa<VectorType>(AT) && cast<VectorType>(AT)->isScalable()) ||
+      DL.getTypeAllocSize(AT).getFixedSize() == 0)
     return false;
 
   bool Changed = false;
@@ -4567,8 +4593,15 @@ PreservedAnalyses SROA::runImpl(Function &F, DominatorTree &RunDT,
   BasicBlock &EntryBB = F.getEntryBlock();
   for (BasicBlock::iterator I = EntryBB.begin(), E = std::prev(EntryBB.end());
        I != E; ++I) {
-    if (AllocaInst *AI = dyn_cast<AllocaInst>(I))
-      Worklist.insert(AI);
+    if (AllocaInst *AI = dyn_cast<AllocaInst>(I)) {
+      if (isa<VectorType>(AI->getAllocatedType()) &&
+          cast<VectorType>(AI->getAllocatedType())->isScalable()) {
+        if (isAllocaPromotable(AI))
+          PromotableAllocas.push_back(AI);
+      } else {
+        Worklist.insert(AI);
+      }
+    }
   }
 
   bool Changed = false;
diff --git a/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp b/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
index 0eca6704b496e..deff56b9e27ed 100644
--- a/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
+++ b/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
@@ -522,7 +522,7 @@ bool ConstantOffsetExtractor::CanTraceInto(bool SignExtended,
     //   sext(a + b) = sext(a) + sext(b)
     // even if the addition is not marked nsw.
     //
-    // Leveraging this invarient, we can trace into an sext'ed inbound GEP
+    // Leveraging this invariant, we can trace into an sext'ed inbound GEP
     // index if the constant offset is non-negative.
     //
     // Verified in @sext_add in split-gep.ll.
@@ -552,6 +552,9 @@ bool ConstantOffsetExtractor::CanTraceInto(bool SignExtended,
 APInt ConstantOffsetExtractor::findInEitherOperand(BinaryOperator *BO,
                                                    bool SignExtended,
                                                    bool ZeroExtended) {
+  // Save off the current height of the chain, in case we need to restore it.
+  size_t ChainLength = UserChain.size();
+
   // BO being non-negative does not shed light on whether its operands are
   // non-negative. Clear the NonNegative flag here.
   APInt ConstantOffset = find(BO->getOperand(0), SignExtended, ZeroExtended,
@@ -562,12 +565,22 @@ APInt ConstantOffsetExtractor::findInEitherOperand(BinaryOperator *BO,
   // However, such cases are probably already handled by -instcombine,
   // given this pass runs after the standard optimizations.
   if (ConstantOffset != 0) return ConstantOffset;
+
+  // Reset the chain back to where it was when we started exploring this node,
+  // since visiting the LHS didn't pan out.
+  UserChain.resize(ChainLength);
+
   ConstantOffset = find(BO->getOperand(1), SignExtended, ZeroExtended,
                         /* NonNegative */ false);
   // If U is a sub operator, negate the constant offset found in the right
   // operand.
   if (BO->getOpcode() == Instruction::Sub)
     ConstantOffset = -ConstantOffset;
+
+  // If RHS wasn't a suitable candidate either, reset the chain again.
+  if (ConstantOffset == 0)
+    UserChain.resize(ChainLength);
+
   return ConstantOffset;
 }
 
diff --git a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
index 15aad801ecc94..84d8eebd97202 100644
--- a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
+++ b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
@@ -44,6 +44,7 @@
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/Transforms/Utils.h"
+#include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Transforms/Utils/SSAUpdater.h"
 #include <algorithm>
 #include <cassert>
@@ -218,8 +219,6 @@ class StructurizeCFG : public RegionPass {
 
   void analyzeLoops(RegionNode *N);
 
-  Value *invert(Value *Condition);
-
   Value *buildCondition(BranchInst *Term, unsigned Idx, bool Invert);
 
   void gatherPredicates(RegionNode *N);
@@ -405,39 +404,6 @@ void StructurizeCFG::analyzeLoops(RegionNode *N) {
   }
 }
 
-/// Invert the given condition
-Value *StructurizeCFG::invert(Value *Condition) {
-  // First: Check if it's a constant
-  if (Constant *C = dyn_cast<Constant>(Condition))
-    return ConstantExpr::getNot(C);
-
-  // Second: If the condition is already inverted, return the original value
-  Value *NotCondition;
-  if (match(Condition, m_Not(m_Value(NotCondition))))
-    return NotCondition;
-
-  if (Instruction *Inst = dyn_cast<Instruction>(Condition)) {
-    // Third: Check all the users for an invert
-    BasicBlock *Parent = Inst->getParent();
-    for (User *U : Condition->users())
-      if (Instruction *I = dyn_cast<Instruction>(U))
-        if (I->getParent() == Parent && match(I, m_Not(m_Specific(Condition))))
-          return I;
-
-    // Last option: Create a new instruction
-    return BinaryOperator::CreateNot(Condition, "", Parent->getTerminator());
-  }
-
-  if (Argument *Arg = dyn_cast<Argument>(Condition)) {
-    BasicBlock &EntryBlock = Arg->getParent()->getEntryBlock();
-    return BinaryOperator::CreateNot(Condition,
-                                     Arg->getName() + ".inv",
-                                     EntryBlock.getTerminator());
-  }
-
-  llvm_unreachable("Unhandled condition to invert");
-}
-
 /// Build the condition for one edge
 Value *StructurizeCFG::buildCondition(BranchInst *Term, unsigned Idx,
                                       bool Invert) {
@@ -446,7 +412,7 @@ Value *StructurizeCFG::buildCondition(BranchInst *Term, unsigned Idx,
     Cond = Term->getCondition();
 
     if (Idx != (unsigned)Invert)
-      Cond = invert(Cond);
+      Cond = invertCondition(Cond);
   }
   return Cond;
 }
diff --git a/llvm/lib/IR/KnowledgeRetention.cpp b/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp
similarity index 53%
rename from llvm/lib/IR/KnowledgeRetention.cpp
rename to llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp
index aa83d0fd6df89..cd677f7d0fb74 100644
--- a/llvm/lib/IR/KnowledgeRetention.cpp
+++ b/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp
@@ -1,4 +1,4 @@
-//===- KnowledgeRetention.h - utilities to preserve informations *- C++ -*-===//
+//===- AssumeBundleBuilder.cpp - tools to preserve informations -*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,8 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/IR/KnowledgeRetention.h"
+#include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
+#include "llvm/Analysis/AssumeBundleQueries.h"
 #include "llvm/ADT/DenseSet.h"
+#include "llvm/IR/Function.h"
 #include "llvm/IR/InstIterator.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Module.h"
@@ -64,14 +66,6 @@ template <> struct DenseMapInfo<AssumedKnowledge> {
 
 namespace {
 
-/// Index of elements in the operand bundle.
-/// If the element exist it is guaranteed to be what is specified in this enum
-/// but it may not exist.
-enum BundleOpInfoElem {
-  BOIE_WasOn = 0,
-  BOIE_Argument = 1,
-};
-
 /// Deterministically compare OperandBundleDef.
 /// The ordering is:
 /// - by the attribute's name aka operand bundle tag, (doesn't change)
@@ -84,13 +78,13 @@ bool isLowerOpBundle(const OperandBundleDef &LHS, const OperandBundleDef &RHS) {
   auto getTuple = [](const OperandBundleDef &Op) {
     return std::make_tuple(
         Op.getTag(),
-        Op.input_size() <= BOIE_Argument
+        Op.input_size() <= ABA_Argument
             ? 0
-            : cast<ConstantInt>(*(Op.input_begin() + BOIE_Argument))
+            : cast<ConstantInt>(*(Op.input_begin() + ABA_Argument))
                   ->getZExtValue(),
-         Op.input_size() <= BOIE_WasOn
+        Op.input_size() <= ABA_WasOn
             ? StringRef("")
-            : (*(Op.input_begin() + BOIE_WasOn))->getName());
+            : (*(Op.input_begin() + ABA_WasOn))->getName());
   };
   return getTuple(LHS) < getTuple(RHS);
 }
@@ -149,7 +143,7 @@ struct AssumeBuilderState {
       addAttrList(Fn->getAttributes());
   }
 
-  CallInst *build() {
+  IntrinsicInst *build() {
     if (AssumedKnowledgeSet.empty())
       return nullptr;
     Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
@@ -169,13 +163,50 @@ struct AssumeBuilderState {
       OpBundle.push_back(OperandBundleDefT<Value *>(Elem.Name, Args));
     }
     llvm::sort(OpBundle, isLowerOpBundle);
-    return CallInst::Create(
-        FnAssume, ArrayRef<Value *>({ConstantInt::getTrue(C)}), OpBundle);
+    return cast<IntrinsicInst>(CallInst::Create(
+        FnAssume, ArrayRef<Value *>({ConstantInt::getTrue(C)}), OpBundle));
+  }
+
+  void addAttr(Attribute::AttrKind Kind, Value *Val, unsigned Argument = 0) {
+    AssumedKnowledge AK;
+    AK.Name = Attribute::getNameFromAttrKind(Kind).data();
+    AK.WasOn.setPointer(Val);
+    if (Attribute::doesAttrKindHaveArgument(Kind)) {
+      AK.Argument =
+          ConstantInt::get(Type::getInt64Ty(M->getContext()), Argument);
+    } else {
+      AK.Argument = nullptr;
+      assert(Argument == 0 && "there should be no argument");
+    }
+    AssumedKnowledgeSet.insert(AK);
+  };
+
+  void addAccessedPtr(Instruction *MemInst, Value *Pointer, Type *AccType,
+                      MaybeAlign MA) {
+    uint64_t DerefSize = MemInst->getModule()
+                             ->getDataLayout()
+                             .getTypeStoreSize(AccType)
+                             .getKnownMinSize();
+    if (DerefSize != 0) {
+      addAttr(Attribute::Dereferenceable, Pointer, DerefSize);
+      if (!NullPointerIsDefined(MemInst->getFunction(),
+                                Pointer->getType()->getPointerAddressSpace()))
+        addAttr(Attribute::NonNull, Pointer);
+    }
+    if (MA.valueOrOne() > 1)
+      addAttr(Attribute::Alignment, Pointer, MA.valueOrOne().value());
   }
 
-  void addInstruction(const Instruction *I) {
+  void addInstruction(Instruction *I) {
     if (auto *Call = dyn_cast<CallBase>(I))
-      addCall(Call);
+      return addCall(Call);
+    if (auto *Load = dyn_cast<LoadInst>(I))
+      return addAccessedPtr(I, Load->getPointerOperand(), Load->getType(),
+                            Load->getAlign());
+    if (auto *Store = dyn_cast<StoreInst>(I))
+      return addAccessedPtr(I, Store->getPointerOperand(),
+                            Store->getValueOperand()->getType(),
+                            Store->getAlign());
     // TODO: Add support for the other Instructions.
     // TODO: Maybe we should look around and merge with other llvm.assume.
   }
@@ -183,125 +214,23 @@ struct AssumeBuilderState {
 
 } // namespace
 
-CallInst *llvm::BuildAssumeFromInst(const Instruction *I, Module *M) {
+IntrinsicInst *llvm::buildAssumeFromInst(Instruction *I) {
   if (!EnableKnowledgeRetention)
     return nullptr;
-  AssumeBuilderState Builder(M);
+  AssumeBuilderState Builder(I->getModule());
   Builder.addInstruction(I);
   return Builder.build();
 }
 
-static bool BundleHasArguement(const CallBase::BundleOpInfo &BOI,
-                               unsigned Idx) {
-  return BOI.End - BOI.Begin > Idx;
-}
-
-static Value *getValueFromBundleOpInfo(IntrinsicInst &Assume,
-                                const CallBase::BundleOpInfo &BOI,
-                                unsigned Idx) {
-  assert(BundleHasArguement(BOI, Idx) && "index out of range");
-  return (Assume.op_begin() + BOI.Begin + Idx)->get();
-}
-
-bool llvm::hasAttributeInAssume(CallInst &AssumeCI, Value *IsOn,
-                                StringRef AttrName, uint64_t *ArgVal,
-                                AssumeQuery AQR) {
-  assert(isa<IntrinsicInst>(AssumeCI) &&
-         "this function is intended to be used on llvm.assume");
-  IntrinsicInst &Assume = cast<IntrinsicInst>(AssumeCI);
-  assert(Assume.getIntrinsicID() == Intrinsic::assume &&
-         "this function is intended to be used on llvm.assume");
-  assert(Attribute::isExistingAttribute(AttrName) &&
-         "this attribute doesn't exist");
-  assert((ArgVal == nullptr || Attribute::doesAttrKindHaveArgument(
-                                   Attribute::getAttrKindFromName(AttrName))) &&
-         "requested value for an attribute that has no argument");
-  if (Assume.bundle_op_infos().empty())
-    return false;
-
-  auto Loop = [&](auto &&Range) {
-    for (auto &BOI : Range) {
-      if (BOI.Tag->getKey() != AttrName)
-        continue;
-      if (IsOn && (BOI.End - BOI.Begin <= BOIE_WasOn ||
-                   IsOn != getValueFromBundleOpInfo(Assume, BOI, BOIE_WasOn)))
-        continue;
-      if (ArgVal) {
-        assert(BOI.End - BOI.Begin > BOIE_Argument);
-        *ArgVal = cast<ConstantInt>(
-                      getValueFromBundleOpInfo(Assume, BOI, BOIE_Argument))
-                      ->getZExtValue();
-      }
-      return true;
-    }
-    return false;
-  };
-
-  if (AQR == AssumeQuery::Lowest)
-    return Loop(Assume.bundle_op_infos());
-  return Loop(reverse(Assume.bundle_op_infos()));
-}
-
-void llvm::fillMapFromAssume(CallInst &AssumeCI, RetainedKnowledgeMap &Result) {
-  IntrinsicInst &Assume = cast<IntrinsicInst>(AssumeCI);
-  assert(Assume.getIntrinsicID() == Intrinsic::assume &&
-         "this function is intended to be used on llvm.assume");
-  for (auto &Bundles : Assume.bundle_op_infos()) {
-    std::pair<Value *, Attribute::AttrKind> Key{
-        nullptr, Attribute::getAttrKindFromName(Bundles.Tag->getKey())};
-    if (BundleHasArguement(Bundles, BOIE_WasOn))
-      Key.first = getValueFromBundleOpInfo(Assume, Bundles, BOIE_WasOn);
-
-    if (Key.first == nullptr && Key.second == Attribute::None)
-      continue;
-    if (!BundleHasArguement(Bundles, BOIE_Argument)) {
-      Result[Key][&Assume] = {0, 0};
-      continue;
-    }
-    unsigned Val = cast<ConstantInt>(
-                       getValueFromBundleOpInfo(Assume, Bundles, BOIE_Argument))
-                       ->getZExtValue();
-    auto Lookup = Result.find(Key);
-    if (Lookup == Result.end() || !Lookup->second.count(&Assume)) {
-      Result[Key][&Assume] = {Val, Val};
-      continue;
-    }
-    Lookup->second[&Assume].Min = std::min(Val, Lookup->second[&Assume].Min);
-    Lookup->second[&Assume].Max = std::max(Val, Lookup->second[&Assume].Max);
-  }
-}
-
-RetainedKnowledge llvm::getKnowledgeFromOperandInAssume(CallInst &AssumeCI,
-                                                        unsigned Idx) {
-  IntrinsicInst &Assume = cast<IntrinsicInst>(AssumeCI);
-  assert(Assume.getIntrinsicID() == Intrinsic::assume &&
-         "this function is intended to be used on llvm.assume");
-  CallBase::BundleOpInfo BOI = Assume.getBundleOpInfoForOperand(Idx);
-  RetainedKnowledge Result;
-  Result.AttrKind = Attribute::getAttrKindFromName(BOI.Tag->getKey());
-  Result.WasOn = getValueFromBundleOpInfo(Assume, BOI, BOIE_WasOn);
-  if (BOI.End - BOI.Begin > BOIE_Argument)
-    Result.ArgValue =
-        cast<ConstantInt>(getValueFromBundleOpInfo(Assume, BOI, BOIE_Argument))
-            ->getZExtValue();
-
-  return Result;
-}
-
-bool llvm::isAssumeWithEmptyBundle(CallInst &CI) {
-  IntrinsicInst &Assume = cast<IntrinsicInst>(CI);
-  assert(Assume.getIntrinsicID() == Intrinsic::assume &&
-         "this function is intended to be used on llvm.assume");
-  return none_of(Assume.bundle_op_infos(),
-                 [](const CallBase::BundleOpInfo &BOI) {
-                   return BOI.Tag->getKey() != "ignore";
-                 });
+void llvm::salvageKnowledge(Instruction *I) {
+  if (Instruction *Intr = buildAssumeFromInst(I))
+    Intr->insertBefore(I);
 }
 
 PreservedAnalyses AssumeBuilderPass::run(Function &F,
                                          FunctionAnalysisManager &AM) {
   for (Instruction &I : instructions(F))
-    if (Instruction *Assume = BuildAssumeFromInst(&I))
+    if (Instruction *Assume = buildAssumeFromInst(&I))
       Assume->insertBefore(&I);
   return PreservedAnalyses::all();
 }
diff --git a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
index 2fc70bcdcf4b8..d72279316b88d 100644
--- a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
+++ b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
@@ -1102,3 +1102,223 @@ Value *llvm::GetIfCondition(BasicBlock *BB, BasicBlock *&IfTrue,
   }
   return BI->getCondition();
 }
+
+// After creating a control flow hub, the operands of PHINodes in an outgoing
+// block Out no longer match the predecessors of that block. Predecessors of Out
+// that are incoming blocks to the hub are now replaced by just one edge from
+// the hub. To match this new control flow, the corresponding values from each
+// PHINode must now be moved a new PHINode in the first guard block of the hub.
+//
+// This operation cannot be performed with SSAUpdater, because it involves one
+// new use: If the block Out is in the list of Incoming blocks, then the newly
+// created PHI in the Hub will use itself along that edge from Out to Hub.
+static void reconnectPhis(BasicBlock *Out, BasicBlock *GuardBlock,
+                          const SetVector<BasicBlock *> &Incoming,
+                          BasicBlock *FirstGuardBlock) {
+  auto I = Out->begin();
+  while (I != Out->end() && isa<PHINode>(I)) {
+    auto Phi = cast<PHINode>(I);
+    auto NewPhi =
+        PHINode::Create(Phi->getType(), Incoming.size(),
+                        Phi->getName() + ".moved", &FirstGuardBlock->back());
+    for (auto In : Incoming) {
+      Value *V = UndefValue::get(Phi->getType());
+      if (In == Out) {
+        V = NewPhi;
+      } else if (Phi->getBasicBlockIndex(In) != -1) {
+        V = Phi->removeIncomingValue(In, false);
+      }
+      NewPhi->addIncoming(V, In);
+    }
+    assert(NewPhi->getNumIncomingValues() == Incoming.size());
+    if (Phi->getNumOperands() == 0) {
+      Phi->replaceAllUsesWith(NewPhi);
+      I = Phi->eraseFromParent();
+      continue;
+    }
+    Phi->addIncoming(NewPhi, GuardBlock);
+    ++I;
+  }
+}
+
+using BBPredicates = DenseMap<BasicBlock *, PHINode *>;
+using BBSetVector = SetVector<BasicBlock *>;
+
+// Collect predicates for each outgoing block. If control reaches the
+// Hub from an incoming block InBB, then the predicate for each
+// outgoing block OutBB decides whether control is forwarded to OutBB.
+//
+// These predicates are not orthogonal. The Hub evaluates them in the
+// same order as the Outgoing set-vector, and control branches to the
+// first outgoing block whose predicate evaluates to true.
+static void createGuardPredicates(BasicBlock *FirstGuardBlock,
+                                  BBPredicates &GuardPredicates,
+                                  SmallVectorImpl<WeakVH> &DeletionCandidates,
+                                  const BBSetVector &Incoming,
+                                  const BBSetVector &Outgoing) {
+  auto &Context = Incoming.front()->getContext();
+  auto BoolTrue = ConstantInt::getTrue(Context);
+  auto BoolFalse = ConstantInt::getFalse(Context);
+
+  // The predicate for the last outgoing is trivially true, and so we
+  // process only the first N-1 successors.
+  for (int i = 0, e = Outgoing.size() - 1; i != e; ++i) {
+    auto Out = Outgoing[i];
+    LLVM_DEBUG(dbgs() << "Creating guard for " << Out->getName() << "\n");
+    auto Phi =
+        PHINode::Create(Type::getInt1Ty(Context), Incoming.size(),
+                        StringRef("Guard.") + Out->getName(), FirstGuardBlock);
+    GuardPredicates[Out] = Phi;
+  }
+
+  for (auto In : Incoming) {
+    auto Branch = cast<BranchInst>(In->getTerminator());
+    BasicBlock *Succ0 = Branch->getSuccessor(0);
+    BasicBlock *Succ1 = nullptr;
+
+    Succ0 = Outgoing.count(Succ0) ? Succ0 : nullptr;
+
+    if (Branch->isUnconditional()) {
+      Branch->setSuccessor(0, FirstGuardBlock);
+      assert(Succ0);
+    } else {
+      Succ1 = Branch->getSuccessor(1);
+      Succ1 = Outgoing.count(Succ1) ? Succ1 : nullptr;
+      assert(Succ0 || Succ1);
+      if (Succ0 && !Succ1) {
+        Branch->setSuccessor(0, FirstGuardBlock);
+      } else if (Succ1 && !Succ0) {
+        Branch->setSuccessor(1, FirstGuardBlock);
+      } else {
+        Branch->eraseFromParent();
+        BranchInst::Create(FirstGuardBlock, In);
+      }
+    }
+
+    assert(Succ0 || Succ1);
+
+    // Optimization: Consider an incoming block A with both successors
+    // Succ0 and Succ1 in the set of outgoing blocks. The predicates
+    // for Succ0 and Succ1 complement each other. If Succ0 is visited
+    // first in the loop below, control will branch to Succ0 using the
+    // corresponding predicate. But if that branch is not taken, then
+    // control must reach Succ1, which means that the predicate for
+    // Succ1 is always true.
+    bool OneSuccessorDone = false;
+    auto Condition = Branch->getCondition();
+    for (int i = 0, e = Outgoing.size() - 1; i != e; ++i) {
+      auto Out = Outgoing[i];
+      auto Phi = GuardPredicates[Out];
+      if (Out != Succ0 && Out != Succ1) {
+        Phi->addIncoming(BoolFalse, In);
+        continue;
+      }
+      // Optimization: When only one successor is an outgoing block,
+      // the predicate is always true.
+      if (!Succ0 || !Succ1 || OneSuccessorDone) {
+        Phi->addIncoming(BoolTrue, In);
+        continue;
+      }
+      assert(Succ0 && Succ1);
+      OneSuccessorDone = true;
+      if (Out == Succ0) {
+        Phi->addIncoming(Condition, In);
+        continue;
+      }
+      auto Inverted = invertCondition(Condition);
+      DeletionCandidates.push_back(Condition);
+      Phi->addIncoming(Inverted, In);
+    }
+  }
+}
+
+// For each outgoing block OutBB, create a guard block in the Hub. The
+// first guard block was already created outside, and available as the
+// first element in the vector of guard blocks.
+//
+// Each guard block terminates in a conditional branch that transfers
+// control to the corresponding outgoing block or the next guard
+// block. The last guard block has two outgoing blocks as successors
+// since the condition for the final outgoing block is trivially
+// true. So we create one less block (including the first guard block)
+// than the number of outgoing blocks.
+static void createGuardBlocks(SmallVectorImpl<BasicBlock *> &GuardBlocks,
+                              Function *F, const BBSetVector &Outgoing,
+                              BBPredicates &GuardPredicates, StringRef Prefix) {
+  for (int i = 0, e = Outgoing.size() - 2; i != e; ++i) {
+    GuardBlocks.push_back(
+        BasicBlock::Create(F->getContext(), Prefix + ".guard", F));
+  }
+  assert(GuardBlocks.size() == GuardPredicates.size());
+
+  // To help keep the loop simple, temporarily append the last
+  // outgoing block to the list of guard blocks.
+  GuardBlocks.push_back(Outgoing.back());
+
+  for (int i = 0, e = GuardBlocks.size() - 1; i != e; ++i) {
+    auto Out = Outgoing[i];
+    assert(GuardPredicates.count(Out));
+    BranchInst::Create(Out, GuardBlocks[i + 1], GuardPredicates[Out],
+                       GuardBlocks[i]);
+  }
+
+  // Remove the last block from the guard list.
+  GuardBlocks.pop_back();
+}
+
+BasicBlock *llvm::CreateControlFlowHub(
+    DomTreeUpdater *DTU, SmallVectorImpl<BasicBlock *> &GuardBlocks,
+    const BBSetVector &Incoming, const BBSetVector &Outgoing,
+    const StringRef Prefix) {
+  auto F = Incoming.front()->getParent();
+  auto FirstGuardBlock =
+      BasicBlock::Create(F->getContext(), Prefix + ".guard", F);
+
+  SmallVector<DominatorTree::UpdateType, 16> Updates;
+  if (DTU) {
+    for (auto In : Incoming) {
+      for (auto Succ : successors(In)) {
+        if (Outgoing.count(Succ))
+          Updates.push_back({DominatorTree::Delete, In, Succ});
+      }
+      Updates.push_back({DominatorTree::Insert, In, FirstGuardBlock});
+    }
+  }
+
+  BBPredicates GuardPredicates;
+  SmallVector<WeakVH, 8> DeletionCandidates;
+  createGuardPredicates(FirstGuardBlock, GuardPredicates, DeletionCandidates,
+                        Incoming, Outgoing);
+
+  GuardBlocks.push_back(FirstGuardBlock);
+  createGuardBlocks(GuardBlocks, F, Outgoing, GuardPredicates, Prefix);
+
+  // Update the PHINodes in each outgoing block to match the new control flow.
+  for (int i = 0, e = GuardBlocks.size(); i != e; ++i) {
+    reconnectPhis(Outgoing[i], GuardBlocks[i], Incoming, FirstGuardBlock);
+  }
+  reconnectPhis(Outgoing.back(), GuardBlocks.back(), Incoming, FirstGuardBlock);
+
+  if (DTU) {
+    int NumGuards = GuardBlocks.size();
+    assert((int)Outgoing.size() == NumGuards + 1);
+    for (int i = 0; i != NumGuards - 1; ++i) {
+      Updates.push_back({DominatorTree::Insert, GuardBlocks[i], Outgoing[i]});
+      Updates.push_back(
+          {DominatorTree::Insert, GuardBlocks[i], GuardBlocks[i + 1]});
+    }
+    Updates.push_back({DominatorTree::Insert, GuardBlocks[NumGuards - 1],
+                       Outgoing[NumGuards - 1]});
+    Updates.push_back({DominatorTree::Insert, GuardBlocks[NumGuards - 1],
+                       Outgoing[NumGuards]});
+    DTU->applyUpdates(Updates);
+  }
+
+  for (auto I : DeletionCandidates) {
+    if (I->use_empty())
+      if (auto Inst = dyn_cast_or_null<Instruction>(I))
+        Inst->eraseFromParent();
+  }
+
+  return FirstGuardBlock;
+}
diff --git a/llvm/lib/Transforms/Utils/BuildLibCalls.cpp b/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
index a385b08d17a98..c64ad147fdfec 100644
--- a/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
@@ -378,6 +378,10 @@ bool llvm::inferLibFuncAttributes(Function &F, const TargetLibraryInfo &TLI) {
     Changed |= setDoesNotCapture(F, 1);
     Changed |= setOnlyReadsMemory(F, 1);
     return Changed;
+  case LibFunc_aligned_alloc:
+    Changed |= setDoesNotThrow(F);
+    Changed |= setRetDoesNotAlias(F);
+    return Changed;
   case LibFunc_bcopy:
     Changed |= setDoesNotThrow(F);
     Changed |= setDoesNotCapture(F, 0);
diff --git a/llvm/lib/Transforms/Utils/CMakeLists.txt b/llvm/lib/Transforms/Utils/CMakeLists.txt
index 56e5071b2e3f3..7d9fcb566dd3c 100644
--- a/llvm/lib/Transforms/Utils/CMakeLists.txt
+++ b/llvm/lib/Transforms/Utils/CMakeLists.txt
@@ -1,7 +1,8 @@
 add_llvm_component_library(LLVMTransformUtils
+  AddDiscriminators.cpp
   AMDGPUEmitPrintf.cpp
   ASanStackFrameLayout.cpp
-  AddDiscriminators.cpp
+  AssumeBundleBuilder.cpp
   BasicBlockUtils.cpp
   BreakCriticalEdges.cpp
   BuildLibCalls.cpp
@@ -62,6 +63,7 @@ add_llvm_component_library(LLVMTransformUtils
   StripNonLineTableDebugInfo.cpp
   SymbolRewriter.cpp
   UnifyFunctionExitNodes.cpp
+  UnifyLoopExits.cpp
   Utils.cpp
   ValueMapper.cpp
   VNCoercion.cpp
diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp
index 3c921060070c6..5fe651f611555 100644
--- a/llvm/lib/Transforms/Utils/InlineFunction.cpp
+++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp
@@ -50,7 +50,6 @@
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
-#include "llvm/IR/KnowledgeRetention.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/Metadata.h"
@@ -61,6 +60,7 @@
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
 #include "llvm/Transforms/Utils/Cloning.h"
 #include "llvm/Transforms/Utils/ValueMapper.h"
 #include <algorithm>
@@ -85,6 +85,16 @@ PreserveAlignmentAssumptions("preserve-alignment-assumptions-during-inlining",
   cl::init(true), cl::Hidden,
   cl::desc("Convert align attributes to assumptions during inlining."));
 
+static cl::opt<bool> UpdateReturnAttributes(
+        "update-return-attrs", cl::init(true), cl::Hidden,
+            cl::desc("Update return attributes on calls within inlined body"));
+
+static cl::opt<unsigned> InlinerAttributeWindow(
+    "max-inst-checked-for-throw-during-inlining", cl::Hidden,
+    cl::desc("the maximum number of instructions analyzed for may throw during "
+             "attribute inference in inlined body"),
+    cl::init(4));
+
 llvm::InlineResult llvm::InlineFunction(CallBase *CB, InlineFunctionInfo &IFI,
                                         AAResults *CalleeAAR,
                                         bool InsertLifetime) {
@@ -1136,6 +1146,97 @@ static void AddAliasScopeMetadata(CallSite CS, ValueToValueMapTy &VMap,
   }
 }
 
+static bool MayContainThrowingOrExitingCall(Instruction *Begin,
+                                            Instruction *End) {
+
+  assert(Begin->getParent() == End->getParent() &&
+         "Expected to be in same basic block!");
+  unsigned NumInstChecked = 0;
+  // Check that all instructions in the range [Begin, End) are guaranteed to
+  // transfer execution to successor.
+  for (auto &I : make_range(Begin->getIterator(), End->getIterator()))
+    if (NumInstChecked++ > InlinerAttributeWindow ||
+        !isGuaranteedToTransferExecutionToSuccessor(&I))
+      return true;
+  return false;
+}
+
+static AttrBuilder IdentifyValidAttributes(CallSite CS) {
+
+  AttrBuilder AB(CS.getAttributes(), AttributeList::ReturnIndex);
+  if (AB.empty())
+    return AB;
+  AttrBuilder Valid;
+  // Only allow these white listed attributes to be propagated back to the
+  // callee. This is because other attributes may only be valid on the call
+  // itself, i.e. attributes such as signext and zeroext.
+  if (auto DerefBytes = AB.getDereferenceableBytes())
+    Valid.addDereferenceableAttr(DerefBytes);
+  if (auto DerefOrNullBytes = AB.getDereferenceableOrNullBytes())
+    Valid.addDereferenceableOrNullAttr(DerefOrNullBytes);
+  if (AB.contains(Attribute::NoAlias))
+    Valid.addAttribute(Attribute::NoAlias);
+  if (AB.contains(Attribute::NonNull))
+    Valid.addAttribute(Attribute::NonNull);
+  return Valid;
+}
+
+static void AddReturnAttributes(CallSite CS, ValueToValueMapTy &VMap) {
+  if (!UpdateReturnAttributes)
+    return;
+
+  AttrBuilder Valid = IdentifyValidAttributes(CS);
+  if (Valid.empty())
+    return;
+  auto *CalledFunction = CS.getCalledFunction();
+  auto &Context = CalledFunction->getContext();
+
+  for (auto &BB : *CalledFunction) {
+    auto *RI = dyn_cast<ReturnInst>(BB.getTerminator());
+    if (!RI || !isa<CallBase>(RI->getOperand(0)))
+      continue;
+    auto *RetVal = cast<CallBase>(RI->getOperand(0));
+    // Sanity check that the cloned RetVal exists and is a call, otherwise we
+    // cannot add the attributes on the cloned RetVal.
+    // Simplification during inlining could have transformed the cloned
+    // instruction.
+    auto *NewRetVal = dyn_cast_or_null<CallBase>(VMap.lookup(RetVal));
+    if (!NewRetVal)
+      continue;
+    // Backward propagation of attributes to the returned value may be incorrect
+    // if it is control flow dependent.
+    // Consider:
+    // @callee {
+    //  %rv = call @foo()
+    //  %rv2 = call @bar()
+    //  if (%rv2 != null)
+    //    return %rv2
+    //  if (%rv == null)
+    //    exit()
+    //  return %rv
+    // }
+    // caller() {
+    //   %val = call nonnull @callee()
+    // }
+    // Here we cannot add the nonnull attribute on either foo or bar. So, we
+    // limit the check to both RetVal and RI are in the same basic block and
+    // there are no throwing/exiting instructions between these instructions.
+    if (RI->getParent() != RetVal->getParent() ||
+        MayContainThrowingOrExitingCall(RetVal, RI))
+      continue;
+    // Add to the existing attributes of NewRetVal, i.e. the cloned call
+    // instruction.
+    // NB! When we have the same attribute already existing on NewRetVal, but
+    // with a differing value, the AttributeList's merge API honours the already
+    // existing attribute value (i.e. attributes such as dereferenceable,
+    // dereferenceable_or_null etc). See AttrBuilder::merge for more details.
+    AttributeList AL = NewRetVal->getAttributes();
+    AttributeList NewAL =
+        AL.addAttributes(Context, AttributeList::ReturnIndex, Valid);
+    NewRetVal->setAttributes(NewAL);
+  }
+}
+
 /// If the inlined function has non-byval align arguments, then
 /// add @llvm.assume-based alignment assumptions to preserve this information.
 static void AddAlignmentAssumptions(CallSite CS, InlineFunctionInfo &IFI) {
@@ -1703,7 +1804,7 @@ llvm::InlineResult llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI,
     AddAlignmentAssumptions(CS, IFI);
 
     /// Preserve all attributes on of the call and its parameters.
-    if (Instruction *Assume = BuildAssumeFromInst(CS.getInstruction()))
+    if (Instruction *Assume = buildAssumeFromInst(CS.getInstruction()))
       Assume->insertBefore(CS.getInstruction());
 
     // We want the inliner to prune the code as it copies.  We would LOVE to
@@ -1801,6 +1902,10 @@ llvm::InlineResult llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI,
     // Add noalias metadata if necessary.
     AddAliasScopeMetadata(CS, VMap, DL, CalleeAAR);
 
+    // Clone return attributes on the callsite into the calls within the inlined
+    // function which feed into its return value.
+    AddReturnAttributes(CS, VMap);
+
     // Propagate llvm.mem.parallel_loop_access if necessary.
     PropagateParallelLoopAccessMetadata(CS, VMap);
 
diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index d5d9dff75ef04..e3c9a3bc87c0c 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -25,6 +25,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/TinyPtrVector.h"
+#include "llvm/Analysis/AssumeBundleQueries.h"
 #include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/EHPersonalities.h"
@@ -59,7 +60,6 @@
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
-#include "llvm/IR/KnowledgeRetention.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/Metadata.h"
@@ -3031,3 +3031,42 @@ AllocaInst *llvm::findAllocaForValue(Value *V,
     AllocaForValue[V] = Res;
   return Res;
 }
+
+Value *llvm::invertCondition(Value *Condition) {
+  // First: Check if it's a constant
+  if (Constant *C = dyn_cast<Constant>(Condition))
+    return ConstantExpr::getNot(C);
+
+  // Second: If the condition is already inverted, return the original value
+  Value *NotCondition;
+  if (match(Condition, m_Not(m_Value(NotCondition))))
+    return NotCondition;
+
+  if (Instruction *Inst = dyn_cast<Instruction>(Condition)) {
+    // Third: Check all the users for an invert
+    BasicBlock *Parent = Inst->getParent();
+    for (User *U : Condition->users())
+      if (Instruction *I = dyn_cast<Instruction>(U))
+        if (I->getParent() == Parent && match(I, m_Not(m_Specific(Condition))))
+          return I;
+
+    // Last option: Create a new instruction
+    auto Inverted = BinaryOperator::CreateNot(Inst, "");
+    if (isa<PHINode>(Inst)) {
+      // FIXME: This fails if the inversion is to be used in a
+      // subsequent PHINode in the same basic block.
+      Inverted->insertBefore(&*Parent->getFirstInsertionPt());
+    } else {
+      Inverted->insertAfter(Inst);
+    }
+    return Inverted;
+  }
+
+  if (Argument *Arg = dyn_cast<Argument>(Condition)) {
+    BasicBlock &EntryBlock = Arg->getParent()->getEntryBlock();
+    return BinaryOperator::CreateNot(Condition, Arg->getName() + ".inv",
+                                     &*EntryBlock.getFirstInsertionPt());
+  }
+
+  llvm_unreachable("Unhandled condition to invert");
+}
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index b86a67faf6465..73e5ac4271a18 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -1359,16 +1359,15 @@ int llvm::rewriteLoopExitValues(Loop *L, LoopInfo *LI, TargetLibraryInfo *TLI,
 
         // Computing the value outside of the loop brings no benefit if it is
         // definitely used inside the loop in a way which can not be optimized
-        // away. Avoid doing so unless either we know we have a value
-        // which computes the ExitValue already, or it is cheap to do so.
-        // TODO: This should be merged into SCEV expander to leverage
-        // its knowledge of existing expressions.
-        bool HighCost = Rewriter.isHighCostExpansion(
-            ExitValue, L, SCEVCheapExpansionBudget, TTI, Inst);
-        if (ReplaceExitValue != AlwaysRepl && HighCost &&
-            hasHardUserWithinLoop(L, Inst))
+        // away. Avoid doing so unless we know we have a value which computes
+        // the ExitValue already. TODO: This should be merged into SCEV
+        // expander to leverage its knowledge of existing expressions.
+        if (ReplaceExitValue != AlwaysRepl && !isa<SCEVConstant>(ExitValue) &&
+            !isa<SCEVUnknown>(ExitValue) && hasHardUserWithinLoop(L, Inst))
           continue;
 
+        bool HighCost = Rewriter.isHighCostExpansion(
+            ExitValue, L, SCEVCheapExpansionBudget, TTI, Inst);
         Value *ExitVal = Rewriter.expandCodeFor(ExitValue, PN->getType(), Inst);
 
         LLVM_DEBUG(dbgs() << "rewriteLoopExitValues: AfterLoopVal = "
diff --git a/llvm/lib/Transforms/Utils/UnifyLoopExits.cpp b/llvm/lib/Transforms/Utils/UnifyLoopExits.cpp
new file mode 100644
index 0000000000000..15c951fb18439
--- /dev/null
+++ b/llvm/lib/Transforms/Utils/UnifyLoopExits.cpp
@@ -0,0 +1,220 @@
+//===- UnifyLoopExits.cpp - Redirect exiting edges to one block -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// For each natural loop with multiple exit blocks, this pass creates a new
+// block N such that all exiting blocks now branch to N, and then control flow
+// is redistributed to all the original exit blocks.
+//
+// Limitation: This assumes that all terminators in the CFG are direct branches
+//             (the "br" instruction). The presence of any other control flow
+//             such as indirectbr, switch or callbr will cause an assert.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Transforms/Utils.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+
+#define DEBUG_TYPE "unify-loop-exits"
+
+using namespace llvm;
+
+namespace {
+struct UnifyLoopExits : public FunctionPass {
+  static char ID;
+  UnifyLoopExits() : FunctionPass(ID) {
+    initializeUnifyLoopExitsPass(*PassRegistry::getPassRegistry());
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const {
+    AU.addRequiredID(LowerSwitchID);
+    AU.addRequired<LoopInfoWrapperPass>();
+    AU.addRequired<DominatorTreeWrapperPass>();
+    AU.addPreservedID(LowerSwitchID);
+    AU.addPreserved<LoopInfoWrapperPass>();
+    AU.addPreserved<DominatorTreeWrapperPass>();
+  }
+
+  bool runOnFunction(Function &F);
+};
+} // namespace
+
+char UnifyLoopExits::ID = 0;
+
+FunctionPass *llvm::createUnifyLoopExitsPass() { return new UnifyLoopExits(); }
+
+INITIALIZE_PASS_BEGIN(UnifyLoopExits, "unify-loop-exits",
+                      "Fixup each natural loop to have a single exit block",
+                      false /* Only looks at CFG */, false /* Analysis Pass */)
+INITIALIZE_PASS_DEPENDENCY(LowerSwitch)
+INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
+INITIALIZE_PASS_END(UnifyLoopExits, "unify-loop-exits",
+                    "Fixup each natural loop to have a single exit block",
+                    false /* Only looks at CFG */, false /* Analysis Pass */)
+
+// The current transform introduces new control flow paths which may break the
+// SSA requirement that every def must dominate all its uses. For example,
+// consider a value D defined inside the loop that is used by some instruction
+// U outside the loop. It follows that D dominates U, since the original
+// program has valid SSA form. After merging the exits, all paths from D to U
+// now flow through the unified exit block. In addition, there may be other
+// paths that do not pass through D, but now reach the unified exit
+// block. Thus, D no longer dominates U.
+//
+// Restore the dominance by creating a phi for each such D at the new unified
+// loop exit. But when doing this, ignore any uses U that are in the new unified
+// loop exit, since those were introduced specially when the block was created.
+//
+// The use of SSAUpdater seems like overkill for this operation. The location
+// for creating the new PHI is well-known, and also the set of incoming blocks
+// to the new PHI.
+static void restoreSSA(const DominatorTree &DT, const Loop *L,
+                       const SetVector<BasicBlock *> &Incoming,
+                       BasicBlock *LoopExitBlock) {
+  using InstVector = SmallVector<Instruction *, 8>;
+  using IIMap = DenseMap<Instruction *, InstVector>;
+  IIMap ExternalUsers;
+  for (auto BB : L->blocks()) {
+    for (auto &I : *BB) {
+      for (auto &U : I.uses()) {
+        auto UserInst = cast<Instruction>(U.getUser());
+        auto UserBlock = UserInst->getParent();
+        if (UserBlock == LoopExitBlock)
+          continue;
+        if (L->contains(UserBlock))
+          continue;
+        LLVM_DEBUG(dbgs() << "added ext use for " << I.getName() << "("
+                          << BB->getName() << ")"
+                          << ": " << UserInst->getName() << "("
+                          << UserBlock->getName() << ")"
+                          << "\n");
+        ExternalUsers[&I].push_back(UserInst);
+      }
+    }
+  }
+
+  for (auto II : ExternalUsers) {
+    // For each Def used outside the loop, create NewPhi in
+    // LoopExitBlock. NewPhi receives Def only along exiting blocks that
+    // dominate it, while the remaining values are undefined since those paths
+    // didn't exist in the original CFG.
+    auto Def = II.first;
+    LLVM_DEBUG(dbgs() << "externally used: " << Def->getName() << "\n");
+    auto NewPhi = PHINode::Create(Def->getType(), Incoming.size(),
+                                  Def->getName() + ".moved",
+                                  LoopExitBlock->getTerminator());
+    for (auto In : Incoming) {
+      LLVM_DEBUG(dbgs() << "predecessor " << In->getName() << ": ");
+      if (Def->getParent() == In || DT.dominates(Def, In)) {
+        LLVM_DEBUG(dbgs() << "dominated\n");
+        NewPhi->addIncoming(Def, In);
+      } else {
+        LLVM_DEBUG(dbgs() << "not dominated\n");
+        NewPhi->addIncoming(UndefValue::get(Def->getType()), In);
+      }
+    }
+
+    LLVM_DEBUG(dbgs() << "external users:");
+    for (auto U : II.second) {
+      LLVM_DEBUG(dbgs() << " " << U->getName());
+      U->replaceUsesOfWith(Def, NewPhi);
+    }
+    LLVM_DEBUG(dbgs() << "\n");
+  }
+}
+
+static bool unifyLoopExits(DominatorTree &DT, LoopInfo &LI, Loop *L) {
+  // To unify the loop exits, we need a list of the exiting blocks as
+  // well as exit blocks. The functions for locating these lists both
+  // traverse the entire loop body. It is more efficient to first
+  // locate the exiting blocks and then examine their successors to
+  // locate the exit blocks.
+  SetVector<BasicBlock *> ExitingBlocks;
+  SetVector<BasicBlock *> Exits;
+
+  // We need SetVectors, but the Loop API takes a vector, so we use a temporary.
+  SmallVector<BasicBlock *, 8> Temp;
+  L->getExitingBlocks(Temp);
+  for (auto BB : Temp) {
+    ExitingBlocks.insert(BB);
+    for (auto S : successors(BB)) {
+      auto SL = LI.getLoopFor(S);
+      // A successor is not an exit if it is directly or indirectly in the
+      // current loop.
+      if (SL == L || L->contains(SL))
+        continue;
+      Exits.insert(S);
+    }
+  }
+
+  LLVM_DEBUG(
+      dbgs() << "Found exit blocks:";
+      for (auto Exit : Exits) {
+        dbgs() << " " << Exit->getName();
+      }
+      dbgs() << "\n";
+
+      dbgs() << "Found exiting blocks:";
+      for (auto EB : ExitingBlocks) {
+        dbgs() << " " << EB->getName();
+      }
+      dbgs() << "\n";);
+
+  if (Exits.size() <= 1) {
+    LLVM_DEBUG(dbgs() << "loop does not have multiple exits; nothing to do\n");
+    return false;
+  }
+
+  SmallVector<BasicBlock *, 8> GuardBlocks;
+  DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);
+  auto LoopExitBlock = CreateControlFlowHub(&DTU, GuardBlocks, ExitingBlocks,
+                                            Exits, "loop.exit");
+
+  restoreSSA(DT, L, ExitingBlocks, LoopExitBlock);
+
+#if defined(EXPENSIVE_CHECKS)
+  assert(DT.verify(DominatorTree::VerificationLevel::Full));
+#else
+  assert(DT.verify(DominatorTree::VerificationLevel::Full));
+#endif // EXPENSIVE_CHECKS
+  L->verifyLoop();
+
+  // The guard blocks were created outside the loop, so they need to become
+  // members of the parent loop.
+  if (auto ParentLoop = L->getParentLoop()) {
+    for (auto G : GuardBlocks) {
+      ParentLoop->addBasicBlockToLoop(G, LI);
+    }
+    ParentLoop->verifyLoop();
+  }
+
+#if defined(EXPENSIVE_CHECKS)
+  LI.verify(DT);
+#endif // EXPENSIVE_CHECKS
+
+  return true;
+}
+
+bool UnifyLoopExits::runOnFunction(Function &F) {
+  LLVM_DEBUG(dbgs() << "===== Unifying loop exits in function " << F.getName()
+                    << "\n");
+  auto &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
+  auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
+
+  bool Changed = false;
+  auto Loops = LI.getLoopsInPreorder();
+  for (auto L : Loops) {
+    LLVM_DEBUG(dbgs() << "Loop: " << L->getHeader()->getName() << " (depth: "
+                      << LI.getLoopDepth(L->getHeader()) << ")\n");
+    Changed |= unifyLoopExits(DT, LI, L);
+  }
+  return Changed;
+}
diff --git a/llvm/lib/Transforms/Utils/Utils.cpp b/llvm/lib/Transforms/Utils/Utils.cpp
index 7769c7493cdab..363110ab8a7d0 100644
--- a/llvm/lib/Transforms/Utils/Utils.cpp
+++ b/llvm/lib/Transforms/Utils/Utils.cpp
@@ -40,6 +40,7 @@ void llvm::initializeTransformUtils(PassRegistry &Registry) {
   initializeStripGCRelocatesPass(Registry);
   initializePredicateInfoPrinterLegacyPassPass(Registry);
   initializeInjectTLIMappingsLegacyPass(Registry);
+  initializeUnifyLoopExitsPass(Registry);
 }
 
 /// LLVMInitializeTransformUtils - C binding for initializeTransformUtilsPasses.
diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
index b0c1a296aace4..d5c3acc4eedf5 100644
--- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
@@ -128,16 +128,10 @@ class Vectorizer {
 private:
   unsigned getPointerAddressSpace(Value *I);
 
-  /// TODO: Remove this function once transition to Align is over.
-  unsigned getAlignment(LoadInst *LI) const { return getAlign(LI).value(); }
-
   Align getAlign(LoadInst *LI) const {
     return DL.getValueOrABITypeAlignment(LI->getAlign(), LI->getType());
   }
 
-  /// TODO: Remove this function once transition to Align is over.
-  unsigned getAlignment(StoreInst *SI) const { return getAlign(SI).value(); }
-
   Align getAlign(StoreInst *SI) const {
     return DL.getValueOrABITypeAlignment(SI->getAlign(),
                                          SI->getValueOperand()->getType());
@@ -1109,7 +1103,7 @@ bool Vectorizer::vectorizeLoadChain(
   unsigned VecRegSize = TTI.getLoadStoreVecRegBitWidth(AS);
   unsigned VF = VecRegSize / Sz;
   unsigned ChainSize = Chain.size();
-  unsigned Alignment = getAlignment(L0);
+  Align Alignment = getAlign(L0);
 
   if (!isPowerOf2_32(Sz) || VF < 2 || ChainSize < 2) {
     InstructionsProcessed->insert(Chain.begin(), Chain.end());
@@ -1159,7 +1153,7 @@ bool Vectorizer::vectorizeLoadChain(
   InstructionsProcessed->insert(Chain.begin(), Chain.end());
 
   // If the load is going to be misaligned, don't vectorize it.
-  if (accessIsMisaligned(SzInBytes, AS, Alignment)) {
+  if (accessIsMisaligned(SzInBytes, AS, Alignment.value())) {
     if (L0->getPointerAddressSpace() != DL.getAllocaAddrSpace()) {
       auto Chains = splitOddVectorElts(Chain, Sz);
       return vectorizeLoadChain(Chains.first, InstructionsProcessed) |
@@ -1168,13 +1162,13 @@ bool Vectorizer::vectorizeLoadChain(
 
     unsigned NewAlign = getOrEnforceKnownAlignment(
       L0->getPointerOperand(), StackAdjustedAlignment, DL, L0, nullptr, &DT);
-    if (NewAlign >= Alignment)
-      Alignment = NewAlign;
+    if (NewAlign >= Alignment.value())
+      Alignment = Align(NewAlign);
     else
       return false;
   }
 
-  if (!TTI.isLegalToVectorizeLoadChain(SzInBytes, Alignment, AS)) {
+  if (!TTI.isLegalToVectorizeLoadChain(SzInBytes, Alignment.value(), AS)) {
     auto Chains = splitOddVectorElts(Chain, Sz);
     return vectorizeLoadChain(Chains.first, InstructionsProcessed) |
            vectorizeLoadChain(Chains.second, InstructionsProcessed);
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 60cf5be692fb1..af51dea002da9 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -486,8 +486,8 @@ class InnerLoopVectorizer {
   /// non-null. Use \p State to translate given VPValues to IR values in the
   /// vectorized loop.
   void vectorizeMemoryInstruction(Instruction *Instr, VPTransformState &State,
-                                  VPValue *Addr,
-                                  VPValue *BlockInMask = nullptr);
+                                  VPValue *Addr, VPValue *StoredValue,
+                                  VPValue *BlockInMask);
 
   /// Set the debug location in the builder using the debug location in
   /// the instruction.
@@ -1806,51 +1806,31 @@ void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV, TruncInst *Trunc) {
   auto ID = II->second;
   assert(IV->getType() == ID.getStartValue()->getType() && "Types must match");
 
-  // The scalar value to broadcast. This will be derived from the canonical
-  // induction variable.
-  Value *ScalarIV = nullptr;
-
   // The value from the original loop to which we are mapping the new induction
   // variable.
   Instruction *EntryVal = Trunc ? cast<Instruction>(Trunc) : IV;
 
-  // True if we have vectorized the induction variable.
-  auto VectorizedIV = false;
-
-  // Determine if we want a scalar version of the induction variable. This is
-  // true if the induction variable itself is not widened, or if it has at
-  // least one user in the loop that is not widened.
-  auto NeedsScalarIV = VF > 1 && needsScalarInduction(EntryVal);
+  auto &DL = OrigLoop->getHeader()->getModule()->getDataLayout();
 
   // Generate code for the induction step. Note that induction steps are
   // required to be loop-invariant
-  assert(PSE.getSE()->isLoopInvariant(ID.getStep(), OrigLoop) &&
-         "Induction step should be loop invariant");
-  auto &DL = OrigLoop->getHeader()->getModule()->getDataLayout();
-  Value *Step = nullptr;
-  if (PSE.getSE()->isSCEVable(IV->getType())) {
-    SCEVExpander Exp(*PSE.getSE(), DL, "induction");
-    Step = Exp.expandCodeFor(ID.getStep(), ID.getStep()->getType(),
-                             LoopVectorPreHeader->getTerminator());
-  } else {
-    Step = cast<SCEVUnknown>(ID.getStep())->getValue();
-  }
-
-  // Try to create a new independent vector induction variable. If we can't
-  // create the phi node, we will splat the scalar induction variable in each
-  // loop iteration.
-  if (VF > 1 && !shouldScalarizeInstruction(EntryVal)) {
-    createVectorIntOrFpInductionPHI(ID, Step, EntryVal);
-    VectorizedIV = true;
-  }
+  auto CreateStepValue = [&](const SCEV *Step) -> Value * {
+    assert(PSE.getSE()->isLoopInvariant(Step, OrigLoop) &&
+           "Induction step should be loop invariant");
+    if (PSE.getSE()->isSCEVable(IV->getType())) {
+      SCEVExpander Exp(*PSE.getSE(), DL, "induction");
+      return Exp.expandCodeFor(Step, Step->getType(),
+                               LoopVectorPreHeader->getTerminator());
+    }
+    return cast<SCEVUnknown>(Step)->getValue();
+  };
 
-  // If we haven't yet vectorized the induction variable, or if we will create
-  // a scalar one, we need to define the scalar induction variable and step
-  // values. If we were given a truncation type, truncate the canonical
+  // The scalar value to broadcast. This is derived from the canonical
+  // induction variable. If a truncation type is given, truncate the canonical
   // induction variable and step. Otherwise, derive these values from the
   // induction descriptor.
-  if (!VectorizedIV || NeedsScalarIV) {
-    ScalarIV = Induction;
+  auto CreateScalarIV = [&](Value *&Step) -> Value * {
+    Value *ScalarIV = Induction;
     if (IV != OldInduction) {
       ScalarIV = IV->getType()->isIntegerTy()
                      ? Builder.CreateSExtOrTrunc(Induction, IV->getType())
@@ -1866,12 +1846,12 @@ void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV, TruncInst *Trunc) {
       ScalarIV = Builder.CreateTrunc(ScalarIV, TruncType);
       Step = Builder.CreateTrunc(Step, TruncType);
     }
-  }
+    return ScalarIV;
+  };
 
-  // If we haven't yet vectorized the induction variable, splat the scalar
-  // induction variable, and build the necessary step vectors.
-  // TODO: Don't do it unless the vectorized IV is really required.
-  if (!VectorizedIV) {
+  // Create the vector values from the scalar IV, in the absence of creating a
+  // vector IV.
+  auto CreateSplatIV = [&](Value *ScalarIV, Value *Step) {
     Value *Broadcasted = getBroadcastInstrs(ScalarIV);
     for (unsigned Part = 0; Part < UF; ++Part) {
       Value *EntryPart =
@@ -1881,16 +1861,45 @@ void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV, TruncInst *Trunc) {
         addMetadata(EntryPart, Trunc);
       recordVectorLoopValueForInductionCast(ID, EntryVal, EntryPart, Part);
     }
+  };
+
+  // Now do the actual transformations, and start with creating the step value.
+  Value *Step = CreateStepValue(ID.getStep());
+  if (VF <= 1) {
+    Value *ScalarIV = CreateScalarIV(Step);
+    CreateSplatIV(ScalarIV, Step);
+    return;
+  }
+
+  // Determine if we want a scalar version of the induction variable. This is
+  // true if the induction variable itself is not widened, or if it has at
+  // least one user in the loop that is not widened.
+  auto NeedsScalarIV = needsScalarInduction(EntryVal);
+  if (!NeedsScalarIV) {
+    createVectorIntOrFpInductionPHI(ID, Step, EntryVal);
+    return;
   }
 
-  // If an induction variable is only used for counting loop iterations or
-  // calculating addresses, it doesn't need to be widened. Create scalar steps
-  // that can be used by instructions we will later scalarize. Note that the
-  // addition of the scalar steps will not increase the number of instructions
-  // in the loop in the common case prior to InstCombine. We will be trading
-  // one vector extract for each scalar step.
-  if (NeedsScalarIV)
+  // Try to create a new independent vector induction variable. If we can't
+  // create the phi node, we will splat the scalar induction variable in each
+  // loop iteration.
+  if (!shouldScalarizeInstruction(EntryVal)) {
+    createVectorIntOrFpInductionPHI(ID, Step, EntryVal);
+    Value *ScalarIV = CreateScalarIV(Step);
+    // Create scalar steps that can be used by instructions we will later
+    // scalarize. Note that the addition of the scalar steps will not increase
+    // the number of instructions in the loop in the common case prior to
+    // InstCombine. We will be trading one vector extract for each scalar step.
     buildScalarSteps(ScalarIV, Step, EntryVal, ID);
+    return;
+  }
+
+  // If we haven't yet vectorized the induction variable, splat the scalar
+  // induction variable, and build the necessary step vectors.
+  // TODO: Don't do it unless the vectorized IV is really required.
+  Value *ScalarIV = CreateScalarIV(Step);
+  CreateSplatIV(ScalarIV, Step);
+  buildScalarSteps(ScalarIV, Step, EntryVal, ID);
 }
 
 Value *InnerLoopVectorizer::getStepVector(Value *Val, int StartIdx, Value *Step,
@@ -2348,12 +2357,15 @@ void InnerLoopVectorizer::vectorizeInterleaveGroup(Instruction *Instr,
 void InnerLoopVectorizer::vectorizeMemoryInstruction(Instruction *Instr,
                                                      VPTransformState &State,
                                                      VPValue *Addr,
+                                                     VPValue *StoredValue,
                                                      VPValue *BlockInMask) {
   // Attempt to issue a wide load.
   LoadInst *LI = dyn_cast<LoadInst>(Instr);
   StoreInst *SI = dyn_cast<StoreInst>(Instr);
 
   assert((LI || SI) && "Invalid Load/Store instruction");
+  assert((!SI || StoredValue) && "No stored value provided for widened store");
+  assert((!LI || !StoredValue) && "Stored value provided for widened load");
 
   LoopVectorizationCostModel::InstWidening Decision =
       Cost->getWideningDecision(Instr, VF);
@@ -2425,7 +2437,7 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction(Instruction *Instr,
 
     for (unsigned Part = 0; Part < UF; ++Part) {
       Instruction *NewSI = nullptr;
-      Value *StoredVal = getOrCreateVectorValue(SI->getValueOperand(), Part);
+      Value *StoredVal = State.get(StoredValue, Part);
       if (CreateGatherScatter) {
         Value *MaskPart = isMaskRequired ? BlockInMaskParts[Part] : nullptr;
         Value *VectorGep = State.get(Addr, Part);
@@ -2571,7 +2583,7 @@ Value *InnerLoopVectorizer::getOrCreateTripCount(Loop *L) {
   // compare. The only way that we get a backedge taken count is that the
   // induction variable was signed and as such will not overflow. In such a case
   // truncation is legal.
-  if (BackedgeTakenCount->getType()->getPrimitiveSizeInBits() >
+  if (SE->getTypeSizeInBits(BackedgeTakenCount->getType()) >
       IdxTy->getPrimitiveSizeInBits())
     BackedgeTakenCount = SE->getTruncateOrNoop(BackedgeTakenCount, IdxTy);
   BackedgeTakenCount = SE->getNoopOrZeroExtend(BackedgeTakenCount, IdxTy);
@@ -3393,7 +3405,7 @@ void InnerLoopVectorizer::truncateToMinimalBitwidths() {
         auto *O1 = B.CreateZExtOrTrunc(
             SI->getOperand(1), VectorType::get(ScalarTruncatedTy, Elements1));
 
-        NewI = B.CreateShuffleVector(O0, O1, SI->getMask());
+        NewI = B.CreateShuffleVector(O0, O1, SI->getShuffleMask());
       } else if (isa<LoadInst>(I) || isa<PHINode>(I)) {
         // Don't do anything with the operands, just extend the result.
         continue;
@@ -6807,7 +6819,12 @@ VPRecipeBuilder::tryToWidenMemory(Instruction *I, VFRange &Range,
     Mask = createBlockInMask(I->getParent(), Plan);
 
   VPValue *Addr = Plan->getOrAddVPValue(getLoadStorePointerOperand(I));
-  return new VPWidenMemoryInstructionRecipe(*I, Addr, Mask);
+  if (LoadInst *Load = dyn_cast<LoadInst>(I))
+    return new VPWidenMemoryInstructionRecipe(*Load, Addr, Mask);
+
+  StoreInst *Store = cast<StoreInst>(I);
+  VPValue *StoredValue = Plan->getOrAddVPValue(Store->getValueOperand());
+  return new VPWidenMemoryInstructionRecipe(*Store, Addr, StoredValue, Mask);
 }
 
 VPWidenIntOrFpInductionRecipe *
@@ -6959,21 +6976,9 @@ bool VPRecipeBuilder::tryToWiden(Instruction *I, VPBasicBlock *VPBB,
 
   if (!LoopVectorizationPlanner::getDecisionAndClampRange(willWiden, Range))
     return false;
-  // If this ingredient's recipe is to be recorded, keep its recipe a singleton
-  // to avoid having to split recipes later.
-  bool IsSingleton = Ingredient2Recipe.count(I);
 
   // Success: widen this instruction.
-
-  // Use the default widening recipe. We optimize the common case where
-  // consecutive instructions can be represented by a single recipe.
-  if (!IsSingleton && !VPBB->empty() && LastExtensibleRecipe == &VPBB->back() &&
-      LastExtensibleRecipe->appendInstruction(I))
-    return true;
-
-  VPWidenRecipe *WidenRecipe = new VPWidenRecipe(I);
-  if (!IsSingleton)
-    LastExtensibleRecipe = WidenRecipe;
+  VPWidenRecipe *WidenRecipe = new VPWidenRecipe(*I);
   setRecipe(I, WidenRecipe);
   VPBB->appendRecipe(WidenRecipe);
   return true;
@@ -7368,8 +7373,7 @@ void VPInterleaveRecipe::print(raw_ostream &O, const Twine &Indent,
 }
 
 void VPWidenRecipe::execute(VPTransformState &State) {
-  for (auto &Instr : make_range(Begin, End))
-    State.ILV->widenInstruction(Instr);
+  State.ILV->widenInstruction(Ingredient);
 }
 
 void VPWidenGEPRecipe::execute(VPTransformState &State) {
@@ -7515,7 +7519,9 @@ void VPPredInstPHIRecipe::execute(VPTransformState &State) {
 }
 
 void VPWidenMemoryInstructionRecipe::execute(VPTransformState &State) {
-  State.ILV->vectorizeMemoryInstruction(&Instr, State, getAddr(), getMask());
+  VPValue *StoredValue = isa<StoreInst>(Instr) ? getStoredValue() : nullptr;
+  State.ILV->vectorizeMemoryInstruction(&Instr, State, getAddr(), StoredValue,
+                                        getMask());
 }
 
 // Determine how to lower the scalar epilogue, which depends on 1) optimising
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 598fb00e956ea..f3172d49cf2af 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -49,11 +49,8 @@ class VPRecipeBuilder {
 
   // VPlan-VPlan transformations support: Hold a mapping from ingredients to
   // their recipe. To save on memory, only do so for selected ingredients,
-  // marked by having a nullptr entry in this map. If those ingredients get a
-  // VPWidenRecipe, also avoid compressing other ingredients into it to avoid
-  // having to split such recipes later.
+  // marked by having a nullptr entry in this map.
   DenseMap<Instruction *, VPRecipeBase *> Ingredient2Recipe;
-  VPWidenRecipe *LastExtensibleRecipe = nullptr;
 
   /// Set the recipe created for given ingredient. This operation is a no-op for
   /// ingredients that were not marked using a nullptr entry in the map.
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index f584b92866944..7944cf5405382 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -715,8 +715,7 @@ void VPlanPrinter::printAsIngredient(raw_ostream &O, Value *V) {
 void VPWidenRecipe::print(raw_ostream &O, const Twine &Indent,
                           VPSlotTracker &SlotTracker) const {
   O << " +\n" << Indent << "\"WIDEN\\l\"";
-  for (auto &Instr : make_range(Begin, End))
-    O << " +\n" << Indent << "\"  " << VPlanIngredient(&Instr) << "\\l\"";
+  O << "\"  " << VPlanIngredient(&Ingredient) << "\\l\"";
 }
 
 void VPWidenIntOrFpInductionRecipe::print(raw_ostream &O, const Twine &Indent,
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 044d2ffc3fc3e..68928d02d56d8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -757,21 +757,16 @@ class VPInstruction : public VPUser, public VPRecipeBase {
   }
 };
 
-/// VPWidenRecipe is a recipe for producing a copy of vector type for each
-/// Instruction in its ingredients independently, in order. This recipe covers
-/// most of the traditional vectorization cases where each ingredient transforms
-/// into a vectorized version of itself.
+/// VPWidenRecipe is a recipe for producing a copy of vector type its
+/// ingredient. This recipe covers most of the traditional vectorization cases
+/// where each ingredient transforms into a vectorized version of itself.
 class VPWidenRecipe : public VPRecipeBase {
 private:
-  /// Hold the ingredients by pointing to their original BasicBlock location.
-  BasicBlock::iterator Begin;
-  BasicBlock::iterator End;
+  /// Hold the instruction to be widened.
+  Instruction &Ingredient;
 
 public:
-  VPWidenRecipe(Instruction *I) : VPRecipeBase(VPWidenSC) {
-    End = I->getIterator();
-    Begin = End++;
-  }
+  VPWidenRecipe(Instruction &I) : VPRecipeBase(VPWidenSC), Ingredient(I) {}
 
   ~VPWidenRecipe() override = default;
 
@@ -783,14 +778,6 @@ class VPWidenRecipe : public VPRecipeBase {
   /// Produce widened copies of all Ingredients.
   void execute(VPTransformState &State) override;
 
-  /// Augment the recipe to include Instr, if it lies at its End.
-  bool appendInstruction(Instruction *Instr) {
-    if (End != Instr->getIterator())
-      return false;
-    End++;
-    return true;
-  }
-
   /// Print the recipe.
   void print(raw_ostream &O, const Twine &Indent,
              VPSlotTracker &SlotTracker) const override;
@@ -1061,6 +1048,9 @@ class VPPredInstPHIRecipe : public VPRecipeBase {
 };
 
 /// A Recipe for widening load/store operations.
+/// The recipe uses the following VPValues:
+/// - For load: Address, optional mask
+/// - For store: Address, stored value, optional mask
 /// TODO: We currently execute only per-part unless a specific instance is
 /// provided.
 class VPWidenMemoryInstructionRecipe : public VPRecipeBase {
@@ -1068,12 +1058,28 @@ class VPWidenMemoryInstructionRecipe : public VPRecipeBase {
   Instruction &Instr;
   VPUser User;
 
+  void setMask(VPValue *Mask) {
+    if (!Mask)
+      return;
+    User.addOperand(Mask);
+  }
+
+  bool isMasked() const {
+    return (isa<LoadInst>(Instr) && User.getNumOperands() == 2) ||
+           (isa<StoreInst>(Instr) && User.getNumOperands() == 3);
+  }
+
 public:
-  VPWidenMemoryInstructionRecipe(Instruction &Instr, VPValue *Addr,
-                                 VPValue *Mask)
-      : VPRecipeBase(VPWidenMemoryInstructionSC), Instr(Instr), User({Addr}) {
-    if (Mask)
-      User.addOperand(Mask);
+  VPWidenMemoryInstructionRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask)
+      : VPRecipeBase(VPWidenMemoryInstructionSC), Instr(Load), User({Addr}) {
+    setMask(Mask);
+  }
+
+  VPWidenMemoryInstructionRecipe(StoreInst &Store, VPValue *Addr,
+                                 VPValue *StoredValue, VPValue *Mask)
+      : VPRecipeBase(VPWidenMemoryInstructionSC), Instr(Store),
+        User({Addr, StoredValue}) {
+    setMask(Mask);
   }
 
   /// Method to support type inquiry through isa, cast, and dyn_cast.
@@ -1089,8 +1095,15 @@ class VPWidenMemoryInstructionRecipe : public VPRecipeBase {
   /// Return the mask used by this recipe. Note that a full mask is represented
   /// by a nullptr.
   VPValue *getMask() const {
-    // Mask is optional and therefore the last, currently 2nd operand.
-    return User.getNumOperands() == 2 ? User.getOperand(1) : nullptr;
+    // Mask is optional and therefore the last operand.
+    return isMasked() ? User.getOperand(User.getNumOperands() - 1) : nullptr;
+  }
+
+  /// Return the address accessed by this recipe.
+  VPValue *getStoredValue() const {
+    assert(isa<StoreInst>(Instr) &&
+           "Stored value only available for store instructions");
+    return User.getOperand(1); // Stored value is the 2nd, mandatory operand.
   }
 
   /// Generate the wide load/store.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 933d87bbe645b..fa864889efa3f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -41,7 +41,6 @@ void VPlanTransforms::VPInstructionsToVPRecipes(
       continue;
 
     VPBasicBlock *VPBB = Base->getEntryBasicBlock();
-    VPRecipeBase *LastRecipe = nullptr;
     // Introduce each ingredient into VPlan.
     for (auto I = VPBB->begin(), E = VPBB->end(); I != E;) {
       VPRecipeBase *Ingredient = &*I++;
@@ -55,10 +54,14 @@ void VPlanTransforms::VPInstructionsToVPRecipes(
 
       VPRecipeBase *NewRecipe = nullptr;
       // Create VPWidenMemoryInstructionRecipe for loads and stores.
-      if (isa<LoadInst>(Inst) || isa<StoreInst>(Inst))
+      if (LoadInst *Load = dyn_cast<LoadInst>(Inst))
         NewRecipe = new VPWidenMemoryInstructionRecipe(
-            *Inst, Plan->getOrAddVPValue(getLoadStorePointerOperand(Inst)),
+            *Load, Plan->getOrAddVPValue(getLoadStorePointerOperand(Inst)),
             nullptr /*Mask*/);
+      else if (StoreInst *Store = dyn_cast<StoreInst>(Inst))
+        NewRecipe = new VPWidenMemoryInstructionRecipe(
+            *Store, Plan->getOrAddVPValue(getLoadStorePointerOperand(Inst)),
+            Plan->getOrAddVPValue(Store->getValueOperand()), nullptr /*Mask*/);
       else if (PHINode *Phi = dyn_cast<PHINode>(Inst)) {
         InductionDescriptor II = Inductions.lookup(Phi);
         if (II.getKind() == InductionDescriptor::IK_IntInduction ||
@@ -68,20 +71,10 @@ void VPlanTransforms::VPInstructionsToVPRecipes(
           NewRecipe = new VPWidenPHIRecipe(Phi);
       } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Inst)) {
         NewRecipe = new VPWidenGEPRecipe(GEP, OrigLoop);
-      } else {
-        // If the last recipe is a VPWidenRecipe, add Inst to it instead of
-        // creating a new recipe.
-        if (VPWidenRecipe *WidenRecipe =
-                dyn_cast_or_null<VPWidenRecipe>(LastRecipe)) {
-          WidenRecipe->appendInstruction(Inst);
-          Ingredient->eraseFromParent();
-          continue;
-        }
-        NewRecipe = new VPWidenRecipe(Inst);
-      }
+      } else
+        NewRecipe = new VPWidenRecipe(*Inst);
 
       NewRecipe->insertBefore(Ingredient);
-      LastRecipe = NewRecipe;
       Ingredient->eraseFromParent();
     }
   }
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index ba3ff87884b0e..444290b2139c1 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -17,6 +17,7 @@
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
+#include "llvm/Analysis/VectorUtils.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IRBuilder.h"
@@ -51,7 +52,8 @@ static cl::opt<bool> DisableBinopExtractShuffle(
 static bool isExtractExtractCheap(Instruction *Ext0, Instruction *Ext1,
                                   unsigned Opcode,
                                   const TargetTransformInfo &TTI,
-                                  Instruction *&ConvertToShuffle) {
+                                  Instruction *&ConvertToShuffle,
+                                  unsigned PreferredExtractIndex) {
   assert(isa<ConstantInt>(Ext0->getOperand(1)) &&
          isa<ConstantInt>(Ext1->getOperand(1)) &&
          "Expected constant extract indexes");
@@ -130,12 +132,17 @@ static bool isExtractExtractCheap(Instruction *Ext0, Instruction *Ext1,
     NewCost +=
         TTI.getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, VecTy);
 
-    // The more expensive extract will be replaced by a shuffle. If the extracts
-    // have the same cost, replace the extract with the higher index.
+    // The more expensive extract will be replaced by a shuffle. If the costs
+    // are equal and there is a preferred extract index, shuffle the opposite
+    // operand. Otherwise, replace the extract with the higher index.
     if (Extract0Cost > Extract1Cost)
       ConvertToShuffle = Ext0;
     else if (Extract1Cost > Extract0Cost)
       ConvertToShuffle = Ext1;
+    else if (PreferredExtractIndex == Ext0Index)
+      ConvertToShuffle = Ext1;
+    else if (PreferredExtractIndex == Ext1Index)
+      ConvertToShuffle = Ext0;
     else
       ConvertToShuffle = Ext0Index > Ext1Index ? Ext0 : Ext1;
   }
@@ -208,8 +215,19 @@ static bool foldExtractExtract(Instruction &I, const TargetTransformInfo &TTI) {
       V0->getType() != V1->getType())
     return false;
 
+  // If the scalar value 'I' is going to be re-inserted into a vector, then try
+  // to create an extract to that same element. The extract/insert can be
+  // reduced to a "select shuffle".
+  // TODO: If we add a larger pattern match that starts from an insert, this
+  //       probably becomes unnecessary.
+  uint64_t InsertIndex = std::numeric_limits<uint64_t>::max();
+  if (I.hasOneUse())
+    match(I.user_back(), m_InsertElement(m_Value(), m_Value(),
+                                         m_ConstantInt(InsertIndex)));
+
   Instruction *ConvertToShuffle;
-  if (isExtractExtractCheap(Ext0, Ext1, I.getOpcode(), TTI, ConvertToShuffle))
+  if (isExtractExtractCheap(Ext0, Ext1, I.getOpcode(), TTI, ConvertToShuffle,
+                            InsertIndex))
     return false;
 
   if (ConvertToShuffle) {
@@ -244,6 +262,49 @@ static bool foldExtractExtract(Instruction &I, const TargetTransformInfo &TTI) {
   return true;
 }
 
+/// If this is a bitcast to narrow elements from a shuffle of wider elements,
+/// try to bitcast the source vector to the narrow type followed by shuffle.
+/// This can enable further transforms by moving bitcasts or shuffles together.
+static bool foldBitcastShuf(Instruction &I, const TargetTransformInfo &TTI) {
+  Value *V;
+  ArrayRef<int> Mask;
+  if (!match(&I, m_BitCast(m_OneUse(m_ShuffleVector(m_Value(V), m_Undef(),
+                                                    m_Mask(Mask))))))
+    return false;
+
+  Type *DestTy = I.getType();
+  Type *SrcTy = V->getType();
+  if (!DestTy->isVectorTy() || I.getOperand(0)->getType() != SrcTy)
+    return false;
+
+  // TODO: Handle bitcast from narrow element type to wide element type.
+  assert(SrcTy->isVectorTy() && "Shuffle of non-vector type?");
+  unsigned DestNumElts = DestTy->getVectorNumElements();
+  unsigned SrcNumElts = SrcTy->getVectorNumElements();
+  if (SrcNumElts > DestNumElts)
+    return false;
+
+  // The new shuffle must not cost more than the old shuffle. The bitcast is
+  // moved ahead of the shuffle, so assume that it has the same cost as before.
+  if (TTI.getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, DestTy) >
+      TTI.getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, SrcTy))
+    return false;
+
+  // Bitcast the source vector and expand the shuffle mask to the equivalent for
+  // narrow elements.
+  // bitcast (shuf V, MaskC) --> shuf (bitcast V), MaskC'
+  IRBuilder<> Builder(&I);
+  Value *CastV = Builder.CreateBitCast(V, DestTy);
+  SmallVector<int, 16> NewMask;
+  assert(DestNumElts % SrcNumElts == 0 && "Unexpected shuffle mask");
+  unsigned ScaleFactor = DestNumElts / SrcNumElts;
+  scaleShuffleMask(ScaleFactor, Mask, NewMask);
+  Value *Shuf = Builder.CreateShuffleVector(CastV, UndefValue::get(DestTy),
+                                            NewMask);
+  I.replaceAllUsesWith(Shuf);
+  return true;
+}
+
 /// This is the entry point for all transforms. Pass manager differences are
 /// handled in the callers of this function.
 static bool runImpl(Function &F, const TargetTransformInfo &TTI,
@@ -261,8 +322,12 @@ static bool runImpl(Function &F, const TargetTransformInfo &TTI,
     // use->defs, so we're more likely to succeed by starting from the bottom.
     // TODO: It could be more efficient to remove dead instructions
     //       iteratively in this loop rather than waiting until the end.
-    for (Instruction &I : make_range(BB.rbegin(), BB.rend()))
+    for (Instruction &I : make_range(BB.rbegin(), BB.rend())) {
+      if (isa<DbgInfoIntrinsic>(I))
+        continue;
       MadeChange |= foldExtractExtract(I, TTI);
+      MadeChange |= foldBitcastShuf(I, TTI);
+    }
   }
 
   // We're done with transforms, so remove dead instructions.
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/fdiv.ll b/llvm/test/Analysis/CostModel/AMDGPU/fdiv.ll
index 6986a31584615..e898f0d2c461c 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/fdiv.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/fdiv.ll
@@ -278,5 +278,5 @@ define amdgpu_kernel void @rcp_v2f16_f32_ftzdaz(<2 x half> addrspace(1)* %out, <
   ret void
 }
 
-attributes #0 = { nounwind "target-features"="+fp32-denormals" }
-attributes #1 = { nounwind "target-features"="-fp32-denormals" }
+attributes #0 = { nounwind "denormal-fp-math-f32"="ieee,ieee" }
+attributes #1 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
diff --git a/llvm/test/Analysis/CostModel/X86/reduce-fmax.ll b/llvm/test/Analysis/CostModel/X86/reduce-fmax.ll
new file mode 100644
index 0000000000000..2f6ae6e60b2b2
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/reduce-fmax.ll
@@ -0,0 +1,111 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
+; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+ssse3 | FileCheck %s --check-prefixes=CHECK,SSE,SSSE3
+; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+sse4.1 | FileCheck %s --check-prefixes=CHECK,SSE,SSE4,SSE41
+; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+sse4.2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE4,SSE42
+; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512DQ
+
+define i32 @reduce_f64(i32 %arg) {
+; SSE-LABEL: 'reduce_f64'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call double @llvm.experimental.vector.reduce.fmax.v1f64(<1 x double> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call double @llvm.experimental.vector.reduce.fmax.v2f64(<2 x double> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call double @llvm.experimental.vector.reduce.fmax.v4f64(<4 x double> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8 = call double @llvm.experimental.vector.reduce.fmax.v8f64(<8 x double> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16 = call double @llvm.experimental.vector.reduce.fmax.v16f64(<16 x double> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'reduce_f64'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call double @llvm.experimental.vector.reduce.fmax.v1f64(<1 x double> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call double @llvm.experimental.vector.reduce.fmax.v2f64(<2 x double> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call double @llvm.experimental.vector.reduce.fmax.v4f64(<4 x double> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call double @llvm.experimental.vector.reduce.fmax.v8f64(<8 x double> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16 = call double @llvm.experimental.vector.reduce.fmax.v16f64(<16 x double> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'reduce_f64'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call double @llvm.experimental.vector.reduce.fmax.v1f64(<1 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call double @llvm.experimental.vector.reduce.fmax.v2f64(<2 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call double @llvm.experimental.vector.reduce.fmax.v4f64(<4 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call double @llvm.experimental.vector.reduce.fmax.v8f64(<8 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call double @llvm.experimental.vector.reduce.fmax.v16f64(<16 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %V1  = call double @llvm.experimental.vector.reduce.fmax.v1f64(<1 x double> undef)
+  %V2  = call double @llvm.experimental.vector.reduce.fmax.v2f64(<2 x double> undef)
+  %V4  = call double @llvm.experimental.vector.reduce.fmax.v4f64(<4 x double> undef)
+  %V8  = call double @llvm.experimental.vector.reduce.fmax.v8f64(<8 x double> undef)
+  %V16 = call double @llvm.experimental.vector.reduce.fmax.v16f64(<16 x double> undef)
+  ret i32 undef
+}
+
+define i32 @reduce_f32(i32 %arg) {
+; SSE2-LABEL: 'reduce_f32'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call float @llvm.experimental.vector.reduce.fmax.v1f32(<1 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call float @llvm.experimental.vector.reduce.fmax.v2f32(<2 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call float @llvm.experimental.vector.reduce.fmax.v4f32(<4 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8 = call float @llvm.experimental.vector.reduce.fmax.v8f32(<8 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16 = call float @llvm.experimental.vector.reduce.fmax.v16f32(<16 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V32 = call float @llvm.experimental.vector.reduce.fmax.v32f32(<32 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'reduce_f32'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call float @llvm.experimental.vector.reduce.fmax.v1f32(<1 x float> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call float @llvm.experimental.vector.reduce.fmax.v2f32(<2 x float> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call float @llvm.experimental.vector.reduce.fmax.v4f32(<4 x float> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8 = call float @llvm.experimental.vector.reduce.fmax.v8f32(<8 x float> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16 = call float @llvm.experimental.vector.reduce.fmax.v16f32(<16 x float> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V32 = call float @llvm.experimental.vector.reduce.fmax.v32f32(<32 x float> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE4-LABEL: 'reduce_f32'
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call float @llvm.experimental.vector.reduce.fmax.v1f32(<1 x float> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call float @llvm.experimental.vector.reduce.fmax.v2f32(<2 x float> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call float @llvm.experimental.vector.reduce.fmax.v4f32(<4 x float> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call float @llvm.experimental.vector.reduce.fmax.v8f32(<8 x float> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16 = call float @llvm.experimental.vector.reduce.fmax.v16f32(<16 x float> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32 = call float @llvm.experimental.vector.reduce.fmax.v32f32(<32 x float> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'reduce_f32'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call float @llvm.experimental.vector.reduce.fmax.v1f32(<1 x float> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call float @llvm.experimental.vector.reduce.fmax.v2f32(<2 x float> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call float @llvm.experimental.vector.reduce.fmax.v4f32(<4 x float> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call float @llvm.experimental.vector.reduce.fmax.v8f32(<8 x float> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call float @llvm.experimental.vector.reduce.fmax.v16f32(<16 x float> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32 = call float @llvm.experimental.vector.reduce.fmax.v32f32(<32 x float> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'reduce_f32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call float @llvm.experimental.vector.reduce.fmax.v1f32(<1 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call float @llvm.experimental.vector.reduce.fmax.v2f32(<2 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call float @llvm.experimental.vector.reduce.fmax.v4f32(<4 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call float @llvm.experimental.vector.reduce.fmax.v8f32(<8 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call float @llvm.experimental.vector.reduce.fmax.v16f32(<16 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32 = call float @llvm.experimental.vector.reduce.fmax.v32f32(<32 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %V1  = call float @llvm.experimental.vector.reduce.fmax.v1f32(<1 x float> undef)
+  %V2  = call float @llvm.experimental.vector.reduce.fmax.v2f32(<2 x float> undef)
+  %V4  = call float @llvm.experimental.vector.reduce.fmax.v4f32(<4 x float> undef)
+  %V8  = call float @llvm.experimental.vector.reduce.fmax.v8f32(<8 x float> undef)
+  %V16 = call float @llvm.experimental.vector.reduce.fmax.v16f32(<16 x float> undef)
+  %V32 = call float @llvm.experimental.vector.reduce.fmax.v32f32(<32 x float> undef)
+  ret i32 undef
+}
+
+declare double @llvm.experimental.vector.reduce.fmax.v1f64(<1 x double>)
+declare double @llvm.experimental.vector.reduce.fmax.v2f64(<2 x double>)
+declare double @llvm.experimental.vector.reduce.fmax.v4f64(<4 x double>)
+declare double @llvm.experimental.vector.reduce.fmax.v8f64(<8 x double>)
+declare double @llvm.experimental.vector.reduce.fmax.v16f64(<16 x double>)
+
+declare float @llvm.experimental.vector.reduce.fmax.v1f32(<1 x float>)
+declare float @llvm.experimental.vector.reduce.fmax.v2f32(<2 x float>)
+declare float @llvm.experimental.vector.reduce.fmax.v4f32(<4 x float>)
+declare float @llvm.experimental.vector.reduce.fmax.v8f32(<8 x float>)
+declare float @llvm.experimental.vector.reduce.fmax.v16f32(<16 x float>)
+declare float @llvm.experimental.vector.reduce.fmax.v32f32(<32 x float>)
diff --git a/llvm/test/Analysis/CostModel/X86/reduce-fmin.ll b/llvm/test/Analysis/CostModel/X86/reduce-fmin.ll
new file mode 100644
index 0000000000000..917da71af9d9e
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/reduce-fmin.ll
@@ -0,0 +1,111 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
+; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+ssse3 | FileCheck %s --check-prefixes=CHECK,SSE,SSSE3
+; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+sse4.1 | FileCheck %s --check-prefixes=CHECK,SSE,SSE4,SSE41
+; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+sse4.2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE4,SSE42
+; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512DQ
+
+define i32 @reduce_f64(i32 %arg) {
+; SSE-LABEL: 'reduce_f64'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call double @llvm.experimental.vector.reduce.fmin.v1f64(<1 x double> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call double @llvm.experimental.vector.reduce.fmin.v2f64(<2 x double> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call double @llvm.experimental.vector.reduce.fmin.v4f64(<4 x double> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8 = call double @llvm.experimental.vector.reduce.fmin.v8f64(<8 x double> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16 = call double @llvm.experimental.vector.reduce.fmin.v16f64(<16 x double> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'reduce_f64'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call double @llvm.experimental.vector.reduce.fmin.v1f64(<1 x double> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call double @llvm.experimental.vector.reduce.fmin.v2f64(<2 x double> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call double @llvm.experimental.vector.reduce.fmin.v4f64(<4 x double> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call double @llvm.experimental.vector.reduce.fmin.v8f64(<8 x double> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16 = call double @llvm.experimental.vector.reduce.fmin.v16f64(<16 x double> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'reduce_f64'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call double @llvm.experimental.vector.reduce.fmin.v1f64(<1 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call double @llvm.experimental.vector.reduce.fmin.v2f64(<2 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call double @llvm.experimental.vector.reduce.fmin.v4f64(<4 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call double @llvm.experimental.vector.reduce.fmin.v8f64(<8 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call double @llvm.experimental.vector.reduce.fmin.v16f64(<16 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %V1  = call double @llvm.experimental.vector.reduce.fmin.v1f64(<1 x double> undef)
+  %V2  = call double @llvm.experimental.vector.reduce.fmin.v2f64(<2 x double> undef)
+  %V4  = call double @llvm.experimental.vector.reduce.fmin.v4f64(<4 x double> undef)
+  %V8  = call double @llvm.experimental.vector.reduce.fmin.v8f64(<8 x double> undef)
+  %V16 = call double @llvm.experimental.vector.reduce.fmin.v16f64(<16 x double> undef)
+  ret i32 undef
+}
+
+define i32 @reduce_f32(i32 %arg) {
+; SSE2-LABEL: 'reduce_f32'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call float @llvm.experimental.vector.reduce.fmin.v1f32(<1 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call float @llvm.experimental.vector.reduce.fmin.v2f32(<2 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call float @llvm.experimental.vector.reduce.fmin.v4f32(<4 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8 = call float @llvm.experimental.vector.reduce.fmin.v8f32(<8 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16 = call float @llvm.experimental.vector.reduce.fmin.v16f32(<16 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V32 = call float @llvm.experimental.vector.reduce.fmin.v32f32(<32 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'reduce_f32'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call float @llvm.experimental.vector.reduce.fmin.v1f32(<1 x float> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call float @llvm.experimental.vector.reduce.fmin.v2f32(<2 x float> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call float @llvm.experimental.vector.reduce.fmin.v4f32(<4 x float> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8 = call float @llvm.experimental.vector.reduce.fmin.v8f32(<8 x float> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16 = call float @llvm.experimental.vector.reduce.fmin.v16f32(<16 x float> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V32 = call float @llvm.experimental.vector.reduce.fmin.v32f32(<32 x float> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE4-LABEL: 'reduce_f32'
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call float @llvm.experimental.vector.reduce.fmin.v1f32(<1 x float> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call float @llvm.experimental.vector.reduce.fmin.v2f32(<2 x float> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call float @llvm.experimental.vector.reduce.fmin.v4f32(<4 x float> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call float @llvm.experimental.vector.reduce.fmin.v8f32(<8 x float> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16 = call float @llvm.experimental.vector.reduce.fmin.v16f32(<16 x float> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32 = call float @llvm.experimental.vector.reduce.fmin.v32f32(<32 x float> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'reduce_f32'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call float @llvm.experimental.vector.reduce.fmin.v1f32(<1 x float> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call float @llvm.experimental.vector.reduce.fmin.v2f32(<2 x float> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call float @llvm.experimental.vector.reduce.fmin.v4f32(<4 x float> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call float @llvm.experimental.vector.reduce.fmin.v8f32(<8 x float> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call float @llvm.experimental.vector.reduce.fmin.v16f32(<16 x float> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32 = call float @llvm.experimental.vector.reduce.fmin.v32f32(<32 x float> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'reduce_f32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call float @llvm.experimental.vector.reduce.fmin.v1f32(<1 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call float @llvm.experimental.vector.reduce.fmin.v2f32(<2 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call float @llvm.experimental.vector.reduce.fmin.v4f32(<4 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call float @llvm.experimental.vector.reduce.fmin.v8f32(<8 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call float @llvm.experimental.vector.reduce.fmin.v16f32(<16 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32 = call float @llvm.experimental.vector.reduce.fmin.v32f32(<32 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %V1  = call float @llvm.experimental.vector.reduce.fmin.v1f32(<1 x float> undef)
+  %V2  = call float @llvm.experimental.vector.reduce.fmin.v2f32(<2 x float> undef)
+  %V4  = call float @llvm.experimental.vector.reduce.fmin.v4f32(<4 x float> undef)
+  %V8  = call float @llvm.experimental.vector.reduce.fmin.v8f32(<8 x float> undef)
+  %V16 = call float @llvm.experimental.vector.reduce.fmin.v16f32(<16 x float> undef)
+  %V32 = call float @llvm.experimental.vector.reduce.fmin.v32f32(<32 x float> undef)
+  ret i32 undef
+}
+
+declare double @llvm.experimental.vector.reduce.fmin.v1f64(<1 x double>)
+declare double @llvm.experimental.vector.reduce.fmin.v2f64(<2 x double>)
+declare double @llvm.experimental.vector.reduce.fmin.v4f64(<4 x double>)
+declare double @llvm.experimental.vector.reduce.fmin.v8f64(<8 x double>)
+declare double @llvm.experimental.vector.reduce.fmin.v16f64(<16 x double>)
+
+declare float @llvm.experimental.vector.reduce.fmin.v1f32(<1 x float>)
+declare float @llvm.experimental.vector.reduce.fmin.v2f32(<2 x float>)
+declare float @llvm.experimental.vector.reduce.fmin.v4f32(<4 x float>)
+declare float @llvm.experimental.vector.reduce.fmin.v8f32(<8 x float>)
+declare float @llvm.experimental.vector.reduce.fmin.v16f32(<16 x float>)
+declare float @llvm.experimental.vector.reduce.fmin.v32f32(<32 x float>)
diff --git a/llvm/test/Analysis/CostModel/X86/reduce-smax.ll b/llvm/test/Analysis/CostModel/X86/reduce-smax.ll
index a578ad67f723d..f121a14b345c4 100644
--- a/llvm/test/Analysis/CostModel/X86/reduce-smax.ll
+++ b/llvm/test/Analysis/CostModel/X86/reduce-smax.ll
@@ -1,7 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
 ; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
 ; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+ssse3 | FileCheck %s --check-prefixes=CHECK,SSE,SSSE3
-; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+sse4.2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
+; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+sse4.1 | FileCheck %s --check-prefixes=CHECK,SSE,SSE4,SSE41
+; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+sse4.2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE4,SSE42
 ; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
 ; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
 ; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
@@ -11,26 +12,34 @@
 define i32 @reduce_i64(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i64'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.smax.v1i64(<1 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.smax.v2i64(<2 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.smax.v4i64(<4 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.smax.v8i64(<8 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.smax.v16i64(<16 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.smax.v2i64(<2 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.smax.v4i64(<4 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.smax.v8i64(<8 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.smax.v16i64(<16 x i64> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i64'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.smax.v1i64(<1 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.smax.v2i64(<2 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.smax.v4i64(<4 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.smax.v8i64(<8 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.smax.v16i64(<16 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.smax.v2i64(<2 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.smax.v4i64(<4 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.smax.v8i64(<8 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.smax.v16i64(<16 x i64> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SSE41-LABEL: 'reduce_i64'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.smax.v1i64(<1 x i64> undef)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.smax.v2i64(<2 x i64> undef)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.smax.v4i64(<4 x i64> undef)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.smax.v8i64(<8 x i64> undef)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.smax.v16i64(<16 x i64> undef)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; SSE42-LABEL: 'reduce_i64'
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.smax.v1i64(<1 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.smax.v2i64(<2 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.smax.v4i64(<4 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.smax.v8i64(<8 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.smax.v16i64(<16 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.smax.v2i64(<2 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.smax.v4i64(<4 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.smax.v8i64(<8 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.smax.v16i64(<16 x i64> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i64'
@@ -67,28 +76,28 @@ define i32 @reduce_i64(i32 %arg) {
 
 define i32 @reduce_i32(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smax.v2i32(<2 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smax.v4i32(<4 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smax.v8i32(<8 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smax.v16i32(<16 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.smax.v32i32(<32 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smax.v2i32(<2 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smax.v4i32(<4 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smax.v8i32(<8 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smax.v16i32(<16 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.smax.v32i32(<32 x i32> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i32'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smax.v2i32(<2 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smax.v4i32(<4 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smax.v8i32(<8 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smax.v16i32(<16 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.smax.v32i32(<32 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smax.v2i32(<2 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smax.v4i32(<4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smax.v8i32(<8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smax.v16i32(<16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.smax.v32i32(<32 x i32> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; SSE42-LABEL: 'reduce_i32'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smax.v2i32(<2 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smax.v4i32(<4 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smax.v8i32(<8 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smax.v16i32(<16 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.smax.v32i32(<32 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; SSE4-LABEL: 'reduce_i32'
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smax.v2i32(<2 x i32> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smax.v4i32(<4 x i32> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smax.v8i32(<8 x i32> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smax.v16i32(<16 x i32> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.smax.v32i32(<32 x i32> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i32'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smax.v2i32(<2 x i32> undef)
@@ -124,31 +133,31 @@ define i32 @reduce_i32(i32 %arg) {
 
 define i32 @reduce_i16(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.v2i16(<2 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smax.v4i16(<4 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smax.v8i16(<8 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smax.v16i16(<16 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smax.v32i16(<32 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smax.v64i16(<64 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.v2i16(<2 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smax.v4i16(<4 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smax.v8i16(<8 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smax.v16i16(<16 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smax.v32i16(<32 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smax.v64i16(<64 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i16'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.v2i16(<2 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smax.v4i16(<4 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smax.v8i16(<8 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smax.v16i16(<16 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smax.v32i16(<32 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smax.v64i16(<64 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.v2i16(<2 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smax.v4i16(<4 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smax.v8i16(<8 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smax.v16i16(<16 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smax.v32i16(<32 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smax.v64i16(<64 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; SSE42-LABEL: 'reduce_i16'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.v2i16(<2 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smax.v4i16(<4 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smax.v8i16(<8 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smax.v16i16(<16 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smax.v32i16(<32 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smax.v64i16(<64 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; SSE4-LABEL: 'reduce_i16'
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.v2i16(<2 x i16> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smax.v4i16(<4 x i16> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smax.v8i16(<8 x i16> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smax.v16i16(<16 x i16> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smax.v32i16(<32 x i16> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smax.v64i16(<64 x i16> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i16'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.v2i16(<2 x i16> undef)
@@ -206,34 +215,34 @@ define i32 @reduce_i16(i32 %arg) {
 
 define i32 @reduce_i8(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i8'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.v2i8(<2 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.v4i8(<4 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smax.v8i8(<8 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smax.v16i8(<16 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smax.v32i8(<32 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smax.v64i8(<64 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smax.v128i8(<128 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.v2i8(<2 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.v4i8(<4 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smax.v8i8(<8 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smax.v16i8(<16 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smax.v32i8(<32 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smax.v64i8(<64 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smax.v128i8(<128 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i8'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.v2i8(<2 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.v4i8(<4 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smax.v8i8(<8 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smax.v16i8(<16 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smax.v32i8(<32 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smax.v64i8(<64 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smax.v128i8(<128 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.v2i8(<2 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.v4i8(<4 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smax.v8i8(<8 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smax.v16i8(<16 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smax.v32i8(<32 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smax.v64i8(<64 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smax.v128i8(<128 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; SSE42-LABEL: 'reduce_i8'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.v2i8(<2 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.v4i8(<4 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smax.v8i8(<8 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smax.v16i8(<16 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smax.v32i8(<32 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smax.v64i8(<64 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smax.v128i8(<128 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; SSE4-LABEL: 'reduce_i8'
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.v2i8(<2 x i8> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.v4i8(<4 x i8> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smax.v8i8(<8 x i8> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smax.v16i8(<16 x i8> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smax.v32i8(<32 x i8> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smax.v64i8(<64 x i8> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smax.v128i8(<128 x i8> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i8'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.v2i8(<2 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/X86/reduce-smin.ll b/llvm/test/Analysis/CostModel/X86/reduce-smin.ll
index 01263daea9be2..71eb73567af1e 100644
--- a/llvm/test/Analysis/CostModel/X86/reduce-smin.ll
+++ b/llvm/test/Analysis/CostModel/X86/reduce-smin.ll
@@ -1,7 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
 ; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
 ; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+ssse3 | FileCheck %s --check-prefixes=CHECK,SSE,SSSE3
-; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+sse4.2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
+; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+sse4.1 | FileCheck %s --check-prefixes=CHECK,SSE,SSE4,SSE41
+; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+sse4.2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE4,SSE42
 ; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
 ; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
 ; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
@@ -11,26 +12,34 @@
 define i32 @reduce_i64(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i64'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.smin.v1i64(<1 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.smin.v2i64(<2 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.smin.v4i64(<4 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.smin.v8i64(<8 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.smin.v16i64(<16 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.smin.v2i64(<2 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.smin.v4i64(<4 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.smin.v8i64(<8 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.smin.v16i64(<16 x i64> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i64'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.smin.v1i64(<1 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.smin.v2i64(<2 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.smin.v4i64(<4 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.smin.v8i64(<8 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.smin.v16i64(<16 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.smin.v2i64(<2 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.smin.v4i64(<4 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.smin.v8i64(<8 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.smin.v16i64(<16 x i64> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SSE41-LABEL: 'reduce_i64'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.smin.v1i64(<1 x i64> undef)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.smin.v2i64(<2 x i64> undef)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.smin.v4i64(<4 x i64> undef)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.smin.v8i64(<8 x i64> undef)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.smin.v16i64(<16 x i64> undef)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; SSE42-LABEL: 'reduce_i64'
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.smin.v1i64(<1 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.smin.v2i64(<2 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.smin.v4i64(<4 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.smin.v8i64(<8 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.smin.v16i64(<16 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.smin.v2i64(<2 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.smin.v4i64(<4 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.smin.v8i64(<8 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.smin.v16i64(<16 x i64> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i64'
@@ -67,28 +76,28 @@ define i32 @reduce_i64(i32 %arg) {
 
 define i32 @reduce_i32(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smin.v2i32(<2 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smin.v4i32(<4 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smin.v8i32(<8 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smin.v16i32(<16 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.smin.v32i32(<32 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smin.v2i32(<2 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smin.v4i32(<4 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smin.v8i32(<8 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smin.v16i32(<16 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.smin.v32i32(<32 x i32> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i32'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smin.v2i32(<2 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smin.v4i32(<4 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smin.v8i32(<8 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smin.v16i32(<16 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.smin.v32i32(<32 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smin.v2i32(<2 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smin.v4i32(<4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smin.v8i32(<8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smin.v16i32(<16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.smin.v32i32(<32 x i32> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; SSE42-LABEL: 'reduce_i32'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smin.v2i32(<2 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smin.v4i32(<4 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smin.v8i32(<8 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smin.v16i32(<16 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.smin.v32i32(<32 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; SSE4-LABEL: 'reduce_i32'
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smin.v2i32(<2 x i32> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smin.v4i32(<4 x i32> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smin.v8i32(<8 x i32> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smin.v16i32(<16 x i32> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.smin.v32i32(<32 x i32> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i32'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smin.v2i32(<2 x i32> undef)
@@ -124,31 +133,31 @@ define i32 @reduce_i32(i32 %arg) {
 
 define i32 @reduce_i16(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smin.v2i16(<2 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smin.v4i16(<4 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smin.v8i16(<8 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smin.v16i16(<16 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smin.v32i16(<32 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smin.v64i16(<64 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smin.v2i16(<2 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smin.v4i16(<4 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smin.v8i16(<8 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smin.v16i16(<16 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smin.v32i16(<32 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smin.v64i16(<64 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i16'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smin.v2i16(<2 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smin.v4i16(<4 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smin.v8i16(<8 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smin.v16i16(<16 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smin.v32i16(<32 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smin.v64i16(<64 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smin.v2i16(<2 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smin.v4i16(<4 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smin.v8i16(<8 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smin.v16i16(<16 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smin.v32i16(<32 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smin.v64i16(<64 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; SSE42-LABEL: 'reduce_i16'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smin.v2i16(<2 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smin.v4i16(<4 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smin.v8i16(<8 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smin.v16i16(<16 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smin.v32i16(<32 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smin.v64i16(<64 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; SSE4-LABEL: 'reduce_i16'
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smin.v2i16(<2 x i16> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smin.v4i16(<4 x i16> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smin.v8i16(<8 x i16> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smin.v16i16(<16 x i16> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smin.v32i16(<32 x i16> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smin.v64i16(<64 x i16> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i16'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smin.v2i16(<2 x i16> undef)
@@ -206,34 +215,34 @@ define i32 @reduce_i16(i32 %arg) {
 
 define i32 @reduce_i8(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i8'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.v2i8(<2 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.v4i8(<4 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smin.v8i8(<8 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smin.v16i8(<16 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smin.v32i8(<32 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smin.v64i8(<64 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smin.v128i8(<128 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.v2i8(<2 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.v4i8(<4 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smin.v8i8(<8 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smin.v16i8(<16 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smin.v32i8(<32 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smin.v64i8(<64 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smin.v128i8(<128 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i8'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.v2i8(<2 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.v4i8(<4 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smin.v8i8(<8 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smin.v16i8(<16 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smin.v32i8(<32 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smin.v64i8(<64 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smin.v128i8(<128 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.v2i8(<2 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.v4i8(<4 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smin.v8i8(<8 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smin.v16i8(<16 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smin.v32i8(<32 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smin.v64i8(<64 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smin.v128i8(<128 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; SSE42-LABEL: 'reduce_i8'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.v2i8(<2 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.v4i8(<4 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smin.v8i8(<8 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smin.v16i8(<16 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smin.v32i8(<32 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smin.v64i8(<64 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smin.v128i8(<128 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; SSE4-LABEL: 'reduce_i8'
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.v2i8(<2 x i8> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.v4i8(<4 x i8> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smin.v8i8(<8 x i8> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smin.v16i8(<16 x i8> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smin.v32i8(<32 x i8> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smin.v64i8(<64 x i8> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smin.v128i8(<128 x i8> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i8'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.v2i8(<2 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/X86/reduce-umax.ll b/llvm/test/Analysis/CostModel/X86/reduce-umax.ll
index f77915e1b3560..66af4c9a8dea6 100644
--- a/llvm/test/Analysis/CostModel/X86/reduce-umax.ll
+++ b/llvm/test/Analysis/CostModel/X86/reduce-umax.ll
@@ -1,7 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
 ; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
 ; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+ssse3 | FileCheck %s --check-prefixes=CHECK,SSE,SSSE3
-; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+sse4.2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
+; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+sse4.1 | FileCheck %s --check-prefixes=CHECK,SSE,SSE4,SSE41
+; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+sse4.2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE4,SSE42
 ; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
 ; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
 ; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
@@ -11,26 +12,34 @@
 define i32 @reduce_i64(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i64'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.umax.v1i64(<1 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.umax.v2i64(<2 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.umax.v4i64(<4 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.umax.v8i64(<8 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.umax.v16i64(<16 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.umax.v2i64(<2 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.umax.v4i64(<4 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.umax.v8i64(<8 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.umax.v16i64(<16 x i64> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i64'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.umax.v1i64(<1 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.umax.v2i64(<2 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.umax.v4i64(<4 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.umax.v8i64(<8 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.umax.v16i64(<16 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.umax.v2i64(<2 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.umax.v4i64(<4 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.umax.v8i64(<8 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.umax.v16i64(<16 x i64> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SSE41-LABEL: 'reduce_i64'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.umax.v1i64(<1 x i64> undef)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.umax.v2i64(<2 x i64> undef)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.umax.v4i64(<4 x i64> undef)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.umax.v8i64(<8 x i64> undef)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.umax.v16i64(<16 x i64> undef)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; SSE42-LABEL: 'reduce_i64'
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.umax.v1i64(<1 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.umax.v2i64(<2 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.umax.v4i64(<4 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.umax.v8i64(<8 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.umax.v16i64(<16 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.umax.v2i64(<2 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.umax.v4i64(<4 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.umax.v8i64(<8 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.umax.v16i64(<16 x i64> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i64'
@@ -67,28 +76,28 @@ define i32 @reduce_i64(i32 %arg) {
 
 define i32 @reduce_i32(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umax.v2i32(<2 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umax.v4i32(<4 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umax.v8i32(<8 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umax.v16i32(<16 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.umax.v32i32(<32 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umax.v2i32(<2 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umax.v4i32(<4 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umax.v8i32(<8 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umax.v16i32(<16 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.umax.v32i32(<32 x i32> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i32'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umax.v2i32(<2 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umax.v4i32(<4 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umax.v8i32(<8 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umax.v16i32(<16 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.umax.v32i32(<32 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umax.v2i32(<2 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umax.v4i32(<4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umax.v8i32(<8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umax.v16i32(<16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.umax.v32i32(<32 x i32> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; SSE42-LABEL: 'reduce_i32'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umax.v2i32(<2 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umax.v4i32(<4 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umax.v8i32(<8 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umax.v16i32(<16 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.umax.v32i32(<32 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; SSE4-LABEL: 'reduce_i32'
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umax.v2i32(<2 x i32> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umax.v4i32(<4 x i32> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umax.v8i32(<8 x i32> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umax.v16i32(<16 x i32> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.umax.v32i32(<32 x i32> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i32'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umax.v2i32(<2 x i32> undef)
@@ -124,31 +133,31 @@ define i32 @reduce_i32(i32 %arg) {
 
 define i32 @reduce_i16(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.v2i16(<2 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umax.v4i16(<4 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umax.v8i16(<8 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umax.v16i16(<16 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umax.v32i16(<32 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umax.v64i16(<64 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.v2i16(<2 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umax.v4i16(<4 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umax.v8i16(<8 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umax.v16i16(<16 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umax.v32i16(<32 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umax.v64i16(<64 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i16'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.v2i16(<2 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umax.v4i16(<4 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umax.v8i16(<8 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umax.v16i16(<16 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umax.v32i16(<32 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umax.v64i16(<64 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.v2i16(<2 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umax.v4i16(<4 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umax.v8i16(<8 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umax.v16i16(<16 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umax.v32i16(<32 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umax.v64i16(<64 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; SSE42-LABEL: 'reduce_i16'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.v2i16(<2 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umax.v4i16(<4 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umax.v8i16(<8 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umax.v16i16(<16 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umax.v32i16(<32 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umax.v64i16(<64 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; SSE4-LABEL: 'reduce_i16'
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.v2i16(<2 x i16> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umax.v4i16(<4 x i16> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umax.v8i16(<8 x i16> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umax.v16i16(<16 x i16> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umax.v32i16(<32 x i16> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umax.v64i16(<64 x i16> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i16'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.v2i16(<2 x i16> undef)
@@ -206,34 +215,34 @@ define i32 @reduce_i16(i32 %arg) {
 
 define i32 @reduce_i8(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i8'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.v2i8(<2 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.v4i8(<4 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.v8i8(<8 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umax.v16i8(<16 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umax.v32i8(<32 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umax.v64i8(<64 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umax.v128i8(<128 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.v2i8(<2 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.v4i8(<4 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.v8i8(<8 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umax.v16i8(<16 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umax.v32i8(<32 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umax.v64i8(<64 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umax.v128i8(<128 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i8'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.v2i8(<2 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.v4i8(<4 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.v8i8(<8 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umax.v16i8(<16 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umax.v32i8(<32 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umax.v64i8(<64 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umax.v128i8(<128 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.v2i8(<2 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.v4i8(<4 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.v8i8(<8 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umax.v16i8(<16 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umax.v32i8(<32 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umax.v64i8(<64 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umax.v128i8(<128 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; SSE42-LABEL: 'reduce_i8'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.v2i8(<2 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.v4i8(<4 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.v8i8(<8 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umax.v16i8(<16 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umax.v32i8(<32 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umax.v64i8(<64 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umax.v128i8(<128 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; SSE4-LABEL: 'reduce_i8'
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.v2i8(<2 x i8> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.v4i8(<4 x i8> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.v8i8(<8 x i8> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umax.v16i8(<16 x i8> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umax.v32i8(<32 x i8> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umax.v64i8(<64 x i8> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umax.v128i8(<128 x i8> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i8'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.v2i8(<2 x i8> undef)
diff --git a/llvm/test/Analysis/CostModel/X86/reduce-umin.ll b/llvm/test/Analysis/CostModel/X86/reduce-umin.ll
index ce35e8b248485..1075e115a13b2 100644
--- a/llvm/test/Analysis/CostModel/X86/reduce-umin.ll
+++ b/llvm/test/Analysis/CostModel/X86/reduce-umin.ll
@@ -1,7 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
 ; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
 ; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+ssse3 | FileCheck %s --check-prefixes=CHECK,SSE,SSSE3
-; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+sse4.2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
+; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+sse4.1 | FileCheck %s --check-prefixes=CHECK,SSE,SSE4,SSE41
+; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+sse4.2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE4,SSE42
 ; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
 ; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
 ; RUN: opt < %s -cost-model -mtriple=x86_64-apple-darwin -analyze -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
@@ -11,26 +12,34 @@
 define i32 @reduce_i64(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i64'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.umin.v1i64(<1 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.umin.v2i64(<2 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.umin.v4i64(<4 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.umin.v8i64(<8 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.umin.v16i64(<16 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.umin.v2i64(<2 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.umin.v4i64(<4 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.umin.v8i64(<8 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.umin.v16i64(<16 x i64> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i64'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.umin.v1i64(<1 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.umin.v2i64(<2 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.umin.v4i64(<4 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.umin.v8i64(<8 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.umin.v16i64(<16 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.umin.v2i64(<2 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.umin.v4i64(<4 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.umin.v8i64(<8 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.umin.v16i64(<16 x i64> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SSE41-LABEL: 'reduce_i64'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.umin.v1i64(<1 x i64> undef)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.umin.v2i64(<2 x i64> undef)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.umin.v4i64(<4 x i64> undef)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.umin.v8i64(<8 x i64> undef)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.umin.v16i64(<16 x i64> undef)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; SSE42-LABEL: 'reduce_i64'
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.umin.v1i64(<1 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.umin.v2i64(<2 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.umin.v4i64(<4 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.umin.v8i64(<8 x i64> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.umin.v16i64(<16 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.umin.v2i64(<2 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.umin.v4i64(<4 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.umin.v8i64(<8 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.umin.v16i64(<16 x i64> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i64'
@@ -67,28 +76,28 @@ define i32 @reduce_i64(i32 %arg) {
 
 define i32 @reduce_i32(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umin.v2i32(<2 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umin.v4i32(<4 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umin.v8i32(<8 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umin.v16i32(<16 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.umin.v32i32(<32 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umin.v2i32(<2 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umin.v4i32(<4 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umin.v8i32(<8 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umin.v16i32(<16 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.umin.v32i32(<32 x i32> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i32'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umin.v2i32(<2 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umin.v4i32(<4 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umin.v8i32(<8 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umin.v16i32(<16 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.umin.v32i32(<32 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umin.v2i32(<2 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umin.v4i32(<4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umin.v8i32(<8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umin.v16i32(<16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.umin.v32i32(<32 x i32> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; SSE42-LABEL: 'reduce_i32'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umin.v2i32(<2 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umin.v4i32(<4 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umin.v8i32(<8 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umin.v16i32(<16 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.umin.v32i32(<32 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; SSE4-LABEL: 'reduce_i32'
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umin.v2i32(<2 x i32> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umin.v4i32(<4 x i32> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umin.v8i32(<8 x i32> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umin.v16i32(<16 x i32> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.umin.v32i32(<32 x i32> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i32'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umin.v2i32(<2 x i32> undef)
@@ -124,31 +133,31 @@ define i32 @reduce_i32(i32 %arg) {
 
 define i32 @reduce_i16(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.v2i16(<2 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umin.v4i16(<4 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umin.v8i16(<8 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umin.v16i16(<16 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umin.v32i16(<32 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umin.v64i16(<64 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.v2i16(<2 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umin.v4i16(<4 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umin.v8i16(<8 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umin.v16i16(<16 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umin.v32i16(<32 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umin.v64i16(<64 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i16'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.v2i16(<2 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umin.v4i16(<4 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umin.v8i16(<8 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umin.v16i16(<16 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umin.v32i16(<32 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umin.v64i16(<64 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.v2i16(<2 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umin.v4i16(<4 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umin.v8i16(<8 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umin.v16i16(<16 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umin.v32i16(<32 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umin.v64i16(<64 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; SSE42-LABEL: 'reduce_i16'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.v2i16(<2 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umin.v4i16(<4 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umin.v8i16(<8 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umin.v16i16(<16 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umin.v32i16(<32 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umin.v64i16(<64 x i16> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; SSE4-LABEL: 'reduce_i16'
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.v2i16(<2 x i16> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umin.v4i16(<4 x i16> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umin.v8i16(<8 x i16> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umin.v16i16(<16 x i16> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umin.v32i16(<32 x i16> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umin.v64i16(<64 x i16> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i16'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.v2i16(<2 x i16> undef)
@@ -206,34 +215,34 @@ define i32 @reduce_i16(i32 %arg) {
 
 define i32 @reduce_i8(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i8'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.v2i8(<2 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.v4i8(<4 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.v8i8(<8 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umin.v16i8(<16 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umin.v32i8(<32 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umin.v64i8(<64 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umin.v128i8(<128 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.v2i8(<2 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.v4i8(<4 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.v8i8(<8 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umin.v16i8(<16 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umin.v32i8(<32 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umin.v64i8(<64 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umin.v128i8(<128 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i8'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.v2i8(<2 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.v4i8(<4 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.v8i8(<8 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umin.v16i8(<16 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umin.v32i8(<32 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umin.v64i8(<64 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umin.v128i8(<128 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.v2i8(<2 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.v4i8(<4 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.v8i8(<8 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umin.v16i8(<16 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umin.v32i8(<32 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umin.v64i8(<64 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umin.v128i8(<128 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; SSE42-LABEL: 'reduce_i8'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.v2i8(<2 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.v4i8(<4 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.v8i8(<8 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umin.v16i8(<16 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umin.v32i8(<32 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umin.v64i8(<64 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umin.v128i8(<128 x i8> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; SSE4-LABEL: 'reduce_i8'
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.v2i8(<2 x i8> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.v4i8(<4 x i8> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.v8i8(<8 x i8> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umin.v16i8(<16 x i8> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umin.v32i8(<32 x i8> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umin.v64i8(<64 x i8> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umin.v128i8(<128 x i8> undef)
+; SSE4-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'reduce_i8'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.v2i8(<2 x i8> undef)
diff --git a/llvm/test/Analysis/DivergenceAnalysis/AMDGPU/always_uniform.ll b/llvm/test/Analysis/DivergenceAnalysis/AMDGPU/always_uniform.ll
index 2720028e5af46..3d94855340914 100644
--- a/llvm/test/Analysis/DivergenceAnalysis/AMDGPU/always_uniform.ll
+++ b/llvm/test/Analysis/DivergenceAnalysis/AMDGPU/always_uniform.ll
@@ -23,6 +23,13 @@ define amdgpu_kernel void @fcmp(float inreg %x, float inreg %y) {
   ret void
 }
 
+; CHECK-LABEL: for function 'ballot':
+define amdgpu_kernel void @ballot(i1 inreg %x) {
+; CHECK-NOT: DIVERGENT:  %ballot = call i64 @llvm.amdgcn.ballot.i32
+  %ballot = call i64 @llvm.amdgcn.ballot.i32(i1 %x)
+  ret void
+}
+
 ; SGPR asm outputs are uniform regardless of the input operands.
 ; CHECK-LABEL: for function 'asm_sgpr':
 ; CHECK: DIVERGENT: i32 %divergent
@@ -49,6 +56,7 @@ declare i32 @llvm.amdgcn.workitem.id.x() #0
 declare i32 @llvm.amdgcn.readfirstlane(i32) #0
 declare i64 @llvm.amdgcn.icmp.i32(i32, i32, i32) #1
 declare i64 @llvm.amdgcn.fcmp.i32(float, float, i32) #1
+declare i64 @llvm.amdgcn.ballot.i32(i1) #1
 
 attributes #0 = { nounwind readnone }
 attributes #1 = { nounwind readnone convergent }
diff --git a/llvm/test/Analysis/ScalarEvolution/pr44605.ll b/llvm/test/Analysis/ScalarEvolution/pr44605.ll
new file mode 100644
index 0000000000000..4d98d8d7f93df
--- /dev/null
+++ b/llvm/test/Analysis/ScalarEvolution/pr44605.ll
@@ -0,0 +1,67 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Only %local_3_4 is important here.
+;       All other instructions are needed to lure LLVM into executing
+;       specific code to trigger a bug.
+; RUN: opt < %s -indvars -S | FileCheck %s
+define i32 @test() {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[OUTER:%.*]]
+; CHECK:       outer:
+; CHECK-NEXT:    [[LOCAL_6_6:%.*]] = phi i32 [ 10, [[ENTRY:%.*]] ], [ [[TMP5:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[LOCAL_4_5:%.*]] = phi i32 [ 56587, [[ENTRY]] ], [ 0, [[LATCH]] ]
+; CHECK-NEXT:    [[LOCAL_3_4:%.*]] = phi i32 [ 2, [[ENTRY]] ], [ [[TMP5]], [[LATCH]] ]
+; CHECK-NEXT:    [[DOTUDIV:%.*]] = udiv i32 [[LOCAL_6_6]], 8361
+; CHECK-NEXT:    br label [[INNER:%.*]]
+; CHECK:       inner:
+; CHECK-NEXT:    [[LOCAL_7_3:%.*]] = phi i32 [ 2, [[OUTER]] ], [ [[TMP3:%.*]], [[INNER]] ]
+; CHECK-NEXT:    [[LOCAL_4_5_PN:%.*]] = phi i32 [ [[LOCAL_4_5]], [[OUTER]] ], [ [[TMP2:%.*]], [[INNER]] ]
+; CHECK-NEXT:    [[LOCAL_3_31:%.*]] = mul i32 [[LOCAL_4_5_PN]], [[DOTUDIV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = mul nuw nsw i32 [[LOCAL_7_3]], [[DOTUDIV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[TMP0]], [[LOCAL_3_4]]
+; CHECK-NEXT:    [[TMP2]] = add i32 [[TMP1]], [[LOCAL_3_31]]
+; CHECK-NEXT:    [[TMP3]] = add nuw nsw i32 [[LOCAL_7_3]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ugt i32 [[LOCAL_7_3]], 4
+; CHECK-NEXT:    br i1 [[TMP4]], label [[LATCH]], label [[INNER]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[DOTLCSSA:%.*]] = phi i32 [ [[TMP2]], [[INNER]] ]
+; CHECK-NEXT:    [[TMP5]] = add nuw nsw i32 [[LOCAL_6_6]], 1
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ugt i32 [[LOCAL_6_6]], 276
+; CHECK-NEXT:    br i1 [[TMP6]], label [[RETURN:%.*]], label [[OUTER]]
+; CHECK:       return:
+; CHECK-NEXT:    [[DOTLCSSA_LCSSA:%.*]] = phi i32 [ [[DOTLCSSA]], [[LATCH]] ]
+; CHECK-NEXT:    ret i32 [[DOTLCSSA_LCSSA]]
+;
+entry:
+  br label %outer
+
+outer:
+  %local_6_6 = phi i32 [ 10, %entry ], [ %5, %latch ]
+  %local_4_5 = phi i32 [ 56587, %entry ], [ 0, %latch ]
+  %local_3_4 = phi i32 [ 2, %entry ], [ %5, %latch ]
+  %.udiv = udiv i32 %local_6_6, 8361
+  br label %inner
+
+inner:
+  %local_7_3 = phi i32 [ 2, %outer ], [ %3, %inner ]
+  %local_4_5.pn = phi i32 [ %local_4_5, %outer ], [ %2, %inner ]
+  %local_3_31 = mul i32 %local_4_5.pn, %.udiv
+  %0 = mul i32 %local_7_3, %.udiv
+  %1 = sub i32 %0, %local_3_4
+  %2 = add i32 %1, %local_3_31
+  %3 = add nuw nsw i32 %local_7_3, 1
+  %4 = icmp ugt i32 %local_7_3, 4
+  br i1 %4, label %latch, label %inner
+
+latch:
+  %.lcssa = phi i32 [ %2, %inner ]
+  %5 = add nuw nsw i32 %local_6_6, 1
+  %6 = icmp ugt i32 %local_6_6, 276
+  br i1 %6, label %return, label %outer
+
+return:
+  %.lcssa.lcssa = phi i32 [ %.lcssa, %latch ]
+  ret i32 %.lcssa.lcssa
+
+}
+
diff --git a/llvm/test/CMakeLists.txt b/llvm/test/CMakeLists.txt
index 2722920334359..ec38d76ce27fb 100644
--- a/llvm/test/CMakeLists.txt
+++ b/llvm/test/CMakeLists.txt
@@ -22,12 +22,23 @@ configure_lit_site_cfg(
   ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
   MAIN_CONFIG
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py
+  PATHS
+  "LLVM_SOURCE_DIR"
+  "LLVM_BINARY_DIR"
+  "LLVM_TOOLS_DIR"
+  "LLVM_LIBRARY_DIR"
+  "SHLIBDIR"
   )
 configure_lit_site_cfg(
   ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.py.in
   ${CMAKE_CURRENT_BINARY_DIR}/Unit/lit.site.cfg.py
   MAIN_CONFIG
   ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.cfg.py
+  PATHS
+  "LLVM_SOURCE_DIR"
+  "LLVM_BINARY_DIR"
+  "LLVM_TOOLS_DIR"
+  "SHLIBDIR"
   )
 
 # Set the depends list as a variable so that it can grow conditionally.
@@ -39,6 +50,7 @@ set(LLVM_TEST_DEPENDS
           UnitTests
           bugpoint
           count
+          file-table-tform
           llc
           lli
           lli-child-target
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll
index 418d09d01fd33..2b498d2ab9a49 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll
@@ -120,7 +120,7 @@ end:
   br label %block
 }
 
-; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to legalize instruction: %2:_(<2 x p0>) = G_INSERT_VECTOR_ELT %0:_, %3:_(p0), %5:_(s32) (in function: vector_of_pointers_insertelement)
+; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to legalize instruction: %2:_(<2 x p0>) = G_INSERT_VECTOR_ELT %0:_, %{{[0-9]+}}:_(p0), %{{[0-9]+}}:_(s32) (in function: vector_of_pointers_insertelement)
 ; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for vector_of_pointers_insertelement
 ; FALLBACK-WITH-REPORT-OUT-LABEL: vector_of_pointers_insertelement:
 define void @vector_of_pointers_insertelement() {
@@ -153,7 +153,9 @@ define void @nonpow2_add_narrowing() {
 define void @nonpow2_or_narrowing() {
   %a = add i128 undef, undef
   %b = trunc i128 %a to i96
-  %dummy = or i96 %b, %b
+  %a2 = add i128 undef, undef
+  %b2 = trunc i128 %a2 to i96
+  %dummy = or i96 %b, %b2
   store i96 %dummy, i96* undef
   ret void
 }
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator-switch.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator-switch.ll
index bc973b34d6fd0..485fa62904f0a 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator-switch.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator-switch.ll
@@ -285,62 +285,62 @@ define void @jt_multiple_jump_tables(%1* %arg, i32 %arg1, i32* %arg2) {
   ; CHECK:   [[C53:%[0-9]+]]:_(s32) = G_CONSTANT i32 4354
   ; CHECK:   [[C54:%[0-9]+]]:_(s32) = G_CONSTANT i32 4355
   ; CHECK:   [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @global
-  ; CHECK:   [[C55:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; CHECK:   [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[C55]](s64)
-  ; CHECK:   [[C56:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-  ; CHECK:   [[C57:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-  ; CHECK:   [[C58:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
-  ; CHECK:   [[C59:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-  ; CHECK:   [[C60:%[0-9]+]]:_(s64) = G_CONSTANT i64 5
-  ; CHECK:   [[C61:%[0-9]+]]:_(s64) = G_CONSTANT i64 6
-  ; CHECK:   [[C62:%[0-9]+]]:_(s64) = G_CONSTANT i64 7
-  ; CHECK:   [[C63:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; CHECK:   [[C64:%[0-9]+]]:_(s64) = G_CONSTANT i64 9
-  ; CHECK:   [[C65:%[0-9]+]]:_(s64) = G_CONSTANT i64 10
-  ; CHECK:   [[C66:%[0-9]+]]:_(s64) = G_CONSTANT i64 11
-  ; CHECK:   [[C67:%[0-9]+]]:_(s64) = G_CONSTANT i64 12
-  ; CHECK:   [[C68:%[0-9]+]]:_(s64) = G_CONSTANT i64 13
-  ; CHECK:   [[C69:%[0-9]+]]:_(s64) = G_CONSTANT i64 14
-  ; CHECK:   [[C70:%[0-9]+]]:_(s64) = G_CONSTANT i64 15
-  ; CHECK:   [[C71:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-  ; CHECK:   [[C72:%[0-9]+]]:_(s64) = G_CONSTANT i64 17
-  ; CHECK:   [[C73:%[0-9]+]]:_(s64) = G_CONSTANT i64 18
-  ; CHECK:   [[C74:%[0-9]+]]:_(s64) = G_CONSTANT i64 19
-  ; CHECK:   [[C75:%[0-9]+]]:_(s64) = G_CONSTANT i64 20
-  ; CHECK:   [[C76:%[0-9]+]]:_(s64) = G_CONSTANT i64 21
-  ; CHECK:   [[C77:%[0-9]+]]:_(s64) = G_CONSTANT i64 22
-  ; CHECK:   [[C78:%[0-9]+]]:_(s64) = G_CONSTANT i64 23
-  ; CHECK:   [[C79:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
-  ; CHECK:   [[C80:%[0-9]+]]:_(s64) = G_CONSTANT i64 25
-  ; CHECK:   [[C81:%[0-9]+]]:_(s64) = G_CONSTANT i64 26
-  ; CHECK:   [[C82:%[0-9]+]]:_(s64) = G_CONSTANT i64 27
-  ; CHECK:   [[C83:%[0-9]+]]:_(s64) = G_CONSTANT i64 28
-  ; CHECK:   [[C84:%[0-9]+]]:_(s64) = G_CONSTANT i64 29
-  ; CHECK:   [[C85:%[0-9]+]]:_(s64) = G_CONSTANT i64 30
-  ; CHECK:   [[C86:%[0-9]+]]:_(s64) = G_CONSTANT i64 31
-  ; CHECK:   [[C87:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
-  ; CHECK:   [[C88:%[0-9]+]]:_(s64) = G_CONSTANT i64 33
-  ; CHECK:   [[C89:%[0-9]+]]:_(s64) = G_CONSTANT i64 34
-  ; CHECK:   [[C90:%[0-9]+]]:_(s64) = G_CONSTANT i64 35
-  ; CHECK:   [[C91:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
-  ; CHECK:   [[C92:%[0-9]+]]:_(s64) = G_CONSTANT i64 37
-  ; CHECK:   [[C93:%[0-9]+]]:_(s64) = G_CONSTANT i64 38
-  ; CHECK:   [[C94:%[0-9]+]]:_(s64) = G_CONSTANT i64 39
-  ; CHECK:   [[C95:%[0-9]+]]:_(s64) = G_CONSTANT i64 40
-  ; CHECK:   [[C96:%[0-9]+]]:_(s64) = G_CONSTANT i64 41
-  ; CHECK:   [[C97:%[0-9]+]]:_(s64) = G_CONSTANT i64 42
-  ; CHECK:   [[C98:%[0-9]+]]:_(s64) = G_CONSTANT i64 43
-  ; CHECK:   [[C99:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
-  ; CHECK:   [[C100:%[0-9]+]]:_(s64) = G_CONSTANT i64 45
-  ; CHECK:   [[C101:%[0-9]+]]:_(s64) = G_CONSTANT i64 46
-  ; CHECK:   [[C102:%[0-9]+]]:_(s64) = G_CONSTANT i64 47
-  ; CHECK:   [[C103:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
-  ; CHECK:   [[C104:%[0-9]+]]:_(s64) = G_CONSTANT i64 49
-  ; CHECK:   [[C105:%[0-9]+]]:_(s64) = G_CONSTANT i64 50
-  ; CHECK:   [[C106:%[0-9]+]]:_(s64) = G_CONSTANT i64 51
-  ; CHECK:   [[C107:%[0-9]+]]:_(s64) = G_CONSTANT i64 52
-  ; CHECK:   [[C108:%[0-9]+]]:_(s64) = G_CONSTANT i64 53
-  ; CHECK:   [[C109:%[0-9]+]]:_(s64) = G_CONSTANT i64 54
+  ; CHECK:   [[C55:%[0-9]+]]:_(p0) = G_CONSTANT i64 0
+  ; CHECK:   [[C56:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[C57:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+  ; CHECK:   [[C58:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+  ; CHECK:   [[C59:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
+  ; CHECK:   [[C60:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; CHECK:   [[C61:%[0-9]+]]:_(s64) = G_CONSTANT i64 5
+  ; CHECK:   [[C62:%[0-9]+]]:_(s64) = G_CONSTANT i64 6
+  ; CHECK:   [[C63:%[0-9]+]]:_(s64) = G_CONSTANT i64 7
+  ; CHECK:   [[C64:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+  ; CHECK:   [[C65:%[0-9]+]]:_(s64) = G_CONSTANT i64 9
+  ; CHECK:   [[C66:%[0-9]+]]:_(s64) = G_CONSTANT i64 10
+  ; CHECK:   [[C67:%[0-9]+]]:_(s64) = G_CONSTANT i64 11
+  ; CHECK:   [[C68:%[0-9]+]]:_(s64) = G_CONSTANT i64 12
+  ; CHECK:   [[C69:%[0-9]+]]:_(s64) = G_CONSTANT i64 13
+  ; CHECK:   [[C70:%[0-9]+]]:_(s64) = G_CONSTANT i64 14
+  ; CHECK:   [[C71:%[0-9]+]]:_(s64) = G_CONSTANT i64 15
+  ; CHECK:   [[C72:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+  ; CHECK:   [[C73:%[0-9]+]]:_(s64) = G_CONSTANT i64 17
+  ; CHECK:   [[C74:%[0-9]+]]:_(s64) = G_CONSTANT i64 18
+  ; CHECK:   [[C75:%[0-9]+]]:_(s64) = G_CONSTANT i64 19
+  ; CHECK:   [[C76:%[0-9]+]]:_(s64) = G_CONSTANT i64 20
+  ; CHECK:   [[C77:%[0-9]+]]:_(s64) = G_CONSTANT i64 21
+  ; CHECK:   [[C78:%[0-9]+]]:_(s64) = G_CONSTANT i64 22
+  ; CHECK:   [[C79:%[0-9]+]]:_(s64) = G_CONSTANT i64 23
+  ; CHECK:   [[C80:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
+  ; CHECK:   [[C81:%[0-9]+]]:_(s64) = G_CONSTANT i64 25
+  ; CHECK:   [[C82:%[0-9]+]]:_(s64) = G_CONSTANT i64 26
+  ; CHECK:   [[C83:%[0-9]+]]:_(s64) = G_CONSTANT i64 27
+  ; CHECK:   [[C84:%[0-9]+]]:_(s64) = G_CONSTANT i64 28
+  ; CHECK:   [[C85:%[0-9]+]]:_(s64) = G_CONSTANT i64 29
+  ; CHECK:   [[C86:%[0-9]+]]:_(s64) = G_CONSTANT i64 30
+  ; CHECK:   [[C87:%[0-9]+]]:_(s64) = G_CONSTANT i64 31
+  ; CHECK:   [[C88:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
+  ; CHECK:   [[C89:%[0-9]+]]:_(s64) = G_CONSTANT i64 33
+  ; CHECK:   [[C90:%[0-9]+]]:_(s64) = G_CONSTANT i64 34
+  ; CHECK:   [[C91:%[0-9]+]]:_(s64) = G_CONSTANT i64 35
+  ; CHECK:   [[C92:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
+  ; CHECK:   [[C93:%[0-9]+]]:_(s64) = G_CONSTANT i64 37
+  ; CHECK:   [[C94:%[0-9]+]]:_(s64) = G_CONSTANT i64 38
+  ; CHECK:   [[C95:%[0-9]+]]:_(s64) = G_CONSTANT i64 39
+  ; CHECK:   [[C96:%[0-9]+]]:_(s64) = G_CONSTANT i64 40
+  ; CHECK:   [[C97:%[0-9]+]]:_(s64) = G_CONSTANT i64 41
+  ; CHECK:   [[C98:%[0-9]+]]:_(s64) = G_CONSTANT i64 42
+  ; CHECK:   [[C99:%[0-9]+]]:_(s64) = G_CONSTANT i64 43
+  ; CHECK:   [[C100:%[0-9]+]]:_(s64) = G_CONSTANT i64 44
+  ; CHECK:   [[C101:%[0-9]+]]:_(s64) = G_CONSTANT i64 45
+  ; CHECK:   [[C102:%[0-9]+]]:_(s64) = G_CONSTANT i64 46
+  ; CHECK:   [[C103:%[0-9]+]]:_(s64) = G_CONSTANT i64 47
+  ; CHECK:   [[C104:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
+  ; CHECK:   [[C105:%[0-9]+]]:_(s64) = G_CONSTANT i64 49
+  ; CHECK:   [[C106:%[0-9]+]]:_(s64) = G_CONSTANT i64 50
+  ; CHECK:   [[C107:%[0-9]+]]:_(s64) = G_CONSTANT i64 51
+  ; CHECK:   [[C108:%[0-9]+]]:_(s64) = G_CONSTANT i64 52
+  ; CHECK:   [[C109:%[0-9]+]]:_(s64) = G_CONSTANT i64 53
+  ; CHECK:   [[C110:%[0-9]+]]:_(s64) = G_CONSTANT i64 54
   ; CHECK:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.tmp
   ; CHECK:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C]]
   ; CHECK:   G_BRCOND [[ICMP]](s1), %bb.56
@@ -778,12 +778,12 @@ define void @jt_multiple_jump_tables(%1* %arg, i32 %arg1, i32* %arg2) {
   ; CHECK:   successors: %bb.56(0x80000000)
   ; CHECK: bb.56.bb57:
   ; CHECK:   successors: %bb.59(0x80000000)
-  ; CHECK:   [[PHI:%[0-9]+]]:_(s64) = G_PHI [[C55]](s64), %bb.1, [[C56]](s64), %bb.2, [[C57]](s64), %bb.3, [[C58]](s64), %bb.4, [[C59]](s64), %bb.5, [[C60]](s64), %bb.6, [[C61]](s64), %bb.7, [[C62]](s64), %bb.8, [[C63]](s64), %bb.9, [[C64]](s64), %bb.10, [[C65]](s64), %bb.11, [[C66]](s64), %bb.12, [[C67]](s64), %bb.13, [[C68]](s64), %bb.14, [[C69]](s64), %bb.15, [[C70]](s64), %bb.16, [[C71]](s64), %bb.17, [[C72]](s64), %bb.18, [[C73]](s64), %bb.19, [[C74]](s64), %bb.20, [[C75]](s64), %bb.21, [[C76]](s64), %bb.22, [[C77]](s64), %bb.23, [[C78]](s64), %bb.24, [[C79]](s64), %bb.25, [[C80]](s64), %bb.26, [[C81]](s64), %bb.27, [[C82]](s64), %bb.28, [[C83]](s64), %bb.29, [[C84]](s64), %bb.30, [[C85]](s64), %bb.31, [[C86]](s64), %bb.32, [[C87]](s64), %bb.33, [[C88]](s64), %bb.34, [[C89]](s64), %bb.35, [[C90]](s64), %bb.36, [[C91]](s64), %bb.37, [[C92]](s64), %bb.38, [[C93]](s64), %bb.39, [[C94]](s64), %bb.40, [[C95]](s64), %bb.41, [[C96]](s64), %bb.42, [[C97]](s64), %bb.43, [[C98]](s64), %bb.44, [[C99]](s64), %bb.45, [[C100]](s64), %bb.46, [[C101]](s64), %bb.47, [[C102]](s64), %bb.48, [[C103]](s64), %bb.49, [[C104]](s64), %bb.50, [[C105]](s64), %bb.51, [[C106]](s64), %bb.52, [[C107]](s64), %bb.53, [[C108]](s64), %bb.54, [[C109]](s64), %bb.55
-  ; CHECK:   [[C110:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
-  ; CHECK:   [[MUL:%[0-9]+]]:_(s64) = G_MUL [[PHI]], [[C110]]
+  ; CHECK:   [[PHI:%[0-9]+]]:_(s64) = G_PHI [[C56]](s64), %bb.1, [[C57]](s64), %bb.2, [[C58]](s64), %bb.3, [[C59]](s64), %bb.4, [[C60]](s64), %bb.5, [[C61]](s64), %bb.6, [[C62]](s64), %bb.7, [[C63]](s64), %bb.8, [[C64]](s64), %bb.9, [[C65]](s64), %bb.10, [[C66]](s64), %bb.11, [[C67]](s64), %bb.12, [[C68]](s64), %bb.13, [[C69]](s64), %bb.14, [[C70]](s64), %bb.15, [[C71]](s64), %bb.16, [[C72]](s64), %bb.17, [[C73]](s64), %bb.18, [[C74]](s64), %bb.19, [[C75]](s64), %bb.20, [[C76]](s64), %bb.21, [[C77]](s64), %bb.22, [[C78]](s64), %bb.23, [[C79]](s64), %bb.24, [[C80]](s64), %bb.25, [[C81]](s64), %bb.26, [[C82]](s64), %bb.27, [[C83]](s64), %bb.28, [[C84]](s64), %bb.29, [[C85]](s64), %bb.30, [[C86]](s64), %bb.31, [[C87]](s64), %bb.32, [[C88]](s64), %bb.33, [[C89]](s64), %bb.34, [[C90]](s64), %bb.35, [[C91]](s64), %bb.36, [[C92]](s64), %bb.37, [[C93]](s64), %bb.38, [[C94]](s64), %bb.39, [[C95]](s64), %bb.40, [[C96]](s64), %bb.41, [[C97]](s64), %bb.42, [[C98]](s64), %bb.43, [[C99]](s64), %bb.44, [[C100]](s64), %bb.45, [[C101]](s64), %bb.46, [[C102]](s64), %bb.47, [[C103]](s64), %bb.48, [[C104]](s64), %bb.49, [[C105]](s64), %bb.50, [[C106]](s64), %bb.51, [[C107]](s64), %bb.52, [[C108]](s64), %bb.53, [[C109]](s64), %bb.54, [[C110]](s64), %bb.55
+  ; CHECK:   [[C111:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+  ; CHECK:   [[MUL:%[0-9]+]]:_(s64) = G_MUL [[PHI]], [[C111]]
   ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[GV]], [[MUL]](s64)
-  ; CHECK:   [[C111:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; CHECK:   [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C111]](s64)
+  ; CHECK:   [[C112:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+  ; CHECK:   [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C112]](s64)
   ; CHECK:   [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[PTR_ADD1]](p0) :: (load 8 from %ir.tmp59)
   ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
   ; CHECK:   $x0 = COPY [[COPY]](p0)
@@ -819,7 +819,7 @@ define void @jt_multiple_jump_tables(%1* %arg, i32 %arg1, i32* %arg2) {
   ; CHECK:   RET_ReallyLR
   ; CHECK: bb.60.bb69:
   ; CHECK:   successors: %bb.58(0x40000000), %bb.57(0x40000000)
-  ; CHECK:   [[ICMP55:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[COPY2]](p0), [[INTTOPTR]]
+  ; CHECK:   [[ICMP55:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[COPY2]](p0), [[C55]]
   ; CHECK:   G_BRCOND [[ICMP55]](s1), %bb.58
   ; CHECK:   G_BR %bb.57
 bb:
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
index f200cf2f0d241..c023b8d2f9e11 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
@@ -594,8 +594,7 @@ define i32 @test_urem(i32 %arg1, i32 %arg2) {
 }
 
 ; CHECK-LABEL: name: test_constant_null
-; CHECK: [[ZERO:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-; CHECK: [[NULL:%[0-9]+]]:_(p0) = G_INTTOPTR [[ZERO]]
+; CHECK: [[NULL:%[0-9]+]]:_(p0) = G_CONSTANT i64 0
 ; CHECK: $x0 = COPY [[NULL]]
 define i8* @test_constant_null() {
   ret i8* null
@@ -1396,6 +1395,30 @@ define i32 @test_bitreverse_intrinsic(i32 %a) {
   ret i32 %res
 }
 
+declare i32 @llvm.fshl.i32(i32, i32, i32)
+define i32 @test_fshl_intrinsic(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: name: test_fshl_intrinsic
+; CHECK: [[A:%[0-9]+]]:_(s32) = COPY $w0
+; CHECK: [[B:%[0-9]+]]:_(s32) = COPY $w1
+; CHECK: [[C:%[0-9]+]]:_(s32) = COPY $w2
+; CHECK: [[RES:%[0-9]+]]:_(s32) = G_FSHL [[A]], [[B]], [[C]]
+; CHECK: $w0 = COPY [[RES]]
+  %res = call i32 @llvm.fshl.i32(i32 %a, i32 %b, i32 %c)
+  ret i32 %res
+}
+
+declare i32 @llvm.fshr.i32(i32, i32, i32)
+define i32 @test_fshr_intrinsic(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: name: test_fshr_intrinsic
+; CHECK: [[A:%[0-9]+]]:_(s32) = COPY $w0
+; CHECK: [[B:%[0-9]+]]:_(s32) = COPY $w1
+; CHECK: [[C:%[0-9]+]]:_(s32) = COPY $w2
+; CHECK: [[RES:%[0-9]+]]:_(s32) = G_FSHR [[A]], [[B]], [[C]]
+; CHECK: $w0 = COPY [[RES]]
+  %res = call i32 @llvm.fshr.i32(i32 %a, i32 %b, i32 %c)
+  ret i32 %res
+}
+
 declare void @llvm.lifetime.start.p0i8(i64, i8*)
 declare void @llvm.lifetime.end.p0i8(i64, i8*)
 define void @test_lifetime_intrin() {
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-tail-call.ll b/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-tail-call.ll
index d028147d5987a..b96b1498e9fef 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-tail-call.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-tail-call.ll
@@ -260,9 +260,8 @@ define void @foo(i32*) {
   ; COMMON: bb.1 (%ir-block.1):
   ; COMMON:   liveins: $x0
   ; COMMON:   [[COPY:%[0-9]+]]:_(p0) = COPY $x0
-  ; COMMON:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; COMMON:   [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[C]](s64)
-  ; COMMON:   $x0 = COPY [[INTTOPTR]](p0)
+  ; COMMON:   [[C:%[0-9]+]]:_(p0) = G_CONSTANT i64 0
+  ; COMMON:   $x0 = COPY [[C]](p0)
   ; COMMON:   TCRETURNdi @must_callee, 0, csr_aarch64_aapcs, implicit $sp, implicit $x0
   musttail call void @must_callee(i8* null)
   ret void
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/call-translator.ll b/llvm/test/CodeGen/AArch64/GlobalISel/call-translator.ll
index c1fe9782580d5..8a96eb8c1cb4e 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/call-translator.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/call-translator.ll
@@ -193,8 +193,7 @@ define void @test_stack_slots([8 x i64], i64 %lhs, i64 %rhs, i64* %addr) {
 ; CHECK-LABEL: name: test_call_stack
 ; CHECK: [[C42:%[0-9]+]]:_(s64) = G_CONSTANT i64 42
 ; CHECK: [[C12:%[0-9]+]]:_(s64) = G_CONSTANT i64 12
-; CHECK: [[ZERO:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-; CHECK: [[PTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[ZERO]]
+; CHECK: [[PTR:%[0-9]+]]:_(p0) = G_CONSTANT i64 0
 ; CHECK: ADJCALLSTACKDOWN 24, 0, implicit-def $sp, implicit $sp
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[C42_OFFS:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/dynamic-alloca.ll b/llvm/test/CodeGen/AArch64/GlobalISel/dynamic-alloca.ll
index 47b3878177998..ad2a5fd8c1f8d 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/dynamic-alloca.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/dynamic-alloca.ll
@@ -13,7 +13,7 @@ define i8* @test_simple_alloca(i32 %numelts) {
   ; CHECK:   [[ADD:%[0-9]+]]:_(s64) = nuw G_ADD [[MUL]], [[C1]]
   ; CHECK:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -16
   ; CHECK:   [[AND:%[0-9]+]]:_(s64) = G_AND [[ADD]], [[C2]]
-  ; CHECK:   [[DYN_STACKALLOC:%[0-9]+]]:_(p0) = G_DYN_STACKALLOC [[AND]](s64), 0
+  ; CHECK:   [[DYN_STACKALLOC:%[0-9]+]]:_(p0) = G_DYN_STACKALLOC [[AND]](s64), 1
   ; CHECK:   $x0 = COPY [[DYN_STACKALLOC]](p0)
   ; CHECK:   RET_ReallyLR implicit $x0
   %addr = alloca i8, i32 %numelts
@@ -51,7 +51,7 @@ define i128* @test_natural_alloca(i32 %numelts) {
   ; CHECK:   [[ADD:%[0-9]+]]:_(s64) = nuw G_ADD [[MUL]], [[C1]]
   ; CHECK:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -16
   ; CHECK:   [[AND:%[0-9]+]]:_(s64) = G_AND [[ADD]], [[C2]]
-  ; CHECK:   [[DYN_STACKALLOC:%[0-9]+]]:_(p0) = G_DYN_STACKALLOC [[AND]](s64), 0
+  ; CHECK:   [[DYN_STACKALLOC:%[0-9]+]]:_(p0) = G_DYN_STACKALLOC [[AND]](s64), 1
   ; CHECK:   $x0 = COPY [[DYN_STACKALLOC]](p0)
   ; CHECK:   RET_ReallyLR implicit $x0
   %addr = alloca i128, i32 %numelts
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-exceptions.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-exceptions.ll
index bb393e08d5f31..a74e001a4928f 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-exceptions.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-exceptions.ll
@@ -59,8 +59,7 @@ continue:
 
 ; CHECK-LABEL: name: test_invoke_varargs
 
-; CHECK: [[ZERO:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-; CHECK: [[NULL:%[0-9]+]]:_(p0) = G_INTTOPTR [[ZERO]]
+; CHECK: [[NULL:%[0-9]+]]:_(p0) = G_CONSTANT i64 0
 ; CHECK: [[ANSWER:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
 ; CHECK: [[ONE:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.0
 
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-tbaa.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-tbaa.ll
index bbf5747039724..33417d8a1fc66 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-tbaa.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-tbaa.ll
@@ -8,8 +8,8 @@ bb:
   %tmp5 = getelementptr i16, i16* null, i64 2
   %tmp6 = load i16, i16* %tmp1, align 2, !tbaa !0
   store i16 %tmp6, i16* %tmp5, align 2, !tbaa !0
-  ; CHECK: %5:_(s16) = G_LOAD %2(p0) :: (load 2 from %ir.tmp1, !tbaa !0)
-  ; CHECK: G_STORE %5(s16), %4(p0) :: (store 2 into %ir.tmp5, !tbaa !0)
+  ; CHECK: [[LOAD:%[0-9]+]]:_(s16) = G_LOAD %{{[0-9]+}}(p0) :: (load 2 from %ir.tmp1, !tbaa !0)
+  ; CHECK: G_STORE [[LOAD]](s16), %{{[0-9]+}}(p0) :: (store 2 into %ir.tmp5, !tbaa !0)
   ret void
 }
 
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index aa0ef61094616..7163ecee5f6e3 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -263,6 +263,12 @@
 # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: G_FSHL (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
+# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: G_FSHR (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
+# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: G_ICMP (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
@@ -314,16 +320,16 @@
 # DEBUG-NEXT: G_SMULH (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. the first uncovered type index: 1, OK
 # DEBUG-NEXT: .. the first uncovered imm index: 0, OK
-# DEBUG-NEXT: G_UADDSAT (opcode 117): 1 type index, 0 imm indices
+# DEBUG-NEXT: G_UADDSAT (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: G_SADDSAT (opcode 118): 1 type index, 0 imm indices
+# DEBUG-NEXT: G_SADDSAT (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: G_USUBSAT (opcode 119): 1 type index, 0 imm indices
+# DEBUG-NEXT: G_USUBSAT (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: G_SSUBSAT (opcode 120): 1 type index, 0 imm indices
+# DEBUG-NEXT: G_SSUBSAT (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: G_FADD (opcode {{[0-9]+}}): 1 type index, 0 imm indices
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-binop-same-val.mir b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-binop-same-val.mir
new file mode 100644
index 0000000000000..3c70c41719839
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-binop-same-val.mir
@@ -0,0 +1,96 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple aarch64 -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+
+name:            or_same
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0
+
+    ; Fold: x or x -> x
+    ; CHECK-LABEL: name: or_same
+    ; CHECK: liveins: $x0
+    ; CHECK: %copy:_(s64) = COPY $x0
+    ; CHECK: $x0 = COPY %copy(s64)
+    ; CHECK: RET_ReallyLR implicit $x0
+    %copy:_(s64) = COPY $x0
+    %or:_(s64) = G_OR %copy, %copy
+    $x0 = COPY %or(s64)
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            and_same
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0
+
+    ; Fold: x and x -> x
+
+    ; CHECK-LABEL: name: and_same
+    ; CHECK: liveins: $x0
+    ; CHECK: %copy:_(s64) = COPY $x0
+    ; CHECK: $x0 = COPY %copy(s64)
+    ; CHECK: RET_ReallyLR implicit $x0
+    %copy:_(s64) = COPY $x0
+    %and:_(s64) = G_AND %copy, %copy
+    $x0 = COPY %and(s64)
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            and_same2
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0, $x1
+
+    ; We can fold when the LHS and RHS are guaranteed to be identical.
+
+    ; CHECK-LABEL: name: and_same2
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK: %copy1:_(s64) = COPY $x0
+    ; CHECK: %copy2:_(s64) = COPY $x1
+    ; CHECK: %or:_(s64) = G_OR %copy1, %copy2
+    ; CHECK: $x0 = COPY %or(s64)
+    ; CHECK: RET_ReallyLR implicit $x0
+    %copy1:_(s64) = COPY $x0
+    %copy2:_(s64) = COPY $x1
+    %or:_(s64) = G_OR %copy1, %copy2
+    %same_as_or:_(s64) = COPY %or(s64)
+    %and:_(s64) = G_AND %or, %same_as_or
+    $x0 = COPY %and(s64)
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            or_and_not_same
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0, $x1, $x2
+
+    ; None of the G_ORs or G_ANDs should be eliminated here, because their LHS
+    ; and RHS values are different.
+
+    ; CHECK-LABEL: name: or_and_not_same
+    ; CHECK: liveins: $x0, $x1, $x2
+    ; CHECK: %copy1:_(s64) = COPY $x0
+    ; CHECK: %copy2:_(s64) = COPY $x1
+    ; CHECK: %copy3:_(s64) = COPY $x2
+    ; CHECK: %or1:_(s64) = G_OR %copy1, %copy2
+    ; CHECK: %or2:_(s64) = G_OR %copy1, %copy3
+    ; CHECK: %and:_(s64) = G_AND %or1, %or2
+    ; CHECK: $x0 = COPY %and(s64)
+    ; CHECK: RET_ReallyLR implicit $x0
+    %copy1:_(s64) = COPY $x0
+    %copy2:_(s64) = COPY $x1
+    %copy3:_(s64) = COPY $x2
+    %or1:_(s64) = G_OR %copy1, %copy2
+    %or2:_(s64) = G_OR %copy1, %copy3
+    %and:_(s64) = G_AND %or1, %or2
+    $x0 = COPY %and(s64)
+    RET_ReallyLR implicit $x0
+
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-select.mir b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-select.mir
index 646d7f8eea3fa..b842b24d4226a 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-select.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-select.mir
@@ -1,10 +1,18 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple aarch64 -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
-
+--- |
+  declare void @foo()
+  define void @self() { ret void }
+  define void @self_with_copy() { ret void }
+  define void @self_not_equivalent_overwrite_w0() { ret void }
+  define void @self_not_equivalent_overwrite_w0_implicit() { ret void }
+  define void @self_not_equivalent_different_copies() { ret void }
+...
+---
 name:            self
 tracksRegLiveness: true
 body:             |
-  bb.1.entry:
+  bb.0:
     liveins: $w0, $w1
     ; Optimize (cond ? %a : %a) -> %a
     ; CHECK-LABEL: name: self
@@ -24,7 +32,7 @@ body:             |
 name:            self_with_copy
 tracksRegLiveness: true
 body:             |
-  bb.1.entry:
+  bb.0:
     liveins: $w0, $w1
     ; Optimize (cond ? %a : %b) -> %a
     ;
@@ -46,22 +54,27 @@ body:             |
 
 ...
 ---
-name:            self_with_equivalent
+name:            self_not_equivalent_overwrite_w0
 tracksRegLiveness: true
 body:             |
-  bb.1.entry:
+  bb.0:
     liveins: $w0, $w1
-    ; Optimize (cond ? %a : %b) -> %a
     ;
-    ; This shows that we can detect when %a == %b, even though they define
-    ; different virtual registers.
+    ; $w0 is overwritten by a copy from $w1, so the copies for %a and %b are
+    ; not the same.
     ;
-    ; CHECK-LABEL: name: self_with_equivalent
+    ; CHECK-LABEL: name: self_not_equivalent_overwrite_w0
     ; CHECK: liveins: $w0, $w1
     ; CHECK: %a:_(s32) = COPY $w0
-    ; CHECK: $w0 = COPY %a(s32)
+    ; CHECK: $w0 = COPY $w1
+    ; CHECK: %b:_(s32) = COPY $w0
+    ; CHECK: %cond_wide:gpr(s32) = COPY $w1
+    ; CHECK: %cond:gpr(s1) = G_TRUNC %cond_wide(s32)
+    ; CHECK: %select:_(s32) = G_SELECT %cond(s1), %a, %b
+    ; CHECK: $w0 = COPY %select(s32)
     ; CHECK: RET_ReallyLR implicit $w0
     %a:_(s32) = COPY $w0
+    $w0 = COPY $w1
     %b:_(s32) = COPY $w0
     %cond_wide:gpr(s32) = COPY $w1
     %cond:gpr(s1) = G_TRUNC %cond_wide(s32)
@@ -71,13 +84,44 @@ body:             |
 
 ...
 ---
-name:            self_not_equivalent
+name:            self_not_equivalent_overwrite_w0_implicit
 tracksRegLiveness: true
 body:             |
-  bb.1.entry:
+  bb.0:
+    liveins: $w0, $w1
+    ;
+    ; $w0 is overwritten by a call which defines it implicitly, so the copies
+    ; are not the same.
+    ;
+    ; CHECK-LABEL: name: self_not_equivalent_overwrite_w0_implicit
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK: %a:_(s32) = COPY $w0
+    ; CHECK: BL @foo, implicit-def $w0
+    ; CHECK: %b:_(s32) = COPY $w0
+    ; CHECK: %cond_wide:gpr(s32) = COPY $w1
+    ; CHECK: %cond:gpr(s1) = G_TRUNC %cond_wide(s32)
+    ; CHECK: %select:_(s32) = G_SELECT %cond(s1), %a, %b
+    ; CHECK: $w0 = COPY %select(s32)
+    ; CHECK: RET_ReallyLR implicit $w0
+    %a:_(s32) = COPY $w0
+    BL @foo, implicit-def $w0
+    %b:_(s32) = COPY $w0
+    %cond_wide:gpr(s32) = COPY $w1
+    %cond:gpr(s1) = G_TRUNC %cond_wide(s32)
+    %select:_(s32) = G_SELECT %cond(s1), %a, %b
+    $w0 = COPY %select(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            self_not_equivalent_different_copies
+tracksRegLiveness: true
+body:             |
+  bb.0:
     liveins: $w0, $w1
     ; In this case, the copies are not equivalent, so there is no optimization.
-    ; CHECK-LABEL: name: self_not_equivalent
+    ;
+    ; CHECK-LABEL: name: self_not_equivalent_different_copies
     ; CHECK: liveins: $w0, $w1
     ; CHECK: %a:_(s32) = COPY $w0
     ; CHECK: %b:_(s32) = COPY $w1
@@ -88,8 +132,11 @@ body:             |
     ; CHECK: RET_ReallyLR implicit $w0
     %a:_(s32) = COPY $w0
     %b:_(s32) = COPY $w1
+    %c:_(s32) = COPY %b
     %cond_wide:gpr(s32) = COPY $w1
     %cond:gpr(s1) = G_TRUNC %cond_wide(s32)
-    %select:_(s32) = G_SELECT %cond(s1), %a, %b
+    %select:_(s32) = G_SELECT %cond(s1), %a, %c
     $w0 = COPY %select(s32)
     RET_ReallyLR implicit $w0
+
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-trivial-arith.mir b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-trivial-arith.mir
new file mode 100644
index 0000000000000..ec9157f190a7c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-trivial-arith.mir
@@ -0,0 +1,163 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple aarch64 -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+
+name:            right_ident_sub
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $w0
+    ; Fold (x - 0) -> x
+    ;
+    ; CHECK-LABEL: name: right_ident_sub
+    ; CHECK: liveins: $w0
+    ; CHECK: %x:_(s32) = COPY $w0
+    ; CHECK: $w0 = COPY %x(s32)
+    ; CHECK: RET_ReallyLR implicit $w0
+    %x:_(s32) = COPY $w0
+    %cst:_(s32) = G_CONSTANT i32 0
+    %op:_(s32) = G_SUB %x(s32), %cst
+    $w0 = COPY %op(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            right_ident_add
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $w0
+    ; Fold (x + 0) -> x
+    ;
+    ; CHECK-LABEL: name: right_ident_add
+    ; CHECK: liveins: $w0
+    ; CHECK: %x:_(s32) = COPY $w0
+    ; CHECK: $w0 = COPY %x(s32)
+    ; CHECK: RET_ReallyLR implicit $w0
+    %x:_(s32) = COPY $w0
+    %cst:_(s32) = G_CONSTANT i32 0
+    %op:_(s32) = G_ADD %x(s32), %cst
+    $w0 = COPY %op(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            right_ident_or
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $w0
+    ; Fold (x || 0) -> x
+    ;
+    ; CHECK-LABEL: name: right_ident_or
+    ; CHECK: liveins: $w0
+    ; CHECK: %x:_(s32) = COPY $w0
+    ; CHECK: $w0 = COPY %x(s32)
+    ; CHECK: RET_ReallyLR implicit $w0
+    %x:_(s32) = COPY $w0
+    %cst:_(s32) = G_CONSTANT i32 0
+    %op:_(s32) = G_OR %x(s32), %cst
+    $w0 = COPY %op(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            right_ident_xor
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $w0
+    ; Fold (x | 0) -> x
+    ;
+    ; CHECK-LABEL: name: right_ident_xor
+    ; CHECK: liveins: $w0
+    ; CHECK: %x:_(s32) = COPY $w0
+    ; CHECK: $w0 = COPY %x(s32)
+    ; CHECK: RET_ReallyLR implicit $w0
+    %x:_(s32) = COPY $w0
+    %cst:_(s32) = G_CONSTANT i32 0
+    %op:_(s32) = G_XOR %x(s32), %cst
+    $w0 = COPY %op(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            right_ident_shl
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $w0
+    ; Fold (x << 0) -> x
+    ;
+    ; CHECK-LABEL: name: right_ident_shl
+    ; CHECK: liveins: $w0
+    ; CHECK: %x:_(s32) = COPY $w0
+    ; CHECK: $w0 = COPY %x(s32)
+    ; CHECK: RET_ReallyLR implicit $w0
+    %x:_(s32) = COPY $w0
+    %cst:_(s32) = G_CONSTANT i32 0
+    %op:_(s32) = G_SHL %x(s32), %cst
+    $w0 = COPY %op(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            right_ident_ashr
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $w0
+    ; Fold (x ashr 0) -> x
+    ;
+    ; CHECK-LABEL: name: right_ident_ashr
+    ; CHECK: liveins: $w0
+    ; CHECK: %x:_(s32) = COPY $w0
+    ; CHECK: $w0 = COPY %x(s32)
+    ; CHECK: RET_ReallyLR implicit $w0
+    %x:_(s32) = COPY $w0
+    %cst:_(s32) = G_CONSTANT i32 0
+    %op:_(s32) = G_ASHR %x(s32), %cst
+    $w0 = COPY %op(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            right_ident_lshr
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $w0
+    ; Fold (x lshr 0) -> x
+    ;
+    ; CHECK-LABEL: name: right_ident_lshr
+    ; CHECK: liveins: $w0
+    ; CHECK: %x:_(s32) = COPY $w0
+    ; CHECK: $w0 = COPY %x(s32)
+    ; CHECK: RET_ReallyLR implicit $w0
+    %x:_(s32) = COPY $w0
+    %cst:_(s32) = G_CONSTANT i32 0
+    %op:_(s32) = G_LSHR %x(s32), %cst
+    $w0 = COPY %op(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            dont_fold_sub
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $w0
+    ; Not an identity, no folding.
+    ;
+    ; CHECK-LABEL: name: dont_fold_sub
+    ; CHECK: liveins: $w0
+    ; CHECK: %x:_(s32) = COPY $w0
+    ; CHECK: %cst:_(s32) = G_CONSTANT i32 1
+    ; CHECK: %op:_(s32) = G_SUB %x, %cst
+    ; CHECK: $w0 = COPY %op(s32)
+    ; CHECK: RET_ReallyLR implicit $w0
+    %x:_(s32) = COPY $w0
+    %cst:_(s32) = G_CONSTANT i32 1
+    %op:_(s32) = G_SUB %x(s32), %cst
+    $w0 = COPY %op(s32)
+    RET_ReallyLR implicit $w0
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-constant.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-constant.mir
index 822ba251bc53d..e25c84958b9db 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/select-constant.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-constant.mir
@@ -8,6 +8,7 @@
   define i16 @const_s16() { ret i16 42 }
   define i32 @const_s32() { ret i32 42 }
   define i64 @const_s64() { ret i64 1234567890123 }
+  define i8* @const_p0_0() { ret i8* null }
 
   define i32 @fconst_s32() { ret i32 42 }
   define i64 @fconst_s64() { ret i64 1234567890123 }
@@ -80,6 +81,19 @@ body:             |
     $x0 = COPY %0(s64)
 ...
 
+---
+name:            const_p0_0
+legalized:       true
+regBankSelected: true
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: const_p0_0
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64 = COPY $xzr
+    ; CHECK: $x0 = COPY [[COPY]]
+    %0:gpr(p0) = G_CONSTANT i64 0
+    $x0 = COPY %0
+...
+
 ---
 name:            fconst_s32
 legalized:       true
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-jump-table-brjt-constrain.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-jump-table-brjt-constrain.mir
new file mode 100644
index 0000000000000..082bf43061da4
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-jump-table-brjt-constrain.mir
@@ -0,0 +1,61 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=aarch64-unknown-unknown -o - -verify-machineinstrs -run-pass=instruction-select %s | FileCheck %s
+
+# When we select the G_ZEXTLOAD, the SUBREG_TO_REG will initially land on a
+# gpr64sp register.
+#
+# This caused a test failure when selecting the G_BRJT, because it was not being
+# constrained. This test checks that the G_BRJT is actually being constrained.
+# As a result, the SUBREG_TO_REG should end up on a gpr64common.
+
+...
+---
+name:            check_constrain
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+jumpTable:
+  kind:            block-address
+  entries:
+    - id:              0
+      blocks:          [ '%bb.4' ]
+body:             |
+  ; CHECK-LABEL: name: check_constrain
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.1(0x40000000)
+  ; CHECK:   [[DEF:%[0-9]+]]:gpr64common = IMPLICIT_DEF
+  ; CHECK:   [[LDRBBui:%[0-9]+]]:gpr32 = LDRBBui [[DEF]], 0 :: (load 1)
+  ; CHECK:   [[SUBREG_TO_REG:%[0-9]+]]:gpr64common = SUBREG_TO_REG 0, [[LDRBBui]], %subreg.sub_32
+  ; CHECK:   $xzr = SUBSXri [[SUBREG_TO_REG]], 8, 0, implicit-def $nzcv
+  ; CHECK:   Bcc 8, %bb.3, implicit $nzcv
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
+  ; CHECK:   [[MOVaddrJT:%[0-9]+]]:gpr64 = MOVaddrJT target-flags(aarch64-page) %jump-table.0, target-flags(aarch64-pageoff, aarch64-nc) %jump-table.0
+  ; CHECK:   early-clobber %6:gpr64, early-clobber %7:gpr64sp = JumpTableDest32 [[MOVaddrJT]], [[SUBREG_TO_REG]], %jump-table.0
+  ; CHECK:   BR %6
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x80000000)
+  ; CHECK:   B %bb.3
+  ; CHECK: bb.3:
+  ; CHECK:   RET_ReallyLR
+  bb.1:
+    %1:gpr(p0) = G_IMPLICIT_DEF
+    %5:gpr(s64) = G_ZEXTLOAD %1(p0) :: (load 1)
+    %7:gpr(s64) = G_CONSTANT i64 8
+    %16:gpr(s32) = G_ICMP intpred(ugt), %5(s64), %7
+    %8:gpr(s1) = G_TRUNC %16(s32)
+    G_BRCOND %8(s1), %bb.4
+
+  bb.2:
+    successors: %bb.3, %bb.4
+
+    %9:gpr(p0) = G_JUMP_TABLE %jump-table.0
+    G_BRJT %9(p0), %jump-table.0, %5(s64)
+
+  bb.3:
+    G_BR %bb.4
+
+  bb.4:
+    RET_ReallyLR
+
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/translate-gep.ll b/llvm/test/CodeGen/AArch64/GlobalISel/translate-gep.ll
index 1f812f95aea1b..351b80d5e2ff7 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/translate-gep.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/translate-gep.ll
@@ -8,9 +8,8 @@ define i8*  @translate_element_size1(i64 %arg) {
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $x0
   ; CHECK:   [[COPY:%[0-9]+]]:_(s64) = COPY $x0
-  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-  ; CHECK:   [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[C]](s64)
-  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[INTTOPTR]], [[COPY]](s64)
+  ; CHECK:   [[C:%[0-9]+]]:_(p0) = G_CONSTANT i64 0
+  ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[C]], [[COPY]](s64)
   ; CHECK:   [[COPY1:%[0-9]+]]:_(p0) = COPY [[PTR_ADD]](p0)
   ; CHECK:   $x0 = COPY [[COPY1]](p0)
   ; CHECK:   RET_ReallyLR implicit $x0
diff --git a/llvm/test/CodeGen/AArch64/funclet-match-add-sub-stack.ll b/llvm/test/CodeGen/AArch64/funclet-match-add-sub-stack.ll
new file mode 100644
index 0000000000000..67e9c49675cfd
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/funclet-match-add-sub-stack.ll
@@ -0,0 +1,62 @@
+; RUN: llc -o - %s -mtriple=aarch64-windows | FileCheck %s
+; Check that the stack bump around a funclet is computed correctly in both the
+; prologue and epilogue in the case we have a MaxCallFrameSize > 0 and are doing alloca
+target datalayout = "e-m:w-p:64:64-i32:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-pc-windows-msvc19.25.28611"
+
+; // requires passing arguments on the stack
+; void test2(void*, int, int, int, int, int, int, int, int);
+;
+; // function with the funclet being checked
+; void test1(size_t bytes)
+; {
+;   // alloca forces a separate callee save bump and stack bump
+;   void *data = _alloca(bytes);
+;   try {
+;     test2(data, 0, 1, 2, 3, 4, 5, 6, 7);
+;   } catch (...) {
+;     // the funclet being checked
+;   }
+; }
+
+; CHECK-LABEL: ?catch$2@?0??test1@@YAX_K@Z@4HA
+; CHECK: sub sp, sp, #16
+; CHECK: add sp, sp, #16
+; Function Attrs: uwtable
+define dso_local void @"?test1@@YAX_K@Z"(i64 %0) #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+  %2 = alloca i64, align 8
+  %3 = alloca i8*, align 8
+  store i64 %0, i64* %2, align 8
+  %4 = load i64, i64* %2, align 8
+  %5 = alloca i8, i64 %4, align 16
+  store i8* %5, i8** %3, align 8
+  %6 = load i8*, i8** %3, align 8
+  invoke void @"?test2@@YAXPEAXHHHHHHHH@Z"(i8* %6, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7)
+          to label %13 unwind label %7
+
+7:                                                ; preds = %1
+  %8 = catchswitch within none [label %9] unwind to caller
+
+9:                                                ; preds = %7
+  %10 = catchpad within %8 [i8* null, i32 64, i8* null]
+  catchret from %10 to label %11
+
+11:                                               ; preds = %9
+  br label %12
+
+12:                                               ; preds = %11, %13
+  ret void
+
+13:                                               ; preds = %1
+  br label %12
+}
+
+declare dso_local void @"?test2@@YAXPEAXHHHHHHHH@Z"(i8*, i32, i32, i32, i32, i32, i32, i32, i32) #1
+
+declare dso_local i32 @__CxxFrameHandler3(...)
+
+attributes #0 = { uwtable }
+
+!llvm.module.flags = !{!0}
+
+!0 = !{i32 1, !"wchar_size", i32 2}
diff --git a/llvm/test/CodeGen/AArch64/seh-finally.ll b/llvm/test/CodeGen/AArch64/seh-finally.ll
index 66558c90a79cd..dbc6c4b0804bf 100644
--- a/llvm/test/CodeGen/AArch64/seh-finally.ll
+++ b/llvm/test/CodeGen/AArch64/seh-finally.ll
@@ -37,7 +37,7 @@ entry:
 ; CHECK-LABEL: simple_seh
 ; CHECK: add     x29, sp, #16
 ; CHECK: mov     x0, #-2
-; CHECK: stur    x0, [x29, #-16]
+; CHECK: stur    x0, [x29, #16]
 ; CHECK: .set .Lsimple_seh$frame_escape_0, -8
 ; CHECK: ldur    w0, [x29, #-8]
 ; CHECK: bl      foo
@@ -87,13 +87,13 @@ define void @stack_realign() #0 personality i8* bitcast (i32 (...)* @__C_specifi
 entry:
 ; CHECK-LABEL: stack_realign
 ; CHECK: mov     x29, sp
-; CHECK: sub     x9, sp, #64
+; CHECK: sub     x9, sp, #16
 ; CHECK: and     sp, x9, #0xffffffffffffffe0
 ; CHECK: mov     x19, sp
 ; CHECK: mov     x0, #-2
-; CHECK: stur    x0, [x19, #16]
-; CHECK: .set .Lstack_realign$frame_escape_0, 32
-; CHECK: ldr     w0, [x19, #32]
+; CHECK: stur    x0, [x29, #32]
+; CHECK: .set .Lstack_realign$frame_escape_0, 0
+; CHECK: ldr     w0, [x19]
 ; CHECK: bl      foo
 
   %o = alloca %struct.S, align 32
@@ -142,7 +142,7 @@ entry:
 ; CHECK-LABEL: vla_present
 ; CHECK: add     x29, sp, #32
 ; CHECK: mov     x1, #-2
-; CHECK: stur    x1, [x29, #-32]
+; CHECK: stur    x1, [x29, #16]
 ; CHECK: .set .Lvla_present$frame_escape_0, -4
 ; CHECK: stur    w0, [x29, #-4]
 ; CHECK: ldur    w8, [x29, #-4]
@@ -206,17 +206,17 @@ define void @vla_and_realign(i32 %n) #0 personality i8* bitcast (i32 (...)* @__C
 entry:
 ; CHECK-LABEL: vla_and_realign
 ; CHECK: mov     x29, sp
-; CHECK: sub     x9, sp, #64
+; CHECK: sub     x9, sp, #48
 ; CHECK: and     sp, x9, #0xffffffffffffffe0
 ; CHECK: mov     x19, sp
 ; CHECK: mov     x1, #-2
-; CHECK: stur    x1, [x19]
+; CHECK: stur    x1, [x29, #32]
 ; CHECK: .set .Lvla_and_realign$frame_escape_0, 32
-; CHECK: str     w0, [x29, #28]
-; CHECK: ldr     w8, [x29, #28]
+; CHECK: str     w0, [x29, #44]
+; CHECK: ldr     w8, [x29, #44]
 ; CHECK: mov     x9, sp
-; CHECK: str     x9, [x19, #24]
-; CHECK: str     x8, [x19, #16]
+; CHECK: str     x9, [x29, #24]
+; CHECK: str     x8, [x19, #24]
 ; CHECK: ldr     w0, [x19, #32]
 ; CHECK: bl      foo
 
diff --git a/llvm/test/CodeGen/AArch64/speculation-hardening.ll b/llvm/test/CodeGen/AArch64/speculation-hardening.ll
index 81aea75e3e12d..23b87563013fd 100644
--- a/llvm/test/CodeGen/AArch64/speculation-hardening.ll
+++ b/llvm/test/CodeGen/AArch64/speculation-hardening.ll
@@ -1,7 +1,7 @@
 ; RUN: sed -e 's/SLHATTR/speculative_load_hardening/' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu | FileCheck %s --check-prefixes=CHECK,SLH --dump-input-on-failure
 ; RUN: sed -e 's/SLHATTR//' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu | FileCheck %s --check-prefixes=CHECK,NOSLH --dump-input-on-failure
 ; RUN: sed -e 's/SLHATTR/speculative_load_hardening/' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -global-isel | FileCheck %s --check-prefixes=CHECK,SLH --dump-input-on-failure
-; RUN sed -e 's/SLHATTR//' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -global-isel | FileCheck %s --check-prefixes=CHECK,NOSLH --dump-input-on-failure
+; RUN: sed -e 's/SLHATTR//' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -global-isel | FileCheck %s --check-prefixes=CHECK,NOSLH --dump-input-on-failure
 ; RUN: sed -e 's/SLHATTR/speculative_load_hardening/' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -fast-isel | FileCheck %s --check-prefixes=CHECK,SLH --dump-input-on-failure
 ; RUN: sed -e 's/SLHATTR//' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -fast-isel | FileCheck %s --check-prefixes=CHECK,NOSLH --dump-input-on-failure
 
diff --git a/llvm/test/CodeGen/AArch64/sve-fcmp.ll b/llvm/test/CodeGen/AArch64/sve-fcmp.ll
new file mode 100644
index 0000000000000..f1426bb3179dc
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-fcmp.ll
@@ -0,0 +1,231 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve < %s | FileCheck %s
+
+define <vscale x 4 x i1> @oeq(<vscale x 4 x float> %x, <vscale x 4 x float> %x2) {
+; CHECK-LABEL: oeq:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fcmeq p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    ret
+  %y = fcmp oeq <vscale x 4 x float> %x, %x2
+  ret <vscale x 4 x i1> %y
+}
+define <vscale x 4 x i1> @ogt(<vscale x 4 x float> %x, <vscale x 4 x float> %x2) {
+; CHECK-LABEL: ogt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fcmgt p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    ret
+  %y = fcmp ogt <vscale x 4 x float> %x, %x2
+  ret <vscale x 4 x i1> %y
+}
+define <vscale x 4 x i1> @oge(<vscale x 4 x float> %x, <vscale x 4 x float> %x2) {
+; CHECK-LABEL: oge:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fcmge p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    ret
+  %y = fcmp oge <vscale x 4 x float> %x, %x2
+  ret <vscale x 4 x i1> %y
+}
+define <vscale x 4 x i1> @olt(<vscale x 4 x float> %x, <vscale x 4 x float> %x2) {
+; CHECK-LABEL: olt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fcmgt p0.s, p0/z, z1.s, z0.s
+; CHECK-NEXT:    ret
+  %y = fcmp olt <vscale x 4 x float> %x, %x2
+  ret <vscale x 4 x i1> %y
+}
+define <vscale x 4 x i1> @ole(<vscale x 4 x float> %x, <vscale x 4 x float> %x2) {
+; CHECK-LABEL: ole:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fcmge p0.s, p0/z, z1.s, z0.s
+; CHECK-NEXT:    ret
+  %y = fcmp ole <vscale x 4 x float> %x, %x2
+  ret <vscale x 4 x i1> %y
+}
+define <vscale x 4 x i1> @one(<vscale x 4 x float> %x, <vscale x 4 x float> %x2) {
+; CHECK-LABEL: one:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fcmne p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    ret
+  %y = fcmp one <vscale x 4 x float> %x, %x2
+  ret <vscale x 4 x i1> %y
+}
+define <vscale x 4 x i1> @ord(<vscale x 4 x float> %x, <vscale x 4 x float> %x2) {
+; CHECK-LABEL: ord:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fcmuo p1.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    not p0.b, p0/z, p1.b
+; CHECK-NEXT:    ret
+  %y = fcmp ord <vscale x 4 x float> %x, %x2
+  ret <vscale x 4 x i1> %y
+}
+define <vscale x 4 x i1> @ueq(<vscale x 4 x float> %x, <vscale x 4 x float> %x2) {
+; CHECK-LABEL: ueq:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fcmne p1.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    not p0.b, p0/z, p1.b
+; CHECK-NEXT:    ret
+  %y = fcmp ueq <vscale x 4 x float> %x, %x2
+  ret <vscale x 4 x i1> %y
+}
+define <vscale x 4 x i1> @ugt(<vscale x 4 x float> %x, <vscale x 4 x float> %x2) {
+; CHECK-LABEL: ugt:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fcmge p1.s, p0/z, z1.s, z0.s
+; CHECK-NEXT:    not p0.b, p0/z, p1.b
+; CHECK-NEXT:    ret
+  %y = fcmp ugt <vscale x 4 x float> %x, %x2
+  ret <vscale x 4 x i1> %y
+}
+define <vscale x 4 x i1> @uge(<vscale x 4 x float> %x, <vscale x 4 x float> %x2) {
+; CHECK-LABEL: uge:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fcmgt p1.s, p0/z, z1.s, z0.s
+; CHECK-NEXT:    not p0.b, p0/z, p1.b
+; CHECK-NEXT:    ret
+  %y = fcmp uge <vscale x 4 x float> %x, %x2
+  ret <vscale x 4 x i1> %y
+}
+define <vscale x 4 x i1> @ult(<vscale x 4 x float> %x, <vscale x 4 x float> %x2) {
+; CHECK-LABEL: ult:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fcmge p1.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    not p0.b, p0/z, p1.b
+; CHECK-NEXT:    ret
+  %y = fcmp ult <vscale x 4 x float> %x, %x2
+  ret <vscale x 4 x i1> %y
+}
+define <vscale x 4 x i1> @ule(<vscale x 4 x float> %x, <vscale x 4 x float> %x2) {
+; CHECK-LABEL: ule:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fcmgt p1.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    not p0.b, p0/z, p1.b
+; CHECK-NEXT:    ret
+  %y = fcmp ule <vscale x 4 x float> %x, %x2
+  ret <vscale x 4 x i1> %y
+}
+define <vscale x 4 x i1> @une(<vscale x 4 x float> %x, <vscale x 4 x float> %x2) {
+; CHECK-LABEL: une:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fcmeq p1.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    not p0.b, p0/z, p1.b
+; CHECK-NEXT:    ret
+  %y = fcmp une <vscale x 4 x float> %x, %x2
+  ret <vscale x 4 x i1> %y
+}
+define <vscale x 4 x i1> @uno(<vscale x 4 x float> %x, <vscale x 4 x float> %x2) {
+; CHECK-LABEL: uno:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fcmuo p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    ret
+  %y = fcmp uno <vscale x 4 x float> %x, %x2
+  ret <vscale x 4 x i1> %y
+}
+define <vscale x 2 x i1> @oeq_2f32(<vscale x 2 x float> %x, <vscale x 2 x float> %x2) {
+; CHECK-LABEL: oeq_2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fcmeq p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    ret
+  %y = fcmp oeq <vscale x 2 x float> %x, %x2
+  ret <vscale x 2 x i1> %y
+}
+define <vscale x 2 x i1> @ueq_2f32(<vscale x 2 x float> %x, <vscale x 2 x float> %x2) {
+; CHECK-LABEL: ueq_2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fcmne p1.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    not p0.b, p0/z, p1.b
+; CHECK-NEXT:    ret
+  %y = fcmp ueq <vscale x 2 x float> %x, %x2
+  ret <vscale x 2 x i1> %y
+}
+define <vscale x 2 x i1> @oeq_2f64(<vscale x 2 x double> %x, <vscale x 2 x double> %x2) {
+; CHECK-LABEL: oeq_2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fcmeq p0.d, p0/z, z0.d, z1.d
+; CHECK-NEXT:    ret
+  %y = fcmp oeq <vscale x 2 x double> %x, %x2
+  ret <vscale x 2 x i1> %y
+}
+define <vscale x 2 x i1> @ueq_2f64(<vscale x 2 x double> %x, <vscale x 2 x double> %x2) {
+; CHECK-LABEL: ueq_2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fcmne p1.d, p0/z, z0.d, z1.d
+; CHECK-NEXT:    not p0.b, p0/z, p1.b
+; CHECK-NEXT:    ret
+  %y = fcmp ueq <vscale x 2 x double> %x, %x2
+  ret <vscale x 2 x i1> %y
+}
+define <vscale x 2 x i1> @oeq_2f16(<vscale x 2 x half> %x, <vscale x 2 x half> %x2) {
+; CHECK-LABEL: oeq_2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fcmeq p0.h, p0/z, z0.h, z1.h
+; CHECK-NEXT:    ret
+  %y = fcmp oeq <vscale x 2 x half> %x, %x2
+  ret <vscale x 2 x i1> %y
+}
+define <vscale x 2 x i1> @ueq_2f16(<vscale x 2 x half> %x, <vscale x 2 x half> %x2) {
+; CHECK-LABEL: ueq_2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fcmne p1.h, p0/z, z0.h, z1.h
+; CHECK-NEXT:    not p0.b, p0/z, p1.b
+; CHECK-NEXT:    ret
+  %y = fcmp ueq <vscale x 2 x half> %x, %x2
+  ret <vscale x 2 x i1> %y
+}
+define <vscale x 4 x i1> @oeq_4f16(<vscale x 4 x half> %x, <vscale x 4 x half> %x2) {
+; CHECK-LABEL: oeq_4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fcmeq p0.h, p0/z, z0.h, z1.h
+; CHECK-NEXT:    ret
+  %y = fcmp oeq <vscale x 4 x half> %x, %x2
+  ret <vscale x 4 x i1> %y
+}
+define <vscale x 4 x i1> @ueq_4f16(<vscale x 4 x half> %x, <vscale x 4 x half> %x2) {
+; CHECK-LABEL: ueq_4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fcmne p1.h, p0/z, z0.h, z1.h
+; CHECK-NEXT:    not p0.b, p0/z, p1.b
+; CHECK-NEXT:    ret
+  %y = fcmp ueq <vscale x 4 x half> %x, %x2
+  ret <vscale x 4 x i1> %y
+}
+define <vscale x 8 x i1> @oeq_8f16(<vscale x 8 x half> %x, <vscale x 8 x half> %x2) {
+; CHECK-LABEL: oeq_8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    fcmeq p0.h, p0/z, z0.h, z1.h
+; CHECK-NEXT:    ret
+  %y = fcmp oeq <vscale x 8 x half> %x, %x2
+  ret <vscale x 8 x i1> %y
+}
+define <vscale x 8 x i1> @ueq_8f16(<vscale x 8 x half> %x, <vscale x 8 x half> %x2) {
+; CHECK-LABEL: ueq_8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    fcmne p1.h, p0/z, z0.h, z1.h
+; CHECK-NEXT:    not p0.b, p0/z, p1.b
+; CHECK-NEXT:    ret
+  %y = fcmp ueq <vscale x 8 x half> %x, %x2
+  ret <vscale x 8 x i1> %y
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-int-log.ll b/llvm/test/CodeGen/AArch64/sve-int-log.ll
index 2e958e79286e1..116112f525e10 100644
--- a/llvm/test/CodeGen/AArch64/sve-int-log.ll
+++ b/llvm/test/CodeGen/AArch64/sve-int-log.ll
@@ -1,96 +1,230 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
 
 define <vscale x 2 x i64> @and_d(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
-; CHECK-LABEL: and_d
-; CHECK: and z0.d, z0.d, z1.d
-; CHECK-NEXT: ret
+; CHECK-LABEL: and_d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
   %res = and <vscale x 2 x i64> %a, %b
   ret <vscale x 2 x i64> %res
 }
 
 define <vscale x 4 x i32> @and_s(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
-; CHECK-LABEL: and_s
-; CHECK: and z0.d, z0.d, z1.d
-; CHECK-NEXT: ret
+; CHECK-LABEL: and_s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
   %res = and <vscale x 4 x i32> %a, %b
   ret <vscale x 4 x i32> %res
 }
 
 define <vscale x 8 x i16> @and_h(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
-; CHECK-LABEL: and_h
-; CHECK: and z0.d, z0.d, z1.d
-; CHECK-NEXT: ret
+; CHECK-LABEL: and_h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
   %res = and <vscale x 8 x i16> %a, %b
   ret <vscale x 8 x i16> %res
 }
 
 define <vscale x 16 x i8> @and_b(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
-; CHECK-LABEL: and_b
-; CHECK: and z0.d, z0.d, z1.d
-; CHECK-NEXT: ret
+; CHECK-LABEL: and_b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
   %res = and <vscale x 16 x i8> %a, %b
   ret <vscale x 16 x i8> %res
-}                                                                                          
+}
+
+define <vscale x 2 x i1> @and_pred_d(<vscale x 2 x i1> %a, <vscale x 2 x i1> %b) {
+; CHECK-LABEL: and_pred_d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p2.d
+; CHECK-NEXT:    and p0.b, p2/z, p0.b, p1.b
+; CHECK-NEXT:    ret
+  %res = and <vscale x 2 x i1> %a, %b
+  ret <vscale x 2 x i1> %res
+}
+
+define <vscale x 4 x i1> @and_pred_s(<vscale x 4 x i1> %a, <vscale x 4 x i1> %b) {
+; CHECK-LABEL: and_pred_s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p2.s
+; CHECK-NEXT:    and p0.b, p2/z, p0.b, p1.b
+; CHECK-NEXT:    ret
+  %res = and <vscale x 4 x i1> %a, %b
+  ret <vscale x 4 x i1> %res
+}
+
+define <vscale x 8 x i1> @and_pred_h(<vscale x 8 x i1> %a, <vscale x 8 x i1> %b) {
+; CHECK-LABEL: and_pred_h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p2.h
+; CHECK-NEXT:    and p0.b, p2/z, p0.b, p1.b
+; CHECK-NEXT:    ret
+  %res = and <vscale x 8 x i1> %a, %b
+  ret <vscale x 8 x i1> %res
+}
+
+define <vscale x 16 x i1> @and_pred_b(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b) {
+; CHECK-LABEL: and_pred_b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p2.b
+; CHECK-NEXT:    and p0.b, p2/z, p0.b, p1.b
+; CHECK-NEXT:    ret
+  %res = and <vscale x 16 x i1> %a, %b
+  ret <vscale x 16 x i1> %res
+}
+
 define <vscale x 2 x i64> @or_d(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
-; CHECK-LABEL: or_d
-; CHECK: orr z0.d, z0.d, z1.d
-; CHECK-NEXT: ret
+; CHECK-LABEL: or_d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    orr z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
   %res = or <vscale x 2 x i64> %a, %b
   ret <vscale x 2 x i64> %res
 }
 
 define <vscale x 4 x i32> @or_s(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
-; CHECK-LABEL: or_s
-; CHECK: orr z0.d, z0.d, z1.d
-; CHECK-NEXT: ret
+; CHECK-LABEL: or_s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    orr z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
   %res = or <vscale x 4 x i32> %a, %b
   ret <vscale x 4 x i32> %res
 }
 
 define <vscale x 8 x i16> @or_h(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
-; CHECK-LABEL: or_h
-; CHECK: orr z0.d, z0.d, z1.d
-; CHECK-NEXT: ret
+; CHECK-LABEL: or_h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    orr z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
   %res = or <vscale x 8 x i16> %a, %b
   ret <vscale x 8 x i16> %res
 }
 
 define <vscale x 16 x i8> @or_b(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
-; CHECK-LABEL: or_b
-; CHECK: orr z0.d, z0.d, z1.d
-; CHECK-NEXT: ret
+; CHECK-LABEL: or_b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    orr z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
   %res = or <vscale x 16 x i8> %a, %b
   ret <vscale x 16 x i8> %res
-}                                                                                          
+}
+
+define <vscale x 2 x i1> @or_pred_d(<vscale x 2 x i1> %a, <vscale x 2 x i1> %b) {
+; CHECK-LABEL: or_pred_d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p2.d
+; CHECK-NEXT:    orr p0.b, p2/z, p0.b, p1.b
+; CHECK-NEXT:    ret
+  %res = or <vscale x 2 x i1> %a, %b
+  ret <vscale x 2 x i1> %res
+}
+
+define <vscale x 4 x i1> @or_pred_s(<vscale x 4 x i1> %a, <vscale x 4 x i1> %b) {
+; CHECK-LABEL: or_pred_s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p2.s
+; CHECK-NEXT:    orr p0.b, p2/z, p0.b, p1.b
+; CHECK-NEXT:    ret
+  %res = or <vscale x 4 x i1> %a, %b
+  ret <vscale x 4 x i1> %res
+}
+
+define <vscale x 8 x i1> @or_pred_h(<vscale x 8 x i1> %a, <vscale x 8 x i1> %b) {
+; CHECK-LABEL: or_pred_h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p2.h
+; CHECK-NEXT:    orr p0.b, p2/z, p0.b, p1.b
+; CHECK-NEXT:    ret
+  %res = or <vscale x 8 x i1> %a, %b
+  ret <vscale x 8 x i1> %res
+}
+
+define <vscale x 16 x i1> @or_pred_b(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b) {
+; CHECK-LABEL: or_pred_b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p2.b
+; CHECK-NEXT:    orr p0.b, p2/z, p0.b, p1.b
+; CHECK-NEXT:    ret
+  %res = or <vscale x 16 x i1> %a, %b
+  ret <vscale x 16 x i1> %res
+}
 
 define <vscale x 2 x i64> @xor_d(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
-; CHECK-LABEL: xor_d
-; CHECK: eor z0.d, z0.d, z1.d
-; CHECK-NEXT: ret
+; CHECK-LABEL: xor_d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    eor z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
   %res = xor <vscale x 2 x i64> %a, %b
   ret <vscale x 2 x i64> %res
 }
 
 define <vscale x 4 x i32> @xor_s(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
-; CHECK-LABEL: xor_s
-; CHECK: eor z0.d, z0.d, z1.d
-; CHECK-NEXT: ret
+; CHECK-LABEL: xor_s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    eor z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
   %res = xor <vscale x 4 x i32> %a, %b
   ret <vscale x 4 x i32> %res
 }
 
 define <vscale x 8 x i16> @xor_h(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
-; CHECK-LABEL: xor_h
-; CHECK: eor z0.d, z0.d, z1.d
-; CHECK-NEXT: ret
+; CHECK-LABEL: xor_h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    eor z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
   %res = xor <vscale x 8 x i16> %a, %b
   ret <vscale x 8 x i16> %res
 }
 
 define <vscale x 16 x i8> @xor_b(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
-; CHECK-LABEL: xor_b
-; CHECK: eor z0.d, z0.d, z1.d
-; CHECK-NEXT: ret
+; CHECK-LABEL: xor_b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    eor z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
   %res = xor <vscale x 16 x i8> %a, %b
   ret <vscale x 16 x i8> %res
 }
+
+define <vscale x 2 x i1> @xor_pred_d(<vscale x 2 x i1> %a, <vscale x 2 x i1> %b) {
+; CHECK-LABEL: xor_pred_d:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p2.d
+; CHECK-NEXT:    eor p0.b, p2/z, p0.b, p1.b
+; CHECK-NEXT:    ret
+  %res = xor <vscale x 2 x i1> %a, %b
+  ret <vscale x 2 x i1> %res
+}
+
+define <vscale x 4 x i1> @xor_pred_s(<vscale x 4 x i1> %a, <vscale x 4 x i1> %b) {
+; CHECK-LABEL: xor_pred_s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p2.s
+; CHECK-NEXT:    eor p0.b, p2/z, p0.b, p1.b
+; CHECK-NEXT:    ret
+  %res = xor <vscale x 4 x i1> %a, %b
+  ret <vscale x 4 x i1> %res
+}
+
+define <vscale x 8 x i1> @xor_pred_h(<vscale x 8 x i1> %a, <vscale x 8 x i1> %b) {
+; CHECK-LABEL: xor_pred_h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p2.h
+; CHECK-NEXT:    eor p0.b, p2/z, p0.b, p1.b
+; CHECK-NEXT:    ret
+  %res = xor <vscale x 8 x i1> %a, %b
+  ret <vscale x 8 x i1> %res
+}
+
+define <vscale x 16 x i1> @xor_pred_b(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b) {
+; CHECK-LABEL: xor_pred_b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p2.b
+; CHECK-NEXT:    eor p0.b, p2/z, p0.b, p1.b
+; CHECK-NEXT:    ret
+  %res = xor <vscale x 16 x i1> %a, %b
+  ret <vscale x 16 x i1> %res
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-stores.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-stores.ll
index 772e36888fbe8..ac2b9a32b14e6 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-stores.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-stores.ll
@@ -1,5 +1,305 @@
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
 
+;
+; ST2B
+;
+
+define void @st2b_i8(<vscale x 16 x i8> %v0, <vscale x 16 x i8> %v1, <vscale x 16 x i1> %pred, <vscale x 16 x i8>* %addr) {
+; CHECK-LABEL: st2b_i8:
+; CHECK: st2b { z0.b, z1.b }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st2.nxv16i8(<vscale x 16 x i8> %v0,
+                                          <vscale x 16 x i8> %v1,
+                                          <vscale x 16 x i1> %pred,
+                                          <vscale x 16 x i8>* %addr)
+  ret void
+}
+
+;
+; ST2H
+;
+
+define void @st2h_i16(<vscale x 8 x i16> %v0, <vscale x 8 x i16> %v1, <vscale x 8 x i1> %pred, <vscale x 8 x i16>* %addr) {
+; CHECK-LABEL: st2h_i16:
+; CHECK: st2h { z0.h, z1.h }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st2.nxv8i16(<vscale x 8 x i16> %v0,
+                                          <vscale x 8 x i16> %v1,
+                                          <vscale x 8 x i1> %pred,
+                                          <vscale x 8 x i16>* %addr)
+  ret void
+}
+
+define void @st2h_f16(<vscale x 8 x half> %v0, <vscale x 8 x half> %v1, <vscale x 8 x i1> %pred, <vscale x 8 x half>* %addr) {
+; CHECK-LABEL: st2h_f16:
+; CHECK: st2h { z0.h, z1.h }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st2.nxv8f16(<vscale x 8 x half> %v0,
+                                          <vscale x 8 x half> %v1,
+                                          <vscale x 8 x i1> %pred,
+                                          <vscale x 8 x half>* %addr)
+  ret void
+}
+
+;
+; ST2W
+;
+
+define void @st2w_i32(<vscale x 4 x i32> %v0, <vscale x 4 x i32> %v1, <vscale x 4 x i1> %pred, <vscale x 4 x i32>* %addr) {
+; CHECK-LABEL: st2w_i32:
+; CHECK: st2w { z0.s, z1.s }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st2.nxv4i32(<vscale x 4 x i32> %v0,
+                                          <vscale x 4 x i32> %v1,
+                                          <vscale x 4 x i1> %pred,
+                                          <vscale x 4 x i32>* %addr)
+  ret void
+}
+
+define void @st2w_f32(<vscale x 4 x float> %v0, <vscale x 4 x float> %v1, <vscale x 4 x i1> %pred, <vscale x 4 x float>* %addr) {
+; CHECK-LABEL: st2w_f32:
+; CHECK: st2w { z0.s, z1.s }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st2.nxv4f32(<vscale x 4 x float> %v0,
+                                          <vscale x 4 x float> %v1,
+                                          <vscale x 4 x i1> %pred,
+                                          <vscale x 4 x float>* %addr)
+  ret void
+}
+
+;
+; ST2D
+;
+
+define void @st2d_i64(<vscale x 2 x i64> %v0, <vscale x 2 x i64> %v1, <vscale x 2 x i1> %pred, <vscale x 2 x i64>* %addr) {
+; CHECK-LABEL: st2d_i64:
+; CHECK: st2d { z0.d, z1.d }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st2.nxv2i64(<vscale x 2 x i64> %v0,
+                                          <vscale x 2 x i64> %v1,
+                                          <vscale x 2 x i1> %pred,
+                                          <vscale x 2 x i64>* %addr)
+  ret void
+}
+
+define void @st2d_f64(<vscale x 2 x double> %v0, <vscale x 2 x double> %v1, <vscale x 2 x i1> %pred, <vscale x 2 x double>* %addr) {
+; CHECK-LABEL: st2d_f64:
+; CHECK: st2d { z0.d, z1.d }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st2.nxv2f64(<vscale x 2 x double> %v0,
+                                          <vscale x 2 x double> %v1,
+                                          <vscale x 2 x i1> %pred,
+                                          <vscale x 2 x double>* %addr)
+  ret void
+}
+
+;
+; ST3B
+;
+
+define void @st3b_i8(<vscale x 16 x i8> %v0, <vscale x 16 x i8> %v1, <vscale x 16 x i8> %v2, <vscale x 16 x i1> %pred, <vscale x 16 x i8>* %addr) {
+; CHECK-LABEL: st3b_i8:
+; CHECK: st3b { z0.b, z1.b, z2.b }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st3.nxv16i8(<vscale x 16 x i8> %v0,
+                                          <vscale x 16 x i8> %v1,
+                                          <vscale x 16 x i8> %v2,
+                                          <vscale x 16 x i1> %pred,
+                                          <vscale x 16 x i8>* %addr)
+  ret void
+}
+
+;
+; ST3H
+;
+
+define void @st3h_i16(<vscale x 8 x i16> %v0, <vscale x 8 x i16> %v1, <vscale x 8 x i16> %v2, <vscale x 8 x i1> %pred, <vscale x 8 x i16>* %addr) {
+; CHECK-LABEL: st3h_i16:
+; CHECK: st3h { z0.h, z1.h, z2.h }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st3.nxv8i16(<vscale x 8 x i16> %v0,
+                                          <vscale x 8 x i16> %v1,
+                                          <vscale x 8 x i16> %v2,
+                                          <vscale x 8 x i1> %pred,
+                                          <vscale x 8 x i16>* %addr)
+  ret void
+}
+
+define void @st3h_f16(<vscale x 8 x half> %v0, <vscale x 8 x half> %v1, <vscale x 8 x half> %v2, <vscale x 8 x i1> %pred, <vscale x 8 x half>* %addr) {
+; CHECK-LABEL: st3h_f16:
+; CHECK: st3h { z0.h, z1.h, z2.h }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st3.nxv8f16(<vscale x 8 x half> %v0,
+                                          <vscale x 8 x half> %v1,
+                                          <vscale x 8 x half> %v2,
+                                          <vscale x 8 x i1> %pred,
+                                          <vscale x 8 x half>* %addr)
+  ret void
+}
+
+;
+; ST3W
+;
+
+define void @st3w_i32(<vscale x 4 x i32> %v0, <vscale x 4 x i32> %v1, <vscale x 4 x i32> %v2, <vscale x 4 x i1> %pred, <vscale x 4 x i32>* %addr) {
+; CHECK-LABEL: st3w_i32:
+; CHECK: st3w { z0.s, z1.s, z2.s }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st3.nxv4i32(<vscale x 4 x i32> %v0,
+                                          <vscale x 4 x i32> %v1,
+                                          <vscale x 4 x i32> %v2,
+                                          <vscale x 4 x i1> %pred,
+                                          <vscale x 4 x i32>* %addr)
+  ret void
+}
+
+define void @st3w_f32(<vscale x 4 x float> %v0, <vscale x 4 x float> %v1, <vscale x 4 x float> %v2, <vscale x 4 x i1> %pred, <vscale x 4 x float>* %addr) {
+; CHECK-LABEL: st3w_f32:
+; CHECK: st3w { z0.s, z1.s, z2.s }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st3.nxv4f32(<vscale x 4 x float> %v0,
+                                          <vscale x 4 x float> %v1,
+                                          <vscale x 4 x float> %v2,
+                                          <vscale x 4 x i1> %pred,
+                                          <vscale x 4 x float>* %addr)
+  ret void
+}
+
+;
+; ST3D
+;
+
+define void @st3d_i64(<vscale x 2 x i64> %v0, <vscale x 2 x i64> %v1, <vscale x 2 x i64> %v2, <vscale x 2 x i1> %pred, <vscale x 2 x i64>* %addr) {
+; CHECK-LABEL: st3d_i64:
+; CHECK: st3d { z0.d, z1.d, z2.d }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st3.nxv2i64(<vscale x 2 x i64> %v0,
+                                          <vscale x 2 x i64> %v1,
+                                          <vscale x 2 x i64> %v2,
+                                          <vscale x 2 x i1> %pred,
+                                          <vscale x 2 x i64>* %addr)
+  ret void
+}
+
+define void @st3d_f64(<vscale x 2 x double> %v0, <vscale x 2 x double> %v1, <vscale x 2 x double> %v2, <vscale x 2 x i1> %pred, <vscale x 2 x double>* %addr) {
+; CHECK-LABEL: st3d_f64:
+; CHECK: st3d { z0.d, z1.d, z2.d }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st3.nxv2f64(<vscale x 2 x double> %v0,
+                                          <vscale x 2 x double> %v1,
+                                          <vscale x 2 x double> %v2,
+                                          <vscale x 2 x i1> %pred,
+                                          <vscale x 2 x double>* %addr)
+  ret void
+}
+
+;
+; ST4B
+;
+
+define void @st4b_i8(<vscale x 16 x i8> %v0, <vscale x 16 x i8> %v1, <vscale x 16 x i8> %v2, <vscale x 16 x i8> %v3, <vscale x 16 x i1> %pred, <vscale x 16 x i8>* %addr) {
+; CHECK-LABEL: st4b_i8:
+; CHECK: st4b { z0.b, z1.b, z2.b, z3.b }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st4.nxv16i8(<vscale x 16 x i8> %v0,
+                                          <vscale x 16 x i8> %v1,
+                                          <vscale x 16 x i8> %v2,
+                                          <vscale x 16 x i8> %v3,
+                                          <vscale x 16 x i1> %pred,
+                                          <vscale x 16 x i8>* %addr)
+  ret void
+}
+
+;
+; ST4H
+;
+
+define void @st4h_i16(<vscale x 8 x i16> %v0, <vscale x 8 x i16> %v1, <vscale x 8 x i16> %v2, <vscale x 8 x i16> %v3, <vscale x 8 x i1> %pred, <vscale x 8 x i16>* %addr) {
+; CHECK-LABEL: st4h_i16:
+; CHECK: st4h { z0.h, z1.h, z2.h, z3.h }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st4.nxv8i16(<vscale x 8 x i16> %v0,
+                                          <vscale x 8 x i16> %v1,
+                                          <vscale x 8 x i16> %v2,
+                                          <vscale x 8 x i16> %v3,
+                                          <vscale x 8 x i1> %pred,
+                                          <vscale x 8 x i16>* %addr)
+  ret void
+}
+
+define void @st4h_f16(<vscale x 8 x half> %v0, <vscale x 8 x half> %v1, <vscale x 8 x half> %v2, <vscale x 8 x half> %v3, <vscale x 8 x i1> %pred, <vscale x 8 x half>* %addr) {
+; CHECK-LABEL: st4h_f16:
+; CHECK: st4h { z0.h, z1.h, z2.h, z3.h }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st4.nxv8f16(<vscale x 8 x half> %v0,
+                                          <vscale x 8 x half> %v1,
+                                          <vscale x 8 x half> %v2,
+                                          <vscale x 8 x half> %v3,
+                                          <vscale x 8 x i1> %pred,
+                                          <vscale x 8 x half>* %addr)
+  ret void
+}
+
+;
+; ST4W
+;
+
+define void @st4w_i32(<vscale x 4 x i32> %v0, <vscale x 4 x i32> %v1, <vscale x 4 x i32> %v2, <vscale x 4 x i32> %v3, <vscale x 4 x i1> %pred, <vscale x 4 x i32>* %addr) {
+; CHECK-LABEL: st4w_i32:
+; CHECK: st4w { z0.s, z1.s, z2.s, z3.s }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st4.nxv4i32(<vscale x 4 x i32> %v0,
+                                          <vscale x 4 x i32> %v1,
+                                          <vscale x 4 x i32> %v2,
+                                          <vscale x 4 x i32> %v3,
+                                          <vscale x 4 x i1> %pred,
+                                          <vscale x 4 x i32>* %addr)
+  ret void
+}
+
+define void @st4w_f32(<vscale x 4 x float> %v0, <vscale x 4 x float> %v1, <vscale x 4 x float> %v2, <vscale x 4 x float> %v3, <vscale x 4 x i1> %pred, <vscale x 4 x float>* %addr) {
+; CHECK-LABEL: st4w_f32:
+; CHECK: st4w { z0.s, z1.s, z2.s, z3.s }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st4.nxv4f32(<vscale x 4 x float> %v0,
+                                          <vscale x 4 x float> %v1,
+                                          <vscale x 4 x float> %v2,
+                                          <vscale x 4 x float> %v3,
+                                          <vscale x 4 x i1> %pred,
+                                          <vscale x 4 x float>* %addr)
+  ret void
+}
+
+;
+; ST4D
+;
+
+define void @st4d_i64(<vscale x 2 x i64> %v0, <vscale x 2 x i64> %v1, <vscale x 2 x i64> %v2, <vscale x 2 x i64> %v3, <vscale x 2 x i1> %pred, <vscale x 2 x i64>* %addr) {
+; CHECK-LABEL: st4d_i64:
+; CHECK: st4d { z0.d, z1.d, z2.d, z3.d }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st4.nxv2i64(<vscale x 2 x i64> %v0,
+                                          <vscale x 2 x i64> %v1,
+                                          <vscale x 2 x i64> %v2,
+                                          <vscale x 2 x i64> %v3,
+                                          <vscale x 2 x i1> %pred,
+                                          <vscale x 2 x i64>* %addr)
+  ret void
+}
+
+define void @st4d_f64(<vscale x 2 x double> %v0, <vscale x 2 x double> %v1, <vscale x 2 x double> %v2, <vscale x 2 x double> %v3, <vscale x 2 x i1> %pred, <vscale x 2 x double>* %addr) {
+; CHECK-LABEL: st4d_f64:
+; CHECK: st4d { z0.d, z1.d, z2.d, z3.d }, p0, [x0]
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.st4.nxv2f64(<vscale x 2 x double> %v0,
+                                          <vscale x 2 x double> %v1,
+                                          <vscale x 2 x double> %v2,
+                                          <vscale x 2 x double> %v3,
+                                          <vscale x 2 x i1> %pred,
+                                          <vscale x 2 x double>* %addr)
+  ret void
+}
+
 ;
 ; STNT1B
 ;
@@ -86,6 +386,31 @@ define void @stnt1d_f64(<vscale x 2 x double> %data, <vscale x 2 x i1> %pred, do
   ret void
 }
 
+
+declare void @llvm.aarch64.sve.st2.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i1>, <vscale x 16 x i8>*)
+declare void @llvm.aarch64.sve.st2.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i1>, <vscale x 8 x i16>*)
+declare void @llvm.aarch64.sve.st2.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x i32>*)
+declare void @llvm.aarch64.sve.st2.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>*)
+declare void @llvm.aarch64.sve.st2.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x i1>, <vscale x 8 x half>*)
+declare void @llvm.aarch64.sve.st2.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x i1>, <vscale x 4 x float>*)
+declare void @llvm.aarch64.sve.st2.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x i1>, <vscale x 2 x double>*)
+
+declare void @llvm.aarch64.sve.st3.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i1>, <vscale x 16 x i8>*)
+declare void @llvm.aarch64.sve.st3.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i1>, <vscale x 8 x i16>*)
+declare void @llvm.aarch64.sve.st3.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x i32>*)
+declare void @llvm.aarch64.sve.st3.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>*)
+declare void @llvm.aarch64.sve.st3.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x i1>, <vscale x 8 x half>*)
+declare void @llvm.aarch64.sve.st3.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x i1>, <vscale x 4 x float>*)
+declare void @llvm.aarch64.sve.st3.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x i1>, <vscale x 2 x double>*)
+
+declare void @llvm.aarch64.sve.st4.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i1>, <vscale x 16 x i8>*)
+declare void @llvm.aarch64.sve.st4.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i1>, <vscale x 8 x i16>*)
+declare void @llvm.aarch64.sve.st4.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x i32>*)
+declare void @llvm.aarch64.sve.st4.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>*)
+declare void @llvm.aarch64.sve.st4.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x i1>, <vscale x 8 x half>*)
+declare void @llvm.aarch64.sve.st4.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x i1>, <vscale x 4 x float>*)
+declare void @llvm.aarch64.sve.st4.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x i1>, <vscale x 2 x double>*)
+
 declare void @llvm.aarch64.sve.stnt1.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, i8*)
 declare void @llvm.aarch64.sve.stnt1.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, i16*)
 declare void @llvm.aarch64.sve.stnt1.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, i32*)
diff --git a/llvm/test/CodeGen/AArch64/swap-compare-operands.ll b/llvm/test/CodeGen/AArch64/swap-compare-operands.ll
index 7c19b911166e3..b106e15c23e30 100644
--- a/llvm/test/CodeGen/AArch64/swap-compare-operands.ll
+++ b/llvm/test/CodeGen/AArch64/swap-compare-operands.ll
@@ -1,7 +1,7 @@
 ; RUN: llc < %s -mtriple=arm64 | FileCheck %s
 
 define i1 @testSwapCmpWithLSL64_1(i64 %a, i64 %b) {
-; CHECK-LABEL testSwapCmpWithLSL64_1:
+; CHECK-LABEL: testSwapCmpWithLSL64_1:
 ; CHECK:      cmp     x1, x0, lsl #1
 ; CHECK-NEXT: cset    w0, gt
 entry:
@@ -11,7 +11,7 @@ entry:
 }
 
 define i1 @testSwapCmpWithLSL64_63(i64 %a, i64 %b) {
-; CHECK-LABEL testSwapCmpWithLSL64_63:
+; CHECK-LABEL: testSwapCmpWithLSL64_63:
 ; CHECK:      cmp     x1, x0, lsl #63
 ; CHECK-NEXT: cset    w0, gt
 entry:
@@ -21,7 +21,7 @@ entry:
 }
 
 define i1 @testSwapCmpWithLSL32_1(i32 %a, i32 %b) {
-; CHECK-LABEL testSwapCmpWithLSL32_1:
+; CHECK-LABEL: testSwapCmpWithLSL32_1:
 ; CHECK:      cmp     w1, w0, lsl #1
 ; CHECK-NEXT: cset    w0, gt
 entry:
@@ -31,7 +31,7 @@ entry:
 }
 
 define i1 @testSwapCmpWithLSL32_31(i32 %a, i32 %b) {
-; CHECK-LABEL testSwapCmpWithLSL32_31:
+; CHECK-LABEL: testSwapCmpWithLSL32_31:
 ; CHECK:      cmp     w1, w0, lsl #31
 ; CHECK-NEXT: cset    w0, gt
 entry:
@@ -41,7 +41,7 @@ entry:
 }
 
 define i1 @testSwapCmpWithLSR64_1(i64 %a, i64 %b) {
-; CHECK-LABEL testSwapCmpWithLSR64_1:
+; CHECK-LABEL: testSwapCmpWithLSR64_1:
 ; CHECK:      cmp     x1, x0, lsr #1
 ; CHECK-NEXT: cset    w0, gt
 entry:
@@ -51,7 +51,7 @@ entry:
 }
 
 define i1 @testSwapCmpWithLSR64_63(i64 %a, i64 %b) {
-; CHECK-LABEL testSwapCmpWithLSR64_63:
+; CHECK-LABEL: testSwapCmpWithLSR64_63:
 ; CHECK:      cmp     x1, x0, lsr #63
 ; CHECK-NEXT: cset    w0, gt
 entry:
@@ -61,7 +61,7 @@ entry:
 }
 
 define i1 @testSwapCmpWithLSR32_1(i32 %a, i32 %b) {
-; CHECK-LABEL testSwapCmpWithLSR32_1:
+; CHECK-LABEL: testSwapCmpWithLSR32_1:
 ; CHECK:      cmp     w1, w0, lsr #1
 ; CHECK-NEXT: cset    w0, gt
 entry:
@@ -71,7 +71,7 @@ entry:
 }
 
 define i1 @testSwapCmpWithLSR32_31(i32 %a, i32 %b) {
-; CHECK-LABEL testSwapCmpWithLSR32_31:
+; CHECK-LABEL: testSwapCmpWithLSR32_31:
 ; CHECK:      cmp     w1, w0, lsr #31
 ; CHECK-NEXT: cset    w0, gt
 entry:
@@ -81,7 +81,7 @@ entry:
 }
 
 define i1 @testSwapCmpWithASR64_1(i64 %a, i64 %b) {
-; CHECK-LABEL testSwapCmpWithASR64_1:
+; CHECK-LABEL: testSwapCmpWithASR64_1:
 ; CHECK:      cmp     x1, x0, asr #1
 ; CHECK-NEXT: cset    w0, gt
 entry:
@@ -91,7 +91,7 @@ entry:
 }
 
 define i1 @testSwapCmpWithASR64_63(i64 %a, i64 %b) {
-; CHECK-LABEL testSwapCmpWithASR64_63:
+; CHECK-LABEL: testSwapCmpWithASR64_63:
 ; CHECK:      cmp     x1, x0, asr #63
 ; CHECK-NEXT: cset    w0, gt
 entry:
@@ -101,7 +101,7 @@ entry:
 }
 
 define i1 @testSwapCmpWithASR32_1(i32 %a, i32 %b) {
-; CHECK-LABEL testSwapCmpWithASR32_1:
+; CHECK-LABEL: testSwapCmpWithASR32_1:
 ; CHECK:      cmp     w1, w0, asr #1
 ; CHECK-NEXT: cset    w0, gt
 entry:
@@ -111,7 +111,7 @@ entry:
 }
 
 define i1 @testSwapCmpWithASR32_31(i32 %a, i32 %b) {
-; CHECK-LABEL testSwapCmpWithASR32_31:
+; CHECK-LABEL: testSwapCmpWithASR32_31:
 ; CHECK:      cmp     w1, w0, asr #31
 ; CHECK-NEXT: cset    w0, gt
 entry:
@@ -121,7 +121,7 @@ entry:
 }
 
 define i1 @testSwapCmpWithShiftedZeroExtend32_64(i32 %a, i64 %b) {
-; CHECK-LABEL testSwapCmpWithShiftedZeroExtend32_64
+; CHECK-LABEL: testSwapCmpWithShiftedZeroExtend32_64
 ; CHECK:      cmp    x1, w0, uxtw #2
 ; CHECK-NEXT: cset   w0, lo
 entry:
@@ -132,7 +132,7 @@ entry:
 }
 
 define i1 @testSwapCmpWithShiftedZeroExtend16_64(i16 %a, i64 %b) {
-; CHECK-LABEL testSwapCmpWithShiftedZeroExtend16_64
+; CHECK-LABEL: testSwapCmpWithShiftedZeroExtend16_64
 ; CHECK:      cmp    x1, w0, uxth #2
 ; CHECK-NEXT: cset   w0, lo
 entry:
@@ -143,7 +143,7 @@ entry:
 }
 
 define i1 @testSwapCmpWithShiftedZeroExtend8_64(i8 %a, i64 %b) {
-; CHECK-LABEL testSwapCmpWithShiftedZeroExtend8_64
+; CHECK-LABEL: testSwapCmpWithShiftedZeroExtend8_64
 ; CHECK:      cmp    x1, w0, uxtb #4
 ; CHECK-NEXT: cset    w0, lo
 entry:
@@ -154,7 +154,7 @@ entry:
 }
 
 define i1 @testSwapCmpWithShiftedZeroExtend16_32(i16 %a, i32 %b) {
-; CHECK-LABEL testSwapCmpWithShiftedZeroExtend8_64
+; CHECK-LABEL: testSwapCmpWithShiftedZeroExtend16_32
 ; CHECK:      cmp    w1, w0, uxth #3
 ; CHECK-NEXT: cset    w0, lo
 entry:
@@ -165,7 +165,7 @@ entry:
 }
 
 define i1 @testSwapCmpWithShiftedZeroExtend8_32(i8 %a, i32 %b) {
-; CHECK-LABEL testSwapCmpWithShiftedZeroExtend8_64
+; CHECK-LABEL: testSwapCmpWithShiftedZeroExtend8_32
 ; CHECK:      cmp    w1, w0, uxtb #4
 ; CHECK-NEXT: cset    w0, lo
 entry:
@@ -176,7 +176,7 @@ entry:
 }
 
 define i1 @testSwapCmpWithTooLargeShiftedZeroExtend8_32(i8 %a, i32 %b) {
-; CHECK-LABEL testSwapCmpWithTooLargeShiftedZeroExtend8_64
+; CHECK-LABEL: testSwapCmpWithTooLargeShiftedZeroExtend8_32
 ; CHECK:      and    [[REG:w[0-9]+]], w0, #0xff
 ; CHECK:      cmp    w1, [[REG]], lsl #5
 ; CHECK-NEXT: cset   w0, lo
@@ -188,7 +188,7 @@ entry:
 }
 
 define i1 @testSwapCmpWithZeroExtend8_32(i8 %a, i32 %b) {
-; CHECK-LABEL testSwapCmpWithZeroExtend8_64
+; CHECK-LABEL: testSwapCmpWithZeroExtend8_32
 ; CHECK:      cmp    w1, w0, uxtb
 ; CHECK-NEXT: cset   w0, lo
 entry:
@@ -198,7 +198,7 @@ entry:
 }
 
 define i1 @testSwapCmpWithShiftedSignExtend32_64(i32 %a, i64 %b) {
-; CHECK-LABEL testSwapCmpWithShiftedZeroExtend32_64
+; CHECK-LABEL: testSwapCmpWithShiftedSignExtend32_64
 ; CHECK:      cmp    x1, w0, sxtw #2
 ; CHECK-NEXT: cset   w0, lo
 entry:
@@ -209,7 +209,7 @@ entry:
 }
 
 define i1 @testSwapCmpWithShiftedSignExtend16_64(i16 %a, i64 %b) {
-; CHECK-LABEL testSwapCmpWithShiftedZeroExtend16_64
+; CHECK-LABEL: testSwapCmpWithShiftedSignExtend16_64
 ; CHECK:      cmp    x1, w0, sxth #2
 ; CHECK-NEXT: cset   w0, lo
 entry:
@@ -220,7 +220,7 @@ entry:
 }
 
 define i1 @testSwapCmpWithShiftedSignExtend8_64(i8 %a, i64 %b) {
-; CHECK-LABEL testSwapCmpWithShiftedSignExtend8_64
+; CHECK-LABEL: testSwapCmpWithShiftedSignExtend8_64
 ; CHECK:      cmp    x1, w0, sxtb #4
 ; CHECK-NEXT: cset    w0, lo
 entry:
@@ -231,7 +231,7 @@ entry:
 }
 
 define i1 @testSwapCmpWithShiftedSignExtend16_32(i16 %a, i32 %b) {
-; CHECK-LABEL testSwapCmpWithShiftedSignExtend8_64
+; CHECK-LABEL: testSwapCmpWithShiftedSignExtend16_32
 ; CHECK:      cmp    w1, w0, sxth #3
 ; CHECK-NEXT: cset    w0, lo
 entry:
@@ -242,7 +242,7 @@ entry:
 }
 
 define i1 @testSwapCmpWithShiftedSignExtend8_32(i8 %a, i32 %b) {
-; CHECK-LABEL testSwapCmpWithShiftedSignExtend8_64
+; CHECK-LABEL: testSwapCmpWithShiftedSignExtend8_32
 ; CHECK:      cmp    w1, w0, sxtb #4
 ; CHECK-NEXT: cset   w0, lo
 entry:
@@ -253,7 +253,7 @@ entry:
 }
 
 define i1 @testSwapCmpWithTooLargeShiftedSignExtend8_32(i8 %a, i32 %b) {
-; CHECK-LABEL testSwapCmpWithTooLargeShiftedSignExtend8_64
+; CHECK-LABEL: testSwapCmpWithTooLargeShiftedSignExtend8_32
 ; CHECK:      sxtb   [[REG:w[0-9]+]], w0
 ; CHECK-NEXT: cmp    w1, [[REG]], lsl #5
 ; CHECK-NEXT: cset   w0, lo
@@ -265,7 +265,7 @@ entry:
 }
 
 define i1 @testSwapCmpWithSignExtend8_32(i8 %a, i32 %b) {
-; CHECK-LABEL testSwapCmpWithSignExtend8_64
+; CHECK-LABEL: testSwapCmpWithSignExtend8_32
 ; CHECK:      cmp    w1, w0, sxtb
 ; CHECK-NEXT: cset   w0, lo
 entry:
@@ -275,7 +275,7 @@ entry:
 }
 
 define i1 @testSwapCmnWithLSL64_1(i64 %a, i64 %b) {
-; CHECK-LABEL testSwapCmnWithLSL64_1:
+; CHECK-LABEL: testSwapCmnWithLSL64_1:
 ; CHECK:      cmn    x1, x0, lsl #1
 ; CHECK-NEXT: cset   w0, ne
 entry:
@@ -287,7 +287,7 @@ entry:
 
 ; Note: testing with a 62 bits shift as 63 has another optimization kicking in.
 define i1 @testSwapCmnWithLSL64_62(i64 %a, i64 %b) {
-; CHECK-LABEL testSwapCmnWithLSL64_62:
+; CHECK-LABEL: testSwapCmnWithLSL64_62:
 ; CHECK:      cmn    x1, x0, lsl #62
 ; CHECK-NEXT: cset   w0, ne
 entry:
@@ -302,7 +302,7 @@ entry:
 ; so that this test can be adapted should the optimization be done with the
 ; operand swap.
 define i1 @testSwapCmnWithLSL64_63(i64 %a, i64 %b) {
-; CHECK-LABEL testSwapCmnWithLSL64_63:
+; CHECK-LABEL: testSwapCmnWithLSL64_63:
 ; CHECK:      cmp    x1, x0, lsl #63
 ; CHECK-NEXT: cset   w0, ne
 entry:
@@ -313,7 +313,7 @@ entry:
 }
 
 define i1 @testSwapCmnWithLSL32_1(i32 %a, i32 %b) {
-; CHECK-LABEL testSwapCmnWithLSL32_1:
+; CHECK-LABEL: testSwapCmnWithLSL32_1:
 ; CHECK:      cmn    w1, w0, lsl #1
 ; CHECK-NEXT: cset   w0, ne
 entry:
@@ -325,7 +325,7 @@ entry:
 
 ; Note: testing with a 30 bits shift as 30 has another optimization kicking in.
 define i1 @testSwapCmnWithLSL32_30(i32 %a, i32 %b) {
-; CHECK-LABEL testSwapCmnWithLSL32_30:
+; CHECK-LABEL: testSwapCmnWithLSL32_30:
 ; CHECK:      cmn    w1, w0, lsl #30
 ; CHECK-NEXT: cset   w0, ne
 entry:
@@ -340,7 +340,7 @@ entry:
 ; so that this test can be adapted should the optimization be done with the
 ; operand swap.
 define i1 @testSwapCmnWithLSL32_31(i32 %a, i32 %b) {
-; CHECK-LABEL testSwapCmnWithLSL32_31:
+; CHECK-LABEL: testSwapCmnWithLSL32_31:
 ; CHECK:      cmp    w1, w0, lsl #31
 ; CHECK-NEXT: cset   w0, ne
 entry:
@@ -351,7 +351,7 @@ entry:
 }
 
 define i1 @testSwapCmnWithLSR64_1(i64 %a, i64 %b) {
-; CHECK-LABEL testSwapCmnWithLSR64_1:
+; CHECK-LABEL: testSwapCmnWithLSR64_1:
 ; CHECK:      cmn    x1, x0, lsr #1
 ; CHECK-NEXT: cset   w0, ne
 entry:
@@ -363,7 +363,7 @@ entry:
 
 ; Note: testing with a 62 bits shift as 63 has another optimization kicking in.
 define i1 @testSwapCmnWithLSR64_62(i64 %a, i64 %b) {
-; CHECK-LABEL testSwapCmnWithLSR64_62:
+; CHECK-LABEL: testSwapCmnWithLSR64_62:
 ; CHECK:      cmn    x1, x0, lsr #62
 ; CHECK-NEXT: cset   w0, ne
 entry:
@@ -378,7 +378,7 @@ entry:
 ; so that this test can be adapted should the optimization be done with the
 ; operand swap.
 define i1 @testSwapCmnWithLSR64_63(i64 %a, i64 %b) {
-; CHECK-LABEL testSwapCmnWithLSR64_63:
+; CHECK-LABEL: testSwapCmnWithLSR64_63:
 ; CHECK:      cmp    x1, x0, asr #63
 ; CHECK-NEXT: cset   w0, ne
 entry:
@@ -389,7 +389,7 @@ entry:
 }
 
 define i1 @testSwapCmnWithLSR32_1(i32 %a, i32 %b) {
-; CHECK-LABEL testSwapCmnWithLSR32_1:
+; CHECK-LABEL: testSwapCmnWithLSR32_1:
 ; CHECK:      cmn    w1, w0, lsr #1
 ; CHECK-NEXT: cset   w0, ne
 entry:
@@ -401,7 +401,7 @@ entry:
 
 ; Note: testing with a 30 bits shift as 31 has another optimization kicking in.
 define i1 @testSwapCmnWithLSR32_30(i32 %a, i32 %b) {
-; CHECK-LABEL testSwapCmnWithLSR32_30:
+; CHECK-LABEL: testSwapCmnWithLSR32_30:
 ; CHECK:      cmn    w1, w0, lsr #30
 ; CHECK-NEXT: cset   w0, ne
 entry:
@@ -416,7 +416,7 @@ entry:
 ; so that this test can be adapted should the optimization be done with the
 ; operand swap.
 define i1 @testSwapCmnWithLSR32_31(i32 %a, i32 %b) {
-; CHECK-LABEL testSwapCmnWithLSR32_31:
+; CHECK-LABEL: testSwapCmnWithLSR32_31:
 ; CHECK:      cmp    w1, w0, asr #31
 ; CHECK-NEXT: cset   w0, ne
 entry:
@@ -427,7 +427,7 @@ entry:
 }
 
 define i1 @testSwapCmnWithASR64_1(i64 %a, i64 %b) {
-; CHECK-LABEL testSwapCmnWithASR64_1:
+; CHECK-LABEL: testSwapCmnWithASR64_1:
 ; CHECK:      cmn    x1, x0, asr #3
 ; CHECK-NEXT: cset   w0, ne
 entry:
@@ -439,7 +439,7 @@ entry:
 
 ; Note: testing with a 62 bits shift as 63 has another optimization kicking in.
 define i1 @testSwapCmnWithASR64_62(i64 %a, i64 %b) {
-; CHECK-LABEL testSwapCmnWithASR64_62:
+; CHECK-LABEL: testSwapCmnWithASR64_62:
 ; CHECK:      cmn    x1, x0, asr #62
 ; CHECK-NEXT: cset   w0, ne
 entry:
@@ -454,7 +454,7 @@ entry:
 ; so that this test can be adapted should the optimization be done with the
 ; operand swap.
 define i1 @testSwapCmnWithASR64_63(i64 %a, i64 %b) {
-; CHECK-LABEL testSwapCmnWithASR64_63:
+; CHECK-LABEL: testSwapCmnWithASR64_63:
 ; CHECK:      cmp    x1, x0, lsr #63
 ; CHECK-NEXT: cset   w0, ne
 entry:
@@ -465,7 +465,7 @@ entry:
 }
 
 define i1 @testSwapCmnWithASR32_1(i32 %a, i32 %b) {
-; CHECK-LABEL testSwapCmnWithASR32_1:
+; CHECK-LABEL: testSwapCmnWithASR32_1:
 ; CHECK:      cmn    w1, w0, asr #1
 ; CHECK-NEXT: cset   w0, eq
 entry:
@@ -477,7 +477,7 @@ entry:
 
 ; Note: testing with a 30 bits shift as 31 has another optimization kicking in.
 define i1 @testSwapCmnWithASR32_30(i32 %a, i32 %b) {
-; CHECK-LABEL testSwapCmnWithASR32_30:
+; CHECK-LABEL: testSwapCmnWithASR32_30:
 ; CHECK:      cmn    w1, w0, asr #30
 ; CHECK-NEXT: cset   w0, ne
 entry:
@@ -492,7 +492,7 @@ entry:
 ; so that this test can be adapted should the optimization be done with the
 ; operand swap.
 define i1 @testSwapCmnWithASR32_31(i32 %a, i32 %b) {
-; CHECK-LABEL testSwapCmnWithASR32_31:
+; CHECK-LABEL: testSwapCmnWithASR32_31:
 ; CHECK:      cmp    w1, w0, lsr #31
 ; CHECK-NEXT: cset   w0, ne
 entry:
@@ -503,7 +503,7 @@ entry:
 }
 
 define i64 @testSwapCmpToCmnWithZeroExtend(i32 %a32, i16 %a16, i8 %a8, i64 %b64, i32 %b32) {
-; CHECK-LABEL testSwapCmpToCmnWithZeroExtend:
+; CHECK-LABEL: testSwapCmpToCmnWithZeroExtend:
 t0:
   %conv0 = zext i32 %a32 to i64
   %shl0 = shl i64 %conv0, 1
@@ -567,7 +567,7 @@ end:
   ret i64 1
 }
 define i64 @testSwapCmpToCmnWithSignExtend(i32 %a32, i16 %a16, i8 %a8, i64 %b64, i32 %b32) {
-; CHECK-LABEL testSwapCmpToCmnWithSignExtend:
+; CHECK-LABEL: testSwapCmpToCmnWithSignExtend:
 t0:
   %conv0 = sext i32 %a32 to i64
   %shl0 = shl i64 %conv0, 1
diff --git a/llvm/test/CodeGen/AArch64/wineh-try-catch-cbz.ll b/llvm/test/CodeGen/AArch64/wineh-try-catch-cbz.ll
index d84c07f8bc1a5..cbed64ab99e3d 100644
--- a/llvm/test/CodeGen/AArch64/wineh-try-catch-cbz.ll
+++ b/llvm/test/CodeGen/AArch64/wineh-try-catch-cbz.ll
@@ -4,11 +4,10 @@
 ; but the original issue only reproduced if the cbz was immediately
 ; after the frame setup.)
 
-; CHECK:      sub     sp, sp, #32
-; CHECK-NEXT: stp     x29, x30, [sp, #16]
-; CHECK-NEXT: add     x29, sp, #16
+; CHECK: stp     x29, x30, [sp, #-32]!
+; CHECK-NEXT: mov     x29, sp
 ; CHECK-NEXT: mov     x1, #-2
-; CHECK-NEXT: stur    x1, [x29, #-16]
+; CHECK-NEXT: stur    x1, [x29, #16]
 ; CHECK-NEXT: cbz     w0, .LBB0_2
 
 target datalayout = "e-m:w-p:64:64-i32:32-i64:64-i128:128-n32:64-S128"
diff --git a/llvm/test/CodeGen/AArch64/wineh-try-catch-realign.ll b/llvm/test/CodeGen/AArch64/wineh-try-catch-realign.ll
index b10a0f3033a02..a66ec10748e77 100644
--- a/llvm/test/CodeGen/AArch64/wineh-try-catch-realign.ll
+++ b/llvm/test/CodeGen/AArch64/wineh-try-catch-realign.ll
@@ -12,7 +12,7 @@
 ; CHECK:      stp     x29, x30, [sp, #-32]!
 ; CHECK-NEXT: str     x28, [sp, #16]
 ; CHECK-NEXT: str     x19, [sp, #24]
-; CHECK-NEXT: add     x0, x19, #64
+; CHECK-NEXT: add     x0, x19, #0
 ; CHECK-NEXT: mov     w1, wzr
 ; CHECK-NEXT: bl      "?bb@@YAXPEAHH@Z"
 ; CHECK-NEXT: adrp    x0, .LBB0_1
diff --git a/llvm/test/CodeGen/AArch64/wineh-try-catch.ll b/llvm/test/CodeGen/AArch64/wineh-try-catch.ll
index 3ae2df37efe4b..73909825d377d 100644
--- a/llvm/test/CodeGen/AArch64/wineh-try-catch.ll
+++ b/llvm/test/CodeGen/AArch64/wineh-try-catch.ll
@@ -11,11 +11,11 @@
 ;    and the parent function.
 
 ; The following checks that the unwind help object has -2 stored into it at
-; fp - 400 - 256 = fp - 656, which is on-entry sp - 48 + 32 - 656 =
-; on-entry sp - 672.  We check this offset in the table later on.
+; fp + 16, which is on-entry sp - 16.
+; We check this offset in the table later on.
 
 ; CHECK-LABEL: "?func@@YAHXZ":
-; CHECK:       stp     x29, x30, [sp, #-48]!
+; CHECK:       stp     x29, x30, [sp, #-64]!
 ; CHECK:       str     x28, [sp, #16]
 ; CHECK:       str     x21, [sp, #24]
 ; CHECK:       stp     x19, x20, [sp, #32]
@@ -23,7 +23,7 @@
 ; CHECK:       sub     sp, sp, #624
 ; CHECK:       mov     x19, sp
 ; CHECK:       mov     x0, #-2
-; CHECK:       stur    x0, [x19]
+; CHECK:       stur    x0, [x29, #48]
 
 ; Now check that x is stored at fp - 20.  We check that this is the same
 ; location accessed from the funclet to retrieve x.
@@ -72,7 +72,7 @@
 
 ; Now check that the offset of the unwind help object from the stack pointer on
 ; entry to func is encoded in cppxdata that is passed to __CxxFrameHandler3.  As
-; computed above, this comes to -672.
+; computed above, this comes to -16.
 ; CHECK-LABEL:        "$cppxdata$?func@@YAHXZ":
 ; CHECK-NEXT:         .word   429065506               ; MagicNumber
 ; CHECK-NEXT:         .word   2                       ; MaxState
@@ -81,7 +81,7 @@
 ; CHECK-NEXT:         .word   ("$tryMap$?func@@YAHXZ")@IMGREL ; TryBlockMap
 ; CHECK-NEXT:         .word   4                       ; IPMapEntries
 ; CHECK-NEXT:         .word   ("$ip2state$?func@@YAHXZ")@IMGREL ; IPToStateXData
-; CHECK-NEXT:         .word   -672                    ; UnwindHelp
+; CHECK-NEXT:         .word   -16                     ; UnwindHelp
 
 ; UNWIND: Function: ?func@@YAHXZ (0x0)
 ; UNWIND: Prologue [
@@ -91,7 +91,7 @@
 ; UNWIND-NEXT: ; stp x19, x20, [sp, #32]
 ; UNWIND-NEXT: ; str x21, [sp, #24]
 ; UNWIND-NEXT: ; str x28, [sp, #16]
-; UNWIND-NEXT: ; stp x29, x30, [sp, #-48]!
+; UNWIND-NEXT: ; stp x29, x30, [sp, #-64]!
 ; UNWIND-NEXT: ; end
 ; UNWIND: Function: ?catch$2@?0??func@@YAHXZ@4HA
 ; UNWIND: Prologue [
diff --git a/llvm/test/CodeGen/AArch64/wineh-unwindhelp-via-fp.ll b/llvm/test/CodeGen/AArch64/wineh-unwindhelp-via-fp.ll
new file mode 100644
index 0000000000000..6ec78087020c4
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/wineh-unwindhelp-via-fp.ll
@@ -0,0 +1,69 @@
+; RUN: llc -o - %s -mtriple=aarch64-windows | FileCheck %s
+; Check that we allocate the unwind help stack object in a fixed location from fp
+; so that the runtime can find it when handling an exception
+target datalayout = "e-m:w-p:64:64-i32:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-pc-windows-msvc19.25.28611"
+
+; Check that the store to the unwind help object for func2 is via FP
+; CHECK-LABEL: ?func2@@YAXXZ
+; CHECK: mov x[[#SCRATCH_REG:]], #-2
+; CHECK: stur x[[#SCRATCH_REG:]], [x29, #[[#]]]
+;
+; // struct that requires greater than stack alignment
+; struct alignas(32) A
+; {
+;     // data that would be invalid for unwind help (> 0)
+;     int _x[4]{42, 42, 42, 42};
+;     ~A() {}
+; };
+; 
+; // cause us to run the funclet in func2
+; void func3()
+; {
+;     throw 1;
+; }
+; 
+; // the funclet that ensures we have the unwind help correct
+; void func2()
+; {
+;     A a;
+;     func3();
+; }
+; 
+; // function to ensure we are misaligned in func2
+; void func1()
+; {
+;     func2();
+; }
+; 
+; // set things up and ensure alignment for func1
+; void test()
+; {
+;     try {
+;         A a;
+;         func1();
+;     } catch(...) {}
+; }
+
+%struct.A = type { [4 x i32], [16 x i8] }
+declare dso_local %struct.A* @"??0A@@QEAA@XZ"(%struct.A* returned %0)
+declare dso_local void @"??1A@@QEAA@XZ"(%struct.A* %0)
+declare dso_local i32 @__CxxFrameHandler3(...)
+declare dso_local void @"?func3@@YAXXZ"()
+
+; Function Attrs: noinline optnone uwtable
+define dso_local void @"?func2@@YAXXZ"() #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+  %1 = alloca %struct.A, align 32
+  %2 = call %struct.A* @"??0A@@QEAA@XZ"(%struct.A* %1) #3
+  invoke void @"?func3@@YAXXZ"()
+          to label %3 unwind label %4
+
+3:                                                ; preds = %0
+  call void @"??1A@@QEAA@XZ"(%struct.A* %1) #3
+  ret void
+
+4:                                                ; preds = %0
+  %5 = cleanuppad within none []
+  call void @"??1A@@QEAA@XZ"(%struct.A* %1) #3 [ "funclet"(token %5) ]
+  cleanupret from %5 unwind to caller
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-itofp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-itofp.mir
new file mode 100644
index 0000000000000..2021107b84485
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-itofp.mir
@@ -0,0 +1,175 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -run-pass=amdgpu-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name: uitofp_char_to_f32
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: uitofp_char_to_f32
+    ; CHECK: liveins: $vgpr0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; CHECK: [[AMDGPU_CVT_F32_UBYTE0_:%[0-9]+]]:_(s32) = G_AMDGPU_CVT_F32_UBYTE0 [[AND]]
+    ; CHECK: $vgpr0 = COPY [[AMDGPU_CVT_F32_UBYTE0_]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_CONSTANT i32 255
+    %2:_(s32) = G_AND %0, %1
+    %3:_(s32) = G_UITOFP %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: uitofp_too_many_bits_to_f32
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: uitofp_too_many_bits_to_f32
+    ; CHECK: liveins: $vgpr0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 256
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; CHECK: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND]](s32)
+    ; CHECK: $vgpr0 = COPY [[UITOFP]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_CONSTANT i32 256
+    %2:_(s32) = G_AND %0, %1
+    %3:_(s32) = G_UITOFP %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: sitofp_char_to_f32
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: sitofp_char_to_f32
+    ; CHECK: liveins: $vgpr0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; CHECK: [[AMDGPU_CVT_F32_UBYTE0_:%[0-9]+]]:_(s32) = G_AMDGPU_CVT_F32_UBYTE0 [[AND]]
+    ; CHECK: $vgpr0 = COPY [[AMDGPU_CVT_F32_UBYTE0_]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_CONSTANT i32 255
+    %2:_(s32) = G_AND %0, %1
+    %3:_(s32) = G_SITOFP %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: sitofp_bits127_to_f32
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: sitofp_bits127_to_f32
+    ; CHECK: liveins: $vgpr0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; CHECK: [[AMDGPU_CVT_F32_UBYTE0_:%[0-9]+]]:_(s32) = G_AMDGPU_CVT_F32_UBYTE0 [[AND]]
+    ; CHECK: $vgpr0 = COPY [[AMDGPU_CVT_F32_UBYTE0_]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_CONSTANT i32 127
+    %2:_(s32) = G_AND %0, %1
+    %3:_(s32) = G_SITOFP %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: sitofp_bits128_to_f32
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: sitofp_bits128_to_f32
+    ; CHECK: liveins: $vgpr0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 128
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; CHECK: [[AMDGPU_CVT_F32_UBYTE0_:%[0-9]+]]:_(s32) = G_AMDGPU_CVT_F32_UBYTE0 [[AND]]
+    ; CHECK: $vgpr0 = COPY [[AMDGPU_CVT_F32_UBYTE0_]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_CONSTANT i32 128
+    %2:_(s32) = G_AND %0, %1
+    %3:_(s32) = G_SITOFP %2
+    $vgpr0 = COPY %3
+...
+---
+name: sitofp_too_many_bits_to_f32
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: sitofp_too_many_bits_to_f32
+    ; CHECK: liveins: $vgpr0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 256
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; CHECK: [[SITOFP:%[0-9]+]]:_(s32) = G_SITOFP [[AND]](s32)
+    ; CHECK: $vgpr0 = COPY [[SITOFP]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_CONSTANT i32 256
+    %2:_(s32) = G_AND %0, %1
+    %3:_(s32) = G_SITOFP %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: uitofp_char_to_f16
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: uitofp_char_to_f16
+    ; CHECK: liveins: $vgpr0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; CHECK: [[AMDGPU_CVT_F32_UBYTE0_:%[0-9]+]]:_(s32) = G_AMDGPU_CVT_F32_UBYTE0 [[AND]]
+    ; CHECK: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[AMDGPU_CVT_F32_UBYTE0_]](s32)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC]](s16)
+    ; CHECK: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_CONSTANT i32 255
+    %2:_(s32) = G_AND %0, %1
+    %3:_(s16) = G_UITOFP %2
+    %4:_(s32) = G_ANYEXT %3
+    $vgpr0 = COPY %4
+...
+
+---
+name: sitofp_char_to_f16
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: sitofp_char_to_f16
+    ; CHECK: liveins: $vgpr0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; CHECK: [[AMDGPU_CVT_F32_UBYTE0_:%[0-9]+]]:_(s32) = G_AMDGPU_CVT_F32_UBYTE0 [[AND]]
+    ; CHECK: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[AMDGPU_CVT_F32_UBYTE0_]](s32)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC]](s16)
+    ; CHECK: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_CONSTANT i32 255
+    %2:_(s32) = G_AND %0, %1
+    %3:_(s16) = G_SITOFP %2
+    %4:_(s32) = G_ANYEXT %3
+    $vgpr0 = COPY %4
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/image_ls_mipmap_zero.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/image_ls_mipmap_zero.a16.ll
new file mode 100644
index 0000000000000..1475ccf83d2a1
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/image_ls_mipmap_zero.a16.ll
@@ -0,0 +1,717 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX10 %s
+
+define amdgpu_ps <4 x float> @load_mip_1d(<8 x i32> inreg %rsrc, i16 %s) {
+  ; GFX9-LABEL: name: load_mip_1d
+  ; GFX9: bb.1.main_body:
+  ; GFX9:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0
+  ; GFX9:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
+  ; GFX9:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
+  ; GFX9:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
+  ; GFX9:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
+  ; GFX9:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
+  ; GFX9:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
+  ; GFX9:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
+  ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
+  ; GFX9:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY [[COPY8]](s32)
+  ; GFX9:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY9]](s32), [[DEF]](s32)
+  ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.1d), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), 0, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
+  ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
+  ; GFX9:   $vgpr0 = COPY [[UV]](s32)
+  ; GFX9:   $vgpr1 = COPY [[UV1]](s32)
+  ; GFX9:   $vgpr2 = COPY [[UV2]](s32)
+  ; GFX9:   $vgpr3 = COPY [[UV3]](s32)
+  ; GFX9:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+  ; GFX10-LABEL: name: load_mip_1d
+  ; GFX10: bb.1.main_body:
+  ; GFX10:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0
+  ; GFX10:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
+  ; GFX10:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
+  ; GFX10:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
+  ; GFX10:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
+  ; GFX10:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
+  ; GFX10:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
+  ; GFX10:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
+  ; GFX10:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
+  ; GFX10:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10:   [[COPY9:%[0-9]+]]:_(s32) = COPY [[COPY8]](s32)
+  ; GFX10:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX10:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY9]](s32), [[DEF]](s32)
+  ; GFX10:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.1d), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), 0, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
+  ; GFX10:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
+  ; GFX10:   $vgpr0 = COPY [[UV]](s32)
+  ; GFX10:   $vgpr1 = COPY [[UV1]](s32)
+  ; GFX10:   $vgpr2 = COPY [[UV2]](s32)
+  ; GFX10:   $vgpr3 = COPY [[UV3]](s32)
+  ; GFX10:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.load.mip.1d.v4f32.i16(i32 15, i16 %s, i16 0, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @load_mip_2d(<8 x i32> inreg %rsrc, i16 %s, i16 %t) {
+  ; GFX9-LABEL: name: load_mip_2d
+  ; GFX9: bb.1.main_body:
+  ; GFX9:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+  ; GFX9:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
+  ; GFX9:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
+  ; GFX9:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
+  ; GFX9:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
+  ; GFX9:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
+  ; GFX9:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
+  ; GFX9:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
+  ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
+  ; GFX9:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[COPY8]](s32)
+  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[COPY9]](s32)
+  ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.2d), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, 0, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
+  ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
+  ; GFX9:   $vgpr0 = COPY [[UV]](s32)
+  ; GFX9:   $vgpr1 = COPY [[UV1]](s32)
+  ; GFX9:   $vgpr2 = COPY [[UV2]](s32)
+  ; GFX9:   $vgpr3 = COPY [[UV3]](s32)
+  ; GFX9:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+  ; GFX10-LABEL: name: load_mip_2d
+  ; GFX10: bb.1.main_body:
+  ; GFX10:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+  ; GFX10:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
+  ; GFX10:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
+  ; GFX10:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
+  ; GFX10:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
+  ; GFX10:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
+  ; GFX10:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
+  ; GFX10:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
+  ; GFX10:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
+  ; GFX10:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[COPY8]](s32)
+  ; GFX10:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[COPY9]](s32)
+  ; GFX10:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX10:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.2d), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, 0, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
+  ; GFX10:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
+  ; GFX10:   $vgpr0 = COPY [[UV]](s32)
+  ; GFX10:   $vgpr1 = COPY [[UV1]](s32)
+  ; GFX10:   $vgpr2 = COPY [[UV2]](s32)
+  ; GFX10:   $vgpr3 = COPY [[UV3]](s32)
+  ; GFX10:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i16(i32 15, i16 %s, i16 %t, i16 0, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @load_mip_3d(<8 x i32> inreg %rsrc, i16 %s, i16 %t, i16 %u) {
+  ; GFX9-LABEL: name: load_mip_3d
+  ; GFX9: bb.1.main_body:
+  ; GFX9:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2
+  ; GFX9:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
+  ; GFX9:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
+  ; GFX9:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
+  ; GFX9:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
+  ; GFX9:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
+  ; GFX9:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
+  ; GFX9:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
+  ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
+  ; GFX9:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[COPY8]](s32)
+  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[COPY9]](s32)
+  ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY11]](s32), [[COPY12]](s32)
+  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[COPY10]](s32)
+  ; GFX9:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY13]](s32), [[DEF]](s32)
+  ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
+  ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.3d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, 0, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
+  ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
+  ; GFX9:   $vgpr0 = COPY [[UV]](s32)
+  ; GFX9:   $vgpr1 = COPY [[UV1]](s32)
+  ; GFX9:   $vgpr2 = COPY [[UV2]](s32)
+  ; GFX9:   $vgpr3 = COPY [[UV3]](s32)
+  ; GFX9:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+  ; GFX10-LABEL: name: load_mip_3d
+  ; GFX10: bb.1.main_body:
+  ; GFX10:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2
+  ; GFX10:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
+  ; GFX10:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
+  ; GFX10:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
+  ; GFX10:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
+  ; GFX10:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
+  ; GFX10:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
+  ; GFX10:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
+  ; GFX10:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
+  ; GFX10:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[COPY8]](s32)
+  ; GFX10:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[COPY9]](s32)
+  ; GFX10:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY11]](s32), [[COPY12]](s32)
+  ; GFX10:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[COPY10]](s32)
+  ; GFX10:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX10:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY13]](s32), [[DEF]](s32)
+  ; GFX10:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.3d), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>), $noreg, 0, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
+  ; GFX10:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
+  ; GFX10:   $vgpr0 = COPY [[UV]](s32)
+  ; GFX10:   $vgpr1 = COPY [[UV1]](s32)
+  ; GFX10:   $vgpr2 = COPY [[UV2]](s32)
+  ; GFX10:   $vgpr3 = COPY [[UV3]](s32)
+  ; GFX10:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.load.mip.3d.v4f32.i16(i32 15, i16 %s, i16 %t, i16 %u, i16 0, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @load_mip_1darray(<8 x i32> inreg %rsrc, i16 %s, i16 %t) {
+  ; GFX9-LABEL: name: load_mip_1darray
+  ; GFX9: bb.1.main_body:
+  ; GFX9:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+  ; GFX9:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
+  ; GFX9:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
+  ; GFX9:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
+  ; GFX9:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
+  ; GFX9:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
+  ; GFX9:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
+  ; GFX9:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
+  ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
+  ; GFX9:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[COPY8]](s32)
+  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[COPY9]](s32)
+  ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.1darray), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, 0, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
+  ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
+  ; GFX9:   $vgpr0 = COPY [[UV]](s32)
+  ; GFX9:   $vgpr1 = COPY [[UV1]](s32)
+  ; GFX9:   $vgpr2 = COPY [[UV2]](s32)
+  ; GFX9:   $vgpr3 = COPY [[UV3]](s32)
+  ; GFX9:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+  ; GFX10-LABEL: name: load_mip_1darray
+  ; GFX10: bb.1.main_body:
+  ; GFX10:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+  ; GFX10:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
+  ; GFX10:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
+  ; GFX10:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
+  ; GFX10:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
+  ; GFX10:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
+  ; GFX10:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
+  ; GFX10:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
+  ; GFX10:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
+  ; GFX10:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[COPY8]](s32)
+  ; GFX10:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[COPY9]](s32)
+  ; GFX10:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX10:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.1darray), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, 0, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
+  ; GFX10:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
+  ; GFX10:   $vgpr0 = COPY [[UV]](s32)
+  ; GFX10:   $vgpr1 = COPY [[UV1]](s32)
+  ; GFX10:   $vgpr2 = COPY [[UV2]](s32)
+  ; GFX10:   $vgpr3 = COPY [[UV3]](s32)
+  ; GFX10:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.load.mip.1darray.v4f32.i16(i32 15, i16 %s, i16 %t, i16 0, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @load_mip_2darray(<8 x i32> inreg %rsrc, i16 %s, i16 %t, i16 %u) {
+  ; GFX9-LABEL: name: load_mip_2darray
+  ; GFX9: bb.1.main_body:
+  ; GFX9:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2
+  ; GFX9:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
+  ; GFX9:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
+  ; GFX9:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
+  ; GFX9:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
+  ; GFX9:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
+  ; GFX9:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
+  ; GFX9:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
+  ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
+  ; GFX9:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[COPY8]](s32)
+  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[COPY9]](s32)
+  ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY11]](s32), [[COPY12]](s32)
+  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[COPY10]](s32)
+  ; GFX9:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY13]](s32), [[DEF]](s32)
+  ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
+  ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.2darray), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, 0, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
+  ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
+  ; GFX9:   $vgpr0 = COPY [[UV]](s32)
+  ; GFX9:   $vgpr1 = COPY [[UV1]](s32)
+  ; GFX9:   $vgpr2 = COPY [[UV2]](s32)
+  ; GFX9:   $vgpr3 = COPY [[UV3]](s32)
+  ; GFX9:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+  ; GFX10-LABEL: name: load_mip_2darray
+  ; GFX10: bb.1.main_body:
+  ; GFX10:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2
+  ; GFX10:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
+  ; GFX10:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
+  ; GFX10:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
+  ; GFX10:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
+  ; GFX10:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
+  ; GFX10:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
+  ; GFX10:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
+  ; GFX10:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
+  ; GFX10:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[COPY8]](s32)
+  ; GFX10:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[COPY9]](s32)
+  ; GFX10:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY11]](s32), [[COPY12]](s32)
+  ; GFX10:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[COPY10]](s32)
+  ; GFX10:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX10:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY13]](s32), [[DEF]](s32)
+  ; GFX10:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.2darray), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>), $noreg, 0, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
+  ; GFX10:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
+  ; GFX10:   $vgpr0 = COPY [[UV]](s32)
+  ; GFX10:   $vgpr1 = COPY [[UV1]](s32)
+  ; GFX10:   $vgpr2 = COPY [[UV2]](s32)
+  ; GFX10:   $vgpr3 = COPY [[UV3]](s32)
+  ; GFX10:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.load.mip.2darray.v4f32.i16(i32 15, i16 %s, i16 %t, i16 %u, i16 0, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @load_mip_cube(<8 x i32> inreg %rsrc, i16 %s, i16 %t, i16 %u) {
+  ; GFX9-LABEL: name: load_mip_cube
+  ; GFX9: bb.1.main_body:
+  ; GFX9:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2
+  ; GFX9:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
+  ; GFX9:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
+  ; GFX9:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
+  ; GFX9:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
+  ; GFX9:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
+  ; GFX9:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
+  ; GFX9:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
+  ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
+  ; GFX9:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[COPY8]](s32)
+  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[COPY9]](s32)
+  ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY11]](s32), [[COPY12]](s32)
+  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[COPY10]](s32)
+  ; GFX9:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY13]](s32), [[DEF]](s32)
+  ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
+  ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.cube), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, 0, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
+  ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
+  ; GFX9:   $vgpr0 = COPY [[UV]](s32)
+  ; GFX9:   $vgpr1 = COPY [[UV1]](s32)
+  ; GFX9:   $vgpr2 = COPY [[UV2]](s32)
+  ; GFX9:   $vgpr3 = COPY [[UV3]](s32)
+  ; GFX9:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+  ; GFX10-LABEL: name: load_mip_cube
+  ; GFX10: bb.1.main_body:
+  ; GFX10:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2
+  ; GFX10:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
+  ; GFX10:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
+  ; GFX10:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
+  ; GFX10:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
+  ; GFX10:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
+  ; GFX10:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
+  ; GFX10:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
+  ; GFX10:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
+  ; GFX10:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10:   [[COPY11:%[0-9]+]]:_(s32) = COPY [[COPY8]](s32)
+  ; GFX10:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[COPY9]](s32)
+  ; GFX10:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY11]](s32), [[COPY12]](s32)
+  ; GFX10:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[COPY10]](s32)
+  ; GFX10:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX10:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY13]](s32), [[DEF]](s32)
+  ; GFX10:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.cube), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>), $noreg, 0, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
+  ; GFX10:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
+  ; GFX10:   $vgpr0 = COPY [[UV]](s32)
+  ; GFX10:   $vgpr1 = COPY [[UV1]](s32)
+  ; GFX10:   $vgpr2 = COPY [[UV2]](s32)
+  ; GFX10:   $vgpr3 = COPY [[UV3]](s32)
+  ; GFX10:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.load.mip.cube.v4f32.i16(i32 15, i16 %s, i16 %t, i16 %u, i16 0, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps void @store_mip_1d(<8 x i32> inreg %rsrc, <4 x float> %vdata, i16 %s) {
+  ; GFX9-LABEL: name: store_mip_1d
+  ; GFX9: bb.1.main_body:
+  ; GFX9:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX9:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
+  ; GFX9:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
+  ; GFX9:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
+  ; GFX9:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
+  ; GFX9:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
+  ; GFX9:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
+  ; GFX9:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
+  ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
+  ; GFX9:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[COPY12]](s32)
+  ; GFX9:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY13]](s32), [[DEF]](s32)
+  ; GFX9:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), 0, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable store 16 into custom "TargetCustom8")
+  ; GFX9:   S_ENDPGM 0
+  ; GFX10-LABEL: name: store_mip_1d
+  ; GFX10: bb.1.main_body:
+  ; GFX10:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX10:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
+  ; GFX10:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
+  ; GFX10:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
+  ; GFX10:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
+  ; GFX10:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
+  ; GFX10:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
+  ; GFX10:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
+  ; GFX10:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
+  ; GFX10:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX10:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GFX10:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX10:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[COPY12]](s32)
+  ; GFX10:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX10:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY13]](s32), [[DEF]](s32)
+  ; GFX10:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), 0, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable store 16 into custom "TargetCustom8")
+  ; GFX10:   S_ENDPGM 0
+main_body:
+  call void @llvm.amdgcn.image.store.mip.1d.v4f32.i16(<4 x float> %vdata, i32 15, i16 %s, i16 0, <8 x i32> %rsrc, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @store_mip_2d(<8 x i32> inreg %rsrc, <4 x float> %vdata, i16 %s, i16 %t) {
+  ; GFX9-LABEL: name: store_mip_2d
+  ; GFX9: bb.1.main_body:
+  ; GFX9:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX9:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
+  ; GFX9:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
+  ; GFX9:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
+  ; GFX9:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
+  ; GFX9:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
+  ; GFX9:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
+  ; GFX9:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
+  ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
+  ; GFX9:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY12]](s32)
+  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
+  ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
+  ; GFX9:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.2d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, 0, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable store 16 into custom "TargetCustom8")
+  ; GFX9:   S_ENDPGM 0
+  ; GFX10-LABEL: name: store_mip_2d
+  ; GFX10: bb.1.main_body:
+  ; GFX10:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX10:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
+  ; GFX10:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
+  ; GFX10:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
+  ; GFX10:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
+  ; GFX10:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
+  ; GFX10:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
+  ; GFX10:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
+  ; GFX10:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
+  ; GFX10:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX10:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GFX10:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; GFX10:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX10:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY12]](s32)
+  ; GFX10:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
+  ; GFX10:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
+  ; GFX10:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.2d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, 0, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable store 16 into custom "TargetCustom8")
+  ; GFX10:   S_ENDPGM 0
+main_body:
+  call void @llvm.amdgcn.image.store.mip.2d.v4f32.i16(<4 x float> %vdata, i32 15, i16 %s, i16 %t, i16 0, <8 x i32> %rsrc, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @store_mip_3d(<8 x i32> inreg %rsrc, <4 x float> %vdata, i16 %s, i16 %t, i16 %u) {
+  ; GFX9-LABEL: name: store_mip_3d
+  ; GFX9: bb.1.main_body:
+  ; GFX9:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX9:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
+  ; GFX9:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
+  ; GFX9:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
+  ; GFX9:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
+  ; GFX9:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
+  ; GFX9:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
+  ; GFX9:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
+  ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
+  ; GFX9:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY12]](s32)
+  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
+  ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY15]](s32), [[COPY16]](s32)
+  ; GFX9:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY14]](s32)
+  ; GFX9:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY17]](s32), [[DEF]](s32)
+  ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
+  ; GFX9:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.3d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, 0, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable store 16 into custom "TargetCustom8")
+  ; GFX9:   S_ENDPGM 0
+  ; GFX10-LABEL: name: store_mip_3d
+  ; GFX10: bb.1.main_body:
+  ; GFX10:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX10:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
+  ; GFX10:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
+  ; GFX10:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
+  ; GFX10:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
+  ; GFX10:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
+  ; GFX10:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
+  ; GFX10:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
+  ; GFX10:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
+  ; GFX10:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX10:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GFX10:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; GFX10:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; GFX10:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX10:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY12]](s32)
+  ; GFX10:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
+  ; GFX10:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY15]](s32), [[COPY16]](s32)
+  ; GFX10:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY14]](s32)
+  ; GFX10:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX10:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY17]](s32), [[DEF]](s32)
+  ; GFX10:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.3d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>), $noreg, 0, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable store 16 into custom "TargetCustom8")
+  ; GFX10:   S_ENDPGM 0
+main_body:
+  call void @llvm.amdgcn.image.store.mip.3d.v4f32.i16(<4 x float> %vdata, i32 15, i16 %s, i16 %t, i16 %u, i16 0, <8 x i32> %rsrc, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @store_mip_1darray(<8 x i32> inreg %rsrc, <4 x float> %vdata, i16 %s, i16 %t) {
+  ; GFX9-LABEL: name: store_mip_1darray
+  ; GFX9: bb.1.main_body:
+  ; GFX9:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX9:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
+  ; GFX9:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
+  ; GFX9:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
+  ; GFX9:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
+  ; GFX9:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
+  ; GFX9:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
+  ; GFX9:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
+  ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
+  ; GFX9:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY12]](s32)
+  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
+  ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
+  ; GFX9:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.1darray), [[BUILD_VECTOR1]](<4 x s32>), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, 0, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable store 16 into custom "TargetCustom8")
+  ; GFX9:   S_ENDPGM 0
+  ; GFX10-LABEL: name: store_mip_1darray
+  ; GFX10: bb.1.main_body:
+  ; GFX10:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX10:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
+  ; GFX10:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
+  ; GFX10:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
+  ; GFX10:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
+  ; GFX10:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
+  ; GFX10:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
+  ; GFX10:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
+  ; GFX10:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
+  ; GFX10:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX10:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GFX10:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; GFX10:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX10:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY12]](s32)
+  ; GFX10:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
+  ; GFX10:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
+  ; GFX10:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.1darray), [[BUILD_VECTOR1]](<4 x s32>), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, 0, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable store 16 into custom "TargetCustom8")
+  ; GFX10:   S_ENDPGM 0
+main_body:
+  call void @llvm.amdgcn.image.store.mip.1darray.v4f32.i16(<4 x float> %vdata, i32 15, i16 %s, i16 %t, i16 0, <8 x i32> %rsrc, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @store_mip_2darray(<8 x i32> inreg %rsrc, <4 x float> %vdata, i16 %s, i16 %t, i16 %u) {
+  ; GFX9-LABEL: name: store_mip_2darray
+  ; GFX9: bb.1.main_body:
+  ; GFX9:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX9:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
+  ; GFX9:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
+  ; GFX9:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
+  ; GFX9:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
+  ; GFX9:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
+  ; GFX9:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
+  ; GFX9:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
+  ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
+  ; GFX9:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY12]](s32)
+  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
+  ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY15]](s32), [[COPY16]](s32)
+  ; GFX9:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY14]](s32)
+  ; GFX9:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY17]](s32), [[DEF]](s32)
+  ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
+  ; GFX9:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.2darray), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, 0, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable store 16 into custom "TargetCustom8")
+  ; GFX9:   S_ENDPGM 0
+  ; GFX10-LABEL: name: store_mip_2darray
+  ; GFX10: bb.1.main_body:
+  ; GFX10:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX10:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
+  ; GFX10:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
+  ; GFX10:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
+  ; GFX10:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
+  ; GFX10:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
+  ; GFX10:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
+  ; GFX10:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
+  ; GFX10:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
+  ; GFX10:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX10:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GFX10:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; GFX10:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; GFX10:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX10:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY12]](s32)
+  ; GFX10:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
+  ; GFX10:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY15]](s32), [[COPY16]](s32)
+  ; GFX10:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY14]](s32)
+  ; GFX10:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX10:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY17]](s32), [[DEF]](s32)
+  ; GFX10:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.2darray), [[BUILD_VECTOR1]](<4 x s32>), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>), $noreg, 0, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable store 16 into custom "TargetCustom8")
+  ; GFX10:   S_ENDPGM 0
+main_body:
+  call void @llvm.amdgcn.image.store.mip.2darray.v4f32.i16(<4 x float> %vdata, i32 15, i16 %s, i16 %t, i16 %u, i16 0, <8 x i32> %rsrc, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @store_mip_cube(<8 x i32> inreg %rsrc, <4 x float> %vdata, i16 %s, i16 %t, i16 %u) {
+  ; GFX9-LABEL: name: store_mip_cube
+  ; GFX9: bb.1.main_body:
+  ; GFX9:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX9:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
+  ; GFX9:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
+  ; GFX9:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
+  ; GFX9:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
+  ; GFX9:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
+  ; GFX9:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
+  ; GFX9:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
+  ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
+  ; GFX9:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY12]](s32)
+  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
+  ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY15]](s32), [[COPY16]](s32)
+  ; GFX9:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY14]](s32)
+  ; GFX9:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY17]](s32), [[DEF]](s32)
+  ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
+  ; GFX9:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.cube), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, 0, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable store 16 into custom "TargetCustom8")
+  ; GFX9:   S_ENDPGM 0
+  ; GFX10-LABEL: name: store_mip_cube
+  ; GFX10: bb.1.main_body:
+  ; GFX10:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX10:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
+  ; GFX10:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
+  ; GFX10:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
+  ; GFX10:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
+  ; GFX10:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
+  ; GFX10:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
+  ; GFX10:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
+  ; GFX10:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
+  ; GFX10:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX10:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GFX10:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; GFX10:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; GFX10:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; GFX10:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY12]](s32)
+  ; GFX10:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
+  ; GFX10:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY15]](s32), [[COPY16]](s32)
+  ; GFX10:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY14]](s32)
+  ; GFX10:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX10:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY17]](s32), [[DEF]](s32)
+  ; GFX10:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.cube), [[BUILD_VECTOR1]](<4 x s32>), 15, [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>), $noreg, 0, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable store 16 into custom "TargetCustom8")
+  ; GFX10:   S_ENDPGM 0
+main_body:
+  call void @llvm.amdgcn.image.store.mip.cube.v4f32.i16(<4 x float> %vdata, i32 15, i16 %s, i16 %t, i16 %u, i16 0, <8 x i32> %rsrc, i32 0, i32 0)
+  ret void
+}
+
+declare <4 x float> @llvm.amdgcn.image.load.mip.1d.v4f32.i16(i32 immarg, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i16(i32 immarg, i16, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.load.mip.3d.v4f32.i16(i32 immarg, i16, i16, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.load.mip.1darray.v4f32.i16(i32 immarg, i16, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.load.mip.2darray.v4f32.i16(i32 immarg, i16, i16, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.load.mip.cube.v4f32.i16(i32 immarg, i16, i16, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare void @llvm.amdgcn.image.store.mip.1d.v4f32.i16(<4 x float>, i32 immarg, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #1
+declare void @llvm.amdgcn.image.store.mip.2d.v4f32.i16(<4 x float>, i32 immarg, i16, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #1
+declare void @llvm.amdgcn.image.store.mip.3d.v4f32.i16(<4 x float>, i32 immarg, i16, i16, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #1
+declare void @llvm.amdgcn.image.store.mip.cube.v4f32.i16(<4 x float>, i32 immarg, i16, i16, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #1
+declare void @llvm.amdgcn.image.store.mip.1darray.v4f32.i16(<4 x float>, i32 immarg, i16, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #1
+declare void @llvm.amdgcn.image.store.mip.2darray.v4f32.i16(<4 x float>, i32 immarg, i16, i16, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #1
+
+attributes #0 = { nounwind readonly }
+attributes #1 = { nounwind writeonly }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/image_ls_mipmap_zero.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/image_ls_mipmap_zero.ll
new file mode 100644
index 0000000000000..afcc55a6c0d59
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/image_ls_mipmap_zero.ll
@@ -0,0 +1,415 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -o - %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -o - %s | FileCheck -check-prefix=GFX10 %s
+
+define amdgpu_ps <4 x float> @load_mip_1d(<8 x i32> inreg %rsrc, i32 %s) {
+; GFX9-LABEL: load_mip_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf unorm
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_mip_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.load.mip.1d.v4f32.i32(i32 15, i32 %s, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @load_mip_2d(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+; GFX9-LABEL: load_mip_2d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf unorm
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_mip_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32(i32 15, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @load_mip_3d(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %u) {
+; GFX9-LABEL: load_mip_3d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_load v[0:3], v[0:2], s[0:7] dmask:0xf unorm
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_mip_3d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:3], v[0:2], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.load.mip.3d.v4f32.i32(i32 15, i32 %s, i32 %t, i32 %u, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @load_mip_1darray(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+; GFX9-LABEL: load_mip_1darray:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf unorm da
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_mip_1darray:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY unorm
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.load.mip.1darray.v4f32.i32(i32 15, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @load_mip_2darray(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %u) {
+; GFX9-LABEL: load_mip_2darray:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_load v[0:3], v[0:2], s[0:7] dmask:0xf unorm da
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_mip_2darray:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:3], v[0:2], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY unorm
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.load.mip.2darray.v4f32.i32(i32 15, i32 %s, i32 %t, i32 %u, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @load_mip_cube(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %u) {
+; GFX9-LABEL: load_mip_cube:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_load v[0:3], v[0:2], s[0:7] dmask:0xf unorm da
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_mip_cube:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:3], v[0:2], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_CUBE unorm
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.load.mip.cube.v4f32.i32(i32 15, i32 %s, i32 %t, i32 %u, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps void @store_mip_1d(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s) {
+; GFX9-LABEL: store_mip_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_store v[0:3], v4, s[0:7] dmask:0xf unorm
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: store_mip_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_store v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm
+; GFX10-NEXT:    s_endpgm
+main_body:
+  call void @llvm.amdgcn.image.store.mip.1d.v4f32.i32(<4 x float> %vdata, i32 15, i32 %s, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @store_mip_2d(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s, i32 %t) {
+; GFX9-LABEL: store_mip_2d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_store v[0:3], v[4:5], s[0:7] dmask:0xf unorm
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: store_mip_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_store v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm
+; GFX10-NEXT:    s_endpgm
+main_body:
+  call void @llvm.amdgcn.image.store.mip.2d.v4f32.i32(<4 x float> %vdata, i32 15, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @store_mip_3d(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s, i32 %t, i32 %u) {
+; GFX9-LABEL: store_mip_3d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_store v[0:3], v[4:6], s[0:7] dmask:0xf unorm
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: store_mip_3d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_store v[0:3], v[4:6], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm
+; GFX10-NEXT:    s_endpgm
+main_body:
+  call void @llvm.amdgcn.image.store.mip.3d.v4f32.i32(<4 x float> %vdata, i32 15, i32 %s, i32 %t, i32 %u, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @store_mip_1darray(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s, i32 %t) {
+; GFX9-LABEL: store_mip_1darray:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_store v[0:3], v[4:5], s[0:7] dmask:0xf unorm da
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: store_mip_1darray:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_store v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY unorm
+; GFX10-NEXT:    s_endpgm
+main_body:
+  call void @llvm.amdgcn.image.store.mip.1darray.v4f32.i32(<4 x float> %vdata, i32 15, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @store_mip_2darray(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s, i32 %t, i32 %u) {
+; GFX9-LABEL: store_mip_2darray:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_store v[0:3], v[4:6], s[0:7] dmask:0xf unorm da
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: store_mip_2darray:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_store v[0:3], v[4:6], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY unorm
+; GFX10-NEXT:    s_endpgm
+main_body:
+  call void @llvm.amdgcn.image.store.mip.2darray.v4f32.i32(<4 x float> %vdata, i32 15, i32 %s, i32 %t, i32 %u, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @store_mip_cube(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s, i32 %t, i32 %u) {
+; GFX9-LABEL: store_mip_cube:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_store v[0:3], v[4:6], s[0:7] dmask:0xf unorm da
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: store_mip_cube:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_store v[0:3], v[4:6], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_CUBE unorm
+; GFX10-NEXT:    s_endpgm
+main_body:
+  call void @llvm.amdgcn.image.store.mip.cube.v4f32.i32(<4 x float> %vdata, i32 15, i32 %s, i32 %t, i32 %u, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  ret void
+}
+
+declare <4 x float> @llvm.amdgcn.image.load.mip.1d.v4f32.i32(i32 immarg, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32(i32 immarg, i32, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.load.mip.3d.v4f32.i32(i32 immarg, i32, i32, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.load.mip.1darray.v4f32.i32(i32 immarg, i32, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.load.mip.2darray.v4f32.i32(i32 immarg, i32, i32, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.load.mip.cube.v4f32.i32(i32 immarg, i32, i32, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare void @llvm.amdgcn.image.store.mip.1d.v4f32.i32(<4 x float>, i32 immarg, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #1
+declare void @llvm.amdgcn.image.store.mip.2d.v4f32.i32(<4 x float>, i32 immarg, i32, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #1
+declare void @llvm.amdgcn.image.store.mip.3d.v4f32.i32(<4 x float>, i32 immarg, i32, i32, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #1
+declare void @llvm.amdgcn.image.store.mip.cube.v4f32.i32(<4 x float>, i32 immarg, i32, i32, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #1
+declare void @llvm.amdgcn.image.store.mip.1darray.v4f32.i32(<4 x float>, i32 immarg, i32, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #1
+declare void @llvm.amdgcn.image.store.mip.2darray.v4f32.i32(<4 x float>, i32 immarg, i32, i32, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #1
+
+attributes #0 = { nounwind readonly }
+attributes #1 = { nounwind writeonly }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.fmad.ftz.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.fmad.ftz.mir
index 05011091a6691..b9f88557678a9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.fmad.ftz.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.fmad.ftz.mir
@@ -1,8 +1,8 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=amdgcn -mcpu=tahiti -mattr=+fp32-denormals -run-pass=instruction-select -verify-machineinstrs %s -o -  | FileCheck -check-prefix=GCN %s
-# RUN: llc -march=amdgcn -mcpu=tahiti -mattr=-fp32-denormals -run-pass=instruction-select -verify-machineinstrs %s -o -  | FileCheck -check-prefix=GCN %s
-# RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=+fp32-denormals -run-pass=instruction-select -verify-machineinstrs %s -o -  | FileCheck -check-prefix=GCN %s
-# RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-fp32-denormals -run-pass=instruction-select -verify-machineinstrs %s -o -  | FileCheck -check-prefix=GCN %s
+# RUN: llc -march=amdgcn -mcpu=tahiti -denormal-fp-math-f32=ieee -run-pass=instruction-select -verify-machineinstrs %s -o -  | FileCheck -check-prefix=GCN %s
+# RUN: llc -march=amdgcn -mcpu=tahiti  -denormal-fp-math-f32=preserve-sign -run-pass=instruction-select -verify-machineinstrs %s -o -  | FileCheck -check-prefix=GCN %s
+# RUN: llc -march=amdgcn -mcpu=gfx900 -denormal-fp-math-f32=ieee -run-pass=instruction-select -verify-machineinstrs %s -o -  | FileCheck -check-prefix=GCN %s
+# RUN: llc -march=amdgcn -mcpu=gfx900 -denormal-fp-math-f32=preserve-sign -run-pass=instruction-select -verify-machineinstrs %s -o -  | FileCheck -check-prefix=GCN %s
 
 ---
 name: fmad_ftz_s32_vvvv
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fshr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fshr.mir
new file mode 100644
index 0000000000000..f4c46a99a4bb7
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fshr.mir
@@ -0,0 +1,29 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=tahiti -run-pass=instruction-select -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=instruction-select -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=instruction-select -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -march=amdgcn -mcpu=gfx1010 -run-pass=instruction-select -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
+
+---
+
+name:            fshr_s32
+legalized:       true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+
+    ; GCN-LABEL: name: fshr_s32
+    ; GCN: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GCN: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GCN: [[V_ALIGNBIT_B32_:%[0-9]+]]:vgpr_32 = V_ALIGNBIT_B32 [[COPY]], [[COPY1]], [[COPY2]], implicit $exec
+    ; GCN: S_ENDPGM 0, implicit [[V_ALIGNBIT_B32_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s32) = COPY $vgpr2
+    %3:vgpr(s32) = G_FSHR %0, %1, %2
+    S_ENDPGM 0, implicit %3
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-insert-vector-elt.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-insert-vector-elt.mir
index ac3d623fb3d04..a64b2a30c65cb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-insert-vector-elt.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-insert-vector-elt.mir
@@ -16,14 +16,14 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[S_INDIRECT_REG_WRITE_B32_V2_:%[0-9]+]]:sreg_64 = S_INDIRECT_REG_WRITE_B32_V2 [[COPY]], [[COPY1]], 1, implicit $m0
+    ; MOVREL: [[S_INDIRECT_REG_WRITE_B32_V2_:%[0-9]+]]:sreg_64 = S_INDIRECT_REG_WRITE_B32_V2 [[COPY]], [[COPY1]], 3, implicit $m0
     ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V2_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_s_s32_v2s32
     ; GPRIDX: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
     ; GPRIDX: $m0 = COPY [[COPY2]]
-    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B32_V2_:%[0-9]+]]:sreg_64 = S_INDIRECT_REG_WRITE_B32_V2 [[COPY]], [[COPY1]], 1, implicit $m0
+    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B32_V2_:%[0-9]+]]:sreg_64 = S_INDIRECT_REG_WRITE_B32_V2 [[COPY]], [[COPY1]], 3, implicit $m0
     ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V2_]]
     %0:sgpr(<2 x s32>) = COPY $sgpr0_sgpr1
     %1:sgpr(s32) = COPY $sgpr2
@@ -46,14 +46,14 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[S_INDIRECT_REG_WRITE_B32_V3_:%[0-9]+]]:sreg_96 = S_INDIRECT_REG_WRITE_B32_V3 [[COPY]], [[COPY1]], 1, implicit $m0
+    ; MOVREL: [[S_INDIRECT_REG_WRITE_B32_V3_:%[0-9]+]]:sreg_96 = S_INDIRECT_REG_WRITE_B32_V3 [[COPY]], [[COPY1]], 3, implicit $m0
     ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V3_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_s_s32_v3s32
     ; GPRIDX: [[COPY:%[0-9]+]]:sreg_96 = COPY $sgpr0_sgpr1_sgpr2
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
     ; GPRIDX: $m0 = COPY [[COPY2]]
-    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B32_V3_:%[0-9]+]]:sreg_96 = S_INDIRECT_REG_WRITE_B32_V3 [[COPY]], [[COPY1]], 1, implicit $m0
+    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B32_V3_:%[0-9]+]]:sreg_96 = S_INDIRECT_REG_WRITE_B32_V3 [[COPY]], [[COPY1]], 3, implicit $m0
     ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V3_]]
     %0:sgpr(<3 x s32>) = COPY $sgpr0_sgpr1_sgpr2
     %1:sgpr(s32) = COPY $sgpr3
@@ -76,14 +76,14 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[S_INDIRECT_REG_WRITE_B32_V4_:%[0-9]+]]:sgpr_128 = S_INDIRECT_REG_WRITE_B32_V4 [[COPY]], [[COPY1]], 1, implicit $m0
+    ; MOVREL: [[S_INDIRECT_REG_WRITE_B32_V4_:%[0-9]+]]:sgpr_128 = S_INDIRECT_REG_WRITE_B32_V4 [[COPY]], [[COPY1]], 3, implicit $m0
     ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V4_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_s_s32_v4s32
     ; GPRIDX: [[COPY:%[0-9]+]]:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
     ; GPRIDX: $m0 = COPY [[COPY2]]
-    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B32_V4_:%[0-9]+]]:sgpr_128 = S_INDIRECT_REG_WRITE_B32_V4 [[COPY]], [[COPY1]], 1, implicit $m0
+    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B32_V4_:%[0-9]+]]:sgpr_128 = S_INDIRECT_REG_WRITE_B32_V4 [[COPY]], [[COPY1]], 3, implicit $m0
     ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V4_]]
     %0:sgpr(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     %1:sgpr(s32) = COPY $sgpr3
@@ -106,14 +106,14 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr5
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr6
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[S_INDIRECT_REG_WRITE_B32_V5_:%[0-9]+]]:sreg_160 = S_INDIRECT_REG_WRITE_B32_V5 [[COPY]], [[COPY1]], 1, implicit $m0
+    ; MOVREL: [[S_INDIRECT_REG_WRITE_B32_V5_:%[0-9]+]]:sreg_160 = S_INDIRECT_REG_WRITE_B32_V5 [[COPY]], [[COPY1]], 3, implicit $m0
     ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V5_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_s_s32_v5s32
     ; GPRIDX: [[COPY:%[0-9]+]]:sreg_160 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr5
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr6
     ; GPRIDX: $m0 = COPY [[COPY2]]
-    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B32_V5_:%[0-9]+]]:sreg_160 = S_INDIRECT_REG_WRITE_B32_V5 [[COPY]], [[COPY1]], 1, implicit $m0
+    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B32_V5_:%[0-9]+]]:sreg_160 = S_INDIRECT_REG_WRITE_B32_V5 [[COPY]], [[COPY1]], 3, implicit $m0
     ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V5_]]
     %0:sgpr(<5 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4
     %1:sgpr(s32) = COPY $sgpr5
@@ -136,14 +136,14 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr8
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr9
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[S_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:sreg_256 = S_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 1, implicit $m0
+    ; MOVREL: [[S_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:sreg_256 = S_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 3, implicit $m0
     ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V8_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_s_s32_v8s32
     ; GPRIDX: [[COPY:%[0-9]+]]:sreg_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr8
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr9
     ; GPRIDX: $m0 = COPY [[COPY2]]
-    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:sreg_256 = S_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 1, implicit $m0
+    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:sreg_256 = S_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 3, implicit $m0
     ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V8_]]
     %0:sgpr(<8 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     %1:sgpr(s32) = COPY $sgpr8
@@ -166,14 +166,14 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr16
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr17
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[S_INDIRECT_REG_WRITE_B32_V16_:%[0-9]+]]:sreg_512 = S_INDIRECT_REG_WRITE_B32_V16 [[COPY]], [[COPY1]], 1, implicit $m0
+    ; MOVREL: [[S_INDIRECT_REG_WRITE_B32_V16_:%[0-9]+]]:sreg_512 = S_INDIRECT_REG_WRITE_B32_V16 [[COPY]], [[COPY1]], 3, implicit $m0
     ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V16_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_s_s32_v16s32
     ; GPRIDX: [[COPY:%[0-9]+]]:sreg_512 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr16
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr17
     ; GPRIDX: $m0 = COPY [[COPY2]]
-    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B32_V16_:%[0-9]+]]:sreg_512 = S_INDIRECT_REG_WRITE_B32_V16 [[COPY]], [[COPY1]], 1, implicit $m0
+    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B32_V16_:%[0-9]+]]:sreg_512 = S_INDIRECT_REG_WRITE_B32_V16 [[COPY]], [[COPY1]], 3, implicit $m0
     ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V16_]]
     %0:sgpr(<16 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
     %1:sgpr(s32) = COPY $sgpr16
@@ -196,14 +196,14 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr40
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr41
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[S_INDIRECT_REG_WRITE_B32_V32_:%[0-9]+]]:sreg_1024 = S_INDIRECT_REG_WRITE_B32_V32 [[COPY]], [[COPY1]], 1, implicit $m0
+    ; MOVREL: [[S_INDIRECT_REG_WRITE_B32_V32_:%[0-9]+]]:sreg_1024 = S_INDIRECT_REG_WRITE_B32_V32 [[COPY]], [[COPY1]], 3, implicit $m0
     ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V32_]]
     ; GPRIDX-LABEL: name: extract_vector_elt_s_s32_v32s32
     ; GPRIDX: [[COPY:%[0-9]+]]:sreg_1024 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr40
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr41
     ; GPRIDX: $m0 = COPY [[COPY2]]
-    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B32_V32_:%[0-9]+]]:sreg_1024 = S_INDIRECT_REG_WRITE_B32_V32 [[COPY]], [[COPY1]], 1, implicit $m0
+    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B32_V32_:%[0-9]+]]:sreg_1024 = S_INDIRECT_REG_WRITE_B32_V32 [[COPY]], [[COPY1]], 3, implicit $m0
     ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V32_]]
     %0:sgpr(<32 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31
     %1:sgpr(s32) = COPY $sgpr40
@@ -226,14 +226,14 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr4_sgpr5
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr6
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[S_INDIRECT_REG_WRITE_B64_V2_:%[0-9]+]]:sgpr_128 = S_INDIRECT_REG_WRITE_B64_V2 [[COPY]], [[COPY1]], 35, implicit $m0
+    ; MOVREL: [[S_INDIRECT_REG_WRITE_B64_V2_:%[0-9]+]]:sgpr_128 = S_INDIRECT_REG_WRITE_B64_V2 [[COPY]], [[COPY1]], 4, implicit $m0
     ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B64_V2_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_s_s64_v2s64
     ; GPRIDX: [[COPY:%[0-9]+]]:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr4_sgpr5
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr6
     ; GPRIDX: $m0 = COPY [[COPY2]]
-    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B64_V2_:%[0-9]+]]:sgpr_128 = S_INDIRECT_REG_WRITE_B64_V2 [[COPY]], [[COPY1]], 35, implicit $m0
+    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B64_V2_:%[0-9]+]]:sgpr_128 = S_INDIRECT_REG_WRITE_B64_V2 [[COPY]], [[COPY1]], 4, implicit $m0
     ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B64_V2_]]
     %0:sgpr(<2 x s64>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     %1:sgpr(s64) = COPY $sgpr4_sgpr5
@@ -256,14 +256,14 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr8_sgpr9
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr10
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[S_INDIRECT_REG_WRITE_B64_V4_:%[0-9]+]]:sreg_256 = S_INDIRECT_REG_WRITE_B64_V4 [[COPY]], [[COPY1]], 35, implicit $m0
+    ; MOVREL: [[S_INDIRECT_REG_WRITE_B64_V4_:%[0-9]+]]:sreg_256 = S_INDIRECT_REG_WRITE_B64_V4 [[COPY]], [[COPY1]], 4, implicit $m0
     ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B64_V4_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_s_s64_v4s64
     ; GPRIDX: [[COPY:%[0-9]+]]:sreg_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr8_sgpr9
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr10
     ; GPRIDX: $m0 = COPY [[COPY2]]
-    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B64_V4_:%[0-9]+]]:sreg_256 = S_INDIRECT_REG_WRITE_B64_V4 [[COPY]], [[COPY1]], 35, implicit $m0
+    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B64_V4_:%[0-9]+]]:sreg_256 = S_INDIRECT_REG_WRITE_B64_V4 [[COPY]], [[COPY1]], 4, implicit $m0
     ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B64_V4_]]
     %0:sgpr(<4 x s64>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     %1:sgpr(s64) = COPY $sgpr8_sgpr9
@@ -286,14 +286,14 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr16_sgpr17
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr18
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[S_INDIRECT_REG_WRITE_B64_V8_:%[0-9]+]]:sreg_512 = S_INDIRECT_REG_WRITE_B64_V8 [[COPY]], [[COPY1]], 35, implicit $m0
+    ; MOVREL: [[S_INDIRECT_REG_WRITE_B64_V8_:%[0-9]+]]:sreg_512 = S_INDIRECT_REG_WRITE_B64_V8 [[COPY]], [[COPY1]], 4, implicit $m0
     ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B64_V8_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_s_s64_v8s64
     ; GPRIDX: [[COPY:%[0-9]+]]:sreg_512 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr16_sgpr17
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr18
     ; GPRIDX: $m0 = COPY [[COPY2]]
-    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B64_V8_:%[0-9]+]]:sreg_512 = S_INDIRECT_REG_WRITE_B64_V8 [[COPY]], [[COPY1]], 35, implicit $m0
+    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B64_V8_:%[0-9]+]]:sreg_512 = S_INDIRECT_REG_WRITE_B64_V8 [[COPY]], [[COPY1]], 4, implicit $m0
     ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B64_V8_]]
     %0:sgpr(<8 x s64>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
     %1:sgpr(s64) = COPY $sgpr16_sgpr17
@@ -316,14 +316,14 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr40_sgpr41
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr42
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[S_INDIRECT_REG_WRITE_B64_V16_:%[0-9]+]]:sreg_1024 = S_INDIRECT_REG_WRITE_B64_V16 [[COPY]], [[COPY1]], 35, implicit $m0
+    ; MOVREL: [[S_INDIRECT_REG_WRITE_B64_V16_:%[0-9]+]]:sreg_1024 = S_INDIRECT_REG_WRITE_B64_V16 [[COPY]], [[COPY1]], 4, implicit $m0
     ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B64_V16_]]
     ; GPRIDX-LABEL: name: extract_vector_elt_s_s64_v16s64
     ; GPRIDX: [[COPY:%[0-9]+]]:sreg_1024 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr40_sgpr41
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr42
     ; GPRIDX: $m0 = COPY [[COPY2]]
-    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B64_V16_:%[0-9]+]]:sreg_1024 = S_INDIRECT_REG_WRITE_B64_V16 [[COPY]], [[COPY1]], 35, implicit $m0
+    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B64_V16_:%[0-9]+]]:sreg_1024 = S_INDIRECT_REG_WRITE_B64_V16 [[COPY]], [[COPY1]], 4, implicit $m0
     ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B64_V16_]]
     %0:sgpr(<16 x s64>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31
     %1:sgpr(s64) = COPY $sgpr40_sgpr41
@@ -346,14 +346,14 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[V_INDIRECT_REG_WRITE_B32_V2_:%[0-9]+]]:vreg_64 = V_INDIRECT_REG_WRITE_B32_V2 [[COPY]], [[COPY1]], 1, implicit $m0, implicit $exec
+    ; MOVREL: [[V_INDIRECT_REG_WRITE_B32_V2_:%[0-9]+]]:vreg_64 = V_INDIRECT_REG_WRITE_B32_V2 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
     ; MOVREL: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V2_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_vvs_s32_v2s32
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GPRIDX: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
     ; GPRIDX: S_SET_GPR_IDX_ON [[COPY2]], 8, implicit-def $m0, implicit $m0
-    ; GPRIDX: [[V_INDIRECT_REG_WRITE_B32_V2_:%[0-9]+]]:vreg_64 = V_INDIRECT_REG_WRITE_B32_V2 [[COPY]], [[COPY1]], 1, implicit $m0, implicit $exec
+    ; GPRIDX: [[V_INDIRECT_REG_WRITE_B32_V2_:%[0-9]+]]:vreg_64 = V_INDIRECT_REG_WRITE_B32_V2 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
     ; GPRIDX: S_SET_GPR_IDX_OFF
     ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V2_]]
     %0:vgpr(<2 x s32>) = COPY $vgpr0_vgpr1
@@ -377,14 +377,14 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr3
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[V_INDIRECT_REG_WRITE_B32_V3_:%[0-9]+]]:vreg_96 = V_INDIRECT_REG_WRITE_B32_V3 [[COPY]], [[COPY1]], 1, implicit $m0, implicit $exec
+    ; MOVREL: [[V_INDIRECT_REG_WRITE_B32_V3_:%[0-9]+]]:vreg_96 = V_INDIRECT_REG_WRITE_B32_V3 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
     ; MOVREL: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V3_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_vvs_s32_v3s32
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_96 = COPY $vgpr0_vgpr1_vgpr2
     ; GPRIDX: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr3
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
     ; GPRIDX: S_SET_GPR_IDX_ON [[COPY2]], 8, implicit-def $m0, implicit $m0
-    ; GPRIDX: [[V_INDIRECT_REG_WRITE_B32_V3_:%[0-9]+]]:vreg_96 = V_INDIRECT_REG_WRITE_B32_V3 [[COPY]], [[COPY1]], 1, implicit $m0, implicit $exec
+    ; GPRIDX: [[V_INDIRECT_REG_WRITE_B32_V3_:%[0-9]+]]:vreg_96 = V_INDIRECT_REG_WRITE_B32_V3 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
     ; GPRIDX: S_SET_GPR_IDX_OFF
     ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V3_]]
     %0:vgpr(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
@@ -408,14 +408,14 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr3
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[V_INDIRECT_REG_WRITE_B32_V4_:%[0-9]+]]:vreg_128 = V_INDIRECT_REG_WRITE_B32_V4 [[COPY]], [[COPY1]], 1, implicit $m0, implicit $exec
+    ; MOVREL: [[V_INDIRECT_REG_WRITE_B32_V4_:%[0-9]+]]:vreg_128 = V_INDIRECT_REG_WRITE_B32_V4 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
     ; MOVREL: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V4_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_vvs_s32_v4s32
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GPRIDX: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr3
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
     ; GPRIDX: S_SET_GPR_IDX_ON [[COPY2]], 8, implicit-def $m0, implicit $m0
-    ; GPRIDX: [[V_INDIRECT_REG_WRITE_B32_V4_:%[0-9]+]]:vreg_128 = V_INDIRECT_REG_WRITE_B32_V4 [[COPY]], [[COPY1]], 1, implicit $m0, implicit $exec
+    ; GPRIDX: [[V_INDIRECT_REG_WRITE_B32_V4_:%[0-9]+]]:vreg_128 = V_INDIRECT_REG_WRITE_B32_V4 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
     ; GPRIDX: S_SET_GPR_IDX_OFF
     ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V4_]]
     %0:vgpr(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
@@ -439,14 +439,14 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr6
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[V_INDIRECT_REG_WRITE_B32_V5_:%[0-9]+]]:vreg_160 = V_INDIRECT_REG_WRITE_B32_V5 [[COPY]], [[COPY1]], 1, implicit $m0, implicit $exec
+    ; MOVREL: [[V_INDIRECT_REG_WRITE_B32_V5_:%[0-9]+]]:vreg_160 = V_INDIRECT_REG_WRITE_B32_V5 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
     ; MOVREL: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V5_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_vvs_s32_v5s32
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_160 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
     ; GPRIDX: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr6
     ; GPRIDX: S_SET_GPR_IDX_ON [[COPY2]], 8, implicit-def $m0, implicit $m0
-    ; GPRIDX: [[V_INDIRECT_REG_WRITE_B32_V5_:%[0-9]+]]:vreg_160 = V_INDIRECT_REG_WRITE_B32_V5 [[COPY]], [[COPY1]], 1, implicit $m0, implicit $exec
+    ; GPRIDX: [[V_INDIRECT_REG_WRITE_B32_V5_:%[0-9]+]]:vreg_160 = V_INDIRECT_REG_WRITE_B32_V5 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
     ; GPRIDX: S_SET_GPR_IDX_OFF
     ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V5_]]
     %0:vgpr(<5 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
@@ -470,14 +470,14 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr9
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[V_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:vreg_256 = V_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 1, implicit $m0, implicit $exec
+    ; MOVREL: [[V_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:vreg_256 = V_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
     ; MOVREL: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V8_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_vvs_s32_v8s32
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_256 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     ; GPRIDX: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr9
     ; GPRIDX: S_SET_GPR_IDX_ON [[COPY2]], 8, implicit-def $m0, implicit $m0
-    ; GPRIDX: [[V_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:vreg_256 = V_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 1, implicit $m0, implicit $exec
+    ; GPRIDX: [[V_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:vreg_256 = V_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
     ; GPRIDX: S_SET_GPR_IDX_OFF
     ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V8_]]
     %0:vgpr(<8 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
@@ -501,14 +501,14 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr9
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[V_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:vreg_256 = V_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 2, implicit $m0, implicit $exec
+    ; MOVREL: [[V_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:vreg_256 = V_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 10, implicit $m0, implicit $exec
     ; MOVREL: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V8_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_vvs_s32_v8s32_add_1
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_256 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     ; GPRIDX: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr9
     ; GPRIDX: S_SET_GPR_IDX_ON [[COPY2]], 8, implicit-def $m0, implicit $m0
-    ; GPRIDX: [[V_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:vreg_256 = V_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 2, implicit $m0, implicit $exec
+    ; GPRIDX: [[V_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:vreg_256 = V_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 10, implicit $m0, implicit $exec
     ; GPRIDX: S_SET_GPR_IDX_OFF
     ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V8_]]
     %0:vgpr(<8 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
@@ -536,7 +536,7 @@ body: |
     ; MOVREL: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 8
     ; MOVREL: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY2]], [[S_MOV_B32_]], implicit-def $scc
     ; MOVREL: $m0 = COPY [[S_ADD_I32_]]
-    ; MOVREL: [[V_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:vreg_256 = V_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 1, implicit $m0, implicit $exec
+    ; MOVREL: [[V_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:vreg_256 = V_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
     ; MOVREL: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V8_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_vvs_s32_v8s32_add_8
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_256 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
@@ -545,7 +545,7 @@ body: |
     ; GPRIDX: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 8
     ; GPRIDX: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY2]], [[S_MOV_B32_]], implicit-def $scc
     ; GPRIDX: S_SET_GPR_IDX_ON [[S_ADD_I32_]], 8, implicit-def $m0, implicit $m0
-    ; GPRIDX: [[V_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:vreg_256 = V_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 1, implicit $m0, implicit $exec
+    ; GPRIDX: [[V_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:vreg_256 = V_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
     ; GPRIDX: S_SET_GPR_IDX_OFF
     ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V8_]]
     %0:vgpr(<8 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
@@ -571,14 +571,14 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr8
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr9
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[S_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:sreg_256 = S_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 2, implicit $m0
+    ; MOVREL: [[S_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:sreg_256 = S_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 10, implicit $m0
     ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V8_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_s_s32_v8s32_add_1
     ; GPRIDX: [[COPY:%[0-9]+]]:sreg_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr8
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr9
     ; GPRIDX: $m0 = COPY [[COPY2]]
-    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:sreg_256 = S_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 2, implicit $m0
+    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:sreg_256 = S_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 10, implicit $m0
     ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V8_]]
     %0:sgpr(<8 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     %1:sgpr(s32) = COPY $sgpr8
@@ -605,7 +605,7 @@ body: |
     ; MOVREL: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 8
     ; MOVREL: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY2]], [[S_MOV_B32_]], implicit-def $scc
     ; MOVREL: $m0 = COPY [[S_ADD_I32_]]
-    ; MOVREL: [[S_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:sreg_256 = S_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 1, implicit $m0
+    ; MOVREL: [[S_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:sreg_256 = S_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 3, implicit $m0
     ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V8_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_s_s32_v8s32_add_8
     ; GPRIDX: [[COPY:%[0-9]+]]:sreg_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
@@ -614,7 +614,7 @@ body: |
     ; GPRIDX: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 8
     ; GPRIDX: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY2]], [[S_MOV_B32_]], implicit-def $scc
     ; GPRIDX: $m0 = COPY [[S_ADD_I32_]]
-    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:sreg_256 = S_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 1, implicit $m0
+    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:sreg_256 = S_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 3, implicit $m0
     ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V8_]]
     %0:sgpr(<8 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     %1:sgpr(s32) = COPY $sgpr8
@@ -641,14 +641,14 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr4
     ; MOVREL: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
     ; MOVREL: $m0 = COPY [[S_MOV_B32_]]
-    ; MOVREL: [[S_INDIRECT_REG_WRITE_B32_V4_:%[0-9]+]]:sgpr_128 = S_INDIRECT_REG_WRITE_B32_V4 [[COPY]], [[COPY1]], 1, implicit $m0
+    ; MOVREL: [[S_INDIRECT_REG_WRITE_B32_V4_:%[0-9]+]]:sgpr_128 = S_INDIRECT_REG_WRITE_B32_V4 [[COPY]], [[COPY1]], 3, implicit $m0
     ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V4_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_s_s32_v4s32_const_idx
     ; GPRIDX: [[COPY:%[0-9]+]]:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr4
     ; GPRIDX: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
     ; GPRIDX: $m0 = COPY [[S_MOV_B32_]]
-    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B32_V4_:%[0-9]+]]:sgpr_128 = S_INDIRECT_REG_WRITE_B32_V4 [[COPY]], [[COPY1]], 1, implicit $m0
+    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B32_V4_:%[0-9]+]]:sgpr_128 = S_INDIRECT_REG_WRITE_B32_V4 [[COPY]], [[COPY1]], 3, implicit $m0
     ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V4_]]
     %0:sgpr(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     %1:sgpr(s32) = COPY $sgpr4
@@ -671,14 +671,14 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr4
     ; MOVREL: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
     ; MOVREL: $m0 = COPY [[S_MOV_B32_]]
-    ; MOVREL: [[V_INDIRECT_REG_WRITE_B32_V4_:%[0-9]+]]:vreg_128 = V_INDIRECT_REG_WRITE_B32_V4 [[COPY]], [[COPY1]], 1, implicit $m0, implicit $exec
+    ; MOVREL: [[V_INDIRECT_REG_WRITE_B32_V4_:%[0-9]+]]:vreg_128 = V_INDIRECT_REG_WRITE_B32_V4 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
     ; MOVREL: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V4_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_v_s32_v4s32_const_idx
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr4
     ; GPRIDX: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
     ; GPRIDX: S_SET_GPR_IDX_ON [[S_MOV_B32_]], 8, implicit-def $m0, implicit $m0
-    ; GPRIDX: [[V_INDIRECT_REG_WRITE_B32_V4_:%[0-9]+]]:vreg_128 = V_INDIRECT_REG_WRITE_B32_V4 [[COPY]], [[COPY1]], 1, implicit $m0, implicit $exec
+    ; GPRIDX: [[V_INDIRECT_REG_WRITE_B32_V4_:%[0-9]+]]:vreg_128 = V_INDIRECT_REG_WRITE_B32_V4 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
     ; GPRIDX: S_SET_GPR_IDX_OFF
     ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V4_]]
     %0:vgpr(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-constant.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-constant.mir
index 74f18b7e7c946..309bd1101a867 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-constant.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-constant.mir
@@ -800,8 +800,9 @@ body: |
     ; GFX10: liveins: $sgpr0_sgpr1
     ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
-    ; GFX10: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 1048575, 0, 0 :: (load 4, addrspace 4)
-    ; GFX10: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    ; GFX10: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1048575
+    ; GFX10: [[S_LOAD_DWORD_SGPR:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_SGPR [[COPY]], [[S_MOV_B32_]], 0, 0 :: (load 4, addrspace 4)
+    ; GFX10: $sgpr0 = COPY [[S_LOAD_DWORD_SGPR]]
     %0:sgpr(p4) = COPY $sgpr0_sgpr1
     %1:sgpr(s64) = G_CONSTANT i64 1048575
     %2:sgpr(p4) = G_PTR_ADD %0, %1
@@ -896,3 +897,137 @@ body: |
     $sgpr0 = COPY %3
 
 ...
+
+---
+
+name: load_constant_s32_from_4_gep_negative_1
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1
+
+    ; GFX6-LABEL: name: load_constant_s32_from_4_gep_negative_1
+    ; GFX6: liveins: $sgpr0_sgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX6: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 -1
+    ; GFX6: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[COPY]].sub0
+    ; GFX6: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[S_MOV_B64_]].sub0
+    ; GFX6: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[COPY]].sub1
+    ; GFX6: [[COPY4:%[0-9]+]]:sreg_32 = COPY [[S_MOV_B64_]].sub1
+    ; GFX6: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY1]], [[COPY2]], implicit-def $scc
+    ; GFX6: [[S_ADDC_U32_:%[0-9]+]]:sreg_32 = S_ADDC_U32 [[COPY3]], [[COPY4]], implicit-def $scc, implicit $scc
+    ; GFX6: [[REG_SEQUENCE:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[S_ADD_U32_]], %subreg.sub0, [[S_ADDC_U32_]], %subreg.sub1
+    ; GFX6: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[REG_SEQUENCE]], 0, 0, 0 :: (load 4, addrspace 4)
+    ; GFX6: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    ; GFX7-LABEL: name: load_constant_s32_from_4_gep_negative_1
+    ; GFX7: liveins: $sgpr0_sgpr1
+    ; GFX7: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX7: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 -1
+    ; GFX7: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[COPY]].sub0
+    ; GFX7: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[S_MOV_B64_]].sub0
+    ; GFX7: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[COPY]].sub1
+    ; GFX7: [[COPY4:%[0-9]+]]:sreg_32 = COPY [[S_MOV_B64_]].sub1
+    ; GFX7: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY1]], [[COPY2]], implicit-def $scc
+    ; GFX7: [[S_ADDC_U32_:%[0-9]+]]:sreg_32 = S_ADDC_U32 [[COPY3]], [[COPY4]], implicit-def $scc, implicit $scc
+    ; GFX7: [[REG_SEQUENCE:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[S_ADD_U32_]], %subreg.sub0, [[S_ADDC_U32_]], %subreg.sub1
+    ; GFX7: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[REG_SEQUENCE]], 0, 0, 0 :: (load 4, addrspace 4)
+    ; GFX7: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    ; GFX8-LABEL: name: load_constant_s32_from_4_gep_negative_1
+    ; GFX8: liveins: $sgpr0_sgpr1
+    ; GFX8: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX8: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 -1
+    ; GFX8: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[COPY]].sub0
+    ; GFX8: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[S_MOV_B64_]].sub0
+    ; GFX8: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[COPY]].sub1
+    ; GFX8: [[COPY4:%[0-9]+]]:sreg_32 = COPY [[S_MOV_B64_]].sub1
+    ; GFX8: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY1]], [[COPY2]], implicit-def $scc
+    ; GFX8: [[S_ADDC_U32_:%[0-9]+]]:sreg_32 = S_ADDC_U32 [[COPY3]], [[COPY4]], implicit-def $scc, implicit $scc
+    ; GFX8: [[REG_SEQUENCE:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[S_ADD_U32_]], %subreg.sub0, [[S_ADDC_U32_]], %subreg.sub1
+    ; GFX8: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[REG_SEQUENCE]], 0, 0, 0 :: (load 4, addrspace 4)
+    ; GFX8: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    ; GFX10-LABEL: name: load_constant_s32_from_4_gep_negative_1
+    ; GFX10: liveins: $sgpr0_sgpr1
+    ; GFX10: $vcc_hi = IMPLICIT_DEF
+    ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX10: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], -1, 0, 0 :: (load 4, addrspace 4)
+    ; GFX10: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    %0:sgpr(p4) = COPY $sgpr0_sgpr1
+    %1:sgpr(s64) = G_CONSTANT i64 -1
+    %2:sgpr(p4) = G_PTR_ADD %0, %1
+    %3:sgpr(s32) = G_LOAD %2 :: (load 4, align 4, addrspace 4)
+    $sgpr0 = COPY %3
+
+...
+
+---
+
+name: load_constant_s32_from_4_gep_negative_524288
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1
+
+    ; GFX6-LABEL: name: load_constant_s32_from_4_gep_negative_524288
+    ; GFX6: liveins: $sgpr0_sgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX6: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4294443008
+    ; GFX6: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
+    ; GFX6: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX6: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[COPY]].sub0
+    ; GFX6: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub0
+    ; GFX6: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[COPY]].sub1
+    ; GFX6: [[COPY4:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub1
+    ; GFX6: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY1]], [[COPY2]], implicit-def $scc
+    ; GFX6: [[S_ADDC_U32_:%[0-9]+]]:sreg_32 = S_ADDC_U32 [[COPY3]], [[COPY4]], implicit-def $scc, implicit $scc
+    ; GFX6: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[S_ADD_U32_]], %subreg.sub0, [[S_ADDC_U32_]], %subreg.sub1
+    ; GFX6: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[REG_SEQUENCE1]], 0, 0, 0 :: (load 4, addrspace 4)
+    ; GFX6: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    ; GFX7-LABEL: name: load_constant_s32_from_4_gep_negative_524288
+    ; GFX7: liveins: $sgpr0_sgpr1
+    ; GFX7: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX7: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4294443008
+    ; GFX7: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
+    ; GFX7: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX7: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[COPY]].sub0
+    ; GFX7: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub0
+    ; GFX7: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[COPY]].sub1
+    ; GFX7: [[COPY4:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub1
+    ; GFX7: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY1]], [[COPY2]], implicit-def $scc
+    ; GFX7: [[S_ADDC_U32_:%[0-9]+]]:sreg_32 = S_ADDC_U32 [[COPY3]], [[COPY4]], implicit-def $scc, implicit $scc
+    ; GFX7: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[S_ADD_U32_]], %subreg.sub0, [[S_ADDC_U32_]], %subreg.sub1
+    ; GFX7: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[REG_SEQUENCE1]], 0, 0, 0 :: (load 4, addrspace 4)
+    ; GFX7: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    ; GFX8-LABEL: name: load_constant_s32_from_4_gep_negative_524288
+    ; GFX8: liveins: $sgpr0_sgpr1
+    ; GFX8: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX8: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4294443008
+    ; GFX8: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
+    ; GFX8: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+    ; GFX8: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[COPY]].sub0
+    ; GFX8: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub0
+    ; GFX8: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[COPY]].sub1
+    ; GFX8: [[COPY4:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub1
+    ; GFX8: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY1]], [[COPY2]], implicit-def $scc
+    ; GFX8: [[S_ADDC_U32_:%[0-9]+]]:sreg_32 = S_ADDC_U32 [[COPY3]], [[COPY4]], implicit-def $scc, implicit $scc
+    ; GFX8: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[S_ADD_U32_]], %subreg.sub0, [[S_ADDC_U32_]], %subreg.sub1
+    ; GFX8: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[REG_SEQUENCE1]], 0, 0, 0 :: (load 4, addrspace 4)
+    ; GFX8: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    ; GFX10-LABEL: name: load_constant_s32_from_4_gep_negative_524288
+    ; GFX10: liveins: $sgpr0_sgpr1
+    ; GFX10: $vcc_hi = IMPLICIT_DEF
+    ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX10: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], -524288, 0, 0 :: (load 4, addrspace 4)
+    ; GFX10: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    %0:sgpr(p4) = COPY $sgpr0_sgpr1
+    %1:sgpr(s64) = G_CONSTANT i64 -524288
+    %2:sgpr(p4) = G_PTR_ADD %0, %1
+    %3:sgpr(s32) = G_LOAD %2 :: (load 4, align 4, addrspace 4)
+    $sgpr0 = COPY %3
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.ll
index fdf98d2acb582..f7e6fd5cd4746 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-function-args.ll
@@ -95,11 +95,10 @@ define void @void_func_i8(i8 %arg0) #0 {
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
   ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
-  ; CHECK:   [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC]](s16)
   ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
   ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
-  ; CHECK:   G_STORE [[TRUNC1]](s8), [[DEF]](p1) :: (store 1 into `i8 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   G_STORE [[TRUNC]](s8), [[DEF]](p1) :: (store 1 into `i8 addrspace(1)* undef`, addrspace 1)
   ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
   ; CHECK:   S_SETPC_B64_return [[COPY2]]
   store i8 %arg0, i8 addrspace(1)* undef
@@ -111,8 +110,8 @@ define void @void_func_i8_zeroext(i8 zeroext %arg0) #0 {
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
   ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
   ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
   ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
   ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; CHECK:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC]](s8)
@@ -131,8 +130,8 @@ define void @void_func_i8_signext(i8 signext %arg0) #0 {
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
   ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
   ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
   ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
   ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; CHECK:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[TRUNC]](s8)
@@ -201,6 +200,51 @@ define void @void_func_i16_signext(i16 signext %arg0) #0 {
   ret void
 }
 
+define void @void_func_i24(i24 %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_i24
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s24) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[TRUNC]](s24), [[DEF]](p1) :: (store 3 into `i24 addrspace(1)* undef`, align 4, addrspace 1)
+  ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
+  ; CHECK:   S_SETPC_B64_return [[COPY2]]
+  store i24 %arg0, i24 addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_i24_zeroext(i24 zeroext %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_i24_zeroext
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s24) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[TRUNC]](s24), [[DEF]](p1) :: (store 3 into `i24 addrspace(1)* undef`, align 4, addrspace 1)
+  ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
+  ; CHECK:   S_SETPC_B64_return [[COPY2]]
+  store i24 %arg0, i24 addrspace(1)* undef
+  ret void
+}
+
+define void @void_func_i24_signext(i24 signext %arg0) #0 {
+  ; CHECK-LABEL: name: void_func_i24_signext
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $vgpr0, $sgpr30_sgpr31
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s24) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK:   G_STORE [[TRUNC]](s24), [[DEF]](p1) :: (store 3 into `i24 addrspace(1)* undef`, align 4, addrspace 1)
+  ; CHECK:   [[COPY2:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY1]]
+  ; CHECK:   S_SETPC_B64_return [[COPY2]]
+  store i24 %arg0, i24 addrspace(1)* undef
+  ret void
+}
+
 define void @void_func_i32(i32 %arg0) #0 {
   ; CHECK-LABEL: name: void_func_i32
   ; CHECK: bb.1 (%ir-block.0):
@@ -1497,12 +1541,11 @@ define void @void_func_struct_i8_i32({ i8, i32 } %arg0) #0 {
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31
   ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
-  ; CHECK:   [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[TRUNC]](s16)
   ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
   ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
-  ; CHECK:   G_STORE [[TRUNC1]](s8), [[DEF]](p1) :: (store 1 into `{ i8, i32 } addrspace(1)* undef`, align 4, addrspace 1)
+  ; CHECK:   G_STORE [[TRUNC]](s8), [[DEF]](p1) :: (store 1 into `{ i8, i32 } addrspace(1)* undef`, align 4, addrspace 1)
   ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
   ; CHECK:   [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[DEF]], [[C]](s64)
   ; CHECK:   G_STORE [[COPY1]](s32), [[PTR_ADD]](p1) :: (store 4 into `{ i8, i32 } addrspace(1)* undef` + 4, addrspace 1)
@@ -1695,13 +1738,14 @@ define void @void_func_v32i32_i1_i8_i16(<32 x i32> %arg0, i1 %arg1, i8 %arg2, i1
   ; CHECK:   [[FRAME_INDEX:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.3
   ; CHECK:   [[LOAD:%[0-9]+]]:_(s1) = G_LOAD [[FRAME_INDEX]](p5) :: (invariant load 1 from %fixed-stack.3, align 16, addrspace 5)
   ; CHECK:   [[FRAME_INDEX1:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.2
-  ; CHECK:   [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX1]](p5) :: (invariant load 1 from %fixed-stack.2, align 4, addrspace 5)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p5) :: (invariant load 4 from %fixed-stack.2, addrspace 5)
   ; CHECK:   [[FRAME_INDEX2:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.1
   ; CHECK:   [[LOAD2:%[0-9]+]]:_(s16) = G_LOAD [[FRAME_INDEX2]](p5) :: (invariant load 2 from %fixed-stack.1, align 8, addrspace 5)
   ; CHECK:   [[FRAME_INDEX3:%[0-9]+]]:_(p5) = G_FRAME_INDEX %fixed-stack.0
   ; CHECK:   [[LOAD3:%[0-9]+]]:_(s16) = G_LOAD [[FRAME_INDEX3]](p5) :: (invariant load 2 from %fixed-stack.0, align 4, addrspace 5)
   ; CHECK:   [[COPY32:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
   ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<32 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32), [[COPY17]](s32), [[COPY18]](s32), [[COPY19]](s32), [[COPY20]](s32), [[COPY21]](s32), [[COPY22]](s32), [[COPY23]](s32), [[COPY24]](s32), [[COPY25]](s32), [[COPY26]](s32), [[COPY27]](s32), [[COPY28]](s32), [[COPY29]](s32), [[COPY30]](s32), [[COPY31]](s32)
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[LOAD1]](s32)
   ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; CHECK:   [[COPY33:%[0-9]+]]:_(p1) = COPY [[DEF]](p1)
   ; CHECK:   [[COPY34:%[0-9]+]]:_(p1) = COPY [[DEF]](p1)
@@ -1709,7 +1753,7 @@ define void @void_func_v32i32_i1_i8_i16(<32 x i32> %arg0, i1 %arg1, i8 %arg2, i1
   ; CHECK:   [[COPY36:%[0-9]+]]:_(p1) = COPY [[DEF]](p1)
   ; CHECK:   G_STORE [[BUILD_VECTOR]](<32 x s32>), [[DEF]](p1) :: (volatile store 128 into `<32 x i32> addrspace(1)* undef`, addrspace 1)
   ; CHECK:   G_STORE [[LOAD]](s1), [[COPY33]](p1) :: (volatile store 1 into `i1 addrspace(1)* undef`, addrspace 1)
-  ; CHECK:   G_STORE [[LOAD1]](s8), [[COPY34]](p1) :: (volatile store 1 into `i8 addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   G_STORE [[TRUNC]](s8), [[COPY34]](p1) :: (volatile store 1 into `i8 addrspace(1)* undef`, addrspace 1)
   ; CHECK:   G_STORE [[LOAD2]](s16), [[COPY35]](p1) :: (volatile store 2 into `i16 addrspace(1)* undef`, addrspace 1)
   ; CHECK:   G_STORE [[LOAD3]](s16), [[COPY36]](p1) :: (volatile store 2 into `half addrspace(1)* undef`, addrspace 1)
   ; CHECK:   [[COPY37:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY32]]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-sat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-sat.ll
index efdecf7fb49fe..2d648580408f7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-sat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-sat.ll
@@ -6,10 +6,10 @@ define i16 @uaddsat_i16(i16 %lhs, i16 %rhs) {
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31
   ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
   ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; CHECK:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
   ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
   ; CHECK:   [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC]], [[TRUNC1]]
   ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UADDSAT]](s16)
   ; CHECK:   $vgpr0 = COPY [[ANYEXT]](s32)
@@ -85,10 +85,10 @@ define i16 @saddsat_i16(i16 %lhs, i16 %rhs) {
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31
   ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
   ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; CHECK:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
   ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
   ; CHECK:   [[SADDSAT:%[0-9]+]]:_(s16) = G_SADDSAT [[TRUNC]], [[TRUNC1]]
   ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SADDSAT]](s16)
   ; CHECK:   $vgpr0 = COPY [[ANYEXT]](s32)
@@ -164,10 +164,10 @@ define i16 @usubsat_i16(i16 %lhs, i16 %rhs) {
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31
   ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
   ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; CHECK:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
   ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
   ; CHECK:   [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC]], [[TRUNC1]]
   ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[USUBSAT]](s16)
   ; CHECK:   $vgpr0 = COPY [[ANYEXT]](s32)
@@ -243,10 +243,10 @@ define i16 @ssubsat_i16(i16 %lhs, i16 %rhs) {
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $vgpr0, $vgpr1, $sgpr30_sgpr31
   ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
   ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; CHECK:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
   ; CHECK:   [[COPY2:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
   ; CHECK:   [[SSUBSAT:%[0-9]+]]:_(s16) = G_SSUBSAT [[TRUNC]], [[TRUNC1]]
   ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SSUBSAT]](s16)
   ; CHECK:   $vgpr0 = COPY [[ANYEXT]](s32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmad.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmad.s16.mir
index 2e96ee4e881f0..441eec00d5e9c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmad.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmad.s16.mir
@@ -396,10 +396,10 @@ body: |
     ; GFX7: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
     ; GFX7: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
     ; GFX7: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32)
-    ; GFX7: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT %6(s16)
-    ; GFX7: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX7: [[FMUL:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC]], [[TRUNC1]]
     ; GFX7: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[FMUL]], [[TRUNC2]]
+    ; GFX7: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FADD]](s16)
+    ; GFX7: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX10-LABEL: name: test_fmad_s16_denorm
     ; GFX10: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
@@ -457,10 +457,10 @@ body: |
     ; GFX7: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
     ; GFX7: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
     ; GFX7: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32)
-    ; GFX7: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT %6(s16)
-    ; GFX7: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX7: [[FMUL:%[0-9]+]]:_(s16) = nnan G_FMUL [[TRUNC]], [[TRUNC1]]
     ; GFX7: [[FADD:%[0-9]+]]:_(s16) = nnan G_FADD [[FMUL]], [[TRUNC2]]
+    ; GFX7: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FADD]](s16)
+    ; GFX7: $vgpr0 = COPY [[ANYEXT]](s32)
     ; GFX10-LABEL: name: test_fmad_s16_denorm_flags
     ; GFX10: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
@@ -550,16 +550,16 @@ body: |
     ; GFX7: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
     ; GFX7: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
     ; GFX7: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
-    ; GFX7: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT %10(s16)
-    ; GFX7: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT %11(s16)
+    ; GFX7: [[FMUL:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC]], [[TRUNC2]]
+    ; GFX7: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[FMUL]], [[TRUNC4]]
+    ; GFX7: [[FMUL1:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC1]], [[TRUNC3]]
+    ; GFX7: [[FADD1:%[0-9]+]]:_(s16) = G_FADD [[FMUL1]], [[TRUNC5]]
+    ; GFX7: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FADD]](s16)
+    ; GFX7: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FADD1]](s16)
     ; GFX7: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
     ; GFX7: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
     ; GFX7: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
     ; GFX7: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
-    ; GFX7: [[FMUL:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC1]], [[TRUNC3]]
-    ; GFX7: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[FMUL]], [[TRUNC5]]
-    ; GFX7: [[FMUL1:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC]], [[TRUNC2]]
-    ; GFX7: [[FADD1:%[0-9]+]]:_(s16) = G_FADD [[FMUL1]], [[TRUNC4]]
     ; GFX10-LABEL: name: test_fmad_v2s16_denorm
     ; GFX10: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
@@ -659,16 +659,16 @@ body: |
     ; GFX7: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST2]](s32)
     ; GFX7: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](s32)
     ; GFX7: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
-    ; GFX7: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT %10(s16)
-    ; GFX7: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT %11(s16)
+    ; GFX7: [[FMUL:%[0-9]+]]:_(s16) = nnan G_FMUL [[TRUNC]], [[TRUNC2]]
+    ; GFX7: [[FADD:%[0-9]+]]:_(s16) = nnan G_FADD [[FMUL]], [[TRUNC4]]
+    ; GFX7: [[FMUL1:%[0-9]+]]:_(s16) = nnan G_FMUL [[TRUNC1]], [[TRUNC3]]
+    ; GFX7: [[FADD1:%[0-9]+]]:_(s16) = nnan G_FADD [[FMUL1]], [[TRUNC5]]
+    ; GFX7: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FADD]](s16)
+    ; GFX7: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FADD1]](s16)
     ; GFX7: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
     ; GFX7: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
     ; GFX7: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
     ; GFX7: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
-    ; GFX7: [[FMUL:%[0-9]+]]:_(s16) = nnan G_FMUL [[TRUNC1]], [[TRUNC3]]
-    ; GFX7: [[FADD:%[0-9]+]]:_(s16) = nnan G_FADD [[FMUL]], [[TRUNC5]]
-    ; GFX7: [[FMUL1:%[0-9]+]]:_(s16) = nnan G_FMUL [[TRUNC]], [[TRUNC2]]
-    ; GFX7: [[FADD1:%[0-9]+]]:_(s16) = nnan G_FADD [[FMUL1]], [[TRUNC4]]
     ; GFX10-LABEL: name: test_fmad_v2s16_denorm_flags
     ; GFX10: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
@@ -820,26 +820,26 @@ body: |
     ; GFX7: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
     ; GFX7: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
     ; GFX7: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](s32)
-    ; GFX7: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT %16(s16)
-    ; GFX7: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT %17(s16)
+    ; GFX7: [[FMUL:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC]], [[TRUNC4]]
+    ; GFX7: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[FMUL]], [[TRUNC8]]
+    ; GFX7: [[FMUL1:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC1]], [[TRUNC5]]
+    ; GFX7: [[FADD1:%[0-9]+]]:_(s16) = G_FADD [[FMUL1]], [[TRUNC9]]
+    ; GFX7: [[FMUL2:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC2]], [[TRUNC6]]
+    ; GFX7: [[FADD2:%[0-9]+]]:_(s16) = G_FADD [[FMUL2]], [[TRUNC10]]
+    ; GFX7: [[FMUL3:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC3]], [[TRUNC7]]
+    ; GFX7: [[FADD3:%[0-9]+]]:_(s16) = G_FADD [[FMUL3]], [[TRUNC11]]
+    ; GFX7: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FADD]](s16)
+    ; GFX7: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FADD1]](s16)
     ; GFX7: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
     ; GFX7: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
     ; GFX7: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
-    ; GFX7: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT %18(s16)
-    ; GFX7: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT %19(s16)
+    ; GFX7: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FADD2]](s16)
+    ; GFX7: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FADD3]](s16)
     ; GFX7: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
     ; GFX7: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
     ; GFX7: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
     ; GFX7: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
     ; GFX7: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
-    ; GFX7: [[FMUL:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC3]], [[TRUNC7]]
-    ; GFX7: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[FMUL]], [[TRUNC11]]
-    ; GFX7: [[FMUL1:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC2]], [[TRUNC6]]
-    ; GFX7: [[FADD1:%[0-9]+]]:_(s16) = G_FADD [[FMUL1]], [[TRUNC10]]
-    ; GFX7: [[FMUL2:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC1]], [[TRUNC5]]
-    ; GFX7: [[FADD2:%[0-9]+]]:_(s16) = G_FADD [[FMUL2]], [[TRUNC9]]
-    ; GFX7: [[FMUL3:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC]], [[TRUNC4]]
-    ; GFX7: [[FADD3:%[0-9]+]]:_(s16) = G_FADD [[FMUL3]], [[TRUNC8]]
     ; GFX10-LABEL: name: test_fmad_v4s16_denorm
     ; GFX10: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
@@ -1015,26 +1015,26 @@ body: |
     ; GFX7: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[BITCAST5]](s32)
     ; GFX7: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST5]], [[C]](s32)
     ; GFX7: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](s32)
-    ; GFX7: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT %16(s16)
-    ; GFX7: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT %17(s16)
+    ; GFX7: [[FMUL:%[0-9]+]]:_(s16) = nnan G_FMUL [[TRUNC]], [[TRUNC4]]
+    ; GFX7: [[FADD:%[0-9]+]]:_(s16) = nnan G_FADD [[FMUL]], [[TRUNC8]]
+    ; GFX7: [[FMUL1:%[0-9]+]]:_(s16) = nnan G_FMUL [[TRUNC1]], [[TRUNC5]]
+    ; GFX7: [[FADD1:%[0-9]+]]:_(s16) = nnan G_FADD [[FMUL1]], [[TRUNC9]]
+    ; GFX7: [[FMUL2:%[0-9]+]]:_(s16) = nnan G_FMUL [[TRUNC2]], [[TRUNC6]]
+    ; GFX7: [[FADD2:%[0-9]+]]:_(s16) = nnan G_FADD [[FMUL2]], [[TRUNC10]]
+    ; GFX7: [[FMUL3:%[0-9]+]]:_(s16) = nnan G_FMUL [[TRUNC3]], [[TRUNC7]]
+    ; GFX7: [[FADD3:%[0-9]+]]:_(s16) = nnan G_FADD [[FMUL3]], [[TRUNC11]]
+    ; GFX7: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FADD]](s16)
+    ; GFX7: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FADD1]](s16)
     ; GFX7: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
     ; GFX7: [[OR:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL]]
     ; GFX7: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32)
-    ; GFX7: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT %18(s16)
-    ; GFX7: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT %19(s16)
+    ; GFX7: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FADD2]](s16)
+    ; GFX7: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FADD3]](s16)
     ; GFX7: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
     ; GFX7: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL1]]
     ; GFX7: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](s32)
     ; GFX7: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
     ; GFX7: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
-    ; GFX7: [[FMUL:%[0-9]+]]:_(s16) = nnan G_FMUL [[TRUNC3]], [[TRUNC7]]
-    ; GFX7: [[FADD:%[0-9]+]]:_(s16) = nnan G_FADD [[FMUL]], [[TRUNC11]]
-    ; GFX7: [[FMUL1:%[0-9]+]]:_(s16) = nnan G_FMUL [[TRUNC2]], [[TRUNC6]]
-    ; GFX7: [[FADD1:%[0-9]+]]:_(s16) = nnan G_FADD [[FMUL1]], [[TRUNC10]]
-    ; GFX7: [[FMUL2:%[0-9]+]]:_(s16) = nnan G_FMUL [[TRUNC1]], [[TRUNC5]]
-    ; GFX7: [[FADD2:%[0-9]+]]:_(s16) = nnan G_FADD [[FMUL2]], [[TRUNC9]]
-    ; GFX7: [[FMUL3:%[0-9]+]]:_(s16) = nnan G_FMUL [[TRUNC]], [[TRUNC4]]
-    ; GFX7: [[FADD3:%[0-9]+]]:_(s16) = nnan G_FADD [[FMUL3]], [[TRUNC8]]
     ; GFX10-LABEL: name: test_fmad_v4s16_denorm_flags
     ; GFX10: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmad.s32.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmad.s32.mir
index 32738d499bda7..95515f3593ace 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmad.s32.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmad.s32.mir
@@ -252,23 +252,23 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; GFX6: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
-    ; GFX6: $vgpr0 = COPY %3(s32)
     ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[COPY]], [[COPY1]]
     ; GFX6: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[COPY2]]
+    ; GFX6: $vgpr0 = COPY [[FADD]](s32)
     ; GFX7-LABEL: name: test_fmad_s32_denorm
     ; GFX7: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX7: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; GFX7: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
-    ; GFX7: $vgpr0 = COPY %3(s32)
     ; GFX7: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[COPY]], [[COPY1]]
     ; GFX7: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[COPY2]]
+    ; GFX7: $vgpr0 = COPY [[FADD]](s32)
     ; GFX10-LABEL: name: test_fmad_s32_denorm
     ; GFX10: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
-    ; GFX10: $vgpr0 = COPY %3(s32)
     ; GFX10: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[COPY]], [[COPY1]]
     ; GFX10: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[COPY2]]
+    ; GFX10: $vgpr0 = COPY [[FADD]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = COPY $vgpr2
@@ -291,23 +291,23 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; GFX6: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
-    ; GFX6: $vgpr0 = COPY %3(s32)
     ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = nnan G_FMUL [[COPY]], [[COPY1]]
     ; GFX6: [[FADD:%[0-9]+]]:_(s32) = nnan G_FADD [[FMUL]], [[COPY2]]
+    ; GFX6: $vgpr0 = COPY [[FADD]](s32)
     ; GFX7-LABEL: name: test_fmad_s32_flags_denorm
     ; GFX7: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX7: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; GFX7: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
-    ; GFX7: $vgpr0 = COPY %3(s32)
     ; GFX7: [[FMUL:%[0-9]+]]:_(s32) = nnan G_FMUL [[COPY]], [[COPY1]]
     ; GFX7: [[FADD:%[0-9]+]]:_(s32) = nnan G_FADD [[FMUL]], [[COPY2]]
+    ; GFX7: $vgpr0 = COPY [[FADD]](s32)
     ; GFX10-LABEL: name: test_fmad_s32_flags_denorm
     ; GFX10: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
-    ; GFX10: $vgpr0 = COPY %3(s32)
     ; GFX10: [[FMUL:%[0-9]+]]:_(s32) = nnan G_FMUL [[COPY]], [[COPY1]]
     ; GFX10: [[FADD:%[0-9]+]]:_(s32) = nnan G_FADD [[FMUL]], [[COPY2]]
+    ; GFX10: $vgpr0 = COPY [[FADD]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = COPY $vgpr2
@@ -333,12 +333,12 @@ body: |
     ; GFX6: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
     ; GFX6: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
     ; GFX6: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<2 x s32>)
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR %10(s32), %11(s32)
+    ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UV]], [[UV2]]
+    ; GFX6: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UV4]]
+    ; GFX6: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[UV1]], [[UV3]]
+    ; GFX6: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL1]], [[UV5]]
+    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FADD]](s32), [[FADD1]](s32)
     ; GFX6: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
-    ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UV1]], [[UV3]]
-    ; GFX6: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UV5]]
-    ; GFX6: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[UV]], [[UV2]]
-    ; GFX6: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL1]], [[UV4]]
     ; GFX7-LABEL: name: test_fmad_v2s32_denorm
     ; GFX7: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
     ; GFX7: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
@@ -346,12 +346,12 @@ body: |
     ; GFX7: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
     ; GFX7: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
     ; GFX7: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<2 x s32>)
-    ; GFX7: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR %10(s32), %11(s32)
+    ; GFX7: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UV]], [[UV2]]
+    ; GFX7: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UV4]]
+    ; GFX7: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[UV1]], [[UV3]]
+    ; GFX7: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL1]], [[UV5]]
+    ; GFX7: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FADD]](s32), [[FADD1]](s32)
     ; GFX7: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
-    ; GFX7: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UV1]], [[UV3]]
-    ; GFX7: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UV5]]
-    ; GFX7: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[UV]], [[UV2]]
-    ; GFX7: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL1]], [[UV4]]
     ; GFX10-LABEL: name: test_fmad_v2s32_denorm
     ; GFX10: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
@@ -359,12 +359,12 @@ body: |
     ; GFX10: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
     ; GFX10: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
     ; GFX10: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<2 x s32>)
-    ; GFX10: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR %10(s32), %11(s32)
+    ; GFX10: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UV]], [[UV2]]
+    ; GFX10: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UV4]]
+    ; GFX10: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[UV1]], [[UV3]]
+    ; GFX10: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL1]], [[UV5]]
+    ; GFX10: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FADD]](s32), [[FADD1]](s32)
     ; GFX10: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
-    ; GFX10: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UV1]], [[UV3]]
-    ; GFX10: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UV5]]
-    ; GFX10: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[UV]], [[UV2]]
-    ; GFX10: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL1]], [[UV4]]
     %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
     %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
     %2:_(<2 x s32>) = COPY $vgpr4_vgpr5
@@ -390,14 +390,14 @@ body: |
     ; GFX6: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
     ; GFX6: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
     ; GFX6: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<3 x s32>)
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR %13(s32), %14(s32), %15(s32)
-    ; GFX6: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
-    ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UV2]], [[UV5]]
-    ; GFX6: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UV8]]
+    ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UV]], [[UV3]]
+    ; GFX6: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UV6]]
     ; GFX6: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[UV1]], [[UV4]]
     ; GFX6: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL1]], [[UV7]]
-    ; GFX6: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[UV]], [[UV3]]
-    ; GFX6: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FMUL2]], [[UV6]]
+    ; GFX6: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[UV2]], [[UV5]]
+    ; GFX6: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FMUL2]], [[UV8]]
+    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FADD]](s32), [[FADD1]](s32), [[FADD2]](s32)
+    ; GFX6: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
     ; GFX7-LABEL: name: test_fmad_v3s32_denorm
     ; GFX7: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
     ; GFX7: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
@@ -405,14 +405,14 @@ body: |
     ; GFX7: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
     ; GFX7: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
     ; GFX7: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<3 x s32>)
-    ; GFX7: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR %13(s32), %14(s32), %15(s32)
-    ; GFX7: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
-    ; GFX7: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UV2]], [[UV5]]
-    ; GFX7: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UV8]]
+    ; GFX7: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UV]], [[UV3]]
+    ; GFX7: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UV6]]
     ; GFX7: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[UV1]], [[UV4]]
     ; GFX7: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL1]], [[UV7]]
-    ; GFX7: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[UV]], [[UV3]]
-    ; GFX7: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FMUL2]], [[UV6]]
+    ; GFX7: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[UV2]], [[UV5]]
+    ; GFX7: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FMUL2]], [[UV8]]
+    ; GFX7: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FADD]](s32), [[FADD1]](s32), [[FADD2]](s32)
+    ; GFX7: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
     ; GFX10-LABEL: name: test_fmad_v3s32_denorm
     ; GFX10: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
     ; GFX10: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
@@ -420,14 +420,14 @@ body: |
     ; GFX10: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
     ; GFX10: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
     ; GFX10: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<3 x s32>)
-    ; GFX10: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR %13(s32), %14(s32), %15(s32)
-    ; GFX10: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
-    ; GFX10: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UV2]], [[UV5]]
-    ; GFX10: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UV8]]
+    ; GFX10: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UV]], [[UV3]]
+    ; GFX10: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UV6]]
     ; GFX10: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[UV1]], [[UV4]]
     ; GFX10: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL1]], [[UV7]]
-    ; GFX10: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[UV]], [[UV3]]
-    ; GFX10: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FMUL2]], [[UV6]]
+    ; GFX10: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[UV2]], [[UV5]]
+    ; GFX10: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FMUL2]], [[UV8]]
+    ; GFX10: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FADD]](s32), [[FADD1]](s32), [[FADD2]](s32)
+    ; GFX10: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
     %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
     %1:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
     %2:_(<3 x s32>) = COPY $vgpr6_vgpr7_vgpr8
@@ -453,16 +453,16 @@ body: |
     ; GFX6: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<4 x s32>)
     ; GFX6: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
     ; GFX6: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<4 x s32>)
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR %16(s32), %17(s32), %18(s32), %19(s32)
+    ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UV]], [[UV4]]
+    ; GFX6: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UV8]]
+    ; GFX6: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[UV1]], [[UV5]]
+    ; GFX6: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL1]], [[UV9]]
+    ; GFX6: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[UV2]], [[UV6]]
+    ; GFX6: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FMUL2]], [[UV10]]
+    ; GFX6: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[UV3]], [[UV7]]
+    ; GFX6: [[FADD3:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[UV11]]
+    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[FADD]](s32), [[FADD1]](s32), [[FADD2]](s32), [[FADD3]](s32)
     ; GFX6: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>)
-    ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UV3]], [[UV7]]
-    ; GFX6: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UV11]]
-    ; GFX6: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[UV2]], [[UV6]]
-    ; GFX6: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL1]], [[UV10]]
-    ; GFX6: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[UV1]], [[UV5]]
-    ; GFX6: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FMUL2]], [[UV9]]
-    ; GFX6: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[UV]], [[UV4]]
-    ; GFX6: [[FADD3:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[UV8]]
     ; GFX7-LABEL: name: test_fmad_v4s32_denorm
     ; GFX7: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX7: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
@@ -470,16 +470,16 @@ body: |
     ; GFX7: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<4 x s32>)
     ; GFX7: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
     ; GFX7: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<4 x s32>)
-    ; GFX7: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR %16(s32), %17(s32), %18(s32), %19(s32)
+    ; GFX7: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UV]], [[UV4]]
+    ; GFX7: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UV8]]
+    ; GFX7: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[UV1]], [[UV5]]
+    ; GFX7: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL1]], [[UV9]]
+    ; GFX7: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[UV2]], [[UV6]]
+    ; GFX7: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FMUL2]], [[UV10]]
+    ; GFX7: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[UV3]], [[UV7]]
+    ; GFX7: [[FADD3:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[UV11]]
+    ; GFX7: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[FADD]](s32), [[FADD1]](s32), [[FADD2]](s32), [[FADD3]](s32)
     ; GFX7: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>)
-    ; GFX7: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UV3]], [[UV7]]
-    ; GFX7: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UV11]]
-    ; GFX7: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[UV2]], [[UV6]]
-    ; GFX7: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL1]], [[UV10]]
-    ; GFX7: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[UV1]], [[UV5]]
-    ; GFX7: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FMUL2]], [[UV9]]
-    ; GFX7: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[UV]], [[UV4]]
-    ; GFX7: [[FADD3:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[UV8]]
     ; GFX10-LABEL: name: test_fmad_v4s32_denorm
     ; GFX10: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX10: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
@@ -487,16 +487,16 @@ body: |
     ; GFX10: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<4 x s32>)
     ; GFX10: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
     ; GFX10: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<4 x s32>)
-    ; GFX10: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR %16(s32), %17(s32), %18(s32), %19(s32)
+    ; GFX10: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UV]], [[UV4]]
+    ; GFX10: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UV8]]
+    ; GFX10: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[UV1]], [[UV5]]
+    ; GFX10: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL1]], [[UV9]]
+    ; GFX10: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[UV2]], [[UV6]]
+    ; GFX10: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FMUL2]], [[UV10]]
+    ; GFX10: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[UV3]], [[UV7]]
+    ; GFX10: [[FADD3:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[UV11]]
+    ; GFX10: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[FADD]](s32), [[FADD1]](s32), [[FADD2]](s32), [[FADD3]](s32)
     ; GFX10: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>)
-    ; GFX10: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UV3]], [[UV7]]
-    ; GFX10: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UV11]]
-    ; GFX10: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[UV2]], [[UV6]]
-    ; GFX10: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL1]], [[UV10]]
-    ; GFX10: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[UV1]], [[UV5]]
-    ; GFX10: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FMUL2]], [[UV9]]
-    ; GFX10: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[UV]], [[UV4]]
-    ; GFX10: [[FADD3:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[UV8]]
     %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     %1:_(<4 x s32>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
     %2:_(<4 x s32>) = COPY $vgpr8_vgpr9_vgpr10_vgpr11
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshr.mir
new file mode 100644
index 0000000000000..b26b28f6bf341
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshr.mir
@@ -0,0 +1,299 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer -global-isel-abort=0 %s -o - | FileCheck -check-prefix=SI %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer  -global-isel-abort=0 %s -o - | FileCheck -check-prefix=VI %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -O0 -run-pass=legalizer  -global-isel-abort=0 %s -o - | FileCheck -check-prefix=GFX9  %s
+
+---
+name: test_fshr_s32_s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+
+    ; SI-LABEL: name: test_fshr_s32_s32
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; SI: [[FSHR:%[0-9]+]]:_(s32) = G_FSHR [[COPY]], [[COPY1]], [[COPY2]](s32)
+    ; SI: $vgpr0 = COPY [[FSHR]](s32)
+    ; VI-LABEL: name: test_fshr_s32_s32
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; VI: [[FSHR:%[0-9]+]]:_(s32) = G_FSHR [[COPY]], [[COPY1]], [[COPY2]](s32)
+    ; VI: $vgpr0 = COPY [[FSHR]](s32)
+    ; GFX9-LABEL: name: test_fshr_s32_s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; GFX9: [[FSHR:%[0-9]+]]:_(s32) = G_FSHR [[COPY]], [[COPY1]], [[COPY2]](s32)
+    ; GFX9: $vgpr0 = COPY [[FSHR]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = G_FSHR %0, %1, %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: test_fshr_v2s32_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_fshr_v2s32_v2s32
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; SI: [[COPY2:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; SI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; SI: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<2 x s32>)
+    ; SI: [[FSHR:%[0-9]+]]:_(s32) = G_FSHR [[UV]], [[UV2]], [[UV4]](s32)
+    ; SI: [[FSHR1:%[0-9]+]]:_(s32) = G_FSHR [[UV1]], [[UV3]], [[UV5]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FSHR]](s32), [[FSHR1]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; VI-LABEL: name: test_fshr_v2s32_v2s32
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; VI: [[COPY2:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; VI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; VI: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<2 x s32>)
+    ; VI: [[FSHR:%[0-9]+]]:_(s32) = G_FSHR [[UV]], [[UV2]], [[UV4]](s32)
+    ; VI: [[FSHR1:%[0-9]+]]:_(s32) = G_FSHR [[UV1]], [[UV3]], [[UV5]](s32)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FSHR]](s32), [[FSHR1]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; GFX9-LABEL: name: test_fshr_v2s32_v2s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; GFX9: [[COPY2:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<2 x s32>)
+    ; GFX9: [[FSHR:%[0-9]+]]:_(s32) = G_FSHR [[UV]], [[UV2]], [[UV4]](s32)
+    ; GFX9: [[FSHR1:%[0-9]+]]:_(s32) = G_FSHR [[UV1]], [[UV3]], [[UV5]](s32)
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FSHR]](s32), [[FSHR1]](s32)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    %3:_(<2 x s32>) = G_FSHR %0, %1, %2
+    $vgpr0_vgpr1 = COPY %3
+...
+
+---
+name: test_fshr_s16_s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+
+    ; SI-LABEL: name: test_fshr_s16_s16
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32)
+    ; SI: [[FSHR:%[0-9]+]]:_(s16) = G_FSHR [[TRUNC]], [[TRUNC1]], [[TRUNC2]](s16)
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FSHR]](s16)
+    ; SI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; VI-LABEL: name: test_fshr_s16_s16
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32)
+    ; VI: [[FSHR:%[0-9]+]]:_(s16) = G_FSHR [[TRUNC]], [[TRUNC1]], [[TRUNC2]](s16)
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FSHR]](s16)
+    ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX9-LABEL: name: test_fshr_s16_s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; GFX9: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32)
+    ; GFX9: [[FSHR:%[0-9]+]]:_(s16) = G_FSHR [[TRUNC]], [[TRUNC1]], [[TRUNC2]](s16)
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FSHR]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s16) = G_TRUNC %0
+    %4:_(s16) = G_TRUNC %1
+    %5:_(s16) = G_TRUNC %2
+    %6:_(s16) = G_FSHR %3, %4, %5
+    %7:_(s32) = G_ANYEXT %6
+    $vgpr0 = COPY %7
+...
+
+---
+name: test_fshr_v2s16_v2s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+
+    ; SI-LABEL: name: test_fshr_v2s16_v2s16
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; SI: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; SI: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
+    ; SI: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY2]](<2 x s16>)
+    ; SI: [[FSHR:%[0-9]+]]:_(s16) = G_FSHR [[UV]], [[UV2]], [[UV4]](s16)
+    ; SI: [[FSHR1:%[0-9]+]]:_(s16) = G_FSHR [[UV1]], [[UV3]], [[UV5]](s16)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[FSHR]](s16), [[FSHR1]](s16)
+    ; SI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; VI-LABEL: name: test_fshr_v2s16_v2s16
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; VI: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; VI: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
+    ; VI: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY2]](<2 x s16>)
+    ; VI: [[FSHR:%[0-9]+]]:_(s16) = G_FSHR [[UV]], [[UV2]], [[UV4]](s16)
+    ; VI: [[FSHR1:%[0-9]+]]:_(s16) = G_FSHR [[UV1]], [[UV3]], [[UV5]](s16)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[FSHR]](s16), [[FSHR1]](s16)
+    ; VI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX9-LABEL: name: test_fshr_v2s16_v2s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; GFX9: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
+    ; GFX9: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY2]](<2 x s16>)
+    ; GFX9: [[FSHR:%[0-9]+]]:_(s16) = G_FSHR [[UV]], [[UV2]], [[UV4]](s16)
+    ; GFX9: [[FSHR1:%[0-9]+]]:_(s16) = G_FSHR [[UV1]], [[UV3]], [[UV5]](s16)
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[FSHR]](s16), [[FSHR1]](s16)
+    ; GFX9: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s16>) = COPY $vgpr1
+    %2:_(<2 x s16>) = COPY $vgpr2
+    %3:_(<2 x s16>) = G_FSHR %0, %1, %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: test_fshr_s64_s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_fshr_s64_s64
+    ; SI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; SI: [[COPY2:%[0-9]+]]:_(s64) = COPY $vgpr4_vgpr5
+    ; SI: [[FSHR:%[0-9]+]]:_(s64) = G_FSHR [[COPY]], [[COPY1]], [[COPY2]](s64)
+    ; SI: $vgpr0_vgpr1 = COPY [[FSHR]](s64)
+    ; VI-LABEL: name: test_fshr_s64_s64
+    ; VI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; VI: [[COPY2:%[0-9]+]]:_(s64) = COPY $vgpr4_vgpr5
+    ; VI: [[FSHR:%[0-9]+]]:_(s64) = G_FSHR [[COPY]], [[COPY1]], [[COPY2]](s64)
+    ; VI: $vgpr0_vgpr1 = COPY [[FSHR]](s64)
+    ; GFX9-LABEL: name: test_fshr_s64_s64
+    ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s64) = COPY $vgpr4_vgpr5
+    ; GFX9: [[FSHR:%[0-9]+]]:_(s64) = G_FSHR [[COPY]], [[COPY1]], [[COPY2]](s64)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[FSHR]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = COPY $vgpr2_vgpr3
+    %2:_(s64) = COPY $vgpr4_vgpr5
+    %3:_(s64) = G_FSHR %0, %1, %2
+    $vgpr0_vgpr1 = COPY %3
+...
+
+---
+name: test_fshr_s8_s8
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+
+    ; SI-LABEL: name: test_fshr_s8_s8
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; SI: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32)
+    ; SI: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[COPY2]](s32)
+    ; SI: [[FSHR:%[0-9]+]]:_(s8) = G_FSHR [[TRUNC]], [[TRUNC1]], [[TRUNC2]](s8)
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FSHR]](s8)
+    ; SI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; VI-LABEL: name: test_fshr_s8_s8
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; VI: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32)
+    ; VI: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[COPY2]](s32)
+    ; VI: [[FSHR:%[0-9]+]]:_(s8) = G_FSHR [[TRUNC]], [[TRUNC1]], [[TRUNC2]](s8)
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FSHR]](s8)
+    ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX9-LABEL: name: test_fshr_s8_s8
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32)
+    ; GFX9: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[COPY2]](s32)
+    ; GFX9: [[FSHR:%[0-9]+]]:_(s8) = G_FSHR [[TRUNC]], [[TRUNC1]], [[TRUNC2]](s8)
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FSHR]](s8)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s8) = G_TRUNC %0
+    %4:_(s8) = G_TRUNC %1
+    %5:_(s8) = G_TRUNC %2
+    %6:_(s8) = G_FSHR %3, %4, %5
+    %7:_(s32) = G_ANYEXT %6
+    $vgpr0 = COPY %7
+...
+
+---
+name: test_fshr_s24_s24
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+
+    ; SI-LABEL: name: test_fshr_s24_s24
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; SI: [[TRUNC:%[0-9]+]]:_(s24) = G_TRUNC [[COPY]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s24) = G_TRUNC [[COPY1]](s32)
+    ; SI: [[TRUNC2:%[0-9]+]]:_(s24) = G_TRUNC [[COPY2]](s32)
+    ; SI: [[FSHR:%[0-9]+]]:_(s24) = G_FSHR [[TRUNC]], [[TRUNC1]], [[TRUNC2]](s24)
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FSHR]](s24)
+    ; SI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; VI-LABEL: name: test_fshr_s24_s24
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; VI: [[TRUNC:%[0-9]+]]:_(s24) = G_TRUNC [[COPY]](s32)
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s24) = G_TRUNC [[COPY1]](s32)
+    ; VI: [[TRUNC2:%[0-9]+]]:_(s24) = G_TRUNC [[COPY2]](s32)
+    ; VI: [[FSHR:%[0-9]+]]:_(s24) = G_FSHR [[TRUNC]], [[TRUNC1]], [[TRUNC2]](s24)
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FSHR]](s24)
+    ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX9-LABEL: name: test_fshr_s24_s24
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s24) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s24) = G_TRUNC [[COPY1]](s32)
+    ; GFX9: [[TRUNC2:%[0-9]+]]:_(s24) = G_TRUNC [[COPY2]](s32)
+    ; GFX9: [[FSHR:%[0-9]+]]:_(s24) = G_FSHR [[TRUNC]], [[TRUNC1]], [[TRUNC2]](s24)
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FSHR]](s24)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s24) = G_TRUNC %0
+    %4:_(s24) = G_TRUNC %1
+    %5:_(s24) = G_TRUNC %2
+    %6:_(s24) = G_FSHR %3, %4, %5
+    %7:_(s32) = G_ANYEXT %6
+    $vgpr0 = COPY %7
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.atomic.dim.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.atomic.dim.a16.ll
index 7f40f1c665104..a0a1366f7408c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.atomic.dim.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.atomic.dim.a16.ll
@@ -536,7 +536,8 @@ define amdgpu_ps float @atomic_cmpswap_1d(<8 x i32> inreg %rsrc, i32 %cmp, i32 %
   ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
   ; GFX9:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY10]](s32)
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.1d), [[COPY8]](s32), [[COPY9]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (volatile dereferenceable load store 4 on custom "TargetCustom8")
+  ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
+  ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.1d), [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (volatile dereferenceable load store 4 on custom "TargetCustom8")
   ; GFX9:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10NSA-LABEL: name: atomic_cmpswap_1d
@@ -555,7 +556,8 @@ define amdgpu_ps float @atomic_cmpswap_1d(<8 x i32> inreg %rsrc, i32 %cmp, i32 %
   ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
   ; GFX10NSA:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY10]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.1d), [[COPY8]](s32), [[COPY9]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (volatile dereferenceable load store 4 on custom "TargetCustom8")
+  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
+  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.1d), [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (volatile dereferenceable load store 4 on custom "TargetCustom8")
   ; GFX10NSA:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
@@ -1014,10 +1016,11 @@ define amdgpu_ps float @atomic_cmpswap_2d(<8 x i32> inreg %rsrc, i32 %cmp, i32 %
   ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
   ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr3
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
   ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[COPY10]](s32)
   ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[COPY11]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY12]](s32), [[COPY13]](s32)
-  ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.2d), [[COPY8]](s32), [[COPY9]](s32), [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (volatile dereferenceable load store 4 on custom "TargetCustom8")
+  ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.2d), [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (volatile dereferenceable load store 4 on custom "TargetCustom8")
   ; GFX9:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10NSA-LABEL: name: atomic_cmpswap_2d
@@ -1036,10 +1039,11 @@ define amdgpu_ps float @atomic_cmpswap_2d(<8 x i32> inreg %rsrc, i32 %cmp, i32 %
   ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
   ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr3
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
   ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY [[COPY10]](s32)
   ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[COPY11]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY12]](s32), [[COPY13]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.2d), [[COPY8]](s32), [[COPY9]](s32), [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (volatile dereferenceable load store 4 on custom "TargetCustom8")
+  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.2d), [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (volatile dereferenceable load store 4 on custom "TargetCustom8")
   ; GFX10NSA:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
@@ -1066,6 +1070,7 @@ define amdgpu_ps float @atomic_cmpswap_3d(<8 x i32> inreg %rsrc, i32 %cmp, i32 %
   ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr3
   ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr4
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
   ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[COPY10]](s32)
   ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY11]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY13]](s32), [[COPY14]](s32)
@@ -1073,7 +1078,7 @@ define amdgpu_ps float @atomic_cmpswap_3d(<8 x i32> inreg %rsrc, i32 %cmp, i32 %
   ; GFX9:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY15]](s32), [[DEF]](s32)
   ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
-  ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.3d), [[COPY8]](s32), [[COPY9]](s32), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (volatile dereferenceable load store 4 on custom "TargetCustom8")
+  ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.3d), [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (volatile dereferenceable load store 4 on custom "TargetCustom8")
   ; GFX9:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10NSA-LABEL: name: atomic_cmpswap_3d
@@ -1093,13 +1098,14 @@ define amdgpu_ps float @atomic_cmpswap_3d(<8 x i32> inreg %rsrc, i32 %cmp, i32 %
   ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr3
   ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr4
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
   ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY [[COPY10]](s32)
   ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY11]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY13]](s32), [[COPY14]](s32)
   ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY12]](s32)
   ; GFX10NSA:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY15]](s32), [[DEF]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.3d), [[COPY8]](s32), [[COPY9]](s32), [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (volatile dereferenceable load store 4 on custom "TargetCustom8")
+  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.3d), [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (volatile dereferenceable load store 4 on custom "TargetCustom8")
   ; GFX10NSA:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
@@ -1127,6 +1133,7 @@ define amdgpu_ps float @atomic_cmpswap_2darraymsaa(<8 x i32> inreg %rsrc, i32 %c
   ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr4
   ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr5
   ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
   ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY10]](s32)
   ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY11]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
@@ -1134,7 +1141,7 @@ define amdgpu_ps float @atomic_cmpswap_2darraymsaa(<8 x i32> inreg %rsrc, i32 %c
   ; GFX9:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
   ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[COPY17]](s32)
   ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
-  ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.2darraymsaa), [[COPY8]](s32), [[COPY9]](s32), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (volatile dereferenceable load store 4 on custom "TargetCustom8")
+  ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.2darraymsaa), [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (volatile dereferenceable load store 4 on custom "TargetCustom8")
   ; GFX9:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10NSA-LABEL: name: atomic_cmpswap_2darraymsaa
@@ -1155,13 +1162,14 @@ define amdgpu_ps float @atomic_cmpswap_2darraymsaa(<8 x i32> inreg %rsrc, i32 %c
   ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr4
   ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr5
   ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
   ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY10]](s32)
   ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY11]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
   ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY12]](s32)
   ; GFX10NSA:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
   ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[COPY17]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.2darraymsaa), [[COPY8]](s32), [[COPY9]](s32), [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (volatile dereferenceable load store 4 on custom "TargetCustom8")
+  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.2darraymsaa), [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (volatile dereferenceable load store 4 on custom "TargetCustom8")
   ; GFX10NSA:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2d.d16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2d.d16.ll
index d9d83883ed011..cd5a5638211d1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2d.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2d.d16.ll
@@ -547,7 +547,7 @@ define amdgpu_ps <2 x half> @image_load_v2f16_dmask_1000(<8 x i32> inreg %rsrc,
   ; UNPACKED:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; UNPACKED:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 4 from custom "TargetCustom8")
+  ; UNPACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 2 from custom "TargetCustom8")
   ; UNPACKED:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
   ; UNPACKED:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; UNPACKED:   [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY10]], [[C]]
@@ -573,7 +573,7 @@ define amdgpu_ps <2 x half> @image_load_v2f16_dmask_1000(<8 x i32> inreg %rsrc,
   ; PACKED:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; PACKED:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s16>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 4 from custom "TargetCustom8")
+  ; PACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s16>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 2 from custom "TargetCustom8")
   ; PACKED:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s16>)
   ; PACKED:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %tex = call <2 x half> @llvm.amdgcn.image.load.2d.v2f16.i32(i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
@@ -633,7 +633,7 @@ define amdgpu_ps <3 x half> @image_load_v3f16_dmask_1100(<8 x i32> inreg %rsrc,
   ; UNPACKED:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; UNPACKED:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 6 from custom "TargetCustom8", align 8)
+  ; UNPACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 4 from custom "TargetCustom8")
   ; UNPACKED:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; UNPACKED:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
   ; UNPACKED:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
@@ -672,7 +672,7 @@ define amdgpu_ps <3 x half> @image_load_v3f16_dmask_1100(<8 x i32> inreg %rsrc,
   ; PACKED:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; PACKED:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s16>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 6 from custom "TargetCustom8", align 8)
+  ; PACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s16>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 4 from custom "TargetCustom8")
   ; PACKED:   [[DEF:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
   ; PACKED:   [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s16>), [[DEF]](<2 x s16>), [[DEF]](<2 x s16>)
   ; PACKED:   [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<6 x s16>)
@@ -703,7 +703,7 @@ define amdgpu_ps <3 x half> @image_load_v3f16_dmask_1000(<8 x i32> inreg %rsrc,
   ; UNPACKED:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; UNPACKED:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 6 from custom "TargetCustom8", align 8)
+  ; UNPACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 2 from custom "TargetCustom8")
   ; UNPACKED:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
   ; UNPACKED:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; UNPACKED:   [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY10]], [[C]]
@@ -738,7 +738,7 @@ define amdgpu_ps <3 x half> @image_load_v3f16_dmask_1000(<8 x i32> inreg %rsrc,
   ; PACKED:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; PACKED:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s16>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 6 from custom "TargetCustom8", align 8)
+  ; PACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s16>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 2 from custom "TargetCustom8")
   ; PACKED:   [[DEF:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
   ; PACKED:   [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s16>), [[DEF]](<2 x s16>), [[DEF]](<2 x s16>)
   ; PACKED:   [[UV:%[0-9]+]]:_(<3 x s16>), [[UV1:%[0-9]+]]:_(<3 x s16>) = G_UNMERGE_VALUES [[CONCAT_VECTORS]](<6 x s16>)
@@ -818,7 +818,7 @@ define amdgpu_ps <4 x half> @image_load_v4f16_dmask_1110(<8 x i32> inreg %rsrc,
   ; UNPACKED:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; UNPACKED:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 8 from custom "TargetCustom8")
+  ; UNPACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 6 from custom "TargetCustom8", align 8)
   ; UNPACKED:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<3 x s32>)
   ; UNPACKED:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
   ; UNPACKED:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
@@ -853,7 +853,7 @@ define amdgpu_ps <4 x half> @image_load_v4f16_dmask_1110(<8 x i32> inreg %rsrc,
   ; PACKED:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; PACKED:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s16>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 8 from custom "TargetCustom8")
+  ; PACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s16>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 6 from custom "TargetCustom8", align 8)
   ; PACKED:   [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s16>)
   ; PACKED:   $vgpr0 = COPY [[UV]](<2 x s16>)
   ; PACKED:   $vgpr1 = COPY [[UV1]](<2 x s16>)
@@ -878,7 +878,7 @@ define amdgpu_ps <4 x half> @image_load_v4f16_dmask_1100(<8 x i32> inreg %rsrc,
   ; UNPACKED:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; UNPACKED:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 8 from custom "TargetCustom8")
+  ; UNPACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 4 from custom "TargetCustom8")
   ; UNPACKED:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; UNPACKED:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
   ; UNPACKED:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
@@ -911,7 +911,7 @@ define amdgpu_ps <4 x half> @image_load_v4f16_dmask_1100(<8 x i32> inreg %rsrc,
   ; PACKED:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; PACKED:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s16>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 8 from custom "TargetCustom8")
+  ; PACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s16>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 4 from custom "TargetCustom8")
   ; PACKED:   [[DEF:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
   ; PACKED:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s16>)
   ; PACKED:   $vgpr1 = COPY [[DEF]](<2 x s16>)
@@ -936,7 +936,7 @@ define amdgpu_ps <4 x half> @image_load_v4f16_dmask_1000(<8 x i32> inreg %rsrc,
   ; UNPACKED:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; UNPACKED:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 8 from custom "TargetCustom8")
+  ; UNPACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 2 from custom "TargetCustom8")
   ; UNPACKED:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
   ; UNPACKED:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; UNPACKED:   [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY10]], [[C]]
@@ -965,7 +965,7 @@ define amdgpu_ps <4 x half> @image_load_v4f16_dmask_1000(<8 x i32> inreg %rsrc,
   ; PACKED:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; PACKED:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s16>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 8 from custom "TargetCustom8")
+  ; PACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s16>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 2 from custom "TargetCustom8")
   ; PACKED:   [[DEF:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
   ; PACKED:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s16>)
   ; PACKED:   $vgpr1 = COPY [[DEF]](<2 x s16>)
@@ -1084,7 +1084,7 @@ define amdgpu_ps <2 x half> @image_load_tfe_v2f16_dmask_1000(<8 x i32> inreg %rs
   ; UNPACKED:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; UNPACKED:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; UNPACKED:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 4 from custom "TargetCustom8")
+  ; UNPACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 2 from custom "TargetCustom8")
   ; UNPACKED:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; UNPACKED:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
   ; UNPACKED:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
@@ -1113,7 +1113,7 @@ define amdgpu_ps <2 x half> @image_load_tfe_v2f16_dmask_1000(<8 x i32> inreg %rs
   ; PACKED:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; PACKED:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; PACKED:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 4 from custom "TargetCustom8")
+  ; PACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 2 from custom "TargetCustom8")
   ; PACKED:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; PACKED:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[UV]](s32)
   ; PACKED:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
@@ -1143,7 +1143,7 @@ define amdgpu_ps <2 x half> @image_load_tfe_v2f16_dmask_0000(<8 x i32> inreg %rs
   ; UNPACKED:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; UNPACKED:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; UNPACKED:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 4 from custom "TargetCustom8")
+  ; UNPACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 2 from custom "TargetCustom8")
   ; UNPACKED:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; UNPACKED:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
   ; UNPACKED:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
@@ -1172,7 +1172,7 @@ define amdgpu_ps <2 x half> @image_load_tfe_v2f16_dmask_0000(<8 x i32> inreg %rs
   ; PACKED:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; PACKED:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; PACKED:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 4 from custom "TargetCustom8")
+  ; PACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 2 from custom "TargetCustom8")
   ; PACKED:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; PACKED:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[UV]](s32)
   ; PACKED:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
@@ -1202,7 +1202,7 @@ define amdgpu_ps <3 x half> @image_load_tfe_v3f16_dmask_1100(<8 x i32> inreg %rs
   ; UNPACKED:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; UNPACKED:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; UNPACKED:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 6 from custom "TargetCustom8", align 8)
+  ; UNPACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 4 from custom "TargetCustom8")
   ; UNPACKED:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<3 x s32>)
   ; UNPACKED:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
   ; UNPACKED:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
@@ -1243,7 +1243,7 @@ define amdgpu_ps <3 x half> @image_load_tfe_v3f16_dmask_1100(<8 x i32> inreg %rs
   ; PACKED:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; PACKED:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; PACKED:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 6 from custom "TargetCustom8", align 8)
+  ; PACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 4 from custom "TargetCustom8")
   ; PACKED:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; PACKED:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[UV]](s32)
   ; PACKED:   [[DEF1:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
@@ -1281,7 +1281,7 @@ define amdgpu_ps <3 x half> @image_load_tfe_v3f16_dmask_1000(<8 x i32> inreg %rs
   ; UNPACKED:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; UNPACKED:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; UNPACKED:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 6 from custom "TargetCustom8", align 8)
+  ; UNPACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 2 from custom "TargetCustom8")
   ; UNPACKED:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; UNPACKED:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
   ; UNPACKED:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
@@ -1319,7 +1319,7 @@ define amdgpu_ps <3 x half> @image_load_tfe_v3f16_dmask_1000(<8 x i32> inreg %rs
   ; PACKED:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; PACKED:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; PACKED:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 6 from custom "TargetCustom8", align 8)
+  ; PACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 2 from custom "TargetCustom8")
   ; PACKED:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; PACKED:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[UV]](s32)
   ; PACKED:   [[DEF1:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
@@ -1357,7 +1357,7 @@ define amdgpu_ps <3 x half> @image_load_tfe_v3f16_dmask_0000(<8 x i32> inreg %rs
   ; UNPACKED:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; UNPACKED:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; UNPACKED:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 6 from custom "TargetCustom8", align 8)
+  ; UNPACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 2 from custom "TargetCustom8")
   ; UNPACKED:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; UNPACKED:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
   ; UNPACKED:   [[COPY10:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
@@ -1395,7 +1395,7 @@ define amdgpu_ps <3 x half> @image_load_tfe_v3f16_dmask_0000(<8 x i32> inreg %rs
   ; PACKED:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; PACKED:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; PACKED:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 6 from custom "TargetCustom8", align 8)
+  ; PACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 2 from custom "TargetCustom8")
   ; PACKED:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; PACKED:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[UV]](s32)
   ; PACKED:   [[DEF1:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
@@ -1433,7 +1433,7 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_1110(<8 x i32> inreg %rs
   ; UNPACKED:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; UNPACKED:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; UNPACKED:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 8 from custom "TargetCustom8")
+  ; UNPACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 6 from custom "TargetCustom8", align 8)
   ; UNPACKED:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; UNPACKED:   G_STORE [[UV3]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
   ; UNPACKED:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
@@ -1470,7 +1470,7 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_1110(<8 x i32> inreg %rs
   ; PACKED:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; PACKED:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; PACKED:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 8 from custom "TargetCustom8")
+  ; PACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 6 from custom "TargetCustom8", align 8)
   ; PACKED:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<3 x s32>)
   ; PACKED:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[UV]](s32)
   ; PACKED:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[UV1]](s32)
@@ -1502,7 +1502,7 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_1100(<8 x i32> inreg %rs
   ; UNPACKED:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; UNPACKED:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; UNPACKED:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 8 from custom "TargetCustom8")
+  ; UNPACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 4 from custom "TargetCustom8")
   ; UNPACKED:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<3 x s32>)
   ; UNPACKED:   G_STORE [[UV2]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
   ; UNPACKED:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
@@ -1537,7 +1537,7 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_1100(<8 x i32> inreg %rs
   ; PACKED:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; PACKED:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; PACKED:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 8 from custom "TargetCustom8")
+  ; PACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 4 from custom "TargetCustom8")
   ; PACKED:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; PACKED:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[UV]](s32)
   ; PACKED:   [[DEF1:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
@@ -1569,7 +1569,7 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_1000(<8 x i32> inreg %rs
   ; UNPACKED:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; UNPACKED:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; UNPACKED:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 8 from custom "TargetCustom8")
+  ; UNPACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 2 from custom "TargetCustom8")
   ; UNPACKED:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; UNPACKED:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
   ; UNPACKED:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
@@ -1601,7 +1601,7 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_1000(<8 x i32> inreg %rs
   ; PACKED:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; PACKED:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; PACKED:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 8 from custom "TargetCustom8")
+  ; PACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 2 from custom "TargetCustom8")
   ; PACKED:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; PACKED:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[UV]](s32)
   ; PACKED:   [[DEF1:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
@@ -1633,7 +1633,7 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_0000(<8 x i32> inreg %rs
   ; UNPACKED:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; UNPACKED:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; UNPACKED:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 8 from custom "TargetCustom8")
+  ; UNPACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 2 from custom "TargetCustom8")
   ; UNPACKED:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; UNPACKED:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
   ; UNPACKED:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
@@ -1665,7 +1665,7 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_0000(<8 x i32> inreg %rs
   ; PACKED:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; PACKED:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; PACKED:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 8 from custom "TargetCustom8")
+  ; PACKED:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 2 from custom "TargetCustom8")
   ; PACKED:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; PACKED:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[UV]](s32)
   ; PACKED:   [[DEF1:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2d.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2d.ll
index e7a8ca40c02a0..4e904e7faf590 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2d.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2d.ll
@@ -261,7 +261,7 @@ define amdgpu_ps <2 x float> @image_load_v2f32_dmask_1000(<8 x i32> inreg %rsrc,
   ; GCN:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GCN:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GCN:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 8 from custom "TargetCustom8")
+  ; GCN:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 4 from custom "TargetCustom8")
   ; GCN:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GCN:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GCN:   $vgpr1 = COPY [[DEF]](s32)
@@ -309,7 +309,7 @@ define amdgpu_ps <3 x float> @image_load_v3f32_dmask_1100(<8 x i32> inreg %rsrc,
   ; GCN:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GCN:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GCN:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 12 from custom "TargetCustom8", align 16)
+  ; GCN:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 8 from custom "TargetCustom8")
   ; GCN:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; GCN:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GCN:   $vgpr0 = COPY [[UV]](s32)
@@ -336,7 +336,7 @@ define amdgpu_ps <3 x float> @image_load_v3f32_dmask_1000(<8 x i32> inreg %rsrc,
   ; GCN:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GCN:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GCN:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 12 from custom "TargetCustom8", align 16)
+  ; GCN:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 4 from custom "TargetCustom8")
   ; GCN:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GCN:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GCN:   $vgpr1 = COPY [[DEF]](s32)
@@ -386,7 +386,7 @@ define amdgpu_ps <4 x float> @image_load_v4f32_dmask_1110(<8 x i32> inreg %rsrc,
   ; GCN:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GCN:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GCN:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
+  ; GCN:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 12 from custom "TargetCustom8", align 16)
   ; GCN:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<3 x s32>)
   ; GCN:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GCN:   $vgpr0 = COPY [[UV]](s32)
@@ -414,7 +414,7 @@ define amdgpu_ps <4 x float> @image_load_v4f32_dmask_1100(<8 x i32> inreg %rsrc,
   ; GCN:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GCN:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GCN:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
+  ; GCN:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 8 from custom "TargetCustom8")
   ; GCN:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; GCN:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GCN:   $vgpr0 = COPY [[UV]](s32)
@@ -442,7 +442,7 @@ define amdgpu_ps <4 x float> @image_load_v4f32_dmask_1000(<8 x i32> inreg %rsrc,
   ; GCN:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GCN:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GCN:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
+  ; GCN:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 4 from custom "TargetCustom8")
   ; GCN:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GCN:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GCN:   $vgpr1 = COPY [[DEF]](s32)
@@ -524,7 +524,7 @@ define amdgpu_ps <2 x float> @image_load_tfe_v2f32_dmask_1000(<8 x i32> inreg %r
   ; GCN:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; GCN:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 8 from custom "TargetCustom8")
+  ; GCN:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 4 from custom "TargetCustom8")
   ; GCN:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; GCN:   [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GCN:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
@@ -555,7 +555,7 @@ define amdgpu_ps <2 x float> @image_load_tfe_v2f32_dmask_0000(<8 x i32> inreg %r
   ; GCN:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; GCN:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 8 from custom "TargetCustom8")
+  ; GCN:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 4 from custom "TargetCustom8")
   ; GCN:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; GCN:   [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GCN:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
@@ -586,7 +586,7 @@ define amdgpu_ps <3 x float> @image_load_tfe_v3f32_dmask_1100(<8 x i32> inreg %r
   ; GCN:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; GCN:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 12 from custom "TargetCustom8", align 16)
+  ; GCN:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 8 from custom "TargetCustom8")
   ; GCN:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<3 x s32>)
   ; GCN:   [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GCN:   G_STORE [[UV2]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
@@ -618,7 +618,7 @@ define amdgpu_ps <3 x float> @image_load_tfe_v3f32_dmask_1000(<8 x i32> inreg %r
   ; GCN:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; GCN:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 12 from custom "TargetCustom8", align 16)
+  ; GCN:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 4 from custom "TargetCustom8")
   ; GCN:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; GCN:   [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GCN:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
@@ -650,7 +650,7 @@ define amdgpu_ps <3 x float> @image_load_tfe_v3f32_dmask_0000(<8 x i32> inreg %r
   ; GCN:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; GCN:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 12 from custom "TargetCustom8", align 16)
+  ; GCN:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 4 from custom "TargetCustom8")
   ; GCN:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; GCN:   [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GCN:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
@@ -682,7 +682,7 @@ define amdgpu_ps <4 x float> @image_load_tfe_v4f32_dmask_1110(<8 x i32> inreg %r
   ; GCN:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; GCN:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
+  ; GCN:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 12 from custom "TargetCustom8", align 16)
   ; GCN:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GCN:   [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GCN:   G_STORE [[UV3]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
@@ -715,7 +715,7 @@ define amdgpu_ps <4 x float> @image_load_tfe_v4f32_dmask_1100(<8 x i32> inreg %r
   ; GCN:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; GCN:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
+  ; GCN:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 8 from custom "TargetCustom8")
   ; GCN:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<3 x s32>)
   ; GCN:   [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GCN:   G_STORE [[UV2]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
@@ -748,7 +748,7 @@ define amdgpu_ps <4 x float> @image_load_tfe_v4f32_dmask_1000(<8 x i32> inreg %r
   ; GCN:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; GCN:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
+  ; GCN:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 4 from custom "TargetCustom8")
   ; GCN:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; GCN:   [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GCN:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
@@ -781,7 +781,7 @@ define amdgpu_ps <4 x float> @image_load_tfe_v4f32_dmask_0000(<8 x i32> inreg %r
   ; GCN:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GCN:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; GCN:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
+  ; GCN:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0 :: (dereferenceable load 4 from custom "TargetCustom8")
   ; GCN:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; GCN:   [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GCN:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir
index 7e0607dc6ddcc..2fb12235ee31d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer -global-isel-abort=0  %s -o - | FileCheck -check-prefix=SI %s
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=bonaire -O0 -run-pass=legalizer -global-isel-abort=0  %s -o - | FileCheck -check-prefix=CI %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=bonaire -mattr=-enable-ds128 -O0 -run-pass=legalizer -global-isel-abort=0  %s -o - | FileCheck -check-prefix=CI %s
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d  -mcpu=bonaire -mattr=+enable-ds128 -O0 -run-pass=legalizer -global-isel-abort=0  %s -o - | FileCheck -check-prefix=CI-DS128 %s
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer  -global-isel-abort=0 %s -o - | FileCheck -check-prefix=VI %s
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -O0 -run-pass=legalizer  -global-isel-abort=0 %s -o - | FileCheck -check-prefix=GFX9 %s
@@ -1759,24 +1759,12 @@ body: |
     ; CI-DS128: $vgpr0_vgpr1_vgpr2 = COPY [[LOAD]](s96)
     ; VI-LABEL: name: test_load_local_s96_align8
     ; VI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; VI: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p3) :: (load 8, addrspace 3)
-    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; VI: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; VI: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p3) :: (load 4, align 8, addrspace 3)
-    ; VI: [[DEF:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF
-    ; VI: [[INSERT:%[0-9]+]]:_(s96) = G_INSERT [[DEF]], [[LOAD]](s64), 0
-    ; VI: [[INSERT1:%[0-9]+]]:_(s96) = G_INSERT [[INSERT]], [[LOAD1]](s32), 64
-    ; VI: $vgpr0_vgpr1_vgpr2 = COPY [[INSERT1]](s96)
+    ; VI: [[LOAD:%[0-9]+]]:_(s96) = G_LOAD [[COPY]](p3) :: (load 12, align 8, addrspace 3)
+    ; VI: $vgpr0_vgpr1_vgpr2 = COPY [[LOAD]](s96)
     ; GFX9-LABEL: name: test_load_local_s96_align8
     ; GFX9: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; GFX9: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p3) :: (load 8, addrspace 3)
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p3) :: (load 4, align 8, addrspace 3)
-    ; GFX9: [[DEF:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF
-    ; GFX9: [[INSERT:%[0-9]+]]:_(s96) = G_INSERT [[DEF]], [[LOAD]](s64), 0
-    ; GFX9: [[INSERT1:%[0-9]+]]:_(s96) = G_INSERT [[INSERT]], [[LOAD1]](s32), 64
-    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[INSERT1]](s96)
+    ; GFX9: [[LOAD:%[0-9]+]]:_(s96) = G_LOAD [[COPY]](p3) :: (load 12, align 8, addrspace 3)
+    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[LOAD]](s96)
     %0:_(p3) = COPY $vgpr0
     %1:_(s96) = G_LOAD %0 :: (load 12, align 8, addrspace 3)
     $vgpr0_vgpr1_vgpr2 = COPY %1
@@ -1814,24 +1802,12 @@ body: |
     ; CI-DS128: $vgpr0_vgpr1_vgpr2 = COPY [[LOAD]](s96)
     ; VI-LABEL: name: test_load_local_s96_align4
     ; VI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; VI: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p3) :: (load 8, align 4, addrspace 3)
-    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; VI: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; VI: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p3) :: (load 4, addrspace 3)
-    ; VI: [[DEF:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF
-    ; VI: [[INSERT:%[0-9]+]]:_(s96) = G_INSERT [[DEF]], [[LOAD]](s64), 0
-    ; VI: [[INSERT1:%[0-9]+]]:_(s96) = G_INSERT [[INSERT]], [[LOAD1]](s32), 64
-    ; VI: $vgpr0_vgpr1_vgpr2 = COPY [[INSERT1]](s96)
+    ; VI: [[LOAD:%[0-9]+]]:_(s96) = G_LOAD [[COPY]](p3) :: (load 12, align 4, addrspace 3)
+    ; VI: $vgpr0_vgpr1_vgpr2 = COPY [[LOAD]](s96)
     ; GFX9-LABEL: name: test_load_local_s96_align4
     ; GFX9: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; GFX9: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p3) :: (load 8, align 4, addrspace 3)
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p3) :: (load 4, addrspace 3)
-    ; GFX9: [[DEF:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF
-    ; GFX9: [[INSERT:%[0-9]+]]:_(s96) = G_INSERT [[DEF]], [[LOAD]](s64), 0
-    ; GFX9: [[INSERT1:%[0-9]+]]:_(s96) = G_INSERT [[INSERT]], [[LOAD1]](s32), 64
-    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[INSERT1]](s96)
+    ; GFX9: [[LOAD:%[0-9]+]]:_(s96) = G_LOAD [[COPY]](p3) :: (load 12, align 4, addrspace 3)
+    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[LOAD]](s96)
     %0:_(p3) = COPY $vgpr0
     %1:_(s96) = G_LOAD %0 :: (load 12, align 4, addrspace 3)
     $vgpr0_vgpr1_vgpr2 = COPY %1
@@ -2969,94 +2945,99 @@ body: |
     ; VI: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
     ; VI: [[PTR_ADD6:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C6]](s32)
     ; VI: [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p3) :: (load 1, addrspace 3)
-    ; VI: [[C7:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; VI: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; VI: [[PTR_ADD7:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C7]](s32)
+    ; VI: [[LOAD8:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD7]](p3) :: (load 1, addrspace 3)
+    ; VI: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 9
+    ; VI: [[PTR_ADD8:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C8]](s32)
+    ; VI: [[LOAD9:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD8]](p3) :: (load 1, addrspace 3)
+    ; VI: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+    ; VI: [[PTR_ADD9:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C9]](s32)
+    ; VI: [[LOAD10:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD9]](p3) :: (load 1, addrspace 3)
+    ; VI: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 11
+    ; VI: [[PTR_ADD10:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C10]](s32)
+    ; VI: [[LOAD11:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p3) :: (load 1, addrspace 3)
+    ; VI: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; VI: [[PTR_ADD11:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C11]](s32)
+    ; VI: [[LOAD12:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD11]](p3) :: (load 1, addrspace 3)
+    ; VI: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 13
+    ; VI: [[PTR_ADD12:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C12]](s32)
+    ; VI: [[LOAD13:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD12]](p3) :: (load 1, addrspace 3)
+    ; VI: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 14
+    ; VI: [[PTR_ADD13:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C13]](s32)
+    ; VI: [[LOAD14:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD13]](p3) :: (load 1, addrspace 3)
+    ; VI: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
+    ; VI: [[PTR_ADD14:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C14]](s32)
+    ; VI: [[LOAD15:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p3) :: (load 1, addrspace 3)
+    ; VI: [[C15:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
     ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
-    ; VI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C7]]
+    ; VI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C15]]
     ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
-    ; VI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C7]]
-    ; VI: [[C8:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C8]](s16)
+    ; VI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C15]]
+    ; VI: [[C16:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C16]](s16)
     ; VI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
     ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
-    ; VI: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C7]]
+    ; VI: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C15]]
     ; VI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD3]](s32)
-    ; VI: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C7]]
-    ; VI: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C8]](s16)
+    ; VI: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C15]]
+    ; VI: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C16]](s16)
     ; VI: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
     ; VI: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD4]](s32)
-    ; VI: [[AND4:%[0-9]+]]:_(s16) = G_AND [[TRUNC4]], [[C7]]
+    ; VI: [[AND4:%[0-9]+]]:_(s16) = G_AND [[TRUNC4]], [[C15]]
     ; VI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD5]](s32)
-    ; VI: [[AND5:%[0-9]+]]:_(s16) = G_AND [[TRUNC5]], [[C7]]
-    ; VI: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND5]], [[C8]](s16)
+    ; VI: [[AND5:%[0-9]+]]:_(s16) = G_AND [[TRUNC5]], [[C15]]
+    ; VI: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND5]], [[C16]](s16)
     ; VI: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND4]], [[SHL2]]
     ; VI: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD6]](s32)
-    ; VI: [[AND6:%[0-9]+]]:_(s16) = G_AND [[TRUNC6]], [[C7]]
+    ; VI: [[AND6:%[0-9]+]]:_(s16) = G_AND [[TRUNC6]], [[C15]]
     ; VI: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD7]](s32)
-    ; VI: [[AND7:%[0-9]+]]:_(s16) = G_AND [[TRUNC7]], [[C7]]
-    ; VI: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND7]], [[C8]](s16)
+    ; VI: [[AND7:%[0-9]+]]:_(s16) = G_AND [[TRUNC7]], [[C15]]
+    ; VI: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND7]], [[C16]](s16)
     ; VI: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND6]], [[SHL3]]
-    ; VI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
-    ; VI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
-    ; VI: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; VI: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C9]](s32)
-    ; VI: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
-    ; VI: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
-    ; VI: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
-    ; VI: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C9]](s32)
-    ; VI: [[OR5:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL5]]
-    ; VI: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR4]](s32), [[OR5]](s32)
-    ; VI: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; VI: [[PTR_ADD7:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C10]](s32)
-    ; VI: [[LOAD8:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD7]](p3) :: (load 1, addrspace 3)
-    ; VI: [[PTR_ADD8:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
-    ; VI: [[LOAD9:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD8]](p3) :: (load 1, addrspace 3)
-    ; VI: [[PTR_ADD9:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C1]](s32)
-    ; VI: [[LOAD10:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD9]](p3) :: (load 1, addrspace 3)
-    ; VI: [[PTR_ADD10:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
-    ; VI: [[LOAD11:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p3) :: (load 1, addrspace 3)
-    ; VI: [[PTR_ADD11:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C3]](s32)
-    ; VI: [[LOAD12:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD11]](p3) :: (load 1, addrspace 3)
-    ; VI: [[PTR_ADD12:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C4]](s32)
-    ; VI: [[LOAD13:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD12]](p3) :: (load 1, addrspace 3)
-    ; VI: [[PTR_ADD13:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C5]](s32)
-    ; VI: [[LOAD14:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD13]](p3) :: (load 1, addrspace 3)
-    ; VI: [[PTR_ADD14:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C6]](s32)
-    ; VI: [[LOAD15:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p3) :: (load 1, addrspace 3)
     ; VI: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD8]](s32)
-    ; VI: [[AND8:%[0-9]+]]:_(s16) = G_AND [[TRUNC8]], [[C7]]
+    ; VI: [[AND8:%[0-9]+]]:_(s16) = G_AND [[TRUNC8]], [[C15]]
     ; VI: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD9]](s32)
-    ; VI: [[AND9:%[0-9]+]]:_(s16) = G_AND [[TRUNC9]], [[C7]]
-    ; VI: [[SHL6:%[0-9]+]]:_(s16) = G_SHL [[AND9]], [[C8]](s16)
-    ; VI: [[OR6:%[0-9]+]]:_(s16) = G_OR [[AND8]], [[SHL6]]
+    ; VI: [[AND9:%[0-9]+]]:_(s16) = G_AND [[TRUNC9]], [[C15]]
+    ; VI: [[SHL4:%[0-9]+]]:_(s16) = G_SHL [[AND9]], [[C16]](s16)
+    ; VI: [[OR4:%[0-9]+]]:_(s16) = G_OR [[AND8]], [[SHL4]]
     ; VI: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD10]](s32)
-    ; VI: [[AND10:%[0-9]+]]:_(s16) = G_AND [[TRUNC10]], [[C7]]
+    ; VI: [[AND10:%[0-9]+]]:_(s16) = G_AND [[TRUNC10]], [[C15]]
     ; VI: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD11]](s32)
-    ; VI: [[AND11:%[0-9]+]]:_(s16) = G_AND [[TRUNC11]], [[C7]]
-    ; VI: [[SHL7:%[0-9]+]]:_(s16) = G_SHL [[AND11]], [[C8]](s16)
-    ; VI: [[OR7:%[0-9]+]]:_(s16) = G_OR [[AND10]], [[SHL7]]
+    ; VI: [[AND11:%[0-9]+]]:_(s16) = G_AND [[TRUNC11]], [[C15]]
+    ; VI: [[SHL5:%[0-9]+]]:_(s16) = G_SHL [[AND11]], [[C16]](s16)
+    ; VI: [[OR5:%[0-9]+]]:_(s16) = G_OR [[AND10]], [[SHL5]]
     ; VI: [[TRUNC12:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD12]](s32)
-    ; VI: [[AND12:%[0-9]+]]:_(s16) = G_AND [[TRUNC12]], [[C7]]
+    ; VI: [[AND12:%[0-9]+]]:_(s16) = G_AND [[TRUNC12]], [[C15]]
     ; VI: [[TRUNC13:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD13]](s32)
-    ; VI: [[AND13:%[0-9]+]]:_(s16) = G_AND [[TRUNC13]], [[C7]]
-    ; VI: [[SHL8:%[0-9]+]]:_(s16) = G_SHL [[AND13]], [[C8]](s16)
-    ; VI: [[OR8:%[0-9]+]]:_(s16) = G_OR [[AND12]], [[SHL8]]
+    ; VI: [[AND13:%[0-9]+]]:_(s16) = G_AND [[TRUNC13]], [[C15]]
+    ; VI: [[SHL6:%[0-9]+]]:_(s16) = G_SHL [[AND13]], [[C16]](s16)
+    ; VI: [[OR6:%[0-9]+]]:_(s16) = G_OR [[AND12]], [[SHL6]]
     ; VI: [[TRUNC14:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD14]](s32)
-    ; VI: [[AND14:%[0-9]+]]:_(s16) = G_AND [[TRUNC14]], [[C7]]
+    ; VI: [[AND14:%[0-9]+]]:_(s16) = G_AND [[TRUNC14]], [[C15]]
     ; VI: [[TRUNC15:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD15]](s32)
-    ; VI: [[AND15:%[0-9]+]]:_(s16) = G_AND [[TRUNC15]], [[C7]]
-    ; VI: [[SHL9:%[0-9]+]]:_(s16) = G_SHL [[AND15]], [[C8]](s16)
-    ; VI: [[OR9:%[0-9]+]]:_(s16) = G_OR [[AND14]], [[SHL9]]
-    ; VI: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[OR6]](s16)
-    ; VI: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[OR7]](s16)
-    ; VI: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C9]](s32)
+    ; VI: [[AND15:%[0-9]+]]:_(s16) = G_AND [[TRUNC15]], [[C15]]
+    ; VI: [[SHL7:%[0-9]+]]:_(s16) = G_SHL [[AND15]], [[C16]](s16)
+    ; VI: [[OR7:%[0-9]+]]:_(s16) = G_OR [[AND14]], [[SHL7]]
+    ; VI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
+    ; VI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
+    ; VI: [[C17:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; VI: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C17]](s32)
+    ; VI: [[OR8:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL8]]
+    ; VI: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
+    ; VI: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
+    ; VI: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C17]](s32)
+    ; VI: [[OR9:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL9]]
+    ; VI: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[OR4]](s16)
+    ; VI: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[OR5]](s16)
+    ; VI: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C17]](s32)
     ; VI: [[OR10:%[0-9]+]]:_(s32) = G_OR [[ZEXT4]], [[SHL10]]
-    ; VI: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[OR8]](s16)
-    ; VI: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[OR9]](s16)
-    ; VI: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[ZEXT7]], [[C9]](s32)
+    ; VI: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[OR6]](s16)
+    ; VI: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[OR7]](s16)
+    ; VI: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[ZEXT7]], [[C17]](s32)
     ; VI: [[OR11:%[0-9]+]]:_(s32) = G_OR [[ZEXT6]], [[SHL11]]
-    ; VI: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR10]](s32), [[OR11]](s32)
-    ; VI: [[MV2:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[MV]](s64), [[MV1]](s64)
-    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV2]](s128)
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[OR8]](s32), [[OR9]](s32), [[OR10]](s32), [[OR11]](s32)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
     ; GFX9-LABEL: name: test_load_local_s128_align16
     ; GFX9: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
     ; GFX9: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p3) :: (load 1, addrspace 3)
@@ -3081,94 +3062,99 @@ body: |
     ; GFX9: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
     ; GFX9: [[PTR_ADD6:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C6]](s32)
     ; GFX9: [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p3) :: (load 1, addrspace 3)
-    ; GFX9: [[C7:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX9: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9: [[PTR_ADD7:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C7]](s32)
+    ; GFX9: [[LOAD8:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD7]](p3) :: (load 1, addrspace 3)
+    ; GFX9: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 9
+    ; GFX9: [[PTR_ADD8:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C8]](s32)
+    ; GFX9: [[LOAD9:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD8]](p3) :: (load 1, addrspace 3)
+    ; GFX9: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+    ; GFX9: [[PTR_ADD9:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C9]](s32)
+    ; GFX9: [[LOAD10:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD9]](p3) :: (load 1, addrspace 3)
+    ; GFX9: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 11
+    ; GFX9: [[PTR_ADD10:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C10]](s32)
+    ; GFX9: [[LOAD11:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p3) :: (load 1, addrspace 3)
+    ; GFX9: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; GFX9: [[PTR_ADD11:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C11]](s32)
+    ; GFX9: [[LOAD12:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD11]](p3) :: (load 1, addrspace 3)
+    ; GFX9: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 13
+    ; GFX9: [[PTR_ADD12:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C12]](s32)
+    ; GFX9: [[LOAD13:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD12]](p3) :: (load 1, addrspace 3)
+    ; GFX9: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 14
+    ; GFX9: [[PTR_ADD13:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C13]](s32)
+    ; GFX9: [[LOAD14:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD13]](p3) :: (load 1, addrspace 3)
+    ; GFX9: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
+    ; GFX9: [[PTR_ADD14:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C14]](s32)
+    ; GFX9: [[LOAD15:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p3) :: (load 1, addrspace 3)
+    ; GFX9: [[C15:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
     ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
-    ; GFX9: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C7]]
+    ; GFX9: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C15]]
     ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
-    ; GFX9: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C7]]
-    ; GFX9: [[C8:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C8]](s16)
+    ; GFX9: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C15]]
+    ; GFX9: [[C16:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C16]](s16)
     ; GFX9: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
     ; GFX9: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
-    ; GFX9: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C7]]
+    ; GFX9: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C15]]
     ; GFX9: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD3]](s32)
-    ; GFX9: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C7]]
-    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C8]](s16)
+    ; GFX9: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C15]]
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C16]](s16)
     ; GFX9: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
     ; GFX9: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD4]](s32)
-    ; GFX9: [[AND4:%[0-9]+]]:_(s16) = G_AND [[TRUNC4]], [[C7]]
+    ; GFX9: [[AND4:%[0-9]+]]:_(s16) = G_AND [[TRUNC4]], [[C15]]
     ; GFX9: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD5]](s32)
-    ; GFX9: [[AND5:%[0-9]+]]:_(s16) = G_AND [[TRUNC5]], [[C7]]
-    ; GFX9: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND5]], [[C8]](s16)
+    ; GFX9: [[AND5:%[0-9]+]]:_(s16) = G_AND [[TRUNC5]], [[C15]]
+    ; GFX9: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND5]], [[C16]](s16)
     ; GFX9: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND4]], [[SHL2]]
     ; GFX9: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD6]](s32)
-    ; GFX9: [[AND6:%[0-9]+]]:_(s16) = G_AND [[TRUNC6]], [[C7]]
+    ; GFX9: [[AND6:%[0-9]+]]:_(s16) = G_AND [[TRUNC6]], [[C15]]
     ; GFX9: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD7]](s32)
-    ; GFX9: [[AND7:%[0-9]+]]:_(s16) = G_AND [[TRUNC7]], [[C7]]
-    ; GFX9: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND7]], [[C8]](s16)
+    ; GFX9: [[AND7:%[0-9]+]]:_(s16) = G_AND [[TRUNC7]], [[C15]]
+    ; GFX9: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND7]], [[C16]](s16)
     ; GFX9: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND6]], [[SHL3]]
-    ; GFX9: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
-    ; GFX9: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
-    ; GFX9: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C9]](s32)
-    ; GFX9: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
-    ; GFX9: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
-    ; GFX9: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
-    ; GFX9: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C9]](s32)
-    ; GFX9: [[OR5:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL5]]
-    ; GFX9: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR4]](s32), [[OR5]](s32)
-    ; GFX9: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9: [[PTR_ADD7:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C10]](s32)
-    ; GFX9: [[LOAD8:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD7]](p3) :: (load 1, addrspace 3)
-    ; GFX9: [[PTR_ADD8:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
-    ; GFX9: [[LOAD9:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD8]](p3) :: (load 1, addrspace 3)
-    ; GFX9: [[PTR_ADD9:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C1]](s32)
-    ; GFX9: [[LOAD10:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD9]](p3) :: (load 1, addrspace 3)
-    ; GFX9: [[PTR_ADD10:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
-    ; GFX9: [[LOAD11:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p3) :: (load 1, addrspace 3)
-    ; GFX9: [[PTR_ADD11:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C3]](s32)
-    ; GFX9: [[LOAD12:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD11]](p3) :: (load 1, addrspace 3)
-    ; GFX9: [[PTR_ADD12:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C4]](s32)
-    ; GFX9: [[LOAD13:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD12]](p3) :: (load 1, addrspace 3)
-    ; GFX9: [[PTR_ADD13:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C5]](s32)
-    ; GFX9: [[LOAD14:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD13]](p3) :: (load 1, addrspace 3)
-    ; GFX9: [[PTR_ADD14:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C6]](s32)
-    ; GFX9: [[LOAD15:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p3) :: (load 1, addrspace 3)
     ; GFX9: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD8]](s32)
-    ; GFX9: [[AND8:%[0-9]+]]:_(s16) = G_AND [[TRUNC8]], [[C7]]
+    ; GFX9: [[AND8:%[0-9]+]]:_(s16) = G_AND [[TRUNC8]], [[C15]]
     ; GFX9: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD9]](s32)
-    ; GFX9: [[AND9:%[0-9]+]]:_(s16) = G_AND [[TRUNC9]], [[C7]]
-    ; GFX9: [[SHL6:%[0-9]+]]:_(s16) = G_SHL [[AND9]], [[C8]](s16)
-    ; GFX9: [[OR6:%[0-9]+]]:_(s16) = G_OR [[AND8]], [[SHL6]]
+    ; GFX9: [[AND9:%[0-9]+]]:_(s16) = G_AND [[TRUNC9]], [[C15]]
+    ; GFX9: [[SHL4:%[0-9]+]]:_(s16) = G_SHL [[AND9]], [[C16]](s16)
+    ; GFX9: [[OR4:%[0-9]+]]:_(s16) = G_OR [[AND8]], [[SHL4]]
     ; GFX9: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD10]](s32)
-    ; GFX9: [[AND10:%[0-9]+]]:_(s16) = G_AND [[TRUNC10]], [[C7]]
+    ; GFX9: [[AND10:%[0-9]+]]:_(s16) = G_AND [[TRUNC10]], [[C15]]
     ; GFX9: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD11]](s32)
-    ; GFX9: [[AND11:%[0-9]+]]:_(s16) = G_AND [[TRUNC11]], [[C7]]
-    ; GFX9: [[SHL7:%[0-9]+]]:_(s16) = G_SHL [[AND11]], [[C8]](s16)
-    ; GFX9: [[OR7:%[0-9]+]]:_(s16) = G_OR [[AND10]], [[SHL7]]
+    ; GFX9: [[AND11:%[0-9]+]]:_(s16) = G_AND [[TRUNC11]], [[C15]]
+    ; GFX9: [[SHL5:%[0-9]+]]:_(s16) = G_SHL [[AND11]], [[C16]](s16)
+    ; GFX9: [[OR5:%[0-9]+]]:_(s16) = G_OR [[AND10]], [[SHL5]]
     ; GFX9: [[TRUNC12:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD12]](s32)
-    ; GFX9: [[AND12:%[0-9]+]]:_(s16) = G_AND [[TRUNC12]], [[C7]]
+    ; GFX9: [[AND12:%[0-9]+]]:_(s16) = G_AND [[TRUNC12]], [[C15]]
     ; GFX9: [[TRUNC13:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD13]](s32)
-    ; GFX9: [[AND13:%[0-9]+]]:_(s16) = G_AND [[TRUNC13]], [[C7]]
-    ; GFX9: [[SHL8:%[0-9]+]]:_(s16) = G_SHL [[AND13]], [[C8]](s16)
-    ; GFX9: [[OR8:%[0-9]+]]:_(s16) = G_OR [[AND12]], [[SHL8]]
+    ; GFX9: [[AND13:%[0-9]+]]:_(s16) = G_AND [[TRUNC13]], [[C15]]
+    ; GFX9: [[SHL6:%[0-9]+]]:_(s16) = G_SHL [[AND13]], [[C16]](s16)
+    ; GFX9: [[OR6:%[0-9]+]]:_(s16) = G_OR [[AND12]], [[SHL6]]
     ; GFX9: [[TRUNC14:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD14]](s32)
-    ; GFX9: [[AND14:%[0-9]+]]:_(s16) = G_AND [[TRUNC14]], [[C7]]
+    ; GFX9: [[AND14:%[0-9]+]]:_(s16) = G_AND [[TRUNC14]], [[C15]]
     ; GFX9: [[TRUNC15:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD15]](s32)
-    ; GFX9: [[AND15:%[0-9]+]]:_(s16) = G_AND [[TRUNC15]], [[C7]]
-    ; GFX9: [[SHL9:%[0-9]+]]:_(s16) = G_SHL [[AND15]], [[C8]](s16)
-    ; GFX9: [[OR9:%[0-9]+]]:_(s16) = G_OR [[AND14]], [[SHL9]]
-    ; GFX9: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[OR6]](s16)
-    ; GFX9: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[OR7]](s16)
-    ; GFX9: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C9]](s32)
+    ; GFX9: [[AND15:%[0-9]+]]:_(s16) = G_AND [[TRUNC15]], [[C15]]
+    ; GFX9: [[SHL7:%[0-9]+]]:_(s16) = G_SHL [[AND15]], [[C16]](s16)
+    ; GFX9: [[OR7:%[0-9]+]]:_(s16) = G_OR [[AND14]], [[SHL7]]
+    ; GFX9: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
+    ; GFX9: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
+    ; GFX9: [[C17:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C17]](s32)
+    ; GFX9: [[OR8:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL8]]
+    ; GFX9: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
+    ; GFX9: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
+    ; GFX9: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C17]](s32)
+    ; GFX9: [[OR9:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL9]]
+    ; GFX9: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[OR4]](s16)
+    ; GFX9: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[OR5]](s16)
+    ; GFX9: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C17]](s32)
     ; GFX9: [[OR10:%[0-9]+]]:_(s32) = G_OR [[ZEXT4]], [[SHL10]]
-    ; GFX9: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[OR8]](s16)
-    ; GFX9: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[OR9]](s16)
-    ; GFX9: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[ZEXT7]], [[C9]](s32)
+    ; GFX9: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[OR6]](s16)
+    ; GFX9: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[OR7]](s16)
+    ; GFX9: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[ZEXT7]], [[C17]](s32)
     ; GFX9: [[OR11:%[0-9]+]]:_(s32) = G_OR [[ZEXT6]], [[SHL11]]
-    ; GFX9: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR10]](s32), [[OR11]](s32)
-    ; GFX9: [[MV2:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[MV]](s64), [[MV1]](s64)
-    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV2]](s128)
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[OR8]](s32), [[OR9]](s32), [[OR10]](s32), [[OR11]](s32)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
     %0:_(p3) = COPY $vgpr0
     %1:_(s128) = G_LOAD %0 :: (load 16, align 1, addrspace 3)
     $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
@@ -3202,20 +3188,12 @@ body: |
     ; CI-DS128: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](s128)
     ; VI-LABEL: name: test_load_local_s128_align8
     ; VI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; VI: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p3) :: (load 8, addrspace 3)
-    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; VI: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; VI: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD]](p3) :: (load 8, addrspace 3)
-    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[LOAD]](s64), [[LOAD1]](s64)
-    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; VI: [[LOAD:%[0-9]+]]:_(s128) = G_LOAD [[COPY]](p3) :: (load 16, align 8, addrspace 3)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](s128)
     ; GFX9-LABEL: name: test_load_local_s128_align8
     ; GFX9: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; GFX9: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p3) :: (load 8, addrspace 3)
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD]](p3) :: (load 8, addrspace 3)
-    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[LOAD]](s64), [[LOAD1]](s64)
-    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; GFX9: [[LOAD:%[0-9]+]]:_(s128) = G_LOAD [[COPY]](p3) :: (load 16, align 8, addrspace 3)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](s128)
     %0:_(p3) = COPY $vgpr0
     %1:_(s128) = G_LOAD %0 :: (load 16, align 8, addrspace 3)
     $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
@@ -3249,20 +3227,12 @@ body: |
     ; CI-DS128: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](s128)
     ; VI-LABEL: name: test_load_local_s128_align4
     ; VI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; VI: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p3) :: (load 8, align 4, addrspace 3)
-    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; VI: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; VI: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD]](p3) :: (load 8, align 4, addrspace 3)
-    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[LOAD]](s64), [[LOAD1]](s64)
-    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; VI: [[LOAD:%[0-9]+]]:_(s128) = G_LOAD [[COPY]](p3) :: (load 16, align 4, addrspace 3)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](s128)
     ; GFX9-LABEL: name: test_load_local_s128_align4
     ; GFX9: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; GFX9: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p3) :: (load 8, align 4, addrspace 3)
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD]](p3) :: (load 8, align 4, addrspace 3)
-    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[LOAD]](s64), [[LOAD1]](s64)
-    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; GFX9: [[LOAD:%[0-9]+]]:_(s128) = G_LOAD [[COPY]](p3) :: (load 16, align 4, addrspace 3)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](s128)
     %0:_(p3) = COPY $vgpr0
     %1:_(s128) = G_LOAD %0 :: (load 16, align 4, addrspace 3)
     $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
@@ -3440,45 +3410,46 @@ body: |
     ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
     ; VI: [[PTR_ADD2:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C2]](s32)
     ; VI: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p3) :: (load 2, addrspace 3)
-    ; VI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; VI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; VI: [[PTR_ADD3:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C3]](s32)
+    ; VI: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p3) :: (load 2, addrspace 3)
+    ; VI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+    ; VI: [[PTR_ADD4:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; VI: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p3) :: (load 2, addrspace 3)
+    ; VI: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; VI: [[PTR_ADD5:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C5]](s32)
+    ; VI: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD5]](p3) :: (load 2, addrspace 3)
+    ; VI: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 14
+    ; VI: [[PTR_ADD6:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C6]](s32)
+    ; VI: [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p3) :: (load 2, addrspace 3)
+    ; VI: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
-    ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C3]]
+    ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C7]]
     ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LOAD1]](s32)
-    ; VI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C3]]
-    ; VI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; VI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+    ; VI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C7]]
+    ; VI: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; VI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C8]](s32)
     ; VI: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
     ; VI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LOAD2]](s32)
-    ; VI: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C3]]
+    ; VI: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C7]]
     ; VI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LOAD3]](s32)
-    ; VI: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C3]]
-    ; VI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C4]](s32)
+    ; VI: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C7]]
+    ; VI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C8]](s32)
     ; VI: [[OR1:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[SHL1]]
-    ; VI: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR]](s32), [[OR1]](s32)
-    ; VI: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; VI: [[PTR_ADD3:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C5]](s32)
-    ; VI: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p3) :: (load 2, addrspace 3)
-    ; VI: [[PTR_ADD4:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; VI: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p3) :: (load 2, addrspace 3)
-    ; VI: [[PTR_ADD5:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD3]], [[C1]](s32)
-    ; VI: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD5]](p3) :: (load 2, addrspace 3)
-    ; VI: [[PTR_ADD6:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
-    ; VI: [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p3) :: (load 2, addrspace 3)
     ; VI: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LOAD4]](s32)
-    ; VI: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C3]]
+    ; VI: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C7]]
     ; VI: [[COPY6:%[0-9]+]]:_(s32) = COPY [[LOAD5]](s32)
-    ; VI: [[AND5:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C3]]
-    ; VI: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C4]](s32)
+    ; VI: [[AND5:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C7]]
+    ; VI: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C8]](s32)
     ; VI: [[OR2:%[0-9]+]]:_(s32) = G_OR [[AND4]], [[SHL2]]
     ; VI: [[COPY7:%[0-9]+]]:_(s32) = COPY [[LOAD6]](s32)
-    ; VI: [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C3]]
+    ; VI: [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C7]]
     ; VI: [[COPY8:%[0-9]+]]:_(s32) = COPY [[LOAD7]](s32)
-    ; VI: [[AND7:%[0-9]+]]:_(s32) = G_AND [[COPY8]], [[C3]]
-    ; VI: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C4]](s32)
+    ; VI: [[AND7:%[0-9]+]]:_(s32) = G_AND [[COPY8]], [[C7]]
+    ; VI: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C8]](s32)
     ; VI: [[OR3:%[0-9]+]]:_(s32) = G_OR [[AND6]], [[SHL3]]
-    ; VI: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](s32), [[OR3]](s32)
-    ; VI: [[MV2:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[MV]](s64), [[MV1]](s64)
-    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV2]](s128)
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[OR]](s32), [[OR1]](s32), [[OR2]](s32), [[OR3]](s32)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
     ; GFX9-LABEL: name: test_load_local_s128_align2
     ; GFX9: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
     ; GFX9: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p3) :: (load 2, addrspace 3)
@@ -3491,45 +3462,46 @@ body: |
     ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
     ; GFX9: [[PTR_ADD2:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C2]](s32)
     ; GFX9: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p3) :: (load 2, addrspace 3)
-    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9: [[PTR_ADD3:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C3]](s32)
+    ; GFX9: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p3) :: (load 2, addrspace 3)
+    ; GFX9: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+    ; GFX9: [[PTR_ADD4:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C4]](s32)
+    ; GFX9: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p3) :: (load 2, addrspace 3)
+    ; GFX9: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; GFX9: [[PTR_ADD5:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C5]](s32)
+    ; GFX9: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD5]](p3) :: (load 2, addrspace 3)
+    ; GFX9: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 14
+    ; GFX9: [[PTR_ADD6:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C6]](s32)
+    ; GFX9: [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p3) :: (load 2, addrspace 3)
+    ; GFX9: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
-    ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C3]]
+    ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C7]]
     ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LOAD1]](s32)
-    ; GFX9: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C3]]
-    ; GFX9: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+    ; GFX9: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C7]]
+    ; GFX9: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C8]](s32)
     ; GFX9: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
     ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LOAD2]](s32)
-    ; GFX9: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C3]]
+    ; GFX9: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C7]]
     ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LOAD3]](s32)
-    ; GFX9: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C3]]
-    ; GFX9: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C4]](s32)
+    ; GFX9: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C7]]
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C8]](s32)
     ; GFX9: [[OR1:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[SHL1]]
-    ; GFX9: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR]](s32), [[OR1]](s32)
-    ; GFX9: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9: [[PTR_ADD3:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C5]](s32)
-    ; GFX9: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p3) :: (load 2, addrspace 3)
-    ; GFX9: [[PTR_ADD4:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32)
-    ; GFX9: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p3) :: (load 2, addrspace 3)
-    ; GFX9: [[PTR_ADD5:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD3]], [[C1]](s32)
-    ; GFX9: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD5]](p3) :: (load 2, addrspace 3)
-    ; GFX9: [[PTR_ADD6:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD3]], [[C2]](s32)
-    ; GFX9: [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p3) :: (load 2, addrspace 3)
     ; GFX9: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LOAD4]](s32)
-    ; GFX9: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C3]]
+    ; GFX9: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C7]]
     ; GFX9: [[COPY6:%[0-9]+]]:_(s32) = COPY [[LOAD5]](s32)
-    ; GFX9: [[AND5:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C3]]
-    ; GFX9: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C4]](s32)
+    ; GFX9: [[AND5:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C7]]
+    ; GFX9: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C8]](s32)
     ; GFX9: [[OR2:%[0-9]+]]:_(s32) = G_OR [[AND4]], [[SHL2]]
     ; GFX9: [[COPY7:%[0-9]+]]:_(s32) = COPY [[LOAD6]](s32)
-    ; GFX9: [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C3]]
+    ; GFX9: [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C7]]
     ; GFX9: [[COPY8:%[0-9]+]]:_(s32) = COPY [[LOAD7]](s32)
-    ; GFX9: [[AND7:%[0-9]+]]:_(s32) = G_AND [[COPY8]], [[C3]]
-    ; GFX9: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C4]](s32)
+    ; GFX9: [[AND7:%[0-9]+]]:_(s32) = G_AND [[COPY8]], [[C7]]
+    ; GFX9: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C8]](s32)
     ; GFX9: [[OR3:%[0-9]+]]:_(s32) = G_OR [[AND6]], [[SHL3]]
-    ; GFX9: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](s32), [[OR3]](s32)
-    ; GFX9: [[MV2:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[MV]](s64), [[MV1]](s64)
-    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV2]](s128)
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[OR]](s32), [[OR1]](s32), [[OR2]](s32), [[OR3]](s32)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
     %0:_(p3) = COPY $vgpr0
     %1:_(s128) = G_LOAD %0 :: (load 16, align 2, addrspace 3)
     $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
@@ -3954,94 +3926,99 @@ body: |
     ; VI: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
     ; VI: [[PTR_ADD6:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C6]](s32)
     ; VI: [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p3) :: (load 1, addrspace 3)
-    ; VI: [[C7:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; VI: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; VI: [[PTR_ADD7:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C7]](s32)
+    ; VI: [[LOAD8:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD7]](p3) :: (load 1, addrspace 3)
+    ; VI: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 9
+    ; VI: [[PTR_ADD8:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C8]](s32)
+    ; VI: [[LOAD9:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD8]](p3) :: (load 1, addrspace 3)
+    ; VI: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+    ; VI: [[PTR_ADD9:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C9]](s32)
+    ; VI: [[LOAD10:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD9]](p3) :: (load 1, addrspace 3)
+    ; VI: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 11
+    ; VI: [[PTR_ADD10:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C10]](s32)
+    ; VI: [[LOAD11:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p3) :: (load 1, addrspace 3)
+    ; VI: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; VI: [[PTR_ADD11:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C11]](s32)
+    ; VI: [[LOAD12:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD11]](p3) :: (load 1, addrspace 3)
+    ; VI: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 13
+    ; VI: [[PTR_ADD12:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C12]](s32)
+    ; VI: [[LOAD13:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD12]](p3) :: (load 1, addrspace 3)
+    ; VI: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 14
+    ; VI: [[PTR_ADD13:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C13]](s32)
+    ; VI: [[LOAD14:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD13]](p3) :: (load 1, addrspace 3)
+    ; VI: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
+    ; VI: [[PTR_ADD14:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C14]](s32)
+    ; VI: [[LOAD15:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p3) :: (load 1, addrspace 3)
+    ; VI: [[C15:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
     ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
-    ; VI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C7]]
+    ; VI: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C15]]
     ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
-    ; VI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C7]]
-    ; VI: [[C8:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C8]](s16)
+    ; VI: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C15]]
+    ; VI: [[C16:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C16]](s16)
     ; VI: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
     ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
-    ; VI: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C7]]
+    ; VI: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C15]]
     ; VI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD3]](s32)
-    ; VI: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C7]]
-    ; VI: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C8]](s16)
+    ; VI: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C15]]
+    ; VI: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C16]](s16)
     ; VI: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
     ; VI: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD4]](s32)
-    ; VI: [[AND4:%[0-9]+]]:_(s16) = G_AND [[TRUNC4]], [[C7]]
+    ; VI: [[AND4:%[0-9]+]]:_(s16) = G_AND [[TRUNC4]], [[C15]]
     ; VI: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD5]](s32)
-    ; VI: [[AND5:%[0-9]+]]:_(s16) = G_AND [[TRUNC5]], [[C7]]
-    ; VI: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND5]], [[C8]](s16)
+    ; VI: [[AND5:%[0-9]+]]:_(s16) = G_AND [[TRUNC5]], [[C15]]
+    ; VI: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND5]], [[C16]](s16)
     ; VI: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND4]], [[SHL2]]
     ; VI: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD6]](s32)
-    ; VI: [[AND6:%[0-9]+]]:_(s16) = G_AND [[TRUNC6]], [[C7]]
+    ; VI: [[AND6:%[0-9]+]]:_(s16) = G_AND [[TRUNC6]], [[C15]]
     ; VI: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD7]](s32)
-    ; VI: [[AND7:%[0-9]+]]:_(s16) = G_AND [[TRUNC7]], [[C7]]
-    ; VI: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND7]], [[C8]](s16)
-    ; VI: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND6]], [[SHL3]]
-    ; VI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
-    ; VI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
-    ; VI: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; VI: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C9]](s32)
-    ; VI: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
-    ; VI: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
-    ; VI: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
-    ; VI: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C9]](s32)
-    ; VI: [[OR5:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL5]]
-    ; VI: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR4]](s32), [[OR5]](s32)
-    ; VI: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; VI: [[PTR_ADD7:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C10]](s32)
-    ; VI: [[LOAD8:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD7]](p3) :: (load 1, addrspace 3)
-    ; VI: [[PTR_ADD8:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
-    ; VI: [[LOAD9:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD8]](p3) :: (load 1, addrspace 3)
-    ; VI: [[PTR_ADD9:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C1]](s32)
-    ; VI: [[LOAD10:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD9]](p3) :: (load 1, addrspace 3)
-    ; VI: [[PTR_ADD10:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
-    ; VI: [[LOAD11:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p3) :: (load 1, addrspace 3)
-    ; VI: [[PTR_ADD11:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C3]](s32)
-    ; VI: [[LOAD12:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD11]](p3) :: (load 1, addrspace 3)
-    ; VI: [[PTR_ADD12:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C4]](s32)
-    ; VI: [[LOAD13:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD12]](p3) :: (load 1, addrspace 3)
-    ; VI: [[PTR_ADD13:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C5]](s32)
-    ; VI: [[LOAD14:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD13]](p3) :: (load 1, addrspace 3)
-    ; VI: [[PTR_ADD14:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C6]](s32)
-    ; VI: [[LOAD15:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p3) :: (load 1, addrspace 3)
+    ; VI: [[AND7:%[0-9]+]]:_(s16) = G_AND [[TRUNC7]], [[C15]]
+    ; VI: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND7]], [[C16]](s16)
+    ; VI: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND6]], [[SHL3]]
     ; VI: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD8]](s32)
-    ; VI: [[AND8:%[0-9]+]]:_(s16) = G_AND [[TRUNC8]], [[C7]]
+    ; VI: [[AND8:%[0-9]+]]:_(s16) = G_AND [[TRUNC8]], [[C15]]
     ; VI: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD9]](s32)
-    ; VI: [[AND9:%[0-9]+]]:_(s16) = G_AND [[TRUNC9]], [[C7]]
-    ; VI: [[SHL6:%[0-9]+]]:_(s16) = G_SHL [[AND9]], [[C8]](s16)
-    ; VI: [[OR6:%[0-9]+]]:_(s16) = G_OR [[AND8]], [[SHL6]]
+    ; VI: [[AND9:%[0-9]+]]:_(s16) = G_AND [[TRUNC9]], [[C15]]
+    ; VI: [[SHL4:%[0-9]+]]:_(s16) = G_SHL [[AND9]], [[C16]](s16)
+    ; VI: [[OR4:%[0-9]+]]:_(s16) = G_OR [[AND8]], [[SHL4]]
     ; VI: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD10]](s32)
-    ; VI: [[AND10:%[0-9]+]]:_(s16) = G_AND [[TRUNC10]], [[C7]]
+    ; VI: [[AND10:%[0-9]+]]:_(s16) = G_AND [[TRUNC10]], [[C15]]
     ; VI: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD11]](s32)
-    ; VI: [[AND11:%[0-9]+]]:_(s16) = G_AND [[TRUNC11]], [[C7]]
-    ; VI: [[SHL7:%[0-9]+]]:_(s16) = G_SHL [[AND11]], [[C8]](s16)
-    ; VI: [[OR7:%[0-9]+]]:_(s16) = G_OR [[AND10]], [[SHL7]]
+    ; VI: [[AND11:%[0-9]+]]:_(s16) = G_AND [[TRUNC11]], [[C15]]
+    ; VI: [[SHL5:%[0-9]+]]:_(s16) = G_SHL [[AND11]], [[C16]](s16)
+    ; VI: [[OR5:%[0-9]+]]:_(s16) = G_OR [[AND10]], [[SHL5]]
     ; VI: [[TRUNC12:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD12]](s32)
-    ; VI: [[AND12:%[0-9]+]]:_(s16) = G_AND [[TRUNC12]], [[C7]]
+    ; VI: [[AND12:%[0-9]+]]:_(s16) = G_AND [[TRUNC12]], [[C15]]
     ; VI: [[TRUNC13:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD13]](s32)
-    ; VI: [[AND13:%[0-9]+]]:_(s16) = G_AND [[TRUNC13]], [[C7]]
-    ; VI: [[SHL8:%[0-9]+]]:_(s16) = G_SHL [[AND13]], [[C8]](s16)
-    ; VI: [[OR8:%[0-9]+]]:_(s16) = G_OR [[AND12]], [[SHL8]]
+    ; VI: [[AND13:%[0-9]+]]:_(s16) = G_AND [[TRUNC13]], [[C15]]
+    ; VI: [[SHL6:%[0-9]+]]:_(s16) = G_SHL [[AND13]], [[C16]](s16)
+    ; VI: [[OR6:%[0-9]+]]:_(s16) = G_OR [[AND12]], [[SHL6]]
     ; VI: [[TRUNC14:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD14]](s32)
-    ; VI: [[AND14:%[0-9]+]]:_(s16) = G_AND [[TRUNC14]], [[C7]]
+    ; VI: [[AND14:%[0-9]+]]:_(s16) = G_AND [[TRUNC14]], [[C15]]
     ; VI: [[TRUNC15:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD15]](s32)
-    ; VI: [[AND15:%[0-9]+]]:_(s16) = G_AND [[TRUNC15]], [[C7]]
-    ; VI: [[SHL9:%[0-9]+]]:_(s16) = G_SHL [[AND15]], [[C8]](s16)
-    ; VI: [[OR9:%[0-9]+]]:_(s16) = G_OR [[AND14]], [[SHL9]]
-    ; VI: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[OR6]](s16)
-    ; VI: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[OR7]](s16)
-    ; VI: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C9]](s32)
+    ; VI: [[AND15:%[0-9]+]]:_(s16) = G_AND [[TRUNC15]], [[C15]]
+    ; VI: [[SHL7:%[0-9]+]]:_(s16) = G_SHL [[AND15]], [[C16]](s16)
+    ; VI: [[OR7:%[0-9]+]]:_(s16) = G_OR [[AND14]], [[SHL7]]
+    ; VI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
+    ; VI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
+    ; VI: [[C17:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; VI: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C17]](s32)
+    ; VI: [[OR8:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL8]]
+    ; VI: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
+    ; VI: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
+    ; VI: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C17]](s32)
+    ; VI: [[OR9:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL9]]
+    ; VI: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[OR4]](s16)
+    ; VI: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[OR5]](s16)
+    ; VI: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C17]](s32)
     ; VI: [[OR10:%[0-9]+]]:_(s32) = G_OR [[ZEXT4]], [[SHL10]]
-    ; VI: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[OR8]](s16)
-    ; VI: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[OR9]](s16)
-    ; VI: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[ZEXT7]], [[C9]](s32)
+    ; VI: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[OR6]](s16)
+    ; VI: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[OR7]](s16)
+    ; VI: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[ZEXT7]], [[C17]](s32)
     ; VI: [[OR11:%[0-9]+]]:_(s32) = G_OR [[ZEXT6]], [[SHL11]]
-    ; VI: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR10]](s32), [[OR11]](s32)
-    ; VI: [[MV2:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[MV]](s64), [[MV1]](s64)
-    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV2]](s128)
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[OR8]](s32), [[OR9]](s32), [[OR10]](s32), [[OR11]](s32)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
     ; GFX9-LABEL: name: test_load_local_s128_align1
     ; GFX9: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
     ; GFX9: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p3) :: (load 1, addrspace 3)
@@ -4066,94 +4043,99 @@ body: |
     ; GFX9: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
     ; GFX9: [[PTR_ADD6:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C6]](s32)
     ; GFX9: [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p3) :: (load 1, addrspace 3)
-    ; GFX9: [[C7:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; GFX9: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9: [[PTR_ADD7:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C7]](s32)
+    ; GFX9: [[LOAD8:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD7]](p3) :: (load 1, addrspace 3)
+    ; GFX9: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 9
+    ; GFX9: [[PTR_ADD8:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C8]](s32)
+    ; GFX9: [[LOAD9:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD8]](p3) :: (load 1, addrspace 3)
+    ; GFX9: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+    ; GFX9: [[PTR_ADD9:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C9]](s32)
+    ; GFX9: [[LOAD10:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD9]](p3) :: (load 1, addrspace 3)
+    ; GFX9: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 11
+    ; GFX9: [[PTR_ADD10:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C10]](s32)
+    ; GFX9: [[LOAD11:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p3) :: (load 1, addrspace 3)
+    ; GFX9: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; GFX9: [[PTR_ADD11:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C11]](s32)
+    ; GFX9: [[LOAD12:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD11]](p3) :: (load 1, addrspace 3)
+    ; GFX9: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 13
+    ; GFX9: [[PTR_ADD12:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C12]](s32)
+    ; GFX9: [[LOAD13:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD12]](p3) :: (load 1, addrspace 3)
+    ; GFX9: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 14
+    ; GFX9: [[PTR_ADD13:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C13]](s32)
+    ; GFX9: [[LOAD14:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD13]](p3) :: (load 1, addrspace 3)
+    ; GFX9: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
+    ; GFX9: [[PTR_ADD14:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C14]](s32)
+    ; GFX9: [[LOAD15:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p3) :: (load 1, addrspace 3)
+    ; GFX9: [[C15:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
     ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
-    ; GFX9: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C7]]
+    ; GFX9: [[AND:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C15]]
     ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD1]](s32)
-    ; GFX9: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C7]]
-    ; GFX9: [[C8:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C8]](s16)
+    ; GFX9: [[AND1:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C15]]
+    ; GFX9: [[C16:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[AND1]], [[C16]](s16)
     ; GFX9: [[OR:%[0-9]+]]:_(s16) = G_OR [[AND]], [[SHL]]
     ; GFX9: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD2]](s32)
-    ; GFX9: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C7]]
+    ; GFX9: [[AND2:%[0-9]+]]:_(s16) = G_AND [[TRUNC2]], [[C15]]
     ; GFX9: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD3]](s32)
-    ; GFX9: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C7]]
-    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C8]](s16)
+    ; GFX9: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C15]]
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C16]](s16)
     ; GFX9: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
     ; GFX9: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD4]](s32)
-    ; GFX9: [[AND4:%[0-9]+]]:_(s16) = G_AND [[TRUNC4]], [[C7]]
+    ; GFX9: [[AND4:%[0-9]+]]:_(s16) = G_AND [[TRUNC4]], [[C15]]
     ; GFX9: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD5]](s32)
-    ; GFX9: [[AND5:%[0-9]+]]:_(s16) = G_AND [[TRUNC5]], [[C7]]
-    ; GFX9: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND5]], [[C8]](s16)
+    ; GFX9: [[AND5:%[0-9]+]]:_(s16) = G_AND [[TRUNC5]], [[C15]]
+    ; GFX9: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND5]], [[C16]](s16)
     ; GFX9: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND4]], [[SHL2]]
     ; GFX9: [[TRUNC6:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD6]](s32)
-    ; GFX9: [[AND6:%[0-9]+]]:_(s16) = G_AND [[TRUNC6]], [[C7]]
+    ; GFX9: [[AND6:%[0-9]+]]:_(s16) = G_AND [[TRUNC6]], [[C15]]
     ; GFX9: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD7]](s32)
-    ; GFX9: [[AND7:%[0-9]+]]:_(s16) = G_AND [[TRUNC7]], [[C7]]
-    ; GFX9: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND7]], [[C8]](s16)
+    ; GFX9: [[AND7:%[0-9]+]]:_(s16) = G_AND [[TRUNC7]], [[C15]]
+    ; GFX9: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND7]], [[C16]](s16)
     ; GFX9: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND6]], [[SHL3]]
-    ; GFX9: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
-    ; GFX9: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
-    ; GFX9: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C9]](s32)
-    ; GFX9: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
-    ; GFX9: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
-    ; GFX9: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
-    ; GFX9: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C9]](s32)
-    ; GFX9: [[OR5:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL5]]
-    ; GFX9: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR4]](s32), [[OR5]](s32)
-    ; GFX9: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9: [[PTR_ADD7:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C10]](s32)
-    ; GFX9: [[LOAD8:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD7]](p3) :: (load 1, addrspace 3)
-    ; GFX9: [[PTR_ADD8:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
-    ; GFX9: [[LOAD9:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD8]](p3) :: (load 1, addrspace 3)
-    ; GFX9: [[PTR_ADD9:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C1]](s32)
-    ; GFX9: [[LOAD10:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD9]](p3) :: (load 1, addrspace 3)
-    ; GFX9: [[PTR_ADD10:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
-    ; GFX9: [[LOAD11:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p3) :: (load 1, addrspace 3)
-    ; GFX9: [[PTR_ADD11:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C3]](s32)
-    ; GFX9: [[LOAD12:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD11]](p3) :: (load 1, addrspace 3)
-    ; GFX9: [[PTR_ADD12:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C4]](s32)
-    ; GFX9: [[LOAD13:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD12]](p3) :: (load 1, addrspace 3)
-    ; GFX9: [[PTR_ADD13:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C5]](s32)
-    ; GFX9: [[LOAD14:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD13]](p3) :: (load 1, addrspace 3)
-    ; GFX9: [[PTR_ADD14:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C6]](s32)
-    ; GFX9: [[LOAD15:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p3) :: (load 1, addrspace 3)
     ; GFX9: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD8]](s32)
-    ; GFX9: [[AND8:%[0-9]+]]:_(s16) = G_AND [[TRUNC8]], [[C7]]
+    ; GFX9: [[AND8:%[0-9]+]]:_(s16) = G_AND [[TRUNC8]], [[C15]]
     ; GFX9: [[TRUNC9:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD9]](s32)
-    ; GFX9: [[AND9:%[0-9]+]]:_(s16) = G_AND [[TRUNC9]], [[C7]]
-    ; GFX9: [[SHL6:%[0-9]+]]:_(s16) = G_SHL [[AND9]], [[C8]](s16)
-    ; GFX9: [[OR6:%[0-9]+]]:_(s16) = G_OR [[AND8]], [[SHL6]]
+    ; GFX9: [[AND9:%[0-9]+]]:_(s16) = G_AND [[TRUNC9]], [[C15]]
+    ; GFX9: [[SHL4:%[0-9]+]]:_(s16) = G_SHL [[AND9]], [[C16]](s16)
+    ; GFX9: [[OR4:%[0-9]+]]:_(s16) = G_OR [[AND8]], [[SHL4]]
     ; GFX9: [[TRUNC10:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD10]](s32)
-    ; GFX9: [[AND10:%[0-9]+]]:_(s16) = G_AND [[TRUNC10]], [[C7]]
+    ; GFX9: [[AND10:%[0-9]+]]:_(s16) = G_AND [[TRUNC10]], [[C15]]
     ; GFX9: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD11]](s32)
-    ; GFX9: [[AND11:%[0-9]+]]:_(s16) = G_AND [[TRUNC11]], [[C7]]
-    ; GFX9: [[SHL7:%[0-9]+]]:_(s16) = G_SHL [[AND11]], [[C8]](s16)
-    ; GFX9: [[OR7:%[0-9]+]]:_(s16) = G_OR [[AND10]], [[SHL7]]
+    ; GFX9: [[AND11:%[0-9]+]]:_(s16) = G_AND [[TRUNC11]], [[C15]]
+    ; GFX9: [[SHL5:%[0-9]+]]:_(s16) = G_SHL [[AND11]], [[C16]](s16)
+    ; GFX9: [[OR5:%[0-9]+]]:_(s16) = G_OR [[AND10]], [[SHL5]]
     ; GFX9: [[TRUNC12:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD12]](s32)
-    ; GFX9: [[AND12:%[0-9]+]]:_(s16) = G_AND [[TRUNC12]], [[C7]]
+    ; GFX9: [[AND12:%[0-9]+]]:_(s16) = G_AND [[TRUNC12]], [[C15]]
     ; GFX9: [[TRUNC13:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD13]](s32)
-    ; GFX9: [[AND13:%[0-9]+]]:_(s16) = G_AND [[TRUNC13]], [[C7]]
-    ; GFX9: [[SHL8:%[0-9]+]]:_(s16) = G_SHL [[AND13]], [[C8]](s16)
-    ; GFX9: [[OR8:%[0-9]+]]:_(s16) = G_OR [[AND12]], [[SHL8]]
+    ; GFX9: [[AND13:%[0-9]+]]:_(s16) = G_AND [[TRUNC13]], [[C15]]
+    ; GFX9: [[SHL6:%[0-9]+]]:_(s16) = G_SHL [[AND13]], [[C16]](s16)
+    ; GFX9: [[OR6:%[0-9]+]]:_(s16) = G_OR [[AND12]], [[SHL6]]
     ; GFX9: [[TRUNC14:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD14]](s32)
-    ; GFX9: [[AND14:%[0-9]+]]:_(s16) = G_AND [[TRUNC14]], [[C7]]
+    ; GFX9: [[AND14:%[0-9]+]]:_(s16) = G_AND [[TRUNC14]], [[C15]]
     ; GFX9: [[TRUNC15:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD15]](s32)
-    ; GFX9: [[AND15:%[0-9]+]]:_(s16) = G_AND [[TRUNC15]], [[C7]]
-    ; GFX9: [[SHL9:%[0-9]+]]:_(s16) = G_SHL [[AND15]], [[C8]](s16)
-    ; GFX9: [[OR9:%[0-9]+]]:_(s16) = G_OR [[AND14]], [[SHL9]]
-    ; GFX9: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[OR6]](s16)
-    ; GFX9: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[OR7]](s16)
-    ; GFX9: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C9]](s32)
+    ; GFX9: [[AND15:%[0-9]+]]:_(s16) = G_AND [[TRUNC15]], [[C15]]
+    ; GFX9: [[SHL7:%[0-9]+]]:_(s16) = G_SHL [[AND15]], [[C16]](s16)
+    ; GFX9: [[OR7:%[0-9]+]]:_(s16) = G_OR [[AND14]], [[SHL7]]
+    ; GFX9: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
+    ; GFX9: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
+    ; GFX9: [[C17:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C17]](s32)
+    ; GFX9: [[OR8:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL8]]
+    ; GFX9: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
+    ; GFX9: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
+    ; GFX9: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C17]](s32)
+    ; GFX9: [[OR9:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL9]]
+    ; GFX9: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[OR4]](s16)
+    ; GFX9: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[OR5]](s16)
+    ; GFX9: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C17]](s32)
     ; GFX9: [[OR10:%[0-9]+]]:_(s32) = G_OR [[ZEXT4]], [[SHL10]]
-    ; GFX9: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[OR8]](s16)
-    ; GFX9: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[OR9]](s16)
-    ; GFX9: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[ZEXT7]], [[C9]](s32)
+    ; GFX9: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[OR6]](s16)
+    ; GFX9: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[OR7]](s16)
+    ; GFX9: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[ZEXT7]], [[C17]](s32)
     ; GFX9: [[OR11:%[0-9]+]]:_(s32) = G_OR [[ZEXT6]], [[SHL11]]
-    ; GFX9: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR10]](s32), [[OR11]](s32)
-    ; GFX9: [[MV2:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[MV]](s64), [[MV1]](s64)
-    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV2]](s128)
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[OR8]](s32), [[OR9]](s32), [[OR10]](s32), [[OR11]](s32)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
     %0:_(p3) = COPY $vgpr0
     %1:_(s128) = G_LOAD %0 :: (load 16, align 1, addrspace 3)
     $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
@@ -5846,33 +5828,38 @@ body: |
     ; VI: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
     ; VI: [[PTR_ADD6:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C6]](s32)
     ; VI: [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p3) :: (load 1, addrspace 3)
-    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
-    ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LOAD1]](s32)
-    ; VI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LOAD2]](s32)
-    ; VI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LOAD3]](s32)
-    ; VI: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LOAD4]](s32)
-    ; VI: [[COPY6:%[0-9]+]]:_(s32) = COPY [[LOAD5]](s32)
-    ; VI: [[COPY7:%[0-9]+]]:_(s32) = COPY [[LOAD6]](s32)
-    ; VI: [[COPY8:%[0-9]+]]:_(s32) = COPY [[LOAD7]](s32)
-    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32)
-    ; VI: [[TRUNC:%[0-9]+]]:_(<8 x s8>) = G_TRUNC [[BUILD_VECTOR]](<8 x s32>)
     ; VI: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
     ; VI: [[PTR_ADD7:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C7]](s32)
     ; VI: [[LOAD8:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD7]](p3) :: (load 1, addrspace 3)
-    ; VI: [[PTR_ADD8:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
+    ; VI: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 9
+    ; VI: [[PTR_ADD8:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C8]](s32)
     ; VI: [[LOAD9:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD8]](p3) :: (load 1, addrspace 3)
-    ; VI: [[PTR_ADD9:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C1]](s32)
+    ; VI: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+    ; VI: [[PTR_ADD9:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C9]](s32)
     ; VI: [[LOAD10:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD9]](p3) :: (load 1, addrspace 3)
-    ; VI: [[PTR_ADD10:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
+    ; VI: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 11
+    ; VI: [[PTR_ADD10:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C10]](s32)
     ; VI: [[LOAD11:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p3) :: (load 1, addrspace 3)
-    ; VI: [[PTR_ADD11:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C3]](s32)
+    ; VI: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; VI: [[PTR_ADD11:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C11]](s32)
     ; VI: [[LOAD12:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD11]](p3) :: (load 1, addrspace 3)
-    ; VI: [[PTR_ADD12:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C4]](s32)
+    ; VI: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 13
+    ; VI: [[PTR_ADD12:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C12]](s32)
     ; VI: [[LOAD13:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD12]](p3) :: (load 1, addrspace 3)
-    ; VI: [[PTR_ADD13:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C5]](s32)
+    ; VI: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 14
+    ; VI: [[PTR_ADD13:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C13]](s32)
     ; VI: [[LOAD14:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD13]](p3) :: (load 1, addrspace 3)
-    ; VI: [[PTR_ADD14:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C6]](s32)
+    ; VI: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
+    ; VI: [[PTR_ADD14:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C14]](s32)
     ; VI: [[LOAD15:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p3) :: (load 1, addrspace 3)
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
+    ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LOAD1]](s32)
+    ; VI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LOAD2]](s32)
+    ; VI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LOAD3]](s32)
+    ; VI: [[COPY5:%[0-9]+]]:_(s32) = COPY [[LOAD4]](s32)
+    ; VI: [[COPY6:%[0-9]+]]:_(s32) = COPY [[LOAD5]](s32)
+    ; VI: [[COPY7:%[0-9]+]]:_(s32) = COPY [[LOAD6]](s32)
+    ; VI: [[COPY8:%[0-9]+]]:_(s32) = COPY [[LOAD7]](s32)
     ; VI: [[COPY9:%[0-9]+]]:_(s32) = COPY [[LOAD8]](s32)
     ; VI: [[COPY10:%[0-9]+]]:_(s32) = COPY [[LOAD9]](s32)
     ; VI: [[COPY11:%[0-9]+]]:_(s32) = COPY [[LOAD10]](s32)
@@ -5881,10 +5868,9 @@ body: |
     ; VI: [[COPY14:%[0-9]+]]:_(s32) = COPY [[LOAD13]](s32)
     ; VI: [[COPY15:%[0-9]+]]:_(s32) = COPY [[LOAD14]](s32)
     ; VI: [[COPY16:%[0-9]+]]:_(s32) = COPY [[LOAD15]](s32)
-    ; VI: [[BUILD_VECTOR1:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32)
-    ; VI: [[TRUNC1:%[0-9]+]]:_(<8 x s8>) = G_TRUNC [[BUILD_VECTOR1]](<8 x s32>)
-    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<16 x s8>) = G_CONCAT_VECTORS [[TRUNC]](<8 x s8>), [[TRUNC1]](<8 x s8>)
-    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[CONCAT_VECTORS]](<16 x s8>)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<16 x s32>) = G_BUILD_VECTOR [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32)
+    ; VI: [[TRUNC:%[0-9]+]]:_(<16 x s8>) = G_TRUNC [[BUILD_VECTOR]](<16 x s32>)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[TRUNC]](<16 x s8>)
     ; GFX9-LABEL: name: test_load_local_v16s8_align16
     ; GFX9: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
     ; GFX9: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p3) :: (load 1, addrspace 3)
@@ -5909,6 +5895,30 @@ body: |
     ; GFX9: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
     ; GFX9: [[PTR_ADD6:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C6]](s32)
     ; GFX9: [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p3) :: (load 1, addrspace 3)
+    ; GFX9: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9: [[PTR_ADD7:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C7]](s32)
+    ; GFX9: [[LOAD8:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD7]](p3) :: (load 1, addrspace 3)
+    ; GFX9: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 9
+    ; GFX9: [[PTR_ADD8:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C8]](s32)
+    ; GFX9: [[LOAD9:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD8]](p3) :: (load 1, addrspace 3)
+    ; GFX9: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+    ; GFX9: [[PTR_ADD9:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C9]](s32)
+    ; GFX9: [[LOAD10:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD9]](p3) :: (load 1, addrspace 3)
+    ; GFX9: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 11
+    ; GFX9: [[PTR_ADD10:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C10]](s32)
+    ; GFX9: [[LOAD11:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p3) :: (load 1, addrspace 3)
+    ; GFX9: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; GFX9: [[PTR_ADD11:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C11]](s32)
+    ; GFX9: [[LOAD12:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD11]](p3) :: (load 1, addrspace 3)
+    ; GFX9: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 13
+    ; GFX9: [[PTR_ADD12:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C12]](s32)
+    ; GFX9: [[LOAD13:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD12]](p3) :: (load 1, addrspace 3)
+    ; GFX9: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 14
+    ; GFX9: [[PTR_ADD13:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C13]](s32)
+    ; GFX9: [[LOAD14:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD13]](p3) :: (load 1, addrspace 3)
+    ; GFX9: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
+    ; GFX9: [[PTR_ADD14:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C14]](s32)
+    ; GFX9: [[LOAD15:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p3) :: (load 1, addrspace 3)
     ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
     ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LOAD1]](s32)
     ; GFX9: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY1]](s32), [[COPY2]](s32)
@@ -5921,25 +5931,6 @@ body: |
     ; GFX9: [[COPY7:%[0-9]+]]:_(s32) = COPY [[LOAD6]](s32)
     ; GFX9: [[COPY8:%[0-9]+]]:_(s32) = COPY [[LOAD7]](s32)
     ; GFX9: [[BUILD_VECTOR_TRUNC3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY7]](s32), [[COPY8]](s32)
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>), [[BUILD_VECTOR_TRUNC2]](<2 x s16>), [[BUILD_VECTOR_TRUNC3]](<2 x s16>)
-    ; GFX9: [[TRUNC:%[0-9]+]]:_(<8 x s8>) = G_TRUNC [[CONCAT_VECTORS]](<8 x s16>)
-    ; GFX9: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9: [[PTR_ADD7:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C7]](s32)
-    ; GFX9: [[LOAD8:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD7]](p3) :: (load 1, addrspace 3)
-    ; GFX9: [[PTR_ADD8:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C]](s32)
-    ; GFX9: [[LOAD9:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD8]](p3) :: (load 1, addrspace 3)
-    ; GFX9: [[PTR_ADD9:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C1]](s32)
-    ; GFX9: [[LOAD10:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD9]](p3) :: (load 1, addrspace 3)
-    ; GFX9: [[PTR_ADD10:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C2]](s32)
-    ; GFX9: [[LOAD11:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD10]](p3) :: (load 1, addrspace 3)
-    ; GFX9: [[PTR_ADD11:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C3]](s32)
-    ; GFX9: [[LOAD12:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD11]](p3) :: (load 1, addrspace 3)
-    ; GFX9: [[PTR_ADD12:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C4]](s32)
-    ; GFX9: [[LOAD13:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD12]](p3) :: (load 1, addrspace 3)
-    ; GFX9: [[PTR_ADD13:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C5]](s32)
-    ; GFX9: [[LOAD14:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD13]](p3) :: (load 1, addrspace 3)
-    ; GFX9: [[PTR_ADD14:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C6]](s32)
-    ; GFX9: [[LOAD15:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD14]](p3) :: (load 1, addrspace 3)
     ; GFX9: [[COPY9:%[0-9]+]]:_(s32) = COPY [[LOAD8]](s32)
     ; GFX9: [[COPY10:%[0-9]+]]:_(s32) = COPY [[LOAD9]](s32)
     ; GFX9: [[BUILD_VECTOR_TRUNC4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY9]](s32), [[COPY10]](s32)
@@ -5952,10 +5943,9 @@ body: |
     ; GFX9: [[COPY15:%[0-9]+]]:_(s32) = COPY [[LOAD14]](s32)
     ; GFX9: [[COPY16:%[0-9]+]]:_(s32) = COPY [[LOAD15]](s32)
     ; GFX9: [[BUILD_VECTOR_TRUNC7:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY15]](s32), [[COPY16]](s32)
-    ; GFX9: [[CONCAT_VECTORS1:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC4]](<2 x s16>), [[BUILD_VECTOR_TRUNC5]](<2 x s16>), [[BUILD_VECTOR_TRUNC6]](<2 x s16>), [[BUILD_VECTOR_TRUNC7]](<2 x s16>)
-    ; GFX9: [[TRUNC1:%[0-9]+]]:_(<8 x s8>) = G_TRUNC [[CONCAT_VECTORS1]](<8 x s16>)
-    ; GFX9: [[CONCAT_VECTORS2:%[0-9]+]]:_(<16 x s8>) = G_CONCAT_VECTORS [[TRUNC]](<8 x s8>), [[TRUNC1]](<8 x s8>)
-    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[CONCAT_VECTORS2]](<16 x s8>)
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<16 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>), [[BUILD_VECTOR_TRUNC2]](<2 x s16>), [[BUILD_VECTOR_TRUNC3]](<2 x s16>), [[BUILD_VECTOR_TRUNC4]](<2 x s16>), [[BUILD_VECTOR_TRUNC5]](<2 x s16>), [[BUILD_VECTOR_TRUNC6]](<2 x s16>), [[BUILD_VECTOR_TRUNC7]](<2 x s16>)
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(<16 x s8>) = G_TRUNC [[CONCAT_VECTORS]](<16 x s16>)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[TRUNC]](<16 x s8>)
     %0:_(p3) = COPY $vgpr0
     %1:_(<16 x s8>) = G_LOAD %0 :: (load 16, align 1, addrspace 3)
     $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
@@ -8187,24 +8177,12 @@ body: |
     ; CI-DS128: $vgpr0_vgpr1_vgpr2 = COPY [[LOAD]](<3 x s32>)
     ; VI-LABEL: name: test_load_local_v3s32_align4
     ; VI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; VI: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p3) :: (load 8, align 4, addrspace 3)
-    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; VI: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; VI: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p3) :: (load 4, addrspace 3)
-    ; VI: [[DEF:%[0-9]+]]:_(<3 x s32>) = G_IMPLICIT_DEF
-    ; VI: [[INSERT:%[0-9]+]]:_(<3 x s32>) = G_INSERT [[DEF]], [[LOAD]](<2 x s32>), 0
-    ; VI: [[INSERT1:%[0-9]+]]:_(<3 x s32>) = G_INSERT [[INSERT]], [[LOAD1]](s32), 64
-    ; VI: $vgpr0_vgpr1_vgpr2 = COPY [[INSERT1]](<3 x s32>)
+    ; VI: [[LOAD:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[COPY]](p3) :: (load 12, align 4, addrspace 3)
+    ; VI: $vgpr0_vgpr1_vgpr2 = COPY [[LOAD]](<3 x s32>)
     ; GFX9-LABEL: name: test_load_local_v3s32_align4
     ; GFX9: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; GFX9: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p3) :: (load 8, align 4, addrspace 3)
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p3) :: (load 4, addrspace 3)
-    ; GFX9: [[DEF:%[0-9]+]]:_(<3 x s32>) = G_IMPLICIT_DEF
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<3 x s32>) = G_INSERT [[DEF]], [[LOAD]](<2 x s32>), 0
-    ; GFX9: [[INSERT1:%[0-9]+]]:_(<3 x s32>) = G_INSERT [[INSERT]], [[LOAD1]](s32), 64
-    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[INSERT1]](<3 x s32>)
+    ; GFX9: [[LOAD:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[COPY]](p3) :: (load 12, align 4, addrspace 3)
+    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[LOAD]](<3 x s32>)
     %0:_(p3) = COPY $vgpr0
     %1:_(<3 x s32>) = G_LOAD %0 :: (load 12, align 4, addrspace 3)
     $vgpr0_vgpr1_vgpr2 = COPY %1
@@ -8238,20 +8216,12 @@ body: |
     ; CI-DS128: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<4 x s32>)
     ; VI-LABEL: name: test_load_local_v4s32_align16
     ; VI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; VI: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p3) :: (load 8, align 16, addrspace 3)
-    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; VI: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; VI: [[LOAD1:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[PTR_ADD]](p3) :: (load 8, addrspace 3)
-    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s32>) = G_CONCAT_VECTORS [[LOAD]](<2 x s32>), [[LOAD1]](<2 x s32>)
-    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[CONCAT_VECTORS]](<4 x s32>)
+    ; VI: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p3) :: (load 16, addrspace 3)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<4 x s32>)
     ; GFX9-LABEL: name: test_load_local_v4s32_align16
     ; GFX9: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; GFX9: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p3) :: (load 8, align 16, addrspace 3)
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9: [[LOAD1:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[PTR_ADD]](p3) :: (load 8, addrspace 3)
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s32>) = G_CONCAT_VECTORS [[LOAD]](<2 x s32>), [[LOAD1]](<2 x s32>)
-    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[CONCAT_VECTORS]](<4 x s32>)
+    ; GFX9: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p3) :: (load 16, addrspace 3)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<4 x s32>)
     %0:_(p3) = COPY $vgpr0
     %1:_(<4 x s32>) = G_LOAD %0 :: (load 16, align 16, addrspace 3)
     $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
@@ -8285,20 +8255,12 @@ body: |
     ; CI-DS128: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<4 x s32>)
     ; VI-LABEL: name: test_load_local_v4s32_align8
     ; VI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; VI: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p3) :: (load 8, addrspace 3)
-    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; VI: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; VI: [[LOAD1:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[PTR_ADD]](p3) :: (load 8, addrspace 3)
-    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s32>) = G_CONCAT_VECTORS [[LOAD]](<2 x s32>), [[LOAD1]](<2 x s32>)
-    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[CONCAT_VECTORS]](<4 x s32>)
+    ; VI: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p3) :: (load 16, align 8, addrspace 3)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<4 x s32>)
     ; GFX9-LABEL: name: test_load_local_v4s32_align8
     ; GFX9: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; GFX9: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p3) :: (load 8, addrspace 3)
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9: [[LOAD1:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[PTR_ADD]](p3) :: (load 8, addrspace 3)
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s32>) = G_CONCAT_VECTORS [[LOAD]](<2 x s32>), [[LOAD1]](<2 x s32>)
-    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[CONCAT_VECTORS]](<4 x s32>)
+    ; GFX9: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p3) :: (load 16, align 8, addrspace 3)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<4 x s32>)
     %0:_(p3) = COPY $vgpr0
     %1:_(<4 x s32>) = G_LOAD %0 :: (load 16, align 8, addrspace 3)
     $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
@@ -8332,20 +8294,12 @@ body: |
     ; CI-DS128: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<4 x s32>)
     ; VI-LABEL: name: test_load_local_v4s32_align4
     ; VI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; VI: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p3) :: (load 8, align 4, addrspace 3)
-    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; VI: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; VI: [[LOAD1:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[PTR_ADD]](p3) :: (load 8, align 4, addrspace 3)
-    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s32>) = G_CONCAT_VECTORS [[LOAD]](<2 x s32>), [[LOAD1]](<2 x s32>)
-    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[CONCAT_VECTORS]](<4 x s32>)
+    ; VI: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p3) :: (load 16, align 4, addrspace 3)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<4 x s32>)
     ; GFX9-LABEL: name: test_load_local_v4s32_align4
     ; GFX9: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; GFX9: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p3) :: (load 8, align 4, addrspace 3)
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9: [[LOAD1:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[PTR_ADD]](p3) :: (load 8, align 4, addrspace 3)
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s32>) = G_CONCAT_VECTORS [[LOAD]](<2 x s32>), [[LOAD1]](<2 x s32>)
-    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[CONCAT_VECTORS]](<4 x s32>)
+    ; GFX9: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p3) :: (load 16, align 4, addrspace 3)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<4 x s32>)
     %0:_(p3) = COPY $vgpr0
     %1:_(<4 x s32>) = G_LOAD %0 :: (load 16, align 4, addrspace 3)
     $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
@@ -9048,7 +9002,6 @@ body: |
     ; VI: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[OR4]](s16)
     ; VI: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C5]](s32)
     ; VI: [[OR5:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL5]]
-    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32)
     ; VI: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
     ; VI: [[PTR_ADD7:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C7]](s32)
     ; VI: [[LOAD8:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD7]](p3) :: (load 1, addrspace 3)
@@ -9074,7 +9027,8 @@ body: |
     ; VI: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[OR7]](s16)
     ; VI: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C5]](s32)
     ; VI: [[OR8:%[0-9]+]]:_(s32) = G_OR [[ZEXT4]], [[SHL8]]
-    ; VI: [[PTR_ADD11:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C6]](s32)
+    ; VI: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; VI: [[PTR_ADD11:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C8]](s32)
     ; VI: [[LOAD12:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD11]](p3) :: (load 1, addrspace 3)
     ; VI: [[PTR_ADD12:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD11]], [[C]](s32)
     ; VI: [[LOAD13:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD12]](p3) :: (load 1, addrspace 3)
@@ -9098,9 +9052,8 @@ body: |
     ; VI: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[OR10]](s16)
     ; VI: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[ZEXT7]], [[C5]](s32)
     ; VI: [[OR11:%[0-9]+]]:_(s32) = G_OR [[ZEXT6]], [[SHL11]]
-    ; VI: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR8]](s32), [[OR11]](s32)
-    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s32>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x s32>), [[BUILD_VECTOR1]](<2 x s32>)
-    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[CONCAT_VECTORS]](<4 x s32>)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32), [[OR11]](s32)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>)
     ; GFX9-LABEL: name: test_load_local_v4s32_align1
     ; GFX9: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
     ; GFX9: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p3) :: (load 1, addrspace 3)
@@ -9157,7 +9110,6 @@ body: |
     ; GFX9: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[OR4]](s16)
     ; GFX9: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C5]](s32)
     ; GFX9: [[OR5:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL5]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32)
     ; GFX9: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
     ; GFX9: [[PTR_ADD7:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C7]](s32)
     ; GFX9: [[LOAD8:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD7]](p3) :: (load 1, addrspace 3)
@@ -9183,7 +9135,8 @@ body: |
     ; GFX9: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[OR7]](s16)
     ; GFX9: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C5]](s32)
     ; GFX9: [[OR8:%[0-9]+]]:_(s32) = G_OR [[ZEXT4]], [[SHL8]]
-    ; GFX9: [[PTR_ADD11:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD7]], [[C6]](s32)
+    ; GFX9: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+    ; GFX9: [[PTR_ADD11:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C8]](s32)
     ; GFX9: [[LOAD12:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD11]](p3) :: (load 1, addrspace 3)
     ; GFX9: [[PTR_ADD12:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD11]], [[C]](s32)
     ; GFX9: [[LOAD13:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD12]](p3) :: (load 1, addrspace 3)
@@ -9207,9 +9160,8 @@ body: |
     ; GFX9: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[OR10]](s16)
     ; GFX9: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[ZEXT7]], [[C5]](s32)
     ; GFX9: [[OR11:%[0-9]+]]:_(s32) = G_OR [[ZEXT6]], [[SHL11]]
-    ; GFX9: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR8]](s32), [[OR11]](s32)
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s32>) = G_CONCAT_VECTORS [[BUILD_VECTOR]](<2 x s32>), [[BUILD_VECTOR1]](<2 x s32>)
-    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[CONCAT_VECTORS]](<4 x s32>)
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32), [[OR11]](s32)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>)
     %0:_(p3) = COPY $vgpr0
     %1:_(<4 x s32>) = G_LOAD %0 :: (load 16, align 1, addrspace 3)
     $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
@@ -9259,31 +9211,19 @@ body: |
     ; CI-DS128: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[CONCAT_VECTORS]](<8 x s32>)
     ; VI-LABEL: name: test_load_local_v8s32_align32
     ; VI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; VI: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p3) :: (load 8, align 32, addrspace 3)
-    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; VI: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p3) :: (load 16, align 32, addrspace 3)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; VI: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; VI: [[LOAD1:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[PTR_ADD]](p3) :: (load 8, addrspace 3)
-    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; VI: [[PTR_ADD1:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; VI: [[LOAD2:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[PTR_ADD1]](p3) :: (load 8, align 16, addrspace 3)
-    ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; VI: [[PTR_ADD2:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; VI: [[LOAD3:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[PTR_ADD2]](p3) :: (load 8, addrspace 3)
-    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s32>) = G_CONCAT_VECTORS [[LOAD]](<2 x s32>), [[LOAD1]](<2 x s32>), [[LOAD2]](<2 x s32>), [[LOAD3]](<2 x s32>)
+    ; VI: [[LOAD1:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[PTR_ADD]](p3) :: (load 16, addrspace 3)
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s32>) = G_CONCAT_VECTORS [[LOAD]](<4 x s32>), [[LOAD1]](<4 x s32>)
     ; VI: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[CONCAT_VECTORS]](<8 x s32>)
     ; GFX9-LABEL: name: test_load_local_v8s32_align32
     ; GFX9: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; GFX9: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p3) :: (load 8, align 32, addrspace 3)
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[COPY]](p3) :: (load 16, align 32, addrspace 3)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; GFX9: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9: [[LOAD1:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[PTR_ADD]](p3) :: (load 8, addrspace 3)
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9: [[PTR_ADD1:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; GFX9: [[LOAD2:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[PTR_ADD1]](p3) :: (load 8, align 16, addrspace 3)
-    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; GFX9: [[PTR_ADD2:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX9: [[LOAD3:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[PTR_ADD2]](p3) :: (load 8, addrspace 3)
-    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s32>) = G_CONCAT_VECTORS [[LOAD]](<2 x s32>), [[LOAD1]](<2 x s32>), [[LOAD2]](<2 x s32>), [[LOAD3]](<2 x s32>)
+    ; GFX9: [[LOAD1:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[PTR_ADD]](p3) :: (load 16, addrspace 3)
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s32>) = G_CONCAT_VECTORS [[LOAD]](<4 x s32>), [[LOAD1]](<4 x s32>)
     ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[CONCAT_VECTORS]](<8 x s32>)
     %0:_(p3) = COPY $vgpr0
     %1:_(<8 x s32>) = G_LOAD %0 :: (load 32, align 32, addrspace 3)
@@ -9349,20 +9289,12 @@ body: |
     ; CI-DS128: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<2 x s64>)
     ; VI-LABEL: name: test_load_local_v2s64_align4
     ; VI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; VI: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p3) :: (load 8, align 4, addrspace 3)
-    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; VI: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; VI: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD]](p3) :: (load 8, align 4, addrspace 3)
-    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[LOAD]](s64), [[LOAD1]](s64)
-    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; VI: [[LOAD:%[0-9]+]]:_(<2 x s64>) = G_LOAD [[COPY]](p3) :: (load 16, align 4, addrspace 3)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<2 x s64>)
     ; GFX9-LABEL: name: test_load_local_v2s64_align4
     ; GFX9: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; GFX9: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p3) :: (load 8, align 4, addrspace 3)
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD]](p3) :: (load 8, align 4, addrspace 3)
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[LOAD]](s64), [[LOAD1]](s64)
-    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; GFX9: [[LOAD:%[0-9]+]]:_(<2 x s64>) = G_LOAD [[COPY]](p3) :: (load 16, align 4, addrspace 3)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<2 x s64>)
     %0:_(p3) = COPY $vgpr0
     %1:_(<2 x s64>) = G_LOAD %0 :: (load 16, align 4, addrspace 3)
     $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
@@ -10033,30 +9965,28 @@ body: |
     ; CI-DS128: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INSERT2]](<4 x s64>)
     ; VI-LABEL: name: test_load_local_v3s64_align32
     ; VI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; VI: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p3) :: (load 8, align 32, addrspace 3)
-    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; VI: [[LOAD:%[0-9]+]]:_(<2 x s64>) = G_LOAD [[COPY]](p3) :: (load 16, align 32, addrspace 3)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; VI: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; VI: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD]](p3) :: (load 8, addrspace 3)
-    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; VI: [[PTR_ADD1:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; VI: [[LOAD2:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD1]](p3) :: (load 8, align 16, addrspace 3)
-    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s64>) = G_BUILD_VECTOR [[LOAD]](s64), [[LOAD1]](s64), [[LOAD2]](s64)
-    ; VI: [[DEF:%[0-9]+]]:_(<4 x s64>) = G_IMPLICIT_DEF
-    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[BUILD_VECTOR]](<3 x s64>), 0
-    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INSERT]](<4 x s64>)
+    ; VI: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD]](p3) :: (load 8, align 16, addrspace 3)
+    ; VI: [[DEF:%[0-9]+]]:_(<3 x s64>) = G_IMPLICIT_DEF
+    ; VI: [[INSERT:%[0-9]+]]:_(<3 x s64>) = G_INSERT [[DEF]], [[LOAD]](<2 x s64>), 0
+    ; VI: [[INSERT1:%[0-9]+]]:_(<3 x s64>) = G_INSERT [[INSERT]], [[LOAD1]](s64), 128
+    ; VI: [[DEF1:%[0-9]+]]:_(<4 x s64>) = G_IMPLICIT_DEF
+    ; VI: [[INSERT2:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF1]], [[INSERT1]](<3 x s64>), 0
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INSERT2]](<4 x s64>)
     ; GFX9-LABEL: name: test_load_local_v3s64_align32
     ; GFX9: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; GFX9: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p3) :: (load 8, align 32, addrspace 3)
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9: [[LOAD:%[0-9]+]]:_(<2 x s64>) = G_LOAD [[COPY]](p3) :: (load 16, align 32, addrspace 3)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; GFX9: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD]](p3) :: (load 8, addrspace 3)
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9: [[PTR_ADD1:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; GFX9: [[LOAD2:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD1]](p3) :: (load 8, align 16, addrspace 3)
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s64>) = G_BUILD_VECTOR [[LOAD]](s64), [[LOAD1]](s64), [[LOAD2]](s64)
-    ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s64>) = G_IMPLICIT_DEF
-    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[BUILD_VECTOR]](<3 x s64>), 0
-    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INSERT]](<4 x s64>)
+    ; GFX9: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD]](p3) :: (load 8, align 16, addrspace 3)
+    ; GFX9: [[DEF:%[0-9]+]]:_(<3 x s64>) = G_IMPLICIT_DEF
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<3 x s64>) = G_INSERT [[DEF]], [[LOAD]](<2 x s64>), 0
+    ; GFX9: [[INSERT1:%[0-9]+]]:_(<3 x s64>) = G_INSERT [[INSERT]], [[LOAD1]](s64), 128
+    ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s64>) = G_IMPLICIT_DEF
+    ; GFX9: [[INSERT2:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF1]], [[INSERT1]](<3 x s64>), 0
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INSERT2]](<4 x s64>)
     %0:_(p3) = COPY $vgpr0
     %1:_(<3 x s64>) = G_LOAD %0 :: (load 24, align 32, addrspace 3)
     %2:_(<4 x s64>) = G_IMPLICIT_DEF
@@ -10108,32 +10038,20 @@ body: |
     ; CI-DS128: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[CONCAT_VECTORS]](<4 x s64>)
     ; VI-LABEL: name: test_load_local_v4s64_align32
     ; VI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; VI: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p3) :: (load 8, align 32, addrspace 3)
-    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; VI: [[LOAD:%[0-9]+]]:_(<2 x s64>) = G_LOAD [[COPY]](p3) :: (load 16, align 32, addrspace 3)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; VI: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; VI: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD]](p3) :: (load 8, addrspace 3)
-    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; VI: [[PTR_ADD1:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; VI: [[LOAD2:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD1]](p3) :: (load 8, align 16, addrspace 3)
-    ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; VI: [[PTR_ADD2:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; VI: [[LOAD3:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD2]](p3) :: (load 8, addrspace 3)
-    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[LOAD]](s64), [[LOAD1]](s64), [[LOAD2]](s64), [[LOAD3]](s64)
-    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
+    ; VI: [[LOAD1:%[0-9]+]]:_(<2 x s64>) = G_LOAD [[PTR_ADD]](p3) :: (load 16, addrspace 3)
+    ; VI: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s64>) = G_CONCAT_VECTORS [[LOAD]](<2 x s64>), [[LOAD1]](<2 x s64>)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[CONCAT_VECTORS]](<4 x s64>)
     ; GFX9-LABEL: name: test_load_local_v4s64_align32
     ; GFX9: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; GFX9: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p3) :: (load 8, align 32, addrspace 3)
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9: [[LOAD:%[0-9]+]]:_(<2 x s64>) = G_LOAD [[COPY]](p3) :: (load 16, align 32, addrspace 3)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; GFX9: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD]](p3) :: (load 8, addrspace 3)
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; GFX9: [[PTR_ADD1:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; GFX9: [[LOAD2:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD1]](p3) :: (load 8, align 16, addrspace 3)
-    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; GFX9: [[PTR_ADD2:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C2]](s32)
-    ; GFX9: [[LOAD3:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD2]](p3) :: (load 8, addrspace 3)
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[LOAD]](s64), [[LOAD1]](s64), [[LOAD2]](s64), [[LOAD3]](s64)
-    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
+    ; GFX9: [[LOAD1:%[0-9]+]]:_(<2 x s64>) = G_LOAD [[PTR_ADD]](p3) :: (load 16, addrspace 3)
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s64>) = G_CONCAT_VECTORS [[LOAD]](<2 x s64>), [[LOAD1]](<2 x s64>)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[CONCAT_VECTORS]](<4 x s64>)
     %0:_(p3) = COPY $vgpr0
     %1:_(<4 x s64>) = G_LOAD %0 :: (load 32, align 32, addrspace 3)
     $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY %1
@@ -10167,20 +10085,12 @@ body: |
     ; CI-DS128: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<2 x p1>)
     ; VI-LABEL: name: test_load_local_v2p1_align4
     ; VI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; VI: [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[COPY]](p3) :: (load 8, align 4, addrspace 3)
-    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; VI: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; VI: [[LOAD1:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p3) :: (load 8, align 4, addrspace 3)
-    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p1>) = G_BUILD_VECTOR [[LOAD]](p1), [[LOAD1]](p1)
-    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x p1>)
+    ; VI: [[LOAD:%[0-9]+]]:_(<2 x p1>) = G_LOAD [[COPY]](p3) :: (load 16, align 4, addrspace 3)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<2 x p1>)
     ; GFX9-LABEL: name: test_load_local_v2p1_align4
     ; GFX9: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; GFX9: [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[COPY]](p3) :: (load 8, align 4, addrspace 3)
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
-    ; GFX9: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9: [[LOAD1:%[0-9]+]]:_(p1) = G_LOAD [[PTR_ADD]](p3) :: (load 8, align 4, addrspace 3)
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p1>) = G_BUILD_VECTOR [[LOAD]](p1), [[LOAD1]](p1)
-    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x p1>)
+    ; GFX9: [[LOAD:%[0-9]+]]:_(<2 x p1>) = G_LOAD [[COPY]](p3) :: (load 16, align 4, addrspace 3)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<2 x p1>)
     %0:_(p3) = COPY $vgpr0
     %1:_(<2 x p1>) = G_LOAD %0 :: (load 16, align 4, addrspace 3)
     $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
@@ -12026,46 +11936,24 @@ body: |
     ; CI-DS128: $vgpr3_vgpr4_vgpr5 = COPY [[COPY2]](s96)
     ; VI-LABEL: name: test_extload_local_v2s96_from_24_align4
     ; VI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; VI: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p3) :: (load 8, align 4, addrspace 3)
-    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; VI: [[LOAD:%[0-9]+]]:_(s96) = G_LOAD [[COPY]](p3) :: (load 12, align 4, addrspace 3)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
     ; VI: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; VI: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p3) :: (load 4, addrspace 3)
-    ; VI: [[DEF:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF
-    ; VI: [[COPY1:%[0-9]+]]:_(s96) = COPY [[DEF]](s96)
-    ; VI: [[INSERT:%[0-9]+]]:_(s96) = G_INSERT [[COPY1]], [[LOAD]](s64), 0
-    ; VI: [[INSERT1:%[0-9]+]]:_(s96) = G_INSERT [[INSERT]], [[LOAD1]](s32), 64
-    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
-    ; VI: [[PTR_ADD1:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; VI: [[LOAD2:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD1]](p3) :: (load 8, align 4, addrspace 3)
-    ; VI: [[PTR_ADD2:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; VI: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p3) :: (load 4, addrspace 3)
-    ; VI: [[INSERT2:%[0-9]+]]:_(s96) = G_INSERT [[DEF]], [[LOAD2]](s64), 0
-    ; VI: [[INSERT3:%[0-9]+]]:_(s96) = G_INSERT [[INSERT2]], [[LOAD3]](s32), 64
-    ; VI: [[COPY2:%[0-9]+]]:_(s96) = COPY [[INSERT1]](s96)
-    ; VI: [[COPY3:%[0-9]+]]:_(s96) = COPY [[INSERT3]](s96)
-    ; VI: $vgpr0_vgpr1_vgpr2 = COPY [[COPY2]](s96)
-    ; VI: $vgpr3_vgpr4_vgpr5 = COPY [[COPY3]](s96)
+    ; VI: [[LOAD1:%[0-9]+]]:_(s96) = G_LOAD [[PTR_ADD]](p3) :: (load 12, align 4, addrspace 3)
+    ; VI: [[COPY1:%[0-9]+]]:_(s96) = COPY [[LOAD]](s96)
+    ; VI: [[COPY2:%[0-9]+]]:_(s96) = COPY [[LOAD1]](s96)
+    ; VI: $vgpr0_vgpr1_vgpr2 = COPY [[COPY1]](s96)
+    ; VI: $vgpr3_vgpr4_vgpr5 = COPY [[COPY2]](s96)
     ; GFX9-LABEL: name: test_extload_local_v2s96_from_24_align4
     ; GFX9: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; GFX9: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p3) :: (load 8, align 4, addrspace 3)
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9: [[LOAD:%[0-9]+]]:_(s96) = G_LOAD [[COPY]](p3) :: (load 12, align 4, addrspace 3)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
     ; GFX9: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p3) :: (load 4, addrspace 3)
-    ; GFX9: [[DEF:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF
-    ; GFX9: [[COPY1:%[0-9]+]]:_(s96) = COPY [[DEF]](s96)
-    ; GFX9: [[INSERT:%[0-9]+]]:_(s96) = G_INSERT [[COPY1]], [[LOAD]](s64), 0
-    ; GFX9: [[INSERT1:%[0-9]+]]:_(s96) = G_INSERT [[INSERT]], [[LOAD1]](s32), 64
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
-    ; GFX9: [[PTR_ADD1:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; GFX9: [[LOAD2:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD1]](p3) :: (load 8, align 4, addrspace 3)
-    ; GFX9: [[PTR_ADD2:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX9: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p3) :: (load 4, addrspace 3)
-    ; GFX9: [[INSERT2:%[0-9]+]]:_(s96) = G_INSERT [[DEF]], [[LOAD2]](s64), 0
-    ; GFX9: [[INSERT3:%[0-9]+]]:_(s96) = G_INSERT [[INSERT2]], [[LOAD3]](s32), 64
-    ; GFX9: [[COPY2:%[0-9]+]]:_(s96) = COPY [[INSERT1]](s96)
-    ; GFX9: [[COPY3:%[0-9]+]]:_(s96) = COPY [[INSERT3]](s96)
-    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[COPY2]](s96)
-    ; GFX9: $vgpr3_vgpr4_vgpr5 = COPY [[COPY3]](s96)
+    ; GFX9: [[LOAD1:%[0-9]+]]:_(s96) = G_LOAD [[PTR_ADD]](p3) :: (load 12, align 4, addrspace 3)
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s96) = COPY [[LOAD]](s96)
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s96) = COPY [[LOAD1]](s96)
+    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[COPY1]](s96)
+    ; GFX9: $vgpr3_vgpr4_vgpr5 = COPY [[COPY2]](s96)
     %0:_(p3) = COPY $vgpr0
     %1:_(<2 x s96>) = G_LOAD %0 :: (load 24, align 4, addrspace 3)
     %2:_(s96) = G_EXTRACT %1, 0
@@ -12134,46 +12022,24 @@ body: |
     ; CI-DS128: $vgpr3_vgpr4_vgpr5 = COPY [[COPY2]](s96)
     ; VI-LABEL: name: test_extload_local_v2s96_from_24_align16
     ; VI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; VI: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p3) :: (load 8, align 16, addrspace 3)
-    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; VI: [[LOAD:%[0-9]+]]:_(s96) = G_LOAD [[COPY]](p3) :: (load 12, align 16, addrspace 3)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
     ; VI: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; VI: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p3) :: (load 4, align 8, addrspace 3)
-    ; VI: [[DEF:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF
-    ; VI: [[COPY1:%[0-9]+]]:_(s96) = COPY [[DEF]](s96)
-    ; VI: [[INSERT:%[0-9]+]]:_(s96) = G_INSERT [[COPY1]], [[LOAD]](s64), 0
-    ; VI: [[INSERT1:%[0-9]+]]:_(s96) = G_INSERT [[INSERT]], [[LOAD1]](s32), 64
-    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
-    ; VI: [[PTR_ADD1:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; VI: [[LOAD2:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD1]](p3) :: (load 8, align 4, addrspace 3)
-    ; VI: [[PTR_ADD2:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; VI: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p3) :: (load 4, addrspace 3)
-    ; VI: [[INSERT2:%[0-9]+]]:_(s96) = G_INSERT [[DEF]], [[LOAD2]](s64), 0
-    ; VI: [[INSERT3:%[0-9]+]]:_(s96) = G_INSERT [[INSERT2]], [[LOAD3]](s32), 64
-    ; VI: [[COPY2:%[0-9]+]]:_(s96) = COPY [[INSERT1]](s96)
-    ; VI: [[COPY3:%[0-9]+]]:_(s96) = COPY [[INSERT3]](s96)
-    ; VI: $vgpr0_vgpr1_vgpr2 = COPY [[COPY2]](s96)
-    ; VI: $vgpr3_vgpr4_vgpr5 = COPY [[COPY3]](s96)
+    ; VI: [[LOAD1:%[0-9]+]]:_(s96) = G_LOAD [[PTR_ADD]](p3) :: (load 12, align 4, addrspace 3)
+    ; VI: [[COPY1:%[0-9]+]]:_(s96) = COPY [[LOAD]](s96)
+    ; VI: [[COPY2:%[0-9]+]]:_(s96) = COPY [[LOAD1]](s96)
+    ; VI: $vgpr0_vgpr1_vgpr2 = COPY [[COPY1]](s96)
+    ; VI: $vgpr3_vgpr4_vgpr5 = COPY [[COPY2]](s96)
     ; GFX9-LABEL: name: test_extload_local_v2s96_from_24_align16
     ; GFX9: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
-    ; GFX9: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p3) :: (load 8, align 16, addrspace 3)
-    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9: [[LOAD:%[0-9]+]]:_(s96) = G_LOAD [[COPY]](p3) :: (load 12, align 16, addrspace 3)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
     ; GFX9: [[PTR_ADD:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
-    ; GFX9: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p3) :: (load 4, align 8, addrspace 3)
-    ; GFX9: [[DEF:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF
-    ; GFX9: [[COPY1:%[0-9]+]]:_(s96) = COPY [[DEF]](s96)
-    ; GFX9: [[INSERT:%[0-9]+]]:_(s96) = G_INSERT [[COPY1]], [[LOAD]](s64), 0
-    ; GFX9: [[INSERT1:%[0-9]+]]:_(s96) = G_INSERT [[INSERT]], [[LOAD1]](s32), 64
-    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
-    ; GFX9: [[PTR_ADD1:%[0-9]+]]:_(p3) = G_PTR_ADD [[COPY]], [[C1]](s32)
-    ; GFX9: [[LOAD2:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD1]](p3) :: (load 8, align 4, addrspace 3)
-    ; GFX9: [[PTR_ADD2:%[0-9]+]]:_(p3) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32)
-    ; GFX9: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p3) :: (load 4, addrspace 3)
-    ; GFX9: [[INSERT2:%[0-9]+]]:_(s96) = G_INSERT [[DEF]], [[LOAD2]](s64), 0
-    ; GFX9: [[INSERT3:%[0-9]+]]:_(s96) = G_INSERT [[INSERT2]], [[LOAD3]](s32), 64
-    ; GFX9: [[COPY2:%[0-9]+]]:_(s96) = COPY [[INSERT1]](s96)
-    ; GFX9: [[COPY3:%[0-9]+]]:_(s96) = COPY [[INSERT3]](s96)
-    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[COPY2]](s96)
-    ; GFX9: $vgpr3_vgpr4_vgpr5 = COPY [[COPY3]](s96)
+    ; GFX9: [[LOAD1:%[0-9]+]]:_(s96) = G_LOAD [[PTR_ADD]](p3) :: (load 12, align 4, addrspace 3)
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s96) = COPY [[LOAD]](s96)
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s96) = COPY [[LOAD1]](s96)
+    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[COPY1]](s96)
+    ; GFX9: $vgpr3_vgpr4_vgpr5 = COPY [[COPY2]](s96)
     %0:_(p3) = COPY $vgpr0
     %1:_(<2 x s96>) = G_LOAD %0 :: (load 24, align 16, addrspace 3)
     %2:_(s96) = G_EXTRACT %1, 0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-udiv.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-udiv.mir
index 5a0d3879e55b2..d9204693d5183 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-udiv.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-udiv.mir
@@ -1,7 +1,7 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX6 %s
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer  -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX8 %s
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -O0 -run-pass=legalizer  -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX9 %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer -o - %s | FileCheck -check-prefix=GFX6 %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer -o - %s | FileCheck -check-prefix=GFX8 %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -O0 -run-pass=legalizer -o - %s | FileCheck -check-prefix=GFX9 %s
 
 ---
 name: test_udiv_s32
@@ -297,18 +297,462 @@ body: |
     ; GFX6-LABEL: name: test_udiv_s64
     ; GFX6: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX6: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX6: [[UDIV:%[0-9]+]]:_(s64) = G_UDIV [[COPY]], [[COPY1]]
-    ; GFX6: $vgpr0_vgpr1 = COPY [[UDIV]](s64)
+    ; GFX6: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV]](s32)
+    ; GFX6: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV1]](s32)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C]]
+    ; GFX6: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX6: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX6: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C2]]
+    ; GFX6: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX6: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C3]]
+    ; GFX6: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX6: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX6: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX6: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C4]](s64)
+    ; GFX6: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX6: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV2]], [[UV4]]
+    ; GFX6: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV3]], [[UV5]], [[USUBO1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX6: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX6: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX6: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX6: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO1]](s1)
+    ; GFX6: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UADDO]], [[UMULH1]]
+    ; GFX6: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO3]](s1)
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX6: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX6: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX6: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX6: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX6: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH3]]
+    ; GFX6: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX6: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[UADDO6]], [[ADD2]]
+    ; GFX6: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX6: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX6: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX6: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX6: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO8]]
+    ; GFX6: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO11]]
+    ; GFX6: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX6: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO10]]
+    ; GFX6: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO10]]
+    ; GFX6: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE]]
+    ; GFX6: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO10]]
+    ; GFX6: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX6: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX6: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[MUL6]]
+    ; GFX6: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO10]], [[ADD8]]
+    ; GFX6: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[MUL6]]
+    ; GFX6: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX6: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX6: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[UADDO12]], [[UMULH6]]
+    ; GFX6: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO15]](s1)
+    ; GFX6: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX6: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[ADD8]]
+    ; GFX6: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[MUL6]]
+    ; GFX6: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[ADD8]]
+    ; GFX6: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX6: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX6: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH8]]
+    ; GFX6: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX6: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX6: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[UADDO18]], [[ADD9]]
+    ; GFX6: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX6: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX6: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[ADD8]]
+    ; GFX6: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX6: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX6: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[UADDO20]]
+    ; GFX6: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO11]]
+    ; GFX6: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[UADDE2]], [[C5]], [[UADDO23]]
+    ; GFX6: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX6: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX6: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV9]], [[UADDO22]]
+    ; GFX6: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV8]], [[UADDE4]]
+    ; GFX6: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV8]], [[UADDO22]]
+    ; GFX6: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX6: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX6: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO24]], [[UMULH10]]
+    ; GFX6: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO27]](s1)
+    ; GFX6: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX6: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV9]], [[UADDE4]]
+    ; GFX6: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV9]], [[UADDO22]]
+    ; GFX6: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV8]], [[UADDE4]]
+    ; GFX6: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX6: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX6: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH12]]
+    ; GFX6: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX6: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX6: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[UADDO30]], [[ADD13]]
+    ; GFX6: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX6: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX6: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV9]], [[UADDE4]]
+    ; GFX6: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX6: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO32]](s32), [[ADD16]](s32)
+    ; GFX6: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX6: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV10]], [[UADDO32]]
+    ; GFX6: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV11]], [[UADDO32]]
+    ; GFX6: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV10]], [[ADD16]]
+    ; GFX6: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV10]], [[UADDO32]]
+    ; GFX6: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX6: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX6: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV6]], [[MUL15]]
+    ; GFX6: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV7]], [[ADD18]], [[USUBO3]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV7]], [[ADD18]]
+    ; GFX6: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV13]]
+    ; GFX6: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV12]]
+    ; GFX6: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV13]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX6: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV12]]
+    ; GFX6: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV13]], [[USUBO3]]
+    ; GFX6: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C5]], [[USUBO5]]
+    ; GFX6: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; GFX6: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C6]](s64)
+    ; GFX6: [[UADDO34:%[0-9]+]]:_(s32), [[UADDO35:%[0-9]+]]:_(s1) = G_UADDO [[UADDO32]], [[UV14]]
+    ; GFX6: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[ADD16]], [[UV15]], [[UADDO35]]
+    ; GFX6: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO34]](s32), [[UADDE6]](s32)
+    ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV13]]
+    ; GFX6: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX6: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV12]]
+    ; GFX6: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX6: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV13]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX6: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C6]](s64)
+    ; GFX6: [[UADDO36:%[0-9]+]]:_(s32), [[UADDO37:%[0-9]+]]:_(s1) = G_UADDO [[UADDO34]], [[UV16]]
+    ; GFX6: [[UADDE8:%[0-9]+]]:_(s32), [[UADDE9:%[0-9]+]]:_(s1) = G_UADDE [[UADDE6]], [[UV17]], [[UADDO37]]
+    ; GFX6: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO36]](s32), [[UADDE8]](s32)
+    ; GFX6: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV12]]
+    ; GFX6: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV13]], [[USUBO5]]
+    ; GFX6: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C5]], [[USUBO7]]
+    ; GFX6: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C5]]
+    ; GFX6: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV2]], [[MV1]]
+    ; GFX6: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C5]]
+    ; GFX6: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV]]
+    ; GFX6: $vgpr0_vgpr1 = COPY [[SELECT3]](s64)
     ; GFX8-LABEL: name: test_udiv_s64
     ; GFX8: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX8: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX8: [[UDIV:%[0-9]+]]:_(s64) = G_UDIV [[COPY]], [[COPY1]]
-    ; GFX8: $vgpr0_vgpr1 = COPY [[UDIV]](s64)
+    ; GFX8: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV]](s32)
+    ; GFX8: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV1]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C]]
+    ; GFX8: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX8: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX8: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C2]]
+    ; GFX8: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX8: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C3]]
+    ; GFX8: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX8: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX8: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX8: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C4]](s64)
+    ; GFX8: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX8: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV2]], [[UV4]]
+    ; GFX8: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV3]], [[UV5]], [[USUBO1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX8: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO1]](s1)
+    ; GFX8: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UADDO]], [[UMULH1]]
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO3]](s1)
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX8: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX8: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX8: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX8: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX8: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH3]]
+    ; GFX8: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX8: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[UADDO6]], [[ADD2]]
+    ; GFX8: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX8: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX8: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX8: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX8: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO8]]
+    ; GFX8: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO11]]
+    ; GFX8: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX8: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO10]]
+    ; GFX8: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO10]]
+    ; GFX8: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE]]
+    ; GFX8: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO10]]
+    ; GFX8: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX8: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX8: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[MUL6]]
+    ; GFX8: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO10]], [[ADD8]]
+    ; GFX8: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[MUL6]]
+    ; GFX8: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX8: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX8: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[UADDO12]], [[UMULH6]]
+    ; GFX8: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO15]](s1)
+    ; GFX8: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX8: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[ADD8]]
+    ; GFX8: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[MUL6]]
+    ; GFX8: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[ADD8]]
+    ; GFX8: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX8: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX8: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH8]]
+    ; GFX8: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX8: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX8: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[UADDO18]], [[ADD9]]
+    ; GFX8: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX8: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX8: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[ADD8]]
+    ; GFX8: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX8: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[UADDO20]]
+    ; GFX8: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO11]]
+    ; GFX8: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[UADDE2]], [[C5]], [[UADDO23]]
+    ; GFX8: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX8: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX8: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV9]], [[UADDO22]]
+    ; GFX8: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV8]], [[UADDE4]]
+    ; GFX8: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV8]], [[UADDO22]]
+    ; GFX8: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX8: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX8: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO24]], [[UMULH10]]
+    ; GFX8: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO27]](s1)
+    ; GFX8: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX8: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV9]], [[UADDE4]]
+    ; GFX8: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV9]], [[UADDO22]]
+    ; GFX8: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV8]], [[UADDE4]]
+    ; GFX8: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX8: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX8: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH12]]
+    ; GFX8: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX8: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX8: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[UADDO30]], [[ADD13]]
+    ; GFX8: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX8: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX8: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV9]], [[UADDE4]]
+    ; GFX8: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX8: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO32]](s32), [[ADD16]](s32)
+    ; GFX8: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX8: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV10]], [[UADDO32]]
+    ; GFX8: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV11]], [[UADDO32]]
+    ; GFX8: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV10]], [[ADD16]]
+    ; GFX8: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV10]], [[UADDO32]]
+    ; GFX8: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX8: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX8: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV6]], [[MUL15]]
+    ; GFX8: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV7]], [[ADD18]], [[USUBO3]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV7]], [[ADD18]]
+    ; GFX8: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV13]]
+    ; GFX8: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV12]]
+    ; GFX8: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV13]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX8: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV12]]
+    ; GFX8: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV13]], [[USUBO3]]
+    ; GFX8: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C5]], [[USUBO5]]
+    ; GFX8: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; GFX8: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C6]](s64)
+    ; GFX8: [[UADDO34:%[0-9]+]]:_(s32), [[UADDO35:%[0-9]+]]:_(s1) = G_UADDO [[UADDO32]], [[UV14]]
+    ; GFX8: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[ADD16]], [[UV15]], [[UADDO35]]
+    ; GFX8: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO34]](s32), [[UADDE6]](s32)
+    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV13]]
+    ; GFX8: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX8: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV12]]
+    ; GFX8: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX8: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV13]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX8: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C6]](s64)
+    ; GFX8: [[UADDO36:%[0-9]+]]:_(s32), [[UADDO37:%[0-9]+]]:_(s1) = G_UADDO [[UADDO34]], [[UV16]]
+    ; GFX8: [[UADDE8:%[0-9]+]]:_(s32), [[UADDE9:%[0-9]+]]:_(s1) = G_UADDE [[UADDE6]], [[UV17]], [[UADDO37]]
+    ; GFX8: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO36]](s32), [[UADDE8]](s32)
+    ; GFX8: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV12]]
+    ; GFX8: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV13]], [[USUBO5]]
+    ; GFX8: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C5]], [[USUBO7]]
+    ; GFX8: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C5]]
+    ; GFX8: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV2]], [[MV1]]
+    ; GFX8: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C5]]
+    ; GFX8: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV]]
+    ; GFX8: $vgpr0_vgpr1 = COPY [[SELECT3]](s64)
     ; GFX9-LABEL: name: test_udiv_s64
     ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX9: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX9: [[UDIV:%[0-9]+]]:_(s64) = G_UDIV [[COPY]], [[COPY1]]
-    ; GFX9: $vgpr0_vgpr1 = COPY [[UDIV]](s64)
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV]](s32)
+    ; GFX9: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV1]](s32)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C]]
+    ; GFX9: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX9: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX9: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C2]]
+    ; GFX9: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX9: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C3]]
+    ; GFX9: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX9: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX9: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C4]](s64)
+    ; GFX9: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX9: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV2]], [[UV4]]
+    ; GFX9: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV3]], [[UV5]], [[USUBO1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX9: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX9: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO1]](s1)
+    ; GFX9: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UADDO]], [[UMULH1]]
+    ; GFX9: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO3]](s1)
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX9: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX9: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX9: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX9: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX9: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH3]]
+    ; GFX9: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX9: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[UADDO6]], [[ADD2]]
+    ; GFX9: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX9: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX9: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX9: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX9: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO8]]
+    ; GFX9: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO11]]
+    ; GFX9: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX9: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO10]]
+    ; GFX9: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO10]]
+    ; GFX9: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE]]
+    ; GFX9: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO10]]
+    ; GFX9: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX9: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX9: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[MUL6]]
+    ; GFX9: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO10]], [[ADD8]]
+    ; GFX9: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[MUL6]]
+    ; GFX9: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX9: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX9: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[UADDO12]], [[UMULH6]]
+    ; GFX9: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO15]](s1)
+    ; GFX9: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX9: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[ADD8]]
+    ; GFX9: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[MUL6]]
+    ; GFX9: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[ADD8]]
+    ; GFX9: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX9: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX9: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH8]]
+    ; GFX9: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX9: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX9: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[UADDO18]], [[ADD9]]
+    ; GFX9: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX9: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX9: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[ADD8]]
+    ; GFX9: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX9: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[UADDO20]]
+    ; GFX9: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO11]]
+    ; GFX9: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[UADDE2]], [[C5]], [[UADDO23]]
+    ; GFX9: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX9: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX9: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV9]], [[UADDO22]]
+    ; GFX9: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV8]], [[UADDE4]]
+    ; GFX9: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV8]], [[UADDO22]]
+    ; GFX9: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX9: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX9: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO24]], [[UMULH10]]
+    ; GFX9: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO27]](s1)
+    ; GFX9: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX9: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV9]], [[UADDE4]]
+    ; GFX9: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV9]], [[UADDO22]]
+    ; GFX9: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV8]], [[UADDE4]]
+    ; GFX9: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX9: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX9: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH12]]
+    ; GFX9: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX9: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX9: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[UADDO30]], [[ADD13]]
+    ; GFX9: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX9: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX9: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV9]], [[UADDE4]]
+    ; GFX9: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO32]](s32), [[ADD16]](s32)
+    ; GFX9: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX9: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV10]], [[UADDO32]]
+    ; GFX9: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV11]], [[UADDO32]]
+    ; GFX9: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV10]], [[ADD16]]
+    ; GFX9: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV10]], [[UADDO32]]
+    ; GFX9: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX9: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX9: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV6]], [[MUL15]]
+    ; GFX9: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV7]], [[ADD18]], [[USUBO3]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV7]], [[ADD18]]
+    ; GFX9: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV13]]
+    ; GFX9: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV12]]
+    ; GFX9: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV13]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX9: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV12]]
+    ; GFX9: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV13]], [[USUBO3]]
+    ; GFX9: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C5]], [[USUBO5]]
+    ; GFX9: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; GFX9: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C6]](s64)
+    ; GFX9: [[UADDO34:%[0-9]+]]:_(s32), [[UADDO35:%[0-9]+]]:_(s1) = G_UADDO [[UADDO32]], [[UV14]]
+    ; GFX9: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[ADD16]], [[UV15]], [[UADDO35]]
+    ; GFX9: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO34]](s32), [[UADDE6]](s32)
+    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV13]]
+    ; GFX9: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX9: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV12]]
+    ; GFX9: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX9: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV13]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX9: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C6]](s64)
+    ; GFX9: [[UADDO36:%[0-9]+]]:_(s32), [[UADDO37:%[0-9]+]]:_(s1) = G_UADDO [[UADDO34]], [[UV16]]
+    ; GFX9: [[UADDE8:%[0-9]+]]:_(s32), [[UADDE9:%[0-9]+]]:_(s1) = G_UADDE [[UADDE6]], [[UV17]], [[UADDO37]]
+    ; GFX9: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO36]](s32), [[UADDE8]](s32)
+    ; GFX9: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV12]]
+    ; GFX9: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV13]], [[USUBO5]]
+    ; GFX9: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C5]], [[USUBO7]]
+    ; GFX9: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C5]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV2]], [[MV1]]
+    ; GFX9: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C5]]
+    ; GFX9: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV]]
+    ; GFX9: $vgpr0_vgpr1 = COPY [[SELECT3]](s64)
     %0:_(s64) = COPY $vgpr0_vgpr1
     %1:_(s64) = COPY $vgpr2_vgpr3
     %2:_(s64) = G_UDIV %0, %1
@@ -326,27 +770,906 @@ body: |
     ; GFX6: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
     ; GFX6: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
     ; GFX6: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
-    ; GFX6: [[UDIV:%[0-9]+]]:_(s64) = G_UDIV [[UV]], [[UV2]]
-    ; GFX6: [[UDIV1:%[0-9]+]]:_(s64) = G_UDIV [[UV1]], [[UV3]]
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[UDIV]](s64), [[UDIV1]](s64)
+    ; GFX6: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV4]](s32)
+    ; GFX6: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV5]](s32)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C]]
+    ; GFX6: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX6: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX6: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C2]]
+    ; GFX6: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX6: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C3]]
+    ; GFX6: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX6: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX6: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX6: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C4]](s64)
+    ; GFX6: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX6: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV6]], [[UV8]]
+    ; GFX6: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV7]], [[UV9]], [[USUBO1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX6: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX6: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX6: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX6: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO1]](s1)
+    ; GFX6: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UADDO]], [[UMULH1]]
+    ; GFX6: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO3]](s1)
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX6: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX6: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX6: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX6: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX6: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH3]]
+    ; GFX6: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX6: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[UADDO6]], [[ADD2]]
+    ; GFX6: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX6: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX6: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX6: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX6: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO8]]
+    ; GFX6: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO11]]
+    ; GFX6: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX6: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO10]]
+    ; GFX6: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO10]]
+    ; GFX6: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE]]
+    ; GFX6: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO10]]
+    ; GFX6: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX6: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX6: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[MUL6]]
+    ; GFX6: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO10]], [[ADD8]]
+    ; GFX6: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[MUL6]]
+    ; GFX6: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX6: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX6: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[UADDO12]], [[UMULH6]]
+    ; GFX6: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO15]](s1)
+    ; GFX6: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX6: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[ADD8]]
+    ; GFX6: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[MUL6]]
+    ; GFX6: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[ADD8]]
+    ; GFX6: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX6: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX6: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH8]]
+    ; GFX6: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX6: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX6: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[UADDO18]], [[ADD9]]
+    ; GFX6: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX6: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX6: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[ADD8]]
+    ; GFX6: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX6: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX6: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[UADDO20]]
+    ; GFX6: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO11]]
+    ; GFX6: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[UADDE2]], [[C5]], [[UADDO23]]
+    ; GFX6: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+    ; GFX6: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+    ; GFX6: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV13]], [[UADDO22]]
+    ; GFX6: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV12]], [[UADDE4]]
+    ; GFX6: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV12]], [[UADDO22]]
+    ; GFX6: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX6: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX6: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO24]], [[UMULH10]]
+    ; GFX6: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO27]](s1)
+    ; GFX6: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX6: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV13]], [[UADDE4]]
+    ; GFX6: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV13]], [[UADDO22]]
+    ; GFX6: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV12]], [[UADDE4]]
+    ; GFX6: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX6: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX6: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH12]]
+    ; GFX6: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX6: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX6: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[UADDO30]], [[ADD13]]
+    ; GFX6: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX6: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX6: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV13]], [[UADDE4]]
+    ; GFX6: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX6: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO32]](s32), [[ADD16]](s32)
+    ; GFX6: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX6: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV14]], [[UADDO32]]
+    ; GFX6: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV15]], [[UADDO32]]
+    ; GFX6: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV14]], [[ADD16]]
+    ; GFX6: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV14]], [[UADDO32]]
+    ; GFX6: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX6: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX6: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV10]], [[MUL15]]
+    ; GFX6: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV11]], [[ADD18]], [[USUBO3]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV11]], [[ADD18]]
+    ; GFX6: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV17]]
+    ; GFX6: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV16]]
+    ; GFX6: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV17]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX6: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV16]]
+    ; GFX6: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV17]], [[USUBO3]]
+    ; GFX6: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C5]], [[USUBO5]]
+    ; GFX6: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; GFX6: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C6]](s64)
+    ; GFX6: [[UADDO34:%[0-9]+]]:_(s32), [[UADDO35:%[0-9]+]]:_(s1) = G_UADDO [[UADDO32]], [[UV18]]
+    ; GFX6: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[ADD16]], [[UV19]], [[UADDO35]]
+    ; GFX6: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO34]](s32), [[UADDE6]](s32)
+    ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV17]]
+    ; GFX6: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX6: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV16]]
+    ; GFX6: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX6: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV17]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX6: [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C6]](s64)
+    ; GFX6: [[UADDO36:%[0-9]+]]:_(s32), [[UADDO37:%[0-9]+]]:_(s1) = G_UADDO [[UADDO34]], [[UV20]]
+    ; GFX6: [[UADDE8:%[0-9]+]]:_(s32), [[UADDE9:%[0-9]+]]:_(s1) = G_UADDE [[UADDE6]], [[UV21]], [[UADDO37]]
+    ; GFX6: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO36]](s32), [[UADDE8]](s32)
+    ; GFX6: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV16]]
+    ; GFX6: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV17]], [[USUBO5]]
+    ; GFX6: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C5]], [[USUBO7]]
+    ; GFX6: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C5]]
+    ; GFX6: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV2]], [[MV1]]
+    ; GFX6: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C5]]
+    ; GFX6: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV]]
+    ; GFX6: [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX6: [[UITOFP2:%[0-9]+]]:_(s32) = G_UITOFP [[UV22]](s32)
+    ; GFX6: [[UITOFP3:%[0-9]+]]:_(s32) = G_UITOFP [[UV23]](s32)
+    ; GFX6: [[C7:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[FMUL4:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP3]], [[C7]]
+    ; GFX6: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FMUL4]], [[UITOFP2]]
+    ; GFX6: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD2]](s32)
+    ; GFX6: [[C8:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX6: [[FMUL5:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C8]]
+    ; GFX6: [[C9:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX6: [[FMUL6:%[0-9]+]]:_(s32) = G_FMUL [[FMUL5]], [[C9]]
+    ; GFX6: [[INTRINSIC_TRUNC1:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL6]]
+    ; GFX6: [[C10:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX6: [[FMUL7:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC1]], [[C10]]
+    ; GFX6: [[FADD3:%[0-9]+]]:_(s32) = G_FADD [[FMUL7]], [[FMUL5]]
+    ; GFX6: [[FPTOUI2:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD3]](s32)
+    ; GFX6: [[FPTOUI3:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC1]](s32)
+    ; GFX6: [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C4]](s64)
+    ; GFX6: [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX6: [[USUBO8:%[0-9]+]]:_(s32), [[USUBO9:%[0-9]+]]:_(s1) = G_USUBO [[UV24]], [[UV26]]
+    ; GFX6: [[USUBE12:%[0-9]+]]:_(s32), [[USUBE13:%[0-9]+]]:_(s1) = G_USUBE [[UV25]], [[UV27]], [[USUBO9]]
+    ; GFX6: [[MUL18:%[0-9]+]]:_(s32) = G_MUL [[USUBO8]], [[FPTOUI2]]
+    ; GFX6: [[MUL19:%[0-9]+]]:_(s32) = G_MUL [[USUBE12]], [[FPTOUI2]]
+    ; GFX6: [[MUL20:%[0-9]+]]:_(s32) = G_MUL [[USUBO8]], [[FPTOUI3]]
+    ; GFX6: [[UMULH15:%[0-9]+]]:_(s32) = G_UMULH [[USUBO8]], [[FPTOUI2]]
+    ; GFX6: [[ADD19:%[0-9]+]]:_(s32) = G_ADD [[MUL19]], [[MUL20]]
+    ; GFX6: [[ADD20:%[0-9]+]]:_(s32) = G_ADD [[ADD19]], [[UMULH15]]
+    ; GFX6: [[MUL21:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI3]], [[MUL18]]
+    ; GFX6: [[MUL22:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI2]], [[ADD20]]
+    ; GFX6: [[UMULH16:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI2]], [[MUL18]]
+    ; GFX6: [[UADDO38:%[0-9]+]]:_(s32), [[UADDO39:%[0-9]+]]:_(s1) = G_UADDO [[MUL21]], [[MUL22]]
+    ; GFX6: [[ZEXT15:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO39]](s1)
+    ; GFX6: [[UADDO40:%[0-9]+]]:_(s32), [[UADDO41:%[0-9]+]]:_(s1) = G_UADDO [[UADDO38]], [[UMULH16]]
+    ; GFX6: [[ZEXT16:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO41]](s1)
+    ; GFX6: [[ADD21:%[0-9]+]]:_(s32) = G_ADD [[ZEXT15]], [[ZEXT16]]
+    ; GFX6: [[MUL23:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI3]], [[ADD20]]
+    ; GFX6: [[UMULH17:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI3]], [[MUL18]]
+    ; GFX6: [[UMULH18:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI2]], [[ADD20]]
+    ; GFX6: [[UADDO42:%[0-9]+]]:_(s32), [[UADDO43:%[0-9]+]]:_(s1) = G_UADDO [[MUL23]], [[UMULH17]]
+    ; GFX6: [[ZEXT17:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO43]](s1)
+    ; GFX6: [[UADDO44:%[0-9]+]]:_(s32), [[UADDO45:%[0-9]+]]:_(s1) = G_UADDO [[UADDO42]], [[UMULH18]]
+    ; GFX6: [[ZEXT18:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO45]](s1)
+    ; GFX6: [[ADD22:%[0-9]+]]:_(s32) = G_ADD [[ZEXT17]], [[ZEXT18]]
+    ; GFX6: [[UADDO46:%[0-9]+]]:_(s32), [[UADDO47:%[0-9]+]]:_(s1) = G_UADDO [[UADDO44]], [[ADD21]]
+    ; GFX6: [[ZEXT19:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO47]](s1)
+    ; GFX6: [[ADD23:%[0-9]+]]:_(s32) = G_ADD [[ADD22]], [[ZEXT19]]
+    ; GFX6: [[UMULH19:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI3]], [[ADD20]]
+    ; GFX6: [[ADD24:%[0-9]+]]:_(s32) = G_ADD [[UMULH19]], [[ADD23]]
+    ; GFX6: [[UADDO48:%[0-9]+]]:_(s32), [[UADDO49:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI2]], [[UADDO46]]
+    ; GFX6: [[UADDE10:%[0-9]+]]:_(s32), [[UADDE11:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI3]], [[ADD24]], [[UADDO49]]
+    ; GFX6: [[ADD25:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI3]], [[ADD24]]
+    ; GFX6: [[MUL24:%[0-9]+]]:_(s32) = G_MUL [[USUBO8]], [[UADDO48]]
+    ; GFX6: [[MUL25:%[0-9]+]]:_(s32) = G_MUL [[USUBE12]], [[UADDO48]]
+    ; GFX6: [[MUL26:%[0-9]+]]:_(s32) = G_MUL [[USUBO8]], [[UADDE10]]
+    ; GFX6: [[UMULH20:%[0-9]+]]:_(s32) = G_UMULH [[USUBO8]], [[UADDO48]]
+    ; GFX6: [[ADD26:%[0-9]+]]:_(s32) = G_ADD [[MUL25]], [[MUL26]]
+    ; GFX6: [[ADD27:%[0-9]+]]:_(s32) = G_ADD [[ADD26]], [[UMULH20]]
+    ; GFX6: [[MUL27:%[0-9]+]]:_(s32) = G_MUL [[UADDE10]], [[MUL24]]
+    ; GFX6: [[MUL28:%[0-9]+]]:_(s32) = G_MUL [[UADDO48]], [[ADD27]]
+    ; GFX6: [[UMULH21:%[0-9]+]]:_(s32) = G_UMULH [[UADDO48]], [[MUL24]]
+    ; GFX6: [[UADDO50:%[0-9]+]]:_(s32), [[UADDO51:%[0-9]+]]:_(s1) = G_UADDO [[MUL27]], [[MUL28]]
+    ; GFX6: [[ZEXT20:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO51]](s1)
+    ; GFX6: [[UADDO52:%[0-9]+]]:_(s32), [[UADDO53:%[0-9]+]]:_(s1) = G_UADDO [[UADDO50]], [[UMULH21]]
+    ; GFX6: [[ZEXT21:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO53]](s1)
+    ; GFX6: [[ADD28:%[0-9]+]]:_(s32) = G_ADD [[ZEXT20]], [[ZEXT21]]
+    ; GFX6: [[MUL29:%[0-9]+]]:_(s32) = G_MUL [[UADDE10]], [[ADD27]]
+    ; GFX6: [[UMULH22:%[0-9]+]]:_(s32) = G_UMULH [[UADDE10]], [[MUL24]]
+    ; GFX6: [[UMULH23:%[0-9]+]]:_(s32) = G_UMULH [[UADDO48]], [[ADD27]]
+    ; GFX6: [[UADDO54:%[0-9]+]]:_(s32), [[UADDO55:%[0-9]+]]:_(s1) = G_UADDO [[MUL29]], [[UMULH22]]
+    ; GFX6: [[ZEXT22:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO55]](s1)
+    ; GFX6: [[UADDO56:%[0-9]+]]:_(s32), [[UADDO57:%[0-9]+]]:_(s1) = G_UADDO [[UADDO54]], [[UMULH23]]
+    ; GFX6: [[ZEXT23:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO57]](s1)
+    ; GFX6: [[ADD29:%[0-9]+]]:_(s32) = G_ADD [[ZEXT22]], [[ZEXT23]]
+    ; GFX6: [[UADDO58:%[0-9]+]]:_(s32), [[UADDO59:%[0-9]+]]:_(s1) = G_UADDO [[UADDO56]], [[ADD28]]
+    ; GFX6: [[ZEXT24:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO59]](s1)
+    ; GFX6: [[ADD30:%[0-9]+]]:_(s32) = G_ADD [[ADD29]], [[ZEXT24]]
+    ; GFX6: [[UMULH24:%[0-9]+]]:_(s32) = G_UMULH [[UADDE10]], [[ADD27]]
+    ; GFX6: [[ADD31:%[0-9]+]]:_(s32) = G_ADD [[UMULH24]], [[ADD30]]
+    ; GFX6: [[UADDO60:%[0-9]+]]:_(s32), [[UADDO61:%[0-9]+]]:_(s1) = G_UADDO [[UADDO48]], [[UADDO58]]
+    ; GFX6: [[UADDE12:%[0-9]+]]:_(s32), [[UADDE13:%[0-9]+]]:_(s1) = G_UADDE [[ADD25]], [[ADD31]], [[UADDO49]]
+    ; GFX6: [[UADDE14:%[0-9]+]]:_(s32), [[UADDE15:%[0-9]+]]:_(s1) = G_UADDE [[UADDE12]], [[C5]], [[UADDO61]]
+    ; GFX6: [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+    ; GFX6: [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+    ; GFX6: [[MUL30:%[0-9]+]]:_(s32) = G_MUL [[UV31]], [[UADDO60]]
+    ; GFX6: [[MUL31:%[0-9]+]]:_(s32) = G_MUL [[UV30]], [[UADDE14]]
+    ; GFX6: [[UMULH25:%[0-9]+]]:_(s32) = G_UMULH [[UV30]], [[UADDO60]]
+    ; GFX6: [[UADDO62:%[0-9]+]]:_(s32), [[UADDO63:%[0-9]+]]:_(s1) = G_UADDO [[MUL30]], [[MUL31]]
+    ; GFX6: [[ZEXT25:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO63]](s1)
+    ; GFX6: [[UADDO64:%[0-9]+]]:_(s32), [[UADDO65:%[0-9]+]]:_(s1) = G_UADDO [[UADDO62]], [[UMULH25]]
+    ; GFX6: [[ZEXT26:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO65]](s1)
+    ; GFX6: [[ADD32:%[0-9]+]]:_(s32) = G_ADD [[ZEXT25]], [[ZEXT26]]
+    ; GFX6: [[MUL32:%[0-9]+]]:_(s32) = G_MUL [[UV31]], [[UADDE14]]
+    ; GFX6: [[UMULH26:%[0-9]+]]:_(s32) = G_UMULH [[UV31]], [[UADDO60]]
+    ; GFX6: [[UMULH27:%[0-9]+]]:_(s32) = G_UMULH [[UV30]], [[UADDE14]]
+    ; GFX6: [[UADDO66:%[0-9]+]]:_(s32), [[UADDO67:%[0-9]+]]:_(s1) = G_UADDO [[MUL32]], [[UMULH26]]
+    ; GFX6: [[ZEXT27:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO67]](s1)
+    ; GFX6: [[UADDO68:%[0-9]+]]:_(s32), [[UADDO69:%[0-9]+]]:_(s1) = G_UADDO [[UADDO66]], [[UMULH27]]
+    ; GFX6: [[ZEXT28:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO69]](s1)
+    ; GFX6: [[ADD33:%[0-9]+]]:_(s32) = G_ADD [[ZEXT27]], [[ZEXT28]]
+    ; GFX6: [[UADDO70:%[0-9]+]]:_(s32), [[UADDO71:%[0-9]+]]:_(s1) = G_UADDO [[UADDO68]], [[ADD32]]
+    ; GFX6: [[ZEXT29:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO71]](s1)
+    ; GFX6: [[ADD34:%[0-9]+]]:_(s32) = G_ADD [[ADD33]], [[ZEXT29]]
+    ; GFX6: [[UMULH28:%[0-9]+]]:_(s32) = G_UMULH [[UV31]], [[UADDE14]]
+    ; GFX6: [[ADD35:%[0-9]+]]:_(s32) = G_ADD [[UMULH28]], [[ADD34]]
+    ; GFX6: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO70]](s32), [[ADD35]](s32)
+    ; GFX6: [[UV32:%[0-9]+]]:_(s32), [[UV33:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX6: [[MUL33:%[0-9]+]]:_(s32) = G_MUL [[UV32]], [[UADDO70]]
+    ; GFX6: [[MUL34:%[0-9]+]]:_(s32) = G_MUL [[UV33]], [[UADDO70]]
+    ; GFX6: [[MUL35:%[0-9]+]]:_(s32) = G_MUL [[UV32]], [[ADD35]]
+    ; GFX6: [[UMULH29:%[0-9]+]]:_(s32) = G_UMULH [[UV32]], [[UADDO70]]
+    ; GFX6: [[ADD36:%[0-9]+]]:_(s32) = G_ADD [[MUL34]], [[MUL35]]
+    ; GFX6: [[ADD37:%[0-9]+]]:_(s32) = G_ADD [[ADD36]], [[UMULH29]]
+    ; GFX6: [[USUBO10:%[0-9]+]]:_(s32), [[USUBO11:%[0-9]+]]:_(s1) = G_USUBO [[UV28]], [[MUL33]]
+    ; GFX6: [[USUBE14:%[0-9]+]]:_(s32), [[USUBE15:%[0-9]+]]:_(s1) = G_USUBE [[UV29]], [[ADD37]], [[USUBO11]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[UV29]], [[ADD37]]
+    ; GFX6: [[UV34:%[0-9]+]]:_(s32), [[UV35:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX6: [[ICMP8:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE14]](s32), [[UV35]]
+    ; GFX6: [[SEXT4:%[0-9]+]]:_(s32) = G_SEXT [[ICMP8]](s1)
+    ; GFX6: [[ICMP9:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO10]](s32), [[UV34]]
+    ; GFX6: [[SEXT5:%[0-9]+]]:_(s32) = G_SEXT [[ICMP9]](s1)
+    ; GFX6: [[ICMP10:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE14]](s32), [[UV35]]
+    ; GFX6: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP10]](s1), [[SEXT5]], [[SEXT4]]
+    ; GFX6: [[USUBO12:%[0-9]+]]:_(s32), [[USUBO13:%[0-9]+]]:_(s1) = G_USUBO [[USUBO10]], [[UV34]]
+    ; GFX6: [[USUBE16:%[0-9]+]]:_(s32), [[USUBE17:%[0-9]+]]:_(s1) = G_USUBE [[SUB1]], [[UV35]], [[USUBO11]]
+    ; GFX6: [[USUBE18:%[0-9]+]]:_(s32), [[USUBE19:%[0-9]+]]:_(s1) = G_USUBE [[USUBE16]], [[C5]], [[USUBO13]]
+    ; GFX6: [[UV36:%[0-9]+]]:_(s32), [[UV37:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C6]](s64)
+    ; GFX6: [[UADDO72:%[0-9]+]]:_(s32), [[UADDO73:%[0-9]+]]:_(s1) = G_UADDO [[UADDO70]], [[UV36]]
+    ; GFX6: [[UADDE16:%[0-9]+]]:_(s32), [[UADDE17:%[0-9]+]]:_(s1) = G_UADDE [[ADD35]], [[UV37]], [[UADDO73]]
+    ; GFX6: [[MV4:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO72]](s32), [[UADDE16]](s32)
+    ; GFX6: [[ICMP11:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE18]](s32), [[UV35]]
+    ; GFX6: [[SEXT6:%[0-9]+]]:_(s32) = G_SEXT [[ICMP11]](s1)
+    ; GFX6: [[ICMP12:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO12]](s32), [[UV34]]
+    ; GFX6: [[SEXT7:%[0-9]+]]:_(s32) = G_SEXT [[ICMP12]](s1)
+    ; GFX6: [[ICMP13:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE18]](s32), [[UV35]]
+    ; GFX6: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP13]](s1), [[SEXT7]], [[SEXT6]]
+    ; GFX6: [[UV38:%[0-9]+]]:_(s32), [[UV39:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C6]](s64)
+    ; GFX6: [[UADDO74:%[0-9]+]]:_(s32), [[UADDO75:%[0-9]+]]:_(s1) = G_UADDO [[UADDO72]], [[UV38]]
+    ; GFX6: [[UADDE18:%[0-9]+]]:_(s32), [[UADDE19:%[0-9]+]]:_(s1) = G_UADDE [[UADDE16]], [[UV39]], [[UADDO75]]
+    ; GFX6: [[MV5:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO74]](s32), [[UADDE18]](s32)
+    ; GFX6: [[USUBO14:%[0-9]+]]:_(s32), [[USUBO15:%[0-9]+]]:_(s1) = G_USUBO [[USUBO12]], [[UV34]]
+    ; GFX6: [[USUBE20:%[0-9]+]]:_(s32), [[USUBE21:%[0-9]+]]:_(s1) = G_USUBE [[USUBE16]], [[UV35]], [[USUBO13]]
+    ; GFX6: [[USUBE22:%[0-9]+]]:_(s32), [[USUBE23:%[0-9]+]]:_(s1) = G_USUBE [[USUBE20]], [[C5]], [[USUBO15]]
+    ; GFX6: [[ICMP14:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT5]](s32), [[C5]]
+    ; GFX6: [[SELECT6:%[0-9]+]]:_(s64) = G_SELECT [[ICMP14]](s1), [[MV5]], [[MV4]]
+    ; GFX6: [[ICMP15:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT4]](s32), [[C5]]
+    ; GFX6: [[SELECT7:%[0-9]+]]:_(s64) = G_SELECT [[ICMP15]](s1), [[SELECT6]], [[MV3]]
+    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SELECT3]](s64), [[SELECT7]](s64)
     ; GFX6: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     ; GFX8-LABEL: name: test_udiv_v2s64
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
     ; GFX8: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
     ; GFX8: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
-    ; GFX8: [[UDIV:%[0-9]+]]:_(s64) = G_UDIV [[UV]], [[UV2]]
-    ; GFX8: [[UDIV1:%[0-9]+]]:_(s64) = G_UDIV [[UV1]], [[UV3]]
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[UDIV]](s64), [[UDIV1]](s64)
+    ; GFX8: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV4]](s32)
+    ; GFX8: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV5]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C]]
+    ; GFX8: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX8: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX8: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C2]]
+    ; GFX8: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX8: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C3]]
+    ; GFX8: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX8: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX8: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX8: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C4]](s64)
+    ; GFX8: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX8: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV6]], [[UV8]]
+    ; GFX8: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV7]], [[UV9]], [[USUBO1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX8: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO1]](s1)
+    ; GFX8: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UADDO]], [[UMULH1]]
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO3]](s1)
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX8: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX8: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX8: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX8: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX8: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH3]]
+    ; GFX8: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX8: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[UADDO6]], [[ADD2]]
+    ; GFX8: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX8: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX8: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX8: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX8: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO8]]
+    ; GFX8: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO11]]
+    ; GFX8: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX8: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO10]]
+    ; GFX8: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO10]]
+    ; GFX8: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE]]
+    ; GFX8: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO10]]
+    ; GFX8: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX8: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX8: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[MUL6]]
+    ; GFX8: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO10]], [[ADD8]]
+    ; GFX8: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[MUL6]]
+    ; GFX8: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX8: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX8: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[UADDO12]], [[UMULH6]]
+    ; GFX8: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO15]](s1)
+    ; GFX8: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX8: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[ADD8]]
+    ; GFX8: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[MUL6]]
+    ; GFX8: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[ADD8]]
+    ; GFX8: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX8: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX8: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH8]]
+    ; GFX8: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX8: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX8: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[UADDO18]], [[ADD9]]
+    ; GFX8: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX8: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX8: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[ADD8]]
+    ; GFX8: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX8: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[UADDO20]]
+    ; GFX8: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO11]]
+    ; GFX8: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[UADDE2]], [[C5]], [[UADDO23]]
+    ; GFX8: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+    ; GFX8: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+    ; GFX8: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV13]], [[UADDO22]]
+    ; GFX8: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV12]], [[UADDE4]]
+    ; GFX8: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV12]], [[UADDO22]]
+    ; GFX8: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX8: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX8: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO24]], [[UMULH10]]
+    ; GFX8: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO27]](s1)
+    ; GFX8: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX8: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV13]], [[UADDE4]]
+    ; GFX8: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV13]], [[UADDO22]]
+    ; GFX8: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV12]], [[UADDE4]]
+    ; GFX8: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX8: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX8: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH12]]
+    ; GFX8: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX8: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX8: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[UADDO30]], [[ADD13]]
+    ; GFX8: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX8: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX8: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV13]], [[UADDE4]]
+    ; GFX8: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX8: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO32]](s32), [[ADD16]](s32)
+    ; GFX8: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX8: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV14]], [[UADDO32]]
+    ; GFX8: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV15]], [[UADDO32]]
+    ; GFX8: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV14]], [[ADD16]]
+    ; GFX8: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV14]], [[UADDO32]]
+    ; GFX8: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX8: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX8: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV10]], [[MUL15]]
+    ; GFX8: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV11]], [[ADD18]], [[USUBO3]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV11]], [[ADD18]]
+    ; GFX8: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV17]]
+    ; GFX8: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV16]]
+    ; GFX8: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV17]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX8: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV16]]
+    ; GFX8: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV17]], [[USUBO3]]
+    ; GFX8: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C5]], [[USUBO5]]
+    ; GFX8: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; GFX8: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C6]](s64)
+    ; GFX8: [[UADDO34:%[0-9]+]]:_(s32), [[UADDO35:%[0-9]+]]:_(s1) = G_UADDO [[UADDO32]], [[UV18]]
+    ; GFX8: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[ADD16]], [[UV19]], [[UADDO35]]
+    ; GFX8: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO34]](s32), [[UADDE6]](s32)
+    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV17]]
+    ; GFX8: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX8: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV16]]
+    ; GFX8: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX8: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV17]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX8: [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C6]](s64)
+    ; GFX8: [[UADDO36:%[0-9]+]]:_(s32), [[UADDO37:%[0-9]+]]:_(s1) = G_UADDO [[UADDO34]], [[UV20]]
+    ; GFX8: [[UADDE8:%[0-9]+]]:_(s32), [[UADDE9:%[0-9]+]]:_(s1) = G_UADDE [[UADDE6]], [[UV21]], [[UADDO37]]
+    ; GFX8: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO36]](s32), [[UADDE8]](s32)
+    ; GFX8: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV16]]
+    ; GFX8: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV17]], [[USUBO5]]
+    ; GFX8: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C5]], [[USUBO7]]
+    ; GFX8: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C5]]
+    ; GFX8: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV2]], [[MV1]]
+    ; GFX8: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C5]]
+    ; GFX8: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV]]
+    ; GFX8: [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX8: [[UITOFP2:%[0-9]+]]:_(s32) = G_UITOFP [[UV22]](s32)
+    ; GFX8: [[UITOFP3:%[0-9]+]]:_(s32) = G_UITOFP [[UV23]](s32)
+    ; GFX8: [[C7:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[FMUL4:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP3]], [[C7]]
+    ; GFX8: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FMUL4]], [[UITOFP2]]
+    ; GFX8: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD2]](s32)
+    ; GFX8: [[C8:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX8: [[FMUL5:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C8]]
+    ; GFX8: [[C9:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX8: [[FMUL6:%[0-9]+]]:_(s32) = G_FMUL [[FMUL5]], [[C9]]
+    ; GFX8: [[INTRINSIC_TRUNC1:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL6]]
+    ; GFX8: [[C10:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX8: [[FMUL7:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC1]], [[C10]]
+    ; GFX8: [[FADD3:%[0-9]+]]:_(s32) = G_FADD [[FMUL7]], [[FMUL5]]
+    ; GFX8: [[FPTOUI2:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD3]](s32)
+    ; GFX8: [[FPTOUI3:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC1]](s32)
+    ; GFX8: [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C4]](s64)
+    ; GFX8: [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX8: [[USUBO8:%[0-9]+]]:_(s32), [[USUBO9:%[0-9]+]]:_(s1) = G_USUBO [[UV24]], [[UV26]]
+    ; GFX8: [[USUBE12:%[0-9]+]]:_(s32), [[USUBE13:%[0-9]+]]:_(s1) = G_USUBE [[UV25]], [[UV27]], [[USUBO9]]
+    ; GFX8: [[MUL18:%[0-9]+]]:_(s32) = G_MUL [[USUBO8]], [[FPTOUI2]]
+    ; GFX8: [[MUL19:%[0-9]+]]:_(s32) = G_MUL [[USUBE12]], [[FPTOUI2]]
+    ; GFX8: [[MUL20:%[0-9]+]]:_(s32) = G_MUL [[USUBO8]], [[FPTOUI3]]
+    ; GFX8: [[UMULH15:%[0-9]+]]:_(s32) = G_UMULH [[USUBO8]], [[FPTOUI2]]
+    ; GFX8: [[ADD19:%[0-9]+]]:_(s32) = G_ADD [[MUL19]], [[MUL20]]
+    ; GFX8: [[ADD20:%[0-9]+]]:_(s32) = G_ADD [[ADD19]], [[UMULH15]]
+    ; GFX8: [[MUL21:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI3]], [[MUL18]]
+    ; GFX8: [[MUL22:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI2]], [[ADD20]]
+    ; GFX8: [[UMULH16:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI2]], [[MUL18]]
+    ; GFX8: [[UADDO38:%[0-9]+]]:_(s32), [[UADDO39:%[0-9]+]]:_(s1) = G_UADDO [[MUL21]], [[MUL22]]
+    ; GFX8: [[ZEXT15:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO39]](s1)
+    ; GFX8: [[UADDO40:%[0-9]+]]:_(s32), [[UADDO41:%[0-9]+]]:_(s1) = G_UADDO [[UADDO38]], [[UMULH16]]
+    ; GFX8: [[ZEXT16:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO41]](s1)
+    ; GFX8: [[ADD21:%[0-9]+]]:_(s32) = G_ADD [[ZEXT15]], [[ZEXT16]]
+    ; GFX8: [[MUL23:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI3]], [[ADD20]]
+    ; GFX8: [[UMULH17:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI3]], [[MUL18]]
+    ; GFX8: [[UMULH18:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI2]], [[ADD20]]
+    ; GFX8: [[UADDO42:%[0-9]+]]:_(s32), [[UADDO43:%[0-9]+]]:_(s1) = G_UADDO [[MUL23]], [[UMULH17]]
+    ; GFX8: [[ZEXT17:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO43]](s1)
+    ; GFX8: [[UADDO44:%[0-9]+]]:_(s32), [[UADDO45:%[0-9]+]]:_(s1) = G_UADDO [[UADDO42]], [[UMULH18]]
+    ; GFX8: [[ZEXT18:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO45]](s1)
+    ; GFX8: [[ADD22:%[0-9]+]]:_(s32) = G_ADD [[ZEXT17]], [[ZEXT18]]
+    ; GFX8: [[UADDO46:%[0-9]+]]:_(s32), [[UADDO47:%[0-9]+]]:_(s1) = G_UADDO [[UADDO44]], [[ADD21]]
+    ; GFX8: [[ZEXT19:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO47]](s1)
+    ; GFX8: [[ADD23:%[0-9]+]]:_(s32) = G_ADD [[ADD22]], [[ZEXT19]]
+    ; GFX8: [[UMULH19:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI3]], [[ADD20]]
+    ; GFX8: [[ADD24:%[0-9]+]]:_(s32) = G_ADD [[UMULH19]], [[ADD23]]
+    ; GFX8: [[UADDO48:%[0-9]+]]:_(s32), [[UADDO49:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI2]], [[UADDO46]]
+    ; GFX8: [[UADDE10:%[0-9]+]]:_(s32), [[UADDE11:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI3]], [[ADD24]], [[UADDO49]]
+    ; GFX8: [[ADD25:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI3]], [[ADD24]]
+    ; GFX8: [[MUL24:%[0-9]+]]:_(s32) = G_MUL [[USUBO8]], [[UADDO48]]
+    ; GFX8: [[MUL25:%[0-9]+]]:_(s32) = G_MUL [[USUBE12]], [[UADDO48]]
+    ; GFX8: [[MUL26:%[0-9]+]]:_(s32) = G_MUL [[USUBO8]], [[UADDE10]]
+    ; GFX8: [[UMULH20:%[0-9]+]]:_(s32) = G_UMULH [[USUBO8]], [[UADDO48]]
+    ; GFX8: [[ADD26:%[0-9]+]]:_(s32) = G_ADD [[MUL25]], [[MUL26]]
+    ; GFX8: [[ADD27:%[0-9]+]]:_(s32) = G_ADD [[ADD26]], [[UMULH20]]
+    ; GFX8: [[MUL27:%[0-9]+]]:_(s32) = G_MUL [[UADDE10]], [[MUL24]]
+    ; GFX8: [[MUL28:%[0-9]+]]:_(s32) = G_MUL [[UADDO48]], [[ADD27]]
+    ; GFX8: [[UMULH21:%[0-9]+]]:_(s32) = G_UMULH [[UADDO48]], [[MUL24]]
+    ; GFX8: [[UADDO50:%[0-9]+]]:_(s32), [[UADDO51:%[0-9]+]]:_(s1) = G_UADDO [[MUL27]], [[MUL28]]
+    ; GFX8: [[ZEXT20:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO51]](s1)
+    ; GFX8: [[UADDO52:%[0-9]+]]:_(s32), [[UADDO53:%[0-9]+]]:_(s1) = G_UADDO [[UADDO50]], [[UMULH21]]
+    ; GFX8: [[ZEXT21:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO53]](s1)
+    ; GFX8: [[ADD28:%[0-9]+]]:_(s32) = G_ADD [[ZEXT20]], [[ZEXT21]]
+    ; GFX8: [[MUL29:%[0-9]+]]:_(s32) = G_MUL [[UADDE10]], [[ADD27]]
+    ; GFX8: [[UMULH22:%[0-9]+]]:_(s32) = G_UMULH [[UADDE10]], [[MUL24]]
+    ; GFX8: [[UMULH23:%[0-9]+]]:_(s32) = G_UMULH [[UADDO48]], [[ADD27]]
+    ; GFX8: [[UADDO54:%[0-9]+]]:_(s32), [[UADDO55:%[0-9]+]]:_(s1) = G_UADDO [[MUL29]], [[UMULH22]]
+    ; GFX8: [[ZEXT22:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO55]](s1)
+    ; GFX8: [[UADDO56:%[0-9]+]]:_(s32), [[UADDO57:%[0-9]+]]:_(s1) = G_UADDO [[UADDO54]], [[UMULH23]]
+    ; GFX8: [[ZEXT23:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO57]](s1)
+    ; GFX8: [[ADD29:%[0-9]+]]:_(s32) = G_ADD [[ZEXT22]], [[ZEXT23]]
+    ; GFX8: [[UADDO58:%[0-9]+]]:_(s32), [[UADDO59:%[0-9]+]]:_(s1) = G_UADDO [[UADDO56]], [[ADD28]]
+    ; GFX8: [[ZEXT24:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO59]](s1)
+    ; GFX8: [[ADD30:%[0-9]+]]:_(s32) = G_ADD [[ADD29]], [[ZEXT24]]
+    ; GFX8: [[UMULH24:%[0-9]+]]:_(s32) = G_UMULH [[UADDE10]], [[ADD27]]
+    ; GFX8: [[ADD31:%[0-9]+]]:_(s32) = G_ADD [[UMULH24]], [[ADD30]]
+    ; GFX8: [[UADDO60:%[0-9]+]]:_(s32), [[UADDO61:%[0-9]+]]:_(s1) = G_UADDO [[UADDO48]], [[UADDO58]]
+    ; GFX8: [[UADDE12:%[0-9]+]]:_(s32), [[UADDE13:%[0-9]+]]:_(s1) = G_UADDE [[ADD25]], [[ADD31]], [[UADDO49]]
+    ; GFX8: [[UADDE14:%[0-9]+]]:_(s32), [[UADDE15:%[0-9]+]]:_(s1) = G_UADDE [[UADDE12]], [[C5]], [[UADDO61]]
+    ; GFX8: [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+    ; GFX8: [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+    ; GFX8: [[MUL30:%[0-9]+]]:_(s32) = G_MUL [[UV31]], [[UADDO60]]
+    ; GFX8: [[MUL31:%[0-9]+]]:_(s32) = G_MUL [[UV30]], [[UADDE14]]
+    ; GFX8: [[UMULH25:%[0-9]+]]:_(s32) = G_UMULH [[UV30]], [[UADDO60]]
+    ; GFX8: [[UADDO62:%[0-9]+]]:_(s32), [[UADDO63:%[0-9]+]]:_(s1) = G_UADDO [[MUL30]], [[MUL31]]
+    ; GFX8: [[ZEXT25:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO63]](s1)
+    ; GFX8: [[UADDO64:%[0-9]+]]:_(s32), [[UADDO65:%[0-9]+]]:_(s1) = G_UADDO [[UADDO62]], [[UMULH25]]
+    ; GFX8: [[ZEXT26:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO65]](s1)
+    ; GFX8: [[ADD32:%[0-9]+]]:_(s32) = G_ADD [[ZEXT25]], [[ZEXT26]]
+    ; GFX8: [[MUL32:%[0-9]+]]:_(s32) = G_MUL [[UV31]], [[UADDE14]]
+    ; GFX8: [[UMULH26:%[0-9]+]]:_(s32) = G_UMULH [[UV31]], [[UADDO60]]
+    ; GFX8: [[UMULH27:%[0-9]+]]:_(s32) = G_UMULH [[UV30]], [[UADDE14]]
+    ; GFX8: [[UADDO66:%[0-9]+]]:_(s32), [[UADDO67:%[0-9]+]]:_(s1) = G_UADDO [[MUL32]], [[UMULH26]]
+    ; GFX8: [[ZEXT27:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO67]](s1)
+    ; GFX8: [[UADDO68:%[0-9]+]]:_(s32), [[UADDO69:%[0-9]+]]:_(s1) = G_UADDO [[UADDO66]], [[UMULH27]]
+    ; GFX8: [[ZEXT28:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO69]](s1)
+    ; GFX8: [[ADD33:%[0-9]+]]:_(s32) = G_ADD [[ZEXT27]], [[ZEXT28]]
+    ; GFX8: [[UADDO70:%[0-9]+]]:_(s32), [[UADDO71:%[0-9]+]]:_(s1) = G_UADDO [[UADDO68]], [[ADD32]]
+    ; GFX8: [[ZEXT29:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO71]](s1)
+    ; GFX8: [[ADD34:%[0-9]+]]:_(s32) = G_ADD [[ADD33]], [[ZEXT29]]
+    ; GFX8: [[UMULH28:%[0-9]+]]:_(s32) = G_UMULH [[UV31]], [[UADDE14]]
+    ; GFX8: [[ADD35:%[0-9]+]]:_(s32) = G_ADD [[UMULH28]], [[ADD34]]
+    ; GFX8: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO70]](s32), [[ADD35]](s32)
+    ; GFX8: [[UV32:%[0-9]+]]:_(s32), [[UV33:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX8: [[MUL33:%[0-9]+]]:_(s32) = G_MUL [[UV32]], [[UADDO70]]
+    ; GFX8: [[MUL34:%[0-9]+]]:_(s32) = G_MUL [[UV33]], [[UADDO70]]
+    ; GFX8: [[MUL35:%[0-9]+]]:_(s32) = G_MUL [[UV32]], [[ADD35]]
+    ; GFX8: [[UMULH29:%[0-9]+]]:_(s32) = G_UMULH [[UV32]], [[UADDO70]]
+    ; GFX8: [[ADD36:%[0-9]+]]:_(s32) = G_ADD [[MUL34]], [[MUL35]]
+    ; GFX8: [[ADD37:%[0-9]+]]:_(s32) = G_ADD [[ADD36]], [[UMULH29]]
+    ; GFX8: [[USUBO10:%[0-9]+]]:_(s32), [[USUBO11:%[0-9]+]]:_(s1) = G_USUBO [[UV28]], [[MUL33]]
+    ; GFX8: [[USUBE14:%[0-9]+]]:_(s32), [[USUBE15:%[0-9]+]]:_(s1) = G_USUBE [[UV29]], [[ADD37]], [[USUBO11]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[UV29]], [[ADD37]]
+    ; GFX8: [[UV34:%[0-9]+]]:_(s32), [[UV35:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX8: [[ICMP8:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE14]](s32), [[UV35]]
+    ; GFX8: [[SEXT4:%[0-9]+]]:_(s32) = G_SEXT [[ICMP8]](s1)
+    ; GFX8: [[ICMP9:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO10]](s32), [[UV34]]
+    ; GFX8: [[SEXT5:%[0-9]+]]:_(s32) = G_SEXT [[ICMP9]](s1)
+    ; GFX8: [[ICMP10:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE14]](s32), [[UV35]]
+    ; GFX8: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP10]](s1), [[SEXT5]], [[SEXT4]]
+    ; GFX8: [[USUBO12:%[0-9]+]]:_(s32), [[USUBO13:%[0-9]+]]:_(s1) = G_USUBO [[USUBO10]], [[UV34]]
+    ; GFX8: [[USUBE16:%[0-9]+]]:_(s32), [[USUBE17:%[0-9]+]]:_(s1) = G_USUBE [[SUB1]], [[UV35]], [[USUBO11]]
+    ; GFX8: [[USUBE18:%[0-9]+]]:_(s32), [[USUBE19:%[0-9]+]]:_(s1) = G_USUBE [[USUBE16]], [[C5]], [[USUBO13]]
+    ; GFX8: [[UV36:%[0-9]+]]:_(s32), [[UV37:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C6]](s64)
+    ; GFX8: [[UADDO72:%[0-9]+]]:_(s32), [[UADDO73:%[0-9]+]]:_(s1) = G_UADDO [[UADDO70]], [[UV36]]
+    ; GFX8: [[UADDE16:%[0-9]+]]:_(s32), [[UADDE17:%[0-9]+]]:_(s1) = G_UADDE [[ADD35]], [[UV37]], [[UADDO73]]
+    ; GFX8: [[MV4:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO72]](s32), [[UADDE16]](s32)
+    ; GFX8: [[ICMP11:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE18]](s32), [[UV35]]
+    ; GFX8: [[SEXT6:%[0-9]+]]:_(s32) = G_SEXT [[ICMP11]](s1)
+    ; GFX8: [[ICMP12:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO12]](s32), [[UV34]]
+    ; GFX8: [[SEXT7:%[0-9]+]]:_(s32) = G_SEXT [[ICMP12]](s1)
+    ; GFX8: [[ICMP13:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE18]](s32), [[UV35]]
+    ; GFX8: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP13]](s1), [[SEXT7]], [[SEXT6]]
+    ; GFX8: [[UV38:%[0-9]+]]:_(s32), [[UV39:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C6]](s64)
+    ; GFX8: [[UADDO74:%[0-9]+]]:_(s32), [[UADDO75:%[0-9]+]]:_(s1) = G_UADDO [[UADDO72]], [[UV38]]
+    ; GFX8: [[UADDE18:%[0-9]+]]:_(s32), [[UADDE19:%[0-9]+]]:_(s1) = G_UADDE [[UADDE16]], [[UV39]], [[UADDO75]]
+    ; GFX8: [[MV5:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO74]](s32), [[UADDE18]](s32)
+    ; GFX8: [[USUBO14:%[0-9]+]]:_(s32), [[USUBO15:%[0-9]+]]:_(s1) = G_USUBO [[USUBO12]], [[UV34]]
+    ; GFX8: [[USUBE20:%[0-9]+]]:_(s32), [[USUBE21:%[0-9]+]]:_(s1) = G_USUBE [[USUBE16]], [[UV35]], [[USUBO13]]
+    ; GFX8: [[USUBE22:%[0-9]+]]:_(s32), [[USUBE23:%[0-9]+]]:_(s1) = G_USUBE [[USUBE20]], [[C5]], [[USUBO15]]
+    ; GFX8: [[ICMP14:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT5]](s32), [[C5]]
+    ; GFX8: [[SELECT6:%[0-9]+]]:_(s64) = G_SELECT [[ICMP14]](s1), [[MV5]], [[MV4]]
+    ; GFX8: [[ICMP15:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT4]](s32), [[C5]]
+    ; GFX8: [[SELECT7:%[0-9]+]]:_(s64) = G_SELECT [[ICMP15]](s1), [[SELECT6]], [[MV3]]
+    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SELECT3]](s64), [[SELECT7]](s64)
     ; GFX8: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     ; GFX9-LABEL: name: test_udiv_v2s64
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
     ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
     ; GFX9: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
-    ; GFX9: [[UDIV:%[0-9]+]]:_(s64) = G_UDIV [[UV]], [[UV2]]
-    ; GFX9: [[UDIV1:%[0-9]+]]:_(s64) = G_UDIV [[UV1]], [[UV3]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[UDIV]](s64), [[UDIV1]](s64)
+    ; GFX9: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV4]](s32)
+    ; GFX9: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV5]](s32)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C]]
+    ; GFX9: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX9: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX9: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C2]]
+    ; GFX9: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX9: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C3]]
+    ; GFX9: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX9: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX9: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C4]](s64)
+    ; GFX9: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX9: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV6]], [[UV8]]
+    ; GFX9: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV7]], [[UV9]], [[USUBO1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX9: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX9: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO1]](s1)
+    ; GFX9: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UADDO]], [[UMULH1]]
+    ; GFX9: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO3]](s1)
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX9: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX9: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX9: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX9: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX9: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH3]]
+    ; GFX9: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX9: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[UADDO6]], [[ADD2]]
+    ; GFX9: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX9: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX9: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX9: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX9: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO8]]
+    ; GFX9: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO11]]
+    ; GFX9: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX9: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO10]]
+    ; GFX9: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO10]]
+    ; GFX9: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE]]
+    ; GFX9: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO10]]
+    ; GFX9: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX9: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX9: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[MUL6]]
+    ; GFX9: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO10]], [[ADD8]]
+    ; GFX9: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[MUL6]]
+    ; GFX9: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX9: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX9: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[UADDO12]], [[UMULH6]]
+    ; GFX9: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO15]](s1)
+    ; GFX9: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX9: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[ADD8]]
+    ; GFX9: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[MUL6]]
+    ; GFX9: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[ADD8]]
+    ; GFX9: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX9: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX9: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH8]]
+    ; GFX9: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX9: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX9: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[UADDO18]], [[ADD9]]
+    ; GFX9: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX9: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX9: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[ADD8]]
+    ; GFX9: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX9: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[UADDO20]]
+    ; GFX9: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO11]]
+    ; GFX9: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[UADDE2]], [[C5]], [[UADDO23]]
+    ; GFX9: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+    ; GFX9: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+    ; GFX9: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV13]], [[UADDO22]]
+    ; GFX9: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV12]], [[UADDE4]]
+    ; GFX9: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV12]], [[UADDO22]]
+    ; GFX9: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX9: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX9: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO24]], [[UMULH10]]
+    ; GFX9: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO27]](s1)
+    ; GFX9: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX9: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV13]], [[UADDE4]]
+    ; GFX9: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV13]], [[UADDO22]]
+    ; GFX9: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV12]], [[UADDE4]]
+    ; GFX9: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX9: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX9: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH12]]
+    ; GFX9: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX9: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX9: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[UADDO30]], [[ADD13]]
+    ; GFX9: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX9: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX9: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV13]], [[UADDE4]]
+    ; GFX9: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO32]](s32), [[ADD16]](s32)
+    ; GFX9: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX9: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV14]], [[UADDO32]]
+    ; GFX9: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV15]], [[UADDO32]]
+    ; GFX9: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV14]], [[ADD16]]
+    ; GFX9: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV14]], [[UADDO32]]
+    ; GFX9: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX9: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX9: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV10]], [[MUL15]]
+    ; GFX9: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV11]], [[ADD18]], [[USUBO3]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV11]], [[ADD18]]
+    ; GFX9: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV17]]
+    ; GFX9: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV16]]
+    ; GFX9: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV17]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX9: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV16]]
+    ; GFX9: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV17]], [[USUBO3]]
+    ; GFX9: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C5]], [[USUBO5]]
+    ; GFX9: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; GFX9: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C6]](s64)
+    ; GFX9: [[UADDO34:%[0-9]+]]:_(s32), [[UADDO35:%[0-9]+]]:_(s1) = G_UADDO [[UADDO32]], [[UV18]]
+    ; GFX9: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[ADD16]], [[UV19]], [[UADDO35]]
+    ; GFX9: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO34]](s32), [[UADDE6]](s32)
+    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV17]]
+    ; GFX9: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX9: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV16]]
+    ; GFX9: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX9: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV17]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX9: [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C6]](s64)
+    ; GFX9: [[UADDO36:%[0-9]+]]:_(s32), [[UADDO37:%[0-9]+]]:_(s1) = G_UADDO [[UADDO34]], [[UV20]]
+    ; GFX9: [[UADDE8:%[0-9]+]]:_(s32), [[UADDE9:%[0-9]+]]:_(s1) = G_UADDE [[UADDE6]], [[UV21]], [[UADDO37]]
+    ; GFX9: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO36]](s32), [[UADDE8]](s32)
+    ; GFX9: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV16]]
+    ; GFX9: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV17]], [[USUBO5]]
+    ; GFX9: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C5]], [[USUBO7]]
+    ; GFX9: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C5]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV2]], [[MV1]]
+    ; GFX9: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C5]]
+    ; GFX9: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV]]
+    ; GFX9: [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX9: [[UITOFP2:%[0-9]+]]:_(s32) = G_UITOFP [[UV22]](s32)
+    ; GFX9: [[UITOFP3:%[0-9]+]]:_(s32) = G_UITOFP [[UV23]](s32)
+    ; GFX9: [[C7:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[FMUL4:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP3]], [[C7]]
+    ; GFX9: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FMUL4]], [[UITOFP2]]
+    ; GFX9: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD2]](s32)
+    ; GFX9: [[C8:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX9: [[FMUL5:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C8]]
+    ; GFX9: [[C9:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX9: [[FMUL6:%[0-9]+]]:_(s32) = G_FMUL [[FMUL5]], [[C9]]
+    ; GFX9: [[INTRINSIC_TRUNC1:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL6]]
+    ; GFX9: [[C10:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX9: [[FMUL7:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC1]], [[C10]]
+    ; GFX9: [[FADD3:%[0-9]+]]:_(s32) = G_FADD [[FMUL7]], [[FMUL5]]
+    ; GFX9: [[FPTOUI2:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD3]](s32)
+    ; GFX9: [[FPTOUI3:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC1]](s32)
+    ; GFX9: [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C4]](s64)
+    ; GFX9: [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX9: [[USUBO8:%[0-9]+]]:_(s32), [[USUBO9:%[0-9]+]]:_(s1) = G_USUBO [[UV24]], [[UV26]]
+    ; GFX9: [[USUBE12:%[0-9]+]]:_(s32), [[USUBE13:%[0-9]+]]:_(s1) = G_USUBE [[UV25]], [[UV27]], [[USUBO9]]
+    ; GFX9: [[MUL18:%[0-9]+]]:_(s32) = G_MUL [[USUBO8]], [[FPTOUI2]]
+    ; GFX9: [[MUL19:%[0-9]+]]:_(s32) = G_MUL [[USUBE12]], [[FPTOUI2]]
+    ; GFX9: [[MUL20:%[0-9]+]]:_(s32) = G_MUL [[USUBO8]], [[FPTOUI3]]
+    ; GFX9: [[UMULH15:%[0-9]+]]:_(s32) = G_UMULH [[USUBO8]], [[FPTOUI2]]
+    ; GFX9: [[ADD19:%[0-9]+]]:_(s32) = G_ADD [[MUL19]], [[MUL20]]
+    ; GFX9: [[ADD20:%[0-9]+]]:_(s32) = G_ADD [[ADD19]], [[UMULH15]]
+    ; GFX9: [[MUL21:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI3]], [[MUL18]]
+    ; GFX9: [[MUL22:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI2]], [[ADD20]]
+    ; GFX9: [[UMULH16:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI2]], [[MUL18]]
+    ; GFX9: [[UADDO38:%[0-9]+]]:_(s32), [[UADDO39:%[0-9]+]]:_(s1) = G_UADDO [[MUL21]], [[MUL22]]
+    ; GFX9: [[ZEXT15:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO39]](s1)
+    ; GFX9: [[UADDO40:%[0-9]+]]:_(s32), [[UADDO41:%[0-9]+]]:_(s1) = G_UADDO [[UADDO38]], [[UMULH16]]
+    ; GFX9: [[ZEXT16:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO41]](s1)
+    ; GFX9: [[ADD21:%[0-9]+]]:_(s32) = G_ADD [[ZEXT15]], [[ZEXT16]]
+    ; GFX9: [[MUL23:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI3]], [[ADD20]]
+    ; GFX9: [[UMULH17:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI3]], [[MUL18]]
+    ; GFX9: [[UMULH18:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI2]], [[ADD20]]
+    ; GFX9: [[UADDO42:%[0-9]+]]:_(s32), [[UADDO43:%[0-9]+]]:_(s1) = G_UADDO [[MUL23]], [[UMULH17]]
+    ; GFX9: [[ZEXT17:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO43]](s1)
+    ; GFX9: [[UADDO44:%[0-9]+]]:_(s32), [[UADDO45:%[0-9]+]]:_(s1) = G_UADDO [[UADDO42]], [[UMULH18]]
+    ; GFX9: [[ZEXT18:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO45]](s1)
+    ; GFX9: [[ADD22:%[0-9]+]]:_(s32) = G_ADD [[ZEXT17]], [[ZEXT18]]
+    ; GFX9: [[UADDO46:%[0-9]+]]:_(s32), [[UADDO47:%[0-9]+]]:_(s1) = G_UADDO [[UADDO44]], [[ADD21]]
+    ; GFX9: [[ZEXT19:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO47]](s1)
+    ; GFX9: [[ADD23:%[0-9]+]]:_(s32) = G_ADD [[ADD22]], [[ZEXT19]]
+    ; GFX9: [[UMULH19:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI3]], [[ADD20]]
+    ; GFX9: [[ADD24:%[0-9]+]]:_(s32) = G_ADD [[UMULH19]], [[ADD23]]
+    ; GFX9: [[UADDO48:%[0-9]+]]:_(s32), [[UADDO49:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI2]], [[UADDO46]]
+    ; GFX9: [[UADDE10:%[0-9]+]]:_(s32), [[UADDE11:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI3]], [[ADD24]], [[UADDO49]]
+    ; GFX9: [[ADD25:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI3]], [[ADD24]]
+    ; GFX9: [[MUL24:%[0-9]+]]:_(s32) = G_MUL [[USUBO8]], [[UADDO48]]
+    ; GFX9: [[MUL25:%[0-9]+]]:_(s32) = G_MUL [[USUBE12]], [[UADDO48]]
+    ; GFX9: [[MUL26:%[0-9]+]]:_(s32) = G_MUL [[USUBO8]], [[UADDE10]]
+    ; GFX9: [[UMULH20:%[0-9]+]]:_(s32) = G_UMULH [[USUBO8]], [[UADDO48]]
+    ; GFX9: [[ADD26:%[0-9]+]]:_(s32) = G_ADD [[MUL25]], [[MUL26]]
+    ; GFX9: [[ADD27:%[0-9]+]]:_(s32) = G_ADD [[ADD26]], [[UMULH20]]
+    ; GFX9: [[MUL27:%[0-9]+]]:_(s32) = G_MUL [[UADDE10]], [[MUL24]]
+    ; GFX9: [[MUL28:%[0-9]+]]:_(s32) = G_MUL [[UADDO48]], [[ADD27]]
+    ; GFX9: [[UMULH21:%[0-9]+]]:_(s32) = G_UMULH [[UADDO48]], [[MUL24]]
+    ; GFX9: [[UADDO50:%[0-9]+]]:_(s32), [[UADDO51:%[0-9]+]]:_(s1) = G_UADDO [[MUL27]], [[MUL28]]
+    ; GFX9: [[ZEXT20:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO51]](s1)
+    ; GFX9: [[UADDO52:%[0-9]+]]:_(s32), [[UADDO53:%[0-9]+]]:_(s1) = G_UADDO [[UADDO50]], [[UMULH21]]
+    ; GFX9: [[ZEXT21:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO53]](s1)
+    ; GFX9: [[ADD28:%[0-9]+]]:_(s32) = G_ADD [[ZEXT20]], [[ZEXT21]]
+    ; GFX9: [[MUL29:%[0-9]+]]:_(s32) = G_MUL [[UADDE10]], [[ADD27]]
+    ; GFX9: [[UMULH22:%[0-9]+]]:_(s32) = G_UMULH [[UADDE10]], [[MUL24]]
+    ; GFX9: [[UMULH23:%[0-9]+]]:_(s32) = G_UMULH [[UADDO48]], [[ADD27]]
+    ; GFX9: [[UADDO54:%[0-9]+]]:_(s32), [[UADDO55:%[0-9]+]]:_(s1) = G_UADDO [[MUL29]], [[UMULH22]]
+    ; GFX9: [[ZEXT22:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO55]](s1)
+    ; GFX9: [[UADDO56:%[0-9]+]]:_(s32), [[UADDO57:%[0-9]+]]:_(s1) = G_UADDO [[UADDO54]], [[UMULH23]]
+    ; GFX9: [[ZEXT23:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO57]](s1)
+    ; GFX9: [[ADD29:%[0-9]+]]:_(s32) = G_ADD [[ZEXT22]], [[ZEXT23]]
+    ; GFX9: [[UADDO58:%[0-9]+]]:_(s32), [[UADDO59:%[0-9]+]]:_(s1) = G_UADDO [[UADDO56]], [[ADD28]]
+    ; GFX9: [[ZEXT24:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO59]](s1)
+    ; GFX9: [[ADD30:%[0-9]+]]:_(s32) = G_ADD [[ADD29]], [[ZEXT24]]
+    ; GFX9: [[UMULH24:%[0-9]+]]:_(s32) = G_UMULH [[UADDE10]], [[ADD27]]
+    ; GFX9: [[ADD31:%[0-9]+]]:_(s32) = G_ADD [[UMULH24]], [[ADD30]]
+    ; GFX9: [[UADDO60:%[0-9]+]]:_(s32), [[UADDO61:%[0-9]+]]:_(s1) = G_UADDO [[UADDO48]], [[UADDO58]]
+    ; GFX9: [[UADDE12:%[0-9]+]]:_(s32), [[UADDE13:%[0-9]+]]:_(s1) = G_UADDE [[ADD25]], [[ADD31]], [[UADDO49]]
+    ; GFX9: [[UADDE14:%[0-9]+]]:_(s32), [[UADDE15:%[0-9]+]]:_(s1) = G_UADDE [[UADDE12]], [[C5]], [[UADDO61]]
+    ; GFX9: [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+    ; GFX9: [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+    ; GFX9: [[MUL30:%[0-9]+]]:_(s32) = G_MUL [[UV31]], [[UADDO60]]
+    ; GFX9: [[MUL31:%[0-9]+]]:_(s32) = G_MUL [[UV30]], [[UADDE14]]
+    ; GFX9: [[UMULH25:%[0-9]+]]:_(s32) = G_UMULH [[UV30]], [[UADDO60]]
+    ; GFX9: [[UADDO62:%[0-9]+]]:_(s32), [[UADDO63:%[0-9]+]]:_(s1) = G_UADDO [[MUL30]], [[MUL31]]
+    ; GFX9: [[ZEXT25:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO63]](s1)
+    ; GFX9: [[UADDO64:%[0-9]+]]:_(s32), [[UADDO65:%[0-9]+]]:_(s1) = G_UADDO [[UADDO62]], [[UMULH25]]
+    ; GFX9: [[ZEXT26:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO65]](s1)
+    ; GFX9: [[ADD32:%[0-9]+]]:_(s32) = G_ADD [[ZEXT25]], [[ZEXT26]]
+    ; GFX9: [[MUL32:%[0-9]+]]:_(s32) = G_MUL [[UV31]], [[UADDE14]]
+    ; GFX9: [[UMULH26:%[0-9]+]]:_(s32) = G_UMULH [[UV31]], [[UADDO60]]
+    ; GFX9: [[UMULH27:%[0-9]+]]:_(s32) = G_UMULH [[UV30]], [[UADDE14]]
+    ; GFX9: [[UADDO66:%[0-9]+]]:_(s32), [[UADDO67:%[0-9]+]]:_(s1) = G_UADDO [[MUL32]], [[UMULH26]]
+    ; GFX9: [[ZEXT27:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO67]](s1)
+    ; GFX9: [[UADDO68:%[0-9]+]]:_(s32), [[UADDO69:%[0-9]+]]:_(s1) = G_UADDO [[UADDO66]], [[UMULH27]]
+    ; GFX9: [[ZEXT28:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO69]](s1)
+    ; GFX9: [[ADD33:%[0-9]+]]:_(s32) = G_ADD [[ZEXT27]], [[ZEXT28]]
+    ; GFX9: [[UADDO70:%[0-9]+]]:_(s32), [[UADDO71:%[0-9]+]]:_(s1) = G_UADDO [[UADDO68]], [[ADD32]]
+    ; GFX9: [[ZEXT29:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO71]](s1)
+    ; GFX9: [[ADD34:%[0-9]+]]:_(s32) = G_ADD [[ADD33]], [[ZEXT29]]
+    ; GFX9: [[UMULH28:%[0-9]+]]:_(s32) = G_UMULH [[UV31]], [[UADDE14]]
+    ; GFX9: [[ADD35:%[0-9]+]]:_(s32) = G_ADD [[UMULH28]], [[ADD34]]
+    ; GFX9: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO70]](s32), [[ADD35]](s32)
+    ; GFX9: [[UV32:%[0-9]+]]:_(s32), [[UV33:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX9: [[MUL33:%[0-9]+]]:_(s32) = G_MUL [[UV32]], [[UADDO70]]
+    ; GFX9: [[MUL34:%[0-9]+]]:_(s32) = G_MUL [[UV33]], [[UADDO70]]
+    ; GFX9: [[MUL35:%[0-9]+]]:_(s32) = G_MUL [[UV32]], [[ADD35]]
+    ; GFX9: [[UMULH29:%[0-9]+]]:_(s32) = G_UMULH [[UV32]], [[UADDO70]]
+    ; GFX9: [[ADD36:%[0-9]+]]:_(s32) = G_ADD [[MUL34]], [[MUL35]]
+    ; GFX9: [[ADD37:%[0-9]+]]:_(s32) = G_ADD [[ADD36]], [[UMULH29]]
+    ; GFX9: [[USUBO10:%[0-9]+]]:_(s32), [[USUBO11:%[0-9]+]]:_(s1) = G_USUBO [[UV28]], [[MUL33]]
+    ; GFX9: [[USUBE14:%[0-9]+]]:_(s32), [[USUBE15:%[0-9]+]]:_(s1) = G_USUBE [[UV29]], [[ADD37]], [[USUBO11]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[UV29]], [[ADD37]]
+    ; GFX9: [[UV34:%[0-9]+]]:_(s32), [[UV35:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX9: [[ICMP8:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE14]](s32), [[UV35]]
+    ; GFX9: [[SEXT4:%[0-9]+]]:_(s32) = G_SEXT [[ICMP8]](s1)
+    ; GFX9: [[ICMP9:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO10]](s32), [[UV34]]
+    ; GFX9: [[SEXT5:%[0-9]+]]:_(s32) = G_SEXT [[ICMP9]](s1)
+    ; GFX9: [[ICMP10:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE14]](s32), [[UV35]]
+    ; GFX9: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP10]](s1), [[SEXT5]], [[SEXT4]]
+    ; GFX9: [[USUBO12:%[0-9]+]]:_(s32), [[USUBO13:%[0-9]+]]:_(s1) = G_USUBO [[USUBO10]], [[UV34]]
+    ; GFX9: [[USUBE16:%[0-9]+]]:_(s32), [[USUBE17:%[0-9]+]]:_(s1) = G_USUBE [[SUB1]], [[UV35]], [[USUBO11]]
+    ; GFX9: [[USUBE18:%[0-9]+]]:_(s32), [[USUBE19:%[0-9]+]]:_(s1) = G_USUBE [[USUBE16]], [[C5]], [[USUBO13]]
+    ; GFX9: [[UV36:%[0-9]+]]:_(s32), [[UV37:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C6]](s64)
+    ; GFX9: [[UADDO72:%[0-9]+]]:_(s32), [[UADDO73:%[0-9]+]]:_(s1) = G_UADDO [[UADDO70]], [[UV36]]
+    ; GFX9: [[UADDE16:%[0-9]+]]:_(s32), [[UADDE17:%[0-9]+]]:_(s1) = G_UADDE [[ADD35]], [[UV37]], [[UADDO73]]
+    ; GFX9: [[MV4:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO72]](s32), [[UADDE16]](s32)
+    ; GFX9: [[ICMP11:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE18]](s32), [[UV35]]
+    ; GFX9: [[SEXT6:%[0-9]+]]:_(s32) = G_SEXT [[ICMP11]](s1)
+    ; GFX9: [[ICMP12:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO12]](s32), [[UV34]]
+    ; GFX9: [[SEXT7:%[0-9]+]]:_(s32) = G_SEXT [[ICMP12]](s1)
+    ; GFX9: [[ICMP13:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE18]](s32), [[UV35]]
+    ; GFX9: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP13]](s1), [[SEXT7]], [[SEXT6]]
+    ; GFX9: [[UV38:%[0-9]+]]:_(s32), [[UV39:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C6]](s64)
+    ; GFX9: [[UADDO74:%[0-9]+]]:_(s32), [[UADDO75:%[0-9]+]]:_(s1) = G_UADDO [[UADDO72]], [[UV38]]
+    ; GFX9: [[UADDE18:%[0-9]+]]:_(s32), [[UADDE19:%[0-9]+]]:_(s1) = G_UADDE [[UADDE16]], [[UV39]], [[UADDO75]]
+    ; GFX9: [[MV5:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO74]](s32), [[UADDE18]](s32)
+    ; GFX9: [[USUBO14:%[0-9]+]]:_(s32), [[USUBO15:%[0-9]+]]:_(s1) = G_USUBO [[USUBO12]], [[UV34]]
+    ; GFX9: [[USUBE20:%[0-9]+]]:_(s32), [[USUBE21:%[0-9]+]]:_(s1) = G_USUBE [[USUBE16]], [[UV35]], [[USUBO13]]
+    ; GFX9: [[USUBE22:%[0-9]+]]:_(s32), [[USUBE23:%[0-9]+]]:_(s1) = G_USUBE [[USUBE20]], [[C5]], [[USUBO15]]
+    ; GFX9: [[ICMP14:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT5]](s32), [[C5]]
+    ; GFX9: [[SELECT6:%[0-9]+]]:_(s64) = G_SELECT [[ICMP14]](s1), [[MV5]], [[MV4]]
+    ; GFX9: [[ICMP15:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT4]](s32), [[C5]]
+    ; GFX9: [[SELECT7:%[0-9]+]]:_(s64) = G_SELECT [[ICMP15]](s1), [[SELECT6]], [[MV3]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SELECT3]](s64), [[SELECT7]](s64)
     ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     %1:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
@@ -968,36 +2291,480 @@ body: |
     ; GFX6-LABEL: name: test_udiv_s33
     ; GFX6: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX6: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX6: [[TRUNC:%[0-9]+]]:_(s33) = G_TRUNC [[COPY]](s64)
-    ; GFX6: [[TRUNC1:%[0-9]+]]:_(s33) = G_TRUNC [[COPY1]](s64)
-    ; GFX6: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[TRUNC]](s33)
-    ; GFX6: [[ZEXT1:%[0-9]+]]:_(s64) = G_ZEXT [[TRUNC1]](s33)
-    ; GFX6: [[UDIV:%[0-9]+]]:_(s64) = G_UDIV [[ZEXT]], [[ZEXT1]]
-    ; GFX6: [[TRUNC2:%[0-9]+]]:_(s33) = G_TRUNC [[UDIV]](s64)
-    ; GFX6: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[TRUNC2]](s33)
-    ; GFX6: $vgpr0_vgpr1 = COPY [[ANYEXT]](s64)
+    ; GFX6: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8589934591
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s64) = COPY [[COPY]](s64)
+    ; GFX6: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C]]
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s64) = COPY [[COPY1]](s64)
+    ; GFX6: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY3]], [[C]]
+    ; GFX6: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND1]](s64)
+    ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV]](s32)
+    ; GFX6: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV1]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C1]]
+    ; GFX6: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX6: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
+    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX6: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C3]]
+    ; GFX6: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX6: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX6: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C4]]
+    ; GFX6: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX6: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX6: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX6: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C5]](s64)
+    ; GFX6: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND1]](s64)
+    ; GFX6: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV2]], [[UV4]]
+    ; GFX6: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV3]], [[UV5]], [[USUBO1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX6: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX6: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX6: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX6: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO1]](s1)
+    ; GFX6: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UADDO]], [[UMULH1]]
+    ; GFX6: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO3]](s1)
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX6: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX6: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX6: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX6: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX6: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH3]]
+    ; GFX6: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX6: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[UADDO6]], [[ADD2]]
+    ; GFX6: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX6: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX6: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX6: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX6: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO8]]
+    ; GFX6: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO11]]
+    ; GFX6: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX6: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO10]]
+    ; GFX6: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO10]]
+    ; GFX6: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE]]
+    ; GFX6: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO10]]
+    ; GFX6: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX6: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX6: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[MUL6]]
+    ; GFX6: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO10]], [[ADD8]]
+    ; GFX6: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[MUL6]]
+    ; GFX6: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX6: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX6: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[UADDO12]], [[UMULH6]]
+    ; GFX6: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO15]](s1)
+    ; GFX6: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX6: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[ADD8]]
+    ; GFX6: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[MUL6]]
+    ; GFX6: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[ADD8]]
+    ; GFX6: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX6: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX6: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH8]]
+    ; GFX6: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX6: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX6: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[UADDO18]], [[ADD9]]
+    ; GFX6: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX6: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX6: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[ADD8]]
+    ; GFX6: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX6: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX6: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[UADDO20]]
+    ; GFX6: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO11]]
+    ; GFX6: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[UADDE2]], [[C6]], [[UADDO23]]
+    ; GFX6: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND]](s64)
+    ; GFX6: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND]](s64)
+    ; GFX6: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV9]], [[UADDO22]]
+    ; GFX6: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV8]], [[UADDE4]]
+    ; GFX6: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV8]], [[UADDO22]]
+    ; GFX6: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX6: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX6: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO24]], [[UMULH10]]
+    ; GFX6: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO27]](s1)
+    ; GFX6: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX6: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV9]], [[UADDE4]]
+    ; GFX6: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV9]], [[UADDO22]]
+    ; GFX6: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV8]], [[UADDE4]]
+    ; GFX6: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX6: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX6: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH12]]
+    ; GFX6: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX6: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX6: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[UADDO30]], [[ADD13]]
+    ; GFX6: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX6: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX6: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV9]], [[UADDE4]]
+    ; GFX6: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX6: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO32]](s32), [[ADD16]](s32)
+    ; GFX6: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND1]](s64)
+    ; GFX6: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV10]], [[UADDO32]]
+    ; GFX6: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV11]], [[UADDO32]]
+    ; GFX6: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV10]], [[ADD16]]
+    ; GFX6: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV10]], [[UADDO32]]
+    ; GFX6: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX6: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX6: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV6]], [[MUL15]]
+    ; GFX6: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV7]], [[ADD18]], [[USUBO3]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV7]], [[ADD18]]
+    ; GFX6: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND1]](s64)
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV13]]
+    ; GFX6: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV12]]
+    ; GFX6: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV13]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX6: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV12]]
+    ; GFX6: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV13]], [[USUBO3]]
+    ; GFX6: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C6]], [[USUBO5]]
+    ; GFX6: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; GFX6: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C7]](s64)
+    ; GFX6: [[UADDO34:%[0-9]+]]:_(s32), [[UADDO35:%[0-9]+]]:_(s1) = G_UADDO [[UADDO32]], [[UV14]]
+    ; GFX6: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[ADD16]], [[UV15]], [[UADDO35]]
+    ; GFX6: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO34]](s32), [[UADDE6]](s32)
+    ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV13]]
+    ; GFX6: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX6: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV12]]
+    ; GFX6: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX6: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV13]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX6: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C7]](s64)
+    ; GFX6: [[UADDO36:%[0-9]+]]:_(s32), [[UADDO37:%[0-9]+]]:_(s1) = G_UADDO [[UADDO34]], [[UV16]]
+    ; GFX6: [[UADDE8:%[0-9]+]]:_(s32), [[UADDE9:%[0-9]+]]:_(s1) = G_UADDE [[UADDE6]], [[UV17]], [[UADDO37]]
+    ; GFX6: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO36]](s32), [[UADDE8]](s32)
+    ; GFX6: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV12]]
+    ; GFX6: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV13]], [[USUBO5]]
+    ; GFX6: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C6]], [[USUBO7]]
+    ; GFX6: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C6]]
+    ; GFX6: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV2]], [[MV1]]
+    ; GFX6: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C6]]
+    ; GFX6: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV]]
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s64) = COPY [[SELECT3]](s64)
+    ; GFX6: $vgpr0_vgpr1 = COPY [[COPY4]](s64)
     ; GFX8-LABEL: name: test_udiv_s33
     ; GFX8: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX8: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX8: [[TRUNC:%[0-9]+]]:_(s33) = G_TRUNC [[COPY]](s64)
-    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s33) = G_TRUNC [[COPY1]](s64)
-    ; GFX8: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[TRUNC]](s33)
-    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s64) = G_ZEXT [[TRUNC1]](s33)
-    ; GFX8: [[UDIV:%[0-9]+]]:_(s64) = G_UDIV [[ZEXT]], [[ZEXT1]]
-    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s33) = G_TRUNC [[UDIV]](s64)
-    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[TRUNC2]](s33)
-    ; GFX8: $vgpr0_vgpr1 = COPY [[ANYEXT]](s64)
+    ; GFX8: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8589934591
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s64) = COPY [[COPY]](s64)
+    ; GFX8: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C]]
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s64) = COPY [[COPY1]](s64)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY3]], [[C]]
+    ; GFX8: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND1]](s64)
+    ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV]](s32)
+    ; GFX8: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV1]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C1]]
+    ; GFX8: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX8: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
+    ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX8: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C3]]
+    ; GFX8: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX8: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX8: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C4]]
+    ; GFX8: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX8: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX8: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX8: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C5]](s64)
+    ; GFX8: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND1]](s64)
+    ; GFX8: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV2]], [[UV4]]
+    ; GFX8: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV3]], [[UV5]], [[USUBO1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX8: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO1]](s1)
+    ; GFX8: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UADDO]], [[UMULH1]]
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO3]](s1)
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX8: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX8: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX8: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX8: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX8: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH3]]
+    ; GFX8: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX8: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[UADDO6]], [[ADD2]]
+    ; GFX8: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX8: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX8: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX8: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX8: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO8]]
+    ; GFX8: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO11]]
+    ; GFX8: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX8: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO10]]
+    ; GFX8: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO10]]
+    ; GFX8: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE]]
+    ; GFX8: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO10]]
+    ; GFX8: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX8: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX8: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[MUL6]]
+    ; GFX8: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO10]], [[ADD8]]
+    ; GFX8: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[MUL6]]
+    ; GFX8: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX8: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX8: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[UADDO12]], [[UMULH6]]
+    ; GFX8: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO15]](s1)
+    ; GFX8: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX8: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[ADD8]]
+    ; GFX8: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[MUL6]]
+    ; GFX8: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[ADD8]]
+    ; GFX8: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX8: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX8: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH8]]
+    ; GFX8: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX8: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX8: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[UADDO18]], [[ADD9]]
+    ; GFX8: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX8: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX8: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[ADD8]]
+    ; GFX8: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX8: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[UADDO20]]
+    ; GFX8: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO11]]
+    ; GFX8: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[UADDE2]], [[C6]], [[UADDO23]]
+    ; GFX8: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND]](s64)
+    ; GFX8: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND]](s64)
+    ; GFX8: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV9]], [[UADDO22]]
+    ; GFX8: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV8]], [[UADDE4]]
+    ; GFX8: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV8]], [[UADDO22]]
+    ; GFX8: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX8: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX8: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO24]], [[UMULH10]]
+    ; GFX8: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO27]](s1)
+    ; GFX8: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX8: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV9]], [[UADDE4]]
+    ; GFX8: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV9]], [[UADDO22]]
+    ; GFX8: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV8]], [[UADDE4]]
+    ; GFX8: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX8: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX8: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH12]]
+    ; GFX8: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX8: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX8: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[UADDO30]], [[ADD13]]
+    ; GFX8: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX8: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX8: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV9]], [[UADDE4]]
+    ; GFX8: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX8: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO32]](s32), [[ADD16]](s32)
+    ; GFX8: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND1]](s64)
+    ; GFX8: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV10]], [[UADDO32]]
+    ; GFX8: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV11]], [[UADDO32]]
+    ; GFX8: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV10]], [[ADD16]]
+    ; GFX8: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV10]], [[UADDO32]]
+    ; GFX8: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX8: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX8: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV6]], [[MUL15]]
+    ; GFX8: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV7]], [[ADD18]], [[USUBO3]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV7]], [[ADD18]]
+    ; GFX8: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND1]](s64)
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV13]]
+    ; GFX8: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV12]]
+    ; GFX8: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV13]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX8: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV12]]
+    ; GFX8: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV13]], [[USUBO3]]
+    ; GFX8: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C6]], [[USUBO5]]
+    ; GFX8: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; GFX8: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C7]](s64)
+    ; GFX8: [[UADDO34:%[0-9]+]]:_(s32), [[UADDO35:%[0-9]+]]:_(s1) = G_UADDO [[UADDO32]], [[UV14]]
+    ; GFX8: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[ADD16]], [[UV15]], [[UADDO35]]
+    ; GFX8: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO34]](s32), [[UADDE6]](s32)
+    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV13]]
+    ; GFX8: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX8: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV12]]
+    ; GFX8: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX8: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV13]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX8: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C7]](s64)
+    ; GFX8: [[UADDO36:%[0-9]+]]:_(s32), [[UADDO37:%[0-9]+]]:_(s1) = G_UADDO [[UADDO34]], [[UV16]]
+    ; GFX8: [[UADDE8:%[0-9]+]]:_(s32), [[UADDE9:%[0-9]+]]:_(s1) = G_UADDE [[UADDE6]], [[UV17]], [[UADDO37]]
+    ; GFX8: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO36]](s32), [[UADDE8]](s32)
+    ; GFX8: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV12]]
+    ; GFX8: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV13]], [[USUBO5]]
+    ; GFX8: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C6]], [[USUBO7]]
+    ; GFX8: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C6]]
+    ; GFX8: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV2]], [[MV1]]
+    ; GFX8: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C6]]
+    ; GFX8: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV]]
+    ; GFX8: [[COPY4:%[0-9]+]]:_(s64) = COPY [[SELECT3]](s64)
+    ; GFX8: $vgpr0_vgpr1 = COPY [[COPY4]](s64)
     ; GFX9-LABEL: name: test_udiv_s33
     ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX9: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX9: [[TRUNC:%[0-9]+]]:_(s33) = G_TRUNC [[COPY]](s64)
-    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s33) = G_TRUNC [[COPY1]](s64)
-    ; GFX9: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[TRUNC]](s33)
-    ; GFX9: [[ZEXT1:%[0-9]+]]:_(s64) = G_ZEXT [[TRUNC1]](s33)
-    ; GFX9: [[UDIV:%[0-9]+]]:_(s64) = G_UDIV [[ZEXT]], [[ZEXT1]]
-    ; GFX9: [[TRUNC2:%[0-9]+]]:_(s33) = G_TRUNC [[UDIV]](s64)
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[TRUNC2]](s33)
-    ; GFX9: $vgpr0_vgpr1 = COPY [[ANYEXT]](s64)
+    ; GFX9: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8589934591
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s64) = COPY [[COPY]](s64)
+    ; GFX9: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C]]
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s64) = COPY [[COPY1]](s64)
+    ; GFX9: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY3]], [[C]]
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND1]](s64)
+    ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV]](s32)
+    ; GFX9: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV1]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C1]]
+    ; GFX9: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX9: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX9: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C3]]
+    ; GFX9: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX9: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX9: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C4]]
+    ; GFX9: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX9: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX9: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C5]](s64)
+    ; GFX9: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND1]](s64)
+    ; GFX9: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV2]], [[UV4]]
+    ; GFX9: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV3]], [[UV5]], [[USUBO1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX9: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX9: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO1]](s1)
+    ; GFX9: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UADDO]], [[UMULH1]]
+    ; GFX9: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO3]](s1)
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX9: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX9: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX9: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX9: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX9: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH3]]
+    ; GFX9: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX9: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[UADDO6]], [[ADD2]]
+    ; GFX9: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX9: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX9: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX9: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX9: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO8]]
+    ; GFX9: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO11]]
+    ; GFX9: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX9: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO10]]
+    ; GFX9: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO10]]
+    ; GFX9: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE]]
+    ; GFX9: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO10]]
+    ; GFX9: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX9: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX9: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[MUL6]]
+    ; GFX9: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO10]], [[ADD8]]
+    ; GFX9: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[MUL6]]
+    ; GFX9: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX9: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX9: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[UADDO12]], [[UMULH6]]
+    ; GFX9: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO15]](s1)
+    ; GFX9: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX9: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[ADD8]]
+    ; GFX9: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[MUL6]]
+    ; GFX9: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[ADD8]]
+    ; GFX9: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX9: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX9: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH8]]
+    ; GFX9: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX9: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX9: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[UADDO18]], [[ADD9]]
+    ; GFX9: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX9: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX9: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[ADD8]]
+    ; GFX9: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX9: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[UADDO20]]
+    ; GFX9: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO11]]
+    ; GFX9: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[UADDE2]], [[C6]], [[UADDO23]]
+    ; GFX9: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND]](s64)
+    ; GFX9: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND]](s64)
+    ; GFX9: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV9]], [[UADDO22]]
+    ; GFX9: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV8]], [[UADDE4]]
+    ; GFX9: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV8]], [[UADDO22]]
+    ; GFX9: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX9: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX9: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO24]], [[UMULH10]]
+    ; GFX9: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO27]](s1)
+    ; GFX9: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX9: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV9]], [[UADDE4]]
+    ; GFX9: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV9]], [[UADDO22]]
+    ; GFX9: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV8]], [[UADDE4]]
+    ; GFX9: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX9: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX9: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH12]]
+    ; GFX9: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX9: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX9: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[UADDO30]], [[ADD13]]
+    ; GFX9: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX9: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX9: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV9]], [[UADDE4]]
+    ; GFX9: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO32]](s32), [[ADD16]](s32)
+    ; GFX9: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND1]](s64)
+    ; GFX9: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV10]], [[UADDO32]]
+    ; GFX9: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV11]], [[UADDO32]]
+    ; GFX9: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV10]], [[ADD16]]
+    ; GFX9: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV10]], [[UADDO32]]
+    ; GFX9: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX9: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX9: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV6]], [[MUL15]]
+    ; GFX9: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV7]], [[ADD18]], [[USUBO3]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV7]], [[ADD18]]
+    ; GFX9: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND1]](s64)
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV13]]
+    ; GFX9: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV12]]
+    ; GFX9: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV13]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX9: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV12]]
+    ; GFX9: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV13]], [[USUBO3]]
+    ; GFX9: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C6]], [[USUBO5]]
+    ; GFX9: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; GFX9: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C7]](s64)
+    ; GFX9: [[UADDO34:%[0-9]+]]:_(s32), [[UADDO35:%[0-9]+]]:_(s1) = G_UADDO [[UADDO32]], [[UV14]]
+    ; GFX9: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[ADD16]], [[UV15]], [[UADDO35]]
+    ; GFX9: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO34]](s32), [[UADDE6]](s32)
+    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV13]]
+    ; GFX9: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX9: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV12]]
+    ; GFX9: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX9: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV13]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX9: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C7]](s64)
+    ; GFX9: [[UADDO36:%[0-9]+]]:_(s32), [[UADDO37:%[0-9]+]]:_(s1) = G_UADDO [[UADDO34]], [[UV16]]
+    ; GFX9: [[UADDE8:%[0-9]+]]:_(s32), [[UADDE9:%[0-9]+]]:_(s1) = G_UADDE [[UADDE6]], [[UV17]], [[UADDO37]]
+    ; GFX9: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UADDO36]](s32), [[UADDE8]](s32)
+    ; GFX9: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV12]]
+    ; GFX9: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV13]], [[USUBO5]]
+    ; GFX9: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C6]], [[USUBO7]]
+    ; GFX9: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C6]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV2]], [[MV1]]
+    ; GFX9: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C6]]
+    ; GFX9: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV]]
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s64) = COPY [[SELECT3]](s64)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[COPY4]](s64)
     %0:_(s64) = COPY $vgpr0_vgpr1
     %1:_(s64) = COPY $vgpr2_vgpr3
     %2:_(s33) = G_TRUNC %0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-urem.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-urem.mir
index 973cbb61c5889..e42fe1400477f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-urem.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-urem.mir
@@ -1,7 +1,7 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX6 %s
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer  -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX8 %s
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -O0 -run-pass=legalizer  -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX9 %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer -o - %s | FileCheck -check-prefix=GFX6 %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer -o - %s | FileCheck -check-prefix=GFX8 %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -O0 -run-pass=legalizer -o - %s | FileCheck -check-prefix=GFX9 %s
 
 ---
 name: test_urem_s32
@@ -291,18 +291,441 @@ body: |
     ; GFX6-LABEL: name: test_urem_s64
     ; GFX6: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX6: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX6: [[UREM:%[0-9]+]]:_(s64) = G_UREM [[COPY]], [[COPY1]]
-    ; GFX6: $vgpr0_vgpr1 = COPY [[UREM]](s64)
+    ; GFX6: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV]](s32)
+    ; GFX6: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV1]](s32)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C]]
+    ; GFX6: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX6: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX6: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C2]]
+    ; GFX6: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX6: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C3]]
+    ; GFX6: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX6: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX6: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX6: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C4]](s64)
+    ; GFX6: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX6: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV2]], [[UV4]]
+    ; GFX6: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV3]], [[UV5]], [[USUBO1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX6: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX6: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX6: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX6: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO1]](s1)
+    ; GFX6: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UADDO]], [[UMULH1]]
+    ; GFX6: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO3]](s1)
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX6: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX6: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX6: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX6: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX6: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH3]]
+    ; GFX6: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX6: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[UADDO6]], [[ADD2]]
+    ; GFX6: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX6: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX6: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX6: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX6: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO8]]
+    ; GFX6: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO11]]
+    ; GFX6: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX6: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO10]]
+    ; GFX6: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO10]]
+    ; GFX6: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE]]
+    ; GFX6: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO10]]
+    ; GFX6: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX6: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX6: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[MUL6]]
+    ; GFX6: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO10]], [[ADD8]]
+    ; GFX6: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[MUL6]]
+    ; GFX6: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX6: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX6: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[UADDO12]], [[UMULH6]]
+    ; GFX6: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO15]](s1)
+    ; GFX6: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX6: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[ADD8]]
+    ; GFX6: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[MUL6]]
+    ; GFX6: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[ADD8]]
+    ; GFX6: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX6: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX6: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH8]]
+    ; GFX6: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX6: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX6: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[UADDO18]], [[ADD9]]
+    ; GFX6: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX6: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX6: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[ADD8]]
+    ; GFX6: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX6: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX6: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[UADDO20]]
+    ; GFX6: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO11]]
+    ; GFX6: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[UADDE2]], [[C5]], [[UADDO23]]
+    ; GFX6: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX6: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX6: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV9]], [[UADDO22]]
+    ; GFX6: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV8]], [[UADDE4]]
+    ; GFX6: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV8]], [[UADDO22]]
+    ; GFX6: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX6: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX6: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO24]], [[UMULH10]]
+    ; GFX6: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO27]](s1)
+    ; GFX6: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX6: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV9]], [[UADDE4]]
+    ; GFX6: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV9]], [[UADDO22]]
+    ; GFX6: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV8]], [[UADDE4]]
+    ; GFX6: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX6: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX6: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH12]]
+    ; GFX6: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX6: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX6: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[UADDO30]], [[ADD13]]
+    ; GFX6: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX6: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX6: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV9]], [[UADDE4]]
+    ; GFX6: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX6: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX6: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV10]], [[UADDO32]]
+    ; GFX6: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV11]], [[UADDO32]]
+    ; GFX6: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV10]], [[ADD16]]
+    ; GFX6: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV10]], [[UADDO32]]
+    ; GFX6: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX6: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX6: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV6]], [[MUL15]]
+    ; GFX6: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV7]], [[ADD18]], [[USUBO3]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV7]], [[ADD18]]
+    ; GFX6: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO2]](s32), [[USUBE2]](s32)
+    ; GFX6: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV13]]
+    ; GFX6: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV12]]
+    ; GFX6: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV13]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX6: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV12]]
+    ; GFX6: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV13]], [[USUBO3]]
+    ; GFX6: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C5]], [[USUBO5]]
+    ; GFX6: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO4]](s32), [[USUBE6]](s32)
+    ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV13]]
+    ; GFX6: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX6: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV12]]
+    ; GFX6: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX6: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV13]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX6: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV12]]
+    ; GFX6: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV13]], [[USUBO5]]
+    ; GFX6: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C5]], [[USUBO7]]
+    ; GFX6: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO6]](s32), [[USUBE10]](s32)
+    ; GFX6: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C5]]
+    ; GFX6: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV2]], [[MV1]]
+    ; GFX6: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C5]]
+    ; GFX6: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV]]
+    ; GFX6: $vgpr0_vgpr1 = COPY [[SELECT3]](s64)
     ; GFX8-LABEL: name: test_urem_s64
     ; GFX8: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX8: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX8: [[UREM:%[0-9]+]]:_(s64) = G_UREM [[COPY]], [[COPY1]]
-    ; GFX8: $vgpr0_vgpr1 = COPY [[UREM]](s64)
+    ; GFX8: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV]](s32)
+    ; GFX8: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV1]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C]]
+    ; GFX8: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX8: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX8: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C2]]
+    ; GFX8: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX8: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C3]]
+    ; GFX8: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX8: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX8: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX8: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C4]](s64)
+    ; GFX8: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX8: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV2]], [[UV4]]
+    ; GFX8: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV3]], [[UV5]], [[USUBO1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX8: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO1]](s1)
+    ; GFX8: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UADDO]], [[UMULH1]]
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO3]](s1)
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX8: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX8: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX8: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX8: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX8: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH3]]
+    ; GFX8: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX8: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[UADDO6]], [[ADD2]]
+    ; GFX8: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX8: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX8: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX8: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX8: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO8]]
+    ; GFX8: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO11]]
+    ; GFX8: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX8: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO10]]
+    ; GFX8: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO10]]
+    ; GFX8: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE]]
+    ; GFX8: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO10]]
+    ; GFX8: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX8: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX8: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[MUL6]]
+    ; GFX8: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO10]], [[ADD8]]
+    ; GFX8: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[MUL6]]
+    ; GFX8: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX8: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX8: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[UADDO12]], [[UMULH6]]
+    ; GFX8: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO15]](s1)
+    ; GFX8: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX8: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[ADD8]]
+    ; GFX8: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[MUL6]]
+    ; GFX8: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[ADD8]]
+    ; GFX8: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX8: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX8: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH8]]
+    ; GFX8: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX8: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX8: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[UADDO18]], [[ADD9]]
+    ; GFX8: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX8: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX8: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[ADD8]]
+    ; GFX8: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX8: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[UADDO20]]
+    ; GFX8: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO11]]
+    ; GFX8: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[UADDE2]], [[C5]], [[UADDO23]]
+    ; GFX8: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX8: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX8: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV9]], [[UADDO22]]
+    ; GFX8: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV8]], [[UADDE4]]
+    ; GFX8: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV8]], [[UADDO22]]
+    ; GFX8: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX8: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX8: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO24]], [[UMULH10]]
+    ; GFX8: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO27]](s1)
+    ; GFX8: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX8: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV9]], [[UADDE4]]
+    ; GFX8: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV9]], [[UADDO22]]
+    ; GFX8: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV8]], [[UADDE4]]
+    ; GFX8: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX8: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX8: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH12]]
+    ; GFX8: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX8: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX8: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[UADDO30]], [[ADD13]]
+    ; GFX8: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX8: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX8: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV9]], [[UADDE4]]
+    ; GFX8: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX8: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX8: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV10]], [[UADDO32]]
+    ; GFX8: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV11]], [[UADDO32]]
+    ; GFX8: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV10]], [[ADD16]]
+    ; GFX8: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV10]], [[UADDO32]]
+    ; GFX8: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX8: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX8: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV6]], [[MUL15]]
+    ; GFX8: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV7]], [[ADD18]], [[USUBO3]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV7]], [[ADD18]]
+    ; GFX8: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO2]](s32), [[USUBE2]](s32)
+    ; GFX8: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV13]]
+    ; GFX8: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV12]]
+    ; GFX8: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV13]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX8: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV12]]
+    ; GFX8: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV13]], [[USUBO3]]
+    ; GFX8: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C5]], [[USUBO5]]
+    ; GFX8: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO4]](s32), [[USUBE6]](s32)
+    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV13]]
+    ; GFX8: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX8: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV12]]
+    ; GFX8: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX8: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV13]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX8: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV12]]
+    ; GFX8: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV13]], [[USUBO5]]
+    ; GFX8: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C5]], [[USUBO7]]
+    ; GFX8: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO6]](s32), [[USUBE10]](s32)
+    ; GFX8: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C5]]
+    ; GFX8: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV2]], [[MV1]]
+    ; GFX8: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C5]]
+    ; GFX8: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV]]
+    ; GFX8: $vgpr0_vgpr1 = COPY [[SELECT3]](s64)
     ; GFX9-LABEL: name: test_urem_s64
     ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX9: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX9: [[UREM:%[0-9]+]]:_(s64) = G_UREM [[COPY]], [[COPY1]]
-    ; GFX9: $vgpr0_vgpr1 = COPY [[UREM]](s64)
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV]](s32)
+    ; GFX9: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV1]](s32)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C]]
+    ; GFX9: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX9: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX9: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C2]]
+    ; GFX9: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX9: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C3]]
+    ; GFX9: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX9: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX9: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C4]](s64)
+    ; GFX9: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX9: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV2]], [[UV4]]
+    ; GFX9: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV3]], [[UV5]], [[USUBO1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX9: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX9: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO1]](s1)
+    ; GFX9: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UADDO]], [[UMULH1]]
+    ; GFX9: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO3]](s1)
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX9: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX9: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX9: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX9: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX9: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH3]]
+    ; GFX9: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX9: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[UADDO6]], [[ADD2]]
+    ; GFX9: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX9: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX9: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX9: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX9: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO8]]
+    ; GFX9: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO11]]
+    ; GFX9: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX9: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO10]]
+    ; GFX9: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO10]]
+    ; GFX9: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE]]
+    ; GFX9: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO10]]
+    ; GFX9: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX9: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX9: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[MUL6]]
+    ; GFX9: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO10]], [[ADD8]]
+    ; GFX9: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[MUL6]]
+    ; GFX9: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX9: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX9: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[UADDO12]], [[UMULH6]]
+    ; GFX9: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO15]](s1)
+    ; GFX9: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX9: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[ADD8]]
+    ; GFX9: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[MUL6]]
+    ; GFX9: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[ADD8]]
+    ; GFX9: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX9: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX9: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH8]]
+    ; GFX9: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX9: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX9: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[UADDO18]], [[ADD9]]
+    ; GFX9: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX9: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX9: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[ADD8]]
+    ; GFX9: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX9: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[UADDO20]]
+    ; GFX9: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO11]]
+    ; GFX9: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[UADDE2]], [[C5]], [[UADDO23]]
+    ; GFX9: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX9: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; GFX9: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV9]], [[UADDO22]]
+    ; GFX9: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV8]], [[UADDE4]]
+    ; GFX9: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV8]], [[UADDO22]]
+    ; GFX9: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX9: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX9: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO24]], [[UMULH10]]
+    ; GFX9: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO27]](s1)
+    ; GFX9: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX9: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV9]], [[UADDE4]]
+    ; GFX9: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV9]], [[UADDO22]]
+    ; GFX9: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV8]], [[UADDE4]]
+    ; GFX9: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX9: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX9: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH12]]
+    ; GFX9: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX9: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX9: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[UADDO30]], [[ADD13]]
+    ; GFX9: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX9: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX9: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV9]], [[UADDE4]]
+    ; GFX9: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX9: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX9: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV10]], [[UADDO32]]
+    ; GFX9: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV11]], [[UADDO32]]
+    ; GFX9: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV10]], [[ADD16]]
+    ; GFX9: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV10]], [[UADDO32]]
+    ; GFX9: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX9: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX9: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV6]], [[MUL15]]
+    ; GFX9: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV7]], [[ADD18]], [[USUBO3]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV7]], [[ADD18]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO2]](s32), [[USUBE2]](s32)
+    ; GFX9: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV13]]
+    ; GFX9: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV12]]
+    ; GFX9: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV13]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX9: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV12]]
+    ; GFX9: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV13]], [[USUBO3]]
+    ; GFX9: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C5]], [[USUBO5]]
+    ; GFX9: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO4]](s32), [[USUBE6]](s32)
+    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV13]]
+    ; GFX9: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX9: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV12]]
+    ; GFX9: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX9: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV13]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX9: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV12]]
+    ; GFX9: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV13]], [[USUBO5]]
+    ; GFX9: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C5]], [[USUBO7]]
+    ; GFX9: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO6]](s32), [[USUBE10]](s32)
+    ; GFX9: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C5]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV2]], [[MV1]]
+    ; GFX9: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C5]]
+    ; GFX9: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV]]
+    ; GFX9: $vgpr0_vgpr1 = COPY [[SELECT3]](s64)
     %0:_(s64) = COPY $vgpr0_vgpr1
     %1:_(s64) = COPY $vgpr2_vgpr3
     %2:_(s64) = G_UREM %0, %1
@@ -320,27 +743,867 @@ body: |
     ; GFX6: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
     ; GFX6: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
     ; GFX6: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
-    ; GFX6: [[UREM:%[0-9]+]]:_(s64) = G_UREM [[UV]], [[UV2]]
-    ; GFX6: [[UREM1:%[0-9]+]]:_(s64) = G_UREM [[UV1]], [[UV3]]
-    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[UREM]](s64), [[UREM1]](s64)
+    ; GFX6: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV4]](s32)
+    ; GFX6: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV5]](s32)
+    ; GFX6: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C]]
+    ; GFX6: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX6: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX6: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C2]]
+    ; GFX6: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX6: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C3]]
+    ; GFX6: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX6: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX6: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX6: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C4]](s64)
+    ; GFX6: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX6: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV6]], [[UV8]]
+    ; GFX6: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV7]], [[UV9]], [[USUBO1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX6: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX6: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX6: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX6: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO1]](s1)
+    ; GFX6: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UADDO]], [[UMULH1]]
+    ; GFX6: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO3]](s1)
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX6: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX6: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX6: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX6: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX6: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH3]]
+    ; GFX6: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX6: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[UADDO6]], [[ADD2]]
+    ; GFX6: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX6: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX6: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX6: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX6: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO8]]
+    ; GFX6: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO11]]
+    ; GFX6: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX6: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO10]]
+    ; GFX6: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO10]]
+    ; GFX6: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE]]
+    ; GFX6: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO10]]
+    ; GFX6: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX6: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX6: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[MUL6]]
+    ; GFX6: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO10]], [[ADD8]]
+    ; GFX6: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[MUL6]]
+    ; GFX6: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX6: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX6: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[UADDO12]], [[UMULH6]]
+    ; GFX6: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO15]](s1)
+    ; GFX6: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX6: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[ADD8]]
+    ; GFX6: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[MUL6]]
+    ; GFX6: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[ADD8]]
+    ; GFX6: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX6: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX6: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH8]]
+    ; GFX6: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX6: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX6: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[UADDO18]], [[ADD9]]
+    ; GFX6: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX6: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX6: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[ADD8]]
+    ; GFX6: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX6: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX6: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[UADDO20]]
+    ; GFX6: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO11]]
+    ; GFX6: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[UADDE2]], [[C5]], [[UADDO23]]
+    ; GFX6: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+    ; GFX6: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+    ; GFX6: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV13]], [[UADDO22]]
+    ; GFX6: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV12]], [[UADDE4]]
+    ; GFX6: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV12]], [[UADDO22]]
+    ; GFX6: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX6: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX6: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO24]], [[UMULH10]]
+    ; GFX6: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO27]](s1)
+    ; GFX6: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX6: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV13]], [[UADDE4]]
+    ; GFX6: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV13]], [[UADDO22]]
+    ; GFX6: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV12]], [[UADDE4]]
+    ; GFX6: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX6: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX6: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH12]]
+    ; GFX6: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX6: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX6: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[UADDO30]], [[ADD13]]
+    ; GFX6: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX6: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX6: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV13]], [[UADDE4]]
+    ; GFX6: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX6: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX6: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV14]], [[UADDO32]]
+    ; GFX6: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV15]], [[UADDO32]]
+    ; GFX6: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV14]], [[ADD16]]
+    ; GFX6: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV14]], [[UADDO32]]
+    ; GFX6: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX6: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX6: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV10]], [[MUL15]]
+    ; GFX6: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV11]], [[ADD18]], [[USUBO3]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV11]], [[ADD18]]
+    ; GFX6: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO2]](s32), [[USUBE2]](s32)
+    ; GFX6: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV17]]
+    ; GFX6: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV16]]
+    ; GFX6: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV17]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX6: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV16]]
+    ; GFX6: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV17]], [[USUBO3]]
+    ; GFX6: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C5]], [[USUBO5]]
+    ; GFX6: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO4]](s32), [[USUBE6]](s32)
+    ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV17]]
+    ; GFX6: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX6: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV16]]
+    ; GFX6: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX6: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV17]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX6: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV16]]
+    ; GFX6: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV17]], [[USUBO5]]
+    ; GFX6: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C5]], [[USUBO7]]
+    ; GFX6: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO6]](s32), [[USUBE10]](s32)
+    ; GFX6: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C5]]
+    ; GFX6: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV2]], [[MV1]]
+    ; GFX6: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C5]]
+    ; GFX6: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV]]
+    ; GFX6: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX6: [[UITOFP2:%[0-9]+]]:_(s32) = G_UITOFP [[UV18]](s32)
+    ; GFX6: [[UITOFP3:%[0-9]+]]:_(s32) = G_UITOFP [[UV19]](s32)
+    ; GFX6: [[C6:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[FMUL4:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP3]], [[C6]]
+    ; GFX6: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FMUL4]], [[UITOFP2]]
+    ; GFX6: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD2]](s32)
+    ; GFX6: [[C7:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX6: [[FMUL5:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C7]]
+    ; GFX6: [[C8:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX6: [[FMUL6:%[0-9]+]]:_(s32) = G_FMUL [[FMUL5]], [[C8]]
+    ; GFX6: [[INTRINSIC_TRUNC1:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL6]]
+    ; GFX6: [[C9:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX6: [[FMUL7:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC1]], [[C9]]
+    ; GFX6: [[FADD3:%[0-9]+]]:_(s32) = G_FADD [[FMUL7]], [[FMUL5]]
+    ; GFX6: [[FPTOUI2:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD3]](s32)
+    ; GFX6: [[FPTOUI3:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC1]](s32)
+    ; GFX6: [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C4]](s64)
+    ; GFX6: [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX6: [[USUBO8:%[0-9]+]]:_(s32), [[USUBO9:%[0-9]+]]:_(s1) = G_USUBO [[UV20]], [[UV22]]
+    ; GFX6: [[USUBE12:%[0-9]+]]:_(s32), [[USUBE13:%[0-9]+]]:_(s1) = G_USUBE [[UV21]], [[UV23]], [[USUBO9]]
+    ; GFX6: [[MUL18:%[0-9]+]]:_(s32) = G_MUL [[USUBO8]], [[FPTOUI2]]
+    ; GFX6: [[MUL19:%[0-9]+]]:_(s32) = G_MUL [[USUBE12]], [[FPTOUI2]]
+    ; GFX6: [[MUL20:%[0-9]+]]:_(s32) = G_MUL [[USUBO8]], [[FPTOUI3]]
+    ; GFX6: [[UMULH15:%[0-9]+]]:_(s32) = G_UMULH [[USUBO8]], [[FPTOUI2]]
+    ; GFX6: [[ADD19:%[0-9]+]]:_(s32) = G_ADD [[MUL19]], [[MUL20]]
+    ; GFX6: [[ADD20:%[0-9]+]]:_(s32) = G_ADD [[ADD19]], [[UMULH15]]
+    ; GFX6: [[MUL21:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI3]], [[MUL18]]
+    ; GFX6: [[MUL22:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI2]], [[ADD20]]
+    ; GFX6: [[UMULH16:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI2]], [[MUL18]]
+    ; GFX6: [[UADDO34:%[0-9]+]]:_(s32), [[UADDO35:%[0-9]+]]:_(s1) = G_UADDO [[MUL21]], [[MUL22]]
+    ; GFX6: [[ZEXT15:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO35]](s1)
+    ; GFX6: [[UADDO36:%[0-9]+]]:_(s32), [[UADDO37:%[0-9]+]]:_(s1) = G_UADDO [[UADDO34]], [[UMULH16]]
+    ; GFX6: [[ZEXT16:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO37]](s1)
+    ; GFX6: [[ADD21:%[0-9]+]]:_(s32) = G_ADD [[ZEXT15]], [[ZEXT16]]
+    ; GFX6: [[MUL23:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI3]], [[ADD20]]
+    ; GFX6: [[UMULH17:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI3]], [[MUL18]]
+    ; GFX6: [[UMULH18:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI2]], [[ADD20]]
+    ; GFX6: [[UADDO38:%[0-9]+]]:_(s32), [[UADDO39:%[0-9]+]]:_(s1) = G_UADDO [[MUL23]], [[UMULH17]]
+    ; GFX6: [[ZEXT17:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO39]](s1)
+    ; GFX6: [[UADDO40:%[0-9]+]]:_(s32), [[UADDO41:%[0-9]+]]:_(s1) = G_UADDO [[UADDO38]], [[UMULH18]]
+    ; GFX6: [[ZEXT18:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO41]](s1)
+    ; GFX6: [[ADD22:%[0-9]+]]:_(s32) = G_ADD [[ZEXT17]], [[ZEXT18]]
+    ; GFX6: [[UADDO42:%[0-9]+]]:_(s32), [[UADDO43:%[0-9]+]]:_(s1) = G_UADDO [[UADDO40]], [[ADD21]]
+    ; GFX6: [[ZEXT19:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO43]](s1)
+    ; GFX6: [[ADD23:%[0-9]+]]:_(s32) = G_ADD [[ADD22]], [[ZEXT19]]
+    ; GFX6: [[UMULH19:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI3]], [[ADD20]]
+    ; GFX6: [[ADD24:%[0-9]+]]:_(s32) = G_ADD [[UMULH19]], [[ADD23]]
+    ; GFX6: [[UADDO44:%[0-9]+]]:_(s32), [[UADDO45:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI2]], [[UADDO42]]
+    ; GFX6: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI3]], [[ADD24]], [[UADDO45]]
+    ; GFX6: [[ADD25:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI3]], [[ADD24]]
+    ; GFX6: [[MUL24:%[0-9]+]]:_(s32) = G_MUL [[USUBO8]], [[UADDO44]]
+    ; GFX6: [[MUL25:%[0-9]+]]:_(s32) = G_MUL [[USUBE12]], [[UADDO44]]
+    ; GFX6: [[MUL26:%[0-9]+]]:_(s32) = G_MUL [[USUBO8]], [[UADDE6]]
+    ; GFX6: [[UMULH20:%[0-9]+]]:_(s32) = G_UMULH [[USUBO8]], [[UADDO44]]
+    ; GFX6: [[ADD26:%[0-9]+]]:_(s32) = G_ADD [[MUL25]], [[MUL26]]
+    ; GFX6: [[ADD27:%[0-9]+]]:_(s32) = G_ADD [[ADD26]], [[UMULH20]]
+    ; GFX6: [[MUL27:%[0-9]+]]:_(s32) = G_MUL [[UADDE6]], [[MUL24]]
+    ; GFX6: [[MUL28:%[0-9]+]]:_(s32) = G_MUL [[UADDO44]], [[ADD27]]
+    ; GFX6: [[UMULH21:%[0-9]+]]:_(s32) = G_UMULH [[UADDO44]], [[MUL24]]
+    ; GFX6: [[UADDO46:%[0-9]+]]:_(s32), [[UADDO47:%[0-9]+]]:_(s1) = G_UADDO [[MUL27]], [[MUL28]]
+    ; GFX6: [[ZEXT20:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO47]](s1)
+    ; GFX6: [[UADDO48:%[0-9]+]]:_(s32), [[UADDO49:%[0-9]+]]:_(s1) = G_UADDO [[UADDO46]], [[UMULH21]]
+    ; GFX6: [[ZEXT21:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO49]](s1)
+    ; GFX6: [[ADD28:%[0-9]+]]:_(s32) = G_ADD [[ZEXT20]], [[ZEXT21]]
+    ; GFX6: [[MUL29:%[0-9]+]]:_(s32) = G_MUL [[UADDE6]], [[ADD27]]
+    ; GFX6: [[UMULH22:%[0-9]+]]:_(s32) = G_UMULH [[UADDE6]], [[MUL24]]
+    ; GFX6: [[UMULH23:%[0-9]+]]:_(s32) = G_UMULH [[UADDO44]], [[ADD27]]
+    ; GFX6: [[UADDO50:%[0-9]+]]:_(s32), [[UADDO51:%[0-9]+]]:_(s1) = G_UADDO [[MUL29]], [[UMULH22]]
+    ; GFX6: [[ZEXT22:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO51]](s1)
+    ; GFX6: [[UADDO52:%[0-9]+]]:_(s32), [[UADDO53:%[0-9]+]]:_(s1) = G_UADDO [[UADDO50]], [[UMULH23]]
+    ; GFX6: [[ZEXT23:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO53]](s1)
+    ; GFX6: [[ADD29:%[0-9]+]]:_(s32) = G_ADD [[ZEXT22]], [[ZEXT23]]
+    ; GFX6: [[UADDO54:%[0-9]+]]:_(s32), [[UADDO55:%[0-9]+]]:_(s1) = G_UADDO [[UADDO52]], [[ADD28]]
+    ; GFX6: [[ZEXT24:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO55]](s1)
+    ; GFX6: [[ADD30:%[0-9]+]]:_(s32) = G_ADD [[ADD29]], [[ZEXT24]]
+    ; GFX6: [[UMULH24:%[0-9]+]]:_(s32) = G_UMULH [[UADDE6]], [[ADD27]]
+    ; GFX6: [[ADD31:%[0-9]+]]:_(s32) = G_ADD [[UMULH24]], [[ADD30]]
+    ; GFX6: [[UADDO56:%[0-9]+]]:_(s32), [[UADDO57:%[0-9]+]]:_(s1) = G_UADDO [[UADDO44]], [[UADDO54]]
+    ; GFX6: [[UADDE8:%[0-9]+]]:_(s32), [[UADDE9:%[0-9]+]]:_(s1) = G_UADDE [[ADD25]], [[ADD31]], [[UADDO45]]
+    ; GFX6: [[UADDE10:%[0-9]+]]:_(s32), [[UADDE11:%[0-9]+]]:_(s1) = G_UADDE [[UADDE8]], [[C5]], [[UADDO57]]
+    ; GFX6: [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+    ; GFX6: [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+    ; GFX6: [[MUL30:%[0-9]+]]:_(s32) = G_MUL [[UV27]], [[UADDO56]]
+    ; GFX6: [[MUL31:%[0-9]+]]:_(s32) = G_MUL [[UV26]], [[UADDE10]]
+    ; GFX6: [[UMULH25:%[0-9]+]]:_(s32) = G_UMULH [[UV26]], [[UADDO56]]
+    ; GFX6: [[UADDO58:%[0-9]+]]:_(s32), [[UADDO59:%[0-9]+]]:_(s1) = G_UADDO [[MUL30]], [[MUL31]]
+    ; GFX6: [[ZEXT25:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO59]](s1)
+    ; GFX6: [[UADDO60:%[0-9]+]]:_(s32), [[UADDO61:%[0-9]+]]:_(s1) = G_UADDO [[UADDO58]], [[UMULH25]]
+    ; GFX6: [[ZEXT26:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO61]](s1)
+    ; GFX6: [[ADD32:%[0-9]+]]:_(s32) = G_ADD [[ZEXT25]], [[ZEXT26]]
+    ; GFX6: [[MUL32:%[0-9]+]]:_(s32) = G_MUL [[UV27]], [[UADDE10]]
+    ; GFX6: [[UMULH26:%[0-9]+]]:_(s32) = G_UMULH [[UV27]], [[UADDO56]]
+    ; GFX6: [[UMULH27:%[0-9]+]]:_(s32) = G_UMULH [[UV26]], [[UADDE10]]
+    ; GFX6: [[UADDO62:%[0-9]+]]:_(s32), [[UADDO63:%[0-9]+]]:_(s1) = G_UADDO [[MUL32]], [[UMULH26]]
+    ; GFX6: [[ZEXT27:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO63]](s1)
+    ; GFX6: [[UADDO64:%[0-9]+]]:_(s32), [[UADDO65:%[0-9]+]]:_(s1) = G_UADDO [[UADDO62]], [[UMULH27]]
+    ; GFX6: [[ZEXT28:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO65]](s1)
+    ; GFX6: [[ADD33:%[0-9]+]]:_(s32) = G_ADD [[ZEXT27]], [[ZEXT28]]
+    ; GFX6: [[UADDO66:%[0-9]+]]:_(s32), [[UADDO67:%[0-9]+]]:_(s1) = G_UADDO [[UADDO64]], [[ADD32]]
+    ; GFX6: [[ZEXT29:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO67]](s1)
+    ; GFX6: [[ADD34:%[0-9]+]]:_(s32) = G_ADD [[ADD33]], [[ZEXT29]]
+    ; GFX6: [[UMULH28:%[0-9]+]]:_(s32) = G_UMULH [[UV27]], [[UADDE10]]
+    ; GFX6: [[ADD35:%[0-9]+]]:_(s32) = G_ADD [[UMULH28]], [[ADD34]]
+    ; GFX6: [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX6: [[MUL33:%[0-9]+]]:_(s32) = G_MUL [[UV28]], [[UADDO66]]
+    ; GFX6: [[MUL34:%[0-9]+]]:_(s32) = G_MUL [[UV29]], [[UADDO66]]
+    ; GFX6: [[MUL35:%[0-9]+]]:_(s32) = G_MUL [[UV28]], [[ADD35]]
+    ; GFX6: [[UMULH29:%[0-9]+]]:_(s32) = G_UMULH [[UV28]], [[UADDO66]]
+    ; GFX6: [[ADD36:%[0-9]+]]:_(s32) = G_ADD [[MUL34]], [[MUL35]]
+    ; GFX6: [[ADD37:%[0-9]+]]:_(s32) = G_ADD [[ADD36]], [[UMULH29]]
+    ; GFX6: [[USUBO10:%[0-9]+]]:_(s32), [[USUBO11:%[0-9]+]]:_(s1) = G_USUBO [[UV24]], [[MUL33]]
+    ; GFX6: [[USUBE14:%[0-9]+]]:_(s32), [[USUBE15:%[0-9]+]]:_(s1) = G_USUBE [[UV25]], [[ADD37]], [[USUBO11]]
+    ; GFX6: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[UV25]], [[ADD37]]
+    ; GFX6: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO10]](s32), [[USUBE14]](s32)
+    ; GFX6: [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX6: [[ICMP8:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE14]](s32), [[UV31]]
+    ; GFX6: [[SEXT4:%[0-9]+]]:_(s32) = G_SEXT [[ICMP8]](s1)
+    ; GFX6: [[ICMP9:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO10]](s32), [[UV30]]
+    ; GFX6: [[SEXT5:%[0-9]+]]:_(s32) = G_SEXT [[ICMP9]](s1)
+    ; GFX6: [[ICMP10:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE14]](s32), [[UV31]]
+    ; GFX6: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP10]](s1), [[SEXT5]], [[SEXT4]]
+    ; GFX6: [[USUBO12:%[0-9]+]]:_(s32), [[USUBO13:%[0-9]+]]:_(s1) = G_USUBO [[USUBO10]], [[UV30]]
+    ; GFX6: [[USUBE16:%[0-9]+]]:_(s32), [[USUBE17:%[0-9]+]]:_(s1) = G_USUBE [[SUB1]], [[UV31]], [[USUBO11]]
+    ; GFX6: [[USUBE18:%[0-9]+]]:_(s32), [[USUBE19:%[0-9]+]]:_(s1) = G_USUBE [[USUBE16]], [[C5]], [[USUBO13]]
+    ; GFX6: [[MV4:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO12]](s32), [[USUBE18]](s32)
+    ; GFX6: [[ICMP11:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE18]](s32), [[UV31]]
+    ; GFX6: [[SEXT6:%[0-9]+]]:_(s32) = G_SEXT [[ICMP11]](s1)
+    ; GFX6: [[ICMP12:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO12]](s32), [[UV30]]
+    ; GFX6: [[SEXT7:%[0-9]+]]:_(s32) = G_SEXT [[ICMP12]](s1)
+    ; GFX6: [[ICMP13:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE18]](s32), [[UV31]]
+    ; GFX6: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP13]](s1), [[SEXT7]], [[SEXT6]]
+    ; GFX6: [[USUBO14:%[0-9]+]]:_(s32), [[USUBO15:%[0-9]+]]:_(s1) = G_USUBO [[USUBO12]], [[UV30]]
+    ; GFX6: [[USUBE20:%[0-9]+]]:_(s32), [[USUBE21:%[0-9]+]]:_(s1) = G_USUBE [[USUBE16]], [[UV31]], [[USUBO13]]
+    ; GFX6: [[USUBE22:%[0-9]+]]:_(s32), [[USUBE23:%[0-9]+]]:_(s1) = G_USUBE [[USUBE20]], [[C5]], [[USUBO15]]
+    ; GFX6: [[MV5:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO14]](s32), [[USUBE22]](s32)
+    ; GFX6: [[ICMP14:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT5]](s32), [[C5]]
+    ; GFX6: [[SELECT6:%[0-9]+]]:_(s64) = G_SELECT [[ICMP14]](s1), [[MV5]], [[MV4]]
+    ; GFX6: [[ICMP15:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT4]](s32), [[C5]]
+    ; GFX6: [[SELECT7:%[0-9]+]]:_(s64) = G_SELECT [[ICMP15]](s1), [[SELECT6]], [[MV3]]
+    ; GFX6: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SELECT3]](s64), [[SELECT7]](s64)
     ; GFX6: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     ; GFX8-LABEL: name: test_urem_v2s64
     ; GFX8: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX8: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
     ; GFX8: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
     ; GFX8: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
-    ; GFX8: [[UREM:%[0-9]+]]:_(s64) = G_UREM [[UV]], [[UV2]]
-    ; GFX8: [[UREM1:%[0-9]+]]:_(s64) = G_UREM [[UV1]], [[UV3]]
-    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[UREM]](s64), [[UREM1]](s64)
+    ; GFX8: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV4]](s32)
+    ; GFX8: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV5]](s32)
+    ; GFX8: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C]]
+    ; GFX8: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX8: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX8: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C2]]
+    ; GFX8: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX8: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C3]]
+    ; GFX8: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX8: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX8: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX8: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C4]](s64)
+    ; GFX8: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX8: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV6]], [[UV8]]
+    ; GFX8: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV7]], [[UV9]], [[USUBO1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX8: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO1]](s1)
+    ; GFX8: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UADDO]], [[UMULH1]]
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO3]](s1)
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX8: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX8: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX8: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX8: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX8: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH3]]
+    ; GFX8: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX8: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[UADDO6]], [[ADD2]]
+    ; GFX8: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX8: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX8: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX8: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX8: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO8]]
+    ; GFX8: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO11]]
+    ; GFX8: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX8: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO10]]
+    ; GFX8: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO10]]
+    ; GFX8: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE]]
+    ; GFX8: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO10]]
+    ; GFX8: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX8: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX8: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[MUL6]]
+    ; GFX8: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO10]], [[ADD8]]
+    ; GFX8: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[MUL6]]
+    ; GFX8: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX8: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX8: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[UADDO12]], [[UMULH6]]
+    ; GFX8: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO15]](s1)
+    ; GFX8: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX8: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[ADD8]]
+    ; GFX8: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[MUL6]]
+    ; GFX8: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[ADD8]]
+    ; GFX8: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX8: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX8: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH8]]
+    ; GFX8: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX8: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX8: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[UADDO18]], [[ADD9]]
+    ; GFX8: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX8: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX8: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[ADD8]]
+    ; GFX8: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX8: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[UADDO20]]
+    ; GFX8: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO11]]
+    ; GFX8: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[UADDE2]], [[C5]], [[UADDO23]]
+    ; GFX8: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+    ; GFX8: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+    ; GFX8: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV13]], [[UADDO22]]
+    ; GFX8: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV12]], [[UADDE4]]
+    ; GFX8: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV12]], [[UADDO22]]
+    ; GFX8: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX8: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX8: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO24]], [[UMULH10]]
+    ; GFX8: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO27]](s1)
+    ; GFX8: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX8: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV13]], [[UADDE4]]
+    ; GFX8: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV13]], [[UADDO22]]
+    ; GFX8: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV12]], [[UADDE4]]
+    ; GFX8: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX8: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX8: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH12]]
+    ; GFX8: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX8: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX8: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[UADDO30]], [[ADD13]]
+    ; GFX8: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX8: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX8: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV13]], [[UADDE4]]
+    ; GFX8: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX8: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX8: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV14]], [[UADDO32]]
+    ; GFX8: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV15]], [[UADDO32]]
+    ; GFX8: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV14]], [[ADD16]]
+    ; GFX8: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV14]], [[UADDO32]]
+    ; GFX8: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX8: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX8: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV10]], [[MUL15]]
+    ; GFX8: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV11]], [[ADD18]], [[USUBO3]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV11]], [[ADD18]]
+    ; GFX8: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO2]](s32), [[USUBE2]](s32)
+    ; GFX8: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV17]]
+    ; GFX8: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV16]]
+    ; GFX8: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV17]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX8: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV16]]
+    ; GFX8: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV17]], [[USUBO3]]
+    ; GFX8: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C5]], [[USUBO5]]
+    ; GFX8: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO4]](s32), [[USUBE6]](s32)
+    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV17]]
+    ; GFX8: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX8: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV16]]
+    ; GFX8: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX8: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV17]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX8: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV16]]
+    ; GFX8: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV17]], [[USUBO5]]
+    ; GFX8: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C5]], [[USUBO7]]
+    ; GFX8: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO6]](s32), [[USUBE10]](s32)
+    ; GFX8: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C5]]
+    ; GFX8: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV2]], [[MV1]]
+    ; GFX8: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C5]]
+    ; GFX8: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV]]
+    ; GFX8: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX8: [[UITOFP2:%[0-9]+]]:_(s32) = G_UITOFP [[UV18]](s32)
+    ; GFX8: [[UITOFP3:%[0-9]+]]:_(s32) = G_UITOFP [[UV19]](s32)
+    ; GFX8: [[C6:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[FMUL4:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP3]], [[C6]]
+    ; GFX8: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FMUL4]], [[UITOFP2]]
+    ; GFX8: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD2]](s32)
+    ; GFX8: [[C7:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX8: [[FMUL5:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C7]]
+    ; GFX8: [[C8:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX8: [[FMUL6:%[0-9]+]]:_(s32) = G_FMUL [[FMUL5]], [[C8]]
+    ; GFX8: [[INTRINSIC_TRUNC1:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL6]]
+    ; GFX8: [[C9:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX8: [[FMUL7:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC1]], [[C9]]
+    ; GFX8: [[FADD3:%[0-9]+]]:_(s32) = G_FADD [[FMUL7]], [[FMUL5]]
+    ; GFX8: [[FPTOUI2:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD3]](s32)
+    ; GFX8: [[FPTOUI3:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC1]](s32)
+    ; GFX8: [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C4]](s64)
+    ; GFX8: [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX8: [[USUBO8:%[0-9]+]]:_(s32), [[USUBO9:%[0-9]+]]:_(s1) = G_USUBO [[UV20]], [[UV22]]
+    ; GFX8: [[USUBE12:%[0-9]+]]:_(s32), [[USUBE13:%[0-9]+]]:_(s1) = G_USUBE [[UV21]], [[UV23]], [[USUBO9]]
+    ; GFX8: [[MUL18:%[0-9]+]]:_(s32) = G_MUL [[USUBO8]], [[FPTOUI2]]
+    ; GFX8: [[MUL19:%[0-9]+]]:_(s32) = G_MUL [[USUBE12]], [[FPTOUI2]]
+    ; GFX8: [[MUL20:%[0-9]+]]:_(s32) = G_MUL [[USUBO8]], [[FPTOUI3]]
+    ; GFX8: [[UMULH15:%[0-9]+]]:_(s32) = G_UMULH [[USUBO8]], [[FPTOUI2]]
+    ; GFX8: [[ADD19:%[0-9]+]]:_(s32) = G_ADD [[MUL19]], [[MUL20]]
+    ; GFX8: [[ADD20:%[0-9]+]]:_(s32) = G_ADD [[ADD19]], [[UMULH15]]
+    ; GFX8: [[MUL21:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI3]], [[MUL18]]
+    ; GFX8: [[MUL22:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI2]], [[ADD20]]
+    ; GFX8: [[UMULH16:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI2]], [[MUL18]]
+    ; GFX8: [[UADDO34:%[0-9]+]]:_(s32), [[UADDO35:%[0-9]+]]:_(s1) = G_UADDO [[MUL21]], [[MUL22]]
+    ; GFX8: [[ZEXT15:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO35]](s1)
+    ; GFX8: [[UADDO36:%[0-9]+]]:_(s32), [[UADDO37:%[0-9]+]]:_(s1) = G_UADDO [[UADDO34]], [[UMULH16]]
+    ; GFX8: [[ZEXT16:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO37]](s1)
+    ; GFX8: [[ADD21:%[0-9]+]]:_(s32) = G_ADD [[ZEXT15]], [[ZEXT16]]
+    ; GFX8: [[MUL23:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI3]], [[ADD20]]
+    ; GFX8: [[UMULH17:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI3]], [[MUL18]]
+    ; GFX8: [[UMULH18:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI2]], [[ADD20]]
+    ; GFX8: [[UADDO38:%[0-9]+]]:_(s32), [[UADDO39:%[0-9]+]]:_(s1) = G_UADDO [[MUL23]], [[UMULH17]]
+    ; GFX8: [[ZEXT17:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO39]](s1)
+    ; GFX8: [[UADDO40:%[0-9]+]]:_(s32), [[UADDO41:%[0-9]+]]:_(s1) = G_UADDO [[UADDO38]], [[UMULH18]]
+    ; GFX8: [[ZEXT18:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO41]](s1)
+    ; GFX8: [[ADD22:%[0-9]+]]:_(s32) = G_ADD [[ZEXT17]], [[ZEXT18]]
+    ; GFX8: [[UADDO42:%[0-9]+]]:_(s32), [[UADDO43:%[0-9]+]]:_(s1) = G_UADDO [[UADDO40]], [[ADD21]]
+    ; GFX8: [[ZEXT19:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO43]](s1)
+    ; GFX8: [[ADD23:%[0-9]+]]:_(s32) = G_ADD [[ADD22]], [[ZEXT19]]
+    ; GFX8: [[UMULH19:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI3]], [[ADD20]]
+    ; GFX8: [[ADD24:%[0-9]+]]:_(s32) = G_ADD [[UMULH19]], [[ADD23]]
+    ; GFX8: [[UADDO44:%[0-9]+]]:_(s32), [[UADDO45:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI2]], [[UADDO42]]
+    ; GFX8: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI3]], [[ADD24]], [[UADDO45]]
+    ; GFX8: [[ADD25:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI3]], [[ADD24]]
+    ; GFX8: [[MUL24:%[0-9]+]]:_(s32) = G_MUL [[USUBO8]], [[UADDO44]]
+    ; GFX8: [[MUL25:%[0-9]+]]:_(s32) = G_MUL [[USUBE12]], [[UADDO44]]
+    ; GFX8: [[MUL26:%[0-9]+]]:_(s32) = G_MUL [[USUBO8]], [[UADDE6]]
+    ; GFX8: [[UMULH20:%[0-9]+]]:_(s32) = G_UMULH [[USUBO8]], [[UADDO44]]
+    ; GFX8: [[ADD26:%[0-9]+]]:_(s32) = G_ADD [[MUL25]], [[MUL26]]
+    ; GFX8: [[ADD27:%[0-9]+]]:_(s32) = G_ADD [[ADD26]], [[UMULH20]]
+    ; GFX8: [[MUL27:%[0-9]+]]:_(s32) = G_MUL [[UADDE6]], [[MUL24]]
+    ; GFX8: [[MUL28:%[0-9]+]]:_(s32) = G_MUL [[UADDO44]], [[ADD27]]
+    ; GFX8: [[UMULH21:%[0-9]+]]:_(s32) = G_UMULH [[UADDO44]], [[MUL24]]
+    ; GFX8: [[UADDO46:%[0-9]+]]:_(s32), [[UADDO47:%[0-9]+]]:_(s1) = G_UADDO [[MUL27]], [[MUL28]]
+    ; GFX8: [[ZEXT20:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO47]](s1)
+    ; GFX8: [[UADDO48:%[0-9]+]]:_(s32), [[UADDO49:%[0-9]+]]:_(s1) = G_UADDO [[UADDO46]], [[UMULH21]]
+    ; GFX8: [[ZEXT21:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO49]](s1)
+    ; GFX8: [[ADD28:%[0-9]+]]:_(s32) = G_ADD [[ZEXT20]], [[ZEXT21]]
+    ; GFX8: [[MUL29:%[0-9]+]]:_(s32) = G_MUL [[UADDE6]], [[ADD27]]
+    ; GFX8: [[UMULH22:%[0-9]+]]:_(s32) = G_UMULH [[UADDE6]], [[MUL24]]
+    ; GFX8: [[UMULH23:%[0-9]+]]:_(s32) = G_UMULH [[UADDO44]], [[ADD27]]
+    ; GFX8: [[UADDO50:%[0-9]+]]:_(s32), [[UADDO51:%[0-9]+]]:_(s1) = G_UADDO [[MUL29]], [[UMULH22]]
+    ; GFX8: [[ZEXT22:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO51]](s1)
+    ; GFX8: [[UADDO52:%[0-9]+]]:_(s32), [[UADDO53:%[0-9]+]]:_(s1) = G_UADDO [[UADDO50]], [[UMULH23]]
+    ; GFX8: [[ZEXT23:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO53]](s1)
+    ; GFX8: [[ADD29:%[0-9]+]]:_(s32) = G_ADD [[ZEXT22]], [[ZEXT23]]
+    ; GFX8: [[UADDO54:%[0-9]+]]:_(s32), [[UADDO55:%[0-9]+]]:_(s1) = G_UADDO [[UADDO52]], [[ADD28]]
+    ; GFX8: [[ZEXT24:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO55]](s1)
+    ; GFX8: [[ADD30:%[0-9]+]]:_(s32) = G_ADD [[ADD29]], [[ZEXT24]]
+    ; GFX8: [[UMULH24:%[0-9]+]]:_(s32) = G_UMULH [[UADDE6]], [[ADD27]]
+    ; GFX8: [[ADD31:%[0-9]+]]:_(s32) = G_ADD [[UMULH24]], [[ADD30]]
+    ; GFX8: [[UADDO56:%[0-9]+]]:_(s32), [[UADDO57:%[0-9]+]]:_(s1) = G_UADDO [[UADDO44]], [[UADDO54]]
+    ; GFX8: [[UADDE8:%[0-9]+]]:_(s32), [[UADDE9:%[0-9]+]]:_(s1) = G_UADDE [[ADD25]], [[ADD31]], [[UADDO45]]
+    ; GFX8: [[UADDE10:%[0-9]+]]:_(s32), [[UADDE11:%[0-9]+]]:_(s1) = G_UADDE [[UADDE8]], [[C5]], [[UADDO57]]
+    ; GFX8: [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+    ; GFX8: [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+    ; GFX8: [[MUL30:%[0-9]+]]:_(s32) = G_MUL [[UV27]], [[UADDO56]]
+    ; GFX8: [[MUL31:%[0-9]+]]:_(s32) = G_MUL [[UV26]], [[UADDE10]]
+    ; GFX8: [[UMULH25:%[0-9]+]]:_(s32) = G_UMULH [[UV26]], [[UADDO56]]
+    ; GFX8: [[UADDO58:%[0-9]+]]:_(s32), [[UADDO59:%[0-9]+]]:_(s1) = G_UADDO [[MUL30]], [[MUL31]]
+    ; GFX8: [[ZEXT25:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO59]](s1)
+    ; GFX8: [[UADDO60:%[0-9]+]]:_(s32), [[UADDO61:%[0-9]+]]:_(s1) = G_UADDO [[UADDO58]], [[UMULH25]]
+    ; GFX8: [[ZEXT26:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO61]](s1)
+    ; GFX8: [[ADD32:%[0-9]+]]:_(s32) = G_ADD [[ZEXT25]], [[ZEXT26]]
+    ; GFX8: [[MUL32:%[0-9]+]]:_(s32) = G_MUL [[UV27]], [[UADDE10]]
+    ; GFX8: [[UMULH26:%[0-9]+]]:_(s32) = G_UMULH [[UV27]], [[UADDO56]]
+    ; GFX8: [[UMULH27:%[0-9]+]]:_(s32) = G_UMULH [[UV26]], [[UADDE10]]
+    ; GFX8: [[UADDO62:%[0-9]+]]:_(s32), [[UADDO63:%[0-9]+]]:_(s1) = G_UADDO [[MUL32]], [[UMULH26]]
+    ; GFX8: [[ZEXT27:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO63]](s1)
+    ; GFX8: [[UADDO64:%[0-9]+]]:_(s32), [[UADDO65:%[0-9]+]]:_(s1) = G_UADDO [[UADDO62]], [[UMULH27]]
+    ; GFX8: [[ZEXT28:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO65]](s1)
+    ; GFX8: [[ADD33:%[0-9]+]]:_(s32) = G_ADD [[ZEXT27]], [[ZEXT28]]
+    ; GFX8: [[UADDO66:%[0-9]+]]:_(s32), [[UADDO67:%[0-9]+]]:_(s1) = G_UADDO [[UADDO64]], [[ADD32]]
+    ; GFX8: [[ZEXT29:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO67]](s1)
+    ; GFX8: [[ADD34:%[0-9]+]]:_(s32) = G_ADD [[ADD33]], [[ZEXT29]]
+    ; GFX8: [[UMULH28:%[0-9]+]]:_(s32) = G_UMULH [[UV27]], [[UADDE10]]
+    ; GFX8: [[ADD35:%[0-9]+]]:_(s32) = G_ADD [[UMULH28]], [[ADD34]]
+    ; GFX8: [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX8: [[MUL33:%[0-9]+]]:_(s32) = G_MUL [[UV28]], [[UADDO66]]
+    ; GFX8: [[MUL34:%[0-9]+]]:_(s32) = G_MUL [[UV29]], [[UADDO66]]
+    ; GFX8: [[MUL35:%[0-9]+]]:_(s32) = G_MUL [[UV28]], [[ADD35]]
+    ; GFX8: [[UMULH29:%[0-9]+]]:_(s32) = G_UMULH [[UV28]], [[UADDO66]]
+    ; GFX8: [[ADD36:%[0-9]+]]:_(s32) = G_ADD [[MUL34]], [[MUL35]]
+    ; GFX8: [[ADD37:%[0-9]+]]:_(s32) = G_ADD [[ADD36]], [[UMULH29]]
+    ; GFX8: [[USUBO10:%[0-9]+]]:_(s32), [[USUBO11:%[0-9]+]]:_(s1) = G_USUBO [[UV24]], [[MUL33]]
+    ; GFX8: [[USUBE14:%[0-9]+]]:_(s32), [[USUBE15:%[0-9]+]]:_(s1) = G_USUBE [[UV25]], [[ADD37]], [[USUBO11]]
+    ; GFX8: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[UV25]], [[ADD37]]
+    ; GFX8: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO10]](s32), [[USUBE14]](s32)
+    ; GFX8: [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX8: [[ICMP8:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE14]](s32), [[UV31]]
+    ; GFX8: [[SEXT4:%[0-9]+]]:_(s32) = G_SEXT [[ICMP8]](s1)
+    ; GFX8: [[ICMP9:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO10]](s32), [[UV30]]
+    ; GFX8: [[SEXT5:%[0-9]+]]:_(s32) = G_SEXT [[ICMP9]](s1)
+    ; GFX8: [[ICMP10:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE14]](s32), [[UV31]]
+    ; GFX8: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP10]](s1), [[SEXT5]], [[SEXT4]]
+    ; GFX8: [[USUBO12:%[0-9]+]]:_(s32), [[USUBO13:%[0-9]+]]:_(s1) = G_USUBO [[USUBO10]], [[UV30]]
+    ; GFX8: [[USUBE16:%[0-9]+]]:_(s32), [[USUBE17:%[0-9]+]]:_(s1) = G_USUBE [[SUB1]], [[UV31]], [[USUBO11]]
+    ; GFX8: [[USUBE18:%[0-9]+]]:_(s32), [[USUBE19:%[0-9]+]]:_(s1) = G_USUBE [[USUBE16]], [[C5]], [[USUBO13]]
+    ; GFX8: [[MV4:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO12]](s32), [[USUBE18]](s32)
+    ; GFX8: [[ICMP11:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE18]](s32), [[UV31]]
+    ; GFX8: [[SEXT6:%[0-9]+]]:_(s32) = G_SEXT [[ICMP11]](s1)
+    ; GFX8: [[ICMP12:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO12]](s32), [[UV30]]
+    ; GFX8: [[SEXT7:%[0-9]+]]:_(s32) = G_SEXT [[ICMP12]](s1)
+    ; GFX8: [[ICMP13:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE18]](s32), [[UV31]]
+    ; GFX8: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP13]](s1), [[SEXT7]], [[SEXT6]]
+    ; GFX8: [[USUBO14:%[0-9]+]]:_(s32), [[USUBO15:%[0-9]+]]:_(s1) = G_USUBO [[USUBO12]], [[UV30]]
+    ; GFX8: [[USUBE20:%[0-9]+]]:_(s32), [[USUBE21:%[0-9]+]]:_(s1) = G_USUBE [[USUBE16]], [[UV31]], [[USUBO13]]
+    ; GFX8: [[USUBE22:%[0-9]+]]:_(s32), [[USUBE23:%[0-9]+]]:_(s1) = G_USUBE [[USUBE20]], [[C5]], [[USUBO15]]
+    ; GFX8: [[MV5:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO14]](s32), [[USUBE22]](s32)
+    ; GFX8: [[ICMP14:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT5]](s32), [[C5]]
+    ; GFX8: [[SELECT6:%[0-9]+]]:_(s64) = G_SELECT [[ICMP14]](s1), [[MV5]], [[MV4]]
+    ; GFX8: [[ICMP15:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT4]](s32), [[C5]]
+    ; GFX8: [[SELECT7:%[0-9]+]]:_(s64) = G_SELECT [[ICMP15]](s1), [[SELECT6]], [[MV3]]
+    ; GFX8: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SELECT3]](s64), [[SELECT7]](s64)
     ; GFX8: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     ; GFX9-LABEL: name: test_urem_v2s64
     ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
     ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
     ; GFX9: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
-    ; GFX9: [[UREM:%[0-9]+]]:_(s64) = G_UREM [[UV]], [[UV2]]
-    ; GFX9: [[UREM1:%[0-9]+]]:_(s64) = G_UREM [[UV1]], [[UV3]]
-    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[UREM]](s64), [[UREM1]](s64)
+    ; GFX9: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV4]](s32)
+    ; GFX9: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV5]](s32)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C]]
+    ; GFX9: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX9: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C1]]
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX9: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C2]]
+    ; GFX9: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX9: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C3]]
+    ; GFX9: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX9: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX9: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C4]](s64)
+    ; GFX9: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX9: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV6]], [[UV8]]
+    ; GFX9: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV7]], [[UV9]], [[USUBO1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX9: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX9: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO1]](s1)
+    ; GFX9: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UADDO]], [[UMULH1]]
+    ; GFX9: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO3]](s1)
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX9: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX9: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX9: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX9: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX9: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH3]]
+    ; GFX9: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX9: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[UADDO6]], [[ADD2]]
+    ; GFX9: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX9: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX9: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX9: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX9: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO8]]
+    ; GFX9: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO11]]
+    ; GFX9: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX9: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO10]]
+    ; GFX9: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO10]]
+    ; GFX9: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE]]
+    ; GFX9: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO10]]
+    ; GFX9: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX9: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX9: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[MUL6]]
+    ; GFX9: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO10]], [[ADD8]]
+    ; GFX9: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[MUL6]]
+    ; GFX9: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX9: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX9: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[UADDO12]], [[UMULH6]]
+    ; GFX9: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO15]](s1)
+    ; GFX9: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX9: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[ADD8]]
+    ; GFX9: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[MUL6]]
+    ; GFX9: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[ADD8]]
+    ; GFX9: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX9: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX9: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH8]]
+    ; GFX9: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX9: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX9: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[UADDO18]], [[ADD9]]
+    ; GFX9: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX9: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX9: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[ADD8]]
+    ; GFX9: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX9: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[UADDO20]]
+    ; GFX9: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO11]]
+    ; GFX9: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[UADDE2]], [[C5]], [[UADDO23]]
+    ; GFX9: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+    ; GFX9: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV]](s64)
+    ; GFX9: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV13]], [[UADDO22]]
+    ; GFX9: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV12]], [[UADDE4]]
+    ; GFX9: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV12]], [[UADDO22]]
+    ; GFX9: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX9: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX9: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO24]], [[UMULH10]]
+    ; GFX9: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO27]](s1)
+    ; GFX9: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX9: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV13]], [[UADDE4]]
+    ; GFX9: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV13]], [[UADDO22]]
+    ; GFX9: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV12]], [[UADDE4]]
+    ; GFX9: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX9: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX9: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH12]]
+    ; GFX9: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX9: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX9: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[UADDO30]], [[ADD13]]
+    ; GFX9: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX9: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX9: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV13]], [[UADDE4]]
+    ; GFX9: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX9: [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX9: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV14]], [[UADDO32]]
+    ; GFX9: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV15]], [[UADDO32]]
+    ; GFX9: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV14]], [[ADD16]]
+    ; GFX9: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV14]], [[UADDO32]]
+    ; GFX9: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX9: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX9: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV10]], [[MUL15]]
+    ; GFX9: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV11]], [[ADD18]], [[USUBO3]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV11]], [[ADD18]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO2]](s32), [[USUBE2]](s32)
+    ; GFX9: [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV2]](s64)
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV17]]
+    ; GFX9: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV16]]
+    ; GFX9: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV17]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX9: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV16]]
+    ; GFX9: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV17]], [[USUBO3]]
+    ; GFX9: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C5]], [[USUBO5]]
+    ; GFX9: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO4]](s32), [[USUBE6]](s32)
+    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV17]]
+    ; GFX9: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX9: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV16]]
+    ; GFX9: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX9: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV17]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX9: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV16]]
+    ; GFX9: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV17]], [[USUBO5]]
+    ; GFX9: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C5]], [[USUBO7]]
+    ; GFX9: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO6]](s32), [[USUBE10]](s32)
+    ; GFX9: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C5]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV2]], [[MV1]]
+    ; GFX9: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C5]]
+    ; GFX9: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV]]
+    ; GFX9: [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX9: [[UITOFP2:%[0-9]+]]:_(s32) = G_UITOFP [[UV18]](s32)
+    ; GFX9: [[UITOFP3:%[0-9]+]]:_(s32) = G_UITOFP [[UV19]](s32)
+    ; GFX9: [[C6:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[FMUL4:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP3]], [[C6]]
+    ; GFX9: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FMUL4]], [[UITOFP2]]
+    ; GFX9: [[AMDGPU_RCP_IFLAG1:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD2]](s32)
+    ; GFX9: [[C7:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX9: [[FMUL5:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG1]], [[C7]]
+    ; GFX9: [[C8:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX9: [[FMUL6:%[0-9]+]]:_(s32) = G_FMUL [[FMUL5]], [[C8]]
+    ; GFX9: [[INTRINSIC_TRUNC1:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL6]]
+    ; GFX9: [[C9:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX9: [[FMUL7:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC1]], [[C9]]
+    ; GFX9: [[FADD3:%[0-9]+]]:_(s32) = G_FADD [[FMUL7]], [[FMUL5]]
+    ; GFX9: [[FPTOUI2:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD3]](s32)
+    ; GFX9: [[FPTOUI3:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC1]](s32)
+    ; GFX9: [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C4]](s64)
+    ; GFX9: [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX9: [[USUBO8:%[0-9]+]]:_(s32), [[USUBO9:%[0-9]+]]:_(s1) = G_USUBO [[UV20]], [[UV22]]
+    ; GFX9: [[USUBE12:%[0-9]+]]:_(s32), [[USUBE13:%[0-9]+]]:_(s1) = G_USUBE [[UV21]], [[UV23]], [[USUBO9]]
+    ; GFX9: [[MUL18:%[0-9]+]]:_(s32) = G_MUL [[USUBO8]], [[FPTOUI2]]
+    ; GFX9: [[MUL19:%[0-9]+]]:_(s32) = G_MUL [[USUBE12]], [[FPTOUI2]]
+    ; GFX9: [[MUL20:%[0-9]+]]:_(s32) = G_MUL [[USUBO8]], [[FPTOUI3]]
+    ; GFX9: [[UMULH15:%[0-9]+]]:_(s32) = G_UMULH [[USUBO8]], [[FPTOUI2]]
+    ; GFX9: [[ADD19:%[0-9]+]]:_(s32) = G_ADD [[MUL19]], [[MUL20]]
+    ; GFX9: [[ADD20:%[0-9]+]]:_(s32) = G_ADD [[ADD19]], [[UMULH15]]
+    ; GFX9: [[MUL21:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI3]], [[MUL18]]
+    ; GFX9: [[MUL22:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI2]], [[ADD20]]
+    ; GFX9: [[UMULH16:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI2]], [[MUL18]]
+    ; GFX9: [[UADDO34:%[0-9]+]]:_(s32), [[UADDO35:%[0-9]+]]:_(s1) = G_UADDO [[MUL21]], [[MUL22]]
+    ; GFX9: [[ZEXT15:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO35]](s1)
+    ; GFX9: [[UADDO36:%[0-9]+]]:_(s32), [[UADDO37:%[0-9]+]]:_(s1) = G_UADDO [[UADDO34]], [[UMULH16]]
+    ; GFX9: [[ZEXT16:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO37]](s1)
+    ; GFX9: [[ADD21:%[0-9]+]]:_(s32) = G_ADD [[ZEXT15]], [[ZEXT16]]
+    ; GFX9: [[MUL23:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI3]], [[ADD20]]
+    ; GFX9: [[UMULH17:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI3]], [[MUL18]]
+    ; GFX9: [[UMULH18:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI2]], [[ADD20]]
+    ; GFX9: [[UADDO38:%[0-9]+]]:_(s32), [[UADDO39:%[0-9]+]]:_(s1) = G_UADDO [[MUL23]], [[UMULH17]]
+    ; GFX9: [[ZEXT17:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO39]](s1)
+    ; GFX9: [[UADDO40:%[0-9]+]]:_(s32), [[UADDO41:%[0-9]+]]:_(s1) = G_UADDO [[UADDO38]], [[UMULH18]]
+    ; GFX9: [[ZEXT18:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO41]](s1)
+    ; GFX9: [[ADD22:%[0-9]+]]:_(s32) = G_ADD [[ZEXT17]], [[ZEXT18]]
+    ; GFX9: [[UADDO42:%[0-9]+]]:_(s32), [[UADDO43:%[0-9]+]]:_(s1) = G_UADDO [[UADDO40]], [[ADD21]]
+    ; GFX9: [[ZEXT19:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO43]](s1)
+    ; GFX9: [[ADD23:%[0-9]+]]:_(s32) = G_ADD [[ADD22]], [[ZEXT19]]
+    ; GFX9: [[UMULH19:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI3]], [[ADD20]]
+    ; GFX9: [[ADD24:%[0-9]+]]:_(s32) = G_ADD [[UMULH19]], [[ADD23]]
+    ; GFX9: [[UADDO44:%[0-9]+]]:_(s32), [[UADDO45:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI2]], [[UADDO42]]
+    ; GFX9: [[UADDE6:%[0-9]+]]:_(s32), [[UADDE7:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI3]], [[ADD24]], [[UADDO45]]
+    ; GFX9: [[ADD25:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI3]], [[ADD24]]
+    ; GFX9: [[MUL24:%[0-9]+]]:_(s32) = G_MUL [[USUBO8]], [[UADDO44]]
+    ; GFX9: [[MUL25:%[0-9]+]]:_(s32) = G_MUL [[USUBE12]], [[UADDO44]]
+    ; GFX9: [[MUL26:%[0-9]+]]:_(s32) = G_MUL [[USUBO8]], [[UADDE6]]
+    ; GFX9: [[UMULH20:%[0-9]+]]:_(s32) = G_UMULH [[USUBO8]], [[UADDO44]]
+    ; GFX9: [[ADD26:%[0-9]+]]:_(s32) = G_ADD [[MUL25]], [[MUL26]]
+    ; GFX9: [[ADD27:%[0-9]+]]:_(s32) = G_ADD [[ADD26]], [[UMULH20]]
+    ; GFX9: [[MUL27:%[0-9]+]]:_(s32) = G_MUL [[UADDE6]], [[MUL24]]
+    ; GFX9: [[MUL28:%[0-9]+]]:_(s32) = G_MUL [[UADDO44]], [[ADD27]]
+    ; GFX9: [[UMULH21:%[0-9]+]]:_(s32) = G_UMULH [[UADDO44]], [[MUL24]]
+    ; GFX9: [[UADDO46:%[0-9]+]]:_(s32), [[UADDO47:%[0-9]+]]:_(s1) = G_UADDO [[MUL27]], [[MUL28]]
+    ; GFX9: [[ZEXT20:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO47]](s1)
+    ; GFX9: [[UADDO48:%[0-9]+]]:_(s32), [[UADDO49:%[0-9]+]]:_(s1) = G_UADDO [[UADDO46]], [[UMULH21]]
+    ; GFX9: [[ZEXT21:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO49]](s1)
+    ; GFX9: [[ADD28:%[0-9]+]]:_(s32) = G_ADD [[ZEXT20]], [[ZEXT21]]
+    ; GFX9: [[MUL29:%[0-9]+]]:_(s32) = G_MUL [[UADDE6]], [[ADD27]]
+    ; GFX9: [[UMULH22:%[0-9]+]]:_(s32) = G_UMULH [[UADDE6]], [[MUL24]]
+    ; GFX9: [[UMULH23:%[0-9]+]]:_(s32) = G_UMULH [[UADDO44]], [[ADD27]]
+    ; GFX9: [[UADDO50:%[0-9]+]]:_(s32), [[UADDO51:%[0-9]+]]:_(s1) = G_UADDO [[MUL29]], [[UMULH22]]
+    ; GFX9: [[ZEXT22:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO51]](s1)
+    ; GFX9: [[UADDO52:%[0-9]+]]:_(s32), [[UADDO53:%[0-9]+]]:_(s1) = G_UADDO [[UADDO50]], [[UMULH23]]
+    ; GFX9: [[ZEXT23:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO53]](s1)
+    ; GFX9: [[ADD29:%[0-9]+]]:_(s32) = G_ADD [[ZEXT22]], [[ZEXT23]]
+    ; GFX9: [[UADDO54:%[0-9]+]]:_(s32), [[UADDO55:%[0-9]+]]:_(s1) = G_UADDO [[UADDO52]], [[ADD28]]
+    ; GFX9: [[ZEXT24:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO55]](s1)
+    ; GFX9: [[ADD30:%[0-9]+]]:_(s32) = G_ADD [[ADD29]], [[ZEXT24]]
+    ; GFX9: [[UMULH24:%[0-9]+]]:_(s32) = G_UMULH [[UADDE6]], [[ADD27]]
+    ; GFX9: [[ADD31:%[0-9]+]]:_(s32) = G_ADD [[UMULH24]], [[ADD30]]
+    ; GFX9: [[UADDO56:%[0-9]+]]:_(s32), [[UADDO57:%[0-9]+]]:_(s1) = G_UADDO [[UADDO44]], [[UADDO54]]
+    ; GFX9: [[UADDE8:%[0-9]+]]:_(s32), [[UADDE9:%[0-9]+]]:_(s1) = G_UADDE [[ADD25]], [[ADD31]], [[UADDO45]]
+    ; GFX9: [[UADDE10:%[0-9]+]]:_(s32), [[UADDE11:%[0-9]+]]:_(s1) = G_UADDE [[UADDE8]], [[C5]], [[UADDO57]]
+    ; GFX9: [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+    ; GFX9: [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV1]](s64)
+    ; GFX9: [[MUL30:%[0-9]+]]:_(s32) = G_MUL [[UV27]], [[UADDO56]]
+    ; GFX9: [[MUL31:%[0-9]+]]:_(s32) = G_MUL [[UV26]], [[UADDE10]]
+    ; GFX9: [[UMULH25:%[0-9]+]]:_(s32) = G_UMULH [[UV26]], [[UADDO56]]
+    ; GFX9: [[UADDO58:%[0-9]+]]:_(s32), [[UADDO59:%[0-9]+]]:_(s1) = G_UADDO [[MUL30]], [[MUL31]]
+    ; GFX9: [[ZEXT25:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO59]](s1)
+    ; GFX9: [[UADDO60:%[0-9]+]]:_(s32), [[UADDO61:%[0-9]+]]:_(s1) = G_UADDO [[UADDO58]], [[UMULH25]]
+    ; GFX9: [[ZEXT26:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO61]](s1)
+    ; GFX9: [[ADD32:%[0-9]+]]:_(s32) = G_ADD [[ZEXT25]], [[ZEXT26]]
+    ; GFX9: [[MUL32:%[0-9]+]]:_(s32) = G_MUL [[UV27]], [[UADDE10]]
+    ; GFX9: [[UMULH26:%[0-9]+]]:_(s32) = G_UMULH [[UV27]], [[UADDO56]]
+    ; GFX9: [[UMULH27:%[0-9]+]]:_(s32) = G_UMULH [[UV26]], [[UADDE10]]
+    ; GFX9: [[UADDO62:%[0-9]+]]:_(s32), [[UADDO63:%[0-9]+]]:_(s1) = G_UADDO [[MUL32]], [[UMULH26]]
+    ; GFX9: [[ZEXT27:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO63]](s1)
+    ; GFX9: [[UADDO64:%[0-9]+]]:_(s32), [[UADDO65:%[0-9]+]]:_(s1) = G_UADDO [[UADDO62]], [[UMULH27]]
+    ; GFX9: [[ZEXT28:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO65]](s1)
+    ; GFX9: [[ADD33:%[0-9]+]]:_(s32) = G_ADD [[ZEXT27]], [[ZEXT28]]
+    ; GFX9: [[UADDO66:%[0-9]+]]:_(s32), [[UADDO67:%[0-9]+]]:_(s1) = G_UADDO [[UADDO64]], [[ADD32]]
+    ; GFX9: [[ZEXT29:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO67]](s1)
+    ; GFX9: [[ADD34:%[0-9]+]]:_(s32) = G_ADD [[ADD33]], [[ZEXT29]]
+    ; GFX9: [[UMULH28:%[0-9]+]]:_(s32) = G_UMULH [[UV27]], [[UADDE10]]
+    ; GFX9: [[ADD35:%[0-9]+]]:_(s32) = G_ADD [[UMULH28]], [[ADD34]]
+    ; GFX9: [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX9: [[MUL33:%[0-9]+]]:_(s32) = G_MUL [[UV28]], [[UADDO66]]
+    ; GFX9: [[MUL34:%[0-9]+]]:_(s32) = G_MUL [[UV29]], [[UADDO66]]
+    ; GFX9: [[MUL35:%[0-9]+]]:_(s32) = G_MUL [[UV28]], [[ADD35]]
+    ; GFX9: [[UMULH29:%[0-9]+]]:_(s32) = G_UMULH [[UV28]], [[UADDO66]]
+    ; GFX9: [[ADD36:%[0-9]+]]:_(s32) = G_ADD [[MUL34]], [[MUL35]]
+    ; GFX9: [[ADD37:%[0-9]+]]:_(s32) = G_ADD [[ADD36]], [[UMULH29]]
+    ; GFX9: [[USUBO10:%[0-9]+]]:_(s32), [[USUBO11:%[0-9]+]]:_(s1) = G_USUBO [[UV24]], [[MUL33]]
+    ; GFX9: [[USUBE14:%[0-9]+]]:_(s32), [[USUBE15:%[0-9]+]]:_(s1) = G_USUBE [[UV25]], [[ADD37]], [[USUBO11]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[UV25]], [[ADD37]]
+    ; GFX9: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO10]](s32), [[USUBE14]](s32)
+    ; GFX9: [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UV3]](s64)
+    ; GFX9: [[ICMP8:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE14]](s32), [[UV31]]
+    ; GFX9: [[SEXT4:%[0-9]+]]:_(s32) = G_SEXT [[ICMP8]](s1)
+    ; GFX9: [[ICMP9:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO10]](s32), [[UV30]]
+    ; GFX9: [[SEXT5:%[0-9]+]]:_(s32) = G_SEXT [[ICMP9]](s1)
+    ; GFX9: [[ICMP10:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE14]](s32), [[UV31]]
+    ; GFX9: [[SELECT4:%[0-9]+]]:_(s32) = G_SELECT [[ICMP10]](s1), [[SEXT5]], [[SEXT4]]
+    ; GFX9: [[USUBO12:%[0-9]+]]:_(s32), [[USUBO13:%[0-9]+]]:_(s1) = G_USUBO [[USUBO10]], [[UV30]]
+    ; GFX9: [[USUBE16:%[0-9]+]]:_(s32), [[USUBE17:%[0-9]+]]:_(s1) = G_USUBE [[SUB1]], [[UV31]], [[USUBO11]]
+    ; GFX9: [[USUBE18:%[0-9]+]]:_(s32), [[USUBE19:%[0-9]+]]:_(s1) = G_USUBE [[USUBE16]], [[C5]], [[USUBO13]]
+    ; GFX9: [[MV4:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO12]](s32), [[USUBE18]](s32)
+    ; GFX9: [[ICMP11:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE18]](s32), [[UV31]]
+    ; GFX9: [[SEXT6:%[0-9]+]]:_(s32) = G_SEXT [[ICMP11]](s1)
+    ; GFX9: [[ICMP12:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO12]](s32), [[UV30]]
+    ; GFX9: [[SEXT7:%[0-9]+]]:_(s32) = G_SEXT [[ICMP12]](s1)
+    ; GFX9: [[ICMP13:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE18]](s32), [[UV31]]
+    ; GFX9: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP13]](s1), [[SEXT7]], [[SEXT6]]
+    ; GFX9: [[USUBO14:%[0-9]+]]:_(s32), [[USUBO15:%[0-9]+]]:_(s1) = G_USUBO [[USUBO12]], [[UV30]]
+    ; GFX9: [[USUBE20:%[0-9]+]]:_(s32), [[USUBE21:%[0-9]+]]:_(s1) = G_USUBE [[USUBE16]], [[UV31]], [[USUBO13]]
+    ; GFX9: [[USUBE22:%[0-9]+]]:_(s32), [[USUBE23:%[0-9]+]]:_(s1) = G_USUBE [[USUBE20]], [[C5]], [[USUBO15]]
+    ; GFX9: [[MV5:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO14]](s32), [[USUBE22]](s32)
+    ; GFX9: [[ICMP14:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT5]](s32), [[C5]]
+    ; GFX9: [[SELECT6:%[0-9]+]]:_(s64) = G_SELECT [[ICMP14]](s1), [[MV5]], [[MV4]]
+    ; GFX9: [[ICMP15:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT4]](s32), [[C5]]
+    ; GFX9: [[SELECT7:%[0-9]+]]:_(s64) = G_SELECT [[ICMP15]](s1), [[SELECT6]], [[MV3]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SELECT3]](s64), [[SELECT7]](s64)
     ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
     %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     %1:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
@@ -950,36 +2213,459 @@ body: |
     ; GFX6-LABEL: name: test_urem_s33
     ; GFX6: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX6: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX6: [[TRUNC:%[0-9]+]]:_(s33) = G_TRUNC [[COPY]](s64)
-    ; GFX6: [[TRUNC1:%[0-9]+]]:_(s33) = G_TRUNC [[COPY1]](s64)
-    ; GFX6: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[TRUNC]](s33)
-    ; GFX6: [[ZEXT1:%[0-9]+]]:_(s64) = G_ZEXT [[TRUNC1]](s33)
-    ; GFX6: [[UREM:%[0-9]+]]:_(s64) = G_UREM [[ZEXT]], [[ZEXT1]]
-    ; GFX6: [[TRUNC2:%[0-9]+]]:_(s33) = G_TRUNC [[UREM]](s64)
-    ; GFX6: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[TRUNC2]](s33)
-    ; GFX6: $vgpr0_vgpr1 = COPY [[ANYEXT]](s64)
+    ; GFX6: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8589934591
+    ; GFX6: [[COPY2:%[0-9]+]]:_(s64) = COPY [[COPY]](s64)
+    ; GFX6: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C]]
+    ; GFX6: [[COPY3:%[0-9]+]]:_(s64) = COPY [[COPY1]](s64)
+    ; GFX6: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY3]], [[C]]
+    ; GFX6: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND1]](s64)
+    ; GFX6: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV]](s32)
+    ; GFX6: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV1]](s32)
+    ; GFX6: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX6: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C1]]
+    ; GFX6: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX6: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX6: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX6: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
+    ; GFX6: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX6: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C3]]
+    ; GFX6: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX6: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX6: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C4]]
+    ; GFX6: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX6: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX6: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX6: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX6: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C5]](s64)
+    ; GFX6: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND1]](s64)
+    ; GFX6: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV2]], [[UV4]]
+    ; GFX6: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV3]], [[UV5]], [[USUBO1]]
+    ; GFX6: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX6: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX6: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX6: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX6: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX6: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX6: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX6: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX6: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX6: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX6: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO1]](s1)
+    ; GFX6: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UADDO]], [[UMULH1]]
+    ; GFX6: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO3]](s1)
+    ; GFX6: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX6: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX6: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX6: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX6: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX6: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX6: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH3]]
+    ; GFX6: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX6: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX6: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[UADDO6]], [[ADD2]]
+    ; GFX6: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX6: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX6: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX6: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX6: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO8]]
+    ; GFX6: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO11]]
+    ; GFX6: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX6: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO10]]
+    ; GFX6: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO10]]
+    ; GFX6: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE]]
+    ; GFX6: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO10]]
+    ; GFX6: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX6: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX6: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[MUL6]]
+    ; GFX6: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO10]], [[ADD8]]
+    ; GFX6: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[MUL6]]
+    ; GFX6: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX6: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX6: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[UADDO12]], [[UMULH6]]
+    ; GFX6: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO15]](s1)
+    ; GFX6: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX6: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[ADD8]]
+    ; GFX6: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[MUL6]]
+    ; GFX6: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[ADD8]]
+    ; GFX6: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX6: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX6: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH8]]
+    ; GFX6: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX6: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX6: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[UADDO18]], [[ADD9]]
+    ; GFX6: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX6: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX6: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[ADD8]]
+    ; GFX6: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX6: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX6: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[UADDO20]]
+    ; GFX6: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO11]]
+    ; GFX6: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[UADDE2]], [[C6]], [[UADDO23]]
+    ; GFX6: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND]](s64)
+    ; GFX6: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND]](s64)
+    ; GFX6: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV9]], [[UADDO22]]
+    ; GFX6: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV8]], [[UADDE4]]
+    ; GFX6: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV8]], [[UADDO22]]
+    ; GFX6: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX6: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX6: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO24]], [[UMULH10]]
+    ; GFX6: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO27]](s1)
+    ; GFX6: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX6: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV9]], [[UADDE4]]
+    ; GFX6: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV9]], [[UADDO22]]
+    ; GFX6: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV8]], [[UADDE4]]
+    ; GFX6: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX6: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX6: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH12]]
+    ; GFX6: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX6: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX6: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[UADDO30]], [[ADD13]]
+    ; GFX6: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX6: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX6: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV9]], [[UADDE4]]
+    ; GFX6: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX6: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND1]](s64)
+    ; GFX6: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV10]], [[UADDO32]]
+    ; GFX6: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV11]], [[UADDO32]]
+    ; GFX6: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV10]], [[ADD16]]
+    ; GFX6: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV10]], [[UADDO32]]
+    ; GFX6: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX6: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX6: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV6]], [[MUL15]]
+    ; GFX6: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV7]], [[ADD18]], [[USUBO3]]
+    ; GFX6: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV7]], [[ADD18]]
+    ; GFX6: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO2]](s32), [[USUBE2]](s32)
+    ; GFX6: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND1]](s64)
+    ; GFX6: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV13]]
+    ; GFX6: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX6: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV12]]
+    ; GFX6: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX6: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV13]]
+    ; GFX6: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX6: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV12]]
+    ; GFX6: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV13]], [[USUBO3]]
+    ; GFX6: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C6]], [[USUBO5]]
+    ; GFX6: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO4]](s32), [[USUBE6]](s32)
+    ; GFX6: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV13]]
+    ; GFX6: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX6: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV12]]
+    ; GFX6: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX6: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV13]]
+    ; GFX6: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX6: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV12]]
+    ; GFX6: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV13]], [[USUBO5]]
+    ; GFX6: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C6]], [[USUBO7]]
+    ; GFX6: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO6]](s32), [[USUBE10]](s32)
+    ; GFX6: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C6]]
+    ; GFX6: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV2]], [[MV1]]
+    ; GFX6: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C6]]
+    ; GFX6: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV]]
+    ; GFX6: [[COPY4:%[0-9]+]]:_(s64) = COPY [[SELECT3]](s64)
+    ; GFX6: $vgpr0_vgpr1 = COPY [[COPY4]](s64)
     ; GFX8-LABEL: name: test_urem_s33
     ; GFX8: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX8: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX8: [[TRUNC:%[0-9]+]]:_(s33) = G_TRUNC [[COPY]](s64)
-    ; GFX8: [[TRUNC1:%[0-9]+]]:_(s33) = G_TRUNC [[COPY1]](s64)
-    ; GFX8: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[TRUNC]](s33)
-    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s64) = G_ZEXT [[TRUNC1]](s33)
-    ; GFX8: [[UREM:%[0-9]+]]:_(s64) = G_UREM [[ZEXT]], [[ZEXT1]]
-    ; GFX8: [[TRUNC2:%[0-9]+]]:_(s33) = G_TRUNC [[UREM]](s64)
-    ; GFX8: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[TRUNC2]](s33)
-    ; GFX8: $vgpr0_vgpr1 = COPY [[ANYEXT]](s64)
+    ; GFX8: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8589934591
+    ; GFX8: [[COPY2:%[0-9]+]]:_(s64) = COPY [[COPY]](s64)
+    ; GFX8: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C]]
+    ; GFX8: [[COPY3:%[0-9]+]]:_(s64) = COPY [[COPY1]](s64)
+    ; GFX8: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY3]], [[C]]
+    ; GFX8: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND1]](s64)
+    ; GFX8: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV]](s32)
+    ; GFX8: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV1]](s32)
+    ; GFX8: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX8: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C1]]
+    ; GFX8: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX8: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX8: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX8: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
+    ; GFX8: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX8: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C3]]
+    ; GFX8: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX8: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX8: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C4]]
+    ; GFX8: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX8: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX8: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX8: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX8: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C5]](s64)
+    ; GFX8: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND1]](s64)
+    ; GFX8: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV2]], [[UV4]]
+    ; GFX8: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV3]], [[UV5]], [[USUBO1]]
+    ; GFX8: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX8: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX8: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX8: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX8: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX8: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX8: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX8: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX8: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX8: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX8: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO1]](s1)
+    ; GFX8: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UADDO]], [[UMULH1]]
+    ; GFX8: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO3]](s1)
+    ; GFX8: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX8: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX8: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX8: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX8: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX8: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX8: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH3]]
+    ; GFX8: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX8: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX8: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[UADDO6]], [[ADD2]]
+    ; GFX8: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX8: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX8: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX8: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX8: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO8]]
+    ; GFX8: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO11]]
+    ; GFX8: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX8: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO10]]
+    ; GFX8: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO10]]
+    ; GFX8: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE]]
+    ; GFX8: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO10]]
+    ; GFX8: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX8: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX8: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[MUL6]]
+    ; GFX8: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO10]], [[ADD8]]
+    ; GFX8: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[MUL6]]
+    ; GFX8: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX8: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX8: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[UADDO12]], [[UMULH6]]
+    ; GFX8: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO15]](s1)
+    ; GFX8: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX8: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[ADD8]]
+    ; GFX8: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[MUL6]]
+    ; GFX8: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[ADD8]]
+    ; GFX8: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX8: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX8: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH8]]
+    ; GFX8: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX8: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX8: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[UADDO18]], [[ADD9]]
+    ; GFX8: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX8: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX8: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[ADD8]]
+    ; GFX8: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX8: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX8: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[UADDO20]]
+    ; GFX8: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO11]]
+    ; GFX8: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[UADDE2]], [[C6]], [[UADDO23]]
+    ; GFX8: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND]](s64)
+    ; GFX8: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND]](s64)
+    ; GFX8: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV9]], [[UADDO22]]
+    ; GFX8: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV8]], [[UADDE4]]
+    ; GFX8: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV8]], [[UADDO22]]
+    ; GFX8: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX8: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX8: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO24]], [[UMULH10]]
+    ; GFX8: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO27]](s1)
+    ; GFX8: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX8: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV9]], [[UADDE4]]
+    ; GFX8: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV9]], [[UADDO22]]
+    ; GFX8: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV8]], [[UADDE4]]
+    ; GFX8: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX8: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX8: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH12]]
+    ; GFX8: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX8: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX8: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[UADDO30]], [[ADD13]]
+    ; GFX8: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX8: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX8: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV9]], [[UADDE4]]
+    ; GFX8: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX8: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND1]](s64)
+    ; GFX8: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV10]], [[UADDO32]]
+    ; GFX8: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV11]], [[UADDO32]]
+    ; GFX8: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV10]], [[ADD16]]
+    ; GFX8: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV10]], [[UADDO32]]
+    ; GFX8: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX8: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX8: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV6]], [[MUL15]]
+    ; GFX8: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV7]], [[ADD18]], [[USUBO3]]
+    ; GFX8: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV7]], [[ADD18]]
+    ; GFX8: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO2]](s32), [[USUBE2]](s32)
+    ; GFX8: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND1]](s64)
+    ; GFX8: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV13]]
+    ; GFX8: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX8: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV12]]
+    ; GFX8: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX8: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV13]]
+    ; GFX8: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX8: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV12]]
+    ; GFX8: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV13]], [[USUBO3]]
+    ; GFX8: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C6]], [[USUBO5]]
+    ; GFX8: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO4]](s32), [[USUBE6]](s32)
+    ; GFX8: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV13]]
+    ; GFX8: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX8: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV12]]
+    ; GFX8: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX8: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV13]]
+    ; GFX8: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX8: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV12]]
+    ; GFX8: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV13]], [[USUBO5]]
+    ; GFX8: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C6]], [[USUBO7]]
+    ; GFX8: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO6]](s32), [[USUBE10]](s32)
+    ; GFX8: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C6]]
+    ; GFX8: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV2]], [[MV1]]
+    ; GFX8: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C6]]
+    ; GFX8: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV]]
+    ; GFX8: [[COPY4:%[0-9]+]]:_(s64) = COPY [[SELECT3]](s64)
+    ; GFX8: $vgpr0_vgpr1 = COPY [[COPY4]](s64)
     ; GFX9-LABEL: name: test_urem_s33
     ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; GFX9: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX9: [[TRUNC:%[0-9]+]]:_(s33) = G_TRUNC [[COPY]](s64)
-    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s33) = G_TRUNC [[COPY1]](s64)
-    ; GFX9: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[TRUNC]](s33)
-    ; GFX9: [[ZEXT1:%[0-9]+]]:_(s64) = G_ZEXT [[TRUNC1]](s33)
-    ; GFX9: [[UREM:%[0-9]+]]:_(s64) = G_UREM [[ZEXT]], [[ZEXT1]]
-    ; GFX9: [[TRUNC2:%[0-9]+]]:_(s33) = G_TRUNC [[UREM]](s64)
-    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[TRUNC2]](s33)
-    ; GFX9: $vgpr0_vgpr1 = COPY [[ANYEXT]](s64)
+    ; GFX9: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8589934591
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s64) = COPY [[COPY]](s64)
+    ; GFX9: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C]]
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s64) = COPY [[COPY1]](s64)
+    ; GFX9: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY3]], [[C]]
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND1]](s64)
+    ; GFX9: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[UV]](s32)
+    ; GFX9: [[UITOFP1:%[0-9]+]]:_(s32) = G_UITOFP [[UV1]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x41F0000000000000
+    ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UITOFP1]], [[C1]]
+    ; GFX9: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FMUL]], [[UITOFP]]
+    ; GFX9: [[AMDGPU_RCP_IFLAG:%[0-9]+]]:_(s32) = G_AMDGPU_RCP_IFLAG [[FADD]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x43EFFFFF80000000
+    ; GFX9: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[AMDGPU_RCP_IFLAG]], [[C2]]
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_FCONSTANT float 0x3DF0000000000000
+    ; GFX9: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FMUL1]], [[C3]]
+    ; GFX9: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FMUL2]]
+    ; GFX9: [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0xC1F0000000000000
+    ; GFX9: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[INTRINSIC_TRUNC]], [[C4]]
+    ; GFX9: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FMUL3]], [[FMUL1]]
+    ; GFX9: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FADD1]](s32)
+    ; GFX9: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[INTRINSIC_TRUNC]](s32)
+    ; GFX9: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C5]](s64)
+    ; GFX9: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND1]](s64)
+    ; GFX9: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[UV2]], [[UV4]]
+    ; GFX9: [[USUBE:%[0-9]+]]:_(s32), [[USUBE1:%[0-9]+]]:_(s1) = G_USUBE [[UV3]], [[UV5]], [[USUBO1]]
+    ; GFX9: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI]]
+    ; GFX9: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[FPTOUI]]
+    ; GFX9: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[FPTOUI1]]
+    ; GFX9: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[FPTOUI]]
+    ; GFX9: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[MUL1]], [[MUL2]]
+    ; GFX9: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[UMULH]]
+    ; GFX9: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[MUL]]
+    ; GFX9: [[MUL4:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI]], [[ADD1]]
+    ; GFX9: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[MUL]]
+    ; GFX9: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[MUL3]], [[MUL4]]
+    ; GFX9: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO1]](s1)
+    ; GFX9: [[UADDO2:%[0-9]+]]:_(s32), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UADDO]], [[UMULH1]]
+    ; GFX9: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO3]](s1)
+    ; GFX9: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+    ; GFX9: [[MUL5:%[0-9]+]]:_(s32) = G_MUL [[FPTOUI1]], [[ADD1]]
+    ; GFX9: [[UMULH2:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[MUL]]
+    ; GFX9: [[UMULH3:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI]], [[ADD1]]
+    ; GFX9: [[UADDO4:%[0-9]+]]:_(s32), [[UADDO5:%[0-9]+]]:_(s1) = G_UADDO [[MUL5]], [[UMULH2]]
+    ; GFX9: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO5]](s1)
+    ; GFX9: [[UADDO6:%[0-9]+]]:_(s32), [[UADDO7:%[0-9]+]]:_(s1) = G_UADDO [[UADDO4]], [[UMULH3]]
+    ; GFX9: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO7]](s1)
+    ; GFX9: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ZEXT3]]
+    ; GFX9: [[UADDO8:%[0-9]+]]:_(s32), [[UADDO9:%[0-9]+]]:_(s1) = G_UADDO [[UADDO6]], [[ADD2]]
+    ; GFX9: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO9]](s1)
+    ; GFX9: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[ZEXT4]]
+    ; GFX9: [[UMULH4:%[0-9]+]]:_(s32) = G_UMULH [[FPTOUI1]], [[ADD1]]
+    ; GFX9: [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UMULH4]], [[ADD4]]
+    ; GFX9: [[UADDO10:%[0-9]+]]:_(s32), [[UADDO11:%[0-9]+]]:_(s1) = G_UADDO [[FPTOUI]], [[UADDO8]]
+    ; GFX9: [[UADDE:%[0-9]+]]:_(s32), [[UADDE1:%[0-9]+]]:_(s1) = G_UADDE [[FPTOUI1]], [[ADD5]], [[UADDO11]]
+    ; GFX9: [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[FPTOUI1]], [[ADD5]]
+    ; GFX9: [[MUL6:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDO10]]
+    ; GFX9: [[MUL7:%[0-9]+]]:_(s32) = G_MUL [[USUBE]], [[UADDO10]]
+    ; GFX9: [[MUL8:%[0-9]+]]:_(s32) = G_MUL [[USUBO]], [[UADDE]]
+    ; GFX9: [[UMULH5:%[0-9]+]]:_(s32) = G_UMULH [[USUBO]], [[UADDO10]]
+    ; GFX9: [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[MUL7]], [[MUL8]]
+    ; GFX9: [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[ADD7]], [[UMULH5]]
+    ; GFX9: [[MUL9:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[MUL6]]
+    ; GFX9: [[MUL10:%[0-9]+]]:_(s32) = G_MUL [[UADDO10]], [[ADD8]]
+    ; GFX9: [[UMULH6:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[MUL6]]
+    ; GFX9: [[UADDO12:%[0-9]+]]:_(s32), [[UADDO13:%[0-9]+]]:_(s1) = G_UADDO [[MUL9]], [[MUL10]]
+    ; GFX9: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO13]](s1)
+    ; GFX9: [[UADDO14:%[0-9]+]]:_(s32), [[UADDO15:%[0-9]+]]:_(s1) = G_UADDO [[UADDO12]], [[UMULH6]]
+    ; GFX9: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO15]](s1)
+    ; GFX9: [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[ZEXT5]], [[ZEXT6]]
+    ; GFX9: [[MUL11:%[0-9]+]]:_(s32) = G_MUL [[UADDE]], [[ADD8]]
+    ; GFX9: [[UMULH7:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[MUL6]]
+    ; GFX9: [[UMULH8:%[0-9]+]]:_(s32) = G_UMULH [[UADDO10]], [[ADD8]]
+    ; GFX9: [[UADDO16:%[0-9]+]]:_(s32), [[UADDO17:%[0-9]+]]:_(s1) = G_UADDO [[MUL11]], [[UMULH7]]
+    ; GFX9: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO17]](s1)
+    ; GFX9: [[UADDO18:%[0-9]+]]:_(s32), [[UADDO19:%[0-9]+]]:_(s1) = G_UADDO [[UADDO16]], [[UMULH8]]
+    ; GFX9: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO19]](s1)
+    ; GFX9: [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[ZEXT7]], [[ZEXT8]]
+    ; GFX9: [[UADDO20:%[0-9]+]]:_(s32), [[UADDO21:%[0-9]+]]:_(s1) = G_UADDO [[UADDO18]], [[ADD9]]
+    ; GFX9: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO21]](s1)
+    ; GFX9: [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[ADD10]], [[ZEXT9]]
+    ; GFX9: [[UMULH9:%[0-9]+]]:_(s32) = G_UMULH [[UADDE]], [[ADD8]]
+    ; GFX9: [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UMULH9]], [[ADD11]]
+    ; GFX9: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[UADDO22:%[0-9]+]]:_(s32), [[UADDO23:%[0-9]+]]:_(s1) = G_UADDO [[UADDO10]], [[UADDO20]]
+    ; GFX9: [[UADDE2:%[0-9]+]]:_(s32), [[UADDE3:%[0-9]+]]:_(s1) = G_UADDE [[ADD6]], [[ADD12]], [[UADDO11]]
+    ; GFX9: [[UADDE4:%[0-9]+]]:_(s32), [[UADDE5:%[0-9]+]]:_(s1) = G_UADDE [[UADDE2]], [[C6]], [[UADDO23]]
+    ; GFX9: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND]](s64)
+    ; GFX9: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND]](s64)
+    ; GFX9: [[MUL12:%[0-9]+]]:_(s32) = G_MUL [[UV9]], [[UADDO22]]
+    ; GFX9: [[MUL13:%[0-9]+]]:_(s32) = G_MUL [[UV8]], [[UADDE4]]
+    ; GFX9: [[UMULH10:%[0-9]+]]:_(s32) = G_UMULH [[UV8]], [[UADDO22]]
+    ; GFX9: [[UADDO24:%[0-9]+]]:_(s32), [[UADDO25:%[0-9]+]]:_(s1) = G_UADDO [[MUL12]], [[MUL13]]
+    ; GFX9: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO25]](s1)
+    ; GFX9: [[UADDO26:%[0-9]+]]:_(s32), [[UADDO27:%[0-9]+]]:_(s1) = G_UADDO [[UADDO24]], [[UMULH10]]
+    ; GFX9: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO27]](s1)
+    ; GFX9: [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[ZEXT10]], [[ZEXT11]]
+    ; GFX9: [[MUL14:%[0-9]+]]:_(s32) = G_MUL [[UV9]], [[UADDE4]]
+    ; GFX9: [[UMULH11:%[0-9]+]]:_(s32) = G_UMULH [[UV9]], [[UADDO22]]
+    ; GFX9: [[UMULH12:%[0-9]+]]:_(s32) = G_UMULH [[UV8]], [[UADDE4]]
+    ; GFX9: [[UADDO28:%[0-9]+]]:_(s32), [[UADDO29:%[0-9]+]]:_(s1) = G_UADDO [[MUL14]], [[UMULH11]]
+    ; GFX9: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO29]](s1)
+    ; GFX9: [[UADDO30:%[0-9]+]]:_(s32), [[UADDO31:%[0-9]+]]:_(s1) = G_UADDO [[UADDO28]], [[UMULH12]]
+    ; GFX9: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO31]](s1)
+    ; GFX9: [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[ZEXT12]], [[ZEXT13]]
+    ; GFX9: [[UADDO32:%[0-9]+]]:_(s32), [[UADDO33:%[0-9]+]]:_(s1) = G_UADDO [[UADDO30]], [[ADD13]]
+    ; GFX9: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO33]](s1)
+    ; GFX9: [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[ADD14]], [[ZEXT14]]
+    ; GFX9: [[UMULH13:%[0-9]+]]:_(s32) = G_UMULH [[UV9]], [[UADDE4]]
+    ; GFX9: [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UMULH13]], [[ADD15]]
+    ; GFX9: [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND1]](s64)
+    ; GFX9: [[MUL15:%[0-9]+]]:_(s32) = G_MUL [[UV10]], [[UADDO32]]
+    ; GFX9: [[MUL16:%[0-9]+]]:_(s32) = G_MUL [[UV11]], [[UADDO32]]
+    ; GFX9: [[MUL17:%[0-9]+]]:_(s32) = G_MUL [[UV10]], [[ADD16]]
+    ; GFX9: [[UMULH14:%[0-9]+]]:_(s32) = G_UMULH [[UV10]], [[UADDO32]]
+    ; GFX9: [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[MUL16]], [[MUL17]]
+    ; GFX9: [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[ADD17]], [[UMULH14]]
+    ; GFX9: [[USUBO2:%[0-9]+]]:_(s32), [[USUBO3:%[0-9]+]]:_(s1) = G_USUBO [[UV6]], [[MUL15]]
+    ; GFX9: [[USUBE2:%[0-9]+]]:_(s32), [[USUBE3:%[0-9]+]]:_(s1) = G_USUBE [[UV7]], [[ADD18]], [[USUBO3]]
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV7]], [[ADD18]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO2]](s32), [[USUBE2]](s32)
+    ; GFX9: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AND1]](s64)
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE2]](s32), [[UV13]]
+    ; GFX9: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO2]](s32), [[UV12]]
+    ; GFX9: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE2]](s32), [[UV13]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[SEXT1]], [[SEXT]]
+    ; GFX9: [[USUBO4:%[0-9]+]]:_(s32), [[USUBO5:%[0-9]+]]:_(s1) = G_USUBO [[USUBO2]], [[UV12]]
+    ; GFX9: [[USUBE4:%[0-9]+]]:_(s32), [[USUBE5:%[0-9]+]]:_(s1) = G_USUBE [[SUB]], [[UV13]], [[USUBO3]]
+    ; GFX9: [[USUBE6:%[0-9]+]]:_(s32), [[USUBE7:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[C6]], [[USUBO5]]
+    ; GFX9: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO4]](s32), [[USUBE6]](s32)
+    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBE6]](s32), [[UV13]]
+    ; GFX9: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[ICMP3]](s1)
+    ; GFX9: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[USUBO4]](s32), [[UV12]]
+    ; GFX9: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[ICMP4]](s1)
+    ; GFX9: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[USUBE6]](s32), [[UV13]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SEXT3]], [[SEXT2]]
+    ; GFX9: [[USUBO6:%[0-9]+]]:_(s32), [[USUBO7:%[0-9]+]]:_(s1) = G_USUBO [[USUBO4]], [[UV12]]
+    ; GFX9: [[USUBE8:%[0-9]+]]:_(s32), [[USUBE9:%[0-9]+]]:_(s1) = G_USUBE [[USUBE4]], [[UV13]], [[USUBO5]]
+    ; GFX9: [[USUBE10:%[0-9]+]]:_(s32), [[USUBE11:%[0-9]+]]:_(s1) = G_USUBE [[USUBE8]], [[C6]], [[USUBO7]]
+    ; GFX9: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[USUBO6]](s32), [[USUBE10]](s32)
+    ; GFX9: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT1]](s32), [[C6]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[MV2]], [[MV1]]
+    ; GFX9: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SELECT]](s32), [[C6]]
+    ; GFX9: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[SELECT2]], [[MV]]
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s64) = COPY [[SELECT3]](s64)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[COPY4]](s64)
     %0:_(s64) = COPY $vgpr0_vgpr1
     %1:_(s64) = COPY $vgpr2_vgpr3
     %2:_(s33) = G_TRUNC %0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.a16.ll
new file mode 100644
index 0000000000000..304dee10c66ce
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.a16.ll
@@ -0,0 +1,1637 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -o - %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -o - %s | FileCheck -check-prefix=GFX10 %s
+
+define amdgpu_ps float @atomic_swap_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s) {
+; GFX9-LABEL: atomic_swap_i32_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_atomic_swap v0, v1, s[0:7] dmask:0x1 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_swap_i32_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_swap v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.swap.1d.i32.i16(i32 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_add_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s) {
+; GFX9-LABEL: atomic_add_i32_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_atomic_add v0, v1, s[0:7] dmask:0x1 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i32_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.add.1d.i32.i16(i32 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_sub_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s) {
+; GFX9-LABEL: atomic_sub_i32_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_atomic_sub v0, v1, s[0:7] dmask:0x1 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_sub_i32_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_sub v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.sub.1d.i32.i16(i32 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_smin_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s) {
+; GFX9-LABEL: atomic_smin_i32_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_atomic_smin v0, v1, s[0:7] dmask:0x1 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_smin_i32_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_smin v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.smin.1d.i32.i16(i32 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_umin_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s) {
+; GFX9-LABEL: atomic_umin_i32_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_atomic_umin v0, v1, s[0:7] dmask:0x1 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_umin_i32_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_umin v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.umin.1d.i32.i16(i32 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_smax_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s) {
+; GFX9-LABEL: atomic_smax_i32_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_atomic_smax v0, v1, s[0:7] dmask:0x1 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_smax_i32_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_smax v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.smax.1d.i32.i16(i32 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_umax_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s) {
+; GFX9-LABEL: atomic_umax_i32_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_atomic_umax v0, v1, s[0:7] dmask:0x1 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_umax_i32_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_umax v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.umax.1d.i32.i16(i32 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_and_i321d(<8 x i32> inreg %rsrc, i32 %data, i16 %s) {
+; GFX9-LABEL: atomic_and_i321d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_atomic_and v0, v1, s[0:7] dmask:0x1 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_and_i321d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_and v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.and.1d.i32.i16(i32 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_or_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s) {
+; GFX9-LABEL: atomic_or_i32_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_atomic_or v0, v1, s[0:7] dmask:0x1 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_or_i32_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_or v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.or.1d.i32.i16(i32 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_xor_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s) {
+; GFX9-LABEL: atomic_xor_i32_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_atomic_xor v0, v1, s[0:7] dmask:0x1 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_xor_i32_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_xor v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.xor.1d.i32.i16(i32 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_inc_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s) {
+; GFX9-LABEL: atomic_inc_i32_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_atomic_inc v0, v1, s[0:7] dmask:0x1 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_inc_i32_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_inc v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.inc.1d.i32.i16(i32 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_dec_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s) {
+; GFX9-LABEL: atomic_dec_i32_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_atomic_dec v0, v1, s[0:7] dmask:0x1 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_dec_i32_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_dec v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.dec.1d.i32.i16(i32 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_cmpswap_i32_1d(<8 x i32> inreg %rsrc, i32 %cmp, i32 %swap, i16 %s) {
+; GFX9-LABEL: atomic_cmpswap_i32_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_cmpswap_i32_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.cmpswap.1d.i32.i16(i32 %cmp, i32 %swap, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_add_i32_2d(<8 x i32> inreg %rsrc, i32 %data, i16 %s, i16 %t) {
+; GFX9-LABEL: atomic_add_i32_2d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    v_and_or_b32 v1, v1, v3, v2
+; GFX9-NEXT:    image_atomic_add v0, v1, s[0:7] dmask:0x1 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i32_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    v_and_or_b32 v1, v1, 0xffff, v2
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.add.2d.i32.i16(i32 %data, i16 %s, i16 %t, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_add_i32_3d(<8 x i32> inreg %rsrc, i32 %data, i16 %s, i16 %t, i16 %r) {
+; GFX9-LABEL: atomic_add_i32_3d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT:    s_lshl_b32 s8, s0, 16
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    v_and_or_b32 v1, v1, v4, v2
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    v_and_or_b32 v2, v3, v4, s8
+; GFX9-NEXT:    image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i32_3d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-NEXT:    s_lshl_b32 s8, s0, 16
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    v_and_or_b32 v1, v1, v4, v2
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    v_and_or_b32 v2, v3, v4, s8
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_3D unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.add.3d.i32.i16(i32 %data, i16 %s, i16 %t, i16 %r, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_add_i32_cube(<8 x i32> inreg %rsrc, i32 %data, i16 %s, i16 %t, i16 %face) {
+; GFX9-LABEL: atomic_add_i32_cube:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT:    s_lshl_b32 s8, s0, 16
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    v_and_or_b32 v1, v1, v4, v2
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    v_and_or_b32 v2, v3, v4, s8
+; GFX9-NEXT:    image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 unorm glc a16 da
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i32_cube:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-NEXT:    s_lshl_b32 s8, s0, 16
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    v_and_or_b32 v1, v1, v4, v2
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    v_and_or_b32 v2, v3, v4, s8
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_CUBE unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.add.cube.i32.i16(i32 %data, i16 %s, i16 %t, i16 %face, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_add_i32_1darray(<8 x i32> inreg %rsrc, i32 %data, i16 %s, i16 %slice) {
+; GFX9-LABEL: atomic_add_i32_1darray:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    v_and_or_b32 v1, v1, v3, v2
+; GFX9-NEXT:    image_atomic_add v0, v1, s[0:7] dmask:0x1 unorm glc a16 da
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i32_1darray:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    v_and_or_b32 v1, v1, 0xffff, v2
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D_ARRAY unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.add.1darray.i32.i16(i32 %data, i16 %s, i16 %slice, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_add_i32_2darray(<8 x i32> inreg %rsrc, i32 %data, i16 %s, i16 %t, i16 %slice) {
+; GFX9-LABEL: atomic_add_i32_2darray:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT:    s_lshl_b32 s8, s0, 16
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    v_and_or_b32 v1, v1, v4, v2
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    v_and_or_b32 v2, v3, v4, s8
+; GFX9-NEXT:    image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 unorm glc a16 da
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i32_2darray:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-NEXT:    s_lshl_b32 s8, s0, 16
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    v_and_or_b32 v1, v1, v4, v2
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    v_and_or_b32 v2, v3, v4, s8
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.add.2darray.i32.i16(i32 %data, i16 %s, i16 %t, i16 %slice, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_add_i32_2dmsaa(<8 x i32> inreg %rsrc, i32 %data, i16 %s, i16 %t, i16 %fragid) {
+; GFX9-LABEL: atomic_add_i32_2dmsaa:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT:    s_lshl_b32 s8, s0, 16
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    v_and_or_b32 v1, v1, v4, v2
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    v_and_or_b32 v2, v3, v4, s8
+; GFX9-NEXT:    image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i32_2dmsaa:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-NEXT:    s_lshl_b32 s8, s0, 16
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    v_and_or_b32 v1, v1, v4, v2
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    v_and_or_b32 v2, v3, v4, s8
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.add.2dmsaa.i32.i16(i32 %data, i16 %s, i16 %t, i16 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_add_i32_2darraymsaa(<8 x i32> inreg %rsrc, i32 %data, i16 %s, i16 %t, i16 %slice, i16 %fragid) {
+; GFX9-LABEL: atomic_add_i32_2darraymsaa:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    v_mov_b32_e32 v5, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT:    v_and_or_b32 v1, v1, v5, v2
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v4
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    v_and_or_b32 v2, v3, v5, v2
+; GFX9-NEXT:    image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 unorm glc a16 da
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i32_2darraymsaa:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    v_mov_b32_e32 v5, 0xffff
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    v_and_or_b32 v1, v1, v5, v2
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    v_and_or_b32 v2, v3, v5, v4
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.add.2darraymsaa.i32.i16(i32 %data, i16 %s, i16 %t, i16 %slice, i16 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_add_i32_1d_slc(<8 x i32> inreg %rsrc, i32 %data, i16 %s) {
+; GFX9-LABEL: atomic_add_i32_1d_slc:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_atomic_add v0, v1, s[0:7] dmask:0x1 unorm glc slc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i32_1d_slc:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc slc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.add.1d.i32.i16(i32 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 2)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps <2 x float> @atomic_swap_i64_1d(<8 x i32> inreg %rsrc, i64 %data, i16 %s) {
+; GFX9-LABEL: atomic_swap_i64_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_atomic_swap v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_swap_i64_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_swap v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.swap.1d.i64.i16(i64 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_add_i64_1d(<8 x i32> inreg %rsrc, i64 %data, i16 %s) {
+; GFX9-LABEL: atomic_add_i64_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i64_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.add.1d.i64.i16(i64 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_sub_i64_1d(<8 x i32> inreg %rsrc, i64 %data, i16 %s) {
+; GFX9-LABEL: atomic_sub_i64_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_atomic_sub v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_sub_i64_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_sub v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.sub.1d.i64.i16(i64 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_smin_i64_1d(<8 x i32> inreg %rsrc, i64 %data, i16 %s) {
+; GFX9-LABEL: atomic_smin_i64_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_atomic_smin v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_smin_i64_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_smin v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.smin.1d.i64.i16(i64 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_umin_i64_1d(<8 x i32> inreg %rsrc, i64 %data, i16 %s) {
+; GFX9-LABEL: atomic_umin_i64_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_atomic_umin v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_umin_i64_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_umin v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.umin.1d.i64.i16(i64 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_smax_i64_1d(<8 x i32> inreg %rsrc, i64 %data, i16 %s) {
+; GFX9-LABEL: atomic_smax_i64_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_atomic_smax v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_smax_i64_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_smax v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.smax.1d.i64.i16(i64 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_umax_i64_1d(<8 x i32> inreg %rsrc, i64 %data, i16 %s) {
+; GFX9-LABEL: atomic_umax_i64_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_atomic_umax v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_umax_i64_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_umax v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.umax.1d.i64.i16(i64 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_and_i64_1d(<8 x i32> inreg %rsrc, i64 %data, i16 %s) {
+; GFX9-LABEL: atomic_and_i64_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_atomic_and v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_and_i64_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_and v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.and.1d.i64.i16(i64 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_or_i64_1d(<8 x i32> inreg %rsrc, i64 %data, i16 %s) {
+; GFX9-LABEL: atomic_or_i64_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_atomic_or v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_or_i64_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_or v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.or.1d.i64.i16(i64 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_xor_i64_1d(<8 x i32> inreg %rsrc, i64 %data, i16 %s) {
+; GFX9-LABEL: atomic_xor_i64_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_atomic_xor v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_xor_i64_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_xor v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.xor.1d.i64.i16(i64 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_inc_i64_1d(<8 x i32> inreg %rsrc, i64 %data, i16 %s) {
+; GFX9-LABEL: atomic_inc_i64_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_atomic_inc v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_inc_i64_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_inc v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.inc.1d.i64.i16(i64 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_dec_i64_1d(<8 x i32> inreg %rsrc, i64 %data, i16 %s) {
+; GFX9-LABEL: atomic_dec_i64_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_atomic_dec v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_dec_i64_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_dec v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.dec.1d.i64.i16(i64 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_cmpswap_i64_1d(<8 x i32> inreg %rsrc, i64 %cmp, i64 %swap, i16 %s) {
+; GFX9-LABEL: atomic_cmpswap_i64_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_cmpswap_i64_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.cmpswap.1d.i64.i16(i64 %cmp, i64 %swap, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_add_i64_2d(<8 x i32> inreg %rsrc, i64 %data, i16 %s, i16 %t) {
+; GFX9-LABEL: atomic_add_i64_2d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    v_and_or_b32 v2, v2, v4, v3
+; GFX9-NEXT:    image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i64_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    v_and_or_b32 v2, v2, 0xffff, v3
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.add.2d.i64.i16(i64 %data, i16 %s, i16 %t, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_add_i64_3d(<8 x i32> inreg %rsrc, i64 %data, i16 %s, i16 %t, i16 %r) {
+; GFX9-LABEL: atomic_add_i64_3d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    v_mov_b32_e32 v5, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT:    s_lshl_b32 s8, s0, 16
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    v_and_or_b32 v2, v2, v5, v3
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    v_and_or_b32 v3, v4, v5, s8
+; GFX9-NEXT:    image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i64_3d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    v_mov_b32_e32 v5, 0xffff
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT:    s_lshl_b32 s8, s0, 16
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    v_and_or_b32 v2, v2, v5, v3
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    v_and_or_b32 v3, v4, v5, s8
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_3D unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.add.3d.i64.i16(i64 %data, i16 %s, i16 %t, i16 %r, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_add_i64_cube(<8 x i32> inreg %rsrc, i64 %data, i16 %s, i16 %t, i16 %face) {
+; GFX9-LABEL: atomic_add_i64_cube:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    v_mov_b32_e32 v5, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT:    s_lshl_b32 s8, s0, 16
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    v_and_or_b32 v2, v2, v5, v3
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    v_and_or_b32 v3, v4, v5, s8
+; GFX9-NEXT:    image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 unorm glc a16 da
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i64_cube:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    v_mov_b32_e32 v5, 0xffff
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT:    s_lshl_b32 s8, s0, 16
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    v_and_or_b32 v2, v2, v5, v3
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    v_and_or_b32 v3, v4, v5, s8
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_CUBE unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.add.cube.i64.i16(i64 %data, i16 %s, i16 %t, i16 %face , <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_add_i64_1darray(<8 x i32> inreg %rsrc, i64 %data, i16 %s, i16 %slice) {
+; GFX9-LABEL: atomic_add_i64_1darray:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    v_and_or_b32 v2, v2, v4, v3
+; GFX9-NEXT:    image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 unorm glc a16 da
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i64_1darray:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    v_and_or_b32 v2, v2, 0xffff, v3
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D_ARRAY unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.add.1darray.i64.i16(i64 %data, i16 %s, i16 %slice, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_add_i64_2darray(<8 x i32> inreg %rsrc, i64 %data, i16 %s, i16 %t, i16 %slice) {
+; GFX9-LABEL: atomic_add_i64_2darray:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    v_mov_b32_e32 v5, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT:    s_lshl_b32 s8, s0, 16
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    v_and_or_b32 v2, v2, v5, v3
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    v_and_or_b32 v3, v4, v5, s8
+; GFX9-NEXT:    image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 unorm glc a16 da
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i64_2darray:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    v_mov_b32_e32 v5, 0xffff
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT:    s_lshl_b32 s8, s0, 16
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    v_and_or_b32 v2, v2, v5, v3
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    v_and_or_b32 v3, v4, v5, s8
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_ARRAY unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.add.2darray.i64.i16(i64 %data, i16 %s, i16 %t, i16 %slice, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_add_i64_2dmsaa(<8 x i32> inreg %rsrc, i64 %data, i16 %s, i16 %t, i16 %fragid) {
+; GFX9-LABEL: atomic_add_i64_2dmsaa:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    v_mov_b32_e32 v5, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT:    s_lshl_b32 s8, s0, 16
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    v_and_or_b32 v2, v2, v5, v3
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    v_and_or_b32 v3, v4, v5, s8
+; GFX9-NEXT:    image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 unorm glc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i64_2dmsaa:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    v_mov_b32_e32 v5, 0xffff
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT:    s_lshl_b32 s8, s0, 16
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    v_and_or_b32 v2, v2, v5, v3
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    v_and_or_b32 v3, v4, v5, s8
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.add.2dmsaa.i64.i16(i64 %data, i16 %s, i16 %t, i16 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_add_i64_2darraymsaa(<8 x i32> inreg %rsrc, i64 %data, i16 %s, i16 %t, i16 %slice, i16 %fragid) {
+; GFX9-LABEL: atomic_add_i64_2darraymsaa:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    v_mov_b32_e32 v6, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT:    v_and_or_b32 v2, v2, v6, v3
+; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 16, v5
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    v_and_or_b32 v3, v4, v6, v3
+; GFX9-NEXT:    image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 unorm glc a16 da
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i64_2darraymsaa:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    v_mov_b32_e32 v6, 0xffff
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    v_and_or_b32 v2, v2, v6, v3
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    v_and_or_b32 v3, v4, v6, v5
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm glc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.add.2darraymsaa.i64.i16(i64 %data, i16 %s, i16 %t, i16 %slice, i16 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_add_i64_1d_slc(<8 x i32> inreg %rsrc, i64 %data, i16 %s) {
+; GFX9-LABEL: atomic_add_i64_1d_slc:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 unorm glc slc a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i64_1d_slc:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc slc a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.add.1d.i64.i16(i64 %data, i16 %s, <8 x i32> %rsrc, i32 0, i32 2)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+declare i32 @llvm.amdgcn.image.atomic.swap.1d.i32.i16(i32, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.add.1d.i32.i16(i32, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.sub.1d.i32.i16(i32, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.smin.1d.i32.i16(i32, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.umin.1d.i32.i16(i32, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.smax.1d.i32.i16(i32, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.umax.1d.i32.i16(i32, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.and.1d.i32.i16(i32, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.or.1d.i32.i16(i32, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.xor.1d.i32.i16(i32, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.inc.1d.i32.i16(i32, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.dec.1d.i32.i16(i32, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.cmpswap.1d.i32.i16(i32, i32, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.add.2d.i32.i16(i32, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.add.3d.i32.i16(i32, i16, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.add.cube.i32.i16(i32, i16, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.add.1darray.i32.i16(i32, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.add.2darray.i32.i16(i32, i16, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.add.2dmsaa.i32.i16(i32, i16, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.add.2darraymsaa.i32.i16(i32, i16, i16, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+
+declare i64 @llvm.amdgcn.image.atomic.swap.1d.i64.i16(i64, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.add.1d.i64.i16(i64, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.sub.1d.i64.i16(i64, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.smin.1d.i64.i16(i64, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.umin.1d.i64.i16(i64, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.smax.1d.i64.i16(i64, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.umax.1d.i64.i16(i64, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.and.1d.i64.i16(i64, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.or.1d.i64.i16(i64, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.xor.1d.i64.i16(i64, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.inc.1d.i64.i16(i64, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.dec.1d.i64.i16(i64, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.cmpswap.1d.i64.i16(i64, i64, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.add.2d.i64.i16(i64, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.add.3d.i64.i16(i64, i16, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.add.cube.i64.i16(i64, i16, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.add.1darray.i64.i16(i64, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.add.2darray.i64.i16(i64, i16, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.add.2dmsaa.i64.i16(i64, i16, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.add.2darraymsaa.i64.i16(i64, i16, i16, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.ll
new file mode 100644
index 0000000000000..48dbdf4248095
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.ll
@@ -0,0 +1,2107 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -o - %s | FileCheck -check-prefix=GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -o - %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -o - %s | FileCheck -check-prefix=GFX10 %s
+
+
+define amdgpu_ps float @atomic_swap_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
+; GFX6-LABEL: atomic_swap_i32_1d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_swap v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_swap_i32_1d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_swap v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_swap_i32_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_swap v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.swap.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_add_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
+; GFX6-LABEL: atomic_add_i32_1d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_add v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_add_i32_1d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_add v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i32_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.add.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_sub_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
+; GFX6-LABEL: atomic_sub_i32_1d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_sub v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_sub_i32_1d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_sub v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_sub_i32_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_sub v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.sub.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_smin_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
+; GFX6-LABEL: atomic_smin_i32_1d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_smin v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_smin_i32_1d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_smin v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_smin_i32_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_smin v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.smin.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_umin_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
+; GFX6-LABEL: atomic_umin_i32_1d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_umin v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_umin_i32_1d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_umin v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_umin_i32_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_umin v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.umin.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_smax_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
+; GFX6-LABEL: atomic_smax_i32_1d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_smax v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_smax_i32_1d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_smax v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_smax_i32_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_smax v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.smax.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_umax_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
+; GFX6-LABEL: atomic_umax_i32_1d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_umax v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_umax_i32_1d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_umax v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_umax_i32_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_umax v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.umax.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_and_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
+; GFX6-LABEL: atomic_and_i32_1d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_and v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_and_i32_1d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_and v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_and_i32_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_and v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.and.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_or_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
+; GFX6-LABEL: atomic_or_i32_1d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_or v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_or_i32_1d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_or v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_or_i32_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_or v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.or.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_xor_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
+; GFX6-LABEL: atomic_xor_i32_1d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_xor v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_xor_i32_1d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_xor v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_xor_i32_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_xor v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.xor.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_inc_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
+; GFX6-LABEL: atomic_inc_i32_1d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_inc v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_inc_i32_1d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_inc v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_inc_i32_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_inc v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.inc.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_dec_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
+; GFX6-LABEL: atomic_dec_i32_1d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_dec v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_dec_i32_1d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_dec v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_dec_i32_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_dec v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.dec.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_cmpswap_i32_1d(<8 x i32> inreg %rsrc, i32 %cmp, i32 %swap, i32 %s) {
+; GFX6-LABEL: atomic_cmpswap_i32_1d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_cmpswap_i32_1d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_cmpswap_i32_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.cmpswap.1d.i32.i32(i32 %cmp, i32 %swap, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_add_i32_2d(<8 x i32> inreg %rsrc, i32 %data, i32 %s, i32 %t) {
+; GFX6-LABEL: atomic_add_i32_2d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_add_i32_2d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i32_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.add.2d.i32.i32(i32 %data, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_add_i32_3d(<8 x i32> inreg %rsrc, i32 %data, i32 %s, i32 %t, i32 %r) {
+; GFX6-LABEL: atomic_add_i32_3d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_add_i32_3d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i32_3d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_3D unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.add.3d.i32.i32(i32 %data, i32 %s, i32 %t, i32 %r, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_add_i32_cube(<8 x i32> inreg %rsrc, i32 %data, i32 %s, i32 %t, i32 %face) {
+; GFX6-LABEL: atomic_add_i32_cube:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 unorm glc da
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_add_i32_cube:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 unorm glc da
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i32_cube:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_CUBE unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.add.cube.i32.i32(i32 %data, i32 %s, i32 %t, i32 %face, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_add_i32_1darray(<8 x i32> inreg %rsrc, i32 %data, i32 %s, i32 %slice) {
+; GFX6-LABEL: atomic_add_i32_1darray:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 unorm glc da
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_add_i32_1darray:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 unorm glc da
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i32_1darray:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D_ARRAY unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.add.1darray.i32.i32(i32 %data, i32 %s, i32 %slice, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_add_i32_2darray(<8 x i32> inreg %rsrc, i32 %data, i32 %s, i32 %t, i32 %slice) {
+; GFX6-LABEL: atomic_add_i32_2darray:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 unorm glc da
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_add_i32_2darray:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 unorm glc da
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i32_2darray:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.add.2darray.i32.i32(i32 %data, i32 %s, i32 %t, i32 %slice, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_add_i32_2dmsaa(<8 x i32> inreg %rsrc, i32 %data, i32 %s, i32 %t, i32 %fragid) {
+; GFX6-LABEL: atomic_add_i32_2dmsaa:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_add_i32_2dmsaa:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i32_2dmsaa:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.add.2dmsaa.i32.i32(i32 %data, i32 %s, i32 %t, i32 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_add_i32_2darraymsaa(<8 x i32> inreg %rsrc, i32 %data, i32 %s, i32 %t, i32 %slice, i32 %fragid) {
+; GFX6-LABEL: atomic_add_i32_2darraymsaa:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_add v0, v[1:4], s[0:7] dmask:0x1 unorm glc da
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_add_i32_2darraymsaa:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_add v0, v[1:4], s[0:7] dmask:0x1 unorm glc da
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i32_2darraymsaa:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v0, v[1:4], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.add.2darraymsaa.i32.i32(i32 %data, i32 %s, i32 %t, i32 %slice, i32 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps float @atomic_add_i32_1d_slc(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
+; GFX6-LABEL: atomic_add_i32_1d_slc:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_add v0, v1, s[0:7] dmask:0x1 unorm glc slc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_add_i32_1d_slc:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_add v0, v1, s[0:7] dmask:0x1 unorm glc slc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i32_1d_slc:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc slc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i32 @llvm.amdgcn.image.atomic.add.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 2)
+  %out = bitcast i32 %v to float
+  ret float %out
+}
+
+define amdgpu_ps <2 x float> @atomic_swap_i64_1d(<8 x i32> inreg %rsrc, i64 %data, i32 %s) {
+; GFX6-LABEL: atomic_swap_i64_1d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_swap v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_swap_i64_1d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_swap v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_swap_i64_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_swap v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.swap.1d.i64.i32(i64 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_add_i64_1d(<8 x i32> inreg %rsrc, i64 %data, i32 %s) {
+; GFX6-LABEL: atomic_add_i64_1d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_add_i64_1d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i64_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.add.1d.i64.i32(i64 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_sub_i64_1d(<8 x i32> inreg %rsrc, i64 %data, i32 %s) {
+; GFX6-LABEL: atomic_sub_i64_1d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_sub v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_sub_i64_1d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_sub v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_sub_i64_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_sub v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.sub.1d.i64.i32(i64 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_smin_i64_1d(<8 x i32> inreg %rsrc, i64 %data, i32 %s) {
+; GFX6-LABEL: atomic_smin_i64_1d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_smin v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_smin_i64_1d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_smin v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_smin_i64_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_smin v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.smin.1d.i64.i32(i64 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_umin_i64_1d(<8 x i32> inreg %rsrc, i64 %data, i32 %s) {
+; GFX6-LABEL: atomic_umin_i64_1d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_umin v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_umin_i64_1d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_umin v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_umin_i64_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_umin v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.umin.1d.i64.i32(i64 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_smax_i64_1d(<8 x i32> inreg %rsrc, i64 %data, i32 %s) {
+; GFX6-LABEL: atomic_smax_i64_1d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_smax v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_smax_i64_1d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_smax v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_smax_i64_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_smax v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.smax.1d.i64.i32(i64 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_umax_i64_1d(<8 x i32> inreg %rsrc, i64 %data, i32 %s) {
+; GFX6-LABEL: atomic_umax_i64_1d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_umax v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_umax_i64_1d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_umax v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_umax_i64_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_umax v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.umax.1d.i64.i32(i64 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_and_i64_1d(<8 x i32> inreg %rsrc, i64 %data, i32 %s) {
+; GFX6-LABEL: atomic_and_i64_1d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_and v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_and_i64_1d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_and v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_and_i64_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_and v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.and.1d.i64.i32(i64 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_or_i64_1d(<8 x i32> inreg %rsrc, i64 %data, i32 %s) {
+; GFX6-LABEL: atomic_or_i64_1d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_or v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_or_i64_1d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_or v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_or_i64_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_or v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.or.1d.i64.i32(i64 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_xor_i64_1d(<8 x i32> inreg %rsrc, i64 %data, i32 %s) {
+; GFX6-LABEL: atomic_xor_i64_1d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_xor v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_xor_i64_1d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_xor v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_xor_i64_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_xor v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.xor.1d.i64.i32(i64 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_inc_i64_1d(<8 x i32> inreg %rsrc, i64 %data, i32 %s) {
+; GFX6-LABEL: atomic_inc_i64_1d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_inc v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_inc_i64_1d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_inc v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_inc_i64_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_inc v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.inc.1d.i64.i32(i64 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_dec_i64_1d(<8 x i32> inreg %rsrc, i64 %data, i32 %s) {
+; GFX6-LABEL: atomic_dec_i64_1d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_dec v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_dec_i64_1d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_dec v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_dec_i64_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_dec v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.dec.1d.i64.i32(i64 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_cmpswap_i64_1d(<8 x i32> inreg %rsrc, i64 %cmp, i64 %swap, i32 %s) {
+; GFX6-LABEL: atomic_cmpswap_i64_1d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_cmpswap_i64_1d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_cmpswap_i64_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.cmpswap.1d.i64.i32(i64 %cmp, i64 %swap, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_add_i64_2d(<8 x i32> inreg %rsrc, i64 %data, i32 %s, i32 %t) {
+; GFX6-LABEL: atomic_add_i64_2d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_add_i64_2d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i64_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.add.2d.i64.i32(i64 %data, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_add_i64_3d(<8 x i32> inreg %rsrc, i64 %data, i32 %s, i32 %t, i32 %r) {
+; GFX6-LABEL: atomic_add_i64_3d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_add_i64_3d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i64_3d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_3D unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.add.3d.i64.i32(i64 %data, i32 %s, i32 %t, i32 %r, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_add_i64_cube(<8 x i32> inreg %rsrc, i64 %data, i32 %s, i32 %t, i32 %face) {
+; GFX6-LABEL: atomic_add_i64_cube:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 unorm glc da
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_add_i64_cube:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 unorm glc da
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i64_cube:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_CUBE unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.add.cube.i64.i32(i64 %data, i32 %s, i32 %t, i32 %face, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_add_i64_1darray(<8 x i32> inreg %rsrc, i64 %data, i32 %s, i32 %slice) {
+; GFX6-LABEL: atomic_add_i64_1darray:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 unorm glc da
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_add_i64_1darray:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 unorm glc da
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i64_1darray:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D_ARRAY unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.add.1darray.i64.i32(i64 %data, i32 %s, i32 %slice, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_add_i64_2darray(<8 x i32> inreg %rsrc, i64 %data, i32 %s, i32 %t, i32 %slice) {
+; GFX6-LABEL: atomic_add_i64_2darray:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 unorm glc da
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_add_i64_2darray:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 unorm glc da
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i64_2darray:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_ARRAY unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.add.2darray.i64.i32(i64 %data, i32 %s, i32 %t, i32 %slice, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_add_i64_2dmsaa(<8 x i32> inreg %rsrc, i64 %data, i32 %s, i32 %t, i32 %fragid) {
+; GFX6-LABEL: atomic_add_i64_2dmsaa:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 unorm glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_add_i64_2dmsaa:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 unorm glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i64_2dmsaa:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.add.2dmsaa.i64.i32(i64 %data, i32 %s, i32 %t, i32 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_add_i64_2darraymsaa(<8 x i32> inreg %rsrc, i64 %data, i32 %s, i32 %t, i32 %slice, i32 %fragid) {
+; GFX6-LABEL: atomic_add_i64_2darraymsaa:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_add v[0:1], v[2:5], s[0:7] dmask:0x3 unorm glc da
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_add_i64_2darraymsaa:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_add v[0:1], v[2:5], s[0:7] dmask:0x3 unorm glc da
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i64_2darraymsaa:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v[0:1], v[2:5], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm glc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.add.2darraymsaa.i64.i32(i64 %data, i32 %s, i32 %t, i32 %slice, i32 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+define amdgpu_ps <2 x float> @atomic_add_i64_1d_slc(<8 x i32> inreg %rsrc, i64 %data, i32 %s) {
+; GFX6-LABEL: atomic_add_i64_1d_slc:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 unorm glc slc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: atomic_add_i64_1d_slc:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 unorm glc slc
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: atomic_add_i64_1d_slc:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc slc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call i64 @llvm.amdgcn.image.atomic.add.1d.i64.i32(i64 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 2)
+  %out = bitcast i64 %v to <2 x float>
+  ret <2 x float> %out
+}
+
+declare i32 @llvm.amdgcn.image.atomic.swap.1d.i32.i32(i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.add.1d.i32.i32(i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.sub.1d.i32.i32(i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.smin.1d.i32.i32(i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.umin.1d.i32.i32(i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.smax.1d.i32.i32(i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.umax.1d.i32.i32(i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.and.1d.i32.i32(i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.or.1d.i32.i32(i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.xor.1d.i32.i32(i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.inc.1d.i32.i32(i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.dec.1d.i32.i32(i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.cmpswap.1d.i32.i32(i32, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.add.2d.i32.i32(i32, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.add.3d.i32.i32(i32, i32, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.add.cube.i32.i32(i32, i32, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.add.1darray.i32.i32(i32, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.add.2darray.i32.i32(i32, i32, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.add.2dmsaa.i32.i32(i32, i32, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i32 @llvm.amdgcn.image.atomic.add.2darraymsaa.i32.i32(i32, i32, i32, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+
+declare i64 @llvm.amdgcn.image.atomic.swap.1d.i64.i32(i64, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.add.1d.i64.i32(i64, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.sub.1d.i64.i32(i64, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.smin.1d.i64.i32(i64, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.umin.1d.i64.i32(i64, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.smax.1d.i64.i32(i64, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.umax.1d.i64.i32(i64, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.and.1d.i64.i32(i64, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.or.1d.i64.i32(i64, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.xor.1d.i64.i32(i64, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.inc.1d.i64.i32(i64, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.dec.1d.i64.i32(i64, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.cmpswap.1d.i64.i32(i64, i64, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.add.2d.i64.i32(i64, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.add.3d.i64.i32(i64, i32, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.add.cube.i64.i32(i64, i32, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.add.1darray.i64.i32(i64, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.add.2darray.i64.i32(i64, i32, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.add.2dmsaa.i64.i32(i64, i32, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare i64 @llvm.amdgcn.image.atomic.add.2darraymsaa.i64.i32(i64, i32, i32, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll
index d798e2687fd42..dd99fbdfedf1a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll
@@ -1,1014 +1,760 @@
-; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX9 %s
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX10NSA %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -o - %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -o - %s | FileCheck -check-prefix=GFX10NSA %s
 
 define amdgpu_ps <4 x float> @gather4_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t) {
-  ; GFX9-LABEL: name: gather4_2d
-  ; GFX9: bb.1.main_body:
-  ; GFX9:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
-  ; GFX9:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX9:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX9:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX9:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX9:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX9:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX9:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX9:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY12]](s32)
-  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
-  ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.2d), 1, [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX9:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX9:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX9:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX9:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX9:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_2d
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY12]](s32)
-  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.2d), 1, [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX9-LABEL: gather4_2d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    s_mov_b32 s8, s10
+; GFX9-NEXT:    s_mov_b32 s9, s11
+; GFX9-NEXT:    s_mov_b64 s[14:15], exec
+; GFX9-NEXT:    s_mov_b32 s10, s12
+; GFX9-NEXT:    s_mov_b32 s11, s13
+; GFX9-NEXT:    s_wqm_b64 exec, exec
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    v_and_or_b32 v0, v0, v2, v1
+; GFX9-NEXT:    s_and_b64 exec, exec, s[14:15]
+; GFX9-NEXT:    image_gather4 v[0:3], v0, s[0:7], s[8:11] dmask:0x1 a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_2d:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10NSA-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX10NSA-NEXT:    v_and_or_b32 v0, v0, 0xffff, v1
+; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT:    image_gather4 v[0:3], v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f16(i32 1, half %s, half %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_cube(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t, half %face) {
-  ; GFX9-LABEL: name: gather4_cube
-  ; GFX9: bb.1.main_body:
-  ; GFX9:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX9:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX9:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX9:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX9:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX9:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX9:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX9:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX9:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY12]](s32)
-  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
-  ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY15]](s32), [[COPY16]](s32)
-  ; GFX9:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY14]](s32)
-  ; GFX9:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-  ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY17]](s32), [[DEF]](s32)
-  ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
-  ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.cube), 1, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX9:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX9:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX9:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX9:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX9:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_cube
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY12]](s32)
-  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY15]](s32), [[COPY16]](s32)
-  ; GFX10NSA:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY14]](s32)
-  ; GFX10NSA:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-  ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY17]](s32), [[DEF]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.cube), 1, [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX9-LABEL: gather4_cube:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    s_mov_b32 s8, s10
+; GFX9-NEXT:    s_mov_b32 s9, s11
+; GFX9-NEXT:    s_mov_b64 s[14:15], exec
+; GFX9-NEXT:    s_mov_b32 s10, s12
+; GFX9-NEXT:    s_mov_b32 s11, s13
+; GFX9-NEXT:    s_wqm_b64 exec, exec
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    s_lshl_b32 s12, s0, 16
+; GFX9-NEXT:    v_and_or_b32 v0, v0, v3, v1
+; GFX9-NEXT:    v_and_or_b32 v1, v2, v3, s12
+; GFX9-NEXT:    s_and_b64 exec, exec, s[14:15]
+; GFX9-NEXT:    image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 a16 da
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_cube:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10NSA-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX10NSA-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX10NSA-NEXT:    s_lshl_b32 s12, s0, 16
+; GFX10NSA-NEXT:    v_and_or_b32 v0, v0, v3, v1
+; GFX10NSA-NEXT:    v_and_or_b32 v1, v2, v3, s12
+; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT:    image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_CUBE a16
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.cube.v4f32.f16(i32 1, half %s, half %t, half %face, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_2darray(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t, half %slice) {
-  ; GFX9-LABEL: name: gather4_2darray
-  ; GFX9: bb.1.main_body:
-  ; GFX9:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX9:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX9:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX9:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX9:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX9:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX9:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX9:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX9:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY12]](s32)
-  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
-  ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY15]](s32), [[COPY16]](s32)
-  ; GFX9:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY14]](s32)
-  ; GFX9:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-  ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY17]](s32), [[DEF]](s32)
-  ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
-  ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.2darray), 1, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX9:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX9:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX9:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX9:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX9:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_2darray
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY12]](s32)
-  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY15]](s32), [[COPY16]](s32)
-  ; GFX10NSA:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY14]](s32)
-  ; GFX10NSA:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-  ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY17]](s32), [[DEF]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.2darray), 1, [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX9-LABEL: gather4_2darray:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    s_mov_b32 s8, s10
+; GFX9-NEXT:    s_mov_b32 s9, s11
+; GFX9-NEXT:    s_mov_b64 s[14:15], exec
+; GFX9-NEXT:    s_mov_b32 s10, s12
+; GFX9-NEXT:    s_mov_b32 s11, s13
+; GFX9-NEXT:    s_wqm_b64 exec, exec
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    s_lshl_b32 s12, s0, 16
+; GFX9-NEXT:    v_and_or_b32 v0, v0, v3, v1
+; GFX9-NEXT:    v_and_or_b32 v1, v2, v3, s12
+; GFX9-NEXT:    s_and_b64 exec, exec, s[14:15]
+; GFX9-NEXT:    image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 a16 da
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_2darray:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10NSA-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX10NSA-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX10NSA-NEXT:    s_lshl_b32 s12, s0, 16
+; GFX10NSA-NEXT:    v_and_or_b32 v0, v0, v3, v1
+; GFX10NSA-NEXT:    v_and_or_b32 v1, v2, v3, s12
+; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT:    image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY a16
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.2darray.v4f32.f16(i32 1, half %s, half %t, half %slice, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %s, half %t) {
-  ; GFX9-LABEL: name: gather4_c_2d
-  ; GFX9: bb.1.main_body:
-  ; GFX9:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX9:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX9:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX9:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX9:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX9:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX9:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX9:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX9:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX9:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY12]](s32)
-  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
-  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY14]](s32)
-  ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY15]](s32), [[COPY16]](s32)
-  ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR_TRUNC]](<2 x s16>)
-  ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.2d), 1, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX9:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX9:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX9:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX9:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX9:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_c_2d
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY12]](s32)
-  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
-  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY14]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY15]](s32), [[COPY16]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.2d), 1, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX9-LABEL: gather4_c_2d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    s_mov_b32 s8, s10
+; GFX9-NEXT:    s_mov_b32 s9, s11
+; GFX9-NEXT:    s_mov_b64 s[14:15], exec
+; GFX9-NEXT:    s_mov_b32 s10, s12
+; GFX9-NEXT:    s_mov_b32 s11, s13
+; GFX9-NEXT:    s_wqm_b64 exec, exec
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT:    v_and_or_b32 v1, v1, v3, v2
+; GFX9-NEXT:    s_and_b64 exec, exec, s[14:15]
+; GFX9-NEXT:    image_gather4_c v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_c_2d:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10NSA-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX10NSA-NEXT:    v_and_or_b32 v1, v1, 0xffff, v2
+; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT:    image_gather4_c v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.c.2d.v4f32.f16(i32 1, float %zcompare, half %s, half %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t, half %clamp) {
-  ; GFX9-LABEL: name: gather4_cl_2d
-  ; GFX9: bb.1.main_body:
-  ; GFX9:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX9:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX9:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX9:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX9:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX9:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX9:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX9:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX9:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY12]](s32)
-  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
-  ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY15]](s32), [[COPY16]](s32)
-  ; GFX9:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY14]](s32)
-  ; GFX9:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-  ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY17]](s32), [[DEF]](s32)
-  ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
-  ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.cl.2d), 1, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX9:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX9:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX9:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX9:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX9:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_cl_2d
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY12]](s32)
-  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY15]](s32), [[COPY16]](s32)
-  ; GFX10NSA:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY14]](s32)
-  ; GFX10NSA:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-  ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY17]](s32), [[DEF]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.cl.2d), 1, [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX9-LABEL: gather4_cl_2d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    s_mov_b32 s8, s10
+; GFX9-NEXT:    s_mov_b32 s9, s11
+; GFX9-NEXT:    s_mov_b64 s[14:15], exec
+; GFX9-NEXT:    s_mov_b32 s10, s12
+; GFX9-NEXT:    s_mov_b32 s11, s13
+; GFX9-NEXT:    s_wqm_b64 exec, exec
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    s_lshl_b32 s12, s0, 16
+; GFX9-NEXT:    v_and_or_b32 v0, v0, v3, v1
+; GFX9-NEXT:    v_and_or_b32 v1, v2, v3, s12
+; GFX9-NEXT:    s_and_b64 exec, exec, s[14:15]
+; GFX9-NEXT:    image_gather4_cl v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_cl_2d:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10NSA-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX10NSA-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX10NSA-NEXT:    s_lshl_b32 s12, s0, 16
+; GFX10NSA-NEXT:    v_and_or_b32 v0, v0, v3, v1
+; GFX10NSA-NEXT:    v_and_or_b32 v1, v2, v3, s12
+; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT:    image_gather4_cl v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.cl.2d.v4f32.f16(i32 1, half %s, half %t, half %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %s, half %t, half %clamp) {
-  ; GFX9-LABEL: name: gather4_c_cl_2d
-  ; GFX9: bb.1.main_body:
-  ; GFX9:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX9:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX9:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX9:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX9:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX9:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX9:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX9:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX9:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX9:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY12]](s32)
-  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
-  ; GFX9:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY14]](s32)
-  ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[COPY17]](s32)
-  ; GFX9:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY15]](s32)
-  ; GFX9:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-  ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY18]](s32), [[DEF]](s32)
-  ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
-  ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.cl.2d), 1, [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX9:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX9:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX9:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX9:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX9:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_c_cl_2d
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY12]](s32)
-  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
-  ; GFX10NSA:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY14]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[COPY17]](s32)
-  ; GFX10NSA:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY15]](s32)
-  ; GFX10NSA:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-  ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY18]](s32), [[DEF]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.cl.2d), 1, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX9-LABEL: gather4_c_cl_2d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    s_mov_b32 s8, s10
+; GFX9-NEXT:    s_mov_b32 s9, s11
+; GFX9-NEXT:    s_mov_b64 s[14:15], exec
+; GFX9-NEXT:    s_mov_b32 s10, s12
+; GFX9-NEXT:    s_mov_b32 s11, s13
+; GFX9-NEXT:    s_wqm_b64 exec, exec
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT:    s_lshl_b32 s12, s0, 16
+; GFX9-NEXT:    v_and_or_b32 v1, v1, v4, v2
+; GFX9-NEXT:    v_and_or_b32 v2, v3, v4, s12
+; GFX9-NEXT:    s_and_b64 exec, exec, s[14:15]
+; GFX9-NEXT:    image_gather4_c_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_c_cl_2d:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10NSA-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX10NSA-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX10NSA-NEXT:    s_lshl_b32 s12, s0, 16
+; GFX10NSA-NEXT:    v_and_or_b32 v1, v1, v4, v2
+; GFX10NSA-NEXT:    v_and_or_b32 v2, v3, v4, s12
+; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT:    image_gather4_c_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.c.cl.2d.v4f32.f16(i32 1, float %zcompare, half %s, half %t, half %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %bias, half %s, half %t) {
-  ; GFX9-LABEL: name: gather4_b_2d
-  ; GFX9: bb.1.main_body:
-  ; GFX9:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX9:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX9:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX9:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX9:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX9:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX9:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX9:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX9:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX9:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY12]](s32)
-  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
-  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY14]](s32)
-  ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY15]](s32), [[COPY16]](s32)
-  ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR_TRUNC]](<2 x s16>)
-  ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.b.2d), 1, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX9:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX9:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX9:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX9:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX9:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_b_2d
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY12]](s32)
-  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
-  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY14]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY15]](s32), [[COPY16]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.b.2d), 1, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX9-LABEL: gather4_b_2d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    s_mov_b32 s8, s10
+; GFX9-NEXT:    s_mov_b32 s9, s11
+; GFX9-NEXT:    s_mov_b64 s[14:15], exec
+; GFX9-NEXT:    s_mov_b32 s10, s12
+; GFX9-NEXT:    s_mov_b32 s11, s13
+; GFX9-NEXT:    s_wqm_b64 exec, exec
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT:    v_and_or_b32 v1, v1, v3, v2
+; GFX9-NEXT:    s_and_b64 exec, exec, s[14:15]
+; GFX9-NEXT:    image_gather4_b v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_b_2d:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10NSA-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX10NSA-NEXT:    v_and_or_b32 v1, v1, 0xffff, v2
+; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT:    image_gather4_b v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.b.2d.v4f32.f32.f16(i32 1, float %bias, half %s, half %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %bias, float %zcompare, half %s, half %t) {
-  ; GFX9-LABEL: name: gather4_c_b_2d
-  ; GFX9: bb.1.main_body:
-  ; GFX9:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX9:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX9:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX9:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX9:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX9:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX9:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX9:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX9:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX9:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY12]](s32)
-  ; GFX9:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY13]](s32)
-  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY14]](s32)
-  ; GFX9:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY15]](s32)
-  ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[COPY17]](s32)
-  ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR_TRUNC]](<2 x s16>)
-  ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.b.2d), 1, [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX9:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX9:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX9:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX9:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX9:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_c_b_2d
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY12]](s32)
-  ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY13]](s32)
-  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY14]](s32)
-  ; GFX10NSA:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY15]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[COPY17]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.b.2d), 1, [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX9-LABEL: gather4_c_b_2d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    s_mov_b32 s8, s10
+; GFX9-NEXT:    s_mov_b32 s9, s11
+; GFX9-NEXT:    s_mov_b64 s[14:15], exec
+; GFX9-NEXT:    s_mov_b32 s10, s12
+; GFX9-NEXT:    s_mov_b32 s11, s13
+; GFX9-NEXT:    s_wqm_b64 exec, exec
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT:    v_and_or_b32 v2, v2, v4, v3
+; GFX9-NEXT:    s_and_b64 exec, exec, s[14:15]
+; GFX9-NEXT:    image_gather4_c_b v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_c_b_2d:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10NSA-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX10NSA-NEXT:    v_and_or_b32 v2, v2, 0xffff, v3
+; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT:    image_gather4_c_b v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.c.b.2d.v4f32.f32.f16(i32 1, float %bias, float %zcompare, half %s, half %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %bias, half %s, half %t, half %clamp) {
-  ; GFX9-LABEL: name: gather4_b_cl_2d
-  ; GFX9: bb.1.main_body:
-  ; GFX9:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX9:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX9:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX9:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX9:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX9:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX9:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX9:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX9:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX9:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY12]](s32)
-  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
-  ; GFX9:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY14]](s32)
-  ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[COPY17]](s32)
-  ; GFX9:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY15]](s32)
-  ; GFX9:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-  ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY18]](s32), [[DEF]](s32)
-  ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
-  ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.b.cl.2d), 1, [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX9:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX9:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX9:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX9:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX9:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_b_cl_2d
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY12]](s32)
-  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
-  ; GFX10NSA:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY14]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[COPY17]](s32)
-  ; GFX10NSA:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY15]](s32)
-  ; GFX10NSA:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-  ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY18]](s32), [[DEF]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.b.cl.2d), 1, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX9-LABEL: gather4_b_cl_2d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    s_mov_b32 s8, s10
+; GFX9-NEXT:    s_mov_b32 s9, s11
+; GFX9-NEXT:    s_mov_b64 s[14:15], exec
+; GFX9-NEXT:    s_mov_b32 s10, s12
+; GFX9-NEXT:    s_mov_b32 s11, s13
+; GFX9-NEXT:    s_wqm_b64 exec, exec
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT:    s_lshl_b32 s12, s0, 16
+; GFX9-NEXT:    v_and_or_b32 v1, v1, v4, v2
+; GFX9-NEXT:    v_and_or_b32 v2, v3, v4, s12
+; GFX9-NEXT:    s_and_b64 exec, exec, s[14:15]
+; GFX9-NEXT:    image_gather4_b_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_b_cl_2d:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10NSA-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX10NSA-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX10NSA-NEXT:    s_lshl_b32 s12, s0, 16
+; GFX10NSA-NEXT:    v_and_or_b32 v1, v1, v4, v2
+; GFX10NSA-NEXT:    v_and_or_b32 v2, v3, v4, s12
+; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT:    image_gather4_b_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.2d.v4f32.f32.f16(i32 1, float %bias, half %s, half %t, half %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %bias, float %zcompare, half %s, half %t, half %clamp) {
-  ; GFX9-LABEL: name: gather4_c_b_cl_2d
-  ; GFX9: bb.1.main_body:
-  ; GFX9:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX9:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX9:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX9:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX9:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX9:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX9:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX9:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX9:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr4
-  ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX9:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY12]](s32)
-  ; GFX9:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY13]](s32)
-  ; GFX9:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY14]](s32)
-  ; GFX9:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY15]](s32)
-  ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY17]](s32), [[COPY18]](s32)
-  ; GFX9:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY16]](s32)
-  ; GFX9:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-  ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY19]](s32), [[DEF]](s32)
-  ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
-  ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.b.cl.2d), 1, [[CONCAT_VECTORS]](<8 x s16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX9:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX9:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX9:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX9:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX9:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_c_b_cl_2d
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr4
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY12]](s32)
-  ; GFX10NSA:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY13]](s32)
-  ; GFX10NSA:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY14]](s32)
-  ; GFX10NSA:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY15]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY17]](s32), [[COPY18]](s32)
-  ; GFX10NSA:   [[COPY19:%[0-9]+]]:_(s32) = COPY [[COPY16]](s32)
-  ; GFX10NSA:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-  ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY19]](s32), [[DEF]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.b.cl.2d), 1, [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX9-LABEL: gather4_c_b_cl_2d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    s_mov_b32 s8, s10
+; GFX9-NEXT:    s_mov_b32 s9, s11
+; GFX9-NEXT:    s_mov_b64 s[14:15], exec
+; GFX9-NEXT:    s_mov_b32 s10, s12
+; GFX9-NEXT:    s_mov_b32 s11, s13
+; GFX9-NEXT:    s_wqm_b64 exec, exec
+; GFX9-NEXT:    v_mov_b32_e32 v5, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT:    s_lshl_b32 s12, s0, 16
+; GFX9-NEXT:    v_and_or_b32 v2, v2, v5, v3
+; GFX9-NEXT:    v_and_or_b32 v3, v4, v5, s12
+; GFX9-NEXT:    s_and_b64 exec, exec, s[14:15]
+; GFX9-NEXT:    image_gather4_c_b_cl v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1 a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_c_b_cl_2d:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10NSA-NEXT:    v_mov_b32_e32 v5, 0xffff
+; GFX10NSA-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX10NSA-NEXT:    s_lshl_b32 s12, s0, 16
+; GFX10NSA-NEXT:    v_and_or_b32 v2, v2, v5, v3
+; GFX10NSA-NEXT:    v_and_or_b32 v3, v4, v5, s12
+; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT:    image_gather4_c_b_cl v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.c.b.cl.2d.v4f32.f32.f16(i32 1, float %bias, float %zcompare, half %s, half %t, half %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t, half %lod) {
-  ; GFX9-LABEL: name: gather4_l_2d
-  ; GFX9: bb.1.main_body:
-  ; GFX9:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX9:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX9:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX9:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX9:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX9:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX9:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX9:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX9:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY12]](s32)
-  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
-  ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY15]](s32), [[COPY16]](s32)
-  ; GFX9:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY14]](s32)
-  ; GFX9:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-  ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY17]](s32), [[DEF]](s32)
-  ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
-  ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.l.2d), 1, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX9:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX9:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX9:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX9:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX9:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_l_2d
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY12]](s32)
-  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY15]](s32), [[COPY16]](s32)
-  ; GFX10NSA:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY14]](s32)
-  ; GFX10NSA:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-  ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY17]](s32), [[DEF]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.l.2d), 1, [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX9-LABEL: gather4_l_2d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s8, s10
+; GFX9-NEXT:    s_mov_b32 s10, s12
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    s_lshl_b32 s12, s0, 16
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    s_mov_b32 s9, s11
+; GFX9-NEXT:    v_and_or_b32 v0, v0, v3, v1
+; GFX9-NEXT:    s_mov_b32 s11, s13
+; GFX9-NEXT:    v_and_or_b32 v1, v2, v3, s12
+; GFX9-NEXT:    image_gather4_l v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_l_2d:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX10NSA-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX10NSA-NEXT:    s_lshl_b32 s12, s0, 16
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    v_and_or_b32 v0, v0, v3, v1
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    v_and_or_b32 v1, v2, v3, s12
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    image_gather4_l v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f16(i32 1, half %s, half %t, half %lod, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %s, half %t, half %lod) {
-  ; GFX9-LABEL: name: gather4_c_l_2d
-  ; GFX9: bb.1.main_body:
-  ; GFX9:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX9:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX9:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX9:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX9:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX9:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX9:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX9:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX9:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX9:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY12]](s32)
-  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
-  ; GFX9:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY14]](s32)
-  ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[COPY17]](s32)
-  ; GFX9:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY15]](s32)
-  ; GFX9:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-  ; GFX9:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY18]](s32), [[DEF]](s32)
-  ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>)
-  ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.l.2d), 1, [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX9:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX9:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX9:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX9:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX9:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_c_l_2d
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY12]](s32)
-  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
-  ; GFX10NSA:   [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY14]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY16]](s32), [[COPY17]](s32)
-  ; GFX10NSA:   [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY15]](s32)
-  ; GFX10NSA:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-  ; GFX10NSA:   [[BUILD_VECTOR_TRUNC1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY18]](s32), [[DEF]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.l.2d), 1, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR_TRUNC]](<2 x s16>), [[BUILD_VECTOR_TRUNC1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX9-LABEL: gather4_c_l_2d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s8, s10
+; GFX9-NEXT:    s_mov_b32 s10, s12
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT:    s_lshl_b32 s12, s0, 16
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    s_mov_b32 s9, s11
+; GFX9-NEXT:    v_and_or_b32 v1, v1, v4, v2
+; GFX9-NEXT:    s_mov_b32 s11, s13
+; GFX9-NEXT:    v_and_or_b32 v2, v3, v4, s12
+; GFX9-NEXT:    image_gather4_c_l v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_c_l_2d:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX10NSA-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX10NSA-NEXT:    s_lshl_b32 s12, s0, 16
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    v_and_or_b32 v1, v1, v4, v2
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    v_and_or_b32 v2, v3, v4, s12
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    image_gather4_c_l v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.c.l.2d.v4f32.f16(i32 1, float %zcompare, half %s, half %t, half %lod, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t) {
-  ; GFX9-LABEL: name: gather4_lz_2d
-  ; GFX9: bb.1.main_body:
-  ; GFX9:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
-  ; GFX9:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX9:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX9:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX9:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX9:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX9:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX9:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX9:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY12]](s32)
-  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
-  ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.lz.2d), 1, [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX9:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX9:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX9:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX9:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX9:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_lz_2d
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY12]](s32)
-  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.lz.2d), 1, [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX9-LABEL: gather4_lz_2d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    s_mov_b32 s8, s10
+; GFX9-NEXT:    s_mov_b32 s9, s11
+; GFX9-NEXT:    s_mov_b32 s10, s12
+; GFX9-NEXT:    s_mov_b32 s11, s13
+; GFX9-NEXT:    v_and_or_b32 v0, v0, v2, v1
+; GFX9-NEXT:    image_gather4_lz v[0:3], v0, s[0:7], s[8:11] dmask:0x1 a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_lz_2d:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    v_and_or_b32 v0, v0, 0xffff, v1
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    image_gather4_lz v[0:3], v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.lz.2d.v4f32.f16(i32 1, half %s, half %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_c_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %s, half %t) {
-  ; GFX9-LABEL: name: gather4_c_lz_2d
-  ; GFX9: bb.1.main_body:
-  ; GFX9:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX9:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX9:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX9:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX9:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX9:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX9:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX9:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX9:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX9:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX9:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX9:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX9:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX9:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX9:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX9:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX9:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX9:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX9:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY12]](s32)
-  ; GFX9:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
-  ; GFX9:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY14]](s32)
-  ; GFX9:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY15]](s32), [[COPY16]](s32)
-  ; GFX9:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR_TRUNC]](<2 x s16>)
-  ; GFX9:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.lz.2d), 1, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX9:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX9:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX9:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX9:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX9:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX9:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_c_lz_2d
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY12]](s32)
-  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY13]](s32)
-  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY14]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC [[COPY15]](s32), [[COPY16]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.lz.2d), 1, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR_TRUNC]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX9-LABEL: gather4_c_lz_2d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    s_mov_b32 s8, s10
+; GFX9-NEXT:    s_mov_b32 s9, s11
+; GFX9-NEXT:    s_mov_b32 s10, s12
+; GFX9-NEXT:    s_mov_b32 s11, s13
+; GFX9-NEXT:    v_and_or_b32 v1, v1, v3, v2
+; GFX9-NEXT:    image_gather4_c_lz v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_c_lz_2d:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    v_and_or_b32 v1, v1, 0xffff, v2
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    image_gather4_c_lz v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.c.lz.2d.v4f32.f16(i32 1, float %zcompare, half %s, half %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.dim.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.dim.ll
index 9e9407f005c6d..bfdbe9a87a0d6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.dim.ll
@@ -1,1046 +1,794 @@
-; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX6 %s
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX10NSA %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -o - %s | FileCheck -check-prefix=GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -o - %s | FileCheck -check-prefix=GFX10NSA %s
 
 define amdgpu_ps <4 x float> @gather4_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t) {
-  ; GFX6-LABEL: name: gather4_2d
-  ; GFX6: bb.1.main_body:
-  ; GFX6:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
-  ; GFX6:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX6:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX6:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX6:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX6:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX6:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX6:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX6:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX6:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX6:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX6:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX6:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX6:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX6:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX6:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX6:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX6:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32)
-  ; GFX6:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.2d), 1, [[BUILD_VECTOR2]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX6:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX6:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX6:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX6:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX6:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX6:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_2d
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.2d), 1, [[BUILD_VECTOR2]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX6-LABEL: gather4_2d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b64 s[14:15], exec
+; GFX6-NEXT:    s_mov_b32 s10, s12
+; GFX6-NEXT:    s_mov_b32 s11, s13
+; GFX6-NEXT:    s_wqm_b64 exec, exec
+; GFX6-NEXT:    s_and_b64 exec, exec, s[14:15]
+; GFX6-NEXT:    image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_2d:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT:    image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f32(i32 1, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_cube(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t, float %face) {
-  ; GFX6-LABEL: name: gather4_cube
-  ; GFX6: bb.1.main_body:
-  ; GFX6:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX6:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX6:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX6:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX6:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX6:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX6:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX6:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX6:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX6:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX6:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX6:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX6:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX6:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX6:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX6:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX6:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX6:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX6:   [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32)
-  ; GFX6:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.cube), 1, [[BUILD_VECTOR2]](<3 x s32>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX6:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX6:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX6:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX6:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX6:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX6:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_cube
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.cube), 1, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX6-LABEL: gather4_cube:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b64 s[14:15], exec
+; GFX6-NEXT:    s_mov_b32 s10, s12
+; GFX6-NEXT:    s_mov_b32 s11, s13
+; GFX6-NEXT:    s_wqm_b64 exec, exec
+; GFX6-NEXT:    s_and_b64 exec, exec, s[14:15]
+; GFX6-NEXT:    image_gather4 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 da
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_cube:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT:    image_gather4 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_CUBE
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.cube.v4f32.f32(i32 1, float %s, float %t, float %face, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_2darray(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t, float %slice) {
-  ; GFX6-LABEL: name: gather4_2darray
-  ; GFX6: bb.1.main_body:
-  ; GFX6:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX6:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX6:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX6:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX6:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX6:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX6:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX6:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX6:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX6:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX6:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX6:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX6:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX6:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX6:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX6:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX6:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX6:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX6:   [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32)
-  ; GFX6:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.2darray), 1, [[BUILD_VECTOR2]](<3 x s32>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX6:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX6:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX6:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX6:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX6:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX6:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_2darray
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.2darray), 1, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX6-LABEL: gather4_2darray:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b64 s[14:15], exec
+; GFX6-NEXT:    s_mov_b32 s10, s12
+; GFX6-NEXT:    s_mov_b32 s11, s13
+; GFX6-NEXT:    s_wqm_b64 exec, exec
+; GFX6-NEXT:    s_and_b64 exec, exec, s[14:15]
+; GFX6-NEXT:    image_gather4 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 da
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_2darray:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT:    image_gather4 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.2darray.v4f32.f32(i32 1, float %s, float %t, float %slice, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %s, float %t) {
-  ; GFX6-LABEL: name: gather4_c_2d
-  ; GFX6: bb.1.main_body:
-  ; GFX6:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX6:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX6:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX6:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX6:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX6:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX6:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX6:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX6:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX6:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX6:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX6:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX6:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX6:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX6:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX6:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX6:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX6:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX6:   [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32)
-  ; GFX6:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.2d), 1, [[BUILD_VECTOR2]](<3 x s32>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX6:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX6:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX6:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX6:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX6:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX6:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_c_2d
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.2d), 1, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX6-LABEL: gather4_c_2d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b64 s[14:15], exec
+; GFX6-NEXT:    s_mov_b32 s10, s12
+; GFX6-NEXT:    s_mov_b32 s11, s13
+; GFX6-NEXT:    s_wqm_b64 exec, exec
+; GFX6-NEXT:    s_and_b64 exec, exec, s[14:15]
+; GFX6-NEXT:    image_gather4_c v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_c_2d:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT:    image_gather4_c v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.c.2d.v4f32.f32(i32 1, float %zcompare, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t, float %clamp) {
-  ; GFX6-LABEL: name: gather4_cl_2d
-  ; GFX6: bb.1.main_body:
-  ; GFX6:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX6:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX6:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX6:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX6:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX6:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX6:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX6:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX6:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX6:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX6:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX6:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX6:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX6:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX6:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX6:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX6:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX6:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX6:   [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32)
-  ; GFX6:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.cl.2d), 1, [[BUILD_VECTOR2]](<3 x s32>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX6:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX6:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX6:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX6:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX6:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX6:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_cl_2d
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.cl.2d), 1, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX6-LABEL: gather4_cl_2d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b64 s[14:15], exec
+; GFX6-NEXT:    s_mov_b32 s10, s12
+; GFX6-NEXT:    s_mov_b32 s11, s13
+; GFX6-NEXT:    s_wqm_b64 exec, exec
+; GFX6-NEXT:    s_and_b64 exec, exec, s[14:15]
+; GFX6-NEXT:    image_gather4_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_cl_2d:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT:    image_gather4_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.cl.2d.v4f32.f32(i32 1, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %s, float %t, float %clamp) {
-  ; GFX6-LABEL: name: gather4_c_cl_2d
-  ; GFX6: bb.1.main_body:
-  ; GFX6:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX6:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX6:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX6:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX6:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX6:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX6:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX6:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX6:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX6:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX6:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX6:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX6:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX6:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX6:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX6:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX6:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX6:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX6:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX6:   [[BUILD_VECTOR2:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX6:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.cl.2d), 1, [[BUILD_VECTOR2]](<4 x s32>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX6:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX6:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX6:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX6:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX6:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX6:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_c_cl_2d
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.cl.2d), 1, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX6-LABEL: gather4_c_cl_2d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b64 s[14:15], exec
+; GFX6-NEXT:    s_mov_b32 s10, s12
+; GFX6-NEXT:    s_mov_b32 s11, s13
+; GFX6-NEXT:    s_wqm_b64 exec, exec
+; GFX6-NEXT:    s_and_b64 exec, exec, s[14:15]
+; GFX6-NEXT:    image_gather4_c_cl v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_c_cl_2d:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT:    image_gather4_c_cl v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.c.cl.2d.v4f32.f32(i32 1, float %zcompare, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %bias, float %s, float %t) {
-  ; GFX6-LABEL: name: gather4_b_2d
-  ; GFX6: bb.1.main_body:
-  ; GFX6:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX6:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX6:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX6:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX6:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX6:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX6:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX6:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX6:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX6:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX6:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX6:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX6:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX6:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX6:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX6:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX6:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX6:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX6:   [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32)
-  ; GFX6:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.b.2d), 1, [[BUILD_VECTOR2]](<3 x s32>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX6:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX6:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX6:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX6:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX6:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX6:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_b_2d
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.b.2d), 1, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX6-LABEL: gather4_b_2d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b64 s[14:15], exec
+; GFX6-NEXT:    s_mov_b32 s10, s12
+; GFX6-NEXT:    s_mov_b32 s11, s13
+; GFX6-NEXT:    s_wqm_b64 exec, exec
+; GFX6-NEXT:    s_and_b64 exec, exec, s[14:15]
+; GFX6-NEXT:    image_gather4_b v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_b_2d:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT:    image_gather4_b v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.b.2d.v4f32.f32.f32(i32 1, float %bias, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %bias, float %zcompare, float %s, float %t) {
-  ; GFX6-LABEL: name: gather4_c_b_2d
-  ; GFX6: bb.1.main_body:
-  ; GFX6:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX6:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX6:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX6:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX6:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX6:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX6:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX6:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX6:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX6:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX6:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX6:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX6:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX6:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX6:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX6:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX6:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX6:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX6:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX6:   [[BUILD_VECTOR2:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX6:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.b.2d), 1, [[BUILD_VECTOR2]](<4 x s32>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX6:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX6:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX6:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX6:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX6:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX6:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_c_b_2d
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.b.2d), 1, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX6-LABEL: gather4_c_b_2d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b64 s[14:15], exec
+; GFX6-NEXT:    s_mov_b32 s10, s12
+; GFX6-NEXT:    s_mov_b32 s11, s13
+; GFX6-NEXT:    s_wqm_b64 exec, exec
+; GFX6-NEXT:    s_and_b64 exec, exec, s[14:15]
+; GFX6-NEXT:    image_gather4_c_b v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_c_b_2d:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT:    image_gather4_c_b v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.c.b.2d.v4f32.f32.f32(i32 1, float %bias, float %zcompare, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %bias, float %s, float %t, float %clamp) {
-  ; GFX6-LABEL: name: gather4_b_cl_2d
-  ; GFX6: bb.1.main_body:
-  ; GFX6:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX6:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX6:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX6:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX6:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX6:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX6:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX6:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX6:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX6:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX6:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX6:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX6:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX6:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX6:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX6:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX6:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX6:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX6:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX6:   [[BUILD_VECTOR2:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX6:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.b.cl.2d), 1, [[BUILD_VECTOR2]](<4 x s32>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX6:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX6:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX6:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX6:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX6:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX6:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_b_cl_2d
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.b.cl.2d), 1, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX6-LABEL: gather4_b_cl_2d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b64 s[14:15], exec
+; GFX6-NEXT:    s_mov_b32 s10, s12
+; GFX6-NEXT:    s_mov_b32 s11, s13
+; GFX6-NEXT:    s_wqm_b64 exec, exec
+; GFX6-NEXT:    s_and_b64 exec, exec, s[14:15]
+; GFX6-NEXT:    image_gather4_b_cl v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_b_cl_2d:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT:    image_gather4_b_cl v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.2d.v4f32.f32.f32(i32 1, float %bias, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %bias, float %zcompare, float %s, float %t, float %clamp) {
-  ; GFX6-LABEL: name: gather4_c_b_cl_2d
-  ; GFX6: bb.1.main_body:
-  ; GFX6:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX6:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX6:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX6:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX6:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX6:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX6:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX6:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX6:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX6:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX6:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX6:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX6:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX6:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX6:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX6:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX6:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX6:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr4
-  ; GFX6:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX6:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX6:   [[BUILD_VECTOR2:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32)
-  ; GFX6:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.b.cl.2d), 1, [[BUILD_VECTOR2]](<5 x s32>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX6:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX6:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX6:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX6:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX6:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX6:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_c_b_cl_2d
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX10NSA:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr4
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.b.cl.2d), 1, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX6-LABEL: gather4_c_b_cl_2d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b64 s[14:15], exec
+; GFX6-NEXT:    s_mov_b32 s10, s12
+; GFX6-NEXT:    s_mov_b32 s11, s13
+; GFX6-NEXT:    s_wqm_b64 exec, exec
+; GFX6-NEXT:    s_and_b64 exec, exec, s[14:15]
+; GFX6-NEXT:    image_gather4_c_b_cl v[0:3], v[0:7], s[0:7], s[8:11] dmask:0x1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_c_b_cl_2d:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT:    image_gather4_c_b_cl v[0:3], v[0:7], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.c.b.cl.2d.v4f32.f32.f32(i32 1, float %bias, float %zcompare, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t, float %lod) {
-  ; GFX6-LABEL: name: gather4_l_2d
-  ; GFX6: bb.1.main_body:
-  ; GFX6:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX6:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX6:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX6:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX6:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX6:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX6:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX6:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX6:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX6:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX6:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX6:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX6:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX6:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX6:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX6:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX6:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX6:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX6:   [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32)
-  ; GFX6:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.l.2d), 1, [[BUILD_VECTOR2]](<3 x s32>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX6:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX6:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX6:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX6:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX6:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX6:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_l_2d
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.l.2d), 1, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX6-LABEL: gather4_l_2d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b32 s10, s12
+; GFX6-NEXT:    s_mov_b32 s11, s13
+; GFX6-NEXT:    image_gather4_l v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_l_2d:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    image_gather4_l v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f32(i32 1, float %s, float %t, float %lod, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %s, float %t, float %lod) {
-  ; GFX6-LABEL: name: gather4_c_l_2d
-  ; GFX6: bb.1.main_body:
-  ; GFX6:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX6:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX6:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX6:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX6:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX6:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX6:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX6:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX6:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX6:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX6:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX6:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX6:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX6:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX6:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX6:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX6:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX6:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX6:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX6:   [[BUILD_VECTOR2:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX6:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.l.2d), 1, [[BUILD_VECTOR2]](<4 x s32>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX6:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX6:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX6:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX6:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX6:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX6:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_c_l_2d
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10NSA:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.l.2d), 1, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX6-LABEL: gather4_c_l_2d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b32 s10, s12
+; GFX6-NEXT:    s_mov_b32 s11, s13
+; GFX6-NEXT:    image_gather4_c_l v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_c_l_2d:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    image_gather4_c_l v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.c.l.2d.v4f32.f32(i32 1, float %zcompare, float %s, float %t, float %lod, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t) {
-  ; GFX6-LABEL: name: gather4_lz_2d
-  ; GFX6: bb.1.main_body:
-  ; GFX6:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
-  ; GFX6:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX6:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX6:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX6:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX6:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX6:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX6:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX6:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX6:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX6:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX6:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX6:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX6:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX6:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX6:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX6:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX6:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32)
-  ; GFX6:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.lz.2d), 1, [[BUILD_VECTOR2]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX6:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX6:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX6:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX6:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX6:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX6:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_lz_2d
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.lz.2d), 1, [[BUILD_VECTOR2]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX6-LABEL: gather4_lz_2d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b32 s10, s12
+; GFX6-NEXT:    s_mov_b32 s11, s13
+; GFX6-NEXT:    image_gather4_lz v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_lz_2d:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    image_gather4_lz v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.lz.2d.v4f32.f32(i32 1, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_c_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %s, float %t) {
-  ; GFX6-LABEL: name: gather4_c_lz_2d
-  ; GFX6: bb.1.main_body:
-  ; GFX6:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX6:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX6:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX6:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX6:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX6:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX6:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX6:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX6:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX6:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX6:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX6:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX6:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX6:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX6:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX6:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX6:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX6:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX6:   [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32)
-  ; GFX6:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.lz.2d), 1, [[BUILD_VECTOR2]](<3 x s32>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX6:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX6:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX6:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX6:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX6:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX6:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_c_lz_2d
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.lz.2d), 1, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX6-LABEL: gather4_c_lz_2d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b32 s10, s12
+; GFX6-NEXT:    s_mov_b32 s11, s13
+; GFX6-NEXT:    image_gather4_c_lz v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_c_lz_2d:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    image_gather4_c_lz v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.c.lz.2d.v4f32.f32(i32 1, float %zcompare, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_2d_dmask_2(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t) {
-  ; GFX6-LABEL: name: gather4_2d_dmask_2
-  ; GFX6: bb.1.main_body:
-  ; GFX6:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
-  ; GFX6:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX6:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX6:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX6:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX6:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX6:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX6:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX6:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX6:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX6:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX6:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX6:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX6:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX6:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX6:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX6:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX6:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32)
-  ; GFX6:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.2d), 2, [[BUILD_VECTOR2]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX6:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX6:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX6:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX6:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX6:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX6:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_2d_dmask_2
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.2d), 2, [[BUILD_VECTOR2]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX6-LABEL: gather4_2d_dmask_2:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b64 s[14:15], exec
+; GFX6-NEXT:    s_mov_b32 s10, s12
+; GFX6-NEXT:    s_mov_b32 s11, s13
+; GFX6-NEXT:    s_wqm_b64 exec, exec
+; GFX6-NEXT:    s_and_b64 exec, exec, s[14:15]
+; GFX6-NEXT:    image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x2
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_2d_dmask_2:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT:    image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x2 dim:SQ_RSRC_IMG_2D
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f32(i32 2, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_2d_dmask_4(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t) {
-  ; GFX6-LABEL: name: gather4_2d_dmask_4
-  ; GFX6: bb.1.main_body:
-  ; GFX6:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
-  ; GFX6:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX6:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX6:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX6:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX6:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX6:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX6:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX6:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX6:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX6:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX6:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX6:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX6:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX6:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX6:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX6:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX6:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32)
-  ; GFX6:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.2d), 4, [[BUILD_VECTOR2]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX6:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX6:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX6:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX6:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX6:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX6:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_2d_dmask_4
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.2d), 4, [[BUILD_VECTOR2]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX6-LABEL: gather4_2d_dmask_4:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b64 s[14:15], exec
+; GFX6-NEXT:    s_mov_b32 s10, s12
+; GFX6-NEXT:    s_mov_b32 s11, s13
+; GFX6-NEXT:    s_wqm_b64 exec, exec
+; GFX6-NEXT:    s_and_b64 exec, exec, s[14:15]
+; GFX6-NEXT:    image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x4
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_2d_dmask_4:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT:    image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f32(i32 4, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_2d_dmask_8(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t) {
-  ; GFX6-LABEL: name: gather4_2d_dmask_8
-  ; GFX6: bb.1.main_body:
-  ; GFX6:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
-  ; GFX6:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX6:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX6:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX6:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX6:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX6:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX6:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX6:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX6:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX6:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX6:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX6:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX6:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX6:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX6:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX6:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX6:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32)
-  ; GFX6:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.2d), 8, [[BUILD_VECTOR2]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX6:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX6:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX6:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX6:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX6:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX6:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10NSA-LABEL: name: gather4_2d_dmask_8
-  ; GFX10NSA: bb.1.main_body:
-  ; GFX10NSA:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
-  ; GFX10NSA:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10NSA:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10NSA:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10NSA:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10NSA:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10NSA:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10NSA:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10NSA:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10NSA:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10NSA:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10NSA:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10NSA:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10NSA:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10NSA:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10NSA:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10NSA:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32)
-  ; GFX10NSA:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.2d), 8, [[BUILD_VECTOR2]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10NSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10NSA:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10NSA:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10NSA:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10NSA:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10NSA:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX6-LABEL: gather4_2d_dmask_8:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b64 s[14:15], exec
+; GFX6-NEXT:    s_mov_b32 s10, s12
+; GFX6-NEXT:    s_mov_b32 s11, s13
+; GFX6-NEXT:    s_wqm_b64 exec, exec
+; GFX6-NEXT:    s_and_b64 exec, exec, s[14:15]
+; GFX6-NEXT:    image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x8
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10NSA-LABEL: gather4_2d_dmask_8:
+; GFX10NSA:       ; %bb.0: ; %main_body
+; GFX10NSA-NEXT:    s_mov_b32 s0, s2
+; GFX10NSA-NEXT:    s_mov_b32 s1, s3
+; GFX10NSA-NEXT:    s_mov_b32 s2, s4
+; GFX10NSA-NEXT:    s_mov_b32 s3, s5
+; GFX10NSA-NEXT:    s_mov_b32 s4, s6
+; GFX10NSA-NEXT:    s_mov_b32 s5, s7
+; GFX10NSA-NEXT:    s_mov_b32 s6, s8
+; GFX10NSA-NEXT:    s_mov_b32 s7, s9
+; GFX10NSA-NEXT:    s_mov_b32 s8, s10
+; GFX10NSA-NEXT:    s_mov_b32 s9, s11
+; GFX10NSA-NEXT:    s_mov_b32 s14, exec_lo
+; GFX10NSA-NEXT:    s_mov_b32 s10, s12
+; GFX10NSA-NEXT:    s_mov_b32 s11, s13
+; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
+; GFX10NSA-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s14
+; GFX10NSA-NEXT:    image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x8 dim:SQ_RSRC_IMG_2D
+; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
+; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f32(i32 8, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.o.dim.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.o.dim.ll
index 5338f1deb8e2a..de5cd806fa295 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.o.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.o.dim.ll
@@ -1,766 +1,548 @@
-; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX6 %s
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX10 %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -o - %s | FileCheck -check-prefix=GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -o - %s | FileCheck -check-prefix=GFX10 %s
 
 define amdgpu_ps <4 x float> @gather4_o_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %s, float %t) {
-  ; GFX6-LABEL: name: gather4_o_2d
-  ; GFX6: bb.1.main_body:
-  ; GFX6:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX6:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX6:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX6:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX6:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX6:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX6:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX6:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX6:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX6:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX6:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX6:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX6:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX6:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX6:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX6:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX6:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX6:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX6:   [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32)
-  ; GFX6:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.o.2d), 1, [[BUILD_VECTOR2]](<3 x s32>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX6:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX6:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX6:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX6:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX6:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX6:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10-LABEL: name: gather4_o_2d
-  ; GFX10: bb.1.main_body:
-  ; GFX10:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX10:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.o.2d), 1, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX6-LABEL: gather4_o_2d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b64 s[14:15], exec
+; GFX6-NEXT:    s_mov_b32 s10, s12
+; GFX6-NEXT:    s_mov_b32 s11, s13
+; GFX6-NEXT:    s_wqm_b64 exec, exec
+; GFX6-NEXT:    s_and_b64 exec, exec, s[14:15]
+; GFX6-NEXT:    image_gather4_o v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: gather4_o_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    s_mov_b32 s8, s10
+; GFX10-NEXT:    s_mov_b32 s9, s11
+; GFX10-NEXT:    s_mov_b32 s14, exec_lo
+; GFX10-NEXT:    s_mov_b32 s10, s12
+; GFX10-NEXT:    s_mov_b32 s11, s13
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s14
+; GFX10-NEXT:    image_gather4_o v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.o.2d.v4f32.f32(i32 1, i32 %offset, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_c_o_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, float %s, float %t) {
-  ; GFX6-LABEL: name: gather4_c_o_2d
-  ; GFX6: bb.1.main_body:
-  ; GFX6:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX6:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX6:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX6:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX6:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX6:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX6:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX6:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX6:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX6:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX6:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX6:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX6:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX6:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX6:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX6:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX6:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX6:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX6:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX6:   [[BUILD_VECTOR2:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX6:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.o.2d), 1, [[BUILD_VECTOR2]](<4 x s32>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX6:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX6:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX6:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX6:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX6:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX6:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10-LABEL: name: gather4_c_o_2d
-  ; GFX10: bb.1.main_body:
-  ; GFX10:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX10:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX10:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.o.2d), 1, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX6-LABEL: gather4_c_o_2d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b64 s[14:15], exec
+; GFX6-NEXT:    s_mov_b32 s10, s12
+; GFX6-NEXT:    s_mov_b32 s11, s13
+; GFX6-NEXT:    s_wqm_b64 exec, exec
+; GFX6-NEXT:    s_and_b64 exec, exec, s[14:15]
+; GFX6-NEXT:    image_gather4_c_o v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: gather4_c_o_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    s_mov_b32 s8, s10
+; GFX10-NEXT:    s_mov_b32 s9, s11
+; GFX10-NEXT:    s_mov_b32 s14, exec_lo
+; GFX10-NEXT:    s_mov_b32 s10, s12
+; GFX10-NEXT:    s_mov_b32 s11, s13
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s14
+; GFX10-NEXT:    image_gather4_c_o v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.c.o.2d.v4f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_cl_o_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %s, float %t, float %clamp) {
-  ; GFX6-LABEL: name: gather4_cl_o_2d
-  ; GFX6: bb.1.main_body:
-  ; GFX6:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX6:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX6:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX6:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX6:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX6:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX6:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX6:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX6:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX6:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX6:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX6:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX6:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX6:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX6:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX6:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX6:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX6:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX6:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX6:   [[BUILD_VECTOR2:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX6:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.cl.o.2d), 1, [[BUILD_VECTOR2]](<4 x s32>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX6:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX6:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX6:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX6:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX6:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX6:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10-LABEL: name: gather4_cl_o_2d
-  ; GFX10: bb.1.main_body:
-  ; GFX10:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX10:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX10:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.cl.o.2d), 1, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX6-LABEL: gather4_cl_o_2d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b64 s[14:15], exec
+; GFX6-NEXT:    s_mov_b32 s10, s12
+; GFX6-NEXT:    s_mov_b32 s11, s13
+; GFX6-NEXT:    s_wqm_b64 exec, exec
+; GFX6-NEXT:    s_and_b64 exec, exec, s[14:15]
+; GFX6-NEXT:    image_gather4_cl_o v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: gather4_cl_o_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    s_mov_b32 s8, s10
+; GFX10-NEXT:    s_mov_b32 s9, s11
+; GFX10-NEXT:    s_mov_b32 s14, exec_lo
+; GFX10-NEXT:    s_mov_b32 s10, s12
+; GFX10-NEXT:    s_mov_b32 s11, s13
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s14
+; GFX10-NEXT:    image_gather4_cl_o v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.cl.o.2d.v4f32.f32(i32 1, i32 %offset, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_c_cl_o_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, float %s, float %t, float %clamp) {
-  ; GFX6-LABEL: name: gather4_c_cl_o_2d
-  ; GFX6: bb.1.main_body:
-  ; GFX6:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX6:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX6:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX6:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX6:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX6:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX6:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX6:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX6:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX6:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX6:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX6:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX6:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX6:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX6:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX6:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX6:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX6:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr4
-  ; GFX6:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX6:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX6:   [[BUILD_VECTOR2:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32)
-  ; GFX6:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.cl.o.2d), 1, [[BUILD_VECTOR2]](<5 x s32>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX6:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX6:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX6:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX6:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX6:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX6:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10-LABEL: name: gather4_c_cl_o_2d
-  ; GFX10: bb.1.main_body:
-  ; GFX10:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX10:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX10:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr4
-  ; GFX10:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.cl.o.2d), 1, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX6-LABEL: gather4_c_cl_o_2d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b64 s[14:15], exec
+; GFX6-NEXT:    s_mov_b32 s10, s12
+; GFX6-NEXT:    s_mov_b32 s11, s13
+; GFX6-NEXT:    s_wqm_b64 exec, exec
+; GFX6-NEXT:    s_and_b64 exec, exec, s[14:15]
+; GFX6-NEXT:    image_gather4_c_cl_o v[0:3], v[0:7], s[0:7], s[8:11] dmask:0x1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: gather4_c_cl_o_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    s_mov_b32 s8, s10
+; GFX10-NEXT:    s_mov_b32 s9, s11
+; GFX10-NEXT:    s_mov_b32 s14, exec_lo
+; GFX10-NEXT:    s_mov_b32 s10, s12
+; GFX10-NEXT:    s_mov_b32 s11, s13
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s14
+; GFX10-NEXT:    image_gather4_c_cl_o v[0:3], v[0:7], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.c.cl.o.2d.v4f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_b_o_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %bias, float %s, float %t) {
-  ; GFX6-LABEL: name: gather4_b_o_2d
-  ; GFX6: bb.1.main_body:
-  ; GFX6:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX6:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX6:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX6:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX6:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX6:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX6:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX6:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX6:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX6:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX6:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX6:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX6:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX6:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX6:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX6:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX6:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX6:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX6:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX6:   [[BUILD_VECTOR2:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX6:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.b.o.2d), 1, [[BUILD_VECTOR2]](<4 x s32>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX6:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX6:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX6:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX6:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX6:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX6:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10-LABEL: name: gather4_b_o_2d
-  ; GFX10: bb.1.main_body:
-  ; GFX10:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX10:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX10:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.b.o.2d), 1, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX6-LABEL: gather4_b_o_2d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b64 s[14:15], exec
+; GFX6-NEXT:    s_mov_b32 s10, s12
+; GFX6-NEXT:    s_mov_b32 s11, s13
+; GFX6-NEXT:    s_wqm_b64 exec, exec
+; GFX6-NEXT:    s_and_b64 exec, exec, s[14:15]
+; GFX6-NEXT:    image_gather4_b_o v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: gather4_b_o_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    s_mov_b32 s8, s10
+; GFX10-NEXT:    s_mov_b32 s9, s11
+; GFX10-NEXT:    s_mov_b32 s14, exec_lo
+; GFX10-NEXT:    s_mov_b32 s10, s12
+; GFX10-NEXT:    s_mov_b32 s11, s13
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s14
+; GFX10-NEXT:    image_gather4_b_o v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.b.o.2d.v4f32.f32.f32(i32 1, i32 %offset, float %bias, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_c_b_o_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %bias, float %zcompare, float %s, float %t) {
-  ; GFX6-LABEL: name: gather4_c_b_o_2d
-  ; GFX6: bb.1.main_body:
-  ; GFX6:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX6:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX6:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX6:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX6:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX6:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX6:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX6:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX6:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX6:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX6:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX6:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX6:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX6:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX6:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX6:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX6:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX6:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr4
-  ; GFX6:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX6:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX6:   [[BUILD_VECTOR2:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32)
-  ; GFX6:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.b.o.2d), 1, [[BUILD_VECTOR2]](<5 x s32>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX6:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX6:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX6:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX6:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX6:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX6:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10-LABEL: name: gather4_c_b_o_2d
-  ; GFX10: bb.1.main_body:
-  ; GFX10:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX10:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX10:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr4
-  ; GFX10:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.b.o.2d), 1, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX6-LABEL: gather4_c_b_o_2d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b64 s[14:15], exec
+; GFX6-NEXT:    s_mov_b32 s10, s12
+; GFX6-NEXT:    s_mov_b32 s11, s13
+; GFX6-NEXT:    s_wqm_b64 exec, exec
+; GFX6-NEXT:    s_and_b64 exec, exec, s[14:15]
+; GFX6-NEXT:    image_gather4_c_b_o v[0:3], v[0:7], s[0:7], s[8:11] dmask:0x1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: gather4_c_b_o_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    s_mov_b32 s8, s10
+; GFX10-NEXT:    s_mov_b32 s9, s11
+; GFX10-NEXT:    s_mov_b32 s14, exec_lo
+; GFX10-NEXT:    s_mov_b32 s10, s12
+; GFX10-NEXT:    s_mov_b32 s11, s13
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s14
+; GFX10-NEXT:    image_gather4_c_b_o v[0:3], v[0:7], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.c.b.o.2d.v4f32.f32.f32(i32 1, i32 %offset, float %bias, float %zcompare, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_b_cl_o_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %bias, float %s, float %t, float %clamp) {
-  ; GFX6-LABEL: name: gather4_b_cl_o_2d
-  ; GFX6: bb.1.main_body:
-  ; GFX6:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX6:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX6:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX6:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX6:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX6:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX6:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX6:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX6:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX6:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX6:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX6:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX6:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX6:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX6:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX6:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX6:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX6:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr4
-  ; GFX6:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX6:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX6:   [[BUILD_VECTOR2:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32)
-  ; GFX6:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.b.cl.o.2d), 1, [[BUILD_VECTOR2]](<5 x s32>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX6:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX6:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX6:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX6:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX6:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX6:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10-LABEL: name: gather4_b_cl_o_2d
-  ; GFX10: bb.1.main_body:
-  ; GFX10:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX10:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX10:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr4
-  ; GFX10:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.b.cl.o.2d), 1, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX6-LABEL: gather4_b_cl_o_2d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b32 s10, s12
+; GFX6-NEXT:    s_mov_b32 s11, s13
+; GFX6-NEXT:    image_gather4_b_cl_o v[0:3], v[0:7], s[0:7], s[8:11] dmask:0x1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: gather4_b_cl_o_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    s_mov_b32 s8, s10
+; GFX10-NEXT:    s_mov_b32 s9, s11
+; GFX10-NEXT:    s_mov_b32 s10, s12
+; GFX10-NEXT:    s_mov_b32 s11, s13
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_gather4_b_cl_o v[0:3], v[0:7], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.o.2d.v4f32.f32.f32(i32 1, i32 %offset, float %bias, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_c_b_cl_o_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %bias, float %zcompare, float %s, float %t, float %clamp) {
-  ; GFX6-LABEL: name: gather4_c_b_cl_o_2d
-  ; GFX6: bb.1.main_body:
-  ; GFX6:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
-  ; GFX6:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX6:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX6:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX6:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX6:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX6:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX6:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX6:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX6:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX6:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX6:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX6:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX6:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX6:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX6:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX6:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX6:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr4
-  ; GFX6:   [[COPY17:%[0-9]+]]:_(s32) = COPY $vgpr5
-  ; GFX6:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX6:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX6:   [[BUILD_VECTOR2:%[0-9]+]]:_(<6 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32), [[COPY17]](s32)
-  ; GFX6:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.b.cl.o.2d), 1, [[BUILD_VECTOR2]](<6 x s32>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX6:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX6:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX6:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX6:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX6:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX6:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10-LABEL: name: gather4_c_b_cl_o_2d
-  ; GFX10: bb.1.main_body:
-  ; GFX10:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
-  ; GFX10:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX10:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr4
-  ; GFX10:   [[COPY17:%[0-9]+]]:_(s32) = COPY $vgpr5
-  ; GFX10:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.b.cl.o.2d), 1, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32), [[COPY17]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX6-LABEL: gather4_c_b_cl_o_2d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b64 s[14:15], exec
+; GFX6-NEXT:    s_mov_b32 s10, s12
+; GFX6-NEXT:    s_mov_b32 s11, s13
+; GFX6-NEXT:    s_wqm_b64 exec, exec
+; GFX6-NEXT:    s_and_b64 exec, exec, s[14:15]
+; GFX6-NEXT:    image_gather4_c_b_cl_o v[0:3], v[0:7], s[0:7], s[8:11] dmask:0x1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: gather4_c_b_cl_o_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    s_mov_b32 s8, s10
+; GFX10-NEXT:    s_mov_b32 s9, s11
+; GFX10-NEXT:    s_mov_b32 s14, exec_lo
+; GFX10-NEXT:    s_mov_b32 s10, s12
+; GFX10-NEXT:    s_mov_b32 s11, s13
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s14
+; GFX10-NEXT:    image_gather4_c_b_cl_o v[0:3], v[0:7], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.c.b.cl.o.2d.v4f32.f32.f32(i32 1, i32 %offset, float %bias, float %zcompare, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_l_o_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %s, float %t, float %lod) {
-  ; GFX6-LABEL: name: gather4_l_o_2d
-  ; GFX6: bb.1.main_body:
-  ; GFX6:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX6:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX6:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX6:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX6:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX6:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX6:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX6:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX6:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX6:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX6:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX6:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX6:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX6:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX6:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX6:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX6:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX6:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX6:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX6:   [[BUILD_VECTOR2:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX6:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.l.o.2d), 1, [[BUILD_VECTOR2]](<4 x s32>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX6:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX6:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX6:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX6:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX6:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX6:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10-LABEL: name: gather4_l_o_2d
-  ; GFX10: bb.1.main_body:
-  ; GFX10:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX10:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX10:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.l.o.2d), 1, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX6-LABEL: gather4_l_o_2d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b32 s10, s12
+; GFX6-NEXT:    s_mov_b32 s11, s13
+; GFX6-NEXT:    image_gather4_l_o v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: gather4_l_o_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    s_mov_b32 s8, s10
+; GFX10-NEXT:    s_mov_b32 s9, s11
+; GFX10-NEXT:    s_mov_b32 s10, s12
+; GFX10-NEXT:    s_mov_b32 s11, s13
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_gather4_l_o v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.l.o.2d.v4f32.f32(i32 1, i32 %offset, float %s, float %t, float %lod, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_c_l_o_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, float %s, float %t, float %lod) {
-  ; GFX6-LABEL: name: gather4_c_l_o_2d
-  ; GFX6: bb.1.main_body:
-  ; GFX6:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX6:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX6:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX6:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX6:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX6:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX6:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX6:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX6:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX6:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX6:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX6:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX6:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX6:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX6:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX6:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX6:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX6:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr4
-  ; GFX6:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX6:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX6:   [[BUILD_VECTOR2:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32)
-  ; GFX6:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.l.o.2d), 1, [[BUILD_VECTOR2]](<5 x s32>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX6:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX6:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX6:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX6:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX6:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX6:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10-LABEL: name: gather4_c_l_o_2d
-  ; GFX10: bb.1.main_body:
-  ; GFX10:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX10:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX10:   [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr4
-  ; GFX10:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.l.o.2d), 1, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX6-LABEL: gather4_c_l_o_2d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b32 s10, s12
+; GFX6-NEXT:    s_mov_b32 s11, s13
+; GFX6-NEXT:    image_gather4_c_l_o v[0:3], v[0:7], s[0:7], s[8:11] dmask:0x1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: gather4_c_l_o_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    s_mov_b32 s8, s10
+; GFX10-NEXT:    s_mov_b32 s9, s11
+; GFX10-NEXT:    s_mov_b32 s10, s12
+; GFX10-NEXT:    s_mov_b32 s11, s13
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_gather4_c_l_o v[0:3], v[0:7], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2d.v4f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %t, float %lod, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_lz_o_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %s, float %t) {
-  ; GFX6-LABEL: name: gather4_lz_o_2d
-  ; GFX6: bb.1.main_body:
-  ; GFX6:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX6:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX6:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX6:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX6:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX6:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX6:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX6:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX6:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX6:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX6:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX6:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX6:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX6:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX6:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX6:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX6:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX6:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX6:   [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32)
-  ; GFX6:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.lz.o.2d), 1, [[BUILD_VECTOR2]](<3 x s32>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX6:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX6:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX6:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX6:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX6:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX6:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10-LABEL: name: gather4_lz_o_2d
-  ; GFX10: bb.1.main_body:
-  ; GFX10:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
-  ; GFX10:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.lz.o.2d), 1, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX6-LABEL: gather4_lz_o_2d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b32 s10, s12
+; GFX6-NEXT:    s_mov_b32 s11, s13
+; GFX6-NEXT:    image_gather4_lz_o v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: gather4_lz_o_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    s_mov_b32 s8, s10
+; GFX10-NEXT:    s_mov_b32 s9, s11
+; GFX10-NEXT:    s_mov_b32 s10, s12
+; GFX10-NEXT:    s_mov_b32 s11, s13
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_gather4_lz_o v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.lz.o.2d.v4f32.f32(i32 1, i32 %offset, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
 }
 
 define amdgpu_ps <4 x float> @gather4_c_lz_o_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, float %s, float %t) {
-  ; GFX6-LABEL: name: gather4_c_lz_o_2d
-  ; GFX6: bb.1.main_body:
-  ; GFX6:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX6:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX6:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX6:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX6:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX6:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX6:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX6:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX6:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX6:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX6:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX6:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX6:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX6:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX6:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX6:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX6:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX6:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX6:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX6:   [[BUILD_VECTOR2:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32)
-  ; GFX6:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.lz.o.2d), 1, [[BUILD_VECTOR2]](<4 x s32>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX6:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX6:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX6:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX6:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX6:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX6:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  ; GFX10-LABEL: name: gather4_c_lz_o_2d
-  ; GFX10: bb.1.main_body:
-  ; GFX10:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX10:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
-  ; GFX10:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr3
-  ; GFX10:   [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr4
-  ; GFX10:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr5
-  ; GFX10:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr6
-  ; GFX10:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr7
-  ; GFX10:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr8
-  ; GFX10:   [[COPY7:%[0-9]+]]:_(s32) = COPY $sgpr9
-  ; GFX10:   [[COPY8:%[0-9]+]]:_(s32) = COPY $sgpr10
-  ; GFX10:   [[COPY9:%[0-9]+]]:_(s32) = COPY $sgpr11
-  ; GFX10:   [[COPY10:%[0-9]+]]:_(s32) = COPY $sgpr12
-  ; GFX10:   [[COPY11:%[0-9]+]]:_(s32) = COPY $sgpr13
-  ; GFX10:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
-  ; GFX10:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; GFX10:   [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10:   [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX10:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
-  ; GFX10:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX10:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4.c.lz.o.2d), 1, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom "TargetCustom8")
-  ; GFX10:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
-  ; GFX10:   $vgpr0 = COPY [[UV]](s32)
-  ; GFX10:   $vgpr1 = COPY [[UV1]](s32)
-  ; GFX10:   $vgpr2 = COPY [[UV2]](s32)
-  ; GFX10:   $vgpr3 = COPY [[UV3]](s32)
-  ; GFX10:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+; GFX6-LABEL: gather4_c_lz_o_2d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b32 s10, s12
+; GFX6-NEXT:    s_mov_b32 s11, s13
+; GFX6-NEXT:    image_gather4_c_lz_o v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: gather4_c_lz_o_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    s_mov_b32 s8, s10
+; GFX10-NEXT:    s_mov_b32 s9, s11
+; GFX10-NEXT:    s_mov_b32 s10, s12
+; GFX10-NEXT:    s_mov_b32 s11, s13
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_gather4_c_lz_o v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.gather4.c.lz.o.2d.v4f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   ret <4 x float> %v
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.getresinfo.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.getresinfo.a16.ll
new file mode 100644
index 0000000000000..d4084bafd93f4
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.getresinfo.a16.ll
@@ -0,0 +1,301 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s
+
+define amdgpu_ps <4 x float> @getresinfo_1d(<8 x i32> inreg %rsrc, i16 %mip) {
+; GFX9-LABEL: getresinfo_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: getresinfo_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.getresinfo.1d.v4f32.i16(i32 15, i16 %mip, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @getresinfo_2d(<8 x i32> inreg %rsrc, i16 %mip) {
+; GFX9-LABEL: getresinfo_2d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: getresinfo_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.getresinfo.2d.v4f32.i16(i32 15, i16 %mip, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @getresinfo_3d(<8 x i32> inreg %rsrc, i16 %mip) {
+; GFX9-LABEL: getresinfo_3d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: getresinfo_3d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.getresinfo.3d.v4f32.i16(i32 15, i16 %mip, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @getresinfo_cube(<8 x i32> inreg %rsrc, i16 %mip) {
+; GFX9-LABEL: getresinfo_cube:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm a16 da
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: getresinfo_cube:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_CUBE unorm a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.getresinfo.cube.v4f32.i16(i32 15, i16 %mip, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @getresinfo_1darray(<8 x i32> inreg %rsrc, i16 %mip) {
+; GFX9-LABEL: getresinfo_1darray:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm a16 da
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: getresinfo_1darray:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY unorm a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.getresinfo.1darray.v4f32.i16(i32 15, i16 %mip, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @getresinfo_2darray(<8 x i32> inreg %rsrc, i16 %mip) {
+; GFX9-LABEL: getresinfo_2darray:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm a16 da
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: getresinfo_2darray:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY unorm a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.getresinfo.2darray.v4f32.i16(i32 15, i16 %mip, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @getresinfo_2dmsaa(<8 x i32> inreg %rsrc, i16 %mip) {
+; GFX9-LABEL: getresinfo_2dmsaa:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: getresinfo_2dmsaa:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA unorm a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.getresinfo.2dmsaa.v4f32.i16(i32 15, i16 %mip, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @getresinfo_2darraymsaa(<8 x i32> inreg %rsrc, i16 %mip) {
+; GFX9-LABEL: getresinfo_2darraymsaa:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm a16 da
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: getresinfo_2darraymsaa:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.getresinfo.2darraymsaa.v4f32.i16(i32 15, i16 %mip, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @getresinfo_dmask0(<8 x i32> inreg %rsrc, <4 x float> %vdata, i16 %mip) {
+; GFX9-LABEL: getresinfo_dmask0:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: getresinfo_dmask0:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %r = call <4 x float> @llvm.amdgcn.image.getresinfo.1d.v4f32.i16(i32 0, i16 %mip, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %r
+}
+
+declare <4 x float> @llvm.amdgcn.image.getresinfo.1d.v4f32.i16(i32 immarg, i16, <8 x i32>, i32 immarg, i32 immarg) #1
+declare <4 x float> @llvm.amdgcn.image.getresinfo.2d.v4f32.i16(i32 immarg, i16, <8 x i32>, i32 immarg, i32 immarg) #1
+declare <4 x float> @llvm.amdgcn.image.getresinfo.3d.v4f32.i16(i32 immarg, i16, <8 x i32>, i32 immarg, i32 immarg) #1
+declare <4 x float> @llvm.amdgcn.image.getresinfo.cube.v4f32.i16(i32 immarg, i16, <8 x i32>, i32 immarg, i32 immarg) #1
+declare <4 x float> @llvm.amdgcn.image.getresinfo.1darray.v4f32.i16(i32 immarg, i16, <8 x i32>, i32 immarg, i32 immarg) #1
+declare <4 x float> @llvm.amdgcn.image.getresinfo.2darray.v4f32.i16(i32 immarg, i16, <8 x i32>, i32 immarg, i32 immarg) #1
+declare <4 x float> @llvm.amdgcn.image.getresinfo.2dmsaa.v4f32.i16(i32 immarg, i16, <8 x i32>, i32 immarg, i32 immarg) #1
+declare <4 x float> @llvm.amdgcn.image.getresinfo.2darraymsaa.v4f32.i16(i32 immarg, i16, <8 x i32>, i32 immarg, i32 immarg) #1
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.getresinfo.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.getresinfo.ll
new file mode 100644
index 0000000000000..6b975309cb332
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.getresinfo.ll
@@ -0,0 +1,418 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s
+
+define amdgpu_ps <4 x float> @getresinfo_1d(<8 x i32> inreg %rsrc, i32 %mip) {
+; GFX6-LABEL: getresinfo_1d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: getresinfo_1d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: getresinfo_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.getresinfo.1d.v4f32.i32(i32 15, i32 %mip, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @getresinfo_2d(<8 x i32> inreg %rsrc, i32 %mip) {
+; GFX6-LABEL: getresinfo_2d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: getresinfo_2d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: getresinfo_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.getresinfo.2d.v4f32.i32(i32 15, i32 %mip, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @getresinfo_3d(<8 x i32> inreg %rsrc, i32 %mip) {
+; GFX6-LABEL: getresinfo_3d:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: getresinfo_3d:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: getresinfo_3d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.getresinfo.3d.v4f32.i32(i32 15, i32 %mip, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @getresinfo_cube(<8 x i32> inreg %rsrc, i32 %mip) {
+; GFX6-LABEL: getresinfo_cube:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm da
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: getresinfo_cube:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm da
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: getresinfo_cube:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_CUBE unorm
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.getresinfo.cube.v4f32.i32(i32 15, i32 %mip, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @getresinfo_1darray(<8 x i32> inreg %rsrc, i32 %mip) {
+; GFX6-LABEL: getresinfo_1darray:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm da
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: getresinfo_1darray:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm da
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: getresinfo_1darray:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY unorm
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.getresinfo.1darray.v4f32.i32(i32 15, i32 %mip, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @getresinfo_2darray(<8 x i32> inreg %rsrc, i32 %mip) {
+; GFX6-LABEL: getresinfo_2darray:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm da
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: getresinfo_2darray:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm da
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: getresinfo_2darray:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY unorm
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.getresinfo.2darray.v4f32.i32(i32 15, i32 %mip, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @getresinfo_2dmsaa(<8 x i32> inreg %rsrc, i32 %mip) {
+; GFX6-LABEL: getresinfo_2dmsaa:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: getresinfo_2dmsaa:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: getresinfo_2dmsaa:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA unorm
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.getresinfo.2dmsaa.v4f32.i32(i32 15, i32 %mip, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @getresinfo_2darraymsaa(<8 x i32> inreg %rsrc, i32 %mip) {
+; GFX6-LABEL: getresinfo_2darraymsaa:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm da
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: getresinfo_2darraymsaa:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm da
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: getresinfo_2darraymsaa:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.getresinfo.2darraymsaa.v4f32.i32(i32 15, i32 %mip, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @getresinfo_dmask0(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %mip) {
+; GFX6-LABEL: getresinfo_dmask0:
+; GFX6:       ; %bb.0: ; %main_body
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: getresinfo_dmask0:
+; GFX8:       ; %bb.0: ; %main_body
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: getresinfo_dmask0:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %r = call <4 x float> @llvm.amdgcn.image.getresinfo.1d.v4f32.i32(i32 0, i32 %mip, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %r
+}
+
+declare <4 x float> @llvm.amdgcn.image.getresinfo.1d.v4f32.i32(i32 immarg, i32, <8 x i32>, i32 immarg, i32 immarg) #1
+declare <4 x float> @llvm.amdgcn.image.getresinfo.2d.v4f32.i32(i32 immarg, i32, <8 x i32>, i32 immarg, i32 immarg) #1
+declare <4 x float> @llvm.amdgcn.image.getresinfo.3d.v4f32.i32(i32 immarg, i32, <8 x i32>, i32 immarg, i32 immarg) #1
+declare <4 x float> @llvm.amdgcn.image.getresinfo.cube.v4f32.i32(i32 immarg, i32, <8 x i32>, i32 immarg, i32 immarg) #1
+declare <4 x float> @llvm.amdgcn.image.getresinfo.1darray.v4f32.i32(i32 immarg, i32, <8 x i32>, i32 immarg, i32 immarg) #1
+declare <4 x float> @llvm.amdgcn.image.getresinfo.2darray.v4f32.i32(i32 immarg, i32, <8 x i32>, i32 immarg, i32 immarg) #1
+declare <4 x float> @llvm.amdgcn.image.getresinfo.2dmsaa.v4f32.i32(i32 immarg, i32, <8 x i32>, i32 immarg, i32 immarg) #1
+declare <4 x float> @llvm.amdgcn.image.getresinfo.2darraymsaa.v4f32.i32(i32 immarg, i32, <8 x i32>, i32 immarg, i32 immarg) #1
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.1d.d16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.1d.d16.ll
new file mode 100644
index 0000000000000..bfb2c84f6c027
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.1d.d16.ll
@@ -0,0 +1,824 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GFX8-UNPACKED %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx810 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX8-PACKED %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s
+
+define amdgpu_ps half @load_1d_f16_x(<8 x i32> inreg %rsrc, i32 %s) {
+; GFX8-UNPACKED-LABEL: load_1d_f16_x:
+; GFX8-UNPACKED:       ; %bb.0:
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s0, s2
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s1, s3
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s2, s4
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s3, s5
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s4, s6
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s5, s7
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s6, s8
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s7, s9
+; GFX8-UNPACKED-NEXT:    image_load v0, v0, s[0:7] dmask:0x1 unorm d16
+; GFX8-UNPACKED-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-NEXT:    ; return to shader part epilog
+;
+; GFX8-PACKED-LABEL: load_1d_f16_x:
+; GFX8-PACKED:       ; %bb.0:
+; GFX8-PACKED-NEXT:    s_mov_b32 s0, s2
+; GFX8-PACKED-NEXT:    s_mov_b32 s1, s3
+; GFX8-PACKED-NEXT:    s_mov_b32 s2, s4
+; GFX8-PACKED-NEXT:    s_mov_b32 s3, s5
+; GFX8-PACKED-NEXT:    s_mov_b32 s4, s6
+; GFX8-PACKED-NEXT:    s_mov_b32 s5, s7
+; GFX8-PACKED-NEXT:    s_mov_b32 s6, s8
+; GFX8-PACKED-NEXT:    s_mov_b32 s7, s9
+; GFX8-PACKED-NEXT:    s_nop 0
+; GFX8-PACKED-NEXT:    s_nop 0
+; GFX8-PACKED-NEXT:    image_load v0, v0, s[0:7] dmask:0x1 unorm d16
+; GFX8-PACKED-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-PACKED-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: load_1d_f16_x:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_load v0, v0, s[0:7] dmask:0x1 unorm d16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_1d_f16_x:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm d16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call half @llvm.amdgcn.image.load.1d.half.i32(i32 1, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  ret half %v
+}
+
+define amdgpu_ps half @load_1d_f16_y(<8 x i32> inreg %rsrc, i32 %s) {
+; GFX8-UNPACKED-LABEL: load_1d_f16_y:
+; GFX8-UNPACKED:       ; %bb.0:
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s0, s2
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s1, s3
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s2, s4
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s3, s5
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s4, s6
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s5, s7
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s6, s8
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s7, s9
+; GFX8-UNPACKED-NEXT:    image_load v0, v0, s[0:7] dmask:0x2 unorm d16
+; GFX8-UNPACKED-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-NEXT:    ; return to shader part epilog
+;
+; GFX8-PACKED-LABEL: load_1d_f16_y:
+; GFX8-PACKED:       ; %bb.0:
+; GFX8-PACKED-NEXT:    s_mov_b32 s0, s2
+; GFX8-PACKED-NEXT:    s_mov_b32 s1, s3
+; GFX8-PACKED-NEXT:    s_mov_b32 s2, s4
+; GFX8-PACKED-NEXT:    s_mov_b32 s3, s5
+; GFX8-PACKED-NEXT:    s_mov_b32 s4, s6
+; GFX8-PACKED-NEXT:    s_mov_b32 s5, s7
+; GFX8-PACKED-NEXT:    s_mov_b32 s6, s8
+; GFX8-PACKED-NEXT:    s_mov_b32 s7, s9
+; GFX8-PACKED-NEXT:    s_nop 0
+; GFX8-PACKED-NEXT:    s_nop 0
+; GFX8-PACKED-NEXT:    image_load v0, v0, s[0:7] dmask:0x2 unorm d16
+; GFX8-PACKED-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-PACKED-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: load_1d_f16_y:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_load v0, v0, s[0:7] dmask:0x2 unorm d16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_1d_f16_y:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v0, v0, s[0:7] dmask:0x2 dim:SQ_RSRC_IMG_1D unorm d16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call half @llvm.amdgcn.image.load.1d.half.i32(i32 2, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  ret half %v
+}
+
+define amdgpu_ps half @load_1d_f16_z(<8 x i32> inreg %rsrc, i32 %s) {
+; GFX8-UNPACKED-LABEL: load_1d_f16_z:
+; GFX8-UNPACKED:       ; %bb.0:
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s0, s2
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s1, s3
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s2, s4
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s3, s5
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s4, s6
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s5, s7
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s6, s8
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s7, s9
+; GFX8-UNPACKED-NEXT:    image_load v0, v0, s[0:7] dmask:0x4 unorm d16
+; GFX8-UNPACKED-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-NEXT:    ; return to shader part epilog
+;
+; GFX8-PACKED-LABEL: load_1d_f16_z:
+; GFX8-PACKED:       ; %bb.0:
+; GFX8-PACKED-NEXT:    s_mov_b32 s0, s2
+; GFX8-PACKED-NEXT:    s_mov_b32 s1, s3
+; GFX8-PACKED-NEXT:    s_mov_b32 s2, s4
+; GFX8-PACKED-NEXT:    s_mov_b32 s3, s5
+; GFX8-PACKED-NEXT:    s_mov_b32 s4, s6
+; GFX8-PACKED-NEXT:    s_mov_b32 s5, s7
+; GFX8-PACKED-NEXT:    s_mov_b32 s6, s8
+; GFX8-PACKED-NEXT:    s_mov_b32 s7, s9
+; GFX8-PACKED-NEXT:    s_nop 0
+; GFX8-PACKED-NEXT:    s_nop 0
+; GFX8-PACKED-NEXT:    image_load v0, v0, s[0:7] dmask:0x4 unorm d16
+; GFX8-PACKED-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-PACKED-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: load_1d_f16_z:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_load v0, v0, s[0:7] dmask:0x4 unorm d16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_1d_f16_z:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v0, v0, s[0:7] dmask:0x4 dim:SQ_RSRC_IMG_1D unorm d16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call half @llvm.amdgcn.image.load.1d.half.i32(i32 4, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  ret half %v
+}
+
+define amdgpu_ps half @load_1d_f16_w(<8 x i32> inreg %rsrc, i32 %s) {
+; GFX8-UNPACKED-LABEL: load_1d_f16_w:
+; GFX8-UNPACKED:       ; %bb.0:
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s0, s2
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s1, s3
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s2, s4
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s3, s5
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s4, s6
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s5, s7
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s6, s8
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s7, s9
+; GFX8-UNPACKED-NEXT:    image_load v0, v0, s[0:7] dmask:0x8 unorm d16
+; GFX8-UNPACKED-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-NEXT:    ; return to shader part epilog
+;
+; GFX8-PACKED-LABEL: load_1d_f16_w:
+; GFX8-PACKED:       ; %bb.0:
+; GFX8-PACKED-NEXT:    s_mov_b32 s0, s2
+; GFX8-PACKED-NEXT:    s_mov_b32 s1, s3
+; GFX8-PACKED-NEXT:    s_mov_b32 s2, s4
+; GFX8-PACKED-NEXT:    s_mov_b32 s3, s5
+; GFX8-PACKED-NEXT:    s_mov_b32 s4, s6
+; GFX8-PACKED-NEXT:    s_mov_b32 s5, s7
+; GFX8-PACKED-NEXT:    s_mov_b32 s6, s8
+; GFX8-PACKED-NEXT:    s_mov_b32 s7, s9
+; GFX8-PACKED-NEXT:    s_nop 0
+; GFX8-PACKED-NEXT:    s_nop 0
+; GFX8-PACKED-NEXT:    image_load v0, v0, s[0:7] dmask:0x8 unorm d16
+; GFX8-PACKED-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-PACKED-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: load_1d_f16_w:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_load v0, v0, s[0:7] dmask:0x8 unorm d16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_1d_f16_w:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v0, v0, s[0:7] dmask:0x8 dim:SQ_RSRC_IMG_1D unorm d16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call half @llvm.amdgcn.image.load.1d.half.i32(i32 8, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  ret half %v
+}
+
+define amdgpu_ps <2 x half> @load_1d_v2f16_xy(<8 x i32> inreg %rsrc, i32 %s) {
+; GFX8-UNPACKED-LABEL: load_1d_v2f16_xy:
+; GFX8-UNPACKED:       ; %bb.0:
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s0, s2
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s1, s3
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s2, s4
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s3, s5
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s4, s6
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s5, s7
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s6, s8
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s7, s9
+; GFX8-UNPACKED-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x3 unorm d16
+; GFX8-UNPACKED-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX8-UNPACKED-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-UNPACKED-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-UNPACKED-NEXT:    ; return to shader part epilog
+;
+; GFX8-PACKED-LABEL: load_1d_v2f16_xy:
+; GFX8-PACKED:       ; %bb.0:
+; GFX8-PACKED-NEXT:    s_mov_b32 s0, s2
+; GFX8-PACKED-NEXT:    s_mov_b32 s1, s3
+; GFX8-PACKED-NEXT:    s_mov_b32 s2, s4
+; GFX8-PACKED-NEXT:    s_mov_b32 s3, s5
+; GFX8-PACKED-NEXT:    s_mov_b32 s4, s6
+; GFX8-PACKED-NEXT:    s_mov_b32 s5, s7
+; GFX8-PACKED-NEXT:    s_mov_b32 s6, s8
+; GFX8-PACKED-NEXT:    s_mov_b32 s7, s9
+; GFX8-PACKED-NEXT:    s_nop 0
+; GFX8-PACKED-NEXT:    s_nop 0
+; GFX8-PACKED-NEXT:    image_load v0, v0, s[0:7] dmask:0x3 unorm d16
+; GFX8-PACKED-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-PACKED-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: load_1d_v2f16_xy:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_load v0, v0, s[0:7] dmask:0x3 unorm d16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_1d_v2f16_xy:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v0, v0, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm d16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call <2 x half> @llvm.amdgcn.image.load.1d.v2f16.i32(i32 3, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <2 x half> %v
+}
+
+define amdgpu_ps <2 x half> @load_1d_v2f16_xz(<8 x i32> inreg %rsrc, i32 %s) {
+; GFX8-UNPACKED-LABEL: load_1d_v2f16_xz:
+; GFX8-UNPACKED:       ; %bb.0:
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s0, s2
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s1, s3
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s2, s4
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s3, s5
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s4, s6
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s5, s7
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s6, s8
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s7, s9
+; GFX8-UNPACKED-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x5 unorm d16
+; GFX8-UNPACKED-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX8-UNPACKED-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-UNPACKED-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-UNPACKED-NEXT:    ; return to shader part epilog
+;
+; GFX8-PACKED-LABEL: load_1d_v2f16_xz:
+; GFX8-PACKED:       ; %bb.0:
+; GFX8-PACKED-NEXT:    s_mov_b32 s0, s2
+; GFX8-PACKED-NEXT:    s_mov_b32 s1, s3
+; GFX8-PACKED-NEXT:    s_mov_b32 s2, s4
+; GFX8-PACKED-NEXT:    s_mov_b32 s3, s5
+; GFX8-PACKED-NEXT:    s_mov_b32 s4, s6
+; GFX8-PACKED-NEXT:    s_mov_b32 s5, s7
+; GFX8-PACKED-NEXT:    s_mov_b32 s6, s8
+; GFX8-PACKED-NEXT:    s_mov_b32 s7, s9
+; GFX8-PACKED-NEXT:    s_nop 0
+; GFX8-PACKED-NEXT:    s_nop 0
+; GFX8-PACKED-NEXT:    image_load v0, v0, s[0:7] dmask:0x5 unorm d16
+; GFX8-PACKED-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-PACKED-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: load_1d_v2f16_xz:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_load v0, v0, s[0:7] dmask:0x5 unorm d16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_1d_v2f16_xz:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v0, v0, s[0:7] dmask:0x5 dim:SQ_RSRC_IMG_1D unorm d16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call <2 x half> @llvm.amdgcn.image.load.1d.v2f16.i32(i32 5, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <2 x half> %v
+}
+
+define amdgpu_ps <2 x half> @load_1d_v2f16_xw(<8 x i32> inreg %rsrc, i32 %s) {
+; GFX8-UNPACKED-LABEL: load_1d_v2f16_xw:
+; GFX8-UNPACKED:       ; %bb.0:
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s0, s2
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s1, s3
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s2, s4
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s3, s5
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s4, s6
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s5, s7
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s6, s8
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s7, s9
+; GFX8-UNPACKED-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x9 unorm d16
+; GFX8-UNPACKED-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX8-UNPACKED-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-UNPACKED-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-UNPACKED-NEXT:    ; return to shader part epilog
+;
+; GFX8-PACKED-LABEL: load_1d_v2f16_xw:
+; GFX8-PACKED:       ; %bb.0:
+; GFX8-PACKED-NEXT:    s_mov_b32 s0, s2
+; GFX8-PACKED-NEXT:    s_mov_b32 s1, s3
+; GFX8-PACKED-NEXT:    s_mov_b32 s2, s4
+; GFX8-PACKED-NEXT:    s_mov_b32 s3, s5
+; GFX8-PACKED-NEXT:    s_mov_b32 s4, s6
+; GFX8-PACKED-NEXT:    s_mov_b32 s5, s7
+; GFX8-PACKED-NEXT:    s_mov_b32 s6, s8
+; GFX8-PACKED-NEXT:    s_mov_b32 s7, s9
+; GFX8-PACKED-NEXT:    s_nop 0
+; GFX8-PACKED-NEXT:    s_nop 0
+; GFX8-PACKED-NEXT:    image_load v0, v0, s[0:7] dmask:0x9 unorm d16
+; GFX8-PACKED-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-PACKED-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: load_1d_v2f16_xw:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_load v0, v0, s[0:7] dmask:0x9 unorm d16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_1d_v2f16_xw:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v0, v0, s[0:7] dmask:0x9 dim:SQ_RSRC_IMG_1D unorm d16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call <2 x half> @llvm.amdgcn.image.load.1d.v2f16.i32(i32 9, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <2 x half> %v
+}
+
+define amdgpu_ps <2 x half> @load_1d_v2f16_yz(<8 x i32> inreg %rsrc, i32 %s) {
+; GFX8-UNPACKED-LABEL: load_1d_v2f16_yz:
+; GFX8-UNPACKED:       ; %bb.0:
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s0, s2
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s1, s3
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s2, s4
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s3, s5
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s4, s6
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s5, s7
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s6, s8
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s7, s9
+; GFX8-UNPACKED-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x6 unorm d16
+; GFX8-UNPACKED-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX8-UNPACKED-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-UNPACKED-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-UNPACKED-NEXT:    ; return to shader part epilog
+;
+; GFX8-PACKED-LABEL: load_1d_v2f16_yz:
+; GFX8-PACKED:       ; %bb.0:
+; GFX8-PACKED-NEXT:    s_mov_b32 s0, s2
+; GFX8-PACKED-NEXT:    s_mov_b32 s1, s3
+; GFX8-PACKED-NEXT:    s_mov_b32 s2, s4
+; GFX8-PACKED-NEXT:    s_mov_b32 s3, s5
+; GFX8-PACKED-NEXT:    s_mov_b32 s4, s6
+; GFX8-PACKED-NEXT:    s_mov_b32 s5, s7
+; GFX8-PACKED-NEXT:    s_mov_b32 s6, s8
+; GFX8-PACKED-NEXT:    s_mov_b32 s7, s9
+; GFX8-PACKED-NEXT:    s_nop 0
+; GFX8-PACKED-NEXT:    s_nop 0
+; GFX8-PACKED-NEXT:    image_load v0, v0, s[0:7] dmask:0x6 unorm d16
+; GFX8-PACKED-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-PACKED-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: load_1d_v2f16_yz:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_load v0, v0, s[0:7] dmask:0x6 unorm d16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_1d_v2f16_yz:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v0, v0, s[0:7] dmask:0x6 dim:SQ_RSRC_IMG_1D unorm d16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call <2 x half> @llvm.amdgcn.image.load.1d.v2f16.i32(i32 6, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <2 x half> %v
+}
+
+; FIXME:
+; define amdgpu_ps <3 x half> @load_1d_v3f16_xyz(<8 x i32> inreg %rsrc, i32 %s) {
+;   %v = call <3 x half> @llvm.amdgcn.image.load.1d.v3f16.i32(i32 7, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+;   ret <3 x half> %v
+; }
+
+define amdgpu_ps <4 x half> @load_1d_v4f16_xyzw(<8 x i32> inreg %rsrc, i32 %s) {
+; GFX8-UNPACKED-LABEL: load_1d_v4f16_xyzw:
+; GFX8-UNPACKED:       ; %bb.0:
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s0, s2
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s1, s3
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s2, s4
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s3, s5
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s4, s6
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s5, s7
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s6, s8
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s7, s9
+; GFX8-UNPACKED-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf unorm d16
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s0, 0xffff
+; GFX8-UNPACKED-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-NEXT:    v_and_b32_e32 v1, s0, v1
+; GFX8-UNPACKED-NEXT:    v_and_b32_e32 v3, s0, v3
+; GFX8-UNPACKED-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-UNPACKED-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX8-UNPACKED-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-UNPACKED-NEXT:    v_or_b32_sdwa v1, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-UNPACKED-NEXT:    ; return to shader part epilog
+;
+; GFX8-PACKED-LABEL: load_1d_v4f16_xyzw:
+; GFX8-PACKED:       ; %bb.0:
+; GFX8-PACKED-NEXT:    s_mov_b32 s0, s2
+; GFX8-PACKED-NEXT:    s_mov_b32 s1, s3
+; GFX8-PACKED-NEXT:    s_mov_b32 s2, s4
+; GFX8-PACKED-NEXT:    s_mov_b32 s3, s5
+; GFX8-PACKED-NEXT:    s_mov_b32 s4, s6
+; GFX8-PACKED-NEXT:    s_mov_b32 s5, s7
+; GFX8-PACKED-NEXT:    s_mov_b32 s6, s8
+; GFX8-PACKED-NEXT:    s_mov_b32 s7, s9
+; GFX8-PACKED-NEXT:    s_nop 0
+; GFX8-PACKED-NEXT:    s_nop 0
+; GFX8-PACKED-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0xf unorm d16
+; GFX8-PACKED-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-PACKED-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: load_1d_v4f16_xyzw:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0xf unorm d16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_1d_v4f16_xyzw:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm d16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call <4 x half> @llvm.amdgcn.image.load.1d.v4f16.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x half> %v
+}
+
+define amdgpu_ps float @load_1d_f16_tfe_dmask_x(<8 x i32> inreg %rsrc, i32 %s) {
+; GFX8-UNPACKED-LABEL: load_1d_f16_tfe_dmask_x:
+; GFX8-UNPACKED:       ; %bb.0:
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s0, s2
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s1, s3
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s2, s4
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s3, s5
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s4, s6
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s5, s7
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s6, s8
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s7, s9
+; GFX8-UNPACKED-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x1 unorm tfe d16
+; GFX8-UNPACKED-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-NEXT:    v_mov_b32_e32 v0, v1
+; GFX8-UNPACKED-NEXT:    ; return to shader part epilog
+;
+; GFX8-PACKED-LABEL: load_1d_f16_tfe_dmask_x:
+; GFX8-PACKED:       ; %bb.0:
+; GFX8-PACKED-NEXT:    s_mov_b32 s0, s2
+; GFX8-PACKED-NEXT:    s_mov_b32 s1, s3
+; GFX8-PACKED-NEXT:    s_mov_b32 s2, s4
+; GFX8-PACKED-NEXT:    s_mov_b32 s3, s5
+; GFX8-PACKED-NEXT:    s_mov_b32 s4, s6
+; GFX8-PACKED-NEXT:    s_mov_b32 s5, s7
+; GFX8-PACKED-NEXT:    s_mov_b32 s6, s8
+; GFX8-PACKED-NEXT:    s_mov_b32 s7, s9
+; GFX8-PACKED-NEXT:    s_nop 0
+; GFX8-PACKED-NEXT:    s_nop 0
+; GFX8-PACKED-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x1 unorm tfe d16
+; GFX8-PACKED-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-PACKED-NEXT:    v_mov_b32_e32 v0, v1
+; GFX8-PACKED-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: load_1d_f16_tfe_dmask_x:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x1 unorm tfe d16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_1d_f16_tfe_dmask_x:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm tfe d16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v0, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call { half, i32 } @llvm.amdgcn.image.load.1d.sl_f16i32s.i32(i32 1, i32 %s, <8 x i32> %rsrc, i32 1, i32 0)
+  %v.err = extractvalue { half, i32 } %v, 1
+  %vv = bitcast i32 %v.err to float
+  ret float %vv
+}
+
+define amdgpu_ps float @load_1d_v2f16_tfe_dmask_xy(<8 x i32> inreg %rsrc, i32 %s) {
+; GFX8-UNPACKED-LABEL: load_1d_v2f16_tfe_dmask_xy:
+; GFX8-UNPACKED:       ; %bb.0:
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s0, s2
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s1, s3
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s2, s4
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s3, s5
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s4, s6
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s5, s7
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s6, s8
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s7, s9
+; GFX8-UNPACKED-NEXT:    image_load v[0:2], v0, s[0:7] dmask:0x3 unorm tfe d16
+; GFX8-UNPACKED-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-NEXT:    v_mov_b32_e32 v0, v2
+; GFX8-UNPACKED-NEXT:    ; return to shader part epilog
+;
+; GFX8-PACKED-LABEL: load_1d_v2f16_tfe_dmask_xy:
+; GFX8-PACKED:       ; %bb.0:
+; GFX8-PACKED-NEXT:    s_mov_b32 s0, s2
+; GFX8-PACKED-NEXT:    s_mov_b32 s1, s3
+; GFX8-PACKED-NEXT:    s_mov_b32 s2, s4
+; GFX8-PACKED-NEXT:    s_mov_b32 s3, s5
+; GFX8-PACKED-NEXT:    s_mov_b32 s4, s6
+; GFX8-PACKED-NEXT:    s_mov_b32 s5, s7
+; GFX8-PACKED-NEXT:    s_mov_b32 s6, s8
+; GFX8-PACKED-NEXT:    s_mov_b32 s7, s9
+; GFX8-PACKED-NEXT:    s_nop 0
+; GFX8-PACKED-NEXT:    s_nop 0
+; GFX8-PACKED-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x3 unorm tfe d16
+; GFX8-PACKED-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-PACKED-NEXT:    v_mov_b32_e32 v0, v1
+; GFX8-PACKED-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: load_1d_v2f16_tfe_dmask_xy:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x3 unorm tfe d16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_1d_v2f16_tfe_dmask_xy:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm tfe d16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v0, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call { <2 x half>, i32 } @llvm.amdgcn.image.load.1d.sl_v2f16i32s.i32(i32 3, i32 %s, <8 x i32> %rsrc, i32 1, i32 0)
+  %v.err = extractvalue { <2 x half>, i32 } %v, 1
+  %vv = bitcast i32 %v.err to float
+  ret float %vv
+}
+
+; FIXME:
+; define amdgpu_ps float @load_1d_v3f16_tfe_dmask_xyz(<8 x i32> inreg %rsrc, i32 %s) {
+;   %v = call { <3 x half>, i32 } @llvm.amdgcn.image.load.1d.sl_v3f16i32s.i32(i32 7, i32 %s, <8 x i32> %rsrc, i32 1, i32 0)
+;   %v.err = extractvalue { <3 x half>, i32 } %v, 1
+;   %vv = bitcast i32 %v.err to float
+;   ret float %vv
+; }
+
+define amdgpu_ps float @load_1d_v4f16_tfe_dmask_xyzw(<8 x i32> inreg %rsrc, i32 %s) {
+; GFX8-UNPACKED-LABEL: load_1d_v4f16_tfe_dmask_xyzw:
+; GFX8-UNPACKED:       ; %bb.0:
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s0, s2
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s1, s3
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s2, s4
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s3, s5
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s4, s6
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s5, s7
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s6, s8
+; GFX8-UNPACKED-NEXT:    s_mov_b32 s7, s9
+; GFX8-UNPACKED-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x10 unorm tfe d16
+; GFX8-UNPACKED-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-NEXT:    v_mov_b32_e32 v0, v1
+; GFX8-UNPACKED-NEXT:    ; return to shader part epilog
+;
+; GFX8-PACKED-LABEL: load_1d_v4f16_tfe_dmask_xyzw:
+; GFX8-PACKED:       ; %bb.0:
+; GFX8-PACKED-NEXT:    s_mov_b32 s0, s2
+; GFX8-PACKED-NEXT:    s_mov_b32 s1, s3
+; GFX8-PACKED-NEXT:    s_mov_b32 s2, s4
+; GFX8-PACKED-NEXT:    s_mov_b32 s3, s5
+; GFX8-PACKED-NEXT:    s_mov_b32 s4, s6
+; GFX8-PACKED-NEXT:    s_mov_b32 s5, s7
+; GFX8-PACKED-NEXT:    s_mov_b32 s6, s8
+; GFX8-PACKED-NEXT:    s_mov_b32 s7, s9
+; GFX8-PACKED-NEXT:    s_nop 0
+; GFX8-PACKED-NEXT:    s_nop 0
+; GFX8-PACKED-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x10 unorm tfe d16
+; GFX8-PACKED-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-PACKED-NEXT:    v_mov_b32_e32 v0, v1
+; GFX8-PACKED-NEXT:    ; return to shader part epilog
+;
+; GFX9-LABEL: load_1d_v4f16_tfe_dmask_xyzw:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x10 unorm tfe d16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, v1
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_1d_v4f16_tfe_dmask_xyzw:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x10 dim:SQ_RSRC_IMG_1D unorm tfe d16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v0, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call { <4 x half>, i32 } @llvm.amdgcn.image.load.1d.sl_v4f16i32s.i32(i32 16, i32 %s, <8 x i32> %rsrc, i32 1, i32 0)
+  %v.err = extractvalue { <4 x half>, i32 } %v, 1
+  %vv = bitcast i32 %v.err to float
+  ret float %vv
+}
+
+declare half @llvm.amdgcn.image.load.1d.half.i32(i32 immarg, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare <2 x half> @llvm.amdgcn.image.load.1d.v2f16.i32(i32 immarg, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare <3 x half> @llvm.amdgcn.image.load.1d.v3f16.i32(i32 immarg, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare <4 x half> @llvm.amdgcn.image.load.1d.v4f16.i32(i32 immarg, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+
+declare { half, i32 } @llvm.amdgcn.image.load.1d.sl_f16i32s.i32(i32 immarg, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare { <2 x half>, i32 } @llvm.amdgcn.image.load.1d.sl_v2f16i32s.i32(i32 immarg, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare { <3 x half>, i32 } @llvm.amdgcn.image.load.1d.sl_v3f16i32s.i32(i32 immarg, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare { <4 x half>, i32 } @llvm.amdgcn.image.load.1d.sl_v4f16i32s.i32(i32 immarg, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+
+attributes #0 = { nounwind readonly }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.1d.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.1d.ll
new file mode 100644
index 0000000000000..0062118cdb494
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.1d.ll
@@ -0,0 +1,746 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s
+
+define amdgpu_ps float @load_1d_f32_x(<8 x i32> inreg %rsrc, i32 %s) {
+; GFX6-LABEL: load_1d_f32_x:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_load v0, v0, s[0:7] dmask:0x1 unorm
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: load_1d_f32_x:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_load v0, v0, s[0:7] dmask:0x1 unorm
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_1d_f32_x:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call float @llvm.amdgcn.image.load.1d.f32.i32(i32 1, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  ret float %v
+}
+
+define amdgpu_ps float @load_1d_f32_y(<8 x i32> inreg %rsrc, i32 %s) {
+; GFX6-LABEL: load_1d_f32_y:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_load v0, v0, s[0:7] dmask:0x2 unorm
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: load_1d_f32_y:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_load v0, v0, s[0:7] dmask:0x2 unorm
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_1d_f32_y:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v0, v0, s[0:7] dmask:0x2 dim:SQ_RSRC_IMG_1D unorm
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call float @llvm.amdgcn.image.load.1d.f32.i32(i32 2, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  ret float %v
+}
+
+define amdgpu_ps float @load_1d_f32_z(<8 x i32> inreg %rsrc, i32 %s) {
+; GFX6-LABEL: load_1d_f32_z:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_load v0, v0, s[0:7] dmask:0x4 unorm
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: load_1d_f32_z:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_load v0, v0, s[0:7] dmask:0x4 unorm
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_1d_f32_z:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v0, v0, s[0:7] dmask:0x4 dim:SQ_RSRC_IMG_1D unorm
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call float @llvm.amdgcn.image.load.1d.f32.i32(i32 4, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  ret float %v
+}
+
+define amdgpu_ps float @load_1d_f32_w(<8 x i32> inreg %rsrc, i32 %s) {
+; GFX6-LABEL: load_1d_f32_w:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_load v0, v0, s[0:7] dmask:0x8 unorm
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: load_1d_f32_w:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_load v0, v0, s[0:7] dmask:0x8 unorm
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_1d_f32_w:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v0, v0, s[0:7] dmask:0x8 dim:SQ_RSRC_IMG_1D unorm
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call float @llvm.amdgcn.image.load.1d.f32.i32(i32 8, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  ret float %v
+}
+
+define amdgpu_ps <2 x float> @load_1d_v2f32_xy(<8 x i32> inreg %rsrc, i32 %s) {
+; GFX6-LABEL: load_1d_v2f32_xy:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x3 unorm
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: load_1d_v2f32_xy:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x3 unorm
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_1d_v2f32_xy:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call <2 x float> @llvm.amdgcn.image.load.1d.v2f32.i32(i32 3, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <2 x float> %v
+}
+
+define amdgpu_ps <2 x float> @load_1d_v2f32_xz(<8 x i32> inreg %rsrc, i32 %s) {
+; GFX6-LABEL: load_1d_v2f32_xz:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x5 unorm
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: load_1d_v2f32_xz:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x5 unorm
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_1d_v2f32_xz:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x5 dim:SQ_RSRC_IMG_1D unorm
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call <2 x float> @llvm.amdgcn.image.load.1d.v2f32.i32(i32 5, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <2 x float> %v
+}
+
+define amdgpu_ps <2 x float> @load_1d_v2f32_xw(<8 x i32> inreg %rsrc, i32 %s) {
+; GFX6-LABEL: load_1d_v2f32_xw:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x9 unorm
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: load_1d_v2f32_xw:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x9 unorm
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_1d_v2f32_xw:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x9 dim:SQ_RSRC_IMG_1D unorm
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call <2 x float> @llvm.amdgcn.image.load.1d.v2f32.i32(i32 9, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <2 x float> %v
+}
+
+define amdgpu_ps <2 x float> @load_1d_v2f32_yz(<8 x i32> inreg %rsrc, i32 %s) {
+; GFX6-LABEL: load_1d_v2f32_yz:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x6 unorm
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: load_1d_v2f32_yz:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x6 unorm
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_1d_v2f32_yz:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x6 dim:SQ_RSRC_IMG_1D unorm
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call <2 x float> @llvm.amdgcn.image.load.1d.v2f32.i32(i32 6, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <2 x float> %v
+}
+
+define amdgpu_ps <3 x float> @load_1d_v3f32_xyz(<8 x i32> inreg %rsrc, i32 %s) {
+; GFX6-LABEL: load_1d_v3f32_xyz:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_load v[0:2], v0, s[0:7] dmask:0x7 unorm
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: load_1d_v3f32_xyz:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_load v[0:2], v0, s[0:7] dmask:0x7 unorm
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_1d_v3f32_xyz:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:2], v0, s[0:7] dmask:0x7 dim:SQ_RSRC_IMG_1D unorm
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call <3 x float> @llvm.amdgcn.image.load.1d.v3f32.i32(i32 7, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <3 x float> %v
+}
+
+define amdgpu_ps <4 x float> @load_1d_v4f32_xyzw(<8 x i32> inreg %rsrc, i32 %s) {
+; GFX6-LABEL: load_1d_v4f32_xyzw:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf unorm
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: load_1d_v4f32_xyzw:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf unorm
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_1d_v4f32_xyzw:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps float @load_1d_f32_tfe_dmask_x(<8 x i32> inreg %rsrc, i32 %s) {
+; GFX6-LABEL: load_1d_f32_tfe_dmask_x:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x1 unorm tfe
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: load_1d_f32_tfe_dmask_x:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x1 unorm tfe
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_1d_f32_tfe_dmask_x:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm tfe
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v0, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call { float, i32 } @llvm.amdgcn.image.load.1d.sl_f32i32s.i32(i32 1, i32 %s, <8 x i32> %rsrc, i32 1, i32 0)
+  %v.err = extractvalue { float, i32 } %v, 1
+  %vv = bitcast i32 %v.err to float
+  ret float %vv
+}
+
+define amdgpu_ps float @load_1d_v2f32_tfe_dmask_xy(<8 x i32> inreg %rsrc, i32 %s) {
+; GFX6-LABEL: load_1d_v2f32_tfe_dmask_xy:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_load v[0:2], v0, s[0:7] dmask:0x3 unorm tfe
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, v2
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: load_1d_v2f32_tfe_dmask_xy:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_load v[0:2], v0, s[0:7] dmask:0x3 unorm tfe
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, v2
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_1d_v2f32_tfe_dmask_xy:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:2], v0, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm tfe
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v0, v2
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call { <2 x float>, i32 } @llvm.amdgcn.image.load.1d.sl_v2f32i32s.i32(i32 3, i32 %s, <8 x i32> %rsrc, i32 1, i32 0)
+  %v.err = extractvalue { <2 x float>, i32 } %v, 1
+  %vv = bitcast i32 %v.err to float
+  ret float %vv
+}
+
+define amdgpu_ps float @load_1d_v3f32_tfe_dmask_xyz(<8 x i32> inreg %rsrc, i32 %s) {
+; GFX6-LABEL: load_1d_v3f32_tfe_dmask_xyz:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0x7 unorm tfe
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, v3
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: load_1d_v3f32_tfe_dmask_xyz:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0x7 unorm tfe
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, v3
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_1d_v3f32_tfe_dmask_xyz:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0x7 dim:SQ_RSRC_IMG_1D unorm tfe
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v0, v3
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call { <3 x float>, i32 } @llvm.amdgcn.image.load.1d.sl_v3f32i32s.i32(i32 7, i32 %s, <8 x i32> %rsrc, i32 1, i32 0)
+  %v.err = extractvalue { <3 x float>, i32 } %v, 1
+  %vv = bitcast i32 %v.err to float
+  ret float %vv
+}
+
+define amdgpu_ps float @load_1d_v4f32_tfe_dmask_xyzw(<8 x i32> inreg %rsrc, i32 %s) {
+; GFX6-LABEL: load_1d_v4f32_tfe_dmask_xyzw:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x10 unorm tfe
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: load_1d_v4f32_tfe_dmask_xyzw:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x10 unorm tfe
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_1d_v4f32_tfe_dmask_xyzw:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x10 dim:SQ_RSRC_IMG_1D unorm tfe
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v0, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call { <4 x float>, i32 } @llvm.amdgcn.image.load.1d.sl_v4f32i32s.i32(i32 16, i32 %s, <8 x i32> %rsrc, i32 1, i32 0)
+  %v.err = extractvalue { <4 x float>, i32 } %v, 1
+  %vv = bitcast i32 %v.err to float
+  ret float %vv
+}
+
+define amdgpu_ps float @load_1d_f32_tfe_dmask_0(<8 x i32> inreg %rsrc, i32 %s) {
+; GFX6-LABEL: load_1d_f32_tfe_dmask_0:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x1 unorm tfe
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, v1
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX8-LABEL: load_1d_f32_tfe_dmask_0:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x1 unorm tfe
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, v1
+; GFX8-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_1d_f32_tfe_dmask_0:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm tfe
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v0, v1
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call { float, i32 } @llvm.amdgcn.image.load.1d.sl_f32i32s.i32(i32 0, i32 %s, <8 x i32> %rsrc, i32 1, i32 0)
+  %v.err = extractvalue { float, i32 } %v, 1
+  %vv = bitcast i32 %v.err to float
+  ret float %vv
+}
+
+declare float @llvm.amdgcn.image.load.1d.f32.i32(i32 immarg, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare <2 x float> @llvm.amdgcn.image.load.1d.v2f32.i32(i32 immarg, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare <3 x float> @llvm.amdgcn.image.load.1d.v3f32.i32(i32 immarg, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 immarg, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+
+declare { float, i32 } @llvm.amdgcn.image.load.1d.sl_f32i32s.i32(i32 immarg, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare { <2 x float>, i32 } @llvm.amdgcn.image.load.1d.sl_v2f32i32s.i32(i32 immarg, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare { <3 x float>, i32 } @llvm.amdgcn.image.load.1d.sl_v3f32i32s.i32(i32 immarg, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare { <4 x float>, i32 } @llvm.amdgcn.image.load.1d.sl_v4f32i32s.i32(i32 immarg, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+
+attributes #0 = { nounwind readonly }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2d.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2d.ll
new file mode 100644
index 0000000000000..fdaca8f4985c7
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2d.ll
@@ -0,0 +1,133 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s
+
+define amdgpu_ps <4 x float> @load_2d_v4f32_xyzw(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+; GFX6-LABEL: load_2d_v4f32_xyzw:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf unorm
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_2d_v4f32_xyzw:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i32(i32 15, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @load_2d_v4f32_xyzw_tfe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %t) {
+; GFX6-LABEL: load_2d_v4f32_xyzw_tfe:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_load v[0:4], v[0:1], s[0:7] dmask:0xf unorm tfe
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b32 s10, -1
+; GFX6-NEXT:    s_mov_b32 s11, 0xf000
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v4, off, s[8:11], 0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_2d_v4f32_xyzw_tfe:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    v_mov_b32_e32 v5, s10
+; GFX10-NEXT:    v_mov_b32_e32 v6, s11
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:4], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm tfe
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_dword v[5:6], v4, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call { <4 x float>, i32 } @llvm.amdgcn.image.load.2d.sl_v4f32i32s.i32(i32 15, i32 %s, i32 %t, <8 x i32> %rsrc, i32 1, i32 0)
+  %v.vec = extractvalue { <4 x float>, i32 } %v, 0
+  %v.err = extractvalue { <4 x float>, i32 } %v, 1
+  store i32 %v.err, i32 addrspace(1)* %out, align 4
+  ret <4 x float> %v.vec
+}
+
+define amdgpu_ps <4 x float> @load_2d_v4f32_xyzw_tfe_lwe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %t) {
+; GFX6-LABEL: load_2d_v4f32_xyzw_tfe_lwe:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_load v[0:4], v[0:1], s[0:7] dmask:0xf unorm tfe lwe
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b32 s10, -1
+; GFX6-NEXT:    s_mov_b32 s11, 0xf000
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v4, off, s[8:11], 0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_2d_v4f32_xyzw_tfe_lwe:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    v_mov_b32_e32 v5, s10
+; GFX10-NEXT:    v_mov_b32_e32 v6, s11
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:4], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm tfe lwe
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_dword v[5:6], v4, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call { <4 x float>, i32 } @llvm.amdgcn.image.load.2d.sl_v4f32i32s.i32(i32 15, i32 %s, i32 %t, <8 x i32> %rsrc, i32 3, i32 0)
+  %v.vec = extractvalue { <4 x float>, i32 } %v, 0
+  %v.err = extractvalue { <4 x float>, i32 } %v, 1
+  store i32 %v.err, i32 addrspace(1)* %out, align 4
+  ret <4 x float> %v.vec
+}
+
+declare <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i32(i32 immarg, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare { <4 x float>, i32 } @llvm.amdgcn.image.load.2d.sl_v4f32i32s.i32(i32 immarg, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+
+attributes #0 = { nounwind readonly }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.a16.ll
new file mode 100644
index 0000000000000..4a20c7df3ba7c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.a16.ll
@@ -0,0 +1,159 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s
+
+define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw(<8 x i32> inreg %rsrc, i16 %s, i16 %t, i16 %slice, i16 %fragid) {
+; GFX9-LABEL: load_2darraymsaa_v4f32_xyzw:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    v_and_or_b32 v0, v0, v4, v1
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    v_and_or_b32 v1, v2, v4, v1
+; GFX9-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf unorm a16 da
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_2darraymsaa_v4f32_xyzw:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    v_and_or_b32 v0, v0, v4, v1
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    v_and_or_b32 v1, v2, v4, v3
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call <4 x float> @llvm.amdgcn.image.load.2darraymsaa.v4f32.i16(i32 15, i16 %s, i16 %t, i16 %slice, i16 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i16 %s, i16 %t, i16 %slice, i16 %fragid) {
+; GFX9-LABEL: load_2darraymsaa_v4f32_xyzw_tfe:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    v_and_or_b32 v0, v0, v4, v1
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    v_and_or_b32 v1, v2, v4, v1
+; GFX9-NEXT:    image_load v[0:4], v[0:1], s[0:7] dmask:0xf unorm a16 tfe da
+; GFX9-NEXT:    v_mov_b32_e32 v5, s10
+; GFX9-NEXT:    v_mov_b32_e32 v6, s11
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_dword v[5:6], v4, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_2darraymsaa_v4f32_xyzw_tfe:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    v_and_or_b32 v0, v0, v4, v1
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    v_and_or_b32 v1, v2, v4, v3
+; GFX10-NEXT:    v_mov_b32_e32 v5, s10
+; GFX10-NEXT:    v_mov_b32_e32 v6, s11
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:4], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16 tfe
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_dword v[5:6], v4, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call { <4 x float>, i32 } @llvm.amdgcn.image.load.2darraymsaa.sl_v4f32i32s.i16(i32 15, i16 %s, i16 %t, i16 %slice, i16 %fragid, <8 x i32> %rsrc, i32 1, i32 0)
+  %v.vec = extractvalue { <4 x float>, i32 } %v, 0
+  %v.err = extractvalue { <4 x float>, i32 } %v, 1
+  store i32 %v.err, i32 addrspace(1)* %out, align 4
+  ret <4 x float> %v.vec
+}
+
+define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe_lwe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i16 %s, i16 %t, i16 %slice, i16 %fragid) {
+; GFX9-LABEL: load_2darraymsaa_v4f32_xyzw_tfe_lwe:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    v_and_or_b32 v0, v0, v4, v1
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v3
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    v_and_or_b32 v1, v2, v4, v1
+; GFX9-NEXT:    image_load v[0:4], v[0:1], s[0:7] dmask:0xf unorm a16 tfe lwe da
+; GFX9-NEXT:    v_mov_b32_e32 v5, s10
+; GFX9-NEXT:    v_mov_b32_e32 v6, s11
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_dword v[5:6], v4, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_2darraymsaa_v4f32_xyzw_tfe_lwe:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    v_and_or_b32 v0, v0, v4, v1
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    v_and_or_b32 v1, v2, v4, v3
+; GFX10-NEXT:    v_mov_b32_e32 v5, s10
+; GFX10-NEXT:    v_mov_b32_e32 v6, s11
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:4], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16 tfe lwe
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_dword v[5:6], v4, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call { <4 x float>, i32 } @llvm.amdgcn.image.load.2darraymsaa.sl_v4f32i32s.i16(i32 15, i16 %s, i16 %t, i16 %slice, i16 %fragid, <8 x i32> %rsrc, i32 3, i32 0)
+  %v.vec = extractvalue { <4 x float>, i32 } %v, 0
+  %v.err = extractvalue { <4 x float>, i32 } %v, 1
+  store i32 %v.err, i32 addrspace(1)* %out, align 4
+  ret <4 x float> %v.vec
+}
+
+declare <4 x float> @llvm.amdgcn.image.load.2darraymsaa.v4f32.i16(i32 immarg, i16, i16, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare { <4 x float>, i32 } @llvm.amdgcn.image.load.2darraymsaa.sl_v4f32i32s.i16(i32 immarg, i16, i16, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+
+attributes #0 = { nounwind readonly }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.ll
new file mode 100644
index 0000000000000..38845b42c666b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.ll
@@ -0,0 +1,133 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s
+
+define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %slice, i32 %fragid) {
+; GFX6-LABEL: load_2darraymsaa_v4f32_xyzw:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_load v[0:3], v[0:3], s[0:7] dmask:0xf unorm da
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_2darraymsaa_v4f32_xyzw:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:3], v[0:3], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call <4 x float> @llvm.amdgcn.image.load.2darraymsaa.v4f32.i32(i32 15, i32 %s, i32 %t, i32 %slice, i32 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %t, i32 %slice, i32 %fragid) {
+; GFX6-LABEL: load_2darraymsaa_v4f32_xyzw_tfe:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_load v[0:4], v[0:3], s[0:7] dmask:0xf unorm tfe da
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b32 s10, -1
+; GFX6-NEXT:    s_mov_b32 s11, 0xf000
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v4, off, s[8:11], 0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_2darraymsaa_v4f32_xyzw_tfe:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    v_mov_b32_e32 v5, s10
+; GFX10-NEXT:    v_mov_b32_e32 v6, s11
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:4], v[0:3], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_dword v[5:6], v4, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call { <4 x float>, i32 } @llvm.amdgcn.image.load.2darraymsaa.sl_v4f32i32s.i32(i32 15, i32 %s, i32 %t, i32 %slice, i32 %fragid, <8 x i32> %rsrc, i32 1, i32 0)
+  %v.vec = extractvalue { <4 x float>, i32 } %v, 0
+  %v.err = extractvalue { <4 x float>, i32 } %v, 1
+  store i32 %v.err, i32 addrspace(1)* %out, align 4
+  ret <4 x float> %v.vec
+}
+
+define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe_lwe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %t, i32 %slice, i32 %fragid) {
+; GFX6-LABEL: load_2darraymsaa_v4f32_xyzw_tfe_lwe:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_load v[0:4], v[0:3], s[0:7] dmask:0xf unorm tfe lwe da
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b32 s10, -1
+; GFX6-NEXT:    s_mov_b32 s11, 0xf000
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v4, off, s[8:11], 0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_2darraymsaa_v4f32_xyzw_tfe_lwe:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    v_mov_b32_e32 v5, s10
+; GFX10-NEXT:    v_mov_b32_e32 v6, s11
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:4], v[0:3], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe lwe
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_dword v[5:6], v4, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call { <4 x float>, i32 } @llvm.amdgcn.image.load.2darraymsaa.sl_v4f32i32s.i32(i32 15, i32 %s, i32 %t, i32 %slice, i32 %fragid, <8 x i32> %rsrc, i32 3, i32 0)
+  %v.vec = extractvalue { <4 x float>, i32 } %v, 0
+  %v.err = extractvalue { <4 x float>, i32 } %v, 1
+  store i32 %v.err, i32 addrspace(1)* %out, align 4
+  ret <4 x float> %v.vec
+}
+
+declare <4 x float> @llvm.amdgcn.image.load.2darraymsaa.v4f32.i32(i32 immarg, i32, i32, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare { <4 x float>, i32 } @llvm.amdgcn.image.load.2darraymsaa.sl_v4f32i32s.i32(i32 immarg, i32, i32, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+
+attributes #0 = { nounwind readonly }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.a16.ll
new file mode 100644
index 0000000000000..cc21bab4db788
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.a16.ll
@@ -0,0 +1,159 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s
+
+define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw(<8 x i32> inreg %rsrc, i16 %s, i16 %t, i16 %r) {
+; GFX9-LABEL: load_3d_v4f32_xyzw:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    s_lshl_b32 s8, s0, 16
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    v_and_or_b32 v0, v0, v3, v1
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    v_and_or_b32 v1, v2, v3, s8
+; GFX9-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf unorm a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_3d_v4f32_xyzw:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT:    s_lshl_b32 s8, s0, 16
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    v_and_or_b32 v0, v0, v3, v1
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    v_and_or_b32 v1, v2, v3, s8
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call <4 x float> @llvm.amdgcn.image.load.3d.v4f32.i16(i32 15, i16 %s, i16 %t, i16 %r, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i16 %s, i16 %t, i16 %r) {
+; GFX9-LABEL: load_3d_v4f32_xyzw_tfe:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    s_lshl_b32 s8, s0, 16
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    v_and_or_b32 v0, v0, v3, v1
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    v_and_or_b32 v1, v2, v3, s8
+; GFX9-NEXT:    image_load v[0:4], v[0:1], s[0:7] dmask:0xf unorm a16 tfe
+; GFX9-NEXT:    v_mov_b32_e32 v5, s10
+; GFX9-NEXT:    v_mov_b32_e32 v6, s11
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_dword v[5:6], v4, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_3d_v4f32_xyzw_tfe:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT:    s_lshl_b32 s8, s0, 16
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    v_and_or_b32 v0, v0, v3, v1
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    v_and_or_b32 v1, v2, v3, s8
+; GFX10-NEXT:    v_mov_b32_e32 v5, s10
+; GFX10-NEXT:    v_mov_b32_e32 v6, s11
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:4], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16 tfe
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_dword v[5:6], v4, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call { <4 x float>, i32 } @llvm.amdgcn.image.load.3d.sl_v4f32i32s.i16(i32 15, i16 %s, i16 %t, i16 %r, <8 x i32> %rsrc, i32 1, i32 0)
+  %v.vec = extractvalue { <4 x float>, i32 } %v, 0
+  %v.err = extractvalue { <4 x float>, i32 } %v, 1
+  store i32 %v.err, i32 addrspace(1)* %out, align 4
+  ret <4 x float> %v.vec
+}
+
+define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe_lwe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i16 %s, i16 %t, i16 %r) {
+; GFX9-LABEL: load_3d_v4f32_xyzw_tfe_lwe:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    s_lshl_b32 s8, s0, 16
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    v_and_or_b32 v0, v0, v3, v1
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    v_and_or_b32 v1, v2, v3, s8
+; GFX9-NEXT:    image_load v[0:4], v[0:1], s[0:7] dmask:0xf unorm a16 tfe lwe
+; GFX9-NEXT:    v_mov_b32_e32 v5, s10
+; GFX9-NEXT:    v_mov_b32_e32 v6, s11
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_store_dword v[5:6], v4, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_3d_v4f32_xyzw_tfe_lwe:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT:    s_lshl_b32 s8, s0, 16
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    v_and_or_b32 v0, v0, v3, v1
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    v_and_or_b32 v1, v2, v3, s8
+; GFX10-NEXT:    v_mov_b32_e32 v5, s10
+; GFX10-NEXT:    v_mov_b32_e32 v6, s11
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:4], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16 tfe lwe
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_dword v[5:6], v4, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call { <4 x float>, i32 } @llvm.amdgcn.image.load.3d.sl_v4f32i32s.i16(i32 15, i16 %s, i16 %t, i16 %r, <8 x i32> %rsrc, i32 3, i32 0)
+  %v.vec = extractvalue { <4 x float>, i32 } %v, 0
+  %v.err = extractvalue { <4 x float>, i32 } %v, 1
+  store i32 %v.err, i32 addrspace(1)* %out, align 4
+  ret <4 x float> %v.vec
+}
+
+declare <4 x float> @llvm.amdgcn.image.load.3d.v4f32.i16(i32 immarg, i16, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+declare { <4 x float>, i32 } @llvm.amdgcn.image.load.3d.sl_v4f32i32s.i16(i32 immarg, i16, i16, i16, <8 x i32>, i32 immarg, i32 immarg) #0
+
+attributes #0 = { nounwind readonly }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.ll
new file mode 100644
index 0000000000000..e3da98720b16a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.ll
@@ -0,0 +1,133 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s
+
+define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %r) {
+; GFX6-LABEL: load_3d_v4f32_xyzw:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_load v[0:3], v[0:2], s[0:7] dmask:0xf unorm
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_3d_v4f32_xyzw:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:3], v[0:2], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call <4 x float> @llvm.amdgcn.image.load.3d.v4f32.i32(i32 15, i32 %s, i32 %t, i32 %r, <8 x i32> %rsrc, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %t, i32 %r) {
+; GFX6-LABEL: load_3d_v4f32_xyzw_tfe:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_load v[0:4], v[0:2], s[0:7] dmask:0xf unorm tfe
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b32 s10, -1
+; GFX6-NEXT:    s_mov_b32 s11, 0xf000
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v4, off, s[8:11], 0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_3d_v4f32_xyzw_tfe:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    v_mov_b32_e32 v5, s10
+; GFX10-NEXT:    v_mov_b32_e32 v6, s11
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:4], v[0:2], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm tfe
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_dword v[5:6], v4, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call { <4 x float>, i32 } @llvm.amdgcn.image.load.3d.sl_v4f32i32s.i32(i32 15, i32 %s, i32 %t, i32 %r, <8 x i32> %rsrc, i32 1, i32 0)
+  %v.vec = extractvalue { <4 x float>, i32 } %v, 0
+  %v.err = extractvalue { <4 x float>, i32 } %v, 1
+  store i32 %v.err, i32 addrspace(1)* %out, align 4
+  ret <4 x float> %v.vec
+}
+
+define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe_lwe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %t, i32 %r) {
+; GFX6-LABEL: load_3d_v4f32_xyzw_tfe_lwe:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_load v[0:4], v[0:2], s[0:7] dmask:0xf unorm tfe lwe
+; GFX6-NEXT:    s_mov_b32 s8, s10
+; GFX6-NEXT:    s_mov_b32 s9, s11
+; GFX6-NEXT:    s_mov_b32 s10, -1
+; GFX6-NEXT:    s_mov_b32 s11, 0xf000
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v4, off, s[8:11], 0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: load_3d_v4f32_xyzw_tfe_lwe:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    v_mov_b32_e32 v5, s10
+; GFX10-NEXT:    v_mov_b32_e32 v6, s11
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_load v[0:4], v[0:2], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm tfe lwe
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_dword v[5:6], v4, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    ; return to shader part epilog
+  %v = call { <4 x float>, i32 } @llvm.amdgcn.image.load.3d.sl_v4f32i32s.i32(i32 15, i32 %s, i32 %t, i32 %r, <8 x i32> %rsrc, i32 3, i32 0)
+  %v.vec = extractvalue { <4 x float>, i32 } %v, 0
+  %v.err = extractvalue { <4 x float>, i32 } %v, 1
+  store i32 %v.err, i32 addrspace(1)* %out, align 4
+  ret <4 x float> %v.vec
+}
+
+declare <4 x float> @llvm.amdgcn.image.load.3d.v4f32.i32(i32 immarg, i32, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare { <4 x float>, i32 } @llvm.amdgcn.image.load.3d.sl_v4f32i32s.i32(i32 immarg, i32, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+
+attributes #0 = { nounwind readonly }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.ltolz.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.ltolz.a16.ll
new file mode 100644
index 0000000000000..d200f7c5c306e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.ltolz.a16.ll
@@ -0,0 +1,577 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -o - %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -o - %s | FileCheck -check-prefix=GFX10 %s
+
+define amdgpu_ps <4 x float> @sample_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %lod) {
+; GFX9-LABEL: sample_l_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    s_mov_b32 s8, s10
+; GFX9-NEXT:    s_mov_b32 s9, s11
+; GFX9-NEXT:    s_mov_b32 s10, s12
+; GFX9-NEXT:    s_mov_b32 s11, s13
+; GFX9-NEXT:    image_sample_lz v[0:3], v0, s[0:7], s[8:11] dmask:0xf a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: sample_l_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    s_mov_b32 s8, s10
+; GFX10-NEXT:    s_mov_b32 s9, s11
+; GFX10-NEXT:    s_mov_b32 s10, s12
+; GFX10-NEXT:    s_mov_b32 s11, s13
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_sample_lz v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.sample.l.1d.v4f32.f16(i32 15, half %s, half 0.000000e+00, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @sample_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t, half %lod) {
+; GFX9-LABEL: sample_l_2d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    s_mov_b32 s8, s10
+; GFX9-NEXT:    s_mov_b32 s9, s11
+; GFX9-NEXT:    s_mov_b32 s10, s12
+; GFX9-NEXT:    s_mov_b32 s11, s13
+; GFX9-NEXT:    v_and_or_b32 v0, v0, v2, v1
+; GFX9-NEXT:    image_sample_lz v[0:3], v0, s[0:7], s[8:11] dmask:0xf a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: sample_l_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    s_mov_b32 s8, s10
+; GFX10-NEXT:    s_mov_b32 s9, s11
+; GFX10-NEXT:    s_mov_b32 s10, s12
+; GFX10-NEXT:    s_mov_b32 s11, s13
+; GFX10-NEXT:    v_and_or_b32 v0, v0, 0xffff, v1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_sample_lz v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.sample.l.2d.v4f32.f16(i32 15, half %s, half %t, half -0.000000e+00, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @sample_c_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %s, half %lod) {
+; GFX9-LABEL: sample_c_l_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s8, s10
+; GFX9-NEXT:    s_mov_b32 s10, s12
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX9-NEXT:    s_lshl_b32 s12, s0, 16
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    s_mov_b32 s9, s11
+; GFX9-NEXT:    s_mov_b32 s11, s13
+; GFX9-NEXT:    v_and_or_b32 v1, v1, v2, s12
+; GFX9-NEXT:    image_sample_c_lz v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: sample_c_l_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s8, s10
+; GFX10-NEXT:    s_mov_b32 s10, s12
+; GFX10-NEXT:    s_lshl_b32 s12, s0, 16
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    s_mov_b32 s9, s11
+; GFX10-NEXT:    s_mov_b32 s11, s13
+; GFX10-NEXT:    v_and_or_b32 v1, v1, 0xffff, s12
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_sample_c_lz v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.l.1d.v4f32.f16(i32 15, float %zcompare, half %s, half -2.000000e+00, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @sample_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %s, half %t, half %lod) {
+; GFX9-LABEL: sample_c_l_2d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    s_mov_b32 s8, s10
+; GFX9-NEXT:    s_mov_b32 s9, s11
+; GFX9-NEXT:    s_mov_b32 s10, s12
+; GFX9-NEXT:    s_mov_b32 s11, s13
+; GFX9-NEXT:    v_and_or_b32 v1, v1, v3, v2
+; GFX9-NEXT:    image_sample_c_lz v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: sample_c_l_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    s_mov_b32 s8, s10
+; GFX10-NEXT:    s_mov_b32 s9, s11
+; GFX10-NEXT:    s_mov_b32 s10, s12
+; GFX10-NEXT:    s_mov_b32 s11, s13
+; GFX10-NEXT:    v_and_or_b32 v1, v1, 0xffff, v2
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_sample_c_lz v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.l.2d.v4f32.f16(i32 15, float %zcompare, half %s, half %t, half 0.000000e+00, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @sample_l_o_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, half %s, half %lod) {
+; GFX9-LABEL: sample_l_o_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s8, s10
+; GFX9-NEXT:    s_mov_b32 s10, s12
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX9-NEXT:    s_lshl_b32 s12, s0, 16
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    s_mov_b32 s9, s11
+; GFX9-NEXT:    s_mov_b32 s11, s13
+; GFX9-NEXT:    v_and_or_b32 v1, v1, v2, s12
+; GFX9-NEXT:    image_sample_lz_o v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: sample_l_o_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s8, s10
+; GFX10-NEXT:    s_mov_b32 s10, s12
+; GFX10-NEXT:    s_lshl_b32 s12, s0, 16
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    s_mov_b32 s9, s11
+; GFX10-NEXT:    s_mov_b32 s11, s13
+; GFX10-NEXT:    v_and_or_b32 v1, v1, 0xffff, s12
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_sample_lz_o v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.sample.l.o.1d.v4f32.f16(i32 15, i32 %offset, half %s, half 0.000000e+00, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @sample_l_o_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, half %s, half %t, half %lod) {
+; GFX9-LABEL: sample_l_o_2d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    s_mov_b32 s8, s10
+; GFX9-NEXT:    s_mov_b32 s9, s11
+; GFX9-NEXT:    s_mov_b32 s10, s12
+; GFX9-NEXT:    s_mov_b32 s11, s13
+; GFX9-NEXT:    v_and_or_b32 v1, v1, v3, v2
+; GFX9-NEXT:    image_sample_lz_o v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: sample_l_o_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    s_mov_b32 s8, s10
+; GFX10-NEXT:    s_mov_b32 s9, s11
+; GFX10-NEXT:    s_mov_b32 s10, s12
+; GFX10-NEXT:    s_mov_b32 s11, s13
+; GFX10-NEXT:    v_and_or_b32 v1, v1, 0xffff, v2
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_sample_lz_o v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.sample.l.o.2d.v4f32.f16(i32 15, i32 %offset, half %s, half %t, half 0.000000e+00, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @sample_c_l_o_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, half %s, half %lod) {
+; GFX9-LABEL: sample_c_l_o_1d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s8, s10
+; GFX9-NEXT:    s_mov_b32 s10, s12
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX9-NEXT:    s_lshl_b32 s12, s0, 16
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    s_mov_b32 s9, s11
+; GFX9-NEXT:    s_mov_b32 s11, s13
+; GFX9-NEXT:    v_and_or_b32 v2, v2, v3, s12
+; GFX9-NEXT:    image_sample_c_lz_o v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: sample_c_l_o_1d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s8, s10
+; GFX10-NEXT:    s_mov_b32 s10, s12
+; GFX10-NEXT:    s_lshl_b32 s12, s0, 16
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    s_mov_b32 s9, s11
+; GFX10-NEXT:    s_mov_b32 s11, s13
+; GFX10-NEXT:    v_and_or_b32 v2, v2, 0xffff, s12
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_sample_c_lz_o v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.l.o.1d.v4f32.f16(i32 15, i32 %offset, float %zcompare, half %s, half 0.000000e+00, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @sample_c_l_o_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, half %s, half %t, half %lod) {
+; GFX9-LABEL: sample_c_l_o_2d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    s_mov_b32 s8, s10
+; GFX9-NEXT:    s_mov_b32 s9, s11
+; GFX9-NEXT:    s_mov_b32 s10, s12
+; GFX9-NEXT:    s_mov_b32 s11, s13
+; GFX9-NEXT:    v_and_or_b32 v2, v2, v4, v3
+; GFX9-NEXT:    image_sample_c_lz_o v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: sample_c_l_o_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    s_mov_b32 s8, s10
+; GFX10-NEXT:    s_mov_b32 s9, s11
+; GFX10-NEXT:    s_mov_b32 s10, s12
+; GFX10-NEXT:    s_mov_b32 s11, s13
+; GFX10-NEXT:    v_and_or_b32 v2, v2, 0xffff, v3
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_sample_c_lz_o v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.l.o.2d.v4f32.f16(i32 15, i32 %offset, float %zcompare, half %s, half %t, half 0.000000e+00, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @gather4_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t, half %lod) {
+; GFX9-LABEL: gather4_l_2d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    s_mov_b32 s8, s10
+; GFX9-NEXT:    s_mov_b32 s9, s11
+; GFX9-NEXT:    s_mov_b32 s10, s12
+; GFX9-NEXT:    s_mov_b32 s11, s13
+; GFX9-NEXT:    v_and_or_b32 v0, v0, v2, v1
+; GFX9-NEXT:    image_gather4_lz v[0:3], v0, s[0:7], s[8:11] dmask:0xf a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: gather4_l_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    s_mov_b32 s8, s10
+; GFX10-NEXT:    s_mov_b32 s9, s11
+; GFX10-NEXT:    s_mov_b32 s10, s12
+; GFX10-NEXT:    s_mov_b32 s11, s13
+; GFX10-NEXT:    v_and_or_b32 v0, v0, 0xffff, v1
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_gather4_lz v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f16(i32 15, half %s, half %t, half 0.000000e+00, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @gather4_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %s, half %t, half %lod) {
+; GFX9-LABEL: gather4_c_l_2d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    s_mov_b32 s8, s10
+; GFX9-NEXT:    s_mov_b32 s9, s11
+; GFX9-NEXT:    s_mov_b32 s10, s12
+; GFX9-NEXT:    s_mov_b32 s11, s13
+; GFX9-NEXT:    v_and_or_b32 v1, v1, v3, v2
+; GFX9-NEXT:    image_gather4_c_lz v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: gather4_c_l_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    s_mov_b32 s8, s10
+; GFX10-NEXT:    s_mov_b32 s9, s11
+; GFX10-NEXT:    s_mov_b32 s10, s12
+; GFX10-NEXT:    s_mov_b32 s11, s13
+; GFX10-NEXT:    v_and_or_b32 v1, v1, 0xffff, v2
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_gather4_c_lz v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.gather4.c.l.2d.v4f32.f16(i32 15, float %zcompare, half %s, half %t, half 0.000000e+00, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @gather4_l_o_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, half %s, half %t, half %lod) {
+; GFX9-LABEL: gather4_l_o_2d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    v_mov_b32_e32 v3, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    s_mov_b32 s8, s10
+; GFX9-NEXT:    s_mov_b32 s9, s11
+; GFX9-NEXT:    s_mov_b32 s10, s12
+; GFX9-NEXT:    s_mov_b32 s11, s13
+; GFX9-NEXT:    v_and_or_b32 v1, v1, v3, v2
+; GFX9-NEXT:    image_gather4_lz_o v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: gather4_l_o_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    s_mov_b32 s8, s10
+; GFX10-NEXT:    s_mov_b32 s9, s11
+; GFX10-NEXT:    s_mov_b32 s10, s12
+; GFX10-NEXT:    s_mov_b32 s11, s13
+; GFX10-NEXT:    v_and_or_b32 v1, v1, 0xffff, v2
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_gather4_lz_o v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.gather4.l.o.2d.v4f32.f16(i32 15, i32 %offset, half %s, half %t, half 0.000000e+00, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @gather4_c_l_o_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, half %s, half %t, half %lod) {
+; GFX9-LABEL: gather4_c_l_o_2d:
+; GFX9:       ; %bb.0: ; %main_body
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT:    s_mov_b32 s0, s2
+; GFX9-NEXT:    s_mov_b32 s1, s3
+; GFX9-NEXT:    s_mov_b32 s2, s4
+; GFX9-NEXT:    s_mov_b32 s3, s5
+; GFX9-NEXT:    s_mov_b32 s4, s6
+; GFX9-NEXT:    s_mov_b32 s5, s7
+; GFX9-NEXT:    s_mov_b32 s6, s8
+; GFX9-NEXT:    s_mov_b32 s7, s9
+; GFX9-NEXT:    s_mov_b32 s8, s10
+; GFX9-NEXT:    s_mov_b32 s9, s11
+; GFX9-NEXT:    s_mov_b32 s10, s12
+; GFX9-NEXT:    s_mov_b32 s11, s13
+; GFX9-NEXT:    v_and_or_b32 v2, v2, v4, v3
+; GFX9-NEXT:    image_gather4_c_lz_o v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf a16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    ; return to shader part epilog
+;
+; GFX10-LABEL: gather4_c_l_o_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    s_mov_b32 s8, s10
+; GFX10-NEXT:    s_mov_b32 s9, s11
+; GFX10-NEXT:    s_mov_b32 s10, s12
+; GFX10-NEXT:    s_mov_b32 s11, s13
+; GFX10-NEXT:    v_and_or_b32 v2, v2, 0xffff, v3
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_gather4_c_lz_o v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2d.v4f32.f16(i32 15, i32 %offset, float %zcompare, half %s, half %t, half 0.000000e+00, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.l.1d.v4f32.f16(i32 immarg, half, half, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.sample.l.2d.v4f32.f16(i32 immarg, half, half, half, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.sample.c.l.1d.v4f32.f16(i32 immarg, float, half, half, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.sample.c.l.2d.v4f32.f16(i32 immarg, float, half, half, half, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.sample.l.o.1d.v4f32.f16(i32 immarg, i32, half, half, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.sample.l.o.2d.v4f32.f16(i32 immarg, i32, half, half, half, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.sample.c.l.o.1d.v4f32.f16(i32 immarg, i32, float, half, half, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.sample.c.l.o.2d.v4f32.f16(i32 immarg, i32, float, half, half, half, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f16(i32 immarg, half, half, half, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.gather4.c.l.2d.v4f32.f16(i32 immarg, float, half, half, half, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.gather4.l.o.2d.v4f32.f16(i32 immarg, i32, half, half, half, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2d.v4f32.f16(i32 immarg, i32, float, half, half, half, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) #0
+
+attributes #0 = { nounwind readonly }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.ltolz.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.ltolz.ll
new file mode 100644
index 0000000000000..516e92e08b16e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.ltolz.ll
@@ -0,0 +1,293 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -o - %s | FileCheck -check-prefix=GCN %s
+
+define amdgpu_ps <4 x float> @sample_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %lod) {
+; GCN-LABEL: sample_l_1d:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    s_mov_b32 s0, s2
+; GCN-NEXT:    s_mov_b32 s1, s3
+; GCN-NEXT:    s_mov_b32 s2, s4
+; GCN-NEXT:    s_mov_b32 s3, s5
+; GCN-NEXT:    s_mov_b32 s4, s6
+; GCN-NEXT:    s_mov_b32 s5, s7
+; GCN-NEXT:    s_mov_b32 s6, s8
+; GCN-NEXT:    s_mov_b32 s7, s9
+; GCN-NEXT:    s_mov_b32 s8, s10
+; GCN-NEXT:    s_mov_b32 s9, s11
+; GCN-NEXT:    s_mov_b32 s10, s12
+; GCN-NEXT:    s_mov_b32 s11, s13
+; GCN-NEXT:    image_sample_lz v[0:3], v0, s[0:7], s[8:11] dmask:0xf
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.sample.l.1d.v4f32.f32(i32 15, float %s, float 0.000000e+00, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @sample_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t, float %lod) {
+; GCN-LABEL: sample_l_2d:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    s_mov_b32 s0, s2
+; GCN-NEXT:    s_mov_b32 s1, s3
+; GCN-NEXT:    s_mov_b32 s2, s4
+; GCN-NEXT:    s_mov_b32 s3, s5
+; GCN-NEXT:    s_mov_b32 s4, s6
+; GCN-NEXT:    s_mov_b32 s5, s7
+; GCN-NEXT:    s_mov_b32 s6, s8
+; GCN-NEXT:    s_mov_b32 s7, s9
+; GCN-NEXT:    s_mov_b32 s8, s10
+; GCN-NEXT:    s_mov_b32 s9, s11
+; GCN-NEXT:    s_mov_b32 s10, s12
+; GCN-NEXT:    s_mov_b32 s11, s13
+; GCN-NEXT:    image_sample_lz v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.sample.l.2d.v4f32.f32(i32 15, float %s, float %t, float -0.000000e+00, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @sample_c_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %s, float %lod) {
+; GCN-LABEL: sample_c_l_1d:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    s_mov_b32 s0, s2
+; GCN-NEXT:    s_mov_b32 s1, s3
+; GCN-NEXT:    s_mov_b32 s2, s4
+; GCN-NEXT:    s_mov_b32 s3, s5
+; GCN-NEXT:    s_mov_b32 s4, s6
+; GCN-NEXT:    s_mov_b32 s5, s7
+; GCN-NEXT:    s_mov_b32 s6, s8
+; GCN-NEXT:    s_mov_b32 s7, s9
+; GCN-NEXT:    s_mov_b32 s8, s10
+; GCN-NEXT:    s_mov_b32 s9, s11
+; GCN-NEXT:    s_mov_b32 s10, s12
+; GCN-NEXT:    s_mov_b32 s11, s13
+; GCN-NEXT:    image_sample_c_lz v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.l.1d.v4f32.f32(i32 15, float %zcompare, float %s, float -2.000000e+00, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @sample_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %s, float %t, float %lod) {
+; GCN-LABEL: sample_c_l_2d:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    s_mov_b32 s0, s2
+; GCN-NEXT:    s_mov_b32 s1, s3
+; GCN-NEXT:    s_mov_b32 s2, s4
+; GCN-NEXT:    s_mov_b32 s3, s5
+; GCN-NEXT:    s_mov_b32 s4, s6
+; GCN-NEXT:    s_mov_b32 s5, s7
+; GCN-NEXT:    s_mov_b32 s6, s8
+; GCN-NEXT:    s_mov_b32 s7, s9
+; GCN-NEXT:    s_mov_b32 s8, s10
+; GCN-NEXT:    s_mov_b32 s9, s11
+; GCN-NEXT:    s_mov_b32 s10, s12
+; GCN-NEXT:    s_mov_b32 s11, s13
+; GCN-NEXT:    image_sample_c_lz v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.l.2d.v4f32.f32(i32 15, float %zcompare, float %s, float %t, float 0.000000e+00, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @sample_l_o_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %s, float %lod) {
+; GCN-LABEL: sample_l_o_1d:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    s_mov_b32 s0, s2
+; GCN-NEXT:    s_mov_b32 s1, s3
+; GCN-NEXT:    s_mov_b32 s2, s4
+; GCN-NEXT:    s_mov_b32 s3, s5
+; GCN-NEXT:    s_mov_b32 s4, s6
+; GCN-NEXT:    s_mov_b32 s5, s7
+; GCN-NEXT:    s_mov_b32 s6, s8
+; GCN-NEXT:    s_mov_b32 s7, s9
+; GCN-NEXT:    s_mov_b32 s8, s10
+; GCN-NEXT:    s_mov_b32 s9, s11
+; GCN-NEXT:    s_mov_b32 s10, s12
+; GCN-NEXT:    s_mov_b32 s11, s13
+; GCN-NEXT:    image_sample_lz_o v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.sample.l.o.1d.v4f32.f32(i32 15, i32 %offset, float %s, float 0.000000e+00, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @sample_l_o_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %s, float %t, float %lod) {
+; GCN-LABEL: sample_l_o_2d:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    s_mov_b32 s0, s2
+; GCN-NEXT:    s_mov_b32 s1, s3
+; GCN-NEXT:    s_mov_b32 s2, s4
+; GCN-NEXT:    s_mov_b32 s3, s5
+; GCN-NEXT:    s_mov_b32 s4, s6
+; GCN-NEXT:    s_mov_b32 s5, s7
+; GCN-NEXT:    s_mov_b32 s6, s8
+; GCN-NEXT:    s_mov_b32 s7, s9
+; GCN-NEXT:    s_mov_b32 s8, s10
+; GCN-NEXT:    s_mov_b32 s9, s11
+; GCN-NEXT:    s_mov_b32 s10, s12
+; GCN-NEXT:    s_mov_b32 s11, s13
+; GCN-NEXT:    image_sample_lz_o v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.sample.l.o.2d.v4f32.f32(i32 15, i32 %offset, float %s, float %t, float 0.000000e+00, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @sample_c_l_o_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, float %s, float %lod) {
+; GCN-LABEL: sample_c_l_o_1d:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    s_mov_b32 s0, s2
+; GCN-NEXT:    s_mov_b32 s1, s3
+; GCN-NEXT:    s_mov_b32 s2, s4
+; GCN-NEXT:    s_mov_b32 s3, s5
+; GCN-NEXT:    s_mov_b32 s4, s6
+; GCN-NEXT:    s_mov_b32 s5, s7
+; GCN-NEXT:    s_mov_b32 s6, s8
+; GCN-NEXT:    s_mov_b32 s7, s9
+; GCN-NEXT:    s_mov_b32 s8, s10
+; GCN-NEXT:    s_mov_b32 s9, s11
+; GCN-NEXT:    s_mov_b32 s10, s12
+; GCN-NEXT:    s_mov_b32 s11, s13
+; GCN-NEXT:    image_sample_c_lz_o v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.l.o.1d.v4f32.f32(i32 15, i32 %offset, float %zcompare, float %s, float 0.000000e+00, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @sample_c_l_o_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, float %s, float %t, float %lod) {
+; GCN-LABEL: sample_c_l_o_2d:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    s_mov_b32 s0, s2
+; GCN-NEXT:    s_mov_b32 s1, s3
+; GCN-NEXT:    s_mov_b32 s2, s4
+; GCN-NEXT:    s_mov_b32 s3, s5
+; GCN-NEXT:    s_mov_b32 s4, s6
+; GCN-NEXT:    s_mov_b32 s5, s7
+; GCN-NEXT:    s_mov_b32 s6, s8
+; GCN-NEXT:    s_mov_b32 s7, s9
+; GCN-NEXT:    s_mov_b32 s8, s10
+; GCN-NEXT:    s_mov_b32 s9, s11
+; GCN-NEXT:    s_mov_b32 s10, s12
+; GCN-NEXT:    s_mov_b32 s11, s13
+; GCN-NEXT:    image_sample_c_lz_o v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.l.o.2d.v4f32.f32(i32 15, i32 %offset, float %zcompare, float %s, float %t, float 0.000000e+00, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @gather4_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t, float %lod) {
+; GCN-LABEL: gather4_l_2d:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    s_mov_b32 s0, s2
+; GCN-NEXT:    s_mov_b32 s1, s3
+; GCN-NEXT:    s_mov_b32 s2, s4
+; GCN-NEXT:    s_mov_b32 s3, s5
+; GCN-NEXT:    s_mov_b32 s4, s6
+; GCN-NEXT:    s_mov_b32 s5, s7
+; GCN-NEXT:    s_mov_b32 s6, s8
+; GCN-NEXT:    s_mov_b32 s7, s9
+; GCN-NEXT:    s_mov_b32 s8, s10
+; GCN-NEXT:    s_mov_b32 s9, s11
+; GCN-NEXT:    s_mov_b32 s10, s12
+; GCN-NEXT:    s_mov_b32 s11, s13
+; GCN-NEXT:    image_gather4_lz v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f32(i32 15, float %s, float %t, float 0.000000e+00, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @gather4_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %s, float %t, float %lod) {
+; GCN-LABEL: gather4_c_l_2d:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    s_mov_b32 s0, s2
+; GCN-NEXT:    s_mov_b32 s1, s3
+; GCN-NEXT:    s_mov_b32 s2, s4
+; GCN-NEXT:    s_mov_b32 s3, s5
+; GCN-NEXT:    s_mov_b32 s4, s6
+; GCN-NEXT:    s_mov_b32 s5, s7
+; GCN-NEXT:    s_mov_b32 s6, s8
+; GCN-NEXT:    s_mov_b32 s7, s9
+; GCN-NEXT:    s_mov_b32 s8, s10
+; GCN-NEXT:    s_mov_b32 s9, s11
+; GCN-NEXT:    s_mov_b32 s10, s12
+; GCN-NEXT:    s_mov_b32 s11, s13
+; GCN-NEXT:    image_gather4_c_lz v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.gather4.c.l.2d.v4f32.f32(i32 15, float %zcompare, float %s, float %t, float 0.000000e+00, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @gather4_l_o_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %s, float %t, float %lod) {
+; GCN-LABEL: gather4_l_o_2d:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    s_mov_b32 s0, s2
+; GCN-NEXT:    s_mov_b32 s1, s3
+; GCN-NEXT:    s_mov_b32 s2, s4
+; GCN-NEXT:    s_mov_b32 s3, s5
+; GCN-NEXT:    s_mov_b32 s4, s6
+; GCN-NEXT:    s_mov_b32 s5, s7
+; GCN-NEXT:    s_mov_b32 s6, s8
+; GCN-NEXT:    s_mov_b32 s7, s9
+; GCN-NEXT:    s_mov_b32 s8, s10
+; GCN-NEXT:    s_mov_b32 s9, s11
+; GCN-NEXT:    s_mov_b32 s10, s12
+; GCN-NEXT:    s_mov_b32 s11, s13
+; GCN-NEXT:    image_gather4_lz_o v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.gather4.l.o.2d.v4f32.f32(i32 15, i32 %offset, float %s, float %t, float 0.000000e+00, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+define amdgpu_ps <4 x float> @gather4_c_l_o_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, float %s, float %t, float %lod) {
+; GCN-LABEL: gather4_c_l_o_2d:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    s_mov_b32 s0, s2
+; GCN-NEXT:    s_mov_b32 s1, s3
+; GCN-NEXT:    s_mov_b32 s2, s4
+; GCN-NEXT:    s_mov_b32 s3, s5
+; GCN-NEXT:    s_mov_b32 s4, s6
+; GCN-NEXT:    s_mov_b32 s5, s7
+; GCN-NEXT:    s_mov_b32 s6, s8
+; GCN-NEXT:    s_mov_b32 s7, s9
+; GCN-NEXT:    s_mov_b32 s8, s10
+; GCN-NEXT:    s_mov_b32 s9, s11
+; GCN-NEXT:    s_mov_b32 s10, s12
+; GCN-NEXT:    s_mov_b32 s11, s13
+; GCN-NEXT:    image_gather4_c_lz_o v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
+main_body:
+  %v = call <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2d.v4f32.f32(i32 15, i32 %offset, float %zcompare, float %s, float %t, float 0.000000e+00, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  ret <4 x float> %v
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.l.1d.v4f32.f32(i32 immarg, float, float, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.sample.l.2d.v4f32.f32(i32 immarg, float, float, float, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.sample.c.l.1d.v4f32.f32(i32 immarg, float, float, float, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.sample.c.l.2d.v4f32.f32(i32 immarg, float, float, float, float, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.sample.l.o.1d.v4f32.f32(i32 immarg, i32, float, float, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.sample.l.o.2d.v4f32.f32(i32 immarg, i32, float, float, float, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.sample.c.l.o.1d.v4f32.f32(i32 immarg, i32, float, float, float, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.sample.c.l.o.2d.v4f32.f32(i32 immarg, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f32(i32 immarg, float, float, float, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.gather4.c.l.2d.v4f32.f32(i32 immarg, float, float, float, float, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.gather4.l.o.2d.v4f32.f32(i32 immarg, i32, float, float, float, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2d.v4f32.f32(i32 immarg, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) #0
+
+attributes #0 = { nounwind readonly }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.store.2d.d16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.store.2d.d16.ll
new file mode 100644
index 0000000000000..a192b372995b7
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.store.2d.d16.ll
@@ -0,0 +1,118 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=tonga -o - %s | FileCheck -check-prefix=UNPACKED %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx810 -o - %s | FileCheck -check-prefix=PACKED %s
+
+define amdgpu_ps void @image_store_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, half %data) {
+; UNPACKED-LABEL: image_store_f16:
+; UNPACKED:       ; %bb.0:
+; UNPACKED-NEXT:    s_mov_b32 s0, s2
+; UNPACKED-NEXT:    s_mov_b32 s1, s3
+; UNPACKED-NEXT:    s_mov_b32 s2, s4
+; UNPACKED-NEXT:    s_mov_b32 s3, s5
+; UNPACKED-NEXT:    s_mov_b32 s4, s6
+; UNPACKED-NEXT:    s_mov_b32 s5, s7
+; UNPACKED-NEXT:    s_mov_b32 s6, s8
+; UNPACKED-NEXT:    s_mov_b32 s7, s9
+; UNPACKED-NEXT:    image_store v2, v[0:1], s[0:7] dmask:0x1 unorm
+; UNPACKED-NEXT:    s_endpgm
+;
+; PACKED-LABEL: image_store_f16:
+; PACKED:       ; %bb.0:
+; PACKED-NEXT:    s_mov_b32 s0, s2
+; PACKED-NEXT:    s_mov_b32 s1, s3
+; PACKED-NEXT:    s_mov_b32 s2, s4
+; PACKED-NEXT:    s_mov_b32 s3, s5
+; PACKED-NEXT:    s_mov_b32 s4, s6
+; PACKED-NEXT:    s_mov_b32 s5, s7
+; PACKED-NEXT:    s_mov_b32 s6, s8
+; PACKED-NEXT:    s_mov_b32 s7, s9
+; PACKED-NEXT:    s_nop 0
+; PACKED-NEXT:    s_nop 0
+; PACKED-NEXT:    image_store v2, v[0:1], s[0:7] dmask:0x1 unorm
+; PACKED-NEXT:    s_endpgm
+  call void @llvm.amdgcn.image.store.2d.f16.i32(half %data, i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @image_store_v2f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, <2 x half> %in) {
+; UNPACKED-LABEL: image_store_v2f16:
+; UNPACKED:       ; %bb.0:
+; UNPACKED-NEXT:    s_mov_b32 s0, s2
+; UNPACKED-NEXT:    s_mov_b32 s1, s3
+; UNPACKED-NEXT:    s_mov_b32 s2, s4
+; UNPACKED-NEXT:    s_mov_b32 s3, s5
+; UNPACKED-NEXT:    s_mov_b32 s4, s6
+; UNPACKED-NEXT:    s_mov_b32 s5, s7
+; UNPACKED-NEXT:    s_mov_b32 s6, s8
+; UNPACKED-NEXT:    s_mov_b32 s7, s9
+; UNPACKED-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; UNPACKED-NEXT:    image_store v[2:3], v[0:1], s[0:7] dmask:0x3 unorm
+; UNPACKED-NEXT:    s_endpgm
+;
+; PACKED-LABEL: image_store_v2f16:
+; PACKED:       ; %bb.0:
+; PACKED-NEXT:    s_mov_b32 s0, s2
+; PACKED-NEXT:    s_mov_b32 s1, s3
+; PACKED-NEXT:    s_mov_b32 s2, s4
+; PACKED-NEXT:    s_mov_b32 s3, s5
+; PACKED-NEXT:    s_mov_b32 s4, s6
+; PACKED-NEXT:    s_mov_b32 s5, s7
+; PACKED-NEXT:    s_mov_b32 s6, s8
+; PACKED-NEXT:    s_mov_b32 s7, s9
+; PACKED-NEXT:    s_nop 0
+; PACKED-NEXT:    s_nop 0
+; PACKED-NEXT:    image_store v2, v[0:1], s[0:7] dmask:0x3 unorm
+; PACKED-NEXT:    s_endpgm
+  call void @llvm.amdgcn.image.store.2d.v2f16.i32(<2 x half> %in, i32 3, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
+  ret void
+}
+
+; FIXME: Broken
+; define amdgpu_ps void @image_store_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, <3 x half> %in) {
+;   call void @llvm.amdgcn.image.store.2d.v3f16.i32(<3 x half> %in, i32 7, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
+;   ret void
+; }
+
+define amdgpu_ps void @image_store_v4f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, <4 x half> %in) {
+; UNPACKED-LABEL: image_store_v4f16:
+; UNPACKED:       ; %bb.0:
+; UNPACKED-NEXT:    v_mov_b32_e32 v6, v1
+; UNPACKED-NEXT:    v_mov_b32_e32 v1, v2
+; UNPACKED-NEXT:    s_mov_b32 s0, s2
+; UNPACKED-NEXT:    s_mov_b32 s1, s3
+; UNPACKED-NEXT:    s_mov_b32 s2, s4
+; UNPACKED-NEXT:    s_mov_b32 s3, s5
+; UNPACKED-NEXT:    s_mov_b32 s4, s6
+; UNPACKED-NEXT:    s_mov_b32 s5, s7
+; UNPACKED-NEXT:    s_mov_b32 s6, s8
+; UNPACKED-NEXT:    s_mov_b32 s7, s9
+; UNPACKED-NEXT:    v_mov_b32_e32 v5, v0
+; UNPACKED-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
+; UNPACKED-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
+; UNPACKED-NEXT:    image_store v[1:4], v[5:6], s[0:7] dmask:0xf unorm
+; UNPACKED-NEXT:    s_endpgm
+;
+; PACKED-LABEL: image_store_v4f16:
+; PACKED:       ; %bb.0:
+; PACKED-NEXT:    s_mov_b32 s0, s2
+; PACKED-NEXT:    s_mov_b32 s1, s3
+; PACKED-NEXT:    s_mov_b32 s2, s4
+; PACKED-NEXT:    s_mov_b32 s3, s5
+; PACKED-NEXT:    s_mov_b32 s4, s6
+; PACKED-NEXT:    s_mov_b32 s5, s7
+; PACKED-NEXT:    s_mov_b32 s6, s8
+; PACKED-NEXT:    s_mov_b32 s7, s9
+; PACKED-NEXT:    s_nop 0
+; PACKED-NEXT:    s_nop 0
+; PACKED-NEXT:    image_store v[2:3], v[0:1], s[0:7] dmask:0xf unorm
+; PACKED-NEXT:    s_endpgm
+  call void @llvm.amdgcn.image.store.2d.v4f16.i32(<4 x half> %in, i32 15, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
+  ret void
+}
+
+declare void @llvm.amdgcn.image.store.2d.f16.i32(half, i32 immarg, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare void @llvm.amdgcn.image.store.2d.v2f16.i32(<2 x half>, i32 immarg, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare void @llvm.amdgcn.image.store.2d.v3f16.i32(<3 x half>, i32 immarg, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare void @llvm.amdgcn.image.store.2d.v4f16.i32(<4 x half>, i32 immarg, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+
+attributes #0 = { nounwind writeonly }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.store.2d.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.store.2d.ll
new file mode 100644
index 0000000000000..ed2ed403c1247
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.store.2d.ll
@@ -0,0 +1,451 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -o - %s | FileCheck -check-prefix=GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -o - %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -o - %s | FileCheck -check-prefix=GFX10 %s
+
+define amdgpu_ps void @image_store_f32(<8 x i32> inreg %rsrc, i32 %s, i32 %t, float %data) {
+; GFX6-LABEL: image_store_f32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_store v2, v[0:1], s[0:7] dmask:0x1 unorm
+; GFX6-NEXT:    s_endpgm
+;
+; GFX8-LABEL: image_store_f32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_store v2, v[0:1], s[0:7] dmask:0x1 unorm
+; GFX8-NEXT:    s_endpgm
+;
+; GFX10-LABEL: image_store_f32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_store v2, v[0:1], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D unorm
+; GFX10-NEXT:    s_endpgm
+  call void @llvm.amdgcn.image.store.2d.f32.i32(float %data, i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @image_store_v2f32(<8 x i32> inreg %rsrc, i32 %s, i32 %t, <2 x float> %in) {
+; GFX6-LABEL: image_store_v2f32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_store v[2:3], v[0:1], s[0:7] dmask:0x3 unorm
+; GFX6-NEXT:    s_endpgm
+;
+; GFX8-LABEL: image_store_v2f32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_store v[2:3], v[0:1], s[0:7] dmask:0x3 unorm
+; GFX8-NEXT:    s_endpgm
+;
+; GFX10-LABEL: image_store_v2f32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_store v[2:3], v[0:1], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D unorm
+; GFX10-NEXT:    s_endpgm
+  call void @llvm.amdgcn.image.store.2d.v2f32.i32(<2 x float> %in, i32 3, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @image_store_v3f32(<8 x i32> inreg %rsrc, i32 %s, i32 %t, <3 x float> %in) {
+; GFX6-LABEL: image_store_v3f32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_store v[2:4], v[0:1], s[0:7] dmask:0x7 unorm
+; GFX6-NEXT:    s_endpgm
+;
+; GFX8-LABEL: image_store_v3f32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_store v[2:4], v[0:1], s[0:7] dmask:0x7 unorm
+; GFX8-NEXT:    s_endpgm
+;
+; GFX10-LABEL: image_store_v3f32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_store v[2:4], v[0:1], s[0:7] dmask:0x7 dim:SQ_RSRC_IMG_2D unorm
+; GFX10-NEXT:    s_endpgm
+  call void @llvm.amdgcn.image.store.2d.v3f32.i32(<3 x float> %in, i32 7, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @image_store_v4f32(<8 x i32> inreg %rsrc, i32 %s, i32 %t, <4 x float> %in) {
+; GFX6-LABEL: image_store_v4f32:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0xf unorm
+; GFX6-NEXT:    s_endpgm
+;
+; GFX8-LABEL: image_store_v4f32:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0xf unorm
+; GFX8-NEXT:    s_endpgm
+;
+; GFX10-LABEL: image_store_v4f32:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm
+; GFX10-NEXT:    s_endpgm
+  call void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float> %in, i32 15, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @image_store_v4f32_dmask_0001(<8 x i32> inreg %rsrc, i32 %s, i32 %t, <4 x float> %in) {
+; GFX6-LABEL: image_store_v4f32_dmask_0001:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0x1 unorm
+; GFX6-NEXT:    s_endpgm
+;
+; GFX8-LABEL: image_store_v4f32_dmask_0001:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0x1 unorm
+; GFX8-NEXT:    s_endpgm
+;
+; GFX10-LABEL: image_store_v4f32_dmask_0001:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D unorm
+; GFX10-NEXT:    s_endpgm
+  call void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float> %in, i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @image_store_v4f32_dmask_0010(<8 x i32> inreg %rsrc, i32 %s, i32 %t, <4 x float> %in) {
+; GFX6-LABEL: image_store_v4f32_dmask_0010:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0x2 unorm
+; GFX6-NEXT:    s_endpgm
+;
+; GFX8-LABEL: image_store_v4f32_dmask_0010:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0x2 unorm
+; GFX8-NEXT:    s_endpgm
+;
+; GFX10-LABEL: image_store_v4f32_dmask_0010:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0x2 dim:SQ_RSRC_IMG_2D unorm
+; GFX10-NEXT:    s_endpgm
+  call void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float> %in, i32 2, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @image_store_v4f32_dmask_0100(<8 x i32> inreg %rsrc, i32 %s, i32 %t, <4 x float> %in) {
+; GFX6-LABEL: image_store_v4f32_dmask_0100:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0x4 unorm
+; GFX6-NEXT:    s_endpgm
+;
+; GFX8-LABEL: image_store_v4f32_dmask_0100:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0x4 unorm
+; GFX8-NEXT:    s_endpgm
+;
+; GFX10-LABEL: image_store_v4f32_dmask_0100:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0x4 dim:SQ_RSRC_IMG_2D unorm
+; GFX10-NEXT:    s_endpgm
+  call void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float> %in, i32 4, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @image_store_v4f32_dmask_1000(<8 x i32> inreg %rsrc, i32 %s, i32 %t, <4 x float> %in) {
+; GFX6-LABEL: image_store_v4f32_dmask_1000:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0x8 unorm
+; GFX6-NEXT:    s_endpgm
+;
+; GFX8-LABEL: image_store_v4f32_dmask_1000:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0x8 unorm
+; GFX8-NEXT:    s_endpgm
+;
+; GFX10-LABEL: image_store_v4f32_dmask_1000:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0x8 dim:SQ_RSRC_IMG_2D unorm
+; GFX10-NEXT:    s_endpgm
+  call void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float> %in, i32 8, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @image_store_v4f32_dmask_0011(<8 x i32> inreg %rsrc, i32 %s, i32 %t, <4 x float> %in) {
+; GFX6-LABEL: image_store_v4f32_dmask_0011:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0x3 unorm
+; GFX6-NEXT:    s_endpgm
+;
+; GFX8-LABEL: image_store_v4f32_dmask_0011:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0x3 unorm
+; GFX8-NEXT:    s_endpgm
+;
+; GFX10-LABEL: image_store_v4f32_dmask_0011:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D unorm
+; GFX10-NEXT:    s_endpgm
+  call void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float> %in, i32 3, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @image_store_v4f32_dmask_0110(<8 x i32> inreg %rsrc, i32 %s, i32 %t, <4 x float> %in) {
+; GFX6-LABEL: image_store_v4f32_dmask_0110:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_mov_b32 s0, s2
+; GFX6-NEXT:    s_mov_b32 s1, s3
+; GFX6-NEXT:    s_mov_b32 s2, s4
+; GFX6-NEXT:    s_mov_b32 s3, s5
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s8
+; GFX6-NEXT:    s_mov_b32 s7, s9
+; GFX6-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0x6 unorm
+; GFX6-NEXT:    s_endpgm
+;
+; GFX8-LABEL: image_store_v4f32_dmask_0110:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_mov_b32 s0, s2
+; GFX8-NEXT:    s_mov_b32 s1, s3
+; GFX8-NEXT:    s_mov_b32 s2, s4
+; GFX8-NEXT:    s_mov_b32 s3, s5
+; GFX8-NEXT:    s_mov_b32 s4, s6
+; GFX8-NEXT:    s_mov_b32 s5, s7
+; GFX8-NEXT:    s_mov_b32 s6, s8
+; GFX8-NEXT:    s_mov_b32 s7, s9
+; GFX8-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0x6 unorm
+; GFX8-NEXT:    s_endpgm
+;
+; GFX10-LABEL: image_store_v4f32_dmask_0110:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s0, s2
+; GFX10-NEXT:    s_mov_b32 s1, s3
+; GFX10-NEXT:    s_mov_b32 s2, s4
+; GFX10-NEXT:    s_mov_b32 s3, s5
+; GFX10-NEXT:    s_mov_b32 s4, s6
+; GFX10-NEXT:    s_mov_b32 s5, s7
+; GFX10-NEXT:    s_mov_b32 s6, s8
+; GFX10-NEXT:    s_mov_b32 s7, s9
+; GFX10-NEXT:    ; implicit-def: $vcc_hi
+; GFX10-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0x6 dim:SQ_RSRC_IMG_2D unorm
+; GFX10-NEXT:    s_endpgm
+  call void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float> %in, i32 6, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
+  ret void
+}
+
+declare void @llvm.amdgcn.image.store.2d.f32.i32(float, i32 immarg, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare void @llvm.amdgcn.image.store.2d.v2f32.i32(<2 x float>, i32 immarg, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare void @llvm.amdgcn.image.store.2d.v3f32.i32(<3 x float>, i32 immarg, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+declare void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float>, i32 immarg, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+
+attributes #0 = { nounwind writeonly }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fshr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fshr.mir
new file mode 100644
index 0000000000000..3a0bd156c0d08
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fshr.mir
@@ -0,0 +1,152 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+
+---
+name: fshr_sss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2
+    ; CHECK-LABEL: name: fshr_sss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[COPY2]](s32)
+    ; CHECK: [[FSHR:%[0-9]+]]:vgpr(s32) = G_FSHR [[COPY3]], [[COPY4]], [[COPY5]](s32)
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $sgpr2
+    %3:_(s32) = G_FSHR %0, %1, %2
+...
+---
+name: fshr_vss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: fshr_vss
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY2]](s32)
+    ; CHECK: [[FSHR:%[0-9]+]]:vgpr(s32) = G_FSHR [[COPY]], [[COPY3]], [[COPY4]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $sgpr0
+    %2:_(s32) = COPY $sgpr1
+    %3:_(s32) = G_FSHR %0, %1, %2
+...
+---
+name: fshr_svs
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0, $sgpr1
+    ; CHECK-LABEL: name: fshr_svs
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY2]](s32)
+    ; CHECK: [[FSHR:%[0-9]+]]:vgpr(s32) = G_FSHR [[COPY3]], [[COPY1]], [[COPY4]](s32)
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(s32) = COPY $sgpr1
+    %3:_(s32) = G_FSHR %0, %1, %2
+...
+---
+name: fshr_ssv
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $vgpr0
+    ; CHECK-LABEL: name: fshr_ssv
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[FSHR:%[0-9]+]]:vgpr(s32) = G_FSHR [[COPY3]], [[COPY4]], [[COPY2]](s32)
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $vgpr0
+    %3:_(s32) = G_FSHR %0, %1, %2
+...
+---
+name: fshr_vvs
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $sgpr0
+    ; CHECK-LABEL: name: fshr_vvs
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY2]](s32)
+    ; CHECK: [[FSHR:%[0-9]+]]:vgpr(s32) = G_FSHR [[COPY]], [[COPY1]], [[COPY3]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $sgpr0
+    %3:_(s32) = G_FSHR %0, %1, %2
+...
+---
+name: fshr_vsv
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $sgpr0, $vgpr1
+    ; CHECK-LABEL: name: fshr_vsv
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[FSHR:%[0-9]+]]:vgpr(s32) = G_FSHR [[COPY]], [[COPY3]], [[COPY2]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $vgpr1
+    %3:_(s32) = G_FSHR %0, %1, %2
+...
+---
+name: fshr_svv
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0, $vgpr1
+    ; CHECK-LABEL: name: fshr_svv
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[FSHR:%[0-9]+]]:vgpr(s32) = G_FSHR [[COPY3]], [[COPY1]], [[COPY2]](s32)
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(s32) = COPY $vgpr1
+    %3:_(s32) = G_FSHR %0, %1, %2
+...
+---
+name: fshr_vvv
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+    ; CHECK-LABEL: name: fshr_vvv
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+    ; CHECK: [[FSHR:%[0-9]+]]:vgpr(s32) = G_FSHR [[COPY]], [[COPY1]], [[COPY2]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = G_FSHR %0, %1, %2
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-load.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-load.mir
index b56a67d5336b1..f221250fc353c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-load.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-load.mir
@@ -455,7 +455,7 @@ body: |
   bb.0:
     liveins: $sgpr0_sgpr1
     ; CHECK-LABEL: name: load_constant_v8i32_uniform
-    ; CHECK (<8 x s32>) = G_LOAD %0 :: (load 32, addrspace 4)
+    ; CHECK: (<8 x s32>) = G_LOAD %0(p4) :: (load 32, addrspace 4)
     %0:_(p4) = COPY $sgpr0_sgpr1
     %1:_(<8 x s32>) = G_LOAD %0 :: (load 32, addrspace 4)
 ...
@@ -468,7 +468,7 @@ body: |
   bb.0:
     liveins: $sgpr0_sgpr1
     ; CHECK-LABEL: name: load_constant_v4i64_uniform
-    ; CHECK (<4 x s64>) = G_LOAD %0 :: (load 32, addrspace 4)
+    ; CHECK: (<4 x s64>) = G_LOAD %0(p4) :: (load 32, addrspace 4)
     %0:_(p4) = COPY $sgpr0_sgpr1
     %1:_(<4 x s64>) = G_LOAD %0 :: (load 32, addrspace 4)
 ...
@@ -481,7 +481,7 @@ body: |
   bb.0:
     liveins: $sgpr0_sgpr1
     ; CHECK-LABEL: name: load_constant_v16i32_uniform
-    ; CHECK (<16 x s32>) = G_LOAD %0 :: (load 64, addrspace 4)
+    ; CHECK: (<16 x s32>) = G_LOAD %0(p4) :: (load 64, addrspace 4)
     %0:_(p4) = COPY $sgpr0_sgpr1
     %1:_(<16 x s32>) = G_LOAD %0 :: (load 64, addrspace 4)
 ...
@@ -494,7 +494,7 @@ body: |
   bb.0:
     liveins: $sgpr0_sgpr1
     ; CHECK-LABEL: name: load_constant_v8i64_uniform
-    ; CHECK (<8 x s64>) = G_LOAD %0 :: (load 64, addrspace 4)
+    ; CHECK: (<8 x s64>) = G_LOAD %0(p4) :: (load 64, addrspace 4)
     %0:_(p4) = COPY $sgpr0_sgpr1
     %1:_(<8 x s64>) = G_LOAD %0 :: (load 64, addrspace 4)
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/smrd.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/smrd.ll
index da386d2949f45..5a720e8412c3d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/smrd.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/smrd.ll
@@ -1,12 +1,14 @@
 ; FIXME: Need to add support for mubuf stores to enable this on SI.
 ; XUN: llc < %s -march=amdgcn -mcpu=tahiti -show-mc-encoding -verify-machineinstrs -global-isel | FileCheck --check-prefix=SI --check-prefix=GCN --check-prefix=SIVI %s
-; RUN: llc < %s -march=amdgcn -mcpu=bonaire -show-mc-encoding -verify-machineinstrs -global-isel | FileCheck --check-prefix=CI --check-prefix=GCN %s
-; RUN: llc < %s -march=amdgcn -mcpu=tonga -show-mc-encoding -verify-machineinstrs -global-isel | FileCheck --check-prefix=VI --check-prefix=GCN --check-prefix=SIVI %s
+; RUN: llc < %s -march=amdgcn -mcpu=bonaire -show-mc-encoding -verify-machineinstrs -global-isel | FileCheck --check-prefix=CI --check-prefix=GCN --check-prefix=SICIVI %s
+; RUN: llc < %s -march=amdgcn -mcpu=tonga -show-mc-encoding -verify-machineinstrs -global-isel | FileCheck --check-prefix=VI --check-prefix=GCN --check-prefix=SIVI --check-prefix=SICIVI %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -show-mc-encoding -verify-machineinstrs -global-isel < %s | FileCheck -check-prefix=GFX9 -check-prefix=GFX9_10 -check-prefix=GCN -check-prefix=VIGFX9_10 -check-prefix=SIVIGFX9_10  %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -show-mc-encoding -verify-machineinstrs -global-isel < %s | FileCheck -check-prefix=GFX10 -check-prefix=GFX9_10 -check-prefix=GCN -check-prefix=VIGFX9_10 -check-prefix=SIVIGFX9_10  %s
 
 ; SMRD load with an immediate offset.
 ; GCN-LABEL: {{^}}smrd0:
 ; SICI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x1 ; encoding: [0x01
-; VI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x4
+; VIGFX9_10: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x4
 define amdgpu_kernel void @smrd0(i32 addrspace(4)* %ptr) {
 entry:
   %0 = getelementptr i32, i32 addrspace(4)* %ptr, i64 1
@@ -18,7 +20,7 @@ entry:
 ; SMRD load with the largest possible immediate offset.
 ; GCN-LABEL: {{^}}smrd1:
 ; SICI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0xff ; encoding: [0xff,0x{{[0-9]+[137]}}
-; VI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x3fc
+; VIGFX9_10: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x3fc
 define amdgpu_kernel void @smrd1(i32 addrspace(4)* %ptr) {
 entry:
   %0 = getelementptr i32, i32 addrspace(4)* %ptr, i64 255
@@ -32,7 +34,7 @@ entry:
 ; SI: s_movk_i32 s[[OFFSET:[0-9]]], 0x400
 ; SI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], s[[OFFSET]] ; encoding: [0x0[[OFFSET]]
 ; CI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x100
-; VI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x400
+; VIGFX9_10: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x400
 ; GCN: s_endpgm
 define amdgpu_kernel void @smrd2(i32 addrspace(4)* %ptr) {
 entry:
@@ -63,6 +65,8 @@ entry:
 ; SI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], [[OFFSET]]
 ; CI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x3ffff
 ; VI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0xffffc
+; GFX9_10: s_mov_b32 [[OFFSET:s[0-9]+]], 0xffffc
+; GFX9_10: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], [[OFFSET]]
 define amdgpu_kernel void @smrd4(i32 addrspace(4)* %ptr) {
 entry:
   %0 = getelementptr i32, i32 addrspace(4)* %ptr, i64 262143
@@ -73,8 +77,8 @@ entry:
 
 ; SMRD load with an offset greater than the largest possible immediate on VI
 ; GCN-LABEL: {{^}}smrd5:
-; SIVI: s_mov_b32 [[OFFSET:s[0-9]+]], 0x100000
-; SIVI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], [[OFFSET]]
+; SIVIGFX9_10: s_mov_b32 [[OFFSET:s[0-9]+]], 0x100000
+; SIVIGFX9_10: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], [[OFFSET]]
 ; CI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x40000
 ; GCN: s_endpgm
 define amdgpu_kernel void @smrd5(i32 addrspace(4)* %ptr) {
@@ -85,3 +89,28 @@ entry:
   ret void
 }
 
+; GFX9_10 can use a signed immediate byte offset
+; GCN-LABEL: {{^}}smrd6:
+; SICIVI: s_add_u32 s{{[0-9]}}, s{{[0-9]}}, -4
+; SICIVI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x0
+; GFX9_10: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0xfffffffc
+define amdgpu_kernel void @smrd6(i32 addrspace(1)* %out, i32 addrspace(4)* %ptr) #0 {
+entry:
+  %tmp = getelementptr i32, i32 addrspace(4)* %ptr, i64 -1
+  %tmp1 = load i32, i32 addrspace(4)* %tmp
+  store i32 %tmp1, i32 addrspace(1)* %out
+  ret void
+}
+
+; Don't use a negative SGPR offset
+; GCN-LABEL: {{^}}smrd7:
+; GCN: s_add_u32 s{{[0-9]}}, s{{[0-9]}}, 0xffe00000
+; SICIVI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x0
+; GFX9_10: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x0
+define amdgpu_kernel void @smrd7(i32 addrspace(1)* %out, i32 addrspace(4)* %ptr) #0 {
+entry:
+  %tmp = getelementptr i32, i32 addrspace(4)* %ptr, i64 -524288
+  %tmp1 = load i32, i32 addrspace(4)* %tmp
+  store i32 %tmp1, i32 addrspace(1)* %out
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll
new file mode 100644
index 0000000000000..51cc92525174e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll
@@ -0,0 +1,3456 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -amdgpu-codegenprepare-disable-idiv-expansion=1 -mtriple=amdgcn-amd-amdhsa -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefixes=CHECK,GISEL %s
+; RUN: llc -global-isel -amdgpu-codegenprepare-disable-idiv-expansion=0 -mtriple=amdgcn-amd-amdhsa -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefixes=CHECK,CGP %s
+
+; The same 32-bit expansion is implemented in the legalizer and in AMDGPUCodeGenPrepare.
+
+define i64 @v_udiv_i64(i64 %num, i64 %den) {
+; CHECK-LABEL: v_udiv_i64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_or_b32_e32 v5, v1, v3
+; CHECK-NEXT:    v_mov_b32_e32 v4, 0
+; CHECK-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[4:5]
+; CHECK-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, 1
+; CHECK-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; CHECK-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; CHECK-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
+; CHECK-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; CHECK-NEXT:    s_cbranch_execnz BB0_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v4, v2
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v5, v3
+; CHECK-NEXT:    v_sub_i32_e32 v6, vcc, 0, v2
+; CHECK-NEXT:    v_subb_u32_e32 v7, vcc, 0, v3, vcc
+; CHECK-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; CHECK-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; CHECK-NEXT:    v_mul_f32_e32 v5, 0x2f800000, v4
+; CHECK-NEXT:    v_trunc_f32_e32 v5, v5
+; CHECK-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v5
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; CHECK-NEXT:    v_mul_lo_u32 v8, v6, v5
+; CHECK-NEXT:    v_mul_lo_u32 v9, v6, v4
+; CHECK-NEXT:    v_mul_lo_u32 v10, v7, v4
+; CHECK-NEXT:    v_mul_hi_u32 v11, v6, v4
+; CHECK-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
+; CHECK-NEXT:    v_mul_lo_u32 v10, v5, v9
+; CHECK-NEXT:    v_mul_hi_u32 v12, v4, v9
+; CHECK-NEXT:    v_mul_hi_u32 v9, v5, v9
+; CHECK-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
+; CHECK-NEXT:    v_mul_lo_u32 v11, v4, v8
+; CHECK-NEXT:    v_mul_lo_u32 v13, v5, v8
+; CHECK-NEXT:    v_mul_hi_u32 v14, v4, v8
+; CHECK-NEXT:    v_mul_hi_u32 v8, v5, v8
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; CHECK-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v9, vcc, v13, v9
+; CHECK-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v9, vcc, v9, v14
+; CHECK-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; CHECK-NEXT:    v_add_i32_e32 v11, vcc, v13, v12
+; CHECK-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; CHECK-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v9
+; CHECK-NEXT:    v_addc_u32_e64 v9, s[4:5], v5, v8, vcc
+; CHECK-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v8
+; CHECK-NEXT:    v_mul_lo_u32 v8, v6, v4
+; CHECK-NEXT:    v_mul_lo_u32 v7, v7, v4
+; CHECK-NEXT:    v_mul_hi_u32 v10, v6, v4
+; CHECK-NEXT:    v_mul_lo_u32 v6, v6, v9
+; CHECK-NEXT:    v_mul_lo_u32 v11, v9, v8
+; CHECK-NEXT:    v_mul_hi_u32 v12, v4, v8
+; CHECK-NEXT:    v_mul_hi_u32 v8, v9, v8
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v7, v6
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v10
+; CHECK-NEXT:    v_mul_lo_u32 v7, v4, v6
+; CHECK-NEXT:    v_mul_lo_u32 v10, v9, v6
+; CHECK-NEXT:    v_mul_hi_u32 v13, v4, v6
+; CHECK-NEXT:    v_mul_hi_u32 v6, v9, v6
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v11, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v8, s[4:5], v10, v8
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v7, v12
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v13
+; CHECK-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v9, v7
+; CHECK-NEXT:    v_add_i32_e64 v9, s[4:5], v10, v11
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v8, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v8, s[4:5], v9, v8
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v8
+; CHECK-NEXT:    v_addc_u32_e32 v5, vcc, v5, v6, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
+; CHECK-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v6, v1, v4
+; CHECK-NEXT:    v_mul_hi_u32 v7, v0, v4
+; CHECK-NEXT:    v_mul_hi_u32 v4, v1, v4
+; CHECK-NEXT:    v_mul_lo_u32 v8, v0, v5
+; CHECK-NEXT:    v_mul_lo_u32 v9, v1, v5
+; CHECK-NEXT:    v_mul_hi_u32 v10, v0, v5
+; CHECK-NEXT:    v_mul_hi_u32 v5, v1, v5
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v9, v4
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v9, v7
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT:    v_mul_lo_u32 v7, v2, v4
+; CHECK-NEXT:    v_mul_lo_u32 v8, v3, v4
+; CHECK-NEXT:    v_mul_hi_u32 v9, v2, v4
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT:    v_mul_lo_u32 v6, v2, v5
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, 1, v4
+; CHECK-NEXT:    v_addc_u32_e32 v11, vcc, 0, v5, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
+; CHECK-NEXT:    v_add_i32_e32 v8, vcc, 1, v10
+; CHECK-NEXT:    v_addc_u32_e32 v12, vcc, 0, v11, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v9
+; CHECK-NEXT:    v_sub_i32_e32 v7, vcc, v0, v7
+; CHECK-NEXT:    v_subb_u32_e64 v9, s[4:5], v1, v6, vcc
+; CHECK-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v6
+; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v7, v2
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[4:5]
+; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v9, v3
+; CHECK-NEXT:    v_cndmask_b32_e64 v13, 0, -1, s[4:5]
+; CHECK-NEXT:    v_subb_u32_e32 v1, vcc, v1, v3, vcc
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, v9, v3
+; CHECK-NEXT:    v_cndmask_b32_e32 v6, v13, v6, vcc
+; CHECK-NEXT:    v_sub_i32_e32 v7, vcc, v7, v2
+; CHECK-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v7, v2
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, -1, vcc
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v3
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v9, v7, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v10, v8, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v3, v11, v12, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; CHECK-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v5, v5, v3, vcc
+; CHECK-NEXT:  BB0_2: ; %Flow
+; CHECK-NEXT:    s_or_saveexec_b64 s[6:7], s[6:7]
+; CHECK-NEXT:    s_xor_b64 exec, exec, s[6:7]
+; CHECK-NEXT:    s_cbranch_execz BB0_4
+; CHECK-NEXT:  ; %bb.3:
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v1, v2
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v1, v1
+; CHECK-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v1
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; CHECK-NEXT:    v_mul_lo_u32 v3, v1, v2
+; CHECK-NEXT:    v_mul_hi_u32 v4, v1, v2
+; CHECK-NEXT:    v_sub_i32_e32 v5, vcc, 0, v3
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; CHECK-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; CHECK-NEXT:    v_mul_hi_u32 v3, v3, v1
+; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v1, v3
+; CHECK-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v3
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
+; CHECK-NEXT:    v_mul_hi_u32 v1, v1, v0
+; CHECK-NEXT:    v_mul_lo_u32 v3, v1, v2
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, 1, v1
+; CHECK-NEXT:    v_subrev_i32_e32 v5, vcc, 1, v1
+; CHECK-NEXT:    v_sub_i32_e32 v6, vcc, v0, v3
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v3
+; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v6, v2
+; CHECK-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
+; CHECK-NEXT:    v_cndmask_b32_e64 v0, v4, v1, s[4:5]
+; CHECK-NEXT:    v_cndmask_b32_e32 v4, v5, v0, vcc
+; CHECK-NEXT:    v_mov_b32_e32 v5, 0
+; CHECK-NEXT:  BB0_4:
+; CHECK-NEXT:    s_or_b64 exec, exec, s[6:7]
+; CHECK-NEXT:    v_mov_b32_e32 v0, v4
+; CHECK-NEXT:    v_mov_b32_e32 v1, v5
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %result = udiv i64 %num, %den
+  ret i64 %result
+}
+
+; FIXME: This is a workaround for not handling uniform VGPR case.
+declare i32 @llvm.amdgcn.readfirstlane(i32)
+
+define amdgpu_ps i64 @s_udiv_i64(i64 inreg %num, i64 inreg %den) {
+; CHECK-LABEL: s_udiv_i64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_or_b64 s[4:5], s[0:1], s[2:3]
+; CHECK-NEXT:    s_mov_b32 s6, 0
+; CHECK-NEXT:    s_mov_b32 s7, -1
+; CHECK-NEXT:    s_and_b64 s[4:5], s[4:5], s[6:7]
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v0, s2
+; CHECK-NEXT:    v_cmp_ne_u64_e64 vcc, s[4:5], 0
+; CHECK-NEXT:    s_cbranch_vccnz BB1_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; CHECK-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; CHECK-NEXT:    v_mul_lo_u32 v1, v0, s2
+; CHECK-NEXT:    v_mul_hi_u32 v2, v0, s2
+; CHECK-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; CHECK-NEXT:    v_mul_hi_u32 v1, v1, v0
+; CHECK-NEXT:    v_add_i32_e64 v2, s[4:5], v0, v1
+; CHECK-NEXT:    v_sub_i32_e64 v0, s[4:5], v0, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; CHECK-NEXT:    v_mul_hi_u32 v0, v0, s0
+; CHECK-NEXT:    v_mul_lo_u32 v1, v0, s2
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
+; CHECK-NEXT:    v_subrev_i32_e32 v3, vcc, 1, v0
+; CHECK-NEXT:    v_sub_i32_e32 v4, vcc, s0, v1
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, s0, v1
+; CHECK-NEXT:    v_cmp_le_u32_e64 s[0:1], s2, v4
+; CHECK-NEXT:    s_and_b64 s[0:1], s[0:1], vcc
+; CHECK-NEXT:    v_cndmask_b32_e64 v0, v2, v0, s[0:1]
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; CHECK-NEXT:    s_branch BB1_3
+; CHECK-NEXT:  BB1_2:
+; CHECK-NEXT:    v_mov_b32_e32 v1, s3
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v2, s3
+; CHECK-NEXT:    s_sub_u32 s6, 0, s2
+; CHECK-NEXT:    s_cselect_b32 s4, 1, 0
+; CHECK-NEXT:    v_mov_b32_e32 v3, s1
+; CHECK-NEXT:    v_mac_f32_e32 v0, 0x4f800000, v2
+; CHECK-NEXT:    s_and_b32 s4, s4, 1
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; CHECK-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
+; CHECK-NEXT:    s_cmp_lg_u32 s4, 0
+; CHECK-NEXT:    s_subb_u32 s7, 0, s3
+; CHECK-NEXT:    v_mul_f32_e32 v2, 0x2f800000, v0
+; CHECK-NEXT:    v_trunc_f32_e32 v2, v2
+; CHECK-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v2
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; CHECK-NEXT:    v_mul_lo_u32 v4, s6, v2
+; CHECK-NEXT:    v_mul_lo_u32 v5, s6, v0
+; CHECK-NEXT:    v_mul_lo_u32 v6, s7, v0
+; CHECK-NEXT:    v_mul_hi_u32 v7, s6, v0
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
+; CHECK-NEXT:    v_mul_lo_u32 v6, v2, v5
+; CHECK-NEXT:    v_mul_hi_u32 v8, v0, v5
+; CHECK-NEXT:    v_mul_hi_u32 v5, v2, v5
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
+; CHECK-NEXT:    v_mul_lo_u32 v7, v0, v4
+; CHECK-NEXT:    v_mul_lo_u32 v9, v2, v4
+; CHECK-NEXT:    v_mul_hi_u32 v10, v0, v4
+; CHECK-NEXT:    v_mul_hi_u32 v4, v2, v4
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v9, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v9, v8
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v0, v5
+; CHECK-NEXT:    v_addc_u32_e64 v5, s[4:5], v2, v4, vcc
+; CHECK-NEXT:    v_add_i32_e64 v2, s[4:5], v2, v4
+; CHECK-NEXT:    v_mul_lo_u32 v4, s6, v0
+; CHECK-NEXT:    v_mul_lo_u32 v6, s7, v0
+; CHECK-NEXT:    v_mul_hi_u32 v7, s6, v0
+; CHECK-NEXT:    v_mul_lo_u32 v8, s6, v5
+; CHECK-NEXT:    v_mul_lo_u32 v9, v5, v4
+; CHECK-NEXT:    v_mul_hi_u32 v10, v0, v4
+; CHECK-NEXT:    v_mul_hi_u32 v4, v5, v4
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v8
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v7
+; CHECK-NEXT:    v_mul_lo_u32 v7, v0, v6
+; CHECK-NEXT:    v_mul_lo_u32 v8, v5, v6
+; CHECK-NEXT:    v_mul_hi_u32 v11, v0, v6
+; CHECK-NEXT:    v_mul_hi_u32 v5, v5, v6
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v9, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v8, v4
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v4, v11
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v7, v6
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v8, v9
+; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v4, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v7, v6
+; CHECK-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v6
+; CHECK-NEXT:    v_addc_u32_e32 v2, vcc, v2, v5, vcc
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; CHECK-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v4, s1, v0
+; CHECK-NEXT:    v_mul_hi_u32 v5, s0, v0
+; CHECK-NEXT:    v_mul_hi_u32 v0, s1, v0
+; CHECK-NEXT:    v_mul_lo_u32 v6, s0, v2
+; CHECK-NEXT:    v_mul_lo_u32 v7, s1, v2
+; CHECK-NEXT:    v_mul_hi_u32 v8, s0, v2
+; CHECK-NEXT:    v_mul_hi_u32 v2, s1, v2
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v7, v0
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v0, v8
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; CHECK-NEXT:    v_mul_lo_u32 v5, s2, v0
+; CHECK-NEXT:    v_mul_lo_u32 v6, s3, v0
+; CHECK-NEXT:    v_mul_hi_u32 v7, s2, v0
+; CHECK-NEXT:    v_add_i32_e32 v8, vcc, 1, v0
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, 1, v8
+; CHECK-NEXT:    v_mul_lo_u32 v2, s2, v2
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v6, v2
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v7
+; CHECK-NEXT:    v_sub_i32_e32 v5, vcc, s0, v5
+; CHECK-NEXT:    v_subb_u32_e64 v3, s[4:5], v3, v2, vcc
+; CHECK-NEXT:    v_sub_i32_e64 v2, s[0:1], s1, v2
+; CHECK-NEXT:    v_cmp_le_u32_e64 s[0:1], s2, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[0:1]
+; CHECK-NEXT:    v_cmp_le_u32_e64 s[0:1], s3, v3
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[0:1]
+; CHECK-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, s3, v3
+; CHECK-NEXT:    v_cndmask_b32_e32 v2, v7, v6, vcc
+; CHECK-NEXT:    v_subrev_i32_e32 v3, vcc, s2, v5
+; CHECK-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s2, v3
+; CHECK-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s3, v1
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, s3, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v5, v3, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v8, v4, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; CHECK-NEXT:  BB1_3:
+; CHECK-NEXT:    v_readfirstlane_b32 s0, v0
+; CHECK-NEXT:    s_mov_b32 s1, s0
+; CHECK-NEXT:    ; return to shader part epilog
+  %result = udiv i64 %num, %den
+  %cast = bitcast i64 %result to <2 x i32>
+  %elt.0 = extractelement <2 x i32> %cast, i32 0
+  %elt.1 = extractelement <2 x i32> %cast, i32 1
+  %res.0 = call i32 @llvm.amdgcn.readfirstlane(i32 %elt.0)
+  %res.1 = call i32 @llvm.amdgcn.readfirstlane(i32 %elt.1)
+  %ins.0 = insertelement <2 x i32> undef, i32 %res.0, i32 0
+  %ins.1 = insertelement <2 x i32> %ins.0, i32 %res.0, i32 1
+  %cast.back = bitcast <2 x i32> %ins.1 to i64
+  ret i64 %cast.back
+}
+
+define <2 x i64> @v_udiv_v2i64(<2 x i64> %num, <2 x i64> %den) {
+; GISEL-LABEL: v_udiv_v2i64:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v8, v4
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v9, v5
+; GISEL-NEXT:    v_mac_f32_e32 v8, 0x4f800000, v9
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v8, v8
+; GISEL-NEXT:    v_mul_f32_e32 v8, 0x5f7ffffc, v8
+; GISEL-NEXT:    v_mul_f32_e32 v9, 0x2f800000, v8
+; GISEL-NEXT:    v_trunc_f32_e32 v9, v9
+; GISEL-NEXT:    v_mac_f32_e32 v8, 0xcf800000, v9
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v8, v8
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v9, v9
+; GISEL-NEXT:    v_sub_i32_e32 v10, vcc, 0, v4
+; GISEL-NEXT:    v_subb_u32_e32 v11, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v12, v10, v8
+; GISEL-NEXT:    v_mul_lo_u32 v13, v11, v8
+; GISEL-NEXT:    v_mul_lo_u32 v14, v10, v9
+; GISEL-NEXT:    v_mul_hi_u32 v15, v10, v8
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v15
+; GISEL-NEXT:    v_mul_lo_u32 v14, v9, v12
+; GISEL-NEXT:    v_mul_lo_u32 v15, v8, v13
+; GISEL-NEXT:    v_mul_hi_u32 v16, v8, v12
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; GISEL-NEXT:    v_mul_lo_u32 v15, v9, v13
+; GISEL-NEXT:    v_mul_hi_u32 v12, v9, v12
+; GISEL-NEXT:    v_mul_hi_u32 v16, v8, v13
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v15, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v15, v16
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; GISEL-NEXT:    v_mul_hi_u32 v13, v9, v13
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
+; GISEL-NEXT:    v_addc_u32_e64 v12, s[4:5], v9, v13, vcc
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v13
+; GISEL-NEXT:    v_mul_lo_u32 v13, v10, v8
+; GISEL-NEXT:    v_mul_lo_u32 v11, v11, v8
+; GISEL-NEXT:    v_mul_lo_u32 v14, v10, v12
+; GISEL-NEXT:    v_mul_hi_u32 v10, v10, v8
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v14
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v11, v10
+; GISEL-NEXT:    v_mul_lo_u32 v11, v12, v13
+; GISEL-NEXT:    v_mul_lo_u32 v14, v8, v10
+; GISEL-NEXT:    v_mul_hi_u32 v15, v8, v13
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v14, v11
+; GISEL-NEXT:    v_mul_lo_u32 v14, v12, v10
+; GISEL-NEXT:    v_mul_hi_u32 v13, v12, v13
+; GISEL-NEXT:    v_mul_hi_u32 v15, v8, v10
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v14, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v14, s[4:5], v14, v15
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v13, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v14, v13
+; GISEL-NEXT:    v_mul_hi_u32 v10, v12, v10
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v13
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v11
+; GISEL-NEXT:    v_addc_u32_e32 v9, vcc, v9, v10, vcc
+; GISEL-NEXT:    v_addc_u32_e64 v9, vcc, 0, v9, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v10, v1, v8
+; GISEL-NEXT:    v_mul_lo_u32 v11, v0, v9
+; GISEL-NEXT:    v_mul_hi_u32 v12, v0, v8
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT:    v_mul_lo_u32 v11, v1, v9
+; GISEL-NEXT:    v_mul_hi_u32 v8, v1, v8
+; GISEL-NEXT:    v_mul_hi_u32 v12, v0, v9
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v11, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT:    v_mul_hi_u32 v9, v1, v9
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT:    v_mul_lo_u32 v10, v4, v8
+; GISEL-NEXT:    v_mul_lo_u32 v11, v5, v8
+; GISEL-NEXT:    v_mul_lo_u32 v12, v4, v9
+; GISEL-NEXT:    v_mul_hi_u32 v13, v4, v8
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v13
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v10
+; GISEL-NEXT:    v_subb_u32_e64 v10, s[4:5], v1, v11, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v11
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v10, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], v10, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, v11, v12, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e64 v0, s[4:5], v0, v4
+; GISEL-NEXT:    v_subb_u32_e32 v1, vcc, v1, v5, vcc
+; GISEL-NEXT:    v_subbrev_u32_e64 v1, vcc, 0, v1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, 1, v8
+; GISEL-NEXT:    v_addc_u32_e32 v12, vcc, 0, v9, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v5
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v13, v0, vcc
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, 1, v11
+; GISEL-NEXT:    v_addc_u32_e32 v4, vcc, 0, v12, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v11, v1, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v12, v4, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, v6
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v5, v7
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; GISEL-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; GISEL-NEXT:    v_mul_f32_e32 v5, 0x2f800000, v4
+; GISEL-NEXT:    v_trunc_f32_e32 v5, v5
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v5
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; GISEL-NEXT:    v_sub_i32_e32 v8, vcc, 0, v6
+; GISEL-NEXT:    v_subb_u32_e32 v9, vcc, 0, v7, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v10, v8, v4
+; GISEL-NEXT:    v_mul_lo_u32 v11, v9, v4
+; GISEL-NEXT:    v_mul_lo_u32 v12, v8, v5
+; GISEL-NEXT:    v_mul_hi_u32 v13, v8, v4
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v13
+; GISEL-NEXT:    v_mul_lo_u32 v12, v5, v10
+; GISEL-NEXT:    v_mul_lo_u32 v13, v4, v11
+; GISEL-NEXT:    v_mul_hi_u32 v14, v4, v10
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; GISEL-NEXT:    v_mul_lo_u32 v13, v5, v11
+; GISEL-NEXT:    v_mul_hi_u32 v10, v5, v10
+; GISEL-NEXT:    v_mul_hi_u32 v14, v4, v11
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v13, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; GISEL-NEXT:    v_mul_hi_u32 v11, v5, v11
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v10
+; GISEL-NEXT:    v_addc_u32_e64 v10, s[4:5], v5, v11, vcc
+; GISEL-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v11
+; GISEL-NEXT:    v_mul_lo_u32 v11, v8, v4
+; GISEL-NEXT:    v_mul_lo_u32 v9, v9, v4
+; GISEL-NEXT:    v_mul_lo_u32 v12, v8, v10
+; GISEL-NEXT:    v_mul_hi_u32 v8, v8, v4
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v12
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v9, v8
+; GISEL-NEXT:    v_mul_lo_u32 v9, v10, v11
+; GISEL-NEXT:    v_mul_lo_u32 v12, v4, v8
+; GISEL-NEXT:    v_mul_hi_u32 v13, v4, v11
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v12, v9
+; GISEL-NEXT:    v_mul_lo_u32 v12, v10, v8
+; GISEL-NEXT:    v_mul_hi_u32 v11, v10, v11
+; GISEL-NEXT:    v_mul_hi_u32 v13, v4, v8
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v12, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v13
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v11, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v12, v11
+; GISEL-NEXT:    v_mul_hi_u32 v8, v10, v8
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v11
+; GISEL-NEXT:    v_add_i32_e64 v4, s[4:5], v4, v9
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, v5, v8, vcc
+; GISEL-NEXT:    v_addc_u32_e64 v5, vcc, 0, v5, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v8, v3, v4
+; GISEL-NEXT:    v_mul_lo_u32 v9, v2, v5
+; GISEL-NEXT:    v_mul_hi_u32 v10, v2, v4
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT:    v_mul_lo_u32 v9, v3, v5
+; GISEL-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GISEL-NEXT:    v_mul_hi_u32 v10, v2, v5
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v9, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT:    v_mul_hi_u32 v5, v3, v5
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v8
+; GISEL-NEXT:    v_mul_lo_u32 v8, v6, v4
+; GISEL-NEXT:    v_mul_lo_u32 v9, v7, v4
+; GISEL-NEXT:    v_mul_lo_u32 v10, v6, v5
+; GISEL-NEXT:    v_mul_hi_u32 v11, v6, v4
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v8
+; GISEL-NEXT:    v_subb_u32_e64 v8, s[4:5], v3, v9, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v9
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v8, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v2, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], v8, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, v9, v10, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e64 v2, s[4:5], v2, v6
+; GISEL-NEXT:    v_subb_u32_e32 v3, vcc, v3, v7, vcc
+; GISEL-NEXT:    v_subbrev_u32_e64 v3, vcc, 0, v3, s[4:5]
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, 1, v4
+; GISEL-NEXT:    v_addc_u32_e32 v10, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v7
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, 1, v9
+; GISEL-NEXT:    v_addc_u32_e32 v6, vcc, 0, v10, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v9, v3, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v10, v6, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_udiv_v2i64:
+; CGP:       ; %bb.0:
+; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT:    v_mov_b32_e32 v8, v0
+; CGP-NEXT:    v_mov_b32_e32 v9, v1
+; CGP-NEXT:    v_or_b32_e32 v1, v9, v5
+; CGP-NEXT:    v_mov_b32_e32 v0, 0
+; CGP-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
+; CGP-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, 1
+; CGP-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; CGP-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; CGP-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
+; CGP-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; CGP-NEXT:    s_cbranch_execnz BB2_2
+; CGP-NEXT:  ; %bb.1:
+; CGP-NEXT:    v_cvt_f32_u32_e32 v0, v4
+; CGP-NEXT:    v_cvt_f32_u32_e32 v1, v5
+; CGP-NEXT:    v_sub_i32_e32 v10, vcc, 0, v4
+; CGP-NEXT:    v_subb_u32_e32 v11, vcc, 0, v5, vcc
+; CGP-NEXT:    v_mac_f32_e32 v0, 0x4f800000, v1
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; CGP-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
+; CGP-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
+; CGP-NEXT:    v_trunc_f32_e32 v1, v1
+; CGP-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
+; CGP-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; CGP-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; CGP-NEXT:    v_mul_lo_u32 v12, v10, v1
+; CGP-NEXT:    v_mul_lo_u32 v13, v10, v0
+; CGP-NEXT:    v_mul_lo_u32 v14, v11, v0
+; CGP-NEXT:    v_mul_hi_u32 v15, v10, v0
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v14, v12
+; CGP-NEXT:    v_mul_lo_u32 v14, v1, v13
+; CGP-NEXT:    v_mul_hi_u32 v16, v0, v13
+; CGP-NEXT:    v_mul_hi_u32 v13, v1, v13
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v12, v15
+; CGP-NEXT:    v_mul_lo_u32 v15, v0, v12
+; CGP-NEXT:    v_mul_lo_u32 v17, v1, v12
+; CGP-NEXT:    v_mul_hi_u32 v18, v0, v12
+; CGP-NEXT:    v_mul_hi_u32 v12, v1, v12
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v17, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v13, v18
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, v17, v16
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v12, v14
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v13
+; CGP-NEXT:    v_addc_u32_e64 v13, s[4:5], v1, v12, vcc
+; CGP-NEXT:    v_add_i32_e64 v1, s[4:5], v1, v12
+; CGP-NEXT:    v_mul_lo_u32 v12, v10, v0
+; CGP-NEXT:    v_mul_lo_u32 v11, v11, v0
+; CGP-NEXT:    v_mul_hi_u32 v14, v10, v0
+; CGP-NEXT:    v_mul_lo_u32 v10, v10, v13
+; CGP-NEXT:    v_mul_lo_u32 v15, v13, v12
+; CGP-NEXT:    v_mul_hi_u32 v16, v0, v12
+; CGP-NEXT:    v_mul_hi_u32 v12, v13, v12
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v11, v10
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v14
+; CGP-NEXT:    v_mul_lo_u32 v11, v0, v10
+; CGP-NEXT:    v_mul_lo_u32 v14, v13, v10
+; CGP-NEXT:    v_mul_hi_u32 v17, v0, v10
+; CGP-NEXT:    v_mul_hi_u32 v10, v13, v10
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v15, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v14, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v17
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v13, v11
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v14, v15
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v12, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v13, v12
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v12
+; CGP-NEXT:    v_addc_u32_e32 v1, vcc, v1, v10, vcc
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v11
+; CGP-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v10, v9, v0
+; CGP-NEXT:    v_mul_hi_u32 v11, v8, v0
+; CGP-NEXT:    v_mul_hi_u32 v0, v9, v0
+; CGP-NEXT:    v_mul_lo_u32 v12, v8, v1
+; CGP-NEXT:    v_mul_lo_u32 v13, v9, v1
+; CGP-NEXT:    v_mul_hi_u32 v14, v8, v1
+; CGP-NEXT:    v_mul_hi_u32 v1, v9, v1
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v13, v0
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v13, v11
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; CGP-NEXT:    v_mul_lo_u32 v11, v4, v0
+; CGP-NEXT:    v_mul_lo_u32 v12, v5, v0
+; CGP-NEXT:    v_mul_hi_u32 v13, v4, v0
+; CGP-NEXT:    v_add_i32_e32 v1, vcc, v1, v10
+; CGP-NEXT:    v_mul_lo_u32 v10, v4, v1
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, 1, v0
+; CGP-NEXT:    v_addc_u32_e32 v15, vcc, 0, v1, vcc
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, 1, v14
+; CGP-NEXT:    v_addc_u32_e32 v16, vcc, 0, v15, vcc
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v13
+; CGP-NEXT:    v_sub_i32_e32 v11, vcc, v8, v11
+; CGP-NEXT:    v_subb_u32_e64 v13, s[4:5], v9, v10, vcc
+; CGP-NEXT:    v_sub_i32_e64 v9, s[4:5], v9, v10
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v11, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v13, v5
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, -1, s[4:5]
+; CGP-NEXT:    v_subb_u32_e32 v9, vcc, v9, v5, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, v13, v5
+; CGP-NEXT:    v_cndmask_b32_e32 v10, v17, v10, vcc
+; CGP-NEXT:    v_sub_i32_e32 v11, vcc, v11, v4
+; CGP-NEXT:    v_subbrev_u32_e32 v9, vcc, 0, v9, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v11, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, -1, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v9, v5
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, -1, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, v9, v5
+; CGP-NEXT:    v_cndmask_b32_e32 v5, v13, v11, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; CGP-NEXT:    v_cndmask_b32_e32 v5, v14, v12, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v9, v15, v16, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
+; CGP-NEXT:  BB2_2: ; %Flow1
+; CGP-NEXT:    s_or_saveexec_b64 s[6:7], s[6:7]
+; CGP-NEXT:    s_xor_b64 exec, exec, s[6:7]
+; CGP-NEXT:    s_cbranch_execz BB2_4
+; CGP-NEXT:  ; %bb.3:
+; CGP-NEXT:    v_cvt_f32_u32_e32 v0, v4
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; CGP-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; CGP-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; CGP-NEXT:    v_mul_lo_u32 v1, v0, v4
+; CGP-NEXT:    v_mul_hi_u32 v5, v0, v4
+; CGP-NEXT:    v_sub_i32_e32 v9, vcc, 0, v1
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
+; CGP-NEXT:    v_mul_hi_u32 v1, v1, v0
+; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v0, v1
+; CGP-NEXT:    v_sub_i32_e64 v0, s[4:5], v0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; CGP-NEXT:    v_mul_hi_u32 v0, v0, v8
+; CGP-NEXT:    v_mul_lo_u32 v1, v0, v4
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, 1, v0
+; CGP-NEXT:    v_subrev_i32_e32 v9, vcc, 1, v0
+; CGP-NEXT:    v_sub_i32_e32 v10, vcc, v8, v1
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v8, v1
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v10, v4
+; CGP-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v0, v5, v0, s[4:5]
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v9, v0, vcc
+; CGP-NEXT:    v_mov_b32_e32 v1, 0
+; CGP-NEXT:  BB2_4:
+; CGP-NEXT:    s_or_b64 exec, exec, s[6:7]
+; CGP-NEXT:    v_or_b32_e32 v5, v3, v7
+; CGP-NEXT:    v_mov_b32_e32 v4, 0
+; CGP-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[4:5]
+; CGP-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, 1
+; CGP-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; CGP-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; CGP-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
+; CGP-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; CGP-NEXT:    s_cbranch_execnz BB2_6
+; CGP-NEXT:  ; %bb.5:
+; CGP-NEXT:    v_cvt_f32_u32_e32 v4, v6
+; CGP-NEXT:    v_cvt_f32_u32_e32 v5, v7
+; CGP-NEXT:    v_sub_i32_e32 v8, vcc, 0, v6
+; CGP-NEXT:    v_subb_u32_e32 v9, vcc, 0, v7, vcc
+; CGP-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; CGP-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; CGP-NEXT:    v_mul_f32_e32 v5, 0x2f800000, v4
+; CGP-NEXT:    v_trunc_f32_e32 v5, v5
+; CGP-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v5
+; CGP-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; CGP-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; CGP-NEXT:    v_mul_lo_u32 v10, v8, v5
+; CGP-NEXT:    v_mul_lo_u32 v11, v8, v4
+; CGP-NEXT:    v_mul_lo_u32 v12, v9, v4
+; CGP-NEXT:    v_mul_hi_u32 v13, v8, v4
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; CGP-NEXT:    v_mul_lo_u32 v12, v5, v11
+; CGP-NEXT:    v_mul_hi_u32 v14, v4, v11
+; CGP-NEXT:    v_mul_hi_u32 v11, v5, v11
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v13
+; CGP-NEXT:    v_mul_lo_u32 v13, v4, v10
+; CGP-NEXT:    v_mul_lo_u32 v15, v5, v10
+; CGP-NEXT:    v_mul_hi_u32 v16, v4, v10
+; CGP-NEXT:    v_mul_hi_u32 v10, v5, v10
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v15, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v12, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v11, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v15, v14
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v11
+; CGP-NEXT:    v_addc_u32_e64 v11, s[4:5], v5, v10, vcc
+; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v10
+; CGP-NEXT:    v_mul_lo_u32 v10, v8, v4
+; CGP-NEXT:    v_mul_lo_u32 v9, v9, v4
+; CGP-NEXT:    v_mul_hi_u32 v12, v8, v4
+; CGP-NEXT:    v_mul_lo_u32 v8, v8, v11
+; CGP-NEXT:    v_mul_lo_u32 v13, v11, v10
+; CGP-NEXT:    v_mul_hi_u32 v14, v4, v10
+; CGP-NEXT:    v_mul_hi_u32 v10, v11, v10
+; CGP-NEXT:    v_add_i32_e64 v8, s[4:5], v9, v8
+; CGP-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v12
+; CGP-NEXT:    v_mul_lo_u32 v9, v4, v8
+; CGP-NEXT:    v_mul_lo_u32 v12, v11, v8
+; CGP-NEXT:    v_mul_hi_u32 v15, v4, v8
+; CGP-NEXT:    v_mul_hi_u32 v8, v11, v8
+; CGP-NEXT:    v_add_i32_e64 v9, s[4:5], v13, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v12, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v9, s[4:5], v11, v9
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v12, v13
+; CGP-NEXT:    v_add_i32_e64 v9, s[4:5], v10, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v11, v10
+; CGP-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v10
+; CGP-NEXT:    v_addc_u32_e32 v5, vcc, v5, v8, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v9
+; CGP-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; CGP-NEXT:    v_mul_lo_u32 v8, v3, v4
+; CGP-NEXT:    v_mul_hi_u32 v9, v2, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT:    v_mul_lo_u32 v10, v2, v5
+; CGP-NEXT:    v_mul_lo_u32 v11, v3, v5
+; CGP-NEXT:    v_mul_hi_u32 v12, v2, v5
+; CGP-NEXT:    v_mul_hi_u32 v5, v3, v5
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v11, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; CGP-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; CGP-NEXT:    v_mul_lo_u32 v9, v6, v4
+; CGP-NEXT:    v_mul_lo_u32 v10, v7, v4
+; CGP-NEXT:    v_mul_hi_u32 v11, v6, v4
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v8
+; CGP-NEXT:    v_mul_lo_u32 v8, v6, v5
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, 1, v4
+; CGP-NEXT:    v_addc_u32_e32 v13, vcc, 0, v5, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, 1, v12
+; CGP-NEXT:    v_addc_u32_e32 v14, vcc, 0, v13, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
+; CGP-NEXT:    v_sub_i32_e32 v9, vcc, v2, v9
+; CGP-NEXT:    v_subb_u32_e64 v11, s[4:5], v3, v8, vcc
+; CGP-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v8
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v9, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v11, v7
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, -1, s[4:5]
+; CGP-NEXT:    v_subb_u32_e32 v3, vcc, v3, v7, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, v11, v7
+; CGP-NEXT:    v_cndmask_b32_e32 v8, v15, v8, vcc
+; CGP-NEXT:    v_sub_i32_e32 v9, vcc, v9, v6
+; CGP-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v9, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, -1, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v7
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, -1, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v7
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v11, v9, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v12, v10, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v7, v13, v14, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v4, v3, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
+; CGP-NEXT:  BB2_6: ; %Flow
+; CGP-NEXT:    s_or_saveexec_b64 s[6:7], s[6:7]
+; CGP-NEXT:    s_xor_b64 exec, exec, s[6:7]
+; CGP-NEXT:    s_cbranch_execz BB2_8
+; CGP-NEXT:  ; %bb.7:
+; CGP-NEXT:    v_cvt_f32_u32_e32 v3, v6
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; CGP-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v3
+; CGP-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; CGP-NEXT:    v_mul_lo_u32 v4, v3, v6
+; CGP-NEXT:    v_mul_hi_u32 v5, v3, v6
+; CGP-NEXT:    v_sub_i32_e32 v7, vcc, 0, v4
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
+; CGP-NEXT:    v_mul_hi_u32 v4, v4, v3
+; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v3, v4
+; CGP-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v4
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; CGP-NEXT:    v_mul_hi_u32 v3, v3, v2
+; CGP-NEXT:    v_mul_lo_u32 v4, v3, v6
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, 1, v3
+; CGP-NEXT:    v_subrev_i32_e32 v7, vcc, 1, v3
+; CGP-NEXT:    v_sub_i32_e32 v8, vcc, v2, v4
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v4
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v8, v6
+; CGP-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v2, v5, v3, s[4:5]
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v7, v2, vcc
+; CGP-NEXT:    v_mov_b32_e32 v5, 0
+; CGP-NEXT:  BB2_8:
+; CGP-NEXT:    s_or_b64 exec, exec, s[6:7]
+; CGP-NEXT:    v_mov_b32_e32 v2, v4
+; CGP-NEXT:    v_mov_b32_e32 v3, v5
+; CGP-NEXT:    s_setpc_b64 s[30:31]
+  %result = udiv <2 x i64> %num, %den
+  ret <2 x i64> %result
+}
+
+define i64 @v_udiv_i64_pow2k_denom(i64 %num) {
+; CHECK-LABEL: v_udiv_i64_pow2k_denom:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_movk_i32 s6, 0x1000
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v2, 0
+; CHECK-NEXT:    s_mov_b32 s7, 0xfffff000
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v3, s6
+; CHECK-NEXT:    v_mac_f32_e32 v3, 0x4f800000, v2
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v2, v3
+; CHECK-NEXT:    v_mul_f32_e32 v2, 0x5f7ffffc, v2
+; CHECK-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v2
+; CHECK-NEXT:    v_trunc_f32_e32 v3, v3
+; CHECK-NEXT:    v_mac_f32_e32 v2, 0xcf800000, v3
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; CHECK-NEXT:    v_mul_lo_u32 v4, s7, v3
+; CHECK-NEXT:    v_mul_lo_u32 v5, s7, v2
+; CHECK-NEXT:    v_mul_lo_u32 v6, -1, v2
+; CHECK-NEXT:    v_mul_hi_u32 v7, s7, v2
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
+; CHECK-NEXT:    v_mul_lo_u32 v6, v3, v5
+; CHECK-NEXT:    v_mul_hi_u32 v8, v2, v5
+; CHECK-NEXT:    v_mul_hi_u32 v5, v3, v5
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
+; CHECK-NEXT:    v_mul_lo_u32 v7, v2, v4
+; CHECK-NEXT:    v_mul_lo_u32 v9, v3, v4
+; CHECK-NEXT:    v_mul_hi_u32 v10, v2, v4
+; CHECK-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v9, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v9, v8
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; CHECK-NEXT:    v_addc_u32_e64 v5, s[4:5], v3, v4, vcc
+; CHECK-NEXT:    v_add_i32_e64 v3, s[4:5], v3, v4
+; CHECK-NEXT:    v_mul_lo_u32 v4, s7, v2
+; CHECK-NEXT:    v_mul_lo_u32 v6, -1, v2
+; CHECK-NEXT:    v_mul_hi_u32 v7, s7, v2
+; CHECK-NEXT:    v_mul_lo_u32 v8, s7, v5
+; CHECK-NEXT:    v_mul_lo_u32 v9, v5, v4
+; CHECK-NEXT:    v_mul_hi_u32 v10, v2, v4
+; CHECK-NEXT:    v_mul_hi_u32 v4, v5, v4
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v8
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v7
+; CHECK-NEXT:    v_mul_lo_u32 v7, v2, v6
+; CHECK-NEXT:    v_mul_lo_u32 v8, v5, v6
+; CHECK-NEXT:    v_mul_hi_u32 v11, v2, v6
+; CHECK-NEXT:    v_mul_hi_u32 v5, v5, v6
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v9, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v8, v4
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v4, v11
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v7, v6
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v8, v9
+; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v4, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v7, v6
+; CHECK-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v6
+; CHECK-NEXT:    v_addc_u32_e32 v3, vcc, v3, v5, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; CHECK-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v4, v1, v2
+; CHECK-NEXT:    v_mul_hi_u32 v5, v0, v2
+; CHECK-NEXT:    v_mul_hi_u32 v2, v1, v2
+; CHECK-NEXT:    v_mul_lo_u32 v6, v0, v3
+; CHECK-NEXT:    v_mul_lo_u32 v7, v1, v3
+; CHECK-NEXT:    v_mul_hi_u32 v8, v0, v3
+; CHECK-NEXT:    v_mul_hi_u32 v3, v1, v3
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v8
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; CHECK-NEXT:    v_mul_lo_u32 v5, s6, v2
+; CHECK-NEXT:    v_mul_lo_u32 v6, 0, v2
+; CHECK-NEXT:    v_mul_hi_u32 v7, s6, v2
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; CHECK-NEXT:    v_mul_lo_u32 v4, s6, v3
+; CHECK-NEXT:    v_add_i32_e32 v8, vcc, 1, v2
+; CHECK-NEXT:    v_addc_u32_e32 v9, vcc, 0, v3, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, 1, v8
+; CHECK-NEXT:    v_addc_u32_e32 v10, vcc, 0, v9, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v5
+; CHECK-NEXT:    v_subb_u32_e64 v5, s[4:5], v1, v4, vcc
+; CHECK-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v4
+; CHECK-NEXT:    v_cmp_le_u32_e64 s[4:5], s6, v0
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[4:5]
+; CHECK-NEXT:    v_cmp_le_u32_e64 s[4:5], 0, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[4:5]
+; CHECK-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; CHECK-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
+; CHECK-NEXT:    v_subrev_i32_e32 v0, vcc, s6, v0
+; CHECK-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s6, v0
+; CHECK-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, 0, v1
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v8, v6, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v9, v10, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %result = udiv i64 %num, 4096
+  ret i64 %result
+}
+
+define <2 x i64> @v_udiv_v2i64_pow2k_denom(<2 x i64> %num) {
+; GISEL-LABEL: v_udiv_v2i64_pow2k_denom:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    s_movk_i32 s10, 0x1000
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, s10
+; GISEL-NEXT:    s_sub_u32 s8, 0, s10
+; GISEL-NEXT:    s_cselect_b32 s4, 1, 0
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v5, 0
+; GISEL-NEXT:    v_mov_b32_e32 v6, v4
+; GISEL-NEXT:    s_and_b32 s4, s4, 1
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
+; GISEL-NEXT:    v_mac_f32_e32 v6, 0x4f800000, v5
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v5, v6
+; GISEL-NEXT:    s_cmp_lg_u32 s4, 0
+; GISEL-NEXT:    s_subb_u32 s9, 0, 0
+; GISEL-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; GISEL-NEXT:    v_mul_f32_e32 v5, 0x5f7ffffc, v5
+; GISEL-NEXT:    v_mul_f32_e32 v6, 0x2f800000, v4
+; GISEL-NEXT:    s_sub_u32 s11, 0, s10
+; GISEL-NEXT:    s_cselect_b32 s4, 1, 0
+; GISEL-NEXT:    v_mul_f32_e32 v7, 0x2f800000, v5
+; GISEL-NEXT:    v_trunc_f32_e32 v6, v6
+; GISEL-NEXT:    s_and_b32 s4, s4, 1
+; GISEL-NEXT:    v_trunc_f32_e32 v7, v7
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v6
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; GISEL-NEXT:    v_mac_f32_e32 v5, 0xcf800000, v7
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; GISEL-NEXT:    s_cmp_lg_u32 s4, 0
+; GISEL-NEXT:    s_subb_u32 s6, 0, 0
+; GISEL-NEXT:    v_mul_lo_u32 v8, s11, v6
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; GISEL-NEXT:    v_mul_lo_u32 v9, s8, v7
+; GISEL-NEXT:    v_mul_lo_u32 v10, s11, v4
+; GISEL-NEXT:    v_mul_lo_u32 v11, s6, v4
+; GISEL-NEXT:    v_mul_hi_u32 v12, s11, v4
+; GISEL-NEXT:    v_mul_lo_u32 v13, s8, v5
+; GISEL-NEXT:    v_mul_lo_u32 v14, s9, v5
+; GISEL-NEXT:    v_mul_hi_u32 v15, s8, v5
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v11, v8
+; GISEL-NEXT:    v_mul_lo_u32 v11, v6, v10
+; GISEL-NEXT:    v_mul_hi_u32 v16, v4, v10
+; GISEL-NEXT:    v_mul_hi_u32 v10, v6, v10
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v14, v9
+; GISEL-NEXT:    v_mul_lo_u32 v14, v7, v13
+; GISEL-NEXT:    v_mul_hi_u32 v17, v5, v13
+; GISEL-NEXT:    v_mul_hi_u32 v13, v7, v13
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v15
+; GISEL-NEXT:    v_mul_lo_u32 v12, v4, v8
+; GISEL-NEXT:    v_mul_lo_u32 v15, v6, v8
+; GISEL-NEXT:    v_mul_hi_u32 v18, v4, v8
+; GISEL-NEXT:    v_mul_hi_u32 v8, v6, v8
+; GISEL-NEXT:    v_mul_lo_u32 v19, v5, v9
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v19
+; GISEL-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v17
+; GISEL-NEXT:    v_mul_lo_u32 v14, v7, v9
+; GISEL-NEXT:    v_mul_hi_u32 v17, v5, v9
+; GISEL-NEXT:    v_mul_hi_u32 v9, v7, v9
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v15, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v14, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v18
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[4:5]
+; GISEL-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v17
+; GISEL-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v15, v16
+; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v19, v18
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v17
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v14, v15
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v12
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v10
+; GISEL-NEXT:    v_addc_u32_e64 v10, s[4:5], v6, v8, vcc
+; GISEL-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v8
+; GISEL-NEXT:    v_mul_lo_u32 v8, s11, v4
+; GISEL-NEXT:    v_mul_lo_u32 v11, s6, v4
+; GISEL-NEXT:    v_mul_hi_u32 v12, s11, v4
+; GISEL-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v13
+; GISEL-NEXT:    v_addc_u32_e64 v13, s[6:7], v7, v9, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v7, s[6:7], v7, v9
+; GISEL-NEXT:    v_mul_lo_u32 v9, s8, v5
+; GISEL-NEXT:    v_mul_lo_u32 v14, s9, v5
+; GISEL-NEXT:    v_mul_hi_u32 v15, s8, v5
+; GISEL-NEXT:    v_mul_lo_u32 v16, s11, v10
+; GISEL-NEXT:    v_mul_lo_u32 v17, v10, v8
+; GISEL-NEXT:    v_mul_hi_u32 v18, v4, v8
+; GISEL-NEXT:    v_mul_hi_u32 v8, v10, v8
+; GISEL-NEXT:    v_mul_lo_u32 v19, s8, v13
+; GISEL-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v16
+; GISEL-NEXT:    v_mul_lo_u32 v16, v13, v9
+; GISEL-NEXT:    v_add_i32_e64 v14, s[6:7], v14, v19
+; GISEL-NEXT:    v_mul_hi_u32 v19, v5, v9
+; GISEL-NEXT:    v_mul_hi_u32 v9, v13, v9
+; GISEL-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v12
+; GISEL-NEXT:    v_add_i32_e64 v12, s[6:7], v14, v15
+; GISEL-NEXT:    v_mul_lo_u32 v14, v4, v11
+; GISEL-NEXT:    v_mul_lo_u32 v15, v5, v12
+; GISEL-NEXT:    v_add_i32_e64 v15, s[6:7], v16, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[6:7]
+; GISEL-NEXT:    v_add_i32_e64 v15, s[6:7], v15, v19
+; GISEL-NEXT:    v_mul_lo_u32 v15, v10, v11
+; GISEL-NEXT:    v_mul_hi_u32 v19, v4, v11
+; GISEL-NEXT:    v_mul_hi_u32 v10, v10, v11
+; GISEL-NEXT:    v_mul_lo_u32 v11, v13, v12
+; GISEL-NEXT:    v_mul_hi_u32 v13, v13, v12
+; GISEL-NEXT:    v_mul_hi_u32 v12, v5, v12
+; GISEL-NEXT:    v_add_i32_e64 v14, s[8:9], v17, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v17, 0, 1, s[8:9]
+; GISEL-NEXT:    v_add_i32_e64 v8, s[8:9], v15, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[8:9]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[8:9], v11, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[8:9]
+; GISEL-NEXT:    v_add_i32_e64 v14, s[8:9], v14, v18
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[8:9]
+; GISEL-NEXT:    v_add_i32_e64 v8, s[8:9], v8, v19
+; GISEL-NEXT:    v_cndmask_b32_e64 v18, 0, 1, s[8:9]
+; GISEL-NEXT:    v_cndmask_b32_e64 v19, 0, 1, s[6:7]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[6:7], v9, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[6:7]
+; GISEL-NEXT:    v_add_i32_e64 v14, s[6:7], v17, v14
+; GISEL-NEXT:    v_add_i32_e64 v15, s[6:7], v15, v18
+; GISEL-NEXT:    v_add_i32_e64 v16, s[6:7], v16, v19
+; GISEL-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v12
+; GISEL-NEXT:    v_add_i32_e64 v8, s[6:7], v8, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[6:7]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[6:7], v9, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[6:7]
+; GISEL-NEXT:    v_add_i32_e64 v12, s[6:7], v15, v12
+; GISEL-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v14
+; GISEL-NEXT:    v_add_i32_e64 v10, s[6:7], v10, v12
+; GISEL-NEXT:    v_add_i32_e64 v11, s[6:7], v13, v11
+; GISEL-NEXT:    v_addc_u32_e32 v6, vcc, v6, v10, vcc
+; GISEL-NEXT:    v_addc_u32_e64 v7, vcc, v7, v11, s[4:5]
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v8, v3, v4
+; GISEL-NEXT:    v_mul_hi_u32 v10, v2, v4
+; GISEL-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
+; GISEL-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v9, v1, v5
+; GISEL-NEXT:    v_mul_hi_u32 v11, v0, v5
+; GISEL-NEXT:    v_mul_hi_u32 v5, v1, v5
+; GISEL-NEXT:    v_mul_lo_u32 v12, v2, v6
+; GISEL-NEXT:    v_mul_lo_u32 v13, v3, v6
+; GISEL-NEXT:    v_mul_hi_u32 v14, v2, v6
+; GISEL-NEXT:    v_mul_hi_u32 v6, v3, v6
+; GISEL-NEXT:    v_mul_lo_u32 v15, v0, v7
+; GISEL-NEXT:    v_mul_lo_u32 v16, v1, v7
+; GISEL-NEXT:    v_mul_hi_u32 v17, v0, v7
+; GISEL-NEXT:    v_mul_hi_u32 v7, v1, v7
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v13, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v16, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v17
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v12, v8
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v13, v10
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v15, v9
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v16, v11
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
+; GISEL-NEXT:    v_mul_lo_u32 v10, s10, v4
+; GISEL-NEXT:    v_mul_lo_u32 v12, 0, v4
+; GISEL-NEXT:    v_mul_hi_u32 v13, s10, v4
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
+; GISEL-NEXT:    v_mul_lo_u32 v11, s10, v5
+; GISEL-NEXT:    v_mul_lo_u32 v14, 0, v5
+; GISEL-NEXT:    v_mul_hi_u32 v15, s10, v5
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
+; GISEL-NEXT:    v_mul_lo_u32 v8, s10, v6
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, 1, v4
+; GISEL-NEXT:    v_addc_u32_e32 v16, vcc, 0, v6, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v17, s10, v7
+; GISEL-NEXT:    v_add_i32_e32 v18, vcc, 1, v5
+; GISEL-NEXT:    v_addc_u32_e32 v19, vcc, 0, v7, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v12, v8
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v14, v17
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, 1, v9
+; GISEL-NEXT:    v_addc_u32_e32 v17, vcc, 0, v16, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v13
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v15
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, 1, v18
+; GISEL-NEXT:    v_addc_u32_e32 v15, vcc, 0, v19, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v10
+; GISEL-NEXT:    v_subb_u32_e64 v10, s[4:5], v3, v8, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v8
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s10, v2
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e64 v0, s[4:5], v0, v11
+; GISEL-NEXT:    v_subb_u32_e64 v11, s[6:7], v1, v12, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e64 v1, s[6:7], v1, v12
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[6:7], 0, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, -1, s[6:7]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[6:7], 0, v10
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[8:9], s10, v0
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[8:9]
+; GISEL-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, v12, v8, s[6:7]
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, 0, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, -1, vcc
+; GISEL-NEXT:    v_subbrev_u32_e64 v1, vcc, 0, v1, s[4:5]
+; GISEL-NEXT:    v_subrev_i32_e32 v2, vcc, s10, v2
+; GISEL-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s10, v2
+; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v11
+; GISEL-NEXT:    v_cndmask_b32_e32 v10, v12, v10, vcc
+; GISEL-NEXT:    v_subrev_i32_e32 v0, vcc, s10, v0
+; GISEL-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s10, v0
+; GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, 0, v3
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, 0, v1
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v12, v0, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v9, v14, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v0
+; GISEL-NEXT:    v_cndmask_b32_e64 v0, v18, v13, s[4:5]
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v16, v17, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v4, v1, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v1, v19, v15, s[4:5]
+; GISEL-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v0, v5, v0, s[4:5]
+; GISEL-NEXT:    v_cndmask_b32_e64 v1, v7, v1, s[4:5]
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v6, v3, vcc
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_udiv_v2i64_pow2k_denom:
+; CGP:       ; %bb.0:
+; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT:    s_movk_i32 s10, 0x1000
+; CGP-NEXT:    v_cvt_f32_u32_e32 v4, 0
+; CGP-NEXT:    v_cvt_f32_u32_e32 v5, s10
+; CGP-NEXT:    s_mov_b32 s8, 0xfffff000
+; CGP-NEXT:    v_mov_b32_e32 v6, v5
+; CGP-NEXT:    v_mac_f32_e32 v5, 0x4f800000, v4
+; CGP-NEXT:    v_mac_f32_e32 v6, 0x4f800000, v4
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v4, v5
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v5, v6
+; CGP-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; CGP-NEXT:    v_mul_f32_e32 v5, 0x5f7ffffc, v5
+; CGP-NEXT:    v_mul_f32_e32 v6, 0x2f800000, v4
+; CGP-NEXT:    v_mul_f32_e32 v7, 0x2f800000, v5
+; CGP-NEXT:    v_trunc_f32_e32 v6, v6
+; CGP-NEXT:    v_trunc_f32_e32 v7, v7
+; CGP-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v6
+; CGP-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; CGP-NEXT:    v_mac_f32_e32 v5, 0xcf800000, v7
+; CGP-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; CGP-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; CGP-NEXT:    v_mul_lo_u32 v8, s8, v6
+; CGP-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; CGP-NEXT:    v_mul_lo_u32 v9, s8, v7
+; CGP-NEXT:    v_mul_lo_u32 v10, s8, v4
+; CGP-NEXT:    v_mul_lo_u32 v11, -1, v4
+; CGP-NEXT:    v_mul_hi_u32 v12, s8, v4
+; CGP-NEXT:    v_mul_lo_u32 v13, s8, v5
+; CGP-NEXT:    v_mul_lo_u32 v14, -1, v5
+; CGP-NEXT:    v_mul_hi_u32 v15, s8, v5
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v11, v8
+; CGP-NEXT:    v_mul_lo_u32 v11, v6, v10
+; CGP-NEXT:    v_mul_hi_u32 v16, v4, v10
+; CGP-NEXT:    v_mul_hi_u32 v10, v6, v10
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v14, v9
+; CGP-NEXT:    v_mul_lo_u32 v14, v7, v13
+; CGP-NEXT:    v_mul_hi_u32 v17, v5, v13
+; CGP-NEXT:    v_mul_hi_u32 v13, v7, v13
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v15
+; CGP-NEXT:    v_mul_lo_u32 v12, v4, v8
+; CGP-NEXT:    v_mul_lo_u32 v15, v6, v8
+; CGP-NEXT:    v_mul_hi_u32 v18, v4, v8
+; CGP-NEXT:    v_mul_hi_u32 v8, v6, v8
+; CGP-NEXT:    v_mul_lo_u32 v19, v5, v9
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v19
+; CGP-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v17
+; CGP-NEXT:    v_mul_lo_u32 v14, v7, v9
+; CGP-NEXT:    v_mul_hi_u32 v17, v5, v9
+; CGP-NEXT:    v_mul_hi_u32 v9, v7, v9
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v15, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v14, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v18
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[4:5]
+; CGP-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v13, v17
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v15, v16
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, v19, v18
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v17
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v13, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v14, v15
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v12
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v10
+; CGP-NEXT:    v_addc_u32_e64 v10, s[4:5], v6, v8, vcc
+; CGP-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v8
+; CGP-NEXT:    v_mul_lo_u32 v8, s8, v4
+; CGP-NEXT:    v_mul_lo_u32 v11, -1, v4
+; CGP-NEXT:    v_mul_hi_u32 v12, s8, v4
+; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v13
+; CGP-NEXT:    v_addc_u32_e64 v13, s[6:7], v7, v9, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v7, s[6:7], v7, v9
+; CGP-NEXT:    v_mul_lo_u32 v9, s8, v5
+; CGP-NEXT:    v_mul_lo_u32 v14, -1, v5
+; CGP-NEXT:    v_mul_hi_u32 v15, s8, v5
+; CGP-NEXT:    v_mul_lo_u32 v16, s8, v10
+; CGP-NEXT:    v_mul_lo_u32 v17, v10, v8
+; CGP-NEXT:    v_mul_hi_u32 v18, v4, v8
+; CGP-NEXT:    v_mul_hi_u32 v8, v10, v8
+; CGP-NEXT:    v_mul_lo_u32 v19, s8, v13
+; CGP-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v16
+; CGP-NEXT:    v_mul_lo_u32 v16, v13, v9
+; CGP-NEXT:    v_add_i32_e64 v14, s[6:7], v14, v19
+; CGP-NEXT:    v_mul_hi_u32 v19, v5, v9
+; CGP-NEXT:    v_mul_hi_u32 v9, v13, v9
+; CGP-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v12
+; CGP-NEXT:    v_add_i32_e64 v12, s[6:7], v14, v15
+; CGP-NEXT:    v_mul_lo_u32 v14, v4, v11
+; CGP-NEXT:    v_mul_lo_u32 v15, v5, v12
+; CGP-NEXT:    v_add_i32_e64 v15, s[6:7], v16, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[6:7]
+; CGP-NEXT:    v_add_i32_e64 v15, s[6:7], v15, v19
+; CGP-NEXT:    v_mul_lo_u32 v15, v10, v11
+; CGP-NEXT:    v_mul_hi_u32 v19, v4, v11
+; CGP-NEXT:    v_mul_hi_u32 v10, v10, v11
+; CGP-NEXT:    v_mul_lo_u32 v11, v13, v12
+; CGP-NEXT:    v_mul_hi_u32 v13, v13, v12
+; CGP-NEXT:    v_mul_hi_u32 v12, v5, v12
+; CGP-NEXT:    v_add_i32_e64 v14, s[8:9], v17, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, 1, s[8:9]
+; CGP-NEXT:    v_add_i32_e64 v8, s[8:9], v15, v8
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[8:9]
+; CGP-NEXT:    v_add_i32_e64 v9, s[8:9], v11, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[8:9]
+; CGP-NEXT:    v_add_i32_e64 v14, s[8:9], v14, v18
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[8:9]
+; CGP-NEXT:    v_add_i32_e64 v8, s[8:9], v8, v19
+; CGP-NEXT:    v_cndmask_b32_e64 v18, 0, 1, s[8:9]
+; CGP-NEXT:    v_cndmask_b32_e64 v19, 0, 1, s[6:7]
+; CGP-NEXT:    v_add_i32_e64 v9, s[6:7], v9, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[6:7]
+; CGP-NEXT:    v_add_i32_e64 v14, s[6:7], v17, v14
+; CGP-NEXT:    v_add_i32_e64 v15, s[6:7], v15, v18
+; CGP-NEXT:    v_add_i32_e64 v16, s[6:7], v16, v19
+; CGP-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v12
+; CGP-NEXT:    v_add_i32_e64 v8, s[6:7], v8, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[6:7]
+; CGP-NEXT:    v_add_i32_e64 v9, s[6:7], v9, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[6:7]
+; CGP-NEXT:    v_add_i32_e64 v12, s[6:7], v15, v12
+; CGP-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v14
+; CGP-NEXT:    v_add_i32_e64 v10, s[6:7], v10, v12
+; CGP-NEXT:    v_add_i32_e64 v11, s[6:7], v13, v11
+; CGP-NEXT:    v_addc_u32_e32 v6, vcc, v6, v10, vcc
+; CGP-NEXT:    v_addc_u32_e64 v7, vcc, v7, v11, s[4:5]
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; CGP-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; CGP-NEXT:    v_mul_lo_u32 v8, v3, v4
+; CGP-NEXT:    v_mul_hi_u32 v10, v2, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
+; CGP-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; CGP-NEXT:    v_mul_lo_u32 v9, v1, v5
+; CGP-NEXT:    v_mul_hi_u32 v11, v0, v5
+; CGP-NEXT:    v_mul_hi_u32 v5, v1, v5
+; CGP-NEXT:    v_mul_lo_u32 v12, v2, v6
+; CGP-NEXT:    v_mul_lo_u32 v13, v3, v6
+; CGP-NEXT:    v_mul_hi_u32 v14, v2, v6
+; CGP-NEXT:    v_mul_hi_u32 v6, v3, v6
+; CGP-NEXT:    v_mul_lo_u32 v15, v0, v7
+; CGP-NEXT:    v_mul_lo_u32 v16, v1, v7
+; CGP-NEXT:    v_mul_hi_u32 v17, v0, v7
+; CGP-NEXT:    v_mul_hi_u32 v7, v1, v7
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v13, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v16, v5
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v17
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v12, v8
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v13, v10
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v15, v9
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v16, v11
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; CGP-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
+; CGP-NEXT:    v_mul_lo_u32 v10, s10, v4
+; CGP-NEXT:    v_mul_lo_u32 v12, 0, v4
+; CGP-NEXT:    v_mul_hi_u32 v13, s10, v4
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
+; CGP-NEXT:    v_mul_lo_u32 v11, s10, v5
+; CGP-NEXT:    v_mul_lo_u32 v14, 0, v5
+; CGP-NEXT:    v_mul_hi_u32 v15, s10, v5
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
+; CGP-NEXT:    v_mul_lo_u32 v8, s10, v6
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, 1, v4
+; CGP-NEXT:    v_addc_u32_e32 v16, vcc, 0, v6, vcc
+; CGP-NEXT:    v_mul_lo_u32 v17, s10, v7
+; CGP-NEXT:    v_add_i32_e32 v18, vcc, 1, v5
+; CGP-NEXT:    v_addc_u32_e32 v19, vcc, 0, v7, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v12, v8
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v14, v17
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, 1, v9
+; CGP-NEXT:    v_addc_u32_e32 v17, vcc, 0, v16, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v13
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v12, v15
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, 1, v18
+; CGP-NEXT:    v_addc_u32_e32 v15, vcc, 0, v19, vcc
+; CGP-NEXT:    v_sub_i32_e32 v2, vcc, v2, v10
+; CGP-NEXT:    v_subb_u32_e64 v10, s[4:5], v3, v8, vcc
+; CGP-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v8
+; CGP-NEXT:    v_cmp_le_u32_e64 s[4:5], s10, v2
+; CGP-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; CGP-NEXT:    v_sub_i32_e64 v0, s[4:5], v0, v11
+; CGP-NEXT:    v_subb_u32_e64 v11, s[6:7], v1, v12, s[4:5]
+; CGP-NEXT:    v_sub_i32_e64 v1, s[6:7], v1, v12
+; CGP-NEXT:    v_cmp_le_u32_e64 s[6:7], 0, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, -1, s[6:7]
+; CGP-NEXT:    v_cmp_eq_u32_e64 s[6:7], 0, v10
+; CGP-NEXT:    v_cmp_le_u32_e64 s[8:9], s10, v0
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[8:9]
+; CGP-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v8, v12, v8, s[6:7]
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, 0, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, -1, vcc
+; CGP-NEXT:    v_subbrev_u32_e64 v1, vcc, 0, v1, s[4:5]
+; CGP-NEXT:    v_subrev_i32_e32 v2, vcc, s10, v2
+; CGP-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s10, v2
+; CGP-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v11
+; CGP-NEXT:    v_cndmask_b32_e32 v10, v12, v10, vcc
+; CGP-NEXT:    v_subrev_i32_e32 v0, vcc, s10, v0
+; CGP-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s10, v0
+; CGP-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, 0, v3
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, -1, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, 0, v1
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, -1, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v12, v0, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v9, v14, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v0
+; CGP-NEXT:    v_cndmask_b32_e64 v0, v18, v13, s[4:5]
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v16, v17, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; CGP-NEXT:    v_cndmask_b32_e32 v2, v4, v1, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v1, v19, v15, s[4:5]
+; CGP-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v0, v5, v0, s[4:5]
+; CGP-NEXT:    v_cndmask_b32_e64 v1, v7, v1, s[4:5]
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v6, v3, vcc
+; CGP-NEXT:    s_setpc_b64 s[30:31]
+  %result = udiv <2 x i64> %num, <i64 4096, i64 4096>
+  ret <2 x i64> %result
+}
+
+define i64 @v_udiv_i64_oddk_denom(i64 %num) {
+; CHECK-LABEL: v_udiv_i64_oddk_denom:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_mov_b32 s6, 0x12d8fb
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v2, 0
+; CHECK-NEXT:    s_mov_b32 s7, 0xffed2705
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v3, s6
+; CHECK-NEXT:    v_mac_f32_e32 v3, 0x4f800000, v2
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v2, v3
+; CHECK-NEXT:    v_mul_f32_e32 v2, 0x5f7ffffc, v2
+; CHECK-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v2
+; CHECK-NEXT:    v_trunc_f32_e32 v3, v3
+; CHECK-NEXT:    v_mac_f32_e32 v2, 0xcf800000, v3
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; CHECK-NEXT:    v_mul_lo_u32 v4, s7, v3
+; CHECK-NEXT:    v_mul_lo_u32 v5, s7, v2
+; CHECK-NEXT:    v_mul_lo_u32 v6, -1, v2
+; CHECK-NEXT:    v_mul_hi_u32 v7, s7, v2
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
+; CHECK-NEXT:    v_mul_lo_u32 v6, v3, v5
+; CHECK-NEXT:    v_mul_hi_u32 v8, v2, v5
+; CHECK-NEXT:    v_mul_hi_u32 v5, v3, v5
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
+; CHECK-NEXT:    v_mul_lo_u32 v7, v2, v4
+; CHECK-NEXT:    v_mul_lo_u32 v9, v3, v4
+; CHECK-NEXT:    v_mul_hi_u32 v10, v2, v4
+; CHECK-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v9, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v9, v8
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; CHECK-NEXT:    v_addc_u32_e64 v5, s[4:5], v3, v4, vcc
+; CHECK-NEXT:    v_add_i32_e64 v3, s[4:5], v3, v4
+; CHECK-NEXT:    v_mul_lo_u32 v4, s7, v2
+; CHECK-NEXT:    v_mul_lo_u32 v6, -1, v2
+; CHECK-NEXT:    v_mul_hi_u32 v7, s7, v2
+; CHECK-NEXT:    v_mul_lo_u32 v8, s7, v5
+; CHECK-NEXT:    v_mul_lo_u32 v9, v5, v4
+; CHECK-NEXT:    v_mul_hi_u32 v10, v2, v4
+; CHECK-NEXT:    v_mul_hi_u32 v4, v5, v4
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v8
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v7
+; CHECK-NEXT:    v_mul_lo_u32 v7, v2, v6
+; CHECK-NEXT:    v_mul_lo_u32 v8, v5, v6
+; CHECK-NEXT:    v_mul_hi_u32 v11, v2, v6
+; CHECK-NEXT:    v_mul_hi_u32 v5, v5, v6
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v9, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v8, v4
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v4, v11
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v7, v6
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v8, v9
+; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v4, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v7, v6
+; CHECK-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v6
+; CHECK-NEXT:    v_addc_u32_e32 v3, vcc, v3, v5, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; CHECK-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v4, v1, v2
+; CHECK-NEXT:    v_mul_hi_u32 v5, v0, v2
+; CHECK-NEXT:    v_mul_hi_u32 v2, v1, v2
+; CHECK-NEXT:    v_mul_lo_u32 v6, v0, v3
+; CHECK-NEXT:    v_mul_lo_u32 v7, v1, v3
+; CHECK-NEXT:    v_mul_hi_u32 v8, v0, v3
+; CHECK-NEXT:    v_mul_hi_u32 v3, v1, v3
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v8
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; CHECK-NEXT:    v_mul_lo_u32 v5, s6, v2
+; CHECK-NEXT:    v_mul_lo_u32 v6, 0, v2
+; CHECK-NEXT:    v_mul_hi_u32 v7, s6, v2
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; CHECK-NEXT:    v_mul_lo_u32 v4, s6, v3
+; CHECK-NEXT:    v_add_i32_e32 v8, vcc, 1, v2
+; CHECK-NEXT:    v_addc_u32_e32 v9, vcc, 0, v3, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, 1, v8
+; CHECK-NEXT:    v_addc_u32_e32 v10, vcc, 0, v9, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v5
+; CHECK-NEXT:    v_subb_u32_e64 v5, s[4:5], v1, v4, vcc
+; CHECK-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v4
+; CHECK-NEXT:    v_cmp_le_u32_e64 s[4:5], s6, v0
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[4:5]
+; CHECK-NEXT:    v_cmp_le_u32_e64 s[4:5], 0, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[4:5]
+; CHECK-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; CHECK-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
+; CHECK-NEXT:    v_subrev_i32_e32 v0, vcc, s6, v0
+; CHECK-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s6, v0
+; CHECK-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, 0, v1
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v8, v6, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v9, v10, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %result = udiv i64 %num, 1235195
+  ret i64 %result
+}
+
+define <2 x i64> @v_udiv_v2i64_oddk_denom(<2 x i64> %num) {
+; GISEL-LABEL: v_udiv_v2i64_oddk_denom:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    s_mov_b32 s10, 0x12d8fb
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, s10
+; GISEL-NEXT:    s_sub_u32 s8, 0, s10
+; GISEL-NEXT:    s_cselect_b32 s4, 1, 0
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v5, 0
+; GISEL-NEXT:    v_mov_b32_e32 v6, v4
+; GISEL-NEXT:    s_and_b32 s4, s4, 1
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
+; GISEL-NEXT:    v_mac_f32_e32 v6, 0x4f800000, v5
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v5, v6
+; GISEL-NEXT:    s_cmp_lg_u32 s4, 0
+; GISEL-NEXT:    s_subb_u32 s9, 0, 0
+; GISEL-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; GISEL-NEXT:    v_mul_f32_e32 v5, 0x5f7ffffc, v5
+; GISEL-NEXT:    v_mul_f32_e32 v6, 0x2f800000, v4
+; GISEL-NEXT:    s_sub_u32 s11, 0, s10
+; GISEL-NEXT:    s_cselect_b32 s4, 1, 0
+; GISEL-NEXT:    v_mul_f32_e32 v7, 0x2f800000, v5
+; GISEL-NEXT:    v_trunc_f32_e32 v6, v6
+; GISEL-NEXT:    s_and_b32 s4, s4, 1
+; GISEL-NEXT:    v_trunc_f32_e32 v7, v7
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v6
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; GISEL-NEXT:    v_mac_f32_e32 v5, 0xcf800000, v7
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; GISEL-NEXT:    s_cmp_lg_u32 s4, 0
+; GISEL-NEXT:    s_subb_u32 s6, 0, 0
+; GISEL-NEXT:    v_mul_lo_u32 v8, s11, v6
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; GISEL-NEXT:    v_mul_lo_u32 v9, s8, v7
+; GISEL-NEXT:    v_mul_lo_u32 v10, s11, v4
+; GISEL-NEXT:    v_mul_lo_u32 v11, s6, v4
+; GISEL-NEXT:    v_mul_hi_u32 v12, s11, v4
+; GISEL-NEXT:    v_mul_lo_u32 v13, s8, v5
+; GISEL-NEXT:    v_mul_lo_u32 v14, s9, v5
+; GISEL-NEXT:    v_mul_hi_u32 v15, s8, v5
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v11, v8
+; GISEL-NEXT:    v_mul_lo_u32 v11, v6, v10
+; GISEL-NEXT:    v_mul_hi_u32 v16, v4, v10
+; GISEL-NEXT:    v_mul_hi_u32 v10, v6, v10
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v14, v9
+; GISEL-NEXT:    v_mul_lo_u32 v14, v7, v13
+; GISEL-NEXT:    v_mul_hi_u32 v17, v5, v13
+; GISEL-NEXT:    v_mul_hi_u32 v13, v7, v13
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v15
+; GISEL-NEXT:    v_mul_lo_u32 v12, v4, v8
+; GISEL-NEXT:    v_mul_lo_u32 v15, v6, v8
+; GISEL-NEXT:    v_mul_hi_u32 v18, v4, v8
+; GISEL-NEXT:    v_mul_hi_u32 v8, v6, v8
+; GISEL-NEXT:    v_mul_lo_u32 v19, v5, v9
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v19
+; GISEL-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v17
+; GISEL-NEXT:    v_mul_lo_u32 v14, v7, v9
+; GISEL-NEXT:    v_mul_hi_u32 v17, v5, v9
+; GISEL-NEXT:    v_mul_hi_u32 v9, v7, v9
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v15, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v14, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v18
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[4:5]
+; GISEL-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v17
+; GISEL-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v15, v16
+; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v19, v18
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v17
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v14, v15
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v12
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v10
+; GISEL-NEXT:    v_addc_u32_e64 v10, s[4:5], v6, v8, vcc
+; GISEL-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v8
+; GISEL-NEXT:    v_mul_lo_u32 v8, s11, v4
+; GISEL-NEXT:    v_mul_lo_u32 v11, s6, v4
+; GISEL-NEXT:    v_mul_hi_u32 v12, s11, v4
+; GISEL-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v13
+; GISEL-NEXT:    v_addc_u32_e64 v13, s[6:7], v7, v9, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v7, s[6:7], v7, v9
+; GISEL-NEXT:    v_mul_lo_u32 v9, s8, v5
+; GISEL-NEXT:    v_mul_lo_u32 v14, s9, v5
+; GISEL-NEXT:    v_mul_hi_u32 v15, s8, v5
+; GISEL-NEXT:    v_mul_lo_u32 v16, s11, v10
+; GISEL-NEXT:    v_mul_lo_u32 v17, v10, v8
+; GISEL-NEXT:    v_mul_hi_u32 v18, v4, v8
+; GISEL-NEXT:    v_mul_hi_u32 v8, v10, v8
+; GISEL-NEXT:    v_mul_lo_u32 v19, s8, v13
+; GISEL-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v16
+; GISEL-NEXT:    v_mul_lo_u32 v16, v13, v9
+; GISEL-NEXT:    v_add_i32_e64 v14, s[6:7], v14, v19
+; GISEL-NEXT:    v_mul_hi_u32 v19, v5, v9
+; GISEL-NEXT:    v_mul_hi_u32 v9, v13, v9
+; GISEL-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v12
+; GISEL-NEXT:    v_add_i32_e64 v12, s[6:7], v14, v15
+; GISEL-NEXT:    v_mul_lo_u32 v14, v4, v11
+; GISEL-NEXT:    v_mul_lo_u32 v15, v5, v12
+; GISEL-NEXT:    v_add_i32_e64 v15, s[6:7], v16, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[6:7]
+; GISEL-NEXT:    v_add_i32_e64 v15, s[6:7], v15, v19
+; GISEL-NEXT:    v_mul_lo_u32 v15, v10, v11
+; GISEL-NEXT:    v_mul_hi_u32 v19, v4, v11
+; GISEL-NEXT:    v_mul_hi_u32 v10, v10, v11
+; GISEL-NEXT:    v_mul_lo_u32 v11, v13, v12
+; GISEL-NEXT:    v_mul_hi_u32 v13, v13, v12
+; GISEL-NEXT:    v_mul_hi_u32 v12, v5, v12
+; GISEL-NEXT:    v_add_i32_e64 v14, s[8:9], v17, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v17, 0, 1, s[8:9]
+; GISEL-NEXT:    v_add_i32_e64 v8, s[8:9], v15, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[8:9]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[8:9], v11, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[8:9]
+; GISEL-NEXT:    v_add_i32_e64 v14, s[8:9], v14, v18
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[8:9]
+; GISEL-NEXT:    v_add_i32_e64 v8, s[8:9], v8, v19
+; GISEL-NEXT:    v_cndmask_b32_e64 v18, 0, 1, s[8:9]
+; GISEL-NEXT:    v_cndmask_b32_e64 v19, 0, 1, s[6:7]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[6:7], v9, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[6:7]
+; GISEL-NEXT:    v_add_i32_e64 v14, s[6:7], v17, v14
+; GISEL-NEXT:    v_add_i32_e64 v15, s[6:7], v15, v18
+; GISEL-NEXT:    v_add_i32_e64 v16, s[6:7], v16, v19
+; GISEL-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v12
+; GISEL-NEXT:    v_add_i32_e64 v8, s[6:7], v8, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[6:7]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[6:7], v9, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[6:7]
+; GISEL-NEXT:    v_add_i32_e64 v12, s[6:7], v15, v12
+; GISEL-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v14
+; GISEL-NEXT:    v_add_i32_e64 v10, s[6:7], v10, v12
+; GISEL-NEXT:    v_add_i32_e64 v11, s[6:7], v13, v11
+; GISEL-NEXT:    v_addc_u32_e32 v6, vcc, v6, v10, vcc
+; GISEL-NEXT:    v_addc_u32_e64 v7, vcc, v7, v11, s[4:5]
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v8, v3, v4
+; GISEL-NEXT:    v_mul_hi_u32 v10, v2, v4
+; GISEL-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
+; GISEL-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v9, v1, v5
+; GISEL-NEXT:    v_mul_hi_u32 v11, v0, v5
+; GISEL-NEXT:    v_mul_hi_u32 v5, v1, v5
+; GISEL-NEXT:    v_mul_lo_u32 v12, v2, v6
+; GISEL-NEXT:    v_mul_lo_u32 v13, v3, v6
+; GISEL-NEXT:    v_mul_hi_u32 v14, v2, v6
+; GISEL-NEXT:    v_mul_hi_u32 v6, v3, v6
+; GISEL-NEXT:    v_mul_lo_u32 v15, v0, v7
+; GISEL-NEXT:    v_mul_lo_u32 v16, v1, v7
+; GISEL-NEXT:    v_mul_hi_u32 v17, v0, v7
+; GISEL-NEXT:    v_mul_hi_u32 v7, v1, v7
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v13, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v16, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v17
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v12, v8
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v13, v10
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v15, v9
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v16, v11
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
+; GISEL-NEXT:    v_mul_lo_u32 v10, s10, v4
+; GISEL-NEXT:    v_mul_lo_u32 v12, 0, v4
+; GISEL-NEXT:    v_mul_hi_u32 v13, s10, v4
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
+; GISEL-NEXT:    v_mul_lo_u32 v11, s10, v5
+; GISEL-NEXT:    v_mul_lo_u32 v14, 0, v5
+; GISEL-NEXT:    v_mul_hi_u32 v15, s10, v5
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
+; GISEL-NEXT:    v_mul_lo_u32 v8, s10, v6
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, 1, v4
+; GISEL-NEXT:    v_addc_u32_e32 v16, vcc, 0, v6, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v17, s10, v7
+; GISEL-NEXT:    v_add_i32_e32 v18, vcc, 1, v5
+; GISEL-NEXT:    v_addc_u32_e32 v19, vcc, 0, v7, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v12, v8
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v14, v17
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, 1, v9
+; GISEL-NEXT:    v_addc_u32_e32 v17, vcc, 0, v16, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v13
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v15
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, 1, v18
+; GISEL-NEXT:    v_addc_u32_e32 v15, vcc, 0, v19, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v10
+; GISEL-NEXT:    v_subb_u32_e64 v10, s[4:5], v3, v8, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v8
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s10, v2
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e64 v0, s[4:5], v0, v11
+; GISEL-NEXT:    v_subb_u32_e64 v11, s[6:7], v1, v12, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e64 v1, s[6:7], v1, v12
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[6:7], 0, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, -1, s[6:7]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[6:7], 0, v10
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[8:9], s10, v0
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[8:9]
+; GISEL-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, v12, v8, s[6:7]
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, 0, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, -1, vcc
+; GISEL-NEXT:    v_subbrev_u32_e64 v1, vcc, 0, v1, s[4:5]
+; GISEL-NEXT:    v_subrev_i32_e32 v2, vcc, s10, v2
+; GISEL-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s10, v2
+; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v11
+; GISEL-NEXT:    v_cndmask_b32_e32 v10, v12, v10, vcc
+; GISEL-NEXT:    v_subrev_i32_e32 v0, vcc, s10, v0
+; GISEL-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s10, v0
+; GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, 0, v3
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, 0, v1
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v12, v0, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v9, v14, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v0
+; GISEL-NEXT:    v_cndmask_b32_e64 v0, v18, v13, s[4:5]
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v16, v17, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v4, v1, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v1, v19, v15, s[4:5]
+; GISEL-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v0, v5, v0, s[4:5]
+; GISEL-NEXT:    v_cndmask_b32_e64 v1, v7, v1, s[4:5]
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v6, v3, vcc
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_udiv_v2i64_oddk_denom:
+; CGP:       ; %bb.0:
+; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT:    s_mov_b32 s10, 0x12d8fb
+; CGP-NEXT:    v_cvt_f32_u32_e32 v4, 0
+; CGP-NEXT:    v_cvt_f32_u32_e32 v5, s10
+; CGP-NEXT:    s_mov_b32 s8, 0xffed2705
+; CGP-NEXT:    v_mov_b32_e32 v6, v5
+; CGP-NEXT:    v_mac_f32_e32 v5, 0x4f800000, v4
+; CGP-NEXT:    v_mac_f32_e32 v6, 0x4f800000, v4
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v4, v5
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v5, v6
+; CGP-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; CGP-NEXT:    v_mul_f32_e32 v5, 0x5f7ffffc, v5
+; CGP-NEXT:    v_mul_f32_e32 v6, 0x2f800000, v4
+; CGP-NEXT:    v_mul_f32_e32 v7, 0x2f800000, v5
+; CGP-NEXT:    v_trunc_f32_e32 v6, v6
+; CGP-NEXT:    v_trunc_f32_e32 v7, v7
+; CGP-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v6
+; CGP-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; CGP-NEXT:    v_mac_f32_e32 v5, 0xcf800000, v7
+; CGP-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; CGP-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; CGP-NEXT:    v_mul_lo_u32 v8, s8, v6
+; CGP-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; CGP-NEXT:    v_mul_lo_u32 v9, s8, v7
+; CGP-NEXT:    v_mul_lo_u32 v10, s8, v4
+; CGP-NEXT:    v_mul_lo_u32 v11, -1, v4
+; CGP-NEXT:    v_mul_hi_u32 v12, s8, v4
+; CGP-NEXT:    v_mul_lo_u32 v13, s8, v5
+; CGP-NEXT:    v_mul_lo_u32 v14, -1, v5
+; CGP-NEXT:    v_mul_hi_u32 v15, s8, v5
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v11, v8
+; CGP-NEXT:    v_mul_lo_u32 v11, v6, v10
+; CGP-NEXT:    v_mul_hi_u32 v16, v4, v10
+; CGP-NEXT:    v_mul_hi_u32 v10, v6, v10
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v14, v9
+; CGP-NEXT:    v_mul_lo_u32 v14, v7, v13
+; CGP-NEXT:    v_mul_hi_u32 v17, v5, v13
+; CGP-NEXT:    v_mul_hi_u32 v13, v7, v13
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v15
+; CGP-NEXT:    v_mul_lo_u32 v12, v4, v8
+; CGP-NEXT:    v_mul_lo_u32 v15, v6, v8
+; CGP-NEXT:    v_mul_hi_u32 v18, v4, v8
+; CGP-NEXT:    v_mul_hi_u32 v8, v6, v8
+; CGP-NEXT:    v_mul_lo_u32 v19, v5, v9
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v19
+; CGP-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v17
+; CGP-NEXT:    v_mul_lo_u32 v14, v7, v9
+; CGP-NEXT:    v_mul_hi_u32 v17, v5, v9
+; CGP-NEXT:    v_mul_hi_u32 v9, v7, v9
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v15, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v14, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v18
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[4:5]
+; CGP-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v13, v17
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v15, v16
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, v19, v18
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v17
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v13, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v14, v15
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v12
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v10
+; CGP-NEXT:    v_addc_u32_e64 v10, s[4:5], v6, v8, vcc
+; CGP-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v8
+; CGP-NEXT:    v_mul_lo_u32 v8, s8, v4
+; CGP-NEXT:    v_mul_lo_u32 v11, -1, v4
+; CGP-NEXT:    v_mul_hi_u32 v12, s8, v4
+; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v13
+; CGP-NEXT:    v_addc_u32_e64 v13, s[6:7], v7, v9, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v7, s[6:7], v7, v9
+; CGP-NEXT:    v_mul_lo_u32 v9, s8, v5
+; CGP-NEXT:    v_mul_lo_u32 v14, -1, v5
+; CGP-NEXT:    v_mul_hi_u32 v15, s8, v5
+; CGP-NEXT:    v_mul_lo_u32 v16, s8, v10
+; CGP-NEXT:    v_mul_lo_u32 v17, v10, v8
+; CGP-NEXT:    v_mul_hi_u32 v18, v4, v8
+; CGP-NEXT:    v_mul_hi_u32 v8, v10, v8
+; CGP-NEXT:    v_mul_lo_u32 v19, s8, v13
+; CGP-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v16
+; CGP-NEXT:    v_mul_lo_u32 v16, v13, v9
+; CGP-NEXT:    v_add_i32_e64 v14, s[6:7], v14, v19
+; CGP-NEXT:    v_mul_hi_u32 v19, v5, v9
+; CGP-NEXT:    v_mul_hi_u32 v9, v13, v9
+; CGP-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v12
+; CGP-NEXT:    v_add_i32_e64 v12, s[6:7], v14, v15
+; CGP-NEXT:    v_mul_lo_u32 v14, v4, v11
+; CGP-NEXT:    v_mul_lo_u32 v15, v5, v12
+; CGP-NEXT:    v_add_i32_e64 v15, s[6:7], v16, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[6:7]
+; CGP-NEXT:    v_add_i32_e64 v15, s[6:7], v15, v19
+; CGP-NEXT:    v_mul_lo_u32 v15, v10, v11
+; CGP-NEXT:    v_mul_hi_u32 v19, v4, v11
+; CGP-NEXT:    v_mul_hi_u32 v10, v10, v11
+; CGP-NEXT:    v_mul_lo_u32 v11, v13, v12
+; CGP-NEXT:    v_mul_hi_u32 v13, v13, v12
+; CGP-NEXT:    v_mul_hi_u32 v12, v5, v12
+; CGP-NEXT:    v_add_i32_e64 v14, s[8:9], v17, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, 1, s[8:9]
+; CGP-NEXT:    v_add_i32_e64 v8, s[8:9], v15, v8
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[8:9]
+; CGP-NEXT:    v_add_i32_e64 v9, s[8:9], v11, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[8:9]
+; CGP-NEXT:    v_add_i32_e64 v14, s[8:9], v14, v18
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[8:9]
+; CGP-NEXT:    v_add_i32_e64 v8, s[8:9], v8, v19
+; CGP-NEXT:    v_cndmask_b32_e64 v18, 0, 1, s[8:9]
+; CGP-NEXT:    v_cndmask_b32_e64 v19, 0, 1, s[6:7]
+; CGP-NEXT:    v_add_i32_e64 v9, s[6:7], v9, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[6:7]
+; CGP-NEXT:    v_add_i32_e64 v14, s[6:7], v17, v14
+; CGP-NEXT:    v_add_i32_e64 v15, s[6:7], v15, v18
+; CGP-NEXT:    v_add_i32_e64 v16, s[6:7], v16, v19
+; CGP-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v12
+; CGP-NEXT:    v_add_i32_e64 v8, s[6:7], v8, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[6:7]
+; CGP-NEXT:    v_add_i32_e64 v9, s[6:7], v9, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[6:7]
+; CGP-NEXT:    v_add_i32_e64 v12, s[6:7], v15, v12
+; CGP-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v14
+; CGP-NEXT:    v_add_i32_e64 v10, s[6:7], v10, v12
+; CGP-NEXT:    v_add_i32_e64 v11, s[6:7], v13, v11
+; CGP-NEXT:    v_addc_u32_e32 v6, vcc, v6, v10, vcc
+; CGP-NEXT:    v_addc_u32_e64 v7, vcc, v7, v11, s[4:5]
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; CGP-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; CGP-NEXT:    v_mul_lo_u32 v8, v3, v4
+; CGP-NEXT:    v_mul_hi_u32 v10, v2, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
+; CGP-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; CGP-NEXT:    v_mul_lo_u32 v9, v1, v5
+; CGP-NEXT:    v_mul_hi_u32 v11, v0, v5
+; CGP-NEXT:    v_mul_hi_u32 v5, v1, v5
+; CGP-NEXT:    v_mul_lo_u32 v12, v2, v6
+; CGP-NEXT:    v_mul_lo_u32 v13, v3, v6
+; CGP-NEXT:    v_mul_hi_u32 v14, v2, v6
+; CGP-NEXT:    v_mul_hi_u32 v6, v3, v6
+; CGP-NEXT:    v_mul_lo_u32 v15, v0, v7
+; CGP-NEXT:    v_mul_lo_u32 v16, v1, v7
+; CGP-NEXT:    v_mul_hi_u32 v17, v0, v7
+; CGP-NEXT:    v_mul_hi_u32 v7, v1, v7
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v13, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v16, v5
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v17
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v12, v8
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v13, v10
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v15, v9
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v16, v11
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; CGP-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
+; CGP-NEXT:    v_mul_lo_u32 v10, s10, v4
+; CGP-NEXT:    v_mul_lo_u32 v12, 0, v4
+; CGP-NEXT:    v_mul_hi_u32 v13, s10, v4
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
+; CGP-NEXT:    v_mul_lo_u32 v11, s10, v5
+; CGP-NEXT:    v_mul_lo_u32 v14, 0, v5
+; CGP-NEXT:    v_mul_hi_u32 v15, s10, v5
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
+; CGP-NEXT:    v_mul_lo_u32 v8, s10, v6
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, 1, v4
+; CGP-NEXT:    v_addc_u32_e32 v16, vcc, 0, v6, vcc
+; CGP-NEXT:    v_mul_lo_u32 v17, s10, v7
+; CGP-NEXT:    v_add_i32_e32 v18, vcc, 1, v5
+; CGP-NEXT:    v_addc_u32_e32 v19, vcc, 0, v7, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v12, v8
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v14, v17
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, 1, v9
+; CGP-NEXT:    v_addc_u32_e32 v17, vcc, 0, v16, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v13
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v12, v15
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, 1, v18
+; CGP-NEXT:    v_addc_u32_e32 v15, vcc, 0, v19, vcc
+; CGP-NEXT:    v_sub_i32_e32 v2, vcc, v2, v10
+; CGP-NEXT:    v_subb_u32_e64 v10, s[4:5], v3, v8, vcc
+; CGP-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v8
+; CGP-NEXT:    v_cmp_le_u32_e64 s[4:5], s10, v2
+; CGP-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; CGP-NEXT:    v_sub_i32_e64 v0, s[4:5], v0, v11
+; CGP-NEXT:    v_subb_u32_e64 v11, s[6:7], v1, v12, s[4:5]
+; CGP-NEXT:    v_sub_i32_e64 v1, s[6:7], v1, v12
+; CGP-NEXT:    v_cmp_le_u32_e64 s[6:7], 0, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, -1, s[6:7]
+; CGP-NEXT:    v_cmp_eq_u32_e64 s[6:7], 0, v10
+; CGP-NEXT:    v_cmp_le_u32_e64 s[8:9], s10, v0
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[8:9]
+; CGP-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v8, v12, v8, s[6:7]
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, 0, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, -1, vcc
+; CGP-NEXT:    v_subbrev_u32_e64 v1, vcc, 0, v1, s[4:5]
+; CGP-NEXT:    v_subrev_i32_e32 v2, vcc, s10, v2
+; CGP-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s10, v2
+; CGP-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v11
+; CGP-NEXT:    v_cndmask_b32_e32 v10, v12, v10, vcc
+; CGP-NEXT:    v_subrev_i32_e32 v0, vcc, s10, v0
+; CGP-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s10, v0
+; CGP-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, 0, v3
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, -1, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, 0, v1
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, -1, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v12, v0, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v9, v14, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v0
+; CGP-NEXT:    v_cndmask_b32_e64 v0, v18, v13, s[4:5]
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v16, v17, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; CGP-NEXT:    v_cndmask_b32_e32 v2, v4, v1, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v1, v19, v15, s[4:5]
+; CGP-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v0, v5, v0, s[4:5]
+; CGP-NEXT:    v_cndmask_b32_e64 v1, v7, v1, s[4:5]
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v6, v3, vcc
+; CGP-NEXT:    s_setpc_b64 s[30:31]
+  %result = udiv <2 x i64> %num, <i64 1235195, i64 1235195>
+  ret <2 x i64> %result
+}
+
+define i64 @v_udiv_i64_pow2_shl_denom(i64 %x, i64 %y) {
+; CHECK-LABEL: v_udiv_i64_pow2_shl_denom:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_movk_i32 s4, 0x1000
+; CHECK-NEXT:    s_mov_b32 s5, 0
+; CHECK-NEXT:    v_mov_b32_e32 v6, 0
+; CHECK-NEXT:    v_lshl_b64 v[4:5], s[4:5], v2
+; CHECK-NEXT:    v_or_b32_e32 v7, v1, v5
+; CHECK-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; CHECK-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, 1
+; CHECK-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; CHECK-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; CHECK-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
+; CHECK-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; CHECK-NEXT:    s_cbranch_execnz BB7_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v2, v4
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v3, v5
+; CHECK-NEXT:    v_sub_i32_e32 v6, vcc, 0, v4
+; CHECK-NEXT:    v_subb_u32_e32 v7, vcc, 0, v5, vcc
+; CHECK-NEXT:    v_mac_f32_e32 v2, 0x4f800000, v3
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; CHECK-NEXT:    v_mul_f32_e32 v2, 0x5f7ffffc, v2
+; CHECK-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v2
+; CHECK-NEXT:    v_trunc_f32_e32 v3, v3
+; CHECK-NEXT:    v_mac_f32_e32 v2, 0xcf800000, v3
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; CHECK-NEXT:    v_mul_lo_u32 v8, v6, v3
+; CHECK-NEXT:    v_mul_lo_u32 v9, v6, v2
+; CHECK-NEXT:    v_mul_lo_u32 v10, v7, v2
+; CHECK-NEXT:    v_mul_hi_u32 v11, v6, v2
+; CHECK-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
+; CHECK-NEXT:    v_mul_lo_u32 v10, v3, v9
+; CHECK-NEXT:    v_mul_hi_u32 v12, v2, v9
+; CHECK-NEXT:    v_mul_hi_u32 v9, v3, v9
+; CHECK-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
+; CHECK-NEXT:    v_mul_lo_u32 v11, v2, v8
+; CHECK-NEXT:    v_mul_lo_u32 v13, v3, v8
+; CHECK-NEXT:    v_mul_hi_u32 v14, v2, v8
+; CHECK-NEXT:    v_mul_hi_u32 v8, v3, v8
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; CHECK-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v9, vcc, v13, v9
+; CHECK-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v9, vcc, v9, v14
+; CHECK-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; CHECK-NEXT:    v_add_i32_e32 v11, vcc, v13, v12
+; CHECK-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; CHECK-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v9
+; CHECK-NEXT:    v_addc_u32_e64 v9, s[4:5], v3, v8, vcc
+; CHECK-NEXT:    v_add_i32_e64 v3, s[4:5], v3, v8
+; CHECK-NEXT:    v_mul_lo_u32 v8, v6, v2
+; CHECK-NEXT:    v_mul_lo_u32 v7, v7, v2
+; CHECK-NEXT:    v_mul_hi_u32 v10, v6, v2
+; CHECK-NEXT:    v_mul_lo_u32 v6, v6, v9
+; CHECK-NEXT:    v_mul_lo_u32 v11, v9, v8
+; CHECK-NEXT:    v_mul_hi_u32 v12, v2, v8
+; CHECK-NEXT:    v_mul_hi_u32 v8, v9, v8
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v7, v6
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v10
+; CHECK-NEXT:    v_mul_lo_u32 v7, v2, v6
+; CHECK-NEXT:    v_mul_lo_u32 v10, v9, v6
+; CHECK-NEXT:    v_mul_hi_u32 v13, v2, v6
+; CHECK-NEXT:    v_mul_hi_u32 v6, v9, v6
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v11, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v8, s[4:5], v10, v8
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v7, v12
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v13
+; CHECK-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v9, v7
+; CHECK-NEXT:    v_add_i32_e64 v9, s[4:5], v10, v11
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v8, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v8, s[4:5], v9, v8
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v8
+; CHECK-NEXT:    v_addc_u32_e32 v3, vcc, v3, v6, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v7
+; CHECK-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v6, v1, v2
+; CHECK-NEXT:    v_mul_hi_u32 v7, v0, v2
+; CHECK-NEXT:    v_mul_hi_u32 v2, v1, v2
+; CHECK-NEXT:    v_mul_lo_u32 v8, v0, v3
+; CHECK-NEXT:    v_mul_lo_u32 v9, v1, v3
+; CHECK-NEXT:    v_mul_hi_u32 v10, v0, v3
+; CHECK-NEXT:    v_mul_hi_u32 v3, v1, v3
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v9, v2
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v9, v7
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT:    v_mul_lo_u32 v7, v4, v2
+; CHECK-NEXT:    v_mul_lo_u32 v8, v5, v2
+; CHECK-NEXT:    v_mul_hi_u32 v9, v4, v2
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
+; CHECK-NEXT:    v_mul_lo_u32 v6, v4, v3
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, 1, v2
+; CHECK-NEXT:    v_addc_u32_e32 v11, vcc, 0, v3, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
+; CHECK-NEXT:    v_add_i32_e32 v8, vcc, 1, v10
+; CHECK-NEXT:    v_addc_u32_e32 v12, vcc, 0, v11, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v9
+; CHECK-NEXT:    v_sub_i32_e32 v7, vcc, v0, v7
+; CHECK-NEXT:    v_subb_u32_e64 v9, s[4:5], v1, v6, vcc
+; CHECK-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v6
+; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v7, v4
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[4:5]
+; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v9, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v13, 0, -1, s[4:5]
+; CHECK-NEXT:    v_subb_u32_e32 v1, vcc, v1, v5, vcc
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, v9, v5
+; CHECK-NEXT:    v_cndmask_b32_e32 v6, v13, v6, vcc
+; CHECK-NEXT:    v_sub_i32_e32 v7, vcc, v7, v4
+; CHECK-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v7, v4
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, -1, vcc
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v5
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v9, v7, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v10, v8, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v5, v11, v12, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; CHECK-NEXT:    v_cndmask_b32_e32 v2, v2, v1, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; CHECK-NEXT:  BB7_2: ; %Flow
+; CHECK-NEXT:    s_or_saveexec_b64 s[6:7], s[6:7]
+; CHECK-NEXT:    s_xor_b64 exec, exec, s[6:7]
+; CHECK-NEXT:    s_cbranch_execz BB7_4
+; CHECK-NEXT:  ; %bb.3:
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v1, v4
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v1, v1
+; CHECK-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v1
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; CHECK-NEXT:    v_mul_lo_u32 v2, v1, v4
+; CHECK-NEXT:    v_mul_hi_u32 v3, v1, v4
+; CHECK-NEXT:    v_sub_i32_e32 v5, vcc, 0, v2
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; CHECK-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
+; CHECK-NEXT:    v_mul_hi_u32 v2, v2, v1
+; CHECK-NEXT:    v_add_i32_e64 v3, s[4:5], v1, v2
+; CHECK-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v2
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; CHECK-NEXT:    v_mul_hi_u32 v1, v1, v0
+; CHECK-NEXT:    v_mul_lo_u32 v2, v1, v4
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, 1, v1
+; CHECK-NEXT:    v_subrev_i32_e32 v5, vcc, 1, v1
+; CHECK-NEXT:    v_sub_i32_e32 v6, vcc, v0, v2
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
+; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v6, v4
+; CHECK-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
+; CHECK-NEXT:    v_cndmask_b32_e64 v0, v3, v1, s[4:5]
+; CHECK-NEXT:    v_cndmask_b32_e32 v2, v5, v0, vcc
+; CHECK-NEXT:    v_mov_b32_e32 v3, 0
+; CHECK-NEXT:  BB7_4:
+; CHECK-NEXT:    s_or_b64 exec, exec, s[6:7]
+; CHECK-NEXT:    v_mov_b32_e32 v0, v2
+; CHECK-NEXT:    v_mov_b32_e32 v1, v3
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %shl.y = shl i64 4096, %y
+  %r = udiv i64 %x, %shl.y
+  ret i64 %r
+}
+
+define <2 x i64> @v_udiv_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
+; GISEL-LABEL: v_udiv_v2i64_pow2_shl_denom:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    s_movk_i32 s4, 0x1000
+; GISEL-NEXT:    s_mov_b32 s5, 0
+; GISEL-NEXT:    v_lshl_b64 v[4:5], s[4:5], v4
+; GISEL-NEXT:    v_lshl_b64 v[6:7], s[4:5], v6
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v8, v4
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v9, v5
+; GISEL-NEXT:    v_mac_f32_e32 v8, 0x4f800000, v9
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v8, v8
+; GISEL-NEXT:    v_mul_f32_e32 v8, 0x5f7ffffc, v8
+; GISEL-NEXT:    v_mul_f32_e32 v9, 0x2f800000, v8
+; GISEL-NEXT:    v_trunc_f32_e32 v9, v9
+; GISEL-NEXT:    v_mac_f32_e32 v8, 0xcf800000, v9
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v8, v8
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v9, v9
+; GISEL-NEXT:    v_sub_i32_e32 v10, vcc, 0, v4
+; GISEL-NEXT:    v_subb_u32_e32 v11, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v12, v10, v8
+; GISEL-NEXT:    v_mul_lo_u32 v13, v11, v8
+; GISEL-NEXT:    v_mul_lo_u32 v14, v10, v9
+; GISEL-NEXT:    v_mul_hi_u32 v15, v10, v8
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v15
+; GISEL-NEXT:    v_mul_lo_u32 v14, v9, v12
+; GISEL-NEXT:    v_mul_lo_u32 v15, v8, v13
+; GISEL-NEXT:    v_mul_hi_u32 v16, v8, v12
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; GISEL-NEXT:    v_mul_lo_u32 v15, v9, v13
+; GISEL-NEXT:    v_mul_hi_u32 v12, v9, v12
+; GISEL-NEXT:    v_mul_hi_u32 v16, v8, v13
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v15, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v15, v16
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; GISEL-NEXT:    v_mul_hi_u32 v13, v9, v13
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
+; GISEL-NEXT:    v_addc_u32_e64 v12, s[4:5], v9, v13, vcc
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v13
+; GISEL-NEXT:    v_mul_lo_u32 v13, v10, v8
+; GISEL-NEXT:    v_mul_lo_u32 v11, v11, v8
+; GISEL-NEXT:    v_mul_lo_u32 v14, v10, v12
+; GISEL-NEXT:    v_mul_hi_u32 v10, v10, v8
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v14
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v11, v10
+; GISEL-NEXT:    v_mul_lo_u32 v11, v12, v13
+; GISEL-NEXT:    v_mul_lo_u32 v14, v8, v10
+; GISEL-NEXT:    v_mul_hi_u32 v15, v8, v13
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v14, v11
+; GISEL-NEXT:    v_mul_lo_u32 v14, v12, v10
+; GISEL-NEXT:    v_mul_hi_u32 v13, v12, v13
+; GISEL-NEXT:    v_mul_hi_u32 v15, v8, v10
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v14, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v14, s[4:5], v14, v15
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v13, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v14, v13
+; GISEL-NEXT:    v_mul_hi_u32 v10, v12, v10
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v13
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v11
+; GISEL-NEXT:    v_addc_u32_e32 v9, vcc, v9, v10, vcc
+; GISEL-NEXT:    v_addc_u32_e64 v9, vcc, 0, v9, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v10, v1, v8
+; GISEL-NEXT:    v_mul_lo_u32 v11, v0, v9
+; GISEL-NEXT:    v_mul_hi_u32 v12, v0, v8
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT:    v_mul_lo_u32 v11, v1, v9
+; GISEL-NEXT:    v_mul_hi_u32 v8, v1, v8
+; GISEL-NEXT:    v_mul_hi_u32 v12, v0, v9
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v11, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT:    v_mul_hi_u32 v9, v1, v9
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT:    v_mul_lo_u32 v10, v4, v8
+; GISEL-NEXT:    v_mul_lo_u32 v11, v5, v8
+; GISEL-NEXT:    v_mul_lo_u32 v12, v4, v9
+; GISEL-NEXT:    v_mul_hi_u32 v13, v4, v8
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v13
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v10
+; GISEL-NEXT:    v_subb_u32_e64 v10, s[4:5], v1, v11, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v11
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v10, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], v10, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, v11, v12, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e64 v0, s[4:5], v0, v4
+; GISEL-NEXT:    v_subb_u32_e32 v1, vcc, v1, v5, vcc
+; GISEL-NEXT:    v_subbrev_u32_e64 v1, vcc, 0, v1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, 1, v8
+; GISEL-NEXT:    v_addc_u32_e32 v12, vcc, 0, v9, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v5
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v13, v0, vcc
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, 1, v11
+; GISEL-NEXT:    v_addc_u32_e32 v4, vcc, 0, v12, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v11, v1, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v12, v4, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, v6
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v5, v7
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; GISEL-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; GISEL-NEXT:    v_mul_f32_e32 v5, 0x2f800000, v4
+; GISEL-NEXT:    v_trunc_f32_e32 v5, v5
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v5
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; GISEL-NEXT:    v_sub_i32_e32 v8, vcc, 0, v6
+; GISEL-NEXT:    v_subb_u32_e32 v9, vcc, 0, v7, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v10, v8, v4
+; GISEL-NEXT:    v_mul_lo_u32 v11, v9, v4
+; GISEL-NEXT:    v_mul_lo_u32 v12, v8, v5
+; GISEL-NEXT:    v_mul_hi_u32 v13, v8, v4
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v13
+; GISEL-NEXT:    v_mul_lo_u32 v12, v5, v10
+; GISEL-NEXT:    v_mul_lo_u32 v13, v4, v11
+; GISEL-NEXT:    v_mul_hi_u32 v14, v4, v10
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; GISEL-NEXT:    v_mul_lo_u32 v13, v5, v11
+; GISEL-NEXT:    v_mul_hi_u32 v10, v5, v10
+; GISEL-NEXT:    v_mul_hi_u32 v14, v4, v11
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v13, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; GISEL-NEXT:    v_mul_hi_u32 v11, v5, v11
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v10
+; GISEL-NEXT:    v_addc_u32_e64 v10, s[4:5], v5, v11, vcc
+; GISEL-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v11
+; GISEL-NEXT:    v_mul_lo_u32 v11, v8, v4
+; GISEL-NEXT:    v_mul_lo_u32 v9, v9, v4
+; GISEL-NEXT:    v_mul_lo_u32 v12, v8, v10
+; GISEL-NEXT:    v_mul_hi_u32 v8, v8, v4
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v12
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v9, v8
+; GISEL-NEXT:    v_mul_lo_u32 v9, v10, v11
+; GISEL-NEXT:    v_mul_lo_u32 v12, v4, v8
+; GISEL-NEXT:    v_mul_hi_u32 v13, v4, v11
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v12, v9
+; GISEL-NEXT:    v_mul_lo_u32 v12, v10, v8
+; GISEL-NEXT:    v_mul_hi_u32 v11, v10, v11
+; GISEL-NEXT:    v_mul_hi_u32 v13, v4, v8
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v12, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v13
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v11, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v12, v11
+; GISEL-NEXT:    v_mul_hi_u32 v8, v10, v8
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v11
+; GISEL-NEXT:    v_add_i32_e64 v4, s[4:5], v4, v9
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, v5, v8, vcc
+; GISEL-NEXT:    v_addc_u32_e64 v5, vcc, 0, v5, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v8, v3, v4
+; GISEL-NEXT:    v_mul_lo_u32 v9, v2, v5
+; GISEL-NEXT:    v_mul_hi_u32 v10, v2, v4
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT:    v_mul_lo_u32 v9, v3, v5
+; GISEL-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GISEL-NEXT:    v_mul_hi_u32 v10, v2, v5
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v9, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT:    v_mul_hi_u32 v5, v3, v5
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v8
+; GISEL-NEXT:    v_mul_lo_u32 v8, v6, v4
+; GISEL-NEXT:    v_mul_lo_u32 v9, v7, v4
+; GISEL-NEXT:    v_mul_lo_u32 v10, v6, v5
+; GISEL-NEXT:    v_mul_hi_u32 v11, v6, v4
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v8
+; GISEL-NEXT:    v_subb_u32_e64 v8, s[4:5], v3, v9, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v9
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v8, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v2, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], v8, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, v9, v10, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e64 v2, s[4:5], v2, v6
+; GISEL-NEXT:    v_subb_u32_e32 v3, vcc, v3, v7, vcc
+; GISEL-NEXT:    v_subbrev_u32_e64 v3, vcc, 0, v3, s[4:5]
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, 1, v4
+; GISEL-NEXT:    v_addc_u32_e32 v10, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v7
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v11, v2, vcc
+; GISEL-NEXT:    v_add_i32_e32 v3, vcc, 1, v9
+; GISEL-NEXT:    v_addc_u32_e32 v6, vcc, 0, v10, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v9, v3, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v10, v6, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_udiv_v2i64_pow2_shl_denom:
+; CGP:       ; %bb.0:
+; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT:    v_mov_b32_e32 v5, v0
+; CGP-NEXT:    v_mov_b32_e32 v7, v1
+; CGP-NEXT:    s_movk_i32 s4, 0x1000
+; CGP-NEXT:    s_mov_b32 s5, 0
+; CGP-NEXT:    v_mov_b32_e32 v0, 0
+; CGP-NEXT:    v_lshl_b64 v[10:11], s[4:5], v4
+; CGP-NEXT:    v_lshl_b64 v[8:9], s[4:5], v6
+; CGP-NEXT:    v_or_b32_e32 v1, v7, v11
+; CGP-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
+; CGP-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, 1
+; CGP-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; CGP-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; CGP-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
+; CGP-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; CGP-NEXT:    s_cbranch_execnz BB8_2
+; CGP-NEXT:  ; %bb.1:
+; CGP-NEXT:    v_cvt_f32_u32_e32 v0, v10
+; CGP-NEXT:    v_cvt_f32_u32_e32 v1, v11
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, 0, v10
+; CGP-NEXT:    v_subb_u32_e32 v6, vcc, 0, v11, vcc
+; CGP-NEXT:    v_mac_f32_e32 v0, 0x4f800000, v1
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; CGP-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
+; CGP-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
+; CGP-NEXT:    v_trunc_f32_e32 v1, v1
+; CGP-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
+; CGP-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; CGP-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; CGP-NEXT:    v_mul_lo_u32 v12, v4, v1
+; CGP-NEXT:    v_mul_lo_u32 v13, v4, v0
+; CGP-NEXT:    v_mul_lo_u32 v14, v6, v0
+; CGP-NEXT:    v_mul_hi_u32 v15, v4, v0
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v14, v12
+; CGP-NEXT:    v_mul_lo_u32 v14, v1, v13
+; CGP-NEXT:    v_mul_hi_u32 v16, v0, v13
+; CGP-NEXT:    v_mul_hi_u32 v13, v1, v13
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v12, v15
+; CGP-NEXT:    v_mul_lo_u32 v15, v0, v12
+; CGP-NEXT:    v_mul_lo_u32 v17, v1, v12
+; CGP-NEXT:    v_mul_hi_u32 v18, v0, v12
+; CGP-NEXT:    v_mul_hi_u32 v12, v1, v12
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v17, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v13, v18
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, v17, v16
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v12, v14
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v13
+; CGP-NEXT:    v_addc_u32_e64 v13, s[4:5], v1, v12, vcc
+; CGP-NEXT:    v_add_i32_e64 v1, s[4:5], v1, v12
+; CGP-NEXT:    v_mul_lo_u32 v12, v4, v0
+; CGP-NEXT:    v_mul_lo_u32 v6, v6, v0
+; CGP-NEXT:    v_mul_hi_u32 v14, v4, v0
+; CGP-NEXT:    v_mul_lo_u32 v4, v4, v13
+; CGP-NEXT:    v_mul_lo_u32 v15, v13, v12
+; CGP-NEXT:    v_mul_hi_u32 v16, v0, v12
+; CGP-NEXT:    v_mul_hi_u32 v12, v13, v12
+; CGP-NEXT:    v_add_i32_e64 v4, s[4:5], v6, v4
+; CGP-NEXT:    v_add_i32_e64 v4, s[4:5], v4, v14
+; CGP-NEXT:    v_mul_lo_u32 v6, v0, v4
+; CGP-NEXT:    v_mul_lo_u32 v14, v13, v4
+; CGP-NEXT:    v_mul_hi_u32 v17, v0, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, v13, v4
+; CGP-NEXT:    v_add_i32_e64 v6, s[4:5], v15, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v14, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v17
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v6, s[4:5], v13, v6
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v14, v15
+; CGP-NEXT:    v_add_i32_e64 v6, s[4:5], v12, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v13, v12
+; CGP-NEXT:    v_add_i32_e64 v4, s[4:5], v4, v12
+; CGP-NEXT:    v_addc_u32_e32 v1, vcc, v1, v4, vcc
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v6
+; CGP-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v4, v7, v0
+; CGP-NEXT:    v_mul_hi_u32 v6, v5, v0
+; CGP-NEXT:    v_mul_hi_u32 v0, v7, v0
+; CGP-NEXT:    v_mul_lo_u32 v12, v5, v1
+; CGP-NEXT:    v_mul_lo_u32 v13, v7, v1
+; CGP-NEXT:    v_mul_hi_u32 v14, v5, v1
+; CGP-NEXT:    v_mul_hi_u32 v1, v7, v1
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v13, v0
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v12, v4
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v13, v6
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
+; CGP-NEXT:    v_mul_lo_u32 v6, v10, v0
+; CGP-NEXT:    v_mul_lo_u32 v12, v11, v0
+; CGP-NEXT:    v_mul_hi_u32 v13, v10, v0
+; CGP-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
+; CGP-NEXT:    v_mul_lo_u32 v4, v10, v1
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, 1, v0
+; CGP-NEXT:    v_addc_u32_e32 v15, vcc, 0, v1, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v12, v4
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, 1, v14
+; CGP-NEXT:    v_addc_u32_e32 v16, vcc, 0, v15, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v13
+; CGP-NEXT:    v_sub_i32_e32 v6, vcc, v5, v6
+; CGP-NEXT:    v_subb_u32_e64 v13, s[4:5], v7, v4, vcc
+; CGP-NEXT:    v_sub_i32_e64 v4, s[4:5], v7, v4
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v6, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v13, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, -1, s[4:5]
+; CGP-NEXT:    v_subb_u32_e32 v4, vcc, v4, v11, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, v13, v11
+; CGP-NEXT:    v_cndmask_b32_e32 v7, v17, v7, vcc
+; CGP-NEXT:    v_sub_i32_e32 v6, vcc, v6, v10
+; CGP-NEXT:    v_subbrev_u32_e32 v4, vcc, 0, v4, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v6, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v4, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, -1, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v11
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v13, v6, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v14, v12, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v6, v15, v16, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; CGP-NEXT:  BB8_2: ; %Flow1
+; CGP-NEXT:    s_or_saveexec_b64 s[6:7], s[6:7]
+; CGP-NEXT:    s_xor_b64 exec, exec, s[6:7]
+; CGP-NEXT:    s_cbranch_execz BB8_4
+; CGP-NEXT:  ; %bb.3:
+; CGP-NEXT:    v_cvt_f32_u32_e32 v0, v10
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; CGP-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; CGP-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; CGP-NEXT:    v_mul_lo_u32 v1, v0, v10
+; CGP-NEXT:    v_mul_hi_u32 v4, v0, v10
+; CGP-NEXT:    v_sub_i32_e32 v6, vcc, 0, v1
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; CGP-NEXT:    v_mul_hi_u32 v1, v1, v0
+; CGP-NEXT:    v_add_i32_e64 v4, s[4:5], v0, v1
+; CGP-NEXT:    v_sub_i32_e64 v0, s[4:5], v0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; CGP-NEXT:    v_mul_hi_u32 v0, v0, v5
+; CGP-NEXT:    v_mul_lo_u32 v1, v0, v10
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
+; CGP-NEXT:    v_subrev_i32_e32 v6, vcc, 1, v0
+; CGP-NEXT:    v_sub_i32_e32 v7, vcc, v5, v1
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v5, v1
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v7, v10
+; CGP-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[4:5]
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; CGP-NEXT:    v_mov_b32_e32 v1, 0
+; CGP-NEXT:  BB8_4:
+; CGP-NEXT:    s_or_b64 exec, exec, s[6:7]
+; CGP-NEXT:    v_or_b32_e32 v5, v3, v9
+; CGP-NEXT:    v_mov_b32_e32 v4, 0
+; CGP-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[4:5]
+; CGP-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, 1
+; CGP-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; CGP-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; CGP-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
+; CGP-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; CGP-NEXT:    s_cbranch_execnz BB8_6
+; CGP-NEXT:  ; %bb.5:
+; CGP-NEXT:    v_cvt_f32_u32_e32 v4, v8
+; CGP-NEXT:    v_cvt_f32_u32_e32 v5, v9
+; CGP-NEXT:    v_sub_i32_e32 v6, vcc, 0, v8
+; CGP-NEXT:    v_subb_u32_e32 v7, vcc, 0, v9, vcc
+; CGP-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; CGP-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; CGP-NEXT:    v_mul_f32_e32 v5, 0x2f800000, v4
+; CGP-NEXT:    v_trunc_f32_e32 v5, v5
+; CGP-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v5
+; CGP-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; CGP-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; CGP-NEXT:    v_mul_lo_u32 v10, v6, v5
+; CGP-NEXT:    v_mul_lo_u32 v11, v6, v4
+; CGP-NEXT:    v_mul_lo_u32 v12, v7, v4
+; CGP-NEXT:    v_mul_hi_u32 v13, v6, v4
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; CGP-NEXT:    v_mul_lo_u32 v12, v5, v11
+; CGP-NEXT:    v_mul_hi_u32 v14, v4, v11
+; CGP-NEXT:    v_mul_hi_u32 v11, v5, v11
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v13
+; CGP-NEXT:    v_mul_lo_u32 v13, v4, v10
+; CGP-NEXT:    v_mul_lo_u32 v15, v5, v10
+; CGP-NEXT:    v_mul_hi_u32 v16, v4, v10
+; CGP-NEXT:    v_mul_hi_u32 v10, v5, v10
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v15, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v12, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v11, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v15, v14
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v11
+; CGP-NEXT:    v_addc_u32_e64 v11, s[4:5], v5, v10, vcc
+; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v10
+; CGP-NEXT:    v_mul_lo_u32 v10, v6, v4
+; CGP-NEXT:    v_mul_lo_u32 v7, v7, v4
+; CGP-NEXT:    v_mul_hi_u32 v12, v6, v4
+; CGP-NEXT:    v_mul_lo_u32 v6, v6, v11
+; CGP-NEXT:    v_mul_lo_u32 v13, v11, v10
+; CGP-NEXT:    v_mul_hi_u32 v14, v4, v10
+; CGP-NEXT:    v_mul_hi_u32 v10, v11, v10
+; CGP-NEXT:    v_add_i32_e64 v6, s[4:5], v7, v6
+; CGP-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v12
+; CGP-NEXT:    v_mul_lo_u32 v7, v4, v6
+; CGP-NEXT:    v_mul_lo_u32 v12, v11, v6
+; CGP-NEXT:    v_mul_hi_u32 v15, v4, v6
+; CGP-NEXT:    v_mul_hi_u32 v6, v11, v6
+; CGP-NEXT:    v_add_i32_e64 v7, s[4:5], v13, v7
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v12, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v7, s[4:5], v7, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v7, s[4:5], v11, v7
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v12, v13
+; CGP-NEXT:    v_add_i32_e64 v7, s[4:5], v10, v7
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v11, v10
+; CGP-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v10
+; CGP-NEXT:    v_addc_u32_e32 v5, vcc, v5, v6, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
+; CGP-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; CGP-NEXT:    v_mul_lo_u32 v6, v3, v4
+; CGP-NEXT:    v_mul_hi_u32 v7, v2, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT:    v_mul_lo_u32 v10, v2, v5
+; CGP-NEXT:    v_mul_lo_u32 v11, v3, v5
+; CGP-NEXT:    v_mul_hi_u32 v12, v2, v5
+; CGP-NEXT:    v_mul_hi_u32 v5, v3, v5
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v11, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
+; CGP-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v10, v6
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v11, v7
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CGP-NEXT:    v_mul_lo_u32 v7, v8, v4
+; CGP-NEXT:    v_mul_lo_u32 v10, v9, v4
+; CGP-NEXT:    v_mul_hi_u32 v11, v8, v4
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; CGP-NEXT:    v_mul_lo_u32 v6, v8, v5
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, 1, v4
+; CGP-NEXT:    v_addc_u32_e32 v13, vcc, 0, v5, vcc
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v10, v6
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, 1, v12
+; CGP-NEXT:    v_addc_u32_e32 v14, vcc, 0, v13, vcc
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v11
+; CGP-NEXT:    v_sub_i32_e32 v7, vcc, v2, v7
+; CGP-NEXT:    v_subb_u32_e64 v11, s[4:5], v3, v6, vcc
+; CGP-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v6
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v7, v8
+; CGP-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v11, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, -1, s[4:5]
+; CGP-NEXT:    v_subb_u32_e32 v3, vcc, v3, v9, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, v11, v9
+; CGP-NEXT:    v_cndmask_b32_e32 v6, v15, v6, vcc
+; CGP-NEXT:    v_sub_i32_e32 v7, vcc, v7, v8
+; CGP-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v7, v8
+; CGP-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, -1, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v9
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v11, v7, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v12, v10, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v7, v13, v14, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v4, v3, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
+; CGP-NEXT:  BB8_6: ; %Flow
+; CGP-NEXT:    s_or_saveexec_b64 s[6:7], s[6:7]
+; CGP-NEXT:    s_xor_b64 exec, exec, s[6:7]
+; CGP-NEXT:    s_cbranch_execz BB8_8
+; CGP-NEXT:  ; %bb.7:
+; CGP-NEXT:    v_cvt_f32_u32_e32 v3, v8
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; CGP-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v3
+; CGP-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; CGP-NEXT:    v_mul_lo_u32 v4, v3, v8
+; CGP-NEXT:    v_mul_hi_u32 v5, v3, v8
+; CGP-NEXT:    v_sub_i32_e32 v6, vcc, 0, v4
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; CGP-NEXT:    v_mul_hi_u32 v4, v4, v3
+; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v3, v4
+; CGP-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v4
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; CGP-NEXT:    v_mul_hi_u32 v3, v3, v2
+; CGP-NEXT:    v_mul_lo_u32 v4, v3, v8
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, 1, v3
+; CGP-NEXT:    v_subrev_i32_e32 v6, vcc, 1, v3
+; CGP-NEXT:    v_sub_i32_e32 v7, vcc, v2, v4
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v4
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v7, v8
+; CGP-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v2, v5, v3, s[4:5]
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v6, v2, vcc
+; CGP-NEXT:    v_mov_b32_e32 v5, 0
+; CGP-NEXT:  BB8_8:
+; CGP-NEXT:    s_or_b64 exec, exec, s[6:7]
+; CGP-NEXT:    v_mov_b32_e32 v2, v4
+; CGP-NEXT:    v_mov_b32_e32 v3, v5
+; CGP-NEXT:    s_setpc_b64 s[30:31]
+  %shl.y = shl <2 x i64> <i64 4096, i64 4096>, %y
+  %r = udiv <2 x i64> %x, %shl.y
+  ret <2 x i64> %r
+}
+
+define i64 @v_udiv_i64_24bit(i64 %num, i64 %den) {
+; GISEL-LABEL: v_udiv_i64_24bit:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    s_mov_b32 s4, 0xffffff
+; GISEL-NEXT:    v_and_b32_e32 v0, s4, v0
+; GISEL-NEXT:    v_and_b32_e32 v1, s4, v2
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v2, v1
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v2
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GISEL-NEXT:    v_mul_lo_u32 v3, v2, v1
+; GISEL-NEXT:    v_mul_hi_u32 v4, v2, v1
+; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, 0, v3
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GISEL-NEXT:    v_mul_hi_u32 v3, v3, v2
+; GISEL-NEXT:    v_add_i32_e64 v4, s[4:5], v2, v3
+; GISEL-NEXT:    v_sub_i32_e64 v2, s[4:5], v2, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GISEL-NEXT:    v_mul_hi_u32 v2, v2, v0
+; GISEL-NEXT:    v_mul_lo_u32 v3, v2, v1
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
+; GISEL-NEXT:    v_subrev_i32_e32 v5, vcc, 1, v2
+; GISEL-NEXT:    v_sub_i32_e32 v6, vcc, v0, v3
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v3
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v6, v1
+; GISEL-NEXT:    s_and_b64 s[4:5], s[4:5], vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v0, v4, v2, s[4:5]
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
+; GISEL-NEXT:    v_mov_b32_e32 v1, 0
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_udiv_i64_24bit:
+; CGP:       ; %bb.0:
+; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT:    s_mov_b32 s4, 0xffffff
+; CGP-NEXT:    v_and_b32_e32 v0, s4, v0
+; CGP-NEXT:    v_and_b32_e32 v1, s4, v2
+; CGP-NEXT:    v_cvt_f32_u32_e32 v0, v0
+; CGP-NEXT:    v_cvt_f32_u32_e32 v1, v1
+; CGP-NEXT:    v_rcp_f32_e32 v2, v1
+; CGP-NEXT:    v_mul_f32_e32 v2, v0, v2
+; CGP-NEXT:    v_trunc_f32_e32 v2, v2
+; CGP-NEXT:    v_mad_f32 v0, -v2, v1, v0
+; CGP-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; CGP-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v0|, v1
+; CGP-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v2, v0
+; CGP-NEXT:    v_and_b32_e32 v0, 0xffffff, v0
+; CGP-NEXT:    v_mov_b32_e32 v1, 0
+; CGP-NEXT:    s_setpc_b64 s[30:31]
+  %num.mask = and i64 %num, 16777215
+  %den.mask = and i64 %den, 16777215
+  %result = udiv i64 %num.mask, %den.mask
+  ret i64 %result
+}
+
+define <2 x i64> @v_udiv_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
+; GISEL-LABEL: v_udiv_v2i64_24bit:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    s_mov_b32 s6, 0xffffff
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v1, 0
+; GISEL-NEXT:    v_and_b32_e32 v3, s6, v4
+; GISEL-NEXT:    v_and_b32_e32 v4, s6, v6
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v5, v3
+; GISEL-NEXT:    v_sub_i32_e32 v6, vcc, 0, v3
+; GISEL-NEXT:    v_subb_u32_e64 v7, s[4:5], 0, 0, vcc
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v8, v4
+; GISEL-NEXT:    v_sub_i32_e32 v9, vcc, 0, v4
+; GISEL-NEXT:    v_subb_u32_e64 v10, s[4:5], 0, 0, vcc
+; GISEL-NEXT:    v_mac_f32_e32 v5, 0x4f800000, v1
+; GISEL-NEXT:    v_mac_f32_e32 v8, 0x4f800000, v1
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v1, v5
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v5, v8
+; GISEL-NEXT:    v_mul_f32_e32 v1, 0x5f7ffffc, v1
+; GISEL-NEXT:    v_mul_f32_e32 v5, 0x5f7ffffc, v5
+; GISEL-NEXT:    v_mul_f32_e32 v8, 0x2f800000, v1
+; GISEL-NEXT:    v_mul_f32_e32 v11, 0x2f800000, v5
+; GISEL-NEXT:    v_trunc_f32_e32 v8, v8
+; GISEL-NEXT:    v_trunc_f32_e32 v11, v11
+; GISEL-NEXT:    v_mac_f32_e32 v1, 0xcf800000, v8
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v8, v8
+; GISEL-NEXT:    v_mac_f32_e32 v5, 0xcf800000, v11
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v11, v11
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GISEL-NEXT:    v_mul_lo_u32 v12, v6, v8
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; GISEL-NEXT:    v_mul_lo_u32 v13, v9, v11
+; GISEL-NEXT:    v_mul_lo_u32 v14, v6, v1
+; GISEL-NEXT:    v_mul_lo_u32 v15, v7, v1
+; GISEL-NEXT:    v_mul_hi_u32 v16, v6, v1
+; GISEL-NEXT:    v_mul_lo_u32 v17, v9, v5
+; GISEL-NEXT:    v_mul_lo_u32 v18, v10, v5
+; GISEL-NEXT:    v_mul_hi_u32 v19, v9, v5
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v15, v12
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v18, v13
+; GISEL-NEXT:    v_mul_lo_u32 v15, v11, v17
+; GISEL-NEXT:    v_mul_hi_u32 v18, v5, v17
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v19
+; GISEL-NEXT:    v_mul_lo_u32 v19, v5, v13
+; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v15, v19
+; GISEL-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v15, v18
+; GISEL-NEXT:    v_mul_lo_u32 v15, v8, v14
+; GISEL-NEXT:    v_mul_hi_u32 v18, v1, v14
+; GISEL-NEXT:    v_mul_hi_u32 v14, v8, v14
+; GISEL-NEXT:    v_mul_hi_u32 v17, v11, v17
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v16
+; GISEL-NEXT:    v_mul_lo_u32 v16, v1, v12
+; GISEL-NEXT:    v_add_i32_e64 v15, s[4:5], v15, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v15, s[4:5], v15, v18
+; GISEL-NEXT:    v_mul_lo_u32 v15, v8, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v18, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v16, s[4:5], v16, v18
+; GISEL-NEXT:    v_mul_hi_u32 v18, v1, v12
+; GISEL-NEXT:    v_add_i32_e64 v14, s[4:5], v15, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v14, s[4:5], v14, v18
+; GISEL-NEXT:    v_cndmask_b32_e64 v18, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v15, s[4:5], v15, v18
+; GISEL-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v18, vcc, v19, v18
+; GISEL-NEXT:    v_mul_lo_u32 v19, v11, v13
+; GISEL-NEXT:    v_add_i32_e32 v17, vcc, v19, v17
+; GISEL-NEXT:    v_mul_hi_u32 v19, v5, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v17, vcc, v17, v19
+; GISEL-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v19, vcc, v20, v19
+; GISEL-NEXT:    v_and_b32_e32 v0, s6, v0
+; GISEL-NEXT:    v_and_b32_e32 v2, s6, v2
+; GISEL-NEXT:    v_mul_hi_u32 v12, v8, v12
+; GISEL-NEXT:    v_mul_hi_u32 v13, v11, v13
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v17, vcc, v17, v18
+; GISEL-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v15, v16
+; GISEL-NEXT:    v_add_i32_e32 v16, vcc, v19, v18
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v15
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v16
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v1, v14
+; GISEL-NEXT:    v_addc_u32_e64 v14, s[4:5], v8, v12, vcc
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v12
+; GISEL-NEXT:    v_mul_lo_u32 v12, v6, v1
+; GISEL-NEXT:    v_mul_lo_u32 v7, v7, v1
+; GISEL-NEXT:    v_mul_hi_u32 v15, v6, v1
+; GISEL-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v17
+; GISEL-NEXT:    v_addc_u32_e64 v16, s[6:7], v11, v13, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v13
+; GISEL-NEXT:    v_mul_lo_u32 v13, v9, v5
+; GISEL-NEXT:    v_mul_lo_u32 v10, v10, v5
+; GISEL-NEXT:    v_mul_hi_u32 v17, v9, v5
+; GISEL-NEXT:    v_mul_lo_u32 v6, v6, v14
+; GISEL-NEXT:    v_mul_lo_u32 v18, v14, v12
+; GISEL-NEXT:    v_mul_hi_u32 v19, v1, v12
+; GISEL-NEXT:    v_mul_hi_u32 v12, v14, v12
+; GISEL-NEXT:    v_mul_lo_u32 v9, v9, v16
+; GISEL-NEXT:    v_add_i32_e64 v6, s[6:7], v7, v6
+; GISEL-NEXT:    v_mul_lo_u32 v7, v16, v13
+; GISEL-NEXT:    v_add_i32_e64 v9, s[6:7], v10, v9
+; GISEL-NEXT:    v_mul_hi_u32 v10, v5, v13
+; GISEL-NEXT:    v_mul_hi_u32 v13, v16, v13
+; GISEL-NEXT:    v_add_i32_e64 v6, s[6:7], v6, v15
+; GISEL-NEXT:    v_add_i32_e64 v9, s[6:7], v9, v17
+; GISEL-NEXT:    v_mul_lo_u32 v15, v1, v6
+; GISEL-NEXT:    v_mul_lo_u32 v17, v5, v9
+; GISEL-NEXT:    v_add_i32_e64 v7, s[6:7], v7, v17
+; GISEL-NEXT:    v_cndmask_b32_e64 v17, 0, 1, s[6:7]
+; GISEL-NEXT:    v_add_i32_e64 v7, s[6:7], v7, v10
+; GISEL-NEXT:    v_mul_lo_u32 v7, v14, v6
+; GISEL-NEXT:    v_mul_hi_u32 v10, v1, v6
+; GISEL-NEXT:    v_mul_hi_u32 v6, v14, v6
+; GISEL-NEXT:    v_mul_lo_u32 v14, v16, v9
+; GISEL-NEXT:    v_mul_hi_u32 v16, v16, v9
+; GISEL-NEXT:    v_mul_hi_u32 v9, v5, v9
+; GISEL-NEXT:    v_add_i32_e64 v15, s[8:9], v18, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v18, 0, 1, s[8:9]
+; GISEL-NEXT:    v_add_i32_e64 v7, s[8:9], v7, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[8:9]
+; GISEL-NEXT:    v_add_i32_e64 v13, s[8:9], v14, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[8:9]
+; GISEL-NEXT:    v_add_i32_e64 v15, s[8:9], v15, v19
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[8:9]
+; GISEL-NEXT:    v_add_i32_e64 v7, s[8:9], v7, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s[8:9]
+; GISEL-NEXT:    v_cndmask_b32_e64 v19, 0, 1, s[6:7]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[6:7], v13, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[6:7]
+; GISEL-NEXT:    v_add_i32_e64 v15, s[6:7], v18, v15
+; GISEL-NEXT:    v_add_i32_e64 v10, s[6:7], v12, v10
+; GISEL-NEXT:    v_add_i32_e64 v12, s[6:7], v17, v19
+; GISEL-NEXT:    v_add_i32_e64 v13, s[6:7], v14, v13
+; GISEL-NEXT:    v_add_i32_e64 v7, s[6:7], v7, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[6:7]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[6:7], v9, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[6:7]
+; GISEL-NEXT:    v_add_i32_e64 v10, s[6:7], v10, v14
+; GISEL-NEXT:    v_add_i32_e64 v12, s[6:7], v13, v12
+; GISEL-NEXT:    v_add_i32_e64 v6, s[6:7], v6, v10
+; GISEL-NEXT:    v_add_i32_e64 v10, s[6:7], v16, v12
+; GISEL-NEXT:    v_addc_u32_e32 v6, vcc, v8, v6, vcc
+; GISEL-NEXT:    v_addc_u32_e64 v8, vcc, v11, v10, s[4:5]
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v1, v7
+; GISEL-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v7, 0, v1
+; GISEL-NEXT:    v_mul_hi_u32 v10, v0, v1
+; GISEL-NEXT:    v_mul_hi_u32 v1, 0, v1
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
+; GISEL-NEXT:    v_addc_u32_e32 v8, vcc, 0, v8, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v9, 0, v5
+; GISEL-NEXT:    v_mul_hi_u32 v11, v2, v5
+; GISEL-NEXT:    v_mul_hi_u32 v5, 0, v5
+; GISEL-NEXT:    v_mul_lo_u32 v12, v0, v6
+; GISEL-NEXT:    v_mul_lo_u32 v13, 0, v6
+; GISEL-NEXT:    v_mul_hi_u32 v14, v0, v6
+; GISEL-NEXT:    v_mul_hi_u32 v6, 0, v6
+; GISEL-NEXT:    v_mul_lo_u32 v15, v2, v8
+; GISEL-NEXT:    v_mul_lo_u32 v16, 0, v8
+; GISEL-NEXT:    v_mul_hi_u32 v17, v2, v8
+; GISEL-NEXT:    v_mul_hi_u32 v8, 0, v8
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v13, v1
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v16, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v1, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v17
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v12, v7
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v13, v10
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v15, v9
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v16, v11
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v1, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v10, v7
+; GISEL-NEXT:    v_mul_lo_u32 v10, v3, v1
+; GISEL-NEXT:    v_mul_lo_u32 v12, 0, v1
+; GISEL-NEXT:    v_mul_hi_u32 v13, v3, v1
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
+; GISEL-NEXT:    v_mul_lo_u32 v11, v4, v5
+; GISEL-NEXT:    v_mul_lo_u32 v14, 0, v5
+; GISEL-NEXT:    v_mul_hi_u32 v15, v4, v5
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v8, v9
+; GISEL-NEXT:    v_mul_lo_u32 v8, v3, v6
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, 1, v1
+; GISEL-NEXT:    v_addc_u32_e32 v16, vcc, 0, v6, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v17, v4, v7
+; GISEL-NEXT:    v_add_i32_e32 v18, vcc, 1, v5
+; GISEL-NEXT:    v_addc_u32_e32 v19, vcc, 0, v7, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v12, v8
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v14, v17
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v13
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v15
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v10
+; GISEL-NEXT:    v_subb_u32_e64 v10, s[4:5], 0, v8, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v3
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, -1, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e64 v2, s[4:5], v2, v11
+; GISEL-NEXT:    v_subb_u32_e64 v11, s[6:7], 0, v12, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[6:7], v2, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, -1, s[6:7]
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[6:7], 0, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, -1, s[6:7]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[6:7], 0, v10
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[8:9], 0, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[8:9]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[8:9], 0, v11
+; GISEL-NEXT:    v_add_i32_e64 v11, s[10:11], 1, v9
+; GISEL-NEXT:    v_addc_u32_e64 v17, s[10:11], 0, v16, s[10:11]
+; GISEL-NEXT:    v_sub_i32_e64 v8, s[10:11], 0, v8
+; GISEL-NEXT:    v_subbrev_u32_e32 v8, vcc, 0, v8, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v3
+; GISEL-NEXT:    v_subbrev_u32_e32 v8, vcc, 0, v8, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v3
+; GISEL-NEXT:    v_add_i32_e64 v0, s[10:11], 1, v18
+; GISEL-NEXT:    v_addc_u32_e64 v3, s[10:11], 0, v19, s[10:11]
+; GISEL-NEXT:    v_sub_i32_e64 v12, s[10:11], 0, v12
+; GISEL-NEXT:    v_subbrev_u32_e64 v12, s[4:5], 0, v12, s[4:5]
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, v15, v13, s[6:7]
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, -1, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, v10, v14, s[8:9]
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v4
+; GISEL-NEXT:    v_subbrev_u32_e32 v12, vcc, 0, v12, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, 0, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v4, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, 0, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v8
+; GISEL-NEXT:    v_cndmask_b32_e32 v4, v4, v15, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v12
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v14, v2, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; GISEL-NEXT:    v_cndmask_b32_e32 v4, v9, v11, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v2
+; GISEL-NEXT:    v_cndmask_b32_e64 v2, v18, v0, s[4:5]
+; GISEL-NEXT:    v_cndmask_b32_e32 v8, v16, v17, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v13
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v1, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v3, v19, v3, s[4:5]
+; GISEL-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[4:5]
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v6, v8, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_udiv_v2i64_24bit:
+; CGP:       ; %bb.0:
+; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT:    s_mov_b32 s6, 0xffffff
+; CGP-NEXT:    v_mov_b32_e32 v1, 0
+; CGP-NEXT:    v_and_b32_e32 v0, s6, v0
+; CGP-NEXT:    v_and_b32_e32 v2, s6, v2
+; CGP-NEXT:    v_and_b32_e32 v3, s6, v4
+; CGP-NEXT:    v_and_b32_e32 v4, s6, v6
+; CGP-NEXT:    v_cvt_f32_u32_e32 v0, v0
+; CGP-NEXT:    v_cvt_f32_u32_e32 v3, v3
+; CGP-NEXT:    v_cvt_f32_u32_e32 v2, v2
+; CGP-NEXT:    v_cvt_f32_u32_e32 v4, v4
+; CGP-NEXT:    v_rcp_f32_e32 v5, v3
+; CGP-NEXT:    v_rcp_f32_e32 v6, v4
+; CGP-NEXT:    v_mul_f32_e32 v5, v0, v5
+; CGP-NEXT:    v_mul_f32_e32 v6, v2, v6
+; CGP-NEXT:    v_trunc_f32_e32 v5, v5
+; CGP-NEXT:    v_trunc_f32_e32 v6, v6
+; CGP-NEXT:    v_mad_f32 v0, -v5, v3, v0
+; CGP-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; CGP-NEXT:    v_mad_f32 v2, -v6, v4, v2
+; CGP-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; CGP-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v0|, v3
+; CGP-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; CGP-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v2|, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v5, v0
+; CGP-NEXT:    v_add_i32_e32 v2, vcc, v6, v2
+; CGP-NEXT:    v_and_b32_e32 v0, s6, v0
+; CGP-NEXT:    v_and_b32_e32 v2, s6, v2
+; CGP-NEXT:    v_mov_b32_e32 v3, v1
+; CGP-NEXT:    s_setpc_b64 s[30:31]
+  %num.mask = and <2 x i64> %num, <i64 16777215, i64 16777215>
+  %den.mask = and <2 x i64> %den, <i64 16777215, i64 16777215>
+  %result = udiv <2 x i64> %num.mask, %den.mask
+  ret <2 x i64> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
new file mode 100644
index 0000000000000..78f03210fb885
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
@@ -0,0 +1,3427 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel -amdgpu-codegenprepare-disable-idiv-expansion=1 -mtriple=amdgcn-amd-amdhsa -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefixes=CHECK,GISEL %s
+; RUN: llc -global-isel -amdgpu-codegenprepare-disable-idiv-expansion=0 -mtriple=amdgcn-amd-amdhsa -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefixes=CHECK,CGP %s
+
+; The same 32-bit expansion is implemented in the legalizer and in AMDGPUCodeGenPrepare.
+
+define i64 @v_urem_i64(i64 %num, i64 %den) {
+; CHECK-LABEL: v_urem_i64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_or_b32_e32 v5, v1, v3
+; CHECK-NEXT:    v_mov_b32_e32 v4, 0
+; CHECK-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[4:5]
+; CHECK-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, 1
+; CHECK-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; CHECK-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; CHECK-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
+; CHECK-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; CHECK-NEXT:    s_cbranch_execnz BB0_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v4, v2
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v5, v3
+; CHECK-NEXT:    v_sub_i32_e32 v6, vcc, 0, v2
+; CHECK-NEXT:    v_subb_u32_e32 v7, vcc, 0, v3, vcc
+; CHECK-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; CHECK-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; CHECK-NEXT:    v_mul_f32_e32 v5, 0x2f800000, v4
+; CHECK-NEXT:    v_trunc_f32_e32 v5, v5
+; CHECK-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v5
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; CHECK-NEXT:    v_mul_lo_u32 v8, v6, v5
+; CHECK-NEXT:    v_mul_lo_u32 v9, v6, v4
+; CHECK-NEXT:    v_mul_lo_u32 v10, v7, v4
+; CHECK-NEXT:    v_mul_hi_u32 v11, v6, v4
+; CHECK-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
+; CHECK-NEXT:    v_mul_lo_u32 v10, v5, v9
+; CHECK-NEXT:    v_mul_hi_u32 v12, v4, v9
+; CHECK-NEXT:    v_mul_hi_u32 v9, v5, v9
+; CHECK-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
+; CHECK-NEXT:    v_mul_lo_u32 v11, v4, v8
+; CHECK-NEXT:    v_mul_lo_u32 v13, v5, v8
+; CHECK-NEXT:    v_mul_hi_u32 v14, v4, v8
+; CHECK-NEXT:    v_mul_hi_u32 v8, v5, v8
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; CHECK-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v9, vcc, v13, v9
+; CHECK-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v9, vcc, v9, v14
+; CHECK-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; CHECK-NEXT:    v_add_i32_e32 v11, vcc, v13, v12
+; CHECK-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; CHECK-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v9
+; CHECK-NEXT:    v_addc_u32_e64 v9, s[4:5], v5, v8, vcc
+; CHECK-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v8
+; CHECK-NEXT:    v_mul_lo_u32 v8, v6, v4
+; CHECK-NEXT:    v_mul_lo_u32 v7, v7, v4
+; CHECK-NEXT:    v_mul_hi_u32 v10, v6, v4
+; CHECK-NEXT:    v_mul_lo_u32 v6, v6, v9
+; CHECK-NEXT:    v_mul_lo_u32 v11, v9, v8
+; CHECK-NEXT:    v_mul_hi_u32 v12, v4, v8
+; CHECK-NEXT:    v_mul_hi_u32 v8, v9, v8
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v7, v6
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v10
+; CHECK-NEXT:    v_mul_lo_u32 v7, v4, v6
+; CHECK-NEXT:    v_mul_lo_u32 v10, v9, v6
+; CHECK-NEXT:    v_mul_hi_u32 v13, v4, v6
+; CHECK-NEXT:    v_mul_hi_u32 v6, v9, v6
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v11, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v8, s[4:5], v10, v8
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v7, v12
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v13
+; CHECK-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v9, v7
+; CHECK-NEXT:    v_add_i32_e64 v9, s[4:5], v10, v11
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v8, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v8, s[4:5], v9, v8
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v8
+; CHECK-NEXT:    v_addc_u32_e32 v5, vcc, v5, v6, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
+; CHECK-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v6, v1, v4
+; CHECK-NEXT:    v_mul_hi_u32 v7, v0, v4
+; CHECK-NEXT:    v_mul_hi_u32 v4, v1, v4
+; CHECK-NEXT:    v_mul_lo_u32 v8, v0, v5
+; CHECK-NEXT:    v_mul_lo_u32 v9, v1, v5
+; CHECK-NEXT:    v_mul_hi_u32 v10, v0, v5
+; CHECK-NEXT:    v_mul_hi_u32 v5, v1, v5
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v9, v4
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v9, v7
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT:    v_mul_lo_u32 v7, v2, v4
+; CHECK-NEXT:    v_mul_lo_u32 v8, v3, v4
+; CHECK-NEXT:    v_mul_hi_u32 v4, v2, v4
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT:    v_mul_lo_u32 v5, v2, v5
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; CHECK-NEXT:    v_sub_i32_e32 v5, vcc, v0, v7
+; CHECK-NEXT:    v_subb_u32_e64 v6, s[4:5], v1, v4, vcc
+; CHECK-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v4
+; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v5, v2
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[4:5]
+; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v6, v3
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[4:5]
+; CHECK-NEXT:    v_subb_u32_e32 v1, vcc, v1, v3, vcc
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, v6, v3
+; CHECK-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
+; CHECK-NEXT:    v_sub_i32_e32 v7, vcc, v5, v2
+; CHECK-NEXT:    v_subbrev_u32_e64 v8, s[4:5], 0, v1, vcc
+; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v7, v2
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; CHECK-NEXT:    v_subb_u32_e32 v1, vcc, v1, v3, vcc
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v8, v3
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, -1, vcc
+; CHECK-NEXT:    v_sub_i32_e32 v11, vcc, v7, v2
+; CHECK-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, v8, v3
+; CHECK-NEXT:    v_cndmask_b32_e32 v3, v10, v9, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
+; CHECK-NEXT:    v_cndmask_b32_e32 v3, v7, v11, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; CHECK-NEXT:    v_cndmask_b32_e32 v4, v5, v3, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v5, v6, v1, vcc
+; CHECK-NEXT:  BB0_2: ; %Flow
+; CHECK-NEXT:    s_or_saveexec_b64 s[8:9], s[6:7]
+; CHECK-NEXT:    s_xor_b64 exec, exec, s[8:9]
+; CHECK-NEXT:    s_cbranch_execz BB0_4
+; CHECK-NEXT:  ; %bb.3:
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v1, v2
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v1, v1
+; CHECK-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v1
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; CHECK-NEXT:    v_mul_lo_u32 v3, v1, v2
+; CHECK-NEXT:    v_mul_hi_u32 v4, v1, v2
+; CHECK-NEXT:    v_sub_i32_e32 v5, vcc, 0, v3
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; CHECK-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; CHECK-NEXT:    v_mul_hi_u32 v3, v3, v1
+; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v1, v3
+; CHECK-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v3
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
+; CHECK-NEXT:    v_mul_hi_u32 v1, v1, v0
+; CHECK-NEXT:    v_mul_lo_u32 v1, v1, v2
+; CHECK-NEXT:    v_sub_i32_e32 v3, vcc, v0, v1
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v2
+; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v3, v2
+; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v1
+; CHECK-NEXT:    v_sub_i32_e64 v0, s[6:7], v3, v2
+; CHECK-NEXT:    s_and_b64 vcc, vcc, s[4:5]
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, v4, v0, s[4:5]
+; CHECK-NEXT:    v_mov_b32_e32 v5, 0
+; CHECK-NEXT:  BB0_4:
+; CHECK-NEXT:    s_or_b64 exec, exec, s[8:9]
+; CHECK-NEXT:    v_mov_b32_e32 v0, v4
+; CHECK-NEXT:    v_mov_b32_e32 v1, v5
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %result = urem i64 %num, %den
+  ret i64 %result
+}
+
+; FIXME: This is a workaround for not handling uniform VGPR case.
+declare i32 @llvm.amdgcn.readfirstlane(i32)
+
+define amdgpu_ps i64 @s_urem_i64(i64 inreg %num, i64 inreg %den) {
+; CHECK-LABEL: s_urem_i64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_or_b64 s[4:5], s[0:1], s[2:3]
+; CHECK-NEXT:    s_mov_b32 s6, 0
+; CHECK-NEXT:    s_mov_b32 s7, -1
+; CHECK-NEXT:    s_and_b64 s[4:5], s[4:5], s[6:7]
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v0, s2
+; CHECK-NEXT:    v_cmp_ne_u64_e64 vcc, s[4:5], 0
+; CHECK-NEXT:    s_cbranch_vccnz BB1_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; CHECK-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; CHECK-NEXT:    v_mul_lo_u32 v1, v0, s2
+; CHECK-NEXT:    v_mul_hi_u32 v2, v0, s2
+; CHECK-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; CHECK-NEXT:    v_mul_hi_u32 v1, v1, v0
+; CHECK-NEXT:    v_add_i32_e64 v2, s[4:5], v0, v1
+; CHECK-NEXT:    v_sub_i32_e64 v0, s[4:5], v0, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; CHECK-NEXT:    v_mul_hi_u32 v0, v0, s0
+; CHECK-NEXT:    v_mul_lo_u32 v0, v0, s2
+; CHECK-NEXT:    v_sub_i32_e32 v1, vcc, s0, v0
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s2, v1
+; CHECK-NEXT:    v_add_i32_e64 v2, s[4:5], s2, v1
+; CHECK-NEXT:    v_cmp_ge_u32_e64 s[0:1], s0, v0
+; CHECK-NEXT:    v_subrev_i32_e64 v0, s[2:3], s2, v1
+; CHECK-NEXT:    s_and_b64 vcc, vcc, s[0:1]
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; CHECK-NEXT:    v_cndmask_b32_e64 v0, v2, v0, s[0:1]
+; CHECK-NEXT:    s_branch BB1_3
+; CHECK-NEXT:  BB1_2:
+; CHECK-NEXT:    v_mov_b32_e32 v1, s3
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v2, s3
+; CHECK-NEXT:    s_sub_u32 s6, 0, s2
+; CHECK-NEXT:    s_cselect_b32 s4, 1, 0
+; CHECK-NEXT:    v_mov_b32_e32 v3, s1
+; CHECK-NEXT:    v_mac_f32_e32 v0, 0x4f800000, v2
+; CHECK-NEXT:    s_and_b32 s4, s4, 1
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; CHECK-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
+; CHECK-NEXT:    s_cmp_lg_u32 s4, 0
+; CHECK-NEXT:    s_subb_u32 s7, 0, s3
+; CHECK-NEXT:    v_mul_f32_e32 v2, 0x2f800000, v0
+; CHECK-NEXT:    v_trunc_f32_e32 v2, v2
+; CHECK-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v2
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; CHECK-NEXT:    v_mul_lo_u32 v4, s6, v2
+; CHECK-NEXT:    v_mul_lo_u32 v5, s6, v0
+; CHECK-NEXT:    v_mul_lo_u32 v6, s7, v0
+; CHECK-NEXT:    v_mul_hi_u32 v7, s6, v0
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
+; CHECK-NEXT:    v_mul_lo_u32 v6, v2, v5
+; CHECK-NEXT:    v_mul_hi_u32 v8, v0, v5
+; CHECK-NEXT:    v_mul_hi_u32 v5, v2, v5
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
+; CHECK-NEXT:    v_mul_lo_u32 v7, v0, v4
+; CHECK-NEXT:    v_mul_lo_u32 v9, v2, v4
+; CHECK-NEXT:    v_mul_hi_u32 v10, v0, v4
+; CHECK-NEXT:    v_mul_hi_u32 v4, v2, v4
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v9, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v9, v8
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v0, v5
+; CHECK-NEXT:    v_addc_u32_e64 v5, s[4:5], v2, v4, vcc
+; CHECK-NEXT:    v_add_i32_e64 v2, s[4:5], v2, v4
+; CHECK-NEXT:    v_mul_lo_u32 v4, s6, v0
+; CHECK-NEXT:    v_mul_lo_u32 v6, s7, v0
+; CHECK-NEXT:    v_mul_hi_u32 v7, s6, v0
+; CHECK-NEXT:    v_mul_lo_u32 v8, s6, v5
+; CHECK-NEXT:    v_mul_lo_u32 v9, v5, v4
+; CHECK-NEXT:    v_mul_hi_u32 v10, v0, v4
+; CHECK-NEXT:    v_mul_hi_u32 v4, v5, v4
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v8
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v7
+; CHECK-NEXT:    v_mul_lo_u32 v7, v0, v6
+; CHECK-NEXT:    v_mul_lo_u32 v8, v5, v6
+; CHECK-NEXT:    v_mul_hi_u32 v11, v0, v6
+; CHECK-NEXT:    v_mul_hi_u32 v5, v5, v6
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v9, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v8, v4
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v4, v11
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v7, v6
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v8, v9
+; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v4, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v7, v6
+; CHECK-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v6
+; CHECK-NEXT:    v_addc_u32_e32 v2, vcc, v2, v5, vcc
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; CHECK-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v4, s1, v0
+; CHECK-NEXT:    v_mul_hi_u32 v5, s0, v0
+; CHECK-NEXT:    v_mul_hi_u32 v0, s1, v0
+; CHECK-NEXT:    v_mul_lo_u32 v6, s0, v2
+; CHECK-NEXT:    v_mul_lo_u32 v7, s1, v2
+; CHECK-NEXT:    v_mul_hi_u32 v8, s0, v2
+; CHECK-NEXT:    v_mul_hi_u32 v2, s1, v2
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v7, v0
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v0, v8
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; CHECK-NEXT:    v_mul_lo_u32 v5, s2, v0
+; CHECK-NEXT:    v_mul_lo_u32 v6, s3, v0
+; CHECK-NEXT:    v_mul_hi_u32 v0, s2, v0
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; CHECK-NEXT:    v_mul_lo_u32 v2, s2, v2
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v6, v2
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v2, v0
+; CHECK-NEXT:    v_sub_i32_e32 v2, vcc, s0, v5
+; CHECK-NEXT:    v_subb_u32_e64 v3, s[4:5], v3, v0, vcc
+; CHECK-NEXT:    v_sub_i32_e64 v0, s[0:1], s1, v0
+; CHECK-NEXT:    v_cmp_le_u32_e64 s[0:1], s2, v2
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[0:1]
+; CHECK-NEXT:    v_cmp_le_u32_e64 s[0:1], s3, v3
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[0:1]
+; CHECK-NEXT:    v_subb_u32_e32 v0, vcc, v0, v1, vcc
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, s3, v3
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v5, v4, vcc
+; CHECK-NEXT:    v_subrev_i32_e32 v3, vcc, s2, v2
+; CHECK-NEXT:    v_subbrev_u32_e32 v0, vcc, 0, v0, vcc
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s2, v3
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, -1, vcc
+; CHECK-NEXT:    v_subrev_i32_e32 v5, vcc, s2, v3
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s3, v0
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, s3, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v6, v4, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v3, v5, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; CHECK-NEXT:  BB1_3:
+; CHECK-NEXT:    v_readfirstlane_b32 s0, v0
+; CHECK-NEXT:    s_mov_b32 s1, s0
+; CHECK-NEXT:    ; return to shader part epilog
+  %result = urem i64 %num, %den
+  %cast = bitcast i64 %result to <2 x i32>
+  %elt.0 = extractelement <2 x i32> %cast, i32 0
+  %elt.1 = extractelement <2 x i32> %cast, i32 1
+  %res.0 = call i32 @llvm.amdgcn.readfirstlane(i32 %elt.0)
+  %res.1 = call i32 @llvm.amdgcn.readfirstlane(i32 %elt.1)
+  %ins.0 = insertelement <2 x i32> undef, i32 %res.0, i32 0
+  %ins.1 = insertelement <2 x i32> %ins.0, i32 %res.0, i32 1
+  %cast.back = bitcast <2 x i32> %ins.1 to i64
+  ret i64 %cast.back
+}
+
+define <2 x i64> @v_urem_v2i64(<2 x i64> %num, <2 x i64> %den) {
+; GISEL-LABEL: v_urem_v2i64:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v8, v4
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v9, v5
+; GISEL-NEXT:    v_mac_f32_e32 v8, 0x4f800000, v9
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v8, v8
+; GISEL-NEXT:    v_mul_f32_e32 v8, 0x5f7ffffc, v8
+; GISEL-NEXT:    v_mul_f32_e32 v9, 0x2f800000, v8
+; GISEL-NEXT:    v_trunc_f32_e32 v9, v9
+; GISEL-NEXT:    v_mac_f32_e32 v8, 0xcf800000, v9
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v8, v8
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v9, v9
+; GISEL-NEXT:    v_sub_i32_e32 v10, vcc, 0, v4
+; GISEL-NEXT:    v_subb_u32_e32 v11, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v12, v10, v8
+; GISEL-NEXT:    v_mul_lo_u32 v13, v11, v8
+; GISEL-NEXT:    v_mul_lo_u32 v14, v10, v9
+; GISEL-NEXT:    v_mul_hi_u32 v15, v10, v8
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v15
+; GISEL-NEXT:    v_mul_lo_u32 v14, v9, v12
+; GISEL-NEXT:    v_mul_lo_u32 v15, v8, v13
+; GISEL-NEXT:    v_mul_hi_u32 v16, v8, v12
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; GISEL-NEXT:    v_mul_lo_u32 v15, v9, v13
+; GISEL-NEXT:    v_mul_hi_u32 v12, v9, v12
+; GISEL-NEXT:    v_mul_hi_u32 v16, v8, v13
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v15, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v15, v16
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; GISEL-NEXT:    v_mul_hi_u32 v13, v9, v13
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
+; GISEL-NEXT:    v_addc_u32_e64 v12, s[4:5], v9, v13, vcc
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v13
+; GISEL-NEXT:    v_mul_lo_u32 v13, v10, v8
+; GISEL-NEXT:    v_mul_lo_u32 v11, v11, v8
+; GISEL-NEXT:    v_mul_lo_u32 v14, v10, v12
+; GISEL-NEXT:    v_mul_hi_u32 v10, v10, v8
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v14
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v11, v10
+; GISEL-NEXT:    v_mul_lo_u32 v11, v12, v13
+; GISEL-NEXT:    v_mul_lo_u32 v14, v8, v10
+; GISEL-NEXT:    v_mul_hi_u32 v15, v8, v13
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v14, v11
+; GISEL-NEXT:    v_mul_lo_u32 v14, v12, v10
+; GISEL-NEXT:    v_mul_hi_u32 v13, v12, v13
+; GISEL-NEXT:    v_mul_hi_u32 v15, v8, v10
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v14, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v14, s[4:5], v14, v15
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v13, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v14, v13
+; GISEL-NEXT:    v_mul_hi_u32 v10, v12, v10
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v13
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v11
+; GISEL-NEXT:    v_addc_u32_e32 v9, vcc, v9, v10, vcc
+; GISEL-NEXT:    v_addc_u32_e64 v9, vcc, 0, v9, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v10, v1, v8
+; GISEL-NEXT:    v_mul_lo_u32 v11, v0, v9
+; GISEL-NEXT:    v_mul_hi_u32 v12, v0, v8
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT:    v_mul_lo_u32 v11, v1, v9
+; GISEL-NEXT:    v_mul_hi_u32 v8, v1, v8
+; GISEL-NEXT:    v_mul_hi_u32 v12, v0, v9
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v11, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT:    v_mul_hi_u32 v9, v1, v9
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT:    v_mul_lo_u32 v10, v4, v8
+; GISEL-NEXT:    v_mul_lo_u32 v11, v5, v8
+; GISEL-NEXT:    v_mul_lo_u32 v9, v4, v9
+; GISEL-NEXT:    v_mul_hi_u32 v8, v4, v8
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v10
+; GISEL-NEXT:    v_subb_u32_e64 v9, s[4:5], v1, v8, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v8
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v9, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], v9, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, v8, v10, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e64 v10, s[4:5], v0, v4
+; GISEL-NEXT:    v_subb_u32_e32 v1, vcc, v1, v5, vcc
+; GISEL-NEXT:    v_subbrev_u32_e64 v11, vcc, 0, v1, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v11, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v10, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v11, v5
+; GISEL-NEXT:    v_cndmask_b32_e32 v12, v12, v13, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v4, vcc, v10, v4
+; GISEL-NEXT:    v_subb_u32_e64 v1, s[4:5], v1, v5, s[4:5]
+; GISEL-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GISEL-NEXT:    v_cndmask_b32_e32 v4, v10, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v11, v1, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, v6
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v5, v7
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; GISEL-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; GISEL-NEXT:    v_mul_f32_e32 v5, 0x2f800000, v4
+; GISEL-NEXT:    v_trunc_f32_e32 v5, v5
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v5
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; GISEL-NEXT:    v_sub_i32_e32 v8, vcc, 0, v6
+; GISEL-NEXT:    v_subb_u32_e32 v9, vcc, 0, v7, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v10, v8, v4
+; GISEL-NEXT:    v_mul_lo_u32 v11, v9, v4
+; GISEL-NEXT:    v_mul_lo_u32 v12, v8, v5
+; GISEL-NEXT:    v_mul_hi_u32 v13, v8, v4
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v13
+; GISEL-NEXT:    v_mul_lo_u32 v12, v5, v10
+; GISEL-NEXT:    v_mul_lo_u32 v13, v4, v11
+; GISEL-NEXT:    v_mul_hi_u32 v14, v4, v10
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; GISEL-NEXT:    v_mul_lo_u32 v13, v5, v11
+; GISEL-NEXT:    v_mul_hi_u32 v10, v5, v10
+; GISEL-NEXT:    v_mul_hi_u32 v14, v4, v11
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v13, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; GISEL-NEXT:    v_mul_hi_u32 v11, v5, v11
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v10
+; GISEL-NEXT:    v_addc_u32_e64 v10, s[4:5], v5, v11, vcc
+; GISEL-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v11
+; GISEL-NEXT:    v_mul_lo_u32 v11, v8, v4
+; GISEL-NEXT:    v_mul_lo_u32 v9, v9, v4
+; GISEL-NEXT:    v_mul_lo_u32 v12, v8, v10
+; GISEL-NEXT:    v_mul_hi_u32 v8, v8, v4
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v12
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v9, v8
+; GISEL-NEXT:    v_mul_lo_u32 v9, v10, v11
+; GISEL-NEXT:    v_mul_lo_u32 v12, v4, v8
+; GISEL-NEXT:    v_mul_hi_u32 v13, v4, v11
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v12, v9
+; GISEL-NEXT:    v_mul_lo_u32 v12, v10, v8
+; GISEL-NEXT:    v_mul_hi_u32 v11, v10, v11
+; GISEL-NEXT:    v_mul_hi_u32 v13, v4, v8
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v12, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v13
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v11, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v12, v11
+; GISEL-NEXT:    v_mul_hi_u32 v8, v10, v8
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v11
+; GISEL-NEXT:    v_add_i32_e64 v4, s[4:5], v4, v9
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, v5, v8, vcc
+; GISEL-NEXT:    v_addc_u32_e64 v5, vcc, 0, v5, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v8, v3, v4
+; GISEL-NEXT:    v_mul_lo_u32 v9, v2, v5
+; GISEL-NEXT:    v_mul_hi_u32 v10, v2, v4
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT:    v_mul_lo_u32 v9, v3, v5
+; GISEL-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GISEL-NEXT:    v_mul_hi_u32 v10, v2, v5
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v9, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT:    v_mul_hi_u32 v5, v3, v5
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v8
+; GISEL-NEXT:    v_mul_lo_u32 v8, v6, v4
+; GISEL-NEXT:    v_mul_lo_u32 v9, v7, v4
+; GISEL-NEXT:    v_mul_lo_u32 v5, v6, v5
+; GISEL-NEXT:    v_mul_hi_u32 v4, v6, v4
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v9, v5
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v8
+; GISEL-NEXT:    v_subb_u32_e64 v5, s[4:5], v3, v4, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v4
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v5, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v2, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], v5, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v4, v4, v8, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e64 v8, s[4:5], v2, v6
+; GISEL-NEXT:    v_subb_u32_e32 v3, vcc, v3, v7, vcc
+; GISEL-NEXT:    v_subbrev_u32_e64 v9, vcc, 0, v3, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v9, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v8, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v9, v7
+; GISEL-NEXT:    v_cndmask_b32_e32 v10, v10, v11, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v6, vcc, v8, v6
+; GISEL-NEXT:    v_subb_u32_e64 v3, s[4:5], v3, v7, s[4:5]
+; GISEL-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GISEL-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_urem_v2i64:
+; CGP:       ; %bb.0:
+; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT:    v_mov_b32_e32 v8, v0
+; CGP-NEXT:    v_mov_b32_e32 v9, v1
+; CGP-NEXT:    v_or_b32_e32 v1, v9, v5
+; CGP-NEXT:    v_mov_b32_e32 v0, 0
+; CGP-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
+; CGP-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, 1
+; CGP-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; CGP-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; CGP-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
+; CGP-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; CGP-NEXT:    s_cbranch_execnz BB2_2
+; CGP-NEXT:  ; %bb.1:
+; CGP-NEXT:    v_cvt_f32_u32_e32 v0, v4
+; CGP-NEXT:    v_cvt_f32_u32_e32 v1, v5
+; CGP-NEXT:    v_sub_i32_e32 v10, vcc, 0, v4
+; CGP-NEXT:    v_subb_u32_e32 v11, vcc, 0, v5, vcc
+; CGP-NEXT:    v_mac_f32_e32 v0, 0x4f800000, v1
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; CGP-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
+; CGP-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
+; CGP-NEXT:    v_trunc_f32_e32 v1, v1
+; CGP-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
+; CGP-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; CGP-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; CGP-NEXT:    v_mul_lo_u32 v12, v10, v1
+; CGP-NEXT:    v_mul_lo_u32 v13, v10, v0
+; CGP-NEXT:    v_mul_lo_u32 v14, v11, v0
+; CGP-NEXT:    v_mul_hi_u32 v15, v10, v0
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v14, v12
+; CGP-NEXT:    v_mul_lo_u32 v14, v1, v13
+; CGP-NEXT:    v_mul_hi_u32 v16, v0, v13
+; CGP-NEXT:    v_mul_hi_u32 v13, v1, v13
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v12, v15
+; CGP-NEXT:    v_mul_lo_u32 v15, v0, v12
+; CGP-NEXT:    v_mul_lo_u32 v17, v1, v12
+; CGP-NEXT:    v_mul_hi_u32 v18, v0, v12
+; CGP-NEXT:    v_mul_hi_u32 v12, v1, v12
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v17, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v13, v18
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, v17, v16
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v12, v14
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v13
+; CGP-NEXT:    v_addc_u32_e64 v13, s[4:5], v1, v12, vcc
+; CGP-NEXT:    v_add_i32_e64 v1, s[4:5], v1, v12
+; CGP-NEXT:    v_mul_lo_u32 v12, v10, v0
+; CGP-NEXT:    v_mul_lo_u32 v11, v11, v0
+; CGP-NEXT:    v_mul_hi_u32 v14, v10, v0
+; CGP-NEXT:    v_mul_lo_u32 v10, v10, v13
+; CGP-NEXT:    v_mul_lo_u32 v15, v13, v12
+; CGP-NEXT:    v_mul_hi_u32 v16, v0, v12
+; CGP-NEXT:    v_mul_hi_u32 v12, v13, v12
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v11, v10
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v14
+; CGP-NEXT:    v_mul_lo_u32 v11, v0, v10
+; CGP-NEXT:    v_mul_lo_u32 v14, v13, v10
+; CGP-NEXT:    v_mul_hi_u32 v17, v0, v10
+; CGP-NEXT:    v_mul_hi_u32 v10, v13, v10
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v15, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v14, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v17
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v13, v11
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v14, v15
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v12, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v13, v12
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v12
+; CGP-NEXT:    v_addc_u32_e32 v1, vcc, v1, v10, vcc
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v11
+; CGP-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v10, v9, v0
+; CGP-NEXT:    v_mul_hi_u32 v11, v8, v0
+; CGP-NEXT:    v_mul_hi_u32 v0, v9, v0
+; CGP-NEXT:    v_mul_lo_u32 v12, v8, v1
+; CGP-NEXT:    v_mul_lo_u32 v13, v9, v1
+; CGP-NEXT:    v_mul_hi_u32 v14, v8, v1
+; CGP-NEXT:    v_mul_hi_u32 v1, v9, v1
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v13, v0
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v13, v11
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; CGP-NEXT:    v_mul_lo_u32 v11, v4, v0
+; CGP-NEXT:    v_mul_lo_u32 v12, v5, v0
+; CGP-NEXT:    v_mul_hi_u32 v0, v4, v0
+; CGP-NEXT:    v_add_i32_e32 v1, vcc, v1, v10
+; CGP-NEXT:    v_mul_lo_u32 v1, v4, v1
+; CGP-NEXT:    v_add_i32_e32 v1, vcc, v12, v1
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v8, v11
+; CGP-NEXT:    v_subb_u32_e64 v10, s[4:5], v9, v0, vcc
+; CGP-NEXT:    v_sub_i32_e64 v0, s[4:5], v9, v0
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v10, v5
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, -1, s[4:5]
+; CGP-NEXT:    v_subb_u32_e32 v0, vcc, v0, v5, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, v10, v5
+; CGP-NEXT:    v_cndmask_b32_e32 v9, v11, v9, vcc
+; CGP-NEXT:    v_sub_i32_e32 v11, vcc, v1, v4
+; CGP-NEXT:    v_subbrev_u32_e64 v12, s[4:5], 0, v0, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v11, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, -1, s[4:5]
+; CGP-NEXT:    v_subb_u32_e32 v0, vcc, v0, v5, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v12, v5
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, -1, vcc
+; CGP-NEXT:    v_sub_i32_e32 v15, vcc, v11, v4
+; CGP-NEXT:    v_subbrev_u32_e32 v0, vcc, 0, v0, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, v12, v5
+; CGP-NEXT:    v_cndmask_b32_e32 v5, v14, v13, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; CGP-NEXT:    v_cndmask_b32_e32 v5, v11, v15, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v11, v12, v0, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v9
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v1, v5, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v10, v11, vcc
+; CGP-NEXT:  BB2_2: ; %Flow1
+; CGP-NEXT:    s_or_saveexec_b64 s[8:9], s[6:7]
+; CGP-NEXT:    s_xor_b64 exec, exec, s[8:9]
+; CGP-NEXT:    s_cbranch_execz BB2_4
+; CGP-NEXT:  ; %bb.3:
+; CGP-NEXT:    v_cvt_f32_u32_e32 v0, v4
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; CGP-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; CGP-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; CGP-NEXT:    v_mul_lo_u32 v1, v0, v4
+; CGP-NEXT:    v_mul_hi_u32 v5, v0, v4
+; CGP-NEXT:    v_sub_i32_e32 v9, vcc, 0, v1
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
+; CGP-NEXT:    v_mul_hi_u32 v1, v1, v0
+; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v0, v1
+; CGP-NEXT:    v_sub_i32_e64 v0, s[4:5], v0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; CGP-NEXT:    v_mul_hi_u32 v0, v0, v8
+; CGP-NEXT:    v_mul_lo_u32 v0, v0, v4
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v8, v0
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v4
+; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v1, v4
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v8, v0
+; CGP-NEXT:    v_sub_i32_e64 v0, s[6:7], v1, v4
+; CGP-NEXT:    s_and_b64 vcc, vcc, s[4:5]
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v0, v5, v0, s[4:5]
+; CGP-NEXT:    v_mov_b32_e32 v1, 0
+; CGP-NEXT:  BB2_4:
+; CGP-NEXT:    s_or_b64 exec, exec, s[8:9]
+; CGP-NEXT:    v_or_b32_e32 v5, v3, v7
+; CGP-NEXT:    v_mov_b32_e32 v4, 0
+; CGP-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[4:5]
+; CGP-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, 1
+; CGP-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; CGP-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; CGP-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
+; CGP-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; CGP-NEXT:    s_cbranch_execnz BB2_6
+; CGP-NEXT:  ; %bb.5:
+; CGP-NEXT:    v_cvt_f32_u32_e32 v4, v6
+; CGP-NEXT:    v_cvt_f32_u32_e32 v5, v7
+; CGP-NEXT:    v_sub_i32_e32 v8, vcc, 0, v6
+; CGP-NEXT:    v_subb_u32_e32 v9, vcc, 0, v7, vcc
+; CGP-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; CGP-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; CGP-NEXT:    v_mul_f32_e32 v5, 0x2f800000, v4
+; CGP-NEXT:    v_trunc_f32_e32 v5, v5
+; CGP-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v5
+; CGP-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; CGP-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; CGP-NEXT:    v_mul_lo_u32 v10, v8, v5
+; CGP-NEXT:    v_mul_lo_u32 v11, v8, v4
+; CGP-NEXT:    v_mul_lo_u32 v12, v9, v4
+; CGP-NEXT:    v_mul_hi_u32 v13, v8, v4
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; CGP-NEXT:    v_mul_lo_u32 v12, v5, v11
+; CGP-NEXT:    v_mul_hi_u32 v14, v4, v11
+; CGP-NEXT:    v_mul_hi_u32 v11, v5, v11
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v13
+; CGP-NEXT:    v_mul_lo_u32 v13, v4, v10
+; CGP-NEXT:    v_mul_lo_u32 v15, v5, v10
+; CGP-NEXT:    v_mul_hi_u32 v16, v4, v10
+; CGP-NEXT:    v_mul_hi_u32 v10, v5, v10
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v15, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v12, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v11, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v15, v14
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v11
+; CGP-NEXT:    v_addc_u32_e64 v11, s[4:5], v5, v10, vcc
+; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v10
+; CGP-NEXT:    v_mul_lo_u32 v10, v8, v4
+; CGP-NEXT:    v_mul_lo_u32 v9, v9, v4
+; CGP-NEXT:    v_mul_hi_u32 v12, v8, v4
+; CGP-NEXT:    v_mul_lo_u32 v8, v8, v11
+; CGP-NEXT:    v_mul_lo_u32 v13, v11, v10
+; CGP-NEXT:    v_mul_hi_u32 v14, v4, v10
+; CGP-NEXT:    v_mul_hi_u32 v10, v11, v10
+; CGP-NEXT:    v_add_i32_e64 v8, s[4:5], v9, v8
+; CGP-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v12
+; CGP-NEXT:    v_mul_lo_u32 v9, v4, v8
+; CGP-NEXT:    v_mul_lo_u32 v12, v11, v8
+; CGP-NEXT:    v_mul_hi_u32 v15, v4, v8
+; CGP-NEXT:    v_mul_hi_u32 v8, v11, v8
+; CGP-NEXT:    v_add_i32_e64 v9, s[4:5], v13, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v12, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v9, s[4:5], v11, v9
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v12, v13
+; CGP-NEXT:    v_add_i32_e64 v9, s[4:5], v10, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v11, v10
+; CGP-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v10
+; CGP-NEXT:    v_addc_u32_e32 v5, vcc, v5, v8, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v9
+; CGP-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; CGP-NEXT:    v_mul_lo_u32 v8, v3, v4
+; CGP-NEXT:    v_mul_hi_u32 v9, v2, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT:    v_mul_lo_u32 v10, v2, v5
+; CGP-NEXT:    v_mul_lo_u32 v11, v3, v5
+; CGP-NEXT:    v_mul_hi_u32 v12, v2, v5
+; CGP-NEXT:    v_mul_hi_u32 v5, v3, v5
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v11, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; CGP-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; CGP-NEXT:    v_mul_lo_u32 v9, v6, v4
+; CGP-NEXT:    v_mul_lo_u32 v10, v7, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, v6, v4
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v8
+; CGP-NEXT:    v_mul_lo_u32 v5, v6, v5
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v10, v5
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; CGP-NEXT:    v_sub_i32_e32 v5, vcc, v2, v9
+; CGP-NEXT:    v_subb_u32_e64 v8, s[4:5], v3, v4, vcc
+; CGP-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v4
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v5, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v8, v7
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; CGP-NEXT:    v_subb_u32_e32 v3, vcc, v3, v7, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, v8, v7
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v9, v4, vcc
+; CGP-NEXT:    v_sub_i32_e32 v9, vcc, v5, v6
+; CGP-NEXT:    v_subbrev_u32_e64 v10, s[4:5], 0, v3, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v9, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, -1, s[4:5]
+; CGP-NEXT:    v_subb_u32_e32 v3, vcc, v3, v7, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v10, v7
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, -1, vcc
+; CGP-NEXT:    v_sub_i32_e32 v13, vcc, v9, v6
+; CGP-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, v10, v7
+; CGP-NEXT:    v_cndmask_b32_e32 v7, v12, v11, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; CGP-NEXT:    v_cndmask_b32_e32 v7, v9, v13, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v5, v7, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v5, v8, v3, vcc
+; CGP-NEXT:  BB2_6: ; %Flow
+; CGP-NEXT:    s_or_saveexec_b64 s[8:9], s[6:7]
+; CGP-NEXT:    s_xor_b64 exec, exec, s[8:9]
+; CGP-NEXT:    s_cbranch_execz BB2_8
+; CGP-NEXT:  ; %bb.7:
+; CGP-NEXT:    v_cvt_f32_u32_e32 v3, v6
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; CGP-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v3
+; CGP-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; CGP-NEXT:    v_mul_lo_u32 v4, v3, v6
+; CGP-NEXT:    v_mul_hi_u32 v5, v3, v6
+; CGP-NEXT:    v_sub_i32_e32 v7, vcc, 0, v4
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
+; CGP-NEXT:    v_mul_hi_u32 v4, v4, v3
+; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v3, v4
+; CGP-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v4
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; CGP-NEXT:    v_mul_hi_u32 v3, v3, v2
+; CGP-NEXT:    v_mul_lo_u32 v3, v3, v6
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, v2, v3
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v4, v6
+; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v4, v6
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v2, v3
+; CGP-NEXT:    v_sub_i32_e64 v2, s[6:7], v4, v6
+; CGP-NEXT:    s_and_b64 vcc, vcc, s[4:5]
+; CGP-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v4, v5, v2, s[4:5]
+; CGP-NEXT:    v_mov_b32_e32 v5, 0
+; CGP-NEXT:  BB2_8:
+; CGP-NEXT:    s_or_b64 exec, exec, s[8:9]
+; CGP-NEXT:    v_mov_b32_e32 v2, v4
+; CGP-NEXT:    v_mov_b32_e32 v3, v5
+; CGP-NEXT:    s_setpc_b64 s[30:31]
+  %result = urem <2 x i64> %num, %den
+  ret <2 x i64> %result
+}
+
+define i64 @v_urem_i64_pow2k_denom(i64 %num) {
+; CHECK-LABEL: v_urem_i64_pow2k_denom:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_movk_i32 s6, 0x1000
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v2, 0
+; CHECK-NEXT:    s_mov_b32 s7, 0xfffff000
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v3, s6
+; CHECK-NEXT:    v_mac_f32_e32 v3, 0x4f800000, v2
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v2, v3
+; CHECK-NEXT:    v_mul_f32_e32 v2, 0x5f7ffffc, v2
+; CHECK-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v2
+; CHECK-NEXT:    v_trunc_f32_e32 v3, v3
+; CHECK-NEXT:    v_mac_f32_e32 v2, 0xcf800000, v3
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; CHECK-NEXT:    v_mul_lo_u32 v4, s7, v3
+; CHECK-NEXT:    v_mul_lo_u32 v5, s7, v2
+; CHECK-NEXT:    v_mul_lo_u32 v6, -1, v2
+; CHECK-NEXT:    v_mul_hi_u32 v7, s7, v2
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
+; CHECK-NEXT:    v_mul_lo_u32 v6, v3, v5
+; CHECK-NEXT:    v_mul_hi_u32 v8, v2, v5
+; CHECK-NEXT:    v_mul_hi_u32 v5, v3, v5
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
+; CHECK-NEXT:    v_mul_lo_u32 v7, v2, v4
+; CHECK-NEXT:    v_mul_lo_u32 v9, v3, v4
+; CHECK-NEXT:    v_mul_hi_u32 v10, v2, v4
+; CHECK-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v9, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v9, v8
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; CHECK-NEXT:    v_addc_u32_e64 v5, s[4:5], v3, v4, vcc
+; CHECK-NEXT:    v_add_i32_e64 v3, s[4:5], v3, v4
+; CHECK-NEXT:    v_mul_lo_u32 v4, s7, v2
+; CHECK-NEXT:    v_mul_lo_u32 v6, -1, v2
+; CHECK-NEXT:    v_mul_hi_u32 v7, s7, v2
+; CHECK-NEXT:    v_mul_lo_u32 v8, s7, v5
+; CHECK-NEXT:    v_mul_lo_u32 v9, v5, v4
+; CHECK-NEXT:    v_mul_hi_u32 v10, v2, v4
+; CHECK-NEXT:    v_mul_hi_u32 v4, v5, v4
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v8
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v7
+; CHECK-NEXT:    v_mul_lo_u32 v7, v2, v6
+; CHECK-NEXT:    v_mul_lo_u32 v8, v5, v6
+; CHECK-NEXT:    v_mul_hi_u32 v11, v2, v6
+; CHECK-NEXT:    v_mul_hi_u32 v5, v5, v6
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v9, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v8, v4
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v4, v11
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v7, v6
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v8, v9
+; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v4, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v7, v6
+; CHECK-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v6
+; CHECK-NEXT:    v_addc_u32_e32 v3, vcc, v3, v5, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; CHECK-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v4, v1, v2
+; CHECK-NEXT:    v_mul_hi_u32 v5, v0, v2
+; CHECK-NEXT:    v_mul_hi_u32 v2, v1, v2
+; CHECK-NEXT:    v_mul_lo_u32 v6, v0, v3
+; CHECK-NEXT:    v_mul_lo_u32 v7, v1, v3
+; CHECK-NEXT:    v_mul_hi_u32 v8, v0, v3
+; CHECK-NEXT:    v_mul_hi_u32 v3, v1, v3
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v8
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; CHECK-NEXT:    v_mul_lo_u32 v5, s6, v2
+; CHECK-NEXT:    v_mul_lo_u32 v6, 0, v2
+; CHECK-NEXT:    v_mul_hi_u32 v2, s6, v2
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; CHECK-NEXT:    v_mul_lo_u32 v3, s6, v3
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v5
+; CHECK-NEXT:    v_subb_u32_e64 v3, s[4:5], v1, v2, vcc
+; CHECK-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v2
+; CHECK-NEXT:    v_cmp_le_u32_e64 s[4:5], s6, v0
+; CHECK-NEXT:    v_cndmask_b32_e64 v2, 0, -1, s[4:5]
+; CHECK-NEXT:    v_cmp_le_u32_e64 s[4:5], 0, v3
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[4:5]
+; CHECK-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; CHECK-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
+; CHECK-NEXT:    v_subrev_i32_e32 v4, vcc, s6, v0
+; CHECK-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s6, v4
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, 0, v1
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; CHECK-NEXT:    v_subrev_i32_e32 v7, vcc, s6, v4
+; CHECK-NEXT:    v_subbrev_u32_e32 v8, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; CHECK-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v1, v8, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %result = urem i64 %num, 4096
+  ret i64 %result
+}
+
+define <2 x i64> @v_urem_v2i64_pow2k_denom(<2 x i64> %num) {
+; GISEL-LABEL: v_urem_v2i64_pow2k_denom:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    s_movk_i32 s10, 0x1000
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, s10
+; GISEL-NEXT:    s_sub_u32 s8, 0, s10
+; GISEL-NEXT:    s_cselect_b32 s4, 1, 0
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v5, 0
+; GISEL-NEXT:    v_mov_b32_e32 v6, v4
+; GISEL-NEXT:    s_and_b32 s4, s4, 1
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
+; GISEL-NEXT:    v_mac_f32_e32 v6, 0x4f800000, v5
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v5, v6
+; GISEL-NEXT:    s_cmp_lg_u32 s4, 0
+; GISEL-NEXT:    s_subb_u32 s9, 0, 0
+; GISEL-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; GISEL-NEXT:    v_mul_f32_e32 v5, 0x5f7ffffc, v5
+; GISEL-NEXT:    v_mul_f32_e32 v6, 0x2f800000, v4
+; GISEL-NEXT:    s_sub_u32 s11, 0, s10
+; GISEL-NEXT:    s_cselect_b32 s4, 1, 0
+; GISEL-NEXT:    v_mul_f32_e32 v7, 0x2f800000, v5
+; GISEL-NEXT:    v_trunc_f32_e32 v6, v6
+; GISEL-NEXT:    s_and_b32 s4, s4, 1
+; GISEL-NEXT:    v_trunc_f32_e32 v7, v7
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v6
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; GISEL-NEXT:    v_mac_f32_e32 v5, 0xcf800000, v7
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; GISEL-NEXT:    s_cmp_lg_u32 s4, 0
+; GISEL-NEXT:    s_subb_u32 s6, 0, 0
+; GISEL-NEXT:    v_mul_lo_u32 v8, s11, v6
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; GISEL-NEXT:    v_mul_lo_u32 v9, s8, v7
+; GISEL-NEXT:    v_mul_lo_u32 v10, s11, v4
+; GISEL-NEXT:    v_mul_lo_u32 v11, s6, v4
+; GISEL-NEXT:    v_mul_hi_u32 v12, s11, v4
+; GISEL-NEXT:    v_mul_lo_u32 v13, s8, v5
+; GISEL-NEXT:    v_mul_lo_u32 v14, s9, v5
+; GISEL-NEXT:    v_mul_hi_u32 v15, s8, v5
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v11, v8
+; GISEL-NEXT:    v_mul_lo_u32 v11, v6, v10
+; GISEL-NEXT:    v_mul_hi_u32 v16, v4, v10
+; GISEL-NEXT:    v_mul_hi_u32 v10, v6, v10
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v14, v9
+; GISEL-NEXT:    v_mul_lo_u32 v14, v7, v13
+; GISEL-NEXT:    v_mul_hi_u32 v17, v5, v13
+; GISEL-NEXT:    v_mul_hi_u32 v13, v7, v13
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v15
+; GISEL-NEXT:    v_mul_lo_u32 v12, v4, v8
+; GISEL-NEXT:    v_mul_lo_u32 v15, v6, v8
+; GISEL-NEXT:    v_mul_hi_u32 v18, v4, v8
+; GISEL-NEXT:    v_mul_hi_u32 v8, v6, v8
+; GISEL-NEXT:    v_mul_lo_u32 v19, v5, v9
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v19
+; GISEL-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v17
+; GISEL-NEXT:    v_mul_lo_u32 v14, v7, v9
+; GISEL-NEXT:    v_mul_hi_u32 v17, v5, v9
+; GISEL-NEXT:    v_mul_hi_u32 v9, v7, v9
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v15, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v14, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v18
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[4:5]
+; GISEL-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v17
+; GISEL-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v15, v16
+; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v19, v18
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v17
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v14, v15
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v12
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v10
+; GISEL-NEXT:    v_addc_u32_e64 v10, s[4:5], v6, v8, vcc
+; GISEL-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v8
+; GISEL-NEXT:    v_mul_lo_u32 v8, s11, v4
+; GISEL-NEXT:    v_mul_lo_u32 v11, s6, v4
+; GISEL-NEXT:    v_mul_hi_u32 v12, s11, v4
+; GISEL-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v13
+; GISEL-NEXT:    v_addc_u32_e64 v13, s[6:7], v7, v9, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v7, s[6:7], v7, v9
+; GISEL-NEXT:    v_mul_lo_u32 v9, s8, v5
+; GISEL-NEXT:    v_mul_lo_u32 v14, s9, v5
+; GISEL-NEXT:    v_mul_hi_u32 v15, s8, v5
+; GISEL-NEXT:    v_mul_lo_u32 v16, s11, v10
+; GISEL-NEXT:    v_mul_lo_u32 v17, v10, v8
+; GISEL-NEXT:    v_mul_hi_u32 v18, v4, v8
+; GISEL-NEXT:    v_mul_hi_u32 v8, v10, v8
+; GISEL-NEXT:    v_mul_lo_u32 v19, s8, v13
+; GISEL-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v16
+; GISEL-NEXT:    v_mul_lo_u32 v16, v13, v9
+; GISEL-NEXT:    v_add_i32_e64 v14, s[6:7], v14, v19
+; GISEL-NEXT:    v_mul_hi_u32 v19, v5, v9
+; GISEL-NEXT:    v_mul_hi_u32 v9, v13, v9
+; GISEL-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v12
+; GISEL-NEXT:    v_add_i32_e64 v12, s[6:7], v14, v15
+; GISEL-NEXT:    v_mul_lo_u32 v14, v4, v11
+; GISEL-NEXT:    v_mul_lo_u32 v15, v5, v12
+; GISEL-NEXT:    v_add_i32_e64 v15, s[6:7], v16, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[6:7]
+; GISEL-NEXT:    v_add_i32_e64 v15, s[6:7], v15, v19
+; GISEL-NEXT:    v_mul_lo_u32 v15, v10, v11
+; GISEL-NEXT:    v_mul_hi_u32 v19, v4, v11
+; GISEL-NEXT:    v_mul_hi_u32 v10, v10, v11
+; GISEL-NEXT:    v_mul_lo_u32 v11, v13, v12
+; GISEL-NEXT:    v_mul_hi_u32 v13, v13, v12
+; GISEL-NEXT:    v_mul_hi_u32 v12, v5, v12
+; GISEL-NEXT:    v_add_i32_e64 v14, s[8:9], v17, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v17, 0, 1, s[8:9]
+; GISEL-NEXT:    v_add_i32_e64 v8, s[8:9], v15, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[8:9]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[8:9], v11, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[8:9]
+; GISEL-NEXT:    v_add_i32_e64 v14, s[8:9], v14, v18
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[8:9]
+; GISEL-NEXT:    v_add_i32_e64 v8, s[8:9], v8, v19
+; GISEL-NEXT:    v_cndmask_b32_e64 v18, 0, 1, s[8:9]
+; GISEL-NEXT:    v_cndmask_b32_e64 v19, 0, 1, s[6:7]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[6:7], v9, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[6:7]
+; GISEL-NEXT:    v_add_i32_e64 v14, s[6:7], v17, v14
+; GISEL-NEXT:    v_add_i32_e64 v15, s[6:7], v15, v18
+; GISEL-NEXT:    v_add_i32_e64 v16, s[6:7], v16, v19
+; GISEL-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v12
+; GISEL-NEXT:    v_add_i32_e64 v8, s[6:7], v8, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[6:7]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[6:7], v9, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[6:7]
+; GISEL-NEXT:    v_add_i32_e64 v12, s[6:7], v15, v12
+; GISEL-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v14
+; GISEL-NEXT:    v_add_i32_e64 v10, s[6:7], v10, v12
+; GISEL-NEXT:    v_add_i32_e64 v11, s[6:7], v13, v11
+; GISEL-NEXT:    v_addc_u32_e32 v6, vcc, v6, v10, vcc
+; GISEL-NEXT:    v_addc_u32_e64 v7, vcc, v7, v11, s[4:5]
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v8, v3, v4
+; GISEL-NEXT:    v_mul_hi_u32 v10, v2, v4
+; GISEL-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
+; GISEL-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v9, v1, v5
+; GISEL-NEXT:    v_mul_hi_u32 v11, v0, v5
+; GISEL-NEXT:    v_mul_hi_u32 v5, v1, v5
+; GISEL-NEXT:    v_mul_lo_u32 v12, v2, v6
+; GISEL-NEXT:    v_mul_lo_u32 v13, v3, v6
+; GISEL-NEXT:    v_mul_hi_u32 v14, v2, v6
+; GISEL-NEXT:    v_mul_hi_u32 v6, v3, v6
+; GISEL-NEXT:    v_mul_lo_u32 v15, v0, v7
+; GISEL-NEXT:    v_mul_lo_u32 v16, v1, v7
+; GISEL-NEXT:    v_mul_hi_u32 v17, v0, v7
+; GISEL-NEXT:    v_mul_hi_u32 v7, v1, v7
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v13, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v16, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v17
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v12, v8
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v13, v10
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v15, v9
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v16, v11
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
+; GISEL-NEXT:    v_mul_lo_u32 v10, s10, v4
+; GISEL-NEXT:    v_mul_lo_u32 v12, 0, v4
+; GISEL-NEXT:    v_mul_hi_u32 v4, s10, v4
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
+; GISEL-NEXT:    v_mul_lo_u32 v11, s10, v5
+; GISEL-NEXT:    v_mul_lo_u32 v13, 0, v5
+; GISEL-NEXT:    v_mul_hi_u32 v5, s10, v5
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
+; GISEL-NEXT:    v_mul_lo_u32 v6, s10, v6
+; GISEL-NEXT:    v_mul_lo_u32 v7, s10, v7
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v12, v6
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v13, v7
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v10
+; GISEL-NEXT:    v_subb_u32_e64 v6, s[4:5], v3, v4, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v4
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s10, v2
+; GISEL-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e64 v0, s[4:5], v0, v11
+; GISEL-NEXT:    v_subb_u32_e64 v7, s[6:7], v1, v5, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e64 v1, s[6:7], v1, v5
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[6:7], s10, v0
+; GISEL-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[6:7]
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[6:7], 0, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[6:7]
+; GISEL-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, 0, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, -1, vcc
+; GISEL-NEXT:    v_subbrev_u32_e64 v1, vcc, 0, v1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v6
+; GISEL-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
+; GISEL-NEXT:    v_subrev_i32_e32 v8, vcc, s10, v2
+; GISEL-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s10, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
+; GISEL-NEXT:    v_cndmask_b32_e32 v5, v9, v5, vcc
+; GISEL-NEXT:    v_subrev_i32_e32 v9, vcc, s10, v0
+; GISEL-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s10, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, 0, v3
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, -1, vcc
+; GISEL-NEXT:    v_subrev_i32_e32 v13, vcc, s10, v8
+; GISEL-NEXT:    v_subbrev_u32_e32 v14, vcc, 0, v3, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, 0, v1
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, -1, vcc
+; GISEL-NEXT:    v_subrev_i32_e32 v16, vcc, s10, v9
+; GISEL-NEXT:    v_subbrev_u32_e32 v17, vcc, 0, v1, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v10, v12, v10, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v11, v15, v11, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GISEL-NEXT:    v_cndmask_b32_e32 v8, v8, v13, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, v9, v16, s[4:5]
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v3, v14, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v1, v1, v17, s[4:5]
+; GISEL-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v0, v0, v9, s[4:5]
+; GISEL-NEXT:    v_cndmask_b32_e64 v1, v7, v1, s[4:5]
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v6, v3, vcc
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_urem_v2i64_pow2k_denom:
+; CGP:       ; %bb.0:
+; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT:    s_movk_i32 s10, 0x1000
+; CGP-NEXT:    v_cvt_f32_u32_e32 v4, 0
+; CGP-NEXT:    v_cvt_f32_u32_e32 v5, s10
+; CGP-NEXT:    s_mov_b32 s8, 0xfffff000
+; CGP-NEXT:    v_mov_b32_e32 v6, v5
+; CGP-NEXT:    v_mac_f32_e32 v5, 0x4f800000, v4
+; CGP-NEXT:    v_mac_f32_e32 v6, 0x4f800000, v4
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v4, v5
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v5, v6
+; CGP-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; CGP-NEXT:    v_mul_f32_e32 v5, 0x5f7ffffc, v5
+; CGP-NEXT:    v_mul_f32_e32 v6, 0x2f800000, v4
+; CGP-NEXT:    v_mul_f32_e32 v7, 0x2f800000, v5
+; CGP-NEXT:    v_trunc_f32_e32 v6, v6
+; CGP-NEXT:    v_trunc_f32_e32 v7, v7
+; CGP-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v6
+; CGP-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; CGP-NEXT:    v_mac_f32_e32 v5, 0xcf800000, v7
+; CGP-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; CGP-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; CGP-NEXT:    v_mul_lo_u32 v8, s8, v6
+; CGP-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; CGP-NEXT:    v_mul_lo_u32 v9, s8, v7
+; CGP-NEXT:    v_mul_lo_u32 v10, s8, v4
+; CGP-NEXT:    v_mul_lo_u32 v11, -1, v4
+; CGP-NEXT:    v_mul_hi_u32 v12, s8, v4
+; CGP-NEXT:    v_mul_lo_u32 v13, s8, v5
+; CGP-NEXT:    v_mul_lo_u32 v14, -1, v5
+; CGP-NEXT:    v_mul_hi_u32 v15, s8, v5
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v11, v8
+; CGP-NEXT:    v_mul_lo_u32 v11, v6, v10
+; CGP-NEXT:    v_mul_hi_u32 v16, v4, v10
+; CGP-NEXT:    v_mul_hi_u32 v10, v6, v10
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v14, v9
+; CGP-NEXT:    v_mul_lo_u32 v14, v7, v13
+; CGP-NEXT:    v_mul_hi_u32 v17, v5, v13
+; CGP-NEXT:    v_mul_hi_u32 v13, v7, v13
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v15
+; CGP-NEXT:    v_mul_lo_u32 v12, v4, v8
+; CGP-NEXT:    v_mul_lo_u32 v15, v6, v8
+; CGP-NEXT:    v_mul_hi_u32 v18, v4, v8
+; CGP-NEXT:    v_mul_hi_u32 v8, v6, v8
+; CGP-NEXT:    v_mul_lo_u32 v19, v5, v9
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v19
+; CGP-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v17
+; CGP-NEXT:    v_mul_lo_u32 v14, v7, v9
+; CGP-NEXT:    v_mul_hi_u32 v17, v5, v9
+; CGP-NEXT:    v_mul_hi_u32 v9, v7, v9
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v15, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v14, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v18
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[4:5]
+; CGP-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v13, v17
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v15, v16
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, v19, v18
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v17
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v13, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v14, v15
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v12
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v10
+; CGP-NEXT:    v_addc_u32_e64 v10, s[4:5], v6, v8, vcc
+; CGP-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v8
+; CGP-NEXT:    v_mul_lo_u32 v8, s8, v4
+; CGP-NEXT:    v_mul_lo_u32 v11, -1, v4
+; CGP-NEXT:    v_mul_hi_u32 v12, s8, v4
+; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v13
+; CGP-NEXT:    v_addc_u32_e64 v13, s[6:7], v7, v9, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v7, s[6:7], v7, v9
+; CGP-NEXT:    v_mul_lo_u32 v9, s8, v5
+; CGP-NEXT:    v_mul_lo_u32 v14, -1, v5
+; CGP-NEXT:    v_mul_hi_u32 v15, s8, v5
+; CGP-NEXT:    v_mul_lo_u32 v16, s8, v10
+; CGP-NEXT:    v_mul_lo_u32 v17, v10, v8
+; CGP-NEXT:    v_mul_hi_u32 v18, v4, v8
+; CGP-NEXT:    v_mul_hi_u32 v8, v10, v8
+; CGP-NEXT:    v_mul_lo_u32 v19, s8, v13
+; CGP-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v16
+; CGP-NEXT:    v_mul_lo_u32 v16, v13, v9
+; CGP-NEXT:    v_add_i32_e64 v14, s[6:7], v14, v19
+; CGP-NEXT:    v_mul_hi_u32 v19, v5, v9
+; CGP-NEXT:    v_mul_hi_u32 v9, v13, v9
+; CGP-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v12
+; CGP-NEXT:    v_add_i32_e64 v12, s[6:7], v14, v15
+; CGP-NEXT:    v_mul_lo_u32 v14, v4, v11
+; CGP-NEXT:    v_mul_lo_u32 v15, v5, v12
+; CGP-NEXT:    v_add_i32_e64 v15, s[6:7], v16, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[6:7]
+; CGP-NEXT:    v_add_i32_e64 v15, s[6:7], v15, v19
+; CGP-NEXT:    v_mul_lo_u32 v15, v10, v11
+; CGP-NEXT:    v_mul_hi_u32 v19, v4, v11
+; CGP-NEXT:    v_mul_hi_u32 v10, v10, v11
+; CGP-NEXT:    v_mul_lo_u32 v11, v13, v12
+; CGP-NEXT:    v_mul_hi_u32 v13, v13, v12
+; CGP-NEXT:    v_mul_hi_u32 v12, v5, v12
+; CGP-NEXT:    v_add_i32_e64 v14, s[8:9], v17, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, 1, s[8:9]
+; CGP-NEXT:    v_add_i32_e64 v8, s[8:9], v15, v8
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[8:9]
+; CGP-NEXT:    v_add_i32_e64 v9, s[8:9], v11, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[8:9]
+; CGP-NEXT:    v_add_i32_e64 v14, s[8:9], v14, v18
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[8:9]
+; CGP-NEXT:    v_add_i32_e64 v8, s[8:9], v8, v19
+; CGP-NEXT:    v_cndmask_b32_e64 v18, 0, 1, s[8:9]
+; CGP-NEXT:    v_cndmask_b32_e64 v19, 0, 1, s[6:7]
+; CGP-NEXT:    v_add_i32_e64 v9, s[6:7], v9, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[6:7]
+; CGP-NEXT:    v_add_i32_e64 v14, s[6:7], v17, v14
+; CGP-NEXT:    v_add_i32_e64 v15, s[6:7], v15, v18
+; CGP-NEXT:    v_add_i32_e64 v16, s[6:7], v16, v19
+; CGP-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v12
+; CGP-NEXT:    v_add_i32_e64 v8, s[6:7], v8, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[6:7]
+; CGP-NEXT:    v_add_i32_e64 v9, s[6:7], v9, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[6:7]
+; CGP-NEXT:    v_add_i32_e64 v12, s[6:7], v15, v12
+; CGP-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v14
+; CGP-NEXT:    v_add_i32_e64 v10, s[6:7], v10, v12
+; CGP-NEXT:    v_add_i32_e64 v11, s[6:7], v13, v11
+; CGP-NEXT:    v_addc_u32_e32 v6, vcc, v6, v10, vcc
+; CGP-NEXT:    v_addc_u32_e64 v7, vcc, v7, v11, s[4:5]
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; CGP-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; CGP-NEXT:    v_mul_lo_u32 v8, v3, v4
+; CGP-NEXT:    v_mul_hi_u32 v10, v2, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
+; CGP-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; CGP-NEXT:    v_mul_lo_u32 v9, v1, v5
+; CGP-NEXT:    v_mul_hi_u32 v11, v0, v5
+; CGP-NEXT:    v_mul_hi_u32 v5, v1, v5
+; CGP-NEXT:    v_mul_lo_u32 v12, v2, v6
+; CGP-NEXT:    v_mul_lo_u32 v13, v3, v6
+; CGP-NEXT:    v_mul_hi_u32 v14, v2, v6
+; CGP-NEXT:    v_mul_hi_u32 v6, v3, v6
+; CGP-NEXT:    v_mul_lo_u32 v15, v0, v7
+; CGP-NEXT:    v_mul_lo_u32 v16, v1, v7
+; CGP-NEXT:    v_mul_hi_u32 v17, v0, v7
+; CGP-NEXT:    v_mul_hi_u32 v7, v1, v7
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v13, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v16, v5
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v17
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v12, v8
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v13, v10
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v15, v9
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v16, v11
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; CGP-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
+; CGP-NEXT:    v_mul_lo_u32 v10, s10, v4
+; CGP-NEXT:    v_mul_lo_u32 v12, 0, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, s10, v4
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
+; CGP-NEXT:    v_mul_lo_u32 v11, s10, v5
+; CGP-NEXT:    v_mul_lo_u32 v13, 0, v5
+; CGP-NEXT:    v_mul_hi_u32 v5, s10, v5
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
+; CGP-NEXT:    v_mul_lo_u32 v6, s10, v6
+; CGP-NEXT:    v_mul_lo_u32 v7, s10, v7
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v12, v6
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v13, v7
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; CGP-NEXT:    v_sub_i32_e32 v2, vcc, v2, v10
+; CGP-NEXT:    v_subb_u32_e64 v6, s[4:5], v3, v4, vcc
+; CGP-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v4
+; CGP-NEXT:    v_cmp_le_u32_e64 s[4:5], s10, v2
+; CGP-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[4:5]
+; CGP-NEXT:    v_sub_i32_e64 v0, s[4:5], v0, v11
+; CGP-NEXT:    v_subb_u32_e64 v7, s[6:7], v1, v5, s[4:5]
+; CGP-NEXT:    v_sub_i32_e64 v1, s[6:7], v1, v5
+; CGP-NEXT:    v_cmp_le_u32_e64 s[6:7], s10, v0
+; CGP-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[6:7]
+; CGP-NEXT:    v_cmp_le_u32_e64 s[6:7], 0, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[6:7]
+; CGP-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, 0, v7
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, -1, vcc
+; CGP-NEXT:    v_subbrev_u32_e64 v1, vcc, 0, v1, s[4:5]
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v6
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
+; CGP-NEXT:    v_subrev_i32_e32 v8, vcc, s10, v2
+; CGP-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s10, v8
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, -1, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
+; CGP-NEXT:    v_cndmask_b32_e32 v5, v9, v5, vcc
+; CGP-NEXT:    v_subrev_i32_e32 v9, vcc, s10, v0
+; CGP-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s10, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, -1, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, 0, v3
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, -1, vcc
+; CGP-NEXT:    v_subrev_i32_e32 v13, vcc, s10, v8
+; CGP-NEXT:    v_subbrev_u32_e32 v14, vcc, 0, v3, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, 0, v1
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, -1, vcc
+; CGP-NEXT:    v_subrev_i32_e32 v16, vcc, s10, v9
+; CGP-NEXT:    v_subbrev_u32_e32 v17, vcc, 0, v1, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v10, v12, v10, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v11, v15, v11, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; CGP-NEXT:    v_cndmask_b32_e32 v8, v8, v13, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v9, v9, v16, s[4:5]
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v3, v14, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; CGP-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v1, v1, v17, s[4:5]
+; CGP-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v5
+; CGP-NEXT:    v_cndmask_b32_e64 v0, v0, v9, s[4:5]
+; CGP-NEXT:    v_cndmask_b32_e64 v1, v7, v1, s[4:5]
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v6, v3, vcc
+; CGP-NEXT:    s_setpc_b64 s[30:31]
+  %result = urem <2 x i64> %num, <i64 4096, i64 4096>
+  ret <2 x i64> %result
+}
+
+define i64 @v_urem_i64_oddk_denom(i64 %num) {
+; CHECK-LABEL: v_urem_i64_oddk_denom:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_mov_b32 s6, 0x12d8fb
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v2, 0
+; CHECK-NEXT:    s_mov_b32 s7, 0xffed2705
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v3, s6
+; CHECK-NEXT:    v_mac_f32_e32 v3, 0x4f800000, v2
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v2, v3
+; CHECK-NEXT:    v_mul_f32_e32 v2, 0x5f7ffffc, v2
+; CHECK-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v2
+; CHECK-NEXT:    v_trunc_f32_e32 v3, v3
+; CHECK-NEXT:    v_mac_f32_e32 v2, 0xcf800000, v3
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; CHECK-NEXT:    v_mul_lo_u32 v4, s7, v3
+; CHECK-NEXT:    v_mul_lo_u32 v5, s7, v2
+; CHECK-NEXT:    v_mul_lo_u32 v6, -1, v2
+; CHECK-NEXT:    v_mul_hi_u32 v7, s7, v2
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
+; CHECK-NEXT:    v_mul_lo_u32 v6, v3, v5
+; CHECK-NEXT:    v_mul_hi_u32 v8, v2, v5
+; CHECK-NEXT:    v_mul_hi_u32 v5, v3, v5
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
+; CHECK-NEXT:    v_mul_lo_u32 v7, v2, v4
+; CHECK-NEXT:    v_mul_lo_u32 v9, v3, v4
+; CHECK-NEXT:    v_mul_hi_u32 v10, v2, v4
+; CHECK-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v9, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v9, v8
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; CHECK-NEXT:    v_addc_u32_e64 v5, s[4:5], v3, v4, vcc
+; CHECK-NEXT:    v_add_i32_e64 v3, s[4:5], v3, v4
+; CHECK-NEXT:    v_mul_lo_u32 v4, s7, v2
+; CHECK-NEXT:    v_mul_lo_u32 v6, -1, v2
+; CHECK-NEXT:    v_mul_hi_u32 v7, s7, v2
+; CHECK-NEXT:    v_mul_lo_u32 v8, s7, v5
+; CHECK-NEXT:    v_mul_lo_u32 v9, v5, v4
+; CHECK-NEXT:    v_mul_hi_u32 v10, v2, v4
+; CHECK-NEXT:    v_mul_hi_u32 v4, v5, v4
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v8
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v7
+; CHECK-NEXT:    v_mul_lo_u32 v7, v2, v6
+; CHECK-NEXT:    v_mul_lo_u32 v8, v5, v6
+; CHECK-NEXT:    v_mul_hi_u32 v11, v2, v6
+; CHECK-NEXT:    v_mul_hi_u32 v5, v5, v6
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v9, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v8, v4
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v4, v11
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v7, v6
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v8, v9
+; CHECK-NEXT:    v_add_i32_e64 v4, s[4:5], v4, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v7, v6
+; CHECK-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v6
+; CHECK-NEXT:    v_addc_u32_e32 v3, vcc, v3, v5, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; CHECK-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v4, v1, v2
+; CHECK-NEXT:    v_mul_hi_u32 v5, v0, v2
+; CHECK-NEXT:    v_mul_hi_u32 v2, v1, v2
+; CHECK-NEXT:    v_mul_lo_u32 v6, v0, v3
+; CHECK-NEXT:    v_mul_lo_u32 v7, v1, v3
+; CHECK-NEXT:    v_mul_hi_u32 v8, v0, v3
+; CHECK-NEXT:    v_mul_hi_u32 v3, v1, v3
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v8
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; CHECK-NEXT:    v_mul_lo_u32 v5, s6, v2
+; CHECK-NEXT:    v_mul_lo_u32 v6, 0, v2
+; CHECK-NEXT:    v_mul_hi_u32 v2, s6, v2
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v3, v4
+; CHECK-NEXT:    v_mul_lo_u32 v3, s6, v3
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v5
+; CHECK-NEXT:    v_subb_u32_e64 v3, s[4:5], v1, v2, vcc
+; CHECK-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v2
+; CHECK-NEXT:    v_cmp_le_u32_e64 s[4:5], s6, v0
+; CHECK-NEXT:    v_cndmask_b32_e64 v2, 0, -1, s[4:5]
+; CHECK-NEXT:    v_cmp_le_u32_e64 s[4:5], 0, v3
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[4:5]
+; CHECK-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; CHECK-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
+; CHECK-NEXT:    v_subrev_i32_e32 v4, vcc, s6, v0
+; CHECK-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s6, v4
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, 0, v1
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; CHECK-NEXT:    v_subrev_i32_e32 v7, vcc, s6, v4
+; CHECK-NEXT:    v_subbrev_u32_e32 v8, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; CHECK-NEXT:    v_cndmask_b32_e32 v5, v6, v5, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; CHECK-NEXT:    v_cndmask_b32_e32 v4, v4, v7, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v1, v8, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %result = urem i64 %num, 1235195
+  ret i64 %result
+}
+
+define <2 x i64> @v_urem_v2i64_oddk_denom(<2 x i64> %num) {
+; GISEL-LABEL: v_urem_v2i64_oddk_denom:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    s_mov_b32 s10, 0x12d8fb
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, s10
+; GISEL-NEXT:    s_sub_u32 s8, 0, s10
+; GISEL-NEXT:    s_cselect_b32 s4, 1, 0
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v5, 0
+; GISEL-NEXT:    v_mov_b32_e32 v6, v4
+; GISEL-NEXT:    s_and_b32 s4, s4, 1
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
+; GISEL-NEXT:    v_mac_f32_e32 v6, 0x4f800000, v5
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v5, v6
+; GISEL-NEXT:    s_cmp_lg_u32 s4, 0
+; GISEL-NEXT:    s_subb_u32 s9, 0, 0
+; GISEL-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; GISEL-NEXT:    v_mul_f32_e32 v5, 0x5f7ffffc, v5
+; GISEL-NEXT:    v_mul_f32_e32 v6, 0x2f800000, v4
+; GISEL-NEXT:    s_sub_u32 s11, 0, s10
+; GISEL-NEXT:    s_cselect_b32 s4, 1, 0
+; GISEL-NEXT:    v_mul_f32_e32 v7, 0x2f800000, v5
+; GISEL-NEXT:    v_trunc_f32_e32 v6, v6
+; GISEL-NEXT:    s_and_b32 s4, s4, 1
+; GISEL-NEXT:    v_trunc_f32_e32 v7, v7
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v6
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; GISEL-NEXT:    v_mac_f32_e32 v5, 0xcf800000, v7
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; GISEL-NEXT:    s_cmp_lg_u32 s4, 0
+; GISEL-NEXT:    s_subb_u32 s6, 0, 0
+; GISEL-NEXT:    v_mul_lo_u32 v8, s11, v6
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; GISEL-NEXT:    v_mul_lo_u32 v9, s8, v7
+; GISEL-NEXT:    v_mul_lo_u32 v10, s11, v4
+; GISEL-NEXT:    v_mul_lo_u32 v11, s6, v4
+; GISEL-NEXT:    v_mul_hi_u32 v12, s11, v4
+; GISEL-NEXT:    v_mul_lo_u32 v13, s8, v5
+; GISEL-NEXT:    v_mul_lo_u32 v14, s9, v5
+; GISEL-NEXT:    v_mul_hi_u32 v15, s8, v5
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v11, v8
+; GISEL-NEXT:    v_mul_lo_u32 v11, v6, v10
+; GISEL-NEXT:    v_mul_hi_u32 v16, v4, v10
+; GISEL-NEXT:    v_mul_hi_u32 v10, v6, v10
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v14, v9
+; GISEL-NEXT:    v_mul_lo_u32 v14, v7, v13
+; GISEL-NEXT:    v_mul_hi_u32 v17, v5, v13
+; GISEL-NEXT:    v_mul_hi_u32 v13, v7, v13
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v15
+; GISEL-NEXT:    v_mul_lo_u32 v12, v4, v8
+; GISEL-NEXT:    v_mul_lo_u32 v15, v6, v8
+; GISEL-NEXT:    v_mul_hi_u32 v18, v4, v8
+; GISEL-NEXT:    v_mul_hi_u32 v8, v6, v8
+; GISEL-NEXT:    v_mul_lo_u32 v19, v5, v9
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v19
+; GISEL-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v17
+; GISEL-NEXT:    v_mul_lo_u32 v14, v7, v9
+; GISEL-NEXT:    v_mul_hi_u32 v17, v5, v9
+; GISEL-NEXT:    v_mul_hi_u32 v9, v7, v9
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v15, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v14, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v18
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[4:5]
+; GISEL-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v17
+; GISEL-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v15, v16
+; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v19, v18
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v17
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v14, v15
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v12
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v10
+; GISEL-NEXT:    v_addc_u32_e64 v10, s[4:5], v6, v8, vcc
+; GISEL-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v8
+; GISEL-NEXT:    v_mul_lo_u32 v8, s11, v4
+; GISEL-NEXT:    v_mul_lo_u32 v11, s6, v4
+; GISEL-NEXT:    v_mul_hi_u32 v12, s11, v4
+; GISEL-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v13
+; GISEL-NEXT:    v_addc_u32_e64 v13, s[6:7], v7, v9, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v7, s[6:7], v7, v9
+; GISEL-NEXT:    v_mul_lo_u32 v9, s8, v5
+; GISEL-NEXT:    v_mul_lo_u32 v14, s9, v5
+; GISEL-NEXT:    v_mul_hi_u32 v15, s8, v5
+; GISEL-NEXT:    v_mul_lo_u32 v16, s11, v10
+; GISEL-NEXT:    v_mul_lo_u32 v17, v10, v8
+; GISEL-NEXT:    v_mul_hi_u32 v18, v4, v8
+; GISEL-NEXT:    v_mul_hi_u32 v8, v10, v8
+; GISEL-NEXT:    v_mul_lo_u32 v19, s8, v13
+; GISEL-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v16
+; GISEL-NEXT:    v_mul_lo_u32 v16, v13, v9
+; GISEL-NEXT:    v_add_i32_e64 v14, s[6:7], v14, v19
+; GISEL-NEXT:    v_mul_hi_u32 v19, v5, v9
+; GISEL-NEXT:    v_mul_hi_u32 v9, v13, v9
+; GISEL-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v12
+; GISEL-NEXT:    v_add_i32_e64 v12, s[6:7], v14, v15
+; GISEL-NEXT:    v_mul_lo_u32 v14, v4, v11
+; GISEL-NEXT:    v_mul_lo_u32 v15, v5, v12
+; GISEL-NEXT:    v_add_i32_e64 v15, s[6:7], v16, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[6:7]
+; GISEL-NEXT:    v_add_i32_e64 v15, s[6:7], v15, v19
+; GISEL-NEXT:    v_mul_lo_u32 v15, v10, v11
+; GISEL-NEXT:    v_mul_hi_u32 v19, v4, v11
+; GISEL-NEXT:    v_mul_hi_u32 v10, v10, v11
+; GISEL-NEXT:    v_mul_lo_u32 v11, v13, v12
+; GISEL-NEXT:    v_mul_hi_u32 v13, v13, v12
+; GISEL-NEXT:    v_mul_hi_u32 v12, v5, v12
+; GISEL-NEXT:    v_add_i32_e64 v14, s[8:9], v17, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v17, 0, 1, s[8:9]
+; GISEL-NEXT:    v_add_i32_e64 v8, s[8:9], v15, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[8:9]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[8:9], v11, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[8:9]
+; GISEL-NEXT:    v_add_i32_e64 v14, s[8:9], v14, v18
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[8:9]
+; GISEL-NEXT:    v_add_i32_e64 v8, s[8:9], v8, v19
+; GISEL-NEXT:    v_cndmask_b32_e64 v18, 0, 1, s[8:9]
+; GISEL-NEXT:    v_cndmask_b32_e64 v19, 0, 1, s[6:7]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[6:7], v9, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[6:7]
+; GISEL-NEXT:    v_add_i32_e64 v14, s[6:7], v17, v14
+; GISEL-NEXT:    v_add_i32_e64 v15, s[6:7], v15, v18
+; GISEL-NEXT:    v_add_i32_e64 v16, s[6:7], v16, v19
+; GISEL-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v12
+; GISEL-NEXT:    v_add_i32_e64 v8, s[6:7], v8, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[6:7]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[6:7], v9, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[6:7]
+; GISEL-NEXT:    v_add_i32_e64 v12, s[6:7], v15, v12
+; GISEL-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v14
+; GISEL-NEXT:    v_add_i32_e64 v10, s[6:7], v10, v12
+; GISEL-NEXT:    v_add_i32_e64 v11, s[6:7], v13, v11
+; GISEL-NEXT:    v_addc_u32_e32 v6, vcc, v6, v10, vcc
+; GISEL-NEXT:    v_addc_u32_e64 v7, vcc, v7, v11, s[4:5]
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v8, v3, v4
+; GISEL-NEXT:    v_mul_hi_u32 v10, v2, v4
+; GISEL-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
+; GISEL-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v9, v1, v5
+; GISEL-NEXT:    v_mul_hi_u32 v11, v0, v5
+; GISEL-NEXT:    v_mul_hi_u32 v5, v1, v5
+; GISEL-NEXT:    v_mul_lo_u32 v12, v2, v6
+; GISEL-NEXT:    v_mul_lo_u32 v13, v3, v6
+; GISEL-NEXT:    v_mul_hi_u32 v14, v2, v6
+; GISEL-NEXT:    v_mul_hi_u32 v6, v3, v6
+; GISEL-NEXT:    v_mul_lo_u32 v15, v0, v7
+; GISEL-NEXT:    v_mul_lo_u32 v16, v1, v7
+; GISEL-NEXT:    v_mul_hi_u32 v17, v0, v7
+; GISEL-NEXT:    v_mul_hi_u32 v7, v1, v7
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v13, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v16, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v17
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v12, v8
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v13, v10
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v15, v9
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v16, v11
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
+; GISEL-NEXT:    v_mul_lo_u32 v10, s10, v4
+; GISEL-NEXT:    v_mul_lo_u32 v12, 0, v4
+; GISEL-NEXT:    v_mul_hi_u32 v4, s10, v4
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
+; GISEL-NEXT:    v_mul_lo_u32 v11, s10, v5
+; GISEL-NEXT:    v_mul_lo_u32 v13, 0, v5
+; GISEL-NEXT:    v_mul_hi_u32 v5, s10, v5
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
+; GISEL-NEXT:    v_mul_lo_u32 v6, s10, v6
+; GISEL-NEXT:    v_mul_lo_u32 v7, s10, v7
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v12, v6
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v13, v7
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v10
+; GISEL-NEXT:    v_subb_u32_e64 v6, s[4:5], v3, v4, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v4
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[4:5], s10, v2
+; GISEL-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e64 v0, s[4:5], v0, v11
+; GISEL-NEXT:    v_subb_u32_e64 v7, s[6:7], v1, v5, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e64 v1, s[6:7], v1, v5
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[6:7], s10, v0
+; GISEL-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[6:7]
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[6:7], 0, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[6:7]
+; GISEL-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, 0, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, -1, vcc
+; GISEL-NEXT:    v_subbrev_u32_e64 v1, vcc, 0, v1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v6
+; GISEL-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
+; GISEL-NEXT:    v_subrev_i32_e32 v8, vcc, s10, v2
+; GISEL-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s10, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
+; GISEL-NEXT:    v_cndmask_b32_e32 v5, v9, v5, vcc
+; GISEL-NEXT:    v_subrev_i32_e32 v9, vcc, s10, v0
+; GISEL-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, s10, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, 0, v3
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, -1, vcc
+; GISEL-NEXT:    v_subrev_i32_e32 v13, vcc, s10, v8
+; GISEL-NEXT:    v_subbrev_u32_e32 v14, vcc, 0, v3, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, 0, v1
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, -1, vcc
+; GISEL-NEXT:    v_subrev_i32_e32 v16, vcc, s10, v9
+; GISEL-NEXT:    v_subbrev_u32_e32 v17, vcc, 0, v1, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v10, v12, v10, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v11, v15, v11, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GISEL-NEXT:    v_cndmask_b32_e32 v8, v8, v13, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, v9, v16, s[4:5]
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v3, v14, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v1, v1, v17, s[4:5]
+; GISEL-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v0, v0, v9, s[4:5]
+; GISEL-NEXT:    v_cndmask_b32_e64 v1, v7, v1, s[4:5]
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v6, v3, vcc
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_urem_v2i64_oddk_denom:
+; CGP:       ; %bb.0:
+; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT:    s_mov_b32 s10, 0x12d8fb
+; CGP-NEXT:    v_cvt_f32_u32_e32 v4, 0
+; CGP-NEXT:    v_cvt_f32_u32_e32 v5, s10
+; CGP-NEXT:    s_mov_b32 s8, 0xffed2705
+; CGP-NEXT:    v_mov_b32_e32 v6, v5
+; CGP-NEXT:    v_mac_f32_e32 v5, 0x4f800000, v4
+; CGP-NEXT:    v_mac_f32_e32 v6, 0x4f800000, v4
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v4, v5
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v5, v6
+; CGP-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; CGP-NEXT:    v_mul_f32_e32 v5, 0x5f7ffffc, v5
+; CGP-NEXT:    v_mul_f32_e32 v6, 0x2f800000, v4
+; CGP-NEXT:    v_mul_f32_e32 v7, 0x2f800000, v5
+; CGP-NEXT:    v_trunc_f32_e32 v6, v6
+; CGP-NEXT:    v_trunc_f32_e32 v7, v7
+; CGP-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v6
+; CGP-NEXT:    v_cvt_u32_f32_e32 v6, v6
+; CGP-NEXT:    v_mac_f32_e32 v5, 0xcf800000, v7
+; CGP-NEXT:    v_cvt_u32_f32_e32 v7, v7
+; CGP-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; CGP-NEXT:    v_mul_lo_u32 v8, s8, v6
+; CGP-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; CGP-NEXT:    v_mul_lo_u32 v9, s8, v7
+; CGP-NEXT:    v_mul_lo_u32 v10, s8, v4
+; CGP-NEXT:    v_mul_lo_u32 v11, -1, v4
+; CGP-NEXT:    v_mul_hi_u32 v12, s8, v4
+; CGP-NEXT:    v_mul_lo_u32 v13, s8, v5
+; CGP-NEXT:    v_mul_lo_u32 v14, -1, v5
+; CGP-NEXT:    v_mul_hi_u32 v15, s8, v5
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v11, v8
+; CGP-NEXT:    v_mul_lo_u32 v11, v6, v10
+; CGP-NEXT:    v_mul_hi_u32 v16, v4, v10
+; CGP-NEXT:    v_mul_hi_u32 v10, v6, v10
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v14, v9
+; CGP-NEXT:    v_mul_lo_u32 v14, v7, v13
+; CGP-NEXT:    v_mul_hi_u32 v17, v5, v13
+; CGP-NEXT:    v_mul_hi_u32 v13, v7, v13
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v15
+; CGP-NEXT:    v_mul_lo_u32 v12, v4, v8
+; CGP-NEXT:    v_mul_lo_u32 v15, v6, v8
+; CGP-NEXT:    v_mul_hi_u32 v18, v4, v8
+; CGP-NEXT:    v_mul_hi_u32 v8, v6, v8
+; CGP-NEXT:    v_mul_lo_u32 v19, v5, v9
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v19
+; CGP-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v17
+; CGP-NEXT:    v_mul_lo_u32 v14, v7, v9
+; CGP-NEXT:    v_mul_hi_u32 v17, v5, v9
+; CGP-NEXT:    v_mul_hi_u32 v9, v7, v9
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v15, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v14, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v18
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[4:5]
+; CGP-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v13, v17
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v15, v16
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, v19, v18
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v17
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v13, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v14, v15
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v12
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v10
+; CGP-NEXT:    v_addc_u32_e64 v10, s[4:5], v6, v8, vcc
+; CGP-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v8
+; CGP-NEXT:    v_mul_lo_u32 v8, s8, v4
+; CGP-NEXT:    v_mul_lo_u32 v11, -1, v4
+; CGP-NEXT:    v_mul_hi_u32 v12, s8, v4
+; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v13
+; CGP-NEXT:    v_addc_u32_e64 v13, s[6:7], v7, v9, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v7, s[6:7], v7, v9
+; CGP-NEXT:    v_mul_lo_u32 v9, s8, v5
+; CGP-NEXT:    v_mul_lo_u32 v14, -1, v5
+; CGP-NEXT:    v_mul_hi_u32 v15, s8, v5
+; CGP-NEXT:    v_mul_lo_u32 v16, s8, v10
+; CGP-NEXT:    v_mul_lo_u32 v17, v10, v8
+; CGP-NEXT:    v_mul_hi_u32 v18, v4, v8
+; CGP-NEXT:    v_mul_hi_u32 v8, v10, v8
+; CGP-NEXT:    v_mul_lo_u32 v19, s8, v13
+; CGP-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v16
+; CGP-NEXT:    v_mul_lo_u32 v16, v13, v9
+; CGP-NEXT:    v_add_i32_e64 v14, s[6:7], v14, v19
+; CGP-NEXT:    v_mul_hi_u32 v19, v5, v9
+; CGP-NEXT:    v_mul_hi_u32 v9, v13, v9
+; CGP-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v12
+; CGP-NEXT:    v_add_i32_e64 v12, s[6:7], v14, v15
+; CGP-NEXT:    v_mul_lo_u32 v14, v4, v11
+; CGP-NEXT:    v_mul_lo_u32 v15, v5, v12
+; CGP-NEXT:    v_add_i32_e64 v15, s[6:7], v16, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[6:7]
+; CGP-NEXT:    v_add_i32_e64 v15, s[6:7], v15, v19
+; CGP-NEXT:    v_mul_lo_u32 v15, v10, v11
+; CGP-NEXT:    v_mul_hi_u32 v19, v4, v11
+; CGP-NEXT:    v_mul_hi_u32 v10, v10, v11
+; CGP-NEXT:    v_mul_lo_u32 v11, v13, v12
+; CGP-NEXT:    v_mul_hi_u32 v13, v13, v12
+; CGP-NEXT:    v_mul_hi_u32 v12, v5, v12
+; CGP-NEXT:    v_add_i32_e64 v14, s[8:9], v17, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, 1, s[8:9]
+; CGP-NEXT:    v_add_i32_e64 v8, s[8:9], v15, v8
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[8:9]
+; CGP-NEXT:    v_add_i32_e64 v9, s[8:9], v11, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[8:9]
+; CGP-NEXT:    v_add_i32_e64 v14, s[8:9], v14, v18
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[8:9]
+; CGP-NEXT:    v_add_i32_e64 v8, s[8:9], v8, v19
+; CGP-NEXT:    v_cndmask_b32_e64 v18, 0, 1, s[8:9]
+; CGP-NEXT:    v_cndmask_b32_e64 v19, 0, 1, s[6:7]
+; CGP-NEXT:    v_add_i32_e64 v9, s[6:7], v9, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[6:7]
+; CGP-NEXT:    v_add_i32_e64 v14, s[6:7], v17, v14
+; CGP-NEXT:    v_add_i32_e64 v15, s[6:7], v15, v18
+; CGP-NEXT:    v_add_i32_e64 v16, s[6:7], v16, v19
+; CGP-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v12
+; CGP-NEXT:    v_add_i32_e64 v8, s[6:7], v8, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[6:7]
+; CGP-NEXT:    v_add_i32_e64 v9, s[6:7], v9, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[6:7]
+; CGP-NEXT:    v_add_i32_e64 v12, s[6:7], v15, v12
+; CGP-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v14
+; CGP-NEXT:    v_add_i32_e64 v10, s[6:7], v10, v12
+; CGP-NEXT:    v_add_i32_e64 v11, s[6:7], v13, v11
+; CGP-NEXT:    v_addc_u32_e32 v6, vcc, v6, v10, vcc
+; CGP-NEXT:    v_addc_u32_e64 v7, vcc, v7, v11, s[4:5]
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; CGP-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; CGP-NEXT:    v_mul_lo_u32 v8, v3, v4
+; CGP-NEXT:    v_mul_hi_u32 v10, v2, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
+; CGP-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; CGP-NEXT:    v_mul_lo_u32 v9, v1, v5
+; CGP-NEXT:    v_mul_hi_u32 v11, v0, v5
+; CGP-NEXT:    v_mul_hi_u32 v5, v1, v5
+; CGP-NEXT:    v_mul_lo_u32 v12, v2, v6
+; CGP-NEXT:    v_mul_lo_u32 v13, v3, v6
+; CGP-NEXT:    v_mul_hi_u32 v14, v2, v6
+; CGP-NEXT:    v_mul_hi_u32 v6, v3, v6
+; CGP-NEXT:    v_mul_lo_u32 v15, v0, v7
+; CGP-NEXT:    v_mul_lo_u32 v16, v1, v7
+; CGP-NEXT:    v_mul_hi_u32 v17, v0, v7
+; CGP-NEXT:    v_mul_hi_u32 v7, v1, v7
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v13, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v16, v5
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v17
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v12, v8
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v13, v10
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v15, v9
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v16, v11
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; CGP-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
+; CGP-NEXT:    v_mul_lo_u32 v10, s10, v4
+; CGP-NEXT:    v_mul_lo_u32 v12, 0, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, s10, v4
+; CGP-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
+; CGP-NEXT:    v_mul_lo_u32 v11, s10, v5
+; CGP-NEXT:    v_mul_lo_u32 v13, 0, v5
+; CGP-NEXT:    v_mul_hi_u32 v5, s10, v5
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
+; CGP-NEXT:    v_mul_lo_u32 v6, s10, v6
+; CGP-NEXT:    v_mul_lo_u32 v7, s10, v7
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v12, v6
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v13, v7
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; CGP-NEXT:    v_sub_i32_e32 v2, vcc, v2, v10
+; CGP-NEXT:    v_subb_u32_e64 v6, s[4:5], v3, v4, vcc
+; CGP-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v4
+; CGP-NEXT:    v_cmp_le_u32_e64 s[4:5], s10, v2
+; CGP-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[4:5]
+; CGP-NEXT:    v_sub_i32_e64 v0, s[4:5], v0, v11
+; CGP-NEXT:    v_subb_u32_e64 v7, s[6:7], v1, v5, s[4:5]
+; CGP-NEXT:    v_sub_i32_e64 v1, s[6:7], v1, v5
+; CGP-NEXT:    v_cmp_le_u32_e64 s[6:7], s10, v0
+; CGP-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[6:7]
+; CGP-NEXT:    v_cmp_le_u32_e64 s[6:7], 0, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[6:7]
+; CGP-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, 0, v7
+; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, -1, vcc
+; CGP-NEXT:    v_subbrev_u32_e64 v1, vcc, 0, v1, s[4:5]
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v6
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
+; CGP-NEXT:    v_subrev_i32_e32 v8, vcc, s10, v2
+; CGP-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s10, v8
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, -1, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
+; CGP-NEXT:    v_cndmask_b32_e32 v5, v9, v5, vcc
+; CGP-NEXT:    v_subrev_i32_e32 v9, vcc, s10, v0
+; CGP-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, s10, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, -1, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, 0, v3
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, -1, vcc
+; CGP-NEXT:    v_subrev_i32_e32 v13, vcc, s10, v8
+; CGP-NEXT:    v_subbrev_u32_e32 v14, vcc, 0, v3, vcc
+; CGP-NEXT:    v_cmp_le_u32_e32 vcc, 0, v1
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, -1, vcc
+; CGP-NEXT:    v_subrev_i32_e32 v16, vcc, s10, v9
+; CGP-NEXT:    v_subbrev_u32_e32 v17, vcc, 0, v1, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; CGP-NEXT:    v_cndmask_b32_e32 v10, v12, v10, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v11, v15, v11, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; CGP-NEXT:    v_cndmask_b32_e32 v8, v8, v13, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v9, v9, v16, s[4:5]
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v3, v14, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; CGP-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v1, v1, v17, s[4:5]
+; CGP-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v5
+; CGP-NEXT:    v_cndmask_b32_e64 v0, v0, v9, s[4:5]
+; CGP-NEXT:    v_cndmask_b32_e64 v1, v7, v1, s[4:5]
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v6, v3, vcc
+; CGP-NEXT:    s_setpc_b64 s[30:31]
+  %result = urem <2 x i64> %num, <i64 1235195, i64 1235195>
+  ret <2 x i64> %result
+}
+
+define i64 @v_urem_i64_pow2_shl_denom(i64 %x, i64 %y) {
+; CHECK-LABEL: v_urem_i64_pow2_shl_denom:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_movk_i32 s4, 0x1000
+; CHECK-NEXT:    s_mov_b32 s5, 0
+; CHECK-NEXT:    v_mov_b32_e32 v6, 0
+; CHECK-NEXT:    v_lshl_b64 v[4:5], s[4:5], v2
+; CHECK-NEXT:    v_or_b32_e32 v7, v1, v5
+; CHECK-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; CHECK-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, 1
+; CHECK-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; CHECK-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; CHECK-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
+; CHECK-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; CHECK-NEXT:    s_cbranch_execnz BB7_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v2, v4
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v3, v5
+; CHECK-NEXT:    v_sub_i32_e32 v6, vcc, 0, v4
+; CHECK-NEXT:    v_subb_u32_e32 v7, vcc, 0, v5, vcc
+; CHECK-NEXT:    v_mac_f32_e32 v2, 0x4f800000, v3
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; CHECK-NEXT:    v_mul_f32_e32 v2, 0x5f7ffffc, v2
+; CHECK-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v2
+; CHECK-NEXT:    v_trunc_f32_e32 v3, v3
+; CHECK-NEXT:    v_mac_f32_e32 v2, 0xcf800000, v3
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; CHECK-NEXT:    v_mul_lo_u32 v8, v6, v3
+; CHECK-NEXT:    v_mul_lo_u32 v9, v6, v2
+; CHECK-NEXT:    v_mul_lo_u32 v10, v7, v2
+; CHECK-NEXT:    v_mul_hi_u32 v11, v6, v2
+; CHECK-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
+; CHECK-NEXT:    v_mul_lo_u32 v10, v3, v9
+; CHECK-NEXT:    v_mul_hi_u32 v12, v2, v9
+; CHECK-NEXT:    v_mul_hi_u32 v9, v3, v9
+; CHECK-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
+; CHECK-NEXT:    v_mul_lo_u32 v11, v2, v8
+; CHECK-NEXT:    v_mul_lo_u32 v13, v3, v8
+; CHECK-NEXT:    v_mul_hi_u32 v14, v2, v8
+; CHECK-NEXT:    v_mul_hi_u32 v8, v3, v8
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; CHECK-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v9, vcc, v13, v9
+; CHECK-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v9, vcc, v9, v14
+; CHECK-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; CHECK-NEXT:    v_add_i32_e32 v11, vcc, v13, v12
+; CHECK-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; CHECK-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v9
+; CHECK-NEXT:    v_addc_u32_e64 v9, s[4:5], v3, v8, vcc
+; CHECK-NEXT:    v_add_i32_e64 v3, s[4:5], v3, v8
+; CHECK-NEXT:    v_mul_lo_u32 v8, v6, v2
+; CHECK-NEXT:    v_mul_lo_u32 v7, v7, v2
+; CHECK-NEXT:    v_mul_hi_u32 v10, v6, v2
+; CHECK-NEXT:    v_mul_lo_u32 v6, v6, v9
+; CHECK-NEXT:    v_mul_lo_u32 v11, v9, v8
+; CHECK-NEXT:    v_mul_hi_u32 v12, v2, v8
+; CHECK-NEXT:    v_mul_hi_u32 v8, v9, v8
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v7, v6
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v10
+; CHECK-NEXT:    v_mul_lo_u32 v7, v2, v6
+; CHECK-NEXT:    v_mul_lo_u32 v10, v9, v6
+; CHECK-NEXT:    v_mul_hi_u32 v13, v2, v6
+; CHECK-NEXT:    v_mul_hi_u32 v6, v9, v6
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v11, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v8, s[4:5], v10, v8
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v7, v12
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v13
+; CHECK-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v9, v7
+; CHECK-NEXT:    v_add_i32_e64 v9, s[4:5], v10, v11
+; CHECK-NEXT:    v_add_i32_e64 v7, s[4:5], v8, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s[4:5]
+; CHECK-NEXT:    v_add_i32_e64 v8, s[4:5], v9, v8
+; CHECK-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v8
+; CHECK-NEXT:    v_addc_u32_e32 v3, vcc, v3, v6, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v7
+; CHECK-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; CHECK-NEXT:    v_mul_lo_u32 v6, v1, v2
+; CHECK-NEXT:    v_mul_hi_u32 v7, v0, v2
+; CHECK-NEXT:    v_mul_hi_u32 v2, v1, v2
+; CHECK-NEXT:    v_mul_lo_u32 v8, v0, v3
+; CHECK-NEXT:    v_mul_lo_u32 v9, v1, v3
+; CHECK-NEXT:    v_mul_hi_u32 v10, v0, v3
+; CHECK-NEXT:    v_mul_hi_u32 v3, v1, v3
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v9, v2
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v10
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v9, v7
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CHECK-NEXT:    v_mul_lo_u32 v7, v4, v2
+; CHECK-NEXT:    v_mul_lo_u32 v8, v5, v2
+; CHECK-NEXT:    v_mul_hi_u32 v2, v4, v2
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
+; CHECK-NEXT:    v_mul_lo_u32 v3, v4, v3
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v8, v3
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; CHECK-NEXT:    v_sub_i32_e32 v3, vcc, v0, v7
+; CHECK-NEXT:    v_subb_u32_e64 v6, s[4:5], v1, v2, vcc
+; CHECK-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v2
+; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v3, v4
+; CHECK-NEXT:    v_cndmask_b32_e64 v2, 0, -1, s[4:5]
+; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v6, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[4:5]
+; CHECK-NEXT:    v_subb_u32_e32 v1, vcc, v1, v5, vcc
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, v6, v5
+; CHECK-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
+; CHECK-NEXT:    v_sub_i32_e32 v7, vcc, v3, v4
+; CHECK-NEXT:    v_subbrev_u32_e64 v8, s[4:5], 0, v1, vcc
+; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v7, v4
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; CHECK-NEXT:    v_subb_u32_e32 v1, vcc, v1, v5, vcc
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v8, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, -1, vcc
+; CHECK-NEXT:    v_sub_i32_e32 v11, vcc, v7, v4
+; CHECK-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, v8, v5
+; CHECK-NEXT:    v_cndmask_b32_e32 v5, v10, v9, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; CHECK-NEXT:    v_cndmask_b32_e32 v5, v7, v11, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
+; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; CHECK-NEXT:    v_cndmask_b32_e32 v2, v3, v5, vcc
+; CHECK-NEXT:    v_cndmask_b32_e32 v3, v6, v1, vcc
+; CHECK-NEXT:  BB7_2: ; %Flow
+; CHECK-NEXT:    s_or_saveexec_b64 s[8:9], s[6:7]
+; CHECK-NEXT:    s_xor_b64 exec, exec, s[8:9]
+; CHECK-NEXT:    s_cbranch_execz BB7_4
+; CHECK-NEXT:  ; %bb.3:
+; CHECK-NEXT:    v_cvt_f32_u32_e32 v1, v4
+; CHECK-NEXT:    v_rcp_iflag_f32_e32 v1, v1
+; CHECK-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v1
+; CHECK-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; CHECK-NEXT:    v_mul_lo_u32 v2, v1, v4
+; CHECK-NEXT:    v_mul_hi_u32 v3, v1, v4
+; CHECK-NEXT:    v_sub_i32_e32 v5, vcc, 0, v2
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; CHECK-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
+; CHECK-NEXT:    v_mul_hi_u32 v2, v2, v1
+; CHECK-NEXT:    v_add_i32_e64 v3, s[4:5], v1, v2
+; CHECK-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v2
+; CHECK-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; CHECK-NEXT:    v_mul_hi_u32 v1, v1, v0
+; CHECK-NEXT:    v_mul_lo_u32 v1, v1, v4
+; CHECK-NEXT:    v_sub_i32_e32 v2, vcc, v0, v1
+; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v4
+; CHECK-NEXT:    v_add_i32_e64 v3, s[4:5], v2, v4
+; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v1
+; CHECK-NEXT:    v_sub_i32_e64 v0, s[6:7], v2, v4
+; CHECK-NEXT:    s_and_b64 vcc, vcc, s[4:5]
+; CHECK-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; CHECK-NEXT:    v_cndmask_b32_e64 v2, v3, v0, s[4:5]
+; CHECK-NEXT:    v_mov_b32_e32 v3, 0
+; CHECK-NEXT:  BB7_4:
+; CHECK-NEXT:    s_or_b64 exec, exec, s[8:9]
+; CHECK-NEXT:    v_mov_b32_e32 v0, v2
+; CHECK-NEXT:    v_mov_b32_e32 v1, v3
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %shl.y = shl i64 4096, %y
+  %r = urem i64 %x, %shl.y
+  ret i64 %r
+}
+
+define <2 x i64> @v_urem_v2i64_pow2_shl_denom(<2 x i64> %x, <2 x i64> %y) {
+; GISEL-LABEL: v_urem_v2i64_pow2_shl_denom:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    s_movk_i32 s4, 0x1000
+; GISEL-NEXT:    s_mov_b32 s5, 0
+; GISEL-NEXT:    v_lshl_b64 v[4:5], s[4:5], v4
+; GISEL-NEXT:    v_lshl_b64 v[6:7], s[4:5], v6
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v8, v4
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v9, v5
+; GISEL-NEXT:    v_mac_f32_e32 v8, 0x4f800000, v9
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v8, v8
+; GISEL-NEXT:    v_mul_f32_e32 v8, 0x5f7ffffc, v8
+; GISEL-NEXT:    v_mul_f32_e32 v9, 0x2f800000, v8
+; GISEL-NEXT:    v_trunc_f32_e32 v9, v9
+; GISEL-NEXT:    v_mac_f32_e32 v8, 0xcf800000, v9
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v8, v8
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v9, v9
+; GISEL-NEXT:    v_sub_i32_e32 v10, vcc, 0, v4
+; GISEL-NEXT:    v_subb_u32_e32 v11, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v12, v10, v8
+; GISEL-NEXT:    v_mul_lo_u32 v13, v11, v8
+; GISEL-NEXT:    v_mul_lo_u32 v14, v10, v9
+; GISEL-NEXT:    v_mul_hi_u32 v15, v10, v8
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v15
+; GISEL-NEXT:    v_mul_lo_u32 v14, v9, v12
+; GISEL-NEXT:    v_mul_lo_u32 v15, v8, v13
+; GISEL-NEXT:    v_mul_hi_u32 v16, v8, v12
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; GISEL-NEXT:    v_mul_lo_u32 v15, v9, v13
+; GISEL-NEXT:    v_mul_hi_u32 v12, v9, v12
+; GISEL-NEXT:    v_mul_hi_u32 v16, v8, v13
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v15, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v15, v16
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; GISEL-NEXT:    v_mul_hi_u32 v13, v9, v13
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
+; GISEL-NEXT:    v_addc_u32_e64 v12, s[4:5], v9, v13, vcc
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v13
+; GISEL-NEXT:    v_mul_lo_u32 v13, v10, v8
+; GISEL-NEXT:    v_mul_lo_u32 v11, v11, v8
+; GISEL-NEXT:    v_mul_lo_u32 v14, v10, v12
+; GISEL-NEXT:    v_mul_hi_u32 v10, v10, v8
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v14
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v11, v10
+; GISEL-NEXT:    v_mul_lo_u32 v11, v12, v13
+; GISEL-NEXT:    v_mul_lo_u32 v14, v8, v10
+; GISEL-NEXT:    v_mul_hi_u32 v15, v8, v13
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v14, v11
+; GISEL-NEXT:    v_mul_lo_u32 v14, v12, v10
+; GISEL-NEXT:    v_mul_hi_u32 v13, v12, v13
+; GISEL-NEXT:    v_mul_hi_u32 v15, v8, v10
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v14, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v14, s[4:5], v14, v15
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v13, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v14, v13
+; GISEL-NEXT:    v_mul_hi_u32 v10, v12, v10
+; GISEL-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v13
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v11
+; GISEL-NEXT:    v_addc_u32_e32 v9, vcc, v9, v10, vcc
+; GISEL-NEXT:    v_addc_u32_e64 v9, vcc, 0, v9, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v10, v1, v8
+; GISEL-NEXT:    v_mul_lo_u32 v11, v0, v9
+; GISEL-NEXT:    v_mul_hi_u32 v12, v0, v8
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT:    v_mul_lo_u32 v11, v1, v9
+; GISEL-NEXT:    v_mul_hi_u32 v8, v1, v8
+; GISEL-NEXT:    v_mul_hi_u32 v12, v0, v9
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v11, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; GISEL-NEXT:    v_mul_hi_u32 v9, v1, v9
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT:    v_mul_lo_u32 v10, v4, v8
+; GISEL-NEXT:    v_mul_lo_u32 v11, v5, v8
+; GISEL-NEXT:    v_mul_lo_u32 v9, v4, v9
+; GISEL-NEXT:    v_mul_hi_u32 v8, v4, v8
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v10
+; GISEL-NEXT:    v_subb_u32_e64 v9, s[4:5], v1, v8, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v1, s[4:5], v1, v8
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v9, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], v9, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, v8, v10, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e64 v10, s[4:5], v0, v4
+; GISEL-NEXT:    v_subb_u32_e32 v1, vcc, v1, v5, vcc
+; GISEL-NEXT:    v_subbrev_u32_e64 v11, vcc, 0, v1, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v11, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v10, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v11, v5
+; GISEL-NEXT:    v_cndmask_b32_e32 v12, v12, v13, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v4, vcc, v10, v4
+; GISEL-NEXT:    v_subb_u32_e64 v1, s[4:5], v1, v5, s[4:5]
+; GISEL-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GISEL-NEXT:    v_cndmask_b32_e32 v4, v10, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v11, v1, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v8
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v9, v1, vcc
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v4, v6
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v5, v7
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; GISEL-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; GISEL-NEXT:    v_mul_f32_e32 v5, 0x2f800000, v4
+; GISEL-NEXT:    v_trunc_f32_e32 v5, v5
+; GISEL-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v5
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; GISEL-NEXT:    v_sub_i32_e32 v8, vcc, 0, v6
+; GISEL-NEXT:    v_subb_u32_e32 v9, vcc, 0, v7, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v10, v8, v4
+; GISEL-NEXT:    v_mul_lo_u32 v11, v9, v4
+; GISEL-NEXT:    v_mul_lo_u32 v12, v8, v5
+; GISEL-NEXT:    v_mul_hi_u32 v13, v8, v4
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v13
+; GISEL-NEXT:    v_mul_lo_u32 v12, v5, v10
+; GISEL-NEXT:    v_mul_lo_u32 v13, v4, v11
+; GISEL-NEXT:    v_mul_hi_u32 v14, v4, v10
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; GISEL-NEXT:    v_mul_lo_u32 v13, v5, v11
+; GISEL-NEXT:    v_mul_hi_u32 v10, v5, v10
+; GISEL-NEXT:    v_mul_hi_u32 v14, v4, v11
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v13, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; GISEL-NEXT:    v_mul_hi_u32 v11, v5, v11
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v10
+; GISEL-NEXT:    v_addc_u32_e64 v10, s[4:5], v5, v11, vcc
+; GISEL-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v11
+; GISEL-NEXT:    v_mul_lo_u32 v11, v8, v4
+; GISEL-NEXT:    v_mul_lo_u32 v9, v9, v4
+; GISEL-NEXT:    v_mul_lo_u32 v12, v8, v10
+; GISEL-NEXT:    v_mul_hi_u32 v8, v8, v4
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v12
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v9, v8
+; GISEL-NEXT:    v_mul_lo_u32 v9, v10, v11
+; GISEL-NEXT:    v_mul_lo_u32 v12, v4, v8
+; GISEL-NEXT:    v_mul_hi_u32 v13, v4, v11
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v9, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v12, v9
+; GISEL-NEXT:    v_mul_lo_u32 v12, v10, v8
+; GISEL-NEXT:    v_mul_hi_u32 v11, v10, v11
+; GISEL-NEXT:    v_mul_hi_u32 v13, v4, v8
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v12, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v13
+; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v11, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v12, v11
+; GISEL-NEXT:    v_mul_hi_u32 v8, v10, v8
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v11
+; GISEL-NEXT:    v_add_i32_e64 v4, s[4:5], v4, v9
+; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, v5, v8, vcc
+; GISEL-NEXT:    v_addc_u32_e64 v5, vcc, 0, v5, s[4:5]
+; GISEL-NEXT:    v_mul_lo_u32 v8, v3, v4
+; GISEL-NEXT:    v_mul_lo_u32 v9, v2, v5
+; GISEL-NEXT:    v_mul_hi_u32 v10, v2, v4
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT:    v_mul_lo_u32 v9, v3, v5
+; GISEL-NEXT:    v_mul_hi_u32 v4, v3, v4
+; GISEL-NEXT:    v_mul_hi_u32 v10, v2, v5
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v9, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GISEL-NEXT:    v_mul_hi_u32 v5, v3, v5
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v8
+; GISEL-NEXT:    v_mul_lo_u32 v8, v6, v4
+; GISEL-NEXT:    v_mul_lo_u32 v9, v7, v4
+; GISEL-NEXT:    v_mul_lo_u32 v5, v6, v5
+; GISEL-NEXT:    v_mul_hi_u32 v4, v6, v4
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v9, v5
+; GISEL-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; GISEL-NEXT:    v_sub_i32_e32 v2, vcc, v2, v8
+; GISEL-NEXT:    v_subb_u32_e64 v5, s[4:5], v3, v4, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v4
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v5, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v2, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], v5, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v4, v4, v8, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e64 v8, s[4:5], v2, v6
+; GISEL-NEXT:    v_subb_u32_e32 v3, vcc, v3, v7, vcc
+; GISEL-NEXT:    v_subbrev_u32_e64 v9, vcc, 0, v3, s[4:5]
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v9, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v8, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, v9, v7
+; GISEL-NEXT:    v_cndmask_b32_e32 v10, v10, v11, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v6, vcc, v8, v6
+; GISEL-NEXT:    v_subb_u32_e64 v3, s[4:5], v3, v7, s[4:5]
+; GISEL-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v10
+; GISEL-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v9, v3, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v5, v3, vcc
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_urem_v2i64_pow2_shl_denom:
+; CGP:       ; %bb.0:
+; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT:    v_mov_b32_e32 v5, v0
+; CGP-NEXT:    v_mov_b32_e32 v7, v1
+; CGP-NEXT:    s_movk_i32 s4, 0x1000
+; CGP-NEXT:    s_mov_b32 s5, 0
+; CGP-NEXT:    v_mov_b32_e32 v0, 0
+; CGP-NEXT:    v_lshl_b64 v[10:11], s[4:5], v4
+; CGP-NEXT:    v_lshl_b64 v[8:9], s[4:5], v6
+; CGP-NEXT:    v_or_b32_e32 v1, v7, v11
+; CGP-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
+; CGP-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, 1
+; CGP-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; CGP-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; CGP-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
+; CGP-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; CGP-NEXT:    s_cbranch_execnz BB8_2
+; CGP-NEXT:  ; %bb.1:
+; CGP-NEXT:    v_cvt_f32_u32_e32 v0, v10
+; CGP-NEXT:    v_cvt_f32_u32_e32 v1, v11
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, 0, v10
+; CGP-NEXT:    v_subb_u32_e32 v6, vcc, 0, v11, vcc
+; CGP-NEXT:    v_mac_f32_e32 v0, 0x4f800000, v1
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; CGP-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
+; CGP-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
+; CGP-NEXT:    v_trunc_f32_e32 v1, v1
+; CGP-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
+; CGP-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; CGP-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; CGP-NEXT:    v_mul_lo_u32 v12, v4, v1
+; CGP-NEXT:    v_mul_lo_u32 v13, v4, v0
+; CGP-NEXT:    v_mul_lo_u32 v14, v6, v0
+; CGP-NEXT:    v_mul_hi_u32 v15, v4, v0
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v14, v12
+; CGP-NEXT:    v_mul_lo_u32 v14, v1, v13
+; CGP-NEXT:    v_mul_hi_u32 v16, v0, v13
+; CGP-NEXT:    v_mul_hi_u32 v13, v1, v13
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v12, v15
+; CGP-NEXT:    v_mul_lo_u32 v15, v0, v12
+; CGP-NEXT:    v_mul_lo_u32 v17, v1, v12
+; CGP-NEXT:    v_mul_hi_u32 v18, v0, v12
+; CGP-NEXT:    v_mul_hi_u32 v12, v1, v12
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v17, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v14, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v13, v18
+; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; CGP-NEXT:    v_add_i32_e32 v15, vcc, v17, v16
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v14, vcc, v15, v14
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v12, v14
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v13
+; CGP-NEXT:    v_addc_u32_e64 v13, s[4:5], v1, v12, vcc
+; CGP-NEXT:    v_add_i32_e64 v1, s[4:5], v1, v12
+; CGP-NEXT:    v_mul_lo_u32 v12, v4, v0
+; CGP-NEXT:    v_mul_lo_u32 v6, v6, v0
+; CGP-NEXT:    v_mul_hi_u32 v14, v4, v0
+; CGP-NEXT:    v_mul_lo_u32 v4, v4, v13
+; CGP-NEXT:    v_mul_lo_u32 v15, v13, v12
+; CGP-NEXT:    v_mul_hi_u32 v16, v0, v12
+; CGP-NEXT:    v_mul_hi_u32 v12, v13, v12
+; CGP-NEXT:    v_add_i32_e64 v4, s[4:5], v6, v4
+; CGP-NEXT:    v_add_i32_e64 v4, s[4:5], v4, v14
+; CGP-NEXT:    v_mul_lo_u32 v6, v0, v4
+; CGP-NEXT:    v_mul_lo_u32 v14, v13, v4
+; CGP-NEXT:    v_mul_hi_u32 v17, v0, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, v13, v4
+; CGP-NEXT:    v_add_i32_e64 v6, s[4:5], v15, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v14, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v17
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v6, s[4:5], v13, v6
+; CGP-NEXT:    v_add_i32_e64 v13, s[4:5], v14, v15
+; CGP-NEXT:    v_add_i32_e64 v6, s[4:5], v12, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v12, s[4:5], v13, v12
+; CGP-NEXT:    v_add_i32_e64 v4, s[4:5], v4, v12
+; CGP-NEXT:    v_addc_u32_e32 v1, vcc, v1, v4, vcc
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v6
+; CGP-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; CGP-NEXT:    v_mul_lo_u32 v4, v7, v0
+; CGP-NEXT:    v_mul_hi_u32 v6, v5, v0
+; CGP-NEXT:    v_mul_hi_u32 v0, v7, v0
+; CGP-NEXT:    v_mul_lo_u32 v12, v5, v1
+; CGP-NEXT:    v_mul_lo_u32 v13, v7, v1
+; CGP-NEXT:    v_mul_hi_u32 v14, v5, v1
+; CGP-NEXT:    v_mul_hi_u32 v1, v7, v1
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v13, v0
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v12, v4
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v13, v6
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
+; CGP-NEXT:    v_mul_lo_u32 v6, v10, v0
+; CGP-NEXT:    v_mul_lo_u32 v12, v11, v0
+; CGP-NEXT:    v_mul_hi_u32 v0, v10, v0
+; CGP-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
+; CGP-NEXT:    v_mul_lo_u32 v1, v10, v1
+; CGP-NEXT:    v_add_i32_e32 v1, vcc, v12, v1
+; CGP-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v5, v6
+; CGP-NEXT:    v_subb_u32_e64 v4, s[4:5], v7, v0, vcc
+; CGP-NEXT:    v_sub_i32_e64 v0, s[4:5], v7, v0
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v1, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v4, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[4:5]
+; CGP-NEXT:    v_subb_u32_e32 v0, vcc, v0, v11, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v11
+; CGP-NEXT:    v_cndmask_b32_e32 v6, v7, v6, vcc
+; CGP-NEXT:    v_sub_i32_e32 v7, vcc, v1, v10
+; CGP-NEXT:    v_subbrev_u32_e64 v12, s[4:5], 0, v0, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v7, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, -1, s[4:5]
+; CGP-NEXT:    v_subb_u32_e32 v0, vcc, v0, v11, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v12, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, -1, vcc
+; CGP-NEXT:    v_sub_i32_e32 v15, vcc, v7, v10
+; CGP-NEXT:    v_subbrev_u32_e32 v0, vcc, 0, v0, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, v12, v11
+; CGP-NEXT:    v_cndmask_b32_e32 v11, v14, v13, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v11
+; CGP-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v11, v12, v0, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v1, v7, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v4, v11, vcc
+; CGP-NEXT:  BB8_2: ; %Flow1
+; CGP-NEXT:    s_or_saveexec_b64 s[8:9], s[6:7]
+; CGP-NEXT:    s_xor_b64 exec, exec, s[8:9]
+; CGP-NEXT:    s_cbranch_execz BB8_4
+; CGP-NEXT:  ; %bb.3:
+; CGP-NEXT:    v_cvt_f32_u32_e32 v0, v10
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; CGP-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; CGP-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; CGP-NEXT:    v_mul_lo_u32 v1, v0, v10
+; CGP-NEXT:    v_mul_hi_u32 v4, v0, v10
+; CGP-NEXT:    v_sub_i32_e32 v6, vcc, 0, v1
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; CGP-NEXT:    v_mul_hi_u32 v1, v1, v0
+; CGP-NEXT:    v_add_i32_e64 v4, s[4:5], v0, v1
+; CGP-NEXT:    v_sub_i32_e64 v0, s[4:5], v0, v1
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; CGP-NEXT:    v_mul_hi_u32 v0, v0, v5
+; CGP-NEXT:    v_mul_lo_u32 v0, v0, v10
+; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v5, v0
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v10
+; CGP-NEXT:    v_add_i32_e64 v4, s[4:5], v1, v10
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v5, v0
+; CGP-NEXT:    v_sub_i32_e64 v0, s[6:7], v1, v10
+; CGP-NEXT:    s_and_b64 vcc, vcc, s[4:5]
+; CGP-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[4:5]
+; CGP-NEXT:    v_mov_b32_e32 v1, 0
+; CGP-NEXT:  BB8_4:
+; CGP-NEXT:    s_or_b64 exec, exec, s[8:9]
+; CGP-NEXT:    v_or_b32_e32 v5, v3, v9
+; CGP-NEXT:    v_mov_b32_e32 v4, 0
+; CGP-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[4:5]
+; CGP-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, 1
+; CGP-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; CGP-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; CGP-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
+; CGP-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; CGP-NEXT:    s_cbranch_execnz BB8_6
+; CGP-NEXT:  ; %bb.5:
+; CGP-NEXT:    v_cvt_f32_u32_e32 v4, v8
+; CGP-NEXT:    v_cvt_f32_u32_e32 v5, v9
+; CGP-NEXT:    v_sub_i32_e32 v6, vcc, 0, v8
+; CGP-NEXT:    v_subb_u32_e32 v7, vcc, 0, v9, vcc
+; CGP-NEXT:    v_mac_f32_e32 v4, 0x4f800000, v5
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; CGP-NEXT:    v_mul_f32_e32 v4, 0x5f7ffffc, v4
+; CGP-NEXT:    v_mul_f32_e32 v5, 0x2f800000, v4
+; CGP-NEXT:    v_trunc_f32_e32 v5, v5
+; CGP-NEXT:    v_mac_f32_e32 v4, 0xcf800000, v5
+; CGP-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; CGP-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; CGP-NEXT:    v_mul_lo_u32 v10, v6, v5
+; CGP-NEXT:    v_mul_lo_u32 v11, v6, v4
+; CGP-NEXT:    v_mul_lo_u32 v12, v7, v4
+; CGP-NEXT:    v_mul_hi_u32 v13, v6, v4
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
+; CGP-NEXT:    v_mul_lo_u32 v12, v5, v11
+; CGP-NEXT:    v_mul_hi_u32 v14, v4, v11
+; CGP-NEXT:    v_mul_hi_u32 v11, v5, v11
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v13
+; CGP-NEXT:    v_mul_lo_u32 v13, v4, v10
+; CGP-NEXT:    v_mul_lo_u32 v15, v5, v10
+; CGP-NEXT:    v_mul_hi_u32 v16, v4, v10
+; CGP-NEXT:    v_mul_hi_u32 v10, v5, v10
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v15, v11
+; CGP-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v12, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v11, v16
+; CGP-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT:    v_add_i32_e32 v13, vcc, v15, v14
+; CGP-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v11
+; CGP-NEXT:    v_addc_u32_e64 v11, s[4:5], v5, v10, vcc
+; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v10
+; CGP-NEXT:    v_mul_lo_u32 v10, v6, v4
+; CGP-NEXT:    v_mul_lo_u32 v7, v7, v4
+; CGP-NEXT:    v_mul_hi_u32 v12, v6, v4
+; CGP-NEXT:    v_mul_lo_u32 v6, v6, v11
+; CGP-NEXT:    v_mul_lo_u32 v13, v11, v10
+; CGP-NEXT:    v_mul_hi_u32 v14, v4, v10
+; CGP-NEXT:    v_mul_hi_u32 v10, v11, v10
+; CGP-NEXT:    v_add_i32_e64 v6, s[4:5], v7, v6
+; CGP-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v12
+; CGP-NEXT:    v_mul_lo_u32 v7, v4, v6
+; CGP-NEXT:    v_mul_lo_u32 v12, v11, v6
+; CGP-NEXT:    v_mul_hi_u32 v15, v4, v6
+; CGP-NEXT:    v_mul_hi_u32 v6, v11, v6
+; CGP-NEXT:    v_add_i32_e64 v7, s[4:5], v13, v7
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v12, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v7, s[4:5], v7, v14
+; CGP-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v10, v15
+; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v7, s[4:5], v11, v7
+; CGP-NEXT:    v_add_i32_e64 v11, s[4:5], v12, v13
+; CGP-NEXT:    v_add_i32_e64 v7, s[4:5], v10, v7
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e64 v10, s[4:5], v11, v10
+; CGP-NEXT:    v_add_i32_e64 v6, s[4:5], v6, v10
+; CGP-NEXT:    v_addc_u32_e32 v5, vcc, v5, v6, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v7
+; CGP-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; CGP-NEXT:    v_mul_lo_u32 v6, v3, v4
+; CGP-NEXT:    v_mul_hi_u32 v7, v2, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CGP-NEXT:    v_mul_lo_u32 v10, v2, v5
+; CGP-NEXT:    v_mul_lo_u32 v11, v3, v5
+; CGP-NEXT:    v_mul_hi_u32 v12, v2, v5
+; CGP-NEXT:    v_mul_hi_u32 v5, v3, v5
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v10
+; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v11, v4
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
+; CGP-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v12
+; CGP-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v10, v6
+; CGP-NEXT:    v_add_i32_e32 v7, vcc, v11, v7
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
+; CGP-NEXT:    v_mul_lo_u32 v7, v8, v4
+; CGP-NEXT:    v_mul_lo_u32 v10, v9, v4
+; CGP-NEXT:    v_mul_hi_u32 v4, v8, v4
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; CGP-NEXT:    v_mul_lo_u32 v5, v8, v5
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v10, v5
+; CGP-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; CGP-NEXT:    v_sub_i32_e32 v5, vcc, v2, v7
+; CGP-NEXT:    v_subb_u32_e64 v6, s[4:5], v3, v4, vcc
+; CGP-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v4
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v5, v8
+; CGP-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[4:5]
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v6, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[4:5]
+; CGP-NEXT:    v_subb_u32_e32 v3, vcc, v3, v9, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, v6, v9
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
+; CGP-NEXT:    v_sub_i32_e32 v7, vcc, v5, v8
+; CGP-NEXT:    v_subbrev_u32_e64 v10, s[4:5], 0, v3, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v7, v8
+; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, -1, s[4:5]
+; CGP-NEXT:    v_subb_u32_e32 v3, vcc, v3, v9, vcc
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v10, v9
+; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, -1, vcc
+; CGP-NEXT:    v_sub_i32_e32 v13, vcc, v7, v8
+; CGP-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v3, vcc
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, v10, v9
+; CGP-NEXT:    v_cndmask_b32_e32 v9, v12, v11, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v9
+; CGP-NEXT:    v_cndmask_b32_e32 v7, v7, v13, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc
+; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v5, v7, vcc
+; CGP-NEXT:    v_cndmask_b32_e32 v5, v6, v3, vcc
+; CGP-NEXT:  BB8_6: ; %Flow
+; CGP-NEXT:    s_or_saveexec_b64 s[8:9], s[6:7]
+; CGP-NEXT:    s_xor_b64 exec, exec, s[8:9]
+; CGP-NEXT:    s_cbranch_execz BB8_8
+; CGP-NEXT:  ; %bb.7:
+; CGP-NEXT:    v_cvt_f32_u32_e32 v3, v8
+; CGP-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; CGP-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v3
+; CGP-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; CGP-NEXT:    v_mul_lo_u32 v4, v3, v8
+; CGP-NEXT:    v_mul_hi_u32 v5, v3, v8
+; CGP-NEXT:    v_sub_i32_e32 v6, vcc, 0, v4
+; CGP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; CGP-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; CGP-NEXT:    v_mul_hi_u32 v4, v4, v3
+; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v3, v4
+; CGP-NEXT:    v_sub_i32_e64 v3, s[4:5], v3, v4
+; CGP-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; CGP-NEXT:    v_mul_hi_u32 v3, v3, v2
+; CGP-NEXT:    v_mul_lo_u32 v3, v3, v8
+; CGP-NEXT:    v_sub_i32_e32 v4, vcc, v2, v3
+; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v4, v8
+; CGP-NEXT:    v_add_i32_e64 v5, s[4:5], v4, v8
+; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v2, v3
+; CGP-NEXT:    v_sub_i32_e64 v2, s[6:7], v4, v8
+; CGP-NEXT:    s_and_b64 vcc, vcc, s[4:5]
+; CGP-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
+; CGP-NEXT:    v_cndmask_b32_e64 v4, v5, v2, s[4:5]
+; CGP-NEXT:    v_mov_b32_e32 v5, 0
+; CGP-NEXT:  BB8_8:
+; CGP-NEXT:    s_or_b64 exec, exec, s[8:9]
+; CGP-NEXT:    v_mov_b32_e32 v2, v4
+; CGP-NEXT:    v_mov_b32_e32 v3, v5
+; CGP-NEXT:    s_setpc_b64 s[30:31]
+  %shl.y = shl <2 x i64> <i64 4096, i64 4096>, %y
+  %r = urem <2 x i64> %x, %shl.y
+  ret <2 x i64> %r
+}
+
+define i64 @v_urem_i64_24bit(i64 %num, i64 %den) {
+; GISEL-LABEL: v_urem_i64_24bit:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    s_mov_b32 s4, 0xffffff
+; GISEL-NEXT:    v_and_b32_e32 v0, s4, v0
+; GISEL-NEXT:    v_and_b32_e32 v1, s4, v2
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v2, v1
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; GISEL-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v2
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GISEL-NEXT:    v_mul_lo_u32 v3, v2, v1
+; GISEL-NEXT:    v_mul_hi_u32 v4, v2, v1
+; GISEL-NEXT:    v_sub_i32_e32 v5, vcc, 0, v3
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GISEL-NEXT:    v_mul_hi_u32 v3, v3, v2
+; GISEL-NEXT:    v_add_i32_e64 v4, s[4:5], v2, v3
+; GISEL-NEXT:    v_sub_i32_e64 v2, s[4:5], v2, v3
+; GISEL-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GISEL-NEXT:    v_mul_hi_u32 v2, v2, v0
+; GISEL-NEXT:    v_mul_lo_u32 v2, v2, v1
+; GISEL-NEXT:    v_sub_i32_e32 v3, vcc, v0, v2
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v1
+; GISEL-NEXT:    v_add_i32_e64 v4, s[4:5], v3, v1
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v2
+; GISEL-NEXT:    v_sub_i32_e64 v0, s[6:7], v3, v1
+; GISEL-NEXT:    s_and_b64 vcc, vcc, s[4:5]
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[4:5]
+; GISEL-NEXT:    v_mov_b32_e32 v1, 0
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_urem_i64_24bit:
+; CGP:       ; %bb.0:
+; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT:    s_mov_b32 s4, 0xffffff
+; CGP-NEXT:    v_and_b32_e32 v0, s4, v0
+; CGP-NEXT:    v_and_b32_e32 v1, s4, v2
+; CGP-NEXT:    v_cvt_f32_u32_e32 v2, v0
+; CGP-NEXT:    v_cvt_f32_u32_e32 v3, v1
+; CGP-NEXT:    v_rcp_f32_e32 v4, v3
+; CGP-NEXT:    v_mul_f32_e32 v4, v2, v4
+; CGP-NEXT:    v_trunc_f32_e32 v4, v4
+; CGP-NEXT:    v_mad_f32 v2, -v4, v3, v2
+; CGP-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; CGP-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v2|, v3
+; CGP-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
+; CGP-NEXT:    v_mul_lo_u32 v1, v2, v1
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; CGP-NEXT:    v_and_b32_e32 v0, 0xffffff, v0
+; CGP-NEXT:    v_mov_b32_e32 v1, 0
+; CGP-NEXT:    s_setpc_b64 s[30:31]
+  %num.mask = and i64 %num, 16777215
+  %den.mask = and i64 %den, 16777215
+  %result = urem i64 %num.mask, %den.mask
+  ret i64 %result
+}
+
+define <2 x i64> @v_urem_v2i64_24bit(<2 x i64> %num, <2 x i64> %den) {
+; GISEL-LABEL: v_urem_v2i64_24bit:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    s_mov_b32 s6, 0xffffff
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v1, 0
+; GISEL-NEXT:    v_and_b32_e32 v3, s6, v4
+; GISEL-NEXT:    v_and_b32_e32 v4, s6, v6
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v5, v3
+; GISEL-NEXT:    v_sub_i32_e32 v6, vcc, 0, v3
+; GISEL-NEXT:    v_subb_u32_e64 v7, s[4:5], 0, 0, vcc
+; GISEL-NEXT:    v_cvt_f32_u32_e32 v8, v4
+; GISEL-NEXT:    v_sub_i32_e32 v9, vcc, 0, v4
+; GISEL-NEXT:    v_subb_u32_e64 v10, s[4:5], 0, 0, vcc
+; GISEL-NEXT:    v_mac_f32_e32 v5, 0x4f800000, v1
+; GISEL-NEXT:    v_mac_f32_e32 v8, 0x4f800000, v1
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v1, v5
+; GISEL-NEXT:    v_rcp_iflag_f32_e32 v5, v8
+; GISEL-NEXT:    v_mul_f32_e32 v1, 0x5f7ffffc, v1
+; GISEL-NEXT:    v_mul_f32_e32 v5, 0x5f7ffffc, v5
+; GISEL-NEXT:    v_mul_f32_e32 v8, 0x2f800000, v1
+; GISEL-NEXT:    v_mul_f32_e32 v11, 0x2f800000, v5
+; GISEL-NEXT:    v_trunc_f32_e32 v8, v8
+; GISEL-NEXT:    v_trunc_f32_e32 v11, v11
+; GISEL-NEXT:    v_mac_f32_e32 v1, 0xcf800000, v8
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v8, v8
+; GISEL-NEXT:    v_mac_f32_e32 v5, 0xcf800000, v11
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v11, v11
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GISEL-NEXT:    v_mul_lo_u32 v12, v6, v8
+; GISEL-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; GISEL-NEXT:    v_mul_lo_u32 v13, v9, v11
+; GISEL-NEXT:    v_mul_lo_u32 v14, v6, v1
+; GISEL-NEXT:    v_mul_lo_u32 v15, v7, v1
+; GISEL-NEXT:    v_mul_hi_u32 v16, v6, v1
+; GISEL-NEXT:    v_mul_lo_u32 v17, v9, v5
+; GISEL-NEXT:    v_mul_lo_u32 v18, v10, v5
+; GISEL-NEXT:    v_mul_hi_u32 v19, v9, v5
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v15, v12
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v18, v13
+; GISEL-NEXT:    v_mul_lo_u32 v15, v11, v17
+; GISEL-NEXT:    v_mul_hi_u32 v18, v5, v17
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v19
+; GISEL-NEXT:    v_mul_lo_u32 v19, v5, v13
+; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v15, v19
+; GISEL-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v15, v18
+; GISEL-NEXT:    v_mul_lo_u32 v15, v8, v14
+; GISEL-NEXT:    v_mul_hi_u32 v18, v1, v14
+; GISEL-NEXT:    v_mul_hi_u32 v14, v8, v14
+; GISEL-NEXT:    v_mul_hi_u32 v17, v11, v17
+; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v16
+; GISEL-NEXT:    v_mul_lo_u32 v16, v1, v12
+; GISEL-NEXT:    v_add_i32_e64 v15, s[4:5], v15, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v15, s[4:5], v15, v18
+; GISEL-NEXT:    v_mul_lo_u32 v15, v8, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v18, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v16, s[4:5], v16, v18
+; GISEL-NEXT:    v_mul_hi_u32 v18, v1, v12
+; GISEL-NEXT:    v_add_i32_e64 v14, s[4:5], v15, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v14, s[4:5], v14, v18
+; GISEL-NEXT:    v_cndmask_b32_e64 v18, 0, 1, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v15, s[4:5], v15, v18
+; GISEL-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v18, vcc, v19, v18
+; GISEL-NEXT:    v_mul_lo_u32 v19, v11, v13
+; GISEL-NEXT:    v_add_i32_e32 v17, vcc, v19, v17
+; GISEL-NEXT:    v_mul_hi_u32 v19, v5, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v17, vcc, v17, v19
+; GISEL-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v19, vcc, v20, v19
+; GISEL-NEXT:    v_and_b32_e32 v0, s6, v0
+; GISEL-NEXT:    v_and_b32_e32 v2, s6, v2
+; GISEL-NEXT:    v_mul_hi_u32 v12, v8, v12
+; GISEL-NEXT:    v_mul_hi_u32 v13, v11, v13
+; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v16
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v17, vcc, v17, v18
+; GISEL-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v15, v16
+; GISEL-NEXT:    v_add_i32_e32 v16, vcc, v19, v18
+; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v15
+; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v16
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v1, v14
+; GISEL-NEXT:    v_addc_u32_e64 v14, s[4:5], v8, v12, vcc
+; GISEL-NEXT:    v_add_i32_e64 v8, s[4:5], v8, v12
+; GISEL-NEXT:    v_mul_lo_u32 v12, v6, v1
+; GISEL-NEXT:    v_mul_lo_u32 v7, v7, v1
+; GISEL-NEXT:    v_mul_hi_u32 v15, v6, v1
+; GISEL-NEXT:    v_add_i32_e64 v5, s[4:5], v5, v17
+; GISEL-NEXT:    v_addc_u32_e64 v16, s[6:7], v11, v13, s[4:5]
+; GISEL-NEXT:    v_add_i32_e64 v11, s[6:7], v11, v13
+; GISEL-NEXT:    v_mul_lo_u32 v13, v9, v5
+; GISEL-NEXT:    v_mul_lo_u32 v10, v10, v5
+; GISEL-NEXT:    v_mul_hi_u32 v17, v9, v5
+; GISEL-NEXT:    v_mul_lo_u32 v6, v6, v14
+; GISEL-NEXT:    v_mul_lo_u32 v18, v14, v12
+; GISEL-NEXT:    v_mul_hi_u32 v19, v1, v12
+; GISEL-NEXT:    v_mul_hi_u32 v12, v14, v12
+; GISEL-NEXT:    v_mul_lo_u32 v9, v9, v16
+; GISEL-NEXT:    v_add_i32_e64 v6, s[6:7], v7, v6
+; GISEL-NEXT:    v_mul_lo_u32 v7, v16, v13
+; GISEL-NEXT:    v_add_i32_e64 v9, s[6:7], v10, v9
+; GISEL-NEXT:    v_mul_hi_u32 v10, v5, v13
+; GISEL-NEXT:    v_mul_hi_u32 v13, v16, v13
+; GISEL-NEXT:    v_add_i32_e64 v6, s[6:7], v6, v15
+; GISEL-NEXT:    v_add_i32_e64 v9, s[6:7], v9, v17
+; GISEL-NEXT:    v_mul_lo_u32 v15, v1, v6
+; GISEL-NEXT:    v_mul_lo_u32 v17, v5, v9
+; GISEL-NEXT:    v_add_i32_e64 v7, s[6:7], v7, v17
+; GISEL-NEXT:    v_cndmask_b32_e64 v17, 0, 1, s[6:7]
+; GISEL-NEXT:    v_add_i32_e64 v7, s[6:7], v7, v10
+; GISEL-NEXT:    v_mul_lo_u32 v7, v14, v6
+; GISEL-NEXT:    v_mul_hi_u32 v10, v1, v6
+; GISEL-NEXT:    v_mul_hi_u32 v6, v14, v6
+; GISEL-NEXT:    v_mul_lo_u32 v14, v16, v9
+; GISEL-NEXT:    v_mul_hi_u32 v16, v16, v9
+; GISEL-NEXT:    v_mul_hi_u32 v9, v5, v9
+; GISEL-NEXT:    v_add_i32_e64 v15, s[8:9], v18, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v18, 0, 1, s[8:9]
+; GISEL-NEXT:    v_add_i32_e64 v7, s[8:9], v7, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[8:9]
+; GISEL-NEXT:    v_add_i32_e64 v13, s[8:9], v14, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[8:9]
+; GISEL-NEXT:    v_add_i32_e64 v15, s[8:9], v15, v19
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, s[8:9]
+; GISEL-NEXT:    v_add_i32_e64 v7, s[8:9], v7, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s[8:9]
+; GISEL-NEXT:    v_cndmask_b32_e64 v19, 0, 1, s[6:7]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[6:7], v13, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[6:7]
+; GISEL-NEXT:    v_add_i32_e64 v15, s[6:7], v18, v15
+; GISEL-NEXT:    v_add_i32_e64 v10, s[6:7], v12, v10
+; GISEL-NEXT:    v_add_i32_e64 v12, s[6:7], v17, v19
+; GISEL-NEXT:    v_add_i32_e64 v13, s[6:7], v14, v13
+; GISEL-NEXT:    v_add_i32_e64 v7, s[6:7], v7, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[6:7]
+; GISEL-NEXT:    v_add_i32_e64 v9, s[6:7], v9, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s[6:7]
+; GISEL-NEXT:    v_add_i32_e64 v10, s[6:7], v10, v14
+; GISEL-NEXT:    v_add_i32_e64 v12, s[6:7], v13, v12
+; GISEL-NEXT:    v_add_i32_e64 v6, s[6:7], v6, v10
+; GISEL-NEXT:    v_add_i32_e64 v10, s[6:7], v16, v12
+; GISEL-NEXT:    v_addc_u32_e32 v6, vcc, v8, v6, vcc
+; GISEL-NEXT:    v_addc_u32_e64 v8, vcc, v11, v10, s[4:5]
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v1, v7
+; GISEL-NEXT:    v_addc_u32_e32 v6, vcc, 0, v6, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v7, 0, v1
+; GISEL-NEXT:    v_mul_hi_u32 v10, v0, v1
+; GISEL-NEXT:    v_mul_hi_u32 v1, 0, v1
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
+; GISEL-NEXT:    v_addc_u32_e32 v8, vcc, 0, v8, vcc
+; GISEL-NEXT:    v_mul_lo_u32 v9, 0, v5
+; GISEL-NEXT:    v_mul_hi_u32 v11, v2, v5
+; GISEL-NEXT:    v_mul_hi_u32 v5, 0, v5
+; GISEL-NEXT:    v_mul_lo_u32 v12, v0, v6
+; GISEL-NEXT:    v_mul_lo_u32 v13, 0, v6
+; GISEL-NEXT:    v_mul_hi_u32 v14, v0, v6
+; GISEL-NEXT:    v_mul_hi_u32 v6, 0, v6
+; GISEL-NEXT:    v_mul_lo_u32 v15, v2, v8
+; GISEL-NEXT:    v_mul_lo_u32 v16, 0, v8
+; GISEL-NEXT:    v_mul_hi_u32 v17, v2, v8
+; GISEL-NEXT:    v_mul_hi_u32 v8, 0, v8
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v12
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v13, v1
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v15
+; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v16, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v10
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v1, v14
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v17
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v12, v7
+; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v13, v10
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v15, v9
+; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v16, v11
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v1, v7
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v10, v7
+; GISEL-NEXT:    v_mul_lo_u32 v10, v3, v1
+; GISEL-NEXT:    v_mul_lo_u32 v12, 0, v1
+; GISEL-NEXT:    v_mul_hi_u32 v1, v3, v1
+; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
+; GISEL-NEXT:    v_mul_lo_u32 v11, v4, v5
+; GISEL-NEXT:    v_mul_lo_u32 v13, 0, v5
+; GISEL-NEXT:    v_mul_hi_u32 v5, v4, v5
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v8, v9
+; GISEL-NEXT:    v_mul_lo_u32 v6, v3, v6
+; GISEL-NEXT:    v_mul_lo_u32 v7, v4, v7
+; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v12, v6
+; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v13, v7
+; GISEL-NEXT:    v_add_i32_e32 v1, vcc, v6, v1
+; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v10
+; GISEL-NEXT:    v_subb_u32_e64 v6, s[4:5], 0, v1, vcc
+; GISEL-NEXT:    v_sub_i32_e64 v1, s[4:5], 0, v1
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v0, v3
+; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e64 v2, s[4:5], v2, v11
+; GISEL-NEXT:    v_subb_u32_e64 v8, s[6:7], 0, v5, s[4:5]
+; GISEL-NEXT:    v_sub_i32_e64 v5, s[6:7], 0, v5
+; GISEL-NEXT:    v_cmp_ge_u32_e64 s[6:7], v2, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[6:7]
+; GISEL-NEXT:    v_cmp_le_u32_e64 s[6:7], 0, v6
+; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[6:7]
+; GISEL-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, 0, v8
+; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, -1, vcc
+; GISEL-NEXT:    v_subbrev_u32_e64 v5, vcc, 0, v5, s[4:5]
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v6
+; GISEL-NEXT:    v_cndmask_b32_e32 v7, v10, v7, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v10, vcc, v0, v3
+; GISEL-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v10, v3
+; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v8
+; GISEL-NEXT:    v_cndmask_b32_e32 v9, v11, v9, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v11, vcc, v2, v4
+; GISEL-NEXT:    v_subbrev_u32_e32 v5, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v11, v4
+; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, -1, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, 0, v1
+; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, -1, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v3, vcc, v10, v3
+; GISEL-NEXT:    v_subbrev_u32_e32 v15, vcc, 0, v1, vcc
+; GISEL-NEXT:    v_cmp_le_u32_e32 vcc, 0, v5
+; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, -1, vcc
+; GISEL-NEXT:    v_sub_i32_e32 v4, vcc, v11, v4
+; GISEL-NEXT:    v_subbrev_u32_e32 v17, vcc, 0, v5, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GISEL-NEXT:    v_cndmask_b32_e32 v12, v14, v12, vcc
+; GISEL-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; GISEL-NEXT:    v_cndmask_b32_e32 v13, v16, v13, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v12
+; GISEL-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v13
+; GISEL-NEXT:    v_cndmask_b32_e64 v4, v11, v4, s[4:5]
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v15, vcc
+; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v3, v5, v17, s[4:5]
+; GISEL-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v9
+; GISEL-NEXT:    v_cndmask_b32_e64 v2, v2, v4, s[4:5]
+; GISEL-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v3, v8, v3, s[4:5]
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; CGP-LABEL: v_urem_v2i64_24bit:
+; CGP:       ; %bb.0:
+; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CGP-NEXT:    s_mov_b32 s6, 0xffffff
+; CGP-NEXT:    v_mov_b32_e32 v1, 0
+; CGP-NEXT:    v_and_b32_e32 v0, s6, v0
+; CGP-NEXT:    v_and_b32_e32 v2, s6, v2
+; CGP-NEXT:    v_and_b32_e32 v3, s6, v4
+; CGP-NEXT:    v_and_b32_e32 v4, s6, v6
+; CGP-NEXT:    v_cvt_f32_u32_e32 v5, v0
+; CGP-NEXT:    v_cvt_f32_u32_e32 v6, v3
+; CGP-NEXT:    v_cvt_f32_u32_e32 v7, v2
+; CGP-NEXT:    v_cvt_f32_u32_e32 v8, v4
+; CGP-NEXT:    v_rcp_f32_e32 v9, v6
+; CGP-NEXT:    v_rcp_f32_e32 v10, v8
+; CGP-NEXT:    v_mul_f32_e32 v9, v5, v9
+; CGP-NEXT:    v_mul_f32_e32 v10, v7, v10
+; CGP-NEXT:    v_trunc_f32_e32 v9, v9
+; CGP-NEXT:    v_trunc_f32_e32 v10, v10
+; CGP-NEXT:    v_mad_f32 v5, -v9, v6, v5
+; CGP-NEXT:    v_cvt_u32_f32_e32 v9, v9
+; CGP-NEXT:    v_mad_f32 v7, -v10, v8, v7
+; CGP-NEXT:    v_cvt_u32_f32_e32 v10, v10
+; CGP-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v5|, v6
+; CGP-NEXT:    v_cndmask_b32_e64 v5, 0, 1, s[4:5]
+; CGP-NEXT:    v_cmp_ge_f32_e64 s[4:5], |v7|, v8
+; CGP-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[4:5]
+; CGP-NEXT:    v_add_i32_e32 v5, vcc, v9, v5
+; CGP-NEXT:    v_add_i32_e32 v6, vcc, v10, v6
+; CGP-NEXT:    v_mul_lo_u32 v3, v5, v3
+; CGP-NEXT:    v_mul_lo_u32 v4, v6, v4
+; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v3
+; CGP-NEXT:    v_sub_i32_e32 v2, vcc, v2, v4
+; CGP-NEXT:    v_and_b32_e32 v0, s6, v0
+; CGP-NEXT:    v_and_b32_e32 v2, s6, v2
+; CGP-NEXT:    v_mov_b32_e32 v3, v1
+; CGP-NEXT:    s_setpc_b64 s[30:31]
+  %num.mask = and <2 x i64> %num, <i64 16777215, i64 16777215>
+  %den.mask = and <2 x i64> %den, <i64 16777215, i64 16777215>
+  %result = urem <2 x i64> %num.mask, %den.mask
+  ret <2 x i64> %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn-ieee.ll b/llvm/test/CodeGen/AMDGPU/amdgcn-ieee.ll
index 1e6cc0523dc29..2ad2ee3502d8a 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn-ieee.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn-ieee.ll
@@ -3,8 +3,8 @@
 ; GCN-LABEL: {{^}}kernel_ieee_mode_default:
 ; GCN: {{buffer|global|flat}}_load_dword [[VAL0:v[0-9]+]]
 ; GCN-NEXT: {{buffer|global|flat}}_load_dword [[VAL1:v[0-9]+]]
-; GCN-DAG: v_mul_f32_e32 [[QUIET0:v[0-9]+]], 1.0, [[VAL0]]
-; GCN-DAG: v_mul_f32_e32 [[QUIET1:v[0-9]+]], 1.0, [[VAL1]]
+; GCN-DAG: v_max_f32_e32 [[QUIET0:v[0-9]+]], [[VAL0]], [[VAL0]]
+; GCN-DAG: v_max_f32_e32 [[QUIET1:v[0-9]+]], [[VAL1]], [[VAL1]]
 ; GCN: v_min_f32_e32 [[MIN:v[0-9]+]], [[QUIET0]], [[QUIET1]]
 ; GCN-NOT: v_mul_f32
 define amdgpu_kernel void @kernel_ieee_mode_default() #0 {
@@ -18,8 +18,8 @@ define amdgpu_kernel void @kernel_ieee_mode_default() #0 {
 ; GCN-LABEL: {{^}}kernel_ieee_mode_on:
 ; GCN: {{buffer|global|flat}}_load_dword [[VAL0:v[0-9]+]]
 ; GCN-NEXT: {{buffer|global|flat}}_load_dword [[VAL1:v[0-9]+]]
-; GCN-DAG: v_mul_f32_e32 [[QUIET0:v[0-9]+]], 1.0, [[VAL0]]
-; GCN-DAG: v_mul_f32_e32 [[QUIET1:v[0-9]+]], 1.0, [[VAL1]]
+; GCN-DAG: v_max_f32_e32 [[QUIET0:v[0-9]+]], [[VAL0]], [[VAL0]]
+; GCN-DAG: v_max_f32_e32 [[QUIET1:v[0-9]+]], [[VAL1]], [[VAL1]]
 ; GCN: v_min_f32_e32 [[MIN:v[0-9]+]], [[QUIET0]], [[QUIET1]]
 ; GCN-NOT: v_mul_f32
 define amdgpu_kernel void @kernel_ieee_mode_on() #1 {
@@ -48,8 +48,8 @@ define amdgpu_kernel void @kernel_ieee_mode_off() #2 {
 ; GCN-LABEL: {{^}}func_ieee_mode_default:
 ; GCN: {{buffer|global|flat}}_load_dword [[VAL0:v[0-9]+]]
 ; GCN-NEXT: {{buffer|global|flat}}_load_dword [[VAL1:v[0-9]+]]
-; GCN-DAG: v_mul_f32_e32 [[QUIET0:v[0-9]+]], 1.0, [[VAL0]]
-; GCN-DAG: v_mul_f32_e32 [[QUIET1:v[0-9]+]], 1.0, [[VAL1]]
+; GCN-DAG: v_max_f32_e32 [[QUIET0:v[0-9]+]], [[VAL0]], [[VAL0]]
+; GCN-DAG: v_max_f32_e32 [[QUIET1:v[0-9]+]], [[VAL1]], [[VAL1]]
 ; GCN: v_min_f32_e32 [[MIN:v[0-9]+]], [[QUIET0]], [[QUIET1]]
 ; GCN-NOT: v_mul_f32
 define void @func_ieee_mode_default() #0 {
@@ -63,8 +63,8 @@ define void @func_ieee_mode_default() #0 {
 ; GCN-LABEL: {{^}}func_ieee_mode_on:
 ; GCN: {{buffer|global|flat}}_load_dword [[VAL0:v[0-9]+]]
 ; GCN-NEXT: {{buffer|global|flat}}_load_dword [[VAL1:v[0-9]+]]
-; GCN-DAG: v_mul_f32_e32 [[QUIET0:v[0-9]+]], 1.0, [[VAL0]]
-; GCN-DAG: v_mul_f32_e32 [[QUIET1:v[0-9]+]], 1.0, [[VAL1]]
+; GCN-DAG: v_max_f32_e32 [[QUIET0:v[0-9]+]], [[VAL0]], [[VAL0]]
+; GCN-DAG: v_max_f32_e32 [[QUIET1:v[0-9]+]], [[VAL1]], [[VAL1]]
 ; GCN: v_min_f32_e32 [[MIN:v[0-9]+]], [[QUIET0]], [[QUIET1]]
 ; GCN-NOT: v_mul_f32
 define void @func_ieee_mode_on() #1 {
@@ -93,8 +93,8 @@ define void @func_ieee_mode_off() #2 {
 ; GCN-LABEL: {{^}}cs_ieee_mode_default:
 ; GCN: {{buffer|global|flat}}_load_dword [[VAL0:v[0-9]+]]
 ; GCN-NEXT: {{buffer|global|flat}}_load_dword [[VAL1:v[0-9]+]]
-; GCN-DAG: v_mul_f32_e32 [[QUIET0:v[0-9]+]], 1.0, [[VAL0]]
-; GCN-DAG: v_mul_f32_e32 [[QUIET1:v[0-9]+]], 1.0, [[VAL1]]
+; GCN-DAG: v_max_f32_e32 [[QUIET0:v[0-9]+]], [[VAL0]], [[VAL0]]
+; GCN-DAG: v_max_f32_e32 [[QUIET1:v[0-9]+]], [[VAL1]], [[VAL1]]
 ; GCN: v_min_f32_e32 [[MIN:v[0-9]+]], [[QUIET0]], [[QUIET1]]
 ; GCN-NOT: v_mul_f32
 define amdgpu_cs void @cs_ieee_mode_default() #0 {
@@ -108,8 +108,8 @@ define amdgpu_cs void @cs_ieee_mode_default() #0 {
 ; GCN-LABEL: {{^}}cs_ieee_mode_on:
 ; GCN: {{buffer|global|flat}}_load_dword [[VAL0:v[0-9]+]]
 ; GCN-NEXT: {{buffer|global|flat}}_load_dword [[VAL1:v[0-9]+]]
-; GCN-DAG: v_mul_f32_e32 [[QUIET0:v[0-9]+]], 1.0, [[VAL0]]
-; GCN-DAG: v_mul_f32_e32 [[QUIET1:v[0-9]+]], 1.0, [[VAL1]]
+; GCN-DAG: v_max_f32_e32 [[QUIET0:v[0-9]+]], [[VAL0]], [[VAL0]]
+; GCN-DAG: v_max_f32_e32 [[QUIET1:v[0-9]+]], [[VAL1]], [[VAL1]]
 ; GCN: v_min_f32_e32 [[MIN:v[0-9]+]], [[QUIET0]], [[QUIET1]]
 ; GCN-NOT: v_mul_f32
 define amdgpu_cs void @cs_ieee_mode_on() #1 {
@@ -153,8 +153,8 @@ define amdgpu_ps void @ps_ieee_mode_default() #0 {
 ; GCN-LABEL: {{^}}ps_ieee_mode_on:
 ; GCN: {{buffer|global|flat}}_load_dword [[VAL0:v[0-9]+]]
 ; GCN-NEXT: {{buffer|global|flat}}_load_dword [[VAL1:v[0-9]+]]
-; GCN-DAG: v_mul_f32_e32 [[QUIET0:v[0-9]+]], 1.0, [[VAL0]]
-; GCN-DAG: v_mul_f32_e32 [[QUIET1:v[0-9]+]], 1.0, [[VAL1]]
+; GCN-DAG: v_max_f32_e32 [[QUIET0:v[0-9]+]], [[VAL0]], [[VAL0]]
+; GCN-DAG: v_max_f32_e32 [[QUIET1:v[0-9]+]], [[VAL1]], [[VAL1]]
 ; GCN: v_min_f32_e32 [[MIN:v[0-9]+]], [[QUIET0]], [[QUIET1]]
 ; GCN-NOT: v_mul_f32
 define amdgpu_ps void @ps_ieee_mode_on() #1 {
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn-load-offset-from-reg.ll b/llvm/test/CodeGen/AMDGPU/amdgcn-load-offset-from-reg.ll
new file mode 100644
index 0000000000000..fde3ab8c6d4a4
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn-load-offset-from-reg.ll
@@ -0,0 +1,62 @@
+; Test that DAG->DAG ISel is able to pick up the S_LOAD_DWORDX4_SGPR instruction that fetches the offset
+; from a register.
+
+; RUN: llc -march=amdgcn -verify-machineinstrs -stop-after=amdgpu-isel -o - %s | FileCheck -check-prefix=GCN %s
+
+; GCN: %[[OFFSET:[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @DescriptorBuffer
+; GCN: %{{[0-9]+}}:sgpr_128 = S_LOAD_DWORDX4_SGPR killed %{{[0-9]+}}, killed %[[OFFSET]], 0, 0 :: (invariant load 16 from %ir.13, addrspace 4)
+
+define amdgpu_cs void @test_load_zext(i32 inreg %0, i32 inreg %1, i32 inreg %resNode0, i32 inreg %resNode1, <3 x i32> inreg %2, i32 inreg %3, <3 x i32> %4) local_unnamed_addr #2 {
+.entry:
+  %5 = call i64 @llvm.amdgcn.s.getpc() #3
+  %6 = bitcast i64 %5 to <2 x i32>
+  %7 = insertelement <2 x i32> %6, i32 %resNode0, i32 0
+  %8 = bitcast <2 x i32> %7 to i64
+  %9 = inttoptr i64 %8 to [4294967295 x i8] addrspace(4)*
+  %10 = call i32 @llvm.amdgcn.reloc.constant(metadata !4)
+  %11 = zext i32 %10 to i64
+  %12 = getelementptr [4294967295 x i8], [4294967295 x i8] addrspace(4)* %9, i64 0, i64 %11
+  %13 = bitcast i8 addrspace(4)* %12 to <4 x i32> addrspace(4)*, !amdgpu.uniform !5
+  %14 = load <4 x i32>, <4 x i32> addrspace(4)* %13, align 16, !invariant.load !5
+  %15 = call <4 x i32> @llvm.amdgcn.s.buffer.load.v4i32(<4 x i32> %14, i32 0, i32 0)
+  call void @llvm.amdgcn.raw.buffer.store.v4i32(<4 x i32> %15, <4 x i32> %14, i32 0, i32 0, i32 0)
+  ret void
+}
+
+declare <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32>, i32, i32, i1, i1) #0
+; Function Attrs: nounwind writeonly
+declare void @llvm.amdgcn.raw.buffer.store.v4i32(<4 x i32>, <4 x i32>, i32, i32, i32 immarg) #1
+
+; Function Attrs: nounwind readnone speculatable
+declare i32 @llvm.amdgcn.reloc.constant(metadata) #3
+
+; Function Attrs: nounwind readnone speculatable
+declare i64 @llvm.amdgcn.s.getpc() #3
+
+; Function Attrs: nounwind readnone
+declare <4 x i32> @llvm.amdgcn.s.buffer.load.v4i32(<4 x i32>, i32, i32 immarg) #1
+
+attributes #0 = { argmemonly nounwind willreturn }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind "amdgpu-unroll-threshold"="700" }
+attributes #3 = { nounwind readnone speculatable }
+attributes #4 = { nounwind writeonly }
+
+!llpc.compute.mode = !{!0}
+!llpc.options = !{!1}
+!llpc.options.CS = !{!2}
+!llpc.user.data.nodes = !{!3, !4, !5, !6}
+!amdgpu.pal.metadata.msgpack = !{!7}
+
+!0 = !{i32 2, i32 3, i32 1}
+!1 = !{i32 245227952, i32 996822128, i32 2024708198, i32 497230408}
+!2 = !{i32 1381820427, i32 1742110173, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 64}
+!3 = !{!"DescriptorTableVaPtr", i32 0, i32 1, i32 1}
+!4 = !{!"DescriptorBuffer", i32 4, i32 8, i32 0, i32 0}
+!5 = !{!"DescriptorTableVaPtr", i32 1, i32 1, i32 1}
+!6 = !{!"DescriptorBuffer", i32 4, i32 8, i32 1, i32 0}
+!7 = !{!"\82\B0amdpal.pipelines\91\88\A4.api\A6Vulkan\B0.hardware_stages\81\A3.cs\82\AB.sgpr_limith\AB.vgpr_limit\CD\01\00\B7.internal_pipeline_hash\92\CF;jLp\0E\9D\E1\B0\CF\1D\A3\22Hx\AE\98f\AA.registers\88\CD.\07\02\CD.\08\03\CD.\09\01\CD.\12\CE\00,\00\00\CD.\13\CD\0F\88\CD.@\CE\10\00\00\00\CD.B\00\CD.C\01\A8.shaders\81\A8.compute\82\B0.api_shader_hash\92\CFg\D6}\DDR\\\E8\0B\00\B1.hardware_mapping\91\A3.cs\B0.spill_threshold\CE\FF\FF\FF\FF\A5.type\A2Cs\B0.user_data_limit\02\AEamdpal.version\92\02\03"}
+!8 = !{i32 5}
+!9 = !{!"doff_0_0_b"}
+!10 = !{}
+!11 = !{!"doff_1_0_b"}
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fdiv.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fdiv.ll
index 3096372394bb7..74228ce7f3444 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fdiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fdiv.ll
@@ -338,8 +338,8 @@ define amdgpu_kernel void @fdiv_fpmath_f32_denormals(float addrspace(1)* %out, f
 }
 
 attributes #0 = { nounwind optnone noinline }
-attributes #1 = { nounwind }
-attributes #2 = { nounwind "target-features"="+fp32-denormals" }
+attributes #1 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
+attributes #2 = { nounwind "denormal-fp-math-f32"="ieee,ieee" }
 
 !0 = !{float 2.500000e+00}
 !1 = !{float 5.000000e-01}
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-reloc-const.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-reloc-const.ll
new file mode 100644
index 0000000000000..569f0101e3bee
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-reloc-const.ll
@@ -0,0 +1,30 @@
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx900 -filetype=obj -o %t.o < %s && llvm-readobj -relocations %t.o | FileCheck --check-prefix=ELF %s
+; GCN-LABEL: {{^}}ps_main:
+; GCN: v_mov_b32_{{.*}} v[[relocreg:[0-9]+]], doff_0_0_b@abs32@lo
+; GCN-NEXT: exp {{.*}} v[[relocreg]], {{.*}}
+; GCN-NEXT: s_endpgm
+; GCN-NEXT: .Lfunc_end
+
+; ELF: Relocations [
+; ELF-NEXT: Section (3) .rel.text {
+; ELF-NEXT: 0x{{[0-9]+}} R_AMDGPU_ABS32 doff_0_0_b {{.*}}
+
+define amdgpu_ps void @ps_main(i32 %arg, i32 inreg %arg1, i32 inreg %arg2) local_unnamed_addr #0 {
+  %rc = call i32 @llvm.amdgcn.reloc.constant(metadata !1)
+  %rcf = bitcast i32 %rc to float
+  call void @llvm.amdgcn.exp.f32(i32 immarg 40, i32 immarg 15, float %rcf, float undef, float undef, float undef, i1 immarg false, i1 immarg false) #0
+  ret void
+}
+
+; Function Attrs: inaccessiblememonly nounwind
+declare void @llvm.amdgcn.exp.f32(i32 immarg, i32 immarg, float, float, float, float, i1 immarg, i1 immarg) #1
+
+; Function Attrs: nounwind readnone speculatable
+declare i32 @llvm.amdgcn.reloc.constant(metadata) #2
+
+attributes #0 = { nounwind }
+attributes #1 = { inaccessiblememonly nounwind }
+attributes #2 = { nounwind readnone speculatable }
+
+!1 = !{!"doff_0_0_b"}
diff --git a/llvm/test/CodeGen/AMDGPU/at-least-one-def-value-assert.mir b/llvm/test/CodeGen/AMDGPU/at-least-one-def-value-assert.mir
index 4503ed12cb6fb..82ee173e12256 100644
--- a/llvm/test/CodeGen/AMDGPU/at-least-one-def-value-assert.mir
+++ b/llvm/test/CodeGen/AMDGPU/at-least-one-def-value-assert.mir
@@ -5,7 +5,7 @@
 # CHECK-NEXT: - basic block: %bb.0
 # CHECK-NEXT: - instruction: 48B	dead undef %2.sub0:vreg_128 = COPY %0.sub0:vreg_128
 # CHECK-NEXT: - operand 1:   %0.sub0:vreg_128
-# CHECK-NEXT: - interval:    %0 [16r,48r:0)  0@16r L0000000000000002 [16r,32r:0)  0@16r weight:0.000000e+00
+# CHECK-NEXT: - interval:    %0 [16r,48r:0)  0@16r L000000000000000C [16r,32r:0)  0@16r weight:0.000000e+00
 
 # This used to assert with: !SR.empty() && "At least one value should be defined by this mask"
 
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
index 72c121187691f..cd88dcd7f79ad 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
@@ -9,14 +9,15 @@ declare i32 @llvm.amdgcn.raw.buffer.atomic.add(i32, <4 x i32>, i32, i32, i32 imm
 declare i32 @llvm.amdgcn.struct.buffer.atomic.add(i32, <4 x i32>, i32, i32, i32, i32 immarg)
 declare i32 @llvm.amdgcn.raw.buffer.atomic.sub(i32, <4 x i32>, i32, i32, i32 immarg)
 
-; Show that what the atomic optimization pass will do for raw buffers.
+; Show what the atomic optimization pass will do for raw buffers.
 
 ; GCN-LABEL: add_i32_constant:
-; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
-; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo
+; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
 ; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc
 ; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
 ; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: v_mul_u32_u24{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[popcount]], 5
@@ -29,11 +30,12 @@ entry:
 }
 
 ; GCN-LABEL: add_i32_uniform:
-; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
-; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo
+; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
 ; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc
 ; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
 ; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: s_mul_i32 s[[scalar_value:[0-9]+]], s{{[0-9]+}}, s[[popcount]]
@@ -108,11 +110,12 @@ entry:
 }
 
 ; GCN-LABEL: sub_i32_constant:
-; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
-; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo
+; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
 ; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc
 ; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
 ; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: v_mul_u32_u24{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[popcount]], 5
@@ -125,11 +128,12 @@ entry:
 }
 
 ; GCN-LABEL: sub_i32_uniform:
-; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
-; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo
+; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
 ; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc
 ; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
 ; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: s_mul_i32 s[[scalar_value:[0-9]+]], s{{[0-9]+}}, s[[popcount]]
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index 24fe357979d05..e7893c7bd6933 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -6,14 +6,15 @@
 
 declare i32 @llvm.amdgcn.workitem.id.x()
 
-; Show that what the atomic optimization pass will do for global pointers.
+; Show what the atomic optimization pass will do for global pointers.
 
 ; GCN-LABEL: add_i32_constant:
-; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
-; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo
+; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
 ; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc
 ; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
 ; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: v_mul_u32_u24{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[popcount]], 5
@@ -26,11 +27,12 @@ entry:
 }
 
 ; GCN-LABEL: add_i32_uniform:
-; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
-; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo
+; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
 ; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc
 ; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
 ; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: s_mul_i32 s[[scalar_value:[0-9]+]], s{{[0-9]+}}, s[[popcount]]
@@ -63,11 +65,12 @@ entry:
 }
 
 ; GCN-LABEL: add_i64_constant:
-; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
-; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo
+; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
 ; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc
 ; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
 ; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: v_mul_hi_u32_u24{{(_e[0-9]+)?}} v[[value_hi:[0-9]+]], s[[popcount]], 5
@@ -81,13 +84,14 @@ entry:
 }
 
 ; GCN-LABEL: add_i64_uniform:
-; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
-; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo
+; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
 ; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
-; GCN32: s_bcnt1_i32_b32 s{{[0-9]+}}, s[[exec_lo]]
-; GCN64: s_bcnt1_i32_b64 s{{[0-9]+}}, s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
+; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc
+; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
+; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: {{flat|buffer|global}}_atomic_add_x2 v{{\[}}{{[0-9]+}}:{{[0-9]+}}{{\]}}
 define amdgpu_kernel void @add_i64_uniform(i64 addrspace(1)* %out, i64 addrspace(1)* %inout, i64 %additive) {
 entry:
@@ -111,11 +115,12 @@ entry:
 }
 
 ; GCN-LABEL: sub_i32_constant:
-; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
-; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo
+; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
 ; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc
 ; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
 ; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: v_mul_u32_u24{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[popcount]], 5
@@ -128,11 +133,12 @@ entry:
 }
 
 ; GCN-LABEL: sub_i32_uniform:
-; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
-; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo
+; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
 ; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc
 ; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
 ; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: s_mul_i32 s[[scalar_value:[0-9]+]], s{{[0-9]+}}, s[[popcount]]
@@ -165,11 +171,12 @@ entry:
 }
 
 ; GCN-LABEL: sub_i64_constant:
-; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
-; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo
+; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
 ; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc
 ; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
 ; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: v_mul_hi_u32_u24{{(_e[0-9]+)?}} v[[value_hi:[0-9]+]], s[[popcount]], 5
@@ -183,13 +190,14 @@ entry:
 }
 
 ; GCN-LABEL: sub_i64_uniform:
-; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
-; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo
+; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
 ; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
-; GCN32: s_bcnt1_i32_b32 s{{[0-9]+}}, s[[exec_lo]]
-; GCN64: s_bcnt1_i32_b64 s{{[0-9]+}}, s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
+; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc
+; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
+; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: {{flat|buffer|global}}_atomic_sub_x2 v{{\[}}{{[0-9]+}}:{{[0-9]+}}{{\]}}
 define amdgpu_kernel void @sub_i64_uniform(i64 addrspace(1)* %out, i64 addrspace(1)* %inout, i64 %subitive) {
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
index 3695586633dfb..eb1618f346e4b 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
@@ -10,32 +10,32 @@ declare i32 @llvm.amdgcn.workitem.id.x()
 @local_var32 = addrspace(3) global i32 undef, align 4
 @local_var64 = addrspace(3) global i64 undef, align 8
 
-; Show that what the atomic optimization pass will do for local pointers.
+; Show what the atomic optimization pass will do for local pointers.
 
 define amdgpu_kernel void @add_i32_constant(i32 addrspace(1)* %out) {
 ;
 ;
 ; GFX7LESS-LABEL: add_i32_constant:
 ; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], exec
 ; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
-; GFX7LESS-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, 0
-; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s4, 0
-; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s5, v0
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-NEXT:    ; implicit-def: $vgpr1
-; GFX7LESS-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX7LESS-NEXT:    s_cbranch_execz BB0_2
 ; GFX7LESS-NEXT:  ; %bb.1:
-; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
+; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
-; GFX7LESS-NEXT:    v_mul_u32_u24_e64 v2, s4, 5
+; GFX7LESS-NEXT:    v_mul_u32_u24_e64 v2, s2, 5
 ; GFX7LESS-NEXT:    s_mov_b32 m0, -1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7LESS-NEXT:    ds_add_rtn_u32 v1, v1, v2
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7LESS-NEXT:    buffer_wbinvl1
 ; GFX7LESS-NEXT:  BB0_2:
-; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7LESS-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
@@ -47,16 +47,16 @@ define amdgpu_kernel void @add_i32_constant(i32 addrspace(1)* %out) {
 ; GFX8-LABEL: add_i32_constant:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX8-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, 0
-; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
-; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
+; GFX8-NEXT:    s_mov_b64 s[2:3], exec
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX8-NEXT:    ; implicit-def: $vgpr1
-; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX8-NEXT:    s_cbranch_execz BB0_2
 ; GFX8-NEXT:  ; %bb.1:
-; GFX8-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
-; GFX8-NEXT:    v_mul_u32_u24_e64 v1, s4, 5
+; GFX8-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX8-NEXT:    v_mul_u32_u24_e64 v1, s2, 5
 ; GFX8-NEXT:    v_mov_b32_e32 v2, local_var32@abs32@lo
 ; GFX8-NEXT:    s_mov_b32 m0, -1
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -64,7 +64,7 @@ define amdgpu_kernel void @add_i32_constant(i32 addrspace(1)* %out) {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
 ; GFX8-NEXT:  BB0_2:
-; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX8-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX8-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
 ; GFX8-NEXT:    s_mov_b32 s3, 0xf000
@@ -77,23 +77,23 @@ define amdgpu_kernel void @add_i32_constant(i32 addrspace(1)* %out) {
 ; GFX9-LABEL: add_i32_constant:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, 0
-; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
-; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
+; GFX9-NEXT:    s_mov_b64 s[2:3], exec
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    ; implicit-def: $vgpr1
-; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_cbranch_execz BB0_2
 ; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
-; GFX9-NEXT:    v_mul_u32_u24_e64 v1, s4, 5
+; GFX9-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX9-NEXT:    v_mul_u32_u24_e64 v1, s2, 5
 ; GFX9-NEXT:    v_mov_b32_e32 v2, local_var32@abs32@lo
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    ds_add_rtn_u32 v1, v2, v1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:  BB0_2:
-; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX9-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
 ; GFX9-NEXT:    s_mov_b32 s3, 0xf000
@@ -105,18 +105,18 @@ define amdgpu_kernel void @add_i32_constant(i32 addrspace(1)* %out) {
 ;
 ; GFX1064-LABEL: add_i32_constant:
 ; GFX1064:       ; %bb.0: ; %entry
-; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, 0
+; GFX1064-NEXT:    s_mov_b64 s[2:3], exec
 ; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX1064-NEXT:    ; implicit-def: $vgpr1
-; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s4, 0
-; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s5, v0
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX1064-NEXT:    s_cbranch_execz BB0_2
 ; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
+; GFX1064-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
 ; GFX1064-NEXT:    v_mov_b32_e32 v2, local_var32@abs32@lo
-; GFX1064-NEXT:    v_mul_u32_u24_e64 v1, s4, 5
+; GFX1064-NEXT:    v_mul_u32_u24_e64 v1, s2, 5
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1064-NEXT:    ds_add_rtn_u32 v1, v2, v1
@@ -125,7 +125,7 @@ define amdgpu_kernel void @add_i32_constant(i32 addrspace(1)* %out) {
 ; GFX1064-NEXT:    buffer_gl1_inv
 ; GFX1064-NEXT:  BB0_2:
 ; GFX1064-NEXT:    v_nop
-; GFX1064-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX1064-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1064-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
@@ -138,17 +138,17 @@ define amdgpu_kernel void @add_i32_constant(i32 addrspace(1)* %out) {
 ; GFX1032-LABEL: add_i32_constant:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX1032-NEXT:    v_cmp_ne_u32_e64 s3, 1, 0
+; GFX1032-NEXT:    s_mov_b32 s2, exec_lo
 ; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    ; implicit-def: $vgpr1
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s3, 0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT:    s_and_saveexec_b32 s2, vcc_lo
+; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz BB0_2
 ; GFX1032-NEXT:  ; %bb.1:
-; GFX1032-NEXT:    s_bcnt1_i32_b32 s3, s3
+; GFX1032-NEXT:    s_bcnt1_i32_b32 s2, s2
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, local_var32@abs32@lo
-; GFX1032-NEXT:    v_mul_u32_u24_e64 v1, s3, 5
+; GFX1032-NEXT:    v_mul_u32_u24_e64 v1, s2, 5
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1032-NEXT:    ds_add_rtn_u32 v1, v2, v1
@@ -157,7 +157,7 @@ define amdgpu_kernel void @add_i32_constant(i32 addrspace(1)* %out) {
 ; GFX1032-NEXT:    buffer_gl1_inv
 ; GFX1032-NEXT:  BB0_2:
 ; GFX1032-NEXT:    v_nop
-; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s3
 ; GFX1032-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1032-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
@@ -177,9 +177,9 @@ define amdgpu_kernel void @add_i32_uniform(i32 addrspace(1)* %out, i32 %additive
 ;
 ; GFX7LESS-LABEL: add_i32_uniform:
 ; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_mov_b64 s[6:7], exec
 ; GFX7LESS-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
 ; GFX7LESS-NEXT:    s_load_dword s2, s[0:1], 0xb
-; GFX7LESS-NEXT:    v_cmp_ne_u32_e64 s[6:7], 1, 0
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
 ; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s7, v0
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
@@ -212,15 +212,15 @@ define amdgpu_kernel void @add_i32_uniform(i32 addrspace(1)* %out, i32 %additive
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
 ; GFX8-NEXT:    s_load_dword s0, s[0:1], 0x2c
-; GFX8-NEXT:    v_cmp_ne_u32_e64 s[6:7], 1, 0
-; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX8-NEXT:    s_mov_b64 s[2:3], exec
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX8-NEXT:    ; implicit-def: $vgpr1
-; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX8-NEXT:    s_and_saveexec_b64 s[6:7], vcc
 ; GFX8-NEXT:    s_cbranch_execz BB1_2
 ; GFX8-NEXT:  ; %bb.1:
-; GFX8-NEXT:    s_bcnt1_i32_b64 s1, s[6:7]
+; GFX8-NEXT:    s_bcnt1_i32_b64 s1, s[2:3]
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_mul_i32 s1, s0, s1
 ; GFX8-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
@@ -231,7 +231,7 @@ define amdgpu_kernel void @add_i32_uniform(i32 addrspace(1)* %out, i32 %additive
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
 ; GFX8-NEXT:  BB1_2:
-; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mul_lo_u32 v0, s0, v0
 ; GFX8-NEXT:    v_readfirstlane_b32 s0, v1
@@ -245,15 +245,15 @@ define amdgpu_kernel void @add_i32_uniform(i32 addrspace(1)* %out, i32 %additive
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
 ; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x2c
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[6:7], 1, 0
-; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX9-NEXT:    s_mov_b64 s[2:3], exec
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    ; implicit-def: $vgpr1
-; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT:    s_and_saveexec_b64 s[6:7], vcc
 ; GFX9-NEXT:    s_cbranch_execz BB1_2
 ; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    s_bcnt1_i32_b64 s1, s[6:7]
+; GFX9-NEXT:    s_bcnt1_i32_b64 s1, s[2:3]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_mul_i32 s1, s0, s1
 ; GFX9-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
@@ -263,7 +263,7 @@ define amdgpu_kernel void @add_i32_uniform(i32 addrspace(1)* %out, i32 %additive
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:  BB1_2:
-; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX9-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mul_lo_u32 v0, s0, v0
 ; GFX9-NEXT:    v_readfirstlane_b32 s0, v1
@@ -275,17 +275,17 @@ define amdgpu_kernel void @add_i32_uniform(i32 addrspace(1)* %out, i32 %additive
 ;
 ; GFX1064-LABEL: add_i32_uniform:
 ; GFX1064:       ; %bb.0: ; %entry
-; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[6:7], 1, 0
+; GFX1064-NEXT:    s_mov_b64 s[2:3], exec
 ; GFX1064-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
 ; GFX1064-NEXT:    s_load_dword s0, s[0:1], 0x2c
 ; GFX1064-NEXT:    ; implicit-def: $vgpr1
-; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s7, v0
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-NEXT:    s_and_saveexec_b64 s[6:7], vcc
 ; GFX1064-NEXT:    s_cbranch_execz BB1_2
 ; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    s_bcnt1_i32_b64 s1, s[6:7]
+; GFX1064-NEXT:    s_bcnt1_i32_b64 s1, s[2:3]
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_mul_i32 s1, s0, s1
@@ -298,7 +298,7 @@ define amdgpu_kernel void @add_i32_uniform(i32 addrspace(1)* %out, i32 %additive
 ; GFX1064-NEXT:    buffer_gl1_inv
 ; GFX1064-NEXT:  BB1_2:
 ; GFX1064-NEXT:    v_nop
-; GFX1064-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    v_mul_lo_u32 v0, s0, v0
 ; GFX1064-NEXT:    v_readfirstlane_b32 s0, v1
@@ -312,7 +312,7 @@ define amdgpu_kernel void @add_i32_uniform(i32 addrspace(1)* %out, i32 %additive
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
 ; GFX1032-NEXT:    s_load_dword s0, s[0:1], 0x2c
-; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    s_mov_b32 s2, exec_lo
 ; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    ; implicit-def: $vgpr1
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
@@ -348,14 +348,6 @@ entry:
   ret void
 }
 
-; GFX7LESS-NOT: v_mbcnt_lo_u32_b32
-; GFX7LESS-NOT: v_mbcnt_hi_u32_b32
-; GFX7LESS-NOT: s_bcnt1_i32_b64
-; DPPCOMB: v_add_u32_dpp
-; DPPCOMB: v_add_u32_dpp
-; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31
-; GFX8MORE: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
-; GFX8MORE: ds_add_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v[[value]]
 define amdgpu_kernel void @add_i32_varying(i32 addrspace(1)* %out) {
 ;
 ;
@@ -376,11 +368,11 @@ define amdgpu_kernel void @add_i32_varying(i32 addrspace(1)* %out) {
 ; GFX8-LABEL: add_i32_varying:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    s_mov_b64 s[2:3], exec
 ; GFX8-NEXT:    v_mov_b32_e32 v2, v0
-; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX8-NEXT:    s_or_saveexec_b64 s[4:5], -1
 ; GFX8-NEXT:    v_mov_b32_e32 v1, 0
-; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX8-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
 ; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX8-NEXT:    s_not_b64 exec, exec
@@ -429,11 +421,11 @@ define amdgpu_kernel void @add_i32_varying(i32 addrspace(1)* %out) {
 ; GFX9-LABEL: add_i32_varying:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_mov_b64 s[2:3], exec
 ; GFX9-NEXT:    v_mov_b32_e32 v2, v0
-; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], -1
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
 ; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX9-NEXT:    s_not_b64 exec, exec
@@ -481,11 +473,11 @@ define amdgpu_kernel void @add_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1064-LABEL: add_i32_varying:
 ; GFX1064:       ; %bb.0: ; %entry
 ; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1064-NEXT:    s_mov_b64 s[2:3], exec
 ; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
-; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
 ; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
 ; GFX1064-NEXT:    s_not_b64 exec, exec
@@ -540,12 +532,12 @@ define amdgpu_kernel void @add_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1032-LABEL: add_i32_varying:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1032-NEXT:    s_mov_b32 s2, exec_lo
 ; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
-; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
+; GFX1032-NEXT:    s_or_saveexec_b32 s3, -1
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s3
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
 ; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
@@ -615,11 +607,11 @@ define amdgpu_kernel void @add_i32_varying_gfx1032(i32 addrspace(1)* %out) {
 ; GFX8-LABEL: add_i32_varying_gfx1032:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    s_mov_b64 s[2:3], exec
 ; GFX8-NEXT:    v_mov_b32_e32 v2, v0
-; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX8-NEXT:    s_or_saveexec_b64 s[4:5], -1
 ; GFX8-NEXT:    v_mov_b32_e32 v1, 0
-; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX8-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
 ; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX8-NEXT:    s_not_b64 exec, exec
@@ -668,11 +660,11 @@ define amdgpu_kernel void @add_i32_varying_gfx1032(i32 addrspace(1)* %out) {
 ; GFX9-LABEL: add_i32_varying_gfx1032:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_mov_b64 s[2:3], exec
 ; GFX9-NEXT:    v_mov_b32_e32 v2, v0
-; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], -1
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
 ; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX9-NEXT:    s_not_b64 exec, exec
@@ -720,11 +712,11 @@ define amdgpu_kernel void @add_i32_varying_gfx1032(i32 addrspace(1)* %out) {
 ; GFX1064-LABEL: add_i32_varying_gfx1032:
 ; GFX1064:       ; %bb.0: ; %entry
 ; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1064-NEXT:    s_mov_b64 s[2:3], exec
 ; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
-; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
 ; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
 ; GFX1064-NEXT:    s_not_b64 exec, exec
@@ -779,12 +771,12 @@ define amdgpu_kernel void @add_i32_varying_gfx1032(i32 addrspace(1)* %out) {
 ; GFX1032-LABEL: add_i32_varying_gfx1032:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1032-NEXT:    s_mov_b32 s2, exec_lo
 ; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
-; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
+; GFX1032-NEXT:    s_or_saveexec_b32 s3, -1
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s3
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
 ; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
@@ -854,11 +846,11 @@ define amdgpu_kernel void @add_i32_varying_gfx1064(i32 addrspace(1)* %out) {
 ; GFX8-LABEL: add_i32_varying_gfx1064:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    s_mov_b64 s[2:3], exec
 ; GFX8-NEXT:    v_mov_b32_e32 v2, v0
-; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX8-NEXT:    s_or_saveexec_b64 s[4:5], -1
 ; GFX8-NEXT:    v_mov_b32_e32 v1, 0
-; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX8-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
 ; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX8-NEXT:    s_not_b64 exec, exec
@@ -907,11 +899,11 @@ define amdgpu_kernel void @add_i32_varying_gfx1064(i32 addrspace(1)* %out) {
 ; GFX9-LABEL: add_i32_varying_gfx1064:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_mov_b64 s[2:3], exec
 ; GFX9-NEXT:    v_mov_b32_e32 v2, v0
-; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], -1
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
 ; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX9-NEXT:    s_not_b64 exec, exec
@@ -959,11 +951,11 @@ define amdgpu_kernel void @add_i32_varying_gfx1064(i32 addrspace(1)* %out) {
 ; GFX1064-LABEL: add_i32_varying_gfx1064:
 ; GFX1064:       ; %bb.0: ; %entry
 ; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1064-NEXT:    s_mov_b64 s[2:3], exec
 ; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
-; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
 ; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
 ; GFX1064-NEXT:    s_not_b64 exec, exec
@@ -1018,12 +1010,12 @@ define amdgpu_kernel void @add_i32_varying_gfx1064(i32 addrspace(1)* %out) {
 ; GFX1032-LABEL: add_i32_varying_gfx1064:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1032-NEXT:    s_mov_b32 s2, exec_lo
 ; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
-; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
+; GFX1032-NEXT:    s_or_saveexec_b32 s3, -1
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s3
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
 ; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
@@ -1078,8 +1070,8 @@ define amdgpu_kernel void @add_i64_constant(i64 addrspace(1)* %out) {
 ;
 ; GFX7LESS-LABEL: add_i64_constant:
 ; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], exec
 ; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
-; GFX7LESS-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, 0
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s4, 0
 ; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s5, v0
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
@@ -1114,7 +1106,7 @@ define amdgpu_kernel void @add_i64_constant(i64 addrspace(1)* %out) {
 ; GFX8-LABEL: add_i64_constant:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX8-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, 0
+; GFX8-NEXT:    s_mov_b64 s[4:5], exec
 ; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
 ; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
@@ -1148,7 +1140,7 @@ define amdgpu_kernel void @add_i64_constant(i64 addrspace(1)* %out) {
 ; GFX9-LABEL: add_i64_constant:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, 0
+; GFX9-NEXT:    s_mov_b64 s[4:5], exec
 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
 ; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
@@ -1180,7 +1172,7 @@ define amdgpu_kernel void @add_i64_constant(i64 addrspace(1)* %out) {
 ;
 ; GFX1064-LABEL: add_i64_constant:
 ; GFX1064:       ; %bb.0: ; %entry
-; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, 0
+; GFX1064-NEXT:    s_mov_b64 s[4:5], exec
 ; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX1064-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s4, 0
@@ -1215,7 +1207,7 @@ define amdgpu_kernel void @add_i64_constant(i64 addrspace(1)* %out) {
 ; GFX1032-LABEL: add_i64_constant:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX1032-NEXT:    v_cmp_ne_u32_e64 s3, 1, 0
+; GFX1032-NEXT:    s_mov_b32 s3, exec_lo
 ; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s3, 0
@@ -1256,8 +1248,8 @@ define amdgpu_kernel void @add_i64_uniform(i64 addrspace(1)* %out, i64 %additive
 ;
 ; GFX7LESS-LABEL: add_i64_uniform:
 ; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_mov_b64 s[6:7], exec
 ; GFX7LESS-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
-; GFX7LESS-NEXT:    v_cmp_ne_u32_e64 s[6:7], 1, 0
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
 ; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s7, v0
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
@@ -1301,7 +1293,7 @@ define amdgpu_kernel void @add_i64_uniform(i64 addrspace(1)* %out, i64 %additive
 ; GFX8-LABEL: add_i64_uniform:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX8-NEXT:    v_cmp_ne_u32_e64 s[6:7], 1, 0
+; GFX8-NEXT:    s_mov_b64 s[6:7], exec
 ; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
 ; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
@@ -1345,7 +1337,7 @@ define amdgpu_kernel void @add_i64_uniform(i64 addrspace(1)* %out, i64 %additive
 ; GFX9-LABEL: add_i64_uniform:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[6:7], 1, 0
+; GFX9-NEXT:    s_mov_b64 s[6:7], exec
 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
 ; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
@@ -1387,7 +1379,7 @@ define amdgpu_kernel void @add_i64_uniform(i64 addrspace(1)* %out, i64 %additive
 ;
 ; GFX1064-LABEL: add_i64_uniform:
 ; GFX1064:       ; %bb.0: ; %entry
-; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[6:7], 1, 0
+; GFX1064-NEXT:    s_mov_b64 s[6:7], exec
 ; GFX1064-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX1064-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
@@ -1431,7 +1423,7 @@ define amdgpu_kernel void @add_i64_uniform(i64 addrspace(1)* %out, i64 %additive
 ; GFX1032-LABEL: add_i64_uniform:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX1032-NEXT:    v_cmp_ne_u32_e64 s5, 1, 0
+; GFX1032-NEXT:    s_mov_b32 s5, exec_lo
 ; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s5, 0
@@ -1571,25 +1563,25 @@ define amdgpu_kernel void @sub_i32_constant(i32 addrspace(1)* %out) {
 ;
 ; GFX7LESS-LABEL: sub_i32_constant:
 ; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], exec
 ; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
-; GFX7LESS-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, 0
-; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s4, 0
-; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s5, v0
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-NEXT:    ; implicit-def: $vgpr1
-; GFX7LESS-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX7LESS-NEXT:    s_cbranch_execz BB8_2
 ; GFX7LESS-NEXT:  ; %bb.1:
-; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
+; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
 ; GFX7LESS-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
-; GFX7LESS-NEXT:    v_mul_u32_u24_e64 v2, s4, 5
+; GFX7LESS-NEXT:    v_mul_u32_u24_e64 v2, s2, 5
 ; GFX7LESS-NEXT:    s_mov_b32 m0, -1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7LESS-NEXT:    ds_sub_rtn_u32 v1, v1, v2
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7LESS-NEXT:    buffer_wbinvl1
 ; GFX7LESS-NEXT:  BB8_2:
-; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX7LESS-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
 ; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
@@ -1602,16 +1594,16 @@ define amdgpu_kernel void @sub_i32_constant(i32 addrspace(1)* %out) {
 ; GFX8-LABEL: sub_i32_constant:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX8-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, 0
-; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
-; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
+; GFX8-NEXT:    s_mov_b64 s[2:3], exec
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX8-NEXT:    ; implicit-def: $vgpr1
-; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX8-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX8-NEXT:    s_cbranch_execz BB8_2
 ; GFX8-NEXT:  ; %bb.1:
-; GFX8-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
-; GFX8-NEXT:    v_mul_u32_u24_e64 v1, s4, 5
+; GFX8-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX8-NEXT:    v_mul_u32_u24_e64 v1, s2, 5
 ; GFX8-NEXT:    v_mov_b32_e32 v2, local_var32@abs32@lo
 ; GFX8-NEXT:    s_mov_b32 m0, -1
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1619,7 +1611,7 @@ define amdgpu_kernel void @sub_i32_constant(i32 addrspace(1)* %out) {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
 ; GFX8-NEXT:  BB8_2:
-; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX8-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX8-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
 ; GFX8-NEXT:    v_sub_u32_e32 v0, vcc, s2, v0
@@ -1633,23 +1625,23 @@ define amdgpu_kernel void @sub_i32_constant(i32 addrspace(1)* %out) {
 ; GFX9-LABEL: sub_i32_constant:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, 0
-; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
-; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
+; GFX9-NEXT:    s_mov_b64 s[2:3], exec
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    ; implicit-def: $vgpr1
-; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX9-NEXT:    s_cbranch_execz BB8_2
 ; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
-; GFX9-NEXT:    v_mul_u32_u24_e64 v1, s4, 5
+; GFX9-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX9-NEXT:    v_mul_u32_u24_e64 v1, s2, 5
 ; GFX9-NEXT:    v_mov_b32_e32 v2, local_var32@abs32@lo
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    ds_sub_rtn_u32 v1, v2, v1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:  BB8_2:
-; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX9-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
 ; GFX9-NEXT:    v_sub_u32_e32 v0, s2, v0
@@ -1662,18 +1654,18 @@ define amdgpu_kernel void @sub_i32_constant(i32 addrspace(1)* %out) {
 ;
 ; GFX1064-LABEL: sub_i32_constant:
 ; GFX1064:       ; %bb.0: ; %entry
-; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, 0
+; GFX1064-NEXT:    s_mov_b64 s[2:3], exec
 ; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX1064-NEXT:    ; implicit-def: $vgpr1
-; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s4, 0
-; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s5, v0
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX1064-NEXT:    s_cbranch_execz BB8_2
 ; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
+; GFX1064-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
 ; GFX1064-NEXT:    v_mov_b32_e32 v2, local_var32@abs32@lo
-; GFX1064-NEXT:    v_mul_u32_u24_e64 v1, s4, 5
+; GFX1064-NEXT:    v_mul_u32_u24_e64 v1, s2, 5
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX1064-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1064-NEXT:    ds_sub_rtn_u32 v1, v2, v1
@@ -1682,7 +1674,7 @@ define amdgpu_kernel void @sub_i32_constant(i32 addrspace(1)* %out) {
 ; GFX1064-NEXT:    buffer_gl1_inv
 ; GFX1064-NEXT:  BB8_2:
 ; GFX1064-NEXT:    v_nop
-; GFX1064-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX1064-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX1064-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
 ; GFX1064-NEXT:    s_mov_b32 s3, 0x31016000
@@ -1696,17 +1688,17 @@ define amdgpu_kernel void @sub_i32_constant(i32 addrspace(1)* %out) {
 ; GFX1032-LABEL: sub_i32_constant:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX1032-NEXT:    v_cmp_ne_u32_e64 s3, 1, 0
+; GFX1032-NEXT:    s_mov_b32 s2, exec_lo
 ; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    ; implicit-def: $vgpr1
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s3, 0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT:    s_and_saveexec_b32 s2, vcc_lo
+; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz BB8_2
 ; GFX1032-NEXT:  ; %bb.1:
-; GFX1032-NEXT:    s_bcnt1_i32_b32 s3, s3
+; GFX1032-NEXT:    s_bcnt1_i32_b32 s2, s2
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, local_var32@abs32@lo
-; GFX1032-NEXT:    v_mul_u32_u24_e64 v1, s3, 5
+; GFX1032-NEXT:    v_mul_u32_u24_e64 v1, s2, 5
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1032-NEXT:    ds_sub_rtn_u32 v1, v2, v1
@@ -1715,7 +1707,7 @@ define amdgpu_kernel void @sub_i32_constant(i32 addrspace(1)* %out) {
 ; GFX1032-NEXT:    buffer_gl1_inv
 ; GFX1032-NEXT:  BB8_2:
 ; GFX1032-NEXT:    v_nop
-; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT:    s_or_b32 exec_lo, exec_lo, s3
 ; GFX1032-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX1032-NEXT:    v_mul_u32_u24_e32 v0, 5, v0
 ; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
@@ -1736,9 +1728,9 @@ define amdgpu_kernel void @sub_i32_uniform(i32 addrspace(1)* %out, i32 %subitive
 ;
 ; GFX7LESS-LABEL: sub_i32_uniform:
 ; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_mov_b64 s[6:7], exec
 ; GFX7LESS-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
 ; GFX7LESS-NEXT:    s_load_dword s2, s[0:1], 0xb
-; GFX7LESS-NEXT:    v_cmp_ne_u32_e64 s[6:7], 1, 0
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
 ; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s7, v0
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
@@ -1771,15 +1763,15 @@ define amdgpu_kernel void @sub_i32_uniform(i32 addrspace(1)* %out, i32 %subitive
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
 ; GFX8-NEXT:    s_load_dword s0, s[0:1], 0x2c
-; GFX8-NEXT:    v_cmp_ne_u32_e64 s[6:7], 1, 0
-; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX8-NEXT:    s_mov_b64 s[2:3], exec
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX8-NEXT:    ; implicit-def: $vgpr1
-; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX8-NEXT:    s_and_saveexec_b64 s[6:7], vcc
 ; GFX8-NEXT:    s_cbranch_execz BB9_2
 ; GFX8-NEXT:  ; %bb.1:
-; GFX8-NEXT:    s_bcnt1_i32_b64 s1, s[6:7]
+; GFX8-NEXT:    s_bcnt1_i32_b64 s1, s[2:3]
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_mul_i32 s1, s0, s1
 ; GFX8-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
@@ -1790,7 +1782,7 @@ define amdgpu_kernel void @sub_i32_uniform(i32 addrspace(1)* %out, i32 %subitive
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
 ; GFX8-NEXT:  BB9_2:
-; GFX8-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mul_lo_u32 v0, s0, v0
 ; GFX8-NEXT:    v_readfirstlane_b32 s0, v1
@@ -1804,15 +1796,15 @@ define amdgpu_kernel void @sub_i32_uniform(i32 addrspace(1)* %out, i32 %subitive
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
 ; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x2c
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[6:7], 1, 0
-; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX9-NEXT:    s_mov_b64 s[2:3], exec
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    ; implicit-def: $vgpr1
-; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT:    s_and_saveexec_b64 s[6:7], vcc
 ; GFX9-NEXT:    s_cbranch_execz BB9_2
 ; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    s_bcnt1_i32_b64 s1, s[6:7]
+; GFX9-NEXT:    s_bcnt1_i32_b64 s1, s[2:3]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_mul_i32 s1, s0, s1
 ; GFX9-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
@@ -1822,7 +1814,7 @@ define amdgpu_kernel void @sub_i32_uniform(i32 addrspace(1)* %out, i32 %subitive
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
 ; GFX9-NEXT:  BB9_2:
-; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX9-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mul_lo_u32 v0, s0, v0
 ; GFX9-NEXT:    v_readfirstlane_b32 s0, v1
@@ -1834,17 +1826,17 @@ define amdgpu_kernel void @sub_i32_uniform(i32 addrspace(1)* %out, i32 %subitive
 ;
 ; GFX1064-LABEL: sub_i32_uniform:
 ; GFX1064:       ; %bb.0: ; %entry
-; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[6:7], 1, 0
+; GFX1064-NEXT:    s_mov_b64 s[2:3], exec
 ; GFX1064-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
 ; GFX1064-NEXT:    s_load_dword s0, s[0:1], 0x2c
 ; GFX1064-NEXT:    ; implicit-def: $vgpr1
-; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s7, v0
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-NEXT:    s_and_saveexec_b64 s[6:7], vcc
 ; GFX1064-NEXT:    s_cbranch_execz BB9_2
 ; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    s_bcnt1_i32_b64 s1, s[6:7]
+; GFX1064-NEXT:    s_bcnt1_i32_b64 s1, s[2:3]
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, local_var32@abs32@lo
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_mul_i32 s1, s0, s1
@@ -1857,7 +1849,7 @@ define amdgpu_kernel void @sub_i32_uniform(i32 addrspace(1)* %out, i32 %subitive
 ; GFX1064-NEXT:    buffer_gl1_inv
 ; GFX1064-NEXT:  BB9_2:
 ; GFX1064-NEXT:    v_nop
-; GFX1064-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX1064-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    v_mul_lo_u32 v0, s0, v0
 ; GFX1064-NEXT:    v_readfirstlane_b32 s0, v1
@@ -1871,7 +1863,7 @@ define amdgpu_kernel void @sub_i32_uniform(i32 addrspace(1)* %out, i32 %subitive
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
 ; GFX1032-NEXT:    s_load_dword s0, s[0:1], 0x2c
-; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    s_mov_b32 s2, exec_lo
 ; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    ; implicit-def: $vgpr1
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
@@ -1907,14 +1899,6 @@ entry:
   ret void
 }
 
-; GFX7LESS-NOT: v_mbcnt_lo_u32_b32
-; GFX7LESS-NOT: v_mbcnt_hi_u32_b32
-; GFX7LESS-NOT: s_bcnt1_i32_b64
-; DPPCOMB: v_add_u32_dpp
-; DPPCOMB: v_add_u32_dpp
-; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31
-; GFX8MORE: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
-; GFX8MORE: ds_sub_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v[[value]]
 define amdgpu_kernel void @sub_i32_varying(i32 addrspace(1)* %out) {
 ;
 ;
@@ -1935,11 +1919,11 @@ define amdgpu_kernel void @sub_i32_varying(i32 addrspace(1)* %out) {
 ; GFX8-LABEL: sub_i32_varying:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    s_mov_b64 s[2:3], exec
 ; GFX8-NEXT:    v_mov_b32_e32 v2, v0
-; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX8-NEXT:    s_or_saveexec_b64 s[4:5], -1
 ; GFX8-NEXT:    v_mov_b32_e32 v1, 0
-; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX8-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
 ; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX8-NEXT:    s_not_b64 exec, exec
@@ -1988,11 +1972,11 @@ define amdgpu_kernel void @sub_i32_varying(i32 addrspace(1)* %out) {
 ; GFX9-LABEL: sub_i32_varying:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_mov_b64 s[2:3], exec
 ; GFX9-NEXT:    v_mov_b32_e32 v2, v0
-; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], -1
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
 ; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX9-NEXT:    s_not_b64 exec, exec
@@ -2040,11 +2024,11 @@ define amdgpu_kernel void @sub_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1064-LABEL: sub_i32_varying:
 ; GFX1064:       ; %bb.0: ; %entry
 ; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1064-NEXT:    s_mov_b64 s[2:3], exec
 ; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
-; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
 ; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
 ; GFX1064-NEXT:    s_not_b64 exec, exec
@@ -2099,12 +2083,12 @@ define amdgpu_kernel void @sub_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1032-LABEL: sub_i32_varying:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1032-NEXT:    s_mov_b32 s2, exec_lo
 ; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
-; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
+; GFX1032-NEXT:    s_or_saveexec_b32 s3, -1
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s3
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
 ; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
@@ -2159,8 +2143,8 @@ define amdgpu_kernel void @sub_i64_constant(i64 addrspace(1)* %out) {
 ;
 ; GFX7LESS-LABEL: sub_i64_constant:
 ; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], exec
 ; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
-; GFX7LESS-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, 0
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s4, 0
 ; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s5, v0
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
@@ -2195,7 +2179,7 @@ define amdgpu_kernel void @sub_i64_constant(i64 addrspace(1)* %out) {
 ; GFX8-LABEL: sub_i64_constant:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX8-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, 0
+; GFX8-NEXT:    s_mov_b64 s[4:5], exec
 ; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
 ; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
@@ -2230,7 +2214,7 @@ define amdgpu_kernel void @sub_i64_constant(i64 addrspace(1)* %out) {
 ; GFX9-LABEL: sub_i64_constant:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, 0
+; GFX9-NEXT:    s_mov_b64 s[4:5], exec
 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
 ; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
@@ -2263,7 +2247,7 @@ define amdgpu_kernel void @sub_i64_constant(i64 addrspace(1)* %out) {
 ;
 ; GFX1064-LABEL: sub_i64_constant:
 ; GFX1064:       ; %bb.0: ; %entry
-; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[4:5], 1, 0
+; GFX1064-NEXT:    s_mov_b64 s[4:5], exec
 ; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX1064-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s4, 0
@@ -2300,7 +2284,7 @@ define amdgpu_kernel void @sub_i64_constant(i64 addrspace(1)* %out) {
 ; GFX1032-LABEL: sub_i64_constant:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX1032-NEXT:    v_cmp_ne_u32_e64 s3, 1, 0
+; GFX1032-NEXT:    s_mov_b32 s3, exec_lo
 ; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s3, 0
@@ -2343,8 +2327,8 @@ define amdgpu_kernel void @sub_i64_uniform(i64 addrspace(1)* %out, i64 %subitive
 ;
 ; GFX7LESS-LABEL: sub_i64_uniform:
 ; GFX7LESS:       ; %bb.0: ; %entry
+; GFX7LESS-NEXT:    s_mov_b64 s[6:7], exec
 ; GFX7LESS-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
-; GFX7LESS-NEXT:    v_cmp_ne_u32_e64 s[6:7], 1, 0
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
 ; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s7, v0
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
@@ -2388,7 +2372,7 @@ define amdgpu_kernel void @sub_i64_uniform(i64 addrspace(1)* %out, i64 %subitive
 ; GFX8-LABEL: sub_i64_uniform:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX8-NEXT:    v_cmp_ne_u32_e64 s[6:7], 1, 0
+; GFX8-NEXT:    s_mov_b64 s[6:7], exec
 ; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
 ; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
@@ -2432,7 +2416,7 @@ define amdgpu_kernel void @sub_i64_uniform(i64 addrspace(1)* %out, i64 %subitive
 ; GFX9-LABEL: sub_i64_uniform:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[6:7], 1, 0
+; GFX9-NEXT:    s_mov_b64 s[6:7], exec
 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s6, 0
 ; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s7, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
@@ -2474,7 +2458,7 @@ define amdgpu_kernel void @sub_i64_uniform(i64 addrspace(1)* %out, i64 %subitive
 ;
 ; GFX1064-LABEL: sub_i64_uniform:
 ; GFX1064:       ; %bb.0: ; %entry
-; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[6:7], 1, 0
+; GFX1064-NEXT:    s_mov_b64 s[6:7], exec
 ; GFX1064-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX1064-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
@@ -2518,7 +2502,7 @@ define amdgpu_kernel void @sub_i64_uniform(i64 addrspace(1)* %out, i64 %subitive
 ; GFX1032-LABEL: sub_i64_uniform:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX1032-NEXT:    v_cmp_ne_u32_e64 s5, 1, 0
+; GFX1032-NEXT:    s_mov_b32 s5, exec_lo
 ; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    ; implicit-def: $vgpr1_vgpr2
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s5, 0
@@ -2653,9 +2637,6 @@ entry:
   ret void
 }
 
-; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31
-; GFX8MORE: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
-; GFX8MORE: ds_and_rtn_b32 v{{[0-9]+}}, v{{[0-9]+}}, v[[value]]
 define amdgpu_kernel void @and_i32_varying(i32 addrspace(1)* %out) {
 ;
 ;
@@ -2676,9 +2657,8 @@ define amdgpu_kernel void @and_i32_varying(i32 addrspace(1)* %out) {
 ; GFX8-LABEL: and_i32_varying:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX8-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
-; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v3, s2, 0
-; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v3, s3, v3
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
 ; GFX8-NEXT:    v_mov_b32_e32 v2, v0
 ; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
 ; GFX8-NEXT:    v_mov_b32_e32 v1, -1
@@ -2729,9 +2709,8 @@ define amdgpu_kernel void @and_i32_varying(i32 addrspace(1)* %out) {
 ; GFX9-LABEL: and_i32_varying:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
-; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, s2, 0
-; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, s3, v3
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
 ; GFX9-NEXT:    v_mov_b32_e32 v2, v0
 ; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
 ; GFX9-NEXT:    v_mov_b32_e32 v1, -1
@@ -2781,10 +2760,9 @@ define amdgpu_kernel void @and_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1064-LABEL: and_i32_varying:
 ; GFX1064:       ; %bb.0: ; %entry
 ; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v4, exec_lo, 0
 ; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
-; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v4, s2, 0
-; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v4, s3, v4
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v4, exec_hi, v4
 ; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, -1
 ; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
@@ -2840,10 +2818,9 @@ define amdgpu_kernel void @and_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1032-LABEL: and_i32_varying:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v4, exec_lo, 0
 ; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v4, s2, 0
 ; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, -1
 ; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
@@ -2895,9 +2872,6 @@ entry:
   ret void
 }
 
-; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31
-; GFX8MORE: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
-; GFX8MORE: ds_or_rtn_b32 v{{[0-9]+}}, v{{[0-9]+}}, v[[value]]
 define amdgpu_kernel void @or_i32_varying(i32 addrspace(1)* %out) {
 ;
 ;
@@ -2918,11 +2892,11 @@ define amdgpu_kernel void @or_i32_varying(i32 addrspace(1)* %out) {
 ; GFX8-LABEL: or_i32_varying:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    s_mov_b64 s[2:3], exec
 ; GFX8-NEXT:    v_mov_b32_e32 v2, v0
-; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX8-NEXT:    s_or_saveexec_b64 s[4:5], -1
 ; GFX8-NEXT:    v_mov_b32_e32 v1, 0
-; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX8-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
 ; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX8-NEXT:    s_not_b64 exec, exec
@@ -2971,11 +2945,11 @@ define amdgpu_kernel void @or_i32_varying(i32 addrspace(1)* %out) {
 ; GFX9-LABEL: or_i32_varying:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_mov_b64 s[2:3], exec
 ; GFX9-NEXT:    v_mov_b32_e32 v2, v0
-; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], -1
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
 ; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX9-NEXT:    s_not_b64 exec, exec
@@ -3023,11 +2997,11 @@ define amdgpu_kernel void @or_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1064-LABEL: or_i32_varying:
 ; GFX1064:       ; %bb.0: ; %entry
 ; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1064-NEXT:    s_mov_b64 s[2:3], exec
 ; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
-; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
 ; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
 ; GFX1064-NEXT:    s_not_b64 exec, exec
@@ -3082,12 +3056,12 @@ define amdgpu_kernel void @or_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1032-LABEL: or_i32_varying:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1032-NEXT:    s_mov_b32 s2, exec_lo
 ; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
-; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
+; GFX1032-NEXT:    s_or_saveexec_b32 s3, -1
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s3
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
 ; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
@@ -3137,9 +3111,6 @@ entry:
   ret void
 }
 
-; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31
-; GFX8MORE: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
-; GFX8MORE: ds_xor_rtn_b32 v{{[0-9]+}}, v{{[0-9]+}}, v[[value]]
 define amdgpu_kernel void @xor_i32_varying(i32 addrspace(1)* %out) {
 ;
 ;
@@ -3160,11 +3131,11 @@ define amdgpu_kernel void @xor_i32_varying(i32 addrspace(1)* %out) {
 ; GFX8-LABEL: xor_i32_varying:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    s_mov_b64 s[2:3], exec
 ; GFX8-NEXT:    v_mov_b32_e32 v2, v0
-; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX8-NEXT:    s_or_saveexec_b64 s[4:5], -1
 ; GFX8-NEXT:    v_mov_b32_e32 v1, 0
-; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX8-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
 ; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX8-NEXT:    s_not_b64 exec, exec
@@ -3213,11 +3184,11 @@ define amdgpu_kernel void @xor_i32_varying(i32 addrspace(1)* %out) {
 ; GFX9-LABEL: xor_i32_varying:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_mov_b64 s[2:3], exec
 ; GFX9-NEXT:    v_mov_b32_e32 v2, v0
-; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], -1
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
 ; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX9-NEXT:    s_not_b64 exec, exec
@@ -3265,11 +3236,11 @@ define amdgpu_kernel void @xor_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1064-LABEL: xor_i32_varying:
 ; GFX1064:       ; %bb.0: ; %entry
 ; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1064-NEXT:    s_mov_b64 s[2:3], exec
 ; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
-; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
 ; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
 ; GFX1064-NEXT:    s_not_b64 exec, exec
@@ -3324,12 +3295,12 @@ define amdgpu_kernel void @xor_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1032-LABEL: xor_i32_varying:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1032-NEXT:    s_mov_b32 s2, exec_lo
 ; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
-; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
+; GFX1032-NEXT:    s_or_saveexec_b32 s3, -1
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s3
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
 ; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
@@ -3379,9 +3350,6 @@ entry:
   ret void
 }
 
-; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31
-; GFX8MORE: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
-; GFX8MORE: ds_max_rtn_i32 v{{[0-9]+}}, v{{[0-9]+}}, v[[value]]
 define amdgpu_kernel void @max_i32_varying(i32 addrspace(1)* %out) {
 ;
 ;
@@ -3402,9 +3370,8 @@ define amdgpu_kernel void @max_i32_varying(i32 addrspace(1)* %out) {
 ; GFX8-LABEL: max_i32_varying:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX8-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
-; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v3, s2, 0
-; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v3, s3, v3
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
 ; GFX8-NEXT:    v_mov_b32_e32 v2, v0
 ; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
 ; GFX8-NEXT:    v_bfrev_b32_e32 v1, 1
@@ -3455,9 +3422,8 @@ define amdgpu_kernel void @max_i32_varying(i32 addrspace(1)* %out) {
 ; GFX9-LABEL: max_i32_varying:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
-; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, s2, 0
-; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, s3, v3
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
 ; GFX9-NEXT:    v_mov_b32_e32 v2, v0
 ; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
 ; GFX9-NEXT:    v_bfrev_b32_e32 v1, 1
@@ -3507,10 +3473,9 @@ define amdgpu_kernel void @max_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1064-LABEL: max_i32_varying:
 ; GFX1064:       ; %bb.0: ; %entry
 ; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v4, exec_lo, 0
 ; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
-; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v4, s2, 0
-; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v4, s3, v4
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v4, exec_hi, v4
 ; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
 ; GFX1064-NEXT:    v_bfrev_b32_e32 v1, 1
 ; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
@@ -3566,10 +3531,9 @@ define amdgpu_kernel void @max_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1032-LABEL: max_i32_varying:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v4, exec_lo, 0
 ; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v4, s2, 0
 ; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
 ; GFX1032-NEXT:    v_bfrev_b32_e32 v1, 1
 ; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
@@ -3627,9 +3591,8 @@ define amdgpu_kernel void @max_i64_constant(i64 addrspace(1)* %out) {
 ; GFX7LESS-LABEL: max_i64_constant:
 ; GFX7LESS:       ; %bb.0: ; %entry
 ; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
-; GFX7LESS-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
-; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX7LESS-NEXT:    s_and_saveexec_b64 s[2:3], vcc
@@ -3664,9 +3627,8 @@ define amdgpu_kernel void @max_i64_constant(i64 addrspace(1)* %out) {
 ; GFX8-LABEL: max_i64_constant:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX8-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
-; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX8-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
@@ -3701,9 +3663,8 @@ define amdgpu_kernel void @max_i64_constant(i64 addrspace(1)* %out) {
 ; GFX9-LABEL: max_i64_constant:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
-; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
@@ -3736,10 +3697,9 @@ define amdgpu_kernel void @max_i64_constant(i64 addrspace(1)* %out) {
 ;
 ; GFX1064-LABEL: max_i64_constant:
 ; GFX1064:       ; %bb.0: ; %entry
-; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
 ; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, exec_hi, v0
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[2:3], vcc
@@ -3773,9 +3733,8 @@ define amdgpu_kernel void @max_i64_constant(i64 addrspace(1)* %out) {
 ; GFX1032-LABEL: max_i64_constant:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
 ; GFX1032-NEXT:    ; implicit-def: $vcc_hi
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX1032-NEXT:    s_and_saveexec_b32 s2, vcc_lo
@@ -3811,9 +3770,6 @@ entry:
   ret void
 }
 
-; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31
-; GFX8MORE: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
-; GFX8MORE: ds_min_rtn_i32 v{{[0-9]+}}, v{{[0-9]+}}, v[[value]]
 define amdgpu_kernel void @min_i32_varying(i32 addrspace(1)* %out) {
 ;
 ;
@@ -3834,9 +3790,8 @@ define amdgpu_kernel void @min_i32_varying(i32 addrspace(1)* %out) {
 ; GFX8-LABEL: min_i32_varying:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX8-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
-; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v3, s2, 0
-; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v3, s3, v3
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
 ; GFX8-NEXT:    v_mov_b32_e32 v2, v0
 ; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
 ; GFX8-NEXT:    v_bfrev_b32_e32 v1, -2
@@ -3887,9 +3842,8 @@ define amdgpu_kernel void @min_i32_varying(i32 addrspace(1)* %out) {
 ; GFX9-LABEL: min_i32_varying:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
-; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, s2, 0
-; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, s3, v3
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
 ; GFX9-NEXT:    v_mov_b32_e32 v2, v0
 ; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
 ; GFX9-NEXT:    v_bfrev_b32_e32 v1, -2
@@ -3939,10 +3893,9 @@ define amdgpu_kernel void @min_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1064-LABEL: min_i32_varying:
 ; GFX1064:       ; %bb.0: ; %entry
 ; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v4, exec_lo, 0
 ; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
-; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v4, s2, 0
-; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v4, s3, v4
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v4, exec_hi, v4
 ; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
 ; GFX1064-NEXT:    v_bfrev_b32_e32 v1, -2
 ; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
@@ -3998,10 +3951,9 @@ define amdgpu_kernel void @min_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1032-LABEL: min_i32_varying:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v4, exec_lo, 0
 ; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v4, s2, 0
 ; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
 ; GFX1032-NEXT:    v_bfrev_b32_e32 v1, -2
 ; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
@@ -4059,9 +4011,8 @@ define amdgpu_kernel void @min_i64_constant(i64 addrspace(1)* %out) {
 ; GFX7LESS-LABEL: min_i64_constant:
 ; GFX7LESS:       ; %bb.0: ; %entry
 ; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
-; GFX7LESS-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
-; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX7LESS-NEXT:    s_and_saveexec_b64 s[2:3], vcc
@@ -4096,9 +4047,8 @@ define amdgpu_kernel void @min_i64_constant(i64 addrspace(1)* %out) {
 ; GFX8-LABEL: min_i64_constant:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX8-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
-; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX8-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
@@ -4133,9 +4083,8 @@ define amdgpu_kernel void @min_i64_constant(i64 addrspace(1)* %out) {
 ; GFX9-LABEL: min_i64_constant:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
-; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
@@ -4168,10 +4117,9 @@ define amdgpu_kernel void @min_i64_constant(i64 addrspace(1)* %out) {
 ;
 ; GFX1064-LABEL: min_i64_constant:
 ; GFX1064:       ; %bb.0: ; %entry
-; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
 ; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, exec_hi, v0
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[2:3], vcc
@@ -4205,9 +4153,8 @@ define amdgpu_kernel void @min_i64_constant(i64 addrspace(1)* %out) {
 ; GFX1032-LABEL: min_i64_constant:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
 ; GFX1032-NEXT:    ; implicit-def: $vcc_hi
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX1032-NEXT:    s_and_saveexec_b32 s2, vcc_lo
@@ -4243,9 +4190,6 @@ entry:
   ret void
 }
 
-; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31
-; GFX8MORE: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
-; GFX8MORE: ds_max_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v[[value]]
 define amdgpu_kernel void @umax_i32_varying(i32 addrspace(1)* %out) {
 ;
 ;
@@ -4266,11 +4210,11 @@ define amdgpu_kernel void @umax_i32_varying(i32 addrspace(1)* %out) {
 ; GFX8-LABEL: umax_i32_varying:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX8-NEXT:    s_mov_b64 s[2:3], exec
 ; GFX8-NEXT:    v_mov_b32_e32 v2, v0
-; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX8-NEXT:    s_or_saveexec_b64 s[4:5], -1
 ; GFX8-NEXT:    v_mov_b32_e32 v1, 0
-; GFX8-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX8-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX8-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
 ; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX8-NEXT:    s_not_b64 exec, exec
@@ -4319,11 +4263,11 @@ define amdgpu_kernel void @umax_i32_varying(i32 addrspace(1)* %out) {
 ; GFX9-LABEL: umax_i32_varying:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_mov_b64 s[2:3], exec
 ; GFX9-NEXT:    v_mov_b32_e32 v2, v0
-; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX9-NEXT:    s_or_saveexec_b64 s[4:5], -1
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX9-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
 ; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
 ; GFX9-NEXT:    s_not_b64 exec, exec
@@ -4371,11 +4315,11 @@ define amdgpu_kernel void @umax_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1064-LABEL: umax_i32_varying:
 ; GFX1064:       ; %bb.0: ; %entry
 ; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1064-NEXT:    s_mov_b64 s[2:3], exec
 ; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
-; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
+; GFX1064-NEXT:    s_or_saveexec_b64 s[4:5], -1
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
-; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX1064-NEXT:    s_mov_b64 exec, s[4:5]
 ; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
 ; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
 ; GFX1064-NEXT:    s_not_b64 exec, exec
@@ -4430,12 +4374,12 @@ define amdgpu_kernel void @umax_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1032-LABEL: umax_i32_varying:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX1032-NEXT:    s_mov_b32 s2, exec_lo
 ; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
-; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
+; GFX1032-NEXT:    s_or_saveexec_b32 s3, -1
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
-; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s3
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
 ; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
@@ -4491,9 +4435,8 @@ define amdgpu_kernel void @umax_i64_constant(i64 addrspace(1)* %out) {
 ; GFX7LESS-LABEL: umax_i64_constant:
 ; GFX7LESS:       ; %bb.0: ; %entry
 ; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
-; GFX7LESS-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
-; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX7LESS-NEXT:    s_and_saveexec_b64 s[2:3], vcc
@@ -4527,9 +4470,8 @@ define amdgpu_kernel void @umax_i64_constant(i64 addrspace(1)* %out) {
 ; GFX8-LABEL: umax_i64_constant:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX8-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
-; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX8-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
@@ -4563,9 +4505,8 @@ define amdgpu_kernel void @umax_i64_constant(i64 addrspace(1)* %out) {
 ; GFX9-LABEL: umax_i64_constant:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
-; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
@@ -4597,10 +4538,9 @@ define amdgpu_kernel void @umax_i64_constant(i64 addrspace(1)* %out) {
 ;
 ; GFX1064-LABEL: umax_i64_constant:
 ; GFX1064:       ; %bb.0: ; %entry
-; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
 ; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, exec_hi, v0
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[2:3], vcc
@@ -4634,9 +4574,8 @@ define amdgpu_kernel void @umax_i64_constant(i64 addrspace(1)* %out) {
 ; GFX1032-LABEL: umax_i64_constant:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
 ; GFX1032-NEXT:    ; implicit-def: $vcc_hi
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX1032-NEXT:    s_and_saveexec_b32 s2, vcc_lo
@@ -4672,9 +4611,6 @@ entry:
   ret void
 }
 
-; GFX8MORE32: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 31
-; GFX8MORE: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
-; GFX8MORE: ds_min_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v[[value]]
 define amdgpu_kernel void @umin_i32_varying(i32 addrspace(1)* %out) {
 ;
 ;
@@ -4695,9 +4631,8 @@ define amdgpu_kernel void @umin_i32_varying(i32 addrspace(1)* %out) {
 ; GFX8-LABEL: umin_i32_varying:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX8-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
-; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v3, s2, 0
-; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v3, s3, v3
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
 ; GFX8-NEXT:    v_mov_b32_e32 v2, v0
 ; GFX8-NEXT:    s_or_saveexec_b64 s[2:3], -1
 ; GFX8-NEXT:    v_mov_b32_e32 v1, -1
@@ -4748,9 +4683,8 @@ define amdgpu_kernel void @umin_i32_varying(i32 addrspace(1)* %out) {
 ; GFX9-LABEL: umin_i32_varying:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
-; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, s2, 0
-; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, s3, v3
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
 ; GFX9-NEXT:    v_mov_b32_e32 v2, v0
 ; GFX9-NEXT:    s_or_saveexec_b64 s[2:3], -1
 ; GFX9-NEXT:    v_mov_b32_e32 v1, -1
@@ -4800,10 +4734,9 @@ define amdgpu_kernel void @umin_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1064-LABEL: umin_i32_varying:
 ; GFX1064:       ; %bb.0: ; %entry
 ; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v4, exec_lo, 0
 ; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
-; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v4, s2, 0
-; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v4, s3, v4
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v4, exec_hi, v4
 ; GFX1064-NEXT:    s_or_saveexec_b64 s[2:3], -1
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, -1
 ; GFX1064-NEXT:    s_mov_b64 exec, s[2:3]
@@ -4859,10 +4792,9 @@ define amdgpu_kernel void @umin_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1032-LABEL: umin_i32_varying:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v4, exec_lo, 0
 ; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v4, s2, 0
 ; GFX1032-NEXT:    s_or_saveexec_b32 s2, -1
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, -1
 ; GFX1032-NEXT:    s_mov_b32 exec_lo, s2
@@ -4920,9 +4852,8 @@ define amdgpu_kernel void @umin_i64_constant(i64 addrspace(1)* %out) {
 ; GFX7LESS-LABEL: umin_i64_constant:
 ; GFX7LESS:       ; %bb.0: ; %entry
 ; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
-; GFX7LESS-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
-; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX7LESS-NEXT:    s_and_saveexec_b64 s[2:3], vcc
@@ -4956,9 +4887,8 @@ define amdgpu_kernel void @umin_i64_constant(i64 addrspace(1)* %out) {
 ; GFX8-LABEL: umin_i64_constant:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX8-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
-; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX8-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX8-NEXT:    s_and_saveexec_b64 s[2:3], vcc
@@ -4992,9 +4922,8 @@ define amdgpu_kernel void @umin_i64_constant(i64 addrspace(1)* %out) {
 ; GFX9-LABEL: umin_i64_constant:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
-; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
@@ -5026,10 +4955,9 @@ define amdgpu_kernel void @umin_i64_constant(i64 addrspace(1)* %out) {
 ;
 ; GFX1064-LABEL: umin_i64_constant:
 ; GFX1064:       ; %bb.0: ; %entry
-; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[2:3], 1, 0
 ; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s3, v0
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, exec_hi, v0
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[2:3], vcc
@@ -5063,9 +4991,8 @@ define amdgpu_kernel void @umin_i64_constant(i64 addrspace(1)* %out) {
 ; GFX1032-LABEL: umin_i64_constant:
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX1032-NEXT:    v_cmp_ne_u32_e64 s2, 1, 0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
 ; GFX1032-NEXT:    ; implicit-def: $vcc_hi
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX1032-NEXT:    s_and_saveexec_b32 s2, vcc_lo
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
index b13bd13c5f645..a90f0a689ac64 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
@@ -19,7 +19,7 @@ define amdgpu_ps void @add_i32_constant(<4 x i32> inreg %out, <4 x i32> inreg %i
 ; GFX7-NEXT:    s_and_saveexec_b64 s[8:9], s[10:11]
 ; GFX7-NEXT:    s_cbranch_execz BB0_4
 ; GFX7-NEXT:  ; %bb.1:
-; GFX7-NEXT:    v_cmp_ne_u32_e64 s[12:13], 1, 0
+; GFX7-NEXT:    s_mov_b64 s[12:13], exec
 ; GFX7-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s12, 0
 ; GFX7-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s13, v0
 ; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
@@ -52,7 +52,7 @@ define amdgpu_ps void @add_i32_constant(<4 x i32> inreg %out, <4 x i32> inreg %i
 ; GFX8-NEXT:    s_and_saveexec_b64 s[8:9], s[10:11]
 ; GFX8-NEXT:    s_cbranch_execz BB0_4
 ; GFX8-NEXT:  ; %bb.1:
-; GFX8-NEXT:    v_cmp_ne_u32_e64 s[12:13], 1, 0
+; GFX8-NEXT:    s_mov_b64 s[12:13], exec
 ; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s12, 0
 ; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s13, v0
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
@@ -85,7 +85,7 @@ define amdgpu_ps void @add_i32_constant(<4 x i32> inreg %out, <4 x i32> inreg %i
 ; GFX9-NEXT:    s_and_saveexec_b64 s[8:9], s[10:11]
 ; GFX9-NEXT:    s_cbranch_execz BB0_4
 ; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[12:13], 1, 0
+; GFX9-NEXT:    s_mov_b64 s[12:13], exec
 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s12, 0
 ; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s13, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
@@ -118,7 +118,7 @@ define amdgpu_ps void @add_i32_constant(<4 x i32> inreg %out, <4 x i32> inreg %i
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[8:9], s[10:11]
 ; GFX1064-NEXT:    s_cbranch_execz BB0_4
 ; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[12:13], 1, 0
+; GFX1064-NEXT:    s_mov_b64 s[12:13], exec
 ; GFX1064-NEXT:    ; implicit-def: $vgpr1
 ; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s12, 0
 ; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s13, v0
@@ -153,7 +153,7 @@ define amdgpu_ps void @add_i32_constant(<4 x i32> inreg %out, <4 x i32> inreg %i
 ; GFX1032-NEXT:    s_and_saveexec_b32 s8, s9
 ; GFX1032-NEXT:    s_cbranch_execz BB0_4
 ; GFX1032-NEXT:  ; %bb.1:
-; GFX1032-NEXT:    v_cmp_ne_u32_e64 s10, 1, 0
+; GFX1032-NEXT:    s_mov_b32 s10, exec_lo
 ; GFX1032-NEXT:    ; implicit-def: $vgpr1
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s10, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -213,10 +213,10 @@ define amdgpu_ps void @add_i32_varying(<4 x i32> inreg %out, <4 x i32> inreg %in
 ; GFX8-NEXT:    s_and_saveexec_b64 s[8:9], s[10:11]
 ; GFX8-NEXT:    s_cbranch_execz BB1_4
 ; GFX8-NEXT:  ; %bb.1:
-; GFX8-NEXT:    s_or_saveexec_b64 s[10:11], -1
+; GFX8-NEXT:    s_mov_b64 s[10:11], exec
+; GFX8-NEXT:    s_or_saveexec_b64 s[12:13], -1
 ; GFX8-NEXT:    v_mov_b32_e32 v1, 0
-; GFX8-NEXT:    s_mov_b64 exec, s[10:11]
-; GFX8-NEXT:    v_cmp_ne_u32_e64 s[10:11], 1, 0
+; GFX8-NEXT:    s_mov_b64 exec, s[12:13]
 ; GFX8-NEXT:    v_mbcnt_lo_u32_b32 v0, s10, 0
 ; GFX8-NEXT:    v_mbcnt_hi_u32_b32 v0, s11, v0
 ; GFX8-NEXT:    s_not_b64 exec, exec
@@ -270,10 +270,10 @@ define amdgpu_ps void @add_i32_varying(<4 x i32> inreg %out, <4 x i32> inreg %in
 ; GFX9-NEXT:    s_and_saveexec_b64 s[8:9], s[10:11]
 ; GFX9-NEXT:    s_cbranch_execz BB1_4
 ; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    s_or_saveexec_b64 s[10:11], -1
+; GFX9-NEXT:    s_mov_b64 s[10:11], exec
+; GFX9-NEXT:    s_or_saveexec_b64 s[12:13], -1
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    s_mov_b64 exec, s[10:11]
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[10:11], 1, 0
+; GFX9-NEXT:    s_mov_b64 exec, s[12:13]
 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s10, 0
 ; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s11, v0
 ; GFX9-NEXT:    s_not_b64 exec, exec
@@ -327,10 +327,10 @@ define amdgpu_ps void @add_i32_varying(<4 x i32> inreg %out, <4 x i32> inreg %in
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[8:9], s[10:11]
 ; GFX1064-NEXT:    s_cbranch_execz BB1_4
 ; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    s_or_saveexec_b64 s[10:11], -1
+; GFX1064-NEXT:    s_mov_b64 s[10:11], exec
+; GFX1064-NEXT:    s_or_saveexec_b64 s[12:13], -1
 ; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    s_mov_b64 exec, s[10:11]
-; GFX1064-NEXT:    v_cmp_ne_u32_e64 s[10:11], 1, 0
+; GFX1064-NEXT:    s_mov_b64 exec, s[12:13]
 ; GFX1064-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s10, 0
 ; GFX1064-NEXT:    v_mbcnt_hi_u32_b32_e64 v0, s11, v0
 ; GFX1064-NEXT:    s_not_b64 exec, exec
@@ -389,10 +389,10 @@ define amdgpu_ps void @add_i32_varying(<4 x i32> inreg %out, <4 x i32> inreg %in
 ; GFX1032-NEXT:    s_and_saveexec_b32 s8, s9
 ; GFX1032-NEXT:    s_cbranch_execz BB1_4
 ; GFX1032-NEXT:  ; %bb.1:
-; GFX1032-NEXT:    s_or_saveexec_b32 s9, -1
+; GFX1032-NEXT:    s_mov_b32 s9, exec_lo
+; GFX1032-NEXT:    s_or_saveexec_b32 s10, -1
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    s_mov_b32 exec_lo, s9
-; GFX1032-NEXT:    v_cmp_ne_u32_e64 s9, 1, 0
+; GFX1032-NEXT:    s_mov_b32 exec_lo, s10
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s9, 0
 ; GFX1032-NEXT:    s_not_b32 exec_lo, exec_lo
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
index 084d901a2b22d..7318e3963bf4e 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
@@ -8,14 +8,15 @@ declare i32 @llvm.amdgcn.workitem.id.x()
 declare i32 @llvm.amdgcn.raw.buffer.atomic.add(i32, <4 x i32>, i32, i32, i32)
 declare i32 @llvm.amdgcn.raw.buffer.atomic.sub(i32, <4 x i32>, i32, i32, i32)
 
-; Show that what the atomic optimization pass will do for raw buffers.
+; Show what the atomic optimization pass will do for raw buffers.
 
 ; GCN-LABEL: add_i32_constant:
-; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
-; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo
+; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
 ; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc
 ; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
 ; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: v_mul_u32_u24{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[popcount]], 5
@@ -28,11 +29,12 @@ entry:
 }
 
 ; GCN-LABEL: add_i32_uniform:
-; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
-; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo
+; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
 ; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc
 ; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
 ; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: s_mul_i32 s[[scalar_value:[0-9]+]], s{{[0-9]+}}, s[[popcount]]
@@ -78,11 +80,12 @@ entry:
 }
 
 ; GCN-LABEL: sub_i32_constant:
-; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
-; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo
+; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
 ; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc
 ; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
 ; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: v_mul_u32_u24{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[popcount]], 5
@@ -95,11 +98,12 @@ entry:
 }
 
 ; GCN-LABEL: sub_i32_uniform:
-; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
-; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo
+; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
 ; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc
 ; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
 ; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: s_mul_i32 s[[scalar_value:[0-9]+]], s{{[0-9]+}}, s[[popcount]]
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
index ec962513116a7..96113977ba6fa 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
@@ -8,14 +8,15 @@ declare i32 @llvm.amdgcn.workitem.id.x()
 declare i32 @llvm.amdgcn.struct.buffer.atomic.add(i32, <4 x i32>, i32, i32, i32, i32)
 declare i32 @llvm.amdgcn.struct.buffer.atomic.sub(i32, <4 x i32>, i32, i32, i32, i32)
 
-; Show that what the atomic optimization pass will do for struct buffers.
+; Show what the atomic optimization pass will do for struct buffers.
 
 ; GCN-LABEL: add_i32_constant:
-; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
-; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo
+; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
 ; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc
 ; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
 ; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: v_mul_u32_u24{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[popcount]], 5
@@ -28,11 +29,12 @@ entry:
 }
 
 ; GCN-LABEL: add_i32_uniform:
-; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
-; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo
+; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
 ; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc
 ; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
 ; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: s_mul_i32 s[[scalar_value:[0-9]+]], s{{[0-9]+}}, s[[popcount]]
@@ -91,11 +93,12 @@ entry:
 }
 
 ; GCN-LABEL: sub_i32_constant:
-; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
-; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo
+; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
 ; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc
 ; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
 ; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: v_mul_u32_u24{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[popcount]], 5
@@ -108,11 +111,12 @@ entry:
 }
 
 ; GCN-LABEL: sub_i32_uniform:
-; GCN32: v_cmp_ne_u32_e64 s[[exec_lo:[0-9]+]], 1, 0
-; GCN64: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GCN32: s_mov_b32 s[[exec_lo:[0-9]+]], exec_lo
+; GCN64: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt:[0-9]+]], s[[exec_lo]], 0
 ; GCN64: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt]], s[[exec_hi]], v[[mbcnt]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc{{(_lo)?}}, 0, v[[mbcnt]]
+; GCN: s_and_saveexec_b{{32|64}} s[[exec:\[?[0-9:]+\]?]], vcc
 ; GCN32: s_bcnt1_i32_b32 s[[popcount:[0-9]+]], s[[exec_lo]]
 ; GCN64: s_bcnt1_i32_b64 s[[popcount:[0-9]+]], s{{\[}}[[exec_lo]]:[[exec_hi]]{{\]}}
 ; GCN: s_mul_i32 s[[scalar_value:[0-9]+]], s{{[0-9]+}}, s[[popcount]]
diff --git a/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size-vgpr-limit.ll b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size-vgpr-limit.ll
new file mode 100644
index 0000000000000..6b159e0bc746b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size-vgpr-limit.ll
@@ -0,0 +1,559 @@
+; -enable-misched=false makes the register usage more predictable
+; -regalloc=fast just makes the test run faster
+; RUN: llc -march=amdgcn -mcpu=gfx900 -amdgpu-function-calls=false -enable-misched=false -regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX9
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -amdgpu-function-calls=false -enable-misched=false -regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10WGP-WAVE32
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize64 -amdgpu-function-calls=false -enable-misched=false -regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10WGP-WAVE64
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=+cumode -amdgpu-function-calls=false -enable-misched=false -regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10CU-WAVE32
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=+cumode,+wavefrontsize64 -amdgpu-function-calls=false -enable-misched=false -regalloc=fast < %s | FileCheck %s --check-prefixes=GCN,GFX10CU-WAVE64
+
+define internal void @use256vgprs() {
+  %v0 = call i32 asm sideeffect "; def $0", "=v"()
+  %v1 = call i32 asm sideeffect "; def $0", "=v"()
+  %v2 = call i32 asm sideeffect "; def $0", "=v"()
+  %v3 = call i32 asm sideeffect "; def $0", "=v"()
+  %v4 = call i32 asm sideeffect "; def $0", "=v"()
+  %v5 = call i32 asm sideeffect "; def $0", "=v"()
+  %v6 = call i32 asm sideeffect "; def $0", "=v"()
+  %v7 = call i32 asm sideeffect "; def $0", "=v"()
+  %v8 = call i32 asm sideeffect "; def $0", "=v"()
+  %v9 = call i32 asm sideeffect "; def $0", "=v"()
+  %v10 = call i32 asm sideeffect "; def $0", "=v"()
+  %v11 = call i32 asm sideeffect "; def $0", "=v"()
+  %v12 = call i32 asm sideeffect "; def $0", "=v"()
+  %v13 = call i32 asm sideeffect "; def $0", "=v"()
+  %v14 = call i32 asm sideeffect "; def $0", "=v"()
+  %v15 = call i32 asm sideeffect "; def $0", "=v"()
+  %v16 = call i32 asm sideeffect "; def $0", "=v"()
+  %v17 = call i32 asm sideeffect "; def $0", "=v"()
+  %v18 = call i32 asm sideeffect "; def $0", "=v"()
+  %v19 = call i32 asm sideeffect "; def $0", "=v"()
+  %v20 = call i32 asm sideeffect "; def $0", "=v"()
+  %v21 = call i32 asm sideeffect "; def $0", "=v"()
+  %v22 = call i32 asm sideeffect "; def $0", "=v"()
+  %v23 = call i32 asm sideeffect "; def $0", "=v"()
+  %v24 = call i32 asm sideeffect "; def $0", "=v"()
+  %v25 = call i32 asm sideeffect "; def $0", "=v"()
+  %v26 = call i32 asm sideeffect "; def $0", "=v"()
+  %v27 = call i32 asm sideeffect "; def $0", "=v"()
+  %v28 = call i32 asm sideeffect "; def $0", "=v"()
+  %v29 = call i32 asm sideeffect "; def $0", "=v"()
+  %v30 = call i32 asm sideeffect "; def $0", "=v"()
+  %v31 = call i32 asm sideeffect "; def $0", "=v"()
+  %v32 = call i32 asm sideeffect "; def $0", "=v"()
+  %v33 = call i32 asm sideeffect "; def $0", "=v"()
+  %v34 = call i32 asm sideeffect "; def $0", "=v"()
+  %v35 = call i32 asm sideeffect "; def $0", "=v"()
+  %v36 = call i32 asm sideeffect "; def $0", "=v"()
+  %v37 = call i32 asm sideeffect "; def $0", "=v"()
+  %v38 = call i32 asm sideeffect "; def $0", "=v"()
+  %v39 = call i32 asm sideeffect "; def $0", "=v"()
+  %v40 = call i32 asm sideeffect "; def $0", "=v"()
+  %v41 = call i32 asm sideeffect "; def $0", "=v"()
+  %v42 = call i32 asm sideeffect "; def $0", "=v"()
+  %v43 = call i32 asm sideeffect "; def $0", "=v"()
+  %v44 = call i32 asm sideeffect "; def $0", "=v"()
+  %v45 = call i32 asm sideeffect "; def $0", "=v"()
+  %v46 = call i32 asm sideeffect "; def $0", "=v"()
+  %v47 = call i32 asm sideeffect "; def $0", "=v"()
+  %v48 = call i32 asm sideeffect "; def $0", "=v"()
+  %v49 = call i32 asm sideeffect "; def $0", "=v"()
+  %v50 = call i32 asm sideeffect "; def $0", "=v"()
+  %v51 = call i32 asm sideeffect "; def $0", "=v"()
+  %v52 = call i32 asm sideeffect "; def $0", "=v"()
+  %v53 = call i32 asm sideeffect "; def $0", "=v"()
+  %v54 = call i32 asm sideeffect "; def $0", "=v"()
+  %v55 = call i32 asm sideeffect "; def $0", "=v"()
+  %v56 = call i32 asm sideeffect "; def $0", "=v"()
+  %v57 = call i32 asm sideeffect "; def $0", "=v"()
+  %v58 = call i32 asm sideeffect "; def $0", "=v"()
+  %v59 = call i32 asm sideeffect "; def $0", "=v"()
+  %v60 = call i32 asm sideeffect "; def $0", "=v"()
+  %v61 = call i32 asm sideeffect "; def $0", "=v"()
+  %v62 = call i32 asm sideeffect "; def $0", "=v"()
+  %v63 = call i32 asm sideeffect "; def $0", "=v"()
+  %v64 = call i32 asm sideeffect "; def $0", "=v"()
+  %v65 = call i32 asm sideeffect "; def $0", "=v"()
+  %v66 = call i32 asm sideeffect "; def $0", "=v"()
+  %v67 = call i32 asm sideeffect "; def $0", "=v"()
+  %v68 = call i32 asm sideeffect "; def $0", "=v"()
+  %v69 = call i32 asm sideeffect "; def $0", "=v"()
+  %v70 = call i32 asm sideeffect "; def $0", "=v"()
+  %v71 = call i32 asm sideeffect "; def $0", "=v"()
+  %v72 = call i32 asm sideeffect "; def $0", "=v"()
+  %v73 = call i32 asm sideeffect "; def $0", "=v"()
+  %v74 = call i32 asm sideeffect "; def $0", "=v"()
+  %v75 = call i32 asm sideeffect "; def $0", "=v"()
+  %v76 = call i32 asm sideeffect "; def $0", "=v"()
+  %v77 = call i32 asm sideeffect "; def $0", "=v"()
+  %v78 = call i32 asm sideeffect "; def $0", "=v"()
+  %v79 = call i32 asm sideeffect "; def $0", "=v"()
+  %v80 = call i32 asm sideeffect "; def $0", "=v"()
+  %v81 = call i32 asm sideeffect "; def $0", "=v"()
+  %v82 = call i32 asm sideeffect "; def $0", "=v"()
+  %v83 = call i32 asm sideeffect "; def $0", "=v"()
+  %v84 = call i32 asm sideeffect "; def $0", "=v"()
+  %v85 = call i32 asm sideeffect "; def $0", "=v"()
+  %v86 = call i32 asm sideeffect "; def $0", "=v"()
+  %v87 = call i32 asm sideeffect "; def $0", "=v"()
+  %v88 = call i32 asm sideeffect "; def $0", "=v"()
+  %v89 = call i32 asm sideeffect "; def $0", "=v"()
+  %v90 = call i32 asm sideeffect "; def $0", "=v"()
+  %v91 = call i32 asm sideeffect "; def $0", "=v"()
+  %v92 = call i32 asm sideeffect "; def $0", "=v"()
+  %v93 = call i32 asm sideeffect "; def $0", "=v"()
+  %v94 = call i32 asm sideeffect "; def $0", "=v"()
+  %v95 = call i32 asm sideeffect "; def $0", "=v"()
+  %v96 = call i32 asm sideeffect "; def $0", "=v"()
+  %v97 = call i32 asm sideeffect "; def $0", "=v"()
+  %v98 = call i32 asm sideeffect "; def $0", "=v"()
+  %v99 = call i32 asm sideeffect "; def $0", "=v"()
+  %v100 = call i32 asm sideeffect "; def $0", "=v"()
+  %v101 = call i32 asm sideeffect "; def $0", "=v"()
+  %v102 = call i32 asm sideeffect "; def $0", "=v"()
+  %v103 = call i32 asm sideeffect "; def $0", "=v"()
+  %v104 = call i32 asm sideeffect "; def $0", "=v"()
+  %v105 = call i32 asm sideeffect "; def $0", "=v"()
+  %v106 = call i32 asm sideeffect "; def $0", "=v"()
+  %v107 = call i32 asm sideeffect "; def $0", "=v"()
+  %v108 = call i32 asm sideeffect "; def $0", "=v"()
+  %v109 = call i32 asm sideeffect "; def $0", "=v"()
+  %v110 = call i32 asm sideeffect "; def $0", "=v"()
+  %v111 = call i32 asm sideeffect "; def $0", "=v"()
+  %v112 = call i32 asm sideeffect "; def $0", "=v"()
+  %v113 = call i32 asm sideeffect "; def $0", "=v"()
+  %v114 = call i32 asm sideeffect "; def $0", "=v"()
+  %v115 = call i32 asm sideeffect "; def $0", "=v"()
+  %v116 = call i32 asm sideeffect "; def $0", "=v"()
+  %v117 = call i32 asm sideeffect "; def $0", "=v"()
+  %v118 = call i32 asm sideeffect "; def $0", "=v"()
+  %v119 = call i32 asm sideeffect "; def $0", "=v"()
+  %v120 = call i32 asm sideeffect "; def $0", "=v"()
+  %v121 = call i32 asm sideeffect "; def $0", "=v"()
+  %v122 = call i32 asm sideeffect "; def $0", "=v"()
+  %v123 = call i32 asm sideeffect "; def $0", "=v"()
+  %v124 = call i32 asm sideeffect "; def $0", "=v"()
+  %v125 = call i32 asm sideeffect "; def $0", "=v"()
+  %v126 = call i32 asm sideeffect "; def $0", "=v"()
+  %v127 = call i32 asm sideeffect "; def $0", "=v"()
+  %v128 = call i32 asm sideeffect "; def $0", "=v"()
+  %v129 = call i32 asm sideeffect "; def $0", "=v"()
+  %v130 = call i32 asm sideeffect "; def $0", "=v"()
+  %v131 = call i32 asm sideeffect "; def $0", "=v"()
+  %v132 = call i32 asm sideeffect "; def $0", "=v"()
+  %v133 = call i32 asm sideeffect "; def $0", "=v"()
+  %v134 = call i32 asm sideeffect "; def $0", "=v"()
+  %v135 = call i32 asm sideeffect "; def $0", "=v"()
+  %v136 = call i32 asm sideeffect "; def $0", "=v"()
+  %v137 = call i32 asm sideeffect "; def $0", "=v"()
+  %v138 = call i32 asm sideeffect "; def $0", "=v"()
+  %v139 = call i32 asm sideeffect "; def $0", "=v"()
+  %v140 = call i32 asm sideeffect "; def $0", "=v"()
+  %v141 = call i32 asm sideeffect "; def $0", "=v"()
+  %v142 = call i32 asm sideeffect "; def $0", "=v"()
+  %v143 = call i32 asm sideeffect "; def $0", "=v"()
+  %v144 = call i32 asm sideeffect "; def $0", "=v"()
+  %v145 = call i32 asm sideeffect "; def $0", "=v"()
+  %v146 = call i32 asm sideeffect "; def $0", "=v"()
+  %v147 = call i32 asm sideeffect "; def $0", "=v"()
+  %v148 = call i32 asm sideeffect "; def $0", "=v"()
+  %v149 = call i32 asm sideeffect "; def $0", "=v"()
+  %v150 = call i32 asm sideeffect "; def $0", "=v"()
+  %v151 = call i32 asm sideeffect "; def $0", "=v"()
+  %v152 = call i32 asm sideeffect "; def $0", "=v"()
+  %v153 = call i32 asm sideeffect "; def $0", "=v"()
+  %v154 = call i32 asm sideeffect "; def $0", "=v"()
+  %v155 = call i32 asm sideeffect "; def $0", "=v"()
+  %v156 = call i32 asm sideeffect "; def $0", "=v"()
+  %v157 = call i32 asm sideeffect "; def $0", "=v"()
+  %v158 = call i32 asm sideeffect "; def $0", "=v"()
+  %v159 = call i32 asm sideeffect "; def $0", "=v"()
+  %v160 = call i32 asm sideeffect "; def $0", "=v"()
+  %v161 = call i32 asm sideeffect "; def $0", "=v"()
+  %v162 = call i32 asm sideeffect "; def $0", "=v"()
+  %v163 = call i32 asm sideeffect "; def $0", "=v"()
+  %v164 = call i32 asm sideeffect "; def $0", "=v"()
+  %v165 = call i32 asm sideeffect "; def $0", "=v"()
+  %v166 = call i32 asm sideeffect "; def $0", "=v"()
+  %v167 = call i32 asm sideeffect "; def $0", "=v"()
+  %v168 = call i32 asm sideeffect "; def $0", "=v"()
+  %v169 = call i32 asm sideeffect "; def $0", "=v"()
+  %v170 = call i32 asm sideeffect "; def $0", "=v"()
+  %v171 = call i32 asm sideeffect "; def $0", "=v"()
+  %v172 = call i32 asm sideeffect "; def $0", "=v"()
+  %v173 = call i32 asm sideeffect "; def $0", "=v"()
+  %v174 = call i32 asm sideeffect "; def $0", "=v"()
+  %v175 = call i32 asm sideeffect "; def $0", "=v"()
+  %v176 = call i32 asm sideeffect "; def $0", "=v"()
+  %v177 = call i32 asm sideeffect "; def $0", "=v"()
+  %v178 = call i32 asm sideeffect "; def $0", "=v"()
+  %v179 = call i32 asm sideeffect "; def $0", "=v"()
+  %v180 = call i32 asm sideeffect "; def $0", "=v"()
+  %v181 = call i32 asm sideeffect "; def $0", "=v"()
+  %v182 = call i32 asm sideeffect "; def $0", "=v"()
+  %v183 = call i32 asm sideeffect "; def $0", "=v"()
+  %v184 = call i32 asm sideeffect "; def $0", "=v"()
+  %v185 = call i32 asm sideeffect "; def $0", "=v"()
+  %v186 = call i32 asm sideeffect "; def $0", "=v"()
+  %v187 = call i32 asm sideeffect "; def $0", "=v"()
+  %v188 = call i32 asm sideeffect "; def $0", "=v"()
+  %v189 = call i32 asm sideeffect "; def $0", "=v"()
+  %v190 = call i32 asm sideeffect "; def $0", "=v"()
+  %v191 = call i32 asm sideeffect "; def $0", "=v"()
+  %v192 = call i32 asm sideeffect "; def $0", "=v"()
+  %v193 = call i32 asm sideeffect "; def $0", "=v"()
+  %v194 = call i32 asm sideeffect "; def $0", "=v"()
+  %v195 = call i32 asm sideeffect "; def $0", "=v"()
+  %v196 = call i32 asm sideeffect "; def $0", "=v"()
+  %v197 = call i32 asm sideeffect "; def $0", "=v"()
+  %v198 = call i32 asm sideeffect "; def $0", "=v"()
+  %v199 = call i32 asm sideeffect "; def $0", "=v"()
+  %v200 = call i32 asm sideeffect "; def $0", "=v"()
+  %v201 = call i32 asm sideeffect "; def $0", "=v"()
+  %v202 = call i32 asm sideeffect "; def $0", "=v"()
+  %v203 = call i32 asm sideeffect "; def $0", "=v"()
+  %v204 = call i32 asm sideeffect "; def $0", "=v"()
+  %v205 = call i32 asm sideeffect "; def $0", "=v"()
+  %v206 = call i32 asm sideeffect "; def $0", "=v"()
+  %v207 = call i32 asm sideeffect "; def $0", "=v"()
+  %v208 = call i32 asm sideeffect "; def $0", "=v"()
+  %v209 = call i32 asm sideeffect "; def $0", "=v"()
+  %v210 = call i32 asm sideeffect "; def $0", "=v"()
+  %v211 = call i32 asm sideeffect "; def $0", "=v"()
+  %v212 = call i32 asm sideeffect "; def $0", "=v"()
+  %v213 = call i32 asm sideeffect "; def $0", "=v"()
+  %v214 = call i32 asm sideeffect "; def $0", "=v"()
+  %v215 = call i32 asm sideeffect "; def $0", "=v"()
+  %v216 = call i32 asm sideeffect "; def $0", "=v"()
+  %v217 = call i32 asm sideeffect "; def $0", "=v"()
+  %v218 = call i32 asm sideeffect "; def $0", "=v"()
+  %v219 = call i32 asm sideeffect "; def $0", "=v"()
+  %v220 = call i32 asm sideeffect "; def $0", "=v"()
+  %v221 = call i32 asm sideeffect "; def $0", "=v"()
+  %v222 = call i32 asm sideeffect "; def $0", "=v"()
+  %v223 = call i32 asm sideeffect "; def $0", "=v"()
+  %v224 = call i32 asm sideeffect "; def $0", "=v"()
+  %v225 = call i32 asm sideeffect "; def $0", "=v"()
+  %v226 = call i32 asm sideeffect "; def $0", "=v"()
+  %v227 = call i32 asm sideeffect "; def $0", "=v"()
+  %v228 = call i32 asm sideeffect "; def $0", "=v"()
+  %v229 = call i32 asm sideeffect "; def $0", "=v"()
+  %v230 = call i32 asm sideeffect "; def $0", "=v"()
+  %v231 = call i32 asm sideeffect "; def $0", "=v"()
+  %v232 = call i32 asm sideeffect "; def $0", "=v"()
+  %v233 = call i32 asm sideeffect "; def $0", "=v"()
+  %v234 = call i32 asm sideeffect "; def $0", "=v"()
+  %v235 = call i32 asm sideeffect "; def $0", "=v"()
+  %v236 = call i32 asm sideeffect "; def $0", "=v"()
+  %v237 = call i32 asm sideeffect "; def $0", "=v"()
+  %v238 = call i32 asm sideeffect "; def $0", "=v"()
+  %v239 = call i32 asm sideeffect "; def $0", "=v"()
+  %v240 = call i32 asm sideeffect "; def $0", "=v"()
+  %v241 = call i32 asm sideeffect "; def $0", "=v"()
+  %v242 = call i32 asm sideeffect "; def $0", "=v"()
+  %v243 = call i32 asm sideeffect "; def $0", "=v"()
+  %v244 = call i32 asm sideeffect "; def $0", "=v"()
+  %v245 = call i32 asm sideeffect "; def $0", "=v"()
+  %v246 = call i32 asm sideeffect "; def $0", "=v"()
+  %v247 = call i32 asm sideeffect "; def $0", "=v"()
+  %v248 = call i32 asm sideeffect "; def $0", "=v"()
+  %v249 = call i32 asm sideeffect "; def $0", "=v"()
+  %v250 = call i32 asm sideeffect "; def $0", "=v"()
+  %v251 = call i32 asm sideeffect "; def $0", "=v"()
+  %v252 = call i32 asm sideeffect "; def $0", "=v"()
+  %v253 = call i32 asm sideeffect "; def $0", "=v"()
+  %v254 = call i32 asm sideeffect "; def $0", "=v"()
+  %v255 = call i32 asm sideeffect "; def $0", "=v"()
+  call void asm sideeffect "; use $0", "v"(i32 %v0)
+  call void asm sideeffect "; use $0", "v"(i32 %v1)
+  call void asm sideeffect "; use $0", "v"(i32 %v2)
+  call void asm sideeffect "; use $0", "v"(i32 %v3)
+  call void asm sideeffect "; use $0", "v"(i32 %v4)
+  call void asm sideeffect "; use $0", "v"(i32 %v5)
+  call void asm sideeffect "; use $0", "v"(i32 %v6)
+  call void asm sideeffect "; use $0", "v"(i32 %v7)
+  call void asm sideeffect "; use $0", "v"(i32 %v8)
+  call void asm sideeffect "; use $0", "v"(i32 %v9)
+  call void asm sideeffect "; use $0", "v"(i32 %v10)
+  call void asm sideeffect "; use $0", "v"(i32 %v11)
+  call void asm sideeffect "; use $0", "v"(i32 %v12)
+  call void asm sideeffect "; use $0", "v"(i32 %v13)
+  call void asm sideeffect "; use $0", "v"(i32 %v14)
+  call void asm sideeffect "; use $0", "v"(i32 %v15)
+  call void asm sideeffect "; use $0", "v"(i32 %v16)
+  call void asm sideeffect "; use $0", "v"(i32 %v17)
+  call void asm sideeffect "; use $0", "v"(i32 %v18)
+  call void asm sideeffect "; use $0", "v"(i32 %v19)
+  call void asm sideeffect "; use $0", "v"(i32 %v20)
+  call void asm sideeffect "; use $0", "v"(i32 %v21)
+  call void asm sideeffect "; use $0", "v"(i32 %v22)
+  call void asm sideeffect "; use $0", "v"(i32 %v23)
+  call void asm sideeffect "; use $0", "v"(i32 %v24)
+  call void asm sideeffect "; use $0", "v"(i32 %v25)
+  call void asm sideeffect "; use $0", "v"(i32 %v26)
+  call void asm sideeffect "; use $0", "v"(i32 %v27)
+  call void asm sideeffect "; use $0", "v"(i32 %v28)
+  call void asm sideeffect "; use $0", "v"(i32 %v29)
+  call void asm sideeffect "; use $0", "v"(i32 %v30)
+  call void asm sideeffect "; use $0", "v"(i32 %v31)
+  call void asm sideeffect "; use $0", "v"(i32 %v32)
+  call void asm sideeffect "; use $0", "v"(i32 %v33)
+  call void asm sideeffect "; use $0", "v"(i32 %v34)
+  call void asm sideeffect "; use $0", "v"(i32 %v35)
+  call void asm sideeffect "; use $0", "v"(i32 %v36)
+  call void asm sideeffect "; use $0", "v"(i32 %v37)
+  call void asm sideeffect "; use $0", "v"(i32 %v38)
+  call void asm sideeffect "; use $0", "v"(i32 %v39)
+  call void asm sideeffect "; use $0", "v"(i32 %v40)
+  call void asm sideeffect "; use $0", "v"(i32 %v41)
+  call void asm sideeffect "; use $0", "v"(i32 %v42)
+  call void asm sideeffect "; use $0", "v"(i32 %v43)
+  call void asm sideeffect "; use $0", "v"(i32 %v44)
+  call void asm sideeffect "; use $0", "v"(i32 %v45)
+  call void asm sideeffect "; use $0", "v"(i32 %v46)
+  call void asm sideeffect "; use $0", "v"(i32 %v47)
+  call void asm sideeffect "; use $0", "v"(i32 %v48)
+  call void asm sideeffect "; use $0", "v"(i32 %v49)
+  call void asm sideeffect "; use $0", "v"(i32 %v50)
+  call void asm sideeffect "; use $0", "v"(i32 %v51)
+  call void asm sideeffect "; use $0", "v"(i32 %v52)
+  call void asm sideeffect "; use $0", "v"(i32 %v53)
+  call void asm sideeffect "; use $0", "v"(i32 %v54)
+  call void asm sideeffect "; use $0", "v"(i32 %v55)
+  call void asm sideeffect "; use $0", "v"(i32 %v56)
+  call void asm sideeffect "; use $0", "v"(i32 %v57)
+  call void asm sideeffect "; use $0", "v"(i32 %v58)
+  call void asm sideeffect "; use $0", "v"(i32 %v59)
+  call void asm sideeffect "; use $0", "v"(i32 %v60)
+  call void asm sideeffect "; use $0", "v"(i32 %v61)
+  call void asm sideeffect "; use $0", "v"(i32 %v62)
+  call void asm sideeffect "; use $0", "v"(i32 %v63)
+  call void asm sideeffect "; use $0", "v"(i32 %v64)
+  call void asm sideeffect "; use $0", "v"(i32 %v65)
+  call void asm sideeffect "; use $0", "v"(i32 %v66)
+  call void asm sideeffect "; use $0", "v"(i32 %v67)
+  call void asm sideeffect "; use $0", "v"(i32 %v68)
+  call void asm sideeffect "; use $0", "v"(i32 %v69)
+  call void asm sideeffect "; use $0", "v"(i32 %v70)
+  call void asm sideeffect "; use $0", "v"(i32 %v71)
+  call void asm sideeffect "; use $0", "v"(i32 %v72)
+  call void asm sideeffect "; use $0", "v"(i32 %v73)
+  call void asm sideeffect "; use $0", "v"(i32 %v74)
+  call void asm sideeffect "; use $0", "v"(i32 %v75)
+  call void asm sideeffect "; use $0", "v"(i32 %v76)
+  call void asm sideeffect "; use $0", "v"(i32 %v77)
+  call void asm sideeffect "; use $0", "v"(i32 %v78)
+  call void asm sideeffect "; use $0", "v"(i32 %v79)
+  call void asm sideeffect "; use $0", "v"(i32 %v80)
+  call void asm sideeffect "; use $0", "v"(i32 %v81)
+  call void asm sideeffect "; use $0", "v"(i32 %v82)
+  call void asm sideeffect "; use $0", "v"(i32 %v83)
+  call void asm sideeffect "; use $0", "v"(i32 %v84)
+  call void asm sideeffect "; use $0", "v"(i32 %v85)
+  call void asm sideeffect "; use $0", "v"(i32 %v86)
+  call void asm sideeffect "; use $0", "v"(i32 %v87)
+  call void asm sideeffect "; use $0", "v"(i32 %v88)
+  call void asm sideeffect "; use $0", "v"(i32 %v89)
+  call void asm sideeffect "; use $0", "v"(i32 %v90)
+  call void asm sideeffect "; use $0", "v"(i32 %v91)
+  call void asm sideeffect "; use $0", "v"(i32 %v92)
+  call void asm sideeffect "; use $0", "v"(i32 %v93)
+  call void asm sideeffect "; use $0", "v"(i32 %v94)
+  call void asm sideeffect "; use $0", "v"(i32 %v95)
+  call void asm sideeffect "; use $0", "v"(i32 %v96)
+  call void asm sideeffect "; use $0", "v"(i32 %v97)
+  call void asm sideeffect "; use $0", "v"(i32 %v98)
+  call void asm sideeffect "; use $0", "v"(i32 %v99)
+  call void asm sideeffect "; use $0", "v"(i32 %v100)
+  call void asm sideeffect "; use $0", "v"(i32 %v101)
+  call void asm sideeffect "; use $0", "v"(i32 %v102)
+  call void asm sideeffect "; use $0", "v"(i32 %v103)
+  call void asm sideeffect "; use $0", "v"(i32 %v104)
+  call void asm sideeffect "; use $0", "v"(i32 %v105)
+  call void asm sideeffect "; use $0", "v"(i32 %v106)
+  call void asm sideeffect "; use $0", "v"(i32 %v107)
+  call void asm sideeffect "; use $0", "v"(i32 %v108)
+  call void asm sideeffect "; use $0", "v"(i32 %v109)
+  call void asm sideeffect "; use $0", "v"(i32 %v110)
+  call void asm sideeffect "; use $0", "v"(i32 %v111)
+  call void asm sideeffect "; use $0", "v"(i32 %v112)
+  call void asm sideeffect "; use $0", "v"(i32 %v113)
+  call void asm sideeffect "; use $0", "v"(i32 %v114)
+  call void asm sideeffect "; use $0", "v"(i32 %v115)
+  call void asm sideeffect "; use $0", "v"(i32 %v116)
+  call void asm sideeffect "; use $0", "v"(i32 %v117)
+  call void asm sideeffect "; use $0", "v"(i32 %v118)
+  call void asm sideeffect "; use $0", "v"(i32 %v119)
+  call void asm sideeffect "; use $0", "v"(i32 %v120)
+  call void asm sideeffect "; use $0", "v"(i32 %v121)
+  call void asm sideeffect "; use $0", "v"(i32 %v122)
+  call void asm sideeffect "; use $0", "v"(i32 %v123)
+  call void asm sideeffect "; use $0", "v"(i32 %v124)
+  call void asm sideeffect "; use $0", "v"(i32 %v125)
+  call void asm sideeffect "; use $0", "v"(i32 %v126)
+  call void asm sideeffect "; use $0", "v"(i32 %v127)
+  call void asm sideeffect "; use $0", "v"(i32 %v128)
+  call void asm sideeffect "; use $0", "v"(i32 %v129)
+  call void asm sideeffect "; use $0", "v"(i32 %v130)
+  call void asm sideeffect "; use $0", "v"(i32 %v131)
+  call void asm sideeffect "; use $0", "v"(i32 %v132)
+  call void asm sideeffect "; use $0", "v"(i32 %v133)
+  call void asm sideeffect "; use $0", "v"(i32 %v134)
+  call void asm sideeffect "; use $0", "v"(i32 %v135)
+  call void asm sideeffect "; use $0", "v"(i32 %v136)
+  call void asm sideeffect "; use $0", "v"(i32 %v137)
+  call void asm sideeffect "; use $0", "v"(i32 %v138)
+  call void asm sideeffect "; use $0", "v"(i32 %v139)
+  call void asm sideeffect "; use $0", "v"(i32 %v140)
+  call void asm sideeffect "; use $0", "v"(i32 %v141)
+  call void asm sideeffect "; use $0", "v"(i32 %v142)
+  call void asm sideeffect "; use $0", "v"(i32 %v143)
+  call void asm sideeffect "; use $0", "v"(i32 %v144)
+  call void asm sideeffect "; use $0", "v"(i32 %v145)
+  call void asm sideeffect "; use $0", "v"(i32 %v146)
+  call void asm sideeffect "; use $0", "v"(i32 %v147)
+  call void asm sideeffect "; use $0", "v"(i32 %v148)
+  call void asm sideeffect "; use $0", "v"(i32 %v149)
+  call void asm sideeffect "; use $0", "v"(i32 %v150)
+  call void asm sideeffect "; use $0", "v"(i32 %v151)
+  call void asm sideeffect "; use $0", "v"(i32 %v152)
+  call void asm sideeffect "; use $0", "v"(i32 %v153)
+  call void asm sideeffect "; use $0", "v"(i32 %v154)
+  call void asm sideeffect "; use $0", "v"(i32 %v155)
+  call void asm sideeffect "; use $0", "v"(i32 %v156)
+  call void asm sideeffect "; use $0", "v"(i32 %v157)
+  call void asm sideeffect "; use $0", "v"(i32 %v158)
+  call void asm sideeffect "; use $0", "v"(i32 %v159)
+  call void asm sideeffect "; use $0", "v"(i32 %v160)
+  call void asm sideeffect "; use $0", "v"(i32 %v161)
+  call void asm sideeffect "; use $0", "v"(i32 %v162)
+  call void asm sideeffect "; use $0", "v"(i32 %v163)
+  call void asm sideeffect "; use $0", "v"(i32 %v164)
+  call void asm sideeffect "; use $0", "v"(i32 %v165)
+  call void asm sideeffect "; use $0", "v"(i32 %v166)
+  call void asm sideeffect "; use $0", "v"(i32 %v167)
+  call void asm sideeffect "; use $0", "v"(i32 %v168)
+  call void asm sideeffect "; use $0", "v"(i32 %v169)
+  call void asm sideeffect "; use $0", "v"(i32 %v170)
+  call void asm sideeffect "; use $0", "v"(i32 %v171)
+  call void asm sideeffect "; use $0", "v"(i32 %v172)
+  call void asm sideeffect "; use $0", "v"(i32 %v173)
+  call void asm sideeffect "; use $0", "v"(i32 %v174)
+  call void asm sideeffect "; use $0", "v"(i32 %v175)
+  call void asm sideeffect "; use $0", "v"(i32 %v176)
+  call void asm sideeffect "; use $0", "v"(i32 %v177)
+  call void asm sideeffect "; use $0", "v"(i32 %v178)
+  call void asm sideeffect "; use $0", "v"(i32 %v179)
+  call void asm sideeffect "; use $0", "v"(i32 %v180)
+  call void asm sideeffect "; use $0", "v"(i32 %v181)
+  call void asm sideeffect "; use $0", "v"(i32 %v182)
+  call void asm sideeffect "; use $0", "v"(i32 %v183)
+  call void asm sideeffect "; use $0", "v"(i32 %v184)
+  call void asm sideeffect "; use $0", "v"(i32 %v185)
+  call void asm sideeffect "; use $0", "v"(i32 %v186)
+  call void asm sideeffect "; use $0", "v"(i32 %v187)
+  call void asm sideeffect "; use $0", "v"(i32 %v188)
+  call void asm sideeffect "; use $0", "v"(i32 %v189)
+  call void asm sideeffect "; use $0", "v"(i32 %v190)
+  call void asm sideeffect "; use $0", "v"(i32 %v191)
+  call void asm sideeffect "; use $0", "v"(i32 %v192)
+  call void asm sideeffect "; use $0", "v"(i32 %v193)
+  call void asm sideeffect "; use $0", "v"(i32 %v194)
+  call void asm sideeffect "; use $0", "v"(i32 %v195)
+  call void asm sideeffect "; use $0", "v"(i32 %v196)
+  call void asm sideeffect "; use $0", "v"(i32 %v197)
+  call void asm sideeffect "; use $0", "v"(i32 %v198)
+  call void asm sideeffect "; use $0", "v"(i32 %v199)
+  call void asm sideeffect "; use $0", "v"(i32 %v200)
+  call void asm sideeffect "; use $0", "v"(i32 %v201)
+  call void asm sideeffect "; use $0", "v"(i32 %v202)
+  call void asm sideeffect "; use $0", "v"(i32 %v203)
+  call void asm sideeffect "; use $0", "v"(i32 %v204)
+  call void asm sideeffect "; use $0", "v"(i32 %v205)
+  call void asm sideeffect "; use $0", "v"(i32 %v206)
+  call void asm sideeffect "; use $0", "v"(i32 %v207)
+  call void asm sideeffect "; use $0", "v"(i32 %v208)
+  call void asm sideeffect "; use $0", "v"(i32 %v209)
+  call void asm sideeffect "; use $0", "v"(i32 %v210)
+  call void asm sideeffect "; use $0", "v"(i32 %v211)
+  call void asm sideeffect "; use $0", "v"(i32 %v212)
+  call void asm sideeffect "; use $0", "v"(i32 %v213)
+  call void asm sideeffect "; use $0", "v"(i32 %v214)
+  call void asm sideeffect "; use $0", "v"(i32 %v215)
+  call void asm sideeffect "; use $0", "v"(i32 %v216)
+  call void asm sideeffect "; use $0", "v"(i32 %v217)
+  call void asm sideeffect "; use $0", "v"(i32 %v218)
+  call void asm sideeffect "; use $0", "v"(i32 %v219)
+  call void asm sideeffect "; use $0", "v"(i32 %v220)
+  call void asm sideeffect "; use $0", "v"(i32 %v221)
+  call void asm sideeffect "; use $0", "v"(i32 %v222)
+  call void asm sideeffect "; use $0", "v"(i32 %v223)
+  call void asm sideeffect "; use $0", "v"(i32 %v224)
+  call void asm sideeffect "; use $0", "v"(i32 %v225)
+  call void asm sideeffect "; use $0", "v"(i32 %v226)
+  call void asm sideeffect "; use $0", "v"(i32 %v227)
+  call void asm sideeffect "; use $0", "v"(i32 %v228)
+  call void asm sideeffect "; use $0", "v"(i32 %v229)
+  call void asm sideeffect "; use $0", "v"(i32 %v230)
+  call void asm sideeffect "; use $0", "v"(i32 %v231)
+  call void asm sideeffect "; use $0", "v"(i32 %v232)
+  call void asm sideeffect "; use $0", "v"(i32 %v233)
+  call void asm sideeffect "; use $0", "v"(i32 %v234)
+  call void asm sideeffect "; use $0", "v"(i32 %v235)
+  call void asm sideeffect "; use $0", "v"(i32 %v236)
+  call void asm sideeffect "; use $0", "v"(i32 %v237)
+  call void asm sideeffect "; use $0", "v"(i32 %v238)
+  call void asm sideeffect "; use $0", "v"(i32 %v239)
+  call void asm sideeffect "; use $0", "v"(i32 %v240)
+  call void asm sideeffect "; use $0", "v"(i32 %v241)
+  call void asm sideeffect "; use $0", "v"(i32 %v242)
+  call void asm sideeffect "; use $0", "v"(i32 %v243)
+  call void asm sideeffect "; use $0", "v"(i32 %v244)
+  call void asm sideeffect "; use $0", "v"(i32 %v245)
+  call void asm sideeffect "; use $0", "v"(i32 %v246)
+  call void asm sideeffect "; use $0", "v"(i32 %v247)
+  call void asm sideeffect "; use $0", "v"(i32 %v248)
+  call void asm sideeffect "; use $0", "v"(i32 %v249)
+  call void asm sideeffect "; use $0", "v"(i32 %v250)
+  call void asm sideeffect "; use $0", "v"(i32 %v251)
+  call void asm sideeffect "; use $0", "v"(i32 %v252)
+  call void asm sideeffect "; use $0", "v"(i32 %v253)
+  call void asm sideeffect "; use $0", "v"(i32 %v254)
+  call void asm sideeffect "; use $0", "v"(i32 %v255)
+  ret void
+}
+
+; GCN-LABEL: {{^}}f256:
+; GFX9: NumVgprs: 256
+; GFX10WGP-WAVE32: NumVgprs: 256
+; GFX10WGP-WAVE64: NumVgprs: 256
+; GFX10CU-WAVE32: NumVgprs: 256
+; GFX10CU-WAVE64: NumVgprs: 256
+define amdgpu_kernel void @f256() #256 {
+  call void @use256vgprs()
+  ret void
+}
+attributes #256 = { nounwind "amdgpu-flat-work-group-size"="256,256" }
+
+; GCN-LABEL: {{^}}f512:
+; GFX9: NumVgprs: 128
+; GFX10WGP-WAVE32: NumVgprs: 256
+; GFX10WGP-WAVE64: NumVgprs: 256
+; GFX10CU-WAVE32: NumVgprs: 128
+; GFX10CU-WAVE64: NumVgprs: 128
+define amdgpu_kernel void @f512() #512 {
+  call void @use256vgprs()
+  ret void
+}
+attributes #512 = { nounwind "amdgpu-flat-work-group-size"="512,512" }
+
+; GCN-LABEL: {{^}}f1024:
+; GFX9: NumVgprs: 64
+; GFX10WGP-WAVE32: NumVgprs: 128
+; GFX10WGP-WAVE64: NumVgprs: 128
+; GFX10CU-WAVE32: NumVgprs: 64
+; GFX10CU-WAVE64: NumVgprs: 64
+define amdgpu_kernel void @f1024() #1024 {
+  call void @use256vgprs()
+  ret void
+}
+attributes #1024 = { nounwind "amdgpu-flat-work-group-size"="1024,1024" }
diff --git a/llvm/test/CodeGen/AMDGPU/clamp-modifier.ll b/llvm/test/CodeGen/AMDGPU/clamp-modifier.ll
index 2bc5db986a975..5a56a1a264afa 100644
--- a/llvm/test/CodeGen/AMDGPU/clamp-modifier.ll
+++ b/llvm/test/CodeGen/AMDGPU/clamp-modifier.ll
@@ -389,10 +389,10 @@ declare <2 x half> @llvm.amdgcn.cvt.pkrtz(float, float) #1
 
 declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1
 
-attributes #0 = { nounwind }
+attributes #0 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { nounwind "target-features"="+fp32-denormals" }
-attributes #3 = { nounwind "target-features"="-fp64-fp16-denormals" }
+attributes #2 = { nounwind "denormal-fp-math-f32"="ieee.ieee" }
+attributes #3 = { nounwind "denormal-fp-math-f32"="ieee,ieee" "denormal-fp-math"="preserve-sign,preserve-sign" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!2, !3}
diff --git a/llvm/test/CodeGen/AMDGPU/clamp.ll b/llvm/test/CodeGen/AMDGPU/clamp.ll
index 1fcdfd2ff24d9..df48096a368fc 100644
--- a/llvm/test/CodeGen/AMDGPU/clamp.ll
+++ b/llvm/test/CodeGen/AMDGPU/clamp.ll
@@ -767,8 +767,8 @@ declare <2 x half> @llvm.fabs.v2f16(<2 x half>) #1
 declare <2 x half> @llvm.minnum.v2f16(<2 x half>, <2 x half>) #1
 declare <2 x half> @llvm.maxnum.v2f16(<2 x half>, <2 x half>) #1
 
-attributes #0 = { nounwind }
+attributes #0 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { nounwind "amdgpu-dx10-clamp"="false" "target-features"="-fp-exceptions" "no-nans-fp-math"="false" }
-attributes #3 = { nounwind "amdgpu-dx10-clamp"="true" "target-features"="+fp-exceptions" "no-nans-fp-math"="false" }
-attributes #4 = { nounwind "amdgpu-dx10-clamp"="false" "target-features"="+fp-exceptions" "no-nans-fp-math"="false" }
+attributes #2 = { nounwind "amdgpu-dx10-clamp"="false" "target-features"="-fp-exceptions" "denormal-fp-math-f32"="preserve-sign,preserve-sign" "no-nans-fp-math"="false" }
+attributes #3 = { nounwind "amdgpu-dx10-clamp"="true" "target-features"="+fp-exceptions" "denormal-fp-math-f32"="preserve-sign,preserve-sign" "no-nans-fp-math"="false" }
+attributes #4 = { nounwind "amdgpu-dx10-clamp"="false" "target-features"="+fp-exceptions" "denormal-fp-math-f32"="preserve-sign,preserve-sign" "no-nans-fp-math"="false" }
diff --git a/llvm/test/CodeGen/AMDGPU/concat_vectors.ll b/llvm/test/CodeGen/AMDGPU/concat_vectors.ll
index e50173b7caf83..4fe3008011a2b 100644
--- a/llvm/test/CodeGen/AMDGPU/concat_vectors.ll
+++ b/llvm/test/CodeGen/AMDGPU/concat_vectors.ll
@@ -309,8 +309,8 @@ define amdgpu_kernel void @concat_vector_crash2(<8 x i8> addrspace(1)* %out, i32
 
 ; GCN-LABEL: {{^}}build_vector_splat_concat_v8i16:
 ; VI: v_mov_b32_e32 v{{[0-9]+}}, 0{{$}}
-; VI: ds_write_b64
-; VI: ds_write2_b64
+; VI: ds_write_b128
+; VI: ds_write_b128
 define amdgpu_kernel void @build_vector_splat_concat_v8i16() {
 entry:
   store <8 x i16> zeroinitializer, <8 x i16> addrspace(3)* undef, align 16
diff --git a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
index 41a6b8c291a9a..c3f213b9c22f5 100644
--- a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
+++ b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
@@ -1,10 +1,386 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgcn-- -mcpu=tahiti -verify-machineinstrs < %s | FileCheck %s -allow-deprecated-dag-overlap -check-prefixes=GCN,SI
-; RUN: llc -mtriple=amdgcn-- -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck %s -allow-deprecated-dag-overlap -check-prefixes=GCN,VI
+; RUN: llc -mtriple=amdgcn-- -mcpu=tahiti -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GCN,SI
+; RUN: llc -mtriple=amdgcn-- -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GCN,VI
 
 declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
 declare i32 @llvm.amdgcn.workitem.id.y() nounwind readnone
 
+define float @v_uitofp_i32_to_f32_mask255(i32 %arg0) nounwind {
+; GCN-LABEL: v_uitofp_i32_to_f32_mask255:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %masked = and i32 %arg0, 255
+  %cvt = uitofp i32 %masked to float
+  ret float %cvt
+}
+
+define float @v_sitofp_i32_to_f32_mask255(i32 %arg0) nounwind {
+; GCN-LABEL: v_sitofp_i32_to_f32_mask255:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %masked = and i32 %arg0, 255
+  %cvt = sitofp i32 %masked to float
+  ret float %cvt
+}
+
+define float @v_uitofp_to_f32_lshr7_mask255(i32 %arg0) nounwind {
+; GCN-LABEL: v_uitofp_to_f32_lshr7_mask255:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v0, 7, v0
+; GCN-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %lshr.7 = lshr i32 %arg0, 7
+  %masked = and i32 %lshr.7, 255
+  %cvt = uitofp i32 %masked to float
+  ret float %cvt
+}
+
+define float @v_uitofp_to_f32_lshr8_mask255(i32 %arg0) nounwind {
+; GCN-LABEL: v_uitofp_to_f32_lshr8_mask255:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cvt_f32_ubyte1_e32 v0, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %lshr.8 = lshr i32 %arg0, 8
+  %masked = and i32 %lshr.8, 255
+  %cvt = uitofp i32 %masked to float
+  ret float %cvt
+}
+
+define float @v_uitofp_to_f32_multi_use_lshr8_mask255(i32 %arg0) nounwind {
+; SI-LABEL: v_uitofp_to_f32_multi_use_lshr8_mask255:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; SI-NEXT:    v_cvt_f32_ubyte1_e32 v0, v0
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    buffer_store_dword v1, off, s[4:7], 0
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: v_uitofp_to_f32_multi_use_lshr8_mask255:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; VI-NEXT:    v_cvt_f32_ubyte1_e32 v0, v0
+; VI-NEXT:    s_mov_b32 s7, 0xf000
+; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    buffer_store_dword v1, off, s[4:7], 0
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    s_setpc_b64 s[30:31]
+  %lshr.8 = lshr i32 %arg0, 8
+  store i32 %lshr.8, i32 addrspace(1)* undef
+  %masked = and i32 %lshr.8, 255
+  %cvt = uitofp i32 %masked to float
+  ret float %cvt
+}
+
+define float @v_uitofp_to_f32_lshr16_mask255(i32 %arg0) nounwind {
+; GCN-LABEL: v_uitofp_to_f32_lshr16_mask255:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cvt_f32_ubyte2_e32 v0, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %lshr.16 = lshr i32 %arg0, 16
+  %masked = and i32 %lshr.16, 255
+  %cvt = uitofp i32 %masked to float
+  ret float %cvt
+}
+
+define float @v_uitofp_to_f32_lshr24_mask255(i32 %arg0) nounwind {
+; GCN-LABEL: v_uitofp_to_f32_lshr24_mask255:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cvt_f32_ubyte3_e32 v0, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %lshr.16 = lshr i32 %arg0, 24
+  %masked = and i32 %lshr.16, 255
+  %cvt = uitofp i32 %masked to float
+  ret float %cvt
+}
+
+define float @v_uitofp_i8_to_f32(i8 %arg0) nounwind {
+; GCN-LABEL: v_uitofp_i8_to_f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %cvt = uitofp i8 %arg0 to float
+  ret float %cvt
+}
+
+define <2 x float> @v_uitofp_v2i8_to_v2f32(i16 %arg0) nounwind {
+; GCN-LABEL: v_uitofp_v2i8_to_v2f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cvt_f32_ubyte0_e32 v2, v0
+; GCN-NEXT:    v_cvt_f32_ubyte1_e32 v1, v0
+; GCN-NEXT:    v_mov_b32_e32 v0, v2
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %val = bitcast i16 %arg0 to <2 x i8>
+  %cvt = uitofp <2 x i8> %val to <2 x float>
+  ret <2 x float> %cvt
+}
+
+define <3 x float> @v_uitofp_v3i8_to_v3f32(i32 %arg0) nounwind {
+; GCN-LABEL: v_uitofp_v3i8_to_v3f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cvt_f32_ubyte0_e32 v3, v0
+; GCN-NEXT:    v_cvt_f32_ubyte1_e32 v1, v0
+; GCN-NEXT:    v_cvt_f32_ubyte2_e32 v2, v0
+; GCN-NEXT:    v_mov_b32_e32 v0, v3
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %trunc = trunc i32 %arg0 to i24
+  %val = bitcast i24 %trunc to <3 x i8>
+  %cvt = uitofp <3 x i8> %val to <3 x float>
+  ret <3 x float> %cvt
+}
+
+define <4 x float> @v_uitofp_v4i8_to_v4f32(i32 %arg0) nounwind {
+; GCN-LABEL: v_uitofp_v4i8_to_v4f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cvt_f32_ubyte0_e32 v4, v0
+; GCN-NEXT:    v_cvt_f32_ubyte1_e32 v1, v0
+; GCN-NEXT:    v_cvt_f32_ubyte2_e32 v2, v0
+; GCN-NEXT:    v_cvt_f32_ubyte3_e32 v3, v0
+; GCN-NEXT:    v_mov_b32_e32 v0, v4
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %val = bitcast i32 %arg0 to <4 x i8>
+  %cvt = uitofp <4 x i8> %val to <4 x float>
+  ret <4 x float> %cvt
+}
+
+define <4 x float> @v_uitofp_unpack_i32_to_v4f32(i32 %arg0) nounwind {
+; GCN-LABEL: v_uitofp_unpack_i32_to_v4f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cvt_f32_ubyte0_e32 v4, v0
+; GCN-NEXT:    v_cvt_f32_ubyte1_e32 v1, v0
+; GCN-NEXT:    v_cvt_f32_ubyte2_e32 v2, v0
+; GCN-NEXT:    v_cvt_f32_ubyte3_e32 v3, v0
+; GCN-NEXT:    v_mov_b32_e32 v0, v4
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %mask.arg0 = and i32 %arg0, 255
+  %cvt0 = uitofp i32 %mask.arg0 to float
+
+  %lshr.8 = lshr i32 %arg0, 8
+  %mask.lshr.8 = and i32 %lshr.8, 255
+  %cvt1 = uitofp i32 %mask.lshr.8 to float
+
+  %lshr.16 = lshr i32 %arg0, 16
+  %mask.lshr.16 = and i32 %lshr.16, 255
+  %cvt2 = uitofp i32 %mask.lshr.16 to float
+
+  %lshr.24 = lshr i32 %arg0, 24
+  %mask.lshr.24 = and i32 %lshr.24, 255
+  %cvt3 = uitofp i32 %mask.lshr.24 to float
+
+  %ins.0 = insertelement <4 x float> undef, float %cvt0, i32 0
+  %ins.1 = insertelement <4 x float> %ins.0, float %cvt1, i32 1
+  %ins.2 = insertelement <4 x float> %ins.1, float %cvt2, i32 2
+  %ins.3 = insertelement <4 x float> %ins.2, float %cvt3, i32 3
+  ret <4 x float> %ins.3
+}
+
+define half @v_uitofp_i32_to_f16_mask255(i32 %arg0) nounwind {
+; SI-LABEL: v_uitofp_i32_to_f16_mask255:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
+; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: v_uitofp_i32_to_f16_mask255:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
+; VI-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; VI-NEXT:    s_setpc_b64 s[30:31]
+  %masked = and i32 %arg0, 255
+  %cvt = uitofp i32 %masked to half
+  ret half %cvt
+}
+
+define half @v_sitofp_i32_to_f16_mask255(i32 %arg0) nounwind {
+; SI-LABEL: v_sitofp_i32_to_f16_mask255:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
+; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: v_sitofp_i32_to_f16_mask255:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
+; VI-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; VI-NEXT:    s_setpc_b64 s[30:31]
+  %masked = and i32 %arg0, 255
+  %cvt = sitofp i32 %masked to half
+  ret half %cvt
+}
+
+define half @v_uitofp_to_f16_lshr8_mask255(i32 %arg0) nounwind {
+; SI-LABEL: v_uitofp_to_f16_lshr8_mask255:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_cvt_f32_ubyte1_e32 v0, v0
+; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: v_uitofp_to_f16_lshr8_mask255:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cvt_f32_ubyte1_e32 v0, v0
+; VI-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; VI-NEXT:    s_setpc_b64 s[30:31]
+  %lshr.8 = lshr i32 %arg0, 8
+  %masked = and i32 %lshr.8, 255
+  %cvt = uitofp i32 %masked to half
+  ret half %cvt
+}
+
+define half @v_uitofp_to_f16_lshr16_mask255(i32 %arg0) nounwind {
+; SI-LABEL: v_uitofp_to_f16_lshr16_mask255:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_cvt_f32_ubyte2_e32 v0, v0
+; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: v_uitofp_to_f16_lshr16_mask255:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cvt_f32_ubyte2_e32 v0, v0
+; VI-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; VI-NEXT:    s_setpc_b64 s[30:31]
+  %lshr.16 = lshr i32 %arg0, 16
+  %masked = and i32 %lshr.16, 255
+  %cvt = uitofp i32 %masked to half
+  ret half %cvt
+}
+
+define half @v_uitofp_to_f16_lshr24_mask255(i32 %arg0) nounwind {
+; SI-LABEL: v_uitofp_to_f16_lshr24_mask255:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_cvt_f32_ubyte3_e32 v0, v0
+; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: v_uitofp_to_f16_lshr24_mask255:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cvt_f32_ubyte3_e32 v0, v0
+; VI-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; VI-NEXT:    s_setpc_b64 s[30:31]
+  %lshr.16 = lshr i32 %arg0, 24
+  %masked = and i32 %lshr.16, 255
+  %cvt = uitofp i32 %masked to half
+  ret half %cvt
+}
+
+define half @v_uitofp_i8_to_f16(i8 %arg0) nounwind {
+; SI-LABEL: v_uitofp_i8_to_f16:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
+; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: v_uitofp_i8_to_f16:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_cvt_f16_u16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
+; VI-NEXT:    s_setpc_b64 s[30:31]
+  %cvt = uitofp i8 %arg0 to half
+  ret half %cvt
+}
+
+define double @v_uitofp_i32_to_f64_mask255(i32 %arg0) nounwind {
+; GCN-LABEL: v_uitofp_i32_to_f64_mask255:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; GCN-NEXT:    v_cvt_f64_u32_e32 v[0:1], v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %masked = and i32 %arg0, 255
+  %cvt = uitofp i32 %masked to double
+  ret double %cvt
+}
+
+define double @v_uitofp_to_f64_lshr8_mask255(i32 %arg0) nounwind {
+; GCN-LABEL: v_uitofp_to_f64_lshr8_mask255:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_bfe_u32 v0, v0, 8, 8
+; GCN-NEXT:    v_cvt_f64_u32_e32 v[0:1], v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %lshr.8 = lshr i32 %arg0, 8
+  %masked = and i32 %lshr.8, 255
+  %cvt = uitofp i32 %masked to double
+  ret double %cvt
+}
+
+define double @v_uitofp_to_f64_lshr16_mask255(i32 %arg0) nounwind {
+; GCN-LABEL: v_uitofp_to_f64_lshr16_mask255:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_bfe_u32 v0, v0, 16, 8
+; GCN-NEXT:    v_cvt_f64_u32_e32 v[0:1], v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %lshr.16 = lshr i32 %arg0, 16
+  %masked = and i32 %lshr.16, 255
+  %cvt = uitofp i32 %masked to double
+  ret double %cvt
+}
+
+define double @v_uitofp_to_f64_lshr24_mask255(i32 %arg0) nounwind {
+; GCN-LABEL: v_uitofp_to_f64_lshr24_mask255:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
+; GCN-NEXT:    v_cvt_f64_u32_e32 v[0:1], v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %lshr.16 = lshr i32 %arg0, 24
+  %masked = and i32 %lshr.16, 255
+  %cvt = uitofp i32 %masked to double
+  ret double %cvt
+}
+
+define double @v_uitofp_i8_to_f64(i8 %arg0) nounwind {
+; SI-LABEL: v_uitofp_i8_to_f64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; SI-NEXT:    v_cvt_f64_u32_e32 v[0:1], v0
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: v_uitofp_i8_to_f64:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, 0xffff
+; VI-NEXT:    v_and_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT:    v_cvt_f64_u32_e32 v[0:1], v0
+; VI-NEXT:    s_setpc_b64 s[30:31]
+  %cvt = uitofp i8 %arg0 to double
+  ret double %cvt
+}
+
 define amdgpu_kernel void @load_i8_to_f32(float addrspace(1)* noalias %out, i8 addrspace(1)* noalias %in) nounwind {
 ; SI-LABEL: load_i8_to_f32:
 ; SI:       ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/debug-value.ll b/llvm/test/CodeGen/AMDGPU/debug-value.ll
index 0eb639e06e6aa..0eff1df7621bd 100644
--- a/llvm/test/CodeGen/AMDGPU/debug-value.ll
+++ b/llvm/test/CodeGen/AMDGPU/debug-value.ll
@@ -89,7 +89,7 @@ declare float @barney() #2
 declare void @eggs(float) #2
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { convergent nounwind "target-cpu"="gfx900" "target-features"="+fp32-denormals" }
+attributes #0 = { convergent nounwind "target-cpu"="gfx900" }
 attributes #1 = { nounwind readnone speculatable }
 attributes #2 = { nounwind }
 
diff --git a/llvm/test/CodeGen/AMDGPU/default-fp-mode.ll b/llvm/test/CodeGen/AMDGPU/default-fp-mode.ll
index fb6491937e5b1..ddc226506357c 100644
--- a/llvm/test/CodeGen/AMDGPU/default-fp-mode.ll
+++ b/llvm/test/CodeGen/AMDGPU/default-fp-mode.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
 ; GCN-LABEL: {{^}}test_default_si:
-; GCN: FloatMode: 192
+; GCN: FloatMode: 240
 ; GCN: IeeeMode: 1
 define amdgpu_kernel void @test_default_si(float addrspace(1)* %out0, double addrspace(1)* %out1) #0 {
   store float 0.0, float addrspace(1)* %out0
@@ -10,7 +10,7 @@ define amdgpu_kernel void @test_default_si(float addrspace(1)* %out0, double add
 }
 
 ; GCN-LABEL: {{^}}test_default_vi:
-; GCN: FloatMode: 192
+; GCN: FloatMode: 240
 ; GCN: IeeeMode: 1
 define amdgpu_kernel void @test_default_vi(float addrspace(1)* %out0, double addrspace(1)* %out1) #1 {
   store float 0.0, float addrspace(1)* %out0
@@ -19,7 +19,7 @@ define amdgpu_kernel void @test_default_vi(float addrspace(1)* %out0, double add
 }
 
 ; GCN-LABEL: {{^}}test_f64_denormals:
-; GCN: FloatMode: 192
+; GCN: FloatMode: 240
 ; GCN: IeeeMode: 1
 define amdgpu_kernel void @test_f64_denormals(float addrspace(1)* %out0, double addrspace(1)* %out1) #2 {
   store float 0.0, float addrspace(1)* %out0
@@ -55,7 +55,7 @@ define amdgpu_kernel void @test_no_denormals(float addrspace(1)* %out0, double a
 }
 
 ; GCN-LABEL: {{^}}test_f16_f64_denormals:
-; GCN: FloatMode: 192
+; GCN: FloatMode: 240
 ; GCN: IeeeMode: 1
 define amdgpu_kernel void @test_f16_f64_denormals(half addrspace(1)* %out0, double addrspace(1)* %out1) #6 {
   store half 0.0, half addrspace(1)* %out0
@@ -64,7 +64,7 @@ define amdgpu_kernel void @test_f16_f64_denormals(half addrspace(1)* %out0, doub
 }
 
 ; GCN-LABEL: {{^}}test_no_f16_f64_denormals:
-; GCN: FloatMode: 0
+; GCN: FloatMode: 48
 ; GCN: IeeeMode: 1
 define amdgpu_kernel void @test_no_f16_f64_denormals(half addrspace(1)* %out0, double addrspace(1)* %out1) #7 {
   store half 0.0, half addrspace(1)* %out0
@@ -82,7 +82,71 @@ define amdgpu_kernel void @test_f32_f16_f64_denormals(half addrspace(1)* %out0,
   ret void
 }
 
+; GCN-LABEL: {{^}}test_just_f32_attr_flush
+; GCN: FloatMode: 192
+; GCN: IeeeMode: 1
+define amdgpu_kernel void @test_just_f32_attr_flush(float addrspace(1)* %out0, double addrspace(1)* %out1) #9 {
+  store float 0.0, float addrspace(1)* %out0
+  store double 0.0, double addrspace(1)* %out1
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_flush_all_outputs:
+; GCN: FloatMode: 80
+; GCN: IeeeMode: 1
+define amdgpu_kernel void @test_flush_all_outputs(float addrspace(1)* %out0, double addrspace(1)* %out1) #10 {
+  store float 0.0, float addrspace(1)* %out0
+  store double 0.0, double addrspace(1)* %out1
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_flush_all_inputs:
+; GCN: FloatMode: 160
+; GCN: IeeeMode: 1
+define amdgpu_kernel void @test_flush_all_inputs(float addrspace(1)* %out0, double addrspace(1)* %out1) #11 {
+  store float 0.0, float addrspace(1)* %out0
+  store double 0.0, double addrspace(1)* %out1
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_flush_f32_inputs:
+; GCN: FloatMode: 224
+; GCN: IeeeMode: 1
+define amdgpu_kernel void @test_flush_f32_inputs(float addrspace(1)* %out0, double addrspace(1)* %out1) #12 {
+  store float 0.0, float addrspace(1)* %out0
+  store double 0.0, double addrspace(1)* %out1
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_flush_f32_outputs:
+; GCN: FloatMode: 208
+; GCN: IeeeMode: 1
+define amdgpu_kernel void @test_flush_f32_outputs(float addrspace(1)* %out0, double addrspace(1)* %out1) #13 {
+  store float 0.0, float addrspace(1)* %out0
+  store double 0.0, double addrspace(1)* %out1
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_flush_f64_inputs:
+; GCN: FloatMode: 176
+; GCN: IeeeMode: 1
+define amdgpu_kernel void @test_flush_f64_inputs(float addrspace(1)* %out0, double addrspace(1)* %out1) #14 {
+  store float 0.0, float addrspace(1)* %out0
+  store double 0.0, double addrspace(1)* %out1
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_flush_f64_outputs:
+; GCN: FloatMode: 112
+; GCN: IeeeMode: 1
+define amdgpu_kernel void @test_flush_f64_outputs(float addrspace(1)* %out0, double addrspace(1)* %out1) #15 {
+  store float 0.0, float addrspace(1)* %out0
+  store double 0.0, double addrspace(1)* %out1
+  ret void
+}
+
 ; GCN-LABEL: {{^}}kill_gs_const:
+; GCN: FloatMode: 240
 ; GCN: IeeeMode: 0
 define amdgpu_gs void @kill_gs_const() {
 main_body:
@@ -94,6 +158,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}kill_vcc_implicit_def:
+; GCN: FloatMode: 240
 ; GCN: IeeeMode: 0
 define amdgpu_ps float @kill_vcc_implicit_def([6 x <16 x i8>] addrspace(4)* inreg, [17 x <16 x i8>] addrspace(4)* inreg, [17 x <4 x i32>] addrspace(4)* inreg, [34 x <8 x i32>] addrspace(4)* inreg, float inreg, i32 inreg, <2 x i32>, <2 x i32>, <2 x i32>, <3 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, float, float, float, float, float, float, i32, float, float) {
 entry:
@@ -107,10 +172,17 @@ declare void @llvm.amdgcn.kill(i1)
 
 attributes #0 = { nounwind "target-cpu"="tahiti" }
 attributes #1 = { nounwind "target-cpu"="fiji" }
-attributes #2 = { nounwind "target-features"="+fp64-denormals" }
-attributes #3 = { nounwind "target-features"="+fp32-denormals" }
-attributes #4 = { nounwind "target-features"="+fp32-denormals,+fp64-denormals" }
-attributes #5 = { nounwind "target-features"="-fp32-denormals,-fp64-fp16-denormals" }
-attributes #6 = { nounwind "target-features"="+fp64-fp16-denormals" }
-attributes #7 = { nounwind "target-features"="-fp64-fp16-denormals" }
-attributes #8 = { nounwind "target-features"="+fp32-denormals,+fp64-fp16-denormals" }
+attributes #2 = { nounwind "denormal-fp-math"="ieee,ieee" }
+attributes #3 = { nounwind "denormal-fp-math-f32"="ieee,ieee" }
+attributes #4 = { nounwind "denormal-fp-math"="ieee,ieee" }
+attributes #5 = { nounwind "denormal-fp-math"="preserve-sign,preserve-sign" }
+attributes #6 = { nounwind "denormal-fp-math"="ieee,ieee" }
+attributes #7 = { nounwind "denormal-fp-math-f32"="ieee,ieee" "denormal-fp-math"="preserve-sign,preserve-sign" }
+attributes #8 = { nounwind "denormal-fp-math"="ieee,ieee" }
+attributes #9 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
+attributes #10 = { nounwind "denormal-fp-math"="preserve-sign,ieee" }
+attributes #11 = { nounwind "denormal-fp-math"="ieee,preserve-sign" }
+attributes #12 = { nounwind "denormal-fp-math-f32"="ieee,preserve-sign" "denormal-fp-math"="ieee,ieee" }
+attributes #13 = { nounwind "denormal-fp-math-f32"="preserve-sign,ieee" "denormal-fp-math"="ieee,ieee" }
+attributes #14 = { nounwind "denormal-fp-math"="ieee,preserve-sign" "denormal-fp-math-f32"="ieee,ieee" }
+attributes #15 = { nounwind "denormal-fp-math"="preserve-sign,ieee" "denormal-fp-math-f32"="ieee,ieee" }
diff --git a/llvm/test/CodeGen/AMDGPU/ds_read2_superreg.ll b/llvm/test/CodeGen/AMDGPU/ds_read2_superreg.ll
index ef4efc6336ce1..5e2fc937a86c6 100644
--- a/llvm/test/CodeGen/AMDGPU/ds_read2_superreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds_read2_superreg.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -check-prefix=CI %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt,-enable-ds128 < %s | FileCheck -check-prefixes=CI,NODS128 %s
 
 @lds = addrspace(3) global [512 x float] undef, align 4
 @lds.v2 = addrspace(3) global [512 x <2 x float>] undef, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/fadd-fma-fmul-combine.ll b/llvm/test/CodeGen/AMDGPU/fadd-fma-fmul-combine.ll
index 501971ea96621..2780a6dec8f0e 100644
--- a/llvm/test/CodeGen/AMDGPU/fadd-fma-fmul-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/fadd-fma-fmul-combine.ll
@@ -1,8 +1,8 @@
-; RUN: llc -march=amdgcn -mcpu=tahiti -mattr=+fast-fmaf,-fp32-denormals -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-FLUSH %s
-; RUN: llc -march=amdgcn -mcpu=tahiti -mattr=-fast-fmaf,-fp32-denormals -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-FLUSH %s
+; RUN: llc -march=amdgcn -mattr=+fast-fmaf -denormal-fp-math-f32=preserve-sign -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-FLUSH %s
+; RUN: llc -march=amdgcn -mattr=-fast-fmaf -denormal-fp-math-f32=preserve-sign -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-FLUSH %s
 
-; RUN: llc -march=amdgcn -mcpu=tahiti -mattr=+fast-fmaf,+fp32-denormals -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-FASTFMA %s
-; RUN: llc -march=amdgcn -mcpu=tahiti -mattr=-fast-fmaf,+fp32-denormals -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-SLOWFMA %s
+; RUN: llc -march=amdgcn -mattr=+fast-fmaf -denormal-fp-math-f32=ieee  -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-FASTFMA %s
+; RUN: llc -march=amdgcn -mattr=-fast-fmaf -denormal-fp-math-f32=ieee -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-SLOWFMA %s
 
 ; FIXME: This should also fold when fma is actually fast if an FMA
 ; exists in the original program.
diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize-elimination.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize-elimination.ll
index 6d44292ffbe84..3d8a1c7d74584 100644
--- a/llvm/test/CodeGen/AMDGPU/fcanonicalize-elimination.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize-elimination.ll
@@ -1,8 +1,8 @@
-; RUN: llc -march=amdgcn -mcpu=gfx801 -verify-machineinstrs -mattr=-fp32-denormals < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,VI-FLUSH,GCN-FLUSH,GCN-NOEXCEPT %s
-; RUN: llc -march=amdgcn -mcpu=gfx801 -verify-machineinstrs -mattr=-fp32-denormals,+fp-exceptions < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-EXCEPT,VI,VI-FLUSH,GCN-FLUSH %s
-; RUN: llc -march=amdgcn -mcpu=gfx801 -verify-machineinstrs -mattr=+fp32-denormals < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,VI-DENORM,GCN-DENORM,GCN-NOEXCEPT %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -mattr=+fp32-denormals < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,GFX9-DENORM,GCN-DENORM,GCN-NOEXCEPT %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -mattr=-fp32-denormals < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,GFX9-FLUSH,GCN-FLUSH,GCN-NOEXCEPT %s
+; RUN: llc -march=amdgcn -mcpu=gfx801 -verify-machineinstrs -denormal-fp-math-f32=preserve-sign < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,VI-FLUSH,GCN-FLUSH,GCN-NOEXCEPT %s
+; RUN: llc -march=amdgcn -mcpu=gfx801 -verify-machineinstrs -mattr=+fp-exceptions -denormal-fp-math-f32=preserve-sign < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-EXCEPT,VI,VI-FLUSH,GCN-FLUSH %s
+; RUN: llc -march=amdgcn -mcpu=gfx801 -verify-machineinstrs -denormal-fp-math-f32=ieee < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,VI-DENORM,GCN-DENORM,GCN-NOEXCEPT %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -denormal-fp-math-f32=ieee < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,GFX9-DENORM,GCN-DENORM,GCN-NOEXCEPT %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -denormal-fp-math-f32=preserve-sign < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,GFX9-FLUSH,GCN-FLUSH,GCN-NOEXCEPT %s
 
 ; GCN-LABEL: {{^}}test_no_fold_canonicalize_loaded_value_f32:
 ; GCN-FLUSH:   v_mul_f32_e32 v{{[0-9]+}}, 1.0, v{{[0-9]+}}
@@ -907,4 +907,4 @@ declare float @llvm.amdgcn.frexp.mant.f32(float) #0
 
 attributes #0 = { nounwind readnone }
 attributes #1 = { "no-nans-fp-math"="true" }
-attributes #2 = { "target-features"="-fp64-fp16-denormals" }
+attributes #2 = { "denormal-fp-math"="preserve-sign,preserve-sign" "denormal-fp-math-f32"="ieee,ieee" }
diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll
index 346430d6e7e60..e179ef37b3f8c 100644
--- a/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll
@@ -739,6 +739,6 @@ define <4 x half> @v_test_canonicalize_reg_undef_reg_reg_v4f16(half %val0, half
 }
 
 attributes #0 = { nounwind readnone }
-attributes #1 = { nounwind }
-attributes #2 = { nounwind "target-features"="-fp64-fp16-denormals" }
-attributes #3 = { nounwind "target-features"="+fp64-fp16-denormals" }
+attributes #1 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
+attributes #2 = { nounwind "denormal-fp-math"="preserve-sign,preserve-sign" }
+attributes #3 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll
index 8ea72ec3fdb81..ad472a92f86e2 100644
--- a/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll
@@ -625,9 +625,9 @@ define <4 x double> @v_test_canonicalize_v4f64(<4 x double> %arg) #1 {
 }
 
 attributes #0 = { nounwind readnone }
-attributes #1 = { nounwind }
-attributes #2 = { nounwind "target-features"="-fp32-denormals,-fp64-fp16-denormals" }
-attributes #3 = { nounwind "target-features"="+fp32-denormals,+fp64-fp16-denormals" }
-attributes #4 = { nounwind "target-features"="-fp32-denormals,-fp64-fp16-denormals" "target-cpu"="tonga" }
-attributes #5 = { nounwind "target-features"="+fp32-denormals,+fp64-fp16-denormals" "target-cpu"="gfx900" }
-attributes #6 = { nounwind "target-features"="-fp32-denormals,-fp64-fp16-denormals" "target-cpu"="gfx900" }
+attributes #1 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
+attributes #2 = { nounwind "denormal-fp-math"="preserve-sign,preserve-sign" }
+attributes #3 = { nounwind "denormal-fp-math"="ieee,ieee" }
+attributes #4 = { nounwind "denormal-fp-math"="preserve-sign,preserve-sign" "target-cpu"="tonga" }
+attributes #5 = { nounwind "denormal-fp-math"="ieee,ieee" "target-cpu"="gfx900" }
+attributes #6 = { nounwind "denormal-fp-math"="preserve-sign,preserve-sign" "target-cpu"="gfx900" }
diff --git a/llvm/test/CodeGen/AMDGPU/fdiv.f16.ll b/llvm/test/CodeGen/AMDGPU/fdiv.f16.ll
index f526ad3ca03b8..29a48397d94e2 100644
--- a/llvm/test/CodeGen/AMDGPU/fdiv.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fdiv.f16.ll
@@ -1,8 +1,8 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -mattr=+fp16-denormals -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX8_9_10 %s
-; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -mattr=-fp16-denormals -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX8_9_10 %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX8_9_10 %s
-; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX8_9_10 %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX8_9_10 %s
+; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX8_9_10 %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX8_9_10 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=-flat-for-global -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX8_9_10 %s
 
 ; Make sure fdiv is promoted to f32.
 
@@ -263,9 +263,9 @@ define amdgpu_kernel void @div_afn_neg_k_x_pat_f16(half addrspace(1)* %out) #0 {
   ret void
 }
 
-declare i32 @llvm.amdgcn.workitem.id.x() #1
-declare half @llvm.sqrt.f16(half) #1
-declare half @llvm.fabs.f16(half) #1
+declare i32 @llvm.amdgcn.workitem.id.x() #2
+declare half @llvm.sqrt.f16(half) #2
+declare half @llvm.fabs.f16(half) #2
 
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readnone }
diff --git a/llvm/test/CodeGen/AMDGPU/fdiv.ll b/llvm/test/CodeGen/AMDGPU/fdiv.ll
index 74ad621590539..1986ecf209292 100644
--- a/llvm/test/CodeGen/AMDGPU/fdiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/fdiv.ll
@@ -345,9 +345,8 @@ entry:
   ret void
 }
 
-
-attributes #0 = { nounwind "enable-unsafe-fp-math"="false" "target-features"="-fp32-denormals,+fp64-fp16-denormals,-flat-for-global" }
-attributes #1 = { nounwind "enable-unsafe-fp-math"="true" "target-features"="-fp32-denormals,-flat-for-global" }
-attributes #2 = { nounwind "enable-unsafe-fp-math"="false" "target-features"="+fp32-denormals,-flat-for-global" }
+attributes #0 = { nounwind "enable-unsafe-fp-math"="false" "denormal-fp-math-f32"="preserve-sign,preserve-sign" "target-features"="-flat-for-global" }
+attributes #1 = { nounwind "enable-unsafe-fp-math"="true" "denormal-fp-math-f32"="preserve-sign,preserve-sign" "target-features"="-flat-for-global" }
+attributes #2 = { nounwind "enable-unsafe-fp-math"="false" "denormal-fp-math-f32"="ieee,ieee" "target-features"="-flat-for-global" }
 
 !0 = !{float 2.500000e+00}
diff --git a/llvm/test/CodeGen/AMDGPU/fdiv32-to-rcp-folding.ll b/llvm/test/CodeGen/AMDGPU/fdiv32-to-rcp-folding.ll
index c02a21efce5fa..197bdd77d2544 100644
--- a/llvm/test/CodeGen/AMDGPU/fdiv32-to-rcp-folding.ll
+++ b/llvm/test/CodeGen/AMDGPU/fdiv32-to-rcp-folding.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=+fp32-denormals < %s | FileCheck --check-prefixes=GCN,GCN-DENORM %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-fp32-denormals < %s | FileCheck --check-prefixes=GCN,GCN-FLUSH %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -denormal-fp-math-f32=ieee < %s | FileCheck --check-prefixes=GCN,GCN-DENORM %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -denormal-fp-math-f32=preserve-sign < %s | FileCheck --check-prefixes=GCN,GCN-FLUSH %s
 
 ; GCN-LABEL: {{^}}div_1_by_x_25ulp:
 ; GCN-DENORM-DAG: v_mov_b32_e32 [[L:v[0-9]+]], 0x6f800000
diff --git a/llvm/test/CodeGen/AMDGPU/fdot2.ll b/llvm/test/CodeGen/AMDGPU/fdot2.ll
index dbfcd5de4a481..7da5dbd987166 100644
--- a/llvm/test/CodeGen/AMDGPU/fdot2.ll
+++ b/llvm/test/CodeGen/AMDGPU/fdot2.ll
@@ -1,10 +1,10 @@
-; RUN: llc -march=amdgcn -mcpu=gfx900 -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GFX900
-; RUN: llc -march=amdgcn -mcpu=gfx906 -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GCN-DL-UNSAFE,GFX906-DL-UNSAFE
-; RUN: llc -march=amdgcn -mcpu=gfx1011 -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GCN-DL-UNSAFE,GFX10-DL-UNSAFE,GFX10-CONTRACT
-; RUN: llc -march=amdgcn -mcpu=gfx1012 -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GCN-DL-UNSAFE,GFX10-DL-UNSAFE,GFX10-CONTRACT
-; RUN: llc -march=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GFX906
-; RUN: llc -march=amdgcn -mcpu=gfx906 -mattr=-fp64-fp16-denormals,-fp32-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GFX906-CONTRACT
-; RUN: llc -march=amdgcn -mcpu=gfx906 -mattr=+fp64-fp16-denormals,+fp32-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GFX906-DENORM-CONTRACT
+; RUN: llc -march=amdgcn -mcpu=gfx900 -denormal-fp-math-f32=preserve-sign -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GFX900
+; RUN: llc -march=amdgcn -mcpu=gfx906 -denormal-fp-math-f32=preserve-sign -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GCN-DL-UNSAFE,GFX906-DL-UNSAFE
+; RUN: llc -march=amdgcn -mcpu=gfx1011 -denormal-fp-math-f32=preserve-sign -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GCN-DL-UNSAFE,GFX10-DL-UNSAFE,GFX10-CONTRACT
+; RUN: llc -march=amdgcn -mcpu=gfx1012 -denormal-fp-math-f32=preserve-sign -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GCN-DL-UNSAFE,GFX10-DL-UNSAFE,GFX10-CONTRACT
+; RUN: llc -march=amdgcn -mcpu=gfx906 -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GFX906
+; RUN: llc -march=amdgcn -mcpu=gfx906 -denormal-fp-math=preserve-sign -fp-contract=fast -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GFX906-CONTRACT
+; RUN: llc -march=amdgcn -mcpu=gfx906 -denormal-fp-math=ieee -fp-contract=fast -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GFX906-DENORM-CONTRACT
 ; (fadd (fmul S1.x, S2.x), (fadd (fmul (S1.y, S2.y), z))) -> (fdot2 S1, S2, z)
 
 ; Tests to make sure fdot2 is not generated when vector elements of dot-product expressions
diff --git a/llvm/test/CodeGen/AMDGPU/fma-combine.ll b/llvm/test/CodeGen/AMDGPU/fma-combine.ll
index 5d78ddada14d6..b624ddf7a6fd0 100644
--- a/llvm/test/CodeGen/AMDGPU/fma-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/fma-combine.ll
@@ -1,6 +1,6 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tahiti -verify-machineinstrs -fp-contract=fast < %s | FileCheck -enable-var-scope -check-prefix=SI-NOFMA -check-prefix=SI-SAFE -check-prefix=SI -check-prefix=FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=verde -verify-machineinstrs -fp-contract=fast < %s | FileCheck -enable-var-scope -check-prefix=SI-NOFMA -check-prefix=SI-SAFE -check-prefix=SI -check-prefix=FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tahiti -verify-machineinstrs -fp-contract=fast -enable-no-infs-fp-math -enable-unsafe-fp-math -mattr=+fp32-denormals < %s | FileCheck -enable-var-scope -check-prefix=SI-FMA -check-prefix=SI-UNSAFE -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign -verify-machineinstrs -fp-contract=fast < %s | FileCheck -enable-var-scope -check-prefix=SI-NOFMA -check-prefix=SI-SAFE -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=verde -denormal-fp-math-f32=preserve-sign -verify-machineinstrs -fp-contract=fast < %s | FileCheck -enable-var-scope -check-prefix=SI-NOFMA -check-prefix=SI-SAFE -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -denormal-fp-math-f32=ieee -verify-machineinstrs -fp-contract=fast -enable-no-infs-fp-math -enable-unsafe-fp-math < %s | FileCheck -enable-var-scope -check-prefix=SI-FMA -check-prefix=SI-UNSAFE -check-prefix=SI -check-prefix=FUNC %s
 
 ; FIXME: Remove enable-unsafe-fp-math in RUN line and add flags to IR instrs
 
diff --git a/llvm/test/CodeGen/AMDGPU/fmaxnum.ll b/llvm/test/CodeGen/AMDGPU/fmaxnum.ll
index 7e16d1b883a01..b37bfbe30d594 100644
--- a/llvm/test/CodeGen/AMDGPU/fmaxnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmaxnum.ll
@@ -205,7 +205,7 @@ define amdgpu_ps float @fmax_literal_var_f32_no_ieee(float inreg %a) #0 {
 ; GCN: v_max_f32_e32
 ; GCN: v_max_f32_e32
 ; GCN-NOT: v_max_f32
-define <3 x float> @test_func_fmax_v3f32(<3 x float> %a, <3 x float> %b) nounwind {
+define <3 x float> @test_func_fmax_v3f32(<3 x float> %a, <3 x float> %b) #0 {
   %val = call <3 x float> @llvm.maxnum.v3f32(<3 x float> %a, <3 x float> %b) #0
   ret <3 x float> %val
 }
@@ -218,5 +218,5 @@ declare <8 x float> @llvm.maxnum.v8f32(<8 x float>, <8 x float>) #1
 declare <16 x float> @llvm.maxnum.v16f32(<16 x float>, <16 x float>) #1
 declare double @llvm.maxnum.f64(double, double)
 
-attributes #0 = { nounwind }
+attributes #0 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
 attributes #1 = { nounwind readnone }
diff --git a/llvm/test/CodeGen/AMDGPU/fminnum.ll b/llvm/test/CodeGen/AMDGPU/fminnum.ll
index a8574b288f57e..84ed73a864376 100644
--- a/llvm/test/CodeGen/AMDGPU/fminnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fminnum.ll
@@ -225,5 +225,5 @@ declare <4 x float> @llvm.minnum.v4f32(<4 x float>, <4 x float>) #1
 declare <8 x float> @llvm.minnum.v8f32(<8 x float>, <8 x float>) #1
 declare <16 x float> @llvm.minnum.v16f32(<16 x float>, <16 x float>) #1
 
-attributes #0 = { nounwind }
+attributes #0 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
 attributes #1 = { nounwind readnone }
diff --git a/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll b/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll
index af54cbf9d5c5c..c9be3952f01a0 100644
--- a/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll
@@ -1,8 +1,8 @@
-; XUN: llc -mtriple=amdgcn-amd-amdhsa -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=tonga -mattr=+fp64-fp16-denormals,-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,SIVI,VI-DENORM %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=tonga -mattr=-fp64-fp16-denormals,-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,SIVI,VI-FLUSH %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+fp64-fp16-denormals,-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX10,GFX8_10,GFX10-DENORM %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=-fp64-fp16-denormals,-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX10,GFX8_10,GFX10-FLUSH %s
+; XUN: llc -mtriple=amdgcn-amd-amdhsa -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=tonga -denormal-fp-math-f32=preserve-sign -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,SIVI,VI-DENORM %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=tonga -denormal-fp-math=preserve-sign -denormal-fp-math-f32=preserve-sign -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,SIVI,VI-FLUSH %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -denormal-fp-math-f32=preserve-sign -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX10,GFX8_10,GFX10-DENORM %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -denormal-fp-math=preserve-sign -denormal-fp-math-f32=preserve-sign -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX10,GFX8_10,GFX10-FLUSH %s
 
 ; Make sure (fmul (fadd x, x), c) -> (fmul x, (fmul 2.0, c)) doesn't
 ; make add an instruction if the fadd has more than one use.
diff --git a/llvm/test/CodeGen/AMDGPU/fmuladd.f16.ll b/llvm/test/CodeGen/AMDGPU/fmuladd.f16.ll
index 248cbe6ab5cc7..58d61ccaf97bb 100644
--- a/llvm/test/CodeGen/AMDGPU/fmuladd.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmuladd.f16.ll
@@ -1,13 +1,13 @@
-; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-fp64-fp16-denormals -fp-contract=on -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-STRICT,VI-FLUSH,VI %s
-; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-fp64-fp16-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-CONTRACT,VI-FLUSH,VI %s
+; RUN: llc -march=amdgcn -mcpu=fiji -denormal-fp-math=preserve-sign -denormal-fp-math-f32=ieee -fp-contract=on -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-STRICT,VI-FLUSH,VI %s
+; RUN: llc -march=amdgcn -mcpu=fiji -denormal-fp-math=preserve-sign -denormal-fp-math-f32=ieee -fp-contract=fast -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-CONTRACT,VI-FLUSH,VI %s
 
-; RUN: llc -march=amdgcn -mcpu=fiji -mattr=+fp64-fp16-denormals -fp-contract=on -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-STRICT,GCN-DENORM,GCN-DENORM-STRICT,VI-DENORM-STRICT,VI-DENORM,VI %s
-; RUN: llc -march=amdgcn -mcpu=fiji -mattr=+fp64-fp16-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-CONTRACT,GCN-DENORM,GCN-DENORM-CONTRACT,VI-DENORM-CONTRACT,VI-DENORM,VI %s
+; RUN: llc -march=amdgcn -mcpu=fiji -denormal-fp-math=ieee -denormal-fp-math-f32=ieee -fp-contract=on -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-STRICT,GCN-DENORM,GCN-DENORM-STRICT,VI-DENORM-STRICT,VI-DENORM,VI %s
+; RUN: llc -march=amdgcn -mcpu=fiji -denormal-fp-math=ieee -denormal-fp-math-f32=ieee -fp-contract=fast -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-CONTRACT,GCN-DENORM,GCN-DENORM-CONTRACT,VI-DENORM-CONTRACT,VI-DENORM,VI %s
 
-; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=-fp64-fp16-denormals -fp-contract=on -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-STRICT,GFX10-FLUSH,GFX10 %s
-; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=-fp64-fp16-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-CONTRACT,GFX10-FLUSH,GFX10 %s
-; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=+fp64-fp16-denormals -fp-contract=on -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-STRICT,GCN-DENORM,GCN-DENORM-STRICT,GFX10-DENORM-STRICT,GFX10-DENORM,GFX10 %s
-; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=+fp64-fp16-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-CONTRACT,GCN-DENORM,GCN-DENORM-CONTRACT,GFX10-DENORM-CONTRACT,GFX10-DENORM,GFX10 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -denormal-fp-math=preserve-sign -denormal-fp-math-f32=ieee -fp-contract=on -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-STRICT,GFX10-FLUSH,GFX10 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -denormal-fp-math=preserve-sign -denormal-fp-math-f32=ieee -fp-contract=fast -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-CONTRACT,GFX10-FLUSH,GFX10 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -denormal-fp-math=ieee -denormal-fp-math-f32=ieee -fp-contract=on -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-STRICT,GCN-DENORM,GCN-DENORM-STRICT,GFX10-DENORM-STRICT,GFX10-DENORM,GFX10 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -denormal-fp-math=ieee -denormal-fp-math-f32=ieee -fp-contract=fast -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-CONTRACT,GCN-DENORM,GCN-DENORM-CONTRACT,GFX10-DENORM-CONTRACT,GFX10-DENORM,GFX10 %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() #1
 declare half @llvm.fmuladd.f16(half, half, half) #1
diff --git a/llvm/test/CodeGen/AMDGPU/fmuladd.f32.ll b/llvm/test/CodeGen/AMDGPU/fmuladd.f32.ll
index 4ef293f9a51f3..a6b374445cfb8 100644
--- a/llvm/test/CodeGen/AMDGPU/fmuladd.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmuladd.f32.ll
@@ -1,21 +1,21 @@
-; RUN: llc -amdgpu-scalarize-global-loads=false -verify-machineinstrs -mcpu=tahiti -mattr=-fp32-denormals,+fast-fmaf -fp-contract=on < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-FLUSH,GCN-FLUSH-STRICT,GCN-FLUSH-MAD,GCN-FLUSH-FASTFMA,GCN-FLUSH-FASTFMA-STRICT,SI %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -verify-machineinstrs -mcpu=tahiti -mattr=+fp32-denormals,+fast-fmaf -fp-contract=on < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-DENORM-STRICT,GCN-DENORM,SI-DENORM,GCN-DENORM-FASTFMA,GCN-DENORM-FASTFMA-STRICT,SI %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -verify-machineinstrs -mcpu=verde -mattr=-fp32-denormals,-fast-fmaf -fp-contract=on < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-FLUSH,GCN-FLUSH-STRICT,GCN-FLUSH-MAD,SI-FLUSH,GCN-FLUSH-SLOWFMA,GCN-FLUSH-SLOWFMA-STRICT,SI %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -verify-machineinstrs -mcpu=verde -mattr=+fp32-denormals,-fast-fmaf -fp-contract=on < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-DENORM-STRICT,GCN-DENORM,SI-DENORM,GCN-DENORM-SLOWFMA,GCN-DENORM-SLOWFMA-STRICT,SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -verify-machineinstrs -mcpu=tahiti -denormal-fp-math-f32=preserve-sign -mattr=+fast-fmaf -fp-contract=on < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-FLUSH,GCN-FLUSH-STRICT,GCN-FLUSH-MAD,GCN-FLUSH-FASTFMA,GCN-FLUSH-FASTFMA-STRICT,SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -verify-machineinstrs -mcpu=tahiti -denormal-fp-math-f32=ieee -mattr=+fast-fmaf -fp-contract=on < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-DENORM-STRICT,GCN-DENORM,SI-DENORM,GCN-DENORM-FASTFMA,GCN-DENORM-FASTFMA-STRICT,SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -verify-machineinstrs -mcpu=verde -denormal-fp-math-f32=preserve-sign -mattr=-fast-fmaf -fp-contract=on < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-FLUSH,GCN-FLUSH-STRICT,GCN-FLUSH-MAD,SI-FLUSH,GCN-FLUSH-SLOWFMA,GCN-FLUSH-SLOWFMA-STRICT,SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -verify-machineinstrs -mcpu=verde -denormal-fp-math-f32=ieee -mattr=-fast-fmaf -fp-contract=on < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-DENORM-STRICT,GCN-DENORM,SI-DENORM,GCN-DENORM-SLOWFMA,GCN-DENORM-SLOWFMA-STRICT,SI %s
 
-; RUN: llc -amdgpu-scalarize-global-loads=false -verify-machineinstrs -mcpu=tahiti -mattr=-fp32-denormals,+fast-fmaf -fp-contract=fast < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-FLUSH,GCN-FLUSH-CONTRACT,GCN-FLUSH-MAD,SI-FLUSH,GCN-FLUSH-FASTFMA,GCN-FLUSH-FASTFMA-CONTRACT,SI %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -verify-machineinstrs -mcpu=tahiti -mattr=+fp32-denormals,+fast-fmaf -fp-contract=fast < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-DENORM-CONTRACT,GCN-DENORM,SI-DENORM,GCN-DENORM-FASTFMA,GCN-DENORM-FASTFMA-CONTRACT,SI %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -verify-machineinstrs -mcpu=verde -mattr=-fp32-denormals,-fast-fmaf -fp-contract=fast < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-FLUSH,GCN-FLUSH-CONTRACT,GCN-FLUSH-MAD,SI-FLUSH,GCN-FLUSH-SLOWFMA,GCN-FLUSH-SLOWFMA-CONTRACT,SI %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -verify-machineinstrs -mcpu=verde -mattr=+fp32-denormals,-fast-fmaf -fp-contract=fast < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-DENORM-CONTRACT,GCN-DENORM,SI-DENORM,GCN-DENORM-SLOWFMA,GCN-DENORM-SLOWFMA-CONTRACT,SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -verify-machineinstrs -mcpu=tahiti -denormal-fp-math-f32=preserve-sign -mattr=+fast-fmaf -fp-contract=fast < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-FLUSH,GCN-FLUSH-CONTRACT,GCN-FLUSH-MAD,SI-FLUSH,GCN-FLUSH-FASTFMA,GCN-FLUSH-FASTFMA-CONTRACT,SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -verify-machineinstrs -mcpu=tahiti -denormal-fp-math-f32=ieee -mattr=+fast-fmaf -fp-contract=fast < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-DENORM-CONTRACT,GCN-DENORM,SI-DENORM,GCN-DENORM-FASTFMA,GCN-DENORM-FASTFMA-CONTRACT,SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -verify-machineinstrs -mcpu=verde -denormal-fp-math-f32=preserve-sign -mattr=-fast-fmaf -fp-contract=fast < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-FLUSH,GCN-FLUSH-CONTRACT,GCN-FLUSH-MAD,SI-FLUSH,GCN-FLUSH-SLOWFMA,GCN-FLUSH-SLOWFMA-CONTRACT,SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -verify-machineinstrs -mcpu=verde -denormal-fp-math-f32=ieee -mattr=-fast-fmaf -fp-contract=fast < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-DENORM-CONTRACT,GCN-DENORM,SI-DENORM,GCN-DENORM-SLOWFMA,GCN-DENORM-SLOWFMA-CONTRACT,SI %s
 
 
-; RUN: llc -amdgpu-scalarize-global-loads=false -verify-machineinstrs -mcpu=gfx900 -mattr=-fp32-denormals -fp-contract=on < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-FLUSH,GCN-FLUSH-STRICT,GCN-FLUSH-MAD,GFX9-FLUSH,GCN-FLUSH-FASTFMA,GCN-FLUSH-FASTFMA-STRICT,GFX900 %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -verify-machineinstrs -mcpu=gfx900 -mattr=+fp32-denormals -fp-contract=on < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-DENORM-STRICT,GCN-DENORM,GFX9-DENORM,GCN-DENORM-FASTFMA,GCN-DENORM-FASTFMA-STRICT,GFX900 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -verify-machineinstrs -mcpu=gfx900 -denormal-fp-math-f32=preserve-sign -fp-contract=on < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-FLUSH,GCN-FLUSH-STRICT,GCN-FLUSH-MAD,GFX9-FLUSH,GCN-FLUSH-FASTFMA,GCN-FLUSH-FASTFMA-STRICT,GFX900 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -verify-machineinstrs -mcpu=gfx900 -denormal-fp-math-f32=ieee -fp-contract=on < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-DENORM-STRICT,GCN-DENORM,GFX9-DENORM,GCN-DENORM-FASTFMA,GCN-DENORM-FASTFMA-STRICT,GFX900 %s
 
-; RUN: llc -amdgpu-scalarize-global-loads=false -verify-machineinstrs -mcpu=gfx906 -mattr=-fp32-denormals -fp-contract=on < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-FLUSH,GCN-FLUSH-STRICT,GCN-FLUSH-FMAC,GFX9-FLUSH,GCN-FLUSH-FASTFMA,GCN-FLUSH-FASTFMA-STRICT,GFX906 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -verify-machineinstrs -mcpu=gfx906 -denormal-fp-math-f32=preserve-sign -fp-contract=on < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-FLUSH,GCN-FLUSH-STRICT,GCN-FLUSH-FMAC,GFX9-FLUSH,GCN-FLUSH-FASTFMA,GCN-FLUSH-FASTFMA-STRICT,GFX906 %s
 
 ; FIXME: Should probably test this, but sometimes selecting fmac is painful to match.
-; XUN: llc -amdgpu-scalarize-global-loads=false -verify-machineinstrs -mcpu=gfx906 -mattr=+fp32-denormals -fp-contract=on < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-DENORM-STRICT,GCN-DENORM,GFX9-DENORM,GCN-DENORM-FASTFMA,GCN-DENORM-FASTFMA-STRICT,GFX906 %s
+; XUN: llc -amdgpu-scalarize-global-loads=false -verify-machineinstrs -mcpu=gfx906 -denormal-fp-math-f32=ieee -fp-contract=on < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-DENORM-STRICT,GCN-DENORM,GFX9-DENORM,GCN-DENORM-FASTFMA,GCN-DENORM-FASTFMA-STRICT,GFX906 %s
 
 
 ; Test all permutations of: fp32 denormals, fast fp contract, fp contract enabled for fmuladd, fmaf fast/slow.
diff --git a/llvm/test/CodeGen/AMDGPU/fmuladd.v2f16.ll b/llvm/test/CodeGen/AMDGPU/fmuladd.v2f16.ll
index d02c399dd0bb0..95ea266c6eb72 100644
--- a/llvm/test/CodeGen/AMDGPU/fmuladd.v2f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmuladd.v2f16.ll
@@ -1,12 +1,12 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx900 -mattr=-fp64-fp16-denormals -fp-contract=on -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-STRICT,GFX9-FLUSH,GFX9 %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx900 -mattr=-fp64-fp16-denormals -fp-contract=on -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-STRICT,GFX9-FLUSH,GFX9 %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx900 -mattr=-fp64-fp16-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-CONTRACT,GFX9-FLUSH,GFX9 %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx900 -mattr=-fp64-fp16-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-CONTRACT,GFX9-FLUSH,GFX9 %s
-
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx900 -mattr=+fp64-fp16-denormals -fp-contract=on -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-STRICT,GFX9-DENORM-STRICT,GFX9-DENORM,GFX9 %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx900 -mattr=+fp64-fp16-denormals -fp-contract=on -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-STRICT,GFX9-DENORM-STRICT,GFX9-DENORM,GFX9 %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx900 -mattr=+fp64-fp16-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-CONTRACT,GFX9-DENORM-CONTRACT,GFX9-DENORM,GFX9 %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx900 -mattr=+fp64-fp16-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-CONTRACT,GFX9-DENORM-CONTRACT,GFX9-DENORM,GFX9 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -denormal-fp-math=preserve-sign -fp-contract=on -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-STRICT,GFX9-FLUSH,GFX9 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -denormal-fp-math=preserve-sign -fp-contract=on -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-STRICT,GFX9-FLUSH,GFX9 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -denormal-fp-math=preserve-sign -fp-contract=fast -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-CONTRACT,GFX9-FLUSH,GFX9 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -denormal-fp-math=preserve-sign -fp-contract=fast -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-CONTRACT,GFX9-FLUSH,GFX9 %s
+
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -denormal-fp-math=ieee -fp-contract=on -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-STRICT,GFX9-DENORM-STRICT,GFX9-DENORM,GFX9 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -denormal-fp-math=ieee -fp-contract=on -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-STRICT,GFX9-DENORM-STRICT,GFX9-DENORM,GFX9 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -denormal-fp-math=ieee -fp-contract=fast -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-CONTRACT,GFX9-DENORM-CONTRACT,GFX9-DENORM,GFX9 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -denormal-fp-math=ieee -fp-contract=fast -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-CONTRACT,GFX9-DENORM-CONTRACT,GFX9-DENORM,GFX9 %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() #1
 declare <2 x half> @llvm.fmuladd.v2f16(<2 x half>, <2 x half>, <2 x half>) #1
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-combines.ll b/llvm/test/CodeGen/AMDGPU/fneg-combines.ll
index 45cfaa7c09dbb..108b5830f22a3 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-combines.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-combines.ll
@@ -2562,6 +2562,6 @@ declare float @llvm.amdgcn.fmul.legacy(float, float) #1
 declare float @llvm.amdgcn.interp.p1(float, i32, i32, i32) #0
 declare float @llvm.amdgcn.interp.p2(float, float, i32, i32, i32) #0
 
-attributes #0 = { nounwind }
+attributes #0 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
 attributes #1 = { nounwind readnone }
 attributes #2 = { nounwind "unsafe-fp-math"="true" }
diff --git a/llvm/test/CodeGen/AMDGPU/force-alwaysinline-lds-global-address.ll b/llvm/test/CodeGen/AMDGPU/force-alwaysinline-lds-global-address.ll
index f525ca5f8036b..8ab59dc2224c3 100644
--- a/llvm/test/CodeGen/AMDGPU/force-alwaysinline-lds-global-address.ll
+++ b/llvm/test/CodeGen/AMDGPU/force-alwaysinline-lds-global-address.ll
@@ -74,4 +74,21 @@ define i32 @recursive_call_lds(i32 %arg0) {
   ret i32 %call
 }
 
+; Test we don't break the IR and have both alwaysinline and noinline
+; FIXME: We should really not override noinline.
+
+; ALL-LABEL: define i32 @load_lds_simple_noinline() #0 {
+define i32 @load_lds_simple_noinline() noinline {
+  %load = load i32, i32 addrspace(3)* @lds0, align 4
+  ret i32 %load
+}
+
+; ALL-LABEL: define i32 @recursive_call_lds_noinline(i32 %arg0) #0 {
+define i32 @recursive_call_lds_noinline(i32 %arg0) noinline {
+  %load = load i32, i32 addrspace(3)* @lds0, align 4
+  %add = add i32 %arg0, %load
+  %call = call i32 @recursive_call_lds(i32 %add)
+  ret i32 %call
+}
+
 ; ALL: attributes #0 = { alwaysinline }
diff --git a/llvm/test/CodeGen/AMDGPU/fpext-free.ll b/llvm/test/CodeGen/AMDGPU/fpext-free.ll
index 1bd4caa180a4b..fd9843400ba56 100644
--- a/llvm/test/CodeGen/AMDGPU/fpext-free.ll
+++ b/llvm/test/CodeGen/AMDGPU/fpext-free.ll
@@ -1,7 +1,7 @@
-; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-fp32-denormals -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX89,GFX9,GFX9-F32FLUSH %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=+fp32-denormals -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX89,GFX9,GFX9-F32DENORM %s
-; RUN: llc -march=amdgcn -mcpu=gfx803 -mattr=-fp32-denormals -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX89,VI,VI-F32FLUSH %s
-; RUN: llc -march=amdgcn -mcpu=gfx803 -mattr=+fp32-denormals -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX89,VI,VI-F32DENORM %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX89,GFX9,GFX9-F32FLUSH %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -denormal-fp-math-f32=ieee -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX89,GFX9,GFX9-F32DENORM %s
+; RUN: llc -march=amdgcn -mcpu=gfx803 -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX89,VI,VI-F32FLUSH %s
+; RUN: llc -march=amdgcn -mcpu=gfx803 -denormal-fp-math-f32=ieee -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX89,VI,VI-F32DENORM %s
 
 ;  fold (fadd (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), z)
 
diff --git a/llvm/test/CodeGen/AMDGPU/frem.ll b/llvm/test/CodeGen/AMDGPU/frem.ll
index 1305dae62995b..bf4f8f8b02e05 100644
--- a/llvm/test/CodeGen/AMDGPU/frem.ll
+++ b/llvm/test/CodeGen/AMDGPU/frem.ll
@@ -1,4 +1,4 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tahiti -verify-machineinstrs  < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=FUNC %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs  < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=FUNC %s
 ; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -check-prefix=CI -check-prefix=GCN -check-prefix=FUNC %s
 ; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=CI -check-prefix=GCN -check-prefix=FUNC %s
 
@@ -109,5 +109,5 @@ define amdgpu_kernel void @frem_v2f64(<2 x double> addrspace(1)* %out, <2 x doub
    ret void
 }
 
-attributes #0 = { nounwind "unsafe-fp-math"="false" }
-attributes #1 = { nounwind "unsafe-fp-math"="true" }
+attributes #0 = { nounwind "unsafe-fp-math"="false" "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
+attributes #1 = { nounwind "unsafe-fp-math"="true" "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-fp-mode.ll b/llvm/test/CodeGen/AMDGPU/hsa-fp-mode.ll
index 53e5dadc78647..0ad55502ac430 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-fp-mode.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-fp-mode.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -mtriple=amdgcn--amdhsa -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
 ; GCN-LABEL: {{^}}test_default_ci:
-; GCN: float_mode = 192
+; GCN: float_mode = 240
 ; GCN: enable_dx10_clamp = 1
 ; GCN: enable_ieee_mode = 1
 define amdgpu_kernel void @test_default_ci(float addrspace(1)* %out0, double addrspace(1)* %out1) #0 {
@@ -11,7 +11,7 @@ define amdgpu_kernel void @test_default_ci(float addrspace(1)* %out0, double add
 }
 
 ; GCN-LABEL: {{^}}test_default_vi:
-; GCN: float_mode = 192
+; GCN: float_mode = 240
 ; GCN: enable_dx10_clamp = 1
 ; GCN: enable_ieee_mode = 1
 define amdgpu_kernel void @test_default_vi(float addrspace(1)* %out0, double addrspace(1)* %out1) #1 {
@@ -61,7 +61,7 @@ define amdgpu_kernel void @test_no_denormals(float addrspace(1)* %out0, double a
 }
 
 ; GCN-LABEL: {{^}}test_no_dx10_clamp_vi:
-; GCN: float_mode = 192
+; GCN: float_mode = 240
 ; GCN: enable_dx10_clamp = 0
 ; GCN: enable_ieee_mode = 1
 define amdgpu_kernel void @test_no_dx10_clamp_vi(float addrspace(1)* %out0, double addrspace(1)* %out1) #6 {
@@ -71,7 +71,7 @@ define amdgpu_kernel void @test_no_dx10_clamp_vi(float addrspace(1)* %out0, doub
 }
 
 ; GCN-LABEL: {{^}}test_no_ieee_mode_vi:
-; GCN: float_mode = 192
+; GCN: float_mode = 240
 ; GCN: enable_dx10_clamp = 1
 ; GCN: enable_ieee_mode = 0
 define amdgpu_kernel void @test_no_ieee_mode_vi(float addrspace(1)* %out0, double addrspace(1)* %out1) #7 {
@@ -81,7 +81,7 @@ define amdgpu_kernel void @test_no_ieee_mode_vi(float addrspace(1)* %out0, doubl
 }
 
 ; GCN-LABEL: {{^}}test_no_ieee_mode_no_dx10_clamp_vi:
-; GCN: float_mode = 192
+; GCN: float_mode = 240
 ; GCN: enable_dx10_clamp = 0
 ; GCN: enable_ieee_mode = 0
 define amdgpu_kernel void @test_no_ieee_mode_no_dx10_clamp_vi(float addrspace(1)* %out0, double addrspace(1)* %out1) #8 {
@@ -92,10 +92,10 @@ define amdgpu_kernel void @test_no_ieee_mode_no_dx10_clamp_vi(float addrspace(1)
 
 attributes #0 = { nounwind "target-cpu"="kaveri" "target-features"="-code-object-v3" }
 attributes #1 = { nounwind "target-cpu"="fiji" "target-features"="-code-object-v3" }
-attributes #2 = { nounwind "target-features"="-code-object-v3,-fp32-denormals,+fp64-fp16-denormals" }
-attributes #3 = { nounwind "target-features"="-code-object-v3,+fp32-denormals,-fp64-fp16-denormals" }
-attributes #4 = { nounwind "target-features"="-code-object-v3,+fp32-denormals,+fp64-fp16-denormals" }
-attributes #5 = { nounwind "target-features"="-code-object-v3,-fp32-denormals,-fp64-fp16-denormals" }
+attributes #2 = { nounwind "target-features"="-code-object-v3" "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
+attributes #3 = { nounwind "target-features"="-code-object-v3" "denormal-fp-math-f32"="ieee,ieee" "denormal-fp-math"="preserve-sign,preserve-sign" }
+attributes #4 = { nounwind "target-features"="-code-object-v3" "denormal-fp-math"="ieee,ieee" }
+attributes #5 = { nounwind "target-features"="-code-object-v3" "denormal-fp-math"="preserve-sign,preserve-sign" }
 attributes #6 = { nounwind "amdgpu-dx10-clamp"="false" "target-cpu"="fiji" "target-features"="-code-object-v3" }
 attributes #7 = { nounwind "amdgpu-ieee"="false" "target-cpu"="fiji" "target-features"="-code-object-v3" }
 attributes #8 = { nounwind "amdgpu-dx10-clamp"="false" "amdgpu-ieee"="false" "target-cpu"="fiji" "target-features"="-code-object-v3" }
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-call.ll b/llvm/test/CodeGen/AMDGPU/indirect-call.ll
index ca033e5289f3f..8432d2961f04f 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-call.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-call.ll
@@ -18,7 +18,7 @@ define amdgpu_kernel void @test_indirect_call_sgpr_ptr() {
 ; GCN-NEXT:     granulated_workitem_vgpr_count = 7
 ; GCN-NEXT:     granulated_wavefront_sgpr_count = 5
 ; GCN-NEXT:     priority = 0
-; GCN-NEXT:     float_mode = 192
+; GCN-NEXT:     float_mode = 240
 ; GCN-NEXT:     priv = 0
 ; GCN-NEXT:     enable_dx10_clamp = 1
 ; GCN-NEXT:     debug_mode = 0
@@ -111,7 +111,7 @@ define amdgpu_kernel void @test_indirect_call_sgpr_ptr_arg() {
 ; GCN-NEXT:     granulated_workitem_vgpr_count = 7
 ; GCN-NEXT:     granulated_wavefront_sgpr_count = 5
 ; GCN-NEXT:     priority = 0
-; GCN-NEXT:     float_mode = 192
+; GCN-NEXT:     float_mode = 240
 ; GCN-NEXT:     priv = 0
 ; GCN-NEXT:     enable_dx10_clamp = 1
 ; GCN-NEXT:     debug_mode = 0
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-private-64.ll b/llvm/test/CodeGen/AMDGPU/indirect-private-64.ll
index 699f2060bf26a..39afd17298ed1 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-private-64.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-private-64.ll
@@ -49,8 +49,8 @@ define amdgpu_kernel void @private_access_f64_alloca(double addrspace(1)* noalia
 ; SI-PROMOTE: ds_write_b64
 ; SI-PROMOTE: ds_read_b64
 ; SI-PROMOTE: ds_read_b64
-; CI-PROMOTE: ds_write2_b64
-; CI-PROMOTE: ds_read2_b64
+; CI-PROMOTE: ds_write_b128
+; CI-PROMOTE: ds_read_b128
 define amdgpu_kernel void @private_access_v2f64_alloca(<2 x double> addrspace(1)* noalias %out, <2 x double> addrspace(1)* noalias %in, i32 %b) #1 {
   %val = load <2 x double>, <2 x double> addrspace(1)* %in, align 16
   %array = alloca [4 x <2 x double>], align 16, addrspace(5)
@@ -107,8 +107,8 @@ define amdgpu_kernel void @private_access_i64_alloca(i64 addrspace(1)* noalias %
 ; SI-PROMOTE: ds_write_b64
 ; SI-PROMOTE: ds_read_b64
 ; SI-PROMOTE: ds_read_b64
-; CI-PROMOTE: ds_write2_b64
-; CI-PROMOTE: ds_read2_b64
+; CI-PROMOTE: ds_write_b128
+; CI-PROMOTE: ds_read_b128
 define amdgpu_kernel void @private_access_v2i64_alloca(<2 x i64> addrspace(1)* noalias %out, <2 x i64> addrspace(1)* noalias %in, i32 %b) #1 {
   %val = load <2 x i64>, <2 x i64> addrspace(1)* %in, align 16
   %array = alloca [4 x <2 x i64>], align 16, addrspace(5)
diff --git a/llvm/test/CodeGen/AMDGPU/inline-attr.ll b/llvm/test/CodeGen/AMDGPU/inline-attr.ll
index b44138d107aee..c73a3c52e9ae7 100644
--- a/llvm/test/CodeGen/AMDGPU/inline-attr.ll
+++ b/llvm/test/CodeGen/AMDGPU/inline-attr.ll
@@ -6,8 +6,8 @@
 ; GCN: define amdgpu_kernel void @caller(float addrspace(1)* nocapture %p) local_unnamed_addr #1 {
 ; GCN: %mul.i = fmul float %load, 1.500000e+01
 
-; UNSAFE: attributes #0 = { norecurse nounwind readnone "less-precise-fpmad"="true" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "unsafe-fp-math"="true" }
-; UNSAFE: attributes #1 = { nofree norecurse nounwind "less-precise-fpmad"="true" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "unsafe-fp-math"="true" }
+; UNSAFE: attributes #0 = { norecurse nounwind readnone "unsafe-fp-math"="true" }
+; UNSAFE: attributes #1 = { nofree norecurse nounwind "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="true" }
 
 ; NOINFS: attributes #0 = { norecurse nounwind readnone "no-infs-fp-math"="true" }
 ; NOINFS: attributes #1 = { nofree norecurse nounwind "less-precise-fpmad"="false" "no-infs-fp-math"="true" "no-nans-fp-math"="false" "unsafe-fp-math"="false" }
diff --git a/llvm/test/CodeGen/AMDGPU/insert-skip-from-vcc.mir b/llvm/test/CodeGen/AMDGPU/insert-skip-from-vcc.mir
index 4d92b06335b27..fabae90490cb5 100644
--- a/llvm/test/CodeGen/AMDGPU/insert-skip-from-vcc.mir
+++ b/llvm/test/CodeGen/AMDGPU/insert-skip-from-vcc.mir
@@ -1,5 +1,5 @@
-# RUN: llc -march=amdgcn -mcpu=fiji -run-pass si-insert-skips -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
-# RUN: llc -march=amdgcn -mcpu=gfx1010 -run-pass si-insert-skips -verify-machineinstrs -o - %s | FileCheck -check-prefix=W32 %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass si-pre-emit-peephole -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -march=amdgcn -mcpu=gfx1010 -run-pass si-pre-emit-peephole -verify-machineinstrs -o - %s | FileCheck -check-prefix=W32 %s
 
 ---
 # GCN-LABEL: name: and_execz_mov_vccz
diff --git a/llvm/test/CodeGen/AMDGPU/insert-subvector-unused-scratch.ll b/llvm/test/CodeGen/AMDGPU/insert-subvector-unused-scratch.ll
index 9a23df97d9a5b..c4599ee2b3534 100644
--- a/llvm/test/CodeGen/AMDGPU/insert-subvector-unused-scratch.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert-subvector-unused-scratch.ll
@@ -18,9 +18,9 @@ define amdgpu_kernel void @store_v3i32(<3 x i32> addrspace(3)* %out, <3 x i32> %
 
 ; GCN-LABEL: store_v5i32:
 ; GCN:        ds_read_b32
-; GCN:        ds_read2_b64
+; GCN:        ds_read_b128
 ; GCN:        ds_write_b32
-; GCN:        ds_write2_b64
+; GCN:        ds_write_b128
 ; GCN: ScratchSize: 0
 define amdgpu_kernel void @store_v5i32(<5 x i32> addrspace(3)* %out, <5 x i32> %a) nounwind {
   %val = load <5 x i32>, <5 x i32> addrspace(3)* %out
@@ -28,5 +28,3 @@ define amdgpu_kernel void @store_v5i32(<5 x i32> addrspace(3)* %out, <5 x i32> %
   store <5 x i32> %val.1, <5 x i32> addrspace(3)* %out, align 16
   ret void
 }
-
-
diff --git a/llvm/test/CodeGen/AMDGPU/ipra-regmask.ll b/llvm/test/CodeGen/AMDGPU/ipra-regmask.ll
index 92c182f80e902..f1132fc773d44 100644
--- a/llvm/test/CodeGen/AMDGPU/ipra-regmask.ll
+++ b/llvm/test/CodeGen/AMDGPU/ipra-regmask.ll
@@ -1,19 +1,19 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -enable-ipra -print-regusage -o /dev/null 2>&1 < %s | FileCheck %s
 ; Make sure the expected regmask is generated for sub/superregisters.
 
-; CHECK-DAG: csr Clobbered Registers: $vgpr0 $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 $vgpr0_vgpr1_vgpr2_vgpr3 $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 $vgpr0_vgpr1 $vgpr0_vgpr1_vgpr2 {{$}}
+; CHECK-DAG: csr Clobbered Registers: $vgpr0 $vgpr0_hi16 $vgpr0_lo16 $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 $vgpr0_vgpr1_vgpr2_vgpr3 $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 $vgpr0_vgpr1 $vgpr0_vgpr1_vgpr2 {{$}}
 define void @csr() #0 {
   call void asm sideeffect "", "~{v0},~{v36},~{v37}"() #0
   ret void
 }
 
-; CHECK-DAG: subregs_for_super Clobbered Registers: $vgpr0 $vgpr1 $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32 $vgpr0_vgpr1_vgpr2_vgpr3 $vgpr1_vgpr2_vgpr3_vgpr4 $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8 $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16 $vgpr0_vgpr1 $vgpr1_vgpr2 $vgpr0_vgpr1_vgpr2 $vgpr1_vgpr2_vgpr3 {{$}}
+; CHECK-DAG: subregs_for_super Clobbered Registers: $vgpr0 $vgpr1 $vgpr0_hi16 $vgpr1_hi16 $vgpr0_lo16 $vgpr1_lo16 $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32 $vgpr0_vgpr1_vgpr2_vgpr3 $vgpr1_vgpr2_vgpr3_vgpr4 $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8 $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16 $vgpr0_vgpr1 $vgpr1_vgpr2 $vgpr0_vgpr1_vgpr2 $vgpr1_vgpr2_vgpr3 {{$}}
 define void @subregs_for_super() #0 {
   call void asm sideeffect "", "~{v0},~{v1}"() #0
   ret void
 }
 
-; CHECK-DAG: Clobbered Registers: $vgpr0 $vgpr1 $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32 $vgpr0_vgpr1_vgpr2_vgpr3 $vgpr1_vgpr2_vgpr3_vgpr4 $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8 $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16 $vgpr0_vgpr1 $vgpr1_vgpr2 $vgpr0_vgpr1_vgpr2 $vgpr1_vgpr2_vgpr3 {{$}}
+; CHECK-DAG: Clobbered Registers: $vgpr0 $vgpr1 $vgpr0_hi16 $vgpr1_hi16 $vgpr0_lo16 $vgpr1_lo16 $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32 $vgpr0_vgpr1_vgpr2_vgpr3 $vgpr1_vgpr2_vgpr3_vgpr4 $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8 $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16 $vgpr0_vgpr1 $vgpr1_vgpr2 $vgpr0_vgpr1_vgpr2 $vgpr1_vgpr2_vgpr3 {{$}}
 define void @clobbered_reg_with_sub() #0 {
   call void asm sideeffect "", "~{v[0:1]}"() #0
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/known-never-snan.ll b/llvm/test/CodeGen/AMDGPU/known-never-snan.ll
index 25f110f9126a9..ecd71887f8e37 100644
--- a/llvm/test/CodeGen/AMDGPU/known-never-snan.ll
+++ b/llvm/test/CodeGen/AMDGPU/known-never-snan.ll
@@ -667,7 +667,7 @@ declare float @llvm.amdgcn.rsq.f32(float) #1
 declare float @llvm.amdgcn.fract.f32(float) #1
 declare float @llvm.amdgcn.cubeid(float, float, float) #0
 
-attributes #0 = { nounwind }
+attributes #0 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
 attributes #1 = { nounwind readnone speculatable }
 
 !0 = !{float 2.500000e+00}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll
new file mode 100644
index 0000000000000..c2aa65bddb7e0
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll
@@ -0,0 +1,93 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 < %s | FileCheck %s
+
+declare i32 @llvm.amdgcn.ballot.i32(i1)
+
+; Test ballot(0)
+
+define i32 @test0() {
+; CHECK-LABEL: test0:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_waitcnt_vscnt null, 0x0
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    ; implicit-def: $vcc_hi
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 0)
+  ret i32 %ballot
+}
+
+; Test ballot(1)
+
+define i32 @test1() {
+; CHECK-LABEL: test1:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_waitcnt_vscnt null, 0x0
+; CHECK-NEXT:    v_mov_b32_e32 v0, exec_lo
+; CHECK-NEXT:    ; implicit-def: $vcc_hi
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 1)
+  ret i32 %ballot
+}
+
+; Test ballot of a non-comparison operation
+
+define i32 @test2(i32 %x) {
+; CHECK-LABEL: test2:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_waitcnt_vscnt null, 0x0
+; CHECK-NEXT:    v_and_b32_e32 v0, 1, v0
+; CHECK-NEXT:    ; implicit-def: $vcc_hi
+; CHECK-NEXT:    v_cmp_ne_u32_e64 s4, 0, v0
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %trunc = trunc i32 %x to i1
+  %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %trunc)
+  ret i32 %ballot
+}
+
+; Test ballot of comparisons
+
+define i32 @test3(i32 %x, i32 %y) {
+; CHECK-LABEL: test3:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_waitcnt_vscnt null, 0x0
+; CHECK-NEXT:    v_cmp_eq_u32_e64 s4, v0, v1
+; CHECK-NEXT:    ; implicit-def: $vcc_hi
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %x, %y
+  %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %cmp)
+  ret i32 %ballot
+}
+
+define i32 @test4(i32 %x) {
+; CHECK-LABEL: test4:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_waitcnt_vscnt null, 0x0
+; CHECK-NEXT:    v_cmp_lt_i32_e64 s4, 0x62, v0
+; CHECK-NEXT:    ; implicit-def: $vcc_hi
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp sge i32 %x, 99
+  %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %cmp)
+  ret i32 %ballot
+}
+
+define i32 @test5(float %x, float %y) {
+; CHECK-LABEL: test5:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_waitcnt_vscnt null, 0x0
+; CHECK-NEXT:    v_cmp_gt_f32_e64 s4, v0, v1
+; CHECK-NEXT:    ; implicit-def: $vcc_hi
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = fcmp ogt float %x, %y
+  %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %cmp)
+  ret i32 %ballot
+}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.ll
new file mode 100644
index 0000000000000..97678bf309cbc
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.ll
@@ -0,0 +1,88 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -march=amdgcn -mcpu=gfx900 < %s | FileCheck %s
+
+declare i64 @llvm.amdgcn.ballot.i64(i1)
+
+; Test ballot(0)
+
+define i64 @test0() {
+; CHECK-LABEL: test0:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 0)
+  ret i64 %ballot
+}
+
+; Test ballot(1)
+
+define i64 @test1() {
+; CHECK-LABEL: test1:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v0, exec_lo
+; CHECK-NEXT:    v_mov_b32_e32 v1, exec_hi
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 1)
+  ret i64 %ballot
+}
+
+; Test ballot of a non-comparison operation
+
+define i64 @test2(i32 %x) {
+; CHECK-LABEL: test2:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_and_b32_e32 v0, 1, v0
+; CHECK-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v0
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    v_mov_b32_e32 v1, s5
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %trunc = trunc i32 %x to i1
+  %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %trunc)
+  ret i64 %ballot
+}
+
+; Test ballot of comparisons
+
+define i64 @test3(i32 %x, i32 %y) {
+; CHECK-LABEL: test3:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_cmp_eq_u32_e64 s[4:5], v0, v1
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    v_mov_b32_e32 v1, s5
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp eq i32 %x, %y
+  %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %cmp)
+  ret i64 %ballot
+}
+
+define i64 @test4(i32 %x) {
+; CHECK-LABEL: test4:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    s_movk_i32 s4, 0x62
+; CHECK-NEXT:    v_cmp_lt_i32_e64 s[4:5], s4, v0
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    v_mov_b32_e32 v1, s5
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = icmp sge i32 %x, 99
+  %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %cmp)
+  ret i64 %ballot
+}
+
+define i64 @test5(float %x, float %y) {
+; CHECK-LABEL: test5:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_cmp_gt_f32_e64 s[4:5], v0, v1
+; CHECK-NEXT:    v_mov_b32_e32 v0, s4
+; CHECK-NEXT:    v_mov_b32_e32 v1, s5
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %cmp = fcmp ogt float %x, %y
+  %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %cmp)
+  ret i64 %ballot
+}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fmad.ftz.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fmad.ftz.f16.ll
index 000353ee1307a..cd9c47a57c522 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fmad.ftz.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fmad.ftz.f16.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX8 %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=+fp32-denormals -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX8 %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=+fp32-denormals -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
+; RUN: llc -march=amdgcn -mcpu=tonga -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX8 %s
+; RUN: llc -march=amdgcn -mcpu=tonga -denormal-fp-math-f32=ieee -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX8 %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -denormal-fp-math-f32=ieee -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
 
 declare half @llvm.amdgcn.fmad.ftz.f16(half %a, half %b, half %c)
 
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fmad.ftz.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fmad.ftz.ll
index ca35fbcb01823..38d1212a70cc6 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fmad.ftz.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fmad.ftz.ll
@@ -1,7 +1,7 @@
-; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=+fp32-denormals -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=+fp32-denormals -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mcpu=tonga -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mcpu=tonga -denormal-fp-math-f32=ieee -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -denormal-fp-math-f32=ieee -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
 
 declare float @llvm.amdgcn.fmad.ftz.f32(float %a, float %b, float %c)
 
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.ll
index a3c08038b87b1..a5585d5d09d36 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.ll
@@ -139,9 +139,9 @@ define amdgpu_kernel void @unsafe_rsq_rcp_pat_f64(double addrspace(1)* %out, dou
 }
 
 attributes #0 = { nounwind readnone }
-attributes #1 = { nounwind "unsafe-fp-math"="false" "target-features"="-fp32-denormals" }
-attributes #2 = { nounwind "unsafe-fp-math"="true" "target-features"="-fp32-denormals" }
-attributes #3 = { nounwind "unsafe-fp-math"="false" "target-features"="+fp32-denormals" }
-attributes #4 = { nounwind "unsafe-fp-math"="true" "target-features"="+fp32-denormals" }
+attributes #1 = { nounwind "unsafe-fp-math"="false" "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
+attributes #2 = { nounwind "unsafe-fp-math"="true" "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
+attributes #3 = { nounwind "unsafe-fp-math"="false" "denormal-fp-math-f32"="ieee,ieee" }
+attributes #4 = { nounwind "unsafe-fp-math"="true" "denormal-fp-math-f32"="ieee,ieee" }
 
 !0 = !{float 2.500000e+00}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.ll
index f35bb54e955c0..69ee09661e13b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.ll
@@ -357,9 +357,9 @@ define amdgpu_ps i32 @s_buffer_load_imm_neg_bit20(<4 x i32> inreg %desc) {
 ; SI: s_mov_b32 [[K:s[0-9]+]], 0x80000{{$}}
 ; SI: s_buffer_load_dword s0, s[0:3], [[K]]{{$}}
 
-; CI s_buffer_load_dword s0, s[0:3], 0x20000{{$}}
+; CI: s_buffer_load_dword s0, s[0:3], 0x20000{{$}}
 
-; VI s_buffer_load_dword s0, s[0:3], 0x20000{{$}}
+; VI: s_buffer_load_dword s0, s[0:3], 0x80000{{$}}
 define amdgpu_ps i32 @s_buffer_load_imm_bit19(<4 x i32> inreg %desc) {
   %load = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %desc, i32 524288, i32 0)
   ret i32 %load
@@ -369,9 +369,10 @@ define amdgpu_ps i32 @s_buffer_load_imm_bit19(<4 x i32> inreg %desc) {
 ; SI: s_mov_b32 [[K:s[0-9]+]], 0xfff80000{{$}}
 ; SI: s_buffer_load_dword s0, s[0:3], [[K]]{{$}}
 
-; CI s_buffer_load_dword s0, s[0:3], 0x20000{{$}}
+; CI: s_buffer_load_dword s0, s[0:3], 0x3ffe0000{{$}}
 
-; VI s_buffer_load_dword s0, s[0:3], 0x20000{{$}}
+; VI: s_mov_b32 [[K:s[0-9]+]], 0xfff80000{{$}}
+; VI: s_buffer_load_dword s0, s[0:3], [[K]]{{$}}
 define amdgpu_ps i32 @s_buffer_load_imm_neg_bit19(<4 x i32> inreg %desc) {
   %load = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %desc, i32 -524288, i32 0)
   ret i32 %load
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.fmuladd.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.fmuladd.f16.ll
index ff644b08095ef..9e1a7b62d03c4 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.fmuladd.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.fmuladd.f16.ll
@@ -1,9 +1,9 @@
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -mattr=-fp64-fp16-denormals -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=SI -check-prefix=SI-FLUSH %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -mattr=-fp64-fp16-denormals,-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI -check-prefix=VI-FLUSH %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -mattr=+fp64-fp16-denormals -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=SI -check-prefix=SI-DENORM %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -mattr=+fp64-fp16-denormals,-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI -check-prefix=VI-DENORM %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx1010 -mattr=-fp64-fp16-denormals,-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=GFX10 -check-prefix=GFX10-FLUSH %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx1010 -mattr=+fp64-fp16-denormals,-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=GFX10 -check-prefix=GFX10-DENORM %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -denormal-fp-math=preserve-sign -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=SI -check-prefix=SI-FLUSH %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -denormal-fp-math=preserve-sign -denormal-fp-math-f32=preserve-sign -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI -check-prefix=VI-FLUSH %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -denormal-fp-math=ieee -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=SI -check-prefix=SI-DENORM %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -denormal-fp-math=ieee -denormal-fp-math-f32=preserve-sign -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI -check-prefix=VI-DENORM %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx1010 -denormal-fp-math=preserve-sign -denormal-fp-math-f32=preserve-sign -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=GFX10 -check-prefix=GFX10-FLUSH %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx1010 -denormal-fp-math=ieee -denormal-fp-math-f32=preserve-sign -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=GFX10 -check-prefix=GFX10-DENORM %s
 
 declare half @llvm.fmuladd.f16(half %a, half %b, half %c)
 declare <2 x half> @llvm.fmuladd.v2f16(<2 x half> %a, <2 x half> %b, <2 x half> %c)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.maxnum.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.maxnum.f16.ll
index a53002197203b..14ca8225273da 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.maxnum.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.maxnum.f16.ll
@@ -88,7 +88,7 @@ define amdgpu_kernel void @maxnum_f16(
 ; GFX9-NEXT:    s_endpgm
     half addrspace(1)* %r,
     half addrspace(1)* %a,
-    half addrspace(1)* %b) {
+    half addrspace(1)* %b) #0 {
 entry:
   %a.val = load volatile half, half addrspace(1)* %a
   %b.val = load volatile half, half addrspace(1)* %b
@@ -157,7 +157,7 @@ define amdgpu_kernel void @maxnum_f16_imm_a(
 ; GFX9-NEXT:    buffer_store_short v0, off, s[0:3], 0
 ; GFX9-NEXT:    s_endpgm
     half addrspace(1)* %r,
-    half addrspace(1)* %b) {
+    half addrspace(1)* %b) #0 {
 entry:
   %b.val = load half, half addrspace(1)* %b
   %r.val = call half @llvm.maxnum.f16(half 3.0, half %b.val)
@@ -225,7 +225,7 @@ define amdgpu_kernel void @maxnum_f16_imm_b(
 ; GFX9-NEXT:    buffer_store_short v0, off, s[0:3], 0
 ; GFX9-NEXT:    s_endpgm
     half addrspace(1)* %r,
-    half addrspace(1)* %a) {
+    half addrspace(1)* %a) #0 {
 entry:
   %a.val = load half, half addrspace(1)* %a
   %r.val = call half @llvm.maxnum.f16(half %a.val, half 4.0)
@@ -308,7 +308,7 @@ define amdgpu_kernel void @maxnum_v2f16(
 ; GFX9-NEXT:    s_endpgm
     <2 x half> addrspace(1)* %r,
     <2 x half> addrspace(1)* %a,
-    <2 x half> addrspace(1)* %b) {
+    <2 x half> addrspace(1)* %b) #0 {
 entry:
   %a.val = load <2 x half>, <2 x half> addrspace(1)* %a
   %b.val = load <2 x half>, <2 x half> addrspace(1)* %b
@@ -376,7 +376,7 @@ define amdgpu_kernel void @maxnum_v2f16_imm_a(
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX9-NEXT:    s_endpgm
     <2 x half> addrspace(1)* %r,
-    <2 x half> addrspace(1)* %b) {
+    <2 x half> addrspace(1)* %b) #0 {
 entry:
   %b.val = load <2 x half>, <2 x half> addrspace(1)* %b
   %r.val = call <2 x half> @llvm.maxnum.v2f16(<2 x half> <half 3.0, half 4.0>, <2 x half> %b.val)
@@ -443,7 +443,7 @@ define amdgpu_kernel void @maxnum_v2f16_imm_b(
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX9-NEXT:    s_endpgm
     <2 x half> addrspace(1)* %r,
-    <2 x half> addrspace(1)* %a) {
+    <2 x half> addrspace(1)* %a) #0 {
 entry:
   %a.val = load <2 x half>, <2 x half> addrspace(1)* %a
   %r.val = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %a.val, <2 x half> <half 4.0, half 3.0>)
@@ -542,7 +542,7 @@ define amdgpu_kernel void @maxnum_v3f16(
 ; GFX9-NEXT:    s_endpgm
     <3 x half> addrspace(1)* %r,
     <3 x half> addrspace(1)* %a,
-    <3 x half> addrspace(1)* %b) {
+    <3 x half> addrspace(1)* %b) #0 {
 entry:
   %a.val = load <3 x half>, <3 x half> addrspace(1)* %a
   %b.val = load <3 x half>, <3 x half> addrspace(1)* %b
@@ -655,7 +655,7 @@ define amdgpu_kernel void @maxnum_v4f16(
 ; GFX9-NEXT:    s_endpgm
     <4 x half> addrspace(1)* %r,
     <4 x half> addrspace(1)* %a,
-    <4 x half> addrspace(1)* %b) {
+    <4 x half> addrspace(1)* %b) #0 {
 entry:
   %a.val = load <4 x half>, <4 x half> addrspace(1)* %a
   %b.val = load <4 x half>, <4 x half> addrspace(1)* %b
@@ -746,10 +746,12 @@ define amdgpu_kernel void @fmax_v4f16_imm_a(
 ; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX9-NEXT:    s_endpgm
     <4 x half> addrspace(1)* %r,
-    <4 x half> addrspace(1)* %b) {
+    <4 x half> addrspace(1)* %b) #0 {
 entry:
   %b.val = load <4 x half>, <4 x half> addrspace(1)* %b
   %r.val = call <4 x half> @llvm.maxnum.v4f16(<4 x half> <half 8.0, half 2.0, half 3.0, half 4.0>, <4 x half> %b.val)
   store <4 x half> %r.val, <4 x half> addrspace(1)* %r
   ret void
 }
+
+attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.minnum.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.minnum.f16.ll
index fcb483f18c9a9..c28fd3abcbadd 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.minnum.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.minnum.f16.ll
@@ -88,7 +88,7 @@ define amdgpu_kernel void @minnum_f16_ieee(
 ; GFX9-NEXT:    s_endpgm
     half addrspace(1)* %r,
     half addrspace(1)* %a,
-    half addrspace(1)* %b) {
+    half addrspace(1)* %b) #0 {
 entry:
   %a.val = load volatile half, half addrspace(1)* %a
   %b.val = load volatile half, half addrspace(1)* %b
@@ -97,7 +97,7 @@ entry:
   ret void
 }
 
-define amdgpu_ps half @minnum_f16_no_ieee(half %a, half %b) {
+define amdgpu_ps half @minnum_f16_no_ieee(half %a, half %b) #0 {
 ; SI-LABEL: minnum_f16_no_ieee:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    v_cvt_f16_f32_e32 v1, v1
@@ -180,7 +180,7 @@ define amdgpu_kernel void @minnum_f16_imm_a(
 ; GFX9-NEXT:    buffer_store_short v0, off, s[0:3], 0
 ; GFX9-NEXT:    s_endpgm
     half addrspace(1)* %r,
-    half addrspace(1)* %b) {
+    half addrspace(1)* %b) #0 {
 entry:
   %b.val = load half, half addrspace(1)* %b
   %r.val = call half @llvm.minnum.f16(half 3.0, half %b.val)
@@ -248,7 +248,7 @@ define amdgpu_kernel void @minnum_f16_imm_b(
 ; GFX9-NEXT:    buffer_store_short v0, off, s[0:3], 0
 ; GFX9-NEXT:    s_endpgm
     half addrspace(1)* %r,
-    half addrspace(1)* %a) {
+    half addrspace(1)* %a) #0 {
 entry:
   %a.val = load half, half addrspace(1)* %a
   %r.val = call half @llvm.minnum.f16(half %a.val, half 4.0)
@@ -331,7 +331,7 @@ define amdgpu_kernel void @minnum_v2f16_ieee(
 ; GFX9-NEXT:    s_endpgm
     <2 x half> addrspace(1)* %r,
     <2 x half> addrspace(1)* %a,
-    <2 x half> addrspace(1)* %b) {
+    <2 x half> addrspace(1)* %b) #0 {
 entry:
   %a.val = load <2 x half>, <2 x half> addrspace(1)* %a
   %b.val = load <2 x half>, <2 x half> addrspace(1)* %b
@@ -340,7 +340,7 @@ entry:
   ret void
 }
 
-define amdgpu_ps <2 x half> @minnum_v2f16_no_ieee(<2 x half> %a, <2 x half> %b) {
+define amdgpu_ps <2 x half> @minnum_v2f16_no_ieee(<2 x half> %a, <2 x half> %b) #0 {
 ; SI-LABEL: minnum_v2f16_no_ieee:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    v_cvt_f16_f32_e32 v3, v3
@@ -429,7 +429,7 @@ define amdgpu_kernel void @minnum_v2f16_imm_a(
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX9-NEXT:    s_endpgm
     <2 x half> addrspace(1)* %r,
-    <2 x half> addrspace(1)* %b) {
+    <2 x half> addrspace(1)* %b) #0 {
 entry:
   %b.val = load <2 x half>, <2 x half> addrspace(1)* %b
   %r.val = call <2 x half> @llvm.minnum.v2f16(<2 x half> <half 3.0, half 4.0>, <2 x half> %b.val)
@@ -496,7 +496,7 @@ define amdgpu_kernel void @minnum_v2f16_imm_b(
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX9-NEXT:    s_endpgm
     <2 x half> addrspace(1)* %r,
-    <2 x half> addrspace(1)* %a) {
+    <2 x half> addrspace(1)* %a) #0 {
 entry:
   %a.val = load <2 x half>, <2 x half> addrspace(1)* %a
   %r.val = call <2 x half> @llvm.minnum.v2f16(<2 x half> %a.val, <2 x half> <half 4.0, half 3.0>)
@@ -595,7 +595,7 @@ define amdgpu_kernel void @minnum_v3f16(
 ; GFX9-NEXT:    s_endpgm
     <3 x half> addrspace(1)* %r,
     <3 x half> addrspace(1)* %a,
-    <3 x half> addrspace(1)* %b) {
+    <3 x half> addrspace(1)* %b) #0 {
 entry:
   %a.val = load <3 x half>, <3 x half> addrspace(1)* %a
   %b.val = load <3 x half>, <3 x half> addrspace(1)* %b
@@ -708,7 +708,7 @@ define amdgpu_kernel void @minnum_v4f16(
 ; GFX9-NEXT:    s_endpgm
     <4 x half> addrspace(1)* %r,
     <4 x half> addrspace(1)* %a,
-    <4 x half> addrspace(1)* %b) {
+    <4 x half> addrspace(1)* %b) #0 {
 entry:
   %a.val = load <4 x half>, <4 x half> addrspace(1)* %a
   %b.val = load <4 x half>, <4 x half> addrspace(1)* %b
@@ -799,10 +799,12 @@ define amdgpu_kernel void @fmin_v4f16_imm_a(
 ; GFX9-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX9-NEXT:    s_endpgm
     <4 x half> addrspace(1)* %r,
-    <4 x half> addrspace(1)* %b) {
+    <4 x half> addrspace(1)* %b) #0 {
 entry:
   %b.val = load <4 x half>, <4 x half> addrspace(1)* %b
   %r.val = call <4 x half> @llvm.minnum.v4f16(<4 x half> <half 8.0, half 2.0, half 3.0, half 4.0>, <4 x half> %b.val)
   store <4 x half> %r.val, <4 x half> addrspace(1)* %r
   ret void
 }
+
+attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
diff --git a/llvm/test/CodeGen/AMDGPU/load-hi16.ll b/llvm/test/CodeGen/AMDGPU/load-hi16.ll
index 37ca76e9489e6..659a98248939e 100644
--- a/llvm/test/CodeGen/AMDGPU/load-hi16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-hi16.ll
@@ -989,4 +989,24 @@ entry:
   ret <2 x i16> %build1
 }
 
+; FIXME: This test should work without copying of v0.
+;        ds_read_u16_d16_hi preserves low 16 bits of the destination
+;        and ds_write_b16 only reads low 16 bits.
+; GCN: s_waitcnt
+; GFX900:      v_mov_b32_e32 [[COPY:v[0-9]+]], v0
+; GFX900-NEXT: ds_read_u16_d16_hi [[COPY]], v1
+; GFX900-NEXT: ds_write_b16 v1, v0
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: v_mov_b32_e32 v0, [[COPY]]
+; GFX900-NEXT: s_waitcnt
+; GFX900-NEXT: s_setpc_b64
+define <2 x i16> @load_local_hi_v2i16_store_local_lo(i16 %reg, i16 addrspace(3)* %in) #0 {
+entry:
+  %load = load i16, i16 addrspace(3)* %in
+  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build1 = insertelement <2 x i16> %build0, i16 %load, i32 1
+  store volatile i16 %reg, i16 addrspace(3)* %in
+  ret <2 x i16> %build1
+}
+
 attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/load-local-f32.ll b/llvm/test/CodeGen/AMDGPU/load-local-f32.ll
index c33c000731565..a0559c17a1613 100644
--- a/llvm/test/CodeGen/AMDGPU/load-local-f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-local-f32.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-enable-ds128 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
 ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefixes=EG,FUNC %s
 
 ; Testing for ds_read/write_128
diff --git a/llvm/test/CodeGen/AMDGPU/load-local-f64.ll b/llvm/test/CodeGen/AMDGPU/load-local-f64.ll
index e313b38b73d7b..7495860107f5b 100644
--- a/llvm/test/CodeGen/AMDGPU/load-local-f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-local-f64.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -mattr=-enable-ds128 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-enable-ds128 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-enable-ds128 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
 ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefixes=EG,FUNC %s
 
 ; Testing for ds_read_b128
diff --git a/llvm/test/CodeGen/AMDGPU/load-local-i16.ll b/llvm/test/CodeGen/AMDGPU/load-local-i16.ll
index d8d7d98e3088f..357141b36a857 100644
--- a/llvm/test/CodeGen/AMDGPU/load-local-i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-local-i16.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=GCN,SI,SICIVI,FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=GCN,SICIVI,GFX89,FUNC %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=GCN,GFX9,GFX89,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-enable-ds128 -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=GCN,SICIVI,GFX89,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-enable-ds128 -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=GCN,GFX9,GFX89,FUNC %s
 ; RUN: llc -march=r600 -mcpu=redwood -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefix=EG -check-prefix=FUNC %s
 
 ; Testing for ds_read/write_b128
diff --git a/llvm/test/CodeGen/AMDGPU/load-local-i32.ll b/llvm/test/CodeGen/AMDGPU/load-local-i32.ll
index 10887885ef672..0063bb0b6d212 100644
--- a/llvm/test/CodeGen/AMDGPU/load-local-i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-local-i32.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,FUNC %s
-; RUN: llc -march=amdgcn -mcpu=gfx908 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global,-enable-ds128 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global,-enable-ds128 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=gfx908 -mattr=-flat-for-global,-enable-ds128 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,FUNC %s
 ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 
 ; Testing for ds_read/write_128
diff --git a/llvm/test/CodeGen/AMDGPU/load-local-i64.ll b/llvm/test/CodeGen/AMDGPU/load-local-i64.ll
index d91c3b5be9284..8a07640e4f4d3 100644
--- a/llvm/test/CodeGen/AMDGPU/load-local-i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-local-i64.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -mattr=-enable-ds128 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-enable-ds128 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-enable-ds128 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
 ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefixes=EG,FUNC %s
 
 ; Testing for ds_read/write_b128
diff --git a/llvm/test/CodeGen/AMDGPU/load-local-i8.ll b/llvm/test/CodeGen/AMDGPU/load-local-i8.ll
index 02477eff677b2..8137ded1454d8 100644
--- a/llvm/test/CodeGen/AMDGPU/load-local-i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-local-i8.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -march=amdgcn -mtriple=amdgcn-- -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=GCN,SI,SICIVI,FUNC %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=tonga -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=GCN,VI,SICIVI,FUNC %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=GCN,GFX9,FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=tonga -mattr=-enable-ds128 -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=GCN,VI,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=gfx900 -mattr=-enable-ds128 -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefixes=GCN,GFX9,FUNC %s
 ; RUN: llc -march=r600 -mtriple=r600-- -mcpu=redwood -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefix=EG -check-prefix=FUNC %s
 
 ; Testing for ds_read/write_b128
diff --git a/llvm/test/CodeGen/AMDGPU/local-64.ll b/llvm/test/CodeGen/AMDGPU/local-64.ll
index f0dca0780aaef..3e85dd5905fca 100644
--- a/llvm/test/CodeGen/AMDGPU/local-64.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-64.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -march=amdgcn -verify-machineinstrs< %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI,SICIVI %s
-; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs< %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI,SICIVI %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs< %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,SICIVI %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs< %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs< %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI,SICIVI,CIPLUS %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs< %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,SICIVI,CIPLUS %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs< %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,CIPLUS %s
 
 ; GCN-LABEL: {{^}}local_i32_load
 ; SICIVI: s_mov_b32 m0
@@ -165,7 +165,8 @@ define amdgpu_kernel void @local_f64_store_0_offset(double addrspace(3)* %out) n
 ; GFX9-NOT: m0
 
 ; GCN-NOT: add
-; GCN: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset0:14 offset1:15
+; SI: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset0:14 offset1:15
+; CIPLUS: ds_write_b128 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}} offset:112
 ; GCN: s_endpgm
 define amdgpu_kernel void @local_v2i64_store(<2 x i64> addrspace(3)* %out) nounwind {
   %gep = getelementptr <2 x i64>, <2 x i64> addrspace(3)* %out, i32 7
@@ -178,7 +179,10 @@ define amdgpu_kernel void @local_v2i64_store(<2 x i64> addrspace(3)* %out) nounw
 ; GFX9-NOT: m0
 
 ; GCN-NOT: add
-; GCN: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset1:1
+
+; SI: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset1:1{{$}}
+; CIPLUS: ds_write_b128 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]$}}
+
 ; GCN: s_endpgm
 define amdgpu_kernel void @local_v2i64_store_0_offset(<2 x i64> addrspace(3)* %out) nounwind {
   store <2 x i64> <i64 1234, i64 1234>, <2 x i64> addrspace(3)* %out, align 16
@@ -190,8 +194,12 @@ define amdgpu_kernel void @local_v2i64_store_0_offset(<2 x i64> addrspace(3)* %o
 ; GFX9-NOT: m0
 
 ; GCN-NOT: add
-; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset0:30 offset1:31
-; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset0:28 offset1:29
+; SI-DAG: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset0:30 offset1:31
+; SI-DAG: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset0:28 offset1:29
+
+; CIPLUS-DAG: ds_write_b128 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}} offset:224{{$}}
+; CIPLUS-DAG: ds_write_b128 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}} offset:240{{$}}
+
 ; GCN: s_endpgm
 define amdgpu_kernel void @local_v4i64_store(<4 x i64> addrspace(3)* %out) nounwind {
   %gep = getelementptr <4 x i64>, <4 x i64> addrspace(3)* %out, i32 7
@@ -204,8 +212,12 @@ define amdgpu_kernel void @local_v4i64_store(<4 x i64> addrspace(3)* %out) nounw
 ; GFX9-NOT: m0
 
 ; GCN-NOT: add
-; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset0:2 offset1:3
-; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset1:1
+; SI-DAG: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset0:2 offset1:3
+; SI-DAG: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset1:1
+
+; CIPLUS-DAG: ds_write_b128 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]$}}
+; CIPLUS-DAG: ds_write_b128 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}} offset:16{{$}}
+
 ; GCN: s_endpgm
 define amdgpu_kernel void @local_v4i64_store_0_offset(<4 x i64> addrspace(3)* %out) nounwind {
   store <4 x i64> <i64 1234, i64 1234, i64 1234, i64 1234>, <4 x i64> addrspace(3)* %out, align 16
diff --git a/llvm/test/CodeGen/AMDGPU/loop_break.ll b/llvm/test/CodeGen/AMDGPU/loop_break.ll
index 6e2868a941677..4b3fb1aa07914 100644
--- a/llvm/test/CodeGen/AMDGPU/loop_break.ll
+++ b/llvm/test/CodeGen/AMDGPU/loop_break.ll
@@ -499,8 +499,8 @@ define amdgpu_kernel void @invert_true_phi_cond_break_loop(i32 %arg) #0 {
 ; GCN-NEXT:    s_or_b64 s[4:5], s[4:5], s[8:9]
 ; GCN-NEXT:  BB5_3: ; %Flow
 ; GCN-NEXT:    ; in Loop: Header=BB5_1 Depth=1
-; GCN-NEXT:    s_add_i32 s6, s6, 1
 ; GCN-NEXT:    s_xor_b64 s[8:9], s[4:5], -1
+; GCN-NEXT:    s_add_i32 s6, s6, 1
 ; GCN-NEXT:    s_and_b64 s[8:9], exec, s[8:9]
 ; GCN-NEXT:    s_or_b64 s[0:1], s[8:9], s[0:1]
 ; GCN-NEXT:    s_andn2_b64 exec, exec, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll b/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll
index 29206d2f5cf8c..b541ed31e6797 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll
@@ -1,17 +1,40 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -amdgpu-lower-intrinsics %s | FileCheck -check-prefix=OPT %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -amdgpu-lower-intrinsics -amdgpu-mem-intrinsic-expand-size=1024 %s | FileCheck -check-prefixes=OPT,MAX1024 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -amdgpu-lower-intrinsics -amdgpu-mem-intrinsic-expand-size=-1 %s | FileCheck -check-prefixes=OPT,ALL %s
 
 declare void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* nocapture, i8 addrspace(1)* nocapture readonly, i64, i1) #1
 declare void @llvm.memcpy.p1i8.p3i8.i32(i8 addrspace(1)* nocapture, i8 addrspace(3)* nocapture readonly, i32, i1) #1
+declare void @llvm.memcpy.p3i8.p1i8.i32(i8 addrspace(3)* nocapture, i8 addrspace(1)* nocapture readonly, i32, i1) #1
+declare void @llvm.memcpy.p5i8.p5i8.i32(i8 addrspace(5)* nocapture, i8 addrspace(5)* nocapture readonly, i32, i1) #1
+declare void @llvm.memcpy.p3i8.p3i8.i32(i8 addrspace(3)* nocapture, i8 addrspace(3)* nocapture readonly, i32, i1) #1
 
 declare void @llvm.memmove.p1i8.p1i8.i64(i8 addrspace(1)* nocapture, i8 addrspace(1)* nocapture readonly, i64, i1) #1
+declare void @llvm.memmove.p1i8.p3i8.i32(i8 addrspace(1)* nocapture, i8 addrspace(3)* nocapture readonly, i32, i1) #1
+declare void @llvm.memmove.p5i8.p5i8.i32(i8 addrspace(5)* nocapture, i8 addrspace(5)* nocapture readonly, i32, i1) #1
+
 declare void @llvm.memset.p1i8.i64(i8 addrspace(1)* nocapture, i8, i64, i1) #1
 
 ; Test the upper bound for sizes to leave
 define amdgpu_kernel void @max_size_small_static_memcpy_caller0(i8 addrspace(1)* %dst, i8 addrspace(1)* %src) #0 {
-; OPT-LABEL: @max_size_small_static_memcpy_caller0(
-; OPT-NEXT:    call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* [[DST:%.*]], i8 addrspace(1)* [[SRC:%.*]], i64 1024, i1 false)
-; OPT-NEXT:    ret void
+; MAX1024-LABEL: @max_size_small_static_memcpy_caller0(
+; MAX1024-NEXT:    call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* [[DST:%.*]], i8 addrspace(1)* [[SRC:%.*]], i64 1024, i1 false)
+; MAX1024-NEXT:    ret void
+;
+; ALL-LABEL: @max_size_small_static_memcpy_caller0(
+; ALL-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SRC:%.*]] to <4 x i32> addrspace(1)*
+; ALL-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(1)* [[DST:%.*]] to <4 x i32> addrspace(1)*
+; ALL-NEXT:    br label [[LOAD_STORE_LOOP:%.*]]
+; ALL:       load-store-loop:
+; ALL-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[LOAD_STORE_LOOP]] ]
+; ALL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP1]], i64 [[LOOP_INDEX]]
+; ALL-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32> addrspace(1)* [[TMP3]], align 1
+; ALL-NEXT:    [[TMP5:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP2]], i64 [[LOOP_INDEX]]
+; ALL-NEXT:    store <4 x i32> [[TMP4]], <4 x i32> addrspace(1)* [[TMP5]], align 1
+; ALL-NEXT:    [[TMP6]] = add i64 [[LOOP_INDEX]], 1
+; ALL-NEXT:    [[TMP7:%.*]] = icmp ult i64 [[TMP6]], 64
+; ALL-NEXT:    br i1 [[TMP7]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
+; ALL:       memcpy-split:
+; ALL-NEXT:    ret void
 ;
   call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* %dst, i8 addrspace(1)* %src, i64 1024, i1 false)
   ret void
@@ -20,17 +43,25 @@ define amdgpu_kernel void @max_size_small_static_memcpy_caller0(i8 addrspace(1)*
 ; Smallest static size which will be expanded
 define amdgpu_kernel void @min_size_large_static_memcpy_caller0(i8 addrspace(1)* %dst, i8 addrspace(1)* %src) #0 {
 ; OPT-LABEL: @min_size_large_static_memcpy_caller0(
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SRC:%.*]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(1)* [[DST:%.*]] to <4 x i32> addrspace(1)*
 ; OPT-NEXT:    br label [[LOAD_STORE_LOOP:%.*]]
 ; OPT:       load-store-loop:
-; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], [[LOAD_STORE_LOOP]] ]
-; OPT-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[SRC:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP2:%.*]] = load i8, i8 addrspace(1)* [[TMP1]], align 1
-; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[DST:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    store i8 [[TMP2]], i8 addrspace(1)* [[TMP3]], align 1
-; OPT-NEXT:    [[TMP4]] = add i64 [[LOOP_INDEX]], 1
-; OPT-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 1025
-; OPT-NEXT:    br i1 [[TMP5]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[LOAD_STORE_LOOP]] ]
+; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP1]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32> addrspace(1)* [[TMP3]], align 1
+; OPT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP2]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    store <4 x i32> [[TMP4]], <4 x i32> addrspace(1)* [[TMP5]], align 1
+; OPT-NEXT:    [[TMP6]] = add i64 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP7:%.*]] = icmp ult i64 [[TMP6]], 64
+; OPT-NEXT:    br i1 [[TMP7]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
 ; OPT:       memcpy-split:
+; OPT-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP1]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP8]], i64 1024
+; OPT-NEXT:    [[TMP10:%.*]] = load i8, i8 addrspace(1)* [[TMP9]], align 1
+; OPT-NEXT:    [[TMP11:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP2]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP11]], i64 1024
+; OPT-NEXT:    store i8 [[TMP10]], i8 addrspace(1)* [[TMP12]], align 1
 ; OPT-NEXT:    ret void
 ;
   call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* %dst, i8 addrspace(1)* %src, i64 1025, i1 false)
@@ -38,9 +69,38 @@ define amdgpu_kernel void @min_size_large_static_memcpy_caller0(i8 addrspace(1)*
 }
 
 define amdgpu_kernel void @max_size_small_static_memmove_caller0(i8 addrspace(1)* %dst, i8 addrspace(1)* %src) #0 {
-; OPT-LABEL: @max_size_small_static_memmove_caller0(
-; OPT-NEXT:    call void @llvm.memmove.p1i8.p1i8.i64(i8 addrspace(1)* [[DST:%.*]], i8 addrspace(1)* [[SRC:%.*]], i64 1024, i1 false)
-; OPT-NEXT:    ret void
+; MAX1024-LABEL: @max_size_small_static_memmove_caller0(
+; MAX1024-NEXT:    call void @llvm.memmove.p1i8.p1i8.i64(i8 addrspace(1)* [[DST:%.*]], i8 addrspace(1)* [[SRC:%.*]], i64 1024, i1 false)
+; MAX1024-NEXT:    ret void
+;
+; ALL-LABEL: @max_size_small_static_memmove_caller0(
+; ALL-NEXT:    [[COMPARE_SRC_DST:%.*]] = icmp ult i8 addrspace(1)* [[SRC:%.*]], [[DST:%.*]]
+; ALL-NEXT:    [[COMPARE_N_TO_0:%.*]] = icmp eq i64 1024, 0
+; ALL-NEXT:    br i1 [[COMPARE_SRC_DST]], label [[COPY_BACKWARDS:%.*]], label [[COPY_FORWARD:%.*]]
+; ALL:       copy_backwards:
+; ALL-NEXT:    br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE:%.*]], label [[COPY_BACKWARDS_LOOP:%.*]]
+; ALL:       copy_backwards_loop:
+; ALL-NEXT:    [[TMP1:%.*]] = phi i64 [ [[INDEX_PTR:%.*]], [[COPY_BACKWARDS_LOOP]] ], [ 1024, [[COPY_BACKWARDS]] ]
+; ALL-NEXT:    [[INDEX_PTR]] = sub i64 [[TMP1]], 1
+; ALL-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[SRC]], i64 [[INDEX_PTR]]
+; ALL-NEXT:    [[ELEMENT:%.*]] = load i8, i8 addrspace(1)* [[TMP2]], align 1
+; ALL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[DST]], i64 [[INDEX_PTR]]
+; ALL-NEXT:    store i8 [[ELEMENT]], i8 addrspace(1)* [[TMP3]], align 1
+; ALL-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_PTR]], 0
+; ALL-NEXT:    br i1 [[TMP4]], label [[MEMMOVE_DONE]], label [[COPY_BACKWARDS_LOOP]]
+; ALL:       copy_forward:
+; ALL-NEXT:    br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP:%.*]]
+; ALL:       copy_forward_loop:
+; ALL-NEXT:    [[INDEX_PTR1:%.*]] = phi i64 [ [[INDEX_INCREMENT:%.*]], [[COPY_FORWARD_LOOP]] ], [ 0, [[COPY_FORWARD]] ]
+; ALL-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[SRC]], i64 [[INDEX_PTR1]]
+; ALL-NEXT:    [[ELEMENT2:%.*]] = load i8, i8 addrspace(1)* [[TMP5]], align 1
+; ALL-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[DST]], i64 [[INDEX_PTR1]]
+; ALL-NEXT:    store i8 [[ELEMENT2]], i8 addrspace(1)* [[TMP6]], align 1
+; ALL-NEXT:    [[INDEX_INCREMENT]] = add i64 [[INDEX_PTR1]], 1
+; ALL-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_INCREMENT]], 1024
+; ALL-NEXT:    br i1 [[TMP7]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP]]
+; ALL:       memmove_done:
+; ALL-NEXT:    ret void
 ;
   call void @llvm.memmove.p1i8.p1i8.i64(i8 addrspace(1)* %dst, i8 addrspace(1)* %src, i64 1024, i1 false)
   ret void
@@ -81,9 +141,21 @@ define amdgpu_kernel void @min_size_large_static_memmove_caller0(i8 addrspace(1)
 }
 
 define amdgpu_kernel void @max_size_small_static_memset_caller0(i8 addrspace(1)* %dst, i8 %val) #0 {
-; OPT-LABEL: @max_size_small_static_memset_caller0(
-; OPT-NEXT:    call void @llvm.memset.p1i8.i64(i8 addrspace(1)* [[DST:%.*]], i8 [[VAL:%.*]], i64 1024, i1 false)
-; OPT-NEXT:    ret void
+; MAX1024-LABEL: @max_size_small_static_memset_caller0(
+; MAX1024-NEXT:    call void @llvm.memset.p1i8.i64(i8 addrspace(1)* [[DST:%.*]], i8 [[VAL:%.*]], i64 1024, i1 false)
+; MAX1024-NEXT:    ret void
+;
+; ALL-LABEL: @max_size_small_static_memset_caller0(
+; ALL-NEXT:    br i1 false, label [[SPLIT:%.*]], label [[LOADSTORELOOP:%.*]]
+; ALL:       loadstoreloop:
+; ALL-NEXT:    [[TMP1:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP3:%.*]], [[LOADSTORELOOP]] ]
+; ALL-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[DST:%.*]], i64 [[TMP1]]
+; ALL-NEXT:    store i8 [[VAL:%.*]], i8 addrspace(1)* [[TMP2]], align 1
+; ALL-NEXT:    [[TMP3]] = add i64 [[TMP1]], 1
+; ALL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP3]], 1024
+; ALL-NEXT:    br i1 [[TMP4]], label [[LOADSTORELOOP]], label [[SPLIT]]
+; ALL:       split:
+; ALL-NEXT:    ret void
 ;
   call void @llvm.memset.p1i8.i64(i8 addrspace(1)* %dst, i8 %val, i64 1024, i1 false)
   ret void
@@ -108,19 +180,39 @@ define amdgpu_kernel void @min_size_large_static_memset_caller0(i8 addrspace(1)*
 
 define amdgpu_kernel void @variable_memcpy_caller0(i8 addrspace(1)* %dst, i8 addrspace(1)* %src, i64 %n) #0 {
 ; OPT-LABEL: @variable_memcpy_caller0(
-; OPT-NEXT:    [[TMP1:%.*]] = icmp ne i64 [[N:%.*]], 0
-; OPT-NEXT:    br i1 [[TMP1]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[POST_LOOP_MEMCPY_EXPANSION:%.*]]
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SRC:%.*]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(1)* [[DST:%.*]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    [[TMP3:%.*]] = udiv i64 [[N:%.*]], 16
+; OPT-NEXT:    [[TMP4:%.*]] = urem i64 [[N]], 16
+; OPT-NEXT:    [[TMP5:%.*]] = sub i64 [[N]], [[TMP4]]
+; OPT-NEXT:    [[TMP6:%.*]] = icmp ne i64 [[TMP3]], 0
+; OPT-NEXT:    br i1 [[TMP6]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[LOOP_MEMCPY_RESIDUAL_HEADER:%.*]]
 ; OPT:       loop-memcpy-expansion:
-; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
-; OPT-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[SRC:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP3:%.*]] = load i8, i8 addrspace(1)* [[TMP2]], align 1
-; OPT-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[DST:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    store i8 [[TMP3]], i8 addrspace(1)* [[TMP4]], align 1
-; OPT-NEXT:    [[TMP5]] = add i64 [[LOOP_INDEX]], 1
-; OPT-NEXT:    [[TMP6:%.*]] = icmp ult i64 [[TMP5]], [[N]]
-; OPT-NEXT:    br i1 [[TMP6]], label [[LOOP_MEMCPY_EXPANSION]], label [[POST_LOOP_MEMCPY_EXPANSION]]
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP10:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
+; OPT-NEXT:    [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP1]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32> addrspace(1)* [[TMP7]], align 1
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP2]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    store <4 x i32> [[TMP8]], <4 x i32> addrspace(1)* [[TMP9]], align 1
+; OPT-NEXT:    [[TMP10]] = add i64 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP11:%.*]] = icmp ult i64 [[TMP10]], [[TMP3]]
+; OPT-NEXT:    br i1 [[TMP11]], label [[LOOP_MEMCPY_EXPANSION]], label [[LOOP_MEMCPY_RESIDUAL_HEADER]]
+; OPT:       loop-memcpy-residual:
+; OPT-NEXT:    [[RESIDUAL_LOOP_INDEX:%.*]] = phi i64 [ 0, [[LOOP_MEMCPY_RESIDUAL_HEADER]] ], [ [[TMP18:%.*]], [[LOOP_MEMCPY_RESIDUAL:%.*]] ]
+; OPT-NEXT:    [[TMP12:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP1]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP13:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP2]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP14:%.*]] = add i64 [[TMP5]], [[RESIDUAL_LOOP_INDEX]]
+; OPT-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP12]], i64 [[TMP14]]
+; OPT-NEXT:    [[TMP16:%.*]] = load i8, i8 addrspace(1)* [[TMP15]], align 1
+; OPT-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP13]], i64 [[TMP14]]
+; OPT-NEXT:    store i8 [[TMP16]], i8 addrspace(1)* [[TMP17]], align 1
+; OPT-NEXT:    [[TMP18]] = add i64 [[RESIDUAL_LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP19:%.*]] = icmp ult i64 [[TMP18]], [[TMP4]]
+; OPT-NEXT:    br i1 [[TMP19]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION:%.*]]
 ; OPT:       post-loop-memcpy-expansion:
 ; OPT-NEXT:    ret void
+; OPT:       loop-memcpy-residual-header:
+; OPT-NEXT:    [[TMP20:%.*]] = icmp ne i64 [[TMP4]], 0
+; OPT-NEXT:    br i1 [[TMP20]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION]]
 ;
   call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* %dst, i8 addrspace(1)* %src, i64 %n, i1 false)
   ret void
@@ -128,19 +220,39 @@ define amdgpu_kernel void @variable_memcpy_caller0(i8 addrspace(1)* %dst, i8 add
 
 define amdgpu_kernel void @variable_memcpy_caller1(i8 addrspace(1)* %dst, i8 addrspace(1)* %src, i64 %n) #0 {
 ; OPT-LABEL: @variable_memcpy_caller1(
-; OPT-NEXT:    [[TMP1:%.*]] = icmp ne i64 [[N:%.*]], 0
-; OPT-NEXT:    br i1 [[TMP1]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[POST_LOOP_MEMCPY_EXPANSION:%.*]]
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SRC:%.*]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(1)* [[DST:%.*]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    [[TMP3:%.*]] = udiv i64 [[N:%.*]], 16
+; OPT-NEXT:    [[TMP4:%.*]] = urem i64 [[N]], 16
+; OPT-NEXT:    [[TMP5:%.*]] = sub i64 [[N]], [[TMP4]]
+; OPT-NEXT:    [[TMP6:%.*]] = icmp ne i64 [[TMP3]], 0
+; OPT-NEXT:    br i1 [[TMP6]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[LOOP_MEMCPY_RESIDUAL_HEADER:%.*]]
 ; OPT:       loop-memcpy-expansion:
-; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
-; OPT-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[SRC:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP3:%.*]] = load i8, i8 addrspace(1)* [[TMP2]], align 1
-; OPT-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[DST:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    store i8 [[TMP3]], i8 addrspace(1)* [[TMP4]], align 1
-; OPT-NEXT:    [[TMP5]] = add i64 [[LOOP_INDEX]], 1
-; OPT-NEXT:    [[TMP6:%.*]] = icmp ult i64 [[TMP5]], [[N]]
-; OPT-NEXT:    br i1 [[TMP6]], label [[LOOP_MEMCPY_EXPANSION]], label [[POST_LOOP_MEMCPY_EXPANSION]]
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP10:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
+; OPT-NEXT:    [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP1]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32> addrspace(1)* [[TMP7]], align 1
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP2]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    store <4 x i32> [[TMP8]], <4 x i32> addrspace(1)* [[TMP9]], align 1
+; OPT-NEXT:    [[TMP10]] = add i64 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP11:%.*]] = icmp ult i64 [[TMP10]], [[TMP3]]
+; OPT-NEXT:    br i1 [[TMP11]], label [[LOOP_MEMCPY_EXPANSION]], label [[LOOP_MEMCPY_RESIDUAL_HEADER]]
+; OPT:       loop-memcpy-residual:
+; OPT-NEXT:    [[RESIDUAL_LOOP_INDEX:%.*]] = phi i64 [ 0, [[LOOP_MEMCPY_RESIDUAL_HEADER]] ], [ [[TMP18:%.*]], [[LOOP_MEMCPY_RESIDUAL:%.*]] ]
+; OPT-NEXT:    [[TMP12:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP1]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP13:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP2]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP14:%.*]] = add i64 [[TMP5]], [[RESIDUAL_LOOP_INDEX]]
+; OPT-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP12]], i64 [[TMP14]]
+; OPT-NEXT:    [[TMP16:%.*]] = load i8, i8 addrspace(1)* [[TMP15]], align 1
+; OPT-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP13]], i64 [[TMP14]]
+; OPT-NEXT:    store i8 [[TMP16]], i8 addrspace(1)* [[TMP17]], align 1
+; OPT-NEXT:    [[TMP18]] = add i64 [[RESIDUAL_LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP19:%.*]] = icmp ult i64 [[TMP18]], [[TMP4]]
+; OPT-NEXT:    br i1 [[TMP19]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION:%.*]]
 ; OPT:       post-loop-memcpy-expansion:
 ; OPT-NEXT:    ret void
+; OPT:       loop-memcpy-residual-header:
+; OPT-NEXT:    [[TMP20:%.*]] = icmp ne i64 [[TMP4]], 0
+; OPT-NEXT:    br i1 [[TMP20]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION]]
 ;
   call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* %dst, i8 addrspace(1)* %src, i64 %n, i1 false)
   ret void
@@ -148,31 +260,71 @@ define amdgpu_kernel void @variable_memcpy_caller1(i8 addrspace(1)* %dst, i8 add
 
 define amdgpu_kernel void @memcpy_multi_use_one_function(i8 addrspace(1)* %dst0, i8 addrspace(1)* %dst1, i8 addrspace(1)* %src, i64 %n, i64 %m) #0 {
 ; OPT-LABEL: @memcpy_multi_use_one_function(
-; OPT-NEXT:    [[TMP1:%.*]] = icmp ne i64 [[N:%.*]], 0
-; OPT-NEXT:    br i1 [[TMP1]], label [[LOOP_MEMCPY_EXPANSION2:%.*]], label [[POST_LOOP_MEMCPY_EXPANSION1:%.*]]
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SRC:%.*]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(1)* [[DST0:%.*]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    [[TMP3:%.*]] = udiv i64 [[N:%.*]], 16
+; OPT-NEXT:    [[TMP4:%.*]] = urem i64 [[N]], 16
+; OPT-NEXT:    [[TMP5:%.*]] = sub i64 [[N]], [[TMP4]]
+; OPT-NEXT:    [[TMP6:%.*]] = icmp ne i64 [[TMP3]], 0
+; OPT-NEXT:    br i1 [[TMP6]], label [[LOOP_MEMCPY_EXPANSION2:%.*]], label [[LOOP_MEMCPY_RESIDUAL_HEADER5:%.*]]
 ; OPT:       loop-memcpy-expansion2:
-; OPT-NEXT:    [[LOOP_INDEX3:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[LOOP_MEMCPY_EXPANSION2]] ]
-; OPT-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[SRC:%.*]], i64 [[LOOP_INDEX3]]
-; OPT-NEXT:    [[TMP3:%.*]] = load i8, i8 addrspace(1)* [[TMP2]], align 1
-; OPT-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[DST0:%.*]], i64 [[LOOP_INDEX3]]
-; OPT-NEXT:    store i8 [[TMP3]], i8 addrspace(1)* [[TMP4]], align 1
-; OPT-NEXT:    [[TMP5]] = add i64 [[LOOP_INDEX3]], 1
-; OPT-NEXT:    [[TMP6:%.*]] = icmp ult i64 [[TMP5]], [[N]]
-; OPT-NEXT:    br i1 [[TMP6]], label [[LOOP_MEMCPY_EXPANSION2]], label [[POST_LOOP_MEMCPY_EXPANSION1]]
+; OPT-NEXT:    [[LOOP_INDEX3:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP10:%.*]], [[LOOP_MEMCPY_EXPANSION2]] ]
+; OPT-NEXT:    [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP1]], i64 [[LOOP_INDEX3]]
+; OPT-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32> addrspace(1)* [[TMP7]], align 1
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP2]], i64 [[LOOP_INDEX3]]
+; OPT-NEXT:    store <4 x i32> [[TMP8]], <4 x i32> addrspace(1)* [[TMP9]], align 1
+; OPT-NEXT:    [[TMP10]] = add i64 [[LOOP_INDEX3]], 1
+; OPT-NEXT:    [[TMP11:%.*]] = icmp ult i64 [[TMP10]], [[TMP3]]
+; OPT-NEXT:    br i1 [[TMP11]], label [[LOOP_MEMCPY_EXPANSION2]], label [[LOOP_MEMCPY_RESIDUAL_HEADER5]]
+; OPT:       loop-memcpy-residual4:
+; OPT-NEXT:    [[RESIDUAL_LOOP_INDEX6:%.*]] = phi i64 [ 0, [[LOOP_MEMCPY_RESIDUAL_HEADER5]] ], [ [[TMP18:%.*]], [[LOOP_MEMCPY_RESIDUAL4:%.*]] ]
+; OPT-NEXT:    [[TMP12:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP1]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP13:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP2]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP14:%.*]] = add i64 [[TMP5]], [[RESIDUAL_LOOP_INDEX6]]
+; OPT-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP12]], i64 [[TMP14]]
+; OPT-NEXT:    [[TMP16:%.*]] = load i8, i8 addrspace(1)* [[TMP15]], align 1
+; OPT-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP13]], i64 [[TMP14]]
+; OPT-NEXT:    store i8 [[TMP16]], i8 addrspace(1)* [[TMP17]], align 1
+; OPT-NEXT:    [[TMP18]] = add i64 [[RESIDUAL_LOOP_INDEX6]], 1
+; OPT-NEXT:    [[TMP19:%.*]] = icmp ult i64 [[TMP18]], [[TMP4]]
+; OPT-NEXT:    br i1 [[TMP19]], label [[LOOP_MEMCPY_RESIDUAL4]], label [[POST_LOOP_MEMCPY_EXPANSION1:%.*]]
 ; OPT:       post-loop-memcpy-expansion1:
-; OPT-NEXT:    [[TMP7:%.*]] = icmp ne i64 [[M:%.*]], 0
-; OPT-NEXT:    br i1 [[TMP7]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[POST_LOOP_MEMCPY_EXPANSION:%.*]]
+; OPT-NEXT:    [[TMP20:%.*]] = bitcast i8 addrspace(1)* [[SRC]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    [[TMP21:%.*]] = bitcast i8 addrspace(1)* [[DST1:%.*]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    [[TMP22:%.*]] = udiv i64 [[M:%.*]], 16
+; OPT-NEXT:    [[TMP23:%.*]] = urem i64 [[M]], 16
+; OPT-NEXT:    [[TMP24:%.*]] = sub i64 [[M]], [[TMP23]]
+; OPT-NEXT:    [[TMP25:%.*]] = icmp ne i64 [[TMP22]], 0
+; OPT-NEXT:    br i1 [[TMP25]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[LOOP_MEMCPY_RESIDUAL_HEADER:%.*]]
 ; OPT:       loop-memcpy-expansion:
-; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[POST_LOOP_MEMCPY_EXPANSION1]] ], [ [[TMP11:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
-; OPT-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[SRC]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP9:%.*]] = load i8, i8 addrspace(1)* [[TMP8]], align 1
-; OPT-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[DST1:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    store i8 [[TMP9]], i8 addrspace(1)* [[TMP10]], align 1
-; OPT-NEXT:    [[TMP11]] = add i64 [[LOOP_INDEX]], 1
-; OPT-NEXT:    [[TMP12:%.*]] = icmp ult i64 [[TMP11]], [[M]]
-; OPT-NEXT:    br i1 [[TMP12]], label [[LOOP_MEMCPY_EXPANSION]], label [[POST_LOOP_MEMCPY_EXPANSION]]
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[POST_LOOP_MEMCPY_EXPANSION1]] ], [ [[TMP29:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
+; OPT-NEXT:    [[TMP26:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP20]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP27:%.*]] = load <4 x i32>, <4 x i32> addrspace(1)* [[TMP26]], align 1
+; OPT-NEXT:    [[TMP28:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP21]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    store <4 x i32> [[TMP27]], <4 x i32> addrspace(1)* [[TMP28]], align 1
+; OPT-NEXT:    [[TMP29]] = add i64 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP30:%.*]] = icmp ult i64 [[TMP29]], [[TMP22]]
+; OPT-NEXT:    br i1 [[TMP30]], label [[LOOP_MEMCPY_EXPANSION]], label [[LOOP_MEMCPY_RESIDUAL_HEADER]]
+; OPT:       loop-memcpy-residual:
+; OPT-NEXT:    [[RESIDUAL_LOOP_INDEX:%.*]] = phi i64 [ 0, [[LOOP_MEMCPY_RESIDUAL_HEADER]] ], [ [[TMP37:%.*]], [[LOOP_MEMCPY_RESIDUAL:%.*]] ]
+; OPT-NEXT:    [[TMP31:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP20]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP32:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP21]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP33:%.*]] = add i64 [[TMP24]], [[RESIDUAL_LOOP_INDEX]]
+; OPT-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP31]], i64 [[TMP33]]
+; OPT-NEXT:    [[TMP35:%.*]] = load i8, i8 addrspace(1)* [[TMP34]], align 1
+; OPT-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP32]], i64 [[TMP33]]
+; OPT-NEXT:    store i8 [[TMP35]], i8 addrspace(1)* [[TMP36]], align 1
+; OPT-NEXT:    [[TMP37]] = add i64 [[RESIDUAL_LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP38:%.*]] = icmp ult i64 [[TMP37]], [[TMP23]]
+; OPT-NEXT:    br i1 [[TMP38]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION:%.*]]
 ; OPT:       post-loop-memcpy-expansion:
 ; OPT-NEXT:    ret void
+; OPT:       loop-memcpy-residual-header:
+; OPT-NEXT:    [[TMP39:%.*]] = icmp ne i64 [[TMP23]], 0
+; OPT-NEXT:    br i1 [[TMP39]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION]]
+; OPT:       loop-memcpy-residual-header5:
+; OPT-NEXT:    [[TMP40:%.*]] = icmp ne i64 [[TMP4]], 0
+; OPT-NEXT:    br i1 [[TMP40]], label [[LOOP_MEMCPY_RESIDUAL4]], label [[POST_LOOP_MEMCPY_EXPANSION1]]
 ;
   call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* %dst0, i8 addrspace(1)* %src, i64 %n, i1 false)
   call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* %dst1, i8 addrspace(1)* %src, i64 %m, i1 false)
@@ -181,19 +333,39 @@ define amdgpu_kernel void @memcpy_multi_use_one_function(i8 addrspace(1)* %dst0,
 
 define amdgpu_kernel void @memcpy_alt_type(i8 addrspace(1)* %dst, i8 addrspace(3)* %src, i32 %n) #0 {
 ; OPT-LABEL: @memcpy_alt_type(
-; OPT-NEXT:    [[TMP1:%.*]] = icmp ne i32 [[N:%.*]], 0
-; OPT-NEXT:    br i1 [[TMP1]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[POST_LOOP_MEMCPY_EXPANSION:%.*]]
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(3)* [[SRC:%.*]] to <2 x i32> addrspace(3)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(1)* [[DST:%.*]] to <2 x i32> addrspace(1)*
+; OPT-NEXT:    [[TMP3:%.*]] = udiv i32 [[N:%.*]], 8
+; OPT-NEXT:    [[TMP4:%.*]] = urem i32 [[N]], 8
+; OPT-NEXT:    [[TMP5:%.*]] = sub i32 [[N]], [[TMP4]]
+; OPT-NEXT:    [[TMP6:%.*]] = icmp ne i32 [[TMP3]], 0
+; OPT-NEXT:    br i1 [[TMP6]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[LOOP_MEMCPY_RESIDUAL_HEADER:%.*]]
 ; OPT:       loop-memcpy-expansion:
-; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
-; OPT-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8 addrspace(3)* [[SRC:%.*]], i32 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP3:%.*]] = load i8, i8 addrspace(3)* [[TMP2]], align 1
-; OPT-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[DST:%.*]], i32 [[LOOP_INDEX]]
-; OPT-NEXT:    store i8 [[TMP3]], i8 addrspace(1)* [[TMP4]], align 1
-; OPT-NEXT:    [[TMP5]] = add i32 [[LOOP_INDEX]], 1
-; OPT-NEXT:    [[TMP6:%.*]] = icmp ult i32 [[TMP5]], [[N]]
-; OPT-NEXT:    br i1 [[TMP6]], label [[LOOP_MEMCPY_EXPANSION]], label [[POST_LOOP_MEMCPY_EXPANSION]]
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP10:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
+; OPT-NEXT:    [[TMP7:%.*]] = getelementptr inbounds <2 x i32>, <2 x i32> addrspace(3)* [[TMP1]], i32 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP8:%.*]] = load <2 x i32>, <2 x i32> addrspace(3)* [[TMP7]], align 1
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds <2 x i32>, <2 x i32> addrspace(1)* [[TMP2]], i32 [[LOOP_INDEX]]
+; OPT-NEXT:    store <2 x i32> [[TMP8]], <2 x i32> addrspace(1)* [[TMP9]], align 1
+; OPT-NEXT:    [[TMP10]] = add i32 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP11:%.*]] = icmp ult i32 [[TMP10]], [[TMP3]]
+; OPT-NEXT:    br i1 [[TMP11]], label [[LOOP_MEMCPY_EXPANSION]], label [[LOOP_MEMCPY_RESIDUAL_HEADER]]
+; OPT:       loop-memcpy-residual:
+; OPT-NEXT:    [[RESIDUAL_LOOP_INDEX:%.*]] = phi i32 [ 0, [[LOOP_MEMCPY_RESIDUAL_HEADER]] ], [ [[TMP18:%.*]], [[LOOP_MEMCPY_RESIDUAL:%.*]] ]
+; OPT-NEXT:    [[TMP12:%.*]] = bitcast <2 x i32> addrspace(3)* [[TMP1]] to i8 addrspace(3)*
+; OPT-NEXT:    [[TMP13:%.*]] = bitcast <2 x i32> addrspace(1)* [[TMP2]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP14:%.*]] = add i32 [[TMP5]], [[RESIDUAL_LOOP_INDEX]]
+; OPT-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, i8 addrspace(3)* [[TMP12]], i32 [[TMP14]]
+; OPT-NEXT:    [[TMP16:%.*]] = load i8, i8 addrspace(3)* [[TMP15]], align 1
+; OPT-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP13]], i32 [[TMP14]]
+; OPT-NEXT:    store i8 [[TMP16]], i8 addrspace(1)* [[TMP17]], align 1
+; OPT-NEXT:    [[TMP18]] = add i32 [[RESIDUAL_LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP19:%.*]] = icmp ult i32 [[TMP18]], [[TMP4]]
+; OPT-NEXT:    br i1 [[TMP19]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION:%.*]]
 ; OPT:       post-loop-memcpy-expansion:
 ; OPT-NEXT:    ret void
+; OPT:       loop-memcpy-residual-header:
+; OPT-NEXT:    [[TMP20:%.*]] = icmp ne i32 [[TMP4]], 0
+; OPT-NEXT:    br i1 [[TMP20]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION]]
 ;
   call void @llvm.memcpy.p1i8.p3i8.i32(i8 addrspace(1)* %dst, i8 addrspace(3)* %src, i32 %n, i1 false)
   ret void
@@ -201,24 +373,1119 @@ define amdgpu_kernel void @memcpy_alt_type(i8 addrspace(1)* %dst, i8 addrspace(3
 
 ; One of the uses in the function should be expanded, the other left alone.
 define amdgpu_kernel void @memcpy_multi_use_one_function_keep_small(i8 addrspace(1)* %dst0, i8 addrspace(1)* %dst1, i8 addrspace(1)* %src, i64 %n) #0 {
-; OPT-LABEL: @memcpy_multi_use_one_function_keep_small(
-; OPT-NEXT:    [[TMP1:%.*]] = icmp ne i64 [[N:%.*]], 0
-; OPT-NEXT:    br i1 [[TMP1]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[POST_LOOP_MEMCPY_EXPANSION:%.*]]
+; MAX1024-LABEL: @memcpy_multi_use_one_function_keep_small(
+; MAX1024-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SRC:%.*]] to <4 x i32> addrspace(1)*
+; MAX1024-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(1)* [[DST0:%.*]] to <4 x i32> addrspace(1)*
+; MAX1024-NEXT:    [[TMP3:%.*]] = udiv i64 [[N:%.*]], 16
+; MAX1024-NEXT:    [[TMP4:%.*]] = urem i64 [[N]], 16
+; MAX1024-NEXT:    [[TMP5:%.*]] = sub i64 [[N]], [[TMP4]]
+; MAX1024-NEXT:    [[TMP6:%.*]] = icmp ne i64 [[TMP3]], 0
+; MAX1024-NEXT:    br i1 [[TMP6]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[LOOP_MEMCPY_RESIDUAL_HEADER:%.*]]
+; MAX1024:       loop-memcpy-expansion:
+; MAX1024-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP10:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
+; MAX1024-NEXT:    [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP1]], i64 [[LOOP_INDEX]]
+; MAX1024-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32> addrspace(1)* [[TMP7]], align 1
+; MAX1024-NEXT:    [[TMP9:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP2]], i64 [[LOOP_INDEX]]
+; MAX1024-NEXT:    store <4 x i32> [[TMP8]], <4 x i32> addrspace(1)* [[TMP9]], align 1
+; MAX1024-NEXT:    [[TMP10]] = add i64 [[LOOP_INDEX]], 1
+; MAX1024-NEXT:    [[TMP11:%.*]] = icmp ult i64 [[TMP10]], [[TMP3]]
+; MAX1024-NEXT:    br i1 [[TMP11]], label [[LOOP_MEMCPY_EXPANSION]], label [[LOOP_MEMCPY_RESIDUAL_HEADER]]
+; MAX1024:       loop-memcpy-residual:
+; MAX1024-NEXT:    [[RESIDUAL_LOOP_INDEX:%.*]] = phi i64 [ 0, [[LOOP_MEMCPY_RESIDUAL_HEADER]] ], [ [[TMP18:%.*]], [[LOOP_MEMCPY_RESIDUAL:%.*]] ]
+; MAX1024-NEXT:    [[TMP12:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP1]] to i8 addrspace(1)*
+; MAX1024-NEXT:    [[TMP13:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP2]] to i8 addrspace(1)*
+; MAX1024-NEXT:    [[TMP14:%.*]] = add i64 [[TMP5]], [[RESIDUAL_LOOP_INDEX]]
+; MAX1024-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP12]], i64 [[TMP14]]
+; MAX1024-NEXT:    [[TMP16:%.*]] = load i8, i8 addrspace(1)* [[TMP15]], align 1
+; MAX1024-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP13]], i64 [[TMP14]]
+; MAX1024-NEXT:    store i8 [[TMP16]], i8 addrspace(1)* [[TMP17]], align 1
+; MAX1024-NEXT:    [[TMP18]] = add i64 [[RESIDUAL_LOOP_INDEX]], 1
+; MAX1024-NEXT:    [[TMP19:%.*]] = icmp ult i64 [[TMP18]], [[TMP4]]
+; MAX1024-NEXT:    br i1 [[TMP19]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION:%.*]]
+; MAX1024:       post-loop-memcpy-expansion:
+; MAX1024-NEXT:    call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* [[DST1:%.*]], i8 addrspace(1)* [[SRC]], i64 102, i1 false)
+; MAX1024-NEXT:    ret void
+; MAX1024:       loop-memcpy-residual-header:
+; MAX1024-NEXT:    [[TMP20:%.*]] = icmp ne i64 [[TMP4]], 0
+; MAX1024-NEXT:    br i1 [[TMP20]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION]]
+;
+; ALL-LABEL: @memcpy_multi_use_one_function_keep_small(
+; ALL-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SRC:%.*]] to <4 x i32> addrspace(1)*
+; ALL-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(1)* [[DST0:%.*]] to <4 x i32> addrspace(1)*
+; ALL-NEXT:    [[TMP3:%.*]] = udiv i64 [[N:%.*]], 16
+; ALL-NEXT:    [[TMP4:%.*]] = urem i64 [[N]], 16
+; ALL-NEXT:    [[TMP5:%.*]] = sub i64 [[N]], [[TMP4]]
+; ALL-NEXT:    [[TMP6:%.*]] = icmp ne i64 [[TMP3]], 0
+; ALL-NEXT:    br i1 [[TMP6]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[LOOP_MEMCPY_RESIDUAL_HEADER:%.*]]
+; ALL:       loop-memcpy-expansion:
+; ALL-NEXT:    [[LOOP_INDEX1:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP10:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
+; ALL-NEXT:    [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP1]], i64 [[LOOP_INDEX1]]
+; ALL-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32> addrspace(1)* [[TMP7]], align 1
+; ALL-NEXT:    [[TMP9:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP2]], i64 [[LOOP_INDEX1]]
+; ALL-NEXT:    store <4 x i32> [[TMP8]], <4 x i32> addrspace(1)* [[TMP9]], align 1
+; ALL-NEXT:    [[TMP10]] = add i64 [[LOOP_INDEX1]], 1
+; ALL-NEXT:    [[TMP11:%.*]] = icmp ult i64 [[TMP10]], [[TMP3]]
+; ALL-NEXT:    br i1 [[TMP11]], label [[LOOP_MEMCPY_EXPANSION]], label [[LOOP_MEMCPY_RESIDUAL_HEADER]]
+; ALL:       loop-memcpy-residual:
+; ALL-NEXT:    [[RESIDUAL_LOOP_INDEX:%.*]] = phi i64 [ 0, [[LOOP_MEMCPY_RESIDUAL_HEADER]] ], [ [[TMP18:%.*]], [[LOOP_MEMCPY_RESIDUAL:%.*]] ]
+; ALL-NEXT:    [[TMP12:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP1]] to i8 addrspace(1)*
+; ALL-NEXT:    [[TMP13:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP2]] to i8 addrspace(1)*
+; ALL-NEXT:    [[TMP14:%.*]] = add i64 [[TMP5]], [[RESIDUAL_LOOP_INDEX]]
+; ALL-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP12]], i64 [[TMP14]]
+; ALL-NEXT:    [[TMP16:%.*]] = load i8, i8 addrspace(1)* [[TMP15]], align 1
+; ALL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP13]], i64 [[TMP14]]
+; ALL-NEXT:    store i8 [[TMP16]], i8 addrspace(1)* [[TMP17]], align 1
+; ALL-NEXT:    [[TMP18]] = add i64 [[RESIDUAL_LOOP_INDEX]], 1
+; ALL-NEXT:    [[TMP19:%.*]] = icmp ult i64 [[TMP18]], [[TMP4]]
+; ALL-NEXT:    br i1 [[TMP19]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION:%.*]]
+; ALL:       post-loop-memcpy-expansion:
+; ALL-NEXT:    [[TMP20:%.*]] = bitcast i8 addrspace(1)* [[SRC]] to <4 x i32> addrspace(1)*
+; ALL-NEXT:    [[TMP21:%.*]] = bitcast i8 addrspace(1)* [[DST1:%.*]] to <4 x i32> addrspace(1)*
+; ALL-NEXT:    br label [[LOAD_STORE_LOOP:%.*]]
+; ALL:       load-store-loop:
+; ALL-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[POST_LOOP_MEMCPY_EXPANSION]] ], [ [[TMP25:%.*]], [[LOAD_STORE_LOOP]] ]
+; ALL-NEXT:    [[TMP22:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP20]], i64 [[LOOP_INDEX]]
+; ALL-NEXT:    [[TMP23:%.*]] = load <4 x i32>, <4 x i32> addrspace(1)* [[TMP22]], align 1
+; ALL-NEXT:    [[TMP24:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP21]], i64 [[LOOP_INDEX]]
+; ALL-NEXT:    store <4 x i32> [[TMP23]], <4 x i32> addrspace(1)* [[TMP24]], align 1
+; ALL-NEXT:    [[TMP25]] = add i64 [[LOOP_INDEX]], 1
+; ALL-NEXT:    [[TMP26:%.*]] = icmp ult i64 [[TMP25]], 6
+; ALL-NEXT:    br i1 [[TMP26]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
+; ALL:       memcpy-split:
+; ALL-NEXT:    [[TMP27:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP20]] to i32 addrspace(1)*
+; ALL-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP27]], i64 24
+; ALL-NEXT:    [[TMP29:%.*]] = load i32, i32 addrspace(1)* [[TMP28]], align 1
+; ALL-NEXT:    [[TMP30:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP21]] to i32 addrspace(1)*
+; ALL-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP30]], i64 24
+; ALL-NEXT:    store i32 [[TMP29]], i32 addrspace(1)* [[TMP31]], align 1
+; ALL-NEXT:    [[TMP32:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP20]] to i16 addrspace(1)*
+; ALL-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i16, i16 addrspace(1)* [[TMP32]], i64 50
+; ALL-NEXT:    [[TMP34:%.*]] = load i16, i16 addrspace(1)* [[TMP33]], align 1
+; ALL-NEXT:    [[TMP35:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP21]] to i16 addrspace(1)*
+; ALL-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i16, i16 addrspace(1)* [[TMP35]], i64 50
+; ALL-NEXT:    store i16 [[TMP34]], i16 addrspace(1)* [[TMP36]], align 1
+; ALL-NEXT:    ret void
+; ALL:       loop-memcpy-residual-header:
+; ALL-NEXT:    [[TMP37:%.*]] = icmp ne i64 [[TMP4]], 0
+; ALL-NEXT:    br i1 [[TMP37]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION]]
+;
+  call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* %dst0, i8 addrspace(1)* %src, i64 %n, i1 false)
+  call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* %dst1, i8 addrspace(1)* %src, i64 102, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_global_align4_global_align4_1028(i8 addrspace(1)* %dst, i8 addrspace(1)* %src) #0 {
+; OPT-LABEL: @memcpy_global_align4_global_align4_1028(
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SRC:%.*]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(1)* [[DST:%.*]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    br label [[LOAD_STORE_LOOP:%.*]]
+; OPT:       load-store-loop:
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[LOAD_STORE_LOOP]] ]
+; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP1]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32> addrspace(1)* [[TMP3]], align 4
+; OPT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP2]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    store <4 x i32> [[TMP4]], <4 x i32> addrspace(1)* [[TMP5]], align 4
+; OPT-NEXT:    [[TMP6]] = add i64 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP7:%.*]] = icmp ult i64 [[TMP6]], 64
+; OPT-NEXT:    br i1 [[TMP7]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
+; OPT:       memcpy-split:
+; OPT-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP1]] to i32 addrspace(1)*
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP8]], i64 256
+; OPT-NEXT:    [[TMP10:%.*]] = load i32, i32 addrspace(1)* [[TMP9]], align 4
+; OPT-NEXT:    [[TMP11:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP2]] to i32 addrspace(1)*
+; OPT-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP11]], i64 256
+; OPT-NEXT:    store i32 [[TMP10]], i32 addrspace(1)* [[TMP12]], align 4
+; OPT-NEXT:    ret void
+;
+  call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 %dst, i8 addrspace(1)* align 4 %src, i64 1028, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_global_align4_global_align4_1025(i8 addrspace(1)* %dst, i8 addrspace(1)* %src) #0 {
+; OPT-LABEL: @memcpy_global_align4_global_align4_1025(
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SRC:%.*]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(1)* [[DST:%.*]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    br label [[LOAD_STORE_LOOP:%.*]]
+; OPT:       load-store-loop:
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[LOAD_STORE_LOOP]] ]
+; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP1]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32> addrspace(1)* [[TMP3]], align 4
+; OPT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP2]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    store <4 x i32> [[TMP4]], <4 x i32> addrspace(1)* [[TMP5]], align 4
+; OPT-NEXT:    [[TMP6]] = add i64 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP7:%.*]] = icmp ult i64 [[TMP6]], 64
+; OPT-NEXT:    br i1 [[TMP7]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
+; OPT:       memcpy-split:
+; OPT-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP1]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP8]], i64 1024
+; OPT-NEXT:    [[TMP10:%.*]] = load i8, i8 addrspace(1)* [[TMP9]], align 4
+; OPT-NEXT:    [[TMP11:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP2]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP11]], i64 1024
+; OPT-NEXT:    store i8 [[TMP10]], i8 addrspace(1)* [[TMP12]], align 4
+; OPT-NEXT:    ret void
+;
+  call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 %dst, i8 addrspace(1)* align 4 %src, i64 1025, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_global_align4_global_align4_1026(i8 addrspace(1)* %dst, i8 addrspace(1)* %src) #0 {
+; OPT-LABEL: @memcpy_global_align4_global_align4_1026(
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SRC:%.*]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(1)* [[DST:%.*]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    br label [[LOAD_STORE_LOOP:%.*]]
+; OPT:       load-store-loop:
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[LOAD_STORE_LOOP]] ]
+; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP1]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32> addrspace(1)* [[TMP3]], align 4
+; OPT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP2]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    store <4 x i32> [[TMP4]], <4 x i32> addrspace(1)* [[TMP5]], align 4
+; OPT-NEXT:    [[TMP6]] = add i64 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP7:%.*]] = icmp ult i64 [[TMP6]], 64
+; OPT-NEXT:    br i1 [[TMP7]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
+; OPT:       memcpy-split:
+; OPT-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP1]] to i16 addrspace(1)*
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i16, i16 addrspace(1)* [[TMP8]], i64 512
+; OPT-NEXT:    [[TMP10:%.*]] = load i16, i16 addrspace(1)* [[TMP9]], align 4
+; OPT-NEXT:    [[TMP11:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP2]] to i16 addrspace(1)*
+; OPT-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i16, i16 addrspace(1)* [[TMP11]], i64 512
+; OPT-NEXT:    store i16 [[TMP10]], i16 addrspace(1)* [[TMP12]], align 4
+; OPT-NEXT:    ret void
+;
+  call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 %dst, i8 addrspace(1)* align 4 %src, i64 1026, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_global_align4_global_align4_1032(i8 addrspace(1)* %dst, i8 addrspace(1)* %src) #0 {
+; OPT-LABEL: @memcpy_global_align4_global_align4_1032(
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SRC:%.*]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(1)* [[DST:%.*]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    br label [[LOAD_STORE_LOOP:%.*]]
+; OPT:       load-store-loop:
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[LOAD_STORE_LOOP]] ]
+; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP1]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32> addrspace(1)* [[TMP3]], align 4
+; OPT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP2]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    store <4 x i32> [[TMP4]], <4 x i32> addrspace(1)* [[TMP5]], align 4
+; OPT-NEXT:    [[TMP6]] = add i64 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP7:%.*]] = icmp ult i64 [[TMP6]], 64
+; OPT-NEXT:    br i1 [[TMP7]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
+; OPT:       memcpy-split:
+; OPT-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP1]] to i64 addrspace(1)*
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, i64 addrspace(1)* [[TMP8]], i64 128
+; OPT-NEXT:    [[TMP10:%.*]] = load i64, i64 addrspace(1)* [[TMP9]], align 4
+; OPT-NEXT:    [[TMP11:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP2]] to i64 addrspace(1)*
+; OPT-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, i64 addrspace(1)* [[TMP11]], i64 128
+; OPT-NEXT:    store i64 [[TMP10]], i64 addrspace(1)* [[TMP12]], align 4
+; OPT-NEXT:    ret void
+;
+  call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 %dst, i8 addrspace(1)* align 4 %src, i64 1032, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_global_align4_global_align4_1034(i8 addrspace(1)* %dst, i8 addrspace(1)* %src) #0 {
+; OPT-LABEL: @memcpy_global_align4_global_align4_1034(
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SRC:%.*]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(1)* [[DST:%.*]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    br label [[LOAD_STORE_LOOP:%.*]]
+; OPT:       load-store-loop:
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[LOAD_STORE_LOOP]] ]
+; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP1]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32> addrspace(1)* [[TMP3]], align 4
+; OPT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP2]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    store <4 x i32> [[TMP4]], <4 x i32> addrspace(1)* [[TMP5]], align 4
+; OPT-NEXT:    [[TMP6]] = add i64 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP7:%.*]] = icmp ult i64 [[TMP6]], 64
+; OPT-NEXT:    br i1 [[TMP7]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
+; OPT:       memcpy-split:
+; OPT-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP1]] to i64 addrspace(1)*
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, i64 addrspace(1)* [[TMP8]], i64 128
+; OPT-NEXT:    [[TMP10:%.*]] = load i64, i64 addrspace(1)* [[TMP9]], align 4
+; OPT-NEXT:    [[TMP11:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP2]] to i64 addrspace(1)*
+; OPT-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, i64 addrspace(1)* [[TMP11]], i64 128
+; OPT-NEXT:    store i64 [[TMP10]], i64 addrspace(1)* [[TMP12]], align 4
+; OPT-NEXT:    [[TMP13:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP1]] to i16 addrspace(1)*
+; OPT-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i16, i16 addrspace(1)* [[TMP13]], i64 516
+; OPT-NEXT:    [[TMP15:%.*]] = load i16, i16 addrspace(1)* [[TMP14]], align 4
+; OPT-NEXT:    [[TMP16:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP2]] to i16 addrspace(1)*
+; OPT-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i16, i16 addrspace(1)* [[TMP16]], i64 516
+; OPT-NEXT:    store i16 [[TMP15]], i16 addrspace(1)* [[TMP17]], align 4
+; OPT-NEXT:    ret void
+;
+  call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 %dst, i8 addrspace(1)* align 4 %src, i64 1034, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_global_align4_global_align4_1035(i8 addrspace(1)* %dst, i8 addrspace(1)* %src) #0 {
+; OPT-LABEL: @memcpy_global_align4_global_align4_1035(
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SRC:%.*]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(1)* [[DST:%.*]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    br label [[LOAD_STORE_LOOP:%.*]]
+; OPT:       load-store-loop:
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[LOAD_STORE_LOOP]] ]
+; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP1]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32> addrspace(1)* [[TMP3]], align 4
+; OPT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP2]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    store <4 x i32> [[TMP4]], <4 x i32> addrspace(1)* [[TMP5]], align 4
+; OPT-NEXT:    [[TMP6]] = add i64 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP7:%.*]] = icmp ult i64 [[TMP6]], 64
+; OPT-NEXT:    br i1 [[TMP7]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
+; OPT:       memcpy-split:
+; OPT-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP1]] to i64 addrspace(1)*
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, i64 addrspace(1)* [[TMP8]], i64 128
+; OPT-NEXT:    [[TMP10:%.*]] = load i64, i64 addrspace(1)* [[TMP9]], align 4
+; OPT-NEXT:    [[TMP11:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP2]] to i64 addrspace(1)*
+; OPT-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, i64 addrspace(1)* [[TMP11]], i64 128
+; OPT-NEXT:    store i64 [[TMP10]], i64 addrspace(1)* [[TMP12]], align 4
+; OPT-NEXT:    [[TMP13:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP1]] to i16 addrspace(1)*
+; OPT-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i16, i16 addrspace(1)* [[TMP13]], i64 516
+; OPT-NEXT:    [[TMP15:%.*]] = load i16, i16 addrspace(1)* [[TMP14]], align 4
+; OPT-NEXT:    [[TMP16:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP2]] to i16 addrspace(1)*
+; OPT-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i16, i16 addrspace(1)* [[TMP16]], i64 516
+; OPT-NEXT:    store i16 [[TMP15]], i16 addrspace(1)* [[TMP17]], align 4
+; OPT-NEXT:    [[TMP18:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP1]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP18]], i64 1034
+; OPT-NEXT:    [[TMP20:%.*]] = load i8, i8 addrspace(1)* [[TMP19]], align 2
+; OPT-NEXT:    [[TMP21:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP2]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP21]], i64 1034
+; OPT-NEXT:    store i8 [[TMP20]], i8 addrspace(1)* [[TMP22]], align 2
+; OPT-NEXT:    ret void
+;
+  call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 %dst, i8 addrspace(1)* align 4 %src, i64 1035, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_global_align4_global_align4_1036(i8 addrspace(1)* %dst, i8 addrspace(1)* %src) #0 {
+; OPT-LABEL: @memcpy_global_align4_global_align4_1036(
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SRC:%.*]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(1)* [[DST:%.*]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    br label [[LOAD_STORE_LOOP:%.*]]
+; OPT:       load-store-loop:
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[LOAD_STORE_LOOP]] ]
+; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP1]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32> addrspace(1)* [[TMP3]], align 4
+; OPT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP2]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    store <4 x i32> [[TMP4]], <4 x i32> addrspace(1)* [[TMP5]], align 4
+; OPT-NEXT:    [[TMP6]] = add i64 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP7:%.*]] = icmp ult i64 [[TMP6]], 64
+; OPT-NEXT:    br i1 [[TMP7]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
+; OPT:       memcpy-split:
+; OPT-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP1]] to i64 addrspace(1)*
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, i64 addrspace(1)* [[TMP8]], i64 128
+; OPT-NEXT:    [[TMP10:%.*]] = load i64, i64 addrspace(1)* [[TMP9]], align 4
+; OPT-NEXT:    [[TMP11:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP2]] to i64 addrspace(1)*
+; OPT-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, i64 addrspace(1)* [[TMP11]], i64 128
+; OPT-NEXT:    store i64 [[TMP10]], i64 addrspace(1)* [[TMP12]], align 4
+; OPT-NEXT:    [[TMP13:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP1]] to i32 addrspace(1)*
+; OPT-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP13]], i64 258
+; OPT-NEXT:    [[TMP15:%.*]] = load i32, i32 addrspace(1)* [[TMP14]], align 4
+; OPT-NEXT:    [[TMP16:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP2]] to i32 addrspace(1)*
+; OPT-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP16]], i64 258
+; OPT-NEXT:    store i32 [[TMP15]], i32 addrspace(1)* [[TMP17]], align 4
+; OPT-NEXT:    ret void
+;
+  call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 %dst, i8 addrspace(1)* align 4 %src, i64 1036, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_global_align4_global_align4_1039(i8 addrspace(1)* %dst, i8 addrspace(1)* %src) #0 {
+; OPT-LABEL: @memcpy_global_align4_global_align4_1039(
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SRC:%.*]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(1)* [[DST:%.*]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    br label [[LOAD_STORE_LOOP:%.*]]
+; OPT:       load-store-loop:
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[LOAD_STORE_LOOP]] ]
+; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP1]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32> addrspace(1)* [[TMP3]], align 4
+; OPT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP2]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    store <4 x i32> [[TMP4]], <4 x i32> addrspace(1)* [[TMP5]], align 4
+; OPT-NEXT:    [[TMP6]] = add i64 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP7:%.*]] = icmp ult i64 [[TMP6]], 64
+; OPT-NEXT:    br i1 [[TMP7]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
+; OPT:       memcpy-split:
+; OPT-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP1]] to i64 addrspace(1)*
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, i64 addrspace(1)* [[TMP8]], i64 128
+; OPT-NEXT:    [[TMP10:%.*]] = load i64, i64 addrspace(1)* [[TMP9]], align 4
+; OPT-NEXT:    [[TMP11:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP2]] to i64 addrspace(1)*
+; OPT-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, i64 addrspace(1)* [[TMP11]], i64 128
+; OPT-NEXT:    store i64 [[TMP10]], i64 addrspace(1)* [[TMP12]], align 4
+; OPT-NEXT:    [[TMP13:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP1]] to i32 addrspace(1)*
+; OPT-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP13]], i64 258
+; OPT-NEXT:    [[TMP15:%.*]] = load i32, i32 addrspace(1)* [[TMP14]], align 4
+; OPT-NEXT:    [[TMP16:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP2]] to i32 addrspace(1)*
+; OPT-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP16]], i64 258
+; OPT-NEXT:    store i32 [[TMP15]], i32 addrspace(1)* [[TMP17]], align 4
+; OPT-NEXT:    [[TMP18:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP1]] to i16 addrspace(1)*
+; OPT-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i16, i16 addrspace(1)* [[TMP18]], i64 518
+; OPT-NEXT:    [[TMP20:%.*]] = load i16, i16 addrspace(1)* [[TMP19]], align 4
+; OPT-NEXT:    [[TMP21:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP2]] to i16 addrspace(1)*
+; OPT-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i16, i16 addrspace(1)* [[TMP21]], i64 518
+; OPT-NEXT:    store i16 [[TMP20]], i16 addrspace(1)* [[TMP22]], align 4
+; OPT-NEXT:    [[TMP23:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP1]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP23]], i64 1038
+; OPT-NEXT:    [[TMP25:%.*]] = load i8, i8 addrspace(1)* [[TMP24]], align 2
+; OPT-NEXT:    [[TMP26:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP2]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP26]], i64 1038
+; OPT-NEXT:    store i8 [[TMP25]], i8 addrspace(1)* [[TMP27]], align 2
+; OPT-NEXT:    ret void
+;
+  call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 %dst, i8 addrspace(1)* align 4 %src, i64 1039, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_global_align2_global_align2_1039(i8 addrspace(1)* %dst, i8 addrspace(1)* %src) #0 {
+; OPT-LABEL: @memcpy_global_align2_global_align2_1039(
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SRC:%.*]] to i16 addrspace(1)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(1)* [[DST:%.*]] to i16 addrspace(1)*
+; OPT-NEXT:    br label [[LOAD_STORE_LOOP:%.*]]
+; OPT:       load-store-loop:
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[LOAD_STORE_LOOP]] ]
+; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i16, i16 addrspace(1)* [[TMP1]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP4:%.*]] = load i16, i16 addrspace(1)* [[TMP3]], align 2
+; OPT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i16, i16 addrspace(1)* [[TMP2]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    store i16 [[TMP4]], i16 addrspace(1)* [[TMP5]], align 2
+; OPT-NEXT:    [[TMP6]] = add i64 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP7:%.*]] = icmp ult i64 [[TMP6]], 519
+; OPT-NEXT:    br i1 [[TMP7]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
+; OPT:       memcpy-split:
+; OPT-NEXT:    [[TMP8:%.*]] = bitcast i16 addrspace(1)* [[TMP1]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP8]], i64 1038
+; OPT-NEXT:    [[TMP10:%.*]] = load i8, i8 addrspace(1)* [[TMP9]], align 2
+; OPT-NEXT:    [[TMP11:%.*]] = bitcast i16 addrspace(1)* [[TMP2]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP11]], i64 1038
+; OPT-NEXT:    store i8 [[TMP10]], i8 addrspace(1)* [[TMP12]], align 2
+; OPT-NEXT:    ret void
+;
+  call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 2 %dst, i8 addrspace(1)* align 2 %src, i64 1039, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_global_align4_global_align4_1027(i8 addrspace(1)* %dst, i8 addrspace(1)* %src) #0 {
+; OPT-LABEL: @memcpy_global_align4_global_align4_1027(
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SRC:%.*]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(1)* [[DST:%.*]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    br label [[LOAD_STORE_LOOP:%.*]]
+; OPT:       load-store-loop:
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[LOAD_STORE_LOOP]] ]
+; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP1]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32> addrspace(1)* [[TMP3]], align 4
+; OPT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP2]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    store <4 x i32> [[TMP4]], <4 x i32> addrspace(1)* [[TMP5]], align 4
+; OPT-NEXT:    [[TMP6]] = add i64 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP7:%.*]] = icmp ult i64 [[TMP6]], 64
+; OPT-NEXT:    br i1 [[TMP7]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
+; OPT:       memcpy-split:
+; OPT-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP1]] to i16 addrspace(1)*
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i16, i16 addrspace(1)* [[TMP8]], i64 512
+; OPT-NEXT:    [[TMP10:%.*]] = load i16, i16 addrspace(1)* [[TMP9]], align 4
+; OPT-NEXT:    [[TMP11:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP2]] to i16 addrspace(1)*
+; OPT-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i16, i16 addrspace(1)* [[TMP11]], i64 512
+; OPT-NEXT:    store i16 [[TMP10]], i16 addrspace(1)* [[TMP12]], align 4
+; OPT-NEXT:    [[TMP13:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP1]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP13]], i64 1026
+; OPT-NEXT:    [[TMP15:%.*]] = load i8, i8 addrspace(1)* [[TMP14]], align 2
+; OPT-NEXT:    [[TMP16:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP2]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP16]], i64 1026
+; OPT-NEXT:    store i8 [[TMP15]], i8 addrspace(1)* [[TMP17]], align 2
+; OPT-NEXT:    ret void
+;
+  call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 %dst, i8 addrspace(1)* align 4 %src, i64 1027, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_global_align2_global_align4_1027(i8 addrspace(1)* %dst, i8 addrspace(1)* %src) #0 {
+; OPT-LABEL: @memcpy_global_align2_global_align4_1027(
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SRC:%.*]] to i16 addrspace(1)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(1)* [[DST:%.*]] to i16 addrspace(1)*
+; OPT-NEXT:    br label [[LOAD_STORE_LOOP:%.*]]
+; OPT:       load-store-loop:
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[LOAD_STORE_LOOP]] ]
+; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i16, i16 addrspace(1)* [[TMP1]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP4:%.*]] = load i16, i16 addrspace(1)* [[TMP3]], align 2
+; OPT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i16, i16 addrspace(1)* [[TMP2]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    store i16 [[TMP4]], i16 addrspace(1)* [[TMP5]], align 2
+; OPT-NEXT:    [[TMP6]] = add i64 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP7:%.*]] = icmp ult i64 [[TMP6]], 513
+; OPT-NEXT:    br i1 [[TMP7]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
+; OPT:       memcpy-split:
+; OPT-NEXT:    [[TMP8:%.*]] = bitcast i16 addrspace(1)* [[TMP1]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP8]], i64 1026
+; OPT-NEXT:    [[TMP10:%.*]] = load i8, i8 addrspace(1)* [[TMP9]], align 2
+; OPT-NEXT:    [[TMP11:%.*]] = bitcast i16 addrspace(1)* [[TMP2]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP11]], i64 1026
+; OPT-NEXT:    store i8 [[TMP10]], i8 addrspace(1)* [[TMP12]], align 2
+; OPT-NEXT:    ret void
+;
+  call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 2 %dst, i8 addrspace(1)* align 4 %src, i64 1027, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_global_align4_global_align2_1027(i8 addrspace(1)* %dst, i8 addrspace(1)* %src) #0 {
+; OPT-LABEL: @memcpy_global_align4_global_align2_1027(
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SRC:%.*]] to i16 addrspace(1)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(1)* [[DST:%.*]] to i16 addrspace(1)*
+; OPT-NEXT:    br label [[LOAD_STORE_LOOP:%.*]]
+; OPT:       load-store-loop:
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[LOAD_STORE_LOOP]] ]
+; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i16, i16 addrspace(1)* [[TMP1]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP4:%.*]] = load i16, i16 addrspace(1)* [[TMP3]], align 2
+; OPT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i16, i16 addrspace(1)* [[TMP2]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    store i16 [[TMP4]], i16 addrspace(1)* [[TMP5]], align 2
+; OPT-NEXT:    [[TMP6]] = add i64 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP7:%.*]] = icmp ult i64 [[TMP6]], 513
+; OPT-NEXT:    br i1 [[TMP7]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
+; OPT:       memcpy-split:
+; OPT-NEXT:    [[TMP8:%.*]] = bitcast i16 addrspace(1)* [[TMP1]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP8]], i64 1026
+; OPT-NEXT:    [[TMP10:%.*]] = load i8, i8 addrspace(1)* [[TMP9]], align 2
+; OPT-NEXT:    [[TMP11:%.*]] = bitcast i16 addrspace(1)* [[TMP2]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP11]], i64 1026
+; OPT-NEXT:    store i8 [[TMP10]], i8 addrspace(1)* [[TMP12]], align 2
+; OPT-NEXT:    ret void
+;
+  call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 %dst, i8 addrspace(1)* align 2 %src, i64 1027, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_private_align4_private_align4_1027(i8 addrspace(5)* %dst, i8 addrspace(5)* %src) #0 {
+; OPT-LABEL: @memcpy_private_align4_private_align4_1027(
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(5)* [[SRC:%.*]] to <4 x i32> addrspace(5)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(5)* [[DST:%.*]] to <4 x i32> addrspace(5)*
+; OPT-NEXT:    br label [[LOAD_STORE_LOOP:%.*]]
+; OPT:       load-store-loop:
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[LOAD_STORE_LOOP]] ]
+; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* [[TMP1]], i32 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32> addrspace(5)* [[TMP3]], align 4
+; OPT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* [[TMP2]], i32 [[LOOP_INDEX]]
+; OPT-NEXT:    store <4 x i32> [[TMP4]], <4 x i32> addrspace(5)* [[TMP5]], align 4
+; OPT-NEXT:    [[TMP6]] = add i32 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP7:%.*]] = icmp ult i32 [[TMP6]], 64
+; OPT-NEXT:    br i1 [[TMP7]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
+; OPT:       memcpy-split:
+; OPT-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> addrspace(5)* [[TMP1]] to i16 addrspace(5)*
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i16, i16 addrspace(5)* [[TMP8]], i32 512
+; OPT-NEXT:    [[TMP10:%.*]] = load i16, i16 addrspace(5)* [[TMP9]], align 4
+; OPT-NEXT:    [[TMP11:%.*]] = bitcast <4 x i32> addrspace(5)* [[TMP2]] to i16 addrspace(5)*
+; OPT-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i16, i16 addrspace(5)* [[TMP11]], i32 512
+; OPT-NEXT:    store i16 [[TMP10]], i16 addrspace(5)* [[TMP12]], align 4
+; OPT-NEXT:    [[TMP13:%.*]] = bitcast <4 x i32> addrspace(5)* [[TMP1]] to i8 addrspace(5)*
+; OPT-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, i8 addrspace(5)* [[TMP13]], i32 1026
+; OPT-NEXT:    [[TMP15:%.*]] = load i8, i8 addrspace(5)* [[TMP14]], align 2
+; OPT-NEXT:    [[TMP16:%.*]] = bitcast <4 x i32> addrspace(5)* [[TMP2]] to i8 addrspace(5)*
+; OPT-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, i8 addrspace(5)* [[TMP16]], i32 1026
+; OPT-NEXT:    store i8 [[TMP15]], i8 addrspace(5)* [[TMP17]], align 2
+; OPT-NEXT:    ret void
+;
+  call void @llvm.memcpy.p5i8.p5i8.i32(i8 addrspace(5)* align 4 %dst, i8 addrspace(5)* align 4 %src, i32 1027, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_private_align2_private_align4_1027(i8 addrspace(5)* %dst, i8 addrspace(5)* %src) #0 {
+; OPT-LABEL: @memcpy_private_align2_private_align4_1027(
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(5)* [[SRC:%.*]] to i16 addrspace(5)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(5)* [[DST:%.*]] to i16 addrspace(5)*
+; OPT-NEXT:    br label [[LOAD_STORE_LOOP:%.*]]
+; OPT:       load-store-loop:
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[LOAD_STORE_LOOP]] ]
+; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i16, i16 addrspace(5)* [[TMP1]], i32 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP4:%.*]] = load i16, i16 addrspace(5)* [[TMP3]], align 2
+; OPT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i16, i16 addrspace(5)* [[TMP2]], i32 [[LOOP_INDEX]]
+; OPT-NEXT:    store i16 [[TMP4]], i16 addrspace(5)* [[TMP5]], align 2
+; OPT-NEXT:    [[TMP6]] = add i32 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP7:%.*]] = icmp ult i32 [[TMP6]], 513
+; OPT-NEXT:    br i1 [[TMP7]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
+; OPT:       memcpy-split:
+; OPT-NEXT:    [[TMP8:%.*]] = bitcast i16 addrspace(5)* [[TMP1]] to i8 addrspace(5)*
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, i8 addrspace(5)* [[TMP8]], i32 1026
+; OPT-NEXT:    [[TMP10:%.*]] = load i8, i8 addrspace(5)* [[TMP9]], align 2
+; OPT-NEXT:    [[TMP11:%.*]] = bitcast i16 addrspace(5)* [[TMP2]] to i8 addrspace(5)*
+; OPT-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, i8 addrspace(5)* [[TMP11]], i32 1026
+; OPT-NEXT:    store i8 [[TMP10]], i8 addrspace(5)* [[TMP12]], align 2
+; OPT-NEXT:    ret void
+;
+  call void @llvm.memcpy.p5i8.p5i8.i32(i8 addrspace(5)* align 2 %dst, i8 addrspace(5)* align 4 %src, i32 1027, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_private_align1_private_align4_1027(i8 addrspace(5)* %dst, i8 addrspace(5)* %src) #0 {
+; OPT-LABEL: @memcpy_private_align1_private_align4_1027(
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(5)* [[SRC:%.*]] to <4 x i32> addrspace(5)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(5)* [[DST:%.*]] to <4 x i32> addrspace(5)*
+; OPT-NEXT:    br label [[LOAD_STORE_LOOP:%.*]]
+; OPT:       load-store-loop:
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[LOAD_STORE_LOOP]] ]
+; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* [[TMP1]], i32 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32> addrspace(5)* [[TMP3]], align 4
+; OPT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* [[TMP2]], i32 [[LOOP_INDEX]]
+; OPT-NEXT:    store <4 x i32> [[TMP4]], <4 x i32> addrspace(5)* [[TMP5]], align 1
+; OPT-NEXT:    [[TMP6]] = add i32 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP7:%.*]] = icmp ult i32 [[TMP6]], 64
+; OPT-NEXT:    br i1 [[TMP7]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
+; OPT:       memcpy-split:
+; OPT-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> addrspace(5)* [[TMP1]] to i16 addrspace(5)*
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i16, i16 addrspace(5)* [[TMP8]], i32 512
+; OPT-NEXT:    [[TMP10:%.*]] = load i16, i16 addrspace(5)* [[TMP9]], align 4
+; OPT-NEXT:    [[TMP11:%.*]] = bitcast <4 x i32> addrspace(5)* [[TMP2]] to i16 addrspace(5)*
+; OPT-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i16, i16 addrspace(5)* [[TMP11]], i32 512
+; OPT-NEXT:    store i16 [[TMP10]], i16 addrspace(5)* [[TMP12]], align 1
+; OPT-NEXT:    [[TMP13:%.*]] = bitcast <4 x i32> addrspace(5)* [[TMP1]] to i8 addrspace(5)*
+; OPT-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, i8 addrspace(5)* [[TMP13]], i32 1026
+; OPT-NEXT:    [[TMP15:%.*]] = load i8, i8 addrspace(5)* [[TMP14]], align 2
+; OPT-NEXT:    [[TMP16:%.*]] = bitcast <4 x i32> addrspace(5)* [[TMP2]] to i8 addrspace(5)*
+; OPT-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, i8 addrspace(5)* [[TMP16]], i32 1026
+; OPT-NEXT:    store i8 [[TMP15]], i8 addrspace(5)* [[TMP17]], align 1
+; OPT-NEXT:    ret void
+;
+  call void @llvm.memcpy.p5i8.p5i8.i32(i8 addrspace(5)* align 1 %dst, i8 addrspace(5)* align 4 %src, i32 1027, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_private_align4_private_align2_1027(i8 addrspace(5)* %dst, i8 addrspace(5)* %src) #0 {
+; OPT-LABEL: @memcpy_private_align4_private_align2_1027(
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(5)* [[SRC:%.*]] to i16 addrspace(5)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(5)* [[DST:%.*]] to i16 addrspace(5)*
+; OPT-NEXT:    br label [[LOAD_STORE_LOOP:%.*]]
+; OPT:       load-store-loop:
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[LOAD_STORE_LOOP]] ]
+; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i16, i16 addrspace(5)* [[TMP1]], i32 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP4:%.*]] = load i16, i16 addrspace(5)* [[TMP3]], align 2
+; OPT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i16, i16 addrspace(5)* [[TMP2]], i32 [[LOOP_INDEX]]
+; OPT-NEXT:    store i16 [[TMP4]], i16 addrspace(5)* [[TMP5]], align 2
+; OPT-NEXT:    [[TMP6]] = add i32 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP7:%.*]] = icmp ult i32 [[TMP6]], 513
+; OPT-NEXT:    br i1 [[TMP7]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
+; OPT:       memcpy-split:
+; OPT-NEXT:    [[TMP8:%.*]] = bitcast i16 addrspace(5)* [[TMP1]] to i8 addrspace(5)*
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, i8 addrspace(5)* [[TMP8]], i32 1026
+; OPT-NEXT:    [[TMP10:%.*]] = load i8, i8 addrspace(5)* [[TMP9]], align 2
+; OPT-NEXT:    [[TMP11:%.*]] = bitcast i16 addrspace(5)* [[TMP2]] to i8 addrspace(5)*
+; OPT-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, i8 addrspace(5)* [[TMP11]], i32 1026
+; OPT-NEXT:    store i8 [[TMP10]], i8 addrspace(5)* [[TMP12]], align 2
+; OPT-NEXT:    ret void
+;
+  call void @llvm.memcpy.p5i8.p5i8.i32(i8 addrspace(5)* align 4 %dst, i8 addrspace(5)* align 2 %src, i32 1027, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_private_align4_private_align1_1027(i8 addrspace(5)* %dst, i8 addrspace(5)* %src) #0 {
+; OPT-LABEL: @memcpy_private_align4_private_align1_1027(
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(5)* [[SRC:%.*]] to <4 x i32> addrspace(5)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(5)* [[DST:%.*]] to <4 x i32> addrspace(5)*
+; OPT-NEXT:    br label [[LOAD_STORE_LOOP:%.*]]
+; OPT:       load-store-loop:
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[LOAD_STORE_LOOP]] ]
+; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* [[TMP1]], i32 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32> addrspace(5)* [[TMP3]], align 1
+; OPT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* [[TMP2]], i32 [[LOOP_INDEX]]
+; OPT-NEXT:    store <4 x i32> [[TMP4]], <4 x i32> addrspace(5)* [[TMP5]], align 4
+; OPT-NEXT:    [[TMP6]] = add i32 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP7:%.*]] = icmp ult i32 [[TMP6]], 64
+; OPT-NEXT:    br i1 [[TMP7]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
+; OPT:       memcpy-split:
+; OPT-NEXT:    [[TMP8:%.*]] = bitcast <4 x i32> addrspace(5)* [[TMP1]] to i16 addrspace(5)*
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i16, i16 addrspace(5)* [[TMP8]], i32 512
+; OPT-NEXT:    [[TMP10:%.*]] = load i16, i16 addrspace(5)* [[TMP9]], align 1
+; OPT-NEXT:    [[TMP11:%.*]] = bitcast <4 x i32> addrspace(5)* [[TMP2]] to i16 addrspace(5)*
+; OPT-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i16, i16 addrspace(5)* [[TMP11]], i32 512
+; OPT-NEXT:    store i16 [[TMP10]], i16 addrspace(5)* [[TMP12]], align 4
+; OPT-NEXT:    [[TMP13:%.*]] = bitcast <4 x i32> addrspace(5)* [[TMP1]] to i8 addrspace(5)*
+; OPT-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, i8 addrspace(5)* [[TMP13]], i32 1026
+; OPT-NEXT:    [[TMP15:%.*]] = load i8, i8 addrspace(5)* [[TMP14]], align 1
+; OPT-NEXT:    [[TMP16:%.*]] = bitcast <4 x i32> addrspace(5)* [[TMP2]] to i8 addrspace(5)*
+; OPT-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, i8 addrspace(5)* [[TMP16]], i32 1026
+; OPT-NEXT:    store i8 [[TMP15]], i8 addrspace(5)* [[TMP17]], align 2
+; OPT-NEXT:    ret void
+;
+  call void @llvm.memcpy.p5i8.p5i8.i32(i8 addrspace(5)* align 4 %dst, i8 addrspace(5)* align 1 %src, i32 1027, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_private_align2_private_align2_1027(i8 addrspace(5)* %dst, i8 addrspace(5)* %src) #0 {
+; OPT-LABEL: @memcpy_private_align2_private_align2_1027(
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(5)* [[SRC:%.*]] to i16 addrspace(5)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(5)* [[DST:%.*]] to i16 addrspace(5)*
+; OPT-NEXT:    br label [[LOAD_STORE_LOOP:%.*]]
+; OPT:       load-store-loop:
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[LOAD_STORE_LOOP]] ]
+; OPT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i16, i16 addrspace(5)* [[TMP1]], i32 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP4:%.*]] = load i16, i16 addrspace(5)* [[TMP3]], align 2
+; OPT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i16, i16 addrspace(5)* [[TMP2]], i32 [[LOOP_INDEX]]
+; OPT-NEXT:    store i16 [[TMP4]], i16 addrspace(5)* [[TMP5]], align 2
+; OPT-NEXT:    [[TMP6]] = add i32 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP7:%.*]] = icmp ult i32 [[TMP6]], 513
+; OPT-NEXT:    br i1 [[TMP7]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
+; OPT:       memcpy-split:
+; OPT-NEXT:    [[TMP8:%.*]] = bitcast i16 addrspace(5)* [[TMP1]] to i8 addrspace(5)*
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, i8 addrspace(5)* [[TMP8]], i32 1026
+; OPT-NEXT:    [[TMP10:%.*]] = load i8, i8 addrspace(5)* [[TMP9]], align 2
+; OPT-NEXT:    [[TMP11:%.*]] = bitcast i16 addrspace(5)* [[TMP2]] to i8 addrspace(5)*
+; OPT-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, i8 addrspace(5)* [[TMP11]], i32 1026
+; OPT-NEXT:    store i8 [[TMP10]], i8 addrspace(5)* [[TMP12]], align 2
+; OPT-NEXT:    ret void
+;
+  call void @llvm.memcpy.p5i8.p5i8.i32(i8 addrspace(5)* align 2 %dst, i8 addrspace(5)* align 2 %src, i32 1027, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_global_align4_global_align4_variable(i8 addrspace(1)* %dst, i8 addrspace(1)* %src, i64 %n) #0 {
+; OPT-LABEL: @memcpy_global_align4_global_align4_variable(
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SRC:%.*]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(1)* [[DST:%.*]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    [[TMP3:%.*]] = udiv i64 [[N:%.*]], 16
+; OPT-NEXT:    [[TMP4:%.*]] = urem i64 [[N]], 16
+; OPT-NEXT:    [[TMP5:%.*]] = sub i64 [[N]], [[TMP4]]
+; OPT-NEXT:    [[TMP6:%.*]] = icmp ne i64 [[TMP3]], 0
+; OPT-NEXT:    br i1 [[TMP6]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[LOOP_MEMCPY_RESIDUAL_HEADER:%.*]]
 ; OPT:       loop-memcpy-expansion:
-; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
-; OPT-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[SRC:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    [[TMP3:%.*]] = load i8, i8 addrspace(1)* [[TMP2]], align 1
-; OPT-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[DST0:%.*]], i64 [[LOOP_INDEX]]
-; OPT-NEXT:    store i8 [[TMP3]], i8 addrspace(1)* [[TMP4]], align 1
-; OPT-NEXT:    [[TMP5]] = add i64 [[LOOP_INDEX]], 1
-; OPT-NEXT:    [[TMP6:%.*]] = icmp ult i64 [[TMP5]], [[N]]
-; OPT-NEXT:    br i1 [[TMP6]], label [[LOOP_MEMCPY_EXPANSION]], label [[POST_LOOP_MEMCPY_EXPANSION]]
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP10:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
+; OPT-NEXT:    [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP1]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32> addrspace(1)* [[TMP7]], align 4
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP2]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    store <4 x i32> [[TMP8]], <4 x i32> addrspace(1)* [[TMP9]], align 4
+; OPT-NEXT:    [[TMP10]] = add i64 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP11:%.*]] = icmp ult i64 [[TMP10]], [[TMP3]]
+; OPT-NEXT:    br i1 [[TMP11]], label [[LOOP_MEMCPY_EXPANSION]], label [[LOOP_MEMCPY_RESIDUAL_HEADER]]
+; OPT:       loop-memcpy-residual:
+; OPT-NEXT:    [[RESIDUAL_LOOP_INDEX:%.*]] = phi i64 [ 0, [[LOOP_MEMCPY_RESIDUAL_HEADER]] ], [ [[TMP18:%.*]], [[LOOP_MEMCPY_RESIDUAL:%.*]] ]
+; OPT-NEXT:    [[TMP12:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP1]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP13:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP2]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP14:%.*]] = add i64 [[TMP5]], [[RESIDUAL_LOOP_INDEX]]
+; OPT-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP12]], i64 [[TMP14]]
+; OPT-NEXT:    [[TMP16:%.*]] = load i8, i8 addrspace(1)* [[TMP15]], align 4
+; OPT-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP13]], i64 [[TMP14]]
+; OPT-NEXT:    store i8 [[TMP16]], i8 addrspace(1)* [[TMP17]], align 4
+; OPT-NEXT:    [[TMP18]] = add i64 [[RESIDUAL_LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP19:%.*]] = icmp ult i64 [[TMP18]], [[TMP4]]
+; OPT-NEXT:    br i1 [[TMP19]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION:%.*]]
 ; OPT:       post-loop-memcpy-expansion:
-; OPT-NEXT:    call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* [[DST1:%.*]], i8 addrspace(1)* [[SRC]], i64 102, i1 false)
 ; OPT-NEXT:    ret void
+; OPT:       loop-memcpy-residual-header:
+; OPT-NEXT:    [[TMP20:%.*]] = icmp ne i64 [[TMP4]], 0
+; OPT-NEXT:    br i1 [[TMP20]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION]]
 ;
-  call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* %dst0, i8 addrspace(1)* %src, i64 %n, i1 false)
-  call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* %dst1, i8 addrspace(1)* %src, i64 102, i1 false)
+  call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 %dst, i8 addrspace(1)* align 4 %src, i64 %n, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_global_align2_global_align2_variable(i8 addrspace(1)* %dst, i8 addrspace(1)* %src, i64 %n) #0 {
+; OPT-LABEL: @memcpy_global_align2_global_align2_variable(
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SRC:%.*]] to i16 addrspace(1)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(1)* [[DST:%.*]] to i16 addrspace(1)*
+; OPT-NEXT:    [[TMP3:%.*]] = udiv i64 [[N:%.*]], 2
+; OPT-NEXT:    [[TMP4:%.*]] = urem i64 [[N]], 2
+; OPT-NEXT:    [[TMP5:%.*]] = sub i64 [[N]], [[TMP4]]
+; OPT-NEXT:    [[TMP6:%.*]] = icmp ne i64 [[TMP3]], 0
+; OPT-NEXT:    br i1 [[TMP6]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[LOOP_MEMCPY_RESIDUAL_HEADER:%.*]]
+; OPT:       loop-memcpy-expansion:
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP10:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
+; OPT-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i16, i16 addrspace(1)* [[TMP1]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP8:%.*]] = load i16, i16 addrspace(1)* [[TMP7]], align 2
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i16, i16 addrspace(1)* [[TMP2]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    store i16 [[TMP8]], i16 addrspace(1)* [[TMP9]], align 2
+; OPT-NEXT:    [[TMP10]] = add i64 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP11:%.*]] = icmp ult i64 [[TMP10]], [[TMP3]]
+; OPT-NEXT:    br i1 [[TMP11]], label [[LOOP_MEMCPY_EXPANSION]], label [[LOOP_MEMCPY_RESIDUAL_HEADER]]
+; OPT:       loop-memcpy-residual:
+; OPT-NEXT:    [[RESIDUAL_LOOP_INDEX:%.*]] = phi i64 [ 0, [[LOOP_MEMCPY_RESIDUAL_HEADER]] ], [ [[TMP18:%.*]], [[LOOP_MEMCPY_RESIDUAL:%.*]] ]
+; OPT-NEXT:    [[TMP12:%.*]] = bitcast i16 addrspace(1)* [[TMP1]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP13:%.*]] = bitcast i16 addrspace(1)* [[TMP2]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP14:%.*]] = add i64 [[TMP5]], [[RESIDUAL_LOOP_INDEX]]
+; OPT-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP12]], i64 [[TMP14]]
+; OPT-NEXT:    [[TMP16:%.*]] = load i8, i8 addrspace(1)* [[TMP15]], align 2
+; OPT-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP13]], i64 [[TMP14]]
+; OPT-NEXT:    store i8 [[TMP16]], i8 addrspace(1)* [[TMP17]], align 2
+; OPT-NEXT:    [[TMP18]] = add i64 [[RESIDUAL_LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP19:%.*]] = icmp ult i64 [[TMP18]], [[TMP4]]
+; OPT-NEXT:    br i1 [[TMP19]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION:%.*]]
+; OPT:       post-loop-memcpy-expansion:
+; OPT-NEXT:    ret void
+; OPT:       loop-memcpy-residual-header:
+; OPT-NEXT:    [[TMP20:%.*]] = icmp ne i64 [[TMP4]], 0
+; OPT-NEXT:    br i1 [[TMP20]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION]]
+;
+  call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 2 %dst, i8 addrspace(1)* align 2 %src, i64 %n, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_global_align1_global_align1_variable(i8 addrspace(1)* %dst, i8 addrspace(1)* %src, i64 %n) #0 {
+; OPT-LABEL: @memcpy_global_align1_global_align1_variable(
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SRC:%.*]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(1)* [[DST:%.*]] to <4 x i32> addrspace(1)*
+; OPT-NEXT:    [[TMP3:%.*]] = udiv i64 [[N:%.*]], 16
+; OPT-NEXT:    [[TMP4:%.*]] = urem i64 [[N]], 16
+; OPT-NEXT:    [[TMP5:%.*]] = sub i64 [[N]], [[TMP4]]
+; OPT-NEXT:    [[TMP6:%.*]] = icmp ne i64 [[TMP3]], 0
+; OPT-NEXT:    br i1 [[TMP6]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[LOOP_MEMCPY_RESIDUAL_HEADER:%.*]]
+; OPT:       loop-memcpy-expansion:
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP10:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
+; OPT-NEXT:    [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP1]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32> addrspace(1)* [[TMP7]], align 1
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP2]], i64 [[LOOP_INDEX]]
+; OPT-NEXT:    store <4 x i32> [[TMP8]], <4 x i32> addrspace(1)* [[TMP9]], align 1
+; OPT-NEXT:    [[TMP10]] = add i64 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP11:%.*]] = icmp ult i64 [[TMP10]], [[TMP3]]
+; OPT-NEXT:    br i1 [[TMP11]], label [[LOOP_MEMCPY_EXPANSION]], label [[LOOP_MEMCPY_RESIDUAL_HEADER]]
+; OPT:       loop-memcpy-residual:
+; OPT-NEXT:    [[RESIDUAL_LOOP_INDEX:%.*]] = phi i64 [ 0, [[LOOP_MEMCPY_RESIDUAL_HEADER]] ], [ [[TMP18:%.*]], [[LOOP_MEMCPY_RESIDUAL:%.*]] ]
+; OPT-NEXT:    [[TMP12:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP1]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP13:%.*]] = bitcast <4 x i32> addrspace(1)* [[TMP2]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP14:%.*]] = add i64 [[TMP5]], [[RESIDUAL_LOOP_INDEX]]
+; OPT-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP12]], i64 [[TMP14]]
+; OPT-NEXT:    [[TMP16:%.*]] = load i8, i8 addrspace(1)* [[TMP15]], align 1
+; OPT-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP13]], i64 [[TMP14]]
+; OPT-NEXT:    store i8 [[TMP16]], i8 addrspace(1)* [[TMP17]], align 1
+; OPT-NEXT:    [[TMP18]] = add i64 [[RESIDUAL_LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP19:%.*]] = icmp ult i64 [[TMP18]], [[TMP4]]
+; OPT-NEXT:    br i1 [[TMP19]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION:%.*]]
+; OPT:       post-loop-memcpy-expansion:
+; OPT-NEXT:    ret void
+; OPT:       loop-memcpy-residual-header:
+; OPT-NEXT:    [[TMP20:%.*]] = icmp ne i64 [[TMP4]], 0
+; OPT-NEXT:    br i1 [[TMP20]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION]]
+;
+  call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 1 %dst, i8 addrspace(1)* align 1 %src, i64 %n, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_local_align4_local_align4_variable(i8 addrspace(3)* %dst, i8 addrspace(3)* %src, i32 %n) #0 {
+; OPT-LABEL: @memcpy_local_align4_local_align4_variable(
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(3)* [[SRC:%.*]] to <2 x i32> addrspace(3)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(3)* [[DST:%.*]] to <2 x i32> addrspace(3)*
+; OPT-NEXT:    [[TMP3:%.*]] = udiv i32 [[N:%.*]], 8
+; OPT-NEXT:    [[TMP4:%.*]] = urem i32 [[N]], 8
+; OPT-NEXT:    [[TMP5:%.*]] = sub i32 [[N]], [[TMP4]]
+; OPT-NEXT:    [[TMP6:%.*]] = icmp ne i32 [[TMP3]], 0
+; OPT-NEXT:    br i1 [[TMP6]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[LOOP_MEMCPY_RESIDUAL_HEADER:%.*]]
+; OPT:       loop-memcpy-expansion:
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP10:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
+; OPT-NEXT:    [[TMP7:%.*]] = getelementptr inbounds <2 x i32>, <2 x i32> addrspace(3)* [[TMP1]], i32 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP8:%.*]] = load <2 x i32>, <2 x i32> addrspace(3)* [[TMP7]], align 4
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds <2 x i32>, <2 x i32> addrspace(3)* [[TMP2]], i32 [[LOOP_INDEX]]
+; OPT-NEXT:    store <2 x i32> [[TMP8]], <2 x i32> addrspace(3)* [[TMP9]], align 4
+; OPT-NEXT:    [[TMP10]] = add i32 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP11:%.*]] = icmp ult i32 [[TMP10]], [[TMP3]]
+; OPT-NEXT:    br i1 [[TMP11]], label [[LOOP_MEMCPY_EXPANSION]], label [[LOOP_MEMCPY_RESIDUAL_HEADER]]
+; OPT:       loop-memcpy-residual:
+; OPT-NEXT:    [[RESIDUAL_LOOP_INDEX:%.*]] = phi i32 [ 0, [[LOOP_MEMCPY_RESIDUAL_HEADER]] ], [ [[TMP18:%.*]], [[LOOP_MEMCPY_RESIDUAL:%.*]] ]
+; OPT-NEXT:    [[TMP12:%.*]] = bitcast <2 x i32> addrspace(3)* [[TMP1]] to i8 addrspace(3)*
+; OPT-NEXT:    [[TMP13:%.*]] = bitcast <2 x i32> addrspace(3)* [[TMP2]] to i8 addrspace(3)*
+; OPT-NEXT:    [[TMP14:%.*]] = add i32 [[TMP5]], [[RESIDUAL_LOOP_INDEX]]
+; OPT-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, i8 addrspace(3)* [[TMP12]], i32 [[TMP14]]
+; OPT-NEXT:    [[TMP16:%.*]] = load i8, i8 addrspace(3)* [[TMP15]], align 4
+; OPT-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, i8 addrspace(3)* [[TMP13]], i32 [[TMP14]]
+; OPT-NEXT:    store i8 [[TMP16]], i8 addrspace(3)* [[TMP17]], align 4
+; OPT-NEXT:    [[TMP18]] = add i32 [[RESIDUAL_LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP19:%.*]] = icmp ult i32 [[TMP18]], [[TMP4]]
+; OPT-NEXT:    br i1 [[TMP19]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION:%.*]]
+; OPT:       post-loop-memcpy-expansion:
+; OPT-NEXT:    ret void
+; OPT:       loop-memcpy-residual-header:
+; OPT-NEXT:    [[TMP20:%.*]] = icmp ne i32 [[TMP4]], 0
+; OPT-NEXT:    br i1 [[TMP20]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION]]
+;
+  call void @llvm.memcpy.p3i8.p3i8.i32(i8 addrspace(3)* align 4 %dst, i8 addrspace(3)* align 4 %src, i32 %n, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_local_align2_local_align2_variable(i8 addrspace(3)* %dst, i8 addrspace(3)* %src, i32 %n) #0 {
+; OPT-LABEL: @memcpy_local_align2_local_align2_variable(
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(3)* [[SRC:%.*]] to i16 addrspace(3)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(3)* [[DST:%.*]] to i16 addrspace(3)*
+; OPT-NEXT:    [[TMP3:%.*]] = udiv i32 [[N:%.*]], 2
+; OPT-NEXT:    [[TMP4:%.*]] = urem i32 [[N]], 2
+; OPT-NEXT:    [[TMP5:%.*]] = sub i32 [[N]], [[TMP4]]
+; OPT-NEXT:    [[TMP6:%.*]] = icmp ne i32 [[TMP3]], 0
+; OPT-NEXT:    br i1 [[TMP6]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[LOOP_MEMCPY_RESIDUAL_HEADER:%.*]]
+; OPT:       loop-memcpy-expansion:
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP10:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
+; OPT-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i16, i16 addrspace(3)* [[TMP1]], i32 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP8:%.*]] = load i16, i16 addrspace(3)* [[TMP7]], align 2
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i16, i16 addrspace(3)* [[TMP2]], i32 [[LOOP_INDEX]]
+; OPT-NEXT:    store i16 [[TMP8]], i16 addrspace(3)* [[TMP9]], align 2
+; OPT-NEXT:    [[TMP10]] = add i32 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP11:%.*]] = icmp ult i32 [[TMP10]], [[TMP3]]
+; OPT-NEXT:    br i1 [[TMP11]], label [[LOOP_MEMCPY_EXPANSION]], label [[LOOP_MEMCPY_RESIDUAL_HEADER]]
+; OPT:       loop-memcpy-residual:
+; OPT-NEXT:    [[RESIDUAL_LOOP_INDEX:%.*]] = phi i32 [ 0, [[LOOP_MEMCPY_RESIDUAL_HEADER]] ], [ [[TMP18:%.*]], [[LOOP_MEMCPY_RESIDUAL:%.*]] ]
+; OPT-NEXT:    [[TMP12:%.*]] = bitcast i16 addrspace(3)* [[TMP1]] to i8 addrspace(3)*
+; OPT-NEXT:    [[TMP13:%.*]] = bitcast i16 addrspace(3)* [[TMP2]] to i8 addrspace(3)*
+; OPT-NEXT:    [[TMP14:%.*]] = add i32 [[TMP5]], [[RESIDUAL_LOOP_INDEX]]
+; OPT-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, i8 addrspace(3)* [[TMP12]], i32 [[TMP14]]
+; OPT-NEXT:    [[TMP16:%.*]] = load i8, i8 addrspace(3)* [[TMP15]], align 2
+; OPT-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, i8 addrspace(3)* [[TMP13]], i32 [[TMP14]]
+; OPT-NEXT:    store i8 [[TMP16]], i8 addrspace(3)* [[TMP17]], align 2
+; OPT-NEXT:    [[TMP18]] = add i32 [[RESIDUAL_LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP19:%.*]] = icmp ult i32 [[TMP18]], [[TMP4]]
+; OPT-NEXT:    br i1 [[TMP19]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION:%.*]]
+; OPT:       post-loop-memcpy-expansion:
+; OPT-NEXT:    ret void
+; OPT:       loop-memcpy-residual-header:
+; OPT-NEXT:    [[TMP20:%.*]] = icmp ne i32 [[TMP4]], 0
+; OPT-NEXT:    br i1 [[TMP20]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION]]
+;
+  call void @llvm.memcpy.p3i8.p3i8.i32(i8 addrspace(3)* align 2 %dst, i8 addrspace(3)* align 2 %src, i32 %n, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_local_align1_local_align1_variable(i8 addrspace(3)* %dst, i8 addrspace(3)* %src, i32 %n) #0 {
+; OPT-LABEL: @memcpy_local_align1_local_align1_variable(
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(3)* [[SRC:%.*]] to <2 x i32> addrspace(3)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(3)* [[DST:%.*]] to <2 x i32> addrspace(3)*
+; OPT-NEXT:    [[TMP3:%.*]] = udiv i32 [[N:%.*]], 8
+; OPT-NEXT:    [[TMP4:%.*]] = urem i32 [[N]], 8
+; OPT-NEXT:    [[TMP5:%.*]] = sub i32 [[N]], [[TMP4]]
+; OPT-NEXT:    [[TMP6:%.*]] = icmp ne i32 [[TMP3]], 0
+; OPT-NEXT:    br i1 [[TMP6]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[LOOP_MEMCPY_RESIDUAL_HEADER:%.*]]
+; OPT:       loop-memcpy-expansion:
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP10:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
+; OPT-NEXT:    [[TMP7:%.*]] = getelementptr inbounds <2 x i32>, <2 x i32> addrspace(3)* [[TMP1]], i32 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP8:%.*]] = load <2 x i32>, <2 x i32> addrspace(3)* [[TMP7]], align 1
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds <2 x i32>, <2 x i32> addrspace(3)* [[TMP2]], i32 [[LOOP_INDEX]]
+; OPT-NEXT:    store <2 x i32> [[TMP8]], <2 x i32> addrspace(3)* [[TMP9]], align 1
+; OPT-NEXT:    [[TMP10]] = add i32 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP11:%.*]] = icmp ult i32 [[TMP10]], [[TMP3]]
+; OPT-NEXT:    br i1 [[TMP11]], label [[LOOP_MEMCPY_EXPANSION]], label [[LOOP_MEMCPY_RESIDUAL_HEADER]]
+; OPT:       loop-memcpy-residual:
+; OPT-NEXT:    [[RESIDUAL_LOOP_INDEX:%.*]] = phi i32 [ 0, [[LOOP_MEMCPY_RESIDUAL_HEADER]] ], [ [[TMP18:%.*]], [[LOOP_MEMCPY_RESIDUAL:%.*]] ]
+; OPT-NEXT:    [[TMP12:%.*]] = bitcast <2 x i32> addrspace(3)* [[TMP1]] to i8 addrspace(3)*
+; OPT-NEXT:    [[TMP13:%.*]] = bitcast <2 x i32> addrspace(3)* [[TMP2]] to i8 addrspace(3)*
+; OPT-NEXT:    [[TMP14:%.*]] = add i32 [[TMP5]], [[RESIDUAL_LOOP_INDEX]]
+; OPT-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, i8 addrspace(3)* [[TMP12]], i32 [[TMP14]]
+; OPT-NEXT:    [[TMP16:%.*]] = load i8, i8 addrspace(3)* [[TMP15]], align 1
+; OPT-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, i8 addrspace(3)* [[TMP13]], i32 [[TMP14]]
+; OPT-NEXT:    store i8 [[TMP16]], i8 addrspace(3)* [[TMP17]], align 1
+; OPT-NEXT:    [[TMP18]] = add i32 [[RESIDUAL_LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP19:%.*]] = icmp ult i32 [[TMP18]], [[TMP4]]
+; OPT-NEXT:    br i1 [[TMP19]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION:%.*]]
+; OPT:       post-loop-memcpy-expansion:
+; OPT-NEXT:    ret void
+; OPT:       loop-memcpy-residual-header:
+; OPT-NEXT:    [[TMP20:%.*]] = icmp ne i32 [[TMP4]], 0
+; OPT-NEXT:    br i1 [[TMP20]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION]]
+;
+  call void @llvm.memcpy.p3i8.p3i8.i32(i8 addrspace(3)* align 1 %dst, i8 addrspace(3)* align 1 %src, i32 %n, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_local_align4_global_align4_variable(i8 addrspace(3)* %dst, i8 addrspace(1)* %src, i32 %n) #0 {
+; OPT-LABEL: @memcpy_local_align4_global_align4_variable(
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SRC:%.*]] to <2 x i32> addrspace(1)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(3)* [[DST:%.*]] to <2 x i32> addrspace(3)*
+; OPT-NEXT:    [[TMP3:%.*]] = udiv i32 [[N:%.*]], 8
+; OPT-NEXT:    [[TMP4:%.*]] = urem i32 [[N]], 8
+; OPT-NEXT:    [[TMP5:%.*]] = sub i32 [[N]], [[TMP4]]
+; OPT-NEXT:    [[TMP6:%.*]] = icmp ne i32 [[TMP3]], 0
+; OPT-NEXT:    br i1 [[TMP6]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[LOOP_MEMCPY_RESIDUAL_HEADER:%.*]]
+; OPT:       loop-memcpy-expansion:
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP10:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
+; OPT-NEXT:    [[TMP7:%.*]] = getelementptr inbounds <2 x i32>, <2 x i32> addrspace(1)* [[TMP1]], i32 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP8:%.*]] = load <2 x i32>, <2 x i32> addrspace(1)* [[TMP7]], align 4
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds <2 x i32>, <2 x i32> addrspace(3)* [[TMP2]], i32 [[LOOP_INDEX]]
+; OPT-NEXT:    store <2 x i32> [[TMP8]], <2 x i32> addrspace(3)* [[TMP9]], align 4
+; OPT-NEXT:    [[TMP10]] = add i32 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP11:%.*]] = icmp ult i32 [[TMP10]], [[TMP3]]
+; OPT-NEXT:    br i1 [[TMP11]], label [[LOOP_MEMCPY_EXPANSION]], label [[LOOP_MEMCPY_RESIDUAL_HEADER]]
+; OPT:       loop-memcpy-residual:
+; OPT-NEXT:    [[RESIDUAL_LOOP_INDEX:%.*]] = phi i32 [ 0, [[LOOP_MEMCPY_RESIDUAL_HEADER]] ], [ [[TMP18:%.*]], [[LOOP_MEMCPY_RESIDUAL:%.*]] ]
+; OPT-NEXT:    [[TMP12:%.*]] = bitcast <2 x i32> addrspace(1)* [[TMP1]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP13:%.*]] = bitcast <2 x i32> addrspace(3)* [[TMP2]] to i8 addrspace(3)*
+; OPT-NEXT:    [[TMP14:%.*]] = add i32 [[TMP5]], [[RESIDUAL_LOOP_INDEX]]
+; OPT-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP12]], i32 [[TMP14]]
+; OPT-NEXT:    [[TMP16:%.*]] = load i8, i8 addrspace(1)* [[TMP15]], align 4
+; OPT-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, i8 addrspace(3)* [[TMP13]], i32 [[TMP14]]
+; OPT-NEXT:    store i8 [[TMP16]], i8 addrspace(3)* [[TMP17]], align 4
+; OPT-NEXT:    [[TMP18]] = add i32 [[RESIDUAL_LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP19:%.*]] = icmp ult i32 [[TMP18]], [[TMP4]]
+; OPT-NEXT:    br i1 [[TMP19]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION:%.*]]
+; OPT:       post-loop-memcpy-expansion:
+; OPT-NEXT:    ret void
+; OPT:       loop-memcpy-residual-header:
+; OPT-NEXT:    [[TMP20:%.*]] = icmp ne i32 [[TMP4]], 0
+; OPT-NEXT:    br i1 [[TMP20]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION]]
+;
+  call void @llvm.memcpy.p3i8.p1i8.i32(i8 addrspace(3)* align 4 %dst, i8 addrspace(1)* align 4 %src, i32 %n, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_global_align4_local_align4_variable(i8 addrspace(1)* %dst, i8 addrspace(3)* %src, i32 %n) #0 {
+; OPT-LABEL: @memcpy_global_align4_local_align4_variable(
+; OPT-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(3)* [[SRC:%.*]] to <2 x i32> addrspace(3)*
+; OPT-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(1)* [[DST:%.*]] to <2 x i32> addrspace(1)*
+; OPT-NEXT:    [[TMP3:%.*]] = udiv i32 [[N:%.*]], 8
+; OPT-NEXT:    [[TMP4:%.*]] = urem i32 [[N]], 8
+; OPT-NEXT:    [[TMP5:%.*]] = sub i32 [[N]], [[TMP4]]
+; OPT-NEXT:    [[TMP6:%.*]] = icmp ne i32 [[TMP3]], 0
+; OPT-NEXT:    br i1 [[TMP6]], label [[LOOP_MEMCPY_EXPANSION:%.*]], label [[LOOP_MEMCPY_RESIDUAL_HEADER:%.*]]
+; OPT:       loop-memcpy-expansion:
+; OPT-NEXT:    [[LOOP_INDEX:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP10:%.*]], [[LOOP_MEMCPY_EXPANSION]] ]
+; OPT-NEXT:    [[TMP7:%.*]] = getelementptr inbounds <2 x i32>, <2 x i32> addrspace(3)* [[TMP1]], i32 [[LOOP_INDEX]]
+; OPT-NEXT:    [[TMP8:%.*]] = load <2 x i32>, <2 x i32> addrspace(3)* [[TMP7]], align 4
+; OPT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds <2 x i32>, <2 x i32> addrspace(1)* [[TMP2]], i32 [[LOOP_INDEX]]
+; OPT-NEXT:    store <2 x i32> [[TMP8]], <2 x i32> addrspace(1)* [[TMP9]], align 4
+; OPT-NEXT:    [[TMP10]] = add i32 [[LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP11:%.*]] = icmp ult i32 [[TMP10]], [[TMP3]]
+; OPT-NEXT:    br i1 [[TMP11]], label [[LOOP_MEMCPY_EXPANSION]], label [[LOOP_MEMCPY_RESIDUAL_HEADER]]
+; OPT:       loop-memcpy-residual:
+; OPT-NEXT:    [[RESIDUAL_LOOP_INDEX:%.*]] = phi i32 [ 0, [[LOOP_MEMCPY_RESIDUAL_HEADER]] ], [ [[TMP18:%.*]], [[LOOP_MEMCPY_RESIDUAL:%.*]] ]
+; OPT-NEXT:    [[TMP12:%.*]] = bitcast <2 x i32> addrspace(3)* [[TMP1]] to i8 addrspace(3)*
+; OPT-NEXT:    [[TMP13:%.*]] = bitcast <2 x i32> addrspace(1)* [[TMP2]] to i8 addrspace(1)*
+; OPT-NEXT:    [[TMP14:%.*]] = add i32 [[TMP5]], [[RESIDUAL_LOOP_INDEX]]
+; OPT-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, i8 addrspace(3)* [[TMP12]], i32 [[TMP14]]
+; OPT-NEXT:    [[TMP16:%.*]] = load i8, i8 addrspace(3)* [[TMP15]], align 4
+; OPT-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP13]], i32 [[TMP14]]
+; OPT-NEXT:    store i8 [[TMP16]], i8 addrspace(1)* [[TMP17]], align 4
+; OPT-NEXT:    [[TMP18]] = add i32 [[RESIDUAL_LOOP_INDEX]], 1
+; OPT-NEXT:    [[TMP19:%.*]] = icmp ult i32 [[TMP18]], [[TMP4]]
+; OPT-NEXT:    br i1 [[TMP19]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION:%.*]]
+; OPT:       post-loop-memcpy-expansion:
+; OPT-NEXT:    ret void
+; OPT:       loop-memcpy-residual-header:
+; OPT-NEXT:    [[TMP20:%.*]] = icmp ne i32 [[TMP4]], 0
+; OPT-NEXT:    br i1 [[TMP20]], label [[LOOP_MEMCPY_RESIDUAL]], label [[POST_LOOP_MEMCPY_EXPANSION]]
+;
+  call void @llvm.memcpy.p1i8.p3i8.i32(i8 addrspace(1)* align 4 %dst, i8 addrspace(3)* align 4 %src, i32 %n, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_global_align4_global_align4_16(i8 addrspace(1)* %dst, i8 addrspace(1)* %src) #0 {
+; MAX1024-LABEL: @memcpy_global_align4_global_align4_16(
+; MAX1024-NEXT:    call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 [[DST:%.*]], i8 addrspace(1)* align 4 [[SRC:%.*]], i64 16, i1 false)
+; MAX1024-NEXT:    ret void
+;
+; ALL-LABEL: @memcpy_global_align4_global_align4_16(
+; ALL-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SRC:%.*]] to <4 x i32> addrspace(1)*
+; ALL-NEXT:    [[TMP2:%.*]] = bitcast i8 addrspace(1)* [[DST:%.*]] to <4 x i32> addrspace(1)*
+; ALL-NEXT:    br label [[LOAD_STORE_LOOP:%.*]]
+; ALL:       load-store-loop:
+; ALL-NEXT:    [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[LOAD_STORE_LOOP]] ]
+; ALL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP1]], i64 [[LOOP_INDEX]]
+; ALL-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32> addrspace(1)* [[TMP3]], align 4
+; ALL-NEXT:    [[TMP5:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* [[TMP2]], i64 [[LOOP_INDEX]]
+; ALL-NEXT:    store <4 x i32> [[TMP4]], <4 x i32> addrspace(1)* [[TMP5]], align 4
+; ALL-NEXT:    [[TMP6]] = add i64 [[LOOP_INDEX]], 1
+; ALL-NEXT:    [[TMP7:%.*]] = icmp ult i64 [[TMP6]], 1
+; ALL-NEXT:    br i1 [[TMP7]], label [[LOAD_STORE_LOOP]], label [[MEMCPY_SPLIT:%.*]]
+; ALL:       memcpy-split:
+; ALL-NEXT:    ret void
+;
+  call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 %dst, i8 addrspace(1)* align 4 %src, i64 16, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_global_align4_global_align4_12(i8 addrspace(1)* %dst, i8 addrspace(1)* %src) #0 {
+; MAX1024-LABEL: @memcpy_global_align4_global_align4_12(
+; MAX1024-NEXT:    call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 [[DST:%.*]], i8 addrspace(1)* align 4 [[SRC:%.*]], i64 12, i1 false)
+; MAX1024-NEXT:    ret void
+;
+; ALL-LABEL: @memcpy_global_align4_global_align4_12(
+; ALL-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SRC:%.*]] to i64 addrspace(1)*
+; ALL-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, i64 addrspace(1)* [[TMP1]], i64 0
+; ALL-NEXT:    [[TMP3:%.*]] = load i64, i64 addrspace(1)* [[TMP2]], align 4
+; ALL-NEXT:    [[TMP4:%.*]] = bitcast i8 addrspace(1)* [[DST:%.*]] to i64 addrspace(1)*
+; ALL-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, i64 addrspace(1)* [[TMP4]], i64 0
+; ALL-NEXT:    store i64 [[TMP3]], i64 addrspace(1)* [[TMP5]], align 4
+; ALL-NEXT:    [[TMP6:%.*]] = bitcast i8 addrspace(1)* [[SRC]] to i32 addrspace(1)*
+; ALL-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP6]], i64 2
+; ALL-NEXT:    [[TMP8:%.*]] = load i32, i32 addrspace(1)* [[TMP7]], align 4
+; ALL-NEXT:    [[TMP9:%.*]] = bitcast i8 addrspace(1)* [[DST]] to i32 addrspace(1)*
+; ALL-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP9]], i64 2
+; ALL-NEXT:    store i32 [[TMP8]], i32 addrspace(1)* [[TMP10]], align 4
+; ALL-NEXT:    ret void
+;
+  call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 %dst, i8 addrspace(1)* align 4 %src, i64 12, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_global_align4_global_align4_8(i8 addrspace(1)* %dst, i8 addrspace(1)* %src) #0 {
+; MAX1024-LABEL: @memcpy_global_align4_global_align4_8(
+; MAX1024-NEXT:    call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 [[DST:%.*]], i8 addrspace(1)* align 4 [[SRC:%.*]], i64 8, i1 false)
+; MAX1024-NEXT:    ret void
+;
+; ALL-LABEL: @memcpy_global_align4_global_align4_8(
+; ALL-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SRC:%.*]] to i64 addrspace(1)*
+; ALL-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, i64 addrspace(1)* [[TMP1]], i64 0
+; ALL-NEXT:    [[TMP3:%.*]] = load i64, i64 addrspace(1)* [[TMP2]], align 4
+; ALL-NEXT:    [[TMP4:%.*]] = bitcast i8 addrspace(1)* [[DST:%.*]] to i64 addrspace(1)*
+; ALL-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, i64 addrspace(1)* [[TMP4]], i64 0
+; ALL-NEXT:    store i64 [[TMP3]], i64 addrspace(1)* [[TMP5]], align 4
+; ALL-NEXT:    ret void
+;
+  call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 %dst, i8 addrspace(1)* align 4 %src, i64 8, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_global_align4_global_align4_10(i8 addrspace(1)* %dst, i8 addrspace(1)* %src) #0 {
+; MAX1024-LABEL: @memcpy_global_align4_global_align4_10(
+; MAX1024-NEXT:    call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 [[DST:%.*]], i8 addrspace(1)* align 4 [[SRC:%.*]], i64 10, i1 false)
+; MAX1024-NEXT:    ret void
+;
+; ALL-LABEL: @memcpy_global_align4_global_align4_10(
+; ALL-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SRC:%.*]] to i64 addrspace(1)*
+; ALL-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, i64 addrspace(1)* [[TMP1]], i64 0
+; ALL-NEXT:    [[TMP3:%.*]] = load i64, i64 addrspace(1)* [[TMP2]], align 4
+; ALL-NEXT:    [[TMP4:%.*]] = bitcast i8 addrspace(1)* [[DST:%.*]] to i64 addrspace(1)*
+; ALL-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, i64 addrspace(1)* [[TMP4]], i64 0
+; ALL-NEXT:    store i64 [[TMP3]], i64 addrspace(1)* [[TMP5]], align 4
+; ALL-NEXT:    [[TMP6:%.*]] = bitcast i8 addrspace(1)* [[SRC]] to i16 addrspace(1)*
+; ALL-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i16, i16 addrspace(1)* [[TMP6]], i64 4
+; ALL-NEXT:    [[TMP8:%.*]] = load i16, i16 addrspace(1)* [[TMP7]], align 4
+; ALL-NEXT:    [[TMP9:%.*]] = bitcast i8 addrspace(1)* [[DST]] to i16 addrspace(1)*
+; ALL-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i16, i16 addrspace(1)* [[TMP9]], i64 4
+; ALL-NEXT:    store i16 [[TMP8]], i16 addrspace(1)* [[TMP10]], align 4
+; ALL-NEXT:    ret void
+;
+  call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 %dst, i8 addrspace(1)* align 4 %src, i64 10, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_global_align4_global_align4_4(i8 addrspace(1)* %dst, i8 addrspace(1)* %src) #0 {
+; MAX1024-LABEL: @memcpy_global_align4_global_align4_4(
+; MAX1024-NEXT:    call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 [[DST:%.*]], i8 addrspace(1)* align 4 [[SRC:%.*]], i64 4, i1 false)
+; MAX1024-NEXT:    ret void
+;
+; ALL-LABEL: @memcpy_global_align4_global_align4_4(
+; ALL-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SRC:%.*]] to i32 addrspace(1)*
+; ALL-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP1]], i64 0
+; ALL-NEXT:    [[TMP3:%.*]] = load i32, i32 addrspace(1)* [[TMP2]], align 4
+; ALL-NEXT:    [[TMP4:%.*]] = bitcast i8 addrspace(1)* [[DST:%.*]] to i32 addrspace(1)*
+; ALL-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP4]], i64 0
+; ALL-NEXT:    store i32 [[TMP3]], i32 addrspace(1)* [[TMP5]], align 4
+; ALL-NEXT:    ret void
+;
+  call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 %dst, i8 addrspace(1)* align 4 %src, i64 4, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_global_align4_global_align4_2(i8 addrspace(1)* %dst, i8 addrspace(1)* %src) #0 {
+; MAX1024-LABEL: @memcpy_global_align4_global_align4_2(
+; MAX1024-NEXT:    call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 [[DST:%.*]], i8 addrspace(1)* align 4 [[SRC:%.*]], i64 2, i1 false)
+; MAX1024-NEXT:    ret void
+;
+; ALL-LABEL: @memcpy_global_align4_global_align4_2(
+; ALL-NEXT:    [[TMP1:%.*]] = bitcast i8 addrspace(1)* [[SRC:%.*]] to i16 addrspace(1)*
+; ALL-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i16, i16 addrspace(1)* [[TMP1]], i64 0
+; ALL-NEXT:    [[TMP3:%.*]] = load i16, i16 addrspace(1)* [[TMP2]], align 4
+; ALL-NEXT:    [[TMP4:%.*]] = bitcast i8 addrspace(1)* [[DST:%.*]] to i16 addrspace(1)*
+; ALL-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i16, i16 addrspace(1)* [[TMP4]], i64 0
+; ALL-NEXT:    store i16 [[TMP3]], i16 addrspace(1)* [[TMP5]], align 4
+; ALL-NEXT:    ret void
+;
+  call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 %dst, i8 addrspace(1)* align 4 %src, i64 2, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_global_align4_global_align4_1(i8 addrspace(1)* %dst, i8 addrspace(1)* %src) #0 {
+; MAX1024-LABEL: @memcpy_global_align4_global_align4_1(
+; MAX1024-NEXT:    call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 [[DST:%.*]], i8 addrspace(1)* align 4 [[SRC:%.*]], i64 1, i1 false)
+; MAX1024-NEXT:    ret void
+;
+; ALL-LABEL: @memcpy_global_align4_global_align4_1(
+; ALL-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[SRC:%.*]], i64 0
+; ALL-NEXT:    [[TMP2:%.*]] = load i8, i8 addrspace(1)* [[TMP1]], align 4
+; ALL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[DST:%.*]], i64 0
+; ALL-NEXT:    store i8 [[TMP2]], i8 addrspace(1)* [[TMP3]], align 4
+; ALL-NEXT:    ret void
+;
+  call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 %dst, i8 addrspace(1)* align 4 %src, i64 1, i1 false)
   ret void
 }
 
diff --git a/llvm/test/CodeGen/AMDGPU/mad-combine.ll b/llvm/test/CodeGen/AMDGPU/mad-combine.ll
index 09bc371876fc2..c90970a93f9fe 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-combine.ll
@@ -1,14 +1,14 @@
 ; Make sure we still form mad even when unsafe math or fp-contract is allowed instead of fma.
 
-; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-STD  -check-prefix=SI-STD-SAFE -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs -fp-contract=fast < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-STD -check-prefix=SI-STD-SAFE -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-STD -check-prefix=SI-STD-UNSAFE -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-STD  -check-prefix=SI-STD-SAFE -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign -verify-machineinstrs -fp-contract=fast < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-STD -check-prefix=SI-STD-SAFE -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-STD -check-prefix=SI-STD-UNSAFE -check-prefix=FUNC %s
 
 ; FIXME: Remove enable-unsafe-fp-math in RUN line and add flags to IR instrs
 
 ; Make sure we don't form mad with denormals
-; RUN: llc -march=amdgcn -mcpu=tahiti -mattr=+fp32-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-DENORM -check-prefix=SI-DENORM-FASTFMAF -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=verde -mattr=+fp32-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-DENORM -check-prefix=SI-DENORM-SLOWFMAF -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -denormal-fp-math-f32=ieee -fp-contract=fast -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-DENORM -check-prefix=SI-DENORM-FASTFMAF -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=verde -denormal-fp-math-f32=ieee -fp-contract=fast -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-DENORM -check-prefix=SI-DENORM-SLOWFMAF -check-prefix=FUNC %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() #0
 declare float @llvm.fabs.f32(float) #0
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll
index 6c27690fb2b77..f20e29f178492 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll
@@ -143,5 +143,5 @@ declare float @llvm.maxnum.f32(float, float) #1
 declare float @llvm.fmuladd.f32(float, float, float) #1
 declare <2 x float> @llvm.fmuladd.v2f32(<2 x float>, <2 x float>, <2 x float>) #1
 
-attributes #0 = { nounwind }
+attributes #0 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
 attributes #1 = { nounwind readnone speculatable }
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
index 43e9611acc2ce..db2ed78a15f0f 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
@@ -310,5 +310,5 @@ declare <2 x float> @llvm.fmuladd.v2f32(<2 x float>, <2 x float>, <2 x float>) #
 declare <3 x float> @llvm.fmuladd.v3f32(<3 x float>, <3 x float>, <3 x float>) #1
 declare <4 x float> @llvm.fmuladd.v4f32(<4 x float>, <4 x float>, <4 x float>) #1
 
-attributes #0 = { nounwind }
+attributes #0 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
 attributes #1 = { nounwind readnone speculatable }
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix.ll b/llvm/test/CodeGen/AMDGPU/mad-mix.ll
index 0116f49d8bced..a7126121f09fa 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix.ll
@@ -557,6 +557,6 @@ declare float @llvm.maxnum.f32(float, float) #2
 declare float @llvm.fmuladd.f32(float, float, float) #2
 declare <2 x float> @llvm.fmuladd.v2f32(<2 x float>, <2 x float>, <2 x float>) #2
 
-attributes #0 = { nounwind "target-features"="-fp32-denormals" }
-attributes #1 = { nounwind "target-features"="+fp32-denormals" }
+attributes #0 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
+attributes #1 = { nounwind "denormal-fp-math-f32"="ieee,ieee" }
 attributes #2 = { nounwind readnone speculatable }
diff --git a/llvm/test/CodeGen/AMDGPU/madak.ll b/llvm/test/CodeGen/AMDGPU/madak.ll
index fe710f7c79bbf..c92244b4f9903 100644
--- a/llvm/test/CodeGen/AMDGPU/madak.ll
+++ b/llvm/test/CodeGen/AMDGPU/madak.ll
@@ -19,7 +19,7 @@ declare float @llvm.fabs.f32(float) nounwind readnone
 ; MAD:   v_madak_f32 {{v[0-9]+}}, [[VA]], [[VB]], 0x41200000
 ; GFX10-MAD:   v_madak_f32 {{v[0-9]+}}, [[VA]], [[VB]], 0x41200000
 ; FMA:   v_fmaak_f32 {{v[0-9]+}}, [[VA]], [[VB]], 0x41200000
-define amdgpu_kernel void @madak_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in.a, float addrspace(1)* noalias %in.b) nounwind {
+define amdgpu_kernel void @madak_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in.a, float addrspace(1)* noalias %in.b) #0 {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %in.a.gep = getelementptr float, float addrspace(1)* %in.a, i32 %tid
   %in.b.gep = getelementptr float, float addrspace(1)* %in.b, i32 %tid
@@ -53,7 +53,7 @@ define amdgpu_kernel void @madak_f32(float addrspace(1)* noalias %out, float add
 ; MAD-DAG:      v_mac_f32_e32 [[VK]], [[VA]], [[VC]]
 ; FMA-DAG:      v_fmac_f32_e32 [[VK]], [[VA]], [[VC]]
 ; GCN:          s_endpgm
-define amdgpu_kernel void @madak_2_use_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) nounwind {
+define amdgpu_kernel void @madak_2_use_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #0 {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
 
   %in.gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
@@ -82,7 +82,7 @@ define amdgpu_kernel void @madak_2_use_f32(float addrspace(1)* noalias %out, flo
 ; MAD: v_madak_f32 {{v[0-9]+}}, 4.0, [[VA]], 0x41200000
 ; GFX10-MAD: v_madak_f32 {{v[0-9]+}}, 4.0, [[VA]], 0x41200000
 ; FMA: v_fmaak_f32 {{v[0-9]+}}, 4.0, [[VA]], 0x41200000
-define amdgpu_kernel void @madak_m_inline_imm_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in.a) nounwind {
+define amdgpu_kernel void @madak_m_inline_imm_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in.a) #0 {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %in.a.gep = getelementptr float, float addrspace(1)* %in.a, i32 %tid
   %out.gep = getelementptr float, float addrspace(1)* %out, i32 %tid
@@ -110,7 +110,7 @@ define amdgpu_kernel void @madak_m_inline_imm_f32(float addrspace(1)* noalias %o
 ; MAD:   v_mad_f32 {{v[0-9]+}}, [[VA]], [[VB]], 4.0
 ; GFX10-MAD:   v_mad_f32 {{v[0-9]+}}, [[VA]], [[VB]], 4.0
 ; FMA:   v_fma_f32 {{v[0-9]+}}, [[VA]], [[VB]], 4.0
-define amdgpu_kernel void @madak_inline_imm_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in.a, float addrspace(1)* noalias %in.b) nounwind {
+define amdgpu_kernel void @madak_inline_imm_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in.a, float addrspace(1)* noalias %in.b) #0 {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %in.a.gep = getelementptr float, float addrspace(1)* %in.a, i32 %tid
   %in.b.gep = getelementptr float, float addrspace(1)* %in.b, i32 %tid
@@ -134,7 +134,7 @@ define amdgpu_kernel void @madak_inline_imm_f32(float addrspace(1)* noalias %out
 ; GFX6_8_9:     v_mac_f32_e32 [[VK]], [[SB]], [[VA]]
 ; GFX10-MAD:    v_mad_f32 v{{[0-9]+}}, [[VA]], [[SB]], 0x41200000
 ; FMA:          v_fma_f32 v{{[0-9]+}}, [[VA]], [[SB]], 0x41200000
-define amdgpu_kernel void @s_v_madak_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in.a, float %b) nounwind {
+define amdgpu_kernel void @s_v_madak_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in.a, float %b) #0 {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %in.a.gep = getelementptr float, float addrspace(1)* %in.a, i32 %tid
   %out.gep = getelementptr float, float addrspace(1)* %out, i32 %tid
@@ -155,7 +155,7 @@ define amdgpu_kernel void @s_v_madak_f32(float addrspace(1)* noalias %out, float
 ; GFX6_8_9:      v_mac_f32_e32 [[VK]], [[SB]], [[VA]]
 ; GFX10-MAD:     v_madak_f32 v{{[0-9]+}}, [[SB]], [[VA]], 0x41200000
 ; FMA:           v_fmaak_f32 v{{[0-9]+}}, [[SB]], [[VA]], 0x41200000
-define amdgpu_kernel void @v_s_madak_f32(float addrspace(1)* noalias %out, float %a, float addrspace(1)* noalias %in.b) nounwind {
+define amdgpu_kernel void @v_s_madak_f32(float addrspace(1)* noalias %out, float %a, float addrspace(1)* noalias %in.b) #0 {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %in.b.gep = getelementptr float, float addrspace(1)* %in.b, i32 %tid
   %out.gep = getelementptr float, float addrspace(1)* %out, i32 %tid
@@ -173,7 +173,7 @@ define amdgpu_kernel void @v_s_madak_f32(float addrspace(1)* noalias %out, float
 ; GFX8_9:  v_mac_f32_e32 {{v[0-9]+}}, {{s[0-9]+}}, {{v[0-9]+}}
 ; GFX10-MAD: v_mac_f32_e64 {{v[0-9]+}}, {{s[0-9]+}}, {{s[0-9]+}}
 ; FMA:       v_fmac_f32_e64 {{v[0-9]+}}, {{s[0-9]+}}, {{s[0-9]+}}
-define amdgpu_kernel void @s_s_madak_f32(float addrspace(1)* %out, float %a, float %b) nounwind {
+define amdgpu_kernel void @s_s_madak_f32(float addrspace(1)* %out, float %a, float %b) #0 {
   %mul = fmul float %a, %b
   %madak = fadd float %mul, 10.0
   store float %madak, float addrspace(1)* %out, align 4
@@ -189,7 +189,7 @@ define amdgpu_kernel void @s_s_madak_f32(float addrspace(1)* %out, float %a, flo
 ; GFX10-MAD: v_mad_f32 {{v[0-9]+}}, |{{v[0-9]+}}|, {{v[0-9]+}}, 0x41200000
 ; FMA:       v_fma_f32 {{v[0-9]+}}, |{{v[0-9]+}}|, {{v[0-9]+}}, 0x41200000
 ; GCN:       s_endpgm
-define amdgpu_kernel void @no_madak_src0_modifier_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in.a, float addrspace(1)* noalias %in.b) nounwind {
+define amdgpu_kernel void @no_madak_src0_modifier_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in.a, float addrspace(1)* noalias %in.b) #0 {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %in.a.gep = getelementptr float, float addrspace(1)* %in.a, i32 %tid
   %in.b.gep = getelementptr float, float addrspace(1)* %in.b, i32 %tid
@@ -215,7 +215,7 @@ define amdgpu_kernel void @no_madak_src0_modifier_f32(float addrspace(1)* noalia
 ; GFX10-MAD: v_mad_f32 {{v[0-9]+}}, {{v[0-9]+}}, |{{v[0-9]+}}|, 0x41200000
 ; FMA:       v_fma_f32 {{v[0-9]+}}, {{v[0-9]+}}, |{{v[0-9]+}}|, 0x41200000
 ; GCN:       s_endpgm
-define amdgpu_kernel void @no_madak_src1_modifier_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in.a, float addrspace(1)* noalias %in.b) nounwind {
+define amdgpu_kernel void @no_madak_src1_modifier_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in.a, float addrspace(1)* noalias %in.b) #0 {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %in.a.gep = getelementptr float, float addrspace(1)* %in.a, i32 %tid
   %in.b.gep = getelementptr float, float addrspace(1)* %in.b, i32 %tid
@@ -265,4 +265,4 @@ bb4:
   ret void
 }
 
-attributes #0 = { nounwind}
+attributes #0 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
diff --git a/llvm/test/CodeGen/AMDGPU/madmk.ll b/llvm/test/CodeGen/AMDGPU/madmk.ll
index c9b9abfabeb2c..0939ba1832568 100644
--- a/llvm/test/CodeGen/AMDGPU/madmk.ll
+++ b/llvm/test/CodeGen/AMDGPU/madmk.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
 ; XUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
 
  ; FIXME: None of these trigger madmk emission anymore. It is still
@@ -12,7 +12,7 @@ declare float @llvm.fabs.f32(float) nounwind readnone
 ; GCN-DAG: buffer_load_dword [[VA:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
 ; GCN-DAG: buffer_load_dword [[VB:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
 ; GCN: v_mac_f32_e32 [[VB]], 0x41200000, [[VA]]
-define amdgpu_kernel void @madmk_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) nounwind {
+define amdgpu_kernel void @madmk_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #0 {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
   %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
@@ -35,7 +35,7 @@ define amdgpu_kernel void @madmk_f32(float addrspace(1)* noalias %out, float add
 ; GCN-DAG: v_mac_f32_e32 [[VB]], [[SK]], [[VA]]
 ; GCN-DAG: v_mac_f32_e32 [[VC]], [[SK]], [[VA]]
 ; GCN: s_endpgm
-define amdgpu_kernel void @madmk_2_use_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) nounwind {
+define amdgpu_kernel void @madmk_2_use_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #0 {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
 
   %in.gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
@@ -64,7 +64,7 @@ define amdgpu_kernel void @madmk_2_use_f32(float addrspace(1)* noalias %out, flo
 ; GCN-DAG: buffer_load_dword [[VA:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
 ; GCN-DAG: buffer_load_dword [[VB:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
 ; GCN: v_mac_f32_e32 [[VB]], 4.0, [[VA]]
-define amdgpu_kernel void @madmk_inline_imm_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) nounwind {
+define amdgpu_kernel void @madmk_inline_imm_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #0 {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
   %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
@@ -83,7 +83,7 @@ define amdgpu_kernel void @madmk_inline_imm_f32(float addrspace(1)* noalias %out
 ; GCN-NOT: v_madmk_f32
 ; GCN: v_mac_f32_e32
 ; GCN: s_endpgm
-define amdgpu_kernel void @s_s_madmk_f32(float addrspace(1)* noalias %out, [8 x i32], float %a, [8 x i32], float %b) nounwind {
+define amdgpu_kernel void @s_s_madmk_f32(float addrspace(1)* noalias %out, [8 x i32], float %a, [8 x i32], float %b) #0 {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %out.gep = getelementptr float, float addrspace(1)* %out, i32 %tid
 
@@ -99,7 +99,7 @@ define amdgpu_kernel void @s_s_madmk_f32(float addrspace(1)* noalias %out, [8 x
 ; GCN: v_mov_b32_e32 [[VREG2:v[0-9]+]], [[SREG]]
 ; GCN: v_mac_f32_e32 [[VREG2]], 0x41200000, [[VREG1]]
 ; GCN: s_endpgm
-define amdgpu_kernel void @v_s_madmk_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in, float %b) nounwind {
+define amdgpu_kernel void @v_s_madmk_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in, float %b) #0 {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
   %out.gep = getelementptr float, float addrspace(1)* %out, i32 %tid
@@ -115,7 +115,7 @@ define amdgpu_kernel void @v_s_madmk_f32(float addrspace(1)* noalias %out, float
 ; GCN-NOT: v_madmk_f32
 ; GCN: v_mac_f32_e32
 ; GCN: s_endpgm
-define amdgpu_kernel void @scalar_vector_madmk_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in, float %a) nounwind {
+define amdgpu_kernel void @scalar_vector_madmk_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in, float %a) #0 {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
   %out.gep = getelementptr float, float addrspace(1)* %out, i32 %tid
@@ -132,7 +132,7 @@ define amdgpu_kernel void @scalar_vector_madmk_f32(float addrspace(1)* noalias %
 ; GCN-DAG: buffer_load_dword [[VB:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
 ; GCN-DAG: s_mov_b32 [[SK:s[0-9]+]], 0x41200000
 ; GCN: v_mad_f32 {{v[0-9]+}}, |[[VA]]|, [[SK]], [[VB]]
-define amdgpu_kernel void @no_madmk_src0_modifier_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) nounwind {
+define amdgpu_kernel void @no_madmk_src0_modifier_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #0 {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
   %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
@@ -153,7 +153,7 @@ define amdgpu_kernel void @no_madmk_src0_modifier_f32(float addrspace(1)* noalia
 ; GCN-DAG: buffer_load_dword [[VA:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
 ; GCN-DAG: buffer_load_dword [[VB:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
 ; GCN: v_mad_f32 {{v[0-9]+}}, {{v[0-9]+}}, {{[sv][0-9]+}}, |{{v[0-9]+}}|
-define amdgpu_kernel void @no_madmk_src2_modifier_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) nounwind {
+define amdgpu_kernel void @no_madmk_src2_modifier_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #0 {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
   %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
@@ -174,7 +174,7 @@ define amdgpu_kernel void @no_madmk_src2_modifier_f32(float addrspace(1)* noalia
 ; GCN: buffer_load_dword [[A:v[0-9]+]]
 ; GCN: s_mov_b32 [[SK:s[0-9]+]], 0x41200000
 ; GCN: v_mad_f32 {{v[0-9]+}}, [[A]], [[SK]], 2.0
-define amdgpu_kernel void @madmk_add_inline_imm_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) nounwind {
+define amdgpu_kernel void @madmk_add_inline_imm_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #0 {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
   %out.gep = getelementptr float, float addrspace(1)* %out, i32 %tid
@@ -191,7 +191,7 @@ define amdgpu_kernel void @madmk_add_inline_imm_f32(float addrspace(1)* noalias
 ; SI: s_or_b64
 ; SI: s_xor_b64
 ; SI: v_mac_f32_e32 {{v[0-9]+}}, 0x472aee8c, {{v[0-9]+}}
-define amdgpu_kernel void @kill_madmk_verifier_error() nounwind {
+define amdgpu_kernel void @kill_madmk_verifier_error() #0 {
 bb:
   br label %bb2
 
@@ -214,4 +214,5 @@ bb6:                                              ; preds = %bb2
 
 declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #1
 
+attributes #0 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
 attributes #1 = { nounwind readnone }
diff --git a/llvm/test/CodeGen/AMDGPU/merge-out-of-order-ldst.ll b/llvm/test/CodeGen/AMDGPU/merge-out-of-order-ldst.ll
new file mode 100644
index 0000000000000..927868c642252
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/merge-out-of-order-ldst.ll
@@ -0,0 +1,28 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+
+@L = external local_unnamed_addr addrspace(3) global [9 x double], align 16
+@Ldisp = external local_unnamed_addr addrspace(3) global [96 x double], align 16
+
+; Stores are reordered during loads merge. This case used to assert while
+; scanning for a paired instruction because it used to expect paired one
+; to follow a base one.
+
+; GCN-LABEL: {{^}}out_of_order_merge:
+; GCN-COUNT2: ds_read2_b64
+; GCN-COUNT3: ds_write_b64
+define amdgpu_kernel void @out_of_order_merge() {
+entry:
+  %gep1 = getelementptr inbounds [96 x double], [96 x double] addrspace(3)* @Ldisp, i32 0, i32 0
+  %gep2 = getelementptr inbounds [96 x double], [96 x double] addrspace(3)* @Ldisp, i32 0, i32 1
+  %tmp12 = load <2 x double>, <2 x double> addrspace(3)* bitcast (double addrspace(3)* getelementptr inbounds ([9 x double], [9 x double] addrspace(3)* @L, i32 0, i32 1) to <2 x double> addrspace(3)*), align 8
+  %tmp14 = extractelement <2 x double> %tmp12, i32 0
+  %tmp15 = extractelement <2 x double> %tmp12, i32 1
+  %add50.i = fadd double %tmp14, %tmp15
+  store double %add50.i, double addrspace(3)* %gep1, align 8
+  %tmp16 = load double, double addrspace(3)* getelementptr inbounds ([9 x double], [9 x double] addrspace(3)* @L, i32 1, i32 0), align 8
+  store double %tmp16, double addrspace(3)* %gep2, align 8
+  %tmp17 = load <2 x double>, <2 x double> addrspace(3)* bitcast (double addrspace(3)* getelementptr inbounds ([9 x double], [9 x double] addrspace(3)* @L, i32 2, i32 1) to <2 x double> addrspace(3)*), align 8
+  %tmp19 = extractelement <2 x double> %tmp17, i32 1
+  store double %tmp19, double addrspace(3)* undef, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/merge-out-of-order-ldst.mir b/llvm/test/CodeGen/AMDGPU/merge-out-of-order-ldst.mir
new file mode 100644
index 0000000000000..17c2e130aa126
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/merge-out-of-order-ldst.mir
@@ -0,0 +1,23 @@
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs -run-pass si-load-store-opt %s -o - | FileCheck -check-prefix=GCN %s
+
+# GCN-LABEL: name: out_of_order_merge
+# GCN: DS_READ2_B64_gfx9
+# GCN: DS_WRITE_B64_gfx9
+# GCN: DS_READ2_B64_gfx9
+# GCN: DS_WRITE_B64_gfx9
+# GCN: DS_WRITE_B64_gfx9
+---
+name:            out_of_order_merge
+body:             |
+  bb.0:
+    %4:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %5:vreg_64 = DS_READ_B64_gfx9 %4, 776, 0, implicit $exec :: (load 8 from `double addrspace(3)* undef`, addrspace 3)
+    %6:vreg_64 = DS_READ_B64_gfx9 %4, 784, 0, implicit $exec :: (load 8 from `double addrspace(3)* undef` + 8, addrspace 3)
+    %17:vreg_64 = DS_READ_B64_gfx9 %4, 840, 0, implicit $exec :: (load 8 from `double addrspace(3)* undef`, addrspace 3)
+    DS_WRITE_B64_gfx9 %4, %17, 8, 0, implicit $exec :: (store 8 into `double addrspace(3)* undef` + 8, addrspace 3)
+    DS_WRITE_B64_gfx9 %4, %6, 0, 0, implicit $exec :: (store 8 into `double addrspace(3)* undef`, align 16, addrspace 3)
+    %24:vreg_64 = DS_READ_B64_gfx9 %4, 928, 0, implicit $exec :: (load 8 from `double addrspace(3)* undef` + 8, addrspace 3)
+    DS_WRITE_B64_gfx9 undef %29:vgpr_32, %5, 0, 0, implicit $exec :: (store 8 into `double addrspace(3)* undef`, addrspace 3)
+    S_ENDPGM 0
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/mfma-loop.ll b/llvm/test/CodeGen/AMDGPU/mfma-loop.ll
index b66c9d41446ff..63c4e5c496562 100644
--- a/llvm/test/CodeGen/AMDGPU/mfma-loop.ll
+++ b/llvm/test/CodeGen/AMDGPU/mfma-loop.ll
@@ -487,5 +487,52 @@ exit:
   ret void
 }
 
+; GCN-LABEL: {{^}}test_mfma_nested_loop_zeroinit:
+
+; GCN-COUNT-32: v_accvgpr_write_b32 a{{[0-9]+}}, 0{{$}}
+
+; Check that we do not copy agprs to vgprs and back in an outer loop.
+
+; GCN: [[OUTER_LOOP:BB[0-9_]+]]:
+; GCN-NOT: v_accvgpr
+; GCN: [[INNER_LOOP:BB[0-9_]+]]:
+; GCN-NOT: v_accvgpr
+; GCN: v_mfma_f32_32x32x1f32
+; GCN-NOT: v_accvgpr
+; GCN: s_cbranch_scc1 [[INNER_LOOP]]
+; GCN-NOT: v_accvgpr
+; GCN: s_cbranch_scc1 [[OUTER_LOOP]]
+
+; Final result should be read only once after the loop.
+
+; GCN-COUNT-32: v_accvgpr_read_b32
+
+define amdgpu_kernel void @test_mfma_nested_loop_zeroinit(<32 x float> addrspace(1)* %arg) {
+entry:
+  br label %for.cond.preheader
+
+for.cond.preheader:
+  %phi.0 = phi <32 x float> [ zeroinitializer, %entry ], [ %mai.1, %inner.exit ]
+  %c.0 = phi i32 [ 0, %entry ], [ %inc.0, %inner.exit ]
+  br label %inner.for.cond.preheader
+
+inner.for.cond.preheader:
+  %phi = phi <32 x float> [ %phi.0, %for.cond.preheader ], [ %mai.1, %inner.for.cond.preheader ]
+  %c = phi i32 [ 0, %for.cond.preheader ], [ %inc, %inner.for.cond.preheader ]
+  %mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %phi, i32 0, i32 0, i32 0)
+  %inc = add nuw nsw i32 %c, 1
+  %cc = icmp eq i32 %inc, 16
+  br i1 %cc, label %inner.exit, label %inner.for.cond.preheader
+
+inner.exit:
+  %inc.0 = add nuw nsw i32 %c.0, 1
+  %cc.0 = icmp eq i32 %inc.0, 16
+  br i1 %cc.0, label %exit, label %for.cond.preheader
+
+exit:
+  store <32 x float> %mai.1, <32 x float> addrspace(1)* %arg
+  ret void
+}
+
 declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float, float, <32 x float>, i32, i32, i32)
 declare i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/mul24-pass-ordering.ll b/llvm/test/CodeGen/AMDGPU/mul24-pass-ordering.ll
index 3eb478896f55f..ffba105f793bd 100644
--- a/llvm/test/CodeGen/AMDGPU/mul24-pass-ordering.ll
+++ b/llvm/test/CodeGen/AMDGPU/mul24-pass-ordering.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 %s
 
 ; Make sure that AMDGPUCodeGenPrepare introduces mul24 intrinsics
 ; after SLSR, as the intrinsics would interfere. It's unclear if these
@@ -254,7 +254,7 @@ define void @slsr1_1(i32 %b.arg, i32 %s.arg) #0 {
 declare void @foo(i32) #0
 declare float @llvm.fmuladd.f32(float, float, float) #1
 
-attributes #0 = { nounwind willreturn }
+attributes #0 = { nounwind willreturn "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
 attributes #1 = { nounwind readnone speculatable }
 
 !0 = !{float 2.500000e+00}
diff --git a/llvm/test/CodeGen/AMDGPU/omod.ll b/llvm/test/CodeGen/AMDGPU/omod.ll
index 19ffc05b66e50..bd9996e1e2298 100644
--- a/llvm/test/CodeGen/AMDGPU/omod.ll
+++ b/llvm/test/CodeGen/AMDGPU/omod.ll
@@ -275,10 +275,10 @@ declare half @llvm.minnum.f16(half, half) #1
 declare half @llvm.maxnum.f16(half, half) #1
 declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1
 
-attributes #0 = { nounwind "no-signed-zeros-fp-math"="true" }
+attributes #0 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" "no-signed-zeros-fp-math"="true" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { nounwind "target-features"="+fp32-denormals" "no-signed-zeros-fp-math"="true" }
-attributes #3 = { nounwind "target-features"="-fp64-fp16-denormals" "no-signed-zeros-fp-math"="true" }
+attributes #2 = { nounwind "denormal-fp-math-f32"="ieee,ieee" "no-signed-zeros-fp-math"="true" }
+attributes #3 = { nounwind "denormal-fp-math"="preserve-sign,preserve-sign" "no-signed-zeros-fp-math"="true" }
 attributes #4 = { nounwind "no-signed-zeros-fp-math"="false" }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/CodeGen/AMDGPU/operand-folding.ll b/llvm/test/CodeGen/AMDGPU/operand-folding.ll
index af00caa9bdbf0..38aeaa8cb9c4c 100644
--- a/llvm/test/CodeGen/AMDGPU/operand-folding.ll
+++ b/llvm/test/CodeGen/AMDGPU/operand-folding.ll
@@ -1,8 +1,8 @@
-; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck %s
 
 ; CHECK-LABEL: {{^}}fold_sgpr:
 ; CHECK: v_add_i32_e32 v{{[0-9]+}}, vcc, s
-define amdgpu_kernel void @fold_sgpr(i32 addrspace(1)* %out, i32 %fold) {
+define amdgpu_kernel void @fold_sgpr(i32 addrspace(1)* %out, i32 %fold) #1 {
 entry:
   %tmp0 = icmp ne i32 %fold, 0
   br i1 %tmp0, label %if, label %endif
@@ -20,7 +20,7 @@ endif:
 
 ; CHECK-LABEL: {{^}}fold_imm:
 ; CHECK: v_or_b32_e32 v{{[0-9]+}}, 5
-define amdgpu_kernel void @fold_imm(i32 addrspace(1)* %out, i32 %cmp) {
+define amdgpu_kernel void @fold_imm(i32 addrspace(1)* %out, i32 %cmp) #1 {
 entry:
   %fold = add i32 3, 2
   %tmp0 = icmp ne i32 %cmp, 0
@@ -46,7 +46,7 @@ endif:
 ; CHECK-DAG: v_mov_b32_e32 v[[VHI:[0-9]+]], [[HI]]
 ; CHECK: buffer_store_dwordx2 v{{\[}}[[VLO]]:[[VHI]]{{\]}},
 
-define amdgpu_kernel void @fold_64bit_constant_add(i64 addrspace(1)* %out, i32 %cmp, i64 %val) {
+define amdgpu_kernel void @fold_64bit_constant_add(i64 addrspace(1)* %out, i32 %cmp, i64 %val) #1 {
 entry:
   %tmp0 = add i64 %val, 1
   store i64 %tmp0, i64 addrspace(1)* %out
@@ -61,7 +61,7 @@ entry:
 ; CHECK: v_xor_b32_e32 v{{[0-9]+}}, 5, v{{[0-9]+}}
 ; CHECK: v_xor_b32_e32 v{{[0-9]+}}, 5, v{{[0-9]+}}
 
-define amdgpu_kernel void @vector_inline(<4 x i32> addrspace(1)* %out) {
+define amdgpu_kernel void @vector_inline(<4 x i32> addrspace(1)* %out) #1 {
 entry:
   %tmp0 = call i32 @llvm.amdgcn.workitem.id.x()
   %tmp1 = add i32 %tmp0, 1
@@ -80,7 +80,7 @@ entry:
 ; CHECK-LABEL: {{^}}imm_one_use:
 ; CHECK: v_xor_b32_e32 v{{[0-9]+}}, 0x64, v{{[0-9]+}}
 
-define amdgpu_kernel void @imm_one_use(i32 addrspace(1)* %out) {
+define amdgpu_kernel void @imm_one_use(i32 addrspace(1)* %out) #1 {
 entry:
   %tmp0 = call i32 @llvm.amdgcn.workitem.id.x()
   %tmp1 = xor i32 %tmp0, 100
@@ -94,7 +94,7 @@ entry:
 ; CHECK: v_xor_b32_e32 v{{[0-9]}}, [[IMM]], v{{[0-9]}}
 ; CHECK: v_xor_b32_e32 v{{[0-9]}}, [[IMM]], v{{[0-9]}}
 
-define amdgpu_kernel void @vector_imm(<4 x i32> addrspace(1)* %out) {
+define amdgpu_kernel void @vector_imm(<4 x i32> addrspace(1)* %out) #1 {
 entry:
   %tmp0 = call i32 @llvm.amdgcn.workitem.id.x()
   %tmp1 = add i32 %tmp0, 1
@@ -114,7 +114,7 @@ entry:
 ; CHECK: buffer_load_dwordx2 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}
 ; CHECK: v_mac_f32_e32 v[[LO]], 0x41200000, v[[HI]]
 ; CHECK: buffer_store_dword v[[LO]]
-define amdgpu_kernel void @no_fold_tied_subregister() {
+define amdgpu_kernel void @no_fold_tied_subregister() #1 {
   %tmp1 = load volatile <2 x float>, <2 x float> addrspace(1)* undef
   %tmp2 = extractelement <2 x float> %tmp1, i32 0
   %tmp3 = extractelement <2 x float> %tmp1, i32 1
@@ -128,7 +128,7 @@ define amdgpu_kernel void @no_fold_tied_subregister() {
 ; CHECK-LABEL: {{^}}no_extra_fold_on_same_opnd
 ; CHECK: v_xor_b32_e32 v{{[0-9]+}}, 0, v{{[0-9]+}}
 ; CHECK: v_xor_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-define void @no_extra_fold_on_same_opnd() {
+define void @no_extra_fold_on_same_opnd() #1 {
 entry:
   %s0 = load i32, i32 addrspace(5)* undef, align 4
   %s0.i64= zext i32 %s0 to i64
@@ -151,3 +151,4 @@ if.else:
 declare i32 @llvm.amdgcn.workitem.id.x() #0
 
 attributes #0 = { nounwind readnone }
+attributes #1 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
diff --git a/llvm/test/CodeGen/AMDGPU/postra-bundle-memops.mir b/llvm/test/CodeGen/AMDGPU/postra-bundle-memops.mir
index 94ed449088a62..2042d8233a5a5 100644
--- a/llvm/test/CodeGen/AMDGPU/postra-bundle-memops.mir
+++ b/llvm/test/CodeGen/AMDGPU/postra-bundle-memops.mir
@@ -9,13 +9,13 @@ body:             |
     ; GCN-LABEL: name: bundle_memops
     ; GCN: $vgpr0 = GLOBAL_LOAD_DWORD undef $vgpr3_vgpr4, 0, 0, 0, 0, implicit $exec
     ; GCN: S_NOP 0
-    ; GCN: BUNDLE implicit-def $vgpr0, implicit-def $vgpr1, implicit undef $vgpr3_vgpr4, implicit $exec {
+    ; GCN: BUNDLE implicit-def $vgpr0, implicit-def $vgpr0_lo16, implicit-def $vgpr0_hi16, implicit-def $vgpr1, implicit-def $vgpr1_lo16, implicit-def $vgpr1_hi16, implicit undef $vgpr3_vgpr4, implicit $exec {
     ; GCN:   $vgpr0 = GLOBAL_LOAD_DWORD undef $vgpr3_vgpr4, 0, 0, 0, 0, implicit $exec
     ; GCN:   $vgpr1 = GLOBAL_LOAD_DWORD undef $vgpr3_vgpr4, 4, 0, 0, 0, implicit $exec
     ; GCN: }
     ; GCN: S_NOP 0
     ; GCN: $vgpr0 = GLOBAL_LOAD_DWORD undef $vgpr3_vgpr4, 0, 0, 0, 0, implicit $exec
-    ; GCN: BUNDLE implicit-def $vgpr1, implicit-def $vgpr2, implicit-def $vgpr5, implicit undef $vgpr0_vgpr1, implicit $exec, implicit undef $vgpr3_vgpr4 {
+    ; GCN: BUNDLE implicit-def $vgpr1, implicit-def $vgpr1_lo16, implicit-def $vgpr1_hi16, implicit-def $vgpr2, implicit-def $vgpr2_lo16, implicit-def $vgpr2_hi16, implicit-def $vgpr5, implicit-def $vgpr5_lo16, implicit-def $vgpr5_hi16, implicit undef $vgpr0_vgpr1, implicit $exec, implicit undef $vgpr3_vgpr4 {
     ; GCN:   $vgpr1 = GLOBAL_LOAD_DWORD undef $vgpr0_vgpr1, 4, 0, 0, 0, implicit $exec
     ; GCN:   $vgpr2 = GLOBAL_LOAD_DWORD undef $vgpr3_vgpr4, 4, 0, 0, 0, implicit $exec
     ; GCN:   $vgpr5 = GLOBAL_LOAD_DWORD undef $vgpr3_vgpr4, 0, 0, 0, 0, implicit $exec
@@ -34,7 +34,7 @@ body:             |
     ; GCN: $vgpr0 = GLOBAL_LOAD_DWORD undef $vgpr3_vgpr4, 0, 0, 0, 0, implicit $exec
     ; GCN: S_NOP 0
     ; GCN: GLOBAL_STORE_DWORD undef $vgpr3_vgpr4, $vgpr0, 4, 0, 0, 0, implicit $exec
-    ; GCN: BUNDLE implicit-def $vgpr2, implicit-def $vgpr3, implicit $vgpr0, implicit $exec, implicit $vgpr1 {
+    ; GCN: BUNDLE implicit-def $vgpr2, implicit-def $vgpr2_lo16, implicit-def $vgpr2_hi16, implicit-def $vgpr3, implicit-def $vgpr3_lo16, implicit-def $vgpr3_hi16, implicit $vgpr0, implicit $exec, implicit $vgpr1 {
     ; GCN:   $vgpr2 = DS_READ_B32_gfx9 $vgpr0, 0, 0, implicit $exec
     ; GCN:   $vgpr3 = DS_READ_B32_gfx9 $vgpr1, 0, 0, implicit $exec
     ; GCN: }
@@ -47,7 +47,7 @@ body:             |
     ; GCN:   $sgpr2 = S_LOAD_DWORD_IMM undef $sgpr0_sgpr1, 0, 0, 0
     ; GCN:   $sgpr3 = S_LOAD_DWORD_SGPR undef $sgpr0_sgpr1, undef $sgpr10, 0, 0
     ; GCN: }
-    ; GCN: BUNDLE implicit-def $vgpr2, implicit-def $vgpr3, implicit $vgpr0, implicit undef $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr2, implicit $exec, implicit $vgpr1 {
+    ; GCN: BUNDLE implicit-def $vgpr2, implicit-def $vgpr2_lo16, implicit-def $vgpr2_hi16, implicit-def $vgpr3, implicit-def $vgpr3_lo16, implicit-def $vgpr3_hi16, implicit $vgpr0, implicit undef $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr2, implicit $exec, implicit $vgpr1 {
     ; GCN:   $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr2, 0, 0, 0, 0, 0, 0, implicit $exec
     ; GCN:   $vgpr3 = BUFFER_LOAD_DWORD_OFFEN $vgpr1, undef $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr2, 0, 0, 0, 0, 0, 0, implicit $exec
     ; GCN: }
@@ -55,7 +55,7 @@ body:             |
     ; GCN:   BUFFER_STORE_DWORD_ADDR64 $vgpr0, $vgpr2_vgpr3, undef $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     ; GCN:   BUFFER_STORE_DWORD_ADDR64 $vgpr0, $vgpr2_vgpr3, undef $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     ; GCN: }
-    ; GCN: BUNDLE implicit-def $vgpr2, implicit-def $vgpr3, implicit undef $vgpr4_vgpr5_vgpr6_vgpr7, implicit undef $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, implicit $exec {
+    ; GCN: BUNDLE implicit-def $vgpr2, implicit-def $vgpr2_lo16, implicit-def $vgpr2_hi16, implicit-def $vgpr3, implicit-def $vgpr3_lo16, implicit-def $vgpr3_hi16, implicit undef $vgpr4_vgpr5_vgpr6_vgpr7, implicit undef $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, implicit $exec {
     ; GCN:   $vgpr2 = IMAGE_LOAD_V1_V4 undef $vgpr4_vgpr5_vgpr6_vgpr7, undef $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, 2, -1, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     ; GCN:   $vgpr3 = IMAGE_LOAD_V1_V4 undef $vgpr4_vgpr5_vgpr6_vgpr7, undef $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, 2, -1, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     ; GCN: }
@@ -67,7 +67,7 @@ body:             |
     ; GCN: $sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71 = S_LOAD_DWORDX8_IMM undef $sgpr10_sgpr11, 464, 0, 0
     ; GCN: $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75 = S_LOAD_DWORDX8_IMM undef $sgpr10_sgpr11, 128, 0, 0
     ; GCN: S_NOP 0
-    ; GCN: BUNDLE implicit-def $vgpr2, implicit-def $vgpr3, implicit $vgpr0, implicit $exec, implicit $vgpr1 {
+    ; GCN: BUNDLE implicit-def $vgpr2, implicit-def $vgpr2_lo16, implicit-def $vgpr2_hi16, implicit-def $vgpr3, implicit-def $vgpr3_lo16, implicit-def $vgpr3_hi16, implicit $vgpr0, implicit $exec, implicit $vgpr1 {
     ; GCN:   $vgpr2 = DS_READ_B32_gfx9 $vgpr0, 0, 0, implicit $exec
     ; GCN:   $vgpr3 = DS_READ_B32_gfx9 $vgpr1, 0, 0, implicit $exec
     ; GCN: }
diff --git a/llvm/test/CodeGen/AMDGPU/postra-machine-sink.mir b/llvm/test/CodeGen/AMDGPU/postra-machine-sink.mir
index c77d6e0eb1be1..ff74c129942a8 100644
--- a/llvm/test/CodeGen/AMDGPU/postra-machine-sink.mir
+++ b/llvm/test/CodeGen/AMDGPU/postra-machine-sink.mir
@@ -5,7 +5,7 @@
 # CHECK-LABEL: bb.0:
 # CHECK: renamable $sgpr1 = COPY renamable $sgpr2
 # CHECK-LABEL: bb.1:
-# CHECK: liveins: $sgpr0_sgpr1:0x0000000000000003
+# CHECK: liveins: $sgpr0_sgpr1:0x000000000000000F
 # CHECK: renamable $vgpr1_vgpr2 = COPY renamable $sgpr0_sgpr1
 
 ---
diff --git a/llvm/test/CodeGen/AMDGPU/propagate-attributes-clone.ll b/llvm/test/CodeGen/AMDGPU/propagate-attributes-clone.ll
index cb0406c5b1f8b..438445279b740 100644
--- a/llvm/test/CodeGen/AMDGPU/propagate-attributes-clone.ll
+++ b/llvm/test/CodeGen/AMDGPU/propagate-attributes-clone.ll
@@ -1,20 +1,65 @@
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -O1 < %s | FileCheck -check-prefix=OPT %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -O1 < %s | FileCheck -check-prefixes=OPT,OPT-EXT %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -O1 --amdgpu-internalize-symbols < %s | FileCheck -check-prefixes=OPT,OPT-INT %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=LLC %s
 
 ; OPT: declare void @foo4() local_unnamed_addr #0
-; OPT: define internal fastcc void @foo3.2() unnamed_addr #1
-; OPT: define void @foo2() local_unnamed_addr #1
-; OPT: define internal fastcc void @foo1.1() unnamed_addr #1
+; OPT: define internal fastcc void @0() unnamed_addr #1
+; OPT-EXT: define void @foo3() local_unnamed_addr #1
+; OPT-INT: define internal fastcc void @foo3.2() unnamed_addr #1
+; OPT-EXT: define void @foo2() local_unnamed_addr #1
+; OPT-INT: define internal fastcc void @foo2.3() unnamed_addr #1
+; OPT-EXT: define void @foo1() local_unnamed_addr #1
+; OPT-EXT:  tail call void @foo4()
+; OPT-EXT:  tail call void @foo3()
+; OPT-EXT:  tail call void @foo2()
+; OPT-EXT:  tail call void @foo2()
+; OPT-EXT:  tail call void @foo1()
+; OPT-EXT:  tail call fastcc void @0()
+; OPT-INT: define internal fastcc void @foo1.1() unnamed_addr #1
+; OPT-INT:  tail call void @foo4()
+; OPT-INT:  tail call fastcc void @foo3.2()
+; OPT-INT:  tail call fastcc void @foo2.3()
+; OPT-INT:  tail call fastcc void @foo2.3()
+; OPT-INT:  tail call fastcc void @foo1.1()
+; OPT-INT:  tail call fastcc void @0()
+; OPT:      ret void
 ; OPT: define amdgpu_kernel void @kernel1() local_unnamed_addr #2
+; OPT-EXT:  tail call fastcc void @foo1.1()
+; OPT-INT:  tail call fastcc void @foo1()
+; OPT:      ret void
 ; OPT: define amdgpu_kernel void @kernel2() local_unnamed_addr #3
+; OPT-EXT:  tail call void @foo2()
+; OPT-INT:  tail call fastcc void @foo2.3()
+; OPT:      ret void
 ; OPT: define amdgpu_kernel void @kernel3() local_unnamed_addr #3
-; OPT: define void @foo1() local_unnamed_addr #4
-; OPT: define void @foo3() local_unnamed_addr #4
-; OPT: attributes #0 = { {{.*}} "target-features"="+wavefrontsize64" }
+; OPT-EXT:  tail call void @foo1()
+; OPT-INT:  tail call fastcc void @foo1.1()
+; OPT:      ret void
+; OPT-EXT: define internal fastcc void @foo1.1() unnamed_addr #4
+; OPT-EXT:  tail call void @foo4()
+; OPT-EXT:  tail call fastcc void @foo3.2()
+; OPT-EXT:  tail call fastcc void @foo2.3()
+; OPT-EXT:  tail call fastcc void @foo2.3()
+; OPT-EXT:  tail call fastcc void @foo1.1()
+; OPT-EXT:  tail call fastcc void @1()
+; OPT-INT: define internal fastcc void @foo1() unnamed_addr #4
+; OPT-INT:  tail call void @foo4()
+; OPT-INT:  tail call fastcc void @foo3()
+; OPT-INT:  tail call fastcc void @foo2()
+; OPT-INT:  tail call fastcc void @foo2()
+; OPT-INT:  tail call fastcc void @foo1()
+; OPT-INT:  tail call fastcc void @1()
+; OPT:      ret void
+; OPT: define internal fastcc void @1() unnamed_addr #4
+; OPT-EXT: define internal fastcc void @foo3.2() unnamed_addr #4
+; OPT-INT: define internal fastcc void @foo3() unnamed_addr #4
+; OPT-EXT: define internal fastcc void @foo2.3() unnamed_addr #4
+; OPT-INT: define internal fastcc void @foo2() unnamed_addr #4
+; OPT: attributes #0 = { {{.*}} "amdgpu-waves-per-eu"="1,1" "target-features"="+wavefrontsize64" }
 ; OPT: attributes #1 = { {{.*}} "target-features"="{{.*}},-wavefrontsize16,-wavefrontsize32,+wavefrontsize64{{.*}}" }
-; OPT: attributes #2 = { {{.*}} "target-features"="+wavefrontsize32" }
+; OPT: attributes #2 = { {{.*}} "amdgpu-waves-per-eu"="2,4" "target-features"="+wavefrontsize32" }
 ; OPT: attributes #3 = { {{.*}} "target-features"="+wavefrontsize64" }
-; OPT: attributes #4 = { {{.*}} "target-features"="{{.*}},-wavefrontsize16,+wavefrontsize32,-wavefrontsize64{{.*}}" }
+; OPT: attributes #4 = { {{.*}} "amdgpu-waves-per-eu"="2,4" "target-features"="{{.*}},-wavefrontsize16,+wavefrontsize32,-wavefrontsize64{{.*}}" }
 
 ; LLC: foo3:
 ; LLC: sample asm
@@ -29,6 +74,8 @@
 ; LLC: foo2@gotpcrel32@hi+4
 ; LLC: foo1@gotpcrel32@lo+4
 ; LLC: foo1@gotpcrel32@hi+4
+; LLC: __unnamed_1@gotpcrel32@lo+4
+; LLC: __unnamed_1@gotpcrel32@hi+4
 ; LLC: kernel1:
 ; LLC: foo1@gotpcrel32@lo+4
 ; LLC: foo1@gotpcrel32@hi+4
@@ -41,7 +88,13 @@
 
 declare void @foo4() #1
 
-define void @foo3() #1 {
+define void @0() #1 {
+entry:
+  call void asm sideeffect "; sample asm", ""()
+  ret void
+}
+
+define void @foo3() #4 {
 entry:
   call void asm sideeffect "; sample asm", ""()
   ret void
@@ -60,6 +113,7 @@ entry:
   tail call void @foo2()
   tail call void @foo2()
   tail call void @foo1()
+  tail call void @0()
   ret void
 }
 
@@ -81,7 +135,8 @@ entry:
   ret void
 }
 
-attributes #0 = { nounwind "target-features"="+wavefrontsize32" }
-attributes #1 = { noinline nounwind "target-features"="+wavefrontsize64" }
+attributes #0 = { nounwind "target-features"="+wavefrontsize32" "amdgpu-waves-per-eu"="2,4" }
+attributes #1 = { noinline nounwind "target-features"="+wavefrontsize64" "amdgpu-waves-per-eu"="1,1" }
 attributes #2 = { nounwind "target-features"="+wavefrontsize64" }
 attributes #3 = { nounwind "target-features"="+wavefrontsize64" }
+attributes #4 = { noinline nounwind "target-features"="+wavefrontsize64" "amdgpu-waves-per-eu"="2,4" }
diff --git a/llvm/test/CodeGen/AMDGPU/rcp-pattern.ll b/llvm/test/CodeGen/AMDGPU/rcp-pattern.ll
index b7552b06f089e..4702cc7000a6a 100644
--- a/llvm/test/CodeGen/AMDGPU/rcp-pattern.ll
+++ b/llvm/test/CodeGen/AMDGPU/rcp-pattern.ll
@@ -150,8 +150,8 @@ define amdgpu_kernel void @div_arcp_neg_k_x_pat_f32(float addrspace(1)* %out) #0
 declare float @llvm.fabs.f32(float) #1
 declare float @llvm.sqrt.f32(float) #1
 
-attributes #0 = { nounwind "unsafe-fp-math"="false" }
+attributes #0 = { nounwind "unsafe-fp-math"="false" "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { nounwind "unsafe-fp-math"="true" }
+attributes #2 = { nounwind "unsafe-fp-math"="true" "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
 
 !0 = !{float 2.500000e+00}
diff --git a/llvm/test/CodeGen/AMDGPU/rcp_iflag.ll b/llvm/test/CodeGen/AMDGPU/rcp_iflag.ll
index badaae3af2376..b95efaf323eda 100644
--- a/llvm/test/CodeGen/AMDGPU/rcp_iflag.ll
+++ b/llvm/test/CodeGen/AMDGPU/rcp_iflag.ll
@@ -2,7 +2,7 @@
 
 ; GCN-LABEL: {{^}}rcp_uint:
 ; GCN: v_rcp_iflag_f32_e32
-define amdgpu_kernel void @rcp_uint(i32 addrspace(1)* %in, float addrspace(1)* %out) {
+define amdgpu_kernel void @rcp_uint(i32 addrspace(1)* %in, float addrspace(1)* %out) #0 {
   %load = load i32, i32 addrspace(1)* %in, align 4
   %cvt = uitofp i32 %load to float
   %div = fdiv float 1.000000e+00, %cvt, !fpmath !0
@@ -12,7 +12,7 @@ define amdgpu_kernel void @rcp_uint(i32 addrspace(1)* %in, float addrspace(1)* %
 
 ; GCN-LABEL: {{^}}rcp_sint:
 ; GCN: v_rcp_iflag_f32_e32
-define amdgpu_kernel void @rcp_sint(i32 addrspace(1)* %in, float addrspace(1)* %out) {
+define amdgpu_kernel void @rcp_sint(i32 addrspace(1)* %in, float addrspace(1)* %out) #0 {
   %load = load i32, i32 addrspace(1)* %in, align 4
   %cvt = sitofp i32 %load to float
   %div = fdiv float 1.000000e+00, %cvt, !fpmath !0
@@ -20,4 +20,27 @@ define amdgpu_kernel void @rcp_sint(i32 addrspace(1)* %in, float addrspace(1)* %
   ret void
 }
 
+; GCN-LABEL: {{^}}rcp_uint_denorm:
+; GCN-NOT: v_rcp_iflag_f32
+define amdgpu_kernel void @rcp_uint_denorm(i32 addrspace(1)* %in, float addrspace(1)* %out) #1 {
+  %load = load i32, i32 addrspace(1)* %in, align 4
+  %cvt = uitofp i32 %load to float
+  %div = fdiv float 1.000000e+00, %cvt
+  store float %div, float addrspace(1)* %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}rcp_sint_denorm:
+; GCN-NOT: v_rcp_iflag_f32
+define amdgpu_kernel void @rcp_sint_denorm(i32 addrspace(1)* %in, float addrspace(1)* %out) #1 {
+  %load = load i32, i32 addrspace(1)* %in, align 4
+  %cvt = sitofp i32 %load to float
+  %div = fdiv float 1.000000e+00, %cvt
+  store float %div, float addrspace(1)* %out, align 4
+  ret void
+}
+
 !0 = !{float 2.500000e+00}
+
+attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
+attributes #1 = { "denormal-fp-math-f32"="ieee,ieee" }
diff --git a/llvm/test/CodeGen/AMDGPU/reorder-stores.ll b/llvm/test/CodeGen/AMDGPU/reorder-stores.ll
index 260b32ed3406c..a379a646bee93 100644
--- a/llvm/test/CodeGen/AMDGPU/reorder-stores.ll
+++ b/llvm/test/CodeGen/AMDGPU/reorder-stores.ll
@@ -1,12 +1,12 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn < %s | FileCheck -check-prefix=SI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefix=SI %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn < %s | FileCheck -check-prefixes=GCN,SI %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GCN,VI %s
 
-; SI-LABEL: {{^}}no_reorder_v2f64_global_load_store:
-; SI: buffer_load_dwordx4
-; SI: buffer_load_dwordx4
-; SI: buffer_store_dwordx4
-; SI: buffer_store_dwordx4
-; SI: s_endpgm
+; GCN-LABEL: {{^}}no_reorder_v2f64_global_load_store:
+; GCN: buffer_load_dwordx4
+; GCN: buffer_load_dwordx4
+; GCN: buffer_store_dwordx4
+; GCN: buffer_store_dwordx4
+; GCN: s_endpgm
 define amdgpu_kernel void @no_reorder_v2f64_global_load_store(<2 x double> addrspace(1)* nocapture %x, <2 x double> addrspace(1)* nocapture %y) nounwind {
   %tmp1 = load <2 x double>, <2 x double> addrspace(1)* %x, align 16
   %tmp4 = load <2 x double>, <2 x double> addrspace(1)* %y, align 16
@@ -15,10 +15,14 @@ define amdgpu_kernel void @no_reorder_v2f64_global_load_store(<2 x double> addrs
   ret void
 }
 
-; SI-LABEL: {{^}}no_reorder_scalarized_v2f64_local_load_store:
+; GCN-LABEL: {{^}}no_reorder_scalarized_v2f64_local_load_store:
 ; SI: ds_read2_b64
 ; SI: ds_write2_b64
-; SI: s_endpgm
+
+; VI: ds_read_b128
+; VI: ds_write_b128
+
+; GCN: s_endpgm
 define amdgpu_kernel void @no_reorder_scalarized_v2f64_local_load_store(<2 x double> addrspace(3)* nocapture %x, <2 x double> addrspace(3)* nocapture %y) nounwind {
   %tmp1 = load <2 x double>, <2 x double> addrspace(3)* %x, align 16
   %tmp4 = load <2 x double>, <2 x double> addrspace(3)* %y, align 16
@@ -27,18 +31,18 @@ define amdgpu_kernel void @no_reorder_scalarized_v2f64_local_load_store(<2 x dou
   ret void
 }
 
-; SI-LABEL: {{^}}no_reorder_split_v8i32_global_load_store:
-; SI: buffer_load_dwordx4
-; SI: buffer_load_dwordx4
-; SI: buffer_load_dwordx4
-; SI: buffer_load_dwordx4
+; GCN-LABEL: {{^}}no_reorder_split_v8i32_global_load_store:
+; GCN: buffer_load_dwordx4
+; GCN: buffer_load_dwordx4
+; GCN: buffer_load_dwordx4
+; GCN: buffer_load_dwordx4
 
 
-; SI: buffer_store_dwordx4
-; SI: buffer_store_dwordx4
-; SI: buffer_store_dwordx4
-; SI: buffer_store_dwordx4
-; SI: s_endpgm
+; GCN: buffer_store_dwordx4
+; GCN: buffer_store_dwordx4
+; GCN: buffer_store_dwordx4
+; GCN: buffer_store_dwordx4
+; GCN: s_endpgm
 define amdgpu_kernel void @no_reorder_split_v8i32_global_load_store(<8 x i32> addrspace(1)* nocapture %x, <8 x i32> addrspace(1)* nocapture %y) nounwind {
   %tmp1 = load <8 x i32>, <8 x i32> addrspace(1)* %x, align 32
   %tmp4 = load <8 x i32>, <8 x i32> addrspace(1)* %y, align 32
@@ -47,13 +51,13 @@ define amdgpu_kernel void @no_reorder_split_v8i32_global_load_store(<8 x i32> ad
   ret void
 }
 
-; SI-LABEL: {{^}}no_reorder_extload_64:
-; SI: ds_read_b64
-; SI: ds_read_b64
-; SI: ds_write_b64
-; SI-NOT: ds_read
-; SI: ds_write_b64
-; SI: s_endpgm
+; GCN-LABEL: {{^}}no_reorder_extload_64:
+; GCN: ds_read_b64
+; GCN: ds_read_b64
+; GCN: ds_write_b64
+; GCN-NOT: ds_read
+; GCN: ds_write_b64
+; GCN: s_endpgm
 define amdgpu_kernel void @no_reorder_extload_64(<2 x i32> addrspace(3)* nocapture %x, <2 x i32> addrspace(3)* nocapture %y) nounwind {
   %tmp1 = load <2 x i32>, <2 x i32> addrspace(3)* %x, align 8
   %tmp4 = load <2 x i32>, <2 x i32> addrspace(3)* %y, align 8
diff --git a/llvm/test/CodeGen/AMDGPU/rsq.ll b/llvm/test/CodeGen/AMDGPU/rsq.ll
index 8480f344601b4..4dd5b55170742 100644
--- a/llvm/test/CodeGen/AMDGPU/rsq.ll
+++ b/llvm/test/CodeGen/AMDGPU/rsq.ll
@@ -1,5 +1,5 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mattr=-fp32-denormals -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=SI-UNSAFE -check-prefix=SI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mattr=-fp32-denormals -verify-machineinstrs < %s | FileCheck -check-prefix=SI-SAFE -check-prefix=SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=SI-UNSAFE -check-prefix=SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI-SAFE -check-prefix=SI %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
 declare float @llvm.sqrt.f32(float) nounwind readnone
@@ -8,7 +8,7 @@ declare double @llvm.sqrt.f64(double) nounwind readnone
 ; SI-LABEL: {{^}}rsq_f32:
 ; SI: v_rsq_f32_e32
 ; SI: s_endpgm
-define amdgpu_kernel void @rsq_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) nounwind {
+define amdgpu_kernel void @rsq_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #0 {
   %val = load float, float addrspace(1)* %in, align 4
   %sqrt = call float @llvm.sqrt.f32(float %val) nounwind readnone
   %div = fdiv float 1.0, %sqrt, !fpmath !0
@@ -20,7 +20,7 @@ define amdgpu_kernel void @rsq_f32(float addrspace(1)* noalias %out, float addrs
 ; SI-UNSAFE: v_rsq_f64_e32
 ; SI-SAFE: v_sqrt_f64_e32
 ; SI: s_endpgm
-define amdgpu_kernel void @rsq_f64(double addrspace(1)* noalias %out, double addrspace(1)* noalias %in) nounwind {
+define amdgpu_kernel void @rsq_f64(double addrspace(1)* noalias %out, double addrspace(1)* noalias %in) #0 {
   %val = load double, double addrspace(1)* %in, align 4
   %sqrt = call double @llvm.sqrt.f64(double %val) nounwind readnone
   %div = fdiv double 1.0, %sqrt
@@ -31,7 +31,7 @@ define amdgpu_kernel void @rsq_f64(double addrspace(1)* noalias %out, double add
 ; SI-LABEL: {{^}}rsq_f32_sgpr:
 ; SI: v_rsq_f32_e32 {{v[0-9]+}}, {{s[0-9]+}}
 ; SI: s_endpgm
-define amdgpu_kernel void @rsq_f32_sgpr(float addrspace(1)* noalias %out, float %val) nounwind {
+define amdgpu_kernel void @rsq_f32_sgpr(float addrspace(1)* noalias %out, float %val) #0 {
   %sqrt = call float @llvm.sqrt.f32(float %val) nounwind readnone
   %div = fdiv float 1.0, %sqrt, !fpmath !0
   store float %div, float addrspace(1)* %out, align 4
@@ -57,7 +57,7 @@ define amdgpu_kernel void @rsq_f32_sgpr(float addrspace(1)* noalias %out, float
 ; SI-SAFE-NOT: v_rsq_f32
 
 ; SI: s_endpgm
-define amdgpu_kernel void @rsqrt_fmul(float addrspace(1)* %out, float addrspace(1)* %in) {
+define amdgpu_kernel void @rsqrt_fmul(float addrspace(1)* %out, float addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %out.gep = getelementptr float, float addrspace(1)* %out, i32 %tid
   %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
@@ -83,7 +83,7 @@ define amdgpu_kernel void @rsqrt_fmul(float addrspace(1)* %out, float addrspace(
 ; SI-UNSAFE: v_sqrt_f32_e32 [[SQRT:v[0-9]+]], v{{[0-9]+}}
 ; SI-UNSAFE: v_rcp_f32_e64 [[RSQ:v[0-9]+]], -[[SQRT]]
 ; SI-UNSAFE: buffer_store_dword [[RSQ]]
-define amdgpu_kernel void @neg_rsq_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) nounwind {
+define amdgpu_kernel void @neg_rsq_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #0 {
   %val = load float, float addrspace(1)* %in, align 4
   %sqrt = call float @llvm.sqrt.f32(float %val)
   %div = fdiv float -1.0, %sqrt, !fpmath !0
@@ -98,7 +98,7 @@ define amdgpu_kernel void @neg_rsq_f32(float addrspace(1)* noalias %out, float a
 ; SI-UNSAFE: v_sqrt_f64_e32 [[SQRT:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}
 ; SI-UNSAFE: v_rcp_f64_e64 [[RCP:v\[[0-9]+:[0-9]+\]]], -[[SQRT]]
 ; SI-UNSAFE: buffer_store_dwordx2 [[RCP]]
-define amdgpu_kernel void @neg_rsq_f64(double addrspace(1)* noalias %out, double addrspace(1)* noalias %in) nounwind {
+define amdgpu_kernel void @neg_rsq_f64(double addrspace(1)* noalias %out, double addrspace(1)* noalias %in) #0 {
   %val = load double, double addrspace(1)* %in, align 4
   %sqrt = call double @llvm.sqrt.f64(double %val)
   %div = fdiv double -1.0, %sqrt
@@ -114,7 +114,7 @@ define amdgpu_kernel void @neg_rsq_f64(double addrspace(1)* noalias %out, double
 ; SI-UNSAFE: v_sqrt_f32_e64 [[SQRT:v[0-9]+]], -v{{[0-9]+}}
 ; SI-UNSAFE: v_rcp_f32_e64 [[RSQ:v[0-9]+]], -[[SQRT]]
 ; SI-UNSAFE: buffer_store_dword [[RSQ]]
-define amdgpu_kernel void @neg_rsq_neg_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) nounwind {
+define amdgpu_kernel void @neg_rsq_neg_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #0 {
   %val = load float, float addrspace(1)* %in, align 4
   %val.fneg = fsub float -0.0, %val
   %sqrt = call float @llvm.sqrt.f32(float %val.fneg)
@@ -130,7 +130,7 @@ define amdgpu_kernel void @neg_rsq_neg_f32(float addrspace(1)* noalias %out, flo
 ; SI-UNSAFE: v_sqrt_f64_e64 [[SQRT:v\[[0-9]+:[0-9]+\]]], -v{{\[[0-9]+:[0-9]+\]}}
 ; SI-UNSAFE: v_rcp_f64_e64 [[RCP:v\[[0-9]+:[0-9]+\]]], -[[SQRT]]
 ; SI-UNSAFE: buffer_store_dwordx2 [[RCP]]
-define amdgpu_kernel void @neg_rsq_neg_f64(double addrspace(1)* noalias %out, double addrspace(1)* noalias %in) nounwind {
+define amdgpu_kernel void @neg_rsq_neg_f64(double addrspace(1)* noalias %out, double addrspace(1)* noalias %in) #0 {
   %val = load double, double addrspace(1)* %in, align 4
   %val.fneg = fsub double -0.0, %val
   %sqrt = call double @llvm.sqrt.f64(double %val.fneg)
@@ -140,3 +140,5 @@ define amdgpu_kernel void @neg_rsq_neg_f64(double addrspace(1)* noalias %out, do
 }
 
 !0 = !{float 2.500000e+00}
+
+attributes #0 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
diff --git a/llvm/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir b/llvm/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir
index b954b778dc653..9d3144196eb17 100644
--- a/llvm/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir
+++ b/llvm/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir
@@ -149,7 +149,7 @@
 
   attributes #0 = { argmemonly nounwind }
   attributes #1 = { nounwind readnone speculatable }
-  attributes #2 = { convergent nounwind "amdgpu-dispatch-ptr" "amdgpu-flat-scratch" "amdgpu-work-item-id-y" "amdgpu-work-item-id-z" "target-cpu"="gfx900" "target-features"="+fp32-denormals" }
+  attributes #2 = { convergent nounwind "amdgpu-dispatch-ptr" "amdgpu-flat-scratch" "amdgpu-work-item-id-y" "amdgpu-work-item-id-z" "target-cpu"="gfx900" }
   attributes #3 = { nounwind }
 
   !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll b/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
index 97d5bcce6190c..ae836e447cc56 100644
--- a/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
@@ -1,7 +1,7 @@
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -amdgpu-sdwa-peephole=0 -mattr=-fp64-fp16-denormals -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=NOSDWA,GCN %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -amdgpu-sdwa-peephole -mattr=-fp64-fp16-denormals -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=VI,GFX89,SDWA,GCN %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -amdgpu-sdwa-peephole -mattr=-fp64-fp16-denormals -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9_10,SDWA,GCN %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx1010 -amdgpu-sdwa-peephole -mattr=-fp64-fp16-denormals -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX10,GFX9_10,SDWA,GCN %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -amdgpu-sdwa-peephole=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=NOSDWA,GCN %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -amdgpu-sdwa-peephole -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=VI,GFX89,SDWA,GCN %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -amdgpu-sdwa-peephole -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9_10,SDWA,GCN %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx1010 -amdgpu-sdwa-peephole -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX10,GFX9_10,SDWA,GCN %s
 
 ; GCN-LABEL: {{^}}add_shr_i32:
 ; NOSDWA: v_lshrrev_b32_e32 v[[DST:[0-9]+]], 16, v{{[0-9]+}}
@@ -12,7 +12,7 @@
 ; GFX9: v_add_u32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
 ; GFX10: v_add_nc_u32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
 
-define amdgpu_kernel void @add_shr_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
+define amdgpu_kernel void @add_shr_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
   %a = load i32, i32 addrspace(1)* %in, align 4
   %shr = lshr i32 %a, 16
   %add = add i32 %a, %shr
@@ -28,7 +28,7 @@ define amdgpu_kernel void @add_shr_i32(i32 addrspace(1)* %out, i32 addrspace(1)*
 ; VI: v_subrev_u32_sdwa v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
 ; GFX9: v_sub_u32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; GFX10: v_sub_nc_u32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-define amdgpu_kernel void @sub_shr_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
+define amdgpu_kernel void @sub_shr_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
   %a = load i32, i32 addrspace(1)* %in, align 4
   %shr = lshr i32 %a, 16
   %sub = sub i32 %shr, %a
@@ -44,7 +44,7 @@ define amdgpu_kernel void @sub_shr_i32(i32 addrspace(1)* %out, i32 addrspace(1)*
 
 ; SDWA: v_mul_u32_u24_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
 
-define amdgpu_kernel void @mul_shr_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in1, i32 addrspace(1)* %in2) {
+define amdgpu_kernel void @mul_shr_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in1, i32 addrspace(1)* %in2) #0 {
   %a = load i32, i32 addrspace(1)* %in1, align 4
   %b = load i32, i32 addrspace(1)* %in2, align 4
   %shra = lshr i32 %a, 16
@@ -61,7 +61,7 @@ define amdgpu_kernel void @mul_shr_i32(i32 addrspace(1)* %out, i32 addrspace(1)*
 ; GFX10: v_mul_lo_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; SDWA-NOT: v_mul_u32_u24_sdwa
 
-define amdgpu_kernel void @mul_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %ina, i16 addrspace(1)* %inb) {
+define amdgpu_kernel void @mul_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %ina, i16 addrspace(1)* %inb) #0 {
 entry:
   %a = load i16, i16 addrspace(1)* %ina, align 4
   %b = load i16, i16 addrspace(1)* %inb, align 4
@@ -84,7 +84,7 @@ entry:
 
 ; GFX9_10: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 
-define amdgpu_kernel void @mul_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %ina, <2 x i16> addrspace(1)* %inb) {
+define amdgpu_kernel void @mul_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %ina, <2 x i16> addrspace(1)* %inb) #0 {
 entry:
   %a = load <2 x i16>, <2 x i16> addrspace(1)* %ina, align 4
   %b = load <2 x i16>, <2 x i16> addrspace(1)* %inb, align 4
@@ -111,7 +111,7 @@ entry:
 ; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 
-define amdgpu_kernel void @mul_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %ina, <4 x i16> addrspace(1)* %inb) {
+define amdgpu_kernel void @mul_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %ina, <4 x i16> addrspace(1)* %inb) #0 {
 entry:
   %a = load <4 x i16>, <4 x i16> addrspace(1)* %ina, align 4
   %b = load <4 x i16>, <4 x i16> addrspace(1)* %inb, align 4
@@ -146,7 +146,7 @@ entry:
 ; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; GFX9_10-DAG: v_pk_mul_lo_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 
-define amdgpu_kernel void @mul_v8i16(<8 x i16> addrspace(1)* %out, <8 x i16> addrspace(1)* %ina, <8 x i16> addrspace(1)* %inb) {
+define amdgpu_kernel void @mul_v8i16(<8 x i16> addrspace(1)* %out, <8 x i16> addrspace(1)* %ina, <8 x i16> addrspace(1)* %inb) #0 {
 entry:
   %a = load <8 x i16>, <8 x i16> addrspace(1)* %ina, align 4
   %b = load <8 x i16>, <8 x i16> addrspace(1)* %inb, align 4
@@ -161,7 +161,7 @@ entry:
 ; SDWA: v_mul_f16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; SDWA-NOT: v_mul_f16_sdwa
 
-define amdgpu_kernel void @mul_half(half addrspace(1)* %out, half addrspace(1)* %ina, half addrspace(1)* %inb) {
+define amdgpu_kernel void @mul_half(half addrspace(1)* %out, half addrspace(1)* %ina, half addrspace(1)* %inb) #0 {
 entry:
   %a = load half, half addrspace(1)* %ina, align 4
   %b = load half, half addrspace(1)* %inb, align 4
@@ -184,7 +184,7 @@ entry:
 
 ; GFX9_10: v_pk_mul_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 
-define amdgpu_kernel void @mul_v2half(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %ina, <2 x half> addrspace(1)* %inb) {
+define amdgpu_kernel void @mul_v2half(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %ina, <2 x half> addrspace(1)* %inb) #0 {
 entry:
   %a = load <2 x half>, <2 x half> addrspace(1)* %ina, align 4
   %b = load <2 x half>, <2 x half> addrspace(1)* %inb, align 4
@@ -209,7 +209,7 @@ entry:
 ; GFX9_10-DAG: v_pk_mul_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; GFX9_10-DAG: v_pk_mul_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 
-define amdgpu_kernel void @mul_v4half(<4 x half> addrspace(1)* %out, <4 x half> addrspace(1)* %ina, <4 x half> addrspace(1)* %inb) {
+define amdgpu_kernel void @mul_v4half(<4 x half> addrspace(1)* %out, <4 x half> addrspace(1)* %ina, <4 x half> addrspace(1)* %inb) #0 {
 entry:
   %a = load <4 x half>, <4 x half> addrspace(1)* %ina, align 4
   %b = load <4 x half>, <4 x half> addrspace(1)* %inb, align 4
@@ -240,7 +240,7 @@ entry:
 ; GFX9_10-DAG: v_pk_mul_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; GFX9_10-DAG: v_pk_mul_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 
-define amdgpu_kernel void @mul_v8half(<8 x half> addrspace(1)* %out, <8 x half> addrspace(1)* %ina, <8 x half> addrspace(1)* %inb) {
+define amdgpu_kernel void @mul_v8half(<8 x half> addrspace(1)* %out, <8 x half> addrspace(1)* %ina, <8 x half> addrspace(1)* %inb) #0 {
 entry:
   %a = load <8 x half>, <8 x half> addrspace(1)* %ina, align 4
   %b = load <8 x half>, <8 x half> addrspace(1)* %inb, align 4
@@ -256,7 +256,7 @@ entry:
 ; GFX10: v_mul_lo_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; SDWA-NOT: v_mul_u32_u24_sdwa
 
-define amdgpu_kernel void @mul_i8(i8 addrspace(1)* %out, i8 addrspace(1)* %ina, i8 addrspace(1)* %inb) {
+define amdgpu_kernel void @mul_i8(i8 addrspace(1)* %out, i8 addrspace(1)* %ina, i8 addrspace(1)* %inb) #0 {
 entry:
   %a = load i8, i8 addrspace(1)* %ina, align 4
   %b = load i8, i8 addrspace(1)* %inb, align 4
@@ -285,7 +285,7 @@ entry:
 
 ; GFX10: v_lshlrev_b16_e64 v{{[0-9]+}}, 8, v
 ; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-define amdgpu_kernel void @mul_v2i8(<2 x i8> addrspace(1)* %out, <2 x i8> addrspace(1)* %ina, <2 x i8> addrspace(1)* %inb) {
+define amdgpu_kernel void @mul_v2i8(<2 x i8> addrspace(1)* %out, <2 x i8> addrspace(1)* %ina, <2 x i8> addrspace(1)* %inb) #0 {
 entry:
   %a = load <2 x i8>, <2 x i8> addrspace(1)* %ina, align 4
   %b = load <2 x i8>, <2 x i8> addrspace(1)* %inb, align 4
@@ -315,7 +315,7 @@ entry:
 ; GFX10-DAG: v_mul_lo_u16_e64
 ; GFX10-DAG: v_mul_lo_u16_e64
 
-define amdgpu_kernel void @mul_v4i8(<4 x i8> addrspace(1)* %out, <4 x i8> addrspace(1)* %ina, <4 x i8> addrspace(1)* %inb) {
+define amdgpu_kernel void @mul_v4i8(<4 x i8> addrspace(1)* %out, <4 x i8> addrspace(1)* %ina, <4 x i8> addrspace(1)* %inb) #0 {
 entry:
   %a = load <4 x i8>, <4 x i8> addrspace(1)* %ina, align 4
   %b = load <4 x i8>, <4 x i8> addrspace(1)* %inb, align 4
@@ -355,7 +355,7 @@ entry:
 ; GFX10-DAG: v_mul_lo_u16_e64
 ; GFX10-DAG: v_mul_lo_u16_e64
 
-define amdgpu_kernel void @mul_v8i8(<8 x i8> addrspace(1)* %out, <8 x i8> addrspace(1)* %ina, <8 x i8> addrspace(1)* %inb) {
+define amdgpu_kernel void @mul_v8i8(<8 x i8> addrspace(1)* %out, <8 x i8> addrspace(1)* %ina, <8 x i8> addrspace(1)* %inb) #0 {
 entry:
   %a = load <8 x i8>, <8 x i8> addrspace(1)* %ina, align 4
   %b = load <8 x i8>, <8 x i8> addrspace(1)* %inb, align 4
@@ -376,7 +376,7 @@ entry:
 ; FIXME: Should be able to avoid or
 define amdgpu_kernel void @sitofp_v2i16_to_v2f16(
     <2 x half> addrspace(1)* %r,
-    <2 x i16> addrspace(1)* %a) {
+    <2 x i16> addrspace(1)* %a) #0 {
 entry:
   %a.val = load <2 x i16>, <2 x i16> addrspace(1)* %a
   %r.val = sitofp <2 x i16> %a.val to <2 x half>
@@ -399,7 +399,7 @@ entry:
 ; GFX9_10: v_pk_mul_f16 v[[DST_MUL:[0-9]+]], v{{[0-9]+}}, v[[SRC:[0-9]+]]
 ; GFX9_10: v_pk_add_f16 v{{[0-9]+}}, v[[DST_MUL]], v[[SRC]]
 
-define amdgpu_kernel void @mac_v2half(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %ina, <2 x half> addrspace(1)* %inb) {
+define amdgpu_kernel void @mac_v2half(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %ina, <2 x half> addrspace(1)* %inb) #0 {
 entry:
   %a = load <2 x half>, <2 x half> addrspace(1)* %ina, align 4
   %b = load <2 x half>, <2 x half> addrspace(1)* %inb, align 4
@@ -421,7 +421,7 @@ entry:
 
 ; GFX10: v_pk_mul_lo_u16 v{{[0-9]+}}, 0x141007b, v{{[0-9]+}}
 
-define amdgpu_kernel void @immediate_mul_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) {
+define amdgpu_kernel void @immediate_mul_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
 entry:
   %a = load <2 x i16>, <2 x i16> addrspace(1)* %in, align 4
   %mul = mul <2 x i16> %a, <i16 123, i16 321>
@@ -443,7 +443,7 @@ entry:
 ; GFX9_10: v_pk_mul_lo_u16 v[[DST1:[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
 ; GFX9_10: v_pk_mul_lo_u16 v{{[0-9]+}}, v[[DST1]], v{{[0-9]+}}
 
-define amdgpu_kernel void @mulmul_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %ina, <2 x i16> addrspace(1)* %inb) {
+define amdgpu_kernel void @mulmul_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %ina, <2 x i16> addrspace(1)* %inb) #0 {
 entry:
   %a = load <2 x i16>, <2 x i16> addrspace(1)* %ina, align 4
   %b = load <2 x i16>, <2 x i16> addrspace(1)* %inb, align 4
@@ -460,7 +460,7 @@ entry:
 
 ; GFX9_10: v_pk_add_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 
-define amdgpu_kernel void @add_bb_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %ina, <2 x i16> addrspace(1)* %inb) {
+define amdgpu_kernel void @add_bb_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %ina, <2 x i16> addrspace(1)* %inb) #0 {
 entry:
   %a = load <2 x i16>, <2 x i16> addrspace(1)* %ina, align 4
   %b = load <2 x i16>, <2 x i16> addrspace(1)* %inb, align 4
@@ -503,7 +503,7 @@ store_label:
 ; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX10: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 
-define amdgpu_kernel void @pulled_out_test(<8 x i8> addrspace(1)* %sourceA, <8 x i8> addrspace(1)* %destValues) {
+define amdgpu_kernel void @pulled_out_test(<8 x i8> addrspace(1)* %sourceA, <8 x i8> addrspace(1)* %destValues) #0 {
 entry:
   %idxprom = ashr exact i64 15, 32
   %arrayidx = getelementptr inbounds <8 x i8>, <8 x i8> addrspace(1)* %sourceA, i64 %idxprom
@@ -564,3 +564,5 @@ bb11:                                             ; preds = %bb10, %bb2
   store volatile <2 x i32> %tmp12, <2 x i32> addrspace(1)* undef
   br label %bb1
 }
+
+attributes #0 = { "denormal-fp-math"="preserve-sign,preserve-sign" }
diff --git a/llvm/test/CodeGen/AMDGPU/shrink-vop3-carry-out.mir b/llvm/test/CodeGen/AMDGPU/shrink-vop3-carry-out.mir
index 42c42a48a6169..0eaa485903a79 100644
--- a/llvm/test/CodeGen/AMDGPU/shrink-vop3-carry-out.mir
+++ b/llvm/test/CodeGen/AMDGPU/shrink-vop3-carry-out.mir
@@ -344,7 +344,7 @@ body:             |
 ---
 # GCN-LABEL: name: shrink_addc_vop3{{$}}
 # GCN: %29:vgpr_32 = V_ADDC_U32_e32 %19, %17, implicit-def $vcc, implicit $vcc, implicit $exec
-# GCN %24 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed $vcc, implicit $exec
+# GCN: %24:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed $vcc, implicit $exec
 
 name:            shrink_addc_vop3
 alignment:       1
diff --git a/llvm/test/CodeGen/AMDGPU/smrd.ll b/llvm/test/CodeGen/AMDGPU/smrd.ll
index 834d53e6d7f7c..1305239e23929 100644
--- a/llvm/test/CodeGen/AMDGPU/smrd.ll
+++ b/llvm/test/CodeGen/AMDGPU/smrd.ll
@@ -1,8 +1,8 @@
 ; RUN: llc -march=amdgcn -mcpu=tahiti  -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefix=SI   -check-prefix=GCN -check-prefix=SICIVI -check-prefix=SICI -check-prefix=SIVIGFX9_10 %s
 ; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefix=CI   -check-prefix=GCN -check-prefix=SICIVI -check-prefix=SICI %s
 ; RUN: llc -march=amdgcn -mcpu=tonga   -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefix=VI   -check-prefix=GCN -check-prefix=SICIVI -check-prefix=VIGFX9_10 -check-prefix=SIVIGFX9_10 %s
-; RUN: llc -march=amdgcn -mcpu=gfx900  -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefix=GFX9 -check-prefix=GCN -check-prefix=VIGFX9_10 -check-prefix=SIVIGFX9_10  %s
-; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefix=GFX10 -check-prefix=GCN -check-prefix=VIGFX9_10 -check-prefix=SIVIGFX9_10  %s
+; RUN: llc -march=amdgcn -mcpu=gfx900  -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefix=GFX9 -check-prefix=GFX9_10 -check-prefix=GCN -check-prefix=VIGFX9_10 -check-prefix=SIVIGFX9_10  %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefix=GFX10 -check-prefix=GFX9_10 -check-prefix=GCN -check-prefix=VIGFX9_10 -check-prefix=SIVIGFX9_10  %s
 
 ; SMRD load with an immediate offset.
 ; GCN-LABEL: {{^}}smrd0:
@@ -63,7 +63,9 @@ entry:
 ; SI: s_mov_b32 [[OFFSET:s[0-9]+]], 0xffffc
 ; SI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], [[OFFSET]]
 ; CI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x3ffff
-; VIGFX9_10: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0xffffc
+; VI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0xffffc
+; GFX9_10: s_mov_b32 [[OFFSET:s[0-9]+]], 0xffffc
+; GFX9_10: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], [[OFFSET]]
 define amdgpu_kernel void @smrd4(i32 addrspace(1)* %out, i32 addrspace(4)* %ptr) #0 {
 entry:
   %tmp = getelementptr i32, i32 addrspace(4)* %ptr, i64 262143
@@ -86,6 +88,32 @@ entry:
   ret void
 }
 
+; GFX9_10 can use a signed immediate byte offset
+; GCN-LABEL: {{^}}smrd6:
+; SICIVI: s_add_u32 s{{[0-9]}}, s{{[0-9]}}, -4
+; SICIVI: s_load_dword s{{[0-9]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; GFX9_10: s_load_dword s{{[0-9]}}, s{{\[[0-9]+:[0-9]+\]}}, 0xfffffffc
+define amdgpu_kernel void @smrd6(i32 addrspace(1)* %out, i32 addrspace(4)* %ptr) #0 {
+entry:
+  %tmp = getelementptr i32, i32 addrspace(4)* %ptr, i64 -1
+  %tmp1 = load i32, i32 addrspace(4)* %tmp
+  store i32 %tmp1, i32 addrspace(1)* %out
+  ret void
+}
+
+; Don't use a negative SGPR offset
+; GCN-LABEL: {{^}}smrd7:
+; GCN: s_add_u32 s{{[0-9]}}, s{{[0-9]}}, 0xffe00000
+; SICIVI: s_load_dword s{{[0-9]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; GFX9_10: s_load_dword s{{[0-9]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+define amdgpu_kernel void @smrd7(i32 addrspace(1)* %out, i32 addrspace(4)* %ptr) #0 {
+entry:
+  %tmp = getelementptr i32, i32 addrspace(4)* %ptr, i64 -524288
+  %tmp1 = load i32, i32 addrspace(4)* %tmp
+  store i32 %tmp1, i32 addrspace(1)* %out
+  ret void
+}
+
 ; GCN-LABEL: {{^}}smrd_hazard:
 ; GCN-DAG: s_mov_b32 s3, 3
 ; GCN-DAG: s_mov_b32 s2, 2
diff --git a/llvm/test/CodeGen/AMDGPU/split-vector-memoperand-offsets.ll b/llvm/test/CodeGen/AMDGPU/split-vector-memoperand-offsets.ll
index c4db4c5734a5f..6cecc23a8810c 100644
--- a/llvm/test/CodeGen/AMDGPU/split-vector-memoperand-offsets.ll
+++ b/llvm/test/CodeGen/AMDGPU/split-vector-memoperand-offsets.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -mcpu=hawaii -enable-amdgpu-aa=0 -verify-machineinstrs -mattr=-promote-alloca,-load-store-opt < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mcpu=hawaii -enable-amdgpu-aa=0 -verify-machineinstrs -mattr=-promote-alloca,-load-store-opt,-enable-ds128 < %s | FileCheck -check-prefix=GCN %s
 
 @sPrivateStorage = internal addrspace(3) global [256 x [8 x <4 x i64>]] undef
 
diff --git a/llvm/test/CodeGen/AMDGPU/stack-realign-kernel.ll b/llvm/test/CodeGen/AMDGPU/stack-realign-kernel.ll
index 93b6eacf34292..2cd61720867a9 100644
--- a/llvm/test/CodeGen/AMDGPU/stack-realign-kernel.ll
+++ b/llvm/test/CodeGen/AMDGPU/stack-realign-kernel.ll
@@ -37,7 +37,7 @@ define amdgpu_kernel void @max_alignment_128() #0 {
 ; VI-NEXT:     .amdhsa_reserve_vcc 0
 ; VI-NEXT:     .amdhsa_float_round_mode_32 0
 ; VI-NEXT:     .amdhsa_float_round_mode_16_64 0
-; VI-NEXT:     .amdhsa_float_denorm_mode_32 0
+; VI-NEXT:     .amdhsa_float_denorm_mode_32 3
 ; VI-NEXT:     .amdhsa_float_denorm_mode_16_64 3
 ; VI-NEXT:     .amdhsa_dx10_clamp 1
 ; VI-NEXT:     .amdhsa_ieee_mode 1
@@ -83,7 +83,7 @@ define amdgpu_kernel void @max_alignment_128() #0 {
 ; GFX9-NEXT:     .amdhsa_reserve_vcc 0
 ; GFX9-NEXT:     .amdhsa_float_round_mode_32 0
 ; GFX9-NEXT:     .amdhsa_float_round_mode_16_64 0
-; GFX9-NEXT:     .amdhsa_float_denorm_mode_32 0
+; GFX9-NEXT:     .amdhsa_float_denorm_mode_32 3
 ; GFX9-NEXT:     .amdhsa_float_denorm_mode_16_64 3
 ; GFX9-NEXT:     .amdhsa_dx10_clamp 1
 ; GFX9-NEXT:     .amdhsa_ieee_mode 1
@@ -136,7 +136,7 @@ define amdgpu_kernel void @stackrealign_attr() #1 {
 ; VI-NEXT:     .amdhsa_reserve_vcc 0
 ; VI-NEXT:     .amdhsa_float_round_mode_32 0
 ; VI-NEXT:     .amdhsa_float_round_mode_16_64 0
-; VI-NEXT:     .amdhsa_float_denorm_mode_32 0
+; VI-NEXT:     .amdhsa_float_denorm_mode_32 3
 ; VI-NEXT:     .amdhsa_float_denorm_mode_16_64 3
 ; VI-NEXT:     .amdhsa_dx10_clamp 1
 ; VI-NEXT:     .amdhsa_ieee_mode 1
@@ -182,7 +182,7 @@ define amdgpu_kernel void @stackrealign_attr() #1 {
 ; GFX9-NEXT:     .amdhsa_reserve_vcc 0
 ; GFX9-NEXT:     .amdhsa_float_round_mode_32 0
 ; GFX9-NEXT:     .amdhsa_float_round_mode_16_64 0
-; GFX9-NEXT:     .amdhsa_float_denorm_mode_32 0
+; GFX9-NEXT:     .amdhsa_float_denorm_mode_32 3
 ; GFX9-NEXT:     .amdhsa_float_denorm_mode_16_64 3
 ; GFX9-NEXT:     .amdhsa_dx10_clamp 1
 ; GFX9-NEXT:     .amdhsa_ieee_mode 1
@@ -235,7 +235,7 @@ define amdgpu_kernel void @alignstack_attr() #2 {
 ; VI-NEXT:     .amdhsa_reserve_vcc 0
 ; VI-NEXT:     .amdhsa_float_round_mode_32 0
 ; VI-NEXT:     .amdhsa_float_round_mode_16_64 0
-; VI-NEXT:     .amdhsa_float_denorm_mode_32 0
+; VI-NEXT:     .amdhsa_float_denorm_mode_32 3
 ; VI-NEXT:     .amdhsa_float_denorm_mode_16_64 3
 ; VI-NEXT:     .amdhsa_dx10_clamp 1
 ; VI-NEXT:     .amdhsa_ieee_mode 1
@@ -281,7 +281,7 @@ define amdgpu_kernel void @alignstack_attr() #2 {
 ; GFX9-NEXT:     .amdhsa_reserve_vcc 0
 ; GFX9-NEXT:     .amdhsa_float_round_mode_32 0
 ; GFX9-NEXT:     .amdhsa_float_round_mode_16_64 0
-; GFX9-NEXT:     .amdhsa_float_denorm_mode_32 0
+; GFX9-NEXT:     .amdhsa_float_denorm_mode_32 3
 ; GFX9-NEXT:     .amdhsa_float_denorm_mode_16_64 3
 ; GFX9-NEXT:     .amdhsa_dx10_clamp 1
 ; GFX9-NEXT:     .amdhsa_ieee_mode 1
diff --git a/llvm/test/CodeGen/AMDGPU/store-local.ll b/llvm/test/CodeGen/AMDGPU/store-local.ll
index f0e76ac59510a..f302ea099b754 100644
--- a/llvm/test/CodeGen/AMDGPU/store-local.ll
+++ b/llvm/test/CodeGen/AMDGPU/store-local.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,VI,FUNC %s
 ; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
 ; RUN: llc -march=r600 -mtriple=r600-- -mcpu=redwood < %s | FileCheck -check-prefixes=EG,FUNC %s
 ; RUN: llc -march=r600 -mtriple=r600-- -mcpu=cayman < %s | FileCheck -check-prefixes=CM,FUNC %s
@@ -156,7 +156,9 @@ entry:
 ; CM: LDS_WRITE
 ; CM: LDS_WRITE
 
-; GCN: ds_write2_b64
+; SI: ds_write2_b32
+; VI: ds_write_b128
+; GFX9: ds_write_b128
 define amdgpu_kernel void @store_local_v4i32(<4 x i32> addrspace(3)* %out, <4 x i32> %in) {
 entry:
   store <4 x i32> %in, <4 x i32> addrspace(3)* %out
diff --git a/llvm/test/CodeGen/AMDGPU/store-v3i64.ll b/llvm/test/CodeGen/AMDGPU/store-v3i64.ll
index 7af1736b3207d..b2e69deb73cfc 100644
--- a/llvm/test/CodeGen/AMDGPU/store-v3i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/store-v3i64.ll
@@ -46,8 +46,14 @@ define amdgpu_kernel void @global_store_v3i64_unaligned(<3 x i64> addrspace(1)*
 }
 
 ; GCN-LABEL: {{^}}local_store_v3i64:
-; GCN: ds_write2_b64
-; GCN: ds_write_b64
+; SI: ds_write2_b64
+; SI: ds_write_b64
+
+; CI: ds_write_b64
+; CI: ds_write_b128
+
+; VI: ds_write_b64
+; VI: ds_write_b128
 define amdgpu_kernel void @local_store_v3i64(<3 x i64> addrspace(3)* %out, <3 x i64> %x) {
   store <3 x i64> %x, <3 x i64> addrspace(3)* %out, align 32
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/trunc-cmp-constant.ll b/llvm/test/CodeGen/AMDGPU/trunc-cmp-constant.ll
index e1644e4a6d241..9b9a807db2302 100644
--- a/llvm/test/CodeGen/AMDGPU/trunc-cmp-constant.ll
+++ b/llvm/test/CodeGen/AMDGPU/trunc-cmp-constant.ll
@@ -3,7 +3,7 @@
 
 declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
 
-; FUNC-LABEL {{^}}sextload_i1_to_i32_trunc_cmp_eq_0:
+; FUNC-LABEL: {{^}}sextload_i1_to_i32_trunc_cmp_eq_0:
 ; SI: buffer_load_ubyte [[LOAD:v[0-9]+]]
 ; SI: v_and_b32_e32 [[TMP:v[0-9]+]], 1, [[LOAD]]
 ; SI: v_cmp_eq_u32_e32 vcc, 0, [[TMP]]{{$}}
@@ -80,9 +80,9 @@ define amdgpu_kernel void @zextload_i1_to_i32_trunc_cmp_eq_neg1(i1 addrspace(1)*
 }
 
 
-; FUNC-LABEL {{^}}sextload_i1_to_i32_trunc_cmp_ne_0:
+; FUNC-LABEL: {{^}}sextload_i1_to_i32_trunc_cmp_ne_0:
 ; SI: buffer_load_ubyte [[LOAD:v[0-9]+]]
-; SI: v_and_b32_e32 [[TMP:v[0-9]+]], 1, [[LOAD]]
+; SI: v_and_b32_e32 [[RESULT:v[0-9]+]], 1, [[LOAD]]
 ; SI: buffer_store_byte [[RESULT]]
 define amdgpu_kernel void @sextload_i1_to_i32_trunc_cmp_ne_0(i1 addrspace(1)* %out, i1 addrspace(1)* %in) nounwind {
   %load = load i1, i1 addrspace(1)* %in
diff --git a/llvm/test/CodeGen/AMDGPU/udiv.ll b/llvm/test/CodeGen/AMDGPU/udiv.ll
index e9a14bbda1ac2..d6c9791e7d926 100644
--- a/llvm/test/CodeGen/AMDGPU/udiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/udiv.ll
@@ -1,7 +1,7 @@
 ; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs -mattr=-fp32-denormals < %s | FileCheck -check-prefix=SI -check-prefix=FUNC -check-prefix=VI %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefix=SI -check-prefix=FUNC -check-prefix=VI %s
 
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=amdgcn--amdhsa -mcpu=fiji -mattr=+fp32-denormals < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=amdgcn--amdhsa -mcpu=fiji -denormal-fp-math-f32=ieee < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
 
 ; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 
diff --git a/llvm/test/CodeGen/AMDGPU/udivrem24.ll b/llvm/test/CodeGen/AMDGPU/udivrem24.ll
index 6e37578ef252e..ec15cc33be42a 100644
--- a/llvm/test/CodeGen/AMDGPU/udivrem24.ll
+++ b/llvm/test/CodeGen/AMDGPU/udivrem24.ll
@@ -21,6 +21,63 @@ define amdgpu_kernel void @udiv24_i8(i8 addrspace(1)* %out, i8 addrspace(1)* %in
   ret void
 }
 
+; FUNC-LABEL: {{^}}udiv24_i8_denorm_flush_in_out:
+; SI: v_cvt_f32_ubyte
+; SI-DAG: v_cvt_f32_ubyte
+; SI-DAG: v_rcp_iflag_f32
+; SI: v_cvt_u32_f32
+
+; EG: UINT_TO_FLT
+; EG-DAG: UINT_TO_FLT
+; EG-DAG: RECIP_IEEE
+; EG: FLT_TO_UINT
+define amdgpu_kernel void @udiv24_i8_denorm_flush_in_out(i8 addrspace(1)* %out, i8 addrspace(1)* %in) #0 {
+  %den_ptr = getelementptr i8, i8 addrspace(1)* %in, i8 1
+  %num = load i8, i8 addrspace(1) * %in
+  %den = load i8, i8 addrspace(1) * %den_ptr
+  %result = udiv i8 %num, %den
+  store i8 %result, i8 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}udiv24_i8_denorm_flush_in:
+; SI: v_cvt_f32_ubyte
+; SI-DAG: v_cvt_f32_ubyte
+; SI-DAG: v_rcp_iflag_f32
+; SI: v_cvt_u32_f32
+
+; EG: UINT_TO_FLT
+; EG-DAG: UINT_TO_FLT
+; EG-DAG: RECIP_IEEE
+; EG: FLT_TO_UINT
+define amdgpu_kernel void @udiv24_i8_denorm_flush_in(i8 addrspace(1)* %out, i8 addrspace(1)* %in) #1 {
+  %den_ptr = getelementptr i8, i8 addrspace(1)* %in, i8 1
+  %num = load i8, i8 addrspace(1) * %in
+  %den = load i8, i8 addrspace(1) * %den_ptr
+  %result = udiv i8 %num, %den
+  store i8 %result, i8 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}udiv24_i8_denorm_flush_out:
+; SI: v_cvt_f32_ubyte
+; SI-DAG: v_cvt_f32_ubyte
+; SI-DAG: v_rcp_iflag_f32
+; SI: v_cvt_u32_f32
+
+; EG: UINT_TO_FLT
+; EG-DAG: UINT_TO_FLT
+; EG-DAG: RECIP_IEEE
+; EG: FLT_TO_UINT
+define amdgpu_kernel void @udiv24_i8_denorm_flush_out(i8 addrspace(1)* %out, i8 addrspace(1)* %in) #2 {
+  %den_ptr = getelementptr i8, i8 addrspace(1)* %in, i8 1
+  %num = load i8, i8 addrspace(1) * %in
+  %den = load i8, i8 addrspace(1) * %den_ptr
+  %result = udiv i8 %num, %den
+  store i8 %result, i8 addrspace(1)* %out
+  ret void
+}
+
 ; FUNC-LABEL: {{^}}udiv24_i16:
 ; SI: v_cvt_f32_u32
 ; SI: v_cvt_f32_u32
@@ -325,3 +382,7 @@ define amdgpu_kernel void @test_udiv24_u23_u16_i32(i32 addrspace(1)* %out, i32 a
   store i32 %result, i32 addrspace(1)* %out, align 4
   ret void
 }
+
+attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
+attributes #1 = { "denormal-fp-math-f32"="ieee,preserve-sign" }
+attributes #2 = { "denormal-fp-math-f32"="preserve-sign,ieee" }
diff --git a/llvm/test/CodeGen/AMDGPU/v_mac.ll b/llvm/test/CodeGen/AMDGPU/v_mac.ll
index e86b885bd152c..03964acb26bc3 100644
--- a/llvm/test/CodeGen/AMDGPU/v_mac.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_mac.ll
@@ -1,6 +1,6 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -mattr=-fp64-fp16-denormals,-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=VI-FLUSH -check-prefix=GCN %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -mattr=+fp64-fp16-denormals,-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=VI-DENORM -check-prefix=GCN %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false -march=amdgcn -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tonga -denormal-fp-math=preserve-sign -denormal-fp-math-f32=preserve-sign -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=VI-FLUSH -check-prefix=GCN %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tonga -denormal-fp-math=ieee -denormal-fp-math-f32=preserve-sign -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=VI-DENORM -check-prefix=GCN %s
 
 ; GCN-LABEL: {{^}}mac_vvv:
 ; GCN: buffer_load_dword [[A:v[0-9]+]], off, s[{{[0-9]+:[0-9]+}}], 0{{$}}
diff --git a/llvm/test/CodeGen/AMDGPU/v_mac_f16.ll b/llvm/test/CodeGen/AMDGPU/v_mac_f16.ll
index f928a320058bf..e10d2d389c695 100644
--- a/llvm/test/CodeGen/AMDGPU/v_mac_f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_mac_f16.ll
@@ -1,5 +1,5 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tahiti -mattr=-fp64-fp16-denormals -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefix=GCN -check-prefix=SI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=fiji -mattr=-fp64-fp16-denormals,-flat-for-global -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefix=GCN -check-prefix=VI %s
 
 ; GCN-LABEL: {{^}}mac_f16:
 ; GCN: {{buffer|flat}}_load_ushort v[[A_F16:[0-9]+]]
@@ -677,6 +677,6 @@ entry:
 
 declare void @llvm.amdgcn.s.barrier() #2
 
-attributes #0 = { nounwind "no-signed-zeros-fp-math"="false" }
-attributes #1 = { nounwind "no-signed-zeros-fp-math"="true" }
+attributes #0 = { nounwind "no-signed-zeros-fp-math"="false" "denormal-fp-math"="preserve-sign,preserve-sign" }
+attributes #1 = { nounwind "no-signed-zeros-fp-math"="true" "denormal-fp-math"="preserve-sign,preserve-sign" }
 attributes #2 = { nounwind convergent }
diff --git a/llvm/test/CodeGen/AMDGPU/v_madak_f16.ll b/llvm/test/CodeGen/AMDGPU/v_madak_f16.ll
index 5f771bdbe6ff5..cf0d456032180 100644
--- a/llvm/test/CodeGen/AMDGPU/v_madak_f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_madak_f16.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=amdgcn-- -mcpu=tahiti -mattr=-fp64-fp16-denormals -verify-machineinstrs | FileCheck %s -check-prefixes=GCN,SI
-; RUN: llc < %s -mtriple=amdgcn-- -mcpu=fiji -mattr=-fp64-fp16-denormals,-flat-for-global -verify-machineinstrs | FileCheck %s -check-prefixes=GCN,VI
+; RUN: llc < %s -mtriple=amdgcn-- -mcpu=tahiti -verify-machineinstrs | FileCheck %s -check-prefixes=GCN,SI
+; RUN: llc < %s -mtriple=amdgcn-- -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs | FileCheck %s -check-prefixes=GCN,VI
 
 define amdgpu_kernel void @madak_f16(
 ; SI-LABEL: madak_f16:
@@ -52,7 +52,7 @@ define amdgpu_kernel void @madak_f16(
 ; VI-NEXT:    s_endpgm
     half addrspace(1)* %r,
     half addrspace(1)* %a,
-    half addrspace(1)* %b) {
+    half addrspace(1)* %b) #0 {
 entry:
   %a.val = load half, half addrspace(1)* %a
   %b.val = load half, half addrspace(1)* %b
@@ -136,7 +136,7 @@ define amdgpu_kernel void @madak_f16_use_2(
     half addrspace(1)* %r1,
     half addrspace(1)* %a,
     half addrspace(1)* %b,
-    half addrspace(1)* %c) {
+    half addrspace(1)* %c) #0 {
 entry:
   %a.val = load volatile half, half addrspace(1)* %a
   %b.val = load volatile half, half addrspace(1)* %b
@@ -151,3 +151,5 @@ entry:
   store half %r1.val, half addrspace(1)* %r1
   ret void
 }
+
+attributes #0 = { "denormal-fp-math"="preserve-sign,preserve-sign" }
diff --git a/llvm/test/CodeGen/ARM/O3-pipeline.ll b/llvm/test/CodeGen/ARM/O3-pipeline.ll
index 44fcf2a756b37..6c1a1b22f7f18 100644
--- a/llvm/test/CodeGen/ARM/O3-pipeline.ll
+++ b/llvm/test/CodeGen/ARM/O3-pipeline.ll
@@ -161,15 +161,15 @@
 ; CHECK-NEXT:      Thumb2 instruction size reduce pass
 ; CHECK-NEXT:      Unpack machine instruction bundles
 ; CHECK-NEXT:      optimise barriers pass
+; CHECK-NEXT:      Contiguously Lay Out Funclets
+; CHECK-NEXT:      StackMap Liveness Analysis
+; CHECK-NEXT:      Live DEBUG_VALUE analysis
 ; CHECK-NEXT:      MachineDominator Tree Construction
 ; CHECK-NEXT:      ARM constant island placement and branch shortening pass
 ; CHECK-NEXT:      MachineDominator Tree Construction
 ; CHECK-NEXT:      Machine Natural Loop Construction
 ; CHECK-NEXT:      ReachingDefAnalysis
 ; CHECK-NEXT:      ARM Low Overhead Loops pass
-; CHECK-NEXT:      Contiguously Lay Out Funclets
-; CHECK-NEXT:      StackMap Liveness Analysis
-; CHECK-NEXT:      Live DEBUG_VALUE analysis
 ; CHECK-NEXT:      Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:      Machine Optimization Remark Emitter
 ; CHECK-NEXT:      ARM Assembly Printer
diff --git a/llvm/test/CodeGen/ARM/acle-intrinsics-v5.ll b/llvm/test/CodeGen/ARM/acle-intrinsics-v5.ll
index 407bea1488630..f9fea403f06bc 100644
--- a/llvm/test/CodeGen/ARM/acle-intrinsics-v5.ll
+++ b/llvm/test/CodeGen/ARM/acle-intrinsics-v5.ll
@@ -80,7 +80,7 @@ define i32 @qsub(i32 %a, i32 %b) nounwind {
 
 define i32 @qdadd(i32 %a, i32 %b) nounwind {
 ; CHECK-LABEL: qdadd
-; CHECK: qdadd r0, r0, r1
+; CHECK: qdadd r0, r1, r0
   %dbl = call i32 @llvm.arm.qadd(i32 %a, i32 %a)
   %add = call i32 @llvm.arm.qadd(i32 %dbl, i32 %b)
   ret i32 %add
diff --git a/llvm/test/CodeGen/ARM/big-endian-neon-fp16-bitconv.ll b/llvm/test/CodeGen/ARM/big-endian-neon-fp16-bitconv.ll
index 86a69be5b4df0..9942d6df99a4b 100644
--- a/llvm/test/CodeGen/ARM/big-endian-neon-fp16-bitconv.ll
+++ b/llvm/test/CodeGen/ARM/big-endian-neon-fp16-bitconv.ll
@@ -98,7 +98,7 @@ entry:
 define void @conv_v4i16_to_v4f16( <4 x i16> %a, <4 x half>* %store ) {
 ; CHECK-LABEL: conv_v4i16_to_v4f16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.i64 d16, #0xffffffff0000
+; CHECK-NEXT:    vmov.i64 d16, #0xffff00000000ffff
 ; CHECK-NEXT:    vldr d17, [r0]
 ; CHECK-NEXT:    vrev64.16 d18, d0
 ; CHECK-NEXT:    vrev64.16 d17, d17
diff --git a/llvm/test/CodeGen/ARM/big-endian-vmov.ll b/llvm/test/CodeGen/ARM/big-endian-vmov.ll
new file mode 100644
index 0000000000000..1b2d4db9b9090
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/big-endian-vmov.ll
@@ -0,0 +1,88 @@
+; RUN: llc < %s -mtriple armv7-eabi -o - | FileCheck %s --check-prefixes=CHECK,CHECK-LE
+; RUN: llc < %s -mtriple armebv7-eabi -o - | FileCheck %s --check-prefixes=CHECK,CHECK-BE
+
+; CHECK-LABEL: vmov_i8
+; CHECK-LE: vmov.i64 d0, #0xff00000000000000{{$}}
+; CHECK-BE: vmov.i64 d0, #0xff{{$}}
+; CHECK-NEXT: bx lr
+define arm_aapcs_vfpcc <8 x i8> @vmov_i8() {
+  ret <8 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 -1>
+}
+
+; CHECK-LABEL: vmov_i16_a:
+; CHECK-LE: vmov.i64 d0, #0xffff000000000000{{$}}
+; CHECK-BE: vmov.i64 d0, #0xffff{{$}}
+; CHECK-NEXT: bx lr
+define arm_aapcs_vfpcc <4 x i16> @vmov_i16_a() {
+  ret <4 x i16> <i16 0, i16 0, i16 0, i16 -1>
+}
+
+; CHECK-LABEL: vmov_i16_b:
+; CHECK-LE: vmov.i64 d0, #0xff000000000000{{$}}
+; CHECK-BE: vmov.i64 d0, #0xff{{$}}
+; CHECK-NEXT: bx lr
+define arm_aapcs_vfpcc <4 x i16> @vmov_i16_b() {
+  ret <4 x i16> <i16 0, i16 0, i16 0, i16 255>
+}
+
+; CHECK-LABEL: vmov_i16_c:
+; CHECK-LE: vmov.i64 d0, #0xff00000000000000{{$}}
+; CHECK-BE: vmov.i64 d0, #0xff00{{$}}
+; CHECK-NEXT: bx lr
+define arm_aapcs_vfpcc <4 x i16> @vmov_i16_c() {
+  ret <4 x i16> <i16 0, i16 0, i16 0, i16 65280>
+}
+
+; CHECK-LABEL: vmov_i32_a:
+; CHECK-LE: vmov.i64 d0, #0xffffffff00000000{{$}}
+; CHECK-BE: vmov.i64 d0, #0xffffffff{{$}}
+; CHECK-NEXT: bx lr
+define arm_aapcs_vfpcc <2 x i32> @vmov_i32_a() {
+  ret <2 x i32> <i32 0, i32 -1>
+}
+
+; CHECK-LABEL: vmov_i32_b:
+; CHECK-LE: vmov.i64 d0, #0xff00000000{{$}}
+; CHECK-BE: vmov.i64 d0, #0xff{{$}}
+; CHECK-NEXT: bx lr
+define arm_aapcs_vfpcc <2 x i32> @vmov_i32_b() {
+  ret <2 x i32> <i32 0, i32 255>
+}
+
+; CHECK-LABEL: vmov_i32_c:
+; CHECK-LE: vmov.i64 d0, #0xff0000000000{{$}}
+; CHECK-BE: vmov.i64 d0, #0xff00{{$}}
+; CHECK-NEXT: bx lr
+define arm_aapcs_vfpcc <2 x i32> @vmov_i32_c() {
+  ret <2 x i32> <i32 0, i32 65280>
+}
+
+; CHECK-LABEL: vmov_i32_d:
+; CHECK-LE: vmov.i64 d0, #0xff000000000000{{$}}
+; CHECK-BE: vmov.i64 d0, #0xff0000{{$}}
+; CHECK-NEXT: bx lr
+define arm_aapcs_vfpcc <2 x i32> @vmov_i32_d() {
+  ret <2 x i32> <i32 0, i32 16711680>
+}
+
+; CHECK-LABEL: vmov_i32_e:
+; CHECK-LE: vmov.i64 d0, #0xff00000000000000{{$}}
+; CHECK-BE: vmov.i64 d0, #0xff000000{{$}}
+; CHECK-NEXT: bx lr
+define arm_aapcs_vfpcc <2 x i32> @vmov_i32_e() {
+  ret <2 x i32> <i32 0, i32 4278190080>
+}
+
+; CHECK-LABEL: vmov_i64_a:
+; CHECK: vmov.i8 d0, #0xff{{$}}
+; CHECK-NEXT: bx lr
+define arm_aapcs_vfpcc <1 x i64> @vmov_i64_a() {
+  ret <1 x i64> <i64 -1>
+}
+
+; CHECK-LABEL: vmov_i64_b:
+; CHECK: vmov.i64 d0, #0xffff00ff0000ff{{$}}
+; CHECK-NEXT: bx lr
+define arm_aapcs_vfpcc <1 x i64> @vmov_i64_b() {
+  ret <1 x i64> <i64 72056498804490495>
+}
diff --git a/llvm/test/CodeGen/ARM/freeze-soften.ll b/llvm/test/CodeGen/ARM/freeze-soften.ll
new file mode 100644
index 0000000000000..4148a750df98e
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/freeze-soften.ll
@@ -0,0 +1,16 @@
+; RUN: llc -mtriple=thumbv8m.main-none-eabi %s -o - | FileCheck %s
+
+; Check that freeze operations on floating types are successfully softened.
+
+; CHECK-LABEL: sitofp_f32_i32:
+; CHECK: bl __aeabi_i2f
+define float @sitofp_f32_i32(i32 %x) #0 {
+  %val = call float @llvm.experimental.constrained.sitofp.f32.i32(i32 %x, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  %val.fr = freeze float %val
+  ret float %val.fr
+}
+
+attributes #0 = { strictfp }
+
+declare float @llvm.experimental.constrained.sitofp.f32.i32(i32, metadata, metadata)
+declare double @llvm.experimental.constrained.sitofp.f64.i32(i32, metadata, metadata)
diff --git a/llvm/test/CodeGen/ARM/qdadd.ll b/llvm/test/CodeGen/ARM/qdadd.ll
index 94442ca93afad..b160f818a0fb5 100644
--- a/llvm/test/CodeGen/ARM/qdadd.ll
+++ b/llvm/test/CodeGen/ARM/qdadd.ll
@@ -36,12 +36,12 @@ define i32 @qdadd(i32 %x, i32 %y) nounwind {
 ;
 ; CHECK-T2DSP-LABEL: qdadd:
 ; CHECK-T2DSP:       @ %bb.0:
-; CHECK-T2DSP-NEXT:    qdadd r0, r0, r1
+; CHECK-T2DSP-NEXT:    qdadd r0, r1, r0
 ; CHECK-T2DSP-NEXT:    bx lr
 ;
 ; CHECK-ARM-LABEL: qdadd:
 ; CHECK-ARM:       @ %bb.0:
-; CHECK-ARM-NEXT:    qdadd r0, r0, r1
+; CHECK-ARM-NEXT:    qdadd r0, r1, r0
 ; CHECK-ARM-NEXT:    bx lr
   %z = call i32 @llvm.sadd.sat.i32(i32 %x, i32 %x)
   %tmp = call i32 @llvm.sadd.sat.i32(i32 %z, i32 %y)
@@ -80,12 +80,12 @@ define i32 @qdadd_c(i32 %x, i32 %y) nounwind {
 ;
 ; CHECK-T2DSP-LABEL: qdadd_c:
 ; CHECK-T2DSP:       @ %bb.0:
-; CHECK-T2DSP-NEXT:    qdadd r0, r0, r1
+; CHECK-T2DSP-NEXT:    qdadd r0, r1, r0
 ; CHECK-T2DSP-NEXT:    bx lr
 ;
 ; CHECK-ARM-LABEL: qdadd_c:
 ; CHECK-ARM:       @ %bb.0:
-; CHECK-ARM-NEXT:    qdadd r0, r0, r1
+; CHECK-ARM-NEXT:    qdadd r0, r1, r0
 ; CHECK-ARM-NEXT:    bx lr
   %z = call i32 @llvm.sadd.sat.i32(i32 %x, i32 %x)
   %tmp = call i32 @llvm.sadd.sat.i32(i32 %y, i32 %z)
diff --git a/llvm/test/CodeGen/ARM/vmov.ll b/llvm/test/CodeGen/ARM/vmov.ll
index b1b4f1a940d4b..995e015b4b819 100644
--- a/llvm/test/CodeGen/ARM/vmov.ll
+++ b/llvm/test/CodeGen/ARM/vmov.ll
@@ -1,332 +1,570 @@
-; RUN: llc -mtriple=arm-eabi -mattr=+neon %s -o - | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=arm-eabi -mattr=+neon,+fullfp16 %s -o - | FileCheck --check-prefixes=CHECK,CHECK-LE %s
+; RUN: llc -mtriple=armeb-eabi -mattr=+neon,+fullfp16 %s -o - | FileCheck --check-prefixes=CHECK,CHECK-BE %s
 
-define <8 x i8> @v_movi8() nounwind {
-;CHECK-LABEL: v_movi8:
-;CHECK: vmov.i8 d{{.*}}, #0x8
+define arm_aapcs_vfpcc <8 x i8> @v_movi8() nounwind {
+; CHECK-LABEL: v_movi8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i8 d0, #0x8
+; CHECK-NEXT:    mov pc, lr
 	ret <8 x i8> < i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8 >
 }
 
-define <4 x i16> @v_movi16a() nounwind {
-;CHECK-LABEL: v_movi16a:
-;CHECK: vmov.i16 d{{.*}}, #0x10
+define arm_aapcs_vfpcc <4 x i16> @v_movi16a() nounwind {
+; CHECK-LABEL: v_movi16a:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i16 d0, #0x10
+; CHECK-NEXT:    mov pc, lr
 	ret <4 x i16> < i16 16, i16 16, i16 16, i16 16 >
 }
 
-define <4 x i16> @v_movi16b() nounwind {
-;CHECK-LABEL: v_movi16b:
-;CHECK: vmov.i16 d{{.*}}, #0x1000
+define arm_aapcs_vfpcc <4 x i16> @v_movi16b() nounwind {
+; CHECK-LABEL: v_movi16b:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i16 d0, #0x1000
+; CHECK-NEXT:    mov pc, lr
 	ret <4 x i16> < i16 4096, i16 4096, i16 4096, i16 4096 >
 }
 
-define <4 x i16> @v_mvni16a() nounwind {
-;CHECK-LABEL: v_mvni16a:
-;CHECK: vmvn.i16 d{{.*}}, #0x10
+define arm_aapcs_vfpcc <4 x i16> @v_mvni16a() nounwind {
+; CHECK-LABEL: v_mvni16a:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmvn.i16 d0, #0x10
+; CHECK-NEXT:    mov pc, lr
 	ret <4 x i16> < i16 65519, i16 65519, i16 65519, i16 65519 >
 }
 
-define <4 x i16> @v_mvni16b() nounwind {
-;CHECK-LABEL: v_mvni16b:
-;CHECK: vmvn.i16 d{{.*}}, #0x1000
+define arm_aapcs_vfpcc <4 x i16> @v_mvni16b() nounwind {
+; CHECK-LABEL: v_mvni16b:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmvn.i16 d0, #0x1000
+; CHECK-NEXT:    mov pc, lr
 	ret <4 x i16> < i16 61439, i16 61439, i16 61439, i16 61439 >
 }
 
-define <2 x i32> @v_movi32a() nounwind {
-;CHECK-LABEL: v_movi32a:
-;CHECK: vmov.i32 d{{.*}}, #0x20
+define arm_aapcs_vfpcc <2 x i32> @v_movi32a() nounwind {
+; CHECK-LABEL: v_movi32a:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i32 d0, #0x20
+; CHECK-NEXT:    mov pc, lr
 	ret <2 x i32> < i32 32, i32 32 >
 }
 
-define <2 x i32> @v_movi32b() nounwind {
-;CHECK-LABEL: v_movi32b:
-;CHECK: vmov.i32 d{{.*}}, #0x2000
+define arm_aapcs_vfpcc <2 x i32> @v_movi32b() nounwind {
+; CHECK-LABEL: v_movi32b:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i32 d0, #0x2000
+; CHECK-NEXT:    mov pc, lr
 	ret <2 x i32> < i32 8192, i32 8192 >
 }
 
-define <2 x i32> @v_movi32c() nounwind {
-;CHECK-LABEL: v_movi32c:
-;CHECK: vmov.i32 d{{.*}}, #0x200000
+define arm_aapcs_vfpcc <2 x i32> @v_movi32c() nounwind {
+; CHECK-LABEL: v_movi32c:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i32 d0, #0x200000
+; CHECK-NEXT:    mov pc, lr
 	ret <2 x i32> < i32 2097152, i32 2097152 >
 }
 
-define <2 x i32> @v_movi32d() nounwind {
-;CHECK-LABEL: v_movi32d:
-;CHECK: vmov.i32 d{{.*}}, #0x20000000
+define arm_aapcs_vfpcc <2 x i32> @v_movi32d() nounwind {
+; CHECK-LABEL: v_movi32d:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i32 d0, #0x20000000
+; CHECK-NEXT:    mov pc, lr
 	ret <2 x i32> < i32 536870912, i32 536870912 >
 }
 
-define <2 x i32> @v_movi32e() nounwind {
-;CHECK-LABEL: v_movi32e:
-;CHECK: vmov.i32 d{{.*}}, #0x20ff
+define arm_aapcs_vfpcc <2 x i32> @v_movi32e() nounwind {
+; CHECK-LABEL: v_movi32e:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i32 d0, #0x20ff
+; CHECK-NEXT:    mov pc, lr
 	ret <2 x i32> < i32 8447, i32 8447 >
 }
 
-define <2 x i32> @v_movi32f() nounwind {
-;CHECK-LABEL: v_movi32f:
-;CHECK: vmov.i32 d{{.*}}, #0x20ffff
+define arm_aapcs_vfpcc <2 x i32> @v_movi32f() nounwind {
+; CHECK-LABEL: v_movi32f:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i32 d0, #0x20ffff
+; CHECK-NEXT:    mov pc, lr
 	ret <2 x i32> < i32 2162687, i32 2162687 >
 }
 
-define <2 x i32> @v_mvni32a() nounwind {
-;CHECK-LABEL: v_mvni32a:
-;CHECK: vmvn.i32 d{{.*}}, #0x20
+define arm_aapcs_vfpcc <2 x i32> @v_mvni32a() nounwind {
+; CHECK-LABEL: v_mvni32a:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmvn.i32 d0, #0x20
+; CHECK-NEXT:    mov pc, lr
 	ret <2 x i32> < i32 4294967263, i32 4294967263 >
 }
 
-define <2 x i32> @v_mvni32b() nounwind {
-;CHECK-LABEL: v_mvni32b:
-;CHECK: vmvn.i32 d{{.*}}, #0x2000
+define arm_aapcs_vfpcc <2 x i32> @v_mvni32b() nounwind {
+; CHECK-LABEL: v_mvni32b:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmvn.i32 d0, #0x2000
+; CHECK-NEXT:    mov pc, lr
 	ret <2 x i32> < i32 4294959103, i32 4294959103 >
 }
 
-define <2 x i32> @v_mvni32c() nounwind {
-;CHECK-LABEL: v_mvni32c:
-;CHECK: vmvn.i32 d{{.*}}, #0x200000
+define arm_aapcs_vfpcc <2 x i32> @v_mvni32c() nounwind {
+; CHECK-LABEL: v_mvni32c:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmvn.i32 d0, #0x200000
+; CHECK-NEXT:    mov pc, lr
 	ret <2 x i32> < i32 4292870143, i32 4292870143 >
 }
 
-define <2 x i32> @v_mvni32d() nounwind {
-;CHECK-LABEL: v_mvni32d:
-;CHECK: vmvn.i32 d{{.*}}, #0x20000000
+define arm_aapcs_vfpcc <2 x i32> @v_mvni32d() nounwind {
+; CHECK-LABEL: v_mvni32d:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmvn.i32 d0, #0x20000000
+; CHECK-NEXT:    mov pc, lr
 	ret <2 x i32> < i32 3758096383, i32 3758096383 >
 }
 
-define <2 x i32> @v_mvni32e() nounwind {
-;CHECK-LABEL: v_mvni32e:
-;CHECK: vmvn.i32 d{{.*}}, #0x20ff
+define arm_aapcs_vfpcc <2 x i32> @v_mvni32e() nounwind {
+; CHECK-LABEL: v_mvni32e:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmvn.i32 d0, #0x20ff
+; CHECK-NEXT:    mov pc, lr
 	ret <2 x i32> < i32 4294958848, i32 4294958848 >
 }
 
-define <2 x i32> @v_mvni32f() nounwind {
-;CHECK-LABEL: v_mvni32f:
-;CHECK: vmvn.i32 d{{.*}}, #0x20ffff
+define arm_aapcs_vfpcc <2 x i32> @v_mvni32f() nounwind {
+; CHECK-LABEL: v_mvni32f:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmvn.i32 d0, #0x20ffff
+; CHECK-NEXT:    mov pc, lr
 	ret <2 x i32> < i32 4292804608, i32 4292804608 >
 }
 
-define <1 x i64> @v_movi64() nounwind {
-;CHECK-LABEL: v_movi64:
-;CHECK: vmov.i64 d{{.*}}, #0xff0000ff0000ffff
+define arm_aapcs_vfpcc <1 x i64> @v_movi64() nounwind {
+; CHECK-LABEL: v_movi64:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i64 d0, #0xff0000ff0000ffff
+; CHECK-NEXT:    mov pc, lr
 	ret <1 x i64> < i64 18374687574888349695 >
 }
 
-define <16 x i8> @v_movQi8() nounwind {
-;CHECK-LABEL: v_movQi8:
-;CHECK: vmov.i8 q{{.*}}, #0x8
+define arm_aapcs_vfpcc <16 x i8> @v_movQi8() nounwind {
+; CHECK-LABEL: v_movQi8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i8 q0, #0x8
+; CHECK-NEXT:    mov pc, lr
 	ret <16 x i8> < i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8 >
 }
 
-define <8 x i16> @v_movQi16a() nounwind {
-;CHECK-LABEL: v_movQi16a:
-;CHECK: vmov.i16 q{{.*}}, #0x10
+define arm_aapcs_vfpcc <8 x i16> @v_movQi16a() nounwind {
+; CHECK-LABEL: v_movQi16a:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i16 q0, #0x10
+; CHECK-NEXT:    mov pc, lr
 	ret <8 x i16> < i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16 >
 }
 
-define <8 x i16> @v_movQi16b() nounwind {
-;CHECK-LABEL: v_movQi16b:
-;CHECK: vmov.i16 q{{.*}}, #0x1000
+define arm_aapcs_vfpcc <8 x i16> @v_movQi16b() nounwind {
+; CHECK-LABEL: v_movQi16b:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i16 q0, #0x1000
+; CHECK-NEXT:    mov pc, lr
 	ret <8 x i16> < i16 4096, i16 4096, i16 4096, i16 4096, i16 4096, i16 4096, i16 4096, i16 4096 >
 }
 
-define <4 x i32> @v_movQi32a() nounwind {
-;CHECK-LABEL: v_movQi32a:
-;CHECK: vmov.i32 q{{.*}}, #0x20
+define arm_aapcs_vfpcc <4 x i32> @v_movQi32a() nounwind {
+; CHECK-LABEL: v_movQi32a:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i32 q0, #0x20
+; CHECK-NEXT:    mov pc, lr
 	ret <4 x i32> < i32 32, i32 32, i32 32, i32 32 >
 }
 
-define <4 x i32> @v_movQi32b() nounwind {
-;CHECK-LABEL: v_movQi32b:
-;CHECK: vmov.i32 q{{.*}}, #0x2000
+define arm_aapcs_vfpcc <4 x i32> @v_movQi32b() nounwind {
+; CHECK-LABEL: v_movQi32b:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i32 q0, #0x2000
+; CHECK-NEXT:    mov pc, lr
 	ret <4 x i32> < i32 8192, i32 8192, i32 8192, i32 8192 >
 }
 
-define <4 x i32> @v_movQi32c() nounwind {
-;CHECK-LABEL: v_movQi32c:
-;CHECK: vmov.i32 q{{.*}}, #0x200000
+define arm_aapcs_vfpcc <4 x i32> @v_movQi32c() nounwind {
+; CHECK-LABEL: v_movQi32c:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i32 q0, #0x200000
+; CHECK-NEXT:    mov pc, lr
 	ret <4 x i32> < i32 2097152, i32 2097152, i32 2097152, i32 2097152 >
 }
 
-define <4 x i32> @v_movQi32d() nounwind {
-;CHECK-LABEL: v_movQi32d:
-;CHECK: vmov.i32 q{{.*}}, #0x20000000
+define arm_aapcs_vfpcc <4 x i32> @v_movQi32d() nounwind {
+; CHECK-LABEL: v_movQi32d:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i32 q0, #0x20000000
+; CHECK-NEXT:    mov pc, lr
 	ret <4 x i32> < i32 536870912, i32 536870912, i32 536870912, i32 536870912 >
 }
 
-define <4 x i32> @v_movQi32e() nounwind {
-;CHECK-LABEL: v_movQi32e:
-;CHECK: vmov.i32 q{{.*}}, #0x20ff
+define arm_aapcs_vfpcc <4 x i32> @v_movQi32e() nounwind {
+; CHECK-LABEL: v_movQi32e:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i32 q0, #0x20ff
+; CHECK-NEXT:    mov pc, lr
 	ret <4 x i32> < i32 8447, i32 8447, i32 8447, i32 8447 >
 }
 
-define <4 x i32> @v_movQi32f() nounwind {
-;CHECK-LABEL: v_movQi32f:
-;CHECK: vmov.i32 q{{.*}}, #0x20ffff
+define arm_aapcs_vfpcc <4 x i32> @v_movQi32f() nounwind {
+; CHECK-LABEL: v_movQi32f:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i32 q0, #0x20ffff
+; CHECK-NEXT:    mov pc, lr
 	ret <4 x i32> < i32 2162687, i32 2162687, i32 2162687, i32 2162687 >
 }
 
-define <2 x i64> @v_movQi64() nounwind {
-;CHECK-LABEL: v_movQi64:
-;CHECK: vmov.i64 q{{.*}}, #0xff0000ff0000ffff
+define arm_aapcs_vfpcc <2 x i64> @v_movQi64() nounwind {
+; CHECK-LABEL: v_movQi64:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i64 q0, #0xff0000ff0000ffff
+; CHECK-NEXT:    mov pc, lr
 	ret <2 x i64> < i64 18374687574888349695, i64 18374687574888349695 >
 }
 
 ; Check for correct assembler printing for immediate values.
 %struct.int8x8_t = type { <8 x i8> }
-define void @vdupn128(%struct.int8x8_t* noalias nocapture sret %agg.result) nounwind {
+define arm_aapcs_vfpcc void @vdupn128(%struct.int8x8_t* noalias nocapture sret %agg.result) nounwind {
+; CHECK-LABEL: vdupn128:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i8 d16, #0x80
+; CHECK-NEXT:    vstr d16, [r0]
+; CHECK-NEXT:    mov pc, lr
 entry:
-;CHECK-LABEL: vdupn128:
-;CHECK: vmov.i8 d{{.*}}, #0x80
   %0 = getelementptr inbounds %struct.int8x8_t, %struct.int8x8_t* %agg.result, i32 0, i32 0 ; <<8 x i8>*> [#uses=1]
   store <8 x i8> <i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>, <8 x i8>* %0, align 8
   ret void
 }
 
-define void @vdupnneg75(%struct.int8x8_t* noalias nocapture sret %agg.result) nounwind {
+define arm_aapcs_vfpcc void @vdupnneg75(%struct.int8x8_t* noalias nocapture sret %agg.result) nounwind {
+; CHECK-LABEL: vdupnneg75:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.i8 d16, #0xb5
+; CHECK-NEXT:    vstr d16, [r0]
+; CHECK-NEXT:    mov pc, lr
 entry:
-;CHECK-LABEL: vdupnneg75:
-;CHECK: vmov.i8 d{{.*}}, #0xb5
   %0 = getelementptr inbounds %struct.int8x8_t, %struct.int8x8_t* %agg.result, i32 0, i32 0 ; <<8 x i8>*> [#uses=1]
   store <8 x i8> <i8 -75, i8 -75, i8 -75, i8 -75, i8 -75, i8 -75, i8 -75, i8 -75>, <8 x i8>* %0, align 8
   ret void
 }
 
-define <8 x i16> @vmovls8(<8 x i8>* %A) nounwind {
-;CHECK-LABEL: vmovls8:
-;CHECK: vmovl.s8
+define arm_aapcs_vfpcc <8 x i16> @vmovls8(<8 x i8>* %A) nounwind {
+; CHECK-LE-LABEL: vmovls8:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vld1.8 {d16}, [r0:64]
+; CHECK-LE-NEXT:    vmovl.s8 q0, d16
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-BE-LABEL: vmovls8:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vld1.8 {d16}, [r0:64]
+; CHECK-BE-NEXT:    vmovl.s8 q8, d16
+; CHECK-BE-NEXT:    vrev64.16 q0, q8
+; CHECK-BE-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i8>, <8 x i8>* %A
 	%tmp2 = sext <8 x i8> %tmp1 to <8 x i16>
 	ret <8 x i16> %tmp2
 }
 
-define <4 x i32> @vmovls16(<4 x i16>* %A) nounwind {
-;CHECK-LABEL: vmovls16:
-;CHECK: vmovl.s16
+define arm_aapcs_vfpcc <4 x i32> @vmovls16(<4 x i16>* %A) nounwind {
+; CHECK-LE-LABEL: vmovls16:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vld1.16 {d16}, [r0:64]
+; CHECK-LE-NEXT:    vmovl.s16 q0, d16
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-BE-LABEL: vmovls16:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vld1.16 {d16}, [r0:64]
+; CHECK-BE-NEXT:    vmovl.s16 q8, d16
+; CHECK-BE-NEXT:    vrev64.32 q0, q8
+; CHECK-BE-NEXT:    mov pc, lr
 	%tmp1 = load <4 x i16>, <4 x i16>* %A
 	%tmp2 = sext <4 x i16> %tmp1 to <4 x i32>
 	ret <4 x i32> %tmp2
 }
 
-define <2 x i64> @vmovls32(<2 x i32>* %A) nounwind {
-;CHECK-LABEL: vmovls32:
-;CHECK: vmovl.s32
+define arm_aapcs_vfpcc <2 x i64> @vmovls32(<2 x i32>* %A) nounwind {
+; CHECK-LABEL: vmovls32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.32 {d16}, [r0:64]
+; CHECK-NEXT:    vmovl.s32 q0, d16
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <2 x i32>, <2 x i32>* %A
 	%tmp2 = sext <2 x i32> %tmp1 to <2 x i64>
 	ret <2 x i64> %tmp2
 }
 
-define <8 x i16> @vmovlu8(<8 x i8>* %A) nounwind {
-;CHECK-LABEL: vmovlu8:
-;CHECK: vmovl.u8
+define arm_aapcs_vfpcc <8 x i16> @vmovlu8(<8 x i8>* %A) nounwind {
+; CHECK-LE-LABEL: vmovlu8:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vld1.8 {d16}, [r0:64]
+; CHECK-LE-NEXT:    vmovl.u8 q0, d16
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-BE-LABEL: vmovlu8:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vld1.8 {d16}, [r0:64]
+; CHECK-BE-NEXT:    vmovl.u8 q8, d16
+; CHECK-BE-NEXT:    vrev64.16 q0, q8
+; CHECK-BE-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i8>, <8 x i8>* %A
 	%tmp2 = zext <8 x i8> %tmp1 to <8 x i16>
 	ret <8 x i16> %tmp2
 }
 
-define <4 x i32> @vmovlu16(<4 x i16>* %A) nounwind {
-;CHECK-LABEL: vmovlu16:
-;CHECK: vmovl.u16
+define arm_aapcs_vfpcc <4 x i32> @vmovlu16(<4 x i16>* %A) nounwind {
+; CHECK-LE-LABEL: vmovlu16:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vld1.16 {d16}, [r0:64]
+; CHECK-LE-NEXT:    vmovl.u16 q0, d16
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-BE-LABEL: vmovlu16:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vld1.16 {d16}, [r0:64]
+; CHECK-BE-NEXT:    vmovl.u16 q8, d16
+; CHECK-BE-NEXT:    vrev64.32 q0, q8
+; CHECK-BE-NEXT:    mov pc, lr
 	%tmp1 = load <4 x i16>, <4 x i16>* %A
 	%tmp2 = zext <4 x i16> %tmp1 to <4 x i32>
 	ret <4 x i32> %tmp2
 }
 
-define <2 x i64> @vmovlu32(<2 x i32>* %A) nounwind {
-;CHECK-LABEL: vmovlu32:
-;CHECK: vmovl.u32
+define arm_aapcs_vfpcc <2 x i64> @vmovlu32(<2 x i32>* %A) nounwind {
+; CHECK-LABEL: vmovlu32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.32 {d16}, [r0:64]
+; CHECK-NEXT:    vmovl.u32 q0, d16
+; CHECK-NEXT:    mov pc, lr
 	%tmp1 = load <2 x i32>, <2 x i32>* %A
 	%tmp2 = zext <2 x i32> %tmp1 to <2 x i64>
 	ret <2 x i64> %tmp2
 }
 
-define <8 x i8> @vmovni16(<8 x i16>* %A) nounwind {
-;CHECK-LABEL: vmovni16:
-;CHECK: vmovn.i16
+define arm_aapcs_vfpcc <8 x i8> @vmovni16(<8 x i16>* %A) nounwind {
+; CHECK-LE-LABEL: vmovni16:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-LE-NEXT:    vmovn.i16 d0, q8
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-BE-LABEL: vmovni16:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-BE-NEXT:    vrev64.16 q8, q8
+; CHECK-BE-NEXT:    vmovn.i16 d16, q8
+; CHECK-BE-NEXT:    vrev64.8 d0, d16
+; CHECK-BE-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i16>, <8 x i16>* %A
 	%tmp2 = trunc <8 x i16> %tmp1 to <8 x i8>
 	ret <8 x i8> %tmp2
 }
 
-define <4 x i16> @vmovni32(<4 x i32>* %A) nounwind {
-;CHECK-LABEL: vmovni32:
-;CHECK: vmovn.i32
+define arm_aapcs_vfpcc <4 x i16> @vmovni32(<4 x i32>* %A) nounwind {
+; CHECK-LE-LABEL: vmovni32:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-LE-NEXT:    vmovn.i32 d0, q8
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-BE-LABEL: vmovni32:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-BE-NEXT:    vrev64.32 q8, q8
+; CHECK-BE-NEXT:    vmovn.i32 d16, q8
+; CHECK-BE-NEXT:    vrev64.16 d0, d16
+; CHECK-BE-NEXT:    mov pc, lr
 	%tmp1 = load <4 x i32>, <4 x i32>* %A
 	%tmp2 = trunc <4 x i32> %tmp1 to <4 x i16>
 	ret <4 x i16> %tmp2
 }
 
-define <2 x i32> @vmovni64(<2 x i64>* %A) nounwind {
-;CHECK-LABEL: vmovni64:
-;CHECK: vmovn.i64
+define arm_aapcs_vfpcc <2 x i32> @vmovni64(<2 x i64>* %A) nounwind {
+; CHECK-LE-LABEL: vmovni64:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-LE-NEXT:    vmovn.i64 d0, q8
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-BE-LABEL: vmovni64:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-BE-NEXT:    vmovn.i64 d16, q8
+; CHECK-BE-NEXT:    vrev64.32 d0, d16
+; CHECK-BE-NEXT:    mov pc, lr
 	%tmp1 = load <2 x i64>, <2 x i64>* %A
 	%tmp2 = trunc <2 x i64> %tmp1 to <2 x i32>
 	ret <2 x i32> %tmp2
 }
 
-define <8 x i8> @vqmovns16(<8 x i16>* %A) nounwind {
-;CHECK-LABEL: vqmovns16:
-;CHECK: vqmovn.s16
+define arm_aapcs_vfpcc <8 x i8> @vqmovns16(<8 x i16>* %A) nounwind {
+; CHECK-LE-LABEL: vqmovns16:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-LE-NEXT:    vqmovn.s16 d0, q8
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-BE-LABEL: vqmovns16:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-BE-NEXT:    vrev64.16 q8, q8
+; CHECK-BE-NEXT:    vqmovn.s16 d16, q8
+; CHECK-BE-NEXT:    vrev64.8 d0, d16
+; CHECK-BE-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i16>, <8 x i16>* %A
 	%tmp2 = call <8 x i8> @llvm.arm.neon.vqmovns.v8i8(<8 x i16> %tmp1)
 	ret <8 x i8> %tmp2
 }
 
-define <4 x i16> @vqmovns32(<4 x i32>* %A) nounwind {
-;CHECK-LABEL: vqmovns32:
-;CHECK: vqmovn.s32
+define arm_aapcs_vfpcc <4 x i16> @vqmovns32(<4 x i32>* %A) nounwind {
+; CHECK-LE-LABEL: vqmovns32:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-LE-NEXT:    vqmovn.s32 d0, q8
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-BE-LABEL: vqmovns32:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-BE-NEXT:    vrev64.32 q8, q8
+; CHECK-BE-NEXT:    vqmovn.s32 d16, q8
+; CHECK-BE-NEXT:    vrev64.16 d0, d16
+; CHECK-BE-NEXT:    mov pc, lr
 	%tmp1 = load <4 x i32>, <4 x i32>* %A
 	%tmp2 = call <4 x i16> @llvm.arm.neon.vqmovns.v4i16(<4 x i32> %tmp1)
 	ret <4 x i16> %tmp2
 }
 
-define <2 x i32> @vqmovns64(<2 x i64>* %A) nounwind {
-;CHECK-LABEL: vqmovns64:
-;CHECK: vqmovn.s64
+define arm_aapcs_vfpcc <2 x i32> @vqmovns64(<2 x i64>* %A) nounwind {
+; CHECK-LE-LABEL: vqmovns64:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-LE-NEXT:    vqmovn.s64 d0, q8
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-BE-LABEL: vqmovns64:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-BE-NEXT:    vqmovn.s64 d16, q8
+; CHECK-BE-NEXT:    vrev64.32 d0, d16
+; CHECK-BE-NEXT:    mov pc, lr
 	%tmp1 = load <2 x i64>, <2 x i64>* %A
 	%tmp2 = call <2 x i32> @llvm.arm.neon.vqmovns.v2i32(<2 x i64> %tmp1)
 	ret <2 x i32> %tmp2
 }
 
-define <8 x i8> @vqmovnu16(<8 x i16>* %A) nounwind {
-;CHECK-LABEL: vqmovnu16:
-;CHECK: vqmovn.u16
+define arm_aapcs_vfpcc <8 x i8> @vqmovnu16(<8 x i16>* %A) nounwind {
+; CHECK-LE-LABEL: vqmovnu16:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-LE-NEXT:    vqmovn.u16 d0, q8
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-BE-LABEL: vqmovnu16:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-BE-NEXT:    vrev64.16 q8, q8
+; CHECK-BE-NEXT:    vqmovn.u16 d16, q8
+; CHECK-BE-NEXT:    vrev64.8 d0, d16
+; CHECK-BE-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i16>, <8 x i16>* %A
 	%tmp2 = call <8 x i8> @llvm.arm.neon.vqmovnu.v8i8(<8 x i16> %tmp1)
 	ret <8 x i8> %tmp2
 }
 
-define <4 x i16> @vqmovnu32(<4 x i32>* %A) nounwind {
-;CHECK-LABEL: vqmovnu32:
-;CHECK: vqmovn.u32
+define arm_aapcs_vfpcc <4 x i16> @vqmovnu32(<4 x i32>* %A) nounwind {
+; CHECK-LE-LABEL: vqmovnu32:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-LE-NEXT:    vqmovn.u32 d0, q8
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-BE-LABEL: vqmovnu32:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-BE-NEXT:    vrev64.32 q8, q8
+; CHECK-BE-NEXT:    vqmovn.u32 d16, q8
+; CHECK-BE-NEXT:    vrev64.16 d0, d16
+; CHECK-BE-NEXT:    mov pc, lr
 	%tmp1 = load <4 x i32>, <4 x i32>* %A
 	%tmp2 = call <4 x i16> @llvm.arm.neon.vqmovnu.v4i16(<4 x i32> %tmp1)
 	ret <4 x i16> %tmp2
 }
 
-define <2 x i32> @vqmovnu64(<2 x i64>* %A) nounwind {
-;CHECK-LABEL: vqmovnu64:
-;CHECK: vqmovn.u64
+define arm_aapcs_vfpcc <2 x i32> @vqmovnu64(<2 x i64>* %A) nounwind {
+; CHECK-LE-LABEL: vqmovnu64:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-LE-NEXT:    vqmovn.u64 d0, q8
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-BE-LABEL: vqmovnu64:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-BE-NEXT:    vqmovn.u64 d16, q8
+; CHECK-BE-NEXT:    vrev64.32 d0, d16
+; CHECK-BE-NEXT:    mov pc, lr
 	%tmp1 = load <2 x i64>, <2 x i64>* %A
 	%tmp2 = call <2 x i32> @llvm.arm.neon.vqmovnu.v2i32(<2 x i64> %tmp1)
 	ret <2 x i32> %tmp2
 }
 
-define <8 x i8> @vqmovuns16(<8 x i16>* %A) nounwind {
-;CHECK-LABEL: vqmovuns16:
-;CHECK: vqmovun.s16
+define arm_aapcs_vfpcc <8 x i8> @vqmovuns16(<8 x i16>* %A) nounwind {
+; CHECK-LE-LABEL: vqmovuns16:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-LE-NEXT:    vqmovun.s16 d0, q8
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-BE-LABEL: vqmovuns16:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-BE-NEXT:    vrev64.16 q8, q8
+; CHECK-BE-NEXT:    vqmovun.s16 d16, q8
+; CHECK-BE-NEXT:    vrev64.8 d0, d16
+; CHECK-BE-NEXT:    mov pc, lr
 	%tmp1 = load <8 x i16>, <8 x i16>* %A
 	%tmp2 = call <8 x i8> @llvm.arm.neon.vqmovnsu.v8i8(<8 x i16> %tmp1)
 	ret <8 x i8> %tmp2
 }
 
-define <4 x i16> @vqmovuns32(<4 x i32>* %A) nounwind {
-;CHECK-LABEL: vqmovuns32:
-;CHECK: vqmovun.s32
+define arm_aapcs_vfpcc <4 x i16> @vqmovuns32(<4 x i32>* %A) nounwind {
+; CHECK-LE-LABEL: vqmovuns32:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-LE-NEXT:    vqmovun.s32 d0, q8
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-BE-LABEL: vqmovuns32:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-BE-NEXT:    vrev64.32 q8, q8
+; CHECK-BE-NEXT:    vqmovun.s32 d16, q8
+; CHECK-BE-NEXT:    vrev64.16 d0, d16
+; CHECK-BE-NEXT:    mov pc, lr
 	%tmp1 = load <4 x i32>, <4 x i32>* %A
 	%tmp2 = call <4 x i16> @llvm.arm.neon.vqmovnsu.v4i16(<4 x i32> %tmp1)
 	ret <4 x i16> %tmp2
 }
 
-define <2 x i32> @vqmovuns64(<2 x i64>* %A) nounwind {
-;CHECK-LABEL: vqmovuns64:
-;CHECK: vqmovun.s64
+define arm_aapcs_vfpcc <2 x i32> @vqmovuns64(<2 x i64>* %A) nounwind {
+; CHECK-LE-LABEL: vqmovuns64:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-LE-NEXT:    vqmovun.s64 d0, q8
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-BE-LABEL: vqmovuns64:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-BE-NEXT:    vqmovun.s64 d16, q8
+; CHECK-BE-NEXT:    vrev64.32 d0, d16
+; CHECK-BE-NEXT:    mov pc, lr
 	%tmp1 = load <2 x i64>, <2 x i64>* %A
 	%tmp2 = call <2 x i32> @llvm.arm.neon.vqmovnsu.v2i32(<2 x i64> %tmp1)
 	ret <2 x i32> %tmp2
@@ -346,8 +584,22 @@ declare <2 x i32> @llvm.arm.neon.vqmovnsu.v2i32(<2 x i64>) nounwind readnone
 
 ; Truncating vector stores are not supported.  The following should not crash.
 ; Radar 8598391.
-define void @noTruncStore(<4 x i32>* %a, <4 x i16>* %b) nounwind {
-;CHECK: vmovn
+define arm_aapcs_vfpcc void @noTruncStore(<4 x i32>* %a, <4 x i16>* %b) nounwind {
+; CHECK-LE-LABEL: noTruncStore:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    vld1.64 {d16, d17}, [r0:128]
+; CHECK-LE-NEXT:    vmovn.i32 d16, q8
+; CHECK-LE-NEXT:    vstr d16, [r1]
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-BE-LABEL: noTruncStore:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    vld1.64 {d16, d17}, [r0:128]
+; CHECK-BE-NEXT:    vrev64.32 q8, q8
+; CHECK-BE-NEXT:    vmovn.i32 d16, q8
+; CHECK-BE-NEXT:    vrev64.16 d16, d16
+; CHECK-BE-NEXT:    vstr d16, [r1]
+; CHECK-BE-NEXT:    mov pc, lr
   %tmp1 = load <4 x i32>, <4 x i32>* %a, align 16
   %tmp2 = trunc <4 x i32> %tmp1 to <4 x i16>
   store <4 x i16> %tmp2, <4 x i16>* %b, align 8
@@ -356,26 +608,53 @@ define void @noTruncStore(<4 x i32>* %a, <4 x i16>* %b) nounwind {
 
 ; Use vmov.f32 to materialize f32 immediate splats
 ; rdar://10437054
-define void @v_mov_v2f32(<2 x float>* nocapture %p) nounwind {
+define arm_aapcs_vfpcc void @v_mov_v2f32(<2 x float>* nocapture %p) nounwind {
+; CHECK-LABEL: v_mov_v2f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.f32 d16, #-1.600000e+01
+; CHECK-NEXT:    vstr d16, [r0]
+; CHECK-NEXT:    mov pc, lr
 entry:
-;CHECK-LABEL: v_mov_v2f32:
-;CHECK: vmov.f32 d{{.*}}, #-1.600000e+01
   store <2 x float> <float -1.600000e+01, float -1.600000e+01>, <2 x float>* %p, align 4
   ret void
 }
 
-define void @v_mov_v4f32(<4 x float>* nocapture %p) nounwind {
+define arm_aapcs_vfpcc void @v_mov_v4f32(<4 x float>* nocapture %p) nounwind {
+; CHECK-LE-LABEL: v_mov_v4f32:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vmov.f32 q8, #3.100000e+01
+; CHECK-LE-NEXT:    vst1.32 {d16, d17}, [r0]
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-BE-LABEL: v_mov_v4f32:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vmov.f32 q8, #3.100000e+01
+; CHECK-BE-NEXT:    vstmia r0, {d16, d17}
+; CHECK-BE-NEXT:    mov pc, lr
 entry:
-;CHECK-LABEL: v_mov_v4f32:
-;CHECK: vmov.f32 q{{.*}}, #3.100000e+01
   store <4 x float> <float 3.100000e+01, float 3.100000e+01, float 3.100000e+01, float 3.100000e+01>, <4 x float>* %p, align 4
   ret void
 }
 
-define void @v_mov_v4f32_undef(<4 x float> * nocapture %p) nounwind {
+define arm_aapcs_vfpcc void @v_mov_v4f32_undef(<4 x float> * nocapture %p) nounwind {
+; CHECK-LE-LABEL: v_mov_v4f32_undef:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vmov.f32 q8, #1.000000e+00
+; CHECK-LE-NEXT:    vld1.64 {d18, d19}, [r0]
+; CHECK-LE-NEXT:    vadd.f32 q8, q9, q8
+; CHECK-LE-NEXT:    vst1.64 {d16, d17}, [r0]
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-BE-LABEL: v_mov_v4f32_undef:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vld1.64 {d16, d17}, [r0]
+; CHECK-BE-NEXT:    vmov.f32 q9, #1.000000e+00
+; CHECK-BE-NEXT:    vrev64.32 q8, q8
+; CHECK-BE-NEXT:    vadd.f32 q8, q8, q9
+; CHECK-BE-NEXT:    vrev64.32 q8, q8
+; CHECK-BE-NEXT:    vst1.64 {d16, d17}, [r0]
+; CHECK-BE-NEXT:    mov pc, lr
 entry:
-;CHECK-LABEL: v_mov_v4f32_undef:
-;CHECK: vmov.f32 q{{.*}}, #1.000000e+00
   %a = load <4 x float> , <4 x float> *%p
   %b = fadd <4 x float> %a, <float undef, float 1.0, float 1.0, float 1.0>
   store <4 x float> %b, <4 x float> *%p
@@ -384,10 +663,28 @@ entry:
 
 ; Vector any_extends must be selected as either vmovl.u or vmovl.s.
 ; rdar://10723651
-define void @any_extend(<4 x i1> %x, <4 x i32> %y) nounwind ssp {
+define arm_aapcs_vfpcc void @any_extend(<4 x i1> %x, <4 x i32> %y) nounwind ssp {
+; CHECK-LE-LABEL: any_extend:
+; CHECK-LE:       @ %bb.0: @ %entry
+; CHECK-LE-NEXT:    vmov.i16 d16, #0x1
+; CHECK-LE-NEXT:    vand d16, d0, d16
+; CHECK-LE-NEXT:    vmovl.u16 q8, d16
+; CHECK-LE-NEXT:    vsub.i32 q8, q8, q1
+; CHECK-LE-NEXT:    vmovn.i32 d16, q8
+; CHECK-LE-NEXT:    vst1.16 {d16}, [r0]
+;
+; CHECK-BE-LABEL: any_extend:
+; CHECK-BE:       @ %bb.0: @ %entry
+; CHECK-BE-NEXT:    vmov.i16 d16, #0x1
+; CHECK-BE-NEXT:    vrev64.32 d17, d0
+; CHECK-BE-NEXT:    vrev64.32 q9, q1
+; CHECK-BE-NEXT:    vand d16, d17, d16
+; CHECK-BE-NEXT:    vrev32.16 d16, d16
+; CHECK-BE-NEXT:    vmovl.u16 q8, d16
+; CHECK-BE-NEXT:    vsub.i32 q8, q8, q9
+; CHECK-BE-NEXT:    vmovn.i32 d16, q8
+; CHECK-BE-NEXT:    vst1.16 {d16}, [r0]
 entry:
-;CHECK-LABEL: any_extend:
-;CHECK: vmovl
   %and.i186 = zext <4 x i1> %x to <4 x i32>
   %add.i185 = sub <4 x i32> %and.i186, %y
   %sub.i = sub <4 x i32> %add.i185, zeroinitializer
@@ -397,4 +694,599 @@ entry:
   unreachable
 }
 
+define arm_aapcs_vfpcc void @v_movi8_sti8(i8* %p) {
+; CHECK-LABEL: v_movi8_sti8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i8 d16, #0x1
+; CHECK-NEXT:    vst1.8 {d16}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  call void @llvm.arm.neon.vst1.p0i8.v8i8(i8* %p, <8 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>, i32 1)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movi8_sti16(i8* %p) {
+; CHECK-LABEL: v_movi8_sti16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i8 d16, #0x1
+; CHECK-NEXT:    vst1.16 {d16}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <8 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1> to <4 x i16>
+  call void @llvm.arm.neon.vst1.p0i8.v4i16(i8* %p, <4 x i16> %val, i32 2)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movi8_stf16(i8* %p) {
+; CHECK-LABEL: v_movi8_stf16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i8 d16, #0x1
+; CHECK-NEXT:    vst1.16 {d16}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <8 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1> to <4 x half>
+  call void @llvm.arm.neon.vst1.p0i8.v4f16(i8* %p, <4 x half> %val, i32 2)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movi8_sti32(i8* %p) {
+; CHECK-LABEL: v_movi8_sti32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i8 d16, #0x1
+; CHECK-NEXT:    vst1.32 {d16}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <8 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1> to <2 x i32>
+  call void @llvm.arm.neon.vst1.p0i8.v2i32(i8* %p, <2 x i32> %val, i32 4)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movi8_stf32(i8* %p) {
+; CHECK-LABEL: v_movi8_stf32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i8 d16, #0x1
+; CHECK-NEXT:    vst1.32 {d16}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <8 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1> to <2 x float>
+  call void @llvm.arm.neon.vst1.p0i8.v2f32(i8* %p, <2 x float> %val, i32 4)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movi8_sti64(i8* %p) {
+; CHECK-LABEL: v_movi8_sti64:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i8 d16, #0x1
+; CHECK-NEXT:    vst1.64 {d16}, [r0:64]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <8 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1> to <1 x i64>
+  call void @llvm.arm.neon.vst1.p0i8.v1i64(i8* %p, <1 x i64> %val, i32 8)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movi16_sti16(i8* %p) {
+; CHECK-LABEL: v_movi16_sti16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i16 d16, #0x1
+; CHECK-NEXT:    vst1.16 {d16}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  call void @llvm.arm.neon.vst1.p0i8.v4i16(i8* %p, <4 x i16> <i16 1, i16 1, i16 1, i16 1>, i32 2)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movi16_stf16(i8* %p) {
+; CHECK-LABEL: v_movi16_stf16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i16 d16, #0x1
+; CHECK-NEXT:    vst1.16 {d16}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <4 x i16> <i16 1, i16 1, i16 1, i16 1> to <4 x half>
+  call void @llvm.arm.neon.vst1.p0i8.v4f16(i8* %p, <4 x half> %val, i32 2)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movi16_sti32(i8* %p) {
+; CHECK-LABEL: v_movi16_sti32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i16 d16, #0x1
+; CHECK-NEXT:    vst1.32 {d16}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <4 x i16> <i16 1, i16 1, i16 1, i16 1> to <2 x i32>
+  call void @llvm.arm.neon.vst1.p0i8.v2i32(i8* %p, <2 x i32> %val, i32 4)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movi16_stf32(i8* %p) {
+; CHECK-LABEL: v_movi16_stf32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i16 d16, #0x1
+; CHECK-NEXT:    vst1.32 {d16}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <4 x i16> <i16 1, i16 1, i16 1, i16 1> to <2 x float>
+  call void @llvm.arm.neon.vst1.p0i8.v2f32(i8* %p, <2 x float> %val, i32 4)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movi16_sti64(i8* %p) {
+; CHECK-LABEL: v_movi16_sti64:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i16 d16, #0x1
+; CHECK-NEXT:    vst1.64 {d16}, [r0:64]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <4 x i16> <i16 1, i16 1, i16 1, i16 1> to <1 x i64>
+  call void @llvm.arm.neon.vst1.p0i8.v1i64(i8* %p, <1 x i64> %val, i32 8)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movi32_sti32(i8* %p) {
+; CHECK-LABEL: v_movi32_sti32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i32 d16, #0x1
+; CHECK-NEXT:    vst1.32 {d16}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  call void @llvm.arm.neon.vst1.p0i8.v2i32(i8* %p, <2 x i32> <i32 1, i32 1>, i32 4)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movi32_stf32(i8* %p) {
+; CHECK-LABEL: v_movi32_stf32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i32 d16, #0x1
+; CHECK-NEXT:    vst1.32 {d16}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <2 x i32> <i32 1, i32 1> to <2 x float>
+  call void @llvm.arm.neon.vst1.p0i8.v2f32(i8* %p, <2 x float> %val, i32 4)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movi32_sti64(i8* %p) {
+; CHECK-LABEL: v_movi32_sti64:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i32 d16, #0x1
+; CHECK-NEXT:    vst1.64 {d16}, [r0:64]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <2 x i32> <i32 1, i32 1> to <1 x i64>
+  call void @llvm.arm.neon.vst1.p0i8.v1i64(i8* %p, <1 x i64> %val, i32 8)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movf32_stf32(i8* %p) {
+; CHECK-LABEL: v_movf32_stf32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.f32 d16, #1.000000e+00
+; CHECK-NEXT:    vst1.32 {d16}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  call void @llvm.arm.neon.vst1.p0i8.v2f32(i8* %p, <2 x float> <float 1.0, float 1.0>, i32 4)
+  ret void
+}
+
+define arm_aapcs_vfpcc void@v_movf32_sti32(i8* %p) {
+; FIXME: We should use vmov.f32 instead of mov then vdup
+; CHECK-LABEL: v_movf32_sti32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    mov r1, #1065353216
+; CHECK-NEXT:    vdup.32 d16, r1
+; CHECK-NEXT:    vst1.32 {d16}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <2 x float> <float 1.0, float 1.0> to <2 x i32>
+  call void @llvm.arm.neon.vst1.p0i8.v2i32(i8* %p, <2 x i32> %val, i32 4)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movf32_sti64(i8* %p) {
+; CHECK-LE-LABEL: v_movf32_sti64:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    mov r1, #1065353216
+; CHECK-LE-NEXT:    vdup.32 d16, r1
+; CHECK-LE-NEXT:    vst1.64 {d16}, [r0:64]
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; FIXME: vrev is not needed here
+; CHECK-BE-LABEL: v_movf32_sti64:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    mov r1, #1065353216
+; CHECK-BE-NEXT:    vdup.32 d16, r1
+; CHECK-BE-NEXT:    vrev64.32 d16, d16
+; CHECK-BE-NEXT:    vst1.64 {d16}, [r0:64]
+; CHECK-BE-NEXT:    mov pc, lr
+  %val = bitcast <2 x float> <float 1.0, float 1.0> to <1 x i64>
+  call void @llvm.arm.neon.vst1.p0i8.v1i64(i8* %p, <1 x i64> %val, i32 8)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movi64_sti64(i8* %p) {
+; CHECK-LABEL: v_movi64_sti64:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i64 d16, #0xff
+; CHECK-NEXT:    vst1.64 {d16}, [r0:64]
+; CHECK-NEXT:    mov pc, lr
+  call void @llvm.arm.neon.vst1.p0i8.v1i64(i8* %p, <1 x i64> <i64 255>, i32 8)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movQi8_sti8(i8* %p) {
+; CHECK-LABEL: v_movQi8_sti8:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i8 q8, #0x1
+; CHECK-NEXT:    vst1.8 {d16, d17}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  call void @llvm.arm.neon.vst1.p0i8.v16i8(i8* %p, <16 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>, i32 1)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movQi8_sti16(i8* %p) {
+; CHECK-LABEL: v_movQi8_sti16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i8 q8, #0x1
+; CHECK-NEXT:    vst1.16 {d16, d17}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <16 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1> to <8 x i16>
+  call void @llvm.arm.neon.vst1.p0i8.v8i16(i8* %p, <8 x i16> %val, i32 2)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movQi8_stf16(i8* %p) {
+; CHECK-LABEL: v_movQi8_stf16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i8 q8, #0x1
+; CHECK-NEXT:    vst1.16 {d16, d17}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <16 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1> to <8 x half>
+  call void @llvm.arm.neon.vst1.p0i8.v8f16(i8* %p, <8 x half> %val, i32 2)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movQi8_sti32(i8* %p) {
+; CHECK-LABEL: v_movQi8_sti32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i8 q8, #0x1
+; CHECK-NEXT:    vst1.32 {d16, d17}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <16 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1> to <4 x i32>
+  call void @llvm.arm.neon.vst1.p0i8.v4i32(i8* %p, <4 x i32> %val, i32 4)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movQi8_stf32(i8* %p) {
+; CHECK-LABEL: v_movQi8_stf32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i8 q8, #0x1
+; CHECK-NEXT:    vst1.32 {d16, d17}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <16 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1> to <4 x float>
+  call void @llvm.arm.neon.vst1.p0i8.v4f32(i8* %p, <4 x float> %val, i32 4)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movQi8_sti64(i8* %p) {
+; CHECK-LABEL: v_movQi8_sti64:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i8 q8, #0x1
+; CHECK-NEXT:    vst1.64 {d16, d17}, [r0:64]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <16 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1> to <2 x i64>
+  call void @llvm.arm.neon.vst1.p0i8.v2i64(i8* %p, <2 x i64> %val, i32 8)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movQi16_sti16(i8* %p) {
+; CHECK-LABEL: v_movQi16_sti16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i16 q8, #0x1
+; CHECK-NEXT:    vst1.16 {d16, d17}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  call void @llvm.arm.neon.vst1.p0i8.v8i16(i8* %p, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>, i32 2)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movQi16_stf16(i8* %p) {
+; CHECK-LABEL: v_movQi16_stf16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i16 q8, #0x1
+; CHECK-NEXT:    vst1.16 {d16, d17}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1> to <8 x half>
+  call void @llvm.arm.neon.vst1.p0i8.v8f16(i8* %p, <8 x half> %val, i32 2)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movQi16_sti32(i8* %p) {
+; CHECK-LABEL: v_movQi16_sti32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i16 q8, #0x1
+; CHECK-NEXT:    vst1.32 {d16, d17}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1> to <4 x i32>
+  call void @llvm.arm.neon.vst1.p0i8.v4i32(i8* %p, <4 x i32> %val, i32 4)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movQi16_stf32(i8* %p) {
+; CHECK-LABEL: v_movQi16_stf32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i16 q8, #0x1
+; CHECK-NEXT:    vst1.32 {d16, d17}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1> to <4 x float>
+  call void @llvm.arm.neon.vst1.p0i8.v4f32(i8* %p, <4 x float> %val, i32 4)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movQi16_sti64(i8* %p) {
+; CHECK-LABEL: v_movQi16_sti64:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i16 q8, #0x1
+; CHECK-NEXT:    vst1.64 {d16, d17}, [r0:64]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1> to <2 x i64>
+  call void @llvm.arm.neon.vst1.p0i8.v2i64(i8* %p, <2 x i64> %val, i32 8)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movQi32_sti32(i8* %p) {
+; CHECK-LABEL: v_movQi32_sti32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i32 q8, #0x1
+; CHECK-NEXT:    vst1.32 {d16, d17}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  call void @llvm.arm.neon.vst1.p0i8.v4i32(i8* %p, <4 x i32> <i32 1, i32 1, i32 1, i32 1>, i32 4)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movQi32_stf32(i8* %p) {
+; CHECK-LABEL: v_movQi32_stf32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i32 q8, #0x1
+; CHECK-NEXT:    vst1.32 {d16, d17}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <4 x i32> <i32 1, i32 1, i32 1, i32 1> to <4 x float>
+  call void @llvm.arm.neon.vst1.p0i8.v4f32(i8* %p, <4 x float> %val, i32 4)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movQi32_sti64(i8* %p) {
+; CHECK-LABEL: v_movQi32_sti64:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i32 q8, #0x1
+; CHECK-NEXT:    vst1.64 {d16, d17}, [r0:64]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <4 x i32> <i32 1, i32 1, i32 1, i32 1> to <2 x i64>
+  call void @llvm.arm.neon.vst1.p0i8.v2i64(i8* %p, <2 x i64> %val, i32 8)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movQf32_stf32(i8* %p) {
+; CHECK-LABEL: v_movQf32_stf32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.f32 q8, #1.000000e+00
+; CHECK-NEXT:    vst1.32 {d16, d17}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  call void @llvm.arm.neon.vst1.p0i8.v4f32(i8* %p, <4 x float> <float 1.0, float 1.0, float 1.0, float 1.0>, i32 4)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movQf32_sti32(i8* %p) {
+; FIXME: We should use vmov.f32 instead of mov then vdup
+; CHECK-LABEL: v_movQf32_sti32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    mov r1, #1065353216
+; CHECK-NEXT:    vdup.32 q8, r1
+; CHECK-NEXT:    vst1.32 {d16, d17}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <4 x float> <float 1.0, float 1.0, float 1.0, float 1.0> to <4 x i32>
+  call void @llvm.arm.neon.vst1.p0i8.v4i32(i8* %p, <4 x i32> %val, i32 4)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movQf32_sti64(i8* %p) {
+; CHECK-LE-LABEL: v_movQf32_sti64:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    mov r1, #1065353216
+; CHECK-LE-NEXT:    vdup.32 q8, r1
+; CHECK-LE-NEXT:    vst1.64 {d16, d17}, [r0:64]
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; FIXME: vrev is not needed here
+; CHECK-BE-LABEL: v_movQf32_sti64:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    mov r1, #1065353216
+; CHECK-BE-NEXT:    vdup.32 q8, r1
+; CHECK-BE-NEXT:    vrev64.32 q8, q8
+; CHECK-BE-NEXT:    vst1.64 {d16, d17}, [r0:64]
+; CHECK-BE-NEXT:    mov pc, lr
+  %val = bitcast <4 x float> <float 1.0, float 1.0, float 1.0, float 1.0> to <2 x i64>
+  call void @llvm.arm.neon.vst1.p0i8.v2i64(i8* %p, <2 x i64> %val, i32 8)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_movQi64_sti64(i8* %p) {
+; CHECK-LABEL: v_movQi64_sti64:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i64 q8, #0xff
+; CHECK-NEXT:    vst1.64 {d16, d17}, [r0:64]
+; CHECK-NEXT:    mov pc, lr
+  call void @llvm.arm.neon.vst1.p0i8.v2i64(i8* %p, <2 x i64> <i64 255, i64 255>, i32 8)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_mvni16_sti16(i8* %p) {
+; CHECK-LABEL: v_mvni16_sti16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmvn.i16 d16, #0xfe
+; CHECK-NEXT:    vst1.16 {d16}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  call void @llvm.arm.neon.vst1.p0i8.v4i16(i8* %p, <4 x i16> <i16 65281, i16 65281, i16 65281, i16 65281>, i32 2)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_mvni16_stf16(i8* %p) {
+; CHECK-LABEL: v_mvni16_stf16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmvn.i16 d16, #0xfe
+; CHECK-NEXT:    vst1.16 {d16}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <4 x i16> <i16 65281, i16 65281, i16 65281, i16 65281> to <4 x half>
+  call void @llvm.arm.neon.vst1.p0i8.v4f16(i8* %p, <4 x half> %val, i32 2)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_mvni16_sti32(i8* %p) {
+; CHECK-LABEL: v_mvni16_sti32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmvn.i16 d16, #0xfe
+; CHECK-NEXT:    vst1.32 {d16}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <4 x i16> <i16 65281, i16 65281, i16 65281, i16 65281> to <2 x i32>
+  call void @llvm.arm.neon.vst1.p0i8.v2i32(i8* %p, <2 x i32> %val, i32 4)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_mvni16_stf32(i8* %p) {
+; CHECK-LABEL: v_mvni16_stf32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmvn.i16 d16, #0xfe
+; CHECK-NEXT:    vst1.32 {d16}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <4 x i16> <i16 65281, i16 65281, i16 65281, i16 65281> to <2 x float>
+  call void @llvm.arm.neon.vst1.p0i8.v2f32(i8* %p, <2 x float> %val, i32 4)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_mvni16_sti64(i8* %p) {
+; CHECK-LABEL: v_mvni16_sti64:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmvn.i16 d16, #0xfe
+; CHECK-NEXT:    vst1.64 {d16}, [r0:64]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <4 x i16> <i16 65281, i16 65281, i16 65281, i16 65281> to <1 x i64>
+  call void @llvm.arm.neon.vst1.p0i8.v1i64(i8* %p, <1 x i64> %val, i32 8)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_mvni32_sti32(i8* %p) {
+; CHECK-LABEL: v_mvni32_sti32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmvn.i32 d16, #0xfe
+; CHECK-NEXT:    vst1.32 {d16}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  call void @llvm.arm.neon.vst1.p0i8.v2i32(i8* %p, <2 x i32> <i32 4294967041, i32 4294967041>, i32 4)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_mvni32_stf32(i8* %p) {
+; CHECK-LABEL: v_mvni32_stf32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmvn.i32 d16, #0xfe
+; CHECK-NEXT:    vst1.32 {d16}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <2 x i32> <i32 4294967041, i32 4294967041> to <2 x float>
+  call void @llvm.arm.neon.vst1.p0i8.v2f32(i8* %p, <2 x float> %val, i32 4)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_mvni32_sti64(i8* %p) {
+; CHECK-LABEL: v_mvni32_sti64:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmvn.i32 d16, #0xfe
+; CHECK-NEXT:    vst1.64 {d16}, [r0:64]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <2 x i32> <i32 4294967041, i32 4294967041> to <1 x i64>
+  call void @llvm.arm.neon.vst1.p0i8.v1i64(i8* %p, <1 x i64> %val, i32 8)
+  ret void
+}
+
+
+define arm_aapcs_vfpcc void @v_mvnQi16_sti16(i8* %p) {
+; CHECK-LABEL: v_mvnQi16_sti16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmvn.i16 q8, #0xfe
+; CHECK-NEXT:    vst1.16 {d16, d17}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  call void @llvm.arm.neon.vst1.p0i8.v8i16(i8* %p, <8 x i16> <i16 65281, i16 65281, i16 65281, i16 65281, i16 65281, i16 65281, i16 65281, i16 65281>, i32 2)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_mvnQi16_stf16(i8* %p) {
+; CHECK-LABEL: v_mvnQi16_stf16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmvn.i16 q8, #0xfe
+; CHECK-NEXT:    vst1.16 {d16, d17}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <8 x i16> <i16 65281, i16 65281, i16 65281, i16 65281, i16 65281, i16 65281, i16 65281, i16 65281> to <8 x half>
+  call void @llvm.arm.neon.vst1.p0i8.v8f16(i8* %p, <8 x half> %val, i32 2)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_mvnQi16_sti32(i8* %p) {
+; CHECK-LABEL: v_mvnQi16_sti32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmvn.i16 q8, #0xfe
+; CHECK-NEXT:    vst1.32 {d16, d17}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <8 x i16> <i16 65281, i16 65281, i16 65281, i16 65281, i16 65281, i16 65281, i16 65281, i16 65281> to <4 x i32>
+  call void @llvm.arm.neon.vst1.p0i8.v4i32(i8* %p, <4 x i32> %val, i32 4)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_mvnQi16_stf32(i8* %p) {
+; CHECK-LABEL: v_mvnQi16_stf32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmvn.i16 q8, #0xfe
+; CHECK-NEXT:    vst1.32 {d16, d17}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <8 x i16> <i16 65281, i16 65281, i16 65281, i16 65281, i16 65281, i16 65281, i16 65281, i16 65281> to <4 x float>
+  call void @llvm.arm.neon.vst1.p0i8.v4f32(i8* %p, <4 x float> %val, i32 4)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_mvnQi16_sti64(i8* %p) {
+; CHECK-LABEL: v_mvnQi16_sti64:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmvn.i16 q8, #0xfe
+; CHECK-NEXT:    vst1.64 {d16, d17}, [r0:64]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <8 x i16> <i16 65281, i16 65281, i16 65281, i16 65281, i16 65281, i16 65281, i16 65281, i16 65281> to <2 x i64>
+  call void @llvm.arm.neon.vst1.p0i8.v2i64(i8* %p, <2 x i64> %val, i32 8)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_mvnQi32_sti32(i8* %p) {
+; CHECK-LABEL: v_mvnQi32_sti32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmvn.i32 q8, #0xfe
+; CHECK-NEXT:    vst1.32 {d16, d17}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  call void @llvm.arm.neon.vst1.p0i8.v4i32(i8* %p, <4 x i32> <i32 4294967041, i32 4294967041, i32 4294967041, i32 4294967041>, i32 4)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_mvnQi32_stf32(i8* %p) {
+; CHECK-LABEL: v_mvnQi32_stf32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmvn.i32 q8, #0xfe
+; CHECK-NEXT:    vst1.32 {d16, d17}, [r0]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <4 x i32> <i32 4294967041, i32 4294967041, i32 4294967041, i32 4294967041> to <4 x float>
+  call void @llvm.arm.neon.vst1.p0i8.v4f32(i8* %p, <4 x float> %val, i32 4)
+  ret void
+}
+
+define arm_aapcs_vfpcc void @v_mvnQi32_sti64(i8* %p) {
+; CHECK-LABEL: v_mvnQi32_sti64:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmvn.i32 q8, #0xfe
+; CHECK-NEXT:    vst1.64 {d16, d17}, [r0:64]
+; CHECK-NEXT:    mov pc, lr
+  %val = bitcast <4 x i32> <i32 4294967041, i32 4294967041, i32 4294967041, i32 4294967041> to <2 x i64>
+  call void @llvm.arm.neon.vst1.p0i8.v2i64(i8* %p, <2 x i64> %val, i32 8)
+  ret void
+}
+
+declare void @llvm.arm.neon.vst1.p0i8.v8i8(i8*, <8 x i8>, i32) nounwind
 declare void @llvm.arm.neon.vst1.p0i8.v4i16(i8*, <4 x i16>, i32) nounwind
+declare void @llvm.arm.neon.vst1.p0i8.v4f16(i8*, <4 x half>, i32) nounwind
+declare void @llvm.arm.neon.vst1.p0i8.v2i32(i8*, <2 x i32>, i32) nounwind
+declare void @llvm.arm.neon.vst1.p0i8.v2f32(i8*, <2 x float>, i32) nounwind
+declare void @llvm.arm.neon.vst1.p0i8.v1i64(i8*, <1 x i64>, i32) nounwind
+
+declare void @llvm.arm.neon.vst1.p0i8.v16i8(i8*, <16 x i8>, i32) nounwind
+declare void @llvm.arm.neon.vst1.p0i8.v8i16(i8*, <8 x i16>, i32) nounwind
+declare void @llvm.arm.neon.vst1.p0i8.v8f16(i8*, <8 x half>, i32) nounwind
+declare void @llvm.arm.neon.vst1.p0i8.v4i32(i8*, <4 x i32>, i32) nounwind
+declare void @llvm.arm.neon.vst1.p0i8.v4f32(i8*, <4 x float>, i32) nounwind
+declare void @llvm.arm.neon.vst1.p0i8.v2i64(i8*, <2 x i64>, i32) nounwind
diff --git a/llvm/test/CodeGen/AVR/interrupts.ll b/llvm/test/CodeGen/AVR/interrupts.ll
index 5dddf0ea70184..b402d867e12b2 100644
--- a/llvm/test/CodeGen/AVR/interrupts.ll
+++ b/llvm/test/CodeGen/AVR/interrupts.ll
@@ -16,6 +16,22 @@ define avr_intrcc void @interrupt_handler() {
   ret void
 }
 
+define void @interrupt_handler_via_ir_attribute() #0 {
+; CHECK-LABEL: interrupt_handler_via_ir_attribute:
+; CHECK: sei
+; CHECK-NEXT: push r0
+; CHECK-NEXT: push r1
+; CHECK-NEXT: in r0, 63
+; CHECK-NEXT: push r0
+; CHECK: clr r0
+; CHECK: pop r0
+; CHECK-NEXT: out 63, r0
+; CHECK-NEXT: pop r1
+; CHECK-NEXT: pop r0
+; CHECK-NEXT: reti
+  ret void
+}
+
 define avr_signalcc void @signal_handler() {
 ; CHECK-LABEL: signal_handler:
 ; CHECK-NOT: sei
@@ -31,3 +47,22 @@ define avr_signalcc void @signal_handler() {
 ; CHECK-NEXT: reti
   ret void
 }
+
+define void @signal_handler_via_attribute() #1 {
+; CHECK-LABEL: signal_handler_via_attribute:
+; CHECK-NOT: sei
+; CHECK: push r0
+; CHECK-NEXT: push r1
+; CHECK-NEXT: in r0, 63
+; CHECK-NEXT: push r0
+; CHECK: clr r0
+; CHECK: pop r0
+; CHECK-NEXT: out 63, r0
+; CHECK-NEXT: pop r1
+; CHECK-NEXT: pop r0
+; CHECK-NEXT: reti
+  ret void
+}
+
+attributes #0 = { "interrupt" }
+attributes #1 = { "signal" }
diff --git a/llvm/test/CodeGen/BPF/i128.ll b/llvm/test/CodeGen/BPF/i128.ll
new file mode 100644
index 0000000000000..c77838fa8bb47
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/i128.ll
@@ -0,0 +1,67 @@
+; RUN: llc -march=bpfel -o - %s | FileCheck %s
+; RUN: llc -march=bpfeb -o - %s | FileCheck %s
+; Source code:
+;   struct ipv6_key_t {
+;     unsigned pid;
+;     unsigned __int128 saddr;
+;     unsigned short lport;
+;   };
+;
+;   extern void test1(void *);
+;   int test(int pid) {
+;     struct ipv6_key_t ipv6_key = {.pid = pid};
+;     test1(&ipv6_key);
+;     return 0;
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -S -emit-llvm t.c
+
+%struct.ipv6_key_t = type { i32, i128, i16 }
+
+; Function Attrs: nounwind
+define dso_local i32 @test(i32 %pid) local_unnamed_addr #0 {
+entry:
+  %ipv6_key = alloca %struct.ipv6_key_t, align 16
+  %0 = bitcast %struct.ipv6_key_t* %ipv6_key to i8*
+  call void @llvm.lifetime.start.p0i8(i64 48, i8* nonnull %0) #4
+  call void @llvm.memset.p0i8.i64(i8* nonnull align 16 dereferenceable(48) %0, i8 0, i64 48, i1 false)
+  %pid1 = getelementptr inbounds %struct.ipv6_key_t, %struct.ipv6_key_t* %ipv6_key, i64 0, i32 0
+  store i32 %pid, i32* %pid1, align 16, !tbaa !2
+  call void @test1(i8* nonnull %0) #4
+  call void @llvm.lifetime.end.p0i8(i64 48, i8* nonnull %0) #4
+  ret i32 0
+}
+
+; CHECK-LABEL: test
+; CHECK:       *(u64 *)(r10 - 48) = r{{[0-9]+}}
+; CHECK:       *(u32 *)(r10 - 48) = r{{[0-9]+}}
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
+
+; Function Attrs: argmemonly nounwind willreturn writeonly
+declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1 immarg) #2
+
+declare dso_local void @test1(i8*) local_unnamed_addr #3
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { argmemonly nounwind willreturn }
+attributes #2 = { argmemonly nounwind willreturn writeonly }
+attributes #3 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { nounwind }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 11.0.0 (https://github.com/llvm/llvm-project.git 55fc7a47f8f18f84b44ff16f4e7a420c0a42ddf1)"}
+!2 = !{!3, !4, i64 0}
+!3 = !{!"ipv6_key_t", !4, i64 0, !7, i64 16, !8, i64 32}
+!4 = !{!"int", !5, i64 0}
+!5 = !{!"omnipotent char", !6, i64 0}
+!6 = !{!"Simple C/C++ TBAA"}
+!7 = !{!"__int128", !5, i64 0}
+!8 = !{!"short", !5, i64 0}
diff --git a/llvm/test/CodeGen/Generic/vector-redux.ll b/llvm/test/CodeGen/Generic/vector-redux.ll
deleted file mode 100644
index 8efdbf85b8c04..0000000000000
--- a/llvm/test/CodeGen/Generic/vector-redux.ll
+++ /dev/null
@@ -1,237 +0,0 @@
-; RUN: llc < %s -debug-only=isel -o /dev/null 2>&1 | FileCheck %s
-; REQUIRES: asserts
-
-@a = global [1024 x i32] zeroinitializer, align 16
-
-define i32 @reduce_add() {
-; CHECK-LABEL: reduce_add
-; CHECK:       Detected a reduction operation: {{.*}} add
-; CHECK:       Detected a reduction operation: {{.*}} add
-; CHECK:       Detected a reduction operation: {{.*}} add
-; CHECK:       Detected a reduction operation: {{.*}} add
-; CHECK:       Detected a reduction operation: {{.*}} add
-; CHECK:       Detected a reduction operation: {{.*}} add
-; CHECK:       Detected a reduction operation: {{.*}} add
-; CHECK:       Detected a reduction operation: {{.*}} add
-; CHECK:       Detected a reduction operation: {{.*}} add
-; CHECK:       Detected a reduction operation: {{.*}} add
-; CHECK:       Detected a reduction operation: {{.*}} add
-
-min.iters.checked:
-  br label %vector.body
-
-vector.body:
-  %index = phi i64 [ 0, %min.iters.checked ], [ %index.next.4, %vector.body ]
-  %vec.phi = phi <4 x i32> [ zeroinitializer, %min.iters.checked ], [ %28, %vector.body ]
-  %vec.phi4 = phi <4 x i32> [ zeroinitializer, %min.iters.checked ], [ %29, %vector.body ]
-  %0 = getelementptr inbounds [1024 x i32], [1024 x i32]* @a, i64 0, i64 %index
-  %1 = bitcast i32* %0 to <4 x i32>*
-  %wide.load = load <4 x i32>, <4 x i32>* %1, align 16
-  %2 = getelementptr i32, i32* %0, i64 4
-  %3 = bitcast i32* %2 to <4 x i32>*
-  %wide.load5 = load <4 x i32>, <4 x i32>* %3, align 16
-  %4 = add nsw <4 x i32> %wide.load, %vec.phi
-  %5 = add nsw <4 x i32> %wide.load5, %vec.phi4
-  %index.next = add nuw nsw i64 %index, 8
-  %6 = getelementptr inbounds [1024 x i32], [1024 x i32]* @a, i64 0, i64 %index.next
-  %7 = bitcast i32* %6 to <4 x i32>*
-  %wide.load.1 = load <4 x i32>, <4 x i32>* %7, align 16
-  %8 = getelementptr i32, i32* %6, i64 4
-  %9 = bitcast i32* %8 to <4 x i32>*
-  %wide.load5.1 = load <4 x i32>, <4 x i32>* %9, align 16
-  %10 = add nsw <4 x i32> %wide.load.1, %4
-  %11 = add nsw <4 x i32> %wide.load5.1, %5
-  %index.next.1 = add nsw i64 %index, 16
-  %12 = getelementptr inbounds [1024 x i32], [1024 x i32]* @a, i64 0, i64 %index.next.1
-  %13 = bitcast i32* %12 to <4 x i32>*
-  %wide.load.2 = load <4 x i32>, <4 x i32>* %13, align 16
-  %14 = getelementptr i32, i32* %12, i64 4
-  %15 = bitcast i32* %14 to <4 x i32>*
-  %wide.load5.2 = load <4 x i32>, <4 x i32>* %15, align 16
-  %16 = add nsw <4 x i32> %wide.load.2, %10
-  %17 = add nsw <4 x i32> %wide.load5.2, %11
-  %index.next.2 = add nsw i64 %index, 24
-  %18 = getelementptr inbounds [1024 x i32], [1024 x i32]* @a, i64 0, i64 %index.next.2
-  %19 = bitcast i32* %18 to <4 x i32>*
-  %wide.load.3 = load <4 x i32>, <4 x i32>* %19, align 16
-  %20 = getelementptr i32, i32* %18, i64 4
-  %21 = bitcast i32* %20 to <4 x i32>*
-  %wide.load5.3 = load <4 x i32>, <4 x i32>* %21, align 16
-  %22 = add nsw <4 x i32> %wide.load.3, %16
-  %23 = add nsw <4 x i32> %wide.load5.3, %17
-  %index.next.3 = add nsw i64 %index, 32
-  %24 = getelementptr inbounds [1024 x i32], [1024 x i32]* @a, i64 0, i64 %index.next.3
-  %25 = bitcast i32* %24 to <4 x i32>*
-  %wide.load.4 = load <4 x i32>, <4 x i32>* %25, align 16
-  %26 = getelementptr i32, i32* %24, i64 4
-  %27 = bitcast i32* %26 to <4 x i32>*
-  %wide.load5.4 = load <4 x i32>, <4 x i32>* %27, align 16
-  %28 = add nsw <4 x i32> %wide.load.4, %22
-  %29 = add nsw <4 x i32> %wide.load5.4, %23
-  %index.next.4 = add nsw i64 %index, 40
-  %30 = icmp eq i64 %index.next.4, 1000
-  br i1 %30, label %middle.block, label %vector.body
-
-middle.block:
-  %.lcssa10 = phi <4 x i32> [ %29, %vector.body ]
-  %.lcssa = phi <4 x i32> [ %28, %vector.body ]
-  %bin.rdx = add <4 x i32> %.lcssa10, %.lcssa
-  %rdx.shuf = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
-  %bin.rdx6 = add <4 x i32> %bin.rdx, %rdx.shuf
-  %rdx.shuf7 = shufflevector <4 x i32> %bin.rdx6, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-  %bin.rdx8 = add <4 x i32> %bin.rdx6, %rdx.shuf7
-  %31 = extractelement <4 x i32> %bin.rdx8, i32 0
-  ret i32 %31
-}
-
-define i32 @reduce_and() {
-; CHECK-LABEL: reduce_and
-; CHECK:       Detected a reduction operation: {{.*}} and
-; CHECK:       Detected a reduction operation: {{.*}} and
-; CHECK:       Detected a reduction operation: {{.*}} and
-; CHECK:       Detected a reduction operation: {{.*}} and
-; CHECK:       Detected a reduction operation: {{.*}} and
-; CHECK:       Detected a reduction operation: {{.*}} and
-; CHECK:       Detected a reduction operation: {{.*}} and
-; CHECK:       Detected a reduction operation: {{.*}} and
-; CHECK:       Detected a reduction operation: {{.*}} and
-
-entry:
-  br label %vector.body
-
-vector.body:
-  %lsr.iv = phi i64 [ %lsr.iv.next, %vector.body ], [ -4096, %entry ]
-  %vec.phi = phi <4 x i32> [ <i32 -1, i32 -1, i32 -1, i32 -1>, %entry ], [ %6, %vector.body ]
-  %vec.phi9 = phi <4 x i32> [ <i32 -1, i32 -1, i32 -1, i32 -1>, %entry ], [ %7, %vector.body ]
-  %uglygep33 = getelementptr i8, i8* bitcast ([1024 x i32]* @a to i8*), i64 %lsr.iv
-  %uglygep3334 = bitcast i8* %uglygep33 to <4 x i32>*
-  %scevgep35 = getelementptr <4 x i32>, <4 x i32>* %uglygep3334, i64 256
-  %wide.load = load <4 x i32>, <4 x i32>* %scevgep35, align 16
-  %scevgep36 = getelementptr <4 x i32>, <4 x i32>* %uglygep3334, i64 257
-  %wide.load10 = load <4 x i32>, <4 x i32>* %scevgep36, align 16
-  %0 = and <4 x i32> %wide.load, %vec.phi
-  %1 = and <4 x i32> %wide.load10, %vec.phi9
-  %uglygep30 = getelementptr i8, i8* bitcast ([1024 x i32]* @a to i8*), i64 %lsr.iv
-  %uglygep3031 = bitcast i8* %uglygep30 to <4 x i32>*
-  %scevgep32 = getelementptr <4 x i32>, <4 x i32>* %uglygep3031, i64 258
-  %wide.load.1 = load <4 x i32>, <4 x i32>* %scevgep32, align 16
-  %uglygep27 = getelementptr i8, i8* bitcast ([1024 x i32]* @a to i8*), i64 %lsr.iv
-  %uglygep2728 = bitcast i8* %uglygep27 to <4 x i32>*
-  %scevgep29 = getelementptr <4 x i32>, <4 x i32>* %uglygep2728, i64 259
-  %wide.load10.1 = load <4 x i32>, <4 x i32>* %scevgep29, align 16
-  %2 = and <4 x i32> %wide.load.1, %0
-  %3 = and <4 x i32> %wide.load10.1, %1
-  %uglygep24 = getelementptr i8, i8* bitcast ([1024 x i32]* @a to i8*), i64 %lsr.iv
-  %uglygep2425 = bitcast i8* %uglygep24 to <4 x i32>*
-  %scevgep26 = getelementptr <4 x i32>, <4 x i32>* %uglygep2425, i64 260
-  %wide.load.2 = load <4 x i32>, <4 x i32>* %scevgep26, align 16
-  %uglygep21 = getelementptr i8, i8* bitcast ([1024 x i32]* @a to i8*), i64 %lsr.iv
-  %uglygep2122 = bitcast i8* %uglygep21 to <4 x i32>*
-  %scevgep23 = getelementptr <4 x i32>, <4 x i32>* %uglygep2122, i64 261
-  %wide.load10.2 = load <4 x i32>, <4 x i32>* %scevgep23, align 16
-  %4 = and <4 x i32> %wide.load.2, %2
-  %5 = and <4 x i32> %wide.load10.2, %3
-  %uglygep18 = getelementptr i8, i8* bitcast ([1024 x i32]* @a to i8*), i64 %lsr.iv
-  %uglygep1819 = bitcast i8* %uglygep18 to <4 x i32>*
-  %scevgep20 = getelementptr <4 x i32>, <4 x i32>* %uglygep1819, i64 262
-  %wide.load.3 = load <4 x i32>, <4 x i32>* %scevgep20, align 16
-  %uglygep = getelementptr i8, i8* bitcast ([1024 x i32]* @a to i8*), i64 %lsr.iv
-  %uglygep17 = bitcast i8* %uglygep to <4 x i32>*
-  %scevgep = getelementptr <4 x i32>, <4 x i32>* %uglygep17, i64 263
-  %wide.load10.3 = load <4 x i32>, <4 x i32>* %scevgep, align 16
-  %6 = and <4 x i32> %wide.load.3, %4
-  %7 = and <4 x i32> %wide.load10.3, %5
-  %lsr.iv.next = add nsw i64 %lsr.iv, 128
-  %8 = icmp eq i64 %lsr.iv.next, 0
-  br i1 %8, label %middle.block, label %vector.body
-
-middle.block:
-  %bin.rdx = and <4 x i32> %7, %6
-  %rdx.shuf = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
-  %bin.rdx11 = and <4 x i32> %bin.rdx, %rdx.shuf
-  %rdx.shuf12 = shufflevector <4 x i32> %bin.rdx11, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-  %bin.rdx13 = and <4 x i32> %bin.rdx11, %rdx.shuf12
-  %9 = extractelement <4 x i32> %bin.rdx13, i32 0
-  ret i32 %9
-}
-
-define float @reduce_add_float(float* nocapture readonly %a) {
-; CHECK-LABEL: reduce_add_float
-; CHECK:       Detected a reduction operation: {{.*}} fadd fast
-; CHECK:       Detected a reduction operation: {{.*}} fadd fast
-; CHECK:       Detected a reduction operation: {{.*}} fadd fast
-; CHECK:       Detected a reduction operation: {{.*}} fadd fast
-; CHECK:       Detected a reduction operation: {{.*}} fadd fast
-; CHECK:       Detected a reduction operation: {{.*}} fadd fast
-; CHECK:       Detected a reduction operation: {{.*}} fadd fast
-; CHECK:       Detected a reduction operation: {{.*}} fadd fast
-; CHECK:       Detected a reduction operation: {{.*}} fadd fast
-; CHECK:       Detected a reduction operation: {{.*}} fadd fast
-; CHECK:       Detected a reduction operation: {{.*}} fadd fast
-;
-entry:
-  br label %vector.body
-
-vector.body:
-  %index = phi i64 [ 0, %entry ], [ %index.next.4, %vector.body ]
-  %vec.phi = phi <4 x float> [ zeroinitializer, %entry ], [ %28, %vector.body ]
-  %vec.phi9 = phi <4 x float> [ zeroinitializer, %entry ], [ %29, %vector.body ]
-  %0 = getelementptr inbounds float, float* %a, i64 %index
-  %1 = bitcast float* %0 to <4 x float>*
-  %wide.load = load <4 x float>, <4 x float>* %1, align 4
-  %2 = getelementptr float, float* %0, i64 4
-  %3 = bitcast float* %2 to <4 x float>*
-  %wide.load10 = load <4 x float>, <4 x float>* %3, align 4
-  %4 = fadd fast <4 x float> %wide.load, %vec.phi
-  %5 = fadd fast <4 x float> %wide.load10, %vec.phi9
-  %index.next = add nuw nsw i64 %index, 8
-  %6 = getelementptr inbounds float, float* %a, i64 %index.next
-  %7 = bitcast float* %6 to <4 x float>*
-  %wide.load.1 = load <4 x float>, <4 x float>* %7, align 4
-  %8 = getelementptr float, float* %6, i64 4
-  %9 = bitcast float* %8 to <4 x float>*
-  %wide.load10.1 = load <4 x float>, <4 x float>* %9, align 4
-  %10 = fadd fast <4 x float> %wide.load.1, %4
-  %11 = fadd fast <4 x float> %wide.load10.1, %5
-  %index.next.1 = add nsw i64 %index, 16
-  %12 = getelementptr inbounds float, float* %a, i64 %index.next.1
-  %13 = bitcast float* %12 to <4 x float>*
-  %wide.load.2 = load <4 x float>, <4 x float>* %13, align 4
-  %14 = getelementptr float, float* %12, i64 4
-  %15 = bitcast float* %14 to <4 x float>*
-  %wide.load10.2 = load <4 x float>, <4 x float>* %15, align 4
-  %16 = fadd fast <4 x float> %wide.load.2, %10
-  %17 = fadd fast <4 x float> %wide.load10.2, %11
-  %index.next.2 = add nsw i64 %index, 24
-  %18 = getelementptr inbounds float, float* %a, i64 %index.next.2
-  %19 = bitcast float* %18 to <4 x float>*
-  %wide.load.3 = load <4 x float>, <4 x float>* %19, align 4
-  %20 = getelementptr float, float* %18, i64 4
-  %21 = bitcast float* %20 to <4 x float>*
-  %wide.load10.3 = load <4 x float>, <4 x float>* %21, align 4
-  %22 = fadd fast <4 x float> %wide.load.3, %16
-  %23 = fadd fast <4 x float> %wide.load10.3, %17
-  %index.next.3 = add nsw i64 %index, 32
-  %24 = getelementptr inbounds float, float* %a, i64 %index.next.3
-  %25 = bitcast float* %24 to <4 x float>*
-  %wide.load.4 = load <4 x float>, <4 x float>* %25, align 4
-  %26 = getelementptr float, float* %24, i64 4
-  %27 = bitcast float* %26 to <4 x float>*
-  %wide.load10.4 = load <4 x float>, <4 x float>* %27, align 4
-  %28 = fadd fast <4 x float> %wide.load.4, %22
-  %29 = fadd fast <4 x float> %wide.load10.4, %23
-  %index.next.4 = add nsw i64 %index, 40
-  %30 = icmp eq i64 %index.next.4, 1000
-  br i1 %30, label %middle.block, label %vector.body
-
-middle.block:
-  %.lcssa15 = phi <4 x float> [ %29, %vector.body ]
-  %.lcssa = phi <4 x float> [ %28, %vector.body ]
-  %bin.rdx = fadd fast <4 x float> %.lcssa15, %.lcssa
-  %rdx.shuf = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
-  %bin.rdx11 = fadd fast <4 x float> %bin.rdx, %rdx.shuf
-  %rdx.shuf12 = shufflevector <4 x float> %bin.rdx11, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-  %bin.rdx13 = fadd fast <4 x float> %bin.rdx11, %rdx.shuf12
-  %31 = extractelement <4 x float> %bin.rdx13, i32 0
-  ret float %31
-}
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info.ll b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info.ll
index 975a4ea19af4f..94f926e451a9e 100644
--- a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info.ll
+++ b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info.ll
@@ -27,8 +27,8 @@
 ; CHECK-NEXT: mode:
 ; CHECK-NEXT: ieee: true
 ; CHECK-NEXT: dx10-clamp: true
-; CHECK-NEXT: fp32-input-denormals: false
-; CHECK-NEXT: fp32-output-denormals: false
+; CHECK-NEXT: fp32-input-denormals: true
+; CHECK-NEXT: fp32-output-denormals: true
 ; CHECK-NEXT: fp64-fp16-input-denormals: true
 ; CHECK-NEXT: fp64-fp16-output-denormals: true
 ; CHECK-NEXT: highBitsOf32BitAddress: 0
@@ -57,8 +57,8 @@ define amdgpu_kernel void @kernel(i32 %arg0, i64 %arg1, <16 x i32> %arg2) {
 ; CHECK-NEXT: mode:
 ; CHECK-NEXT: ieee: false
 ; CHECK-NEXT: dx10-clamp: true
-; CHECK-NEXT: fp32-input-denormals: false
-; CHECK-NEXT: fp32-output-denormals: false
+; CHECK-NEXT: fp32-input-denormals: true
+; CHECK-NEXT: fp32-output-denormals: true
 ; CHECK-NEXT: fp64-fp16-input-denormals: true
 ; CHECK-NEXT: fp64-fp16-output-denormals: true
 ; CHECK-NEXT: highBitsOf32BitAddress: 0
@@ -84,8 +84,8 @@ define amdgpu_ps void @ps_shader(i32 %arg0, i32 inreg %arg1) {
 ; CHECK-NEXT: mode:
 ; CHECK-NEXT: ieee: true
 ; CHECK-NEXT: dx10-clamp: true
-; CHECK-NEXT: fp32-input-denormals: false
-; CHECK-NEXT: fp32-output-denormals: false
+; CHECK-NEXT: fp32-input-denormals: true
+; CHECK-NEXT: fp32-output-denormals: true
 ; CHECK-NEXT: fp64-fp16-input-denormals: true
 ; CHECK-NEXT: fp64-fp16-output-denormals: true
 ; CHECK-NEXT: highBitsOf32BitAddress: 0
@@ -111,8 +111,8 @@ define void @function() {
 ; CHECK-NEXT: mode:
 ; CHECK-NEXT: ieee: true
 ; CHECK-NEXT: dx10-clamp: true
-; CHECK-NEXT: fp32-input-denormals: false
-; CHECK-NEXT: fp32-output-denormals: false
+; CHECK-NEXT: fp32-input-denormals: true
+; CHECK-NEXT: fp32-output-denormals: true
 ; CHECK-NEXT: fp64-fp16-input-denormals: true
 ; CHECK-NEXT: fp64-fp16-output-denormals: true
 ; CHECK-NEXT: highBitsOf32BitAddress: 0
@@ -125,8 +125,8 @@ define void @function_nsz() #0 {
 ; CHECK: mode:
 ; CHECK-NEXT: ieee: true
 ; CHECK-NEXT: dx10-clamp: false
-; CHECK-NEXT: fp32-input-denormals: false
-; CHECK-NEXT: fp32-output-denormals: false
+; CHECK-NEXT: fp32-input-denormals: true
+; CHECK-NEXT: fp32-output-denormals: true
 ; CHECK-NEXT: fp64-fp16-input-denormals: true
 ; CHECK-NEXT: fp64-fp16-output-denormals: true
 define void @function_dx10_clamp_off() #1 {
@@ -137,8 +137,8 @@ define void @function_dx10_clamp_off() #1 {
 ; CHECK: mode:
 ; CHECK-NEXT: ieee: false
 ; CHECK-NEXT: dx10-clamp: true
-; CHECK-NEXT: fp32-input-denormals: false
-; CHECK-NEXT: fp32-output-denormals: false
+; CHECK-NEXT: fp32-input-denormals: true
+; CHECK-NEXT: fp32-output-denormals: true
 ; CHECK-NEXT: fp64-fp16-input-denormals: true
 ; CHECK-NEXT: fp64-fp16-output-denormals: true
 define void @function_ieee_off() #2 {
@@ -149,8 +149,8 @@ define void @function_ieee_off() #2 {
 ; CHECK: mode:
 ; CHECK-NEXT: ieee: false
 ; CHECK-NEXT: dx10-clamp: false
-; CHECK-NEXT: fp32-input-denormals: false
-; CHECK-NEXT: fp32-output-denormals: false
+; CHECK-NEXT: fp32-input-denormals: true
+; CHECK-NEXT: fp32-output-denormals: true
 ; CHECK-NEXT: fp64-fp16-input-denormals: true
 ; CHECK-NEXT: fp64-fp16-output-denormals: true
 define void @function_ieee_off_dx10_clamp_off() #3 {
diff --git a/llvm/test/CodeGen/Mips/longbranch/long-branch-octeon.ll b/llvm/test/CodeGen/Mips/longbranch/long-branch-octeon.ll
new file mode 100644
index 0000000000000..0c8c3a35768ea
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/longbranch/long-branch-octeon.ll
@@ -0,0 +1,105 @@
+;; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+;; Test that Octeon BBIT family of branch can be replaced by
+;; the long branch expansion pass.
+
+; RUN: llc -O3 -mtriple=mips64-octeon-linux -mcpu=octeon -force-mips-long-branch < %s -o - | FileCheck %s
+
+define i64 @bbit1(i64 %a) nounwind {
+; CHECK-LABEL: bbit1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    bbit0 $4, 3, .LBB0_2
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    j .LBB0_3
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  .LBB0_2: # %endif
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    daddiu $2, $zero, 12
+; CHECK-NEXT:  .LBB0_3: # %if
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    daddiu $2, $zero, 48
+entry:
+  %bit = and i64 %a, 8
+  %res = icmp eq i64 %bit, 0
+  br i1 %res, label %endif, label %if
+if:
+  ret i64 48
+
+endif:
+  ret i64 12
+}
+
+define i64 @bbit132(i64 %a) nounwind {
+; CHECK-LABEL: bbit132:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    bbit032 $4, 3, .LBB1_2
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    j .LBB1_3
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  .LBB1_2: # %endif
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    daddiu $2, $zero, 12
+; CHECK-NEXT:  .LBB1_3: # %if
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    daddiu $2, $zero, 48
+entry:
+  %bit = and i64 %a, 34359738368
+  %res = icmp eq i64 %bit, 0
+  br i1 %res, label %endif, label %if
+if:
+  ret i64 48
+
+endif:
+  ret i64 12
+}
+
+define i64 @bbit0(i64 %a) nounwind {
+; CHECK-LABEL: bbit0:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    bbit1 $4, 3, .LBB2_2
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    j .LBB2_3
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  .LBB2_2: # %endif
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    daddiu $2, $zero, 12
+; CHECK-NEXT:  .LBB2_3: # %if
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    daddiu $2, $zero, 48
+entry:
+  %bit = and i64 %a, 8
+  %res = icmp ne i64 %bit, 0
+  br i1 %res, label %endif, label %if
+if:
+  ret i64 48
+
+endif:
+  ret i64 12
+}
+
+define i64 @bbit032(i64 %a) nounwind {
+; CHECK-LABEL: bbit032:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    bbit132 $4, 3, .LBB3_2
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    j .LBB3_3
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  .LBB3_2: # %endif
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    daddiu $2, $zero, 12
+; CHECK-NEXT:  .LBB3_3: # %if
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    daddiu $2, $zero, 48
+entry:
+  %bit = and i64 %a, 34359738368
+  %res = icmp ne i64 %bit, 0
+  br i1 %res, label %endif, label %if
+if:
+  ret i64 48
+
+endif:
+  ret i64 12
+}
diff --git a/llvm/test/CodeGen/NVPTX/f16-instructions.ll b/llvm/test/CodeGen/NVPTX/f16-instructions.ll
index 9aa81dac1262a..3856cb63350df 100644
--- a/llvm/test/CodeGen/NVPTX/f16-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/f16-instructions.ll
@@ -1,1140 +1,1140 @@
-; ## Full FP16 support enabled by default.
-; RUN: llc < %s -mtriple=nvptx64-nvidia-cuda -mcpu=sm_53 -asm-verbose=false \
-; RUN:          -O0 -disable-post-ra -frame-pointer=all -verify-machineinstrs \
-; RUN: | FileCheck -check-prefixes CHECK,CHECK-NOFTZ,CHECK-F16,CHECK-F16-NOFTZ %s
-; ## Full FP16 with FTZ
-; RUN: llc < %s -mtriple=nvptx64-nvidia-cuda -mcpu=sm_53 -asm-verbose=false \
-; RUN:          -O0 -disable-post-ra -frame-pointer=all -verify-machineinstrs \
-; RUN:          -nvptx-f32ftz \
-; RUN: | FileCheck -check-prefixes CHECK,CHECK-F16,CHECK-F16-FTZ %s
-; ## FP16 support explicitly disabled.
-; RUN: llc < %s -mtriple=nvptx64-nvidia-cuda -mcpu=sm_53 -asm-verbose=false \
-; RUN:          -O0 -disable-post-ra -frame-pointer=all --nvptx-no-f16-math \
-; RUN:           -verify-machineinstrs \
-; RUN: | FileCheck -check-prefixes CHECK,CHECK-NOFTZ,CHECK-NOF16 %s
-; ## FP16 is not supported by hardware.
-; RUN: llc < %s -O0 -mtriple=nvptx64-nvidia-cuda -mcpu=sm_52 -asm-verbose=false \
-; RUN:          -disable-post-ra -frame-pointer=all -verify-machineinstrs \
-; RUN: | FileCheck -check-prefixes CHECK,CHECK-NOFTZ,CHECK-NOF16 %s
-
-target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
-
-; CHECK-LABEL: test_ret_const(
-; CHECK:      mov.b16         [[R:%h[0-9]+]], 0x3C00;
-; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define half @test_ret_const() #0 {
-  ret half 1.0
-}
-
-; CHECK-LABEL: test_fadd(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fadd_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fadd_param_1];
-; CHECK-F16-NOFTZ-NEXT:   add.rn.f16     [[R:%h[0-9]+]], [[A]], [[B]];
-; CHECK-F16-FTZ-NEXT:   add.rn.ftz.f16     [[R:%h[0-9]+]], [[A]], [[B]];
-; CHECK-NOF16-DAG:  cvt.f32.f16    [[A32:%f[0-9]+]], [[A]]
-; CHECK-NOF16-DAG:  cvt.f32.f16    [[B32:%f[0-9]+]], [[B]]
-; CHECK-NOF16-NEXT: add.rn.f32     [[R32:%f[0-9]+]], [[A32]], [[B32]];
-; CHECK-NOF16-NEXT: cvt.rn.f16.f32 [[R:%h[0-9]+]], [[R32]]
-; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define half @test_fadd(half %a, half %b) #0 {
-  %r = fadd half %a, %b
-  ret half %r
-}
-
-; CHECK-LABEL: test_fadd_v1f16(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fadd_v1f16_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fadd_v1f16_param_1];
-; CHECK-F16-NOFTZ-NEXT:   add.rn.f16     [[R:%h[0-9]+]], [[A]], [[B]];
-; CHECK-F16-FTZ-NEXT:   add.rn.ftz.f16     [[R:%h[0-9]+]], [[A]], [[B]];
-; CHECK-NOF16-DAG:  cvt.f32.f16    [[A32:%f[0-9]+]], [[A]]
-; CHECK-NOF16-DAG:  cvt.f32.f16    [[B32:%f[0-9]+]], [[B]]
-; CHECK-NOF16-NEXT: add.rn.f32     [[R32:%f[0-9]+]], [[A32]], [[B32]];
-; CHECK-NOF16-NEXT: cvt.rn.f16.f32 [[R:%h[0-9]+]], [[R32]]
-; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define <1 x half> @test_fadd_v1f16(<1 x half> %a, <1 x half> %b) #0 {
-  %r = fadd <1 x half> %a, %b
-  ret <1 x half> %r
-}
-
-; Check that we can lower fadd with immediate arguments.
-; CHECK-LABEL: test_fadd_imm_0(
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fadd_imm_0_param_0];
-; CHECK-F16-NOFTZ-DAG:    mov.b16        [[A:%h[0-9]+]], 0x3C00;
-; CHECK-F16-NOFTZ-NEXT:   add.rn.f16     [[R:%h[0-9]+]], [[B]], [[A]];
-; CHECK-F16-FTZ-DAG:    mov.b16        [[A:%h[0-9]+]], 0x3C00;
-; CHECK-F16-FTZ-NEXT:   add.rn.ftz.f16     [[R:%h[0-9]+]], [[B]], [[A]];
-; CHECK-NOF16-DAG:  cvt.f32.f16    [[B32:%f[0-9]+]], [[B]]
-; CHECK-NOF16-NEXT: add.rn.f32     [[R32:%f[0-9]+]], [[B32]], 0f3F800000;
-; CHECK-NOF16-NEXT: cvt.rn.f16.f32 [[R:%h[0-9]+]], [[R32]]
-; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define half @test_fadd_imm_0(half %b) #0 {
-  %r = fadd half 1.0, %b
-  ret half %r
-}
-
-; CHECK-LABEL: test_fadd_imm_1(
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fadd_imm_1_param_0];
-; CHECK-F16-NOFTZ-DAG:    mov.b16        [[A:%h[0-9]+]], 0x3C00;
-; CHECK-F16-NOFTZ-NEXT:   add.rn.f16     [[R:%h[0-9]+]], [[B]], [[A]];
-; CHECK-F16-FTZ-DAG:    mov.b16        [[A:%h[0-9]+]], 0x3C00;
-; CHECK-F16-FTZ-NEXT:   add.rn.ftz.f16     [[R:%h[0-9]+]], [[B]], [[A]];
-; CHECK-NOF16-DAG:  cvt.f32.f16    [[B32:%f[0-9]+]], [[B]]
-; CHECK-NOF16-NEXT: add.rn.f32     [[R32:%f[0-9]+]], [[B32]], 0f3F800000;
-; CHECK-NOF16-NEXT: cvt.rn.f16.f32 [[R:%h[0-9]+]], [[R32]]
-; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define half @test_fadd_imm_1(half %a) #0 {
-  %r = fadd half %a, 1.0
-  ret half %r
-}
-
-; CHECK-LABEL: test_fsub(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fsub_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fsub_param_1];
-; CHECK-F16-NOFTZ-NEXT:   sub.rn.f16     [[R:%h[0-9]+]], [[A]], [[B]];
-; CHECK-F16-FTZ-NEXT:   sub.rn.ftz.f16     [[R:%h[0-9]+]], [[A]], [[B]];
-; CHECK-NOF16-DAG:  cvt.f32.f16    [[A32:%f[0-9]+]], [[A]]
-; CHECK-NOF16-DAG:  cvt.f32.f16    [[B32:%f[0-9]+]], [[B]]
-; CHECK-NOF16-NEXT: sub.rn.f32     [[R32:%f[0-9]+]], [[A32]], [[B32]];
-; CHECK-NOF16-NEXT: cvt.rn.f16.f32 [[R:%h[0-9]+]], [[R32]]
-; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define half @test_fsub(half %a, half %b) #0 {
-  %r = fsub half %a, %b
-  ret half %r
-}
-
-; CHECK-LABEL: test_fneg(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fneg_param_0];
-; CHECK-F16-NOFTZ-NEXT:   mov.b16        [[Z:%h[0-9]+]], 0x0000
-; CHECK-F16-NOFTZ-NEXT:   sub.rn.f16     [[R:%h[0-9]+]], [[Z]], [[A]];
-; CHECK-F16-FTZ-NEXT:   mov.b16        [[Z:%h[0-9]+]], 0x0000
-; CHECK-F16-FTZ-NEXT:   sub.rn.ftz.f16     [[R:%h[0-9]+]], [[Z]], [[A]];
-; CHECK-NOF16-DAG:  cvt.f32.f16    [[A32:%f[0-9]+]], [[A]]
-; CHECK-NOF16-DAG:  mov.f32        [[Z:%f[0-9]+]], 0f00000000;
-; CHECK-NOF16-NEXT: sub.rn.f32     [[R32:%f[0-9]+]], [[Z]], [[A32]];
-; CHECK-NOF16-NEXT: cvt.rn.f16.f32 [[R:%h[0-9]+]], [[R32]]
-; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define half @test_fneg(half %a) #0 {
-  %r = fsub half 0.0, %a
-  ret half %r
-}
-
-; CHECK-LABEL: test_fmul(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fmul_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fmul_param_1];
-; CHECK-F16-NOFTZ-NEXT: mul.rn.f16      [[R:%h[0-9]+]], [[A]], [[B]];
-; CHECK-F16-FTZ-NEXT: mul.rn.ftz.f16      [[R:%h[0-9]+]], [[A]], [[B]];
-; CHECK-NOF16-DAG:  cvt.f32.f16    [[A32:%f[0-9]+]], [[A]]
-; CHECK-NOF16-DAG:  cvt.f32.f16    [[B32:%f[0-9]+]], [[B]]
-; CHECK-NOF16-NEXT: mul.rn.f32     [[R32:%f[0-9]+]], [[A32]], [[B32]];
-; CHECK-NOF16-NEXT: cvt.rn.f16.f32 [[R:%h[0-9]+]], [[R32]]
-; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define half @test_fmul(half %a, half %b) #0 {
-  %r = fmul half %a, %b
-  ret half %r
-}
-
-; CHECK-LABEL: test_fdiv(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fdiv_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fdiv_param_1];
-; CHECK-NOFTZ-DAG:  cvt.f32.f16     [[F0:%f[0-9]+]], [[A]];
-; CHECK-NOFTZ-DAG:  cvt.f32.f16     [[F1:%f[0-9]+]], [[B]];
-; CHECK-NOFTZ-NEXT: div.rn.f32      [[FR:%f[0-9]+]], [[F0]], [[F1]];
-; CHECK-F16-FTZ-DAG:  cvt.ftz.f32.f16     [[F0:%f[0-9]+]], [[A]];
-; CHECK-F16-FTZ-DAG:  cvt.ftz.f32.f16     [[F1:%f[0-9]+]], [[B]];
-; CHECK-F16-FTZ-NEXT: div.rn.ftz.f32      [[FR:%f[0-9]+]], [[F0]], [[F1]];
-; CHECK-NEXT: cvt.rn.f16.f32  [[R:%h[0-9]+]], [[FR]];
-; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define half @test_fdiv(half %a, half %b) #0 {
-  %r = fdiv half %a, %b
-  ret half %r
-}
-
-; CHECK-LABEL: test_frem(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_frem_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_frem_param_1];
-; CHECK-NOFTZ-DAG:  cvt.f32.f16     [[FA:%f[0-9]+]], [[A]];
-; CHECK-NOFTZ-DAG:  cvt.f32.f16     [[FB:%f[0-9]+]], [[B]];
-; CHECK-NOFTZ-NEXT: div.rn.f32      [[D:%f[0-9]+]], [[FA]], [[FB]];
-; CHECK-NOFTZ-NEXT: cvt.rmi.f32.f32 [[DI:%f[0-9]+]], [[D]];
-; CHECK-NOFTZ-NEXT: mul.f32         [[RI:%f[0-9]+]], [[DI]], [[FB]];
-; CHECK-NOFTZ-NEXT: sub.f32         [[RF:%f[0-9]+]], [[FA]], [[RI]];
-; CHECK-F16-FTZ-DAG:  cvt.ftz.f32.f16     [[FA:%f[0-9]+]], [[A]];
-; CHECK-F16-FTZ-DAG:  cvt.ftz.f32.f16     [[FB:%f[0-9]+]], [[B]];
-; CHECK-F16-FTZ-NEXT: div.rn.ftz.f32      [[D:%f[0-9]+]], [[FA]], [[FB]];
-; CHECK-F16-FTZ-NEXT: cvt.rmi.ftz.f32.f32 [[DI:%f[0-9]+]], [[D]];
-; CHECK-F16-FTZ-NEXT: mul.ftz.f32         [[RI:%f[0-9]+]], [[DI]], [[FB]];
-; CHECK-F16-FTZ-NEXT: sub.ftz.f32         [[RF:%f[0-9]+]], [[FA]], [[RI]];
-; CHECK-NEXT: cvt.rn.f16.f32  [[R:%h[0-9]+]], [[RF]];
-; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define half @test_frem(half %a, half %b) #0 {
-  %r = frem half %a, %b
-  ret half %r
-}
-
-; CHECK-LABEL: test_store(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_store_param_0];
-; CHECK-DAG:  ld.param.u64    %[[PTR:rd[0-9]+]], [test_store_param_1];
-; CHECK-NEXT: st.b16          [%[[PTR]]], [[A]];
-; CHECK-NEXT: ret;
-define void @test_store(half %a, half* %b) #0 {
-  store half %a, half* %b
-  ret void
-}
-
-; CHECK-LABEL: test_load(
-; CHECK:      ld.param.u64    %[[PTR:rd[0-9]+]], [test_load_param_0];
-; CHECK-NEXT: ld.b16          [[R:%h[0-9]+]], [%[[PTR]]];
-; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define half @test_load(half* %a) #0 {
-  %r = load half, half* %a
-  ret half %r
-}
-
-; CHECK-LABEL: .visible .func test_halfp0a1(
-; CHECK-DAG: ld.param.u64 %[[FROM:rd?[0-9]+]], [test_halfp0a1_param_0];
-; CHECK-DAG: ld.param.u64 %[[TO:rd?[0-9]+]], [test_halfp0a1_param_1];
-; CHECK-DAG: ld.u8        [[B0:%r[sd]?[0-9]+]], [%[[FROM]]]
-; CHECK-DAG: st.u8        [%[[TO]]], [[B0]]
-; CHECK-DAG: ld.u8        [[B1:%r[sd]?[0-9]+]], [%[[FROM]]+1]
-; CHECK-DAG: st.u8        [%[[TO]]+1], [[B1]]
-; CHECK: ret
-define void @test_halfp0a1(half * noalias readonly %from, half * %to) {
-  %1 = load half, half * %from , align 1
-  store half %1, half * %to , align 1
-  ret void
-}
-
-declare half @test_callee(half %a, half %b) #0
-
-; CHECK-LABEL: test_call(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_call_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_call_param_1];
-; CHECK:      {
-; CHECK-DAG:  .param .b32 param0;
-; CHECK-DAG:  .param .b32 param1;
-; CHECK-DAG:  st.param.b16    [param0+0], [[A]];
-; CHECK-DAG:  st.param.b16    [param1+0], [[B]];
-; CHECK-DAG:  .param .b32 retval0;
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT:        test_callee,
-; CHECK:      );
-; CHECK-NEXT: ld.param.b16    [[R:%h[0-9]+]], [retval0+0];
-; CHECK-NEXT: }
-; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define half @test_call(half %a, half %b) #0 {
-  %r = call half @test_callee(half %a, half %b)
-  ret half %r
-}
-
-; CHECK-LABEL: test_call_flipped(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_call_flipped_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_call_flipped_param_1];
-; CHECK:      {
-; CHECK-DAG:  .param .b32 param0;
-; CHECK-DAG:  .param .b32 param1;
-; CHECK-DAG:  st.param.b16    [param0+0], [[B]];
-; CHECK-DAG:  st.param.b16    [param1+0], [[A]];
-; CHECK-DAG:  .param .b32 retval0;
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT:        test_callee,
-; CHECK:      );
-; CHECK-NEXT: ld.param.b16    [[R:%h[0-9]+]], [retval0+0];
-; CHECK-NEXT: }
-; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define half @test_call_flipped(half %a, half %b) #0 {
-  %r = call half @test_callee(half %b, half %a)
-  ret half %r
-}
-
-; CHECK-LABEL: test_tailcall_flipped(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_tailcall_flipped_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_tailcall_flipped_param_1];
-; CHECK:      {
-; CHECK-DAG:  .param .b32 param0;
-; CHECK-DAG:  .param .b32 param1;
-; CHECK-DAG:  st.param.b16    [param0+0], [[B]];
-; CHECK-DAG:  st.param.b16    [param1+0], [[A]];
-; CHECK-DAG:  .param .b32 retval0;
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT:        test_callee,
-; CHECK:      );
-; CHECK-NEXT: ld.param.b16    [[R:%h[0-9]+]], [retval0+0];
-; CHECK-NEXT: }
-; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define half @test_tailcall_flipped(half %a, half %b) #0 {
-  %r = tail call half @test_callee(half %b, half %a)
-  ret half %r
-}
-
-; CHECK-LABEL: test_select(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_select_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_select_param_1];
-; CHECK-DAG:  setp.eq.b16     [[PRED:%p[0-9]+]], %rs{{.*}}, 1;
-; CHECK-NEXT: selp.b16        [[R:%h[0-9]+]], [[A]], [[B]], [[PRED]];
-; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define half @test_select(half %a, half %b, i1 zeroext %c) #0 {
-  %r = select i1 %c, half %a, half %b
-  ret half %r
-}
-
-; CHECK-LABEL: test_select_cc(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_select_cc_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_select_cc_param_1];
-; CHECK-DAG:  ld.param.b16    [[C:%h[0-9]+]], [test_select_cc_param_2];
-; CHECK-DAG:  ld.param.b16    [[D:%h[0-9]+]], [test_select_cc_param_3];
-; CHECK-F16-NOFTZ:  setp.neu.f16    [[PRED:%p[0-9]+]], [[C]], [[D]]
-; CHECK-NOF16-DAG: cvt.f32.f16 [[DF:%f[0-9]+]], [[D]];
-; CHECK-NOF16-DAG: cvt.f32.f16 [[CF:%f[0-9]+]], [[C]];
-; CHECK-NOF16: setp.neu.f32    [[PRED:%p[0-9]+]], [[CF]], [[DF]]
-; CHECK:      selp.b16        [[R:%h[0-9]+]], [[A]], [[B]], [[PRED]];
-; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define half @test_select_cc(half %a, half %b, half %c, half %d) #0 {
-  %cc = fcmp une half %c, %d
-  %r = select i1 %cc, half %a, half %b
-  ret half %r
-}
-
-; CHECK-LABEL: test_select_cc_f32_f16(
-; CHECK-DAG:  ld.param.f32    [[A:%f[0-9]+]], [test_select_cc_f32_f16_param_0];
-; CHECK-DAG:  ld.param.f32    [[B:%f[0-9]+]], [test_select_cc_f32_f16_param_1];
-; CHECK-DAG:  ld.param.b16    [[C:%h[0-9]+]], [test_select_cc_f32_f16_param_2];
-; CHECK-DAG:  ld.param.b16    [[D:%h[0-9]+]], [test_select_cc_f32_f16_param_3];
-; CHECK-F16-NOFTZ:  setp.neu.f16    [[PRED:%p[0-9]+]], [[C]], [[D]]
-; CHECK-F16-FTZ:  setp.neu.ftz.f16    [[PRED:%p[0-9]+]], [[C]], [[D]]
-; CHECK-NOF16-DAG: cvt.f32.f16 [[DF:%f[0-9]+]], [[D]];
-; CHECK-NOF16-DAG: cvt.f32.f16 [[CF:%f[0-9]+]], [[C]];
-; CHECK-NOF16: setp.neu.f32    [[PRED:%p[0-9]+]], [[CF]], [[DF]]
-; CHECK-NEXT: selp.f32        [[R:%f[0-9]+]], [[A]], [[B]], [[PRED]];
-; CHECK-NEXT: st.param.f32    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define float @test_select_cc_f32_f16(float %a, float %b, half %c, half %d) #0 {
-  %cc = fcmp une half %c, %d
-  %r = select i1 %cc, float %a, float %b
-  ret float %r
-}
-
-; CHECK-LABEL: test_select_cc_f16_f32(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_select_cc_f16_f32_param_0];
-; CHECK-DAG:  ld.param.f32    [[C:%f[0-9]+]], [test_select_cc_f16_f32_param_2];
-; CHECK-DAG:  ld.param.f32    [[D:%f[0-9]+]], [test_select_cc_f16_f32_param_3];
-; CHECK-NOFTZ-DAG:  setp.neu.f32    [[PRED:%p[0-9]+]], [[C]], [[D]]
-; CHECK-F16-FTZ-DAG:  setp.neu.ftz.f32    [[PRED:%p[0-9]+]], [[C]], [[D]]
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_select_cc_f16_f32_param_1];
-; CHECK-NEXT: selp.b16        [[R:%h[0-9]+]], [[A]], [[B]], [[PRED]];
-; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define half @test_select_cc_f16_f32(half %a, half %b, float %c, float %d) #0 {
-  %cc = fcmp une float %c, %d
-  %r = select i1 %cc, half %a, half %b
-  ret half %r
-}
-
-; CHECK-LABEL: test_fcmp_une(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fcmp_une_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fcmp_une_param_1];
-; CHECK-F16-NOFTZ:  setp.neu.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
-; CHECK-F16-FTZ:  setp.neu.ftz.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
-; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%f[0-9]+]], [[A]];
-; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%f[0-9]+]], [[B]];
-; CHECK-NOF16: setp.neu.f32   [[PRED:%p[0-9]+]], [[AF]], [[BF]]
-; CHECK-NEXT: selp.u32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
-; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define i1 @test_fcmp_une(half %a, half %b) #0 {
-  %r = fcmp une half %a, %b
-  ret i1 %r
-}
-
-; CHECK-LABEL: test_fcmp_ueq(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fcmp_ueq_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fcmp_ueq_param_1];
-; CHECK-F16-NOFTZ:  setp.equ.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
-; CHECK-F16-FTZ:  setp.equ.ftz.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
-; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%f[0-9]+]], [[A]];
-; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%f[0-9]+]], [[B]];
-; CHECK-NOF16: setp.equ.f32   [[PRED:%p[0-9]+]], [[AF]], [[BF]]
-; CHECK-NEXT: selp.u32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
-; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define i1 @test_fcmp_ueq(half %a, half %b) #0 {
-  %r = fcmp ueq half %a, %b
-  ret i1 %r
-}
-
-; CHECK-LABEL: test_fcmp_ugt(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fcmp_ugt_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fcmp_ugt_param_1];
-; CHECK-F16-NOFTZ:  setp.gtu.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
-; CHECK-F16-FTZ:  setp.gtu.ftz.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
-; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%f[0-9]+]], [[A]];
-; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%f[0-9]+]], [[B]];
-; CHECK-NOF16: setp.gtu.f32   [[PRED:%p[0-9]+]], [[AF]], [[BF]]
-; CHECK-NEXT: selp.u32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
-; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define i1 @test_fcmp_ugt(half %a, half %b) #0 {
-  %r = fcmp ugt half %a, %b
-  ret i1 %r
-}
-
-; CHECK-LABEL: test_fcmp_uge(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fcmp_uge_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fcmp_uge_param_1];
-; CHECK-F16-NOFTZ:  setp.geu.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
-; CHECK-F16-FTZ:  setp.geu.ftz.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
-; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%f[0-9]+]], [[A]];
-; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%f[0-9]+]], [[B]];
-; CHECK-NOF16: setp.geu.f32   [[PRED:%p[0-9]+]], [[AF]], [[BF]]
-; CHECK-NEXT: selp.u32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
-; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define i1 @test_fcmp_uge(half %a, half %b) #0 {
-  %r = fcmp uge half %a, %b
-  ret i1 %r
-}
-
-; CHECK-LABEL: test_fcmp_ult(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fcmp_ult_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fcmp_ult_param_1];
-; CHECK-F16-NOFTZ:  setp.ltu.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
-; CHECK-F16-FTZ:  setp.ltu.ftz.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
-; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%f[0-9]+]], [[A]];
-; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%f[0-9]+]], [[B]];
-; CHECK-NOF16: setp.ltu.f32   [[PRED:%p[0-9]+]], [[AF]], [[BF]]
-; CHECK-NEXT: selp.u32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
-; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define i1 @test_fcmp_ult(half %a, half %b) #0 {
-  %r = fcmp ult half %a, %b
-  ret i1 %r
-}
-
-; CHECK-LABEL: test_fcmp_ule(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fcmp_ule_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fcmp_ule_param_1];
-; CHECK-F16-NOFTZ:  setp.leu.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
-; CHECK-F16-FTZ:  setp.leu.ftz.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
-; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%f[0-9]+]], [[A]];
-; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%f[0-9]+]], [[B]];
-; CHECK-NOF16: setp.leu.f32   [[PRED:%p[0-9]+]], [[AF]], [[BF]]
-; CHECK-NEXT: selp.u32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
-; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define i1 @test_fcmp_ule(half %a, half %b) #0 {
-  %r = fcmp ule half %a, %b
-  ret i1 %r
-}
-
-
-; CHECK-LABEL: test_fcmp_uno(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fcmp_uno_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fcmp_uno_param_1];
-; CHECK-F16-NOFTZ:  setp.nan.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
-; CHECK-F16-FTZ:  setp.nan.ftz.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
-; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%f[0-9]+]], [[A]];
-; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%f[0-9]+]], [[B]];
-; CHECK-NOF16: setp.nan.f32   [[PRED:%p[0-9]+]], [[AF]], [[BF]]
-; CHECK-NEXT: selp.u32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
-; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define i1 @test_fcmp_uno(half %a, half %b) #0 {
-  %r = fcmp uno half %a, %b
-  ret i1 %r
-}
-
-; CHECK-LABEL: test_fcmp_one(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fcmp_one_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fcmp_one_param_1];
-; CHECK-F16-NOFTZ:  setp.ne.f16     [[PRED:%p[0-9]+]], [[A]], [[B]]
-; CHECK-F16-FTZ:  setp.ne.ftz.f16     [[PRED:%p[0-9]+]], [[A]], [[B]]
-; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%f[0-9]+]], [[A]];
-; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%f[0-9]+]], [[B]];
-; CHECK-NOF16: setp.ne.f32    [[PRED:%p[0-9]+]], [[AF]], [[BF]]
-; CHECK-NEXT: selp.u32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
-; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define i1 @test_fcmp_one(half %a, half %b) #0 {
-  %r = fcmp one half %a, %b
-  ret i1 %r
-}
-
-; CHECK-LABEL: test_fcmp_oeq(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fcmp_oeq_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fcmp_oeq_param_1];
-; CHECK-F16-NOFTZ:  setp.eq.f16     [[PRED:%p[0-9]+]], [[A]], [[B]]
-; CHECK-F16-FTZ:  setp.eq.ftz.f16     [[PRED:%p[0-9]+]], [[A]], [[B]]
-; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%f[0-9]+]], [[A]];
-; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%f[0-9]+]], [[B]];
-; CHECK-NOF16: setp.eq.f32    [[PRED:%p[0-9]+]], [[AF]], [[BF]]
-; CHECK-NEXT: selp.u32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
-; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define i1 @test_fcmp_oeq(half %a, half %b) #0 {
-  %r = fcmp oeq half %a, %b
-  ret i1 %r
-}
-
-; CHECK-LABEL: test_fcmp_ogt(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fcmp_ogt_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fcmp_ogt_param_1];
-; CHECK-F16-NOFTZ:  setp.gt.f16     [[PRED:%p[0-9]+]], [[A]], [[B]]
-; CHECK-F16-FTZ:  setp.gt.ftz.f16     [[PRED:%p[0-9]+]], [[A]], [[B]]
-; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%f[0-9]+]], [[A]];
-; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%f[0-9]+]], [[B]];
-; CHECK-NOF16: setp.gt.f32    [[PRED:%p[0-9]+]], [[AF]], [[BF]]
-; CHECK-NEXT: selp.u32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
-; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define i1 @test_fcmp_ogt(half %a, half %b) #0 {
-  %r = fcmp ogt half %a, %b
-  ret i1 %r
-}
-
-; CHECK-LABEL: test_fcmp_oge(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fcmp_oge_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fcmp_oge_param_1];
-; CHECK-F16-NOFTZ:  setp.ge.f16     [[PRED:%p[0-9]+]], [[A]], [[B]]
-; CHECK-F16-FTZ:  setp.ge.ftz.f16     [[PRED:%p[0-9]+]], [[A]], [[B]]
-; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%f[0-9]+]], [[A]];
-; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%f[0-9]+]], [[B]];
-; CHECK-NOF16: setp.ge.f32    [[PRED:%p[0-9]+]], [[AF]], [[BF]]
-; CHECK-NEXT: selp.u32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
-; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define i1 @test_fcmp_oge(half %a, half %b) #0 {
-  %r = fcmp oge half %a, %b
-  ret i1 %r
-}
-
-; XCHECK-LABEL: test_fcmp_olt(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fcmp_olt_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fcmp_olt_param_1];
-; CHECK-F16-NOFTZ:  setp.lt.f16     [[PRED:%p[0-9]+]], [[A]], [[B]]
-; CHECK-F16-FTZ:  setp.lt.ftz.f16     [[PRED:%p[0-9]+]], [[A]], [[B]]
-; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%f[0-9]+]], [[A]];
-; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%f[0-9]+]], [[B]];
-; CHECK-NOF16: setp.lt.f32    [[PRED:%p[0-9]+]], [[AF]], [[BF]]
-; CHECK-NEXT: selp.u32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
-; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define i1 @test_fcmp_olt(half %a, half %b) #0 {
-  %r = fcmp olt half %a, %b
-  ret i1 %r
-}
-
-; XCHECK-LABEL: test_fcmp_ole(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fcmp_ole_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fcmp_ole_param_1];
-; CHECK-F16-NOFTZ:  setp.le.f16     [[PRED:%p[0-9]+]], [[A]], [[B]]
-; CHECK-F16-FTZ:  setp.le.ftz.f16     [[PRED:%p[0-9]+]], [[A]], [[B]]
-; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%f[0-9]+]], [[A]];
-; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%f[0-9]+]], [[B]];
-; CHECK-NOF16: setp.le.f32    [[PRED:%p[0-9]+]], [[AF]], [[BF]]
-; CHECK-NEXT: selp.u32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
-; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define i1 @test_fcmp_ole(half %a, half %b) #0 {
-  %r = fcmp ole half %a, %b
-  ret i1 %r
-}
-
-; CHECK-LABEL: test_fcmp_ord(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fcmp_ord_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fcmp_ord_param_1];
-; CHECK-F16-NOFTZ:  setp.num.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
-; CHECK-F16-FTZ:  setp.num.ftz.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
-; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%f[0-9]+]], [[A]];
-; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%f[0-9]+]], [[B]];
-; CHECK-NOF16: setp.num.f32   [[PRED:%p[0-9]+]], [[AF]], [[BF]]
-; CHECK-NEXT: selp.u32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
-; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-define i1 @test_fcmp_ord(half %a, half %b) #0 {
-  %r = fcmp ord half %a, %b
-  ret i1 %r
-}
-
-; CHECK-LABEL: test_br_cc(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_br_cc_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_br_cc_param_1];
-; CHECK-DAG:  ld.param.u64    %[[C:rd[0-9]+]], [test_br_cc_param_2];
-; CHECK-DAG:  ld.param.u64    %[[D:rd[0-9]+]], [test_br_cc_param_3];
-; CHECK-F16-NOFTZ:  setp.lt.f16     [[PRED:%p[0-9]+]], [[A]], [[B]]
-; CHECK-F16-FTZ:  setp.lt.ftz.f16     [[PRED:%p[0-9]+]], [[A]], [[B]]
-; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%f[0-9]+]], [[A]];
-; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%f[0-9]+]], [[B]];
-; CHECK-NOF16: setp.lt.f32    [[PRED:%p[0-9]+]], [[AF]], [[BF]]
-; CHECK-NEXT: @[[PRED]] bra   [[LABEL:LBB.*]];
-; CHECK:      st.u32  [%[[C]]],
-; CHECK:      [[LABEL]]:
-; CHECK:      st.u32  [%[[D]]],
-; CHECK:      ret;
-define void @test_br_cc(half %a, half %b, i32* %p1, i32* %p2) #0 {
-  %c = fcmp uge half %a, %b
-  br i1 %c, label %then, label %else
-then:
-  store i32 0, i32* %p1
-  ret void
-else:
-  store i32 0, i32* %p2
-  ret void
-}
-
-; CHECK-LABEL: test_phi(
-; CHECK:      ld.param.u64    %[[P1:rd[0-9]+]], [test_phi_param_0];
-; CHECK:      ld.b16  {{%h[0-9]+}}, [%[[P1]]];
-; CHECK: [[LOOP:LBB[0-9_]+]]:
-; CHECK:      mov.b16 [[R:%h[0-9]+]], [[AB:%h[0-9]+]];
-; CHECK:      ld.b16  [[AB:%h[0-9]+]], [%[[P1]]];
-; CHECK:      {
-; CHECK:      st.param.b64    [param0+0], %[[P1]];
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_dummy
-; CHECK:      }
-; CHECK:      setp.eq.b32     [[PRED:%p[0-9]+]], %r{{[0-9]+}}, 1;
-; CHECK:      @[[PRED]] bra   [[LOOP]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define half @test_phi(half* %p1) #0 {
-entry:
-  %a = load half, half* %p1
-  br label %loop
-loop:
-  %r = phi half [%a, %entry], [%b, %loop]
-  %b = load half, half* %p1
-  %c = call i1 @test_dummy(half* %p1)
-  br i1 %c, label %loop, label %return
-return:
-  ret half %r
-}
-declare i1 @test_dummy(half* %p1) #0
-
-; CHECK-LABEL: test_fptosi_i32(
-; CHECK:      ld.param.b16    [[A:%h[0-9]+]], [test_fptosi_i32_param_0];
-; CHECK:      cvt.rzi.s32.f16 [[R:%r[0-9]+]], [[A]];
-; CHECK:      st.param.b32    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define i32 @test_fptosi_i32(half %a) #0 {
-  %r = fptosi half %a to i32
-  ret i32 %r
-}
-
-; CHECK-LABEL: test_fptosi_i64(
-; CHECK:      ld.param.b16    [[A:%h[0-9]+]], [test_fptosi_i64_param_0];
-; CHECK:      cvt.rzi.s64.f16 [[R:%rd[0-9]+]], [[A]];
-; CHECK:      st.param.b64    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define i64 @test_fptosi_i64(half %a) #0 {
-  %r = fptosi half %a to i64
-  ret i64 %r
-}
-
-; CHECK-LABEL: test_fptoui_i32(
-; CHECK:      ld.param.b16    [[A:%h[0-9]+]], [test_fptoui_i32_param_0];
-; CHECK:      cvt.rzi.u32.f16 [[R:%r[0-9]+]], [[A]];
-; CHECK:      st.param.b32    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define i32 @test_fptoui_i32(half %a) #0 {
-  %r = fptoui half %a to i32
-  ret i32 %r
-}
-
-; CHECK-LABEL: test_fptoui_i64(
-; CHECK:      ld.param.b16    [[A:%h[0-9]+]], [test_fptoui_i64_param_0];
-; CHECK:      cvt.rzi.u64.f16 [[R:%rd[0-9]+]], [[A]];
-; CHECK:      st.param.b64    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define i64 @test_fptoui_i64(half %a) #0 {
-  %r = fptoui half %a to i64
-  ret i64 %r
-}
-
-; CHECK-LABEL: test_uitofp_i32(
-; CHECK:      ld.param.u32    [[A:%r[0-9]+]], [test_uitofp_i32_param_0];
-; CHECK:      cvt.rn.f16.u32  [[R:%h[0-9]+]], [[A]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define half @test_uitofp_i32(i32 %a) #0 {
-  %r = uitofp i32 %a to half
-  ret half %r
-}
-
-; CHECK-LABEL: test_uitofp_i64(
-; CHECK:      ld.param.u64    [[A:%rd[0-9]+]], [test_uitofp_i64_param_0];
-; CHECK:      cvt.rn.f16.u64  [[R:%h[0-9]+]], [[A]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define half @test_uitofp_i64(i64 %a) #0 {
-  %r = uitofp i64 %a to half
-  ret half %r
-}
-
-; CHECK-LABEL: test_sitofp_i32(
-; CHECK:      ld.param.u32    [[A:%r[0-9]+]], [test_sitofp_i32_param_0];
-; CHECK:      cvt.rn.f16.s32  [[R:%h[0-9]+]], [[A]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define half @test_sitofp_i32(i32 %a) #0 {
-  %r = sitofp i32 %a to half
-  ret half %r
-}
-
-; CHECK-LABEL: test_sitofp_i64(
-; CHECK:      ld.param.u64    [[A:%rd[0-9]+]], [test_sitofp_i64_param_0];
-; CHECK:      cvt.rn.f16.s64  [[R:%h[0-9]+]], [[A]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define half @test_sitofp_i64(i64 %a) #0 {
-  %r = sitofp i64 %a to half
-  ret half %r
-}
-
-; CHECK-LABEL: test_uitofp_i32_fadd(
-; CHECK-DAG:  ld.param.u32    [[A:%r[0-9]+]], [test_uitofp_i32_fadd_param_0];
-; CHECK-DAG:  cvt.rn.f16.u32  [[C:%h[0-9]+]], [[A]];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_uitofp_i32_fadd_param_1];
-; CHECK-F16-NOFTZ:       add.rn.f16      [[R:%h[0-9]+]], [[B]], [[C]];
-; CHECK-F16-FTZ:       add.rn.ftz.f16      [[R:%h[0-9]+]], [[B]], [[C]];
-; CHECK-NOF16-DAG:  cvt.f32.f16    [[B32:%f[0-9]+]], [[B]]
-; CHECK-NOF16-DAG:  cvt.f32.f16    [[C32:%f[0-9]+]], [[C]]
-; CHECK-NOF16-NEXT: add.rn.f32     [[R32:%f[0-9]+]], [[B32]], [[C32]];
-; CHECK-NOF16-NEXT: cvt.rn.f16.f32 [[R:%h[0-9]+]], [[R32]]
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define half @test_uitofp_i32_fadd(i32 %a, half %b) #0 {
-  %c = uitofp i32 %a to half
-  %r = fadd half %b, %c
-  ret half %r
-}
-
-; CHECK-LABEL: test_sitofp_i32_fadd(
-; CHECK-DAG:  ld.param.u32    [[A:%r[0-9]+]], [test_sitofp_i32_fadd_param_0];
-; CHECK-DAG:  cvt.rn.f16.s32  [[C:%h[0-9]+]], [[A]];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_sitofp_i32_fadd_param_1];
-; CHECK-F16-NOFTZ:         add.rn.f16     [[R:%h[0-9]+]], [[B]], [[C]];
-; CHECK-F16-FTZ:         add.rn.ftz.f16     [[R:%h[0-9]+]], [[B]], [[C]];
-; XCHECK-NOF16-DAG:  cvt.f32.f16    [[B32:%f[0-9]+]], [[B]]
-; XCHECK-NOF16-DAG:  cvt.f32.f16    [[C32:%f[0-9]+]], [[C]]
-; XCHECK-NOF16-NEXT: add.rn.f32     [[R32:%f[0-9]+]], [[B32]], [[C32]];
-; XCHECK-NOF16-NEXT: cvt.rn.f16.f32 [[R:%h[0-9]+]], [[R32]]
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define half @test_sitofp_i32_fadd(i32 %a, half %b) #0 {
-  %c = sitofp i32 %a to half
-  %r = fadd half %b, %c
-  ret half %r
-}
-
-; CHECK-LABEL: test_fptrunc_float(
-; CHECK:      ld.param.f32    [[A:%f[0-9]+]], [test_fptrunc_float_param_0];
-; CHECK:      cvt.rn.f16.f32  [[R:%h[0-9]+]], [[A]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define half @test_fptrunc_float(float %a) #0 {
-  %r = fptrunc float %a to half
-  ret half %r
-}
-
-; CHECK-LABEL: test_fptrunc_double(
-; CHECK:      ld.param.f64    [[A:%fd[0-9]+]], [test_fptrunc_double_param_0];
-; CHECK:      cvt.rn.f16.f64  [[R:%h[0-9]+]], [[A]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define half @test_fptrunc_double(double %a) #0 {
-  %r = fptrunc double %a to half
-  ret half %r
-}
-
-; CHECK-LABEL: test_fpext_float(
-; CHECK:      ld.param.b16    [[A:%h[0-9]+]], [test_fpext_float_param_0];
-; CHECK-NOFTZ:      cvt.f32.f16     [[R:%f[0-9]+]], [[A]];
-; CHECK-F16-FTZ:      cvt.ftz.f32.f16     [[R:%f[0-9]+]], [[A]];
-; CHECK:      st.param.f32    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define float @test_fpext_float(half %a) #0 {
-  %r = fpext half %a to float
-  ret float %r
-}
-
-; CHECK-LABEL: test_fpext_double(
-; CHECK:      ld.param.b16    [[A:%h[0-9]+]], [test_fpext_double_param_0];
-; CHECK:      cvt.f64.f16     [[R:%fd[0-9]+]], [[A]];
-; CHECK:      st.param.f64    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define double @test_fpext_double(half %a) #0 {
-  %r = fpext half %a to double
-  ret double %r
-}
-
-
-; CHECK-LABEL: test_bitcast_halftoi16(
-; CHECK:      ld.param.b16    [[AH:%h[0-9]+]], [test_bitcast_halftoi16_param_0];
-; CHECK:      mov.b16         [[AS:%rs[0-9]+]], [[AH]]
-; CHECK:      cvt.u32.u16     [[R:%r[0-9]+]], [[AS]]
-; CHECK:      st.param.b32    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define i16 @test_bitcast_halftoi16(half %a) #0 {
-  %r = bitcast half %a to i16
-  ret i16 %r
-}
-
-; CHECK-LABEL: test_bitcast_i16tohalf(
-; CHECK:      ld.param.u16    [[AS:%rs[0-9]+]], [test_bitcast_i16tohalf_param_0];
-; CHECK:      mov.b16         [[AH:%h[0-9]+]], [[AS]]
-; CHECK:      st.param.b16    [func_retval0+0], [[AH]];
-; CHECK:      ret;
-define half @test_bitcast_i16tohalf(i16 %a) #0 {
-  %r = bitcast i16 %a to half
-  ret half %r
-}
-
-
-declare half @llvm.sqrt.f16(half %a) #0
-declare half @llvm.powi.f16(half %a, i32 %b) #0
-declare half @llvm.sin.f16(half %a) #0
-declare half @llvm.cos.f16(half %a) #0
-declare half @llvm.pow.f16(half %a, half %b) #0
-declare half @llvm.exp.f16(half %a) #0
-declare half @llvm.exp2.f16(half %a) #0
-declare half @llvm.log.f16(half %a) #0
-declare half @llvm.log10.f16(half %a) #0
-declare half @llvm.log2.f16(half %a) #0
-declare half @llvm.fma.f16(half %a, half %b, half %c) #0
-declare half @llvm.fabs.f16(half %a) #0
-declare half @llvm.minnum.f16(half %a, half %b) #0
-declare half @llvm.maxnum.f16(half %a, half %b) #0
-declare half @llvm.copysign.f16(half %a, half %b) #0
-declare half @llvm.floor.f16(half %a) #0
-declare half @llvm.ceil.f16(half %a) #0
-declare half @llvm.trunc.f16(half %a) #0
-declare half @llvm.rint.f16(half %a) #0
-declare half @llvm.nearbyint.f16(half %a) #0
-declare half @llvm.round.f16(half %a) #0
-declare half @llvm.fmuladd.f16(half %a, half %b, half %c) #0
-
-; CHECK-LABEL: test_sqrt(
-; CHECK:      ld.param.b16    [[A:%h[0-9]+]], [test_sqrt_param_0];
-; CHECK-NOFTZ:      cvt.f32.f16     [[AF:%f[0-9]+]], [[A]];
-; CHECK-NOFTZ:      sqrt.rn.f32     [[RF:%f[0-9]+]], [[AF]];
-; CHECK-F16-FTZ:      cvt.ftz.f32.f16     [[AF:%f[0-9]+]], [[A]];
-; CHECK-F16-FTZ:      sqrt.rn.ftz.f32     [[RF:%f[0-9]+]], [[AF]];
-; CHECK:      cvt.rn.f16.f32  [[R:%h[0-9]+]], [[RF]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define half @test_sqrt(half %a) #0 {
-  %r = call half @llvm.sqrt.f16(half %a)
-  ret half %r
-}
-
-;;; Can't do this yet: requires libcall.
-; XCHECK-LABEL: test_powi(
-;define half @test_powi(half %a, i32 %b) #0 {
-;  %r = call half @llvm.powi.f16(half %a, i32 %b)
-;  ret half %r
-;}
-
-; CHECK-LABEL: test_sin(
-; CHECK:      ld.param.b16    [[A:%h[0-9]+]], [test_sin_param_0];
-; CHECK-NOFTZ:      cvt.f32.f16     [[AF:%f[0-9]+]], [[A]];
-; CHECK-F16-FTZ:      cvt.ftz.f32.f16     [[AF:%f[0-9]+]], [[A]];
-; CHECK:      sin.approx.f32  [[RF:%f[0-9]+]], [[AF]];
-; CHECK:      cvt.rn.f16.f32  [[R:%h[0-9]+]], [[RF]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define half @test_sin(half %a) #0 #1 {
-  %r = call half @llvm.sin.f16(half %a)
-  ret half %r
-}
-
-; CHECK-LABEL: test_cos(
-; CHECK:      ld.param.b16    [[A:%h[0-9]+]], [test_cos_param_0];
-; CHECK-NOFTZ:      cvt.f32.f16     [[AF:%f[0-9]+]], [[A]];
-; CHECK-F16-FTZ:      cvt.ftz.f32.f16     [[AF:%f[0-9]+]], [[A]];
-; CHECK:      cos.approx.f32  [[RF:%f[0-9]+]], [[AF]];
-; CHECK:      cvt.rn.f16.f32  [[R:%h[0-9]+]], [[RF]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define half @test_cos(half %a) #0 #1 {
-  %r = call half @llvm.cos.f16(half %a)
-  ret half %r
-}
-
-;;; Can't do this yet: requires libcall.
-; XCHECK-LABEL: test_pow(
-;define half @test_pow(half %a, half %b) #0 {
-;  %r = call half @llvm.pow.f16(half %a, half %b)
-;  ret half %r
-;}
-
-;;; Can't do this yet: requires libcall.
-; XCHECK-LABEL: test_exp(
-;define half @test_exp(half %a) #0 {
-;  %r = call half @llvm.exp.f16(half %a)
-;  ret half %r
-;}
-
-;;; Can't do this yet: requires libcall.
-; XCHECK-LABEL: test_exp2(
-;define half @test_exp2(half %a) #0 {
-;  %r = call half @llvm.exp2.f16(half %a)
-;  ret half %r
-;}
-
-;;; Can't do this yet: requires libcall.
-; XCHECK-LABEL: test_log(
-;define half @test_log(half %a) #0 {
-;  %r = call half @llvm.log.f16(half %a)
-;  ret half %r
-;}
-
-;;; Can't do this yet: requires libcall.
-; XCHECK-LABEL: test_log10(
-;define half @test_log10(half %a) #0 {
-;  %r = call half @llvm.log10.f16(half %a)
-;  ret half %r
-;}
-
-;;; Can't do this yet: requires libcall.
-; XCHECK-LABEL: test_log2(
-;define half @test_log2(half %a) #0 {
-;  %r = call half @llvm.log2.f16(half %a)
-;  ret half %r
-;}
-
-; CHECK-LABEL: test_fma(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fma_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fma_param_1];
-; CHECK-DAG:  ld.param.b16    [[C:%h[0-9]+]], [test_fma_param_2];
-; CHECK-F16-NOFTZ:      fma.rn.f16      [[R:%h[0-9]+]], [[A]], [[B]], [[C]];
-; CHECK-F16-FTZ:      fma.rn.ftz.f16      [[R:%h[0-9]+]], [[A]], [[B]], [[C]];
-; CHECK-NOF16-DAG:  cvt.f32.f16    [[A32:%f[0-9]+]], [[A]]
-; CHECK-NOF16-DAG:  cvt.f32.f16    [[B32:%f[0-9]+]], [[B]]
-; CHECK-NOF16-DAG:  cvt.f32.f16    [[C32:%f[0-9]+]], [[C]]
-; CHECK-NOF16-NEXT: fma.rn.f32     [[R32:%f[0-9]+]], [[A32]], [[B32]], [[C32]];
-; CHECK-NOF16-NEXT: cvt.rn.f16.f32 [[R:%h[0-9]+]], [[R32]]
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret
-define half @test_fma(half %a, half %b, half %c) #0 {
-  %r = call half @llvm.fma.f16(half %a, half %b, half %c)
-  ret half %r
-}
-
-; CHECK-LABEL: test_fabs(
-; CHECK:      ld.param.b16    [[A:%h[0-9]+]], [test_fabs_param_0];
-; CHECK-NOFTZ:      cvt.f32.f16     [[AF:%f[0-9]+]], [[A]];
-; CHECK-NOFTZ:      abs.f32         [[RF:%f[0-9]+]], [[AF]];
-; CHECK-F16-FTZ:      cvt.ftz.f32.f16     [[AF:%f[0-9]+]], [[A]];
-; CHECK-F16-FTZ:      abs.ftz.f32         [[RF:%f[0-9]+]], [[AF]];
-; CHECK:      cvt.rn.f16.f32  [[R:%h[0-9]+]], [[RF]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define half @test_fabs(half %a) #0 {
-  %r = call half @llvm.fabs.f16(half %a)
-  ret half %r
-}
-
-; CHECK-LABEL: test_minnum(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_minnum_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_minnum_param_1];
-; CHECK-NOFTZ-DAG:  cvt.f32.f16     [[AF:%f[0-9]+]], [[A]];
-; CHECK-NOFTZ-DAG:  cvt.f32.f16     [[BF:%f[0-9]+]], [[B]];
-; CHECK-NOFTZ:      min.f32         [[RF:%f[0-9]+]], [[AF]], [[BF]];
-; CHECK-F16-FTZ-DAG:  cvt.ftz.f32.f16     [[AF:%f[0-9]+]], [[A]];
-; CHECK-F16-FTZ-DAG:  cvt.ftz.f32.f16     [[BF:%f[0-9]+]], [[B]];
-; CHECK-F16-FTZ:      min.ftz.f32         [[RF:%f[0-9]+]], [[AF]], [[BF]];
-; CHECK:      cvt.rn.f16.f32  [[R:%h[0-9]+]], [[RF]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define half @test_minnum(half %a, half %b) #0 {
-  %r = call half @llvm.minnum.f16(half %a, half %b)
-  ret half %r
-}
-
-; CHECK-LABEL: test_maxnum(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_maxnum_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_maxnum_param_1];
-; CHECK-NOFTZ-DAG:  cvt.f32.f16     [[AF:%f[0-9]+]], [[A]];
-; CHECK-NOFTZ-DAG:  cvt.f32.f16     [[BF:%f[0-9]+]], [[B]];
-; CHECK-NOFTZ:      max.f32         [[RF:%f[0-9]+]], [[AF]], [[BF]];
-; CHECK-F16-FTZ-DAG:  cvt.ftz.f32.f16     [[AF:%f[0-9]+]], [[A]];
-; CHECK-F16-FTZ-DAG:  cvt.ftz.f32.f16     [[BF:%f[0-9]+]], [[B]];
-; CHECK-F16-FTZ:      max.ftz.f32         [[RF:%f[0-9]+]], [[AF]], [[BF]];
-; CHECK:      cvt.rn.f16.f32  [[R:%h[0-9]+]], [[RF]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define half @test_maxnum(half %a, half %b) #0 {
-  %r = call half @llvm.maxnum.f16(half %a, half %b)
-  ret half %r
-}
-
-; CHECK-LABEL: test_copysign(
-; CHECK-DAG:  ld.param.b16    [[AH:%h[0-9]+]], [test_copysign_param_0];
-; CHECK-DAG:  ld.param.b16    [[BH:%h[0-9]+]], [test_copysign_param_1];
-; CHECK-DAG:  mov.b16         [[AS:%rs[0-9]+]], [[AH]];
-; CHECK-DAG:  mov.b16         [[BS:%rs[0-9]+]], [[BH]];
-; CHECK-DAG:  and.b16         [[AX:%rs[0-9]+]], [[AS]], 32767;
-; CHECK-DAG:  and.b16         [[BX:%rs[0-9]+]], [[BS]], -32768;
-; CHECK:      or.b16          [[RX:%rs[0-9]+]], [[AX]], [[BX]];
-; CHECK:      mov.b16         [[R:%h[0-9]+]], [[RX]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define half @test_copysign(half %a, half %b) #0 {
-  %r = call half @llvm.copysign.f16(half %a, half %b)
-  ret half %r
-}
-
-; CHECK-LABEL: test_copysign_f32(
-; CHECK-DAG:  ld.param.b16    [[AH:%h[0-9]+]], [test_copysign_f32_param_0];
-; CHECK-DAG:  ld.param.f32    [[BF:%f[0-9]+]], [test_copysign_f32_param_1];
-; CHECK-DAG:  mov.b16         [[A:%rs[0-9]+]], [[AH]];
-; CHECK-DAG:  mov.b32         [[B:%r[0-9]+]], [[BF]];
-; CHECK-DAG:  and.b16         [[AX:%rs[0-9]+]], [[A]], 32767;
-; CHECK-DAG:  and.b32         [[BX0:%r[0-9]+]], [[B]], -2147483648;
-; CHECK-DAG:  shr.u32         [[BX1:%r[0-9]+]], [[BX0]], 16;
-; CHECK-DAG:  cvt.u16.u32     [[BX2:%rs[0-9]+]], [[BX1]];
-; CHECK:      or.b16          [[RX:%rs[0-9]+]], [[AX]], [[BX2]];
-; CHECK:      mov.b16         [[R:%h[0-9]+]], [[RX]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define half @test_copysign_f32(half %a, float %b) #0 {
-  %tb = fptrunc float %b to half
-  %r = call half @llvm.copysign.f16(half %a, half %tb)
-  ret half %r
-}
-
-; CHECK-LABEL: test_copysign_f64(
-; CHECK-DAG:  ld.param.b16    [[AH:%h[0-9]+]], [test_copysign_f64_param_0];
-; CHECK-DAG:  ld.param.f64    [[BD:%fd[0-9]+]], [test_copysign_f64_param_1];
-; CHECK-DAG:  mov.b16         [[A:%rs[0-9]+]], [[AH]];
-; CHECK-DAG:  mov.b64         [[B:%rd[0-9]+]], [[BD]];
-; CHECK-DAG:  and.b16         [[AX:%rs[0-9]+]], [[A]], 32767;
-; CHECK-DAG:  and.b64         [[BX0:%rd[0-9]+]], [[B]], -9223372036854775808;
-; CHECK-DAG:  shr.u64         [[BX1:%rd[0-9]+]], [[BX0]], 48;
-; CHECK-DAG:  cvt.u16.u64     [[BX2:%rs[0-9]+]], [[BX1]];
-; CHECK:      or.b16          [[RX:%rs[0-9]+]], [[AX]], [[BX2]];
-; CHECK:      mov.b16         [[R:%h[0-9]+]], [[RX]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define half @test_copysign_f64(half %a, double %b) #0 {
-  %tb = fptrunc double %b to half
-  %r = call half @llvm.copysign.f16(half %a, half %tb)
-  ret half %r
-}
-
-; CHECK-LABEL: test_copysign_extended(
-; CHECK-DAG:  ld.param.b16    [[AH:%h[0-9]+]], [test_copysign_extended_param_0];
-; CHECK-DAG:  ld.param.b16    [[BH:%h[0-9]+]], [test_copysign_extended_param_1];
-; CHECK-DAG:  mov.b16         [[AS:%rs[0-9]+]], [[AH]];
-; CHECK-DAG:  mov.b16         [[BS:%rs[0-9]+]], [[BH]];
-; CHECK-DAG:  and.b16         [[AX:%rs[0-9]+]], [[AS]], 32767;
-; CHECK-DAG:  and.b16         [[BX:%rs[0-9]+]], [[BS]], -32768;
-; CHECK:      or.b16          [[RX:%rs[0-9]+]], [[AX]], [[BX]];
-; CHECK:      mov.b16         [[R:%h[0-9]+]], [[RX]];
-; CHECK-NOFTZ: cvt.f32.f16     [[XR:%f[0-9]+]], [[R]];
-; CHECK-F16-FTZ:   cvt.ftz.f32.f16 [[XR:%f[0-9]+]], [[R]];
-; CHECK:      st.param.f32    [func_retval0+0], [[XR]];
-; CHECK:      ret;
-define float @test_copysign_extended(half %a, half %b) #0 {
-  %r = call half @llvm.copysign.f16(half %a, half %b)
-  %xr = fpext half %r to float
-  ret float %xr
-}
-
-; CHECK-LABEL: test_floor(
-; CHECK:      ld.param.b16    [[A:%h[0-9]+]], [test_floor_param_0];
-; CHECK:      cvt.rmi.f16.f16 [[R:%h[0-9]+]], [[A]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define half @test_floor(half %a) #0 {
-  %r = call half @llvm.floor.f16(half %a)
-  ret half %r
-}
-
-; CHECK-LABEL: test_ceil(
-; CHECK:      ld.param.b16    [[A:%h[0-9]+]], [test_ceil_param_0];
-; CHECK:      cvt.rpi.f16.f16 [[R:%h[0-9]+]], [[A]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define half @test_ceil(half %a) #0 {
-  %r = call half @llvm.ceil.f16(half %a)
-  ret half %r
-}
-
-; CHECK-LABEL: test_trunc(
-; CHECK:      ld.param.b16    [[A:%h[0-9]+]], [test_trunc_param_0];
-; CHECK:      cvt.rzi.f16.f16 [[R:%h[0-9]+]], [[A]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define half @test_trunc(half %a) #0 {
-  %r = call half @llvm.trunc.f16(half %a)
-  ret half %r
-}
-
-; CHECK-LABEL: test_rint(
-; CHECK:      ld.param.b16    [[A:%h[0-9]+]], [test_rint_param_0];
-; CHECK:      cvt.rni.f16.f16 [[R:%h[0-9]+]], [[A]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define half @test_rint(half %a) #0 {
-  %r = call half @llvm.rint.f16(half %a)
-  ret half %r
-}
-
-; CHECK-LABEL: test_nearbyint(
-; CHECK:      ld.param.b16    [[A:%h[0-9]+]], [test_nearbyint_param_0];
-; CHECK:      cvt.rni.f16.f16 [[R:%h[0-9]+]], [[A]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define half @test_nearbyint(half %a) #0 {
-  %r = call half @llvm.nearbyint.f16(half %a)
-  ret half %r
-}
-
-; CHECK-LABEL: test_round(
-; CHECK:      ld.param.b16    {{.*}}, [test_round_param_0];
-; check the use of sign mask and 0.5 to implement round
-; CHECK:      and.b32 [[R:%r[0-9]+]], {{.*}}, -2147483648;
-; CHECK:      or.b32 {{.*}}, [[R]], 1056964608;
-; CHECK:      st.param.b16    [func_retval0+0], {{.*}};
-; CHECK:      ret;
-define half @test_round(half %a) #0 {
-  %r = call half @llvm.round.f16(half %a)
-  ret half %r
-}
-
-; CHECK-LABEL: test_fmuladd(
-; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fmuladd_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fmuladd_param_1];
-; CHECK-DAG:  ld.param.b16    [[C:%h[0-9]+]], [test_fmuladd_param_2];
-; CHECK-F16-NOFTZ:        fma.rn.f16     [[R:%h[0-9]+]], [[A]], [[B]], [[C]];
-; CHECK-F16-FTZ:        fma.rn.ftz.f16     [[R:%h[0-9]+]], [[A]], [[B]], [[C]];
-; CHECK-NOF16-DAG:  cvt.f32.f16    [[A32:%f[0-9]+]], [[A]]
-; CHECK-NOF16-DAG:  cvt.f32.f16    [[B32:%f[0-9]+]], [[B]]
-; CHECK-NOF16-DAG:  cvt.f32.f16    [[C32:%f[0-9]+]], [[C]]
-; CHECK-NOF16-NEXT: fma.rn.f32     [[R32:%f[0-9]+]], [[A32]], [[B32]], [[C32]];
-; CHECK-NOF16-NEXT: cvt.rn.f16.f32 [[R:%h[0-9]+]], [[R32]]
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
-define half @test_fmuladd(half %a, half %b, half %c) #0 {
-  %r = call half @llvm.fmuladd.f16(half %a, half %b, half %c)
-  ret half %r
-}
-
-attributes #0 = { nounwind }
-attributes #1 = { "unsafe-fp-math" = "true" }
+; ## Full FP16 support enabled by default.
+; RUN: llc < %s -mtriple=nvptx64-nvidia-cuda -mcpu=sm_53 -asm-verbose=false \
+; RUN:          -O0 -disable-post-ra -frame-pointer=all -verify-machineinstrs \
+; RUN: | FileCheck -check-prefixes CHECK,CHECK-NOFTZ,CHECK-F16,CHECK-F16-NOFTZ %s
+; ## Full FP16 with FTZ
+; RUN: llc < %s -mtriple=nvptx64-nvidia-cuda -mcpu=sm_53 -asm-verbose=false \
+; RUN:          -O0 -disable-post-ra -frame-pointer=all -verify-machineinstrs \
+; RUN:          -denormal-fp-math-f32=preserve-sign \
+; RUN: | FileCheck -check-prefixes CHECK,CHECK-F16,CHECK-F16-FTZ %s
+; ## FP16 support explicitly disabled.
+; RUN: llc < %s -mtriple=nvptx64-nvidia-cuda -mcpu=sm_53 -asm-verbose=false \
+; RUN:          -O0 -disable-post-ra -frame-pointer=all --nvptx-no-f16-math \
+; RUN:           -verify-machineinstrs \
+; RUN: | FileCheck -check-prefixes CHECK,CHECK-NOFTZ,CHECK-NOF16 %s
+; ## FP16 is not supported by hardware.
+; RUN: llc < %s -O0 -mtriple=nvptx64-nvidia-cuda -mcpu=sm_52 -asm-verbose=false \
+; RUN:          -disable-post-ra -frame-pointer=all -verify-machineinstrs \
+; RUN: | FileCheck -check-prefixes CHECK,CHECK-NOFTZ,CHECK-NOF16 %s
+
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+
+; CHECK-LABEL: test_ret_const(
+; CHECK:      mov.b16         [[R:%h[0-9]+]], 0x3C00;
+; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define half @test_ret_const() #0 {
+  ret half 1.0
+}
+
+; CHECK-LABEL: test_fadd(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fadd_param_0];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fadd_param_1];
+; CHECK-F16-NOFTZ-NEXT:   add.rn.f16     [[R:%h[0-9]+]], [[A]], [[B]];
+; CHECK-F16-FTZ-NEXT:   add.rn.ftz.f16     [[R:%h[0-9]+]], [[A]], [[B]];
+; CHECK-NOF16-DAG:  cvt.f32.f16    [[A32:%f[0-9]+]], [[A]]
+; CHECK-NOF16-DAG:  cvt.f32.f16    [[B32:%f[0-9]+]], [[B]]
+; CHECK-NOF16-NEXT: add.rn.f32     [[R32:%f[0-9]+]], [[A32]], [[B32]];
+; CHECK-NOF16-NEXT: cvt.rn.f16.f32 [[R:%h[0-9]+]], [[R32]]
+; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define half @test_fadd(half %a, half %b) #0 {
+  %r = fadd half %a, %b
+  ret half %r
+}
+
+; CHECK-LABEL: test_fadd_v1f16(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fadd_v1f16_param_0];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fadd_v1f16_param_1];
+; CHECK-F16-NOFTZ-NEXT:   add.rn.f16     [[R:%h[0-9]+]], [[A]], [[B]];
+; CHECK-F16-FTZ-NEXT:   add.rn.ftz.f16     [[R:%h[0-9]+]], [[A]], [[B]];
+; CHECK-NOF16-DAG:  cvt.f32.f16    [[A32:%f[0-9]+]], [[A]]
+; CHECK-NOF16-DAG:  cvt.f32.f16    [[B32:%f[0-9]+]], [[B]]
+; CHECK-NOF16-NEXT: add.rn.f32     [[R32:%f[0-9]+]], [[A32]], [[B32]];
+; CHECK-NOF16-NEXT: cvt.rn.f16.f32 [[R:%h[0-9]+]], [[R32]]
+; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define <1 x half> @test_fadd_v1f16(<1 x half> %a, <1 x half> %b) #0 {
+  %r = fadd <1 x half> %a, %b
+  ret <1 x half> %r
+}
+
+; Check that we can lower fadd with immediate arguments.
+; CHECK-LABEL: test_fadd_imm_0(
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fadd_imm_0_param_0];
+; CHECK-F16-NOFTZ-DAG:    mov.b16        [[A:%h[0-9]+]], 0x3C00;
+; CHECK-F16-NOFTZ-NEXT:   add.rn.f16     [[R:%h[0-9]+]], [[B]], [[A]];
+; CHECK-F16-FTZ-DAG:    mov.b16        [[A:%h[0-9]+]], 0x3C00;
+; CHECK-F16-FTZ-NEXT:   add.rn.ftz.f16     [[R:%h[0-9]+]], [[B]], [[A]];
+; CHECK-NOF16-DAG:  cvt.f32.f16    [[B32:%f[0-9]+]], [[B]]
+; CHECK-NOF16-NEXT: add.rn.f32     [[R32:%f[0-9]+]], [[B32]], 0f3F800000;
+; CHECK-NOF16-NEXT: cvt.rn.f16.f32 [[R:%h[0-9]+]], [[R32]]
+; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define half @test_fadd_imm_0(half %b) #0 {
+  %r = fadd half 1.0, %b
+  ret half %r
+}
+
+; CHECK-LABEL: test_fadd_imm_1(
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fadd_imm_1_param_0];
+; CHECK-F16-NOFTZ-DAG:    mov.b16        [[A:%h[0-9]+]], 0x3C00;
+; CHECK-F16-NOFTZ-NEXT:   add.rn.f16     [[R:%h[0-9]+]], [[B]], [[A]];
+; CHECK-F16-FTZ-DAG:    mov.b16        [[A:%h[0-9]+]], 0x3C00;
+; CHECK-F16-FTZ-NEXT:   add.rn.ftz.f16     [[R:%h[0-9]+]], [[B]], [[A]];
+; CHECK-NOF16-DAG:  cvt.f32.f16    [[B32:%f[0-9]+]], [[B]]
+; CHECK-NOF16-NEXT: add.rn.f32     [[R32:%f[0-9]+]], [[B32]], 0f3F800000;
+; CHECK-NOF16-NEXT: cvt.rn.f16.f32 [[R:%h[0-9]+]], [[R32]]
+; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define half @test_fadd_imm_1(half %a) #0 {
+  %r = fadd half %a, 1.0
+  ret half %r
+}
+
+; CHECK-LABEL: test_fsub(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fsub_param_0];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fsub_param_1];
+; CHECK-F16-NOFTZ-NEXT:   sub.rn.f16     [[R:%h[0-9]+]], [[A]], [[B]];
+; CHECK-F16-FTZ-NEXT:   sub.rn.ftz.f16     [[R:%h[0-9]+]], [[A]], [[B]];
+; CHECK-NOF16-DAG:  cvt.f32.f16    [[A32:%f[0-9]+]], [[A]]
+; CHECK-NOF16-DAG:  cvt.f32.f16    [[B32:%f[0-9]+]], [[B]]
+; CHECK-NOF16-NEXT: sub.rn.f32     [[R32:%f[0-9]+]], [[A32]], [[B32]];
+; CHECK-NOF16-NEXT: cvt.rn.f16.f32 [[R:%h[0-9]+]], [[R32]]
+; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define half @test_fsub(half %a, half %b) #0 {
+  %r = fsub half %a, %b
+  ret half %r
+}
+
+; CHECK-LABEL: test_fneg(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fneg_param_0];
+; CHECK-F16-NOFTZ-NEXT:   mov.b16        [[Z:%h[0-9]+]], 0x0000
+; CHECK-F16-NOFTZ-NEXT:   sub.rn.f16     [[R:%h[0-9]+]], [[Z]], [[A]];
+; CHECK-F16-FTZ-NEXT:   mov.b16        [[Z:%h[0-9]+]], 0x0000
+; CHECK-F16-FTZ-NEXT:   sub.rn.ftz.f16     [[R:%h[0-9]+]], [[Z]], [[A]];
+; CHECK-NOF16-DAG:  cvt.f32.f16    [[A32:%f[0-9]+]], [[A]]
+; CHECK-NOF16-DAG:  mov.f32        [[Z:%f[0-9]+]], 0f00000000;
+; CHECK-NOF16-NEXT: sub.rn.f32     [[R32:%f[0-9]+]], [[Z]], [[A32]];
+; CHECK-NOF16-NEXT: cvt.rn.f16.f32 [[R:%h[0-9]+]], [[R32]]
+; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define half @test_fneg(half %a) #0 {
+  %r = fsub half 0.0, %a
+  ret half %r
+}
+
+; CHECK-LABEL: test_fmul(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fmul_param_0];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fmul_param_1];
+; CHECK-F16-NOFTZ-NEXT: mul.rn.f16      [[R:%h[0-9]+]], [[A]], [[B]];
+; CHECK-F16-FTZ-NEXT: mul.rn.ftz.f16      [[R:%h[0-9]+]], [[A]], [[B]];
+; CHECK-NOF16-DAG:  cvt.f32.f16    [[A32:%f[0-9]+]], [[A]]
+; CHECK-NOF16-DAG:  cvt.f32.f16    [[B32:%f[0-9]+]], [[B]]
+; CHECK-NOF16-NEXT: mul.rn.f32     [[R32:%f[0-9]+]], [[A32]], [[B32]];
+; CHECK-NOF16-NEXT: cvt.rn.f16.f32 [[R:%h[0-9]+]], [[R32]]
+; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define half @test_fmul(half %a, half %b) #0 {
+  %r = fmul half %a, %b
+  ret half %r
+}
+
+; CHECK-LABEL: test_fdiv(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fdiv_param_0];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fdiv_param_1];
+; CHECK-NOFTZ-DAG:  cvt.f32.f16     [[F0:%f[0-9]+]], [[A]];
+; CHECK-NOFTZ-DAG:  cvt.f32.f16     [[F1:%f[0-9]+]], [[B]];
+; CHECK-NOFTZ-NEXT: div.rn.f32      [[FR:%f[0-9]+]], [[F0]], [[F1]];
+; CHECK-F16-FTZ-DAG:  cvt.ftz.f32.f16     [[F0:%f[0-9]+]], [[A]];
+; CHECK-F16-FTZ-DAG:  cvt.ftz.f32.f16     [[F1:%f[0-9]+]], [[B]];
+; CHECK-F16-FTZ-NEXT: div.rn.ftz.f32      [[FR:%f[0-9]+]], [[F0]], [[F1]];
+; CHECK-NEXT: cvt.rn.f16.f32  [[R:%h[0-9]+]], [[FR]];
+; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define half @test_fdiv(half %a, half %b) #0 {
+  %r = fdiv half %a, %b
+  ret half %r
+}
+
+; CHECK-LABEL: test_frem(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_frem_param_0];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_frem_param_1];
+; CHECK-NOFTZ-DAG:  cvt.f32.f16     [[FA:%f[0-9]+]], [[A]];
+; CHECK-NOFTZ-DAG:  cvt.f32.f16     [[FB:%f[0-9]+]], [[B]];
+; CHECK-NOFTZ-NEXT: div.rn.f32      [[D:%f[0-9]+]], [[FA]], [[FB]];
+; CHECK-NOFTZ-NEXT: cvt.rmi.f32.f32 [[DI:%f[0-9]+]], [[D]];
+; CHECK-NOFTZ-NEXT: mul.f32         [[RI:%f[0-9]+]], [[DI]], [[FB]];
+; CHECK-NOFTZ-NEXT: sub.f32         [[RF:%f[0-9]+]], [[FA]], [[RI]];
+; CHECK-F16-FTZ-DAG:  cvt.ftz.f32.f16     [[FA:%f[0-9]+]], [[A]];
+; CHECK-F16-FTZ-DAG:  cvt.ftz.f32.f16     [[FB:%f[0-9]+]], [[B]];
+; CHECK-F16-FTZ-NEXT: div.rn.ftz.f32      [[D:%f[0-9]+]], [[FA]], [[FB]];
+; CHECK-F16-FTZ-NEXT: cvt.rmi.ftz.f32.f32 [[DI:%f[0-9]+]], [[D]];
+; CHECK-F16-FTZ-NEXT: mul.ftz.f32         [[RI:%f[0-9]+]], [[DI]], [[FB]];
+; CHECK-F16-FTZ-NEXT: sub.ftz.f32         [[RF:%f[0-9]+]], [[FA]], [[RI]];
+; CHECK-NEXT: cvt.rn.f16.f32  [[R:%h[0-9]+]], [[RF]];
+; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define half @test_frem(half %a, half %b) #0 {
+  %r = frem half %a, %b
+  ret half %r
+}
+
+; CHECK-LABEL: test_store(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_store_param_0];
+; CHECK-DAG:  ld.param.u64    %[[PTR:rd[0-9]+]], [test_store_param_1];
+; CHECK-NEXT: st.b16          [%[[PTR]]], [[A]];
+; CHECK-NEXT: ret;
+define void @test_store(half %a, half* %b) #0 {
+  store half %a, half* %b
+  ret void
+}
+
+; CHECK-LABEL: test_load(
+; CHECK:      ld.param.u64    %[[PTR:rd[0-9]+]], [test_load_param_0];
+; CHECK-NEXT: ld.b16          [[R:%h[0-9]+]], [%[[PTR]]];
+; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define half @test_load(half* %a) #0 {
+  %r = load half, half* %a
+  ret half %r
+}
+
+; CHECK-LABEL: .visible .func test_halfp0a1(
+; CHECK-DAG: ld.param.u64 %[[FROM:rd?[0-9]+]], [test_halfp0a1_param_0];
+; CHECK-DAG: ld.param.u64 %[[TO:rd?[0-9]+]], [test_halfp0a1_param_1];
+; CHECK-DAG: ld.u8        [[B0:%r[sd]?[0-9]+]], [%[[FROM]]]
+; CHECK-DAG: st.u8        [%[[TO]]], [[B0]]
+; CHECK-DAG: ld.u8        [[B1:%r[sd]?[0-9]+]], [%[[FROM]]+1]
+; CHECK-DAG: st.u8        [%[[TO]]+1], [[B1]]
+; CHECK: ret
+define void @test_halfp0a1(half * noalias readonly %from, half * %to) {
+  %1 = load half, half * %from , align 1
+  store half %1, half * %to , align 1
+  ret void
+}
+
+declare half @test_callee(half %a, half %b) #0
+
+; CHECK-LABEL: test_call(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_call_param_0];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_call_param_1];
+; CHECK:      {
+; CHECK-DAG:  .param .b32 param0;
+; CHECK-DAG:  .param .b32 param1;
+; CHECK-DAG:  st.param.b16    [param0+0], [[A]];
+; CHECK-DAG:  st.param.b16    [param1+0], [[B]];
+; CHECK-DAG:  .param .b32 retval0;
+; CHECK:      call.uni (retval0),
+; CHECK-NEXT:        test_callee,
+; CHECK:      );
+; CHECK-NEXT: ld.param.b16    [[R:%h[0-9]+]], [retval0+0];
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define half @test_call(half %a, half %b) #0 {
+  %r = call half @test_callee(half %a, half %b)
+  ret half %r
+}
+
+; CHECK-LABEL: test_call_flipped(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_call_flipped_param_0];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_call_flipped_param_1];
+; CHECK:      {
+; CHECK-DAG:  .param .b32 param0;
+; CHECK-DAG:  .param .b32 param1;
+; CHECK-DAG:  st.param.b16    [param0+0], [[B]];
+; CHECK-DAG:  st.param.b16    [param1+0], [[A]];
+; CHECK-DAG:  .param .b32 retval0;
+; CHECK:      call.uni (retval0),
+; CHECK-NEXT:        test_callee,
+; CHECK:      );
+; CHECK-NEXT: ld.param.b16    [[R:%h[0-9]+]], [retval0+0];
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define half @test_call_flipped(half %a, half %b) #0 {
+  %r = call half @test_callee(half %b, half %a)
+  ret half %r
+}
+
+; CHECK-LABEL: test_tailcall_flipped(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_tailcall_flipped_param_0];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_tailcall_flipped_param_1];
+; CHECK:      {
+; CHECK-DAG:  .param .b32 param0;
+; CHECK-DAG:  .param .b32 param1;
+; CHECK-DAG:  st.param.b16    [param0+0], [[B]];
+; CHECK-DAG:  st.param.b16    [param1+0], [[A]];
+; CHECK-DAG:  .param .b32 retval0;
+; CHECK:      call.uni (retval0),
+; CHECK-NEXT:        test_callee,
+; CHECK:      );
+; CHECK-NEXT: ld.param.b16    [[R:%h[0-9]+]], [retval0+0];
+; CHECK-NEXT: }
+; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define half @test_tailcall_flipped(half %a, half %b) #0 {
+  %r = tail call half @test_callee(half %b, half %a)
+  ret half %r
+}
+
+; CHECK-LABEL: test_select(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_select_param_0];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_select_param_1];
+; CHECK-DAG:  setp.eq.b16     [[PRED:%p[0-9]+]], %rs{{.*}}, 1;
+; CHECK-NEXT: selp.b16        [[R:%h[0-9]+]], [[A]], [[B]], [[PRED]];
+; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define half @test_select(half %a, half %b, i1 zeroext %c) #0 {
+  %r = select i1 %c, half %a, half %b
+  ret half %r
+}
+
+; CHECK-LABEL: test_select_cc(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_select_cc_param_0];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_select_cc_param_1];
+; CHECK-DAG:  ld.param.b16    [[C:%h[0-9]+]], [test_select_cc_param_2];
+; CHECK-DAG:  ld.param.b16    [[D:%h[0-9]+]], [test_select_cc_param_3];
+; CHECK-F16-NOFTZ:  setp.neu.f16    [[PRED:%p[0-9]+]], [[C]], [[D]]
+; CHECK-NOF16-DAG: cvt.f32.f16 [[DF:%f[0-9]+]], [[D]];
+; CHECK-NOF16-DAG: cvt.f32.f16 [[CF:%f[0-9]+]], [[C]];
+; CHECK-NOF16: setp.neu.f32    [[PRED:%p[0-9]+]], [[CF]], [[DF]]
+; CHECK:      selp.b16        [[R:%h[0-9]+]], [[A]], [[B]], [[PRED]];
+; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define half @test_select_cc(half %a, half %b, half %c, half %d) #0 {
+  %cc = fcmp une half %c, %d
+  %r = select i1 %cc, half %a, half %b
+  ret half %r
+}
+
+; CHECK-LABEL: test_select_cc_f32_f16(
+; CHECK-DAG:  ld.param.f32    [[A:%f[0-9]+]], [test_select_cc_f32_f16_param_0];
+; CHECK-DAG:  ld.param.f32    [[B:%f[0-9]+]], [test_select_cc_f32_f16_param_1];
+; CHECK-DAG:  ld.param.b16    [[C:%h[0-9]+]], [test_select_cc_f32_f16_param_2];
+; CHECK-DAG:  ld.param.b16    [[D:%h[0-9]+]], [test_select_cc_f32_f16_param_3];
+; CHECK-F16-NOFTZ:  setp.neu.f16    [[PRED:%p[0-9]+]], [[C]], [[D]]
+; CHECK-F16-FTZ:  setp.neu.ftz.f16    [[PRED:%p[0-9]+]], [[C]], [[D]]
+; CHECK-NOF16-DAG: cvt.f32.f16 [[DF:%f[0-9]+]], [[D]];
+; CHECK-NOF16-DAG: cvt.f32.f16 [[CF:%f[0-9]+]], [[C]];
+; CHECK-NOF16: setp.neu.f32    [[PRED:%p[0-9]+]], [[CF]], [[DF]]
+; CHECK-NEXT: selp.f32        [[R:%f[0-9]+]], [[A]], [[B]], [[PRED]];
+; CHECK-NEXT: st.param.f32    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define float @test_select_cc_f32_f16(float %a, float %b, half %c, half %d) #0 {
+  %cc = fcmp une half %c, %d
+  %r = select i1 %cc, float %a, float %b
+  ret float %r
+}
+
+; CHECK-LABEL: test_select_cc_f16_f32(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_select_cc_f16_f32_param_0];
+; CHECK-DAG:  ld.param.f32    [[C:%f[0-9]+]], [test_select_cc_f16_f32_param_2];
+; CHECK-DAG:  ld.param.f32    [[D:%f[0-9]+]], [test_select_cc_f16_f32_param_3];
+; CHECK-NOFTZ-DAG:  setp.neu.f32    [[PRED:%p[0-9]+]], [[C]], [[D]]
+; CHECK-F16-FTZ-DAG:  setp.neu.ftz.f32    [[PRED:%p[0-9]+]], [[C]], [[D]]
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_select_cc_f16_f32_param_1];
+; CHECK-NEXT: selp.b16        [[R:%h[0-9]+]], [[A]], [[B]], [[PRED]];
+; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define half @test_select_cc_f16_f32(half %a, half %b, float %c, float %d) #0 {
+  %cc = fcmp une float %c, %d
+  %r = select i1 %cc, half %a, half %b
+  ret half %r
+}
+
+; CHECK-LABEL: test_fcmp_une(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fcmp_une_param_0];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fcmp_une_param_1];
+; CHECK-F16-NOFTZ:  setp.neu.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
+; CHECK-F16-FTZ:  setp.neu.ftz.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
+; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%f[0-9]+]], [[A]];
+; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%f[0-9]+]], [[B]];
+; CHECK-NOF16: setp.neu.f32   [[PRED:%p[0-9]+]], [[AF]], [[BF]]
+; CHECK-NEXT: selp.u32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
+; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define i1 @test_fcmp_une(half %a, half %b) #0 {
+  %r = fcmp une half %a, %b
+  ret i1 %r
+}
+
+; CHECK-LABEL: test_fcmp_ueq(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fcmp_ueq_param_0];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fcmp_ueq_param_1];
+; CHECK-F16-NOFTZ:  setp.equ.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
+; CHECK-F16-FTZ:  setp.equ.ftz.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
+; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%f[0-9]+]], [[A]];
+; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%f[0-9]+]], [[B]];
+; CHECK-NOF16: setp.equ.f32   [[PRED:%p[0-9]+]], [[AF]], [[BF]]
+; CHECK-NEXT: selp.u32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
+; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define i1 @test_fcmp_ueq(half %a, half %b) #0 {
+  %r = fcmp ueq half %a, %b
+  ret i1 %r
+}
+
+; CHECK-LABEL: test_fcmp_ugt(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fcmp_ugt_param_0];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fcmp_ugt_param_1];
+; CHECK-F16-NOFTZ:  setp.gtu.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
+; CHECK-F16-FTZ:  setp.gtu.ftz.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
+; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%f[0-9]+]], [[A]];
+; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%f[0-9]+]], [[B]];
+; CHECK-NOF16: setp.gtu.f32   [[PRED:%p[0-9]+]], [[AF]], [[BF]]
+; CHECK-NEXT: selp.u32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
+; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define i1 @test_fcmp_ugt(half %a, half %b) #0 {
+  %r = fcmp ugt half %a, %b
+  ret i1 %r
+}
+
+; CHECK-LABEL: test_fcmp_uge(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fcmp_uge_param_0];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fcmp_uge_param_1];
+; CHECK-F16-NOFTZ:  setp.geu.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
+; CHECK-F16-FTZ:  setp.geu.ftz.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
+; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%f[0-9]+]], [[A]];
+; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%f[0-9]+]], [[B]];
+; CHECK-NOF16: setp.geu.f32   [[PRED:%p[0-9]+]], [[AF]], [[BF]]
+; CHECK-NEXT: selp.u32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
+; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define i1 @test_fcmp_uge(half %a, half %b) #0 {
+  %r = fcmp uge half %a, %b
+  ret i1 %r
+}
+
+; CHECK-LABEL: test_fcmp_ult(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fcmp_ult_param_0];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fcmp_ult_param_1];
+; CHECK-F16-NOFTZ:  setp.ltu.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
+; CHECK-F16-FTZ:  setp.ltu.ftz.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
+; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%f[0-9]+]], [[A]];
+; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%f[0-9]+]], [[B]];
+; CHECK-NOF16: setp.ltu.f32   [[PRED:%p[0-9]+]], [[AF]], [[BF]]
+; CHECK-NEXT: selp.u32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
+; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define i1 @test_fcmp_ult(half %a, half %b) #0 {
+  %r = fcmp ult half %a, %b
+  ret i1 %r
+}
+
+; CHECK-LABEL: test_fcmp_ule(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fcmp_ule_param_0];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fcmp_ule_param_1];
+; CHECK-F16-NOFTZ:  setp.leu.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
+; CHECK-F16-FTZ:  setp.leu.ftz.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
+; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%f[0-9]+]], [[A]];
+; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%f[0-9]+]], [[B]];
+; CHECK-NOF16: setp.leu.f32   [[PRED:%p[0-9]+]], [[AF]], [[BF]]
+; CHECK-NEXT: selp.u32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
+; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define i1 @test_fcmp_ule(half %a, half %b) #0 {
+  %r = fcmp ule half %a, %b
+  ret i1 %r
+}
+
+
+; CHECK-LABEL: test_fcmp_uno(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fcmp_uno_param_0];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fcmp_uno_param_1];
+; CHECK-F16-NOFTZ:  setp.nan.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
+; CHECK-F16-FTZ:  setp.nan.ftz.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
+; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%f[0-9]+]], [[A]];
+; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%f[0-9]+]], [[B]];
+; CHECK-NOF16: setp.nan.f32   [[PRED:%p[0-9]+]], [[AF]], [[BF]]
+; CHECK-NEXT: selp.u32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
+; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define i1 @test_fcmp_uno(half %a, half %b) #0 {
+  %r = fcmp uno half %a, %b
+  ret i1 %r
+}
+
+; CHECK-LABEL: test_fcmp_one(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fcmp_one_param_0];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fcmp_one_param_1];
+; CHECK-F16-NOFTZ:  setp.ne.f16     [[PRED:%p[0-9]+]], [[A]], [[B]]
+; CHECK-F16-FTZ:  setp.ne.ftz.f16     [[PRED:%p[0-9]+]], [[A]], [[B]]
+; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%f[0-9]+]], [[A]];
+; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%f[0-9]+]], [[B]];
+; CHECK-NOF16: setp.ne.f32    [[PRED:%p[0-9]+]], [[AF]], [[BF]]
+; CHECK-NEXT: selp.u32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
+; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define i1 @test_fcmp_one(half %a, half %b) #0 {
+  %r = fcmp one half %a, %b
+  ret i1 %r
+}
+
+; CHECK-LABEL: test_fcmp_oeq(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fcmp_oeq_param_0];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fcmp_oeq_param_1];
+; CHECK-F16-NOFTZ:  setp.eq.f16     [[PRED:%p[0-9]+]], [[A]], [[B]]
+; CHECK-F16-FTZ:  setp.eq.ftz.f16     [[PRED:%p[0-9]+]], [[A]], [[B]]
+; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%f[0-9]+]], [[A]];
+; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%f[0-9]+]], [[B]];
+; CHECK-NOF16: setp.eq.f32    [[PRED:%p[0-9]+]], [[AF]], [[BF]]
+; CHECK-NEXT: selp.u32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
+; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define i1 @test_fcmp_oeq(half %a, half %b) #0 {
+  %r = fcmp oeq half %a, %b
+  ret i1 %r
+}
+
+; CHECK-LABEL: test_fcmp_ogt(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fcmp_ogt_param_0];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fcmp_ogt_param_1];
+; CHECK-F16-NOFTZ:  setp.gt.f16     [[PRED:%p[0-9]+]], [[A]], [[B]]
+; CHECK-F16-FTZ:  setp.gt.ftz.f16     [[PRED:%p[0-9]+]], [[A]], [[B]]
+; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%f[0-9]+]], [[A]];
+; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%f[0-9]+]], [[B]];
+; CHECK-NOF16: setp.gt.f32    [[PRED:%p[0-9]+]], [[AF]], [[BF]]
+; CHECK-NEXT: selp.u32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
+; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define i1 @test_fcmp_ogt(half %a, half %b) #0 {
+  %r = fcmp ogt half %a, %b
+  ret i1 %r
+}
+
+; CHECK-LABEL: test_fcmp_oge(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fcmp_oge_param_0];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fcmp_oge_param_1];
+; CHECK-F16-NOFTZ:  setp.ge.f16     [[PRED:%p[0-9]+]], [[A]], [[B]]
+; CHECK-F16-FTZ:  setp.ge.ftz.f16     [[PRED:%p[0-9]+]], [[A]], [[B]]
+; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%f[0-9]+]], [[A]];
+; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%f[0-9]+]], [[B]];
+; CHECK-NOF16: setp.ge.f32    [[PRED:%p[0-9]+]], [[AF]], [[BF]]
+; CHECK-NEXT: selp.u32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
+; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define i1 @test_fcmp_oge(half %a, half %b) #0 {
+  %r = fcmp oge half %a, %b
+  ret i1 %r
+}
+
+; XCHECK-LABEL: test_fcmp_olt(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fcmp_olt_param_0];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fcmp_olt_param_1];
+; CHECK-F16-NOFTZ:  setp.lt.f16     [[PRED:%p[0-9]+]], [[A]], [[B]]
+; CHECK-F16-FTZ:  setp.lt.ftz.f16     [[PRED:%p[0-9]+]], [[A]], [[B]]
+; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%f[0-9]+]], [[A]];
+; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%f[0-9]+]], [[B]];
+; CHECK-NOF16: setp.lt.f32    [[PRED:%p[0-9]+]], [[AF]], [[BF]]
+; CHECK-NEXT: selp.u32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
+; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define i1 @test_fcmp_olt(half %a, half %b) #0 {
+  %r = fcmp olt half %a, %b
+  ret i1 %r
+}
+
+; XCHECK-LABEL: test_fcmp_ole(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fcmp_ole_param_0];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fcmp_ole_param_1];
+; CHECK-F16-NOFTZ:  setp.le.f16     [[PRED:%p[0-9]+]], [[A]], [[B]]
+; CHECK-F16-FTZ:  setp.le.ftz.f16     [[PRED:%p[0-9]+]], [[A]], [[B]]
+; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%f[0-9]+]], [[A]];
+; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%f[0-9]+]], [[B]];
+; CHECK-NOF16: setp.le.f32    [[PRED:%p[0-9]+]], [[AF]], [[BF]]
+; CHECK-NEXT: selp.u32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
+; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define i1 @test_fcmp_ole(half %a, half %b) #0 {
+  %r = fcmp ole half %a, %b
+  ret i1 %r
+}
+
+; CHECK-LABEL: test_fcmp_ord(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fcmp_ord_param_0];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fcmp_ord_param_1];
+; CHECK-F16-NOFTZ:  setp.num.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
+; CHECK-F16-FTZ:  setp.num.ftz.f16    [[PRED:%p[0-9]+]], [[A]], [[B]]
+; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%f[0-9]+]], [[A]];
+; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%f[0-9]+]], [[B]];
+; CHECK-NOF16: setp.num.f32   [[PRED:%p[0-9]+]], [[AF]], [[BF]]
+; CHECK-NEXT: selp.u32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
+; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
+; CHECK-NEXT: ret;
+define i1 @test_fcmp_ord(half %a, half %b) #0 {
+  %r = fcmp ord half %a, %b
+  ret i1 %r
+}
+
+; CHECK-LABEL: test_br_cc(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_br_cc_param_0];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_br_cc_param_1];
+; CHECK-DAG:  ld.param.u64    %[[C:rd[0-9]+]], [test_br_cc_param_2];
+; CHECK-DAG:  ld.param.u64    %[[D:rd[0-9]+]], [test_br_cc_param_3];
+; CHECK-F16-NOFTZ:  setp.lt.f16     [[PRED:%p[0-9]+]], [[A]], [[B]]
+; CHECK-F16-FTZ:  setp.lt.ftz.f16     [[PRED:%p[0-9]+]], [[A]], [[B]]
+; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%f[0-9]+]], [[A]];
+; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%f[0-9]+]], [[B]];
+; CHECK-NOF16: setp.lt.f32    [[PRED:%p[0-9]+]], [[AF]], [[BF]]
+; CHECK-NEXT: @[[PRED]] bra   [[LABEL:LBB.*]];
+; CHECK:      st.u32  [%[[C]]],
+; CHECK:      [[LABEL]]:
+; CHECK:      st.u32  [%[[D]]],
+; CHECK:      ret;
+define void @test_br_cc(half %a, half %b, i32* %p1, i32* %p2) #0 {
+  %c = fcmp uge half %a, %b
+  br i1 %c, label %then, label %else
+then:
+  store i32 0, i32* %p1
+  ret void
+else:
+  store i32 0, i32* %p2
+  ret void
+}
+
+; CHECK-LABEL: test_phi(
+; CHECK:      ld.param.u64    %[[P1:rd[0-9]+]], [test_phi_param_0];
+; CHECK:      ld.b16  {{%h[0-9]+}}, [%[[P1]]];
+; CHECK: [[LOOP:LBB[0-9_]+]]:
+; CHECK:      mov.b16 [[R:%h[0-9]+]], [[AB:%h[0-9]+]];
+; CHECK:      ld.b16  [[AB:%h[0-9]+]], [%[[P1]]];
+; CHECK:      {
+; CHECK:      st.param.b64    [param0+0], %[[P1]];
+; CHECK:      call.uni (retval0),
+; CHECK-NEXT: test_dummy
+; CHECK:      }
+; CHECK:      setp.eq.b32     [[PRED:%p[0-9]+]], %r{{[0-9]+}}, 1;
+; CHECK:      @[[PRED]] bra   [[LOOP]];
+; CHECK:      st.param.b16    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define half @test_phi(half* %p1) #0 {
+entry:
+  %a = load half, half* %p1
+  br label %loop
+loop:
+  %r = phi half [%a, %entry], [%b, %loop]
+  %b = load half, half* %p1
+  %c = call i1 @test_dummy(half* %p1)
+  br i1 %c, label %loop, label %return
+return:
+  ret half %r
+}
+declare i1 @test_dummy(half* %p1) #0
+
+; CHECK-LABEL: test_fptosi_i32(
+; CHECK:      ld.param.b16    [[A:%h[0-9]+]], [test_fptosi_i32_param_0];
+; CHECK:      cvt.rzi.s32.f16 [[R:%r[0-9]+]], [[A]];
+; CHECK:      st.param.b32    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define i32 @test_fptosi_i32(half %a) #0 {
+  %r = fptosi half %a to i32
+  ret i32 %r
+}
+
+; CHECK-LABEL: test_fptosi_i64(
+; CHECK:      ld.param.b16    [[A:%h[0-9]+]], [test_fptosi_i64_param_0];
+; CHECK:      cvt.rzi.s64.f16 [[R:%rd[0-9]+]], [[A]];
+; CHECK:      st.param.b64    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define i64 @test_fptosi_i64(half %a) #0 {
+  %r = fptosi half %a to i64
+  ret i64 %r
+}
+
+; CHECK-LABEL: test_fptoui_i32(
+; CHECK:      ld.param.b16    [[A:%h[0-9]+]], [test_fptoui_i32_param_0];
+; CHECK:      cvt.rzi.u32.f16 [[R:%r[0-9]+]], [[A]];
+; CHECK:      st.param.b32    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define i32 @test_fptoui_i32(half %a) #0 {
+  %r = fptoui half %a to i32
+  ret i32 %r
+}
+
+; CHECK-LABEL: test_fptoui_i64(
+; CHECK:      ld.param.b16    [[A:%h[0-9]+]], [test_fptoui_i64_param_0];
+; CHECK:      cvt.rzi.u64.f16 [[R:%rd[0-9]+]], [[A]];
+; CHECK:      st.param.b64    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define i64 @test_fptoui_i64(half %a) #0 {
+  %r = fptoui half %a to i64
+  ret i64 %r
+}
+
+; CHECK-LABEL: test_uitofp_i32(
+; CHECK:      ld.param.u32    [[A:%r[0-9]+]], [test_uitofp_i32_param_0];
+; CHECK:      cvt.rn.f16.u32  [[R:%h[0-9]+]], [[A]];
+; CHECK:      st.param.b16    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define half @test_uitofp_i32(i32 %a) #0 {
+  %r = uitofp i32 %a to half
+  ret half %r
+}
+
+; CHECK-LABEL: test_uitofp_i64(
+; CHECK:      ld.param.u64    [[A:%rd[0-9]+]], [test_uitofp_i64_param_0];
+; CHECK:      cvt.rn.f16.u64  [[R:%h[0-9]+]], [[A]];
+; CHECK:      st.param.b16    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define half @test_uitofp_i64(i64 %a) #0 {
+  %r = uitofp i64 %a to half
+  ret half %r
+}
+
+; CHECK-LABEL: test_sitofp_i32(
+; CHECK:      ld.param.u32    [[A:%r[0-9]+]], [test_sitofp_i32_param_0];
+; CHECK:      cvt.rn.f16.s32  [[R:%h[0-9]+]], [[A]];
+; CHECK:      st.param.b16    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define half @test_sitofp_i32(i32 %a) #0 {
+  %r = sitofp i32 %a to half
+  ret half %r
+}
+
+; CHECK-LABEL: test_sitofp_i64(
+; CHECK:      ld.param.u64    [[A:%rd[0-9]+]], [test_sitofp_i64_param_0];
+; CHECK:      cvt.rn.f16.s64  [[R:%h[0-9]+]], [[A]];
+; CHECK:      st.param.b16    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define half @test_sitofp_i64(i64 %a) #0 {
+  %r = sitofp i64 %a to half
+  ret half %r
+}
+
+; CHECK-LABEL: test_uitofp_i32_fadd(
+; CHECK-DAG:  ld.param.u32    [[A:%r[0-9]+]], [test_uitofp_i32_fadd_param_0];
+; CHECK-DAG:  cvt.rn.f16.u32  [[C:%h[0-9]+]], [[A]];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_uitofp_i32_fadd_param_1];
+; CHECK-F16-NOFTZ:       add.rn.f16      [[R:%h[0-9]+]], [[B]], [[C]];
+; CHECK-F16-FTZ:       add.rn.ftz.f16      [[R:%h[0-9]+]], [[B]], [[C]];
+; CHECK-NOF16-DAG:  cvt.f32.f16    [[B32:%f[0-9]+]], [[B]]
+; CHECK-NOF16-DAG:  cvt.f32.f16    [[C32:%f[0-9]+]], [[C]]
+; CHECK-NOF16-NEXT: add.rn.f32     [[R32:%f[0-9]+]], [[B32]], [[C32]];
+; CHECK-NOF16-NEXT: cvt.rn.f16.f32 [[R:%h[0-9]+]], [[R32]]
+; CHECK:      st.param.b16    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define half @test_uitofp_i32_fadd(i32 %a, half %b) #0 {
+  %c = uitofp i32 %a to half
+  %r = fadd half %b, %c
+  ret half %r
+}
+
+; CHECK-LABEL: test_sitofp_i32_fadd(
+; CHECK-DAG:  ld.param.u32    [[A:%r[0-9]+]], [test_sitofp_i32_fadd_param_0];
+; CHECK-DAG:  cvt.rn.f16.s32  [[C:%h[0-9]+]], [[A]];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_sitofp_i32_fadd_param_1];
+; CHECK-F16-NOFTZ:         add.rn.f16     [[R:%h[0-9]+]], [[B]], [[C]];
+; CHECK-F16-FTZ:         add.rn.ftz.f16     [[R:%h[0-9]+]], [[B]], [[C]];
+; XCHECK-NOF16-DAG:  cvt.f32.f16    [[B32:%f[0-9]+]], [[B]]
+; XCHECK-NOF16-DAG:  cvt.f32.f16    [[C32:%f[0-9]+]], [[C]]
+; XCHECK-NOF16-NEXT: add.rn.f32     [[R32:%f[0-9]+]], [[B32]], [[C32]];
+; XCHECK-NOF16-NEXT: cvt.rn.f16.f32 [[R:%h[0-9]+]], [[R32]]
+; CHECK:      st.param.b16    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define half @test_sitofp_i32_fadd(i32 %a, half %b) #0 {
+  %c = sitofp i32 %a to half
+  %r = fadd half %b, %c
+  ret half %r
+}
+
+; CHECK-LABEL: test_fptrunc_float(
+; CHECK:      ld.param.f32    [[A:%f[0-9]+]], [test_fptrunc_float_param_0];
+; CHECK:      cvt.rn.f16.f32  [[R:%h[0-9]+]], [[A]];
+; CHECK:      st.param.b16    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define half @test_fptrunc_float(float %a) #0 {
+  %r = fptrunc float %a to half
+  ret half %r
+}
+
+; CHECK-LABEL: test_fptrunc_double(
+; CHECK:      ld.param.f64    [[A:%fd[0-9]+]], [test_fptrunc_double_param_0];
+; CHECK:      cvt.rn.f16.f64  [[R:%h[0-9]+]], [[A]];
+; CHECK:      st.param.b16    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define half @test_fptrunc_double(double %a) #0 {
+  %r = fptrunc double %a to half
+  ret half %r
+}
+
+; CHECK-LABEL: test_fpext_float(
+; CHECK:      ld.param.b16    [[A:%h[0-9]+]], [test_fpext_float_param_0];
+; CHECK-NOFTZ:      cvt.f32.f16     [[R:%f[0-9]+]], [[A]];
+; CHECK-F16-FTZ:      cvt.ftz.f32.f16     [[R:%f[0-9]+]], [[A]];
+; CHECK:      st.param.f32    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define float @test_fpext_float(half %a) #0 {
+  %r = fpext half %a to float
+  ret float %r
+}
+
+; CHECK-LABEL: test_fpext_double(
+; CHECK:      ld.param.b16    [[A:%h[0-9]+]], [test_fpext_double_param_0];
+; CHECK:      cvt.f64.f16     [[R:%fd[0-9]+]], [[A]];
+; CHECK:      st.param.f64    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define double @test_fpext_double(half %a) #0 {
+  %r = fpext half %a to double
+  ret double %r
+}
+
+
+; CHECK-LABEL: test_bitcast_halftoi16(
+; CHECK:      ld.param.b16    [[AH:%h[0-9]+]], [test_bitcast_halftoi16_param_0];
+; CHECK:      mov.b16         [[AS:%rs[0-9]+]], [[AH]]
+; CHECK:      cvt.u32.u16     [[R:%r[0-9]+]], [[AS]]
+; CHECK:      st.param.b32    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define i16 @test_bitcast_halftoi16(half %a) #0 {
+  %r = bitcast half %a to i16
+  ret i16 %r
+}
+
+; CHECK-LABEL: test_bitcast_i16tohalf(
+; CHECK:      ld.param.u16    [[AS:%rs[0-9]+]], [test_bitcast_i16tohalf_param_0];
+; CHECK:      mov.b16         [[AH:%h[0-9]+]], [[AS]]
+; CHECK:      st.param.b16    [func_retval0+0], [[AH]];
+; CHECK:      ret;
+define half @test_bitcast_i16tohalf(i16 %a) #0 {
+  %r = bitcast i16 %a to half
+  ret half %r
+}
+
+
+declare half @llvm.sqrt.f16(half %a) #0
+declare half @llvm.powi.f16(half %a, i32 %b) #0
+declare half @llvm.sin.f16(half %a) #0
+declare half @llvm.cos.f16(half %a) #0
+declare half @llvm.pow.f16(half %a, half %b) #0
+declare half @llvm.exp.f16(half %a) #0
+declare half @llvm.exp2.f16(half %a) #0
+declare half @llvm.log.f16(half %a) #0
+declare half @llvm.log10.f16(half %a) #0
+declare half @llvm.log2.f16(half %a) #0
+declare half @llvm.fma.f16(half %a, half %b, half %c) #0
+declare half @llvm.fabs.f16(half %a) #0
+declare half @llvm.minnum.f16(half %a, half %b) #0
+declare half @llvm.maxnum.f16(half %a, half %b) #0
+declare half @llvm.copysign.f16(half %a, half %b) #0
+declare half @llvm.floor.f16(half %a) #0
+declare half @llvm.ceil.f16(half %a) #0
+declare half @llvm.trunc.f16(half %a) #0
+declare half @llvm.rint.f16(half %a) #0
+declare half @llvm.nearbyint.f16(half %a) #0
+declare half @llvm.round.f16(half %a) #0
+declare half @llvm.fmuladd.f16(half %a, half %b, half %c) #0
+
+; CHECK-LABEL: test_sqrt(
+; CHECK:      ld.param.b16    [[A:%h[0-9]+]], [test_sqrt_param_0];
+; CHECK-NOFTZ:      cvt.f32.f16     [[AF:%f[0-9]+]], [[A]];
+; CHECK-NOFTZ:      sqrt.rn.f32     [[RF:%f[0-9]+]], [[AF]];
+; CHECK-F16-FTZ:      cvt.ftz.f32.f16     [[AF:%f[0-9]+]], [[A]];
+; CHECK-F16-FTZ:      sqrt.rn.ftz.f32     [[RF:%f[0-9]+]], [[AF]];
+; CHECK:      cvt.rn.f16.f32  [[R:%h[0-9]+]], [[RF]];
+; CHECK:      st.param.b16    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define half @test_sqrt(half %a) #0 {
+  %r = call half @llvm.sqrt.f16(half %a)
+  ret half %r
+}
+
+;;; Can't do this yet: requires libcall.
+; XCHECK-LABEL: test_powi(
+;define half @test_powi(half %a, i32 %b) #0 {
+;  %r = call half @llvm.powi.f16(half %a, i32 %b)
+;  ret half %r
+;}
+
+; CHECK-LABEL: test_sin(
+; CHECK:      ld.param.b16    [[A:%h[0-9]+]], [test_sin_param_0];
+; CHECK-NOFTZ:      cvt.f32.f16     [[AF:%f[0-9]+]], [[A]];
+; CHECK-F16-FTZ:      cvt.ftz.f32.f16     [[AF:%f[0-9]+]], [[A]];
+; CHECK:      sin.approx.f32  [[RF:%f[0-9]+]], [[AF]];
+; CHECK:      cvt.rn.f16.f32  [[R:%h[0-9]+]], [[RF]];
+; CHECK:      st.param.b16    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define half @test_sin(half %a) #0 #1 {
+  %r = call half @llvm.sin.f16(half %a)
+  ret half %r
+}
+
+; CHECK-LABEL: test_cos(
+; CHECK:      ld.param.b16    [[A:%h[0-9]+]], [test_cos_param_0];
+; CHECK-NOFTZ:      cvt.f32.f16     [[AF:%f[0-9]+]], [[A]];
+; CHECK-F16-FTZ:      cvt.ftz.f32.f16     [[AF:%f[0-9]+]], [[A]];
+; CHECK:      cos.approx.f32  [[RF:%f[0-9]+]], [[AF]];
+; CHECK:      cvt.rn.f16.f32  [[R:%h[0-9]+]], [[RF]];
+; CHECK:      st.param.b16    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define half @test_cos(half %a) #0 #1 {
+  %r = call half @llvm.cos.f16(half %a)
+  ret half %r
+}
+
+;;; Can't do this yet: requires libcall.
+; XCHECK-LABEL: test_pow(
+;define half @test_pow(half %a, half %b) #0 {
+;  %r = call half @llvm.pow.f16(half %a, half %b)
+;  ret half %r
+;}
+
+;;; Can't do this yet: requires libcall.
+; XCHECK-LABEL: test_exp(
+;define half @test_exp(half %a) #0 {
+;  %r = call half @llvm.exp.f16(half %a)
+;  ret half %r
+;}
+
+;;; Can't do this yet: requires libcall.
+; XCHECK-LABEL: test_exp2(
+;define half @test_exp2(half %a) #0 {
+;  %r = call half @llvm.exp2.f16(half %a)
+;  ret half %r
+;}
+
+;;; Can't do this yet: requires libcall.
+; XCHECK-LABEL: test_log(
+;define half @test_log(half %a) #0 {
+;  %r = call half @llvm.log.f16(half %a)
+;  ret half %r
+;}
+
+;;; Can't do this yet: requires libcall.
+; XCHECK-LABEL: test_log10(
+;define half @test_log10(half %a) #0 {
+;  %r = call half @llvm.log10.f16(half %a)
+;  ret half %r
+;}
+
+;;; Can't do this yet: requires libcall.
+; XCHECK-LABEL: test_log2(
+;define half @test_log2(half %a) #0 {
+;  %r = call half @llvm.log2.f16(half %a)
+;  ret half %r
+;}
+
+; CHECK-LABEL: test_fma(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fma_param_0];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fma_param_1];
+; CHECK-DAG:  ld.param.b16    [[C:%h[0-9]+]], [test_fma_param_2];
+; CHECK-F16-NOFTZ:      fma.rn.f16      [[R:%h[0-9]+]], [[A]], [[B]], [[C]];
+; CHECK-F16-FTZ:      fma.rn.ftz.f16      [[R:%h[0-9]+]], [[A]], [[B]], [[C]];
+; CHECK-NOF16-DAG:  cvt.f32.f16    [[A32:%f[0-9]+]], [[A]]
+; CHECK-NOF16-DAG:  cvt.f32.f16    [[B32:%f[0-9]+]], [[B]]
+; CHECK-NOF16-DAG:  cvt.f32.f16    [[C32:%f[0-9]+]], [[C]]
+; CHECK-NOF16-NEXT: fma.rn.f32     [[R32:%f[0-9]+]], [[A32]], [[B32]], [[C32]];
+; CHECK-NOF16-NEXT: cvt.rn.f16.f32 [[R:%h[0-9]+]], [[R32]]
+; CHECK:      st.param.b16    [func_retval0+0], [[R]];
+; CHECK:      ret
+define half @test_fma(half %a, half %b, half %c) #0 {
+  %r = call half @llvm.fma.f16(half %a, half %b, half %c)
+  ret half %r
+}
+
+; CHECK-LABEL: test_fabs(
+; CHECK:      ld.param.b16    [[A:%h[0-9]+]], [test_fabs_param_0];
+; CHECK-NOFTZ:      cvt.f32.f16     [[AF:%f[0-9]+]], [[A]];
+; CHECK-NOFTZ:      abs.f32         [[RF:%f[0-9]+]], [[AF]];
+; CHECK-F16-FTZ:      cvt.ftz.f32.f16     [[AF:%f[0-9]+]], [[A]];
+; CHECK-F16-FTZ:      abs.ftz.f32         [[RF:%f[0-9]+]], [[AF]];
+; CHECK:      cvt.rn.f16.f32  [[R:%h[0-9]+]], [[RF]];
+; CHECK:      st.param.b16    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define half @test_fabs(half %a) #0 {
+  %r = call half @llvm.fabs.f16(half %a)
+  ret half %r
+}
+
+; CHECK-LABEL: test_minnum(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_minnum_param_0];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_minnum_param_1];
+; CHECK-NOFTZ-DAG:  cvt.f32.f16     [[AF:%f[0-9]+]], [[A]];
+; CHECK-NOFTZ-DAG:  cvt.f32.f16     [[BF:%f[0-9]+]], [[B]];
+; CHECK-NOFTZ:      min.f32         [[RF:%f[0-9]+]], [[AF]], [[BF]];
+; CHECK-F16-FTZ-DAG:  cvt.ftz.f32.f16     [[AF:%f[0-9]+]], [[A]];
+; CHECK-F16-FTZ-DAG:  cvt.ftz.f32.f16     [[BF:%f[0-9]+]], [[B]];
+; CHECK-F16-FTZ:      min.ftz.f32         [[RF:%f[0-9]+]], [[AF]], [[BF]];
+; CHECK:      cvt.rn.f16.f32  [[R:%h[0-9]+]], [[RF]];
+; CHECK:      st.param.b16    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define half @test_minnum(half %a, half %b) #0 {
+  %r = call half @llvm.minnum.f16(half %a, half %b)
+  ret half %r
+}
+
+; CHECK-LABEL: test_maxnum(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_maxnum_param_0];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_maxnum_param_1];
+; CHECK-NOFTZ-DAG:  cvt.f32.f16     [[AF:%f[0-9]+]], [[A]];
+; CHECK-NOFTZ-DAG:  cvt.f32.f16     [[BF:%f[0-9]+]], [[B]];
+; CHECK-NOFTZ:      max.f32         [[RF:%f[0-9]+]], [[AF]], [[BF]];
+; CHECK-F16-FTZ-DAG:  cvt.ftz.f32.f16     [[AF:%f[0-9]+]], [[A]];
+; CHECK-F16-FTZ-DAG:  cvt.ftz.f32.f16     [[BF:%f[0-9]+]], [[B]];
+; CHECK-F16-FTZ:      max.ftz.f32         [[RF:%f[0-9]+]], [[AF]], [[BF]];
+; CHECK:      cvt.rn.f16.f32  [[R:%h[0-9]+]], [[RF]];
+; CHECK:      st.param.b16    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define half @test_maxnum(half %a, half %b) #0 {
+  %r = call half @llvm.maxnum.f16(half %a, half %b)
+  ret half %r
+}
+
+; CHECK-LABEL: test_copysign(
+; CHECK-DAG:  ld.param.b16    [[AH:%h[0-9]+]], [test_copysign_param_0];
+; CHECK-DAG:  ld.param.b16    [[BH:%h[0-9]+]], [test_copysign_param_1];
+; CHECK-DAG:  mov.b16         [[AS:%rs[0-9]+]], [[AH]];
+; CHECK-DAG:  mov.b16         [[BS:%rs[0-9]+]], [[BH]];
+; CHECK-DAG:  and.b16         [[AX:%rs[0-9]+]], [[AS]], 32767;
+; CHECK-DAG:  and.b16         [[BX:%rs[0-9]+]], [[BS]], -32768;
+; CHECK:      or.b16          [[RX:%rs[0-9]+]], [[AX]], [[BX]];
+; CHECK:      mov.b16         [[R:%h[0-9]+]], [[RX]];
+; CHECK:      st.param.b16    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define half @test_copysign(half %a, half %b) #0 {
+  %r = call half @llvm.copysign.f16(half %a, half %b)
+  ret half %r
+}
+
+; CHECK-LABEL: test_copysign_f32(
+; CHECK-DAG:  ld.param.b16    [[AH:%h[0-9]+]], [test_copysign_f32_param_0];
+; CHECK-DAG:  ld.param.f32    [[BF:%f[0-9]+]], [test_copysign_f32_param_1];
+; CHECK-DAG:  mov.b16         [[A:%rs[0-9]+]], [[AH]];
+; CHECK-DAG:  mov.b32         [[B:%r[0-9]+]], [[BF]];
+; CHECK-DAG:  and.b16         [[AX:%rs[0-9]+]], [[A]], 32767;
+; CHECK-DAG:  and.b32         [[BX0:%r[0-9]+]], [[B]], -2147483648;
+; CHECK-DAG:  shr.u32         [[BX1:%r[0-9]+]], [[BX0]], 16;
+; CHECK-DAG:  cvt.u16.u32     [[BX2:%rs[0-9]+]], [[BX1]];
+; CHECK:      or.b16          [[RX:%rs[0-9]+]], [[AX]], [[BX2]];
+; CHECK:      mov.b16         [[R:%h[0-9]+]], [[RX]];
+; CHECK:      st.param.b16    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define half @test_copysign_f32(half %a, float %b) #0 {
+  %tb = fptrunc float %b to half
+  %r = call half @llvm.copysign.f16(half %a, half %tb)
+  ret half %r
+}
+
+; CHECK-LABEL: test_copysign_f64(
+; CHECK-DAG:  ld.param.b16    [[AH:%h[0-9]+]], [test_copysign_f64_param_0];
+; CHECK-DAG:  ld.param.f64    [[BD:%fd[0-9]+]], [test_copysign_f64_param_1];
+; CHECK-DAG:  mov.b16         [[A:%rs[0-9]+]], [[AH]];
+; CHECK-DAG:  mov.b64         [[B:%rd[0-9]+]], [[BD]];
+; CHECK-DAG:  and.b16         [[AX:%rs[0-9]+]], [[A]], 32767;
+; CHECK-DAG:  and.b64         [[BX0:%rd[0-9]+]], [[B]], -9223372036854775808;
+; CHECK-DAG:  shr.u64         [[BX1:%rd[0-9]+]], [[BX0]], 48;
+; CHECK-DAG:  cvt.u16.u64     [[BX2:%rs[0-9]+]], [[BX1]];
+; CHECK:      or.b16          [[RX:%rs[0-9]+]], [[AX]], [[BX2]];
+; CHECK:      mov.b16         [[R:%h[0-9]+]], [[RX]];
+; CHECK:      st.param.b16    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define half @test_copysign_f64(half %a, double %b) #0 {
+  %tb = fptrunc double %b to half
+  %r = call half @llvm.copysign.f16(half %a, half %tb)
+  ret half %r
+}
+
+; CHECK-LABEL: test_copysign_extended(
+; CHECK-DAG:  ld.param.b16    [[AH:%h[0-9]+]], [test_copysign_extended_param_0];
+; CHECK-DAG:  ld.param.b16    [[BH:%h[0-9]+]], [test_copysign_extended_param_1];
+; CHECK-DAG:  mov.b16         [[AS:%rs[0-9]+]], [[AH]];
+; CHECK-DAG:  mov.b16         [[BS:%rs[0-9]+]], [[BH]];
+; CHECK-DAG:  and.b16         [[AX:%rs[0-9]+]], [[AS]], 32767;
+; CHECK-DAG:  and.b16         [[BX:%rs[0-9]+]], [[BS]], -32768;
+; CHECK:      or.b16          [[RX:%rs[0-9]+]], [[AX]], [[BX]];
+; CHECK:      mov.b16         [[R:%h[0-9]+]], [[RX]];
+; CHECK-NOFTZ: cvt.f32.f16     [[XR:%f[0-9]+]], [[R]];
+; CHECK-F16-FTZ:   cvt.ftz.f32.f16 [[XR:%f[0-9]+]], [[R]];
+; CHECK:      st.param.f32    [func_retval0+0], [[XR]];
+; CHECK:      ret;
+define float @test_copysign_extended(half %a, half %b) #0 {
+  %r = call half @llvm.copysign.f16(half %a, half %b)
+  %xr = fpext half %r to float
+  ret float %xr
+}
+
+; CHECK-LABEL: test_floor(
+; CHECK:      ld.param.b16    [[A:%h[0-9]+]], [test_floor_param_0];
+; CHECK:      cvt.rmi.f16.f16 [[R:%h[0-9]+]], [[A]];
+; CHECK:      st.param.b16    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define half @test_floor(half %a) #0 {
+  %r = call half @llvm.floor.f16(half %a)
+  ret half %r
+}
+
+; CHECK-LABEL: test_ceil(
+; CHECK:      ld.param.b16    [[A:%h[0-9]+]], [test_ceil_param_0];
+; CHECK:      cvt.rpi.f16.f16 [[R:%h[0-9]+]], [[A]];
+; CHECK:      st.param.b16    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define half @test_ceil(half %a) #0 {
+  %r = call half @llvm.ceil.f16(half %a)
+  ret half %r
+}
+
+; CHECK-LABEL: test_trunc(
+; CHECK:      ld.param.b16    [[A:%h[0-9]+]], [test_trunc_param_0];
+; CHECK:      cvt.rzi.f16.f16 [[R:%h[0-9]+]], [[A]];
+; CHECK:      st.param.b16    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define half @test_trunc(half %a) #0 {
+  %r = call half @llvm.trunc.f16(half %a)
+  ret half %r
+}
+
+; CHECK-LABEL: test_rint(
+; CHECK:      ld.param.b16    [[A:%h[0-9]+]], [test_rint_param_0];
+; CHECK:      cvt.rni.f16.f16 [[R:%h[0-9]+]], [[A]];
+; CHECK:      st.param.b16    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define half @test_rint(half %a) #0 {
+  %r = call half @llvm.rint.f16(half %a)
+  ret half %r
+}
+
+; CHECK-LABEL: test_nearbyint(
+; CHECK:      ld.param.b16    [[A:%h[0-9]+]], [test_nearbyint_param_0];
+; CHECK:      cvt.rni.f16.f16 [[R:%h[0-9]+]], [[A]];
+; CHECK:      st.param.b16    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define half @test_nearbyint(half %a) #0 {
+  %r = call half @llvm.nearbyint.f16(half %a)
+  ret half %r
+}
+
+; CHECK-LABEL: test_round(
+; CHECK:      ld.param.b16    {{.*}}, [test_round_param_0];
+; check the use of sign mask and 0.5 to implement round
+; CHECK:      and.b32 [[R:%r[0-9]+]], {{.*}}, -2147483648;
+; CHECK:      or.b32 {{.*}}, [[R]], 1056964608;
+; CHECK:      st.param.b16    [func_retval0+0], {{.*}};
+; CHECK:      ret;
+define half @test_round(half %a) #0 {
+  %r = call half @llvm.round.f16(half %a)
+  ret half %r
+}
+
+; CHECK-LABEL: test_fmuladd(
+; CHECK-DAG:  ld.param.b16    [[A:%h[0-9]+]], [test_fmuladd_param_0];
+; CHECK-DAG:  ld.param.b16    [[B:%h[0-9]+]], [test_fmuladd_param_1];
+; CHECK-DAG:  ld.param.b16    [[C:%h[0-9]+]], [test_fmuladd_param_2];
+; CHECK-F16-NOFTZ:        fma.rn.f16     [[R:%h[0-9]+]], [[A]], [[B]], [[C]];
+; CHECK-F16-FTZ:        fma.rn.ftz.f16     [[R:%h[0-9]+]], [[A]], [[B]], [[C]];
+; CHECK-NOF16-DAG:  cvt.f32.f16    [[A32:%f[0-9]+]], [[A]]
+; CHECK-NOF16-DAG:  cvt.f32.f16    [[B32:%f[0-9]+]], [[B]]
+; CHECK-NOF16-DAG:  cvt.f32.f16    [[C32:%f[0-9]+]], [[C]]
+; CHECK-NOF16-NEXT: fma.rn.f32     [[R32:%f[0-9]+]], [[A32]], [[B32]], [[C32]];
+; CHECK-NOF16-NEXT: cvt.rn.f16.f32 [[R:%h[0-9]+]], [[R32]]
+; CHECK:      st.param.b16    [func_retval0+0], [[R]];
+; CHECK:      ret;
+define half @test_fmuladd(half %a, half %b, half %c) #0 {
+  %r = call half @llvm.fmuladd.f16(half %a, half %b, half %c)
+  ret half %r
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { "unsafe-fp-math" = "true" }
diff --git a/llvm/test/CodeGen/NVPTX/fast-math.ll b/llvm/test/CodeGen/NVPTX/fast-math.ll
index 900521664e0c9..db5fb63f4e766 100644
--- a/llvm/test/CodeGen/NVPTX/fast-math.ll
+++ b/llvm/test/CodeGen/NVPTX/fast-math.ll
@@ -13,7 +13,7 @@ define float @sqrt_div(float %a, float %b) {
 }
 
 ; CHECK-LABEL: sqrt_div_fast(
-; CHECK: sqrt.approx.f32
+; CHECK: sqrt.rn.f32
 ; CHECK: div.approx.f32
 define float @sqrt_div_fast(float %a, float %b) #0 {
   %t1 = tail call float @llvm.sqrt.f32(float %a)
@@ -21,6 +21,15 @@ define float @sqrt_div_fast(float %a, float %b) #0 {
   ret float %t2
 }
 
+; CHECK-LABEL: sqrt_div_fast_ninf(
+; CHECK: sqrt.approx.f32
+; CHECK: div.approx.f32
+define float @sqrt_div_fast_ninf(float %a, float %b) #0 {
+  %t1 = tail call ninf float @llvm.sqrt.f32(float %a)
+  %t2 = fdiv float %t1, %b
+  ret float %t2
+}
+
 ; CHECK-LABEL: sqrt_div_ftz(
 ; CHECK: sqrt.rn.ftz.f32
 ; CHECK: div.rn.ftz.f32
@@ -31,7 +40,7 @@ define float @sqrt_div_ftz(float %a, float %b) #1 {
 }
 
 ; CHECK-LABEL: sqrt_div_fast_ftz(
-; CHECK: sqrt.approx.ftz.f32
+; CHECK: sqrt.rn.ftz.f32
 ; CHECK: div.approx.ftz.f32
 define float @sqrt_div_fast_ftz(float %a, float %b) #0 #1 {
   %t1 = tail call float @llvm.sqrt.f32(float %a)
@@ -39,12 +48,20 @@ define float @sqrt_div_fast_ftz(float %a, float %b) #0 #1 {
   ret float %t2
 }
 
+; CHECK-LABEL: sqrt_div_fast_ftz_ninf(
+; CHECK: sqrt.approx.ftz.f32
+; CHECK: div.approx.ftz.f32
+define float @sqrt_div_fast_ftz_ninf(float %a, float %b) #0 #1 {
+  %t1 = tail call ninf float @llvm.sqrt.f32(float %a)
+  %t2 = fdiv float %t1, %b
+  ret float %t2
+}
+
 ; There are no fast-math or ftz versions of sqrt and div for f64.  We use
 ; reciprocal(rsqrt(x)) for sqrt(x), and emit a vanilla divide.
 
 ; CHECK-LABEL: sqrt_div_fast_ftz_f64(
-; CHECK: rsqrt.approx.f64
-; CHECK: rcp.approx.ftz.f64
+; CHECK: sqrt.rn.f64
 ; CHECK: div.rn.f64
 define double @sqrt_div_fast_ftz_f64(double %a, double %b) #0 #1 {
   %t1 = tail call double @llvm.sqrt.f64(double %a)
@@ -52,6 +69,16 @@ define double @sqrt_div_fast_ftz_f64(double %a, double %b) #0 #1 {
   ret double %t2
 }
 
+; CHECK-LABEL: sqrt_div_fast_ftz_f64_ninf(
+; CHECK: rsqrt.approx.f64
+; CHECK: rcp.approx.ftz.f64
+; CHECK: div.rn.f64
+define double @sqrt_div_fast_ftz_f64_ninf(double %a, double %b) #0 #1 {
+  %t1 = tail call ninf double @llvm.sqrt.f64(double %a)
+  %t2 = fdiv double %t1, %b
+  ret double %t2
+}
+
 ; CHECK-LABEL: rsqrt(
 ; CHECK-NOT: rsqrt.approx
 ; CHECK: sqrt.rn.f32
diff --git a/llvm/test/CodeGen/NVPTX/sqrt-approx.ll b/llvm/test/CodeGen/NVPTX/sqrt-approx.ll
index a8590b7c43ab0..465b696c7610c 100644
--- a/llvm/test/CodeGen/NVPTX/sqrt-approx.ll
+++ b/llvm/test/CodeGen/NVPTX/sqrt-approx.ll
@@ -45,35 +45,63 @@ define double @test_rsqrt64_ftz(double %a) #0 #1 {
 
 ; CHECK-LABEL test_sqrt32
 define float @test_sqrt32(float %a) #0 {
-; CHECK: sqrt.approx.f32
+; CHECK: sqrt.rn.f32
   %ret = tail call float @llvm.sqrt.f32(float %a)
   ret float %ret
 }
 
+; CHECK-LABEL test_sqrt32_ninf
+define float @test_sqrt32_ninf(float %a) #0 {
+; CHECK: sqrt.approx.f32
+  %ret = tail call ninf float @llvm.sqrt.f32(float %a)
+  ret float %ret
+}
+
 ; CHECK-LABEL test_sqrt_ftz
 define float @test_sqrt_ftz(float %a) #0 #1 {
-; CHECK: sqrt.approx.ftz.f32
+; CHECK: sqrt.rn.ftz.f32
   %ret = tail call float @llvm.sqrt.f32(float %a)
   ret float %ret
 }
 
+; CHECK-LABEL test_sqrt_ftz_ninf
+define float @test_sqrt_ftz_ninf(float %a) #0 #1 {
+; CHECK: sqrt.approx.ftz.f32
+  %ret = tail call ninf float @llvm.sqrt.f32(float %a)
+  ret float %ret
+}
+
 ; CHECK-LABEL test_sqrt64
 define double @test_sqrt64(double %a) #0 {
+; CHECK: sqrt.rn.f64
+  %ret = tail call double @llvm.sqrt.f64(double %a)
+  ret double %ret
+}
+
+; CHECK-LABEL test_sqrt64_ninf
+define double @test_sqrt64_ninf(double %a) #0 {
 ; There's no sqrt.approx.f64 instruction; we emit
 ; reciprocal(rsqrt.approx.f64(x)).  There's no non-ftz approximate reciprocal,
 ; so we just use the ftz version.
 ; CHECK: rsqrt.approx.f64
 ; CHECK: rcp.approx.ftz.f64
-  %ret = tail call double @llvm.sqrt.f64(double %a)
+  %ret = tail call ninf double @llvm.sqrt.f64(double %a)
   ret double %ret
 }
 
 ; CHECK-LABEL test_sqrt64_ftz
 define double @test_sqrt64_ftz(double %a) #0 #1 {
+; CHECK: sqrt.rn.f64
+  %ret = tail call double @llvm.sqrt.f64(double %a)
+  ret double %ret
+}
+
+; CHECK-LABEL test_sqrt64_ftz_ninf
+define double @test_sqrt64_ftz_ninf(double %a) #0 #1 {
 ; There's no sqrt.approx.ftz.f64 instruction; we just use the non-ftz version.
 ; CHECK: rsqrt.approx.f64
 ; CHECK: rcp.approx.ftz.f64
-  %ret = tail call double @llvm.sqrt.f64(double %a)
+  %ret = tail call ninf double @llvm.sqrt.f64(double %a)
   ret double %ret
 }
 
@@ -92,11 +120,18 @@ define float @test_rsqrt32_refined(float %a) #0 #2 {
 
 ; CHECK-LABEL: test_sqrt32_refined
 define float @test_sqrt32_refined(float %a) #0 #2 {
-; CHECK: rsqrt.approx.f32
+; CHECK: sqrt.rn.f32
   %ret = tail call float @llvm.sqrt.f32(float %a)
   ret float %ret
 }
 
+; CHECK-LABEL: test_sqrt32_refined_ninf
+define float @test_sqrt32_refined_ninf(float %a) #0 #2 {
+; CHECK: rsqrt.approx.f32
+  %ret = tail call ninf float @llvm.sqrt.f32(float %a)
+  ret float %ret
+}
+
 ; CHECK-LABEL: test_rsqrt64_refined
 define double @test_rsqrt64_refined(double %a) #0 #2 {
 ; CHECK: rsqrt.approx.f64
@@ -107,11 +142,18 @@ define double @test_rsqrt64_refined(double %a) #0 #2 {
 
 ; CHECK-LABEL: test_sqrt64_refined
 define double @test_sqrt64_refined(double %a) #0 #2 {
-; CHECK: rsqrt.approx.f64
+; CHECK: sqrt.rn.f64
   %ret = tail call double @llvm.sqrt.f64(double %a)
   ret double %ret
 }
 
+; CHECK-LABEL: test_sqrt64_refined_ninf
+define double @test_sqrt64_refined_ninf(double %a) #0 #2 {
+; CHECK: rsqrt.approx.f64
+  %ret = tail call ninf double @llvm.sqrt.f64(double %a)
+  ret double %ret
+}
+
 ; -- refined sqrt and rsqrt with ftz enabled --
 
 ; CHECK-LABEL: test_rsqrt32_refined_ftz
@@ -124,11 +166,18 @@ define float @test_rsqrt32_refined_ftz(float %a) #0 #1 #2 {
 
 ; CHECK-LABEL: test_sqrt32_refined_ftz
 define float @test_sqrt32_refined_ftz(float %a) #0 #1 #2 {
-; CHECK: rsqrt.approx.ftz.f32
+; CHECK: sqrt.rn.ftz.f32
   %ret = tail call float @llvm.sqrt.f32(float %a)
   ret float %ret
 }
 
+; CHECK-LABEL: test_sqrt32_refined_ftz_ninf
+define float @test_sqrt32_refined_ftz_ninf(float %a) #0 #1 #2 {
+; CHECK: rsqrt.approx.ftz.f32
+  %ret = tail call ninf float @llvm.sqrt.f32(float %a)
+  ret float %ret
+}
+
 ; CHECK-LABEL: test_rsqrt64_refined_ftz
 define double @test_rsqrt64_refined_ftz(double %a) #0 #1 #2 {
 ; There's no rsqrt.approx.ftz.f64, so we just use the non-ftz version.
@@ -140,11 +189,18 @@ define double @test_rsqrt64_refined_ftz(double %a) #0 #1 #2 {
 
 ; CHECK-LABEL: test_sqrt64_refined_ftz
 define double @test_sqrt64_refined_ftz(double %a) #0 #1 #2 {
-; CHECK: rsqrt.approx.f64
+; CHECK: sqrt.rn.f64
   %ret = tail call double @llvm.sqrt.f64(double %a)
   ret double %ret
 }
 
+; CHECK-LABEL: test_sqrt64_refined_ftz_ninf
+define double @test_sqrt64_refined_ftz_ninf(double %a) #0 #1 #2 {
+; CHECK: rsqrt.approx.f64
+  %ret = tail call ninf double @llvm.sqrt.f64(double %a)
+  ret double %ret
+}
+
 attributes #0 = { "unsafe-fp-math" = "true" }
 attributes #1 = { "denormal-fp-math-f32" = "preserve-sign,preserve-sign" }
 attributes #2 = { "reciprocal-estimates" = "rsqrtf:1,rsqrtd:1,sqrtf:1,sqrtd:1" }
diff --git a/llvm/test/CodeGen/PowerPC/aix-cc-byval.ll b/llvm/test/CodeGen/PowerPC/aix-cc-byval.ll
index 55713dfc70e69..87ae98ad4d189 100644
--- a/llvm/test/CodeGen/PowerPC/aix-cc-byval.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-cc-byval.ll
@@ -22,11 +22,10 @@
 define void @call_test_byval_1Byte() {
 entry:
   %s0 = alloca %struct.S0, align 8
-  call void @test_byval_1Byte(%struct.S0* byval(%struct.S0) align 1 %s0, %struct.S1* byval(%struct.S1) align 1 @gS1)
+  %call = call zeroext i8 @test_byval_1Byte(%struct.S0* byval(%struct.S0) align 1 %s0, %struct.S1* byval(%struct.S1) align 1 @gS1)
   ret void
 }
 
-declare void @test_byval_1Byte(%struct.S0* byval(%struct.S0) align 1, %struct.S1* byval(%struct.S1) align 1)
 
 ; CHECK-LABEL: name: call_test_byval_1Byte{{.*}}
 
@@ -63,18 +62,59 @@ declare void @test_byval_1Byte(%struct.S0* byval(%struct.S0) align 1, %struct.S1
 ; ASM64-NEXT:  nop
 ; ASM64-NEXT:  addi 1, 1, 128
 
+
+define zeroext i8 @test_byval_1Byte(%struct.S0* byval(%struct.S0) align 1 %s0, %struct.S1* byval(%struct.S1) align 1 %s) {
+entry:
+  %arrayidx = getelementptr inbounds %struct.S1, %struct.S1* %s, i32 0, i32 0, i32 0
+  %0 = load i8, i8* %arrayidx, align 1
+  ret i8 %0
+}
+
+; CHECK-LABEL: name:            test_byval_1Byte
+
+; 32BIT:       fixedStack:
+; 32BIT-NEXT:    - { id: 0, type: default, offset: 24, size: 4, alignment: 8, stack-id: default,
+; 32BIT-NEXT:        isImmutable: false, isAliased: true, callee-saved-register: '', callee-saved-restored: true,
+; 32BIT:         - { id: 1, type: default, offset: 24, size: 4, alignment: 8, stack-id: default,
+; 32BIT-NEXT:        isImmutable: false, isAliased: true, callee-saved-register: '', callee-saved-restored: true,
+
+; 32BIT:       bb.0.entry:
+; 32BIT-NEXT:    liveins: $r3
+; 32BIT:         STW killed renamable $r3, 0, %fixed-stack.0 :: (store 4 into %fixed-stack.0, align 8)
+; 32BIT-NEXT:    renamable $r3 = LBZ 0,  %fixed-stack.0 :: (dereferenceable load 1
+; 32BIT-NEXT:    BLR
+
+; 64BIT:       fixedStack:
+; 64BIT-NEXT:    - { id: 0, type: default, offset: 48, size: 8, alignment: 16, stack-id: default,
+; 64BIT-NEXT:        isImmutable: false, isAliased: true, callee-saved-register: '', callee-saved-restored: true,
+; 64BIT:         - { id: 1, type: default, offset: 48, size: 8, alignment: 16, stack-id: default,
+; 64BIT-NEXT:        isImmutable: false, isAliased: true, callee-saved-register: '', callee-saved-restored: true,
+
+; 64BIT:      bb.0.entry:
+; 64BIT-NEXT:   liveins: $x3
+; 64BIT:        STD killed renamable $x3, 0, %fixed-stack.0 :: (store 8 into %fixed-stack.0, align 16)
+; 64BIT-NEXT:   renamable $x3 = LBZ8 0, %fixed-stack.0 :: (dereferenceable load 1
+
+; CHECKASM-LABEL: .test_byval_1Byte:
+
+; ASM32:        stw 3, 24(1)
+; ASM32-NEXT:   lbz 3, 24(1)
+; ASM32-NEXT:   blr
+
+; ASM64:        std 3, 48(1)
+; ASM64-NEXT:   lbz 3, 48(1)
+; ASM64-NEXT:   blr
+
 %struct.S2 = type { [2 x i8] }
 
 @gS2 = external global %struct.S2, align 1
 
 define void @call_test_byval_2Byte() {
 entry:
-  call void @test_byval_2Byte(%struct.S2* byval(%struct.S2) align 1 @gS2)
+  %call = call zeroext i8 @test_byval_2Byte(%struct.S2* byval(%struct.S2) align 1 @gS2)
   ret void
 }
 
-declare void @test_byval_2Byte(%struct.S2* byval(%struct.S2) align 1)
-
 ; CHECK-LABEL: name: call_test_byval_2Byte{{.*}}
 
 ; 32BIT:       ADJCALLSTACKDOWN 56, 0, implicit-def dead $r1, implicit $r1
@@ -110,18 +150,51 @@ declare void @test_byval_2Byte(%struct.S2* byval(%struct.S2) align 1)
 ; ASM64-NEXT:  nop
 ; ASM64-NEXT:  addi 1, 1, 112
 
-%struct.S3 = type <{ i8, i16 }>
 
+define zeroext i8 @test_byval_2Byte(%struct.S2* byval(%struct.S2) align 1 %s) {
+entry:
+  %arrayidx = getelementptr inbounds %struct.S2, %struct.S2* %s, i32 0, i32 0, i32 1
+  %0 = load i8, i8* %arrayidx, align 1
+  ret i8 %0
+}
+
+; CHECK-LABEL: name:            test_byval_2Byte
+; 32BIT:      fixedStack:
+; 32BIT-NEXT:   - { id: 0, type: default, offset: 24, size: 4, alignment: 8, stack-id: default,
+
+; 32BIT:      bb.0.entry:
+; 32BIT-NEXT:   liveins: $r3
+; 32BIT:        STW killed renamable $r3, 0, %fixed-stack.0 :: (store 4 into %fixed-stack.0, align 8)
+; 32BIT-NEXT:   renamable $r3 = LBZ 1, %fixed-stack.0 :: (dereferenceable load 1
+
+; 64BIT:      fixedStack:
+; 64BIT-NEXT:   - { id: 0, type: default, offset: 48, size: 8, alignment: 16, stack-id: default,
+
+; 64BIT:      bb.0.entry:
+; 64BIT-NEXT:   liveins: $x3
+; 64BIT:        STD killed renamable $x3, 0, %fixed-stack.0 :: (store 8 into %fixed-stack.0, align 16)
+; 64BIT-NEXT:   renamable $x3 = LBZ8 1, %fixed-stack.0 :: (dereferenceable load 1
+
+; CHECKASM-LABEL: .test_byval_2Byte:
+
+; ASM32:        stw 3, 24(1)
+; ASM32-NEXT:   lbz 3, 25(1)
+; ASM32-NEXT:   blr
+
+; ASM64:        std 3, 48(1)
+; ASM64-NEXT:   lbz 3, 49(1)
+; ASM64-NEXT:   blr
+
+
+%struct.S3 = type <{ i8, i16 }>
 @gS3 = external global %struct.S3, align 1
 
 define void @call_test_byval_3Byte() {
 entry:
-  call void @test_byval_3Byte(%struct.S3* byval(%struct.S3) align 1 @gS3)
+  %call = call zeroext i16 @test_byval_3Byte(%struct.S3* byval(%struct.S3) align 1 @gS3)
   ret void
 }
 
-declare void @test_byval_3Byte(%struct.S3* byval(%struct.S3) align 1)
-
 ; CHECK-LABEL: name: call_test_byval_3Byte{{.*}}
 
 ; The DAG block permits some invalid inputs for the benefit of allowing more valid orderings.
@@ -166,6 +239,44 @@ declare void @test_byval_3Byte(%struct.S3* byval(%struct.S3) align 1)
 ; ASM64-NEXT:  bl .test_byval_3Byte
 ; ASM64-NEXT:  nop
 
+
+define zeroext i16 @test_byval_3Byte(%struct.S3* byval(%struct.S3) align 1 %s) {
+entry:
+  %gep = getelementptr inbounds %struct.S3, %struct.S3* %s, i32 0, i32 1
+  %0 = load i16, i16* %gep, align 1
+  ret i16 %0
+}
+
+; CHECK-LABEL: name:            test_byval_3Byte
+
+; 32BIT:      fixedStack:
+; 32BIT-NEXT:   - { id: 0, type: default, offset: 24, size: 4, alignment: 8, stack-id: default,
+
+; 32BIT:      bb.0.entry:
+; 32BIT-NEXT:   liveins: $r3
+; 32BIT:        STW killed renamable $r3, 0, %fixed-stack.0 :: (store 4 into %fixed-stack.0, align 8)
+; 32BIT-NEXT:   renamable $r3 = LHZ 1, %fixed-stack.0 :: (dereferenceable load 2
+
+; 64BIT:      fixedStack:
+; 64BIT-NEXT:   - { id: 0, type: default, offset: 48, size: 8, alignment: 16, stack-id: default,
+
+; 64BIT:      bb.0.entry:
+; 64BIT-NEXT:   liveins: $x3
+; 64BIT:        STD killed renamable $x3, 0, %fixed-stack.0 :: (store 8 into %fixed-stack.0, align 16)
+; 64BIT-NEXT:   renamable $x3 = LHZ8 1, %fixed-stack.0 :: (dereferenceable load 2
+
+
+; CHECKASM-LABEL: .test_byval_3Byte:
+
+; ASM32:        stw 3, 24(1)
+; ASM32-NEXT:   lhz 3, 25(1)
+; ASM32-NEXT:   blr
+
+; ASM64:        std 3, 48(1)
+; ASM64-NEXT:   lhz 3, 49(1)
+; ASM64-NEXT:   blr
+
+
 %struct.S4 = type { [4 x i8] }
 %struct.S4A = type { i32 }
 
@@ -175,12 +286,10 @@ define void @call_test_byval_4Byte() {
 entry:
   %s0 = alloca %struct.S0, align 8
   %s4a = alloca %struct.S4A, align 4
-  call void @test_byval_4Byte(%struct.S4* byval(%struct.S4) align 1 @gS4, %struct.S0* byval(%struct.S0) align 1 %s0, %struct.S4A* byval(%struct.S4A) align 4 %s4a)
+  %call = call signext i32 @test_byval_4Byte(%struct.S4* byval(%struct.S4) align 1 @gS4, %struct.S0* byval(%struct.S0) align 1 %s0, %struct.S4A* byval(%struct.S4A) align 4 %s4a)
   ret void
 }
 
-declare void @test_byval_4Byte(%struct.S4* byval(%struct.S4) align 1, %struct.S0* byval(%struct.S0) align 1, %struct.S4A* byval(%struct.S4A) align 4)
-
 ; CHECK-LABEL: name: call_test_byval_4Byte{{.*}}
 
 ; 32BIT:       ADJCALLSTACKDOWN 56, 0, implicit-def dead $r1, implicit $r1
@@ -219,3 +328,66 @@ declare void @test_byval_4Byte(%struct.S4* byval(%struct.S4) align 1, %struct.S0
 ; ASM64-NEXT:  nop
 ; ASM64-NEXT:  addi 1, 1, 128
 
+
+define signext i32 @test_byval_4Byte(%struct.S4* byval(%struct.S4) align 1 %s, %struct.S0* byval(%struct.S0) align 1, %struct.S4A* byval(%struct.S4A) align 4 %s4a) {
+entry:
+  %arrayidx = getelementptr inbounds %struct.S4, %struct.S4* %s, i32 0, i32 0, i32 3
+  %gep = getelementptr inbounds %struct.S4A, %struct.S4A* %s4a, i32 0, i32 0
+  %1 = load i8, i8* %arrayidx, align 1
+  %2 = load i32, i32* %gep, align 4
+  %conv = zext i8 %1 to i32
+  %add = add nsw i32 %2, %conv
+  ret i32 %add
+}
+
+; CHECK-LABEL: name:            test_byval_4Byte
+
+; 32BIT:      fixedStack:
+; 32BIT-NEXT:   - { id: 0, type: default, offset: 28, size: 4, alignment: 4, stack-id: default,
+; 32BIT-NEXT:       isImmutable: false, isAliased: true, callee-saved-register: '', callee-saved-restored: true,
+; 32BIT:        - { id: 1, type: default, offset: 28, size: 4, alignment: 4, stack-id: default,
+; 32BIT-NEXT:       isImmutable: false, isAliased: true, callee-saved-register: '', callee-saved-restored: true,
+; 32BIT:        - { id: 2, type: default, offset: 24, size: 4, alignment: 8, stack-id: default,
+; 32BIT-NEXT:       isImmutable: false, isAliased: true, callee-saved-register: '', callee-saved-restored: true,
+
+; 32BIT:      bb.0.entry:
+; 32BIT-NEXT:   liveins: $r3
+; 32BIT:        STW renamable $r3, 0, %fixed-stack.2 :: (store 4 into %fixed-stack.2, align 8)
+; 32BIT-DAG:    STW killed renamable $r4, 0, %fixed-stack.0 :: (store 4 into %fixed-stack.0)
+; 32BIT-DAG:    renamable $r[[SCRATCH:[0-9]+]] = RLWINM killed renamable $r3, 0, 24, 31
+; 32BIT-DAG:    renamable $r3 = nsw ADD4 renamable $r4, killed renamable $r[[SCRATCH]]
+; 32BIT:        BLR
+
+; 64BIT:      fixedStack:
+; 64BIT-NEXT: - { id: 0, type: default, offset: 56, size: 8, alignment: 8, stack-id: default,
+; 64BIT-NEXT:     isImmutable: false, isAliased: true, callee-saved-register: '', callee-saved-restored: true,
+; 64BIT:      - { id: 1, type: default, offset: 56, size: 8, alignment: 8, stack-id: default,
+; 64BIT-NEXT:     isImmutable: false, isAliased: true, callee-saved-register: '', callee-saved-restored: true,
+; 64BIT:      - { id: 2, type: default, offset: 48, size: 8, alignment: 16, stack-id: default,
+; 64BIT-NEXT:     isImmutable: false, isAliased: true, callee-saved-register: '', callee-saved-restored: true,
+
+; 64BIT:      bb.0.entry:
+; 64BIT-NEXT:   liveins: $x3
+; 64BIT:        STD killed renamable $x3, 0, %fixed-stack.2 :: (store 8 into %fixed-stack.2, align 16)
+; 64BIT-NEXT:   STD killed renamable $x4, 0, %fixed-stack.0 :: (store 8 into %fixed-stack.0)
+; 64BIT-DAG:    renamable $r[[SCRATCH1:[0-9]+]] = LBZ 3, %fixed-stack.2 :: (dereferenceable load 1
+; 64BIT-DAG:    renamable $r[[SCRATCH2:[0-9]+]] = LWZ 0, %fixed-stack.0 :: (dereferenceable load 4
+; 64BIT-NEXT:   renamable $r[[SCRATCH3:[0-9]+]] = nsw ADD4 killed renamable $r[[SCRATCH2]], killed renamable $r[[SCRATCH1]]
+; 64BIT-NEXT:   renamable $x3 = EXTSW_32_64 killed renamable $r[[SCRATCH3]]
+; 64BIT-NEXT:   BLR8
+
+; CHECKASM-LABEL: .test_byval_4Byte:
+
+; ASM32:        stw 3, 24(1)
+; ASM32-DAG:    stw 4, 28(1)
+; ASM32-DAG:    clrlwi  [[SCRATCH:[0-9]+]], 3, 24
+; ASM32-DAG:    add 3, 4, [[SCRATCH]]
+; ASM32-NEXT:   blr
+
+; ASM64:        std 3, 48(1)
+; ASM64-NEXT:   std 4, 56(1)
+; ASM64-DAG:    lbz [[SCRATCH1:[0-9]+]], 51(1)
+; ASM64-DAG:    lwz [[SCRATCH2:[0-9]+]], 56(1)
+; ASM64-NEXT:   add [[SCRATCH3:[0-9]+]], [[SCRATCH2]], [[SCRATCH1]]
+; ASM64-NEXT:   extsw 3, [[SCRATCH3]]
+; ASM64-NEXT:   blr
diff --git a/llvm/test/CodeGen/PowerPC/aix-func-dsc-gen.ll b/llvm/test/CodeGen/PowerPC/aix-func-dsc-gen.ll
index d983c8de54a9f..27cd5b5ba190e 100644
--- a/llvm/test/CodeGen/PowerPC/aix-func-dsc-gen.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-func-dsc-gen.ll
@@ -56,7 +56,7 @@ entry:
 ; CHECK-NEXT:     Value (RelocatableAddress): 0x4
 ; CHECK-NEXT:     Section: .data
 ; CHECK-NEXT:     Type: 0x0
-; CHECK-NEXT:     StorageClass: C_HIDEXT (0x6B)
+; CHECK-NEXT:     StorageClass: C_EXT (0x2)
 ; CHECK-NEXT:     NumberOfAuxEntries: 1
 ; CHECK-NEXT:     CSECT Auxiliary Entry {
 ; CHECK-NEXT:       Index: [[#Index+5]]
@@ -72,26 +72,6 @@ entry:
 ; CHECK-NEXT:   }
 ; CHECK-NEXT:   Symbol {
 ; CHECK-NEXT:     Index: [[#Index+6]]
-; CHECK-NEXT:     Name: foo
-; CHECK-NEXT:     Value (RelocatableAddress): 0x4
-; CHECK-NEXT:     Section: .data
-; CHECK-NEXT:     Type: 0x0
-; CHECK-NEXT:     StorageClass: C_EXT (0x2)
-; CHECK-NEXT:     NumberOfAuxEntries: 1
-; CHECK-NEXT:     CSECT Auxiliary Entry {
-; CHECK-NEXT:       Index: [[#Index+7]]
-; CHECK-NEXT:       ContainingCsectSymbolIndex: [[#Index+4]]
-; CHECK-NEXT:       ParameterHashIndex: 0x0
-; CHECK-NEXT:       TypeChkSectNum: 0x0
-; CHECK-NEXT:       SymbolAlignmentLog2: 0
-; CHECK-NEXT:       SymbolType: XTY_LD (0x2)
-; CHECK-NEXT:       StorageMappingClass: XMC_DS (0xA)
-; CHECK-NEXT:       StabInfoIndex: 0x0
-; CHECK-NEXT:       StabSectNum: 0x0
-; CHECK-NEXT:     }
-; CHECK-NEXT:   }
-; CHECK-NEXT:   Symbol {
-; CHECK-NEXT:     Index: [[#Index+8]]
 ; CHECK-NEXT:     Name: TOC
 ; CHECK-NEXT:     Value (RelocatableAddress): 0x10
 ; CHECK-NEXT:     Section: .data
@@ -99,7 +79,7 @@ entry:
 ; CHECK-NEXT:     StorageClass: C_HIDEXT (0x6B)
 ; CHECK-NEXT:     NumberOfAuxEntries: 1
 ; CHECK-NEXT:     CSECT Auxiliary Entry {
-; CHECK-NEXT:       Index: [[#Index+9]]
+; CHECK-NEXT:       Index: [[#Index+7]]
 ; CHECK-NEXT:       SectionLen: 0
 ; CHECK-NEXT:       ParameterHashIndex: 0x0
 ; CHECK-NEXT:       TypeChkSectNum: 0x0
diff --git a/llvm/test/CodeGen/PowerPC/aix-return55.ll b/llvm/test/CodeGen/PowerPC/aix-return55.ll
index a09857169a307..a18a211b46b2b 100644
--- a/llvm/test/CodeGen/PowerPC/aix-return55.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-return55.ll
@@ -31,7 +31,7 @@ entry:
 ;CHECKOBJ-NEXT:      18: 00 01 23 45                   <unknown>
 ;CHECKOBJ-NEXT:      1c: 67 8a bc de                   oris 10, 28, 48350{{[[:space:]] *}}
 ;CHECKOBJ-NEXT: 00000020 <d>:
-;CHECKOBJ-NEXT:      20: 40 14 00 00                   bdnzf   20, .+0
+;CHECKOBJ-NEXT:      20: 40 14 00 00                   bdnzf   20, 0x20
 ;CHECKOBJ-NEXT:      24: 00 00 00 00                   <unknown>{{[[:space:]] *}}
 ;CHECKOBJ-NEXT: 00000028 <foo>:
 ;CHECKOBJ-NEXT:      28: 00 00 00 00                   <unknown>
diff --git a/llvm/test/CodeGen/PowerPC/aix-user-defined-memcpy.ll b/llvm/test/CodeGen/PowerPC/aix-user-defined-memcpy.ll
index bafda1ca889c5..0e6fbcb6a3efa 100644
--- a/llvm/test/CodeGen/PowerPC/aix-user-defined-memcpy.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-user-defined-memcpy.ll
@@ -68,7 +68,7 @@ declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture writeonly, i8* nocapture r
 ; 32-REL-NEXT:  }
 ; 32-REL-NEXT:  Relocation {
 ; 32-REL-NEXT:    Virtual Address: 0x38
-; 32-REL-NEXT:    Symbol: TOC (14)
+; 32-REL-NEXT:    Symbol: TOC (10)
 ; 32-REL-NEXT:    IsSigned: No
 ; 32-REL-NEXT:    FixupBitValue: 0
 ; 32-REL-NEXT:    Length: 32
@@ -84,7 +84,7 @@ declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture writeonly, i8* nocapture r
 ; 32-REL-NEXT:  }
 ; 32-REL-NEXT:  Relocation {
 ; 32-REL-NEXT:    Virtual Address: 0x44
-; 32-REL-NEXT:    Symbol: TOC (14)
+; 32-REL-NEXT:    Symbol: TOC (10)
 ; 32-REL-NEXT:    IsSigned: No
 ; 32-REL-NEXT:    FixupBitValue: 0
 ; 32-REL-NEXT:    Length: 32
@@ -105,7 +105,7 @@ declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture writeonly, i8* nocapture r
 ; 32-DIS-NEXT:       10: 7c 08 02 a6                   mflr 0
 ; 32-DIS-NEXT:       14: 90 01 00 08                   stw 0, 8(1)
 ; 32-DIS-NEXT:       18: 94 21 ff c0                   stwu 1, -64(1)
-; 32-DIS-NEXT:       1c: 4b ff ff e5                   bl .-28
+; 32-DIS-NEXT:       1c: 4b ff ff e5                   bl 0x0
 ; 32-DIS-NEXT:       20: 60 00 00 00                   nop
 ; 32-DIS-NEXT:       24: 38 21 00 40                   addi 1, 1, 64
 ; 32-DIS-NEXT:       28: 80 01 00 08                   lwz 0, 8(1)
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-reloc.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-reloc.ll
index c68017e3150d1..04ce0f9307e4d 100644
--- a/llvm/test/CodeGen/PowerPC/aix-xcoff-reloc.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-reloc.ll
@@ -4,6 +4,7 @@
 ; RUN: llvm-readobj --relocs --expand-relocs %t.o | FileCheck --check-prefix=RELOC %s
 ; RUN: llvm-readobj -t %t.o | FileCheck --check-prefix=SYM %s
 ; RUN: llvm-objdump -D %t.o | FileCheck --check-prefix=DIS %s
+; RUN: llvm-objdump -r %t.o | FileCheck --check-prefix=DIS_REL %s
 
 ; RUN: not --crash llc -verify-machineinstrs -mcpu=pwr4 -mtriple powerpc64-ibm-aix-xcoff -mattr=-altivec -filetype=obj < %s 2>&1 | \
 ; RUN: FileCheck --check-prefix=XCOFF64 %s
@@ -35,7 +36,7 @@ declare i32 @bar(i32)
 ; OBJ-NEXT:   NumberOfSections: 2
 ; OBJ-NEXT:   TimeStamp: None (0x0)
 ; OBJ-NEXT:   SymbolTableOffset: 0x13C
-; OBJ-NEXT:   SymbolTableEntries: 26
+; OBJ-NEXT:   SymbolTableEntries: 24
 ; OBJ-NEXT:   OptionalHeaderSize: 0x0
 ; OBJ-NEXT:   Flags: 0x0
 ; OBJ-NEXT: }
@@ -85,7 +86,7 @@ declare i32 @bar(i32)
 ; RELOC-NEXT:   }
 ; RELOC-NEXT:   Relocation {
 ; RELOC-NEXT:     Virtual Address: 0x1A
-; RELOC-NEXT:     Symbol: globalA (22)
+; RELOC-NEXT:     Symbol: globalA (20)
 ; RELOC-NEXT:     IsSigned: No
 ; RELOC-NEXT:     FixupBitValue: 0
 ; RELOC-NEXT:     Length: 16
@@ -93,7 +94,7 @@ declare i32 @bar(i32)
 ; RELOC-NEXT:   }
 ; RELOC-NEXT:   Relocation {
 ; RELOC-NEXT:     Virtual Address: 0x1E
-; RELOC-NEXT:     Symbol: globalB (24)
+; RELOC-NEXT:     Symbol: globalB (22)
 ; RELOC-NEXT:     IsSigned: No
 ; RELOC-NEXT:     FixupBitValue: 0
 ; RELOC-NEXT:     Length: 16
@@ -119,7 +120,7 @@ declare i32 @bar(i32)
 ; RELOC-NEXT: }
 ; RELOC-NEXT: Relocation {
 ; RELOC-NEXT:   Virtual Address: 0x78
-; RELOC-NEXT:   Symbol: TOC (20)
+; RELOC-NEXT:   Symbol: TOC (18)
 ; RELOC-NEXT:   IsSigned: No
 ; RELOC-NEXT:   FixupBitValue: 0
 ; RELOC-NEXT:   Length: 32
@@ -311,7 +312,7 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     Value (RelocatableAddress): 0x74
 ; SYM-NEXT:     Section: .data
 ; SYM-NEXT:     Type: 0x0
-; SYM-NEXT:     StorageClass: C_HIDEXT (0x6B)
+; SYM-NEXT:     StorageClass: C_EXT (0x2)
 ; SYM-NEXT:     NumberOfAuxEntries: 1
 ; SYM-NEXT:     CSECT Auxiliary Entry {
 ; SYM-NEXT:       Index: 17
@@ -327,26 +328,6 @@ declare i32 @bar(i32)
 ; SYM-NEXT:   }
 ; SYM-NEXT:   Symbol {
 ; SYM-NEXT:     Index: 18
-; SYM-NEXT:     Name: foo
-; SYM-NEXT:     Value (RelocatableAddress): 0x74
-; SYM-NEXT:     Section: .data
-; SYM-NEXT:     Type: 0x0
-; SYM-NEXT:     StorageClass: C_EXT (0x2)
-; SYM-NEXT:     NumberOfAuxEntries: 1
-; SYM-NEXT:     CSECT Auxiliary Entry {
-; SYM-NEXT:       Index: 19
-; SYM-NEXT:       ContainingCsectSymbolIndex: 16
-; SYM-NEXT:       ParameterHashIndex: 0x0
-; SYM-NEXT:       TypeChkSectNum: 0x0
-; SYM-NEXT:       SymbolAlignmentLog2: 0
-; SYM-NEXT:       SymbolType: XTY_LD (0x2)
-; SYM-NEXT:       StorageMappingClass: XMC_DS (0xA)
-; SYM-NEXT:       StabInfoIndex: 0x0
-; SYM-NEXT:       StabSectNum: 0x0
-; SYM-NEXT:     }
-; SYM-NEXT:   }
-; SYM-NEXT:   Symbol {
-; SYM-NEXT:     Index: 20
 ; SYM-NEXT:     Name: TOC
 ; SYM-NEXT:     Value (RelocatableAddress): 0x80
 ; SYM-NEXT:     Section: .data
@@ -354,7 +335,7 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     StorageClass: C_HIDEXT (0x6B)
 ; SYM-NEXT:     NumberOfAuxEntries: 1
 ; SYM-NEXT:     CSECT Auxiliary Entry {
-; SYM-NEXT:       Index: 21
+; SYM-NEXT:       Index: 19
 ; SYM-NEXT:       SectionLen: 0
 ; SYM-NEXT:       ParameterHashIndex: 0x0
 ; SYM-NEXT:       TypeChkSectNum: 0x0
@@ -366,7 +347,7 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     }
 ; SYM-NEXT:   }
 ; SYM-NEXT:   Symbol {
-; SYM-NEXT:     Index: 22
+; SYM-NEXT:     Index: 20
 ; SYM-NEXT:     Name: globalA
 ; SYM-NEXT:     Value (RelocatableAddress): 0x80
 ; SYM-NEXT:     Section: .data
@@ -374,7 +355,7 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     StorageClass: C_HIDEXT (0x6B)
 ; SYM-NEXT:     NumberOfAuxEntries: 1
 ; SYM-NEXT:     CSECT Auxiliary Entry {
-; SYM-NEXT:       Index: 23
+; SYM-NEXT:       Index: 21
 ; SYM-NEXT:       SectionLen: 4
 ; SYM-NEXT:       ParameterHashIndex: 0x0
 ; SYM-NEXT:       TypeChkSectNum: 0x0
@@ -386,7 +367,7 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     }
 ; SYM-NEXT:   }
 ; SYM-NEXT:   Symbol {
-; SYM-NEXT:     Index: 24
+; SYM-NEXT:     Index: 22
 ; SYM-NEXT:     Name: globalB
 ; SYM-NEXT:     Value (RelocatableAddress): 0x84
 ; SYM-NEXT:     Section: .data
@@ -394,7 +375,7 @@ declare i32 @bar(i32)
 ; SYM-NEXT:     StorageClass: C_HIDEXT (0x6B)
 ; SYM-NEXT:     NumberOfAuxEntries: 1
 ; SYM-NEXT:     CSECT Auxiliary Entry {
-; SYM-NEXT:       Index: 25
+; SYM-NEXT:       Index: 23
 ; SYM-NEXT:       SectionLen: 4
 ; SYM-NEXT:       ParameterHashIndex: 0x0
 ; SYM-NEXT:       TypeChkSectNum: 0x0
@@ -415,7 +396,7 @@ declare i32 @bar(i32)
 ; DIS-NEXT:        4: 90 01 00 08                   stw 0, 8(1)
 ; DIS-NEXT:        8: 94 21 ff c0                   stwu 1, -64(1)
 ; DIS-NEXT:        c: 38 60 00 01                   li 3, 1
-; DIS-NEXT:       10: 4b ff ff f1                   bl .-16
+; DIS-NEXT:       10: 4b ff ff f1                   bl 0x0
 ; DIS-NEXT:       14: 60 00 00 00                   nop
 ; DIS-NEXT:       18: 80 82 00 00                   lwz 4, 0(2)
 ; DIS-NEXT:       1c: 80 a2 00 04                   lwz 5, 4(2)
@@ -446,3 +427,17 @@ declare i32 @bar(i32)
 ; DIS-NEXT:       80: 00 00 00 40                   <unknown>
 ; DIS:      00000084 <globalB>:
 ; DIS-NEXT:       84: 00 00 00 44                   <unknown>
+
+; DIS_REL:       {{.*}}aix-xcoff-reloc.ll.tmp.o:   file format aixcoff-rs6000
+; DIS_REL:       RELOCATION RECORDS FOR [.text]:
+; DIS_REL-NEXT:  OFFSET   TYPE                     VALUE
+; DIS_REL-NEXT:  00000010 R_RBR                    .bar
+; DIS_REL-NEXT:  0000001a R_TOC                    globalA
+; DIS_REL-NEXT:  0000001e R_TOC                    globalB
+; DIS_REL:       RELOCATION RECORDS FOR [.data]:
+; DIS_REL-NEXT:  OFFSET   TYPE                     VALUE
+; DIS_REL-NEXT:  00000030 R_POS                    arr
+; DIS_REL-NEXT:  00000034 R_POS                    .foo
+; DIS_REL-NEXT:  00000038 R_POS                    TOC
+; DIS_REL-NEXT:  00000040 R_POS                    globalA
+; DIS_REL-NEXT:  00000044 R_POS                    globalB
diff --git a/llvm/test/CodeGen/PowerPC/aix64-cc-byval.ll b/llvm/test/CodeGen/PowerPC/aix64-cc-byval.ll
index e8135af274b54..6f2384ff8b297 100644
--- a/llvm/test/CodeGen/PowerPC/aix64-cc-byval.ll
+++ b/llvm/test/CodeGen/PowerPC/aix64-cc-byval.ll
@@ -12,12 +12,10 @@
 
 define void @call_test_byval_5Byte() {
 entry:
-  call void @test_byval_5Byte(%struct.S5* byval(%struct.S5) align 1 @gS5)
+  %call = call zeroext i8 @test_byval_5Byte(%struct.S5* byval(%struct.S5) align 1 @gS5)
   ret void
 }
 
-declare void @test_byval_5Byte(%struct.S5* byval(%struct.S5) align 1)
-
 ; CHECK-LABEL: name: call_test_byval_5Byte{{.*}}
 
 ; ASM-LABEL: .call_test_byval_5Byte:
@@ -42,18 +40,40 @@ declare void @test_byval_5Byte(%struct.S5* byval(%struct.S5) align 1)
 ; ASM-NEXT:  bl .test_byval_5Byte
 ; ASM-NEXT:  nop
 
+
+define zeroext i8 @test_byval_5Byte(%struct.S5* byval(%struct.S5) align 1 %s) {
+entry:
+  %arrayidx = getelementptr inbounds %struct.S5, %struct.S5* %s, i32 0, i32 0, i32 4
+  %0 = load i8, i8* %arrayidx, align 1
+  ret i8 %0
+}
+
+; CHECK-LABEL: name:            test_byval_5Byte
+
+; CHECK:      fixedStack:
+; CHECK-NEXT:   - { id: 0, type: default, offset: 48, size: 8, alignment: 16,
+; CHECK:        bb.0.entry:
+; CHECK-NEXT:     liveins: $x3
+; CHECK:          STD killed renamable $x3, 0, %fixed-stack.0 :: (store 8 into %fixed-stack.0, align 16)
+; CHECK-NEXT:     renamable $x3 = LBZ8 4, %fixed-stack.0 :: (dereferenceable load 1
+
+; CHECKASM-LABEL: .test_byval_5Byte:
+
+; ASM:       std 3, 48(1)
+; ASM-NEXT:  lbz 3, 52(1)
+; ASM-NEXT:  blr
+
+
 %struct.S6 = type { [6 x i8] }
 
 @gS6 = external global %struct.S6, align 1
 
 define void @call_test_byval_6Byte() {
 entry:
-  call void @test_byval_6Byte(%struct.S6* byval(%struct.S6) align 1 @gS6)
+  %call = call zeroext i8 @test_byval_6Byte(%struct.S6* byval(%struct.S6) align 1 @gS6)
   ret void
 }
 
-declare void @test_byval_6Byte(%struct.S6* byval(%struct.S6) align 1)
-
 ; CHECK-LABEL: name: call_test_byval_6Byte{{.*}}
 
 ; ASM-LABEL: .call_test_byval_6Byte:
@@ -78,18 +98,39 @@ declare void @test_byval_6Byte(%struct.S6* byval(%struct.S6) align 1)
 ; ASM-NEXT:  bl .test_byval_6Byte
 ; ASM-NEXT:  nop
 
+
+define zeroext i8 @test_byval_6Byte(%struct.S6* byval(%struct.S6) align 1 %s) {
+entry:
+  %arrayidx = getelementptr inbounds %struct.S6, %struct.S6* %s, i32 0, i32 0, i32 5
+  %0 = load i8, i8* %arrayidx, align 1
+  ret i8 %0
+}
+
+; CHECK-LABEL: name:            test_byval_6Byte
+
+; CHECK:      fixedStack:
+; CHECK-NEXT:   - { id: 0, type: default, offset: 48, size: 8, alignment: 16,
+; CHECK:        bb.0.entry:
+; CHECK-NEXT:     liveins: $x3
+; CHECK:          STD killed renamable $x3, 0, %fixed-stack.0 :: (store 8 into %fixed-stack.0, align 16)
+; CHECK-NEXT:     renamable $x3 = LBZ8 5, %fixed-stack.0 :: (dereferenceable load 1
+
+; CHECKASM-LABEL: .test_byval_6Byte:
+
+; ASM:       std 3, 48(1)
+; ASM-NEXT:  lbz 3, 53(1)
+; ASM-NEXT:  blr
+
 %struct.S7 = type { [7 x i8] }
 
 @gS7 = external global %struct.S7, align 1
 
 define void @call_test_byval_7Byte() {
 entry:
-  call void @test_byval_7Byte(%struct.S7* byval(%struct.S7) align 1 @gS7)
+  %call = call zeroext i8 @test_byval_7Byte(%struct.S7* byval(%struct.S7) align 1 @gS7)
   ret void
 }
 
-declare void @test_byval_7Byte(%struct.S7* byval(%struct.S7) align 1)
-
 ; CHECK-LABEL: name: call_test_byval_7Byte{{.*}}
 
 ; ASM-LABEL: .call_test_byval_7Byte:
@@ -118,18 +159,40 @@ declare void @test_byval_7Byte(%struct.S7* byval(%struct.S7) align 1)
 ; ASM-NEXT:  bl .test_byval_7Byte
 ; ASM-NEXT:  nop
 
+
+define zeroext i8 @test_byval_7Byte(%struct.S7* byval(%struct.S7) align 1 %s) {
+entry:
+  %arrayidx = getelementptr inbounds %struct.S7, %struct.S7* %s, i32 0, i32 0, i32 6
+  %0 = load i8, i8* %arrayidx, align 1
+  ret i8 %0
+}
+
+; CHECK-LABEL: name:            test_byval_7Byte
+
+; CHECK:      fixedStack:
+; CHECK-NEXT:   - { id: 0, type: default, offset: 48, size: 8, alignment: 16,
+; CHECK:        bb.0.entry:
+; CHECK-NEXT:     liveins: $x3
+; CHECK:          STD killed renamable $x3, 0, %fixed-stack.0 :: (store 8 into %fixed-stack.0, align 16)
+; CHECK-NEXT:     renamable $x3 = LBZ8 6, %fixed-stack.0 :: (dereferenceable load 1
+
+; CHECKASM-LABEL: .test_byval_7Byte:
+
+; ASM:       std 3, 48(1)
+; ASM-NEXT:  lbz 3, 54(1)
+; ASM-NEXT:  blr
+
+
 %struct.S8 = type { [8 x i8] }
 
 @gS8 = external global %struct.S8, align 1
 
 define void @call_test_byval_8Byte() {
 entry:
-  call void @test_byval_8Byte(%struct.S8* byval(%struct.S8) align 1 @gS8)
+  %call = call zeroext i8 @test_byval_8Byte(%struct.S8* byval(%struct.S8) align 1 @gS8)
   ret void
 }
 
-declare void @test_byval_8Byte(%struct.S8* byval(%struct.S8) align 1)
-
 ; CHECK-LABEL: name: call_test_byval_8Byte{{.*}}
 
 ; ASM-LABEL: .call_test_byval_8Byte:
@@ -145,3 +208,29 @@ declare void @test_byval_8Byte(%struct.S8* byval(%struct.S8) align 1)
 ; ASM-NEXT:  ld 3, 0([[REGADDR]])
 ; ASM-NEXT:  bl .test_byval_8Byte
 ; ASM-NEXT:  nop
+
+
+define zeroext i8 @test_byval_8Byte(%struct.S8* byval(%struct.S8) align 1 %s) {
+entry:
+  %arrayidx = getelementptr inbounds %struct.S8, %struct.S8* %s, i32 0, i32 0, i32 7
+  %0 = load i8, i8* %arrayidx, align 1
+  ret i8 %0
+}
+
+; CHECK-LABEL: name:            test_byval_8Byte
+
+; CHECK:      fixedStack:
+; CHECK-NEXT:   - { id: 0, type: default, offset: 48, size: 8, alignment: 16,
+; CHECK:        bb.0.entry:
+; CHECK-NEXT:     liveins: $x3
+; CHECK:          renamable $x[[SCRATCH:[0-9]+]] = COPY $x3
+; CHECK-DAG:      renamable $x3 = RLDICL $x3, 0, 56
+; CHECK-DAG:      STD killed renamable $x[[SCRATCH]], 0, %fixed-stack.0 :: (store 8 into %fixed-stack.0, align 16)
+
+
+; CHECKASM-LABEL: .test_byval_8Byte:
+
+; ASM:       mr [[SCRATCH:[0-9]+]], 3
+; ASM-DAG:   clrldi  3, 3, 56
+; ASM-DAG:   std [[SCRATCH]], 48(1)
+; ASM-NEXT:  blr
diff --git a/llvm/test/CodeGen/PowerPC/alignlongjumptest.mir b/llvm/test/CodeGen/PowerPC/alignlongjumptest.mir
index 2ec09b0fb26b7..314844a2f3bf8 100644
--- a/llvm/test/CodeGen/PowerPC/alignlongjumptest.mir
+++ b/llvm/test/CodeGen/PowerPC/alignlongjumptest.mir
@@ -70,13 +70,13 @@ body:             |
 ...
 
 # Check for the long branch.
-# CHECK-LE:         08 00 82 4{{[01]}}   b{{[tf]}}  2, .+8
-# CHECK-LE-NEXT:    fc 7f 00 48   b .+32764
+# CHECK-LE:         08 00 82 4{{[01]}}   b{{[tf]}}  2, 0xc
+# CHECK-LE-NEXT:    fc 7f 00 48   b 0x8004
 # CHECK-LE-DAG:     paddi 3, 3, 13, 0
 # CHECK-LE-DAG:     paddi 3, 3, 21, 0
 # CHECK-LE:         blr
-# CHECK-BE:         4{{[01]}} 82 00 08   b{{[tf]}}  2, .+8
-# CHECK-BE-NEXT:    48 00 7f fc   b .+32764
+# CHECK-BE:         4{{[01]}} 82 00 08   b{{[tf]}}  2, 0xc
+# CHECK-BE-NEXT:    48 00 7f fc   b 0x8004
 # CHECK-BE-DAG:     paddi 3, 3, 13, 0
 # CHECK-BE-DAG:     paddi 3, 3, 21, 0
 # CHECK-BE:         blr
diff --git a/llvm/test/CodeGen/PowerPC/atomics-regression.ll b/llvm/test/CodeGen/PowerPC/atomics-regression.ll
index 7cfa4d2f6f026..673219e390b8a 100644
--- a/llvm/test/CodeGen/PowerPC/atomics-regression.ll
+++ b/llvm/test/CodeGen/PowerPC/atomics-regression.ll
@@ -401,11 +401,11 @@ define void @test40(i8* %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-LABEL: test40:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 24, 31
-; PPC64LE-NEXT:  .LBB40_1:
+; PPC64LE-NEXT:  .LBB40_1: #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB40_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
 ; PPC64LE-NEXT:    beqlr 0
 ; PPC64LE-NEXT:    b .LBB40_1
@@ -420,11 +420,11 @@ define void @test41(i8* %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-LABEL: test41:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 24, 31
-; PPC64LE-NEXT:  .LBB41_1:
+; PPC64LE-NEXT:  .LBB41_1: #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB41_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB41_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -442,11 +442,11 @@ define void @test42(i8* %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-LABEL: test42:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 24, 31
-; PPC64LE-NEXT:  .LBB42_1:
+; PPC64LE-NEXT:  .LBB42_1: #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB42_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB42_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -465,11 +465,11 @@ define void @test43(i8* %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 24, 31
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB43_1:
+; PPC64LE-NEXT:  .LBB43_1: #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB43_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
 ; PPC64LE-NEXT:    beqlr 0
 ; PPC64LE-NEXT:    b .LBB43_1
@@ -485,11 +485,11 @@ define void @test44(i8* %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 24, 31
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB44_1:
+; PPC64LE-NEXT:  .LBB44_1: #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB44_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
 ; PPC64LE-NEXT:    beqlr 0
 ; PPC64LE-NEXT:    b .LBB44_1
@@ -505,11 +505,11 @@ define void @test45(i8* %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 24, 31
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB45_1:
+; PPC64LE-NEXT:  .LBB45_1: #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB45_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB45_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -528,11 +528,11 @@ define void @test46(i8* %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 24, 31
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB46_1:
+; PPC64LE-NEXT:  .LBB46_1: #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB46_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB46_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -551,11 +551,11 @@ define void @test47(i8* %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 24, 31
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB47_1:
+; PPC64LE-NEXT:  .LBB47_1: #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB47_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB47_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -574,11 +574,11 @@ define void @test48(i8* %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 24, 31
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB48_1:
+; PPC64LE-NEXT:  .LBB48_1: #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB48_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB48_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -597,11 +597,11 @@ define void @test49(i8* %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 24, 31
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB49_1:
+; PPC64LE-NEXT:  .LBB49_1: #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB49_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB49_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -619,11 +619,11 @@ define void @test50(i16* %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-LABEL: test50:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 16, 31
-; PPC64LE-NEXT:  .LBB50_1:
+; PPC64LE-NEXT:  .LBB50_1: #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB50_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
 ; PPC64LE-NEXT:    beqlr 0
 ; PPC64LE-NEXT:    b .LBB50_1
@@ -638,11 +638,11 @@ define void @test51(i16* %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-LABEL: test51:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 16, 31
-; PPC64LE-NEXT:  .LBB51_1:
+; PPC64LE-NEXT:  .LBB51_1: #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB51_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB51_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -660,11 +660,11 @@ define void @test52(i16* %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-LABEL: test52:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 16, 31
-; PPC64LE-NEXT:  .LBB52_1:
+; PPC64LE-NEXT:  .LBB52_1: #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB52_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB52_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -683,11 +683,11 @@ define void @test53(i16* %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 16, 31
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB53_1:
+; PPC64LE-NEXT:  .LBB53_1: #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB53_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
 ; PPC64LE-NEXT:    beqlr 0
 ; PPC64LE-NEXT:    b .LBB53_1
@@ -703,11 +703,11 @@ define void @test54(i16* %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 16, 31
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB54_1:
+; PPC64LE-NEXT:  .LBB54_1: #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB54_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
 ; PPC64LE-NEXT:    beqlr 0
 ; PPC64LE-NEXT:    b .LBB54_1
@@ -723,11 +723,11 @@ define void @test55(i16* %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 16, 31
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB55_1:
+; PPC64LE-NEXT:  .LBB55_1: #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB55_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB55_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -746,11 +746,11 @@ define void @test56(i16* %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 16, 31
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB56_1:
+; PPC64LE-NEXT:  .LBB56_1: #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB56_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB56_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -769,11 +769,11 @@ define void @test57(i16* %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 16, 31
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB57_1:
+; PPC64LE-NEXT:  .LBB57_1: #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB57_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB57_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -792,11 +792,11 @@ define void @test58(i16* %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 16, 31
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB58_1:
+; PPC64LE-NEXT:  .LBB58_1: #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB58_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB58_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -815,11 +815,11 @@ define void @test59(i16* %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 16, 31
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB59_1:
+; PPC64LE-NEXT:  .LBB59_1: #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB59_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB59_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -836,11 +836,11 @@ define void @test59(i16* %ptr, i16 %cmp, i16 %val) {
 define void @test60(i32* %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test60:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB60_1:
+; PPC64LE-NEXT:  .LBB60_1: #
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB60_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
 ; PPC64LE-NEXT:    beqlr 0
 ; PPC64LE-NEXT:    b .LBB60_1
@@ -854,11 +854,11 @@ define void @test60(i32* %ptr, i32 %cmp, i32 %val) {
 define void @test61(i32* %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test61:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB61_1:
+; PPC64LE-NEXT:  .LBB61_1: #
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB61_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB61_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -875,11 +875,11 @@ define void @test61(i32* %ptr, i32 %cmp, i32 %val) {
 define void @test62(i32* %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test62:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB62_1:
+; PPC64LE-NEXT:  .LBB62_1: #
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB62_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB62_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -897,11 +897,11 @@ define void @test63(i32* %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test63:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB63_1:
+; PPC64LE-NEXT:  .LBB63_1: #
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB63_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
 ; PPC64LE-NEXT:    beqlr 0
 ; PPC64LE-NEXT:    b .LBB63_1
@@ -916,11 +916,11 @@ define void @test64(i32* %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test64:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB64_1:
+; PPC64LE-NEXT:  .LBB64_1: #
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB64_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
 ; PPC64LE-NEXT:    beqlr 0
 ; PPC64LE-NEXT:    b .LBB64_1
@@ -935,11 +935,11 @@ define void @test65(i32* %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test65:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB65_1:
+; PPC64LE-NEXT:  .LBB65_1: #
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB65_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB65_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -957,11 +957,11 @@ define void @test66(i32* %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test66:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB66_1:
+; PPC64LE-NEXT:  .LBB66_1: #
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB66_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB66_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -979,11 +979,11 @@ define void @test67(i32* %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test67:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB67_1:
+; PPC64LE-NEXT:  .LBB67_1: #
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB67_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB67_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1001,11 +1001,11 @@ define void @test68(i32* %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test68:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB68_1:
+; PPC64LE-NEXT:  .LBB68_1: #
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB68_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB68_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1023,11 +1023,11 @@ define void @test69(i32* %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test69:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB69_1:
+; PPC64LE-NEXT:  .LBB69_1: #
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB69_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB69_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1044,11 +1044,11 @@ define void @test69(i32* %ptr, i32 %cmp, i32 %val) {
 define void @test70(i64* %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test70:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB70_1:
+; PPC64LE-NEXT:  .LBB70_1: #
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB70_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
 ; PPC64LE-NEXT:    beqlr 0
 ; PPC64LE-NEXT:    b .LBB70_1
@@ -1062,11 +1062,11 @@ define void @test70(i64* %ptr, i64 %cmp, i64 %val) {
 define void @test71(i64* %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test71:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB71_1:
+; PPC64LE-NEXT:  .LBB71_1: #
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB71_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB71_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1083,11 +1083,11 @@ define void @test71(i64* %ptr, i64 %cmp, i64 %val) {
 define void @test72(i64* %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test72:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB72_1:
+; PPC64LE-NEXT:  .LBB72_1: #
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB72_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB72_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1105,11 +1105,11 @@ define void @test73(i64* %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test73:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB73_1:
+; PPC64LE-NEXT:  .LBB73_1: #
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB73_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
 ; PPC64LE-NEXT:    beqlr 0
 ; PPC64LE-NEXT:    b .LBB73_1
@@ -1124,11 +1124,11 @@ define void @test74(i64* %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test74:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB74_1:
+; PPC64LE-NEXT:  .LBB74_1: #
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB74_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
 ; PPC64LE-NEXT:    beqlr 0
 ; PPC64LE-NEXT:    b .LBB74_1
@@ -1143,11 +1143,11 @@ define void @test75(i64* %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test75:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB75_1:
+; PPC64LE-NEXT:  .LBB75_1: #
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB75_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB75_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1165,11 +1165,11 @@ define void @test76(i64* %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test76:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB76_1:
+; PPC64LE-NEXT:  .LBB76_1: #
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB76_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB76_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1187,11 +1187,11 @@ define void @test77(i64* %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test77:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB77_1:
+; PPC64LE-NEXT:  .LBB77_1: #
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB77_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB77_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1209,11 +1209,11 @@ define void @test78(i64* %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test78:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB78_1:
+; PPC64LE-NEXT:  .LBB78_1: #
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB78_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB78_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1231,11 +1231,11 @@ define void @test79(i64* %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test79:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB79_1:
+; PPC64LE-NEXT:  .LBB79_1: #
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB79_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB79_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1253,11 +1253,11 @@ define void @test80(i8* %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-LABEL: test80:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 24, 31
-; PPC64LE-NEXT:  .LBB80_1:
+; PPC64LE-NEXT:  .LBB80_1: #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB80_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
 ; PPC64LE-NEXT:    beqlr 0
 ; PPC64LE-NEXT:    b .LBB80_1
@@ -1272,11 +1272,11 @@ define void @test81(i8* %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-LABEL: test81:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 24, 31
-; PPC64LE-NEXT:  .LBB81_1:
+; PPC64LE-NEXT:  .LBB81_1: #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB81_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB81_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1294,11 +1294,11 @@ define void @test82(i8* %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-LABEL: test82:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 24, 31
-; PPC64LE-NEXT:  .LBB82_1:
+; PPC64LE-NEXT:  .LBB82_1: #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB82_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB82_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1317,11 +1317,11 @@ define void @test83(i8* %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 24, 31
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB83_1:
+; PPC64LE-NEXT:  .LBB83_1: #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB83_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
 ; PPC64LE-NEXT:    beqlr 0
 ; PPC64LE-NEXT:    b .LBB83_1
@@ -1337,11 +1337,11 @@ define void @test84(i8* %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 24, 31
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB84_1:
+; PPC64LE-NEXT:  .LBB84_1: #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB84_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
 ; PPC64LE-NEXT:    beqlr 0
 ; PPC64LE-NEXT:    b .LBB84_1
@@ -1357,11 +1357,11 @@ define void @test85(i8* %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 24, 31
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB85_1:
+; PPC64LE-NEXT:  .LBB85_1: #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB85_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB85_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1380,11 +1380,11 @@ define void @test86(i8* %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 24, 31
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB86_1:
+; PPC64LE-NEXT:  .LBB86_1: #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB86_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB86_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1403,11 +1403,11 @@ define void @test87(i8* %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 24, 31
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB87_1:
+; PPC64LE-NEXT:  .LBB87_1: #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB87_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB87_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1426,11 +1426,11 @@ define void @test88(i8* %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 24, 31
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB88_1:
+; PPC64LE-NEXT:  .LBB88_1: #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB88_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB88_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1449,11 +1449,11 @@ define void @test89(i8* %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 24, 31
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB89_1:
+; PPC64LE-NEXT:  .LBB89_1: #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB89_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB89_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1471,14 +1471,14 @@ define void @test90(i16* %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-LABEL: test90:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 16, 31
-; PPC64LE-NEXT:  .LBB90_1:
+; PPC64LE-NEXT:  .LBB90_1: #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB90_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
 ; PPC64LE-NEXT:    beqlr 0
-; PPC64LE-NEXT:    b 
+; PPC64LE-NEXT:    b .LBB90_1
 ; PPC64LE-NEXT:  .LBB90_3:
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
 ; PPC64LE-NEXT:    blr
@@ -1490,11 +1490,11 @@ define void @test91(i16* %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-LABEL: test91:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 16, 31
-; PPC64LE-NEXT:  .LBB91_1:
+; PPC64LE-NEXT:  .LBB91_1: #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB91_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB91_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1512,11 +1512,11 @@ define void @test92(i16* %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-LABEL: test92:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 16, 31
-; PPC64LE-NEXT:  .LBB92_1:
+; PPC64LE-NEXT:  .LBB92_1: #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB92_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB92_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1535,11 +1535,11 @@ define void @test93(i16* %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 16, 31
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB93_1:
+; PPC64LE-NEXT:  .LBB93_1: #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB93_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
 ; PPC64LE-NEXT:    beqlr 0
 ; PPC64LE-NEXT:    b .LBB93_1
@@ -1555,11 +1555,11 @@ define void @test94(i16* %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 16, 31
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB94_1:
+; PPC64LE-NEXT:  .LBB94_1: #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB94_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
 ; PPC64LE-NEXT:    beqlr 0
 ; PPC64LE-NEXT:    b .LBB94_1
@@ -1575,11 +1575,11 @@ define void @test95(i16* %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 16, 31
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB95_1:
+; PPC64LE-NEXT:  .LBB95_1: #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB95_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB95_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1598,11 +1598,11 @@ define void @test96(i16* %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 16, 31
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB96_1:
+; PPC64LE-NEXT:  .LBB96_1: #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB96_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB96_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1621,11 +1621,11 @@ define void @test97(i16* %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 16, 31
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB97_1:
+; PPC64LE-NEXT:  .LBB97_1: #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB97_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB97_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1644,11 +1644,11 @@ define void @test98(i16* %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 16, 31
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB98_1:
+; PPC64LE-NEXT:  .LBB98_1: #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB98_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB98_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1667,11 +1667,11 @@ define void @test99(i16* %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    rlwinm 4, 4, 0, 16, 31
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB99_1:
+; PPC64LE-NEXT:  .LBB99_1: #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB99_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB99_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1688,11 +1688,11 @@ define void @test99(i16* %ptr, i16 %cmp, i16 %val) {
 define void @test100(i32* %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test100:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB100_1:
+; PPC64LE-NEXT:  .LBB100_1: #
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB100_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
 ; PPC64LE-NEXT:    beqlr 0
 ; PPC64LE-NEXT:    b .LBB100_1
@@ -1706,11 +1706,11 @@ define void @test100(i32* %ptr, i32 %cmp, i32 %val) {
 define void @test101(i32* %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test101:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB101_1:
+; PPC64LE-NEXT:  .LBB101_1: #
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB101_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB101_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1727,11 +1727,11 @@ define void @test101(i32* %ptr, i32 %cmp, i32 %val) {
 define void @test102(i32* %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test102:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB102_1:
+; PPC64LE-NEXT:  .LBB102_1: #
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB102_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB102_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1749,11 +1749,11 @@ define void @test103(i32* %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test103:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB103_1:
+; PPC64LE-NEXT:  .LBB103_1: #
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB103_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
 ; PPC64LE-NEXT:    beqlr 0
 ; PPC64LE-NEXT:    b .LBB103_1
@@ -1768,11 +1768,11 @@ define void @test104(i32* %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test104:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB104_1:
+; PPC64LE-NEXT:  .LBB104_1: #
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB104_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
 ; PPC64LE-NEXT:    beqlr 0
 ; PPC64LE-NEXT:    b .LBB104_1
@@ -1787,11 +1787,11 @@ define void @test105(i32* %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test105:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB105_1:
+; PPC64LE-NEXT:  .LBB105_1: #
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB105_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB105_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1809,11 +1809,11 @@ define void @test106(i32* %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test106:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB106_1:
+; PPC64LE-NEXT:  .LBB106_1: #
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB106_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB106_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1831,11 +1831,11 @@ define void @test107(i32* %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test107:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB107_1:
+; PPC64LE-NEXT:  .LBB107_1: #
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB107_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB107_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1853,11 +1853,11 @@ define void @test108(i32* %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test108:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB108_1:
+; PPC64LE-NEXT:  .LBB108_1: #
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB108_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB108_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1875,11 +1875,11 @@ define void @test109(i32* %ptr, i32 %cmp, i32 %val) {
 ; PPC64LE-LABEL: test109:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB109_1:
+; PPC64LE-NEXT:  .LBB109_1: #
 ; PPC64LE-NEXT:    lwarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB109_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB109_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1896,11 +1896,11 @@ define void @test109(i32* %ptr, i32 %cmp, i32 %val) {
 define void @test110(i64* %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test110:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB110_1:
+; PPC64LE-NEXT:  .LBB110_1: #
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB110_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
 ; PPC64LE-NEXT:    beqlr 0
 ; PPC64LE-NEXT:    b .LBB110_1
@@ -1914,11 +1914,11 @@ define void @test110(i64* %ptr, i64 %cmp, i64 %val) {
 define void @test111(i64* %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test111:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB111_1:
+; PPC64LE-NEXT:  .LBB111_1: #
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB111_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB111_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1935,11 +1935,11 @@ define void @test111(i64* %ptr, i64 %cmp, i64 %val) {
 define void @test112(i64* %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test112:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB112_1:
+; PPC64LE-NEXT:  .LBB112_1: #
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB112_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB112_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -1957,11 +1957,11 @@ define void @test113(i64* %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test113:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB113_1:
+; PPC64LE-NEXT:  .LBB113_1: #
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB113_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
 ; PPC64LE-NEXT:    beqlr 0
 ; PPC64LE-NEXT:    b .LBB113_1
@@ -1976,11 +1976,11 @@ define void @test114(i64* %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test114:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB114_1:
+; PPC64LE-NEXT:  .LBB114_1: #
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB114_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
 ; PPC64LE-NEXT:    beqlr 0
 ; PPC64LE-NEXT:    b .LBB114_1
@@ -1995,11 +1995,11 @@ define void @test115(i64* %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test115:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB115_1:
+; PPC64LE-NEXT:  .LBB115_1: #
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB115_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB115_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -2017,11 +2017,11 @@ define void @test116(i64* %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test116:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB116_1:
+; PPC64LE-NEXT:  .LBB116_1: #
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB116_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB116_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -2039,11 +2039,11 @@ define void @test117(i64* %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test117:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB117_1:
+; PPC64LE-NEXT:  .LBB117_1: #
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB117_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB117_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -2061,11 +2061,11 @@ define void @test118(i64* %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test118:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB118_1:
+; PPC64LE-NEXT:  .LBB118_1: #
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB118_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB118_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -2083,11 +2083,11 @@ define void @test119(i64* %ptr, i64 %cmp, i64 %val) {
 ; PPC64LE-LABEL: test119:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB119_1:
+; PPC64LE-NEXT:  .LBB119_1: #
 ; PPC64LE-NEXT:    ldarx 6, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 6
 ; PPC64LE-NEXT:    bne 0, .LBB119_4
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB119_1
 ; PPC64LE-NEXT:  # %bb.3:
@@ -2104,7 +2104,7 @@ define void @test119(i64* %ptr, i64 %cmp, i64 %val) {
 define i8 @test120(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test120:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB120_1:
+; PPC64LE-NEXT:  .LBB120_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB120_1
@@ -2119,7 +2119,7 @@ define i8 @test121(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test121:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB121_1:
+; PPC64LE-NEXT:  .LBB121_1: #
 ; PPC64LE-NEXT:    lbarx 3, 0, 5
 ; PPC64LE-NEXT:    stbcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB121_1
@@ -2134,7 +2134,7 @@ define i8 @test122(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test122:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB122_1:
+; PPC64LE-NEXT:  .LBB122_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB122_1
@@ -2149,7 +2149,7 @@ define i8 @test123(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test123:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB123_1:
+; PPC64LE-NEXT:  .LBB123_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB123_1
@@ -2165,7 +2165,7 @@ define i8 @test124(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test124:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB124_1:
+; PPC64LE-NEXT:  .LBB124_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB124_1
@@ -2180,7 +2180,7 @@ define i8 @test124(i8* %ptr, i8 %val) {
 define i16 @test125(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test125:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB125_1:
+; PPC64LE-NEXT:  .LBB125_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB125_1
@@ -2195,7 +2195,7 @@ define i16 @test126(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test126:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB126_1:
+; PPC64LE-NEXT:  .LBB126_1: #
 ; PPC64LE-NEXT:    lharx 3, 0, 5
 ; PPC64LE-NEXT:    sthcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB126_1
@@ -2210,7 +2210,7 @@ define i16 @test127(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test127:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB127_1:
+; PPC64LE-NEXT:  .LBB127_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB127_1
@@ -2225,7 +2225,7 @@ define i16 @test128(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test128:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB128_1:
+; PPC64LE-NEXT:  .LBB128_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB128_1
@@ -2241,7 +2241,7 @@ define i16 @test129(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test129:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB129_1:
+; PPC64LE-NEXT:  .LBB129_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB129_1
@@ -2256,7 +2256,7 @@ define i16 @test129(i16* %ptr, i16 %val) {
 define i32 @test130(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test130:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB130_1:
+; PPC64LE-NEXT:  .LBB130_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB130_1
@@ -2271,7 +2271,7 @@ define i32 @test131(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test131:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB131_1:
+; PPC64LE-NEXT:  .LBB131_1: #
 ; PPC64LE-NEXT:    lwarx 3, 0, 5
 ; PPC64LE-NEXT:    stwcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB131_1
@@ -2286,7 +2286,7 @@ define i32 @test132(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test132:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB132_1:
+; PPC64LE-NEXT:  .LBB132_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB132_1
@@ -2301,7 +2301,7 @@ define i32 @test133(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test133:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB133_1:
+; PPC64LE-NEXT:  .LBB133_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB133_1
@@ -2317,7 +2317,7 @@ define i32 @test134(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test134:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB134_1:
+; PPC64LE-NEXT:  .LBB134_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB134_1
@@ -2332,7 +2332,7 @@ define i32 @test134(i32* %ptr, i32 %val) {
 define i64 @test135(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test135:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB135_1:
+; PPC64LE-NEXT:  .LBB135_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB135_1
@@ -2347,7 +2347,7 @@ define i64 @test136(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test136:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB136_1:
+; PPC64LE-NEXT:  .LBB136_1: #
 ; PPC64LE-NEXT:    ldarx 3, 0, 5
 ; PPC64LE-NEXT:    stdcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB136_1
@@ -2362,7 +2362,7 @@ define i64 @test137(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test137:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB137_1:
+; PPC64LE-NEXT:  .LBB137_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB137_1
@@ -2377,7 +2377,7 @@ define i64 @test138(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test138:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB138_1:
+; PPC64LE-NEXT:  .LBB138_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB138_1
@@ -2393,7 +2393,7 @@ define i64 @test139(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test139:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB139_1:
+; PPC64LE-NEXT:  .LBB139_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB139_1
@@ -2408,7 +2408,7 @@ define i64 @test139(i64* %ptr, i64 %val) {
 define i8 @test140(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test140:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB140_1:
+; PPC64LE-NEXT:  .LBB140_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -2424,7 +2424,7 @@ define i8 @test141(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test141:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB141_1:
+; PPC64LE-NEXT:  .LBB141_1: #
 ; PPC64LE-NEXT:    lbarx 3, 0, 5
 ; PPC64LE-NEXT:    add 6, 4, 3
 ; PPC64LE-NEXT:    stbcx. 6, 0, 5
@@ -2440,7 +2440,7 @@ define i8 @test142(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test142:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB142_1:
+; PPC64LE-NEXT:  .LBB142_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -2456,7 +2456,7 @@ define i8 @test143(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test143:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB143_1:
+; PPC64LE-NEXT:  .LBB143_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -2473,7 +2473,7 @@ define i8 @test144(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test144:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB144_1:
+; PPC64LE-NEXT:  .LBB144_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -2489,7 +2489,7 @@ define i8 @test144(i8* %ptr, i8 %val) {
 define i16 @test145(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test145:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB145_1:
+; PPC64LE-NEXT:  .LBB145_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -2505,7 +2505,7 @@ define i16 @test146(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test146:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB146_1:
+; PPC64LE-NEXT:  .LBB146_1: #
 ; PPC64LE-NEXT:    lharx 3, 0, 5
 ; PPC64LE-NEXT:    add 6, 4, 3
 ; PPC64LE-NEXT:    sthcx. 6, 0, 5
@@ -2521,7 +2521,7 @@ define i16 @test147(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test147:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB147_1:
+; PPC64LE-NEXT:  .LBB147_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -2537,7 +2537,7 @@ define i16 @test148(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test148:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB148_1:
+; PPC64LE-NEXT:  .LBB148_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -2554,7 +2554,7 @@ define i16 @test149(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test149:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB149_1:
+; PPC64LE-NEXT:  .LBB149_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -2570,7 +2570,7 @@ define i16 @test149(i16* %ptr, i16 %val) {
 define i32 @test150(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test150:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB150_1:
+; PPC64LE-NEXT:  .LBB150_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -2586,7 +2586,7 @@ define i32 @test151(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test151:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB151_1:
+; PPC64LE-NEXT:  .LBB151_1: #
 ; PPC64LE-NEXT:    lwarx 3, 0, 5
 ; PPC64LE-NEXT:    add 6, 4, 3
 ; PPC64LE-NEXT:    stwcx. 6, 0, 5
@@ -2602,7 +2602,7 @@ define i32 @test152(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test152:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB152_1:
+; PPC64LE-NEXT:  .LBB152_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -2618,7 +2618,7 @@ define i32 @test153(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test153:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB153_1:
+; PPC64LE-NEXT:  .LBB153_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -2635,7 +2635,7 @@ define i32 @test154(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test154:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB154_1:
+; PPC64LE-NEXT:  .LBB154_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -2651,7 +2651,7 @@ define i32 @test154(i32* %ptr, i32 %val) {
 define i64 @test155(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test155:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB155_1:
+; PPC64LE-NEXT:  .LBB155_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -2667,7 +2667,7 @@ define i64 @test156(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test156:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB156_1:
+; PPC64LE-NEXT:  .LBB156_1: #
 ; PPC64LE-NEXT:    ldarx 3, 0, 5
 ; PPC64LE-NEXT:    add 6, 4, 3
 ; PPC64LE-NEXT:    stdcx. 6, 0, 5
@@ -2683,7 +2683,7 @@ define i64 @test157(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test157:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB157_1:
+; PPC64LE-NEXT:  .LBB157_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -2699,7 +2699,7 @@ define i64 @test158(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test158:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB158_1:
+; PPC64LE-NEXT:  .LBB158_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -2716,7 +2716,7 @@ define i64 @test159(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test159:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB159_1:
+; PPC64LE-NEXT:  .LBB159_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -2732,7 +2732,7 @@ define i64 @test159(i64* %ptr, i64 %val) {
 define i8 @test160(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test160:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB160_1:
+; PPC64LE-NEXT:  .LBB160_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    subf 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -2748,7 +2748,7 @@ define i8 @test161(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test161:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB161_1:
+; PPC64LE-NEXT:  .LBB161_1: #
 ; PPC64LE-NEXT:    lbarx 3, 0, 5
 ; PPC64LE-NEXT:    subf 6, 4, 3
 ; PPC64LE-NEXT:    stbcx. 6, 0, 5
@@ -2764,7 +2764,7 @@ define i8 @test162(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test162:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB162_1:
+; PPC64LE-NEXT:  .LBB162_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    subf 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -2780,7 +2780,7 @@ define i8 @test163(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test163:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB163_1:
+; PPC64LE-NEXT:  .LBB163_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    subf 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -2797,7 +2797,7 @@ define i8 @test164(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test164:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB164_1:
+; PPC64LE-NEXT:  .LBB164_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    subf 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -2813,7 +2813,7 @@ define i8 @test164(i8* %ptr, i8 %val) {
 define i16 @test165(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test165:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB165_1:
+; PPC64LE-NEXT:  .LBB165_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    subf 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -2829,7 +2829,7 @@ define i16 @test166(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test166:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB166_1:
+; PPC64LE-NEXT:  .LBB166_1: #
 ; PPC64LE-NEXT:    lharx 3, 0, 5
 ; PPC64LE-NEXT:    subf 6, 4, 3
 ; PPC64LE-NEXT:    sthcx. 6, 0, 5
@@ -2845,7 +2845,7 @@ define i16 @test167(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test167:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB167_1:
+; PPC64LE-NEXT:  .LBB167_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    subf 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -2861,7 +2861,7 @@ define i16 @test168(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test168:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB168_1:
+; PPC64LE-NEXT:  .LBB168_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    subf 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -2878,7 +2878,7 @@ define i16 @test169(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test169:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB169_1:
+; PPC64LE-NEXT:  .LBB169_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    subf 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -2894,7 +2894,7 @@ define i16 @test169(i16* %ptr, i16 %val) {
 define i32 @test170(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test170:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB170_1:
+; PPC64LE-NEXT:  .LBB170_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    subf 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -2910,7 +2910,7 @@ define i32 @test171(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test171:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB171_1:
+; PPC64LE-NEXT:  .LBB171_1: #
 ; PPC64LE-NEXT:    lwarx 3, 0, 5
 ; PPC64LE-NEXT:    subf 6, 4, 3
 ; PPC64LE-NEXT:    stwcx. 6, 0, 5
@@ -2926,7 +2926,7 @@ define i32 @test172(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test172:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB172_1:
+; PPC64LE-NEXT:  .LBB172_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    subf 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -2942,7 +2942,7 @@ define i32 @test173(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test173:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB173_1:
+; PPC64LE-NEXT:  .LBB173_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    subf 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -2959,7 +2959,7 @@ define i32 @test174(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test174:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB174_1:
+; PPC64LE-NEXT:  .LBB174_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    subf 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -2975,7 +2975,7 @@ define i32 @test174(i32* %ptr, i32 %val) {
 define i64 @test175(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test175:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB175_1:
+; PPC64LE-NEXT:  .LBB175_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    sub 6, 5, 4
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -2991,7 +2991,7 @@ define i64 @test176(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test176:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB176_1:
+; PPC64LE-NEXT:  .LBB176_1: #
 ; PPC64LE-NEXT:    ldarx 3, 0, 5
 ; PPC64LE-NEXT:    sub 6, 3, 4
 ; PPC64LE-NEXT:    stdcx. 6, 0, 5
@@ -3007,7 +3007,7 @@ define i64 @test177(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test177:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB177_1:
+; PPC64LE-NEXT:  .LBB177_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    sub 6, 5, 4
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -3023,7 +3023,7 @@ define i64 @test178(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test178:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB178_1:
+; PPC64LE-NEXT:  .LBB178_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    sub 6, 5, 4
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -3040,7 +3040,7 @@ define i64 @test179(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test179:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB179_1:
+; PPC64LE-NEXT:  .LBB179_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    sub 6, 5, 4
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -3056,7 +3056,7 @@ define i64 @test179(i64* %ptr, i64 %val) {
 define i8 @test180(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test180:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB180_1:
+; PPC64LE-NEXT:  .LBB180_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -3072,7 +3072,7 @@ define i8 @test181(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test181:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB181_1:
+; PPC64LE-NEXT:  .LBB181_1: #
 ; PPC64LE-NEXT:    lbarx 3, 0, 5
 ; PPC64LE-NEXT:    and 6, 4, 3
 ; PPC64LE-NEXT:    stbcx. 6, 0, 5
@@ -3088,7 +3088,7 @@ define i8 @test182(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test182:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB182_1:
+; PPC64LE-NEXT:  .LBB182_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -3104,7 +3104,7 @@ define i8 @test183(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test183:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB183_1:
+; PPC64LE-NEXT:  .LBB183_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -3121,7 +3121,7 @@ define i8 @test184(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test184:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB184_1:
+; PPC64LE-NEXT:  .LBB184_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -3137,7 +3137,7 @@ define i8 @test184(i8* %ptr, i8 %val) {
 define i16 @test185(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test185:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB185_1:
+; PPC64LE-NEXT:  .LBB185_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -3153,7 +3153,7 @@ define i16 @test186(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test186:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB186_1:
+; PPC64LE-NEXT:  .LBB186_1: #
 ; PPC64LE-NEXT:    lharx 3, 0, 5
 ; PPC64LE-NEXT:    and 6, 4, 3
 ; PPC64LE-NEXT:    sthcx. 6, 0, 5
@@ -3169,7 +3169,7 @@ define i16 @test187(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test187:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB187_1:
+; PPC64LE-NEXT:  .LBB187_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -3185,7 +3185,7 @@ define i16 @test188(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test188:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB188_1:
+; PPC64LE-NEXT:  .LBB188_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -3202,7 +3202,7 @@ define i16 @test189(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test189:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB189_1:
+; PPC64LE-NEXT:  .LBB189_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -3218,7 +3218,7 @@ define i16 @test189(i16* %ptr, i16 %val) {
 define i32 @test190(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test190:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB190_1:
+; PPC64LE-NEXT:  .LBB190_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -3234,7 +3234,7 @@ define i32 @test191(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test191:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB191_1:
+; PPC64LE-NEXT:  .LBB191_1: #
 ; PPC64LE-NEXT:    lwarx 3, 0, 5
 ; PPC64LE-NEXT:    and 6, 4, 3
 ; PPC64LE-NEXT:    stwcx. 6, 0, 5
@@ -3250,7 +3250,7 @@ define i32 @test192(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test192:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB192_1:
+; PPC64LE-NEXT:  .LBB192_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -3266,7 +3266,7 @@ define i32 @test193(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test193:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB193_1:
+; PPC64LE-NEXT:  .LBB193_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -3283,7 +3283,7 @@ define i32 @test194(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test194:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB194_1:
+; PPC64LE-NEXT:  .LBB194_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -3299,7 +3299,7 @@ define i32 @test194(i32* %ptr, i32 %val) {
 define i64 @test195(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test195:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB195_1:
+; PPC64LE-NEXT:  .LBB195_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -3315,7 +3315,7 @@ define i64 @test196(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test196:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB196_1:
+; PPC64LE-NEXT:  .LBB196_1: #
 ; PPC64LE-NEXT:    ldarx 3, 0, 5
 ; PPC64LE-NEXT:    and 6, 4, 3
 ; PPC64LE-NEXT:    stdcx. 6, 0, 5
@@ -3331,7 +3331,7 @@ define i64 @test197(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test197:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB197_1:
+; PPC64LE-NEXT:  .LBB197_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -3347,7 +3347,7 @@ define i64 @test198(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test198:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB198_1:
+; PPC64LE-NEXT:  .LBB198_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -3364,7 +3364,7 @@ define i64 @test199(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test199:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB199_1:
+; PPC64LE-NEXT:  .LBB199_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -3380,7 +3380,7 @@ define i64 @test199(i64* %ptr, i64 %val) {
 define i8 @test200(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test200:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB200_1:
+; PPC64LE-NEXT:  .LBB200_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -3396,7 +3396,7 @@ define i8 @test201(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test201:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB201_1:
+; PPC64LE-NEXT:  .LBB201_1: #
 ; PPC64LE-NEXT:    lbarx 3, 0, 5
 ; PPC64LE-NEXT:    nand 6, 4, 3
 ; PPC64LE-NEXT:    stbcx. 6, 0, 5
@@ -3412,7 +3412,7 @@ define i8 @test202(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test202:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB202_1:
+; PPC64LE-NEXT:  .LBB202_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -3428,7 +3428,7 @@ define i8 @test203(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test203:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB203_1:
+; PPC64LE-NEXT:  .LBB203_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -3445,7 +3445,7 @@ define i8 @test204(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test204:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB204_1:
+; PPC64LE-NEXT:  .LBB204_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -3461,7 +3461,7 @@ define i8 @test204(i8* %ptr, i8 %val) {
 define i16 @test205(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test205:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB205_1:
+; PPC64LE-NEXT:  .LBB205_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -3477,7 +3477,7 @@ define i16 @test206(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test206:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB206_1:
+; PPC64LE-NEXT:  .LBB206_1: #
 ; PPC64LE-NEXT:    lharx 3, 0, 5
 ; PPC64LE-NEXT:    nand 6, 4, 3
 ; PPC64LE-NEXT:    sthcx. 6, 0, 5
@@ -3493,7 +3493,7 @@ define i16 @test207(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test207:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB207_1:
+; PPC64LE-NEXT:  .LBB207_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -3509,7 +3509,7 @@ define i16 @test208(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test208:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB208_1:
+; PPC64LE-NEXT:  .LBB208_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -3526,7 +3526,7 @@ define i16 @test209(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test209:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB209_1:
+; PPC64LE-NEXT:  .LBB209_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -3542,7 +3542,7 @@ define i16 @test209(i16* %ptr, i16 %val) {
 define i32 @test210(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test210:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB210_1:
+; PPC64LE-NEXT:  .LBB210_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -3558,7 +3558,7 @@ define i32 @test211(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test211:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB211_1:
+; PPC64LE-NEXT:  .LBB211_1: #
 ; PPC64LE-NEXT:    lwarx 3, 0, 5
 ; PPC64LE-NEXT:    nand 6, 4, 3
 ; PPC64LE-NEXT:    stwcx. 6, 0, 5
@@ -3574,7 +3574,7 @@ define i32 @test212(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test212:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB212_1:
+; PPC64LE-NEXT:  .LBB212_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -3590,7 +3590,7 @@ define i32 @test213(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test213:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB213_1:
+; PPC64LE-NEXT:  .LBB213_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -3607,7 +3607,7 @@ define i32 @test214(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test214:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB214_1:
+; PPC64LE-NEXT:  .LBB214_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -3623,7 +3623,7 @@ define i32 @test214(i32* %ptr, i32 %val) {
 define i64 @test215(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test215:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB215_1:
+; PPC64LE-NEXT:  .LBB215_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -3639,7 +3639,7 @@ define i64 @test216(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test216:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB216_1:
+; PPC64LE-NEXT:  .LBB216_1: #
 ; PPC64LE-NEXT:    ldarx 3, 0, 5
 ; PPC64LE-NEXT:    nand 6, 4, 3
 ; PPC64LE-NEXT:    stdcx. 6, 0, 5
@@ -3655,7 +3655,7 @@ define i64 @test217(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test217:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB217_1:
+; PPC64LE-NEXT:  .LBB217_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -3671,7 +3671,7 @@ define i64 @test218(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test218:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB218_1:
+; PPC64LE-NEXT:  .LBB218_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -3688,7 +3688,7 @@ define i64 @test219(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test219:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB219_1:
+; PPC64LE-NEXT:  .LBB219_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -3704,7 +3704,7 @@ define i64 @test219(i64* %ptr, i64 %val) {
 define i8 @test220(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test220:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB220_1:
+; PPC64LE-NEXT:  .LBB220_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -3720,7 +3720,7 @@ define i8 @test221(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test221:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB221_1:
+; PPC64LE-NEXT:  .LBB221_1: #
 ; PPC64LE-NEXT:    lbarx 3, 0, 5
 ; PPC64LE-NEXT:    or 6, 4, 3
 ; PPC64LE-NEXT:    stbcx. 6, 0, 5
@@ -3736,7 +3736,7 @@ define i8 @test222(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test222:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB222_1:
+; PPC64LE-NEXT:  .LBB222_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -3752,7 +3752,7 @@ define i8 @test223(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test223:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB223_1:
+; PPC64LE-NEXT:  .LBB223_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -3769,7 +3769,7 @@ define i8 @test224(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test224:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB224_1:
+; PPC64LE-NEXT:  .LBB224_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -3785,7 +3785,7 @@ define i8 @test224(i8* %ptr, i8 %val) {
 define i16 @test225(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test225:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB225_1:
+; PPC64LE-NEXT:  .LBB225_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -3801,7 +3801,7 @@ define i16 @test226(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test226:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB226_1:
+; PPC64LE-NEXT:  .LBB226_1: #
 ; PPC64LE-NEXT:    lharx 3, 0, 5
 ; PPC64LE-NEXT:    or 6, 4, 3
 ; PPC64LE-NEXT:    sthcx. 6, 0, 5
@@ -3817,7 +3817,7 @@ define i16 @test227(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test227:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB227_1:
+; PPC64LE-NEXT:  .LBB227_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -3833,7 +3833,7 @@ define i16 @test228(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test228:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB228_1:
+; PPC64LE-NEXT:  .LBB228_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -3850,7 +3850,7 @@ define i16 @test229(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test229:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB229_1:
+; PPC64LE-NEXT:  .LBB229_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -3866,7 +3866,7 @@ define i16 @test229(i16* %ptr, i16 %val) {
 define i32 @test230(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test230:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB230_1:
+; PPC64LE-NEXT:  .LBB230_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -3882,7 +3882,7 @@ define i32 @test231(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test231:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB231_1:
+; PPC64LE-NEXT:  .LBB231_1: #
 ; PPC64LE-NEXT:    lwarx 3, 0, 5
 ; PPC64LE-NEXT:    or 6, 4, 3
 ; PPC64LE-NEXT:    stwcx. 6, 0, 5
@@ -3898,7 +3898,7 @@ define i32 @test232(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test232:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB232_1:
+; PPC64LE-NEXT:  .LBB232_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -3914,7 +3914,7 @@ define i32 @test233(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test233:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB233_1:
+; PPC64LE-NEXT:  .LBB233_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -3931,7 +3931,7 @@ define i32 @test234(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test234:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB234_1:
+; PPC64LE-NEXT:  .LBB234_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -3947,7 +3947,7 @@ define i32 @test234(i32* %ptr, i32 %val) {
 define i64 @test235(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test235:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB235_1:
+; PPC64LE-NEXT:  .LBB235_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -3963,7 +3963,7 @@ define i64 @test236(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test236:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB236_1:
+; PPC64LE-NEXT:  .LBB236_1: #
 ; PPC64LE-NEXT:    ldarx 3, 0, 5
 ; PPC64LE-NEXT:    or 6, 4, 3
 ; PPC64LE-NEXT:    stdcx. 6, 0, 5
@@ -3979,7 +3979,7 @@ define i64 @test237(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test237:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB237_1:
+; PPC64LE-NEXT:  .LBB237_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -3995,7 +3995,7 @@ define i64 @test238(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test238:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB238_1:
+; PPC64LE-NEXT:  .LBB238_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -4012,7 +4012,7 @@ define i64 @test239(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test239:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB239_1:
+; PPC64LE-NEXT:  .LBB239_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -4028,7 +4028,7 @@ define i64 @test239(i64* %ptr, i64 %val) {
 define i8 @test240(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test240:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB240_1:
+; PPC64LE-NEXT:  .LBB240_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -4044,7 +4044,7 @@ define i8 @test241(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test241:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB241_1:
+; PPC64LE-NEXT:  .LBB241_1: #
 ; PPC64LE-NEXT:    lbarx 3, 0, 5
 ; PPC64LE-NEXT:    xor 6, 4, 3
 ; PPC64LE-NEXT:    stbcx. 6, 0, 5
@@ -4060,7 +4060,7 @@ define i8 @test242(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test242:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB242_1:
+; PPC64LE-NEXT:  .LBB242_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -4076,7 +4076,7 @@ define i8 @test243(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test243:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB243_1:
+; PPC64LE-NEXT:  .LBB243_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -4093,7 +4093,7 @@ define i8 @test244(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test244:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB244_1:
+; PPC64LE-NEXT:  .LBB244_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -4109,7 +4109,7 @@ define i8 @test244(i8* %ptr, i8 %val) {
 define i16 @test245(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test245:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB245_1:
+; PPC64LE-NEXT:  .LBB245_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -4125,7 +4125,7 @@ define i16 @test246(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test246:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB246_1:
+; PPC64LE-NEXT:  .LBB246_1: #
 ; PPC64LE-NEXT:    lharx 3, 0, 5
 ; PPC64LE-NEXT:    xor 6, 4, 3
 ; PPC64LE-NEXT:    sthcx. 6, 0, 5
@@ -4141,7 +4141,7 @@ define i16 @test247(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test247:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB247_1:
+; PPC64LE-NEXT:  .LBB247_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -4157,7 +4157,7 @@ define i16 @test248(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test248:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB248_1:
+; PPC64LE-NEXT:  .LBB248_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -4174,7 +4174,7 @@ define i16 @test249(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test249:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB249_1:
+; PPC64LE-NEXT:  .LBB249_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -4190,7 +4190,7 @@ define i16 @test249(i16* %ptr, i16 %val) {
 define i32 @test250(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test250:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB250_1:
+; PPC64LE-NEXT:  .LBB250_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -4206,7 +4206,7 @@ define i32 @test251(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test251:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB251_1:
+; PPC64LE-NEXT:  .LBB251_1: #
 ; PPC64LE-NEXT:    lwarx 3, 0, 5
 ; PPC64LE-NEXT:    xor 6, 4, 3
 ; PPC64LE-NEXT:    stwcx. 6, 0, 5
@@ -4222,7 +4222,7 @@ define i32 @test252(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test252:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB252_1:
+; PPC64LE-NEXT:  .LBB252_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -4238,7 +4238,7 @@ define i32 @test253(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test253:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB253_1:
+; PPC64LE-NEXT:  .LBB253_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -4255,7 +4255,7 @@ define i32 @test254(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test254:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB254_1:
+; PPC64LE-NEXT:  .LBB254_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -4271,7 +4271,7 @@ define i32 @test254(i32* %ptr, i32 %val) {
 define i64 @test255(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test255:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB255_1:
+; PPC64LE-NEXT:  .LBB255_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -4287,7 +4287,7 @@ define i64 @test256(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test256:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB256_1:
+; PPC64LE-NEXT:  .LBB256_1: #
 ; PPC64LE-NEXT:    ldarx 3, 0, 5
 ; PPC64LE-NEXT:    xor 6, 4, 3
 ; PPC64LE-NEXT:    stdcx. 6, 0, 5
@@ -4303,7 +4303,7 @@ define i64 @test257(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test257:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB257_1:
+; PPC64LE-NEXT:  .LBB257_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -4319,7 +4319,7 @@ define i64 @test258(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test258:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB258_1:
+; PPC64LE-NEXT:  .LBB258_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -4336,7 +4336,7 @@ define i64 @test259(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test259:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB259_1:
+; PPC64LE-NEXT:  .LBB259_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -4352,12 +4352,12 @@ define i64 @test259(i64* %ptr, i64 %val) {
 define i8 @test260(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test260:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB260_1:
+; PPC64LE-NEXT:  .LBB260_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    extsb 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    ble 0, .LBB260_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB260_1
 ; PPC64LE-NEXT:  .LBB260_3:
@@ -4371,12 +4371,12 @@ define i8 @test261(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test261:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB261_1:
+; PPC64LE-NEXT:  .LBB261_1: #
 ; PPC64LE-NEXT:    lbarx 3, 0, 5
 ; PPC64LE-NEXT:    extsb 6, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    ble 0, .LBB261_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB261_1
 ; PPC64LE-NEXT:  .LBB261_3:
@@ -4390,12 +4390,12 @@ define i8 @test262(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test262:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB262_1:
+; PPC64LE-NEXT:  .LBB262_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    extsb 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    ble 0, .LBB262_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB262_1
 ; PPC64LE-NEXT:  .LBB262_3:
@@ -4409,12 +4409,12 @@ define i8 @test263(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test263:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB263_1:
+; PPC64LE-NEXT:  .LBB263_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    extsb 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    ble 0, .LBB263_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB263_1
 ; PPC64LE-NEXT:  .LBB263_3:
@@ -4429,12 +4429,12 @@ define i8 @test264(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test264:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB264_1:
+; PPC64LE-NEXT:  .LBB264_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    extsb 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    ble 0, .LBB264_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB264_1
 ; PPC64LE-NEXT:  .LBB264_3:
@@ -4448,12 +4448,12 @@ define i8 @test264(i8* %ptr, i8 %val) {
 define i16 @test265(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test265:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB265_1:
+; PPC64LE-NEXT:  .LBB265_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    extsh 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    ble 0, .LBB265_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB265_1
 ; PPC64LE-NEXT:  .LBB265_3:
@@ -4467,12 +4467,12 @@ define i16 @test266(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test266:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB266_1:
+; PPC64LE-NEXT:  .LBB266_1: #
 ; PPC64LE-NEXT:    lharx 3, 0, 5
 ; PPC64LE-NEXT:    extsh 6, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    ble 0, .LBB266_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB266_1
 ; PPC64LE-NEXT:  .LBB266_3:
@@ -4486,12 +4486,12 @@ define i16 @test267(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test267:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB267_1:
+; PPC64LE-NEXT:  .LBB267_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    extsh 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    ble 0, .LBB267_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB267_1
 ; PPC64LE-NEXT:  .LBB267_3:
@@ -4505,12 +4505,12 @@ define i16 @test268(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test268:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB268_1:
+; PPC64LE-NEXT:  .LBB268_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    extsh 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    ble 0, .LBB268_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB268_1
 ; PPC64LE-NEXT:  .LBB268_3:
@@ -4525,12 +4525,12 @@ define i16 @test269(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test269:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB269_1:
+; PPC64LE-NEXT:  .LBB269_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    extsh 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    ble 0, .LBB269_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB269_1
 ; PPC64LE-NEXT:  .LBB269_3:
@@ -4544,11 +4544,11 @@ define i16 @test269(i16* %ptr, i16 %val) {
 define i32 @test270(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test270:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB270_1:
+; PPC64LE-NEXT:  .LBB270_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB270_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB270_1
 ; PPC64LE-NEXT:  .LBB270_3:
@@ -4562,11 +4562,11 @@ define i32 @test271(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test271:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB271_1:
+; PPC64LE-NEXT:  .LBB271_1: #
 ; PPC64LE-NEXT:    lwarx 3, 0, 5
 ; PPC64LE-NEXT:    cmpw 4, 3
 ; PPC64LE-NEXT:    ble 0, .LBB271_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB271_1
 ; PPC64LE-NEXT:  .LBB271_3:
@@ -4580,11 +4580,11 @@ define i32 @test272(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test272:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB272_1:
+; PPC64LE-NEXT:  .LBB272_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB272_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB272_1
 ; PPC64LE-NEXT:  .LBB272_3:
@@ -4598,11 +4598,11 @@ define i32 @test273(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test273:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB273_1:
+; PPC64LE-NEXT:  .LBB273_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB273_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB273_1
 ; PPC64LE-NEXT:  .LBB273_3:
@@ -4617,11 +4617,11 @@ define i32 @test274(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test274:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB274_1:
+; PPC64LE-NEXT:  .LBB274_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB274_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB274_1
 ; PPC64LE-NEXT:  .LBB274_3:
@@ -4635,11 +4635,11 @@ define i32 @test274(i32* %ptr, i32 %val) {
 define i64 @test275(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test275:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB275_1:
+; PPC64LE-NEXT:  .LBB275_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB275_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB275_1
 ; PPC64LE-NEXT:  .LBB275_3:
@@ -4653,11 +4653,11 @@ define i64 @test276(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test276:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB276_1:
+; PPC64LE-NEXT:  .LBB276_1: #
 ; PPC64LE-NEXT:    ldarx 3, 0, 5
 ; PPC64LE-NEXT:    cmpd 4, 3
 ; PPC64LE-NEXT:    ble 0, .LBB276_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB276_1
 ; PPC64LE-NEXT:  .LBB276_3:
@@ -4671,11 +4671,11 @@ define i64 @test277(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test277:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB277_1:
+; PPC64LE-NEXT:  .LBB277_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB277_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB277_1
 ; PPC64LE-NEXT:  .LBB277_3:
@@ -4689,11 +4689,11 @@ define i64 @test278(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test278:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB278_1:
+; PPC64LE-NEXT:  .LBB278_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB278_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB278_1
 ; PPC64LE-NEXT:  .LBB278_3:
@@ -4708,11 +4708,11 @@ define i64 @test279(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test279:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB279_1:
+; PPC64LE-NEXT:  .LBB279_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB279_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB279_1
 ; PPC64LE-NEXT:  .LBB279_3:
@@ -4726,12 +4726,12 @@ define i64 @test279(i64* %ptr, i64 %val) {
 define i8 @test280(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test280:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB280_1:
+; PPC64LE-NEXT:  .LBB280_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    extsb 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bge 0, .LBB280_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB280_1
 ; PPC64LE-NEXT:  .LBB280_3:
@@ -4745,12 +4745,12 @@ define i8 @test281(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test281:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB281_1:
+; PPC64LE-NEXT:  .LBB281_1: #
 ; PPC64LE-NEXT:    lbarx 3, 0, 5
 ; PPC64LE-NEXT:    extsb 6, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bge 0, .LBB281_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB281_1
 ; PPC64LE-NEXT:  .LBB281_3:
@@ -4764,12 +4764,12 @@ define i8 @test282(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test282:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB282_1:
+; PPC64LE-NEXT:  .LBB282_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    extsb 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bge 0, .LBB282_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB282_1
 ; PPC64LE-NEXT:  .LBB282_3:
@@ -4783,12 +4783,12 @@ define i8 @test283(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test283:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB283_1:
+; PPC64LE-NEXT:  .LBB283_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    extsb 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bge 0, .LBB283_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB283_1
 ; PPC64LE-NEXT:  .LBB283_3:
@@ -4803,12 +4803,12 @@ define i8 @test284(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test284:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB284_1:
+; PPC64LE-NEXT:  .LBB284_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    extsb 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bge 0, .LBB284_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB284_1
 ; PPC64LE-NEXT:  .LBB284_3:
@@ -4822,12 +4822,12 @@ define i8 @test284(i8* %ptr, i8 %val) {
 define i16 @test285(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test285:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB285_1:
+; PPC64LE-NEXT:  .LBB285_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    extsh 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bge 0, .LBB285_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB285_1
 ; PPC64LE-NEXT:  .LBB285_3:
@@ -4841,12 +4841,12 @@ define i16 @test286(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test286:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB286_1:
+; PPC64LE-NEXT:  .LBB286_1: #
 ; PPC64LE-NEXT:    lharx 3, 0, 5
 ; PPC64LE-NEXT:    extsh 6, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bge 0, .LBB286_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB286_1
 ; PPC64LE-NEXT:  .LBB286_3:
@@ -4860,12 +4860,12 @@ define i16 @test287(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test287:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB287_1:
+; PPC64LE-NEXT:  .LBB287_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    extsh 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bge 0, .LBB287_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB287_1
 ; PPC64LE-NEXT:  .LBB287_3:
@@ -4879,12 +4879,12 @@ define i16 @test288(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test288:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB288_1:
+; PPC64LE-NEXT:  .LBB288_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    extsh 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bge 0, .LBB288_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB288_1
 ; PPC64LE-NEXT:  .LBB288_3:
@@ -4899,12 +4899,12 @@ define i16 @test289(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test289:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB289_1:
+; PPC64LE-NEXT:  .LBB289_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    extsh 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bge 0, .LBB289_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB289_1
 ; PPC64LE-NEXT:  .LBB289_3:
@@ -4918,11 +4918,11 @@ define i16 @test289(i16* %ptr, i16 %val) {
 define i32 @test290(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test290:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB290_1:
+; PPC64LE-NEXT:  .LBB290_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB290_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB290_1
 ; PPC64LE-NEXT:  .LBB290_3:
@@ -4936,11 +4936,11 @@ define i32 @test291(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test291:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB291_1:
+; PPC64LE-NEXT:  .LBB291_1: #
 ; PPC64LE-NEXT:    lwarx 3, 0, 5
 ; PPC64LE-NEXT:    cmpw 4, 3
 ; PPC64LE-NEXT:    bge 0, .LBB291_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB291_1
 ; PPC64LE-NEXT:  .LBB291_3:
@@ -4954,11 +4954,11 @@ define i32 @test292(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test292:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB292_1:
+; PPC64LE-NEXT:  .LBB292_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB292_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB292_1
 ; PPC64LE-NEXT:  .LBB292_3:
@@ -4972,11 +4972,11 @@ define i32 @test293(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test293:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB293_1:
+; PPC64LE-NEXT:  .LBB293_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB293_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB293_1
 ; PPC64LE-NEXT:  .LBB293_3:
@@ -4991,11 +4991,11 @@ define i32 @test294(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test294:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB294_1:
+; PPC64LE-NEXT:  .LBB294_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB294_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB294_1
 ; PPC64LE-NEXT:  .LBB294_3:
@@ -5009,11 +5009,11 @@ define i32 @test294(i32* %ptr, i32 %val) {
 define i64 @test295(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test295:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB295_1:
+; PPC64LE-NEXT:  .LBB295_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB295_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB295_1
 ; PPC64LE-NEXT:  .LBB295_3:
@@ -5027,11 +5027,11 @@ define i64 @test296(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test296:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB296_1:
+; PPC64LE-NEXT:  .LBB296_1: #
 ; PPC64LE-NEXT:    ldarx 3, 0, 5
 ; PPC64LE-NEXT:    cmpd 4, 3
 ; PPC64LE-NEXT:    bge 0, .LBB296_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB296_1
 ; PPC64LE-NEXT:  .LBB296_3:
@@ -5045,11 +5045,11 @@ define i64 @test297(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test297:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB297_1:
+; PPC64LE-NEXT:  .LBB297_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB297_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB297_1
 ; PPC64LE-NEXT:  .LBB297_3:
@@ -5063,11 +5063,11 @@ define i64 @test298(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test298:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB298_1:
+; PPC64LE-NEXT:  .LBB298_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB298_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB298_1
 ; PPC64LE-NEXT:  .LBB298_3:
@@ -5082,11 +5082,11 @@ define i64 @test299(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test299:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB299_1:
+; PPC64LE-NEXT:  .LBB299_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB299_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB299_1
 ; PPC64LE-NEXT:  .LBB299_3:
@@ -5100,11 +5100,11 @@ define i64 @test299(i64* %ptr, i64 %val) {
 define i8 @test300(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test300:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB300_1:
+; PPC64LE-NEXT:  .LBB300_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB300_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB300_1
 ; PPC64LE-NEXT:  .LBB300_3:
@@ -5118,11 +5118,11 @@ define i8 @test301(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test301:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB301_1:
+; PPC64LE-NEXT:  .LBB301_1: #
 ; PPC64LE-NEXT:    lbarx 3, 0, 5
 ; PPC64LE-NEXT:    cmplw 4, 3
 ; PPC64LE-NEXT:    ble 0, .LBB301_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB301_1
 ; PPC64LE-NEXT:  .LBB301_3:
@@ -5136,11 +5136,11 @@ define i8 @test302(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test302:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB302_1:
+; PPC64LE-NEXT:  .LBB302_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB302_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB302_1
 ; PPC64LE-NEXT:  .LBB302_3:
@@ -5154,11 +5154,11 @@ define i8 @test303(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test303:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB303_1:
+; PPC64LE-NEXT:  .LBB303_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB303_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB303_1
 ; PPC64LE-NEXT:  .LBB303_3:
@@ -5173,11 +5173,11 @@ define i8 @test304(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test304:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB304_1:
+; PPC64LE-NEXT:  .LBB304_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB304_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB304_1
 ; PPC64LE-NEXT:  .LBB304_3:
@@ -5191,11 +5191,11 @@ define i8 @test304(i8* %ptr, i8 %val) {
 define i16 @test305(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test305:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB305_1:
+; PPC64LE-NEXT:  .LBB305_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB305_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB305_1
 ; PPC64LE-NEXT:  .LBB305_3:
@@ -5209,11 +5209,11 @@ define i16 @test306(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test306:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB306_1:
+; PPC64LE-NEXT:  .LBB306_1: #
 ; PPC64LE-NEXT:    lharx 3, 0, 5
 ; PPC64LE-NEXT:    cmplw 4, 3
 ; PPC64LE-NEXT:    ble 0, .LBB306_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB306_1
 ; PPC64LE-NEXT:  .LBB306_3:
@@ -5227,11 +5227,11 @@ define i16 @test307(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test307:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB307_1:
+; PPC64LE-NEXT:  .LBB307_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB307_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB307_1
 ; PPC64LE-NEXT:  .LBB307_3:
@@ -5245,11 +5245,11 @@ define i16 @test308(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test308:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB308_1:
+; PPC64LE-NEXT:  .LBB308_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB308_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB308_1
 ; PPC64LE-NEXT:  .LBB308_3:
@@ -5264,11 +5264,11 @@ define i16 @test309(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test309:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB309_1:
+; PPC64LE-NEXT:  .LBB309_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB309_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB309_1
 ; PPC64LE-NEXT:  .LBB309_3:
@@ -5282,11 +5282,11 @@ define i16 @test309(i16* %ptr, i16 %val) {
 define i32 @test310(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test310:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB310_1:
+; PPC64LE-NEXT:  .LBB310_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB310_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB310_1
 ; PPC64LE-NEXT:  .LBB310_3:
@@ -5300,11 +5300,11 @@ define i32 @test311(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test311:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB311_1:
+; PPC64LE-NEXT:  .LBB311_1: #
 ; PPC64LE-NEXT:    lwarx 3, 0, 5
 ; PPC64LE-NEXT:    cmplw 4, 3
 ; PPC64LE-NEXT:    ble 0, .LBB311_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB311_1
 ; PPC64LE-NEXT:  .LBB311_3:
@@ -5318,11 +5318,11 @@ define i32 @test312(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test312:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB312_1:
+; PPC64LE-NEXT:  .LBB312_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB312_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB312_1
 ; PPC64LE-NEXT:  .LBB312_3:
@@ -5336,11 +5336,11 @@ define i32 @test313(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test313:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB313_1:
+; PPC64LE-NEXT:  .LBB313_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB313_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB313_1
 ; PPC64LE-NEXT:  .LBB313_3:
@@ -5355,11 +5355,11 @@ define i32 @test314(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test314:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB314_1:
+; PPC64LE-NEXT:  .LBB314_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB314_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB314_1
 ; PPC64LE-NEXT:  .LBB314_3:
@@ -5373,11 +5373,11 @@ define i32 @test314(i32* %ptr, i32 %val) {
 define i64 @test315(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test315:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB315_1:
+; PPC64LE-NEXT:  .LBB315_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpld 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB315_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB315_1
 ; PPC64LE-NEXT:  .LBB315_3:
@@ -5391,11 +5391,11 @@ define i64 @test316(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test316:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB316_1:
+; PPC64LE-NEXT:  .LBB316_1: #
 ; PPC64LE-NEXT:    ldarx 3, 0, 5
 ; PPC64LE-NEXT:    cmpld 4, 3
 ; PPC64LE-NEXT:    ble 0, .LBB316_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB316_1
 ; PPC64LE-NEXT:  .LBB316_3:
@@ -5409,11 +5409,11 @@ define i64 @test317(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test317:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB317_1:
+; PPC64LE-NEXT:  .LBB317_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpld 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB317_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB317_1
 ; PPC64LE-NEXT:  .LBB317_3:
@@ -5427,11 +5427,11 @@ define i64 @test318(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test318:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB318_1:
+; PPC64LE-NEXT:  .LBB318_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpld 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB318_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB318_1
 ; PPC64LE-NEXT:  .LBB318_3:
@@ -5446,11 +5446,11 @@ define i64 @test319(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test319:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB319_1:
+; PPC64LE-NEXT:  .LBB319_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpld 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB319_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB319_1
 ; PPC64LE-NEXT:  .LBB319_3:
@@ -5464,11 +5464,11 @@ define i64 @test319(i64* %ptr, i64 %val) {
 define i8 @test320(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test320:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB320_1:
+; PPC64LE-NEXT:  .LBB320_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB320_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB320_1
 ; PPC64LE-NEXT:  .LBB320_3:
@@ -5482,11 +5482,11 @@ define i8 @test321(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test321:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB321_1:
+; PPC64LE-NEXT:  .LBB321_1: #
 ; PPC64LE-NEXT:    lbarx 3, 0, 5
 ; PPC64LE-NEXT:    cmplw 4, 3
 ; PPC64LE-NEXT:    bge 0, .LBB321_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB321_1
 ; PPC64LE-NEXT:  .LBB321_3:
@@ -5500,11 +5500,11 @@ define i8 @test322(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test322:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB322_1:
+; PPC64LE-NEXT:  .LBB322_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB322_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB322_1
 ; PPC64LE-NEXT:  .LBB322_3:
@@ -5518,11 +5518,11 @@ define i8 @test323(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test323:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB323_1:
+; PPC64LE-NEXT:  .LBB323_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB323_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB323_1
 ; PPC64LE-NEXT:  .LBB323_3:
@@ -5537,11 +5537,11 @@ define i8 @test324(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test324:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB324_1:
+; PPC64LE-NEXT:  .LBB324_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB324_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB324_1
 ; PPC64LE-NEXT:  .LBB324_3:
@@ -5555,11 +5555,11 @@ define i8 @test324(i8* %ptr, i8 %val) {
 define i16 @test325(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test325:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB325_1:
+; PPC64LE-NEXT:  .LBB325_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB325_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB325_1
 ; PPC64LE-NEXT:  .LBB325_3:
@@ -5573,11 +5573,11 @@ define i16 @test326(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test326:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB326_1:
+; PPC64LE-NEXT:  .LBB326_1: #
 ; PPC64LE-NEXT:    lharx 3, 0, 5
 ; PPC64LE-NEXT:    cmplw 4, 3
 ; PPC64LE-NEXT:    bge 0, .LBB326_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB326_1
 ; PPC64LE-NEXT:  .LBB326_3:
@@ -5591,11 +5591,11 @@ define i16 @test327(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test327:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB327_1:
+; PPC64LE-NEXT:  .LBB327_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB327_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB327_1
 ; PPC64LE-NEXT:  .LBB327_3:
@@ -5609,11 +5609,11 @@ define i16 @test328(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test328:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB328_1:
+; PPC64LE-NEXT:  .LBB328_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB328_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB328_1
 ; PPC64LE-NEXT:  .LBB328_3:
@@ -5628,11 +5628,11 @@ define i16 @test329(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test329:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB329_1:
+; PPC64LE-NEXT:  .LBB329_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB329_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB329_1
 ; PPC64LE-NEXT:  .LBB329_3:
@@ -5646,11 +5646,11 @@ define i16 @test329(i16* %ptr, i16 %val) {
 define i32 @test330(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test330:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB330_1:
+; PPC64LE-NEXT:  .LBB330_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB330_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB330_1
 ; PPC64LE-NEXT:  .LBB330_3:
@@ -5664,11 +5664,11 @@ define i32 @test331(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test331:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB331_1:
+; PPC64LE-NEXT:  .LBB331_1: #
 ; PPC64LE-NEXT:    lwarx 3, 0, 5
 ; PPC64LE-NEXT:    cmplw 4, 3
 ; PPC64LE-NEXT:    bge 0, .LBB331_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB331_1
 ; PPC64LE-NEXT:  .LBB331_3:
@@ -5682,11 +5682,11 @@ define i32 @test332(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test332:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB332_1:
+; PPC64LE-NEXT:  .LBB332_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB332_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB332_1
 ; PPC64LE-NEXT:  .LBB332_3:
@@ -5700,11 +5700,11 @@ define i32 @test333(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test333:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB333_1:
+; PPC64LE-NEXT:  .LBB333_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB333_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB333_1
 ; PPC64LE-NEXT:  .LBB333_3:
@@ -5719,11 +5719,11 @@ define i32 @test334(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test334:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB334_1:
+; PPC64LE-NEXT:  .LBB334_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB334_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB334_1
 ; PPC64LE-NEXT:  .LBB334_3:
@@ -5737,11 +5737,11 @@ define i32 @test334(i32* %ptr, i32 %val) {
 define i64 @test335(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test335:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB335_1:
+; PPC64LE-NEXT:  .LBB335_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpld 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB335_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB335_1
 ; PPC64LE-NEXT:  .LBB335_3:
@@ -5755,11 +5755,11 @@ define i64 @test336(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test336:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB336_1:
+; PPC64LE-NEXT:  .LBB336_1: #
 ; PPC64LE-NEXT:    ldarx 3, 0, 5
 ; PPC64LE-NEXT:    cmpld 4, 3
 ; PPC64LE-NEXT:    bge 0, .LBB336_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB336_1
 ; PPC64LE-NEXT:  .LBB336_3:
@@ -5773,11 +5773,11 @@ define i64 @test337(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test337:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB337_1:
+; PPC64LE-NEXT:  .LBB337_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpld 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB337_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB337_1
 ; PPC64LE-NEXT:  .LBB337_3:
@@ -5791,11 +5791,11 @@ define i64 @test338(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test338:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB338_1:
+; PPC64LE-NEXT:  .LBB338_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpld 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB338_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB338_1
 ; PPC64LE-NEXT:  .LBB338_3:
@@ -5810,11 +5810,11 @@ define i64 @test339(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test339:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB339_1:
+; PPC64LE-NEXT:  .LBB339_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpld 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB339_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB339_1
 ; PPC64LE-NEXT:  .LBB339_3:
@@ -5828,7 +5828,7 @@ define i64 @test339(i64* %ptr, i64 %val) {
 define i8 @test340(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test340:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB340_1:
+; PPC64LE-NEXT:  .LBB340_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB340_1
@@ -5843,7 +5843,7 @@ define i8 @test341(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test341:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB341_1:
+; PPC64LE-NEXT:  .LBB341_1: #
 ; PPC64LE-NEXT:    lbarx 3, 0, 5
 ; PPC64LE-NEXT:    stbcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB341_1
@@ -5858,7 +5858,7 @@ define i8 @test342(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test342:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB342_1:
+; PPC64LE-NEXT:  .LBB342_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB342_1
@@ -5873,7 +5873,7 @@ define i8 @test343(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test343:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB343_1:
+; PPC64LE-NEXT:  .LBB343_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB343_1
@@ -5889,7 +5889,7 @@ define i8 @test344(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test344:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB344_1:
+; PPC64LE-NEXT:  .LBB344_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB344_1
@@ -5904,7 +5904,7 @@ define i8 @test344(i8* %ptr, i8 %val) {
 define i16 @test345(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test345:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB345_1:
+; PPC64LE-NEXT:  .LBB345_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB345_1
@@ -5919,7 +5919,7 @@ define i16 @test346(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test346:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB346_1:
+; PPC64LE-NEXT:  .LBB346_1: #
 ; PPC64LE-NEXT:    lharx 3, 0, 5
 ; PPC64LE-NEXT:    sthcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB346_1
@@ -5934,7 +5934,7 @@ define i16 @test347(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test347:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB347_1:
+; PPC64LE-NEXT:  .LBB347_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB347_1
@@ -5949,7 +5949,7 @@ define i16 @test348(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test348:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB348_1:
+; PPC64LE-NEXT:  .LBB348_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB348_1
@@ -5965,7 +5965,7 @@ define i16 @test349(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test349:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB349_1:
+; PPC64LE-NEXT:  .LBB349_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB349_1
@@ -5980,7 +5980,7 @@ define i16 @test349(i16* %ptr, i16 %val) {
 define i32 @test350(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test350:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB350_1:
+; PPC64LE-NEXT:  .LBB350_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB350_1
@@ -5995,7 +5995,7 @@ define i32 @test351(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test351:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB351_1:
+; PPC64LE-NEXT:  .LBB351_1: #
 ; PPC64LE-NEXT:    lwarx 3, 0, 5
 ; PPC64LE-NEXT:    stwcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB351_1
@@ -6010,7 +6010,7 @@ define i32 @test352(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test352:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB352_1:
+; PPC64LE-NEXT:  .LBB352_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB352_1
@@ -6025,7 +6025,7 @@ define i32 @test353(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test353:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB353_1:
+; PPC64LE-NEXT:  .LBB353_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB353_1
@@ -6041,7 +6041,7 @@ define i32 @test354(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test354:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB354_1:
+; PPC64LE-NEXT:  .LBB354_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB354_1
@@ -6056,7 +6056,7 @@ define i32 @test354(i32* %ptr, i32 %val) {
 define i64 @test355(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test355:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB355_1:
+; PPC64LE-NEXT:  .LBB355_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB355_1
@@ -6071,7 +6071,7 @@ define i64 @test356(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test356:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB356_1:
+; PPC64LE-NEXT:  .LBB356_1: #
 ; PPC64LE-NEXT:    ldarx 3, 0, 5
 ; PPC64LE-NEXT:    stdcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB356_1
@@ -6086,7 +6086,7 @@ define i64 @test357(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test357:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB357_1:
+; PPC64LE-NEXT:  .LBB357_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB357_1
@@ -6101,7 +6101,7 @@ define i64 @test358(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test358:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB358_1:
+; PPC64LE-NEXT:  .LBB358_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB358_1
@@ -6117,7 +6117,7 @@ define i64 @test359(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test359:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB359_1:
+; PPC64LE-NEXT:  .LBB359_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB359_1
@@ -6132,7 +6132,7 @@ define i64 @test359(i64* %ptr, i64 %val) {
 define i8 @test360(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test360:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB360_1:
+; PPC64LE-NEXT:  .LBB360_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -6148,7 +6148,7 @@ define i8 @test361(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test361:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB361_1:
+; PPC64LE-NEXT:  .LBB361_1: #
 ; PPC64LE-NEXT:    lbarx 3, 0, 5
 ; PPC64LE-NEXT:    add 6, 4, 3
 ; PPC64LE-NEXT:    stbcx. 6, 0, 5
@@ -6164,7 +6164,7 @@ define i8 @test362(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test362:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB362_1:
+; PPC64LE-NEXT:  .LBB362_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -6180,7 +6180,7 @@ define i8 @test363(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test363:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB363_1:
+; PPC64LE-NEXT:  .LBB363_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -6197,7 +6197,7 @@ define i8 @test364(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test364:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB364_1:
+; PPC64LE-NEXT:  .LBB364_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -6213,7 +6213,7 @@ define i8 @test364(i8* %ptr, i8 %val) {
 define i16 @test365(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test365:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB365_1:
+; PPC64LE-NEXT:  .LBB365_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -6229,7 +6229,7 @@ define i16 @test366(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test366:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB366_1:
+; PPC64LE-NEXT:  .LBB366_1: #
 ; PPC64LE-NEXT:    lharx 3, 0, 5
 ; PPC64LE-NEXT:    add 6, 4, 3
 ; PPC64LE-NEXT:    sthcx. 6, 0, 5
@@ -6245,7 +6245,7 @@ define i16 @test367(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test367:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB367_1:
+; PPC64LE-NEXT:  .LBB367_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -6261,7 +6261,7 @@ define i16 @test368(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test368:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB368_1:
+; PPC64LE-NEXT:  .LBB368_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -6278,7 +6278,7 @@ define i16 @test369(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test369:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB369_1:
+; PPC64LE-NEXT:  .LBB369_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -6294,7 +6294,7 @@ define i16 @test369(i16* %ptr, i16 %val) {
 define i32 @test370(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test370:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB370_1:
+; PPC64LE-NEXT:  .LBB370_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -6310,7 +6310,7 @@ define i32 @test371(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test371:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB371_1:
+; PPC64LE-NEXT:  .LBB371_1: #
 ; PPC64LE-NEXT:    lwarx 3, 0, 5
 ; PPC64LE-NEXT:    add 6, 4, 3
 ; PPC64LE-NEXT:    stwcx. 6, 0, 5
@@ -6326,7 +6326,7 @@ define i32 @test372(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test372:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB372_1:
+; PPC64LE-NEXT:  .LBB372_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -6342,7 +6342,7 @@ define i32 @test373(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test373:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB373_1:
+; PPC64LE-NEXT:  .LBB373_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -6359,7 +6359,7 @@ define i32 @test374(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test374:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB374_1:
+; PPC64LE-NEXT:  .LBB374_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -6375,7 +6375,7 @@ define i32 @test374(i32* %ptr, i32 %val) {
 define i64 @test375(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test375:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB375_1:
+; PPC64LE-NEXT:  .LBB375_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -6391,7 +6391,7 @@ define i64 @test376(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test376:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB376_1:
+; PPC64LE-NEXT:  .LBB376_1: #
 ; PPC64LE-NEXT:    ldarx 3, 0, 5
 ; PPC64LE-NEXT:    add 6, 4, 3
 ; PPC64LE-NEXT:    stdcx. 6, 0, 5
@@ -6407,7 +6407,7 @@ define i64 @test377(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test377:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB377_1:
+; PPC64LE-NEXT:  .LBB377_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -6423,7 +6423,7 @@ define i64 @test378(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test378:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB378_1:
+; PPC64LE-NEXT:  .LBB378_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -6440,7 +6440,7 @@ define i64 @test379(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test379:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB379_1:
+; PPC64LE-NEXT:  .LBB379_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    add 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -6456,7 +6456,7 @@ define i64 @test379(i64* %ptr, i64 %val) {
 define i8 @test380(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test380:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB380_1:
+; PPC64LE-NEXT:  .LBB380_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    subf 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -6472,7 +6472,7 @@ define i8 @test381(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test381:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB381_1:
+; PPC64LE-NEXT:  .LBB381_1: #
 ; PPC64LE-NEXT:    lbarx 3, 0, 5
 ; PPC64LE-NEXT:    subf 6, 4, 3
 ; PPC64LE-NEXT:    stbcx. 6, 0, 5
@@ -6488,7 +6488,7 @@ define i8 @test382(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test382:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB382_1:
+; PPC64LE-NEXT:  .LBB382_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    subf 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -6504,7 +6504,7 @@ define i8 @test383(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test383:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB383_1:
+; PPC64LE-NEXT:  .LBB383_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    subf 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -6521,7 +6521,7 @@ define i8 @test384(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test384:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB384_1:
+; PPC64LE-NEXT:  .LBB384_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    subf 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -6537,7 +6537,7 @@ define i8 @test384(i8* %ptr, i8 %val) {
 define i16 @test385(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test385:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB385_1:
+; PPC64LE-NEXT:  .LBB385_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    subf 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -6553,7 +6553,7 @@ define i16 @test386(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test386:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB386_1:
+; PPC64LE-NEXT:  .LBB386_1: #
 ; PPC64LE-NEXT:    lharx 3, 0, 5
 ; PPC64LE-NEXT:    subf 6, 4, 3
 ; PPC64LE-NEXT:    sthcx. 6, 0, 5
@@ -6569,7 +6569,7 @@ define i16 @test387(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test387:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB387_1:
+; PPC64LE-NEXT:  .LBB387_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    subf 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -6585,7 +6585,7 @@ define i16 @test388(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test388:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB388_1:
+; PPC64LE-NEXT:  .LBB388_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    subf 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -6602,7 +6602,7 @@ define i16 @test389(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test389:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB389_1:
+; PPC64LE-NEXT:  .LBB389_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    subf 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -6618,7 +6618,7 @@ define i16 @test389(i16* %ptr, i16 %val) {
 define i32 @test390(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test390:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB390_1:
+; PPC64LE-NEXT:  .LBB390_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    subf 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -6634,7 +6634,7 @@ define i32 @test391(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test391:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB391_1:
+; PPC64LE-NEXT:  .LBB391_1: #
 ; PPC64LE-NEXT:    lwarx 3, 0, 5
 ; PPC64LE-NEXT:    subf 6, 4, 3
 ; PPC64LE-NEXT:    stwcx. 6, 0, 5
@@ -6650,7 +6650,7 @@ define i32 @test392(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test392:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB392_1:
+; PPC64LE-NEXT:  .LBB392_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    subf 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -6666,7 +6666,7 @@ define i32 @test393(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test393:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB393_1:
+; PPC64LE-NEXT:  .LBB393_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    subf 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -6683,7 +6683,7 @@ define i32 @test394(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test394:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB394_1:
+; PPC64LE-NEXT:  .LBB394_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    subf 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -6699,7 +6699,7 @@ define i32 @test394(i32* %ptr, i32 %val) {
 define i64 @test395(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test395:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB395_1:
+; PPC64LE-NEXT:  .LBB395_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    sub 6, 5, 4
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -6715,7 +6715,7 @@ define i64 @test396(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test396:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB396_1:
+; PPC64LE-NEXT:  .LBB396_1: #
 ; PPC64LE-NEXT:    ldarx 3, 0, 5
 ; PPC64LE-NEXT:    sub 6, 3, 4
 ; PPC64LE-NEXT:    stdcx. 6, 0, 5
@@ -6731,7 +6731,7 @@ define i64 @test397(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test397:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB397_1:
+; PPC64LE-NEXT:  .LBB397_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    sub 6, 5, 4
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -6747,7 +6747,7 @@ define i64 @test398(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test398:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB398_1:
+; PPC64LE-NEXT:  .LBB398_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    sub 6, 5, 4
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -6764,7 +6764,7 @@ define i64 @test399(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test399:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB399_1:
+; PPC64LE-NEXT:  .LBB399_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    sub 6, 5, 4
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -6780,7 +6780,7 @@ define i64 @test399(i64* %ptr, i64 %val) {
 define i8 @test400(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test400:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB400_1:
+; PPC64LE-NEXT:  .LBB400_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -6796,7 +6796,7 @@ define i8 @test401(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test401:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB401_1:
+; PPC64LE-NEXT:  .LBB401_1: #
 ; PPC64LE-NEXT:    lbarx 3, 0, 5
 ; PPC64LE-NEXT:    and 6, 4, 3
 ; PPC64LE-NEXT:    stbcx. 6, 0, 5
@@ -6812,7 +6812,7 @@ define i8 @test402(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test402:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB402_1:
+; PPC64LE-NEXT:  .LBB402_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -6828,7 +6828,7 @@ define i8 @test403(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test403:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB403_1:
+; PPC64LE-NEXT:  .LBB403_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -6845,7 +6845,7 @@ define i8 @test404(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test404:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB404_1:
+; PPC64LE-NEXT:  .LBB404_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -6861,7 +6861,7 @@ define i8 @test404(i8* %ptr, i8 %val) {
 define i16 @test405(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test405:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB405_1:
+; PPC64LE-NEXT:  .LBB405_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -6877,7 +6877,7 @@ define i16 @test406(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test406:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB406_1:
+; PPC64LE-NEXT:  .LBB406_1: #
 ; PPC64LE-NEXT:    lharx 3, 0, 5
 ; PPC64LE-NEXT:    and 6, 4, 3
 ; PPC64LE-NEXT:    sthcx. 6, 0, 5
@@ -6893,7 +6893,7 @@ define i16 @test407(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test407:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB407_1:
+; PPC64LE-NEXT:  .LBB407_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -6909,7 +6909,7 @@ define i16 @test408(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test408:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB408_1:
+; PPC64LE-NEXT:  .LBB408_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -6926,7 +6926,7 @@ define i16 @test409(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test409:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB409_1:
+; PPC64LE-NEXT:  .LBB409_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -6942,7 +6942,7 @@ define i16 @test409(i16* %ptr, i16 %val) {
 define i32 @test410(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test410:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB410_1:
+; PPC64LE-NEXT:  .LBB410_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -6958,7 +6958,7 @@ define i32 @test411(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test411:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB411_1:
+; PPC64LE-NEXT:  .LBB411_1: #
 ; PPC64LE-NEXT:    lwarx 3, 0, 5
 ; PPC64LE-NEXT:    and 6, 4, 3
 ; PPC64LE-NEXT:    stwcx. 6, 0, 5
@@ -6974,7 +6974,7 @@ define i32 @test412(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test412:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB412_1:
+; PPC64LE-NEXT:  .LBB412_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -6990,7 +6990,7 @@ define i32 @test413(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test413:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB413_1:
+; PPC64LE-NEXT:  .LBB413_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -7007,7 +7007,7 @@ define i32 @test414(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test414:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB414_1:
+; PPC64LE-NEXT:  .LBB414_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -7023,7 +7023,7 @@ define i32 @test414(i32* %ptr, i32 %val) {
 define i64 @test415(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test415:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB415_1:
+; PPC64LE-NEXT:  .LBB415_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -7039,7 +7039,7 @@ define i64 @test416(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test416:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB416_1:
+; PPC64LE-NEXT:  .LBB416_1: #
 ; PPC64LE-NEXT:    ldarx 3, 0, 5
 ; PPC64LE-NEXT:    and 6, 4, 3
 ; PPC64LE-NEXT:    stdcx. 6, 0, 5
@@ -7055,7 +7055,7 @@ define i64 @test417(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test417:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB417_1:
+; PPC64LE-NEXT:  .LBB417_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -7071,7 +7071,7 @@ define i64 @test418(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test418:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB418_1:
+; PPC64LE-NEXT:  .LBB418_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -7088,7 +7088,7 @@ define i64 @test419(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test419:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB419_1:
+; PPC64LE-NEXT:  .LBB419_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    and 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -7104,7 +7104,7 @@ define i64 @test419(i64* %ptr, i64 %val) {
 define i8 @test420(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test420:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB420_1:
+; PPC64LE-NEXT:  .LBB420_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -7120,7 +7120,7 @@ define i8 @test421(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test421:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB421_1:
+; PPC64LE-NEXT:  .LBB421_1: #
 ; PPC64LE-NEXT:    lbarx 3, 0, 5
 ; PPC64LE-NEXT:    nand 6, 4, 3
 ; PPC64LE-NEXT:    stbcx. 6, 0, 5
@@ -7136,7 +7136,7 @@ define i8 @test422(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test422:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB422_1:
+; PPC64LE-NEXT:  .LBB422_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -7152,7 +7152,7 @@ define i8 @test423(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test423:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB423_1:
+; PPC64LE-NEXT:  .LBB423_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -7169,7 +7169,7 @@ define i8 @test424(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test424:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB424_1:
+; PPC64LE-NEXT:  .LBB424_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -7185,7 +7185,7 @@ define i8 @test424(i8* %ptr, i8 %val) {
 define i16 @test425(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test425:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB425_1:
+; PPC64LE-NEXT:  .LBB425_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -7201,7 +7201,7 @@ define i16 @test426(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test426:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB426_1:
+; PPC64LE-NEXT:  .LBB426_1: #
 ; PPC64LE-NEXT:    lharx 3, 0, 5
 ; PPC64LE-NEXT:    nand 6, 4, 3
 ; PPC64LE-NEXT:    sthcx. 6, 0, 5
@@ -7217,7 +7217,7 @@ define i16 @test427(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test427:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB427_1:
+; PPC64LE-NEXT:  .LBB427_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -7233,7 +7233,7 @@ define i16 @test428(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test428:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB428_1:
+; PPC64LE-NEXT:  .LBB428_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -7250,7 +7250,7 @@ define i16 @test429(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test429:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB429_1:
+; PPC64LE-NEXT:  .LBB429_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -7266,7 +7266,7 @@ define i16 @test429(i16* %ptr, i16 %val) {
 define i32 @test430(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test430:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB430_1:
+; PPC64LE-NEXT:  .LBB430_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -7282,7 +7282,7 @@ define i32 @test431(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test431:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB431_1:
+; PPC64LE-NEXT:  .LBB431_1: #
 ; PPC64LE-NEXT:    lwarx 3, 0, 5
 ; PPC64LE-NEXT:    nand 6, 4, 3
 ; PPC64LE-NEXT:    stwcx. 6, 0, 5
@@ -7298,7 +7298,7 @@ define i32 @test432(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test432:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB432_1:
+; PPC64LE-NEXT:  .LBB432_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -7314,7 +7314,7 @@ define i32 @test433(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test433:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB433_1:
+; PPC64LE-NEXT:  .LBB433_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -7331,7 +7331,7 @@ define i32 @test434(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test434:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB434_1:
+; PPC64LE-NEXT:  .LBB434_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -7347,7 +7347,7 @@ define i32 @test434(i32* %ptr, i32 %val) {
 define i64 @test435(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test435:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB435_1:
+; PPC64LE-NEXT:  .LBB435_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -7363,7 +7363,7 @@ define i64 @test436(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test436:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB436_1:
+; PPC64LE-NEXT:  .LBB436_1: #
 ; PPC64LE-NEXT:    ldarx 3, 0, 5
 ; PPC64LE-NEXT:    nand 6, 4, 3
 ; PPC64LE-NEXT:    stdcx. 6, 0, 5
@@ -7379,7 +7379,7 @@ define i64 @test437(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test437:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB437_1:
+; PPC64LE-NEXT:  .LBB437_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -7395,7 +7395,7 @@ define i64 @test438(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test438:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB438_1:
+; PPC64LE-NEXT:  .LBB438_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -7412,7 +7412,7 @@ define i64 @test439(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test439:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB439_1:
+; PPC64LE-NEXT:  .LBB439_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    nand 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -7428,7 +7428,7 @@ define i64 @test439(i64* %ptr, i64 %val) {
 define i8 @test440(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test440:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB440_1:
+; PPC64LE-NEXT:  .LBB440_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -7444,7 +7444,7 @@ define i8 @test441(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test441:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB441_1:
+; PPC64LE-NEXT:  .LBB441_1: #
 ; PPC64LE-NEXT:    lbarx 3, 0, 5
 ; PPC64LE-NEXT:    or 6, 4, 3
 ; PPC64LE-NEXT:    stbcx. 6, 0, 5
@@ -7460,7 +7460,7 @@ define i8 @test442(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test442:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB442_1:
+; PPC64LE-NEXT:  .LBB442_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -7476,7 +7476,7 @@ define i8 @test443(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test443:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB443_1:
+; PPC64LE-NEXT:  .LBB443_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -7493,7 +7493,7 @@ define i8 @test444(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test444:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB444_1:
+; PPC64LE-NEXT:  .LBB444_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -7509,7 +7509,7 @@ define i8 @test444(i8* %ptr, i8 %val) {
 define i16 @test445(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test445:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB445_1:
+; PPC64LE-NEXT:  .LBB445_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -7525,7 +7525,7 @@ define i16 @test446(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test446:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB446_1:
+; PPC64LE-NEXT:  .LBB446_1: #
 ; PPC64LE-NEXT:    lharx 3, 0, 5
 ; PPC64LE-NEXT:    or 6, 4, 3
 ; PPC64LE-NEXT:    sthcx. 6, 0, 5
@@ -7541,7 +7541,7 @@ define i16 @test447(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test447:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB447_1:
+; PPC64LE-NEXT:  .LBB447_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -7557,7 +7557,7 @@ define i16 @test448(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test448:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB448_1:
+; PPC64LE-NEXT:  .LBB448_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -7574,7 +7574,7 @@ define i16 @test449(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test449:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB449_1:
+; PPC64LE-NEXT:  .LBB449_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -7590,7 +7590,7 @@ define i16 @test449(i16* %ptr, i16 %val) {
 define i32 @test450(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test450:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB450_1:
+; PPC64LE-NEXT:  .LBB450_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -7606,7 +7606,7 @@ define i32 @test451(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test451:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB451_1:
+; PPC64LE-NEXT:  .LBB451_1: #
 ; PPC64LE-NEXT:    lwarx 3, 0, 5
 ; PPC64LE-NEXT:    or 6, 4, 3
 ; PPC64LE-NEXT:    stwcx. 6, 0, 5
@@ -7622,7 +7622,7 @@ define i32 @test452(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test452:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB452_1:
+; PPC64LE-NEXT:  .LBB452_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -7638,7 +7638,7 @@ define i32 @test453(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test453:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB453_1:
+; PPC64LE-NEXT:  .LBB453_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -7655,7 +7655,7 @@ define i32 @test454(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test454:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB454_1:
+; PPC64LE-NEXT:  .LBB454_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -7671,7 +7671,7 @@ define i32 @test454(i32* %ptr, i32 %val) {
 define i64 @test455(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test455:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB455_1:
+; PPC64LE-NEXT:  .LBB455_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -7687,7 +7687,7 @@ define i64 @test456(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test456:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB456_1:
+; PPC64LE-NEXT:  .LBB456_1: #
 ; PPC64LE-NEXT:    ldarx 3, 0, 5
 ; PPC64LE-NEXT:    or 6, 4, 3
 ; PPC64LE-NEXT:    stdcx. 6, 0, 5
@@ -7703,7 +7703,7 @@ define i64 @test457(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test457:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB457_1:
+; PPC64LE-NEXT:  .LBB457_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -7719,7 +7719,7 @@ define i64 @test458(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test458:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB458_1:
+; PPC64LE-NEXT:  .LBB458_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -7736,7 +7736,7 @@ define i64 @test459(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test459:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB459_1:
+; PPC64LE-NEXT:  .LBB459_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    or 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -7752,7 +7752,7 @@ define i64 @test459(i64* %ptr, i64 %val) {
 define i8 @test460(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test460:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB460_1:
+; PPC64LE-NEXT:  .LBB460_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -7768,7 +7768,7 @@ define i8 @test461(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test461:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB461_1:
+; PPC64LE-NEXT:  .LBB461_1: #
 ; PPC64LE-NEXT:    lbarx 3, 0, 5
 ; PPC64LE-NEXT:    xor 6, 4, 3
 ; PPC64LE-NEXT:    stbcx. 6, 0, 5
@@ -7784,7 +7784,7 @@ define i8 @test462(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test462:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB462_1:
+; PPC64LE-NEXT:  .LBB462_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -7800,7 +7800,7 @@ define i8 @test463(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test463:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB463_1:
+; PPC64LE-NEXT:  .LBB463_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -7817,7 +7817,7 @@ define i8 @test464(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test464:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB464_1:
+; PPC64LE-NEXT:  .LBB464_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    stbcx. 6, 0, 3
@@ -7833,7 +7833,7 @@ define i8 @test464(i8* %ptr, i8 %val) {
 define i16 @test465(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test465:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB465_1:
+; PPC64LE-NEXT:  .LBB465_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -7849,7 +7849,7 @@ define i16 @test466(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test466:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB466_1:
+; PPC64LE-NEXT:  .LBB466_1: #
 ; PPC64LE-NEXT:    lharx 3, 0, 5
 ; PPC64LE-NEXT:    xor 6, 4, 3
 ; PPC64LE-NEXT:    sthcx. 6, 0, 5
@@ -7865,7 +7865,7 @@ define i16 @test467(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test467:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB467_1:
+; PPC64LE-NEXT:  .LBB467_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -7881,7 +7881,7 @@ define i16 @test468(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test468:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB468_1:
+; PPC64LE-NEXT:  .LBB468_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -7898,7 +7898,7 @@ define i16 @test469(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test469:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB469_1:
+; PPC64LE-NEXT:  .LBB469_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    sthcx. 6, 0, 3
@@ -7914,7 +7914,7 @@ define i16 @test469(i16* %ptr, i16 %val) {
 define i32 @test470(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test470:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB470_1:
+; PPC64LE-NEXT:  .LBB470_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -7930,7 +7930,7 @@ define i32 @test471(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test471:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB471_1:
+; PPC64LE-NEXT:  .LBB471_1: #
 ; PPC64LE-NEXT:    lwarx 3, 0, 5
 ; PPC64LE-NEXT:    xor 6, 4, 3
 ; PPC64LE-NEXT:    stwcx. 6, 0, 5
@@ -7946,7 +7946,7 @@ define i32 @test472(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test472:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB472_1:
+; PPC64LE-NEXT:  .LBB472_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -7962,7 +7962,7 @@ define i32 @test473(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test473:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB473_1:
+; PPC64LE-NEXT:  .LBB473_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -7979,7 +7979,7 @@ define i32 @test474(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test474:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB474_1:
+; PPC64LE-NEXT:  .LBB474_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    stwcx. 6, 0, 3
@@ -7995,7 +7995,7 @@ define i32 @test474(i32* %ptr, i32 %val) {
 define i64 @test475(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test475:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB475_1:
+; PPC64LE-NEXT:  .LBB475_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -8011,7 +8011,7 @@ define i64 @test476(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test476:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB476_1:
+; PPC64LE-NEXT:  .LBB476_1: #
 ; PPC64LE-NEXT:    ldarx 3, 0, 5
 ; PPC64LE-NEXT:    xor 6, 4, 3
 ; PPC64LE-NEXT:    stdcx. 6, 0, 5
@@ -8027,7 +8027,7 @@ define i64 @test477(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test477:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB477_1:
+; PPC64LE-NEXT:  .LBB477_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -8043,7 +8043,7 @@ define i64 @test478(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test478:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB478_1:
+; PPC64LE-NEXT:  .LBB478_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -8060,7 +8060,7 @@ define i64 @test479(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test479:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB479_1:
+; PPC64LE-NEXT:  .LBB479_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    xor 6, 4, 5
 ; PPC64LE-NEXT:    stdcx. 6, 0, 3
@@ -8076,12 +8076,12 @@ define i64 @test479(i64* %ptr, i64 %val) {
 define i8 @test480(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test480:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB480_1:
+; PPC64LE-NEXT:  .LBB480_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    extsb 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    ble 0, .LBB480_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB480_1
 ; PPC64LE-NEXT:  .LBB480_3:
@@ -8095,12 +8095,12 @@ define i8 @test481(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test481:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB481_1:
+; PPC64LE-NEXT:  .LBB481_1: #
 ; PPC64LE-NEXT:    lbarx 3, 0, 5
 ; PPC64LE-NEXT:    extsb 6, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    ble 0, .LBB481_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB481_1
 ; PPC64LE-NEXT:  .LBB481_3:
@@ -8114,12 +8114,12 @@ define i8 @test482(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test482:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB482_1:
+; PPC64LE-NEXT:  .LBB482_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    extsb 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    ble 0, .LBB482_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB482_1
 ; PPC64LE-NEXT:  .LBB482_3:
@@ -8133,12 +8133,12 @@ define i8 @test483(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test483:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB483_1:
+; PPC64LE-NEXT:  .LBB483_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    extsb 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    ble 0, .LBB483_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB483_1
 ; PPC64LE-NEXT:  .LBB483_3:
@@ -8153,12 +8153,12 @@ define i8 @test484(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test484:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB484_1:
+; PPC64LE-NEXT:  .LBB484_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    extsb 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    ble 0, .LBB484_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB484_1
 ; PPC64LE-NEXT:  .LBB484_3:
@@ -8172,12 +8172,12 @@ define i8 @test484(i8* %ptr, i8 %val) {
 define i16 @test485(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test485:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB485_1:
+; PPC64LE-NEXT:  .LBB485_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    extsh 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    ble 0, .LBB485_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB485_1
 ; PPC64LE-NEXT:  .LBB485_3:
@@ -8191,12 +8191,12 @@ define i16 @test486(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test486:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB486_1:
+; PPC64LE-NEXT:  .LBB486_1: #
 ; PPC64LE-NEXT:    lharx 3, 0, 5
 ; PPC64LE-NEXT:    extsh 6, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    ble 0, .LBB486_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB486_1
 ; PPC64LE-NEXT:  .LBB486_3:
@@ -8210,12 +8210,12 @@ define i16 @test487(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test487:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB487_1:
+; PPC64LE-NEXT:  .LBB487_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    extsh 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    ble 0, .LBB487_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB487_1
 ; PPC64LE-NEXT:  .LBB487_3:
@@ -8229,12 +8229,12 @@ define i16 @test488(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test488:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB488_1:
+; PPC64LE-NEXT:  .LBB488_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    extsh 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    ble 0, .LBB488_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB488_1
 ; PPC64LE-NEXT:  .LBB488_3:
@@ -8249,12 +8249,12 @@ define i16 @test489(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test489:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB489_1:
+; PPC64LE-NEXT:  .LBB489_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    extsh 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    ble 0, .LBB489_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB489_1
 ; PPC64LE-NEXT:  .LBB489_3:
@@ -8268,11 +8268,11 @@ define i16 @test489(i16* %ptr, i16 %val) {
 define i32 @test490(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test490:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB490_1:
+; PPC64LE-NEXT:  .LBB490_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB490_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB490_1
 ; PPC64LE-NEXT:  .LBB490_3:
@@ -8286,11 +8286,11 @@ define i32 @test491(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test491:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB491_1:
+; PPC64LE-NEXT:  .LBB491_1: #
 ; PPC64LE-NEXT:    lwarx 3, 0, 5
 ; PPC64LE-NEXT:    cmpw 4, 3
 ; PPC64LE-NEXT:    ble 0, .LBB491_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB491_1
 ; PPC64LE-NEXT:  .LBB491_3:
@@ -8304,11 +8304,11 @@ define i32 @test492(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test492:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB492_1:
+; PPC64LE-NEXT:  .LBB492_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB492_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB492_1
 ; PPC64LE-NEXT:  .LBB492_3:
@@ -8322,11 +8322,11 @@ define i32 @test493(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test493:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB493_1:
+; PPC64LE-NEXT:  .LBB493_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB493_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB493_1
 ; PPC64LE-NEXT:  .LBB493_3:
@@ -8341,11 +8341,11 @@ define i32 @test494(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test494:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB494_1:
+; PPC64LE-NEXT:  .LBB494_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB494_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB494_1
 ; PPC64LE-NEXT:  .LBB494_3:
@@ -8359,11 +8359,11 @@ define i32 @test494(i32* %ptr, i32 %val) {
 define i64 @test495(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test495:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB495_1:
+; PPC64LE-NEXT:  .LBB495_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB495_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB495_1
 ; PPC64LE-NEXT:  .LBB495_3:
@@ -8377,11 +8377,11 @@ define i64 @test496(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test496:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB496_1:
+; PPC64LE-NEXT:  .LBB496_1: #
 ; PPC64LE-NEXT:    ldarx 3, 0, 5
 ; PPC64LE-NEXT:    cmpd 4, 3
 ; PPC64LE-NEXT:    ble 0, .LBB496_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB496_1
 ; PPC64LE-NEXT:  .LBB496_3:
@@ -8395,11 +8395,11 @@ define i64 @test497(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test497:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB497_1:
+; PPC64LE-NEXT:  .LBB497_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB497_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB497_1
 ; PPC64LE-NEXT:  .LBB497_3:
@@ -8413,11 +8413,11 @@ define i64 @test498(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test498:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB498_1:
+; PPC64LE-NEXT:  .LBB498_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB498_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB498_1
 ; PPC64LE-NEXT:  .LBB498_3:
@@ -8432,11 +8432,11 @@ define i64 @test499(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test499:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB499_1:
+; PPC64LE-NEXT:  .LBB499_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB499_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB499_1
 ; PPC64LE-NEXT:  .LBB499_3:
@@ -8450,12 +8450,12 @@ define i64 @test499(i64* %ptr, i64 %val) {
 define i8 @test500(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test500:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB500_1:
+; PPC64LE-NEXT:  .LBB500_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    extsb 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bge 0, .LBB500_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB500_1
 ; PPC64LE-NEXT:  .LBB500_3:
@@ -8469,12 +8469,12 @@ define i8 @test501(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test501:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB501_1:
+; PPC64LE-NEXT:  .LBB501_1: #
 ; PPC64LE-NEXT:    lbarx 3, 0, 5
 ; PPC64LE-NEXT:    extsb 6, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bge 0, .LBB501_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB501_1
 ; PPC64LE-NEXT:  .LBB501_3:
@@ -8488,12 +8488,12 @@ define i8 @test502(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test502:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB502_1:
+; PPC64LE-NEXT:  .LBB502_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    extsb 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bge 0, .LBB502_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB502_1
 ; PPC64LE-NEXT:  .LBB502_3:
@@ -8507,12 +8507,12 @@ define i8 @test503(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test503:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB503_1:
+; PPC64LE-NEXT:  .LBB503_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    extsb 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bge 0, .LBB503_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB503_1
 ; PPC64LE-NEXT:  .LBB503_3:
@@ -8527,12 +8527,12 @@ define i8 @test504(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test504:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB504_1:
+; PPC64LE-NEXT:  .LBB504_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    extsb 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bge 0, .LBB504_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB504_1
 ; PPC64LE-NEXT:  .LBB504_3:
@@ -8546,12 +8546,12 @@ define i8 @test504(i8* %ptr, i8 %val) {
 define i16 @test505(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test505:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB505_1:
+; PPC64LE-NEXT:  .LBB505_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    extsh 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bge 0, .LBB505_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB505_1
 ; PPC64LE-NEXT:  .LBB505_3:
@@ -8565,12 +8565,12 @@ define i16 @test506(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test506:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB506_1:
+; PPC64LE-NEXT:  .LBB506_1: #
 ; PPC64LE-NEXT:    lharx 3, 0, 5
 ; PPC64LE-NEXT:    extsh 6, 3
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bge 0, .LBB506_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB506_1
 ; PPC64LE-NEXT:  .LBB506_3:
@@ -8584,12 +8584,12 @@ define i16 @test507(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test507:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB507_1:
+; PPC64LE-NEXT:  .LBB507_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    extsh 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bge 0, .LBB507_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB507_1
 ; PPC64LE-NEXT:  .LBB507_3:
@@ -8603,12 +8603,12 @@ define i16 @test508(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test508:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB508_1:
+; PPC64LE-NEXT:  .LBB508_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    extsh 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bge 0, .LBB508_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB508_1
 ; PPC64LE-NEXT:  .LBB508_3:
@@ -8623,12 +8623,12 @@ define i16 @test509(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test509:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB509_1:
+; PPC64LE-NEXT:  .LBB509_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    extsh 6, 5
 ; PPC64LE-NEXT:    cmpw 4, 6
 ; PPC64LE-NEXT:    bge 0, .LBB509_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB509_1
 ; PPC64LE-NEXT:  .LBB509_3:
@@ -8642,11 +8642,11 @@ define i16 @test509(i16* %ptr, i16 %val) {
 define i32 @test510(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test510:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB510_1:
+; PPC64LE-NEXT:  .LBB510_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB510_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB510_1
 ; PPC64LE-NEXT:  .LBB510_3:
@@ -8660,11 +8660,11 @@ define i32 @test511(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test511:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB511_1:
+; PPC64LE-NEXT:  .LBB511_1: #
 ; PPC64LE-NEXT:    lwarx 3, 0, 5
 ; PPC64LE-NEXT:    cmpw 4, 3
 ; PPC64LE-NEXT:    bge 0, .LBB511_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB511_1
 ; PPC64LE-NEXT:  .LBB511_3:
@@ -8678,11 +8678,11 @@ define i32 @test512(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test512:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB512_1:
+; PPC64LE-NEXT:  .LBB512_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB512_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB512_1
 ; PPC64LE-NEXT:  .LBB512_3:
@@ -8696,11 +8696,11 @@ define i32 @test513(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test513:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB513_1:
+; PPC64LE-NEXT:  .LBB513_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB513_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB513_1
 ; PPC64LE-NEXT:  .LBB513_3:
@@ -8715,11 +8715,11 @@ define i32 @test514(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test514:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB514_1:
+; PPC64LE-NEXT:  .LBB514_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB514_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB514_1
 ; PPC64LE-NEXT:  .LBB514_3:
@@ -8733,11 +8733,11 @@ define i32 @test514(i32* %ptr, i32 %val) {
 define i64 @test515(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test515:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB515_1:
+; PPC64LE-NEXT:  .LBB515_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB515_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB515_1
 ; PPC64LE-NEXT:  .LBB515_3:
@@ -8751,11 +8751,11 @@ define i64 @test516(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test516:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB516_1:
+; PPC64LE-NEXT:  .LBB516_1: #
 ; PPC64LE-NEXT:    ldarx 3, 0, 5
 ; PPC64LE-NEXT:    cmpd 4, 3
 ; PPC64LE-NEXT:    bge 0, .LBB516_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB516_1
 ; PPC64LE-NEXT:  .LBB516_3:
@@ -8769,11 +8769,11 @@ define i64 @test517(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test517:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB517_1:
+; PPC64LE-NEXT:  .LBB517_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB517_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB517_1
 ; PPC64LE-NEXT:  .LBB517_3:
@@ -8787,11 +8787,11 @@ define i64 @test518(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test518:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB518_1:
+; PPC64LE-NEXT:  .LBB518_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB518_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB518_1
 ; PPC64LE-NEXT:  .LBB518_3:
@@ -8806,11 +8806,11 @@ define i64 @test519(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test519:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB519_1:
+; PPC64LE-NEXT:  .LBB519_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpd 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB519_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB519_1
 ; PPC64LE-NEXT:  .LBB519_3:
@@ -8824,11 +8824,11 @@ define i64 @test519(i64* %ptr, i64 %val) {
 define i8 @test520(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test520:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB520_1:
+; PPC64LE-NEXT:  .LBB520_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB520_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB520_1
 ; PPC64LE-NEXT:  .LBB520_3:
@@ -8842,11 +8842,11 @@ define i8 @test521(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test521:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB521_1:
+; PPC64LE-NEXT:  .LBB521_1: #
 ; PPC64LE-NEXT:    lbarx 3, 0, 5
 ; PPC64LE-NEXT:    cmplw 4, 3
 ; PPC64LE-NEXT:    ble 0, .LBB521_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB521_1
 ; PPC64LE-NEXT:  .LBB521_3:
@@ -8860,11 +8860,11 @@ define i8 @test522(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test522:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB522_1:
+; PPC64LE-NEXT:  .LBB522_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB522_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB522_1
 ; PPC64LE-NEXT:  .LBB522_3:
@@ -8878,11 +8878,11 @@ define i8 @test523(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test523:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB523_1:
+; PPC64LE-NEXT:  .LBB523_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB523_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB523_1
 ; PPC64LE-NEXT:  .LBB523_3:
@@ -8897,11 +8897,11 @@ define i8 @test524(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test524:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB524_1:
+; PPC64LE-NEXT:  .LBB524_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB524_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB524_1
 ; PPC64LE-NEXT:  .LBB524_3:
@@ -8915,11 +8915,11 @@ define i8 @test524(i8* %ptr, i8 %val) {
 define i16 @test525(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test525:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB525_1:
+; PPC64LE-NEXT:  .LBB525_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB525_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB525_1
 ; PPC64LE-NEXT:  .LBB525_3:
@@ -8933,11 +8933,11 @@ define i16 @test526(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test526:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB526_1:
+; PPC64LE-NEXT:  .LBB526_1: #
 ; PPC64LE-NEXT:    lharx 3, 0, 5
 ; PPC64LE-NEXT:    cmplw 4, 3
 ; PPC64LE-NEXT:    ble 0, .LBB526_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB526_1
 ; PPC64LE-NEXT:  .LBB526_3:
@@ -8951,11 +8951,11 @@ define i16 @test527(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test527:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB527_1:
+; PPC64LE-NEXT:  .LBB527_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB527_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB527_1
 ; PPC64LE-NEXT:  .LBB527_3:
@@ -8969,11 +8969,11 @@ define i16 @test528(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test528:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB528_1:
+; PPC64LE-NEXT:  .LBB528_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB528_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB528_1
 ; PPC64LE-NEXT:  .LBB528_3:
@@ -8988,11 +8988,11 @@ define i16 @test529(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test529:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB529_1:
+; PPC64LE-NEXT:  .LBB529_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB529_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB529_1
 ; PPC64LE-NEXT:  .LBB529_3:
@@ -9006,11 +9006,11 @@ define i16 @test529(i16* %ptr, i16 %val) {
 define i32 @test530(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test530:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB530_1:
+; PPC64LE-NEXT:  .LBB530_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB530_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB530_1
 ; PPC64LE-NEXT:  .LBB530_3:
@@ -9024,11 +9024,11 @@ define i32 @test531(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test531:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB531_1:
+; PPC64LE-NEXT:  .LBB531_1: #
 ; PPC64LE-NEXT:    lwarx 3, 0, 5
 ; PPC64LE-NEXT:    cmplw 4, 3
 ; PPC64LE-NEXT:    ble 0, .LBB531_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB531_1
 ; PPC64LE-NEXT:  .LBB531_3:
@@ -9042,11 +9042,11 @@ define i32 @test532(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test532:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB532_1:
+; PPC64LE-NEXT:  .LBB532_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB532_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB532_1
 ; PPC64LE-NEXT:  .LBB532_3:
@@ -9060,11 +9060,11 @@ define i32 @test533(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test533:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB533_1:
+; PPC64LE-NEXT:  .LBB533_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB533_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB533_1
 ; PPC64LE-NEXT:  .LBB533_3:
@@ -9079,11 +9079,11 @@ define i32 @test534(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test534:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB534_1:
+; PPC64LE-NEXT:  .LBB534_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB534_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB534_1
 ; PPC64LE-NEXT:  .LBB534_3:
@@ -9097,11 +9097,11 @@ define i32 @test534(i32* %ptr, i32 %val) {
 define i64 @test535(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test535:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB535_1:
+; PPC64LE-NEXT:  .LBB535_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpld 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB535_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB535_1
 ; PPC64LE-NEXT:  .LBB535_3:
@@ -9115,11 +9115,11 @@ define i64 @test536(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test536:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB536_1:
+; PPC64LE-NEXT:  .LBB536_1: #
 ; PPC64LE-NEXT:    ldarx 3, 0, 5
 ; PPC64LE-NEXT:    cmpld 4, 3
 ; PPC64LE-NEXT:    ble 0, .LBB536_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB536_1
 ; PPC64LE-NEXT:  .LBB536_3:
@@ -9133,11 +9133,11 @@ define i64 @test537(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test537:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB537_1:
+; PPC64LE-NEXT:  .LBB537_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpld 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB537_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB537_1
 ; PPC64LE-NEXT:  .LBB537_3:
@@ -9151,11 +9151,11 @@ define i64 @test538(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test538:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB538_1:
+; PPC64LE-NEXT:  .LBB538_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpld 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB538_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB538_1
 ; PPC64LE-NEXT:  .LBB538_3:
@@ -9170,11 +9170,11 @@ define i64 @test539(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test539:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB539_1:
+; PPC64LE-NEXT:  .LBB539_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpld 4, 5
 ; PPC64LE-NEXT:    ble 0, .LBB539_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB539_1
 ; PPC64LE-NEXT:  .LBB539_3:
@@ -9188,11 +9188,11 @@ define i64 @test539(i64* %ptr, i64 %val) {
 define i8 @test540(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test540:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB540_1:
+; PPC64LE-NEXT:  .LBB540_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB540_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB540_1
 ; PPC64LE-NEXT:  .LBB540_3:
@@ -9206,11 +9206,11 @@ define i8 @test541(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test541:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB541_1:
+; PPC64LE-NEXT:  .LBB541_1: #
 ; PPC64LE-NEXT:    lbarx 3, 0, 5
 ; PPC64LE-NEXT:    cmplw 4, 3
 ; PPC64LE-NEXT:    bge 0, .LBB541_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB541_1
 ; PPC64LE-NEXT:  .LBB541_3:
@@ -9224,11 +9224,11 @@ define i8 @test542(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test542:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB542_1:
+; PPC64LE-NEXT:  .LBB542_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB542_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB542_1
 ; PPC64LE-NEXT:  .LBB542_3:
@@ -9242,11 +9242,11 @@ define i8 @test543(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test543:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB543_1:
+; PPC64LE-NEXT:  .LBB543_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB543_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB543_1
 ; PPC64LE-NEXT:  .LBB543_3:
@@ -9261,11 +9261,11 @@ define i8 @test544(i8* %ptr, i8 %val) {
 ; PPC64LE-LABEL: test544:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB544_1:
+; PPC64LE-NEXT:  .LBB544_1: #
 ; PPC64LE-NEXT:    lbarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB544_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stbcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB544_1
 ; PPC64LE-NEXT:  .LBB544_3:
@@ -9279,11 +9279,11 @@ define i8 @test544(i8* %ptr, i8 %val) {
 define i16 @test545(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test545:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB545_1:
+; PPC64LE-NEXT:  .LBB545_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB545_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB545_1
 ; PPC64LE-NEXT:  .LBB545_3:
@@ -9297,11 +9297,11 @@ define i16 @test546(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test546:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB546_1:
+; PPC64LE-NEXT:  .LBB546_1: #
 ; PPC64LE-NEXT:    lharx 3, 0, 5
 ; PPC64LE-NEXT:    cmplw 4, 3
 ; PPC64LE-NEXT:    bge 0, .LBB546_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB546_1
 ; PPC64LE-NEXT:  .LBB546_3:
@@ -9315,11 +9315,11 @@ define i16 @test547(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test547:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB547_1:
+; PPC64LE-NEXT:  .LBB547_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB547_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB547_1
 ; PPC64LE-NEXT:  .LBB547_3:
@@ -9333,11 +9333,11 @@ define i16 @test548(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test548:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB548_1:
+; PPC64LE-NEXT:  .LBB548_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB548_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB548_1
 ; PPC64LE-NEXT:  .LBB548_3:
@@ -9352,11 +9352,11 @@ define i16 @test549(i16* %ptr, i16 %val) {
 ; PPC64LE-LABEL: test549:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB549_1:
+; PPC64LE-NEXT:  .LBB549_1: #
 ; PPC64LE-NEXT:    lharx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB549_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    sthcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB549_1
 ; PPC64LE-NEXT:  .LBB549_3:
@@ -9370,11 +9370,11 @@ define i16 @test549(i16* %ptr, i16 %val) {
 define i32 @test550(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test550:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB550_1:
+; PPC64LE-NEXT:  .LBB550_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB550_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB550_1
 ; PPC64LE-NEXT:  .LBB550_3:
@@ -9388,11 +9388,11 @@ define i32 @test551(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test551:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB551_1:
+; PPC64LE-NEXT:  .LBB551_1: #
 ; PPC64LE-NEXT:    lwarx 3, 0, 5
 ; PPC64LE-NEXT:    cmplw 4, 3
 ; PPC64LE-NEXT:    bge 0, .LBB551_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB551_1
 ; PPC64LE-NEXT:  .LBB551_3:
@@ -9406,11 +9406,11 @@ define i32 @test552(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test552:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB552_1:
+; PPC64LE-NEXT:  .LBB552_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB552_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB552_1
 ; PPC64LE-NEXT:  .LBB552_3:
@@ -9424,11 +9424,11 @@ define i32 @test553(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test553:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB553_1:
+; PPC64LE-NEXT:  .LBB553_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB553_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB553_1
 ; PPC64LE-NEXT:  .LBB553_3:
@@ -9443,11 +9443,11 @@ define i32 @test554(i32* %ptr, i32 %val) {
 ; PPC64LE-LABEL: test554:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB554_1:
+; PPC64LE-NEXT:  .LBB554_1: #
 ; PPC64LE-NEXT:    lwarx 5, 0, 3
 ; PPC64LE-NEXT:    cmplw 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB554_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stwcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB554_1
 ; PPC64LE-NEXT:  .LBB554_3:
@@ -9461,11 +9461,11 @@ define i32 @test554(i32* %ptr, i32 %val) {
 define i64 @test555(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test555:
 ; PPC64LE:       # %bb.0:
-; PPC64LE-NEXT:  .LBB555_1:
+; PPC64LE-NEXT:  .LBB555_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpld 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB555_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB555_1
 ; PPC64LE-NEXT:  .LBB555_3:
@@ -9479,11 +9479,11 @@ define i64 @test556(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test556:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    mr 5, 3
-; PPC64LE-NEXT:  .LBB556_1:
+; PPC64LE-NEXT:  .LBB556_1: #
 ; PPC64LE-NEXT:    ldarx 3, 0, 5
 ; PPC64LE-NEXT:    cmpld 4, 3
 ; PPC64LE-NEXT:    bge 0, .LBB556_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 5
 ; PPC64LE-NEXT:    bne 0, .LBB556_1
 ; PPC64LE-NEXT:  .LBB556_3:
@@ -9497,11 +9497,11 @@ define i64 @test557(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test557:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB557_1:
+; PPC64LE-NEXT:  .LBB557_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpld 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB557_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB557_1
 ; PPC64LE-NEXT:  .LBB557_3:
@@ -9515,11 +9515,11 @@ define i64 @test558(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test558:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    lwsync
-; PPC64LE-NEXT:  .LBB558_1:
+; PPC64LE-NEXT:  .LBB558_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpld 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB558_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB558_1
 ; PPC64LE-NEXT:  .LBB558_3:
@@ -9534,11 +9534,11 @@ define i64 @test559(i64* %ptr, i64 %val) {
 ; PPC64LE-LABEL: test559:
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    sync
-; PPC64LE-NEXT:  .LBB559_1:
+; PPC64LE-NEXT:  .LBB559_1: #
 ; PPC64LE-NEXT:    ldarx 5, 0, 3
 ; PPC64LE-NEXT:    cmpld 4, 5
 ; PPC64LE-NEXT:    bge 0, .LBB559_3
-; PPC64LE-NEXT:  # %bb.2:
+; PPC64LE-NEXT:  # %bb.2: #
 ; PPC64LE-NEXT:    stdcx. 4, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB559_1
 ; PPC64LE-NEXT:  .LBB559_3:
diff --git a/llvm/test/CodeGen/PowerPC/fmf-propagation.ll b/llvm/test/CodeGen/PowerPC/fmf-propagation.ll
index 222583638d591..59a7d233c0c3b 100644
--- a/llvm/test/CodeGen/PowerPC/fmf-propagation.ll
+++ b/llvm/test/CodeGen/PowerPC/fmf-propagation.ll
@@ -270,11 +270,11 @@ define float @fmul_fma_fast2(float %x) {
 ; Reduced precision for sqrt is allowed - should use estimate and NR iterations.
 
 ; FMFDEBUG-LABEL: Optimized lowered selection DAG: %bb.0 'sqrt_afn_ieee:'
-; FMFDEBUG:         fmul afn {{t[0-9]+}}
+; FMFDEBUG:         fmul ninf afn {{t[0-9]+}}
 ; FMFDEBUG:       Type-legalized selection DAG: %bb.0 'sqrt_afn_ieee:'
 
 ; GLOBALDEBUG-LABEL: Optimized lowered selection DAG: %bb.0 'sqrt_afn_ieee:'
-; GLOBALDEBUG:         fmul afn {{t[0-9]+}}
+; GLOBALDEBUG:         fmul ninf afn {{t[0-9]+}}
 ; GLOBALDEBUG:       Type-legalized selection DAG: %bb.0 'sqrt_afn_ieee:'
 
 define float @sqrt_afn_ieee(float %x) #0 {
@@ -321,17 +321,31 @@ define float @sqrt_afn_ieee(float %x) #0 {
 ; GLOBAL-NEXT:    xsmulsp 0, 0, 2
 ; GLOBAL-NEXT:  .LBB10_2:
 ; GLOBAL-NEXT:    fmr 1, 0
+; GLOBAL-NEXT:    blr
+  %rt = call afn ninf float @llvm.sqrt.f32(float %x)
+  ret float %rt
+}
+
+define float @sqrt_afn_ieee_inf(float %x) #0 {
+; FMF-LABEL: sqrt_afn_ieee_inf:
+; FMF:       # %bb.0:
+; FMF-NEXT:    xssqrtsp 1, 1
+; FMF-NEXT:    blr
+;
+; GLOBAL-LABEL: sqrt_afn_ieee_inf:
+; GLOBAL:       # %bb.0:
+; GLOBAL-NEXT:    xssqrtsp 1, 1
 ; GLOBAL-NEXT:    blr
   %rt = call afn float @llvm.sqrt.f32(float %x)
   ret float %rt
 }
 
 ; FMFDEBUG-LABEL: Optimized lowered selection DAG: %bb.0 'sqrt_afn_preserve_sign:'
-; FMFDEBUG:         fmul afn {{t[0-9]+}}
+; FMFDEBUG:         fmul ninf afn {{t[0-9]+}}
 ; FMFDEBUG:       Type-legalized selection DAG: %bb.0 'sqrt_afn_preserve_sign:'
 
 ; GLOBALDEBUG-LABEL: Optimized lowered selection DAG: %bb.0 'sqrt_afn_preserve_sign:'
-; GLOBALDEBUG:         fmul afn {{t[0-9]+}}
+; GLOBALDEBUG:         fmul ninf afn {{t[0-9]+}}
 ; GLOBALDEBUG:       Type-legalized selection DAG: %bb.0 'sqrt_afn_preserve_sign:'
 
 define float @sqrt_afn_preserve_sign(float %x) #1 {
@@ -339,19 +353,19 @@ define float @sqrt_afn_preserve_sign(float %x) #1 {
 ; FMF:       # %bb.0:
 ; FMF-NEXT:    xxlxor 0, 0, 0
 ; FMF-NEXT:    fcmpu 0, 1, 0
-; FMF-NEXT:    beq 0, .LBB11_2
+; FMF-NEXT:    beq 0, .LBB12_2
 ; FMF-NEXT:  # %bb.1:
 ; FMF-NEXT:    xsrsqrtesp 0, 1
-; FMF-NEXT:    addis 3, 2, .LCPI11_0@toc@ha
-; FMF-NEXT:    addis 4, 2, .LCPI11_1@toc@ha
-; FMF-NEXT:    lfs 2, .LCPI11_0@toc@l(3)
-; FMF-NEXT:    lfs 3, .LCPI11_1@toc@l(4)
+; FMF-NEXT:    addis 3, 2, .LCPI12_0@toc@ha
+; FMF-NEXT:    addis 4, 2, .LCPI12_1@toc@ha
+; FMF-NEXT:    lfs 2, .LCPI12_0@toc@l(3)
+; FMF-NEXT:    lfs 3, .LCPI12_1@toc@l(4)
 ; FMF-NEXT:    xsmulsp 1, 1, 0
 ; FMF-NEXT:    xsmulsp 0, 1, 0
 ; FMF-NEXT:    xsmulsp 1, 1, 2
 ; FMF-NEXT:    xsaddsp 0, 0, 3
 ; FMF-NEXT:    xsmulsp 0, 1, 0
-; FMF-NEXT:  .LBB11_2:
+; FMF-NEXT:  .LBB12_2:
 ; FMF-NEXT:    fmr 1, 0
 ; FMF-NEXT:    blr
 ;
@@ -359,19 +373,33 @@ define float @sqrt_afn_preserve_sign(float %x) #1 {
 ; GLOBAL:       # %bb.0:
 ; GLOBAL-NEXT:    xxlxor 0, 0, 0
 ; GLOBAL-NEXT:    fcmpu 0, 1, 0
-; GLOBAL-NEXT:    beq 0, .LBB11_2
+; GLOBAL-NEXT:    beq 0, .LBB12_2
 ; GLOBAL-NEXT:  # %bb.1:
 ; GLOBAL-NEXT:    xsrsqrtesp 0, 1
-; GLOBAL-NEXT:    addis 3, 2, .LCPI11_0@toc@ha
-; GLOBAL-NEXT:    addis 4, 2, .LCPI11_1@toc@ha
-; GLOBAL-NEXT:    lfs 2, .LCPI11_0@toc@l(3)
-; GLOBAL-NEXT:    lfs 3, .LCPI11_1@toc@l(4)
+; GLOBAL-NEXT:    addis 3, 2, .LCPI12_0@toc@ha
+; GLOBAL-NEXT:    addis 4, 2, .LCPI12_1@toc@ha
+; GLOBAL-NEXT:    lfs 2, .LCPI12_0@toc@l(3)
+; GLOBAL-NEXT:    lfs 3, .LCPI12_1@toc@l(4)
 ; GLOBAL-NEXT:    xsmulsp 1, 1, 0
 ; GLOBAL-NEXT:    xsmaddasp 2, 1, 0
 ; GLOBAL-NEXT:    xsmulsp 0, 1, 3
 ; GLOBAL-NEXT:    xsmulsp 0, 0, 2
-; GLOBAL-NEXT:  .LBB11_2:
+; GLOBAL-NEXT:  .LBB12_2:
 ; GLOBAL-NEXT:    fmr 1, 0
+; GLOBAL-NEXT:    blr
+  %rt = call afn ninf float @llvm.sqrt.f32(float %x)
+  ret float %rt
+}
+
+define float @sqrt_afn_preserve_sign_inf(float %x) #1 {
+; FMF-LABEL: sqrt_afn_preserve_sign_inf:
+; FMF:       # %bb.0:
+; FMF-NEXT:    xssqrtsp 1, 1
+; FMF-NEXT:    blr
+;
+; GLOBAL-LABEL: sqrt_afn_preserve_sign_inf:
+; GLOBAL:       # %bb.0:
+; GLOBAL-NEXT:    xssqrtsp 1, 1
 ; GLOBAL-NEXT:    blr
   %rt = call afn float @llvm.sqrt.f32(float %x)
   ret float %rt
@@ -390,45 +418,45 @@ define float @sqrt_afn_preserve_sign(float %x) #1 {
 define float @sqrt_fast_ieee(float %x) #0 {
 ; FMF-LABEL: sqrt_fast_ieee:
 ; FMF:       # %bb.0:
-; FMF-NEXT:    addis 3, 2, .LCPI12_2@toc@ha
+; FMF-NEXT:    addis 3, 2, .LCPI14_2@toc@ha
 ; FMF-NEXT:    fabs 0, 1
-; FMF-NEXT:    lfs 2, .LCPI12_2@toc@l(3)
+; FMF-NEXT:    lfs 2, .LCPI14_2@toc@l(3)
 ; FMF-NEXT:    fcmpu 0, 0, 2
 ; FMF-NEXT:    xxlxor 0, 0, 0
-; FMF-NEXT:    blt 0, .LBB12_2
+; FMF-NEXT:    blt 0, .LBB14_2
 ; FMF-NEXT:  # %bb.1:
 ; FMF-NEXT:    xsrsqrtesp 0, 1
-; FMF-NEXT:    addis 3, 2, .LCPI12_0@toc@ha
-; FMF-NEXT:    addis 4, 2, .LCPI12_1@toc@ha
-; FMF-NEXT:    lfs 2, .LCPI12_0@toc@l(3)
-; FMF-NEXT:    lfs 3, .LCPI12_1@toc@l(4)
+; FMF-NEXT:    addis 3, 2, .LCPI14_0@toc@ha
+; FMF-NEXT:    addis 4, 2, .LCPI14_1@toc@ha
+; FMF-NEXT:    lfs 2, .LCPI14_0@toc@l(3)
+; FMF-NEXT:    lfs 3, .LCPI14_1@toc@l(4)
 ; FMF-NEXT:    xsmulsp 1, 1, 0
 ; FMF-NEXT:    xsmaddasp 2, 1, 0
 ; FMF-NEXT:    xsmulsp 0, 1, 3
 ; FMF-NEXT:    xsmulsp 0, 0, 2
-; FMF-NEXT:  .LBB12_2:
+; FMF-NEXT:  .LBB14_2:
 ; FMF-NEXT:    fmr 1, 0
 ; FMF-NEXT:    blr
 ;
 ; GLOBAL-LABEL: sqrt_fast_ieee:
 ; GLOBAL:       # %bb.0:
-; GLOBAL-NEXT:    addis 3, 2, .LCPI12_2@toc@ha
+; GLOBAL-NEXT:    addis 3, 2, .LCPI14_2@toc@ha
 ; GLOBAL-NEXT:    fabs 0, 1
-; GLOBAL-NEXT:    lfs 2, .LCPI12_2@toc@l(3)
+; GLOBAL-NEXT:    lfs 2, .LCPI14_2@toc@l(3)
 ; GLOBAL-NEXT:    fcmpu 0, 0, 2
 ; GLOBAL-NEXT:    xxlxor 0, 0, 0
-; GLOBAL-NEXT:    blt 0, .LBB12_2
+; GLOBAL-NEXT:    blt 0, .LBB14_2
 ; GLOBAL-NEXT:  # %bb.1:
 ; GLOBAL-NEXT:    xsrsqrtesp 0, 1
-; GLOBAL-NEXT:    addis 3, 2, .LCPI12_0@toc@ha
-; GLOBAL-NEXT:    addis 4, 2, .LCPI12_1@toc@ha
-; GLOBAL-NEXT:    lfs 2, .LCPI12_0@toc@l(3)
-; GLOBAL-NEXT:    lfs 3, .LCPI12_1@toc@l(4)
+; GLOBAL-NEXT:    addis 3, 2, .LCPI14_0@toc@ha
+; GLOBAL-NEXT:    addis 4, 2, .LCPI14_1@toc@ha
+; GLOBAL-NEXT:    lfs 2, .LCPI14_0@toc@l(3)
+; GLOBAL-NEXT:    lfs 3, .LCPI14_1@toc@l(4)
 ; GLOBAL-NEXT:    xsmulsp 1, 1, 0
 ; GLOBAL-NEXT:    xsmaddasp 2, 1, 0
 ; GLOBAL-NEXT:    xsmulsp 0, 1, 3
 ; GLOBAL-NEXT:    xsmulsp 0, 0, 2
-; GLOBAL-NEXT:  .LBB12_2:
+; GLOBAL-NEXT:  .LBB14_2:
 ; GLOBAL-NEXT:    fmr 1, 0
 ; GLOBAL-NEXT:    blr
   %rt = call fast float @llvm.sqrt.f32(float %x)
@@ -450,18 +478,18 @@ define float @sqrt_fast_preserve_sign(float %x) #1 {
 ; FMF:       # %bb.0:
 ; FMF-NEXT:    xxlxor 0, 0, 0
 ; FMF-NEXT:    fcmpu 0, 1, 0
-; FMF-NEXT:    beq 0, .LBB13_2
+; FMF-NEXT:    beq 0, .LBB15_2
 ; FMF-NEXT:  # %bb.1:
 ; FMF-NEXT:    xsrsqrtesp 0, 1
-; FMF-NEXT:    addis 3, 2, .LCPI13_0@toc@ha
-; FMF-NEXT:    addis 4, 2, .LCPI13_1@toc@ha
-; FMF-NEXT:    lfs 2, .LCPI13_0@toc@l(3)
-; FMF-NEXT:    lfs 3, .LCPI13_1@toc@l(4)
+; FMF-NEXT:    addis 3, 2, .LCPI15_0@toc@ha
+; FMF-NEXT:    addis 4, 2, .LCPI15_1@toc@ha
+; FMF-NEXT:    lfs 2, .LCPI15_0@toc@l(3)
+; FMF-NEXT:    lfs 3, .LCPI15_1@toc@l(4)
 ; FMF-NEXT:    xsmulsp 1, 1, 0
 ; FMF-NEXT:    xsmaddasp 2, 1, 0
 ; FMF-NEXT:    xsmulsp 0, 1, 3
 ; FMF-NEXT:    xsmulsp 0, 0, 2
-; FMF-NEXT:  .LBB13_2:
+; FMF-NEXT:  .LBB15_2:
 ; FMF-NEXT:    fmr 1, 0
 ; FMF-NEXT:    blr
 ;
@@ -469,18 +497,18 @@ define float @sqrt_fast_preserve_sign(float %x) #1 {
 ; GLOBAL:       # %bb.0:
 ; GLOBAL-NEXT:    xxlxor 0, 0, 0
 ; GLOBAL-NEXT:    fcmpu 0, 1, 0
-; GLOBAL-NEXT:    beq 0, .LBB13_2
+; GLOBAL-NEXT:    beq 0, .LBB15_2
 ; GLOBAL-NEXT:  # %bb.1:
 ; GLOBAL-NEXT:    xsrsqrtesp 0, 1
-; GLOBAL-NEXT:    addis 3, 2, .LCPI13_0@toc@ha
-; GLOBAL-NEXT:    addis 4, 2, .LCPI13_1@toc@ha
-; GLOBAL-NEXT:    lfs 2, .LCPI13_0@toc@l(3)
-; GLOBAL-NEXT:    lfs 3, .LCPI13_1@toc@l(4)
+; GLOBAL-NEXT:    addis 3, 2, .LCPI15_0@toc@ha
+; GLOBAL-NEXT:    addis 4, 2, .LCPI15_1@toc@ha
+; GLOBAL-NEXT:    lfs 2, .LCPI15_0@toc@l(3)
+; GLOBAL-NEXT:    lfs 3, .LCPI15_1@toc@l(4)
 ; GLOBAL-NEXT:    xsmulsp 1, 1, 0
 ; GLOBAL-NEXT:    xsmaddasp 2, 1, 0
 ; GLOBAL-NEXT:    xsmulsp 0, 1, 3
 ; GLOBAL-NEXT:    xsmulsp 0, 0, 2
-; GLOBAL-NEXT:  .LBB13_2:
+; GLOBAL-NEXT:  .LBB15_2:
 ; GLOBAL-NEXT:    fmr 1, 0
 ; GLOBAL-NEXT:    blr
   %rt = call fast float @llvm.sqrt.f32(float %x)
@@ -502,10 +530,10 @@ define double @fcmp_nnan(double %a, double %y, double %z) {
 ; FMF:       # %bb.0:
 ; FMF-NEXT:    xxlxor 0, 0, 0
 ; FMF-NEXT:    xscmpudp 0, 1, 0
-; FMF-NEXT:    blt 0, .LBB14_2
+; FMF-NEXT:    blt 0, .LBB16_2
 ; FMF-NEXT:  # %bb.1:
 ; FMF-NEXT:    fmr 3, 2
-; FMF-NEXT:  .LBB14_2:
+; FMF-NEXT:  .LBB16_2:
 ; FMF-NEXT:    fmr 1, 3
 ; FMF-NEXT:    blr
 ;
@@ -513,10 +541,10 @@ define double @fcmp_nnan(double %a, double %y, double %z) {
 ; GLOBAL:       # %bb.0:
 ; GLOBAL-NEXT:    xxlxor 0, 0, 0
 ; GLOBAL-NEXT:    xscmpudp 0, 1, 0
-; GLOBAL-NEXT:    blt 0, .LBB14_2
+; GLOBAL-NEXT:    blt 0, .LBB16_2
 ; GLOBAL-NEXT:  # %bb.1:
 ; GLOBAL-NEXT:    fmr 3, 2
-; GLOBAL-NEXT:  .LBB14_2:
+; GLOBAL-NEXT:  .LBB16_2:
 ; GLOBAL-NEXT:    fmr 1, 3
 ; GLOBAL-NEXT:    blr
   %cmp = fcmp nnan ult double %a, 0.0
diff --git a/llvm/test/CodeGen/PowerPC/fold-frame-offset-using-rr.mir b/llvm/test/CodeGen/PowerPC/fold-frame-offset-using-rr.mir
index b8b9660e3e147..810612340a512 100644
--- a/llvm/test/CodeGen/PowerPC/fold-frame-offset-using-rr.mir
+++ b/llvm/test/CodeGen/PowerPC/fold-frame-offset-using-rr.mir
@@ -152,3 +152,16 @@ body: |
     ; CHECK: $x6 = LD 4, killed $x4
     BLR8 implicit $lr8, implicit $rm
 ...
+---
+name: testR0
+# Give up the folding if the register is R0/X0
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $f1, $x0, $x3
+    $x4 = ADDI8 killed $x3, -8
+    $x4 = ADD8 killed $x4, $x0
+    STFD killed $f1, -8, killed $x4
+    ; CHECK-NOT: STFDX
+    BLR8 implicit $lr8, implicit $rm
+...
diff --git a/llvm/test/CodeGen/PowerPC/loop-comment.ll b/llvm/test/CodeGen/PowerPC/loop-comment.ll
new file mode 100644
index 0000000000000..b33d62bf96c69
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/loop-comment.ll
@@ -0,0 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=powerpc64le-linux-gnu < %s | FileCheck %s -check-prefix=PPC64LE
+
+define void @test(i8* %ptr, i8 %cmp, i8 %val) {
+; PPC64LE-LABEL: test:
+; PPC64LE:       # %bb.0:
+; PPC64LE-NEXT:    rlwinm 4, 4, 0, 24, 31
+; PPC64LE-NEXT:  .LBB0_1: #
+; PPC64LE-NEXT:    lbarx 6, 0, 3
+; PPC64LE-NEXT:    cmpw 4, 6
+; PPC64LE-NEXT:    bne 0, .LBB0_3
+; PPC64LE-NEXT:  # %bb.2: #
+; PPC64LE-NEXT:    stbcx. 5, 0, 3
+; PPC64LE-NEXT:    beqlr 0
+; PPC64LE-NEXT:    b .LBB0_1
+; PPC64LE-NEXT:  .LBB0_3:
+; PPC64LE-NEXT:    stbcx. 6, 0, 3
+; PPC64LE-NEXT:    blr
+  %res = cmpxchg i8* %ptr, i8 %cmp, i8 %val monotonic monotonic
+  ret void
+}
diff --git a/llvm/test/CodeGen/PowerPC/mi-simplify-code.mir b/llvm/test/CodeGen/PowerPC/mi-simplify-code.mir
new file mode 100644
index 0000000000000..0cfeb944a5dbc
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/mi-simplify-code.mir
@@ -0,0 +1,63 @@
+# RUN: llc -mtriple powerpc64le-unknown-linux-gnu -mcpu=pwr8 -x mir < %s \
+# RUN:   -verify-machineinstrs -start-before=ppc-mi-peepholes | FileCheck %s
+
+---
+name: remove_frsp
+alignment:       16
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x3, $x4
+
+    %1:g8rc = COPY $x4
+    %0:g8rc_and_g8rc_nox0 = COPY $x3
+    %2:g8rc = RLDICR %1, 2, 61
+    %3:f8rc, %4:g8rc_and_g8rc_nox0 = LFSUX %0, killed %2
+    %5:f4rc = FRSP killed %3, implicit $rm
+    %22:vslrc = SUBREG_TO_REG 1, %5, %subreg.sub_64
+    %7:g8rc = LI8 8
+    %8:vssrc = XFLOADf32 %4, killed %7
+    %23:vslrc = SUBREG_TO_REG 1, %8, %subreg.sub_64
+    %10:vsrc = XXPERMDI %23, %22, 0
+    %11:vrrc = XVCVDPSP killed %10, implicit $rm
+    $v2 = COPY %11
+    BLR8 implicit $lr8, implicit $rm, implicit $v2
+...
+# CHECK-LABEL: remove_frsp
+# CHECK: sldi 4, 4, 2
+# CHECK-NEXT: lfsux 0, 3, 4
+# CHECK-NOT: frsp
+# CHECK-NEXT: lfs 1, 8(3)
+# CHECK-NEXT: xxmrghd	0, 1, 0
+# CHECK-NEXT: xvcvdpsp 34, 0
+# CHECK-NEXT: blr
+
+---
+name: remove_xsrsp
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x3, $x4
+
+    %1:g8rc = COPY $x4
+    %0:g8rc_and_g8rc_nox0 = COPY $x3
+    %2:g8rc = RLDICR %1, 2, 61
+    %3:f8rc, %4:g8rc_and_g8rc_nox0 = LFSUX %0, killed %2
+    %5:vssrc = XSRSP killed %3
+    %22:vslrc = SUBREG_TO_REG 1, %5, %subreg.sub_64
+    %7:g8rc = LI8 8
+    %8:vssrc = XFLOADf32 %4, killed %7
+    %23:vslrc = SUBREG_TO_REG 1, %8, %subreg.sub_64
+    %10:vsrc = XXPERMDI %23, %22, 0
+    %11:vrrc = XVCVDPSP killed %10, implicit $rm
+    $v2 = COPY %11
+    BLR8 implicit $lr8, implicit $rm, implicit $v2
+...
+# CHECK-LABEL: remove_xsrsp
+# CHECK: sldi 4, 4, 2
+# CHECK-NEXT: lfsux 0, 3, 4
+# CHECK-NOT: xsrsp
+# CHECK-NEXT: lfs 1, 8(3)
+# CHECK-NEXT: xxmrghd	0, 1, 0
+# CHECK-NEXT: xvcvdpsp 34, 0
+# CHECK-NEXT: blr
diff --git a/llvm/test/CodeGen/PowerPC/ori_imm32.ll b/llvm/test/CodeGen/PowerPC/ori_imm32.ll
index 5444a5bc7e65e..245c9bdbce77b 100644
--- a/llvm/test/CodeGen/PowerPC/ori_imm32.ll
+++ b/llvm/test/CodeGen/PowerPC/ori_imm32.ll
@@ -1,96 +1,111 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64le-unknown-linux-gnu | FileCheck %s
 ; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu | FileCheck %s
 
 define i64 @ori_test_a(i64 %a) {
+; CHECK-LABEL: ori_test_a:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    ori 3, 3, 65535
+; CHECK-NEXT:    oris 3, 3, 65535
+; CHECK-NEXT:    blr
 entry:
-; CHECK-LABEL: @ori_test_a
-; CHECK-DAG:  ori 3, 3, 65535
-; CHECK-DAG:  oris 3, 3, 65535
-; CHECK-NEXT:  blr
   %or = or i64 %a, 4294967295
   ret i64 %or
 }
 
 define i64 @ori_test_b(i64 %a) {
+; CHECK-LABEL: ori_test_b:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 1
+; CHECK-NEXT:    sldi 4, 4, 32
+; CHECK-NEXT:    or 3, 3, 4
+; CHECK-NEXT:    blr
 entry:
-; CHECK-LABEL: @ori_test_b
-; CHECK:  or 3, 3, {{[0-9]+}}
-; CHECK-NEXT:  blr
   %or = or i64 %a, 4294967296
   ret i64 %or
 }
 
 define i64 @ori_test_c(i64 %a) {
+; CHECK-LABEL: ori_test_c:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    ori 3, 3, 65535
+; CHECK-NEXT:    blr
 entry:
-; CHECK-LABEL: @ori_test_c
-; CHECK:  ori 3, 3, 65535
-; CHECK-NEXT:  blr
   %or = or i64 %a, 65535
   ret i64 %or
 }
 
 define i64 @ori_test_d(i64 %a) {
+; CHECK-LABEL: ori_test_d:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    oris 3, 3, 1
+; CHECK-NEXT:    blr
 entry:
-; CHECK-LABEL: @ori_test_d
-; CHECK:  oris 3, 3, 1
-; CHECK-NEXT:  blr
   %or = or i64 %a, 65536
   ret i64 %or
 }
 
 define zeroext i32 @ori_test_e(i32 zeroext %a) {
+; CHECK-LABEL: ori_test_e:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    ori 3, 3, 65535
+; CHECK-NEXT:    oris 3, 3, 255
+; CHECK-NEXT:    blr
 entry:
-; CHECK-LABEL: @ori_test_e
-; CHECK-DAG:  ori 3, 3, 65535
-; CHECK-DAG:  oris 3, 3, 255
-; CHECK-NEXT:  blr
   %or = or i32 %a, 16777215
   ret i32 %or
 }
 
 define i64 @xori_test_a(i64 %a) {
+; CHECK-LABEL: xori_test_a:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xori 3, 3, 65535
+; CHECK-NEXT:    xoris 3, 3, 65535
+; CHECK-NEXT:    blr
 entry:
-; CHECK-LABEL: @xori_test_a
-; CHECK-DAG:  xori 3, 3, 65535
-; CHECK-DAG:  xoris 3, 3, 65535
-; CHECK-NEXT:  blr
   %xor = xor i64 %a, 4294967295
   ret i64 %xor
 }
 
 define i64 @xori_test_b(i64 %a) {
+; CHECK-LABEL: xori_test_b:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 1
+; CHECK-NEXT:    sldi 4, 4, 32
+; CHECK-NEXT:    xor 3, 3, 4
+; CHECK-NEXT:    blr
 entry:
-; CHECK-LABEL: @xori_test_b
-; CHECK:  xor 3, 3, {{[0-9]+}}
-; CHECK-NEXT:  blr
   %xor = xor i64 %a, 4294967296
   ret i64 %xor
 }
 
 define i64 @xori_test_c(i64 %a) {
+; CHECK-LABEL: xori_test_c:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xori 3, 3, 65535
+; CHECK-NEXT:    blr
 entry:
-; CHECK-LABEL: @xori_test_c
-; CHECK:  xori 3, 3, 65535
-; CHECK-NEXT:  blr
   %xor = xor i64 %a, 65535
   ret i64 %xor
 }
 
 define i64 @xori_test_d(i64 %a) {
+; CHECK-LABEL: xori_test_d:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xoris 3, 3, 1
+; CHECK-NEXT:    blr
 entry:
-; CHECK-LABEL: @xori_test_d
-; CHECK:  xoris 3, 3, 1
-; CHECK-NEXT:  blr
   %xor = xor i64 %a, 65536
   ret i64 %xor
 }
 
 define zeroext i32 @xori_test_e(i32 zeroext %a) {
+; CHECK-LABEL: xori_test_e:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xori 3, 3, 65535
+; CHECK-NEXT:    xoris 3, 3, 255
+; CHECK-NEXT:    blr
 entry:
-; CHECK-LABEL: @xori_test_e
-; CHECK-DAG:  xori 3, 3, 65535
-; CHECK-DAG:  xoris 3, 3, 255
-; CHECK-NEXT:  blr
   %xor = xor i32 %a, 16777215
   ret i32 %xor
 }
diff --git a/llvm/test/CodeGen/PowerPC/ori_imm64.ll b/llvm/test/CodeGen/PowerPC/ori_imm64.ll
new file mode 100644
index 0000000000000..d2ea911c6f140
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/ori_imm64.ll
@@ -0,0 +1,52 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64le-unknown-linux-gnu | FileCheck %s
+; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu | FileCheck %s
+
+define i64 @ori_test_1(i64 %a) {
+; CHECK-LABEL: ori_test_1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 3, -1
+; CHECK-NEXT:    blr
+entry:
+  %or = or i64 %a, 18446744073709551615 ; 0xffffffffffffffff
+  ret i64 %or
+}
+
+define i64 @ori_test_2(i64 %a) {
+; CHECK-LABEL: ori_test_2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lis 4, 15
+; CHECK-NEXT:    ori 4, 4, 65535
+; CHECK-NEXT:    sldi 4, 4, 29
+; CHECK-NEXT:    or 3, 3, 4
+; CHECK-NEXT:    blr
+entry:
+  %or = or i64 %a, 562949416550400 ; 0x1ffffe0000000
+  ret i64 %or
+}
+
+define i64 @ori_test_3(i64 %a) {
+; CHECK-LABEL: ori_test_3:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lis 4, -32768
+; CHECK-NEXT:    rldicr 4, 4, 36, 63
+; CHECK-NEXT:    or 3, 3, 4
+; CHECK-NEXT:    blr
+entry:
+  %or = or i64 %a, 68719476728 ; 0xffffffff8
+  ret i64 %or
+}
+
+define i64 @ori_test_4(i64 %a) {
+; CHECK-LABEL: ori_test_4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, 4112
+; CHECK-NEXT:    sldi 4, 4, 32
+; CHECK-NEXT:    oris 4, 4, 4112
+; CHECK-NEXT:    ori 4, 4, 65535
+; CHECK-NEXT:    or 3, 3, 4
+; CHECK-NEXT:    blr
+entry:
+  %or = or i64 %a, 17661175070719 ; 0x10101010ffff
+  ret i64 %or
+}
diff --git a/llvm/test/CodeGen/PowerPC/ppcf128-constrained-fp-intrinsics.ll b/llvm/test/CodeGen/PowerPC/ppcf128-constrained-fp-intrinsics.ll
index e87620a37b9aa..8284607482a36 100644
--- a/llvm/test/CodeGen/PowerPC/ppcf128-constrained-fp-intrinsics.ll
+++ b/llvm/test/CodeGen/PowerPC/ppcf128-constrained-fp-intrinsics.ll
@@ -402,7 +402,6 @@ define ppc_fp128 @test_powi_ppc_fp128(ppc_fp128 %first, i32 %second) nounwind {
 ; PC64-NEXT:    ld 0, 16(1)
 ; PC64-NEXT:    mtlr 0
 ; PC64-NEXT:    blr
-; PC64LE9     :    clrldi 5, 5, 32
 entry:
   %powi = call ppc_fp128 @llvm.experimental.constrained.powi.ppcf128(
                     ppc_fp128 %first,
diff --git a/llvm/test/CodeGen/PowerPC/pr45297.ll b/llvm/test/CodeGen/PowerPC/pr45297.ll
index 71c19744fb706..39583d5a04cc0 100644
--- a/llvm/test/CodeGen/PowerPC/pr45297.ll
+++ b/llvm/test/CodeGen/PowerPC/pr45297.ll
@@ -1,10 +1,22 @@
-; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
-; RUN:   -mattr=+altivec -mattr=-power8-vector -mattr=-vsx < %s | FileCheck %s
-; XFAIL: *
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -ppc-asm-full-reg-names \
+; RUN:   -mtriple=powerpc64le-unknown-linux-gnu -mattr=+altivec \
+; RUN:   -mattr=-power8-vector -mattr=-vsx < %s 2>&1 | FileCheck %s
+
+@Global = dso_local global i32 55, align 4
 
 define dso_local void @test(float %0) local_unnamed_addr {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fctiwz f0, f1
+; CHECK-NEXT:    addi r3, r1, -4
+; CHECK-NEXT:    addis r4, r2, Global@toc@ha
+; CHECK-NEXT:    stfiwx f0, 0, r3
+; CHECK-NEXT:    lwz r3, -4(r1)
+; CHECK-NEXT:    stw r3, Global@toc@l(r4)
+; CHECK-NEXT:    blr
 entry:
   %1 = fptosi float %0 to i32
-  store i32 %1, i32* undef, align 4
+  store i32 %1, i32* @Global, align 4
   ret void
 }
diff --git a/llvm/test/CodeGen/PowerPC/pr45301.ll b/llvm/test/CodeGen/PowerPC/pr45301.ll
new file mode 100644
index 0000000000000..ee0c6c341cc0c
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/pr45301.ll
@@ -0,0 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=powerpc64-- -verify-machineinstrs \
+; RUN:   -ppc-asm-full-reg-names < %s | FileCheck %s
+%struct.e.0.1.2.3.12.29 = type { [10 x i32] }
+
+define dso_local void @g(%struct.e.0.1.2.3.12.29* %agg.result) local_unnamed_addr #0 {
+; CHECK-LABEL: g:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    std r0, 16(r1)
+; CHECK-NEXT:    stdu r1, -112(r1)
+; CHECK-NEXT:    bl i
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addis r4, r2, g@toc@ha
+; CHECK-NEXT:    addi r4, r4, g@toc@l
+; CHECK-NEXT:    ld r5, 0(r4)
+; CHECK-NEXT:    std r5, 0(r3)
+; CHECK-NEXT:    ld r5, 16(r4)
+; CHECK-NEXT:    std r5, 16(r3)
+; CHECK-NEXT:    ld r6, 8(r4)
+; CHECK-NEXT:    std r6, 8(r3)
+; CHECK-NEXT:    ld r6, 24(r4)
+; CHECK-NEXT:    std r6, 24(r3)
+; CHECK-NEXT:    lwz r6, 0(r3)
+; CHECK-NEXT:    ld r4, 32(r4)
+; CHECK-NEXT:    std r4, 32(r3)
+; CHECK-NEXT:    li r4, 20
+; CHECK-NEXT:    stwbrx r6, 0, r3
+; CHECK-NEXT:    stwbrx r5, r3, r4
+; CHECK-NEXT:    addi r1, r1, 112
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+entry:
+  %call = tail call signext i32 bitcast (i32 (...)* @i to i32 ()*)()
+  %conv = sext i32 %call to i64
+  %0 = inttoptr i64 %conv to i8*
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 dereferenceable(40) %0, i8* nonnull align 4 dereferenceable(40) bitcast (void (%struct.e.0.1.2.3.12.29*)* @g to i8*), i64 40, i1 false)
+  %1 = inttoptr i64 %conv to i32*
+  %2 = load i32, i32* %1, align 4
+  %rev.i = tail call i32 @llvm.bswap.i32(i32 %2)
+  store i32 %rev.i, i32* %1, align 4
+  %incdec.ptr.i.4 = getelementptr inbounds i32, i32* %1, i64 5
+  %3 = load i32, i32* %incdec.ptr.i.4, align 4
+  %rev.i.5 = tail call i32 @llvm.bswap.i32(i32 %3)
+  store i32 %rev.i.5, i32* %incdec.ptr.i.4, align 4
+  ret void
+}
+
+declare i32 @i(...) local_unnamed_addr
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* noalias nocapture writeonly, i8* noalias nocapture readonly, i64, i1 immarg) #1
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare i32 @llvm.bswap.i32(i32)
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/PowerPC/test_func_desc.ll b/llvm/test/CodeGen/PowerPC/test_func_desc.ll
index 3cd16b94fb238..c770194874931 100644
--- a/llvm/test/CodeGen/PowerPC/test_func_desc.ll
+++ b/llvm/test/CodeGen/PowerPC/test_func_desc.ll
@@ -27,10 +27,9 @@ entry:
   ret i32 3
 }
 
-; CHECK: .globl foo
+; CHECK: .globl foo[DS]
 ; CHECK: .globl .foo
 ; CHECK: .csect foo[DS]
-; CHECK-NEXT: foo:
 ; 32BIT: .long .foo
 ; 32BIT-NEXT: .long TOC[TC0]
 ; 32BIT-NEXT: .long 0
@@ -40,10 +39,9 @@ entry:
 ; CHECK-NEXT: .csect .text[PR]
 ; CHECK-LABEL: .foo:
 
-; CHECK: .globl main
+; CHECK: .globl main[DS]
 ; CHECK: .globl .main
 ; CHECK: .csect main[DS]
-; CHECK-NEXT: main:
 ; 32BIT: .long .main
 ; 32BIT-NEXT: .long TOC[TC0]
 ; 32BIT-NEXT: .long 0
@@ -58,7 +56,6 @@ entry:
 
 ; CHECK: .lglobl .static_foo
 ; CHECK: .csect static_foo[DS]
-; CHECK-NEXT: static_foo:
 ; 32BIT: .long .static_foo
 ; 32BIT-NEXT: .long TOC[TC0]
 ; 32BIT-NEXT: .long 0
diff --git a/llvm/test/CodeGen/PowerPC/tls.ll b/llvm/test/CodeGen/PowerPC/tls.ll
index 18ca23b5e81dc..c5570e175f90c 100644
--- a/llvm/test/CodeGen/PowerPC/tls.ll
+++ b/llvm/test/CodeGen/PowerPC/tls.ll
@@ -11,11 +11,11 @@
 ;OPT1-LABEL:          localexec:
 define i32 @localexec() nounwind {
 entry:
-;OPT0:          addis [[REG1:[0-9]+]], 13, a@tprel@ha
-;OPT0-NEXT:     addi [[REG2:[0-9]+]], [[REG1]], a@tprel@l
+;OPT0:          addis [[REG1:[1-9][0-9]*]], 13, a@tprel@ha
+;OPT0-NEXT:     addi [[REG2:[1-9][0-9]*]], [[REG1]], a@tprel@l
 ;OPT0-NEXT:     li [[REG3:[0-9]+]], 42
 ;OPT0:          stw [[REG3]], 0([[REG2]])
-;OPT1:          addis [[REG1:[0-9]+]], 13, a@tprel@ha
+;OPT1:          addis [[REG1:[1-9][0-9]*]], 13, a@tprel@ha
 ;OPT1-NEXT:     li [[REG3:[0-9]+]], 42
 ;OPT1:     stw [[REG3]], a@tprel@l([[REG1]])
   store i32 42, i32* @a, align 4
@@ -36,19 +36,19 @@ entry:
 }
 
 ; OPT1-LABEL: main2:
-; OPT1: addis [[REG1:[0-9]+]], 2, a2@got@tprel@ha
+; OPT1: addis [[REG1:[1-9][0-9]*]], 2, a2@got@tprel@ha
 ; OPT1: ld [[REG2:[0-9]+]], a2@got@tprel@l([[REG1]])
 ; OPT1: add {{[0-9]+}}, [[REG2]], a2@tls
 
 ;OPT0-PPC32-LABEL:    main2:
-;OPT0-PPC32:       li [[REG1:[0-9]+]], _GLOBAL_OFFSET_TABLE_@l
+;OPT0-PPC32:       li [[REG1:[1-9][0-9]*]], _GLOBAL_OFFSET_TABLE_@l
 ;OPT0-PPC32:       addis [[REG1]], [[REG1]], _GLOBAL_OFFSET_TABLE_@ha
-;OPT0-PPC32:       lwz [[REG2:[0-9]+]], a2@got@tprel([[REG1]])
+;OPT0-PPC32:       lwz [[REG2:[1-9][0-9]*]], a2@got@tprel([[REG1]])
 ;OPT0-PPC32:       add 3, [[REG2]], a2@tls
 
 ;OPT0-PPC32-PIC-LABEL:  main2:
 ;OPT0-PPC32-PIC:        .long _GLOBAL_OFFSET_TABLE_-{{.*}}
 ;OPT0-PPC32-PIC-NOT:    li {{[0-9]+}}, _GLOBAL_OFFSET_TABLE_@l
-;OPT0-PPC32-PIC-NOT:    addis {{[0-9]+}}, {{[0-9+]}}, _GLOBAL_OFFSET_TABLE_@ha
+;OPT0-PPC32-PIC-NOT:    addis {{[0-9]+}}, {{[1-9][0-9*]}}, _GLOBAL_OFFSET_TABLE_@ha
 ;OPT0-PPC32-PIC-NOT:    bl __tls_get_addr(a2@tlsgd)@PLT
-;OPT0-PPC32-PIC:        lwz {{[0-9]+}}, a2@got@tprel({{[0-9]+}})
+;OPT0-PPC32-PIC:        lwz {{[0-9]+}}, a2@got@tprel({{[1-9][0-9]*}})
diff --git a/llvm/test/CodeGen/PowerPC/vector-constrained-fp-intrinsics.ll b/llvm/test/CodeGen/PowerPC/vector-constrained-fp-intrinsics.ll
index 0d8795c82914e..1e5000a9854f3 100644
--- a/llvm/test/CodeGen/PowerPC/vector-constrained-fp-intrinsics.ll
+++ b/llvm/test/CodeGen/PowerPC/vector-constrained-fp-intrinsics.ll
@@ -6373,19 +6373,17 @@ define <1 x float> @constrained_vector_ceil_v1f32() #0 {
 ; PC64LE-LABEL: constrained_vector_ceil_v1f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI103_0@toc@ha
-; PC64LE-NEXT:    lfs 0, .LCPI103_0@toc@l(3)
-; PC64LE-NEXT:    xsrdpip 0, 0
-; PC64LE-NEXT:    xscvdpspn 0, 0
-; PC64LE-NEXT:    xxsldwi 34, 0, 0, 1
+; PC64LE-NEXT:    addi 3, 3, .LCPI103_0@toc@l
+; PC64LE-NEXT:    lfiwzx 0, 0, 3
+; PC64LE-NEXT:    xxpermdi 34, 0, 0, 2
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_ceil_v1f32:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI103_0@toc@ha
-; PC64LE9-NEXT:    lfs 0, .LCPI103_0@toc@l(3)
-; PC64LE9-NEXT:    xsrdpip 0, 0
-; PC64LE9-NEXT:    xscvdpspn 0, 0
-; PC64LE9-NEXT:    xxsldwi 34, 0, 0, 1
+; PC64LE9-NEXT:    addi 3, 3, .LCPI103_0@toc@l
+; PC64LE9-NEXT:    lfiwzx 0, 0, 3
+; PC64LE9-NEXT:    xxpermdi 34, 0, 0, 2
 ; PC64LE9-NEXT:    blr
 entry:
   %ceil = call <1 x float> @llvm.experimental.constrained.ceil.v1f32(
@@ -6400,16 +6398,14 @@ define <2 x double> @constrained_vector_ceil_v2f64() #0 {
 ; PC64LE-NEXT:    addis 3, 2, .LCPI104_0@toc@ha
 ; PC64LE-NEXT:    addi 3, 3, .LCPI104_0@toc@l
 ; PC64LE-NEXT:    lxvd2x 0, 0, 3
-; PC64LE-NEXT:    xxswapd 0, 0
-; PC64LE-NEXT:    xvrdpip 34, 0
+; PC64LE-NEXT:    xxswapd 34, 0
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_ceil_v2f64:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI104_0@toc@ha
 ; PC64LE9-NEXT:    addi 3, 3, .LCPI104_0@toc@l
-; PC64LE9-NEXT:    lxvx 0, 0, 3
-; PC64LE9-NEXT:    xvrdpip 34, 0
+; PC64LE9-NEXT:    lxvx 34, 0, 3
 ; PC64LE9-NEXT:    blr
 entry:
   %ceil = call <2 x double> @llvm.experimental.constrained.ceil.v2f64(
@@ -6421,50 +6417,16 @@ entry:
 define <3 x float> @constrained_vector_ceil_v3f32() #0 {
 ; PC64LE-LABEL: constrained_vector_ceil_v3f32:
 ; PC64LE:       # %bb.0: # %entry
-; PC64LE-NEXT:    addis 3, 2, .LCPI105_2@toc@ha
-; PC64LE-NEXT:    addis 4, 2, .LCPI105_1@toc@ha
-; PC64LE-NEXT:    lfs 0, .LCPI105_2@toc@l(3)
-; PC64LE-NEXT:    lfs 1, .LCPI105_1@toc@l(4)
 ; PC64LE-NEXT:    addis 3, 2, .LCPI105_0@toc@ha
-; PC64LE-NEXT:    xsrdpip 0, 0
-; PC64LE-NEXT:    lfs 2, .LCPI105_0@toc@l(3)
-; PC64LE-NEXT:    addis 3, 2, .LCPI105_3@toc@ha
-; PC64LE-NEXT:    xsrdpip 1, 1
-; PC64LE-NEXT:    addi 3, 3, .LCPI105_3@toc@l
-; PC64LE-NEXT:    xsrdpip 2, 2
-; PC64LE-NEXT:    xscvdpspn 0, 0
-; PC64LE-NEXT:    xscvdpspn 1, 1
-; PC64LE-NEXT:    xxsldwi 34, 0, 0, 1
-; PC64LE-NEXT:    xscvdpspn 0, 2
-; PC64LE-NEXT:    xxsldwi 35, 1, 1, 1
-; PC64LE-NEXT:    vmrglw 2, 3, 2
-; PC64LE-NEXT:    lvx 3, 0, 3
-; PC64LE-NEXT:    xxsldwi 36, 0, 0, 1
-; PC64LE-NEXT:    vperm 2, 4, 2, 3
+; PC64LE-NEXT:    addi 3, 3, .LCPI105_0@toc@l
+; PC64LE-NEXT:    lvx 2, 0, 3
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_ceil_v3f32:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI105_0@toc@ha
-; PC64LE9-NEXT:    lfs 0, .LCPI105_0@toc@l(3)
-; PC64LE9-NEXT:    addis 3, 2, .LCPI105_1@toc@ha
-; PC64LE9-NEXT:    lfs 1, .LCPI105_1@toc@l(3)
-; PC64LE9-NEXT:    addis 3, 2, .LCPI105_2@toc@ha
-; PC64LE9-NEXT:    xsrdpip 0, 0
-; PC64LE9-NEXT:    lfs 2, .LCPI105_2@toc@l(3)
-; PC64LE9-NEXT:    addis 3, 2, .LCPI105_3@toc@ha
-; PC64LE9-NEXT:    addi 3, 3, .LCPI105_3@toc@l
-; PC64LE9-NEXT:    xsrdpip 1, 1
-; PC64LE9-NEXT:    xsrdpip 2, 2
-; PC64LE9-NEXT:    xscvdpspn 0, 0
-; PC64LE9-NEXT:    xscvdpspn 1, 1
-; PC64LE9-NEXT:    xscvdpspn 2, 2
-; PC64LE9-NEXT:    xxsldwi 36, 0, 0, 1
-; PC64LE9-NEXT:    xxsldwi 35, 1, 1, 1
-; PC64LE9-NEXT:    xxsldwi 34, 2, 2, 1
-; PC64LE9-NEXT:    vmrglw 2, 3, 2
-; PC64LE9-NEXT:    lxvx 35, 0, 3
-; PC64LE9-NEXT:    vperm 2, 4, 2, 3
+; PC64LE9-NEXT:    addi 3, 3, .LCPI105_0@toc@l
+; PC64LE9-NEXT:    lxvx 34, 0, 3
 ; PC64LE9-NEXT:    blr
 entry:
   %ceil = call <3 x float> @llvm.experimental.constrained.ceil.v3f32(
@@ -6476,31 +6438,18 @@ entry:
 define <3 x double> @constrained_vector_ceil_v3f64() #0 {
 ; PC64LE-LABEL: constrained_vector_ceil_v3f64:
 ; PC64LE:       # %bb.0: # %entry
-; PC64LE-NEXT:    addis 3, 2, .LCPI106_1@toc@ha
-; PC64LE-NEXT:    addi 3, 3, .LCPI106_1@toc@l
-; PC64LE-NEXT:    lxvd2x 0, 0, 3
 ; PC64LE-NEXT:    addis 3, 2, .LCPI106_0@toc@ha
 ; PC64LE-NEXT:    lfs 1, .LCPI106_0@toc@l(3)
-; PC64LE-NEXT:    xxswapd 0, 0
-; PC64LE-NEXT:    xsrdpip 3, 1
-; PC64LE-NEXT:    xvrdpip 2, 0
-; PC64LE-NEXT:    xxswapd 1, 2
-; PC64LE-NEXT:    # kill: def $f2 killed $f2 killed $vsl2
-; PC64LE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; PC64LE-NEXT:    fmr 2, 1
+; PC64LE-NEXT:    fmr 3, 1
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_ceil_v3f64:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI106_0@toc@ha
-; PC64LE9-NEXT:    lfs 0, .LCPI106_0@toc@l(3)
-; PC64LE9-NEXT:    addis 3, 2, .LCPI106_1@toc@ha
-; PC64LE9-NEXT:    addi 3, 3, .LCPI106_1@toc@l
-; PC64LE9-NEXT:    xsrdpip 3, 0
-; PC64LE9-NEXT:    lxvx 0, 0, 3
-; PC64LE9-NEXT:    xvrdpip 2, 0
-; PC64LE9-NEXT:    xxswapd 1, 2
-; PC64LE9-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
-; PC64LE9-NEXT:    # kill: def $f2 killed $f2 killed $vsl2
+; PC64LE9-NEXT:    lfs 1, .LCPI106_0@toc@l(3)
+; PC64LE9-NEXT:    fmr 2, 1
+; PC64LE9-NEXT:    fmr 3, 1
 ; PC64LE9-NEXT:    blr
 entry:
   %ceil = call <3 x double> @llvm.experimental.constrained.ceil.v3f64(
@@ -6513,19 +6462,17 @@ define <1 x float> @constrained_vector_floor_v1f32() #0 {
 ; PC64LE-LABEL: constrained_vector_floor_v1f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI107_0@toc@ha
-; PC64LE-NEXT:    lfs 0, .LCPI107_0@toc@l(3)
-; PC64LE-NEXT:    xsrdpim 0, 0
-; PC64LE-NEXT:    xscvdpspn 0, 0
-; PC64LE-NEXT:    xxsldwi 34, 0, 0, 1
+; PC64LE-NEXT:    addi 3, 3, .LCPI107_0@toc@l
+; PC64LE-NEXT:    lfiwzx 0, 0, 3
+; PC64LE-NEXT:    xxpermdi 34, 0, 0, 2
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_floor_v1f32:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI107_0@toc@ha
-; PC64LE9-NEXT:    lfs 0, .LCPI107_0@toc@l(3)
-; PC64LE9-NEXT:    xsrdpim 0, 0
-; PC64LE9-NEXT:    xscvdpspn 0, 0
-; PC64LE9-NEXT:    xxsldwi 34, 0, 0, 1
+; PC64LE9-NEXT:    addi 3, 3, .LCPI107_0@toc@l
+; PC64LE9-NEXT:    lfiwzx 0, 0, 3
+; PC64LE9-NEXT:    xxpermdi 34, 0, 0, 2
 ; PC64LE9-NEXT:    blr
 entry:
   %floor = call <1 x float> @llvm.experimental.constrained.floor.v1f32(
@@ -6541,16 +6488,14 @@ define <2 x double> @constrained_vector_floor_v2f64() #0 {
 ; PC64LE-NEXT:    addis 3, 2, .LCPI108_0@toc@ha
 ; PC64LE-NEXT:    addi 3, 3, .LCPI108_0@toc@l
 ; PC64LE-NEXT:    lxvd2x 0, 0, 3
-; PC64LE-NEXT:    xxswapd 0, 0
-; PC64LE-NEXT:    xvrdpim 34, 0
+; PC64LE-NEXT:    xxswapd 34, 0
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_floor_v2f64:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI108_0@toc@ha
 ; PC64LE9-NEXT:    addi 3, 3, .LCPI108_0@toc@l
-; PC64LE9-NEXT:    lxvx 0, 0, 3
-; PC64LE9-NEXT:    xvrdpim 34, 0
+; PC64LE9-NEXT:    lxvx 34, 0, 3
 ; PC64LE9-NEXT:    blr
 entry:
   %floor = call <2 x double> @llvm.experimental.constrained.floor.v2f64(
@@ -6562,50 +6507,16 @@ entry:
 define <3 x float> @constrained_vector_floor_v3f32() #0 {
 ; PC64LE-LABEL: constrained_vector_floor_v3f32:
 ; PC64LE:       # %bb.0: # %entry
-; PC64LE-NEXT:    addis 3, 2, .LCPI109_2@toc@ha
-; PC64LE-NEXT:    addis 4, 2, .LCPI109_1@toc@ha
-; PC64LE-NEXT:    lfs 0, .LCPI109_2@toc@l(3)
-; PC64LE-NEXT:    lfs 1, .LCPI109_1@toc@l(4)
 ; PC64LE-NEXT:    addis 3, 2, .LCPI109_0@toc@ha
-; PC64LE-NEXT:    xsrdpim 0, 0
-; PC64LE-NEXT:    lfs 2, .LCPI109_0@toc@l(3)
-; PC64LE-NEXT:    addis 3, 2, .LCPI109_3@toc@ha
-; PC64LE-NEXT:    xsrdpim 1, 1
-; PC64LE-NEXT:    addi 3, 3, .LCPI109_3@toc@l
-; PC64LE-NEXT:    xsrdpim 2, 2
-; PC64LE-NEXT:    xscvdpspn 0, 0
-; PC64LE-NEXT:    xscvdpspn 1, 1
-; PC64LE-NEXT:    xxsldwi 34, 0, 0, 1
-; PC64LE-NEXT:    xscvdpspn 0, 2
-; PC64LE-NEXT:    xxsldwi 35, 1, 1, 1
-; PC64LE-NEXT:    vmrglw 2, 3, 2
-; PC64LE-NEXT:    lvx 3, 0, 3
-; PC64LE-NEXT:    xxsldwi 36, 0, 0, 1
-; PC64LE-NEXT:    vperm 2, 4, 2, 3
+; PC64LE-NEXT:    addi 3, 3, .LCPI109_0@toc@l
+; PC64LE-NEXT:    lvx 2, 0, 3
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_floor_v3f32:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI109_0@toc@ha
-; PC64LE9-NEXT:    lfs 0, .LCPI109_0@toc@l(3)
-; PC64LE9-NEXT:    addis 3, 2, .LCPI109_1@toc@ha
-; PC64LE9-NEXT:    lfs 1, .LCPI109_1@toc@l(3)
-; PC64LE9-NEXT:    addis 3, 2, .LCPI109_2@toc@ha
-; PC64LE9-NEXT:    xsrdpim 0, 0
-; PC64LE9-NEXT:    lfs 2, .LCPI109_2@toc@l(3)
-; PC64LE9-NEXT:    addis 3, 2, .LCPI109_3@toc@ha
-; PC64LE9-NEXT:    addi 3, 3, .LCPI109_3@toc@l
-; PC64LE9-NEXT:    xsrdpim 1, 1
-; PC64LE9-NEXT:    xsrdpim 2, 2
-; PC64LE9-NEXT:    xscvdpspn 0, 0
-; PC64LE9-NEXT:    xscvdpspn 1, 1
-; PC64LE9-NEXT:    xscvdpspn 2, 2
-; PC64LE9-NEXT:    xxsldwi 36, 0, 0, 1
-; PC64LE9-NEXT:    xxsldwi 35, 1, 1, 1
-; PC64LE9-NEXT:    xxsldwi 34, 2, 2, 1
-; PC64LE9-NEXT:    vmrglw 2, 3, 2
-; PC64LE9-NEXT:    lxvx 35, 0, 3
-; PC64LE9-NEXT:    vperm 2, 4, 2, 3
+; PC64LE9-NEXT:    addi 3, 3, .LCPI109_0@toc@l
+; PC64LE9-NEXT:    lxvx 34, 0, 3
 ; PC64LE9-NEXT:    blr
 entry:
   %floor = call <3 x float> @llvm.experimental.constrained.floor.v3f32(
@@ -6617,31 +6528,18 @@ entry:
 define <3 x double> @constrained_vector_floor_v3f64() #0 {
 ; PC64LE-LABEL: constrained_vector_floor_v3f64:
 ; PC64LE:       # %bb.0: # %entry
-; PC64LE-NEXT:    addis 3, 2, .LCPI110_1@toc@ha
-; PC64LE-NEXT:    addi 3, 3, .LCPI110_1@toc@l
-; PC64LE-NEXT:    lxvd2x 0, 0, 3
 ; PC64LE-NEXT:    addis 3, 2, .LCPI110_0@toc@ha
 ; PC64LE-NEXT:    lfs 1, .LCPI110_0@toc@l(3)
-; PC64LE-NEXT:    xxswapd 0, 0
-; PC64LE-NEXT:    xsrdpim 3, 1
-; PC64LE-NEXT:    xvrdpim 2, 0
-; PC64LE-NEXT:    xxswapd 1, 2
-; PC64LE-NEXT:    # kill: def $f2 killed $f2 killed $vsl2
-; PC64LE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; PC64LE-NEXT:    fmr 2, 1
+; PC64LE-NEXT:    fmr 3, 1
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_floor_v3f64:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI110_0@toc@ha
-; PC64LE9-NEXT:    lfs 0, .LCPI110_0@toc@l(3)
-; PC64LE9-NEXT:    addis 3, 2, .LCPI110_1@toc@ha
-; PC64LE9-NEXT:    addi 3, 3, .LCPI110_1@toc@l
-; PC64LE9-NEXT:    xsrdpim 3, 0
-; PC64LE9-NEXT:    lxvx 0, 0, 3
-; PC64LE9-NEXT:    xvrdpim 2, 0
-; PC64LE9-NEXT:    xxswapd 1, 2
-; PC64LE9-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
-; PC64LE9-NEXT:    # kill: def $f2 killed $f2 killed $vsl2
+; PC64LE9-NEXT:    lfs 1, .LCPI110_0@toc@l(3)
+; PC64LE9-NEXT:    fmr 2, 1
+; PC64LE9-NEXT:    fmr 3, 1
 ; PC64LE9-NEXT:    blr
 entry:
   %floor = call <3 x double> @llvm.experimental.constrained.floor.v3f64(
@@ -6654,19 +6552,17 @@ define <1 x float> @constrained_vector_round_v1f32() #0 {
 ; PC64LE-LABEL: constrained_vector_round_v1f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI111_0@toc@ha
-; PC64LE-NEXT:    lfs 0, .LCPI111_0@toc@l(3)
-; PC64LE-NEXT:    xsrdpi 0, 0
-; PC64LE-NEXT:    xscvdpspn 0, 0
-; PC64LE-NEXT:    xxsldwi 34, 0, 0, 1
+; PC64LE-NEXT:    addi 3, 3, .LCPI111_0@toc@l
+; PC64LE-NEXT:    lfiwzx 0, 0, 3
+; PC64LE-NEXT:    xxpermdi 34, 0, 0, 2
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_round_v1f32:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI111_0@toc@ha
-; PC64LE9-NEXT:    lfs 0, .LCPI111_0@toc@l(3)
-; PC64LE9-NEXT:    xsrdpi 0, 0
-; PC64LE9-NEXT:    xscvdpspn 0, 0
-; PC64LE9-NEXT:    xxsldwi 34, 0, 0, 1
+; PC64LE9-NEXT:    addi 3, 3, .LCPI111_0@toc@l
+; PC64LE9-NEXT:    lfiwzx 0, 0, 3
+; PC64LE9-NEXT:    xxpermdi 34, 0, 0, 2
 ; PC64LE9-NEXT:    blr
 entry:
   %round = call <1 x float> @llvm.experimental.constrained.round.v1f32(
@@ -6681,16 +6577,14 @@ define <2 x double> @constrained_vector_round_v2f64() #0 {
 ; PC64LE-NEXT:    addis 3, 2, .LCPI112_0@toc@ha
 ; PC64LE-NEXT:    addi 3, 3, .LCPI112_0@toc@l
 ; PC64LE-NEXT:    lxvd2x 0, 0, 3
-; PC64LE-NEXT:    xxswapd 0, 0
-; PC64LE-NEXT:    xvrdpi 34, 0
+; PC64LE-NEXT:    xxswapd 34, 0
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_round_v2f64:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI112_0@toc@ha
 ; PC64LE9-NEXT:    addi 3, 3, .LCPI112_0@toc@l
-; PC64LE9-NEXT:    lxvx 0, 0, 3
-; PC64LE9-NEXT:    xvrdpi 34, 0
+; PC64LE9-NEXT:    lxvx 34, 0, 3
 ; PC64LE9-NEXT:    blr
 entry:
   %round = call <2 x double> @llvm.experimental.constrained.round.v2f64(
@@ -6702,50 +6596,16 @@ entry:
 define <3 x float> @constrained_vector_round_v3f32() #0 {
 ; PC64LE-LABEL: constrained_vector_round_v3f32:
 ; PC64LE:       # %bb.0: # %entry
-; PC64LE-NEXT:    addis 3, 2, .LCPI113_2@toc@ha
-; PC64LE-NEXT:    addis 4, 2, .LCPI113_1@toc@ha
-; PC64LE-NEXT:    lfs 0, .LCPI113_2@toc@l(3)
-; PC64LE-NEXT:    lfs 1, .LCPI113_1@toc@l(4)
 ; PC64LE-NEXT:    addis 3, 2, .LCPI113_0@toc@ha
-; PC64LE-NEXT:    xsrdpi 0, 0
-; PC64LE-NEXT:    lfs 2, .LCPI113_0@toc@l(3)
-; PC64LE-NEXT:    addis 3, 2, .LCPI113_3@toc@ha
-; PC64LE-NEXT:    xsrdpi 1, 1
-; PC64LE-NEXT:    addi 3, 3, .LCPI113_3@toc@l
-; PC64LE-NEXT:    xsrdpi 2, 2
-; PC64LE-NEXT:    xscvdpspn 0, 0
-; PC64LE-NEXT:    xscvdpspn 1, 1
-; PC64LE-NEXT:    xxsldwi 34, 0, 0, 1
-; PC64LE-NEXT:    xscvdpspn 0, 2
-; PC64LE-NEXT:    xxsldwi 35, 1, 1, 1
-; PC64LE-NEXT:    vmrglw 2, 3, 2
-; PC64LE-NEXT:    lvx 3, 0, 3
-; PC64LE-NEXT:    xxsldwi 36, 0, 0, 1
-; PC64LE-NEXT:    vperm 2, 4, 2, 3
+; PC64LE-NEXT:    addi 3, 3, .LCPI113_0@toc@l
+; PC64LE-NEXT:    lvx 2, 0, 3
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_round_v3f32:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI113_0@toc@ha
-; PC64LE9-NEXT:    lfs 0, .LCPI113_0@toc@l(3)
-; PC64LE9-NEXT:    addis 3, 2, .LCPI113_1@toc@ha
-; PC64LE9-NEXT:    lfs 1, .LCPI113_1@toc@l(3)
-; PC64LE9-NEXT:    addis 3, 2, .LCPI113_2@toc@ha
-; PC64LE9-NEXT:    xsrdpi 0, 0
-; PC64LE9-NEXT:    lfs 2, .LCPI113_2@toc@l(3)
-; PC64LE9-NEXT:    addis 3, 2, .LCPI113_3@toc@ha
-; PC64LE9-NEXT:    addi 3, 3, .LCPI113_3@toc@l
-; PC64LE9-NEXT:    xsrdpi 1, 1
-; PC64LE9-NEXT:    xsrdpi 2, 2
-; PC64LE9-NEXT:    xscvdpspn 0, 0
-; PC64LE9-NEXT:    xscvdpspn 1, 1
-; PC64LE9-NEXT:    xscvdpspn 2, 2
-; PC64LE9-NEXT:    xxsldwi 36, 0, 0, 1
-; PC64LE9-NEXT:    xxsldwi 35, 1, 1, 1
-; PC64LE9-NEXT:    xxsldwi 34, 2, 2, 1
-; PC64LE9-NEXT:    vmrglw 2, 3, 2
-; PC64LE9-NEXT:    lxvx 35, 0, 3
-; PC64LE9-NEXT:    vperm 2, 4, 2, 3
+; PC64LE9-NEXT:    addi 3, 3, .LCPI113_0@toc@l
+; PC64LE9-NEXT:    lxvx 34, 0, 3
 ; PC64LE9-NEXT:    blr
 entry:
   %round = call <3 x float> @llvm.experimental.constrained.round.v3f32(
@@ -6758,31 +6618,20 @@ entry:
 define <3 x double> @constrained_vector_round_v3f64() #0 {
 ; PC64LE-LABEL: constrained_vector_round_v3f64:
 ; PC64LE:       # %bb.0: # %entry
-; PC64LE-NEXT:    addis 3, 2, .LCPI114_1@toc@ha
-; PC64LE-NEXT:    addi 3, 3, .LCPI114_1@toc@l
-; PC64LE-NEXT:    lxvd2x 0, 0, 3
+; PC64LE-NEXT:    addis 4, 2, .LCPI114_1@toc@ha
 ; PC64LE-NEXT:    addis 3, 2, .LCPI114_0@toc@ha
+; PC64LE-NEXT:    lfs 2, .LCPI114_1@toc@l(4)
 ; PC64LE-NEXT:    lfs 1, .LCPI114_0@toc@l(3)
-; PC64LE-NEXT:    xxswapd 0, 0
-; PC64LE-NEXT:    xsrdpi 3, 1
-; PC64LE-NEXT:    xvrdpi 2, 0
-; PC64LE-NEXT:    xxswapd 1, 2
-; PC64LE-NEXT:    # kill: def $f2 killed $f2 killed $vsl2
-; PC64LE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; PC64LE-NEXT:    fmr 3, 2
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_round_v3f64:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI114_0@toc@ha
-; PC64LE9-NEXT:    lfs 0, .LCPI114_0@toc@l(3)
+; PC64LE9-NEXT:    lfs 1, .LCPI114_0@toc@l(3)
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI114_1@toc@ha
-; PC64LE9-NEXT:    addi 3, 3, .LCPI114_1@toc@l
-; PC64LE9-NEXT:    xsrdpi 3, 0
-; PC64LE9-NEXT:    lxvx 0, 0, 3
-; PC64LE9-NEXT:    xvrdpi 2, 0
-; PC64LE9-NEXT:    xxswapd 1, 2
-; PC64LE9-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
-; PC64LE9-NEXT:    # kill: def $f2 killed $f2 killed $vsl2
+; PC64LE9-NEXT:    lfs 2, .LCPI114_1@toc@l(3)
+; PC64LE9-NEXT:    fmr 3, 2
 ; PC64LE9-NEXT:    blr
 entry:
   %round = call <3 x double> @llvm.experimental.constrained.round.v3f64(
@@ -6795,19 +6644,17 @@ define <1 x float> @constrained_vector_trunc_v1f32() #0 {
 ; PC64LE-LABEL: constrained_vector_trunc_v1f32:
 ; PC64LE:       # %bb.0: # %entry
 ; PC64LE-NEXT:    addis 3, 2, .LCPI115_0@toc@ha
-; PC64LE-NEXT:    lfs 0, .LCPI115_0@toc@l(3)
-; PC64LE-NEXT:    xsrdpiz 0, 0
-; PC64LE-NEXT:    xscvdpspn 0, 0
-; PC64LE-NEXT:    xxsldwi 34, 0, 0, 1
+; PC64LE-NEXT:    addi 3, 3, .LCPI115_0@toc@l
+; PC64LE-NEXT:    lfiwzx 0, 0, 3
+; PC64LE-NEXT:    xxpermdi 34, 0, 0, 2
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_trunc_v1f32:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI115_0@toc@ha
-; PC64LE9-NEXT:    lfs 0, .LCPI115_0@toc@l(3)
-; PC64LE9-NEXT:    xsrdpiz 0, 0
-; PC64LE9-NEXT:    xscvdpspn 0, 0
-; PC64LE9-NEXT:    xxsldwi 34, 0, 0, 1
+; PC64LE9-NEXT:    addi 3, 3, .LCPI115_0@toc@l
+; PC64LE9-NEXT:    lfiwzx 0, 0, 3
+; PC64LE9-NEXT:    xxpermdi 34, 0, 0, 2
 ; PC64LE9-NEXT:    blr
 entry:
   %trunc = call <1 x float> @llvm.experimental.constrained.trunc.v1f32(
@@ -6822,16 +6669,14 @@ define <2 x double> @constrained_vector_trunc_v2f64() #0 {
 ; PC64LE-NEXT:    addis 3, 2, .LCPI116_0@toc@ha
 ; PC64LE-NEXT:    addi 3, 3, .LCPI116_0@toc@l
 ; PC64LE-NEXT:    lxvd2x 0, 0, 3
-; PC64LE-NEXT:    xxswapd 0, 0
-; PC64LE-NEXT:    xvrdpiz 34, 0
+; PC64LE-NEXT:    xxswapd 34, 0
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_trunc_v2f64:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI116_0@toc@ha
 ; PC64LE9-NEXT:    addi 3, 3, .LCPI116_0@toc@l
-; PC64LE9-NEXT:    lxvx 0, 0, 3
-; PC64LE9-NEXT:    xvrdpiz 34, 0
+; PC64LE9-NEXT:    lxvx 34, 0, 3
 ; PC64LE9-NEXT:    blr
 entry:
   %trunc = call <2 x double> @llvm.experimental.constrained.trunc.v2f64(
@@ -6843,50 +6688,16 @@ entry:
 define <3 x float> @constrained_vector_trunc_v3f32() #0 {
 ; PC64LE-LABEL: constrained_vector_trunc_v3f32:
 ; PC64LE:       # %bb.0: # %entry
-; PC64LE-NEXT:    addis 3, 2, .LCPI117_2@toc@ha
-; PC64LE-NEXT:    addis 4, 2, .LCPI117_1@toc@ha
-; PC64LE-NEXT:    lfs 0, .LCPI117_2@toc@l(3)
-; PC64LE-NEXT:    lfs 1, .LCPI117_1@toc@l(4)
 ; PC64LE-NEXT:    addis 3, 2, .LCPI117_0@toc@ha
-; PC64LE-NEXT:    xsrdpiz 0, 0
-; PC64LE-NEXT:    lfs 2, .LCPI117_0@toc@l(3)
-; PC64LE-NEXT:    addis 3, 2, .LCPI117_3@toc@ha
-; PC64LE-NEXT:    xsrdpiz 1, 1
-; PC64LE-NEXT:    addi 3, 3, .LCPI117_3@toc@l
-; PC64LE-NEXT:    xsrdpiz 2, 2
-; PC64LE-NEXT:    xscvdpspn 0, 0
-; PC64LE-NEXT:    xscvdpspn 1, 1
-; PC64LE-NEXT:    xxsldwi 34, 0, 0, 1
-; PC64LE-NEXT:    xscvdpspn 0, 2
-; PC64LE-NEXT:    xxsldwi 35, 1, 1, 1
-; PC64LE-NEXT:    vmrglw 2, 3, 2
-; PC64LE-NEXT:    lvx 3, 0, 3
-; PC64LE-NEXT:    xxsldwi 36, 0, 0, 1
-; PC64LE-NEXT:    vperm 2, 4, 2, 3
+; PC64LE-NEXT:    addi 3, 3, .LCPI117_0@toc@l
+; PC64LE-NEXT:    lvx 2, 0, 3
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_trunc_v3f32:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI117_0@toc@ha
-; PC64LE9-NEXT:    lfs 0, .LCPI117_0@toc@l(3)
-; PC64LE9-NEXT:    addis 3, 2, .LCPI117_1@toc@ha
-; PC64LE9-NEXT:    lfs 1, .LCPI117_1@toc@l(3)
-; PC64LE9-NEXT:    addis 3, 2, .LCPI117_2@toc@ha
-; PC64LE9-NEXT:    xsrdpiz 0, 0
-; PC64LE9-NEXT:    lfs 2, .LCPI117_2@toc@l(3)
-; PC64LE9-NEXT:    addis 3, 2, .LCPI117_3@toc@ha
-; PC64LE9-NEXT:    addi 3, 3, .LCPI117_3@toc@l
-; PC64LE9-NEXT:    xsrdpiz 1, 1
-; PC64LE9-NEXT:    xsrdpiz 2, 2
-; PC64LE9-NEXT:    xscvdpspn 0, 0
-; PC64LE9-NEXT:    xscvdpspn 1, 1
-; PC64LE9-NEXT:    xscvdpspn 2, 2
-; PC64LE9-NEXT:    xxsldwi 36, 0, 0, 1
-; PC64LE9-NEXT:    xxsldwi 35, 1, 1, 1
-; PC64LE9-NEXT:    xxsldwi 34, 2, 2, 1
-; PC64LE9-NEXT:    vmrglw 2, 3, 2
-; PC64LE9-NEXT:    lxvx 35, 0, 3
-; PC64LE9-NEXT:    vperm 2, 4, 2, 3
+; PC64LE9-NEXT:    addi 3, 3, .LCPI117_0@toc@l
+; PC64LE9-NEXT:    lxvx 34, 0, 3
 ; PC64LE9-NEXT:    blr
 entry:
   %trunc = call <3 x float> @llvm.experimental.constrained.trunc.v3f32(
@@ -6898,31 +6709,18 @@ entry:
 define <3 x double> @constrained_vector_trunc_v3f64() #0 {
 ; PC64LE-LABEL: constrained_vector_trunc_v3f64:
 ; PC64LE:       # %bb.0: # %entry
-; PC64LE-NEXT:    addis 3, 2, .LCPI118_1@toc@ha
-; PC64LE-NEXT:    addi 3, 3, .LCPI118_1@toc@l
-; PC64LE-NEXT:    lxvd2x 0, 0, 3
 ; PC64LE-NEXT:    addis 3, 2, .LCPI118_0@toc@ha
 ; PC64LE-NEXT:    lfs 1, .LCPI118_0@toc@l(3)
-; PC64LE-NEXT:    xxswapd 0, 0
-; PC64LE-NEXT:    xsrdpiz 3, 1
-; PC64LE-NEXT:    xvrdpiz 2, 0
-; PC64LE-NEXT:    xxswapd 1, 2
-; PC64LE-NEXT:    # kill: def $f2 killed $f2 killed $vsl2
-; PC64LE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; PC64LE-NEXT:    fmr 2, 1
+; PC64LE-NEXT:    fmr 3, 1
 ; PC64LE-NEXT:    blr
 ;
 ; PC64LE9-LABEL: constrained_vector_trunc_v3f64:
 ; PC64LE9:       # %bb.0: # %entry
 ; PC64LE9-NEXT:    addis 3, 2, .LCPI118_0@toc@ha
-; PC64LE9-NEXT:    lfs 0, .LCPI118_0@toc@l(3)
-; PC64LE9-NEXT:    addis 3, 2, .LCPI118_1@toc@ha
-; PC64LE9-NEXT:    addi 3, 3, .LCPI118_1@toc@l
-; PC64LE9-NEXT:    xsrdpiz 3, 0
-; PC64LE9-NEXT:    lxvx 0, 0, 3
-; PC64LE9-NEXT:    xvrdpiz 2, 0
-; PC64LE9-NEXT:    xxswapd 1, 2
-; PC64LE9-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
-; PC64LE9-NEXT:    # kill: def $f2 killed $f2 killed $vsl2
+; PC64LE9-NEXT:    lfs 1, .LCPI118_0@toc@l(3)
+; PC64LE9-NEXT:    fmr 2, 1
+; PC64LE9-NEXT:    fmr 3, 1
 ; PC64LE9-NEXT:    blr
 entry:
   %trunc = call <3 x double> @llvm.experimental.constrained.trunc.v3f64(
diff --git a/llvm/test/CodeGen/PowerPC/vmladduhm.ll b/llvm/test/CodeGen/PowerPC/vmladduhm.ll
index f2475d9e24907..3fae99829abcb 100644
--- a/llvm/test/CodeGen/PowerPC/vmladduhm.ll
+++ b/llvm/test/CodeGen/PowerPC/vmladduhm.ll
@@ -4,7 +4,7 @@
 define <8 x i16> @mul(<8 x i16> %m, <8 x i16> %n) {
 ; CHECK-LABEL: mul:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    xxlxor 36, 36, 36
+; CHECK-NEXT:    vxor 4, 4, 4
 ; CHECK-NEXT:    vmladduhm 2, 2, 3, 4
 ; CHECK-NEXT:    blr
 entry:
@@ -15,9 +15,7 @@ entry:
 define <8 x i16> @madd(<8 x i16> %m, <8 x i16> %n, <8 x i16> %o) {
 ; CHECK-LABEL: madd:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    xxlxor 37, 37, 37
-; CHECK-NEXT:    vmladduhm 2, 2, 3, 5
-; CHECK-NEXT:    vadduhm 2, 2, 4
+; CHECK-NEXT:    vmladduhm 2, 2, 3, 4
 ; CHECK-NEXT:    blr
 entry:
   %0 = mul <8 x i16> %m, %n
diff --git a/llvm/test/CodeGen/RISCV/atomic-cmpxchg.ll b/llvm/test/CodeGen/RISCV/atomic-cmpxchg.ll
index 43da05ebe7c71..f2691ba1a771f 100644
--- a/llvm/test/CodeGen/RISCV/atomic-cmpxchg.ll
+++ b/llvm/test/CodeGen/RISCV/atomic-cmpxchg.ll
@@ -1628,6 +1628,7 @@ define void @cmpxchg_i32_monotonic_monotonic(i32* %ptr, i32 %cmp, i32 %val) noun
 ;
 ; RV64IA-LABEL: cmpxchg_i32_monotonic_monotonic:
 ; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    sext.w a1, a1
 ; RV64IA-NEXT:  .LBB20_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w a3, (a0)
 ; RV64IA-NEXT:    bne a3, a1, .LBB20_3
@@ -1680,6 +1681,7 @@ define void @cmpxchg_i32_acquire_monotonic(i32* %ptr, i32 %cmp, i32 %val) nounwi
 ;
 ; RV64IA-LABEL: cmpxchg_i32_acquire_monotonic:
 ; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    sext.w a1, a1
 ; RV64IA-NEXT:  .LBB21_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aq a3, (a0)
 ; RV64IA-NEXT:    bne a3, a1, .LBB21_3
@@ -1732,6 +1734,7 @@ define void @cmpxchg_i32_acquire_acquire(i32* %ptr, i32 %cmp, i32 %val) nounwind
 ;
 ; RV64IA-LABEL: cmpxchg_i32_acquire_acquire:
 ; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    sext.w a1, a1
 ; RV64IA-NEXT:  .LBB22_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aq a3, (a0)
 ; RV64IA-NEXT:    bne a3, a1, .LBB22_3
@@ -1784,6 +1787,7 @@ define void @cmpxchg_i32_release_monotonic(i32* %ptr, i32 %cmp, i32 %val) nounwi
 ;
 ; RV64IA-LABEL: cmpxchg_i32_release_monotonic:
 ; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    sext.w a1, a1
 ; RV64IA-NEXT:  .LBB23_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w a3, (a0)
 ; RV64IA-NEXT:    bne a3, a1, .LBB23_3
@@ -1836,6 +1840,7 @@ define void @cmpxchg_i32_release_acquire(i32* %ptr, i32 %cmp, i32 %val) nounwind
 ;
 ; RV64IA-LABEL: cmpxchg_i32_release_acquire:
 ; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    sext.w a1, a1
 ; RV64IA-NEXT:  .LBB24_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w a3, (a0)
 ; RV64IA-NEXT:    bne a3, a1, .LBB24_3
@@ -1888,6 +1893,7 @@ define void @cmpxchg_i32_acq_rel_monotonic(i32* %ptr, i32 %cmp, i32 %val) nounwi
 ;
 ; RV64IA-LABEL: cmpxchg_i32_acq_rel_monotonic:
 ; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    sext.w a1, a1
 ; RV64IA-NEXT:  .LBB25_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aq a3, (a0)
 ; RV64IA-NEXT:    bne a3, a1, .LBB25_3
@@ -1940,6 +1946,7 @@ define void @cmpxchg_i32_acq_rel_acquire(i32* %ptr, i32 %cmp, i32 %val) nounwind
 ;
 ; RV64IA-LABEL: cmpxchg_i32_acq_rel_acquire:
 ; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    sext.w a1, a1
 ; RV64IA-NEXT:  .LBB26_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aq a3, (a0)
 ; RV64IA-NEXT:    bne a3, a1, .LBB26_3
@@ -1992,6 +1999,7 @@ define void @cmpxchg_i32_seq_cst_monotonic(i32* %ptr, i32 %cmp, i32 %val) nounwi
 ;
 ; RV64IA-LABEL: cmpxchg_i32_seq_cst_monotonic:
 ; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    sext.w a1, a1
 ; RV64IA-NEXT:  .LBB27_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aqrl a3, (a0)
 ; RV64IA-NEXT:    bne a3, a1, .LBB27_3
@@ -2044,6 +2052,7 @@ define void @cmpxchg_i32_seq_cst_acquire(i32* %ptr, i32 %cmp, i32 %val) nounwind
 ;
 ; RV64IA-LABEL: cmpxchg_i32_seq_cst_acquire:
 ; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    sext.w a1, a1
 ; RV64IA-NEXT:  .LBB28_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aqrl a3, (a0)
 ; RV64IA-NEXT:    bne a3, a1, .LBB28_3
@@ -2096,6 +2105,7 @@ define void @cmpxchg_i32_seq_cst_seq_cst(i32* %ptr, i32 %cmp, i32 %val) nounwind
 ;
 ; RV64IA-LABEL: cmpxchg_i32_seq_cst_seq_cst:
 ; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    sext.w a1, a1
 ; RV64IA-NEXT:  .LBB29_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aqrl a3, (a0)
 ; RV64IA-NEXT:    bne a3, a1, .LBB29_3
diff --git a/llvm/test/CodeGen/RISCV/attributes.ll b/llvm/test/CodeGen/RISCV/attributes.ll
new file mode 100644
index 0000000000000..b36b4430f157a
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/attributes.ll
@@ -0,0 +1,28 @@
+;; Generate ELF attributes from llc.
+
+; RUN: llc -mtriple=riscv32 -mattr=+m %s -o - | FileCheck --check-prefix=RV32M %s
+; RUN: llc -mtriple=riscv32 -mattr=+a %s -o - | FileCheck --check-prefix=RV32A %s
+; RUN: llc -mtriple=riscv32 -mattr=+f %s -o - | FileCheck --check-prefix=RV32F %s
+; RUN: llc -mtriple=riscv32 -mattr=+d %s -o - | FileCheck --check-prefix=RV32D %s
+; RUN: llc -mtriple=riscv32 -mattr=+c %s -o - | FileCheck --check-prefix=RV32C %s
+; RUN: llc -mtriple=riscv64 -mattr=+m %s -o - | FileCheck --check-prefix=RV64M %s
+; RUN: llc -mtriple=riscv64 -mattr=+a %s -o - | FileCheck --check-prefix=RV64A %s
+; RUN: llc -mtriple=riscv64 -mattr=+f %s -o - | FileCheck --check-prefix=RV64F %s
+; RUN: llc -mtriple=riscv64 -mattr=+d %s -o - | FileCheck --check-prefix=RV64D %s
+; RUN: llc -mtriple=riscv64 -mattr=+c %s -o - | FileCheck --check-prefix=RV64C %s
+
+; RV32M: .attribute 5, "rv32i2p0_m2p0"
+; RV32A: .attribute 5, "rv32i2p0_a2p0"
+; RV32F: .attribute 5, "rv32i2p0_f2p0"
+; RV32D: .attribute 5, "rv32i2p0_f2p0_d2p0"
+; RV32C: .attribute 5, "rv32i2p0_c2p0"
+; RV64M: .attribute 5, "rv64i2p0_m2p0"
+; RV64A: .attribute 5, "rv64i2p0_a2p0"
+; RV64F: .attribute 5, "rv64i2p0_f2p0"
+; RV64D: .attribute 5, "rv64i2p0_f2p0_d2p0"
+; RV64C: .attribute 5, "rv64i2p0_c2p0"
+
+define i32 @addi(i32 %a) {
+  %1 = add i32 %a, 1
+  ret i32 %1
+}
diff --git a/llvm/test/CodeGen/RISCV/thread-pointer.ll b/llvm/test/CodeGen/RISCV/thread-pointer.ll
new file mode 100644
index 0000000000000..ec495a49d2267
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/thread-pointer.ll
@@ -0,0 +1,14 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=riscv64 | FileCheck %s
+; RUN: llc < %s -mtriple=riscv32 | FileCheck %s
+
+declare i8* @llvm.thread.pointer()
+
+define i8* @thread_pointer() nounwind {
+; CHECK-LABEL: thread_pointer:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mv a0, tp
+; CHECK-NEXT:    ret
+  %1 = tail call i8* @llvm.thread.pointer()
+  ret i8* %1
+}
diff --git a/llvm/test/CodeGen/SystemZ/foldmemop-imm-01.ll b/llvm/test/CodeGen/SystemZ/foldmemop-imm-01.ll
new file mode 100644
index 0000000000000..08a32a39519bd
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/foldmemop-imm-01.ll
@@ -0,0 +1,55 @@
+; RUN: llc < %s -mtriple=s390x-linux-gnu -O3 -mcpu=z10 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -O3 -mcpu=z14 | FileCheck %s
+;
+; Test folding of spilled immediate loads and compares.
+
+define i32 @fun0(i32 *%src, i32 %arg) nounwind {
+; CHECK-LABEL: fun0:
+; CHECK: 	mvhi	164(%r15), 0            # 4-byte Folded Spill
+; CHECK:	mvc	164(4,%r15), 0(%r2)     # 4-byte Folded Spill
+; CHECK-LABEL: .LBB0_2:
+; CHECK:	chsi	164(%r15), 2            # 4-byte Folded Reload
+
+entry:
+  %cmp  = icmp eq i32 %arg, 0
+  br i1 %cmp, label %cond, label %exit
+
+cond:
+  %val0 = load i32, i32 *%src
+  call void asm sideeffect "", "~{r0},~{r1},~{r2},~{r3},~{r4},~{r5},~{r6},~{r7},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"() nounwind
+  br label %exit
+
+exit:
+  %tmp0 = phi i32 [0, %entry], [%val0, %cond]
+  call void asm sideeffect "", "~{r0},~{r1},~{r2},~{r3},~{r4},~{r5},~{r6},~{r7},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"() nounwind
+  %cmp0 = icmp ne i32 %tmp0, 2
+  %zxt0 = zext i1 %cmp0 to i32
+  %and0 = and i32 %arg, %zxt0
+
+  ret i32 %and0
+}
+
+define i64 @fun1(i64 *%src, i64 %arg) nounwind {
+; CHECK-LABEL: fun1:
+; CHECK: 	mvghi	168(%r15), 0            # 8-byte Folded Spill
+; CHECK:	mvc	168(8,%r15), 0(%r2)     # 8-byte Folded Spill
+; CHECK-LABEL: .LBB1_2:
+; CHECK:	cghsi	168(%r15), 2            # 8-byte Folded Reload
+entry:
+  %cmp  = icmp eq i64 %arg, 0
+  br i1 %cmp, label %cond, label %exit
+
+cond:
+  %val0 = load i64, i64 *%src
+  call void asm sideeffect "", "~{r0},~{r1},~{r2},~{r3},~{r4},~{r5},~{r6},~{r7},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"() nounwind
+  br label %exit
+
+exit:
+  %tmp0 = phi i64 [0, %entry], [%val0, %cond]
+  call void asm sideeffect "", "~{r0},~{r1},~{r2},~{r3},~{r4},~{r5},~{r6},~{r7},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"() nounwind
+  %cmp0 = icmp ne i64 %tmp0, 2
+  %zxt0 = zext i1 %cmp0 to i64
+  %and0 = and i64 %arg, %zxt0
+
+  ret i64 %and0
+}
diff --git a/llvm/test/CodeGen/SystemZ/foldmemop-imm-02.mir b/llvm/test/CodeGen/SystemZ/foldmemop-imm-02.mir
new file mode 100644
index 0000000000000..bf3347eb5c45d
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/foldmemop-imm-02.mir
@@ -0,0 +1,233 @@
+# RUN: llc -mtriple=s390x-linux-gnu -mcpu=z14 -start-before=greedy %s -o - \
+# RUN:   | FileCheck %s
+#
+# Test folding of a memory operand into logical compare with an immediate.
+
+--- |
+  define i32 @fun0(i32* %src, i32 %arg) { ret i32 0 }
+  define i64 @fun1(i64* %src, i64 %arg) { ret i64 0 }
+  define i32 @fun2(i32* %src, i32 %arg) { ret i32 0 }
+  define i64 @fun3(i64* %src, i64 %arg) { ret i64 0 }
+...
+
+
+# CHECK-LABEL: fun0:
+# CHECK: 	mvhi	164(%r15), 0            # 4-byte Folded Spill
+# CHECK:	mvc	164(4,%r15), 0(%r2)     # 4-byte Folded Spill
+# CHECK-LABEL: .LBB0_2:
+# CHECK:	clfhsi	164(%r15), 2            # 4-byte Folded Reload
+---
+name:            fun0
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: grx32bit }
+  - { id: 1, class: grx32bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: gr32bit }
+  - { id: 4, class: grx32bit }
+  - { id: 5, class: grx32bit }
+  - { id: 6, class: gr32bit }
+  - { id: 7, class: gr32bit }
+  - { id: 8, class: grx32bit }
+liveins:
+  - { reg: '$r2d', virtual-reg: '%2' }
+  - { reg: '$r3l', virtual-reg: '%3' }
+frameInfo:
+  maxAlignment:    1
+  hasOpaqueSPAdjustment: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    successors: %bb.1(0x30000000), %bb.2(0x50000000)
+    liveins: $r2d, $r3l
+  
+    %3:gr32bit = COPY $r3l
+    %2:addr64bit = COPY $r2d
+    %6:gr32bit = LHIMux 0
+    CHIMux %3, 0, implicit-def $cc
+    %8:grx32bit = LHIMux 0
+    BRC 14, 6, %bb.2, implicit killed $cc
+    J %bb.1
+  
+  bb.1:
+    %8:grx32bit = LMux %2, 0, $noreg :: (load 4 from %ir.src)
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def early-clobber $r15d
+  
+  bb.2:
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def early-clobber $r15d
+    CLFIMux %8, 2, implicit-def $cc
+    %6:gr32bit = LOCHIMux %6, 1, 14, 6, implicit killed $cc
+    %7:gr32bit = NRK %3, %6, implicit-def dead $cc
+    $r2l = COPY %7
+    Return implicit $r2l
+
+...
+
+
+# CHECK-LABEL: fun1:
+# CHECK: 	mvghi	168(%r15), 0            # 8-byte Folded Spill
+# CHECK:	mvc	168(8,%r15), 0(%r2)     # 8-byte Folded Spill
+# CHECK-LABEL: .LBB1_2:
+# CHECK:	clghsi	168(%r15), 2            # 8-byte Folded Reload
+---
+name:            fun1
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gr64bit }
+  - { id: 1, class: gr64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: gr64bit }
+  - { id: 4, class: gr64bit }
+  - { id: 5, class: gr64bit }
+  - { id: 6, class: gr64bit }
+  - { id: 7, class: gr64bit }
+  - { id: 8, class: gr64bit }
+liveins:
+  - { reg: '$r2d', virtual-reg: '%2' }
+  - { reg: '$r3d', virtual-reg: '%3' }
+frameInfo:
+  maxAlignment:    1
+  hasOpaqueSPAdjustment: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    successors: %bb.1(0x30000000), %bb.2(0x50000000)
+    liveins: $r2d, $r3d
+  
+    %3:gr64bit = COPY $r3d
+    %2:addr64bit = COPY $r2d
+    %6:gr64bit = LGHI 0
+    CGHI %3, 0, implicit-def $cc
+    %8:gr64bit = LGHI 0
+    BRC 14, 6, %bb.2, implicit killed $cc
+    J %bb.1
+  
+  bb.1:
+    %8:gr64bit = LG %2, 0, $noreg :: (load 8 from %ir.src)
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def early-clobber $r15d
+  
+  bb.2:
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def early-clobber $r15d
+    CLGFI %8, 2, implicit-def $cc
+    %6:gr64bit = LOCGHI %6, 1, 14, 6, implicit killed $cc
+    %7:gr64bit = NGRK %3, %6, implicit-def dead $cc
+    $r2d = COPY %7
+    Return implicit $r2d
+
+...
+
+
+# 17-bit immediate can not be folded
+
+# CHECK-LABEL: fun2:
+# CHECK: 	mvhi	164(%r15), 0            # 4-byte Folded Spill
+# CHECK:	mvc	164(4,%r15), 0(%r2)     # 4-byte Folded Spill
+# CHECK-LABEL: .LBB2_2:
+# CHECK:        l       %r0, 164(%r15)          # 4-byte Folded Reload
+# CHECK:        clfi    %r0, 65536
+---
+name:            fun2
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: grx32bit }
+  - { id: 1, class: grx32bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: gr32bit }
+  - { id: 4, class: grx32bit }
+  - { id: 5, class: grx32bit }
+  - { id: 6, class: gr32bit }
+  - { id: 7, class: gr32bit }
+  - { id: 8, class: grx32bit }
+liveins:
+  - { reg: '$r2d', virtual-reg: '%2' }
+  - { reg: '$r3l', virtual-reg: '%3' }
+frameInfo:
+  maxAlignment:    1
+  hasOpaqueSPAdjustment: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    successors: %bb.1(0x30000000), %bb.2(0x50000000)
+    liveins: $r2d, $r3l
+  
+    %3:gr32bit = COPY $r3l
+    %2:addr64bit = COPY $r2d
+    %6:gr32bit = LHIMux 0
+    CHIMux %3, 0, implicit-def $cc
+    %8:grx32bit = LHIMux 0
+    BRC 14, 6, %bb.2, implicit killed $cc
+    J %bb.1
+  
+  bb.1:
+    %8:grx32bit = LMux %2, 0, $noreg :: (load 4 from %ir.src)
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def early-clobber $r15d
+  
+  bb.2:
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def early-clobber $r15d
+    CLFIMux %8, 65536, implicit-def $cc
+    %6:gr32bit = LOCHIMux %6, 1, 14, 6, implicit killed $cc
+    %7:gr32bit = NRK %3, %6, implicit-def dead $cc
+    $r2l = COPY %7
+    Return implicit $r2l
+
+...
+
+
+# 17-bit immediate can not be folded
+
+# CHECK-LABEL: fun3:
+# CHECK: 	mvghi	168(%r15), 0            # 8-byte Folded Spill
+# CHECK:	mvc	168(8,%r15), 0(%r2)     # 8-byte Folded Spill
+# CHECK-LABEL: .LBB3_2:
+# CHECK:        lg      %r0, 168(%r15)          # 8-byte Folded Reload
+# CHECK:        clgfi   %r0, 65536
+---
+name:            fun3
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gr64bit }
+  - { id: 1, class: gr64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: gr64bit }
+  - { id: 4, class: gr64bit }
+  - { id: 5, class: gr64bit }
+  - { id: 6, class: gr64bit }
+  - { id: 7, class: gr64bit }
+  - { id: 8, class: gr64bit }
+liveins:
+  - { reg: '$r2d', virtual-reg: '%2' }
+  - { reg: '$r3d', virtual-reg: '%3' }
+frameInfo:
+  maxAlignment:    1
+  hasOpaqueSPAdjustment: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    successors: %bb.1(0x30000000), %bb.2(0x50000000)
+    liveins: $r2d, $r3d
+  
+    %3:gr64bit = COPY $r3d
+    %2:addr64bit = COPY $r2d
+    %6:gr64bit = LGHI 0
+    CGHI %3, 0, implicit-def $cc
+    %8:gr64bit = LGHI 0
+    BRC 14, 6, %bb.2, implicit killed $cc
+    J %bb.1
+  
+  bb.1:
+    %8:gr64bit = LG %2, 0, $noreg :: (load 8 from %ir.src)
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def early-clobber $r15d
+  
+  bb.2:
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def early-clobber $r15d
+    CLGFI %8, 65536, implicit-def $cc
+    %6:gr64bit = LOCGHI %6, 1, 14, 6, implicit killed $cc
+    %7:gr64bit = NGRK %3, %6, implicit-def dead $cc
+    $r2d = COPY %7
+    Return implicit $r2d
+
+...
diff --git a/llvm/test/CodeGen/SystemZ/foldmemop-msc.mir b/llvm/test/CodeGen/SystemZ/foldmemop-msc.mir
new file mode 100644
index 0000000000000..4541605182212
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/foldmemop-msc.mir
@@ -0,0 +1,219 @@
+# RUN: llc -mtriple=s390x-linux-gnu -mcpu=z14 -start-before=greedy %s -o - \
+# RUN:   | FileCheck %s
+#
+# Test folding of a memory operand into logical compare with an immediate.
+
+--- |
+  define i32 @fun0(i32* %src, i32 %arg) { ret i32 0 }
+  define i64 @fun1(i64* %src, i64 %arg) { ret i64 0 }
+  define i32 @fun2(i32* %src, i32 %arg) { ret i32 0 }
+  define i64 @fun3(i64* %src, i64 %arg) { ret i64 0 }
+...
+
+
+# CHECK-LABEL: fun0:
+# CHECK-LABEL: .LBB0_2:
+# CHECK:	msc     %r0, 164(%r15)                  # 4-byte Folded Reload
+---
+name:            fun0
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: grx32bit }
+  - { id: 1, class: grx32bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: gr32bit }
+  - { id: 4, class: grx32bit }
+  - { id: 5, class: grx32bit }
+  - { id: 6, class: gr32bit }
+  - { id: 7, class: gr32bit }
+  - { id: 8, class: gr32bit }
+liveins:
+  - { reg: '$r2d', virtual-reg: '%2' }
+  - { reg: '$r3l', virtual-reg: '%3' }
+frameInfo:
+  maxAlignment:    1
+  hasOpaqueSPAdjustment: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    successors: %bb.1(0x30000000), %bb.2(0x50000000)
+    liveins: $r2d, $r3l
+  
+    %3:gr32bit = COPY $r3l
+    %2:addr64bit = COPY $r2d
+    %6:gr32bit = LHIMux 0
+    CHIMux %3, 0, implicit-def $cc
+    %8:gr32bit = LHIMux 0
+    BRC 14, 6, %bb.2, implicit killed $cc
+    J %bb.1
+  
+  bb.1:
+    %8:gr32bit = LMux %2, 0, $noreg :: (load 4 from %ir.src)
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def early-clobber $r15d
+  
+  bb.2:
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def early-clobber $r15d
+    %6:gr32bit = MSRKC %8, %6, implicit-def $cc
+    %6:gr32bit = LOCHIMux %6, 1, 14, 6, implicit killed $cc
+    %7:gr32bit = NRK %3, %6, implicit-def dead $cc
+    $r2l = COPY %7
+    Return implicit $r2l
+
+...
+
+
+# CHECK-LABEL: fun1:
+# CHECK-LABEL: .LBB1_2:
+# CHECK:	msc     %r0, 164(%r15)                  # 4-byte Folded Reload
+---
+name:            fun1
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: grx32bit }
+  - { id: 1, class: grx32bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: gr32bit }
+  - { id: 4, class: grx32bit }
+  - { id: 5, class: grx32bit }
+  - { id: 6, class: gr32bit }
+  - { id: 7, class: gr32bit }
+  - { id: 8, class: gr32bit }
+liveins:
+  - { reg: '$r2d', virtual-reg: '%2' }
+  - { reg: '$r3l', virtual-reg: '%3' }
+frameInfo:
+  maxAlignment:    1
+  hasOpaqueSPAdjustment: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    successors: %bb.1(0x30000000), %bb.2(0x50000000)
+    liveins: $r2d, $r3l
+  
+    %3:gr32bit = COPY $r3l
+    %2:addr64bit = COPY $r2d
+    %6:gr32bit = LHIMux 0
+    CHIMux %3, 0, implicit-def $cc
+    %8:gr32bit = LHIMux 0
+    BRC 14, 6, %bb.2, implicit killed $cc
+    J %bb.1
+  
+  bb.1:
+    %8:gr32bit = LMux %2, 0, $noreg :: (load 4 from %ir.src)
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def early-clobber $r15d
+  
+  bb.2:
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def early-clobber $r15d
+    %6:gr32bit = MSRKC %6, %8, implicit-def $cc
+    %6:gr32bit = LOCHIMux %6, 1, 14, 6, implicit killed $cc
+    %7:gr32bit = NRK %3, %6, implicit-def dead $cc
+    $r2l = COPY %7
+    Return implicit $r2l
+
+...
+
+
+# CHECK-LABEL: fun2:
+# CHECK-LABEL: .LBB2_2:
+# CHECK:	msgc    %r0, 168(%r15)                  # 8-byte Folded Reload
+---
+name:            fun2
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gr64bit }
+  - { id: 1, class: gr64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: gr64bit }
+  - { id: 4, class: gr64bit }
+  - { id: 5, class: gr64bit }
+  - { id: 6, class: gr64bit }
+  - { id: 7, class: gr64bit }
+  - { id: 8, class: gr64bit }
+liveins:
+  - { reg: '$r2d', virtual-reg: '%2' }
+  - { reg: '$r3d', virtual-reg: '%3' }
+frameInfo:
+  maxAlignment:    1
+  hasOpaqueSPAdjustment: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    successors: %bb.1(0x30000000), %bb.2(0x50000000)
+    liveins: $r2d, $r3d
+  
+    %3:gr64bit = COPY $r3d
+    %2:addr64bit = COPY $r2d
+    %6:gr64bit = LGHI 0
+    CGHI %3, 0, implicit-def $cc
+    %8:gr64bit = LGHI 0
+    BRC 14, 6, %bb.2, implicit killed $cc
+    J %bb.1
+  
+  bb.1:
+    %8:gr64bit = LG %2, 0, $noreg :: (load 8 from %ir.src)
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def early-clobber $r15d
+  
+  bb.2:
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def early-clobber $r15d
+    %6:gr64bit = MSGRKC %8, %6, implicit-def $cc
+    %6:gr64bit = LOCGHI %6, 1, 14, 6, implicit killed $cc
+    %7:gr64bit = NGRK %3, %6, implicit-def dead $cc
+    $r2d = COPY %7
+    Return implicit $r2d
+
+...
+
+
+# CHECK-LABEL: fun3:
+# CHECK-LABEL: .LBB3_2:
+# CHECK:	msgc    %r0, 168(%r15)                  # 8-byte Folded Reload
+---
+name:            fun3
+alignment:       16
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gr64bit }
+  - { id: 1, class: gr64bit }
+  - { id: 2, class: addr64bit }
+  - { id: 3, class: gr64bit }
+  - { id: 4, class: gr64bit }
+  - { id: 5, class: gr64bit }
+  - { id: 6, class: gr64bit }
+  - { id: 7, class: gr64bit }
+  - { id: 8, class: gr64bit }
+liveins:
+  - { reg: '$r2d', virtual-reg: '%2' }
+  - { reg: '$r3d', virtual-reg: '%3' }
+frameInfo:
+  maxAlignment:    1
+  hasOpaqueSPAdjustment: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    successors: %bb.1(0x30000000), %bb.2(0x50000000)
+    liveins: $r2d, $r3d
+  
+    %3:gr64bit = COPY $r3d
+    %2:addr64bit = COPY $r2d
+    %6:gr64bit = LGHI 0
+    CGHI %3, 0, implicit-def $cc
+    %8:gr64bit = LGHI 0
+    BRC 14, 6, %bb.2, implicit killed $cc
+    J %bb.1
+  
+  bb.1:
+    %8:gr64bit = LG %2, 0, $noreg :: (load 8 from %ir.src)
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def early-clobber $r15d
+  
+  bb.2:
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $r0d, 12, implicit-def dead early-clobber $r1d, 12, implicit-def dead early-clobber $r2d, 12, implicit-def dead early-clobber $r3d, 12, implicit-def dead early-clobber $r4d, 12, implicit-def dead early-clobber $r5d, 12, implicit-def dead early-clobber $r6d, 12, implicit-def dead early-clobber $r7d, 12, implicit-def dead early-clobber $r8d, 12, implicit-def dead early-clobber $r9d, 12, implicit-def dead early-clobber $r10d, 12, implicit-def dead early-clobber $r11d, 12, implicit-def dead early-clobber $r12d, 12, implicit-def dead early-clobber $r13d, 12, implicit-def dead early-clobber $r14d, 12, implicit-def early-clobber $r15d
+    %6:gr64bit = MSGRKC %6, %8, implicit-def $cc
+    %6:gr64bit = LOCGHI %6, 1, 14, 6, implicit killed $cc
+    %7:gr64bit = NGRK %3, %6, implicit-def dead $cc
+    $r2d = COPY %7
+    Return implicit $r2d
+
+...
diff --git a/llvm/test/CodeGen/SystemZ/prefetch-02.ll b/llvm/test/CodeGen/SystemZ/prefetch-02.ll
new file mode 100644
index 0000000000000..5f417699f98a3
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/prefetch-02.ll
@@ -0,0 +1,33 @@
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 -prefetch-distance=100 \
+; RUN:   -stop-after=loop-data-prefetch | FileCheck %s -check-prefix=FAR-PREFETCH
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 -prefetch-distance=20 \
+; RUN:   -stop-after=loop-data-prefetch | FileCheck %s -check-prefix=NEAR-PREFETCH
+;
+; Check that prefetches are not emitted when the known constant trip count of
+; the loop is smaller than the estimated "iterations ahead" of the prefetch.
+;
+; FAR-PREFETCH-LABEL: fun
+; FAR-PREFETCH-NOT: call void @llvm.prefetch
+
+; NEAR-PREFETCH-LABEL: fun
+; NEAR-PREFETCH: call void @llvm.prefetch
+
+
+define void @fun(i32* nocapture %Src, i32* nocapture readonly %Dst) {
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next.9, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %Dst, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %Src, i64 %indvars.iv
+  store i32 %0, i32* %arrayidx2, align 4
+  %indvars.iv.next.9 = add nuw nsw i64 %indvars.iv, 1600
+  %cmp.9 = icmp ult i64 %indvars.iv.next.9, 11200
+  br i1 %cmp.9, label %for.body, label %for.cond.cleanup
+}
+
diff --git a/llvm/test/CodeGen/SystemZ/prefetch-03.ll b/llvm/test/CodeGen/SystemZ/prefetch-03.ll
new file mode 100644
index 0000000000000..9c2e92689caf7
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/prefetch-03.ll
@@ -0,0 +1,46 @@
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 -prefetch-distance=50 \
+; RUN:   -loop-prefetch-writes -stop-after=loop-data-prefetch | FileCheck %s
+;
+; Check that prefetches are emitted in a position that is executed each
+; iteration for each targeted memory instruction. The two stores in %true and
+; %false are within one cache line in memory, so they should get a single
+; prefetch in %for.body.
+;
+; CHECK-LABEL: for.body
+; CHECK: call void @llvm.prefetch.p0i8(i8* {{.*}}, i32 0
+; CHECK: call void @llvm.prefetch.p0i8(i8* {{.*}}, i32 1
+; CHECK-LABEL: true
+; CHECK-LABEL: false
+; CHECK-LABEL: latch
+
+define void @fun(i32* nocapture %Src, i32* nocapture readonly %Dst) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next.9, %latch ]
+  %arrayidx = getelementptr inbounds i32, i32* %Dst, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp = icmp sgt i32 %0, 0
+  br i1 %cmp, label %true, label %false
+
+true:  
+  %arrayidx2 = getelementptr inbounds i32, i32* %Src, i64 %indvars.iv
+  store i32 %0, i32* %arrayidx2, align 4
+  br label %latch
+
+false:
+  %a = add i64 %indvars.iv, 8
+  %arrayidx3 = getelementptr inbounds i32, i32* %Src, i64 %a
+  store i32 %0, i32* %arrayidx3, align 4
+  br label %latch
+
+latch:
+  %indvars.iv.next.9 = add nuw nsw i64 %indvars.iv, 1600
+  %cmp.9 = icmp ult i64 %indvars.iv.next.9, 11200
+  br i1 %cmp.9, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/SystemZ/prefetch-04.ll b/llvm/test/CodeGen/SystemZ/prefetch-04.ll
new file mode 100644
index 0000000000000..af101ec7fa34d
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/prefetch-04.ll
@@ -0,0 +1,28 @@
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 -prefetch-distance=20 \
+; RUN:   -loop-prefetch-writes -stop-after=loop-data-prefetch | FileCheck %s
+;
+; Check that for a load followed by a store to the same address gets a single
+; write prefetch.
+;
+; CHECK-LABEL: for.body
+; CHECK: call void @llvm.prefetch.p0i8(i8* %scevgep{{.*}}, i32 1, i32 3, i32 1
+; CHECK-not: call void @llvm.prefetch
+
+define void @fun(i32* nocapture %Src, i32* nocapture readonly %Dst) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next.9, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %Dst, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %a = add i32 %0, 128
+  store i32 %a, i32* %arrayidx, align 4
+  %indvars.iv.next.9 = add nuw nsw i64 %indvars.iv, 1600
+  %cmp.9 = icmp ult i64 %indvars.iv.next.9, 11200
+  br i1 %cmp.9, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vaddv.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vaddv.mir
index 42c6f4323d101..ae1c783c0606d 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vaddv.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vaddv.mir
@@ -751,6 +751,47 @@
     ret i32 %res
   }
 
+  define hidden i32 @illegal_vmull_non_zero(i16* %x, i16* %y, i16* %z, i32 %n) {
+  entry:
+    %cmp22 = icmp sgt i32 %n, 0
+    %0 = add i32 %n, 7
+    %1 = icmp slt i32 %n, 8
+    %smin = select i1 %1, i32 %n, i32 8
+    %2 = sub i32 %0, %smin
+    %3 = lshr i32 %2, 3
+    %4 = add nuw nsw i32 %3, 1
+    br i1 %cmp22, label %while.body.preheader, label %while.end
+
+  while.body.preheader:                             ; preds = %entry
+    call void @llvm.set.loop.iterations.i32(i32 %4)
+    br label %while.body
+
+  while.body:                                       ; preds = %while.body.preheader, %while.body
+    %x.addr.026 = phi i16* [ %add.ptr, %while.body ], [ %x, %while.body.preheader ]
+    %y.addr.025 = phi i16* [ %add.ptr4, %while.body ], [ %y, %while.body.preheader ]
+    %n.addr.023 = phi i32 [ %sub, %while.body ], [ %n, %while.body.preheader ]
+    %acc = phi i32 [ %acc.next, %while.body ], [ 0, %while.body.preheader ]
+    %5 = phi i32 [ %4, %while.body.preheader ], [ %6, %while.body ]
+    %tmp3 = bitcast i16* %y.addr.025 to <8 x i16>*
+    %tmp1 = bitcast i16* %x.addr.026 to <8 x i16>*
+    %tmp = tail call <8 x i1> @llvm.arm.mve.vctp16(i32 %n.addr.023)
+    %tmp2 = tail call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %tmp1, i32 2, <8 x i1> %tmp, <8 x i16> zeroinitializer)
+    %tmp4 = tail call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %tmp3, i32 2, <8 x i1> %tmp, <8 x i16> zeroinitializer)
+    %mul = tail call <4 x i32> @llvm.arm.mve.vmull.v4i32.v8i16(<8 x i16> %tmp2, <8 x i16> %tmp4, i32 0, i32 1)
+    %reduce = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %mul)
+    %acc.next = add i32 %reduce, %acc
+    %add.ptr = getelementptr inbounds i16, i16* %x.addr.026, i32 8
+    %add.ptr4 = getelementptr inbounds i16, i16* %y.addr.025, i32 8
+    %sub = add nsw i32 %n.addr.023, -8
+    %6 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %5, i32 1)
+    %7 = icmp ne i32 %6, 0
+    br i1 %7, label %while.body, label %while.end
+
+  while.end:                                        ; preds = %while.body, %entry
+    %res = phi i32 [ 0, %entry ], [ %acc.next, %while.body ]
+    ret i32 %res
+  }
+
   declare <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>*, i32 immarg, <8 x i1>, <8 x i8>)
   declare <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>*, i32 immarg, <4 x i1>, <4 x i16>)
   declare <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>*, i32 immarg, <8 x i1>, <8 x i16>)
@@ -812,25 +853,14 @@ body:             |
   ; CHECK:   tCMPi8 renamable $r2, 0, 14 /* CC::al */, $noreg, implicit-def $cpsr
   ; CHECK:   t2IT 0, 8, implicit-def $itstate
   ; CHECK:   tPOP_RET 0 /* CC::eq */, killed $cpsr, def $r7, def $pc, implicit killed $itstate
-  ; CHECK:   renamable $r3, dead $cpsr = tADDi3 renamable $r2, 3, 14 /* CC::al */, $noreg
-  ; CHECK:   renamable $r3 = t2BICri killed renamable $r3, 3, 14 /* CC::al */, $noreg, $noreg
-  ; CHECK:   renamable $r12 = t2SUBri killed renamable $r3, 4, 14 /* CC::al */, $noreg, $noreg
-  ; CHECK:   renamable $r3, dead $cpsr = tMOVi8 1, 14 /* CC::al */, $noreg
-  ; CHECK:   renamable $r12 = nuw nsw t2ADDrs killed renamable $r3, killed renamable $r12, 19, 14 /* CC::al */, $noreg, $noreg
-  ; CHECK:   dead $lr = t2DLS renamable $r12
-  ; CHECK:   $r3 = tMOVr killed $r12, 14 /* CC::al */, $noreg
+  ; CHECK:   $lr = MVE_DLSTP_32 killed renamable $r2
   ; CHECK: bb.1.vector.body:
   ; CHECK:   successors: %bb.1(0x7c000000), %bb.2(0x04000000)
-  ; CHECK:   liveins: $r0, $r1, $r2, $r3
-  ; CHECK:   renamable $vpr = MVE_VCTP32 renamable $r2, 0, $noreg
-  ; CHECK:   MVE_VPST 8, implicit $vpr
-  ; CHECK:   renamable $r0, renamable $q0 = MVE_VLDRHS32_post killed renamable $r0, 8, 1, killed renamable $vpr :: (load 8 from %ir.lsr.iv17, align 2)
+  ; CHECK:   liveins: $lr, $r0, $r1
+  ; CHECK:   renamable $r0, renamable $q0 = MVE_VLDRHS32_post killed renamable $r0, 8, 0, killed $noreg :: (load 8 from %ir.lsr.iv17, align 2)
   ; CHECK:   renamable $r12 = MVE_VADDVu32no_acc killed renamable $q0, 0, $noreg
-  ; CHECK:   $lr = tMOVr $r3, 14 /* CC::al */, $noreg
   ; CHECK:   early-clobber renamable $r1 = t2STR_POST killed renamable $r12, killed renamable $r1, 4, 14 /* CC::al */, $noreg :: (store 4 into %ir.store.addr)
-  ; CHECK:   renamable $r3, dead $cpsr = nsw tSUBi8 killed $r3, 1, 14 /* CC::al */, $noreg
-  ; CHECK:   renamable $r2, dead $cpsr = tSUBi8 killed renamable $r2, 4, 14 /* CC::al */, $noreg
-  ; CHECK:   dead $lr = t2LEUpdate killed renamable $lr, %bb.1
+  ; CHECK:   $lr = MVE_LETP killed renamable $lr, %bb.1
   ; CHECK: bb.2.exit:
   ; CHECK:   tPOP_RET 14 /* CC::al */, $noreg, def $r7, def $pc
   bb.0.entry:
@@ -914,25 +944,14 @@ body:             |
   ; CHECK:   tCMPi8 renamable $r2, 0, 14 /* CC::al */, $noreg, implicit-def $cpsr
   ; CHECK:   t2IT 0, 8, implicit-def $itstate
   ; CHECK:   tPOP_RET 0 /* CC::eq */, killed $cpsr, def $r7, def $pc, implicit killed $itstate
-  ; CHECK:   renamable $r3, dead $cpsr = tADDi3 renamable $r2, 3, 14 /* CC::al */, $noreg
-  ; CHECK:   renamable $r3 = t2BICri killed renamable $r3, 3, 14 /* CC::al */, $noreg, $noreg
-  ; CHECK:   renamable $r12 = t2SUBri killed renamable $r3, 4, 14 /* CC::al */, $noreg, $noreg
-  ; CHECK:   renamable $r3, dead $cpsr = tMOVi8 1, 14 /* CC::al */, $noreg
-  ; CHECK:   renamable $r12 = nuw nsw t2ADDrs killed renamable $r3, killed renamable $r12, 19, 14 /* CC::al */, $noreg, $noreg
-  ; CHECK:   dead $lr = t2DLS renamable $r12
-  ; CHECK:   $r3 = tMOVr killed $r12, 14 /* CC::al */, $noreg
+  ; CHECK:   $lr = MVE_DLSTP_16 killed renamable $r2
   ; CHECK: bb.1.vector.body:
   ; CHECK:   successors: %bb.1(0x7c000000), %bb.2(0x04000000)
-  ; CHECK:   liveins: $r0, $r1, $r2, $r3
-  ; CHECK:   renamable $vpr = MVE_VCTP16 renamable $r2, 0, $noreg
-  ; CHECK:   MVE_VPST 8, implicit $vpr
-  ; CHECK:   renamable $r0, renamable $q0 = MVE_VLDRHU16_post killed renamable $r0, 16, 1, killed renamable $vpr :: (load 16 from %ir.lsr.iv17, align 2)
+  ; CHECK:   liveins: $lr, $r0, $r1
+  ; CHECK:   renamable $r0, renamable $q0 = MVE_VLDRHU16_post killed renamable $r0, 16, 0, killed $noreg :: (load 16 from %ir.lsr.iv17, align 2)
   ; CHECK:   renamable $r12 = MVE_VADDVs16no_acc killed renamable $q0, 0, $noreg
-  ; CHECK:   $lr = tMOVr $r3, 14 /* CC::al */, $noreg
   ; CHECK:   early-clobber renamable $r1 = t2STR_POST killed renamable $r12, killed renamable $r1, 4, 14 /* CC::al */, $noreg :: (store 4 into %ir.store.addr)
-  ; CHECK:   renamable $r3, dead $cpsr = nsw tSUBi8 killed $r3, 1, 14 /* CC::al */, $noreg
-  ; CHECK:   renamable $r2, dead $cpsr = tSUBi8 killed renamable $r2, 8, 14 /* CC::al */, $noreg
-  ; CHECK:   dead $lr = t2LEUpdate killed renamable $lr, %bb.1
+  ; CHECK:   $lr = MVE_LETP killed renamable $lr, %bb.1
   ; CHECK: bb.2.exit:
   ; CHECK:   tPOP_RET 14 /* CC::al */, $noreg, def $r7, def $pc
   bb.0.entry:
@@ -1016,25 +1035,14 @@ body:             |
   ; CHECK:   tCMPi8 renamable $r2, 0, 14 /* CC::al */, $noreg, implicit-def $cpsr
   ; CHECK:   t2IT 0, 8, implicit-def $itstate
   ; CHECK:   tPOP_RET 0 /* CC::eq */, killed $cpsr, def $r7, def $pc, implicit killed $itstate
-  ; CHECK:   renamable $r3, dead $cpsr = tADDi3 renamable $r2, 7, 14 /* CC::al */, $noreg
-  ; CHECK:   renamable $r3 = t2BICri killed renamable $r3, 7, 14 /* CC::al */, $noreg, $noreg
-  ; CHECK:   renamable $r12 = t2SUBri killed renamable $r3, 7, 14 /* CC::al */, $noreg, $noreg
-  ; CHECK:   renamable $r3, dead $cpsr = tMOVi8 1, 14 /* CC::al */, $noreg
-  ; CHECK:   renamable $r12 = nuw nsw t2ADDrs killed renamable $r3, killed renamable $r12, 27, 14 /* CC::al */, $noreg, $noreg
-  ; CHECK:   dead $lr = t2DLS renamable $r12
-  ; CHECK:   $r3 = tMOVr killed $r12, 14 /* CC::al */, $noreg
+  ; CHECK:   $lr = MVE_DLSTP_8 killed renamable $r2
   ; CHECK: bb.1.vector.body:
   ; CHECK:   successors: %bb.1(0x7c000000), %bb.2(0x04000000)
-  ; CHECK:   liveins: $r0, $r1, $r2, $r3
-  ; CHECK:   renamable $vpr = MVE_VCTP8 renamable $r2, 0, $noreg
-  ; CHECK:   MVE_VPST 8, implicit $vpr
-  ; CHECK:   renamable $r0, renamable $q0 = MVE_VLDRBU8_post killed renamable $r0, 16, 1, killed renamable $vpr :: (load 16 from %ir.lsr.iv17, align 1)
+  ; CHECK:   liveins: $lr, $r0, $r1
+  ; CHECK:   renamable $r0, renamable $q0 = MVE_VLDRBU8_post killed renamable $r0, 16, 0, killed $noreg :: (load 16 from %ir.lsr.iv17, align 1)
   ; CHECK:   renamable $r12 = MVE_VADDVs8no_acc killed renamable $q0, 0, $noreg
-  ; CHECK:   $lr = tMOVr $r3, 14 /* CC::al */, $noreg
   ; CHECK:   early-clobber renamable $r1 = t2STR_POST killed renamable $r12, killed renamable $r1, 4, 14 /* CC::al */, $noreg :: (store 4 into %ir.store.addr)
-  ; CHECK:   renamable $r3, dead $cpsr = nsw tSUBi8 killed $r3, 1, 14 /* CC::al */, $noreg
-  ; CHECK:   renamable $r2, dead $cpsr = tSUBi8 killed renamable $r2, 16, 14 /* CC::al */, $noreg
-  ; CHECK:   dead $lr = t2LEUpdate killed renamable $lr, %bb.1
+  ; CHECK:   $lr = MVE_LETP killed renamable $lr, %bb.1
   ; CHECK: bb.2.exit:
   ; CHECK:   tPOP_RET 14 /* CC::al */, $noreg, def $r7, def $pc
   bb.0.entry:
@@ -1118,25 +1126,14 @@ body:             |
   ; CHECK: bb.1.vector.ph:
   ; CHECK:   successors: %bb.2(0x80000000)
   ; CHECK:   liveins: $r0, $r1
-  ; CHECK:   renamable $r2, dead $cpsr = tADDi3 renamable $r1, 3, 14 /* CC::al */, $noreg
-  ; CHECK:   renamable $r3, dead $cpsr = tMOVi8 1, 14 /* CC::al */, $noreg
-  ; CHECK:   renamable $r2 = t2BICri killed renamable $r2, 3, 14 /* CC::al */, $noreg, $noreg
-  ; CHECK:   renamable $r2, dead $cpsr = tSUBi8 killed renamable $r2, 4, 14 /* CC::al */, $noreg
-  ; CHECK:   renamable $r2 = nuw nsw t2ADDrs killed renamable $r3, killed renamable $r2, 19, 14 /* CC::al */, $noreg, $noreg
-  ; CHECK:   dead $lr = t2DLS renamable $r2
-  ; CHECK:   $r3 = tMOVr killed $r2, 14 /* CC::al */, $noreg
+  ; CHECK:   $lr = MVE_DLSTP_32 killed renamable $r1
   ; CHECK:   renamable $r2, dead $cpsr = tMOVi8 0, 14 /* CC::al */, $noreg
   ; CHECK: bb.2.vector.body:
   ; CHECK:   successors: %bb.2(0x7c000000), %bb.3(0x04000000)
-  ; CHECK:   liveins: $r0, $r1, $r2, $r3
-  ; CHECK:   renamable $vpr = MVE_VCTP32 renamable $r1, 0, $noreg
-  ; CHECK:   $lr = tMOVr $r3, 14 /* CC::al */, $noreg
-  ; CHECK:   renamable $r3, dead $cpsr = nsw tSUBi8 killed $r3, 1, 14 /* CC::al */, $noreg
-  ; CHECK:   renamable $r1, dead $cpsr = tSUBi8 killed renamable $r1, 4, 14 /* CC::al */, $noreg
-  ; CHECK:   MVE_VPST 8, implicit $vpr
-  ; CHECK:   renamable $r0, renamable $q0 = MVE_VLDRHS32_post killed renamable $r0, 8, 1, killed renamable $vpr :: (load 8 from %ir.lsr.iv17, align 2)
+  ; CHECK:   liveins: $lr, $r0, $r2
+  ; CHECK:   renamable $r0, renamable $q0 = MVE_VLDRHS32_post killed renamable $r0, 8, 0, killed $noreg :: (load 8 from %ir.lsr.iv17, align 2)
   ; CHECK:   renamable $r2 = MVE_VADDVu32acc killed renamable $r2, killed renamable $q0, 0, $noreg
-  ; CHECK:   dead $lr = t2LEUpdate killed renamable $lr, %bb.2
+  ; CHECK:   $lr = MVE_LETP killed renamable $lr, %bb.2
   ; CHECK: bb.3.exit:
   ; CHECK:   liveins: $r2
   ; CHECK:   $r0 = tMOVr killed $r2, 14 /* CC::al */, $noreg
@@ -2664,32 +2661,21 @@ body:             |
   ; CHECK:   frame-setup CFI_INSTRUCTION def_cfa_offset 8
   ; CHECK:   frame-setup CFI_INSTRUCTION offset $lr, -4
   ; CHECK:   frame-setup CFI_INSTRUCTION offset $r7, -8
-  ; CHECK:   tCMPi8 renamable $r2, 4, 14 /* CC::al */, $noreg, implicit-def $cpsr
-  ; CHECK:   $r3 = tMOVr $r2, 14 /* CC::al */, $noreg
-  ; CHECK:   t2IT 10, 8, implicit-def $itstate
-  ; CHECK:   renamable $r3 = tMOVi8 $noreg, 4, 10 /* CC::ge */, killed $cpsr, implicit killed renamable $r3, implicit killed $itstate
   ; CHECK:   tCMPi8 renamable $r2, 1, 14 /* CC::al */, $noreg, implicit-def $cpsr
   ; CHECK:   tBcc %bb.4, 11 /* CC::lt */, killed $cpsr
   ; CHECK: bb.1.while.body.preheader:
   ; CHECK:   successors: %bb.2(0x80000000)
-  ; CHECK:   liveins: $r0, $r1, $r2, $r3
-  ; CHECK:   renamable $r3, dead $cpsr = tSUBrr renamable $r2, killed renamable $r3, 14 /* CC::al */, $noreg
-  ; CHECK:   renamable $r12 = t2ADDri killed renamable $r3, 3, 14 /* CC::al */, $noreg, $noreg
-  ; CHECK:   renamable $r3, dead $cpsr = tMOVi8 1, 14 /* CC::al */, $noreg
-  ; CHECK:   renamable $lr = nuw nsw t2ADDrs killed renamable $r3, killed renamable $r12, 19, 14 /* CC::al */, $noreg, $noreg
+  ; CHECK:   liveins: $r0, $r1, $r2
   ; CHECK:   renamable $r12 = t2MOVi 0, 14 /* CC::al */, $noreg, $noreg
-  ; CHECK:   $lr = t2DLS killed renamable $lr
+  ; CHECK:   $lr = MVE_DLSTP_32 killed renamable $r2
   ; CHECK: bb.2.while.body:
   ; CHECK:   successors: %bb.2(0x7c000000), %bb.3(0x04000000)
-  ; CHECK:   liveins: $lr, $r0, $r1, $r2, $r12
-  ; CHECK:   renamable $vpr = MVE_VCTP32 renamable $r2, 0, $noreg
-  ; CHECK:   MVE_VPST 4, implicit $vpr
-  ; CHECK:   renamable $r1, renamable $q0 = MVE_VLDRHU32_post killed renamable $r1, 8, 1, renamable $vpr :: (load 8 from %ir.tmp3, align 2)
-  ; CHECK:   renamable $r0, renamable $q1 = MVE_VLDRHU32_post killed renamable $r0, 8, 1, killed renamable $vpr :: (load 8 from %ir.tmp1, align 2)
+  ; CHECK:   liveins: $lr, $r0, $r1, $r12
+  ; CHECK:   renamable $r1, renamable $q0 = MVE_VLDRHU32_post killed renamable $r1, 8, 0, $noreg :: (load 8 from %ir.tmp3, align 2)
+  ; CHECK:   renamable $r0, renamable $q1 = MVE_VLDRHU32_post killed renamable $r0, 8, 0, killed $noreg :: (load 8 from %ir.tmp1, align 2)
   ; CHECK:   renamable $q0 = MVE_VORR killed renamable $q1, killed renamable $q0, 0, $noreg, undef renamable $q0
-  ; CHECK:   renamable $r2, dead $cpsr = nsw tSUBi8 killed renamable $r2, 4, 14 /* CC::al */, $noreg
   ; CHECK:   renamable $r12 = MVE_VADDVu32acc killed renamable $r12, killed renamable $q0, 0, $noreg
-  ; CHECK:   $lr = t2LEUpdate killed renamable $lr, %bb.2
+  ; CHECK:   $lr = MVE_LETP killed renamable $lr, %bb.2
   ; CHECK: bb.3.while.end:
   ; CHECK:   liveins: $r12
   ; CHECK:   $r0 = tMOVr killed $r12, 14 /* CC::al */, $noreg
@@ -2790,33 +2776,22 @@ body:             |
   ; CHECK:   frame-setup CFI_INSTRUCTION def_cfa_offset 8
   ; CHECK:   frame-setup CFI_INSTRUCTION offset $lr, -4
   ; CHECK:   frame-setup CFI_INSTRUCTION offset $r7, -8
-  ; CHECK:   tCMPi8 renamable $r2, 8, 14 /* CC::al */, $noreg, implicit-def $cpsr
-  ; CHECK:   $r3 = tMOVr $r2, 14 /* CC::al */, $noreg
-  ; CHECK:   t2IT 10, 8, implicit-def $itstate
-  ; CHECK:   renamable $r3 = tMOVi8 $noreg, 8, 10 /* CC::ge */, killed $cpsr, implicit killed renamable $r3, implicit killed $itstate
   ; CHECK:   tCMPi8 renamable $r2, 1, 14 /* CC::al */, $noreg, implicit-def $cpsr
   ; CHECK:   tBcc %bb.4, 11 /* CC::lt */, killed $cpsr
   ; CHECK: bb.1.while.body.preheader:
   ; CHECK:   successors: %bb.2(0x80000000)
-  ; CHECK:   liveins: $r0, $r1, $r2, $r3
-  ; CHECK:   renamable $r3, dead $cpsr = tSUBrr renamable $r2, killed renamable $r3, 14 /* CC::al */, $noreg
-  ; CHECK:   renamable $r12 = t2ADDri killed renamable $r3, 7, 14 /* CC::al */, $noreg, $noreg
-  ; CHECK:   renamable $r3, dead $cpsr = tMOVi8 1, 14 /* CC::al */, $noreg
-  ; CHECK:   renamable $lr = nuw nsw t2ADDrs killed renamable $r3, killed renamable $r12, 27, 14 /* CC::al */, $noreg, $noreg
+  ; CHECK:   liveins: $r0, $r1, $r2
   ; CHECK:   renamable $r3, dead $cpsr = tMOVi8 0, 14 /* CC::al */, $noreg
-  ; CHECK:   $lr = t2DLS killed renamable $lr
+  ; CHECK:   $lr = MVE_DLSTP_16 killed renamable $r2
   ; CHECK: bb.2.while.body:
   ; CHECK:   successors: %bb.2(0x7c000000), %bb.3(0x04000000)
-  ; CHECK:   liveins: $lr, $r0, $r1, $r2, $r3
-  ; CHECK:   renamable $vpr = MVE_VCTP16 renamable $r2, 0, $noreg
-  ; CHECK:   MVE_VPST 4, implicit $vpr
-  ; CHECK:   renamable $r1, renamable $q0 = MVE_VLDRHU16_post killed renamable $r1, 16, 1, renamable $vpr :: (load 16 from %ir.tmp3, align 2)
-  ; CHECK:   renamable $r0, renamable $q1 = MVE_VLDRHU16_post killed renamable $r0, 16, 1, killed renamable $vpr :: (load 16 from %ir.tmp1, align 2)
+  ; CHECK:   liveins: $lr, $r0, $r1, $r3
+  ; CHECK:   renamable $r1, renamable $q0 = MVE_VLDRHU16_post killed renamable $r1, 16, 0, $noreg :: (load 16 from %ir.tmp3, align 2)
+  ; CHECK:   renamable $r0, renamable $q1 = MVE_VLDRHU16_post killed renamable $r0, 16, 0, killed $noreg :: (load 16 from %ir.tmp1, align 2)
   ; CHECK:   renamable $q0 = MVE_VORR killed renamable $q1, killed renamable $q0, 0, $noreg, undef renamable $q0
-  ; CHECK:   renamable $r2, dead $cpsr = nsw tSUBi8 killed renamable $r2, 8, 14 /* CC::al */, $noreg
   ; CHECK:   renamable $r12 = MVE_VADDVu16no_acc killed renamable $q0, 0, $noreg
   ; CHECK:   renamable $r3 = t2UXTAH killed renamable $r3, killed renamable $r12, 0, 14 /* CC::al */, $noreg
-  ; CHECK:   $lr = t2LEUpdate killed renamable $lr, %bb.2
+  ; CHECK:   $lr = MVE_LETP killed renamable $lr, %bb.2
   ; CHECK: bb.3.while.end:
   ; CHECK:   liveins: $r3
   ; CHECK:   $r0 = tMOVr killed $r3, 14 /* CC::al */, $noreg
@@ -3009,3 +2984,129 @@ body:             |
     tPOP_RET 14 /* CC::al */, $noreg, def $r7, def $pc, implicit killed $r0
 
 ...
+---
+name:            illegal_vmull_non_zero
+alignment:       2
+tracksRegLiveness: true
+registers:       []
+liveins:
+  - { reg: '$r0', virtual-reg: '' }
+  - { reg: '$r1', virtual-reg: '' }
+  - { reg: '$r3', virtual-reg: '' }
+frameInfo:
+  stackSize:       8
+  offsetAdjustment: 0
+  maxAlignment:    4
+fixedStack:      []
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '$lr', callee-saved-restored: false,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 1, name: '', type: spill-slot, offset: -8, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '$r7', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+callSites:       []
+constants:       []
+machineFunctionInfo: {}
+body:             |
+  ; CHECK-LABEL: name: illegal_vmull_non_zero
+  ; CHECK: bb.0.entry:
+  ; CHECK:   successors: %bb.1(0x50000000), %bb.4(0x30000000)
+  ; CHECK:   liveins: $lr, $r0, $r1, $r3, $r7
+  ; CHECK:   frame-setup tPUSH 14 /* CC::al */, $noreg, killed $r7, killed $lr, implicit-def $sp, implicit $sp
+  ; CHECK:   frame-setup CFI_INSTRUCTION def_cfa_offset 8
+  ; CHECK:   frame-setup CFI_INSTRUCTION offset $lr, -4
+  ; CHECK:   frame-setup CFI_INSTRUCTION offset $r7, -8
+  ; CHECK:   tCMPi8 renamable $r3, 8, 14 /* CC::al */, $noreg, implicit-def $cpsr
+  ; CHECK:   $r2 = tMOVr $r3, 14 /* CC::al */, $noreg
+  ; CHECK:   t2IT 10, 8, implicit-def $itstate
+  ; CHECK:   renamable $r2 = tMOVi8 $noreg, 8, 10 /* CC::ge */, killed $cpsr, implicit killed renamable $r2, implicit killed $itstate
+  ; CHECK:   tCMPi8 renamable $r3, 1, 14 /* CC::al */, $noreg, implicit-def $cpsr
+  ; CHECK:   tBcc %bb.4, 11 /* CC::lt */, killed $cpsr
+  ; CHECK: bb.1.while.body.preheader:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $r0, $r1, $r2, $r3
+  ; CHECK:   renamable $r2, dead $cpsr = tSUBrr renamable $r3, killed renamable $r2, 14 /* CC::al */, $noreg
+  ; CHECK:   renamable $r12 = t2ADDri killed renamable $r2, 7, 14 /* CC::al */, $noreg, $noreg
+  ; CHECK:   renamable $r2, dead $cpsr = tMOVi8 1, 14 /* CC::al */, $noreg
+  ; CHECK:   renamable $r2 = nuw nsw t2ADDrs killed renamable $r2, killed renamable $r12, 27, 14 /* CC::al */, $noreg, $noreg
+  ; CHECK:   dead $lr = t2DLS renamable $r2
+  ; CHECK:   $r12 = tMOVr killed $r2, 14 /* CC::al */, $noreg
+  ; CHECK:   renamable $r2, dead $cpsr = tMOVi8 0, 14 /* CC::al */, $noreg
+  ; CHECK: bb.2.while.body:
+  ; CHECK:   successors: %bb.2(0x7c000000), %bb.3(0x04000000)
+  ; CHECK:   liveins: $r0, $r1, $r2, $r3, $r12
+  ; CHECK:   renamable $vpr = MVE_VCTP16 renamable $r3, 0, $noreg
+  ; CHECK:   MVE_VPST 4, implicit $vpr
+  ; CHECK:   renamable $r1, renamable $q0 = MVE_VLDRHU16_post killed renamable $r1, 16, 1, renamable $vpr :: (load 16 from %ir.tmp3, align 2)
+  ; CHECK:   renamable $r0, renamable $q1 = MVE_VLDRHU16_post killed renamable $r0, 16, 1, killed renamable $vpr :: (load 16 from %ir.tmp1, align 2)
+  ; CHECK:   $lr = tMOVr $r12, 14 /* CC::al */, $noreg
+  ; CHECK:   renamable $q0 = MVE_VMULLTs16 killed renamable $q1, killed renamable $q0, 0, $noreg, undef renamable $q0
+  ; CHECK:   renamable $r12 = nsw t2SUBri killed $r12, 1, 14 /* CC::al */, $noreg, $noreg
+  ; CHECK:   renamable $r3, dead $cpsr = nsw tSUBi8 killed renamable $r3, 8, 14 /* CC::al */, $noreg
+  ; CHECK:   renamable $r2 = MVE_VADDVu32acc killed renamable $r2, killed renamable $q0, 0, $noreg
+  ; CHECK:   dead $lr = t2LEUpdate killed renamable $lr, %bb.2
+  ; CHECK: bb.3.while.end:
+  ; CHECK:   liveins: $r2
+  ; CHECK:   $r0 = tMOVr killed $r2, 14 /* CC::al */, $noreg
+  ; CHECK:   frame-destroy tPOP_RET 14 /* CC::al */, $noreg, def $r7, def $pc, implicit killed $r0
+  ; CHECK: bb.4:
+  ; CHECK:   renamable $r2, dead $cpsr = tMOVi8 0, 14 /* CC::al */, $noreg
+  ; CHECK:   $r0 = tMOVr killed $r2, 14 /* CC::al */, $noreg
+  ; CHECK:   frame-destroy tPOP_RET 14 /* CC::al */, $noreg, def $r7, def $pc, implicit killed $r0
+  bb.0.entry:
+    successors: %bb.1(0x50000000), %bb.4(0x30000000)
+    liveins: $r0, $r1, $r3, $r7, $lr
+
+    frame-setup tPUSH 14 /* CC::al */, $noreg, killed $r7, killed $lr, implicit-def $sp, implicit $sp
+    frame-setup CFI_INSTRUCTION def_cfa_offset 8
+    frame-setup CFI_INSTRUCTION offset $lr, -4
+    frame-setup CFI_INSTRUCTION offset $r7, -8
+    tCMPi8 renamable $r3, 8, 14 /* CC::al */, $noreg, implicit-def $cpsr
+    $r2 = tMOVr $r3, 14 /* CC::al */, $noreg
+    t2IT 10, 8, implicit-def $itstate
+    renamable $r2 = tMOVi8 $noreg, 8, 10 /* CC::ge */, killed $cpsr, implicit killed renamable $r2, implicit killed $itstate
+    tCMPi8 renamable $r3, 1, 14 /* CC::al */, $noreg, implicit-def $cpsr
+    tBcc %bb.4, 11 /* CC::lt */, killed $cpsr
+
+  bb.1.while.body.preheader:
+    successors: %bb.2(0x80000000)
+    liveins: $r0, $r1, $r2, $r3
+
+    renamable $r2, dead $cpsr = tSUBrr renamable $r3, killed renamable $r2, 14 /* CC::al */, $noreg
+    renamable $r12 = t2ADDri killed renamable $r2, 7, 14 /* CC::al */, $noreg, $noreg
+    renamable $r2, dead $cpsr = tMOVi8 1, 14 /* CC::al */, $noreg
+    renamable $r2 = nuw nsw t2ADDrs killed renamable $r2, killed renamable $r12, 27, 14 /* CC::al */, $noreg, $noreg
+    t2DoLoopStart renamable $r2
+    $r12 = tMOVr killed $r2, 14 /* CC::al */, $noreg
+    renamable $r2, dead $cpsr = tMOVi8 0, 14 /* CC::al */, $noreg
+
+  bb.2.while.body:
+    successors: %bb.2(0x7c000000), %bb.3(0x04000000)
+    liveins: $r0, $r1, $r2, $r3, $r12
+
+    renamable $vpr = MVE_VCTP16 renamable $r3, 0, $noreg
+    MVE_VPST 4, implicit $vpr
+    renamable $r1, renamable $q0 = MVE_VLDRHU16_post killed renamable $r1, 16, 1, renamable $vpr :: (load 16 from %ir.tmp3, align 2)
+    renamable $r0, renamable $q1 = MVE_VLDRHU16_post killed renamable $r0, 16, 1, killed renamable $vpr :: (load 16 from %ir.tmp1, align 2)
+    $lr = tMOVr $r12, 14 /* CC::al */, $noreg
+    renamable $q0 = MVE_VMULLTs16 killed renamable $q1, killed renamable $q0, 0, $noreg, undef renamable $q0
+    renamable $r12 = nsw t2SUBri killed $r12, 1, 14 /* CC::al */, $noreg, $noreg
+    renamable $r3, dead $cpsr = nsw tSUBi8 killed renamable $r3, 8, 14 /* CC::al */, $noreg
+    renamable $r2 = MVE_VADDVu32acc killed renamable $r2, killed renamable $q0, 0, $noreg
+    renamable $lr = t2LoopDec killed renamable $lr, 1
+    t2LoopEnd killed renamable $lr, %bb.2, implicit-def dead $cpsr
+    tB %bb.3, 14 /* CC::al */, $noreg
+
+  bb.3.while.end:
+    liveins: $r2
+
+    $r0 = tMOVr killed $r2, 14 /* CC::al */, $noreg
+    frame-destroy tPOP_RET 14 /* CC::al */, $noreg, def $r7, def $pc, implicit killed $r0
+
+  bb.4:
+    renamable $r2, dead $cpsr = tMOVi8 0, 14 /* CC::al */, $noreg
+    $r0 = tMOVr killed $r2, 14 /* CC::al */, $noreg
+    frame-destroy tPOP_RET 14 /* CC::al */, $noreg, def $r7, def $pc, implicit killed $r0
+
+...
diff --git a/llvm/test/CodeGen/Thumb2/cde-vec.ll b/llvm/test/CodeGen/Thumb2/cde-vec.ll
index 5dfd11180e7f6..3c0c0473bb8f8 100644
--- a/llvm/test/CodeGen/Thumb2/cde-vec.ll
+++ b/llvm/test/CodeGen/Thumb2/cde-vec.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=thumbv8.1m.main -mattr=+cdecp0 -mattr=+cdecp1 -mattr=+mve -verify-machineinstrs -o - %s | FileCheck %s
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+cdecp0 -mattr=+cdecp1 -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
 
 declare <16 x i8> @llvm.arm.cde.vcx1q(i32 immarg, i32 immarg)
 declare <16 x i8> @llvm.arm.cde.vcx1qa(i32 immarg, <16 x i8>, i32 immarg)
@@ -112,3 +112,103 @@ entry:
   %2 = call <16 x i8> @llvm.arm.cde.vcx3qa(i32 1, <16 x i8> %acc, <16 x i8> %0, <16 x i8> %1, i32 13)
   ret <16 x i8> %2
 }
+
+declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32)
+declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32)
+declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32)
+declare <8 x i16> @llvm.arm.cde.vcx1q.predicated.v8i16.v8i1(i32 immarg, <8 x i16>, i32 immarg, <8 x i1>)
+declare <16 x i8> @llvm.arm.cde.vcx1qa.predicated.v16i8.v16i1(i32 immarg, <16 x i8>, i32 immarg, <16 x i1>)
+declare <4 x i32> @llvm.arm.cde.vcx2q.predicated.v4i32.v4i1(i32 immarg, <4 x i32>, <16 x i8>, i32 immarg, <4 x i1>)
+declare <4 x float> @llvm.arm.cde.vcx2qa.predicated.v4f32.v4i1(i32 immarg, <4 x float>, <16 x i8>, i32 immarg, <4 x i1>)
+declare <2 x i64> @llvm.arm.cde.vcx3q.predicated.v2i64.v4i1(i32 immarg, <2 x i64>, <16 x i8>, <16 x i8>, i32 immarg, <4 x i1>)
+declare <4 x float> @llvm.arm.cde.vcx3qa.predicated.v4f32.v4i1(i32 immarg, <4 x float>, <16 x i8>, <16 x i8>, i32 immarg, <4 x i1>)
+
+define arm_aapcs_vfpcc <8 x i16> @test_vcx1q_m(<8 x i16> %inactive, i16 zeroext %p) {
+; CHECK-LABEL: test_vcx1q_m:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcx1t p0, q0, #1111
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
+  %2 = call <8 x i16> @llvm.arm.cde.vcx1q.predicated.v8i16.v8i1(i32 0, <8 x i16> %inactive, i32 1111, <8 x i1> %1)
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @test_vcx1qa_m(<16 x i8> %acc, i16 zeroext %p) {
+; CHECK-LABEL: test_vcx1qa_m:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcx1at p1, q0, #1112
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext i16 %p to i32
+  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
+  %2 = call <16 x i8> @llvm.arm.cde.vcx1qa.predicated.v16i8.v16i1(i32 1, <16 x i8> %acc, i32 1112, <16 x i1> %1)
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @test_vcx2q_m(<4 x i32> %inactive, <4 x float> %n, i16 zeroext %p) {
+; CHECK-LABEL: test_vcx2q_m:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcx2t p0, q0, q1, #111
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast <4 x float> %n to <16 x i8>
+  %1 = zext i16 %p to i32
+  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  %3 = call <4 x i32> @llvm.arm.cde.vcx2q.predicated.v4i32.v4i1(i32 0, <4 x i32> %inactive, <16 x i8> %0, i32 111, <4 x i1> %2)
+  ret <4 x i32> %3
+}
+
+define arm_aapcs_vfpcc <4 x float> @test_vcx2qa_m(<4 x float> %acc, <8 x half> %n, i16 zeroext %p) {
+; CHECK-LABEL: test_vcx2qa_m:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcx2at p0, q0, q1, #112
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast <8 x half> %n to <16 x i8>
+  %1 = zext i16 %p to i32
+  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  %3 = call <4 x float> @llvm.arm.cde.vcx2qa.predicated.v4f32.v4i1(i32 0, <4 x float> %acc, <16 x i8> %0, i32 112, <4 x i1> %2)
+  ret <4 x float> %3
+}
+
+define arm_aapcs_vfpcc <2 x i64> @test_vcx3q_m(<2 x i64> %inactive, <4 x float> %n, <16 x i8> %m, i16 zeroext %p) {
+; CHECK-LABEL: test_vcx3q_m:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcx3t p0, q0, q1, q2, #11
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast <4 x float> %n to <16 x i8>
+  %1 = zext i16 %p to i32
+  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
+  %3 = call <2 x i64> @llvm.arm.cde.vcx3q.predicated.v2i64.v4i1(i32 0, <2 x i64> %inactive, <16 x i8> %0, <16 x i8> %m, i32 11, <4 x i1> %2)
+  ret <2 x i64> %3
+}
+
+define arm_aapcs_vfpcc <8 x half> @test_vcx3qa_m(<4 x float> %inactive, <8 x half> %n, <4 x i32> %m, i16 zeroext %p) {
+; CHECK-LABEL: test_vcx3qa_m:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcx3at p0, q0, q1, q2, #12
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = bitcast <8 x half> %n to <16 x i8>
+  %1 = bitcast <4 x i32> %m to <16 x i8>
+  %2 = zext i16 %p to i32
+  %3 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %2)
+  %4 = call <4 x float> @llvm.arm.cde.vcx3qa.predicated.v4f32.v4i1(i32 0, <4 x float> %inactive, <16 x i8> %0, <16 x i8> %1, i32 12, <4 x i1> %3)
+  %5 = bitcast <4 x float> %4 to <8 x half>
+  ret <8 x half> %5
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-float16regloops.ll b/llvm/test/CodeGen/Thumb2/mve-float16regloops.ll
index 8161b1326b6a9..38083b7edcc2d 100644
--- a/llvm/test/CodeGen/Thumb2/mve-float16regloops.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-float16regloops.ll
@@ -9,13 +9,12 @@ define arm_aapcs_vfpcc void @test_fadd(half* noalias nocapture readonly %A, half
 ; CHECK-NEXT:    bxlt lr
 ; CHECK-NEXT:    vldr.16 s0, [r1]
 ; CHECK-NEXT:    vmov.f16 r1, s0
-; CHECK-NEXT:    vdup.16 q0, r1
 ; CHECK-NEXT:  .LBB0_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
 ; CHECK-NEXT:    subs r3, #8
-; CHECK-NEXT:    vadd.f16 q1, q1, q0
-; CHECK-NEXT:    vstrb.8 q1, [r2], #16
+; CHECK-NEXT:    vadd.f16 q0, q0, r1
+; CHECK-NEXT:    vstrb.8 q0, [r2], #16
 ; CHECK-NEXT:    bne .LBB0_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    bx lr
@@ -57,13 +56,12 @@ define arm_aapcs_vfpcc void @test_fadd_r(half* noalias nocapture readonly %A, ha
 ; CHECK-NEXT:    bxlt lr
 ; CHECK-NEXT:    vldr.16 s0, [r1]
 ; CHECK-NEXT:    vmov.f16 r1, s0
-; CHECK-NEXT:    vdup.16 q0, r1
 ; CHECK-NEXT:  .LBB1_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
 ; CHECK-NEXT:    subs r3, #8
-; CHECK-NEXT:    vadd.f16 q1, q0, q1
-; CHECK-NEXT:    vstrb.8 q1, [r2], #16
+; CHECK-NEXT:    vadd.f16 q0, q0, r1
+; CHECK-NEXT:    vstrb.8 q0, [r2], #16
 ; CHECK-NEXT:    bne .LBB1_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    bx lr
@@ -105,13 +103,12 @@ define arm_aapcs_vfpcc void @test_fmul(half* noalias nocapture readonly %A, half
 ; CHECK-NEXT:    bxlt lr
 ; CHECK-NEXT:    vldr.16 s0, [r1]
 ; CHECK-NEXT:    vmov.f16 r1, s0
-; CHECK-NEXT:    vdup.16 q0, r1
 ; CHECK-NEXT:  .LBB2_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
 ; CHECK-NEXT:    subs r3, #8
-; CHECK-NEXT:    vmul.f16 q1, q1, q0
-; CHECK-NEXT:    vstrb.8 q1, [r2], #16
+; CHECK-NEXT:    vmul.f16 q0, q0, r1
+; CHECK-NEXT:    vstrb.8 q0, [r2], #16
 ; CHECK-NEXT:    bne .LBB2_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    bx lr
@@ -153,13 +150,12 @@ define arm_aapcs_vfpcc void @test_fmul_r(half* noalias nocapture readonly %A, ha
 ; CHECK-NEXT:    bxlt lr
 ; CHECK-NEXT:    vldr.16 s0, [r1]
 ; CHECK-NEXT:    vmov.f16 r1, s0
-; CHECK-NEXT:    vdup.16 q0, r1
 ; CHECK-NEXT:  .LBB3_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
 ; CHECK-NEXT:    subs r3, #8
-; CHECK-NEXT:    vmul.f16 q1, q0, q1
-; CHECK-NEXT:    vstrb.8 q1, [r2], #16
+; CHECK-NEXT:    vmul.f16 q0, q0, r1
+; CHECK-NEXT:    vstrb.8 q0, [r2], #16
 ; CHECK-NEXT:    bne .LBB3_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    bx lr
@@ -201,13 +197,12 @@ define arm_aapcs_vfpcc void @test_fsub(half* noalias nocapture readonly %A, half
 ; CHECK-NEXT:    bxlt lr
 ; CHECK-NEXT:    vldr.16 s0, [r1]
 ; CHECK-NEXT:    vmov.f16 r1, s0
-; CHECK-NEXT:    vdup.16 q0, r1
 ; CHECK-NEXT:  .LBB4_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
 ; CHECK-NEXT:    subs r3, #8
-; CHECK-NEXT:    vsub.f16 q1, q1, q0
-; CHECK-NEXT:    vstrb.8 q1, [r2], #16
+; CHECK-NEXT:    vsub.f16 q0, q0, r1
+; CHECK-NEXT:    vstrb.8 q0, [r2], #16
 ; CHECK-NEXT:    bne .LBB4_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    bx lr
@@ -299,15 +294,13 @@ define arm_aapcs_vfpcc void @test_fmas(half* noalias nocapture readonly %A, half
 ; CHECK-NEXT:    bxlt lr
 ; CHECK-NEXT:    vldr.16 s0, [r2]
 ; CHECK-NEXT:    vmov.f16 r2, s0
-; CHECK-NEXT:    vdup.16 q0, r2
 ; CHECK-NEXT:  .LBB6_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
-; CHECK-NEXT:    vldrw.u32 q2, [r1], #16
-; CHECK-NEXT:    vmov q3, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
 ; CHECK-NEXT:    subs.w r12, r12, #8
-; CHECK-NEXT:    vfma.f16 q3, q2, q1
-; CHECK-NEXT:    vstrb.8 q3, [r3], #16
+; CHECK-NEXT:    vfmas.f16 q1, q0, r2
+; CHECK-NEXT:    vstrb.8 q1, [r3], #16
 ; CHECK-NEXT:    bne .LBB6_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    bx lr
@@ -354,15 +347,13 @@ define arm_aapcs_vfpcc void @test_fmas_r(half* noalias nocapture readonly %A, ha
 ; CHECK-NEXT:    bxlt lr
 ; CHECK-NEXT:    vldr.16 s0, [r2]
 ; CHECK-NEXT:    vmov.f16 r2, s0
-; CHECK-NEXT:    vdup.16 q0, r2
 ; CHECK-NEXT:  .LBB7_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
-; CHECK-NEXT:    vldrw.u32 q2, [r1], #16
-; CHECK-NEXT:    vmov q3, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
 ; CHECK-NEXT:    subs.w r12, r12, #8
-; CHECK-NEXT:    vfma.f16 q3, q2, q1
-; CHECK-NEXT:    vstrb.8 q3, [r3], #16
+; CHECK-NEXT:    vfmas.f16 q1, q0, r2
+; CHECK-NEXT:    vstrb.8 q1, [r3], #16
 ; CHECK-NEXT:    bne .LBB7_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    bx lr
@@ -409,14 +400,13 @@ define arm_aapcs_vfpcc void @test_fma(half* noalias nocapture readonly %A, half*
 ; CHECK-NEXT:    bxlt lr
 ; CHECK-NEXT:    vldr.16 s0, [r2]
 ; CHECK-NEXT:    vmov.f16 r2, s0
-; CHECK-NEXT:    vdup.16 q0, r2
 ; CHECK-NEXT:  .LBB8_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
-; CHECK-NEXT:    vldrw.u32 q2, [r1], #16
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
 ; CHECK-NEXT:    subs.w r12, r12, #8
-; CHECK-NEXT:    vfma.f16 q2, q1, q0
-; CHECK-NEXT:    vstrb.8 q2, [r3], #16
+; CHECK-NEXT:    vfma.f16 q1, q0, r2
+; CHECK-NEXT:    vstrb.8 q1, [r3], #16
 ; CHECK-NEXT:    bne .LBB8_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    bx lr
@@ -463,14 +453,13 @@ define arm_aapcs_vfpcc void @test_fma_r(half* noalias nocapture readonly %A, hal
 ; CHECK-NEXT:    bxlt lr
 ; CHECK-NEXT:    vldr.16 s0, [r2]
 ; CHECK-NEXT:    vmov.f16 r2, s0
-; CHECK-NEXT:    vdup.16 q0, r2
 ; CHECK-NEXT:  .LBB9_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
-; CHECK-NEXT:    vldrw.u32 q2, [r1], #16
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
 ; CHECK-NEXT:    subs.w r12, r12, #8
-; CHECK-NEXT:    vfma.f16 q2, q0, q1
-; CHECK-NEXT:    vstrb.8 q2, [r3], #16
+; CHECK-NEXT:    vfma.f16 q1, q0, r2
+; CHECK-NEXT:    vstrb.8 q1, [r3], #16
 ; CHECK-NEXT:    bne .LBB9_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    bx lr
@@ -629,15 +618,14 @@ define arm_aapcs_vfpcc void @test_fms(half* noalias nocapture readonly %A, half*
 ; CHECK-NEXT:    bxlt lr
 ; CHECK-NEXT:    vldr.16 s0, [r2]
 ; CHECK-NEXT:    vmov.f16 r2, s0
-; CHECK-NEXT:    vdup.16 q0, r2
 ; CHECK-NEXT:  .LBB12_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
-; CHECK-NEXT:    vldrw.u32 q2, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
+; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
 ; CHECK-NEXT:    subs.w r12, r12, #8
-; CHECK-NEXT:    vneg.f16 q1, q1
-; CHECK-NEXT:    vfma.f16 q1, q2, q0
-; CHECK-NEXT:    vstrb.8 q1, [r3], #16
+; CHECK-NEXT:    vneg.f16 q0, q0
+; CHECK-NEXT:    vfma.f16 q0, q1, r2
+; CHECK-NEXT:    vstrb.8 q0, [r3], #16
 ; CHECK-NEXT:    bne .LBB12_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    bx lr
@@ -684,15 +672,14 @@ define arm_aapcs_vfpcc void @test_fms_r(half* noalias nocapture readonly %A, hal
 ; CHECK-NEXT:    bxlt lr
 ; CHECK-NEXT:    vldr.16 s0, [r2]
 ; CHECK-NEXT:    vmov.f16 r2, s0
-; CHECK-NEXT:    vdup.16 q0, r2
 ; CHECK-NEXT:  .LBB13_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
-; CHECK-NEXT:    vldrw.u32 q2, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
+; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
 ; CHECK-NEXT:    subs.w r12, r12, #8
-; CHECK-NEXT:    vneg.f16 q1, q1
-; CHECK-NEXT:    vfma.f16 q1, q0, q2
-; CHECK-NEXT:    vstrb.8 q1, [r3], #16
+; CHECK-NEXT:    vneg.f16 q0, q0
+; CHECK-NEXT:    vfma.f16 q0, q1, r2
+; CHECK-NEXT:    vstrb.8 q0, [r3], #16
 ; CHECK-NEXT:    bne .LBB13_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    bx lr
@@ -820,105 +807,116 @@ for.end14:                                        ; preds = %for.cond6.for.end_c
 define void @arm_fir_f32_1_4_mve(%struct.arm_fir_instance_f32* nocapture readonly %S, half* nocapture readonly %pSrc, half* %pDst, i32 %blockSize) {
 ; CHECK-LABEL: arm_fir_f32_1_4_mve:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, lr}
-; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
-; CHECK-NEXT:    vpush {d8, d9, d10, d11}
-; CHECK-NEXT:    ldrh.w r10, [r0]
-; CHECK-NEXT:    ldr.w r12, [r0, #4]
-; CHECK-NEXT:    sub.w r7, r10, #1
+; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
+; CHECK-NEXT:    ldrh r4, [r0]
+; CHECK-NEXT:    ldr r5, [r0, #4]
+; CHECK-NEXT:    subs r7, r4, #1
 ; CHECK-NEXT:    cmp r7, #3
 ; CHECK-NEXT:    bhi .LBB15_6
 ; CHECK-NEXT:  @ %bb.1: @ %if.then
 ; CHECK-NEXT:    ldr r6, [r0, #8]
+; CHECK-NEXT:    add.w r11, r5, r7, lsl #1
 ; CHECK-NEXT:    lsr.w lr, r3, #2
-; CHECK-NEXT:    ldrh r4, [r6, #6]
-; CHECK-NEXT:    vdup.16 q0, r4
-; CHECK-NEXT:    ldrh r4, [r6, #4]
-; CHECK-NEXT:    vdup.16 q1, r4
-; CHECK-NEXT:    ldrh r4, [r6, #2]
-; CHECK-NEXT:    ldrh r6, [r6]
-; CHECK-NEXT:    vdup.16 q2, r4
-; CHECK-NEXT:    add.w r4, r12, r7, lsl #1
-; CHECK-NEXT:    vdup.16 q3, r6
+; CHECK-NEXT:    vldr.16 s0, [r6, #6]
+; CHECK-NEXT:    vldr.16 s2, [r6, #4]
+; CHECK-NEXT:    vldr.16 s4, [r6, #2]
+; CHECK-NEXT:    vldr.16 s6, [r6]
 ; CHECK-NEXT:    wls lr, lr, .LBB15_5
 ; CHECK-NEXT:  @ %bb.2: @ %while.body.lr.ph
-; CHECK-NEXT:    bic r9, r3, #3
+; CHECK-NEXT:    strd r3, r4, [sp, #8] @ 8-byte Folded Spill
+; CHECK-NEXT:    vmov.f16 r10, s6
+; CHECK-NEXT:    vmov.f16 r12, s4
+; CHECK-NEXT:    bic r3, r3, #3
+; CHECK-NEXT:    vmov.f16 r4, s2
+; CHECK-NEXT:    str r3, [sp] @ 4-byte Spill
+; CHECK-NEXT:    vmov.f16 r8, s0
+; CHECK-NEXT:    add.w r3, r2, r3, lsl #1
+; CHECK-NEXT:    str r3, [sp, #4] @ 4-byte Spill
 ; CHECK-NEXT:    movs r6, #0
-; CHECK-NEXT:    add.w r8, r2, r9, lsl #1
+; CHECK-NEXT:    mov r3, r5
 ; CHECK-NEXT:  .LBB15_3: @ %while.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    adds r7, r1, r6
-; CHECK-NEXT:    vldrw.u32 q4, [r7]
-; CHECK-NEXT:    adds r7, r4, r6
-; CHECK-NEXT:    vstrw.32 q4, [r7]
-; CHECK-NEXT:    add.w r7, r12, r6
-; CHECK-NEXT:    vldrw.u32 q4, [r7]
+; CHECK-NEXT:    add.w r9, r1, r6
+; CHECK-NEXT:    add.w r7, r11, r6
+; CHECK-NEXT:    vldrw.u32 q2, [r9]
+; CHECK-NEXT:    vstrw.32 q2, [r7]
+; CHECK-NEXT:    adds r7, r3, r6
+; CHECK-NEXT:    vldrw.u32 q2, [r7]
 ; CHECK-NEXT:    adds r5, r7, #2
-; CHECK-NEXT:    vldrw.u32 q5, [r5]
+; CHECK-NEXT:    vldrw.u32 q3, [r5]
 ; CHECK-NEXT:    adds r5, r7, #6
-; CHECK-NEXT:    vmul.f16 q4, q4, q3
-; CHECK-NEXT:    vfma.f16 q4, q5, q2
-; CHECK-NEXT:    vldrw.u32 q5, [r7, #4]
-; CHECK-NEXT:    vfma.f16 q4, q5, q1
-; CHECK-NEXT:    vldrw.u32 q5, [r5]
+; CHECK-NEXT:    vmul.f16 q2, q2, r10
+; CHECK-NEXT:    vfma.f16 q2, q3, r12
+; CHECK-NEXT:    vldrw.u32 q3, [r7, #4]
+; CHECK-NEXT:    vfma.f16 q2, q3, r4
+; CHECK-NEXT:    vldrw.u32 q3, [r5]
 ; CHECK-NEXT:    adds r5, r2, r6
 ; CHECK-NEXT:    adds r6, #8
-; CHECK-NEXT:    vfma.f16 q4, q5, q0
-; CHECK-NEXT:    vstrw.32 q4, [r5]
+; CHECK-NEXT:    vfma.f16 q2, q3, r8
+; CHECK-NEXT:    vstrw.32 q2, [r5]
 ; CHECK-NEXT:    le lr, .LBB15_3
 ; CHECK-NEXT:  @ %bb.4: @ %while.end.loopexit
-; CHECK-NEXT:    add r4, r6
-; CHECK-NEXT:    add.w r12, r12, r9, lsl #1
-; CHECK-NEXT:    add.w r1, r1, r9, lsl #1
-; CHECK-NEXT:    mov r2, r8
+; CHECK-NEXT:    ldr r2, [sp] @ 4-byte Reload
+; CHECK-NEXT:    add r4, sp, #4
+; CHECK-NEXT:    add r11, r6
+; CHECK-NEXT:    add.w r5, r3, r2, lsl #1
+; CHECK-NEXT:    add.w r1, r1, r2, lsl #1
+; CHECK-NEXT:    ldm r4, {r2, r3, r4} @ 12-byte Folded Reload
 ; CHECK-NEXT:  .LBB15_5: @ %while.end
 ; CHECK-NEXT:    and r7, r3, #3
-; CHECK-NEXT:    vldrw.u32 q4, [r1]
+; CHECK-NEXT:    vldrw.u32 q2, [r1]
 ; CHECK-NEXT:    vctp.16 r7
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vstrht.16 q4, [r4]
-; CHECK-NEXT:    vldrw.u32 q4, [r12]
-; CHECK-NEXT:    add.w r1, r12, #2
-; CHECK-NEXT:    vmul.f16 q3, q4, q3
-; CHECK-NEXT:    vldrw.u32 q4, [r1]
-; CHECK-NEXT:    add.w r1, r12, #6
-; CHECK-NEXT:    vfma.f16 q3, q4, q2
-; CHECK-NEXT:    vldrw.u32 q2, [r12, #4]
-; CHECK-NEXT:    vfma.f16 q3, q2, q1
-; CHECK-NEXT:    vldrw.u32 q1, [r1]
-; CHECK-NEXT:    vfma.f16 q3, q1, q0
+; CHECK-NEXT:    vstrht.16 q2, [r11]
+; CHECK-NEXT:    vldrw.u32 q2, [r5]
+; CHECK-NEXT:    vmov.f16 r1, s6
+; CHECK-NEXT:    adds r7, r5, #2
+; CHECK-NEXT:    vmul.f16 q2, q2, r1
+; CHECK-NEXT:    vmov.f16 r1, s4
+; CHECK-NEXT:    vldrw.u32 q1, [r7]
+; CHECK-NEXT:    adds r7, r5, #6
+; CHECK-NEXT:    vfma.f16 q2, q1, r1
+; CHECK-NEXT:    vldrw.u32 q1, [r5, #4]
+; CHECK-NEXT:    vmov.f16 r1, s2
+; CHECK-NEXT:    vfma.f16 q2, q1, r1
+; CHECK-NEXT:    vmov.f16 r1, s0
+; CHECK-NEXT:    vldrw.u32 q0, [r7]
+; CHECK-NEXT:    vfma.f16 q2, q0, r1
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vstrht.16 q3, [r2]
-; CHECK-NEXT:    ldr.w r12, [r0, #4]
+; CHECK-NEXT:    vstrht.16 q2, [r2]
+; CHECK-NEXT:    ldr r5, [r0, #4]
 ; CHECK-NEXT:  .LBB15_6: @ %if.end
-; CHECK-NEXT:    add.w r0, r12, r3, lsl #1
-; CHECK-NEXT:    lsr.w lr, r10, #2
+; CHECK-NEXT:    add.w r0, r5, r3, lsl #1
+; CHECK-NEXT:    mov r2, r5
+; CHECK-NEXT:    lsr.w lr, r4, #2
 ; CHECK-NEXT:    wls lr, lr, .LBB15_10
 ; CHECK-NEXT:  @ %bb.7: @ %while.body51.preheader
-; CHECK-NEXT:    bic r2, r10, #3
-; CHECK-NEXT:    adds r1, r2, r3
-; CHECK-NEXT:    mov r3, r12
-; CHECK-NEXT:    add.w r1, r12, r1, lsl #1
+; CHECK-NEXT:    bic r7, r4, #3
+; CHECK-NEXT:    adds r1, r7, r3
+; CHECK-NEXT:    mov r3, r2
+; CHECK-NEXT:    add.w r1, r2, r1, lsl #1
 ; CHECK-NEXT:  .LBB15_8: @ %while.body51
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrw.u32 q0, [r0], #8
 ; CHECK-NEXT:    vstrb.8 q0, [r3], #8
 ; CHECK-NEXT:    le lr, .LBB15_8
 ; CHECK-NEXT:  @ %bb.9: @ %while.end55.loopexit
-; CHECK-NEXT:    add.w r12, r12, r2, lsl #1
+; CHECK-NEXT:    add.w r2, r2, r7, lsl #1
 ; CHECK-NEXT:    mov r0, r1
 ; CHECK-NEXT:  .LBB15_10: @ %while.end55
-; CHECK-NEXT:    ands r1, r10, #3
+; CHECK-NEXT:    ands r1, r4, #3
 ; CHECK-NEXT:    beq .LBB15_12
 ; CHECK-NEXT:  @ %bb.11: @ %if.then59
 ; CHECK-NEXT:    vldrw.u32 q0, [r0]
 ; CHECK-NEXT:    vctp.16 r1
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vstrht.16 q0, [r12]
+; CHECK-NEXT:    vstrht.16 q0, [r2]
 ; CHECK-NEXT:  .LBB15_12: @ %if.end61
-; CHECK-NEXT:    vpop {d8, d9, d10, d11}
-; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, pc}
+; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 entry:
   %pState1 = getelementptr inbounds %struct.arm_fir_instance_f32, %struct.arm_fir_instance_f32* %S, i32 0, i32 1
   %0 = load half*, half** %pState1, align 4
diff --git a/llvm/test/CodeGen/Thumb2/mve-float32regloops.ll b/llvm/test/CodeGen/Thumb2/mve-float32regloops.ll
index 417c32d646238..f6d1d9ab88391 100644
--- a/llvm/test/CodeGen/Thumb2/mve-float32regloops.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-float32regloops.ll
@@ -8,13 +8,12 @@ define arm_aapcs_vfpcc void @test_fadd(float* noalias nocapture readonly %A, flo
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    bxlt lr
 ; CHECK-NEXT:    vmov r3, s0
-; CHECK-NEXT:    vdup.32 q0, r3
 ; CHECK-NEXT:  .LBB0_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
 ; CHECK-NEXT:    subs r2, #4
-; CHECK-NEXT:    vadd.f32 q1, q1, q0
-; CHECK-NEXT:    vstrb.8 q1, [r1], #16
+; CHECK-NEXT:    vadd.f32 q0, q0, r3
+; CHECK-NEXT:    vstrb.8 q0, [r1], #16
 ; CHECK-NEXT:    bne .LBB0_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    bx lr
@@ -54,13 +53,12 @@ define arm_aapcs_vfpcc void @test_fadd_r(float* noalias nocapture readonly %A, f
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    bxlt lr
 ; CHECK-NEXT:    vmov r3, s0
-; CHECK-NEXT:    vdup.32 q0, r3
 ; CHECK-NEXT:  .LBB1_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
 ; CHECK-NEXT:    subs r2, #4
-; CHECK-NEXT:    vadd.f32 q1, q0, q1
-; CHECK-NEXT:    vstrb.8 q1, [r1], #16
+; CHECK-NEXT:    vadd.f32 q0, q0, r3
+; CHECK-NEXT:    vstrb.8 q0, [r1], #16
 ; CHECK-NEXT:    bne .LBB1_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    bx lr
@@ -100,13 +98,12 @@ define arm_aapcs_vfpcc void @test_fmul(float* noalias nocapture readonly %A, flo
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    bxlt lr
 ; CHECK-NEXT:    vmov r3, s0
-; CHECK-NEXT:    vdup.32 q0, r3
 ; CHECK-NEXT:  .LBB2_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
 ; CHECK-NEXT:    subs r2, #4
-; CHECK-NEXT:    vmul.f32 q1, q1, q0
-; CHECK-NEXT:    vstrb.8 q1, [r1], #16
+; CHECK-NEXT:    vmul.f32 q0, q0, r3
+; CHECK-NEXT:    vstrb.8 q0, [r1], #16
 ; CHECK-NEXT:    bne .LBB2_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    bx lr
@@ -146,13 +143,12 @@ define arm_aapcs_vfpcc void @test_fmul_r(float* noalias nocapture readonly %A, f
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    bxlt lr
 ; CHECK-NEXT:    vmov r3, s0
-; CHECK-NEXT:    vdup.32 q0, r3
 ; CHECK-NEXT:  .LBB3_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
 ; CHECK-NEXT:    subs r2, #4
-; CHECK-NEXT:    vmul.f32 q1, q0, q1
-; CHECK-NEXT:    vstrb.8 q1, [r1], #16
+; CHECK-NEXT:    vmul.f32 q0, q0, r3
+; CHECK-NEXT:    vstrb.8 q0, [r1], #16
 ; CHECK-NEXT:    bne .LBB3_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    bx lr
@@ -192,13 +188,12 @@ define arm_aapcs_vfpcc void @test_fsub(float* noalias nocapture readonly %A, flo
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    bxlt lr
 ; CHECK-NEXT:    vmov r3, s0
-; CHECK-NEXT:    vdup.32 q0, r3
 ; CHECK-NEXT:  .LBB4_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
 ; CHECK-NEXT:    subs r2, #4
-; CHECK-NEXT:    vsub.f32 q1, q1, q0
-; CHECK-NEXT:    vstrb.8 q1, [r1], #16
+; CHECK-NEXT:    vsub.f32 q0, q0, r3
+; CHECK-NEXT:    vstrb.8 q0, [r1], #16
 ; CHECK-NEXT:    bne .LBB4_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    bx lr
@@ -285,15 +280,13 @@ define arm_aapcs_vfpcc void @test_fmas(float* noalias nocapture readonly %A, flo
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    bxlt lr
 ; CHECK-NEXT:    vmov r12, s0
-; CHECK-NEXT:    vdup.32 q0, r12
 ; CHECK-NEXT:  .LBB6_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
-; CHECK-NEXT:    vldrw.u32 q2, [r1], #16
-; CHECK-NEXT:    vmov q3, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
 ; CHECK-NEXT:    subs r3, #4
-; CHECK-NEXT:    vfma.f32 q3, q2, q1
-; CHECK-NEXT:    vstrb.8 q3, [r2], #16
+; CHECK-NEXT:    vfmas.f32 q1, q0, r12
+; CHECK-NEXT:    vstrb.8 q1, [r2], #16
 ; CHECK-NEXT:    bne .LBB6_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    bx lr
@@ -337,15 +330,13 @@ define arm_aapcs_vfpcc void @test_fmas_r(float* noalias nocapture readonly %A, f
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    bxlt lr
 ; CHECK-NEXT:    vmov r12, s0
-; CHECK-NEXT:    vdup.32 q0, r12
 ; CHECK-NEXT:  .LBB7_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
-; CHECK-NEXT:    vldrw.u32 q2, [r1], #16
-; CHECK-NEXT:    vmov q3, q0
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
 ; CHECK-NEXT:    subs r3, #4
-; CHECK-NEXT:    vfma.f32 q3, q2, q1
-; CHECK-NEXT:    vstrb.8 q3, [r2], #16
+; CHECK-NEXT:    vfmas.f32 q1, q0, r12
+; CHECK-NEXT:    vstrb.8 q1, [r2], #16
 ; CHECK-NEXT:    bne .LBB7_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    bx lr
@@ -389,14 +380,13 @@ define arm_aapcs_vfpcc void @test_fma(float* noalias nocapture readonly %A, floa
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    bxlt lr
 ; CHECK-NEXT:    vmov r12, s0
-; CHECK-NEXT:    vdup.32 q0, r12
 ; CHECK-NEXT:  .LBB8_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
-; CHECK-NEXT:    vldrw.u32 q2, [r1], #16
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
 ; CHECK-NEXT:    subs r3, #4
-; CHECK-NEXT:    vfma.f32 q2, q1, q0
-; CHECK-NEXT:    vstrb.8 q2, [r2], #16
+; CHECK-NEXT:    vfma.f32 q1, q0, r12
+; CHECK-NEXT:    vstrb.8 q1, [r2], #16
 ; CHECK-NEXT:    bne .LBB8_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    bx lr
@@ -440,14 +430,13 @@ define arm_aapcs_vfpcc void @test_fma_r(float* noalias nocapture readonly %A, fl
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    bxlt lr
 ; CHECK-NEXT:    vmov r12, s0
-; CHECK-NEXT:    vdup.32 q0, r12
 ; CHECK-NEXT:  .LBB9_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
-; CHECK-NEXT:    vldrw.u32 q2, [r1], #16
+; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
 ; CHECK-NEXT:    subs r3, #4
-; CHECK-NEXT:    vfma.f32 q2, q0, q1
-; CHECK-NEXT:    vstrb.8 q2, [r2], #16
+; CHECK-NEXT:    vfma.f32 q1, q0, r12
+; CHECK-NEXT:    vstrb.8 q1, [r2], #16
 ; CHECK-NEXT:    bne .LBB9_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    bx lr
@@ -597,15 +586,14 @@ define arm_aapcs_vfpcc void @test_fms(float* noalias nocapture readonly %A, floa
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    bxlt lr
 ; CHECK-NEXT:    vmov r12, s0
-; CHECK-NEXT:    vdup.32 q0, r12
 ; CHECK-NEXT:  .LBB12_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
-; CHECK-NEXT:    vldrw.u32 q2, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
+; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
 ; CHECK-NEXT:    subs r3, #4
-; CHECK-NEXT:    vneg.f32 q1, q1
-; CHECK-NEXT:    vfma.f32 q1, q2, q0
-; CHECK-NEXT:    vstrb.8 q1, [r2], #16
+; CHECK-NEXT:    vneg.f32 q0, q0
+; CHECK-NEXT:    vfma.f32 q0, q1, r12
+; CHECK-NEXT:    vstrb.8 q0, [r2], #16
 ; CHECK-NEXT:    bne .LBB12_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    bx lr
@@ -649,15 +637,14 @@ define arm_aapcs_vfpcc void @test_fms_r(float* noalias nocapture readonly %A, fl
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    bxlt lr
 ; CHECK-NEXT:    vmov r12, s0
-; CHECK-NEXT:    vdup.32 q0, r12
 ; CHECK-NEXT:  .LBB13_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
-; CHECK-NEXT:    vldrw.u32 q2, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
+; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
 ; CHECK-NEXT:    subs r3, #4
-; CHECK-NEXT:    vneg.f32 q1, q1
-; CHECK-NEXT:    vfma.f32 q1, q0, q2
-; CHECK-NEXT:    vstrb.8 q1, [r2], #16
+; CHECK-NEXT:    vneg.f32 q0, q0
+; CHECK-NEXT:    vfma.f32 q0, q1, r12
+; CHECK-NEXT:    vstrb.8 q0, [r2], #16
 ; CHECK-NEXT:    bne .LBB13_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    bx lr
@@ -783,99 +770,118 @@ for.end14:                                        ; preds = %for.cond6.for.end_c
 define void @arm_fir_f32_1_4_mve(%struct.arm_fir_instance_f32* nocapture readonly %S, float* nocapture readonly %pSrc, float* %pDst, i32 %blockSize) {
 ; CHECK-LABEL: arm_fir_f32_1_4_mve:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, lr}
-; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
-; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
-; CHECK-NEXT:    ldrh.w r9, [r0]
-; CHECK-NEXT:    ldr.w r12, [r0, #4]
-; CHECK-NEXT:    sub.w r7, r9, #1
+; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; CHECK-NEXT:    .pad #4
+; CHECK-NEXT:    sub sp, #4
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    .pad #24
+; CHECK-NEXT:    sub sp, #24
+; CHECK-NEXT:    ldrh r4, [r0]
+; CHECK-NEXT:    ldr.w r11, [r0, #4]
+; CHECK-NEXT:    subs r7, r4, #1
 ; CHECK-NEXT:    cmp r7, #3
 ; CHECK-NEXT:    bhi .LBB15_6
 ; CHECK-NEXT:  @ %bb.1: @ %if.then
 ; CHECK-NEXT:    ldr r6, [r0, #8]
-; CHECK-NEXT:    add.w r4, r12, r7, lsl #2
-; CHECK-NEXT:    ldrd lr, r8, [r6]
-; CHECK-NEXT:    ldrd r5, r6, [r6, #8]
-; CHECK-NEXT:    vdup.32 q3, lr
-; CHECK-NEXT:    vdup.32 q2, r8
-; CHECK-NEXT:    vdup.32 q0, r6
-; CHECK-NEXT:    vdup.32 q1, r5
+; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    add.w r0, r11, r7, lsl #2
 ; CHECK-NEXT:    lsr.w lr, r3, #2
+; CHECK-NEXT:    vldr s0, [r6]
+; CHECK-NEXT:    vldr s2, [r6, #4]
+; CHECK-NEXT:    vldr s4, [r6, #8]
+; CHECK-NEXT:    vldr s6, [r6, #12]
 ; CHECK-NEXT:    wls lr, lr, .LBB15_5
 ; CHECK-NEXT:  @ %bb.2: @ %while.body.lr.ph
-; CHECK-NEXT:    bic r10, r3, #3
+; CHECK-NEXT:    strd r5, r3, [sp, #12] @ 8-byte Folded Spill
+; CHECK-NEXT:    vmov r7, s4
+; CHECK-NEXT:    str r4, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT:    vmov r5, s6
+; CHECK-NEXT:    vmov r4, s2
+; CHECK-NEXT:    bic r3, r3, #3
+; CHECK-NEXT:    vmov r8, s0
 ; CHECK-NEXT:    movs r6, #0
-; CHECK-NEXT:    add.w r8, r2, r10, lsl #2
+; CHECK-NEXT:    str r3, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    add.w r3, r2, r3, lsl #2
+; CHECK-NEXT:    str r3, [sp, #8] @ 4-byte Spill
 ; CHECK-NEXT:  .LBB15_3: @ %while.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    adds r7, r1, r6
-; CHECK-NEXT:    adds r5, r2, r6
-; CHECK-NEXT:    vldrw.u32 q4, [r7]
-; CHECK-NEXT:    adds r7, r4, r6
-; CHECK-NEXT:    vstrw.32 q4, [r7]
-; CHECK-NEXT:    add.w r7, r12, r6
-; CHECK-NEXT:    vldrw.u32 q4, [r7]
-; CHECK-NEXT:    vldrw.u32 q5, [r7, #4]
-; CHECK-NEXT:    vldrw.u32 q6, [r7, #12]
+; CHECK-NEXT:    add.w r10, r1, r6
+; CHECK-NEXT:    adds r3, r0, r6
+; CHECK-NEXT:    vldrw.u32 q2, [r10]
+; CHECK-NEXT:    add.w r12, r11, r6
+; CHECK-NEXT:    add.w r9, r2, r6
 ; CHECK-NEXT:    adds r6, #16
-; CHECK-NEXT:    vmul.f32 q4, q4, q3
-; CHECK-NEXT:    vfma.f32 q4, q5, q2
-; CHECK-NEXT:    vldrw.u32 q5, [r7, #8]
-; CHECK-NEXT:    vfma.f32 q4, q5, q1
-; CHECK-NEXT:    vfma.f32 q4, q6, q0
-; CHECK-NEXT:    vstrw.32 q4, [r5]
+; CHECK-NEXT:    vstrw.32 q2, [r3]
+; CHECK-NEXT:    vldrw.u32 q2, [r12]
+; CHECK-NEXT:    vldrw.u32 q3, [r12, #4]
+; CHECK-NEXT:    vldrw.u32 q4, [r12, #12]
+; CHECK-NEXT:    vmul.f32 q2, q2, r8
+; CHECK-NEXT:    vfma.f32 q2, q3, r4
+; CHECK-NEXT:    vldrw.u32 q3, [r12, #8]
+; CHECK-NEXT:    vfma.f32 q2, q3, r7
+; CHECK-NEXT:    vfma.f32 q2, q4, r5
+; CHECK-NEXT:    vstrw.32 q2, [r9]
 ; CHECK-NEXT:    le lr, .LBB15_3
 ; CHECK-NEXT:  @ %bb.4: @ %while.end.loopexit
-; CHECK-NEXT:    add r4, r6
-; CHECK-NEXT:    add.w r12, r12, r10, lsl #2
-; CHECK-NEXT:    add.w r1, r1, r10, lsl #2
-; CHECK-NEXT:    mov r2, r8
+; CHECK-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    add r0, r6
+; CHECK-NEXT:    ldrd r3, r4, [sp, #16] @ 8-byte Folded Reload
+; CHECK-NEXT:    add.w r11, r11, r2, lsl #2
+; CHECK-NEXT:    add.w r1, r1, r2, lsl #2
+; CHECK-NEXT:    ldrd r2, r5, [sp, #8] @ 8-byte Folded Reload
 ; CHECK-NEXT:  .LBB15_5: @ %while.end
-; CHECK-NEXT:    and r7, r3, #3
-; CHECK-NEXT:    vldrw.u32 q4, [r1]
-; CHECK-NEXT:    vctp.32 r7
+; CHECK-NEXT:    and r6, r3, #3
+; CHECK-NEXT:    vmov r12, s6
+; CHECK-NEXT:    vmov lr, s4
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    vctp.32 r6
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vstrwt.32 q4, [r4]
-; CHECK-NEXT:    vldrw.u32 q4, [r12]
-; CHECK-NEXT:    vmul.f32 q3, q4, q3
-; CHECK-NEXT:    vldrw.u32 q4, [r12, #4]
-; CHECK-NEXT:    vfma.f32 q3, q4, q2
-; CHECK-NEXT:    vldrw.u32 q2, [r12, #8]
-; CHECK-NEXT:    vfma.f32 q3, q2, q1
-; CHECK-NEXT:    vldrw.u32 q1, [r12, #12]
-; CHECK-NEXT:    vfma.f32 q3, q1, q0
+; CHECK-NEXT:    vstrwt.32 q1, [r0]
+; CHECK-NEXT:    vmov r9, s2
+; CHECK-NEXT:    vldrw.u32 q1, [r11, #4]
+; CHECK-NEXT:    vmov r7, s0
+; CHECK-NEXT:    vldrw.u32 q0, [r11]
+; CHECK-NEXT:    vmul.f32 q0, q0, r7
+; CHECK-NEXT:    vfma.f32 q0, q1, r9
+; CHECK-NEXT:    vldrw.u32 q1, [r11, #8]
+; CHECK-NEXT:    vfma.f32 q0, q1, lr
+; CHECK-NEXT:    vldrw.u32 q1, [r11, #12]
+; CHECK-NEXT:    vfma.f32 q0, q1, r12
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vstrwt.32 q3, [r2]
-; CHECK-NEXT:    ldr.w r12, [r0, #4]
+; CHECK-NEXT:    vstrwt.32 q0, [r2]
+; CHECK-NEXT:    ldr.w r11, [r5, #4]
 ; CHECK-NEXT:  .LBB15_6: @ %if.end
-; CHECK-NEXT:    add.w r0, r12, r3, lsl #2
-; CHECK-NEXT:    lsr.w lr, r9, #2
+; CHECK-NEXT:    add.w r0, r11, r3, lsl #2
+; CHECK-NEXT:    lsr.w lr, r4, #2
 ; CHECK-NEXT:    wls lr, lr, .LBB15_10
 ; CHECK-NEXT:  @ %bb.7: @ %while.body51.preheader
-; CHECK-NEXT:    bic r2, r9, #3
-; CHECK-NEXT:    adds r1, r2, r3
-; CHECK-NEXT:    mov r3, r12
-; CHECK-NEXT:    add.w r1, r12, r1, lsl #2
+; CHECK-NEXT:    bic r7, r4, #3
+; CHECK-NEXT:    adds r1, r7, r3
+; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    add.w r1, r11, r1, lsl #2
 ; CHECK-NEXT:  .LBB15_8: @ %while.body51
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
 ; CHECK-NEXT:    vstrb.8 q0, [r3], #16
 ; CHECK-NEXT:    le lr, .LBB15_8
 ; CHECK-NEXT:  @ %bb.9: @ %while.end55.loopexit
-; CHECK-NEXT:    add.w r12, r12, r2, lsl #2
+; CHECK-NEXT:    add.w r11, r11, r7, lsl #2
 ; CHECK-NEXT:    mov r0, r1
 ; CHECK-NEXT:  .LBB15_10: @ %while.end55
-; CHECK-NEXT:    ands r1, r9, #3
+; CHECK-NEXT:    ands r1, r4, #3
 ; CHECK-NEXT:    beq .LBB15_12
 ; CHECK-NEXT:  @ %bb.11: @ %if.then59
 ; CHECK-NEXT:    vldrw.u32 q0, [r0]
 ; CHECK-NEXT:    vctp.32 r1
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vstrwt.32 q0, [r12]
+; CHECK-NEXT:    vstrwt.32 q0, [r11]
 ; CHECK-NEXT:  .LBB15_12: @ %if.end61
-; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
-; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, pc}
+; CHECK-NEXT:    add sp, #24
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    add sp, #4
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 entry:
   %pState1 = getelementptr inbounds %struct.arm_fir_instance_f32, %struct.arm_fir_instance_f32* %S, i32 0, i32 1
   %0 = load float*, float** %pState1, align 4
diff --git a/llvm/test/CodeGen/Thumb2/mve-gather-ptrs.ll b/llvm/test/CodeGen/Thumb2/mve-gather-ptrs.ll
index e35938894a6a0..c44668c7f5b38 100644
--- a/llvm/test/CodeGen/Thumb2/mve-gather-ptrs.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-gather-ptrs.ll
@@ -251,21 +251,13 @@ define arm_aapcs_vfpcc <2 x i32> @ptr_v2i16_zext(<2 x i16*>* %offptr) {
 ; CHECK-LABEL: ptr_v2i16_zext:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    ldrd r1, r0, [r0]
-; CHECK-NEXT:    adr r2, .LCPI9_0
+; CHECK-NEXT:    vmov.i64 q0, #0xffff
 ; CHECK-NEXT:    ldrh r0, [r0]
-; CHECK-NEXT:    vldrw.u32 q0, [r2]
 ; CHECK-NEXT:    ldrh r1, [r1]
 ; CHECK-NEXT:    vmov.32 q1[0], r1
 ; CHECK-NEXT:    vmov.32 q1[2], r0
 ; CHECK-NEXT:    vand q0, q1, q0
 ; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 4
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI9_0:
-; CHECK-NEXT:    .long 65535 @ 0xffff
-; CHECK-NEXT:    .long 0 @ 0x0
-; CHECK-NEXT:    .long 65535 @ 0xffff
-; CHECK-NEXT:    .long 0 @ 0x0
 entry:
   %offs = load <2 x i16*>, <2 x i16*>* %offptr, align 4
   %gather = call <2 x i16> @llvm.masked.gather.v2i16.v2p0i16(<2 x i16*> %offs, i32 2, <2 x i1> <i1 true, i1 true>, <2 x i16> undef)
diff --git a/llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll b/llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll
index 3b49649e8cdfd..3814b0b2766c9 100644
--- a/llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll
@@ -408,6 +408,7 @@ define void @foo_zext_v2i64_v2i32(<2 x i64> *%dest, <2 x i32> *%mask, <2 x i32>
 ; CHECK-LE-NEXT:    ldrd lr, r12, [r1]
 ; CHECK-LE-NEXT:    movs r1, #0
 ; CHECK-LE-NEXT:    @ implicit-def: $q1
+; CHECK-LE-NEXT:    vmov.i64 q2, #0xffffffff
 ; CHECK-LE-NEXT:    rsbs.w r3, lr, #0
 ; CHECK-LE-NEXT:    vmov.32 q0[0], lr
 ; CHECK-LE-NEXT:    sbcs.w r3, r1, lr, asr #31
@@ -424,23 +425,21 @@ define void @foo_zext_v2i64_v2i32(<2 x i64> *%dest, <2 x i32> *%mask, <2 x i32>
 ; CHECK-LE-NEXT:    bfi r1, lr, #0, #1
 ; CHECK-LE-NEXT:    vmov.32 q0[2], r12
 ; CHECK-LE-NEXT:    and r3, r1, #3
-; CHECK-LE-NEXT:    adr.w r12, .LCPI7_0
+; CHECK-LE-NEXT:    mov.w r12, #0
 ; CHECK-LE-NEXT:    lsls r1, r1, #31
 ; CHECK-LE-NEXT:    itt ne
 ; CHECK-LE-NEXT:    ldrne r1, [r2]
 ; CHECK-LE-NEXT:    vmovne.32 q1[0], r1
 ; CHECK-LE-NEXT:    lsls r1, r3, #30
-; CHECK-LE-NEXT:    vmov r3, s0
 ; CHECK-LE-NEXT:    itt mi
 ; CHECK-LE-NEXT:    ldrmi r1, [r2, #4]
 ; CHECK-LE-NEXT:    vmovmi.32 q1[2], r1
+; CHECK-LE-NEXT:    vmov r1, s0
 ; CHECK-LE-NEXT:    movs r2, #0
-; CHECK-LE-NEXT:    vldrw.u32 q2, [r12]
-; CHECK-LE-NEXT:    mov.w r12, #0
 ; CHECK-LE-NEXT:    vand q1, q1, q2
-; CHECK-LE-NEXT:    rsbs r1, r3, #0
-; CHECK-LE-NEXT:    sbcs.w r1, r2, r3, asr #31
+; CHECK-LE-NEXT:    rsbs r3, r1, #0
 ; CHECK-LE-NEXT:    vmov r3, s2
+; CHECK-LE-NEXT:    sbcs.w r1, r2, r1, asr #31
 ; CHECK-LE-NEXT:    it lt
 ; CHECK-LE-NEXT:    movlt.w r12, #1
 ; CHECK-LE-NEXT:    rsbs r1, r3, #0
@@ -460,13 +459,6 @@ define void @foo_zext_v2i64_v2i32(<2 x i64> *%dest, <2 x i32> *%mask, <2 x i32>
 ; CHECK-LE-NEXT:    vstrmi d3, [r0, #8]
 ; CHECK-LE-NEXT:    add sp, #4
 ; CHECK-LE-NEXT:    pop {r7, pc}
-; CHECK-LE-NEXT:    .p2align 4
-; CHECK-LE-NEXT:  @ %bb.1:
-; CHECK-LE-NEXT:  .LCPI7_0:
-; CHECK-LE-NEXT:    .long 4294967295 @ 0xffffffff
-; CHECK-LE-NEXT:    .long 0 @ 0x0
-; CHECK-LE-NEXT:    .long 4294967295 @ 0xffffffff
-; CHECK-LE-NEXT:    .long 0 @ 0x0
 ;
 ; CHECK-BE-LABEL: foo_zext_v2i64_v2i32:
 ; CHECK-BE:       @ %bb.0: @ %entry
@@ -511,15 +503,13 @@ define void @foo_zext_v2i64_v2i32(<2 x i64> *%dest, <2 x i32> *%mask, <2 x i32>
 ; CHECK-BE-NEXT:  .LBB7_4: @ %else2
 ; CHECK-BE-NEXT:    vrev64.32 q3, q2
 ; CHECK-BE-NEXT:    movs r2, #0
-; CHECK-BE-NEXT:    vmov r3, s15
-; CHECK-BE-NEXT:    adr.w r12, .LCPI7_0
-; CHECK-BE-NEXT:    vldrb.u8 q0, [r12]
+; CHECK-BE-NEXT:    vmov r1, s15
 ; CHECK-BE-NEXT:    mov.w r12, #0
-; CHECK-BE-NEXT:    vrev64.8 q2, q0
-; CHECK-BE-NEXT:    vand q0, q1, q2
-; CHECK-BE-NEXT:    rsbs r1, r3, #0
-; CHECK-BE-NEXT:    sbcs.w r1, r2, r3, asr #31
+; CHECK-BE-NEXT:    vmov.i64 q0, #0xffffffff
+; CHECK-BE-NEXT:    vand q0, q1, q0
+; CHECK-BE-NEXT:    rsbs r3, r1, #0
 ; CHECK-BE-NEXT:    vmov r3, s13
+; CHECK-BE-NEXT:    sbcs.w r1, r2, r1, asr #31
 ; CHECK-BE-NEXT:    it lt
 ; CHECK-BE-NEXT:    movlt.w r12, #1
 ; CHECK-BE-NEXT:    rsbs r1, r3, #0
@@ -539,13 +529,6 @@ define void @foo_zext_v2i64_v2i32(<2 x i64> *%dest, <2 x i32> *%mask, <2 x i32>
 ; CHECK-BE-NEXT:    vstrmi d1, [r0, #8]
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    pop {r7, pc}
-; CHECK-BE-NEXT:    .p2align 4
-; CHECK-BE-NEXT:  @ %bb.5:
-; CHECK-BE-NEXT:  .LCPI7_0:
-; CHECK-BE-NEXT:    .long 0 @ 0x0
-; CHECK-BE-NEXT:    .long 4294967295 @ 0xffffffff
-; CHECK-BE-NEXT:    .long 0 @ 0x0
-; CHECK-BE-NEXT:    .long 4294967295 @ 0xffffffff
 entry:
   %0 = load <2 x i32>, <2 x i32>* %mask, align 4
   %1 = icmp sgt <2 x i32> %0, zeroinitializer
@@ -565,6 +548,7 @@ define void @foo_zext_v2i64_v2i32_unaligned(<2 x i64> *%dest, <2 x i32> *%mask,
 ; CHECK-LE-NEXT:    ldrd lr, r12, [r1]
 ; CHECK-LE-NEXT:    movs r1, #0
 ; CHECK-LE-NEXT:    @ implicit-def: $q1
+; CHECK-LE-NEXT:    vmov.i64 q2, #0xffffffff
 ; CHECK-LE-NEXT:    rsbs.w r3, lr, #0
 ; CHECK-LE-NEXT:    vmov.32 q0[0], lr
 ; CHECK-LE-NEXT:    sbcs.w r3, r1, lr, asr #31
@@ -581,23 +565,21 @@ define void @foo_zext_v2i64_v2i32_unaligned(<2 x i64> *%dest, <2 x i32> *%mask,
 ; CHECK-LE-NEXT:    bfi r1, lr, #0, #1
 ; CHECK-LE-NEXT:    vmov.32 q0[2], r12
 ; CHECK-LE-NEXT:    and r3, r1, #3
-; CHECK-LE-NEXT:    adr.w r12, .LCPI8_0
+; CHECK-LE-NEXT:    mov.w r12, #0
 ; CHECK-LE-NEXT:    lsls r1, r1, #31
 ; CHECK-LE-NEXT:    itt ne
 ; CHECK-LE-NEXT:    ldrne r1, [r2]
 ; CHECK-LE-NEXT:    vmovne.32 q1[0], r1
 ; CHECK-LE-NEXT:    lsls r1, r3, #30
-; CHECK-LE-NEXT:    vmov r3, s0
 ; CHECK-LE-NEXT:    itt mi
 ; CHECK-LE-NEXT:    ldrmi r1, [r2, #4]
 ; CHECK-LE-NEXT:    vmovmi.32 q1[2], r1
+; CHECK-LE-NEXT:    vmov r1, s0
 ; CHECK-LE-NEXT:    movs r2, #0
-; CHECK-LE-NEXT:    vldrw.u32 q2, [r12]
-; CHECK-LE-NEXT:    mov.w r12, #0
 ; CHECK-LE-NEXT:    vand q1, q1, q2
-; CHECK-LE-NEXT:    rsbs r1, r3, #0
-; CHECK-LE-NEXT:    sbcs.w r1, r2, r3, asr #31
+; CHECK-LE-NEXT:    rsbs r3, r1, #0
 ; CHECK-LE-NEXT:    vmov r3, s2
+; CHECK-LE-NEXT:    sbcs.w r1, r2, r1, asr #31
 ; CHECK-LE-NEXT:    it lt
 ; CHECK-LE-NEXT:    movlt.w r12, #1
 ; CHECK-LE-NEXT:    rsbs r1, r3, #0
@@ -619,13 +601,6 @@ define void @foo_zext_v2i64_v2i32_unaligned(<2 x i64> *%dest, <2 x i32> *%mask,
 ; CHECK-LE-NEXT:    strdmi r1, r2, [r0, #8]
 ; CHECK-LE-NEXT:    add sp, #4
 ; CHECK-LE-NEXT:    pop {r7, pc}
-; CHECK-LE-NEXT:    .p2align 4
-; CHECK-LE-NEXT:  @ %bb.1:
-; CHECK-LE-NEXT:  .LCPI8_0:
-; CHECK-LE-NEXT:    .long 4294967295 @ 0xffffffff
-; CHECK-LE-NEXT:    .long 0 @ 0x0
-; CHECK-LE-NEXT:    .long 4294967295 @ 0xffffffff
-; CHECK-LE-NEXT:    .long 0 @ 0x0
 ;
 ; CHECK-BE-LABEL: foo_zext_v2i64_v2i32_unaligned:
 ; CHECK-BE:       @ %bb.0: @ %entry
@@ -670,15 +645,13 @@ define void @foo_zext_v2i64_v2i32_unaligned(<2 x i64> *%dest, <2 x i32> *%mask,
 ; CHECK-BE-NEXT:  .LBB8_4: @ %else2
 ; CHECK-BE-NEXT:    vrev64.32 q3, q2
 ; CHECK-BE-NEXT:    movs r2, #0
-; CHECK-BE-NEXT:    vmov r3, s15
-; CHECK-BE-NEXT:    adr.w r12, .LCPI8_0
-; CHECK-BE-NEXT:    vldrb.u8 q0, [r12]
+; CHECK-BE-NEXT:    vmov r1, s15
 ; CHECK-BE-NEXT:    mov.w r12, #0
-; CHECK-BE-NEXT:    vrev64.8 q2, q0
-; CHECK-BE-NEXT:    vand q0, q1, q2
-; CHECK-BE-NEXT:    rsbs r1, r3, #0
-; CHECK-BE-NEXT:    sbcs.w r1, r2, r3, asr #31
+; CHECK-BE-NEXT:    vmov.i64 q0, #0xffffffff
+; CHECK-BE-NEXT:    vand q0, q1, q0
+; CHECK-BE-NEXT:    rsbs r3, r1, #0
 ; CHECK-BE-NEXT:    vmov r3, s13
+; CHECK-BE-NEXT:    sbcs.w r1, r2, r1, asr #31
 ; CHECK-BE-NEXT:    it lt
 ; CHECK-BE-NEXT:    movlt.w r12, #1
 ; CHECK-BE-NEXT:    rsbs r1, r3, #0
@@ -700,13 +673,6 @@ define void @foo_zext_v2i64_v2i32_unaligned(<2 x i64> *%dest, <2 x i32> *%mask,
 ; CHECK-BE-NEXT:    strdmi r2, r1, [r0, #8]
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    pop {r7, pc}
-; CHECK-BE-NEXT:    .p2align 4
-; CHECK-BE-NEXT:  @ %bb.5:
-; CHECK-BE-NEXT:  .LCPI8_0:
-; CHECK-BE-NEXT:    .long 0 @ 0x0
-; CHECK-BE-NEXT:    .long 4294967295 @ 0xffffffff
-; CHECK-BE-NEXT:    .long 0 @ 0x0
-; CHECK-BE-NEXT:    .long 4294967295 @ 0xffffffff
 entry:
   %0 = load <2 x i32>, <2 x i32>* %mask, align 4
   %1 = icmp sgt <2 x i32> %0, zeroinitializer
diff --git a/llvm/test/CodeGen/Thumb2/mve-masked-load.ll b/llvm/test/CodeGen/Thumb2/mve-masked-load.ll
index 5bddc5ed83337..ca51edb92a2a2 100644
--- a/llvm/test/CodeGen/Thumb2/mve-masked-load.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-masked-load.ll
@@ -1830,8 +1830,7 @@ define arm_aapcs_vfpcc <2 x i64> @masked_v2i64_align4_zero(<2 x i64> *%dest, <2
 ; CHECK-BE-NEXT:    vldr d0, [r0]
 ; CHECK-BE-NEXT:    b .LBB49_3
 ; CHECK-BE-NEXT:  .LBB49_2:
-; CHECK-BE-NEXT:    vmov.i32 q1, #0x0
-; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-BE-NEXT:  .LBB49_3: @ %else
 ; CHECK-BE-NEXT:    lsls r1, r1, #30
 ; CHECK-BE-NEXT:    it mi
@@ -1924,8 +1923,7 @@ define arm_aapcs_vfpcc <2 x double> @masked_v2f64_align4_zero(<2 x double> *%des
 ; CHECK-BE-NEXT:    vldr d0, [r0]
 ; CHECK-BE-NEXT:    b .LBB50_3
 ; CHECK-BE-NEXT:  .LBB50_2:
-; CHECK-BE-NEXT:    vmov.i32 q1, #0x0
-; CHECK-BE-NEXT:    vrev64.32 q0, q1
+; CHECK-BE-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-BE-NEXT:  .LBB50_3: @ %else
 ; CHECK-BE-NEXT:    lsls r1, r1, #30
 ; CHECK-BE-NEXT:    it mi
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-not.ll b/llvm/test/CodeGen/Thumb2/mve-pred-not.ll
index bc94f8ba91876..70fc0e4ab1cb5 100644
--- a/llvm/test/CodeGen/Thumb2/mve-pred-not.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-not.ll
@@ -383,11 +383,9 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @vpnot_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: vpnot_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vpt.s32 lt, q0, zr
+; CHECK-NEXT:    vpte.s32 lt, q0, zr
 ; CHECK-NEXT:    vcmpt.s32 gt, q1, zr
-; CHECK-NEXT:    vpnot
-; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vcmpt.i32 eq, q2, zr
+; CHECK-NEXT:    vcmpe.i32 eq, q2, zr
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -400,3 +398,73 @@ entry:
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
+
+declare <4 x i32> @llvm.arm.mve.max.predicated.v4i32.v4i1(<4 x i32>, <4 x i32>, i32, <4 x i1>, <4 x i32>)
+declare <4 x i32> @llvm.arm.mve.orr.predicated.v4i32.v4i1(<4 x i32>, <4 x i32>, <4 x i1>, <4 x i32>)
+
+define arm_aapcs_vfpcc <4 x i32> @vpttet_v4i1(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
+; CHECK-LABEL: vpttet_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov q3, q2
+; CHECK-NEXT:    vpttet.s32 ge, q0, q2
+; CHECK-NEXT:    vmaxt.s32 q3, q0, q1
+; CHECK-NEXT:    vcmpt.s32 gt, q0, zr
+; CHECK-NEXT:    vcmpe.s32 gt, q1, zr
+; CHECK-NEXT:    vmovt q3, q2
+; CHECK-NEXT:    vmov q0, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = icmp sge <4 x i32> %x, %z
+  %1 = tail call <4 x i32> @llvm.arm.mve.max.predicated.v4i32.v4i1(<4 x i32> %x, <4 x i32> %y, i32 0, <4 x i1> %0, <4 x i32> %z)
+  %2 = icmp sgt <4 x i32> %x, zeroinitializer
+  %3 = and <4 x i1> %0, %2
+  %4 = xor <4 x i1> %3, <i1 true, i1 true, i1 true, i1 true>
+  %5 = icmp sgt <4 x i32> %y, zeroinitializer
+  %6 = and <4 x i1> %5, %4
+  %7 = tail call <4 x i32> @llvm.arm.mve.orr.predicated.v4i32.v4i1(<4 x i32> %z, <4 x i32> %z, <4 x i1> %6, <4 x i32> %1)
+  ret <4 x i32> %7
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vpttee_v4i1(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
+; CHECK-LABEL: vpttee_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov q3, q2
+; CHECK-NEXT:    vpttee.s32 ge, q0, q2
+; CHECK-NEXT:    vmaxt.s32 q3, q0, q1
+; CHECK-NEXT:    vcmpt.s32 gt, q0, zr
+; CHECK-NEXT:    vmove q3, q2
+; CHECK-NEXT:    vmove q3, q2
+; CHECK-NEXT:    vmov q0, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = icmp sge <4 x i32> %x, %z
+  %1 = tail call <4 x i32> @llvm.arm.mve.max.predicated.v4i32.v4i1(<4 x i32> %x, <4 x i32> %y, i32 0, <4 x i1> %0, <4 x i32> %z)
+  %2 = icmp sgt <4 x i32> %x, zeroinitializer
+  %3 = and <4 x i1> %0, %2
+  %4 = xor <4 x i1> %3, <i1 true, i1 true, i1 true, i1 true>
+  %5 = tail call <4 x i32> @llvm.arm.mve.orr.predicated.v4i32.v4i1(<4 x i32> %z, <4 x i32> %z, <4 x i1> %4, <4 x i32> %1)
+  %6 = tail call <4 x i32> @llvm.arm.mve.orr.predicated.v4i32.v4i1(<4 x i32> %z, <4 x i32> %z, <4 x i1> %4, <4 x i32> %5)
+  ret <4 x i32> %6
+}
+
+define arm_aapcs_vfpcc <4 x i32> @vpttte_v4i1(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
+; CHECK-LABEL: vpttte_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov q3, q2
+; CHECK-NEXT:    vpttte.s32 ge, q0, q2
+; CHECK-NEXT:    vmaxt.s32 q3, q0, q1
+; CHECK-NEXT:    vcmpt.s32 gt, q0, zr
+; CHECK-NEXT:    vmovt q3, q2
+; CHECK-NEXT:    vmove q3, q2
+; CHECK-NEXT:    vmov q0, q3
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = icmp sge <4 x i32> %x, %z
+  %1 = tail call <4 x i32> @llvm.arm.mve.max.predicated.v4i32.v4i1(<4 x i32> %x, <4 x i32> %y, i32 0, <4 x i1> %0, <4 x i32> %z)
+  %2 = icmp sgt <4 x i32> %x, zeroinitializer
+  %3 = and <4 x i1> %0, %2
+  %4 = tail call <4 x i32> @llvm.arm.mve.orr.predicated.v4i32.v4i1(<4 x i32> %z, <4 x i32> %z, <4 x i1> %3, <4 x i32> %1)
+  %5 = xor <4 x i1> %3, <i1 true, i1 true, i1 true, i1 true>
+  %6 = tail call <4 x i32> @llvm.arm.mve.orr.predicated.v4i32.v4i1(<4 x i32> %z, <4 x i32> %z, <4 x i1> %5, <4 x i32> %4)
+  ret <4 x i32> %6
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-threshold.ll b/llvm/test/CodeGen/Thumb2/mve-pred-threshold.ll
index 0fb5efb4bd99a..df211f1efebc4 100644
--- a/llvm/test/CodeGen/Thumb2/mve-pred-threshold.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-threshold.ll
@@ -19,11 +19,9 @@ define arm_aapcs_vfpcc void @thres_i32(i32* %data, i16 zeroext %N, i32 %T) {
 ; CHECK-NEXT:  .LBB0_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrw.u32 q1, [r0]
-; CHECK-NEXT:    vpt.s32 ge, q1, r2
+; CHECK-NEXT:    vpte.s32 ge, q1, r2
 ; CHECK-NEXT:    vcmpt.s32 le, q1, r1
-; CHECK-NEXT:    vpnot
-; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vstrwt.32 q0, [r0], #16
+; CHECK-NEXT:    vstrwe.32 q0, [r0], #16
 ; CHECK-NEXT:    le lr, .LBB0_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
@@ -77,11 +75,9 @@ define arm_aapcs_vfpcc void @thresh_i16(i16* %data, i16 zeroext %N, i16 signext
 ; CHECK-NEXT:  .LBB1_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrh.u16 q1, [r0]
-; CHECK-NEXT:    vpt.s16 ge, q1, r2
+; CHECK-NEXT:    vpte.s16 ge, q1, r2
 ; CHECK-NEXT:    vcmpt.s16 le, q1, r1
-; CHECK-NEXT:    vpnot
-; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vstrht.16 q0, [r0], #16
+; CHECK-NEXT:    vstrhe.16 q0, [r0], #16
 ; CHECK-NEXT:    le lr, .LBB1_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
@@ -135,11 +131,9 @@ define arm_aapcs_vfpcc void @thresh_i8(i8* %data, i16 zeroext %N, i8 signext %T)
 ; CHECK-NEXT:  .LBB2_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrb.u8 q1, [r0]
-; CHECK-NEXT:    vpt.s8 ge, q1, r2
+; CHECK-NEXT:    vpte.s8 ge, q1, r2
 ; CHECK-NEXT:    vcmpt.s8 le, q1, r1
-; CHECK-NEXT:    vpnot
-; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vstrbt.8 q0, [r0], #16
+; CHECK-NEXT:    vstrbe.8 q0, [r0], #16
 ; CHECK-NEXT:    le lr, .LBB2_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
@@ -183,24 +177,21 @@ define arm_aapcs_vfpcc void @thresh_f32(float* %data, i16 zeroext %N, float %T)
 ; CHECK-NEXT:    cmp r1, #0
 ; CHECK-NEXT:    it eq
 ; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    vneg.f32 s2, s0
 ; CHECK-NEXT:    mvn r2, #3
 ; CHECK-NEXT:    add.w r1, r2, r1, lsl #2
 ; CHECK-NEXT:    movs r2, #1
-; CHECK-NEXT:    vmov.i32 q2, #0x0
 ; CHECK-NEXT:    add.w lr, r2, r1, lsr #2
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vdup.32 q0, r1
+; CHECK-NEXT:    vmov r2, s0
 ; CHECK-NEXT:    dls lr, lr
-; CHECK-NEXT:    eor r1, r1, #-2147483648
-; CHECK-NEXT:    vdup.32 q1, r1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:  .LBB3_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q3, [r0]
-; CHECK-NEXT:    vpt.f32 le, q0, q3
-; CHECK-NEXT:    vcmpt.f32 le, q3, q1
-; CHECK-NEXT:    vpnot
-; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vstrwt.32 q2, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vpte.f32 ge, q1, r2
+; CHECK-NEXT:    vcmpt.f32 le, q1, r1
+; CHECK-NEXT:    vstrwe.32 q0, [r0], #16
 ; CHECK-NEXT:    le lr, .LBB3_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
@@ -247,22 +238,18 @@ define arm_aapcs_vfpcc void @thresh_f16(half* %data, i16 zeroext %N, float %T.co
 ; CHECK-NEXT:    mvn r2, #7
 ; CHECK-NEXT:    add.w r1, r2, r1, lsl #3
 ; CHECK-NEXT:    movs r2, #1
-; CHECK-NEXT:    vmov.i32 q2, #0x0
+; CHECK-NEXT:    vneg.f16 s2, s0
 ; CHECK-NEXT:    add.w lr, r2, r1, lsr #3
-; CHECK-NEXT:    vmov.f16 r1, s0
-; CHECK-NEXT:    vneg.f16 s0, s0
-; CHECK-NEXT:    vdup.16 q1, r1
+; CHECK-NEXT:    vmov.f16 r1, s2
 ; CHECK-NEXT:    vmov.f16 r2, s0
+; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:    dls lr, lr
-; CHECK-NEXT:    vdup.16 q0, r2
 ; CHECK-NEXT:  .LBB4_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrh.u16 q3, [r0]
-; CHECK-NEXT:    vpt.f16 le, q1, q3
-; CHECK-NEXT:    vcmpt.f16 le, q3, q0
-; CHECK-NEXT:    vpnot
-; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vstrht.16 q2, [r0], #16
+; CHECK-NEXT:    vldrh.u16 q1, [r0]
+; CHECK-NEXT:    vpte.f16 ge, q1, r2
+; CHECK-NEXT:    vcmpt.f16 le, q1, r1
+; CHECK-NEXT:    vstrhe.16 q0, [r0], #16
 ; CHECK-NEXT:    le lr, .LBB4_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
@@ -321,11 +308,9 @@ define arm_aapcs_vfpcc void @thres_rev_i32(i32* %data, i16 zeroext %N, i32 %T) {
 ; CHECK-NEXT:  .LBB5_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrw.u32 q1, [r0]
-; CHECK-NEXT:    vpt.s32 ge, q1, r2
+; CHECK-NEXT:    vpte.s32 ge, q1, r2
 ; CHECK-NEXT:    vcmpt.s32 le, q1, r1
-; CHECK-NEXT:    vpnot
-; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vstrwt.32 q0, [r0], #16
+; CHECK-NEXT:    vstrwe.32 q0, [r0], #16
 ; CHECK-NEXT:    le lr, .LBB5_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
@@ -379,11 +364,9 @@ define arm_aapcs_vfpcc void @thresh_rev_i16(i16* %data, i16 zeroext %N, i16 sign
 ; CHECK-NEXT:  .LBB6_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrh.u16 q1, [r0]
-; CHECK-NEXT:    vpt.s16 ge, q1, r2
+; CHECK-NEXT:    vpte.s16 ge, q1, r2
 ; CHECK-NEXT:    vcmpt.s16 le, q1, r1
-; CHECK-NEXT:    vpnot
-; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vstrht.16 q0, [r0], #16
+; CHECK-NEXT:    vstrhe.16 q0, [r0], #16
 ; CHECK-NEXT:    le lr, .LBB6_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
@@ -437,11 +420,9 @@ define arm_aapcs_vfpcc void @thresh_rev_i8(i8* %data, i16 zeroext %N, i8 signext
 ; CHECK-NEXT:  .LBB7_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrb.u8 q1, [r0]
-; CHECK-NEXT:    vpt.s8 ge, q1, r2
+; CHECK-NEXT:    vpte.s8 ge, q1, r2
 ; CHECK-NEXT:    vcmpt.s8 le, q1, r1
-; CHECK-NEXT:    vpnot
-; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vstrbt.8 q0, [r0], #16
+; CHECK-NEXT:    vstrbe.8 q0, [r0], #16
 ; CHECK-NEXT:    le lr, .LBB7_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
@@ -485,24 +466,21 @@ define arm_aapcs_vfpcc void @thresh_rev_f32(float* %data, i16 zeroext %N, float
 ; CHECK-NEXT:    cmp r1, #0
 ; CHECK-NEXT:    it eq
 ; CHECK-NEXT:    popeq {r7, pc}
+; CHECK-NEXT:    vneg.f32 s2, s0
 ; CHECK-NEXT:    mvn r2, #3
 ; CHECK-NEXT:    add.w r1, r2, r1, lsl #2
 ; CHECK-NEXT:    movs r2, #1
-; CHECK-NEXT:    vmov.i32 q2, #0x0
 ; CHECK-NEXT:    add.w lr, r2, r1, lsr #2
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vdup.32 q0, r1
+; CHECK-NEXT:    vmov r2, s0
 ; CHECK-NEXT:    dls lr, lr
-; CHECK-NEXT:    eor r1, r1, #-2147483648
-; CHECK-NEXT:    vdup.32 q1, r1
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:  .LBB8_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q3, [r0]
-; CHECK-NEXT:    vpt.f32 le, q0, q3
-; CHECK-NEXT:    vcmpt.f32 le, q3, q1
-; CHECK-NEXT:    vpnot
-; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vstrwt.32 q2, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vpte.f32 ge, q1, r2
+; CHECK-NEXT:    vcmpt.f32 le, q1, r1
+; CHECK-NEXT:    vstrwe.32 q0, [r0], #16
 ; CHECK-NEXT:    le lr, .LBB8_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
@@ -549,22 +527,18 @@ define arm_aapcs_vfpcc void @thresh_rev_f16(half* %data, i16 zeroext %N, float %
 ; CHECK-NEXT:    mvn r2, #7
 ; CHECK-NEXT:    add.w r1, r2, r1, lsl #3
 ; CHECK-NEXT:    movs r2, #1
-; CHECK-NEXT:    vmov.i32 q2, #0x0
+; CHECK-NEXT:    vneg.f16 s2, s0
 ; CHECK-NEXT:    add.w lr, r2, r1, lsr #3
-; CHECK-NEXT:    vmov.f16 r1, s0
-; CHECK-NEXT:    vneg.f16 s0, s0
-; CHECK-NEXT:    vdup.16 q1, r1
+; CHECK-NEXT:    vmov.f16 r1, s2
 ; CHECK-NEXT:    vmov.f16 r2, s0
+; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:    dls lr, lr
-; CHECK-NEXT:    vdup.16 q0, r2
 ; CHECK-NEXT:  .LBB9_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrh.u16 q3, [r0]
-; CHECK-NEXT:    vpt.f16 le, q1, q3
-; CHECK-NEXT:    vcmpt.f16 le, q3, q0
-; CHECK-NEXT:    vpnot
-; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vstrht.16 q2, [r0], #16
+; CHECK-NEXT:    vldrh.u16 q1, [r0]
+; CHECK-NEXT:    vpte.f16 ge, q1, r2
+; CHECK-NEXT:    vcmpt.f16 le, q1, r1
+; CHECK-NEXT:    vstrhe.16 q0, [r0], #16
 ; CHECK-NEXT:    le lr, .LBB9_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
diff --git a/llvm/test/CodeGen/Thumb2/mve-sext.ll b/llvm/test/CodeGen/Thumb2/mve-sext.ll
index ffade88a94a0f..f351e6dc01f58 100644
--- a/llvm/test/CodeGen/Thumb2/mve-sext.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-sext.ll
@@ -430,17 +430,9 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @zext_v2i32_v2i64(<2 x i32> %src) {
 ; CHECK-LABEL: zext_v2i32_v2i64:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adr r0, .LCPI20_0
-; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vmov.i64 q1, #0xffffffff
 ; CHECK-NEXT:    vand q0, q0, q1
 ; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 4
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI20_0:
-; CHECK-NEXT:    .long 4294967295 @ 0xffffffff
-; CHECK-NEXT:    .long 0 @ 0x0
-; CHECK-NEXT:    .long 4294967295 @ 0xffffffff
-; CHECK-NEXT:    .long 0 @ 0x0
 entry:
   %0 = zext <2 x i32> %src to <2 x i64>
   ret <2 x i64> %0
diff --git a/llvm/test/CodeGen/Thumb2/mve-vcmpfz.ll b/llvm/test/CodeGen/Thumb2/mve-vcmpfz.ll
index 82ef5df349aaf..b6c28c3e1016c 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vcmpfz.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vcmpfz.ll
@@ -1010,7 +1010,7 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVEFP-LABEL: vcmp_one_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vpt.f16 ge, q0, zr
-; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f16 le, q0, zr
 ; CHECK-MVEFP-NEXT:    vpnot
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
@@ -1628,7 +1628,7 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVEFP-LABEL: vcmp_ueq_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vpt.f16 ge, q0, zr
-; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f16 le, q0, zr
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
@@ -2353,7 +2353,7 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVEFP-LABEL: vcmp_ord_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vpt.f16 ge, q0, zr
-; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f16 lt, q0, zr
 ; CHECK-MVEFP-NEXT:    vpnot
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
@@ -2475,7 +2475,7 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVEFP-LABEL: vcmp_uno_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vpt.f16 ge, q0, zr
-; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f16 lt, q0, zr
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
@@ -3495,7 +3495,7 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_r_one_v8f16(<8 x half> %src, <8 x half>
 ; CHECK-MVEFP-LABEL: vcmp_r_one_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vpt.f16 le, q0, zr
-; CHECK-MVEFP-NEXT:    vcmpt.f32 ge, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f16 ge, q0, zr
 ; CHECK-MVEFP-NEXT:    vpnot
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
@@ -4113,7 +4113,7 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_r_ueq_v8f16(<8 x half> %src, <8 x half>
 ; CHECK-MVEFP-LABEL: vcmp_r_ueq_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vpt.f16 le, q0, zr
-; CHECK-MVEFP-NEXT:    vcmpt.f32 ge, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f16 ge, q0, zr
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
@@ -4838,7 +4838,7 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_r_ord_v8f16(<8 x half> %src, <8 x half>
 ; CHECK-MVEFP-LABEL: vcmp_r_ord_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vpt.f16 le, q0, zr
-; CHECK-MVEFP-NEXT:    vcmpt.f32 gt, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f16 gt, q0, zr
 ; CHECK-MVEFP-NEXT:    vpnot
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
@@ -4960,7 +4960,7 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_r_uno_v8f16(<8 x half> %src, <8 x half>
 ; CHECK-MVEFP-LABEL: vcmp_r_uno_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vpt.f16 le, q0, zr
-; CHECK-MVEFP-NEXT:    vcmpt.f32 gt, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f16 gt, q0, zr
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
diff --git a/llvm/test/CodeGen/Thumb2/mve-vecreduce-add.ll b/llvm/test/CodeGen/Thumb2/mve-vecreduce-add.ll
index ced01f0606c7d..7ea29277cc465 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vecreduce-add.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vecreduce-add.ll
@@ -36,8 +36,7 @@ entry:
 define arm_aapcs_vfpcc i64 @add_v2i32_v2i64_zext(<2 x i32> %x) {
 ; CHECK-LABEL: add_v2i32_v2i64_zext:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adr r0, .LCPI3_0
-; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vmov.i64 q1, #0xffffffff
 ; CHECK-NEXT:    vand q0, q0, q1
 ; CHECK-NEXT:    vmov r0, s2
 ; CHECK-NEXT:    vmov r3, s0
@@ -46,13 +45,6 @@ define arm_aapcs_vfpcc i64 @add_v2i32_v2i64_zext(<2 x i32> %x) {
 ; CHECK-NEXT:    adds r0, r0, r3
 ; CHECK-NEXT:    adcs r1, r2
 ; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 4
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI3_0:
-; CHECK-NEXT:    .long 4294967295 @ 0xffffffff
-; CHECK-NEXT:    .long 0 @ 0x0
-; CHECK-NEXT:    .long 4294967295 @ 0xffffffff
-; CHECK-NEXT:    .long 0 @ 0x0
 entry:
   %xx = zext <2 x i32> %x to <2 x i64>
   %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %xx)
@@ -138,11 +130,10 @@ define arm_aapcs_vfpcc i64 @add_v8i16_v8i64_zext(<8 x i16> %x) {
 ; CHECK-LABEL: add_v8i16_v8i64_zext:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vmov.u16 r0, q0[0]
+; CHECK-NEXT:    vmov.i64 q1, #0xffff
 ; CHECK-NEXT:    vmov.32 q2[0], r0
 ; CHECK-NEXT:    vmov.u16 r0, q0[1]
 ; CHECK-NEXT:    vmov.32 q2[2], r0
-; CHECK-NEXT:    adr r0, .LCPI10_0
-; CHECK-NEXT:    vldrw.u32 q1, [r0]
 ; CHECK-NEXT:    vand q2, q2, q1
 ; CHECK-NEXT:    vmov r0, s10
 ; CHECK-NEXT:    vmov r1, s8
@@ -182,13 +173,6 @@ define arm_aapcs_vfpcc i64 @add_v8i16_v8i64_zext(<8 x i16> %x) {
 ; CHECK-NEXT:    adds r0, r0, r3
 ; CHECK-NEXT:    adcs r1, r2
 ; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 4
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI10_0:
-; CHECK-NEXT:    .long 65535 @ 0xffff
-; CHECK-NEXT:    .long 0 @ 0x0
-; CHECK-NEXT:    .long 65535 @ 0xffff
-; CHECK-NEXT:    .long 0 @ 0x0
 entry:
   %xx = zext <8 x i16> %x to <8 x i64>
   %z = call i64 @llvm.experimental.vector.reduce.add.v8i64(<8 x i64> %xx)
@@ -265,21 +249,13 @@ entry:
 define arm_aapcs_vfpcc i64 @add_v2i16_v2i64_zext(<2 x i16> %x) {
 ; CHECK-LABEL: add_v2i16_v2i64_zext:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adr r0, .LCPI12_0
-; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vmov.i64 q1, #0xffff
 ; CHECK-NEXT:    vand q0, q0, q1
 ; CHECK-NEXT:    vmov r0, s2
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    add r0, r1
 ; CHECK-NEXT:    vmov r1, s3
 ; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 4
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI12_0:
-; CHECK-NEXT:    .long 65535 @ 0xffff
-; CHECK-NEXT:    .long 0 @ 0x0
-; CHECK-NEXT:    .long 65535 @ 0xffff
-; CHECK-NEXT:    .long 0 @ 0x0
 entry:
   %xx = zext <2 x i16> %x to <2 x i64>
   %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %xx)
@@ -489,11 +465,10 @@ define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_zext(<16 x i8> %x) {
 ; CHECK-LABEL: add_v16i8_v16i64_zext:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vmov.u8 r0, q0[0]
+; CHECK-NEXT:    vmov.i64 q1, #0xff
 ; CHECK-NEXT:    vmov.32 q2[0], r0
 ; CHECK-NEXT:    vmov.u8 r0, q0[1]
 ; CHECK-NEXT:    vmov.32 q2[2], r0
-; CHECK-NEXT:    adr r0, .LCPI23_0
-; CHECK-NEXT:    vldrw.u32 q1, [r0]
 ; CHECK-NEXT:    vand q2, q2, q1
 ; CHECK-NEXT:    vmov r0, s10
 ; CHECK-NEXT:    vmov r1, s8
@@ -585,13 +560,6 @@ define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_zext(<16 x i8> %x) {
 ; CHECK-NEXT:    adds r0, r0, r3
 ; CHECK-NEXT:    adcs r1, r2
 ; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 4
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI23_0:
-; CHECK-NEXT:    .long 255 @ 0xff
-; CHECK-NEXT:    .long 0 @ 0x0
-; CHECK-NEXT:    .long 255 @ 0xff
-; CHECK-NEXT:    .long 0 @ 0x0
 entry:
   %xx = zext <16 x i8> %x to <16 x i64>
   %z = call i64 @llvm.experimental.vector.reduce.add.v16i64(<16 x i64> %xx)
@@ -736,21 +704,13 @@ entry:
 define arm_aapcs_vfpcc i64 @add_v2i8_v2i64_zext(<2 x i8> %x) {
 ; CHECK-LABEL: add_v2i8_v2i64_zext:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adr r0, .LCPI25_0
-; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vmov.i64 q1, #0xff
 ; CHECK-NEXT:    vand q0, q0, q1
 ; CHECK-NEXT:    vmov r0, s2
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    add r0, r1
 ; CHECK-NEXT:    vmov r1, s3
 ; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 4
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI25_0:
-; CHECK-NEXT:    .long 255 @ 0xff
-; CHECK-NEXT:    .long 0 @ 0x0
-; CHECK-NEXT:    .long 255 @ 0xff
-; CHECK-NEXT:    .long 0 @ 0x0
 entry:
   %xx = zext <2 x i8> %x to <2 x i64>
   %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %xx)
@@ -832,8 +792,7 @@ define arm_aapcs_vfpcc i64 @add_v2i32_v2i64_acc_zext(<2 x i32> %x, i64 %a) {
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    adr r2, .LCPI31_0
-; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vmov.i64 q1, #0xffffffff
 ; CHECK-NEXT:    vand q0, q0, q1
 ; CHECK-NEXT:    vmov r2, s2
 ; CHECK-NEXT:    vmov r3, s0
@@ -844,13 +803,6 @@ define arm_aapcs_vfpcc i64 @add_v2i32_v2i64_acc_zext(<2 x i32> %x, i64 %a) {
 ; CHECK-NEXT:    adds r0, r0, r2
 ; CHECK-NEXT:    adcs r1, r3
 ; CHECK-NEXT:    pop {r7, pc}
-; CHECK-NEXT:    .p2align 4
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI31_0:
-; CHECK-NEXT:    .long 4294967295 @ 0xffffffff
-; CHECK-NEXT:    .long 0 @ 0x0
-; CHECK-NEXT:    .long 4294967295 @ 0xffffffff
-; CHECK-NEXT:    .long 0 @ 0x0
 entry:
   %xx = zext <2 x i32> %x to <2 x i64>
   %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %xx)
@@ -947,11 +899,10 @@ define arm_aapcs_vfpcc i64 @add_v8i16_v8i64_acc_zext(<8 x i16> %x, i64 %a) {
 ; CHECK-NEXT:    .save {r4, lr}
 ; CHECK-NEXT:    push {r4, lr}
 ; CHECK-NEXT:    vmov.u16 r2, q0[0]
+; CHECK-NEXT:    vmov.i64 q1, #0xffff
 ; CHECK-NEXT:    vmov.32 q2[0], r2
 ; CHECK-NEXT:    vmov.u16 r2, q0[1]
 ; CHECK-NEXT:    vmov.32 q2[2], r2
-; CHECK-NEXT:    adr r2, .LCPI38_0
-; CHECK-NEXT:    vldrw.u32 q1, [r2]
 ; CHECK-NEXT:    vand q2, q2, q1
 ; CHECK-NEXT:    vmov r2, s10
 ; CHECK-NEXT:    vmov r3, s8
@@ -993,13 +944,6 @@ define arm_aapcs_vfpcc i64 @add_v8i16_v8i64_acc_zext(<8 x i16> %x, i64 %a) {
 ; CHECK-NEXT:    adds r0, r0, r2
 ; CHECK-NEXT:    adcs r1, r3
 ; CHECK-NEXT:    pop {r4, pc}
-; CHECK-NEXT:    .p2align 4
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI38_0:
-; CHECK-NEXT:    .long 65535 @ 0xffff
-; CHECK-NEXT:    .long 0 @ 0x0
-; CHECK-NEXT:    .long 65535 @ 0xffff
-; CHECK-NEXT:    .long 0 @ 0x0
 entry:
   %xx = zext <8 x i16> %x to <8 x i64>
   %z = call i64 @llvm.experimental.vector.reduce.add.v8i64(<8 x i64> %xx)
@@ -1082,8 +1026,7 @@ entry:
 define arm_aapcs_vfpcc i64 @add_v2i16_v2i64_acc_zext(<2 x i16> %x, i64 %a) {
 ; CHECK-LABEL: add_v2i16_v2i64_acc_zext:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adr r2, .LCPI40_0
-; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vmov.i64 q1, #0xffff
 ; CHECK-NEXT:    vand q0, q0, q1
 ; CHECK-NEXT:    vmov r2, s2
 ; CHECK-NEXT:    vmov r3, s0
@@ -1092,13 +1035,6 @@ define arm_aapcs_vfpcc i64 @add_v2i16_v2i64_acc_zext(<2 x i16> %x, i64 %a) {
 ; CHECK-NEXT:    adds r0, r0, r2
 ; CHECK-NEXT:    adcs r1, r3
 ; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 4
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI40_0:
-; CHECK-NEXT:    .long 65535 @ 0xffff
-; CHECK-NEXT:    .long 0 @ 0x0
-; CHECK-NEXT:    .long 65535 @ 0xffff
-; CHECK-NEXT:    .long 0 @ 0x0
 entry:
   %xx = zext <2 x i16> %x to <2 x i64>
   %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %xx)
@@ -1323,11 +1259,10 @@ define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_acc_zext(<16 x i8> %x, i64 %a) {
 ; CHECK-NEXT:    .save {r4, lr}
 ; CHECK-NEXT:    push {r4, lr}
 ; CHECK-NEXT:    vmov.u8 r2, q0[0]
+; CHECK-NEXT:    vmov.i64 q1, #0xff
 ; CHECK-NEXT:    vmov.32 q2[0], r2
 ; CHECK-NEXT:    vmov.u8 r2, q0[1]
 ; CHECK-NEXT:    vmov.32 q2[2], r2
-; CHECK-NEXT:    adr r2, .LCPI51_0
-; CHECK-NEXT:    vldrw.u32 q1, [r2]
 ; CHECK-NEXT:    vand q2, q2, q1
 ; CHECK-NEXT:    vmov r2, s10
 ; CHECK-NEXT:    vmov r3, s8
@@ -1421,13 +1356,6 @@ define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_acc_zext(<16 x i8> %x, i64 %a) {
 ; CHECK-NEXT:    adds r0, r0, r2
 ; CHECK-NEXT:    adcs r1, r3
 ; CHECK-NEXT:    pop {r4, pc}
-; CHECK-NEXT:    .p2align 4
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI51_0:
-; CHECK-NEXT:    .long 255 @ 0xff
-; CHECK-NEXT:    .long 0 @ 0x0
-; CHECK-NEXT:    .long 255 @ 0xff
-; CHECK-NEXT:    .long 0 @ 0x0
 entry:
   %xx = zext <16 x i8> %x to <16 x i64>
   %z = call i64 @llvm.experimental.vector.reduce.add.v16i64(<16 x i64> %xx)
@@ -1578,8 +1506,7 @@ entry:
 define arm_aapcs_vfpcc i64 @add_v2i8_v2i64_acc_zext(<2 x i8> %x, i64 %a) {
 ; CHECK-LABEL: add_v2i8_v2i64_acc_zext:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adr r2, .LCPI53_0
-; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    vmov.i64 q1, #0xff
 ; CHECK-NEXT:    vand q0, q0, q1
 ; CHECK-NEXT:    vmov r2, s2
 ; CHECK-NEXT:    vmov r3, s0
@@ -1588,13 +1515,6 @@ define arm_aapcs_vfpcc i64 @add_v2i8_v2i64_acc_zext(<2 x i8> %x, i64 %a) {
 ; CHECK-NEXT:    adds r0, r0, r2
 ; CHECK-NEXT:    adcs r1, r3
 ; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 4
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI53_0:
-; CHECK-NEXT:    .long 255 @ 0xff
-; CHECK-NEXT:    .long 0 @ 0x0
-; CHECK-NEXT:    .long 255 @ 0xff
-; CHECK-NEXT:    .long 0 @ 0x0
 entry:
   %xx = zext <2 x i8> %x to <2 x i64>
   %z = call i64 @llvm.experimental.vector.reduce.add.v2i64(<2 x i64> %xx)
diff --git a/llvm/test/CodeGen/Thumb2/mve-vecreduce-mla.ll b/llvm/test/CodeGen/Thumb2/mve-vecreduce-mla.ll
index 9b6d668f899e1..dea70df68773b 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vecreduce-mla.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vecreduce-mla.ll
@@ -41,12 +41,13 @@ entry:
 define arm_aapcs_vfpcc i64 @add_v2i32_v2i64_zext(<2 x i32> %x, <2 x i32> %y) {
 ; CHECK-LABEL: add_v2i32_v2i64_zext:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov r0, s4
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vmov r2, s6
-; CHECK-NEXT:    vmov r3, s2
-; CHECK-NEXT:    umull r0, r1, r1, r0
-; CHECK-NEXT:    umlal r0, r1, r3, r2
+; CHECK-NEXT:    vmullb.u32 q2, q0, q1
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    vmov r3, s8
+; CHECK-NEXT:    vmov r1, s11
+; CHECK-NEXT:    vmov r2, s9
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    adcs r1, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %xx = zext <2 x i32> %x to <2 x i64>
@@ -59,12 +60,13 @@ entry:
 define arm_aapcs_vfpcc i64 @add_v2i32_v2i64_sext(<2 x i32> %x, <2 x i32> %y) {
 ; CHECK-LABEL: add_v2i32_v2i64_sext:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov r0, s4
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    vmov r2, s6
-; CHECK-NEXT:    vmov r3, s2
-; CHECK-NEXT:    smull r0, r1, r1, r0
-; CHECK-NEXT:    smlal r0, r1, r3, r2
+; CHECK-NEXT:    vmullb.s32 q2, q0, q1
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    vmov r3, s8
+; CHECK-NEXT:    vmov r1, s11
+; CHECK-NEXT:    vmov r2, s9
+; CHECK-NEXT:    adds r0, r0, r3
+; CHECK-NEXT:    adcs r1, r2
 ; CHECK-NEXT:    bx lr
 entry:
   %xx = sext <2 x i32> %x to <2 x i64>
@@ -172,8 +174,7 @@ entry:
 define arm_aapcs_vfpcc i64 @add_v2i16_v2i64_zext(<2 x i16> %x, <2 x i16> %y) {
 ; CHECK-LABEL: add_v2i16_v2i64_zext:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adr r0, .LCPI12_0
-; CHECK-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-NEXT:    vmov.i64 q2, #0xffff
 ; CHECK-NEXT:    vand q1, q1, q2
 ; CHECK-NEXT:    vand q0, q0, q2
 ; CHECK-NEXT:    vmov r0, s4
@@ -183,13 +184,6 @@ define arm_aapcs_vfpcc i64 @add_v2i16_v2i64_zext(<2 x i16> %x, <2 x i16> %y) {
 ; CHECK-NEXT:    umull r0, r1, r1, r0
 ; CHECK-NEXT:    umlal r0, r1, r3, r2
 ; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 4
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI12_0:
-; CHECK-NEXT:    .long 65535 @ 0xffff
-; CHECK-NEXT:    .long 0 @ 0x0
-; CHECK-NEXT:    .long 65535 @ 0xffff
-; CHECK-NEXT:    .long 0 @ 0x0
 entry:
   %xx = zext <2 x i16> %x to <2 x i64>
   %yy = zext <2 x i16> %y to <2 x i64>
@@ -314,10 +308,8 @@ define arm_aapcs_vfpcc zeroext i16 @add_v16i8_v16i16_zext(<16 x i8> %x, <16 x i8
 ; CHECK-NEXT:    vmov.16 q3[6], r0
 ; CHECK-NEXT:    vmov.u8 r0, q0[15]
 ; CHECK-NEXT:    vmov.16 q3[7], r0
-; CHECK-NEXT:    vmovlb.u8 q2, q2
-; CHECK-NEXT:    vmovlb.u8 q3, q3
 ; CHECK-NEXT:    vmov.u8 r0, q1[0]
-; CHECK-NEXT:    vmul.i16 q2, q3, q2
+; CHECK-NEXT:    vmullb.u8 q2, q3, q2
 ; CHECK-NEXT:    vmov.16 q3[0], r0
 ; CHECK-NEXT:    vmov.u8 r0, q1[1]
 ; CHECK-NEXT:    vmov.16 q3[1], r0
@@ -334,24 +326,22 @@ define arm_aapcs_vfpcc zeroext i16 @add_v16i8_v16i16_zext(<16 x i8> %x, <16 x i8
 ; CHECK-NEXT:    vmov.u8 r0, q1[7]
 ; CHECK-NEXT:    vmov.16 q3[7], r0
 ; CHECK-NEXT:    vmov.u8 r0, q0[0]
-; CHECK-NEXT:    vmovlb.u8 q1, q3
-; CHECK-NEXT:    vmov.16 q3[0], r0
+; CHECK-NEXT:    vmov.16 q1[0], r0
 ; CHECK-NEXT:    vmov.u8 r0, q0[1]
-; CHECK-NEXT:    vmov.16 q3[1], r0
+; CHECK-NEXT:    vmov.16 q1[1], r0
 ; CHECK-NEXT:    vmov.u8 r0, q0[2]
-; CHECK-NEXT:    vmov.16 q3[2], r0
+; CHECK-NEXT:    vmov.16 q1[2], r0
 ; CHECK-NEXT:    vmov.u8 r0, q0[3]
-; CHECK-NEXT:    vmov.16 q3[3], r0
+; CHECK-NEXT:    vmov.16 q1[3], r0
 ; CHECK-NEXT:    vmov.u8 r0, q0[4]
-; CHECK-NEXT:    vmov.16 q3[4], r0
+; CHECK-NEXT:    vmov.16 q1[4], r0
 ; CHECK-NEXT:    vmov.u8 r0, q0[5]
-; CHECK-NEXT:    vmov.16 q3[5], r0
+; CHECK-NEXT:    vmov.16 q1[5], r0
 ; CHECK-NEXT:    vmov.u8 r0, q0[6]
-; CHECK-NEXT:    vmov.16 q3[6], r0
+; CHECK-NEXT:    vmov.16 q1[6], r0
 ; CHECK-NEXT:    vmov.u8 r0, q0[7]
-; CHECK-NEXT:    vmov.16 q3[7], r0
-; CHECK-NEXT:    vmovlb.u8 q0, q3
-; CHECK-NEXT:    vmul.i16 q0, q0, q1
+; CHECK-NEXT:    vmov.16 q1[7], r0
+; CHECK-NEXT:    vmullb.u8 q0, q1, q3
 ; CHECK-NEXT:    vadd.i16 q0, q0, q2
 ; CHECK-NEXT:    vaddv.u16 r0, q0
 ; CHECK-NEXT:    uxth r0, r0
@@ -399,10 +389,8 @@ define arm_aapcs_vfpcc signext i16 @add_v16i8_v16i16_sext(<16 x i8> %x, <16 x i8
 ; CHECK-NEXT:    vmov.16 q3[6], r0
 ; CHECK-NEXT:    vmov.u8 r0, q0[15]
 ; CHECK-NEXT:    vmov.16 q3[7], r0
-; CHECK-NEXT:    vmovlb.s8 q2, q2
-; CHECK-NEXT:    vmovlb.s8 q3, q3
 ; CHECK-NEXT:    vmov.u8 r0, q1[0]
-; CHECK-NEXT:    vmul.i16 q2, q3, q2
+; CHECK-NEXT:    vmullb.s8 q2, q3, q2
 ; CHECK-NEXT:    vmov.16 q3[0], r0
 ; CHECK-NEXT:    vmov.u8 r0, q1[1]
 ; CHECK-NEXT:    vmov.16 q3[1], r0
@@ -419,24 +407,22 @@ define arm_aapcs_vfpcc signext i16 @add_v16i8_v16i16_sext(<16 x i8> %x, <16 x i8
 ; CHECK-NEXT:    vmov.u8 r0, q1[7]
 ; CHECK-NEXT:    vmov.16 q3[7], r0
 ; CHECK-NEXT:    vmov.u8 r0, q0[0]
-; CHECK-NEXT:    vmovlb.s8 q1, q3
-; CHECK-NEXT:    vmov.16 q3[0], r0
+; CHECK-NEXT:    vmov.16 q1[0], r0
 ; CHECK-NEXT:    vmov.u8 r0, q0[1]
-; CHECK-NEXT:    vmov.16 q3[1], r0
+; CHECK-NEXT:    vmov.16 q1[1], r0
 ; CHECK-NEXT:    vmov.u8 r0, q0[2]
-; CHECK-NEXT:    vmov.16 q3[2], r0
+; CHECK-NEXT:    vmov.16 q1[2], r0
 ; CHECK-NEXT:    vmov.u8 r0, q0[3]
-; CHECK-NEXT:    vmov.16 q3[3], r0
+; CHECK-NEXT:    vmov.16 q1[3], r0
 ; CHECK-NEXT:    vmov.u8 r0, q0[4]
-; CHECK-NEXT:    vmov.16 q3[4], r0
+; CHECK-NEXT:    vmov.16 q1[4], r0
 ; CHECK-NEXT:    vmov.u8 r0, q0[5]
-; CHECK-NEXT:    vmov.16 q3[5], r0
+; CHECK-NEXT:    vmov.16 q1[5], r0
 ; CHECK-NEXT:    vmov.u8 r0, q0[6]
-; CHECK-NEXT:    vmov.16 q3[6], r0
+; CHECK-NEXT:    vmov.16 q1[6], r0
 ; CHECK-NEXT:    vmov.u8 r0, q0[7]
-; CHECK-NEXT:    vmov.16 q3[7], r0
-; CHECK-NEXT:    vmovlb.s8 q0, q3
-; CHECK-NEXT:    vmul.i16 q0, q0, q1
+; CHECK-NEXT:    vmov.16 q1[7], r0
+; CHECK-NEXT:    vmullb.s8 q0, q1, q3
 ; CHECK-NEXT:    vadd.i16 q0, q0, q2
 ; CHECK-NEXT:    vaddv.u16 r0, q0
 ; CHECK-NEXT:    sxth r0, r0
@@ -452,9 +438,7 @@ entry:
 define arm_aapcs_vfpcc zeroext i16 @add_v8i8_v8i16_zext(<8 x i8> %x, <8 x i8> %y) {
 ; CHECK-LABEL: add_v8i8_v8i16_zext:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovlb.u8 q1, q1
-; CHECK-NEXT:    vmovlb.u8 q0, q0
-; CHECK-NEXT:    vmul.i16 q0, q0, q1
+; CHECK-NEXT:    vmullb.u8 q0, q0, q1
 ; CHECK-NEXT:    vaddv.u16 r0, q0
 ; CHECK-NEXT:    uxth r0, r0
 ; CHECK-NEXT:    bx lr
@@ -469,9 +453,7 @@ entry:
 define arm_aapcs_vfpcc signext i16 @add_v8i8_v8i16_sext(<8 x i8> %x, <8 x i8> %y) {
 ; CHECK-LABEL: add_v8i8_v8i16_sext:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovlb.s8 q1, q1
-; CHECK-NEXT:    vmovlb.s8 q0, q0
-; CHECK-NEXT:    vmul.i16 q0, q0, q1
+; CHECK-NEXT:    vmullb.s8 q0, q0, q1
 ; CHECK-NEXT:    vaddv.u16 r0, q0
 ; CHECK-NEXT:    sxth r0, r0
 ; CHECK-NEXT:    bx lr
@@ -507,11 +489,10 @@ define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_zext(<16 x i8> %x, <16 x i8> %y) {
 ; CHECK-NEXT:    vmov.u8 r1, q0[0]
 ; CHECK-NEXT:    vmov.32 q3[0], r0
 ; CHECK-NEXT:    vmov.u8 r0, q1[1]
-; CHECK-NEXT:    vmov.32 q3[2], r0
-; CHECK-NEXT:    adr r0, .LCPI23_0
-; CHECK-NEXT:    vldrw.u32 q2, [r0]
 ; CHECK-NEXT:    vmov.32 q4[0], r1
 ; CHECK-NEXT:    vmov.u8 r1, q0[1]
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.i64 q2, #0xff
 ; CHECK-NEXT:    vmov.32 q4[2], r1
 ; CHECK-NEXT:    vand q3, q3, q2
 ; CHECK-NEXT:    vand q4, q4, q2
@@ -703,13 +684,6 @@ define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_zext(<16 x i8> %x, <16 x i8> %y) {
 ; CHECK-NEXT:    umlal r0, r1, r3, r2
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-NEXT:    pop {r7, pc}
-; CHECK-NEXT:    .p2align 4
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI23_0:
-; CHECK-NEXT:    .long 255 @ 0xff
-; CHECK-NEXT:    .long 0 @ 0x0
-; CHECK-NEXT:    .long 255 @ 0xff
-; CHECK-NEXT:    .long 0 @ 0x0
 entry:
   %xx = zext <16 x i8> %x to <16 x i64>
   %yy = zext <16 x i8> %y to <16 x i64>
@@ -888,8 +862,7 @@ entry:
 define arm_aapcs_vfpcc i64 @add_v2i8_v2i64_zext(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: add_v2i8_v2i64_zext:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adr r0, .LCPI25_0
-; CHECK-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-NEXT:    vmov.i64 q2, #0xff
 ; CHECK-NEXT:    vand q1, q1, q2
 ; CHECK-NEXT:    vand q0, q0, q2
 ; CHECK-NEXT:    vmov r0, s6
@@ -901,13 +874,6 @@ define arm_aapcs_vfpcc i64 @add_v2i8_v2i64_zext(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-NEXT:    add r0, r2
 ; CHECK-NEXT:    orrs r1, r3
 ; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 4
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI25_0:
-; CHECK-NEXT:    .long 255 @ 0xff
-; CHECK-NEXT:    .long 0 @ 0x0
-; CHECK-NEXT:    .long 255 @ 0xff
-; CHECK-NEXT:    .long 0 @ 0x0
 entry:
   %xx = zext <2 x i8> %x to <2 x i64>
   %yy = zext <2 x i8> %y to <2 x i64>
@@ -1014,14 +980,15 @@ define arm_aapcs_vfpcc i64 @add_v2i32_v2i64_acc_zext(<2 x i32> %x, <2 x i32> %y,
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    vmov r2, s4
-; CHECK-NEXT:    vmov r3, s0
-; CHECK-NEXT:    vmov r12, s6
-; CHECK-NEXT:    umull r2, lr, r3, r2
-; CHECK-NEXT:    vmov r3, s2
-; CHECK-NEXT:    umlal r2, lr, r3, r12
+; CHECK-NEXT:    vmullb.u32 q2, q0, q1
+; CHECK-NEXT:    vmov r2, s10
+; CHECK-NEXT:    vmov r3, s8
+; CHECK-NEXT:    vmov r12, s11
+; CHECK-NEXT:    vmov lr, s9
+; CHECK-NEXT:    adds r2, r2, r3
+; CHECK-NEXT:    adc.w r3, lr, r12
 ; CHECK-NEXT:    adds r0, r0, r2
-; CHECK-NEXT:    adc.w r1, r1, lr
+; CHECK-NEXT:    adcs r1, r3
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   %xx = zext <2 x i32> %x to <2 x i64>
@@ -1037,14 +1004,15 @@ define arm_aapcs_vfpcc i64 @add_v2i32_v2i64_acc_sext(<2 x i32> %x, <2 x i32> %y,
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    vmov r2, s4
-; CHECK-NEXT:    vmov r3, s0
-; CHECK-NEXT:    vmov r12, s6
-; CHECK-NEXT:    smull r2, lr, r3, r2
-; CHECK-NEXT:    vmov r3, s2
-; CHECK-NEXT:    smlal r2, lr, r3, r12
+; CHECK-NEXT:    vmullb.s32 q2, q0, q1
+; CHECK-NEXT:    vmov r2, s10
+; CHECK-NEXT:    vmov r3, s8
+; CHECK-NEXT:    vmov r12, s11
+; CHECK-NEXT:    vmov lr, s9
+; CHECK-NEXT:    adds r2, r2, r3
+; CHECK-NEXT:    adc.w r3, lr, r12
 ; CHECK-NEXT:    adds r0, r0, r2
-; CHECK-NEXT:    adc.w r1, r1, lr
+; CHECK-NEXT:    adcs r1, r3
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   %xx = sext <2 x i32> %x to <2 x i64>
@@ -1162,8 +1130,7 @@ define arm_aapcs_vfpcc i64 @add_v2i16_v2i64_acc_zext(<2 x i16> %x, <2 x i16> %y,
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    adr r2, .LCPI40_0
-; CHECK-NEXT:    vldrw.u32 q2, [r2]
+; CHECK-NEXT:    vmov.i64 q2, #0xffff
 ; CHECK-NEXT:    vand q1, q1, q2
 ; CHECK-NEXT:    vand q0, q0, q2
 ; CHECK-NEXT:    vmov r2, s4
@@ -1175,13 +1142,6 @@ define arm_aapcs_vfpcc i64 @add_v2i16_v2i64_acc_zext(<2 x i16> %x, <2 x i16> %y,
 ; CHECK-NEXT:    adds r0, r0, r2
 ; CHECK-NEXT:    adc.w r1, r1, lr
 ; CHECK-NEXT:    pop {r7, pc}
-; CHECK-NEXT:    .p2align 4
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI40_0:
-; CHECK-NEXT:    .long 65535 @ 0xffff
-; CHECK-NEXT:    .long 0 @ 0x0
-; CHECK-NEXT:    .long 65535 @ 0xffff
-; CHECK-NEXT:    .long 0 @ 0x0
 entry:
   %xx = zext <2 x i16> %x to <2 x i64>
   %yy = zext <2 x i16> %y to <2 x i64>
@@ -1316,10 +1276,8 @@ define arm_aapcs_vfpcc zeroext i16 @add_v16i8_v16i16_acc_zext(<16 x i8> %x, <16
 ; CHECK-NEXT:    vmov.16 q3[6], r1
 ; CHECK-NEXT:    vmov.u8 r1, q0[15]
 ; CHECK-NEXT:    vmov.16 q3[7], r1
-; CHECK-NEXT:    vmovlb.u8 q2, q2
-; CHECK-NEXT:    vmovlb.u8 q3, q3
 ; CHECK-NEXT:    vmov.u8 r1, q1[0]
-; CHECK-NEXT:    vmul.i16 q2, q3, q2
+; CHECK-NEXT:    vmullb.u8 q2, q3, q2
 ; CHECK-NEXT:    vmov.16 q3[0], r1
 ; CHECK-NEXT:    vmov.u8 r1, q1[1]
 ; CHECK-NEXT:    vmov.16 q3[1], r1
@@ -1336,24 +1294,22 @@ define arm_aapcs_vfpcc zeroext i16 @add_v16i8_v16i16_acc_zext(<16 x i8> %x, <16
 ; CHECK-NEXT:    vmov.u8 r1, q1[7]
 ; CHECK-NEXT:    vmov.16 q3[7], r1
 ; CHECK-NEXT:    vmov.u8 r1, q0[0]
-; CHECK-NEXT:    vmovlb.u8 q1, q3
-; CHECK-NEXT:    vmov.16 q3[0], r1
+; CHECK-NEXT:    vmov.16 q1[0], r1
 ; CHECK-NEXT:    vmov.u8 r1, q0[1]
-; CHECK-NEXT:    vmov.16 q3[1], r1
+; CHECK-NEXT:    vmov.16 q1[1], r1
 ; CHECK-NEXT:    vmov.u8 r1, q0[2]
-; CHECK-NEXT:    vmov.16 q3[2], r1
+; CHECK-NEXT:    vmov.16 q1[2], r1
 ; CHECK-NEXT:    vmov.u8 r1, q0[3]
-; CHECK-NEXT:    vmov.16 q3[3], r1
+; CHECK-NEXT:    vmov.16 q1[3], r1
 ; CHECK-NEXT:    vmov.u8 r1, q0[4]
-; CHECK-NEXT:    vmov.16 q3[4], r1
+; CHECK-NEXT:    vmov.16 q1[4], r1
 ; CHECK-NEXT:    vmov.u8 r1, q0[5]
-; CHECK-NEXT:    vmov.16 q3[5], r1
+; CHECK-NEXT:    vmov.16 q1[5], r1
 ; CHECK-NEXT:    vmov.u8 r1, q0[6]
-; CHECK-NEXT:    vmov.16 q3[6], r1
+; CHECK-NEXT:    vmov.16 q1[6], r1
 ; CHECK-NEXT:    vmov.u8 r1, q0[7]
-; CHECK-NEXT:    vmov.16 q3[7], r1
-; CHECK-NEXT:    vmovlb.u8 q0, q3
-; CHECK-NEXT:    vmul.i16 q0, q0, q1
+; CHECK-NEXT:    vmov.16 q1[7], r1
+; CHECK-NEXT:    vmullb.u8 q0, q1, q3
 ; CHECK-NEXT:    vadd.i16 q0, q0, q2
 ; CHECK-NEXT:    vaddva.u16 r0, q0
 ; CHECK-NEXT:    uxth r0, r0
@@ -1402,10 +1358,8 @@ define arm_aapcs_vfpcc signext i16 @add_v16i8_v16i16_acc_sext(<16 x i8> %x, <16
 ; CHECK-NEXT:    vmov.16 q3[6], r1
 ; CHECK-NEXT:    vmov.u8 r1, q0[15]
 ; CHECK-NEXT:    vmov.16 q3[7], r1
-; CHECK-NEXT:    vmovlb.s8 q2, q2
-; CHECK-NEXT:    vmovlb.s8 q3, q3
 ; CHECK-NEXT:    vmov.u8 r1, q1[0]
-; CHECK-NEXT:    vmul.i16 q2, q3, q2
+; CHECK-NEXT:    vmullb.s8 q2, q3, q2
 ; CHECK-NEXT:    vmov.16 q3[0], r1
 ; CHECK-NEXT:    vmov.u8 r1, q1[1]
 ; CHECK-NEXT:    vmov.16 q3[1], r1
@@ -1422,24 +1376,22 @@ define arm_aapcs_vfpcc signext i16 @add_v16i8_v16i16_acc_sext(<16 x i8> %x, <16
 ; CHECK-NEXT:    vmov.u8 r1, q1[7]
 ; CHECK-NEXT:    vmov.16 q3[7], r1
 ; CHECK-NEXT:    vmov.u8 r1, q0[0]
-; CHECK-NEXT:    vmovlb.s8 q1, q3
-; CHECK-NEXT:    vmov.16 q3[0], r1
+; CHECK-NEXT:    vmov.16 q1[0], r1
 ; CHECK-NEXT:    vmov.u8 r1, q0[1]
-; CHECK-NEXT:    vmov.16 q3[1], r1
+; CHECK-NEXT:    vmov.16 q1[1], r1
 ; CHECK-NEXT:    vmov.u8 r1, q0[2]
-; CHECK-NEXT:    vmov.16 q3[2], r1
+; CHECK-NEXT:    vmov.16 q1[2], r1
 ; CHECK-NEXT:    vmov.u8 r1, q0[3]
-; CHECK-NEXT:    vmov.16 q3[3], r1
+; CHECK-NEXT:    vmov.16 q1[3], r1
 ; CHECK-NEXT:    vmov.u8 r1, q0[4]
-; CHECK-NEXT:    vmov.16 q3[4], r1
+; CHECK-NEXT:    vmov.16 q1[4], r1
 ; CHECK-NEXT:    vmov.u8 r1, q0[5]
-; CHECK-NEXT:    vmov.16 q3[5], r1
+; CHECK-NEXT:    vmov.16 q1[5], r1
 ; CHECK-NEXT:    vmov.u8 r1, q0[6]
-; CHECK-NEXT:    vmov.16 q3[6], r1
+; CHECK-NEXT:    vmov.16 q1[6], r1
 ; CHECK-NEXT:    vmov.u8 r1, q0[7]
-; CHECK-NEXT:    vmov.16 q3[7], r1
-; CHECK-NEXT:    vmovlb.s8 q0, q3
-; CHECK-NEXT:    vmul.i16 q0, q0, q1
+; CHECK-NEXT:    vmov.16 q1[7], r1
+; CHECK-NEXT:    vmullb.s8 q0, q1, q3
 ; CHECK-NEXT:    vadd.i16 q0, q0, q2
 ; CHECK-NEXT:    vaddva.u16 r0, q0
 ; CHECK-NEXT:    sxth r0, r0
@@ -1456,9 +1408,7 @@ entry:
 define arm_aapcs_vfpcc zeroext i16 @add_v8i8_v8i16_acc_zext(<8 x i8> %x, <8 x i8> %y, i16 %a) {
 ; CHECK-LABEL: add_v8i8_v8i16_acc_zext:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovlb.u8 q1, q1
-; CHECK-NEXT:    vmovlb.u8 q0, q0
-; CHECK-NEXT:    vmul.i16 q0, q0, q1
+; CHECK-NEXT:    vmullb.u8 q0, q0, q1
 ; CHECK-NEXT:    vaddva.u16 r0, q0
 ; CHECK-NEXT:    uxth r0, r0
 ; CHECK-NEXT:    bx lr
@@ -1474,9 +1424,7 @@ entry:
 define arm_aapcs_vfpcc signext i16 @add_v8i8_v8i16_acc_sext(<8 x i8> %x, <8 x i8> %y, i16 %a) {
 ; CHECK-LABEL: add_v8i8_v8i16_acc_sext:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovlb.s8 q1, q1
-; CHECK-NEXT:    vmovlb.s8 q0, q0
-; CHECK-NEXT:    vmul.i16 q0, q0, q1
+; CHECK-NEXT:    vmullb.s8 q0, q0, q1
 ; CHECK-NEXT:    vaddva.u16 r0, q0
 ; CHECK-NEXT:    sxth r0, r0
 ; CHECK-NEXT:    bx lr
@@ -1514,17 +1462,16 @@ define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_acc_zext(<16 x i8> %x, <16 x i8> %y
 ; CHECK-NEXT:    vmov.u8 r3, q0[0]
 ; CHECK-NEXT:    vmov.32 q3[0], r2
 ; CHECK-NEXT:    vmov.u8 r2, q1[1]
-; CHECK-NEXT:    vmov.32 q3[2], r2
-; CHECK-NEXT:    adr r2, .LCPI51_0
-; CHECK-NEXT:    vldrw.u32 q2, [r2]
 ; CHECK-NEXT:    vmov.32 q4[0], r3
 ; CHECK-NEXT:    vmov.u8 r3, q0[1]
-; CHECK-NEXT:    vmov.u8 r4, q0[2]
+; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    vmov.i64 q2, #0xff
 ; CHECK-NEXT:    vmov.32 q4[2], r3
 ; CHECK-NEXT:    vand q3, q3, q2
 ; CHECK-NEXT:    vand q4, q4, q2
 ; CHECK-NEXT:    vmov r2, s14
 ; CHECK-NEXT:    vmov r3, s18
+; CHECK-NEXT:    vmov.u8 r4, q0[2]
 ; CHECK-NEXT:    umull r12, lr, r3, r2
 ; CHECK-NEXT:    vmov r3, s16
 ; CHECK-NEXT:    vmov r2, s12
@@ -1712,13 +1659,6 @@ define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_acc_zext(<16 x i8> %x, <16 x i8> %y
 ; CHECK-NEXT:    adcs r1, r3
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-NEXT:    pop {r4, r5, r7, pc}
-; CHECK-NEXT:    .p2align 4
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI51_0:
-; CHECK-NEXT:    .long 255 @ 0xff
-; CHECK-NEXT:    .long 0 @ 0x0
-; CHECK-NEXT:    .long 255 @ 0xff
-; CHECK-NEXT:    .long 0 @ 0x0
 entry:
   %xx = zext <16 x i8> %x to <16 x i64>
   %yy = zext <16 x i8> %y to <16 x i64>
@@ -1905,8 +1845,7 @@ define arm_aapcs_vfpcc i64 @add_v2i8_v2i64_acc_zext(<2 x i8> %x, <2 x i8> %y, i6
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    adr r2, .LCPI53_0
-; CHECK-NEXT:    vldrw.u32 q2, [r2]
+; CHECK-NEXT:    vmov.i64 q2, #0xff
 ; CHECK-NEXT:    vand q1, q1, q2
 ; CHECK-NEXT:    vand q0, q0, q2
 ; CHECK-NEXT:    vmov r2, s6
@@ -1920,13 +1859,6 @@ define arm_aapcs_vfpcc i64 @add_v2i8_v2i64_acc_zext(<2 x i8> %x, <2 x i8> %y, i6
 ; CHECK-NEXT:    adds r0, r0, r2
 ; CHECK-NEXT:    adcs r1, r3
 ; CHECK-NEXT:    pop {r7, pc}
-; CHECK-NEXT:    .p2align 4
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI53_0:
-; CHECK-NEXT:    .long 255 @ 0xff
-; CHECK-NEXT:    .long 0 @ 0x0
-; CHECK-NEXT:    .long 255 @ 0xff
-; CHECK-NEXT:    .long 0 @ 0x0
 entry:
   %xx = zext <2 x i8> %x to <2 x i64>
   %yy = zext <2 x i8> %y to <2 x i64>
diff --git a/llvm/test/CodeGen/Thumb2/mve-vld3.ll b/llvm/test/CodeGen/Thumb2/mve-vld3.ll
index c7f3c1d2f436c..3b49d77ba6ff6 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vld3.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vld3.ll
@@ -84,17 +84,17 @@ define void @vld3_v8i32(<24 x i32> *%src, <8 x i32> *%dst) {
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    vldrw.u32 q1, [r0, #48]
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #64]
 ; CHECK-NEXT:    vldrw.u32 q2, [r0, #80]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #64]
 ; CHECK-NEXT:    vmov.f32 s12, s5
-; CHECK-NEXT:    vmov.f32 s13, s0
 ; CHECK-NEXT:    vmov.32 r2, q2[2]
+; CHECK-NEXT:    vmov.f32 s13, s0
 ; CHECK-NEXT:    vdup.32 q4, r2
 ; CHECK-NEXT:    vmov.f32 s14, s3
-; CHECK-NEXT:    vmov.f32 s15, s19
 ; CHECK-NEXT:    vmov.32 r2, q2[1]
-; CHECK-NEXT:    vmov.f64 d8, d2
+; CHECK-NEXT:    vmov.f32 s15, s19
 ; CHECK-NEXT:    vdup.32 q5, r2
+; CHECK-NEXT:    vmov.f64 d8, d2
 ; CHECK-NEXT:    vmov.f32 s17, s7
 ; CHECK-NEXT:    vmov.f32 s18, s2
 ; CHECK-NEXT:    vmov.f32 s0, s6
@@ -108,6 +108,7 @@ define void @vld3_v8i32(<24 x i32> *%src, <8 x i32> *%dst) {
 ; CHECK-NEXT:    vadd.i32 q0, q3, q0
 ; CHECK-NEXT:    vldrw.u32 q3, [r0, #32]
 ; CHECK-NEXT:    vmov.f32 s16, s9
+; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
 ; CHECK-NEXT:    vmov.f32 s17, s4
 ; CHECK-NEXT:    vmov.32 r0, q3[2]
 ; CHECK-NEXT:    vdup.32 q5, r0
@@ -116,7 +117,6 @@ define void @vld3_v8i32(<24 x i32> *%src, <8 x i32> *%dst) {
 ; CHECK-NEXT:    vmov.32 r0, q3[1]
 ; CHECK-NEXT:    vmov.f64 d10, d4
 ; CHECK-NEXT:    vdup.32 q6, r0
-; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
 ; CHECK-NEXT:    vmov.f32 s21, s11
 ; CHECK-NEXT:    vmov.f32 s22, s6
 ; CHECK-NEXT:    vmov.f32 s4, s10
@@ -148,17 +148,17 @@ define void @vld3_v16i32(<48 x i32> *%src, <16 x i32> *%dst) {
 ; CHECK-NEXT:    .pad #16
 ; CHECK-NEXT:    sub sp, #16
 ; CHECK-NEXT:    vldrw.u32 q1, [r0, #48]
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #64]
 ; CHECK-NEXT:    vldrw.u32 q2, [r0, #80]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #64]
 ; CHECK-NEXT:    vmov.f32 s12, s5
-; CHECK-NEXT:    vmov.f32 s13, s0
 ; CHECK-NEXT:    vmov.32 r2, q2[2]
+; CHECK-NEXT:    vmov.f32 s13, s0
 ; CHECK-NEXT:    vdup.32 q4, r2
 ; CHECK-NEXT:    vmov.f32 s14, s3
-; CHECK-NEXT:    vmov.f32 s15, s19
 ; CHECK-NEXT:    vmov.32 r2, q2[1]
-; CHECK-NEXT:    vmov.f64 d8, d2
+; CHECK-NEXT:    vmov.f32 s15, s19
 ; CHECK-NEXT:    vdup.32 q5, r2
+; CHECK-NEXT:    vmov.f64 d8, d2
 ; CHECK-NEXT:    vmov.f32 s17, s7
 ; CHECK-NEXT:    vmov.f32 s18, s2
 ; CHECK-NEXT:    vmov.f32 s0, s6
@@ -181,6 +181,7 @@ define void @vld3_v16i32(<48 x i32> *%src, <16 x i32> *%dst) {
 ; CHECK-NEXT:    vmov.32 r2, q3[1]
 ; CHECK-NEXT:    vmov.f64 d10, d4
 ; CHECK-NEXT:    vdup.32 q6, r2
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #128]
 ; CHECK-NEXT:    vmov.f32 s21, s11
 ; CHECK-NEXT:    vmov.f32 s22, s6
 ; CHECK-NEXT:    vmov.f32 s4, s10
@@ -202,7 +203,6 @@ define void @vld3_v16i32(<48 x i32> *%src, <16 x i32> *%dst) {
 ; CHECK-NEXT:    vmov.32 r2, q4[1]
 ; CHECK-NEXT:    vmov.f64 d12, d6
 ; CHECK-NEXT:    vdup.32 q7, r2
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #128]
 ; CHECK-NEXT:    vmov.f32 s25, s15
 ; CHECK-NEXT:    vmov.f32 s26, s10
 ; CHECK-NEXT:    vmov.f32 s8, s14
@@ -1125,17 +1125,17 @@ define void @vld3_v8f32(<24 x float> *%src, <8 x float> *%dst) {
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
 ; CHECK-NEXT:    vldrw.u32 q1, [r0, #48]
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #64]
 ; CHECK-NEXT:    vldrw.u32 q2, [r0, #80]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #64]
 ; CHECK-NEXT:    vmov.f32 s12, s5
-; CHECK-NEXT:    vmov.f32 s13, s0
 ; CHECK-NEXT:    vmov.32 r2, q2[2]
+; CHECK-NEXT:    vmov.f32 s13, s0
 ; CHECK-NEXT:    vdup.32 q4, r2
 ; CHECK-NEXT:    vmov.f32 s14, s3
-; CHECK-NEXT:    vmov.f32 s15, s19
 ; CHECK-NEXT:    vmov.32 r2, q2[1]
-; CHECK-NEXT:    vmov.f64 d8, d2
+; CHECK-NEXT:    vmov.f32 s15, s19
 ; CHECK-NEXT:    vdup.32 q5, r2
+; CHECK-NEXT:    vmov.f64 d8, d2
 ; CHECK-NEXT:    vmov.f32 s17, s7
 ; CHECK-NEXT:    vmov.f32 s18, s2
 ; CHECK-NEXT:    vmov.f32 s0, s6
@@ -1149,6 +1149,7 @@ define void @vld3_v8f32(<24 x float> *%src, <8 x float> *%dst) {
 ; CHECK-NEXT:    vadd.f32 q0, q3, q0
 ; CHECK-NEXT:    vldrw.u32 q3, [r0, #32]
 ; CHECK-NEXT:    vmov.f32 s16, s9
+; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
 ; CHECK-NEXT:    vmov.f32 s17, s4
 ; CHECK-NEXT:    vmov.32 r0, q3[2]
 ; CHECK-NEXT:    vdup.32 q5, r0
@@ -1157,7 +1158,6 @@ define void @vld3_v8f32(<24 x float> *%src, <8 x float> *%dst) {
 ; CHECK-NEXT:    vmov.32 r0, q3[1]
 ; CHECK-NEXT:    vmov.f64 d10, d4
 ; CHECK-NEXT:    vdup.32 q6, r0
-; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
 ; CHECK-NEXT:    vmov.f32 s21, s11
 ; CHECK-NEXT:    vmov.f32 s22, s6
 ; CHECK-NEXT:    vmov.f32 s4, s10
@@ -1189,17 +1189,17 @@ define void @vld3_v16f32(<48 x float> *%src, <16 x float> *%dst) {
 ; CHECK-NEXT:    .pad #16
 ; CHECK-NEXT:    sub sp, #16
 ; CHECK-NEXT:    vldrw.u32 q1, [r0, #48]
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #64]
 ; CHECK-NEXT:    vldrw.u32 q2, [r0, #80]
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #64]
 ; CHECK-NEXT:    vmov.f32 s12, s5
-; CHECK-NEXT:    vmov.f32 s13, s0
 ; CHECK-NEXT:    vmov.32 r2, q2[2]
+; CHECK-NEXT:    vmov.f32 s13, s0
 ; CHECK-NEXT:    vdup.32 q4, r2
 ; CHECK-NEXT:    vmov.f32 s14, s3
-; CHECK-NEXT:    vmov.f32 s15, s19
 ; CHECK-NEXT:    vmov.32 r2, q2[1]
-; CHECK-NEXT:    vmov.f64 d8, d2
+; CHECK-NEXT:    vmov.f32 s15, s19
 ; CHECK-NEXT:    vdup.32 q5, r2
+; CHECK-NEXT:    vmov.f64 d8, d2
 ; CHECK-NEXT:    vmov.f32 s17, s7
 ; CHECK-NEXT:    vmov.f32 s18, s2
 ; CHECK-NEXT:    vmov.f32 s0, s6
@@ -1222,6 +1222,7 @@ define void @vld3_v16f32(<48 x float> *%src, <16 x float> *%dst) {
 ; CHECK-NEXT:    vmov.32 r2, q3[1]
 ; CHECK-NEXT:    vmov.f64 d10, d4
 ; CHECK-NEXT:    vdup.32 q6, r2
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #128]
 ; CHECK-NEXT:    vmov.f32 s21, s11
 ; CHECK-NEXT:    vmov.f32 s22, s6
 ; CHECK-NEXT:    vmov.f32 s4, s10
@@ -1243,7 +1244,6 @@ define void @vld3_v16f32(<48 x float> *%src, <16 x float> *%dst) {
 ; CHECK-NEXT:    vmov.32 r2, q4[1]
 ; CHECK-NEXT:    vmov.f64 d12, d6
 ; CHECK-NEXT:    vdup.32 q7, r2
-; CHECK-NEXT:    vldrw.u32 q0, [r0, #128]
 ; CHECK-NEXT:    vmov.f32 s25, s15
 ; CHECK-NEXT:    vmov.f32 s26, s10
 ; CHECK-NEXT:    vmov.f32 s8, s14
diff --git a/llvm/test/CodeGen/Thumb2/mve-vld4.ll b/llvm/test/CodeGen/Thumb2/mve-vld4.ll
index 41743220a81f7..9a1320986940b 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vld4.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vld4.ll
@@ -22,10 +22,10 @@ define void @vld4_v2i32(<8 x i32> *%src, <2 x i32> *%dst) {
 ; CHECK-NEXT:    add r2, r3
 ; CHECK-NEXT:    vmov r3, s0
 ; CHECK-NEXT:    add r0, r2
-; CHECK-NEXT:    str r0, [r1, #4]
 ; CHECK-NEXT:    vmov.32 r2, q0[2]
-; CHECK-NEXT:    vmov r0, s8
 ; CHECK-NEXT:    vdup.32 q1, r2
+; CHECK-NEXT:    str r0, [r1, #4]
+; CHECK-NEXT:    vmov r0, s8
 ; CHECK-NEXT:    vmov r2, s4
 ; CHECK-NEXT:    add r0, r2
 ; CHECK-NEXT:    vmov r2, s12
diff --git a/llvm/test/CodeGen/Thumb2/mve-vmovimm.ll b/llvm/test/CodeGen/Thumb2/mve-vmovimm.ll
index 94721a54b942e..aad885b9b18a3 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vmovimm.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vmovimm.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s
-; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp -verify-machineinstrs %s -o - | FileCheck %s
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECKLE
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECKLE
+; RUN: llc -mtriple=thumbebv8.1m.main-arm-none-eabi -mattr=+mve.fp -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECKBE
 
 define arm_aapcs_vfpcc <16 x i8> @mov_int8_1() {
 ; CHECK-LABEL: mov_int8_1:
@@ -57,18 +58,32 @@ entry:
 }
 
 define arm_aapcs_vfpcc <8 x i16> @mov_int16_258() {
-; CHECK-LABEL: mov_int16_258:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adr r0, .LCPI6_0
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 4
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI6_0:
-; CHECK-NEXT:    .long 16908546 @ double 8.204306265173532E-304
-; CHECK-NEXT:    .long 16908546
-; CHECK-NEXT:    .long 16908546 @ double 8.204306265173532E-304
-; CHECK-NEXT:    .long 16908546
+; CHECKLE-LABEL: mov_int16_258:
+; CHECKLE:       @ %bb.0: @ %entry
+; CHECKLE-NEXT:    adr r0, .LCPI6_0
+; CHECKLE-NEXT:    vldrw.u32 q0, [r0]
+; CHECKLE-NEXT:    bx lr
+; CHECKLE-NEXT:    .p2align 4
+; CHECKLE-NEXT:  @ %bb.1:
+; CHECKLE-NEXT:  .LCPI6_0:
+; CHECKLE-NEXT:    .long 16908546 @ double 8.204306265173532E-304
+; CHECKLE-NEXT:    .long 16908546
+; CHECKLE-NEXT:    .long 16908546 @ double 8.204306265173532E-304
+; CHECKLE-NEXT:    .long 16908546
+;
+; CHECKBE-LABEL: mov_int16_258:
+; CHECKBE:       @ %bb.0: @ %entry
+; CHECKBE-NEXT:    adr r0, .LCPI6_0
+; CHECKBE-NEXT:    vldrb.u8 q1, [r0]
+; CHECKBE-NEXT:    vrev64.8 q0, q1
+; CHECKBE-NEXT:    bx lr
+; CHECKBE-NEXT:    .p2align 4
+; CHECKBE-NEXT:  @ %bb.1:
+; CHECKBE-NEXT:  .LCPI6_0:
+; CHECKBE-NEXT:    .long 16908546 @ double 8.204306265173532E-304
+; CHECKBE-NEXT:    .long 16908546
+; CHECKBE-NEXT:    .long 16908546 @ double 8.204306265173532E-304
+; CHECKBE-NEXT:    .long 16908546
 entry:
   ret <8 x i16> <i16 258, i16 258, i16 258, i16 258, i16 258, i16 258, i16 258, i16 258>
 }
@@ -110,18 +125,32 @@ entry:
 }
 
 define arm_aapcs_vfpcc <4 x i32> @mov_int32_16777217() {
-; CHECK-LABEL: mov_int32_16777217:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adr r0, .LCPI11_0
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 4
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI11_0:
-; CHECK-NEXT:    .long 16777217 @ double 7.2911290000737531E-304
-; CHECK-NEXT:    .long 16777217
-; CHECK-NEXT:    .long 16777217 @ double 7.2911290000737531E-304
-; CHECK-NEXT:    .long 16777217
+; CHECKLE-LABEL: mov_int32_16777217:
+; CHECKLE:       @ %bb.0: @ %entry
+; CHECKLE-NEXT:    adr r0, .LCPI11_0
+; CHECKLE-NEXT:    vldrw.u32 q0, [r0]
+; CHECKLE-NEXT:    bx lr
+; CHECKLE-NEXT:    .p2align 4
+; CHECKLE-NEXT:  @ %bb.1:
+; CHECKLE-NEXT:  .LCPI11_0:
+; CHECKLE-NEXT:    .long 16777217 @ double 7.2911290000737531E-304
+; CHECKLE-NEXT:    .long 16777217
+; CHECKLE-NEXT:    .long 16777217 @ double 7.2911290000737531E-304
+; CHECKLE-NEXT:    .long 16777217
+;
+; CHECKBE-LABEL: mov_int32_16777217:
+; CHECKBE:       @ %bb.0: @ %entry
+; CHECKBE-NEXT:    adr r0, .LCPI11_0
+; CHECKBE-NEXT:    vldrb.u8 q1, [r0]
+; CHECKBE-NEXT:    vrev64.8 q0, q1
+; CHECKBE-NEXT:    bx lr
+; CHECKBE-NEXT:    .p2align 4
+; CHECKBE-NEXT:  @ %bb.1:
+; CHECKBE-NEXT:  .LCPI11_0:
+; CHECKBE-NEXT:    .long 16777217 @ double 7.2911290000737531E-304
+; CHECKBE-NEXT:    .long 16777217
+; CHECKBE-NEXT:    .long 16777217 @ double 7.2911290000737531E-304
+; CHECKBE-NEXT:    .long 16777217
 entry:
   ret <4 x i32> <i32 16777217, i32 16777217, i32 16777217, i32 16777217>
 }
@@ -163,18 +192,32 @@ entry:
 }
 
 define arm_aapcs_vfpcc <4 x i32> @mov_int32_4278190335() {
-; CHECK-LABEL: mov_int32_4278190335:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adr r0, .LCPI16_0
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 4
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI16_0:
-; CHECK-NEXT:    .long 4278190335 @ double -5.4874634341155774E+303
-; CHECK-NEXT:    .long 4278190335
-; CHECK-NEXT:    .long 4278190335 @ double -5.4874634341155774E+303
-; CHECK-NEXT:    .long 4278190335
+; CHECKLE-LABEL: mov_int32_4278190335:
+; CHECKLE:       @ %bb.0: @ %entry
+; CHECKLE-NEXT:    adr r0, .LCPI16_0
+; CHECKLE-NEXT:    vldrw.u32 q0, [r0]
+; CHECKLE-NEXT:    bx lr
+; CHECKLE-NEXT:    .p2align 4
+; CHECKLE-NEXT:  @ %bb.1:
+; CHECKLE-NEXT:  .LCPI16_0:
+; CHECKLE-NEXT:    .long 4278190335 @ double -5.4874634341155774E+303
+; CHECKLE-NEXT:    .long 4278190335
+; CHECKLE-NEXT:    .long 4278190335 @ double -5.4874634341155774E+303
+; CHECKLE-NEXT:    .long 4278190335
+;
+; CHECKBE-LABEL: mov_int32_4278190335:
+; CHECKBE:       @ %bb.0: @ %entry
+; CHECKBE-NEXT:    adr r0, .LCPI16_0
+; CHECKBE-NEXT:    vldrb.u8 q1, [r0]
+; CHECKBE-NEXT:    vrev64.8 q0, q1
+; CHECKBE-NEXT:    bx lr
+; CHECKBE-NEXT:    .p2align 4
+; CHECKBE-NEXT:  @ %bb.1:
+; CHECKBE-NEXT:  .LCPI16_0:
+; CHECKBE-NEXT:    .long 4278190335 @ double -5.4874634341155774E+303
+; CHECKBE-NEXT:    .long 4278190335
+; CHECKBE-NEXT:    .long 4278190335 @ double -5.4874634341155774E+303
+; CHECKBE-NEXT:    .long 4278190335
 entry:
   ret <4 x i32> <i32 4278190335, i32 4278190335, i32 4278190335, i32 4278190335>
 }
@@ -189,20 +232,43 @@ entry:
 }
 
 define arm_aapcs_vfpcc <2 x i64> @mov_int64_1() {
-; CHECK-LABEL: mov_int64_1:
+; CHECKLE-LABEL: mov_int64_1:
+; CHECKLE:       @ %bb.0: @ %entry
+; CHECKLE-NEXT:    adr r0, .LCPI18_0
+; CHECKLE-NEXT:    vldrw.u32 q0, [r0]
+; CHECKLE-NEXT:    bx lr
+; CHECKLE-NEXT:    .p2align 4
+; CHECKLE-NEXT:  @ %bb.1:
+; CHECKLE-NEXT:  .LCPI18_0:
+; CHECKLE-NEXT:    .long 1 @ double 4.9406564584124654E-324
+; CHECKLE-NEXT:    .long 0
+; CHECKLE-NEXT:    .long 1 @ double 4.9406564584124654E-324
+; CHECKLE-NEXT:    .long 0
+;
+; CHECKBE-LABEL: mov_int64_1:
+; CHECKBE:       @ %bb.0: @ %entry
+; CHECKBE-NEXT:    adr r0, .LCPI18_0
+; CHECKBE-NEXT:    vldrb.u8 q1, [r0]
+; CHECKBE-NEXT:    vrev64.8 q0, q1
+; CHECKBE-NEXT:    bx lr
+; CHECKBE-NEXT:    .p2align 4
+; CHECKBE-NEXT:  @ %bb.1:
+; CHECKBE-NEXT:  .LCPI18_0:
+; CHECKBE-NEXT:    .long 0 @ double 4.9406564584124654E-324
+; CHECKBE-NEXT:    .long 1
+; CHECKBE-NEXT:    .long 0 @ double 4.9406564584124654E-324
+; CHECKBE-NEXT:    .long 1
+entry:
+  ret <2 x i64> <i64 1, i64 1>
+}
+
+define arm_aapcs_vfpcc <2 x i64> @mov_int64_ff() {
+; CHECK-LABEL: mov_int64_ff:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adr r0, .LCPI18_0
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vmov.i64 q0, #0xff
 ; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 4
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI18_0:
-; CHECK-NEXT:    .long 1 @ double 4.9406564584124654E-324
-; CHECK-NEXT:    .long 0
-; CHECK-NEXT:    .long 1 @ double 4.9406564584124654E-324
-; CHECK-NEXT:    .long 0
 entry:
-  ret <2 x i64> <i64 1, i64 1>
+  ret <2 x i64> < i64 255, i64 255 >
 }
 
 define arm_aapcs_vfpcc <2 x i64> @mov_int64_m1() {
@@ -211,39 +277,149 @@ define arm_aapcs_vfpcc <2 x i64> @mov_int64_m1() {
 ; CHECK-NEXT:    vmov.i8 q0, #0xff
 ; CHECK-NEXT:    bx lr
 entry:
-  ret <2 x i64> <i64 -1, i64 -1>
+  ret <2 x i64> < i64 -1, i64 -1 >
 }
 
-define arm_aapcs_vfpcc <4 x float> @mov_float_1() {
-; CHECK-LABEL: mov_float_1:
+define arm_aapcs_vfpcc <2 x i64> @mov_int64_ff0000ff0000ffff() {
+; CHECK-LABEL: mov_int64_ff0000ff0000ffff:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adr r0, .LCPI20_0
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vmov.i64 q0, #0xff0000ff0000ffff
 ; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 4
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI20_0:
-; CHECK-NEXT:    .long 1065353216 @ double 0.007812501848093234
-; CHECK-NEXT:    .long 1065353216
-; CHECK-NEXT:    .long 1065353216 @ double 0.007812501848093234
-; CHECK-NEXT:    .long 1065353216
+entry:
+  ret <2 x i64> < i64 18374687574888349695, i64 18374687574888349695 >
+}
+
+define arm_aapcs_vfpcc <2 x i64> @mov_int64_f_0() {
+; CHECKLE-LABEL: mov_int64_f_0:
+; CHECKLE:       @ %bb.0: @ %entry
+; CHECKLE-NEXT:    adr r0, .LCPI22_0
+; CHECKLE-NEXT:    vldrw.u32 q0, [r0]
+; CHECKLE-NEXT:    bx lr
+; CHECKLE-NEXT:    .p2align 4
+; CHECKLE-NEXT:  @ %bb.1:
+; CHECKLE-NEXT:  .LCPI22_0:
+; CHECKLE-NEXT:    .long 255 @ double 1.2598673968951787E-321
+; CHECKLE-NEXT:    .long 0
+; CHECKLE-NEXT:    .long 0 @ double 0
+; CHECKLE-NEXT:    .long 0
+;
+; CHECKBE-LABEL: mov_int64_f_0:
+; CHECKBE:       @ %bb.0: @ %entry
+; CHECKBE-NEXT:    adr r0, .LCPI22_0
+; CHECKBE-NEXT:    vldrb.u8 q1, [r0]
+; CHECKBE-NEXT:    vrev64.8 q0, q1
+; CHECKBE-NEXT:    bx lr
+; CHECKBE-NEXT:    .p2align 4
+; CHECKBE-NEXT:  @ %bb.1:
+; CHECKBE-NEXT:  .LCPI22_0:
+; CHECKBE-NEXT:    .long 0 @ double 1.2598673968951787E-321
+; CHECKBE-NEXT:    .long 255
+; CHECKBE-NEXT:    .long 0 @ double 0
+; CHECKBE-NEXT:    .long 0
+entry:
+  ret <2 x i64> < i64 255, i64 0 >
+}
+
+define arm_aapcs_vfpcc <16 x i8> @mov_int64_0f000f0f() {
+; CHECKLE-LABEL: mov_int64_0f000f0f:
+; CHECKLE:       @ %bb.0: @ %entry
+; CHECKLE-NEXT:    vmov.i64 q0, #0xff000000ff00ff
+; CHECKLE-NEXT:    bx lr
+;
+; CHECKBE-LABEL: mov_int64_0f000f0f:
+; CHECKBE:       @ %bb.0: @ %entry
+; CHECKBE-NEXT:    vmov.i64 q0, #0xff00ff000000ff00
+; CHECKBE-NEXT:    bx lr
+entry:
+  ret <16 x i8> <i8 -1, i8 0, i8 -1, i8 0, i8 0, i8 0, i8 -1, i8 0, i8 -1, i8 0, i8 -1, i8 0, i8 0, i8 0, i8 -1, i8 0>
+}
+
+define arm_aapcs_vfpcc <8 x i16> @mov_int64_ff00ffff() {
+; CHECKLE-LABEL: mov_int64_ff00ffff:
+; CHECKLE:       @ %bb.0: @ %entry
+; CHECKLE-NEXT:    vmov.i64 q0, #0xffffffff0000ffff
+; CHECKLE-NEXT:    bx lr
+;
+; CHECKBE-LABEL: mov_int64_ff00ffff:
+; CHECKBE:       @ %bb.0: @ %entry
+; CHECKBE-NEXT:    vmov.i64 q0, #0xffff0000ffffffff
+; CHECKBE-NEXT:    bx lr
+entry:
+  ret <8 x i16> <i16 -1, i16 0, i16 -1, i16 -1, i16 -1, i16 0, i16 -1, i16 -1>
+}
+
+define arm_aapcs_vfpcc <16 x i8> @mov_int64_0f0f0f0f0f0f0f0f() {
+; CHECKLE-LABEL: mov_int64_0f0f0f0f0f0f0f0f:
+; CHECKLE:       @ %bb.0: @ %entry
+; CHECKLE-NEXT:    vmov.i16 q0, #0xff
+; CHECKLE-NEXT:    bx lr
+;
+; CHECKBE-LABEL: mov_int64_0f0f0f0f0f0f0f0f:
+; CHECKBE:       @ %bb.0: @ %entry
+; CHECKBE-NEXT:    vmov.i16 q0, #0xff00
+; CHECKBE-NEXT:    bx lr
+entry:
+  ret <16 x i8> <i8 -1, i8 0, i8 -1, i8 0, i8 -1, i8 0, i8 -1, i8 0, i8 -1, i8 0, i8 -1, i8 0, i8 -1, i8 0, i8 -1, i8 0>
+}
+
+define arm_aapcs_vfpcc <4 x float> @mov_float_1() {
+; CHECKLE-LABEL: mov_float_1:
+; CHECKLE:       @ %bb.0: @ %entry
+; CHECKLE-NEXT:    adr r0, .LCPI26_0
+; CHECKLE-NEXT:    vldrw.u32 q0, [r0]
+; CHECKLE-NEXT:    bx lr
+; CHECKLE-NEXT:    .p2align 4
+; CHECKLE-NEXT:  @ %bb.1:
+; CHECKLE-NEXT:  .LCPI26_0:
+; CHECKLE-NEXT:    .long 1065353216 @ double 0.007812501848093234
+; CHECKLE-NEXT:    .long 1065353216
+; CHECKLE-NEXT:    .long 1065353216 @ double 0.007812501848093234
+; CHECKLE-NEXT:    .long 1065353216
+;
+; CHECKBE-LABEL: mov_float_1:
+; CHECKBE:       @ %bb.0: @ %entry
+; CHECKBE-NEXT:    adr r0, .LCPI26_0
+; CHECKBE-NEXT:    vldrb.u8 q1, [r0]
+; CHECKBE-NEXT:    vrev64.8 q0, q1
+; CHECKBE-NEXT:    bx lr
+; CHECKBE-NEXT:    .p2align 4
+; CHECKBE-NEXT:  @ %bb.1:
+; CHECKBE-NEXT:  .LCPI26_0:
+; CHECKBE-NEXT:    .long 1065353216 @ double 0.007812501848093234
+; CHECKBE-NEXT:    .long 1065353216
+; CHECKBE-NEXT:    .long 1065353216 @ double 0.007812501848093234
+; CHECKBE-NEXT:    .long 1065353216
 entry:
   ret <4 x float> <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
 }
 
 define arm_aapcs_vfpcc <4 x float> @mov_float_m3() {
-; CHECK-LABEL: mov_float_m3:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adr r0, .LCPI21_0
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 4
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI21_0:
-; CHECK-NEXT:    .long 3225419776 @ double -32.000022917985916
-; CHECK-NEXT:    .long 3225419776
-; CHECK-NEXT:    .long 3225419776 @ double -32.000022917985916
-; CHECK-NEXT:    .long 3225419776
+; CHECKLE-LABEL: mov_float_m3:
+; CHECKLE:       @ %bb.0: @ %entry
+; CHECKLE-NEXT:    adr r0, .LCPI27_0
+; CHECKLE-NEXT:    vldrw.u32 q0, [r0]
+; CHECKLE-NEXT:    bx lr
+; CHECKLE-NEXT:    .p2align 4
+; CHECKLE-NEXT:  @ %bb.1:
+; CHECKLE-NEXT:  .LCPI27_0:
+; CHECKLE-NEXT:    .long 3225419776 @ double -32.000022917985916
+; CHECKLE-NEXT:    .long 3225419776
+; CHECKLE-NEXT:    .long 3225419776 @ double -32.000022917985916
+; CHECKLE-NEXT:    .long 3225419776
+;
+; CHECKBE-LABEL: mov_float_m3:
+; CHECKBE:       @ %bb.0: @ %entry
+; CHECKBE-NEXT:    adr r0, .LCPI27_0
+; CHECKBE-NEXT:    vldrb.u8 q1, [r0]
+; CHECKBE-NEXT:    vrev64.8 q0, q1
+; CHECKBE-NEXT:    bx lr
+; CHECKBE-NEXT:    .p2align 4
+; CHECKBE-NEXT:  @ %bb.1:
+; CHECKBE-NEXT:  .LCPI27_0:
+; CHECKBE-NEXT:    .long 3225419776 @ double -32.000022917985916
+; CHECKBE-NEXT:    .long 3225419776
+; CHECKBE-NEXT:    .long 3225419776 @ double -32.000022917985916
+; CHECKBE-NEXT:    .long 3225419776
 entry:
   ret <4 x float> <float -3.000000e+00, float -3.000000e+00, float -3.000000e+00, float -3.000000e+00>
 }
@@ -269,18 +445,72 @@ entry:
 }
 
 define arm_aapcs_vfpcc <2 x double> @mov_double_1() {
-; CHECK-LABEL: mov_double_1:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adr r0, .LCPI24_0
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    bx lr
-; CHECK-NEXT:    .p2align 4
-; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI24_0:
-; CHECK-NEXT:    .long 0 @ double 1
-; CHECK-NEXT:    .long 1072693248
-; CHECK-NEXT:    .long 0 @ double 1
-; CHECK-NEXT:    .long 1072693248
+; CHECKLE-LABEL: mov_double_1:
+; CHECKLE:       @ %bb.0: @ %entry
+; CHECKLE-NEXT:    adr r0, .LCPI30_0
+; CHECKLE-NEXT:    vldrw.u32 q0, [r0]
+; CHECKLE-NEXT:    bx lr
+; CHECKLE-NEXT:    .p2align 4
+; CHECKLE-NEXT:  @ %bb.1:
+; CHECKLE-NEXT:  .LCPI30_0:
+; CHECKLE-NEXT:    .long 0 @ double 1
+; CHECKLE-NEXT:    .long 1072693248
+; CHECKLE-NEXT:    .long 0 @ double 1
+; CHECKLE-NEXT:    .long 1072693248
+;
+; CHECKBE-LABEL: mov_double_1:
+; CHECKBE:       @ %bb.0: @ %entry
+; CHECKBE-NEXT:    adr r0, .LCPI30_0
+; CHECKBE-NEXT:    vldrb.u8 q1, [r0]
+; CHECKBE-NEXT:    vrev64.8 q0, q1
+; CHECKBE-NEXT:    bx lr
+; CHECKBE-NEXT:    .p2align 4
+; CHECKBE-NEXT:  @ %bb.1:
+; CHECKBE-NEXT:  .LCPI30_0:
+; CHECKBE-NEXT:    .long 1072693248 @ double 1
+; CHECKBE-NEXT:    .long 0
+; CHECKBE-NEXT:    .long 1072693248 @ double 1
+; CHECKBE-NEXT:    .long 0
 entry:
   ret <2 x double> <double 1.000000e+00, double 1.000000e+00>
 }
+
+define arm_aapcs_vfpcc <16 x i8> @test(<16 x i8> %i) {
+; CHECKLE-LABEL: test:
+; CHECKLE:       @ %bb.0: @ %entry
+; CHECKLE-NEXT:    vmov.i64 q1, #0xff000000ff00ff
+; CHECKLE-NEXT:    vorr q0, q0, q1
+; CHECKLE-NEXT:    bx lr
+;
+; CHECKBE-LABEL: test:
+; CHECKBE:       @ %bb.0: @ %entry
+; CHECKBE-NEXT:    vmov.i64 q1, #0xff00ff000000ff00
+; CHECKBE-NEXT:    vrev64.8 q2, q1
+; CHECKBE-NEXT:    vrev64.8 q1, q0
+; CHECKBE-NEXT:    vorr q1, q1, q2
+; CHECKBE-NEXT:    vrev64.8 q0, q1
+; CHECKBE-NEXT:    bx lr
+entry:
+  %o = or <16 x i8> %i, <i8 -1, i8 0, i8 -1, i8 0, i8 0, i8 0, i8 -1, i8 0, i8 -1, i8 0, i8 -1, i8 0, i8 0, i8 0, i8 -1, i8 0>
+  ret <16 x i8> %o
+}
+
+define arm_aapcs_vfpcc <8 x i16> @test2(<8 x i16> %i) {
+; CHECKLE-LABEL: test2:
+; CHECKLE:       @ %bb.0: @ %entry
+; CHECKLE-NEXT:    vmov.i64 q1, #0xffffffff0000ffff
+; CHECKLE-NEXT:    vorr q0, q0, q1
+; CHECKLE-NEXT:    bx lr
+;
+; CHECKBE-LABEL: test2:
+; CHECKBE:       @ %bb.0: @ %entry
+; CHECKBE-NEXT:    vmov.i64 q1, #0xffff0000ffffffff
+; CHECKBE-NEXT:    vrev64.16 q2, q1
+; CHECKBE-NEXT:    vrev64.16 q1, q0
+; CHECKBE-NEXT:    vorr q1, q1, q2
+; CHECKBE-NEXT:    vrev64.16 q0, q1
+; CHECKBE-NEXT:    bx lr
+entry:
+  %o = or <8 x i16> %i, <i16 -1, i16 0, i16 -1, i16 -1, i16 -1, i16 0, i16 -1, i16 -1>
+  ret <8 x i16> %o
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-vmull-loop.ll b/llvm/test/CodeGen/Thumb2/mve-vmull-loop.ll
new file mode 100644
index 0000000000000..927a91233d5b3
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vmull-loop.ll
@@ -0,0 +1,207 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK
+
+define arm_aapcs_vfpcc void @test32(i32* noalias nocapture readonly %x, i32* noalias nocapture readonly %y, i32* nocapture %z, i32 %n) {
+; CHECK-LABEL: test32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r5, lr}
+; CHECK-NEXT:    push {r5, lr}
+; CHECK-NEXT:    cmp r3, #1
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    poplt {r5, pc}
+; CHECK-NEXT:  .LBB0_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
+; CHECK-NEXT:    vldrw.u32 q2, [r1], #16
+; CHECK-NEXT:    subs r3, #4
+; CHECK-NEXT:    vmullt.s32 q3, q2, q1
+; CHECK-NEXT:    vmov r5, s13
+; CHECK-NEXT:    vmov r12, s12
+; CHECK-NEXT:    lsrl r12, r5, #31
+; CHECK-NEXT:    vmov.32 q0[0], r12
+; CHECK-NEXT:    vmov r12, s14
+; CHECK-NEXT:    vmov.32 q0[1], r5
+; CHECK-NEXT:    vmov r5, s15
+; CHECK-NEXT:    lsrl r12, r5, #31
+; CHECK-NEXT:    vmullb.s32 q3, q2, q1
+; CHECK-NEXT:    vmov.32 q0[2], r12
+; CHECK-NEXT:    vmov r12, s12
+; CHECK-NEXT:    vmov.32 q0[3], r5
+; CHECK-NEXT:    vmov r5, s13
+; CHECK-NEXT:    lsrl r12, r5, #31
+; CHECK-NEXT:    vmov.32 q1[0], r12
+; CHECK-NEXT:    vmov r12, s14
+; CHECK-NEXT:    vmov.32 q1[1], r5
+; CHECK-NEXT:    vmov r5, s15
+; CHECK-NEXT:    lsrl r12, r5, #31
+; CHECK-NEXT:    vmov.32 q1[2], r12
+; CHECK-NEXT:    vmov.32 q1[3], r5
+; CHECK-NEXT:    vmov.f32 s8, s6
+; CHECK-NEXT:    vmov.f32 s9, s7
+; CHECK-NEXT:    vmov.f32 s6, s0
+; CHECK-NEXT:    vmov.f32 s7, s1
+; CHECK-NEXT:    vmov.f32 s10, s2
+; CHECK-NEXT:    vmov.f32 s5, s6
+; CHECK-NEXT:    vmov.f32 s11, s3
+; CHECK-NEXT:    vmov.f32 s6, s8
+; CHECK-NEXT:    vmov.f32 s7, s10
+; CHECK-NEXT:    vstrb.8 q1, [r2], #16
+; CHECK-NEXT:    bne .LBB0_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r5, pc}
+entry:
+  %0 = and i32 %n, 3
+  %cmp = icmp eq i32 %0, 0
+  %cmp113 = icmp sgt i32 %n, 0
+  br i1 %cmp113, label %vector.body, label %for.cond.cleanup
+
+vector.body:                                      ; preds = %vector.body, %entry
+  %index = phi i32 [ %index.next, %vector.body ], [ 0, %entry ]
+  %1 = getelementptr inbounds i32, i32* %x, i32 %index
+  %2 = bitcast i32* %1 to <4 x i32>*
+  %wide.load = load <4 x i32>, <4 x i32>* %2, align 4
+  %3 = shufflevector <4 x i32> %wide.load, <4 x i32> %wide.load, <2 x i32> <i32 0, i32 2>
+  %4 = shufflevector <4 x i32> %wide.load, <4 x i32> %wide.load, <2 x i32> <i32 1, i32 3>
+  %5 = sext <2 x i32> %3 to <2 x i64>
+  %6 = sext <2 x i32> %4 to <2 x i64>
+  %7 = getelementptr inbounds i32, i32* %y, i32 %index
+  %8 = bitcast i32* %7 to <4 x i32>*
+  %wide.load15 = load <4 x i32>, <4 x i32>* %8, align 4
+  %9 = shufflevector <4 x i32> %wide.load15, <4 x i32> %wide.load15, <2 x i32> <i32 0, i32 2>
+  %10 = shufflevector <4 x i32> %wide.load15, <4 x i32> %wide.load15, <2 x i32> <i32 1, i32 3>
+  %11 = sext <2 x i32> %9 to <2 x i64>
+  %12 = sext <2 x i32> %10 to <2 x i64>
+  %13 = mul <2 x i64> %11, %5
+  %14 = mul <2 x i64> %12, %6
+  %15 = lshr <2 x i64> %13, <i64 31, i64 31>
+  %16 = lshr <2 x i64> %14, <i64 31, i64 31>
+  %17 = shufflevector <2 x i64> %15, <2 x i64> %16, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+  %18 = trunc <4 x i64> %17 to <4 x i32>
+  %19 = getelementptr inbounds i32, i32* %z, i32 %index
+  %20 = bitcast i32* %19 to <4 x i32>*
+  store <4 x i32> %18, <4 x i32>* %20, align 4
+  %index.next = add i32 %index, 4
+  %21 = icmp eq i32 %index.next, %n
+  br i1 %21, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test16(i16* noalias nocapture readonly %x, i16* noalias nocapture readonly %y, i16* nocapture %z, i32 %n) {
+; CHECK-LABEL: test16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r3, #1
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    bxlt lr
+; CHECK-NEXT:  .LBB1_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrh.u16 q0, [r0], #16
+; CHECK-NEXT:    vldrh.u16 q1, [r1], #16
+; CHECK-NEXT:    subs r3, #8
+; CHECK-NEXT:    vmullt.s16 q2, q1, q0
+; CHECK-NEXT:    vmullb.s16 q0, q1, q0
+; CHECK-NEXT:    vshr.u32 q2, q2, #15
+; CHECK-NEXT:    vshr.u32 q0, q0, #15
+; CHECK-NEXT:    vmovnt.i32 q0, q2
+; CHECK-NEXT:    vstrb.8 q0, [r2], #16
+; CHECK-NEXT:    bne .LBB1_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = and i32 %n, 7
+  %cmp = icmp eq i32 %0, 0
+  %cmp113 = icmp sgt i32 %n, 0
+  br i1 %cmp113, label %vector.body, label %for.cond.cleanup
+
+vector.body:                                      ; preds = %vector.body, %entry
+  %index = phi i32 [ %index.next, %vector.body ], [ 0, %entry ]
+  %1 = getelementptr inbounds i16, i16* %x, i32 %index
+  %2 = bitcast i16* %1 to <8 x i16>*
+  %wide.load = load <8 x i16>, <8 x i16>* %2, align 2
+  %3 = shufflevector <8 x i16> %wide.load, <8 x i16> %wide.load, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %4 = shufflevector <8 x i16> %wide.load, <8 x i16> %wide.load, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %5 = sext <4 x i16> %3 to <4 x i32>
+  %6 = sext <4 x i16> %4 to <4 x i32>
+  %7 = getelementptr inbounds i16, i16* %y, i32 %index
+  %8 = bitcast i16* %7 to <8 x i16>*
+  %wide.load15 = load <8 x i16>, <8 x i16>* %8, align 2
+  %9 = shufflevector <8 x i16> %wide.load15, <8 x i16> %wide.load15, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %10 = shufflevector <8 x i16> %wide.load15, <8 x i16> %wide.load15, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %11 = sext <4 x i16> %9 to <4 x i32>
+  %12 = sext <4 x i16> %10 to <4 x i32>
+  %13 = mul <4 x i32> %11, %5
+  %14 = mul <4 x i32> %12, %6
+  %15 = lshr <4 x i32> %13, <i32 15, i32 15, i32 15, i32 15>
+  %16 = lshr <4 x i32> %14, <i32 15, i32 15, i32 15, i32 15>
+  %17 = shufflevector <4 x i32> %15, <4 x i32> %16, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+  %18 = trunc <8 x i32> %17 to <8 x i16>
+  %19 = getelementptr inbounds i16, i16* %z, i32 %index
+  %20 = bitcast i16* %19 to <8 x i16>*
+  store <8 x i16> %18, <8 x i16>* %20, align 2
+  %index.next = add i32 %index, 8
+  %21 = icmp eq i32 %index.next, %n
+  br i1 %21, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test8(i8* noalias nocapture readonly %x, i8* noalias nocapture readonly %y, i8* nocapture %z, i32 %n) {
+; CHECK-LABEL: test8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r3, #1
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    bxlt lr
+; CHECK-NEXT:  .LBB2_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrb.u8 q0, [r0], #16
+; CHECK-NEXT:    vldrb.u8 q1, [r1], #16
+; CHECK-NEXT:    subs r3, #16
+; CHECK-NEXT:    vmullt.u8 q2, q1, q0
+; CHECK-NEXT:    vmullb.u8 q0, q1, q0
+; CHECK-NEXT:    vshr.u16 q2, q2, #7
+; CHECK-NEXT:    vshr.u16 q0, q0, #7
+; CHECK-NEXT:    vmovnt.i16 q0, q2
+; CHECK-NEXT:    vstrb.8 q0, [r2], #16
+; CHECK-NEXT:    bne .LBB2_1
+; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = and i32 %n, 15
+  %cmp = icmp eq i32 %0, 0
+  %cmp117 = icmp sgt i32 %n, 0
+  br i1 %cmp117, label %vector.body, label %for.cond.cleanup
+
+vector.body:                                      ; preds = %vector.body, %entry
+  %index = phi i32 [ %index.next, %vector.body ], [ 0, %entry ]
+  %1 = getelementptr inbounds i8, i8* %x, i32 %index
+  %2 = bitcast i8* %1 to <16 x i8>*
+  %wide.load = load <16 x i8>, <16 x i8>* %2, align 1
+  %3 = shufflevector <16 x i8> %wide.load, <16 x i8> %wide.load, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  %4 = shufflevector <16 x i8> %wide.load, <16 x i8> %wide.load, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+  %5 = zext <8 x i8> %3 to <8 x i16>
+  %6 = zext <8 x i8> %4 to <8 x i16>
+  %7 = getelementptr inbounds i8, i8* %y, i32 %index
+  %8 = bitcast i8* %7 to <16 x i8>*
+  %wide.load19 = load <16 x i8>, <16 x i8>* %8, align 1
+  %9 = shufflevector <16 x i8> %wide.load19, <16 x i8> %wide.load19, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  %10 = shufflevector <16 x i8> %wide.load19, <16 x i8> %wide.load19, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+  %11 = zext <8 x i8> %9 to <8 x i16>
+  %12 = zext <8 x i8> %10 to <8 x i16>
+  %13 = mul <8 x i16> %11, %5
+  %14 = mul <8 x i16> %12, %6
+  %15 = lshr <8 x i16> %13, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
+  %16 = lshr <8 x i16> %14, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
+  %17 = shufflevector <8 x i16> %15, <8 x i16> %16, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+  %18 = trunc <16 x i16> %17 to <16 x i8>
+  %19 = getelementptr inbounds i8, i8* %z, i32 %index
+  %20 = bitcast i8* %19 to <16 x i8>*
+  store <16 x i8> %18, <16 x i8>* %20, align 1
+  %index.next = add i32 %index, 16
+  %21 = icmp eq i32 %index.next, %n
+  br i1 %21, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-vmull.ll b/llvm/test/CodeGen/Thumb2/mve-vmull.ll
index c1e720fa3ab6c..6d0b6622b8a43 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vmull.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vmull.ll
@@ -4,16 +4,7 @@
 define arm_aapcs_vfpcc <2 x i64> @sext_02(<4 x i32> %src1, <4 x i32> %src2) {
 ; CHECK-LABEL: sext_02:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov r0, s4
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    smull r0, r1, r1, r0
-; CHECK-NEXT:    vmov.32 q2[0], r0
-; CHECK-NEXT:    vmov r0, s6
-; CHECK-NEXT:    vmov.32 q2[1], r1
-; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    smull r0, r1, r1, r0
-; CHECK-NEXT:    vmov.32 q2[2], r0
-; CHECK-NEXT:    vmov.32 q2[3], r1
+; CHECK-NEXT:    vmullb.s32 q2, q0, q1
 ; CHECK-NEXT:    vmov q0, q2
 ; CHECK-NEXT:    bx lr
 entry:
@@ -28,18 +19,8 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @sext_13(<4 x i32> %src1, <4 x i32> %src2) {
 ; CHECK-LABEL: sext_13:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vrev64.32 q2, q1
-; CHECK-NEXT:    vrev64.32 q1, q0
-; CHECK-NEXT:    vmov r0, s8
-; CHECK-NEXT:    vmov r1, s4
-; CHECK-NEXT:    smull r0, r1, r1, r0
-; CHECK-NEXT:    vmov.32 q0[0], r0
-; CHECK-NEXT:    vmov r0, s10
-; CHECK-NEXT:    vmov.32 q0[1], r1
-; CHECK-NEXT:    vmov r1, s6
-; CHECK-NEXT:    smull r0, r1, r1, r0
-; CHECK-NEXT:    vmov.32 q0[2], r0
-; CHECK-NEXT:    vmov.32 q0[3], r1
+; CHECK-NEXT:    vmullt.s32 q2, q0, q1
+; CHECK-NEXT:    vmov q0, q2
 ; CHECK-NEXT:    bx lr
 entry:
   %shuf1 = shufflevector <4 x i32> %src1, <4 x i32> undef, <2 x i32> <i32 1, i32 3>
@@ -53,16 +34,7 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @zext_02(<4 x i32> %src1, <4 x i32> %src2) {
 ; CHECK-LABEL: zext_02:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov r0, s4
-; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    umull r0, r1, r1, r0
-; CHECK-NEXT:    vmov.32 q2[0], r0
-; CHECK-NEXT:    vmov r0, s6
-; CHECK-NEXT:    vmov.32 q2[1], r1
-; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    umull r0, r1, r1, r0
-; CHECK-NEXT:    vmov.32 q2[2], r0
-; CHECK-NEXT:    vmov.32 q2[3], r1
+; CHECK-NEXT:    vmullb.u32 q2, q0, q1
 ; CHECK-NEXT:    vmov q0, q2
 ; CHECK-NEXT:    bx lr
 entry:
@@ -77,18 +49,8 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @zext_13(<4 x i32> %src1, <4 x i32> %src2) {
 ; CHECK-LABEL: zext_13:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vrev64.32 q2, q1
-; CHECK-NEXT:    vrev64.32 q1, q0
-; CHECK-NEXT:    vmov r0, s8
-; CHECK-NEXT:    vmov r1, s4
-; CHECK-NEXT:    umull r0, r1, r1, r0
-; CHECK-NEXT:    vmov.32 q0[0], r0
-; CHECK-NEXT:    vmov r0, s10
-; CHECK-NEXT:    vmov.32 q0[1], r1
-; CHECK-NEXT:    vmov r1, s6
-; CHECK-NEXT:    umull r0, r1, r1, r0
-; CHECK-NEXT:    vmov.32 q0[2], r0
-; CHECK-NEXT:    vmov.32 q0[3], r1
+; CHECK-NEXT:    vmullt.u32 q2, q0, q1
+; CHECK-NEXT:    vmov q0, q2
 ; CHECK-NEXT:    bx lr
 entry:
   %shuf1 = shufflevector <4 x i32> %src1, <4 x i32> undef, <2 x i32> <i32 1, i32 3>
@@ -103,9 +65,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @sext_0246(<8 x i16> %src1, <8 x i16> %src2) {
 ; CHECK-LABEL: sext_0246:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovlb.s16 q1, q1
-; CHECK-NEXT:    vmovlb.s16 q0, q0
-; CHECK-NEXT:    vmul.i32 q0, q0, q1
+; CHECK-NEXT:    vmullb.s16 q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %shuf1 = shufflevector <8 x i16> %src1, <8 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
@@ -119,9 +79,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @sext_1357(<8 x i16> %src1, <8 x i16> %src2) {
 ; CHECK-LABEL: sext_1357:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovlt.s16 q1, q1
-; CHECK-NEXT:    vmovlt.s16 q0, q0
-; CHECK-NEXT:    vmul.i32 q0, q0, q1
+; CHECK-NEXT:    vmullt.s16 q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %shuf1 = shufflevector <8 x i16> %src1, <8 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
@@ -135,9 +93,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @zext_0246(<8 x i16> %src1, <8 x i16> %src2) {
 ; CHECK-LABEL: zext_0246:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovlb.u16 q1, q1
-; CHECK-NEXT:    vmovlb.u16 q0, q0
-; CHECK-NEXT:    vmul.i32 q0, q0, q1
+; CHECK-NEXT:    vmullb.u16 q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %shuf1 = shufflevector <8 x i16> %src1, <8 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
@@ -151,9 +107,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @zext_1357(<8 x i16> %src1, <8 x i16> %src2) {
 ; CHECK-LABEL: zext_1357:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovlt.u16 q1, q1
-; CHECK-NEXT:    vmovlt.u16 q0, q0
-; CHECK-NEXT:    vmul.i32 q0, q0, q1
+; CHECK-NEXT:    vmullt.u16 q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %shuf1 = shufflevector <8 x i16> %src1, <8 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
@@ -167,9 +121,7 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @sext_02468101214(<16 x i8> %src1, <16 x i8> %src2) {
 ; CHECK-LABEL: sext_02468101214:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovlb.s8 q1, q1
-; CHECK-NEXT:    vmovlb.s8 q0, q0
-; CHECK-NEXT:    vmul.i16 q0, q0, q1
+; CHECK-NEXT:    vmullb.s8 q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %shuf1 = shufflevector <16 x i8> %src1, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
@@ -183,9 +135,7 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @sext_13579111315(<16 x i8> %src1, <16 x i8> %src2) {
 ; CHECK-LABEL: sext_13579111315:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovlt.s8 q1, q1
-; CHECK-NEXT:    vmovlt.s8 q0, q0
-; CHECK-NEXT:    vmul.i16 q0, q0, q1
+; CHECK-NEXT:    vmullt.s8 q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %shuf1 = shufflevector <16 x i8> %src1, <16 x i8> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
@@ -199,9 +149,7 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @zext_02468101214(<16 x i8> %src1, <16 x i8> %src2) {
 ; CHECK-LABEL: zext_02468101214:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovlb.u8 q1, q1
-; CHECK-NEXT:    vmovlb.u8 q0, q0
-; CHECK-NEXT:    vmul.i16 q0, q0, q1
+; CHECK-NEXT:    vmullb.u8 q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %shuf1 = shufflevector <16 x i8> %src1, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
@@ -215,9 +163,7 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @zext_13579111315(<16 x i8> %src1, <16 x i8> %src2) {
 ; CHECK-LABEL: zext_13579111315:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmovlt.u8 q1, q1
-; CHECK-NEXT:    vmovlt.u8 q0, q0
-; CHECK-NEXT:    vmul.i16 q0, q0, q1
+; CHECK-NEXT:    vmullt.u8 q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %shuf1 = shufflevector <16 x i8> %src1, <16 x i8> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-3-blocks-kill-vpr.mir b/llvm/test/CodeGen/Thumb2/mve-vpt-3-blocks-kill-vpr.mir
index 347c4870ab6a5..7401d771d1b62 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vpt-3-blocks-kill-vpr.mir
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-3-blocks-kill-vpr.mir
@@ -68,14 +68,10 @@ body:             |
     ; CHECK: liveins: $q0, $q1, $q2, $r0
     ; CHECK: $vpr = VMSR_P0 killed $r0, 14 /* CC::al */, $noreg
     ; CHECK: $q3 = MVE_VORR $q0, $q0, 0, $noreg, undef $q3
-    ; CHECK: BUNDLE implicit-def $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit $vpr, implicit killed $q1, implicit $q2, implicit killed $q3 {
-    ; CHECK:   MVE_VPST 8, implicit $vpr
+    ; CHECK: BUNDLE implicit-def dead $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit-def $vpr, implicit-def $q1, implicit-def $d2, implicit-def $s4, implicit-def $s5, implicit-def $d3, implicit-def $s6, implicit-def $s7, implicit killed $vpr, implicit killed $q1, implicit $q2, implicit killed $q3 {
+    ; CHECK:   MVE_VPST 12, implicit $vpr
     ; CHECK:   renamable $q3 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q1, renamable $q2, 1, renamable $vpr, killed renamable $q3
-    ; CHECK: }
-    ; CHECK: renamable $vpr = MVE_VPNOT killed renamable $vpr, 0, $noreg
-    ; CHECK: BUNDLE implicit-def $q1, implicit-def $d2, implicit-def $s4, implicit-def $s5, implicit-def $d3, implicit-def $s6, implicit-def $s7, implicit $vpr, implicit killed $q3, implicit undef $q1 {
-    ; CHECK:   MVE_VPST 8, implicit $vpr
-    ; CHECK:   renamable $q1 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q3, renamable $q3, 1, renamable $vpr, undef renamable $q1
+    ; CHECK:   renamable $q1 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q3, internal renamable $q3, 2, internal renamable $vpr, undef renamable $q1
     ; CHECK: }
     ; CHECK: $q3 = MVE_VORR $q0, $q0, 0, $noreg, undef $q3
     ; CHECK: BUNDLE implicit-def dead $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $s1, implicit-def $d1, implicit-def $s2, implicit-def $s3, implicit killed $vpr, implicit killed $q1, implicit killed $q2, implicit killed $q3, implicit killed $q0 {
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-block-elses.mir b/llvm/test/CodeGen/Thumb2/mve-vpt-block-elses.mir
new file mode 100644
index 0000000000000..765d3a4de831a
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-block-elses.mir
@@ -0,0 +1,231 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -run-pass arm-mve-vpt %s -o - | FileCheck %s
+
+--- |
+  target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+  target triple = "thumbv8.1m.main-arm-none-eabi"
+
+  define hidden arm_aapcs_vfpcc <4 x float> @vpt_block_else(<4 x float> %inactive1, <4 x float> %inactive2, <4 x float> %a, <4 x float> %b, i16 zeroext %p) local_unnamed_addr #0 {
+  entry:
+    %conv.i = zext i16 %p to i32
+    %0 = tail call nnan ninf nsz <4 x float> @llvm.arm.mve.vminnm.m.v4f32.v4f32.v4f32.v4f32.i32(<4 x float> undef, <4 x float> %a, <4 x float> %b, i32 %conv.i) #2
+    %1 = tail call nnan ninf nsz <4 x float> @llvm.arm.mve.vminnm.m.v4f32.v4f32.v4f32.v4f32.i32(<4 x float> undef, <4 x float> %0, <4 x float> %0, i32 %conv.i) #2
+    %2 = tail call nnan ninf nsz <4 x float> @llvm.arm.mve.vminnm.m.v4f32.v4f32.v4f32.v4f32.i32(<4 x float> %inactive1, <4 x float> %1, <4 x float> %b, i32 %conv.i) #2
+    %3 = tail call nnan ninf nsz <4 x float> @llvm.arm.mve.vminnm.m.v4f32.v4f32.v4f32.v4f32.i32(<4 x float> %inactive2, <4 x float> %2, <4 x float> %b, i32 %conv.i) #2
+    ret <4 x float> %3
+  }
+
+  declare <4 x float> @llvm.arm.mve.vminnm.m.v4f32.v4f32.v4f32.v4f32.i32(<4 x float>, <4 x float>, <4 x float>, i32) #1
+
+  attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="preserve-sign" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="128" "frame-pointer"="none" "no-infs-fp-math"="true" "no-jump-tables"="false" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+armv8.1-m.main,+hwdiv,+mve.fp,+ras,+thumb-mode" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #1 = { nounwind readnone }
+  attributes #2 = { nounwind }
+
+...
+---
+name:            vpt_block_else
+alignment:       4
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+failedISel:      false
+tracksRegLiveness: true
+hasWinCFI:       false
+registers:       []
+liveins:
+  - { reg: '$q0', virtual-reg: '' }
+  - { reg: '$q1', virtual-reg: '' }
+  - { reg: '$q2', virtual-reg: '' }
+  - { reg: '$q3', virtual-reg: '' }
+  - { reg: '$r0', virtual-reg: '' }
+frameInfo:
+  isFrameAddressTaken: false
+  isReturnAddressTaken: false
+  hasStackMap:     false
+  hasPatchPoint:   false
+  stackSize:       0
+  offsetAdjustment: 0
+  maxAlignment:    0
+  adjustsStack:    false
+  hasCalls:        false
+  stackProtector:  ''
+  maxCallFrameSize: 0
+  cvBytesOfCalleeSavedRegisters: 0
+  hasOpaqueSPAdjustment: false
+  hasVAStart:      false
+  hasMustTailInVarArgFunc: false
+  localFrameSize:  0
+  savePoint:       ''
+  restorePoint:    ''
+fixedStack:      []
+stack:           []
+constants:       []
+body:             |
+  bb.0.entry:
+    liveins: $q0, $q1, $q2
+
+    ; CHECK-LABEL: name: vpt_block_else
+    ; CHECK: liveins: $q0, $q1, $q2
+    ; CHECK: $q3 = MVE_VORR $q2, $q2, 0, $noreg, undef $q3
+    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit killed $q0, implicit $q2, implicit $q1, implicit killed $q3, implicit $zr {
+    ; CHECK:   MVE_VPTv4s32 5, renamable $q0, renamable $q2, 10, implicit-def $vpr
+    ; CHECK:   renamable $q3 = MVE_VMAXs32 renamable $q0, renamable $q1, 1, internal renamable $vpr, killed renamable $q3
+    ; CHECK:   renamable $vpr = MVE_VCMPs32r killed renamable $q0, $zr, 12, 1, internal killed renamable $vpr
+    ; CHECK:   renamable $vpr = MVE_VCMPs32r renamable $q1, $zr, 12, 2, internal killed renamable $vpr
+    ; CHECK:   renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 1, internal killed renamable $vpr, internal renamable $q3
+    ; CHECK: }
+    ; CHECK: $q0 = MVE_VORR $q3, $q3, 0, $noreg, undef $q0
+    ; CHECK: $q3 = MVE_VORR $q2, $q2, 0, $noreg, undef $q3
+    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit killed $q0, implicit $q2, implicit $q1, implicit killed $q3, implicit $zr {
+    ; CHECK:   MVE_VPTv4s32 7, renamable $q0, renamable $q2, 10, implicit-def $vpr
+    ; CHECK:   renamable $q3 = MVE_VMAXs32 renamable $q0, renamable $q1, 1, internal renamable $vpr, killed renamable $q3
+    ; CHECK:   renamable $vpr = MVE_VCMPs32r killed renamable $q0, $zr, 12, 1, internal killed renamable $vpr
+    ; CHECK:   renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 2, internal renamable $vpr, internal renamable $q3
+    ; CHECK:   renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 2, internal killed renamable $vpr, internal renamable $q3
+    ; CHECK: }
+    ; CHECK: $q0 = MVE_VORR $q3, $q3, 0, $noreg, undef $q0
+    ; CHECK: $q3 = MVE_VORR $q2, $q2, 0, $noreg, undef $q3
+    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit $q0, implicit $q2, implicit $q1, implicit killed $q3, implicit $zr {
+    ; CHECK:   MVE_VPTv4s32 13, renamable $q0, renamable $q2, 10, implicit-def $vpr
+    ; CHECK:   renamable $q3 = MVE_VMAXs32 renamable $q0, renamable $q1, 1, internal renamable $vpr, killed renamable $q3
+    ; CHECK:   renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 2, internal renamable $vpr, internal renamable $q3
+    ; CHECK:   renamable $vpr = MVE_VCMPs32r renamable $q1, $zr, 12, 2, internal killed renamable $vpr
+    ; CHECK:   renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 1, internal killed renamable $vpr, internal killed renamable $q3
+    ; CHECK: }
+    ; CHECK: $q0 = MVE_VORR killed $q3, killed $q3, 0, $noreg, undef $q0
+    ; CHECK: $q3 = MVE_VORR $q2, $q2, 0, $noreg, undef $q3
+    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit $q0, implicit $q2, implicit $q1, implicit killed $q3, implicit $zr {
+    ; CHECK:   MVE_VPTv4s32 9, renamable $q0, renamable $q2, 10, implicit-def $vpr
+    ; CHECK:   renamable $q3 = MVE_VMAXs32 renamable $q0, renamable $q1, 1, internal renamable $vpr, killed renamable $q3
+    ; CHECK:   renamable $vpr = MVE_VCMPs32r renamable $q1, $zr, 12, 2, internal killed renamable $vpr
+    ; CHECK:   renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 1, internal renamable $vpr, internal renamable $q3
+    ; CHECK:   renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 1, internal killed renamable $vpr, internal killed renamable $q3
+    ; CHECK: }
+    ; CHECK: $q0 = MVE_VORR killed $q3, killed $q3, 0, $noreg, undef $q0
+    ; CHECK: $q3 = MVE_VORR $q2, $q2, 0, $noreg, undef $q3
+    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit $q0, implicit $q2, implicit $q1, implicit killed $q3 {
+    ; CHECK:   MVE_VPTv4s32 15, renamable $q0, renamable $q2, 10, implicit-def $vpr
+    ; CHECK:   renamable $q3 = MVE_VMAXs32 renamable $q0, renamable $q1, 1, internal renamable $vpr, killed renamable $q3
+    ; CHECK:   renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 2, internal renamable $vpr, internal renamable $q3
+    ; CHECK:   renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 2, internal renamable $vpr, internal renamable $q3
+    ; CHECK:   renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 2, internal killed renamable $vpr, internal killed renamable $q3
+    ; CHECK: }
+    ; CHECK: $q0 = MVE_VORR killed $q3, killed $q3, 0, $noreg, undef $q0
+    ; CHECK: $q3 = MVE_VORR $q2, $q2, 0, $noreg, undef $q3
+    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit $q0, implicit $q2, implicit $q1, implicit killed $q3 {
+    ; CHECK:   MVE_VPTv4s32 14, renamable $q0, renamable $q2, 10, implicit-def $vpr
+    ; CHECK:   renamable $q3 = MVE_VMAXs32 renamable $q0, renamable $q1, 1, internal renamable $vpr, killed renamable $q3
+    ; CHECK:   renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 2, internal renamable $vpr, internal renamable $q3
+    ; CHECK:   renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 2, internal killed renamable $vpr, internal killed renamable $q3
+    ; CHECK: }
+    ; CHECK: $q0 = MVE_VORR killed $q3, killed $q3, 0, $noreg, undef $q0
+    ; CHECK: $q3 = MVE_VORR $q2, $q2, 0, $noreg, undef $q3
+    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit $q0, implicit $q2, implicit $q1, implicit killed $q3, implicit $zr {
+    ; CHECK:   MVE_VPTv4s32 10, renamable $q0, renamable $q2, 10, implicit-def $vpr
+    ; CHECK:   renamable $q3 = MVE_VMAXs32 renamable $q0, renamable $q1, 1, internal renamable $vpr, killed renamable $q3
+    ; CHECK:   renamable $vpr = MVE_VCMPs32r renamable $q1, $zr, 12, 2, internal killed renamable $vpr
+    ; CHECK:   renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 1, internal killed renamable $vpr, internal killed renamable $q3
+    ; CHECK: }
+    ; CHECK: $q0 = MVE_VORR killed $q3, killed $q3, 0, $noreg, undef $q0
+    ; CHECK: $q3 = MVE_VORR $q2, $q2, 0, $noreg, undef $q3
+    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit $q0, implicit $q2, implicit $q1, implicit killed $q3 {
+    ; CHECK:   MVE_VPTv4s32 6, renamable $q0, renamable $q2, 10, implicit-def $vpr
+    ; CHECK:   renamable $q3 = MVE_VMAXs32 renamable $q0, renamable $q1, 1, internal renamable $vpr, killed renamable $q3
+    ; CHECK:   renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 1, internal renamable $vpr, internal killed renamable $q3
+    ; CHECK:   renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 2, internal killed renamable $vpr, internal killed renamable $q3
+    ; CHECK: }
+    ; CHECK: $q0 = MVE_VORR killed $q3, killed $q3, 0, $noreg, undef $q0
+    ; CHECK: $q3 = MVE_VORR $q2, $q2, 0, $noreg, undef $q3
+    ; CHECK: BUNDLE implicit-def $vpr, implicit-def $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit $q0, implicit $q2, implicit killed $q3 {
+    ; CHECK:   MVE_VPTv4s32 11, renamable $q0, renamable $q2, 10, implicit-def $vpr
+    ; CHECK:   renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 1, internal renamable $vpr, killed renamable $q3
+    ; CHECK:   renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 2, internal renamable $vpr, internal killed renamable $q3
+    ; CHECK:   renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 1, internal renamable $vpr, internal killed renamable $q3
+    ; CHECK:   renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 2, internal renamable $vpr, internal killed renamable $q3
+    ; CHECK: }
+    ; CHECK: $q0 = MVE_VORR killed $q3, killed $q3, 0, $noreg, undef $q0
+    ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $q0
+    renamable $vpr = MVE_VCMPs32 renamable $q0, renamable $q2, 10, 0, $noreg
+    $q3 = MVE_VORR $q2, $q2, 0, $noreg, undef $q3
+    renamable $q3 = MVE_VMAXs32 renamable $q0, renamable $q1, 1, renamable $vpr, killed renamable $q3
+    renamable $vpr = MVE_VCMPs32r killed renamable $q0, $zr, 12, 1, killed renamable $vpr
+    renamable $vpr = MVE_VPNOT killed renamable $vpr, 0, $noreg
+    renamable $vpr = MVE_VCMPs32r renamable $q1, $zr, 12, 1, killed renamable $vpr
+    renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 1, killed renamable $vpr, renamable $q3
+    $q0 = MVE_VORR $q3, $q3, 0, $noreg, undef $q0
+
+    renamable $vpr = MVE_VCMPs32 renamable $q0, renamable $q2, 10, 0, $noreg
+    $q3 = MVE_VORR $q2, $q2, 0, $noreg, undef $q3
+    renamable $q3 = MVE_VMAXs32 renamable $q0, renamable $q1, 1, renamable $vpr, killed renamable $q3
+    renamable $vpr = MVE_VCMPs32r killed renamable $q0, $zr, 12, 1, killed renamable $vpr
+    renamable $vpr = MVE_VPNOT killed renamable $vpr, 0, $noreg
+    renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 1, renamable $vpr, renamable $q3
+    renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 1, killed renamable $vpr, renamable $q3
+    $q0 = MVE_VORR $q3, $q3, 0, $noreg, undef $q0
+
+    renamable $vpr = MVE_VCMPs32 renamable $q0, renamable $q2, 10, 0, $noreg
+    $q3 = MVE_VORR $q2, $q2, 0, $noreg, undef $q3
+    renamable $q3 = MVE_VMAXs32 renamable $q0, renamable $q1, 1, renamable $vpr, killed renamable $q3
+    renamable $vpr = MVE_VPNOT killed renamable $vpr, 0, $noreg
+    renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 1, renamable $vpr, renamable $q3
+    renamable $vpr = MVE_VCMPs32r renamable $q1, $zr, 12, 1, killed renamable $vpr
+    renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 1, killed renamable $vpr, killed renamable $q3
+    $q0 = MVE_VORR killed $q3, killed $q3, 0, $noreg, undef $q0
+
+    renamable $vpr = MVE_VCMPs32 renamable $q0, renamable $q2, 10, 0, $noreg
+    $q3 = MVE_VORR $q2, $q2, 0, $noreg, undef $q3
+    renamable $q3 = MVE_VMAXs32 renamable $q0, renamable $q1, 1, renamable $vpr, killed renamable $q3
+    renamable $vpr = MVE_VPNOT killed renamable $vpr, 0, $noreg
+    renamable $vpr = MVE_VCMPs32r renamable $q1, $zr, 12, 1, killed renamable $vpr
+    renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 1, renamable $vpr, renamable $q3
+    renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 1, killed renamable $vpr, killed renamable $q3
+    $q0 = MVE_VORR killed $q3, killed $q3, 0, $noreg, undef $q0
+
+    renamable $vpr = MVE_VCMPs32 renamable $q0, renamable $q2, 10, 0, $noreg
+    $q3 = MVE_VORR $q2, $q2, 0, $noreg, undef $q3
+    renamable $q3 = MVE_VMAXs32 renamable $q0, renamable $q1, 1, renamable $vpr, killed renamable $q3
+    renamable $vpr = MVE_VPNOT killed renamable $vpr, 0, $noreg
+    renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 1, renamable $vpr, renamable $q3
+    renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 1, renamable $vpr, renamable $q3
+    renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 1, killed renamable $vpr, killed renamable $q3
+    $q0 = MVE_VORR killed $q3, killed $q3, 0, $noreg, undef $q0
+
+    renamable $vpr = MVE_VCMPs32 renamable $q0, renamable $q2, 10, 0, $noreg
+    $q3 = MVE_VORR $q2, $q2, 0, $noreg, undef $q3
+    renamable $q3 = MVE_VMAXs32 renamable $q0, renamable $q1, 1, renamable $vpr, killed renamable $q3
+    renamable $vpr = MVE_VPNOT killed renamable $vpr, 0, $noreg
+    renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 1, renamable $vpr, renamable $q3
+    renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 1, killed renamable $vpr, killed renamable $q3
+    $q0 = MVE_VORR killed $q3, killed $q3, 0, $noreg, undef $q0
+
+    renamable $vpr = MVE_VCMPs32 renamable $q0, renamable $q2, 10, 0, $noreg
+    $q3 = MVE_VORR $q2, $q2, 0, $noreg, undef $q3
+    renamable $q3 = MVE_VMAXs32 renamable $q0, renamable $q1, 1, renamable $vpr, killed renamable $q3
+    renamable $vpr = MVE_VPNOT killed renamable $vpr, 0, $noreg
+    renamable $vpr = MVE_VCMPs32r renamable $q1, $zr, 12, 1, killed renamable $vpr
+    renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 1, killed renamable $vpr, killed renamable $q3
+    $q0 = MVE_VORR killed $q3, killed $q3, 0, $noreg, undef $q0
+
+    renamable $vpr = MVE_VCMPs32 renamable $q0, renamable $q2, 10, 0, $noreg
+    $q3 = MVE_VORR $q2, $q2, 0, $noreg, undef $q3
+    renamable $q3 = MVE_VMAXs32 renamable $q0, renamable $q1, 1, renamable $vpr, killed renamable $q3
+    renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 1, renamable $vpr, killed renamable $q3
+    renamable $vpr = MVE_VPNOT killed renamable $vpr, 0, $noreg
+    renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 1, killed renamable $vpr, killed renamable $q3
+    $q0 = MVE_VORR killed $q3, killed $q3, 0, $noreg, undef $q0
+
+    $q3 = MVE_VORR $q2, $q2, 0, $noreg, undef $q3
+    renamable $vpr = MVE_VCMPs32 renamable $q0, renamable $q2, 10, 0, $noreg
+    renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 1, renamable $vpr, killed renamable $q3
+    renamable $vpr = MVE_VPNOT killed renamable $vpr, 0, $noreg
+    renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 1, renamable $vpr, killed renamable $q3
+    renamable $vpr = MVE_VPNOT killed renamable $vpr, 0, $noreg
+    renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 1, renamable $vpr, killed renamable $q3
+    renamable $vpr = MVE_VPNOT killed renamable $vpr, 0, $noreg
+    renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 1, renamable $vpr, killed renamable $q3
+    $q0 = MVE_VORR killed $q3, killed $q3, 0, $noreg, undef $q0
+
+    tBX_RET 14, $noreg, implicit $q0
+
+...
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-nots.mir b/llvm/test/CodeGen/Thumb2/mve-vpt-nots.mir
index b3e953964b199..8bc7a0b535989 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vpt-nots.mir
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-nots.mir
@@ -61,14 +61,10 @@ body:             |
 
     ; CHECK-LABEL: name: vpnot
     ; CHECK: liveins: $q0, $q1, $q2
-    ; CHECK: BUNDLE implicit-def $vpr, implicit $q0, implicit $zr, implicit $q1 {
-    ; CHECK:   MVE_VPTv4s32r 8, renamable $q0, $zr, 11, implicit-def $vpr
+    ; CHECK: BUNDLE implicit-def $vpr, implicit $q0, implicit $zr, implicit $q1, implicit killed $q2 {
+    ; CHECK:   MVE_VPTv4s32r 12, renamable $q0, $zr, 11, implicit-def $vpr
     ; CHECK:   renamable $vpr = MVE_VCMPs32r renamable $q1, $zr, 12, 1, internal killed renamable $vpr
-    ; CHECK: }
-    ; CHECK: renamable $vpr = MVE_VPNOT killed renamable $vpr, 0, $noreg
-    ; CHECK: BUNDLE implicit-def $vpr, implicit killed $vpr, implicit killed $q2, implicit $zr {
-    ; CHECK:   MVE_VPST 8, implicit $vpr
-    ; CHECK:   renamable $vpr = MVE_VCMPi32r killed renamable $q2, $zr, 0, 1, killed renamable $vpr
+    ; CHECK:   renamable $vpr = MVE_VCMPi32r killed renamable $q2, $zr, 0, 2, internal killed renamable $vpr
     ; CHECK: }
     ; CHECK: renamable $q0 = MVE_VPSEL killed renamable $q0, killed renamable $q1, 0, killed renamable $vpr
     ; CHECK: tBX_RET 14 /* CC::al */, $noreg, implicit $q0
@@ -244,14 +240,10 @@ body:             |
     ; CHECK: liveins: $q0, $q1, $q2
     ; CHECK: renamable $vpr = MVE_VCMPs32r renamable $q0, $zr, 11, 0, $noreg
     ; CHECK: renamable $vpr = MVE_VPNOT killed renamable $vpr, 0, $noreg
-    ; CHECK: BUNDLE implicit-def $vpr, implicit killed $vpr, implicit $q1, implicit $zr {
-    ; CHECK:   MVE_VPST 8, implicit $vpr
+    ; CHECK: BUNDLE implicit-def $vpr, implicit killed $vpr, implicit $q1, implicit $zr, implicit killed $q2 {
+    ; CHECK:   MVE_VPST 12, implicit $vpr
     ; CHECK:   renamable $vpr = MVE_VCMPs32r renamable $q1, $zr, 12, 1, killed renamable $vpr
-    ; CHECK: }
-    ; CHECK: renamable $vpr = MVE_VPNOT killed renamable $vpr, 0, $noreg
-    ; CHECK: BUNDLE implicit-def $vpr, implicit killed $vpr, implicit killed $q2, implicit $zr {
-    ; CHECK:   MVE_VPST 8, implicit $vpr
-    ; CHECK:   renamable $vpr = MVE_VCMPi32r killed renamable $q2, $zr, 0, 1, killed renamable $vpr
+    ; CHECK:   renamable $vpr = MVE_VCMPi32r killed renamable $q2, $zr, 0, 2, internal killed renamable $vpr
     ; CHECK: }
     ; CHECK: renamable $vpr = MVE_VPNOT killed renamable $vpr, 0, $noreg
     ; CHECK: renamable $q0 = MVE_VPSEL killed renamable $q0, killed renamable $q1, 0, killed renamable $vpr
diff --git a/llvm/test/CodeGen/Thumb2/mve-vst3.ll b/llvm/test/CodeGen/Thumb2/mve-vst3.ll
index 97c595c014482..614db5ff829cf 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vst3.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vst3.ll
@@ -93,20 +93,21 @@ define void @vst3_v8i32(<8 x i32> *%src, <24 x i32> *%dst) {
 ; CHECK-NEXT:    vldrw.u32 q0, [r0, #80]
 ; CHECK-NEXT:    vldrw.u32 q5, [r0, #16]
 ; CHECK-NEXT:    vldrw.u32 q1, [r0, #48]
-; CHECK-NEXT:    vmov.f64 d6, d1
-; CHECK-NEXT:    vmov.32 r2, q1[3]
 ; CHECK-NEXT:    vldrw.u32 q4, [r0]
+; CHECK-NEXT:    vmov.f64 d6, d1
 ; CHECK-NEXT:    vldrw.u32 q6, [r0, #32]
-; CHECK-NEXT:    vdup.32 q2, r2
+; CHECK-NEXT:    vmov.32 r2, q1[3]
 ; CHECK-NEXT:    vldrw.u32 q7, [r0, #64]
+; CHECK-NEXT:    vdup.32 q2, r2
 ; CHECK-NEXT:    vstrw.32 q4, [sp, #32] @ 16-byte Spill
+; CHECK-NEXT:    vstrw.32 q5, [sp] @ 16-byte Spill
+; CHECK-NEXT:    vmov.32 r0, q7[0]
 ; CHECK-NEXT:    vmov.f32 s13, s23
 ; CHECK-NEXT:    vstrw.32 q6, [sp, #16] @ 16-byte Spill
 ; CHECK-NEXT:    vmov.f32 s15, s3
-; CHECK-NEXT:    vstrw.32 q5, [sp] @ 16-byte Spill
 ; CHECK-NEXT:    vmov.f32 s14, s10
-; CHECK-NEXT:    vmov.32 r0, q7[0]
 ; CHECK-NEXT:    vmov.f64 d4, d8
+; CHECK-NEXT:    vstrw.32 q3, [r1, #80]
 ; CHECK-NEXT:    vmov.f32 s9, s24
 ; CHECK-NEXT:    vmov.f32 s11, s17
 ; CHECK-NEXT:    vmov q4, q5
@@ -120,20 +121,19 @@ define void @vst3_v8i32(<8 x i32> *%src, <24 x i32> *%dst) {
 ; CHECK-NEXT:    vmov.f32 s3, s6
 ; CHECK-NEXT:    vldrw.u32 q1, [sp] @ 16-byte Reload
 ; CHECK-NEXT:    vdup.32 q6, r0
+; CHECK-NEXT:    vmov.32 r0, q4[3]
 ; CHECK-NEXT:    vmov.f32 s22, s26
 ; CHECK-NEXT:    vldrw.u32 q6, [sp, #32] @ 16-byte Reload
 ; CHECK-NEXT:    vmov.f32 s2, s6
-; CHECK-NEXT:    vmov.32 r0, q4[3]
-; CHECK-NEXT:    vmov.f32 s4, s17
 ; CHECK-NEXT:    vstrw.32 q5, [r1, #48]
-; CHECK-NEXT:    vmov.f32 s5, s29
+; CHECK-NEXT:    vmov.f32 s4, s17
 ; CHECK-NEXT:    vstrw.32 q0, [r1, #64]
+; CHECK-NEXT:    vmov.f32 s5, s29
+; CHECK-NEXT:    vstrw.32 q2, [r1]
 ; CHECK-NEXT:    vmov.f32 s28, s30
-; CHECK-NEXT:    vstrw.32 q3, [r1, #80]
 ; CHECK-NEXT:    vmov.f32 s7, s18
 ; CHECK-NEXT:    vdup.32 q4, r0
 ; CHECK-NEXT:    vmov.f32 s29, s27
-; CHECK-NEXT:    vstrw.32 q2, [r1]
 ; CHECK-NEXT:    vmov.f32 s6, s26
 ; CHECK-NEXT:    vmov.f32 s30, s18
 ; CHECK-NEXT:    vstrw.32 q1, [r1, #16]
@@ -185,32 +185,32 @@ define void @vst3_v16i32(<16 x i32> *%src, <48 x i32> *%dst) {
 ; CHECK-NEXT:    vstrw.32 q2, [sp, #16] @ 16-byte Spill
 ; CHECK-NEXT:    vmov.f64 d4, d15
 ; CHECK-NEXT:    vldrw.u32 q4, [r0, #176]
-; CHECK-NEXT:    vstrw.32 q0, [sp, #32] @ 16-byte Spill
-; CHECK-NEXT:    vstrw.32 q1, [r1, #16]
 ; CHECK-NEXT:    vmov.32 r0, q6[3]
+; CHECK-NEXT:    vstrw.32 q1, [r1, #16]
 ; CHECK-NEXT:    vdup.32 q1, r0
+; CHECK-NEXT:    vmov.32 r0, q3[3]
+; CHECK-NEXT:    vstrw.32 q0, [sp, #32] @ 16-byte Spill
 ; CHECK-NEXT:    vmov.f32 s9, s23
 ; CHECK-NEXT:    vmov.f32 s11, s31
 ; CHECK-NEXT:    vmov.f32 s10, s6
 ; CHECK-NEXT:    vmov.f64 d2, d9
 ; CHECK-NEXT:    vstrw.32 q2, [r1, #32]
-; CHECK-NEXT:    vmov.32 r0, q3[3]
 ; CHECK-NEXT:    vmov.f32 s5, s3
 ; CHECK-NEXT:    vmov.f32 s7, s19
 ; CHECK-NEXT:    vmov q2, q1
 ; CHECK-NEXT:    vdup.32 q1, r0
 ; CHECK-NEXT:    vmov.f32 s10, s6
+; CHECK-NEXT:    vmov.32 r0, q7[0]
 ; CHECK-NEXT:    vmov.f64 d2, d10
 ; CHECK-NEXT:    vstrw.32 q2, [sp, #80] @ 16-byte Spill
-; CHECK-NEXT:    vmov.32 r0, q7[0]
 ; CHECK-NEXT:    vldrw.u32 q2, [sp, #32] @ 16-byte Reload
 ; CHECK-NEXT:    vdup.32 q0, r0
+; CHECK-NEXT:    vmov.32 r0, q4[0]
 ; CHECK-NEXT:    vmov.f32 s5, s24
 ; CHECK-NEXT:    vmov.f32 s7, s21
 ; CHECK-NEXT:    vmov.f32 s6, s2
 ; CHECK-NEXT:    vmov.f64 d0, d4
 ; CHECK-NEXT:    vstrw.32 q1, [sp, #64] @ 16-byte Spill
-; CHECK-NEXT:    vmov.32 r0, q4[0]
 ; CHECK-NEXT:    vmov.f32 s1, s12
 ; CHECK-NEXT:    vmov.f32 s3, s9
 ; CHECK-NEXT:    vmov q1, q0
@@ -227,9 +227,9 @@ define void @vst3_v16i32(<16 x i32> *%src, <48 x i32> *%dst) {
 ; CHECK-NEXT:    vmov.f32 s27, s6
 ; CHECK-NEXT:    vldrw.u32 q1, [sp, #16] @ 16-byte Reload
 ; CHECK-NEXT:    vmov.f32 s18, s10
+; CHECK-NEXT:    vmov.32 r0, q3[3]
 ; CHECK-NEXT:    vmov.f64 d4, d1
 ; CHECK-NEXT:    vstrw.32 q4, [sp, #32] @ 16-byte Spill
-; CHECK-NEXT:    vmov.32 r0, q3[3]
 ; CHECK-NEXT:    vmov q5, q1
 ; CHECK-NEXT:    vldrw.u32 q4, [sp, #144] @ 16-byte Reload
 ; CHECK-NEXT:    vmov.f32 s9, s7
@@ -242,6 +242,8 @@ define void @vst3_v16i32(<16 x i32> *%src, <48 x i32> *%dst) {
 ; CHECK-NEXT:    vmov.f64 d6, d1
 ; CHECK-NEXT:    vmov.32 r0, q4[3]
 ; CHECK-NEXT:    vmov q7, q0
+; CHECK-NEXT:    vstrw.32 q6, [r1, #112]
+; CHECK-NEXT:    vstrw.32 q2, [r1, #128]
 ; CHECK-NEXT:    vmov.f32 s13, s7
 ; CHECK-NEXT:    vldrw.u32 q1, [sp, #96] @ 16-byte Reload
 ; CHECK-NEXT:    vmov.f32 s15, s3
@@ -249,18 +251,16 @@ define void @vst3_v16i32(<16 x i32> *%src, <48 x i32> *%dst) {
 ; CHECK-NEXT:    vmov.f32 s14, s2
 ; CHECK-NEXT:    vmov q0, q5
 ; CHECK-NEXT:    vmov.f32 s21, s4
+; CHECK-NEXT:    vstrw.32 q3, [r1, #80]
 ; CHECK-NEXT:    vmov.f32 s23, s1
 ; CHECK-NEXT:    vldrw.u32 q0, [sp, #112] @ 16-byte Reload
 ; CHECK-NEXT:    vmov.32 r0, q0[0]
 ; CHECK-NEXT:    vdup.32 q0, r0
+; CHECK-NEXT:    vmov.32 r0, q7[0]
 ; CHECK-NEXT:    vmov.f32 s22, s2
 ; CHECK-NEXT:    vldrw.u32 q0, [sp, #128] @ 16-byte Reload
-; CHECK-NEXT:    vmov.32 r0, q7[0]
-; CHECK-NEXT:    vstrw.32 q3, [r1, #80]
-; CHECK-NEXT:    vmov.f64 d2, d0
 ; CHECK-NEXT:    vstrw.32 q5, [r1, #96]
-; CHECK-NEXT:    vstrw.32 q6, [r1, #112]
-; CHECK-NEXT:    vstrw.32 q2, [r1, #128]
+; CHECK-NEXT:    vmov.f64 d2, d0
 ; CHECK-NEXT:    vmov.f32 s5, s16
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vmov.f32 s7, s1
@@ -503,13 +503,14 @@ define void @vst3_v16i16(<16 x i16> *%src, <48 x i16> *%dst) {
 ; CHECK-NEXT:    vmov q7, q1
 ; CHECK-NEXT:    vmov.f32 s9, s0
 ; CHECK-NEXT:    vldrw.u32 q0, [r0, #80]
-; CHECK-NEXT:    vstrw.32 q2, [sp, #96] @ 16-byte Spill
 ; CHECK-NEXT:    vmov.u16 r2, q2[3]
+; CHECK-NEXT:    vmov q3, q2
 ; CHECK-NEXT:    vmov.32 r3, q0[0]
 ; CHECK-NEXT:    vmov q1, q0
 ; CHECK-NEXT:    vdup.32 q0, r3
-; CHECK-NEXT:    vmov q3, q2
+; CHECK-NEXT:    vstrw.32 q2, [sp, #96] @ 16-byte Spill
 ; CHECK-NEXT:    vmov.u16 r3, q0[2]
+; CHECK-NEXT:    vstrw.32 q7, [sp, #32] @ 16-byte Spill
 ; CHECK-NEXT:    vmov.16 q2[2], r3
 ; CHECK-NEXT:    vmov.16 q2[3], r2
 ; CHECK-NEXT:    vmov.u16 r2, q3[4]
@@ -578,17 +579,17 @@ define void @vst3_v16i16(<16 x i16> *%src, <48 x i16> *%dst) {
 ; CHECK-NEXT:    vmov.16 q0[6], r0
 ; CHECK-NEXT:    vmov.u16 r0, q1[7]
 ; CHECK-NEXT:    vldrw.u32 q1, [sp, #128] @ 16-byte Reload
-; CHECK-NEXT:    vstrw.32 q2, [sp, #64] @ 16-byte Spill
-; CHECK-NEXT:    vstrw.32 q7, [sp, #32] @ 16-byte Spill
 ; CHECK-NEXT:    vmov.16 q0[7], r0
-; CHECK-NEXT:    vmov.32 r2, q1[3]
 ; CHECK-NEXT:    vmov.f32 s2, s19
-; CHECK-NEXT:    vdup.32 q7, r2
+; CHECK-NEXT:    vstrw.32 q2, [sp, #64] @ 16-byte Spill
+; CHECK-NEXT:    vmov.32 r2, q1[3]
 ; CHECK-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-NEXT:    vmov.u16 r2, q7[2]
+; CHECK-NEXT:    vdup.32 q7, r2
 ; CHECK-NEXT:    vrev32.16 q3, q3
-; CHECK-NEXT:    vmov.16 q2[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q7[2]
 ; CHECK-NEXT:    vstrw.32 q3, [sp] @ 16-byte Spill
+; CHECK-NEXT:    vmov.16 q2[2], r2
+; CHECK-NEXT:    vstrw.32 q4, [sp, #16] @ 16-byte Spill
 ; CHECK-NEXT:    vmov.16 q2[3], r0
 ; CHECK-NEXT:    vmov.u16 r0, q0[4]
 ; CHECK-NEXT:    vmov.16 q2[4], r0
@@ -603,7 +604,6 @@ define void @vst3_v16i16(<16 x i16> *%src, <48 x i16> *%dst) {
 ; CHECK-NEXT:    vmov.16 q3[6], r0
 ; CHECK-NEXT:    vmov.u16 r0, q1[5]
 ; CHECK-NEXT:    vldrw.u32 q1, [sp, #144] @ 16-byte Reload
-; CHECK-NEXT:    vstrw.32 q4, [sp, #16] @ 16-byte Spill
 ; CHECK-NEXT:    vldrw.u32 q4, [sp, #48] @ 16-byte Reload
 ; CHECK-NEXT:    vrev32.16 q7, q7
 ; CHECK-NEXT:    vmov.16 q3[7], r0
@@ -611,34 +611,35 @@ define void @vst3_v16i16(<16 x i16> *%src, <48 x i16> *%dst) {
 ; CHECK-NEXT:    vstrw.32 q7, [sp, #32] @ 16-byte Spill
 ; CHECK-NEXT:    vmov.16 q7[0], r0
 ; CHECK-NEXT:    vmov.u16 r0, q4[3]
+; CHECK-NEXT:    vmov.f32 s1, s9
 ; CHECK-NEXT:    vmov.16 q7[1], r0
 ; CHECK-NEXT:    vmov.u16 r0, q1[4]
 ; CHECK-NEXT:    vldrw.u32 q1, [sp, #64] @ 16-byte Reload
-; CHECK-NEXT:    vmov.f32 s1, s9
 ; CHECK-NEXT:    vmov.f32 s2, s10
 ; CHECK-NEXT:    vldrw.u32 q2, [sp, #96] @ 16-byte Reload
-; CHECK-NEXT:    vmov.f32 s25, s5
 ; CHECK-NEXT:    vmov.16 q7[6], r0
+; CHECK-NEXT:    vmov.f32 s25, s5
+; CHECK-NEXT:    vmov.u16 r0, q4[5]
 ; CHECK-NEXT:    vmov.f32 s26, s6
 ; CHECK-NEXT:    vldrw.u32 q1, [sp, #80] @ 16-byte Reload
-; CHECK-NEXT:    vmov.u16 r0, q4[5]
 ; CHECK-NEXT:    vstrw.32 q0, [r1, #80]
-; CHECK-NEXT:    vmov.f32 s21, s5
 ; CHECK-NEXT:    vldrw.u32 q0, [sp] @ 16-byte Reload
+; CHECK-NEXT:    vmov.f32 s21, s5
+; CHECK-NEXT:    vmov.16 q7[7], r0
 ; CHECK-NEXT:    vmov.f32 s22, s6
 ; CHECK-NEXT:    vldrw.u32 q1, [sp, #112] @ 16-byte Reload
-; CHECK-NEXT:    vmov.16 q7[7], r0
 ; CHECK-NEXT:    vmov.u16 r2, q0[2]
-; CHECK-NEXT:    vmov.f32 s9, s5
 ; CHECK-NEXT:    vmov.u16 r0, q0[5]
+; CHECK-NEXT:    vmov.f32 s9, s5
+; CHECK-NEXT:    vmov.16 q0[2], r2
 ; CHECK-NEXT:    vmov.f32 s10, s6
 ; CHECK-NEXT:    vldrw.u32 q1, [sp, #144] @ 16-byte Reload
-; CHECK-NEXT:    vmov.16 q0[2], r2
 ; CHECK-NEXT:    vstrw.32 q5, [r1, #32]
+; CHECK-NEXT:    vstrw.32 q2, [r1, #48]
 ; CHECK-NEXT:    vmov.f32 s29, s5
 ; CHECK-NEXT:    vldrw.u32 q1, [sp, #16] @ 16-byte Reload
 ; CHECK-NEXT:    vmov.f32 s30, s18
-; CHECK-NEXT:    vstrw.32 q2, [r1, #48]
+; CHECK-NEXT:    vstrw.32 q6, [r1]
 ; CHECK-NEXT:    vmov.u16 r2, q7[3]
 ; CHECK-NEXT:    vmov.f32 s13, s5
 ; CHECK-NEXT:    vmov.16 q0[3], r2
@@ -646,7 +647,6 @@ define void @vst3_v16i16(<16 x i16> *%src, <48 x i16> *%dst) {
 ; CHECK-NEXT:    vmov.16 q0[4], r2
 ; CHECK-NEXT:    vldrw.u32 q1, [sp, #128] @ 16-byte Reload
 ; CHECK-NEXT:    vmov.16 q0[5], r0
-; CHECK-NEXT:    vstrw.32 q6, [r1]
 ; CHECK-NEXT:    vmov.f32 s29, s1
 ; CHECK-NEXT:    vmov.f32 s30, s2
 ; CHECK-NEXT:    vldrw.u32 q0, [sp, #32] @ 16-byte Reload
@@ -1251,20 +1251,21 @@ define void @vst3_v8f32(<8 x float> *%src, <24 x float> *%dst) {
 ; CHECK-NEXT:    vldrw.u32 q0, [r0, #80]
 ; CHECK-NEXT:    vldrw.u32 q5, [r0, #16]
 ; CHECK-NEXT:    vldrw.u32 q1, [r0, #48]
-; CHECK-NEXT:    vmov.f64 d6, d1
-; CHECK-NEXT:    vmov.32 r2, q1[3]
 ; CHECK-NEXT:    vldrw.u32 q4, [r0]
+; CHECK-NEXT:    vmov.f64 d6, d1
 ; CHECK-NEXT:    vldrw.u32 q6, [r0, #32]
-; CHECK-NEXT:    vdup.32 q2, r2
+; CHECK-NEXT:    vmov.32 r2, q1[3]
 ; CHECK-NEXT:    vldrw.u32 q7, [r0, #64]
+; CHECK-NEXT:    vdup.32 q2, r2
 ; CHECK-NEXT:    vstrw.32 q4, [sp, #32] @ 16-byte Spill
+; CHECK-NEXT:    vstrw.32 q5, [sp] @ 16-byte Spill
+; CHECK-NEXT:    vmov.32 r0, q7[0]
 ; CHECK-NEXT:    vmov.f32 s13, s23
 ; CHECK-NEXT:    vstrw.32 q6, [sp, #16] @ 16-byte Spill
 ; CHECK-NEXT:    vmov.f32 s15, s3
-; CHECK-NEXT:    vstrw.32 q5, [sp] @ 16-byte Spill
 ; CHECK-NEXT:    vmov.f32 s14, s10
-; CHECK-NEXT:    vmov.32 r0, q7[0]
 ; CHECK-NEXT:    vmov.f64 d4, d8
+; CHECK-NEXT:    vstrw.32 q3, [r1, #80]
 ; CHECK-NEXT:    vmov.f32 s9, s24
 ; CHECK-NEXT:    vmov.f32 s11, s17
 ; CHECK-NEXT:    vmov q4, q5
@@ -1278,20 +1279,19 @@ define void @vst3_v8f32(<8 x float> *%src, <24 x float> *%dst) {
 ; CHECK-NEXT:    vmov.f32 s3, s6
 ; CHECK-NEXT:    vldrw.u32 q1, [sp] @ 16-byte Reload
 ; CHECK-NEXT:    vdup.32 q6, r0
+; CHECK-NEXT:    vmov.32 r0, q4[3]
 ; CHECK-NEXT:    vmov.f32 s22, s26
 ; CHECK-NEXT:    vldrw.u32 q6, [sp, #32] @ 16-byte Reload
 ; CHECK-NEXT:    vmov.f32 s2, s6
-; CHECK-NEXT:    vmov.32 r0, q4[3]
-; CHECK-NEXT:    vmov.f32 s4, s17
 ; CHECK-NEXT:    vstrw.32 q5, [r1, #48]
-; CHECK-NEXT:    vmov.f32 s5, s29
+; CHECK-NEXT:    vmov.f32 s4, s17
 ; CHECK-NEXT:    vstrw.32 q0, [r1, #64]
+; CHECK-NEXT:    vmov.f32 s5, s29
+; CHECK-NEXT:    vstrw.32 q2, [r1]
 ; CHECK-NEXT:    vmov.f32 s28, s30
-; CHECK-NEXT:    vstrw.32 q3, [r1, #80]
 ; CHECK-NEXT:    vmov.f32 s7, s18
 ; CHECK-NEXT:    vdup.32 q4, r0
 ; CHECK-NEXT:    vmov.f32 s29, s27
-; CHECK-NEXT:    vstrw.32 q2, [r1]
 ; CHECK-NEXT:    vmov.f32 s6, s26
 ; CHECK-NEXT:    vmov.f32 s30, s18
 ; CHECK-NEXT:    vstrw.32 q1, [r1, #16]
@@ -1343,32 +1343,32 @@ define void @vst3_v16f32(<16 x float> *%src, <48 x float> *%dst) {
 ; CHECK-NEXT:    vstrw.32 q2, [sp, #16] @ 16-byte Spill
 ; CHECK-NEXT:    vmov.f64 d4, d15
 ; CHECK-NEXT:    vldrw.u32 q4, [r0, #176]
-; CHECK-NEXT:    vstrw.32 q0, [sp, #32] @ 16-byte Spill
-; CHECK-NEXT:    vstrw.32 q1, [r1, #16]
 ; CHECK-NEXT:    vmov.32 r0, q6[3]
+; CHECK-NEXT:    vstrw.32 q1, [r1, #16]
 ; CHECK-NEXT:    vdup.32 q1, r0
+; CHECK-NEXT:    vmov.32 r0, q3[3]
+; CHECK-NEXT:    vstrw.32 q0, [sp, #32] @ 16-byte Spill
 ; CHECK-NEXT:    vmov.f32 s9, s23
 ; CHECK-NEXT:    vmov.f32 s11, s31
 ; CHECK-NEXT:    vmov.f32 s10, s6
 ; CHECK-NEXT:    vmov.f64 d2, d9
 ; CHECK-NEXT:    vstrw.32 q2, [r1, #32]
-; CHECK-NEXT:    vmov.32 r0, q3[3]
 ; CHECK-NEXT:    vmov.f32 s5, s3
 ; CHECK-NEXT:    vmov.f32 s7, s19
 ; CHECK-NEXT:    vmov q2, q1
 ; CHECK-NEXT:    vdup.32 q1, r0
 ; CHECK-NEXT:    vmov.f32 s10, s6
+; CHECK-NEXT:    vmov.32 r0, q7[0]
 ; CHECK-NEXT:    vmov.f64 d2, d10
 ; CHECK-NEXT:    vstrw.32 q2, [sp, #80] @ 16-byte Spill
-; CHECK-NEXT:    vmov.32 r0, q7[0]
 ; CHECK-NEXT:    vldrw.u32 q2, [sp, #32] @ 16-byte Reload
 ; CHECK-NEXT:    vdup.32 q0, r0
+; CHECK-NEXT:    vmov.32 r0, q4[0]
 ; CHECK-NEXT:    vmov.f32 s5, s24
 ; CHECK-NEXT:    vmov.f32 s7, s21
 ; CHECK-NEXT:    vmov.f32 s6, s2
 ; CHECK-NEXT:    vmov.f64 d0, d4
 ; CHECK-NEXT:    vstrw.32 q1, [sp, #64] @ 16-byte Spill
-; CHECK-NEXT:    vmov.32 r0, q4[0]
 ; CHECK-NEXT:    vmov.f32 s1, s12
 ; CHECK-NEXT:    vmov.f32 s3, s9
 ; CHECK-NEXT:    vmov q1, q0
@@ -1385,9 +1385,9 @@ define void @vst3_v16f32(<16 x float> *%src, <48 x float> *%dst) {
 ; CHECK-NEXT:    vmov.f32 s27, s6
 ; CHECK-NEXT:    vldrw.u32 q1, [sp, #16] @ 16-byte Reload
 ; CHECK-NEXT:    vmov.f32 s18, s10
+; CHECK-NEXT:    vmov.32 r0, q3[3]
 ; CHECK-NEXT:    vmov.f64 d4, d1
 ; CHECK-NEXT:    vstrw.32 q4, [sp, #32] @ 16-byte Spill
-; CHECK-NEXT:    vmov.32 r0, q3[3]
 ; CHECK-NEXT:    vmov q5, q1
 ; CHECK-NEXT:    vldrw.u32 q4, [sp, #144] @ 16-byte Reload
 ; CHECK-NEXT:    vmov.f32 s9, s7
@@ -1400,6 +1400,8 @@ define void @vst3_v16f32(<16 x float> *%src, <48 x float> *%dst) {
 ; CHECK-NEXT:    vmov.f64 d6, d1
 ; CHECK-NEXT:    vmov.32 r0, q4[3]
 ; CHECK-NEXT:    vmov q7, q0
+; CHECK-NEXT:    vstrw.32 q6, [r1, #112]
+; CHECK-NEXT:    vstrw.32 q2, [r1, #128]
 ; CHECK-NEXT:    vmov.f32 s13, s7
 ; CHECK-NEXT:    vldrw.u32 q1, [sp, #96] @ 16-byte Reload
 ; CHECK-NEXT:    vmov.f32 s15, s3
@@ -1407,18 +1409,16 @@ define void @vst3_v16f32(<16 x float> *%src, <48 x float> *%dst) {
 ; CHECK-NEXT:    vmov.f32 s14, s2
 ; CHECK-NEXT:    vmov q0, q5
 ; CHECK-NEXT:    vmov.f32 s21, s4
+; CHECK-NEXT:    vstrw.32 q3, [r1, #80]
 ; CHECK-NEXT:    vmov.f32 s23, s1
 ; CHECK-NEXT:    vldrw.u32 q0, [sp, #112] @ 16-byte Reload
 ; CHECK-NEXT:    vmov.32 r0, q0[0]
 ; CHECK-NEXT:    vdup.32 q0, r0
+; CHECK-NEXT:    vmov.32 r0, q7[0]
 ; CHECK-NEXT:    vmov.f32 s22, s2
 ; CHECK-NEXT:    vldrw.u32 q0, [sp, #128] @ 16-byte Reload
-; CHECK-NEXT:    vmov.32 r0, q7[0]
-; CHECK-NEXT:    vstrw.32 q3, [r1, #80]
-; CHECK-NEXT:    vmov.f64 d2, d0
 ; CHECK-NEXT:    vstrw.32 q5, [r1, #96]
-; CHECK-NEXT:    vstrw.32 q6, [r1, #112]
-; CHECK-NEXT:    vstrw.32 q2, [r1, #128]
+; CHECK-NEXT:    vmov.f64 d2, d0
 ; CHECK-NEXT:    vmov.f32 s5, s16
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vmov.f32 s7, s1
@@ -1573,11 +1573,11 @@ define void @vst3_v8f16(<8 x half> *%src, <24 x half> *%dst) {
 ; CHECK-NEXT:    vmovx.f16 s12, s4
 ; CHECK-NEXT:    vmov r2, s4
 ; CHECK-NEXT:    vmov.16 q0[0], r3
-; CHECK-NEXT:    vmov.32 r0, q5[0]
 ; CHECK-NEXT:    vmov.16 q0[1], r2
 ; CHECK-NEXT:    vmov r2, s12
-; CHECK-NEXT:    vdup.32 q4, r0
+; CHECK-NEXT:    vmov.32 r0, q5[0]
 ; CHECK-NEXT:    vmov.16 q0[4], r2
+; CHECK-NEXT:    vdup.32 q4, r0
 ; CHECK-NEXT:    vmov r2, s9
 ; CHECK-NEXT:    vmov.16 q0[6], r2
 ; CHECK-NEXT:    vmov r2, s5
@@ -1606,17 +1606,17 @@ define void @vst3_v8f16(<8 x half> *%src, <24 x half> *%dst) {
 ; CHECK-NEXT:    vmovx.f16 s24, s23
 ; CHECK-NEXT:    vmov.16 q4[6], r0
 ; CHECK-NEXT:    vmov r0, s24
-; CHECK-NEXT:    vstrw.32 q1, [sp] @ 16-byte Spill
-; CHECK-NEXT:    vmov.16 q4[7], r0
 ; CHECK-NEXT:    vmov.32 r2, q2[3]
-; CHECK-NEXT:    vmov.f32 s18, s23
+; CHECK-NEXT:    vmov.16 q4[7], r0
 ; CHECK-NEXT:    vdup.32 q7, r2
+; CHECK-NEXT:    vmov.f32 s18, s23
 ; CHECK-NEXT:    vmovx.f16 s24, s17
 ; CHECK-NEXT:    vmov r2, s29
+; CHECK-NEXT:    vmovx.f16 s28, s30
 ; CHECK-NEXT:    vmov r0, s24
 ; CHECK-NEXT:    vmov.16 q6[2], r2
 ; CHECK-NEXT:    vmov.16 q6[3], r0
-; CHECK-NEXT:    vmovx.f16 s28, s30
+; CHECK-NEXT:    vstrw.32 q1, [sp] @ 16-byte Spill
 ; CHECK-NEXT:    vmovx.f16 s4, s10
 ; CHECK-NEXT:    vmov.f32 s1, s13
 ; CHECK-NEXT:    vmov.f32 s2, s14
@@ -1696,12 +1696,12 @@ define void @vst3_v16f16(<16 x half> *%src, <48 x half> *%dst) {
 ; CHECK-NEXT:    vmov r2, s0
 ; CHECK-NEXT:    vldrw.u32 q0, [r0, #64]
 ; CHECK-NEXT:    vmov.f32 s5, s12
+; CHECK-NEXT:    vmov q5, q3
 ; CHECK-NEXT:    vstrw.32 q3, [sp, #32] @ 16-byte Spill
-; CHECK-NEXT:    vstrw.32 q1, [sp, #128] @ 16-byte Spill
 ; CHECK-NEXT:    vmov.32 r3, q0[0]
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vdup.32 q0, r3
-; CHECK-NEXT:    vmov q5, q3
+; CHECK-NEXT:    vstrw.32 q1, [sp, #128] @ 16-byte Spill
 ; CHECK-NEXT:    vmov r3, s1
 ; CHECK-NEXT:    vmovx.f16 s0, s2
 ; CHECK-NEXT:    vmov.16 q3[2], r3
@@ -1725,23 +1725,23 @@ define void @vst3_v16f16(<16 x half> *%src, <48 x half> *%dst) {
 ; CHECK-NEXT:    vmov.16 q1[6], r2
 ; CHECK-NEXT:    vmov r2, s0
 ; CHECK-NEXT:    vmov.16 q1[7], r2
-; CHECK-NEXT:    vstrw.32 q3, [sp, #112] @ 16-byte Spill
+; CHECK-NEXT:    vmov.32 r3, q5[3]
 ; CHECK-NEXT:    vmov.f32 s6, s19
 ; CHECK-NEXT:    vmovx.f16 s0, s5
-; CHECK-NEXT:    vstrw.32 q4, [sp, #48] @ 16-byte Spill
-; CHECK-NEXT:    vstrw.32 q1, [sp, #96] @ 16-byte Spill
-; CHECK-NEXT:    vmov.32 r3, q5[3]
 ; CHECK-NEXT:    vmov r2, s0
 ; CHECK-NEXT:    vdup.32 q0, r3
-; CHECK-NEXT:    vldrw.u32 q5, [r0, #16]
 ; CHECK-NEXT:    vmov r3, s1
 ; CHECK-NEXT:    vmovx.f16 s0, s2
 ; CHECK-NEXT:    vmov.16 q2[2], r3
-; CHECK-NEXT:    vmov r3, s20
+; CHECK-NEXT:    vldrw.u32 q5, [r0, #16]
 ; CHECK-NEXT:    vmov.16 q2[3], r2
+; CHECK-NEXT:    vstrw.32 q3, [sp, #112] @ 16-byte Spill
+; CHECK-NEXT:    vmov r3, s20
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #80]
 ; CHECK-NEXT:    vmov.16 q7[0], r3
+; CHECK-NEXT:    vstrw.32 q1, [sp, #96] @ 16-byte Spill
 ; CHECK-NEXT:    vmov r2, s6
-; CHECK-NEXT:    vldrw.u32 q3, [r0, #80]
+; CHECK-NEXT:    vstrw.32 q4, [sp, #48] @ 16-byte Spill
 ; CHECK-NEXT:    vmov.16 q2[4], r2
 ; CHECK-NEXT:    vmov r2, s0
 ; CHECK-NEXT:    vmov.16 q2[5], r2
diff --git a/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh.ll b/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh.ll
index efdb2c6d684cd..a4d8537343cac 100644
--- a/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh.ll
+++ b/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh.ll
@@ -729,18 +729,98 @@ terminate:                                        ; preds = %entry
   unreachable
 }
 
+%class.MyClass = type { i32 }
+
+; This crashed on debug mode (= when NDEBUG is not defined) when the logic for
+; computing the innermost region was not correct, in which a loop region
+; contains an exception region. This should pass CFGSort without crashing.
+define void @test12() personality i8* bitcast (i32 (...)* @__gxx_wasm_personality_v0 to i8*) {
+entry:
+  %e = alloca %class.MyClass, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %cmp = icmp slt i32 %i.0, 9
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  invoke void @quux(i32 %i.0)
+          to label %for.inc unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %for.body
+  %0 = catchswitch within none [label %catch.start] unwind to caller
+
+catch.start:                                      ; preds = %catch.dispatch
+  %1 = catchpad within %0 [i8* bitcast ({ i8*, i8* }* @_ZTI7MyClass to i8*)]
+  %2 = call i8* @llvm.wasm.get.exception(token %1)
+  %3 = call i32 @llvm.wasm.get.ehselector(token %1)
+  %4 = call i32 @llvm.eh.typeid.for(i8* bitcast ({ i8*, i8* }* @_ZTI7MyClass to i8*)) #3
+  %matches = icmp eq i32 %3, %4
+  br i1 %matches, label %catch, label %rethrow
+
+catch:                                            ; preds = %catch.start
+  %5 = call i8* @__cxa_get_exception_ptr(i8* %2) #3 [ "funclet"(token %1) ]
+  %6 = bitcast i8* %5 to %class.MyClass*
+  %call = call %class.MyClass* @_ZN7MyClassC2ERKS_(%class.MyClass* %e, %class.MyClass* dereferenceable(4) %6) [ "funclet"(token %1) ]
+  %7 = call i8* @__cxa_begin_catch(i8* %2) #3 [ "funclet"(token %1) ]
+  %x = getelementptr inbounds %class.MyClass, %class.MyClass* %e, i32 0, i32 0
+  %8 = load i32, i32* %x, align 4
+  invoke void @quux(i32 %8) [ "funclet"(token %1) ]
+          to label %invoke.cont2 unwind label %ehcleanup
+
+invoke.cont2:                                     ; preds = %catch
+  %call3 = call %class.MyClass* @_ZN7MyClassD2Ev(%class.MyClass* %e) #3 [ "funclet"(token %1) ]
+  call void @__cxa_end_catch() [ "funclet"(token %1) ]
+  catchret from %1 to label %for.inc
+
+rethrow:                                          ; preds = %catch.start
+  call void @llvm.wasm.rethrow.in.catch() #6 [ "funclet"(token %1) ]
+  unreachable
+
+for.inc:                                          ; preds = %invoke.cont2, %for.body
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond
+
+ehcleanup:                                        ; preds = %catch
+  %9 = cleanuppad within %1 []
+  %call4 = call %class.MyClass* @_ZN7MyClassD2Ev(%class.MyClass* %e) #3 [ "funclet"(token %9) ]
+  invoke void @__cxa_end_catch() [ "funclet"(token %9) ]
+          to label %invoke.cont6 unwind label %terminate7
+
+invoke.cont6:                                     ; preds = %ehcleanup
+  cleanupret from %9 unwind to caller
+
+for.end:                                          ; preds = %for.cond
+  ret void
+
+terminate7:                                       ; preds = %ehcleanup
+  %10 = cleanuppad within %9 []
+  %11 = call i8* @llvm.wasm.get.exception(token %10)
+  call void @__clang_call_terminate(i8* %11) #7 [ "funclet"(token %10) ]
+  unreachable
+}
+
 ; Check if the unwind destination mismatch stats are correct
-; NOSORT-STAT: 11 wasm-cfg-stackify    - Number of EH pad unwind mismatches found
+; NOSORT-STAT: 14 wasm-cfg-stackify    - Number of EH pad unwind mismatches found
 
 declare void @foo()
 declare void @bar()
 declare i32 @baz()
+declare void @quux(i32)
 declare void @fun(i32)
 ; Function Attrs: nounwind
 declare void @nothrow(i32) #0
 declare i32 @nothrow_i32() #0
+
 ; Function Attrs: nounwind
 declare %class.Object* @_ZN6ObjectD2Ev(%class.Object* returned) #0
+@_ZTI7MyClass = external constant { i8*, i8* }, align 4
+; Function Attrs: nounwind
+declare %class.MyClass* @_ZN7MyClassD2Ev(%class.MyClass* returned) #0
+; Function Attrs: nounwind
+declare %class.MyClass* @_ZN7MyClassC2ERKS_(%class.MyClass* returned, %class.MyClass* dereferenceable(4)) #0
+
 declare i32 @__gxx_wasm_personality_v0(...)
 declare i8* @llvm.wasm.get.exception(token)
 declare i32 @llvm.wasm.get.ehselector(token)
@@ -748,6 +828,7 @@ declare void @llvm.wasm.rethrow.in.catch()
 declare i32 @llvm.eh.typeid.for(i8*)
 declare i8* @__cxa_begin_catch(i8*)
 declare void @__cxa_end_catch()
+declare i8* @__cxa_get_exception_ptr(i8*)
 declare void @__clang_call_terminate(i8*)
 declare void @_ZSt9terminatev()
 ; Function Attrs: nounwind
diff --git a/llvm/test/CodeGen/WebAssembly/lower-em-sjlj.ll b/llvm/test/CodeGen/WebAssembly/lower-em-sjlj.ll
index cad6c4ac855bf..ad42cc5f86159 100644
--- a/llvm/test/CodeGen/WebAssembly/lower-em-sjlj.ll
+++ b/llvm/test/CodeGen/WebAssembly/lower-em-sjlj.ll
@@ -308,6 +308,10 @@ attributes #0 = { returns_twice }
 attributes #1 = { noreturn }
 attributes #2 = { nounwind }
 attributes #3 = { allocsize(0) }
+; CHECK: attributes #{{[0-9]+}} = { "wasm-import-module"="env" "wasm-import-name"="__invoke_void" }
+; CHECK: attributes #{{[0-9]+}} = { "wasm-import-module"="env" "wasm-import-name"="__cxa_find_matching_catch_3" }
+; CHECK: attributes #{{[0-9]+}} = { "wasm-import-module"="env" "wasm-import-name"="__invoke_i8*_i32_%struct.__jmp_buf_tag*" }
+; CHECK: attributes #{{[0-9]+}} = { "wasm-import-module"="env" "wasm-import-name"="__invoke_void_%struct.__jmp_buf_tag*_i32" }
 ; CHECK: attributes #[[ALLOCSIZE_ATTR]] = { allocsize(1) }
 
 !llvm.dbg.cu = !{!2}
diff --git a/llvm/test/CodeGen/WebAssembly/lower-global-dtors.ll b/llvm/test/CodeGen/WebAssembly/lower-global-dtors.ll
index 57338e9109f66..38eec787db124 100644
--- a/llvm/test/CodeGen/WebAssembly/lower-global-dtors.ll
+++ b/llvm/test/CodeGen/WebAssembly/lower-global-dtors.ll
@@ -13,11 +13,18 @@ declare void @orig_dtor1b()
 declare void @orig_dtor1c0()
 declare void @orig_dtor1c1a()
 declare void @orig_dtor1c1b()
-declare void @orig_dtor65536()
+declare void @orig_dtor1c2a()
+declare void @orig_dtor1c2b()
+declare void @orig_dtor1c3()
+declare void @orig_dtor1d()
+declare void @orig_dtor65535()
+declare void @orig_dtor65535c0()
 declare void @after_the_null()
 
-@associated1c0 = external global i8
-@associated1c1 = external global i8
+@associatedc0 = external global i8
+@associatedc1 = external global i8
+@associatedc2 = global i8 42
+@associatedc3 = global i8 84
 
 @llvm.global_ctors = appending global
 [1 x { i32, void ()*, i8* }]
@@ -26,15 +33,20 @@ declare void @after_the_null()
 ]
 
 @llvm.global_dtors = appending global
-[9 x { i32, void ()*, i8* }]
+[14 x { i32, void ()*, i8* }]
 [
   { i32, void ()*, i8* } { i32 0, void ()* @orig_dtor0, i8* null },
   { i32, void ()*, i8* } { i32 1, void ()* @orig_dtor1a, i8* null },
   { i32, void ()*, i8* } { i32 1, void ()* @orig_dtor1b, i8* null },
-  { i32, void ()*, i8* } { i32 1, void ()* @orig_dtor1c0, i8* @associated1c0 },
-  { i32, void ()*, i8* } { i32 1, void ()* @orig_dtor1c1a, i8* @associated1c1 },
-  { i32, void ()*, i8* } { i32 1, void ()* @orig_dtor1c1b, i8* @associated1c1 },
-  { i32, void ()*, i8* } { i32 65535, void ()* @orig_dtor65536, i8* null },
+  { i32, void ()*, i8* } { i32 1, void ()* @orig_dtor1c0, i8* @associatedc0 },
+  { i32, void ()*, i8* } { i32 1, void ()* @orig_dtor1c1a, i8* @associatedc1 },
+  { i32, void ()*, i8* } { i32 1, void ()* @orig_dtor1c1b, i8* @associatedc1 },
+  { i32, void ()*, i8* } { i32 1, void ()* @orig_dtor1c2a, i8* @associatedc2 },
+  { i32, void ()*, i8* } { i32 1, void ()* @orig_dtor1c2b, i8* @associatedc2 },
+  { i32, void ()*, i8* } { i32 1, void ()* @orig_dtor1c3, i8* @associatedc3 },
+  { i32, void ()*, i8* } { i32 1, void ()* @orig_dtor1d, i8* null },
+  { i32, void ()*, i8* } { i32 65535, void ()* @orig_dtor65535c0, i8* @associatedc0 },
+  { i32, void ()*, i8* } { i32 65535, void ()* @orig_dtor65535, i8* null },
   { i32, void ()*, i8* } { i32 65535, void ()* null, i8* null },
   { i32, void ()*, i8* } { i32 65535, void ()* @after_the_null, i8* null }
 ]
@@ -54,14 +66,14 @@ declare void @after_the_null()
 ; CHECK-NEXT: unreachable
 ;      CHECK: end_block
 
-; CHECK-LABEL: .Lcall_dtors.1:
-; CHECK-NEXT: .functype .Lcall_dtors.1 (i32) -> (){{$}}
-; CHECK-NEXT: call            orig_dtor1a{{$}}
+; CHECK-LABEL: .Lcall_dtors.1$0:
+; CHECK-NEXT: .functype .Lcall_dtors.1$0 (i32) -> (){{$}}
 ; CHECK-NEXT: call            orig_dtor1b{{$}}
+; CHECK-NEXT: call            orig_dtor1a{{$}}
 
-; CHECK-LABEL: .Lregister_call_dtors.1:
+; CHECK-LABEL: .Lregister_call_dtors.1$0:
 ; CHECK:      block
-; CHECK-NEXT: i32.const       $push2=, .Lcall_dtors.1{{$}}
+; CHECK-NEXT: i32.const       $push2=, .Lcall_dtors.1$0{{$}}
 ; CHECK-NEXT: i32.const       $push1=, 0
 ; CHECK-NEXT: i32.const       $push0=, __dso_handle
 ; CHECK-NEXT: call            $push3=, __cxa_atexit, $pop2, $pop1, $pop0{{$}}
@@ -70,13 +82,13 @@ declare void @after_the_null()
 ; CHECK-NEXT: unreachable
 ;      CHECK: end_block
 
-; CHECK-LABEL: .Lcall_dtors.1.associated1c0:
-; CHECK-NEXT: .functype .Lcall_dtors.1.associated1c0 (i32) -> (){{$}}
+; CHECK-LABEL: .Lcall_dtors.1$1.associatedc0:
+; CHECK-NEXT: .functype .Lcall_dtors.1$1.associatedc0 (i32) -> (){{$}}
 ; CHECK-NEXT: call            orig_dtor1c0{{$}}
 
-; CHECK-LABEL: .Lregister_call_dtors.1.associated1c0:
+; CHECK-LABEL: .Lregister_call_dtors.1$1.associatedc0:
 ; CHECK:      block
-; CHECK-NEXT: i32.const       $push2=, .Lcall_dtors.1.associated1c0{{$}}
+; CHECK-NEXT: i32.const       $push2=, .Lcall_dtors.1$1.associatedc0{{$}}
 ; CHECK-NEXT: i32.const       $push1=, 0
 ; CHECK-NEXT: i32.const       $push0=, __dso_handle
 ; CHECK-NEXT: call            $push3=, __cxa_atexit, $pop2, $pop1, $pop0{{$}}
@@ -84,14 +96,57 @@ declare void @after_the_null()
 ; CHECK-NEXT: br_if           0, $pop4
 ; CHECK-NEXT: unreachable
 
-; CHECK-LABEL: .Lcall_dtors.1.associated1c1:
-; CHECK-NEXT: .functype .Lcall_dtors.1.associated1c1 (i32) -> (){{$}}
-; CHECK-NEXT: call            orig_dtor1c1a{{$}}
+; CHECK-LABEL: .Lcall_dtors.1$2.associatedc1:
+; CHECK-NEXT: .functype .Lcall_dtors.1$2.associatedc1 (i32) -> (){{$}}
 ; CHECK-NEXT: call            orig_dtor1c1b{{$}}
+; CHECK-NEXT: call            orig_dtor1c1a{{$}}
+
+; CHECK-LABEL: .Lregister_call_dtors.1$2.associatedc1:
+; CHECK:      block
+; CHECK-NEXT: i32.const       $push2=, .Lcall_dtors.1$2.associatedc1{{$}}
+; CHECK-NEXT: i32.const       $push1=, 0
+; CHECK-NEXT: i32.const       $push0=, __dso_handle
+; CHECK-NEXT: call            $push3=, __cxa_atexit, $pop2, $pop1, $pop0{{$}}
+; CHECK-NEXT: i32.eqz         $push4=, $pop3
+; CHECK-NEXT: br_if           0, $pop4
+; CHECK-NEXT: unreachable
+
+; CHECK-LABEL: .Lcall_dtors.1$3.associatedc2:
+; CHECK-NEXT: .functype .Lcall_dtors.1$3.associatedc2 (i32) -> (){{$}}
+; CHECK-NEXT: call            orig_dtor1c2b{{$}}
+; CHECK-NEXT: call            orig_dtor1c2a{{$}}
+
+; CHECK-LABEL: .Lregister_call_dtors.1$3.associatedc2:
+; CHECK:      block
+; CHECK-NEXT: i32.const       $push2=, .Lcall_dtors.1$3.associatedc2{{$}}
+; CHECK-NEXT: i32.const       $push1=, 0
+; CHECK-NEXT: i32.const       $push0=, __dso_handle
+; CHECK-NEXT: call            $push3=, __cxa_atexit, $pop2, $pop1, $pop0{{$}}
+; CHECK-NEXT: i32.eqz         $push4=, $pop3
+; CHECK-NEXT: br_if           0, $pop4
+; CHECK-NEXT: unreachable
+
+; CHECK-LABEL: .Lcall_dtors.1$4.associatedc3:
+; CHECK-NEXT: .functype .Lcall_dtors.1$4.associatedc3 (i32) -> (){{$}}
+; CHECK-NEXT: call            orig_dtor1c3{{$}}
+
+; CHECK-LABEL: .Lregister_call_dtors.1$4.associatedc3:
+; CHECK:      block
+; CHECK-NEXT: i32.const       $push2=, .Lcall_dtors.1$4.associatedc3{{$}}
+; CHECK-NEXT: i32.const       $push1=, 0
+; CHECK-NEXT: i32.const       $push0=, __dso_handle
+; CHECK-NEXT: call            $push3=, __cxa_atexit, $pop2, $pop1, $pop0{{$}}
+; CHECK-NEXT: i32.eqz         $push4=, $pop3
+; CHECK-NEXT: br_if           0, $pop4
+; CHECK-NEXT: unreachable
 
-; CHECK-LABEL: .Lregister_call_dtors.1.associated1c1:
+; CHECK-LABEL: .Lcall_dtors.1$5:
+; CHECK-NEXT: .functype .Lcall_dtors.1$5 (i32) -> (){{$}}
+; CHECK-NEXT: call            orig_dtor1d{{$}}
+
+; CHECK-LABEL: .Lregister_call_dtors.1$5:
 ; CHECK:      block
-; CHECK-NEXT: i32.const       $push2=, .Lcall_dtors.1.associated1c1{{$}}
+; CHECK-NEXT: i32.const       $push2=, .Lcall_dtors.1$5{{$}}
 ; CHECK-NEXT: i32.const       $push1=, 0
 ; CHECK-NEXT: i32.const       $push0=, __dso_handle
 ; CHECK-NEXT: call            $push3=, __cxa_atexit, $pop2, $pop1, $pop0{{$}}
@@ -99,13 +154,17 @@ declare void @after_the_null()
 ; CHECK-NEXT: br_if           0, $pop4
 ; CHECK-NEXT: unreachable
 
-; CHECK-LABEL: .Lcall_dtors:
-; CHECK-NEXT: .functype .Lcall_dtors (i32) -> (){{$}}
-; CHECK-NEXT: call            orig_dtor65536{{$}}
+; CHECK-LABEL: .Lcall_dtors$0.associatedc0:
+; CHECK-NEXT: .functype .Lcall_dtors$0.associatedc0 (i32) -> (){{$}}
+; CHECK-NEXT: call            orig_dtor65535c0
+
+; CHECK-LABEL: .Lcall_dtors$1:
+; CHECK-NEXT: .functype .Lcall_dtors$1 (i32) -> (){{$}}
+; CHECK-NEXT: call            orig_dtor65535{{$}}
 
-; CHECK-LABEL: .Lregister_call_dtors:
+; CHECK-LABEL: .Lregister_call_dtors$1:
 ; CHECK:      block
-; CHECK-NEXT: i32.const       $push2=, .Lcall_dtors{{$}}
+; CHECK-NEXT: i32.const       $push2=, .Lcall_dtors$1{{$}}
 ; CHECK-NEXT: i32.const       $push1=, 0
 ; CHECK-NEXT: i32.const       $push0=, __dso_handle
 ; CHECK-NEXT: call            $push3=, __cxa_atexit, $pop2, $pop1, $pop0{{$}}
@@ -116,11 +175,14 @@ declare void @after_the_null()
 ; CHECK-LABEL: .section .init_array.0,"",@
 ;      CHECK: .int32  .Lregister_call_dtors.0{{$}}
 ; CHECK-LABEL: .section .init_array.1,"",@
-;      CHECK: .int32  .Lregister_call_dtors.1{{$}}
+;      CHECK: .int32  .Lregister_call_dtors.1$0{{$}}
+; CHECK-NEXT: .int32  .Lregister_call_dtors.1$3.associatedc2{{$}}
+; CHECK-NEXT: .int32  .Lregister_call_dtors.1$4.associatedc3{{$}}
+; CHECK-NEXT: .int32  .Lregister_call_dtors.1$5{{$}}
 ; CHECK-LABEL: .section .init_array.200,"",@
 ;      CHECK: .int32  orig_ctor{{$}}
 ; CHECK-LABEL: .section .init_array,"",@
-;      CHECK: .int32  .Lregister_call_dtors{{$}}
+;      CHECK: .int32  .Lregister_call_dtors$1{{$}}
 
 ; CHECK-LABEL: .weak __dso_handle
 
diff --git a/llvm/test/CodeGen/X86/GlobalISel/select-copy.mir b/llvm/test/CodeGen/X86/GlobalISel/select-copy.mir
index c4d6be6ca16f9..72a6ed15f63b5 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/select-copy.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/select-copy.mir
@@ -40,10 +40,10 @@ regBankSelected: true
 registers:
   - { id: 0, class: gpr, preferred-register: '' }
   - { id: 1, class: gpr, preferred-register: '' }
-# ALL           %0:gr8 = COPY $al
-# ALL-NEXT      %1:gr32 = MOVZX32rr8 %0
-# ALL-NEXT      $eax = COPY %1
-# ALL-NEXT      RET 0, implicit $eax
+# ALL:          %0:gr8 = COPY $al
+# ALL-NEXT:     %1:gr32 = MOVZX32rr8 %0
+# ALL-NEXT:     $eax = COPY %1
+# ALL-NEXT:     RET 0, implicit $eax
 body:             |
   bb.1 (%ir-block.0):
     liveins: $eax
@@ -95,11 +95,13 @@ registers:
   - { id: 0, class: gpr, preferred-register: '' }
   - { id: 1, class: gpr, preferred-register: '' }
   - { id: 2, class: gpr, preferred-register: '' }
-# ALL           %0:gr16 = COPY $ax
-# ALL-NEXT      %1:gr8 = COPY %0.sub_8bit
-# ALL-NEXT      %2:gr32 = MOVZX32rr8 %1
-# ALL-NEXT      $eax = COPY %2
-# ALL-NEXT      RET 0, implicit $eax
+# ALL:          %0:gr16 = COPY $ax
+# X32-NEXT:     %3:gr16_abcd = COPY %0
+# X32-NEXT:     %1:gr8_abcd_l = COPY %3.sub_8bit
+# X64-NEXT:     %1:gr8 = COPY %0.sub_8bit
+# ALL-NEXT:     %2:gr32 = MOVZX32rr8 %1
+# ALL-NEXT:     $eax = COPY %2
+# ALL-NEXT:     RET 0, implicit $eax
 body:             |
   bb.1 (%ir-block.0):
     liveins: $eax
@@ -125,11 +127,11 @@ registers:
   - { id: 0, class: gpr, preferred-register: '' }
   - { id: 1, class: gpr, preferred-register: '' }
   - { id: 2, class: gpr, preferred-register: '' }
-# ALL           %0:gr32 = COPY $eax
-# ALL-NEXT      %1:gr16 = COPY %0.sub_16bit
-# ALL-NEXT      %2:gr32 = MOVZX32rr16 %1
-# ALL-NEXT      $eax = COPY %2
-# ALL-NEXT      RET 0, implicit $eax
+# ALL:          %0:gr32 = COPY $eax
+# ALL-NEXT:     %1:gr16 = COPY %0.sub_16bit
+# ALL-NEXT:     %2:gr32 = MOVZX32rr16 %1
+# ALL-NEXT:     $eax = COPY %2
+# ALL-NEXT:     RET 0, implicit $eax
 body:             |
   bb.1 (%ir-block.0):
     liveins: $eax
@@ -156,11 +158,13 @@ registers:
   - { id: 0, class: gpr, preferred-register: '' }
   - { id: 1, class: gpr, preferred-register: '' }
   - { id: 2, class: gpr, preferred-register: '' }
-# ALL           %0:gr32[[ABCD]] = COPY $edx
-# ALL-NEXT      %1:gr8 = COPY %0.sub_8bit
-# ALL-NEXT      %2:gr32 = MOVZX32rr8 %1
-# ALL-NEXT      $eax = COPY %2
-# ALL-NEXT      RET 0, implicit $eax
+# ALL:          %0:gr32 = COPY $edx
+# X32-NEXT:     %3:gr32_abcd = COPY %0
+# X32-NEXT:     %1:gr8_abcd_l = COPY %3.sub_8bit
+# X64-NEXT:     %1:gr8 = COPY %0.sub_8bit
+# ALL-NEXT:     %2:gr32 = MOVZX32rr8 %1
+# ALL-NEXT:     $eax = COPY %2
+# ALL-NEXT:     RET 0, implicit $eax
 body:             |
   bb.1 (%ir-block.0):
     liveins: $eax,$edx
diff --git a/llvm/test/CodeGen/X86/GlobalISel/select-fconstant.mir b/llvm/test/CodeGen/X86/GlobalISel/select-fconstant.mir
index 47be0d593e823..216cd2c4143f8 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/select-fconstant.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/select-fconstant.mir
@@ -35,7 +35,7 @@ body:             |
     ; CHECK_NOPIC64: RET 0, implicit $xmm0
     ; CHECK_LARGE64-LABEL: name: test_float
     ; CHECK_LARGE64: [[MOV64ri:%[0-9]+]]:gr64 = MOV64ri %const.0
-    ; CHECK_LARGE64: [[MOVSSrm_alt:%[0-9]+]]:fr32 = MOVSSrm_alt [[MOV64ri]], 1, $noreg, 0, $noreg :: (load 8 from constant-pool, align 32)
+    ; CHECK_LARGE64: [[MOVSSrm_alt:%[0-9]+]]:fr32 = MOVSSrm_alt [[MOV64ri]], 1, $noreg, 0, $noreg :: (load 8 from constant-pool, align 4)
     ; CHECK_LARGE64: [[COPY:%[0-9]+]]:vr128 = COPY [[MOVSSrm_alt]]
     ; CHECK_LARGE64: $xmm0 = COPY [[COPY]]
     ; CHECK_LARGE64: RET 0, implicit $xmm0
@@ -82,7 +82,7 @@ body:             |
     ; CHECK_NOPIC64: RET 0, implicit $xmm0
     ; CHECK_LARGE64-LABEL: name: test_double
     ; CHECK_LARGE64: [[MOV64ri:%[0-9]+]]:gr64 = MOV64ri %const.0
-    ; CHECK_LARGE64: [[MOVSDrm_alt:%[0-9]+]]:fr64 = MOVSDrm_alt [[MOV64ri]], 1, $noreg, 0, $noreg :: (load 8 from constant-pool, align 64)
+    ; CHECK_LARGE64: [[MOVSDrm_alt:%[0-9]+]]:fr64 = MOVSDrm_alt [[MOV64ri]], 1, $noreg, 0, $noreg :: (load 8 from constant-pool)
     ; CHECK_LARGE64: [[COPY:%[0-9]+]]:vr128 = COPY [[MOVSDrm_alt]]
     ; CHECK_LARGE64: $xmm0 = COPY [[COPY]]
     ; CHECK_LARGE64: RET 0, implicit $xmm0
diff --git a/llvm/test/CodeGen/X86/O0-pipeline.ll b/llvm/test/CodeGen/X86/O0-pipeline.ll
index 0bcf0da3c9221..6b6fed0c7f203 100644
--- a/llvm/test/CodeGen/X86/O0-pipeline.ll
+++ b/llvm/test/CodeGen/X86/O0-pipeline.ll
@@ -55,6 +55,10 @@
 ; CHECK-NEXT:       Fast Register Allocator
 ; CHECK-NEXT:       Bundle Machine CFG Edges
 ; CHECK-NEXT:       X86 FP Stackifier
+; CHECK-NEXT:       MachineDominator Tree Construction
+; CHECK-NEXT:       Machine Natural Loop Construction
+; CHECK-NEXT:       Machine Dominance Frontier Construction
+; CHECK-NEXT:       X86 Load Value Injection (LVI) Load Hardening
 ; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:       Machine Optimization Remark Emitter
 ; CHECK-NEXT:       Prologue/Epilogue Insertion & Frame Finalization
@@ -72,8 +76,9 @@
 ; CHECK-NEXT:       Contiguously Lay Out Funclets
 ; CHECK-NEXT:       StackMap Liveness Analysis
 ; CHECK-NEXT:       Live DEBUG_VALUE analysis
-; CHECK-NEXT:       X86 Retpoline Thunks
+; CHECK-NEXT:       X86 Indirect Thunks
 ; CHECK-NEXT:       Check CFA info and insert CFI instructions if needed
+; CHECK-NEXT:       X86 Load Value Injection (LVI) Ret-Hardening
 ; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:       Machine Optimization Remark Emitter
 ; CHECK-NEXT:       X86 Assembly Printer
diff --git a/llvm/test/CodeGen/X86/O3-pipeline.ll b/llvm/test/CodeGen/X86/O3-pipeline.ll
index c72676b24cfd3..92b2b818743b8 100644
--- a/llvm/test/CodeGen/X86/O3-pipeline.ll
+++ b/llvm/test/CodeGen/X86/O3-pipeline.ll
@@ -51,6 +51,7 @@
 ; CHECK-NEXT:       Expand reduction intrinsics
 ; CHECK-NEXT:       Dominator Tree Construction
 ; CHECK-NEXT:       Interleaved Access Pass
+; CHECK-NEXT:       X86 Partial Reduction
 ; CHECK-NEXT:       Expand indirectbr instructions
 ; CHECK-NEXT:       Dominator Tree Construction
 ; CHECK-NEXT:       Natural Loop Information
@@ -138,9 +139,11 @@
 ; CHECK-NEXT:       Machine Loop Invariant Code Motion
 ; CHECK-NEXT:       Bundle Machine CFG Edges
 ; CHECK-NEXT:       X86 FP Stackifier
+; CHECK-NEXT:       MachineDominator Tree Construction
+; CHECK-NEXT:       Machine Dominance Frontier Construction
+; CHECK-NEXT:       X86 Load Value Injection (LVI) Load Hardening
 ; CHECK-NEXT:       PostRA Machine Sink
 ; CHECK-NEXT:       Machine Block Frequency Analysis
-; CHECK-NEXT:       MachineDominator Tree Construction
 ; CHECK-NEXT:       MachinePostDominator Tree Construction
 ; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:       Machine Optimization Remark Emitter
@@ -181,8 +184,9 @@
 ; CHECK-NEXT:       Contiguously Lay Out Funclets
 ; CHECK-NEXT:       StackMap Liveness Analysis
 ; CHECK-NEXT:       Live DEBUG_VALUE analysis
-; CHECK-NEXT:       X86 Retpoline Thunks
+; CHECK-NEXT:       X86 Indirect Thunks
 ; CHECK-NEXT:       Check CFA info and insert CFI instructions if needed
+; CHECK-NEXT:       X86 Load Value Injection (LVI) Ret-Hardening
 ; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:       Machine Optimization Remark Emitter
 ; CHECK-NEXT:       X86 Assembly Printer
diff --git a/llvm/test/CodeGen/X86/atomic-idempotent.ll b/llvm/test/CodeGen/X86/atomic-idempotent.ll
index 50c51fc0555c9..5d93e23b3015c 100644
--- a/llvm/test/CodeGen/X86/atomic-idempotent.ll
+++ b/llvm/test/CodeGen/X86/atomic-idempotent.ll
@@ -1,6 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs | FileCheck %s --check-prefix=CHECK --check-prefix=X64
-; RUN: llc < %s -mtriple=i686-- -mattr=+sse2 -verify-machineinstrs | FileCheck %s --check-prefix=CHECK --check-prefix=X86
+; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs                           | FileCheck %s --check-prefixes=CHECK,X64
+; RUN: llc < %s -mtriple=i686-- -verify-machineinstrs           -mattr=+sse2      | FileCheck %s --check-prefixes=CHECK,X86,X86-GENERIC,X86-SSE2
+; RUN: llc < %s -mtriple=i686-- -verify-machineinstrs -mcpu=slm -mattr=-sse2      | FileCheck %s --check-prefixes=CHECK,X86,X86-GENERIC,X86-SLM
+; RUN: llc < %s -mtriple=i686-- -verify-machineinstrs -mcpu=goldmont -mattr=-sse2 | FileCheck %s --check-prefixes=CHECK,X86,X86-GENERIC,X86-SLM
+; RUN: llc < %s -mtriple=i686-- -verify-machineinstrs -mcpu=knl -mattr=-sse2      | FileCheck %s --check-prefixes=CHECK,X86,X86-GENERIC,X86-SLM
+; RUN: llc < %s -mtriple=i686-- -verify-machineinstrs -mcpu=atom -mattr=-sse2     | FileCheck %s --check-prefixes=CHECK,X86,X86-ATOM
 
 ; On x86, an atomic rmw operation that does not modify the value in memory
 ; (such as atomic add 0) can be replaced by an mfence followed by a mov.
@@ -14,12 +18,30 @@ define i8 @add8(i8* %p) {
 ; X64-NEXT:    movb (%rdi), %al
 ; X64-NEXT:    retq
 ;
-; X86-LABEL: add8:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    mfence
-; X86-NEXT:    movb (%eax), %al
-; X86-NEXT:    retl
+; X86-SSE2-LABEL: add8:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    mfence
+; X86-SSE2-NEXT:    movb (%eax), %al
+; X86-SSE2-NEXT:    retl
+;
+; X86-SLM-LABEL: add8:
+; X86-SLM:       # %bb.0:
+; X86-SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SLM-NEXT:    xorl %eax, %eax
+; X86-SLM-NEXT:    lock xaddb %al, (%ecx)
+; X86-SLM-NEXT:    # kill: def $al killed $al killed $eax
+; X86-SLM-NEXT:    retl
+;
+; X86-ATOM-LABEL: add8:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-ATOM-NEXT:    xorl %eax, %eax
+; X86-ATOM-NEXT:    lock xaddb %al, (%ecx)
+; X86-ATOM-NEXT:    # kill: def $al killed $al killed $eax
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    retl
   %1 = atomicrmw add i8* %p, i8 0 monotonic
   ret i8 %1
 }
@@ -31,12 +53,36 @@ define i16 @or16(i16* %p) {
 ; X64-NEXT:    movzwl (%rdi), %eax
 ; X64-NEXT:    retq
 ;
-; X86-LABEL: or16:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    mfence
-; X86-NEXT:    movzwl (%eax), %eax
-; X86-NEXT:    retl
+; X86-SSE2-LABEL: or16:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    mfence
+; X86-SSE2-NEXT:    movzwl (%eax), %eax
+; X86-SSE2-NEXT:    retl
+;
+; X86-SLM-LABEL: or16:
+; X86-SLM:       # %bb.0:
+; X86-SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SLM-NEXT:    movzwl (%ecx), %eax
+; X86-SLM-NEXT:    .p2align 4, 0x90
+; X86-SLM-NEXT:  .LBB1_1: # %atomicrmw.start
+; X86-SLM-NEXT:    # =>This Inner Loop Header: Depth=1
+; X86-SLM-NEXT:    lock cmpxchgw %ax, (%ecx)
+; X86-SLM-NEXT:    jne .LBB1_1
+; X86-SLM-NEXT:  # %bb.2: # %atomicrmw.end
+; X86-SLM-NEXT:    retl
+;
+; X86-ATOM-LABEL: or16:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-ATOM-NEXT:    movzwl (%ecx), %eax
+; X86-ATOM-NEXT:    .p2align 4, 0x90
+; X86-ATOM-NEXT:  .LBB1_1: # %atomicrmw.start
+; X86-ATOM-NEXT:    # =>This Inner Loop Header: Depth=1
+; X86-ATOM-NEXT:    lock cmpxchgw %ax, (%ecx)
+; X86-ATOM-NEXT:    jne .LBB1_1
+; X86-ATOM-NEXT:  # %bb.2: # %atomicrmw.end
+; X86-ATOM-NEXT:    retl
   %1 = atomicrmw or i16* %p, i16 0 acquire
   ret i16 %1
 }
@@ -48,12 +94,36 @@ define i32 @xor32(i32* %p) {
 ; X64-NEXT:    movl (%rdi), %eax
 ; X64-NEXT:    retq
 ;
-; X86-LABEL: xor32:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    mfence
-; X86-NEXT:    movl (%eax), %eax
-; X86-NEXT:    retl
+; X86-SSE2-LABEL: xor32:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    mfence
+; X86-SSE2-NEXT:    movl (%eax), %eax
+; X86-SSE2-NEXT:    retl
+;
+; X86-SLM-LABEL: xor32:
+; X86-SLM:       # %bb.0:
+; X86-SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SLM-NEXT:    movl (%ecx), %eax
+; X86-SLM-NEXT:    .p2align 4, 0x90
+; X86-SLM-NEXT:  .LBB2_1: # %atomicrmw.start
+; X86-SLM-NEXT:    # =>This Inner Loop Header: Depth=1
+; X86-SLM-NEXT:    lock cmpxchgl %eax, (%ecx)
+; X86-SLM-NEXT:    jne .LBB2_1
+; X86-SLM-NEXT:  # %bb.2: # %atomicrmw.end
+; X86-SLM-NEXT:    retl
+;
+; X86-ATOM-LABEL: xor32:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-ATOM-NEXT:    movl (%ecx), %eax
+; X86-ATOM-NEXT:    .p2align 4, 0x90
+; X86-ATOM-NEXT:  .LBB2_1: # %atomicrmw.start
+; X86-ATOM-NEXT:    # =>This Inner Loop Header: Depth=1
+; X86-ATOM-NEXT:    lock cmpxchgl %eax, (%ecx)
+; X86-ATOM-NEXT:    jne .LBB2_1
+; X86-ATOM-NEXT:  # %bb.2: # %atomicrmw.end
+; X86-ATOM-NEXT:    retl
   %1 = atomicrmw xor i32* %p, i32 0 release
   ret i32 %1
 }
@@ -105,44 +175,124 @@ define i128 @or128(i128* %p) {
 ; X64-NEXT:    .cfi_def_cfa_offset 8
 ; X64-NEXT:    retq
 ;
-; X86-LABEL: or128:
-; X86:       # %bb.0:
-; X86-NEXT:    pushl %ebp
-; X86-NEXT:    .cfi_def_cfa_offset 8
-; X86-NEXT:    .cfi_offset %ebp, -8
-; X86-NEXT:    movl %esp, %ebp
-; X86-NEXT:    .cfi_def_cfa_register %ebp
-; X86-NEXT:    pushl %edi
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    andl $-8, %esp
-; X86-NEXT:    subl $16, %esp
-; X86-NEXT:    .cfi_offset %esi, -16
-; X86-NEXT:    .cfi_offset %edi, -12
-; X86-NEXT:    movl 8(%ebp), %esi
-; X86-NEXT:    movl %esp, %eax
-; X86-NEXT:    pushl $0
-; X86-NEXT:    pushl $0
-; X86-NEXT:    pushl $0
-; X86-NEXT:    pushl $0
-; X86-NEXT:    pushl 12(%ebp)
-; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll __sync_fetch_and_or_16
-; X86-NEXT:    addl $20, %esp
-; X86-NEXT:    movl (%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl %edi, 8(%esi)
-; X86-NEXT:    movl %edx, 12(%esi)
-; X86-NEXT:    movl %eax, (%esi)
-; X86-NEXT:    movl %ecx, 4(%esi)
-; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    leal -8(%ebp), %esp
-; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
-; X86-NEXT:    popl %ebp
-; X86-NEXT:    .cfi_def_cfa %esp, 4
-; X86-NEXT:    retl $4
+; X86-SSE2-LABEL: or128:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE2-NEXT:    .cfi_offset %ebp, -8
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    .cfi_def_cfa_register %ebp
+; X86-SSE2-NEXT:    pushl %edi
+; X86-SSE2-NEXT:    pushl %esi
+; X86-SSE2-NEXT:    andl $-8, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    .cfi_offset %esi, -16
+; X86-SSE2-NEXT:    .cfi_offset %edi, -12
+; X86-SSE2-NEXT:    movl 8(%ebp), %esi
+; X86-SSE2-NEXT:    movl %esp, %eax
+; X86-SSE2-NEXT:    pushl $0
+; X86-SSE2-NEXT:    pushl $0
+; X86-SSE2-NEXT:    pushl $0
+; X86-SSE2-NEXT:    pushl $0
+; X86-SSE2-NEXT:    pushl 12(%ebp)
+; X86-SSE2-NEXT:    pushl %eax
+; X86-SSE2-NEXT:    calll __sync_fetch_and_or_16
+; X86-SSE2-NEXT:    addl $20, %esp
+; X86-SSE2-NEXT:    movl (%esp), %eax
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-SSE2-NEXT:    movl %edi, 8(%esi)
+; X86-SSE2-NEXT:    movl %edx, 12(%esi)
+; X86-SSE2-NEXT:    movl %eax, (%esi)
+; X86-SSE2-NEXT:    movl %ecx, 4(%esi)
+; X86-SSE2-NEXT:    movl %esi, %eax
+; X86-SSE2-NEXT:    leal -8(%ebp), %esp
+; X86-SSE2-NEXT:    popl %esi
+; X86-SSE2-NEXT:    popl %edi
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    .cfi_def_cfa %esp, 4
+; X86-SSE2-NEXT:    retl $4
+;
+; X86-SLM-LABEL: or128:
+; X86-SLM:       # %bb.0:
+; X86-SLM-NEXT:    pushl %ebp
+; X86-SLM-NEXT:    .cfi_def_cfa_offset 8
+; X86-SLM-NEXT:    .cfi_offset %ebp, -8
+; X86-SLM-NEXT:    movl %esp, %ebp
+; X86-SLM-NEXT:    .cfi_def_cfa_register %ebp
+; X86-SLM-NEXT:    pushl %edi
+; X86-SLM-NEXT:    pushl %esi
+; X86-SLM-NEXT:    andl $-8, %esp
+; X86-SLM-NEXT:    subl $16, %esp
+; X86-SLM-NEXT:    .cfi_offset %esi, -16
+; X86-SLM-NEXT:    .cfi_offset %edi, -12
+; X86-SLM-NEXT:    movl 8(%ebp), %esi
+; X86-SLM-NEXT:    movl 12(%ebp), %eax
+; X86-SLM-NEXT:    movl %esp, %ecx
+; X86-SLM-NEXT:    pushl $0
+; X86-SLM-NEXT:    pushl $0
+; X86-SLM-NEXT:    pushl $0
+; X86-SLM-NEXT:    pushl $0
+; X86-SLM-NEXT:    pushl %eax
+; X86-SLM-NEXT:    pushl %ecx
+; X86-SLM-NEXT:    calll __sync_fetch_and_or_16
+; X86-SLM-NEXT:    addl $20, %esp
+; X86-SLM-NEXT:    movl (%esp), %eax
+; X86-SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SLM-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-SLM-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-SLM-NEXT:    movl %edi, 8(%esi)
+; X86-SLM-NEXT:    movl %edx, 12(%esi)
+; X86-SLM-NEXT:    movl %eax, (%esi)
+; X86-SLM-NEXT:    movl %ecx, 4(%esi)
+; X86-SLM-NEXT:    movl %esi, %eax
+; X86-SLM-NEXT:    leal -8(%ebp), %esp
+; X86-SLM-NEXT:    popl %esi
+; X86-SLM-NEXT:    popl %edi
+; X86-SLM-NEXT:    popl %ebp
+; X86-SLM-NEXT:    .cfi_def_cfa %esp, 4
+; X86-SLM-NEXT:    retl $4
+;
+; X86-ATOM-LABEL: or128:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    pushl %ebp
+; X86-ATOM-NEXT:    .cfi_def_cfa_offset 8
+; X86-ATOM-NEXT:    .cfi_offset %ebp, -8
+; X86-ATOM-NEXT:    leal (%esp), %ebp
+; X86-ATOM-NEXT:    .cfi_def_cfa_register %ebp
+; X86-ATOM-NEXT:    pushl %edi
+; X86-ATOM-NEXT:    pushl %esi
+; X86-ATOM-NEXT:    andl $-8, %esp
+; X86-ATOM-NEXT:    leal -{{[0-9]+}}(%esp), %esp
+; X86-ATOM-NEXT:    .cfi_offset %esi, -16
+; X86-ATOM-NEXT:    .cfi_offset %edi, -12
+; X86-ATOM-NEXT:    movl 8(%ebp), %esi
+; X86-ATOM-NEXT:    movl 12(%ebp), %eax
+; X86-ATOM-NEXT:    movl %esp, %ecx
+; X86-ATOM-NEXT:    pushl $0
+; X86-ATOM-NEXT:    pushl $0
+; X86-ATOM-NEXT:    pushl $0
+; X86-ATOM-NEXT:    pushl $0
+; X86-ATOM-NEXT:    pushl %eax
+; X86-ATOM-NEXT:    pushl %ecx
+; X86-ATOM-NEXT:    calll __sync_fetch_and_or_16
+; X86-ATOM-NEXT:    leal {{[0-9]+}}(%esp), %esp
+; X86-ATOM-NEXT:    movl (%esp), %ecx
+; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-ATOM-NEXT:    movl %eax, 8(%esi)
+; X86-ATOM-NEXT:    movl %edi, 12(%esi)
+; X86-ATOM-NEXT:    movl %ecx, (%esi)
+; X86-ATOM-NEXT:    movl %esi, %eax
+; X86-ATOM-NEXT:    movl %edx, 4(%esi)
+; X86-ATOM-NEXT:    leal -8(%ebp), %esp
+; X86-ATOM-NEXT:    popl %esi
+; X86-ATOM-NEXT:    popl %edi
+; X86-ATOM-NEXT:    popl %ebp
+; X86-ATOM-NEXT:    .cfi_def_cfa %esp, 4
+; X86-ATOM-NEXT:    retl $4
   %1 = atomicrmw or i128* %p, i128 0 monotonic
   ret i128 %1
 }
@@ -155,49 +305,137 @@ define i32 @and32 (i32* %p) {
 ; X64-NEXT:    movl (%rdi), %eax
 ; X64-NEXT:    retq
 ;
-; X86-LABEL: and32:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    mfence
-; X86-NEXT:    movl (%eax), %eax
-; X86-NEXT:    retl
+; X86-SSE2-LABEL: and32:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    mfence
+; X86-SSE2-NEXT:    movl (%eax), %eax
+; X86-SSE2-NEXT:    retl
+;
+; X86-SLM-LABEL: and32:
+; X86-SLM:       # %bb.0:
+; X86-SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SLM-NEXT:    movl (%ecx), %eax
+; X86-SLM-NEXT:    .p2align 4, 0x90
+; X86-SLM-NEXT:  .LBB5_1: # %atomicrmw.start
+; X86-SLM-NEXT:    # =>This Inner Loop Header: Depth=1
+; X86-SLM-NEXT:    lock cmpxchgl %eax, (%ecx)
+; X86-SLM-NEXT:    jne .LBB5_1
+; X86-SLM-NEXT:  # %bb.2: # %atomicrmw.end
+; X86-SLM-NEXT:    retl
+;
+; X86-ATOM-LABEL: and32:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-ATOM-NEXT:    movl (%ecx), %eax
+; X86-ATOM-NEXT:    .p2align 4, 0x90
+; X86-ATOM-NEXT:  .LBB5_1: # %atomicrmw.start
+; X86-ATOM-NEXT:    # =>This Inner Loop Header: Depth=1
+; X86-ATOM-NEXT:    lock cmpxchgl %eax, (%ecx)
+; X86-ATOM-NEXT:    jne .LBB5_1
+; X86-ATOM-NEXT:  # %bb.2: # %atomicrmw.end
+; X86-ATOM-NEXT:    retl
   %1 = atomicrmw and i32* %p, i32 -1 acq_rel
   ret i32 %1
 }
 
 define void @or32_nouse_monotonic(i32* %p) {
-; CHECK-LABEL: or32_nouse_monotonic:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    #MEMBARRIER
-; CHECK-NEXT:    ret{{[l|q]}}
+; X64-LABEL: or32_nouse_monotonic:
+; X64:       # %bb.0:
+; X64-NEXT:    #MEMBARRIER
+; X64-NEXT:    retq
+;
+; X86-GENERIC-LABEL: or32_nouse_monotonic:
+; X86-GENERIC:       # %bb.0:
+; X86-GENERIC-NEXT:    #MEMBARRIER
+; X86-GENERIC-NEXT:    retl
+;
+; X86-ATOM-LABEL: or32_nouse_monotonic:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    #MEMBARRIER
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    retl
   atomicrmw or i32* %p, i32 0 monotonic
   ret void
 }
 
 
 define void @or32_nouse_acquire(i32* %p) {
-; CHECK-LABEL: or32_nouse_acquire:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    #MEMBARRIER
-; CHECK-NEXT:    ret{{[l|q]}}
+; X64-LABEL: or32_nouse_acquire:
+; X64:       # %bb.0:
+; X64-NEXT:    #MEMBARRIER
+; X64-NEXT:    retq
+;
+; X86-GENERIC-LABEL: or32_nouse_acquire:
+; X86-GENERIC:       # %bb.0:
+; X86-GENERIC-NEXT:    #MEMBARRIER
+; X86-GENERIC-NEXT:    retl
+;
+; X86-ATOM-LABEL: or32_nouse_acquire:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    #MEMBARRIER
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    retl
   atomicrmw or i32* %p, i32 0 acquire
   ret void
 }
 
 define void @or32_nouse_release(i32* %p) {
-; CHECK-LABEL: or32_nouse_release:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    #MEMBARRIER
-; CHECK-NEXT:    ret{{[l|q]}}
+; X64-LABEL: or32_nouse_release:
+; X64:       # %bb.0:
+; X64-NEXT:    #MEMBARRIER
+; X64-NEXT:    retq
+;
+; X86-GENERIC-LABEL: or32_nouse_release:
+; X86-GENERIC:       # %bb.0:
+; X86-GENERIC-NEXT:    #MEMBARRIER
+; X86-GENERIC-NEXT:    retl
+;
+; X86-ATOM-LABEL: or32_nouse_release:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    #MEMBARRIER
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    retl
   atomicrmw or i32* %p, i32 0 release
   ret void
 }
 
 define void @or32_nouse_acq_rel(i32* %p) {
-; CHECK-LABEL: or32_nouse_acq_rel:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    #MEMBARRIER
-; CHECK-NEXT:    ret{{[l|q]}}
+; X64-LABEL: or32_nouse_acq_rel:
+; X64:       # %bb.0:
+; X64-NEXT:    #MEMBARRIER
+; X64-NEXT:    retq
+;
+; X86-GENERIC-LABEL: or32_nouse_acq_rel:
+; X86-GENERIC:       # %bb.0:
+; X86-GENERIC-NEXT:    #MEMBARRIER
+; X86-GENERIC-NEXT:    retl
+;
+; X86-ATOM-LABEL: or32_nouse_acq_rel:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    #MEMBARRIER
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    retl
   atomicrmw or i32* %p, i32 0 acq_rel
   ret void
 }
@@ -208,10 +446,21 @@ define void @or32_nouse_seq_cst(i32* %p) {
 ; X64-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    retq
 ;
-; X86-LABEL: or32_nouse_seq_cst:
-; X86:       # %bb.0:
-; X86-NEXT:    lock orl $0, (%esp)
-; X86-NEXT:    retl
+; X86-GENERIC-LABEL: or32_nouse_seq_cst:
+; X86-GENERIC:       # %bb.0:
+; X86-GENERIC-NEXT:    lock orl $0, (%esp)
+; X86-GENERIC-NEXT:    retl
+;
+; X86-ATOM-LABEL: or32_nouse_seq_cst:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    lock orl $0, (%esp)
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    retl
   atomicrmw or i32* %p, i32 0 seq_cst
   ret void
 }
@@ -264,28 +513,76 @@ define void @or128_nouse_seq_cst(i128* %p) {
 ; X64-NEXT:    .cfi_def_cfa_offset 8
 ; X64-NEXT:    retq
 ;
-; X86-LABEL: or128_nouse_seq_cst:
-; X86:       # %bb.0:
-; X86-NEXT:    pushl %ebp
-; X86-NEXT:    .cfi_def_cfa_offset 8
-; X86-NEXT:    .cfi_offset %ebp, -8
-; X86-NEXT:    movl %esp, %ebp
-; X86-NEXT:    .cfi_def_cfa_register %ebp
-; X86-NEXT:    andl $-8, %esp
-; X86-NEXT:    subl $16, %esp
-; X86-NEXT:    movl %esp, %eax
-; X86-NEXT:    pushl $0
-; X86-NEXT:    pushl $0
-; X86-NEXT:    pushl $0
-; X86-NEXT:    pushl $0
-; X86-NEXT:    pushl 8(%ebp)
-; X86-NEXT:    pushl %eax
-; X86-NEXT:    calll __sync_fetch_and_or_16
-; X86-NEXT:    addl $20, %esp
-; X86-NEXT:    movl %ebp, %esp
-; X86-NEXT:    popl %ebp
-; X86-NEXT:    .cfi_def_cfa %esp, 4
-; X86-NEXT:    retl
+; X86-SSE2-LABEL: or128_nouse_seq_cst:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE2-NEXT:    .cfi_offset %ebp, -8
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    .cfi_def_cfa_register %ebp
+; X86-SSE2-NEXT:    andl $-8, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    movl %esp, %eax
+; X86-SSE2-NEXT:    pushl $0
+; X86-SSE2-NEXT:    pushl $0
+; X86-SSE2-NEXT:    pushl $0
+; X86-SSE2-NEXT:    pushl $0
+; X86-SSE2-NEXT:    pushl 8(%ebp)
+; X86-SSE2-NEXT:    pushl %eax
+; X86-SSE2-NEXT:    calll __sync_fetch_and_or_16
+; X86-SSE2-NEXT:    addl $20, %esp
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    .cfi_def_cfa %esp, 4
+; X86-SSE2-NEXT:    retl
+;
+; X86-SLM-LABEL: or128_nouse_seq_cst:
+; X86-SLM:       # %bb.0:
+; X86-SLM-NEXT:    pushl %ebp
+; X86-SLM-NEXT:    .cfi_def_cfa_offset 8
+; X86-SLM-NEXT:    .cfi_offset %ebp, -8
+; X86-SLM-NEXT:    movl %esp, %ebp
+; X86-SLM-NEXT:    .cfi_def_cfa_register %ebp
+; X86-SLM-NEXT:    andl $-8, %esp
+; X86-SLM-NEXT:    subl $16, %esp
+; X86-SLM-NEXT:    movl 8(%ebp), %eax
+; X86-SLM-NEXT:    movl %esp, %ecx
+; X86-SLM-NEXT:    pushl $0
+; X86-SLM-NEXT:    pushl $0
+; X86-SLM-NEXT:    pushl $0
+; X86-SLM-NEXT:    pushl $0
+; X86-SLM-NEXT:    pushl %eax
+; X86-SLM-NEXT:    pushl %ecx
+; X86-SLM-NEXT:    calll __sync_fetch_and_or_16
+; X86-SLM-NEXT:    addl $20, %esp
+; X86-SLM-NEXT:    movl %ebp, %esp
+; X86-SLM-NEXT:    popl %ebp
+; X86-SLM-NEXT:    .cfi_def_cfa %esp, 4
+; X86-SLM-NEXT:    retl
+;
+; X86-ATOM-LABEL: or128_nouse_seq_cst:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    pushl %ebp
+; X86-ATOM-NEXT:    .cfi_def_cfa_offset 8
+; X86-ATOM-NEXT:    .cfi_offset %ebp, -8
+; X86-ATOM-NEXT:    leal (%esp), %ebp
+; X86-ATOM-NEXT:    .cfi_def_cfa_register %ebp
+; X86-ATOM-NEXT:    andl $-8, %esp
+; X86-ATOM-NEXT:    leal -{{[0-9]+}}(%esp), %esp
+; X86-ATOM-NEXT:    movl 8(%ebp), %eax
+; X86-ATOM-NEXT:    movl %esp, %ecx
+; X86-ATOM-NEXT:    pushl $0
+; X86-ATOM-NEXT:    pushl $0
+; X86-ATOM-NEXT:    pushl $0
+; X86-ATOM-NEXT:    pushl $0
+; X86-ATOM-NEXT:    pushl %eax
+; X86-ATOM-NEXT:    pushl %ecx
+; X86-ATOM-NEXT:    calll __sync_fetch_and_or_16
+; X86-ATOM-NEXT:    leal {{[0-9]+}}(%esp), %esp
+; X86-ATOM-NEXT:    movl %ebp, %esp
+; X86-ATOM-NEXT:    popl %ebp
+; X86-ATOM-NEXT:    .cfi_def_cfa %esp, 4
+; X86-ATOM-NEXT:    retl
   atomicrmw or i128* %p, i128 0 seq_cst
   ret void
 }
@@ -297,10 +594,21 @@ define void @or16_nouse_seq_cst(i16* %p) {
 ; X64-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    retq
 ;
-; X86-LABEL: or16_nouse_seq_cst:
-; X86:       # %bb.0:
-; X86-NEXT:    lock orl $0, (%esp)
-; X86-NEXT:    retl
+; X86-GENERIC-LABEL: or16_nouse_seq_cst:
+; X86-GENERIC:       # %bb.0:
+; X86-GENERIC-NEXT:    lock orl $0, (%esp)
+; X86-GENERIC-NEXT:    retl
+;
+; X86-ATOM-LABEL: or16_nouse_seq_cst:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    lock orl $0, (%esp)
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    retl
   atomicrmw or i16* %p, i16 0 seq_cst
   ret void
 }
@@ -311,10 +619,21 @@ define void @or8_nouse_seq_cst(i8* %p) {
 ; X64-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    retq
 ;
-; X86-LABEL: or8_nouse_seq_cst:
-; X86:       # %bb.0:
-; X86-NEXT:    lock orl $0, (%esp)
-; X86-NEXT:    retl
+; X86-GENERIC-LABEL: or8_nouse_seq_cst:
+; X86-GENERIC:       # %bb.0:
+; X86-GENERIC-NEXT:    lock orl $0, (%esp)
+; X86-GENERIC-NEXT:    retl
+;
+; X86-ATOM-LABEL: or8_nouse_seq_cst:
+; X86-ATOM:       # %bb.0:
+; X86-ATOM-NEXT:    lock orl $0, (%esp)
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    nop
+; X86-ATOM-NEXT:    retl
   atomicrmw or i8* %p, i8 0 seq_cst
   ret void
 }
diff --git a/llvm/test/CodeGen/X86/avg.ll b/llvm/test/CodeGen/X86/avg.ll
index 88b08cc1273cb..3ce8489fcc6d3 100644
--- a/llvm/test/CodeGen/X86/avg.ll
+++ b/llvm/test/CodeGen/X86/avg.ll
@@ -140,7 +140,7 @@ define void @avg_v24i8(<24 x i8>* %a, <24 x i8>* %b) nounwind {
 ; SSE2-NEXT:    psrld $1, %xmm4
 ; SSE2-NEXT:    psrld $1, %xmm2
 ; SSE2-NEXT:    psrld $1, %xmm3
-; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [255,0,255,0,255,0,255,0]
 ; SSE2-NEXT:    pand %xmm6, %xmm3
 ; SSE2-NEXT:    pand %xmm6, %xmm2
 ; SSE2-NEXT:    packuswb %xmm3, %xmm2
@@ -413,7 +413,7 @@ define void @avg_v48i8(<48 x i8>* %a, <48 x i8>* %b) nounwind {
 ; SSE2-NEXT:    psubd %xmm5, %xmm0
 ; SSE2-NEXT:    psrld $1, %xmm3
 ; SSE2-NEXT:    psrld $1, %xmm8
-; SSE2-NEXT:    movdqa {{.*#+}} xmm7 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm7 = [255,0,255,0,255,0,255,0]
 ; SSE2-NEXT:    pand %xmm7, %xmm8
 ; SSE2-NEXT:    pand %xmm7, %xmm3
 ; SSE2-NEXT:    packuswb %xmm8, %xmm3
diff --git a/llvm/test/CodeGen/X86/avx-fp2int.ll b/llvm/test/CodeGen/X86/avx-fp2int.ll
index ac5fcfe4d5fce..195b6f660e927 100644
--- a/llvm/test/CodeGen/X86/avx-fp2int.ll
+++ b/llvm/test/CodeGen/X86/avx-fp2int.ll
@@ -7,7 +7,8 @@ define <4 x i8> @test1(<4 x double> %d) {
 ; CHECK-LABEL: test1:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vcvttpd2dq %ymm0, %xmm0
-; CHECK-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retl
   %c = fptoui <4 x double> %d to <4 x i8>
@@ -17,7 +18,8 @@ define <4 x i8> @test2(<4 x double> %d) {
 ; CHECK-LABEL: test2:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vcvttpd2dq %ymm0, %xmm0
-; CHECK-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retl
   %c = fptosi <4 x double> %d to <4 x i8>
diff --git a/llvm/test/CodeGen/X86/avx-vperm2x128.ll b/llvm/test/CodeGen/X86/avx-vperm2x128.ll
index 24ac0988721de..63445066934e1 100644
--- a/llvm/test/CodeGen/X86/avx-vperm2x128.ll
+++ b/llvm/test/CodeGen/X86/avx-vperm2x128.ll
@@ -452,7 +452,7 @@ define <4 x double> @shuffle_v4f64_01zz_optsize(<4 x double> %a) optsize {
 define <4 x double> @shuffle_v4f64_23zz(<4 x double> %a) {
 ; ALL-LABEL: shuffle_v4f64_23zz:
 ; ALL:       # %bb.0:
-; ALL-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],zero,zero
+; ALL-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; ALL-NEXT:    retq
   %s = shufflevector <4 x double> %a, <4 x double> <double 0.0, double 0.0, double undef, double undef>, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
   ret <4 x double> %s
@@ -460,7 +460,7 @@ define <4 x double> @shuffle_v4f64_23zz(<4 x double> %a) {
 define <4 x double> @shuffle_v4f64_23zz_optsize(<4 x double> %a) optsize {
 ; ALL-LABEL: shuffle_v4f64_23zz_optsize:
 ; ALL:       # %bb.0:
-; ALL-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],zero,zero
+; ALL-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; ALL-NEXT:    retq
   %s = shufflevector <4 x double> %a, <4 x double> <double 0.0, double 0.0, double undef, double undef>, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
   ret <4 x double> %s
@@ -486,7 +486,7 @@ define <4 x double> @shuffle_v4f64_45zz_optsize(<4 x double> %a) optsize {
 define <4 x double> @shuffle_v4f64_67zz(<4 x double> %a) {
 ; ALL-LABEL: shuffle_v4f64_67zz:
 ; ALL:       # %bb.0:
-; ALL-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],zero,zero
+; ALL-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; ALL-NEXT:    retq
   %s = shufflevector <4 x double> <double 0.0, double 0.0, double undef, double undef>, <4 x double> %a, <4 x i32> <i32 6, i32 7, i32 0, i32 1>
   ret <4 x double> %s
@@ -494,7 +494,7 @@ define <4 x double> @shuffle_v4f64_67zz(<4 x double> %a) {
 define <4 x double> @shuffle_v4f64_67zz_optsize(<4 x double> %a) optsize {
 ; ALL-LABEL: shuffle_v4f64_67zz_optsize:
 ; ALL:       # %bb.0:
-; ALL-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],zero,zero
+; ALL-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; ALL-NEXT:    retq
   %s = shufflevector <4 x double> <double 0.0, double 0.0, double undef, double undef>, <4 x double> %a, <4 x i32> <i32 6, i32 7, i32 0, i32 1>
   ret <4 x double> %s
@@ -512,7 +512,7 @@ define <4 x i64> @shuffle_v4i64_67zz(<4 x i64> %a, <4 x i64> %b) {
 ;
 ; AVX2-LABEL: shuffle_v4i64_67zz:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],zero,zero
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX2-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
   %s = shufflevector <4 x i64> <i64 0, i64 0, i64 undef, i64 undef>, <4 x i64> %a, <4 x i32> <i32 6, i32 7, i32 0, i32 1>
diff --git a/llvm/test/CodeGen/X86/avx512-gfni-intrinsics.ll b/llvm/test/CodeGen/X86/avx512-gfni-intrinsics.ll
index fbb28766f653b..241a6ea9995e7 100644
--- a/llvm/test/CodeGen/X86/avx512-gfni-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512-gfni-intrinsics.ll
@@ -8,8 +8,8 @@ define <16 x i8> @test_vgf2p8affineinvqb_128(<16 x i8> %src1, <16 x i8> %src2, <
 ; X86:       # %bb.0:
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vgf2p8affineinvqb $3, %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0xf9,0xcf,0xd9,0x03]
-; X86-NEXT:    vgf2p8affineinvqb $3, %xmm1, %xmm0, %xmm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0x89,0xcf,0xe1,0x03]
-; X86-NEXT:    vgf2p8affineinvqb $3, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0xcf,0xd1,0x03]
+; X86-NEXT:    vgf2p8affineinvqb $4, %xmm1, %xmm0, %xmm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0x89,0xcf,0xe1,0x04]
+; X86-NEXT:    vgf2p8affineinvqb $5, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0xcf,0xd1,0x05]
 ; X86-NEXT:    vpxor %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xc3]
 ; X86-NEXT:    vpxor %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xef,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
@@ -18,18 +18,20 @@ define <16 x i8> @test_vgf2p8affineinvqb_128(<16 x i8> %src1, <16 x i8> %src2, <
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vgf2p8affineinvqb $3, %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0xf9,0xcf,0xd9,0x03]
-; X64-NEXT:    vgf2p8affineinvqb $3, %xmm1, %xmm0, %xmm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0x89,0xcf,0xe1,0x03]
-; X64-NEXT:    vgf2p8affineinvqb $3, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0xcf,0xd1,0x03]
+; X64-NEXT:    vgf2p8affineinvqb $4, %xmm1, %xmm0, %xmm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0x89,0xcf,0xe1,0x04]
+; X64-NEXT:    vgf2p8affineinvqb $5, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0xcf,0xd1,0x05]
 ; X64-NEXT:    vpxor %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xc3]
 ; X64-NEXT:    vpxor %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xef,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = bitcast i16 %mask to <16 x i1>
   %2 = call <16 x i8> @llvm.x86.vgf2p8affineinvqb.128(<16 x i8> %src1, <16 x i8> %src2, i8 3)
-  %3 = select <16 x i1> %1, <16 x i8> %2, <16 x i8> zeroinitializer
-  %4 = select <16 x i1> %1, <16 x i8> %2, <16 x i8> %passthru
-  %5 = xor <16 x i8> %3, %4
-  %6 = xor <16 x i8> %5, %2
-  ret <16 x i8> %6
+  %3 = call <16 x i8> @llvm.x86.vgf2p8affineinvqb.128(<16 x i8> %src1, <16 x i8> %src2, i8 4)
+  %4 = call <16 x i8> @llvm.x86.vgf2p8affineinvqb.128(<16 x i8> %src1, <16 x i8> %src2, i8 5)
+  %5 = select <16 x i1> %1, <16 x i8> %3, <16 x i8> zeroinitializer
+  %6 = select <16 x i1> %1, <16 x i8> %4, <16 x i8> %passthru
+  %7 = xor <16 x i8> %5, %6
+  %8 = xor <16 x i8> %7, %2
+  ret <16 x i8> %8
 }
 
 declare <32 x i8> @llvm.x86.vgf2p8affineinvqb.256(<32 x i8>, <32 x i8>, i8)
@@ -38,8 +40,8 @@ define <32 x i8> @test_vgf2p8affineinvqb_256(<32 x i8> %src1, <32 x i8> %src2, <
 ; X86:       # %bb.0:
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vgf2p8affineinvqb $3, %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0xfd,0xcf,0xd9,0x03]
-; X86-NEXT:    vgf2p8affineinvqb $3, %ymm1, %ymm0, %ymm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xa9,0xcf,0xe1,0x03]
-; X86-NEXT:    vgf2p8affineinvqb $3, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0xcf,0xd1,0x03]
+; X86-NEXT:    vgf2p8affineinvqb $4, %ymm1, %ymm0, %ymm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xa9,0xcf,0xe1,0x04]
+; X86-NEXT:    vgf2p8affineinvqb $5, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0xcf,0xd1,0x05]
 ; X86-NEXT:    vpxor %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xef,0xc3]
 ; X86-NEXT:    vpxor %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xef,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
@@ -48,18 +50,20 @@ define <32 x i8> @test_vgf2p8affineinvqb_256(<32 x i8> %src1, <32 x i8> %src2, <
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vgf2p8affineinvqb $3, %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0xfd,0xcf,0xd9,0x03]
-; X64-NEXT:    vgf2p8affineinvqb $3, %ymm1, %ymm0, %ymm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xa9,0xcf,0xe1,0x03]
-; X64-NEXT:    vgf2p8affineinvqb $3, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0xcf,0xd1,0x03]
+; X64-NEXT:    vgf2p8affineinvqb $4, %ymm1, %ymm0, %ymm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xa9,0xcf,0xe1,0x04]
+; X64-NEXT:    vgf2p8affineinvqb $5, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0xcf,0xd1,0x05]
 ; X64-NEXT:    vpxor %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xef,0xc3]
 ; X64-NEXT:    vpxor %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xef,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = bitcast i32 %mask to <32 x i1>
   %2 = call <32 x i8> @llvm.x86.vgf2p8affineinvqb.256(<32 x i8> %src1, <32 x i8> %src2, i8 3)
-  %3 = select <32 x i1> %1, <32 x i8> %2, <32 x i8> zeroinitializer
-  %4 = select <32 x i1> %1, <32 x i8> %2, <32 x i8> %passthru
-  %5 = xor <32 x i8> %3, %4
-  %6 = xor <32 x i8> %5, %2
-  ret <32 x i8> %6
+  %3 = call <32 x i8> @llvm.x86.vgf2p8affineinvqb.256(<32 x i8> %src1, <32 x i8> %src2, i8 4)
+  %4 = call <32 x i8> @llvm.x86.vgf2p8affineinvqb.256(<32 x i8> %src1, <32 x i8> %src2, i8 5)
+  %5 = select <32 x i1> %1, <32 x i8> %3, <32 x i8> zeroinitializer
+  %6 = select <32 x i1> %1, <32 x i8> %4, <32 x i8> %passthru
+  %7 = xor <32 x i8> %5, %6
+  %8 = xor <32 x i8> %7, %2
+  ret <32 x i8> %8
 }
 
 declare <64 x i8> @llvm.x86.vgf2p8affineinvqb.512(<64 x i8>, <64 x i8>, i8)
@@ -68,8 +72,8 @@ define <64 x i8> @test_vgf2p8affineinvqb_512(<64 x i8> %src1, <64 x i8> %src2, <
 ; X86:       # %bb.0:
 ; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vgf2p8affineinvqb $3, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0xcf,0xd9,0x03]
-; X86-NEXT:    vgf2p8affineinvqb $3, %zmm1, %zmm0, %zmm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xc9,0xcf,0xe1,0x03]
-; X86-NEXT:    vgf2p8affineinvqb $3, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0xcf,0xd1,0x03]
+; X86-NEXT:    vgf2p8affineinvqb $4, %zmm1, %zmm0, %zmm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xc9,0xcf,0xe1,0x04]
+; X86-NEXT:    vgf2p8affineinvqb $5, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0xcf,0xd1,0x05]
 ; X86-NEXT:    vpxorq %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xef,0xc3]
 ; X86-NEXT:    vpxorq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xef,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
@@ -78,18 +82,20 @@ define <64 x i8> @test_vgf2p8affineinvqb_512(<64 x i8> %src1, <64 x i8> %src2, <
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovq %rdi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xcf]
 ; X64-NEXT:    vgf2p8affineinvqb $3, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0xcf,0xd9,0x03]
-; X64-NEXT:    vgf2p8affineinvqb $3, %zmm1, %zmm0, %zmm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xc9,0xcf,0xe1,0x03]
-; X64-NEXT:    vgf2p8affineinvqb $3, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0xcf,0xd1,0x03]
+; X64-NEXT:    vgf2p8affineinvqb $4, %zmm1, %zmm0, %zmm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xc9,0xcf,0xe1,0x04]
+; X64-NEXT:    vgf2p8affineinvqb $5, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0xcf,0xd1,0x05]
 ; X64-NEXT:    vpxorq %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xef,0xc3]
 ; X64-NEXT:    vpxorq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xef,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = bitcast i64 %mask to <64 x i1>
   %2 = call <64 x i8> @llvm.x86.vgf2p8affineinvqb.512(<64 x i8> %src1, <64 x i8> %src2, i8 3)
-  %3 = select <64 x i1> %1, <64 x i8> %2, <64 x i8> zeroinitializer
-  %4 = select <64 x i1> %1, <64 x i8> %2, <64 x i8> %passthru
-  %5 = xor <64 x i8> %3, %4
-  %6 = xor <64 x i8> %5, %2
-  ret <64 x i8> %6
+  %3 = call <64 x i8> @llvm.x86.vgf2p8affineinvqb.512(<64 x i8> %src1, <64 x i8> %src2, i8 4)
+  %4 = call <64 x i8> @llvm.x86.vgf2p8affineinvqb.512(<64 x i8> %src1, <64 x i8> %src2, i8 5)
+  %5 = select <64 x i1> %1, <64 x i8> %3, <64 x i8> zeroinitializer
+  %6 = select <64 x i1> %1, <64 x i8> %4, <64 x i8> %passthru
+  %7 = xor <64 x i8> %5, %6
+  %8 = xor <64 x i8> %7, %2
+  ret <64 x i8> %8
 }
 
 declare <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8>, <16 x i8>, i8)
@@ -98,8 +104,8 @@ define <16 x i8> @test_vgf2p8affineqb_128(<16 x i8> %src1, <16 x i8> %src2, <16
 ; X86:       # %bb.0:
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vgf2p8affineqb $3, %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0xf9,0xce,0xd9,0x03]
-; X86-NEXT:    vgf2p8affineqb $3, %xmm1, %xmm0, %xmm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0x89,0xce,0xe1,0x03]
-; X86-NEXT:    vgf2p8affineqb $3, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0xce,0xd1,0x03]
+; X86-NEXT:    vgf2p8affineqb $4, %xmm1, %xmm0, %xmm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0x89,0xce,0xe1,0x04]
+; X86-NEXT:    vgf2p8affineqb $5, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0xce,0xd1,0x05]
 ; X86-NEXT:    vpxor %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xc3]
 ; X86-NEXT:    vpxor %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xef,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
@@ -108,18 +114,20 @@ define <16 x i8> @test_vgf2p8affineqb_128(<16 x i8> %src1, <16 x i8> %src2, <16
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vgf2p8affineqb $3, %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0xf9,0xce,0xd9,0x03]
-; X64-NEXT:    vgf2p8affineqb $3, %xmm1, %xmm0, %xmm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0x89,0xce,0xe1,0x03]
-; X64-NEXT:    vgf2p8affineqb $3, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0xce,0xd1,0x03]
+; X64-NEXT:    vgf2p8affineqb $4, %xmm1, %xmm0, %xmm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0x89,0xce,0xe1,0x04]
+; X64-NEXT:    vgf2p8affineqb $5, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0xce,0xd1,0x05]
 ; X64-NEXT:    vpxor %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xc3]
 ; X64-NEXT:    vpxor %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xef,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = bitcast i16 %mask to <16 x i1>
   %2 = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %src1, <16 x i8> %src2, i8 3)
-  %3 = select <16 x i1> %1, <16 x i8> %2, <16 x i8> zeroinitializer
-  %4 = select <16 x i1> %1, <16 x i8> %2, <16 x i8> %passthru
-  %5 = xor <16 x i8> %3, %4
-  %6 = xor <16 x i8> %5, %2
-  ret <16 x i8> %6
+  %3 = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %src1, <16 x i8> %src2, i8 4)
+  %4 = call <16 x i8> @llvm.x86.vgf2p8affineqb.128(<16 x i8> %src1, <16 x i8> %src2, i8 5)
+  %5 = select <16 x i1> %1, <16 x i8> %3, <16 x i8> zeroinitializer
+  %6 = select <16 x i1> %1, <16 x i8> %4, <16 x i8> %passthru
+  %7 = xor <16 x i8> %5, %6
+  %8 = xor <16 x i8> %7, %2
+  ret <16 x i8> %8
 }
 
 declare <32 x i8> @llvm.x86.vgf2p8affineqb.256(<32 x i8>, <32 x i8>, i8)
@@ -128,8 +136,8 @@ define <32 x i8> @test_vgf2p8affineqb_256(<32 x i8> %src1, <32 x i8> %src2, <32
 ; X86:       # %bb.0:
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vgf2p8affineqb $3, %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0xfd,0xce,0xd9,0x03]
-; X86-NEXT:    vgf2p8affineqb $3, %ymm1, %ymm0, %ymm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xa9,0xce,0xe1,0x03]
-; X86-NEXT:    vgf2p8affineqb $3, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0xce,0xd1,0x03]
+; X86-NEXT:    vgf2p8affineqb $4, %ymm1, %ymm0, %ymm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xa9,0xce,0xe1,0x04]
+; X86-NEXT:    vgf2p8affineqb $5, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0xce,0xd1,0x05]
 ; X86-NEXT:    vpxor %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xef,0xc3]
 ; X86-NEXT:    vpxor %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xef,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
@@ -138,18 +146,20 @@ define <32 x i8> @test_vgf2p8affineqb_256(<32 x i8> %src1, <32 x i8> %src2, <32
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vgf2p8affineqb $3, %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0xfd,0xce,0xd9,0x03]
-; X64-NEXT:    vgf2p8affineqb $3, %ymm1, %ymm0, %ymm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xa9,0xce,0xe1,0x03]
-; X64-NEXT:    vgf2p8affineqb $3, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0xce,0xd1,0x03]
+; X64-NEXT:    vgf2p8affineqb $4, %ymm1, %ymm0, %ymm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xa9,0xce,0xe1,0x04]
+; X64-NEXT:    vgf2p8affineqb $5, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0xce,0xd1,0x05]
 ; X64-NEXT:    vpxor %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xef,0xc3]
 ; X64-NEXT:    vpxor %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xef,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = bitcast i32 %mask to <32 x i1>
   %2 = call <32 x i8> @llvm.x86.vgf2p8affineqb.256(<32 x i8> %src1, <32 x i8> %src2, i8 3)
-  %3 = select <32 x i1> %1, <32 x i8> %2, <32 x i8> zeroinitializer
-  %4 = select <32 x i1> %1, <32 x i8> %2, <32 x i8> %passthru
-  %5 = xor <32 x i8> %3, %4
-  %6 = xor <32 x i8> %5, %2
-  ret <32 x i8> %6
+  %3 = call <32 x i8> @llvm.x86.vgf2p8affineqb.256(<32 x i8> %src1, <32 x i8> %src2, i8 4)
+  %4 = call <32 x i8> @llvm.x86.vgf2p8affineqb.256(<32 x i8> %src1, <32 x i8> %src2, i8 5)
+  %5 = select <32 x i1> %1, <32 x i8> %3, <32 x i8> zeroinitializer
+  %6 = select <32 x i1> %1, <32 x i8> %4, <32 x i8> %passthru
+  %7 = xor <32 x i8> %5, %6
+  %8 = xor <32 x i8> %7, %2
+  ret <32 x i8> %8
 }
 
 declare <64 x i8> @llvm.x86.vgf2p8affineqb.512(<64 x i8>, <64 x i8>, i8)
@@ -158,8 +168,8 @@ define <64 x i8> @test_vgf2p8affineqb_512(<64 x i8> %src1, <64 x i8> %src2, <64
 ; X86:       # %bb.0:
 ; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vgf2p8affineqb $3, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0xce,0xd9,0x03]
-; X86-NEXT:    vgf2p8affineqb $3, %zmm1, %zmm0, %zmm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xc9,0xce,0xe1,0x03]
-; X86-NEXT:    vgf2p8affineqb $3, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0xce,0xd1,0x03]
+; X86-NEXT:    vgf2p8affineqb $4, %zmm1, %zmm0, %zmm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xc9,0xce,0xe1,0x04]
+; X86-NEXT:    vgf2p8affineqb $5, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0xce,0xd1,0x05]
 ; X86-NEXT:    vpxorq %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xef,0xc3]
 ; X86-NEXT:    vpxorq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xef,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
@@ -168,107 +178,162 @@ define <64 x i8> @test_vgf2p8affineqb_512(<64 x i8> %src1, <64 x i8> %src2, <64
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovq %rdi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xcf]
 ; X64-NEXT:    vgf2p8affineqb $3, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0xce,0xd9,0x03]
-; X64-NEXT:    vgf2p8affineqb $3, %zmm1, %zmm0, %zmm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xc9,0xce,0xe1,0x03]
-; X64-NEXT:    vgf2p8affineqb $3, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0xce,0xd1,0x03]
+; X64-NEXT:    vgf2p8affineqb $4, %zmm1, %zmm0, %zmm4 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0xc9,0xce,0xe1,0x04]
+; X64-NEXT:    vgf2p8affineqb $5, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0xce,0xd1,0x05]
 ; X64-NEXT:    vpxorq %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xef,0xc3]
 ; X64-NEXT:    vpxorq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xef,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = bitcast i64 %mask to <64 x i1>
   %2 = call <64 x i8> @llvm.x86.vgf2p8affineqb.512(<64 x i8> %src1, <64 x i8> %src2, i8 3)
-  %3 = select <64 x i1> %1, <64 x i8> %2, <64 x i8> zeroinitializer
-  %4 = select <64 x i1> %1, <64 x i8> %2, <64 x i8> %passthru
-  %5 = xor <64 x i8> %3, %4
-  %6 = xor <64 x i8> %5, %2
-  ret <64 x i8> %6
+  %3 = call <64 x i8> @llvm.x86.vgf2p8affineqb.512(<64 x i8> %src1, <64 x i8> %src2, i8 4)
+  %4 = call <64 x i8> @llvm.x86.vgf2p8affineqb.512(<64 x i8> %src1, <64 x i8> %src2, i8 5)
+  %5 = select <64 x i1> %1, <64 x i8> %3, <64 x i8> zeroinitializer
+  %6 = select <64 x i1> %1, <64 x i8> %4, <64 x i8> %passthru
+  %7 = xor <64 x i8> %5, %6
+  %8 = xor <64 x i8> %7, %2
+  ret <64 x i8> %8
 }
 
 declare <16 x i8> @llvm.x86.vgf2p8mulb.128(<16 x i8>, <16 x i8>)
-define <16 x i8> @test_vgf2p8mulb_128(<16 x i8> %src1, <16 x i8> %src2, <16 x i8> %passthru, i16 %mask) {
-; X86-LABEL: test_vgf2p8mulb_128:
+define <16 x i8> @test_vgf2p8mulb_128(<16 x i8> %src1, <16 x i8> %src2) {
+; CHECK-LABEL: test_vgf2p8mulb_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vgf2p8mulb %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0xcf,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <16 x i8> @llvm.x86.vgf2p8mulb.128(<16 x i8> %src1, <16 x i8> %src2)
+  ret <16 x i8> %1
+}
+
+define <16 x i8> @test_vgf2p8mulb_128_mask(<16 x i8> %src1, <16 x i8> %src2, <16 x i8> %passthru, i16 %mask) {
+; X86-LABEL: test_vgf2p8mulb_128_mask:
 ; X86:       # %bb.0:
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vgf2p8mulb %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0xcf,0xd9]
-; X86-NEXT:    vgf2p8mulb %xmm1, %xmm0, %xmm4 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0xcf,0xe1]
 ; X86-NEXT:    vgf2p8mulb %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0xcf,0xd1]
-; X86-NEXT:    vpxor %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xc3]
-; X86-NEXT:    vpxor %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xef,0xc0]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_vgf2p8mulb_128:
+; X64-LABEL: test_vgf2p8mulb_128_mask:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vgf2p8mulb %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0xcf,0xd9]
-; X64-NEXT:    vgf2p8mulb %xmm1, %xmm0, %xmm4 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0xcf,0xe1]
 ; X64-NEXT:    vgf2p8mulb %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0xcf,0xd1]
-; X64-NEXT:    vpxor %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xc3]
-; X64-NEXT:    vpxor %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xef,0xc0]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i16 %mask to <16 x i1>
+  %2 = call <16 x i8> @llvm.x86.vgf2p8mulb.128(<16 x i8> %src1, <16 x i8> %src2)
+  %3 = select <16 x i1> %1, <16 x i8> %2, <16 x i8> %passthru
+  ret <16 x i8> %3
+}
+
+define <16 x i8> @test_vgf2p8mulb_128_maskz(<16 x i8> %src1, <16 x i8> %src2, i16 %mask) {
+; X86-LABEL: test_vgf2p8mulb_128_maskz:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vgf2p8mulb %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0xcf,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_vgf2p8mulb_128_maskz:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vgf2p8mulb %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0xcf,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = bitcast i16 %mask to <16 x i1>
   %2 = call <16 x i8> @llvm.x86.vgf2p8mulb.128(<16 x i8> %src1, <16 x i8> %src2)
   %3 = select <16 x i1> %1, <16 x i8> %2, <16 x i8> zeroinitializer
-  %4 = select <16 x i1> %1, <16 x i8> %2, <16 x i8> %passthru
-  %5 = xor <16 x i8> %3, %4
-  %6 = xor <16 x i8> %5, %2
-  ret <16 x i8> %6
+  ret <16 x i8> %3
 }
 
 declare <32 x i8> @llvm.x86.vgf2p8mulb.256(<32 x i8>, <32 x i8>)
-define <32 x i8> @test_vgf2p8mulb_256(<32 x i8> %src1, <32 x i8> %src2, <32 x i8> %passthru, i32 %mask) {
-; X86-LABEL: test_vgf2p8mulb_256:
+define <32 x i8> @test_vgf2p8mulb_256(<32 x i8> %src1, <32 x i8> %src2) {
+; CHECK-LABEL: test_vgf2p8mulb_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vgf2p8mulb %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0xcf,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <32 x i8> @llvm.x86.vgf2p8mulb.256(<32 x i8> %src1, <32 x i8> %src2)
+  ret <32 x i8> %1
+}
+
+define <32 x i8> @test_vgf2p8mulb_256_mask(<32 x i8> %src1, <32 x i8> %src2, <32 x i8> %passthru, i32 %mask) {
+; X86-LABEL: test_vgf2p8mulb_256_mask:
 ; X86:       # %bb.0:
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vgf2p8mulb %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0xcf,0xd9]
-; X86-NEXT:    vgf2p8mulb %ymm1, %ymm0, %ymm4 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0xcf,0xe1]
 ; X86-NEXT:    vgf2p8mulb %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0xcf,0xd1]
-; X86-NEXT:    vpxor %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xef,0xc3]
-; X86-NEXT:    vpxor %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xef,0xc0]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_vgf2p8mulb_256:
+; X64-LABEL: test_vgf2p8mulb_256_mask:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vgf2p8mulb %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0xcf,0xd9]
-; X64-NEXT:    vgf2p8mulb %ymm1, %ymm0, %ymm4 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0xcf,0xe1]
 ; X64-NEXT:    vgf2p8mulb %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0xcf,0xd1]
-; X64-NEXT:    vpxor %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xef,0xc3]
-; X64-NEXT:    vpxor %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xef,0xc0]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i32 %mask to <32 x i1>
+  %2 = call <32 x i8> @llvm.x86.vgf2p8mulb.256(<32 x i8> %src1, <32 x i8> %src2)
+  %3 = select <32 x i1> %1, <32 x i8> %2, <32 x i8> %passthru
+  ret <32 x i8> %3
+}
+
+define <32 x i8> @test_vgf2p8mulb_256_maskz(<32 x i8> %src1, <32 x i8> %src2, i32 %mask) {
+; X86-LABEL: test_vgf2p8mulb_256_maskz:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vgf2p8mulb %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0xcf,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_vgf2p8mulb_256_maskz:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vgf2p8mulb %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0xcf,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = bitcast i32 %mask to <32 x i1>
   %2 = call <32 x i8> @llvm.x86.vgf2p8mulb.256(<32 x i8> %src1, <32 x i8> %src2)
   %3 = select <32 x i1> %1, <32 x i8> %2, <32 x i8> zeroinitializer
-  %4 = select <32 x i1> %1, <32 x i8> %2, <32 x i8> %passthru
-  %5 = xor <32 x i8> %3, %4
-  %6 = xor <32 x i8> %5, %2
-  ret <32 x i8> %6
+  ret <32 x i8> %3
 }
 
 declare <64 x i8> @llvm.x86.vgf2p8mulb.512(<64 x i8>, <64 x i8>)
-define <64 x i8> @test_vgf2p8mulb_512(<64 x i8> %src1, <64 x i8> %src2, <64 x i8> %passthru, i64 %mask) {
-; X86-LABEL: test_vgf2p8mulb_512:
+define <64 x i8> @test_vgf2p8mulb_512(<64 x i8> %src1, <64 x i8> %src2) {
+; CHECK-LABEL: test_vgf2p8mulb_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vgf2p8mulb %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf2,0x7d,0x48,0xcf,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <64 x i8> @llvm.x86.vgf2p8mulb.512(<64 x i8> %src1, <64 x i8> %src2)
+  ret <64 x i8> %1
+}
+
+define <64 x i8> @test_vgf2p8mulb_512_mask(<64 x i8> %src1, <64 x i8> %src2, <64 x i8> %passthru, i64 %mask) {
+; X86-LABEL: test_vgf2p8mulb_512_mask:
 ; X86:       # %bb.0:
 ; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vgf2p8mulb %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf2,0x7d,0x48,0xcf,0xd9]
-; X86-NEXT:    vgf2p8mulb %zmm1, %zmm0, %zmm4 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xc9,0xcf,0xe1]
 ; X86-NEXT:    vgf2p8mulb %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0xcf,0xd1]
-; X86-NEXT:    vpxorq %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xef,0xc3]
-; X86-NEXT:    vpxorq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xef,0xc0]
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_vgf2p8mulb_512:
+; X64-LABEL: test_vgf2p8mulb_512_mask:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovq %rdi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xcf]
-; X64-NEXT:    vgf2p8mulb %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf2,0x7d,0x48,0xcf,0xd9]
-; X64-NEXT:    vgf2p8mulb %zmm1, %zmm0, %zmm4 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xc9,0xcf,0xe1]
 ; X64-NEXT:    vgf2p8mulb %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0xcf,0xd1]
-; X64-NEXT:    vpxorq %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xef,0xc3]
-; X64-NEXT:    vpxorq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xef,0xc0]
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = bitcast i64 %mask to <64 x i1>
   %2 = call <64 x i8> @llvm.x86.vgf2p8mulb.512(<64 x i8> %src1, <64 x i8> %src2)
-  %3 = select <64 x i1> %1, <64 x i8> %2, <64 x i8> zeroinitializer
-  %4 = select <64 x i1> %1, <64 x i8> %2, <64 x i8> %passthru
-  %5 = xor <64 x i8> %3, %4
-  %6 = xor <64 x i8> %5, %2
-  ret <64 x i8> %6
+  %3 = select <64 x i1> %1, <64 x i8> %2, <64 x i8> %passthru
+  ret <64 x i8> %3
 }
 
+define <64 x i8> @test_vgf2p8mulb_512_maskz(<64 x i8> %src1, <64 x i8> %src2, i64 %mask) {
+; X86-LABEL: test_vgf2p8mulb_512_maskz:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vgf2p8mulb %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xc9,0xcf,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_vgf2p8mulb_512_maskz:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovq %rdi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xcf]
+; X64-NEXT:    vgf2p8mulb %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xc9,0xcf,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i64 %mask to <64 x i1>
+  %2 = call <64 x i8> @llvm.x86.vgf2p8mulb.512(<64 x i8> %src1, <64 x i8> %src2)
+  %3 = select <64 x i1> %1, <64 x i8> %2, <64 x i8> zeroinitializer
+  ret <64 x i8> %3
+}
diff --git a/llvm/test/CodeGen/X86/avx512-intel-ocl.ll b/llvm/test/CodeGen/X86/avx512-intel-ocl.ll
index 751d610c2ca79..232b598c61097 100644
--- a/llvm/test/CodeGen/X86/avx512-intel-ocl.ll
+++ b/llvm/test/CodeGen/X86/avx512-intel-ocl.ll
@@ -294,8 +294,7 @@ define intel_ocl_bicc <16 x float> @test_prolog_epilog(<16 x float> %a, <16 x fl
 ; X64-KNL-LABEL: test_prolog_epilog:
 ; X64-KNL:       ## %bb.0:
 ; X64-KNL-NEXT:    pushq %rsi
-; X64-KNL-NEXT:    pushq %rdi
-; X64-KNL-NEXT:    subq $1064, %rsp ## imm = 0x428
+; X64-KNL-NEXT:    subq $1072, %rsp ## imm = 0x430
 ; X64-KNL-NEXT:    kmovw %k7, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
 ; X64-KNL-NEXT:    kmovw %k6, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
 ; X64-KNL-NEXT:    kmovw %k5, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
@@ -337,16 +336,14 @@ define intel_ocl_bicc <16 x float> @test_prolog_epilog(<16 x float> %a, <16 x fl
 ; X64-KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 ## 2-byte Reload
 ; X64-KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k6 ## 2-byte Reload
 ; X64-KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
-; X64-KNL-NEXT:    addq $1064, %rsp ## imm = 0x428
-; X64-KNL-NEXT:    popq %rdi
+; X64-KNL-NEXT:    addq $1072, %rsp ## imm = 0x430
 ; X64-KNL-NEXT:    popq %rsi
 ; X64-KNL-NEXT:    retq
 ;
 ; X64-SKX-LABEL: test_prolog_epilog:
 ; X64-SKX:       ## %bb.0:
 ; X64-SKX-NEXT:    pushq %rsi
-; X64-SKX-NEXT:    pushq %rdi
-; X64-SKX-NEXT:    subq $1064, %rsp ## imm = 0x428
+; X64-SKX-NEXT:    subq $1072, %rsp ## imm = 0x430
 ; X64-SKX-NEXT:    kmovq %k7, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
 ; X64-SKX-NEXT:    kmovq %k6, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
 ; X64-SKX-NEXT:    kmovq %k5, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
@@ -388,8 +385,7 @@ define intel_ocl_bicc <16 x float> @test_prolog_epilog(<16 x float> %a, <16 x fl
 ; X64-SKX-NEXT:    kmovq {{[-0-9]+}}(%r{{[sb]}}p), %k5 ## 8-byte Reload
 ; X64-SKX-NEXT:    kmovq {{[-0-9]+}}(%r{{[sb]}}p), %k6 ## 8-byte Reload
 ; X64-SKX-NEXT:    kmovq {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 8-byte Reload
-; X64-SKX-NEXT:    addq $1064, %rsp ## imm = 0x428
-; X64-SKX-NEXT:    popq %rdi
+; X64-SKX-NEXT:    addq $1072, %rsp ## imm = 0x430
 ; X64-SKX-NEXT:    popq %rsi
 ; X64-SKX-NEXT:    retq
    %c = call <16 x float> @func_float16(<16 x float> %a, <16 x float> %b)
diff --git a/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
index 851ce75b4632f..a0631f1e8d5ef 100644
--- a/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
@@ -317,74 +317,102 @@ define <8 x i64>@test_int_x86_avx512_maskz_pbroadcastq_512(<2 x i64> %x0, i8 %ma
 
 declare <16 x float> @llvm.x86.avx512.mask.movsldup.512(<16 x float>, <16 x float>, i16)
 
+define <16 x float>@test_int_x86_avx512_movsldup_512(<16 x float> %x0, <16 x float> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_movsldup_512:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vmovsldup %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7e,0x48,0x12,0xc0]
+; CHECK-NEXT:    ## zmm0 = zmm0[0,0,2,2,4,4,6,6,8,8,10,10,12,12,14,14]
+; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+  %res = call <16 x float> @llvm.x86.avx512.mask.movsldup.512(<16 x float> %x0, <16 x float> %x1, i16 -1)
+  ret <16 x float> %res
+}
+
 define <16 x float>@test_int_x86_avx512_mask_movsldup_512(<16 x float> %x0, <16 x float> %x1, i16 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_movsldup_512:
 ; X86:       ## %bb.0:
-; X86-NEXT:    vmovsldup %zmm0, %zmm2 ## encoding: [0x62,0xf1,0x7e,0x48,0x12,0xd0]
-; X86-NEXT:    ## zmm2 = zmm0[0,0,2,2,4,4,6,6,8,8,10,10,12,12,14,14]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vmovsldup %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x7e,0x49,0x12,0xc8]
 ; X86-NEXT:    ## zmm1 {%k1} = zmm0[0,0,2,2,4,4,6,6,8,8,10,10,12,12,14,14]
-; X86-NEXT:    vaddps %zmm2, %zmm1, %zmm1 ## encoding: [0x62,0xf1,0x74,0x48,0x58,0xca]
-; X86-NEXT:    vmovsldup %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7e,0xc9,0x12,0xc0]
-; X86-NEXT:    ## zmm0 {%k1} {z} = zmm0[0,0,2,2,4,4,6,6,8,8,10,10,12,12,14,14]
-; X86-NEXT:    vaddps %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7c,0x48,0x58,0xc1]
+; X86-NEXT:    vmovaps %zmm1, %zmm0 ## encoding: [0x62,0xf1,0x7c,0x48,0x28,0xc1]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_movsldup_512:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vmovsldup %zmm0, %zmm2 ## encoding: [0x62,0xf1,0x7e,0x48,0x12,0xd0]
-; X64-NEXT:    ## zmm2 = zmm0[0,0,2,2,4,4,6,6,8,8,10,10,12,12,14,14]
 ; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vmovsldup %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x7e,0x49,0x12,0xc8]
 ; X64-NEXT:    ## zmm1 {%k1} = zmm0[0,0,2,2,4,4,6,6,8,8,10,10,12,12,14,14]
-; X64-NEXT:    vaddps %zmm2, %zmm1, %zmm1 ## encoding: [0x62,0xf1,0x74,0x48,0x58,0xca]
+; X64-NEXT:    vmovaps %zmm1, %zmm0 ## encoding: [0x62,0xf1,0x7c,0x48,0x28,0xc1]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %res = call <16 x float> @llvm.x86.avx512.mask.movsldup.512(<16 x float> %x0, <16 x float> %x1, i16 %x2)
+  ret <16 x float> %res
+}
+
+define <16 x float>@test_int_x86_avx512_maskz_movsldup_512(<16 x float> %x0, i16 %x2) {
+; X86-LABEL: test_int_x86_avx512_maskz_movsldup_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vmovsldup %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7e,0xc9,0x12,0xc0]
+; X86-NEXT:    ## zmm0 {%k1} {z} = zmm0[0,0,2,2,4,4,6,6,8,8,10,10,12,12,14,14]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_movsldup_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vmovsldup %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7e,0xc9,0x12,0xc0]
 ; X64-NEXT:    ## zmm0 {%k1} {z} = zmm0[0,0,2,2,4,4,6,6,8,8,10,10,12,12,14,14]
-; X64-NEXT:    vaddps %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7c,0x48,0x58,0xc1]
 ; X64-NEXT:    retq ## encoding: [0xc3]
-  %res = call <16 x float> @llvm.x86.avx512.mask.movsldup.512(<16 x float> %x0, <16 x float> %x1, i16 %x2)
-  %res1 = call <16 x float> @llvm.x86.avx512.mask.movsldup.512(<16 x float> %x0, <16 x float> %x1, i16 -1)
-  %res2 = call <16 x float> @llvm.x86.avx512.mask.movsldup.512(<16 x float> %x0, <16 x float> zeroinitializer, i16 %x2)
-  %res3 = fadd <16 x float> %res, %res1
-  %res4 = fadd <16 x float> %res2, %res3
-  ret <16 x float> %res4
+  %res = call <16 x float> @llvm.x86.avx512.mask.movsldup.512(<16 x float> %x0, <16 x float> zeroinitializer, i16 %x2)
+  ret <16 x float> %res
 }
 
 declare <16 x float> @llvm.x86.avx512.mask.movshdup.512(<16 x float>, <16 x float>, i16)
 
+define <16 x float>@test_int_x86_avx512_movshdup_512(<16 x float> %x0, <16 x float> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_movshdup_512:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vmovshdup %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7e,0x48,0x16,0xc0]
+; CHECK-NEXT:    ## zmm0 = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
+; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+  %res = call <16 x float> @llvm.x86.avx512.mask.movshdup.512(<16 x float> %x0, <16 x float> %x1, i16 -1)
+  ret <16 x float> %res
+}
+
 define <16 x float>@test_int_x86_avx512_mask_movshdup_512(<16 x float> %x0, <16 x float> %x1, i16 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_movshdup_512:
 ; X86:       ## %bb.0:
-; X86-NEXT:    vmovshdup %zmm0, %zmm2 ## encoding: [0x62,0xf1,0x7e,0x48,0x16,0xd0]
-; X86-NEXT:    ## zmm2 = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vmovshdup %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x7e,0x49,0x16,0xc8]
 ; X86-NEXT:    ## zmm1 {%k1} = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
-; X86-NEXT:    vaddps %zmm2, %zmm1, %zmm1 ## encoding: [0x62,0xf1,0x74,0x48,0x58,0xca]
-; X86-NEXT:    vmovshdup %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7e,0xc9,0x16,0xc0]
-; X86-NEXT:    ## zmm0 {%k1} {z} = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
-; X86-NEXT:    vaddps %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7c,0x48,0x58,0xc1]
+; X86-NEXT:    vmovaps %zmm1, %zmm0 ## encoding: [0x62,0xf1,0x7c,0x48,0x28,0xc1]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_movshdup_512:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vmovshdup %zmm0, %zmm2 ## encoding: [0x62,0xf1,0x7e,0x48,0x16,0xd0]
-; X64-NEXT:    ## zmm2 = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
 ; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vmovshdup %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x7e,0x49,0x16,0xc8]
 ; X64-NEXT:    ## zmm1 {%k1} = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
-; X64-NEXT:    vaddps %zmm2, %zmm1, %zmm1 ## encoding: [0x62,0xf1,0x74,0x48,0x58,0xca]
+; X64-NEXT:    vmovaps %zmm1, %zmm0 ## encoding: [0x62,0xf1,0x7c,0x48,0x28,0xc1]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %res = call <16 x float> @llvm.x86.avx512.mask.movshdup.512(<16 x float> %x0, <16 x float> %x1, i16 %x2)
+  ret <16 x float> %res
+}
+
+define <16 x float>@test_int_x86_avx512_maskz_movshdup_512(<16 x float> %x0, i16 %x2) {
+; X86-LABEL: test_int_x86_avx512_maskz_movshdup_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vmovshdup %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7e,0xc9,0x16,0xc0]
+; X86-NEXT:    ## zmm0 {%k1} {z} = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_movshdup_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vmovshdup %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7e,0xc9,0x16,0xc0]
 ; X64-NEXT:    ## zmm0 {%k1} {z} = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
-; X64-NEXT:    vaddps %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7c,0x48,0x58,0xc1]
 ; X64-NEXT:    retq ## encoding: [0xc3]
-  %res = call <16 x float> @llvm.x86.avx512.mask.movshdup.512(<16 x float> %x0, <16 x float> %x1, i16 %x2)
-  %res1 = call <16 x float> @llvm.x86.avx512.mask.movshdup.512(<16 x float> %x0, <16 x float> %x1, i16 -1)
-  %res2 = call <16 x float> @llvm.x86.avx512.mask.movshdup.512(<16 x float> %x0, <16 x float> zeroinitializer, i16 %x2)
-  %res3 = fadd <16 x float> %res, %res1
-  %res4 = fadd <16 x float> %res2, %res3
-  ret <16 x float> %res4
+  %res = call <16 x float> @llvm.x86.avx512.mask.movshdup.512(<16 x float> %x0, <16 x float> zeroinitializer, i16 %x2)
+  ret <16 x float> %res
 }
 
 declare <8 x double> @llvm.x86.avx512.mask.movddup.512(<8 x double>, <8 x double>, i8)
@@ -3904,140 +3932,202 @@ define <8 x i64>@test_int_x86_avx512_maskz_pmovsxw_q_512(<8 x i16> %x0, i8 %x2)
 
 declare <16 x i32> @llvm.x86.avx512.prolv.d.512(<16 x i32>, <16 x i32>)
 
-define <16 x i32>@test_int_x86_avx512_prolv_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) {
-; X86-LABEL: test_int_x86_avx512_prolv_d_512:
+define <16 x i32>@test_int_x86_avx512_prolv_d_512(<16 x i32> %x0, <16 x i32> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_prolv_d_512:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vprolvd %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf2,0x7d,0x48,0x15,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+  %1 = call <16 x i32> @llvm.x86.avx512.prolv.d.512(<16 x i32> %x0, <16 x i32> %x1)
+  ret <16 x i32> %1
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_prolv_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_prolv_d_512:
 ; X86:       ## %bb.0:
-; X86-NEXT:    vprolvd %zmm1, %zmm0, %zmm3 ## encoding: [0x62,0xf2,0x7d,0x48,0x15,0xd9]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vprolvd %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x15,0xd1]
-; X86-NEXT:    vprolvd %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x15,0xc1]
-; X86-NEXT:    vpaddd %zmm3, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc3]
-; X86-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_prolv_d_512:
+; X64-LABEL: test_int_x86_avx512_mask_prolv_d_512:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vprolvd %zmm1, %zmm0, %zmm3 ## encoding: [0x62,0xf2,0x7d,0x48,0x15,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vprolvd %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x15,0xd1]
-; X64-NEXT:    vprolvd %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x15,0xc1]
-; X64-NEXT:    vpaddd %zmm3, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc3]
-; X64-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %1 = call <16 x i32> @llvm.x86.avx512.prolv.d.512(<16 x i32> %x0, <16 x i32> %x1)
   %2 = bitcast i16 %x3 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x2
-  %4 = call <16 x i32> @llvm.x86.avx512.prolv.d.512(<16 x i32> %x0, <16 x i32> %x1)
-  %5 = bitcast i16 %x3 to <16 x i1>
-  %6 = select <16 x i1> %5, <16 x i32> %4, <16 x i32> zeroinitializer
-  %7 = call <16 x i32> @llvm.x86.avx512.prolv.d.512(<16 x i32> %x0, <16 x i32> %x1)
-  %res3 = add <16 x i32> %3, %6
-  %res4 = add <16 x i32> %res3, %7
-  ret <16 x i32> %res4
+  ret <16 x i32> %3
+}
+
+define <16 x i32>@test_int_x86_avx512_maskz_prolv_d_512(<16 x i32> %x0, <16 x i32> %x1, i16 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_prolv_d_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vprolvd %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x15,0xc1]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_prolv_d_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vprolvd %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x15,0xc1]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %1 = call <16 x i32> @llvm.x86.avx512.prolv.d.512(<16 x i32> %x0, <16 x i32> %x1)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> zeroinitializer
+  ret <16 x i32> %3
 }
 
 declare <8 x i64> @llvm.x86.avx512.prolv.q.512(<8 x i64>, <8 x i64>)
 
-define <8 x i64>@test_int_x86_avx512_prolv_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_prolv_q_512:
+define <8 x i64>@test_int_x86_avx512_prolv_q_512(<8 x i64> %x0, <8 x i64> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_prolv_q_512:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vprolvq %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf2,0xfd,0x48,0x15,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+  %1 = call <8 x i64> @llvm.x86.avx512.prolv.q.512(<8 x i64> %x0, <8 x i64> %x1)
+  ret <8 x i64> %1
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_prolv_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_prolv_q_512:
 ; X86:       ## %bb.0:
-; X86-NEXT:    vprolvq %zmm1, %zmm0, %zmm3 ## encoding: [0x62,0xf2,0xfd,0x48,0x15,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprolvq %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x15,0xd1]
-; X86-NEXT:    vprolvq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x15,0xc1]
-; X86-NEXT:    vpaddq %zmm3, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc3]
-; X86-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_prolv_q_512:
+; X64-LABEL: test_int_x86_avx512_mask_prolv_q_512:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vprolvq %zmm1, %zmm0, %zmm3 ## encoding: [0x62,0xf2,0xfd,0x48,0x15,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vprolvq %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x15,0xd1]
-; X64-NEXT:    vprolvq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x15,0xc1]
-; X64-NEXT:    vpaddq %zmm3, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc3]
-; X64-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %1 = call <8 x i64> @llvm.x86.avx512.prolv.q.512(<8 x i64> %x0, <8 x i64> %x1)
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x2
-  %4 = call <8 x i64> @llvm.x86.avx512.prolv.q.512(<8 x i64> %x0, <8 x i64> %x1)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %6 = select <8 x i1> %5, <8 x i64> %4, <8 x i64> zeroinitializer
-  %7 = call <8 x i64> @llvm.x86.avx512.prolv.q.512(<8 x i64> %x0, <8 x i64> %x1)
-  %res3 = add <8 x i64> %3, %6
-  %res4 = add <8 x i64> %res3, %7
-  ret <8 x i64> %res4
+  ret <8 x i64> %3
+}
+
+define <8 x i64>@test_int_x86_avx512_maskz_prolv_q_512(<8 x i64> %x0, <8 x i64> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_prolv_q_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vprolvq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x15,0xc1]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_prolv_q_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vprolvq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x15,0xc1]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %1 = call <8 x i64> @llvm.x86.avx512.prolv.q.512(<8 x i64> %x0, <8 x i64> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> zeroinitializer
+  ret <8 x i64> %3
 }
 
 declare <16 x i32> @llvm.x86.avx512.prorv.d.512(<16 x i32>, <16 x i32>)
 
-define <16 x i32>@test_int_x86_avx512_prorv_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) {
-; X86-LABEL: test_int_x86_avx512_prorv_d_512:
+define <16 x i32>@test_int_x86_avx512_prorv_d_512(<16 x i32> %x0, <16 x i32> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_prorv_d_512:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vprorvd %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf2,0x7d,0x48,0x14,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+  %1 = call <16 x i32> @llvm.x86.avx512.prorv.d.512(<16 x i32> %x0, <16 x i32> %x1)
+  ret <16 x i32> %1
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_prorv_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_prorv_d_512:
 ; X86:       ## %bb.0:
-; X86-NEXT:    vprorvd %zmm1, %zmm0, %zmm3 ## encoding: [0x62,0xf2,0x7d,0x48,0x14,0xd9]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vprorvd %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x14,0xd1]
-; X86-NEXT:    vprorvd %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x14,0xc1]
-; X86-NEXT:    vpaddd %zmm3, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc3]
-; X86-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_prorv_d_512:
+; X64-LABEL: test_int_x86_avx512_mask_prorv_d_512:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vprorvd %zmm1, %zmm0, %zmm3 ## encoding: [0x62,0xf2,0x7d,0x48,0x14,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vprorvd %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x14,0xd1]
-; X64-NEXT:    vprorvd %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x14,0xc1]
-; X64-NEXT:    vpaddd %zmm3, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc3]
-; X64-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %1 = call <16 x i32> @llvm.x86.avx512.prorv.d.512(<16 x i32> %x0, <16 x i32> %x1)
   %2 = bitcast i16 %x3 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x2
-  %4 = call <16 x i32> @llvm.x86.avx512.prorv.d.512(<16 x i32> %x0, <16 x i32> %x1)
-  %5 = bitcast i16 %x3 to <16 x i1>
-  %6 = select <16 x i1> %5, <16 x i32> %4, <16 x i32> zeroinitializer
-  %7 = call <16 x i32> @llvm.x86.avx512.prorv.d.512(<16 x i32> %x0, <16 x i32> %x1)
-  %res3 = add <16 x i32> %3, %6
-  %res4 = add <16 x i32> %res3, %7
-  ret <16 x i32> %res4
+  ret <16 x i32> %3
+}
+
+define <16 x i32>@test_int_x86_avx512_maskz_prorv_d_512(<16 x i32> %x0, <16 x i32> %x1, i16 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_prorv_d_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vprorvd %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x14,0xc1]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_prorv_d_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vprorvd %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x14,0xc1]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %1 = call <16 x i32> @llvm.x86.avx512.prorv.d.512(<16 x i32> %x0, <16 x i32> %x1)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> zeroinitializer
+  ret <16 x i32> %3
 }
 
 declare <8 x i64> @llvm.x86.avx512.prorv.q.512(<8 x i64>, <8 x i64>)
 
-define <8 x i64>@test_int_x86_avx512_prorv_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_prorv_q_512:
+define <8 x i64>@test_int_x86_avx512_prorv_q_512(<8 x i64> %x0, <8 x i64> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_prorv_q_512:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vprorvq %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf2,0xfd,0x48,0x14,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+  %1 = call <8 x i64> @llvm.x86.avx512.prorv.q.512(<8 x i64> %x0, <8 x i64> %x1)
+  ret <8 x i64> %1
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_prorv_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_prorv_q_512:
 ; X86:       ## %bb.0:
-; X86-NEXT:    vprorvq %zmm1, %zmm0, %zmm3 ## encoding: [0x62,0xf2,0xfd,0x48,0x14,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprorvq %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x14,0xd1]
-; X86-NEXT:    vprorvq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x14,0xc1]
-; X86-NEXT:    vpaddq %zmm3, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc3]
-; X86-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_prorv_q_512:
+; X64-LABEL: test_int_x86_avx512_mask_prorv_q_512:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vprorvq %zmm1, %zmm0, %zmm3 ## encoding: [0x62,0xf2,0xfd,0x48,0x14,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vprorvq %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x14,0xd1]
-; X64-NEXT:    vprorvq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x14,0xc1]
-; X64-NEXT:    vpaddq %zmm3, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc3]
-; X64-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %1 = call <8 x i64> @llvm.x86.avx512.prorv.q.512(<8 x i64> %x0, <8 x i64> %x1)
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x2
-  %4 = call <8 x i64> @llvm.x86.avx512.prorv.q.512(<8 x i64> %x0, <8 x i64> %x1)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %6 = select <8 x i1> %5, <8 x i64> %4, <8 x i64> zeroinitializer
-  %7 = call <8 x i64> @llvm.x86.avx512.prorv.q.512(<8 x i64> %x0, <8 x i64> %x1)
-  %res3 = add <8 x i64> %3, %6
-  %res4 = add <8 x i64> %res3, %7
-  ret <8 x i64> %res4
+  ret <8 x i64> %3
+}
+
+define <8 x i64>@test_int_x86_avx512_maskz_prorv_q_512(<8 x i64> %x0, <8 x i64> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_prorv_q_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vprorvq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x14,0xc1]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_prorv_q_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vprorvq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x14,0xc1]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %1 = call <8 x i64> @llvm.x86.avx512.prorv.q.512(<8 x i64> %x0, <8 x i64> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> zeroinitializer
+  ret <8 x i64> %3
 }
 
 declare <16 x i32> @llvm.x86.avx512.prol.d.512(<16 x i32>, i32)
@@ -4045,30 +4135,30 @@ declare <16 x i32> @llvm.x86.avx512.prol.d.512(<16 x i32>, i32)
 define <16 x i32>@test_int_x86_avx512_prol_d_512(<16 x i32> %x0, i32 %x1, <16 x i32> %x2, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_prol_d_512:
 ; X86:       ## %bb.0:
-; X86-NEXT:    vprold $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0x6d,0x48,0x72,0xc8,0x03]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vprold $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x75,0x49,0x72,0xc8,0x03]
-; X86-NEXT:    vprold $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0x72,0xc8,0x03]
-; X86-NEXT:    vpaddd %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc2]
+; X86-NEXT:    vprold $4, %zmm0, %zmm2 {%k1} {z} ## encoding: [0x62,0xf1,0x6d,0xc9,0x72,0xc8,0x04]
+; X86-NEXT:    vprold $5, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0x72,0xc8,0x05]
+; X86-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0x75,0x48,0xfe,0xc0]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_prol_d_512:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vprold $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0x6d,0x48,0x72,0xc8,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 ## encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vprold $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x75,0x49,0x72,0xc8,0x03]
-; X64-NEXT:    vprold $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0x72,0xc8,0x03]
-; X64-NEXT:    vpaddd %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc2]
+; X64-NEXT:    vprold $4, %zmm0, %zmm2 {%k1} {z} ## encoding: [0x62,0xf1,0x6d,0xc9,0x72,0xc8,0x04]
+; X64-NEXT:    vprold $5, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0x72,0xc8,0x05]
+; X64-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0x75,0x48,0xfe,0xc0]
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %1 = call <16 x i32> @llvm.x86.avx512.prol.d.512(<16 x i32> %x0, i32 3)
   %2 = bitcast i16 %x3 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x2
-  %4 = call <16 x i32> @llvm.x86.avx512.prol.d.512(<16 x i32> %x0, i32 3)
+  %4 = call <16 x i32> @llvm.x86.avx512.prol.d.512(<16 x i32> %x0, i32 4)
   %5 = bitcast i16 %x3 to <16 x i1>
   %6 = select <16 x i1> %5, <16 x i32> %4, <16 x i32> zeroinitializer
-  %7 = call <16 x i32> @llvm.x86.avx512.prol.d.512(<16 x i32> %x0, i32 3)
+  %7 = call <16 x i32> @llvm.x86.avx512.prol.d.512(<16 x i32> %x0, i32 5)
   %res3 = add <16 x i32> %3, %6
   %res4 = add <16 x i32> %res3, %7
   ret <16 x i32> %res4
@@ -4079,31 +4169,31 @@ declare <8 x i64> @llvm.x86.avx512.prol.q.512(<8 x i64>, i32)
 define <8 x i64>@test_int_x86_avx512_prol_q_512(<8 x i64> %x0, i32 %x1, <8 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_prol_q_512:
 ; X86:       ## %bb.0:
-; X86-NEXT:    vprolq $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0xed,0x48,0x72,0xc8,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprolq $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xf5,0x49,0x72,0xc8,0x03]
-; X86-NEXT:    vprolq $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0x72,0xc8,0x03]
-; X86-NEXT:    vpaddq %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc2]
+; X86-NEXT:    vprolq $4, %zmm0, %zmm2 {%k1} {z} ## encoding: [0x62,0xf1,0xed,0xc9,0x72,0xc8,0x04]
+; X86-NEXT:    vprolq $5, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x72,0xc8,0x05]
+; X86-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xf5,0x48,0xd4,0xc0]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_prol_q_512:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vprolq $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0xed,0x48,0x72,0xc8,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 ## encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vprolq $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xf5,0x49,0x72,0xc8,0x03]
-; X64-NEXT:    vprolq $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0x72,0xc8,0x03]
-; X64-NEXT:    vpaddq %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc2]
+; X64-NEXT:    vprolq $4, %zmm0, %zmm2 {%k1} {z} ## encoding: [0x62,0xf1,0xed,0xc9,0x72,0xc8,0x04]
+; X64-NEXT:    vprolq $5, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x72,0xc8,0x05]
+; X64-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xf5,0x48,0xd4,0xc0]
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %1 = call <8 x i64> @llvm.x86.avx512.prol.q.512(<8 x i64> %x0, i32 3)
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x2
-  %4 = call <8 x i64> @llvm.x86.avx512.prol.q.512(<8 x i64> %x0, i32 3)
+  %4 = call <8 x i64> @llvm.x86.avx512.prol.q.512(<8 x i64> %x0, i32 4)
   %5 = bitcast i8 %x3 to <8 x i1>
   %6 = select <8 x i1> %5, <8 x i64> %4, <8 x i64> zeroinitializer
-  %7 = call <8 x i64> @llvm.x86.avx512.prol.q.512(<8 x i64> %x0, i32 3)
+  %7 = call <8 x i64> @llvm.x86.avx512.prol.q.512(<8 x i64> %x0, i32 5)
   %res3 = add <8 x i64> %3, %6
   %res4 = add <8 x i64> %res3, %7
   ret <8 x i64> %res4
@@ -4114,30 +4204,30 @@ declare <16 x i32> @llvm.x86.avx512.pror.d.512(<16 x i32>, i32)
 define <16 x i32>@test_int_x86_avx512_pror_d_512(<16 x i32> %x0, i32 %x1, <16 x i32> %x2, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_pror_d_512:
 ; X86:       ## %bb.0:
-; X86-NEXT:    vprord $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0x6d,0x48,0x72,0xc0,0x03]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vprord $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x75,0x49,0x72,0xc0,0x03]
-; X86-NEXT:    vprord $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0x72,0xc0,0x03]
-; X86-NEXT:    vpaddd %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc2]
+; X86-NEXT:    vprord $4, %zmm0, %zmm2 {%k1} {z} ## encoding: [0x62,0xf1,0x6d,0xc9,0x72,0xc0,0x04]
+; X86-NEXT:    vprord $5, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0x72,0xc0,0x05]
+; X86-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0x75,0x48,0xfe,0xc0]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_pror_d_512:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vprord $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0x6d,0x48,0x72,0xc0,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 ## encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vprord $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x75,0x49,0x72,0xc0,0x03]
-; X64-NEXT:    vprord $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0x72,0xc0,0x03]
-; X64-NEXT:    vpaddd %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc2]
+; X64-NEXT:    vprord $4, %zmm0, %zmm2 {%k1} {z} ## encoding: [0x62,0xf1,0x6d,0xc9,0x72,0xc0,0x04]
+; X64-NEXT:    vprord $5, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0x72,0xc0,0x05]
+; X64-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0x75,0x48,0xfe,0xc0]
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %1 = call <16 x i32> @llvm.x86.avx512.pror.d.512(<16 x i32> %x0, i32 3)
   %2 = bitcast i16 %x3 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x2
-  %4 = call <16 x i32> @llvm.x86.avx512.pror.d.512(<16 x i32> %x0, i32 3)
+  %4 = call <16 x i32> @llvm.x86.avx512.pror.d.512(<16 x i32> %x0, i32 4)
   %5 = bitcast i16 %x3 to <16 x i1>
   %6 = select <16 x i1> %5, <16 x i32> %4, <16 x i32> zeroinitializer
-  %7 = call <16 x i32> @llvm.x86.avx512.pror.d.512(<16 x i32> %x0, i32 3)
+  %7 = call <16 x i32> @llvm.x86.avx512.pror.d.512(<16 x i32> %x0, i32 5)
   %res3 = add <16 x i32> %3, %6
   %res4 = add <16 x i32> %res3, %7
   ret <16 x i32> %res4
@@ -4148,31 +4238,31 @@ declare <8 x i64> @llvm.x86.avx512.pror.q.512(<8 x i64>, i32)
 define <8 x i64>@test_int_x86_avx512_pror_q_512(<8 x i64> %x0, i32 %x1, <8 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_pror_q_512:
 ; X86:       ## %bb.0:
-; X86-NEXT:    vprorq $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0xed,0x48,0x72,0xc0,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprorq $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xf5,0x49,0x72,0xc0,0x03]
-; X86-NEXT:    vprorq $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0x72,0xc0,0x03]
-; X86-NEXT:    vpaddq %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc2]
+; X86-NEXT:    vprorq $4, %zmm0, %zmm2 {%k1} {z} ## encoding: [0x62,0xf1,0xed,0xc9,0x72,0xc0,0x04]
+; X86-NEXT:    vprorq $5, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x72,0xc0,0x05]
+; X86-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xf5,0x48,0xd4,0xc0]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_pror_q_512:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vprorq $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0xed,0x48,0x72,0xc0,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 ## encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vprorq $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xf5,0x49,0x72,0xc0,0x03]
-; X64-NEXT:    vprorq $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0x72,0xc0,0x03]
-; X64-NEXT:    vpaddq %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc2]
+; X64-NEXT:    vprorq $4, %zmm0, %zmm2 {%k1} {z} ## encoding: [0x62,0xf1,0xed,0xc9,0x72,0xc0,0x04]
+; X64-NEXT:    vprorq $5, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x72,0xc0,0x05]
+; X64-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xf5,0x48,0xd4,0xc0]
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %1 = call <8 x i64> @llvm.x86.avx512.pror.q.512(<8 x i64> %x0, i32 3)
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x2
-  %4 = call <8 x i64> @llvm.x86.avx512.pror.q.512(<8 x i64> %x0, i32 3)
+  %4 = call <8 x i64> @llvm.x86.avx512.pror.q.512(<8 x i64> %x0, i32 4)
   %5 = bitcast i8 %x3 to <8 x i1>
   %6 = select <8 x i1> %5, <8 x i64> %4, <8 x i64> zeroinitializer
-  %7 = call <8 x i64> @llvm.x86.avx512.pror.q.512(<8 x i64> %x0, i32 3)
+  %7 = call <8 x i64> @llvm.x86.avx512.pror.q.512(<8 x i64> %x0, i32 5)
   %res3 = add <8 x i64> %3, %6
   %res4 = add <8 x i64> %res3, %7
   ret <8 x i64> %res4
@@ -4183,27 +4273,27 @@ declare <8 x i64> @llvm.x86.avx512.mask.psrl.qi.512(<8 x i64>, i32, <8 x i64>, i
 define <8 x i64>@test_int_x86_avx512_mask_psrl_qi_512(<8 x i64> %x0, i32 %x1, <8 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrl_qi_512:
 ; X86:       ## %bb.0:
-; X86-NEXT:    vpsrlq $4, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0xed,0x48,0x73,0xd0,0x04]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpsrlq $4, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xf5,0x49,0x73,0xd0,0x04]
-; X86-NEXT:    vpsrlq $4, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0x73,0xd0,0x04]
+; X86-NEXT:    vpsrlq $5, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0xed,0x48,0x73,0xd0,0x05]
+; X86-NEXT:    vpsrlq $6, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0x73,0xd0,0x06]
 ; X86-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xf5,0x48,0xd4,0xc0]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrl_qi_512:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vpsrlq $4, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0xed,0x48,0x73,0xd0,0x04]
 ; X64-NEXT:    kmovw %esi, %k1 ## encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vpsrlq $4, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xf5,0x49,0x73,0xd0,0x04]
-; X64-NEXT:    vpsrlq $4, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0x73,0xd0,0x04]
+; X64-NEXT:    vpsrlq $5, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0xed,0x48,0x73,0xd0,0x05]
+; X64-NEXT:    vpsrlq $6, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0x73,0xd0,0x06]
 ; X64-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xf5,0x48,0xd4,0xc0]
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %res = call <8 x i64> @llvm.x86.avx512.mask.psrl.qi.512(<8 x i64> %x0, i32 4, <8 x i64> %x2, i8 %x3)
-  %res1 = call <8 x i64> @llvm.x86.avx512.mask.psrl.qi.512(<8 x i64> %x0, i32 4, <8 x i64> %x2, i8 -1)
-  %res2 = call <8 x i64> @llvm.x86.avx512.mask.psrl.qi.512(<8 x i64> %x0, i32 4, <8 x i64> zeroinitializer, i8 %x3)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.psrl.qi.512(<8 x i64> %x0, i32 5, <8 x i64> %x2, i8 -1)
+  %res2 = call <8 x i64> @llvm.x86.avx512.mask.psrl.qi.512(<8 x i64> %x0, i32 6, <8 x i64> zeroinitializer, i8 %x3)
   %res3 = add <8 x i64> %res, %res1
   %res4 = add <8 x i64> %res3, %res2
   ret <8 x i64> %res4
@@ -4214,26 +4304,26 @@ declare <16 x i32> @llvm.x86.avx512.mask.psrl.di.512(<16 x i32>, i32, <16 x i32>
 define <16 x i32>@test_int_x86_avx512_mask_psrl_di_512(<16 x i32> %x0, i32 %x1, <16 x i32> %x2, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrl_di_512:
 ; X86:       ## %bb.0:
-; X86-NEXT:    vpsrld $4, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0x6d,0x48,0x72,0xd0,0x04]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vpsrld $4, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x75,0x49,0x72,0xd0,0x04]
-; X86-NEXT:    vpsrld $4, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0x72,0xd0,0x04]
+; X86-NEXT:    vpsrld $5, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0x6d,0x48,0x72,0xd0,0x05]
+; X86-NEXT:    vpsrld $6, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0x72,0xd0,0x06]
 ; X86-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0x75,0x48,0xfe,0xc0]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrl_di_512:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vpsrld $4, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0x6d,0x48,0x72,0xd0,0x04]
 ; X64-NEXT:    kmovw %esi, %k1 ## encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vpsrld $4, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x75,0x49,0x72,0xd0,0x04]
-; X64-NEXT:    vpsrld $4, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0x72,0xd0,0x04]
+; X64-NEXT:    vpsrld $5, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0x6d,0x48,0x72,0xd0,0x05]
+; X64-NEXT:    vpsrld $6, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0x72,0xd0,0x06]
 ; X64-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0x75,0x48,0xfe,0xc0]
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %res = call <16 x i32> @llvm.x86.avx512.mask.psrl.di.512(<16 x i32> %x0, i32 4, <16 x i32> %x2, i16 %x3)
-  %res1 = call <16 x i32> @llvm.x86.avx512.mask.psrl.di.512(<16 x i32> %x0, i32 4, <16 x i32> %x2, i16 -1)
-  %res2 = call <16 x i32> @llvm.x86.avx512.mask.psrl.di.512(<16 x i32> %x0, i32 4, <16 x i32> zeroinitializer, i16 %x3)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.psrl.di.512(<16 x i32> %x0, i32 5, <16 x i32> %x2, i16 -1)
+  %res2 = call <16 x i32> @llvm.x86.avx512.mask.psrl.di.512(<16 x i32> %x0, i32 6, <16 x i32> zeroinitializer, i16 %x3)
   %res3 = add <16 x i32> %res, %res1
   %res4 = add <16 x i32> %res3, %res2
   ret <16 x i32> %res4
@@ -4244,26 +4334,26 @@ declare <16 x i32> @llvm.x86.avx512.mask.psra.di.512(<16 x i32>, i32, <16 x i32>
 define <16 x i32>@test_int_x86_avx512_mask_psra_di_512(<16 x i32> %x0, i32 %x1, <16 x i32> %x2, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psra_di_512:
 ; X86:       ## %bb.0:
-; X86-NEXT:    vpsrad $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0x6d,0x48,0x72,0xe0,0x03]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vpsrad $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x75,0x49,0x72,0xe0,0x03]
-; X86-NEXT:    vpsrad $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0x72,0xe0,0x03]
-; X86-NEXT:    vpaddd %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc2]
+; X86-NEXT:    vpsrad $4, %zmm0, %zmm2 {%k1} {z} ## encoding: [0x62,0xf1,0x6d,0xc9,0x72,0xe0,0x04]
+; X86-NEXT:    vpsrad $5, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0x72,0xe0,0x05]
+; X86-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0x75,0x48,0xfe,0xc0]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psra_di_512:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vpsrad $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0x6d,0x48,0x72,0xe0,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 ## encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vpsrad $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x75,0x49,0x72,0xe0,0x03]
-; X64-NEXT:    vpsrad $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0x72,0xe0,0x03]
-; X64-NEXT:    vpaddd %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc2]
+; X64-NEXT:    vpsrad $4, %zmm0, %zmm2 {%k1} {z} ## encoding: [0x62,0xf1,0x6d,0xc9,0x72,0xe0,0x04]
+; X64-NEXT:    vpsrad $5, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0x72,0xe0,0x05]
+; X64-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0x75,0x48,0xfe,0xc0]
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %res = call <16 x i32> @llvm.x86.avx512.mask.psra.di.512(<16 x i32> %x0, i32 3, <16 x i32> %x2, i16 %x3)
-  %res1 = call <16 x i32> @llvm.x86.avx512.mask.psra.di.512(<16 x i32> %x0, i32 3, <16 x i32> zeroinitializer, i16 %x3)
-  %res2 = call <16 x i32> @llvm.x86.avx512.mask.psra.di.512(<16 x i32> %x0, i32 3, <16 x i32> %x2, i16 -1)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.psra.di.512(<16 x i32> %x0, i32 4, <16 x i32> zeroinitializer, i16 %x3)
+  %res2 = call <16 x i32> @llvm.x86.avx512.mask.psra.di.512(<16 x i32> %x0, i32 5, <16 x i32> %x2, i16 -1)
   %res3 = add <16 x i32> %res, %res1
   %res4 = add <16 x i32> %res3, %res2
   ret <16 x i32> %res4
@@ -4274,27 +4364,27 @@ declare <8 x i64> @llvm.x86.avx512.mask.psra.qi.512(<8 x i64>, i32, <8 x i64>, i
 define <8 x i64>@test_int_x86_avx512_mask_psra_qi_512(<8 x i64> %x0, i32 %x1, <8 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psra_qi_512:
 ; X86:       ## %bb.0:
-; X86-NEXT:    vpsraq $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0xed,0x48,0x72,0xe0,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpsraq $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xf5,0x49,0x72,0xe0,0x03]
-; X86-NEXT:    vpsraq $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0x72,0xe0,0x03]
-; X86-NEXT:    vpaddq %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc2]
+; X86-NEXT:    vpsraq $4, %zmm0, %zmm2 {%k1} {z} ## encoding: [0x62,0xf1,0xed,0xc9,0x72,0xe0,0x04]
+; X86-NEXT:    vpsraq $5, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x72,0xe0,0x05]
+; X86-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xf5,0x48,0xd4,0xc0]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psra_qi_512:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vpsraq $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0xed,0x48,0x72,0xe0,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 ## encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vpsraq $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xf5,0x49,0x72,0xe0,0x03]
-; X64-NEXT:    vpsraq $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0x72,0xe0,0x03]
-; X64-NEXT:    vpaddq %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc2]
+; X64-NEXT:    vpsraq $4, %zmm0, %zmm2 {%k1} {z} ## encoding: [0x62,0xf1,0xed,0xc9,0x72,0xe0,0x04]
+; X64-NEXT:    vpsraq $5, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x72,0xe0,0x05]
+; X64-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xf5,0x48,0xd4,0xc0]
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %res = call <8 x i64> @llvm.x86.avx512.mask.psra.qi.512(<8 x i64> %x0, i32 3, <8 x i64> %x2, i8 %x3)
-  %res1 = call <8 x i64> @llvm.x86.avx512.mask.psra.qi.512(<8 x i64> %x0, i32 3, <8 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <8 x i64> @llvm.x86.avx512.mask.psra.qi.512(<8 x i64> %x0, i32 3, <8 x i64> %x2, i8 -1)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.psra.qi.512(<8 x i64> %x0, i32 4, <8 x i64> zeroinitializer, i8 %x3)
+  %res2 = call <8 x i64> @llvm.x86.avx512.mask.psra.qi.512(<8 x i64> %x0, i32 5, <8 x i64> %x2, i8 -1)
   %res3 = add <8 x i64> %res, %res1
   %res4 = add <8 x i64> %res3, %res2
   ret <8 x i64> %res4
@@ -4305,26 +4395,26 @@ declare <16 x i32> @llvm.x86.avx512.mask.psll.di.512(<16 x i32>, i32, <16 x i32>
 define <16 x i32>@test_int_x86_avx512_mask_psll_di_512(<16 x i32> %x0, i32 %x1, <16 x i32> %x2, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psll_di_512:
 ; X86:       ## %bb.0:
-; X86-NEXT:    vpslld $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0x6d,0x48,0x72,0xf0,0x03]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vpslld $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x75,0x49,0x72,0xf0,0x03]
-; X86-NEXT:    vpslld $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0x72,0xf0,0x03]
-; X86-NEXT:    vpaddd %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc2]
+; X86-NEXT:    vpslld $4, %zmm0, %zmm2 {%k1} {z} ## encoding: [0x62,0xf1,0x6d,0xc9,0x72,0xf0,0x04]
+; X86-NEXT:    vpslld $5, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0x72,0xf0,0x05]
+; X86-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0x75,0x48,0xfe,0xc0]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psll_di_512:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vpslld $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0x6d,0x48,0x72,0xf0,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 ## encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vpslld $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x75,0x49,0x72,0xf0,0x03]
-; X64-NEXT:    vpslld $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0x72,0xf0,0x03]
-; X64-NEXT:    vpaddd %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc2]
+; X64-NEXT:    vpslld $4, %zmm0, %zmm2 {%k1} {z} ## encoding: [0x62,0xf1,0x6d,0xc9,0x72,0xf0,0x04]
+; X64-NEXT:    vpslld $5, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0x72,0xf0,0x05]
+; X64-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0x75,0x48,0xfe,0xc0]
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %res = call <16 x i32> @llvm.x86.avx512.mask.psll.di.512(<16 x i32> %x0, i32 3, <16 x i32> %x2, i16 %x3)
-  %res1 = call <16 x i32> @llvm.x86.avx512.mask.psll.di.512(<16 x i32> %x0, i32 3, <16 x i32> zeroinitializer, i16 %x3)
-  %res2 = call <16 x i32> @llvm.x86.avx512.mask.psll.di.512(<16 x i32> %x0, i32 3, <16 x i32> %x2, i16 -1)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.psll.di.512(<16 x i32> %x0, i32 4, <16 x i32> zeroinitializer, i16 %x3)
+  %res2 = call <16 x i32> @llvm.x86.avx512.mask.psll.di.512(<16 x i32> %x0, i32 5, <16 x i32> %x2, i16 -1)
   %res3 = add <16 x i32> %res, %res1
   %res4 = add <16 x i32> %res3, %res2
   ret <16 x i32> %res4
@@ -4335,27 +4425,27 @@ declare <8 x i64> @llvm.x86.avx512.mask.psll.qi.512(<8 x i64>, i32, <8 x i64>, i
 define <8 x i64>@test_int_x86_avx512_mask_psll_qi_512(<8 x i64> %x0, i32 %x1, <8 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psll_qi_512:
 ; X86:       ## %bb.0:
-; X86-NEXT:    vpsllq $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0xed,0x48,0x73,0xf0,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpsllq $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xf5,0x49,0x73,0xf0,0x03]
-; X86-NEXT:    vpsllq $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0x73,0xf0,0x03]
-; X86-NEXT:    vpaddq %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc2]
+; X86-NEXT:    vpsllq $4, %zmm0, %zmm2 {%k1} {z} ## encoding: [0x62,0xf1,0xed,0xc9,0x73,0xf0,0x04]
+; X86-NEXT:    vpsllq $5, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x73,0xf0,0x05]
+; X86-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xf5,0x48,0xd4,0xc0]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psll_qi_512:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vpsllq $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0xed,0x48,0x73,0xf0,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 ## encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vpsllq $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xf5,0x49,0x73,0xf0,0x03]
-; X64-NEXT:    vpsllq $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0x73,0xf0,0x03]
-; X64-NEXT:    vpaddq %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc2]
+; X64-NEXT:    vpsllq $4, %zmm0, %zmm2 {%k1} {z} ## encoding: [0x62,0xf1,0xed,0xc9,0x73,0xf0,0x04]
+; X64-NEXT:    vpsllq $5, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x73,0xf0,0x05]
+; X64-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xf5,0x48,0xd4,0xc0]
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %res = call <8 x i64> @llvm.x86.avx512.mask.psll.qi.512(<8 x i64> %x0, i32 3, <8 x i64> %x2, i8 %x3)
-  %res1 = call <8 x i64> @llvm.x86.avx512.mask.psll.qi.512(<8 x i64> %x0, i32 3, <8 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <8 x i64> @llvm.x86.avx512.mask.psll.qi.512(<8 x i64> %x0, i32 3, <8 x i64> %x2, i8 -1)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.psll.qi.512(<8 x i64> %x0, i32 4, <8 x i64> zeroinitializer, i8 %x3)
+  %res2 = call <8 x i64> @llvm.x86.avx512.mask.psll.qi.512(<8 x i64> %x0, i32 5, <8 x i64> %x2, i8 -1)
   %res3 = add <8 x i64> %res, %res1
   %res4 = add <8 x i64> %res3, %res2
   ret <8 x i64> %res4
@@ -5213,14 +5303,14 @@ define <16 x float>@test_int_x86_avx512_mask_vpermilvar_ps_512_constant_pool(<16
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpermilps {{.*#+}} zmm2 {%k1} = zmm0[2,3,0,1,7,6,5,4,9,8,11,10,12,13,14,15]
 ; X86-NEXT:    ## encoding: [0x62,0xf2,0x7d,0x49,0x0c,0x15,A,A,A,A]
-; X86-NEXT:    ## fixup A - offset: 6, value: LCPI291_0, kind: FK_Data_4
+; X86-NEXT:    ## fixup A - offset: 6, value: LCPI303_0, kind: FK_Data_4
 ; X86-NEXT:    vpermilps {{.*#+}} zmm1 {%k1} {z} = zmm0[1,0,3,2,4,5,6,7,9,8,11,10,12,13,14,15]
 ; X86-NEXT:    ## encoding: [0x62,0xf2,0x7d,0xc9,0x0c,0x0d,A,A,A,A]
-; X86-NEXT:    ## fixup A - offset: 6, value: LCPI291_1, kind: FK_Data_4
+; X86-NEXT:    ## fixup A - offset: 6, value: LCPI303_1, kind: FK_Data_4
 ; X86-NEXT:    vaddps %zmm1, %zmm2, %zmm1 ## encoding: [0x62,0xf1,0x6c,0x48,0x58,0xc9]
 ; X86-NEXT:    vpermilps {{.*#+}} zmm0 = zmm0[1,0,3,2,4,5,6,7,10,11,8,9,14,15,13,12]
 ; X86-NEXT:    ## encoding: [0x62,0xf2,0x7d,0x48,0x0c,0x05,A,A,A,A]
-; X86-NEXT:    ## fixup A - offset: 6, value: LCPI291_2, kind: FK_Data_4
+; X86-NEXT:    ## fixup A - offset: 6, value: LCPI303_2, kind: FK_Data_4
 ; X86-NEXT:    vaddps %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7c,0x48,0x58,0xc1]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
@@ -5229,14 +5319,14 @@ define <16 x float>@test_int_x86_avx512_mask_vpermilvar_ps_512_constant_pool(<16
 ; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpermilps {{.*#+}} zmm2 {%k1} = zmm0[2,3,0,1,7,6,5,4,9,8,11,10,12,13,14,15]
 ; X64-NEXT:    ## encoding: [0x62,0xf2,0x7d,0x49,0x0c,0x15,A,A,A,A]
-; X64-NEXT:    ## fixup A - offset: 6, value: LCPI291_0-4, kind: reloc_riprel_4byte
+; X64-NEXT:    ## fixup A - offset: 6, value: LCPI303_0-4, kind: reloc_riprel_4byte
 ; X64-NEXT:    vpermilps {{.*#+}} zmm1 {%k1} {z} = zmm0[1,0,3,2,4,5,6,7,9,8,11,10,12,13,14,15]
 ; X64-NEXT:    ## encoding: [0x62,0xf2,0x7d,0xc9,0x0c,0x0d,A,A,A,A]
-; X64-NEXT:    ## fixup A - offset: 6, value: LCPI291_1-4, kind: reloc_riprel_4byte
+; X64-NEXT:    ## fixup A - offset: 6, value: LCPI303_1-4, kind: reloc_riprel_4byte
 ; X64-NEXT:    vaddps %zmm1, %zmm2, %zmm1 ## encoding: [0x62,0xf1,0x6c,0x48,0x58,0xc9]
 ; X64-NEXT:    vpermilps {{.*#+}} zmm0 = zmm0[1,0,3,2,4,5,6,7,10,11,8,9,14,15,13,12]
 ; X64-NEXT:    ## encoding: [0x62,0xf2,0x7d,0x48,0x0c,0x05,A,A,A,A]
-; X64-NEXT:    ## fixup A - offset: 6, value: LCPI291_2-4, kind: reloc_riprel_4byte
+; X64-NEXT:    ## fixup A - offset: 6, value: LCPI303_2-4, kind: reloc_riprel_4byte
 ; X64-NEXT:    vaddps %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7c,0x48,0x58,0xc1]
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %res = call <16 x float> @llvm.x86.avx512.mask.vpermilvar.ps.512(<16 x float> %x0, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 3, i32 2, i32 1, i32 0, i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3>, <16 x float> %x2, i16 %x3)
@@ -6601,51 +6691,61 @@ define <8 x i64>@test_int_x86_avx512_mask_broadcasti64x4_512_load(<4 x i64>* %x0
 
 declare <16 x i32> @llvm.x86.avx512.mask.pabs.d.512(<16 x i32>, <16 x i32>, i16)
 
+define <16 x i32>@test_int_x86_avx512_pabs_d_512(<16 x i32> %x0, <16 x i32> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_pabs_d_512:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpabsd %zmm0, %zmm0 ## encoding: [0x62,0xf2,0x7d,0x48,0x1e,0xc0]
+; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+  %res = call <16 x i32> @llvm.x86.avx512.mask.pabs.d.512(<16 x i32> %x0, <16 x i32> %x1, i16 -1)
+  ret <16 x i32> %res
+}
+
 define <16 x i32>@test_int_x86_avx512_mask_pabs_d_512(<16 x i32> %x0, <16 x i32> %x1, i16 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_pabs_d_512:
 ; X86:       ## %bb.0:
-; X86-NEXT:    vpabsd %zmm0, %zmm2 ## encoding: [0x62,0xf2,0x7d,0x48,0x1e,0xd0]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpabsd %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x1e,0xc8]
-; X86-NEXT:    vpaddd %zmm2, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0x75,0x48,0xfe,0xc2]
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pabs_d_512:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vpabsd %zmm0, %zmm2 ## encoding: [0x62,0xf2,0x7d,0x48,0x1e,0xd0]
 ; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpabsd %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x1e,0xc8]
-; X64-NEXT:    vpaddd %zmm2, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0x75,0x48,0xfe,0xc2]
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %res = call <16 x i32> @llvm.x86.avx512.mask.pabs.d.512(<16 x i32> %x0, <16 x i32> %x1, i16 %x2)
-  %res1 = call <16 x i32> @llvm.x86.avx512.mask.pabs.d.512(<16 x i32> %x0, <16 x i32> %x1, i16 -1)
-  %res2 = add <16 x i32> %res, %res1
-  ret <16 x i32> %res2
+  ret <16 x i32> %res
 }
 
 declare <8 x i64> @llvm.x86.avx512.mask.pabs.q.512(<8 x i64>, <8 x i64>, i8)
 
+define <8 x i64>@test_int_x86_avx512_pabs_q_512(<8 x i64> %x0, <8 x i64> %x1, i8 %x2) {
+; CHECK-LABEL: test_int_x86_avx512_pabs_q_512:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpabsq %zmm0, %zmm0 ## encoding: [0x62,0xf2,0xfd,0x48,0x1f,0xc0]
+; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+  %res = call <8 x i64> @llvm.x86.avx512.mask.pabs.q.512(<8 x i64> %x0, <8 x i64> %x1, i8 -1)
+  ret <8 x i64> %res
+}
+
 define <8 x i64>@test_int_x86_avx512_mask_pabs_q_512(<8 x i64> %x0, <8 x i64> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_pabs_q_512:
 ; X86:       ## %bb.0:
-; X86-NEXT:    vpabsq %zmm0, %zmm2 ## encoding: [0x62,0xf2,0xfd,0x48,0x1f,0xd0]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpabsq %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x1f,0xc8]
-; X86-NEXT:    vpaddq %zmm2, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xf5,0x48,0xd4,0xc2]
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pabs_q_512:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vpabsq %zmm0, %zmm2 ## encoding: [0x62,0xf2,0xfd,0x48,0x1f,0xd0]
 ; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpabsq %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x1f,0xc8]
-; X64-NEXT:    vpaddq %zmm2, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xf5,0x48,0xd4,0xc2]
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %res = call <8 x i64> @llvm.x86.avx512.mask.pabs.q.512(<8 x i64> %x0, <8 x i64> %x1, i8 %x2)
-  %res1 = call <8 x i64> @llvm.x86.avx512.mask.pabs.q.512(<8 x i64> %x0, <8 x i64> %x1, i8 -1)
-  %res2 = add <8 x i64> %res, %res1
-  ret <8 x i64> %res2
+  ret <8 x i64> %res
 }
 
 define i8 @test_vptestmq(<8 x i64> %a0, <8 x i64> %a1, i8 %m) {
@@ -7642,26 +7742,18 @@ declare <8 x i64> @llvm.x86.avx512.maskz.pternlog.q.512(<8 x i64>, <8 x i64>, <8
 define <8 x i64>@test_int_x86_avx512_maskz_pternlog_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_maskz_pternlog_q_512:
 ; X86:       ## %bb.0:
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
-; X86-NEXT:    vpternlogq $33, %zmm2, %zmm1, %zmm3 ## encoding: [0x62,0xf3,0xf5,0x48,0x25,0xda,0x21]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogq $33, %zmm2, %zmm1, %zmm0 {%k1} {z} ## encoding: [0x62,0xf3,0xf5,0xc9,0x25,0xc2,0x21]
-; X86-NEXT:    vpaddq %zmm3, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc3]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_pternlog_q_512:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
-; X64-NEXT:    vpternlogq $33, %zmm2, %zmm1, %zmm3 ## encoding: [0x62,0xf3,0xf5,0x48,0x25,0xda,0x21]
 ; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogq $33, %zmm2, %zmm1, %zmm0 {%k1} {z} ## encoding: [0x62,0xf3,0xf5,0xc9,0x25,0xc2,0x21]
-; X64-NEXT:    vpaddq %zmm3, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc3]
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %res = call <8 x i64> @llvm.x86.avx512.maskz.pternlog.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i32 33, i8 %x4)
-  %res1 = call <8 x i64> @llvm.x86.avx512.maskz.pternlog.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i32 33, i8 -1)
-  %res2 = add <8 x i64> %res, %res1
-  ret <8 x i64> %res2
+  ret <8 x i64> %res
 }
 
 declare <16 x i32> @llvm.x86.avx512.mask.vpermi2var.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
@@ -9487,124 +9579,186 @@ declare <16 x float> @llvm.x86.avx512.mask.sqrt.ps.512(<16 x float>, <16 x float
 
 declare <16 x i32> @llvm.x86.avx512.mask.prolv.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
 
-define <16 x i32>@test_int_x86_avx512_mask_prolv_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask_prolv_d_512:
+define <16 x i32>@test_int_x86_avx512_prolv_d_512_old(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_prolv_d_512_old:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vprolvd %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf2,0x7d,0x48,0x15,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+  %res = call <16 x i32> @llvm.x86.avx512.mask.prolv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_prolv_d_512_old(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_prolv_d_512_old:
 ; X86:       ## %bb.0:
-; X86-NEXT:    vprolvd %zmm1, %zmm0, %zmm3 ## encoding: [0x62,0xf2,0x7d,0x48,0x15,0xd9]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vprolvd %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x15,0xd1]
-; X86-NEXT:    vprolvd %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x15,0xc1]
-; X86-NEXT:    vpaddd %zmm3, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc3]
-; X86-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask_prolv_d_512:
+; X64-LABEL: test_int_x86_avx512_mask_prolv_d_512_old:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vprolvd %zmm1, %zmm0, %zmm3 ## encoding: [0x62,0xf2,0x7d,0x48,0x15,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vprolvd %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x15,0xd1]
-; X64-NEXT:    vprolvd %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x15,0xc1]
-; X64-NEXT:    vpaddd %zmm3, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc3]
-; X64-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %res = call <16 x i32> @llvm.x86.avx512.mask.prolv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
-  %res1 = call <16 x i32> @llvm.x86.avx512.mask.prolv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> zeroinitializer, i16 %x3)
-  %res2 = call <16 x i32> @llvm.x86.avx512.mask.prolv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
-  %res3 = add <16 x i32> %res, %res1
-  %res4 = add <16 x i32> %res3, %res2
-  ret <16 x i32> %res4
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_maskz_prolv_d_512_old(<16 x i32> %x0, <16 x i32> %x1, i16 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_prolv_d_512_old:
+; X86:       ## %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vprolvd %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x15,0xc1]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_prolv_d_512_old:
+; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vprolvd %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x15,0xc1]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %res = call <16 x i32> @llvm.x86.avx512.mask.prolv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> zeroinitializer, i16 %x3)
+  ret <16 x i32> %res
 }
 
 declare <8 x i64> @llvm.x86.avx512.mask.prolv.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
 
-define <8 x i64>@test_int_x86_avx512_mask_prolv_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask_prolv_q_512:
+define <8 x i64>@test_int_x86_avx512_prolv_q_512_old(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_prolv_q_512_old:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vprolvq %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf2,0xfd,0x48,0x15,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+  %res = call <8 x i64> @llvm.x86.avx512.mask.prolv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_prolv_q_512_old(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_prolv_q_512_old:
 ; X86:       ## %bb.0:
-; X86-NEXT:    vprolvq %zmm1, %zmm0, %zmm3 ## encoding: [0x62,0xf2,0xfd,0x48,0x15,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprolvq %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x15,0xd1]
-; X86-NEXT:    vprolvq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x15,0xc1]
-; X86-NEXT:    vpaddq %zmm3, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc3]
-; X86-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask_prolv_q_512:
+; X64-LABEL: test_int_x86_avx512_mask_prolv_q_512_old:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vprolvq %zmm1, %zmm0, %zmm3 ## encoding: [0x62,0xf2,0xfd,0x48,0x15,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vprolvq %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x15,0xd1]
-; X64-NEXT:    vprolvq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x15,0xc1]
-; X64-NEXT:    vpaddq %zmm3, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc3]
-; X64-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %res = call <8 x i64> @llvm.x86.avx512.mask.prolv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
-  %res1 = call <8 x i64> @llvm.x86.avx512.mask.prolv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <8 x i64> @llvm.x86.avx512.mask.prolv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1)
-  %res3 = add <8 x i64> %res, %res1
-  %res4 = add <8 x i64> %res3, %res2
-  ret <8 x i64> %res4
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_maskz_prolv_q_512_old(<8 x i64> %x0, <8 x i64> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_prolv_q_512_old:
+; X86:       ## %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vprolvq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x15,0xc1]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_prolv_q_512_old:
+; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vprolvq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x15,0xc1]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %res = call <8 x i64> @llvm.x86.avx512.mask.prolv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> zeroinitializer, i8 %x3)
+  ret <8 x i64> %res
 }
 
 declare <16 x i32> @llvm.x86.avx512.mask.prorv.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
 
-define <16 x i32>@test_int_x86_avx512_mask_prorv_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask_prorv_d_512:
+define <16 x i32>@test_int_x86_avx512_prorv_d_512_old(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_prorv_d_512_old:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vprorvd %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf2,0x7d,0x48,0x14,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+  %res = call <16 x i32> @llvm.x86.avx512.mask.prorv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_mask_prorv_d_512_old(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_prorv_d_512_old:
 ; X86:       ## %bb.0:
-; X86-NEXT:    vprorvd %zmm1, %zmm0, %zmm3 ## encoding: [0x62,0xf2,0x7d,0x48,0x14,0xd9]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vprorvd %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x14,0xd1]
-; X86-NEXT:    vprorvd %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x14,0xc1]
-; X86-NEXT:    vpaddd %zmm3, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc3]
-; X86-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask_prorv_d_512:
+; X64-LABEL: test_int_x86_avx512_mask_prorv_d_512_old:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vprorvd %zmm1, %zmm0, %zmm3 ## encoding: [0x62,0xf2,0x7d,0x48,0x14,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vprorvd %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x14,0xd1]
-; X64-NEXT:    vprorvd %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x14,0xc1]
-; X64-NEXT:    vpaddd %zmm3, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc3]
-; X64-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %res = call <16 x i32> @llvm.x86.avx512.mask.prorv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
-  %res1 = call <16 x i32> @llvm.x86.avx512.mask.prorv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> zeroinitializer, i16 %x3)
-  %res2 = call <16 x i32> @llvm.x86.avx512.mask.prorv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
-  %res3 = add <16 x i32> %res, %res1
-  %res4 = add <16 x i32> %res3, %res2
-  ret <16 x i32> %res4
+  ret <16 x i32> %res
+}
+
+define <16 x i32>@test_int_x86_avx512_maskz_prorv_d_512_old(<16 x i32> %x0, <16 x i32> %x1, i16 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_prorv_d_512_old:
+; X86:       ## %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vprorvd %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x14,0xc1]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_prorv_d_512_old:
+; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vprorvd %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x14,0xc1]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %res = call <16 x i32> @llvm.x86.avx512.mask.prorv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> zeroinitializer, i16 %x3)
+  ret <16 x i32> %res
 }
 
 declare <8 x i64> @llvm.x86.avx512.mask.prorv.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
 
-define <8 x i64>@test_int_x86_avx512_mask_prorv_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask_prorv_q_512:
+define <8 x i64>@test_int_x86_avx512_prorv_q_512_old(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_prorv_q_512_old:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vprorvq %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf2,0xfd,0x48,0x14,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+  %res = call <8 x i64> @llvm.x86.avx512.mask.prorv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_mask_prorv_q_512_old(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_prorv_q_512_old:
 ; X86:       ## %bb.0:
-; X86-NEXT:    vprorvq %zmm1, %zmm0, %zmm3 ## encoding: [0x62,0xf2,0xfd,0x48,0x14,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprorvq %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x14,0xd1]
-; X86-NEXT:    vprorvq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x14,0xc1]
-; X86-NEXT:    vpaddq %zmm3, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc3]
-; X86-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask_prorv_q_512:
+; X64-LABEL: test_int_x86_avx512_mask_prorv_q_512_old:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vprorvq %zmm1, %zmm0, %zmm3 ## encoding: [0x62,0xf2,0xfd,0x48,0x14,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vprorvq %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x14,0xd1]
-; X64-NEXT:    vprorvq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x14,0xc1]
-; X64-NEXT:    vpaddq %zmm3, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc3]
-; X64-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %res = call <8 x i64> @llvm.x86.avx512.mask.prorv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
-  %res1 = call <8 x i64> @llvm.x86.avx512.mask.prorv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <8 x i64> @llvm.x86.avx512.mask.prorv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1)
-  %res3 = add <8 x i64> %res, %res1
-  %res4 = add <8 x i64> %res3, %res2
-  ret <8 x i64> %res4
+  ret <8 x i64> %res
+}
+
+define <8 x i64>@test_int_x86_avx512_maskz_prorv_q_512_old(<8 x i64> %x0, <8 x i64> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_prorv_q_512_old:
+; X86:       ## %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vprorvq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x14,0xc1]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_prorv_q_512_old:
+; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vprorvq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x14,0xc1]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %res = call <8 x i64> @llvm.x86.avx512.mask.prorv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> zeroinitializer, i8 %x3)
+  ret <8 x i64> %res
 }
 
 declare <16 x i32> @llvm.x86.avx512.mask.prol.d.512(<16 x i32>, i32, <16 x i32>, i16)
@@ -9612,26 +9766,26 @@ declare <16 x i32> @llvm.x86.avx512.mask.prol.d.512(<16 x i32>, i32, <16 x i32>,
 define <16 x i32>@test_int_x86_avx512_mask_prol_d_512(<16 x i32> %x0, i32 %x1, <16 x i32> %x2, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_prol_d_512:
 ; X86:       ## %bb.0:
-; X86-NEXT:    vprold $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0x6d,0x48,0x72,0xc8,0x03]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vprold $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x75,0x49,0x72,0xc8,0x03]
-; X86-NEXT:    vprold $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0x72,0xc8,0x03]
-; X86-NEXT:    vpaddd %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc2]
+; X86-NEXT:    vprold $4, %zmm0, %zmm2 {%k1} {z} ## encoding: [0x62,0xf1,0x6d,0xc9,0x72,0xc8,0x04]
+; X86-NEXT:    vprold $5, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0x72,0xc8,0x05]
+; X86-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0x75,0x48,0xfe,0xc0]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_prol_d_512:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vprold $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0x6d,0x48,0x72,0xc8,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 ## encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vprold $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x75,0x49,0x72,0xc8,0x03]
-; X64-NEXT:    vprold $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0x72,0xc8,0x03]
-; X64-NEXT:    vpaddd %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc2]
+; X64-NEXT:    vprold $4, %zmm0, %zmm2 {%k1} {z} ## encoding: [0x62,0xf1,0x6d,0xc9,0x72,0xc8,0x04]
+; X64-NEXT:    vprold $5, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0x72,0xc8,0x05]
+; X64-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0x75,0x48,0xfe,0xc0]
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %res = call <16 x i32> @llvm.x86.avx512.mask.prol.d.512(<16 x i32> %x0, i32 3, <16 x i32> %x2, i16 %x3)
-  %res1 = call <16 x i32> @llvm.x86.avx512.mask.prol.d.512(<16 x i32> %x0, i32 3, <16 x i32> zeroinitializer, i16 %x3)
-  %res2 = call <16 x i32> @llvm.x86.avx512.mask.prol.d.512(<16 x i32> %x0, i32 3, <16 x i32> %x2, i16 -1)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.prol.d.512(<16 x i32> %x0, i32 4, <16 x i32> zeroinitializer, i16 %x3)
+  %res2 = call <16 x i32> @llvm.x86.avx512.mask.prol.d.512(<16 x i32> %x0, i32 5, <16 x i32> %x2, i16 -1)
   %res3 = add <16 x i32> %res, %res1
   %res4 = add <16 x i32> %res3, %res2
   ret <16 x i32> %res4
@@ -9642,27 +9796,27 @@ declare <8 x i64> @llvm.x86.avx512.mask.prol.q.512(<8 x i64>, i32, <8 x i64>, i8
 define <8 x i64>@test_int_x86_avx512_mask_prol_q_512(<8 x i64> %x0, i32 %x1, <8 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_prol_q_512:
 ; X86:       ## %bb.0:
-; X86-NEXT:    vprolq $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0xed,0x48,0x72,0xc8,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprolq $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xf5,0x49,0x72,0xc8,0x03]
-; X86-NEXT:    vprolq $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0x72,0xc8,0x03]
-; X86-NEXT:    vpaddq %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc2]
+; X86-NEXT:    vprolq $4, %zmm0, %zmm2 {%k1} {z} ## encoding: [0x62,0xf1,0xed,0xc9,0x72,0xc8,0x04]
+; X86-NEXT:    vprolq $5, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x72,0xc8,0x05]
+; X86-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xf5,0x48,0xd4,0xc0]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_prol_q_512:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vprolq $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0xed,0x48,0x72,0xc8,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 ## encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vprolq $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xf5,0x49,0x72,0xc8,0x03]
-; X64-NEXT:    vprolq $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0x72,0xc8,0x03]
-; X64-NEXT:    vpaddq %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc2]
+; X64-NEXT:    vprolq $4, %zmm0, %zmm2 {%k1} {z} ## encoding: [0x62,0xf1,0xed,0xc9,0x72,0xc8,0x04]
+; X64-NEXT:    vprolq $5, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x72,0xc8,0x05]
+; X64-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xf5,0x48,0xd4,0xc0]
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %res = call <8 x i64> @llvm.x86.avx512.mask.prol.q.512(<8 x i64> %x0, i32 3, <8 x i64> %x2, i8 %x3)
-  %res1 = call <8 x i64> @llvm.x86.avx512.mask.prol.q.512(<8 x i64> %x0, i32 3, <8 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <8 x i64> @llvm.x86.avx512.mask.prol.q.512(<8 x i64> %x0, i32 3, <8 x i64> %x2, i8 -1)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.prol.q.512(<8 x i64> %x0, i32 4, <8 x i64> zeroinitializer, i8 %x3)
+  %res2 = call <8 x i64> @llvm.x86.avx512.mask.prol.q.512(<8 x i64> %x0, i32 5, <8 x i64> %x2, i8 -1)
   %res3 = add <8 x i64> %res, %res1
   %res4 = add <8 x i64> %res3, %res2
   ret <8 x i64> %res4
@@ -9673,26 +9827,26 @@ declare <16 x i32> @llvm.x86.avx512.mask.pror.d.512(<16 x i32>, i32, <16 x i32>,
 define <16 x i32>@test_int_x86_avx512_mask_pror_d_512(<16 x i32> %x0, i32 %x1, <16 x i32> %x2, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_pror_d_512:
 ; X86:       ## %bb.0:
-; X86-NEXT:    vprord $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0x6d,0x48,0x72,0xc0,0x03]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vprord $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x75,0x49,0x72,0xc0,0x03]
-; X86-NEXT:    vprord $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0x72,0xc0,0x03]
-; X86-NEXT:    vpaddd %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc2]
+; X86-NEXT:    vprord $4, %zmm0, %zmm2 {%k1} {z} ## encoding: [0x62,0xf1,0x6d,0xc9,0x72,0xc0,0x04]
+; X86-NEXT:    vprord $5, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0x72,0xc0,0x05]
+; X86-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0x75,0x48,0xfe,0xc0]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pror_d_512:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vprord $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0x6d,0x48,0x72,0xc0,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 ## encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vprord $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x75,0x49,0x72,0xc0,0x03]
-; X64-NEXT:    vprord $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0x72,0xc0,0x03]
-; X64-NEXT:    vpaddd %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc2]
+; X64-NEXT:    vprord $4, %zmm0, %zmm2 {%k1} {z} ## encoding: [0x62,0xf1,0x6d,0xc9,0x72,0xc0,0x04]
+; X64-NEXT:    vprord $5, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0x72,0xc0,0x05]
+; X64-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0x75,0x48,0xfe,0xc0]
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %res = call <16 x i32> @llvm.x86.avx512.mask.pror.d.512(<16 x i32> %x0, i32 3, <16 x i32> %x2, i16 %x3)
-  %res1 = call <16 x i32> @llvm.x86.avx512.mask.pror.d.512(<16 x i32> %x0, i32 3, <16 x i32> zeroinitializer, i16 %x3)
-  %res2 = call <16 x i32> @llvm.x86.avx512.mask.pror.d.512(<16 x i32> %x0, i32 3, <16 x i32> %x2, i16 -1)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.pror.d.512(<16 x i32> %x0, i32 4, <16 x i32> zeroinitializer, i16 %x3)
+  %res2 = call <16 x i32> @llvm.x86.avx512.mask.pror.d.512(<16 x i32> %x0, i32 5, <16 x i32> %x2, i16 -1)
   %res3 = add <16 x i32> %res, %res1
   %res4 = add <16 x i32> %res3, %res2
   ret <16 x i32> %res4
@@ -9703,27 +9857,27 @@ declare <8 x i64> @llvm.x86.avx512.mask.pror.q.512(<8 x i64>, i32, <8 x i64>, i8
 define <8 x i64>@test_int_x86_avx512_mask_pror_q_512(<8 x i64> %x0, i32 %x1, <8 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_pror_q_512:
 ; X86:       ## %bb.0:
-; X86-NEXT:    vprorq $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0xed,0x48,0x72,0xc0,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprorq $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xf5,0x49,0x72,0xc0,0x03]
-; X86-NEXT:    vprorq $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0x72,0xc0,0x03]
-; X86-NEXT:    vpaddq %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc2]
+; X86-NEXT:    vprorq $4, %zmm0, %zmm2 {%k1} {z} ## encoding: [0x62,0xf1,0xed,0xc9,0x72,0xc0,0x04]
+; X86-NEXT:    vprorq $5, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x72,0xc0,0x05]
+; X86-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xf5,0x48,0xd4,0xc0]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pror_q_512:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vprorq $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0xed,0x48,0x72,0xc0,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 ## encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vprorq $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xf5,0x49,0x72,0xc0,0x03]
-; X64-NEXT:    vprorq $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0x72,0xc0,0x03]
-; X64-NEXT:    vpaddq %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc2]
+; X64-NEXT:    vprorq $4, %zmm0, %zmm2 {%k1} {z} ## encoding: [0x62,0xf1,0xed,0xc9,0x72,0xc0,0x04]
+; X64-NEXT:    vprorq $5, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x72,0xc0,0x05]
+; X64-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xf5,0x48,0xd4,0xc0]
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %res = call <8 x i64> @llvm.x86.avx512.mask.pror.q.512(<8 x i64> %x0, i32 3, <8 x i64> %x2, i8 %x3)
-  %res1 = call <8 x i64> @llvm.x86.avx512.mask.pror.q.512(<8 x i64> %x0, i32 3, <8 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <8 x i64> @llvm.x86.avx512.mask.pror.q.512(<8 x i64> %x0, i32 3, <8 x i64> %x2, i8 -1)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.pror.q.512(<8 x i64> %x0, i32 4, <8 x i64> zeroinitializer, i8 %x3)
+  %res2 = call <8 x i64> @llvm.x86.avx512.mask.pror.q.512(<8 x i64> %x0, i32 5, <8 x i64> %x2, i8 -1)
   %res3 = add <8 x i64> %res, %res1
   %res4 = add <8 x i64> %res3, %res2
   ret <8 x i64> %res4
@@ -9735,45 +9889,35 @@ define <2 x double>@test_int_x86_avx512_mask_vfmadd_sd(<2 x double> %x0, <2 x do
 ; X86-LABEL: test_int_x86_avx512_mask_vfmadd_sd:
 ; X86:       ## %bb.0:
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al ## encoding: [0x8a,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vmovapd %xmm0, %xmm3 ## encoding: [0xc5,0xf9,0x28,0xd8]
-; X86-NEXT:    vfmadd213sd %xmm2, %xmm1, %xmm3 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf1,0xa9,0xda]
+; X86-NEXT:    vfmadd213sd %xmm2, %xmm1, %xmm3 {%k1} ## encoding: [0x62,0xf2,0xf5,0x09,0xa9,0xda]
 ; X86-NEXT:    ## xmm3 = (xmm1 * xmm3) + xmm2
-; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovapd %xmm0, %xmm4 ## encoding: [0xc5,0xf9,0x28,0xe0]
-; X86-NEXT:    vfmadd213sd %xmm2, %xmm1, %xmm4 {%k1} ## encoding: [0x62,0xf2,0xf5,0x09,0xa9,0xe2]
-; X86-NEXT:    ## xmm4 = (xmm1 * xmm4) + xmm2
-; X86-NEXT:    vaddpd %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe1,0x58,0xdc]
 ; X86-NEXT:    vmovapd %xmm0, %xmm4 ## encoding: [0xc5,0xf9,0x28,0xe0]
 ; X86-NEXT:    vfmadd213sd {rz-sae}, %xmm2, %xmm1, %xmm4 ## encoding: [0x62,0xf2,0xf5,0x78,0xa9,0xe2]
-; X86-NEXT:    vfmadd213sd {rz-sae}, %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x79,0xa9,0xc2]
-; X86-NEXT:    vaddpd %xmm0, %xmm4, %xmm0 ## encoding: [0xc5,0xd9,0x58,0xc0]
-; X86-NEXT:    vaddpd %xmm0, %xmm3, %xmm0 ## encoding: [0xc5,0xe1,0x58,0xc0]
+; X86-NEXT:    vaddpd %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe1,0x58,0xdc]
+; X86-NEXT:    vfmadd213sd {ru-sae}, %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x59,0xa9,0xc2]
+; X86-NEXT:    vaddpd %xmm3, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0x58,0xc3]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vfmadd_sd:
 ; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vmovapd %xmm0, %xmm3 ## encoding: [0xc5,0xf9,0x28,0xd8]
-; X64-NEXT:    vfmadd213sd %xmm2, %xmm1, %xmm3 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf1,0xa9,0xda]
+; X64-NEXT:    vfmadd213sd %xmm2, %xmm1, %xmm3 {%k1} ## encoding: [0x62,0xf2,0xf5,0x09,0xa9,0xda]
 ; X64-NEXT:    ## xmm3 = (xmm1 * xmm3) + xmm2
-; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovapd %xmm0, %xmm4 ## encoding: [0xc5,0xf9,0x28,0xe0]
-; X64-NEXT:    vfmadd213sd %xmm2, %xmm1, %xmm4 {%k1} ## encoding: [0x62,0xf2,0xf5,0x09,0xa9,0xe2]
-; X64-NEXT:    ## xmm4 = (xmm1 * xmm4) + xmm2
-; X64-NEXT:    vaddpd %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe1,0x58,0xdc]
 ; X64-NEXT:    vmovapd %xmm0, %xmm4 ## encoding: [0xc5,0xf9,0x28,0xe0]
 ; X64-NEXT:    vfmadd213sd {rz-sae}, %xmm2, %xmm1, %xmm4 ## encoding: [0x62,0xf2,0xf5,0x78,0xa9,0xe2]
-; X64-NEXT:    vfmadd213sd {rz-sae}, %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x79,0xa9,0xc2]
-; X64-NEXT:    vaddpd %xmm0, %xmm4, %xmm0 ## encoding: [0xc5,0xd9,0x58,0xc0]
-; X64-NEXT:    vaddpd %xmm0, %xmm3, %xmm0 ## encoding: [0xc5,0xe1,0x58,0xc0]
+; X64-NEXT:    vaddpd %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe1,0x58,0xdc]
+; X64-NEXT:    vfmadd213sd {ru-sae}, %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x59,0xa9,0xc2]
+; X64-NEXT:    vaddpd %xmm3, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0x58,0xc3]
 ; X64-NEXT:    retq ## encoding: [0xc3]
-  %res = call <2 x double> @llvm.x86.avx512.mask.vfmadd.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1, i32 4)
-  %res1 = call <2 x double> @llvm.x86.avx512.mask.vfmadd.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 4)
-  %res2 = call <2 x double> @llvm.x86.avx512.mask.vfmadd.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1, i32 11)
-  %res3 = call <2 x double> @llvm.x86.avx512.mask.vfmadd.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 11)
-  %res4 = fadd <2 x double> %res, %res1
-  %res5 = fadd <2 x double> %res2, %res3
-  %res6 = fadd <2 x double> %res4, %res5
-  ret <2 x double> %res6
+  %res = call <2 x double> @llvm.x86.avx512.mask.vfmadd.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 4)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.vfmadd.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1, i32 11)
+  %res2 = call <2 x double> @llvm.x86.avx512.mask.vfmadd.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 10)
+  %res3 = fadd <2 x double> %res, %res1
+  %res4 = fadd <2 x double> %res2, %res3
+  ret <2 x double> %res4
 }
 
 declare <4 x float> @llvm.x86.avx512.mask.vfmadd.ss(<4 x float>, <4 x float>, <4 x float>, i8, i32)
@@ -9782,45 +9926,35 @@ define <4 x float>@test_int_x86_avx512_mask_vfmadd_ss(<4 x float> %x0, <4 x floa
 ; X86-LABEL: test_int_x86_avx512_mask_vfmadd_ss:
 ; X86:       ## %bb.0:
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al ## encoding: [0x8a,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vmovaps %xmm0, %xmm3 ## encoding: [0xc5,0xf8,0x28,0xd8]
-; X86-NEXT:    vfmadd213ss %xmm2, %xmm1, %xmm3 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0xa9,0xda]
+; X86-NEXT:    vfmadd213ss %xmm2, %xmm1, %xmm3 {%k1} ## encoding: [0x62,0xf2,0x75,0x09,0xa9,0xda]
 ; X86-NEXT:    ## xmm3 = (xmm1 * xmm3) + xmm2
-; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovaps %xmm0, %xmm4 ## encoding: [0xc5,0xf8,0x28,0xe0]
-; X86-NEXT:    vfmadd213ss %xmm2, %xmm1, %xmm4 {%k1} ## encoding: [0x62,0xf2,0x75,0x09,0xa9,0xe2]
-; X86-NEXT:    ## xmm4 = (xmm1 * xmm4) + xmm2
-; X86-NEXT:    vaddps %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe0,0x58,0xdc]
 ; X86-NEXT:    vmovaps %xmm0, %xmm4 ## encoding: [0xc5,0xf8,0x28,0xe0]
 ; X86-NEXT:    vfmadd213ss {rz-sae}, %xmm2, %xmm1, %xmm4 ## encoding: [0x62,0xf2,0x75,0x78,0xa9,0xe2]
-; X86-NEXT:    vfmadd213ss {rz-sae}, %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x79,0xa9,0xc2]
-; X86-NEXT:    vaddps %xmm0, %xmm4, %xmm0 ## encoding: [0xc5,0xd8,0x58,0xc0]
-; X86-NEXT:    vaddps %xmm0, %xmm3, %xmm0 ## encoding: [0xc5,0xe0,0x58,0xc0]
+; X86-NEXT:    vaddps %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe0,0x58,0xdc]
+; X86-NEXT:    vfmadd213ss {ru-sae}, %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x59,0xa9,0xc2]
+; X86-NEXT:    vaddps %xmm3, %xmm0, %xmm0 ## encoding: [0xc5,0xf8,0x58,0xc3]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vfmadd_ss:
 ; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vmovaps %xmm0, %xmm3 ## encoding: [0xc5,0xf8,0x28,0xd8]
-; X64-NEXT:    vfmadd213ss %xmm2, %xmm1, %xmm3 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0xa9,0xda]
+; X64-NEXT:    vfmadd213ss %xmm2, %xmm1, %xmm3 {%k1} ## encoding: [0x62,0xf2,0x75,0x09,0xa9,0xda]
 ; X64-NEXT:    ## xmm3 = (xmm1 * xmm3) + xmm2
-; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovaps %xmm0, %xmm4 ## encoding: [0xc5,0xf8,0x28,0xe0]
-; X64-NEXT:    vfmadd213ss %xmm2, %xmm1, %xmm4 {%k1} ## encoding: [0x62,0xf2,0x75,0x09,0xa9,0xe2]
-; X64-NEXT:    ## xmm4 = (xmm1 * xmm4) + xmm2
-; X64-NEXT:    vaddps %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe0,0x58,0xdc]
 ; X64-NEXT:    vmovaps %xmm0, %xmm4 ## encoding: [0xc5,0xf8,0x28,0xe0]
 ; X64-NEXT:    vfmadd213ss {rz-sae}, %xmm2, %xmm1, %xmm4 ## encoding: [0x62,0xf2,0x75,0x78,0xa9,0xe2]
-; X64-NEXT:    vfmadd213ss {rz-sae}, %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x79,0xa9,0xc2]
-; X64-NEXT:    vaddps %xmm0, %xmm4, %xmm0 ## encoding: [0xc5,0xd8,0x58,0xc0]
-; X64-NEXT:    vaddps %xmm0, %xmm3, %xmm0 ## encoding: [0xc5,0xe0,0x58,0xc0]
+; X64-NEXT:    vaddps %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe0,0x58,0xdc]
+; X64-NEXT:    vfmadd213ss {ru-sae}, %xmm2, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x59,0xa9,0xc2]
+; X64-NEXT:    vaddps %xmm3, %xmm0, %xmm0 ## encoding: [0xc5,0xf8,0x58,0xc3]
 ; X64-NEXT:    retq ## encoding: [0xc3]
-  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1, i32 4)
-  %res1 = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 4)
-  %res2 = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1, i32 11)
-  %res3 = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 11)
-  %res4 = fadd <4 x float> %res, %res1
-  %res5 = fadd <4 x float> %res2, %res3
-  %res6 = fadd <4 x float> %res4, %res5
-  ret <4 x float> %res6
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 4)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1, i32 11)
+  %res2 = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 10)
+  %res3 = fadd <4 x float> %res, %res1
+  %res4 = fadd <4 x float> %res2, %res3
+  ret <4 x float> %res4
 }
 
 declare <2 x double> @llvm.x86.avx512.maskz.vfmadd.sd(<2 x double>, <2 x double>, <2 x double>, i8, i32)
@@ -9880,45 +10014,35 @@ define <2 x double>@test_int_x86_avx512_mask3_vfmadd_sd(<2 x double> %x0, <2 x d
 ; X86-LABEL: test_int_x86_avx512_mask3_vfmadd_sd:
 ; X86:       ## %bb.0:
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al ## encoding: [0x8a,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vmovapd %xmm2, %xmm3 ## encoding: [0xc5,0xf9,0x28,0xda]
-; X86-NEXT:    vfmadd231sd %xmm1, %xmm0, %xmm3 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf9,0xb9,0xd9]
+; X86-NEXT:    vfmadd231sd %xmm1, %xmm0, %xmm3 {%k1} ## encoding: [0x62,0xf2,0xfd,0x09,0xb9,0xd9]
 ; X86-NEXT:    ## xmm3 = (xmm0 * xmm1) + xmm3
-; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovapd %xmm2, %xmm4 ## encoding: [0xc5,0xf9,0x28,0xe2]
-; X86-NEXT:    vfmadd231sd %xmm1, %xmm0, %xmm4 {%k1} ## encoding: [0x62,0xf2,0xfd,0x09,0xb9,0xe1]
-; X86-NEXT:    ## xmm4 = (xmm0 * xmm1) + xmm4
-; X86-NEXT:    vaddpd %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe1,0x58,0xdc]
 ; X86-NEXT:    vmovapd %xmm2, %xmm4 ## encoding: [0xc5,0xf9,0x28,0xe2]
 ; X86-NEXT:    vfmadd231sd {rz-sae}, %xmm1, %xmm0, %xmm4 ## encoding: [0x62,0xf2,0xfd,0x78,0xb9,0xe1]
-; X86-NEXT:    vfmadd231sd {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf2,0xfd,0x79,0xb9,0xd1]
-; X86-NEXT:    vaddpd %xmm2, %xmm4, %xmm0 ## encoding: [0xc5,0xd9,0x58,0xc2]
-; X86-NEXT:    vaddpd %xmm0, %xmm3, %xmm0 ## encoding: [0xc5,0xe1,0x58,0xc0]
+; X86-NEXT:    vaddpd %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe1,0x58,0xdc]
+; X86-NEXT:    vfmadd231sd {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf2,0xfd,0x59,0xb9,0xd1]
+; X86-NEXT:    vaddpd %xmm3, %xmm2, %xmm0 ## encoding: [0xc5,0xe9,0x58,0xc3]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask3_vfmadd_sd:
 ; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vmovapd %xmm2, %xmm3 ## encoding: [0xc5,0xf9,0x28,0xda]
-; X64-NEXT:    vfmadd231sd %xmm1, %xmm0, %xmm3 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf9,0xb9,0xd9]
+; X64-NEXT:    vfmadd231sd %xmm1, %xmm0, %xmm3 {%k1} ## encoding: [0x62,0xf2,0xfd,0x09,0xb9,0xd9]
 ; X64-NEXT:    ## xmm3 = (xmm0 * xmm1) + xmm3
-; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovapd %xmm2, %xmm4 ## encoding: [0xc5,0xf9,0x28,0xe2]
-; X64-NEXT:    vfmadd231sd %xmm1, %xmm0, %xmm4 {%k1} ## encoding: [0x62,0xf2,0xfd,0x09,0xb9,0xe1]
-; X64-NEXT:    ## xmm4 = (xmm0 * xmm1) + xmm4
-; X64-NEXT:    vaddpd %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe1,0x58,0xdc]
 ; X64-NEXT:    vmovapd %xmm2, %xmm4 ## encoding: [0xc5,0xf9,0x28,0xe2]
 ; X64-NEXT:    vfmadd231sd {rz-sae}, %xmm1, %xmm0, %xmm4 ## encoding: [0x62,0xf2,0xfd,0x78,0xb9,0xe1]
-; X64-NEXT:    vfmadd231sd {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf2,0xfd,0x79,0xb9,0xd1]
-; X64-NEXT:    vaddpd %xmm2, %xmm4, %xmm0 ## encoding: [0xc5,0xd9,0x58,0xc2]
-; X64-NEXT:    vaddpd %xmm0, %xmm3, %xmm0 ## encoding: [0xc5,0xe1,0x58,0xc0]
+; X64-NEXT:    vaddpd %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe1,0x58,0xdc]
+; X64-NEXT:    vfmadd231sd {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf2,0xfd,0x59,0xb9,0xd1]
+; X64-NEXT:    vaddpd %xmm3, %xmm2, %xmm0 ## encoding: [0xc5,0xe9,0x58,0xc3]
 ; X64-NEXT:    retq ## encoding: [0xc3]
-  %res = call <2 x double> @llvm.x86.avx512.mask3.vfmadd.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1, i32 4)
-  %res1 = call <2 x double> @llvm.x86.avx512.mask3.vfmadd.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 4)
-  %res2 = call <2 x double> @llvm.x86.avx512.mask3.vfmadd.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1, i32 11)
-  %res3 = call <2 x double> @llvm.x86.avx512.mask3.vfmadd.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 11)
-  %res4 = fadd <2 x double> %res, %res1
-  %res5 = fadd <2 x double> %res2, %res3
-  %res6 = fadd <2 x double> %res4, %res5
-  ret <2 x double> %res6
+  %res = call <2 x double> @llvm.x86.avx512.mask3.vfmadd.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 4)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask3.vfmadd.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1, i32 11)
+  %res2 = call <2 x double> @llvm.x86.avx512.mask3.vfmadd.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 10)
+  %res3 = fadd <2 x double> %res, %res1
+  %res4 = fadd <2 x double> %res2, %res3
+  ret <2 x double> %res4
 }
 
 declare <4 x float> @llvm.x86.avx512.mask3.vfmadd.ss(<4 x float>, <4 x float>, <4 x float>, i8, i32)
@@ -9927,45 +10051,35 @@ define <4 x float>@test_int_x86_avx512_mask3_vfmadd_ss(<4 x float> %x0, <4 x flo
 ; X86-LABEL: test_int_x86_avx512_mask3_vfmadd_ss:
 ; X86:       ## %bb.0:
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al ## encoding: [0x8a,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vmovaps %xmm2, %xmm3 ## encoding: [0xc5,0xf8,0x28,0xda]
-; X86-NEXT:    vfmadd231ss %xmm1, %xmm0, %xmm3 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0xb9,0xd9]
+; X86-NEXT:    vfmadd231ss %xmm1, %xmm0, %xmm3 {%k1} ## encoding: [0x62,0xf2,0x7d,0x09,0xb9,0xd9]
 ; X86-NEXT:    ## xmm3 = (xmm0 * xmm1) + xmm3
-; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovaps %xmm2, %xmm4 ## encoding: [0xc5,0xf8,0x28,0xe2]
-; X86-NEXT:    vfmadd231ss %xmm1, %xmm0, %xmm4 {%k1} ## encoding: [0x62,0xf2,0x7d,0x09,0xb9,0xe1]
-; X86-NEXT:    ## xmm4 = (xmm0 * xmm1) + xmm4
-; X86-NEXT:    vaddps %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe0,0x58,0xdc]
 ; X86-NEXT:    vmovaps %xmm2, %xmm4 ## encoding: [0xc5,0xf8,0x28,0xe2]
 ; X86-NEXT:    vfmadd231ss {rz-sae}, %xmm1, %xmm0, %xmm4 ## encoding: [0x62,0xf2,0x7d,0x78,0xb9,0xe1]
-; X86-NEXT:    vfmadd231ss {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf2,0x7d,0x79,0xb9,0xd1]
-; X86-NEXT:    vaddps %xmm2, %xmm4, %xmm0 ## encoding: [0xc5,0xd8,0x58,0xc2]
-; X86-NEXT:    vaddps %xmm0, %xmm3, %xmm0 ## encoding: [0xc5,0xe0,0x58,0xc0]
+; X86-NEXT:    vaddps %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe0,0x58,0xdc]
+; X86-NEXT:    vfmadd231ss {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf2,0x7d,0x59,0xb9,0xd1]
+; X86-NEXT:    vaddps %xmm3, %xmm2, %xmm0 ## encoding: [0xc5,0xe8,0x58,0xc3]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask3_vfmadd_ss:
 ; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vmovaps %xmm2, %xmm3 ## encoding: [0xc5,0xf8,0x28,0xda]
-; X64-NEXT:    vfmadd231ss %xmm1, %xmm0, %xmm3 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0xb9,0xd9]
+; X64-NEXT:    vfmadd231ss %xmm1, %xmm0, %xmm3 {%k1} ## encoding: [0x62,0xf2,0x7d,0x09,0xb9,0xd9]
 ; X64-NEXT:    ## xmm3 = (xmm0 * xmm1) + xmm3
-; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovaps %xmm2, %xmm4 ## encoding: [0xc5,0xf8,0x28,0xe2]
-; X64-NEXT:    vfmadd231ss %xmm1, %xmm0, %xmm4 {%k1} ## encoding: [0x62,0xf2,0x7d,0x09,0xb9,0xe1]
-; X64-NEXT:    ## xmm4 = (xmm0 * xmm1) + xmm4
-; X64-NEXT:    vaddps %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe0,0x58,0xdc]
 ; X64-NEXT:    vmovaps %xmm2, %xmm4 ## encoding: [0xc5,0xf8,0x28,0xe2]
 ; X64-NEXT:    vfmadd231ss {rz-sae}, %xmm1, %xmm0, %xmm4 ## encoding: [0x62,0xf2,0x7d,0x78,0xb9,0xe1]
-; X64-NEXT:    vfmadd231ss {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf2,0x7d,0x79,0xb9,0xd1]
-; X64-NEXT:    vaddps %xmm2, %xmm4, %xmm0 ## encoding: [0xc5,0xd8,0x58,0xc2]
-; X64-NEXT:    vaddps %xmm0, %xmm3, %xmm0 ## encoding: [0xc5,0xe0,0x58,0xc0]
+; X64-NEXT:    vaddps %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe0,0x58,0xdc]
+; X64-NEXT:    vfmadd231ss {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf2,0x7d,0x59,0xb9,0xd1]
+; X64-NEXT:    vaddps %xmm3, %xmm2, %xmm0 ## encoding: [0xc5,0xe8,0x58,0xc3]
 ; X64-NEXT:    retq ## encoding: [0xc3]
-  %res = call <4 x float> @llvm.x86.avx512.mask3.vfmadd.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1, i32 4)
-  %res1 = call <4 x float> @llvm.x86.avx512.mask3.vfmadd.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 4)
-  %res2 = call <4 x float> @llvm.x86.avx512.mask3.vfmadd.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1, i32 11)
-  %res3 = call <4 x float> @llvm.x86.avx512.mask3.vfmadd.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 11)
-  %res4 = fadd <4 x float> %res, %res1
-  %res5 = fadd <4 x float> %res2, %res3
-  %res6 = fadd <4 x float> %res4, %res5
-  ret <4 x float> %res6
+  %res = call <4 x float> @llvm.x86.avx512.mask3.vfmadd.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 4)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask3.vfmadd.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1, i32 11)
+  %res2 = call <4 x float> @llvm.x86.avx512.mask3.vfmadd.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 10)
+  %res3 = fadd <4 x float> %res, %res1
+  %res4 = fadd <4 x float> %res2, %res3
+  ret <4 x float> %res4
 }
 
 define void @fmadd_ss_mask_memfold(float* %a, float* %b, i8 %c) {
@@ -10150,45 +10264,35 @@ define <2 x double>@test_int_x86_avx512_mask3_vfmsub_sd(<2 x double> %x0, <2 x d
 ; X86-LABEL: test_int_x86_avx512_mask3_vfmsub_sd:
 ; X86:       ## %bb.0:
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al ## encoding: [0x8a,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vmovapd %xmm2, %xmm3 ## encoding: [0xc5,0xf9,0x28,0xda]
-; X86-NEXT:    vfmsub231sd %xmm1, %xmm0, %xmm3 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf9,0xbb,0xd9]
+; X86-NEXT:    vfmsub231sd %xmm1, %xmm0, %xmm3 {%k1} ## encoding: [0x62,0xf2,0xfd,0x09,0xbb,0xd9]
 ; X86-NEXT:    ## xmm3 = (xmm0 * xmm1) - xmm3
-; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovapd %xmm2, %xmm4 ## encoding: [0xc5,0xf9,0x28,0xe2]
-; X86-NEXT:    vfmsub231sd %xmm1, %xmm0, %xmm4 {%k1} ## encoding: [0x62,0xf2,0xfd,0x09,0xbb,0xe1]
-; X86-NEXT:    ## xmm4 = (xmm0 * xmm1) - xmm4
-; X86-NEXT:    vaddpd %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe1,0x58,0xdc]
 ; X86-NEXT:    vmovapd %xmm2, %xmm4 ## encoding: [0xc5,0xf9,0x28,0xe2]
 ; X86-NEXT:    vfmsub231sd {rz-sae}, %xmm1, %xmm0, %xmm4 ## encoding: [0x62,0xf2,0xfd,0x78,0xbb,0xe1]
-; X86-NEXT:    vfmsub231sd {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf2,0xfd,0x79,0xbb,0xd1]
-; X86-NEXT:    vaddpd %xmm2, %xmm4, %xmm0 ## encoding: [0xc5,0xd9,0x58,0xc2]
-; X86-NEXT:    vaddpd %xmm0, %xmm3, %xmm0 ## encoding: [0xc5,0xe1,0x58,0xc0]
+; X86-NEXT:    vaddpd %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe1,0x58,0xdc]
+; X86-NEXT:    vfmsub231sd {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf2,0xfd,0x59,0xbb,0xd1]
+; X86-NEXT:    vaddpd %xmm3, %xmm2, %xmm0 ## encoding: [0xc5,0xe9,0x58,0xc3]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask3_vfmsub_sd:
 ; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vmovapd %xmm2, %xmm3 ## encoding: [0xc5,0xf9,0x28,0xda]
-; X64-NEXT:    vfmsub231sd %xmm1, %xmm0, %xmm3 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf9,0xbb,0xd9]
+; X64-NEXT:    vfmsub231sd %xmm1, %xmm0, %xmm3 {%k1} ## encoding: [0x62,0xf2,0xfd,0x09,0xbb,0xd9]
 ; X64-NEXT:    ## xmm3 = (xmm0 * xmm1) - xmm3
-; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovapd %xmm2, %xmm4 ## encoding: [0xc5,0xf9,0x28,0xe2]
-; X64-NEXT:    vfmsub231sd %xmm1, %xmm0, %xmm4 {%k1} ## encoding: [0x62,0xf2,0xfd,0x09,0xbb,0xe1]
-; X64-NEXT:    ## xmm4 = (xmm0 * xmm1) - xmm4
-; X64-NEXT:    vaddpd %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe1,0x58,0xdc]
 ; X64-NEXT:    vmovapd %xmm2, %xmm4 ## encoding: [0xc5,0xf9,0x28,0xe2]
 ; X64-NEXT:    vfmsub231sd {rz-sae}, %xmm1, %xmm0, %xmm4 ## encoding: [0x62,0xf2,0xfd,0x78,0xbb,0xe1]
-; X64-NEXT:    vfmsub231sd {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf2,0xfd,0x79,0xbb,0xd1]
-; X64-NEXT:    vaddpd %xmm2, %xmm4, %xmm0 ## encoding: [0xc5,0xd9,0x58,0xc2]
-; X64-NEXT:    vaddpd %xmm0, %xmm3, %xmm0 ## encoding: [0xc5,0xe1,0x58,0xc0]
+; X64-NEXT:    vaddpd %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe1,0x58,0xdc]
+; X64-NEXT:    vfmsub231sd {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf2,0xfd,0x59,0xbb,0xd1]
+; X64-NEXT:    vaddpd %xmm3, %xmm2, %xmm0 ## encoding: [0xc5,0xe9,0x58,0xc3]
 ; X64-NEXT:    retq ## encoding: [0xc3]
-  %res = call <2 x double> @llvm.x86.avx512.mask3.vfmsub.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1, i32 4)
-  %res1 = call <2 x double> @llvm.x86.avx512.mask3.vfmsub.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 4)
-  %res2 = call <2 x double> @llvm.x86.avx512.mask3.vfmsub.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1, i32 11)
-  %res3 = call <2 x double> @llvm.x86.avx512.mask3.vfmsub.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 11)
-  %res4 = fadd <2 x double> %res, %res1
-  %res5 = fadd <2 x double> %res2, %res3
-  %res6 = fadd <2 x double> %res4, %res5
-  ret <2 x double> %res6
+  %res = call <2 x double> @llvm.x86.avx512.mask3.vfmsub.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 4)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask3.vfmsub.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1, i32 11)
+  %res2 = call <2 x double> @llvm.x86.avx512.mask3.vfmsub.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 10)
+  %res3 = fadd <2 x double> %res, %res1
+  %res4 = fadd <2 x double> %res2, %res3
+  ret <2 x double> %res4
 }
 
 declare <4 x float> @llvm.x86.avx512.mask3.vfmsub.ss(<4 x float>, <4 x float>, <4 x float>, i8, i32)
@@ -10197,45 +10301,35 @@ define <4 x float>@test_int_x86_avx512_mask3_vfmsub_ss(<4 x float> %x0, <4 x flo
 ; X86-LABEL: test_int_x86_avx512_mask3_vfmsub_ss:
 ; X86:       ## %bb.0:
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al ## encoding: [0x8a,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vmovaps %xmm2, %xmm3 ## encoding: [0xc5,0xf8,0x28,0xda]
-; X86-NEXT:    vfmsub231ss %xmm1, %xmm0, %xmm3 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0xbb,0xd9]
+; X86-NEXT:    vfmsub231ss %xmm1, %xmm0, %xmm3 {%k1} ## encoding: [0x62,0xf2,0x7d,0x09,0xbb,0xd9]
 ; X86-NEXT:    ## xmm3 = (xmm0 * xmm1) - xmm3
-; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovaps %xmm2, %xmm4 ## encoding: [0xc5,0xf8,0x28,0xe2]
-; X86-NEXT:    vfmsub231ss %xmm1, %xmm0, %xmm4 {%k1} ## encoding: [0x62,0xf2,0x7d,0x09,0xbb,0xe1]
-; X86-NEXT:    ## xmm4 = (xmm0 * xmm1) - xmm4
-; X86-NEXT:    vaddps %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe0,0x58,0xdc]
 ; X86-NEXT:    vmovaps %xmm2, %xmm4 ## encoding: [0xc5,0xf8,0x28,0xe2]
 ; X86-NEXT:    vfmsub231ss {rz-sae}, %xmm1, %xmm0, %xmm4 ## encoding: [0x62,0xf2,0x7d,0x78,0xbb,0xe1]
-; X86-NEXT:    vfmsub231ss {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf2,0x7d,0x79,0xbb,0xd1]
-; X86-NEXT:    vaddps %xmm2, %xmm4, %xmm0 ## encoding: [0xc5,0xd8,0x58,0xc2]
-; X86-NEXT:    vaddps %xmm0, %xmm3, %xmm0 ## encoding: [0xc5,0xe0,0x58,0xc0]
+; X86-NEXT:    vaddps %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe0,0x58,0xdc]
+; X86-NEXT:    vfmsub231ss {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf2,0x7d,0x59,0xbb,0xd1]
+; X86-NEXT:    vaddps %xmm3, %xmm2, %xmm0 ## encoding: [0xc5,0xe8,0x58,0xc3]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask3_vfmsub_ss:
 ; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vmovaps %xmm2, %xmm3 ## encoding: [0xc5,0xf8,0x28,0xda]
-; X64-NEXT:    vfmsub231ss %xmm1, %xmm0, %xmm3 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0xbb,0xd9]
+; X64-NEXT:    vfmsub231ss %xmm1, %xmm0, %xmm3 {%k1} ## encoding: [0x62,0xf2,0x7d,0x09,0xbb,0xd9]
 ; X64-NEXT:    ## xmm3 = (xmm0 * xmm1) - xmm3
-; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovaps %xmm2, %xmm4 ## encoding: [0xc5,0xf8,0x28,0xe2]
-; X64-NEXT:    vfmsub231ss %xmm1, %xmm0, %xmm4 {%k1} ## encoding: [0x62,0xf2,0x7d,0x09,0xbb,0xe1]
-; X64-NEXT:    ## xmm4 = (xmm0 * xmm1) - xmm4
-; X64-NEXT:    vaddps %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe0,0x58,0xdc]
 ; X64-NEXT:    vmovaps %xmm2, %xmm4 ## encoding: [0xc5,0xf8,0x28,0xe2]
 ; X64-NEXT:    vfmsub231ss {rz-sae}, %xmm1, %xmm0, %xmm4 ## encoding: [0x62,0xf2,0x7d,0x78,0xbb,0xe1]
-; X64-NEXT:    vfmsub231ss {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf2,0x7d,0x79,0xbb,0xd1]
-; X64-NEXT:    vaddps %xmm2, %xmm4, %xmm0 ## encoding: [0xc5,0xd8,0x58,0xc2]
-; X64-NEXT:    vaddps %xmm0, %xmm3, %xmm0 ## encoding: [0xc5,0xe0,0x58,0xc0]
+; X64-NEXT:    vaddps %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe0,0x58,0xdc]
+; X64-NEXT:    vfmsub231ss {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf2,0x7d,0x59,0xbb,0xd1]
+; X64-NEXT:    vaddps %xmm3, %xmm2, %xmm0 ## encoding: [0xc5,0xe8,0x58,0xc3]
 ; X64-NEXT:    retq ## encoding: [0xc3]
-  %res = call <4 x float> @llvm.x86.avx512.mask3.vfmsub.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1, i32 4)
-  %res1 = call <4 x float> @llvm.x86.avx512.mask3.vfmsub.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 4)
-  %res2 = call <4 x float> @llvm.x86.avx512.mask3.vfmsub.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1, i32 11)
-  %res3 = call <4 x float> @llvm.x86.avx512.mask3.vfmsub.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 11)
-  %res4 = fadd <4 x float> %res, %res1
-  %res5 = fadd <4 x float> %res2, %res3
-  %res6 = fadd <4 x float> %res4, %res5
-  ret <4 x float> %res6
+  %res = call <4 x float> @llvm.x86.avx512.mask3.vfmsub.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 4)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask3.vfmsub.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1, i32 11)
+  %res2 = call <4 x float> @llvm.x86.avx512.mask3.vfmsub.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 10)
+  %res3 = fadd <4 x float> %res, %res1
+  %res4 = fadd <4 x float> %res2, %res3
+  ret <4 x float> %res4
 }
 
 declare <2 x double> @llvm.x86.avx512.mask3.vfnmsub.sd(<2 x double>, <2 x double>, <2 x double>, i8, i32)
@@ -10244,45 +10338,35 @@ define <2 x double>@test_int_x86_avx512_mask3_vfnmsub_sd(<2 x double> %x0, <2 x
 ; X86-LABEL: test_int_x86_avx512_mask3_vfnmsub_sd:
 ; X86:       ## %bb.0:
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al ## encoding: [0x8a,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vmovapd %xmm2, %xmm3 ## encoding: [0xc5,0xf9,0x28,0xda]
-; X86-NEXT:    vfnmsub231sd %xmm1, %xmm0, %xmm3 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf9,0xbf,0xd9]
+; X86-NEXT:    vfnmsub231sd %xmm1, %xmm0, %xmm3 {%k1} ## encoding: [0x62,0xf2,0xfd,0x09,0xbf,0xd9]
 ; X86-NEXT:    ## xmm3 = -(xmm0 * xmm1) - xmm3
-; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovapd %xmm2, %xmm4 ## encoding: [0xc5,0xf9,0x28,0xe2]
-; X86-NEXT:    vfnmsub231sd %xmm1, %xmm0, %xmm4 {%k1} ## encoding: [0x62,0xf2,0xfd,0x09,0xbf,0xe1]
-; X86-NEXT:    ## xmm4 = -(xmm0 * xmm1) - xmm4
-; X86-NEXT:    vaddpd %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe1,0x58,0xdc]
 ; X86-NEXT:    vmovapd %xmm2, %xmm4 ## encoding: [0xc5,0xf9,0x28,0xe2]
 ; X86-NEXT:    vfnmsub231sd {rz-sae}, %xmm1, %xmm0, %xmm4 ## encoding: [0x62,0xf2,0xfd,0x78,0xbf,0xe1]
-; X86-NEXT:    vfnmsub231sd {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf2,0xfd,0x79,0xbf,0xd1]
-; X86-NEXT:    vaddpd %xmm2, %xmm4, %xmm0 ## encoding: [0xc5,0xd9,0x58,0xc2]
-; X86-NEXT:    vaddpd %xmm0, %xmm3, %xmm0 ## encoding: [0xc5,0xe1,0x58,0xc0]
+; X86-NEXT:    vaddpd %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe1,0x58,0xdc]
+; X86-NEXT:    vfnmsub231sd {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf2,0xfd,0x59,0xbf,0xd1]
+; X86-NEXT:    vaddpd %xmm3, %xmm2, %xmm0 ## encoding: [0xc5,0xe9,0x58,0xc3]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask3_vfnmsub_sd:
 ; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vmovapd %xmm2, %xmm3 ## encoding: [0xc5,0xf9,0x28,0xda]
-; X64-NEXT:    vfnmsub231sd %xmm1, %xmm0, %xmm3 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf9,0xbf,0xd9]
+; X64-NEXT:    vfnmsub231sd %xmm1, %xmm0, %xmm3 {%k1} ## encoding: [0x62,0xf2,0xfd,0x09,0xbf,0xd9]
 ; X64-NEXT:    ## xmm3 = -(xmm0 * xmm1) - xmm3
-; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovapd %xmm2, %xmm4 ## encoding: [0xc5,0xf9,0x28,0xe2]
-; X64-NEXT:    vfnmsub231sd %xmm1, %xmm0, %xmm4 {%k1} ## encoding: [0x62,0xf2,0xfd,0x09,0xbf,0xe1]
-; X64-NEXT:    ## xmm4 = -(xmm0 * xmm1) - xmm4
-; X64-NEXT:    vaddpd %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe1,0x58,0xdc]
 ; X64-NEXT:    vmovapd %xmm2, %xmm4 ## encoding: [0xc5,0xf9,0x28,0xe2]
 ; X64-NEXT:    vfnmsub231sd {rz-sae}, %xmm1, %xmm0, %xmm4 ## encoding: [0x62,0xf2,0xfd,0x78,0xbf,0xe1]
-; X64-NEXT:    vfnmsub231sd {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf2,0xfd,0x79,0xbf,0xd1]
-; X64-NEXT:    vaddpd %xmm2, %xmm4, %xmm0 ## encoding: [0xc5,0xd9,0x58,0xc2]
-; X64-NEXT:    vaddpd %xmm0, %xmm3, %xmm0 ## encoding: [0xc5,0xe1,0x58,0xc0]
+; X64-NEXT:    vaddpd %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe1,0x58,0xdc]
+; X64-NEXT:    vfnmsub231sd {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf2,0xfd,0x59,0xbf,0xd1]
+; X64-NEXT:    vaddpd %xmm3, %xmm2, %xmm0 ## encoding: [0xc5,0xe9,0x58,0xc3]
 ; X64-NEXT:    retq ## encoding: [0xc3]
-  %res = call <2 x double> @llvm.x86.avx512.mask3.vfnmsub.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1, i32 4)
-  %res1 = call <2 x double> @llvm.x86.avx512.mask3.vfnmsub.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 4)
-  %res2 = call <2 x double> @llvm.x86.avx512.mask3.vfnmsub.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1, i32 11)
-  %res3 = call <2 x double> @llvm.x86.avx512.mask3.vfnmsub.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 11)
-  %res4 = fadd <2 x double> %res, %res1
-  %res5 = fadd <2 x double> %res2, %res3
-  %res6 = fadd <2 x double> %res4, %res5
-  ret <2 x double> %res6
+  %res = call <2 x double> @llvm.x86.avx512.mask3.vfnmsub.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 4)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask3.vfnmsub.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1, i32 11)
+  %res2 = call <2 x double> @llvm.x86.avx512.mask3.vfnmsub.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 10)
+  %res3 = fadd <2 x double> %res, %res1
+  %res4 = fadd <2 x double> %res2, %res3
+  ret <2 x double> %res4
 }
 
 declare <4 x float> @llvm.x86.avx512.mask3.vfnmsub.ss(<4 x float>, <4 x float>, <4 x float>, i8, i32)
@@ -10291,45 +10375,35 @@ define <4 x float>@test_int_x86_avx512_mask3_vfnmsub_ss(<4 x float> %x0, <4 x fl
 ; X86-LABEL: test_int_x86_avx512_mask3_vfnmsub_ss:
 ; X86:       ## %bb.0:
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al ## encoding: [0x8a,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vmovaps %xmm2, %xmm3 ## encoding: [0xc5,0xf8,0x28,0xda]
-; X86-NEXT:    vfnmsub231ss %xmm1, %xmm0, %xmm3 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0xbf,0xd9]
+; X86-NEXT:    vfnmsub231ss %xmm1, %xmm0, %xmm3 {%k1} ## encoding: [0x62,0xf2,0x7d,0x09,0xbf,0xd9]
 ; X86-NEXT:    ## xmm3 = -(xmm0 * xmm1) - xmm3
-; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovaps %xmm2, %xmm4 ## encoding: [0xc5,0xf8,0x28,0xe2]
-; X86-NEXT:    vfnmsub231ss %xmm1, %xmm0, %xmm4 {%k1} ## encoding: [0x62,0xf2,0x7d,0x09,0xbf,0xe1]
-; X86-NEXT:    ## xmm4 = -(xmm0 * xmm1) - xmm4
-; X86-NEXT:    vaddps %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe0,0x58,0xdc]
 ; X86-NEXT:    vmovaps %xmm2, %xmm4 ## encoding: [0xc5,0xf8,0x28,0xe2]
 ; X86-NEXT:    vfnmsub231ss {rz-sae}, %xmm1, %xmm0, %xmm4 ## encoding: [0x62,0xf2,0x7d,0x78,0xbf,0xe1]
-; X86-NEXT:    vfnmsub231ss {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf2,0x7d,0x79,0xbf,0xd1]
-; X86-NEXT:    vaddps %xmm2, %xmm4, %xmm0 ## encoding: [0xc5,0xd8,0x58,0xc2]
-; X86-NEXT:    vaddps %xmm0, %xmm3, %xmm0 ## encoding: [0xc5,0xe0,0x58,0xc0]
+; X86-NEXT:    vaddps %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe0,0x58,0xdc]
+; X86-NEXT:    vfnmsub231ss {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf2,0x7d,0x59,0xbf,0xd1]
+; X86-NEXT:    vaddps %xmm3, %xmm2, %xmm0 ## encoding: [0xc5,0xe8,0x58,0xc3]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask3_vfnmsub_ss:
 ; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vmovaps %xmm2, %xmm3 ## encoding: [0xc5,0xf8,0x28,0xda]
-; X64-NEXT:    vfnmsub231ss %xmm1, %xmm0, %xmm3 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0xbf,0xd9]
+; X64-NEXT:    vfnmsub231ss %xmm1, %xmm0, %xmm3 {%k1} ## encoding: [0x62,0xf2,0x7d,0x09,0xbf,0xd9]
 ; X64-NEXT:    ## xmm3 = -(xmm0 * xmm1) - xmm3
-; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovaps %xmm2, %xmm4 ## encoding: [0xc5,0xf8,0x28,0xe2]
-; X64-NEXT:    vfnmsub231ss %xmm1, %xmm0, %xmm4 {%k1} ## encoding: [0x62,0xf2,0x7d,0x09,0xbf,0xe1]
-; X64-NEXT:    ## xmm4 = -(xmm0 * xmm1) - xmm4
-; X64-NEXT:    vaddps %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe0,0x58,0xdc]
 ; X64-NEXT:    vmovaps %xmm2, %xmm4 ## encoding: [0xc5,0xf8,0x28,0xe2]
 ; X64-NEXT:    vfnmsub231ss {rz-sae}, %xmm1, %xmm0, %xmm4 ## encoding: [0x62,0xf2,0x7d,0x78,0xbf,0xe1]
-; X64-NEXT:    vfnmsub231ss {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf2,0x7d,0x79,0xbf,0xd1]
-; X64-NEXT:    vaddps %xmm2, %xmm4, %xmm0 ## encoding: [0xc5,0xd8,0x58,0xc2]
-; X64-NEXT:    vaddps %xmm0, %xmm3, %xmm0 ## encoding: [0xc5,0xe0,0x58,0xc0]
-; X64-NEXT:    retq ## encoding: [0xc3]
-  %res = call <4 x float> @llvm.x86.avx512.mask3.vfnmsub.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1, i32 4)
-  %res1 = call <4 x float> @llvm.x86.avx512.mask3.vfnmsub.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 4)
-  %res2 = call <4 x float> @llvm.x86.avx512.mask3.vfnmsub.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1, i32 11)
-  %res3 = call <4 x float> @llvm.x86.avx512.mask3.vfnmsub.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 11)
-  %res4 = fadd <4 x float> %res, %res1
-  %res5 = fadd <4 x float> %res2, %res3
-  %res6 = fadd <4 x float> %res4, %res5
-  ret <4 x float> %res6
+; X64-NEXT:    vaddps %xmm4, %xmm3, %xmm3 ## encoding: [0xc5,0xe0,0x58,0xdc]
+; X64-NEXT:    vfnmsub231ss {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf2,0x7d,0x59,0xbf,0xd1]
+; X64-NEXT:    vaddps %xmm3, %xmm2, %xmm0 ## encoding: [0xc5,0xe8,0x58,0xc3]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %res = call <4 x float> @llvm.x86.avx512.mask3.vfnmsub.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 4)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask3.vfnmsub.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1, i32 11)
+  %res2 = call <4 x float> @llvm.x86.avx512.mask3.vfnmsub.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 10)
+  %res3 = fadd <4 x float> %res, %res1
+  %res4 = fadd <4 x float> %res2, %res3
+  ret <4 x float> %res4
 }
 
 define <4 x float>@test_int_x86_avx512_mask3_vfmadd_ss_rm(<4 x float> %x0, <4 x float> %x1, float *%ptr_b ,i8 %x3,i32 %x4) {
diff --git a/llvm/test/CodeGen/X86/avx512-intrinsics.ll b/llvm/test/CodeGen/X86/avx512-intrinsics.ll
index 07278bde99c22..762a0909fb2c1 100644
--- a/llvm/test/CodeGen/X86/avx512-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512-intrinsics.ll
@@ -4409,81 +4409,107 @@ define <16 x i32>@test_int_x86_avx512_mask_cvtt_ps2udq_512(<16 x float> %x0, <16
 
 declare <4 x float> @llvm.x86.avx512.mask.getexp.ss(<4 x float>, <4 x float>, <4 x float>, i8, i32) nounwind readnone
 
-define <4 x float> @test_getexp_ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
-; X64-LABEL: test_getexp_ss:
+define <4 x float> @test_getexp_ss(<4 x float> %a0, <4 x float> %a1) {
+; CHECK-LABEL: test_getexp_ss:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vgetexpss {sae}, %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <4 x float> @llvm.x86.avx512.mask.getexp.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 -1, i32 8)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_getexp_ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
+; X64-LABEL: test_mask_getexp_ss:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vmovaps %xmm2, %xmm3
 ; X64-NEXT:    vgetexpss %xmm1, %xmm0, %xmm3 {%k1}
-; X64-NEXT:    vgetexpss {sae}, %xmm1, %xmm0, %xmm4 {%k1} {z}
-; X64-NEXT:    vgetexpss {sae}, %xmm1, %xmm0, %xmm5
-; X64-NEXT:    vaddps %xmm5, %xmm4, %xmm4
 ; X64-NEXT:    vgetexpss {sae}, %xmm1, %xmm0, %xmm2 {%k1}
 ; X64-NEXT:    vaddps %xmm2, %xmm3, %xmm0
-; X64-NEXT:    vaddps %xmm4, %xmm0, %xmm0
 ; X64-NEXT:    retq
 ;
-; X86-LABEL: test_getexp_ss:
+; X86-LABEL: test_mask_getexp_ss:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vmovaps %xmm2, %xmm3
 ; X86-NEXT:    vgetexpss %xmm1, %xmm0, %xmm3 {%k1}
 ; X86-NEXT:    vgetexpss {sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; X86-NEXT:    vaddps %xmm2, %xmm3, %xmm2
-; X86-NEXT:    vgetexpss {sae}, %xmm1, %xmm0, %xmm3 {%k1} {z}
-; X86-NEXT:    vgetexpss {sae}, %xmm1, %xmm0, %xmm0
-; X86-NEXT:    vaddps %xmm0, %xmm3, %xmm0
-; X86-NEXT:    vaddps %xmm0, %xmm2, %xmm0
+; X86-NEXT:    vaddps %xmm2, %xmm3, %xmm0
 ; X86-NEXT:    retl
   %res0 = call <4 x float> @llvm.x86.avx512.mask.getexp.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 4)
   %res1 = call <4 x float> @llvm.x86.avx512.mask.getexp.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 8)
-  %res2 = call <4 x float> @llvm.x86.avx512.mask.getexp.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 %mask, i32 8)
-  %res3 = call <4 x float> @llvm.x86.avx512.mask.getexp.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 -1, i32 8)
-
   %res.1 = fadd <4 x float> %res0, %res1
-  %res.2 = fadd <4 x float> %res2, %res3
-  %res   = fadd <4 x float> %res.1, %res.2
+  ret <4 x float> %res.1
+}
+
+define <4 x float> @test_maskz_getexp_ss(<4 x float> %a0, <4 x float> %a1, i8 %mask) {
+; X64-LABEL: test_maskz_getexp_ss:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vgetexpss {sae}, %xmm1, %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_maskz_getexp_ss:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vgetexpss {sae}, %xmm1, %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    retl
+  %res = call <4 x float> @llvm.x86.avx512.mask.getexp.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 %mask, i32 8)
   ret <4 x float> %res
 }
 
 declare <2 x double> @llvm.x86.avx512.mask.getexp.sd(<2 x double>, <2 x double>, <2 x double>, i8, i32) nounwind readnone
 
-define <2 x double> @test_getexp_sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
-; X64-LABEL: test_getexp_sd:
+define <2 x double> @test_getexp_sd(<2 x double> %a0, <2 x double> %a1) {
+; CHECK-LABEL: test_getexp_sd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vgetexpsd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <2 x double> @llvm.x86.avx512.mask.getexp.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 -1, i32 4)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_mask_getexp_sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
+; X64-LABEL: test_mask_getexp_sd:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1
-; X64-NEXT:    vgetexpsd %xmm1, %xmm0, %xmm3
-; X64-NEXT:    vmovapd %xmm2, %xmm4
-; X64-NEXT:    vgetexpsd %xmm1, %xmm0, %xmm4 {%k1}
-; X64-NEXT:    vgetexpsd {sae}, %xmm1, %xmm0, %xmm5 {%k1} {z}
-; X64-NEXT:    vaddpd %xmm3, %xmm5, %xmm3
+; X64-NEXT:    vmovapd %xmm2, %xmm3
+; X64-NEXT:    vgetexpsd %xmm1, %xmm0, %xmm3 {%k1}
 ; X64-NEXT:    vgetexpsd {sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; X64-NEXT:    vaddpd %xmm2, %xmm4, %xmm0
-; X64-NEXT:    vaddpd %xmm3, %xmm0, %xmm0
+; X64-NEXT:    vaddpd %xmm2, %xmm3, %xmm0
 ; X64-NEXT:    retq
 ;
-; X86-LABEL: test_getexp_sd:
+; X86-LABEL: test_mask_getexp_sd:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vmovapd %xmm2, %xmm3
-; X86-NEXT:    vgetexpsd {sae}, %xmm1, %xmm0, %xmm3 {%k1}
-; X86-NEXT:    vgetexpsd {sae}, %xmm1, %xmm0, %xmm4 {%k1} {z}
-; X86-NEXT:    vgetexpsd %xmm1, %xmm0, %xmm2 {%k1}
-; X86-NEXT:    vaddpd %xmm3, %xmm2, %xmm2
-; X86-NEXT:    vgetexpsd %xmm1, %xmm0, %xmm0
-; X86-NEXT:    vaddpd %xmm0, %xmm4, %xmm0
-; X86-NEXT:    vaddpd %xmm0, %xmm2, %xmm0
+; X86-NEXT:    vgetexpsd %xmm1, %xmm0, %xmm3 {%k1}
+; X86-NEXT:    vgetexpsd {sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vaddpd %xmm2, %xmm3, %xmm0
 ; X86-NEXT:    retl
   %res0 = call <2 x double> @llvm.x86.avx512.mask.getexp.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 4)
   %res1 = call <2 x double> @llvm.x86.avx512.mask.getexp.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 8)
-  %res2 = call <2 x double> @llvm.x86.avx512.mask.getexp.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 %mask, i32 8)
-  %res3 = call <2 x double> @llvm.x86.avx512.mask.getexp.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 -1, i32 4)
-
   %res.1 = fadd <2 x double> %res0, %res1
-  %res.2 = fadd <2 x double> %res2, %res3
-  %res   = fadd <2 x double> %res.1, %res.2
+  ret <2 x double> %res.1
+}
+
+define <2 x double> @test_maskz_getexp_sd(<2 x double> %a0, <2 x double> %a1, i8 %mask) {
+; X64-LABEL: test_maskz_getexp_sd:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vgetexpsd {sae}, %xmm1, %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_maskz_getexp_sd:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vgetexpsd {sae}, %xmm1, %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    retl
+  %res = call <2 x double> @llvm.x86.avx512.mask.getexp.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 %mask, i32 8)
   ret <2 x double> %res
 }
 
@@ -4694,14 +4720,14 @@ define <2 x double>@test_int_x86_avx512_mask_getmant_sd(<2 x double> %x0, <2 x d
 ; X64-LABEL: test_int_x86_avx512_mask_getmant_sd:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1
-; X64-NEXT:    vgetmantsd $11, %xmm1, %xmm0, %xmm3
-; X64-NEXT:    vmovapd %xmm2, %xmm4
-; X64-NEXT:    vgetmantsd $11, %xmm1, %xmm0, %xmm4 {%k1}
-; X64-NEXT:    vgetmantsd $11, %xmm1, %xmm0, %xmm5 {%k1} {z}
-; X64-NEXT:    vaddpd %xmm5, %xmm4, %xmm4
-; X64-NEXT:    vgetmantsd $11, {sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; X64-NEXT:    vaddpd %xmm3, %xmm2, %xmm0
-; X64-NEXT:    vaddpd %xmm0, %xmm4, %xmm0
+; X64-NEXT:    vmovapd %xmm2, %xmm3
+; X64-NEXT:    vgetmantsd $11, %xmm1, %xmm0, %xmm3 {%k1}
+; X64-NEXT:    vgetmantsd $12, %xmm1, %xmm0, %xmm4 {%k1} {z}
+; X64-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
+; X64-NEXT:    vgetmantsd $13, {sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vgetmantsd $14, %xmm1, %xmm0, %xmm0
+; X64-NEXT:    vaddpd %xmm0, %xmm2, %xmm0
+; X64-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: test_int_x86_avx512_mask_getmant_sd:
@@ -4710,17 +4736,17 @@ define <2 x double>@test_int_x86_avx512_mask_getmant_sd(<2 x double> %x0, <2 x d
 ; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vmovapd %xmm2, %xmm3
 ; X86-NEXT:    vgetmantsd $11, %xmm1, %xmm0, %xmm3 {%k1}
-; X86-NEXT:    vgetmantsd $11, %xmm1, %xmm0, %xmm4 {%k1} {z}
+; X86-NEXT:    vgetmantsd $12, %xmm1, %xmm0, %xmm4 {%k1} {z}
 ; X86-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
-; X86-NEXT:    vgetmantsd $11, %xmm1, %xmm0, %xmm4
-; X86-NEXT:    vgetmantsd $11, {sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; X86-NEXT:    vaddpd %xmm4, %xmm2, %xmm0
+; X86-NEXT:    vgetmantsd $13, {sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vgetmantsd $14, %xmm1, %xmm0, %xmm0
+; X86-NEXT:    vaddpd %xmm0, %xmm2, %xmm0
 ; X86-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
 ; X86-NEXT:    retl
   %res  = call <2 x double> @llvm.x86.avx512.mask.getmant.sd(<2 x double> %x0, <2 x double> %x1, i32 11, <2 x double> %x2, i8 %x3, i32 4)
-  %res1 = call <2 x double> @llvm.x86.avx512.mask.getmant.sd(<2 x double> %x0, <2 x double> %x1, i32 11, <2 x double> zeroinitializer, i8 %x3, i32 4)
-  %res2 = call <2 x double> @llvm.x86.avx512.mask.getmant.sd(<2 x double> %x0, <2 x double> %x1, i32 11, <2 x double> %x2, i8 %x3, i32 8)
-  %res3 = call <2 x double> @llvm.x86.avx512.mask.getmant.sd(<2 x double> %x0, <2 x double> %x1, i32 11, <2 x double> %x2, i8 -1, i32 4)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.getmant.sd(<2 x double> %x0, <2 x double> %x1, i32 12, <2 x double> zeroinitializer, i8 %x3, i32 4)
+  %res2 = call <2 x double> @llvm.x86.avx512.mask.getmant.sd(<2 x double> %x0, <2 x double> %x1, i32 13, <2 x double> %x2, i8 %x3, i32 8)
+  %res3 = call <2 x double> @llvm.x86.avx512.mask.getmant.sd(<2 x double> %x0, <2 x double> %x1, i32 14, <2 x double> %x2, i8 -1, i32 4)
   %res11 = fadd <2 x double> %res, %res1
   %res12 = fadd <2 x double> %res2, %res3
   %res13 = fadd <2 x double> %res11, %res12
@@ -4733,12 +4759,12 @@ define <4 x float>@test_int_x86_avx512_mask_getmant_ss(<4 x float> %x0, <4 x flo
 ; X64-LABEL: test_int_x86_avx512_mask_getmant_ss:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1
-; X64-NEXT:    vgetmantss $11, %xmm1, %xmm0, %xmm3
 ; X64-NEXT:    vgetmantss $11, %xmm1, %xmm0, %xmm2 {%k1}
-; X64-NEXT:    vgetmantss $11, %xmm1, %xmm0, %xmm4 {%k1} {z}
-; X64-NEXT:    vaddps %xmm4, %xmm2, %xmm2
-; X64-NEXT:    vgetmantss $11, {sae}, %xmm1, %xmm0, %xmm0
-; X64-NEXT:    vaddps %xmm3, %xmm0, %xmm0
+; X64-NEXT:    vgetmantss $12, %xmm1, %xmm0, %xmm3 {%k1} {z}
+; X64-NEXT:    vaddps %xmm3, %xmm2, %xmm2
+; X64-NEXT:    vgetmantss $13, {sae}, %xmm1, %xmm0, %xmm3
+; X64-NEXT:    vgetmantss $14, %xmm1, %xmm0, %xmm0
+; X64-NEXT:    vaddps %xmm0, %xmm3, %xmm0
 ; X64-NEXT:    vaddps %xmm0, %xmm2, %xmm0
 ; X64-NEXT:    retq
 ;
@@ -4747,17 +4773,17 @@ define <4 x float>@test_int_x86_avx512_mask_getmant_ss(<4 x float> %x0, <4 x flo
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vgetmantss $11, %xmm1, %xmm0, %xmm2 {%k1}
-; X86-NEXT:    vgetmantss $11, %xmm1, %xmm0, %xmm3 {%k1} {z}
+; X86-NEXT:    vgetmantss $12, %xmm1, %xmm0, %xmm3 {%k1} {z}
 ; X86-NEXT:    vaddps %xmm3, %xmm2, %xmm2
-; X86-NEXT:    vgetmantss $11, %xmm1, %xmm0, %xmm3
-; X86-NEXT:    vgetmantss $11, {sae}, %xmm1, %xmm0, %xmm0
-; X86-NEXT:    vaddps %xmm3, %xmm0, %xmm0
+; X86-NEXT:    vgetmantss $13, {sae}, %xmm1, %xmm0, %xmm3
+; X86-NEXT:    vgetmantss $14, %xmm1, %xmm0, %xmm0
+; X86-NEXT:    vaddps %xmm0, %xmm3, %xmm0
 ; X86-NEXT:    vaddps %xmm0, %xmm2, %xmm0
 ; X86-NEXT:    retl
   %res  = call <4 x float> @llvm.x86.avx512.mask.getmant.ss(<4 x float> %x0, <4 x float> %x1, i32 11, <4 x float> %x2, i8 %x3, i32 4)
-  %res1 = call <4 x float> @llvm.x86.avx512.mask.getmant.ss(<4 x float> %x0, <4 x float> %x1, i32 11, <4 x float> zeroinitializer, i8 %x3, i32 4)
-  %res2 = call <4 x float> @llvm.x86.avx512.mask.getmant.ss(<4 x float> %x0, <4 x float> %x1, i32 11, <4 x float> %x2, i8 -1, i32 8)
-  %res3 = call <4 x float> @llvm.x86.avx512.mask.getmant.ss(<4 x float> %x0, <4 x float> %x1, i32 11, <4 x float> %x2, i8 -1, i32 4)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.getmant.ss(<4 x float> %x0, <4 x float> %x1, i32 12, <4 x float> zeroinitializer, i8 %x3, i32 4)
+  %res2 = call <4 x float> @llvm.x86.avx512.mask.getmant.ss(<4 x float> %x0, <4 x float> %x1, i32 13, <4 x float> %x2, i8 -1, i32 8)
+  %res3 = call <4 x float> @llvm.x86.avx512.mask.getmant.ss(<4 x float> %x0, <4 x float> %x1, i32 14, <4 x float> %x2, i8 -1, i32 4)
   %res11 = fadd <4 x float> %res, %res1
   %res12 = fadd <4 x float> %res2, %res3
   %res13 = fadd <4 x float> %res11, %res12
@@ -4980,112 +5006,98 @@ define <4 x float>@test_int_x86_avx512_mask_cvt_sd2ss_round(<4 x float> %x0,<2 x
 
 declare <16 x i32> @llvm.x86.avx512.pternlog.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i32)
 
+define <16 x i32>@test_int_x86_avx512_pternlog_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_pternlog_d_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpternlogd $33, %zmm2, %zmm1, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 = call <16 x i32> @llvm.x86.avx512.pternlog.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i32 33)
+  ret <16 x i32> %1
+}
+
 define <16 x i32>@test_int_x86_avx512_mask_pternlog_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x4) {
 ; X64-LABEL: test_int_x86_avx512_mask_pternlog_d_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm3
-; X64-NEXT:    vpternlogd $33, %zmm2, %zmm1, %zmm3
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vpternlogd $33, %zmm2, %zmm1, %zmm0 {%k1}
-; X64-NEXT:    vpaddd %zmm3, %zmm0, %zmm0
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: test_int_x86_avx512_mask_pternlog_d_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm3
-; X86-NEXT:    vpternlogd $33, %zmm2, %zmm1, %zmm3
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
 ; X86-NEXT:    vpternlogd $33, %zmm2, %zmm1, %zmm0 {%k1}
-; X86-NEXT:    vpaddd %zmm3, %zmm0, %zmm0
 ; X86-NEXT:    retl
   %1 = call <16 x i32> @llvm.x86.avx512.pternlog.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i32 33)
   %2 = bitcast i16 %x4 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x0
-  %4 = call <16 x i32> @llvm.x86.avx512.pternlog.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i32 33)
-  %res2 = add <16 x i32> %3, %4
-  ret <16 x i32> %res2
+  ret <16 x i32> %3
 }
 
 define <16 x i32>@test_int_x86_avx512_maskz_pternlog_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x4) {
 ; X64-LABEL: test_int_x86_avx512_maskz_pternlog_d_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm3
-; X64-NEXT:    vpternlogd $33, %zmm2, %zmm1, %zmm3
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vpternlogd $33, %zmm2, %zmm1, %zmm0 {%k1} {z}
-; X64-NEXT:    vpaddd %zmm3, %zmm0, %zmm0
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: test_int_x86_avx512_maskz_pternlog_d_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm3
-; X86-NEXT:    vpternlogd $33, %zmm2, %zmm1, %zmm3
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
 ; X86-NEXT:    vpternlogd $33, %zmm2, %zmm1, %zmm0 {%k1} {z}
-; X86-NEXT:    vpaddd %zmm3, %zmm0, %zmm0
 ; X86-NEXT:    retl
   %1 = call <16 x i32> @llvm.x86.avx512.pternlog.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i32 33)
   %2 = bitcast i16 %x4 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> zeroinitializer
-  %4 = call <16 x i32> @llvm.x86.avx512.pternlog.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i32 33)
-  %res2 = add <16 x i32> %3, %4
-  ret <16 x i32> %res2
+  ret <16 x i32> %3
 }
 
 declare <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i32)
 
+define <8 x i64>@test_int_x86_avx512_pternlog_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_pternlog_q_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpternlogq $33, %zmm2, %zmm1, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 = call <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i32 33)
+  ret <8 x i64> %1
+}
+
 define <8 x i64>@test_int_x86_avx512_mask_pternlog_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x4) {
 ; X64-LABEL: test_int_x86_avx512_mask_pternlog_q_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm3
-; X64-NEXT:    vpternlogq $33, %zmm2, %zmm1, %zmm3
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vpternlogq $33, %zmm2, %zmm1, %zmm0 {%k1}
-; X64-NEXT:    vpaddq %zmm3, %zmm0, %zmm0
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: test_int_x86_avx512_mask_pternlog_q_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm3
-; X86-NEXT:    vpternlogq $33, %zmm2, %zmm1, %zmm3
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpternlogq $33, %zmm2, %zmm1, %zmm0 {%k1}
-; X86-NEXT:    vpaddq %zmm3, %zmm0, %zmm0
 ; X86-NEXT:    retl
   %1 = call <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i32 33)
   %2 = bitcast i8 %x4 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x0
-  %4 = call <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i32 33)
-  %res2 = add <8 x i64> %3, %4
-  ret <8 x i64> %res2
+  ret <8 x i64> %3
 }
 
 define <8 x i64>@test_int_x86_avx512_maskz_pternlog_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x4) {
 ; X64-LABEL: test_int_x86_avx512_maskz_pternlog_q_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm3
-; X64-NEXT:    vpternlogq $33, %zmm2, %zmm1, %zmm3
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vpternlogq $33, %zmm2, %zmm1, %zmm0 {%k1} {z}
-; X64-NEXT:    vpaddq %zmm3, %zmm0, %zmm0
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: test_int_x86_avx512_maskz_pternlog_q_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm3
-; X86-NEXT:    vpternlogq $33, %zmm2, %zmm1, %zmm3
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpternlogq $33, %zmm2, %zmm1, %zmm0 {%k1} {z}
-; X86-NEXT:    vpaddq %zmm3, %zmm0, %zmm0
 ; X86-NEXT:    retl
   %1 = call <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i32 33)
   %2 = bitcast i8 %x4 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> zeroinitializer
-  %4 = call <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i32 33)
-  %res2 = add <8 x i64> %3, %4
-  ret <8 x i64> %res2
+  ret <8 x i64> %3
 }
 
 define i32 @test_x86_avx512_comi_sd_eq_sae(<2 x double> %a0, <2 x double> %a1) {
@@ -5514,13 +5526,13 @@ define <4 x float>@test_int_x86_avx512_maskz_fixupimm_ss(<4 x float> %x0, <4 x f
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vmovaps %xmm0, %xmm3
-; X64-NEXT:    vfixupimmss $5, %xmm2, %xmm1, %xmm3
-; X64-NEXT:    vmovaps %xmm0, %xmm4
-; X64-NEXT:    vfixupimmss $5, %xmm2, %xmm1, %xmm4 {%k1} {z}
-; X64-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; X64-NEXT:    vfixupimmss $5, {sae}, %xmm2, %xmm1, %xmm0 {%k1} {z}
-; X64-NEXT:    vaddps %xmm0, %xmm4, %xmm0
-; X64-NEXT:    vaddps %xmm3, %xmm0, %xmm0
+; X64-NEXT:    vfixupimmss $5, %xmm2, %xmm1, %xmm3 {%k1} {z}
+; X64-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; X64-NEXT:    vmovaps %xmm0, %xmm5
+; X64-NEXT:    vfixupimmss $5, {sae}, %xmm4, %xmm1, %xmm5 {%k1} {z}
+; X64-NEXT:    vaddps %xmm5, %xmm3, %xmm3
+; X64-NEXT:    vfixupimmss $6, %xmm2, %xmm1, %xmm0
+; X64-NEXT:    vaddps %xmm0, %xmm3, %xmm0
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: test_int_x86_avx512_maskz_fixupimm_ss:
@@ -5529,16 +5541,16 @@ define <4 x float>@test_int_x86_avx512_maskz_fixupimm_ss(<4 x float> %x0, <4 x f
 ; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vmovaps %xmm0, %xmm3
 ; X86-NEXT:    vfixupimmss $5, %xmm2, %xmm1, %xmm3 {%k1} {z}
-; X86-NEXT:    vmovaps %xmm0, %xmm4
-; X86-NEXT:    vfixupimmss $5, %xmm2, %xmm1, %xmm4
-; X86-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; X86-NEXT:    vfixupimmss $5, {sae}, %xmm2, %xmm1, %xmm0 {%k1} {z}
+; X86-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; X86-NEXT:    vmovaps %xmm0, %xmm5
+; X86-NEXT:    vfixupimmss $5, {sae}, %xmm4, %xmm1, %xmm5 {%k1} {z}
+; X86-NEXT:    vaddps %xmm5, %xmm3, %xmm3
+; X86-NEXT:    vfixupimmss $6, %xmm2, %xmm1, %xmm0
 ; X86-NEXT:    vaddps %xmm0, %xmm3, %xmm0
-; X86-NEXT:    vaddps %xmm4, %xmm0, %xmm0
 ; X86-NEXT:    retl
   %res = call <4 x float> @llvm.x86.avx512.maskz.fixupimm.ss(<4 x float> %x0, <4 x float> %x1, <4 x i32> %x2, i32 5, i8 %x4, i32 4)
   %res1 = call <4 x float> @llvm.x86.avx512.maskz.fixupimm.ss(<4 x float> %x0, <4 x float> %x1, <4 x i32> zeroinitializer, i32 5, i8 %x4, i32 8)
-  %res2 = call <4 x float> @llvm.x86.avx512.maskz.fixupimm.ss(<4 x float> %x0, <4 x float> %x1, <4 x i32> %x2, i32 5, i8 -1, i32 4)
+  %res2 = call <4 x float> @llvm.x86.avx512.maskz.fixupimm.ss(<4 x float> %x0, <4 x float> %x1, <4 x i32> %x2, i32 6, i8 -1, i32 4)
   %res3 = fadd <4 x float> %res, %res1
   %res4 = fadd <4 x float> %res3, %res2
   ret <4 x float> %res4
@@ -5603,13 +5615,13 @@ define <16 x float>@test_int_x86_avx512_maskz_fixupimm_ps_512(<16 x float> %x0,
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vmovaps %zmm0, %zmm3
-; X64-NEXT:    vfixupimmps $5, %zmm2, %zmm1, %zmm3
-; X64-NEXT:    vmovaps %zmm0, %zmm4
-; X64-NEXT:    vfixupimmps $5, %zmm2, %zmm1, %zmm4 {%k1} {z}
-; X64-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; X64-NEXT:    vfixupimmps $5, {sae}, %zmm2, %zmm1, %zmm0 {%k1} {z}
-; X64-NEXT:    vaddps %zmm0, %zmm4, %zmm0
-; X64-NEXT:    vaddps %zmm3, %zmm0, %zmm0
+; X64-NEXT:    vfixupimmps $5, %zmm2, %zmm1, %zmm3 {%k1} {z}
+; X64-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; X64-NEXT:    vmovaps %zmm0, %zmm5
+; X64-NEXT:    vfixupimmps $6, {sae}, %zmm4, %zmm1, %zmm5 {%k1} {z}
+; X64-NEXT:    vaddps %zmm5, %zmm3, %zmm3
+; X64-NEXT:    vfixupimmps $7, %zmm2, %zmm1, %zmm0
+; X64-NEXT:    vaddps %zmm0, %zmm3, %zmm0
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: test_int_x86_avx512_maskz_fixupimm_ps_512:
@@ -5617,16 +5629,16 @@ define <16 x float>@test_int_x86_avx512_maskz_fixupimm_ps_512(<16 x float> %x0,
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
 ; X86-NEXT:    vmovaps %zmm0, %zmm3
 ; X86-NEXT:    vfixupimmps $5, %zmm2, %zmm1, %zmm3 {%k1} {z}
-; X86-NEXT:    vmovaps %zmm0, %zmm4
-; X86-NEXT:    vfixupimmps $5, %zmm2, %zmm1, %zmm4
-; X86-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; X86-NEXT:    vfixupimmps $5, {sae}, %zmm2, %zmm1, %zmm0 {%k1} {z}
+; X86-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; X86-NEXT:    vmovaps %zmm0, %zmm5
+; X86-NEXT:    vfixupimmps $6, {sae}, %zmm4, %zmm1, %zmm5 {%k1} {z}
+; X86-NEXT:    vaddps %zmm5, %zmm3, %zmm3
+; X86-NEXT:    vfixupimmps $7, %zmm2, %zmm1, %zmm0
 ; X86-NEXT:    vaddps %zmm0, %zmm3, %zmm0
-; X86-NEXT:    vaddps %zmm4, %zmm0, %zmm0
 ; X86-NEXT:    retl
   %res = call <16 x float> @llvm.x86.avx512.maskz.fixupimm.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x i32> %x2, i32 5, i16 %x4, i32 4)
-  %res1 = call <16 x float> @llvm.x86.avx512.maskz.fixupimm.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x i32> zeroinitializer, i32 5, i16 %x4, i32 8)
-  %res2 = call <16 x float> @llvm.x86.avx512.maskz.fixupimm.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x i32> %x2, i32 5, i16 -1, i32 4)
+  %res1 = call <16 x float> @llvm.x86.avx512.maskz.fixupimm.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x i32> zeroinitializer, i32 6, i16 %x4, i32 8)
+  %res2 = call <16 x float> @llvm.x86.avx512.maskz.fixupimm.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x i32> %x2, i32 7, i16 -1, i32 4)
   %res3 = fadd <16 x float> %res, %res1
   %res4 = fadd <16 x float> %res3, %res2
   ret <16 x float> %res4
@@ -5639,13 +5651,13 @@ define <2 x double>@test_int_x86_avx512_mask_fixupimm_sd(<2 x double> %x0, <2 x
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vmovapd %xmm0, %xmm3
-; X64-NEXT:    vfixupimmsd $5, %xmm2, %xmm1, %xmm3
-; X64-NEXT:    vmovapd %xmm0, %xmm4
-; X64-NEXT:    vfixupimmsd $5, %xmm2, %xmm1, %xmm4 {%k1}
-; X64-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
-; X64-NEXT:    vfixupimmsd $5, {sae}, %xmm2, %xmm1, %xmm0 {%k1}
-; X64-NEXT:    vaddpd %xmm0, %xmm4, %xmm0
-; X64-NEXT:    vaddpd %xmm3, %xmm0, %xmm0
+; X64-NEXT:    vfixupimmsd $5, %xmm2, %xmm1, %xmm3 {%k1}
+; X64-NEXT:    vxorpd %xmm4, %xmm4, %xmm4
+; X64-NEXT:    vmovapd %xmm0, %xmm5
+; X64-NEXT:    vfixupimmsd $5, {sae}, %xmm4, %xmm1, %xmm5 {%k1}
+; X64-NEXT:    vaddpd %xmm5, %xmm3, %xmm3
+; X64-NEXT:    vfixupimmsd $6, %xmm2, %xmm1, %xmm0
+; X64-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: test_int_x86_avx512_mask_fixupimm_sd:
@@ -5654,16 +5666,16 @@ define <2 x double>@test_int_x86_avx512_mask_fixupimm_sd(<2 x double> %x0, <2 x
 ; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vmovapd %xmm0, %xmm3
 ; X86-NEXT:    vfixupimmsd $5, %xmm2, %xmm1, %xmm3 {%k1}
-; X86-NEXT:    vmovapd %xmm0, %xmm4
-; X86-NEXT:    vfixupimmsd $5, %xmm2, %xmm1, %xmm4
-; X86-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
-; X86-NEXT:    vfixupimmsd $5, {sae}, %xmm2, %xmm1, %xmm0 {%k1}
+; X86-NEXT:    vxorpd %xmm4, %xmm4, %xmm4
+; X86-NEXT:    vmovapd %xmm0, %xmm5
+; X86-NEXT:    vfixupimmsd $5, {sae}, %xmm4, %xmm1, %xmm5 {%k1}
+; X86-NEXT:    vaddpd %xmm5, %xmm3, %xmm3
+; X86-NEXT:    vfixupimmsd $6, %xmm2, %xmm1, %xmm0
 ; X86-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
-; X86-NEXT:    vaddpd %xmm4, %xmm0, %xmm0
 ; X86-NEXT:    retl
   %res = call <2 x double> @llvm.x86.avx512.mask.fixupimm.sd(<2 x double> %x0, <2 x double> %x1, <2 x i64> %x2, i32 5, i8 %x4, i32 4)
   %res1 = call <2 x double> @llvm.x86.avx512.mask.fixupimm.sd(<2 x double> %x0, <2 x double> %x1, <2 x i64> zeroinitializer, i32 5, i8 %x4, i32 8)
-  %res2 = call <2 x double> @llvm.x86.avx512.mask.fixupimm.sd(<2 x double> %x0, <2 x double> %x1, <2 x i64> %x2, i32 5, i8 -1, i32 4)
+  %res2 = call <2 x double> @llvm.x86.avx512.mask.fixupimm.sd(<2 x double> %x0, <2 x double> %x1, <2 x i64> %x2, i32 6, i8 -1, i32 4)
   %res3 = fadd <2 x double> %res, %res1
   %res4 = fadd <2 x double> %res3, %res2
   ret <2 x double> %res4
@@ -5709,130 +5721,106 @@ define <2 x double>@test_int_x86_avx512_maskz_fixupimm_sd(<2 x double> %x0, <2 x
 declare double @llvm.fma.f64(double, double, double) #1
 declare double @llvm.x86.avx512.vfmadd.f64(double, double, double, i32) #0
 
-define <2 x double>@test_int_x86_avx512_mask_vfmadd_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3,i32 %x4 ){
+define <2 x double> @test_int_x86_avx512_mask_vfmadd_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 %x4) {
 ; X64-LABEL: test_int_x86_avx512_mask_vfmadd_sd:
 ; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vmovapd %xmm0, %xmm3
 ; X64-NEXT:    vfmadd213sd {{.*#+}} xmm3 = (xmm1 * xmm3) + xmm2
-; X64-NEXT:    kmovw %edi, %k1
-; X64-NEXT:    vmovapd %xmm0, %xmm4
-; X64-NEXT:    vfmadd213sd {{.*#+}} xmm4 = (xmm1 * xmm4) + xmm2
-; X64-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
 ; X64-NEXT:    vmovapd %xmm0, %xmm4
 ; X64-NEXT:    vfmadd213sd {rz-sae}, %xmm2, %xmm1, %xmm4
-; X64-NEXT:    vfmadd213sd {rz-sae}, %xmm2, %xmm1, %xmm0 {%k1}
-; X64-NEXT:    vaddpd %xmm0, %xmm4, %xmm0
-; X64-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; X64-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
+; X64-NEXT:    vfmadd213sd {ru-sae}, %xmm2, %xmm1, %xmm0 {%k1}
+; X64-NEXT:    vaddpd %xmm3, %xmm0, %xmm0
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: test_int_x86_avx512_mask_vfmadd_sd:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vmovapd %xmm0, %xmm3
 ; X86-NEXT:    vfmadd213sd {{.*#+}} xmm3 = (xmm1 * xmm3) + xmm2
-; X86-NEXT:    kmovw %eax, %k1
-; X86-NEXT:    vmovapd %xmm0, %xmm4
-; X86-NEXT:    vfmadd213sd {{.*#+}} xmm4 = (xmm1 * xmm4) + xmm2
-; X86-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
 ; X86-NEXT:    vmovapd %xmm0, %xmm4
 ; X86-NEXT:    vfmadd213sd {rz-sae}, %xmm2, %xmm1, %xmm4
-; X86-NEXT:    vfmadd213sd {rz-sae}, %xmm2, %xmm1, %xmm0 {%k1}
-; X86-NEXT:    vaddpd %xmm0, %xmm4, %xmm0
-; X86-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; X86-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
+; X86-NEXT:    vfmadd213sd {ru-sae}, %xmm2, %xmm1, %xmm0 {%k1}
+; X86-NEXT:    vaddpd %xmm3, %xmm0, %xmm0
 ; X86-NEXT:    retl
   %1 = extractelement <2 x double> %x0, i64 0
   %2 = extractelement <2 x double> %x1, i64 0
   %3 = extractelement <2 x double> %x2, i64 0
   %4 = call double @llvm.fma.f64(double %1, double %2, double %3)
-  %5 = insertelement <2 x double> %x0, double %4, i64 0
-  %6 = extractelement <2 x double> %x0, i64 0
-  %7 = extractelement <2 x double> %x1, i64 0
-  %8 = extractelement <2 x double> %x2, i64 0
-  %9 = call double @llvm.fma.f64(double %6, double %7, double %8)
-  %10 = bitcast i8 %x3 to <8 x i1>
-  %11 = extractelement <8 x i1> %10, i64 0
-  %12 = select i1 %11, double %9, double %6
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = extractelement <8 x i1> %5, i64 0
+  %7 = select i1 %6, double %4, double %1
+  %8 = insertelement <2 x double> %x0, double %7, i64 0
+  %9 = extractelement <2 x double> %x0, i64 0
+  %10 = extractelement <2 x double> %x1, i64 0
+  %11 = extractelement <2 x double> %x2, i64 0
+  %12 = call double @llvm.x86.avx512.vfmadd.f64(double %9, double %10, double %11, i32 11)
   %13 = insertelement <2 x double> %x0, double %12, i64 0
   %14 = extractelement <2 x double> %x0, i64 0
   %15 = extractelement <2 x double> %x1, i64 0
   %16 = extractelement <2 x double> %x2, i64 0
-  %17 = call double @llvm.x86.avx512.vfmadd.f64(double %14, double %15, double %16, i32 11)
-  %18 = insertelement <2 x double> %x0, double %17, i64 0
-  %19 = extractelement <2 x double> %x0, i64 0
-  %20 = extractelement <2 x double> %x1, i64 0
-  %21 = extractelement <2 x double> %x2, i64 0
-  %22 = call double @llvm.x86.avx512.vfmadd.f64(double %19, double %20, double %21, i32 11)
-  %23 = bitcast i8 %x3 to <8 x i1>
-  %24 = extractelement <8 x i1> %23, i64 0
-  %25 = select i1 %24, double %22, double %19
-  %26 = insertelement <2 x double> %x0, double %25, i64 0
-  %res4 = fadd <2 x double> %5, %13
-  %res5 = fadd <2 x double> %18, %26
-  %res6 = fadd <2 x double> %res4, %res5
-  ret <2 x double> %res6
-}
-
-define <4 x float>@test_int_x86_avx512_mask_vfmadd_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3,i32 %x4 ){
+  %17 = call double @llvm.x86.avx512.vfmadd.f64(double %14, double %15, double %16, i32 10)
+  %18 = bitcast i8 %x3 to <8 x i1>
+  %19 = extractelement <8 x i1> %18, i64 0
+  %20 = select i1 %19, double %17, double %14
+  %21 = insertelement <2 x double> %x0, double %20, i64 0
+  %res3 = fadd <2 x double> %8, %13
+  %res4 = fadd <2 x double> %21, %res3
+  ret <2 x double> %res4
+}
+
+define <4 x float> @test_int_x86_avx512_mask_vfmadd_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 %x4) {
 ; X64-LABEL: test_int_x86_avx512_mask_vfmadd_ss:
 ; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vmovaps %xmm0, %xmm3
 ; X64-NEXT:    vfmadd213ss {{.*#+}} xmm3 = (xmm1 * xmm3) + xmm2
-; X64-NEXT:    kmovw %edi, %k1
-; X64-NEXT:    vmovaps %xmm0, %xmm4
-; X64-NEXT:    vfmadd213ss {{.*#+}} xmm4 = (xmm1 * xmm4) + xmm2
-; X64-NEXT:    vaddps %xmm4, %xmm3, %xmm3
 ; X64-NEXT:    vmovaps %xmm0, %xmm4
 ; X64-NEXT:    vfmadd213ss {rz-sae}, %xmm2, %xmm1, %xmm4
-; X64-NEXT:    vfmadd213ss {rz-sae}, %xmm2, %xmm1, %xmm0 {%k1}
-; X64-NEXT:    vaddps %xmm0, %xmm4, %xmm0
-; X64-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; X64-NEXT:    vaddps %xmm4, %xmm3, %xmm3
+; X64-NEXT:    vfmadd213ss {ru-sae}, %xmm2, %xmm1, %xmm0 {%k1}
+; X64-NEXT:    vaddps %xmm3, %xmm0, %xmm0
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: test_int_x86_avx512_mask_vfmadd_ss:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vmovaps %xmm0, %xmm3
 ; X86-NEXT:    vfmadd213ss {{.*#+}} xmm3 = (xmm1 * xmm3) + xmm2
-; X86-NEXT:    kmovw %eax, %k1
-; X86-NEXT:    vmovaps %xmm0, %xmm4
-; X86-NEXT:    vfmadd213ss {{.*#+}} xmm4 = (xmm1 * xmm4) + xmm2
-; X86-NEXT:    vaddps %xmm4, %xmm3, %xmm3
 ; X86-NEXT:    vmovaps %xmm0, %xmm4
 ; X86-NEXT:    vfmadd213ss {rz-sae}, %xmm2, %xmm1, %xmm4
-; X86-NEXT:    vfmadd213ss {rz-sae}, %xmm2, %xmm1, %xmm0 {%k1}
-; X86-NEXT:    vaddps %xmm0, %xmm4, %xmm0
-; X86-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; X86-NEXT:    vaddps %xmm4, %xmm3, %xmm3
+; X86-NEXT:    vfmadd213ss {ru-sae}, %xmm2, %xmm1, %xmm0 {%k1}
+; X86-NEXT:    vaddps %xmm3, %xmm0, %xmm0
 ; X86-NEXT:    retl
   %1 = extractelement <4 x float> %x0, i64 0
   %2 = extractelement <4 x float> %x1, i64 0
   %3 = extractelement <4 x float> %x2, i64 0
   %4 = call float @llvm.fma.f32(float %1, float %2, float %3)
-  %5 = insertelement <4 x float> %x0, float %4, i64 0
-  %6 = extractelement <4 x float> %x0, i64 0
-  %7 = extractelement <4 x float> %x1, i64 0
-  %8 = extractelement <4 x float> %x2, i64 0
-  %9 = call float @llvm.fma.f32(float %6, float %7, float %8)
-  %10 = bitcast i8 %x3 to <8 x i1>
-  %11 = extractelement <8 x i1> %10, i64 0
-  %12 = select i1 %11, float %9, float %6
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = extractelement <8 x i1> %5, i64 0
+  %7 = select i1 %6, float %4, float %1
+  %8 = insertelement <4 x float> %x0, float %7, i64 0
+  %9 = extractelement <4 x float> %x0, i64 0
+  %10 = extractelement <4 x float> %x1, i64 0
+  %11 = extractelement <4 x float> %x2, i64 0
+  %12 = call float @llvm.x86.avx512.vfmadd.f32(float %9, float %10, float %11, i32 11)
   %13 = insertelement <4 x float> %x0, float %12, i64 0
   %14 = extractelement <4 x float> %x0, i64 0
   %15 = extractelement <4 x float> %x1, i64 0
   %16 = extractelement <4 x float> %x2, i64 0
-  %17 = call float @llvm.x86.avx512.vfmadd.f32(float %14, float %15, float %16, i32 11)
-  %18 = insertelement <4 x float> %x0, float %17, i64 0
-  %19 = extractelement <4 x float> %x0, i64 0
-  %20 = extractelement <4 x float> %x1, i64 0
-  %21 = extractelement <4 x float> %x2, i64 0
-  %22 = call float @llvm.x86.avx512.vfmadd.f32(float %19, float %20, float %21, i32 11)
-  %23 = bitcast i8 %x3 to <8 x i1>
-  %24 = extractelement <8 x i1> %23, i64 0
-  %25 = select i1 %24, float %22, float %19
-  %26 = insertelement <4 x float> %x0, float %25, i64 0
-  %res4 = fadd <4 x float> %5, %13
-  %res5 = fadd <4 x float> %18, %26
-  %res6 = fadd <4 x float> %res4, %res5
-  ret <4 x float> %res6
+  %17 = call float @llvm.x86.avx512.vfmadd.f32(float %14, float %15, float %16, i32 10)
+  %18 = bitcast i8 %x3 to <8 x i1>
+  %19 = extractelement <8 x i1> %18, i64 0
+  %20 = select i1 %19, float %17, float %14
+  %21 = insertelement <4 x float> %x0, float %20, i64 0
+  %res3 = fadd <4 x float> %8, %13
+  %res4 = fadd <4 x float> %21, %res3
+  ret <4 x float> %res4
 }
 
 define <2 x double>@test_int_x86_avx512_maskz_vfmadd_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3,i32 %x4 ){
@@ -5945,130 +5933,106 @@ define <4 x float> @test_int_x86_avx512_maskz_vfmadd_ss_load0(i8 zeroext %0, <4
   ret <4 x float> %11
 }
 
-define <2 x double>@test_int_x86_avx512_mask3_vfmadd_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3,i32 %x4 ){
+define <2 x double> @test_int_x86_avx512_mask3_vfmadd_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 %x4) {
 ; X64-LABEL: test_int_x86_avx512_mask3_vfmadd_sd:
 ; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vmovapd %xmm2, %xmm3
 ; X64-NEXT:    vfmadd231sd {{.*#+}} xmm3 = (xmm0 * xmm1) + xmm3
-; X64-NEXT:    kmovw %edi, %k1
-; X64-NEXT:    vmovapd %xmm2, %xmm4
-; X64-NEXT:    vfmadd231sd {{.*#+}} xmm4 = (xmm0 * xmm1) + xmm4
-; X64-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
 ; X64-NEXT:    vmovapd %xmm2, %xmm4
 ; X64-NEXT:    vfmadd231sd {rz-sae}, %xmm1, %xmm0, %xmm4
-; X64-NEXT:    vfmadd231sd {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; X64-NEXT:    vaddpd %xmm2, %xmm4, %xmm0
-; X64-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; X64-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
+; X64-NEXT:    vfmadd231sd {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vaddpd %xmm3, %xmm2, %xmm0
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: test_int_x86_avx512_mask3_vfmadd_sd:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vmovapd %xmm2, %xmm3
 ; X86-NEXT:    vfmadd231sd {{.*#+}} xmm3 = (xmm0 * xmm1) + xmm3
-; X86-NEXT:    kmovw %eax, %k1
-; X86-NEXT:    vmovapd %xmm2, %xmm4
-; X86-NEXT:    vfmadd231sd {{.*#+}} xmm4 = (xmm0 * xmm1) + xmm4
-; X86-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
 ; X86-NEXT:    vmovapd %xmm2, %xmm4
 ; X86-NEXT:    vfmadd231sd {rz-sae}, %xmm1, %xmm0, %xmm4
-; X86-NEXT:    vfmadd231sd {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; X86-NEXT:    vaddpd %xmm2, %xmm4, %xmm0
-; X86-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; X86-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
+; X86-NEXT:    vfmadd231sd {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vaddpd %xmm3, %xmm2, %xmm0
 ; X86-NEXT:    retl
   %1 = extractelement <2 x double> %x0, i64 0
   %2 = extractelement <2 x double> %x1, i64 0
   %3 = extractelement <2 x double> %x2, i64 0
   %4 = call double @llvm.fma.f64(double %1, double %2, double %3)
-  %5 = insertelement <2 x double> %x2, double %4, i64 0
-  %6 = extractelement <2 x double> %x0, i64 0
-  %7 = extractelement <2 x double> %x1, i64 0
-  %8 = extractelement <2 x double> %x2, i64 0
-  %9 = call double @llvm.fma.f64(double %6, double %7, double %8)
-  %10 = bitcast i8 %x3 to <8 x i1>
-  %11 = extractelement <8 x i1> %10, i64 0
-  %12 = select i1 %11, double %9, double %8
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = extractelement <8 x i1> %5, i64 0
+  %7 = select i1 %6, double %4, double %3
+  %8 = insertelement <2 x double> %x2, double %7, i64 0
+  %9 = extractelement <2 x double> %x0, i64 0
+  %10 = extractelement <2 x double> %x1, i64 0
+  %11 = extractelement <2 x double> %x2, i64 0
+  %12 = call double @llvm.x86.avx512.vfmadd.f64(double %9, double %10, double %11, i32 11)
   %13 = insertelement <2 x double> %x2, double %12, i64 0
   %14 = extractelement <2 x double> %x0, i64 0
   %15 = extractelement <2 x double> %x1, i64 0
   %16 = extractelement <2 x double> %x2, i64 0
-  %17 = call double @llvm.x86.avx512.vfmadd.f64(double %14, double %15, double %16, i32 11)
-  %18 = insertelement <2 x double> %x2, double %17, i64 0
-  %19 = extractelement <2 x double> %x0, i64 0
-  %20 = extractelement <2 x double> %x1, i64 0
-  %21 = extractelement <2 x double> %x2, i64 0
-  %22 = call double @llvm.x86.avx512.vfmadd.f64(double %19, double %20, double %21, i32 11)
-  %23 = bitcast i8 %x3 to <8 x i1>
-  %24 = extractelement <8 x i1> %23, i64 0
-  %25 = select i1 %24, double %22, double %21
-  %26 = insertelement <2 x double> %x2, double %25, i64 0
-  %res4 = fadd <2 x double> %5, %13
-  %res5 = fadd <2 x double> %18, %26
-  %res6 = fadd <2 x double> %res4, %res5
-  ret <2 x double> %res6
-}
-
-define <4 x float>@test_int_x86_avx512_mask3_vfmadd_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3,i32 %x4 ){
+  %17 = call double @llvm.x86.avx512.vfmadd.f64(double %14, double %15, double %16, i32 10)
+  %18 = bitcast i8 %x3 to <8 x i1>
+  %19 = extractelement <8 x i1> %18, i64 0
+  %20 = select i1 %19, double %17, double %16
+  %21 = insertelement <2 x double> %x2, double %20, i64 0
+  %res3 = fadd <2 x double> %8, %13
+  %res4 = fadd <2 x double> %21, %res3
+  ret <2 x double> %res4
+}
+
+define <4 x float> @test_int_x86_avx512_mask3_vfmadd_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 %x4) {
 ; X64-LABEL: test_int_x86_avx512_mask3_vfmadd_ss:
 ; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vmovaps %xmm2, %xmm3
 ; X64-NEXT:    vfmadd231ss {{.*#+}} xmm3 = (xmm0 * xmm1) + xmm3
-; X64-NEXT:    kmovw %edi, %k1
-; X64-NEXT:    vmovaps %xmm2, %xmm4
-; X64-NEXT:    vfmadd231ss {{.*#+}} xmm4 = (xmm0 * xmm1) + xmm4
-; X64-NEXT:    vaddps %xmm4, %xmm3, %xmm3
 ; X64-NEXT:    vmovaps %xmm2, %xmm4
 ; X64-NEXT:    vfmadd231ss {rz-sae}, %xmm1, %xmm0, %xmm4
-; X64-NEXT:    vfmadd231ss {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; X64-NEXT:    vaddps %xmm2, %xmm4, %xmm0
-; X64-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; X64-NEXT:    vaddps %xmm4, %xmm3, %xmm3
+; X64-NEXT:    vfmadd231ss {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vaddps %xmm3, %xmm2, %xmm0
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: test_int_x86_avx512_mask3_vfmadd_ss:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vmovaps %xmm2, %xmm3
 ; X86-NEXT:    vfmadd231ss {{.*#+}} xmm3 = (xmm0 * xmm1) + xmm3
-; X86-NEXT:    kmovw %eax, %k1
-; X86-NEXT:    vmovaps %xmm2, %xmm4
-; X86-NEXT:    vfmadd231ss {{.*#+}} xmm4 = (xmm0 * xmm1) + xmm4
-; X86-NEXT:    vaddps %xmm4, %xmm3, %xmm3
 ; X86-NEXT:    vmovaps %xmm2, %xmm4
 ; X86-NEXT:    vfmadd231ss {rz-sae}, %xmm1, %xmm0, %xmm4
-; X86-NEXT:    vfmadd231ss {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; X86-NEXT:    vaddps %xmm2, %xmm4, %xmm0
-; X86-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; X86-NEXT:    vaddps %xmm4, %xmm3, %xmm3
+; X86-NEXT:    vfmadd231ss {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vaddps %xmm3, %xmm2, %xmm0
 ; X86-NEXT:    retl
   %1 = extractelement <4 x float> %x0, i64 0
   %2 = extractelement <4 x float> %x1, i64 0
   %3 = extractelement <4 x float> %x2, i64 0
   %4 = call float @llvm.fma.f32(float %1, float %2, float %3)
-  %5 = insertelement <4 x float> %x2, float %4, i64 0
-  %6 = extractelement <4 x float> %x0, i64 0
-  %7 = extractelement <4 x float> %x1, i64 0
-  %8 = extractelement <4 x float> %x2, i64 0
-  %9 = call float @llvm.fma.f32(float %6, float %7, float %8)
-  %10 = bitcast i8 %x3 to <8 x i1>
-  %11 = extractelement <8 x i1> %10, i64 0
-  %12 = select i1 %11, float %9, float %8
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = extractelement <8 x i1> %5, i64 0
+  %7 = select i1 %6, float %4, float %3
+  %8 = insertelement <4 x float> %x2, float %7, i64 0
+  %9 = extractelement <4 x float> %x0, i64 0
+  %10 = extractelement <4 x float> %x1, i64 0
+  %11 = extractelement <4 x float> %x2, i64 0
+  %12 = call float @llvm.x86.avx512.vfmadd.f32(float %9, float %10, float %11, i32 11)
   %13 = insertelement <4 x float> %x2, float %12, i64 0
   %14 = extractelement <4 x float> %x0, i64 0
   %15 = extractelement <4 x float> %x1, i64 0
   %16 = extractelement <4 x float> %x2, i64 0
-  %17 = call float @llvm.x86.avx512.vfmadd.f32(float %14, float %15, float %16, i32 11)
-  %18 = insertelement <4 x float> %x2, float %17, i64 0
-  %19 = extractelement <4 x float> %x0, i64 0
-  %20 = extractelement <4 x float> %x1, i64 0
-  %21 = extractelement <4 x float> %x2, i64 0
-  %22 = call float @llvm.x86.avx512.vfmadd.f32(float %19, float %20, float %21, i32 11)
-  %23 = bitcast i8 %x3 to <8 x i1>
-  %24 = extractelement <8 x i1> %23, i64 0
-  %25 = select i1 %24, float %22, float %21
-  %26 = insertelement <4 x float> %x2, float %25, i64 0
-  %res4 = fadd <4 x float> %5, %13
-  %res5 = fadd <4 x float> %18, %26
-  %res6 = fadd <4 x float> %res4, %res5
-  ret <4 x float> %res6
+  %17 = call float @llvm.x86.avx512.vfmadd.f32(float %14, float %15, float %16, i32 10)
+  %18 = bitcast i8 %x3 to <8 x i1>
+  %19 = extractelement <8 x i1> %18, i64 0
+  %20 = select i1 %19, float %17, float %16
+  %21 = insertelement <4 x float> %x2, float %20, i64 0
+  %res3 = fadd <4 x float> %8, %13
+  %res4 = fadd <4 x float> %21, %res3
+  ret <4 x float> %res4
 }
 
 define void @fmadd_ss_mask_memfold(float* %a, float* %b, i8 %c) {
@@ -6247,296 +6211,238 @@ define void @fmadd_sd_maskz_memfold(double* %a, double* %b, i8 %c) {
   ret void
 }
 
-define <2 x double>@test_int_x86_avx512_mask3_vfmsub_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3,i32 %x4 ){
+define <2 x double> @test_int_x86_avx512_mask3_vfmsub_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 %x4) {
 ; X64-LABEL: test_int_x86_avx512_mask3_vfmsub_sd:
 ; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vmovapd %xmm2, %xmm3
 ; X64-NEXT:    vfmsub231sd {{.*#+}} xmm3 = (xmm0 * xmm1) - xmm3
-; X64-NEXT:    kmovw %edi, %k1
-; X64-NEXT:    vmovapd %xmm2, %xmm4
-; X64-NEXT:    vfmsub231sd {{.*#+}} xmm4 = (xmm0 * xmm1) - xmm4
-; X64-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
 ; X64-NEXT:    vmovapd %xmm2, %xmm4
 ; X64-NEXT:    vfmsub231sd {rz-sae}, %xmm1, %xmm0, %xmm4
-; X64-NEXT:    vfmsub231sd {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; X64-NEXT:    vaddpd %xmm2, %xmm4, %xmm0
-; X64-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; X64-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
+; X64-NEXT:    vfmsub231sd {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vaddpd %xmm3, %xmm2, %xmm0
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: test_int_x86_avx512_mask3_vfmsub_sd:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vmovapd %xmm2, %xmm3
 ; X86-NEXT:    vfmsub231sd {{.*#+}} xmm3 = (xmm0 * xmm1) - xmm3
-; X86-NEXT:    kmovw %eax, %k1
-; X86-NEXT:    vmovapd %xmm2, %xmm4
-; X86-NEXT:    vfmsub231sd {{.*#+}} xmm4 = (xmm0 * xmm1) - xmm4
-; X86-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
 ; X86-NEXT:    vmovapd %xmm2, %xmm4
 ; X86-NEXT:    vfmsub231sd {rz-sae}, %xmm1, %xmm0, %xmm4
-; X86-NEXT:    vfmsub231sd {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; X86-NEXT:    vaddpd %xmm2, %xmm4, %xmm0
-; X86-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; X86-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
+; X86-NEXT:    vfmsub231sd {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vaddpd %xmm3, %xmm2, %xmm0
 ; X86-NEXT:    retl
-  %1 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %x2
+  %1 = fneg <2 x double> %x2
   %2 = extractelement <2 x double> %x0, i64 0
   %3 = extractelement <2 x double> %x1, i64 0
   %4 = extractelement <2 x double> %1, i64 0
   %5 = call double @llvm.fma.f64(double %2, double %3, double %4)
   %6 = extractelement <2 x double> %x2, i64 0
-  %7 = insertelement <2 x double> %x2, double %5, i64 0
-  %8 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %x2
-  %9 = extractelement <2 x double> %x0, i64 0
-  %10 = extractelement <2 x double> %x1, i64 0
-  %11 = extractelement <2 x double> %8, i64 0
-  %12 = call double @llvm.fma.f64(double %9, double %10, double %11)
-  %13 = extractelement <2 x double> %x2, i64 0
-  %14 = bitcast i8 %x3 to <8 x i1>
-  %15 = extractelement <8 x i1> %14, i64 0
-  %16 = select i1 %15, double %12, double %13
-  %17 = insertelement <2 x double> %x2, double %16, i64 0
-  %18 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %x2
+  %7 = bitcast i8 %x3 to <8 x i1>
+  %8 = extractelement <8 x i1> %7, i64 0
+  %9 = select i1 %8, double %5, double %6
+  %10 = insertelement <2 x double> %x2, double %9, i64 0
+  %11 = fneg <2 x double> %x2
+  %12 = extractelement <2 x double> %x0, i64 0
+  %13 = extractelement <2 x double> %x1, i64 0
+  %14 = extractelement <2 x double> %11, i64 0
+  %15 = call double @llvm.x86.avx512.vfmadd.f64(double %12, double %13, double %14, i32 11)
+  %16 = extractelement <2 x double> %x2, i64 0
+  %17 = insertelement <2 x double> %x2, double %15, i64 0
+  %18 = fneg <2 x double> %x2
   %19 = extractelement <2 x double> %x0, i64 0
   %20 = extractelement <2 x double> %x1, i64 0
   %21 = extractelement <2 x double> %18, i64 0
-  %22 = call double @llvm.x86.avx512.vfmadd.f64(double %19, double %20, double %21, i32 11)
+  %22 = call double @llvm.x86.avx512.vfmadd.f64(double %19, double %20, double %21, i32 10)
   %23 = extractelement <2 x double> %x2, i64 0
-  %24 = insertelement <2 x double> %x2, double %22, i64 0
-  %25 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %x2
-  %26 = extractelement <2 x double> %x0, i64 0
-  %27 = extractelement <2 x double> %x1, i64 0
-  %28 = extractelement <2 x double> %25, i64 0
-  %29 = call double @llvm.x86.avx512.vfmadd.f64(double %26, double %27, double %28, i32 11)
-  %30 = extractelement <2 x double> %x2, i64 0
-  %31 = bitcast i8 %x3 to <8 x i1>
-  %32 = extractelement <8 x i1> %31, i64 0
-  %33 = select i1 %32, double %29, double %30
-  %34 = insertelement <2 x double> %x2, double %33, i64 0
-  %res4 = fadd <2 x double> %7, %17
-  %res5 = fadd <2 x double> %24, %34
-  %res6 = fadd <2 x double> %res4, %res5
-  ret <2 x double> %res6
-}
-
-define <4 x float>@test_int_x86_avx512_mask3_vfmsub_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3,i32 %x4 ){
+  %24 = bitcast i8 %x3 to <8 x i1>
+  %25 = extractelement <8 x i1> %24, i64 0
+  %26 = select i1 %25, double %22, double %23
+  %27 = insertelement <2 x double> %x2, double %26, i64 0
+  %res3 = fadd <2 x double> %10, %17
+  %res4 = fadd <2 x double> %27, %res3
+  ret <2 x double> %res4
+}
+
+define <4 x float> @test_int_x86_avx512_mask3_vfmsub_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 %x4) {
 ; X64-LABEL: test_int_x86_avx512_mask3_vfmsub_ss:
 ; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vmovaps %xmm2, %xmm3
 ; X64-NEXT:    vfmsub231ss {{.*#+}} xmm3 = (xmm0 * xmm1) - xmm3
-; X64-NEXT:    kmovw %edi, %k1
-; X64-NEXT:    vmovaps %xmm2, %xmm4
-; X64-NEXT:    vfmsub231ss {{.*#+}} xmm4 = (xmm0 * xmm1) - xmm4
-; X64-NEXT:    vaddps %xmm4, %xmm3, %xmm3
 ; X64-NEXT:    vmovaps %xmm2, %xmm4
 ; X64-NEXT:    vfmsub231ss {rz-sae}, %xmm1, %xmm0, %xmm4
-; X64-NEXT:    vfmsub231ss {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; X64-NEXT:    vaddps %xmm2, %xmm4, %xmm0
-; X64-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; X64-NEXT:    vaddps %xmm4, %xmm3, %xmm3
+; X64-NEXT:    vfmsub231ss {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vaddps %xmm3, %xmm2, %xmm0
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: test_int_x86_avx512_mask3_vfmsub_ss:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vmovaps %xmm2, %xmm3
 ; X86-NEXT:    vfmsub231ss {{.*#+}} xmm3 = (xmm0 * xmm1) - xmm3
-; X86-NEXT:    kmovw %eax, %k1
-; X86-NEXT:    vmovaps %xmm2, %xmm4
-; X86-NEXT:    vfmsub231ss {{.*#+}} xmm4 = (xmm0 * xmm1) - xmm4
-; X86-NEXT:    vaddps %xmm4, %xmm3, %xmm3
 ; X86-NEXT:    vmovaps %xmm2, %xmm4
 ; X86-NEXT:    vfmsub231ss {rz-sae}, %xmm1, %xmm0, %xmm4
-; X86-NEXT:    vfmsub231ss {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; X86-NEXT:    vaddps %xmm2, %xmm4, %xmm0
-; X86-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; X86-NEXT:    vaddps %xmm4, %xmm3, %xmm3
+; X86-NEXT:    vfmsub231ss {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vaddps %xmm3, %xmm2, %xmm0
 ; X86-NEXT:    retl
-  %1 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %x2
+  %1 = fneg <4 x float> %x2
   %2 = extractelement <4 x float> %x0, i64 0
   %3 = extractelement <4 x float> %x1, i64 0
   %4 = extractelement <4 x float> %1, i64 0
   %5 = call float @llvm.fma.f32(float %2, float %3, float %4)
   %6 = extractelement <4 x float> %x2, i64 0
-  %7 = insertelement <4 x float> %x2, float %5, i64 0
-  %8 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %x2
-  %9 = extractelement <4 x float> %x0, i64 0
-  %10 = extractelement <4 x float> %x1, i64 0
-  %11 = extractelement <4 x float> %8, i64 0
-  %12 = call float @llvm.fma.f32(float %9, float %10, float %11)
-  %13 = extractelement <4 x float> %x2, i64 0
-  %14 = bitcast i8 %x3 to <8 x i1>
-  %15 = extractelement <8 x i1> %14, i64 0
-  %16 = select i1 %15, float %12, float %13
-  %17 = insertelement <4 x float> %x2, float %16, i64 0
-  %18 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %x2
+  %7 = bitcast i8 %x3 to <8 x i1>
+  %8 = extractelement <8 x i1> %7, i64 0
+  %9 = select i1 %8, float %5, float %6
+  %10 = insertelement <4 x float> %x2, float %9, i64 0
+  %11 = fneg <4 x float> %x2
+  %12 = extractelement <4 x float> %x0, i64 0
+  %13 = extractelement <4 x float> %x1, i64 0
+  %14 = extractelement <4 x float> %11, i64 0
+  %15 = call float @llvm.x86.avx512.vfmadd.f32(float %12, float %13, float %14, i32 11)
+  %16 = extractelement <4 x float> %x2, i64 0
+  %17 = insertelement <4 x float> %x2, float %15, i64 0
+  %18 = fneg <4 x float> %x2
   %19 = extractelement <4 x float> %x0, i64 0
   %20 = extractelement <4 x float> %x1, i64 0
   %21 = extractelement <4 x float> %18, i64 0
-  %22 = call float @llvm.x86.avx512.vfmadd.f32(float %19, float %20, float %21, i32 11)
+  %22 = call float @llvm.x86.avx512.vfmadd.f32(float %19, float %20, float %21, i32 10)
   %23 = extractelement <4 x float> %x2, i64 0
-  %24 = insertelement <4 x float> %x2, float %22, i64 0
-  %25 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %x2
-  %26 = extractelement <4 x float> %x0, i64 0
-  %27 = extractelement <4 x float> %x1, i64 0
-  %28 = extractelement <4 x float> %25, i64 0
-  %29 = call float @llvm.x86.avx512.vfmadd.f32(float %26, float %27, float %28, i32 11)
-  %30 = extractelement <4 x float> %x2, i64 0
-  %31 = bitcast i8 %x3 to <8 x i1>
-  %32 = extractelement <8 x i1> %31, i64 0
-  %33 = select i1 %32, float %29, float %30
-  %34 = insertelement <4 x float> %x2, float %33, i64 0
-  %res4 = fadd <4 x float> %7, %17
-  %res5 = fadd <4 x float> %24, %34
-  %res6 = fadd <4 x float> %res4, %res5
-  ret <4 x float> %res6
-}
-
-define <2 x double>@test_int_x86_avx512_mask3_vfnmsub_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3,i32 %x4 ){
+  %24 = bitcast i8 %x3 to <8 x i1>
+  %25 = extractelement <8 x i1> %24, i64 0
+  %26 = select i1 %25, float %22, float %23
+  %27 = insertelement <4 x float> %x2, float %26, i64 0
+  %res3 = fadd <4 x float> %10, %17
+  %res4 = fadd <4 x float> %27, %res3
+  ret <4 x float> %res4
+}
+
+define <2 x double> @test_int_x86_avx512_mask3_vfnmsub_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3, i32 %x4) {
 ; X64-LABEL: test_int_x86_avx512_mask3_vfnmsub_sd:
 ; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vmovapd %xmm2, %xmm3
 ; X64-NEXT:    vfnmsub231sd {{.*#+}} xmm3 = -(xmm0 * xmm1) - xmm3
-; X64-NEXT:    kmovw %edi, %k1
-; X64-NEXT:    vmovapd %xmm2, %xmm4
-; X64-NEXT:    vfnmsub231sd {{.*#+}} xmm4 = -(xmm0 * xmm1) - xmm4
-; X64-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
 ; X64-NEXT:    vmovapd %xmm2, %xmm4
 ; X64-NEXT:    vfnmsub231sd {rz-sae}, %xmm1, %xmm0, %xmm4
-; X64-NEXT:    vfnmsub231sd {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; X64-NEXT:    vaddpd %xmm2, %xmm4, %xmm0
-; X64-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; X64-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
+; X64-NEXT:    vfnmsub231sd {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vaddpd %xmm3, %xmm2, %xmm0
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: test_int_x86_avx512_mask3_vfnmsub_sd:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vmovapd %xmm2, %xmm3
 ; X86-NEXT:    vfnmsub231sd {{.*#+}} xmm3 = -(xmm0 * xmm1) - xmm3
-; X86-NEXT:    kmovw %eax, %k1
-; X86-NEXT:    vmovapd %xmm2, %xmm4
-; X86-NEXT:    vfnmsub231sd {{.*#+}} xmm4 = -(xmm0 * xmm1) - xmm4
-; X86-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
 ; X86-NEXT:    vmovapd %xmm2, %xmm4
 ; X86-NEXT:    vfnmsub231sd {rz-sae}, %xmm1, %xmm0, %xmm4
-; X86-NEXT:    vfnmsub231sd {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; X86-NEXT:    vaddpd %xmm2, %xmm4, %xmm0
-; X86-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; X86-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
+; X86-NEXT:    vfnmsub231sd {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vaddpd %xmm3, %xmm2, %xmm0
 ; X86-NEXT:    retl
-  %1 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %x0
-  %2 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %x2
+  %1 = fneg <2 x double> %x0
+  %2 = fneg <2 x double> %x2
   %3 = extractelement <2 x double> %1, i64 0
   %4 = extractelement <2 x double> %x1, i64 0
   %5 = extractelement <2 x double> %2, i64 0
   %6 = call double @llvm.fma.f64(double %3, double %4, double %5)
   %7 = extractelement <2 x double> %x2, i64 0
-  %8 = insertelement <2 x double> %x2, double %6, i64 0
-  %9 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %x0
-  %10 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %x2
-  %11 = extractelement <2 x double> %9, i64 0
-  %12 = extractelement <2 x double> %x1, i64 0
-  %13 = extractelement <2 x double> %10, i64 0
-  %14 = call double @llvm.fma.f64(double %11, double %12, double %13)
-  %15 = extractelement <2 x double> %x2, i64 0
-  %16 = bitcast i8 %x3 to <8 x i1>
-  %17 = extractelement <8 x i1> %16, i64 0
-  %18 = select i1 %17, double %14, double %15
-  %19 = insertelement <2 x double> %x2, double %18, i64 0
-  %20 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %x0
-  %21 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %x2
+  %8 = bitcast i8 %x3 to <8 x i1>
+  %9 = extractelement <8 x i1> %8, i64 0
+  %10 = select i1 %9, double %6, double %7
+  %11 = insertelement <2 x double> %x2, double %10, i64 0
+  %12 = fneg <2 x double> %x0
+  %13 = fneg <2 x double> %x2
+  %14 = extractelement <2 x double> %12, i64 0
+  %15 = extractelement <2 x double> %x1, i64 0
+  %16 = extractelement <2 x double> %13, i64 0
+  %17 = call double @llvm.x86.avx512.vfmadd.f64(double %14, double %15, double %16, i32 11)
+  %18 = extractelement <2 x double> %x2, i64 0
+  %19 = insertelement <2 x double> %x2, double %17, i64 0
+  %20 = fneg <2 x double> %x0
+  %21 = fneg <2 x double> %x2
   %22 = extractelement <2 x double> %20, i64 0
   %23 = extractelement <2 x double> %x1, i64 0
   %24 = extractelement <2 x double> %21, i64 0
-  %25 = call double @llvm.x86.avx512.vfmadd.f64(double %22, double %23, double %24, i32 11)
+  %25 = call double @llvm.x86.avx512.vfmadd.f64(double %22, double %23, double %24, i32 10)
   %26 = extractelement <2 x double> %x2, i64 0
-  %27 = insertelement <2 x double> %x2, double %25, i64 0
-  %28 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %x0
-  %29 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %x2
-  %30 = extractelement <2 x double> %28, i64 0
-  %31 = extractelement <2 x double> %x1, i64 0
-  %32 = extractelement <2 x double> %29, i64 0
-  %33 = call double @llvm.x86.avx512.vfmadd.f64(double %30, double %31, double %32, i32 11)
-  %34 = extractelement <2 x double> %x2, i64 0
-  %35 = bitcast i8 %x3 to <8 x i1>
-  %36 = extractelement <8 x i1> %35, i64 0
-  %37 = select i1 %36, double %33, double %34
-  %38 = insertelement <2 x double> %x2, double %37, i64 0
-  %res4 = fadd <2 x double> %8, %19
-  %res5 = fadd <2 x double> %27, %38
-  %res6 = fadd <2 x double> %res4, %res5
-  ret <2 x double> %res6
-}
-
-define <4 x float>@test_int_x86_avx512_mask3_vfnmsub_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3,i32 %x4 ){
+  %27 = bitcast i8 %x3 to <8 x i1>
+  %28 = extractelement <8 x i1> %27, i64 0
+  %29 = select i1 %28, double %25, double %26
+  %30 = insertelement <2 x double> %x2, double %29, i64 0
+  %res3 = fadd <2 x double> %11, %19
+  %res4 = fadd <2 x double> %30, %res3
+  ret <2 x double> %res4
+}
+
+define <4 x float> @test_int_x86_avx512_mask3_vfnmsub_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3, i32 %x4) {
 ; X64-LABEL: test_int_x86_avx512_mask3_vfnmsub_ss:
 ; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vmovaps %xmm2, %xmm3
 ; X64-NEXT:    vfnmsub231ss {{.*#+}} xmm3 = -(xmm0 * xmm1) - xmm3
-; X64-NEXT:    kmovw %edi, %k1
-; X64-NEXT:    vmovaps %xmm2, %xmm4
-; X64-NEXT:    vfnmsub231ss {{.*#+}} xmm4 = -(xmm0 * xmm1) - xmm4
-; X64-NEXT:    vaddps %xmm4, %xmm3, %xmm3
 ; X64-NEXT:    vmovaps %xmm2, %xmm4
 ; X64-NEXT:    vfnmsub231ss {rz-sae}, %xmm1, %xmm0, %xmm4
-; X64-NEXT:    vfnmsub231ss {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; X64-NEXT:    vaddps %xmm2, %xmm4, %xmm0
-; X64-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; X64-NEXT:    vaddps %xmm4, %xmm3, %xmm3
+; X64-NEXT:    vfnmsub231ss {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vaddps %xmm3, %xmm2, %xmm0
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: test_int_x86_avx512_mask3_vfnmsub_ss:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vmovaps %xmm2, %xmm3
 ; X86-NEXT:    vfnmsub231ss {{.*#+}} xmm3 = -(xmm0 * xmm1) - xmm3
-; X86-NEXT:    kmovw %eax, %k1
-; X86-NEXT:    vmovaps %xmm2, %xmm4
-; X86-NEXT:    vfnmsub231ss {{.*#+}} xmm4 = -(xmm0 * xmm1) - xmm4
-; X86-NEXT:    vaddps %xmm4, %xmm3, %xmm3
 ; X86-NEXT:    vmovaps %xmm2, %xmm4
 ; X86-NEXT:    vfnmsub231ss {rz-sae}, %xmm1, %xmm0, %xmm4
-; X86-NEXT:    vfnmsub231ss {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; X86-NEXT:    vaddps %xmm2, %xmm4, %xmm0
-; X86-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; X86-NEXT:    vaddps %xmm4, %xmm3, %xmm3
+; X86-NEXT:    vfnmsub231ss {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vaddps %xmm3, %xmm2, %xmm0
 ; X86-NEXT:    retl
-  %1 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %x0
-  %2 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %x2
+  %1 = fneg <4 x float> %x0
+  %2 = fneg <4 x float> %x2
   %3 = extractelement <4 x float> %1, i64 0
   %4 = extractelement <4 x float> %x1, i64 0
   %5 = extractelement <4 x float> %2, i64 0
   %6 = call float @llvm.fma.f32(float %3, float %4, float %5)
   %7 = extractelement <4 x float> %x2, i64 0
-  %8 = insertelement <4 x float> %x2, float %6, i64 0
-  %9 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %x0
-  %10 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %x2
-  %11 = extractelement <4 x float> %9, i64 0
-  %12 = extractelement <4 x float> %x1, i64 0
-  %13 = extractelement <4 x float> %10, i64 0
-  %14 = call float @llvm.fma.f32(float %11, float %12, float %13)
-  %15 = extractelement <4 x float> %x2, i64 0
-  %16 = bitcast i8 %x3 to <8 x i1>
-  %17 = extractelement <8 x i1> %16, i64 0
-  %18 = select i1 %17, float %14, float %15
-  %19 = insertelement <4 x float> %x2, float %18, i64 0
-  %20 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %x0
-  %21 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %x2
+  %8 = bitcast i8 %x3 to <8 x i1>
+  %9 = extractelement <8 x i1> %8, i64 0
+  %10 = select i1 %9, float %6, float %7
+  %11 = insertelement <4 x float> %x2, float %10, i64 0
+  %12 = fneg <4 x float> %x0
+  %13 = fneg <4 x float> %x2
+  %14 = extractelement <4 x float> %12, i64 0
+  %15 = extractelement <4 x float> %x1, i64 0
+  %16 = extractelement <4 x float> %13, i64 0
+  %17 = call float @llvm.x86.avx512.vfmadd.f32(float %14, float %15, float %16, i32 11)
+  %18 = extractelement <4 x float> %x2, i64 0
+  %19 = insertelement <4 x float> %x2, float %17, i64 0
+  %20 = fneg <4 x float> %x0
+  %21 = fneg <4 x float> %x2
   %22 = extractelement <4 x float> %20, i64 0
   %23 = extractelement <4 x float> %x1, i64 0
   %24 = extractelement <4 x float> %21, i64 0
-  %25 = call float @llvm.x86.avx512.vfmadd.f32(float %22, float %23, float %24, i32 11)
+  %25 = call float @llvm.x86.avx512.vfmadd.f32(float %22, float %23, float %24, i32 10)
   %26 = extractelement <4 x float> %x2, i64 0
-  %27 = insertelement <4 x float> %x2, float %25, i64 0
-  %28 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %x0
-  %29 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %x2
-  %30 = extractelement <4 x float> %28, i64 0
-  %31 = extractelement <4 x float> %x1, i64 0
-  %32 = extractelement <4 x float> %29, i64 0
-  %33 = call float @llvm.x86.avx512.vfmadd.f32(float %30, float %31, float %32, i32 11)
-  %34 = extractelement <4 x float> %x2, i64 0
-  %35 = bitcast i8 %x3 to <8 x i1>
-  %36 = extractelement <8 x i1> %35, i64 0
-  %37 = select i1 %36, float %33, float %34
-  %38 = insertelement <4 x float> %x2, float %37, i64 0
-  %res4 = fadd <4 x float> %8, %19
-  %res5 = fadd <4 x float> %27, %38
-  %res6 = fadd <4 x float> %res4, %res5
-  ret <4 x float> %res6
+  %27 = bitcast i8 %x3 to <8 x i1>
+  %28 = extractelement <8 x i1> %27, i64 0
+  %29 = select i1 %28, float %25, float %26
+  %30 = insertelement <4 x float> %x2, float %29, i64 0
+  %res3 = fadd <4 x float> %11, %19
+  %res4 = fadd <4 x float> %30, %res3
+  ret <4 x float> %res4
 }
 
 define <4 x float>@test_int_x86_avx512_mask3_vfmadd_ss_rm(<4 x float> %x0, <4 x float> %x1, float *%ptr_b ,i8 %x3,i32 %x4) {
diff --git a/llvm/test/CodeGen/X86/avx512-memfold.ll b/llvm/test/CodeGen/X86/avx512-memfold.ll
index 6feb622a00de9..3b1fe7cec0021 100644
--- a/llvm/test/CodeGen/X86/avx512-memfold.ll
+++ b/llvm/test/CodeGen/X86/avx512-memfold.ll
@@ -68,14 +68,13 @@ define <2 x double> @test_int_x86_avx512_mask_vfmadd_sd(<2 x double> %a, <2 x do
 }
 
 ; Test what happens when the load when we have multiple uses of the fadds DAG node via separate vselect nodes.
-; TODO: We shouldn't fold the load twice here.
 define <4 x float> @test_mask_add_ss_double_use(<4 x float> %a, float* %b, i8 %mask, <4 x float> %c) {
 ; CHECK-LABEL: test_mask_add_ss_double_use:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    kmovw %esi, %k1
 ; CHECK-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; CHECK-NEXT:    vaddss %xmm2, %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vaddss %xmm2, %xmm0, %xmm0 {%k1} {z}
+; CHECK-NEXT:    vaddss {rn-sae}, %xmm2, %xmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vmulps %xmm0, %xmm1, %xmm0
 ; CHECK-NEXT:    retq
   %b.val = load float, float* %b
@@ -84,7 +83,7 @@ define <4 x float> @test_mask_add_ss_double_use(<4 x float> %a, float* %b, i8 %m
   %bv2 = insertelement <4 x float> %bv1, float 0.000000e+00, i32 2
   %bv  = insertelement <4 x float> %bv2, float 0.000000e+00, i32 3
   %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> %a, <4 x float> %bv, <4 x float> %c, i8 %mask, i32 4)
-  %res2 = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> %a, <4 x float> %bv, <4 x float> zeroinitializer, i8 %mask, i32 4)
+  %res2 = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> %a, <4 x float> %bv, <4 x float> zeroinitializer, i8 %mask, i32 8)
   %res3 = fmul <4 x float> %res, %res2
   ret <4 x float> %res3
 }
diff --git a/llvm/test/CodeGen/X86/avx512-rotate.ll b/llvm/test/CodeGen/X86/avx512-rotate.ll
index 71e3d9b3cca52..7f69c5011f48a 100644
--- a/llvm/test/CodeGen/X86/avx512-rotate.ll
+++ b/llvm/test/CodeGen/X86/avx512-rotate.ll
@@ -15,26 +15,26 @@ declare <16 x i32> @llvm.x86.avx512.maskz.vpermt2var.d.512(<16 x i32>, <16 x i32
 define <16 x i32> @test_splat_rol_v16i32(<16 x i32> %x0, <16 x i32> %x1, i16 %x2) {
 ; KNL-LABEL: test_splat_rol_v16i32:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vprold $5, %zmm0, %zmm2
 ; KNL-NEXT:    kmovw %edi, %k1
 ; KNL-NEXT:    vprold $5, %zmm0, %zmm1 {%k1}
-; KNL-NEXT:    vprold $5, %zmm0, %zmm0 {%k1} {z}
+; KNL-NEXT:    vprold $6, %zmm0, %zmm2 {%k1} {z}
+; KNL-NEXT:    vpaddd %zmm2, %zmm1, %zmm1
+; KNL-NEXT:    vprold $7, %zmm0, %zmm0
 ; KNL-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
-; KNL-NEXT:    vpaddd %zmm2, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test_splat_rol_v16i32:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vprold $5, %zmm0, %zmm2
 ; SKX-NEXT:    kmovd %edi, %k1
 ; SKX-NEXT:    vprold $5, %zmm0, %zmm1 {%k1}
-; SKX-NEXT:    vprold $5, %zmm0, %zmm0 {%k1} {z}
+; SKX-NEXT:    vprold $6, %zmm0, %zmm2 {%k1} {z}
+; SKX-NEXT:    vpaddd %zmm2, %zmm1, %zmm1
+; SKX-NEXT:    vprold $7, %zmm0, %zmm0
 ; SKX-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
-; SKX-NEXT:    vpaddd %zmm2, %zmm0, %zmm0
 ; SKX-NEXT:    retq
   %res = call <16 x i32> @llvm.x86.avx512.mask.prolv.d.512(<16 x i32> %x0, <16 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>, <16 x i32> %x1, i16 %x2)
-  %res1 = call <16 x i32> @llvm.x86.avx512.mask.prolv.d.512(<16 x i32> %x0, <16 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>, <16 x i32> zeroinitializer, i16 %x2)
-  %res2 = call <16 x i32> @llvm.x86.avx512.mask.prolv.d.512(<16 x i32> %x0, <16 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>, <16 x i32> %x1, i16 -1)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.prolv.d.512(<16 x i32> %x0, <16 x i32> <i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6>, <16 x i32> zeroinitializer, i16 %x2)
+  %res2 = call <16 x i32> @llvm.x86.avx512.mask.prolv.d.512(<16 x i32> %x0, <16 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>, <16 x i32> %x1, i16 -1)
   %res3 = add <16 x i32> %res, %res1
   %res4 = add <16 x i32> %res3, %res2
   ret <16 x i32> %res4
@@ -43,26 +43,26 @@ define <16 x i32> @test_splat_rol_v16i32(<16 x i32> %x0, <16 x i32> %x1, i16 %x2
 define <8 x i64>@test_splat_rol_v8i64(<8 x i64> %x0, <8 x i64> %x1, i8 %x2) {
 ; KNL-LABEL: test_splat_rol_v8i64:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vprolq $5, %zmm0, %zmm2
 ; KNL-NEXT:    kmovw %edi, %k1
 ; KNL-NEXT:    vprolq $5, %zmm0, %zmm1 {%k1}
-; KNL-NEXT:    vprolq $5, %zmm0, %zmm0 {%k1} {z}
+; KNL-NEXT:    vprolq $6, %zmm0, %zmm2 {%k1} {z}
+; KNL-NEXT:    vpaddq %zmm2, %zmm1, %zmm1
+; KNL-NEXT:    vprolq $7, %zmm0, %zmm0
 ; KNL-NEXT:    vpaddq %zmm0, %zmm1, %zmm0
-; KNL-NEXT:    vpaddq %zmm2, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test_splat_rol_v8i64:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vprolq $5, %zmm0, %zmm2
 ; SKX-NEXT:    kmovd %edi, %k1
 ; SKX-NEXT:    vprolq $5, %zmm0, %zmm1 {%k1}
-; SKX-NEXT:    vprolq $5, %zmm0, %zmm0 {%k1} {z}
+; SKX-NEXT:    vprolq $6, %zmm0, %zmm2 {%k1} {z}
+; SKX-NEXT:    vpaddq %zmm2, %zmm1, %zmm1
+; SKX-NEXT:    vprolq $7, %zmm0, %zmm0
 ; SKX-NEXT:    vpaddq %zmm0, %zmm1, %zmm0
-; SKX-NEXT:    vpaddq %zmm2, %zmm0, %zmm0
 ; SKX-NEXT:    retq
   %res = call <8 x i64> @llvm.x86.avx512.mask.prolv.q.512(<8 x i64> %x0, <8 x i64> <i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5>, <8 x i64> %x1, i8 %x2)
-  %res1 = call <8 x i64> @llvm.x86.avx512.mask.prolv.q.512(<8 x i64> %x0, <8 x i64> <i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5>, <8 x i64> zeroinitializer, i8 %x2)
-  %res2 = call <8 x i64> @llvm.x86.avx512.mask.prolv.q.512(<8 x i64> %x0, <8 x i64> <i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5>, <8 x i64> %x1, i8 -1)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.prolv.q.512(<8 x i64> %x0, <8 x i64> <i64 6, i64 6, i64 6, i64 6, i64 6, i64 6, i64 6, i64 6>, <8 x i64> zeroinitializer, i8 %x2)
+  %res2 = call <8 x i64> @llvm.x86.avx512.mask.prolv.q.512(<8 x i64> %x0, <8 x i64> <i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7>, <8 x i64> %x1, i8 -1)
   %res3 = add <8 x i64> %res, %res1
   %res4 = add <8 x i64> %res3, %res2
   ret <8 x i64> %res4
@@ -71,26 +71,26 @@ define <8 x i64>@test_splat_rol_v8i64(<8 x i64> %x0, <8 x i64> %x1, i8 %x2) {
 define <16 x i32> @test_splat_ror_v16i32(<16 x i32> %x0, <16 x i32> %x1, i16 %x2) {
 ; KNL-LABEL: test_splat_ror_v16i32:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vprord $5, %zmm0, %zmm2
 ; KNL-NEXT:    kmovw %edi, %k1
 ; KNL-NEXT:    vprord $5, %zmm0, %zmm1 {%k1}
-; KNL-NEXT:    vprord $5, %zmm0, %zmm0 {%k1} {z}
+; KNL-NEXT:    vprord $6, %zmm0, %zmm2 {%k1} {z}
+; KNL-NEXT:    vpaddd %zmm2, %zmm1, %zmm1
+; KNL-NEXT:    vprord $7, %zmm0, %zmm0
 ; KNL-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
-; KNL-NEXT:    vpaddd %zmm2, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test_splat_ror_v16i32:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vprord $5, %zmm0, %zmm2
 ; SKX-NEXT:    kmovd %edi, %k1
 ; SKX-NEXT:    vprord $5, %zmm0, %zmm1 {%k1}
-; SKX-NEXT:    vprord $5, %zmm0, %zmm0 {%k1} {z}
+; SKX-NEXT:    vprord $6, %zmm0, %zmm2 {%k1} {z}
+; SKX-NEXT:    vpaddd %zmm2, %zmm1, %zmm1
+; SKX-NEXT:    vprord $7, %zmm0, %zmm0
 ; SKX-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
-; SKX-NEXT:    vpaddd %zmm2, %zmm0, %zmm0
 ; SKX-NEXT:    retq
   %res = call <16 x i32> @llvm.x86.avx512.mask.prorv.d.512(<16 x i32> %x0, <16 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>, <16 x i32> %x1, i16 %x2)
-  %res1 = call <16 x i32> @llvm.x86.avx512.mask.prorv.d.512(<16 x i32> %x0, <16 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>, <16 x i32> zeroinitializer, i16 %x2)
-  %res2 = call <16 x i32> @llvm.x86.avx512.mask.prorv.d.512(<16 x i32> %x0, <16 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>, <16 x i32> %x1, i16 -1)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.prorv.d.512(<16 x i32> %x0, <16 x i32> <i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6>, <16 x i32> zeroinitializer, i16 %x2)
+  %res2 = call <16 x i32> @llvm.x86.avx512.mask.prorv.d.512(<16 x i32> %x0, <16 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>, <16 x i32> %x1, i16 -1)
   %res3 = add <16 x i32> %res, %res1
   %res4 = add <16 x i32> %res3, %res2
   ret <16 x i32> %res4
@@ -99,26 +99,26 @@ define <16 x i32> @test_splat_ror_v16i32(<16 x i32> %x0, <16 x i32> %x1, i16 %x2
 define <8 x i64>@test_splat_ror_v8i64(<8 x i64> %x0, <8 x i64> %x1, i8 %x2) {
 ; KNL-LABEL: test_splat_ror_v8i64:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vprorq $5, %zmm0, %zmm2
 ; KNL-NEXT:    kmovw %edi, %k1
 ; KNL-NEXT:    vprorq $5, %zmm0, %zmm1 {%k1}
-; KNL-NEXT:    vprorq $5, %zmm0, %zmm0 {%k1} {z}
+; KNL-NEXT:    vprorq $6, %zmm0, %zmm2 {%k1} {z}
+; KNL-NEXT:    vpaddq %zmm2, %zmm1, %zmm1
+; KNL-NEXT:    vprorq $7, %zmm0, %zmm0
 ; KNL-NEXT:    vpaddq %zmm0, %zmm1, %zmm0
-; KNL-NEXT:    vpaddq %zmm2, %zmm0, %zmm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test_splat_ror_v8i64:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vprorq $5, %zmm0, %zmm2
 ; SKX-NEXT:    kmovd %edi, %k1
 ; SKX-NEXT:    vprorq $5, %zmm0, %zmm1 {%k1}
-; SKX-NEXT:    vprorq $5, %zmm0, %zmm0 {%k1} {z}
+; SKX-NEXT:    vprorq $6, %zmm0, %zmm2 {%k1} {z}
+; SKX-NEXT:    vpaddq %zmm2, %zmm1, %zmm1
+; SKX-NEXT:    vprorq $7, %zmm0, %zmm0
 ; SKX-NEXT:    vpaddq %zmm0, %zmm1, %zmm0
-; SKX-NEXT:    vpaddq %zmm2, %zmm0, %zmm0
 ; SKX-NEXT:    retq
   %res = call <8 x i64> @llvm.x86.avx512.mask.prorv.q.512(<8 x i64> %x0, <8 x i64> <i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5>, <8 x i64> %x1, i8 %x2)
-  %res1 = call <8 x i64> @llvm.x86.avx512.mask.prorv.q.512(<8 x i64> %x0, <8 x i64> <i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5>, <8 x i64> zeroinitializer, i8 %x2)
-  %res2 = call <8 x i64> @llvm.x86.avx512.mask.prorv.q.512(<8 x i64> %x0, <8 x i64> <i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5>, <8 x i64> %x1, i8 -1)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.prorv.q.512(<8 x i64> %x0, <8 x i64> <i64 6, i64 6, i64 6, i64 6, i64 6, i64 6, i64 6, i64 6>, <8 x i64> zeroinitializer, i8 %x2)
+  %res2 = call <8 x i64> @llvm.x86.avx512.mask.prorv.q.512(<8 x i64> %x0, <8 x i64> <i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7>, <8 x i64> %x1, i8 -1)
   %res3 = add <8 x i64> %res, %res1
   %res4 = add <8 x i64> %res3, %res2
   ret <8 x i64> %res4
diff --git a/llvm/test/CodeGen/X86/avx512-trunc.ll b/llvm/test/CodeGen/X86/avx512-trunc.ll
index bf49987aba4f7..171fd45419b36 100644
--- a/llvm/test/CodeGen/X86/avx512-trunc.ll
+++ b/llvm/test/CodeGen/X86/avx512-trunc.ll
@@ -91,10 +91,15 @@ define void @trunc_qb_256_mem(<4 x i64> %i, <4 x i8>* %res) #0 {
 }
 
 define <2 x i8> @trunc_qb_128(<2 x i64> %i) #0 {
-; ALL-LABEL: trunc_qb_128:
-; ALL:       ## %bb.0:
-; ALL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; ALL-NEXT:    retq
+; KNL-LABEL: trunc_qb_128:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: trunc_qb_128:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vpmovqb %xmm0, %xmm0
+; SKX-NEXT:    retq
   %x = trunc <2 x i64> %i to <2 x i8>
   ret <2 x i8> %x
 }
@@ -344,10 +349,15 @@ define void @trunc_db_256_mem(<8 x i32> %i, <8 x i8>* %res) #0 {
 }
 
 define <4 x i8> @trunc_db_128(<4 x i32> %i) #0 {
-; ALL-LABEL: trunc_db_128:
-; ALL:       ## %bb.0:
-; ALL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
-; ALL-NEXT:    retq
+; KNL-LABEL: trunc_db_128:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: trunc_db_128:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vpmovdb %xmm0, %xmm0
+; SKX-NEXT:    retq
   %x = trunc <4 x i32> %i to <4 x i8>
   ret <4 x i8> %x
 }
@@ -537,10 +547,15 @@ define <16 x i8> @trunc_wb_256_mem_and_ret(<16 x i16> %i, <16 x i8>* %res) #0 {
 }
 
 define <8 x i8> @trunc_wb_128(<8 x i16> %i) #0 {
-; ALL-LABEL: trunc_wb_128:
-; ALL:       ## %bb.0:
-; ALL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
-; ALL-NEXT:    retq
+; KNL-LABEL: trunc_wb_128:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: trunc_wb_128:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vpmovwb %xmm0, %xmm0
+; SKX-NEXT:    retq
   %x = trunc <8 x i16> %i to <8 x i8>
   ret <8 x i8> %x
 }
diff --git a/llvm/test/CodeGen/X86/avx512-vec-cmp.ll b/llvm/test/CodeGen/X86/avx512-vec-cmp.ll
index 8b9a7a03d502c..e43c6f591574d 100644
--- a/llvm/test/CodeGen/X86/avx512-vec-cmp.ll
+++ b/llvm/test/CodeGen/X86/avx512-vec-cmp.ll
@@ -315,9 +315,9 @@ define <16 x i32> @test13(<16 x float>%a, <16 x float>%b)
 define <16 x i32> @test14(<16 x i32>%a, <16 x i32>%b) {
 ; CHECK-LABEL: test14:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpsubd %zmm1, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0x7d,0x48,0xfa,0xd1]
-; CHECK-NEXT:    vpcmpgtd %zmm0, %zmm2, %k1 ## encoding: [0x62,0xf1,0x6d,0x48,0x66,0xc8]
-; CHECK-NEXT:    vpsubd %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0xfa,0xc1]
+; CHECK-NEXT:    vpsubd %zmm1, %zmm0, %zmm1 ## encoding: [0x62,0xf1,0x7d,0x48,0xfa,0xc9]
+; CHECK-NEXT:    vpcmpgtd %zmm0, %zmm1, %k1 ## encoding: [0x62,0xf1,0x75,0x48,0x66,0xc8]
+; CHECK-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0x6f,0xc1]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %sub_r = sub <16 x i32> %a, %b
   %cmp.i2.i = icmp sgt <16 x i32> %sub_r, %a
@@ -330,9 +330,9 @@ define <16 x i32> @test14(<16 x i32>%a, <16 x i32>%b) {
 define <8 x i64> @test15(<8 x i64>%a, <8 x i64>%b) {
 ; CHECK-LABEL: test15:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpsubq %zmm1, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0xfd,0x48,0xfb,0xd1]
-; CHECK-NEXT:    vpcmpgtq %zmm0, %zmm2, %k1 ## encoding: [0x62,0xf2,0xed,0x48,0x37,0xc8]
-; CHECK-NEXT:    vpsubq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0xfb,0xc1]
+; CHECK-NEXT:    vpsubq %zmm1, %zmm0, %zmm1 ## encoding: [0x62,0xf1,0xfd,0x48,0xfb,0xc9]
+; CHECK-NEXT:    vpcmpgtq %zmm0, %zmm1, %k1 ## encoding: [0x62,0xf2,0xf5,0x48,0x37,0xc8]
+; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0x6f,0xc1]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %sub_r = sub <8 x i64> %a, %b
   %cmp.i2.i = icmp sgt <8 x i64> %sub_r, %a
@@ -1554,9 +1554,7 @@ define <8 x i64> @cmp_swap_bug(<16 x i8>* %x, <8 x i64> %y, <8 x i64> %z) {
 ; SKX-LABEL: cmp_swap_bug:
 ; SKX:       ## %bb.0: ## %entry
 ; SKX-NEXT:    vmovdqa (%rdi), %xmm2 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0x17]
-; SKX-NEXT:    vpshufb {{.*}}(%rip), %xmm2, %xmm2 ## EVEX TO VEX Compression xmm2 = xmm2[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
-; SKX-NEXT:    ## encoding: [0xc4,0xe2,0x69,0x00,0x15,A,A,A,A]
-; SKX-NEXT:    ## fixup A - offset: 5, value: LCPI69_0-4, kind: reloc_riprel_4byte
+; SKX-NEXT:    vpmovwb %xmm2, %xmm2 ## encoding: [0x62,0xf2,0x7e,0x08,0x30,0xd2]
 ; SKX-NEXT:    vpmovb2m %xmm2, %k1 ## encoding: [0x62,0xf2,0x7e,0x08,0x29,0xca]
 ; SKX-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0x64,0xc0]
 ; SKX-NEXT:    retq ## encoding: [0xc3]
diff --git a/llvm/test/CodeGen/X86/avx512bw-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512bw-intrinsics-upgrade.ll
index c87f04fbb841b..5f35c21839e80 100644
--- a/llvm/test/CodeGen/X86/avx512bw-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512bw-intrinsics-upgrade.ll
@@ -1081,32 +1081,47 @@ define <32 x i16>@test_int_x86_avx512_maskz_pmovsxb_w_512(<32 x i8> %x0, i32 %x2
 
 declare <32 x i16> @llvm.x86.avx512.mask.psrl.w.512(<32 x i16>, <8 x i16>, <32 x i16>, i32)
 
+define <32 x i16>@test_int_x86_avx512_psrl_w_512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psrl_w_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsrlw %xmm1, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xd1,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psrl.w.512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
 define <32 x i16>@test_int_x86_avx512_mask_psrl_w_512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> %x2, i32 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrl_w_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsrlw %xmm1, %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7d,0x48,0xd1,0xd9]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpsrlw %xmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xd1,0xd1]
-; X86-NEXT:    vpsrlw %xmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xd1,0xc1]
-; X86-NEXT:    vpaddw %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfd,0xc0]
-; X86-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrl_w_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsrlw %xmm1, %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7d,0x48,0xd1,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpsrlw %xmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xd1,0xd1]
-; X64-NEXT:    vpsrlw %xmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xd1,0xc1]
-; X64-NEXT:    vpaddw %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfd,0xc0]
-; X64-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <32 x i16> @llvm.x86.avx512.mask.psrl.w.512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> %x2, i32 %x3)
-  %res1 = call <32 x i16> @llvm.x86.avx512.mask.psrl.w.512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> %x2, i32 -1)
-  %res2 = call <32 x i16> @llvm.x86.avx512.mask.psrl.w.512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> zeroinitializer, i32 %x3)
-  %res3 = add <32 x i16> %res, %res1
-  %res4 = add <32 x i16> %res3, %res2
-  ret <32 x i16> %res4
+  ret <32 x i16> %res
+}
+
+define <32 x i16>@test_int_x86_avx512_maskz_psrl_w_512(<32 x i16> %x0, <8 x i16> %x1, i32 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psrl_w_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsrlw %xmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xd1,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psrl_w_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsrlw %xmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xd1,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psrl.w.512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> zeroinitializer, i32 %x3)
+  ret <32 x i16> %res
 }
 
 declare <32 x i16> @llvm.x86.avx512.mask.psrl.wi.512(<32 x i16>, i32, <32 x i16>, i32)
@@ -1114,26 +1129,26 @@ declare <32 x i16> @llvm.x86.avx512.mask.psrl.wi.512(<32 x i16>, i32, <32 x i16>
 define <32 x i16>@test_int_x86_avx512_mask_psrl_wi_512(<32 x i16> %x0, i32 %x1, <32 x i16> %x2, i32 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrl_wi_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsrlw $3, %zmm0, %zmm2 # encoding: [0x62,0xf1,0x6d,0x48,0x71,0xd0,0x03]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vpsrlw $3, %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x49,0x71,0xd0,0x03]
-; X86-NEXT:    vpsrlw $3, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0x71,0xd0,0x03]
+; X86-NEXT:    vpsrlw $4, %zmm0, %zmm2 # encoding: [0x62,0xf1,0x6d,0x48,0x71,0xd0,0x04]
+; X86-NEXT:    vpsrlw $5, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0x71,0xd0,0x05]
 ; X86-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
 ; X86-NEXT:    vpaddw %zmm0, %zmm1, %zmm0 # encoding: [0x62,0xf1,0x75,0x48,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrl_wi_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsrlw $3, %zmm0, %zmm2 # encoding: [0x62,0xf1,0x6d,0x48,0x71,0xd0,0x03]
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vpsrlw $3, %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x49,0x71,0xd0,0x03]
-; X64-NEXT:    vpsrlw $3, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0x71,0xd0,0x03]
+; X64-NEXT:    vpsrlw $4, %zmm0, %zmm2 # encoding: [0x62,0xf1,0x6d,0x48,0x71,0xd0,0x04]
+; X64-NEXT:    vpsrlw $5, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0x71,0xd0,0x05]
 ; X64-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
 ; X64-NEXT:    vpaddw %zmm0, %zmm1, %zmm0 # encoding: [0x62,0xf1,0x75,0x48,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <32 x i16> @llvm.x86.avx512.mask.psrl.wi.512(<32 x i16> %x0, i32 3, <32 x i16> %x2, i32 %x3)
-  %res1 = call <32 x i16> @llvm.x86.avx512.mask.psrl.wi.512(<32 x i16> %x0, i32 3, <32 x i16> %x2, i32 -1)
-  %res2 = call <32 x i16> @llvm.x86.avx512.mask.psrl.wi.512(<32 x i16> %x0, i32 3, <32 x i16> zeroinitializer, i32 %x3)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.psrl.wi.512(<32 x i16> %x0, i32 4, <32 x i16> %x2, i32 -1)
+  %res2 = call <32 x i16> @llvm.x86.avx512.mask.psrl.wi.512(<32 x i16> %x0, i32 5, <32 x i16> zeroinitializer, i32 %x3)
   %res3 = add <32 x i16> %res, %res1
   %res4 = add <32 x i16> %res3, %res2
   ret <32 x i16> %res4
@@ -1141,32 +1156,47 @@ define <32 x i16>@test_int_x86_avx512_mask_psrl_wi_512(<32 x i16> %x0, i32 %x1,
 
 declare <32 x i16> @llvm.x86.avx512.mask.psra.w.512(<32 x i16>, <8 x i16>, <32 x i16>, i32)
 
+define <32 x i16>@test_int_x86_avx512_psra_w_512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psra_w_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsraw %xmm1, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xe1,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psra.w.512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
 define <32 x i16>@test_int_x86_avx512_mask_psra_w_512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> %x2, i32 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psra_w_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsraw %xmm1, %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7d,0x48,0xe1,0xd9]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpsraw %xmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xe1,0xd1]
-; X86-NEXT:    vpsraw %xmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xe1,0xc1]
-; X86-NEXT:    vpaddw %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc3]
-; X86-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psra_w_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsraw %xmm1, %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7d,0x48,0xe1,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpsraw %xmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xe1,0xd1]
-; X64-NEXT:    vpsraw %xmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xe1,0xc1]
-; X64-NEXT:    vpaddw %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc3]
-; X64-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <32 x i16> @llvm.x86.avx512.mask.psra.w.512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> %x2, i32 %x3)
-  %res1 = call <32 x i16> @llvm.x86.avx512.mask.psra.w.512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> zeroinitializer, i32 %x3)
-  %res2 = call <32 x i16> @llvm.x86.avx512.mask.psra.w.512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> %x2, i32 -1)
-  %res3 = add <32 x i16> %res, %res1
-  %res4 = add <32 x i16> %res3, %res2
-  ret <32 x i16> %res4
+  ret <32 x i16> %res
+}
+
+define <32 x i16>@test_int_x86_avx512_maskz_psra_w_512(<32 x i16> %x0, <8 x i16> %x1, i32 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psra_w_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsraw %xmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xe1,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psra_w_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsraw %xmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xe1,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psra.w.512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> zeroinitializer, i32 %x3)
+  ret <32 x i16> %res
 }
 
 declare <32 x i16> @llvm.x86.avx512.mask.psra.wi.512(<32 x i16>, i32, <32 x i16>, i32)
@@ -1174,26 +1204,26 @@ declare <32 x i16> @llvm.x86.avx512.mask.psra.wi.512(<32 x i16>, i32, <32 x i16>
 define <32 x i16>@test_int_x86_avx512_mask_psra_wi_512(<32 x i16> %x0, i32 %x1, <32 x i16> %x2, i32 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psra_wi_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsraw $3, %zmm0, %zmm2 # encoding: [0x62,0xf1,0x6d,0x48,0x71,0xe0,0x03]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vpsraw $3, %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x49,0x71,0xe0,0x03]
-; X86-NEXT:    vpsraw $3, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0x71,0xe0,0x03]
-; X86-NEXT:    vpaddw %zmm2, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc2]
+; X86-NEXT:    vpsraw $4, %zmm0, %zmm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0xc9,0x71,0xe0,0x04]
+; X86-NEXT:    vpsraw $5, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0x71,0xe0,0x05]
+; X86-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
 ; X86-NEXT:    vpaddw %zmm0, %zmm1, %zmm0 # encoding: [0x62,0xf1,0x75,0x48,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psra_wi_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsraw $3, %zmm0, %zmm2 # encoding: [0x62,0xf1,0x6d,0x48,0x71,0xe0,0x03]
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vpsraw $3, %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x49,0x71,0xe0,0x03]
-; X64-NEXT:    vpsraw $3, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0x71,0xe0,0x03]
-; X64-NEXT:    vpaddw %zmm2, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc2]
+; X64-NEXT:    vpsraw $4, %zmm0, %zmm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0xc9,0x71,0xe0,0x04]
+; X64-NEXT:    vpsraw $5, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0x71,0xe0,0x05]
+; X64-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
 ; X64-NEXT:    vpaddw %zmm0, %zmm1, %zmm0 # encoding: [0x62,0xf1,0x75,0x48,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <32 x i16> @llvm.x86.avx512.mask.psra.wi.512(<32 x i16> %x0, i32 3, <32 x i16> %x2, i32 %x3)
-  %res1 = call <32 x i16> @llvm.x86.avx512.mask.psra.wi.512(<32 x i16> %x0, i32 3, <32 x i16> zeroinitializer, i32 %x3)
-  %res2 = call <32 x i16> @llvm.x86.avx512.mask.psra.wi.512(<32 x i16> %x0, i32 3, <32 x i16> %x2, i32 -1)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.psra.wi.512(<32 x i16> %x0, i32 4, <32 x i16> zeroinitializer, i32 %x3)
+  %res2 = call <32 x i16> @llvm.x86.avx512.mask.psra.wi.512(<32 x i16> %x0, i32 5, <32 x i16> %x2, i32 -1)
   %res3 = add <32 x i16> %res, %res1
   %res4 = add <32 x i16> %res3, %res2
   ret <32 x i16> %res4
@@ -1201,32 +1231,47 @@ define <32 x i16>@test_int_x86_avx512_mask_psra_wi_512(<32 x i16> %x0, i32 %x1,
 
 declare <32 x i16> @llvm.x86.avx512.mask.psll.w.512(<32 x i16>, <8 x i16>, <32 x i16>, i32)
 
+define <32 x i16>@test_int_x86_avx512_psll_w_512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psll_w_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsllw %xmm1, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xf1,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psll.w.512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
 define <32 x i16>@test_int_x86_avx512_mask_psll_w_512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> %x2, i32 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psll_w_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsllw %xmm1, %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7d,0x48,0xf1,0xd9]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpsllw %xmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xf1,0xd1]
-; X86-NEXT:    vpsllw %xmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xf1,0xc1]
-; X86-NEXT:    vpaddw %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc3]
-; X86-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psll_w_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsllw %xmm1, %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7d,0x48,0xf1,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpsllw %xmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xf1,0xd1]
-; X64-NEXT:    vpsllw %xmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xf1,0xc1]
-; X64-NEXT:    vpaddw %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc3]
-; X64-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <32 x i16> @llvm.x86.avx512.mask.psll.w.512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> %x2, i32 %x3)
-  %res1 = call <32 x i16> @llvm.x86.avx512.mask.psll.w.512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> zeroinitializer, i32 %x3)
-  %res2 = call <32 x i16> @llvm.x86.avx512.mask.psll.w.512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> %x2, i32 -1)
-  %res3 = add <32 x i16> %res, %res1
-  %res4 = add <32 x i16> %res3, %res2
-  ret <32 x i16> %res4
+  ret <32 x i16> %res
+}
+
+define <32 x i16>@test_int_x86_avx512_maskz_psll_w_512(<32 x i16> %x0, <8 x i16> %x1, i32 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psll_w_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsllw %xmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xf1,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psll_w_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsllw %xmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xf1,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psll.w.512(<32 x i16> %x0, <8 x i16> %x1, <32 x i16> zeroinitializer, i32 %x3)
+  ret <32 x i16> %res
 }
 
 declare <32 x i16> @llvm.x86.avx512.mask.psll.wi.512(<32 x i16>, i32, <32 x i16>, i32)
@@ -1234,26 +1279,26 @@ declare <32 x i16> @llvm.x86.avx512.mask.psll.wi.512(<32 x i16>, i32, <32 x i16>
 define <32 x i16>@test_int_x86_avx512_mask_psll_wi_512(<32 x i16> %x0, i32 %x1, <32 x i16> %x2, i32 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psll_wi_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsllw $3, %zmm0, %zmm2 # encoding: [0x62,0xf1,0x6d,0x48,0x71,0xf0,0x03]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vpsllw $3, %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x49,0x71,0xf0,0x03]
-; X86-NEXT:    vpsllw $3, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0x71,0xf0,0x03]
-; X86-NEXT:    vpaddw %zmm2, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc2]
+; X86-NEXT:    vpsllw $4, %zmm0, %zmm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0xc9,0x71,0xf0,0x04]
+; X86-NEXT:    vpsllw $5, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0x71,0xf0,0x05]
+; X86-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
 ; X86-NEXT:    vpaddw %zmm0, %zmm1, %zmm0 # encoding: [0x62,0xf1,0x75,0x48,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psll_wi_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsllw $3, %zmm0, %zmm2 # encoding: [0x62,0xf1,0x6d,0x48,0x71,0xf0,0x03]
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vpsllw $3, %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x49,0x71,0xf0,0x03]
-; X64-NEXT:    vpsllw $3, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0x71,0xf0,0x03]
-; X64-NEXT:    vpaddw %zmm2, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc2]
+; X64-NEXT:    vpsllw $4, %zmm0, %zmm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0xc9,0x71,0xf0,0x04]
+; X64-NEXT:    vpsllw $5, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0x71,0xf0,0x05]
+; X64-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
 ; X64-NEXT:    vpaddw %zmm0, %zmm1, %zmm0 # encoding: [0x62,0xf1,0x75,0x48,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <32 x i16> @llvm.x86.avx512.mask.psll.wi.512(<32 x i16> %x0, i32 3, <32 x i16> %x2, i32 %x3)
-  %res1 = call <32 x i16> @llvm.x86.avx512.mask.psll.wi.512(<32 x i16> %x0, i32 3, <32 x i16> zeroinitializer, i32 %x3)
-  %res2 = call <32 x i16> @llvm.x86.avx512.mask.psll.wi.512(<32 x i16> %x0, i32 3, <32 x i16> %x2, i32 -1)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.psll.wi.512(<32 x i16> %x0, i32 4, <32 x i16> zeroinitializer, i32 %x3)
+  %res2 = call <32 x i16> @llvm.x86.avx512.mask.psll.wi.512(<32 x i16> %x0, i32 5, <32 x i16> %x2, i32 -1)
   %res3 = add <32 x i16> %res, %res1
   %res4 = add <32 x i16> %res3, %res2
   ret <32 x i16> %res4
@@ -2541,98 +2586,117 @@ declare i32 @llvm.x86.avx512.mask.ucmp.w.512(<32 x i16>, <32 x i16>, i32, i32) n
 declare <64 x i8> @llvm.x86.avx512.mask.pavg.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64)
 
 define <64 x i8>@mm512_avg_epu8(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) {
-; X86-LABEL: mm512_avg_epu8:
+; CHECK-LABEL: mm512_avg_epu8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpavgb %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xe0,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <64 x i8> @llvm.x86.avx512.mask.pavg.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 -1)
+  ret <64 x i8> %res
+}
+
+define <64 x i8>@mm512_mask_avg_epu8(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) {
+; X86-LABEL: mm512_mask_avg_epu8:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpavgb %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7d,0x48,0xe0,0xd9]
 ; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpavgb %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xe0,0xd1]
-; X86-NEXT:    vpaddb %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfc,0xc3]
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: mm512_avg_epu8:
+; X64-LABEL: mm512_mask_avg_epu8:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpavgb %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7d,0x48,0xe0,0xd9]
 ; X64-NEXT:    kmovq %rdi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpavgb %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xe0,0xd1]
-; X64-NEXT:    vpaddb %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfc,0xc3]
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <64 x i8> @llvm.x86.avx512.mask.pavg.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3)
-  %res1 = call <64 x i8> @llvm.x86.avx512.mask.pavg.b.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 -1)
-  %res2 = add <64 x i8> %res, %res1
-  ret <64 x i8> %res2
+  ret <64 x i8> %res
 }
 
-
 declare <32 x i16> @llvm.x86.avx512.mask.pavg.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
 
 define <32 x i16>@mm512_avg_epu16(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
-; X86-LABEL: mm512_avg_epu16:
+; CHECK-LABEL: mm512_avg_epu16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpavgw %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xe3,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pavg.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16>@mm512_mask_avg_epu16(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
+; X86-LABEL: mm512_mask_avg_epu16:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpavgw %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7d,0x48,0xe3,0xd9]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpavgw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xe3,0xd1]
-; X86-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: mm512_avg_epu16:
+; X64-LABEL: mm512_mask_avg_epu16:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpavgw %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7d,0x48,0xe3,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpavgw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xe3,0xd1]
-; X64-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <32 x i16> @llvm.x86.avx512.mask.pavg.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
-  %res1 = call <32 x i16> @llvm.x86.avx512.mask.pavg.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
-  %res2 = add <32 x i16> %res, %res1
-  ret <32 x i16> %res2
+  ret <32 x i16> %res
 }
 
 declare <32 x i16> @llvm.x86.avx512.mask.pabs.w.512(<32 x i16>, <32 x i16>, i32)
 
+define <32 x i16>@test_int_x86_avx512_pabs_w_512(<32 x i16> %x0, <32 x i16> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_pabs_w_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpabsw %zmm0, %zmm0 # encoding: [0x62,0xf2,0x7d,0x48,0x1d,0xc0]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.pabs.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 -1)
+  ret <32 x i16> %res
+}
+
 define <32 x i16>@test_int_x86_avx512_mask_pabs_w_512(<32 x i16> %x0, <32 x i16> %x1, i32 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_pabs_w_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpabsw %zmm0, %zmm2 # encoding: [0x62,0xf2,0x7d,0x48,0x1d,0xd0]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpabsw %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x1d,0xc8]
-; X86-NEXT:    vpaddw %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf1,0x75,0x48,0xfd,0xc2]
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pabs_w_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpabsw %zmm0, %zmm2 # encoding: [0x62,0xf2,0x7d,0x48,0x1d,0xd0]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpabsw %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x1d,0xc8]
-; X64-NEXT:    vpaddw %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf1,0x75,0x48,0xfd,0xc2]
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <32 x i16> @llvm.x86.avx512.mask.pabs.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 %x2)
-  %res1 = call <32 x i16> @llvm.x86.avx512.mask.pabs.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 -1)
-  %res2 = add <32 x i16> %res, %res1
-  ret <32 x i16> %res2
+  ret <32 x i16> %res
 }
 
 declare <64 x i8> @llvm.x86.avx512.mask.pabs.b.512(<64 x i8>, <64 x i8>, i64)
 
+define <64 x i8>@test_int_x86_avx512_pabs_b_512(<64 x i8> %x0, <64 x i8> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_pabs_b_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpabsb %zmm0, %zmm0 # encoding: [0x62,0xf2,0x7d,0x48,0x1c,0xc0]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <64 x i8> @llvm.x86.avx512.mask.pabs.b.512(<64 x i8> %x0, <64 x i8> %x1, i64 -1)
+  ret <64 x i8> %res
+}
+
 define <64 x i8>@test_int_x86_avx512_mask_pabs_b_512(<64 x i8> %x0, <64 x i8> %x1, i64 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_pabs_b_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpabsb %zmm0, %zmm2 # encoding: [0x62,0xf2,0x7d,0x48,0x1c,0xd0]
 ; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpabsb %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x1c,0xc8]
-; X86-NEXT:    vpaddb %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf1,0x75,0x48,0xfc,0xc2]
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pabs_b_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpabsb %zmm0, %zmm2 # encoding: [0x62,0xf2,0x7d,0x48,0x1c,0xd0]
 ; X64-NEXT:    kmovq %rdi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpabsb %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x1c,0xc8]
-; X64-NEXT:    vpaddb %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf1,0x75,0x48,0xfc,0xc2]
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <64 x i8> @llvm.x86.avx512.mask.pabs.b.512(<64 x i8> %x0, <64 x i8> %x1, i64 %x2)
-  %res1 = call <64 x i8> @llvm.x86.avx512.mask.pabs.b.512(<64 x i8> %x0, <64 x i8> %x1, i64 -1)
-  %res2 = add <64 x i8> %res, %res1
-  ret <64 x i8> %res2
+  ret <64 x i8> %res
 }
 
 declare i64 @llvm.x86.avx512.ptestm.b.512(<64 x i8>, <64 x i8>, i64)
@@ -3069,26 +3133,26 @@ declare <32 x i16> @llvm.x86.avx512.mask.dbpsadbw.512(<64 x i8>, <64 x i8>, i32,
 define <32 x i16>@test_int_x86_avx512_mask_dbpsadbw_512(<64 x i8> %x0, <64 x i8> %x1, <32 x i16> %x3, i32 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_dbpsadbw_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vdbpsadbw $2, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0x7d,0x48,0x42,0xd9,0x02]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vdbpsadbw $2, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x49,0x42,0xd1,0x02]
-; X86-NEXT:    vdbpsadbw $2, %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0xc9,0x42,0xc1,0x02]
-; X86-NEXT:    vpaddw %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc3]
+; X86-NEXT:    vdbpsadbw $3, %zmm1, %zmm0, %zmm3 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0xc9,0x42,0xd9,0x03]
+; X86-NEXT:    vdbpsadbw $4, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0x7d,0x48,0x42,0xc1,0x04]
+; X86-NEXT:    vpaddw %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfd,0xc0]
 ; X86-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_dbpsadbw_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vdbpsadbw $2, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0x7d,0x48,0x42,0xd9,0x02]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vdbpsadbw $2, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x49,0x42,0xd1,0x02]
-; X64-NEXT:    vdbpsadbw $2, %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0xc9,0x42,0xc1,0x02]
-; X64-NEXT:    vpaddw %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc3]
+; X64-NEXT:    vdbpsadbw $3, %zmm1, %zmm0, %zmm3 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0xc9,0x42,0xd9,0x03]
+; X64-NEXT:    vdbpsadbw $4, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0x7d,0x48,0x42,0xc1,0x04]
+; X64-NEXT:    vpaddw %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfd,0xc0]
 ; X64-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <32 x i16> @llvm.x86.avx512.mask.dbpsadbw.512(<64 x i8> %x0, <64 x i8> %x1, i32 2, <32 x i16> %x3, i32 %x4)
-  %res1 = call <32 x i16> @llvm.x86.avx512.mask.dbpsadbw.512(<64 x i8> %x0, <64 x i8> %x1, i32 2, <32 x i16> zeroinitializer, i32 %x4)
-  %res2 = call <32 x i16> @llvm.x86.avx512.mask.dbpsadbw.512(<64 x i8> %x0, <64 x i8> %x1, i32 2, <32 x i16> %x3, i32 -1)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.dbpsadbw.512(<64 x i8> %x0, <64 x i8> %x1, i32 3, <32 x i16> zeroinitializer, i32 %x4)
+  %res2 = call <32 x i16> @llvm.x86.avx512.mask.dbpsadbw.512(<64 x i8> %x0, <64 x i8> %x1, i32 4, <32 x i16> %x3, i32 -1)
   %res3 = add <32 x i16> %res, %res1
   %res4 = add <32 x i16> %res3, %res2
   ret <32 x i16> %res4
@@ -4102,92 +4166,137 @@ declare <64 x i8> @llvm.x86.avx512.mask.psubs.b.512(<64 x i8>, <64 x i8>, <64 x
 
 declare <32 x i16> @llvm.x86.avx512.mask.psrlv32hi(<32 x i16>, <32 x i16>, <32 x i16>, i32)
 
+define <32 x i16>@test_int_x86_avx512_psrlv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psrlv32hi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf2,0xfd,0x48,0x10,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psrlv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
 define <32 x i16>@test_int_x86_avx512_mask_psrlv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrlv32hi:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf2,0xfd,0x48,0x10,0xd9]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x10,0xd1]
-; X86-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x10,0xc1]
-; X86-NEXT:    vpaddw %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc3]
-; X86-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrlv32hi:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf2,0xfd,0x48,0x10,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x10,0xd1]
-; X64-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x10,0xc1]
-; X64-NEXT:    vpaddw %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc3]
-; X64-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <32 x i16> @llvm.x86.avx512.mask.psrlv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
-  %res1 = call <32 x i16> @llvm.x86.avx512.mask.psrlv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> zeroinitializer, i32 %x3)
-  %res2 = call <32 x i16> @llvm.x86.avx512.mask.psrlv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
-  %res3 = add <32 x i16> %res, %res1
-  %res4 = add <32 x i16> %res3, %res2
-  ret <32 x i16> %res4
+  ret <32 x i16> %res
+}
+
+define <32 x i16>@test_int_x86_avx512_maskz_psrlv32hi(<32 x i16> %x0, <32 x i16> %x1, i32 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psrlv32hi:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x10,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psrlv32hi:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x10,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psrlv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> zeroinitializer, i32 %x3)
+  ret <32 x i16> %res
 }
 
 declare <32 x i16> @llvm.x86.avx512.mask.psrav32.hi(<32 x i16>, <32 x i16>, <32 x i16>, i32)
 
+define <32 x i16>@test_int_x86_avx512_psrav32_hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psrav32_hi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsravw %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf2,0xfd,0x48,0x11,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psrav32.hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
 define <32 x i16>@test_int_x86_avx512_mask_psrav32_hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrav32_hi:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsravw %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf2,0xfd,0x48,0x11,0xd9]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpsravw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x11,0xd1]
-; X86-NEXT:    vpsravw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x11,0xc1]
-; X86-NEXT:    vpaddw %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc3]
-; X86-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrav32_hi:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsravw %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf2,0xfd,0x48,0x11,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpsravw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x11,0xd1]
-; X64-NEXT:    vpsravw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x11,0xc1]
-; X64-NEXT:    vpaddw %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc3]
-; X64-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <32 x i16> @llvm.x86.avx512.mask.psrav32.hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
-  %res1 = call <32 x i16> @llvm.x86.avx512.mask.psrav32.hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> zeroinitializer, i32 %x3)
-  %res2 = call <32 x i16> @llvm.x86.avx512.mask.psrav32.hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
-  %res3 = add <32 x i16> %res, %res1
-  %res4 = add <32 x i16> %res3, %res2
-  ret <32 x i16> %res4
+  ret <32 x i16> %res
+}
+
+define <32 x i16>@test_int_x86_avx512_maskz_psrav32_hi(<32 x i16> %x0, <32 x i16> %x1, i32 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psrav32_hi:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsravw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x11,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psrav32_hi:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsravw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x11,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psrav32.hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> zeroinitializer, i32 %x3)
+  ret <32 x i16> %res
 }
 
 declare <32 x i16> @llvm.x86.avx512.mask.psllv32hi(<32 x i16>, <32 x i16>, <32 x i16>, i32)
 
+define <32 x i16>@test_int_x86_avx512_psllv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psllv32hi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf2,0xfd,0x48,0x12,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psllv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
 define <32 x i16>@test_int_x86_avx512_mask_psllv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psllv32hi:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsllvw %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf2,0xfd,0x48,0x12,0xd9]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpsllvw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x12,0xd1]
-; X86-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x12,0xc1]
-; X86-NEXT:    vpaddw %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc3]
-; X86-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psllv32hi:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsllvw %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf2,0xfd,0x48,0x12,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpsllvw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x12,0xd1]
-; X64-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x12,0xc1]
-; X64-NEXT:    vpaddw %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc3]
-; X64-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <32 x i16> @llvm.x86.avx512.mask.psllv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
-  %res1 = call <32 x i16> @llvm.x86.avx512.mask.psllv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> zeroinitializer, i32 %x3)
-  %res2 = call <32 x i16> @llvm.x86.avx512.mask.psllv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
-  %res3 = add <32 x i16> %res, %res1
-  %res4 = add <32 x i16> %res3, %res2
-  ret <32 x i16> %res4
+  ret <32 x i16> %res
+}
+
+define <32 x i16>@test_int_x86_avx512_maskz_psllv32hi(<32 x i16> %x0, <32 x i16> %x1, i32 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psllv32hi:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x12,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psllv32hi:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x12,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psllv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> zeroinitializer, i32 %x3)
+  ret <32 x i16> %res
 }
 
 declare <32 x i8> @llvm.x86.avx512.mask.pmov.wb.512(<32 x i16>, <32 x i8>, i32)
diff --git a/llvm/test/CodeGen/X86/avx512bw-intrinsics.ll b/llvm/test/CodeGen/X86/avx512bw-intrinsics.ll
index e84d9d37c8222..0ed73f142a846 100644
--- a/llvm/test/CodeGen/X86/avx512bw-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512bw-intrinsics.ll
@@ -763,54 +763,64 @@ define <32 x i16>@test_int_x86_avx512_mask_vpermi2var_hi_512(<32 x i16> %x0, <32
 
 declare <64 x i8> @llvm.x86.avx512.pavg.b.512(<64 x i8>, <64 x i8>)
 
+define <64 x i8> @test_int_x86_avx512_pavg_b_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_pavg_b_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpavgb %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xe0,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <64 x i8> @llvm.x86.avx512.pavg.b.512(<64 x i8> %x0, <64 x i8> %x1)
+  ret <64 x i8> %1
+}
+
 define <64 x i8> @test_int_x86_avx512_mask_pavg_b_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_pavg_b_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpavgb %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7d,0x48,0xe0,0xd9]
 ; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpavgb %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xe0,0xd1]
-; X86-NEXT:    vpaddb %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfc,0xc3]
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pavg_b_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpavgb %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7d,0x48,0xe0,0xd9]
 ; X64-NEXT:    kmovq %rdi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpavgb %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xe0,0xd1]
-; X64-NEXT:    vpaddb %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfc,0xc3]
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <64 x i8> @llvm.x86.avx512.pavg.b.512(<64 x i8> %x0, <64 x i8> %x1)
   %2 = bitcast i64 %x3 to <64 x i1>
   %3 = select <64 x i1> %2, <64 x i8> %1, <64 x i8> %x2
-  %4 = call <64 x i8> @llvm.x86.avx512.pavg.b.512(<64 x i8> %x0, <64 x i8> %x1)
-  %res2 = add <64 x i8> %3, %4
-  ret <64 x i8> %res2
+  ret <64 x i8> %3
 }
 
 declare <32 x i16> @llvm.x86.avx512.pavg.w.512(<32 x i16>, <32 x i16>)
 
+define <32 x i16> @test_int_x86_avx512_pavg_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_pavg_w_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpavgw %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xe3,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <32 x i16> @llvm.x86.avx512.pavg.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  ret <32 x i16> %1
+}
+
 define <32 x i16> @test_int_x86_avx512_mask_pavg_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_pavg_w_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpavgw %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7d,0x48,0xe3,0xd9]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpavgw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xe3,0xd1]
-; X86-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pavg_w_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpavgw %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7d,0x48,0xe3,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpavgw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xe3,0xd1]
-; X64-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <32 x i16> @llvm.x86.avx512.pavg.w.512(<32 x i16> %x0, <32 x i16> %x1)
   %2 = bitcast i32 %x3 to <32 x i1>
   %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x2
-  %4 = call <32 x i16> @llvm.x86.avx512.pavg.w.512(<32 x i16> %x0, <32 x i16> %x1)
-  %res2 = add <32 x i16> %3, %4
-  ret <32 x i16> %res2
+  ret <32 x i16> %3
 }
 
 declare <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8>, <64 x i8>)
@@ -864,80 +874,95 @@ define <64 x i8>@test_int_x86_avx512_pshuf_b_512_maskz(<64 x i8> %x0, <64 x i8>
 
 declare <32 x i16> @llvm.x86.avx512.pmulhu.w.512(<32 x i16>, <32 x i16>)
 
+define <32 x i16> @test_int_x86_avx512_pmulhu_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_pmulhu_w_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmulhuw %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xe4,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <32 x i16> @llvm.x86.avx512.pmulhu.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  ret <32 x i16> %1
+}
+
 define <32 x i16> @test_int_x86_avx512_mask_pmulhu_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmulhu_w_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpmulhuw %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7d,0x48,0xe4,0xd9]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpmulhuw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xe4,0xd1]
-; X86-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmulhu_w_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpmulhuw %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7d,0x48,0xe4,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpmulhuw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xe4,0xd1]
-; X64-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <32 x i16> @llvm.x86.avx512.pmulhu.w.512(<32 x i16> %x0, <32 x i16> %x1)
   %2 = bitcast i32 %x3 to <32 x i1>
   %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x2
-  %4 = call <32 x i16> @llvm.x86.avx512.pmulhu.w.512(<32 x i16> %x0, <32 x i16> %x1)
-  %res2 = add <32 x i16> %3, %4
-  ret <32 x i16> %res2
+  ret <32 x i16> %3
 }
 
 declare <32 x i16> @llvm.x86.avx512.pmulh.w.512(<32 x i16>, <32 x i16>)
 
+define <32 x i16> @test_int_x86_avx512_pmulh_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_pmulh_w_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmulhw %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xe5,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <32 x i16> @llvm.x86.avx512.pmulh.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  ret <32 x i16> %1
+}
+
 define <32 x i16> @test_int_x86_avx512_mask_pmulh_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmulh_w_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpmulhw %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7d,0x48,0xe5,0xd9]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpmulhw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xe5,0xd1]
-; X86-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmulh_w_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpmulhw %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7d,0x48,0xe5,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpmulhw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xe5,0xd1]
-; X64-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <32 x i16> @llvm.x86.avx512.pmulh.w.512(<32 x i16> %x0, <32 x i16> %x1)
   %2 = bitcast i32 %x3 to <32 x i1>
   %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x2
-  %4 = call <32 x i16> @llvm.x86.avx512.pmulh.w.512(<32 x i16> %x0, <32 x i16> %x1)
-  %res2 = add <32 x i16> %3, %4
-  ret <32 x i16> %res2
+  ret <32 x i16> %3
 }
 
 declare <32 x i16> @llvm.x86.avx512.pmul.hr.sw.512(<32 x i16>, <32 x i16>)
 
+define <32 x i16> @test_int_x86_avx512_pmulhr_sw_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_pmulhr_sw_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmulhrsw %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf2,0x7d,0x48,0x0b,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <32 x i16> @llvm.x86.avx512.pmul.hr.sw.512(<32 x i16> %x0, <32 x i16> %x1)
+  ret <32 x i16> %1
+}
+
 define <32 x i16> @test_int_x86_avx512_mask_pmulhr_sw_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmulhr_sw_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpmulhrsw %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf2,0x7d,0x48,0x0b,0xd9]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpmulhrsw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x0b,0xd1]
-; X86-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmulhr_sw_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpmulhrsw %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf2,0x7d,0x48,0x0b,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpmulhrsw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x0b,0xd1]
-; X64-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <32 x i16> @llvm.x86.avx512.pmul.hr.sw.512(<32 x i16> %x0, <32 x i16> %x1)
   %2 = bitcast i32 %x3 to <32 x i1>
   %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x2
-  %4 = call <32 x i16> @llvm.x86.avx512.pmul.hr.sw.512(<32 x i16> %x0, <32 x i16> %x1)
-  %res2 = add <32 x i16> %3, %4
-  ret <32 x i16> %res2
+  ret <32 x i16> %3
 }
 
 define <32 x i8>@test_int_x86_avx512_pmov_wb_512(<32 x i16> %x0) {
@@ -1151,54 +1176,64 @@ define void @test_int_x86_avx512_mask_pmovus_wb_mem_512(i8* %ptr, <32 x i16> %x1
 
 declare <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8>, <64 x i8>)
 
+define <32 x i16> @test_int_x86_avx512_pmaddubs_w_512(<64 x i8> %x0, <64 x i8> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_pmaddubs_w_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmaddubsw %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf2,0x7d,0x48,0x04,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %x0, <64 x i8> %x1)
+  ret <32 x i16> %1
+}
+
 define <32 x i16> @test_int_x86_avx512_mask_pmaddubs_w_512(<64 x i8> %x0, <64 x i8> %x1, <32 x i16> %x2, i32 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmaddubs_w_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpmaddubsw %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf2,0x7d,0x48,0x04,0xd9]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpmaddubsw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x04,0xd1]
-; X86-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmaddubs_w_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpmaddubsw %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf2,0x7d,0x48,0x04,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpmaddubsw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x04,0xd1]
-; X64-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %x0, <64 x i8> %x1)
   %2 = bitcast i32 %x3 to <32 x i1>
   %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x2
-  %4 = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %x0, <64 x i8> %x1)
-  %res2 = add <32 x i16> %3, %4
-  ret <32 x i16> %res2
+  ret <32 x i16> %3
 }
 
 declare <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16>, <32 x i16>)
 
+define <16 x i32> @test_int_x86_avx512_pmaddw_d_512(<32 x i16> %x0, <32 x i16> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_pmaddw_d_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmaddwd %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xf5,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> %x0, <32 x i16> %x1)
+  ret <16 x i32> %1
+}
+
 define <16 x i32> @test_int_x86_avx512_mask_pmaddw_d_512(<32 x i16> %x0, <32 x i16> %x1, <16 x i32> %x2, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmaddw_d_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpmaddwd %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7d,0x48,0xf5,0xd9]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpmaddwd %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xf5,0xd1]
-; X86-NEXT:    vpaddd %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc3]
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmaddw_d_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpmaddwd %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf1,0x7d,0x48,0xf5,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpmaddwd %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xf5,0xd1]
-; X64-NEXT:    vpaddd %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc3]
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> %x0, <32 x i16> %x1)
   %2 = bitcast i16 %x3 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x2
-  %4 = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> %x0, <32 x i16> %x1)
-  %res2 = add <16 x i32> %3, %4
-  ret <16 x i32> %res2
+  ret <16 x i32> %3
 }
 
 declare <32 x i16> @llvm.x86.avx512.dbpsadbw.512(<64 x i8>, <64 x i8>, i32)
@@ -1206,30 +1241,30 @@ declare <32 x i16> @llvm.x86.avx512.dbpsadbw.512(<64 x i8>, <64 x i8>, i32)
 define <32 x i16>@test_int_x86_avx512_mask_dbpsadbw_512(<64 x i8> %x0, <64 x i8> %x1, <32 x i16> %x3, i32 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_dbpsadbw_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vdbpsadbw $2, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0x7d,0x48,0x42,0xd9,0x02]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vdbpsadbw $2, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x49,0x42,0xd1,0x02]
-; X86-NEXT:    vdbpsadbw $2, %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0xc9,0x42,0xc1,0x02]
-; X86-NEXT:    vpaddw %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc3]
+; X86-NEXT:    vdbpsadbw $3, %zmm1, %zmm0, %zmm3 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0xc9,0x42,0xd9,0x03]
+; X86-NEXT:    vdbpsadbw $4, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0x7d,0x48,0x42,0xc1,0x04]
+; X86-NEXT:    vpaddw %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfd,0xc0]
 ; X86-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_dbpsadbw_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vdbpsadbw $2, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0x7d,0x48,0x42,0xd9,0x02]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vdbpsadbw $2, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x49,0x42,0xd1,0x02]
-; X64-NEXT:    vdbpsadbw $2, %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0xc9,0x42,0xc1,0x02]
-; X64-NEXT:    vpaddw %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc3]
+; X64-NEXT:    vdbpsadbw $3, %zmm1, %zmm0, %zmm3 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0xc9,0x42,0xd9,0x03]
+; X64-NEXT:    vdbpsadbw $4, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0x7d,0x48,0x42,0xc1,0x04]
+; X64-NEXT:    vpaddw %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfd,0xc0]
 ; X64-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <32 x i16> @llvm.x86.avx512.dbpsadbw.512(<64 x i8> %x0, <64 x i8> %x1, i32 2)
   %2 = bitcast i32 %x4 to <32 x i1>
   %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x3
-  %4 = call <32 x i16> @llvm.x86.avx512.dbpsadbw.512(<64 x i8> %x0, <64 x i8> %x1, i32 2)
+  %4 = call <32 x i16> @llvm.x86.avx512.dbpsadbw.512(<64 x i8> %x0, <64 x i8> %x1, i32 3)
   %5 = bitcast i32 %x4 to <32 x i1>
   %6 = select <32 x i1> %5, <32 x i16> %4, <32 x i16> zeroinitializer
-  %7 = call <32 x i16> @llvm.x86.avx512.dbpsadbw.512(<64 x i8> %x0, <64 x i8> %x1, i32 2)
+  %7 = call <32 x i16> @llvm.x86.avx512.dbpsadbw.512(<64 x i8> %x0, <64 x i8> %x1, i32 4)
   %res3 = add <32 x i16> %3, %6
   %res4 = add <32 x i16> %res3, %7
   ret <32 x i16> %res4
@@ -1274,70 +1309,100 @@ define <32 x i16> @test_x86_avx512_psrlv_w_512_const() optsize {
   ret <32 x i16> %res1
 }
 
+define <32 x i16>@test_int_x86_avx512_psrlv32hi(<32 x i16> %x0, <32 x i16> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_psrlv32hi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf2,0xfd,0x48,0x10,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  ret <32 x i16> %1
+}
+
 define <32 x i16>@test_int_x86_avx512_mask_psrlv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrlv32hi:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf2,0xfd,0x48,0x10,0xd9]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x10,0xd1]
-; X86-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x10,0xc1]
-; X86-NEXT:    vpaddw %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc3]
-; X86-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrlv32hi:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf2,0xfd,0x48,0x10,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x10,0xd1]
-; X64-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x10,0xc1]
-; X64-NEXT:    vpaddw %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc3]
-; X64-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> %x0, <32 x i16> %x1)
   %2 = bitcast i32 %x3 to <32 x i1>
   %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x2
-  %4 = call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> %x0, <32 x i16> %x1)
-  %5 = bitcast i32 %x3 to <32 x i1>
-  %6 = select <32 x i1> %5, <32 x i16> %4, <32 x i16> zeroinitializer
-  %7 = call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> %x0, <32 x i16> %x1)
-  %res3 = add <32 x i16> %3, %6
-  %res4 = add <32 x i16> %res3, %7
-  ret <32 x i16> %res4
+  ret <32 x i16> %3
+}
+
+define <32 x i16>@test_int_x86_avx512_maskz_psrlv32hi(<32 x i16> %x0, <32 x i16> %x1, i32 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psrlv32hi:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x10,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psrlv32hi:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x10,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  %2 = bitcast i32 %x3 to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> zeroinitializer
+  ret <32 x i16> %3
 }
 
 declare <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16>, <32 x i16>)
 
+define <32 x i16>@test_int_x86_avx512_psrav32_hi(<32 x i16> %x0, <32 x i16> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_psrav32_hi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsravw %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf2,0xfd,0x48,0x11,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  ret <32 x i16> %1
+}
+
 define <32 x i16>@test_int_x86_avx512_mask_psrav32_hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrav32_hi:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsravw %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf2,0xfd,0x48,0x11,0xd9]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpsravw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x11,0xd1]
-; X86-NEXT:    vpsravw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x11,0xc1]
-; X86-NEXT:    vpaddw %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc3]
-; X86-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrav32_hi:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsravw %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf2,0xfd,0x48,0x11,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpsravw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x11,0xd1]
-; X64-NEXT:    vpsravw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x11,0xc1]
-; X64-NEXT:    vpaddw %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc3]
-; X64-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> %x0, <32 x i16> %x1)
   %2 = bitcast i32 %x3 to <32 x i1>
   %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x2
-  %4 = call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> %x0, <32 x i16> %x1)
-  %5 = bitcast i32 %x3 to <32 x i1>
-  %6 = select <32 x i1> %5, <32 x i16> %4, <32 x i16> zeroinitializer
-  %7 = call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> %x0, <32 x i16> %x1)
-  %res3 = add <32 x i16> %3, %6
-  %res4 = add <32 x i16> %res3, %7
-  ret <32 x i16> %res4
+  ret <32 x i16> %3
+}
+
+define <32 x i16>@test_int_x86_avx512_maskz_psrav32_hi(<32 x i16> %x0, <32 x i16> %x1, i32 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psrav32_hi:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsravw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x11,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psrav32_hi:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsravw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x11,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  %2 = bitcast i32 %x3 to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> zeroinitializer
+  ret <32 x i16> %3
 }
 
 define <32 x i16>@test_int_x86_avx512_mask_psrav32_hi_const(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
@@ -1362,36 +1427,51 @@ define <32 x i16>@test_int_x86_avx512_mask_psrav32_hi_const(<32 x i16> %x0, <32
   ret <32 x i16> %1
 }
 
+define <32 x i16>@test_int_x86_avx512_psllv32hi(<32 x i16> %x0, <32 x i16> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_psllv32hi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf2,0xfd,0x48,0x12,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  ret <32 x i16> %1
+}
+
 define <32 x i16>@test_int_x86_avx512_mask_psllv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psllv32hi:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsllvw %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf2,0xfd,0x48,0x12,0xd9]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpsllvw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x12,0xd1]
-; X86-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x12,0xc1]
-; X86-NEXT:    vpaddw %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc3]
-; X86-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psllv32hi:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsllvw %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf2,0xfd,0x48,0x12,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpsllvw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x12,0xd1]
-; X64-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x12,0xc1]
-; X64-NEXT:    vpaddw %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc3]
-; X64-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> %x0, <32 x i16> %x1)
   %2 = bitcast i32 %x3 to <32 x i1>
   %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x2
-  %4 = call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> %x0, <32 x i16> %x1)
-  %5 = bitcast i32 %x3 to <32 x i1>
-  %6 = select <32 x i1> %5, <32 x i16> %4, <32 x i16> zeroinitializer
-  %7 = call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> %x0, <32 x i16> %x1)
-  %res3 = add <32 x i16> %3, %6
-  %res4 = add <32 x i16> %res3, %7
-  ret <32 x i16> %res4
+  ret <32 x i16> %3
+}
+
+define <32 x i16>@test_int_x86_avx512_maskz_psllv32hi(<32 x i16> %x0, <32 x i16> %x1, i32 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psllv32hi:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x12,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psllv32hi:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x12,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  %2 = bitcast i32 %x3 to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> zeroinitializer
+  ret <32 x i16> %3
 }
 
 declare <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16>, <32 x i16>)
diff --git a/llvm/test/CodeGen/X86/avx512bwvl-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/avx512bwvl-intrinsics-fast-isel.ll
index effa39bb8f1f0..11dd987729d92 100644
--- a/llvm/test/CodeGen/X86/avx512bwvl-intrinsics-fast-isel.ll
+++ b/llvm/test/CodeGen/X86/avx512bwvl-intrinsics-fast-isel.ll
@@ -736,7 +736,7 @@ define <4 x i64> @test_mm256_maskz_broadcastw_epi16(i16 %a0, <2 x i64> %a1) {
 define <2 x i64> @test_mm_cvtepi16_epi8(<2 x i64> %__A) {
 ; CHECK-LABEL: test_mm_cvtepi16_epi8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14],zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT:    vpmovwb %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %0 = bitcast <2 x i64> %__A to <8 x i16>
diff --git a/llvm/test/CodeGen/X86/avx512bwvl-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512bwvl-intrinsics-upgrade.ll
index 1f4185ed6dc20..313f04e446a21 100644
--- a/llvm/test/CodeGen/X86/avx512bwvl-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512bwvl-intrinsics-upgrade.ll
@@ -2944,185 +2944,279 @@ define <16 x i16>@test_int_x86_avx512_maskz_pminu_w_256(<16 x i16> %x0, <16 x i1
 
 declare <8 x i16> @llvm.x86.avx512.mask.psrl.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
 
+define <8 x i16>@test_int_x86_avx512_psrl_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psrl_w_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd1,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psrl.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
+  ret <8 x i16> %res
+}
+
 define <8 x i16>@test_int_x86_avx512_mask_psrl_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrl_w_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsrlw %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd1,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpsrlw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xd1,0xd1]
-; X86-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xd1,0xc1]
-; X86-NEXT:    vpaddw %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfd,0xc0]
-; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrl_w_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsrlw %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd1,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpsrlw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xd1,0xd1]
-; X64-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xd1,0xc1]
-; X64-NEXT:    vpaddw %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfd,0xc0]
-; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i16> @llvm.x86.avx512.mask.psrl.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
-  %res1 = call <8 x i16> @llvm.x86.avx512.mask.psrl.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
-  %res2 = call <8 x i16> @llvm.x86.avx512.mask.psrl.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> zeroinitializer, i8 %x3)
-  %res3 = add <8 x i16> %res, %res1
-  %res4 = add <8 x i16> %res2, %res3
-  ret <8 x i16> %res4
+  ret <8 x i16> %res
+}
+
+
+define <8 x i16>@test_int_x86_avx512_maskz_psrl_w_128(<8 x i16> %x0, <8 x i16> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psrl_w_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xd1,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psrl_w_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xd1,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psrl.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> zeroinitializer, i8 %x3)
+  ret <8 x i16> %res
 }
 
 declare <16 x i16> @llvm.x86.avx512.mask.psrl.w.256(<16 x i16>, <8 x i16>, <16 x i16>, i16)
 
+define <16 x i16>@test_int_x86_avx512_psrl_w_256(<16 x i16> %x0, <8 x i16> %x1, <16 x i16> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psrl_w_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd1,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psrl.w.256(<16 x i16> %x0, <8 x i16> %x1, <16 x i16> %x2, i16 -1)
+  ret <16 x i16> %res
+}
+
 define <16 x i16>@test_int_x86_avx512_mask_psrl_w_256(<16 x i16> %x0, <8 x i16> %x1, <16 x i16> %x2, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrl_w_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsrlw %xmm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd1,0xd9]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpsrlw %xmm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xd1,0xd1]
-; X86-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xd1,0xc1]
-; X86-NEXT:    vpaddw %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfd,0xc0]
-; X86-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrl_w_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsrlw %xmm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd1,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpsrlw %xmm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xd1,0xd1]
-; X64-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xd1,0xc1]
-; X64-NEXT:    vpaddw %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfd,0xc0]
-; X64-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx512.mask.psrl.w.256(<16 x i16> %x0, <8 x i16> %x1, <16 x i16> %x2, i16 %x3)
-  %res1 = call <16 x i16> @llvm.x86.avx512.mask.psrl.w.256(<16 x i16> %x0, <8 x i16> %x1, <16 x i16> %x2, i16 -1)
-  %res2 = call <16 x i16> @llvm.x86.avx512.mask.psrl.w.256(<16 x i16> %x0, <8 x i16> %x1, <16 x i16> zeroinitializer, i16 %x3)
-  %res3 = add <16 x i16> %res, %res1
-  %res4 = add <16 x i16> %res3, %res2
-  ret <16 x i16> %res4
+  ret <16 x i16> %res
+}
+
+define <16 x i16>@test_int_x86_avx512_maskz_psrl_w_256(<16 x i16> %x0, <8 x i16> %x1, i16 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psrl_w_256:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xd1,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psrl_w_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xd1,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psrl.w.256(<16 x i16> %x0, <8 x i16> %x1, <16 x i16> zeroinitializer, i16 %x3)
+  ret <16 x i16> %res
 }
 
 declare <8 x i16> @llvm.x86.avx512.mask.psra.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
 
+define <8 x i16>@test_int_x86_avx512_psra_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psra_w_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsraw %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe1,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psra.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
+  ret <8 x i16> %res
+}
+
 define <8 x i16>@test_int_x86_avx512_mask_psra_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psra_w_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsraw %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe1,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpsraw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe1,0xd1]
-; X86-NEXT:    vpsraw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe1,0xc1]
-; X86-NEXT:    vpaddw %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc3]
-; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psra_w_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsraw %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe1,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpsraw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe1,0xd1]
-; X64-NEXT:    vpsraw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe1,0xc1]
-; X64-NEXT:    vpaddw %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc3]
-; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i16> @llvm.x86.avx512.mask.psra.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
-  %res1 = call <8 x i16> @llvm.x86.avx512.mask.psra.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> zeroinitializer, i8 %x3)
-  %res2 = call <8 x i16> @llvm.x86.avx512.mask.psra.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
-  %res3 = add <8 x i16> %res, %res1
-  %res4 = add <8 x i16> %res3, %res2
-  ret <8 x i16> %res4
+  ret <8 x i16> %res
+}
+
+define <8 x i16>@test_int_x86_avx512_maskz_psra_w_128(<8 x i16> %x0, <8 x i16> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psra_w_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpsraw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe1,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psra_w_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsraw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe1,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psra.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> zeroinitializer, i8 %x3)
+  ret <8 x i16> %res
 }
 
 declare <16 x i16> @llvm.x86.avx512.mask.psra.w.256(<16 x i16>, <8 x i16>, <16 x i16>, i16)
 
+define <16 x i16>@test_int_x86_avx512_psra_w_256(<16 x i16> %x0, <8 x i16> %x1, <16 x i16> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psra_w_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsraw %xmm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe1,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psra.w.256(<16 x i16> %x0, <8 x i16> %x1, <16 x i16> %x2, i16 -1)
+  ret <16 x i16> %res
+}
+
 define <16 x i16>@test_int_x86_avx512_mask_psra_w_256(<16 x i16> %x0, <8 x i16> %x1, <16 x i16> %x2, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psra_w_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsraw %xmm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe1,0xd9]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpsraw %xmm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe1,0xd1]
-; X86-NEXT:    vpsraw %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe1,0xc1]
-; X86-NEXT:    vpaddw %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc3]
-; X86-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psra_w_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsraw %xmm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe1,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpsraw %xmm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe1,0xd1]
-; X64-NEXT:    vpsraw %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe1,0xc1]
-; X64-NEXT:    vpaddw %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc3]
-; X64-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx512.mask.psra.w.256(<16 x i16> %x0, <8 x i16> %x1, <16 x i16> %x2, i16 %x3)
-  %res1 = call <16 x i16> @llvm.x86.avx512.mask.psra.w.256(<16 x i16> %x0, <8 x i16> %x1, <16 x i16> zeroinitializer, i16 %x3)
-  %res2 = call <16 x i16> @llvm.x86.avx512.mask.psra.w.256(<16 x i16> %x0, <8 x i16> %x1, <16 x i16> %x2, i16 -1)
-  %res3 = add <16 x i16> %res, %res1
-  %res4 = add <16 x i16> %res3, %res2
-  ret <16 x i16> %res4
+  ret <16 x i16> %res
+}
+
+define <16 x i16>@test_int_x86_avx512_maskz_psra_w_256(<16 x i16> %x0, <8 x i16> %x1, i16 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psra_w_256:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsraw %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe1,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psra_w_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsraw %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe1,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psra.w.256(<16 x i16> %x0, <8 x i16> %x1, <16 x i16> zeroinitializer, i16 %x3)
+  ret <16 x i16> %res
 }
 
 declare <8 x i16> @llvm.x86.avx512.mask.psll.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
 
+define <8 x i16>@test_int_x86_avx512_psll_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psll_w_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsllw %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xf1,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psll.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
+  ret <8 x i16> %res
+}
+
 define <8 x i16>@test_int_x86_avx512_mask_psll_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psll_w_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsllw %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xf1,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpsllw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xf1,0xd1]
-; X86-NEXT:    vpsllw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xf1,0xc1]
-; X86-NEXT:    vpaddw %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc3]
-; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psll_w_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsllw %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xf1,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpsllw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xf1,0xd1]
-; X64-NEXT:    vpsllw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xf1,0xc1]
-; X64-NEXT:    vpaddw %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc3]
-; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i16> @llvm.x86.avx512.mask.psll.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
-  %res1 = call <8 x i16> @llvm.x86.avx512.mask.psll.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> zeroinitializer, i8 %x3)
-  %res2 = call <8 x i16> @llvm.x86.avx512.mask.psll.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
-  %res3 = add <8 x i16> %res, %res1
-  %res4 = add <8 x i16> %res3, %res2
-  ret <8 x i16> %res4
+  ret <8 x i16> %res
+}
+
+define <8 x i16>@test_int_x86_avx512_maskz_psll_w_128(<8 x i16> %x0, <8 x i16> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psll_w_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpsllw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xf1,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psll_w_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsllw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xf1,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psll.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> zeroinitializer, i8 %x3)
+  ret <8 x i16> %res
 }
 
 declare <16 x i16> @llvm.x86.avx512.mask.psll.w.256(<16 x i16>, <8 x i16>, <16 x i16>, i16)
 
+define <16 x i16>@test_int_x86_avx512_psll_w_256(<16 x i16> %x0, <8 x i16> %x1, <16 x i16> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psll_w_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsllw %xmm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xf1,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psll.w.256(<16 x i16> %x0, <8 x i16> %x1, <16 x i16> %x2, i16 -1)
+  ret <16 x i16> %res
+}
+
 define <16 x i16>@test_int_x86_avx512_mask_psll_w_256(<16 x i16> %x0, <8 x i16> %x1, <16 x i16> %x2, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psll_w_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsllw %xmm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xf1,0xd9]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpsllw %xmm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xf1,0xd1]
-; X86-NEXT:    vpsllw %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xf1,0xc1]
-; X86-NEXT:    vpaddw %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc3]
-; X86-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psll_w_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsllw %xmm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xf1,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpsllw %xmm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xf1,0xd1]
-; X64-NEXT:    vpsllw %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xf1,0xc1]
-; X64-NEXT:    vpaddw %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc3]
-; X64-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx512.mask.psll.w.256(<16 x i16> %x0, <8 x i16> %x1, <16 x i16> %x2, i16 %x3)
-  %res1 = call <16 x i16> @llvm.x86.avx512.mask.psll.w.256(<16 x i16> %x0, <8 x i16> %x1, <16 x i16> zeroinitializer, i16 %x3)
-  %res2 = call <16 x i16> @llvm.x86.avx512.mask.psll.w.256(<16 x i16> %x0, <8 x i16> %x1, <16 x i16> %x2, i16 -1)
-  %res3 = add <16 x i16> %res, %res1
-  %res4 = add <16 x i16> %res3, %res2
-  ret <16 x i16> %res4
+  ret <16 x i16> %res
+}
+
+define <16 x i16>@test_int_x86_avx512_maskz_psll_w_256(<16 x i16> %x0, <8 x i16> %x1, i16 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psll_w_256:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsllw %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xf1,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psll_w_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsllw %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xf1,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psll.w.256(<16 x i16> %x0, <8 x i16> %x1, <16 x i16> zeroinitializer, i16 %x3)
+  ret <16 x i16> %res
 }
 
 declare <8 x i16> @llvm.x86.avx512.mask.psrl.wi.128(<8 x i16>, i32, <8 x i16>, i8)
@@ -3130,27 +3224,27 @@ declare <8 x i16> @llvm.x86.avx512.mask.psrl.wi.128(<8 x i16>, i32, <8 x i16>, i
 define <8 x i16>@test_int_x86_avx512_mask_psrl_wi_128(<8 x i16> %x0, i32 %x1, <8 x i16> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrl_wi_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsrlw $3, %xmm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0x71,0xd0,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpsrlw $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x09,0x71,0xd0,0x03]
-; X86-NEXT:    vpsrlw $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x71,0xd0,0x03]
+; X86-NEXT:    vpsrlw $4, %xmm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0x71,0xd0,0x04]
+; X86-NEXT:    vpsrlw $5, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x71,0xd0,0x05]
 ; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X86-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrl_wi_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsrlw $3, %xmm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0x71,0xd0,0x03]
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vpsrlw $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x09,0x71,0xd0,0x03]
-; X64-NEXT:    vpsrlw $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x71,0xd0,0x03]
+; X64-NEXT:    vpsrlw $4, %xmm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0x71,0xd0,0x04]
+; X64-NEXT:    vpsrlw $5, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x71,0xd0,0x05]
 ; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X64-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i16> @llvm.x86.avx512.mask.psrl.wi.128(<8 x i16> %x0, i32 3, <8 x i16> %x2, i8 %x3)
-  %res1 = call <8 x i16> @llvm.x86.avx512.mask.psrl.wi.128(<8 x i16> %x0, i32 3, <8 x i16> %x2, i8 -1)
-  %res2 = call <8 x i16> @llvm.x86.avx512.mask.psrl.wi.128(<8 x i16> %x0, i32 3, <8 x i16> zeroinitializer, i8 %x3)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.psrl.wi.128(<8 x i16> %x0, i32 4, <8 x i16> %x2, i8 -1)
+  %res2 = call <8 x i16> @llvm.x86.avx512.mask.psrl.wi.128(<8 x i16> %x0, i32 5, <8 x i16> zeroinitializer, i8 %x3)
   %res3 = add <8 x i16> %res, %res1
   %res4 = add <8 x i16> %res2, %res3
   ret <8 x i16> %res4
@@ -3161,26 +3255,26 @@ declare <16 x i16> @llvm.x86.avx512.mask.psrl.wi.256(<16 x i16>, i32, <16 x i16>
 define <16 x i16>@test_int_x86_avx512_mask_psrl_wi_256(<16 x i16> %x0, i32 %x1, <16 x i16> %x2, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrl_wi_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsrlw $3, %ymm0, %ymm2 # EVEX TO VEX Compression encoding: [0xc5,0xed,0x71,0xd0,0x03]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vpsrlw $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x75,0x29,0x71,0xd0,0x03]
-; X86-NEXT:    vpsrlw $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x71,0xd0,0x03]
+; X86-NEXT:    vpsrlw $4, %ymm0, %ymm2 # EVEX TO VEX Compression encoding: [0xc5,0xed,0x71,0xd0,0x04]
+; X86-NEXT:    vpsrlw $5, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x71,0xd0,0x05]
 ; X86-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
 ; X86-NEXT:    vpaddw %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrl_wi_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsrlw $3, %ymm0, %ymm2 # EVEX TO VEX Compression encoding: [0xc5,0xed,0x71,0xd0,0x03]
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vpsrlw $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x75,0x29,0x71,0xd0,0x03]
-; X64-NEXT:    vpsrlw $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x71,0xd0,0x03]
+; X64-NEXT:    vpsrlw $4, %ymm0, %ymm2 # EVEX TO VEX Compression encoding: [0xc5,0xed,0x71,0xd0,0x04]
+; X64-NEXT:    vpsrlw $5, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x71,0xd0,0x05]
 ; X64-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
 ; X64-NEXT:    vpaddw %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx512.mask.psrl.wi.256(<16 x i16> %x0, i32 3, <16 x i16> %x2, i16 %x3)
-  %res1 = call <16 x i16> @llvm.x86.avx512.mask.psrl.wi.256(<16 x i16> %x0, i32 3, <16 x i16> %x2, i16 -1)
-  %res2 = call <16 x i16> @llvm.x86.avx512.mask.psrl.wi.256(<16 x i16> %x0, i32 3, <16 x i16> zeroinitializer, i16 %x3)
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.psrl.wi.256(<16 x i16> %x0, i32 4, <16 x i16> %x2, i16 -1)
+  %res2 = call <16 x i16> @llvm.x86.avx512.mask.psrl.wi.256(<16 x i16> %x0, i32 5, <16 x i16> zeroinitializer, i16 %x3)
   %res3 = add <16 x i16> %res, %res1
   %res4 = add <16 x i16> %res3, %res2
   ret <16 x i16> %res4
@@ -3191,27 +3285,27 @@ declare <8 x i16> @llvm.x86.avx512.mask.psra.wi.128(<8 x i16>, i32, <8 x i16>, i
 define <8 x i16>@test_int_x86_avx512_mask_psra_wi_128(<8 x i16> %x0, i32 %x1, <8 x i16> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psra_wi_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsraw $3, %xmm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0x71,0xe0,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpsraw $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x09,0x71,0xe0,0x03]
-; X86-NEXT:    vpsraw $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x71,0xe0,0x03]
-; X86-NEXT:    vpaddw %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc2]
+; X86-NEXT:    vpsraw $4, %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0x89,0x71,0xe0,0x04]
+; X86-NEXT:    vpsraw $5, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x71,0xe0,0x05]
+; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X86-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psra_wi_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsraw $3, %xmm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0x71,0xe0,0x03]
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vpsraw $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x09,0x71,0xe0,0x03]
-; X64-NEXT:    vpsraw $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x71,0xe0,0x03]
-; X64-NEXT:    vpaddw %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc2]
+; X64-NEXT:    vpsraw $4, %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0x89,0x71,0xe0,0x04]
+; X64-NEXT:    vpsraw $5, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x71,0xe0,0x05]
+; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X64-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i16> @llvm.x86.avx512.mask.psra.wi.128(<8 x i16> %x0, i32 3, <8 x i16> %x2, i8 %x3)
-  %res1 = call <8 x i16> @llvm.x86.avx512.mask.psra.wi.128(<8 x i16> %x0, i32 3, <8 x i16> zeroinitializer, i8 %x3)
-  %res2 = call <8 x i16> @llvm.x86.avx512.mask.psra.wi.128(<8 x i16> %x0, i32 3, <8 x i16> %x2, i8 -1)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.psra.wi.128(<8 x i16> %x0, i32 4, <8 x i16> zeroinitializer, i8 %x3)
+  %res2 = call <8 x i16> @llvm.x86.avx512.mask.psra.wi.128(<8 x i16> %x0, i32 5, <8 x i16> %x2, i8 -1)
   %res3 = add <8 x i16> %res, %res1
   %res4 = add <8 x i16> %res3, %res2
   ret <8 x i16> %res4
@@ -3222,26 +3316,26 @@ declare <16 x i16> @llvm.x86.avx512.mask.psra.wi.256(<16 x i16>, i32, <16 x i16>
 define <16 x i16>@test_int_x86_avx512_mask_psra_wi_256(<16 x i16> %x0, i32 %x1, <16 x i16> %x2, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psra_wi_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsraw $3, %ymm0, %ymm2 # EVEX TO VEX Compression encoding: [0xc5,0xed,0x71,0xe0,0x03]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vpsraw $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x75,0x29,0x71,0xe0,0x03]
-; X86-NEXT:    vpsraw $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x71,0xe0,0x03]
-; X86-NEXT:    vpaddw %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc2]
+; X86-NEXT:    vpsraw $4, %ymm0, %ymm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0xa9,0x71,0xe0,0x04]
+; X86-NEXT:    vpsraw $5, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x71,0xe0,0x05]
+; X86-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
 ; X86-NEXT:    vpaddw %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psra_wi_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsraw $3, %ymm0, %ymm2 # EVEX TO VEX Compression encoding: [0xc5,0xed,0x71,0xe0,0x03]
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vpsraw $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x75,0x29,0x71,0xe0,0x03]
-; X64-NEXT:    vpsraw $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x71,0xe0,0x03]
-; X64-NEXT:    vpaddw %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc2]
+; X64-NEXT:    vpsraw $4, %ymm0, %ymm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0xa9,0x71,0xe0,0x04]
+; X64-NEXT:    vpsraw $5, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x71,0xe0,0x05]
+; X64-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
 ; X64-NEXT:    vpaddw %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx512.mask.psra.wi.256(<16 x i16> %x0, i32 3, <16 x i16> %x2, i16 %x3)
-  %res1 = call <16 x i16> @llvm.x86.avx512.mask.psra.wi.256(<16 x i16> %x0, i32 3, <16 x i16> zeroinitializer, i16 %x3)
-  %res2 = call <16 x i16> @llvm.x86.avx512.mask.psra.wi.256(<16 x i16> %x0, i32 3, <16 x i16> %x2, i16 -1)
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.psra.wi.256(<16 x i16> %x0, i32 4, <16 x i16> zeroinitializer, i16 %x3)
+  %res2 = call <16 x i16> @llvm.x86.avx512.mask.psra.wi.256(<16 x i16> %x0, i32 5, <16 x i16> %x2, i16 -1)
   %res3 = add <16 x i16> %res, %res1
   %res4 = add <16 x i16> %res3, %res2
   ret <16 x i16> %res4
@@ -3252,27 +3346,27 @@ declare <8 x i16> @llvm.x86.avx512.mask.psll.wi.128(<8 x i16>, i32, <8 x i16>, i
 define <8 x i16>@test_int_x86_avx512_mask_psll_wi_128(<8 x i16> %x0, i32 %x1, <8 x i16> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psll_wi_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsllw $3, %xmm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0x71,0xf0,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpsllw $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x09,0x71,0xf0,0x03]
-; X86-NEXT:    vpsllw $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x71,0xf0,0x03]
-; X86-NEXT:    vpaddw %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc2]
+; X86-NEXT:    vpsllw $4, %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0x89,0x71,0xf0,0x04]
+; X86-NEXT:    vpsllw $5, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x71,0xf0,0x05]
+; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X86-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psll_wi_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsllw $3, %xmm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0x71,0xf0,0x03]
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vpsllw $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x09,0x71,0xf0,0x03]
-; X64-NEXT:    vpsllw $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x71,0xf0,0x03]
-; X64-NEXT:    vpaddw %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc2]
+; X64-NEXT:    vpsllw $4, %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0x89,0x71,0xf0,0x04]
+; X64-NEXT:    vpsllw $5, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x71,0xf0,0x05]
+; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X64-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i16> @llvm.x86.avx512.mask.psll.wi.128(<8 x i16> %x0, i32 3, <8 x i16> %x2, i8 %x3)
-  %res1 = call <8 x i16> @llvm.x86.avx512.mask.psll.wi.128(<8 x i16> %x0, i32 3, <8 x i16> zeroinitializer, i8 %x3)
-  %res2 = call <8 x i16> @llvm.x86.avx512.mask.psll.wi.128(<8 x i16> %x0, i32 3, <8 x i16> %x2, i8 -1)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.psll.wi.128(<8 x i16> %x0, i32 4, <8 x i16> zeroinitializer, i8 %x3)
+  %res2 = call <8 x i16> @llvm.x86.avx512.mask.psll.wi.128(<8 x i16> %x0, i32 5, <8 x i16> %x2, i8 -1)
   %res3 = add <8 x i16> %res, %res1
   %res4 = add <8 x i16> %res3, %res2
   ret <8 x i16> %res4
@@ -3283,26 +3377,26 @@ declare <16 x i16> @llvm.x86.avx512.mask.psll.wi.256(<16 x i16>, i32, <16 x i16>
 define <16 x i16>@test_int_x86_avx512_mask_psll_wi_256(<16 x i16> %x0, i32 %x1, <16 x i16> %x2, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psll_wi_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsllw $3, %ymm0, %ymm2 # EVEX TO VEX Compression encoding: [0xc5,0xed,0x71,0xf0,0x03]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vpsllw $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x75,0x29,0x71,0xf0,0x03]
-; X86-NEXT:    vpsllw $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x71,0xf0,0x03]
-; X86-NEXT:    vpaddw %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc2]
+; X86-NEXT:    vpsllw $4, %ymm0, %ymm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0xa9,0x71,0xf0,0x04]
+; X86-NEXT:    vpsllw $5, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x71,0xf0,0x05]
+; X86-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
 ; X86-NEXT:    vpaddw %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psll_wi_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsllw $3, %ymm0, %ymm2 # EVEX TO VEX Compression encoding: [0xc5,0xed,0x71,0xf0,0x03]
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vpsllw $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x75,0x29,0x71,0xf0,0x03]
-; X64-NEXT:    vpsllw $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x71,0xf0,0x03]
-; X64-NEXT:    vpaddw %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc2]
+; X64-NEXT:    vpsllw $4, %ymm0, %ymm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0xa9,0x71,0xf0,0x04]
+; X64-NEXT:    vpsllw $5, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x71,0xf0,0x05]
+; X64-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
 ; X64-NEXT:    vpaddw %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx512.mask.psll.wi.256(<16 x i16> %x0, i32 3, <16 x i16> %x2, i16 %x3)
-  %res1 = call <16 x i16> @llvm.x86.avx512.mask.psll.wi.256(<16 x i16> %x0, i32 3, <16 x i16> zeroinitializer, i16 %x3)
-  %res2 = call <16 x i16> @llvm.x86.avx512.mask.psll.wi.256(<16 x i16> %x0, i32 3, <16 x i16> %x2, i16 -1)
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.psll.wi.256(<16 x i16> %x0, i32 4, <16 x i16> zeroinitializer, i16 %x3)
+  %res2 = call <16 x i16> @llvm.x86.avx512.mask.psll.wi.256(<16 x i16> %x0, i32 5, <16 x i16> %x2, i16 -1)
   %res3 = add <16 x i16> %res, %res1
   %res4 = add <16 x i16> %res3, %res2
   ret <16 x i16> %res4
@@ -5853,196 +5947,236 @@ define <8 x i8> @test_mask_ucmp_w_128(<8 x i16> %a0, <8 x i16> %a1, i8 %mask) {
 
 declare i8 @llvm.x86.avx512.mask.ucmp.w.128(<8 x i16>, <8 x i16>, i32, i8) nounwind readnone
 
+define <16 x i8>@mm_avg_epu8(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2) {
+; CHECK-LABEL: mm_avg_epu8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpavgb %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe0,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <16 x i8> @llvm.x86.avx512.mask.pavg.b.128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 -1)
+  ret <16 x i8> %res
+}
+
 define <16 x i8>@mm_mask_avg_epu8(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 %x3) {
 ; X86-LABEL: mm_mask_avg_epu8:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpavgb %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe0,0xd9]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpavgb %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe0,0xd1]
-; X86-NEXT:    vpaddb %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfc,0xc3]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: mm_mask_avg_epu8:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpavgb %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe0,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpavgb %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe0,0xd1]
-; X64-NEXT:    vpaddb %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfc,0xc3]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <16 x i8> @llvm.x86.avx512.mask.pavg.b.128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 %x3)
-  %res1 = call <16 x i8> @llvm.x86.avx512.mask.pavg.b.128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 -1)
-   %res2 = add <16 x i8> %res, %res1
-   ret <16 x i8> %res2
+  ret <16 x i8> %res
 }
 
 declare <16 x i8> @llvm.x86.avx512.mask.pabs.b.128(<16 x i8>, <16 x i8>, i16)
 
+define <16 x i8>@test_int_x86_avx512_pabs_b_128(<16 x i8> %x0, <16 x i8> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_pabs_b_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpabsb %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x1c,0xc0]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <16 x i8> @llvm.x86.avx512.mask.pabs.b.128(<16 x i8> %x0, <16 x i8> %x1, i16 -1)
+  ret <16 x i8> %res
+}
+
 define <16 x i8>@test_int_x86_avx512_mask_pabs_b_128(<16 x i8> %x0, <16 x i8> %x1, i16 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_pabs_b_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpabsb %xmm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x1c,0xd0]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpabsb %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x1c,0xc8]
-; X86-NEXT:    vpaddb %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfc,0xc2]
+; X86-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pabs_b_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpabsb %xmm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x1c,0xd0]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpabsb %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x1c,0xc8]
-; X64-NEXT:    vpaddb %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfc,0xc2]
+; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <16 x i8> @llvm.x86.avx512.mask.pabs.b.128(<16 x i8> %x0, <16 x i8> %x1, i16 %x2)
-  %res1 = call <16 x i8> @llvm.x86.avx512.mask.pabs.b.128(<16 x i8> %x0, <16 x i8> %x1, i16 -1)
-  %res2 = add <16 x i8> %res, %res1
-  ret <16 x i8> %res2
+  ret <16 x i8> %res
 }
 
 declare <16 x i8> @llvm.x86.avx512.mask.pavg.b.128(<16 x i8>, <16 x i8>, <16 x i8>, i16)
 
+define <32 x i8>@mm256_avg_epu8(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 %x3) {
+; CHECK-LABEL: mm256_avg_epu8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpavgb %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe0,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <32 x i8> @llvm.x86.avx512.mask.pavg.b.256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 -1)
+  ret <32 x i8> %res
+}
+
 define <32 x i8>@mm256_mask_avg_epu8(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 %x3) {
 ; X86-LABEL: mm256_mask_avg_epu8:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpavgb %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe0,0xd9]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpavgb %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe0,0xd1]
-; X86-NEXT:    vpaddb %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfc,0xc3]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: mm256_mask_avg_epu8:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpavgb %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe0,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpavgb %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe0,0xd1]
-; X64-NEXT:    vpaddb %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfc,0xc3]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <32 x i8> @llvm.x86.avx512.mask.pavg.b.256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 %x3)
-  %res1 = call <32 x i8> @llvm.x86.avx512.mask.pavg.b.256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 -1)
-  %res2 = add <32 x i8> %res, %res1
-  ret <32 x i8> %res2
+  ret <32 x i8> %res
 }
 
 declare <32 x i8> @llvm.x86.avx512.mask.pabs.b.256(<32 x i8>, <32 x i8>, i32)
 
+define <32 x i8>@test_int_x86_avx512_pabs_b_256(<32 x i8> %x0, <32 x i8> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_pabs_b_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpabsb %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x1c,0xc0]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <32 x i8> @llvm.x86.avx512.mask.pabs.b.256(<32 x i8> %x0, <32 x i8> %x1, i32 -1)
+  ret <32 x i8> %res
+}
+
 define <32 x i8>@test_int_x86_avx512_mask_pabs_b_256(<32 x i8> %x0, <32 x i8> %x1, i32 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_pabs_b_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpabsb %ymm0, %ymm2 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x1c,0xd0]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpabsb %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x1c,0xc8]
-; X86-NEXT:    vpaddb %ymm2, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfc,0xc2]
+; X86-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pabs_b_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpabsb %ymm0, %ymm2 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x1c,0xd0]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpabsb %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x1c,0xc8]
-; X64-NEXT:    vpaddb %ymm2, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfc,0xc2]
+; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <32 x i8> @llvm.x86.avx512.mask.pabs.b.256(<32 x i8> %x0, <32 x i8> %x1, i32 %x2)
-  %res1 = call <32 x i8> @llvm.x86.avx512.mask.pabs.b.256(<32 x i8> %x0, <32 x i8> %x1, i32 -1)
-  %res2 = add <32 x i8> %res, %res1
-  ret <32 x i8> %res2
+  ret <32 x i8> %res
 }
 
 declare <32 x i8> @llvm.x86.avx512.mask.pavg.b.256(<32 x i8>, <32 x i8>, <32 x i8>, i32)
 
+define <8 x i16>@mm_avg_epu16(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2) {
+; CHECK-LABEL: mm_avg_epu16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpavgw %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe3,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.pavg.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
+  ret <8 x i16> %res
+}
+
 define <8 x i16>@mm_mask_avg_epu16(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
 ; X86-LABEL: mm_mask_avg_epu16:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpavgw %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe3,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpavgw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe3,0xd1]
-; X86-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: mm_mask_avg_epu16:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpavgw %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe3,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpavgw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe3,0xd1]
-; X64-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i16> @llvm.x86.avx512.mask.pavg.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
-  %res1 = call <8 x i16> @llvm.x86.avx512.mask.pavg.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
-  %res2 = add <8 x i16> %res, %res1
-  ret <8 x i16> %res2
+  ret <8 x i16> %res
 }
 
 declare <8 x i16> @llvm.x86.avx512.mask.pabs.w.128(<8 x i16>, <8 x i16>, i8)
 
+define <8 x i16>@test_int_x86_avx512_pabs_w_128(<8 x i16> %x0, <8 x i16> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_pabs_w_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpabsw %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x1d,0xc0]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.pabs.w.128(<8 x i16> %x0, <8 x i16> %x1, i8 -1)
+  ret <8 x i16> %res
+}
+
 define <8 x i16>@test_int_x86_avx512_mask_pabs_w_128(<8 x i16> %x0, <8 x i16> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_pabs_w_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpabsw %xmm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x1d,0xd0]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpabsw %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x1d,0xc8]
-; X86-NEXT:    vpaddw %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfd,0xc2]
+; X86-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pabs_w_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpabsw %xmm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x1d,0xd0]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpabsw %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x1d,0xc8]
-; X64-NEXT:    vpaddw %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfd,0xc2]
+; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i16> @llvm.x86.avx512.mask.pabs.w.128(<8 x i16> %x0, <8 x i16> %x1, i8 %x2)
-  %res1 = call <8 x i16> @llvm.x86.avx512.mask.pabs.w.128(<8 x i16> %x0, <8 x i16> %x1, i8 -1)
-  %res2 = add <8 x i16> %res, %res1
-  ret <8 x i16> %res2
+  ret <8 x i16> %res
 }
 
 declare <8 x i16> @llvm.x86.avx512.mask.pavg.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
 
+define <16 x i16>@mm256_avg_epu16(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2) {
+; CHECK-LABEL: mm256_avg_epu16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpavgw %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe3,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.pavg.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 -1)
+  ret <16 x i16> %res
+}
+
 define <16 x i16>@mm256_mask_avg_epu16(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3) {
 ; X86-LABEL: mm256_mask_avg_epu16:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpavgw %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe3,0xd9]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpavgw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe3,0xd1]
-; X86-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: mm256_mask_avg_epu16:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpavgw %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe3,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpavgw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe3,0xd1]
-; X64-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx512.mask.pavg.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3)
-  %res1 = call <16 x i16> @llvm.x86.avx512.mask.pavg.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 -1)
-  %res2 = add <16 x i16> %res, %res1
-  ret <16 x i16> %res2
+  ret <16 x i16> %res
 }
 
 declare <16 x i16> @llvm.x86.avx512.mask.pabs.w.256(<16 x i16>, <16 x i16>, i16)
 
+define <16 x i16>@test_int_x86_avx512_pabs_w_256(<16 x i16> %x0, <16 x i16> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_pabs_w_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpabsw %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x1d,0xc0]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.pabs.w.256(<16 x i16> %x0, <16 x i16> %x1, i16 -1)
+  ret <16 x i16> %res
+}
+
 define <16 x i16>@test_int_x86_avx512_mask_pabs_w_256(<16 x i16> %x0, <16 x i16> %x1, i16 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_pabs_w_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpabsw %ymm0, %ymm2 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x1d,0xd0]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpabsw %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x1d,0xc8]
-; X86-NEXT:    vpaddw %ymm2, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfd,0xc2]
+; X86-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pabs_w_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpabsw %ymm0, %ymm2 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x1d,0xd0]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpabsw %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x1d,0xc8]
-; X64-NEXT:    vpaddw %ymm2, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfd,0xc2]
+; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx512.mask.pabs.w.256(<16 x i16> %x0, <16 x i16> %x1, i16 %x2)
-  %res1 = call <16 x i16> @llvm.x86.avx512.mask.pabs.w.256(<16 x i16> %x0, <16 x i16> %x1, i16 -1)
-  %res2 = add <16 x i16> %res, %res1
-  ret <16 x i16> %res2
+  ret <16 x i16> %res
 }
 
 declare <16 x i16> @llvm.x86.avx512.mask.pavg.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
@@ -6866,27 +7000,27 @@ declare <8 x i16> @llvm.x86.avx512.mask.dbpsadbw.128(<16 x i8>, <16 x i8>, i32,
 define <8 x i16>@test_int_x86_avx512_mask_dbpsadbw_128(<16 x i8> %x0, <16 x i8> %x1, <8 x i16> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_dbpsadbw_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vdbpsadbw $2, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x42,0xd9,0x02]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vdbpsadbw $2, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x42,0xd1,0x02]
-; X86-NEXT:    vdbpsadbw $2, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x42,0xc1,0x02]
-; X86-NEXT:    vpaddw %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc3]
+; X86-NEXT:    vdbpsadbw $3, %xmm1, %xmm0, %xmm3 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x42,0xd9,0x03]
+; X86-NEXT:    vdbpsadbw $4, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0x7d,0x08,0x42,0xc1,0x04]
+; X86-NEXT:    vpaddw %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfd,0xc0]
 ; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_dbpsadbw_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vdbpsadbw $2, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x42,0xd9,0x02]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vdbpsadbw $2, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x42,0xd1,0x02]
-; X64-NEXT:    vdbpsadbw $2, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x42,0xc1,0x02]
-; X64-NEXT:    vpaddw %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc3]
+; X64-NEXT:    vdbpsadbw $3, %xmm1, %xmm0, %xmm3 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x42,0xd9,0x03]
+; X64-NEXT:    vdbpsadbw $4, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0x7d,0x08,0x42,0xc1,0x04]
+; X64-NEXT:    vpaddw %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfd,0xc0]
 ; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i16> @llvm.x86.avx512.mask.dbpsadbw.128(<16 x i8> %x0, <16 x i8> %x1, i32 2, <8 x i16> %x3, i8 %x4)
-  %res1 = call <8 x i16> @llvm.x86.avx512.mask.dbpsadbw.128(<16 x i8> %x0, <16 x i8> %x1, i32 2, <8 x i16> zeroinitializer, i8 %x4)
-  %res2 = call <8 x i16> @llvm.x86.avx512.mask.dbpsadbw.128(<16 x i8> %x0, <16 x i8> %x1, i32 2, <8 x i16> %x3, i8 -1)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.dbpsadbw.128(<16 x i8> %x0, <16 x i8> %x1, i32 3, <8 x i16> zeroinitializer, i8 %x4)
+  %res2 = call <8 x i16> @llvm.x86.avx512.mask.dbpsadbw.128(<16 x i8> %x0, <16 x i8> %x1, i32 4, <8 x i16> %x3, i8 -1)
   %res3 = add <8 x i16> %res, %res1
   %res4 = add <8 x i16> %res2, %res3
   ret <8 x i16> %res4
@@ -6897,26 +7031,26 @@ declare <16 x i16> @llvm.x86.avx512.mask.dbpsadbw.256(<32 x i8>, <32 x i8>, i32,
 define <16 x i16>@test_int_x86_avx512_mask_dbpsadbw_256(<32 x i8> %x0, <32 x i8> %x1, <16 x i16> %x3, i16 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_dbpsadbw_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vdbpsadbw $2, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0x7d,0x28,0x42,0xd9,0x02]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vdbpsadbw $2, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x42,0xd1,0x02]
-; X86-NEXT:    vdbpsadbw $2, %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0xa9,0x42,0xc1,0x02]
-; X86-NEXT:    vpaddw %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc3]
+; X86-NEXT:    vdbpsadbw $3, %ymm1, %ymm0, %ymm3 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0xa9,0x42,0xd9,0x03]
+; X86-NEXT:    vdbpsadbw $4, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0x7d,0x28,0x42,0xc1,0x04]
+; X86-NEXT:    vpaddw %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfd,0xc0]
 ; X86-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_dbpsadbw_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vdbpsadbw $2, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0x7d,0x28,0x42,0xd9,0x02]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vdbpsadbw $2, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x42,0xd1,0x02]
-; X64-NEXT:    vdbpsadbw $2, %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0xa9,0x42,0xc1,0x02]
-; X64-NEXT:    vpaddw %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc3]
+; X64-NEXT:    vdbpsadbw $3, %ymm1, %ymm0, %ymm3 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0xa9,0x42,0xd9,0x03]
+; X64-NEXT:    vdbpsadbw $4, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0x7d,0x28,0x42,0xc1,0x04]
+; X64-NEXT:    vpaddw %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfd,0xc0]
 ; X64-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx512.mask.dbpsadbw.256(<32 x i8> %x0, <32 x i8> %x1, i32 2, <16 x i16> %x3, i16 %x4)
-  %res1 = call <16 x i16> @llvm.x86.avx512.mask.dbpsadbw.256(<32 x i8> %x0, <32 x i8> %x1, i32 2, <16 x i16> zeroinitializer, i16 %x4)
-  %res2 = call <16 x i16> @llvm.x86.avx512.mask.dbpsadbw.256(<32 x i8> %x0, <32 x i8> %x1, i32 2, <16 x i16> %x3, i16 -1)
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.dbpsadbw.256(<32 x i8> %x0, <32 x i8> %x1, i32 3, <16 x i16> zeroinitializer, i16 %x4)
+  %res2 = call <16 x i16> @llvm.x86.avx512.mask.dbpsadbw.256(<32 x i8> %x0, <32 x i8> %x1, i32 4, <16 x i16> %x3, i16 -1)
   %res3 = add <16 x i16> %res, %res1
   %res4 = add <16 x i16> %res3, %res2
   ret <16 x i16> %res4
@@ -9204,185 +9338,278 @@ declare <32 x i8> @llvm.x86.avx512.mask.psubs.b.256(<32 x i8>, <32 x i8>, <32 x
 
 declare <16 x i16> @llvm.x86.avx512.mask.psrav16.hi(<16 x i16>, <16 x i16>, <16 x i16>, i16)
 
+define <16 x i16>@test_int_x86_avx512_psrav16_hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psrav16_hi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsravw %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x11,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psrav16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 -1)
+  ret <16 x i16> %res
+}
+
 define <16 x i16>@test_int_x86_avx512_mask_psrav16_hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrav16_hi:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsravw %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x11,0xd9]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpsravw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x11,0xd1]
-; X86-NEXT:    vpsravw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x11,0xc1]
-; X86-NEXT:    vpaddw %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc3]
-; X86-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrav16_hi:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsravw %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x11,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpsravw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x11,0xd1]
-; X64-NEXT:    vpsravw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x11,0xc1]
-; X64-NEXT:    vpaddw %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc3]
-; X64-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx512.mask.psrav16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3)
-  %res1 = call <16 x i16> @llvm.x86.avx512.mask.psrav16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> zeroinitializer, i16 %x3)
-  %res2 = call <16 x i16> @llvm.x86.avx512.mask.psrav16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 -1)
-  %res3 = add <16 x i16> %res, %res1
-  %res4 = add <16 x i16> %res3, %res2
-  ret <16 x i16> %res4
+  ret <16 x i16> %res
+}
+
+define <16 x i16>@test_int_x86_avx512_maskz_psrav16_hi(<16 x i16> %x0, <16 x i16> %x1, i16 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psrav16_hi:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsravw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x11,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psrav16_hi:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsravw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x11,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psrav16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> zeroinitializer, i16 %x3)
+  ret <16 x i16> %res
 }
 
 declare <8 x i16> @llvm.x86.avx512.mask.psrav8.hi(<8 x i16>, <8 x i16>, <8 x i16>, i8)
 
+define <8 x i16>@test_int_x86_avx512_psrav8_hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psrav8_hi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsravw %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x11,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psrav8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
+  ret <8 x i16> %res
+}
+
 define <8 x i16>@test_int_x86_avx512_mask_psrav8_hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrav8_hi:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsravw %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x11,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpsravw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x11,0xd1]
-; X86-NEXT:    vpsravw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x11,0xc1]
-; X86-NEXT:    vpaddw %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc3]
-; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrav8_hi:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsravw %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x11,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpsravw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x11,0xd1]
-; X64-NEXT:    vpsravw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x11,0xc1]
-; X64-NEXT:    vpaddw %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc3]
-; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i16> @llvm.x86.avx512.mask.psrav8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
-  %res1 = call <8 x i16> @llvm.x86.avx512.mask.psrav8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> zeroinitializer, i8 %x3)
-  %res2 = call <8 x i16> @llvm.x86.avx512.mask.psrav8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
-  %res3 = add <8 x i16> %res, %res1
-  %res4 = add <8 x i16> %res3, %res2
-  ret <8 x i16> %res4
+  ret <8 x i16> %res
+}
+
+define <8 x i16>@test_int_x86_avx512_maskz_psrav8_hi(<8 x i16> %x0, <8 x i16> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psrav8_hi:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpsravw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x11,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psrav8_hi:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsravw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x11,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psrav8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> zeroinitializer, i8 %x3)
+  ret <8 x i16> %res
 }
 
 declare <16 x i16> @llvm.x86.avx512.mask.psllv16.hi(<16 x i16>, <16 x i16>, <16 x i16>, i16)
 
+define <16 x i16>@test_int_x86_avx512_psllv16_hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psllv16_hi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsllvw %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x12,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psllv16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 -1)
+  ret <16 x i16> %res
+}
+
 define <16 x i16>@test_int_x86_avx512_mask_psllv16_hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psllv16_hi:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsllvw %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x12,0xd9]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpsllvw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x12,0xd1]
-; X86-NEXT:    vpsllvw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x12,0xc1]
-; X86-NEXT:    vpaddw %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc3]
-; X86-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psllv16_hi:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsllvw %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x12,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpsllvw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x12,0xd1]
-; X64-NEXT:    vpsllvw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x12,0xc1]
-; X64-NEXT:    vpaddw %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc3]
-; X64-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx512.mask.psllv16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3)
-  %res1 = call <16 x i16> @llvm.x86.avx512.mask.psllv16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> zeroinitializer, i16 %x3)
-  %res2 = call <16 x i16> @llvm.x86.avx512.mask.psllv16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 -1)
-  %res3 = add <16 x i16> %res, %res1
-  %res4 = add <16 x i16> %res3, %res2
-  ret <16 x i16> %res4
+  ret <16 x i16> %res
+}
+
+define <16 x i16>@test_int_x86_avx512_maskz_psllv16_hi(<16 x i16> %x0, <16 x i16> %x1, i16 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psllv16_hi:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsllvw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x12,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psllv16_hi:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsllvw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x12,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psllv16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> zeroinitializer, i16 %x3)
+  ret <16 x i16> %res
 }
 
 declare <8 x i16> @llvm.x86.avx512.mask.psllv8.hi(<8 x i16>, <8 x i16>, <8 x i16>, i8)
 
+define <8 x i16>@test_int_x86_avx512_psllv8_hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psllv8_hi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsllvw %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x12,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psllv8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
+  ret <8 x i16> %res
+}
+
 define <8 x i16>@test_int_x86_avx512_mask_psllv8_hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psllv8_hi:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsllvw %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x12,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpsllvw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x12,0xd1]
-; X86-NEXT:    vpsllvw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x12,0xc1]
-; X86-NEXT:    vpaddw %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc3]
-; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psllv8_hi:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsllvw %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x12,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpsllvw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x12,0xd1]
-; X64-NEXT:    vpsllvw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x12,0xc1]
-; X64-NEXT:    vpaddw %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc3]
-; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i16> @llvm.x86.avx512.mask.psllv8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
-  %res1 = call <8 x i16> @llvm.x86.avx512.mask.psllv8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> zeroinitializer, i8 %x3)
-  %res2 = call <8 x i16> @llvm.x86.avx512.mask.psllv8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
-  %res3 = add <8 x i16> %res, %res1
-  %res4 = add <8 x i16> %res3, %res2
-  ret <8 x i16> %res4
+  ret <8 x i16> %res
+}
+
+define <8 x i16>@test_int_x86_avx512_maskz_psllv8_hi(<8 x i16> %x0, <8 x i16> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psllv8_hi:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpsllvw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x12,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psllv8_hi:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsllvw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x12,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psllv8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> zeroinitializer, i8 %x3)
+  ret <8 x i16> %res
 }
 
 declare <16 x i16> @llvm.x86.avx512.mask.psrlv16.hi(<16 x i16>, <16 x i16>, <16 x i16>, i16)
 
+define <16 x i16>@test_int_x86_avx512_psrlv16_hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psrlv16_hi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x10,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psrlv16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 -1)
+  ret <16 x i16> %res
+}
+
 define <16 x i16>@test_int_x86_avx512_mask_psrlv16_hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrlv16_hi:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x10,0xd9]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x10,0xd1]
-; X86-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x10,0xc1]
-; X86-NEXT:    vpaddw %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc3]
-; X86-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrlv16_hi:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x10,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x10,0xd1]
-; X64-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x10,0xc1]
-; X64-NEXT:    vpaddw %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc3]
-; X64-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx512.mask.psrlv16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3)
-  %res1 = call <16 x i16> @llvm.x86.avx512.mask.psrlv16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> zeroinitializer, i16 %x3)
-  %res2 = call <16 x i16> @llvm.x86.avx512.mask.psrlv16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 -1)
-  %res3 = add <16 x i16> %res, %res1
-  %res4 = add <16 x i16> %res3, %res2
-  ret <16 x i16> %res4
+  ret <16 x i16> %res
+}
+
+define <16 x i16>@test_int_x86_avx512_maskz_psrlv16_hi(<16 x i16> %x0, <16 x i16> %x1, i16 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psrlv16_hi:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x10,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psrlv16_hi:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x10,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psrlv16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> zeroinitializer, i16 %x3)
+  ret <16 x i16> %res
 }
 
 declare <8 x i16> @llvm.x86.avx512.mask.psrlv8.hi(<8 x i16>, <8 x i16>, <8 x i16>, i8)
 
+define <8 x i16>@test_int_x86_avx512_psrlv8_hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psrlv8_hi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x10,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psrlv8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
+  ret <8 x i16> %res
+}
+
 define <8 x i16>@test_int_x86_avx512_mask_psrlv8_hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrlv8_hi:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x10,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x10,0xd1]
-; X86-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x10,0xc1]
-; X86-NEXT:    vpaddw %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc3]
-; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrlv8_hi:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x10,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x10,0xd1]
-; X64-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x10,0xc1]
-; X64-NEXT:    vpaddw %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc3]
-; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i16> @llvm.x86.avx512.mask.psrlv8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
-  %res1 = call <8 x i16> @llvm.x86.avx512.mask.psrlv8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> zeroinitializer, i8 %x3)
-  %res2 = call <8 x i16> @llvm.x86.avx512.mask.psrlv8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
-  %res3 = add <8 x i16> %res, %res1
-  %res4 = add <8 x i16> %res3, %res2
-  ret <8 x i16> %res4
+  ret <8 x i16> %res
+}
+
+define <8 x i16>@test_int_x86_avx512_maskz_psrlv8_hi(<8 x i16> %x0, <8 x i16> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psrlv8_hi:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x10,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psrlv8_hi:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x10,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psrlv8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> zeroinitializer, i8 %x3)
+  ret <8 x i16> %res
 }
 
 declare <16 x i8> @llvm.x86.avx512.mask.pmov.wb.256(<16 x i16>, <16 x i8>, i16)
diff --git a/llvm/test/CodeGen/X86/avx512bwvl-intrinsics.ll b/llvm/test/CodeGen/X86/avx512bwvl-intrinsics.ll
index 73008582aca28..ab1af2c44a5d1 100644
--- a/llvm/test/CodeGen/X86/avx512bwvl-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512bwvl-intrinsics.ll
@@ -1291,25 +1291,21 @@ declare <16 x i8> @llvm.x86.sse2.pavg.b(<16 x i8>, <16 x i8>)
 define <16 x i8> @test_int_x86_avx512_mask_pavg_b_128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_pavg_b_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpavgb %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe0,0xd9]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpavgb %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe0,0xd1]
-; X86-NEXT:    vpaddb %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfc,0xc3]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pavg_b_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpavgb %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe0,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpavgb %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe0,0xd1]
-; X64-NEXT:    vpaddb %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfc,0xc3]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <16 x i8> @llvm.x86.sse2.pavg.b(<16 x i8> %x0, <16 x i8> %x1)
   %2 = bitcast i16 %x3 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> %x2
-  %4 = call <16 x i8> @llvm.x86.sse2.pavg.b(<16 x i8> %x0, <16 x i8> %x1)
-  %res2 = add <16 x i8> %3, %4
-  ret <16 x i8> %res2
+  ret <16 x i8> %3
 }
 
 declare <32 x i8> @llvm.x86.avx2.pavg.b(<32 x i8>, <32 x i8>)
@@ -1317,25 +1313,21 @@ declare <32 x i8> @llvm.x86.avx2.pavg.b(<32 x i8>, <32 x i8>)
 define <32 x i8> @test_int_x86_avx512_mask_pavg_b_256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_pavg_b_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpavgb %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe0,0xd9]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpavgb %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe0,0xd1]
-; X86-NEXT:    vpaddb %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfc,0xc3]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pavg_b_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpavgb %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe0,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpavgb %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe0,0xd1]
-; X64-NEXT:    vpaddb %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfc,0xc3]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <32 x i8> @llvm.x86.avx2.pavg.b(<32 x i8> %x0, <32 x i8> %x1)
   %2 = bitcast i32 %x3 to <32 x i1>
   %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> %x2
-  %4 = call <32 x i8> @llvm.x86.avx2.pavg.b(<32 x i8> %x0, <32 x i8> %x1)
-  %res2 = add <32 x i8> %3, %4
-  ret <32 x i8> %res2
+  ret <32 x i8> %3
 }
 
 declare <8 x i16> @llvm.x86.sse2.pavg.w(<8 x i16>, <8 x i16>)
@@ -1343,26 +1335,22 @@ declare <8 x i16> @llvm.x86.sse2.pavg.w(<8 x i16>, <8 x i16>)
 define <8 x i16> @test_int_x86_avx512_mask_pavg_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_pavg_w_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpavgw %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe3,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpavgw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe3,0xd1]
-; X86-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pavg_w_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpavgw %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe3,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpavgw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe3,0xd1]
-; X64-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <8 x i16> @llvm.x86.sse2.pavg.w(<8 x i16> %x0, <8 x i16> %x1)
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %x2
-  %4 = call <8 x i16> @llvm.x86.sse2.pavg.w(<8 x i16> %x0, <8 x i16> %x1)
-  %res2 = add <8 x i16> %3, %4
-  ret <8 x i16> %res2
+  ret <8 x i16> %3
 }
 
 declare <16 x i16> @llvm.x86.avx2.pavg.w(<16 x i16>, <16 x i16>)
@@ -1370,25 +1358,21 @@ declare <16 x i16> @llvm.x86.avx2.pavg.w(<16 x i16>, <16 x i16>)
 define <16 x i16> @test_int_x86_avx512_mask_pavg_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_pavg_w_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpavgw %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe3,0xd9]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpavgw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe3,0xd1]
-; X86-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pavg_w_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpavgw %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe3,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpavgw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe3,0xd1]
-; X64-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <16 x i16> @llvm.x86.avx2.pavg.w(<16 x i16> %x0, <16 x i16> %x1)
   %2 = bitcast i16 %x3 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %x2
-  %4 = call <16 x i16> @llvm.x86.avx2.pavg.w(<16 x i16> %x0, <16 x i16> %x1)
-  %res2 = add <16 x i16> %3, %4
-  ret <16 x i16> %res2
+  ret <16 x i16> %3
 }
 
 declare <8 x i16> @llvm.x86.sse2.pmulhu.w(<8 x i16>, <8 x i16>)
@@ -1396,26 +1380,22 @@ declare <8 x i16> @llvm.x86.sse2.pmulhu.w(<8 x i16>, <8 x i16>)
 define <8 x i16> @test_int_x86_avx512_mask_pmulhu_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmulhu_w_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe4,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe4,0xd1]
-; X86-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmulhu_w_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe4,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe4,0xd1]
-; X64-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <8 x i16> @llvm.x86.sse2.pmulhu.w(<8 x i16> %x0, <8 x i16> %x1)
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %x2
-  %4 = call <8 x i16> @llvm.x86.sse2.pmulhu.w(<8 x i16> %x0, <8 x i16> %x1)
-  %res2 = add <8 x i16> %3, %4
-  ret <8 x i16> %res2
+  ret <8 x i16> %3
 }
 
 declare <16 x i16> @llvm.x86.avx2.pmulhu.w(<16 x i16>, <16 x i16>)
@@ -1423,25 +1403,21 @@ declare <16 x i16> @llvm.x86.avx2.pmulhu.w(<16 x i16>, <16 x i16>)
 define <16 x i16> @test_int_x86_avx512_mask_pmulhu_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmulhu_w_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe4,0xd9]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe4,0xd1]
-; X86-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmulhu_w_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe4,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe4,0xd1]
-; X64-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <16 x i16> @llvm.x86.avx2.pmulhu.w(<16 x i16> %x0, <16 x i16> %x1)
   %2 = bitcast i16 %x3 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %x2
-  %4 = call <16 x i16> @llvm.x86.avx2.pmulhu.w(<16 x i16> %x0, <16 x i16> %x1)
-  %res2 = add <16 x i16> %3, %4
-  ret <16 x i16> %res2
+  ret <16 x i16> %3
 }
 
 declare <8 x i16> @llvm.x86.sse2.pmulh.w(<8 x i16>, <8 x i16>)
@@ -1449,26 +1425,22 @@ declare <8 x i16> @llvm.x86.sse2.pmulh.w(<8 x i16>, <8 x i16>)
 define <8 x i16> @test_int_x86_avx512_mask_pmulh_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmulh_w_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpmulhw %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe5,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpmulhw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe5,0xd1]
-; X86-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmulh_w_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpmulhw %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe5,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpmulhw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe5,0xd1]
-; X64-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <8 x i16> @llvm.x86.sse2.pmulh.w(<8 x i16> %x0, <8 x i16> %x1)
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %x2
-  %4 = call <8 x i16> @llvm.x86.sse2.pmulh.w(<8 x i16> %x0, <8 x i16> %x1)
-  %res2 = add <8 x i16> %3, %4
-  ret <8 x i16> %res2
+  ret <8 x i16> %3
 }
 
 declare <16 x i16> @llvm.x86.avx2.pmulh.w(<16 x i16>, <16 x i16>)
@@ -1476,25 +1448,21 @@ declare <16 x i16> @llvm.x86.avx2.pmulh.w(<16 x i16>, <16 x i16>)
 define <16 x i16> @test_int_x86_avx512_mask_pmulh_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmulh_w_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpmulhw %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe5,0xd9]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpmulhw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe5,0xd1]
-; X86-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmulh_w_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpmulhw %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe5,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpmulhw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe5,0xd1]
-; X64-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <16 x i16> @llvm.x86.avx2.pmulh.w(<16 x i16> %x0, <16 x i16> %x1)
   %2 = bitcast i16 %x3 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %x2
-  %4 = call <16 x i16> @llvm.x86.avx2.pmulh.w(<16 x i16> %x0, <16 x i16> %x1)
-  %res2 = add <16 x i16> %3, %4
-  ret <16 x i16> %res2
+  ret <16 x i16> %3
 }
 
 declare <8 x i16> @llvm.x86.ssse3.pmul.hr.sw.128(<8 x i16>, <8 x i16>)
@@ -1502,26 +1470,22 @@ declare <8 x i16> @llvm.x86.ssse3.pmul.hr.sw.128(<8 x i16>, <8 x i16>)
 define <8 x i16> @test_int_x86_avx512_mask_pmulhr_sw_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmulhr_sw_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpmulhrsw %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x0b,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpmulhrsw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x0b,0xd1]
-; X86-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmulhr_sw_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpmulhrsw %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x0b,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpmulhrsw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x0b,0xd1]
-; X64-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <8 x i16> @llvm.x86.ssse3.pmul.hr.sw.128(<8 x i16> %x0, <8 x i16> %x1)
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %x2
-  %4 = call <8 x i16> @llvm.x86.ssse3.pmul.hr.sw.128(<8 x i16> %x0, <8 x i16> %x1)
-  %res2 = add <8 x i16> %3, %4
-  ret <8 x i16> %res2
+  ret <8 x i16> %3
 }
 
 declare <16 x i16> @llvm.x86.avx2.pmul.hr.sw(<16 x i16>, <16 x i16>)
@@ -1529,25 +1493,21 @@ declare <16 x i16> @llvm.x86.avx2.pmul.hr.sw(<16 x i16>, <16 x i16>)
 define <16 x i16> @test_int_x86_avx512_mask_pmulhr_sw_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmulhr_sw_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpmulhrsw %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x0b,0xd9]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpmulhrsw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x0b,0xd1]
-; X86-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmulhr_sw_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpmulhrsw %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x0b,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpmulhrsw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x0b,0xd1]
-; X64-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <16 x i16> @llvm.x86.avx2.pmul.hr.sw(<16 x i16> %x0, <16 x i16> %x1)
   %2 = bitcast i16 %x3 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %x2
-  %4 = call <16 x i16> @llvm.x86.avx2.pmul.hr.sw(<16 x i16> %x0, <16 x i16> %x1)
-  %res2 = add <16 x i16> %3, %4
-  ret <16 x i16> %res2
+  ret <16 x i16> %3
 }
 
 declare <16 x i8> @llvm.x86.avx512.mask.pmov.wb.128(<8 x i16>, <16 x i8>, i8)
@@ -1941,27 +1901,23 @@ declare <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16>, <8 x i16>)
 define <4 x i32> @test_int_x86_avx512_mask_pmaddw_d_128(<8 x i16> %x0, <8 x i16> %x1, <4 x i32> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmaddw_d_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpmaddwd %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xf5,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpmaddwd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xf5,0xd1]
-; X86-NEXT:    vpaddd %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc3]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmaddw_d_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpmaddwd %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xf5,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpmaddwd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xf5,0xd1]
-; X64-NEXT:    vpaddd %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc3]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %x0, <8 x i16> %x1)
   %2 = bitcast i8 %x3 to <8 x i1>
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract, <4 x i32> %1, <4 x i32> %x2
-  %4 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %x0, <8 x i16> %x1)
-  %res2 = add <4 x i32> %3, %4
-  ret <4 x i32> %res2
+  ret <4 x i32> %3
 }
 
 declare <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16>, <16 x i16>)
@@ -1969,26 +1925,22 @@ declare <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16>, <16 x i16>)
 define <8 x i32> @test_int_x86_avx512_mask_pmaddw_d_256(<16 x i16> %x0, <16 x i16> %x1, <8 x i32> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmaddw_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpmaddwd %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xf5,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpmaddwd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xf5,0xd1]
-; X86-NEXT:    vpaddd %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc3]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmaddw_d_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpmaddwd %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xf5,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpmaddwd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xf5,0xd1]
-; X64-NEXT:    vpaddd %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc3]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %x0, <16 x i16> %x1)
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x2
-  %4 = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %x0, <16 x i16> %x1)
-  %res2 = add <8 x i32> %3, %4
-  ret <8 x i32> %res2
+  ret <8 x i32> %3
 }
 
 declare <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8>, <16 x i8>)
@@ -1996,26 +1948,22 @@ declare <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8>, <16 x i8>)
 define <8 x i16> @test_int_x86_avx512_mask_pmaddubs_w_128(<16 x i8> %x0, <16 x i8> %x1, <8 x i16> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmaddubs_w_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpmaddubsw %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x04,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpmaddubsw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x04,0xd1]
-; X86-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmaddubs_w_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpmaddubsw %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x04,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpmaddubsw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x04,0xd1]
-; X64-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %x0, <16 x i8> %x1)
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %x2
-  %4 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %x0, <16 x i8> %x1)
-  %res2 = add <8 x i16> %3, %4
-  ret <8 x i16> %res2
+  ret <8 x i16> %3
 }
 
 declare <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8>, <32 x i8>)
@@ -2023,25 +1971,21 @@ declare <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8>, <32 x i8>)
 define <16 x i16> @test_int_x86_avx512_mask_pmaddubs_w_256(<32 x i8> %x0, <32 x i8> %x1, <16 x i16> %x2, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmaddubs_w_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpmaddubsw %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x04,0xd9]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpmaddubsw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x04,0xd1]
-; X86-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmaddubs_w_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpmaddubsw %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x04,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpmaddubsw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x04,0xd1]
-; X64-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %x0, <32 x i8> %x1)
   %2 = bitcast i16 %x3 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %x2
-  %4 = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %x0, <32 x i8> %x1)
-  %res2 = add <16 x i16> %3, %4
-  ret <16 x i16> %res2
+  ret <16 x i16> %3
 }
 
 declare <8 x i16> @llvm.x86.avx512.dbpsadbw.128(<16 x i8>, <16 x i8>, i32)
@@ -2049,31 +1993,31 @@ declare <8 x i16> @llvm.x86.avx512.dbpsadbw.128(<16 x i8>, <16 x i8>, i32)
 define <8 x i16>@test_int_x86_avx512_mask_dbpsadbw_128(<16 x i8> %x0, <16 x i8> %x1, <8 x i16> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_dbpsadbw_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vdbpsadbw $2, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x42,0xd9,0x02]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vdbpsadbw $2, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x42,0xd1,0x02]
-; X86-NEXT:    vdbpsadbw $2, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x42,0xc1,0x02]
-; X86-NEXT:    vpaddw %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc3]
+; X86-NEXT:    vdbpsadbw $3, %xmm1, %xmm0, %xmm3 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x42,0xd9,0x03]
+; X86-NEXT:    vdbpsadbw $4, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0x7d,0x08,0x42,0xc1,0x04]
+; X86-NEXT:    vpaddw %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfd,0xc0]
 ; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_dbpsadbw_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vdbpsadbw $2, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x42,0xd9,0x02]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vdbpsadbw $2, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x42,0xd1,0x02]
-; X64-NEXT:    vdbpsadbw $2, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x42,0xc1,0x02]
-; X64-NEXT:    vpaddw %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc3]
+; X64-NEXT:    vdbpsadbw $3, %xmm1, %xmm0, %xmm3 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x42,0xd9,0x03]
+; X64-NEXT:    vdbpsadbw $4, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0x7d,0x08,0x42,0xc1,0x04]
+; X64-NEXT:    vpaddw %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfd,0xc0]
 ; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <8 x i16> @llvm.x86.avx512.dbpsadbw.128(<16 x i8> %x0, <16 x i8> %x1, i32 2)
   %2 = bitcast i8 %x4 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %x3
-  %4 = call <8 x i16> @llvm.x86.avx512.dbpsadbw.128(<16 x i8> %x0, <16 x i8> %x1, i32 2)
+  %4 = call <8 x i16> @llvm.x86.avx512.dbpsadbw.128(<16 x i8> %x0, <16 x i8> %x1, i32 3)
   %5 = bitcast i8 %x4 to <8 x i1>
   %6 = select <8 x i1> %5, <8 x i16> %4, <8 x i16> zeroinitializer
-  %7 = call <8 x i16> @llvm.x86.avx512.dbpsadbw.128(<16 x i8> %x0, <16 x i8> %x1, i32 2)
+  %7 = call <8 x i16> @llvm.x86.avx512.dbpsadbw.128(<16 x i8> %x0, <16 x i8> %x1, i32 4)
   %res3 = add <8 x i16> %3, %6
   %res4 = add <8 x i16> %7, %res3
   ret <8 x i16> %res4
@@ -2084,30 +2028,30 @@ declare <16 x i16> @llvm.x86.avx512.dbpsadbw.256(<32 x i8>, <32 x i8>, i32)
 define <16 x i16>@test_int_x86_avx512_mask_dbpsadbw_256(<32 x i8> %x0, <32 x i8> %x1, <16 x i16> %x3, i16 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_dbpsadbw_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vdbpsadbw $2, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0x7d,0x28,0x42,0xd9,0x02]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vdbpsadbw $2, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x42,0xd1,0x02]
-; X86-NEXT:    vdbpsadbw $2, %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0xa9,0x42,0xc1,0x02]
-; X86-NEXT:    vpaddw %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc3]
+; X86-NEXT:    vdbpsadbw $3, %ymm1, %ymm0, %ymm3 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0xa9,0x42,0xd9,0x03]
+; X86-NEXT:    vdbpsadbw $4, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0x7d,0x28,0x42,0xc1,0x04]
+; X86-NEXT:    vpaddw %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfd,0xc0]
 ; X86-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_dbpsadbw_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vdbpsadbw $2, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0x7d,0x28,0x42,0xd9,0x02]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vdbpsadbw $2, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x42,0xd1,0x02]
-; X64-NEXT:    vdbpsadbw $2, %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0xa9,0x42,0xc1,0x02]
-; X64-NEXT:    vpaddw %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc3]
+; X64-NEXT:    vdbpsadbw $3, %ymm1, %ymm0, %ymm3 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0xa9,0x42,0xd9,0x03]
+; X64-NEXT:    vdbpsadbw $4, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0x7d,0x28,0x42,0xc1,0x04]
+; X64-NEXT:    vpaddw %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfd,0xc0]
 ; X64-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <16 x i16> @llvm.x86.avx512.dbpsadbw.256(<32 x i8> %x0, <32 x i8> %x1, i32 2)
   %2 = bitcast i16 %x4 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %x3
-  %4 = call <16 x i16> @llvm.x86.avx512.dbpsadbw.256(<32 x i8> %x0, <32 x i8> %x1, i32 2)
+  %4 = call <16 x i16> @llvm.x86.avx512.dbpsadbw.256(<32 x i8> %x0, <32 x i8> %x1, i32 3)
   %5 = bitcast i16 %x4 to <16 x i1>
   %6 = select <16 x i1> %5, <16 x i16> %4, <16 x i16> zeroinitializer
-  %7 = call <16 x i16> @llvm.x86.avx512.dbpsadbw.256(<32 x i8> %x0, <32 x i8> %x1, i32 2)
+  %7 = call <16 x i16> @llvm.x86.avx512.dbpsadbw.256(<32 x i8> %x0, <32 x i8> %x1, i32 4)
   %res3 = add <16 x i16> %3, %6
   %res4 = add <16 x i16> %res3, %7
   ret <16 x i16> %res4
@@ -2115,63 +2059,94 @@ define <16 x i16>@test_int_x86_avx512_mask_dbpsadbw_256(<32 x i8> %x0, <32 x i8>
 
 declare <16 x i16> @llvm.x86.avx512.mask.psrlv16.hi(<16 x i16>, <16 x i16>, <16 x i16>, i16)
 
+define <16 x i16>@test_int_x86_avx512_psrlv16_hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psrlv16_hi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x10,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psrlv16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 -1)
+  ret <16 x i16> %res
+}
+
 define <16 x i16>@test_int_x86_avx512_mask_psrlv16_hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrlv16_hi:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x10,0xd9]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x10,0xd1]
-; X86-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x10,0xc1]
-; X86-NEXT:    vpaddw %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc3]
-; X86-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrlv16_hi:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x10,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x10,0xd1]
-; X64-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x10,0xc1]
-; X64-NEXT:    vpaddw %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc3]
-; X64-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx512.mask.psrlv16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3)
-  %res1 = call <16 x i16> @llvm.x86.avx512.mask.psrlv16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> zeroinitializer, i16 %x3)
-  %res2 = call <16 x i16> @llvm.x86.avx512.mask.psrlv16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 -1)
-  %res3 = add <16 x i16> %res, %res1
-  %res4 = add <16 x i16> %res3, %res2
-  ret <16 x i16> %res4
+  ret <16 x i16> %res
+}
+
+define <16 x i16>@test_int_x86_avx512_maskz_psrlv16_hi(<16 x i16> %x0, <16 x i16> %x1, i16 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psrlv16_hi:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x10,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psrlv16_hi:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x10,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psrlv16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> zeroinitializer, i16 %x3)
+  ret <16 x i16> %res
 }
 
 declare <8 x i16> @llvm.x86.avx512.mask.psrlv8.hi(<8 x i16>, <8 x i16>, <8 x i16>, i8)
 
+define <8 x i16>@test_int_x86_avx512_psrlv8_hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psrlv8_hi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x10,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psrlv8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
+  ret <8 x i16> %res
+}
+
 define <8 x i16>@test_int_x86_avx512_mask_psrlv8_hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrlv8_hi:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x10,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x10,0xd1]
-; X86-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x10,0xc1]
-; X86-NEXT:    vpaddw %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc3]
-; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrlv8_hi:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x10,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x10,0xd1]
-; X64-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x10,0xc1]
-; X64-NEXT:    vpaddw %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc3]
-; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i16> @llvm.x86.avx512.mask.psrlv8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
-  %res1 = call <8 x i16> @llvm.x86.avx512.mask.psrlv8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> zeroinitializer, i8 %x3)
-  %res2 = call <8 x i16> @llvm.x86.avx512.mask.psrlv8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
-  %res3 = add <8 x i16> %res, %res1
-  %res4 = add <8 x i16> %res3, %res2
-  ret <8 x i16> %res4
+  ret <8 x i16> %res
+}
+
+define <8 x i16>@test_int_x86_avx512_maskz_psrlv8_hi(<8 x i16> %x0, <8 x i16> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psrlv8_hi:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x10,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psrlv8_hi:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x10,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psrlv8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> zeroinitializer, i8 %x3)
+  ret <8 x i16> %res
 }
 
 
@@ -2225,136 +2200,198 @@ declare <16 x i16> @llvm.x86.avx512.psrlv.w.256(<16 x i16>, <16 x i16>)
 
 declare <16 x i16> @llvm.x86.avx512.psrav.w.256(<16 x i16>, <16 x i16>)
 
+define <16 x i16>@test_int_x86_avx512_psrav16_hi(<16 x i16> %x0, <16 x i16> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_psrav16_hi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsravw %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x11,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <16 x i16> @llvm.x86.avx512.psrav.w.256(<16 x i16> %x0, <16 x i16> %x1)
+  ret <16 x i16> %1
+}
+
 define <16 x i16>@test_int_x86_avx512_mask_psrav16_hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrav16_hi:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsravw %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x11,0xd9]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpsravw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x11,0xd1]
-; X86-NEXT:    vpsravw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x11,0xc1]
-; X86-NEXT:    vpaddw %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc3]
-; X86-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrav16_hi:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsravw %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x11,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpsravw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x11,0xd1]
-; X64-NEXT:    vpsravw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x11,0xc1]
-; X64-NEXT:    vpaddw %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc3]
-; X64-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <16 x i16> @llvm.x86.avx512.psrav.w.256(<16 x i16> %x0, <16 x i16> %x1)
   %2 = bitcast i16 %x3 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %x2
-  %4 = call <16 x i16> @llvm.x86.avx512.psrav.w.256(<16 x i16> %x0, <16 x i16> %x1)
-  %5 = bitcast i16 %x3 to <16 x i1>
-  %6 = select <16 x i1> %5, <16 x i16> %4, <16 x i16> zeroinitializer
-  %7 = call <16 x i16> @llvm.x86.avx512.psrav.w.256(<16 x i16> %x0, <16 x i16> %x1)
-  %res3 = add <16 x i16> %3, %6
-  %res4 = add <16 x i16> %res3, %7
-  ret <16 x i16> %res4
+  ret <16 x i16> %3
+}
+
+define <16 x i16>@test_int_x86_avx512_maskz_psrav16_hi(<16 x i16> %x0, <16 x i16> %x1, i16 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psrav16_hi:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsravw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x11,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psrav16_hi:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsravw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x11,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <16 x i16> @llvm.x86.avx512.psrav.w.256(<16 x i16> %x0, <16 x i16> %x1)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> zeroinitializer
+  ret <16 x i16> %3
 }
 
 declare <8 x i16> @llvm.x86.avx512.psrav.w.128(<8 x i16>, <8 x i16>)
 
+define <8 x i16>@test_int_x86_avx512_psrav8_hi(<8 x i16> %x0, <8 x i16> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_psrav8_hi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsravw %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x11,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <8 x i16> @llvm.x86.avx512.psrav.w.128(<8 x i16> %x0, <8 x i16> %x1)
+  ret <8 x i16> %1
+}
+
 define <8 x i16>@test_int_x86_avx512_mask_psrav8_hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrav8_hi:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsravw %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x11,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpsravw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x11,0xd1]
-; X86-NEXT:    vpsravw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x11,0xc1]
-; X86-NEXT:    vpaddw %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc3]
-; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrav8_hi:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsravw %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x11,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpsravw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x11,0xd1]
-; X64-NEXT:    vpsravw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x11,0xc1]
-; X64-NEXT:    vpaddw %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc3]
-; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <8 x i16> @llvm.x86.avx512.psrav.w.128(<8 x i16> %x0, <8 x i16> %x1)
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %x2
-  %4 = call <8 x i16> @llvm.x86.avx512.psrav.w.128(<8 x i16> %x0, <8 x i16> %x1)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %6 = select <8 x i1> %5, <8 x i16> %4, <8 x i16> zeroinitializer
-  %7 = call <8 x i16> @llvm.x86.avx512.psrav.w.128(<8 x i16> %x0, <8 x i16> %x1)
-  %res3 = add <8 x i16> %3, %6
-  %res4 = add <8 x i16> %res3, %7
-  ret <8 x i16> %res4
+  ret <8 x i16> %3
+}
+
+define <8 x i16>@test_int_x86_avx512_maskz_psrav8_hi(<8 x i16> %x0, <8 x i16> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psrav8_hi:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpsravw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x11,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psrav8_hi:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsravw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x11,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <8 x i16> @llvm.x86.avx512.psrav.w.128(<8 x i16> %x0, <8 x i16> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> zeroinitializer
+  ret <8 x i16> %3
+}
+
+define <16 x i16>@test_int_x86_avx512_psllv16_hi(<16 x i16> %x0, <16 x i16> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_psllv16_hi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsllvw %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x12,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16> %x0, <16 x i16> %x1)
+  ret <16 x i16> %1
 }
 
 define <16 x i16>@test_int_x86_avx512_mask_psllv16_hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psllv16_hi:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsllvw %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x12,0xd9]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpsllvw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x12,0xd1]
-; X86-NEXT:    vpsllvw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x12,0xc1]
-; X86-NEXT:    vpaddw %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc3]
-; X86-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psllv16_hi:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsllvw %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x12,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpsllvw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x12,0xd1]
-; X64-NEXT:    vpsllvw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x12,0xc1]
-; X64-NEXT:    vpaddw %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc3]
-; X64-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16> %x0, <16 x i16> %x1)
   %2 = bitcast i16 %x3 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %x2
-  %4 = call <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16> %x0, <16 x i16> %x1)
-  %5 = bitcast i16 %x3 to <16 x i1>
-  %6 = select <16 x i1> %5, <16 x i16> %4, <16 x i16> zeroinitializer
-  %7 = call <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16> %x0, <16 x i16> %x1)
-  %res3 = add <16 x i16> %3, %6
-  %res4 = add <16 x i16> %res3, %7
-  ret <16 x i16> %res4
+  ret <16 x i16> %3
+}
+
+define <16 x i16>@test_int_x86_avx512_maskz_psllv16_hi(<16 x i16> %x0, <16 x i16> %x1, i16 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psllv16_hi:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsllvw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x12,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psllv16_hi:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsllvw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x12,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16> %x0, <16 x i16> %x1)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> zeroinitializer
+  ret <16 x i16> %3
+}
+
+define <8 x i16>@test_int_x86_avx512_psllv8_hi(<8 x i16> %x0, <8 x i16> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_psllv8_hi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsllvw %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x12,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16> %x0, <8 x i16> %x1)
+  ret <8 x i16> %1
 }
 
 define <8 x i16>@test_int_x86_avx512_mask_psllv8_hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psllv8_hi:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsllvw %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x12,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpsllvw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x12,0xd1]
-; X86-NEXT:    vpsllvw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x12,0xc1]
-; X86-NEXT:    vpaddw %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc3]
-; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psllv8_hi:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsllvw %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x12,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpsllvw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x12,0xd1]
-; X64-NEXT:    vpsllvw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x12,0xc1]
-; X64-NEXT:    vpaddw %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc3]
-; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16> %x0, <8 x i16> %x1)
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %x2
-  %4 = call <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16> %x0, <8 x i16> %x1)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %6 = select <8 x i1> %5, <8 x i16> %4, <8 x i16> zeroinitializer
-  %7 = call <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16> %x0, <8 x i16> %x1)
-  %res3 = add <8 x i16> %3, %6
-  %res4 = add <8 x i16> %res3, %7
-  ret <8 x i16> %res4
+  ret <8 x i16> %3
+}
+
+define <8 x i16>@test_int_x86_avx512_maskz_psllv8_hi(<8 x i16> %x0, <8 x i16> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psllv8_hi:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpsllvw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x12,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psllv8_hi:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsllvw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x12,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16> %x0, <8 x i16> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> zeroinitializer
+  ret <8 x i16> %3
 }
 
 define <8 x i16> @test_int_x86_avx512_psllv_w_128_const() optsize {
diff --git a/llvm/test/CodeGen/X86/avx512cdvl-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512cdvl-intrinsics-upgrade.ll
index 6f4fc11495f20..ec07484bce4ad 100644
--- a/llvm/test/CodeGen/X86/avx512cdvl-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512cdvl-intrinsics-upgrade.ll
@@ -4,108 +4,139 @@
 
 declare <4 x i32> @llvm.x86.avx512.mask.lzcnt.d.128(<4 x i32>, <4 x i32>, i8)
 
+define <4 x i32>@test_int_x86_avx512_vplzcnt_d_128(<4 x i32> %x0, <4 x i32> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_vplzcnt_d_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vplzcntd %xmm0, %xmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <4 x i32> @llvm.x86.avx512.mask.lzcnt.d.128(<4 x i32> %x0, <4 x i32> %x1, i8 -1)
+  ret <4 x i32> %res
+}
+
 define <4 x i32>@test_int_x86_avx512_mask_vplzcnt_d_128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_vplzcnt_d_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vplzcntd %xmm0, %xmm2
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vplzcntd %xmm0, %xmm1 {%k1}
-; X86-NEXT:    vplzcntd %xmm0, %xmm0 {%k1} {z}
-; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
-; X86-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmovdqa %xmm1, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vplzcnt_d_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vplzcntd %xmm0, %xmm2
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vplzcntd %xmm0, %xmm1 {%k1}
-; X64-NEXT:    vplzcntd %xmm0, %xmm0 {%k1} {z}
-; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
-; X64-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; X64-NEXT:    vmovdqa %xmm1, %xmm0
 ; X64-NEXT:    retq
   %res = call <4 x i32> @llvm.x86.avx512.mask.lzcnt.d.128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2)
-  %res1 = call <4 x i32> @llvm.x86.avx512.mask.lzcnt.d.128(<4 x i32> %x0, <4 x i32> %x1, i8 -1)
-  %res3 = call <4 x i32> @llvm.x86.avx512.mask.lzcnt.d.128(<4 x i32> %x0, <4 x i32> zeroinitializer, i8 %x2)
-  %res2 = add <4 x i32> %res, %res1
-  %res4 = add <4 x i32> %res2, %res3
-  ret <4 x i32> %res4
+  ret <4 x i32> %res
+}
+
+define <4 x i32>@test_int_x86_avx512_maskz_vplzcnt_d_128(<4 x i32> %x0, i8 %x2) {
+; X86-LABEL: test_int_x86_avx512_maskz_vplzcnt_d_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vplzcntd %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_int_x86_avx512_maskz_vplzcnt_d_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vplzcntd %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    retq
+  %res = call <4 x i32> @llvm.x86.avx512.mask.lzcnt.d.128(<4 x i32> %x0, <4 x i32> zeroinitializer, i8 %x2)
+  ret <4 x i32> %res
 }
 
 declare <8 x i32> @llvm.x86.avx512.mask.lzcnt.d.256(<8 x i32>, <8 x i32>, i8)
 
+define <8 x i32>@test_int_x86_avx512_vplzcnt_d_256(<8 x i32> %x0, <8 x i32> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_vplzcnt_d_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vplzcntd %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <8 x i32> @llvm.x86.avx512.mask.lzcnt.d.256(<8 x i32> %x0, <8 x i32> %x1, i8 -1)
+  ret <8 x i32> %res
+}
+
 define <8 x i32>@test_int_x86_avx512_mask_vplzcnt_d_256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_vplzcnt_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vplzcntd %ymm0, %ymm2
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vplzcntd %ymm0, %ymm1 {%k1}
-; X86-NEXT:    vpaddd %ymm2, %ymm1, %ymm0
+; X86-NEXT:    vmovdqa %ymm1, %ymm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vplzcnt_d_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vplzcntd %ymm0, %ymm2
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vplzcntd %ymm0, %ymm1 {%k1}
-; X64-NEXT:    vpaddd %ymm2, %ymm1, %ymm0
+; X64-NEXT:    vmovdqa %ymm1, %ymm0
 ; X64-NEXT:    retq
   %res = call <8 x i32> @llvm.x86.avx512.mask.lzcnt.d.256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2)
-  %res1 = call <8 x i32> @llvm.x86.avx512.mask.lzcnt.d.256(<8 x i32> %x0, <8 x i32> %x1, i8 -1)
-  %res2 = add <8 x i32> %res, %res1
-  ret <8 x i32> %res2
+  ret <8 x i32> %res
 }
 
 declare <2 x i64> @llvm.x86.avx512.mask.lzcnt.q.128(<2 x i64>, <2 x i64>, i8)
 
+define <2 x i64>@test_int_x86_avx512_vplzcnt_q_128(<2 x i64> %x0, <2 x i64> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_vplzcnt_q_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vplzcntq %xmm0, %xmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <2 x i64> @llvm.x86.avx512.mask.lzcnt.q.128(<2 x i64> %x0, <2 x i64> %x1, i8 -1)
+  ret <2 x i64> %res
+}
+
 define <2 x i64>@test_int_x86_avx512_mask_vplzcnt_q_128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_vplzcnt_q_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vplzcntq %xmm0, %xmm2
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vplzcntq %xmm0, %xmm1 {%k1}
-; X86-NEXT:    vpaddq %xmm2, %xmm1, %xmm0
+; X86-NEXT:    vmovdqa %xmm1, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vplzcnt_q_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vplzcntq %xmm0, %xmm2
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vplzcntq %xmm0, %xmm1 {%k1}
-; X64-NEXT:    vpaddq %xmm2, %xmm1, %xmm0
+; X64-NEXT:    vmovdqa %xmm1, %xmm0
 ; X64-NEXT:    retq
   %res = call <2 x i64> @llvm.x86.avx512.mask.lzcnt.q.128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2)
-  %res1 = call <2 x i64> @llvm.x86.avx512.mask.lzcnt.q.128(<2 x i64> %x0, <2 x i64> %x1, i8 -1)
-  %res2 = add <2 x i64> %res, %res1
-  ret <2 x i64> %res2
+  ret <2 x i64> %res
 }
 
 declare <4 x i64> @llvm.x86.avx512.mask.lzcnt.q.256(<4 x i64>, <4 x i64>, i8)
 
+define <4 x i64>@test_int_x86_avx512_vplzcnt_q_256(<4 x i64> %x0, <4 x i64> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_vplzcnt_q_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vplzcntq %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <4 x i64> @llvm.x86.avx512.mask.lzcnt.q.256(<4 x i64> %x0, <4 x i64> %x1, i8 -1)
+  ret <4 x i64> %res
+}
+
 define <4 x i64>@test_int_x86_avx512_mask_vplzcnt_q_256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_vplzcnt_q_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vplzcntq %ymm0, %ymm2
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vplzcntq %ymm0, %ymm1 {%k1}
-; X86-NEXT:    vpaddq %ymm2, %ymm1, %ymm0
+; X86-NEXT:    vmovdqa %ymm1, %ymm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vplzcnt_q_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vplzcntq %ymm0, %ymm2
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vplzcntq %ymm0, %ymm1 {%k1}
-; X64-NEXT:    vpaddq %ymm2, %ymm1, %ymm0
+; X64-NEXT:    vmovdqa %ymm1, %ymm0
 ; X64-NEXT:    retq
   %res = call <4 x i64> @llvm.x86.avx512.mask.lzcnt.q.256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2)
-  %res1 = call <4 x i64> @llvm.x86.avx512.mask.lzcnt.q.256(<4 x i64> %x0, <4 x i64> %x1, i8 -1)
-  %res2 = add <4 x i64> %res, %res1
-  ret <4 x i64> %res2
+  ret <4 x i64> %res
 }
 
 define <8 x i32> @test_x86_vbroadcastmw_256(i16 %a0) {
@@ -182,125 +213,189 @@ declare <2 x i64> @llvm.x86.avx512.broadcastmb.128(i8)
 
 declare <4 x i32> @llvm.x86.avx512.mask.conflict.d.128(<4 x i32>, <4 x i32>, i8)
 
+define <4 x i32> @test_int_x86_avx512_vpconflict_d_128(<4 x i32> %x0, <4 x i32> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_vpconflict_d_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpconflictd %xmm0, %xmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <4 x i32> @llvm.x86.avx512.mask.conflict.d.128(<4 x i32> %x0, <4 x i32> %x1, i8 -1)
+  ret <4 x i32> %res
+}
+
 define <4 x i32> @test_int_x86_avx512_mask_vpconflict_d_128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpconflict_d_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpconflictd %xmm0, %xmm2
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpconflictd %xmm0, %xmm1 {%k1}
-; X86-NEXT:    vpconflictd %xmm0, %xmm0 {%k1} {z}
-; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
-; X86-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmovdqa %xmm1, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpconflict_d_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpconflictd %xmm0, %xmm2
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vpconflictd %xmm0, %xmm1 {%k1}
-; X64-NEXT:    vpconflictd %xmm0, %xmm0 {%k1} {z}
-; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
-; X64-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; X64-NEXT:    vmovdqa %xmm1, %xmm0
 ; X64-NEXT:    retq
   %res = call <4 x i32> @llvm.x86.avx512.mask.conflict.d.128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2)
-  %res1 = call <4 x i32> @llvm.x86.avx512.mask.conflict.d.128(<4 x i32> %x0, <4 x i32> %x1, i8 -1)
-  %res3 = call <4 x i32> @llvm.x86.avx512.mask.conflict.d.128(<4 x i32> %x0, <4 x i32> zeroinitializer, i8 %x2)
-  %res2 = add <4 x i32> %res, %res1
-  %res4 = add <4 x i32> %res2, %res3
-  ret <4 x i32> %res4
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_int_x86_avx512_maskz_vpconflict_d_128(<4 x i32> %x0, i8 %x2) {
+; X86-LABEL: test_int_x86_avx512_maskz_vpconflict_d_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpconflictd %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_int_x86_avx512_maskz_vpconflict_d_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpconflictd %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    retq
+  %res = call <4 x i32> @llvm.x86.avx512.mask.conflict.d.128(<4 x i32> %x0, <4 x i32> zeroinitializer, i8 %x2)
+  ret <4 x i32> %res
 }
 
 declare <8 x i32> @llvm.x86.avx512.mask.conflict.d.256(<8 x i32>, <8 x i32>, i8)
 
+define <8 x i32> @test_int_x86_avx512_vpconflict_d_256(<8 x i32> %x0, <8 x i32> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_vpconflict_d_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpconflictd %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <8 x i32> @llvm.x86.avx512.mask.conflict.d.256(<8 x i32> %x0, <8 x i32> %x1, i8 -1)
+  ret <8 x i32> %res
+}
+
 define <8 x i32> @test_int_x86_avx512_mask_vpconflict_d_256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpconflict_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpconflictd %ymm0, %ymm2
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpconflictd %ymm0, %ymm1 {%k1}
-; X86-NEXT:    vpconflictd %ymm0, %ymm0 {%k1} {z}
-; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
-; X86-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
+; X86-NEXT:    vmovdqa %ymm1, %ymm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpconflict_d_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpconflictd %ymm0, %ymm2
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vpconflictd %ymm0, %ymm1 {%k1}
-; X64-NEXT:    vpconflictd %ymm0, %ymm0 {%k1} {z}
-; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
-; X64-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
+; X64-NEXT:    vmovdqa %ymm1, %ymm0
 ; X64-NEXT:    retq
   %res = call <8 x i32> @llvm.x86.avx512.mask.conflict.d.256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2)
-  %res1 = call <8 x i32> @llvm.x86.avx512.mask.conflict.d.256(<8 x i32> %x0, <8 x i32> %x1, i8 -1)
-  %res2 = call <8 x i32> @llvm.x86.avx512.mask.conflict.d.256(<8 x i32> %x0, <8 x i32> zeroinitializer, i8 %x2)
-  %res3 = add <8 x i32> %res, %res1
-  %res4 = add <8 x i32> %res2, %res3
-  ret <8 x i32> %res4
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_int_x86_avx512_maskz_vpconflict_d_256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2) {
+; X86-LABEL: test_int_x86_avx512_maskz_vpconflict_d_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpconflictd %ymm0, %ymm0 {%k1} {z}
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_int_x86_avx512_maskz_vpconflict_d_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpconflictd %ymm0, %ymm0 {%k1} {z}
+; X64-NEXT:    retq
+  %res = call <8 x i32> @llvm.x86.avx512.mask.conflict.d.256(<8 x i32> %x0, <8 x i32> zeroinitializer, i8 %x2)
+  ret <8 x i32> %res
 }
 
 declare <2 x i64> @llvm.x86.avx512.mask.conflict.q.128(<2 x i64>, <2 x i64>, i8)
 
+define <2 x i64> @test_int_x86_avx512_vpconflict_q_128(<2 x i64> %x0, <2 x i64> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_vpconflict_q_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpconflictq %xmm0, %xmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <2 x i64> @llvm.x86.avx512.mask.conflict.q.128(<2 x i64> %x0, <2 x i64> %x1, i8 -1)
+  ret <2 x i64> %res
+}
+
 define <2 x i64> @test_int_x86_avx512_mask_vpconflict_q_128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpconflict_q_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpconflictq %xmm0, %xmm2
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpconflictq %xmm0, %xmm1 {%k1}
-; X86-NEXT:    vpconflictq %xmm0, %xmm0 {%k1} {z}
-; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
-; X86-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmovdqa %xmm1, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpconflict_q_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpconflictq %xmm0, %xmm2
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vpconflictq %xmm0, %xmm1 {%k1}
-; X64-NEXT:    vpconflictq %xmm0, %xmm0 {%k1} {z}
-; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
-; X64-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; X64-NEXT:    vmovdqa %xmm1, %xmm0
 ; X64-NEXT:    retq
   %res = call <2 x i64> @llvm.x86.avx512.mask.conflict.q.128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2)
-  %res1 = call <2 x i64> @llvm.x86.avx512.mask.conflict.q.128(<2 x i64> %x0, <2 x i64> %x1, i8 -1)
-  %res2 = call <2 x i64> @llvm.x86.avx512.mask.conflict.q.128(<2 x i64> %x0, <2 x i64> zeroinitializer, i8 %x2)
-  %res3 = add <2 x i64> %res, %res1
-  %res4 = add <2 x i64> %res2, %res3
-  ret <2 x i64> %res4
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_int_x86_avx512_maskz_vpconflict_q_128(<2 x i64> %x0, i8 %x2) {
+; X86-LABEL: test_int_x86_avx512_maskz_vpconflict_q_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpconflictq %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_int_x86_avx512_maskz_vpconflict_q_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpconflictq %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    retq
+  %res = call <2 x i64> @llvm.x86.avx512.mask.conflict.q.128(<2 x i64> %x0, <2 x i64> zeroinitializer, i8 %x2)
+  ret <2 x i64> %res
 }
 
 declare <4 x i64> @llvm.x86.avx512.mask.conflict.q.256(<4 x i64>, <4 x i64>, i8)
 
+define <4 x i64> @test_int_x86_avx512_vpconflict_q_256(<4 x i64> %x0, <4 x i64> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_vpconflict_q_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpconflictq %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <4 x i64> @llvm.x86.avx512.mask.conflict.q.256(<4 x i64> %x0, <4 x i64> %x1, i8 -1)
+  ret <4 x i64> %res
+}
+
 define <4 x i64> @test_int_x86_avx512_mask_vpconflict_q_256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpconflict_q_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpconflictq %ymm0, %ymm2
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpconflictq %ymm0, %ymm1 {%k1}
-; X86-NEXT:    vpconflictq %ymm0, %ymm0 {%k1} {z}
-; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
-; X86-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; X86-NEXT:    vmovdqa %ymm1, %ymm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpconflict_q_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpconflictq %ymm0, %ymm2
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vpconflictq %ymm0, %ymm1 {%k1}
-; X64-NEXT:    vpconflictq %ymm0, %ymm0 {%k1} {z}
-; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
-; X64-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; X64-NEXT:    vmovdqa %ymm1, %ymm0
 ; X64-NEXT:    retq
   %res = call <4 x i64> @llvm.x86.avx512.mask.conflict.q.256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2)
-  %res1 = call <4 x i64> @llvm.x86.avx512.mask.conflict.q.256(<4 x i64> %x0, <4 x i64> %x1, i8 -1)
-  %res2 = call <4 x i64> @llvm.x86.avx512.mask.conflict.q.256(<4 x i64> %x0, <4 x i64> zeroinitializer, i8 %x2)
-  %res3 = add <4 x i64> %res, %res1
-  %res4 = add <4 x i64> %res2, %res3
-  ret <4 x i64> %res4
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_int_x86_avx512_maskz_vpconflict_q_256(<4 x i64> %x0, i8 %x2) {
+; X86-LABEL: test_int_x86_avx512_maskz_vpconflict_q_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpconflictq %ymm0, %ymm0 {%k1} {z}
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_int_x86_avx512_maskz_vpconflict_q_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpconflictq %ymm0, %ymm0 {%k1} {z}
+; X64-NEXT:    retq
+  %res = call <4 x i64> @llvm.x86.avx512.mask.conflict.q.256(<4 x i64> %x0, <4 x i64> zeroinitializer, i8 %x2)
+  ret <4 x i64> %res
 }
 
diff --git a/llvm/test/CodeGen/X86/avx512cdvl-intrinsics.ll b/llvm/test/CodeGen/X86/avx512cdvl-intrinsics.ll
index 4d7e19c38a225..7f3a19d92b4c3 100644
--- a/llvm/test/CodeGen/X86/avx512cdvl-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512cdvl-intrinsics.ll
@@ -5,255 +5,314 @@
 define <4 x i32> @test_int_x86_avx512_mask_vplzcnt_d_128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_vplzcnt_d_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vplzcntd %xmm0, %xmm2
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vplzcntd %xmm0, %xmm1 {%k1}
-; X86-NEXT:    vplzcntd %xmm0, %xmm0 {%k1} {z}
-; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
-; X86-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmovdqa %xmm1, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vplzcnt_d_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vplzcntd %xmm0, %xmm2
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vplzcntd %xmm0, %xmm1 {%k1}
-; X64-NEXT:    vplzcntd %xmm0, %xmm0 {%k1} {z}
-; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
-; X64-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; X64-NEXT:    vmovdqa %xmm1, %xmm0
 ; X64-NEXT:    retq
   %1 = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %x0, i1 false)
   %2 = bitcast i8 %x2 to <8 x i1>
   %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract1, <4 x i32> %1, <4 x i32> %x1
-  %4 = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %x0, i1 false)
-  %5 = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %x0, i1 false)
-  %6 = bitcast i8 %x2 to <8 x i1>
-  %extract = shufflevector <8 x i1> %6, <8 x i1> %6, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %7 = select <4 x i1> %extract, <4 x i32> %5, <4 x i32> zeroinitializer
-  %res2 = add <4 x i32> %3, %4
-  %res4 = add <4 x i32> %res2, %7
-  ret <4 x i32> %res4
+  ret <4 x i32> %3
+}
+
+define <4 x i32> @test_int_x86_avx512_maskz_vplzcnt_d_128(<4 x i32> %x0, i8 %x2) {
+; X86-LABEL: test_int_x86_avx512_maskz_vplzcnt_d_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vplzcntd %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_int_x86_avx512_maskz_vplzcnt_d_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vplzcntd %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    retq
+  %1 = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %x0, i1 false)
+  %2 = bitcast i8 %x2 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract1, <4 x i32> %1, <4 x i32> zeroinitializer
+  ret <4 x i32> %3
 }
 declare <4 x i32> @llvm.ctlz.v4i32(<4 x i32>, i1) #0
 
 define <8 x i32> @test_int_x86_avx512_mask_vplzcnt_d_256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_vplzcnt_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vplzcntd %ymm0, %ymm2
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vplzcntd %ymm0, %ymm1 {%k1}
-; X86-NEXT:    vpaddd %ymm2, %ymm1, %ymm0
+; X86-NEXT:    vmovdqa %ymm1, %ymm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vplzcnt_d_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vplzcntd %ymm0, %ymm2
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vplzcntd %ymm0, %ymm1 {%k1}
-; X64-NEXT:    vpaddd %ymm2, %ymm1, %ymm0
+; X64-NEXT:    vmovdqa %ymm1, %ymm0
 ; X64-NEXT:    retq
   %1 = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> %x0, i1 false)
   %2 = bitcast i8 %x2 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x1
-  %4 = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> %x0, i1 false)
-  %res2 = add <8 x i32> %3, %4
-  ret <8 x i32> %res2
+  ret <8 x i32> %3
 }
 declare <8 x i32> @llvm.ctlz.v8i32(<8 x i32>, i1) #0
 
 define <2 x i64> @test_int_x86_avx512_mask_vplzcnt_q_128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_vplzcnt_q_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vplzcntq %xmm0, %xmm2
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vplzcntq %xmm0, %xmm1 {%k1}
-; X86-NEXT:    vpaddq %xmm2, %xmm1, %xmm0
+; X86-NEXT:    vmovdqa %xmm1, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vplzcnt_q_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vplzcntq %xmm0, %xmm2
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vplzcntq %xmm0, %xmm1 {%k1}
-; X64-NEXT:    vpaddq %xmm2, %xmm1, %xmm0
+; X64-NEXT:    vmovdqa %xmm1, %xmm0
 ; X64-NEXT:    retq
   %1 = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %x0, i1 false)
   %2 = bitcast i8 %x2 to <8 x i1>
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
   %3 = select <2 x i1> %extract, <2 x i64> %1, <2 x i64> %x1
-  %4 = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %x0, i1 false)
-  %res2 = add <2 x i64> %3, %4
-  ret <2 x i64> %res2
+  ret <2 x i64> %3
 }
 declare <2 x i64> @llvm.ctlz.v2i64(<2 x i64>, i1) #0
 
 define <4 x i64> @test_int_x86_avx512_mask_vplzcnt_q_256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_vplzcnt_q_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vplzcntq %ymm0, %ymm2
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vplzcntq %ymm0, %ymm1 {%k1}
-; X86-NEXT:    vpaddq %ymm2, %ymm1, %ymm0
+; X86-NEXT:    vmovdqa %ymm1, %ymm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vplzcnt_q_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vplzcntq %ymm0, %ymm2
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vplzcntq %ymm0, %ymm1 {%k1}
-; X64-NEXT:    vpaddq %ymm2, %ymm1, %ymm0
+; X64-NEXT:    vmovdqa %ymm1, %ymm0
 ; X64-NEXT:    retq
   %1 = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> %x0, i1 false)
   %2 = bitcast i8 %x2 to <8 x i1>
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract, <4 x i64> %1, <4 x i64> %x1
-  %4 = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> %x0, i1 false)
-  %res2 = add <4 x i64> %3, %4
-  ret <4 x i64> %res2
+  ret <4 x i64> %3
 }
 declare <4 x i64> @llvm.ctlz.v4i64(<4 x i64>, i1) #0
 
+define <4 x i32> @test_int_x86_avx512_vpconflict_d_128(<4 x i32> %x0, <4 x i32> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_vpconflict_d_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpconflictd %xmm0, %xmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 = call <4 x i32> @llvm.x86.avx512.conflict.d.128(<4 x i32> %x0)
+  ret <4 x i32> %1
+}
+
 define <4 x i32> @test_int_x86_avx512_mask_vpconflict_d_128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpconflict_d_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpconflictd %xmm0, %xmm2
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpconflictd %xmm0, %xmm1 {%k1}
-; X86-NEXT:    vpconflictd %xmm0, %xmm0 {%k1} {z}
-; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
-; X86-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmovdqa %xmm1, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpconflict_d_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpconflictd %xmm0, %xmm2
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vpconflictd %xmm0, %xmm1 {%k1}
-; X64-NEXT:    vpconflictd %xmm0, %xmm0 {%k1} {z}
-; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
-; X64-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; X64-NEXT:    vmovdqa %xmm1, %xmm0
 ; X64-NEXT:    retq
   %1 = call <4 x i32> @llvm.x86.avx512.conflict.d.128(<4 x i32> %x0)
   %2 = bitcast i8 %x2 to <8 x i1>
   %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract1, <4 x i32> %1, <4 x i32> %x1
-  %4 = call <4 x i32> @llvm.x86.avx512.conflict.d.128(<4 x i32> %x0)
-  %5 = call <4 x i32> @llvm.x86.avx512.conflict.d.128(<4 x i32> %x0)
-  %6 = bitcast i8 %x2 to <8 x i1>
-  %extract = shufflevector <8 x i1> %6, <8 x i1> %6, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %7 = select <4 x i1> %extract, <4 x i32> %5, <4 x i32> zeroinitializer
-  %res2 = add <4 x i32> %3, %4
-  %res4 = add <4 x i32> %res2, %7
-  ret <4 x i32> %res4
+  ret <4 x i32> %3
+}
+
+define <4 x i32> @test_int_x86_avx512_maskz_vpconflict_d_128(<4 x i32> %x0, i8 %x2) {
+; X86-LABEL: test_int_x86_avx512_maskz_vpconflict_d_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpconflictd %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_int_x86_avx512_maskz_vpconflict_d_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpconflictd %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    retq
+  %1 = call <4 x i32> @llvm.x86.avx512.conflict.d.128(<4 x i32> %x0)
+  %2 = bitcast i8 %x2 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract1, <4 x i32> %1, <4 x i32> zeroinitializer
+  ret <4 x i32> %3
+}
+
+define <8 x i32> @test_int_x86_avx512_vpconflict_d_256(<8 x i32> %x0, <8 x i32> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_vpconflict_d_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpconflictd %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 = call <8 x i32> @llvm.x86.avx512.conflict.d.256(<8 x i32> %x0)
+  ret <8 x i32> %1
 }
 
 define <8 x i32> @test_int_x86_avx512_mask_vpconflict_d_256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpconflict_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpconflictd %ymm0, %ymm2
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpconflictd %ymm0, %ymm1 {%k1}
-; X86-NEXT:    vpconflictd %ymm0, %ymm0 {%k1} {z}
-; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
-; X86-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
+; X86-NEXT:    vmovdqa %ymm1, %ymm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpconflict_d_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpconflictd %ymm0, %ymm2
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vpconflictd %ymm0, %ymm1 {%k1}
-; X64-NEXT:    vpconflictd %ymm0, %ymm0 {%k1} {z}
-; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
-; X64-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
+; X64-NEXT:    vmovdqa %ymm1, %ymm0
 ; X64-NEXT:    retq
   %1 = call <8 x i32> @llvm.x86.avx512.conflict.d.256(<8 x i32> %x0)
   %2 = bitcast i8 %x2 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x1
-  %4 = call <8 x i32> @llvm.x86.avx512.conflict.d.256(<8 x i32> %x0)
-  %5 = call <8 x i32> @llvm.x86.avx512.conflict.d.256(<8 x i32> %x0)
-  %6 = bitcast i8 %x2 to <8 x i1>
-  %7 = select <8 x i1> %6, <8 x i32> %5, <8 x i32> zeroinitializer
-  %res3 = add <8 x i32> %3, %4
-  %res4 = add <8 x i32> %7, %res3
-  ret <8 x i32> %res4
+  ret <8 x i32> %3
+}
+
+define <8 x i32> @test_int_x86_avx512_maskz_vpconflict_d_256(<8 x i32> %x0, i8 %x2) {
+; X86-LABEL: test_int_x86_avx512_maskz_vpconflict_d_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpconflictd %ymm0, %ymm0 {%k1} {z}
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_int_x86_avx512_maskz_vpconflict_d_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpconflictd %ymm0, %ymm0 {%k1} {z}
+; X64-NEXT:    retq
+  %1 = call <8 x i32> @llvm.x86.avx512.conflict.d.256(<8 x i32> %x0)
+  %2 = bitcast i8 %x2 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> zeroinitializer
+  ret <8 x i32> %3
+}
+
+define <2 x i64> @test_int_x86_avx512_vpconflict_q_128(<2 x i64> %x0, <2 x i64> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_vpconflict_q_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpconflictq %xmm0, %xmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 = call <2 x i64> @llvm.x86.avx512.conflict.q.128(<2 x i64> %x0)
+  ret <2 x i64> %1
 }
 
 define <2 x i64> @test_int_x86_avx512_mask_vpconflict_q_128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpconflict_q_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpconflictq %xmm0, %xmm2
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpconflictq %xmm0, %xmm1 {%k1}
-; X86-NEXT:    vpconflictq %xmm0, %xmm0 {%k1} {z}
-; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
-; X86-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmovdqa %xmm1, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpconflict_q_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpconflictq %xmm0, %xmm2
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vpconflictq %xmm0, %xmm1 {%k1}
-; X64-NEXT:    vpconflictq %xmm0, %xmm0 {%k1} {z}
-; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
-; X64-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; X64-NEXT:    vmovdqa %xmm1, %xmm0
 ; X64-NEXT:    retq
   %1 = call <2 x i64> @llvm.x86.avx512.conflict.q.128(<2 x i64> %x0)
   %2 = bitcast i8 %x2 to <8 x i1>
   %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
   %3 = select <2 x i1> %extract1, <2 x i64> %1, <2 x i64> %x1
-  %4 = call <2 x i64> @llvm.x86.avx512.conflict.q.128(<2 x i64> %x0)
-  %5 = call <2 x i64> @llvm.x86.avx512.conflict.q.128(<2 x i64> %x0)
-  %6 = bitcast i8 %x2 to <8 x i1>
-  %extract = shufflevector <8 x i1> %6, <8 x i1> %6, <2 x i32> <i32 0, i32 1>
-  %7 = select <2 x i1> %extract, <2 x i64> %5, <2 x i64> zeroinitializer
-  %res3 = add <2 x i64> %3, %4
-  %res4 = add <2 x i64> %7, %res3
-  ret <2 x i64> %res4
+  ret <2 x i64> %3
+}
+
+define <2 x i64> @test_int_x86_avx512_maskz_vpconflict_q_128(<2 x i64> %x0, i8 %x2) {
+; X86-LABEL: test_int_x86_avx512_maskz_vpconflict_q_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpconflictq %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_int_x86_avx512_maskz_vpconflict_q_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpconflictq %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    retq
+  %1 = call <2 x i64> @llvm.x86.avx512.conflict.q.128(<2 x i64> %x0)
+  %2 = bitcast i8 %x2 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
+  %3 = select <2 x i1> %extract1, <2 x i64> %1, <2 x i64> zeroinitializer
+  ret <2 x i64> %3
+}
+
+define <4 x i64> @test_int_x86_avx512_vpconflict_q_256(<4 x i64> %x0, <4 x i64> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_vpconflict_q_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpconflictq %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 = call <4 x i64> @llvm.x86.avx512.conflict.q.256(<4 x i64> %x0)
+  ret <4 x i64> %1
 }
 
 define <4 x i64> @test_int_x86_avx512_mask_vpconflict_q_256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpconflict_q_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpconflictq %ymm0, %ymm2
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpconflictq %ymm0, %ymm1 {%k1}
-; X86-NEXT:    vpconflictq %ymm0, %ymm0 {%k1} {z}
-; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
-; X86-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; X86-NEXT:    vmovdqa %ymm1, %ymm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpconflict_q_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpconflictq %ymm0, %ymm2
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vpconflictq %ymm0, %ymm1 {%k1}
-; X64-NEXT:    vpconflictq %ymm0, %ymm0 {%k1} {z}
-; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
-; X64-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; X64-NEXT:    vmovdqa %ymm1, %ymm0
 ; X64-NEXT:    retq
   %1 = call <4 x i64> @llvm.x86.avx512.conflict.q.256(<4 x i64> %x0)
   %2 = bitcast i8 %x2 to <8 x i1>
   %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract1, <4 x i64> %1, <4 x i64> %x1
-  %4 = call <4 x i64> @llvm.x86.avx512.conflict.q.256(<4 x i64> %x0)
-  %5 = call <4 x i64> @llvm.x86.avx512.conflict.q.256(<4 x i64> %x0)
-  %6 = bitcast i8 %x2 to <8 x i1>
-  %extract = shufflevector <8 x i1> %6, <8 x i1> %6, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %7 = select <4 x i1> %extract, <4 x i64> %5, <4 x i64> zeroinitializer
-  %res3 = add <4 x i64> %3, %4
-  %res4 = add <4 x i64> %7, %res3
-  ret <4 x i64> %res4
+  ret <4 x i64> %3
+}
+
+define <4 x i64> @test_int_x86_avx512_maskz_vpconflict_q_256(<4 x i64> %x0, i8 %x2) {
+; X86-LABEL: test_int_x86_avx512_maskz_vpconflict_q_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpconflictq %ymm0, %ymm0 {%k1} {z}
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_int_x86_avx512_maskz_vpconflict_q_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpconflictq %ymm0, %ymm0 {%k1} {z}
+; X64-NEXT:    retq
+  %1 = call <4 x i64> @llvm.x86.avx512.conflict.q.256(<4 x i64> %x0)
+  %2 = bitcast i8 %x2 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract1, <4 x i64> %1, <4 x i64> zeroinitializer
+  ret <4 x i64> %3
 }
 
 declare <4 x i32> @llvm.x86.avx512.conflict.d.128(<4 x i32>)
diff --git a/llvm/test/CodeGen/X86/avx512dq-intrinsics.ll b/llvm/test/CodeGen/X86/avx512dq-intrinsics.ll
index 506db572671ab..ee8c794522357 100644
--- a/llvm/test/CodeGen/X86/avx512dq-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512dq-intrinsics.ll
@@ -587,9 +587,9 @@ define <4 x float>@test_int_x86_avx512_mask_range_ss(<4 x float> %x0, <4 x float
 ; X86-AVX512DQ:       # %bb.0:
 ; X86-AVX512DQ-NEXT:    kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-AVX512DQ-NEXT:    vrangess $4, {sae}, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x19,0x51,0xd1,0x04]
-; X86-AVX512DQ-NEXT:    vrangess $4, {sae}, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x18,0x51,0xd9,0x04]
+; X86-AVX512DQ-NEXT:    vrangess $5, {sae}, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x18,0x51,0xd9,0x05]
 ; X86-AVX512DQ-NEXT:    vaddps %xmm3, %xmm2, %xmm2 # encoding: [0xc5,0xe8,0x58,0xd3]
-; X86-AVX512DQ-NEXT:    vrangess $4, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0x7d,0x08,0x51,0xc1,0x04]
+; X86-AVX512DQ-NEXT:    vrangess $6, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0x7d,0x08,0x51,0xc1,0x06]
 ; X86-AVX512DQ-NEXT:    vaddps %xmm2, %xmm0, %xmm0 # encoding: [0xc5,0xf8,0x58,0xc2]
 ; X86-AVX512DQ-NEXT:    retl # encoding: [0xc3]
 ;
@@ -597,9 +597,9 @@ define <4 x float>@test_int_x86_avx512_mask_range_ss(<4 x float> %x0, <4 x float
 ; X86-AVX512DQVL:       # %bb.0:
 ; X86-AVX512DQVL-NEXT:    kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-AVX512DQVL-NEXT:    vrangess $4, {sae}, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x19,0x51,0xd1,0x04]
-; X86-AVX512DQVL-NEXT:    vrangess $4, {sae}, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x18,0x51,0xd9,0x04]
+; X86-AVX512DQVL-NEXT:    vrangess $5, {sae}, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x18,0x51,0xd9,0x05]
 ; X86-AVX512DQVL-NEXT:    vaddps %xmm3, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe8,0x58,0xd3]
-; X86-AVX512DQVL-NEXT:    vrangess $4, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0x7d,0x08,0x51,0xc1,0x04]
+; X86-AVX512DQVL-NEXT:    vrangess $6, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0x7d,0x08,0x51,0xc1,0x06]
 ; X86-AVX512DQVL-NEXT:    vaddps %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x58,0xc2]
 ; X86-AVX512DQVL-NEXT:    retl # encoding: [0xc3]
 ;
@@ -607,9 +607,9 @@ define <4 x float>@test_int_x86_avx512_mask_range_ss(<4 x float> %x0, <4 x float
 ; X64-AVX512DQ:       # %bb.0:
 ; X64-AVX512DQ-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-AVX512DQ-NEXT:    vrangess $4, {sae}, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x19,0x51,0xd1,0x04]
-; X64-AVX512DQ-NEXT:    vrangess $4, {sae}, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x18,0x51,0xd9,0x04]
+; X64-AVX512DQ-NEXT:    vrangess $5, {sae}, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x18,0x51,0xd9,0x05]
 ; X64-AVX512DQ-NEXT:    vaddps %xmm3, %xmm2, %xmm2 # encoding: [0xc5,0xe8,0x58,0xd3]
-; X64-AVX512DQ-NEXT:    vrangess $4, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0x7d,0x08,0x51,0xc1,0x04]
+; X64-AVX512DQ-NEXT:    vrangess $6, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0x7d,0x08,0x51,0xc1,0x06]
 ; X64-AVX512DQ-NEXT:    vaddps %xmm2, %xmm0, %xmm0 # encoding: [0xc5,0xf8,0x58,0xc2]
 ; X64-AVX512DQ-NEXT:    retq # encoding: [0xc3]
 ;
@@ -617,14 +617,14 @@ define <4 x float>@test_int_x86_avx512_mask_range_ss(<4 x float> %x0, <4 x float
 ; X64-AVX512DQVL:       # %bb.0:
 ; X64-AVX512DQVL-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-AVX512DQVL-NEXT:    vrangess $4, {sae}, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x19,0x51,0xd1,0x04]
-; X64-AVX512DQVL-NEXT:    vrangess $4, {sae}, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x18,0x51,0xd9,0x04]
+; X64-AVX512DQVL-NEXT:    vrangess $5, {sae}, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x18,0x51,0xd9,0x05]
 ; X64-AVX512DQVL-NEXT:    vaddps %xmm3, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe8,0x58,0xd3]
-; X64-AVX512DQVL-NEXT:    vrangess $4, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0x7d,0x08,0x51,0xc1,0x04]
+; X64-AVX512DQVL-NEXT:    vrangess $6, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0x7d,0x08,0x51,0xc1,0x06]
 ; X64-AVX512DQVL-NEXT:    vaddps %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x58,0xc2]
 ; X64-AVX512DQVL-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x float> @llvm.x86.avx512.mask.range.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x3, i8 %x4, i32 4, i32 8)
-  %res1 = call <4 x float> @llvm.x86.avx512.mask.range.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x3, i8 -1, i32 4, i32 8)
-  %res2 = call <4 x float> @llvm.x86.avx512.mask.range.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x3, i8 -1, i32 4, i32 4)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.range.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x3, i8 -1, i32 5, i32 8)
+  %res2 = call <4 x float> @llvm.x86.avx512.mask.range.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x3, i8 -1, i32 6, i32 4)
   %res3 = fadd <4 x float> %res, %res1
   %res4 = fadd <4 x float> %res2, %res3
   ret <4 x float> %res4
@@ -677,44 +677,44 @@ define <2 x double>@test_int_x86_avx512_mask_range_sd(<2 x double> %x0, <2 x dou
 ; X86-AVX512DQ:       # %bb.0:
 ; X86-AVX512DQ-NEXT:    kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-AVX512DQ-NEXT:    vrangesd $4, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x51,0xd1,0x04]
-; X86-AVX512DQ-NEXT:    vrangesd $4, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x51,0xd9,0x04]
-; X86-AVX512DQ-NEXT:    vrangesd $4, {sae}, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x18,0x51,0xc1,0x04]
-; X86-AVX512DQ-NEXT:    vaddpd %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0x58,0xc0]
-; X86-AVX512DQ-NEXT:    vaddpd %xmm0, %xmm3, %xmm0 # encoding: [0xc5,0xe1,0x58,0xc0]
+; X86-AVX512DQ-NEXT:    vrangesd $5, {sae}, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x18,0x51,0xd9,0x05]
+; X86-AVX512DQ-NEXT:    vaddpd %xmm3, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0x58,0xd3]
+; X86-AVX512DQ-NEXT:    vrangesd $6, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x51,0xc1,0x06]
+; X86-AVX512DQ-NEXT:    vaddpd %xmm2, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x58,0xc2]
 ; X86-AVX512DQ-NEXT:    retl # encoding: [0xc3]
 ;
 ; X86-AVX512DQVL-LABEL: test_int_x86_avx512_mask_range_sd:
 ; X86-AVX512DQVL:       # %bb.0:
 ; X86-AVX512DQVL-NEXT:    kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-AVX512DQVL-NEXT:    vrangesd $4, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x51,0xd1,0x04]
-; X86-AVX512DQVL-NEXT:    vrangesd $4, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x51,0xd9,0x04]
-; X86-AVX512DQVL-NEXT:    vrangesd $4, {sae}, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x18,0x51,0xc1,0x04]
-; X86-AVX512DQVL-NEXT:    vaddpd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0x58,0xc0]
-; X86-AVX512DQVL-NEXT:    vaddpd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0x58,0xc0]
+; X86-AVX512DQVL-NEXT:    vrangesd $5, {sae}, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x18,0x51,0xd9,0x05]
+; X86-AVX512DQVL-NEXT:    vaddpd %xmm3, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0x58,0xd3]
+; X86-AVX512DQVL-NEXT:    vrangesd $6, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x51,0xc1,0x06]
+; X86-AVX512DQVL-NEXT:    vaddpd %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x58,0xc2]
 ; X86-AVX512DQVL-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-AVX512DQ-LABEL: test_int_x86_avx512_mask_range_sd:
 ; X64-AVX512DQ:       # %bb.0:
 ; X64-AVX512DQ-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-AVX512DQ-NEXT:    vrangesd $4, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x51,0xd9,0x04]
 ; X64-AVX512DQ-NEXT:    vrangesd $4, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x51,0xd1,0x04]
-; X64-AVX512DQ-NEXT:    vrangesd $4, {sae}, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x18,0x51,0xc1,0x04]
-; X64-AVX512DQ-NEXT:    vaddpd %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0x58,0xc0]
-; X64-AVX512DQ-NEXT:    vaddpd %xmm0, %xmm3, %xmm0 # encoding: [0xc5,0xe1,0x58,0xc0]
+; X64-AVX512DQ-NEXT:    vrangesd $5, {sae}, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x18,0x51,0xd9,0x05]
+; X64-AVX512DQ-NEXT:    vaddpd %xmm3, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0x58,0xd3]
+; X64-AVX512DQ-NEXT:    vrangesd $6, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x51,0xc1,0x06]
+; X64-AVX512DQ-NEXT:    vaddpd %xmm2, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x58,0xc2]
 ; X64-AVX512DQ-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512DQVL-LABEL: test_int_x86_avx512_mask_range_sd:
 ; X64-AVX512DQVL:       # %bb.0:
 ; X64-AVX512DQVL-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-AVX512DQVL-NEXT:    vrangesd $4, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x51,0xd9,0x04]
 ; X64-AVX512DQVL-NEXT:    vrangesd $4, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x51,0xd1,0x04]
-; X64-AVX512DQVL-NEXT:    vrangesd $4, {sae}, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x18,0x51,0xc1,0x04]
-; X64-AVX512DQVL-NEXT:    vaddpd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0x58,0xc0]
-; X64-AVX512DQVL-NEXT:    vaddpd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0x58,0xc0]
+; X64-AVX512DQVL-NEXT:    vrangesd $5, {sae}, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x18,0x51,0xd9,0x05]
+; X64-AVX512DQVL-NEXT:    vaddpd %xmm3, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0x58,0xd3]
+; X64-AVX512DQVL-NEXT:    vrangesd $6, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x51,0xc1,0x06]
+; X64-AVX512DQVL-NEXT:    vaddpd %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x58,0xc2]
 ; X64-AVX512DQVL-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x double> @llvm.x86.avx512.mask.range.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x3, i8 %x4, i32 4, i32 4)
-  %res1 = call <2 x double> @llvm.x86.avx512.mask.range.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x3, i8 -1, i32 4, i32 8)
-  %res2 = call <2 x double> @llvm.x86.avx512.mask.range.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x3, i8 -1, i32 4, i32 4)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.range.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x3, i8 -1, i32 5, i32 8)
+  %res2 = call <2 x double> @llvm.x86.avx512.mask.range.sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x3, i8 -1, i32 6, i32 4)
   %res3 = fadd <2 x double> %res, %res1
   %res4 = fadd <2 x double> %res2, %res3
   ret <2 x double> %res4
diff --git a/llvm/test/CodeGen/X86/avx512dqvl-intrinsics.ll b/llvm/test/CodeGen/X86/avx512dqvl-intrinsics.ll
index 169ce93170bf6..0563b8509d217 100644
--- a/llvm/test/CodeGen/X86/avx512dqvl-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512dqvl-intrinsics.ll
@@ -1381,7 +1381,7 @@ define <8 x float>@test_int_x86_avx512_mask_reduce_ps_256(<8 x float> %x0, <8 x
 ; X86:       # %bb.0:
 ; X86-NEXT:    kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vreduceps $11, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x56,0xc8,0x0b]
-; X86-NEXT:    vreduceps $11, %ymm0, %ymm0 # encoding: [0x62,0xf3,0x7d,0x28,0x56,0xc0,0x0b]
+; X86-NEXT:    vreduceps $12, %ymm0, %ymm0 # encoding: [0x62,0xf3,0x7d,0x28,0x56,0xc0,0x0c]
 ; X86-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf4,0x58,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1389,11 +1389,11 @@ define <8 x float>@test_int_x86_avx512_mask_reduce_ps_256(<8 x float> %x0, <8 x
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vreduceps $11, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x56,0xc8,0x0b]
-; X64-NEXT:    vreduceps $11, %ymm0, %ymm0 # encoding: [0x62,0xf3,0x7d,0x28,0x56,0xc0,0x0b]
+; X64-NEXT:    vreduceps $12, %ymm0, %ymm0 # encoding: [0x62,0xf3,0x7d,0x28,0x56,0xc0,0x0c]
 ; X64-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf4,0x58,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x float> @llvm.x86.avx512.mask.reduce.ps.256(<8 x float> %x0, i32 11, <8 x float> %x2, i8 %x3)
-  %res1 = call <8 x float> @llvm.x86.avx512.mask.reduce.ps.256(<8 x float> %x0, i32 11, <8 x float> %x2, i8 -1)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask.reduce.ps.256(<8 x float> %x0, i32 12, <8 x float> %x2, i8 -1)
   %res2 = fadd <8 x float> %res, %res1
   ret <8 x float> %res2
 }
diff --git a/llvm/test/CodeGen/X86/avx512ifma-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512ifma-intrinsics-upgrade.ll
index bf85814f31973..f4e746064b38e 100644
--- a/llvm/test/CodeGen/X86/avx512ifma-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512ifma-intrinsics-upgrade.ll
@@ -4,46 +4,32 @@
 
 declare <8 x i64> @llvm.x86.avx512.mask.vpmadd52h.uq.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
 
+define <8 x i64>@test_int_x86_avx512_vpmadd52h_uq_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpmadd52h_uq_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf2,0xf5,0x48,0xb5,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+
+  %res = call <8 x i64> @llvm.x86.avx512.mask.vpmadd52h.uq.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1)
+  ret <8 x i64> %res
+}
+
 define <8 x i64>@test_int_x86_avx512_mask_vpmadd52h_uq_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpmadd52h_uq_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
-; X86-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm3 # encoding: [0x62,0xf2,0xf5,0x48,0xb5,0xda]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm4 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0xb5,0xe2]
-; X86-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0xef,0xd2]
 ; X86-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0xb5,0xc2]
-; X86-NEXT:    vpaddq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xd4,0xc0]
-; X86-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb5,0xd2]
-; X86-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
-; X86-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpmadd52h_uq_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
-; X64-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm3 # encoding: [0x62,0xf2,0xf5,0x48,0xb5,0xda]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X64-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm4 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0xb5,0xe2]
-; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0xef,0xd2]
 ; X64-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0xb5,0xc2]
-; X64-NEXT:    vpaddq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xd4,0xc0]
-; X64-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb5,0xd2]
-; X64-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
-; X64-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
 
   %res = call <8 x i64> @llvm.x86.avx512.mask.vpmadd52h.uq.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
-  %res1 = call <8 x i64> @llvm.x86.avx512.mask.vpmadd52h.uq.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <8 x i64> @llvm.x86.avx512.mask.vpmadd52h.uq.512(<8 x i64> zeroinitializer, <8 x i64> %x1, <8 x i64> zeroinitializer, i8 %x3)
-  %res3 = call <8 x i64> @llvm.x86.avx512.mask.vpmadd52h.uq.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1)
-  %res4 = add <8 x i64> %res, %res1
-  %res5 = add <8 x i64> %res3, %res2
-  %res6 = add <8 x i64> %res5, %res4
-  ret <8 x i64> %res6
+  ret <8 x i64> %res
 }
 
 declare <8 x i64> @llvm.x86.avx512.maskz.vpmadd52h.uq.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
@@ -51,87 +37,49 @@ declare <8 x i64> @llvm.x86.avx512.maskz.vpmadd52h.uq.512(<8 x i64>, <8 x i64>,
 define <8 x i64>@test_int_x86_avx512_maskz_vpmadd52h_uq_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_maskz_vpmadd52h_uq_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
-; X86-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm3 # encoding: [0x62,0xf2,0xf5,0x48,0xb5,0xda]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb5,0xe2]
-; X86-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0xef,0xd2]
 ; X86-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb5,0xc2]
-; X86-NEXT:    vpaddq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xd4,0xc0]
-; X86-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb5,0xd2]
-; X86-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
-; X86-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_vpmadd52h_uq_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
-; X64-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm3 # encoding: [0x62,0xf2,0xf5,0x48,0xb5,0xda]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X64-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb5,0xe2]
-; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0xef,0xd2]
 ; X64-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb5,0xc2]
-; X64-NEXT:    vpaddq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xd4,0xc0]
-; X64-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb5,0xd2]
-; X64-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
-; X64-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
 
   %res = call <8 x i64> @llvm.x86.avx512.maskz.vpmadd52h.uq.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
-  %res1 = call <8 x i64> @llvm.x86.avx512.maskz.vpmadd52h.uq.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <8 x i64> @llvm.x86.avx512.maskz.vpmadd52h.uq.512(<8 x i64> zeroinitializer, <8 x i64> %x1, <8 x i64> zeroinitializer, i8 %x3)
-  %res3 = call <8 x i64> @llvm.x86.avx512.maskz.vpmadd52h.uq.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1)
-  %res4 = add <8 x i64> %res, %res1
-  %res5 = add <8 x i64> %res3, %res2
-  %res6 = add <8 x i64> %res5, %res4
-  ret <8 x i64> %res6
+  ret <8 x i64> %res
 }
 
 declare <8 x i64> @llvm.x86.avx512.mask.vpmadd52l.uq.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
 
+define <8 x i64>@test_int_x86_avx512_vpmadd52l_uq_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpmadd52l_uq_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf2,0xf5,0x48,0xb4,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+
+  %res = call <8 x i64> @llvm.x86.avx512.mask.vpmadd52l.uq.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1)
+  ret <8 x i64> %res
+}
+
 define <8 x i64>@test_int_x86_avx512_mask_vpmadd52l_uq_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpmadd52l_uq_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
-; X86-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm3 # encoding: [0x62,0xf2,0xf5,0x48,0xb4,0xda]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm4 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0xb4,0xe2]
-; X86-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0xef,0xd2]
 ; X86-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0xb4,0xc2]
-; X86-NEXT:    vpaddq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xd4,0xc0]
-; X86-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb4,0xd2]
-; X86-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
-; X86-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpmadd52l_uq_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
-; X64-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm3 # encoding: [0x62,0xf2,0xf5,0x48,0xb4,0xda]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X64-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm4 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0xb4,0xe2]
-; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0xef,0xd2]
 ; X64-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0xb4,0xc2]
-; X64-NEXT:    vpaddq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xd4,0xc0]
-; X64-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb4,0xd2]
-; X64-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
-; X64-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
 
   %res = call <8 x i64> @llvm.x86.avx512.mask.vpmadd52l.uq.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
-  %res1 = call <8 x i64> @llvm.x86.avx512.mask.vpmadd52l.uq.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <8 x i64> @llvm.x86.avx512.mask.vpmadd52l.uq.512(<8 x i64> zeroinitializer, <8 x i64> %x1, <8 x i64> zeroinitializer, i8 %x3)
-  %res3 = call <8 x i64> @llvm.x86.avx512.mask.vpmadd52l.uq.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1)
-  %res4 = add <8 x i64> %res, %res1
-  %res5 = add <8 x i64> %res3, %res2
-  %res6 = add <8 x i64> %res5, %res4
-  ret <8 x i64> %res6
+  ret <8 x i64> %res
 }
 
 declare <8 x i64> @llvm.x86.avx512.maskz.vpmadd52l.uq.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
@@ -139,43 +87,19 @@ declare <8 x i64> @llvm.x86.avx512.maskz.vpmadd52l.uq.512(<8 x i64>, <8 x i64>,
 define <8 x i64>@test_int_x86_avx512_maskz_vpmadd52l_uq_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_maskz_vpmadd52l_uq_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
-; X86-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm3 # encoding: [0x62,0xf2,0xf5,0x48,0xb4,0xda]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb4,0xe2]
-; X86-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0xef,0xd2]
 ; X86-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb4,0xc2]
-; X86-NEXT:    vpaddq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xd4,0xc0]
-; X86-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb4,0xd2]
-; X86-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
-; X86-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_vpmadd52l_uq_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
-; X64-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm3 # encoding: [0x62,0xf2,0xf5,0x48,0xb4,0xda]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X64-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb4,0xe2]
-; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0xef,0xd2]
 ; X64-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb4,0xc2]
-; X64-NEXT:    vpaddq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xd4,0xc0]
-; X64-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb4,0xd2]
-; X64-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
-; X64-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
 
   %res = call <8 x i64> @llvm.x86.avx512.maskz.vpmadd52l.uq.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
-  %res1 = call <8 x i64> @llvm.x86.avx512.maskz.vpmadd52l.uq.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <8 x i64> @llvm.x86.avx512.maskz.vpmadd52l.uq.512(<8 x i64> zeroinitializer, <8 x i64> %x1, <8 x i64> zeroinitializer, i8 %x3)
-  %res3 = call <8 x i64> @llvm.x86.avx512.maskz.vpmadd52l.uq.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1)
-  %res4 = add <8 x i64> %res, %res1
-  %res5 = add <8 x i64> %res3, %res2
-  %res6 = add <8 x i64> %res5, %res4
-  ret <8 x i64> %res6
+  ret <8 x i64> %res
 }
 
 define <8 x i64>@test_int_x86_avx512_vpmadd52h_uq_512_load(<8 x i64> %x0, <8 x i64> %x1, <8 x i64>* %x2ptr) {
diff --git a/llvm/test/CodeGen/X86/avx512ifma-intrinsics.ll b/llvm/test/CodeGen/X86/avx512ifma-intrinsics.ll
index 6884666a296ce..b7af9b8b63d0e 100644
--- a/llvm/test/CodeGen/X86/avx512ifma-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512ifma-intrinsics.ll
@@ -4,198 +4,106 @@
 
 declare <8 x i64> @llvm.x86.avx512.vpmadd52h.uq.512(<8 x i64>, <8 x i64>, <8 x i64>)
 
+define <8 x i64>@test_int_x86_avx512_vpmadd52h_uq_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpmadd52h_uq_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf2,0xf5,0x48,0xb5,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+
+  %1 = call <8 x i64> @llvm.x86.avx512.vpmadd52h.uq.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2)
+  ret <8 x i64> %1
+}
+
 define <8 x i64>@test_int_x86_avx512_mask_vpmadd52h_uq_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpmadd52h_uq_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
-; X86-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm3 # encoding: [0x62,0xf2,0xf5,0x48,0xb5,0xda]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm4 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0xb5,0xe2]
-; X86-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0xef,0xd2]
 ; X86-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0xb5,0xc2]
-; X86-NEXT:    vpaddq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xd4,0xc0]
-; X86-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb5,0xd2]
-; X86-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
-; X86-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpmadd52h_uq_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
-; X64-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm3 # encoding: [0x62,0xf2,0xf5,0x48,0xb5,0xda]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X64-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm4 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0xb5,0xe2]
-; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0xef,0xd2]
 ; X64-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0xb5,0xc2]
-; X64-NEXT:    vpaddq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xd4,0xc0]
-; X64-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb5,0xd2]
-; X64-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
-; X64-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
 
   %1 = call <8 x i64> @llvm.x86.avx512.vpmadd52h.uq.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2)
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x0
-  %4 = call <8 x i64> @llvm.x86.avx512.vpmadd52h.uq.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> zeroinitializer)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %6 = select <8 x i1> %5, <8 x i64> %4, <8 x i64> %x0
-  %7 = call <8 x i64> @llvm.x86.avx512.vpmadd52h.uq.512(<8 x i64> zeroinitializer, <8 x i64> %x1, <8 x i64> zeroinitializer)
-  %8 = bitcast i8 %x3 to <8 x i1>
-  %9 = select <8 x i1> %8, <8 x i64> %7, <8 x i64> zeroinitializer
-  %10 = call <8 x i64> @llvm.x86.avx512.vpmadd52h.uq.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2)
-  %res4 = add <8 x i64> %3, %6
-  %res5 = add <8 x i64> %10, %9
-  %res6 = add <8 x i64> %res5, %res4
-  ret <8 x i64> %res6
+  ret <8 x i64> %3
 }
 
 define <8 x i64>@test_int_x86_avx512_maskz_vpmadd52h_uq_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_maskz_vpmadd52h_uq_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
-; X86-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm3 # encoding: [0x62,0xf2,0xf5,0x48,0xb5,0xda]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb5,0xe2]
-; X86-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0xef,0xd2]
 ; X86-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb5,0xc2]
-; X86-NEXT:    vpaddq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xd4,0xc0]
-; X86-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb5,0xd2]
-; X86-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
-; X86-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_vpmadd52h_uq_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
-; X64-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm3 # encoding: [0x62,0xf2,0xf5,0x48,0xb5,0xda]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X64-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb5,0xe2]
-; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0xef,0xd2]
 ; X64-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb5,0xc2]
-; X64-NEXT:    vpaddq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xd4,0xc0]
-; X64-NEXT:    vpmadd52huq %zmm2, %zmm1, %zmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb5,0xd2]
-; X64-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
-; X64-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
 
   %1 = call <8 x i64> @llvm.x86.avx512.vpmadd52h.uq.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2)
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> zeroinitializer
-  %4 = call <8 x i64> @llvm.x86.avx512.vpmadd52h.uq.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> zeroinitializer)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %6 = select <8 x i1> %5, <8 x i64> %4, <8 x i64> zeroinitializer
-  %7 = call <8 x i64> @llvm.x86.avx512.vpmadd52h.uq.512(<8 x i64> zeroinitializer, <8 x i64> %x1, <8 x i64> zeroinitializer)
-  %8 = bitcast i8 %x3 to <8 x i1>
-  %9 = select <8 x i1> %8, <8 x i64> %7, <8 x i64> zeroinitializer
-  %10 = call <8 x i64> @llvm.x86.avx512.vpmadd52h.uq.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2)
-  %res4 = add <8 x i64> %3, %6
-  %res5 = add <8 x i64> %10, %9
-  %res6 = add <8 x i64> %res5, %res4
-  ret <8 x i64> %res6
+  ret <8 x i64> %3
 }
 
 declare <8 x i64> @llvm.x86.avx512.vpmadd52l.uq.512(<8 x i64>, <8 x i64>, <8 x i64>)
 
+define <8 x i64>@test_int_x86_avx512_vpmadd52l_uq_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpmadd52l_uq_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf2,0xf5,0x48,0xb4,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+
+  %1 = call <8 x i64> @llvm.x86.avx512.vpmadd52l.uq.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2)
+  ret <8 x i64> %1
+}
+
 define <8 x i64>@test_int_x86_avx512_mask_vpmadd52l_uq_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpmadd52l_uq_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
-; X86-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm3 # encoding: [0x62,0xf2,0xf5,0x48,0xb4,0xda]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm4 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0xb4,0xe2]
-; X86-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0xef,0xd2]
 ; X86-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0xb4,0xc2]
-; X86-NEXT:    vpaddq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xd4,0xc0]
-; X86-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb4,0xd2]
-; X86-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
-; X86-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpmadd52l_uq_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
-; X64-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm3 # encoding: [0x62,0xf2,0xf5,0x48,0xb4,0xda]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X64-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm4 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0xb4,0xe2]
-; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0xef,0xd2]
 ; X64-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0xb4,0xc2]
-; X64-NEXT:    vpaddq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xd4,0xc0]
-; X64-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb4,0xd2]
-; X64-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
-; X64-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
 
   %1 = call <8 x i64> @llvm.x86.avx512.vpmadd52l.uq.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2)
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x0
-  %4 = call <8 x i64> @llvm.x86.avx512.vpmadd52l.uq.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> zeroinitializer)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %6 = select <8 x i1> %5, <8 x i64> %4, <8 x i64> %x0
-  %7 = call <8 x i64> @llvm.x86.avx512.vpmadd52l.uq.512(<8 x i64> zeroinitializer, <8 x i64> %x1, <8 x i64> zeroinitializer)
-  %8 = bitcast i8 %x3 to <8 x i1>
-  %9 = select <8 x i1> %8, <8 x i64> %7, <8 x i64> zeroinitializer
-  %10 = call <8 x i64> @llvm.x86.avx512.vpmadd52l.uq.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2)
-  %res4 = add <8 x i64> %3, %6
-  %res5 = add <8 x i64> %10, %9
-  %res6 = add <8 x i64> %res5, %res4
-  ret <8 x i64> %res6
+  ret <8 x i64> %3
 }
 
 define <8 x i64>@test_int_x86_avx512_maskz_vpmadd52l_uq_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_maskz_vpmadd52l_uq_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
-; X86-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm3 # encoding: [0x62,0xf2,0xf5,0x48,0xb4,0xda]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb4,0xe2]
-; X86-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0xef,0xd2]
 ; X86-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb4,0xc2]
-; X86-NEXT:    vpaddq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xd4,0xc0]
-; X86-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb4,0xd2]
-; X86-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
-; X86-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_vpmadd52l_uq_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
-; X64-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm3 # encoding: [0x62,0xf2,0xf5,0x48,0xb4,0xda]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X64-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb4,0xe2]
-; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0xef,0xd2]
 ; X64-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb4,0xc2]
-; X64-NEXT:    vpaddq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xd4,0xc0]
-; X64-NEXT:    vpmadd52luq %zmm2, %zmm1, %zmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0xb4,0xd2]
-; X64-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
-; X64-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
 
   %1 = call <8 x i64> @llvm.x86.avx512.vpmadd52l.uq.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2)
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> zeroinitializer
-  %4 = call <8 x i64> @llvm.x86.avx512.vpmadd52l.uq.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> zeroinitializer)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %6 = select <8 x i1> %5, <8 x i64> %4, <8 x i64> zeroinitializer
-  %7 = call <8 x i64> @llvm.x86.avx512.vpmadd52l.uq.512(<8 x i64> zeroinitializer, <8 x i64> %x1, <8 x i64> zeroinitializer)
-  %8 = bitcast i8 %x3 to <8 x i1>
-  %9 = select <8 x i1> %8, <8 x i64> %7, <8 x i64> zeroinitializer
-  %10 = call <8 x i64> @llvm.x86.avx512.vpmadd52l.uq.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2)
-  %res4 = add <8 x i64> %3, %6
-  %res5 = add <8 x i64> %10, %9
-  %res6 = add <8 x i64> %res5, %res4
-  ret <8 x i64> %res6
+  ret <8 x i64> %3
 }
 
 define <8 x i64>@test_int_x86_avx512_vpmadd52h_uq_512_load(<8 x i64> %x0, <8 x i64> %x1, <8 x i64>* %x2ptr) {
diff --git a/llvm/test/CodeGen/X86/avx512ifmavl-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512ifmavl-intrinsics-upgrade.ll
index b417e51370c21..0d59948caafde 100644
--- a/llvm/test/CodeGen/X86/avx512ifmavl-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512ifmavl-intrinsics-upgrade.ll
@@ -4,90 +4,62 @@
 
 declare <2 x i64> @llvm.x86.avx512.mask.vpmadd52h.uq.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
 
+define <2 x i64>@test_int_x86_avx512_vpmadd52h_uq_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpmadd52h_uq_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf2,0xf5,0x08,0xb5,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+
+  %res = call <2 x i64> @llvm.x86.avx512.mask.vpmadd52h.uq.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1)
+  ret <2 x i64> %res
+}
+
 define <2 x i64>@test_int_x86_avx512_mask_vpmadd52h_uq_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpmadd52h_uq_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X86-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf2,0xf5,0x08,0xb5,0xda]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm4 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0xb5,0xe2]
-; X86-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X86-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0xb5,0xc2]
-; X86-NEXT:    vpaddq %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc0]
-; X86-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb5,0xd2]
-; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
-; X86-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpmadd52h_uq_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X64-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf2,0xf5,0x08,0xb5,0xda]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X64-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm4 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0xb5,0xe2]
-; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X64-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0xb5,0xc2]
-; X64-NEXT:    vpaddq %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc0]
-; X64-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb5,0xd2]
-; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
-; X64-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
 
   %res = call <2 x i64> @llvm.x86.avx512.mask.vpmadd52h.uq.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3)
-  %res1 = call <2 x i64> @llvm.x86.avx512.mask.vpmadd52h.uq.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <2 x i64> @llvm.x86.avx512.mask.vpmadd52h.uq.128(<2 x i64> zeroinitializer, <2 x i64> %x1, <2 x i64> zeroinitializer, i8 %x3)
-  %res3 = call <2 x i64> @llvm.x86.avx512.mask.vpmadd52h.uq.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1)
-  %res4 = add <2 x i64> %res, %res1
-  %res5 = add <2 x i64> %res3, %res2
-  %res6 = add <2 x i64> %res5, %res4
-  ret <2 x i64> %res6
+  ret <2 x i64> %res
 }
 
 declare <4 x i64> @llvm.x86.avx512.mask.vpmadd52h.uq.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
 
+define <4 x i64>@test_int_x86_avx512_vpmadd52h_uq_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpmadd52h_uq_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf2,0xf5,0x28,0xb5,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+
+  %res = call <4 x i64> @llvm.x86.avx512.mask.vpmadd52h.uq.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 -1)
+  ret <4 x i64> %res
+}
+
 define <4 x i64>@test_int_x86_avx512_mask_vpmadd52h_uq_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpmadd52h_uq_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X86-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf2,0xf5,0x28,0xb5,0xda]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm4 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0xb5,0xe2]
-; X86-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X86-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0xb5,0xc2]
-; X86-NEXT:    vpaddq %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc0]
-; X86-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb5,0xd2]
-; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
-; X86-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpmadd52h_uq_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X64-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf2,0xf5,0x28,0xb5,0xda]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X64-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm4 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0xb5,0xe2]
-; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X64-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0xb5,0xc2]
-; X64-NEXT:    vpaddq %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc0]
-; X64-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb5,0xd2]
-; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
-; X64-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
 
   %res = call <4 x i64> @llvm.x86.avx512.mask.vpmadd52h.uq.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3)
-  %res1 = call <4 x i64> @llvm.x86.avx512.mask.vpmadd52h.uq.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <4 x i64> @llvm.x86.avx512.mask.vpmadd52h.uq.256(<4 x i64> zeroinitializer, <4 x i64> %x1, <4 x i64> zeroinitializer, i8 %x3)
-  %res3 = call <4 x i64> @llvm.x86.avx512.mask.vpmadd52h.uq.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 -1)
-  %res4 = add <4 x i64> %res, %res1
-  %res5 = add <4 x i64> %res3, %res2
-  %res6 = add <4 x i64> %res5, %res4
-  ret <4 x i64> %res6
+  ret <4 x i64> %res
 }
 
 declare <2 x i64> @llvm.x86.avx512.maskz.vpmadd52h.uq.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
@@ -95,43 +67,19 @@ declare <2 x i64> @llvm.x86.avx512.maskz.vpmadd52h.uq.128(<2 x i64>, <2 x i64>,
 define <2 x i64>@test_int_x86_avx512_maskz_vpmadd52h_uq_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_maskz_vpmadd52h_uq_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X86-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf2,0xf5,0x08,0xb5,0xda]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb5,0xe2]
-; X86-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X86-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb5,0xc2]
-; X86-NEXT:    vpaddq %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc0]
-; X86-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb5,0xd2]
-; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
-; X86-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_vpmadd52h_uq_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X64-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf2,0xf5,0x08,0xb5,0xda]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X64-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb5,0xe2]
-; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X64-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb5,0xc2]
-; X64-NEXT:    vpaddq %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc0]
-; X64-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb5,0xd2]
-; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
-; X64-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
 
   %res = call <2 x i64> @llvm.x86.avx512.maskz.vpmadd52h.uq.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3)
-  %res1 = call <2 x i64> @llvm.x86.avx512.maskz.vpmadd52h.uq.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <2 x i64> @llvm.x86.avx512.maskz.vpmadd52h.uq.128(<2 x i64> zeroinitializer, <2 x i64> %x1, <2 x i64> zeroinitializer, i8 %x3)
-  %res3 = call <2 x i64> @llvm.x86.avx512.maskz.vpmadd52h.uq.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1)
-  %res4 = add <2 x i64> %res, %res1
-  %res5 = add <2 x i64> %res3, %res2
-  %res6 = add <2 x i64> %res5, %res4
-  ret <2 x i64> %res6
+  ret <2 x i64> %res
 }
 
 declare <4 x i64> @llvm.x86.avx512.maskz.vpmadd52h.uq.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
@@ -139,131 +87,79 @@ declare <4 x i64> @llvm.x86.avx512.maskz.vpmadd52h.uq.256(<4 x i64>, <4 x i64>,
 define <4 x i64>@test_int_x86_avx512_maskz_vpmadd52h_uq_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_maskz_vpmadd52h_uq_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X86-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf2,0xf5,0x28,0xb5,0xda]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb5,0xe2]
-; X86-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X86-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb5,0xc2]
-; X86-NEXT:    vpaddq %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc0]
-; X86-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb5,0xd2]
-; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
-; X86-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_vpmadd52h_uq_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X64-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf2,0xf5,0x28,0xb5,0xda]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X64-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb5,0xe2]
-; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X64-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb5,0xc2]
-; X64-NEXT:    vpaddq %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc0]
-; X64-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb5,0xd2]
-; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
-; X64-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
 
   %res = call <4 x i64> @llvm.x86.avx512.maskz.vpmadd52h.uq.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3)
-  %res1 = call <4 x i64> @llvm.x86.avx512.maskz.vpmadd52h.uq.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <4 x i64> @llvm.x86.avx512.maskz.vpmadd52h.uq.256(<4 x i64> zeroinitializer, <4 x i64> %x1, <4 x i64> zeroinitializer, i8 %x3)
-  %res3 = call <4 x i64> @llvm.x86.avx512.maskz.vpmadd52h.uq.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 -1)
-  %res4 = add <4 x i64> %res, %res1
-  %res5 = add <4 x i64> %res3, %res2
-  %res6 = add <4 x i64> %res5, %res4
-  ret <4 x i64> %res6
+  ret <4 x i64> %res
 }
 
 declare <2 x i64> @llvm.x86.avx512.mask.vpmadd52l.uq.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
 
+define <2 x i64>@test_int_x86_avx512_vpmadd52l_uq_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpmadd52l_uq_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf2,0xf5,0x08,0xb4,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+
+  %res = call <2 x i64> @llvm.x86.avx512.mask.vpmadd52l.uq.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1)
+  ret <2 x i64> %res
+}
+
 define <2 x i64>@test_int_x86_avx512_mask_vpmadd52l_uq_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpmadd52l_uq_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X86-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf2,0xf5,0x08,0xb4,0xda]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm4 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0xb4,0xe2]
-; X86-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X86-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0xb4,0xc2]
-; X86-NEXT:    vpaddq %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc0]
-; X86-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb4,0xd2]
-; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
-; X86-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpmadd52l_uq_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X64-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf2,0xf5,0x08,0xb4,0xda]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X64-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm4 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0xb4,0xe2]
-; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X64-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0xb4,0xc2]
-; X64-NEXT:    vpaddq %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc0]
-; X64-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb4,0xd2]
-; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
-; X64-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
 
   %res = call <2 x i64> @llvm.x86.avx512.mask.vpmadd52l.uq.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3)
-  %res1 = call <2 x i64> @llvm.x86.avx512.mask.vpmadd52l.uq.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <2 x i64> @llvm.x86.avx512.mask.vpmadd52l.uq.128(<2 x i64> zeroinitializer, <2 x i64> %x1, <2 x i64> zeroinitializer, i8 %x3)
-  %res3 = call <2 x i64> @llvm.x86.avx512.mask.vpmadd52l.uq.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1)
-  %res4 = add <2 x i64> %res, %res1
-  %res5 = add <2 x i64> %res3, %res2
-  %res6 = add <2 x i64> %res5, %res4
-  ret <2 x i64> %res6
+  ret <2 x i64> %res
 }
 
 declare <4 x i64> @llvm.x86.avx512.mask.vpmadd52l.uq.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
 
+define <4 x i64>@test_int_x86_avx512_vpmadd52l_uq_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpmadd52l_uq_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf2,0xf5,0x28,0xb4,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+
+  %res = call <4 x i64> @llvm.x86.avx512.mask.vpmadd52l.uq.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 -1)
+  ret <4 x i64> %res
+}
+
 define <4 x i64>@test_int_x86_avx512_mask_vpmadd52l_uq_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpmadd52l_uq_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X86-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf2,0xf5,0x28,0xb4,0xda]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm4 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0xb4,0xe2]
-; X86-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X86-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0xb4,0xc2]
-; X86-NEXT:    vpaddq %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc0]
-; X86-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb4,0xd2]
-; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
-; X86-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpmadd52l_uq_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X64-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf2,0xf5,0x28,0xb4,0xda]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X64-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm4 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0xb4,0xe2]
-; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X64-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0xb4,0xc2]
-; X64-NEXT:    vpaddq %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc0]
-; X64-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb4,0xd2]
-; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
-; X64-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
 
   %res = call <4 x i64> @llvm.x86.avx512.mask.vpmadd52l.uq.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3)
-  %res1 = call <4 x i64> @llvm.x86.avx512.mask.vpmadd52l.uq.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <4 x i64> @llvm.x86.avx512.mask.vpmadd52l.uq.256(<4 x i64> zeroinitializer, <4 x i64> %x1, <4 x i64> zeroinitializer, i8 %x3)
-  %res3 = call <4 x i64> @llvm.x86.avx512.mask.vpmadd52l.uq.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 -1)
-  %res4 = add <4 x i64> %res, %res1
-  %res5 = add <4 x i64> %res3, %res2
-  %res6 = add <4 x i64> %res5, %res4
-  ret <4 x i64> %res6
+  ret <4 x i64> %res
 }
 
 declare <2 x i64> @llvm.x86.avx512.maskz.vpmadd52l.uq.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
@@ -271,43 +167,19 @@ declare <2 x i64> @llvm.x86.avx512.maskz.vpmadd52l.uq.128(<2 x i64>, <2 x i64>,
 define <2 x i64>@test_int_x86_avx512_maskz_vpmadd52l_uq_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_maskz_vpmadd52l_uq_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X86-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf2,0xf5,0x08,0xb4,0xda]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb4,0xe2]
-; X86-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X86-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb4,0xc2]
-; X86-NEXT:    vpaddq %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc0]
-; X86-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb4,0xd2]
-; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
-; X86-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_vpmadd52l_uq_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X64-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf2,0xf5,0x08,0xb4,0xda]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X64-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb4,0xe2]
-; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X64-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb4,0xc2]
-; X64-NEXT:    vpaddq %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc0]
-; X64-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb4,0xd2]
-; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
-; X64-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
 
   %res = call <2 x i64> @llvm.x86.avx512.maskz.vpmadd52l.uq.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3)
-  %res1 = call <2 x i64> @llvm.x86.avx512.maskz.vpmadd52l.uq.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <2 x i64> @llvm.x86.avx512.maskz.vpmadd52l.uq.128(<2 x i64> zeroinitializer, <2 x i64> %x1, <2 x i64> zeroinitializer, i8 %x3)
-  %res3 = call <2 x i64> @llvm.x86.avx512.maskz.vpmadd52l.uq.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1)
-  %res4 = add <2 x i64> %res, %res1
-  %res5 = add <2 x i64> %res3, %res2
-  %res6 = add <2 x i64> %res5, %res4
-  ret <2 x i64> %res6
+  ret <2 x i64> %res
 }
 
 declare <4 x i64> @llvm.x86.avx512.maskz.vpmadd52l.uq.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
@@ -315,41 +187,17 @@ declare <4 x i64> @llvm.x86.avx512.maskz.vpmadd52l.uq.256(<4 x i64>, <4 x i64>,
 define <4 x i64>@test_int_x86_avx512_maskz_vpmadd52l_uq_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_maskz_vpmadd52l_uq_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X86-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf2,0xf5,0x28,0xb4,0xda]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb4,0xe2]
-; X86-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X86-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb4,0xc2]
-; X86-NEXT:    vpaddq %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc0]
-; X86-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb4,0xd2]
-; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
-; X86-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_vpmadd52l_uq_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X64-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf2,0xf5,0x28,0xb4,0xda]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X64-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb4,0xe2]
-; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X64-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb4,0xc2]
-; X64-NEXT:    vpaddq %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc0]
-; X64-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb4,0xd2]
-; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
-; X64-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
 
   %res = call <4 x i64> @llvm.x86.avx512.maskz.vpmadd52l.uq.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3)
-  %res1 = call <4 x i64> @llvm.x86.avx512.maskz.vpmadd52l.uq.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <4 x i64> @llvm.x86.avx512.maskz.vpmadd52l.uq.256(<4 x i64> zeroinitializer, <4 x i64> %x1, <4 x i64> zeroinitializer, i8 %x3)
-  %res3 = call <4 x i64> @llvm.x86.avx512.maskz.vpmadd52l.uq.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 -1)
-  %res4 = add <4 x i64> %res, %res1
-  %res5 = add <4 x i64> %res3, %res2
-  %res6 = add <4 x i64> %res5, %res4
-  ret <4 x i64> %res6
+  ret <4 x i64> %res
 }
diff --git a/llvm/test/CodeGen/X86/avx512ifmavl-intrinsics.ll b/llvm/test/CodeGen/X86/avx512ifmavl-intrinsics.ll
index 3533827319d5a..26e269d2d8116 100644
--- a/llvm/test/CodeGen/X86/avx512ifmavl-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512ifmavl-intrinsics.ll
@@ -4,416 +4,216 @@
 
 declare <2 x i64> @llvm.x86.avx512.vpmadd52h.uq.128(<2 x i64>, <2 x i64>, <2 x i64>)
 
+define <2 x i64>@test_int_x86_avx512_vpmadd52h_uq_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpmadd52h_uq_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf2,0xf5,0x08,0xb5,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+
+  %1 = call <2 x i64> @llvm.x86.avx512.vpmadd52h.uq.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2)
+  ret <2 x i64> %1
+}
+
 define <2 x i64>@test_int_x86_avx512_mask_vpmadd52h_uq_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpmadd52h_uq_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X86-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf2,0xf5,0x08,0xb5,0xda]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm4 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0xb5,0xe2]
-; X86-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X86-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0xb5,0xc2]
-; X86-NEXT:    vpaddq %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc0]
-; X86-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb5,0xd2]
-; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
-; X86-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpmadd52h_uq_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X64-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf2,0xf5,0x08,0xb5,0xda]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X64-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm4 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0xb5,0xe2]
-; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X64-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0xb5,0xc2]
-; X64-NEXT:    vpaddq %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc0]
-; X64-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb5,0xd2]
-; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
-; X64-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
 
   %1 = call <2 x i64> @llvm.x86.avx512.vpmadd52h.uq.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2)
   %2 = bitcast i8 %x3 to <8 x i1>
   %extract2 = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
   %3 = select <2 x i1> %extract2, <2 x i64> %1, <2 x i64> %x0
-  %4 = call <2 x i64> @llvm.x86.avx512.vpmadd52h.uq.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> zeroinitializer)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %5, <8 x i1> %5, <2 x i32> <i32 0, i32 1>
-  %6 = select <2 x i1> %extract1, <2 x i64> %4, <2 x i64> %x0
-  %7 = call <2 x i64> @llvm.x86.avx512.vpmadd52h.uq.128(<2 x i64> zeroinitializer, <2 x i64> %x1, <2 x i64> zeroinitializer)
-  %8 = bitcast i8 %x3 to <8 x i1>
-  %extract = shufflevector <8 x i1> %8, <8 x i1> %8, <2 x i32> <i32 0, i32 1>
-  %9 = select <2 x i1> %extract, <2 x i64> %7, <2 x i64> zeroinitializer
-  %10 = call <2 x i64> @llvm.x86.avx512.vpmadd52h.uq.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2)
-  %res4 = add <2 x i64> %3, %6
-  %res5 = add <2 x i64> %10, %9
-  %res6 = add <2 x i64> %res5, %res4
-  ret <2 x i64> %res6
+  ret <2 x i64> %3
 }
 
 declare <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64>, <4 x i64>, <4 x i64>)
 
+define <4 x i64>@test_int_x86_avx512_vpmadd52h_uq_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpmadd52h_uq_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf2,0xf5,0x28,0xb5,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+
+  %1 = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2)
+  ret <4 x i64> %1
+}
+
 define <4 x i64>@test_int_x86_avx512_mask_vpmadd52h_uq_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpmadd52h_uq_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X86-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf2,0xf5,0x28,0xb5,0xda]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm4 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0xb5,0xe2]
-; X86-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X86-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0xb5,0xc2]
-; X86-NEXT:    vpaddq %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc0]
-; X86-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb5,0xd2]
-; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
-; X86-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpmadd52h_uq_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X64-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf2,0xf5,0x28,0xb5,0xda]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X64-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm4 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0xb5,0xe2]
-; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X64-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0xb5,0xc2]
-; X64-NEXT:    vpaddq %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc0]
-; X64-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb5,0xd2]
-; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
-; X64-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
 
   %1 = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2)
   %2 = bitcast i8 %x3 to <8 x i1>
   %extract2 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract2, <4 x i64> %1, <4 x i64> %x0
-  %4 = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> zeroinitializer)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %6 = select <4 x i1> %extract1, <4 x i64> %4, <4 x i64> %x0
-  %7 = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> zeroinitializer, <4 x i64> %x1, <4 x i64> zeroinitializer)
-  %8 = bitcast i8 %x3 to <8 x i1>
-  %extract = shufflevector <8 x i1> %8, <8 x i1> %8, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %9 = select <4 x i1> %extract, <4 x i64> %7, <4 x i64> zeroinitializer
-  %10 = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2)
-  %res4 = add <4 x i64> %3, %6
-  %res5 = add <4 x i64> %10, %9
-  %res6 = add <4 x i64> %res5, %res4
-  ret <4 x i64> %res6
+  ret <4 x i64> %3
 }
 
 define <2 x i64>@test_int_x86_avx512_maskz_vpmadd52h_uq_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_maskz_vpmadd52h_uq_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X86-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf2,0xf5,0x08,0xb5,0xda]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb5,0xe2]
-; X86-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X86-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb5,0xc2]
-; X86-NEXT:    vpaddq %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc0]
-; X86-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb5,0xd2]
-; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
-; X86-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_vpmadd52h_uq_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X64-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf2,0xf5,0x08,0xb5,0xda]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X64-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb5,0xe2]
-; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X64-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb5,0xc2]
-; X64-NEXT:    vpaddq %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc0]
-; X64-NEXT:    vpmadd52huq %xmm2, %xmm1, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb5,0xd2]
-; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
-; X64-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
 
   %1 = call <2 x i64> @llvm.x86.avx512.vpmadd52h.uq.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2)
   %2 = bitcast i8 %x3 to <8 x i1>
   %extract2 = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
   %3 = select <2 x i1> %extract2, <2 x i64> %1, <2 x i64> zeroinitializer
-  %4 = call <2 x i64> @llvm.x86.avx512.vpmadd52h.uq.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> zeroinitializer)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %5, <8 x i1> %5, <2 x i32> <i32 0, i32 1>
-  %6 = select <2 x i1> %extract1, <2 x i64> %4, <2 x i64> zeroinitializer
-  %7 = call <2 x i64> @llvm.x86.avx512.vpmadd52h.uq.128(<2 x i64> zeroinitializer, <2 x i64> %x1, <2 x i64> zeroinitializer)
-  %8 = bitcast i8 %x3 to <8 x i1>
-  %extract = shufflevector <8 x i1> %8, <8 x i1> %8, <2 x i32> <i32 0, i32 1>
-  %9 = select <2 x i1> %extract, <2 x i64> %7, <2 x i64> zeroinitializer
-  %10 = call <2 x i64> @llvm.x86.avx512.vpmadd52h.uq.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2)
-  %res4 = add <2 x i64> %3, %6
-  %res5 = add <2 x i64> %10, %9
-  %res6 = add <2 x i64> %res5, %res4
-  ret <2 x i64> %res6
+  ret <2 x i64> %3
 }
 
 define <4 x i64>@test_int_x86_avx512_maskz_vpmadd52h_uq_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_maskz_vpmadd52h_uq_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X86-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf2,0xf5,0x28,0xb5,0xda]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb5,0xe2]
-; X86-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X86-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb5,0xc2]
-; X86-NEXT:    vpaddq %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc0]
-; X86-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb5,0xd2]
-; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
-; X86-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_vpmadd52h_uq_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X64-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf2,0xf5,0x28,0xb5,0xda]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X64-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb5,0xe2]
-; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X64-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb5,0xc2]
-; X64-NEXT:    vpaddq %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc0]
-; X64-NEXT:    vpmadd52huq %ymm2, %ymm1, %ymm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb5,0xd2]
-; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
-; X64-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
 
   %1 = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2)
   %2 = bitcast i8 %x3 to <8 x i1>
   %extract2 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract2, <4 x i64> %1, <4 x i64> zeroinitializer
-  %4 = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> zeroinitializer)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %6 = select <4 x i1> %extract1, <4 x i64> %4, <4 x i64> zeroinitializer
-  %7 = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> zeroinitializer, <4 x i64> %x1, <4 x i64> zeroinitializer)
-  %8 = bitcast i8 %x3 to <8 x i1>
-  %extract = shufflevector <8 x i1> %8, <8 x i1> %8, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %9 = select <4 x i1> %extract, <4 x i64> %7, <4 x i64> zeroinitializer
-  %10 = call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2)
-  %res4 = add <4 x i64> %3, %6
-  %res5 = add <4 x i64> %10, %9
-  %res6 = add <4 x i64> %res5, %res4
-  ret <4 x i64> %res6
+  ret <4 x i64> %3
 }
 
 declare <2 x i64> @llvm.x86.avx512.vpmadd52l.uq.128(<2 x i64>, <2 x i64>, <2 x i64>)
 
+define <2 x i64>@test_int_x86_avx512_vpmadd52l_uq_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpmadd52l_uq_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf2,0xf5,0x08,0xb4,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+
+  %1 = call <2 x i64> @llvm.x86.avx512.vpmadd52l.uq.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2)
+  ret <2 x i64> %1
+}
+
 define <2 x i64>@test_int_x86_avx512_mask_vpmadd52l_uq_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpmadd52l_uq_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X86-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf2,0xf5,0x08,0xb4,0xda]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm4 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0xb4,0xe2]
-; X86-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X86-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0xb4,0xc2]
-; X86-NEXT:    vpaddq %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc0]
-; X86-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb4,0xd2]
-; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
-; X86-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpmadd52l_uq_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X64-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf2,0xf5,0x08,0xb4,0xda]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X64-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm4 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0xb4,0xe2]
-; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X64-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0xb4,0xc2]
-; X64-NEXT:    vpaddq %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc0]
-; X64-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb4,0xd2]
-; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
-; X64-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
 
   %1 = call <2 x i64> @llvm.x86.avx512.vpmadd52l.uq.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2)
   %2 = bitcast i8 %x3 to <8 x i1>
   %extract2 = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
   %3 = select <2 x i1> %extract2, <2 x i64> %1, <2 x i64> %x0
-  %4 = call <2 x i64> @llvm.x86.avx512.vpmadd52l.uq.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> zeroinitializer)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %5, <8 x i1> %5, <2 x i32> <i32 0, i32 1>
-  %6 = select <2 x i1> %extract1, <2 x i64> %4, <2 x i64> %x0
-  %7 = call <2 x i64> @llvm.x86.avx512.vpmadd52l.uq.128(<2 x i64> zeroinitializer, <2 x i64> %x1, <2 x i64> zeroinitializer)
-  %8 = bitcast i8 %x3 to <8 x i1>
-  %extract = shufflevector <8 x i1> %8, <8 x i1> %8, <2 x i32> <i32 0, i32 1>
-  %9 = select <2 x i1> %extract, <2 x i64> %7, <2 x i64> zeroinitializer
-  %10 = call <2 x i64> @llvm.x86.avx512.vpmadd52l.uq.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2)
-  %res4 = add <2 x i64> %3, %6
-  %res5 = add <2 x i64> %10, %9
-  %res6 = add <2 x i64> %res5, %res4
-  ret <2 x i64> %res6
+  ret <2 x i64> %3
 }
 
 declare <4 x i64> @llvm.x86.avx512.vpmadd52l.uq.256(<4 x i64>, <4 x i64>, <4 x i64>)
 
+define <4 x i64>@test_int_x86_avx512_vpmadd52l_uq_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpmadd52l_uq_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf2,0xf5,0x28,0xb4,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+
+  %1 = call <4 x i64> @llvm.x86.avx512.vpmadd52l.uq.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2)
+  ret <4 x i64> %1
+}
+
 define <4 x i64>@test_int_x86_avx512_mask_vpmadd52l_uq_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpmadd52l_uq_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X86-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf2,0xf5,0x28,0xb4,0xda]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm4 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0xb4,0xe2]
-; X86-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X86-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0xb4,0xc2]
-; X86-NEXT:    vpaddq %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc0]
-; X86-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb4,0xd2]
-; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
-; X86-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpmadd52l_uq_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X64-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf2,0xf5,0x28,0xb4,0xda]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X64-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm4 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0xb4,0xe2]
-; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X64-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0xb4,0xc2]
-; X64-NEXT:    vpaddq %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc0]
-; X64-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb4,0xd2]
-; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
-; X64-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
 
   %1 = call <4 x i64> @llvm.x86.avx512.vpmadd52l.uq.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2)
   %2 = bitcast i8 %x3 to <8 x i1>
   %extract2 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract2, <4 x i64> %1, <4 x i64> %x0
-  %4 = call <4 x i64> @llvm.x86.avx512.vpmadd52l.uq.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> zeroinitializer)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %6 = select <4 x i1> %extract1, <4 x i64> %4, <4 x i64> %x0
-  %7 = call <4 x i64> @llvm.x86.avx512.vpmadd52l.uq.256(<4 x i64> zeroinitializer, <4 x i64> %x1, <4 x i64> zeroinitializer)
-  %8 = bitcast i8 %x3 to <8 x i1>
-  %extract = shufflevector <8 x i1> %8, <8 x i1> %8, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %9 = select <4 x i1> %extract, <4 x i64> %7, <4 x i64> zeroinitializer
-  %10 = call <4 x i64> @llvm.x86.avx512.vpmadd52l.uq.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2)
-  %res4 = add <4 x i64> %3, %6
-  %res5 = add <4 x i64> %10, %9
-  %res6 = add <4 x i64> %res5, %res4
-  ret <4 x i64> %res6
+  ret <4 x i64> %3
 }
 
 define <2 x i64>@test_int_x86_avx512_maskz_vpmadd52l_uq_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_maskz_vpmadd52l_uq_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X86-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf2,0xf5,0x08,0xb4,0xda]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb4,0xe2]
-; X86-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X86-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb4,0xc2]
-; X86-NEXT:    vpaddq %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc0]
-; X86-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb4,0xd2]
-; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
-; X86-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_vpmadd52l_uq_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X64-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf2,0xf5,0x08,0xb4,0xda]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X64-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb4,0xe2]
-; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X64-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb4,0xc2]
-; X64-NEXT:    vpaddq %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc0]
-; X64-NEXT:    vpmadd52luq %xmm2, %xmm1, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xb4,0xd2]
-; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
-; X64-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
 
   %1 = call <2 x i64> @llvm.x86.avx512.vpmadd52l.uq.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2)
   %2 = bitcast i8 %x3 to <8 x i1>
   %extract2 = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
   %3 = select <2 x i1> %extract2, <2 x i64> %1, <2 x i64> zeroinitializer
-  %4 = call <2 x i64> @llvm.x86.avx512.vpmadd52l.uq.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> zeroinitializer)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %5, <8 x i1> %5, <2 x i32> <i32 0, i32 1>
-  %6 = select <2 x i1> %extract1, <2 x i64> %4, <2 x i64> zeroinitializer
-  %7 = call <2 x i64> @llvm.x86.avx512.vpmadd52l.uq.128(<2 x i64> zeroinitializer, <2 x i64> %x1, <2 x i64> zeroinitializer)
-  %8 = bitcast i8 %x3 to <8 x i1>
-  %extract = shufflevector <8 x i1> %8, <8 x i1> %8, <2 x i32> <i32 0, i32 1>
-  %9 = select <2 x i1> %extract, <2 x i64> %7, <2 x i64> zeroinitializer
-  %10 = call <2 x i64> @llvm.x86.avx512.vpmadd52l.uq.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2)
-  %res4 = add <2 x i64> %3, %6
-  %res5 = add <2 x i64> %10, %9
-  %res6 = add <2 x i64> %res5, %res4
-  ret <2 x i64> %res6
+  ret <2 x i64> %3
 }
 
 define <4 x i64>@test_int_x86_avx512_maskz_vpmadd52l_uq_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_maskz_vpmadd52l_uq_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X86-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf2,0xf5,0x28,0xb4,0xda]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb4,0xe2]
-; X86-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X86-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb4,0xc2]
-; X86-NEXT:    vpaddq %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc0]
-; X86-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb4,0xd2]
-; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
-; X86-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_vpmadd52l_uq_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X64-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf2,0xf5,0x28,0xb4,0xda]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X64-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb4,0xe2]
-; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
 ; X64-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb4,0xc2]
-; X64-NEXT:    vpaddq %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc0]
-; X64-NEXT:    vpmadd52luq %ymm2, %ymm1, %ymm2 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xb4,0xd2]
-; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
-; X64-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
 
   %1 = call <4 x i64> @llvm.x86.avx512.vpmadd52l.uq.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2)
   %2 = bitcast i8 %x3 to <8 x i1>
   %extract2 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract2, <4 x i64> %1, <4 x i64> zeroinitializer
-  %4 = call <4 x i64> @llvm.x86.avx512.vpmadd52l.uq.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> zeroinitializer)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %6 = select <4 x i1> %extract1, <4 x i64> %4, <4 x i64> zeroinitializer
-  %7 = call <4 x i64> @llvm.x86.avx512.vpmadd52l.uq.256(<4 x i64> zeroinitializer, <4 x i64> %x1, <4 x i64> zeroinitializer)
-  %8 = bitcast i8 %x3 to <8 x i1>
-  %extract = shufflevector <8 x i1> %8, <8 x i1> %8, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %9 = select <4 x i1> %extract, <4 x i64> %7, <4 x i64> zeroinitializer
-  %10 = call <4 x i64> @llvm.x86.avx512.vpmadd52l.uq.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2)
-  %res4 = add <4 x i64> %3, %6
-  %res5 = add <4 x i64> %10, %9
-  %res6 = add <4 x i64> %res5, %res4
-  ret <4 x i64> %res6
+  ret <4 x i64> %3
 }
diff --git a/llvm/test/CodeGen/X86/avx512vbmi2-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512vbmi2-intrinsics-upgrade.ll
index 96eba89f6a40e..dc34be5a7b836 100644
--- a/llvm/test/CodeGen/X86/avx512vbmi2-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512vbmi2-intrinsics-upgrade.ll
@@ -367,21 +367,21 @@ declare <64 x i8> @llvm.x86.avx512.mask.compress.b.512(<64 x i8> %data, <64 x i8
 define <16 x i32>@test_int_x86_avx512_mask_vpshld_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x3, i16 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_d_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldd $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0x7d,0x48,0x71,0xd9,0x16]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpshldd $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x49,0x71,0xd1,0x16]
-; X86-NEXT:    vpaddd %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc3]
+; X86-NEXT:    vpshldd $23, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0x7d,0x48,0x71,0xc1,0x17]
+; X86-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_d_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldd $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0x7d,0x48,0x71,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshldd $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x49,0x71,0xd1,0x16]
-; X64-NEXT:    vpaddd %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc3]
+; X64-NEXT:    vpshldd $23, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0x7d,0x48,0x71,0xc1,0x17]
+; X64-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <16 x i32> @llvm.x86.avx512.mask.vpshld.d.512(<16 x i32> %x0, <16 x i32> %x1, i32 22, <16 x i32> %x3, i16 %x4)
-  %res1 = call <16 x i32> @llvm.x86.avx512.mask.vpshld.d.512(<16 x i32> %x0, <16 x i32> %x1, i32 22, <16 x i32> %x3, i16 -1)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.vpshld.d.512(<16 x i32> %x0, <16 x i32> %x1, i32 23, <16 x i32> %x3, i16 -1)
   %res2 = add <16 x i32> %res, %res1
   ret <16 x i32> %res2
 }
@@ -390,22 +390,22 @@ declare <16 x i32> @llvm.x86.avx512.mask.vpshld.d.512(<16 x i32>, <16 x i32>, i3
 define <8 x i64>@test_int_x86_avx512_mask_vpshld_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_q_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldq $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x71,0xd9,0x16]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshldq $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x71,0xd1,0x16]
-; X86-NEXT:    vpaddq %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc3]
+; X86-NEXT:    vpshldq $23, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0xfd,0x48,0x71,0xc1,0x17]
+; X86-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_q_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldq $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x71,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshldq $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x71,0xd1,0x16]
-; X64-NEXT:    vpaddq %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc3]
+; X64-NEXT:    vpshldq $23, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0xfd,0x48,0x71,0xc1,0x17]
+; X64-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i64> @llvm.x86.avx512.mask.vpshld.q.512(<8 x i64> %x0, <8 x i64> %x1, i32 22, <8 x i64> %x3, i8 %x4)
-  %res1 = call <8 x i64> @llvm.x86.avx512.mask.vpshld.q.512(<8 x i64> %x0, <8 x i64> %x1, i32 22, <8 x i64> %x3, i8 -1)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.vpshld.q.512(<8 x i64> %x0, <8 x i64> %x1, i32 23, <8 x i64> %x3, i8 -1)
   %res2 = add <8 x i64> %res, %res1
   ret <8 x i64> %res2
 }
@@ -414,21 +414,21 @@ declare <8 x i64> @llvm.x86.avx512.mask.vpshld.q.512(<8 x i64>, <8 x i64>, i32,
 define <32 x i16>@test_int_x86_avx512_mask_vpshld_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x3, i32 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_w_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldw $6, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x70,0xd9,0x06]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpshldw $6, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x70,0xd1,0x06]
-; X86-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
+; X86-NEXT:    vpshldw $7, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0xfd,0x48,0x70,0xc1,0x07]
+; X86-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_w_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldw $6, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x70,0xd9,0x06]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshldw $6, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x70,0xd1,0x06]
-; X64-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
+; X64-NEXT:    vpshldw $7, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0xfd,0x48,0x70,0xc1,0x07]
+; X64-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <32 x i16> @llvm.x86.avx512.mask.vpshld.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 6, <32 x i16> %x3, i32 %x4)
-  %res1 = call <32 x i16> @llvm.x86.avx512.mask.vpshld.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 6, <32 x i16> %x3, i32 -1)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.vpshld.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 7, <32 x i16> %x3, i32 -1)
   %res2 = add <32 x i16> %res, %res1
   ret <32 x i16> %res2
 }
@@ -437,21 +437,21 @@ declare <32 x i16> @llvm.x86.avx512.mask.vpshld.w.512(<32 x i16>, <32 x i16>, i3
 define <16 x i32>@test_int_x86_avx512_mask_vpshrd_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x3, i16 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_d_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdd $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0x7d,0x48,0x73,0xd9,0x16]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpshrdd $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x49,0x73,0xd1,0x16]
-; X86-NEXT:    vpaddd %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc3]
+; X86-NEXT:    vpshrdd $23, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0x7d,0x48,0x73,0xc1,0x17]
+; X86-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_d_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdd $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0x7d,0x48,0x73,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshrdd $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x49,0x73,0xd1,0x16]
-; X64-NEXT:    vpaddd %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc3]
+; X64-NEXT:    vpshrdd $23, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0x7d,0x48,0x73,0xc1,0x17]
+; X64-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <16 x i32> @llvm.x86.avx512.mask.vpshrd.d.512(<16 x i32> %x0, <16 x i32> %x1, i32 22, <16 x i32> %x3, i16 %x4)
-  %res1 = call <16 x i32> @llvm.x86.avx512.mask.vpshrd.d.512(<16 x i32> %x0, <16 x i32> %x1, i32 22, <16 x i32> %x3, i16 -1)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.vpshrd.d.512(<16 x i32> %x0, <16 x i32> %x1, i32 23, <16 x i32> %x3, i16 -1)
   %res2 = add <16 x i32> %res, %res1
   ret <16 x i32> %res2
 }
@@ -460,22 +460,22 @@ declare <16 x i32> @llvm.x86.avx512.mask.vpshrd.d.512(<16 x i32>, <16 x i32>, i3
 define <8 x i64>@test_int_x86_avx512_mask_vpshrd_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_q_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdq $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x73,0xd9,0x16]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshrdq $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x73,0xd1,0x16]
-; X86-NEXT:    vpaddq %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc3]
+; X86-NEXT:    vpshrdq $23, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0xfd,0x48,0x73,0xc1,0x17]
+; X86-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_q_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdq $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x73,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshrdq $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x73,0xd1,0x16]
-; X64-NEXT:    vpaddq %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc3]
+; X64-NEXT:    vpshrdq $23, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0xfd,0x48,0x73,0xc1,0x17]
+; X64-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i64> @llvm.x86.avx512.mask.vpshrd.q.512(<8 x i64> %x0, <8 x i64> %x1, i32 22, <8 x i64> %x3, i8 %x4)
-  %res1 = call <8 x i64> @llvm.x86.avx512.mask.vpshrd.q.512(<8 x i64> %x0, <8 x i64> %x1, i32 22, <8 x i64> %x3, i8 -1)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.vpshrd.q.512(<8 x i64> %x0, <8 x i64> %x1, i32 23, <8 x i64> %x3, i8 -1)
   %res2 = add <8 x i64> %res, %res1
   ret <8 x i64> %res2
 }
@@ -484,21 +484,21 @@ declare <8 x i64> @llvm.x86.avx512.mask.vpshrd.q.512(<8 x i64>, <8 x i64>, i32,
 define <32 x i16>@test_int_x86_avx512_mask_vpshrd_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x3, i32 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_w_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdw $6, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x72,0xd9,0x06]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpshrdw $6, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x72,0xd1,0x06]
-; X86-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
+; X86-NEXT:    vpshrdw $7, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0xfd,0x48,0x72,0xc1,0x07]
+; X86-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_w_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdw $6, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x72,0xd9,0x06]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshrdw $6, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x72,0xd1,0x06]
-; X64-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
+; X64-NEXT:    vpshrdw $7, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0xfd,0x48,0x72,0xc1,0x07]
+; X64-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <32 x i16> @llvm.x86.avx512.mask.vpshrd.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 6, <32 x i16> %x3, i32 %x4)
-  %res1 = call <32 x i16> @llvm.x86.avx512.mask.vpshrd.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 6, <32 x i16> %x3, i32 -1)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.vpshrd.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 7, <32 x i16> %x3, i32 -1)
   %res2 = add <32 x i16> %res, %res1
   ret <32 x i16> %res2
 }
@@ -507,23 +507,23 @@ declare <32 x i16> @llvm.x86.avx512.mask.vpshrd.w.512(<32 x i16>, <32 x i16>, i3
 define <16 x i32>@test_int_x86_avx512_mask_vpshld_d_512_2(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x3, i16 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_d_512_2:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldd $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0x7d,0x48,0x71,0xd9,0x16]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpshldd $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x49,0x71,0xd1,0x16]
-; X86-NEXT:    vpaddd %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc3]
+; X86-NEXT:    vpshldd $23, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0x7d,0x48,0x71,0xc1,0x17]
+; X86-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_d_512_2:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldd $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0x7d,0x48,0x71,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshldd $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x49,0x71,0xd1,0x16]
-; X64-NEXT:    vpaddd %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc3]
+; X64-NEXT:    vpshldd $23, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0x7d,0x48,0x71,0xc1,0x17]
+; X64-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <16 x i32> @llvm.x86.avx512.vpshld.d.512(<16 x i32> %x0, <16 x i32> %x1, i32 22)
   %2 = bitcast i16 %x4 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x3
-  %4 = call <16 x i32> @llvm.x86.avx512.vpshld.d.512(<16 x i32> %x0, <16 x i32> %x1, i32 22)
+  %4 = call <16 x i32> @llvm.x86.avx512.vpshld.d.512(<16 x i32> %x0, <16 x i32> %x1, i32 23)
   %res2 = add <16 x i32> %3, %4
   ret <16 x i32> %res2
 }
@@ -532,24 +532,24 @@ declare <16 x i32> @llvm.x86.avx512.vpshld.d.512(<16 x i32>, <16 x i32>, i32)
 define <8 x i64>@test_int_x86_avx512_mask_vpshld_q_512_2(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_q_512_2:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldq $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x71,0xd9,0x16]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshldq $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x71,0xd1,0x16]
-; X86-NEXT:    vpaddq %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc3]
+; X86-NEXT:    vpshldq $23, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0xfd,0x48,0x71,0xc1,0x17]
+; X86-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_q_512_2:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldq $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x71,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshldq $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x71,0xd1,0x16]
-; X64-NEXT:    vpaddq %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc3]
+; X64-NEXT:    vpshldq $23, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0xfd,0x48,0x71,0xc1,0x17]
+; X64-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <8 x i64> @llvm.x86.avx512.vpshld.q.512(<8 x i64> %x0, <8 x i64> %x1, i32 22)
   %2 = bitcast i8 %x4 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x3
-  %4 = call <8 x i64> @llvm.x86.avx512.vpshld.q.512(<8 x i64> %x0, <8 x i64> %x1, i32 22)
+  %4 = call <8 x i64> @llvm.x86.avx512.vpshld.q.512(<8 x i64> %x0, <8 x i64> %x1, i32 23)
   %res2 = add <8 x i64> %3, %4
   ret <8 x i64> %res2
 }
@@ -558,23 +558,23 @@ declare <8 x i64> @llvm.x86.avx512.vpshld.q.512(<8 x i64>, <8 x i64>, i32)
 define <32 x i16>@test_int_x86_avx512_mask_vpshld_w_512_2(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x3, i32 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_w_512_2:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldw $6, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x70,0xd9,0x06]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpshldw $6, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x70,0xd1,0x06]
-; X86-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
+; X86-NEXT:    vpshldw $7, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0xfd,0x48,0x70,0xc1,0x07]
+; X86-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_w_512_2:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldw $6, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x70,0xd9,0x06]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshldw $6, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x70,0xd1,0x06]
-; X64-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
+; X64-NEXT:    vpshldw $7, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0xfd,0x48,0x70,0xc1,0x07]
+; X64-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <32 x i16> @llvm.x86.avx512.vpshld.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 6)
   %2 = bitcast i32 %x4 to <32 x i1>
   %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x3
-  %4 = call <32 x i16> @llvm.x86.avx512.vpshld.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 6)
+  %4 = call <32 x i16> @llvm.x86.avx512.vpshld.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 7)
   %res2 = add <32 x i16> %3, %4
   ret <32 x i16> %res2
 }
@@ -583,23 +583,23 @@ declare <32 x i16> @llvm.x86.avx512.vpshld.w.512(<32 x i16>, <32 x i16>, i32)
 define <16 x i32>@test_int_x86_avx512_mask_vpshrd_d_512_2(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x3, i16 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_d_512_2:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdd $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0x7d,0x48,0x73,0xd9,0x16]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpshrdd $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x49,0x73,0xd1,0x16]
-; X86-NEXT:    vpaddd %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc3]
+; X86-NEXT:    vpshrdd $23, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0x7d,0x48,0x73,0xc1,0x17]
+; X86-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_d_512_2:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdd $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0x7d,0x48,0x73,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshrdd $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x49,0x73,0xd1,0x16]
-; X64-NEXT:    vpaddd %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc3]
+; X64-NEXT:    vpshrdd $23, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0x7d,0x48,0x73,0xc1,0x17]
+; X64-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <16 x i32> @llvm.x86.avx512.vpshrd.d.512(<16 x i32> %x0, <16 x i32> %x1, i32 22)
   %2 = bitcast i16 %x4 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x3
-  %4 = call <16 x i32> @llvm.x86.avx512.vpshrd.d.512(<16 x i32> %x0, <16 x i32> %x1, i32 22)
+  %4 = call <16 x i32> @llvm.x86.avx512.vpshrd.d.512(<16 x i32> %x0, <16 x i32> %x1, i32 23)
   %res2 = add <16 x i32> %3, %4
   ret <16 x i32> %res2
 }
@@ -608,24 +608,24 @@ declare <16 x i32> @llvm.x86.avx512.vpshrd.d.512(<16 x i32>, <16 x i32>, i32)
 define <8 x i64>@test_int_x86_avx512_mask_vpshrd_q_512_2(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_q_512_2:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdq $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x73,0xd9,0x16]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshrdq $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x73,0xd1,0x16]
-; X86-NEXT:    vpaddq %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc3]
+; X86-NEXT:    vpshrdq $23, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0xfd,0x48,0x73,0xc1,0x17]
+; X86-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_q_512_2:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdq $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x73,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshrdq $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x73,0xd1,0x16]
-; X64-NEXT:    vpaddq %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc3]
+; X64-NEXT:    vpshrdq $23, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0xfd,0x48,0x73,0xc1,0x17]
+; X64-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <8 x i64> @llvm.x86.avx512.vpshrd.q.512(<8 x i64> %x0, <8 x i64> %x1, i32 22)
   %2 = bitcast i8 %x4 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x3
-  %4 = call <8 x i64> @llvm.x86.avx512.vpshrd.q.512(<8 x i64> %x0, <8 x i64> %x1, i32 22)
+  %4 = call <8 x i64> @llvm.x86.avx512.vpshrd.q.512(<8 x i64> %x0, <8 x i64> %x1, i32 23)
   %res2 = add <8 x i64> %3, %4
   ret <8 x i64> %res2
 }
@@ -634,23 +634,23 @@ declare <8 x i64> @llvm.x86.avx512.vpshrd.q.512(<8 x i64>, <8 x i64>, i32)
 define <32 x i16>@test_int_x86_avx512_mask_vpshrd_w_512_2(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x3, i32 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_w_512_2:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdw $6, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x72,0xd9,0x06]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpshrdw $6, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x72,0xd1,0x06]
-; X86-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
+; X86-NEXT:    vpshrdw $7, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0xfd,0x48,0x72,0xc1,0x07]
+; X86-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_w_512_2:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdw $6, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x72,0xd9,0x06]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshrdw $6, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x72,0xd1,0x06]
-; X64-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
+; X64-NEXT:    vpshrdw $7, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0xfd,0x48,0x72,0xc1,0x07]
+; X64-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <32 x i16> @llvm.x86.avx512.vpshrd.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 6)
   %2 = bitcast i32 %x4 to <32 x i1>
   %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x3
-  %4 = call <32 x i16> @llvm.x86.avx512.vpshrd.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 6)
+  %4 = call <32 x i16> @llvm.x86.avx512.vpshrd.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 7)
   %res2 = add <32 x i16> %3, %4
   ret <32 x i16> %res2
 }
@@ -659,6 +659,15 @@ declare <32 x i16> @llvm.x86.avx512.vpshrd.w.512(<32 x i16>, <32 x i16>, i32)
 declare <16 x i32> @llvm.x86.avx512.mask.vpshrdv.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
 declare <16 x i32> @llvm.x86.avx512.maskz.vpshrdv.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
 
+define <16 x i32>@test_int_x86_avx512_vpshrdv_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpshrdv_d_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpshrdvd %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf2,0x75,0x48,0x73,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <16 x i32> @llvm.x86.avx512.mask.vpshrdv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  ret <16 x i32> %res
+}
+
 define <16 x i32>@test_int_x86_avx512_mask_vpshrdv_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32>* %x2p, <16 x i32> %x4, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrdv_d_512:
 ; X86:       # %bb.0:
@@ -666,10 +675,7 @@ define <16 x i32>@test_int_x86_avx512_mask_vpshrdv_d_512(<16 x i32> %x0, <16 x i
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpshrdvd (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x73,0x18]
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpshrdvd %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x73,0xe2]
 ; X86-NEXT:    vpshrdvd %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x73,0xc2]
-; X86-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -678,24 +684,28 @@ define <16 x i32>@test_int_x86_avx512_mask_vpshrdv_d_512(<16 x i32> %x0, <16 x i
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X64-NEXT:    vpshrdvd (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x73,0x1f]
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X64-NEXT:    vpshrdvd %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x73,0xe2]
 ; X64-NEXT:    vpshrdvd %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x73,0xc2]
-; X64-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <16 x i32>, <16 x i32>* %x2p
   %res = call <16 x i32> @llvm.x86.avx512.mask.vpshrdv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
-  %res1 = call <16 x i32> @llvm.x86.avx512.mask.vpshrdv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16 -1)
-  %res2 = call <16 x i32> @llvm.x86.avx512.maskz.vpshrdv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16  %x3)
-  %res3 = add <16 x i32> %res, %res1
-  %res4 = add <16 x i32> %res2, %res3
-  ret <16 x i32> %res4
+  %res1 = call <16 x i32> @llvm.x86.avx512.maskz.vpshrdv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16  %x3)
+  %res2 = add <16 x i32> %res, %res1
+  ret <16 x i32> %res2
 }
 
 declare <8 x i64> @llvm.x86.avx512.mask.vpshrdv.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
 declare <8 x i64> @llvm.x86.avx512.maskz.vpshrdv.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
 
+define <8 x i64>@test_int_x86_avx512_vpshrdv_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpshrdv_q_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpshrdvq %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf2,0xf5,0x48,0x73,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i64> @llvm.x86.avx512.mask.vpshrdv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1)
+  ret <8 x i64> %res
+}
+
 define <8 x i64>@test_int_x86_avx512_mask_vpshrdv_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64>* %x2p, <8 x i64> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrdv_q_512:
 ; X86:       # %bb.0:
@@ -704,10 +714,7 @@ define <8 x i64>@test_int_x86_avx512_mask_vpshrdv_q_512(<8 x i64> %x0, <8 x i64>
 ; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpshrdvq (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0x73,0x18]
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpshrdvq %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0xf5,0x48,0x73,0xe2]
 ; X86-NEXT:    vpshrdvq %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x73,0xc2]
-; X86-NEXT:    vpaddq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -716,61 +723,59 @@ define <8 x i64>@test_int_x86_avx512_mask_vpshrdv_q_512(<8 x i64> %x0, <8 x i64>
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X64-NEXT:    vpshrdvq (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0x73,0x1f]
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X64-NEXT:    vpshrdvq %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0xf5,0x48,0x73,0xe2]
 ; X64-NEXT:    vpshrdvq %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x73,0xc2]
-; X64-NEXT:    vpaddq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <8 x i64>, <8 x i64>* %x2p
   %res = call <8 x i64> @llvm.x86.avx512.mask.vpshrdv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
-  %res1 = call <8 x i64> @llvm.x86.avx512.mask.vpshrdv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x4, i8 -1)
-  %res2 = call <8 x i64> @llvm.x86.avx512.maskz.vpshrdv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x4, i8  %x3)
-  %res3 = add <8 x i64> %res, %res1
-  %res4 = add <8 x i64> %res2, %res3
-  ret <8 x i64> %res4
+  %res1 = call <8 x i64> @llvm.x86.avx512.maskz.vpshrdv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x4, i8  %x3)
+  %res2 = add <8 x i64> %res, %res1
+  ret <8 x i64> %res2
 }
 
 declare <32 x i16> @llvm.x86.avx512.mask.vpshrdv.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
 declare <32 x i16> @llvm.x86.avx512.maskz.vpshrdv.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
 
+define <32 x i16>@test_int_x86_avx512_vpshrdv_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpshrdv_w_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpshrdvw %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf2,0xf5,0x48,0x72,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.vpshrdv.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
 define <32 x i16>@test_int_x86_avx512_mask_vpshrdv_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16>* %x2p, <32 x i16> %x4, i32 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrdv_w_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
-; X86-NEXT:    vpshrdvw (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0x72,0x18]
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpshrdvw %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0xf5,0x48,0x72,0xe2]
 ; X86-NEXT:    vpshrdvw %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x72,0xc2]
-; X86-NEXT:    vpaddw %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfd,0xc0]
-; X86-NEXT:    vpaddw %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrdv_w_512:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
-; X64-NEXT:    vpshrdvw (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0x72,0x1f]
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X64-NEXT:    vpshrdvw %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0xf5,0x48,0x72,0xe2]
 ; X64-NEXT:    vpshrdvw %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x72,0xc2]
-; X64-NEXT:    vpaddw %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfd,0xc0]
-; X64-NEXT:    vpaddw %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <32 x i16>, <32 x i16>* %x2p
   %res = call <32 x i16> @llvm.x86.avx512.mask.vpshrdv.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
-  %res1 = call <32 x i16> @llvm.x86.avx512.mask.vpshrdv.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x4, i32 -1)
-  %res2 = call <32 x i16> @llvm.x86.avx512.maskz.vpshrdv.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x4, i32  %x3)
-  %res3 = add <32 x i16> %res, %res1
-  %res4 = add <32 x i16> %res2, %res3
-  ret <32 x i16> %res4
+  %res1 = call <32 x i16> @llvm.x86.avx512.maskz.vpshrdv.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x4, i32  %x3)
+  %res2 = add <32 x i16> %res, %res1
+  ret <32 x i16> %res1
 }
 
 declare <16 x i32> @llvm.x86.avx512.mask.vpshldv.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
 declare <16 x i32> @llvm.x86.avx512.maskz.vpshldv.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
 
+define <16 x i32>@test_int_x86_avx512_vpshldv_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpshldv_d_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpshldvd %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf2,0x75,0x48,0x71,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <16 x i32> @llvm.x86.avx512.mask.vpshldv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  ret <16 x i32> %res
+}
+
 define <16 x i32>@test_int_x86_avx512_mask_vpshldv_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32>* %x2p, <16 x i32> %x4, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshldv_d_512:
 ; X86:       # %bb.0:
@@ -778,10 +783,7 @@ define <16 x i32>@test_int_x86_avx512_mask_vpshldv_d_512(<16 x i32> %x0, <16 x i
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpshldvd (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x71,0x18]
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpshldvd %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x71,0xe2]
 ; X86-NEXT:    vpshldvd %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x71,0xc2]
-; X86-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -790,24 +792,28 @@ define <16 x i32>@test_int_x86_avx512_mask_vpshldv_d_512(<16 x i32> %x0, <16 x i
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X64-NEXT:    vpshldvd (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x71,0x1f]
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X64-NEXT:    vpshldvd %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x71,0xe2]
 ; X64-NEXT:    vpshldvd %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x71,0xc2]
-; X64-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <16 x i32>, <16 x i32>* %x2p
   %res = call <16 x i32> @llvm.x86.avx512.mask.vpshldv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
-  %res1 = call <16 x i32> @llvm.x86.avx512.mask.vpshldv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16 -1)
-  %res2 = call <16 x i32> @llvm.x86.avx512.maskz.vpshldv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16  %x3)
-  %res3 = add <16 x i32> %res, %res1
-  %res4 = add <16 x i32> %res2, %res3
-  ret <16 x i32> %res4
+  %res1 = call <16 x i32> @llvm.x86.avx512.maskz.vpshldv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16  %x3)
+  %res2 = add <16 x i32> %res, %res1
+  ret <16 x i32> %res2
 }
 
 declare <8 x i64> @llvm.x86.avx512.mask.vpshldv.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
 declare <8 x i64> @llvm.x86.avx512.maskz.vpshldv.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
 
+define <8 x i64>@test_int_x86_avx512_vpshldv_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpshldv_q_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpshldvq %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf2,0xf5,0x48,0x71,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i64> @llvm.x86.avx512.mask.vpshldv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 -1)
+  ret <8 x i64> %res
+}
+
 define <8 x i64>@test_int_x86_avx512_mask_vpshldv_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64>* %x2p, <8 x i64> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshldv_q_512:
 ; X86:       # %bb.0:
@@ -816,10 +822,7 @@ define <8 x i64>@test_int_x86_avx512_mask_vpshldv_q_512(<8 x i64> %x0, <8 x i64>
 ; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpshldvq (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0x71,0x18]
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpshldvq %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0xf5,0x48,0x71,0xe2]
 ; X86-NEXT:    vpshldvq %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x71,0xc2]
-; X86-NEXT:    vpaddq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -828,24 +831,28 @@ define <8 x i64>@test_int_x86_avx512_mask_vpshldv_q_512(<8 x i64> %x0, <8 x i64>
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X64-NEXT:    vpshldvq (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0x71,0x1f]
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X64-NEXT:    vpshldvq %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0xf5,0x48,0x71,0xe2]
 ; X64-NEXT:    vpshldvq %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x71,0xc2]
-; X64-NEXT:    vpaddq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <8 x i64>, <8 x i64>* %x2p
   %res = call <8 x i64> @llvm.x86.avx512.mask.vpshldv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
-  %res1 = call <8 x i64> @llvm.x86.avx512.mask.vpshldv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x4, i8 -1)
-  %res2 = call <8 x i64> @llvm.x86.avx512.maskz.vpshldv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x4, i8  %x3)
-  %res3 = add <8 x i64> %res, %res1
-  %res4 = add <8 x i64> %res2, %res3
-  ret <8 x i64> %res4
+  %res1 = call <8 x i64> @llvm.x86.avx512.maskz.vpshldv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x4, i8  %x3)
+  %res2 = add <8 x i64> %res, %res1
+  ret <8 x i64> %res2
 }
 
 declare <32 x i16> @llvm.x86.avx512.mask.vpshldv.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
 declare <32 x i16> @llvm.x86.avx512.maskz.vpshldv.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
 
+define <32 x i16>@test_int_x86_avx512_vpshldv_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpshldv_w_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpshldvw %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf2,0xf5,0x48,0x70,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.vpshldv.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  ret <32 x i16> %res
+}
+
 define <32 x i16>@test_int_x86_avx512_mask_vpshldv_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16>* %x2p, <32 x i16> %x4, i32 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshldv_w_512:
 ; X86:       # %bb.0:
@@ -853,10 +860,7 @@ define <32 x i16>@test_int_x86_avx512_mask_vpshldv_w_512(<32 x i16> %x0, <32 x i
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpshldvw (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0x70,0x18]
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpshldvw %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0xf5,0x48,0x70,0xe2]
 ; X86-NEXT:    vpshldvw %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x70,0xc2]
-; X86-NEXT:    vpaddw %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfd,0xc0]
 ; X86-NEXT:    vpaddw %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -865,17 +869,12 @@ define <32 x i16>@test_int_x86_avx512_mask_vpshldv_w_512(<32 x i16> %x0, <32 x i
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X64-NEXT:    vpshldvw (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0x70,0x1f]
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X64-NEXT:    vpshldvw %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0xf5,0x48,0x70,0xe2]
 ; X64-NEXT:    vpshldvw %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x70,0xc2]
-; X64-NEXT:    vpaddw %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfd,0xc0]
 ; X64-NEXT:    vpaddw %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <32 x i16>, <32 x i16>* %x2p
   %res = call <32 x i16> @llvm.x86.avx512.mask.vpshldv.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
-  %res1 = call <32 x i16> @llvm.x86.avx512.mask.vpshldv.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x4, i32 -1)
-  %res2 = call <32 x i16> @llvm.x86.avx512.maskz.vpshldv.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x4, i32  %x3)
-  %res3 = add <32 x i16> %res, %res1
-  %res4 = add <32 x i16> %res2, %res3
-  ret <32 x i16> %res4
+  %res1 = call <32 x i16> @llvm.x86.avx512.maskz.vpshldv.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x4, i32  %x3)
+  %res2 = add <32 x i16> %res, %res1
+  ret <32 x i16> %res2
 }
diff --git a/llvm/test/CodeGen/X86/avx512vbmi2-intrinsics.ll b/llvm/test/CodeGen/X86/avx512vbmi2-intrinsics.ll
index a16b91b6218e3..c1e164252249b 100644
--- a/llvm/test/CodeGen/X86/avx512vbmi2-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512vbmi2-intrinsics.ll
@@ -370,23 +370,23 @@ define <64 x i8> @test_compress_b_512(<64 x i8> %data) {
 define <16 x i32> @test_int_x86_avx512_mask_vpshld_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x3, i16 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_d_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldd $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0x7d,0x48,0x71,0xd9,0x16]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpshldd $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x49,0x71,0xd1,0x16]
-; X86-NEXT:    vpaddd %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc3]
+; X86-NEXT:    vpshldd $23, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0x7d,0x48,0x71,0xc1,0x17]
+; X86-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_d_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldd $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0x7d,0x48,0x71,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshldd $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x49,0x71,0xd1,0x16]
-; X64-NEXT:    vpaddd %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc3]
+; X64-NEXT:    vpshldd $23, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0x7d,0x48,0x71,0xc1,0x17]
+; X64-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> <i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22>)
   %2 = bitcast i16 %x4 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x3
-  %4 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> <i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22>)
+  %4 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> <i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23>)
   %res2 = add <16 x i32> %3, %4
   ret <16 x i32> %res2
 }
@@ -394,24 +394,24 @@ define <16 x i32> @test_int_x86_avx512_mask_vpshld_d_512(<16 x i32> %x0, <16 x i
 define <8 x i64> @test_int_x86_avx512_mask_vpshld_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_q_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldq $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x71,0xd9,0x16]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshldq $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x71,0xd1,0x16]
-; X86-NEXT:    vpaddq %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc3]
+; X86-NEXT:    vpshldq $23, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0xfd,0x48,0x71,0xc1,0x17]
+; X86-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_q_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldq $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x71,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshldq $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x71,0xd1,0x16]
-; X64-NEXT:    vpaddq %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc3]
+; X64-NEXT:    vpshldq $23, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0xfd,0x48,0x71,0xc1,0x17]
+; X64-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> <i64 22, i64 22, i64 22, i64 22, i64 22, i64 22, i64 22, i64 22>)
   %2 = bitcast i8 %x4 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x3
-  %4 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> <i64 22, i64 22, i64 22, i64 22, i64 22, i64 22, i64 22, i64 22>)
+  %4 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> <i64 23, i64 23, i64 23, i64 23, i64 23, i64 23, i64 23, i64 23>)
   %res2 = add <8 x i64> %3, %4
   ret <8 x i64> %res2
 }
@@ -419,23 +419,23 @@ define <8 x i64> @test_int_x86_avx512_mask_vpshld_q_512(<8 x i64> %x0, <8 x i64>
 define <32 x i16> @test_int_x86_avx512_mask_vpshld_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x3, i32 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_w_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldw $6, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x70,0xd9,0x06]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpshldw $6, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x70,0xd1,0x06]
-; X86-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
+; X86-NEXT:    vpshldw $7, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0xfd,0x48,0x70,0xc1,0x07]
+; X86-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_w_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldw $6, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x70,0xd9,0x06]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshldw $6, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x70,0xd1,0x06]
-; X64-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
+; X64-NEXT:    vpshldw $7, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0xfd,0x48,0x70,0xc1,0x07]
+; X64-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> <i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6>)
   %2 = bitcast i32 %x4 to <32 x i1>
   %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x3
-  %4 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> <i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6>)
+  %4 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>)
   %res2 = add <32 x i16> %3, %4
   ret <32 x i16> %res2
 }
@@ -443,23 +443,23 @@ define <32 x i16> @test_int_x86_avx512_mask_vpshld_w_512(<32 x i16> %x0, <32 x i
 define <16 x i32> @test_int_x86_avx512_mask_vpshrd_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x3, i16 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_d_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdd $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0x7d,0x48,0x73,0xd9,0x16]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpshrdd $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x49,0x73,0xd1,0x16]
-; X86-NEXT:    vpaddd %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc3]
+; X86-NEXT:    vpshrdd $23, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0x7d,0x48,0x73,0xc1,0x17]
+; X86-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_d_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdd $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0x7d,0x48,0x73,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshrdd $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x49,0x73,0xd1,0x16]
-; X64-NEXT:    vpaddd %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc3]
+; X64-NEXT:    vpshrdd $23, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0x7d,0x48,0x73,0xc1,0x17]
+; X64-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %x1, <16 x i32> %x0, <16 x i32> <i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22>)
   %2 = bitcast i16 %x4 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x3
-  %4 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %x1, <16 x i32> %x0, <16 x i32> <i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22>)
+  %4 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %x1, <16 x i32> %x0, <16 x i32> <i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23>)
   %res2 = add <16 x i32> %3, %4
   ret <16 x i32> %res2
 }
@@ -467,24 +467,24 @@ define <16 x i32> @test_int_x86_avx512_mask_vpshrd_d_512(<16 x i32> %x0, <16 x i
 define <8 x i64> @test_int_x86_avx512_mask_vpshrd_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_q_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdq $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x73,0xd9,0x16]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshrdq $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x73,0xd1,0x16]
-; X86-NEXT:    vpaddq %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc3]
+; X86-NEXT:    vpshrdq $23, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0xfd,0x48,0x73,0xc1,0x17]
+; X86-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_q_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdq $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x73,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshrdq $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x73,0xd1,0x16]
-; X64-NEXT:    vpaddq %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc3]
+; X64-NEXT:    vpshrdq $23, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0xfd,0x48,0x73,0xc1,0x17]
+; X64-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %x1, <8 x i64> %x0, <8 x i64> <i64 22, i64 22, i64 22, i64 22, i64 22, i64 22, i64 22, i64 22>)
   %2 = bitcast i8 %x4 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x3
-  %4 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %x1, <8 x i64> %x0, <8 x i64> <i64 22, i64 22, i64 22, i64 22, i64 22, i64 22, i64 22, i64 22>)
+  %4 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %x1, <8 x i64> %x0, <8 x i64> <i64 23, i64 23, i64 23, i64 23, i64 23, i64 23, i64 23, i64 23>)
   %res2 = add <8 x i64> %3, %4
   ret <8 x i64> %res2
 }
@@ -492,23 +492,23 @@ define <8 x i64> @test_int_x86_avx512_mask_vpshrd_q_512(<8 x i64> %x0, <8 x i64>
 define <32 x i16> @test_int_x86_avx512_mask_vpshrd_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x3, i32 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_w_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdw $6, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x72,0xd9,0x06]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpshrdw $6, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x72,0xd1,0x06]
-; X86-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
+; X86-NEXT:    vpshrdw $7, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0xfd,0x48,0x72,0xc1,0x07]
+; X86-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_w_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdw $6, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x72,0xd9,0x06]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshrdw $6, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x72,0xd1,0x06]
-; X64-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
+; X64-NEXT:    vpshrdw $7, %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf3,0xfd,0x48,0x72,0xc1,0x07]
+; X64-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %x1, <32 x i16> %x0, <32 x i16> <i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6>)
   %2 = bitcast i32 %x4 to <32 x i1>
   %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x3
-  %4 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %x1, <32 x i16> %x0, <32 x i16> <i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6>)
+  %4 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %x1, <32 x i16> %x0, <32 x i16> <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>)
   %res2 = add <32 x i16> %3, %4
   ret <32 x i16> %res2
 }
@@ -520,10 +520,7 @@ define <16 x i32> @test_int_x86_avx512_mask_vpshrdv_d_512(<16 x i32> %x0, <16 x
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpshrdvd (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x73,0x18]
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpshrdvd %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x73,0xe2]
 ; X86-NEXT:    vpshrdvd %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x73,0xc2]
-; X86-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -532,10 +529,7 @@ define <16 x i32> @test_int_x86_avx512_mask_vpshrdv_d_512(<16 x i32> %x0, <16 x
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X64-NEXT:    vpshrdvd (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x73,0x1f]
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X64-NEXT:    vpshrdvd %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x73,0xe2]
 ; X64-NEXT:    vpshrdvd %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x73,0xc2]
-; X64-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <16 x i32>, <16 x i32>* %x2p
@@ -543,12 +537,10 @@ define <16 x i32> @test_int_x86_avx512_mask_vpshrdv_d_512(<16 x i32> %x0, <16 x
   %2 = bitcast i16 %x3 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x0
   %4 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %x1, <16 x i32> %x0, <16 x i32> %x4)
-  %5 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %x1, <16 x i32> %x0, <16 x i32> %x4)
-  %6 = bitcast i16 %x3 to <16 x i1>
-  %7 = select <16 x i1> %6, <16 x i32> %5, <16 x i32> zeroinitializer
-  %res3 = add <16 x i32> %3, %4
-  %res4 = add <16 x i32> %7, %res3
-  ret <16 x i32> %res4
+  %5 = bitcast i16 %x3 to <16 x i1>
+  %6 = select <16 x i1> %5, <16 x i32> %4, <16 x i32> zeroinitializer
+  %res3 = add <16 x i32> %3, %6
+  ret <16 x i32> %res3
 }
 
 define <8 x i64> @test_int_x86_avx512_mask_vpshrdv_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64>* %x2p, <8 x i64> %x4, i8 %x3) {
@@ -559,10 +551,7 @@ define <8 x i64> @test_int_x86_avx512_mask_vpshrdv_q_512(<8 x i64> %x0, <8 x i64
 ; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpshrdvq (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0x73,0x18]
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpshrdvq %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0xf5,0x48,0x73,0xe2]
 ; X86-NEXT:    vpshrdvq %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x73,0xc2]
-; X86-NEXT:    vpaddq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -571,10 +560,7 @@ define <8 x i64> @test_int_x86_avx512_mask_vpshrdv_q_512(<8 x i64> %x0, <8 x i64
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X64-NEXT:    vpshrdvq (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0x73,0x1f]
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X64-NEXT:    vpshrdvq %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0xf5,0x48,0x73,0xe2]
 ; X64-NEXT:    vpshrdvq %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x73,0xc2]
-; X64-NEXT:    vpaddq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <8 x i64>, <8 x i64>* %x2p
@@ -582,12 +568,10 @@ define <8 x i64> @test_int_x86_avx512_mask_vpshrdv_q_512(<8 x i64> %x0, <8 x i64
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x0
   %4 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %x1, <8 x i64> %x0, <8 x i64> %x4)
-  %5 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %x1, <8 x i64> %x0, <8 x i64> %x4)
-  %6 = bitcast i8 %x3 to <8 x i1>
-  %7 = select <8 x i1> %6, <8 x i64> %5, <8 x i64> zeroinitializer
-  %res3 = add <8 x i64> %3, %4
-  %res4 = add <8 x i64> %7, %res3
-  ret <8 x i64> %res4
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = select <8 x i1> %5, <8 x i64> %4, <8 x i64> zeroinitializer
+  %res3 = add <8 x i64> %3, %6
+  ret <8 x i64> %res3
 }
 
 define <32 x i16> @test_int_x86_avx512_mask_vpshrdv_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16>* %x2p, <32 x i16> %x4, i32 %x3) {
@@ -597,10 +581,7 @@ define <32 x i16> @test_int_x86_avx512_mask_vpshrdv_w_512(<32 x i16> %x0, <32 x
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpshrdvw (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0x72,0x18]
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpshrdvw %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0xf5,0x48,0x72,0xe2]
 ; X86-NEXT:    vpshrdvw %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x72,0xc2]
-; X86-NEXT:    vpaddw %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfd,0xc0]
 ; X86-NEXT:    vpaddw %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -609,10 +590,7 @@ define <32 x i16> @test_int_x86_avx512_mask_vpshrdv_w_512(<32 x i16> %x0, <32 x
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X64-NEXT:    vpshrdvw (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0x72,0x1f]
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X64-NEXT:    vpshrdvw %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0xf5,0x48,0x72,0xe2]
 ; X64-NEXT:    vpshrdvw %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x72,0xc2]
-; X64-NEXT:    vpaddw %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfd,0xc0]
 ; X64-NEXT:    vpaddw %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <32 x i16>, <32 x i16>* %x2p
@@ -620,12 +598,10 @@ define <32 x i16> @test_int_x86_avx512_mask_vpshrdv_w_512(<32 x i16> %x0, <32 x
   %2 = bitcast i32 %x3 to <32 x i1>
   %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x0
   %4 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %x1, <32 x i16> %x0, <32 x i16> %x4)
-  %5 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %x1, <32 x i16> %x0, <32 x i16> %x4)
-  %6 = bitcast i32 %x3 to <32 x i1>
-  %7 = select <32 x i1> %6, <32 x i16> %5, <32 x i16> zeroinitializer
-  %res3 = add <32 x i16> %3, %4
-  %res4 = add <32 x i16> %7, %res3
-  ret <32 x i16> %res4
+  %5 = bitcast i32 %x3 to <32 x i1>
+  %6 = select <32 x i1> %5, <32 x i16> %4, <32 x i16> zeroinitializer
+  %res3 = add <32 x i16> %3, %6
+  ret <32 x i16> %res3
 }
 
 define <16 x i32> @test_int_x86_avx512_mask_vpshldv_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32>* %x2p, <16 x i32> %x4, i16 %x3) {
@@ -635,10 +611,7 @@ define <16 x i32> @test_int_x86_avx512_mask_vpshldv_d_512(<16 x i32> %x0, <16 x
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpshldvd (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x71,0x18]
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpshldvd %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x71,0xe2]
 ; X86-NEXT:    vpshldvd %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x71,0xc2]
-; X86-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -647,10 +620,7 @@ define <16 x i32> @test_int_x86_avx512_mask_vpshldv_d_512(<16 x i32> %x0, <16 x
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X64-NEXT:    vpshldvd (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x71,0x1f]
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X64-NEXT:    vpshldvd %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x71,0xe2]
 ; X64-NEXT:    vpshldvd %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x71,0xc2]
-; X64-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <16 x i32>, <16 x i32>* %x2p
@@ -658,12 +628,10 @@ define <16 x i32> @test_int_x86_avx512_mask_vpshldv_d_512(<16 x i32> %x0, <16 x
   %2 = bitcast i16 %x3 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x0
   %4 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4)
-  %5 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4)
-  %6 = bitcast i16 %x3 to <16 x i1>
-  %7 = select <16 x i1> %6, <16 x i32> %5, <16 x i32> zeroinitializer
-  %res3 = add <16 x i32> %3, %4
-  %res4 = add <16 x i32> %7, %res3
-  ret <16 x i32> %res4
+  %5 = bitcast i16 %x3 to <16 x i1>
+  %6 = select <16 x i1> %5, <16 x i32> %4, <16 x i32> zeroinitializer
+  %res3 = add <16 x i32> %3, %6
+  ret <16 x i32> %res3
 }
 
 define <8 x i64> @test_int_x86_avx512_mask_vpshldv_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64>* %x2p, <8 x i64> %x4, i8 %x3) {
@@ -674,10 +642,7 @@ define <8 x i64> @test_int_x86_avx512_mask_vpshldv_q_512(<8 x i64> %x0, <8 x i64
 ; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpshldvq (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0x71,0x18]
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpshldvq %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0xf5,0x48,0x71,0xe2]
 ; X86-NEXT:    vpshldvq %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x71,0xc2]
-; X86-NEXT:    vpaddq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -686,10 +651,7 @@ define <8 x i64> @test_int_x86_avx512_mask_vpshldv_q_512(<8 x i64> %x0, <8 x i64
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X64-NEXT:    vpshldvq (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0x71,0x1f]
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X64-NEXT:    vpshldvq %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0xf5,0x48,0x71,0xe2]
 ; X64-NEXT:    vpshldvq %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x71,0xc2]
-; X64-NEXT:    vpaddq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <8 x i64>, <8 x i64>* %x2p
@@ -697,12 +659,10 @@ define <8 x i64> @test_int_x86_avx512_mask_vpshldv_q_512(<8 x i64> %x0, <8 x i64
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x0
   %4 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x4)
-  %5 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x4)
-  %6 = bitcast i8 %x3 to <8 x i1>
-  %7 = select <8 x i1> %6, <8 x i64> %5, <8 x i64> zeroinitializer
-  %res3 = add <8 x i64> %3, %4
-  %res4 = add <8 x i64> %7, %res3
-  ret <8 x i64> %res4
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = select <8 x i1> %5, <8 x i64> %4, <8 x i64> zeroinitializer
+  %res3 = add <8 x i64> %3, %6
+  ret <8 x i64> %res3
 }
 
 define <32 x i16> @test_int_x86_avx512_mask_vpshldv_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16>* %x2p, <32 x i16> %x4, i32 %x3) {
@@ -712,10 +672,7 @@ define <32 x i16> @test_int_x86_avx512_mask_vpshldv_w_512(<32 x i16> %x0, <32 x
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpshldvw (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0x70,0x18]
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpshldvw %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0xf5,0x48,0x70,0xe2]
 ; X86-NEXT:    vpshldvw %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x70,0xc2]
-; X86-NEXT:    vpaddw %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfd,0xc0]
 ; X86-NEXT:    vpaddw %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -724,10 +681,7 @@ define <32 x i16> @test_int_x86_avx512_mask_vpshldv_w_512(<32 x i16> %x0, <32 x
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X64-NEXT:    vpshldvw (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0x70,0x1f]
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X64-NEXT:    vpshldvw %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0xf5,0x48,0x70,0xe2]
 ; X64-NEXT:    vpshldvw %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x70,0xc2]
-; X64-NEXT:    vpaddw %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfd,0xc0]
 ; X64-NEXT:    vpaddw %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <32 x i16>, <32 x i16>* %x2p
@@ -735,12 +689,10 @@ define <32 x i16> @test_int_x86_avx512_mask_vpshldv_w_512(<32 x i16> %x0, <32 x
   %2 = bitcast i32 %x3 to <32 x i1>
   %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x0
   %4 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x4)
-  %5 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x4)
-  %6 = bitcast i32 %x3 to <32 x i1>
-  %7 = select <32 x i1> %6, <32 x i16> %5, <32 x i16> zeroinitializer
-  %res3 = add <32 x i16> %3, %4
-  %res4 = add <32 x i16> %7, %res3
-  ret <32 x i16> %res4
+  %5 = bitcast i32 %x3 to <32 x i1>
+  %6 = select <32 x i1> %5, <32 x i16> %4, <32 x i16> zeroinitializer
+  %res3 = add <32 x i16> %3, %6
+  ret <32 x i16> %res3
 }
 
 declare <16 x i32> @llvm.fshl.v16i32(<16 x i32>, <16 x i32>, <16 x i32>)
diff --git a/llvm/test/CodeGen/X86/avx512vbmi2vl-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512vbmi2vl-intrinsics-upgrade.ll
index 9c4ff05031618..8ac14762cab79 100644
--- a/llvm/test/CodeGen/X86/avx512vbmi2vl-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512vbmi2vl-intrinsics-upgrade.ll
@@ -728,27 +728,27 @@ declare <32 x i8> @llvm.x86.avx512.mask.compress.b.256(<32 x i8> %data, <32 x i8
 define <4 x i32>@test_int_x86_avx512_mask_vpshld_d_128(<4 x i32> %x0, <4 x i32> %x1,<4 x i32> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_d_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldd $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x71,0xd9,0x16]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshldd $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x71,0xd1,0x16]
-; X86-NEXT:    vpshldd $22, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x71,0xc1,0x16]
+; X86-NEXT:    vpshldd $23, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x71,0xd9,0x17]
+; X86-NEXT:    vpshldd $24, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x71,0xc1,0x18]
 ; X86-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_d_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldd $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x71,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshldd $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x71,0xd1,0x16]
-; X64-NEXT:    vpshldd $22, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x71,0xc1,0x16]
+; X64-NEXT:    vpshldd $23, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x71,0xd9,0x17]
+; X64-NEXT:    vpshldd $24, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x71,0xc1,0x18]
 ; X64-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx512.mask.vpshld.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 22, <4 x i32> %x3, i8 %x4)
-  %res1 = call <4 x i32> @llvm.x86.avx512.mask.vpshld.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 22, <4 x i32> %x3, i8 -1)
-  %res2 = call <4 x i32> @llvm.x86.avx512.mask.vpshld.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 22, <4 x i32> zeroinitializer,i8 %x4)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.vpshld.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 23, <4 x i32> %x3, i8 -1)
+  %res2 = call <4 x i32> @llvm.x86.avx512.mask.vpshld.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 24, <4 x i32> zeroinitializer,i8 %x4)
   %res3 = add <4 x i32> %res, %res1
   %res4 = add <4 x i32> %res3, %res2
   ret <4 x i32> %res4
@@ -758,22 +758,22 @@ declare <4 x i32> @llvm.x86.avx512.mask.vpshld.d.128(<4 x i32>, <4 x i32>, i32,
 define <8 x i32>@test_int_x86_avx512_mask_vpshld_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldd $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0x7d,0x28,0x71,0xd9,0x16]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshldd $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x71,0xd1,0x16]
-; X86-NEXT:    vpaddd %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc3]
+; X86-NEXT:    vpshldd $23, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0x7d,0x28,0x71,0xc1,0x17]
+; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_d_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldd $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0x7d,0x28,0x71,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshldd $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x71,0xd1,0x16]
-; X64-NEXT:    vpaddd %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc3]
+; X64-NEXT:    vpshldd $23, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0x7d,0x28,0x71,0xc1,0x17]
+; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx512.mask.vpshld.d.256(<8 x i32> %x0, <8 x i32> %x1, i32 22, <8 x i32> %x3, i8 %x4)
-  %res1 = call <8 x i32> @llvm.x86.avx512.mask.vpshld.d.256(<8 x i32> %x0, <8 x i32> %x1, i32 22, <8 x i32> %x3, i8 -1)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.vpshld.d.256(<8 x i32> %x0, <8 x i32> %x1, i32 23, <8 x i32> %x3, i8 -1)
   %res2 = add <8 x i32> %res, %res1
   ret <8 x i32> %res2
 }
@@ -782,22 +782,22 @@ declare <8 x i32> @llvm.x86.avx512.mask.vpshld.d.256(<8 x i32>, <8 x i32>, i32,
 define <2 x i64>@test_int_x86_avx512_mask_vpshld_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_q_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldq $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x71,0xd9,0x16]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshldq $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x71,0xd1,0x16]
-; X86-NEXT:    vpaddq %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc3]
+; X86-NEXT:    vpshldq $23, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x71,0xc1,0x17]
+; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_q_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldq $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x71,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshldq $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x71,0xd1,0x16]
-; X64-NEXT:    vpaddq %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc3]
+; X64-NEXT:    vpshldq $23, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x71,0xc1,0x17]
+; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx512.mask.vpshld.q.128(<2 x i64> %x0, <2 x i64> %x1, i32 22, <2 x i64> %x3, i8 %x4)
-  %res1 = call <2 x i64> @llvm.x86.avx512.mask.vpshld.q.128(<2 x i64> %x0, <2 x i64> %x1, i32 22, <2 x i64> %x3, i8 -1)
+  %res1 = call <2 x i64> @llvm.x86.avx512.mask.vpshld.q.128(<2 x i64> %x0, <2 x i64> %x1, i32 23, <2 x i64> %x3, i8 -1)
   %res2 = add <2 x i64> %res, %res1
   ret <2 x i64> %res2
 }
@@ -806,22 +806,22 @@ declare <2 x i64> @llvm.x86.avx512.mask.vpshld.q.128(<2 x i64>, <2 x i64>, i32,
 define <4 x i64>@test_int_x86_avx512_mask_vpshld_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_q_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldq $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x71,0xd9,0x16]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshldq $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x71,0xd1,0x16]
-; X86-NEXT:    vpaddq %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc3]
+; X86-NEXT:    vpshldq $23, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0xfd,0x28,0x71,0xc1,0x17]
+; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_q_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldq $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x71,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshldq $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x71,0xd1,0x16]
-; X64-NEXT:    vpaddq %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc3]
+; X64-NEXT:    vpshldq $23, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0xfd,0x28,0x71,0xc1,0x17]
+; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx512.mask.vpshld.q.256(<4 x i64> %x0, <4 x i64> %x1, i32 22, <4 x i64> %x3, i8 %x4)
-  %res1 = call <4 x i64> @llvm.x86.avx512.mask.vpshld.q.256(<4 x i64> %x0, <4 x i64> %x1, i32 22, <4 x i64> %x3, i8 -1)
+  %res1 = call <4 x i64> @llvm.x86.avx512.mask.vpshld.q.256(<4 x i64> %x0, <4 x i64> %x1, i32 23, <4 x i64> %x3, i8 -1)
   %res2 = add <4 x i64> %res, %res1
   ret <4 x i64> %res2
 }
@@ -830,22 +830,22 @@ declare <4 x i64> @llvm.x86.avx512.mask.vpshld.q.256(<4 x i64>, <4 x i64>, i32,
 define <8 x i16>@test_int_x86_avx512_mask_vpshld_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_w_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldw $6, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x70,0xd9,0x06]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshldw $6, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x70,0xd1,0x06]
-; X86-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
+; X86-NEXT:    vpshldw $7, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x70,0xc1,0x07]
+; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_w_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldw $6, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x70,0xd9,0x06]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshldw $6, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x70,0xd1,0x06]
-; X64-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
+; X64-NEXT:    vpshldw $7, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x70,0xc1,0x07]
+; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i16> @llvm.x86.avx512.mask.vpshld.w.128(<8 x i16> %x0, <8 x i16> %x1, i32 6, <8 x i16> %x3, i8 %x4)
-  %res1 = call <8 x i16> @llvm.x86.avx512.mask.vpshld.w.128(<8 x i16> %x0, <8 x i16> %x1, i32 6, <8 x i16> %x3, i8 -1)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.vpshld.w.128(<8 x i16> %x0, <8 x i16> %x1, i32 7, <8 x i16> %x3, i8 -1)
   %res2 = add <8 x i16> %res, %res1
   ret <8 x i16> %res2
 }
@@ -854,21 +854,21 @@ declare <8 x i16> @llvm.x86.avx512.mask.vpshld.w.128(<8 x i16>, <8 x i16>, i32,
 define <16 x i16>@test_int_x86_avx512_mask_vpshld_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x3, i16 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_w_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldw $6, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x70,0xd9,0x06]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpshldw $6, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x70,0xd1,0x06]
-; X86-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
+; X86-NEXT:    vpshldw $7, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0xfd,0x28,0x70,0xc1,0x07]
+; X86-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_w_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldw $6, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x70,0xd9,0x06]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshldw $6, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x70,0xd1,0x06]
-; X64-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
+; X64-NEXT:    vpshldw $7, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0xfd,0x28,0x70,0xc1,0x07]
+; X64-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx512.mask.vpshld.w.256(<16 x i16> %x0, <16 x i16> %x1, i32 6, <16 x i16> %x3, i16 %x4)
-  %res1 = call <16 x i16> @llvm.x86.avx512.mask.vpshld.w.256(<16 x i16> %x0, <16 x i16> %x1, i32 6, <16 x i16> %x3, i16 -1)
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.vpshld.w.256(<16 x i16> %x0, <16 x i16> %x1, i32 7, <16 x i16> %x3, i16 -1)
   %res2 = add <16 x i16> %res, %res1
   ret <16 x i16> %res2
 }
@@ -877,27 +877,27 @@ declare <16 x i16> @llvm.x86.avx512.mask.vpshld.w.256(<16 x i16>, <16 x i16>, i3
 define <4 x i32>@test_int_x86_avx512_mask_vpshrd_d_128(<4 x i32> %x0, <4 x i32> %x1,<4 x i32> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_d_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdd $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x73,0xd9,0x16]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshrdd $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x73,0xd1,0x16]
-; X86-NEXT:    vpshrdd $22, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x73,0xc1,0x16]
+; X86-NEXT:    vpshrdd $23, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x73,0xd9,0x17]
+; X86-NEXT:    vpshrdd $24, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x73,0xc1,0x18]
 ; X86-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_d_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdd $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x73,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshrdd $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x73,0xd1,0x16]
-; X64-NEXT:    vpshrdd $22, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x73,0xc1,0x16]
+; X64-NEXT:    vpshrdd $23, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x73,0xd9,0x17]
+; X64-NEXT:    vpshrdd $24, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x73,0xc1,0x18]
 ; X64-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx512.mask.vpshrd.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 22, <4 x i32> %x3, i8 %x4)
-  %res1 = call <4 x i32> @llvm.x86.avx512.mask.vpshrd.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 22, <4 x i32> %x3, i8 -1)
-  %res2 = call <4 x i32> @llvm.x86.avx512.mask.vpshrd.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 22, <4 x i32> zeroinitializer,i8 %x4)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.vpshrd.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 23, <4 x i32> %x3, i8 -1)
+  %res2 = call <4 x i32> @llvm.x86.avx512.mask.vpshrd.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 24, <4 x i32> zeroinitializer,i8 %x4)
   %res3 = add <4 x i32> %res, %res1
   %res4 = add <4 x i32> %res3, %res2
   ret <4 x i32> %res4
@@ -907,22 +907,22 @@ declare <4 x i32> @llvm.x86.avx512.mask.vpshrd.d.128(<4 x i32>, <4 x i32>, i32,
 define <8 x i32>@test_int_x86_avx512_mask_vpshrd_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdd $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0x7d,0x28,0x73,0xd9,0x16]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshrdd $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x73,0xd1,0x16]
-; X86-NEXT:    vpaddd %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc3]
+; X86-NEXT:    vpshrdd $23, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0x7d,0x28,0x73,0xc1,0x17]
+; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_d_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdd $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0x7d,0x28,0x73,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshrdd $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x73,0xd1,0x16]
-; X64-NEXT:    vpaddd %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc3]
+; X64-NEXT:    vpshrdd $23, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0x7d,0x28,0x73,0xc1,0x17]
+; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx512.mask.vpshrd.d.256(<8 x i32> %x0, <8 x i32> %x1, i32 22, <8 x i32> %x3, i8 %x4)
-  %res1 = call <8 x i32> @llvm.x86.avx512.mask.vpshrd.d.256(<8 x i32> %x0, <8 x i32> %x1, i32 22, <8 x i32> %x3, i8 -1)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.vpshrd.d.256(<8 x i32> %x0, <8 x i32> %x1, i32 23, <8 x i32> %x3, i8 -1)
   %res2 = add <8 x i32> %res, %res1
   ret <8 x i32> %res2
 }
@@ -931,22 +931,22 @@ declare <8 x i32> @llvm.x86.avx512.mask.vpshrd.d.256(<8 x i32>, <8 x i32>, i32,
 define <2 x i64>@test_int_x86_avx512_mask_vpshrd_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_q_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdq $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x73,0xd9,0x16]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshrdq $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x73,0xd1,0x16]
-; X86-NEXT:    vpaddq %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc3]
+; X86-NEXT:    vpshrdq $23, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x73,0xc1,0x17]
+; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_q_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdq $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x73,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshrdq $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x73,0xd1,0x16]
-; X64-NEXT:    vpaddq %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc3]
+; X64-NEXT:    vpshrdq $23, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x73,0xc1,0x17]
+; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx512.mask.vpshrd.q.128(<2 x i64> %x0, <2 x i64> %x1, i32 22, <2 x i64> %x3, i8 %x4)
-  %res1 = call <2 x i64> @llvm.x86.avx512.mask.vpshrd.q.128(<2 x i64> %x0, <2 x i64> %x1, i32 22, <2 x i64> %x3, i8 -1)
+  %res1 = call <2 x i64> @llvm.x86.avx512.mask.vpshrd.q.128(<2 x i64> %x0, <2 x i64> %x1, i32 23, <2 x i64> %x3, i8 -1)
   %res2 = add <2 x i64> %res, %res1
   ret <2 x i64> %res2
 }
@@ -955,22 +955,22 @@ declare <2 x i64> @llvm.x86.avx512.mask.vpshrd.q.128(<2 x i64>, <2 x i64>, i32,
 define <4 x i64>@test_int_x86_avx512_mask_vpshrd_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_q_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdq $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x73,0xd9,0x16]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshrdq $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x73,0xd1,0x16]
-; X86-NEXT:    vpaddq %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc3]
+; X86-NEXT:    vpshrdq $23, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0xfd,0x28,0x73,0xc1,0x17]
+; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_q_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdq $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x73,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshrdq $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x73,0xd1,0x16]
-; X64-NEXT:    vpaddq %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc3]
+; X64-NEXT:    vpshrdq $23, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0xfd,0x28,0x73,0xc1,0x17]
+; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx512.mask.vpshrd.q.256(<4 x i64> %x0, <4 x i64> %x1, i32 22, <4 x i64> %x3, i8 %x4)
-  %res1 = call <4 x i64> @llvm.x86.avx512.mask.vpshrd.q.256(<4 x i64> %x0, <4 x i64> %x1, i32 22, <4 x i64> %x3, i8 -1)
+  %res1 = call <4 x i64> @llvm.x86.avx512.mask.vpshrd.q.256(<4 x i64> %x0, <4 x i64> %x1, i32 23, <4 x i64> %x3, i8 -1)
   %res2 = add <4 x i64> %res, %res1
   ret <4 x i64> %res2
 }
@@ -979,22 +979,22 @@ declare <4 x i64> @llvm.x86.avx512.mask.vpshrd.q.256(<4 x i64>, <4 x i64>, i32,
 define <8 x i16>@test_int_x86_avx512_mask_vpshrd_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_w_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdw $6, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x72,0xd9,0x06]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshrdw $6, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x72,0xd1,0x06]
-; X86-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
+; X86-NEXT:    vpshrdw $7, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x72,0xc1,0x07]
+; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_w_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdw $6, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x72,0xd9,0x06]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshrdw $6, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x72,0xd1,0x06]
-; X64-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
+; X64-NEXT:    vpshrdw $7, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x72,0xc1,0x07]
+; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i16> @llvm.x86.avx512.mask.vpshrd.w.128(<8 x i16> %x0, <8 x i16> %x1, i32 6, <8 x i16> %x3, i8 %x4)
-  %res1 = call <8 x i16> @llvm.x86.avx512.mask.vpshrd.w.128(<8 x i16> %x0, <8 x i16> %x1, i32 6, <8 x i16> %x3, i8 -1)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.vpshrd.w.128(<8 x i16> %x0, <8 x i16> %x1, i32 7, <8 x i16> %x3, i8 -1)
   %res2 = add <8 x i16> %res, %res1
   ret <8 x i16> %res2
 }
@@ -1003,21 +1003,21 @@ declare <8 x i16> @llvm.x86.avx512.mask.vpshrd.w.128(<8 x i16>, <8 x i16>, i32,
 define <16 x i16>@test_int_x86_avx512_mask_vpshrd_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x3, i16 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_w_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdw $6, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x72,0xd9,0x06]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpshrdw $6, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x72,0xd1,0x06]
-; X86-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
+; X86-NEXT:    vpshrdw $7, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0xfd,0x28,0x72,0xc1,0x07]
+; X86-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_w_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdw $6, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x72,0xd9,0x06]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshrdw $6, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x72,0xd1,0x06]
-; X64-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
+; X64-NEXT:    vpshrdw $7, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0xfd,0x28,0x72,0xc1,0x07]
+; X64-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx512.mask.vpshrd.w.256(<16 x i16> %x0, <16 x i16> %x1, i32 6, <16 x i16> %x3, i16 %x4)
-  %res1 = call <16 x i16> @llvm.x86.avx512.mask.vpshrd.w.256(<16 x i16> %x0, <16 x i16> %x1, i32 6, <16 x i16> %x3, i16 -1)
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.vpshrd.w.256(<16 x i16> %x0, <16 x i16> %x1, i32 7, <16 x i16> %x3, i16 -1)
   %res2 = add <16 x i16> %res, %res1
   ret <16 x i16> %res2
 }
@@ -1026,21 +1026,21 @@ declare <16 x i16> @llvm.x86.avx512.mask.vpshrd.w.256(<16 x i16>, <16 x i16>, i3
 define <4 x i32>@test_int_x86_avx512_mask_vpshld_d_128_2(<4 x i32> %x0, <4 x i32> %x1,<4 x i32> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_d_128_2:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldd $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x71,0xd9,0x16]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshldd $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x71,0xd1,0x16]
-; X86-NEXT:    vpshldd $22, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x71,0xc1,0x16]
+; X86-NEXT:    vpshldd $23, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x71,0xd9,0x17]
+; X86-NEXT:    vpshldd $24, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x71,0xc1,0x18]
 ; X86-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_d_128_2:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldd $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x71,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshldd $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x71,0xd1,0x16]
-; X64-NEXT:    vpshldd $22, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x71,0xc1,0x16]
+; X64-NEXT:    vpshldd $23, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x71,0xd9,0x17]
+; X64-NEXT:    vpshldd $24, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x71,0xc1,0x18]
 ; X64-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
@@ -1048,8 +1048,8 @@ define <4 x i32>@test_int_x86_avx512_mask_vpshld_d_128_2(<4 x i32> %x0, <4 x i32
   %2 = bitcast i8 %x4 to <8 x i1>
   %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract1, <4 x i32> %1, <4 x i32> %x3
-  %4 = call <4 x i32> @llvm.x86.avx512.vpshld.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 22)
-  %5 = call <4 x i32> @llvm.x86.avx512.vpshld.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 22)
+  %4 = call <4 x i32> @llvm.x86.avx512.vpshld.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 23)
+  %5 = call <4 x i32> @llvm.x86.avx512.vpshld.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 24)
   %6 = bitcast i8 %x4 to <8 x i1>
   %extract = shufflevector <8 x i1> %6, <8 x i1> %6, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %7 = select <4 x i1> %extract, <4 x i32> %5, <4 x i32> zeroinitializer
@@ -1062,24 +1062,24 @@ declare <4 x i32> @llvm.x86.avx512.vpshld.d.128(<4 x i32>, <4 x i32>, i32)
 define <8 x i32>@test_int_x86_avx512_mask_vpshld_d_256_2(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_d_256_2:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldd $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0x7d,0x28,0x71,0xd9,0x16]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshldd $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x71,0xd1,0x16]
-; X86-NEXT:    vpaddd %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc3]
+; X86-NEXT:    vpshldd $23, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0x7d,0x28,0x71,0xc1,0x17]
+; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_d_256_2:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldd $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0x7d,0x28,0x71,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshldd $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x71,0xd1,0x16]
-; X64-NEXT:    vpaddd %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc3]
+; X64-NEXT:    vpshldd $23, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0x7d,0x28,0x71,0xc1,0x17]
+; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <8 x i32> @llvm.x86.avx512.vpshld.d.256(<8 x i32> %x0, <8 x i32> %x1, i32 22)
   %2 = bitcast i8 %x4 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x3
-  %4 = call <8 x i32> @llvm.x86.avx512.vpshld.d.256(<8 x i32> %x0, <8 x i32> %x1, i32 22)
+  %4 = call <8 x i32> @llvm.x86.avx512.vpshld.d.256(<8 x i32> %x0, <8 x i32> %x1, i32 23)
   %res2 = add <8 x i32> %3, %4
   ret <8 x i32> %res2
 }
@@ -1088,25 +1088,25 @@ declare <8 x i32> @llvm.x86.avx512.vpshld.d.256(<8 x i32>, <8 x i32>, i32)
 define <2 x i64>@test_int_x86_avx512_mask_vpshld_q_128_2(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_q_128_2:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldq $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x71,0xd9,0x16]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshldq $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x71,0xd1,0x16]
-; X86-NEXT:    vpaddq %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc3]
+; X86-NEXT:    vpshldq $23, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x71,0xc1,0x17]
+; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_q_128_2:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldq $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x71,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshldq $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x71,0xd1,0x16]
-; X64-NEXT:    vpaddq %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc3]
+; X64-NEXT:    vpshldq $23, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x71,0xc1,0x17]
+; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <2 x i64> @llvm.x86.avx512.vpshld.q.128(<2 x i64> %x0, <2 x i64> %x1, i32 22)
   %2 = bitcast i8 %x4 to <8 x i1>
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
   %3 = select <2 x i1> %extract, <2 x i64> %1, <2 x i64> %x3
-  %4 = call <2 x i64> @llvm.x86.avx512.vpshld.q.128(<2 x i64> %x0, <2 x i64> %x1, i32 22)
+  %4 = call <2 x i64> @llvm.x86.avx512.vpshld.q.128(<2 x i64> %x0, <2 x i64> %x1, i32 23)
   %res2 = add <2 x i64> %3, %4
   ret <2 x i64> %res2
 }
@@ -1115,25 +1115,25 @@ declare <2 x i64> @llvm.x86.avx512.vpshld.q.128(<2 x i64>, <2 x i64>, i32)
 define <4 x i64>@test_int_x86_avx512_mask_vpshld_q_256_2(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_q_256_2:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldq $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x71,0xd9,0x16]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshldq $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x71,0xd1,0x16]
-; X86-NEXT:    vpaddq %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc3]
+; X86-NEXT:    vpshldq $23, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0xfd,0x28,0x71,0xc1,0x17]
+; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_q_256_2:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldq $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x71,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshldq $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x71,0xd1,0x16]
-; X64-NEXT:    vpaddq %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc3]
+; X64-NEXT:    vpshldq $23, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0xfd,0x28,0x71,0xc1,0x17]
+; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <4 x i64> @llvm.x86.avx512.vpshld.q.256(<4 x i64> %x0, <4 x i64> %x1, i32 22)
   %2 = bitcast i8 %x4 to <8 x i1>
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract, <4 x i64> %1, <4 x i64> %x3
-  %4 = call <4 x i64> @llvm.x86.avx512.vpshld.q.256(<4 x i64> %x0, <4 x i64> %x1, i32 22)
+  %4 = call <4 x i64> @llvm.x86.avx512.vpshld.q.256(<4 x i64> %x0, <4 x i64> %x1, i32 23)
   %res2 = add <4 x i64> %3, %4
   ret <4 x i64> %res2
 }
@@ -1142,24 +1142,24 @@ declare <4 x i64> @llvm.x86.avx512.vpshld.q.256(<4 x i64>, <4 x i64>, i32)
 define <8 x i16>@test_int_x86_avx512_mask_vpshld_w_128_2(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_w_128_2:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldw $6, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x70,0xd9,0x06]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshldw $6, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x70,0xd1,0x06]
-; X86-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
+; X86-NEXT:    vpshldw $7, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x70,0xc1,0x07]
+; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_w_128_2:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldw $6, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x70,0xd9,0x06]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshldw $6, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x70,0xd1,0x06]
-; X64-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
+; X64-NEXT:    vpshldw $7, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x70,0xc1,0x07]
+; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <8 x i16> @llvm.x86.avx512.vpshld.w.128(<8 x i16> %x0, <8 x i16> %x1, i32 6)
   %2 = bitcast i8 %x4 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %x3
-  %4 = call <8 x i16> @llvm.x86.avx512.vpshld.w.128(<8 x i16> %x0, <8 x i16> %x1, i32 6)
+  %4 = call <8 x i16> @llvm.x86.avx512.vpshld.w.128(<8 x i16> %x0, <8 x i16> %x1, i32 7)
   %res2 = add <8 x i16> %3, %4
   ret <8 x i16> %res2
 }
@@ -1168,23 +1168,23 @@ declare <8 x i16> @llvm.x86.avx512.vpshld.w.128(<8 x i16>, <8 x i16>, i32)
 define <16 x i16>@test_int_x86_avx512_mask_vpshld_w_256_2(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x3, i16 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_w_256_2:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldw $6, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x70,0xd9,0x06]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpshldw $6, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x70,0xd1,0x06]
-; X86-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
+; X86-NEXT:    vpshldw $7, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0xfd,0x28,0x70,0xc1,0x07]
+; X86-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_w_256_2:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldw $6, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x70,0xd9,0x06]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshldw $6, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x70,0xd1,0x06]
-; X64-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
+; X64-NEXT:    vpshldw $7, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0xfd,0x28,0x70,0xc1,0x07]
+; X64-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <16 x i16> @llvm.x86.avx512.vpshld.w.256(<16 x i16> %x0, <16 x i16> %x1, i32 6)
   %2 = bitcast i16 %x4 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %x3
-  %4 = call <16 x i16> @llvm.x86.avx512.vpshld.w.256(<16 x i16> %x0, <16 x i16> %x1, i32 6)
+  %4 = call <16 x i16> @llvm.x86.avx512.vpshld.w.256(<16 x i16> %x0, <16 x i16> %x1, i32 7)
   %res2 = add <16 x i16> %3, %4
   ret <16 x i16> %res2
 }
@@ -1193,21 +1193,21 @@ declare <16 x i16> @llvm.x86.avx512.vpshld.w.256(<16 x i16>, <16 x i16>, i32)
 define <4 x i32>@test_int_x86_avx512_mask_vpshrd_d_128_2(<4 x i32> %x0, <4 x i32> %x1,<4 x i32> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_d_128_2:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdd $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x73,0xd9,0x16]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshrdd $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x73,0xd1,0x16]
-; X86-NEXT:    vpshrdd $22, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x73,0xc1,0x16]
+; X86-NEXT:    vpshrdd $23, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x73,0xd9,0x17]
+; X86-NEXT:    vpshrdd $24, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x73,0xc1,0x18]
 ; X86-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_d_128_2:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdd $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x73,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshrdd $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x73,0xd1,0x16]
-; X64-NEXT:    vpshrdd $22, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x73,0xc1,0x16]
+; X64-NEXT:    vpshrdd $23, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x73,0xd9,0x17]
+; X64-NEXT:    vpshrdd $24, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x73,0xc1,0x18]
 ; X64-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
@@ -1215,8 +1215,8 @@ define <4 x i32>@test_int_x86_avx512_mask_vpshrd_d_128_2(<4 x i32> %x0, <4 x i32
   %2 = bitcast i8 %x4 to <8 x i1>
   %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract1, <4 x i32> %1, <4 x i32> %x3
-  %4 = call <4 x i32> @llvm.x86.avx512.vpshrd.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 22)
-  %5 = call <4 x i32> @llvm.x86.avx512.vpshrd.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 22)
+  %4 = call <4 x i32> @llvm.x86.avx512.vpshrd.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 23)
+  %5 = call <4 x i32> @llvm.x86.avx512.vpshrd.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 24)
   %6 = bitcast i8 %x4 to <8 x i1>
   %extract = shufflevector <8 x i1> %6, <8 x i1> %6, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %7 = select <4 x i1> %extract, <4 x i32> %5, <4 x i32> zeroinitializer
@@ -1229,24 +1229,24 @@ declare <4 x i32> @llvm.x86.avx512.vpshrd.d.128(<4 x i32>, <4 x i32>, i32)
 define <8 x i32>@test_int_x86_avx512_mask_vpshrd_d_256_2(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_d_256_2:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdd $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0x7d,0x28,0x73,0xd9,0x16]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshrdd $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x73,0xd1,0x16]
-; X86-NEXT:    vpaddd %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc3]
+; X86-NEXT:    vpshrdd $23, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0x7d,0x28,0x73,0xc1,0x17]
+; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_d_256_2:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdd $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0x7d,0x28,0x73,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshrdd $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x73,0xd1,0x16]
-; X64-NEXT:    vpaddd %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc3]
+; X64-NEXT:    vpshrdd $23, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0x7d,0x28,0x73,0xc1,0x17]
+; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <8 x i32> @llvm.x86.avx512.vpshrd.d.256(<8 x i32> %x0, <8 x i32> %x1, i32 22)
   %2 = bitcast i8 %x4 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x3
-  %4 = call <8 x i32> @llvm.x86.avx512.vpshrd.d.256(<8 x i32> %x0, <8 x i32> %x1, i32 22)
+  %4 = call <8 x i32> @llvm.x86.avx512.vpshrd.d.256(<8 x i32> %x0, <8 x i32> %x1, i32 23)
   %res2 = add <8 x i32> %3, %4
   ret <8 x i32> %res2
 }
@@ -1255,25 +1255,25 @@ declare <8 x i32> @llvm.x86.avx512.vpshrd.d.256(<8 x i32>, <8 x i32>, i32)
 define <2 x i64>@test_int_x86_avx512_mask_vpshrd_q_128_2(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_q_128_2:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdq $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x73,0xd9,0x16]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshrdq $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x73,0xd1,0x16]
-; X86-NEXT:    vpaddq %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc3]
+; X86-NEXT:    vpshrdq $23, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x73,0xc1,0x17]
+; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_q_128_2:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdq $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x73,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshrdq $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x73,0xd1,0x16]
-; X64-NEXT:    vpaddq %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc3]
+; X64-NEXT:    vpshrdq $23, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x73,0xc1,0x17]
+; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <2 x i64> @llvm.x86.avx512.vpshrd.q.128(<2 x i64> %x0, <2 x i64> %x1, i32 22)
   %2 = bitcast i8 %x4 to <8 x i1>
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
   %3 = select <2 x i1> %extract, <2 x i64> %1, <2 x i64> %x3
-  %4 = call <2 x i64> @llvm.x86.avx512.vpshrd.q.128(<2 x i64> %x0, <2 x i64> %x1, i32 22)
+  %4 = call <2 x i64> @llvm.x86.avx512.vpshrd.q.128(<2 x i64> %x0, <2 x i64> %x1, i32 23)
   %res2 = add <2 x i64> %3, %4
   ret <2 x i64> %res2
 }
@@ -1282,25 +1282,25 @@ declare <2 x i64> @llvm.x86.avx512.vpshrd.q.128(<2 x i64>, <2 x i64>, i32)
 define <4 x i64>@test_int_x86_avx512_mask_vpshrd_q_256_2(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_q_256_2:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdq $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x73,0xd9,0x16]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshrdq $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x73,0xd1,0x16]
-; X86-NEXT:    vpaddq %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc3]
+; X86-NEXT:    vpshrdq $23, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0xfd,0x28,0x73,0xc1,0x17]
+; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_q_256_2:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdq $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x73,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshrdq $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x73,0xd1,0x16]
-; X64-NEXT:    vpaddq %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc3]
+; X64-NEXT:    vpshrdq $23, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0xfd,0x28,0x73,0xc1,0x17]
+; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <4 x i64> @llvm.x86.avx512.vpshrd.q.256(<4 x i64> %x0, <4 x i64> %x1, i32 22)
   %2 = bitcast i8 %x4 to <8 x i1>
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract, <4 x i64> %1, <4 x i64> %x3
-  %4 = call <4 x i64> @llvm.x86.avx512.vpshrd.q.256(<4 x i64> %x0, <4 x i64> %x1, i32 22)
+  %4 = call <4 x i64> @llvm.x86.avx512.vpshrd.q.256(<4 x i64> %x0, <4 x i64> %x1, i32 23)
   %res2 = add <4 x i64> %3, %4
   ret <4 x i64> %res2
 }
@@ -1309,24 +1309,24 @@ declare <4 x i64> @llvm.x86.avx512.vpshrd.q.256(<4 x i64>, <4 x i64>, i32)
 define <8 x i16>@test_int_x86_avx512_mask_vpshrd_w_128_2(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_w_128_2:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdw $6, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x72,0xd9,0x06]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshrdw $6, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x72,0xd1,0x06]
-; X86-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
+; X86-NEXT:    vpshrdw $7, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x72,0xc1,0x07]
+; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_w_128_2:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdw $6, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x72,0xd9,0x06]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshrdw $6, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x72,0xd1,0x06]
-; X64-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
+; X64-NEXT:    vpshrdw $7, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x72,0xc1,0x07]
+; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <8 x i16> @llvm.x86.avx512.vpshrd.w.128(<8 x i16> %x0, <8 x i16> %x1, i32 6)
   %2 = bitcast i8 %x4 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %x3
-  %4 = call <8 x i16> @llvm.x86.avx512.vpshrd.w.128(<8 x i16> %x0, <8 x i16> %x1, i32 6)
+  %4 = call <8 x i16> @llvm.x86.avx512.vpshrd.w.128(<8 x i16> %x0, <8 x i16> %x1, i32 7)
   %res2 = add <8 x i16> %3, %4
   ret <8 x i16> %res2
 }
@@ -1335,23 +1335,23 @@ declare <8 x i16> @llvm.x86.avx512.vpshrd.w.128(<8 x i16>, <8 x i16>, i32)
 define <16 x i16>@test_int_x86_avx512_mask_vpshrd_w_256_2(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x3, i16 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_w_256_2:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdw $6, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x72,0xd9,0x06]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpshrdw $6, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x72,0xd1,0x06]
-; X86-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
+; X86-NEXT:    vpshrdw $7, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0xfd,0x28,0x72,0xc1,0x07]
+; X86-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_w_256_2:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdw $6, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x72,0xd9,0x06]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshrdw $6, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x72,0xd1,0x06]
-; X64-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
+; X64-NEXT:    vpshrdw $7, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0xfd,0x28,0x72,0xc1,0x07]
+; X64-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <16 x i16> @llvm.x86.avx512.vpshrd.w.256(<16 x i16> %x0, <16 x i16> %x1, i32 6)
   %2 = bitcast i16 %x4 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %x3
-  %4 = call <16 x i16> @llvm.x86.avx512.vpshrd.w.256(<16 x i16> %x0, <16 x i16> %x1, i32 6)
+  %4 = call <16 x i16> @llvm.x86.avx512.vpshrd.w.256(<16 x i16> %x0, <16 x i16> %x1, i32 7)
   %res2 = add <16 x i16> %3, %4
   ret <16 x i16> %res2
 }
@@ -1360,6 +1360,15 @@ declare <16 x i16> @llvm.x86.avx512.vpshrd.w.256(<16 x i16>, <16 x i16>, i32)
 declare <8 x i32> @llvm.x86.avx512.mask.vpshrdv.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
 declare <8 x i32> @llvm.x86.avx512.maskz.vpshrdv.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
 
+define <8 x i32>@test_int_x86_avx512_vpshrdv_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpshrdv_d_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpshrdvd %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf2,0x75,0x28,0x73,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.vpshrdv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1)
+  ret <8 x i32> %res
+}
+
 define <8 x i32>@test_int_x86_avx512_mask_vpshrdv_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32>* %x2p, <8 x i32> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrdv_d_256:
 ; X86:       # %bb.0:
@@ -1368,10 +1377,7 @@ define <8 x i32>@test_int_x86_avx512_mask_vpshrdv_d_256(<8 x i32> %x0, <8 x i32>
 ; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpshrdvd (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x73,0x18]
-; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpshrdvd %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x73,0xe2]
 ; X86-NEXT:    vpshrdvd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x73,0xc2]
-; X86-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1380,24 +1386,28 @@ define <8 x i32>@test_int_x86_avx512_mask_vpshrdv_d_256(<8 x i32> %x0, <8 x i32>
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X64-NEXT:    vpshrdvd (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x73,0x1f]
-; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X64-NEXT:    vpshrdvd %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x73,0xe2]
 ; X64-NEXT:    vpshrdvd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x73,0xc2]
-; X64-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <8 x i32>, <8 x i32>* %x2p
   %res = call <8 x i32> @llvm.x86.avx512.mask.vpshrdv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
-  %res1 = call <8 x i32> @llvm.x86.avx512.mask.vpshrdv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8 -1)
-  %res2 = call <8 x i32> @llvm.x86.avx512.maskz.vpshrdv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8  %x3)
-  %res3 = add <8 x i32> %res, %res1
-  %res4 = add <8 x i32> %res2, %res3
-  ret <8 x i32> %res4
+  %res1 = call <8 x i32> @llvm.x86.avx512.maskz.vpshrdv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8  %x3)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
 }
 
 declare <4 x i32> @llvm.x86.avx512.mask.vpshrdv.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
 declare <4 x i32> @llvm.x86.avx512.maskz.vpshrdv.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
 
+define <4 x i32>@test_int_x86_avx512_vpshrdv_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpshrdv_d_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpshrdvd %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf2,0x75,0x08,0x73,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.vpshrdv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1)
+  ret <4 x i32> %res
+}
+
 define <4 x i32>@test_int_x86_avx512_mask_vpshrdv_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32>* %x2p, <4 x i32> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrdv_d_128:
 ; X86:       # %bb.0:
@@ -1406,10 +1416,7 @@ define <4 x i32>@test_int_x86_avx512_mask_vpshrdv_d_128(<4 x i32> %x0, <4 x i32>
 ; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpshrdvd (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x73,0x18]
-; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpshrdvd %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x73,0xe2]
 ; X86-NEXT:    vpshrdvd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x73,0xc2]
-; X86-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1418,24 +1425,28 @@ define <4 x i32>@test_int_x86_avx512_mask_vpshrdv_d_128(<4 x i32> %x0, <4 x i32>
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X64-NEXT:    vpshrdvd (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x73,0x1f]
-; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X64-NEXT:    vpshrdvd %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x73,0xe2]
 ; X64-NEXT:    vpshrdvd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x73,0xc2]
-; X64-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <4 x i32>, <4 x i32>* %x2p
   %res = call <4 x i32> @llvm.x86.avx512.mask.vpshrdv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3)
-  %res1 = call <4 x i32> @llvm.x86.avx512.mask.vpshrdv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8 -1)
-  %res2 = call <4 x i32> @llvm.x86.avx512.maskz.vpshrdv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8  %x3)
-  %res3 = add <4 x i32> %res, %res1
-  %res4 = add <4 x i32> %res2, %res3
-  ret <4 x i32> %res4
+  %res1 = call <4 x i32> @llvm.x86.avx512.maskz.vpshrdv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8  %x3)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
 }
 
 declare <4 x i64> @llvm.x86.avx512.mask.vpshrdv.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
 declare <4 x i64> @llvm.x86.avx512.maskz.vpshrdv.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
 
+define <4 x i64>@test_int_x86_avx512_vpshrdv_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpshrdv_q_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpshrdvq %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf2,0xf5,0x28,0x73,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.vpshrdv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 -1)
+  ret <4 x i64> %res
+}
+
 define <4 x i64>@test_int_x86_avx512_mask_vpshrdv_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64>* %x2p, <4 x i64> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrdv_q_256:
 ; X86:       # %bb.0:
@@ -1444,10 +1455,7 @@ define <4 x i64>@test_int_x86_avx512_mask_vpshrdv_q_256(<4 x i64> %x0, <4 x i64>
 ; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpshrdvq (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0x73,0x18]
-; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpshrdvq %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0xf5,0x28,0x73,0xe2]
 ; X86-NEXT:    vpshrdvq %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x73,0xc2]
-; X86-NEXT:    vpaddq %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1456,24 +1464,28 @@ define <4 x i64>@test_int_x86_avx512_mask_vpshrdv_q_256(<4 x i64> %x0, <4 x i64>
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X64-NEXT:    vpshrdvq (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0x73,0x1f]
-; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X64-NEXT:    vpshrdvq %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0xf5,0x28,0x73,0xe2]
 ; X64-NEXT:    vpshrdvq %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x73,0xc2]
-; X64-NEXT:    vpaddq %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <4 x i64>, <4 x i64>* %x2p
   %res = call <4 x i64> @llvm.x86.avx512.mask.vpshrdv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3)
-  %res1 = call <4 x i64> @llvm.x86.avx512.mask.vpshrdv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x4, i8 -1)
-  %res2 = call <4 x i64> @llvm.x86.avx512.maskz.vpshrdv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x4, i8  %x3)
-  %res3 = add <4 x i64> %res, %res1
-  %res4 = add <4 x i64> %res2, %res3
-  ret <4 x i64> %res4
+  %res1 = call <4 x i64> @llvm.x86.avx512.maskz.vpshrdv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x4, i8  %x3)
+  %res2 = add <4 x i64> %res, %res1
+  ret <4 x i64> %res2
 }
 
 declare <2 x i64> @llvm.x86.avx512.mask.vpshrdv.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
 declare <2 x i64> @llvm.x86.avx512.maskz.vpshrdv.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
 
+define <2 x i64>@test_int_x86_avx512_vpshrdv_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpshrdv_q_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpshrdvq %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf2,0xf5,0x08,0x73,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.vpshrdv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1)
+  ret <2 x i64> %res
+}
+
 define <2 x i64>@test_int_x86_avx512_mask_vpshrdv_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64>* %x2p, <2 x i64> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrdv_q_128:
 ; X86:       # %bb.0:
@@ -1482,10 +1494,7 @@ define <2 x i64>@test_int_x86_avx512_mask_vpshrdv_q_128(<2 x i64> %x0, <2 x i64>
 ; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpshrdvq (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0x73,0x18]
-; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpshrdvq %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0xf5,0x08,0x73,0xe2]
 ; X86-NEXT:    vpshrdvq %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x73,0xc2]
-; X86-NEXT:    vpaddq %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1494,24 +1503,28 @@ define <2 x i64>@test_int_x86_avx512_mask_vpshrdv_q_128(<2 x i64> %x0, <2 x i64>
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X64-NEXT:    vpshrdvq (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0x73,0x1f]
-; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X64-NEXT:    vpshrdvq %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0xf5,0x08,0x73,0xe2]
 ; X64-NEXT:    vpshrdvq %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x73,0xc2]
-; X64-NEXT:    vpaddq %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <2 x i64>, <2 x i64>* %x2p
   %res = call <2 x i64> @llvm.x86.avx512.mask.vpshrdv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3)
-  %res1 = call <2 x i64> @llvm.x86.avx512.mask.vpshrdv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x4, i8 -1)
-  %res2 = call <2 x i64> @llvm.x86.avx512.maskz.vpshrdv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x4, i8  %x3)
-  %res3 = add <2 x i64> %res, %res1
-  %res4 = add <2 x i64> %res2, %res3
-  ret <2 x i64> %res4
+  %res1 = call <2 x i64> @llvm.x86.avx512.maskz.vpshrdv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x4, i8  %x3)
+  %res2 = add <2 x i64> %res, %res1
+  ret <2 x i64> %res2
 }
 
 declare <16 x i16> @llvm.x86.avx512.mask.vpshrdv.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
 declare <16 x i16> @llvm.x86.avx512.maskz.vpshrdv.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
 
+define <16 x i16>@test_int_x86_avx512_vpshrdv_w_256(<16 x i16> %x0, <16 x i16> %x1,  <16 x i16> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpshrdv_w_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpshrdvw %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf2,0xf5,0x28,0x72,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.vpshrdv.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 -1)
+  ret <16 x i16> %res
+}
+
 define <16 x i16>@test_int_x86_avx512_mask_vpshrdv_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16>* %x2p, <16 x i16> %x4, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrdv_w_256:
 ; X86:       # %bb.0:
@@ -1519,10 +1532,7 @@ define <16 x i16>@test_int_x86_avx512_mask_vpshrdv_w_256(<16 x i16> %x0, <16 x i
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpshrdvw (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0x72,0x18]
-; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpshrdvw %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0xf5,0x28,0x72,0xe2]
 ; X86-NEXT:    vpshrdvw %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x72,0xc2]
-; X86-NEXT:    vpaddw %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfd,0xc0]
 ; X86-NEXT:    vpaddw %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1531,24 +1541,28 @@ define <16 x i16>@test_int_x86_avx512_mask_vpshrdv_w_256(<16 x i16> %x0, <16 x i
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X64-NEXT:    vpshrdvw (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0x72,0x1f]
-; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X64-NEXT:    vpshrdvw %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0xf5,0x28,0x72,0xe2]
 ; X64-NEXT:    vpshrdvw %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x72,0xc2]
-; X64-NEXT:    vpaddw %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfd,0xc0]
 ; X64-NEXT:    vpaddw %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <16 x i16>, <16 x i16>* %x2p
   %res = call <16 x i16> @llvm.x86.avx512.mask.vpshrdv.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3)
-  %res1 = call <16 x i16> @llvm.x86.avx512.mask.vpshrdv.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x4, i16 -1)
-  %res2 = call <16 x i16> @llvm.x86.avx512.maskz.vpshrdv.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x4, i16  %x3)
-  %res3 = add <16 x i16> %res, %res1
-  %res4 = add <16 x i16> %res2, %res3
-  ret <16 x i16> %res4
+  %res1 = call <16 x i16> @llvm.x86.avx512.maskz.vpshrdv.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x4, i16  %x3)
+  %res2 = add <16 x i16> %res, %res1
+  ret <16 x i16> %res2
 }
 
 declare <8 x i16> @llvm.x86.avx512.mask.vpshrdv.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
 declare <8 x i16> @llvm.x86.avx512.maskz.vpshrdv.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
 
+define <8 x i16>@test_int_x86_avx512_vpshrdv_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpshrdv_w_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpshrdvw %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf2,0xf5,0x08,0x72,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.vpshrdv.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
+  ret <8 x i16> %res
+}
+
 define <8 x i16>@test_int_x86_avx512_mask_vpshrdv_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16>* %x2p, <8 x i16> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrdv_w_128:
 ; X86:       # %bb.0:
@@ -1557,10 +1571,7 @@ define <8 x i16>@test_int_x86_avx512_mask_vpshrdv_w_128(<8 x i16> %x0, <8 x i16>
 ; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpshrdvw (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0x72,0x18]
-; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpshrdvw %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0xf5,0x08,0x72,0xe2]
 ; X86-NEXT:    vpshrdvw %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x72,0xc2]
-; X86-NEXT:    vpaddw %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfd,0xc0]
 ; X86-NEXT:    vpaddw %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1569,24 +1580,28 @@ define <8 x i16>@test_int_x86_avx512_mask_vpshrdv_w_128(<8 x i16> %x0, <8 x i16>
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X64-NEXT:    vpshrdvw (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0x72,0x1f]
-; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X64-NEXT:    vpshrdvw %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0xf5,0x08,0x72,0xe2]
 ; X64-NEXT:    vpshrdvw %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x72,0xc2]
-; X64-NEXT:    vpaddw %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfd,0xc0]
 ; X64-NEXT:    vpaddw %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <8 x i16>, <8 x i16>* %x2p
   %res = call <8 x i16> @llvm.x86.avx512.mask.vpshrdv.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
-  %res1 = call <8 x i16> @llvm.x86.avx512.mask.vpshrdv.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x4, i8 -1)
-  %res2 = call <8 x i16> @llvm.x86.avx512.maskz.vpshrdv.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x4, i8  %x3)
-  %res3 = add <8 x i16> %res, %res1
-  %res4 = add <8 x i16> %res2, %res3
-  ret <8 x i16> %res4
+  %res1 = call <8 x i16> @llvm.x86.avx512.maskz.vpshrdv.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x4, i8  %x3)
+  %res2 = add <8 x i16> %res, %res1
+  ret <8 x i16> %res2
 }
 
 declare <8 x i32> @llvm.x86.avx512.mask.vpshldv.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
 declare <8 x i32> @llvm.x86.avx512.maskz.vpshldv.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
 
+define <8 x i32>@test_int_x86_avx512_vpshldv_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpshldv_d_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpshldvd %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf2,0x75,0x28,0x71,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.vpshldv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1)
+  ret <8 x i32> %res
+}
+
 define <8 x i32>@test_int_x86_avx512_mask_vpshldv_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32>* %x2p, <8 x i32> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshldv_d_256:
 ; X86:       # %bb.0:
@@ -1595,10 +1610,7 @@ define <8 x i32>@test_int_x86_avx512_mask_vpshldv_d_256(<8 x i32> %x0, <8 x i32>
 ; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpshldvd (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x71,0x18]
-; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpshldvd %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x71,0xe2]
 ; X86-NEXT:    vpshldvd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x71,0xc2]
-; X86-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1607,24 +1619,28 @@ define <8 x i32>@test_int_x86_avx512_mask_vpshldv_d_256(<8 x i32> %x0, <8 x i32>
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X64-NEXT:    vpshldvd (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x71,0x1f]
-; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X64-NEXT:    vpshldvd %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x71,0xe2]
 ; X64-NEXT:    vpshldvd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x71,0xc2]
-; X64-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <8 x i32>, <8 x i32>* %x2p
   %res = call <8 x i32> @llvm.x86.avx512.mask.vpshldv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
-  %res1 = call <8 x i32> @llvm.x86.avx512.mask.vpshldv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8 -1)
-  %res2 = call <8 x i32> @llvm.x86.avx512.maskz.vpshldv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8  %x3)
-  %res3 = add <8 x i32> %res, %res1
-  %res4 = add <8 x i32> %res2, %res3
-  ret <8 x i32> %res4
+  %res1 = call <8 x i32> @llvm.x86.avx512.maskz.vpshldv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8  %x3)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
 }
 
 declare <4 x i32> @llvm.x86.avx512.mask.vpshldv.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
 declare <4 x i32> @llvm.x86.avx512.maskz.vpshldv.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
 
+define <4 x i32>@test_int_x86_avx512_vpshldv_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpshldv_d_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpshldvd %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf2,0x75,0x08,0x71,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.vpshldv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1)
+  ret <4 x i32> %res
+}
+
 define <4 x i32>@test_int_x86_avx512_mask_vpshldv_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32>* %x2p, <4 x i32> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshldv_d_128:
 ; X86:       # %bb.0:
@@ -1633,10 +1649,7 @@ define <4 x i32>@test_int_x86_avx512_mask_vpshldv_d_128(<4 x i32> %x0, <4 x i32>
 ; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpshldvd (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x71,0x18]
-; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpshldvd %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x71,0xe2]
 ; X86-NEXT:    vpshldvd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x71,0xc2]
-; X86-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1645,24 +1658,28 @@ define <4 x i32>@test_int_x86_avx512_mask_vpshldv_d_128(<4 x i32> %x0, <4 x i32>
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X64-NEXT:    vpshldvd (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x71,0x1f]
-; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X64-NEXT:    vpshldvd %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x71,0xe2]
 ; X64-NEXT:    vpshldvd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x71,0xc2]
-; X64-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <4 x i32>, <4 x i32>* %x2p
   %res = call <4 x i32> @llvm.x86.avx512.mask.vpshldv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3)
-  %res1 = call <4 x i32> @llvm.x86.avx512.mask.vpshldv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8 -1)
-  %res2 = call <4 x i32> @llvm.x86.avx512.maskz.vpshldv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8  %x3)
-  %res3 = add <4 x i32> %res, %res1
-  %res4 = add <4 x i32> %res2, %res3
-  ret <4 x i32> %res4
+  %res1 = call <4 x i32> @llvm.x86.avx512.maskz.vpshldv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8  %x3)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
 }
 
 declare <4 x i64> @llvm.x86.avx512.mask.vpshldv.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
 declare <4 x i64> @llvm.x86.avx512.maskz.vpshldv.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
 
+define <4 x i64>@test_int_x86_avx512_vpshldv_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpshldv_q_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpshldvq %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf2,0xf5,0x28,0x71,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.vpshldv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 -1)
+  ret <4 x i64> %res
+}
+
 define <4 x i64>@test_int_x86_avx512_mask_vpshldv_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64>* %x2p, <4 x i64> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshldv_q_256:
 ; X86:       # %bb.0:
@@ -1671,10 +1688,7 @@ define <4 x i64>@test_int_x86_avx512_mask_vpshldv_q_256(<4 x i64> %x0, <4 x i64>
 ; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpshldvq (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0x71,0x18]
-; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpshldvq %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0xf5,0x28,0x71,0xe2]
 ; X86-NEXT:    vpshldvq %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x71,0xc2]
-; X86-NEXT:    vpaddq %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1683,24 +1697,28 @@ define <4 x i64>@test_int_x86_avx512_mask_vpshldv_q_256(<4 x i64> %x0, <4 x i64>
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X64-NEXT:    vpshldvq (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0x71,0x1f]
-; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X64-NEXT:    vpshldvq %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0xf5,0x28,0x71,0xe2]
 ; X64-NEXT:    vpshldvq %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x71,0xc2]
-; X64-NEXT:    vpaddq %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <4 x i64>, <4 x i64>* %x2p
   %res = call <4 x i64> @llvm.x86.avx512.mask.vpshldv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3)
-  %res1 = call <4 x i64> @llvm.x86.avx512.mask.vpshldv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x4, i8 -1)
-  %res2 = call <4 x i64> @llvm.x86.avx512.maskz.vpshldv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x4, i8  %x3)
-  %res3 = add <4 x i64> %res, %res1
-  %res4 = add <4 x i64> %res2, %res3
-  ret <4 x i64> %res4
+  %res1 = call <4 x i64> @llvm.x86.avx512.maskz.vpshldv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x4, i8  %x3)
+  %res2 = add <4 x i64> %res, %res1
+  ret <4 x i64> %res2
 }
 
 declare <2 x i64> @llvm.x86.avx512.mask.vpshldv.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
 declare <2 x i64> @llvm.x86.avx512.maskz.vpshldv.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
 
+define <2 x i64>@test_int_x86_avx512_vpshldv_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpshldv_q_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpshldvq %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf2,0xf5,0x08,0x71,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.vpshldv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1)
+  ret <2 x i64> %res
+}
+
 define <2 x i64>@test_int_x86_avx512_mask_vpshldv_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64>* %x2p, <2 x i64> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshldv_q_128:
 ; X86:       # %bb.0:
@@ -1709,10 +1727,7 @@ define <2 x i64>@test_int_x86_avx512_mask_vpshldv_q_128(<2 x i64> %x0, <2 x i64>
 ; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpshldvq (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0x71,0x18]
-; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpshldvq %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0xf5,0x08,0x71,0xe2]
 ; X86-NEXT:    vpshldvq %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x71,0xc2]
-; X86-NEXT:    vpaddq %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1721,24 +1736,28 @@ define <2 x i64>@test_int_x86_avx512_mask_vpshldv_q_128(<2 x i64> %x0, <2 x i64>
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X64-NEXT:    vpshldvq (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0x71,0x1f]
-; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X64-NEXT:    vpshldvq %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0xf5,0x08,0x71,0xe2]
 ; X64-NEXT:    vpshldvq %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x71,0xc2]
-; X64-NEXT:    vpaddq %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <2 x i64>, <2 x i64>* %x2p
   %res = call <2 x i64> @llvm.x86.avx512.mask.vpshldv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3)
-  %res1 = call <2 x i64> @llvm.x86.avx512.mask.vpshldv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x4, i8 -1)
-  %res2 = call <2 x i64> @llvm.x86.avx512.maskz.vpshldv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x4, i8  %x3)
-  %res3 = add <2 x i64> %res, %res1
-  %res4 = add <2 x i64> %res2, %res3
-  ret <2 x i64> %res4
+  %res1 = call <2 x i64> @llvm.x86.avx512.maskz.vpshldv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x4, i8  %x3)
+  %res2 = add <2 x i64> %res, %res1
+  ret <2 x i64> %res2
 }
 
 declare <16 x i16> @llvm.x86.avx512.mask.vpshldv.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
 declare <16 x i16> @llvm.x86.avx512.maskz.vpshldv.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
 
+define <16 x i16>@test_int_x86_avx512_vpshldv_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpshldv_w_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpshldvw %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf2,0xf5,0x28,0x70,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.vpshldv.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 -1)
+  ret <16 x i16> %res
+}
+
 define <16 x i16>@test_int_x86_avx512_mask_vpshldv_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16>* %x2p, <16 x i16> %x4, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshldv_w_256:
 ; X86:       # %bb.0:
@@ -1746,10 +1765,7 @@ define <16 x i16>@test_int_x86_avx512_mask_vpshldv_w_256(<16 x i16> %x0, <16 x i
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpshldvw (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0x70,0x18]
-; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpshldvw %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0xf5,0x28,0x70,0xe2]
 ; X86-NEXT:    vpshldvw %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x70,0xc2]
-; X86-NEXT:    vpaddw %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfd,0xc0]
 ; X86-NEXT:    vpaddw %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1758,24 +1774,28 @@ define <16 x i16>@test_int_x86_avx512_mask_vpshldv_w_256(<16 x i16> %x0, <16 x i
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X64-NEXT:    vpshldvw (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0x70,0x1f]
-; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X64-NEXT:    vpshldvw %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0xf5,0x28,0x70,0xe2]
 ; X64-NEXT:    vpshldvw %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x70,0xc2]
-; X64-NEXT:    vpaddw %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfd,0xc0]
 ; X64-NEXT:    vpaddw %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <16 x i16>, <16 x i16>* %x2p
   %res = call <16 x i16> @llvm.x86.avx512.mask.vpshldv.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3)
-  %res1 = call <16 x i16> @llvm.x86.avx512.mask.vpshldv.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x4, i16 -1)
-  %res2 = call <16 x i16> @llvm.x86.avx512.maskz.vpshldv.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x4, i16  %x3)
-  %res3 = add <16 x i16> %res, %res1
-  %res4 = add <16 x i16> %res2, %res3
-  ret <16 x i16> %res4
+  %res1 = call <16 x i16> @llvm.x86.avx512.maskz.vpshldv.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x4, i16  %x3)
+  %res2 = add <16 x i16> %res, %res1
+  ret <16 x i16> %res2
 }
 
 declare <8 x i16> @llvm.x86.avx512.mask.vpshldv.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
 declare <8 x i16> @llvm.x86.avx512.maskz.vpshldv.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
 
+define <8 x i16>@test_int_x86_avx512_vpshldv_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpshldv_w_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpshldvw %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf2,0xf5,0x08,0x70,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.vpshldv.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
+  ret <8 x i16> %res
+}
+
 define <8 x i16>@test_int_x86_avx512_mask_vpshldv_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16>* %x2p, <8 x i16> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshldv_w_128:
 ; X86:       # %bb.0:
@@ -1784,10 +1804,7 @@ define <8 x i16>@test_int_x86_avx512_mask_vpshldv_w_128(<8 x i16> %x0, <8 x i16>
 ; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpshldvw (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0x70,0x18]
-; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpshldvw %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0xf5,0x08,0x70,0xe2]
 ; X86-NEXT:    vpshldvw %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x70,0xc2]
-; X86-NEXT:    vpaddw %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfd,0xc0]
 ; X86-NEXT:    vpaddw %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1796,17 +1813,12 @@ define <8 x i16>@test_int_x86_avx512_mask_vpshldv_w_128(<8 x i16> %x0, <8 x i16>
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X64-NEXT:    vpshldvw (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0x70,0x1f]
-; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X64-NEXT:    vpshldvw %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0xf5,0x08,0x70,0xe2]
 ; X64-NEXT:    vpshldvw %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x70,0xc2]
-; X64-NEXT:    vpaddw %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfd,0xc0]
 ; X64-NEXT:    vpaddw %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <8 x i16>, <8 x i16>* %x2p
   %res = call <8 x i16> @llvm.x86.avx512.mask.vpshldv.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
-  %res1 = call <8 x i16> @llvm.x86.avx512.mask.vpshldv.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x4, i8 -1)
-  %res2 = call <8 x i16> @llvm.x86.avx512.maskz.vpshldv.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x4, i8  %x3)
-  %res3 = add <8 x i16> %res, %res1
-  %res4 = add <8 x i16> %res2, %res3
-  ret <8 x i16> %res4
+  %res1 = call <8 x i16> @llvm.x86.avx512.maskz.vpshldv.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x4, i8  %x3)
+  %res2 = add <8 x i16> %res, %res1
+  ret <8 x i16> %res2
 }
diff --git a/llvm/test/CodeGen/X86/avx512vbmi2vl-intrinsics.ll b/llvm/test/CodeGen/X86/avx512vbmi2vl-intrinsics.ll
index 371499c476e24..b82260248e51a 100644
--- a/llvm/test/CodeGen/X86/avx512vbmi2vl-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512vbmi2vl-intrinsics.ll
@@ -734,21 +734,21 @@ define <32 x i8> @test_compress_b_256(<32 x i8> %data) {
 define <4 x i32> @test_int_x86_avx512_mask_vpshld_d_128(<4 x i32> %x0, <4 x i32> %x1,<4 x i32> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_d_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldd $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x71,0xd9,0x16]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshldd $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x71,0xd1,0x16]
-; X86-NEXT:    vpshldd $22, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x71,0xc1,0x16]
+; X86-NEXT:    vpshldd $23, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x71,0xd9,0x17]
+; X86-NEXT:    vpshldd $24, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x71,0xc1,0x18]
 ; X86-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_d_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldd $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x71,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshldd $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x71,0xd1,0x16]
-; X64-NEXT:    vpshldd $22, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x71,0xc1,0x16]
+; X64-NEXT:    vpshldd $23, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x71,0xd9,0x17]
+; X64-NEXT:    vpshldd $24, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x71,0xc1,0x18]
 ; X64-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
@@ -756,8 +756,8 @@ define <4 x i32> @test_int_x86_avx512_mask_vpshld_d_128(<4 x i32> %x0, <4 x i32>
   %2 = bitcast i8 %x4 to <8 x i1>
   %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract1, <4 x i32> %1, <4 x i32> %x3
-  %4 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> <i32 22, i32 22, i32 22, i32 22>)
-  %5 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> <i32 22, i32 22, i32 22, i32 22>)
+  %4 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> <i32 23, i32 23, i32 23, i32 23>)
+  %5 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> <i32 24, i32 24, i32 24, i32 24>)
   %6 = bitcast i8 %x4 to <8 x i1>
   %extract = shufflevector <8 x i1> %6, <8 x i1> %6, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %7 = select <4 x i1> %extract, <4 x i32> %5, <4 x i32> zeroinitializer
@@ -769,24 +769,24 @@ define <4 x i32> @test_int_x86_avx512_mask_vpshld_d_128(<4 x i32> %x0, <4 x i32>
 define <8 x i32> @test_int_x86_avx512_mask_vpshld_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldd $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0x7d,0x28,0x71,0xd9,0x16]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshldd $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x71,0xd1,0x16]
-; X86-NEXT:    vpaddd %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc3]
+; X86-NEXT:    vpshldd $23, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0x7d,0x28,0x71,0xc1,0x17]
+; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_d_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldd $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0x7d,0x28,0x71,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshldd $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x71,0xd1,0x16]
-; X64-NEXT:    vpaddd %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc3]
+; X64-NEXT:    vpshldd $23, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0x7d,0x28,0x71,0xc1,0x17]
+; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> <i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22>)
   %2 = bitcast i8 %x4 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x3
-  %4 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> <i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22>)
+  %4 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> <i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23>)
   %res2 = add <8 x i32> %3, %4
   ret <8 x i32> %res2
 }
@@ -794,25 +794,25 @@ define <8 x i32> @test_int_x86_avx512_mask_vpshld_d_256(<8 x i32> %x0, <8 x i32>
 define <2 x i64> @test_int_x86_avx512_mask_vpshld_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_q_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldq $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x71,0xd9,0x16]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshldq $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x71,0xd1,0x16]
-; X86-NEXT:    vpaddq %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc3]
+; X86-NEXT:    vpshldq $23, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x71,0xc1,0x17]
+; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_q_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldq $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x71,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshldq $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x71,0xd1,0x16]
-; X64-NEXT:    vpaddq %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc3]
+; X64-NEXT:    vpshldq $23, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x71,0xc1,0x17]
+; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> <i64 22, i64 22>)
   %2 = bitcast i8 %x4 to <8 x i1>
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
   %3 = select <2 x i1> %extract, <2 x i64> %1, <2 x i64> %x3
-  %4 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> <i64 22, i64 22>)
+  %4 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> <i64 23, i64 23>)
   %res2 = add <2 x i64> %3, %4
   ret <2 x i64> %res2
 }
@@ -820,25 +820,25 @@ define <2 x i64> @test_int_x86_avx512_mask_vpshld_q_128(<2 x i64> %x0, <2 x i64>
 define <4 x i64> @test_int_x86_avx512_mask_vpshld_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_q_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldq $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x71,0xd9,0x16]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshldq $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x71,0xd1,0x16]
-; X86-NEXT:    vpaddq %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc3]
+; X86-NEXT:    vpshldq $23, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0xfd,0x28,0x71,0xc1,0x17]
+; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_q_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldq $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x71,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshldq $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x71,0xd1,0x16]
-; X64-NEXT:    vpaddq %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc3]
+; X64-NEXT:    vpshldq $23, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0xfd,0x28,0x71,0xc1,0x17]
+; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> <i64 22, i64 22, i64 22, i64 22>)
   %2 = bitcast i8 %x4 to <8 x i1>
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract, <4 x i64> %1, <4 x i64> %x3
-  %4 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> <i64 22, i64 22, i64 22, i64 22>)
+  %4 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> <i64 23, i64 23, i64 23, i64 23>)
   %res2 = add <4 x i64> %3, %4
   ret <4 x i64> %res2
 }
@@ -846,24 +846,24 @@ define <4 x i64> @test_int_x86_avx512_mask_vpshld_q_256(<4 x i64> %x0, <4 x i64>
 define <8 x i16> @test_int_x86_avx512_mask_vpshld_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_w_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldw $6, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x70,0xd9,0x06]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshldw $6, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x70,0xd1,0x06]
-; X86-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
+; X86-NEXT:    vpshldw $7, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x70,0xc1,0x07]
+; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_w_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldw $6, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x70,0xd9,0x06]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshldw $6, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x70,0xd1,0x06]
-; X64-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
+; X64-NEXT:    vpshldw $7, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x70,0xc1,0x07]
+; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> <i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6>)
   %2 = bitcast i8 %x4 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %x3
-  %4 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> <i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6>)
+  %4 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>)
   %res2 = add <8 x i16> %3, %4
   ret <8 x i16> %res2
 }
@@ -871,23 +871,23 @@ define <8 x i16> @test_int_x86_avx512_mask_vpshld_w_128(<8 x i16> %x0, <8 x i16>
 define <16 x i16> @test_int_x86_avx512_mask_vpshld_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x3, i16 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_w_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldw $6, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x70,0xd9,0x06]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpshldw $6, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x70,0xd1,0x06]
-; X86-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
+; X86-NEXT:    vpshldw $7, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0xfd,0x28,0x70,0xc1,0x07]
+; X86-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_w_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldw $6, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x70,0xd9,0x06]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshldw $6, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x70,0xd1,0x06]
-; X64-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
+; X64-NEXT:    vpshldw $7, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0xfd,0x28,0x70,0xc1,0x07]
+; X64-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> <i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6>)
   %2 = bitcast i16 %x4 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %x3
-  %4 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> <i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6>)
+  %4 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>)
   %res2 = add <16 x i16> %3, %4
   ret <16 x i16> %res2
 }
@@ -895,21 +895,21 @@ define <16 x i16> @test_int_x86_avx512_mask_vpshld_w_256(<16 x i16> %x0, <16 x i
 define <4 x i32> @test_int_x86_avx512_mask_vpshrd_d_128(<4 x i32> %x0, <4 x i32> %x1,<4 x i32> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_d_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdd $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x73,0xd9,0x16]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshrdd $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x73,0xd1,0x16]
-; X86-NEXT:    vpshrdd $22, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x73,0xc1,0x16]
+; X86-NEXT:    vpshrdd $23, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x73,0xd9,0x17]
+; X86-NEXT:    vpshrdd $24, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x73,0xc1,0x18]
 ; X86-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_d_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdd $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x73,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshrdd $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x73,0xd1,0x16]
-; X64-NEXT:    vpshrdd $22, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x73,0xc1,0x16]
+; X64-NEXT:    vpshrdd $23, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x73,0xd9,0x17]
+; X64-NEXT:    vpshrdd $24, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x73,0xc1,0x18]
 ; X64-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
@@ -917,8 +917,8 @@ define <4 x i32> @test_int_x86_avx512_mask_vpshrd_d_128(<4 x i32> %x0, <4 x i32>
   %2 = bitcast i8 %x4 to <8 x i1>
   %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract1, <4 x i32> %1, <4 x i32> %x3
-  %4 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x1, <4 x i32> %x0, <4 x i32> <i32 22, i32 22, i32 22, i32 22>)
-  %5 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x1, <4 x i32> %x0, <4 x i32> <i32 22, i32 22, i32 22, i32 22>)
+  %4 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x1, <4 x i32> %x0, <4 x i32> <i32 23, i32 23, i32 23, i32 23>)
+  %5 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x1, <4 x i32> %x0, <4 x i32> <i32 24, i32 24, i32 24, i32 24>)
   %6 = bitcast i8 %x4 to <8 x i1>
   %extract = shufflevector <8 x i1> %6, <8 x i1> %6, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %7 = select <4 x i1> %extract, <4 x i32> %5, <4 x i32> zeroinitializer
@@ -930,24 +930,24 @@ define <4 x i32> @test_int_x86_avx512_mask_vpshrd_d_128(<4 x i32> %x0, <4 x i32>
 define <8 x i32> @test_int_x86_avx512_mask_vpshrd_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdd $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0x7d,0x28,0x73,0xd9,0x16]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshrdd $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x73,0xd1,0x16]
-; X86-NEXT:    vpaddd %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc3]
+; X86-NEXT:    vpshrdd $23, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0x7d,0x28,0x73,0xc1,0x17]
+; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_d_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdd $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0x7d,0x28,0x73,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshrdd $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x73,0xd1,0x16]
-; X64-NEXT:    vpaddd %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc3]
+; X64-NEXT:    vpshrdd $23, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0x7d,0x28,0x73,0xc1,0x17]
+; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %x1, <8 x i32> %x0, <8 x i32> <i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22>)
   %2 = bitcast i8 %x4 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x3
-  %4 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %x1, <8 x i32> %x0, <8 x i32> <i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22>)
+  %4 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %x1, <8 x i32> %x0, <8 x i32> <i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23>)
   %res2 = add <8 x i32> %3, %4
   ret <8 x i32> %res2
 }
@@ -955,25 +955,25 @@ define <8 x i32> @test_int_x86_avx512_mask_vpshrd_d_256(<8 x i32> %x0, <8 x i32>
 define <2 x i64> @test_int_x86_avx512_mask_vpshrd_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_q_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdq $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x73,0xd9,0x16]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshrdq $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x73,0xd1,0x16]
-; X86-NEXT:    vpaddq %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc3]
+; X86-NEXT:    vpshrdq $23, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x73,0xc1,0x17]
+; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_q_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdq $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x73,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshrdq $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x73,0xd1,0x16]
-; X64-NEXT:    vpaddq %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc3]
+; X64-NEXT:    vpshrdq $23, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x73,0xc1,0x17]
+; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %x1, <2 x i64> %x0, <2 x i64> <i64 22, i64 22>)
   %2 = bitcast i8 %x4 to <8 x i1>
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
   %3 = select <2 x i1> %extract, <2 x i64> %1, <2 x i64> %x3
-  %4 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %x1, <2 x i64> %x0, <2 x i64> <i64 22, i64 22>)
+  %4 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %x1, <2 x i64> %x0, <2 x i64> <i64 23, i64 23>)
   %res2 = add <2 x i64> %3, %4
   ret <2 x i64> %res2
 }
@@ -981,25 +981,25 @@ define <2 x i64> @test_int_x86_avx512_mask_vpshrd_q_128(<2 x i64> %x0, <2 x i64>
 define <4 x i64> @test_int_x86_avx512_mask_vpshrd_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_q_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdq $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x73,0xd9,0x16]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshrdq $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x73,0xd1,0x16]
-; X86-NEXT:    vpaddq %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc3]
+; X86-NEXT:    vpshrdq $23, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0xfd,0x28,0x73,0xc1,0x17]
+; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_q_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdq $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x73,0xd9,0x16]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshrdq $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x73,0xd1,0x16]
-; X64-NEXT:    vpaddq %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc3]
+; X64-NEXT:    vpshrdq $23, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0xfd,0x28,0x73,0xc1,0x17]
+; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %x1, <4 x i64> %x0, <4 x i64> <i64 22, i64 22, i64 22, i64 22>)
   %2 = bitcast i8 %x4 to <8 x i1>
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract, <4 x i64> %1, <4 x i64> %x3
-  %4 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %x1, <4 x i64> %x0, <4 x i64> <i64 22, i64 22, i64 22, i64 22>)
+  %4 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %x1, <4 x i64> %x0, <4 x i64> <i64 23, i64 23, i64 23, i64 23>)
   %res2 = add <4 x i64> %3, %4
   ret <4 x i64> %res2
 }
@@ -1007,24 +1007,24 @@ define <4 x i64> @test_int_x86_avx512_mask_vpshrd_q_256(<4 x i64> %x0, <4 x i64>
 define <8 x i16> @test_int_x86_avx512_mask_vpshrd_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_w_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdw $6, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x72,0xd9,0x06]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    vpshrdw $6, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x72,0xd1,0x06]
-; X86-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
+; X86-NEXT:    vpshrdw $7, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x72,0xc1,0x07]
+; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_w_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdw $6, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x72,0xd9,0x06]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshrdw $6, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x72,0xd1,0x06]
-; X64-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
+; X64-NEXT:    vpshrdw $7, %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x72,0xc1,0x07]
+; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %x1, <8 x i16> %x0, <8 x i16> <i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6>)
   %2 = bitcast i8 %x4 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %x3
-  %4 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %x1, <8 x i16> %x0, <8 x i16> <i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6>)
+  %4 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %x1, <8 x i16> %x0, <8 x i16> <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>)
   %res2 = add <8 x i16> %3, %4
   ret <8 x i16> %res2
 }
@@ -1032,23 +1032,23 @@ define <8 x i16> @test_int_x86_avx512_mask_vpshrd_w_128(<8 x i16> %x0, <8 x i16>
 define <16 x i16> @test_int_x86_avx512_mask_vpshrd_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x3, i16 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_w_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdw $6, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x72,0xd9,0x06]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpshrdw $6, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x72,0xd1,0x06]
-; X86-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
+; X86-NEXT:    vpshrdw $7, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0xfd,0x28,0x72,0xc1,0x07]
+; X86-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_w_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdw $6, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x72,0xd9,0x06]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpshrdw $6, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x72,0xd1,0x06]
-; X64-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
+; X64-NEXT:    vpshrdw $7, %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf3,0xfd,0x28,0x72,0xc1,0x07]
+; X64-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %x1, <16 x i16> %x0, <16 x i16> <i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6>)
   %2 = bitcast i16 %x4 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %x3
-  %4 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %x1, <16 x i16> %x0, <16 x i16> <i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6>)
+  %4 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %x1, <16 x i16> %x0, <16 x i16> <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>)
   %res2 = add <16 x i16> %3, %4
   ret <16 x i16> %res2
 }
@@ -1061,10 +1061,7 @@ define <8 x i32> @test_int_x86_avx512_mask_vpshrdv_d_256(<8 x i32> %x0, <8 x i32
 ; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpshrdvd (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x73,0x18]
-; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpshrdvd %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x73,0xe2]
 ; X86-NEXT:    vpshrdvd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x73,0xc2]
-; X86-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1073,10 +1070,7 @@ define <8 x i32> @test_int_x86_avx512_mask_vpshrdv_d_256(<8 x i32> %x0, <8 x i32
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X64-NEXT:    vpshrdvd (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x73,0x1f]
-; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X64-NEXT:    vpshrdvd %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x73,0xe2]
 ; X64-NEXT:    vpshrdvd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x73,0xc2]
-; X64-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <8 x i32>, <8 x i32>* %x2p
@@ -1084,12 +1078,10 @@ define <8 x i32> @test_int_x86_avx512_mask_vpshrdv_d_256(<8 x i32> %x0, <8 x i32
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x0
   %4 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %x1, <8 x i32> %x0, <8 x i32> %x4)
-  %5 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %x1, <8 x i32> %x0, <8 x i32> %x4)
-  %6 = bitcast i8 %x3 to <8 x i1>
-  %7 = select <8 x i1> %6, <8 x i32> %5, <8 x i32> zeroinitializer
-  %res3 = add <8 x i32> %3, %4
-  %res4 = add <8 x i32> %7, %res3
-  ret <8 x i32> %res4
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = select <8 x i1> %5, <8 x i32> %4, <8 x i32> zeroinitializer
+  %res3 = add <8 x i32> %3, %6
+  ret <8 x i32> %res3
 }
 
 define <4 x i32> @test_int_x86_avx512_mask_vpshrdv_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32>* %x2p, <4 x i32> %x4, i8 %x3) {
@@ -1100,10 +1092,7 @@ define <4 x i32> @test_int_x86_avx512_mask_vpshrdv_d_128(<4 x i32> %x0, <4 x i32
 ; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpshrdvd (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x73,0x18]
-; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpshrdvd %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x73,0xe2]
 ; X86-NEXT:    vpshrdvd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x73,0xc2]
-; X86-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1112,10 +1101,7 @@ define <4 x i32> @test_int_x86_avx512_mask_vpshrdv_d_128(<4 x i32> %x0, <4 x i32
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X64-NEXT:    vpshrdvd (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x73,0x1f]
-; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X64-NEXT:    vpshrdvd %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x73,0xe2]
 ; X64-NEXT:    vpshrdvd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x73,0xc2]
-; X64-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <4 x i32>, <4 x i32>* %x2p
@@ -1124,13 +1110,11 @@ define <4 x i32> @test_int_x86_avx512_mask_vpshrdv_d_128(<4 x i32> %x0, <4 x i32
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract, <4 x i32> %1, <4 x i32> %x0
   %4 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x1, <4 x i32> %x0, <4 x i32> %x4)
-  %5 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x1, <4 x i32> %x0, <4 x i32> %x4)
-  %6 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %6, <8 x i1> %6, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %7 = select <4 x i1> %extract1, <4 x i32> %5, <4 x i32> zeroinitializer
-  %res3 = add <4 x i32> %3, %4
-  %res4 = add <4 x i32> %7, %res3
-  ret <4 x i32> %res4
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %6 = select <4 x i1> %extract1, <4 x i32> %4, <4 x i32> zeroinitializer
+  %res3 = add <4 x i32> %3, %6
+  ret <4 x i32> %res3
 }
 
 define <4 x i64> @test_int_x86_avx512_mask_vpshrdv_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64>* %x2p, <4 x i64> %x4, i8 %x3) {
@@ -1141,10 +1125,7 @@ define <4 x i64> @test_int_x86_avx512_mask_vpshrdv_q_256(<4 x i64> %x0, <4 x i64
 ; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpshrdvq (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0x73,0x18]
-; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpshrdvq %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0xf5,0x28,0x73,0xe2]
 ; X86-NEXT:    vpshrdvq %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x73,0xc2]
-; X86-NEXT:    vpaddq %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1153,10 +1134,7 @@ define <4 x i64> @test_int_x86_avx512_mask_vpshrdv_q_256(<4 x i64> %x0, <4 x i64
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X64-NEXT:    vpshrdvq (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0x73,0x1f]
-; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X64-NEXT:    vpshrdvq %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0xf5,0x28,0x73,0xe2]
 ; X64-NEXT:    vpshrdvq %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x73,0xc2]
-; X64-NEXT:    vpaddq %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <4 x i64>, <4 x i64>* %x2p
@@ -1165,13 +1143,11 @@ define <4 x i64> @test_int_x86_avx512_mask_vpshrdv_q_256(<4 x i64> %x0, <4 x i64
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract, <4 x i64> %1, <4 x i64> %x0
   %4 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %x1, <4 x i64> %x0, <4 x i64> %x4)
-  %5 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %x1, <4 x i64> %x0, <4 x i64> %x4)
-  %6 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %6, <8 x i1> %6, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %7 = select <4 x i1> %extract1, <4 x i64> %5, <4 x i64> zeroinitializer
-  %res3 = add <4 x i64> %3, %4
-  %res4 = add <4 x i64> %7, %res3
-  ret <4 x i64> %res4
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %6 = select <4 x i1> %extract1, <4 x i64> %4, <4 x i64> zeroinitializer
+  %res3 = add <4 x i64> %3, %6
+  ret <4 x i64> %res3
 }
 
 define <2 x i64> @test_int_x86_avx512_mask_vpshrdv_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64>* %x2p, <2 x i64> %x4, i8 %x3) {
@@ -1182,10 +1158,7 @@ define <2 x i64> @test_int_x86_avx512_mask_vpshrdv_q_128(<2 x i64> %x0, <2 x i64
 ; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpshrdvq (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0x73,0x18]
-; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpshrdvq %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0xf5,0x08,0x73,0xe2]
 ; X86-NEXT:    vpshrdvq %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x73,0xc2]
-; X86-NEXT:    vpaddq %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1194,10 +1167,7 @@ define <2 x i64> @test_int_x86_avx512_mask_vpshrdv_q_128(<2 x i64> %x0, <2 x i64
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X64-NEXT:    vpshrdvq (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0x73,0x1f]
-; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X64-NEXT:    vpshrdvq %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0xf5,0x08,0x73,0xe2]
 ; X64-NEXT:    vpshrdvq %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x73,0xc2]
-; X64-NEXT:    vpaddq %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <2 x i64>, <2 x i64>* %x2p
@@ -1206,13 +1176,11 @@ define <2 x i64> @test_int_x86_avx512_mask_vpshrdv_q_128(<2 x i64> %x0, <2 x i64
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
   %3 = select <2 x i1> %extract, <2 x i64> %1, <2 x i64> %x0
   %4 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %x1, <2 x i64> %x0, <2 x i64> %x4)
-  %5 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %x1, <2 x i64> %x0, <2 x i64> %x4)
-  %6 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %6, <8 x i1> %6, <2 x i32> <i32 0, i32 1>
-  %7 = select <2 x i1> %extract1, <2 x i64> %5, <2 x i64> zeroinitializer
-  %res3 = add <2 x i64> %3, %4
-  %res4 = add <2 x i64> %7, %res3
-  ret <2 x i64> %res4
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %5, <8 x i1> %5, <2 x i32> <i32 0, i32 1>
+  %6 = select <2 x i1> %extract1, <2 x i64> %4, <2 x i64> zeroinitializer
+  %res3 = add <2 x i64> %3, %6
+  ret <2 x i64> %res3
 }
 
 define <16 x i16> @test_int_x86_avx512_mask_vpshrdv_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16>* %x2p, <16 x i16> %x4, i16 %x3) {
@@ -1222,10 +1190,7 @@ define <16 x i16> @test_int_x86_avx512_mask_vpshrdv_w_256(<16 x i16> %x0, <16 x
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpshrdvw (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0x72,0x18]
-; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpshrdvw %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0xf5,0x28,0x72,0xe2]
-; X86-NEXT:    vpshrdvw %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x72,0xc2]
-; X86-NEXT:    vpaddw %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfd,0xc0]
+; X86-NEXT:    vpshrdvw %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf2,0xf5,0x28,0x72,0xc2]
 ; X86-NEXT:    vpaddw %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1234,10 +1199,7 @@ define <16 x i16> @test_int_x86_avx512_mask_vpshrdv_w_256(<16 x i16> %x0, <16 x
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X64-NEXT:    vpshrdvw (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0x72,0x1f]
-; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X64-NEXT:    vpshrdvw %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0xf5,0x28,0x72,0xe2]
-; X64-NEXT:    vpshrdvw %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x72,0xc2]
-; X64-NEXT:    vpaddw %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfd,0xc0]
+; X64-NEXT:    vpshrdvw %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf2,0xf5,0x28,0x72,0xc2]
 ; X64-NEXT:    vpaddw %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <16 x i16>, <16 x i16>* %x2p
@@ -1245,12 +1207,10 @@ define <16 x i16> @test_int_x86_avx512_mask_vpshrdv_w_256(<16 x i16> %x0, <16 x
   %2 = bitcast i16 %x3 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %x0
   %4 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %x1, <16 x i16> %x0, <16 x i16> %x4)
-  %5 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %x1, <16 x i16> %x0, <16 x i16> %x4)
-  %6 = bitcast i16 %x3 to <16 x i1>
-  %7 = select <16 x i1> %6, <16 x i16> %5, <16 x i16> zeroinitializer
+  %5 = bitcast i16 %x3 to <16 x i1>
+  %6 = select <16 x i1> %5, <16 x i16> %4, <16 x i16> zeroinitializer
   %res3 = add <16 x i16> %3, %4
-  %res4 = add <16 x i16> %7, %res3
-  ret <16 x i16> %res4
+  ret <16 x i16> %res3
 }
 
 define <8 x i16> @test_int_x86_avx512_mask_vpshrdv_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16>* %x2p, <8 x i16> %x4, i8 %x3) {
@@ -1261,10 +1221,7 @@ define <8 x i16> @test_int_x86_avx512_mask_vpshrdv_w_128(<8 x i16> %x0, <8 x i16
 ; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpshrdvw (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0x72,0x18]
-; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpshrdvw %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0xf5,0x08,0x72,0xe2]
 ; X86-NEXT:    vpshrdvw %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x72,0xc2]
-; X86-NEXT:    vpaddw %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfd,0xc0]
 ; X86-NEXT:    vpaddw %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1273,10 +1230,7 @@ define <8 x i16> @test_int_x86_avx512_mask_vpshrdv_w_128(<8 x i16> %x0, <8 x i16
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X64-NEXT:    vpshrdvw (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0x72,0x1f]
-; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X64-NEXT:    vpshrdvw %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0xf5,0x08,0x72,0xe2]
 ; X64-NEXT:    vpshrdvw %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x72,0xc2]
-; X64-NEXT:    vpaddw %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfd,0xc0]
 ; X64-NEXT:    vpaddw %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <8 x i16>, <8 x i16>* %x2p
@@ -1284,12 +1238,10 @@ define <8 x i16> @test_int_x86_avx512_mask_vpshrdv_w_128(<8 x i16> %x0, <8 x i16
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %x0
   %4 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %x1, <8 x i16> %x0, <8 x i16> %x4)
-  %5 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %x1, <8 x i16> %x0, <8 x i16> %x4)
-  %6 = bitcast i8 %x3 to <8 x i1>
-  %7 = select <8 x i1> %6, <8 x i16> %5, <8 x i16> zeroinitializer
-  %res3 = add <8 x i16> %3, %4
-  %res4 = add <8 x i16> %7, %res3
-  ret <8 x i16> %res4
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = select <8 x i1> %5, <8 x i16> %4, <8 x i16> zeroinitializer
+  %res3 = add <8 x i16> %3, %6
+  ret <8 x i16> %res3
 }
 
 define <8 x i32> @test_int_x86_avx512_mask_vpshldv_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32>* %x2p, <8 x i32> %x4, i8 %x3) {
@@ -1300,10 +1252,7 @@ define <8 x i32> @test_int_x86_avx512_mask_vpshldv_d_256(<8 x i32> %x0, <8 x i32
 ; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpshldvd (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x71,0x18]
-; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpshldvd %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x71,0xe2]
 ; X86-NEXT:    vpshldvd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x71,0xc2]
-; X86-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1312,10 +1261,7 @@ define <8 x i32> @test_int_x86_avx512_mask_vpshldv_d_256(<8 x i32> %x0, <8 x i32
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X64-NEXT:    vpshldvd (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x71,0x1f]
-; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X64-NEXT:    vpshldvd %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x71,0xe2]
 ; X64-NEXT:    vpshldvd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x71,0xc2]
-; X64-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <8 x i32>, <8 x i32>* %x2p
@@ -1323,12 +1269,10 @@ define <8 x i32> @test_int_x86_avx512_mask_vpshldv_d_256(<8 x i32> %x0, <8 x i32
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x0
   %4 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4)
-  %5 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4)
-  %6 = bitcast i8 %x3 to <8 x i1>
-  %7 = select <8 x i1> %6, <8 x i32> %5, <8 x i32> zeroinitializer
-  %res3 = add <8 x i32> %3, %4
-  %res4 = add <8 x i32> %7, %res3
-  ret <8 x i32> %res4
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = select <8 x i1> %5, <8 x i32> %4, <8 x i32> zeroinitializer
+  %res3 = add <8 x i32> %3, %6
+  ret <8 x i32> %res3
 }
 
 define <4 x i32> @test_int_x86_avx512_mask_vpshldv_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32>* %x2p, <4 x i32> %x4, i8 %x3) {
@@ -1339,10 +1283,7 @@ define <4 x i32> @test_int_x86_avx512_mask_vpshldv_d_128(<4 x i32> %x0, <4 x i32
 ; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpshldvd (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x71,0x18]
-; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpshldvd %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x71,0xe2]
 ; X86-NEXT:    vpshldvd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x71,0xc2]
-; X86-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1351,10 +1292,7 @@ define <4 x i32> @test_int_x86_avx512_mask_vpshldv_d_128(<4 x i32> %x0, <4 x i32
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X64-NEXT:    vpshldvd (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x71,0x1f]
-; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X64-NEXT:    vpshldvd %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x71,0xe2]
 ; X64-NEXT:    vpshldvd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x71,0xc2]
-; X64-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <4 x i32>, <4 x i32>* %x2p
@@ -1363,13 +1301,11 @@ define <4 x i32> @test_int_x86_avx512_mask_vpshldv_d_128(<4 x i32> %x0, <4 x i32
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract, <4 x i32> %1, <4 x i32> %x0
   %4 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4)
-  %5 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4)
-  %6 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %6, <8 x i1> %6, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %7 = select <4 x i1> %extract1, <4 x i32> %5, <4 x i32> zeroinitializer
-  %res3 = add <4 x i32> %3, %4
-  %res4 = add <4 x i32> %7, %res3
-  ret <4 x i32> %res4
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %6 = select <4 x i1> %extract1, <4 x i32> %4, <4 x i32> zeroinitializer
+  %res3 = add <4 x i32> %3, %6
+  ret <4 x i32> %res3
 }
 
 define <4 x i64> @test_int_x86_avx512_mask_vpshldv_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64>* %x2p, <4 x i64> %x4, i8 %x3) {
@@ -1380,10 +1316,7 @@ define <4 x i64> @test_int_x86_avx512_mask_vpshldv_q_256(<4 x i64> %x0, <4 x i64
 ; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpshldvq (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0x71,0x18]
-; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpshldvq %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0xf5,0x28,0x71,0xe2]
 ; X86-NEXT:    vpshldvq %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x71,0xc2]
-; X86-NEXT:    vpaddq %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1392,10 +1325,7 @@ define <4 x i64> @test_int_x86_avx512_mask_vpshldv_q_256(<4 x i64> %x0, <4 x i64
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X64-NEXT:    vpshldvq (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0x71,0x1f]
-; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X64-NEXT:    vpshldvq %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0xf5,0x28,0x71,0xe2]
 ; X64-NEXT:    vpshldvq %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x71,0xc2]
-; X64-NEXT:    vpaddq %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <4 x i64>, <4 x i64>* %x2p
@@ -1404,13 +1334,11 @@ define <4 x i64> @test_int_x86_avx512_mask_vpshldv_q_256(<4 x i64> %x0, <4 x i64
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract, <4 x i64> %1, <4 x i64> %x0
   %4 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x4)
-  %5 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x4)
-  %6 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %6, <8 x i1> %6, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %7 = select <4 x i1> %extract1, <4 x i64> %5, <4 x i64> zeroinitializer
-  %res3 = add <4 x i64> %3, %4
-  %res4 = add <4 x i64> %7, %res3
-  ret <4 x i64> %res4
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %6 = select <4 x i1> %extract1, <4 x i64> %4, <4 x i64> zeroinitializer
+  %res3 = add <4 x i64> %3, %6
+  ret <4 x i64> %res3
 }
 
 define <2 x i64> @test_int_x86_avx512_mask_vpshldv_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64>* %x2p, <2 x i64> %x4, i8 %x3) {
@@ -1421,10 +1349,7 @@ define <2 x i64> @test_int_x86_avx512_mask_vpshldv_q_128(<2 x i64> %x0, <2 x i64
 ; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpshldvq (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0x71,0x18]
-; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpshldvq %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0xf5,0x08,0x71,0xe2]
 ; X86-NEXT:    vpshldvq %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x71,0xc2]
-; X86-NEXT:    vpaddq %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1433,10 +1358,7 @@ define <2 x i64> @test_int_x86_avx512_mask_vpshldv_q_128(<2 x i64> %x0, <2 x i64
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X64-NEXT:    vpshldvq (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0x71,0x1f]
-; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X64-NEXT:    vpshldvq %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0xf5,0x08,0x71,0xe2]
 ; X64-NEXT:    vpshldvq %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x71,0xc2]
-; X64-NEXT:    vpaddq %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <2 x i64>, <2 x i64>* %x2p
@@ -1445,13 +1367,11 @@ define <2 x i64> @test_int_x86_avx512_mask_vpshldv_q_128(<2 x i64> %x0, <2 x i64
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
   %3 = select <2 x i1> %extract, <2 x i64> %1, <2 x i64> %x0
   %4 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x4)
-  %5 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x4)
-  %6 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %6, <8 x i1> %6, <2 x i32> <i32 0, i32 1>
-  %7 = select <2 x i1> %extract1, <2 x i64> %5, <2 x i64> zeroinitializer
-  %res3 = add <2 x i64> %3, %4
-  %res4 = add <2 x i64> %7, %res3
-  ret <2 x i64> %res4
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %5, <8 x i1> %5, <2 x i32> <i32 0, i32 1>
+  %6 = select <2 x i1> %extract1, <2 x i64> %4, <2 x i64> zeroinitializer
+  %res3 = add <2 x i64> %3, %6
+  ret <2 x i64> %res3
 }
 
 define <16 x i16> @test_int_x86_avx512_mask_vpshldv_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16>* %x2p, <16 x i16> %x4, i16 %x3) {
@@ -1461,10 +1381,7 @@ define <16 x i16> @test_int_x86_avx512_mask_vpshldv_w_256(<16 x i16> %x0, <16 x
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpshldvw (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0x70,0x18]
-; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpshldvw %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0xf5,0x28,0x70,0xe2]
 ; X86-NEXT:    vpshldvw %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x70,0xc2]
-; X86-NEXT:    vpaddw %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfd,0xc0]
 ; X86-NEXT:    vpaddw %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1473,10 +1390,7 @@ define <16 x i16> @test_int_x86_avx512_mask_vpshldv_w_256(<16 x i16> %x0, <16 x
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X64-NEXT:    vpshldvw (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0x70,0x1f]
-; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X64-NEXT:    vpshldvw %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0xf5,0x28,0x70,0xe2]
 ; X64-NEXT:    vpshldvw %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x70,0xc2]
-; X64-NEXT:    vpaddw %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfd,0xc0]
 ; X64-NEXT:    vpaddw %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <16 x i16>, <16 x i16>* %x2p
@@ -1484,12 +1398,10 @@ define <16 x i16> @test_int_x86_avx512_mask_vpshldv_w_256(<16 x i16> %x0, <16 x
   %2 = bitcast i16 %x3 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %x0
   %4 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x4)
-  %5 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x4)
-  %6 = bitcast i16 %x3 to <16 x i1>
-  %7 = select <16 x i1> %6, <16 x i16> %5, <16 x i16> zeroinitializer
-  %res3 = add <16 x i16> %3, %4
-  %res4 = add <16 x i16> %7, %res3
-  ret <16 x i16> %res4
+  %5 = bitcast i16 %x3 to <16 x i1>
+  %6 = select <16 x i1> %5, <16 x i16> %4, <16 x i16> zeroinitializer
+  %res3 = add <16 x i16> %3, %6
+  ret <16 x i16> %res3
 }
 
 define <8 x i16> @test_int_x86_avx512_mask_vpshldv_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16>* %x2p, <8 x i16> %x4, i8 %x3) {
@@ -1500,10 +1412,7 @@ define <8 x i16> @test_int_x86_avx512_mask_vpshldv_w_128(<8 x i16> %x0, <8 x i16
 ; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpshldvw (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0x70,0x18]
-; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpshldvw %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0xf5,0x08,0x70,0xe2]
-; X86-NEXT:    vpshldvw %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x70,0xc2]
-; X86-NEXT:    vpaddw %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfd,0xc0]
+; X86-NEXT:    vpshldvw %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf2,0xf5,0x08,0x70,0xc2]
 ; X86-NEXT:    vpaddw %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1512,10 +1421,7 @@ define <8 x i16> @test_int_x86_avx512_mask_vpshldv_w_128(<8 x i16> %x0, <8 x i16
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
 ; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X64-NEXT:    vpshldvw (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0x70,0x1f]
-; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X64-NEXT:    vpshldvw %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0xf5,0x08,0x70,0xe2]
-; X64-NEXT:    vpshldvw %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x70,0xc2]
-; X64-NEXT:    vpaddw %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfd,0xc0]
+; X64-NEXT:    vpshldvw %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf2,0xf5,0x08,0x70,0xc2]
 ; X64-NEXT:    vpaddw %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <8 x i16>, <8 x i16>* %x2p
@@ -1523,12 +1429,10 @@ define <8 x i16> @test_int_x86_avx512_mask_vpshldv_w_128(<8 x i16> %x0, <8 x i16
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %x0
   %4 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x4)
-  %5 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x4)
-  %6 = bitcast i8 %x3 to <8 x i1>
-  %7 = select <8 x i1> %6, <8 x i16> %5, <8 x i16> zeroinitializer
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = select <8 x i1> %5, <8 x i16> %4, <8 x i16> zeroinitializer
   %res3 = add <8 x i16> %3, %4
-  %res4 = add <8 x i16> %7, %res3
-  ret <8 x i16> %res4
+  ret <8 x i16> %res3
 }
 
 declare <4 x i32> @llvm.fshl.v4i32(<4 x i32>, <4 x i32>, <4 x i32>)
diff --git a/llvm/test/CodeGen/X86/avx512vl-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/avx512vl-intrinsics-fast-isel.ll
index 3386e4043da7b..0e48424a2d255 100644
--- a/llvm/test/CodeGen/X86/avx512vl-intrinsics-fast-isel.ll
+++ b/llvm/test/CodeGen/X86/avx512vl-intrinsics-fast-isel.ll
@@ -3236,7 +3236,7 @@ entry:
 define <2 x i64> @test_mm_cvtepi32_epi8(<2 x i64> %__A) {
 ; CHECK-LABEL: test_mm_cvtepi32_epi8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT:    vpmovdb %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %0 = bitcast <2 x i64> %__A to <4 x i32>
@@ -3249,7 +3249,7 @@ entry:
 define <2 x i64> @test_mm_cvtepi32_epi16(<2 x i64> %__A) {
 ; CHECK-LABEL: test_mm_cvtepi32_epi16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT:    vpmovdw %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %0 = bitcast <2 x i64> %__A to <4 x i32>
@@ -3262,7 +3262,7 @@ entry:
 define <2 x i64> @test_mm_cvtepi64_epi8(<2 x i64> %__A) {
 ; CHECK-LABEL: test_mm_cvtepi64_epi8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT:    vpmovqb %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %conv.i = trunc <2 x i64> %__A to <2 x i8>
@@ -3274,7 +3274,7 @@ entry:
 define <2 x i64> @test_mm_cvtepi64_epi16(<2 x i64> %__A) {
 ; CHECK-LABEL: test_mm_cvtepi64_epi16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT:    vpmovqw %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %conv.i = trunc <2 x i64> %__A to <2 x i16>
@@ -3286,7 +3286,7 @@ entry:
 define <2 x i64> @test_mm_cvtepi64_epi32(<2 x i64> %__A) {
 ; CHECK-LABEL: test_mm_cvtepi64_epi32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,2],zero,zero
+; CHECK-NEXT:    vpmovqd %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %conv.i = trunc <2 x i64> %__A to <2 x i32>
diff --git a/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll
index 3e9ad09b8d587..352a7d6722d1e 100644
--- a/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll
@@ -548,151 +548,212 @@ define <4 x float> @test_x86_maskz_vbroadcast_ss_ps_128(<4 x float> %a0, i8 %mas
 
 declare <4 x float> @llvm.x86.avx512.mask.movsldup.128(<4 x float>, <4 x float>, i8)
 
+define <4 x float>@test_int_x86_avx512_movsldup_128(<4 x float> %x0, <4 x float> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_movsldup_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovsldup %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xfa,0x12,0xc0]
+; CHECK-NEXT:    # xmm0 = xmm0[0,0,2,2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x float> @llvm.x86.avx512.mask.movsldup.128(<4 x float> %x0, <4 x float> %x1, i8 -1)
+  ret <4 x float> %res
+}
+
 define <4 x float>@test_int_x86_avx512_mask_movsldup_128(<4 x float> %x0, <4 x float> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_movsldup_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovsldup %xmm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xfa,0x12,0xd0]
-; X86-NEXT:    # xmm2 = xmm0[0,0,2,2]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vmovsldup %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x7e,0x09,0x12,0xc8]
 ; X86-NEXT:    # xmm1 {%k1} = xmm0[0,0,2,2]
-; X86-NEXT:    vaddps %xmm2, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf0,0x58,0xca]
-; X86-NEXT:    vmovsldup %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7e,0x89,0x12,0xc0]
-; X86-NEXT:    # xmm0 {%k1} {z} = xmm0[0,0,2,2]
-; X86-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x58,0xc1]
+; X86-NEXT:    vmovaps %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_movsldup_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovsldup %xmm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xfa,0x12,0xd0]
-; X64-NEXT:    # xmm2 = xmm0[0,0,2,2]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vmovsldup %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x7e,0x09,0x12,0xc8]
 ; X64-NEXT:    # xmm1 {%k1} = xmm0[0,0,2,2]
-; X64-NEXT:    vaddps %xmm2, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf0,0x58,0xca]
+; X64-NEXT:    vmovaps %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x float> @llvm.x86.avx512.mask.movsldup.128(<4 x float> %x0, <4 x float> %x1, i8 %x2)
+  ret <4 x float> %res
+}
+
+define <4 x float>@test_int_x86_avx512_maskz_movsldup_128(<4 x float> %x0, i8 %x2) {
+; X86-LABEL: test_int_x86_avx512_maskz_movsldup_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vmovsldup %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7e,0x89,0x12,0xc0]
+; X86-NEXT:    # xmm0 {%k1} {z} = xmm0[0,0,2,2]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_movsldup_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vmovsldup %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7e,0x89,0x12,0xc0]
 ; X64-NEXT:    # xmm0 {%k1} {z} = xmm0[0,0,2,2]
-; X64-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x58,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x float> @llvm.x86.avx512.mask.movsldup.128(<4 x float> %x0, <4 x float> %x1, i8 %x2)
-  %res1 = call <4 x float> @llvm.x86.avx512.mask.movsldup.128(<4 x float> %x0, <4 x float> %x1, i8 -1)
-  %res2 = call <4 x float> @llvm.x86.avx512.mask.movsldup.128(<4 x float> %x0, <4 x float> zeroinitializer, i8 %x2)
-  %res3 = fadd <4 x float> %res, %res1
-  %res4 = fadd <4 x float> %res2, %res3
-  ret <4 x float> %res4
+  %res = call <4 x float> @llvm.x86.avx512.mask.movsldup.128(<4 x float> %x0, <4 x float> zeroinitializer, i8 %x2)
+  ret <4 x float> %res
 }
 
 declare <8 x float> @llvm.x86.avx512.mask.movsldup.256(<8 x float>, <8 x float>, i8)
 
+define <8 x float>@test_int_x86_avx512_movsldup_256(<8 x float> %x0, <8 x float> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_movsldup_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovsldup %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfe,0x12,0xc0]
+; CHECK-NEXT:    # ymm0 = ymm0[0,0,2,2,4,4,6,6]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x float> @llvm.x86.avx512.mask.movsldup.256(<8 x float> %x0, <8 x float> %x1, i8 -1)
+  ret <8 x float> %res
+}
+
 define <8 x float>@test_int_x86_avx512_mask_movsldup_256(<8 x float> %x0, <8 x float> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_movsldup_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovsldup %ymm0, %ymm2 # EVEX TO VEX Compression encoding: [0xc5,0xfe,0x12,0xd0]
-; X86-NEXT:    # ymm2 = ymm0[0,0,2,2,4,4,6,6]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vmovsldup %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x7e,0x29,0x12,0xc8]
 ; X86-NEXT:    # ymm1 {%k1} = ymm0[0,0,2,2,4,4,6,6]
-; X86-NEXT:    vaddps %ymm2, %ymm1, %ymm1 # EVEX TO VEX Compression encoding: [0xc5,0xf4,0x58,0xca]
-; X86-NEXT:    vmovsldup %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7e,0xa9,0x12,0xc0]
-; X86-NEXT:    # ymm0 {%k1} {z} = ymm0[0,0,2,2,4,4,6,6]
-; X86-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x58,0xc1]
+; X86-NEXT:    vmovaps %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_movsldup_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovsldup %ymm0, %ymm2 # EVEX TO VEX Compression encoding: [0xc5,0xfe,0x12,0xd0]
-; X64-NEXT:    # ymm2 = ymm0[0,0,2,2,4,4,6,6]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vmovsldup %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x7e,0x29,0x12,0xc8]
 ; X64-NEXT:    # ymm1 {%k1} = ymm0[0,0,2,2,4,4,6,6]
-; X64-NEXT:    vaddps %ymm2, %ymm1, %ymm1 # EVEX TO VEX Compression encoding: [0xc5,0xf4,0x58,0xca]
+; X64-NEXT:    vmovaps %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x float> @llvm.x86.avx512.mask.movsldup.256(<8 x float> %x0, <8 x float> %x1, i8 %x2)
+  ret <8 x float> %res
+}
+
+define <8 x float>@test_int_x86_avx512_maskz_movsldup_256(<8 x float> %x0, i8 %x2) {
+; X86-LABEL: test_int_x86_avx512_maskz_movsldup_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vmovsldup %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7e,0xa9,0x12,0xc0]
+; X86-NEXT:    # ymm0 {%k1} {z} = ymm0[0,0,2,2,4,4,6,6]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_movsldup_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vmovsldup %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7e,0xa9,0x12,0xc0]
 ; X64-NEXT:    # ymm0 {%k1} {z} = ymm0[0,0,2,2,4,4,6,6]
-; X64-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x58,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x float> @llvm.x86.avx512.mask.movsldup.256(<8 x float> %x0, <8 x float> %x1, i8 %x2)
-  %res1 = call <8 x float> @llvm.x86.avx512.mask.movsldup.256(<8 x float> %x0, <8 x float> %x1, i8 -1)
-  %res2 = call <8 x float> @llvm.x86.avx512.mask.movsldup.256(<8 x float> %x0, <8 x float> zeroinitializer, i8 %x2)
-  %res3 = fadd <8 x float> %res, %res1
-  %res4 = fadd <8 x float> %res2, %res3
-  ret <8 x float> %res4
+  %res = call <8 x float> @llvm.x86.avx512.mask.movsldup.256(<8 x float> %x0, <8 x float> zeroinitializer, i8 %x2)
+  ret <8 x float> %res
 }
 
 declare <4 x float> @llvm.x86.avx512.mask.movshdup.128(<4 x float>, <4 x float>, i8)
 
+define <4 x float>@test_int_x86_avx512_movshdup_128(<4 x float> %x0, <4 x float> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_movshdup_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovshdup %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xfa,0x16,0xc0]
+; CHECK-NEXT:    # xmm0 = xmm0[1,1,3,3]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x float> @llvm.x86.avx512.mask.movshdup.128(<4 x float> %x0, <4 x float> %x1, i8 -1)
+  ret <4 x float> %res
+}
+
 define <4 x float>@test_int_x86_avx512_mask_movshdup_128(<4 x float> %x0, <4 x float> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_movshdup_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovshdup %xmm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xfa,0x16,0xd0]
-; X86-NEXT:    # xmm2 = xmm0[1,1,3,3]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vmovshdup %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x7e,0x09,0x16,0xc8]
 ; X86-NEXT:    # xmm1 {%k1} = xmm0[1,1,3,3]
-; X86-NEXT:    vaddps %xmm2, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf0,0x58,0xca]
-; X86-NEXT:    vmovshdup %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7e,0x89,0x16,0xc0]
-; X86-NEXT:    # xmm0 {%k1} {z} = xmm0[1,1,3,3]
-; X86-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x58,0xc1]
+; X86-NEXT:    vmovaps %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_movshdup_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovshdup %xmm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xfa,0x16,0xd0]
-; X64-NEXT:    # xmm2 = xmm0[1,1,3,3]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vmovshdup %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x7e,0x09,0x16,0xc8]
 ; X64-NEXT:    # xmm1 {%k1} = xmm0[1,1,3,3]
-; X64-NEXT:    vaddps %xmm2, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf0,0x58,0xca]
+; X64-NEXT:    vmovaps %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x float> @llvm.x86.avx512.mask.movshdup.128(<4 x float> %x0, <4 x float> %x1, i8 %x2)
+  ret <4 x float> %res
+}
+
+define <4 x float>@test_int_x86_avx512_maskz_movshdup_128(<4 x float> %x0, i8 %x2) {
+; X86-LABEL: test_int_x86_avx512_maskz_movshdup_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vmovshdup %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7e,0x89,0x16,0xc0]
+; X86-NEXT:    # xmm0 {%k1} {z} = xmm0[1,1,3,3]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_movshdup_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vmovshdup %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7e,0x89,0x16,0xc0]
 ; X64-NEXT:    # xmm0 {%k1} {z} = xmm0[1,1,3,3]
-; X64-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x58,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x float> @llvm.x86.avx512.mask.movshdup.128(<4 x float> %x0, <4 x float> %x1, i8 %x2)
-  %res1 = call <4 x float> @llvm.x86.avx512.mask.movshdup.128(<4 x float> %x0, <4 x float> %x1, i8 -1)
-  %res2 = call <4 x float> @llvm.x86.avx512.mask.movshdup.128(<4 x float> %x0, <4 x float> zeroinitializer, i8 %x2)
-  %res3 = fadd <4 x float> %res, %res1
-  %res4 = fadd <4 x float> %res2, %res3
-  ret <4 x float> %res4
+  %res = call <4 x float> @llvm.x86.avx512.mask.movshdup.128(<4 x float> %x0, <4 x float> zeroinitializer, i8 %x2)
+  ret <4 x float> %res
 }
 
 declare <8 x float> @llvm.x86.avx512.mask.movshdup.256(<8 x float>, <8 x float>, i8)
 
+define <8 x float>@test_int_x86_avx512_movshdup_256(<8 x float> %x0, <8 x float> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_movshdup_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovshdup %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfe,0x16,0xc0]
+; CHECK-NEXT:    # ymm0 = ymm0[1,1,3,3,5,5,7,7]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x float> @llvm.x86.avx512.mask.movshdup.256(<8 x float> %x0, <8 x float> %x1, i8 -1)
+  ret <8 x float> %res
+}
+
 define <8 x float>@test_int_x86_avx512_mask_movshdup_256(<8 x float> %x0, <8 x float> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_movshdup_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovshdup %ymm0, %ymm2 # EVEX TO VEX Compression encoding: [0xc5,0xfe,0x16,0xd0]
-; X86-NEXT:    # ymm2 = ymm0[1,1,3,3,5,5,7,7]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vmovshdup %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x7e,0x29,0x16,0xc8]
 ; X86-NEXT:    # ymm1 {%k1} = ymm0[1,1,3,3,5,5,7,7]
-; X86-NEXT:    vaddps %ymm2, %ymm1, %ymm1 # EVEX TO VEX Compression encoding: [0xc5,0xf4,0x58,0xca]
-; X86-NEXT:    vmovshdup %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7e,0xa9,0x16,0xc0]
-; X86-NEXT:    # ymm0 {%k1} {z} = ymm0[1,1,3,3,5,5,7,7]
-; X86-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x58,0xc1]
+; X86-NEXT:    vmovaps %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_movshdup_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovshdup %ymm0, %ymm2 # EVEX TO VEX Compression encoding: [0xc5,0xfe,0x16,0xd0]
-; X64-NEXT:    # ymm2 = ymm0[1,1,3,3,5,5,7,7]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vmovshdup %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x7e,0x29,0x16,0xc8]
 ; X64-NEXT:    # ymm1 {%k1} = ymm0[1,1,3,3,5,5,7,7]
-; X64-NEXT:    vaddps %ymm2, %ymm1, %ymm1 # EVEX TO VEX Compression encoding: [0xc5,0xf4,0x58,0xca]
+; X64-NEXT:    vmovaps %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x float> @llvm.x86.avx512.mask.movshdup.256(<8 x float> %x0, <8 x float> %x1, i8 %x2)
+  ret <8 x float> %res
+}
+
+define <8 x float>@test_int_x86_avx512_maskz_movshdup_256(<8 x float> %x0, i8 %x2) {
+; X86-LABEL: test_int_x86_avx512_maskz_movshdup_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vmovshdup %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7e,0xa9,0x16,0xc0]
+; X86-NEXT:    # ymm0 {%k1} {z} = ymm0[1,1,3,3,5,5,7,7]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_movshdup_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vmovshdup %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7e,0xa9,0x16,0xc0]
 ; X64-NEXT:    # ymm0 {%k1} {z} = ymm0[1,1,3,3,5,5,7,7]
-; X64-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x58,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x float> @llvm.x86.avx512.mask.movshdup.256(<8 x float> %x0, <8 x float> %x1, i8 %x2)
-  %res1 = call <8 x float> @llvm.x86.avx512.mask.movshdup.256(<8 x float> %x0, <8 x float> %x1, i8 -1)
-  %res2 = call <8 x float> @llvm.x86.avx512.mask.movshdup.256(<8 x float> %x0, <8 x float> zeroinitializer, i8 %x2)
-  %res3 = fadd <8 x float> %res, %res1
-  %res4 = fadd <8 x float> %res2, %res3
-  ret <8 x float> %res4
+  %res = call <8 x float> @llvm.x86.avx512.mask.movshdup.256(<8 x float> %x0, <8 x float> zeroinitializer, i8 %x2)
+  ret <8 x float> %res
 }
+
 declare <2 x double> @llvm.x86.avx512.mask.movddup.128(<2 x double>, <2 x double>, i8)
 
 define <2 x double>@test_int_x86_avx512_movddup_128(<2 x double> %x0, <2 x double> %x1) {
@@ -6441,281 +6502,425 @@ define <4 x i64>@test_int_x86_avx512_maskz_pminu_q_256(<4 x i64> %x0, <4 x i64>
 
 declare <2 x i64> @llvm.x86.avx512.mask.psrl.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
 
+define <2 x i64>@test_int_x86_avx512_psrl_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psrl_q_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd3,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.psrl.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1)
+  ret <2 x i64> %res
+}
+
 define <2 x i64>@test_int_x86_avx512_mask_psrl_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrl_q_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsrlq %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd3,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpsrlq %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0xfd,0x09,0xd3,0xd1]
-; X86-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0xd3,0xc1]
-; X86-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
-; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrl_q_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsrlq %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd3,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpsrlq %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0xfd,0x09,0xd3,0xd1]
-; X64-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0xd3,0xc1]
-; X64-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
-; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx512.mask.psrl.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3)
-  %res1 = call <2 x i64> @llvm.x86.avx512.mask.psrl.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1)
-  %res2 = call <2 x i64> @llvm.x86.avx512.mask.psrl.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> zeroinitializer, i8 %x3)
-  %res3 = add <2 x i64> %res, %res1
-  %res4 = add <2 x i64> %res3, %res2
-  ret <2 x i64> %res4
+  ret <2 x i64> %res
+}
+
+define <2 x i64>@test_int_x86_avx512_maskz_psrl_q_128(<2 x i64> %x0, <2 x i64> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psrl_q_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0xd3,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psrl_q_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0xd3,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.psrl.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> zeroinitializer, i8 %x3)
+  ret <2 x i64> %res
 }
 
 declare <4 x i64> @llvm.x86.avx512.mask.psrl.q.256(<4 x i64>, <2 x i64>, <4 x i64>, i8)
 
+define <4 x i64>@test_int_x86_avx512_psrl_q_256(<4 x i64> %x0, <2 x i64> %x1, <4 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psrl_q_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd3,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.psrl.q.256(<4 x i64> %x0, <2 x i64> %x1, <4 x i64> %x2, i8 -1)
+  ret <4 x i64> %res
+}
+
 define <4 x i64>@test_int_x86_avx512_mask_psrl_q_256(<4 x i64> %x0, <2 x i64> %x1, <4 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrl_q_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsrlq %xmm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd3,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpsrlq %xmm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0xfd,0x29,0xd3,0xd1]
-; X86-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0xd3,0xc1]
-; X86-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
-; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrl_q_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsrlq %xmm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd3,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpsrlq %xmm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0xfd,0x29,0xd3,0xd1]
-; X64-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0xd3,0xc1]
-; X64-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
-; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx512.mask.psrl.q.256(<4 x i64> %x0, <2 x i64> %x1, <4 x i64> %x2, i8 %x3)
-  %res1 = call <4 x i64> @llvm.x86.avx512.mask.psrl.q.256(<4 x i64> %x0, <2 x i64> %x1, <4 x i64> %x2, i8 -1)
-  %res2 = call <4 x i64> @llvm.x86.avx512.mask.psrl.q.256(<4 x i64> %x0, <2 x i64> %x1, <4 x i64> zeroinitializer, i8 %x3)
-  %res3 = add <4 x i64> %res, %res1
-  %res4 = add <4 x i64> %res3, %res2
-  ret <4 x i64> %res4
+  ret <4 x i64> %res
+}
+
+define <4 x i64>@test_int_x86_avx512_maskz_psrl_q_256(<4 x i64> %x0, <2 x i64> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psrl_q_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0xd3,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psrl_q_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0xd3,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.psrl.q.256(<4 x i64> %x0, <2 x i64> %x1, <4 x i64> zeroinitializer, i8 %x3)
+  ret <4 x i64> %res
 }
 
 declare <4 x i32> @llvm.x86.avx512.mask.psrl.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
 
+define <4 x i32>@test_int_x86_avx512_psrl_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psrl_d_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsrld %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd2,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.psrl.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1)
+  ret <4 x i32> %res
+}
+
 define <4 x i32>@test_int_x86_avx512_mask_psrl_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrl_d_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsrld %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd2,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpsrld %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xd2,0xd1]
-; X86-NEXT:    vpsrld %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xd2,0xc1]
-; X86-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
-; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrl_d_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsrld %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd2,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpsrld %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xd2,0xd1]
-; X64-NEXT:    vpsrld %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xd2,0xc1]
-; X64-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
-; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx512.mask.psrl.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3)
-  %res1 = call <4 x i32> @llvm.x86.avx512.mask.psrl.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1)
-  %res2 = call <4 x i32> @llvm.x86.avx512.mask.psrl.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> zeroinitializer, i8 %x3)
-  %res3 = add <4 x i32> %res, %res1
-  %res4 = add <4 x i32> %res3, %res2
-  ret <4 x i32> %res4
+  ret <4 x i32> %res
+}
+
+define <4 x i32>@test_int_x86_avx512_maskz_psrl_d_128(<4 x i32> %x0, <4 x i32> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psrl_d_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpsrld %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xd2,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psrl_d_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpsrld %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xd2,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.psrl.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> zeroinitializer, i8 %x3)
+  ret <4 x i32> %res
 }
 
 declare <8 x i32> @llvm.x86.avx512.mask.psrl.d.256(<8 x i32>, <4 x i32>, <8 x i32>, i8)
 
+define <8 x i32>@test_int_x86_avx512_psrl_d_256(<8 x i32> %x0, <4 x i32> %x1, <8 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psrl_d_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsrld %xmm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd2,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.psrl.d.256(<8 x i32> %x0, <4 x i32> %x1, <8 x i32> %x2, i8 -1)
+  ret <8 x i32> %res
+}
+
 define <8 x i32>@test_int_x86_avx512_mask_psrl_d_256(<8 x i32> %x0, <4 x i32> %x1, <8 x i32> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrl_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsrld %xmm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd2,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpsrld %xmm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xd2,0xd1]
-; X86-NEXT:    vpsrld %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xd2,0xc1]
-; X86-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
-; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrl_d_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsrld %xmm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd2,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpsrld %xmm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xd2,0xd1]
-; X64-NEXT:    vpsrld %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xd2,0xc1]
-; X64-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
-; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx512.mask.psrl.d.256(<8 x i32> %x0, <4 x i32> %x1, <8 x i32> %x2, i8 %x3)
-  %res1 = call <8 x i32> @llvm.x86.avx512.mask.psrl.d.256(<8 x i32> %x0, <4 x i32> %x1, <8 x i32> %x2, i8 -1)
-  %res2 = call <8 x i32> @llvm.x86.avx512.mask.psrl.d.256(<8 x i32> %x0, <4 x i32> %x1, <8 x i32> zeroinitializer, i8 %x3)
-  %res3 = add <8 x i32> %res, %res1
-  %res4 = add <8 x i32> %res2, %res3
-  ret <8 x i32> %res4
+  ret <8 x i32> %res
+}
+
+define <8 x i32>@test_int_x86_avx512_maskz_psrl_d_256(<8 x i32> %x0, <4 x i32> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psrl_d_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpsrld %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xd2,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psrl_d_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpsrld %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xd2,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.psrl.d.256(<8 x i32> %x0, <4 x i32> %x1, <8 x i32> zeroinitializer, i8 %x3)
+  ret <8 x i32> %res
 }
 
 declare <4 x i32> @llvm.x86.avx512.mask.psra.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
 
+define <4 x i32>@test_int_x86_avx512_ask_psra_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_ask_psra_d_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsrad %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe2,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.psra.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1)
+  ret <4 x i32> %res
+}
+
 define <4 x i32>@test_int_x86_avx512_mask_psra_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psra_d_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsrad %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe2,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpsrad %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe2,0xd1]
-; X86-NEXT:    vpsrad %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe2,0xc1]
-; X86-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
-; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psra_d_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsrad %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe2,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpsrad %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe2,0xd1]
-; X64-NEXT:    vpsrad %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe2,0xc1]
-; X64-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
-; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx512.mask.psra.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3)
-  %res1 = call <4 x i32> @llvm.x86.avx512.mask.psra.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> zeroinitializer, i8 %x3)
-  %res2 = call <4 x i32> @llvm.x86.avx512.mask.psra.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1)
-  %res3 = add <4 x i32> %res, %res1
-  %res4 = add <4 x i32> %res3, %res2
-  ret <4 x i32> %res4
+  ret <4 x i32> %res
+}
+
+define <4 x i32>@test_int_x86_avx512_maskz_psra_d_128(<4 x i32> %x0, <4 x i32> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psra_d_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpsrad %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe2,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psra_d_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpsrad %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe2,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.psra.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> zeroinitializer, i8 %x3)
+  ret <4 x i32> %res
 }
 
 declare <8 x i32> @llvm.x86.avx512.mask.psra.d.256(<8 x i32>, <4 x i32>, <8 x i32>, i8)
 
+define <8 x i32>@test_int_x86_avx512_psra_d_256(<8 x i32> %x0, <4 x i32> %x1, <8 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psra_d_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsrad %xmm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe2,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.psra.d.256(<8 x i32> %x0, <4 x i32> %x1, <8 x i32> %x2, i8 -1)
+  ret <8 x i32> %res
+}
+
 define <8 x i32>@test_int_x86_avx512_mask_psra_d_256(<8 x i32> %x0, <4 x i32> %x1, <8 x i32> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psra_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsrad %xmm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe2,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpsrad %xmm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe2,0xd1]
-; X86-NEXT:    vpsrad %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe2,0xc1]
-; X86-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
-; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psra_d_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsrad %xmm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe2,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpsrad %xmm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe2,0xd1]
-; X64-NEXT:    vpsrad %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe2,0xc1]
-; X64-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
-; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx512.mask.psra.d.256(<8 x i32> %x0, <4 x i32> %x1, <8 x i32> %x2, i8 %x3)
-  %res1 = call <8 x i32> @llvm.x86.avx512.mask.psra.d.256(<8 x i32> %x0, <4 x i32> %x1, <8 x i32> zeroinitializer, i8 %x3)
-  %res2 = call <8 x i32> @llvm.x86.avx512.mask.psra.d.256(<8 x i32> %x0, <4 x i32> %x1, <8 x i32> %x2, i8 -1)
-  %res3 = add <8 x i32> %res, %res1
-  %res4 = add <8 x i32> %res3, %res2
-  ret <8 x i32> %res4
+  ret <8 x i32> %res
+}
+
+define <8 x i32>@test_int_x86_avx512_maskz_psra_d_256(<8 x i32> %x0, <4 x i32> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psra_d_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpsrad %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe2,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psra_d_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpsrad %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe2,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.psra.d.256(<8 x i32> %x0, <4 x i32> %x1, <8 x i32> zeroinitializer, i8 %x3)
+  ret <8 x i32> %res
 }
 
 declare <4 x i32> @llvm.x86.avx512.mask.psll.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
 
+define <4 x i32>@test_int_x86_avx512_psll_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psll_d_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpslld %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xf2,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.psll.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1)
+  ret <4 x i32> %res
+}
+
 define <4 x i32>@test_int_x86_avx512_mask_psll_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psll_d_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpslld %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xf2,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpslld %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xf2,0xd1]
-; X86-NEXT:    vpslld %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xf2,0xc1]
-; X86-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
-; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psll_d_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpslld %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xf2,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpslld %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xf2,0xd1]
-; X64-NEXT:    vpslld %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xf2,0xc1]
-; X64-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
-; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx512.mask.psll.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3)
-  %res1 = call <4 x i32> @llvm.x86.avx512.mask.psll.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> zeroinitializer, i8 %x3)
-  %res2 = call <4 x i32> @llvm.x86.avx512.mask.psll.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1)
-  %res3 = add <4 x i32> %res, %res1
-  %res4 = add <4 x i32> %res3, %res2
-  ret <4 x i32> %res4
+  ret <4 x i32> %res
+}
+
+define <4 x i32>@test_int_x86_avx512_maskz_psll_d_128(<4 x i32> %x0, <4 x i32> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psll_d_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpslld %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xf2,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psll_d_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpslld %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xf2,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.psll.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> zeroinitializer, i8 %x3)
+  ret <4 x i32> %res
 }
 
 declare <8 x i32> @llvm.x86.avx512.mask.psll.d.256(<8 x i32>, <4 x i32>, <8 x i32>, i8)
 
+define <8 x i32>@test_int_x86_avx512_psll_d_256(<8 x i32> %x0, <4 x i32> %x1, <8 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psll_d_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpslld %xmm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xf2,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.psll.d.256(<8 x i32> %x0, <4 x i32> %x1, <8 x i32> %x2, i8 -1)
+  ret <8 x i32> %res
+}
+
 define <8 x i32>@test_int_x86_avx512_mask_psll_d_256(<8 x i32> %x0, <4 x i32> %x1, <8 x i32> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psll_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpslld %xmm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xf2,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpslld %xmm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xf2,0xd1]
-; X86-NEXT:    vpslld %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xf2,0xc1]
-; X86-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
-; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psll_d_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpslld %xmm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xf2,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpslld %xmm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xf2,0xd1]
-; X64-NEXT:    vpslld %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xf2,0xc1]
-; X64-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
-; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx512.mask.psll.d.256(<8 x i32> %x0, <4 x i32> %x1, <8 x i32> %x2, i8 %x3)
-  %res1 = call <8 x i32> @llvm.x86.avx512.mask.psll.d.256(<8 x i32> %x0, <4 x i32> %x1, <8 x i32> zeroinitializer, i8 %x3)
-  %res2 = call <8 x i32> @llvm.x86.avx512.mask.psll.d.256(<8 x i32> %x0, <4 x i32> %x1, <8 x i32> %x2, i8 -1)
-  %res3 = add <8 x i32> %res, %res1
-  %res4 = add <8 x i32> %res3, %res2
-  ret <8 x i32> %res4
+  ret <8 x i32> %res
+}
+
+define <8 x i32>@test_int_x86_avx512_maskz_psll_d_256(<8 x i32> %x0, <4 x i32> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psll_d_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpslld %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xf2,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psll_d_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpslld %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xf2,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.psll.d.256(<8 x i32> %x0, <4 x i32> %x1, <8 x i32> zeroinitializer, i8 %x3)
+  ret <8 x i32> %res
 }
 
 declare <4 x i64> @llvm.x86.avx512.mask.psll.q.256(<4 x i64>, <2 x i64>, <4 x i64>, i8)
 
+define <4 x i64>@test_int_x86_avx512_psll_q_256(<4 x i64> %x0, <2 x i64> %x1, <4 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psll_q_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsllq %xmm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xf3,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.psll.q.256(<4 x i64> %x0, <2 x i64> %x1, <4 x i64> %x2, i8 -1)
+  ret <4 x i64> %res
+}
+
 define <4 x i64>@test_int_x86_avx512_mask_psll_q_256(<4 x i64> %x0, <2 x i64> %x1, <4 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psll_q_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsllq %xmm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xf3,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpsllq %xmm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0xfd,0x29,0xf3,0xd1]
-; X86-NEXT:    vpsllq %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0xf3,0xc1]
-; X86-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
-; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psll_q_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsllq %xmm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xf3,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpsllq %xmm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0xfd,0x29,0xf3,0xd1]
-; X64-NEXT:    vpsllq %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0xf3,0xc1]
-; X64-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
-; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx512.mask.psll.q.256(<4 x i64> %x0, <2 x i64> %x1, <4 x i64> %x2, i8 %x3)
-  %res1 = call <4 x i64> @llvm.x86.avx512.mask.psll.q.256(<4 x i64> %x0, <2 x i64> %x1, <4 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <4 x i64> @llvm.x86.avx512.mask.psll.q.256(<4 x i64> %x0, <2 x i64> %x1, <4 x i64> %x2, i8 -1)
-  %res3 = add <4 x i64> %res, %res1
-  %res4 = add <4 x i64> %res3, %res2
-  ret <4 x i64> %res4
+  ret <4 x i64> %res
+}
+
+define <4 x i64>@test_int_x86_avx512_maskz_psll_q_256(<4 x i64> %x0, <2 x i64> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psll_q_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpsllq %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0xf3,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psll_q_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpsllq %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0xf3,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.psll.q.256(<4 x i64> %x0, <2 x i64> %x1, <4 x i64> zeroinitializer, i8 %x3)
+  ret <4 x i64> %res
 }
 
 declare <2 x i64> @llvm.x86.avx512.mask.psrl.qi.128(<2 x i64>, i32, <2 x i64>, i8)
@@ -6723,27 +6928,27 @@ declare <2 x i64> @llvm.x86.avx512.mask.psrl.qi.128(<2 x i64>, i32, <2 x i64>, i
 define <2 x i64>@test_int_x86_avx512_mask_psrl_qi_128(<2 x i64> %x0, i32 %x1, <2 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrl_qi_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsrlq $3, %xmm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0x73,0xd0,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpsrlq $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x09,0x73,0xd0,0x03]
-; X86-NEXT:    vpsrlq $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0x73,0xd0,0x03]
+; X86-NEXT:    vpsrlq $4, %xmm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0x73,0xd0,0x04]
+; X86-NEXT:    vpsrlq $5, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0x73,0xd0,0x05]
 ; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrl_qi_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsrlq $3, %xmm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0x73,0xd0,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vpsrlq $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x09,0x73,0xd0,0x03]
-; X64-NEXT:    vpsrlq $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0x73,0xd0,0x03]
+; X64-NEXT:    vpsrlq $4, %xmm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0x73,0xd0,0x04]
+; X64-NEXT:    vpsrlq $5, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0x73,0xd0,0x05]
 ; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx512.mask.psrl.qi.128(<2 x i64> %x0, i32 3, <2 x i64> %x2, i8 %x3)
-  %res1 = call <2 x i64> @llvm.x86.avx512.mask.psrl.qi.128(<2 x i64> %x0, i32 3, <2 x i64> %x2, i8 -1)
-  %res2 = call <2 x i64> @llvm.x86.avx512.mask.psrl.qi.128(<2 x i64> %x0, i32 3, <2 x i64> zeroinitializer, i8 %x3)
+  %res1 = call <2 x i64> @llvm.x86.avx512.mask.psrl.qi.128(<2 x i64> %x0, i32 4, <2 x i64> %x2, i8 -1)
+  %res2 = call <2 x i64> @llvm.x86.avx512.mask.psrl.qi.128(<2 x i64> %x0, i32 5, <2 x i64> zeroinitializer, i8 %x3)
   %res3 = add <2 x i64> %res, %res1
   %res4 = add <2 x i64> %res2, %res3
   ret <2 x i64> %res4
@@ -6754,27 +6959,27 @@ declare <4 x i64> @llvm.x86.avx512.mask.psrl.qi.256(<4 x i64>, i32, <4 x i64>, i
 define <4 x i64>@test_int_x86_avx512_mask_psrl_qi_256(<4 x i64> %x0, i32 %x1, <4 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrl_qi_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsrlq $3, %ymm0, %ymm2 # EVEX TO VEX Compression encoding: [0xc5,0xed,0x73,0xd0,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpsrlq $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x29,0x73,0xd0,0x03]
-; X86-NEXT:    vpsrlq $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0x73,0xd0,0x03]
+; X86-NEXT:    vpsrlq $4, %ymm0, %ymm2 # EVEX TO VEX Compression encoding: [0xc5,0xed,0x73,0xd0,0x04]
+; X86-NEXT:    vpsrlq $5, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0x73,0xd0,0x05]
 ; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrl_qi_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsrlq $3, %ymm0, %ymm2 # EVEX TO VEX Compression encoding: [0xc5,0xed,0x73,0xd0,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vpsrlq $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x29,0x73,0xd0,0x03]
-; X64-NEXT:    vpsrlq $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0x73,0xd0,0x03]
+; X64-NEXT:    vpsrlq $4, %ymm0, %ymm2 # EVEX TO VEX Compression encoding: [0xc5,0xed,0x73,0xd0,0x04]
+; X64-NEXT:    vpsrlq $5, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0x73,0xd0,0x05]
 ; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx512.mask.psrl.qi.256(<4 x i64> %x0, i32 3, <4 x i64> %x2, i8 %x3)
-  %res1 = call <4 x i64> @llvm.x86.avx512.mask.psrl.qi.256(<4 x i64> %x0, i32 3, <4 x i64> %x2, i8 -1)
-  %res2 = call <4 x i64> @llvm.x86.avx512.mask.psrl.qi.256(<4 x i64> %x0, i32 3, <4 x i64> zeroinitializer, i8 %x3)
+  %res1 = call <4 x i64> @llvm.x86.avx512.mask.psrl.qi.256(<4 x i64> %x0, i32 4, <4 x i64> %x2, i8 -1)
+  %res2 = call <4 x i64> @llvm.x86.avx512.mask.psrl.qi.256(<4 x i64> %x0, i32 5, <4 x i64> zeroinitializer, i8 %x3)
   %res3 = add <4 x i64> %res, %res1
   %res4 = add <4 x i64> %res2, %res3
   ret <4 x i64> %res4
@@ -6785,27 +6990,27 @@ declare <4 x i32> @llvm.x86.avx512.mask.psrl.di.128(<4 x i32>, i32, <4 x i32>, i
 define <4 x i32>@test_int_x86_avx512_mask_psrl_di_128(<4 x i32> %x0, i32 %x1, <4 x i32> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrl_di_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsrld $3, %xmm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0x72,0xd0,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpsrld $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x09,0x72,0xd0,0x03]
-; X86-NEXT:    vpsrld $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x72,0xd0,0x03]
+; X86-NEXT:    vpsrld $4, %xmm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0x72,0xd0,0x04]
+; X86-NEXT:    vpsrld $5, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x72,0xd0,0x05]
 ; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrl_di_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsrld $3, %xmm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0x72,0xd0,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vpsrld $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x09,0x72,0xd0,0x03]
-; X64-NEXT:    vpsrld $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x72,0xd0,0x03]
+; X64-NEXT:    vpsrld $4, %xmm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0x72,0xd0,0x04]
+; X64-NEXT:    vpsrld $5, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x72,0xd0,0x05]
 ; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx512.mask.psrl.di.128(<4 x i32> %x0, i32 3, <4 x i32> %x2, i8 %x3)
-  %res1 = call <4 x i32> @llvm.x86.avx512.mask.psrl.di.128(<4 x i32> %x0, i32 3, <4 x i32> %x2, i8 -1)
-  %res2 = call <4 x i32> @llvm.x86.avx512.mask.psrl.di.128(<4 x i32> %x0, i32 3, <4 x i32> zeroinitializer, i8 %x3)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.psrl.di.128(<4 x i32> %x0, i32 4, <4 x i32> %x2, i8 -1)
+  %res2 = call <4 x i32> @llvm.x86.avx512.mask.psrl.di.128(<4 x i32> %x0, i32 5, <4 x i32> zeroinitializer, i8 %x3)
   %res3 = add <4 x i32> %res, %res1
   %res4 = add <4 x i32> %res2, %res3
   ret <4 x i32> %res4
@@ -6816,27 +7021,27 @@ declare <8 x i32> @llvm.x86.avx512.mask.psrl.di.256(<8 x i32>, i32, <8 x i32>, i
 define <8 x i32>@test_int_x86_avx512_mask_psrl_di_256(<8 x i32> %x0, i32 %x1, <8 x i32> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrl_di_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsrld $3, %ymm0, %ymm2 # EVEX TO VEX Compression encoding: [0xc5,0xed,0x72,0xd0,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpsrld $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x75,0x29,0x72,0xd0,0x03]
-; X86-NEXT:    vpsrld $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x72,0xd0,0x03]
+; X86-NEXT:    vpsrld $4, %ymm0, %ymm2 # EVEX TO VEX Compression encoding: [0xc5,0xed,0x72,0xd0,0x04]
+; X86-NEXT:    vpsrld $5, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x72,0xd0,0x05]
 ; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrl_di_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsrld $3, %ymm0, %ymm2 # EVEX TO VEX Compression encoding: [0xc5,0xed,0x72,0xd0,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vpsrld $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x75,0x29,0x72,0xd0,0x03]
-; X64-NEXT:    vpsrld $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x72,0xd0,0x03]
+; X64-NEXT:    vpsrld $4, %ymm0, %ymm2 # EVEX TO VEX Compression encoding: [0xc5,0xed,0x72,0xd0,0x04]
+; X64-NEXT:    vpsrld $5, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x72,0xd0,0x05]
 ; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx512.mask.psrl.di.256(<8 x i32> %x0, i32 3, <8 x i32> %x2, i8 %x3)
-  %res1 = call <8 x i32> @llvm.x86.avx512.mask.psrl.di.256(<8 x i32> %x0, i32 3, <8 x i32> %x2, i8 -1)
-  %res2 = call <8 x i32> @llvm.x86.avx512.mask.psrl.di.256(<8 x i32> %x0, i32 3, <8 x i32> zeroinitializer, i8 %x3)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.psrl.di.256(<8 x i32> %x0, i32 4, <8 x i32> %x2, i8 -1)
+  %res2 = call <8 x i32> @llvm.x86.avx512.mask.psrl.di.256(<8 x i32> %x0, i32 5, <8 x i32> zeroinitializer, i8 %x3)
   %res3 = add <8 x i32> %res, %res1
   %res4 = add <8 x i32> %res2, %res3
   ret <8 x i32> %res4
@@ -6847,27 +7052,27 @@ declare <4 x i32> @llvm.x86.avx512.mask.psll.di.128(<4 x i32>, i32, <4 x i32>, i
 define <4 x i32>@test_int_x86_avx512_mask_psll_di_128(<4 x i32> %x0, i32 %x1, <4 x i32> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psll_di_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpslld $3, %xmm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0x72,0xf0,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpslld $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x09,0x72,0xf0,0x03]
-; X86-NEXT:    vpslld $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x72,0xf0,0x03]
-; X86-NEXT:    vpaddd %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc2]
+; X86-NEXT:    vpslld $4, %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0x89,0x72,0xf0,0x04]
+; X86-NEXT:    vpslld $5, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x72,0xf0,0x05]
+; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psll_di_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpslld $3, %xmm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0x72,0xf0,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vpslld $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x09,0x72,0xf0,0x03]
-; X64-NEXT:    vpslld $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x72,0xf0,0x03]
-; X64-NEXT:    vpaddd %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc2]
+; X64-NEXT:    vpslld $4, %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0x89,0x72,0xf0,0x04]
+; X64-NEXT:    vpslld $5, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x72,0xf0,0x05]
+; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx512.mask.psll.di.128(<4 x i32> %x0, i32 3, <4 x i32> %x2, i8 %x3)
-  %res1 = call <4 x i32> @llvm.x86.avx512.mask.psll.di.128(<4 x i32> %x0, i32 3, <4 x i32> zeroinitializer, i8 %x3)
-  %res2 = call <4 x i32> @llvm.x86.avx512.mask.psll.di.128(<4 x i32> %x0, i32 3, <4 x i32> %x2, i8 -1)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.psll.di.128(<4 x i32> %x0, i32 4, <4 x i32> zeroinitializer, i8 %x3)
+  %res2 = call <4 x i32> @llvm.x86.avx512.mask.psll.di.128(<4 x i32> %x0, i32 5, <4 x i32> %x2, i8 -1)
   %res3 = add <4 x i32> %res, %res1
   %res4 = add <4 x i32> %res3, %res2
   ret <4 x i32> %res4
@@ -6878,27 +7083,27 @@ declare <8 x i32> @llvm.x86.avx512.mask.psll.di.256(<8 x i32>, i32, <8 x i32>, i
 define <8 x i32>@test_int_x86_avx512_mask_psll_di_256(<8 x i32> %x0, i32 %x1, <8 x i32> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psll_di_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpslld $3, %ymm0, %ymm2 # EVEX TO VEX Compression encoding: [0xc5,0xed,0x72,0xf0,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpslld $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x75,0x29,0x72,0xf0,0x03]
-; X86-NEXT:    vpslld $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x72,0xf0,0x03]
-; X86-NEXT:    vpaddd %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc2]
+; X86-NEXT:    vpslld $4, %ymm0, %ymm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0xa9,0x72,0xf0,0x04]
+; X86-NEXT:    vpslld $5, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x72,0xf0,0x05]
+; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psll_di_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpslld $3, %ymm0, %ymm2 # EVEX TO VEX Compression encoding: [0xc5,0xed,0x72,0xf0,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vpslld $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x75,0x29,0x72,0xf0,0x03]
-; X64-NEXT:    vpslld $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x72,0xf0,0x03]
-; X64-NEXT:    vpaddd %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc2]
+; X64-NEXT:    vpslld $4, %ymm0, %ymm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0xa9,0x72,0xf0,0x04]
+; X64-NEXT:    vpslld $5, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x72,0xf0,0x05]
+; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx512.mask.psll.di.256(<8 x i32> %x0, i32 3, <8 x i32> %x2, i8 %x3)
-  %res1 = call <8 x i32> @llvm.x86.avx512.mask.psll.di.256(<8 x i32> %x0, i32 3, <8 x i32> zeroinitializer, i8 %x3)
-  %res2 = call <8 x i32> @llvm.x86.avx512.mask.psll.di.256(<8 x i32> %x0, i32 3, <8 x i32> %x2, i8 -1)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.psll.di.256(<8 x i32> %x0, i32 4, <8 x i32> zeroinitializer, i8 %x3)
+  %res2 = call <8 x i32> @llvm.x86.avx512.mask.psll.di.256(<8 x i32> %x0, i32 5, <8 x i32> %x2, i8 -1)
   %res3 = add <8 x i32> %res, %res1
   %res4 = add <8 x i32> %res3, %res2
   ret <8 x i32> %res4
@@ -6906,188 +7111,284 @@ define <8 x i32>@test_int_x86_avx512_mask_psll_di_256(<8 x i32> %x0, i32 %x1, <8
 
 declare <2 x i64> @llvm.x86.avx512.mask.psrlv2.di(<2 x i64>, <2 x i64>, <2 x i64>, i8)
 
+define <2 x i64>@test_int_x86_avx512_psrlv2_di(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psrlv2_di:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf9,0x45,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.psrlv2.di(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1)
+  ret <2 x i64> %res
+}
+
 define <2 x i64>@test_int_x86_avx512_mask_psrlv2_di(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrlv2_di:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf9,0x45,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x45,0xd1]
-; X86-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x45,0xc1]
-; X86-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
-; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrlv2_di:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf9,0x45,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x45,0xd1]
-; X64-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x45,0xc1]
-; X64-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
-; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx512.mask.psrlv2.di(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3)
-  %res1 = call <2 x i64> @llvm.x86.avx512.mask.psrlv2.di(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <2 x i64> @llvm.x86.avx512.mask.psrlv2.di(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1)
-  %res3 = add <2 x i64> %res, %res1
-  %res4 = add <2 x i64> %res3, %res2
-  ret <2 x i64> %res4
+  ret <2 x i64> %res
+}
+
+define <2 x i64>@test_int_x86_avx512_maskz_psrlv2_di(<2 x i64> %x0, <2 x i64> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psrlv2_di:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x45,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psrlv2_di:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x45,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.psrlv2.di(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> zeroinitializer, i8 %x3)
+  ret <2 x i64> %res
 }
 
 declare <4 x i64> @llvm.x86.avx512.mask.psrlv4.di(<4 x i64>, <4 x i64>, <4 x i64>, i8)
 
+define <4 x i64>@test_int_x86_avx512_psrlv4_di(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psrlv4_di:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xfd,0x45,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.psrlv4.di(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 -1)
+  ret <4 x i64> %res
+}
+
 define <4 x i64>@test_int_x86_avx512_mask_psrlv4_di(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrlv4_di:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xfd,0x45,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x45,0xd1]
-; X86-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x45,0xc1]
-; X86-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
-; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrlv4_di:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xfd,0x45,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x45,0xd1]
-; X64-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x45,0xc1]
-; X64-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
-; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx512.mask.psrlv4.di(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3)
-  %res1 = call <4 x i64> @llvm.x86.avx512.mask.psrlv4.di(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <4 x i64> @llvm.x86.avx512.mask.psrlv4.di(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 -1)
-  %res3 = add <4 x i64> %res, %res1
-  %res4 = add <4 x i64> %res3, %res2
-  ret <4 x i64> %res4
+  ret <4 x i64> %res
+}
+
+define <4 x i64>@test_int_x86_avx512_maskz_psrlv4_di(<4 x i64> %x0, <4 x i64> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psrlv4_di:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x45,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psrlv4_di:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x45,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.psrlv4.di(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> zeroinitializer, i8 %x3)
+  ret <4 x i64> %res
 }
 
 declare <4 x i32> @llvm.x86.avx512.mask.psrlv4.si(<4 x i32>, <4 x i32>, <4 x i32>, i8)
 
+define <4 x i32>@test_int_x86_avx512_psrlv4_si(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psrlv4_si:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x45,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.psrlv4.si(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1)
+  ret <4 x i32> %res
+}
+
 define <4 x i32>@test_int_x86_avx512_mask_psrlv4_si(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrlv4_si:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x45,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x45,0xd1]
-; X86-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x45,0xc1]
-; X86-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
-; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrlv4_si:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x45,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x45,0xd1]
-; X64-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x45,0xc1]
-; X64-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
-; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx512.mask.psrlv4.si(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3)
-  %res1 = call <4 x i32> @llvm.x86.avx512.mask.psrlv4.si(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> zeroinitializer, i8 %x3)
-  %res2 = call <4 x i32> @llvm.x86.avx512.mask.psrlv4.si(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1)
-  %res3 = add <4 x i32> %res, %res1
-  %res4 = add <4 x i32> %res3, %res2
-  ret <4 x i32> %res4
+  ret <4 x i32> %res
+}
+
+define <4 x i32>@test_int_x86_avx512_maskz_psrlv4_si(<4 x i32> %x0, <4 x i32> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psrlv4_si:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x45,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psrlv4_si:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x45,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.psrlv4.si(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> zeroinitializer, i8 %x3)
+  ret <4 x i32> %res
 }
 
 declare <8 x i32> @llvm.x86.avx512.mask.psrlv8.si(<8 x i32>, <8 x i32>, <8 x i32>, i8)
 
+define <8 x i32>@test_int_x86_avx512_psrlv8_si(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psrlv8_si:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x45,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.psrlv8.si(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1)
+  ret <8 x i32> %res
+}
+
 define <8 x i32>@test_int_x86_avx512_mask_psrlv8_si(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrlv8_si:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x45,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x45,0xd1]
-; X86-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x45,0xc1]
-; X86-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
-; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrlv8_si:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x45,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x45,0xd1]
-; X64-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x45,0xc1]
-; X64-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
-; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx512.mask.psrlv8.si(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
-  %res1 = call <8 x i32> @llvm.x86.avx512.mask.psrlv8.si(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> zeroinitializer, i8 %x3)
-  %res2 = call <8 x i32> @llvm.x86.avx512.mask.psrlv8.si(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1)
-  %res3 = add <8 x i32> %res, %res1
-  %res4 = add <8 x i32> %res3, %res2
-  ret <8 x i32> %res4
+  ret <8 x i32> %res
+}
+
+define <8 x i32>@test_int_x86_avx512_maskz_psrlv8_si(<8 x i32> %x0, <8 x i32> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psrlv8_si:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x45,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psrlv8_si:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x45,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.psrlv8.si(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> zeroinitializer, i8 %x3)
+  ret <8 x i32> %res
 }
 
 declare <4 x i32> @llvm.x86.avx512.mask.psrav4.si(<4 x i32>, <4 x i32>, <4 x i32>, i8)
 
+define <4 x i32>@test_int_x86_avx512_psrav4_si(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psrav4_si:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsravd %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x46,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.psrav4.si(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1)
+  ret <4 x i32> %res
+}
+
 define <4 x i32>@test_int_x86_avx512_mask_psrav4_si(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrav4_si:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsravd %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x46,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpsravd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x46,0xd1]
-; X86-NEXT:    vpsravd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x46,0xc1]
-; X86-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
-; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrav4_si:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsravd %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x46,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpsravd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x46,0xd1]
-; X64-NEXT:    vpsravd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x46,0xc1]
-; X64-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
-; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx512.mask.psrav4.si(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3)
-  %res1 = call <4 x i32> @llvm.x86.avx512.mask.psrav4.si(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> zeroinitializer, i8 %x3)
-  %res2 = call <4 x i32> @llvm.x86.avx512.mask.psrav4.si(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1)
-  %res3 = add <4 x i32> %res, %res1
-  %res4 = add <4 x i32> %res3, %res2
-  ret <4 x i32> %res4
+  ret <4 x i32> %res
+}
+
+define <4 x i32>@test_int_x86_avx512_maskz_psrav4_si(<4 x i32> %x0, <4 x i32> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psrav4_si:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpsravd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x46,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psrav4_si:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpsravd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x46,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.psrav4.si(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> zeroinitializer, i8 %x3)
+  ret <4 x i32> %res
 }
 
 declare <8 x i32> @llvm.x86.avx512.mask.psrav8.si(<8 x i32>, <8 x i32>, <8 x i32>, i8)
 
+define <8 x i32>@test_int_x86_avx512_psrav8_si(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psrav8_si:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsravd %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x46,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.psrav8.si(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1)
+  ret <8 x i32> %res
+}
+
 define <8 x i32>@test_int_x86_avx512_mask_psrav8_si(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrav8_si:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsravd %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x46,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpsravd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x46,0xd1]
-; X86-NEXT:    vpsravd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x46,0xc1]
-; X86-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
-; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrav8_si:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsravd %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x46,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpsravd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x46,0xd1]
-; X64-NEXT:    vpsravd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x46,0xc1]
-; X64-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
-; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx512.mask.psrav8.si(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
-  %res1 = call <8 x i32> @llvm.x86.avx512.mask.psrav8.si(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> zeroinitializer, i8 %x3)
-  %res2 = call <8 x i32> @llvm.x86.avx512.mask.psrav8.si(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1)
-  %res3 = add <8 x i32> %res, %res1
-  %res4 = add <8 x i32> %res3, %res2
-  ret <8 x i32> %res4
+  ret <8 x i32> %res
+}
+
+define <8 x i32>@test_int_x86_avx512_maskz_psrav8_si(<8 x i32> %x0, <8 x i32> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psrav8_si:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpsravd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x46,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psrav8_si:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpsravd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x46,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.psrav8.si(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> zeroinitializer, i8 %x3)
+  ret <8 x i32> %res
 }
 
 define <8 x i32>@test_int_x86_avx512_mask_psrav8_si_const() {
@@ -7114,126 +7415,190 @@ define <8 x i32>@test_int_x86_avx512_mask_psrav8_si_const() {
 
 declare <2 x i64> @llvm.x86.avx512.mask.psllv2.di(<2 x i64>, <2 x i64>, <2 x i64>, i8)
 
+define <2 x i64>@test_int_x86_avx512_psllv2_di(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psllv2_di:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf9,0x47,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.psllv2.di(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1)
+  ret <2 x i64> %res
+}
+
 define <2 x i64>@test_int_x86_avx512_mask_psllv2_di(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psllv2_di:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsllvq %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf9,0x47,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpsllvq %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x47,0xd1]
-; X86-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x47,0xc1]
-; X86-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
-; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psllv2_di:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsllvq %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf9,0x47,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpsllvq %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x47,0xd1]
-; X64-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x47,0xc1]
-; X64-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
-; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx512.mask.psllv2.di(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3)
-  %res1 = call <2 x i64> @llvm.x86.avx512.mask.psllv2.di(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <2 x i64> @llvm.x86.avx512.mask.psllv2.di(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1)
-  %res3 = add <2 x i64> %res, %res1
-  %res4 = add <2 x i64> %res3, %res2
-  ret <2 x i64> %res4
+  ret <2 x i64> %res
+}
+
+define <2 x i64>@test_int_x86_avx512_maskz_psllv2_di(<2 x i64> %x0, <2 x i64> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psllv2_di:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x47,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psllv2_di:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x47,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.psllv2.di(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> zeroinitializer, i8 %x3)
+  ret <2 x i64> %res
 }
 
 declare <4 x i64> @llvm.x86.avx512.mask.psllv4.di(<4 x i64>, <4 x i64>, <4 x i64>, i8)
 
+define <4 x i64>@test_int_x86_avx512_psllv4_di(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psllv4_di:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsllvq %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xfd,0x47,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.psllv4.di(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 -1)
+  ret <4 x i64> %res
+}
+
 define <4 x i64>@test_int_x86_avx512_mask_psllv4_di(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psllv4_di:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsllvq %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xfd,0x47,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpsllvq %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x47,0xd1]
-; X86-NEXT:    vpsllvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x47,0xc1]
-; X86-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
-; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psllv4_di:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsllvq %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xfd,0x47,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpsllvq %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x47,0xd1]
-; X64-NEXT:    vpsllvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x47,0xc1]
-; X64-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
-; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx512.mask.psllv4.di(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3)
-  %res1 = call <4 x i64> @llvm.x86.avx512.mask.psllv4.di(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <4 x i64> @llvm.x86.avx512.mask.psllv4.di(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 -1)
-  %res3 = add <4 x i64> %res, %res1
-  %res4 = add <4 x i64> %res3, %res2
-  ret <4 x i64> %res4
+  ret <4 x i64> %res
+}
+
+define <4 x i64>@test_int_x86_avx512_maskz_psllv4_di(<4 x i64> %x0, <4 x i64> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psllv4_di:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpsllvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x47,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psllv4_di:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpsllvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x47,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.psllv4.di(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> zeroinitializer, i8 %x3)
+  ret <4 x i64> %res
 }
 
 declare <4 x i32> @llvm.x86.avx512.mask.psllv4.si(<4 x i32>, <4 x i32>, <4 x i32>, i8)
 
+define <4 x i32>@test_int_x86_avx512_psllv4_si(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psllv4_si:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x47,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.psllv4.si(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1)
+  ret <4 x i32> %res
+}
+
 define <4 x i32>@test_int_x86_avx512_mask_psllv4_si(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psllv4_si:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsllvd %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x47,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpsllvd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x47,0xd1]
-; X86-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x47,0xc1]
-; X86-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
-; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psllv4_si:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsllvd %xmm1, %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x47,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpsllvd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x47,0xd1]
-; X64-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x47,0xc1]
-; X64-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
-; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx512.mask.psllv4.si(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3)
-  %res1 = call <4 x i32> @llvm.x86.avx512.mask.psllv4.si(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> zeroinitializer, i8 %x3)
-  %res2 = call <4 x i32> @llvm.x86.avx512.mask.psllv4.si(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1)
-  %res3 = add <4 x i32> %res, %res1
-  %res4 = add <4 x i32> %res3, %res2
-  ret <4 x i32> %res4
+  ret <4 x i32> %res
+}
+
+define <4 x i32>@test_int_x86_avx512_maskz_psllv4_si(<4 x i32> %x0, <4 x i32> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psllv4_si:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x47,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psllv4_si:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x47,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.psllv4.si(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> zeroinitializer, i8 %x3)
+  ret <4 x i32> %res
 }
 
 declare <8 x i32> @llvm.x86.avx512.mask.psllv8.si(<8 x i32>, <8 x i32>, <8 x i32>, i8)
 
+define <8 x i32>@test_int_x86_avx512_psllv8_si(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psllv8_si:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x47,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.psllv8.si(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1)
+  ret <8 x i32> %res
+}
+
 define <8 x i32>@test_int_x86_avx512_mask_psllv8_si(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psllv8_si:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsllvd %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x47,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpsllvd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x47,0xd1]
-; X86-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x47,0xc1]
-; X86-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
-; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psllv8_si:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsllvd %ymm1, %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x47,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpsllvd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x47,0xd1]
-; X64-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x47,0xc1]
-; X64-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
-; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx512.mask.psllv8.si(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
-  %res1 = call <8 x i32> @llvm.x86.avx512.mask.psllv8.si(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> zeroinitializer, i8 %x3)
-  %res2 = call <8 x i32> @llvm.x86.avx512.mask.psllv8.si(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1)
-  %res3 = add <8 x i32> %res, %res1
-  %res4 = add <8 x i32> %res3, %res2
-  ret <8 x i32> %res4
+  ret <8 x i32> %res
+}
+
+define <8 x i32>@test_int_x86_avx512_maskz_psllv8_si(<8 x i32> %x0, <8 x i32> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psllv8_si:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x47,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psllv8_si:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x47,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.psllv8.si(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> zeroinitializer, i8 %x3)
+  ret <8 x i32> %res
 }
 
 declare <4 x i32> @llvm.x86.avx512.mask.pmovzxb.d.128(<16 x i8>, <4 x i32>, i8)
@@ -8136,64 +8501,96 @@ define <4 x i64>@test_int_x86_avx512_maskz_pmovsxw_q_256(<8 x i16> %x0, i8 %x2)
 
 declare <2 x i64> @llvm.x86.avx512.mask.psra.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
 
+define <2 x i64>@test_int_x86_avx512_psra_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psra_q_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsraq %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf1,0xfd,0x08,0xe2,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.psra.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1)
+  ret <2 x i64> %res
+}
+
 define <2 x i64>@test_int_x86_avx512_mask_psra_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psra_q_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsraq %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf1,0xfd,0x08,0xe2,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpsraq %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0xfd,0x09,0xe2,0xd1]
-; X86-NEXT:    vpsraq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0xe2,0xc1]
-; X86-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
-; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psra_q_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsraq %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf1,0xfd,0x08,0xe2,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpsraq %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0xfd,0x09,0xe2,0xd1]
-; X64-NEXT:    vpsraq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0xe2,0xc1]
-; X64-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
-; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx512.mask.psra.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3)
-  %res1 = call <2 x i64> @llvm.x86.avx512.mask.psra.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <2 x i64> @llvm.x86.avx512.mask.psra.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1)
-  %res3 = add <2 x i64> %res, %res1
-  %res4 = add <2 x i64> %res3, %res2
-  ret <2 x i64> %res4
+  ret <2 x i64> %res
+}
+
+define <2 x i64>@test_int_x86_avx512_maskz_psra_q_128(<2 x i64> %x0, <2 x i64> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psra_q_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpsraq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0xe2,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psra_q_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpsraq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0xe2,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.psra.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> zeroinitializer, i8 %x3)
+  ret <2 x i64> %res
 }
 
 declare <4 x i64> @llvm.x86.avx512.mask.psra.q.256(<4 x i64>, <2 x i64>, <4 x i64>, i8)
 
+define <4 x i64>@test_int_x86_avx512_psra_q_256(<4 x i64> %x0, <2 x i64> %x1, <4 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psra_q_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsraq %xmm1, %ymm0, %ymm0 # encoding: [0x62,0xf1,0xfd,0x28,0xe2,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.psra.q.256(<4 x i64> %x0, <2 x i64> %x1, <4 x i64> %x2, i8 -1)
+  ret <4 x i64> %res
+}
+
 define <4 x i64>@test_int_x86_avx512_mask_psra_q_256(<4 x i64> %x0, <2 x i64> %x1, <4 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psra_q_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsraq %xmm1, %ymm0, %ymm3 # encoding: [0x62,0xf1,0xfd,0x28,0xe2,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpsraq %xmm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0xfd,0x29,0xe2,0xd1]
-; X86-NEXT:    vpsraq %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0xe2,0xc1]
-; X86-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
-; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psra_q_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsraq %xmm1, %ymm0, %ymm3 # encoding: [0x62,0xf1,0xfd,0x28,0xe2,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpsraq %xmm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0xfd,0x29,0xe2,0xd1]
-; X64-NEXT:    vpsraq %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0xe2,0xc1]
-; X64-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
-; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx512.mask.psra.q.256(<4 x i64> %x0, <2 x i64> %x1, <4 x i64> %x2, i8 %x3)
-  %res1 = call <4 x i64> @llvm.x86.avx512.mask.psra.q.256(<4 x i64> %x0, <2 x i64> %x1, <4 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <4 x i64> @llvm.x86.avx512.mask.psra.q.256(<4 x i64> %x0, <2 x i64> %x1, <4 x i64> %x2, i8 -1)
-  %res3 = add <4 x i64> %res, %res1
-  %res4 = add <4 x i64> %res3, %res2
-  ret <4 x i64> %res4
+  ret <4 x i64> %res
+}
+
+define <4 x i64>@test_int_x86_avx512_maskz_psra_q_256(<4 x i64> %x0, <2 x i64> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psra_q_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpsraq %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0xe2,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psra_q_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpsraq %xmm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0xe2,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.psra.q.256(<4 x i64> %x0, <2 x i64> %x1, <4 x i64> zeroinitializer, i8 %x3)
+  ret <4 x i64> %res
 }
 
 declare <2 x i64> @llvm.x86.avx512.mask.psra.qi.128(<2 x i64>, i32, <2 x i64>, i8)
@@ -8201,27 +8598,27 @@ declare <2 x i64> @llvm.x86.avx512.mask.psra.qi.128(<2 x i64>, i32, <2 x i64>, i
 define <2 x i64>@test_int_x86_avx512_mask_psra_qi_128(<2 x i64> %x0, i32 %x1, <2 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psra_qi_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsraq $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0xed,0x08,0x72,0xe0,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpsraq $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x09,0x72,0xe0,0x03]
-; X86-NEXT:    vpsraq $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0x72,0xe0,0x03]
-; X86-NEXT:    vpaddq %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc2]
+; X86-NEXT:    vpsraq $4, %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0xed,0x89,0x72,0xe0,0x04]
+; X86-NEXT:    vpsraq $5, %xmm0, %xmm0 # encoding: [0x62,0xf1,0xfd,0x08,0x72,0xe0,0x05]
+; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psra_qi_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsraq $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0xed,0x08,0x72,0xe0,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vpsraq $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x09,0x72,0xe0,0x03]
-; X64-NEXT:    vpsraq $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0x72,0xe0,0x03]
-; X64-NEXT:    vpaddq %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc2]
+; X64-NEXT:    vpsraq $4, %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0xed,0x89,0x72,0xe0,0x04]
+; X64-NEXT:    vpsraq $5, %xmm0, %xmm0 # encoding: [0x62,0xf1,0xfd,0x08,0x72,0xe0,0x05]
+; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx512.mask.psra.qi.128(<2 x i64> %x0, i32 3, <2 x i64> %x2, i8 %x3)
-  %res1 = call <2 x i64> @llvm.x86.avx512.mask.psra.qi.128(<2 x i64> %x0, i32 3, <2 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <2 x i64> @llvm.x86.avx512.mask.psra.qi.128(<2 x i64> %x0, i32 3, <2 x i64> %x2, i8 -1)
+  %res1 = call <2 x i64> @llvm.x86.avx512.mask.psra.qi.128(<2 x i64> %x0, i32 4, <2 x i64> zeroinitializer, i8 %x3)
+  %res2 = call <2 x i64> @llvm.x86.avx512.mask.psra.qi.128(<2 x i64> %x0, i32 5, <2 x i64> %x2, i8 -1)
   %res3 = add <2 x i64> %res, %res1
   %res4 = add <2 x i64> %res3, %res2
   ret <2 x i64> %res4
@@ -8232,27 +8629,27 @@ declare <4 x i64> @llvm.x86.avx512.mask.psra.qi.256(<4 x i64>, i32, <4 x i64>, i
 define <4 x i64>@test_int_x86_avx512_mask_psra_qi_256(<4 x i64> %x0, i32 %x1, <4 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psra_qi_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsraq $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0xed,0x28,0x72,0xe0,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpsraq $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x29,0x72,0xe0,0x03]
-; X86-NEXT:    vpsraq $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0x72,0xe0,0x03]
-; X86-NEXT:    vpaddq %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc2]
+; X86-NEXT:    vpsraq $4, %ymm0, %ymm2 {%k1} {z} # encoding: [0x62,0xf1,0xed,0xa9,0x72,0xe0,0x04]
+; X86-NEXT:    vpsraq $5, %ymm0, %ymm0 # encoding: [0x62,0xf1,0xfd,0x28,0x72,0xe0,0x05]
+; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psra_qi_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsraq $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0xed,0x28,0x72,0xe0,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vpsraq $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x29,0x72,0xe0,0x03]
-; X64-NEXT:    vpsraq $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0x72,0xe0,0x03]
-; X64-NEXT:    vpaddq %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc2]
+; X64-NEXT:    vpsraq $4, %ymm0, %ymm2 {%k1} {z} # encoding: [0x62,0xf1,0xed,0xa9,0x72,0xe0,0x04]
+; X64-NEXT:    vpsraq $5, %ymm0, %ymm0 # encoding: [0x62,0xf1,0xfd,0x28,0x72,0xe0,0x05]
+; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx512.mask.psra.qi.256(<4 x i64> %x0, i32 3, <4 x i64> %x2, i8 %x3)
-  %res1 = call <4 x i64> @llvm.x86.avx512.mask.psra.qi.256(<4 x i64> %x0, i32 3, <4 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <4 x i64> @llvm.x86.avx512.mask.psra.qi.256(<4 x i64> %x0, i32 3, <4 x i64> %x2, i8 -1)
+  %res1 = call <4 x i64> @llvm.x86.avx512.mask.psra.qi.256(<4 x i64> %x0, i32 4, <4 x i64> zeroinitializer, i8 %x3)
+  %res2 = call <4 x i64> @llvm.x86.avx512.mask.psra.qi.256(<4 x i64> %x0, i32 5, <4 x i64> %x2, i8 -1)
   %res3 = add <4 x i64> %res, %res1
   %res4 = add <4 x i64> %res3, %res2
   ret <4 x i64> %res4
@@ -8260,33 +8657,49 @@ define <4 x i64>@test_int_x86_avx512_mask_psra_qi_256(<4 x i64> %x0, i32 %x1, <4
 
 declare <2 x i64> @llvm.x86.avx512.mask.psrav.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
 
+define <2 x i64>@test_int_x86_avx512_psrav_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psrav_q_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsravq %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x46,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.psrav.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1)
+  ret <2 x i64> %res
+}
+
 define <2 x i64>@test_int_x86_avx512_mask_psrav_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrav_q_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsravq %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x46,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpsravq %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x46,0xd1]
-; X86-NEXT:    vpsravq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x46,0xc1]
-; X86-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
-; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrav_q_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsravq %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x46,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpsravq %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x46,0xd1]
-; X64-NEXT:    vpsravq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x46,0xc1]
-; X64-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
-; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx512.mask.psrav.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3)
-  %res1 = call <2 x i64> @llvm.x86.avx512.mask.psrav.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <2 x i64> @llvm.x86.avx512.mask.psrav.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1)
-  %res3 = add <2 x i64> %res, %res1
-  %res4 = add <2 x i64> %res3, %res2
-  ret <2 x i64> %res4
+  ret <2 x i64> %res
+}
+
+define <2 x i64>@test_int_x86_avx512_maskz_psrav_q_128(<2 x i64> %x0, <2 x i64> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psrav_q_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpsravq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x46,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psrav_q_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpsravq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x46,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.psrav.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> zeroinitializer, i8 %x3)
+  ret <2 x i64> %res
 }
 
 define <2 x i64>@test_int_x86_avx512_mask_psrav_q_128_const(i8 %x3) {
@@ -8313,33 +8726,49 @@ define <2 x i64>@test_int_x86_avx512_mask_psrav_q_128_const(i8 %x3) {
 
 declare <4 x i64> @llvm.x86.avx512.mask.psrav.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
 
+define <4 x i64>@test_int_x86_avx512_psrav_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_psrav_q_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsravq %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x46,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.psrav.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 -1)
+  ret <4 x i64> %res
+}
+
 define <4 x i64>@test_int_x86_avx512_mask_psrav_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrav_q_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpsravq %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x46,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpsravq %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x46,0xd1]
-; X86-NEXT:    vpsravq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x46,0xc1]
-; X86-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
-; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_psrav_q_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsravq %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x46,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpsravq %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x46,0xd1]
-; X64-NEXT:    vpsravq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x46,0xc1]
-; X64-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
-; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx512.mask.psrav.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3)
-  %res1 = call <4 x i64> @llvm.x86.avx512.mask.psrav.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <4 x i64> @llvm.x86.avx512.mask.psrav.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 -1)
-  %res3 = add <4 x i64> %res, %res1
-  %res4 = add <4 x i64> %res3, %res2
-  ret <4 x i64> %res4
+  ret <4 x i64> %res
+}
+
+define <4 x i64>@test_int_x86_avx512_maskz_psrav_q_256(<4 x i64> %x0, <4 x i64> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_psrav_q_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpsravq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x46,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_psrav_q_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpsravq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x46,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.psrav.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> zeroinitializer, i8 %x3)
+  ret <4 x i64> %res
 }
 
 declare <2 x double> @llvm.x86.avx512.mask.cvtdq2pd.128(<4 x i32>, <2 x double>, i8)
@@ -10237,102 +10666,122 @@ define <8 x i32>@test_int_x86_avx512_mask_broadcasti32x4_256_load(<4 x i32>* %x0
 
 declare <2 x i64> @llvm.x86.avx512.mask.pabs.q.128(<2 x i64>, <2 x i64>, i8)
 
+define <2 x i64>@test_int_x86_avx512_pabs_q_128(<2 x i64> %x0, <2 x i64> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_pabs_q_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpabsq %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x1f,0xc0]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.pabs.q.128(<2 x i64> %x0, <2 x i64> %x1, i8 -1)
+  ret <2 x i64> %res
+}
+
 define <2 x i64>@test_int_x86_avx512_mask_pabs_q_128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_pabs_q_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpabsq %xmm0, %xmm2 # encoding: [0x62,0xf2,0xfd,0x08,0x1f,0xd0]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpabsq %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x1f,0xc8]
-; X86-NEXT:    vpaddq %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xd4,0xc2]
+; X86-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pabs_q_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpabsq %xmm0, %xmm2 # encoding: [0x62,0xf2,0xfd,0x08,0x1f,0xd0]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpabsq %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x1f,0xc8]
-; X64-NEXT:    vpaddq %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xd4,0xc2]
+; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx512.mask.pabs.q.128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2)
-  %res1 = call <2 x i64> @llvm.x86.avx512.mask.pabs.q.128(<2 x i64> %x0, <2 x i64> %x1, i8 -1)
-  %res2 = add <2 x i64> %res, %res1
-  ret <2 x i64> %res2
+  ret <2 x i64> %res
 }
 
 declare <4 x i64> @llvm.x86.avx512.mask.pabs.q.256(<4 x i64>, <4 x i64>, i8)
 
-define <4 x i64>@test_int_x86_avx512_mask_pabs_q_256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2) {
-; X86-LABEL: test_int_x86_avx512_mask_pabs_q_256:
-; X86:       # %bb.0:
-; X86-NEXT:    vpabsq %ymm0, %ymm2 # encoding: [0x62,0xf2,0xfd,0x28,0x1f,0xd0]
+define <4 x i64>@test_int_x86_avx512_pabs_q_256(<4 x i64> %x0, <4 x i64> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_pabs_q_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpabsq %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x1f,0xc0]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.pabs.q.256(<4 x i64> %x0, <4 x i64> %x1, i8 -1)
+  ret <4 x i64> %res
+}
+
+define <4 x i64>@test_int_x86_avx512_mask_pabs_q_256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2) {
+; X86-LABEL: test_int_x86_avx512_mask_pabs_q_256:
+; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpabsq %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x1f,0xc8]
-; X86-NEXT:    vpaddq %ymm2, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xd4,0xc2]
+; X86-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pabs_q_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpabsq %ymm0, %ymm2 # encoding: [0x62,0xf2,0xfd,0x28,0x1f,0xd0]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpabsq %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x1f,0xc8]
-; X64-NEXT:    vpaddq %ymm2, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xd4,0xc2]
+; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx512.mask.pabs.q.256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2)
-  %res1 = call <4 x i64> @llvm.x86.avx512.mask.pabs.q.256(<4 x i64> %x0, <4 x i64> %x1, i8 -1)
-  %res2 = add <4 x i64> %res, %res1
-  ret <4 x i64> %res2
+  ret <4 x i64> %res
 }
 
 declare <4 x i32> @llvm.x86.avx512.mask.pabs.d.128(<4 x i32>, <4 x i32>, i8)
 
+define <4 x i32>@test_int_x86_avx512_pabs_d_128(<4 x i32> %x0, <4 x i32> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_pabs_d_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpabsd %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x1e,0xc0]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pabs.d.128(<4 x i32> %x0, <4 x i32> %x1, i8 -1)
+  ret <4 x i32> %res
+}
+
 define <4 x i32>@test_int_x86_avx512_mask_pabs_d_128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_pabs_d_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpabsd %xmm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x1e,0xd0]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpabsd %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x1e,0xc8]
-; X86-NEXT:    vpaddd %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc2]
+; X86-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pabs_d_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpabsd %xmm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x1e,0xd0]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpabsd %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x1e,0xc8]
-; X64-NEXT:    vpaddd %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc2]
+; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx512.mask.pabs.d.128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2)
-  %res1 = call <4 x i32> @llvm.x86.avx512.mask.pabs.d.128(<4 x i32> %x0, <4 x i32> %x1, i8 -1)
-  %res2 = add <4 x i32> %res, %res1
-  ret <4 x i32> %res2
+  ret <4 x i32> %res
 }
 
 declare <8 x i32> @llvm.x86.avx512.mask.pabs.d.256(<8 x i32>, <8 x i32>, i8)
 
+define <8 x i32>@test_int_x86_avx512_pabs_d_256(<8 x i32> %x0, <8 x i32> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_pabs_d_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpabsd %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x1e,0xc0]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pabs.d.256(<8 x i32> %x0, <8 x i32> %x1, i8 -1)
+  ret <8 x i32> %res
+}
+
 define <8 x i32>@test_int_x86_avx512_mask_pabs_d_256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_pabs_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpabsd %ymm0, %ymm2 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x1e,0xd0]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpabsd %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x1e,0xc8]
-; X86-NEXT:    vpaddd %ymm2, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfe,0xc2]
+; X86-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pabs_d_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpabsd %ymm0, %ymm2 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x1e,0xd0]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpabsd %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x1e,0xc8]
-; X64-NEXT:    vpaddd %ymm2, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfe,0xc2]
+; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx512.mask.pabs.d.256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2)
-  %res1 = call <8 x i32> @llvm.x86.avx512.mask.pabs.d.256(<8 x i32> %x0, <8 x i32> %x1, i8 -1)
-  %res2 = add <8 x i32> %res, %res1
-  ret <8 x i32> %res2
+  ret <8 x i32> %res
 }
 
 declare i8 @llvm.x86.avx512.ptestm.d.128(<4 x i32>, <4 x i32>,i8)
@@ -11878,29 +12327,30 @@ define <4 x i64>@test_int_x86_avx512_maskz_permvar_di_256(<4 x i64> %x0, <4 x i6
 
 declare <4 x i32> @llvm.x86.avx512.mask.pternlog.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i32, i8)
 
+define <4 x i32>@test_int_x86_avx512_pternlog_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_pternlog_d_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpternlogd $33, %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf3,0x75,0x08,0x25,0xc2,0x21]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.pternlog.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i32 33, i8 -1)
+  ret <4 x i32> %res
+}
+
 define <4 x i32>@test_int_x86_avx512_mask_pternlog_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_pternlog_d_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X86-NEXT:    vpternlogd $33, %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf3,0x75,0x08,0x25,0xda,0x21]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogd $33, %xmm2, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf3,0x75,0x09,0x25,0xc2,0x21]
-; X86-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pternlog_d_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X64-NEXT:    vpternlogd $33, %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf3,0x75,0x08,0x25,0xda,0x21]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogd $33, %xmm2, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf3,0x75,0x09,0x25,0xc2,0x21]
-; X64-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx512.mask.pternlog.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i32 33, i8 %x4)
-  %res1 = call <4 x i32> @llvm.x86.avx512.mask.pternlog.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i32 33, i8 -1)
-  %res2 = add <4 x i32> %res, %res1
-  ret <4 x i32> %res2
+  ret <4 x i32> %res
 }
 
 declare <4 x i32> @llvm.x86.avx512.maskz.pternlog.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i32, i8)
@@ -11908,53 +12358,46 @@ declare <4 x i32> @llvm.x86.avx512.maskz.pternlog.d.128(<4 x i32>, <4 x i32>, <4
 define <4 x i32>@test_int_x86_avx512_maskz_pternlog_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_maskz_pternlog_d_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X86-NEXT:    vpternlogd $33, %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf3,0x75,0x08,0x25,0xda,0x21]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogd $33, %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x75,0x89,0x25,0xc2,0x21]
-; X86-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_pternlog_d_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X64-NEXT:    vpternlogd $33, %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf3,0x75,0x08,0x25,0xda,0x21]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogd $33, %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x75,0x89,0x25,0xc2,0x21]
-; X64-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx512.maskz.pternlog.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i32 33, i8 %x4)
-  %res1 = call <4 x i32> @llvm.x86.avx512.maskz.pternlog.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i32 33, i8 -1)
-  %res2 = add <4 x i32> %res, %res1
-  ret <4 x i32> %res2
+  ret <4 x i32> %res
 }
 
 declare <8 x i32> @llvm.x86.avx512.mask.pternlog.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i32, i8)
 
+define <8 x i32>@test_int_x86_avx512_pternlog_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_pternlog_d_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpternlogd $33, %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf3,0x75,0x28,0x25,0xc2,0x21]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.pternlog.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i32 33, i8 -1)
+  ret <8 x i32> %res
+}
+
 define <8 x i32>@test_int_x86_avx512_mask_pternlog_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_pternlog_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X86-NEXT:    vpternlogd $33, %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf3,0x75,0x28,0x25,0xda,0x21]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogd $33, %ymm2, %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf3,0x75,0x29,0x25,0xc2,0x21]
-; X86-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pternlog_d_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X64-NEXT:    vpternlogd $33, %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf3,0x75,0x28,0x25,0xda,0x21]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogd $33, %ymm2, %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf3,0x75,0x29,0x25,0xc2,0x21]
-; X64-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx512.mask.pternlog.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i32 33, i8 %x4)
-  %res1 = call <8 x i32> @llvm.x86.avx512.mask.pternlog.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i32 33, i8 -1)
-  %res2 = add <8 x i32> %res, %res1
-  ret <8 x i32> %res2
+  ret <8 x i32> %res
 }
 
 declare <8 x i32> @llvm.x86.avx512.maskz.pternlog.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i32, i8)
@@ -11962,53 +12405,46 @@ declare <8 x i32> @llvm.x86.avx512.maskz.pternlog.d.256(<8 x i32>, <8 x i32>, <8
 define <8 x i32>@test_int_x86_avx512_maskz_pternlog_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_maskz_pternlog_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X86-NEXT:    vpternlogd $33, %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf3,0x75,0x28,0x25,0xda,0x21]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogd $33, %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf3,0x75,0xa9,0x25,0xc2,0x21]
-; X86-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_pternlog_d_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X64-NEXT:    vpternlogd $33, %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf3,0x75,0x28,0x25,0xda,0x21]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogd $33, %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf3,0x75,0xa9,0x25,0xc2,0x21]
-; X64-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx512.maskz.pternlog.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i32 33, i8 %x4)
-  %res1 = call <8 x i32> @llvm.x86.avx512.maskz.pternlog.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i32 33, i8 -1)
-  %res2 = add <8 x i32> %res, %res1
-  ret <8 x i32> %res2
+  ret <8 x i32> %res
 }
 
 declare <2 x i64> @llvm.x86.avx512.mask.pternlog.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i32, i8)
 
+define <2 x i64>@test_int_x86_avx512_pternlog_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_pternlog_q_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpternlogq $33, %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf3,0xf5,0x08,0x25,0xc2,0x21]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.pternlog.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i32 33, i8 -1)
+  ret <2 x i64> %res
+}
+
 define <2 x i64>@test_int_x86_avx512_mask_pternlog_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_pternlog_q_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X86-NEXT:    vpternlogq $33, %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf3,0xf5,0x08,0x25,0xda,0x21]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogq $33, %xmm2, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf3,0xf5,0x09,0x25,0xc2,0x21]
-; X86-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pternlog_q_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X64-NEXT:    vpternlogq $33, %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf3,0xf5,0x08,0x25,0xda,0x21]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogq $33, %xmm2, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf3,0xf5,0x09,0x25,0xc2,0x21]
-; X64-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx512.mask.pternlog.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i32 33, i8 %x4)
-  %res1 = call <2 x i64> @llvm.x86.avx512.mask.pternlog.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i32 33, i8 -1)
-  %res2 = add <2 x i64> %res, %res1
-  ret <2 x i64> %res2
+  ret <2 x i64> %res
 }
 
 declare <2 x i64> @llvm.x86.avx512.maskz.pternlog.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i32, i8)
@@ -12016,53 +12452,46 @@ declare <2 x i64> @llvm.x86.avx512.maskz.pternlog.q.128(<2 x i64>, <2 x i64>, <2
 define <2 x i64>@test_int_x86_avx512_maskz_pternlog_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_maskz_pternlog_q_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X86-NEXT:    vpternlogq $33, %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf3,0xf5,0x08,0x25,0xda,0x21]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogq $33, %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0xf5,0x89,0x25,0xc2,0x21]
-; X86-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_pternlog_q_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X64-NEXT:    vpternlogq $33, %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf3,0xf5,0x08,0x25,0xda,0x21]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogq $33, %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0xf5,0x89,0x25,0xc2,0x21]
-; X64-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx512.maskz.pternlog.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i32 33, i8 %x4)
-  %res1 = call <2 x i64> @llvm.x86.avx512.maskz.pternlog.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i32 33, i8 -1)
-  %res2 = add <2 x i64> %res, %res1
-  ret <2 x i64> %res2
+  ret <2 x i64> %res
 }
 
 declare <4 x i64> @llvm.x86.avx512.mask.pternlog.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i32, i8)
 
+define <4 x i64>@test_int_x86_avx512_pternlog_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_pternlog_q_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpternlogq $33, %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf3,0xf5,0x28,0x25,0xc2,0x21]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.pternlog.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i32 33, i8 -1)
+  ret <4 x i64> %res
+}
+
 define <4 x i64>@test_int_x86_avx512_mask_pternlog_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_pternlog_q_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X86-NEXT:    vpternlogq $33, %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf3,0xf5,0x28,0x25,0xda,0x21]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogq $33, %ymm2, %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf3,0xf5,0x29,0x25,0xc2,0x21]
-; X86-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pternlog_q_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X64-NEXT:    vpternlogq $33, %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf3,0xf5,0x28,0x25,0xda,0x21]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogq $33, %ymm2, %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf3,0xf5,0x29,0x25,0xc2,0x21]
-; X64-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx512.mask.pternlog.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i32 33, i8 %x4)
-  %res1 = call <4 x i64> @llvm.x86.avx512.mask.pternlog.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i32 33, i8 -1)
-  %res2 = add <4 x i64> %res, %res1
-  ret <4 x i64> %res2
+  ret <4 x i64> %res
 }
 
 declare <4 x i64> @llvm.x86.avx512.maskz.pternlog.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i32, i8)
@@ -12070,26 +12499,18 @@ declare <4 x i64> @llvm.x86.avx512.maskz.pternlog.q.256(<4 x i64>, <4 x i64>, <4
 define <4 x i64>@test_int_x86_avx512_maskz_pternlog_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_maskz_pternlog_q_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X86-NEXT:    vpternlogq $33, %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf3,0xf5,0x28,0x25,0xda,0x21]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogq $33, %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf3,0xf5,0xa9,0x25,0xc2,0x21]
-; X86-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_pternlog_q_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X64-NEXT:    vpternlogq $33, %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf3,0xf5,0x28,0x25,0xda,0x21]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogq $33, %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf3,0xf5,0xa9,0x25,0xc2,0x21]
-; X64-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx512.maskz.pternlog.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i32 33, i8 %x4)
-  %res1 = call <4 x i64> @llvm.x86.avx512.maskz.pternlog.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i32 33, i8 -1)
-  %res2 = add <4 x i64> %res, %res1
-  ret <4 x i64> %res2
+  ret <4 x i64> %res
 }
 
 declare <4 x float> @llvm.x86.avx512.mask.cvtudq2ps.128(<4 x i32>, <4 x float>, i8)
@@ -13399,126 +13820,190 @@ declare <8 x float> @llvm.x86.avx512.mask.sqrt.ps.256(<8 x float>, <8 x float>,
 
 declare <4 x i32> @llvm.x86.avx512.mask.prorv.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
 
-define <4 x i32>@test_int_x86_avx512_mask_prorv_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask_prorv_d_128:
+define <4 x i32>@test_int_x86_avx512_prorv_d_128_old(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_prorv_d_128_old:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vprorvd %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7d,0x08,0x14,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.prorv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32>@test_int_x86_avx512_mask_prorv_d_128_old(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_prorv_d_128_old:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprorvd %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0x7d,0x08,0x14,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprorvd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x14,0xd1]
-; X86-NEXT:    vprorvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x14,0xc1]
-; X86-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
-; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask_prorv_d_128:
+; X64-LABEL: test_int_x86_avx512_mask_prorv_d_128_old:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprorvd %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0x7d,0x08,0x14,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vprorvd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x14,0xd1]
-; X64-NEXT:    vprorvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x14,0xc1]
-; X64-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
-; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx512.mask.prorv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3)
-  %res1 = call <4 x i32> @llvm.x86.avx512.mask.prorv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> zeroinitializer, i8 %x3)
-  %res2 = call <4 x i32> @llvm.x86.avx512.mask.prorv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1)
-  %res3 = add <4 x i32> %res, %res1
-  %res4 = add <4 x i32> %res3, %res2
-  ret <4 x i32> %res4
+  ret <4 x i32> %res
+}
+
+define <4 x i32>@test_int_x86_avx512_maskz_prorv_d_128_old(<4 x i32> %x0, <4 x i32> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_prorv_d_128_old:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vprorvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x14,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_prorv_d_128_old:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vprorvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x14,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.prorv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> zeroinitializer, i8 %x3)
+  ret <4 x i32> %res
 }
 
 declare <8 x i32> @llvm.x86.avx512.mask.prorv.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
 
-define <8 x i32>@test_int_x86_avx512_mask_prorv_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask_prorv_d_256:
+define <8 x i32>@test_int_x86_avx512_prorv_d_256_old(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_prorv_d_256_old:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vprorvd %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf2,0x7d,0x28,0x14,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.prorv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1)
+  ret <8 x i32> %res
+}
+
+define <8 x i32>@test_int_x86_avx512_mask_prorv_d_256_old(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_prorv_d_256_old:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprorvd %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0x7d,0x28,0x14,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprorvd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x14,0xd1]
-; X86-NEXT:    vprorvd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x14,0xc1]
-; X86-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
-; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask_prorv_d_256:
+; X64-LABEL: test_int_x86_avx512_mask_prorv_d_256_old:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprorvd %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0x7d,0x28,0x14,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vprorvd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x14,0xd1]
-; X64-NEXT:    vprorvd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x14,0xc1]
-; X64-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
-; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx512.mask.prorv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
-  %res1 = call <8 x i32> @llvm.x86.avx512.mask.prorv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> zeroinitializer, i8 %x3)
-  %res2 = call <8 x i32> @llvm.x86.avx512.mask.prorv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1)
-  %res3 = add <8 x i32> %res, %res1
-  %res4 = add <8 x i32> %res3, %res2
-  ret <8 x i32> %res4
+  ret <8 x i32> %res
+}
+
+define <8 x i32>@test_int_x86_avx512_maskz_prorv_d_256_old(<8 x i32> %x0, <8 x i32> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_prorv_d_256_old:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vprorvd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x14,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_prorv_d_256_old:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vprorvd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x14,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.prorv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> zeroinitializer, i8 %x3)
+  ret <8 x i32> %res
 }
 
 declare <2 x i64> @llvm.x86.avx512.mask.prorv.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
 
-define <2 x i64>@test_int_x86_avx512_mask_prorv_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask_prorv_q_128:
+define <2 x i64>@test_int_x86_avx512_prorv_q_128_old(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_prorv_q_128_old:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vprorvq %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x14,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.prorv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1)
+  ret <2 x i64> %res
+}
+
+define <2 x i64>@test_int_x86_avx512_mask_prorv_q_128_old(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_prorv_q_128_old:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprorvq %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x14,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprorvq %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x14,0xd1]
-; X86-NEXT:    vprorvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x14,0xc1]
-; X86-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
-; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask_prorv_q_128:
+; X64-LABEL: test_int_x86_avx512_mask_prorv_q_128_old:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprorvq %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x14,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vprorvq %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x14,0xd1]
-; X64-NEXT:    vprorvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x14,0xc1]
-; X64-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
-; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx512.mask.prorv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3)
-  %res1 = call <2 x i64> @llvm.x86.avx512.mask.prorv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <2 x i64> @llvm.x86.avx512.mask.prorv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1)
-  %res3 = add <2 x i64> %res, %res1
-  %res4 = add <2 x i64> %res3, %res2
-  ret <2 x i64> %res4
+  ret <2 x i64> %res
+}
+
+define <2 x i64>@test_int_x86_avx512_maskz_prorv_q_128_old(<2 x i64> %x0, <2 x i64> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_prorv_q_128_old:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vprorvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x14,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_prorv_q_128_old:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vprorvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x14,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.prorv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> zeroinitializer, i8 %x3)
+  ret <2 x i64> %res
 }
 
 declare <4 x i64> @llvm.x86.avx512.mask.prorv.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
 
-define <4 x i64>@test_int_x86_avx512_mask_prorv_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask_prorv_q_256:
+define <4 x i64>@test_int_x86_avx512_prorv_q_256_old(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_prorv_q_256_old:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vprorvq %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x14,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.prorv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 -1)
+  ret <4 x i64> %res
+}
+
+define <4 x i64>@test_int_x86_avx512_mask_prorv_q_256_old(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_prorv_q_256_old:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprorvq %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x14,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprorvq %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x14,0xd1]
-; X86-NEXT:    vprorvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x14,0xc1]
-; X86-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
-; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask_prorv_q_256:
+; X64-LABEL: test_int_x86_avx512_mask_prorv_q_256_old:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprorvq %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x14,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vprorvq %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x14,0xd1]
-; X64-NEXT:    vprorvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x14,0xc1]
-; X64-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
-; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx512.mask.prorv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3)
-  %res1 = call <4 x i64> @llvm.x86.avx512.mask.prorv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <4 x i64> @llvm.x86.avx512.mask.prorv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 -1)
-  %res3 = add <4 x i64> %res, %res1
-  %res4 = add <4 x i64> %res3, %res2
-  ret <4 x i64> %res4
+  ret <4 x i64> %res
+}
+
+define <4 x i64>@test_int_x86_avx512_maskz_prorv_q_256_old(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_prorv_q_256_old:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vprorvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x14,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_prorv_q_256_old:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vprorvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x14,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.prorv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> zeroinitializer, i8 %x3)
+  ret <4 x i64> %res
 }
 
 declare <4 x i32> @llvm.x86.avx512.mask.prol.d.128(<4 x i32>, i32, <4 x i32>, i8)
@@ -13526,27 +14011,27 @@ declare <4 x i32> @llvm.x86.avx512.mask.prol.d.128(<4 x i32>, i32, <4 x i32>, i8
 define <4 x i32>@test_int_x86_avx512_mask_prol_d_128(<4 x i32> %x0, i32 %x1, <4 x i32> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_prol_d_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprold $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0x6d,0x08,0x72,0xc8,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprold $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x09,0x72,0xc8,0x03]
-; X86-NEXT:    vprold $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x72,0xc8,0x03]
-; X86-NEXT:    vpaddd %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc2]
+; X86-NEXT:    vprold $4, %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0x89,0x72,0xc8,0x04]
+; X86-NEXT:    vprold $5, %xmm0, %xmm0 # encoding: [0x62,0xf1,0x7d,0x08,0x72,0xc8,0x05]
+; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_prol_d_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprold $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0x6d,0x08,0x72,0xc8,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vprold $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x09,0x72,0xc8,0x03]
-; X64-NEXT:    vprold $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x72,0xc8,0x03]
-; X64-NEXT:    vpaddd %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc2]
+; X64-NEXT:    vprold $4, %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0x89,0x72,0xc8,0x04]
+; X64-NEXT:    vprold $5, %xmm0, %xmm0 # encoding: [0x62,0xf1,0x7d,0x08,0x72,0xc8,0x05]
+; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx512.mask.prol.d.128(<4 x i32> %x0, i32 3, <4 x i32> %x2, i8 %x3)
-  %res1 = call <4 x i32> @llvm.x86.avx512.mask.prol.d.128(<4 x i32> %x0, i32 3, <4 x i32> zeroinitializer, i8 %x3)
-  %res2 = call <4 x i32> @llvm.x86.avx512.mask.prol.d.128(<4 x i32> %x0, i32 3, <4 x i32> %x2, i8 -1)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.prol.d.128(<4 x i32> %x0, i32 4, <4 x i32> zeroinitializer, i8 %x3)
+  %res2 = call <4 x i32> @llvm.x86.avx512.mask.prol.d.128(<4 x i32> %x0, i32 5, <4 x i32> %x2, i8 -1)
   %res3 = add <4 x i32> %res, %res1
   %res4 = add <4 x i32> %res3, %res2
   ret <4 x i32> %res4
@@ -13557,27 +14042,27 @@ declare <8 x i32> @llvm.x86.avx512.mask.prol.d.256(<8 x i32>, i32, <8 x i32>, i8
 define <8 x i32>@test_int_x86_avx512_mask_prol_d_256(<8 x i32> %x0, i32 %x1, <8 x i32> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_prol_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprold $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0x6d,0x28,0x72,0xc8,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprold $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x75,0x29,0x72,0xc8,0x03]
-; X86-NEXT:    vprold $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x72,0xc8,0x03]
-; X86-NEXT:    vpaddd %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc2]
+; X86-NEXT:    vprold $4, %ymm0, %ymm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0xa9,0x72,0xc8,0x04]
+; X86-NEXT:    vprold $5, %ymm0, %ymm0 # encoding: [0x62,0xf1,0x7d,0x28,0x72,0xc8,0x05]
+; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_prol_d_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprold $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0x6d,0x28,0x72,0xc8,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vprold $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x75,0x29,0x72,0xc8,0x03]
-; X64-NEXT:    vprold $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x72,0xc8,0x03]
-; X64-NEXT:    vpaddd %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc2]
+; X64-NEXT:    vprold $4, %ymm0, %ymm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0xa9,0x72,0xc8,0x04]
+; X64-NEXT:    vprold $5, %ymm0, %ymm0 # encoding: [0x62,0xf1,0x7d,0x28,0x72,0xc8,0x05]
+; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx512.mask.prol.d.256(<8 x i32> %x0, i32 3, <8 x i32> %x2, i8 %x3)
-  %res1 = call <8 x i32> @llvm.x86.avx512.mask.prol.d.256(<8 x i32> %x0, i32 3, <8 x i32> zeroinitializer, i8 %x3)
-  %res2 = call <8 x i32> @llvm.x86.avx512.mask.prol.d.256(<8 x i32> %x0, i32 3, <8 x i32> %x2, i8 -1)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.prol.d.256(<8 x i32> %x0, i32 4, <8 x i32> zeroinitializer, i8 %x3)
+  %res2 = call <8 x i32> @llvm.x86.avx512.mask.prol.d.256(<8 x i32> %x0, i32 5, <8 x i32> %x2, i8 -1)
   %res3 = add <8 x i32> %res, %res1
   %res4 = add <8 x i32> %res3, %res2
   ret <8 x i32> %res4
@@ -13588,27 +14073,27 @@ declare <2 x i64> @llvm.x86.avx512.mask.prol.q.128(<2 x i64>, i32, <2 x i64>, i8
 define <2 x i64>@test_int_x86_avx512_mask_prol_q_128(<2 x i64> %x0, i32 %x1, <2 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_prol_q_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprolq $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0xed,0x08,0x72,0xc8,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprolq $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x09,0x72,0xc8,0x03]
-; X86-NEXT:    vprolq $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0x72,0xc8,0x03]
-; X86-NEXT:    vpaddq %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc2]
+; X86-NEXT:    vprolq $4, %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0xed,0x89,0x72,0xc8,0x04]
+; X86-NEXT:    vprolq $5, %xmm0, %xmm0 # encoding: [0x62,0xf1,0xfd,0x08,0x72,0xc8,0x05]
+; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_prol_q_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprolq $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0xed,0x08,0x72,0xc8,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vprolq $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x09,0x72,0xc8,0x03]
-; X64-NEXT:    vprolq $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0x72,0xc8,0x03]
-; X64-NEXT:    vpaddq %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc2]
+; X64-NEXT:    vprolq $4, %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0xed,0x89,0x72,0xc8,0x04]
+; X64-NEXT:    vprolq $5, %xmm0, %xmm0 # encoding: [0x62,0xf1,0xfd,0x08,0x72,0xc8,0x05]
+; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx512.mask.prol.q.128(<2 x i64> %x0, i32 3, <2 x i64> %x2, i8 %x3)
-  %res1 = call <2 x i64> @llvm.x86.avx512.mask.prol.q.128(<2 x i64> %x0, i32 3, <2 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <2 x i64> @llvm.x86.avx512.mask.prol.q.128(<2 x i64> %x0, i32 3, <2 x i64> %x2, i8 -1)
+  %res1 = call <2 x i64> @llvm.x86.avx512.mask.prol.q.128(<2 x i64> %x0, i32 4, <2 x i64> zeroinitializer, i8 %x3)
+  %res2 = call <2 x i64> @llvm.x86.avx512.mask.prol.q.128(<2 x i64> %x0, i32 5, <2 x i64> %x2, i8 -1)
   %res3 = add <2 x i64> %res, %res1
   %res4 = add <2 x i64> %res3, %res2
   ret <2 x i64> %res4
@@ -13619,27 +14104,27 @@ declare <4 x i64> @llvm.x86.avx512.mask.prol.q.256(<4 x i64>, i32, <4 x i64>, i8
 define <4 x i64>@test_int_x86_avx512_mask_prol_q_256(<4 x i64> %x0, i32 %x1, <4 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_prol_q_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprolq $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0xed,0x28,0x72,0xc8,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprolq $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x29,0x72,0xc8,0x03]
-; X86-NEXT:    vprolq $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0x72,0xc8,0x03]
-; X86-NEXT:    vpaddq %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc2]
+; X86-NEXT:    vprolq $4, %ymm0, %ymm2 {%k1} {z} # encoding: [0x62,0xf1,0xed,0xa9,0x72,0xc8,0x04]
+; X86-NEXT:    vprolq $5, %ymm0, %ymm0 # encoding: [0x62,0xf1,0xfd,0x28,0x72,0xc8,0x05]
+; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_prol_q_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprolq $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0xed,0x28,0x72,0xc8,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vprolq $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x29,0x72,0xc8,0x03]
-; X64-NEXT:    vprolq $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0x72,0xc8,0x03]
-; X64-NEXT:    vpaddq %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc2]
+; X64-NEXT:    vprolq $4, %ymm0, %ymm2 {%k1} {z} # encoding: [0x62,0xf1,0xed,0xa9,0x72,0xc8,0x04]
+; X64-NEXT:    vprolq $5, %ymm0, %ymm0 # encoding: [0x62,0xf1,0xfd,0x28,0x72,0xc8,0x05]
+; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx512.mask.prol.q.256(<4 x i64> %x0, i32 3, <4 x i64> %x2, i8 %x3)
-  %res1 = call <4 x i64> @llvm.x86.avx512.mask.prol.q.256(<4 x i64> %x0, i32 3, <4 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <4 x i64> @llvm.x86.avx512.mask.prol.q.256(<4 x i64> %x0, i32 3, <4 x i64> %x2, i8 -1)
+  %res1 = call <4 x i64> @llvm.x86.avx512.mask.prol.q.256(<4 x i64> %x0, i32 4, <4 x i64> zeroinitializer, i8 %x3)
+  %res2 = call <4 x i64> @llvm.x86.avx512.mask.prol.q.256(<4 x i64> %x0, i32 5, <4 x i64> %x2, i8 -1)
   %res3 = add <4 x i64> %res, %res1
   %res4 = add <4 x i64> %res3, %res2
   ret <4 x i64> %res4
@@ -13647,126 +14132,190 @@ define <4 x i64>@test_int_x86_avx512_mask_prol_q_256(<4 x i64> %x0, i32 %x1, <4
 
 declare <4 x i32> @llvm.x86.avx512.mask.prolv.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
 
-define <4 x i32>@test_int_x86_avx512_mask_prolv_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask_prolv_d_128:
+define <4 x i32>@test_int_x86_avx512_prolv_d_128_old(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_prolv_d_128_old:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vprolvd %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7d,0x08,0x15,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.prolv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32>@test_int_x86_avx512_mask_prolv_d_128_old(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_prolv_d_128_old:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprolvd %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0x7d,0x08,0x15,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprolvd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x15,0xd1]
-; X86-NEXT:    vprolvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x15,0xc1]
-; X86-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
-; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask_prolv_d_128:
+; X64-LABEL: test_int_x86_avx512_mask_prolv_d_128_old:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprolvd %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0x7d,0x08,0x15,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vprolvd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x15,0xd1]
-; X64-NEXT:    vprolvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x15,0xc1]
-; X64-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
-; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx512.mask.prolv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3)
-  %res1 = call <4 x i32> @llvm.x86.avx512.mask.prolv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> zeroinitializer, i8 %x3)
-  %res2 = call <4 x i32> @llvm.x86.avx512.mask.prolv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1)
-  %res3 = add <4 x i32> %res, %res1
-  %res4 = add <4 x i32> %res3, %res2
-  ret <4 x i32> %res4
+  ret <4 x i32> %res
+}
+
+define <4 x i32>@test_int_x86_avx512_maskz_prolv_d_128_old(<4 x i32> %x0, <4 x i32> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_prolv_d_128_old:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vprolvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x15,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_prolv_d_128_old:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vprolvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x15,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.prolv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> zeroinitializer, i8 %x3)
+  ret <4 x i32> %res
 }
 
 declare <8 x i32> @llvm.x86.avx512.mask.prolv.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
 
-define <8 x i32>@test_int_x86_avx512_mask_prolv_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask_prolv_d_256:
+define <8 x i32>@test_int_x86_avx512_prolv_d_256_old(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_prolv_d_256_old:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vprolvd %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf2,0x7d,0x28,0x15,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.prolv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1)
+  ret <8 x i32> %res
+}
+
+define <8 x i32>@test_int_x86_avx512_mask_prolv_d_256_old(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_prolv_d_256_old:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprolvd %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0x7d,0x28,0x15,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprolvd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x15,0xd1]
-; X86-NEXT:    vprolvd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x15,0xc1]
-; X86-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
-; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask_prolv_d_256:
+; X64-LABEL: test_int_x86_avx512_mask_prolv_d_256_old:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprolvd %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0x7d,0x28,0x15,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vprolvd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x15,0xd1]
-; X64-NEXT:    vprolvd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x15,0xc1]
-; X64-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
-; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx512.mask.prolv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
-  %res1 = call <8 x i32> @llvm.x86.avx512.mask.prolv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> zeroinitializer, i8 %x3)
-  %res2 = call <8 x i32> @llvm.x86.avx512.mask.prolv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1)
-  %res3 = add <8 x i32> %res, %res1
-  %res4 = add <8 x i32> %res3, %res2
-  ret <8 x i32> %res4
+  ret <8 x i32> %res
+}
+
+define <8 x i32>@test_int_x86_avx512_maskz_prolv_d_256_old(<8 x i32> %x0, <8 x i32> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_prolv_d_256_old:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vprolvd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x15,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_prolv_d_256_old:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vprolvd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x15,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.prolv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> zeroinitializer, i8 %x3)
+  ret <8 x i32> %res
 }
 
 declare <2 x i64> @llvm.x86.avx512.mask.prolv.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
 
-define <2 x i64>@test_int_x86_avx512_mask_prolv_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask_prolv_q_128:
+define <2 x i64>@test_int_x86_avx512_prolv_q_128_old(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_prolv_q_128_old:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vprolvq %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x15,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.prolv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1)
+  ret <2 x i64> %res
+}
+
+define <2 x i64>@test_int_x86_avx512_mask_prolv_q_128_old(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_prolv_q_128_old:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprolvq %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x15,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprolvq %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x15,0xd1]
-; X86-NEXT:    vprolvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x15,0xc1]
-; X86-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
-; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask_prolv_q_128:
+; X64-LABEL: test_int_x86_avx512_mask_prolv_q_128_old:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprolvq %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x15,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vprolvq %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x15,0xd1]
-; X64-NEXT:    vprolvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x15,0xc1]
-; X64-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
-; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx512.mask.prolv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3)
-  %res1 = call <2 x i64> @llvm.x86.avx512.mask.prolv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <2 x i64> @llvm.x86.avx512.mask.prolv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 -1)
-  %res3 = add <2 x i64> %res, %res1
-  %res4 = add <2 x i64> %res3, %res2
-  ret <2 x i64> %res4
+  ret <2 x i64> %res
+}
+
+define <2 x i64>@test_int_x86_avx512_maskz_prolv_q_128_old(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_prolv_q_128_old:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vprolvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x15,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_prolv_q_128_old:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vprolvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x15,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.prolv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> zeroinitializer, i8 %x3)
+  ret <2 x i64> %res
 }
 
 declare <4 x i64> @llvm.x86.avx512.mask.prolv.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
 
-define <4 x i64>@test_int_x86_avx512_mask_prolv_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask_prolv_q_256:
+define <4 x i64>@test_int_x86_avx512_prolv_q_256_old(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_prolv_q_256_old:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vprolvq %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x15,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.prolv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 -1)
+  ret <4 x i64> %res
+}
+
+define <4 x i64>@test_int_x86_avx512_mask_prolv_q_256_old(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_prolv_q_256_old:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprolvq %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x15,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprolvq %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x15,0xd1]
-; X86-NEXT:    vprolvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x15,0xc1]
-; X86-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
-; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask_prolv_q_256:
+; X64-LABEL: test_int_x86_avx512_mask_prolv_q_256_old:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprolvq %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x15,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vprolvq %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x15,0xd1]
-; X64-NEXT:    vprolvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x15,0xc1]
-; X64-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
-; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx512.mask.prolv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3)
-  %res1 = call <4 x i64> @llvm.x86.avx512.mask.prolv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <4 x i64> @llvm.x86.avx512.mask.prolv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 -1)
-  %res3 = add <4 x i64> %res, %res1
-  %res4 = add <4 x i64> %res3, %res2
-  ret <4 x i64> %res4
+  ret <4 x i64> %res
+}
+
+define <4 x i64>@test_int_x86_avx512_maskz_prolv_q_256_old(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_prolv_q_256_old:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vprolvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x15,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_prolv_q_256_old:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vprolvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x15,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.prolv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> zeroinitializer, i8 %x3)
+  ret <4 x i64> %res
 }
 
 declare <4 x i32> @llvm.x86.avx512.mask.pror.d.128(<4 x i32>, i32, <4 x i32>, i8)
@@ -13774,27 +14323,27 @@ declare <4 x i32> @llvm.x86.avx512.mask.pror.d.128(<4 x i32>, i32, <4 x i32>, i8
 define <4 x i32>@test_int_x86_avx512_mask_pror_d_128(<4 x i32> %x0, i32 %x1, <4 x i32> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_pror_d_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprord $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0x6d,0x08,0x72,0xc0,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprord $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x09,0x72,0xc0,0x03]
-; X86-NEXT:    vprord $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x72,0xc0,0x03]
-; X86-NEXT:    vpaddd %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc2]
+; X86-NEXT:    vprord $4, %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0x89,0x72,0xc0,0x04]
+; X86-NEXT:    vprord $5, %xmm0, %xmm0 # encoding: [0x62,0xf1,0x7d,0x08,0x72,0xc0,0x05]
+; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pror_d_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprord $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0x6d,0x08,0x72,0xc0,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vprord $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x09,0x72,0xc0,0x03]
-; X64-NEXT:    vprord $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x72,0xc0,0x03]
-; X64-NEXT:    vpaddd %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc2]
+; X64-NEXT:    vprord $4, %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0x89,0x72,0xc0,0x04]
+; X64-NEXT:    vprord $5, %xmm0, %xmm0 # encoding: [0x62,0xf1,0x7d,0x08,0x72,0xc0,0x05]
+; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx512.mask.pror.d.128(<4 x i32> %x0, i32 3, <4 x i32> %x2, i8 %x3)
-  %res1 = call <4 x i32> @llvm.x86.avx512.mask.pror.d.128(<4 x i32> %x0, i32 3, <4 x i32> zeroinitializer, i8 %x3)
-  %res2 = call <4 x i32> @llvm.x86.avx512.mask.pror.d.128(<4 x i32> %x0, i32 3, <4 x i32> %x2, i8 -1)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.pror.d.128(<4 x i32> %x0, i32 4, <4 x i32> zeroinitializer, i8 %x3)
+  %res2 = call <4 x i32> @llvm.x86.avx512.mask.pror.d.128(<4 x i32> %x0, i32 5, <4 x i32> %x2, i8 -1)
   %res3 = add <4 x i32> %res, %res1
   %res4 = add <4 x i32> %res3, %res2
   ret <4 x i32> %res4
@@ -13805,27 +14354,27 @@ declare <8 x i32> @llvm.x86.avx512.mask.pror.d.256(<8 x i32>, i32, <8 x i32>, i8
 define <8 x i32>@test_int_x86_avx512_mask_pror_d_256(<8 x i32> %x0, i32 %x1, <8 x i32> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_pror_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprord $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0x6d,0x28,0x72,0xc0,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprord $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x75,0x29,0x72,0xc0,0x03]
-; X86-NEXT:    vprord $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x72,0xc0,0x03]
-; X86-NEXT:    vpaddd %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc2]
+; X86-NEXT:    vprord $4, %ymm0, %ymm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0xa9,0x72,0xc0,0x04]
+; X86-NEXT:    vprord $5, %ymm0, %ymm0 # encoding: [0x62,0xf1,0x7d,0x28,0x72,0xc0,0x05]
+; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pror_d_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprord $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0x6d,0x28,0x72,0xc0,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vprord $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x75,0x29,0x72,0xc0,0x03]
-; X64-NEXT:    vprord $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x72,0xc0,0x03]
-; X64-NEXT:    vpaddd %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc2]
+; X64-NEXT:    vprord $4, %ymm0, %ymm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0xa9,0x72,0xc0,0x04]
+; X64-NEXT:    vprord $5, %ymm0, %ymm0 # encoding: [0x62,0xf1,0x7d,0x28,0x72,0xc0,0x05]
+; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx512.mask.pror.d.256(<8 x i32> %x0, i32 3, <8 x i32> %x2, i8 %x3)
-  %res1 = call <8 x i32> @llvm.x86.avx512.mask.pror.d.256(<8 x i32> %x0, i32 3, <8 x i32> zeroinitializer, i8 %x3)
-  %res2 = call <8 x i32> @llvm.x86.avx512.mask.pror.d.256(<8 x i32> %x0, i32 3, <8 x i32> %x2, i8 -1)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.pror.d.256(<8 x i32> %x0, i32 4, <8 x i32> zeroinitializer, i8 %x3)
+  %res2 = call <8 x i32> @llvm.x86.avx512.mask.pror.d.256(<8 x i32> %x0, i32 5, <8 x i32> %x2, i8 -1)
   %res3 = add <8 x i32> %res, %res1
   %res4 = add <8 x i32> %res3, %res2
   ret <8 x i32> %res4
@@ -13836,27 +14385,27 @@ declare <2 x i64> @llvm.x86.avx512.mask.pror.q.128(<2 x i64>, i32, <2 x i64>, i8
 define <2 x i64>@test_int_x86_avx512_mask_pror_q_128(<2 x i64> %x0, i32 %x1, <2 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_pror_q_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprorq $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0xed,0x08,0x72,0xc0,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprorq $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x09,0x72,0xc0,0x03]
-; X86-NEXT:    vprorq $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0x72,0xc0,0x03]
-; X86-NEXT:    vpaddq %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc2]
+; X86-NEXT:    vprorq $4, %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0xed,0x89,0x72,0xc0,0x04]
+; X86-NEXT:    vprorq $5, %xmm0, %xmm0 # encoding: [0x62,0xf1,0xfd,0x08,0x72,0xc0,0x05]
+; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pror_q_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprorq $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0xed,0x08,0x72,0xc0,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vprorq $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x09,0x72,0xc0,0x03]
-; X64-NEXT:    vprorq $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0x72,0xc0,0x03]
-; X64-NEXT:    vpaddq %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc2]
+; X64-NEXT:    vprorq $4, %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0xed,0x89,0x72,0xc0,0x04]
+; X64-NEXT:    vprorq $5, %xmm0, %xmm0 # encoding: [0x62,0xf1,0xfd,0x08,0x72,0xc0,0x05]
+; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx512.mask.pror.q.128(<2 x i64> %x0, i32 3, <2 x i64> %x2, i8 %x3)
-  %res1 = call <2 x i64> @llvm.x86.avx512.mask.pror.q.128(<2 x i64> %x0, i32 3, <2 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <2 x i64> @llvm.x86.avx512.mask.pror.q.128(<2 x i64> %x0, i32 3, <2 x i64> %x2, i8 -1)
+  %res1 = call <2 x i64> @llvm.x86.avx512.mask.pror.q.128(<2 x i64> %x0, i32 4, <2 x i64> zeroinitializer, i8 %x3)
+  %res2 = call <2 x i64> @llvm.x86.avx512.mask.pror.q.128(<2 x i64> %x0, i32 5, <2 x i64> %x2, i8 -1)
   %res3 = add <2 x i64> %res, %res1
   %res4 = add <2 x i64> %res3, %res2
   ret <2 x i64> %res4
@@ -13867,27 +14416,27 @@ declare <4 x i64> @llvm.x86.avx512.mask.pror.q.256(<4 x i64>, i32, <4 x i64>, i8
 define <4 x i64>@test_int_x86_avx512_mask_pror_q_256(<4 x i64> %x0, i32 %x1, <4 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_pror_q_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprorq $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0xed,0x28,0x72,0xc0,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprorq $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x29,0x72,0xc0,0x03]
-; X86-NEXT:    vprorq $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0x72,0xc0,0x03]
-; X86-NEXT:    vpaddq %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc2]
+; X86-NEXT:    vprorq $4, %ymm0, %ymm2 {%k1} {z} # encoding: [0x62,0xf1,0xed,0xa9,0x72,0xc0,0x04]
+; X86-NEXT:    vprorq $5, %ymm0, %ymm0 # encoding: [0x62,0xf1,0xfd,0x28,0x72,0xc0,0x05]
+; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pror_q_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprorq $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0xed,0x28,0x72,0xc0,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vprorq $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x29,0x72,0xc0,0x03]
-; X64-NEXT:    vprorq $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0x72,0xc0,0x03]
-; X64-NEXT:    vpaddq %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc2]
+; X64-NEXT:    vprorq $4, %ymm0, %ymm2 {%k1} {z} # encoding: [0x62,0xf1,0xed,0xa9,0x72,0xc0,0x04]
+; X64-NEXT:    vprorq $5, %ymm0, %ymm0 # encoding: [0x62,0xf1,0xfd,0x28,0x72,0xc0,0x05]
+; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx512.mask.pror.q.256(<4 x i64> %x0, i32 3, <4 x i64> %x2, i8 %x3)
-  %res1 = call <4 x i64> @llvm.x86.avx512.mask.pror.q.256(<4 x i64> %x0, i32 3, <4 x i64> zeroinitializer, i8 %x3)
-  %res2 = call <4 x i64> @llvm.x86.avx512.mask.pror.q.256(<4 x i64> %x0, i32 3, <4 x i64> %x2, i8 -1)
+  %res1 = call <4 x i64> @llvm.x86.avx512.mask.pror.q.256(<4 x i64> %x0, i32 4, <4 x i64> zeroinitializer, i8 %x3)
+  %res2 = call <4 x i64> @llvm.x86.avx512.mask.pror.q.256(<4 x i64> %x0, i32 5, <4 x i64> %x2, i8 -1)
   %res3 = add <4 x i64> %res, %res1
   %res4 = add <4 x i64> %res3, %res2
   ret <4 x i64> %res4
@@ -13895,148 +14444,212 @@ define <4 x i64>@test_int_x86_avx512_mask_pror_q_256(<4 x i64> %x0, i32 %x1, <4
 
 declare <4 x i32> @llvm.x86.avx512.prorv.d.128(<4 x i32>, <4 x i32>)
 
-define <4 x i32>@test_int_x86_avx512_prorv_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_prorv_d_128:
+define <4 x i32>@test_int_x86_avx512_prorv_d_128(<4 x i32> %x0, <4 x i32> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_prorv_d_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vprorvd %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7d,0x08,0x14,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <4 x i32> @llvm.x86.avx512.prorv.d.128(<4 x i32> %x0, <4 x i32> %x1)
+  ret <4 x i32> %1
+}
+
+define <4 x i32>@test_int_x86_avx512_mask_prorv_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_prorv_d_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprorvd %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0x7d,0x08,0x14,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprorvd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x14,0xd1]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_prorv_d_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vprorvd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x14,0xd1]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <4 x i32> @llvm.x86.avx512.prorv.d.128(<4 x i32> %x0, <4 x i32> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract1, <4 x i32> %1, <4 x i32> %x2
+  ret <4 x i32> %3
+}
+
+define <4 x i32>@test_int_x86_avx512_maskz_prorv_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_prorv_d_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprorvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x14,0xc1]
-; X86-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
-; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_prorv_d_128:
+; X64-LABEL: test_int_x86_avx512_maskz_prorv_d_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vprorvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x14,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <4 x i32> @llvm.x86.avx512.prorv.d.128(<4 x i32> %x0, <4 x i32> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract1, <4 x i32> %1, <4 x i32> zeroinitializer
+  ret <4 x i32> %3
+}
+
+declare <8 x i32> @llvm.x86.avx512.prorv.d.256(<8 x i32>, <8 x i32>)
+
+define <8 x i32>@test_int_x86_avx512_prorv_d_256(<8 x i32> %x0, <8 x i32> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_prorv_d_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vprorvd %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf2,0x7d,0x28,0x14,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <8 x i32> @llvm.x86.avx512.prorv.d.256(<8 x i32> %x0, <8 x i32> %x1)
+  ret <8 x i32> %1
+}
+
+define <8 x i32>@test_int_x86_avx512_mask_prorv_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_prorv_d_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vprorvd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x14,0xd1]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_prorv_d_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprorvd %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0x7d,0x08,0x14,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vprorvd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x14,0xd1]
-; X64-NEXT:    vprorvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x14,0xc1]
-; X64-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
-; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
+; X64-NEXT:    vprorvd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x14,0xd1]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <4 x i32> @llvm.x86.avx512.prorv.d.128(<4 x i32> %x0, <4 x i32> %x1)
+  %1 = call <8 x i32> @llvm.x86.avx512.prorv.d.256(<8 x i32> %x0, <8 x i32> %x1)
   %2 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %3 = select <4 x i1> %extract1, <4 x i32> %1, <4 x i32> %x2
-  %4 = call <4 x i32> @llvm.x86.avx512.prorv.d.128(<4 x i32> %x0, <4 x i32> %x1)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %6 = select <4 x i1> %extract, <4 x i32> %4, <4 x i32> zeroinitializer
-  %7 = call <4 x i32> @llvm.x86.avx512.prorv.d.128(<4 x i32> %x0, <4 x i32> %x1)
-  %res3 = add <4 x i32> %3, %6
-  %res4 = add <4 x i32> %res3, %7
-  ret <4 x i32> %res4
+  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x2
+  ret <8 x i32> %3
 }
 
-declare <8 x i32> @llvm.x86.avx512.prorv.d.256(<8 x i32>, <8 x i32>)
-
-define <8 x i32>@test_int_x86_avx512_prorv_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_prorv_d_256:
+define <8 x i32>@test_int_x86_avx512_maskz_prorv_d_256(<8 x i32> %x0, <8 x i32> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_prorv_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprorvd %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0x7d,0x28,0x14,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vprorvd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x14,0xd1]
 ; X86-NEXT:    vprorvd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x14,0xc1]
-; X86-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
-; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_prorv_d_256:
+; X64-LABEL: test_int_x86_avx512_maskz_prorv_d_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprorvd %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0x7d,0x28,0x14,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vprorvd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x14,0xd1]
 ; X64-NEXT:    vprorvd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x14,0xc1]
-; X64-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
-; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <8 x i32> @llvm.x86.avx512.prorv.d.256(<8 x i32> %x0, <8 x i32> %x1)
   %2 = bitcast i8 %x3 to <8 x i1>
-  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x2
-  %4 = call <8 x i32> @llvm.x86.avx512.prorv.d.256(<8 x i32> %x0, <8 x i32> %x1)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %6 = select <8 x i1> %5, <8 x i32> %4, <8 x i32> zeroinitializer
-  %7 = call <8 x i32> @llvm.x86.avx512.prorv.d.256(<8 x i32> %x0, <8 x i32> %x1)
-  %res3 = add <8 x i32> %3, %6
-  %res4 = add <8 x i32> %res3, %7
-  ret <8 x i32> %res4
+  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> zeroinitializer
+  ret <8 x i32> %3
 }
 
 declare <2 x i64> @llvm.x86.avx512.prorv.q.128(<2 x i64>, <2 x i64>)
 
-define <2 x i64>@test_int_x86_avx512_prorv_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_prorv_q_128:
+define <2 x i64>@test_int_x86_avx512_prorv_q_128(<2 x i64> %x0, <2 x i64> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_prorv_q_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vprorvq %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x14,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <2 x i64> @llvm.x86.avx512.prorv.q.128(<2 x i64> %x0, <2 x i64> %x1)
+  ret <2 x i64> %1
+}
+
+define <2 x i64>@test_int_x86_avx512_mask_prorv_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_prorv_q_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprorvq %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x14,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprorvq %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x14,0xd1]
-; X86-NEXT:    vprorvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x14,0xc1]
-; X86-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
-; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_prorv_q_128:
+; X64-LABEL: test_int_x86_avx512_mask_prorv_q_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprorvq %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x14,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vprorvq %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x14,0xd1]
-; X64-NEXT:    vprorvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x14,0xc1]
-; X64-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
-; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <2 x i64> @llvm.x86.avx512.prorv.q.128(<2 x i64> %x0, <2 x i64> %x1)
   %2 = bitcast i8 %x3 to <8 x i1>
   %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
   %3 = select <2 x i1> %extract1, <2 x i64> %1, <2 x i64> %x2
-  %4 = call <2 x i64> @llvm.x86.avx512.prorv.q.128(<2 x i64> %x0, <2 x i64> %x1)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <2 x i32> <i32 0, i32 1>
-  %6 = select <2 x i1> %extract, <2 x i64> %4, <2 x i64> zeroinitializer
-  %7 = call <2 x i64> @llvm.x86.avx512.prorv.q.128(<2 x i64> %x0, <2 x i64> %x1)
-  %res3 = add <2 x i64> %3, %6
-  %res4 = add <2 x i64> %res3, %7
-  ret <2 x i64> %res4
+  ret <2 x i64> %3
+}
+
+define <2 x i64>@test_int_x86_avx512_maskz_prorv_q_128(<2 x i64> %x0, <2 x i64> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_prorv_q_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vprorvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x14,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_prorv_q_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vprorvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x14,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <2 x i64> @llvm.x86.avx512.prorv.q.128(<2 x i64> %x0, <2 x i64> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
+  %3 = select <2 x i1> %extract1, <2 x i64> %1, <2 x i64> zeroinitializer
+  ret <2 x i64> %3
 }
 
 declare <4 x i64> @llvm.x86.avx512.prorv.q.256(<4 x i64>, <4 x i64>)
 
-define <4 x i64>@test_int_x86_avx512_prorv_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_prorv_q_256:
+define <4 x i64>@test_int_x86_avx512_prorv_q_256(<4 x i64> %x0, <4 x i64> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_prorv_q_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vprorvq %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x14,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <4 x i64> @llvm.x86.avx512.prorv.q.256(<4 x i64> %x0, <4 x i64> %x1)
+  ret <4 x i64> %1
+}
+
+define <4 x i64>@test_int_x86_avx512_mask_prorv_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_prorv_q_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprorvq %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x14,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprorvq %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x14,0xd1]
-; X86-NEXT:    vprorvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x14,0xc1]
-; X86-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
-; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_prorv_q_256:
+; X64-LABEL: test_int_x86_avx512_mask_prorv_q_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprorvq %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x14,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vprorvq %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x14,0xd1]
-; X64-NEXT:    vprorvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x14,0xc1]
-; X64-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
-; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <4 x i64> @llvm.x86.avx512.prorv.q.256(<4 x i64> %x0, <4 x i64> %x1)
   %2 = bitcast i8 %x3 to <8 x i1>
   %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract1, <4 x i64> %1, <4 x i64> %x2
-  %4 = call <4 x i64> @llvm.x86.avx512.prorv.q.256(<4 x i64> %x0, <4 x i64> %x1)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %6 = select <4 x i1> %extract, <4 x i64> %4, <4 x i64> zeroinitializer
-  %7 = call <4 x i64> @llvm.x86.avx512.prorv.q.256(<4 x i64> %x0, <4 x i64> %x1)
-  %res3 = add <4 x i64> %3, %6
-  %res4 = add <4 x i64> %res3, %7
-  ret <4 x i64> %res4
+  ret <4 x i64> %3
+}
+
+define <4 x i64>@test_int_x86_avx512_maskz_prorv_q_256(<4 x i64> %x0, <4 x i64> %x1,  i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_prorv_q_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vprorvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x14,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_prorv_q_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vprorvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x14,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <4 x i64> @llvm.x86.avx512.prorv.q.256(<4 x i64> %x0, <4 x i64> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract1, <4 x i64> %1, <4 x i64> zeroinitializer
+  ret <4 x i64> %3
 }
 
 declare <4 x i32> @llvm.x86.avx512.prol.d.128(<4 x i32>, i32)
@@ -14044,33 +14657,33 @@ declare <4 x i32> @llvm.x86.avx512.prol.d.128(<4 x i32>, i32)
 define <4 x i32>@test_int_x86_avx512_prol_d_128(<4 x i32> %x0, i32 %x1, <4 x i32> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_prol_d_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprold $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0x6d,0x08,0x72,0xc8,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprold $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x09,0x72,0xc8,0x03]
-; X86-NEXT:    vprold $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x72,0xc8,0x03]
-; X86-NEXT:    vpaddd %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc2]
+; X86-NEXT:    vprold $4, %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0x89,0x72,0xc8,0x04]
+; X86-NEXT:    vprold $5, %xmm0, %xmm0 # encoding: [0x62,0xf1,0x7d,0x08,0x72,0xc8,0x05]
+; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_prol_d_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprold $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0x6d,0x08,0x72,0xc8,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vprold $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x09,0x72,0xc8,0x03]
-; X64-NEXT:    vprold $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x72,0xc8,0x03]
-; X64-NEXT:    vpaddd %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc2]
+; X64-NEXT:    vprold $4, %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0x89,0x72,0xc8,0x04]
+; X64-NEXT:    vprold $5, %xmm0, %xmm0 # encoding: [0x62,0xf1,0x7d,0x08,0x72,0xc8,0x05]
+; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <4 x i32> @llvm.x86.avx512.prol.d.128(<4 x i32> %x0, i32 3)
   %2 = bitcast i8 %x3 to <8 x i1>
   %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract1, <4 x i32> %1, <4 x i32> %x2
-  %4 = call <4 x i32> @llvm.x86.avx512.prol.d.128(<4 x i32> %x0, i32 3)
+  %4 = call <4 x i32> @llvm.x86.avx512.prol.d.128(<4 x i32> %x0, i32 4)
   %5 = bitcast i8 %x3 to <8 x i1>
   %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %6 = select <4 x i1> %extract, <4 x i32> %4, <4 x i32> zeroinitializer
-  %7 = call <4 x i32> @llvm.x86.avx512.prol.d.128(<4 x i32> %x0, i32 3)
+  %7 = call <4 x i32> @llvm.x86.avx512.prol.d.128(<4 x i32> %x0, i32 5)
   %res3 = add <4 x i32> %3, %6
   %res4 = add <4 x i32> %res3, %7
   ret <4 x i32> %res4
@@ -14081,31 +14694,31 @@ declare <8 x i32> @llvm.x86.avx512.prol.d.256(<8 x i32>, i32)
 define <8 x i32>@test_int_x86_avx512_prol_d_256(<8 x i32> %x0, i32 %x1, <8 x i32> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_prol_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprold $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0x6d,0x28,0x72,0xc8,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprold $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x75,0x29,0x72,0xc8,0x03]
-; X86-NEXT:    vprold $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x72,0xc8,0x03]
-; X86-NEXT:    vpaddd %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc2]
+; X86-NEXT:    vprold $4, %ymm0, %ymm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0xa9,0x72,0xc8,0x04]
+; X86-NEXT:    vprold $5, %ymm0, %ymm0 # encoding: [0x62,0xf1,0x7d,0x28,0x72,0xc8,0x05]
+; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_prol_d_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprold $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0x6d,0x28,0x72,0xc8,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vprold $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x75,0x29,0x72,0xc8,0x03]
-; X64-NEXT:    vprold $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x72,0xc8,0x03]
-; X64-NEXT:    vpaddd %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc2]
+; X64-NEXT:    vprold $4, %ymm0, %ymm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0xa9,0x72,0xc8,0x04]
+; X64-NEXT:    vprold $5, %ymm0, %ymm0 # encoding: [0x62,0xf1,0x7d,0x28,0x72,0xc8,0x05]
+; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <8 x i32> @llvm.x86.avx512.prol.d.256(<8 x i32> %x0, i32 3)
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x2
-  %4 = call <8 x i32> @llvm.x86.avx512.prol.d.256(<8 x i32> %x0, i32 3)
+  %4 = call <8 x i32> @llvm.x86.avx512.prol.d.256(<8 x i32> %x0, i32 4)
   %5 = bitcast i8 %x3 to <8 x i1>
   %6 = select <8 x i1> %5, <8 x i32> %4, <8 x i32> zeroinitializer
-  %7 = call <8 x i32> @llvm.x86.avx512.prol.d.256(<8 x i32> %x0, i32 3)
+  %7 = call <8 x i32> @llvm.x86.avx512.prol.d.256(<8 x i32> %x0, i32 5)
   %res3 = add <8 x i32> %3, %6
   %res4 = add <8 x i32> %res3, %7
   ret <8 x i32> %res4
@@ -14116,33 +14729,33 @@ declare <2 x i64> @llvm.x86.avx512.prol.q.128(<2 x i64>, i32)
 define <2 x i64>@test_int_x86_avx512_prol_q_128(<2 x i64> %x0, i32 %x1, <2 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_prol_q_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprolq $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0xed,0x08,0x72,0xc8,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprolq $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x09,0x72,0xc8,0x03]
-; X86-NEXT:    vprolq $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0x72,0xc8,0x03]
-; X86-NEXT:    vpaddq %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc2]
+; X86-NEXT:    vprolq $4, %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0xed,0x89,0x72,0xc8,0x04]
+; X86-NEXT:    vprolq $5, %xmm0, %xmm0 # encoding: [0x62,0xf1,0xfd,0x08,0x72,0xc8,0x05]
+; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_prol_q_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprolq $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0xed,0x08,0x72,0xc8,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vprolq $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x09,0x72,0xc8,0x03]
-; X64-NEXT:    vprolq $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0x72,0xc8,0x03]
-; X64-NEXT:    vpaddq %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc2]
+; X64-NEXT:    vprolq $4, %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0xed,0x89,0x72,0xc8,0x04]
+; X64-NEXT:    vprolq $5, %xmm0, %xmm0 # encoding: [0x62,0xf1,0xfd,0x08,0x72,0xc8,0x05]
+; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <2 x i64> @llvm.x86.avx512.prol.q.128(<2 x i64> %x0, i32 3)
   %2 = bitcast i8 %x3 to <8 x i1>
   %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
   %3 = select <2 x i1> %extract1, <2 x i64> %1, <2 x i64> %x2
-  %4 = call <2 x i64> @llvm.x86.avx512.prol.q.128(<2 x i64> %x0, i32 3)
+  %4 = call <2 x i64> @llvm.x86.avx512.prol.q.128(<2 x i64> %x0, i32 4)
   %5 = bitcast i8 %x3 to <8 x i1>
   %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <2 x i32> <i32 0, i32 1>
   %6 = select <2 x i1> %extract, <2 x i64> %4, <2 x i64> zeroinitializer
-  %7 = call <2 x i64> @llvm.x86.avx512.prol.q.128(<2 x i64> %x0, i32 3)
+  %7 = call <2 x i64> @llvm.x86.avx512.prol.q.128(<2 x i64> %x0, i32 5)
   %res3 = add <2 x i64> %3, %6
   %res4 = add <2 x i64> %res3, %7
   ret <2 x i64> %res4
@@ -14153,33 +14766,33 @@ declare <4 x i64> @llvm.x86.avx512.prol.q.256(<4 x i64>, i32)
 define <4 x i64>@test_int_x86_avx512_prol_q_256(<4 x i64> %x0, i32 %x1, <4 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_prol_q_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprolq $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0xed,0x28,0x72,0xc8,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprolq $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x29,0x72,0xc8,0x03]
-; X86-NEXT:    vprolq $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0x72,0xc8,0x03]
-; X86-NEXT:    vpaddq %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc2]
+; X86-NEXT:    vprolq $4, %ymm0, %ymm2 {%k1} {z} # encoding: [0x62,0xf1,0xed,0xa9,0x72,0xc8,0x04]
+; X86-NEXT:    vprolq $5, %ymm0, %ymm0 # encoding: [0x62,0xf1,0xfd,0x28,0x72,0xc8,0x05]
+; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_prol_q_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprolq $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0xed,0x28,0x72,0xc8,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vprolq $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x29,0x72,0xc8,0x03]
-; X64-NEXT:    vprolq $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0x72,0xc8,0x03]
-; X64-NEXT:    vpaddq %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc2]
+; X64-NEXT:    vprolq $4, %ymm0, %ymm2 {%k1} {z} # encoding: [0x62,0xf1,0xed,0xa9,0x72,0xc8,0x04]
+; X64-NEXT:    vprolq $5, %ymm0, %ymm0 # encoding: [0x62,0xf1,0xfd,0x28,0x72,0xc8,0x05]
+; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <4 x i64> @llvm.x86.avx512.prol.q.256(<4 x i64> %x0, i32 3)
   %2 = bitcast i8 %x3 to <8 x i1>
   %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract1, <4 x i64> %1, <4 x i64> %x2
-  %4 = call <4 x i64> @llvm.x86.avx512.prol.q.256(<4 x i64> %x0, i32 3)
+  %4 = call <4 x i64> @llvm.x86.avx512.prol.q.256(<4 x i64> %x0, i32 4)
   %5 = bitcast i8 %x3 to <8 x i1>
   %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %6 = select <4 x i1> %extract, <4 x i64> %4, <4 x i64> zeroinitializer
-  %7 = call <4 x i64> @llvm.x86.avx512.prol.q.256(<4 x i64> %x0, i32 3)
+  %7 = call <4 x i64> @llvm.x86.avx512.prol.q.256(<4 x i64> %x0, i32 5)
   %res3 = add <4 x i64> %3, %6
   %res4 = add <4 x i64> %res3, %7
   ret <4 x i64> %res4
@@ -14187,148 +14800,212 @@ define <4 x i64>@test_int_x86_avx512_prol_q_256(<4 x i64> %x0, i32 %x1, <4 x i64
 
 declare <4 x i32> @llvm.x86.avx512.prolv.d.128(<4 x i32>, <4 x i32>)
 
-define <4 x i32>@test_int_x86_avx512_prolv_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_prolv_d_128:
+define <4 x i32>@test_int_x86_avx512_prolv_d_128(<4 x i32> %x0, <4 x i32> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_prolv_d_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vprolvd %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7d,0x08,0x15,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <4 x i32> @llvm.x86.avx512.prolv.d.128(<4 x i32> %x0, <4 x i32> %x1)
+  ret <4 x i32> %1
+}
+
+define <4 x i32>@test_int_x86_avx512_mask_prolv_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_prolv_d_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprolvd %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0x7d,0x08,0x15,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprolvd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x15,0xd1]
-; X86-NEXT:    vprolvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x15,0xc1]
-; X86-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
-; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_prolv_d_128:
+; X64-LABEL: test_int_x86_avx512_mask_prolv_d_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprolvd %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0x7d,0x08,0x15,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vprolvd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x15,0xd1]
-; X64-NEXT:    vprolvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x15,0xc1]
-; X64-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
-; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <4 x i32> @llvm.x86.avx512.prolv.d.128(<4 x i32> %x0, <4 x i32> %x1)
   %2 = bitcast i8 %x3 to <8 x i1>
   %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract1, <4 x i32> %1, <4 x i32> %x2
-  %4 = call <4 x i32> @llvm.x86.avx512.prolv.d.128(<4 x i32> %x0, <4 x i32> %x1)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %6 = select <4 x i1> %extract, <4 x i32> %4, <4 x i32> zeroinitializer
-  %7 = call <4 x i32> @llvm.x86.avx512.prolv.d.128(<4 x i32> %x0, <4 x i32> %x1)
-  %res3 = add <4 x i32> %3, %6
-  %res4 = add <4 x i32> %res3, %7
-  ret <4 x i32> %res4
+  ret <4 x i32> %3
+}
+
+define <4 x i32>@test_int_x86_avx512_maskz_prolv_d_128(<4 x i32> %x0, <4 x i32> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_prolv_d_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vprolvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x15,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_prolv_d_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vprolvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x15,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <4 x i32> @llvm.x86.avx512.prolv.d.128(<4 x i32> %x0, <4 x i32> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract1, <4 x i32> %1, <4 x i32> zeroinitializer
+  ret <4 x i32> %3
 }
 
 declare <8 x i32> @llvm.x86.avx512.prolv.d.256(<8 x i32>, <8 x i32>)
 
-define <8 x i32>@test_int_x86_avx512_prolv_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_prolv_d_256:
+define <8 x i32>@test_int_x86_avx512_prolv_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_prolv_d_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vprolvd %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf2,0x7d,0x28,0x15,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <8 x i32> @llvm.x86.avx512.prolv.d.256(<8 x i32> %x0, <8 x i32> %x1)
+  ret <8 x i32> %1
+}
+
+define <8 x i32>@test_int_x86_avx512_mask_prolv_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_prolv_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprolvd %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0x7d,0x28,0x15,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprolvd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x15,0xd1]
-; X86-NEXT:    vprolvd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x15,0xc1]
-; X86-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
-; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_prolv_d_256:
+; X64-LABEL: test_int_x86_avx512_mask_prolv_d_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprolvd %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0x7d,0x28,0x15,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vprolvd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x15,0xd1]
-; X64-NEXT:    vprolvd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x15,0xc1]
-; X64-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
-; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <8 x i32> @llvm.x86.avx512.prolv.d.256(<8 x i32> %x0, <8 x i32> %x1)
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x2
-  %4 = call <8 x i32> @llvm.x86.avx512.prolv.d.256(<8 x i32> %x0, <8 x i32> %x1)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %6 = select <8 x i1> %5, <8 x i32> %4, <8 x i32> zeroinitializer
-  %7 = call <8 x i32> @llvm.x86.avx512.prolv.d.256(<8 x i32> %x0, <8 x i32> %x1)
-  %res3 = add <8 x i32> %3, %6
-  %res4 = add <8 x i32> %res3, %7
-  ret <8 x i32> %res4
+  ret <8 x i32> %3
+}
+
+define <8 x i32>@test_int_x86_avx512_maskz_prolv_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_prolv_d_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vprolvd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x15,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_prolv_d_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vprolvd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x15,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <8 x i32> @llvm.x86.avx512.prolv.d.256(<8 x i32> %x0, <8 x i32> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> zeroinitializer
+  ret <8 x i32> %3
 }
 
 declare <2 x i64> @llvm.x86.avx512.prolv.q.128(<2 x i64>, <2 x i64>)
 
-define <2 x i64>@test_int_x86_avx512_prolv_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_prolv_q_128:
+define <2 x i64>@test_int_x86_avx512_prolv_q_128(<2 x i64> %x0, <2 x i64> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_prolv_q_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vprolvq %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x15,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <2 x i64> @llvm.x86.avx512.prolv.q.128(<2 x i64> %x0, <2 x i64> %x1)
+  ret <2 x i64> %1
+}
+
+define <2 x i64>@test_int_x86_avx512_mask_prolv_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_prolv_q_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprolvq %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x15,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprolvq %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x15,0xd1]
-; X86-NEXT:    vprolvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x15,0xc1]
-; X86-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
-; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_prolv_q_128:
+; X64-LABEL: test_int_x86_avx512_mask_prolv_q_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprolvq %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x15,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vprolvq %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x15,0xd1]
-; X64-NEXT:    vprolvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x15,0xc1]
-; X64-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
-; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <2 x i64> @llvm.x86.avx512.prolv.q.128(<2 x i64> %x0, <2 x i64> %x1)
   %2 = bitcast i8 %x3 to <8 x i1>
   %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
   %3 = select <2 x i1> %extract1, <2 x i64> %1, <2 x i64> %x2
-  %4 = call <2 x i64> @llvm.x86.avx512.prolv.q.128(<2 x i64> %x0, <2 x i64> %x1)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <2 x i32> <i32 0, i32 1>
-  %6 = select <2 x i1> %extract, <2 x i64> %4, <2 x i64> zeroinitializer
-  %7 = call <2 x i64> @llvm.x86.avx512.prolv.q.128(<2 x i64> %x0, <2 x i64> %x1)
-  %res3 = add <2 x i64> %3, %6
-  %res4 = add <2 x i64> %res3, %7
-  ret <2 x i64> %res4
+  ret <2 x i64> %3
+}
+
+define <2 x i64>@test_int_x86_avx512_maskz_prolv_q_128(<2 x i64> %x0, <2 x i64> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_prolv_q_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vprolvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x15,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_prolv_q_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vprolvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x15,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <2 x i64> @llvm.x86.avx512.prolv.q.128(<2 x i64> %x0, <2 x i64> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
+  %3 = select <2 x i1> %extract1, <2 x i64> %1, <2 x i64> zeroinitializer
+  ret <2 x i64> %3
 }
 
 declare <4 x i64> @llvm.x86.avx512.prolv.q.256(<4 x i64>, <4 x i64>)
 
-define <4 x i64>@test_int_x86_avx512_prolv_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_prolv_q_256:
+define <4 x i64>@test_int_x86_avx512_prolv_q_256(<4 x i64> %x0, <4 x i64> %x1) {
+; CHECK-LABEL: test_int_x86_avx512_prolv_q_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vprolvq %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x15,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <4 x i64> @llvm.x86.avx512.prolv.q.256(<4 x i64> %x0, <4 x i64> %x1)
+  ret <4 x i64> %1
+}
+
+define <4 x i64>@test_int_x86_avx512_mask_prolv_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_prolv_q_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprolvq %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x15,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprolvq %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x15,0xd1]
-; X86-NEXT:    vprolvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x15,0xc1]
-; X86-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
-; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_prolv_q_256:
+; X64-LABEL: test_int_x86_avx512_mask_prolv_q_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprolvq %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x15,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vprolvq %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x15,0xd1]
-; X64-NEXT:    vprolvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x15,0xc1]
-; X64-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
-; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <4 x i64> @llvm.x86.avx512.prolv.q.256(<4 x i64> %x0, <4 x i64> %x1)
   %2 = bitcast i8 %x3 to <8 x i1>
   %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract1, <4 x i64> %1, <4 x i64> %x2
-  %4 = call <4 x i64> @llvm.x86.avx512.prolv.q.256(<4 x i64> %x0, <4 x i64> %x1)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %6 = select <4 x i1> %extract, <4 x i64> %4, <4 x i64> zeroinitializer
-  %7 = call <4 x i64> @llvm.x86.avx512.prolv.q.256(<4 x i64> %x0, <4 x i64> %x1)
-  %res3 = add <4 x i64> %3, %6
-  %res4 = add <4 x i64> %res3, %7
-  ret <4 x i64> %res4
+  ret <4 x i64> %3
+}
+
+define <4 x i64>@test_int_x86_avx512_maskz_prolv_q_256(<4 x i64> %x0, <4 x i64> %x1, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_prolv_q_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vprolvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x15,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_prolv_q_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vprolvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x15,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <4 x i64> @llvm.x86.avx512.prolv.q.256(<4 x i64> %x0, <4 x i64> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract1, <4 x i64> %1, <4 x i64> zeroinitializer
+  ret <4 x i64> %3
 }
 
 declare <4 x i32> @llvm.x86.avx512.pror.d.128(<4 x i32>, i32)
@@ -14336,33 +15013,33 @@ declare <4 x i32> @llvm.x86.avx512.pror.d.128(<4 x i32>, i32)
 define <4 x i32>@test_int_x86_avx512_pror_d_128(<4 x i32> %x0, i32 %x1, <4 x i32> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_pror_d_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprord $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0x6d,0x08,0x72,0xc0,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprord $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x09,0x72,0xc0,0x03]
-; X86-NEXT:    vprord $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x72,0xc0,0x03]
-; X86-NEXT:    vpaddd %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc2]
+; X86-NEXT:    vprord $4, %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0x89,0x72,0xc0,0x04]
+; X86-NEXT:    vprord $5, %xmm0, %xmm0 # encoding: [0x62,0xf1,0x7d,0x08,0x72,0xc0,0x05]
+; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_pror_d_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprord $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0x6d,0x08,0x72,0xc0,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vprord $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x09,0x72,0xc0,0x03]
-; X64-NEXT:    vprord $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x72,0xc0,0x03]
-; X64-NEXT:    vpaddd %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc2]
+; X64-NEXT:    vprord $4, %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0x89,0x72,0xc0,0x04]
+; X64-NEXT:    vprord $5, %xmm0, %xmm0 # encoding: [0x62,0xf1,0x7d,0x08,0x72,0xc0,0x05]
+; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <4 x i32> @llvm.x86.avx512.pror.d.128(<4 x i32> %x0, i32 3)
   %2 = bitcast i8 %x3 to <8 x i1>
   %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract1, <4 x i32> %1, <4 x i32> %x2
-  %4 = call <4 x i32> @llvm.x86.avx512.pror.d.128(<4 x i32> %x0, i32 3)
+  %4 = call <4 x i32> @llvm.x86.avx512.pror.d.128(<4 x i32> %x0, i32 4)
   %5 = bitcast i8 %x3 to <8 x i1>
   %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %6 = select <4 x i1> %extract, <4 x i32> %4, <4 x i32> zeroinitializer
-  %7 = call <4 x i32> @llvm.x86.avx512.pror.d.128(<4 x i32> %x0, i32 3)
+  %7 = call <4 x i32> @llvm.x86.avx512.pror.d.128(<4 x i32> %x0, i32 5)
   %res3 = add <4 x i32> %3, %6
   %res4 = add <4 x i32> %res3, %7
   ret <4 x i32> %res4
@@ -14373,31 +15050,31 @@ declare <8 x i32> @llvm.x86.avx512.pror.d.256(<8 x i32>, i32)
 define <8 x i32>@test_int_x86_avx512_pror_d_256(<8 x i32> %x0, i32 %x1, <8 x i32> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_pror_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprord $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0x6d,0x28,0x72,0xc0,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprord $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x75,0x29,0x72,0xc0,0x03]
-; X86-NEXT:    vprord $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x72,0xc0,0x03]
-; X86-NEXT:    vpaddd %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc2]
+; X86-NEXT:    vprord $4, %ymm0, %ymm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0xa9,0x72,0xc0,0x04]
+; X86-NEXT:    vprord $5, %ymm0, %ymm0 # encoding: [0x62,0xf1,0x7d,0x28,0x72,0xc0,0x05]
+; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_pror_d_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprord $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0x6d,0x28,0x72,0xc0,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vprord $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x75,0x29,0x72,0xc0,0x03]
-; X64-NEXT:    vprord $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x72,0xc0,0x03]
-; X64-NEXT:    vpaddd %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc2]
+; X64-NEXT:    vprord $4, %ymm0, %ymm2 {%k1} {z} # encoding: [0x62,0xf1,0x6d,0xa9,0x72,0xc0,0x04]
+; X64-NEXT:    vprord $5, %ymm0, %ymm0 # encoding: [0x62,0xf1,0x7d,0x28,0x72,0xc0,0x05]
+; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <8 x i32> @llvm.x86.avx512.pror.d.256(<8 x i32> %x0, i32 3)
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x2
-  %4 = call <8 x i32> @llvm.x86.avx512.pror.d.256(<8 x i32> %x0, i32 3)
+  %4 = call <8 x i32> @llvm.x86.avx512.pror.d.256(<8 x i32> %x0, i32 4)
   %5 = bitcast i8 %x3 to <8 x i1>
   %6 = select <8 x i1> %5, <8 x i32> %4, <8 x i32> zeroinitializer
-  %7 = call <8 x i32> @llvm.x86.avx512.pror.d.256(<8 x i32> %x0, i32 3)
+  %7 = call <8 x i32> @llvm.x86.avx512.pror.d.256(<8 x i32> %x0, i32 5)
   %res3 = add <8 x i32> %3, %6
   %res4 = add <8 x i32> %res3, %7
   ret <8 x i32> %res4
@@ -14408,33 +15085,33 @@ declare <2 x i64> @llvm.x86.avx512.pror.q.128(<2 x i64>, i32)
 define <2 x i64>@test_int_x86_avx512_pror_q_128(<2 x i64> %x0, i32 %x1, <2 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_pror_q_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprorq $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0xed,0x08,0x72,0xc0,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprorq $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x09,0x72,0xc0,0x03]
-; X86-NEXT:    vprorq $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0x72,0xc0,0x03]
-; X86-NEXT:    vpaddq %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc2]
+; X86-NEXT:    vprorq $4, %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0xed,0x89,0x72,0xc0,0x04]
+; X86-NEXT:    vprorq $5, %xmm0, %xmm0 # encoding: [0x62,0xf1,0xfd,0x08,0x72,0xc0,0x05]
+; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_pror_q_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprorq $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0xed,0x08,0x72,0xc0,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vprorq $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x09,0x72,0xc0,0x03]
-; X64-NEXT:    vprorq $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0x72,0xc0,0x03]
-; X64-NEXT:    vpaddq %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc2]
+; X64-NEXT:    vprorq $4, %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0xed,0x89,0x72,0xc0,0x04]
+; X64-NEXT:    vprorq $5, %xmm0, %xmm0 # encoding: [0x62,0xf1,0xfd,0x08,0x72,0xc0,0x05]
+; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <2 x i64> @llvm.x86.avx512.pror.q.128(<2 x i64> %x0, i32 3)
   %2 = bitcast i8 %x3 to <8 x i1>
   %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
   %3 = select <2 x i1> %extract1, <2 x i64> %1, <2 x i64> %x2
-  %4 = call <2 x i64> @llvm.x86.avx512.pror.q.128(<2 x i64> %x0, i32 3)
+  %4 = call <2 x i64> @llvm.x86.avx512.pror.q.128(<2 x i64> %x0, i32 4)
   %5 = bitcast i8 %x3 to <8 x i1>
   %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <2 x i32> <i32 0, i32 1>
   %6 = select <2 x i1> %extract, <2 x i64> %4, <2 x i64> zeroinitializer
-  %7 = call <2 x i64> @llvm.x86.avx512.pror.q.128(<2 x i64> %x0, i32 3)
+  %7 = call <2 x i64> @llvm.x86.avx512.pror.q.128(<2 x i64> %x0, i32 5)
   %res3 = add <2 x i64> %3, %6
   %res4 = add <2 x i64> %res3, %7
   ret <2 x i64> %res4
@@ -14445,33 +15122,33 @@ declare <4 x i64> @llvm.x86.avx512.pror.q.256(<4 x i64>, i32)
 define <4 x i64>@test_int_x86_avx512_pror_q_256(<4 x i64> %x0, i32 %x1, <4 x i64> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_pror_q_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vprorq $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0xed,0x28,0x72,0xc0,0x03]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vprorq $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x29,0x72,0xc0,0x03]
-; X86-NEXT:    vprorq $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0x72,0xc0,0x03]
-; X86-NEXT:    vpaddq %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc2]
+; X86-NEXT:    vprorq $4, %ymm0, %ymm2 {%k1} {z} # encoding: [0x62,0xf1,0xed,0xa9,0x72,0xc0,0x04]
+; X86-NEXT:    vprorq $5, %ymm0, %ymm0 # encoding: [0x62,0xf1,0xfd,0x28,0x72,0xc0,0x05]
+; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_pror_q_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vprorq $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0xed,0x28,0x72,0xc0,0x03]
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vprorq $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x29,0x72,0xc0,0x03]
-; X64-NEXT:    vprorq $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0x72,0xc0,0x03]
-; X64-NEXT:    vpaddq %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc2]
+; X64-NEXT:    vprorq $4, %ymm0, %ymm2 {%k1} {z} # encoding: [0x62,0xf1,0xed,0xa9,0x72,0xc0,0x04]
+; X64-NEXT:    vprorq $5, %ymm0, %ymm0 # encoding: [0x62,0xf1,0xfd,0x28,0x72,0xc0,0x05]
+; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
 ; X64-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <4 x i64> @llvm.x86.avx512.pror.q.256(<4 x i64> %x0, i32 3)
   %2 = bitcast i8 %x3 to <8 x i1>
   %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract1, <4 x i64> %1, <4 x i64> %x2
-  %4 = call <4 x i64> @llvm.x86.avx512.pror.q.256(<4 x i64> %x0, i32 3)
+  %4 = call <4 x i64> @llvm.x86.avx512.pror.q.256(<4 x i64> %x0, i32 4)
   %5 = bitcast i8 %x3 to <8 x i1>
   %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %6 = select <4 x i1> %extract, <4 x i64> %4, <4 x i64> zeroinitializer
-  %7 = call <4 x i64> @llvm.x86.avx512.pror.q.256(<4 x i64> %x0, i32 3)
+  %7 = call <4 x i64> @llvm.x86.avx512.pror.q.256(<4 x i64> %x0, i32 5)
   %res3 = add <4 x i64> %3, %6
   %res4 = add <4 x i64> %res3, %7
   ret <4 x i64> %res4
diff --git a/llvm/test/CodeGen/X86/avx512vl-intrinsics.ll b/llvm/test/CodeGen/X86/avx512vl-intrinsics.ll
index 2d2f88db3d93c..6fb460b5cc665 100644
--- a/llvm/test/CodeGen/X86/avx512vl-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512vl-intrinsics.ll
@@ -1538,102 +1538,122 @@ define <4 x i64>@test_int_x86_avx512_maskz_vpermt2var_q_256(<4 x i64> %x0, <4 x
 
 declare <2 x double> @llvm.x86.avx512.mask.scalef.pd.128(<2 x double>, <2 x double>, <2 x double>, i8)
 
+define <2 x double>@test_int_x86_avx512_scalef_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_scalef_pd_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vscalefpd %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x2c,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <2 x double> @llvm.x86.avx512.mask.scalef.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1)
+  ret <2 x double> %res
+}
+
 define <2 x double>@test_int_x86_avx512_mask_scalef_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_scalef_pd_128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vscalefpd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x2c,0xd1]
-; X86-NEXT:    vscalefpd %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x2c,0xc1]
-; X86-NEXT:    vaddpd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0x58,0xc0]
+; X86-NEXT:    vmovapd %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x28,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_scalef_pd_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vscalefpd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x2c,0xd1]
-; X64-NEXT:    vscalefpd %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x2c,0xc1]
-; X64-NEXT:    vaddpd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0x58,0xc0]
+; X64-NEXT:    vmovapd %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x28,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x double> @llvm.x86.avx512.mask.scalef.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3)
-  %res1 = call <2 x double> @llvm.x86.avx512.mask.scalef.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1)
-  %res2 = fadd <2 x double> %res, %res1
-  ret <2 x double> %res2
+  ret <2 x double> %res
 }
 
 declare <4 x double> @llvm.x86.avx512.mask.scalef.pd.256(<4 x double>, <4 x double>, <4 x double>, i8)
 
+define <4 x double>@test_int_x86_avx512_scalef_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_scalef_pd_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vscalefpd %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x2c,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x double> @llvm.x86.avx512.mask.scalef.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 -1)
+  ret <4 x double> %res
+}
+
 define <4 x double>@test_int_x86_avx512_mask_scalef_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_scalef_pd_256:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vscalefpd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x2c,0xd1]
-; X86-NEXT:    vscalefpd %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x2c,0xc1]
-; X86-NEXT:    vaddpd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0x58,0xc0]
+; X86-NEXT:    vmovapd %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x28,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_scalef_pd_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vscalefpd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x2c,0xd1]
-; X64-NEXT:    vscalefpd %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x2c,0xc1]
-; X64-NEXT:    vaddpd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0x58,0xc0]
+; X64-NEXT:    vmovapd %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x28,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x double> @llvm.x86.avx512.mask.scalef.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3)
-  %res1 = call <4 x double> @llvm.x86.avx512.mask.scalef.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 -1)
-  %res2 = fadd <4 x double> %res, %res1
-  ret <4 x double> %res2
+  ret <4 x double> %res
 }
 
 declare <4 x float> @llvm.x86.avx512.mask.scalef.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
 
+define <4 x float>@test_int_x86_avx512_scalef_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_scalef_ps_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vscalefps %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7d,0x08,0x2c,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x float> @llvm.x86.avx512.mask.scalef.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1)
+  ret <4 x float> %res
+}
+
 define <4 x float>@test_int_x86_avx512_mask_scalef_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_scalef_ps_128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vscalefps %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x2c,0xd1]
-; X86-NEXT:    vscalefps %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7d,0x08,0x2c,0xc1]
-; X86-NEXT:    vaddps %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe8,0x58,0xc0]
+; X86-NEXT:    vmovaps %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_scalef_ps_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vscalefps %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x2c,0xd1]
-; X64-NEXT:    vscalefps %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7d,0x08,0x2c,0xc1]
-; X64-NEXT:    vaddps %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe8,0x58,0xc0]
+; X64-NEXT:    vmovaps %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x float> @llvm.x86.avx512.mask.scalef.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3)
-  %res1 = call <4 x float> @llvm.x86.avx512.mask.scalef.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1)
-  %res2 = fadd <4 x float> %res, %res1
-  ret <4 x float> %res2
+  ret <4 x float> %res
 }
 
 declare <8 x float> @llvm.x86.avx512.mask.scalef.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
 
+define <8 x float>@test_int_x86_avx512_scalef_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_scalef_ps_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vscalefps %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf2,0x7d,0x28,0x2c,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x float> @llvm.x86.avx512.mask.scalef.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 -1)
+  ret <8 x float> %res
+}
+
 define <8 x float>@test_int_x86_avx512_mask_scalef_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_scalef_ps_256:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vscalefps %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x2c,0xd1]
-; X86-NEXT:    vscalefps %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf2,0x7d,0x28,0x2c,0xc1]
-; X86-NEXT:    vaddps %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xec,0x58,0xc0]
+; X86-NEXT:    vmovaps %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_scalef_ps_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vscalefps %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x2c,0xd1]
-; X64-NEXT:    vscalefps %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf2,0x7d,0x28,0x2c,0xc1]
-; X64-NEXT:    vaddps %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xec,0x58,0xc0]
+; X64-NEXT:    vmovaps %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x float> @llvm.x86.avx512.mask.scalef.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3)
-  %res1 = call <8 x float> @llvm.x86.avx512.mask.scalef.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 -1)
-  %res2 = fadd <8 x float> %res, %res1
-  ret <8 x float> %res2
+  ret <8 x float> %res
 }
 
 declare <16 x i8> @llvm.x86.avx512.mask.pmov.qb.128(<2 x i64>, <16 x i8>, i8)
@@ -4026,8 +4046,8 @@ define <2 x double>@test_int_x86_avx512_mask_getmant_pd_128(<2 x double> %x0, <2
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vgetmantpd $11, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x26,0xc8,0x0b]
-; X86-NEXT:    vgetmantpd $11, %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0x89,0x26,0xd0,0x0b]
-; X86-NEXT:    vgetmantpd $11, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x26,0xc0,0x0b]
+; X86-NEXT:    vgetmantpd $12, %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0x89,0x26,0xd0,0x0c]
+; X86-NEXT:    vgetmantpd $13, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x26,0xc0,0x0d]
 ; X86-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0x58,0xc0]
 ; X86-NEXT:    vaddpd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0x58,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
@@ -4035,15 +4055,15 @@ define <2 x double>@test_int_x86_avx512_mask_getmant_pd_128(<2 x double> %x0, <2
 ; X64-LABEL: test_int_x86_avx512_mask_getmant_pd_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vgetmantpd $11, %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0x89,0x26,0xd0,0x0b]
 ; X64-NEXT:    vgetmantpd $11, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x26,0xc8,0x0b]
-; X64-NEXT:    vgetmantpd $11, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x26,0xc0,0x0b]
+; X64-NEXT:    vgetmantpd $12, %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf3,0xfd,0x89,0x26,0xd0,0x0c]
+; X64-NEXT:    vgetmantpd $13, %xmm0, %xmm0 # encoding: [0x62,0xf3,0xfd,0x08,0x26,0xc0,0x0d]
 ; X64-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0x58,0xc0]
 ; X64-NEXT:    vaddpd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0x58,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x double> @llvm.x86.avx512.mask.getmant.pd.128(<2 x double> %x0, i32 11, <2 x double> %x2, i8 %x3)
-  %res2 = call <2 x double> @llvm.x86.avx512.mask.getmant.pd.128(<2 x double> %x0, i32 11, <2 x double> zeroinitializer, i8 %x3)
-  %res1 = call <2 x double> @llvm.x86.avx512.mask.getmant.pd.128(<2 x double> %x0, i32 11, <2 x double> %x2, i8 -1)
+  %res2 = call <2 x double> @llvm.x86.avx512.mask.getmant.pd.128(<2 x double> %x0, i32 12, <2 x double> zeroinitializer, i8 %x3)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.getmant.pd.128(<2 x double> %x0, i32 13, <2 x double> %x2, i8 -1)
   %res3 = fadd <2 x double> %res, %res1
   %res4 = fadd <2 x double> %res2, %res3
   ret <2 x double> %res4
@@ -4057,7 +4077,7 @@ define <4 x double>@test_int_x86_avx512_mask_getmant_pd_256(<4 x double> %x0, <4
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vgetmantpd $11, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x26,0xc8,0x0b]
-; X86-NEXT:    vgetmantpd $11, %ymm0, %ymm0 # encoding: [0x62,0xf3,0xfd,0x28,0x26,0xc0,0x0b]
+; X86-NEXT:    vgetmantpd $12, %ymm0, %ymm0 # encoding: [0x62,0xf3,0xfd,0x28,0x26,0xc0,0x0c]
 ; X86-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0x58,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -4065,11 +4085,11 @@ define <4 x double>@test_int_x86_avx512_mask_getmant_pd_256(<4 x double> %x0, <4
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vgetmantpd $11, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x26,0xc8,0x0b]
-; X64-NEXT:    vgetmantpd $11, %ymm0, %ymm0 # encoding: [0x62,0xf3,0xfd,0x28,0x26,0xc0,0x0b]
+; X64-NEXT:    vgetmantpd $12, %ymm0, %ymm0 # encoding: [0x62,0xf3,0xfd,0x28,0x26,0xc0,0x0c]
 ; X64-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0x58,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x double> @llvm.x86.avx512.mask.getmant.pd.256(<4 x double> %x0, i32 11, <4 x double> %x2, i8 %x3)
-  %res1 = call <4 x double> @llvm.x86.avx512.mask.getmant.pd.256(<4 x double> %x0, i32 11, <4 x double> %x2, i8 -1)
+  %res1 = call <4 x double> @llvm.x86.avx512.mask.getmant.pd.256(<4 x double> %x0, i32 12, <4 x double> %x2, i8 -1)
   %res2 = fadd <4 x double> %res, %res1
   ret <4 x double> %res2
 }
@@ -4082,7 +4102,7 @@ define <4 x float>@test_int_x86_avx512_mask_getmant_ps_128(<4 x float> %x0, <4 x
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vgetmantps $11, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x26,0xc8,0x0b]
-; X86-NEXT:    vgetmantps $11, %xmm0, %xmm0 # encoding: [0x62,0xf3,0x7d,0x08,0x26,0xc0,0x0b]
+; X86-NEXT:    vgetmantps $12, %xmm0, %xmm0 # encoding: [0x62,0xf3,0x7d,0x08,0x26,0xc0,0x0c]
 ; X86-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf0,0x58,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -4090,11 +4110,11 @@ define <4 x float>@test_int_x86_avx512_mask_getmant_ps_128(<4 x float> %x0, <4 x
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vgetmantps $11, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x26,0xc8,0x0b]
-; X64-NEXT:    vgetmantps $11, %xmm0, %xmm0 # encoding: [0x62,0xf3,0x7d,0x08,0x26,0xc0,0x0b]
+; X64-NEXT:    vgetmantps $12, %xmm0, %xmm0 # encoding: [0x62,0xf3,0x7d,0x08,0x26,0xc0,0x0c]
 ; X64-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf0,0x58,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x float> @llvm.x86.avx512.mask.getmant.ps.128(<4 x float> %x0, i32 11, <4 x float> %x2, i8 %x3)
-  %res1 = call <4 x float> @llvm.x86.avx512.mask.getmant.ps.128(<4 x float> %x0, i32 11, <4 x float> %x2, i8 -1)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.getmant.ps.128(<4 x float> %x0, i32 12, <4 x float> %x2, i8 -1)
   %res2 = fadd <4 x float> %res, %res1
   ret <4 x float> %res2
 }
@@ -4107,7 +4127,7 @@ define <8 x float>@test_int_x86_avx512_mask_getmant_ps_256(<8 x float> %x0, <8 x
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vgetmantps $11, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x26,0xc8,0x0b]
-; X86-NEXT:    vgetmantps $11, %ymm0, %ymm0 # encoding: [0x62,0xf3,0x7d,0x28,0x26,0xc0,0x0b]
+; X86-NEXT:    vgetmantps $12, %ymm0, %ymm0 # encoding: [0x62,0xf3,0x7d,0x28,0x26,0xc0,0x0c]
 ; X86-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf4,0x58,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -4115,43 +4135,44 @@ define <8 x float>@test_int_x86_avx512_mask_getmant_ps_256(<8 x float> %x0, <8 x
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vgetmantps $11, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x26,0xc8,0x0b]
-; X64-NEXT:    vgetmantps $11, %ymm0, %ymm0 # encoding: [0x62,0xf3,0x7d,0x28,0x26,0xc0,0x0b]
+; X64-NEXT:    vgetmantps $12, %ymm0, %ymm0 # encoding: [0x62,0xf3,0x7d,0x28,0x26,0xc0,0x0c]
 ; X64-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf4,0x58,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x float> @llvm.x86.avx512.mask.getmant.ps.256(<8 x float> %x0, i32 11, <8 x float> %x2, i8 %x3)
-  %res1 = call <8 x float> @llvm.x86.avx512.mask.getmant.ps.256(<8 x float> %x0, i32 11, <8 x float> %x2, i8 -1)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask.getmant.ps.256(<8 x float> %x0, i32 12, <8 x float> %x2, i8 -1)
   %res2 = fadd <8 x float> %res, %res1
   ret <8 x float> %res2
 }
 
 declare <4 x i32> @llvm.x86.avx512.pternlog.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i32)
 
+define <4 x i32>@test_int_x86_avx512_pternlog_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_pternlog_d_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpternlogd $33, %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf3,0x75,0x08,0x25,0xc2,0x21]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <4 x i32> @llvm.x86.avx512.pternlog.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i32 33)
+  ret <4 x i32> %1
+}
+
 define <4 x i32>@test_int_x86_avx512_mask_pternlog_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_pternlog_d_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X86-NEXT:    vpternlogd $33, %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf3,0x75,0x08,0x25,0xda,0x21]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogd $33, %xmm2, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf3,0x75,0x09,0x25,0xc2,0x21]
-; X86-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pternlog_d_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X64-NEXT:    vpternlogd $33, %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf3,0x75,0x08,0x25,0xda,0x21]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogd $33, %xmm2, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf3,0x75,0x09,0x25,0xc2,0x21]
-; X64-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <4 x i32> @llvm.x86.avx512.pternlog.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i32 33)
   %2 = bitcast i8 %x4 to <8 x i1>
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract, <4 x i32> %1, <4 x i32> %x0
-  %4 = call <4 x i32> @llvm.x86.avx512.pternlog.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i32 33)
-  %res2 = add <4 x i32> %3, %4
-  ret <4 x i32> %res2
+  ret <4 x i32> %3
 }
 
 declare <4 x i32> @llvm.x86.avx512.maskz.pternlog.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i32, i8)
@@ -4159,58 +4180,51 @@ declare <4 x i32> @llvm.x86.avx512.maskz.pternlog.d.128(<4 x i32>, <4 x i32>, <4
 define <4 x i32>@test_int_x86_avx512_maskz_pternlog_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_maskz_pternlog_d_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X86-NEXT:    vpternlogd $33, %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf3,0x75,0x08,0x25,0xda,0x21]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogd $33, %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x75,0x89,0x25,0xc2,0x21]
-; X86-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_pternlog_d_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X64-NEXT:    vpternlogd $33, %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf3,0x75,0x08,0x25,0xda,0x21]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogd $33, %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x75,0x89,0x25,0xc2,0x21]
-; X64-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <4 x i32> @llvm.x86.avx512.pternlog.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i32 33)
   %2 = bitcast i8 %x4 to <8 x i1>
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract, <4 x i32> %1, <4 x i32> zeroinitializer
-  %4 = call <4 x i32> @llvm.x86.avx512.pternlog.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i32 33)
-  %res2 = add <4 x i32> %3, %4
-  ret <4 x i32> %res2
+  ret <4 x i32> %3
 }
 
 declare <8 x i32> @llvm.x86.avx512.pternlog.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i32)
 
+define <8 x i32>@test_int_x86_avx512_pternlog_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_pternlog_d_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpternlogd $33, %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf3,0x75,0x28,0x25,0xc2,0x21]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <8 x i32> @llvm.x86.avx512.pternlog.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i32 33)
+  ret <8 x i32> %1
+}
+
 define <8 x i32>@test_int_x86_avx512_mask_pternlog_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_pternlog_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X86-NEXT:    vpternlogd $33, %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf3,0x75,0x28,0x25,0xda,0x21]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogd $33, %ymm2, %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf3,0x75,0x29,0x25,0xc2,0x21]
-; X86-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pternlog_d_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X64-NEXT:    vpternlogd $33, %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf3,0x75,0x28,0x25,0xda,0x21]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogd $33, %ymm2, %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf3,0x75,0x29,0x25,0xc2,0x21]
-; X64-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <8 x i32> @llvm.x86.avx512.pternlog.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i32 33)
   %2 = bitcast i8 %x4 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x0
-  %4 = call <8 x i32> @llvm.x86.avx512.pternlog.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i32 33)
-  %res2 = add <8 x i32> %3, %4
-  ret <8 x i32> %res2
+  ret <8 x i32> %3
 }
 
 declare <8 x i32> @llvm.x86.avx512.maskz.pternlog.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i32, i8)
@@ -4218,144 +4232,122 @@ declare <8 x i32> @llvm.x86.avx512.maskz.pternlog.d.256(<8 x i32>, <8 x i32>, <8
 define <8 x i32>@test_int_x86_avx512_maskz_pternlog_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_maskz_pternlog_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X86-NEXT:    vpternlogd $33, %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf3,0x75,0x28,0x25,0xda,0x21]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogd $33, %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf3,0x75,0xa9,0x25,0xc2,0x21]
-; X86-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_pternlog_d_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X64-NEXT:    vpternlogd $33, %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf3,0x75,0x28,0x25,0xda,0x21]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogd $33, %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf3,0x75,0xa9,0x25,0xc2,0x21]
-; X64-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <8 x i32> @llvm.x86.avx512.pternlog.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i32 33)
   %2 = bitcast i8 %x4 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> zeroinitializer
-  %4 = call <8 x i32> @llvm.x86.avx512.pternlog.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i32 33)
-  %res2 = add <8 x i32> %3, %4
-  ret <8 x i32> %res2
+  ret <8 x i32> %3
 }
 
 declare <2 x i64> @llvm.x86.avx512.pternlog.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i32)
 
+define <2 x i64>@test_int_x86_avx512_pternlog_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_pternlog_q_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpternlogq $33, %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf3,0xf5,0x08,0x25,0xc2,0x21]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <2 x i64> @llvm.x86.avx512.pternlog.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i32 33)
+  ret <2 x i64> %1
+}
+
 define <2 x i64>@test_int_x86_avx512_mask_pternlog_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_pternlog_q_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X86-NEXT:    vpternlogq $33, %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf3,0xf5,0x08,0x25,0xda,0x21]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogq $33, %xmm2, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf3,0xf5,0x09,0x25,0xc2,0x21]
-; X86-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pternlog_q_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X64-NEXT:    vpternlogq $33, %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf3,0xf5,0x08,0x25,0xda,0x21]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogq $33, %xmm2, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf3,0xf5,0x09,0x25,0xc2,0x21]
-; X64-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <2 x i64> @llvm.x86.avx512.pternlog.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i32 33)
   %2 = bitcast i8 %x4 to <8 x i1>
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
   %3 = select <2 x i1> %extract, <2 x i64> %1, <2 x i64> %x0
-  %4 = call <2 x i64> @llvm.x86.avx512.pternlog.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i32 33)
-  %res2 = add <2 x i64> %3, %4
-  ret <2 x i64> %res2
+  ret <2 x i64> %3
 }
 
 define <2 x i64>@test_int_x86_avx512_maskz_pternlog_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_maskz_pternlog_q_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X86-NEXT:    vpternlogq $33, %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf3,0xf5,0x08,0x25,0xda,0x21]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogq $33, %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0xf5,0x89,0x25,0xc2,0x21]
-; X86-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_pternlog_q_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
-; X64-NEXT:    vpternlogq $33, %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf3,0xf5,0x08,0x25,0xda,0x21]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogq $33, %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0xf5,0x89,0x25,0xc2,0x21]
-; X64-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <2 x i64> @llvm.x86.avx512.pternlog.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i32 33)
   %2 = bitcast i8 %x4 to <8 x i1>
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
   %3 = select <2 x i1> %extract, <2 x i64> %1, <2 x i64> zeroinitializer
-  %4 = call <2 x i64> @llvm.x86.avx512.pternlog.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i32 33)
-  %res2 = add <2 x i64> %3, %4
-  ret <2 x i64> %res2
+  ret <2 x i64> %3
 }
 
 declare <4 x i64> @llvm.x86.avx512.pternlog.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i32)
 
+define <4 x i64>@test_int_x86_avx512_pternlog_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_pternlog_q_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpternlogq $33, %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf3,0xf5,0x28,0x25,0xc2,0x21]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <4 x i64> @llvm.x86.avx512.pternlog.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i32 33)
+  ret <4 x i64> %1
+}
+
 define <4 x i64>@test_int_x86_avx512_mask_pternlog_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_pternlog_q_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X86-NEXT:    vpternlogq $33, %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf3,0xf5,0x28,0x25,0xda,0x21]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogq $33, %ymm2, %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf3,0xf5,0x29,0x25,0xc2,0x21]
-; X86-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pternlog_q_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X64-NEXT:    vpternlogq $33, %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf3,0xf5,0x28,0x25,0xda,0x21]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogq $33, %ymm2, %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf3,0xf5,0x29,0x25,0xc2,0x21]
-; X64-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <4 x i64> @llvm.x86.avx512.pternlog.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i32 33)
   %2 = bitcast i8 %x4 to <8 x i1>
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract, <4 x i64> %1, <4 x i64> %x0
-  %4 = call <4 x i64> @llvm.x86.avx512.pternlog.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i32 33)
-  %res2 = add <4 x i64> %3, %4
-  ret <4 x i64> %res2
+  ret <4 x i64> %3
 }
 
 define <4 x i64>@test_int_x86_avx512_maskz_pternlog_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_maskz_pternlog_q_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X86-NEXT:    vpternlogq $33, %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf3,0xf5,0x28,0x25,0xda,0x21]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpternlogq $33, %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf3,0xf5,0xa9,0x25,0xc2,0x21]
-; X86-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_pternlog_q_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
-; X64-NEXT:    vpternlogq $33, %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf3,0xf5,0x28,0x25,0xda,0x21]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpternlogq $33, %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf3,0xf5,0xa9,0x25,0xc2,0x21]
-; X64-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %1 = call <4 x i64> @llvm.x86.avx512.pternlog.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i32 33)
   %2 = bitcast i8 %x4 to <8 x i1>
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract, <4 x i64> %1, <4 x i64> zeroinitializer
-  %4 = call <4 x i64> @llvm.x86.avx512.pternlog.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i32 33)
-  %res2 = add <4 x i64> %3, %4
-  ret <4 x i64> %res2
+  ret <4 x i64> %3
 }
 
 define <8 x i16> @test_x86_vcvtps2ph_128(<4 x float> %a0, i8 %mask, <8 x i16> %src) {
@@ -5047,29 +5039,29 @@ define <4 x float>@test_int_x86_avx512_mask_fixupimm_ps_128(<4 x float> %x0, <4
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vmovaps %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xd8]
 ; X86-NEXT:    vfixupimmps $5, %xmm2, %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf3,0x75,0x09,0x54,0xda,0x05]
-; X86-NEXT:    vmovaps %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xe0]
-; X86-NEXT:    vfixupimmps $5, %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf3,0x75,0x08,0x54,0xe2,0x05]
-; X86-NEXT:    vxorps %xmm2, %xmm2, %xmm2 # encoding: [0xc5,0xe8,0x57,0xd2]
-; X86-NEXT:    vfixupimmps $5, %xmm2, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf3,0x75,0x09,0x54,0xc2,0x05]
+; X86-NEXT:    vxorps %xmm4, %xmm4, %xmm4 # encoding: [0xc5,0xd8,0x57,0xe4]
+; X86-NEXT:    vmovaps %xmm0, %xmm5 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xe8]
+; X86-NEXT:    vfixupimmps $6, %xmm4, %xmm1, %xmm5 {%k1} # encoding: [0x62,0xf3,0x75,0x09,0x54,0xec,0x06]
+; X86-NEXT:    vaddps %xmm5, %xmm3, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xe0,0x58,0xdd]
+; X86-NEXT:    vfixupimmps $7, %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf3,0x75,0x08,0x54,0xc2,0x07]
 ; X86-NEXT:    vaddps %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe0,0x58,0xc0]
-; X86-NEXT:    vaddps %xmm4, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x58,0xc4]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_fixupimm_ps_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vmovaps %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xd8]
-; X64-NEXT:    vfixupimmps $5, %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf3,0x75,0x08,0x54,0xda,0x05]
-; X64-NEXT:    vmovaps %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xe0]
-; X64-NEXT:    vfixupimmps $5, %xmm2, %xmm1, %xmm4 {%k1} # encoding: [0x62,0xf3,0x75,0x09,0x54,0xe2,0x05]
-; X64-NEXT:    vxorps %xmm2, %xmm2, %xmm2 # encoding: [0xc5,0xe8,0x57,0xd2]
-; X64-NEXT:    vfixupimmps $5, %xmm2, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf3,0x75,0x09,0x54,0xc2,0x05]
-; X64-NEXT:    vaddps %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd8,0x58,0xc0]
-; X64-NEXT:    vaddps %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x58,0xc3]
+; X64-NEXT:    vfixupimmps $5, %xmm2, %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf3,0x75,0x09,0x54,0xda,0x05]
+; X64-NEXT:    vxorps %xmm4, %xmm4, %xmm4 # encoding: [0xc5,0xd8,0x57,0xe4]
+; X64-NEXT:    vmovaps %xmm0, %xmm5 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xe8]
+; X64-NEXT:    vfixupimmps $6, %xmm4, %xmm1, %xmm5 {%k1} # encoding: [0x62,0xf3,0x75,0x09,0x54,0xec,0x06]
+; X64-NEXT:    vaddps %xmm5, %xmm3, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xe0,0x58,0xdd]
+; X64-NEXT:    vfixupimmps $7, %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf3,0x75,0x08,0x54,0xc2,0x07]
+; X64-NEXT:    vaddps %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe0,0x58,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x float> @llvm.x86.avx512.mask.fixupimm.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x i32> %x2, i32 5, i8 %x4)
-  %res1 = call <4 x float> @llvm.x86.avx512.mask.fixupimm.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x i32> zeroinitializer, i32 5, i8 %x4)
-  %res2 = call <4 x float> @llvm.x86.avx512.mask.fixupimm.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x i32> %x2, i32 5, i8 -1)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.fixupimm.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x i32> zeroinitializer, i32 6, i8 %x4)
+  %res2 = call <4 x float> @llvm.x86.avx512.mask.fixupimm.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x i32> %x2, i32 7, i8 -1)
   %res3 = fadd <4 x float> %res, %res1
   %res4 = fadd <4 x float> %res3, %res2
   ret <4 x float> %res4
@@ -5084,29 +5076,29 @@ define <4 x float>@test_int_x86_avx512_maskz_fixupimm_ps_128(<4 x float> %x0, <4
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vmovaps %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xd8]
 ; X86-NEXT:    vfixupimmps $5, %xmm2, %xmm1, %xmm3 {%k1} {z} # encoding: [0x62,0xf3,0x75,0x89,0x54,0xda,0x05]
-; X86-NEXT:    vmovaps %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xe0]
-; X86-NEXT:    vfixupimmps $5, %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf3,0x75,0x08,0x54,0xe2,0x05]
-; X86-NEXT:    vxorps %xmm2, %xmm2, %xmm2 # encoding: [0xc5,0xe8,0x57,0xd2]
-; X86-NEXT:    vfixupimmps $5, %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x75,0x89,0x54,0xc2,0x05]
+; X86-NEXT:    vxorps %xmm4, %xmm4, %xmm4 # encoding: [0xc5,0xd8,0x57,0xe4]
+; X86-NEXT:    vmovaps %xmm0, %xmm5 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xe8]
+; X86-NEXT:    vfixupimmps $6, %xmm4, %xmm1, %xmm5 {%k1} {z} # encoding: [0x62,0xf3,0x75,0x89,0x54,0xec,0x06]
+; X86-NEXT:    vaddps %xmm5, %xmm3, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xe0,0x58,0xdd]
+; X86-NEXT:    vfixupimmps $7, %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf3,0x75,0x08,0x54,0xc2,0x07]
 ; X86-NEXT:    vaddps %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe0,0x58,0xc0]
-; X86-NEXT:    vaddps %xmm4, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x58,0xc4]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_fixupimm_ps_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vmovaps %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xd8]
-; X64-NEXT:    vfixupimmps $5, %xmm2, %xmm1, %xmm3 # encoding: [0x62,0xf3,0x75,0x08,0x54,0xda,0x05]
-; X64-NEXT:    vmovaps %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xe0]
-; X64-NEXT:    vfixupimmps $5, %xmm2, %xmm1, %xmm4 {%k1} {z} # encoding: [0x62,0xf3,0x75,0x89,0x54,0xe2,0x05]
-; X64-NEXT:    vxorps %xmm2, %xmm2, %xmm2 # encoding: [0xc5,0xe8,0x57,0xd2]
-; X64-NEXT:    vfixupimmps $5, %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x75,0x89,0x54,0xc2,0x05]
-; X64-NEXT:    vaddps %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd8,0x58,0xc0]
-; X64-NEXT:    vaddps %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x58,0xc3]
+; X64-NEXT:    vfixupimmps $5, %xmm2, %xmm1, %xmm3 {%k1} {z} # encoding: [0x62,0xf3,0x75,0x89,0x54,0xda,0x05]
+; X64-NEXT:    vxorps %xmm4, %xmm4, %xmm4 # encoding: [0xc5,0xd8,0x57,0xe4]
+; X64-NEXT:    vmovaps %xmm0, %xmm5 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xe8]
+; X64-NEXT:    vfixupimmps $6, %xmm4, %xmm1, %xmm5 {%k1} {z} # encoding: [0x62,0xf3,0x75,0x89,0x54,0xec,0x06]
+; X64-NEXT:    vaddps %xmm5, %xmm3, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xe0,0x58,0xdd]
+; X64-NEXT:    vfixupimmps $7, %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf3,0x75,0x08,0x54,0xc2,0x07]
+; X64-NEXT:    vaddps %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe0,0x58,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x float> @llvm.x86.avx512.maskz.fixupimm.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x i32> %x2, i32 5, i8 %x4)
-  %res1 = call <4 x float> @llvm.x86.avx512.maskz.fixupimm.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x i32> zeroinitializer, i32 5, i8 %x4)
-  %res2 = call <4 x float> @llvm.x86.avx512.maskz.fixupimm.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x i32> %x2, i32 5, i8 -1)
+  %res1 = call <4 x float> @llvm.x86.avx512.maskz.fixupimm.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x i32> zeroinitializer, i32 6, i8 %x4)
+  %res2 = call <4 x float> @llvm.x86.avx512.maskz.fixupimm.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x i32> %x2, i32 7, i8 -1)
   %res3 = fadd <4 x float> %res, %res1
   %res4 = fadd <4 x float> %res3, %res2
   ret <4 x float> %res4
@@ -5121,29 +5113,29 @@ define <8 x float>@test_int_x86_avx512_mask_fixupimm_ps_256(<8 x float> %x0, <8
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vmovaps %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xd8]
 ; X86-NEXT:    vfixupimmps $5, %ymm2, %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf3,0x75,0x29,0x54,0xda,0x05]
-; X86-NEXT:    vmovaps %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xe0]
-; X86-NEXT:    vfixupimmps $5, %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf3,0x75,0x28,0x54,0xe2,0x05]
-; X86-NEXT:    vxorps %xmm2, %xmm2, %xmm2 # encoding: [0xc5,0xe8,0x57,0xd2]
-; X86-NEXT:    vfixupimmps $5, %ymm2, %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf3,0x75,0x29,0x54,0xc2,0x05]
+; X86-NEXT:    vxorps %xmm4, %xmm4, %xmm4 # encoding: [0xc5,0xd8,0x57,0xe4]
+; X86-NEXT:    vmovaps %ymm0, %ymm5 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xe8]
+; X86-NEXT:    vfixupimmps $6, %ymm4, %ymm1, %ymm5 {%k1} # encoding: [0x62,0xf3,0x75,0x29,0x54,0xec,0x06]
+; X86-NEXT:    vaddps %ymm5, %ymm3, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xe4,0x58,0xdd]
+; X86-NEXT:    vfixupimmps $7, %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf3,0x75,0x28,0x54,0xc2,0x07]
 ; X86-NEXT:    vaddps %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe4,0x58,0xc0]
-; X86-NEXT:    vaddps %ymm4, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x58,0xc4]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_fixupimm_ps_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vmovaps %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xd8]
-; X64-NEXT:    vfixupimmps $5, %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf3,0x75,0x28,0x54,0xda,0x05]
-; X64-NEXT:    vmovaps %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xe0]
-; X64-NEXT:    vfixupimmps $5, %ymm2, %ymm1, %ymm4 {%k1} # encoding: [0x62,0xf3,0x75,0x29,0x54,0xe2,0x05]
-; X64-NEXT:    vxorps %xmm2, %xmm2, %xmm2 # encoding: [0xc5,0xe8,0x57,0xd2]
-; X64-NEXT:    vfixupimmps $5, %ymm2, %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf3,0x75,0x29,0x54,0xc2,0x05]
-; X64-NEXT:    vaddps %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdc,0x58,0xc0]
-; X64-NEXT:    vaddps %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x58,0xc3]
+; X64-NEXT:    vfixupimmps $5, %ymm2, %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf3,0x75,0x29,0x54,0xda,0x05]
+; X64-NEXT:    vxorps %xmm4, %xmm4, %xmm4 # encoding: [0xc5,0xd8,0x57,0xe4]
+; X64-NEXT:    vmovaps %ymm0, %ymm5 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xe8]
+; X64-NEXT:    vfixupimmps $6, %ymm4, %ymm1, %ymm5 {%k1} # encoding: [0x62,0xf3,0x75,0x29,0x54,0xec,0x06]
+; X64-NEXT:    vaddps %ymm5, %ymm3, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xe4,0x58,0xdd]
+; X64-NEXT:    vfixupimmps $7, %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf3,0x75,0x28,0x54,0xc2,0x07]
+; X64-NEXT:    vaddps %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe4,0x58,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x float> @llvm.x86.avx512.mask.fixupimm.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x i32> %x2, i32 5, i8 %x4)
-  %res1 = call <8 x float> @llvm.x86.avx512.mask.fixupimm.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x i32> zeroinitializer, i32 5, i8 %x4)
-  %res2 = call <8 x float> @llvm.x86.avx512.mask.fixupimm.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x i32> %x2, i32 5, i8 -1)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask.fixupimm.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x i32> zeroinitializer, i32 6, i8 %x4)
+  %res2 = call <8 x float> @llvm.x86.avx512.mask.fixupimm.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x i32> %x2, i32 7, i8 -1)
   %res3 = fadd <8 x float> %res, %res1
   %res4 = fadd <8 x float> %res3, %res2
   ret <8 x float> %res4
@@ -5158,29 +5150,29 @@ define <8 x float>@test_int_x86_avx512_maskz_fixupimm_ps_256(<8 x float> %x0, <8
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vmovaps %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xd8]
 ; X86-NEXT:    vfixupimmps $5, %ymm2, %ymm1, %ymm3 {%k1} {z} # encoding: [0x62,0xf3,0x75,0xa9,0x54,0xda,0x05]
-; X86-NEXT:    vmovaps %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xe0]
-; X86-NEXT:    vfixupimmps $5, %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf3,0x75,0x28,0x54,0xe2,0x05]
-; X86-NEXT:    vxorps %xmm2, %xmm2, %xmm2 # encoding: [0xc5,0xe8,0x57,0xd2]
-; X86-NEXT:    vfixupimmps $5, %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf3,0x75,0xa9,0x54,0xc2,0x05]
+; X86-NEXT:    vxorps %xmm4, %xmm4, %xmm4 # encoding: [0xc5,0xd8,0x57,0xe4]
+; X86-NEXT:    vmovaps %ymm0, %ymm5 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xe8]
+; X86-NEXT:    vfixupimmps $6, %ymm4, %ymm1, %ymm5 {%k1} {z} # encoding: [0x62,0xf3,0x75,0xa9,0x54,0xec,0x06]
+; X86-NEXT:    vaddps %ymm5, %ymm3, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xe4,0x58,0xdd]
+; X86-NEXT:    vfixupimmps $7, %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf3,0x75,0x28,0x54,0xc2,0x07]
 ; X86-NEXT:    vaddps %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe4,0x58,0xc0]
-; X86-NEXT:    vaddps %ymm4, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x58,0xc4]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_maskz_fixupimm_ps_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vmovaps %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xd8]
-; X64-NEXT:    vfixupimmps $5, %ymm2, %ymm1, %ymm3 # encoding: [0x62,0xf3,0x75,0x28,0x54,0xda,0x05]
-; X64-NEXT:    vmovaps %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xe0]
-; X64-NEXT:    vfixupimmps $5, %ymm2, %ymm1, %ymm4 {%k1} {z} # encoding: [0x62,0xf3,0x75,0xa9,0x54,0xe2,0x05]
-; X64-NEXT:    vxorps %xmm2, %xmm2, %xmm2 # encoding: [0xc5,0xe8,0x57,0xd2]
-; X64-NEXT:    vfixupimmps $5, %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf3,0x75,0xa9,0x54,0xc2,0x05]
-; X64-NEXT:    vaddps %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdc,0x58,0xc0]
-; X64-NEXT:    vaddps %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x58,0xc3]
+; X64-NEXT:    vfixupimmps $5, %ymm2, %ymm1, %ymm3 {%k1} {z} # encoding: [0x62,0xf3,0x75,0xa9,0x54,0xda,0x05]
+; X64-NEXT:    vxorps %xmm4, %xmm4, %xmm4 # encoding: [0xc5,0xd8,0x57,0xe4]
+; X64-NEXT:    vmovaps %ymm0, %ymm5 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xe8]
+; X64-NEXT:    vfixupimmps $6, %ymm4, %ymm1, %ymm5 {%k1} {z} # encoding: [0x62,0xf3,0x75,0xa9,0x54,0xec,0x06]
+; X64-NEXT:    vaddps %ymm5, %ymm3, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xe4,0x58,0xdd]
+; X64-NEXT:    vfixupimmps $7, %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf3,0x75,0x28,0x54,0xc2,0x07]
+; X64-NEXT:    vaddps %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe4,0x58,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x float> @llvm.x86.avx512.maskz.fixupimm.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x i32> %x2, i32 5, i8 %x4)
-  %res1 = call <8 x float> @llvm.x86.avx512.maskz.fixupimm.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x i32> zeroinitializer, i32 5, i8 %x4)
-  %res2 = call <8 x float> @llvm.x86.avx512.maskz.fixupimm.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x i32> %x2, i32 5, i8 -1)
+  %res1 = call <8 x float> @llvm.x86.avx512.maskz.fixupimm.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x i32> zeroinitializer, i32 6, i8 %x4)
+  %res2 = call <8 x float> @llvm.x86.avx512.maskz.fixupimm.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x i32> %x2, i32 7, i8 -1)
   %res3 = fadd <8 x float> %res, %res1
   %res4 = fadd <8 x float> %res3, %res2
   ret <8 x float> %res4
diff --git a/llvm/test/CodeGen/X86/avx512vl_vnni-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512vl_vnni-intrinsics-upgrade.ll
index 4305259b0249e..d815bc939504d 100644
--- a/llvm/test/CodeGen/X86/avx512vl_vnni-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512vl_vnni-intrinsics-upgrade.ll
@@ -5,6 +5,15 @@
 declare <8 x i32> @llvm.x86.avx512.mask.vpdpbusd.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
 declare <8 x i32> @llvm.x86.avx512.maskz.vpdpbusd.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
 
+define <8 x i32>@test_int_x86_avx512_vpdpbusd_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpdpbusd_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpbusd %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf2,0x75,0x28,0x50,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.vpdpbusd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1)
+  ret <8 x i32> %res
+}
+
 define <8 x i32>@test_int_x86_avx512_mask_vpdpbusd_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32>* %x2p, <8 x i32> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpdpbusd_256:
 ; X86:       # %bb.0:
@@ -13,10 +22,7 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpbusd_256(<8 x i32> %x0, <8 x i32>
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpdpbusd (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x50,0x18]
-; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpdpbusd %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x50,0xe2]
 ; X86-NEXT:    vpdpbusd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x50,0xc2]
-; X86-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -25,24 +31,28 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpbusd_256(<8 x i32> %x0, <8 x i32>
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X64-NEXT:    vpdpbusd (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x50,0x1f]
-; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X64-NEXT:    vpdpbusd %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x50,0xe2]
 ; X64-NEXT:    vpdpbusd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x50,0xc2]
-; X64-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <8 x i32>, <8 x i32>* %x2p
   %res = call <8 x i32> @llvm.x86.avx512.mask.vpdpbusd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
-  %res1 = call <8 x i32> @llvm.x86.avx512.mask.vpdpbusd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8 -1)
-  %res2 = call <8 x i32> @llvm.x86.avx512.maskz.vpdpbusd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8  %x3)
-  %res3 = add <8 x i32> %res, %res1
-  %res4 = add <8 x i32> %res2, %res3
-  ret <8 x i32> %res4
+  %res1 = call <8 x i32> @llvm.x86.avx512.maskz.vpdpbusd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8  %x3)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
 }
 
 declare <4 x i32> @llvm.x86.avx512.mask.vpdpbusd.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
 declare <4 x i32> @llvm.x86.avx512.maskz.vpdpbusd.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
 
+define <4 x i32>@test_int_x86_avx512_vpdpbusd_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpdpbusd_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpbusd %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf2,0x75,0x08,0x50,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.vpdpbusd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1)
+  ret <4 x i32> %res
+}
+
 define <4 x i32>@test_int_x86_avx512_mask_vpdpbusd_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32>* %x2p, <4 x i32> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpdpbusd_128:
 ; X86:       # %bb.0:
@@ -51,10 +61,7 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpbusd_128(<4 x i32> %x0, <4 x i32>
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpdpbusd (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x50,0x18]
-; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpdpbusd %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x50,0xe2]
 ; X86-NEXT:    vpdpbusd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x50,0xc2]
-; X86-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -63,24 +70,28 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpbusd_128(<4 x i32> %x0, <4 x i32>
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X64-NEXT:    vpdpbusd (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x50,0x1f]
-; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X64-NEXT:    vpdpbusd %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x50,0xe2]
 ; X64-NEXT:    vpdpbusd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x50,0xc2]
-; X64-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <4 x i32>, <4 x i32>* %x2p
   %res = call <4 x i32> @llvm.x86.avx512.mask.vpdpbusd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3)
-  %res1 = call <4 x i32> @llvm.x86.avx512.mask.vpdpbusd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8 -1)
-  %res2 = call <4 x i32> @llvm.x86.avx512.maskz.vpdpbusd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8  %x3)
-  %res3 = add <4 x i32> %res, %res1
-  %res4 = add <4 x i32> %res2, %res3
-  ret <4 x i32> %res4
+  %res1 = call <4 x i32> @llvm.x86.avx512.maskz.vpdpbusd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8  %x3)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
 }
 
 declare <8 x i32> @llvm.x86.avx512.mask.vpdpbusds.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
 declare <8 x i32> @llvm.x86.avx512.maskz.vpdpbusds.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
 
+define <8 x i32>@test_int_x86_avx512_vpdpbusds_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpdpbusds_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpbusds %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf2,0x75,0x28,0x51,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.vpdpbusds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1)
+  ret <8 x i32> %res
+}
+
 define <8 x i32>@test_int_x86_avx512_mask_vpdpbusds_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32>* %x2p, <8 x i32> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpdpbusds_256:
 ; X86:       # %bb.0:
@@ -89,10 +100,7 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpbusds_256(<8 x i32> %x0, <8 x i32>
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpdpbusds (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x51,0x18]
-; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpdpbusds %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x51,0xe2]
 ; X86-NEXT:    vpdpbusds %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x51,0xc2]
-; X86-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -101,24 +109,28 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpbusds_256(<8 x i32> %x0, <8 x i32>
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X64-NEXT:    vpdpbusds (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x51,0x1f]
-; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X64-NEXT:    vpdpbusds %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x51,0xe2]
 ; X64-NEXT:    vpdpbusds %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x51,0xc2]
-; X64-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <8 x i32>, <8 x i32>* %x2p
   %res = call <8 x i32> @llvm.x86.avx512.mask.vpdpbusds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
-  %res1 = call <8 x i32> @llvm.x86.avx512.mask.vpdpbusds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8 -1)
-  %res2 = call <8 x i32> @llvm.x86.avx512.maskz.vpdpbusds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8  %x3)
-  %res3 = add <8 x i32> %res, %res1
-  %res4 = add <8 x i32> %res2, %res3
-  ret <8 x i32> %res4
+  %res1 = call <8 x i32> @llvm.x86.avx512.maskz.vpdpbusds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8  %x3)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
 }
 
 declare <4 x i32> @llvm.x86.avx512.mask.vpdpbusds.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
 declare <4 x i32> @llvm.x86.avx512.maskz.vpdpbusds.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
 
+define <4 x i32>@test_int_x86_avx512_vpdpbusds_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpdpbusds_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpbusds %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf2,0x75,0x08,0x51,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.vpdpbusds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1)
+  ret <4 x i32> %res
+}
+
 define <4 x i32>@test_int_x86_avx512_mask_vpdpbusds_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32>* %x2p, <4 x i32> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpdpbusds_128:
 ; X86:       # %bb.0:
@@ -127,10 +139,7 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpbusds_128(<4 x i32> %x0, <4 x i32>
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpdpbusds (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x51,0x18]
-; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpdpbusds %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x51,0xe2]
 ; X86-NEXT:    vpdpbusds %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x51,0xc2]
-; X86-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -139,24 +148,28 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpbusds_128(<4 x i32> %x0, <4 x i32>
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X64-NEXT:    vpdpbusds (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x51,0x1f]
-; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X64-NEXT:    vpdpbusds %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x51,0xe2]
 ; X64-NEXT:    vpdpbusds %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x51,0xc2]
-; X64-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <4 x i32>, <4 x i32>* %x2p
   %res = call <4 x i32> @llvm.x86.avx512.mask.vpdpbusds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3)
-  %res1 = call <4 x i32> @llvm.x86.avx512.mask.vpdpbusds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8 -1)
-  %res2 = call <4 x i32> @llvm.x86.avx512.maskz.vpdpbusds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8  %x3)
-  %res3 = add <4 x i32> %res, %res1
-  %res4 = add <4 x i32> %res2, %res3
-  ret <4 x i32> %res4
+  %res1 = call <4 x i32> @llvm.x86.avx512.maskz.vpdpbusds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8  %x3)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
 }
 
 declare <8 x i32> @llvm.x86.avx512.mask.vpdpwssd.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
 declare <8 x i32> @llvm.x86.avx512.maskz.vpdpwssd.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
 
+define <8 x i32>@test_int_x86_avx512_vpdpwssd_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpdpwssd_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpwssd %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf2,0x75,0x28,0x52,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.vpdpwssd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1)
+  ret <8 x i32> %res
+}
+
 define <8 x i32>@test_int_x86_avx512_mask_vpdpwssd_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32>* %x2p, <8 x i32> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpdpwssd_256:
 ; X86:       # %bb.0:
@@ -165,10 +178,7 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpwssd_256(<8 x i32> %x0, <8 x i32>
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpdpwssd (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x52,0x18]
-; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpdpwssd %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x52,0xe2]
 ; X86-NEXT:    vpdpwssd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x52,0xc2]
-; X86-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -177,24 +187,28 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpwssd_256(<8 x i32> %x0, <8 x i32>
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X64-NEXT:    vpdpwssd (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x52,0x1f]
-; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X64-NEXT:    vpdpwssd %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x52,0xe2]
 ; X64-NEXT:    vpdpwssd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x52,0xc2]
-; X64-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <8 x i32>, <8 x i32>* %x2p
   %res = call <8 x i32> @llvm.x86.avx512.mask.vpdpwssd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
-  %res1 = call <8 x i32> @llvm.x86.avx512.mask.vpdpwssd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8 -1)
-  %res2 = call <8 x i32> @llvm.x86.avx512.maskz.vpdpwssd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8  %x3)
-  %res3 = add <8 x i32> %res, %res1
-  %res4 = add <8 x i32> %res2, %res3
-  ret <8 x i32> %res4
+  %res1 = call <8 x i32> @llvm.x86.avx512.maskz.vpdpwssd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8  %x3)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
 }
 
 declare <4 x i32> @llvm.x86.avx512.mask.vpdpwssd.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
 declare <4 x i32> @llvm.x86.avx512.maskz.vpdpwssd.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
 
+define <4 x i32>@test_int_x86_avx512_vpdpwssd_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpdpwssd_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpwssd %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf2,0x75,0x08,0x52,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.vpdpwssd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1)
+  ret <4 x i32> %res
+}
+
 define <4 x i32>@test_int_x86_avx512_mask_vpdpwssd_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32>* %x2p, <4 x i32> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpdpwssd_128:
 ; X86:       # %bb.0:
@@ -203,10 +217,7 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpwssd_128(<4 x i32> %x0, <4 x i32>
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpdpwssd (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x52,0x18]
-; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpdpwssd %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x52,0xe2]
 ; X86-NEXT:    vpdpwssd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x52,0xc2]
-; X86-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -215,25 +226,29 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpwssd_128(<4 x i32> %x0, <4 x i32>
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X64-NEXT:    vpdpwssd (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x52,0x1f]
-; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X64-NEXT:    vpdpwssd %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x52,0xe2]
 ; X64-NEXT:    vpdpwssd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x52,0xc2]
-; X64-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <4 x i32>, <4 x i32>* %x2p
   %res = call <4 x i32> @llvm.x86.avx512.mask.vpdpwssd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3)
-  %res1 = call <4 x i32> @llvm.x86.avx512.mask.vpdpwssd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8 -1)
-  %res2 = call <4 x i32> @llvm.x86.avx512.maskz.vpdpwssd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8  %x3)
-  %res3 = add <4 x i32> %res, %res1
-  %res4 = add <4 x i32> %res2, %res3
-  ret <4 x i32> %res4
+  %res1 = call <4 x i32> @llvm.x86.avx512.maskz.vpdpwssd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8  %x3)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
 }
 
 
 declare <8 x i32> @llvm.x86.avx512.mask.vpdpwssds.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
 declare <8 x i32> @llvm.x86.avx512.maskz.vpdpwssds.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
 
+define <8 x i32>@test_int_x86_avx512_vpdpwssds_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpdpwssds_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpwssds %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf2,0x75,0x28,0x53,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.vpdpwssds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1)
+  ret <8 x i32> %res
+}
+
 define <8 x i32>@test_int_x86_avx512_mask_vpdpwssds_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32>* %x2p, <8 x i32> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpdpwssds_256:
 ; X86:       # %bb.0:
@@ -242,10 +257,7 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpwssds_256(<8 x i32> %x0, <8 x i32>
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpdpwssds (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x53,0x18]
-; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpdpwssds %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x53,0xe2]
 ; X86-NEXT:    vpdpwssds %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x53,0xc2]
-; X86-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -254,24 +266,28 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpwssds_256(<8 x i32> %x0, <8 x i32>
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X64-NEXT:    vpdpwssds (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x53,0x1f]
-; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X64-NEXT:    vpdpwssds %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x53,0xe2]
 ; X64-NEXT:    vpdpwssds %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x53,0xc2]
-; X64-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <8 x i32>, <8 x i32>* %x2p
   %res = call <8 x i32> @llvm.x86.avx512.mask.vpdpwssds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
-  %res1 = call <8 x i32> @llvm.x86.avx512.mask.vpdpwssds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8 -1)
-  %res2 = call <8 x i32> @llvm.x86.avx512.maskz.vpdpwssds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8  %x3)
-  %res3 = add <8 x i32> %res, %res1
-  %res4 = add <8 x i32> %res2, %res3
-  ret <8 x i32> %res4
+  %res1 = call <8 x i32> @llvm.x86.avx512.maskz.vpdpwssds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8  %x3)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
 }
 
 declare <4 x i32> @llvm.x86.avx512.mask.vpdpwssds.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
 declare <4 x i32> @llvm.x86.avx512.maskz.vpdpwssds.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
 
+define <4 x i32>@test_int_x86_avx512_vpdpwssds_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpdpwssds_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpwssds %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf2,0x75,0x08,0x53,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.vpdpwssds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1)
+  ret <4 x i32> %res
+}
+
 define <4 x i32>@test_int_x86_avx512_mask_vpdpwssds_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32>* %x2p, <4 x i32> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpdpwssds_128:
 ; X86:       # %bb.0:
@@ -280,10 +296,7 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpwssds_128(<4 x i32> %x0, <4 x i32>
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpdpwssds (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x53,0x18]
-; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpdpwssds %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x53,0xe2]
 ; X86-NEXT:    vpdpwssds %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x53,0xc2]
-; X86-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -292,18 +305,12 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpwssds_128(<4 x i32> %x0, <4 x i32>
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X64-NEXT:    vpdpwssds (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x53,0x1f]
-; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X64-NEXT:    vpdpwssds %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x53,0xe2]
 ; X64-NEXT:    vpdpwssds %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x53,0xc2]
-; X64-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <4 x i32>, <4 x i32>* %x2p
   %res = call <4 x i32> @llvm.x86.avx512.mask.vpdpwssds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3)
-  %res1 = call <4 x i32> @llvm.x86.avx512.mask.vpdpwssds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8 -1)
-  %res2 = call <4 x i32> @llvm.x86.avx512.maskz.vpdpwssds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8  %x3)
-  %res3 = add <4 x i32> %res, %res1
-  %res4 = add <4 x i32> %res2, %res3
-  ret <4 x i32> %res4
+  %res1 = call <4 x i32> @llvm.x86.avx512.maskz.vpdpwssds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8  %x3)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
 }
-
diff --git a/llvm/test/CodeGen/X86/avx512vl_vnni-intrinsics.ll b/llvm/test/CodeGen/X86/avx512vl_vnni-intrinsics.ll
index 68bda52ac8017..5f6f32f5d8bea 100644
--- a/llvm/test/CodeGen/X86/avx512vl_vnni-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512vl_vnni-intrinsics.ll
@@ -4,6 +4,15 @@
 
 declare <8 x i32> @llvm.x86.avx512.vpdpbusd.256(<8 x i32>, <8 x i32>, <8 x i32>)
 
+define <8 x i32>@test_int_x86_avx512_vpdpbusd_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpdpbusd_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpbusd %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf2,0x75,0x28,0x50,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <8 x i32> @llvm.x86.avx512.vpdpbusd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2)
+  ret <8 x i32> %1
+}
+
 define <8 x i32>@test_int_x86_avx512_mask_vpdpbusd_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32>* %x2p, <8 x i32> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpdpbusd_256:
 ; X86:       # %bb.0:
@@ -12,10 +21,7 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpbusd_256(<8 x i32> %x0, <8 x i32>
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpdpbusd (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x50,0x18]
-; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpdpbusd %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x50,0xe2]
 ; X86-NEXT:    vpdpbusd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x50,0xc2]
-; X86-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -24,10 +30,7 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpbusd_256(<8 x i32> %x0, <8 x i32>
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X64-NEXT:    vpdpbusd (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x50,0x1f]
-; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X64-NEXT:    vpdpbusd %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x50,0xe2]
 ; X64-NEXT:    vpdpbusd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x50,0xc2]
-; X64-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <8 x i32>, <8 x i32>* %x2p
@@ -35,16 +38,23 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpbusd_256(<8 x i32> %x0, <8 x i32>
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x0
   %4 = call <8 x i32> @llvm.x86.avx512.vpdpbusd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4)
-  %5 = call <8 x i32> @llvm.x86.avx512.vpdpbusd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4)
-  %6 = bitcast i8 %x3 to <8 x i1>
-  %7 = select <8 x i1> %6, <8 x i32> %5, <8 x i32> zeroinitializer
-  %res3 = add <8 x i32> %3, %4
-  %res4 = add <8 x i32> %7, %res3
-  ret <8 x i32> %res4
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = select <8 x i1> %5, <8 x i32> %4, <8 x i32> zeroinitializer
+  %res3 = add <8 x i32> %3, %6
+  ret <8 x i32> %res3
 }
 
 declare <4 x i32> @llvm.x86.avx512.vpdpbusd.128(<4 x i32>, <4 x i32>, <4 x i32>)
 
+define <4 x i32>@test_int_x86_avx512_vpdpbusd_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpdpbusd_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpbusd %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf2,0x75,0x08,0x50,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <4 x i32> @llvm.x86.avx512.vpdpbusd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2)
+  ret <4 x i32> %1
+}
+
 define <4 x i32>@test_int_x86_avx512_mask_vpdpbusd_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32>* %x2p, <4 x i32> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpdpbusd_128:
 ; X86:       # %bb.0:
@@ -53,10 +63,7 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpbusd_128(<4 x i32> %x0, <4 x i32>
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpdpbusd (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x50,0x18]
-; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpdpbusd %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x50,0xe2]
 ; X86-NEXT:    vpdpbusd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x50,0xc2]
-; X86-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -65,10 +72,7 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpbusd_128(<4 x i32> %x0, <4 x i32>
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X64-NEXT:    vpdpbusd (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x50,0x1f]
-; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X64-NEXT:    vpdpbusd %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x50,0xe2]
 ; X64-NEXT:    vpdpbusd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x50,0xc2]
-; X64-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <4 x i32>, <4 x i32>* %x2p
@@ -77,17 +81,24 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpbusd_128(<4 x i32> %x0, <4 x i32>
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract, <4 x i32> %1, <4 x i32> %x0
   %4 = call <4 x i32> @llvm.x86.avx512.vpdpbusd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4)
-  %5 = call <4 x i32> @llvm.x86.avx512.vpdpbusd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4)
-  %6 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %6, <8 x i1> %6, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %7 = select <4 x i1> %extract1, <4 x i32> %5, <4 x i32> zeroinitializer
-  %res3 = add <4 x i32> %3, %4
-  %res4 = add <4 x i32> %7, %res3
-  ret <4 x i32> %res4
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %6 = select <4 x i1> %extract1, <4 x i32> %4, <4 x i32> zeroinitializer
+  %res3 = add <4 x i32> %3, %6
+  ret <4 x i32> %res3
 }
 
 declare <8 x i32> @llvm.x86.avx512.vpdpbusds.256(<8 x i32>, <8 x i32>, <8 x i32>)
 
+define <8 x i32>@test_int_x86_avx512_vpdpbusds_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpdpbusds_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpbusds %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf2,0x75,0x28,0x51,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <8 x i32> @llvm.x86.avx512.vpdpbusds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2)
+  ret <8 x i32> %1
+}
+
 define <8 x i32>@test_int_x86_avx512_mask_vpdpbusds_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32>* %x2p, <8 x i32> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpdpbusds_256:
 ; X86:       # %bb.0:
@@ -96,10 +107,7 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpbusds_256(<8 x i32> %x0, <8 x i32>
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpdpbusds (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x51,0x18]
-; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpdpbusds %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x51,0xe2]
 ; X86-NEXT:    vpdpbusds %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x51,0xc2]
-; X86-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -108,10 +116,7 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpbusds_256(<8 x i32> %x0, <8 x i32>
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X64-NEXT:    vpdpbusds (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x51,0x1f]
-; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X64-NEXT:    vpdpbusds %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x51,0xe2]
 ; X64-NEXT:    vpdpbusds %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x51,0xc2]
-; X64-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <8 x i32>, <8 x i32>* %x2p
@@ -119,16 +124,23 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpbusds_256(<8 x i32> %x0, <8 x i32>
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x0
   %4 = call <8 x i32> @llvm.x86.avx512.vpdpbusds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4)
-  %5 = call <8 x i32> @llvm.x86.avx512.vpdpbusds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4)
-  %6 = bitcast i8 %x3 to <8 x i1>
-  %7 = select <8 x i1> %6, <8 x i32> %5, <8 x i32> zeroinitializer
-  %res3 = add <8 x i32> %3, %4
-  %res4 = add <8 x i32> %7, %res3
-  ret <8 x i32> %res4
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = select <8 x i1> %5, <8 x i32> %4, <8 x i32> zeroinitializer
+  %res3 = add <8 x i32> %3, %6
+  ret <8 x i32> %res3
 }
 
 declare <4 x i32> @llvm.x86.avx512.vpdpbusds.128(<4 x i32>, <4 x i32>, <4 x i32>)
 
+define <4 x i32>@test_int_x86_avx512_vpdpbusds_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpdpbusds_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpbusds %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf2,0x75,0x08,0x51,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <4 x i32> @llvm.x86.avx512.vpdpbusds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2)
+  ret <4 x i32> %1
+}
+
 define <4 x i32>@test_int_x86_avx512_mask_vpdpbusds_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32>* %x2p, <4 x i32> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpdpbusds_128:
 ; X86:       # %bb.0:
@@ -137,10 +149,7 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpbusds_128(<4 x i32> %x0, <4 x i32>
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpdpbusds (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x51,0x18]
-; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpdpbusds %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x51,0xe2]
 ; X86-NEXT:    vpdpbusds %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x51,0xc2]
-; X86-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -149,10 +158,7 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpbusds_128(<4 x i32> %x0, <4 x i32>
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X64-NEXT:    vpdpbusds (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x51,0x1f]
-; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X64-NEXT:    vpdpbusds %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x51,0xe2]
 ; X64-NEXT:    vpdpbusds %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x51,0xc2]
-; X64-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <4 x i32>, <4 x i32>* %x2p
@@ -161,17 +167,24 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpbusds_128(<4 x i32> %x0, <4 x i32>
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract, <4 x i32> %1, <4 x i32> %x0
   %4 = call <4 x i32> @llvm.x86.avx512.vpdpbusds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4)
-  %5 = call <4 x i32> @llvm.x86.avx512.vpdpbusds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4)
-  %6 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %6, <8 x i1> %6, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %7 = select <4 x i1> %extract1, <4 x i32> %5, <4 x i32> zeroinitializer
-  %res3 = add <4 x i32> %3, %4
-  %res4 = add <4 x i32> %7, %res3
-  ret <4 x i32> %res4
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %6 = select <4 x i1> %extract1, <4 x i32> %4, <4 x i32> zeroinitializer
+  %res3 = add <4 x i32> %3, %6
+  ret <4 x i32> %res3
 }
 
 declare <8 x i32> @llvm.x86.avx512.vpdpwssd.256(<8 x i32>, <8 x i32>, <8 x i32>)
 
+define <8 x i32>@test_int_x86_avx512_vpdpwssd_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpdpwssd_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpwssd %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf2,0x75,0x28,0x52,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <8 x i32> @llvm.x86.avx512.vpdpwssd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2)
+  ret <8 x i32> %1
+}
+
 define <8 x i32>@test_int_x86_avx512_mask_vpdpwssd_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32>* %x2p, <8 x i32> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpdpwssd_256:
 ; X86:       # %bb.0:
@@ -180,10 +193,7 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpwssd_256(<8 x i32> %x0, <8 x i32>
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpdpwssd (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x52,0x18]
-; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpdpwssd %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x52,0xe2]
 ; X86-NEXT:    vpdpwssd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x52,0xc2]
-; X86-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -192,10 +202,7 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpwssd_256(<8 x i32> %x0, <8 x i32>
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X64-NEXT:    vpdpwssd (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x52,0x1f]
-; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X64-NEXT:    vpdpwssd %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x52,0xe2]
 ; X64-NEXT:    vpdpwssd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x52,0xc2]
-; X64-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <8 x i32>, <8 x i32>* %x2p
@@ -203,16 +210,23 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpwssd_256(<8 x i32> %x0, <8 x i32>
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x0
   %4 = call <8 x i32> @llvm.x86.avx512.vpdpwssd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4)
-  %5 = call <8 x i32> @llvm.x86.avx512.vpdpwssd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4)
-  %6 = bitcast i8 %x3 to <8 x i1>
-  %7 = select <8 x i1> %6, <8 x i32> %5, <8 x i32> zeroinitializer
-  %res3 = add <8 x i32> %3, %4
-  %res4 = add <8 x i32> %7, %res3
-  ret <8 x i32> %res4
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = select <8 x i1> %5, <8 x i32> %4, <8 x i32> zeroinitializer
+  %res3 = add <8 x i32> %3, %6
+  ret <8 x i32> %res3
 }
 
 declare <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32>, <4 x i32>, <4 x i32>)
 
+define <4 x i32>@test_int_x86_avx512_vpdpwssd_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpdpwssd_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpwssd %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf2,0x75,0x08,0x52,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2)
+  ret <4 x i32> %1
+}
+
 define <4 x i32>@test_int_x86_avx512_mask_vpdpwssd_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32>* %x2p, <4 x i32> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpdpwssd_128:
 ; X86:       # %bb.0:
@@ -221,10 +235,7 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpwssd_128(<4 x i32> %x0, <4 x i32>
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpdpwssd (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x52,0x18]
-; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpdpwssd %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x52,0xe2]
 ; X86-NEXT:    vpdpwssd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x52,0xc2]
-; X86-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -233,10 +244,7 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpwssd_128(<4 x i32> %x0, <4 x i32>
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X64-NEXT:    vpdpwssd (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x52,0x1f]
-; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X64-NEXT:    vpdpwssd %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x52,0xe2]
 ; X64-NEXT:    vpdpwssd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x52,0xc2]
-; X64-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <4 x i32>, <4 x i32>* %x2p
@@ -245,17 +253,24 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpwssd_128(<4 x i32> %x0, <4 x i32>
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract, <4 x i32> %1, <4 x i32> %x0
   %4 = call <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4)
-  %5 = call <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4)
-  %6 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %6, <8 x i1> %6, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %7 = select <4 x i1> %extract1, <4 x i32> %5, <4 x i32> zeroinitializer
-  %res3 = add <4 x i32> %3, %4
-  %res4 = add <4 x i32> %7, %res3
-  ret <4 x i32> %res4
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %6 = select <4 x i1> %extract1, <4 x i32> %4, <4 x i32> zeroinitializer
+  %res3 = add <4 x i32> %3, %6
+  ret <4 x i32> %res3
 }
 
 declare <8 x i32> @llvm.x86.avx512.vpdpwssds.256(<8 x i32>, <8 x i32>, <8 x i32>)
 
+define <8 x i32>@test_int_x86_avx512_vpdpwssds_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpdpwssds_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpwssds %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf2,0x75,0x28,0x53,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <8 x i32> @llvm.x86.avx512.vpdpwssds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2)
+  ret <8 x i32> %1
+}
+
 define <8 x i32>@test_int_x86_avx512_mask_vpdpwssds_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32>* %x2p, <8 x i32> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpdpwssds_256:
 ; X86:       # %bb.0:
@@ -264,10 +279,7 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpwssds_256(<8 x i32> %x0, <8 x i32>
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpdpwssds (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x53,0x18]
-; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpdpwssds %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x53,0xe2]
 ; X86-NEXT:    vpdpwssds %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x53,0xc2]
-; X86-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -276,10 +288,7 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpwssds_256(<8 x i32> %x0, <8 x i32>
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X64-NEXT:    vpdpwssds (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x53,0x1f]
-; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X64-NEXT:    vpdpwssds %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x53,0xe2]
 ; X64-NEXT:    vpdpwssds %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x53,0xc2]
-; X64-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <8 x i32>, <8 x i32>* %x2p
@@ -287,16 +296,30 @@ define <8 x i32>@test_int_x86_avx512_mask_vpdpwssds_256(<8 x i32> %x0, <8 x i32>
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x0
   %4 = call <8 x i32> @llvm.x86.avx512.vpdpwssds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4)
-  %5 = call <8 x i32> @llvm.x86.avx512.vpdpwssds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4)
-  %6 = bitcast i8 %x3 to <8 x i1>
-  %7 = select <8 x i1> %6, <8 x i32> %5, <8 x i32> zeroinitializer
-  %res3 = add <8 x i32> %3, %4
-  %res4 = add <8 x i32> %7, %res3
-  ret <8 x i32> %res4
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = select <8 x i1> %5, <8 x i32> %4, <8 x i32> zeroinitializer
+  %res3 = add <8 x i32> %3, %6
+  ret <8 x i32> %res3
 }
 
 declare <4 x i32> @llvm.x86.avx512.vpdpwssds.128(<4 x i32>, <4 x i32>, <4 x i32>)
 
+define <4 x i32>@test_int_x86_avx512_vpdpwssds_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32>* %x2p) {
+; X86-LABEL: test_int_x86_avx512_vpdpwssds_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vpdpwssds (%eax), %xmm1, %xmm0 # encoding: [0x62,0xf2,0x75,0x08,0x53,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_vpdpwssds_128:
+; X64:       # %bb.0:
+; X64-NEXT:    vpdpwssds (%rdi), %xmm1, %xmm0 # encoding: [0x62,0xf2,0x75,0x08,0x53,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %x2 = load <4 x i32>, <4 x i32>* %x2p
+  %1 = call <4 x i32> @llvm.x86.avx512.vpdpwssds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2)
+  ret <4 x i32> %1
+}
+
 define <4 x i32>@test_int_x86_avx512_mask_vpdpwssds_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32>* %x2p, <4 x i32> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpdpwssds_128:
 ; X86:       # %bb.0:
@@ -305,10 +328,7 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpwssds_128(<4 x i32> %x0, <4 x i32>
 ; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpdpwssds (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x53,0x18]
-; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpdpwssds %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x53,0xe2]
 ; X86-NEXT:    vpdpwssds %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x53,0xc2]
-; X86-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -317,10 +337,7 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpwssds_128(<4 x i32> %x0, <4 x i32>
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X64-NEXT:    vpdpwssds (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x53,0x1f]
-; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X64-NEXT:    vpdpwssds %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x53,0xe2]
 ; X64-NEXT:    vpdpwssds %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x53,0xc2]
-; X64-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <4 x i32>, <4 x i32>* %x2p
@@ -329,12 +346,9 @@ define <4 x i32>@test_int_x86_avx512_mask_vpdpwssds_128(<4 x i32> %x0, <4 x i32>
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract, <4 x i32> %1, <4 x i32> %x0
   %4 = call <4 x i32> @llvm.x86.avx512.vpdpwssds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4)
-  %5 = call <4 x i32> @llvm.x86.avx512.vpdpwssds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4)
-  %6 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %6, <8 x i1> %6, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %7 = select <4 x i1> %extract1, <4 x i32> %5, <4 x i32> zeroinitializer
-  %res3 = add <4 x i32> %3, %4
-  %res4 = add <4 x i32> %7, %res3
-  ret <4 x i32> %res4
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %6 = select <4 x i1> %extract1, <4 x i32> %4, <4 x i32> zeroinitializer
+  %res3 = add <4 x i32> %3, %6
+  ret <4 x i32> %res3
 }
-
diff --git a/llvm/test/CodeGen/X86/avx512vnni-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512vnni-intrinsics-upgrade.ll
index bd3d406715bda..57d259d7ed677 100644
--- a/llvm/test/CodeGen/X86/avx512vnni-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512vnni-intrinsics-upgrade.ll
@@ -5,6 +5,15 @@
 declare <16 x i32> @llvm.x86.avx512.mask.vpdpbusd.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
 declare <16 x i32> @llvm.x86.avx512.maskz.vpdpbusd.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
 
+define <16 x i32>@test_int_x86_avx512_vpdpbusd_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpdpbusd_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpbusd %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf2,0x75,0x48,0x50,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <16 x i32> @llvm.x86.avx512.mask.vpdpbusd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  ret <16 x i32> %res
+}
+
 define <16 x i32>@test_int_x86_avx512_mask_vpdpbusd_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32>* %x2p, <16 x i32> %x4, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpdpbusd_512:
 ; X86:       # %bb.0:
@@ -12,10 +21,7 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpbusd_512(<16 x i32> %x0, <16 x i3
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpdpbusd (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x50,0x18]
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpdpbusd %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x50,0xe2]
 ; X86-NEXT:    vpdpbusd %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x50,0xc2]
-; X86-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -24,24 +30,28 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpbusd_512(<16 x i32> %x0, <16 x i3
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X64-NEXT:    vpdpbusd (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x50,0x1f]
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X64-NEXT:    vpdpbusd %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x50,0xe2]
 ; X64-NEXT:    vpdpbusd %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x50,0xc2]
-; X64-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <16 x i32>, <16 x i32>* %x2p
   %res = call <16 x i32> @llvm.x86.avx512.mask.vpdpbusd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
-  %res1 = call <16 x i32> @llvm.x86.avx512.mask.vpdpbusd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16 -1)
-  %res2 = call <16 x i32> @llvm.x86.avx512.maskz.vpdpbusd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16  %x3)
-  %res3 = add <16 x i32> %res, %res1
-  %res4 = add <16 x i32> %res2, %res3
-  ret <16 x i32> %res4
+  %res1 = call <16 x i32> @llvm.x86.avx512.maskz.vpdpbusd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16  %x3)
+  %res2 = add <16 x i32> %res, %res1
+  ret <16 x i32> %res2
 }
 
 declare <16 x i32> @llvm.x86.avx512.mask.vpdpbusds.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
 declare <16 x i32> @llvm.x86.avx512.maskz.vpdpbusds.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
 
+define <16 x i32>@test_int_x86_avx512_vpdpbusds_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpdpbusds_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpbusds %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf2,0x75,0x48,0x51,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <16 x i32> @llvm.x86.avx512.mask.vpdpbusds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  ret <16 x i32> %res
+}
+
 define <16 x i32>@test_int_x86_avx512_mask_vpdpbusds_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32>* %x2p, <16 x i32> %x4, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpdpbusds_512:
 ; X86:       # %bb.0:
@@ -49,10 +59,7 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpbusds_512(<16 x i32> %x0, <16 x i
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpdpbusds (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x51,0x18]
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpdpbusds %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x51,0xe2]
 ; X86-NEXT:    vpdpbusds %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x51,0xc2]
-; X86-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -61,24 +68,28 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpbusds_512(<16 x i32> %x0, <16 x i
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X64-NEXT:    vpdpbusds (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x51,0x1f]
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X64-NEXT:    vpdpbusds %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x51,0xe2]
 ; X64-NEXT:    vpdpbusds %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x51,0xc2]
-; X64-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <16 x i32>, <16 x i32>* %x2p
   %res = call <16 x i32> @llvm.x86.avx512.mask.vpdpbusds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
-  %res1 = call <16 x i32> @llvm.x86.avx512.mask.vpdpbusds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16 -1)
-  %res2 = call <16 x i32> @llvm.x86.avx512.maskz.vpdpbusds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16  %x3)
-  %res3 = add <16 x i32> %res, %res1
-  %res4 = add <16 x i32> %res2, %res3
-  ret <16 x i32> %res4
+  %res1 = call <16 x i32> @llvm.x86.avx512.maskz.vpdpbusds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16  %x3)
+  %res2 = add <16 x i32> %res, %res1
+  ret <16 x i32> %res2
 }
 
 declare <16 x i32> @llvm.x86.avx512.mask.vpdpwssd.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
 declare <16 x i32> @llvm.x86.avx512.maskz.vpdpwssd.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
 
+define <16 x i32>@test_int_x86_avx512_vpdpwssd_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpdpwssd_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpwssd %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf2,0x75,0x48,0x52,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <16 x i32> @llvm.x86.avx512.mask.vpdpwssd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  ret <16 x i32> %res
+}
+
 define <16 x i32>@test_int_x86_avx512_mask_vpdpwssd_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32>* %x2p, <16 x i32> %x4, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpdpwssd_512:
 ; X86:       # %bb.0:
@@ -86,10 +97,7 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpwssd_512(<16 x i32> %x0, <16 x i3
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpdpwssd (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x52,0x18]
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpdpwssd %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x52,0xe2]
 ; X86-NEXT:    vpdpwssd %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x52,0xc2]
-; X86-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -98,24 +106,28 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpwssd_512(<16 x i32> %x0, <16 x i3
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X64-NEXT:    vpdpwssd (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x52,0x1f]
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X64-NEXT:    vpdpwssd %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x52,0xe2]
 ; X64-NEXT:    vpdpwssd %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x52,0xc2]
-; X64-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <16 x i32>, <16 x i32>* %x2p
   %res = call <16 x i32> @llvm.x86.avx512.mask.vpdpwssd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
-  %res1 = call <16 x i32> @llvm.x86.avx512.mask.vpdpwssd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16 -1)
-  %res2 = call <16 x i32> @llvm.x86.avx512.maskz.vpdpwssd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16  %x3)
-  %res3 = add <16 x i32> %res, %res1
-  %res4 = add <16 x i32> %res2, %res3
-  ret <16 x i32> %res4
+  %res1 = call <16 x i32> @llvm.x86.avx512.maskz.vpdpwssd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16  %x3)
+  %res2 = add <16 x i32> %res, %res1
+  ret <16 x i32> %res2
 }
 
 declare <16 x i32> @llvm.x86.avx512.mask.vpdpwssds.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
 declare <16 x i32> @llvm.x86.avx512.maskz.vpdpwssds.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
 
+define <16 x i32>@test_int_x86_avx512_vpdpwssds_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpdpwssds_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpwssds %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf2,0x75,0x48,0x53,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <16 x i32> @llvm.x86.avx512.mask.vpdpwssds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  ret <16 x i32> %res
+}
+
 define <16 x i32>@test_int_x86_avx512_mask_vpdpwssds_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32>* %x2p, <16 x i32> %x4, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpdpwssds_512:
 ; X86:       # %bb.0:
@@ -123,10 +135,7 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpwssds_512(<16 x i32> %x0, <16 x i
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpdpwssds (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x53,0x18]
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpdpwssds %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x53,0xe2]
 ; X86-NEXT:    vpdpwssds %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x53,0xc2]
-; X86-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -135,18 +144,13 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpwssds_512(<16 x i32> %x0, <16 x i
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X64-NEXT:    vpdpwssds (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x53,0x1f]
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X64-NEXT:    vpdpwssds %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x53,0xe2]
 ; X64-NEXT:    vpdpwssds %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x53,0xc2]
-; X64-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <16 x i32>, <16 x i32>* %x2p
   %res = call <16 x i32> @llvm.x86.avx512.mask.vpdpwssds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
-  %res1 = call <16 x i32> @llvm.x86.avx512.mask.vpdpwssds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16 -1)
-  %res2 = call <16 x i32> @llvm.x86.avx512.maskz.vpdpwssds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16  %x3)
-  %res3 = add <16 x i32> %res, %res1
-  %res4 = add <16 x i32> %res2, %res3
-  ret <16 x i32> %res4
+  %res1 = call <16 x i32> @llvm.x86.avx512.maskz.vpdpwssds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16  %x3)
+  %res2 = add <16 x i32> %res, %res1
+  ret <16 x i32> %res2
 }
 
diff --git a/llvm/test/CodeGen/X86/avx512vnni-intrinsics.ll b/llvm/test/CodeGen/X86/avx512vnni-intrinsics.ll
index 47fce1801a440..ebf5fb150e485 100644
--- a/llvm/test/CodeGen/X86/avx512vnni-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512vnni-intrinsics.ll
@@ -4,6 +4,15 @@
 
 declare <16 x i32> @llvm.x86.avx512.vpdpbusd.512(<16 x i32>, <16 x i32>, <16 x i32>)
 
+define <16 x i32>@test_int_x86_avx512_ask_vpdpbusd_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_ask_vpdpbusd_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpbusd %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf2,0x75,0x48,0x50,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <16 x i32> @llvm.x86.avx512.vpdpbusd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)
+  ret <16 x i32> %1
+}
+
 define <16 x i32>@test_int_x86_avx512_mask_vpdpbusd_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32>* %x2p, <16 x i32> %x4, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpdpbusd_512:
 ; X86:       # %bb.0:
@@ -11,10 +20,7 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpbusd_512(<16 x i32> %x0, <16 x i3
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpdpbusd (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x50,0x18]
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpdpbusd %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x50,0xe2]
 ; X86-NEXT:    vpdpbusd %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x50,0xc2]
-; X86-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -23,10 +29,7 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpbusd_512(<16 x i32> %x0, <16 x i3
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X64-NEXT:    vpdpbusd (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x50,0x1f]
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X64-NEXT:    vpdpbusd %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x50,0xe2]
 ; X64-NEXT:    vpdpbusd %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x50,0xc2]
-; X64-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <16 x i32>, <16 x i32>* %x2p
@@ -34,16 +37,23 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpbusd_512(<16 x i32> %x0, <16 x i3
   %2 = bitcast i16 %x3 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x0
   %4 = call <16 x i32> @llvm.x86.avx512.vpdpbusd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4)
-  %5 = call <16 x i32> @llvm.x86.avx512.vpdpbusd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4)
-  %6 = bitcast i16 %x3 to <16 x i1>
-  %7 = select <16 x i1> %6, <16 x i32> %5, <16 x i32> zeroinitializer
-  %res3 = add <16 x i32> %3, %4
-  %res4 = add <16 x i32> %7, %res3
-  ret <16 x i32> %res4
+  %5 = bitcast i16 %x3 to <16 x i1>
+  %6 = select <16 x i1> %5, <16 x i32> %4, <16 x i32> zeroinitializer
+  %res3 = add <16 x i32> %3, %6
+  ret <16 x i32> %res3
 }
 
 declare <16 x i32> @llvm.x86.avx512.vpdpbusds.512(<16 x i32>, <16 x i32>, <16 x i32>)
 
+define <16 x i32>@test_int_x86_avx512_vpdpbusds_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpdpbusds_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpbusds %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf2,0x75,0x48,0x51,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <16 x i32> @llvm.x86.avx512.vpdpbusds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)
+  ret <16 x i32> %1
+}
+
 define <16 x i32>@test_int_x86_avx512_mask_vpdpbusds_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32>* %x2p, <16 x i32> %x4, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpdpbusds_512:
 ; X86:       # %bb.0:
@@ -51,10 +61,7 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpbusds_512(<16 x i32> %x0, <16 x i
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpdpbusds (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x51,0x18]
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpdpbusds %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x51,0xe2]
 ; X86-NEXT:    vpdpbusds %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x51,0xc2]
-; X86-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -63,10 +70,7 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpbusds_512(<16 x i32> %x0, <16 x i
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X64-NEXT:    vpdpbusds (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x51,0x1f]
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X64-NEXT:    vpdpbusds %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x51,0xe2]
 ; X64-NEXT:    vpdpbusds %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x51,0xc2]
-; X64-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <16 x i32>, <16 x i32>* %x2p
@@ -74,16 +78,23 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpbusds_512(<16 x i32> %x0, <16 x i
   %2 = bitcast i16 %x3 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x0
   %4 = call <16 x i32> @llvm.x86.avx512.vpdpbusds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4)
-  %5 = call <16 x i32> @llvm.x86.avx512.vpdpbusds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4)
-  %6 = bitcast i16 %x3 to <16 x i1>
-  %7 = select <16 x i1> %6, <16 x i32> %5, <16 x i32> zeroinitializer
-  %res3 = add <16 x i32> %3, %4
-  %res4 = add <16 x i32> %7, %res3
-  ret <16 x i32> %res4
+  %5 = bitcast i16 %x3 to <16 x i1>
+  %6 = select <16 x i1> %5, <16 x i32> %4, <16 x i32> zeroinitializer
+  %res3 = add <16 x i32> %3, %6
+  ret <16 x i32> %res3
 }
 
 declare <16 x i32> @llvm.x86.avx512.vpdpwssd.512(<16 x i32>, <16 x i32>, <16 x i32>)
 
+define <16 x i32>@test_int_x86_avx512_vpdpwssd_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_vpdpwssd_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpwssd %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf2,0x75,0x48,0x52,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <16 x i32> @llvm.x86.avx512.vpdpwssd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)
+  ret <16 x i32> %1
+}
+
 define <16 x i32>@test_int_x86_avx512_mask_vpdpwssd_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32>* %x2p, <16 x i32> %x4, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpdpwssd_512:
 ; X86:       # %bb.0:
@@ -91,10 +102,7 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpwssd_512(<16 x i32> %x0, <16 x i3
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpdpwssd (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x52,0x18]
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpdpwssd %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x52,0xe2]
 ; X86-NEXT:    vpdpwssd %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x52,0xc2]
-; X86-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -103,10 +111,7 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpwssd_512(<16 x i32> %x0, <16 x i3
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X64-NEXT:    vpdpwssd (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x52,0x1f]
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X64-NEXT:    vpdpwssd %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x52,0xe2]
 ; X64-NEXT:    vpdpwssd %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x52,0xc2]
-; X64-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <16 x i32>, <16 x i32>* %x2p
@@ -114,16 +119,23 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpwssd_512(<16 x i32> %x0, <16 x i3
   %2 = bitcast i16 %x3 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x0
   %4 = call <16 x i32> @llvm.x86.avx512.vpdpwssd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4)
-  %5 = call <16 x i32> @llvm.x86.avx512.vpdpwssd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4)
-  %6 = bitcast i16 %x3 to <16 x i1>
-  %7 = select <16 x i1> %6, <16 x i32> %5, <16 x i32> zeroinitializer
-  %res3 = add <16 x i32> %3, %4
-  %res4 = add <16 x i32> %7, %res3
-  ret <16 x i32> %res4
+  %5 = bitcast i16 %x3 to <16 x i1>
+  %6 = select <16 x i1> %5, <16 x i32> %4, <16 x i32> zeroinitializer
+  %res3 = add <16 x i32> %3, %6
+  ret <16 x i32> %res3
 }
 
 declare <16 x i32> @llvm.x86.avx512.vpdpwssds.512(<16 x i32>, <16 x i32>, <16 x i32>)
 
+define <16 x i32>@test_int_x86_avx512_ask_vpdpwssds_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) {
+; CHECK-LABEL: test_int_x86_avx512_ask_vpdpwssds_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpdpwssds %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf2,0x75,0x48,0x53,0xc2]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <16 x i32> @llvm.x86.avx512.vpdpwssds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)
+  ret <16 x i32> %1
+}
+
 define <16 x i32>@test_int_x86_avx512_mask_vpdpwssds_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32>* %x2p, <16 x i32> %x4, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpdpwssds_512:
 ; X86:       # %bb.0:
@@ -131,10 +143,7 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpwssds_512(<16 x i32> %x0, <16 x i
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpdpwssds (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x53,0x18]
-; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpdpwssds %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x53,0xe2]
 ; X86-NEXT:    vpdpwssds %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x53,0xc2]
-; X86-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -143,10 +152,7 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpwssds_512(<16 x i32> %x0, <16 x i
 ; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
 ; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X64-NEXT:    vpdpwssds (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x53,0x1f]
-; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X64-NEXT:    vpdpwssds %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x53,0xe2]
 ; X64-NEXT:    vpdpwssds %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x53,0xc2]
-; X64-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <16 x i32>, <16 x i32>* %x2p
@@ -154,11 +160,8 @@ define <16 x i32>@test_int_x86_avx512_mask_vpdpwssds_512(<16 x i32> %x0, <16 x i
   %2 = bitcast i16 %x3 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x0
   %4 = call <16 x i32> @llvm.x86.avx512.vpdpwssds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4)
-  %5 = call <16 x i32> @llvm.x86.avx512.vpdpwssds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4)
-  %6 = bitcast i16 %x3 to <16 x i1>
-  %7 = select <16 x i1> %6, <16 x i32> %5, <16 x i32> zeroinitializer
-  %res3 = add <16 x i32> %3, %4
-  %res4 = add <16 x i32> %7, %res3
-  ret <16 x i32> %res4
+  %5 = bitcast i16 %x3 to <16 x i1>
+  %6 = select <16 x i1> %5, <16 x i32> %4, <16 x i32> zeroinitializer
+  %res3 = add <16 x i32> %3, %6
+  ret <16 x i32> %res3
 }
-
diff --git a/llvm/test/CodeGen/X86/bitcast-and-setcc-512.ll b/llvm/test/CodeGen/X86/bitcast-and-setcc-512.ll
index 3852b0efd4b17..588eddc3c93d1 100644
--- a/llvm/test/CodeGen/X86/bitcast-and-setcc-512.ll
+++ b/llvm/test/CodeGen/X86/bitcast-and-setcc-512.ll
@@ -13,38 +13,26 @@ define i8 @v8i64(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c, <8 x i64> %d) {
 ; SSE-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm10
 ; SSE-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm11
 ; SSE-NEXT:    pcmpgtq %xmm7, %xmm3
-; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
-; SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,1,0,2,4,5,6,7]
 ; SSE-NEXT:    pcmpgtq %xmm6, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
-; SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,1,0,2,4,5,6,7]
-; SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SSE-NEXT:    packssdw %xmm3, %xmm2
+; SSE-NEXT:    packssdw %xmm2, %xmm2
 ; SSE-NEXT:    pcmpgtq %xmm5, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
 ; SSE-NEXT:    pcmpgtq %xmm4, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
-; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-NEXT:    packssdw %xmm1, %xmm0
+; SSE-NEXT:    packssdw %xmm0, %xmm0
 ; SSE-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
 ; SSE-NEXT:    pcmpgtq {{[0-9]+}}(%rsp), %xmm11
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm11[0,2,2,3]
-; SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,2,4,5,6,7]
 ; SSE-NEXT:    pcmpgtq {{[0-9]+}}(%rsp), %xmm10
-; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm10[0,2,2,3]
-; SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,1,0,2,4,5,6,7]
-; SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE-NEXT:    packssdw %xmm11, %xmm10
+; SSE-NEXT:    packssdw %xmm10, %xmm1
 ; SSE-NEXT:    pcmpgtq {{[0-9]+}}(%rsp), %xmm9
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm9[0,2,2,3]
-; SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
 ; SSE-NEXT:    pcmpgtq {{[0-9]+}}(%rsp), %xmm8
-; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm8[0,2,2,3]
-; SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
-; SSE-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
-; SSE-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm2[4,5,6,7]
-; SSE-NEXT:    pand %xmm0, %xmm3
-; SSE-NEXT:    packsswb %xmm0, %xmm3
-; SSE-NEXT:    pmovmskb %xmm3, %eax
+; SSE-NEXT:    packssdw %xmm9, %xmm8
+; SSE-NEXT:    packssdw %xmm0, %xmm8
+; SSE-NEXT:    pblendw {{.*#+}} xmm8 = xmm8[0,1,2,3],xmm1[4,5,6,7]
+; SSE-NEXT:    pand %xmm0, %xmm8
+; SSE-NEXT:    packsswb %xmm0, %xmm8
+; SSE-NEXT:    pmovmskb %xmm8, %eax
 ; SSE-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE-NEXT:    retq
 ;
@@ -125,38 +113,26 @@ define i8 @v8f64(<8 x double> %a, <8 x double> %b, <8 x double> %c, <8 x double>
 ; SSE-NEXT:    movapd {{[0-9]+}}(%rsp), %xmm10
 ; SSE-NEXT:    movapd {{[0-9]+}}(%rsp), %xmm11
 ; SSE-NEXT:    cmpltpd %xmm3, %xmm7
-; SSE-NEXT:    shufps {{.*#+}} xmm7 = xmm7[0,2,2,3]
-; SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm7[0,1,0,2,4,5,6,7]
 ; SSE-NEXT:    cmpltpd %xmm2, %xmm6
-; SSE-NEXT:    shufps {{.*#+}} xmm6 = xmm6[0,2,2,3]
-; SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm6[0,1,0,2,4,5,6,7]
-; SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SSE-NEXT:    packssdw %xmm7, %xmm6
+; SSE-NEXT:    packssdw %xmm6, %xmm2
 ; SSE-NEXT:    cmpltpd %xmm1, %xmm5
-; SSE-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,2,2,3]
-; SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm5[0,2,2,3,4,5,6,7]
 ; SSE-NEXT:    cmpltpd %xmm0, %xmm4
-; SSE-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,2,2,3]
-; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm4[0,2,2,3,4,5,6,7]
-; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
+; SSE-NEXT:    packssdw %xmm5, %xmm4
+; SSE-NEXT:    packssdw %xmm0, %xmm4
+; SSE-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm2[4,5,6,7]
 ; SSE-NEXT:    cmpltpd {{[0-9]+}}(%rsp), %xmm11
-; SSE-NEXT:    shufps {{.*#+}} xmm11 = xmm11[0,2,2,3]
-; SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm11[0,1,0,2,4,5,6,7]
 ; SSE-NEXT:    cmpltpd {{[0-9]+}}(%rsp), %xmm10
-; SSE-NEXT:    shufps {{.*#+}} xmm10 = xmm10[0,2,2,3]
-; SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm10[0,1,0,2,4,5,6,7]
-; SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE-NEXT:    packssdw %xmm11, %xmm10
+; SSE-NEXT:    packssdw %xmm10, %xmm0
 ; SSE-NEXT:    cmpltpd {{[0-9]+}}(%rsp), %xmm9
-; SSE-NEXT:    shufps {{.*#+}} xmm9 = xmm9[0,2,2,3]
-; SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm9[0,2,2,3,4,5,6,7]
 ; SSE-NEXT:    cmpltpd {{[0-9]+}}(%rsp), %xmm8
-; SSE-NEXT:    shufps {{.*#+}} xmm8 = xmm8[0,2,2,3]
-; SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm8[0,2,2,3,4,5,6,7]
-; SSE-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
-; SSE-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm2[4,5,6,7]
-; SSE-NEXT:    pand %xmm0, %xmm3
-; SSE-NEXT:    packsswb %xmm0, %xmm3
-; SSE-NEXT:    pmovmskb %xmm3, %eax
+; SSE-NEXT:    packssdw %xmm9, %xmm8
+; SSE-NEXT:    packssdw %xmm0, %xmm8
+; SSE-NEXT:    pblendw {{.*#+}} xmm8 = xmm8[0,1,2,3],xmm0[4,5,6,7]
+; SSE-NEXT:    pand %xmm4, %xmm8
+; SSE-NEXT:    packsswb %xmm0, %xmm8
+; SSE-NEXT:    pmovmskb %xmm8, %eax
 ; SSE-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE-NEXT:    retq
 ;
@@ -337,33 +313,23 @@ define i16 @v16i32(<16 x i32> %a, <16 x i32> %b, <16 x i32> %c, <16 x i32> %d) {
 ; SSE-LABEL: v16i32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm8
-; SSE-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm10
 ; SSE-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
+; SSE-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm10
 ; SSE-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm11
 ; SSE-NEXT:    pcmpgtd %xmm7, %xmm3
-; SSE-NEXT:    movdqa {{.*#+}} xmm7 = <u,u,u,u,0,4,8,12,u,u,u,u,u,u,u,u>
-; SSE-NEXT:    pshufb %xmm7, %xmm3
 ; SSE-NEXT:    pcmpgtd %xmm6, %xmm2
-; SSE-NEXT:    pshufb %xmm7, %xmm2
-; SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SSE-NEXT:    packssdw %xmm3, %xmm2
 ; SSE-NEXT:    pcmpgtd %xmm5, %xmm1
-; SSE-NEXT:    movdqa {{.*#+}} xmm3 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
-; SSE-NEXT:    pshufb %xmm3, %xmm1
 ; SSE-NEXT:    pcmpgtd %xmm4, %xmm0
-; SSE-NEXT:    pshufb %xmm3, %xmm0
-; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
+; SSE-NEXT:    packssdw %xmm1, %xmm0
+; SSE-NEXT:    packsswb %xmm2, %xmm0
 ; SSE-NEXT:    pcmpgtd {{[0-9]+}}(%rsp), %xmm11
-; SSE-NEXT:    pshufb %xmm7, %xmm11
-; SSE-NEXT:    pcmpgtd {{[0-9]+}}(%rsp), %xmm9
-; SSE-NEXT:    pshufb %xmm7, %xmm9
-; SSE-NEXT:    punpckldq {{.*#+}} xmm9 = xmm9[0],xmm11[0],xmm9[1],xmm11[1]
 ; SSE-NEXT:    pcmpgtd {{[0-9]+}}(%rsp), %xmm10
-; SSE-NEXT:    pshufb %xmm3, %xmm10
+; SSE-NEXT:    packssdw %xmm11, %xmm10
+; SSE-NEXT:    pcmpgtd {{[0-9]+}}(%rsp), %xmm9
 ; SSE-NEXT:    pcmpgtd {{[0-9]+}}(%rsp), %xmm8
-; SSE-NEXT:    pshufb %xmm3, %xmm8
-; SSE-NEXT:    punpckldq {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1]
-; SSE-NEXT:    pblendw {{.*#+}} xmm8 = xmm8[0,1,2,3],xmm9[4,5,6,7]
+; SSE-NEXT:    packssdw %xmm9, %xmm8
+; SSE-NEXT:    packsswb %xmm10, %xmm8
 ; SSE-NEXT:    pand %xmm0, %xmm8
 ; SSE-NEXT:    pmovmskb %xmm8, %eax
 ; SSE-NEXT:    # kill: def $ax killed $ax killed $eax
@@ -449,33 +415,23 @@ define i16 @v16f32(<16 x float> %a, <16 x float> %b, <16 x float> %c, <16 x floa
 ; SSE-LABEL: v16f32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm8
-; SSE-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm10
 ; SSE-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm9
+; SSE-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm10
 ; SSE-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm11
 ; SSE-NEXT:    cmpltps %xmm3, %xmm7
-; SSE-NEXT:    movdqa {{.*#+}} xmm3 = <u,u,u,u,0,4,8,12,u,u,u,u,u,u,u,u>
-; SSE-NEXT:    pshufb %xmm3, %xmm7
 ; SSE-NEXT:    cmpltps %xmm2, %xmm6
-; SSE-NEXT:    pshufb %xmm3, %xmm6
-; SSE-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
+; SSE-NEXT:    packssdw %xmm7, %xmm6
 ; SSE-NEXT:    cmpltps %xmm1, %xmm5
-; SSE-NEXT:    movdqa {{.*#+}} xmm1 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
-; SSE-NEXT:    pshufb %xmm1, %xmm5
 ; SSE-NEXT:    cmpltps %xmm0, %xmm4
-; SSE-NEXT:    pshufb %xmm1, %xmm4
-; SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; SSE-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm6[4,5,6,7]
+; SSE-NEXT:    packssdw %xmm5, %xmm4
+; SSE-NEXT:    packsswb %xmm6, %xmm4
 ; SSE-NEXT:    cmpltps {{[0-9]+}}(%rsp), %xmm11
-; SSE-NEXT:    pshufb %xmm3, %xmm11
-; SSE-NEXT:    cmpltps {{[0-9]+}}(%rsp), %xmm9
-; SSE-NEXT:    pshufb %xmm3, %xmm9
-; SSE-NEXT:    punpckldq {{.*#+}} xmm9 = xmm9[0],xmm11[0],xmm9[1],xmm11[1]
 ; SSE-NEXT:    cmpltps {{[0-9]+}}(%rsp), %xmm10
-; SSE-NEXT:    pshufb %xmm1, %xmm10
+; SSE-NEXT:    packssdw %xmm11, %xmm10
+; SSE-NEXT:    cmpltps {{[0-9]+}}(%rsp), %xmm9
 ; SSE-NEXT:    cmpltps {{[0-9]+}}(%rsp), %xmm8
-; SSE-NEXT:    pshufb %xmm1, %xmm8
-; SSE-NEXT:    punpckldq {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1]
-; SSE-NEXT:    pblendw {{.*#+}} xmm8 = xmm8[0,1,2,3],xmm9[4,5,6,7]
+; SSE-NEXT:    packssdw %xmm9, %xmm8
+; SSE-NEXT:    packsswb %xmm10, %xmm8
 ; SSE-NEXT:    pand %xmm4, %xmm8
 ; SSE-NEXT:    pmovmskb %xmm8, %eax
 ; SSE-NEXT:    # kill: def $ax killed $ax killed $eax
diff --git a/llvm/test/CodeGen/X86/buildvec-extract.ll b/llvm/test/CodeGen/X86/buildvec-extract.ll
index 218701250e43c..863ab4dee1238 100644
--- a/llvm/test/CodeGen/X86/buildvec-extract.ll
+++ b/llvm/test/CodeGen/X86/buildvec-extract.ll
@@ -293,24 +293,19 @@ define <2 x i64> @extract2_i32_zext_insert1_i64_undef(<4 x i32> %x) {
 define <2 x i64> @extract2_i32_zext_insert1_i64_zero(<4 x i32> %x) {
 ; SSE2-LABEL: extract2_i32_zext_insert1_i64_zero:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    movq %rax, %xmm0
-; SSE2-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; SSE2-NEXT:    andps {{.*}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: extract2_i32_zext_insert1_i64_zero:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    extractps $2, %xmm0, %eax
-; SSE41-NEXT:    movq %rax, %xmm0
-; SSE41-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; SSE41-NEXT:    xorps %xmm1, %xmm1
+; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3]
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: extract2_i32_zext_insert1_i64_zero:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vextractps $2, %xmm0, %eax
-; AVX-NEXT:    vmovq %rax, %xmm0
-; AVX-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3]
 ; AVX-NEXT:    retq
   %e = extractelement <4 x i32> %x, i32 2
   %z = zext i32 %e to i64
@@ -386,16 +381,22 @@ define <2 x i64> @extract0_i16_zext_insert0_i64_undef(<8 x i16> %x) {
 }
 
 define <2 x i64> @extract0_i16_zext_insert0_i64_zero(<8 x i16> %x) {
-; SSE-LABEL: extract0_i16_zext_insert0_i64_zero:
-; SSE:       # %bb.0:
-; SSE-NEXT:    pextrw $0, %xmm0, %eax
-; SSE-NEXT:    movd %eax, %xmm0
-; SSE-NEXT:    retq
+; SSE2-LABEL: extract0_i16_zext_insert0_i64_zero:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pextrw $0, %xmm0, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: extract0_i16_zext_insert0_i64_zero:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: extract0_i16_zext_insert0_i64_zero:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpextrw $0, %xmm0, %eax
-; AVX-NEXT:    vmovd %eax, %xmm0
+; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
 ; AVX-NEXT:    retq
   %e = extractelement <8 x i16> %x, i32 0
   %z = zext i16 %e to i64
diff --git a/llvm/test/CodeGen/X86/buildvec-insertvec.ll b/llvm/test/CodeGen/X86/buildvec-insertvec.ll
index 3add65914b58a..0ee2f424af1a2 100644
--- a/llvm/test/CodeGen/X86/buildvec-insertvec.ll
+++ b/llvm/test/CodeGen/X86/buildvec-insertvec.ll
@@ -21,10 +21,7 @@ define void @foo(<3 x float> %in, <4 x i8>* nocapture %out) nounwind {
 ; SSE41-LABEL: foo:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    cvttps2dq %xmm0, %xmm0
-; SSE41-NEXT:    pextrb $8, %xmm0, %eax
-; SSE41-NEXT:    pextrb $4, %xmm0, %ecx
-; SSE41-NEXT:    pinsrb $1, %ecx, %xmm0
-; SSE41-NEXT:    pinsrb $2, %eax, %xmm0
+; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,3,u,u,u,u,u,u,u,u,u,u,u,u]
 ; SSE41-NEXT:    movl $255, %eax
 ; SSE41-NEXT:    pinsrb $3, %eax, %xmm0
 ; SSE41-NEXT:    movd %xmm0, (%rdi)
diff --git a/llvm/test/CodeGen/X86/callbr-asm-obj-file.ll b/llvm/test/CodeGen/X86/callbr-asm-obj-file.ll
index a8a392541b496..938eabfc7d99c 100644
--- a/llvm/test/CodeGen/X86/callbr-asm-obj-file.ll
+++ b/llvm/test/CodeGen/X86/callbr-asm-obj-file.ll
@@ -3,7 +3,7 @@
 ; RUN:  | FileCheck %s
 
 ; CHECK: 0000000000000000 <test1>:
-; CHECK-NEXT:   0: 74 00 je 0 <test1+0x2>
+; CHECK-NEXT:   0: 74 00 je 0x2 <test1+0x2>
 ; CHECK-NEXT:   2: c3    retq
 
 define void @test1() {
diff --git a/llvm/test/CodeGen/X86/coff-linkonce.ll b/llvm/test/CodeGen/X86/coff-linkonce.ll
new file mode 100644
index 0000000000000..369750147f292
--- /dev/null
+++ b/llvm/test/CodeGen/X86/coff-linkonce.ll
@@ -0,0 +1,9 @@
+; RUN: llc -function-sections -o - %s | FileCheck %s
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+; CHECK: .section{{.*}}one_only
+define linkonce_odr void @foo() {
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/coff-weak.ll b/llvm/test/CodeGen/X86/coff-weak.ll
index 369750147f292..485e8f6a3847b 100644
--- a/llvm/test/CodeGen/X86/coff-weak.ll
+++ b/llvm/test/CodeGen/X86/coff-weak.ll
@@ -1,9 +1,50 @@
-; RUN: llc -function-sections -o - %s | FileCheck %s
+; RUN: llc -mtriple=i686-windows-msvc -o - %s | FileCheck -check-prefix=X86 %s
+; RUN: llc -mtriple=x86_64-windows-msvc -o - %s | FileCheck -check-prefix=X64 %s
 
-target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-pc-windows-msvc"
+; X86: .weak _foo
+; X64: .weak foo
+define weak void @foo() {
+  ret void
+}
+
+; X86: .weak _bar
+; X64: .weak bar
+define weak_odr void @bar() {
+  ret void
+}
+
+; X86-NOT: .weak _bar_comdat
+; X64-NOT: .weak bar_comdat
+$bar_comdat = comdat any
+
+define weak_odr void @bar_comdat() comdat {
+  ret void
+}
+
+; X86: .weak _baz
+; X64: .weak baz
+define linkonce void @baz() {
+  ret void
+}
+
+; X86-NOT: .weak _baz_comdat
+; X64-NOT: .weak baz_comdat
+$baz_comdat = comdat any
+
+define linkonce void @baz_comdat() comdat {
+  ret void
+}
+
+; X86: .weak _quux
+; X64: .weak quux
+define linkonce_odr void @quux() {
+  ret void
+}
+
+; X86-NOT: .weak _quux_comdat
+; X64-NOT: .weak quux_comdat
+$quux_comdat = comdat any
 
-; CHECK: .section{{.*}}one_only
-define linkonce_odr void @foo() {
+define linkonce_odr void @quux_comdat() comdat {
   ret void
 }
diff --git a/llvm/test/CodeGen/X86/combine-ptest.ll b/llvm/test/CodeGen/X86/combine-ptest.ll
new file mode 100644
index 0000000000000..e12a4152ac7ed
--- /dev/null
+++ b/llvm/test/CodeGen/X86/combine-ptest.ll
@@ -0,0 +1,238 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s
+
+;
+; testz(~X,Y) -> testc(X,Y)
+;
+
+define i32 @ptestz_128_invert0(<2 x i64> %c, <2 x i64> %d, i32 %a, i32 %b) {
+; CHECK-LABEL: ptestz_128_invert0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    vptest %xmm1, %xmm0
+; CHECK-NEXT:    cmovael %esi, %eax
+; CHECK-NEXT:    retq
+  %t1 = xor <2 x i64> %c, <i64 -1, i64 -1>
+  %t2 = call i32 @llvm.x86.sse41.ptestz(<2 x i64> %t1, <2 x i64> %d)
+  %t3 = icmp ne i32 %t2, 0
+  %t4 = select i1 %t3, i32 %a, i32 %b
+  ret i32 %t4
+}
+
+define i32 @ptestz_256_invert0(<4 x i64> %c, <4 x i64> %d, i32 %a, i32 %b) {
+; CHECK-LABEL: ptestz_256_invert0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    vptest %ymm1, %ymm0
+; CHECK-NEXT:    cmovael %esi, %eax
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %t1 = xor <4 x i64> %c, <i64 -1, i64 -1, i64 -1, i64 -1>
+  %t2 = call i32 @llvm.x86.avx.ptestz.256(<4 x i64> %t1, <4 x i64> %d)
+  %t3 = icmp ne i32 %t2, 0
+  %t4 = select i1 %t3, i32 %a, i32 %b
+  ret i32 %t4
+}
+
+;
+; testz(X,~Y) -> testc(Y,X)
+;
+
+define i32 @ptestz_128_invert1(<2 x i64> %c, <2 x i64> %d, i32 %a, i32 %b) {
+; CHECK-LABEL: ptestz_128_invert1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    vptest %xmm0, %xmm1
+; CHECK-NEXT:    cmovael %esi, %eax
+; CHECK-NEXT:    retq
+  %t1 = xor <2 x i64> %d, <i64 -1, i64 -1>
+  %t2 = call i32 @llvm.x86.sse41.ptestz(<2 x i64> %c, <2 x i64> %t1)
+  %t3 = icmp ne i32 %t2, 0
+  %t4 = select i1 %t3, i32 %a, i32 %b
+  ret i32 %t4
+}
+
+define i32 @ptestz_256_invert1(<4 x i64> %c, <4 x i64> %d, i32 %a, i32 %b) {
+; CHECK-LABEL: ptestz_256_invert1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    vptest %ymm0, %ymm1
+; CHECK-NEXT:    cmovael %esi, %eax
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %t1 = xor <4 x i64> %d, <i64 -1, i64 -1, i64 -1, i64 -1>
+  %t2 = call i32 @llvm.x86.avx.ptestz.256(<4 x i64> %c, <4 x i64> %t1)
+  %t3 = icmp ne i32 %t2, 0
+  %t4 = select i1 %t3, i32 %a, i32 %b
+  ret i32 %t4
+}
+
+;
+; testc(~X,Y) -> testz(X,Y)
+;
+
+define i32 @ptestc_128_invert0(<2 x i64> %c, <2 x i64> %d, i32 %a, i32 %b) {
+; CHECK-LABEL: ptestc_128_invert0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    vptest %xmm1, %xmm0
+; CHECK-NEXT:    cmovnel %esi, %eax
+; CHECK-NEXT:    retq
+  %t1 = xor <2 x i64> %c, <i64 -1, i64 -1>
+  %t2 = call i32 @llvm.x86.sse41.ptestc(<2 x i64> %t1, <2 x i64> %d)
+  %t3 = icmp ne i32 %t2, 0
+  %t4 = select i1 %t3, i32 %a, i32 %b
+  ret i32 %t4
+}
+
+define i32 @ptestc_256_invert0(<4 x i64> %c, <4 x i64> %d, i32 %a, i32 %b) {
+; CHECK-LABEL: ptestc_256_invert0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    vptest %ymm1, %ymm0
+; CHECK-NEXT:    cmovnel %esi, %eax
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %t1 = xor <4 x i64> %c, <i64 -1, i64 -1, i64 -1, i64 -1>
+  %t2 = call i32 @llvm.x86.avx.ptestc.256(<4 x i64> %t1, <4 x i64> %d)
+  %t3 = icmp ne i32 %t2, 0
+  %t4 = select i1 %t3, i32 %a, i32 %b
+  ret i32 %t4
+}
+
+;
+; testnzc(~X,Y) -> testnzc(X,Y)
+;
+
+define i32 @ptestnzc_128_invert0(<2 x i64> %c, <2 x i64> %d, i32 %a, i32 %b) {
+; CHECK-LABEL: ptestnzc_128_invert0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    vptest %xmm1, %xmm0
+; CHECK-NEXT:    cmovnel %esi, %eax
+; CHECK-NEXT:    retq
+  %t1 = xor <2 x i64> %c, <i64 -1, i64 -1>
+  %t2 = call i32 @llvm.x86.sse41.ptestc(<2 x i64> %t1, <2 x i64> %d)
+  %t3 = icmp ne i32 %t2, 0
+  %t4 = select i1 %t3, i32 %a, i32 %b
+  ret i32 %t4
+}
+
+define i32 @ptestnzc_256_invert0(<4 x i64> %c, <4 x i64> %d, i32 %a, i32 %b) {
+; CHECK-LABEL: ptestnzc_256_invert0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    vptest %ymm1, %ymm0
+; CHECK-NEXT:    cmovbel %esi, %eax
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %t1 = xor <4 x i64> %c, <i64 -1, i64 -1, i64 -1, i64 -1>
+  %t2 = call i32 @llvm.x86.avx.ptestnzc.256(<4 x i64> %t1, <4 x i64> %d)
+  %t3 = icmp ne i32 %t2, 0
+  %t4 = select i1 %t3, i32 %a, i32 %b
+  ret i32 %t4
+}
+
+define i32 @ptestnzc_256_invert0_commute(<4 x i64> %c, <4 x i64> %d, i32 %a, i32 %b) {
+; CHECK-LABEL: ptestnzc_256_invert0_commute:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    vptest %ymm1, %ymm0
+; CHECK-NEXT:    cmoval %esi, %eax
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %t1 = xor <4 x i64> %c, <i64 -1, i64 -1, i64 -1, i64 -1>
+  %t2 = call i32 @llvm.x86.avx.ptestnzc.256(<4 x i64> %t1, <4 x i64> %d)
+  %t3 = icmp eq i32 %t2, 0
+  %t4 = select i1 %t3, i32 %a, i32 %b
+  ret i32 %t4
+}
+
+;
+; testz(-1,X) -> testz(X,X)
+;
+
+define i32 @ptestz_128_allones0(<2 x i64> %c, i32 %a, i32 %b) {
+; CHECK-LABEL: ptestz_128_allones0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    vptest %xmm0, %xmm0
+; CHECK-NEXT:    cmovnel %esi, %eax
+; CHECK-NEXT:    retq
+  %t1 = call i32 @llvm.x86.sse41.ptestz(<2 x i64> <i64 -1, i64 -1>, <2 x i64> %c)
+  %t2 = icmp ne i32 %t1, 0
+  %t3 = select i1 %t2, i32 %a, i32 %b
+  ret i32 %t3
+}
+
+define i32 @ptestz_256_allones0(<4 x i64> %c, i32 %a, i32 %b) {
+; CHECK-LABEL: ptestz_256_allones0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    vptest %ymm0, %ymm0
+; CHECK-NEXT:    cmovnel %esi, %eax
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %t1 = call i32 @llvm.x86.avx.ptestz.256(<4 x i64> <i64 -1, i64 -1, i64 -1, i64 -1>, <4 x i64> %c)
+  %t2 = icmp ne i32 %t1, 0
+  %t3 = select i1 %t2, i32 %a, i32 %b
+  ret i32 %t3
+}
+
+;
+; testz(X,-1) -> testz(X,X)
+;
+
+define i32 @ptestz_128_allones1(<2 x i64> %c, i32 %a, i32 %b) {
+; CHECK-LABEL: ptestz_128_allones1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    vptest %xmm0, %xmm0
+; CHECK-NEXT:    cmovnel %esi, %eax
+; CHECK-NEXT:    retq
+  %t1 = call i32 @llvm.x86.sse41.ptestz(<2 x i64> %c, <2 x i64> <i64 -1, i64 -1>)
+  %t2 = icmp ne i32 %t1, 0
+  %t3 = select i1 %t2, i32 %a, i32 %b
+  ret i32 %t3
+}
+
+define i32 @ptestz_256_allones1(<4 x i64> %c, i32 %a, i32 %b) {
+; CHECK-LABEL: ptestz_256_allones1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    vptest %ymm0, %ymm0
+; CHECK-NEXT:    cmovnel %esi, %eax
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %t1 = call i32 @llvm.x86.avx.ptestz.256(<4 x i64> %c, <4 x i64> <i64 -1, i64 -1, i64 -1, i64 -1>)
+  %t2 = icmp ne i32 %t1, 0
+  %t3 = select i1 %t2, i32 %a, i32 %b
+  ret i32 %t3
+}
+
+define zeroext i1 @PR38522(<16 x i8>* %x, <16 x i8>* %y) {
+; CHECK-LABEL: PR38522:
+; CHECK:       # %bb.0: # %start
+; CHECK-NEXT:    vmovdqa (%rdi), %xmm0
+; CHECK-NEXT:    vpcmpgtb (%rsi), %xmm0, %xmm0
+; CHECK-NEXT:    vptest %xmm0, %xmm0
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    retq
+start:
+  %0 = load <16 x i8>, <16 x i8>* %x, align 16
+  %1 = load <16 x i8>, <16 x i8>* %y, align 16
+  %2 = icmp sle <16 x i8> %0, %1
+  %3 = sext <16 x i1> %2 to <16 x i8>
+  %4 = bitcast <16 x i8> %3 to <2 x i64>
+  %5 = tail call i32 @llvm.x86.sse41.ptestc(<2 x i64> %4, <2 x i64> <i64 -1, i64 -1>)
+  %6 = icmp eq i32 %5, 1
+  ret i1 %6
+}
+
+declare i32 @llvm.x86.sse41.ptestz(<2 x i64>, <2 x i64>) nounwind readnone
+declare i32 @llvm.x86.sse41.ptestc(<2 x i64>, <2 x i64>) nounwind readnone
+declare i32 @llvm.x86.sse41.ptestnzc(<2 x i64>, <2 x i64>) nounwind readnone
+
+declare i32 @llvm.x86.avx.ptestz.256(<4 x i64>, <4 x i64>)
+declare i32 @llvm.x86.avx.ptestc.256(<4 x i64>, <4 x i64>)
+declare i32 @llvm.x86.avx.ptestnzc.256(<4 x i64>, <4 x i64>)
diff --git a/llvm/test/CodeGen/X86/combine-testpd.ll b/llvm/test/CodeGen/X86/combine-testpd.ll
new file mode 100644
index 0000000000000..9ae3d80e59cd7
--- /dev/null
+++ b/llvm/test/CodeGen/X86/combine-testpd.ll
@@ -0,0 +1,158 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s
+
+;
+; testz(~X,Y) -> testc(X,Y)
+;
+
+define i32 @testpdz_128_invert0(<2 x double> %c, <2 x double> %d, i32 %a, i32 %b) {
+; CHECK-LABEL: testpdz_128_invert0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    vtestpd %xmm1, %xmm0
+; CHECK-NEXT:    cmovael %esi, %eax
+; CHECK-NEXT:    retq
+  %t0 = bitcast <2 x double> %c to <2 x i64>
+  %t1 = xor <2 x i64> %t0, <i64 -1, i64 -1>
+  %t2 = bitcast <2 x i64> %t1 to <2 x double>
+  %t3 = call i32 @llvm.x86.avx.vtestz.pd(<2 x double> %t2, <2 x double> %d)
+  %t4 = icmp ne i32 %t3, 0
+  %t5 = select i1 %t4, i32 %a, i32 %b
+  ret i32 %t5
+}
+
+define i32 @testpdz_256_invert0(<4 x double> %c, <4 x double> %d, i32 %a, i32 %b) {
+; CHECK-LABEL: testpdz_256_invert0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    vtestpd %ymm1, %ymm0
+; CHECK-NEXT:    cmovael %esi, %eax
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %t0 = bitcast <4 x double> %c to <4 x i64>
+  %t1 = xor <4 x i64> %t0, <i64 -1, i64 -1, i64 -1, i64 -1>
+  %t2 = bitcast <4 x i64> %t1 to <4 x double>
+  %t3 = call i32 @llvm.x86.avx.vtestz.pd.256(<4 x double> %t2, <4 x double> %d)
+  %t4 = icmp ne i32 %t3, 0
+  %t5 = select i1 %t4, i32 %a, i32 %b
+  ret i32 %t5
+}
+
+;
+; testz(X,~Y) -> testc(Y,X)
+;
+
+define i32 @testpdz_128_invert1(<2 x double> %c, <2 x double> %d, i32 %a, i32 %b) {
+; CHECK-LABEL: testpdz_128_invert1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    vtestpd %xmm0, %xmm1
+; CHECK-NEXT:    cmovael %esi, %eax
+; CHECK-NEXT:    retq
+  %t0 = bitcast <2 x double> %d to <2 x i64>
+  %t1 = xor <2 x i64> %t0, <i64 -1, i64 -1>
+  %t2 = bitcast <2 x i64> %t1 to <2 x double>
+  %t3 = call i32 @llvm.x86.avx.vtestz.pd(<2 x double> %c, <2 x double> %t2)
+  %t4 = icmp ne i32 %t3, 0
+  %t5 = select i1 %t4, i32 %a, i32 %b
+  ret i32 %t5
+}
+
+define i32 @testpdz_256_invert1(<4 x double> %c, <4 x double> %d, i32 %a, i32 %b) {
+; CHECK-LABEL: testpdz_256_invert1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    vtestpd %ymm0, %ymm1
+; CHECK-NEXT:    cmovael %esi, %eax
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %t0 = bitcast <4 x double> %d to <4 x i64>
+  %t1 = xor <4 x i64> %t0, <i64 -1, i64 -1, i64 -1, i64 -1>
+  %t2 = bitcast <4 x i64> %t1 to <4 x double>
+  %t3 = call i32 @llvm.x86.avx.vtestz.pd.256(<4 x double> %c, <4 x double> %t2)
+  %t4 = icmp ne i32 %t3, 0
+  %t5 = select i1 %t4, i32 %a, i32 %b
+  ret i32 %t5
+}
+
+;
+; testc(~X,Y) -> testz(X,Y)
+;
+
+define i32 @testpdc_128_invert0(<2 x double> %c, <2 x double> %d, i32 %a, i32 %b) {
+; CHECK-LABEL: testpdc_128_invert0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    vtestpd %xmm1, %xmm0
+; CHECK-NEXT:    cmovnel %esi, %eax
+; CHECK-NEXT:    retq
+  %t0 = bitcast <2 x double> %c to <2 x i64>
+  %t1 = xor <2 x i64> %t0, <i64 -1, i64 -1>
+  %t2 = bitcast <2 x i64> %t1 to <2 x double>
+  %t3 = call i32 @llvm.x86.avx.vtestc.pd(<2 x double> %t2, <2 x double> %d)
+  %t4 = icmp ne i32 %t3, 0
+  %t5 = select i1 %t4, i32 %a, i32 %b
+  ret i32 %t5
+}
+
+define i32 @testpdc_256_invert0(<4 x double> %c, <4 x double> %d, i32 %a, i32 %b) {
+; CHECK-LABEL: testpdc_256_invert0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    vtestpd %ymm1, %ymm0
+; CHECK-NEXT:    cmovnel %esi, %eax
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %t0 = bitcast <4 x double> %c to <4 x i64>
+  %t1 = xor <4 x i64> %t0, <i64 -1, i64 -1, i64 -1, i64 -1>
+  %t2 = bitcast <4 x i64> %t1 to <4 x double>
+  %t3 = call i32 @llvm.x86.avx.vtestc.pd.256(<4 x double> %t2, <4 x double> %d)
+  %t4 = icmp ne i32 %t3, 0
+  %t5 = select i1 %t4, i32 %a, i32 %b
+  ret i32 %t5
+}
+
+;
+; testnzc(~X,Y) -> testnzc(X,Y)
+;
+
+define i32 @testpdnzc_128_invert0(<2 x double> %c, <2 x double> %d, i32 %a, i32 %b) {
+; CHECK-LABEL: testpdnzc_128_invert0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    vtestpd %xmm1, %xmm0
+; CHECK-NEXT:    cmovbel %esi, %eax
+; CHECK-NEXT:    retq
+  %t0 = bitcast <2 x double> %c to <2 x i64>
+  %t1 = xor <2 x i64> %t0, <i64 -1, i64 -1>
+  %t2 = bitcast <2 x i64> %t1 to <2 x double>
+  %t3 = call i32 @llvm.x86.avx.vtestnzc.pd(<2 x double> %t2, <2 x double> %d)
+  %t4 = icmp ne i32 %t3, 0
+  %t5 = select i1 %t4, i32 %a, i32 %b
+  ret i32 %t5
+}
+
+define i32 @testpdnzc_256_invert0(<4 x double> %c, <4 x double> %d, i32 %a, i32 %b) {
+; CHECK-LABEL: testpdnzc_256_invert0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    vtestpd %ymm1, %ymm0
+; CHECK-NEXT:    cmovbel %esi, %eax
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %t0 = bitcast <4 x double> %c to <4 x i64>
+  %t1 = xor <4 x i64> %t0, <i64 -1, i64 -1, i64 -1, i64 -1>
+  %t2 = bitcast <4 x i64> %t1 to <4 x double>
+  %t3 = call i32 @llvm.x86.avx.vtestnzc.pd.256(<4 x double> %t2, <4 x double> %d)
+  %t4 = icmp ne i32 %t3, 0
+  %t5 = select i1 %t4, i32 %a, i32 %b
+  ret i32 %t5
+}
+
+declare i32 @llvm.x86.avx.vtestz.pd(<2 x double>, <2 x double>) nounwind readnone
+declare i32 @llvm.x86.avx.vtestc.pd(<2 x double>, <2 x double>) nounwind readnone
+declare i32 @llvm.x86.avx.vtestnzc.pd(<2 x double>, <2 x double>) nounwind readnone
+
+declare i32 @llvm.x86.avx.vtestz.pd.256(<4 x double>, <4 x double>) nounwind readnone
+declare i32 @llvm.x86.avx.vtestc.pd.256(<4 x double>, <4 x double>) nounwind readnone
+declare i32 @llvm.x86.avx.vtestnzc.pd.256(<4 x double>, <4 x double>) nounwind readnone
diff --git a/llvm/test/CodeGen/X86/combine-testps.ll b/llvm/test/CodeGen/X86/combine-testps.ll
new file mode 100644
index 0000000000000..f3605441348df
--- /dev/null
+++ b/llvm/test/CodeGen/X86/combine-testps.ll
@@ -0,0 +1,158 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s
+
+;
+; testz(~X,Y) -> testc(X,Y)
+;
+
+define i32 @testpsz_128_invert0(<4 x float> %c, <4 x float> %d, i32 %a, i32 %b) {
+; CHECK-LABEL: testpsz_128_invert0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    vtestps %xmm1, %xmm0
+; CHECK-NEXT:    cmovael %esi, %eax
+; CHECK-NEXT:    retq
+  %t0 = bitcast <4 x float> %c to <2 x i64>
+  %t1 = xor <2 x i64> %t0, <i64 -1, i64 -1>
+  %t2 = bitcast <2 x i64> %t1 to <4 x float>
+  %t3 = call i32 @llvm.x86.avx.vtestz.ps(<4 x float> %t2, <4 x float> %d)
+  %t4 = icmp ne i32 %t3, 0
+  %t5 = select i1 %t4, i32 %a, i32 %b
+  ret i32 %t5
+}
+
+define i32 @testpsz_256_invert0(<8 x float> %c, <8 x float> %d, i32 %a, i32 %b) {
+; CHECK-LABEL: testpsz_256_invert0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    vtestps %ymm1, %ymm0
+; CHECK-NEXT:    cmovael %esi, %eax
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %t0 = bitcast <8 x float> %c to <4 x i64>
+  %t1 = xor <4 x i64> %t0, <i64 -1, i64 -1, i64 -1, i64 -1>
+  %t2 = bitcast <4 x i64> %t1 to <8 x float>
+  %t3 = call i32 @llvm.x86.avx.vtestz.ps.256(<8 x float> %t2, <8 x float> %d)
+  %t4 = icmp ne i32 %t3, 0
+  %t5 = select i1 %t4, i32 %a, i32 %b
+  ret i32 %t5
+}
+
+;
+; testz(X,~Y) -> testc(Y,X)
+;
+
+define i32 @testpsz_128_invert1(<4 x float> %c, <4 x float> %d, i32 %a, i32 %b) {
+; CHECK-LABEL: testpsz_128_invert1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    vtestps %xmm0, %xmm1
+; CHECK-NEXT:    cmovael %esi, %eax
+; CHECK-NEXT:    retq
+  %t0 = bitcast <4 x float> %d to <2 x i64>
+  %t1 = xor <2 x i64> %t0, <i64 -1, i64 -1>
+  %t2 = bitcast <2 x i64> %t1 to <4 x float>
+  %t3 = call i32 @llvm.x86.avx.vtestz.ps(<4 x float> %c, <4 x float> %t2)
+  %t4 = icmp ne i32 %t3, 0
+  %t5 = select i1 %t4, i32 %a, i32 %b
+  ret i32 %t5
+}
+
+define i32 @testpsz_256_invert1(<8 x float> %c, <8 x float> %d, i32 %a, i32 %b) {
+; CHECK-LABEL: testpsz_256_invert1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    vtestps %ymm0, %ymm1
+; CHECK-NEXT:    cmovael %esi, %eax
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %t0 = bitcast <8 x float> %d to <4 x i64>
+  %t1 = xor <4 x i64> %t0, <i64 -1, i64 -1, i64 -1, i64 -1>
+  %t2 = bitcast <4 x i64> %t1 to <8 x float>
+  %t3 = call i32 @llvm.x86.avx.vtestz.ps.256(<8 x float> %c, <8 x float> %t2)
+  %t4 = icmp ne i32 %t3, 0
+  %t5 = select i1 %t4, i32 %a, i32 %b
+  ret i32 %t5
+}
+
+;
+; testc(~X,Y) -> testz(X,Y)
+;
+
+define i32 @testpsc_128_invert0(<4 x float> %c, <4 x float> %d, i32 %a, i32 %b) {
+; CHECK-LABEL: testpsc_128_invert0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    vtestps %xmm1, %xmm0
+; CHECK-NEXT:    cmovnel %esi, %eax
+; CHECK-NEXT:    retq
+  %t0 = bitcast <4 x float> %c to <2 x i64>
+  %t1 = xor <2 x i64> %t0, <i64 -1, i64 -1>
+  %t2 = bitcast <2 x i64> %t1 to <4 x float>
+  %t3 = call i32 @llvm.x86.avx.vtestc.ps(<4 x float> %t2, <4 x float> %d)
+  %t4 = icmp ne i32 %t3, 0
+  %t5 = select i1 %t4, i32 %a, i32 %b
+  ret i32 %t5
+}
+
+define i32 @testpsc_256_invert0(<8 x float> %c, <8 x float> %d, i32 %a, i32 %b) {
+; CHECK-LABEL: testpsc_256_invert0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    vtestps %ymm1, %ymm0
+; CHECK-NEXT:    cmovnel %esi, %eax
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %t0 = bitcast <8 x float> %c to <4 x i64>
+  %t1 = xor <4 x i64> %t0, <i64 -1, i64 -1, i64 -1, i64 -1>
+  %t2 = bitcast <4 x i64> %t1 to <8 x float>
+  %t3 = call i32 @llvm.x86.avx.vtestc.ps.256(<8 x float> %t2, <8 x float> %d)
+  %t4 = icmp ne i32 %t3, 0
+  %t5 = select i1 %t4, i32 %a, i32 %b
+  ret i32 %t5
+}
+
+;
+; testnzc(~X,Y) -> testnzc(X,Y)
+;
+
+define i32 @testpsnzc_128_invert0(<4 x float> %c, <4 x float> %d, i32 %a, i32 %b) {
+; CHECK-LABEL: testpsnzc_128_invert0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    vtestps %xmm1, %xmm0
+; CHECK-NEXT:    cmovbel %esi, %eax
+; CHECK-NEXT:    retq
+  %t0 = bitcast <4 x float> %c to <2 x i64>
+  %t1 = xor <2 x i64> %t0, <i64 -1, i64 -1>
+  %t2 = bitcast <2 x i64> %t1 to <4 x float>
+  %t3 = call i32 @llvm.x86.avx.vtestnzc.ps(<4 x float> %t2, <4 x float> %d)
+  %t4 = icmp ne i32 %t3, 0
+  %t5 = select i1 %t4, i32 %a, i32 %b
+  ret i32 %t5
+}
+
+define i32 @testpsnzc_256_invert0(<8 x float> %c, <8 x float> %d, i32 %a, i32 %b) {
+; CHECK-LABEL: testpsnzc_256_invert0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    vtestps %ymm1, %ymm0
+; CHECK-NEXT:    cmovbel %esi, %eax
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %t0 = bitcast <8 x float> %c to <4 x i64>
+  %t1 = xor <4 x i64> %t0, <i64 -1, i64 -1, i64 -1, i64 -1>
+  %t2 = bitcast <4 x i64> %t1 to <8 x float>
+  %t3 = call i32 @llvm.x86.avx.vtestnzc.ps.256(<8 x float> %t2, <8 x float> %d)
+  %t4 = icmp ne i32 %t3, 0
+  %t5 = select i1 %t4, i32 %a, i32 %b
+  ret i32 %t5
+}
+
+declare i32 @llvm.x86.avx.vtestz.ps(<4 x float>, <4 x float>) nounwind readnone
+declare i32 @llvm.x86.avx.vtestc.ps(<4 x float>, <4 x float>) nounwind readnone
+declare i32 @llvm.x86.avx.vtestnzc.ps(<4 x float>, <4 x float>) nounwind readnone
+
+declare i32 @llvm.x86.avx.vtestz.ps.256(<8 x float>, <8 x float>) nounwind readnone
+declare i32 @llvm.x86.avx.vtestc.ps.256(<8 x float>, <8 x float>) nounwind readnone
+declare i32 @llvm.x86.avx.vtestnzc.ps.256(<8 x float>, <8 x float>) nounwind readnone
diff --git a/llvm/test/CodeGen/X86/dllexport-x86_64.ll b/llvm/test/CodeGen/X86/dllexport-x86_64.ll
index a2825bc0458a1..9f427f60e6230 100644
--- a/llvm/test/CodeGen/X86/dllexport-x86_64.ll
+++ b/llvm/test/CodeGen/X86/dllexport-x86_64.ll
@@ -20,16 +20,20 @@ define dllexport void @f2() unnamed_addr {
 }
 
 ; CHECK: .globl lnk1
-define linkonce_odr dllexport void @lnk1() {
+$lnk1 = comdat any
+
+define linkonce_odr dllexport void @lnk1() comdat {
 	ret void
 }
 
 ; CHECK: .globl lnk2
-define linkonce_odr dllexport void @lnk2() alwaysinline {
+$lnk2 = comdat any
+
+define linkonce_odr dllexport void @lnk2() alwaysinline comdat {
 	ret void
 }
 
-; CHECK: .globl weak1
+; CHECK: .weak weak1
 define weak_odr dllexport void @weak1() {
 	ret void
 }
@@ -46,14 +50,14 @@ define weak_odr dllexport void @weak1() {
 ; CHECK: .comm Var3
 @Var3 = common dllexport global i32 0, align 4
 
-; CHECK: .globl WeakVar1
+; CHECK: .weak WeakVar1
 @WeakVar1 = weak_odr dllexport global i32 1, align 4
 
-; CHECK: .globl WeakVar2
+; CHECK: .weak WeakVar2
 @WeakVar2 = weak_odr dllexport unnamed_addr constant i32 1
 
 ; CHECK: .bss
-; CHECK: .globl WeakVar3
+; CHECK: .weak WeakVar3
 @WeakVar3 = weak_odr dllexport global i32 0, align 4
 
 
diff --git a/llvm/test/CodeGen/X86/dllexport.ll b/llvm/test/CodeGen/X86/dllexport.ll
index 999733949ab2d..f08395a8d3be7 100644
--- a/llvm/test/CodeGen/X86/dllexport.ll
+++ b/llvm/test/CodeGen/X86/dllexport.ll
@@ -45,16 +45,20 @@ define dllexport x86_thiscallcc void @thisfun() nounwind {
 }
 
 ; CHECK: .globl _lnk1
-define linkonce_odr dllexport void @lnk1() {
+$lnk1 = comdat any
+
+define linkonce_odr dllexport void @lnk1() comdat {
 	ret void
 }
 
 ; CHECK: .globl _lnk2
-define linkonce_odr dllexport void @lnk2() alwaysinline {
+$lnk2 = comdat any
+
+define linkonce_odr dllexport void @lnk2() alwaysinline comdat {
 	ret void
 }
 
-; CHECK: .globl _weak1
+; CHECK: .weak _weak1
 define weak_odr dllexport void @weak1() {
 	ret void
 }
@@ -71,10 +75,10 @@ define weak_odr dllexport void @weak1() {
 ; CHECK: .comm _Var3
 @Var3 = common dllexport global i32 0, align 4
 
-; CHECK: .globl _WeakVar1
+; CHECK: .weak _WeakVar1
 @WeakVar1 = weak_odr dllexport global i32 1, align 4
 
-; CHECK: .globl _WeakVar2
+; CHECK: .weak _WeakVar2
 @WeakVar2 = weak_odr dllexport unnamed_addr constant i32 1
 
 
diff --git a/llvm/test/CodeGen/X86/emutls.ll b/llvm/test/CodeGen/X86/emutls.ll
index 67e80a360c053..8d836ef733b5e 100644
--- a/llvm/test/CodeGen/X86/emutls.ll
+++ b/llvm/test/CodeGen/X86/emutls.ll
@@ -25,6 +25,7 @@ define i32 @my_get_xyz() {
 ; X32-NEXT:    addl $12, %esp
 ; X32-NEXT:    .cfi_def_cfa_offset 4
 ; X32-NEXT:    retl
+;
 ; X64-LABEL: my_get_xyz:
 ; X64:         movl $my_emutls_v_xyz, %edi
 ; X64-NEXT:    callq my_emutls_get_address
@@ -32,7 +33,6 @@ define i32 @my_get_xyz() {
 ; X64-NEXT:    popq %rcx
 ; X64-NEXT:    .cfi_def_cfa_offset 8
 ; X64-NEXT:    retq
-
 entry:
   %call = call i8* @my_emutls_get_address(i8* bitcast (i8** @my_emutls_v_xyz to i8*))
   %0 = bitcast i8* %call to i32*
@@ -56,6 +56,7 @@ define i32 @f1() {
 ; X32-NEXT:    addl $12, %esp
 ; X32-NEXT:    .cfi_def_cfa_offset 4
 ; X32-NEXT:    retl
+;
 ; X64-LABEL: f1:
 ; X64:         movl $__emutls_v.i1, %edi
 ; X64-NEXT:    callq __emutls_get_address
@@ -63,7 +64,6 @@ define i32 @f1() {
 ; X64-NEXT:    popq %rcx
 ; X64-NEXT:    .cfi_def_cfa_offset 8
 ; X64-NEXT:    retq
-
 entry:
   %tmp1 = load i32, i32* @i1
   ret i32 %tmp1
@@ -76,13 +76,13 @@ define i32* @f2() {
 ; X32-NEXT:    addl $12, %esp
 ; X32-NEXT:    .cfi_def_cfa_offset 4
 ; X32-NEXT:    retl
+;
 ; X64-LABEL: f2:
 ; X64:         movl $__emutls_v.i1, %edi
 ; X64-NEXT:    callq __emutls_get_address
 ; X64-NEXT:    popq %rcx
 ; X64-NEXT:    .cfi_def_cfa_offset 8
 ; X64-NEXT:    retq
-
 entry:
   ret i32* @i1
 }
@@ -94,7 +94,6 @@ define i32 @f3() nounwind {
 ; X32-NEXT:    movl (%eax), %eax
 ; X32-NEXT:    addl $12, %esp
 ; X32-NEXT:    retl
-
 entry:
   %tmp1 = load i32, i32* @i2
   ret i32 %tmp1
@@ -107,7 +106,6 @@ define i32* @f4() {
 ; X32-NEXT:    addl $12, %esp
 ; X32-NEXT:    .cfi_def_cfa_offset 4
 ; X32-NEXT:    retl
-
 entry:
   ret i32* @i2
 }
@@ -119,7 +117,6 @@ define i32 @f5() nounwind {
 ; X32-NEXT:    movl (%eax), %eax
 ; X32-NEXT:    addl $12, %esp
 ; X32-NEXT:    retl
-
 entry:
   %tmp1 = load i32, i32* @i3
   ret i32 %tmp1
@@ -132,7 +129,6 @@ define i32* @f6() {
 ; X32-NEXT:    addl $12, %esp
 ; X32-NEXT:    .cfi_def_cfa_offset 4
 ; X32-NEXT:    retl
-
 entry:
   ret i32* @i3
 }
@@ -145,7 +141,6 @@ define i32 @f7() {
 ; X32-NEXT:    addl $12, %esp
 ; X32-NEXT:    .cfi_def_cfa_offset 4
 ; X32-NEXT:    retl
-
 entry:
   %tmp1 = load i32, i32* @i4
   ret i32 %tmp1
@@ -158,7 +153,6 @@ define i32* @f8() {
 ; X32-NEXT:    addl $12, %esp
 ; X32-NEXT:    .cfi_def_cfa_offset 4
 ; X32-NEXT:    retl
-
 entry:
   ret i32* @i4
 }
@@ -171,7 +165,6 @@ define i32 @f9() {
 ; X32-NEXT:    addl $12, %esp
 ; X32-NEXT:    .cfi_def_cfa_offset 4
 ; X32-NEXT:    retl
-
 entry:
   %tmp1 = load i32, i32* @i5
   ret i32 %tmp1
@@ -184,7 +177,6 @@ define i32* @f10() {
 ; X32-NEXT:    addl $12, %esp
 ; X32-NEXT:    .cfi_def_cfa_offset 4
 ; X32-NEXT:    retl
-
 entry:
   ret i32* @i5
 }
@@ -197,7 +189,6 @@ define i16 @f11() {
 ; X32-NEXT:    addl $12, %esp
 ; X32-NEXT:    .cfi_def_cfa_offset 4
 ; X32-NEXT:    retl
-
 entry:
   %tmp1 = load i16, i16* @s1
   ret i16 %tmp1
@@ -211,7 +202,6 @@ define i32 @f12() {
 ; X32-NEXT:    addl $12, %esp
 ; X32-NEXT:    .cfi_def_cfa_offset 4
 ; X32-NEXT:    retl
-
 entry:
   %tmp1 = load i16, i16* @s1
   %tmp2 = sext i16 %tmp1 to i32
@@ -226,7 +216,6 @@ define i8 @f13() {
 ; X32-NEXT:    addl $12, %esp
 ; X32-NEXT:    .cfi_def_cfa_offset 4
 ; X32-NEXT:    retl
-
 entry:
   %tmp1 = load i8, i8* @b1
   ret i8 %tmp1
@@ -240,7 +229,6 @@ define i32 @f14() {
 ; X32-NEXT:    addl $12, %esp
 ; X32-NEXT:    .cfi_def_cfa_offset 4
 ; X32-NEXT:    retl
-
 entry:
   %tmp1 = load i8, i8* @b1
   %tmp2 = sext i8 %tmp1 to i32
@@ -249,31 +237,26 @@ entry:
 
 ;;;;;;;;;;;;;; 32-bit __emutls_v. and __emutls_t.
 
-; X32       .section .data.rel.local,
 ; X32-LABEL: __emutls_v.i1:
 ; X32-NEXT: .long 4
 ; X32-NEXT: .long 4
 ; X32-NEXT: .long 0
 ; X32-NEXT: .long __emutls_t.i1
 
-; X32       .section .rodata,
 ; X32-LABEL: __emutls_t.i1:
 ; X32-NEXT: .long 15
 
 ; X32-NOT:   __emutls_v.i2
 
-; X32       .section .data.rel.local,
 ; X32-LABEL: __emutls_v.i3:
 ; X32-NEXT: .long 4
 ; X32-NEXT: .long 4
 ; X32-NEXT: .long 0
 ; X32-NEXT: .long __emutls_t.i3
 
-; X32       .section .rodata,
 ; X32-LABEL: __emutls_t.i3:
 ; X32-NEXT: .long 15
 
-; X32       .section .data.rel.local,
 ; X32-LABEL: __emutls_v.i4:
 ; X32-NEXT: .L__emutls_v.i4$local:
 ; X32-NEXT: .long 4
@@ -281,27 +264,23 @@ entry:
 ; X32-NEXT: .long 0
 ; X32-NEXT: .long __emutls_t.i4
 
-; X32       .section .rodata,
 ; X32-LABEL: __emutls_t.i4:
 ; X32-NEXT: .L__emutls_t.i4$local:
 ; X32-NEXT: .long 15
 
 ; X32-NOT:   __emutls_v.i5:
-; X32       .hidden __emutls_v.i5
+; X32:      .hidden __emutls_v.i5
 ; X32-NOT:   __emutls_v.i5:
 
-; X32 .section .data.rel.local,
 ; X32-LABEL: __emutls_v.s1:
 ; X32-NEXT: .long 2
 ; X32-NEXT: .long 2
 ; X32-NEXT: .long 0
 ; X32-NEXT: .long __emutls_t.s1
 
-; X32 .section .rodata,
 ; X32-LABEL: __emutls_t.s1:
 ; X32-NEXT: .short 15
 
-; X32 .section .data.rel.local,
 ; X32-LABEL: __emutls_v.b1:
 ; X32-NEXT: .long 1
 ; X32-NEXT: .long 1
@@ -312,31 +291,26 @@ entry:
 
 ;;;;;;;;;;;;;; 64-bit __emutls_v. and __emutls_t.
 
-; X64       .section .data.rel.local,
 ; X64-LABEL: __emutls_v.i1:
 ; X64-NEXT: .quad 4
 ; X64-NEXT: .quad 4
 ; X64-NEXT: .quad 0
 ; X64-NEXT: .quad __emutls_t.i1
 
-; X64       .section .rodata,
 ; X64-LABEL: __emutls_t.i1:
 ; X64-NEXT: .long 15
 
 ; X64-NOT:   __emutls_v.i2
 
-; X64       .section .data.rel.local,
 ; X64-LABEL: __emutls_v.i3:
 ; X64-NEXT: .quad 4
 ; X64-NEXT: .quad 4
 ; X64-NEXT: .quad 0
 ; X64-NEXT: .quad __emutls_t.i3
 
-; X64       .section .rodata,
 ; X64-LABEL: __emutls_t.i3:
 ; X64-NEXT: .long 15
 
-; X64       .section .data.rel.local,
 ; X64-LABEL: __emutls_v.i4:
 ; X64-NEXT: .L__emutls_v.i4$local:
 ; X64-NEXT: .quad 4
@@ -344,27 +318,23 @@ entry:
 ; X64-NEXT: .quad 0
 ; X64-NEXT: .quad __emutls_t.i4
 
-; X64       .section .rodata,
 ; X64-LABEL: __emutls_t.i4:
 ; X64-NEXT: .L__emutls_t.i4$local:
 ; X64-NEXT: .long 15
 
 ; X64-NOT:   __emutls_v.i5:
-; X64       .hidden __emutls_v.i5
+; X64:      .hidden __emutls_v.i5
 ; X64-NOT:   __emutls_v.i5:
 
-; X64       .section .data.rel.local,
 ; X64-LABEL: __emutls_v.s1:
 ; X64-NEXT: .quad 2
 ; X64-NEXT: .quad 2
 ; X64-NEXT: .quad 0
 ; X64-NEXT: .quad __emutls_t.s1
 
-; X64       .section .rodata,
 ; X64-LABEL: __emutls_t.s1:
 ; X64-NEXT: .short 15
 
-; X64       .section .data.rel.local,
 ; X64-LABEL: __emutls_v.b1:
 ; X64-NEXT: .quad 1
 ; X64-NEXT: .quad 1
diff --git a/llvm/test/CodeGen/X86/extract-concat.ll b/llvm/test/CodeGen/X86/extract-concat.ll
index b860b7281ee61..39362e375cdcc 100644
--- a/llvm/test/CodeGen/X86/extract-concat.ll
+++ b/llvm/test/CodeGen/X86/extract-concat.ll
@@ -8,10 +8,7 @@ define void @foo(<4 x float> %in, <4 x i8>* %out) {
 ; SSE42-LABEL: foo:
 ; SSE42:       # %bb.0:
 ; SSE42-NEXT:    cvttps2dq %xmm0, %xmm0
-; SSE42-NEXT:    pextrb $8, %xmm0, %eax
-; SSE42-NEXT:    pextrb $4, %xmm0, %ecx
-; SSE42-NEXT:    pinsrb $1, %ecx, %xmm0
-; SSE42-NEXT:    pinsrb $2, %eax, %xmm0
+; SSE42-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,3,u,u,u,u,u,u,u,u,u,u,u,u]
 ; SSE42-NEXT:    movl $255, %eax
 ; SSE42-NEXT:    pinsrb $3, %eax, %xmm0
 ; SSE42-NEXT:    movd %xmm0, (%rdi)
@@ -20,10 +17,7 @@ define void @foo(<4 x float> %in, <4 x i8>* %out) {
 ; AVX-LABEL: foo:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vcvttps2dq %xmm0, %xmm0
-; AVX-NEXT:    vpextrb $8, %xmm0, %eax
-; AVX-NEXT:    vpextrb $4, %xmm0, %ecx
-; AVX-NEXT:    vpinsrb $1, %ecx, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $2, %eax, %xmm0, %xmm0
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,3,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX-NEXT:    movl $255, %eax
 ; AVX-NEXT:    vpinsrb $3, %eax, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, (%rdi)
diff --git a/llvm/test/CodeGen/X86/fast-isel-freeze.ll b/llvm/test/CodeGen/X86/fast-isel-freeze.ll
index fee53ca93f542..8308a28e00a1d 100644
--- a/llvm/test/CodeGen/X86/fast-isel-freeze.ll
+++ b/llvm/test/CodeGen/X86/fast-isel-freeze.ll
@@ -5,8 +5,8 @@
 define i32 @freeze(i32 %t) {
 ; SDAG-LABEL: freeze:
 ; SDAG:       # %bb.0:
-; SDAG-NEXT:    movl $10, %eax
-; SDAG-NEXT:    xorl %edi, %eax
+; SDAG-NEXT:    movl %edi, %eax
+; SDAG-NEXT:    xorl $10, %eax
 ; SDAG-NEXT:    retq
 ;
 ; FAST-LABEL: freeze:
diff --git a/llvm/test/CodeGen/X86/freeze-combine.ll b/llvm/test/CodeGen/X86/freeze-combine.ll
new file mode 100644
index 0000000000000..b037a6d9a1b93
--- /dev/null
+++ b/llvm/test/CodeGen/X86/freeze-combine.ll
@@ -0,0 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -stop-after=finalize-isel -mtriple=x86_64-unknown < %s 2>&1 | FileCheck %s
+define i32 @const() {
+  ; CHECK-LABEL: name: const
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK:   [[MOV32ri:%[0-9]+]]:gr32 = MOV32ri 1
+  ; CHECK:   $eax = COPY [[MOV32ri]]
+  ; CHECK:   RET 0, $eax
+  %y = freeze i32 1
+  ret i32 %y
+}
+
+define i32 @fold(i32 %x) {
+  ; CHECK-LABEL: name: fold
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK:   liveins: $edi
+  ; CHECK:   [[COPY:%[0-9]+]]:gr32 = COPY $edi
+  ; CHECK:   [[COPY1:%[0-9]+]]:gr32 = COPY [[COPY]]
+  ; CHECK:   $eax = COPY [[COPY1]]
+  ; CHECK:   RET 0, $eax
+  %y = freeze i32 %x
+  %z = freeze i32 %y
+  ret i32 %z
+}
diff --git a/llvm/test/CodeGen/X86/freeze-legalize.ll b/llvm/test/CodeGen/X86/freeze-legalize.ll
index 6bbd0b8e59493..4fe60bc4cce6c 100644
--- a/llvm/test/CodeGen/X86/freeze-legalize.ll
+++ b/llvm/test/CodeGen/X86/freeze-legalize.ll
@@ -5,12 +5,8 @@
 define i64 @expand(i32 %x) {
 ; CHECK-LABEL: expand:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    movl $303174162, %eax ## imm = 0x12121212
-; CHECK-NEXT:    movl $875836468, %ecx ## imm = 0x34343434
-; CHECK-NEXT:    movl $1448498774, %edx ## imm = 0x56565656
-; CHECK-NEXT:    xorl %eax, %edx
-; CHECK-NEXT:    movl $2021161080, %eax ## imm = 0x78787878
-; CHECK-NEXT:    xorl %ecx, %eax
+; CHECK-NEXT:    movl $1280068684, %eax ## imm = 0x4C4C4C4C
+; CHECK-NEXT:    movl $1145324612, %edx ## imm = 0x44444444
 ; CHECK-NEXT:    retl
   %y1 = freeze i64 1302123111658042420 ; 0x1212121234343434
   %y2 = freeze i64 6221254864647256184 ; 0x5656565678787878
@@ -22,29 +18,11 @@ define i64 @expand(i32 %x) {
 define <2 x i64> @expand_vec(i32 %x) nounwind {
 ; CHECK-LABEL: expand_vec:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    pushl %ebx
-; CHECK-NEXT:    pushl %edi
-; CHECK-NEXT:    pushl %esi
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; CHECK-NEXT:    movl $16843009, %ecx ## imm = 0x1010101
-; CHECK-NEXT:    movl $589505315, %edx ## imm = 0x23232323
-; CHECK-NEXT:    movl $303174162, %esi ## imm = 0x12121212
-; CHECK-NEXT:    movl $875836468, %edi ## imm = 0x34343434
-; CHECK-NEXT:    movl $1162167621, %ebx ## imm = 0x45454545
-; CHECK-NEXT:    xorl %ecx, %ebx
-; CHECK-NEXT:    movl $1734829927, %ecx ## imm = 0x67676767
-; CHECK-NEXT:    xorl %edx, %ecx
-; CHECK-NEXT:    movl $1448498774, %edx ## imm = 0x56565656
-; CHECK-NEXT:    xorl %esi, %edx
-; CHECK-NEXT:    movl $2021161080, %esi ## imm = 0x78787878
-; CHECK-NEXT:    xorl %edi, %esi
-; CHECK-NEXT:    movl %ebx, 12(%eax)
-; CHECK-NEXT:    movl %ecx, 8(%eax)
-; CHECK-NEXT:    movl %edx, 4(%eax)
-; CHECK-NEXT:    movl %esi, (%eax)
-; CHECK-NEXT:    popl %esi
-; CHECK-NEXT:    popl %edi
-; CHECK-NEXT:    popl %ebx
+; CHECK-NEXT:    movl $1145324612, 12(%eax) ## imm = 0x44444444
+; CHECK-NEXT:    movl $1145324612, 8(%eax) ## imm = 0x44444444
+; CHECK-NEXT:    movl $1145324612, 4(%eax) ## imm = 0x44444444
+; CHECK-NEXT:    movl $1280068684, (%eax) ## imm = 0x4C4C4C4C
 ; CHECK-NEXT:    retl $4
   ; <0x1212121234343434, 0x101010123232323>
   %y1 = freeze <2 x i64> <i64 1302123111658042420, i64 72340173410738979>
@@ -57,10 +35,7 @@ define <2 x i64> @expand_vec(i32 %x) nounwind {
 define i10 @promote() {
 ; CHECK-LABEL: promote:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    movw $682, %cx ## imm = 0x2AA
-; CHECK-NEXT:    movw $992, %ax ## imm = 0x3E0
-; CHECK-NEXT:    addl %ecx, %eax
-; CHECK-NEXT:    ## kill: def $ax killed $ax killed $eax
+; CHECK-NEXT:    movw $650, %ax ## imm = 0x28A
 ; CHECK-NEXT:    retl
   %a = freeze i10 682
   %b = freeze i10 992
@@ -71,14 +46,8 @@ define i10 @promote() {
 define <2 x i10> @promote_vec() {
 ; CHECK-LABEL: promote_vec:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    movw $125, %ax
-; CHECK-NEXT:    movw $682, %cx ## imm = 0x2AA
-; CHECK-NEXT:    movw $393, %dx ## imm = 0x189
-; CHECK-NEXT:    addl %eax, %edx
-; CHECK-NEXT:    movw $992, %ax ## imm = 0x3E0
-; CHECK-NEXT:    addl %ecx, %eax
-; CHECK-NEXT:    ## kill: def $ax killed $ax killed $eax
-; CHECK-NEXT:    ## kill: def $dx killed $dx killed $edx
+; CHECK-NEXT:    movw $1674, %ax ## imm = 0x68A
+; CHECK-NEXT:    movw $518, %dx ## imm = 0x206
 ; CHECK-NEXT:    retl
   %a = freeze <2 x i10> <i10 682, i10 125>
   %b = freeze <2 x i10> <i10 992, i10 393>
diff --git a/llvm/test/CodeGen/X86/indirect-branch-tracking-cm-lager.ll b/llvm/test/CodeGen/X86/indirect-branch-tracking-cm-lager.ll
new file mode 100644
index 0000000000000..f515c1a34c733
--- /dev/null
+++ b/llvm/test/CodeGen/X86/indirect-branch-tracking-cm-lager.ll
@@ -0,0 +1,36 @@
+; RUN: llc -mtriple=x86_64-unknown-unknown -code-model=large < %s | FileCheck %s
+
+; In large code model indirect branches are needed when branching to addresses
+; whose offset from the current instruction pointer is unknown.
+;CHECK-COUNT-3: endbr
+
+@a = dso_local local_unnamed_addr global i32 1, align 4
+
+; Function Attrs: nofree noinline norecurse nounwind uwtable writeonly
+define dso_local void @ext() local_unnamed_addr #0 {
+entry:
+  store i32 0, i32* @a, align 4
+  ret void
+}
+
+; Function Attrs: nofree norecurse nounwind uwtable
+define dso_local i32 @main() local_unnamed_addr #1 {
+entry:
+  tail call fastcc void @foo()
+  %0 = load i32, i32* @a, align 4
+  ret i32 %0
+}
+
+; Function Attrs: nofree noinline norecurse nounwind uwtable writeonly
+define internal fastcc void @foo() unnamed_addr #0 {
+entry:
+  tail call void @ext()
+  ret void
+}
+
+!llvm.module.flags = !{!0, !1, !2, !3}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"cf-protection-return", i32 1}
+!2 = !{i32 4, !"cf-protection-branch", i32 1}
+!3 = !{i32 1, !"Code Model", i32 4}
diff --git a/llvm/test/CodeGen/X86/linux-preemption.ll b/llvm/test/CodeGen/X86/linux-preemption.ll
index e876469f8fa6b..7d22b75132186 100644
--- a/llvm/test/CodeGen/X86/linux-preemption.ll
+++ b/llvm/test/CodeGen/X86/linux-preemption.ll
@@ -73,9 +73,9 @@ define i32* @get_strong_preemptable_global() {
 define i32* @get_weak_preemptable_global() {
   ret i32* @weak_preemptable_global
 }
-; CHECK ;ADD_LABEL_BACK;  movq weak_preemptable_global@GOTPCREL(%rip), %rax
-; STATIC ;ADD_LABEL_BACK; movq weak_preemptable_global@GOTPCREL, %rax
-; CHECK32 ;ADD_LABEL_BACK; movl weak_preemptable_global@GOT(%eax), %eax
+; CHECK: movq weak_preemptable_global@GOTPCREL(%rip), %rax
+; STATIC: movl $weak_preemptable_global, %eax
+; CHECK32: movl weak_preemptable_global@GOT(%eax), %eax
 
 @external_preemptable_global = external dso_preemptable global i32
 define i32* @get_external_preemptable_global() {
@@ -174,7 +174,7 @@ define void()* @get_strong_local_function() {
   ret void()* @strong_local_function
 }
 ; COMMON:     {{^}}strong_local_function:
-; COMMON-NEXT .Lstrong_local_function:
+; COMMON-NEXT: .Lstrong_local_function$local:
 ; CHECK: leaq .Lstrong_local_function$local(%rip), %rax
 ; STATIC: movl $.Lstrong_local_function$local, %eax
 ; CHECK32: leal .Lstrong_local_function$local@GOTOFF(%eax), %eax
@@ -227,7 +227,7 @@ define void()* @get_external_preemptable_function() {
 ; CHECK32: movl external_preemptable_function@GOT(%eax), %eax
 
 ; COMMON:     {{^}}strong_local_global:
-; COMMON-NEXT .Lstrong_local_global:
+; COMMON-NEXT: .Lstrong_local_global$local:
 
 ; COMMON:      .globl strong_default_alias
 ; COMMON-NEXT: .set strong_default_alias, aliasee
diff --git a/llvm/test/CodeGen/X86/lvi-hardening-gadget-graph.ll b/llvm/test/CodeGen/X86/lvi-hardening-gadget-graph.ll
new file mode 100644
index 0000000000000..ba2ce26142b5a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/lvi-hardening-gadget-graph.ll
@@ -0,0 +1,129 @@
+; RUN: llc -verify-machineinstrs -mtriple=x86_64-unknown -x86-lvi-load-dot-verify -o %t < %s | FileCheck %s
+
+; Function Attrs: noinline nounwind optnone uwtable
+define dso_local i32 @test(i32* %untrusted_user_ptr, i32* %secret, i32 %secret_size) #0 {
+entry:
+  %untrusted_user_ptr.addr = alloca i32*, align 8
+  %secret.addr = alloca i32*, align 8
+  %secret_size.addr = alloca i32, align 4
+  %ret_val = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32* %untrusted_user_ptr, i32** %untrusted_user_ptr.addr, align 8
+  store i32* %secret, i32** %secret.addr, align 8
+  store i32 %secret_size, i32* %secret_size.addr, align 4
+  store i32 0, i32* %ret_val, align 4
+  call void @llvm.x86.sse2.lfence()
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %1 = load i32, i32* %secret_size.addr, align 4
+  %cmp = icmp slt i32 %0, %1
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %2 = load i32, i32* %i, align 4
+  %rem = srem i32 %2, 2
+  %cmp1 = icmp eq i32 %rem, 0
+  br i1 %cmp1, label %if.then, label %if.else
+
+if.then:                                          ; preds = %for.body
+  %3 = load i32*, i32** %secret.addr, align 8
+  %4 = load i32, i32* %ret_val, align 4
+  %idxprom = sext i32 %4 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %3, i64 %idxprom
+  %5 = load i32, i32* %arrayidx, align 4
+  %6 = load i32*, i32** %untrusted_user_ptr.addr, align 8
+  store i32 %5, i32* %6, align 4
+  br label %if.end
+
+if.else:                                          ; preds = %for.body
+  %7 = load i32*, i32** %secret.addr, align 8
+  %8 = load i32, i32* %ret_val, align 4
+  %idxprom2 = sext i32 %8 to i64
+  %arrayidx3 = getelementptr inbounds i32, i32* %7, i64 %idxprom2
+  store i32 42, i32* %arrayidx3, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %9 = load i32*, i32** %untrusted_user_ptr.addr, align 8
+  %10 = load i32, i32* %9, align 4
+  store i32 %10, i32* %ret_val, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.end
+  %11 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %11, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  %12 = load i32, i32* %ret_val, align 4
+  ret i32 %12
+}
+
+; CHECK:      digraph "Speculative gadgets for \"test\" function" {
+; CHECK-NEXT: label="Speculative gadgets for \"test\" function";
+; CHECK:      Node0x{{[0-9a-f]+}} [shape=record,color = green,label="{LFENCE\n}"];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 0];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{renamable $eax = MOV32rm %stack.4.i, 1, $noreg, 0, $noreg :: (dereferenceable load 4 from %ir.i)\n}"];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[color = red, style = "dashed"];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{JCC_1 %bb.6, 13, implicit killed $eflags\n}"];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{CMP32rm killed renamable $eax, %stack.2.secret_size.addr, 1, $noreg, 0, $noreg, implicit-def $eflags :: (dereferenceable load 4 from %ir.secret_size.addr)\n}"];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[color = red, style = "dashed"];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{renamable $eax = MOV32rm %stack.4.i, 1, $noreg, 0, $noreg :: (dereferenceable load 4 from %ir.i)\n}"];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[color = red, style = "dashed"];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{JCC_1 %bb.4, 5, implicit killed $eflags\n}"];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{renamable $rax = MOV64rm %stack.1.secret.addr, 1, $noreg, 0, $noreg :: (dereferenceable load 8 from %ir.secret.addr)\n}"];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[color = red, style = "dashed"];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{renamable $eax = MOV32rm killed renamable $rax, 4, killed renamable $rcx, 0, $noreg :: (load 4 from %ir.arrayidx)\n}"];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{renamable $rcx = MOVSX64rm32 %stack.3.ret_val, 1, $noreg, 0, $noreg :: (dereferenceable load 4 from %ir.ret_val)\n}"];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[color = red, style = "dashed"];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{renamable $rcx = MOV64rm %stack.0.untrusted_user_ptr.addr, 1, $noreg, 0, $noreg :: (dereferenceable load 8 from %ir.untrusted_user_ptr.addr)\n}"];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[color = red, style = "dashed"];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{MOV32mr killed renamable $rcx, 1, $noreg, 0, $noreg, killed renamable $eax :: (store 4 into %ir.6)\n}"];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{renamable $rax = MOV64rm %stack.1.secret.addr, 1, $noreg, 0, $noreg :: (dereferenceable load 8 from %ir.secret.addr)\n}"];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[color = red, style = "dashed"];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{MOV32mi killed renamable $rax, 4, killed renamable $rcx, 0, $noreg, 42 :: (store 4 into %ir.arrayidx3)\n}"];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{renamable $rcx = MOVSX64rm32 %stack.3.ret_val, 1, $noreg, 0, $noreg :: (dereferenceable load 4 from %ir.ret_val)\n}"];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[color = red, style = "dashed"];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{renamable $rax = MOV64rm %stack.0.untrusted_user_ptr.addr, 1, $noreg, 0, $noreg :: (dereferenceable load 8 from %ir.untrusted_user_ptr.addr)\n}"];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[color = red, style = "dashed"];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{renamable $eax = MOV32rm killed renamable $rax, 1, $noreg, 0, $noreg :: (load 4 from %ir.9)\n}"];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,color = blue,label="{ARGS}"];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 0];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{MOV64mr %stack.0.untrusted_user_ptr.addr, 1, $noreg, 0, $noreg, killed renamable $rdi :: (store 8 into %ir.untrusted_user_ptr.addr)\n}"];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 0];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{JMP_1 %bb.5\n}"];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{JMP_1 %bb.1\n}"];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 1];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{renamable $eax = MOV32rm %stack.3.ret_val, 1, $noreg, 0, $noreg :: (dereferenceable load 4 from %ir.ret_val)\n}"];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} -> Node0x{{[0-9a-f]+}}[label = 0];
+; CHECK-NEXT: Node0x{{[0-9a-f]+}} [shape=record,label="{RET 0, $eax\n}"];
+; CHECK-NEXT: }
+
+; Function Attrs: nounwind
+declare void @llvm.x86.sse2.lfence() #1
+
+attributes #0 = { "target-features"="+lvi-cfi"
+                  "target-features"="+lvi-load-hardening" }
+attributes #1 = { nounwind }
diff --git a/llvm/test/CodeGen/X86/lvi-hardening-indirectbr.ll b/llvm/test/CodeGen/X86/lvi-hardening-indirectbr.ll
new file mode 100644
index 0000000000000..d2caf6e1e9eb0
--- /dev/null
+++ b/llvm/test/CodeGen/X86/lvi-hardening-indirectbr.ll
@@ -0,0 +1,281 @@
+; RUN: llc -verify-machineinstrs -mtriple=x86_64-unknown -mattr=+lvi-cfi < %s | FileCheck %s --check-prefix=X64
+; RUN: llc -verify-machineinstrs -mtriple=x86_64-unknown -mattr=+lvi-cfi -O0 < %s | FileCheck %s --check-prefix=X64FAST
+;
+; Note that a lot of this code was lifted from retpoline.ll.
+
+declare void @bar(i32)
+
+; Test a simple indirect call and tail call.
+define void @icall_reg(void (i32)* %fp, i32 %x) {
+entry:
+  tail call void @bar(i32 %x)
+  tail call void %fp(i32 %x)
+  tail call void @bar(i32 %x)
+  tail call void %fp(i32 %x)
+  ret void
+}
+
+; X64-LABEL: icall_reg:
+; X64-DAG:   movq %rdi, %[[fp:[^ ]*]]
+; X64-DAG:   movl %esi, %[[x:[^ ]*]]
+; X64:       movl %esi, %edi
+; X64:       callq bar
+; X64-DAG:   movl %[[x]], %edi
+; X64-DAG:   movq %[[fp]], %r11
+; X64:       callq __llvm_lvi_thunk_r11
+; X64:       movl %[[x]], %edi
+; X64:       callq bar
+; X64-DAG:   movl %[[x]], %edi
+; X64-DAG:   movq %[[fp]], %r11
+; X64:       jmp __llvm_lvi_thunk_r11 # TAILCALL
+
+; X64FAST-LABEL: icall_reg:
+; X64FAST:       callq bar
+; X64FAST:       callq __llvm_lvi_thunk_r11
+; X64FAST:       callq bar
+; X64FAST:       jmp __llvm_lvi_thunk_r11 # TAILCALL
+
+
+@global_fp = external global void (i32)*
+
+; Test an indirect call through a global variable.
+define void @icall_global_fp(i32 %x, void (i32)** %fpp) #0 {
+  %fp1 = load void (i32)*, void (i32)** @global_fp
+  call void %fp1(i32 %x)
+  %fp2 = load void (i32)*, void (i32)** @global_fp
+  tail call void %fp2(i32 %x)
+  ret void
+}
+
+; X64-LABEL: icall_global_fp:
+; X64-DAG:   movl %edi, %[[x:[^ ]*]]
+; X64-DAG:   movq global_fp(%rip), %r11
+; X64:       callq __llvm_lvi_thunk_r11
+; X64-DAG:   movl %[[x]], %edi
+; X64-DAG:   movq global_fp(%rip), %r11
+; X64:       jmp __llvm_lvi_thunk_r11 # TAILCALL
+
+; X64FAST-LABEL: icall_global_fp:
+; X64FAST:       movq global_fp(%rip), %r11
+; X64FAST:       callq __llvm_lvi_thunk_r11
+; X64FAST:       movq global_fp(%rip), %r11
+; X64FAST:       jmp __llvm_lvi_thunk_r11 # TAILCALL
+
+
+%struct.Foo = type { void (%struct.Foo*)** }
+
+; Test an indirect call through a vtable.
+define void @vcall(%struct.Foo* %obj) #0 {
+  %vptr_field = getelementptr %struct.Foo, %struct.Foo* %obj, i32 0, i32 0
+  %vptr = load void (%struct.Foo*)**, void (%struct.Foo*)*** %vptr_field
+  %vslot = getelementptr void(%struct.Foo*)*, void(%struct.Foo*)** %vptr, i32 1
+  %fp = load void(%struct.Foo*)*, void(%struct.Foo*)** %vslot
+  tail call void %fp(%struct.Foo* %obj)
+  tail call void %fp(%struct.Foo* %obj)
+  ret void
+}
+
+; X64-LABEL: vcall:
+; X64:       movq %rdi, %[[obj:[^ ]*]]
+; X64:       movq (%rdi), %[[vptr:[^ ]*]]
+; X64:       movq 8(%[[vptr]]), %[[fp:[^ ]*]]
+; X64:       movq %[[fp]], %r11
+; X64:       callq __llvm_lvi_thunk_r11
+; X64-DAG:   movq %[[obj]], %rdi
+; X64-DAG:   movq %[[fp]], %r11
+; X64:       jmp __llvm_lvi_thunk_r11 # TAILCALL
+
+; X64FAST-LABEL: vcall:
+; X64FAST:       callq __llvm_lvi_thunk_r11
+; X64FAST:       jmp __llvm_lvi_thunk_r11 # TAILCALL
+
+
+declare void @direct_callee()
+
+define void @direct_tail() #0 {
+  tail call void @direct_callee()
+  ret void
+}
+
+; X64-LABEL: direct_tail:
+; X64:       jmp direct_callee # TAILCALL
+; X64FAST-LABEL: direct_tail:
+; X64FAST:   jmp direct_callee # TAILCALL
+
+
+declare void @nonlazybind_callee() #1
+
+define void @nonlazybind_caller() #0 {
+  call void @nonlazybind_callee()
+  tail call void @nonlazybind_callee()
+  ret void
+}
+
+; X64-LABEL: nonlazybind_caller:
+; X64:       movq nonlazybind_callee@GOTPCREL(%rip), %[[REG:.*]]
+; X64:       movq %[[REG]], %r11
+; X64:       callq __llvm_lvi_thunk_r11
+; X64:       movq %[[REG]], %r11
+; X64:       jmp __llvm_lvi_thunk_r11 # TAILCALL
+; X64FAST-LABEL: nonlazybind_caller:
+; X64FAST:   movq nonlazybind_callee@GOTPCREL(%rip), %r11
+; X64FAST:   callq __llvm_lvi_thunk_r11
+; X64FAST:   movq nonlazybind_callee@GOTPCREL(%rip), %r11
+; X64FAST:   jmp __llvm_lvi_thunk_r11 # TAILCALL
+
+
+; Check that a switch gets lowered using a jump table
+define void @switch_jumptable(i32* %ptr, i64* %sink) #0 {
+; X64-LABEL: switch_jumptable:
+; X64_NOT:      jmpq *
+entry:
+  br label %header
+
+header:
+  %i = load volatile i32, i32* %ptr
+  switch i32 %i, label %bb0 [
+    i32 1, label %bb1
+    i32 2, label %bb2
+    i32 3, label %bb3
+    i32 4, label %bb4
+    i32 5, label %bb5
+    i32 6, label %bb6
+    i32 7, label %bb7
+    i32 8, label %bb8
+    i32 9, label %bb9
+  ]
+
+bb0:
+  store volatile i64 0, i64* %sink
+  br label %header
+
+bb1:
+  store volatile i64 1, i64* %sink
+  br label %header
+
+bb2:
+  store volatile i64 2, i64* %sink
+  br label %header
+
+bb3:
+  store volatile i64 3, i64* %sink
+  br label %header
+
+bb4:
+  store volatile i64 4, i64* %sink
+  br label %header
+
+bb5:
+  store volatile i64 5, i64* %sink
+  br label %header
+
+bb6:
+  store volatile i64 6, i64* %sink
+  br label %header
+
+bb7:
+  store volatile i64 7, i64* %sink
+  br label %header
+
+bb8:
+  store volatile i64 8, i64* %sink
+  br label %header
+
+bb9:
+  store volatile i64 9, i64* %sink
+  br label %header
+}
+
+
+@indirectbr_rewrite.targets = constant [10 x i8*] [i8* blockaddress(@indirectbr_rewrite, %bb0),
+                                                   i8* blockaddress(@indirectbr_rewrite, %bb1),
+                                                   i8* blockaddress(@indirectbr_rewrite, %bb2),
+                                                   i8* blockaddress(@indirectbr_rewrite, %bb3),
+                                                   i8* blockaddress(@indirectbr_rewrite, %bb4),
+                                                   i8* blockaddress(@indirectbr_rewrite, %bb5),
+                                                   i8* blockaddress(@indirectbr_rewrite, %bb6),
+                                                   i8* blockaddress(@indirectbr_rewrite, %bb7),
+                                                   i8* blockaddress(@indirectbr_rewrite, %bb8),
+                                                   i8* blockaddress(@indirectbr_rewrite, %bb9)]
+
+; Check that when thunks are enabled the indirectbr instruction gets
+; rewritten to use switch, and that in turn doesn't get lowered as a jump
+; table.
+define void @indirectbr_rewrite(i64* readonly %p, i64* %sink) #0 {
+; X64-LABEL: indirectbr_rewrite:
+; X64-NOT:     jmpq *
+entry:
+  %i0 = load i64, i64* %p
+  %target.i0 = getelementptr [10 x i8*], [10 x i8*]* @indirectbr_rewrite.targets, i64 0, i64 %i0
+  %target0 = load i8*, i8** %target.i0
+  indirectbr i8* %target0, [label %bb1, label %bb3]
+
+bb0:
+  store volatile i64 0, i64* %sink
+  br label %latch
+
+bb1:
+  store volatile i64 1, i64* %sink
+  br label %latch
+
+bb2:
+  store volatile i64 2, i64* %sink
+  br label %latch
+
+bb3:
+  store volatile i64 3, i64* %sink
+  br label %latch
+
+bb4:
+  store volatile i64 4, i64* %sink
+  br label %latch
+
+bb5:
+  store volatile i64 5, i64* %sink
+  br label %latch
+
+bb6:
+  store volatile i64 6, i64* %sink
+  br label %latch
+
+bb7:
+  store volatile i64 7, i64* %sink
+  br label %latch
+
+bb8:
+  store volatile i64 8, i64* %sink
+  br label %latch
+
+bb9:
+  store volatile i64 9, i64* %sink
+  br label %latch
+
+latch:
+  %i.next = load i64, i64* %p
+  %target.i.next = getelementptr [10 x i8*], [10 x i8*]* @indirectbr_rewrite.targets, i64 0, i64 %i.next
+  %target.next = load i8*, i8** %target.i.next
+  ; Potentially hit a full 10 successors here so that even if we rewrite as
+  ; a switch it will try to be lowered with a jump table.
+  indirectbr i8* %target.next, [label %bb0,
+                                label %bb1,
+                                label %bb2,
+                                label %bb3,
+                                label %bb4,
+                                label %bb5,
+                                label %bb6,
+                                label %bb7,
+                                label %bb8,
+                                label %bb9]
+}
+
+; Lastly check that the necessary thunks were emitted.
+;
+; X64-LABEL:         .section        .text.__llvm_lvi_thunk_r11,{{.*}},__llvm_lvi_thunk_r11,comdat
+; X64-NEXT:          .hidden __llvm_lvi_thunk_r11
+; X64-NEXT:          .weak   __llvm_lvi_thunk_r11
+; X64:       __llvm_lvi_thunk_r11:
+; X64-NEXT:  # {{.*}}                                # %entry
+; X64-NEXT:          lfence
+; X64-NEXT:          jmpq     *%r11
+
+attributes #1 = { nonlazybind }
diff --git a/llvm/test/CodeGen/X86/lvi-hardening-ret.ll b/llvm/test/CodeGen/X86/lvi-hardening-ret.ll
new file mode 100644
index 0000000000000..9f2b028b30344
--- /dev/null
+++ b/llvm/test/CodeGen/X86/lvi-hardening-ret.ll
@@ -0,0 +1,72 @@
+; RUN: llc -verify-machineinstrs -mtriple=x86_64-unknown < %s | FileCheck %s
+
+define dso_local void @one_instruction() #0 {
+; CHECK-LABEL: one_instruction:
+entry:
+  ret void
+; CHECK-NOT:   retq
+; CHECK:       popq %[[x:[^ ]*]]
+; CHECK-NEXT:  lfence
+; CHECK-NEXT:  jmpq *%[[x]]
+}
+
+; Function Attrs: noinline nounwind optnone uwtable
+define dso_local i32 @ordinary_function(i32 %x, i32 %y) #0 {
+; CHECK-LABEL: ordinary_function:
+entry:
+  %x.addr = alloca i32, align 4
+  %y.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  store i32 %y, i32* %y.addr, align 4
+  %0 = load i32, i32* %x.addr, align 4
+  %1 = load i32, i32* %y.addr, align 4
+  %add = add nsw i32 %0, %1
+  ret i32 %add
+; CHECK-NOT:  retq
+; CHECK:      popq %[[x:[^ ]*]]
+; CHECK-NEXT: lfence
+; CHECK-NEXT: jmpq *%[[x]]
+}
+
+; Function Attrs: noinline nounwind optnone uwtable
+define dso_local i32 @no_caller_saved_registers_function(i32 %x, i32 %y) #1 {
+; CHECK-LABEL: no_caller_saved_registers_function:
+entry:
+  %x.addr = alloca i32, align 4
+  %y.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  store i32 %y, i32* %y.addr, align 4
+  %0 = load i32, i32* %x.addr, align 4
+  %1 = load i32, i32* %y.addr, align 4
+  %add = add nsw i32 %0, %1
+  ret i32 %add
+; CHECK-NOT:  retq
+; CHECK:      shlq $0, (%{{[^ ]*}})
+; CHECK-NEXT: lfence
+; CHECK-NEXT: retq
+}
+
+; Function Attrs: noinline nounwind optnone uwtable
+define dso_local preserve_mostcc void @preserve_most() #0 {
+; CHECK-LABEL: preserve_most:
+entry:
+  ret void
+; CHECK-NOT:  retq
+; CHECK:      popq %r11
+; CHECK-NEXT: lfence
+; CHECK-NEXT: jmpq *%r11
+}
+
+; Function Attrs: noinline nounwind optnone uwtable
+define dso_local preserve_allcc void @preserve_all() #0 {
+; CHECK-LABEL: preserve_all:
+entry:
+  ret void
+; CHECK-NOT:  retq
+; CHECK:      popq %r11
+; CHECK-NEXT: lfence
+; CHECK-NEXT: jmpq *%r11
+}
+
+attributes #0 = { "target-features"="+lvi-cfi" }
+attributes #1 = { "no_caller_saved_registers" "target-features"="+lvi-cfi" }
diff --git a/llvm/test/CodeGen/X86/madd.ll b/llvm/test/CodeGen/X86/madd.ll
index 43bb8ee004a2e..cad6f61e55453 100644
--- a/llvm/test/CodeGen/X86/madd.ll
+++ b/llvm/test/CodeGen/X86/madd.ll
@@ -236,10 +236,10 @@ define i32 @_Z10test_shortPsS_i_512(i16* nocapture readonly, i16* nocapture read
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX1-NEXT:    vmovdqu (%rsi,%rcx,2), %xmm2
 ; AVX1-NEXT:    vmovdqu 16(%rsi,%rcx,2), %xmm3
+; AVX1-NEXT:    vpmaddwd (%rdi,%rcx,2), %xmm2, %xmm2
 ; AVX1-NEXT:    vpmaddwd 16(%rdi,%rcx,2), %xmm3, %xmm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
 ; AVX1-NEXT:    vpaddd %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpmaddwd (%rdi,%rcx,2), %xmm2, %xmm2
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
 ; AVX1-NEXT:    addq $16, %rcx
@@ -407,16 +407,16 @@ define i32 @_Z10test_shortPsS_i_1024(i16* nocapture readonly, i16* nocapture rea
 ; AVX1-NEXT:    vmovdqu 16(%rsi,%rcx,2), %xmm4
 ; AVX1-NEXT:    vmovdqu 32(%rsi,%rcx,2), %xmm5
 ; AVX1-NEXT:    vmovdqu 48(%rsi,%rcx,2), %xmm6
+; AVX1-NEXT:    vpmaddwd (%rdi,%rcx,2), %xmm3, %xmm3
+; AVX1-NEXT:    vpmaddwd 16(%rdi,%rcx,2), %xmm4, %xmm4
+; AVX1-NEXT:    vpmaddwd 32(%rdi,%rcx,2), %xmm5, %xmm5
 ; AVX1-NEXT:    vpmaddwd 48(%rdi,%rcx,2), %xmm6, %xmm6
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm7
 ; AVX1-NEXT:    vpaddd %xmm7, %xmm6, %xmm6
-; AVX1-NEXT:    vpmaddwd 32(%rdi,%rcx,2), %xmm5, %xmm5
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm5, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm2, %ymm2
-; AVX1-NEXT:    vpmaddwd 16(%rdi,%rcx,2), %xmm4, %xmm4
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
 ; AVX1-NEXT:    vpaddd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpmaddwd (%rdi,%rcx,2), %xmm3, %xmm3
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm3, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
 ; AVX1-NEXT:    addq $16, %rcx
@@ -453,10 +453,10 @@ define i32 @_Z10test_shortPsS_i_1024(i16* nocapture readonly, i16* nocapture rea
 ; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX2-NEXT:    vmovdqu (%rsi,%rcx,2), %ymm3
 ; AVX2-NEXT:    vmovdqu 32(%rsi,%rcx,2), %ymm4
-; AVX2-NEXT:    vpmaddwd 32(%rdi,%rcx,2), %ymm4, %ymm4
-; AVX2-NEXT:    vpaddd %ymm2, %ymm4, %ymm2
 ; AVX2-NEXT:    vpmaddwd (%rdi,%rcx,2), %ymm3, %ymm3
 ; AVX2-NEXT:    vpaddd %ymm1, %ymm3, %ymm1
+; AVX2-NEXT:    vpmaddwd 32(%rdi,%rcx,2), %ymm4, %ymm3
+; AVX2-NEXT:    vpaddd %ymm2, %ymm3, %ymm2
 ; AVX2-NEXT:    addq $16, %rcx
 ; AVX2-NEXT:    cmpq %rcx, %rax
 ; AVX2-NEXT:    jne .LBB3_1
@@ -779,18 +779,18 @@ define i32 @_Z9test_charPcS_i_512(i8* nocapture readonly, i8* nocapture readonly
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movdqu (%rdi,%rcx), %xmm3
 ; SSE2-NEXT:    movdqu (%rsi,%rcx), %xmm4
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm3[8],xmm5[9],xmm3[9],xmm5[10],xmm3[10],xmm5[11],xmm3[11],xmm5[12],xmm3[12],xmm5[13],xmm3[13],xmm5[14],xmm3[14],xmm5[15],xmm3[15]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3],xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
 ; SSE2-NEXT:    psraw $8, %xmm5
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm4[8],xmm6[9],xmm4[9],xmm6[10],xmm4[10],xmm6[11],xmm4[11],xmm6[12],xmm4[12],xmm6[13],xmm4[13],xmm6[14],xmm4[14],xmm6[15],xmm4[15]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3],xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]
 ; SSE2-NEXT:    psraw $8, %xmm6
 ; SSE2-NEXT:    pmaddwd %xmm5, %xmm6
-; SSE2-NEXT:    paddd %xmm6, %xmm1
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    paddd %xmm6, %xmm2
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE2-NEXT:    psraw $8, %xmm3
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE2-NEXT:    psraw $8, %xmm4
 ; SSE2-NEXT:    pmaddwd %xmm3, %xmm4
-; SSE2-NEXT:    paddd %xmm4, %xmm2
+; SSE2-NEXT:    paddd %xmm4, %xmm1
 ; SSE2-NEXT:    addq $16, %rcx
 ; SSE2-NEXT:    cmpq %rcx, %rax
 ; SSE2-NEXT:    jne .LBB6_1
@@ -814,16 +814,16 @@ define i32 @_Z9test_charPcS_i_512(i8* nocapture readonly, i8* nocapture readonly
 ; AVX1-NEXT:    .p2align 4, 0x90
 ; AVX1-NEXT:  .LBB6_1: # %vector.body
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
-; AVX1-NEXT:    vpmovsxbw (%rdi,%rcx), %xmm2
-; AVX1-NEXT:    vpmovsxbw 8(%rdi,%rcx), %xmm3
-; AVX1-NEXT:    vpmovsxbw (%rsi,%rcx), %xmm4
-; AVX1-NEXT:    vpmaddwd %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpmovsxbw 8(%rdi,%rcx), %xmm2
+; AVX1-NEXT:    vpmovsxbw (%rdi,%rcx), %xmm3
 ; AVX1-NEXT:    vpmovsxbw 8(%rsi,%rcx), %xmm4
+; AVX1-NEXT:    vpmaddwd %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpmovsxbw (%rsi,%rcx), %xmm4
 ; AVX1-NEXT:    vpmaddwd %xmm3, %xmm4, %xmm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
-; AVX1-NEXT:    vpaddd %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX1-NEXT:    vpaddd %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddd %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
 ; AVX1-NEXT:    addq $16, %rcx
 ; AVX1-NEXT:    cmpq %rcx, %rax
 ; AVX1-NEXT:    jne .LBB6_1
@@ -943,34 +943,34 @@ define i32 @_Z9test_charPcS_i_1024(i8* nocapture readonly, i8* nocapture readonl
 ; SSE2-NEXT:    .p2align 4, 0x90
 ; SSE2-NEXT:  .LBB7_1: # %vector.body
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
-; SSE2-NEXT:    movdqu (%rdi,%rcx), %xmm10
-; SSE2-NEXT:    movdqu 16(%rdi,%rcx), %xmm7
-; SSE2-NEXT:    movdqu (%rsi,%rcx), %xmm9
-; SSE2-NEXT:    movdqu 16(%rsi,%rcx), %xmm0
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm7[8],xmm5[9],xmm7[9],xmm5[10],xmm7[10],xmm5[11],xmm7[11],xmm5[12],xmm7[12],xmm5[13],xmm7[13],xmm5[14],xmm7[14],xmm5[15],xmm7[15]
+; SSE2-NEXT:    movdqu (%rdi,%rcx), %xmm7
+; SSE2-NEXT:    movdqu 16(%rdi,%rcx), %xmm10
+; SSE2-NEXT:    movdqu (%rsi,%rcx), %xmm0
+; SSE2-NEXT:    movdqu 16(%rsi,%rcx), %xmm9
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3],xmm5[4],xmm7[4],xmm5[5],xmm7[5],xmm5[6],xmm7[6],xmm5[7],xmm7[7]
 ; SSE2-NEXT:    psraw $8, %xmm5
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3],xmm6[4],xmm0[4],xmm6[5],xmm0[5],xmm6[6],xmm0[6],xmm6[7],xmm0[7]
 ; SSE2-NEXT:    psraw $8, %xmm6
 ; SSE2-NEXT:    pmaddwd %xmm5, %xmm6
-; SSE2-NEXT:    paddd %xmm6, %xmm3
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3],xmm5[4],xmm7[4],xmm5[5],xmm7[5],xmm5[6],xmm7[6],xmm5[7],xmm7[7]
+; SSE2-NEXT:    paddd %xmm6, %xmm2
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm7[8],xmm5[9],xmm7[9],xmm5[10],xmm7[10],xmm5[11],xmm7[11],xmm5[12],xmm7[12],xmm5[13],xmm7[13],xmm5[14],xmm7[14],xmm5[15],xmm7[15]
 ; SSE2-NEXT:    psraw $8, %xmm5
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE2-NEXT:    psraw $8, %xmm0
 ; SSE2-NEXT:    pmaddwd %xmm5, %xmm0
-; SSE2-NEXT:    paddd %xmm0, %xmm1
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm10[8],xmm0[9],xmm10[9],xmm0[10],xmm10[10],xmm0[11],xmm10[11],xmm0[12],xmm10[12],xmm0[13],xmm10[13],xmm0[14],xmm10[14],xmm0[15],xmm10[15]
+; SSE2-NEXT:    paddd %xmm0, %xmm4
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3],xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]
 ; SSE2-NEXT:    psraw $8, %xmm0
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm9[8],xmm5[9],xmm9[9],xmm5[10],xmm9[10],xmm5[11],xmm9[11],xmm5[12],xmm9[12],xmm5[13],xmm9[13],xmm5[14],xmm9[14],xmm5[15],xmm9[15]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm9[0],xmm5[1],xmm9[1],xmm5[2],xmm9[2],xmm5[3],xmm9[3],xmm5[4],xmm9[4],xmm5[5],xmm9[5],xmm5[6],xmm9[6],xmm5[7],xmm9[7]
 ; SSE2-NEXT:    psraw $8, %xmm5
 ; SSE2-NEXT:    pmaddwd %xmm0, %xmm5
-; SSE2-NEXT:    paddd %xmm5, %xmm4
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3],xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]
+; SSE2-NEXT:    paddd %xmm5, %xmm1
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm10[8],xmm0[9],xmm10[9],xmm0[10],xmm10[10],xmm0[11],xmm10[11],xmm0[12],xmm10[12],xmm0[13],xmm10[13],xmm0[14],xmm10[14],xmm0[15],xmm10[15]
 ; SSE2-NEXT:    psraw $8, %xmm0
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm9[0],xmm5[1],xmm9[1],xmm5[2],xmm9[2],xmm5[3],xmm9[3],xmm5[4],xmm9[4],xmm5[5],xmm9[5],xmm5[6],xmm9[6],xmm5[7],xmm9[7]
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm9[8],xmm5[9],xmm9[9],xmm5[10],xmm9[10],xmm5[11],xmm9[11],xmm5[12],xmm9[12],xmm5[13],xmm9[13],xmm5[14],xmm9[14],xmm5[15],xmm9[15]
 ; SSE2-NEXT:    psraw $8, %xmm5
 ; SSE2-NEXT:    pmaddwd %xmm0, %xmm5
-; SSE2-NEXT:    paddd %xmm5, %xmm2
+; SSE2-NEXT:    paddd %xmm5, %xmm3
 ; SSE2-NEXT:    addq $32, %rcx
 ; SSE2-NEXT:    cmpq %rcx, %rax
 ; SSE2-NEXT:    jne .LBB7_1
@@ -999,26 +999,26 @@ define i32 @_Z9test_charPcS_i_1024(i8* nocapture readonly, i8* nocapture readonl
 ; AVX1-NEXT:    .p2align 4, 0x90
 ; AVX1-NEXT:  .LBB7_1: # %vector.body
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
-; AVX1-NEXT:    vpmovsxbw (%rdi,%rcx), %xmm3
-; AVX1-NEXT:    vpmovsxbw 8(%rdi,%rcx), %xmm4
-; AVX1-NEXT:    vpmovsxbw 16(%rdi,%rcx), %xmm5
-; AVX1-NEXT:    vpmovsxbw 24(%rdi,%rcx), %xmm6
-; AVX1-NEXT:    vpmovsxbw (%rsi,%rcx), %xmm7
+; AVX1-NEXT:    vpmovsxbw 24(%rdi,%rcx), %xmm3
+; AVX1-NEXT:    vpmovsxbw 16(%rdi,%rcx), %xmm4
+; AVX1-NEXT:    vpmovsxbw 8(%rdi,%rcx), %xmm5
+; AVX1-NEXT:    vpmovsxbw (%rdi,%rcx), %xmm6
+; AVX1-NEXT:    vpmovsxbw 24(%rsi,%rcx), %xmm7
 ; AVX1-NEXT:    vpmaddwd %xmm3, %xmm7, %xmm3
-; AVX1-NEXT:    vpmovsxbw 8(%rsi,%rcx), %xmm7
-; AVX1-NEXT:    vpmaddwd %xmm4, %xmm7, %xmm4
 ; AVX1-NEXT:    vpmovsxbw 16(%rsi,%rcx), %xmm7
+; AVX1-NEXT:    vpmaddwd %xmm4, %xmm7, %xmm4
+; AVX1-NEXT:    vpmovsxbw 8(%rsi,%rcx), %xmm7
 ; AVX1-NEXT:    vpmaddwd %xmm5, %xmm7, %xmm5
-; AVX1-NEXT:    vpmovsxbw 24(%rsi,%rcx), %xmm7
+; AVX1-NEXT:    vpmovsxbw (%rsi,%rcx), %xmm7
 ; AVX1-NEXT:    vpmaddwd %xmm6, %xmm7, %xmm6
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm7
-; AVX1-NEXT:    vpaddd %xmm7, %xmm6, %xmm6
-; AVX1-NEXT:    vpaddd %xmm2, %xmm5, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm2, %ymm2
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
-; AVX1-NEXT:    vpaddd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpaddd %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
+; AVX1-NEXT:    vpaddd %xmm7, %xmm3, %xmm3
+; AVX1-NEXT:    vpaddd %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vpaddd %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpaddd %xmm1, %xmm6, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
 ; AVX1-NEXT:    addq $32, %rcx
 ; AVX1-NEXT:    cmpq %rcx, %rax
 ; AVX1-NEXT:    jne .LBB7_1
@@ -1051,14 +1051,14 @@ define i32 @_Z9test_charPcS_i_1024(i8* nocapture readonly, i8* nocapture readonl
 ; AVX2-NEXT:    .p2align 4, 0x90
 ; AVX2-NEXT:  .LBB7_1: # %vector.body
 ; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
-; AVX2-NEXT:    vpmovsxbw (%rdi,%rcx), %ymm3
-; AVX2-NEXT:    vpmovsxbw 16(%rdi,%rcx), %ymm4
-; AVX2-NEXT:    vpmovsxbw (%rsi,%rcx), %ymm5
+; AVX2-NEXT:    vpmovsxbw 16(%rdi,%rcx), %ymm3
+; AVX2-NEXT:    vpmovsxbw (%rdi,%rcx), %ymm4
+; AVX2-NEXT:    vpmovsxbw 16(%rsi,%rcx), %ymm5
 ; AVX2-NEXT:    vpmaddwd %ymm3, %ymm5, %ymm3
-; AVX2-NEXT:    vpaddd %ymm1, %ymm3, %ymm1
-; AVX2-NEXT:    vpmovsxbw 16(%rsi,%rcx), %ymm3
-; AVX2-NEXT:    vpmaddwd %ymm4, %ymm3, %ymm3
 ; AVX2-NEXT:    vpaddd %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vpmovsxbw (%rsi,%rcx), %ymm3
+; AVX2-NEXT:    vpmaddwd %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpaddd %ymm1, %ymm3, %ymm1
 ; AVX2-NEXT:    addq $32, %rcx
 ; AVX2-NEXT:    cmpq %rcx, %rax
 ; AVX2-NEXT:    jne .LBB7_1
@@ -1913,9 +1913,9 @@ define <8 x i32> @pmaddwd_16(<16 x i16> %A, <16 x i16> %B) {
 ;
 ; AVX1-LABEL: pmaddwd_16:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vpmaddwd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpmaddwd %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpmaddwd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -1944,16 +1944,16 @@ define <16 x i32> @pmaddwd_32(<32 x i16> %A, <32 x i16> %B) {
 ;
 ; AVX1-LABEL: pmaddwd_32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm6
-; AVX1-NEXT:    vpmaddwd %xmm6, %xmm4, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm6
-; AVX1-NEXT:    vpmaddwd %xmm6, %xmm5, %xmm5
+; AVX1-NEXT:    vpmaddwd %xmm4, %xmm5, %xmm4
 ; AVX1-NEXT:    vpmaddwd %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm0, %ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vpmaddwd %xmm2, %xmm4, %xmm2
 ; AVX1-NEXT:    vpmaddwd %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: pmaddwd_32:
@@ -1964,9 +1964,9 @@ define <16 x i32> @pmaddwd_32(<32 x i16> %A, <32 x i16> %B) {
 ;
 ; AVX512F-LABEL: pmaddwd_32:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
-; AVX512F-NEXT:    vpmaddwd %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512F-NEXT:    vpmaddwd %ymm2, %ymm3, %ymm2
 ; AVX512F-NEXT:    vpmaddwd %ymm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
@@ -2126,9 +2126,9 @@ define <8 x i32> @jumbled_indices8(<16 x i16> %A, <16 x i16> %B) {
 ;
 ; AVX1-LABEL: jumbled_indices8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vpmaddwd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpmaddwd %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpmaddwd %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -2157,16 +2157,16 @@ define <16 x i32> @jumbled_indices16(<32 x i16> %A, <32 x i16> %B) {
 ;
 ; AVX1-LABEL: jumbled_indices16:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm6
-; AVX1-NEXT:    vpmaddwd %xmm6, %xmm4, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm6
-; AVX1-NEXT:    vpmaddwd %xmm6, %xmm5, %xmm5
+; AVX1-NEXT:    vpmaddwd %xmm4, %xmm5, %xmm4
 ; AVX1-NEXT:    vpmaddwd %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm0, %ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vpmaddwd %xmm2, %xmm4, %xmm2
 ; AVX1-NEXT:    vpmaddwd %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: jumbled_indices16:
@@ -2177,9 +2177,9 @@ define <16 x i32> @jumbled_indices16(<32 x i16> %A, <32 x i16> %B) {
 ;
 ; AVX512F-LABEL: jumbled_indices16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
-; AVX512F-NEXT:    vpmaddwd %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512F-NEXT:    vpmaddwd %ymm2, %ymm3, %ymm2
 ; AVX512F-NEXT:    vpmaddwd %ymm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
@@ -2221,26 +2221,26 @@ define <32 x i32> @jumbled_indices32(<64 x i16> %A, <64 x i16> %B) {
 ;
 ; AVX1-LABEL: jumbled_indices32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm8
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm9
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm10
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm11
-; AVX1-NEXT:    vextractf128 $1, %ymm7, %xmm12
-; AVX1-NEXT:    vpmaddwd %xmm12, %xmm8, %xmm8
-; AVX1-NEXT:    vextractf128 $1, %ymm6, %xmm12
-; AVX1-NEXT:    vpmaddwd %xmm12, %xmm9, %xmm9
-; AVX1-NEXT:    vextractf128 $1, %ymm5, %xmm12
-; AVX1-NEXT:    vpmaddwd %xmm12, %xmm10, %xmm10
-; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm12
-; AVX1-NEXT:    vpmaddwd %xmm12, %xmm11, %xmm11
+; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm8
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm9
+; AVX1-NEXT:    vpmaddwd %xmm8, %xmm9, %xmm8
 ; AVX1-NEXT:    vpmaddwd %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm11, %ymm0, %ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm5, %xmm8
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vpmaddwd %xmm4, %xmm8, %xmm4
 ; AVX1-NEXT:    vpmaddwd %xmm5, %xmm1, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm10, %ymm1, %ymm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
+; AVX1-NEXT:    vextractf128 $1, %ymm6, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm5
+; AVX1-NEXT:    vpmaddwd %xmm4, %xmm5, %xmm4
 ; AVX1-NEXT:    vpmaddwd %xmm6, %xmm2, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm2, %ymm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
+; AVX1-NEXT:    vextractf128 $1, %ymm7, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm5
+; AVX1-NEXT:    vpmaddwd %xmm4, %xmm5, %xmm4
 ; AVX1-NEXT:    vpmaddwd %xmm7, %xmm3, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm3, %ymm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: jumbled_indices32:
@@ -2656,7 +2656,7 @@ define i32 @madd_double_reduction(<8 x i16>* %arg, <8 x i16>* %arg1, <8 x i16>*
 ; AVX-NEXT:    vmovdqu (%rdx), %xmm1
 ; AVX-NEXT:    vpmaddwd (%rcx), %xmm1, %xmm1
 ; AVX-NEXT:    vpmaddwd (%rsi), %xmm0, %xmm0
-; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
@@ -2698,14 +2698,14 @@ define i32 @madd_quad_reduction(<8 x i16>* %arg, <8 x i16>* %arg1, <8 x i16>* %a
 ; SSE2-NEXT:    movdqu (%r8), %xmm0
 ; SSE2-NEXT:    movdqu (%r9), %xmm3
 ; SSE2-NEXT:    pmaddwd %xmm0, %xmm3
-; SSE2-NEXT:    paddd %xmm2, %xmm3
+; SSE2-NEXT:    paddd %xmm1, %xmm3
 ; SSE2-NEXT:    movdqu (%rax), %xmm0
-; SSE2-NEXT:    movdqu (%r10), %xmm2
-; SSE2-NEXT:    pmaddwd %xmm0, %xmm2
-; SSE2-NEXT:    paddd %xmm3, %xmm2
-; SSE2-NEXT:    paddd %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
-; SSE2-NEXT:    paddd %xmm2, %xmm0
+; SSE2-NEXT:    movdqu (%r10), %xmm1
+; SSE2-NEXT:    pmaddwd %xmm0, %xmm1
+; SSE2-NEXT:    paddd %xmm3, %xmm1
+; SSE2-NEXT:    paddd %xmm2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE2-NEXT:    paddd %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
@@ -2721,11 +2721,11 @@ define i32 @madd_quad_reduction(<8 x i16>* %arg, <8 x i16>* %arg1, <8 x i16>* %a
 ; AVX-NEXT:    vpmaddwd (%rsi), %xmm0, %xmm0
 ; AVX-NEXT:    vmovdqu (%r8), %xmm2
 ; AVX-NEXT:    vpmaddwd (%r9), %xmm2, %xmm2
-; AVX-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vmovdqu (%rax), %xmm2
 ; AVX-NEXT:    vpmaddwd (%r10), %xmm2, %xmm2
-; AVX-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
diff --git a/llvm/test/CodeGen/X86/masked_store_trunc.ll b/llvm/test/CodeGen/X86/masked_store_trunc.ll
index 2feb0382d8c9e..3e2a76b66719d 100644
--- a/llvm/test/CodeGen/X86/masked_store_trunc.ll
+++ b/llvm/test/CodeGen/X86/masked_store_trunc.ll
@@ -1372,7 +1372,7 @@ define void @truncstore_v4i64_v4i8(<4 x i64> %x, <4 x i8>* %p, <4 x i32> %mask)
 ; SSE2-LABEL: truncstore_v4i64_v4i8:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,0,0,0,255,0,0,0]
 ; SSE2-NEXT:    pand %xmm4, %xmm1
 ; SSE2-NEXT:    pand %xmm4, %xmm0
 ; SSE2-NEXT:    packuswb %xmm1, %xmm0
@@ -4435,7 +4435,7 @@ define void @truncstore_v32i16_v32i8(<32 x i16> %x, <32 x i8>* %p, <32 x i8> %ma
 ; SSE2-LABEL: truncstore_v32i16_v32i8:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm7, %xmm7
-; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [255,255,255,255,255,255,255,255]
 ; SSE2-NEXT:    pand %xmm6, %xmm1
 ; SSE2-NEXT:    pand %xmm6, %xmm0
 ; SSE2-NEXT:    packuswb %xmm1, %xmm0
@@ -4652,10 +4652,10 @@ define void @truncstore_v32i16_v32i8(<32 x i16> %x, <32 x i8>* %p, <32 x i8> %ma
 ; SSE4-LABEL: truncstore_v32i16_v32i8:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    pxor %xmm7, %xmm7
-; SSE4-NEXT:    movdqa {{.*#+}} xmm6 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; SSE4-NEXT:    pshufb %xmm6, %xmm1
-; SSE4-NEXT:    pshufb %xmm6, %xmm0
-; SSE4-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE4-NEXT:    movdqa {{.*#+}} xmm6 = [255,255,255,255,255,255,255,255]
+; SSE4-NEXT:    pand %xmm6, %xmm1
+; SSE4-NEXT:    pand %xmm6, %xmm0
+; SSE4-NEXT:    packuswb %xmm1, %xmm0
 ; SSE4-NEXT:    pcmpeqb %xmm7, %xmm4
 ; SSE4-NEXT:    pmovmskb %xmm4, %ecx
 ; SSE4-NEXT:    xorl $65535, %ecx # imm = 0xFFFF
@@ -4711,14 +4711,14 @@ define void @truncstore_v32i16_v32i8(<32 x i16> %x, <32 x i8>* %p, <32 x i8> %ma
 ; SSE4-NEXT:  .LBB15_29: # %cond.store27
 ; SSE4-NEXT:    pextrb $14, %xmm0, 14(%rdi)
 ; SSE4-NEXT:  .LBB15_30: # %else28
-; SSE4-NEXT:    pshufb %xmm6, %xmm3
-; SSE4-NEXT:    pshufb %xmm6, %xmm2
+; SSE4-NEXT:    pand %xmm6, %xmm3
+; SSE4-NEXT:    pand %xmm6, %xmm2
 ; SSE4-NEXT:    testl $32768, %eax # imm = 0x8000
 ; SSE4-NEXT:    je .LBB15_32
 ; SSE4-NEXT:  # %bb.31: # %cond.store29
 ; SSE4-NEXT:    pextrb $15, %xmm0, 15(%rdi)
 ; SSE4-NEXT:  .LBB15_32: # %else30
-; SSE4-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; SSE4-NEXT:    packuswb %xmm3, %xmm2
 ; SSE4-NEXT:    testl $65536, %eax # imm = 0x10000
 ; SSE4-NEXT:    jne .LBB15_33
 ; SSE4-NEXT:  # %bb.34: # %else32
@@ -5638,7 +5638,7 @@ define void @truncstore_v16i16_v16i8(<16 x i16> %x, <16 x i8>* %p, <16 x i8> %ma
 ; SSE2-LABEL: truncstore_v16i16_v16i8:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
 ; SSE2-NEXT:    pand %xmm4, %xmm1
 ; SSE2-NEXT:    pand %xmm4, %xmm0
 ; SSE2-NEXT:    packuswb %xmm1, %xmm0
@@ -5750,10 +5750,10 @@ define void @truncstore_v16i16_v16i8(<16 x i16> %x, <16 x i8>* %p, <16 x i8> %ma
 ; SSE4-LABEL: truncstore_v16i16_v16i8:
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    pxor %xmm3, %xmm3
-; SSE4-NEXT:    movdqa {{.*#+}} xmm4 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; SSE4-NEXT:    pshufb %xmm4, %xmm1
-; SSE4-NEXT:    pshufb %xmm4, %xmm0
-; SSE4-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE4-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
+; SSE4-NEXT:    pand %xmm4, %xmm1
+; SSE4-NEXT:    pand %xmm4, %xmm0
+; SSE4-NEXT:    packuswb %xmm1, %xmm0
 ; SSE4-NEXT:    pcmpeqb %xmm2, %xmm3
 ; SSE4-NEXT:    pmovmskb %xmm3, %eax
 ; SSE4-NEXT:    xorl $65535, %eax # imm = 0xFFFF
diff --git a/llvm/test/CodeGen/X86/masked_store_trunc_ssat.ll b/llvm/test/CodeGen/X86/masked_store_trunc_ssat.ll
index bde5ea6e9bde5..f2f9cc36ff543 100644
--- a/llvm/test/CodeGen/X86/masked_store_trunc_ssat.ll
+++ b/llvm/test/CodeGen/X86/masked_store_trunc_ssat.ll
@@ -2138,7 +2138,7 @@ define void @truncstore_v4i64_v4i8(<4 x i64> %x, <4 x i8>* %p, <4 x i32> %mask)
 ; SSE2-NEXT:    pand %xmm1, %xmm5
 ; SSE2-NEXT:    pandn %xmm8, %xmm1
 ; SSE2-NEXT:    por %xmm5, %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [255,0,0,0,255,0,0,0]
 ; SSE2-NEXT:    pand %xmm3, %xmm1
 ; SSE2-NEXT:    pand %xmm3, %xmm0
 ; SSE2-NEXT:    packuswb %xmm1, %xmm0
@@ -5263,7 +5263,8 @@ define void @truncstore_v4i32_v4i8(<4 x i32> %x, <4 x i8>* %p, <4 x i32> %mask)
 ; SSE4-NEXT:    pxor %xmm2, %xmm2
 ; SSE4-NEXT:    pminsd {{.*}}(%rip), %xmm0
 ; SSE4-NEXT:    pmaxsd {{.*}}(%rip), %xmm0
-; SSE4-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSE4-NEXT:    packssdw %xmm0, %xmm0
+; SSE4-NEXT:    packsswb %xmm0, %xmm0
 ; SSE4-NEXT:    pcmpeqd %xmm1, %xmm2
 ; SSE4-NEXT:    movmskps %xmm2, %eax
 ; SSE4-NEXT:    xorl $15, %eax
@@ -5301,7 +5302,8 @@ define void @truncstore_v4i32_v4i8(<4 x i32> %x, <4 x i8>* %p, <4 x i32> %mask)
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpminsd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vpmaxsd {{.*}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vmovmskps %xmm1, %eax
 ; AVX1-NEXT:    xorl $15, %eax
@@ -5341,7 +5343,8 @@ define void @truncstore_v4i32_v4i8(<4 x i32> %x, <4 x i8>* %p, <4 x i32> %mask)
 ; AVX2-NEXT:    vpminsd %xmm3, %xmm0, %xmm0
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [4294967168,4294967168,4294967168,4294967168]
 ; AVX2-NEXT:    vpmaxsd %xmm3, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
 ; AVX2-NEXT:    vmovmskps %xmm1, %eax
 ; AVX2-NEXT:    xorl $15, %eax
diff --git a/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll b/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll
index 84958fef6a325..be5823df716e4 100644
--- a/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll
+++ b/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll
@@ -4438,12 +4438,10 @@ define void @truncstore_v8i32_v8i8(<8 x i32> %x, <8 x i8>* %p, <8 x i32> %mask)
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    pxor %xmm4, %xmm4
 ; SSE4-NEXT:    movdqa {{.*#+}} xmm5 = [255,255,255,255]
-; SSE4-NEXT:    pminud %xmm5, %xmm0
 ; SSE4-NEXT:    pminud %xmm5, %xmm1
-; SSE4-NEXT:    movdqa {{.*#+}} xmm5 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
-; SSE4-NEXT:    pshufb %xmm5, %xmm1
-; SSE4-NEXT:    pshufb %xmm5, %xmm0
-; SSE4-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE4-NEXT:    pminud %xmm5, %xmm0
+; SSE4-NEXT:    packusdw %xmm1, %xmm0
+; SSE4-NEXT:    packuswb %xmm0, %xmm0
 ; SSE4-NEXT:    pcmpeqd %xmm4, %xmm3
 ; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
 ; SSE4-NEXT:    pxor %xmm1, %xmm3
@@ -4511,14 +4509,12 @@ define void @truncstore_v8i32_v8i8(<8 x i32> %x, <8 x i8>* %p, <8 x i32> %mask)
 ;
 ; AVX1-LABEL: truncstore_v8i32_v8i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,255,255]
-; AVX1-NEXT:    vpminud %xmm2, %xmm0, %xmm3
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpminud %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX1-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255]
+; AVX1-NEXT:    vpminud %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpminud %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpackusdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2
@@ -4591,10 +4587,8 @@ define void @truncstore_v8i32_v8i8(<8 x i32> %x, <8 x i8>* %p, <8 x i32> %mask)
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255]
 ; AVX2-NEXT:    vpminud %ymm3, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX2-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX2-NEXT:    vpshufb %xmm4, %xmm0, %xmm0
-; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; AVX2-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm1, %ymm1
 ; AVX2-NEXT:    vmovmskps %ymm1, %eax
 ; AVX2-NEXT:    notl %eax
@@ -5030,7 +5024,8 @@ define void @truncstore_v4i32_v4i8(<4 x i32> %x, <4 x i8>* %p, <4 x i32> %mask)
 ; SSE4:       # %bb.0:
 ; SSE4-NEXT:    pxor %xmm2, %xmm2
 ; SSE4-NEXT:    pminud {{.*}}(%rip), %xmm0
-; SSE4-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSE4-NEXT:    packusdw %xmm0, %xmm0
+; SSE4-NEXT:    packuswb %xmm0, %xmm0
 ; SSE4-NEXT:    pcmpeqd %xmm1, %xmm2
 ; SSE4-NEXT:    movmskps %xmm2, %eax
 ; SSE4-NEXT:    xorl $15, %eax
@@ -5067,7 +5062,8 @@ define void @truncstore_v4i32_v4i8(<4 x i32> %x, <4 x i8>* %p, <4 x i32> %mask)
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vmovmskps %xmm1, %eax
 ; AVX1-NEXT:    xorl $15, %eax
@@ -5105,7 +5101,8 @@ define void @truncstore_v4i32_v4i8(<4 x i32> %x, <4 x i8>* %p, <4 x i32> %mask)
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [255,255,255,255]
 ; AVX2-NEXT:    vpminud %xmm3, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX2-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm1, %xmm1
 ; AVX2-NEXT:    vmovmskps %xmm1, %eax
 ; AVX2-NEXT:    xorl $15, %eax
diff --git a/llvm/test/CodeGen/X86/materialize.ll b/llvm/test/CodeGen/X86/materialize.ll
index 429d578feccc5..1381a951a01de 100644
--- a/llvm/test/CodeGen/X86/materialize.ll
+++ b/llvm/test/CodeGen/X86/materialize.ll
@@ -2,6 +2,13 @@
 ; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+cmov %s -o - | FileCheck %s --check-prefix=CHECK64
 ; RUN: llc -mtriple=x86_64-pc-win32 -mattr=+cmov %s -o - | FileCheck %s --check-prefix=CHECKWIN64
 
+; RUN: llc -mtriple=i686-unknown-linux-gnu -mattr=+cmov %s -o /dev/null \
+; RUN:     -print-after postrapseudos -filter-print-funcs pr26023 2>&1 \
+; RUN:    | FileCheck %s --check-prefix=OPERAND32
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+cmov %s -o /dev/null \
+; RUN:     -print-after postrapseudos -filter-print-funcs one64_minsize 2>&1 \
+; RUN:    | FileCheck %s --check-prefix=OPERAND64
+
 define i32 @one32_nooptsize() {
 entry:
   ret i32 1
@@ -92,6 +99,12 @@ entry:
 ; CHECK32:       pushl $5
 ; CHECK32:       popl %ecx
 ; CHECK32:       retl
+
+; Check push/pop have implicit def/use of $esp
+; OPERAND32:      PUSH32i8 5, implicit-def $esp, implicit $esp
+; OPERAND32-NEXT: CFI_INSTRUCTION adjust_cfa_offset 4
+; OPERAND32-NEXT: renamable $ecx = POP32r implicit-def $esp, implicit $esp
+; OPERAND32-NEXT: CFI_INSTRUCTION adjust_cfa_offset -4
 }
 
 
@@ -110,6 +123,13 @@ entry:
 ; CHECKWIN64-LABEL: one64_minsize:
 ; CHECKWIN64:       movl $1, %eax
 ; CHECKWIN64-NEXT:  retq
+
+; Check push/pop have implicit def/use of $rsp
+; OPERAND64:      PUSH64i8 1, implicit-def $rsp, implicit $rsp
+; OPERAND64-NEXT: CFI_INSTRUCTION adjust_cfa_offset 8
+; OPERAND64-NEXT: $rax = POP64r implicit-def $rsp, implicit $rsp
+; OPERAND64-NEXT: CFI_INSTRUCTION adjust_cfa_offset -8
+; OPERAND64-NEXT: RET 0, $rax
 }
 
 define i32 @minus_one32() optsize {
diff --git a/llvm/test/CodeGen/X86/memcmp-more-load-pairs.ll b/llvm/test/CodeGen/X86/memcmp-more-load-pairs.ll
index 21aef9cbb8e86..374f573eed7b2 100644
--- a/llvm/test/CodeGen/X86/memcmp-more-load-pairs.ll
+++ b/llvm/test/CodeGen/X86/memcmp-more-load-pairs.ll
@@ -577,64 +577,53 @@ define i32 @length7(i8* %X, i8* %Y) nounwind {
 ; X86-LABEL: length7:
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl (%esi), %ecx
 ; X86-NEXT:    movl (%eax), %edx
-; X86-NEXT:    movl (%ecx), %esi
+; X86-NEXT:    bswapl %ecx
 ; X86-NEXT:    bswapl %edx
-; X86-NEXT:    bswapl %esi
-; X86-NEXT:    cmpl %esi, %edx
-; X86-NEXT:    jne .LBB19_4
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    jne .LBB19_2
 ; X86-NEXT:  # %bb.1: # %loadbb1
-; X86-NEXT:    movzwl 4(%eax), %edx
-; X86-NEXT:    movzwl 4(%ecx), %esi
-; X86-NEXT:    rolw $8, %dx
-; X86-NEXT:    rolw $8, %si
-; X86-NEXT:    movzwl %dx, %edx
-; X86-NEXT:    movzwl %si, %esi
-; X86-NEXT:    cmpl %esi, %edx
-; X86-NEXT:    jne .LBB19_4
-; X86-NEXT:  # %bb.2: # %loadbb2
-; X86-NEXT:    movzbl 6(%eax), %eax
-; X86-NEXT:    movzbl 6(%ecx), %ecx
-; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    popl %esi
-; X86-NEXT:    retl
-; X86-NEXT:  .LBB19_4: # %res_block
+; X86-NEXT:    movl 3(%esi), %ecx
+; X86-NEXT:    movl 3(%eax), %edx
+; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    bswapl %edx
 ; X86-NEXT:    xorl %eax, %eax
-; X86-NEXT:    cmpl %esi, %edx
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    je .LBB19_3
+; X86-NEXT:  .LBB19_2: # %res_block
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl %edx, %ecx
 ; X86-NEXT:    setae %al
 ; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:  .LBB19_3: # %endblock
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: length7:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    movl (%rsi), %ecx
-; X64-NEXT:    bswapl %eax
+; X64-NEXT:    movl (%rdi), %ecx
+; X64-NEXT:    movl (%rsi), %edx
 ; X64-NEXT:    bswapl %ecx
-; X64-NEXT:    cmpl %ecx, %eax
-; X64-NEXT:    jne .LBB19_4
+; X64-NEXT:    bswapl %edx
+; X64-NEXT:    cmpl %edx, %ecx
+; X64-NEXT:    jne .LBB19_2
 ; X64-NEXT:  # %bb.1: # %loadbb1
-; X64-NEXT:    movzwl 4(%rdi), %eax
-; X64-NEXT:    movzwl 4(%rsi), %ecx
-; X64-NEXT:    rolw $8, %ax
-; X64-NEXT:    rolw $8, %cx
-; X64-NEXT:    movzwl %ax, %eax
-; X64-NEXT:    movzwl %cx, %ecx
-; X64-NEXT:    cmpl %ecx, %eax
-; X64-NEXT:    jne .LBB19_4
-; X64-NEXT:  # %bb.2: # %loadbb2
-; X64-NEXT:    movzbl 6(%rdi), %eax
-; X64-NEXT:    movzbl 6(%rsi), %ecx
-; X64-NEXT:    subl %ecx, %eax
-; X64-NEXT:    retq
-; X64-NEXT:  .LBB19_4: # %res_block
-; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    cmpl %ecx, %eax
-; X64-NEXT:    setae %dl
-; X64-NEXT:    leal -1(%rdx,%rdx), %eax
+; X64-NEXT:    movl 3(%rdi), %ecx
+; X64-NEXT:    movl 3(%rsi), %edx
+; X64-NEXT:    bswapl %ecx
+; X64-NEXT:    bswapl %edx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpl %edx, %ecx
+; X64-NEXT:    je .LBB19_3
+; X64-NEXT:  .LBB19_2: # %res_block
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpl %edx, %ecx
+; X64-NEXT:    setae %al
+; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:  .LBB19_3: # %endblock
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 7) nounwind
   ret i32 %m
@@ -671,31 +660,25 @@ define i1 @length7_lt(i8* %X, i8* %Y) nounwind {
 ; X86-LABEL: length7_lt:
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl (%esi), %ecx
 ; X86-NEXT:    movl (%eax), %edx
-; X86-NEXT:    movl (%ecx), %esi
+; X86-NEXT:    bswapl %ecx
 ; X86-NEXT:    bswapl %edx
-; X86-NEXT:    bswapl %esi
-; X86-NEXT:    cmpl %esi, %edx
-; X86-NEXT:    jne .LBB21_4
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    jne .LBB21_2
 ; X86-NEXT:  # %bb.1: # %loadbb1
-; X86-NEXT:    movzwl 4(%eax), %edx
-; X86-NEXT:    movzwl 4(%ecx), %esi
-; X86-NEXT:    rolw $8, %dx
-; X86-NEXT:    rolw $8, %si
-; X86-NEXT:    movzwl %dx, %edx
-; X86-NEXT:    movzwl %si, %esi
-; X86-NEXT:    cmpl %esi, %edx
-; X86-NEXT:    jne .LBB21_4
-; X86-NEXT:  # %bb.2: # %loadbb2
-; X86-NEXT:    movzbl 6(%eax), %eax
-; X86-NEXT:    movzbl 6(%ecx), %ecx
-; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    jmp .LBB21_3
-; X86-NEXT:  .LBB21_4: # %res_block
+; X86-NEXT:    movl 3(%esi), %ecx
+; X86-NEXT:    movl 3(%eax), %edx
+; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    bswapl %edx
 ; X86-NEXT:    xorl %eax, %eax
-; X86-NEXT:    cmpl %esi, %edx
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    je .LBB21_3
+; X86-NEXT:  .LBB21_2: # %res_block
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl %edx, %ecx
 ; X86-NEXT:    setae %al
 ; X86-NEXT:    leal -1(%eax,%eax), %eax
 ; X86-NEXT:  .LBB21_3: # %endblock
@@ -706,33 +689,26 @@ define i1 @length7_lt(i8* %X, i8* %Y) nounwind {
 ;
 ; X64-LABEL: length7_lt:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    movl (%rsi), %ecx
-; X64-NEXT:    bswapl %eax
+; X64-NEXT:    movl (%rdi), %ecx
+; X64-NEXT:    movl (%rsi), %edx
 ; X64-NEXT:    bswapl %ecx
-; X64-NEXT:    cmpl %ecx, %eax
-; X64-NEXT:    jne .LBB21_3
+; X64-NEXT:    bswapl %edx
+; X64-NEXT:    cmpl %edx, %ecx
+; X64-NEXT:    jne .LBB21_2
 ; X64-NEXT:  # %bb.1: # %loadbb1
-; X64-NEXT:    movzwl 4(%rdi), %eax
-; X64-NEXT:    movzwl 4(%rsi), %ecx
-; X64-NEXT:    rolw $8, %ax
-; X64-NEXT:    rolw $8, %cx
-; X64-NEXT:    movzwl %ax, %eax
-; X64-NEXT:    movzwl %cx, %ecx
-; X64-NEXT:    cmpl %ecx, %eax
-; X64-NEXT:    jne .LBB21_3
-; X64-NEXT:  # %bb.2: # %loadbb2
-; X64-NEXT:    movzbl 6(%rdi), %eax
-; X64-NEXT:    movzbl 6(%rsi), %ecx
-; X64-NEXT:    subl %ecx, %eax
-; X64-NEXT:    shrl $31, %eax
-; X64-NEXT:    # kill: def $al killed $al killed $eax
-; X64-NEXT:    retq
-; X64-NEXT:  .LBB21_3: # %res_block
-; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    cmpl %ecx, %eax
-; X64-NEXT:    setae %dl
-; X64-NEXT:    leal -1(%rdx,%rdx), %eax
+; X64-NEXT:    movl 3(%rdi), %ecx
+; X64-NEXT:    movl 3(%rsi), %edx
+; X64-NEXT:    bswapl %ecx
+; X64-NEXT:    bswapl %edx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpl %edx, %ecx
+; X64-NEXT:    je .LBB21_3
+; X64-NEXT:  .LBB21_2: # %res_block
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpl %edx, %ecx
+; X64-NEXT:    setae %al
+; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:  .LBB21_3: # %endblock
 ; X64-NEXT:    shrl $31, %eax
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq
@@ -1931,8 +1907,41 @@ define i32 @length31(i8* %X, i8* %Y) nounwind {
 ;
 ; X64-LABEL: length31:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl $31, %edx
-; X64-NEXT:    jmp memcmp # TAILCALL
+; X64-NEXT:    movq (%rdi), %rcx
+; X64-NEXT:    movq (%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    jne .LBB43_4
+; X64-NEXT:  # %bb.1: # %loadbb1
+; X64-NEXT:    movq 8(%rdi), %rcx
+; X64-NEXT:    movq 8(%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    jne .LBB43_4
+; X64-NEXT:  # %bb.2: # %loadbb2
+; X64-NEXT:    movq 16(%rdi), %rcx
+; X64-NEXT:    movq 16(%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    jne .LBB43_4
+; X64-NEXT:  # %bb.3: # %loadbb3
+; X64-NEXT:    movq 23(%rdi), %rcx
+; X64-NEXT:    movq 23(%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    je .LBB43_5
+; X64-NEXT:  .LBB43_4: # %res_block
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    setae %al
+; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:  .LBB43_5: # %endblock
+; X64-NEXT:    retq
   %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 31) nounwind
   ret i32 %m
 }
@@ -2063,12 +2072,42 @@ define i1 @length31_lt(i8* %x, i8* %y) nounwind {
 ;
 ; X64-LABEL: length31_lt:
 ; X64:       # %bb.0:
-; X64-NEXT:    pushq %rax
-; X64-NEXT:    movl $31, %edx
-; X64-NEXT:    callq memcmp
+; X64-NEXT:    movq (%rdi), %rcx
+; X64-NEXT:    movq (%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    jne .LBB45_4
+; X64-NEXT:  # %bb.1: # %loadbb1
+; X64-NEXT:    movq 8(%rdi), %rcx
+; X64-NEXT:    movq 8(%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    jne .LBB45_4
+; X64-NEXT:  # %bb.2: # %loadbb2
+; X64-NEXT:    movq 16(%rdi), %rcx
+; X64-NEXT:    movq 16(%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    jne .LBB45_4
+; X64-NEXT:  # %bb.3: # %loadbb3
+; X64-NEXT:    movq 23(%rdi), %rcx
+; X64-NEXT:    movq 23(%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    je .LBB45_5
+; X64-NEXT:  .LBB45_4: # %res_block
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    setae %al
+; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:  .LBB45_5: # %endblock
 ; X64-NEXT:    shrl $31, %eax
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
-; X64-NEXT:    popq %rcx
 ; X64-NEXT:    retq
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 31) nounwind
   %cmp = icmp slt i32 %call, 0
@@ -2090,12 +2129,42 @@ define i1 @length31_gt(i8* %x, i8* %y) nounwind {
 ;
 ; X64-LABEL: length31_gt:
 ; X64:       # %bb.0:
-; X64-NEXT:    pushq %rax
-; X64-NEXT:    movl $31, %edx
-; X64-NEXT:    callq memcmp
-; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    movq (%rdi), %rax
+; X64-NEXT:    movq (%rsi), %rcx
+; X64-NEXT:    bswapq %rax
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    cmpq %rcx, %rax
+; X64-NEXT:    jne .LBB46_4
+; X64-NEXT:  # %bb.1: # %loadbb1
+; X64-NEXT:    movq 8(%rdi), %rax
+; X64-NEXT:    movq 8(%rsi), %rcx
+; X64-NEXT:    bswapq %rax
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    cmpq %rcx, %rax
+; X64-NEXT:    jne .LBB46_4
+; X64-NEXT:  # %bb.2: # %loadbb2
+; X64-NEXT:    movq 16(%rdi), %rax
+; X64-NEXT:    movq 16(%rsi), %rcx
+; X64-NEXT:    bswapq %rax
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    cmpq %rcx, %rax
+; X64-NEXT:    jne .LBB46_4
+; X64-NEXT:  # %bb.3: # %loadbb3
+; X64-NEXT:    movq 23(%rdi), %rax
+; X64-NEXT:    movq 23(%rsi), %rcx
+; X64-NEXT:    bswapq %rax
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    cmpq %rcx, %rax
+; X64-NEXT:    je .LBB46_5
+; X64-NEXT:  .LBB46_4: # %res_block
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    cmpq %rcx, %rax
+; X64-NEXT:    setae %dl
+; X64-NEXT:    leal -1(%rdx,%rdx), %edx
+; X64-NEXT:  .LBB46_5: # %endblock
+; X64-NEXT:    testl %edx, %edx
 ; X64-NEXT:    setg %al
-; X64-NEXT:    popq %rcx
 ; X64-NEXT:    retq
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 31) nounwind
   %cmp = icmp sgt i32 %call, 0
diff --git a/llvm/test/CodeGen/X86/memcmp.ll b/llvm/test/CodeGen/X86/memcmp.ll
index 64c8b8c9f6d96..de604ded08d1a 100644
--- a/llvm/test/CodeGen/X86/memcmp.ll
+++ b/llvm/test/CodeGen/X86/memcmp.ll
@@ -622,18 +622,55 @@ define i1 @length5_lt(i8* %X, i8* %Y) nounwind {
 define i32 @length7(i8* %X, i8* %Y) nounwind {
 ; X86-LABEL: length7:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl $0
-; X86-NEXT:    pushl $7
-; X86-NEXT:    pushl {{[0-9]+}}(%esp)
-; X86-NEXT:    pushl {{[0-9]+}}(%esp)
-; X86-NEXT:    calll memcmp
-; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl (%esi), %ecx
+; X86-NEXT:    movl (%eax), %edx
+; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    bswapl %edx
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    jne .LBB21_2
+; X86-NEXT:  # %bb.1: # %loadbb1
+; X86-NEXT:    movl 3(%esi), %ecx
+; X86-NEXT:    movl 3(%eax), %edx
+; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    bswapl %edx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    je .LBB21_3
+; X86-NEXT:  .LBB21_2: # %res_block
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    setae %al
+; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:  .LBB21_3: # %endblock
+; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: length7:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl $7, %edx
-; X64-NEXT:    jmp memcmp # TAILCALL
+; X64-NEXT:    movl (%rdi), %ecx
+; X64-NEXT:    movl (%rsi), %edx
+; X64-NEXT:    bswapl %ecx
+; X64-NEXT:    bswapl %edx
+; X64-NEXT:    cmpl %edx, %ecx
+; X64-NEXT:    jne .LBB21_2
+; X64-NEXT:  # %bb.1: # %loadbb1
+; X64-NEXT:    movl 3(%rdi), %ecx
+; X64-NEXT:    movl 3(%rsi), %edx
+; X64-NEXT:    bswapl %ecx
+; X64-NEXT:    bswapl %edx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpl %edx, %ecx
+; X64-NEXT:    je .LBB21_3
+; X64-NEXT:  .LBB21_2: # %res_block
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpl %edx, %ecx
+; X64-NEXT:    setae %al
+; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:  .LBB21_3: # %endblock
+; X64-NEXT:    retq
   %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 7) nounwind
   ret i32 %m
 }
@@ -641,24 +678,58 @@ define i32 @length7(i8* %X, i8* %Y) nounwind {
 define i1 @length7_lt(i8* %X, i8* %Y) nounwind {
 ; X86-LABEL: length7_lt:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl $0
-; X86-NEXT:    pushl $7
-; X86-NEXT:    pushl {{[0-9]+}}(%esp)
-; X86-NEXT:    pushl {{[0-9]+}}(%esp)
-; X86-NEXT:    calll memcmp
-; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl (%esi), %ecx
+; X86-NEXT:    movl (%eax), %edx
+; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    bswapl %edx
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    jne .LBB22_2
+; X86-NEXT:  # %bb.1: # %loadbb1
+; X86-NEXT:    movl 3(%esi), %ecx
+; X86-NEXT:    movl 3(%eax), %edx
+; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    bswapl %edx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    je .LBB22_3
+; X86-NEXT:  .LBB22_2: # %res_block
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    setae %al
+; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:  .LBB22_3: # %endblock
 ; X86-NEXT:    shrl $31, %eax
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: length7_lt:
 ; X64:       # %bb.0:
-; X64-NEXT:    pushq %rax
-; X64-NEXT:    movl $7, %edx
-; X64-NEXT:    callq memcmp
+; X64-NEXT:    movl (%rdi), %ecx
+; X64-NEXT:    movl (%rsi), %edx
+; X64-NEXT:    bswapl %ecx
+; X64-NEXT:    bswapl %edx
+; X64-NEXT:    cmpl %edx, %ecx
+; X64-NEXT:    jne .LBB22_2
+; X64-NEXT:  # %bb.1: # %loadbb1
+; X64-NEXT:    movl 3(%rdi), %ecx
+; X64-NEXT:    movl 3(%rsi), %edx
+; X64-NEXT:    bswapl %ecx
+; X64-NEXT:    bswapl %edx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpl %edx, %ecx
+; X64-NEXT:    je .LBB22_3
+; X64-NEXT:  .LBB22_2: # %res_block
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpl %edx, %ecx
+; X64-NEXT:    setae %al
+; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:  .LBB22_3: # %endblock
 ; X64-NEXT:    shrl $31, %eax
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
-; X64-NEXT:    popq %rcx
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 7) nounwind
   %c = icmp slt i32 %m, 0
@@ -998,8 +1069,27 @@ define i32 @length15(i8* %X, i8* %Y) nounwind {
 ;
 ; X64-LABEL: length15:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl $15, %edx
-; X64-NEXT:    jmp memcmp # TAILCALL
+; X64-NEXT:    movq (%rdi), %rcx
+; X64-NEXT:    movq (%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    jne .LBB34_2
+; X64-NEXT:  # %bb.1: # %loadbb1
+; X64-NEXT:    movq 7(%rdi), %rcx
+; X64-NEXT:    movq 7(%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    je .LBB34_3
+; X64-NEXT:  .LBB34_2: # %res_block
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    setae %al
+; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:  .LBB34_3: # %endblock
+; X64-NEXT:    retq
   %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 15) nounwind
   ret i32 %m
 }
@@ -1019,12 +1109,28 @@ define i1 @length15_lt(i8* %X, i8* %Y) nounwind {
 ;
 ; X64-LABEL: length15_lt:
 ; X64:       # %bb.0:
-; X64-NEXT:    pushq %rax
-; X64-NEXT:    movl $15, %edx
-; X64-NEXT:    callq memcmp
+; X64-NEXT:    movq (%rdi), %rcx
+; X64-NEXT:    movq (%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    jne .LBB35_2
+; X64-NEXT:  # %bb.1: # %loadbb1
+; X64-NEXT:    movq 7(%rdi), %rcx
+; X64-NEXT:    movq 7(%rsi), %rdx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    je .LBB35_3
+; X64-NEXT:  .LBB35_2: # %res_block
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    setae %al
+; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:  .LBB35_3: # %endblock
 ; X64-NEXT:    shrl $31, %eax
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
-; X64-NEXT:    popq %rcx
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 15) nounwind
   %c = icmp slt i32 %m, 0
@@ -1044,9 +1150,25 @@ define i32 @length15_const(i8* %X, i8* %Y) nounwind {
 ;
 ; X64-LABEL: length15_const:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl $.L.str+1, %esi
-; X64-NEXT:    movl $15, %edx
-; X64-NEXT:    jmp memcmp # TAILCALL
+; X64-NEXT:    movabsq $3544952156018063160, %rcx # imm = 0x3132333435363738
+; X64-NEXT:    movq (%rdi), %rdx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    cmpq %rcx, %rdx
+; X64-NEXT:    jne .LBB36_2
+; X64-NEXT:  # %bb.1: # %loadbb1
+; X64-NEXT:    movabsq $4051322327650219061, %rcx # imm = 0x3839303132333435
+; X64-NEXT:    movq 7(%rdi), %rdx
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rcx, %rdx
+; X64-NEXT:    je .LBB36_3
+; X64-NEXT:  .LBB36_2: # %res_block
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rcx, %rdx
+; X64-NEXT:    setae %al
+; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:  .LBB36_3: # %endblock
+; X64-NEXT:    retq
   %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 1), i64 15) nounwind
   ret i32 %m
 }
@@ -1093,13 +1215,26 @@ define i1 @length15_gt_const(i8* %X, i8* %Y) nounwind {
 ;
 ; X64-LABEL: length15_gt_const:
 ; X64:       # %bb.0:
-; X64-NEXT:    pushq %rax
-; X64-NEXT:    movl $.L.str+1, %esi
-; X64-NEXT:    movl $15, %edx
-; X64-NEXT:    callq memcmp
-; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    movabsq $3544952156018063160, %rax # imm = 0x3132333435363738
+; X64-NEXT:    movq (%rdi), %rcx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    cmpq %rax, %rcx
+; X64-NEXT:    jne .LBB38_2
+; X64-NEXT:  # %bb.1: # %loadbb1
+; X64-NEXT:    movabsq $4051322327650219061, %rax # imm = 0x3839303132333435
+; X64-NEXT:    movq 7(%rdi), %rcx
+; X64-NEXT:    bswapq %rcx
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    cmpq %rax, %rcx
+; X64-NEXT:    je .LBB38_3
+; X64-NEXT:  .LBB38_2: # %res_block
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    cmpq %rax, %rcx
+; X64-NEXT:    setae %dl
+; X64-NEXT:    leal -1(%rdx,%rdx), %edx
+; X64-NEXT:  .LBB38_3: # %endblock
+; X64-NEXT:    testl %edx, %edx
 ; X64-NEXT:    setg %al
-; X64-NEXT:    popq %rcx
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 1), i64 15) nounwind
   %c = icmp sgt i32 %m, 0
diff --git a/llvm/test/CodeGen/X86/min-legal-vector-width.ll b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
index 543da1e999c6b..81de0ee4ebaab 100644
--- a/llvm/test/CodeGen/X86/min-legal-vector-width.ll
+++ b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
@@ -180,14 +180,14 @@ define i32 @_Z9test_charPcS_i_256(i8* nocapture readonly, i8* nocapture readonly
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB8_1: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vpmovsxbw (%rdi,%rcx), %ymm3
-; CHECK-NEXT:    vpmovsxbw 16(%rdi,%rcx), %ymm4
-; CHECK-NEXT:    vpmovsxbw (%rsi,%rcx), %ymm5
+; CHECK-NEXT:    vpmovsxbw 16(%rdi,%rcx), %ymm3
+; CHECK-NEXT:    vpmovsxbw (%rdi,%rcx), %ymm4
+; CHECK-NEXT:    vpmovsxbw 16(%rsi,%rcx), %ymm5
 ; CHECK-NEXT:    vpmaddwd %ymm3, %ymm5, %ymm3
-; CHECK-NEXT:    vpaddd %ymm1, %ymm3, %ymm1
-; CHECK-NEXT:    vpmovsxbw 16(%rsi,%rcx), %ymm3
-; CHECK-NEXT:    vpmaddwd %ymm4, %ymm3, %ymm3
 ; CHECK-NEXT:    vpaddd %ymm2, %ymm3, %ymm2
+; CHECK-NEXT:    vpmovsxbw (%rsi,%rcx), %ymm3
+; CHECK-NEXT:    vpmaddwd %ymm4, %ymm3, %ymm3
+; CHECK-NEXT:    vpaddd %ymm1, %ymm3, %ymm1
 ; CHECK-NEXT:    addq $32, %rcx
 ; CHECK-NEXT:    cmpq %rcx, %rax
 ; CHECK-NEXT:    jne .LBB8_1
@@ -1600,3 +1600,43 @@ define i32 @v64i1_inline_asm() "min-legal-vector-width"="256" {
   %4 = load i32, i32* %1, align 4
   ret i32 %4
 }
+
+define void @cmp_v8i64_sext(<8 x i64>* %xptr, <8 x i64>* %yptr, <8 x i64>* %zptr) "min-legal-vector-width"="256" {
+; CHECK-LABEL: cmp_v8i64_sext:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa (%rsi), %ymm0
+; CHECK-NEXT:    vmovdqa 32(%rsi), %ymm1
+; CHECK-NEXT:    vpcmpgtq 32(%rdi), %ymm1, %ymm1
+; CHECK-NEXT:    vpcmpgtq (%rdi), %ymm0, %ymm0
+; CHECK-NEXT:    vmovdqa %ymm0, (%rdx)
+; CHECK-NEXT:    vmovdqa %ymm1, 32(%rdx)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %x = load <8 x i64>, <8 x i64>* %xptr
+  %y = load <8 x i64>, <8 x i64>* %yptr
+  %cmp = icmp slt <8 x i64> %x, %y
+  %ext = sext <8 x i1> %cmp to <8 x i64>
+  store <8 x i64> %ext, <8 x i64>* %zptr
+  ret void
+}
+
+define void @cmp_v8i64_zext(<8 x i64>* %xptr, <8 x i64>* %yptr, <8 x i64>* %zptr) "min-legal-vector-width"="256" {
+; CHECK-LABEL: cmp_v8i64_zext:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa (%rsi), %ymm0
+; CHECK-NEXT:    vmovdqa 32(%rsi), %ymm1
+; CHECK-NEXT:    vpcmpgtq 32(%rdi), %ymm1, %ymm1
+; CHECK-NEXT:    vpcmpgtq (%rdi), %ymm0, %ymm0
+; CHECK-NEXT:    vpsrlq $63, %ymm1, %ymm1
+; CHECK-NEXT:    vpsrlq $63, %ymm0, %ymm0
+; CHECK-NEXT:    vmovdqa %ymm0, (%rdx)
+; CHECK-NEXT:    vmovdqa %ymm1, 32(%rdx)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %x = load <8 x i64>, <8 x i64>* %xptr
+  %y = load <8 x i64>, <8 x i64>* %yptr
+  %cmp = icmp slt <8 x i64> %x, %y
+  %ext = zext <8 x i1> %cmp to <8 x i64>
+  store <8 x i64> %ext, <8 x i64>* %zptr
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/psubus.ll b/llvm/test/CodeGen/X86/psubus.ll
index c765412a71ed2..1d45732d33a9c 100644
--- a/llvm/test/CodeGen/X86/psubus.ll
+++ b/llvm/test/CodeGen/X86/psubus.ll
@@ -622,34 +622,30 @@ define <16 x i8> @test14(<16 x i8> %x, <16 x i32> %y) nounwind {
 ; SSE2-NEXT:    psubd %xmm5, %xmm4
 ; SSE2-NEXT:    por %xmm0, %xmm5
 ; SSE2-NEXT:    pcmpgtd %xmm9, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
-; SSE2-NEXT:    pand %xmm9, %xmm5
 ; SSE2-NEXT:    movdqa %xmm3, %xmm7
 ; SSE2-NEXT:    pxor %xmm0, %xmm7
 ; SSE2-NEXT:    psubd %xmm10, %xmm3
 ; SSE2-NEXT:    por %xmm0, %xmm10
 ; SSE2-NEXT:    pcmpgtd %xmm7, %xmm10
-; SSE2-NEXT:    pand %xmm9, %xmm10
-; SSE2-NEXT:    packuswb %xmm5, %xmm10
+; SSE2-NEXT:    packssdw %xmm5, %xmm10
 ; SSE2-NEXT:    movdqa %xmm2, %xmm5
 ; SSE2-NEXT:    pxor %xmm0, %xmm5
 ; SSE2-NEXT:    psubd %xmm6, %xmm2
 ; SSE2-NEXT:    por %xmm0, %xmm6
 ; SSE2-NEXT:    pcmpgtd %xmm5, %xmm6
-; SSE2-NEXT:    pand %xmm9, %xmm6
 ; SSE2-NEXT:    movdqa %xmm1, %xmm5
 ; SSE2-NEXT:    pxor %xmm0, %xmm5
 ; SSE2-NEXT:    por %xmm8, %xmm0
 ; SSE2-NEXT:    pcmpgtd %xmm5, %xmm0
-; SSE2-NEXT:    pand %xmm9, %xmm0
-; SSE2-NEXT:    packuswb %xmm6, %xmm0
-; SSE2-NEXT:    packuswb %xmm10, %xmm0
+; SSE2-NEXT:    packssdw %xmm6, %xmm0
+; SSE2-NEXT:    packsswb %xmm10, %xmm0
 ; SSE2-NEXT:    psubd %xmm8, %xmm1
-; SSE2-NEXT:    pand %xmm9, %xmm4
-; SSE2-NEXT:    pand %xmm9, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
+; SSE2-NEXT:    pand %xmm5, %xmm4
+; SSE2-NEXT:    pand %xmm5, %xmm3
 ; SSE2-NEXT:    packuswb %xmm4, %xmm3
-; SSE2-NEXT:    pand %xmm9, %xmm2
-; SSE2-NEXT:    pand %xmm9, %xmm1
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    pand %xmm5, %xmm1
 ; SSE2-NEXT:    packuswb %xmm2, %xmm1
 ; SSE2-NEXT:    packuswb %xmm3, %xmm1
 ; SSE2-NEXT:    pandn %xmm1, %xmm0
@@ -657,48 +653,41 @@ define <16 x i8> @test14(<16 x i8> %x, <16 x i32> %y) nounwind {
 ;
 ; SSSE3-LABEL: test14:
 ; SSSE3:       # %bb.0: # %vector.ph
-; SSSE3-NEXT:    pxor %xmm7, %xmm7
-; SSSE3-NEXT:    movdqa %xmm0, %xmm11
-; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm7[8],xmm11[9],xmm7[9],xmm11[10],xmm7[10],xmm11[11],xmm7[11],xmm11[12],xmm7[12],xmm11[13],xmm7[13],xmm11[14],xmm7[14],xmm11[15],xmm7[15]
-; SSSE3-NEXT:    movdqa %xmm11, %xmm8
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]
-; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm7[4],xmm11[5],xmm7[5],xmm11[6],xmm7[6],xmm11[7],xmm7[7]
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[1],xmm7[1],xmm0[2],xmm7[2],xmm0[3],xmm7[3],xmm0[4],xmm7[4],xmm0[5],xmm7[5],xmm0[6],xmm7[6],xmm0[7],xmm7[7]
-; SSSE3-NEXT:    movdqa %xmm0, %xmm10
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm7[0],xmm10[1],xmm7[1],xmm10[2],xmm7[2],xmm10[3],xmm7[3]
-; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm7[4],xmm0[5],xmm7[5],xmm0[6],xmm7[6],xmm0[7],xmm7[7]
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm7 = [2147483648,2147483648,2147483648,2147483648]
-; SSSE3-NEXT:    movdqa %xmm2, %xmm9
-; SSSE3-NEXT:    pxor %xmm7, %xmm9
-; SSSE3-NEXT:    psubd %xmm0, %xmm2
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm5
-; SSSE3-NEXT:    por %xmm7, %xmm5
+; SSSE3-NEXT:    pxor %xmm0, %xmm0
+; SSSE3-NEXT:    movdqa %xmm5, %xmm6
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3],xmm6[4],xmm0[4],xmm6[5],xmm0[5],xmm6[6],xmm0[6],xmm6[7],xmm0[7]
+; SSSE3-NEXT:    movdqa %xmm6, %xmm8
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1],xmm8[2],xmm0[2],xmm8[3],xmm0[3]
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm0[4],xmm6[5],xmm0[5],xmm6[6],xmm0[6],xmm6[7],xmm0[7]
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
+; SSSE3-NEXT:    movdqa %xmm5, %xmm10
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm0[0],xmm10[1],xmm0[1],xmm10[2],xmm0[2],xmm10[3],xmm0[3]
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm4, %xmm9
+; SSSE3-NEXT:    pxor %xmm0, %xmm9
+; SSSE3-NEXT:    psubd %xmm5, %xmm4
+; SSSE3-NEXT:    por %xmm0, %xmm5
 ; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm5
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
-; SSSE3-NEXT:    pshufb %xmm9, %xmm5
-; SSSE3-NEXT:    movdqa %xmm1, %xmm6
-; SSSE3-NEXT:    pxor %xmm7, %xmm6
-; SSSE3-NEXT:    psubd %xmm10, %xmm1
-; SSSE3-NEXT:    movdqa %xmm10, %xmm0
-; SSSE3-NEXT:    por %xmm7, %xmm0
-; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm0
-; SSSE3-NEXT:    pshufb %xmm9, %xmm0
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1]
-; SSSE3-NEXT:    movdqa %xmm4, %xmm5
-; SSSE3-NEXT:    pxor %xmm7, %xmm5
-; SSSE3-NEXT:    psubd %xmm11, %xmm4
-; SSSE3-NEXT:    por %xmm7, %xmm11
-; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm11
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = <u,u,u,u,0,4,8,12,u,u,u,u,u,u,u,u>
-; SSSE3-NEXT:    pshufb %xmm5, %xmm11
-; SSSE3-NEXT:    movdqa %xmm3, %xmm6
-; SSSE3-NEXT:    pxor %xmm7, %xmm6
-; SSSE3-NEXT:    por %xmm8, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSSE3-NEXT:    pshufb %xmm5, %xmm7
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm11[0],xmm7[1],xmm11[1]
-; SSSE3-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm7[2,3]
-; SSSE3-NEXT:    psubd %xmm8, %xmm3
+; SSSE3-NEXT:    movdqa %xmm3, %xmm7
+; SSSE3-NEXT:    pxor %xmm0, %xmm7
+; SSSE3-NEXT:    psubd %xmm10, %xmm3
+; SSSE3-NEXT:    por %xmm0, %xmm10
+; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm10
+; SSSE3-NEXT:    packssdw %xmm5, %xmm10
+; SSSE3-NEXT:    movdqa %xmm2, %xmm5
+; SSSE3-NEXT:    pxor %xmm0, %xmm5
+; SSSE3-NEXT:    psubd %xmm6, %xmm2
+; SSSE3-NEXT:    por %xmm0, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm6
+; SSSE3-NEXT:    movdqa %xmm1, %xmm5
+; SSSE3-NEXT:    pxor %xmm0, %xmm5
+; SSSE3-NEXT:    por %xmm8, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm0
+; SSSE3-NEXT:    packssdw %xmm6, %xmm0
+; SSSE3-NEXT:    packsswb %xmm10, %xmm0
+; SSSE3-NEXT:    psubd %xmm8, %xmm1
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
 ; SSSE3-NEXT:    pand %xmm5, %xmm4
 ; SSSE3-NEXT:    pand %xmm5, %xmm3
@@ -724,27 +713,21 @@ define <16 x i8> @test14(<16 x i8> %x, <16 x i32> %y) nounwind {
 ; SSE41-NEXT:    pcmpeqd %xmm4, %xmm0
 ; SSE41-NEXT:    pcmpeqd %xmm6, %xmm6
 ; SSE41-NEXT:    pxor %xmm6, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm7 = <u,u,u,u,0,4,8,12,u,u,u,u,u,u,u,u>
-; SSE41-NEXT:    pshufb %xmm7, %xmm0
-; SSE41-NEXT:    movdqa %xmm3, %xmm5
-; SSE41-NEXT:    pmaxud %xmm9, %xmm5
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm5
-; SSE41-NEXT:    pxor %xmm6, %xmm5
-; SSE41-NEXT:    pshufb %xmm7, %xmm5
-; SSE41-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
+; SSE41-NEXT:    movdqa %xmm3, %xmm7
+; SSE41-NEXT:    pmaxud %xmm9, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm7
+; SSE41-NEXT:    pxor %xmm6, %xmm7
+; SSE41-NEXT:    packssdw %xmm0, %xmm7
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    pmaxud %xmm8, %xmm0
 ; SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
 ; SSE41-NEXT:    pxor %xmm6, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm12 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
-; SSE41-NEXT:    pshufb %xmm12, %xmm0
-; SSE41-NEXT:    movdqa %xmm2, %xmm7
-; SSE41-NEXT:    pmaxud %xmm11, %xmm7
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm7
-; SSE41-NEXT:    pxor %xmm6, %xmm7
-; SSE41-NEXT:    pshufb %xmm12, %xmm7
-; SSE41-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[1],xmm7[1]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm5[4,5,6,7]
+; SSE41-NEXT:    movdqa %xmm2, %xmm5
+; SSE41-NEXT:    pmaxud %xmm11, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm5
+; SSE41-NEXT:    pxor %xmm6, %xmm5
+; SSE41-NEXT:    packssdw %xmm5, %xmm0
+; SSE41-NEXT:    packsswb %xmm7, %xmm0
 ; SSE41-NEXT:    psubd %xmm11, %xmm2
 ; SSE41-NEXT:    psubd %xmm8, %xmm1
 ; SSE41-NEXT:    psubd %xmm9, %xmm3
diff --git a/llvm/test/CodeGen/X86/rotate_vec.ll b/llvm/test/CodeGen/X86/rotate_vec.ll
index d2d6462486160..fbaf2d0f09141 100644
--- a/llvm/test/CodeGen/X86/rotate_vec.ll
+++ b/llvm/test/CodeGen/X86/rotate_vec.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=bdver4 | FileCheck %s --check-prefixes=CHECK,XOP
+; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=bdver2 | FileCheck %s --check-prefixes=CHECK,XOP,XOPAVX1
+; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=bdver4 | FileCheck %s --check-prefixes=CHECK,XOP,XOPAVX2
 ; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 | FileCheck %s --check-prefixes=CHECK,AVX512
 
 define <4 x i32> @rot_v4i32_splat(<4 x i32> %x) {
@@ -77,10 +78,20 @@ define <4 x i32> @rot_v4i32_non_splat_2masks(<4 x i32> %x) {
 }
 
 define <4 x i32> @rot_v4i32_zero_non_splat(<4 x i32> %x) {
-; CHECK-LABEL: rot_v4i32_zero_non_splat:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vbroadcastss %xmm0, %xmm0
-; CHECK-NEXT:    retq
+; XOPAVX1-LABEL: rot_v4i32_zero_non_splat:
+; XOPAVX1:       # %bb.0:
+; XOPAVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; XOPAVX1-NEXT:    retq
+;
+; XOPAVX2-LABEL: rot_v4i32_zero_non_splat:
+; XOPAVX2:       # %bb.0:
+; XOPAVX2-NEXT:    vbroadcastss %xmm0, %xmm0
+; XOPAVX2-NEXT:    retq
+;
+; AVX512-LABEL: rot_v4i32_zero_non_splat:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vbroadcastss %xmm0, %xmm0
+; AVX512-NEXT:    retq
   %1 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x, <4 x i32> %x, <4 x i32> <i32 0, i32 1, i32 2, i32 3>)
   %2 = shufflevector <4 x i32> %1, <4 x i32> undef, <4 x i32> zeroinitializer
   ret <4 x i32> %2
@@ -97,12 +108,19 @@ define <4 x i32> @rot_v4i32_allsignbits(<4 x i32> %x, <4 x i32> %y) {
 }
 
 define <4 x i32> @rot_v4i32_mask_ashr0(<4 x i32> %a0) {
-; XOP-LABEL: rot_v4i32_mask_ashr0:
-; XOP:       # %bb.0:
-; XOP-NEXT:    vpsravd {{.*}}(%rip), %xmm0, %xmm0
-; XOP-NEXT:    vprotd $1, %xmm0, %xmm0
-; XOP-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; XOP-NEXT:    retq
+; XOPAVX1-LABEL: rot_v4i32_mask_ashr0:
+; XOPAVX1:       # %bb.0:
+; XOPAVX1-NEXT:    vpshad {{.*}}(%rip), %xmm0, %xmm0
+; XOPAVX1-NEXT:    vprotd $1, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; XOPAVX1-NEXT:    retq
+;
+; XOPAVX2-LABEL: rot_v4i32_mask_ashr0:
+; XOPAVX2:       # %bb.0:
+; XOPAVX2-NEXT:    vpsravd {{.*}}(%rip), %xmm0, %xmm0
+; XOPAVX2-NEXT:    vprotd $1, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; XOPAVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: rot_v4i32_mask_ashr0:
 ; AVX512:       # %bb.0:
@@ -118,13 +136,21 @@ define <4 x i32> @rot_v4i32_mask_ashr0(<4 x i32> %a0) {
 }
 
 define <4 x i32> @rot_v4i32_mask_ashr1(<4 x i32> %a0) {
-; XOP-LABEL: rot_v4i32_mask_ashr1:
-; XOP:       # %bb.0:
-; XOP-NEXT:    vpsrad $25, %xmm0, %xmm0
-; XOP-NEXT:    vprotd $1, %xmm0, %xmm0
-; XOP-NEXT:    vpbroadcastd %xmm0, %xmm0
-; XOP-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; XOP-NEXT:    retq
+; XOPAVX1-LABEL: rot_v4i32_mask_ashr1:
+; XOPAVX1:       # %bb.0:
+; XOPAVX1-NEXT:    vpsrad $25, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vprotd $1, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; XOPAVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; XOPAVX1-NEXT:    retq
+;
+; XOPAVX2-LABEL: rot_v4i32_mask_ashr1:
+; XOPAVX2:       # %bb.0:
+; XOPAVX2-NEXT:    vpsrad $25, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vprotd $1, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vpbroadcastd %xmm0, %xmm0
+; XOPAVX2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; XOPAVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: rot_v4i32_mask_ashr1:
 ; AVX512:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/sad.ll b/llvm/test/CodeGen/X86/sad.ll
index 64845c847dff9..6a742068f7019 100644
--- a/llvm/test/CodeGen/X86/sad.ll
+++ b/llvm/test/CodeGen/X86/sad.ll
@@ -158,12 +158,12 @@ define i32 @sad_32i8() nounwind {
 ; SSE2-NEXT:    .p2align 4, 0x90
 ; SSE2-NEXT:  .LBB1_1: # %vector.body
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
-; SSE2-NEXT:    movdqa a+1040(%rax), %xmm3
-; SSE2-NEXT:    psadbw b+1040(%rax), %xmm3
-; SSE2-NEXT:    paddd %xmm3, %xmm1
 ; SSE2-NEXT:    movdqa a+1024(%rax), %xmm3
 ; SSE2-NEXT:    psadbw b+1024(%rax), %xmm3
 ; SSE2-NEXT:    paddd %xmm3, %xmm2
+; SSE2-NEXT:    movdqa a+1040(%rax), %xmm3
+; SSE2-NEXT:    psadbw b+1040(%rax), %xmm3
+; SSE2-NEXT:    paddd %xmm3, %xmm1
 ; SSE2-NEXT:    addq $4, %rax
 ; SSE2-NEXT:    jne .LBB1_1
 ; SSE2-NEXT:  # %bb.2: # %middle.block
@@ -188,14 +188,14 @@ define i32 @sad_32i8() nounwind {
 ; AVX1-NEXT:    .p2align 4, 0x90
 ; AVX1-NEXT:  .LBB1_1: # %vector.body
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
-; AVX1-NEXT:    vmovdqa a+1040(%rax), %xmm2
-; AVX1-NEXT:    vpsadbw b+1040(%rax), %xmm2, %xmm2
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vmovdqa a+1024(%rax), %xmm3
-; AVX1-NEXT:    vpsadbw b+1024(%rax), %xmm3, %xmm3
-; AVX1-NEXT:    vpaddd %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    vmovdqa a+1024(%rax), %xmm2
+; AVX1-NEXT:    vpsadbw b+1024(%rax), %xmm2, %xmm2
+; AVX1-NEXT:    vmovdqa a+1040(%rax), %xmm3
+; AVX1-NEXT:    vpsadbw b+1040(%rax), %xmm3, %xmm3
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vpaddd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
 ; AVX1-NEXT:    addq $4, %rax
 ; AVX1-NEXT:    jne .LBB1_1
 ; AVX1-NEXT:  # %bb.2: # %middle.block
@@ -320,15 +320,15 @@ define i32 @sad_avx64i8() nounwind {
 ; SSE2-NEXT:    .p2align 4, 0x90
 ; SSE2-NEXT:  .LBB2_1: # %vector.body
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
-; SSE2-NEXT:    movdqa a+1056(%rax), %xmm5
-; SSE2-NEXT:    psadbw b+1056(%rax), %xmm5
-; SSE2-NEXT:    paddd %xmm5, %xmm2
-; SSE2-NEXT:    movdqa a+1040(%rax), %xmm5
-; SSE2-NEXT:    psadbw b+1040(%rax), %xmm5
-; SSE2-NEXT:    paddd %xmm5, %xmm3
 ; SSE2-NEXT:    movdqa a+1024(%rax), %xmm5
 ; SSE2-NEXT:    psadbw b+1024(%rax), %xmm5
 ; SSE2-NEXT:    paddd %xmm5, %xmm0
+; SSE2-NEXT:    movdqa a+1040(%rax), %xmm5
+; SSE2-NEXT:    psadbw b+1040(%rax), %xmm5
+; SSE2-NEXT:    paddd %xmm5, %xmm3
+; SSE2-NEXT:    movdqa a+1056(%rax), %xmm5
+; SSE2-NEXT:    psadbw b+1056(%rax), %xmm5
+; SSE2-NEXT:    paddd %xmm5, %xmm2
 ; SSE2-NEXT:    movdqa a+1072(%rax), %xmm5
 ; SSE2-NEXT:    psadbw b+1072(%rax), %xmm5
 ; SSE2-NEXT:    paddd %xmm5, %xmm1
@@ -364,22 +364,22 @@ define i32 @sad_avx64i8() nounwind {
 ; AVX1-NEXT:    .p2align 4, 0x90
 ; AVX1-NEXT:  .LBB2_1: # %vector.body
 ; AVX1-NEXT:    # =>This Inner Loop Header: Depth=1
-; AVX1-NEXT:    vmovdqa a+1072(%rax), %xmm3
-; AVX1-NEXT:    vpsadbw b+1072(%rax), %xmm3, %xmm3
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
-; AVX1-NEXT:    vpaddd %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vmovdqa a+1056(%rax), %xmm4
-; AVX1-NEXT:    vpsadbw b+1056(%rax), %xmm4, %xmm4
-; AVX1-NEXT:    vpaddd %xmm2, %xmm4, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
-; AVX1-NEXT:    vmovdqa a+1040(%rax), %xmm3
-; AVX1-NEXT:    vpsadbw b+1040(%rax), %xmm3, %xmm3
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
-; AVX1-NEXT:    vpaddd %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vmovdqa a+1024(%rax), %xmm4
-; AVX1-NEXT:    vpsadbw b+1024(%rax), %xmm4, %xmm4
-; AVX1-NEXT:    vpaddd %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; AVX1-NEXT:    vmovdqa a+1024(%rax), %xmm3
+; AVX1-NEXT:    vpsadbw b+1024(%rax), %xmm3, %xmm3
+; AVX1-NEXT:    vmovdqa a+1040(%rax), %xmm4
+; AVX1-NEXT:    vpsadbw b+1040(%rax), %xmm4, %xmm4
+; AVX1-NEXT:    vmovdqa a+1056(%rax), %xmm5
+; AVX1-NEXT:    vpsadbw b+1056(%rax), %xmm5, %xmm5
+; AVX1-NEXT:    vmovdqa a+1072(%rax), %xmm6
+; AVX1-NEXT:    vpsadbw b+1072(%rax), %xmm6, %xmm6
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm7
+; AVX1-NEXT:    vpaddd %xmm7, %xmm6, %xmm6
+; AVX1-NEXT:    vpaddd %xmm2, %xmm5, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm2, %ymm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
+; AVX1-NEXT:    vpaddd %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpaddd %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
 ; AVX1-NEXT:    addq $4, %rax
 ; AVX1-NEXT:    jne .LBB2_1
 ; AVX1-NEXT:  # %bb.2: # %middle.block
@@ -416,12 +416,12 @@ define i32 @sad_avx64i8() nounwind {
 ; AVX2-NEXT:    .p2align 4, 0x90
 ; AVX2-NEXT:  .LBB2_1: # %vector.body
 ; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
-; AVX2-NEXT:    vmovdqa a+1056(%rax), %ymm3
-; AVX2-NEXT:    vpsadbw b+1056(%rax), %ymm3, %ymm3
-; AVX2-NEXT:    vpaddd %ymm2, %ymm3, %ymm2
 ; AVX2-NEXT:    vmovdqa a+1024(%rax), %ymm3
 ; AVX2-NEXT:    vpsadbw b+1024(%rax), %ymm3, %ymm3
 ; AVX2-NEXT:    vpaddd %ymm1, %ymm3, %ymm1
+; AVX2-NEXT:    vmovdqa a+1056(%rax), %ymm3
+; AVX2-NEXT:    vpsadbw b+1056(%rax), %ymm3, %ymm3
+; AVX2-NEXT:    vpaddd %ymm2, %ymm3, %ymm2
 ; AVX2-NEXT:    addq $4, %rax
 ; AVX2-NEXT:    jne .LBB2_1
 ; AVX2-NEXT:  # %bb.2: # %middle.block
@@ -449,11 +449,11 @@ define i32 @sad_avx64i8() nounwind {
 ; AVX512F-NEXT:    .p2align 4, 0x90
 ; AVX512F-NEXT:  .LBB2_1: # %vector.body
 ; AVX512F-NEXT:    # =>This Inner Loop Header: Depth=1
-; AVX512F-NEXT:    vmovdqa a+1056(%rax), %ymm2
-; AVX512F-NEXT:    vpsadbw b+1056(%rax), %ymm2, %ymm2
-; AVX512F-NEXT:    vmovdqa a+1024(%rax), %ymm3
-; AVX512F-NEXT:    vpsadbw b+1024(%rax), %ymm3, %ymm3
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
+; AVX512F-NEXT:    vmovdqa a+1024(%rax), %ymm2
+; AVX512F-NEXT:    vpsadbw b+1024(%rax), %ymm2, %ymm2
+; AVX512F-NEXT:    vmovdqa a+1056(%rax), %ymm3
+; AVX512F-NEXT:    vpsadbw b+1056(%rax), %ymm3, %ymm3
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm2
 ; AVX512F-NEXT:    vpaddd %zmm1, %zmm2, %zmm1
 ; AVX512F-NEXT:    addq $4, %rax
 ; AVX512F-NEXT:    jne .LBB2_1
@@ -554,10 +554,10 @@ define i32 @sad_2i8() nounwind {
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; SSE2-NEXT:    pand %xmm1, %xmm3
 ; SSE2-NEXT:    pand %xmm1, %xmm2
-; SSE2-NEXT:    psadbw %xmm3, %xmm2
-; SSE2-NEXT:    paddd %xmm2, %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm3
+; SSE2-NEXT:    psadbw %xmm2, %xmm3
+; SSE2-NEXT:    paddd %xmm3, %xmm0
 ; SSE2-NEXT:    addq $4, %rax
 ; SSE2-NEXT:    jne .LBB3_1
 ; SSE2-NEXT:  # %bb.2: # %middle.block
@@ -576,8 +576,8 @@ define i32 @sad_2i8() nounwind {
 ; AVX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; AVX-NEXT:    vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; AVX-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0],xmm0[1,2,3,4,5,6,7]
 ; AVX-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm0[1,2,3,4,5,6,7]
+; AVX-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0],xmm0[1,2,3,4,5,6,7]
 ; AVX-NEXT:    vpsadbw %xmm3, %xmm2, %xmm2
 ; AVX-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
 ; AVX-NEXT:    addq $4, %rax
@@ -649,7 +649,7 @@ define i32 @sad_4i8() nounwind {
 ; AVX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; AVX-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; AVX-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX-NEXT:    vpsadbw %xmm1, %xmm2, %xmm1
+; AVX-NEXT:    vpsadbw %xmm2, %xmm1, %xmm1
 ; AVX-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    addq $4, %rax
 ; AVX-NEXT:    jne .LBB4_1
@@ -987,75 +987,36 @@ define i32 @sad_unroll_nonzero_initial(<16 x i8>* %arg, <16 x i8>* %arg1, <16 x
 ; SSE2-NEXT:    movdqu (%rdi), %xmm0
 ; SSE2-NEXT:    movdqu (%rsi), %xmm1
 ; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    movdqu (%rdx), %xmm0
+; SSE2-NEXT:    movdqu (%rcx), %xmm2
+; SSE2-NEXT:    psadbw %xmm0, %xmm2
 ; SSE2-NEXT:    movl $1, %eax
 ; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    movdqu (%rdx), %xmm2
-; SSE2-NEXT:    movdqu (%rcx), %xmm3
-; SSE2-NEXT:    psadbw %xmm2, %xmm3
-; SSE2-NEXT:    paddd %xmm0, %xmm3
-; SSE2-NEXT:    paddd %xmm1, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,0,1]
-; SSE2-NEXT:    paddd %xmm3, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE2-NEXT:    paddd %xmm2, %xmm0
+; SSE2-NEXT:    paddd %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; SSE2-NEXT:    paddd %xmm0, %xmm1
-; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    paddd %xmm1, %xmm0
+; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    retq
 ;
-; AVX1-LABEL: sad_unroll_nonzero_initial:
-; AVX1:       # %bb.0: # %bb
-; AVX1-NEXT:    vmovdqu (%rdi), %xmm0
-; AVX1-NEXT:    vpsadbw (%rsi), %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqu (%rdx), %xmm1
-; AVX1-NEXT:    vpsadbw (%rcx), %xmm1, %xmm1
-; AVX1-NEXT:    movl $1, %eax
-; AVX1-NEXT:    vmovd %eax, %xmm2
-; AVX1-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovd %xmm0, %eax
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: sad_unroll_nonzero_initial:
-; AVX2:       # %bb.0: # %bb
-; AVX2-NEXT:    vmovdqu (%rdi), %xmm0
-; AVX2-NEXT:    vpsadbw (%rsi), %xmm0, %xmm0
-; AVX2-NEXT:    movl $1, %eax
-; AVX2-NEXT:    vmovd %eax, %xmm1
-; AVX2-NEXT:    vmovdqu (%rdx), %xmm2
-; AVX2-NEXT:    vpsadbw (%rcx), %xmm2, %xmm2
-; AVX2-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: sad_unroll_nonzero_initial:
-; AVX512:       # %bb.0: # %bb
-; AVX512-NEXT:    vmovdqu (%rdi), %xmm0
-; AVX512-NEXT:    vpsadbw (%rsi), %xmm0, %xmm0
-; AVX512-NEXT:    movl $1, %eax
-; AVX512-NEXT:    vmovd %eax, %xmm1
-; AVX512-NEXT:    vmovdqu (%rdx), %xmm2
-; AVX512-NEXT:    vpsadbw (%rcx), %xmm2, %xmm2
-; AVX512-NEXT:    vpaddd %zmm2, %zmm1, %zmm1
-; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX-LABEL: sad_unroll_nonzero_initial:
+; AVX:       # %bb.0: # %bb
+; AVX-NEXT:    vmovdqu (%rdi), %xmm0
+; AVX-NEXT:    vpsadbw (%rsi), %xmm0, %xmm0
+; AVX-NEXT:    vmovdqu (%rdx), %xmm1
+; AVX-NEXT:    vpsadbw (%rcx), %xmm1, %xmm1
+; AVX-NEXT:    movl $1, %eax
+; AVX-NEXT:    vmovd %eax, %xmm2
+; AVX-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    retq
 bb:
   %tmp = load <16 x i8>, <16 x i8>* %arg, align 1
   %tmp4 = load <16 x i8>, <16 x i8>* %arg1, align 1
@@ -1112,7 +1073,7 @@ define i32 @sad_double_reduction(<16 x i8>* %arg, <16 x i8>* %arg1, <16 x i8>* %
 ; AVX-NEXT:    vmovdqu (%rdx), %xmm1
 ; AVX-NEXT:    vpsadbw (%rcx), %xmm1, %xmm1
 ; AVX-NEXT:    vpsadbw (%rsi), %xmm0, %xmm0
-; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
diff --git a/llvm/test/CodeGen/X86/serialize-intrinsic.ll b/llvm/test/CodeGen/X86/serialize-intrinsic.ll
new file mode 100644
index 0000000000000..629bb6dabd289
--- /dev/null
+++ b/llvm/test/CodeGen/X86/serialize-intrinsic.ll
@@ -0,0 +1,26 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+serialize | FileCheck %s --check-prefix=X86_64
+; RUN: llc < %s -mtriple=i386-unknown-unknown -mattr=+serialize | FileCheck %s --check-prefix=X86
+; RUN: llc < %s -mtriple=x86_64-linux-gnux32 -mattr=+serialize | FileCheck %s --check-prefix=X32
+
+define void @test_serialize() {
+; X86_64-LABEL: test_serialize:
+; X86_64:       # %bb.0: # %entry
+; X86_64-NEXT:    serialize
+; X86_64-NEXT:    retq
+;
+; X86-LABEL: test_serialize:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    serialize
+; X86-NEXT:    retl
+;
+; X32-LABEL: test_serialize:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    serialize
+; X32-NEXT:    retq
+entry:
+  call void @llvm.x86.serialize()
+  ret void
+}
+
+declare void @llvm.x86.serialize()
diff --git a/llvm/test/CodeGen/X86/shuffle-combine-crash-3.ll b/llvm/test/CodeGen/X86/shuffle-combine-crash-3.ll
new file mode 100644
index 0000000000000..7ddc0f1b2e4ac
--- /dev/null
+++ b/llvm/test/CodeGen/X86/shuffle-combine-crash-3.ll
@@ -0,0 +1,31 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s
+
+; Verify that we don't crash when compiling this. We used to hit an
+; assert like this
+;
+;   llc: ../include/llvm/CodeGen/ValueTypes.h:251: llvm::MVT llvm::EVT::getSimpleVT() const: Assertion `isSimple() && "Expected a SimpleValueType!"' failed.
+;
+; due to getFauxShuffleMask not checking that the VT was simple before a call
+; to getSimpleValueType().
+
+define i1 @dont_hit_assert(i24 signext %d) {
+; CHECK-LABEL: dont_hit_assert:
+; CHECK:       # %bb.0: # %for.cond
+; CHECK-NEXT:    movb $-1, %al
+; CHECK-NEXT:    negb %al
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    retq
+for.cond:
+  %t0 = insertelement <8 x i24> zeroinitializer, i24 1, i32 0
+  %t5 = icmp slt <8 x i24> %t0, zeroinitializer
+  %t7 = icmp slt i24 0, %d
+  %rdx.shuf = shufflevector <8 x i1> %t5, <8 x i1> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+  %bin.rdx = and <8 x i1> %t5, %rdx.shuf
+  %rdx.shuf22 = shufflevector <8 x i1> %bin.rdx, <8 x i1> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %bin.rdx23 = and <8 x i1> %bin.rdx, %rdx.shuf22
+  %rdx.shuf24 = shufflevector <8 x i1> %bin.rdx23, <8 x i1> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %bin.rdx25 = and <8 x i1> %bin.rdx23, %rdx.shuf24
+  %t8 = extractelement <8 x i1> %bin.rdx25, i32 0
+  ret i1 %t8
+}
diff --git a/llvm/test/CodeGen/X86/shuffle-vs-trunc-128.ll b/llvm/test/CodeGen/X86/shuffle-vs-trunc-128.ll
index 7805c46e5c343..87e22321a3fc6 100644
--- a/llvm/test/CodeGen/X86/shuffle-vs-trunc-128.ll
+++ b/llvm/test/CodeGen/X86/shuffle-vs-trunc-128.ll
@@ -37,12 +37,33 @@ define void @shuffle_v16i8_to_v8i8(<16 x i8>* %L, <8 x i8>* %S) nounwind {
 ; AVX-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: shuffle_v16i8_to_v8i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
-; AVX512-NEXT:    vmovq %xmm0, (%rsi)
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: shuffle_v16i8_to_v8i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vmovq %xmm0, (%rsi)
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: shuffle_v16i8_to_v8i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; AVX512VL-NEXT:    vmovq %xmm0, (%rsi)
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: shuffle_v16i8_to_v8i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT:    vmovq %xmm0, (%rsi)
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: shuffle_v16i8_to_v8i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512BWVL-NEXT:    vpmovwb %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vmovq %xmm0, (%rsi)
+; AVX512BWVL-NEXT:    retq
   %vec = load <16 x i8>, <16 x i8>* %L
   %strided.vec = shufflevector <16 x i8> %vec, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
   store <8 x i8> %strided.vec, <8 x i8>* %S
@@ -286,12 +307,33 @@ define void @shuffle_v16i8_to_v4i8(<16 x i8>* %L, <4 x i8>* %S) nounwind {
 ; AVX-NEXT:    vmovd %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: shuffle_v16i8_to_v4i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512-NEXT:    vmovd %xmm0, (%rsi)
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: shuffle_v16i8_to_v4i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vmovd %xmm0, (%rsi)
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: shuffle_v16i8_to_v4i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512VL-NEXT:    vpmovdb %xmm0, %xmm0
+; AVX512VL-NEXT:    vmovd %xmm0, (%rsi)
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: shuffle_v16i8_to_v4i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT:    vmovd %xmm0, (%rsi)
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: shuffle_v16i8_to_v4i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512BWVL-NEXT:    vpmovdb %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vmovd %xmm0, (%rsi)
+; AVX512BWVL-NEXT:    retq
   %vec = load <16 x i8>, <16 x i8>* %L
   %strided.vec = shufflevector <16 x i8> %vec, <16 x i8> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
   store <4 x i8> %strided.vec, <4 x i8>* %S
@@ -503,12 +545,33 @@ define void @shuffle_v16i8_to_v2i8(<16 x i8>* %L, <2 x i8>* %S) nounwind {
 ; AVX-NEXT:    vpextrw $0, %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: shuffle_v16i8_to_v2i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512-NEXT:    vpextrw $0, %xmm0, (%rsi)
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: shuffle_v16i8_to_v2i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rsi)
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: shuffle_v16i8_to_v2i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512VL-NEXT:    vpmovqb %xmm0, %xmm0
+; AVX512VL-NEXT:    vpextrw $0, %xmm0, (%rsi)
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: shuffle_v16i8_to_v2i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT:    vpextrw $0, %xmm0, (%rsi)
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: shuffle_v16i8_to_v2i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512BWVL-NEXT:    vpmovqb %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpextrw $0, %xmm0, (%rsi)
+; AVX512BWVL-NEXT:    retq
   %vec = load <16 x i8>, <16 x i8>* %L
   %strided.vec = shufflevector <16 x i8> %vec, <16 x i8> undef, <2 x i32> <i32 0, i32 8>
   store <2 x i8> %strided.vec, <2 x i8>* %S
diff --git a/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll b/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
index 01890e2adba93..5e074b4ab6db2 100644
--- a/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
+++ b/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
@@ -15,65 +15,52 @@
 define void @shuffle_v32i8_to_v16i8(<32 x i8>* %L, <16 x i8>* %S) nounwind {
 ; AVX-LABEL: shuffle_v32i8_to_v16i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; AVX-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
+; AVX-NEXT:    vpand 16(%rdi), %xmm0, %xmm1
+; AVX-NEXT:    vpand (%rdi), %xmm0, %xmm0
+; AVX-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovdqa %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
 ; AVX512F-LABEL: shuffle_v32i8_to_v16i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; AVX512F-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
+; AVX512F-NEXT:    vpand 16(%rdi), %xmm0, %xmm1
+; AVX512F-NEXT:    vpand (%rdi), %xmm0, %xmm0
+; AVX512F-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vmovdqa %xmm0, (%rsi)
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: shuffle_v32i8_to_v16i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
+; AVX512VL-NEXT:    vpand 16(%rdi), %xmm0, %xmm1
+; AVX512VL-NEXT:    vpand (%rdi), %xmm0, %xmm0
+; AVX512VL-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vmovdqa %xmm0, (%rsi)
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: shuffle_v32i8_to_v16i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
+; AVX512BW-NEXT:    vpand 16(%rdi), %xmm0, %xmm1
+; AVX512BW-NEXT:    vpand (%rdi), %xmm0, %xmm0
+; AVX512BW-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vmovdqa %xmm0, (%rsi)
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: shuffle_v32i8_to_v16i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; AVX512BWVL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512BWVL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512BWVL-NEXT:    vmovdqa %xmm0, (%rsi)
+; AVX512BWVL-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, (%rsi)
+; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
 ; AVX512VBMIVL-LABEL: shuffle_v32i8_to_v16i8:
 ; AVX512VBMIVL:       # %bb.0:
-; AVX512VBMIVL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VBMIVL-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30]
-; AVX512VBMIVL-NEXT:    vpermi2b 16(%rdi), %xmm0, %xmm1
-; AVX512VBMIVL-NEXT:    vmovdqa %xmm1, (%rsi)
+; AVX512VBMIVL-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512VBMIVL-NEXT:    vpmovwb %ymm0, (%rsi)
+; AVX512VBMIVL-NEXT:    vzeroupper
 ; AVX512VBMIVL-NEXT:    retq
   %vec = load <32 x i8>, <32 x i8>* %L
   %strided.vec = shufflevector <32 x i8> %vec, <32 x i8> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
@@ -169,13 +156,9 @@ define void @shuffle_v16i16_to_v8i16(<16 x i16>* %L, <8 x i16>* %S) nounwind {
 ;
 ; AVX512VL-LABEL: shuffle_v16i16_to_v8i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512VL-NEXT:    vmovdqa %xmm0, (%rsi)
+; AVX512VL-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512VL-NEXT:    vpmovdw %ymm0, (%rsi)
+; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: shuffle_v16i16_to_v8i16:
@@ -191,18 +174,16 @@ define void @shuffle_v16i16_to_v8i16(<16 x i16>* %L, <8 x i16>* %S) nounwind {
 ;
 ; AVX512BWVL-LABEL: shuffle_v16i16_to_v8i16:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,2,4,6,8,10,12,14]
-; AVX512BWVL-NEXT:    vpermi2w 16(%rdi), %xmm0, %xmm1
-; AVX512BWVL-NEXT:    vmovdqa %xmm1, (%rsi)
+; AVX512BWVL-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512BWVL-NEXT:    vpmovdw %ymm0, (%rsi)
+; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
 ; AVX512VBMIVL-LABEL: shuffle_v16i16_to_v8i16:
 ; AVX512VBMIVL:       # %bb.0:
-; AVX512VBMIVL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VBMIVL-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,2,4,6,8,10,12,14]
-; AVX512VBMIVL-NEXT:    vpermi2w 16(%rdi), %xmm0, %xmm1
-; AVX512VBMIVL-NEXT:    vmovdqa %xmm1, (%rsi)
+; AVX512VBMIVL-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512VBMIVL-NEXT:    vpmovdw %ymm0, (%rsi)
+; AVX512VBMIVL-NEXT:    vzeroupper
 ; AVX512VBMIVL-NEXT:    retq
   %vec = load <16 x i16>, <16 x i16>* %L
   %strided.vec = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
@@ -282,12 +263,40 @@ define void @shuffle_v8i32_to_v4i32(<8 x i32>* %L, <4 x i32>* %S) nounwind {
 ; AVX-NEXT:    vmovaps %xmm0, (%rsi)
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: shuffle_v8i32_to_v4i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovaps (%rdi), %xmm0
-; AVX512-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],mem[0,2]
-; AVX512-NEXT:    vmovaps %xmm0, (%rsi)
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: shuffle_v8i32_to_v4i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovaps (%rdi), %xmm0
+; AVX512F-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],mem[0,2]
+; AVX512F-NEXT:    vmovaps %xmm0, (%rsi)
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: shuffle_v8i32_to_v4i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512VL-NEXT:    vpmovqd %ymm0, (%rsi)
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: shuffle_v8i32_to_v4i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovaps (%rdi), %xmm0
+; AVX512BW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],mem[0,2]
+; AVX512BW-NEXT:    vmovaps %xmm0, (%rsi)
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: shuffle_v8i32_to_v4i32:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512BWVL-NEXT:    vpmovqd %ymm0, (%rsi)
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+;
+; AVX512VBMIVL-LABEL: shuffle_v8i32_to_v4i32:
+; AVX512VBMIVL:       # %bb.0:
+; AVX512VBMIVL-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512VBMIVL-NEXT:    vpmovqd %ymm0, (%rsi)
+; AVX512VBMIVL-NEXT:    vzeroupper
+; AVX512VBMIVL-NEXT:    retq
   %vec = load <8 x i32>, <8 x i32>* %L
   %strided.vec = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
   store <4 x i32> %strided.vec, <4 x i32>* %S
@@ -387,9 +396,8 @@ define void @shuffle_v32i8_to_v8i8(<32 x i8>* %L, <8 x i8>* %S) nounwind {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512VL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmovdb %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmovdb %xmm0, %xmm0
 ; AVX512VL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; AVX512VL-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX512VL-NEXT:    retq
@@ -409,9 +417,8 @@ define void @shuffle_v32i8_to_v8i8(<32 x i8>* %L, <8 x i8>* %S) nounwind {
 ; AVX512BWVL:       # %bb.0:
 ; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512BWVL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512BWVL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512BWVL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovdb %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmovdb %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; AVX512BWVL-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX512BWVL-NEXT:    retq
@@ -419,9 +426,11 @@ define void @shuffle_v32i8_to_v8i8(<32 x i8>* %L, <8 x i8>* %S) nounwind {
 ; AVX512VBMIVL-LABEL: shuffle_v32i8_to_v8i8:
 ; AVX512VBMIVL:       # %bb.0:
 ; AVX512VBMIVL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VBMIVL-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [2024390091656922112,2024390091656922112]
-; AVX512VBMIVL-NEXT:    vpermi2b 16(%rdi), %xmm0, %xmm1
-; AVX512VBMIVL-NEXT:    vmovq %xmm1, (%rsi)
+; AVX512VBMIVL-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX512VBMIVL-NEXT:    vpmovdb %xmm1, %xmm1
+; AVX512VBMIVL-NEXT:    vpmovdb %xmm0, %xmm0
+; AVX512VBMIVL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX512VBMIVL-NEXT:    vmovq %xmm0, (%rsi)
 ; AVX512VBMIVL-NEXT:    retq
   %vec = load <32 x i8>, <32 x i8>* %L
   %strided.vec = shufflevector <32 x i8> %vec, <32 x i8> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
@@ -1278,9 +1287,8 @@ define void @shuffle_v32i8_to_v4i8(<32 x i8>* %L, <4 x i8>* %S) nounwind {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512VL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmovqb %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmovqb %xmm0, %xmm0
 ; AVX512VL-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; AVX512VL-NEXT:    vmovd %xmm0, (%rsi)
 ; AVX512VL-NEXT:    retq
@@ -1300,9 +1308,8 @@ define void @shuffle_v32i8_to_v4i8(<32 x i8>* %L, <4 x i8>* %S) nounwind {
 ; AVX512BWVL:       # %bb.0:
 ; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512BWVL-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512BWVL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512BWVL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovqb %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmovqb %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; AVX512BWVL-NEXT:    vmovd %xmm0, (%rsi)
 ; AVX512BWVL-NEXT:    retq
@@ -1310,9 +1317,11 @@ define void @shuffle_v32i8_to_v4i8(<32 x i8>* %L, <4 x i8>* %S) nounwind {
 ; AVX512VBMIVL-LABEL: shuffle_v32i8_to_v4i8:
 ; AVX512VBMIVL:       # %bb.0:
 ; AVX512VBMIVL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VBMIVL-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [403703808,403703808,403703808,403703808]
-; AVX512VBMIVL-NEXT:    vpermi2b 16(%rdi), %xmm0, %xmm1
-; AVX512VBMIVL-NEXT:    vmovd %xmm1, (%rsi)
+; AVX512VBMIVL-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX512VBMIVL-NEXT:    vpmovqb %xmm1, %xmm1
+; AVX512VBMIVL-NEXT:    vpmovqb %xmm0, %xmm0
+; AVX512VBMIVL-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512VBMIVL-NEXT:    vmovd %xmm0, (%rsi)
 ; AVX512VBMIVL-NEXT:    retq
   %vec = load <32 x i8>, <32 x i8>* %L
   %strided.vec = shufflevector <32 x i8> %vec, <32 x i8> undef, <4 x i32> <i32 0, i32 8, i32 16, i32 24>
diff --git a/llvm/test/CodeGen/X86/shuffle-vs-trunc-512.ll b/llvm/test/CodeGen/X86/shuffle-vs-trunc-512.ll
index 7f00b49b81f85..d0e8dc2100518 100644
--- a/llvm/test/CodeGen/X86/shuffle-vs-trunc-512.ll
+++ b/llvm/test/CodeGen/X86/shuffle-vs-trunc-512.ll
@@ -37,46 +37,29 @@ define void @shuffle_v64i8_to_v32i8(<64 x i8>* %L, <32 x i8>* %S) nounwind {
 ;
 ; AVX512BW-LABEL: shuffle_v64i8_to_v32i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512BW-NEXT:    vmovdqa 32(%rdi), %ymm1
-; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30]
-; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; AVX512BW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512BW-NEXT:    vmovdqa %ymm0, (%rsi)
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, (%rsi)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512BWVL-LABEL: shuffle_v64i8_to_v32i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512BWVL-NEXT:    vmovdqa 32(%rdi), %ymm1
-; AVX512BWVL-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30]
-; AVX512BWVL-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,2,5,7]
-; AVX512BWVL-NEXT:    vpermi2q %ymm1, %ymm0, %ymm2
-; AVX512BWVL-NEXT:    vmovdqa %ymm2, (%rsi)
+; AVX512BWVL-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512BWVL-NEXT:    vpmovwb %zmm0, (%rsi)
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
 ; AVX512VBMI-LABEL: shuffle_v64i8_to_v32i8:
 ; AVX512VBMI:       # %bb.0:
-; AVX512VBMI-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512VBMI-NEXT:    vmovdqa 32(%rdi), %ymm1
-; AVX512VBMI-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30]
-; AVX512VBMI-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
-; AVX512VBMI-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; AVX512VBMI-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512VBMI-NEXT:    vmovdqa %ymm0, (%rsi)
+; AVX512VBMI-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512VBMI-NEXT:    vpmovwb %zmm0, (%rsi)
 ; AVX512VBMI-NEXT:    vzeroupper
 ; AVX512VBMI-NEXT:    retq
 ;
 ; AVX512VBMIVL-LABEL: shuffle_v64i8_to_v32i8:
 ; AVX512VBMIVL:       # %bb.0:
-; AVX512VBMIVL-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512VBMIVL-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62]
-; AVX512VBMIVL-NEXT:    vpermi2b 32(%rdi), %ymm0, %ymm1
-; AVX512VBMIVL-NEXT:    vmovdqa %ymm1, (%rsi)
+; AVX512VBMIVL-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512VBMIVL-NEXT:    vpmovwb %zmm0, (%rsi)
 ; AVX512VBMIVL-NEXT:    vzeroupper
 ; AVX512VBMIVL-NEXT:    retq
   %vec = load <64 x i8>, <64 x i8>* %L
@@ -139,74 +122,12 @@ define void @trunc_v32i16_to_v32i8(<64 x i8>* %L, <32 x i8>* %S) nounwind {
 }
 
 define void @shuffle_v32i16_to_v16i16(<32 x i16>* %L, <16 x i16>* %S) nounwind {
-; AVX512F-LABEL: shuffle_v32i16_to_v16i16:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpshuflw {{.*#+}} ymm0 = mem[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
-; AVX512F-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
-; AVX512F-NEXT:    vpshuflw {{.*#+}} ymm1 = mem[0,2,2,3,4,5,6,7,8,10,10,11,12,13,14,15]
-; AVX512F-NEXT:    vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,6,6,7,8,9,10,11,12,14,14,15]
-; AVX512F-NEXT:    vshufps {{.*#+}} ymm0 = ymm1[0,2],ymm0[0,2],ymm1[4,6],ymm0[4,6]
-; AVX512F-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512F-NEXT:    vmovaps %ymm0, (%rsi)
-; AVX512F-NEXT:    vzeroupper
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v32i16_to_v16i16:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512VL-NEXT:    vmovdqa 32(%rdi), %ymm1
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,4,5,4,5,6,7,8,9,12,13,12,13,14,15,16,17,20,21,20,21,22,23,24,25,28,29,28,29,30,31]
-; AVX512VL-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,2,4,6,8,10,12,14]
-; AVX512VL-NEXT:    vpermi2d %ymm1, %ymm0, %ymm2
-; AVX512VL-NEXT:    vmovdqa %ymm2, (%rsi)
-; AVX512VL-NEXT:    vzeroupper
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v32i16_to_v16i16:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512BW-NEXT:    vmovdqa 32(%rdi), %ymm1
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,4,5,4,5,6,7,8,9,12,13,12,13,14,15,16,17,20,21,20,21,22,23,24,25,28,29,28,29,30,31]
-; AVX512BW-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX512BW-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
-; AVX512BW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm1[0,2],ymm0[4,6],ymm1[4,6]
-; AVX512BW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512BW-NEXT:    vmovaps %ymm0, (%rsi)
-; AVX512BW-NEXT:    vzeroupper
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v32i16_to_v16i16:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30]
-; AVX512BWVL-NEXT:    vpermi2w 32(%rdi), %ymm0, %ymm1
-; AVX512BWVL-NEXT:    vmovdqa %ymm1, (%rsi)
-; AVX512BWVL-NEXT:    vzeroupper
-; AVX512BWVL-NEXT:    retq
-;
-; AVX512VBMI-LABEL: shuffle_v32i16_to_v16i16:
-; AVX512VBMI:       # %bb.0:
-; AVX512VBMI-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512VBMI-NEXT:    vmovdqa 32(%rdi), %ymm1
-; AVX512VBMI-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,4,5,4,5,6,7,8,9,12,13,12,13,14,15,16,17,20,21,20,21,22,23,24,25,28,29,28,29,30,31]
-; AVX512VBMI-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX512VBMI-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
-; AVX512VBMI-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm1[0,2],ymm0[4,6],ymm1[4,6]
-; AVX512VBMI-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512VBMI-NEXT:    vmovaps %ymm0, (%rsi)
-; AVX512VBMI-NEXT:    vzeroupper
-; AVX512VBMI-NEXT:    retq
-;
-; AVX512VBMIVL-LABEL: shuffle_v32i16_to_v16i16:
-; AVX512VBMIVL:       # %bb.0:
-; AVX512VBMIVL-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512VBMIVL-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30]
-; AVX512VBMIVL-NEXT:    vpermi2w 32(%rdi), %ymm0, %ymm1
-; AVX512VBMIVL-NEXT:    vmovdqa %ymm1, (%rsi)
-; AVX512VBMIVL-NEXT:    vzeroupper
-; AVX512VBMIVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v32i16_to_v16i16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512-NEXT:    vpmovdw %zmm0, (%rsi)
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
   %vec = load <32 x i16>, <32 x i16>* %L
   %strided.vec = shufflevector <32 x i16> %vec, <32 x i16> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
   store <16 x i16> %strided.vec, <16 x i16>* %S
@@ -228,59 +149,12 @@ define void @trunc_v16i32_to_v16i16(<32 x i16>* %L, <16 x i16>* %S) nounwind {
 }
 
 define void @shuffle_v16i32_to_v8i32(<16 x i32>* %L, <8 x i32>* %S) nounwind {
-; AVX512F-LABEL: shuffle_v16i32_to_v8i32:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovaps (%rdi), %ymm0
-; AVX512F-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],mem[0,2],ymm0[4,6],mem[4,6]
-; AVX512F-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512F-NEXT:    vmovaps %ymm0, (%rsi)
-; AVX512F-NEXT:    vzeroupper
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: shuffle_v16i32_to_v8i32:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,2,4,6,8,10,12,14]
-; AVX512VL-NEXT:    vpermi2d 32(%rdi), %ymm0, %ymm1
-; AVX512VL-NEXT:    vmovdqa %ymm1, (%rsi)
-; AVX512VL-NEXT:    vzeroupper
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: shuffle_v16i32_to_v8i32:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovaps (%rdi), %ymm0
-; AVX512BW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],mem[0,2],ymm0[4,6],mem[4,6]
-; AVX512BW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512BW-NEXT:    vmovaps %ymm0, (%rsi)
-; AVX512BW-NEXT:    vzeroupper
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: shuffle_v16i32_to_v8i32:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,2,4,6,8,10,12,14]
-; AVX512BWVL-NEXT:    vpermi2d 32(%rdi), %ymm0, %ymm1
-; AVX512BWVL-NEXT:    vmovdqa %ymm1, (%rsi)
-; AVX512BWVL-NEXT:    vzeroupper
-; AVX512BWVL-NEXT:    retq
-;
-; AVX512VBMI-LABEL: shuffle_v16i32_to_v8i32:
-; AVX512VBMI:       # %bb.0:
-; AVX512VBMI-NEXT:    vmovaps (%rdi), %ymm0
-; AVX512VBMI-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2],mem[0,2],ymm0[4,6],mem[4,6]
-; AVX512VBMI-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512VBMI-NEXT:    vmovaps %ymm0, (%rsi)
-; AVX512VBMI-NEXT:    vzeroupper
-; AVX512VBMI-NEXT:    retq
-;
-; AVX512VBMIVL-LABEL: shuffle_v16i32_to_v8i32:
-; AVX512VBMIVL:       # %bb.0:
-; AVX512VBMIVL-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512VBMIVL-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,2,4,6,8,10,12,14]
-; AVX512VBMIVL-NEXT:    vpermi2d 32(%rdi), %ymm0, %ymm1
-; AVX512VBMIVL-NEXT:    vmovdqa %ymm1, (%rsi)
-; AVX512VBMIVL-NEXT:    vzeroupper
-; AVX512VBMIVL-NEXT:    retq
+; AVX512-LABEL: shuffle_v16i32_to_v8i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512-NEXT:    vpmovqd %zmm0, (%rsi)
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
   %vec = load <16 x i32>, <16 x i32>* %L
   %strided.vec = shufflevector <16 x i32> %vec, <16 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
   store <8 x i32> %strided.vec, <8 x i32>* %S
@@ -322,20 +196,17 @@ define void @shuffle_v64i8_to_v16i8(<64 x i8>* %L, <16 x i8>* %S) nounwind {
 ;
 ; AVX512VL-LABEL: shuffle_v64i8_to_v16i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX512VL-NEXT:    vmovdqa 48(%rdi), %xmm0
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm1 = <u,u,u,u,0,4,8,12,u,u,u,u,u,u,u,u>
+; AVX512VL-NEXT:    vpshufb %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512VL-NEXT:    vmovdqa 48(%rdi), %xmm3
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,u,u,0,4,8,12,u,u,u,u,u,u,u,u>
-; AVX512VL-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX512VL-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512VL-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3]
+; AVX512VL-NEXT:    vpshufb %xmm1, %xmm2, %xmm1
+; AVX512VL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX512VL-NEXT:    vmovdqa (%rdi), %ymm1
+; AVX512VL-NEXT:    vpmovdb %ymm1, %xmm1
+; AVX512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
 ; AVX512VL-NEXT:    vmovdqa %xmm0, (%rsi)
+; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: shuffle_v64i8_to_v16i8:
@@ -358,20 +229,17 @@ define void @shuffle_v64i8_to_v16i8(<64 x i8>* %L, <16 x i8>* %S) nounwind {
 ;
 ; AVX512BWVL-LABEL: shuffle_v64i8_to_v16i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BWVL-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX512BWVL-NEXT:    vmovdqa 48(%rdi), %xmm0
+; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm1 = <u,u,u,u,0,4,8,12,u,u,u,u,u,u,u,u>
+; AVX512BWVL-NEXT:    vpshufb %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX512BWVL-NEXT:    vmovdqa 48(%rdi), %xmm3
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm4 = <u,u,u,u,0,4,8,12,u,u,u,u,u,u,u,u>
-; AVX512BWVL-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX512BWVL-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512BWVL-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm3 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512BWVL-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX512BWVL-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX512BWVL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3]
+; AVX512BWVL-NEXT:    vpshufb %xmm1, %xmm2, %xmm1
+; AVX512BWVL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX512BWVL-NEXT:    vmovdqa (%rdi), %ymm1
+; AVX512BWVL-NEXT:    vpmovdb %ymm1, %xmm1
+; AVX512BWVL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
 ; AVX512BWVL-NEXT:    vmovdqa %xmm0, (%rsi)
+; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
 ; AVX512VBMI-LABEL: shuffle_v64i8_to_v16i8:
diff --git a/llvm/test/CodeGen/X86/sink-local-value.ll b/llvm/test/CodeGen/X86/sink-local-value.ll
index b0e511ac1189e..f7d861ac9b6ca 100644
--- a/llvm/test/CodeGen/X86/sink-local-value.ll
+++ b/llvm/test/CodeGen/X86/sink-local-value.ll
@@ -145,6 +145,42 @@ try.cont:                                         ; preds = %entry, %lpad
 ; CHECK:         retl
 
 
+define i32 @lpad_phi() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  store i32 42, i32* @sink_across
+  invoke void @may_throw()
+          to label %try.cont unwind label %lpad
+
+lpad:                                             ; preds = %entry
+  %p = phi i32 [ 11, %entry ]  ; Trivial, but -O0 keeps it
+  %0 = landingpad { i8*, i32 }
+          catch i8* null
+  store i32 %p, i32* @sink_across
+  br label %try.cont
+
+try.cont:                                         ; preds = %entry, %lpad
+  %r.0 = phi i32 [ 13, %entry ], [ 55, %lpad ]
+  ret i32 %r.0
+}
+
+; The constant materialization should be *after* the stores to sink_across, but
+; before any EH_LABEL.
+
+; CHECK-LABEL: lpad_phi:
+; CHECK:         movl    $42, sink_across
+; CHECK:         movl    $13, %{{[a-z]*}}
+; CHECK: .Ltmp{{.*}}:
+; CHECK:         calll   may_throw
+; CHECK: .Ltmp{{.*}}:
+; CHECK:         jmp     .LBB{{.*}}
+; CHECK: .LBB{{.*}}:                                # %lpad
+; CHECK-NEXT: .Ltmp{{.*}}:
+; CHECK:         movl    {{.*}}, sink_across
+; CHECK:         movl    $55, %{{[a-z]*}}
+; CHECK: .LBB{{.*}}:                                # %try.cont
+; CHECK:         retl
+
+
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.value(metadata, metadata, metadata) #0
 
diff --git a/llvm/test/CodeGen/X86/soft-fp-legal-in-HW-reg.ll b/llvm/test/CodeGen/X86/soft-fp-legal-in-HW-reg.ll
index ae516c3bf933c..cf33a4a20524b 100644
--- a/llvm/test/CodeGen/X86/soft-fp-legal-in-HW-reg.ll
+++ b/llvm/test/CodeGen/X86/soft-fp-legal-in-HW-reg.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-linux-android -mattr=+mmx -enable-legalize-types-checking | FileCheck %s
 ;
 ; D31946
@@ -8,48 +9,68 @@ declare fp128 @llvm.fabs.f128(fp128)
 declare fp128 @llvm.copysign.f128(fp128, fp128)
 
 define fp128 @TestSelect(fp128 %a, fp128 %b) {
+; CHECK-LABEL: TestSelect:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    subq $32, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    .cfi_offset %rbx, -16
+; CHECK-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    callq __gttf2
+; CHECK-NEXT:    movl %eax, %ebx
+; CHECK-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    callq __subtf3
+; CHECK-NEXT:    testl %ebx, %ebx
+; CHECK-NEXT:    jg .LBB0_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:  .LBB0_2:
+; CHECK-NEXT:    addq $32, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %cmp = fcmp ogt fp128 %a, %b
   %sub = fsub fp128 %a, %b
   %res = select i1 %cmp, fp128 %sub, fp128 0xL00000000000000000000000000000000
   ret fp128 %res
-; CHECK-LABEL: TestSelect:
-; CHECK        movaps 16(%rsp), %xmm1
-; CHECK-NEXT   callq __subtf3
-; CHECK-NEXT   testl %ebx, %ebx
-; CHECK-NEXT   jg .LBB0_2
-; CHECK-NEXT # %bb.1:
-; CHECK-NEXT   movaps .LCPI0_0(%rip), %xmm0
-; CHECK-NEXT .LBB0_2:
-; CHECK-NEXT   addq $32, %rsp
-; CHECK-NEXT   popq %rbx
-; CHECK-NEXT   retq
 }
 
 define fp128 @TestFabs(fp128 %a) {
+; CHECK-LABEL: TestFabs:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    andps {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    retq
   %res = call fp128 @llvm.fabs.f128(fp128 %a)
   ret fp128 %res
-; CHECK-LABEL: TestFabs:
-; CHECK      andps .LCPI1_0(%rip), %xmm0
-; CHECK-NEXT retq
 }
 
 define fp128 @TestCopysign(fp128 %a, fp128 %b) {
+; CHECK-LABEL: TestCopysign:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    andps {{.*}}(%rip), %xmm1
+; CHECK-NEXT:    andps {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    orps %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %res = call fp128 @llvm.copysign.f128(fp128 %a, fp128 %b)
   ret fp128 %res
-; CHECK-LABEL: TestCopysign:
-; CHECK      andps .LCPI2_1(%rip), %xmm0
-; CHECK-NEXT orps %xmm1, %xmm0
-; CHECK-NEXT retq
 }
 
 define fp128 @TestFneg(fp128 %a) {
+; CHECK-LABEL: TestFneg:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    movaps %xmm0, %xmm1
+; CHECK-NEXT:    callq __multf3
+; CHECK-NEXT:    xorps {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %mul = fmul fp128 %a, %a
   %res = fsub fp128 0xL00000000000000008000000000000000, %mul
   ret fp128 %res
-; CHECK-LABEL: TestFneg:
-; CHECK      movaps %xmm0, %xmm1
-; CHECK-NEXT callq __multf3
-; CHECK-NEXT xorps .LCPI3_0(%rip), %xmm0
-; CHECK-NEXT popq %rax
-; CHECK-NEXT retq
 }
diff --git a/llvm/test/CodeGen/X86/sqrt-fastmath-mir.ll b/llvm/test/CodeGen/X86/sqrt-fastmath-mir.ll
index 7be19c07da803..4483de105385d 100644
--- a/llvm/test/CodeGen/X86/sqrt-fastmath-mir.ll
+++ b/llvm/test/CodeGen/X86/sqrt-fastmath-mir.ll
@@ -9,17 +9,30 @@ define float @sqrt_ieee(float %f) #0 {
   ; CHECK:   liveins: $xmm0
   ; CHECK:   [[COPY:%[0-9]+]]:fr32 = COPY $xmm0
   ; CHECK:   [[DEF:%[0-9]+]]:fr32 = IMPLICIT_DEF
+  ; CHECK:   %1:fr32 = nofpexcept VSQRTSSr killed [[DEF]], [[COPY]], implicit $mxcsr
+  ; CHECK:   $xmm0 = COPY %1
+  ; CHECK:   RET 0, $xmm0
+  %call = tail call float @llvm.sqrt.f32(float %f)
+  ret float %call
+}
+
+define float @sqrt_ieee_ninf(float %f) #0 {
+  ; CHECK-LABEL: name: sqrt_ieee_ninf
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK:   liveins: $xmm0
+  ; CHECK:   [[COPY:%[0-9]+]]:fr32 = COPY $xmm0
+  ; CHECK:   [[DEF:%[0-9]+]]:fr32 = IMPLICIT_DEF
   ; CHECK:   [[VRSQRTSSr:%[0-9]+]]:fr32 = VRSQRTSSr killed [[DEF]], [[COPY]]
-  ; CHECK:   %3:fr32 = nofpexcept VMULSSrr [[COPY]], [[VRSQRTSSr]], implicit $mxcsr
+  ; CHECK:   %3:fr32 = ninf nofpexcept VMULSSrr [[COPY]], [[VRSQRTSSr]], implicit $mxcsr
   ; CHECK:   [[VMOVSSrm_alt:%[0-9]+]]:fr32 = VMOVSSrm_alt $rip, 1, $noreg, %const.0, $noreg :: (load 4 from constant-pool)
-  ; CHECK:   %5:fr32 = nofpexcept VFMADD213SSr [[VRSQRTSSr]], killed %3, [[VMOVSSrm_alt]], implicit $mxcsr
+  ; CHECK:   %5:fr32 = ninf nofpexcept VFMADD213SSr [[VRSQRTSSr]], killed %3, [[VMOVSSrm_alt]], implicit $mxcsr
   ; CHECK:   [[VMOVSSrm_alt1:%[0-9]+]]:fr32 = VMOVSSrm_alt $rip, 1, $noreg, %const.1, $noreg :: (load 4 from constant-pool)
-  ; CHECK:   %7:fr32 = nofpexcept VMULSSrr [[VRSQRTSSr]], [[VMOVSSrm_alt1]], implicit $mxcsr
-  ; CHECK:   %8:fr32 = nofpexcept VMULSSrr killed %7, killed %5, implicit $mxcsr
-  ; CHECK:   %9:fr32 = nofpexcept VMULSSrr [[COPY]], %8, implicit $mxcsr
-  ; CHECK:   %10:fr32 = nofpexcept VFMADD213SSr %8, %9, [[VMOVSSrm_alt]], implicit $mxcsr
-  ; CHECK:   %11:fr32 = nofpexcept VMULSSrr %9, [[VMOVSSrm_alt1]], implicit $mxcsr
-  ; CHECK:   %12:fr32 = nofpexcept VMULSSrr killed %11, killed %10, implicit $mxcsr
+  ; CHECK:   %7:fr32 = ninf nofpexcept VMULSSrr [[VRSQRTSSr]], [[VMOVSSrm_alt1]], implicit $mxcsr
+  ; CHECK:   %8:fr32 = ninf nofpexcept VMULSSrr killed %7, killed %5, implicit $mxcsr
+  ; CHECK:   %9:fr32 = ninf nofpexcept VMULSSrr [[COPY]], %8, implicit $mxcsr
+  ; CHECK:   %10:fr32 = ninf nofpexcept VFMADD213SSr %8, %9, [[VMOVSSrm_alt]], implicit $mxcsr
+  ; CHECK:   %11:fr32 = ninf nofpexcept VMULSSrr %9, [[VMOVSSrm_alt1]], implicit $mxcsr
+  ; CHECK:   %12:fr32 = ninf nofpexcept VMULSSrr killed %11, killed %10, implicit $mxcsr
   ; CHECK:   [[COPY1:%[0-9]+]]:vr128 = COPY %12
   ; CHECK:   [[COPY2:%[0-9]+]]:vr128 = COPY [[COPY]]
   ; CHECK:   [[VPBROADCASTDrm:%[0-9]+]]:vr128 = VPBROADCASTDrm $rip, 1, $noreg, %const.2, $noreg :: (load 4 from constant-pool)
@@ -31,7 +44,7 @@ define float @sqrt_ieee(float %f) #0 {
   ; CHECK:   [[COPY5:%[0-9]+]]:fr32 = COPY [[VPANDNrr]]
   ; CHECK:   $xmm0 = COPY [[COPY5]]
   ; CHECK:   RET 0, $xmm0
-  %call = tail call float @llvm.sqrt.f32(float %f)
+  %call = tail call ninf float @llvm.sqrt.f32(float %f)
   ret float %call
 }
 
@@ -41,17 +54,30 @@ define float @sqrt_daz(float %f) #1 {
   ; CHECK:   liveins: $xmm0
   ; CHECK:   [[COPY:%[0-9]+]]:fr32 = COPY $xmm0
   ; CHECK:   [[DEF:%[0-9]+]]:fr32 = IMPLICIT_DEF
+  ; CHECK:   %1:fr32 = nofpexcept VSQRTSSr killed [[DEF]], [[COPY]], implicit $mxcsr
+  ; CHECK:   $xmm0 = COPY %1
+  ; CHECK:   RET 0, $xmm0
+  %call = tail call float @llvm.sqrt.f32(float %f)
+  ret float %call
+}
+
+define float @sqrt_daz_ninf(float %f) #1 {
+  ; CHECK-LABEL: name: sqrt_daz_ninf
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK:   liveins: $xmm0
+  ; CHECK:   [[COPY:%[0-9]+]]:fr32 = COPY $xmm0
+  ; CHECK:   [[DEF:%[0-9]+]]:fr32 = IMPLICIT_DEF
   ; CHECK:   [[VRSQRTSSr:%[0-9]+]]:fr32 = VRSQRTSSr killed [[DEF]], [[COPY]]
-  ; CHECK:   %3:fr32 = nofpexcept VMULSSrr [[COPY]], [[VRSQRTSSr]], implicit $mxcsr
+  ; CHECK:   %3:fr32 = ninf nofpexcept VMULSSrr [[COPY]], [[VRSQRTSSr]], implicit $mxcsr
   ; CHECK:   [[VMOVSSrm_alt:%[0-9]+]]:fr32 = VMOVSSrm_alt $rip, 1, $noreg, %const.0, $noreg :: (load 4 from constant-pool)
-  ; CHECK:   %5:fr32 = nofpexcept VFMADD213SSr [[VRSQRTSSr]], killed %3, [[VMOVSSrm_alt]], implicit $mxcsr
+  ; CHECK:   %5:fr32 = ninf nofpexcept VFMADD213SSr [[VRSQRTSSr]], killed %3, [[VMOVSSrm_alt]], implicit $mxcsr
   ; CHECK:   [[VMOVSSrm_alt1:%[0-9]+]]:fr32 = VMOVSSrm_alt $rip, 1, $noreg, %const.1, $noreg :: (load 4 from constant-pool)
-  ; CHECK:   %7:fr32 = nofpexcept VMULSSrr [[VRSQRTSSr]], [[VMOVSSrm_alt1]], implicit $mxcsr
-  ; CHECK:   %8:fr32 = nofpexcept VMULSSrr killed %7, killed %5, implicit $mxcsr
-  ; CHECK:   %9:fr32 = nofpexcept VMULSSrr [[COPY]], %8, implicit $mxcsr
-  ; CHECK:   %10:fr32 = nofpexcept VFMADD213SSr %8, %9, [[VMOVSSrm_alt]], implicit $mxcsr
-  ; CHECK:   %11:fr32 = nofpexcept VMULSSrr %9, [[VMOVSSrm_alt1]], implicit $mxcsr
-  ; CHECK:   %12:fr32 = nofpexcept VMULSSrr killed %11, killed %10, implicit $mxcsr
+  ; CHECK:   %7:fr32 = ninf nofpexcept VMULSSrr [[VRSQRTSSr]], [[VMOVSSrm_alt1]], implicit $mxcsr
+  ; CHECK:   %8:fr32 = ninf nofpexcept VMULSSrr killed %7, killed %5, implicit $mxcsr
+  ; CHECK:   %9:fr32 = ninf nofpexcept VMULSSrr [[COPY]], %8, implicit $mxcsr
+  ; CHECK:   %10:fr32 = ninf nofpexcept VFMADD213SSr %8, %9, [[VMOVSSrm_alt]], implicit $mxcsr
+  ; CHECK:   %11:fr32 = ninf nofpexcept VMULSSrr %9, [[VMOVSSrm_alt1]], implicit $mxcsr
+  ; CHECK:   %12:fr32 = ninf nofpexcept VMULSSrr killed %11, killed %10, implicit $mxcsr
   ; CHECK:   [[COPY1:%[0-9]+]]:vr128 = COPY %12
   ; CHECK:   [[FsFLD0SS:%[0-9]+]]:fr32 = FsFLD0SS
   ; CHECK:   %15:fr32 = nofpexcept VCMPSSrr [[COPY]], killed [[FsFLD0SS]], 0, implicit $mxcsr
@@ -60,7 +86,7 @@ define float @sqrt_daz(float %f) #1 {
   ; CHECK:   [[COPY3:%[0-9]+]]:fr32 = COPY [[VPANDNrr]]
   ; CHECK:   $xmm0 = COPY [[COPY3]]
   ; CHECK:   RET 0, $xmm0
-  %call = tail call float @llvm.sqrt.f32(float %f)
+  %call = tail call ninf float @llvm.sqrt.f32(float %f)
   ret float %call
 }
 
diff --git a/llvm/test/CodeGen/X86/sqrt-fastmath-tune.ll b/llvm/test/CodeGen/X86/sqrt-fastmath-tune.ll
index 0b6fb97ef913d..4c8e50468fd7c 100644
--- a/llvm/test/CodeGen/X86/sqrt-fastmath-tune.ll
+++ b/llvm/test/CodeGen/X86/sqrt-fastmath-tune.ll
@@ -87,8 +87,33 @@ define <4 x float> @v4f32_no_daz(<4 x float> %f) #0 {
 define <8 x float> @v8f32_no_daz(<8 x float> %f) #0 {
 ; NHM-LABEL: v8f32_no_daz:
 ; NHM:       # %bb.0:
-; NHM-NEXT:    sqrtps %xmm0, %xmm0
-; NHM-NEXT:    sqrtps %xmm1, %xmm1
+; NHM-NEXT:    movaps %xmm0, %xmm2
+; NHM-NEXT:    rsqrtps %xmm0, %xmm3
+; NHM-NEXT:    mulps %xmm3, %xmm0
+; NHM-NEXT:    movaps {{.*#+}} xmm4 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; NHM-NEXT:    movaps %xmm0, %xmm5
+; NHM-NEXT:    mulps %xmm4, %xmm5
+; NHM-NEXT:    mulps %xmm3, %xmm0
+; NHM-NEXT:    movaps {{.*#+}} xmm3 = [-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0]
+; NHM-NEXT:    addps %xmm3, %xmm0
+; NHM-NEXT:    mulps %xmm5, %xmm0
+; NHM-NEXT:    movaps {{.*#+}} xmm5 = [NaN,NaN,NaN,NaN]
+; NHM-NEXT:    andps %xmm5, %xmm2
+; NHM-NEXT:    movaps {{.*#+}} xmm6 = [1.17549435E-38,1.17549435E-38,1.17549435E-38,1.17549435E-38]
+; NHM-NEXT:    movaps %xmm6, %xmm7
+; NHM-NEXT:    cmpleps %xmm2, %xmm7
+; NHM-NEXT:    andps %xmm7, %xmm0
+; NHM-NEXT:    rsqrtps %xmm1, %xmm7
+; NHM-NEXT:    movaps %xmm1, %xmm2
+; NHM-NEXT:    mulps %xmm7, %xmm2
+; NHM-NEXT:    mulps %xmm2, %xmm4
+; NHM-NEXT:    mulps %xmm7, %xmm2
+; NHM-NEXT:    addps %xmm3, %xmm2
+; NHM-NEXT:    mulps %xmm4, %xmm2
+; NHM-NEXT:    andps %xmm5, %xmm1
+; NHM-NEXT:    cmpleps %xmm1, %xmm6
+; NHM-NEXT:    andps %xmm6, %xmm2
+; NHM-NEXT:    movaps %xmm2, %xmm1
 ; NHM-NEXT:    retq
 ;
 ; SNB-LABEL: v8f32_no_daz:
@@ -209,8 +234,28 @@ define <4 x float> @v4f32_daz(<4 x float> %f) #1 {
 define <8 x float> @v8f32_daz(<8 x float> %f) #1 {
 ; NHM-LABEL: v8f32_daz:
 ; NHM:       # %bb.0:
-; NHM-NEXT:    sqrtps %xmm0, %xmm0
-; NHM-NEXT:    sqrtps %xmm1, %xmm1
+; NHM-NEXT:    rsqrtps %xmm0, %xmm2
+; NHM-NEXT:    movaps %xmm0, %xmm3
+; NHM-NEXT:    mulps %xmm2, %xmm3
+; NHM-NEXT:    movaps {{.*#+}} xmm4 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; NHM-NEXT:    movaps %xmm3, %xmm5
+; NHM-NEXT:    mulps %xmm4, %xmm5
+; NHM-NEXT:    mulps %xmm2, %xmm3
+; NHM-NEXT:    movaps {{.*#+}} xmm2 = [-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0]
+; NHM-NEXT:    addps %xmm2, %xmm3
+; NHM-NEXT:    mulps %xmm5, %xmm3
+; NHM-NEXT:    xorps %xmm5, %xmm5
+; NHM-NEXT:    cmpneqps %xmm5, %xmm0
+; NHM-NEXT:    andps %xmm3, %xmm0
+; NHM-NEXT:    rsqrtps %xmm1, %xmm3
+; NHM-NEXT:    movaps %xmm1, %xmm6
+; NHM-NEXT:    mulps %xmm3, %xmm6
+; NHM-NEXT:    mulps %xmm6, %xmm4
+; NHM-NEXT:    mulps %xmm3, %xmm6
+; NHM-NEXT:    addps %xmm2, %xmm6
+; NHM-NEXT:    mulps %xmm4, %xmm6
+; NHM-NEXT:    cmpneqps %xmm5, %xmm1
+; NHM-NEXT:    andps %xmm6, %xmm1
 ; NHM-NEXT:    retq
 ;
 ; SNB-LABEL: v8f32_daz:
diff --git a/llvm/test/CodeGen/X86/sqrt-fastmath.ll b/llvm/test/CodeGen/X86/sqrt-fastmath.ll
index f10199ce958f0..b2593bc43578b 100644
--- a/llvm/test/CodeGen/X86/sqrt-fastmath.ll
+++ b/llvm/test/CodeGen/X86/sqrt-fastmath.ll
@@ -59,6 +59,20 @@ define float @finite_f32_no_estimate(float %f) #0 {
 define float @finite_f32_estimate_ieee(float %f) #1 {
 ; SSE-LABEL: finite_f32_estimate_ieee:
 ; SSE:       # %bb.0:
+; SSE-NEXT:    sqrtss %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: finite_f32_estimate_ieee:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vsqrtss %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %call = tail call float @__sqrtf_finite(float %f) #2
+  ret float %call
+}
+
+define float @finite_f32_estimate_ieee_ninf(float %f) #1 {
+; SSE-LABEL: finite_f32_estimate_ieee_ninf:
+; SSE:       # %bb.0:
 ; SSE-NEXT:    rsqrtss %xmm0, %xmm1
 ; SSE-NEXT:    movaps %xmm0, %xmm2
 ; SSE-NEXT:    mulss %xmm1, %xmm2
@@ -72,7 +86,7 @@ define float @finite_f32_estimate_ieee(float %f) #1 {
 ; SSE-NEXT:    andnps %xmm2, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: finite_f32_estimate_ieee:
+; AVX1-LABEL: finite_f32_estimate_ieee_ninf:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vrsqrtss %xmm0, %xmm0, %xmm1
 ; AVX1-NEXT:    vmulss %xmm1, %xmm0, %xmm2
@@ -85,7 +99,7 @@ define float @finite_f32_estimate_ieee(float %f) #1 {
 ; AVX1-NEXT:    vandnps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
-; AVX512-LABEL: finite_f32_estimate_ieee:
+; AVX512-LABEL: finite_f32_estimate_ieee_ninf:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vrsqrtss %xmm0, %xmm0, %xmm1
 ; AVX512-NEXT:    vmulss %xmm1, %xmm0, %xmm2
@@ -99,13 +113,27 @@ define float @finite_f32_estimate_ieee(float %f) #1 {
 ; AVX512-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm1, %xmm0
 ; AVX512-NEXT:    retq
-  %call = tail call float @__sqrtf_finite(float %f) #2
+  %call = tail call ninf float @__sqrtf_finite(float %f) #2
   ret float %call
 }
 
 define float @finite_f32_estimate_daz(float %f) #4 {
 ; SSE-LABEL: finite_f32_estimate_daz:
 ; SSE:       # %bb.0:
+; SSE-NEXT:    sqrtss %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: finite_f32_estimate_daz:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vsqrtss %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %call = tail call float @__sqrtf_finite(float %f) #2
+  ret float %call
+}
+
+define float @finite_f32_estimate_daz_ninf(float %f) #4 {
+; SSE-LABEL: finite_f32_estimate_daz_ninf:
+; SSE:       # %bb.0:
 ; SSE-NEXT:    rsqrtss %xmm0, %xmm1
 ; SSE-NEXT:    movaps %xmm0, %xmm2
 ; SSE-NEXT:    mulss %xmm1, %xmm2
@@ -119,7 +147,7 @@ define float @finite_f32_estimate_daz(float %f) #4 {
 ; SSE-NEXT:    andnps %xmm2, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: finite_f32_estimate_daz:
+; AVX1-LABEL: finite_f32_estimate_daz_ninf:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vrsqrtss %xmm0, %xmm0, %xmm1
 ; AVX1-NEXT:    vmulss %xmm1, %xmm0, %xmm2
@@ -132,7 +160,7 @@ define float @finite_f32_estimate_daz(float %f) #4 {
 ; AVX1-NEXT:    vandnps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
-; AVX512-LABEL: finite_f32_estimate_daz:
+; AVX512-LABEL: finite_f32_estimate_daz_ninf:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vrsqrtss %xmm0, %xmm0, %xmm1
 ; AVX512-NEXT:    vmulss %xmm1, %xmm0, %xmm2
@@ -144,7 +172,7 @@ define float @finite_f32_estimate_daz(float %f) #4 {
 ; AVX512-NEXT:    vmovss %xmm2, %xmm1, %xmm1 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm1, %xmm0
 ; AVX512-NEXT:    retq
-  %call = tail call float @__sqrtf_finite(float %f) #2
+  %call = tail call ninf float @__sqrtf_finite(float %f) #2
   ret float %call
 }
 
@@ -175,6 +203,20 @@ define x86_fp80 @finite_f80_estimate_but_no(x86_fp80 %ld) #1 {
 define float @sqrtf_check_denorms(float %x) #3 {
 ; SSE-LABEL: sqrtf_check_denorms:
 ; SSE:       # %bb.0:
+; SSE-NEXT:    sqrtss %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sqrtf_check_denorms:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vsqrtss %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %call = tail call float @__sqrtf_finite(float %x) #2
+  ret float %call
+}
+
+define float @sqrtf_check_denorms_ninf(float %x) #3 {
+; SSE-LABEL: sqrtf_check_denorms_ninf:
+; SSE:       # %bb.0:
 ; SSE-NEXT:    rsqrtss %xmm0, %xmm1
 ; SSE-NEXT:    movaps %xmm0, %xmm2
 ; SSE-NEXT:    mulss %xmm1, %xmm2
@@ -188,7 +230,7 @@ define float @sqrtf_check_denorms(float %x) #3 {
 ; SSE-NEXT:    andnps %xmm2, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: sqrtf_check_denorms:
+; AVX1-LABEL: sqrtf_check_denorms_ninf:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vrsqrtss %xmm0, %xmm0, %xmm1
 ; AVX1-NEXT:    vmulss %xmm1, %xmm0, %xmm2
@@ -201,7 +243,7 @@ define float @sqrtf_check_denorms(float %x) #3 {
 ; AVX1-NEXT:    vandnps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
-; AVX512-LABEL: sqrtf_check_denorms:
+; AVX512-LABEL: sqrtf_check_denorms_ninf:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vrsqrtss %xmm0, %xmm0, %xmm1
 ; AVX512-NEXT:    vmulss %xmm1, %xmm0, %xmm2
@@ -215,13 +257,27 @@ define float @sqrtf_check_denorms(float %x) #3 {
 ; AVX512-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm1, %xmm0
 ; AVX512-NEXT:    retq
-  %call = tail call float @__sqrtf_finite(float %x) #2
+  %call = tail call ninf float @__sqrtf_finite(float %x) #2
   ret float %call
 }
 
 define <4 x float> @sqrt_v4f32_check_denorms(<4 x float> %x) #3 {
 ; SSE-LABEL: sqrt_v4f32_check_denorms:
 ; SSE:       # %bb.0:
+; SSE-NEXT:    sqrtps %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sqrt_v4f32_check_denorms:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vsqrtps %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %call = tail call <4 x float> @llvm.sqrt.v4f32(<4 x float> %x) #2
+  ret <4 x float> %call
+}
+
+define <4 x float> @sqrt_v4f32_check_denorms_ninf(<4 x float> %x) #3 {
+; SSE-LABEL: sqrt_v4f32_check_denorms_ninf:
+; SSE:       # %bb.0:
 ; SSE-NEXT:    rsqrtps %xmm0, %xmm2
 ; SSE-NEXT:    movaps %xmm0, %xmm1
 ; SSE-NEXT:    mulps %xmm2, %xmm1
@@ -237,7 +293,7 @@ define <4 x float> @sqrt_v4f32_check_denorms(<4 x float> %x) #3 {
 ; SSE-NEXT:    movaps %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: sqrt_v4f32_check_denorms:
+; AVX1-LABEL: sqrt_v4f32_check_denorms_ninf:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vrsqrtps %xmm0, %xmm1
 ; AVX1-NEXT:    vmulps %xmm1, %xmm0, %xmm2
@@ -251,7 +307,7 @@ define <4 x float> @sqrt_v4f32_check_denorms(<4 x float> %x) #3 {
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
-; AVX512-LABEL: sqrt_v4f32_check_denorms:
+; AVX512-LABEL: sqrt_v4f32_check_denorms_ninf:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vrsqrtps %xmm0, %xmm1
 ; AVX512-NEXT:    vmulps %xmm1, %xmm0, %xmm2
@@ -266,7 +322,7 @@ define <4 x float> @sqrt_v4f32_check_denorms(<4 x float> %x) #3 {
 ; AVX512-NEXT:    vcmpleps %xmm0, %xmm2, %xmm0
 ; AVX512-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
-  %call = tail call <4 x float> @llvm.sqrt.v4f32(<4 x float> %x) #2
+  %call = tail call ninf <4 x float> @llvm.sqrt.v4f32(<4 x float> %x) #2
   ret <4 x float> %call
 }
 
diff --git a/llvm/test/CodeGen/X86/stack-clash-medium-natural-probes-mutliple-objects.ll b/llvm/test/CodeGen/X86/stack-clash-medium-natural-probes-mutliple-objects.ll
index 1a595f639f26e..0fe492a93d0e1 100644
--- a/llvm/test/CodeGen/X86/stack-clash-medium-natural-probes-mutliple-objects.ll
+++ b/llvm/test/CodeGen/X86/stack-clash-medium-natural-probes-mutliple-objects.ll
@@ -1,24 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s | FileCheck %s
 
-
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define i32 @foo() local_unnamed_addr #0 {
-
 ; CHECK-LABEL: foo:
-; CHECK:         # %bb.0:
-; CHECK-NEXT	subq	$4096, %rsp # imm = 0x1000
-; CHECK-NEXT	.cfi_def_cfa_offset 5888
-; CHECK-NEXT	movl	$1, 2088(%rsp)
-; CHECK-NEXT	subq	$1784, %rsp # imm = 0x6F8
-; CHECK-NEXT	movl	$2, 672(%rsp)
-; CHECK-NEXT	movl	1872(%rsp), %eax
-; CHECK-NEXT	addq	$5880, %rsp # imm = 0x16F8
-; CHECK-NEXT	.cfi_def_cfa_offset 8
-; CHECK-NEXT	retq
-
-
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $4096, %rsp # imm = 0x1000
+; CHECK-NEXT:    movq $0, (%rsp)
+; CHECK-NEXT:    subq $1784, %rsp # imm = 0x6F8
+; CHECK-NEXT:    .cfi_def_cfa_offset 5888
+; CHECK-NEXT:    movl $1, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movl $2, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    addq $5880, %rsp # imm = 0x16F8
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %a = alloca i32, i64 1000, align 16
   %b = alloca i32, i64 500, align 16
   %a0 = getelementptr inbounds i32, i32* %a, i64 500
diff --git a/llvm/test/CodeGen/X86/vec-strict-fptoint-128.ll b/llvm/test/CodeGen/X86/vec-strict-fptoint-128.ll
index 0d5566028e8e0..b3a91e657c427 100644
--- a/llvm/test/CodeGen/X86/vec-strict-fptoint-128.ll
+++ b/llvm/test/CodeGen/X86/vec-strict-fptoint-128.ll
@@ -1587,7 +1587,7 @@ define <2 x i8> @strict_vector_fptosi_v2f64_to_v2i8(<2 x double> %a) #0 {
 ; AVX512VL-LABEL: strict_vector_fptosi_v2f64_to_v2i8:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vcvttpd2dq %xmm0, %xmm0
-; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512VL-NEXT:    vpmovdb %xmm0, %xmm0
 ; AVX512VL-NEXT:    ret{{[l|q]}}
 ;
 ; AVX512DQ-LABEL: strict_vector_fptosi_v2f64_to_v2i8:
@@ -1599,7 +1599,7 @@ define <2 x i8> @strict_vector_fptosi_v2f64_to_v2i8(<2 x double> %a) #0 {
 ; AVX512VLDQ-LABEL: strict_vector_fptosi_v2f64_to_v2i8:
 ; AVX512VLDQ:       # %bb.0:
 ; AVX512VLDQ-NEXT:    vcvttpd2dq %xmm0, %xmm0
-; AVX512VLDQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512VLDQ-NEXT:    vpmovdb %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    ret{{[l|q]}}
   %ret = call <2 x i8> @llvm.experimental.constrained.fptosi.v2i8.v2f64(<2 x double> %a,
                                               metadata !"fpexcept.strict")
@@ -1638,7 +1638,7 @@ define <2 x i8> @strict_vector_fptoui_v2f64_to_v2i8(<2 x double> %a) #0 {
 ; AVX512VL-LABEL: strict_vector_fptoui_v2f64_to_v2i8:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vcvttpd2dq %xmm0, %xmm0
-; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512VL-NEXT:    vpmovdb %xmm0, %xmm0
 ; AVX512VL-NEXT:    ret{{[l|q]}}
 ;
 ; AVX512DQ-LABEL: strict_vector_fptoui_v2f64_to_v2i8:
@@ -1650,7 +1650,7 @@ define <2 x i8> @strict_vector_fptoui_v2f64_to_v2i8(<2 x double> %a) #0 {
 ; AVX512VLDQ-LABEL: strict_vector_fptoui_v2f64_to_v2i8:
 ; AVX512VLDQ:       # %bb.0:
 ; AVX512VLDQ-NEXT:    vcvttpd2dq %xmm0, %xmm0
-; AVX512VLDQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512VLDQ-NEXT:    vpmovdb %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    ret{{[l|q]}}
   %ret = call <2 x i8> @llvm.experimental.constrained.fptoui.v2i8.v2f64(<2 x double> %a,
                                               metadata !"fpexcept.strict")
@@ -1694,7 +1694,7 @@ define <2 x i8> @strict_vector_fptosi_v2f32_to_v2i8(<2 x float> %a) #0 {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512VL-NEXT:    vcvttps2dq %xmm0, %xmm0
-; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512VL-NEXT:    vpmovdb %xmm0, %xmm0
 ; AVX512VL-NEXT:    ret{{[l|q]}}
 ;
 ; AVX512DQ-LABEL: strict_vector_fptosi_v2f32_to_v2i8:
@@ -1708,7 +1708,7 @@ define <2 x i8> @strict_vector_fptosi_v2f32_to_v2i8(<2 x float> %a) #0 {
 ; AVX512VLDQ:       # %bb.0:
 ; AVX512VLDQ-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512VLDQ-NEXT:    vcvttps2dq %xmm0, %xmm0
-; AVX512VLDQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512VLDQ-NEXT:    vpmovdb %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    ret{{[l|q]}}
   %ret = call <2 x i8> @llvm.experimental.constrained.fptosi.v2i8.v2f32(<2 x float> %a,
                                               metadata !"fpexcept.strict")
@@ -1752,7 +1752,7 @@ define <2 x i8> @strict_vector_fptoui_v2f32_to_v2i8(<2 x float> %a) #0 {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512VL-NEXT:    vcvttps2dq %xmm0, %xmm0
-; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512VL-NEXT:    vpmovdb %xmm0, %xmm0
 ; AVX512VL-NEXT:    ret{{[l|q]}}
 ;
 ; AVX512DQ-LABEL: strict_vector_fptoui_v2f32_to_v2i8:
@@ -1766,7 +1766,7 @@ define <2 x i8> @strict_vector_fptoui_v2f32_to_v2i8(<2 x float> %a) #0 {
 ; AVX512VLDQ:       # %bb.0:
 ; AVX512VLDQ-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
 ; AVX512VLDQ-NEXT:    vcvttps2dq %xmm0, %xmm0
-; AVX512VLDQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512VLDQ-NEXT:    vpmovdb %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    ret{{[l|q]}}
   %ret = call <2 x i8> @llvm.experimental.constrained.fptoui.v2i8.v2f32(<2 x float> %a,
                                               metadata !"fpexcept.strict")
@@ -2638,47 +2638,48 @@ define <4 x i8> @strict_vector_fptosi_v4f32_to_v4i8(<4 x float> %a) #0 {
 ; SSE-32-LABEL: strict_vector_fptosi_v4f32_to_v4i8:
 ; SSE-32:       # %bb.0:
 ; SSE-32-NEXT:    cvttps2dq %xmm0, %xmm0
-; SSE-32-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; SSE-32-NEXT:    packuswb %xmm0, %xmm0
-; SSE-32-NEXT:    packuswb %xmm0, %xmm0
+; SSE-32-NEXT:    packssdw %xmm0, %xmm0
+; SSE-32-NEXT:    packsswb %xmm0, %xmm0
 ; SSE-32-NEXT:    retl
 ;
 ; SSE-64-LABEL: strict_vector_fptosi_v4f32_to_v4i8:
 ; SSE-64:       # %bb.0:
 ; SSE-64-NEXT:    cvttps2dq %xmm0, %xmm0
-; SSE-64-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE-64-NEXT:    packuswb %xmm0, %xmm0
-; SSE-64-NEXT:    packuswb %xmm0, %xmm0
+; SSE-64-NEXT:    packssdw %xmm0, %xmm0
+; SSE-64-NEXT:    packsswb %xmm0, %xmm0
 ; SSE-64-NEXT:    retq
 ;
 ; AVX-LABEL: strict_vector_fptosi_v4f32_to_v4i8:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vcvttps2dq %xmm0, %xmm0
-; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    ret{{[l|q]}}
 ;
 ; AVX512F-LABEL: strict_vector_fptosi_v4f32_to_v4i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vcvttps2dq %xmm0, %xmm0
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX512F-NEXT:    ret{{[l|q]}}
 ;
 ; AVX512VL-LABEL: strict_vector_fptosi_v4f32_to_v4i8:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vcvttps2dq %xmm0, %xmm0
-; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512VL-NEXT:    vpmovdb %xmm0, %xmm0
 ; AVX512VL-NEXT:    ret{{[l|q]}}
 ;
 ; AVX512DQ-LABEL: strict_vector_fptosi_v4f32_to_v4i8:
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vcvttps2dq %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    ret{{[l|q]}}
 ;
 ; AVX512VLDQ-LABEL: strict_vector_fptosi_v4f32_to_v4i8:
 ; AVX512VLDQ:       # %bb.0:
 ; AVX512VLDQ-NEXT:    vcvttps2dq %xmm0, %xmm0
-; AVX512VLDQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512VLDQ-NEXT:    vpmovdb %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    ret{{[l|q]}}
   %ret = call <4 x i8> @llvm.experimental.constrained.fptosi.v4i8.v4f32(<4 x float> %a,
                                               metadata !"fpexcept.strict")
@@ -2689,7 +2690,6 @@ define <4 x i8> @strict_vector_fptoui_v4f32_to_v4i8(<4 x float> %a) #0 {
 ; SSE-32-LABEL: strict_vector_fptoui_v4f32_to_v4i8:
 ; SSE-32:       # %bb.0:
 ; SSE-32-NEXT:    cvttps2dq %xmm0, %xmm0
-; SSE-32-NEXT:    pand {{\.LCPI.*}}, %xmm0
 ; SSE-32-NEXT:    packuswb %xmm0, %xmm0
 ; SSE-32-NEXT:    packuswb %xmm0, %xmm0
 ; SSE-32-NEXT:    retl
@@ -2697,7 +2697,6 @@ define <4 x i8> @strict_vector_fptoui_v4f32_to_v4i8(<4 x float> %a) #0 {
 ; SSE-64-LABEL: strict_vector_fptoui_v4f32_to_v4i8:
 ; SSE-64:       # %bb.0:
 ; SSE-64-NEXT:    cvttps2dq %xmm0, %xmm0
-; SSE-64-NEXT:    pand {{.*}}(%rip), %xmm0
 ; SSE-64-NEXT:    packuswb %xmm0, %xmm0
 ; SSE-64-NEXT:    packuswb %xmm0, %xmm0
 ; SSE-64-NEXT:    retq
@@ -2705,31 +2704,34 @@ define <4 x i8> @strict_vector_fptoui_v4f32_to_v4i8(<4 x float> %a) #0 {
 ; AVX-LABEL: strict_vector_fptoui_v4f32_to_v4i8:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vcvttps2dq %xmm0, %xmm0
-; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX-NEXT:    ret{{[l|q]}}
 ;
 ; AVX512F-LABEL: strict_vector_fptoui_v4f32_to_v4i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vcvttps2dq %xmm0, %xmm0
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX512F-NEXT:    ret{{[l|q]}}
 ;
 ; AVX512VL-LABEL: strict_vector_fptoui_v4f32_to_v4i8:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vcvttps2dq %xmm0, %xmm0
-; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512VL-NEXT:    vpmovdb %xmm0, %xmm0
 ; AVX512VL-NEXT:    ret{{[l|q]}}
 ;
 ; AVX512DQ-LABEL: strict_vector_fptoui_v4f32_to_v4i8:
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vcvttps2dq %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    ret{{[l|q]}}
 ;
 ; AVX512VLDQ-LABEL: strict_vector_fptoui_v4f32_to_v4i8:
 ; AVX512VLDQ:       # %bb.0:
 ; AVX512VLDQ-NEXT:    vcvttps2dq %xmm0, %xmm0
-; AVX512VLDQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512VLDQ-NEXT:    vpmovdb %xmm0, %xmm0
 ; AVX512VLDQ-NEXT:    ret{{[l|q]}}
   %ret = call <4 x i8> @llvm.experimental.constrained.fptoui.v4i8.v4f32(<4 x float> %a,
                                               metadata !"fpexcept.strict")
diff --git a/llvm/test/CodeGen/X86/vec-strict-fptoint-256.ll b/llvm/test/CodeGen/X86/vec-strict-fptoint-256.ll
index 38f19e9ed2d52..00aee49f64cb0 100644
--- a/llvm/test/CodeGen/X86/vec-strict-fptoint-256.ll
+++ b/llvm/test/CodeGen/X86/vec-strict-fptoint-256.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx -O3 | FileCheck %s --check-prefixes=CHECK,AVX-32
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx -O3 | FileCheck %s --check-prefixes=CHECK,AVX-64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx -O3 | FileCheck %s --check-prefixes=CHECK,AVX,AVX-32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx -O3 | FileCheck %s --check-prefixes=CHECK,AVX,AVX-64
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512f -O3 | FileCheck %s --check-prefixes=CHECK,AVX512F,AVX512F-32
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f -O3 | FileCheck %s --check-prefixes=CHECK,AVX512F,AVX512F-64
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512vl -O3 | FileCheck %s --check-prefixes=CHECK,AVX512VL,AVX512VL-32
@@ -1173,39 +1173,22 @@ define <4 x i32> @strict_vector_fptosi_v4f64_to_v4i32(<4 x double> %a) #0 {
 }
 
 define <4 x i32> @strict_vector_fptoui_v4f64_to_v4i32(<4 x double> %a) #0 {
-; AVX-32-LABEL: strict_vector_fptoui_v4f64_to_v4i32:
-; AVX-32:       # %bb.0:
-; AVX-32-NEXT:    vmovapd {{.*#+}} ymm1 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
-; AVX-32-NEXT:    vcmpltpd %ymm1, %ymm0, %ymm2
-; AVX-32-NEXT:    vextractf128 $1, %ymm2, %xmm3
-; AVX-32-NEXT:    vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm3[0,2]
-; AVX-32-NEXT:    vxorps %xmm4, %xmm4, %xmm4
-; AVX-32-NEXT:    vmovaps {{.*#+}} xmm5 = [2147483648,2147483648,2147483648,2147483648]
-; AVX-32-NEXT:    vblendvps %xmm3, %xmm4, %xmm5, %xmm3
-; AVX-32-NEXT:    vxorps %xmm4, %xmm4, %xmm4
-; AVX-32-NEXT:    vblendvpd %ymm2, %ymm4, %ymm1, %ymm1
-; AVX-32-NEXT:    vsubpd %ymm1, %ymm0, %ymm0
-; AVX-32-NEXT:    vcvttpd2dq %ymm0, %xmm0
-; AVX-32-NEXT:    vxorpd %xmm3, %xmm0, %xmm0
-; AVX-32-NEXT:    vzeroupper
-; AVX-32-NEXT:    retl
-;
-; AVX-64-LABEL: strict_vector_fptoui_v4f64_to_v4i32:
-; AVX-64:       # %bb.0:
-; AVX-64-NEXT:    vmovapd {{.*#+}} ymm1 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
-; AVX-64-NEXT:    vcmpltpd %ymm1, %ymm0, %ymm2
-; AVX-64-NEXT:    vextractf128 $1, %ymm2, %xmm3
-; AVX-64-NEXT:    vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm3[0,2]
-; AVX-64-NEXT:    vxorps %xmm4, %xmm4, %xmm4
-; AVX-64-NEXT:    vmovaps {{.*#+}} xmm5 = [2147483648,2147483648,2147483648,2147483648]
-; AVX-64-NEXT:    vblendvps %xmm3, %xmm4, %xmm5, %xmm3
-; AVX-64-NEXT:    vxorps %xmm4, %xmm4, %xmm4
-; AVX-64-NEXT:    vblendvpd %ymm2, %ymm4, %ymm1, %ymm1
-; AVX-64-NEXT:    vsubpd %ymm1, %ymm0, %ymm0
-; AVX-64-NEXT:    vcvttpd2dq %ymm0, %xmm0
-; AVX-64-NEXT:    vxorpd %xmm3, %xmm0, %xmm0
-; AVX-64-NEXT:    vzeroupper
-; AVX-64-NEXT:    retq
+; AVX-LABEL: strict_vector_fptoui_v4f64_to_v4i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovapd {{.*#+}} ymm1 = [2.147483648E+9,2.147483648E+9,2.147483648E+9,2.147483648E+9]
+; AVX-NEXT:    vcmpltpd %ymm1, %ymm0, %ymm2
+; AVX-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm3[0,2]
+; AVX-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; AVX-NEXT:    vmovaps {{.*#+}} xmm5 = [2147483648,2147483648,2147483648,2147483648]
+; AVX-NEXT:    vblendvps %xmm3, %xmm4, %xmm5, %xmm3
+; AVX-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; AVX-NEXT:    vblendvpd %ymm2, %ymm4, %ymm1, %ymm1
+; AVX-NEXT:    vsubpd %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX-NEXT:    vxorpd %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    ret{{[l|q]}}
 ;
 ; AVX512F-LABEL: strict_vector_fptoui_v4f64_to_v4i32:
 ; AVX512F:       # %bb.0:
@@ -1264,41 +1247,97 @@ define <4 x i16> @strict_vector_fptoui_v4f64_to_v4i16(<4 x double> %a) #0 {
 }
 
 define <4 x i8> @strict_vector_fptosi_v4f64_to_v4i8(<4 x double> %a) #0 {
-; CHECK-LABEL: strict_vector_fptosi_v4f64_to_v4i8:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vcvttpd2dq %ymm0, %xmm0
-; CHECK-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    ret{{[l|q]}}
+; AVX-LABEL: strict_vector_fptosi_v4f64_to_v4i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    ret{{[l|q]}}
+;
+; AVX512F-LABEL: strict_vector_fptosi_v4f64_to_v4i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX512F-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptosi_v4f64_to_v4i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX512VL-NEXT:    vpmovdb %xmm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptosi_v4f64_to_v4i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX512DQ-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: strict_vector_fptosi_v4f64_to_v4i8:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX512DQVL-NEXT:    vpmovdb %xmm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
   %ret = call <4 x i8> @llvm.experimental.constrained.fptosi.v4i8.v4f64(<4 x double> %a,
                                               metadata !"fpexcept.strict")
   ret <4 x i8> %ret
 }
 
 define <4 x i8> @strict_vector_fptoui_v4f64_to_v4i8(<4 x double> %a) #0 {
-; CHECK-LABEL: strict_vector_fptoui_v4f64_to_v4i8:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vcvttpd2dq %ymm0, %xmm0
-; CHECK-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    ret{{[l|q]}}
+; AVX-LABEL: strict_vector_fptoui_v4f64_to_v4i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    ret{{[l|q]}}
+;
+; AVX512F-LABEL: strict_vector_fptoui_v4f64_to_v4i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX512F-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    ret{{[l|q]}}
+;
+; AVX512VL-LABEL: strict_vector_fptoui_v4f64_to_v4i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX512VL-NEXT:    vpmovdb %xmm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQ-LABEL: strict_vector_fptoui_v4f64_to_v4i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX512DQ-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    ret{{[l|q]}}
+;
+; AVX512DQVL-LABEL: strict_vector_fptoui_v4f64_to_v4i8:
+; AVX512DQVL:       # %bb.0:
+; AVX512DQVL-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX512DQVL-NEXT:    vpmovdb %xmm0, %xmm0
+; AVX512DQVL-NEXT:    vzeroupper
+; AVX512DQVL-NEXT:    ret{{[l|q]}}
   %ret = call <4 x i8> @llvm.experimental.constrained.fptoui.v4i8.v4f64(<4 x double> %a,
                                               metadata !"fpexcept.strict")
   ret <4 x i8> %ret
 }
 
 define <4 x i1> @strict_vector_fptosi_v4f64_to_v4i1(<4 x double> %a) #0 {
-; AVX-32-LABEL: strict_vector_fptosi_v4f64_to_v4i1:
-; AVX-32:       # %bb.0:
-; AVX-32-NEXT:    vcvttpd2dq %ymm0, %xmm0
-; AVX-32-NEXT:    vzeroupper
-; AVX-32-NEXT:    retl
-;
-; AVX-64-LABEL: strict_vector_fptosi_v4f64_to_v4i1:
-; AVX-64:       # %bb.0:
-; AVX-64-NEXT:    vcvttpd2dq %ymm0, %xmm0
-; AVX-64-NEXT:    vzeroupper
-; AVX-64-NEXT:    retq
+; AVX-LABEL: strict_vector_fptosi_v4f64_to_v4i1:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    ret{{[l|q]}}
 ;
 ; AVX512F-LABEL: strict_vector_fptosi_v4f64_to_v4i1:
 ; AVX512F:       # %bb.0:
@@ -1340,17 +1379,11 @@ define <4 x i1> @strict_vector_fptosi_v4f64_to_v4i1(<4 x double> %a) #0 {
 }
 
 define <4 x i1> @strict_vector_fptoui_v4f64_to_v4i1(<4 x double> %a) #0 {
-; AVX-32-LABEL: strict_vector_fptoui_v4f64_to_v4i1:
-; AVX-32:       # %bb.0:
-; AVX-32-NEXT:    vcvttpd2dq %ymm0, %xmm0
-; AVX-32-NEXT:    vzeroupper
-; AVX-32-NEXT:    retl
-;
-; AVX-64-LABEL: strict_vector_fptoui_v4f64_to_v4i1:
-; AVX-64:       # %bb.0:
-; AVX-64-NEXT:    vcvttpd2dq %ymm0, %xmm0
-; AVX-64-NEXT:    vzeroupper
-; AVX-64-NEXT:    retq
+; AVX-LABEL: strict_vector_fptoui_v4f64_to_v4i1:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvttpd2dq %ymm0, %xmm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    ret{{[l|q]}}
 ;
 ; AVX512F-LABEL: strict_vector_fptoui_v4f64_to_v4i1:
 ; AVX512F:       # %bb.0:
@@ -1406,31 +1439,18 @@ define <8 x i32> @strict_vector_fptosi_v8f32_to_v8i32(<8 x float> %a) #0 {
 }
 
 define <8 x i32> @strict_vector_fptoui_v8f32_to_v8i32(<8 x float> %a) #0 {
-; AVX-32-LABEL: strict_vector_fptoui_v8f32_to_v8i32:
-; AVX-32:       # %bb.0:
-; AVX-32-NEXT:    vmovaps {{.*#+}} ymm1 = [2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9]
-; AVX-32-NEXT:    vcmpltps %ymm1, %ymm0, %ymm2
-; AVX-32-NEXT:    vxorps %xmm3, %xmm3, %xmm3
-; AVX-32-NEXT:    vmovaps {{.*#+}} ymm4 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
-; AVX-32-NEXT:    vblendvps %ymm2, %ymm3, %ymm4, %ymm4
-; AVX-32-NEXT:    vblendvps %ymm2, %ymm3, %ymm1, %ymm1
-; AVX-32-NEXT:    vsubps %ymm1, %ymm0, %ymm0
-; AVX-32-NEXT:    vcvttps2dq %ymm0, %ymm0
-; AVX-32-NEXT:    vxorps %ymm4, %ymm0, %ymm0
-; AVX-32-NEXT:    retl
-;
-; AVX-64-LABEL: strict_vector_fptoui_v8f32_to_v8i32:
-; AVX-64:       # %bb.0:
-; AVX-64-NEXT:    vmovaps {{.*#+}} ymm1 = [2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9]
-; AVX-64-NEXT:    vcmpltps %ymm1, %ymm0, %ymm2
-; AVX-64-NEXT:    vxorps %xmm3, %xmm3, %xmm3
-; AVX-64-NEXT:    vmovaps {{.*#+}} ymm4 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
-; AVX-64-NEXT:    vblendvps %ymm2, %ymm3, %ymm4, %ymm4
-; AVX-64-NEXT:    vblendvps %ymm2, %ymm3, %ymm1, %ymm1
-; AVX-64-NEXT:    vsubps %ymm1, %ymm0, %ymm0
-; AVX-64-NEXT:    vcvttps2dq %ymm0, %ymm0
-; AVX-64-NEXT:    vxorps %ymm4, %ymm0, %ymm0
-; AVX-64-NEXT:    retq
+; AVX-LABEL: strict_vector_fptoui_v8f32_to_v8i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovaps {{.*#+}} ymm1 = [2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9]
+; AVX-NEXT:    vcmpltps %ymm1, %ymm0, %ymm2
+; AVX-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; AVX-NEXT:    vmovaps {{.*#+}} ymm4 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX-NEXT:    vblendvps %ymm2, %ymm3, %ymm4, %ymm4
+; AVX-NEXT:    vblendvps %ymm2, %ymm3, %ymm1, %ymm1
+; AVX-NEXT:    vsubps %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX-NEXT:    vxorps %ymm4, %ymm0, %ymm0
+; AVX-NEXT:    ret{{[l|q]}}
 ;
 ; AVX512F-LABEL: strict_vector_fptoui_v8f32_to_v8i32:
 ; AVX512F:       # %bb.0:
@@ -1461,21 +1481,13 @@ define <8 x i32> @strict_vector_fptoui_v8f32_to_v8i32(<8 x float> %a) #0 {
 }
 
 define <8 x i16> @strict_vector_fptosi_v8f32_to_v8i16(<8 x float> %a) #0 {
-; AVX-32-LABEL: strict_vector_fptosi_v8f32_to_v8i16:
-; AVX-32:       # %bb.0:
-; AVX-32-NEXT:    vcvttps2dq %ymm0, %ymm0
-; AVX-32-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-32-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; AVX-32-NEXT:    vzeroupper
-; AVX-32-NEXT:    retl
-;
-; AVX-64-LABEL: strict_vector_fptosi_v8f32_to_v8i16:
-; AVX-64:       # %bb.0:
-; AVX-64-NEXT:    vcvttps2dq %ymm0, %ymm0
-; AVX-64-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-64-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; AVX-64-NEXT:    vzeroupper
-; AVX-64-NEXT:    retq
+; AVX-LABEL: strict_vector_fptosi_v8f32_to_v8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    ret{{[l|q]}}
 ;
 ; AVX512F-LABEL: strict_vector_fptosi_v8f32_to_v8i16:
 ; AVX512F:       # %bb.0:
@@ -1512,21 +1524,13 @@ define <8 x i16> @strict_vector_fptosi_v8f32_to_v8i16(<8 x float> %a) #0 {
 }
 
 define <8 x i16> @strict_vector_fptoui_v8f32_to_v8i16(<8 x float> %a) #0 {
-; AVX-32-LABEL: strict_vector_fptoui_v8f32_to_v8i16:
-; AVX-32:       # %bb.0:
-; AVX-32-NEXT:    vcvttps2dq %ymm0, %ymm0
-; AVX-32-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-32-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
-; AVX-32-NEXT:    vzeroupper
-; AVX-32-NEXT:    retl
-;
-; AVX-64-LABEL: strict_vector_fptoui_v8f32_to_v8i16:
-; AVX-64:       # %bb.0:
-; AVX-64-NEXT:    vcvttps2dq %ymm0, %ymm0
-; AVX-64-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-64-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
-; AVX-64-NEXT:    vzeroupper
-; AVX-64-NEXT:    retq
+; AVX-LABEL: strict_vector_fptoui_v8f32_to_v8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    ret{{[l|q]}}
 ;
 ; AVX512F-LABEL: strict_vector_fptoui_v8f32_to_v8i16:
 ; AVX512F:       # %bb.0:
@@ -1563,23 +1567,14 @@ define <8 x i16> @strict_vector_fptoui_v8f32_to_v8i16(<8 x float> %a) #0 {
 }
 
 define <8 x i8> @strict_vector_fptosi_v8f32_to_v8i8(<8 x float> %a) #0 {
-; AVX-32-LABEL: strict_vector_fptosi_v8f32_to_v8i8:
-; AVX-32:       # %bb.0:
-; AVX-32-NEXT:    vcvttps2dq %ymm0, %ymm0
-; AVX-32-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-32-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; AVX-32-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX-32-NEXT:    vzeroupper
-; AVX-32-NEXT:    retl
-;
-; AVX-64-LABEL: strict_vector_fptosi_v8f32_to_v8i8:
-; AVX-64:       # %bb.0:
-; AVX-64-NEXT:    vcvttps2dq %ymm0, %ymm0
-; AVX-64-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-64-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; AVX-64-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
-; AVX-64-NEXT:    vzeroupper
-; AVX-64-NEXT:    retq
+; AVX-LABEL: strict_vector_fptosi_v8f32_to_v8i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    ret{{[l|q]}}
 ;
 ; AVX512F-LABEL: strict_vector_fptosi_v8f32_to_v8i8:
 ; AVX512F:       # %bb.0:
@@ -1614,23 +1609,14 @@ define <8 x i8> @strict_vector_fptosi_v8f32_to_v8i8(<8 x float> %a) #0 {
 }
 
 define <8 x i8> @strict_vector_fptoui_v8f32_to_v8i8(<8 x float> %a) #0 {
-; AVX-32-LABEL: strict_vector_fptoui_v8f32_to_v8i8:
-; AVX-32:       # %bb.0:
-; AVX-32-NEXT:    vcvttps2dq %ymm0, %ymm0
-; AVX-32-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-32-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; AVX-32-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
-; AVX-32-NEXT:    vzeroupper
-; AVX-32-NEXT:    retl
-;
-; AVX-64-LABEL: strict_vector_fptoui_v8f32_to_v8i8:
-; AVX-64:       # %bb.0:
-; AVX-64-NEXT:    vcvttps2dq %ymm0, %ymm0
-; AVX-64-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-64-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; AVX-64-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
-; AVX-64-NEXT:    vzeroupper
-; AVX-64-NEXT:    retq
+; AVX-LABEL: strict_vector_fptoui_v8f32_to_v8i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    ret{{[l|q]}}
 ;
 ; AVX512F-LABEL: strict_vector_fptoui_v8f32_to_v8i8:
 ; AVX512F:       # %bb.0:
@@ -1665,21 +1651,13 @@ define <8 x i8> @strict_vector_fptoui_v8f32_to_v8i8(<8 x float> %a) #0 {
 }
 
 define <8 x i1> @strict_vector_fptosi_v8f32_to_v8i1(<8 x float> %a) #0 {
-; AVX-32-LABEL: strict_vector_fptosi_v8f32_to_v8i1:
-; AVX-32:       # %bb.0:
-; AVX-32-NEXT:    vcvttps2dq %ymm0, %ymm0
-; AVX-32-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-32-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; AVX-32-NEXT:    vzeroupper
-; AVX-32-NEXT:    retl
-;
-; AVX-64-LABEL: strict_vector_fptosi_v8f32_to_v8i1:
-; AVX-64:       # %bb.0:
-; AVX-64-NEXT:    vcvttps2dq %ymm0, %ymm0
-; AVX-64-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-64-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; AVX-64-NEXT:    vzeroupper
-; AVX-64-NEXT:    retq
+; AVX-LABEL: strict_vector_fptosi_v8f32_to_v8i1:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    ret{{[l|q]}}
 ;
 ; AVX512F-LABEL: strict_vector_fptosi_v8f32_to_v8i1:
 ; AVX512F:       # %bb.0:
@@ -1725,21 +1703,13 @@ define <8 x i1> @strict_vector_fptosi_v8f32_to_v8i1(<8 x float> %a) #0 {
 }
 
 define <8 x i1> @strict_vector_fptoui_v8f32_to_v8i1(<8 x float> %a) #0 {
-; AVX-32-LABEL: strict_vector_fptoui_v8f32_to_v8i1:
-; AVX-32:       # %bb.0:
-; AVX-32-NEXT:    vcvttps2dq %ymm0, %ymm0
-; AVX-32-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-32-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
-; AVX-32-NEXT:    vzeroupper
-; AVX-32-NEXT:    retl
-;
-; AVX-64-LABEL: strict_vector_fptoui_v8f32_to_v8i1:
-; AVX-64:       # %bb.0:
-; AVX-64-NEXT:    vcvttps2dq %ymm0, %ymm0
-; AVX-64-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-64-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
-; AVX-64-NEXT:    vzeroupper
-; AVX-64-NEXT:    retq
+; AVX-LABEL: strict_vector_fptoui_v8f32_to_v8i1:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvttps2dq %ymm0, %ymm0
+; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    ret{{[l|q]}}
 ;
 ; AVX512F-LABEL: strict_vector_fptoui_v8f32_to_v8i1:
 ; AVX512F:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vec_cast2.ll b/llvm/test/CodeGen/X86/vec_cast2.ll
index e43216dcdbdea..ed703f1708478 100644
--- a/llvm/test/CodeGen/X86/vec_cast2.ll
+++ b/llvm/test/CodeGen/X86/vec_cast2.ll
@@ -122,7 +122,8 @@ define <4 x i8> @cvt_v4f32_v4i8(<4 x float> %src) {
 ; CHECK-LABEL: cvt_v4f32_v4i8:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vcvttps2dq %xmm0, %xmm0
-; CHECK-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    retl
   %res = fptosi <4 x float> %src to <4 x i8>
   ret <4 x i8> %res
@@ -167,7 +168,8 @@ define <4 x i8> @cvt_v4f32_v4u8(<4 x float> %src) {
 ; CHECK-LABEL: cvt_v4f32_v4u8:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vcvttps2dq %xmm0, %xmm0
-; CHECK-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    retl
   %res = fptoui <4 x float> %src to <4 x i8>
   ret <4 x i8> %res
diff --git a/llvm/test/CodeGen/X86/vec_fp_to_int.ll b/llvm/test/CodeGen/X86/vec_fp_to_int.ll
index d412a48c03b53..e42742fd008de 100644
--- a/llvm/test/CodeGen/X86/vec_fp_to_int.ll
+++ b/llvm/test/CodeGen/X86/vec_fp_to_int.ll
@@ -2302,11 +2302,35 @@ define <2 x i8> @fptosi_2f32_to_2i8(<2 x float> %a) {
 ; SSE-NEXT:    packuswb %xmm0, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: fptosi_2f32_to_2i8:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vcvttps2dq %xmm0, %xmm0
-; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX-NEXT:    retq
+; VEX-LABEL: fptosi_2f32_to_2i8:
+; VEX:       # %bb.0:
+; VEX-NEXT:    vcvttps2dq %xmm0, %xmm0
+; VEX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; VEX-NEXT:    retq
+;
+; AVX512F-LABEL: fptosi_2f32_to_2i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: fptosi_2f32_to_2i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmovdb %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512DQ-LABEL: fptosi_2f32_to_2i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-NEXT:    retq
+;
+; AVX512VLDQ-LABEL: fptosi_2f32_to_2i8:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vpmovdb %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    retq
   %cvt = fptosi <2 x float> %a to <2 x i8>
   ret <2 x i8> %cvt
 }
@@ -2336,11 +2360,35 @@ define <2 x i8> @fptoui_2f32_to_2i8(<2 x float> %a) {
 ; SSE-NEXT:    packuswb %xmm0, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: fptoui_2f32_to_2i8:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vcvttps2dq %xmm0, %xmm0
-; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX-NEXT:    retq
+; VEX-LABEL: fptoui_2f32_to_2i8:
+; VEX:       # %bb.0:
+; VEX-NEXT:    vcvttps2dq %xmm0, %xmm0
+; VEX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; VEX-NEXT:    retq
+;
+; AVX512F-LABEL: fptoui_2f32_to_2i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: fptoui_2f32_to_2i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmovdb %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512DQ-LABEL: fptoui_2f32_to_2i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-NEXT:    retq
+;
+; AVX512VLDQ-LABEL: fptoui_2f32_to_2i8:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vcvttps2dq %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vpmovdb %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    retq
   %cvt = fptoui <2 x float> %a to <2 x i8>
   ret <2 x i8> %cvt
 }
@@ -2370,11 +2418,35 @@ define <2 x i8> @fptosi_2f64_to_2i8(<2 x double> %a) {
 ; SSE-NEXT:    packuswb %xmm0, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: fptosi_2f64_to_2i8:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vcvttpd2dq %xmm0, %xmm0
-; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX-NEXT:    retq
+; VEX-LABEL: fptosi_2f64_to_2i8:
+; VEX:       # %bb.0:
+; VEX-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; VEX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; VEX-NEXT:    retq
+;
+; AVX512F-LABEL: fptosi_2f64_to_2i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: fptosi_2f64_to_2i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmovdb %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512DQ-LABEL: fptosi_2f64_to_2i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-NEXT:    retq
+;
+; AVX512VLDQ-LABEL: fptosi_2f64_to_2i8:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vpmovdb %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    retq
   %cvt = fptosi <2 x double> %a to <2 x i8>
   ret <2 x i8> %cvt
 }
@@ -2404,11 +2476,35 @@ define <2 x i8> @fptoui_2f64_to_2i8(<2 x double> %a) {
 ; SSE-NEXT:    packuswb %xmm0, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: fptoui_2f64_to_2i8:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vcvttpd2dq %xmm0, %xmm0
-; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX-NEXT:    retq
+; VEX-LABEL: fptoui_2f64_to_2i8:
+; VEX:       # %bb.0:
+; VEX-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; VEX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; VEX-NEXT:    retq
+;
+; AVX512F-LABEL: fptoui_2f64_to_2i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: fptoui_2f64_to_2i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmovdb %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512DQ-LABEL: fptoui_2f64_to_2i8:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-NEXT:    retq
+;
+; AVX512VLDQ-LABEL: fptoui_2f64_to_2i8:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vcvttpd2dq %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    vpmovdb %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    retq
   %cvt = fptoui <2 x double> %a to <2 x i8>
   ret <2 x i8> %cvt
 }
diff --git a/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll b/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll
index 3457450a3ee02..8bb2be88c6741 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-and-bool.ll
@@ -356,29 +356,17 @@ define i1 @trunc_v8i32_v8i1(<8 x i32>) {
 }
 
 define i1 @trunc_v16i16_v16i1(<16 x i16>) {
-; SSE2-LABEL: trunc_v16i16_v16i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    packuswb %xmm1, %xmm0
-; SSE2-NEXT:    psllw $7, %xmm0
-; SSE2-NEXT:    pmovmskb %xmm0, %eax
-; SSE2-NEXT:    cmpw $-1, %ax
-; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: trunc_v16i16_v16i1:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; SSE41-NEXT:    pshufb %xmm2, %xmm1
-; SSE41-NEXT:    pshufb %xmm2, %xmm0
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE41-NEXT:    psllw $7, %xmm0
-; SSE41-NEXT:    pmovmskb %xmm0, %eax
-; SSE41-NEXT:    cmpw $-1, %ax
-; SSE41-NEXT:    sete %al
-; SSE41-NEXT:    retq
+; SSE-LABEL: trunc_v16i16_v16i1:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE-NEXT:    pand %xmm2, %xmm1
+; SSE-NEXT:    pand %xmm2, %xmm0
+; SSE-NEXT:    packuswb %xmm1, %xmm0
+; SSE-NEXT:    psllw $7, %xmm0
+; SSE-NEXT:    pmovmskb %xmm0, %eax
+; SSE-NEXT:    cmpw $-1, %ax
+; SSE-NEXT:    sete %al
+; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_v16i16_v16i1:
 ; AVX1:       # %bb.0:
@@ -695,37 +683,21 @@ define i1 @trunc_v16i32_v16i1(<16 x i32>) {
 }
 
 define i1 @trunc_v32i16_v32i1(<32 x i16>) {
-; SSE2-LABEL: trunc_v32i16_v32i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; SSE2-NEXT:    pand %xmm4, %xmm3
-; SSE2-NEXT:    pand %xmm4, %xmm2
-; SSE2-NEXT:    packuswb %xmm3, %xmm2
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    packuswb %xmm1, %xmm0
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    psllw $7, %xmm0
-; SSE2-NEXT:    pmovmskb %xmm0, %eax
-; SSE2-NEXT:    cmpw $-1, %ax
-; SSE2-NEXT:    sete %al
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: trunc_v32i16_v32i1:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; SSE41-NEXT:    pshufb %xmm4, %xmm3
-; SSE41-NEXT:    pshufb %xmm4, %xmm2
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE41-NEXT:    pshufb %xmm4, %xmm1
-; SSE41-NEXT:    pshufb %xmm4, %xmm0
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE41-NEXT:    pand %xmm2, %xmm0
-; SSE41-NEXT:    psllw $7, %xmm0
-; SSE41-NEXT:    pmovmskb %xmm0, %eax
-; SSE41-NEXT:    cmpw $-1, %ax
-; SSE41-NEXT:    sete %al
-; SSE41-NEXT:    retq
+; SSE-LABEL: trunc_v32i16_v32i1:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
+; SSE-NEXT:    pand %xmm4, %xmm3
+; SSE-NEXT:    pand %xmm4, %xmm2
+; SSE-NEXT:    packuswb %xmm3, %xmm2
+; SSE-NEXT:    pand %xmm4, %xmm1
+; SSE-NEXT:    pand %xmm4, %xmm0
+; SSE-NEXT:    packuswb %xmm1, %xmm0
+; SSE-NEXT:    pand %xmm2, %xmm0
+; SSE-NEXT:    psllw $7, %xmm0
+; SSE-NEXT:    pmovmskb %xmm0, %eax
+; SSE-NEXT:    cmpw $-1, %ax
+; SSE-NEXT:    sete %al
+; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_v32i16_v32i1:
 ; AVX1:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-reduce-mul.ll b/llvm/test/CodeGen/X86/vector-reduce-mul.ll
index 9bdd722f64021..050ce4d9dd4cd 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-mul.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-mul.ll
@@ -2376,24 +2376,21 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512BWVL-NEXT:    vextracti128 $1, %ymm1, %xmm1
 ; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
 ; AVX512BWVL-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; AVX512BWVL-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; AVX512BWVL-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512BWVL-NEXT:    vpmovwb %ymm2, %xmm2
 ; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
 ; AVX512BWVL-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpand %xmm3, %xmm0, %xmm1
-; AVX512BWVL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512BWVL-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
 ; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
 ; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpand %xmm3, %xmm0, %xmm1
-; AVX512BWVL-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
 ; AVX512BWVL-NEXT:    vpsrld $16, %xmm1, %xmm1
 ; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpand %xmm3, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
 ; AVX512BWVL-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vmovd %xmm0, %eax
@@ -2440,37 +2437,32 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512DQVL-LABEL: test_v64i8:
 ; AVX512DQVL:       # %bb.0:
 ; AVX512DQVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
-; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
-; AVX512DQVL-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQVL-NEXT:    vpand %ymm3, %ymm2, %ymm4
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
+; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm0 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpand %ymm3, %ymm0, %ymm1
-; AVX512DQVL-NEXT:    vpackuswb %ymm4, %ymm1, %ymm1
-; AVX512DQVL-NEXT:    vextracti128 $1, %ymm1, %xmm1
-; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
-; AVX512DQVL-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
-; AVX512DQVL-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; AVX512DQVL-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX512DQVL-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
-; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpand %xmm3, %xmm0, %xmm1
-; AVX512DQVL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512DQVL-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpand %xmm3, %xmm0, %xmm1
-; AVX512DQVL-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
-; AVX512DQVL-NEXT:    vpsrld $16, %xmm1, %xmm1
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpand %xmm3, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
+; AVX512DQVL-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
+; AVX512DQVL-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX512DQVL-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX512DQVL-NEXT:    vpmullw %xmm0, %xmm3, %xmm0
+; AVX512DQVL-NEXT:    vpmullw %xmm0, %xmm2, %xmm0
+; AVX512DQVL-NEXT:    vpand %xmm1, %xmm0, %xmm2
+; AVX512DQVL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX512DQVL-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
+; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512DQVL-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
+; AVX512DQVL-NEXT:    vpand %xmm1, %xmm0, %xmm2
+; AVX512DQVL-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
+; AVX512DQVL-NEXT:    vpsrld $16, %xmm2, %xmm2
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512DQVL-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
+; AVX512DQVL-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX512DQVL-NEXT:    vpackuswb %xmm3, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vmovd %xmm0, %eax
@@ -2723,33 +2715,28 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ;
 ; AVX512BW-LABEL: test_v128i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
 ; AVX512BW-NEXT:    vpmullw %zmm2, %zmm3, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512BW-NEXT:    vpandq %zmm3, %zmm2, %zmm4
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm1 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm0 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
 ; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm1
-; AVX512BW-NEXT:    vpackuswb %zmm4, %zmm1, %zmm1
-; AVX512BW-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
-; AVX512BW-NEXT:    vpmullw %zmm3, %zmm2, %zmm2
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512BW-NEXT:    vpand %ymm3, %ymm2, %ymm4
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
+; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpand %ymm3, %ymm0, %ymm1
-; AVX512BW-NEXT:    vpackuswb %ymm4, %ymm1, %ymm1
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512BW-NEXT:    vpand %ymm1, %ymm0, %ymm3
+; AVX512BW-NEXT:    vextracti64x4 $1, %zmm2, %ymm4
+; AVX512BW-NEXT:    vpmullw %zmm4, %zmm2, %zmm2
+; AVX512BW-NEXT:    vpand %ymm1, %ymm2, %ymm1
+; AVX512BW-NEXT:    vpackuswb %ymm3, %ymm1, %ymm1
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm1, %xmm1
 ; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
-; AVX512BW-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
+; AVX512BW-NEXT:    vpmullw %xmm3, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; AVX512BW-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; AVX512BW-NEXT:    vpand %xmm3, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX512BW-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpmullw %xmm0, %xmm1, %xmm0
+; AVX512BW-NEXT:    vpmullw %xmm0, %xmm2, %xmm0
 ; AVX512BW-NEXT:    vpand %xmm3, %xmm0, %xmm1
 ; AVX512BW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX512BW-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
@@ -2772,46 +2759,38 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ;
 ; AVX512BWVL-LABEL: test_v128i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
-; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
 ; AVX512BWVL-NEXT:    vpmullw %zmm2, %zmm3, %zmm2
-; AVX512BWVL-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm2, %zmm4
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm1 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
+; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm0 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
 ; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm0, %zmm1
-; AVX512BWVL-NEXT:    vpackuswb %zmm4, %zmm1, %zmm1
-; AVX512BWVL-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
-; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
-; AVX512BWVL-NEXT:    vpmullw %zmm3, %zmm2, %zmm2
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512BWVL-NEXT:    vpand %ymm3, %ymm2, %ymm4
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
+; AVX512BWVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpand %ymm3, %ymm0, %ymm1
-; AVX512BWVL-NEXT:    vpackuswb %ymm4, %ymm1, %ymm1
+; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512BWVL-NEXT:    vpand %ymm1, %ymm0, %ymm3
+; AVX512BWVL-NEXT:    vextracti64x4 $1, %zmm2, %ymm4
+; AVX512BWVL-NEXT:    vpmullw %zmm4, %zmm2, %zmm2
+; AVX512BWVL-NEXT:    vpand %ymm1, %ymm2, %ymm1
+; AVX512BWVL-NEXT:    vpackuswb %ymm3, %ymm1, %ymm1
 ; AVX512BWVL-NEXT:    vextracti128 $1, %ymm1, %xmm1
 ; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
-; AVX512BWVL-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; AVX512BWVL-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; AVX512BWVL-NEXT:    vpmullw %xmm3, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
 ; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX512BWVL-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
-; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpand %xmm3, %xmm0, %xmm1
-; AVX512BWVL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512BWVL-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX512BWVL-NEXT:    vpmullw %xmm0, %xmm1, %xmm0
+; AVX512BWVL-NEXT:    vpmullw %xmm0, %xmm2, %xmm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
 ; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
 ; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpand %xmm3, %xmm0, %xmm1
-; AVX512BWVL-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
 ; AVX512BWVL-NEXT:    vpsrld $16, %xmm1, %xmm1
 ; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpand %xmm3, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
 ; AVX512BWVL-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vmovd %xmm0, %eax
@@ -2864,32 +2843,27 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ;
 ; AVX512DQVL-LABEL: test_v128i8:
 ; AVX512DQVL:       # %bb.0:
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
+; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
+; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
 ; AVX512DQVL-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
 ; AVX512DQVL-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512DQVL-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
-; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
-; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
 ; AVX512DQVL-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
-; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
 ; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm2, %ymm1
-; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm0 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vmovdqa {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQVL-NEXT:    vpand %ymm1, %ymm0, %ymm2
-; AVX512DQVL-NEXT:    vpand %ymm1, %ymm4, %ymm1
-; AVX512DQVL-NEXT:    vpackuswb %ymm2, %ymm1, %ymm1
-; AVX512DQVL-NEXT:    vextracti128 $1, %ymm1, %xmm1
-; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
-; AVX512DQVL-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
+; AVX512DQVL-NEXT:    vextracti128 $1, %ymm4, %xmm1
+; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm4, %xmm1
 ; AVX512DQVL-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
-; AVX512DQVL-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; AVX512DQVL-NEXT:    vpmullw %xmm0, %xmm1, %xmm0
-; AVX512DQVL-NEXT:    vpmullw %xmm0, %xmm4, %xmm0
+; AVX512DQVL-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm3, %xmm1
+; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpand %xmm2, %xmm0, %xmm1
 ; AVX512DQVL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX512DQVL-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
diff --git a/llvm/test/CodeGen/X86/vector-reduce-or-bool.ll b/llvm/test/CodeGen/X86/vector-reduce-or-bool.ll
index 5a0deab79ae98..23fab58d3f8b0 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-or-bool.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-or-bool.ll
@@ -350,29 +350,17 @@ define i1 @trunc_v8i32_v8i1(<8 x i32>) {
 }
 
 define i1 @trunc_v16i16_v16i1(<16 x i16>) {
-; SSE2-LABEL: trunc_v16i16_v16i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    packuswb %xmm1, %xmm0
-; SSE2-NEXT:    psllw $7, %xmm0
-; SSE2-NEXT:    pmovmskb %xmm0, %eax
-; SSE2-NEXT:    testw %ax, %ax
-; SSE2-NEXT:    setne %al
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: trunc_v16i16_v16i1:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; SSE41-NEXT:    pshufb %xmm2, %xmm1
-; SSE41-NEXT:    pshufb %xmm2, %xmm0
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE41-NEXT:    psllw $7, %xmm0
-; SSE41-NEXT:    pmovmskb %xmm0, %eax
-; SSE41-NEXT:    testw %ax, %ax
-; SSE41-NEXT:    setne %al
-; SSE41-NEXT:    retq
+; SSE-LABEL: trunc_v16i16_v16i1:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE-NEXT:    pand %xmm2, %xmm1
+; SSE-NEXT:    pand %xmm2, %xmm0
+; SSE-NEXT:    packuswb %xmm1, %xmm0
+; SSE-NEXT:    psllw $7, %xmm0
+; SSE-NEXT:    pmovmskb %xmm0, %eax
+; SSE-NEXT:    testw %ax, %ax
+; SSE-NEXT:    setne %al
+; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_v16i16_v16i1:
 ; AVX1:       # %bb.0:
@@ -689,37 +677,21 @@ define i1 @trunc_v16i32_v16i1(<16 x i32>) {
 }
 
 define i1 @trunc_v32i16_v32i1(<32 x i16>) {
-; SSE2-LABEL: trunc_v32i16_v32i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; SSE2-NEXT:    pand %xmm4, %xmm3
-; SSE2-NEXT:    pand %xmm4, %xmm2
-; SSE2-NEXT:    packuswb %xmm3, %xmm2
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    packuswb %xmm1, %xmm0
-; SSE2-NEXT:    por %xmm2, %xmm0
-; SSE2-NEXT:    psllw $7, %xmm0
-; SSE2-NEXT:    pmovmskb %xmm0, %eax
-; SSE2-NEXT:    testw %ax, %ax
-; SSE2-NEXT:    setne %al
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: trunc_v32i16_v32i1:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; SSE41-NEXT:    pshufb %xmm4, %xmm3
-; SSE41-NEXT:    pshufb %xmm4, %xmm2
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE41-NEXT:    pshufb %xmm4, %xmm1
-; SSE41-NEXT:    pshufb %xmm4, %xmm0
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE41-NEXT:    por %xmm2, %xmm0
-; SSE41-NEXT:    psllw $7, %xmm0
-; SSE41-NEXT:    pmovmskb %xmm0, %eax
-; SSE41-NEXT:    testw %ax, %ax
-; SSE41-NEXT:    setne %al
-; SSE41-NEXT:    retq
+; SSE-LABEL: trunc_v32i16_v32i1:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
+; SSE-NEXT:    pand %xmm4, %xmm3
+; SSE-NEXT:    pand %xmm4, %xmm2
+; SSE-NEXT:    packuswb %xmm3, %xmm2
+; SSE-NEXT:    pand %xmm4, %xmm1
+; SSE-NEXT:    pand %xmm4, %xmm0
+; SSE-NEXT:    packuswb %xmm1, %xmm0
+; SSE-NEXT:    por %xmm2, %xmm0
+; SSE-NEXT:    psllw $7, %xmm0
+; SSE-NEXT:    pmovmskb %xmm0, %eax
+; SSE-NEXT:    testw %ax, %ax
+; SSE-NEXT:    setne %al
+; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_v32i16_v32i1:
 ; AVX1:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-reduce-smax.ll b/llvm/test/CodeGen/X86/vector-reduce-smax.ll
index 1160fbbc48f61..c0705ab96e039 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-smax.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-smax.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE4 --check-prefix=SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.2 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE4 --check-prefix=SSE42
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512BW
@@ -50,6 +51,15 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ; SSE41-NEXT:    movq %xmm2, %rax
 ; SSE41-NEXT:    retq
 ;
+; SSE42-LABEL: test_v2i64:
+; SSE42:       # %bb.0:
+; SSE42-NEXT:    movdqa %xmm0, %xmm1
+; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE42-NEXT:    pcmpgtq %xmm2, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE42-NEXT:    movq %xmm2, %rax
+; SSE42-NEXT:    retq
+;
 ; AVX-LABEL: test_v2i64:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
@@ -143,6 +153,18 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; SSE41-NEXT:    movq %xmm2, %rax
 ; SSE41-NEXT:    retq
 ;
+; SSE42-LABEL: test_v4i64:
+; SSE42:       # %bb.0:
+; SSE42-NEXT:    movdqa %xmm0, %xmm2
+; SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE42-NEXT:    movdqa %xmm1, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm2, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE42-NEXT:    movq %xmm2, %rax
+; SSE42-NEXT:    retq
+;
 ; AVX1-LABEL: test_v4i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
@@ -309,6 +331,25 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; SSE41-NEXT:    movq %xmm1, %rax
 ; SSE41-NEXT:    retq
 ;
+; SSE42-LABEL: test_v8i64:
+; SSE42:       # %bb.0:
+; SSE42-NEXT:    movdqa %xmm0, %xmm4
+; SSE42-NEXT:    movdqa %xmm1, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm3, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
+; SSE42-NEXT:    movdqa %xmm4, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm2, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
+; SSE42-NEXT:    movapd %xmm2, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm3, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
+; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
+; SSE42-NEXT:    movdqa %xmm3, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
+; SSE42-NEXT:    movq %xmm1, %rax
+; SSE42-NEXT:    retq
+;
 ; AVX1-LABEL: test_v8i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
@@ -589,6 +630,37 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; SSE41-NEXT:    movq %xmm1, %rax
 ; SSE41-NEXT:    retq
 ;
+; SSE42-LABEL: test_v16i64:
+; SSE42:       # %bb.0:
+; SSE42-NEXT:    movdqa %xmm0, %xmm8
+; SSE42-NEXT:    movdqa %xmm2, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm6, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm6
+; SSE42-NEXT:    movdqa %xmm8, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm4, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm8, %xmm4
+; SSE42-NEXT:    movdqa %xmm3, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm7, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm3, %xmm7
+; SSE42-NEXT:    movdqa %xmm1, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm5, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm1, %xmm5
+; SSE42-NEXT:    movapd %xmm5, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm7, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm5, %xmm7
+; SSE42-NEXT:    movapd %xmm4, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm6, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm4, %xmm6
+; SSE42-NEXT:    movapd %xmm6, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm7, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm6, %xmm7
+; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[2,3,0,1]
+; SSE42-NEXT:    movdqa %xmm7, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm7, %xmm1
+; SSE42-NEXT:    movq %xmm1, %rax
+; SSE42-NEXT:    retq
+;
 ; AVX1-LABEL: test_v16i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm11
@@ -679,12 +751,12 @@ define i32 @test_v2i32(<2 x i32> %a0) {
 ; SSE2-NEXT:    movd %xmm2, %eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v2i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE41-NEXT:    pmaxsd %xmm0, %xmm1
-; SSE41-NEXT:    movd %xmm1, %eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v2i32:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pmaxsd %xmm0, %xmm1
+; SSE4-NEXT:    movd %xmm1, %eax
+; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i32:
 ; AVX:       # %bb.0:
@@ -721,14 +793,14 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v4i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pmaxsd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE41-NEXT:    pmaxsd %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v4i32:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE4-NEXT:    pmaxsd %xmm0, %xmm1
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE4-NEXT:    pmaxsd %xmm1, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: test_v4i32:
 ; AVX:       # %bb.0:
@@ -774,15 +846,15 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE2-NEXT:    movd %xmm2, %eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v8i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmaxsd %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pmaxsd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE41-NEXT:    pmaxsd %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v8i32:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pmaxsd %xmm1, %xmm0
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE4-NEXT:    pmaxsd %xmm0, %xmm1
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE4-NEXT:    pmaxsd %xmm1, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v8i32:
 ; AVX1:       # %bb.0:
@@ -856,17 +928,17 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v16i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmaxsd %xmm3, %xmm1
-; SSE41-NEXT:    pmaxsd %xmm2, %xmm1
-; SSE41-NEXT:    pmaxsd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
-; SSE41-NEXT:    pmaxsd %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE41-NEXT:    pmaxsd %xmm0, %xmm1
-; SSE41-NEXT:    movd %xmm1, %eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v16i32:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pmaxsd %xmm3, %xmm1
+; SSE4-NEXT:    pmaxsd %xmm2, %xmm1
+; SSE4-NEXT:    pmaxsd %xmm0, %xmm1
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE4-NEXT:    pmaxsd %xmm1, %xmm0
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pmaxsd %xmm0, %xmm1
+; SSE4-NEXT:    movd %xmm1, %eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v16i32:
 ; AVX1:       # %bb.0:
@@ -966,21 +1038,21 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE2-NEXT:    movd %xmm2, %eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v32i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmaxsd %xmm6, %xmm2
-; SSE41-NEXT:    pmaxsd %xmm7, %xmm3
-; SSE41-NEXT:    pmaxsd %xmm5, %xmm3
-; SSE41-NEXT:    pmaxsd %xmm1, %xmm3
-; SSE41-NEXT:    pmaxsd %xmm4, %xmm2
-; SSE41-NEXT:    pmaxsd %xmm3, %xmm2
-; SSE41-NEXT:    pmaxsd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
-; SSE41-NEXT:    pmaxsd %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE41-NEXT:    pmaxsd %xmm0, %xmm1
-; SSE41-NEXT:    movd %xmm1, %eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v32i32:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pmaxsd %xmm6, %xmm2
+; SSE4-NEXT:    pmaxsd %xmm7, %xmm3
+; SSE4-NEXT:    pmaxsd %xmm5, %xmm3
+; SSE4-NEXT:    pmaxsd %xmm1, %xmm3
+; SSE4-NEXT:    pmaxsd %xmm4, %xmm2
+; SSE4-NEXT:    pmaxsd %xmm3, %xmm2
+; SSE4-NEXT:    pmaxsd %xmm0, %xmm2
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE4-NEXT:    pmaxsd %xmm2, %xmm0
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pmaxsd %xmm0, %xmm1
+; SSE4-NEXT:    movd %xmm1, %eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v32i32:
 ; AVX1:       # %bb.0:
@@ -1118,14 +1190,14 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v8i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    xorl $32767, %eax # imm = 0x7FFF
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v8i16:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pxor {{.*}}(%rip), %xmm0
+; SSE4-NEXT:    phminposuw %xmm0, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    xorl $32767, %eax # imm = 0x7FFF
+; SSE4-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: test_v8i16:
 ; AVX:       # %bb.0:
@@ -1163,15 +1235,15 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v16i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmaxsw %xmm1, %xmm0
-; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    xorl $32767, %eax # imm = 0x7FFF
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v16i16:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pmaxsw %xmm1, %xmm0
+; SSE4-NEXT:    pxor {{.*}}(%rip), %xmm0
+; SSE4-NEXT:    phminposuw %xmm0, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    xorl $32767, %eax # imm = 0x7FFF
+; SSE4-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v16i16:
 ; AVX1:       # %bb.0:
@@ -1229,17 +1301,17 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v32i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmaxsw %xmm3, %xmm1
-; SSE41-NEXT:    pmaxsw %xmm2, %xmm1
-; SSE41-NEXT:    pmaxsw %xmm0, %xmm1
-; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm1
-; SSE41-NEXT:    phminposuw %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    xorl $32767, %eax # imm = 0x7FFF
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v32i16:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pmaxsw %xmm3, %xmm1
+; SSE4-NEXT:    pmaxsw %xmm2, %xmm1
+; SSE4-NEXT:    pmaxsw %xmm0, %xmm1
+; SSE4-NEXT:    pxor {{.*}}(%rip), %xmm1
+; SSE4-NEXT:    phminposuw %xmm1, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    xorl $32767, %eax # imm = 0x7FFF
+; SSE4-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v32i16:
 ; AVX1:       # %bb.0:
@@ -1307,21 +1379,21 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v64i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmaxsw %xmm7, %xmm3
-; SSE41-NEXT:    pmaxsw %xmm5, %xmm3
-; SSE41-NEXT:    pmaxsw %xmm1, %xmm3
-; SSE41-NEXT:    pmaxsw %xmm6, %xmm2
-; SSE41-NEXT:    pmaxsw %xmm4, %xmm2
-; SSE41-NEXT:    pmaxsw %xmm3, %xmm2
-; SSE41-NEXT:    pmaxsw %xmm0, %xmm2
-; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm2
-; SSE41-NEXT:    phminposuw %xmm2, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    xorl $32767, %eax # imm = 0x7FFF
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v64i16:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pmaxsw %xmm7, %xmm3
+; SSE4-NEXT:    pmaxsw %xmm5, %xmm3
+; SSE4-NEXT:    pmaxsw %xmm1, %xmm3
+; SSE4-NEXT:    pmaxsw %xmm6, %xmm2
+; SSE4-NEXT:    pmaxsw %xmm4, %xmm2
+; SSE4-NEXT:    pmaxsw %xmm3, %xmm2
+; SSE4-NEXT:    pmaxsw %xmm0, %xmm2
+; SSE4-NEXT:    pxor {{.*}}(%rip), %xmm2
+; SSE4-NEXT:    phminposuw %xmm2, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    xorl $32767, %eax # imm = 0x7FFF
+; SSE4-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v64i16:
 ; AVX1:       # %bb.0:
@@ -1395,14 +1467,14 @@ define i8 @test_v2i8(<2 x i8> %a0) {
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v2i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrlw $8, %xmm1
-; SSE41-NEXT:    pmaxsb %xmm0, %xmm1
-; SSE41-NEXT:    movd %xmm1, %eax
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v2i8:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    movdqa %xmm0, %xmm1
+; SSE4-NEXT:    psrlw $8, %xmm1
+; SSE4-NEXT:    pmaxsb %xmm0, %xmm1
+; SSE4-NEXT:    movd %xmm1, %eax
+; SSE4-NEXT:    # kill: def $al killed $al killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i8:
 ; AVX:       # %bb.0:
@@ -1444,17 +1516,17 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v4i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrld $16, %xmm1
-; SSE41-NEXT:    pmaxsb %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrlw $8, %xmm0
-; SSE41-NEXT:    pmaxsb %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v4i8:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    movdqa %xmm0, %xmm1
+; SSE4-NEXT:    psrld $16, %xmm1
+; SSE4-NEXT:    pmaxsb %xmm0, %xmm1
+; SSE4-NEXT:    movdqa %xmm1, %xmm0
+; SSE4-NEXT:    psrlw $8, %xmm0
+; SSE4-NEXT:    pmaxsb %xmm1, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    # kill: def $al killed $al killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: test_v4i8:
 ; AVX:       # %bb.0:
@@ -1506,19 +1578,19 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v8i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE41-NEXT:    pmaxsb %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrld $16, %xmm0
-; SSE41-NEXT:    pmaxsb %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrlw $8, %xmm1
-; SSE41-NEXT:    pmaxsb %xmm0, %xmm1
-; SSE41-NEXT:    movd %xmm1, %eax
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v8i8:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pmaxsb %xmm0, %xmm1
+; SSE4-NEXT:    movdqa %xmm1, %xmm0
+; SSE4-NEXT:    psrld $16, %xmm0
+; SSE4-NEXT:    pmaxsb %xmm1, %xmm0
+; SSE4-NEXT:    movdqa %xmm0, %xmm1
+; SSE4-NEXT:    psrlw $8, %xmm1
+; SSE4-NEXT:    pmaxsb %xmm0, %xmm1
+; SSE4-NEXT:    movd %xmm1, %eax
+; SSE4-NEXT:    # kill: def $al killed $al killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: test_v8i8:
 ; AVX:       # %bb.0:
@@ -1580,17 +1652,17 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v16i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrlw $8, %xmm1
-; SSE41-NEXT:    pminub %xmm0, %xmm1
-; SSE41-NEXT:    phminposuw %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    xorb $127, %al
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v16i8:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pxor {{.*}}(%rip), %xmm0
+; SSE4-NEXT:    movdqa %xmm0, %xmm1
+; SSE4-NEXT:    psrlw $8, %xmm1
+; SSE4-NEXT:    pminub %xmm0, %xmm1
+; SSE4-NEXT:    phminposuw %xmm1, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    xorb $127, %al
+; SSE4-NEXT:    # kill: def $al killed $al killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: test_v16i8:
 ; AVX:       # %bb.0:
@@ -1655,18 +1727,18 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v32i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmaxsb %xmm1, %xmm0
-; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrlw $8, %xmm1
-; SSE41-NEXT:    pminub %xmm0, %xmm1
-; SSE41-NEXT:    phminposuw %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    xorb $127, %al
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v32i8:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pmaxsb %xmm1, %xmm0
+; SSE4-NEXT:    pxor {{.*}}(%rip), %xmm0
+; SSE4-NEXT:    movdqa %xmm0, %xmm1
+; SSE4-NEXT:    psrlw $8, %xmm1
+; SSE4-NEXT:    pminub %xmm0, %xmm1
+; SSE4-NEXT:    phminposuw %xmm1, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    xorb $127, %al
+; SSE4-NEXT:    # kill: def $al killed $al killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v32i8:
 ; AVX1:       # %bb.0:
@@ -1761,20 +1833,20 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v64i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmaxsb %xmm3, %xmm1
-; SSE41-NEXT:    pmaxsb %xmm2, %xmm1
-; SSE41-NEXT:    pmaxsb %xmm0, %xmm1
-; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrlw $8, %xmm0
-; SSE41-NEXT:    pminub %xmm1, %xmm0
-; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    xorb $127, %al
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v64i8:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pmaxsb %xmm3, %xmm1
+; SSE4-NEXT:    pmaxsb %xmm2, %xmm1
+; SSE4-NEXT:    pmaxsb %xmm0, %xmm1
+; SSE4-NEXT:    pxor {{.*}}(%rip), %xmm1
+; SSE4-NEXT:    movdqa %xmm1, %xmm0
+; SSE4-NEXT:    psrlw $8, %xmm0
+; SSE4-NEXT:    pminub %xmm1, %xmm0
+; SSE4-NEXT:    phminposuw %xmm0, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    xorb $127, %al
+; SSE4-NEXT:    # kill: def $al killed $al killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v64i8:
 ; AVX1:       # %bb.0:
@@ -1895,24 +1967,24 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v128i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmaxsb %xmm7, %xmm3
-; SSE41-NEXT:    pmaxsb %xmm5, %xmm3
-; SSE41-NEXT:    pmaxsb %xmm1, %xmm3
-; SSE41-NEXT:    pmaxsb %xmm6, %xmm2
-; SSE41-NEXT:    pmaxsb %xmm4, %xmm2
-; SSE41-NEXT:    pmaxsb %xmm3, %xmm2
-; SSE41-NEXT:    pmaxsb %xmm0, %xmm2
-; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm2
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    psrlw $8, %xmm0
-; SSE41-NEXT:    pminub %xmm2, %xmm0
-; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    xorb $127, %al
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v128i8:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pmaxsb %xmm7, %xmm3
+; SSE4-NEXT:    pmaxsb %xmm5, %xmm3
+; SSE4-NEXT:    pmaxsb %xmm1, %xmm3
+; SSE4-NEXT:    pmaxsb %xmm6, %xmm2
+; SSE4-NEXT:    pmaxsb %xmm4, %xmm2
+; SSE4-NEXT:    pmaxsb %xmm3, %xmm2
+; SSE4-NEXT:    pmaxsb %xmm0, %xmm2
+; SSE4-NEXT:    pxor {{.*}}(%rip), %xmm2
+; SSE4-NEXT:    movdqa %xmm2, %xmm0
+; SSE4-NEXT:    psrlw $8, %xmm0
+; SSE4-NEXT:    pminub %xmm2, %xmm0
+; SSE4-NEXT:    phminposuw %xmm0, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    xorb $127, %al
+; SSE4-NEXT:    # kill: def $al killed $al killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v128i8:
 ; AVX1:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-reduce-smin.ll b/llvm/test/CodeGen/X86/vector-reduce-smin.ll
index edd6b7f7fd329..47a6ac4ce45df 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-smin.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-smin.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE4 --check-prefix=SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.2 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE4 --check-prefix=SSE42
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512BW
@@ -49,6 +50,16 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ; SSE41-NEXT:    movq %xmm2, %rax
 ; SSE41-NEXT:    retq
 ;
+; SSE42-LABEL: test_v2i64:
+; SSE42:       # %bb.0:
+; SSE42-NEXT:    movdqa %xmm0, %xmm1
+; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE42-NEXT:    movdqa %xmm2, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE42-NEXT:    movq %xmm2, %rax
+; SSE42-NEXT:    retq
+;
 ; AVX-LABEL: test_v2i64:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
@@ -141,6 +152,19 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; SSE41-NEXT:    movq %xmm2, %rax
 ; SSE41-NEXT:    retq
 ;
+; SSE42-LABEL: test_v4i64:
+; SSE42:       # %bb.0:
+; SSE42-NEXT:    movdqa %xmm0, %xmm2
+; SSE42-NEXT:    movdqa %xmm1, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm2, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE42-NEXT:    movdqa %xmm2, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE42-NEXT:    movq %xmm2, %rax
+; SSE42-NEXT:    retq
+;
 ; AVX1-LABEL: test_v4i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
@@ -307,6 +331,25 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; SSE41-NEXT:    movq %xmm1, %rax
 ; SSE41-NEXT:    retq
 ;
+; SSE42-LABEL: test_v8i64:
+; SSE42:       # %bb.0:
+; SSE42-NEXT:    movdqa %xmm0, %xmm4
+; SSE42-NEXT:    movdqa %xmm2, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm4, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
+; SSE42-NEXT:    movdqa %xmm3, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
+; SSE42-NEXT:    movapd %xmm3, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm2, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
+; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
+; SSE42-NEXT:    movdqa %xmm1, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm3, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
+; SSE42-NEXT:    movq %xmm1, %rax
+; SSE42-NEXT:    retq
+;
 ; AVX1-LABEL: test_v8i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
@@ -587,6 +630,37 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; SSE41-NEXT:    movq %xmm1, %rax
 ; SSE41-NEXT:    retq
 ;
+; SSE42-LABEL: test_v16i64:
+; SSE42:       # %bb.0:
+; SSE42-NEXT:    movdqa %xmm0, %xmm8
+; SSE42-NEXT:    movdqa %xmm5, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm1, %xmm5
+; SSE42-NEXT:    movdqa %xmm7, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm3, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm3, %xmm7
+; SSE42-NEXT:    movdqa %xmm4, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm8, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm8, %xmm4
+; SSE42-NEXT:    movdqa %xmm6, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm2, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm6
+; SSE42-NEXT:    movapd %xmm6, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm4, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm4, %xmm6
+; SSE42-NEXT:    movapd %xmm7, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm5, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm5, %xmm7
+; SSE42-NEXT:    movapd %xmm7, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm6, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm6, %xmm7
+; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[2,3,0,1]
+; SSE42-NEXT:    movdqa %xmm1, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm7, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm7, %xmm1
+; SSE42-NEXT:    movq %xmm1, %rax
+; SSE42-NEXT:    retq
+;
 ; AVX1-LABEL: test_v16i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm11
@@ -677,12 +751,12 @@ define i32 @test_v2i32(<2 x i32> %a0) {
 ; SSE2-NEXT:    movd %xmm2, %eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v2i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE41-NEXT:    pminsd %xmm0, %xmm1
-; SSE41-NEXT:    movd %xmm1, %eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v2i32:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pminsd %xmm0, %xmm1
+; SSE4-NEXT:    movd %xmm1, %eax
+; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i32:
 ; AVX:       # %bb.0:
@@ -719,14 +793,14 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v4i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pminsd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE41-NEXT:    pminsd %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v4i32:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE4-NEXT:    pminsd %xmm0, %xmm1
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE4-NEXT:    pminsd %xmm1, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: test_v4i32:
 ; AVX:       # %bb.0:
@@ -772,15 +846,15 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE2-NEXT:    movd %xmm2, %eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v8i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pminsd %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pminsd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE41-NEXT:    pminsd %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v8i32:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pminsd %xmm1, %xmm0
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE4-NEXT:    pminsd %xmm0, %xmm1
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE4-NEXT:    pminsd %xmm1, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v8i32:
 ; AVX1:       # %bb.0:
@@ -854,17 +928,17 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v16i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pminsd %xmm3, %xmm1
-; SSE41-NEXT:    pminsd %xmm2, %xmm1
-; SSE41-NEXT:    pminsd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
-; SSE41-NEXT:    pminsd %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE41-NEXT:    pminsd %xmm0, %xmm1
-; SSE41-NEXT:    movd %xmm1, %eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v16i32:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pminsd %xmm3, %xmm1
+; SSE4-NEXT:    pminsd %xmm2, %xmm1
+; SSE4-NEXT:    pminsd %xmm0, %xmm1
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE4-NEXT:    pminsd %xmm1, %xmm0
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pminsd %xmm0, %xmm1
+; SSE4-NEXT:    movd %xmm1, %eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v16i32:
 ; AVX1:       # %bb.0:
@@ -964,21 +1038,21 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v32i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pminsd %xmm6, %xmm2
-; SSE41-NEXT:    pminsd %xmm7, %xmm3
-; SSE41-NEXT:    pminsd %xmm5, %xmm3
-; SSE41-NEXT:    pminsd %xmm1, %xmm3
-; SSE41-NEXT:    pminsd %xmm4, %xmm2
-; SSE41-NEXT:    pminsd %xmm3, %xmm2
-; SSE41-NEXT:    pminsd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
-; SSE41-NEXT:    pminsd %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE41-NEXT:    pminsd %xmm0, %xmm1
-; SSE41-NEXT:    movd %xmm1, %eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v32i32:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pminsd %xmm6, %xmm2
+; SSE4-NEXT:    pminsd %xmm7, %xmm3
+; SSE4-NEXT:    pminsd %xmm5, %xmm3
+; SSE4-NEXT:    pminsd %xmm1, %xmm3
+; SSE4-NEXT:    pminsd %xmm4, %xmm2
+; SSE4-NEXT:    pminsd %xmm3, %xmm2
+; SSE4-NEXT:    pminsd %xmm0, %xmm2
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE4-NEXT:    pminsd %xmm2, %xmm0
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pminsd %xmm0, %xmm1
+; SSE4-NEXT:    movd %xmm1, %eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v32i32:
 ; AVX1:       # %bb.0:
@@ -1116,14 +1190,14 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v8i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    xorl $32768, %eax # imm = 0x8000
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v8i16:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pxor {{.*}}(%rip), %xmm0
+; SSE4-NEXT:    phminposuw %xmm0, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    xorl $32768, %eax # imm = 0x8000
+; SSE4-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: test_v8i16:
 ; AVX:       # %bb.0:
@@ -1161,15 +1235,15 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v16i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pminsw %xmm1, %xmm0
-; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    xorl $32768, %eax # imm = 0x8000
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v16i16:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pminsw %xmm1, %xmm0
+; SSE4-NEXT:    pxor {{.*}}(%rip), %xmm0
+; SSE4-NEXT:    phminposuw %xmm0, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    xorl $32768, %eax # imm = 0x8000
+; SSE4-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v16i16:
 ; AVX1:       # %bb.0:
@@ -1227,17 +1301,17 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v32i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pminsw %xmm3, %xmm1
-; SSE41-NEXT:    pminsw %xmm2, %xmm1
-; SSE41-NEXT:    pminsw %xmm0, %xmm1
-; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm1
-; SSE41-NEXT:    phminposuw %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    xorl $32768, %eax # imm = 0x8000
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v32i16:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pminsw %xmm3, %xmm1
+; SSE4-NEXT:    pminsw %xmm2, %xmm1
+; SSE4-NEXT:    pminsw %xmm0, %xmm1
+; SSE4-NEXT:    pxor {{.*}}(%rip), %xmm1
+; SSE4-NEXT:    phminposuw %xmm1, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    xorl $32768, %eax # imm = 0x8000
+; SSE4-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v32i16:
 ; AVX1:       # %bb.0:
@@ -1305,21 +1379,21 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v64i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pminsw %xmm7, %xmm3
-; SSE41-NEXT:    pminsw %xmm5, %xmm3
-; SSE41-NEXT:    pminsw %xmm1, %xmm3
-; SSE41-NEXT:    pminsw %xmm6, %xmm2
-; SSE41-NEXT:    pminsw %xmm4, %xmm2
-; SSE41-NEXT:    pminsw %xmm3, %xmm2
-; SSE41-NEXT:    pminsw %xmm0, %xmm2
-; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm2
-; SSE41-NEXT:    phminposuw %xmm2, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    xorl $32768, %eax # imm = 0x8000
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v64i16:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pminsw %xmm7, %xmm3
+; SSE4-NEXT:    pminsw %xmm5, %xmm3
+; SSE4-NEXT:    pminsw %xmm1, %xmm3
+; SSE4-NEXT:    pminsw %xmm6, %xmm2
+; SSE4-NEXT:    pminsw %xmm4, %xmm2
+; SSE4-NEXT:    pminsw %xmm3, %xmm2
+; SSE4-NEXT:    pminsw %xmm0, %xmm2
+; SSE4-NEXT:    pxor {{.*}}(%rip), %xmm2
+; SSE4-NEXT:    phminposuw %xmm2, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    xorl $32768, %eax # imm = 0x8000
+; SSE4-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v64i16:
 ; AVX1:       # %bb.0:
@@ -1393,14 +1467,14 @@ define i8 @test_v2i8(<2 x i8> %a0) {
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v2i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrlw $8, %xmm1
-; SSE41-NEXT:    pminsb %xmm0, %xmm1
-; SSE41-NEXT:    movd %xmm1, %eax
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v2i8:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    movdqa %xmm0, %xmm1
+; SSE4-NEXT:    psrlw $8, %xmm1
+; SSE4-NEXT:    pminsb %xmm0, %xmm1
+; SSE4-NEXT:    movd %xmm1, %eax
+; SSE4-NEXT:    # kill: def $al killed $al killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i8:
 ; AVX:       # %bb.0:
@@ -1442,17 +1516,17 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v4i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrld $16, %xmm1
-; SSE41-NEXT:    pminsb %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrlw $8, %xmm0
-; SSE41-NEXT:    pminsb %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v4i8:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    movdqa %xmm0, %xmm1
+; SSE4-NEXT:    psrld $16, %xmm1
+; SSE4-NEXT:    pminsb %xmm0, %xmm1
+; SSE4-NEXT:    movdqa %xmm1, %xmm0
+; SSE4-NEXT:    psrlw $8, %xmm0
+; SSE4-NEXT:    pminsb %xmm1, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    # kill: def $al killed $al killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: test_v4i8:
 ; AVX:       # %bb.0:
@@ -1504,19 +1578,19 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v8i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE41-NEXT:    pminsb %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrld $16, %xmm0
-; SSE41-NEXT:    pminsb %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrlw $8, %xmm1
-; SSE41-NEXT:    pminsb %xmm0, %xmm1
-; SSE41-NEXT:    movd %xmm1, %eax
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v8i8:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pminsb %xmm0, %xmm1
+; SSE4-NEXT:    movdqa %xmm1, %xmm0
+; SSE4-NEXT:    psrld $16, %xmm0
+; SSE4-NEXT:    pminsb %xmm1, %xmm0
+; SSE4-NEXT:    movdqa %xmm0, %xmm1
+; SSE4-NEXT:    psrlw $8, %xmm1
+; SSE4-NEXT:    pminsb %xmm0, %xmm1
+; SSE4-NEXT:    movd %xmm1, %eax
+; SSE4-NEXT:    # kill: def $al killed $al killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: test_v8i8:
 ; AVX:       # %bb.0:
@@ -1578,17 +1652,17 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v16i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrlw $8, %xmm1
-; SSE41-NEXT:    pminub %xmm0, %xmm1
-; SSE41-NEXT:    phminposuw %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    xorb $-128, %al
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v16i8:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pxor {{.*}}(%rip), %xmm0
+; SSE4-NEXT:    movdqa %xmm0, %xmm1
+; SSE4-NEXT:    psrlw $8, %xmm1
+; SSE4-NEXT:    pminub %xmm0, %xmm1
+; SSE4-NEXT:    phminposuw %xmm1, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    xorb $-128, %al
+; SSE4-NEXT:    # kill: def $al killed $al killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: test_v16i8:
 ; AVX:       # %bb.0:
@@ -1653,18 +1727,18 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v32i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pminsb %xmm1, %xmm0
-; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrlw $8, %xmm1
-; SSE41-NEXT:    pminub %xmm0, %xmm1
-; SSE41-NEXT:    phminposuw %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    xorb $-128, %al
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v32i8:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pminsb %xmm1, %xmm0
+; SSE4-NEXT:    pxor {{.*}}(%rip), %xmm0
+; SSE4-NEXT:    movdqa %xmm0, %xmm1
+; SSE4-NEXT:    psrlw $8, %xmm1
+; SSE4-NEXT:    pminub %xmm0, %xmm1
+; SSE4-NEXT:    phminposuw %xmm1, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    xorb $-128, %al
+; SSE4-NEXT:    # kill: def $al killed $al killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v32i8:
 ; AVX1:       # %bb.0:
@@ -1759,20 +1833,20 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v64i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pminsb %xmm3, %xmm1
-; SSE41-NEXT:    pminsb %xmm2, %xmm1
-; SSE41-NEXT:    pminsb %xmm0, %xmm1
-; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrlw $8, %xmm0
-; SSE41-NEXT:    pminub %xmm1, %xmm0
-; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    xorb $-128, %al
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v64i8:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pminsb %xmm3, %xmm1
+; SSE4-NEXT:    pminsb %xmm2, %xmm1
+; SSE4-NEXT:    pminsb %xmm0, %xmm1
+; SSE4-NEXT:    pxor {{.*}}(%rip), %xmm1
+; SSE4-NEXT:    movdqa %xmm1, %xmm0
+; SSE4-NEXT:    psrlw $8, %xmm0
+; SSE4-NEXT:    pminub %xmm1, %xmm0
+; SSE4-NEXT:    phminposuw %xmm0, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    xorb $-128, %al
+; SSE4-NEXT:    # kill: def $al killed $al killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v64i8:
 ; AVX1:       # %bb.0:
@@ -1893,24 +1967,24 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v128i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pminsb %xmm7, %xmm3
-; SSE41-NEXT:    pminsb %xmm5, %xmm3
-; SSE41-NEXT:    pminsb %xmm1, %xmm3
-; SSE41-NEXT:    pminsb %xmm6, %xmm2
-; SSE41-NEXT:    pminsb %xmm4, %xmm2
-; SSE41-NEXT:    pminsb %xmm3, %xmm2
-; SSE41-NEXT:    pminsb %xmm0, %xmm2
-; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm2
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    psrlw $8, %xmm0
-; SSE41-NEXT:    pminub %xmm2, %xmm0
-; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    xorb $-128, %al
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v128i8:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pminsb %xmm7, %xmm3
+; SSE4-NEXT:    pminsb %xmm5, %xmm3
+; SSE4-NEXT:    pminsb %xmm1, %xmm3
+; SSE4-NEXT:    pminsb %xmm6, %xmm2
+; SSE4-NEXT:    pminsb %xmm4, %xmm2
+; SSE4-NEXT:    pminsb %xmm3, %xmm2
+; SSE4-NEXT:    pminsb %xmm0, %xmm2
+; SSE4-NEXT:    pxor {{.*}}(%rip), %xmm2
+; SSE4-NEXT:    movdqa %xmm2, %xmm0
+; SSE4-NEXT:    psrlw $8, %xmm0
+; SSE4-NEXT:    pminub %xmm2, %xmm0
+; SSE4-NEXT:    phminposuw %xmm0, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    xorb $-128, %al
+; SSE4-NEXT:    # kill: def $al killed $al killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v128i8:
 ; AVX1:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-reduce-umax.ll b/llvm/test/CodeGen/X86/vector-reduce-umax.ll
index 59bcf2da6049c..75e8133581338 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-umax.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-umax.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE4 --check-prefix=SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.2 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE4 --check-prefix=SSE42
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512BW
@@ -50,6 +51,18 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ; SSE41-NEXT:    movq %xmm2, %rax
 ; SSE41-NEXT:    retq
 ;
+; SSE42-LABEL: test_v2i64:
+; SSE42:       # %bb.0:
+; SSE42-NEXT:    movdqa %xmm0, %xmm1
+; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE42-NEXT:    movdqa {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
+; SSE42-NEXT:    pxor %xmm3, %xmm0
+; SSE42-NEXT:    pxor %xmm2, %xmm3
+; SSE42-NEXT:    pcmpgtq %xmm3, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE42-NEXT:    movq %xmm2, %rax
+; SSE42-NEXT:    retq
+;
 ; AVX-LABEL: test_v2i64:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
@@ -146,6 +159,24 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; SSE41-NEXT:    movq %xmm2, %rax
 ; SSE41-NEXT:    retq
 ;
+; SSE42-LABEL: test_v4i64:
+; SSE42:       # %bb.0:
+; SSE42-NEXT:    movdqa %xmm0, %xmm2
+; SSE42-NEXT:    movdqa {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
+; SSE42-NEXT:    movdqa %xmm1, %xmm4
+; SSE42-NEXT:    pxor %xmm3, %xmm4
+; SSE42-NEXT:    pxor %xmm3, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm4, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE42-NEXT:    movdqa %xmm1, %xmm0
+; SSE42-NEXT:    pxor %xmm3, %xmm0
+; SSE42-NEXT:    pxor %xmm2, %xmm3
+; SSE42-NEXT:    pcmpgtq %xmm3, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE42-NEXT:    movq %xmm2, %rax
+; SSE42-NEXT:    retq
+;
 ; AVX1-LABEL: test_v4i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
@@ -322,6 +353,36 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; SSE41-NEXT:    movq %xmm1, %rax
 ; SSE41-NEXT:    retq
 ;
+; SSE42-LABEL: test_v8i64:
+; SSE42:       # %bb.0:
+; SSE42-NEXT:    movdqa %xmm0, %xmm4
+; SSE42-NEXT:    movdqa {{.*#+}} xmm5 = [9223372036854775808,9223372036854775808]
+; SSE42-NEXT:    movdqa %xmm2, %xmm6
+; SSE42-NEXT:    pxor %xmm5, %xmm6
+; SSE42-NEXT:    pxor %xmm5, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm6, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
+; SSE42-NEXT:    movdqa %xmm3, %xmm4
+; SSE42-NEXT:    pxor %xmm5, %xmm4
+; SSE42-NEXT:    movdqa %xmm1, %xmm0
+; SSE42-NEXT:    pxor %xmm5, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm4, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
+; SSE42-NEXT:    movapd %xmm3, %xmm1
+; SSE42-NEXT:    xorpd %xmm5, %xmm1
+; SSE42-NEXT:    movapd %xmm2, %xmm0
+; SSE42-NEXT:    xorpd %xmm5, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
+; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
+; SSE42-NEXT:    movdqa %xmm3, %xmm0
+; SSE42-NEXT:    pxor %xmm5, %xmm0
+; SSE42-NEXT:    pxor %xmm1, %xmm5
+; SSE42-NEXT:    pcmpgtq %xmm5, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
+; SSE42-NEXT:    movq %xmm1, %rax
+; SSE42-NEXT:    retq
+;
 ; AVX1-LABEL: test_v8i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
@@ -618,6 +679,61 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; SSE41-NEXT:    movq %xmm1, %rax
 ; SSE41-NEXT:    retq
 ;
+; SSE42-LABEL: test_v16i64:
+; SSE42:       # %bb.0:
+; SSE42-NEXT:    movdqa %xmm0, %xmm8
+; SSE42-NEXT:    movdqa {{.*#+}} xmm9 = [9223372036854775808,9223372036854775808]
+; SSE42-NEXT:    movdqa %xmm5, %xmm10
+; SSE42-NEXT:    pxor %xmm9, %xmm10
+; SSE42-NEXT:    movdqa %xmm1, %xmm0
+; SSE42-NEXT:    pxor %xmm9, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm10, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm1, %xmm5
+; SSE42-NEXT:    movdqa %xmm7, %xmm1
+; SSE42-NEXT:    pxor %xmm9, %xmm1
+; SSE42-NEXT:    movdqa %xmm3, %xmm0
+; SSE42-NEXT:    pxor %xmm9, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm3, %xmm7
+; SSE42-NEXT:    movdqa %xmm4, %xmm1
+; SSE42-NEXT:    pxor %xmm9, %xmm1
+; SSE42-NEXT:    movdqa %xmm8, %xmm0
+; SSE42-NEXT:    pxor %xmm9, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm8, %xmm4
+; SSE42-NEXT:    movdqa %xmm6, %xmm1
+; SSE42-NEXT:    pxor %xmm9, %xmm1
+; SSE42-NEXT:    movdqa %xmm2, %xmm0
+; SSE42-NEXT:    pxor %xmm9, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm6
+; SSE42-NEXT:    movapd %xmm6, %xmm1
+; SSE42-NEXT:    xorpd %xmm9, %xmm1
+; SSE42-NEXT:    movapd %xmm4, %xmm0
+; SSE42-NEXT:    xorpd %xmm9, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm4, %xmm6
+; SSE42-NEXT:    movapd %xmm7, %xmm1
+; SSE42-NEXT:    xorpd %xmm9, %xmm1
+; SSE42-NEXT:    movapd %xmm5, %xmm0
+; SSE42-NEXT:    xorpd %xmm9, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm5, %xmm7
+; SSE42-NEXT:    movapd %xmm7, %xmm1
+; SSE42-NEXT:    xorpd %xmm9, %xmm1
+; SSE42-NEXT:    movapd %xmm6, %xmm0
+; SSE42-NEXT:    xorpd %xmm9, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm6, %xmm7
+; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[2,3,0,1]
+; SSE42-NEXT:    movdqa %xmm7, %xmm0
+; SSE42-NEXT:    pxor %xmm9, %xmm0
+; SSE42-NEXT:    pxor %xmm1, %xmm9
+; SSE42-NEXT:    pcmpgtq %xmm9, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm7, %xmm1
+; SSE42-NEXT:    movq %xmm1, %rax
+; SSE42-NEXT:    retq
+;
 ; AVX1-LABEL: test_v16i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm8
@@ -739,12 +855,12 @@ define i32 @test_v2i32(<2 x i32> %a0) {
 ; SSE2-NEXT:    movd %xmm3, %eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v2i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE41-NEXT:    pmaxud %xmm0, %xmm1
-; SSE41-NEXT:    movd %xmm1, %eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v2i32:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pmaxud %xmm0, %xmm1
+; SSE4-NEXT:    movd %xmm1, %eax
+; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i32:
 ; AVX:       # %bb.0:
@@ -787,14 +903,14 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v4i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pmaxud %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE41-NEXT:    pmaxud %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v4i32:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE4-NEXT:    pmaxud %xmm0, %xmm1
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE4-NEXT:    pmaxud %xmm1, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: test_v4i32:
 ; AVX:       # %bb.0:
@@ -849,15 +965,15 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE2-NEXT:    movd %xmm3, %eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v8i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmaxud %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pmaxud %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE41-NEXT:    pmaxud %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v8i32:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pmaxud %xmm1, %xmm0
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE4-NEXT:    pmaxud %xmm0, %xmm1
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE4-NEXT:    pmaxud %xmm1, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v8i32:
 ; AVX1:       # %bb.0:
@@ -946,17 +1062,17 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v16i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmaxud %xmm3, %xmm1
-; SSE41-NEXT:    pmaxud %xmm2, %xmm1
-; SSE41-NEXT:    pmaxud %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
-; SSE41-NEXT:    pmaxud %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE41-NEXT:    pmaxud %xmm0, %xmm1
-; SSE41-NEXT:    movd %xmm1, %eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v16i32:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pmaxud %xmm3, %xmm1
+; SSE4-NEXT:    pmaxud %xmm2, %xmm1
+; SSE4-NEXT:    pmaxud %xmm0, %xmm1
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE4-NEXT:    pmaxud %xmm1, %xmm0
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pmaxud %xmm0, %xmm1
+; SSE4-NEXT:    movd %xmm1, %eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v16i32:
 ; AVX1:       # %bb.0:
@@ -1083,21 +1199,21 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE2-NEXT:    movd %xmm2, %eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v32i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmaxud %xmm6, %xmm2
-; SSE41-NEXT:    pmaxud %xmm7, %xmm3
-; SSE41-NEXT:    pmaxud %xmm5, %xmm3
-; SSE41-NEXT:    pmaxud %xmm1, %xmm3
-; SSE41-NEXT:    pmaxud %xmm4, %xmm2
-; SSE41-NEXT:    pmaxud %xmm3, %xmm2
-; SSE41-NEXT:    pmaxud %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
-; SSE41-NEXT:    pmaxud %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE41-NEXT:    pmaxud %xmm0, %xmm1
-; SSE41-NEXT:    movd %xmm1, %eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v32i32:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pmaxud %xmm6, %xmm2
+; SSE4-NEXT:    pmaxud %xmm7, %xmm3
+; SSE4-NEXT:    pmaxud %xmm5, %xmm3
+; SSE4-NEXT:    pmaxud %xmm1, %xmm3
+; SSE4-NEXT:    pmaxud %xmm4, %xmm2
+; SSE4-NEXT:    pmaxud %xmm3, %xmm2
+; SSE4-NEXT:    pmaxud %xmm0, %xmm2
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE4-NEXT:    pmaxud %xmm2, %xmm0
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pmaxud %xmm0, %xmm1
+; SSE4-NEXT:    movd %xmm1, %eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v32i32:
 ; AVX1:       # %bb.0:
@@ -1171,14 +1287,14 @@ define i16 @test_v2i16(<2 x i16> %a0) {
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v2i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrld $16, %xmm1
-; SSE41-NEXT:    pmaxuw %xmm0, %xmm1
-; SSE41-NEXT:    movd %xmm1, %eax
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v2i16:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    movdqa %xmm0, %xmm1
+; SSE4-NEXT:    psrld $16, %xmm1
+; SSE4-NEXT:    pmaxuw %xmm0, %xmm1
+; SSE4-NEXT:    movd %xmm1, %eax
+; SSE4-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i16:
 ; AVX:       # %bb.0:
@@ -1215,16 +1331,16 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v4i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE41-NEXT:    pmaxuw %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrld $16, %xmm0
-; SSE41-NEXT:    pmaxuw %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v4i16:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pmaxuw %xmm0, %xmm1
+; SSE4-NEXT:    movdqa %xmm1, %xmm0
+; SSE4-NEXT:    psrld $16, %xmm0
+; SSE4-NEXT:    pmaxuw %xmm1, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: test_v4i16:
 ; AVX:       # %bb.0:
@@ -1267,15 +1383,15 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v8i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT:    pxor %xmm0, %xmm1
-; SSE41-NEXT:    phminposuw %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    notl %eax
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v8i16:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE4-NEXT:    pxor %xmm0, %xmm1
+; SSE4-NEXT:    phminposuw %xmm1, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    notl %eax
+; SSE4-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: test_v8i16:
 ; AVX:       # %bb.0:
@@ -1329,16 +1445,16 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v16i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmaxuw %xmm1, %xmm0
-; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT:    pxor %xmm0, %xmm1
-; SSE41-NEXT:    phminposuw %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    notl %eax
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v16i16:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pmaxuw %xmm1, %xmm0
+; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE4-NEXT:    pxor %xmm0, %xmm1
+; SSE4-NEXT:    phminposuw %xmm1, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    notl %eax
+; SSE4-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v16i16:
 ; AVX1:       # %bb.0:
@@ -1416,18 +1532,18 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v32i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmaxuw %xmm3, %xmm1
-; SSE41-NEXT:    pmaxuw %xmm2, %xmm1
-; SSE41-NEXT:    pmaxuw %xmm0, %xmm1
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    notl %eax
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v32i16:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pmaxuw %xmm3, %xmm1
+; SSE4-NEXT:    pmaxuw %xmm2, %xmm1
+; SSE4-NEXT:    pmaxuw %xmm0, %xmm1
+; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
+; SSE4-NEXT:    pxor %xmm1, %xmm0
+; SSE4-NEXT:    phminposuw %xmm0, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    notl %eax
+; SSE4-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v32i16:
 ; AVX1:       # %bb.0:
@@ -1521,22 +1637,22 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v64i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmaxuw %xmm7, %xmm3
-; SSE41-NEXT:    pmaxuw %xmm5, %xmm3
-; SSE41-NEXT:    pmaxuw %xmm1, %xmm3
-; SSE41-NEXT:    pmaxuw %xmm6, %xmm2
-; SSE41-NEXT:    pmaxuw %xmm4, %xmm2
-; SSE41-NEXT:    pmaxuw %xmm3, %xmm2
-; SSE41-NEXT:    pmaxuw %xmm0, %xmm2
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm2, %xmm0
-; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    notl %eax
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v64i16:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pmaxuw %xmm7, %xmm3
+; SSE4-NEXT:    pmaxuw %xmm5, %xmm3
+; SSE4-NEXT:    pmaxuw %xmm1, %xmm3
+; SSE4-NEXT:    pmaxuw %xmm6, %xmm2
+; SSE4-NEXT:    pmaxuw %xmm4, %xmm2
+; SSE4-NEXT:    pmaxuw %xmm3, %xmm2
+; SSE4-NEXT:    pmaxuw %xmm0, %xmm2
+; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
+; SSE4-NEXT:    pxor %xmm2, %xmm0
+; SSE4-NEXT:    phminposuw %xmm0, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    notl %eax
+; SSE4-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v64i16:
 ; AVX1:       # %bb.0:
@@ -1737,18 +1853,18 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v16i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT:    pxor %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrlw $8, %xmm0
-; SSE41-NEXT:    pminub %xmm1, %xmm0
-; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    notb %al
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v16i8:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE4-NEXT:    pxor %xmm0, %xmm1
+; SSE4-NEXT:    movdqa %xmm1, %xmm0
+; SSE4-NEXT:    psrlw $8, %xmm0
+; SSE4-NEXT:    pminub %xmm1, %xmm0
+; SSE4-NEXT:    phminposuw %xmm0, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    notb %al
+; SSE4-NEXT:    # kill: def $al killed $al killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: test_v16i8:
 ; AVX:       # %bb.0:
@@ -1807,19 +1923,19 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v32i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmaxub %xmm1, %xmm0
-; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT:    pxor %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrlw $8, %xmm0
-; SSE41-NEXT:    pminub %xmm1, %xmm0
-; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    notb %al
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v32i8:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pmaxub %xmm1, %xmm0
+; SSE4-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE4-NEXT:    pxor %xmm0, %xmm1
+; SSE4-NEXT:    movdqa %xmm1, %xmm0
+; SSE4-NEXT:    psrlw $8, %xmm0
+; SSE4-NEXT:    pminub %xmm1, %xmm0
+; SSE4-NEXT:    phminposuw %xmm0, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    notb %al
+; SSE4-NEXT:    # kill: def $al killed $al killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v32i8:
 ; AVX1:       # %bb.0:
@@ -1902,21 +2018,21 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v64i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmaxub %xmm3, %xmm1
-; SSE41-NEXT:    pmaxub %xmm2, %xmm1
-; SSE41-NEXT:    pmaxub %xmm0, %xmm1
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrlw $8, %xmm1
-; SSE41-NEXT:    pminub %xmm0, %xmm1
-; SSE41-NEXT:    phminposuw %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    notb %al
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v64i8:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pmaxub %xmm3, %xmm1
+; SSE4-NEXT:    pmaxub %xmm2, %xmm1
+; SSE4-NEXT:    pmaxub %xmm0, %xmm1
+; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
+; SSE4-NEXT:    pxor %xmm1, %xmm0
+; SSE4-NEXT:    movdqa %xmm0, %xmm1
+; SSE4-NEXT:    psrlw $8, %xmm1
+; SSE4-NEXT:    pminub %xmm0, %xmm1
+; SSE4-NEXT:    phminposuw %xmm1, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    notb %al
+; SSE4-NEXT:    # kill: def $al killed $al killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v64i8:
 ; AVX1:       # %bb.0:
@@ -2011,25 +2127,25 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v128i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmaxub %xmm7, %xmm3
-; SSE41-NEXT:    pmaxub %xmm5, %xmm3
-; SSE41-NEXT:    pmaxub %xmm1, %xmm3
-; SSE41-NEXT:    pmaxub %xmm6, %xmm2
-; SSE41-NEXT:    pmaxub %xmm4, %xmm2
-; SSE41-NEXT:    pmaxub %xmm3, %xmm2
-; SSE41-NEXT:    pmaxub %xmm0, %xmm2
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm2, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrlw $8, %xmm1
-; SSE41-NEXT:    pminub %xmm0, %xmm1
-; SSE41-NEXT:    phminposuw %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    notb %al
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v128i8:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pmaxub %xmm7, %xmm3
+; SSE4-NEXT:    pmaxub %xmm5, %xmm3
+; SSE4-NEXT:    pmaxub %xmm1, %xmm3
+; SSE4-NEXT:    pmaxub %xmm6, %xmm2
+; SSE4-NEXT:    pmaxub %xmm4, %xmm2
+; SSE4-NEXT:    pmaxub %xmm3, %xmm2
+; SSE4-NEXT:    pmaxub %xmm0, %xmm2
+; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
+; SSE4-NEXT:    pxor %xmm2, %xmm0
+; SSE4-NEXT:    movdqa %xmm0, %xmm1
+; SSE4-NEXT:    psrlw $8, %xmm1
+; SSE4-NEXT:    pminub %xmm0, %xmm1
+; SSE4-NEXT:    phminposuw %xmm1, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    notb %al
+; SSE4-NEXT:    # kill: def $al killed $al killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v128i8:
 ; AVX1:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-reduce-umin.ll b/llvm/test/CodeGen/X86/vector-reduce-umin.ll
index 25e3938a68811..8f6a7266d97b9 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-umin.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-umin.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE4 --check-prefix=SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.2 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE4 --check-prefix=SSE42
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512BW
@@ -49,6 +50,19 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ; SSE41-NEXT:    movq %xmm2, %rax
 ; SSE41-NEXT:    retq
 ;
+; SSE42-LABEL: test_v2i64:
+; SSE42:       # %bb.0:
+; SSE42-NEXT:    movdqa %xmm0, %xmm1
+; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE42-NEXT:    movdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808]
+; SSE42-NEXT:    movdqa %xmm1, %xmm3
+; SSE42-NEXT:    pxor %xmm0, %xmm3
+; SSE42-NEXT:    pxor %xmm2, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm3, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE42-NEXT:    movq %xmm2, %rax
+; SSE42-NEXT:    retq
+;
 ; AVX-LABEL: test_v2i64:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
@@ -144,6 +158,26 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; SSE41-NEXT:    movq %xmm2, %rax
 ; SSE41-NEXT:    retq
 ;
+; SSE42-LABEL: test_v4i64:
+; SSE42:       # %bb.0:
+; SSE42-NEXT:    movdqa %xmm0, %xmm2
+; SSE42-NEXT:    movdqa {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
+; SSE42-NEXT:    movdqa %xmm0, %xmm4
+; SSE42-NEXT:    pxor %xmm3, %xmm4
+; SSE42-NEXT:    movdqa %xmm1, %xmm0
+; SSE42-NEXT:    pxor %xmm3, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm4, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; SSE42-NEXT:    movdqa %xmm1, %xmm0
+; SSE42-NEXT:    pxor %xmm3, %xmm0
+; SSE42-NEXT:    pxor %xmm2, %xmm3
+; SSE42-NEXT:    pcmpgtq %xmm0, %xmm3
+; SSE42-NEXT:    movdqa %xmm3, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE42-NEXT:    movq %xmm2, %rax
+; SSE42-NEXT:    retq
+;
 ; AVX1-LABEL: test_v4i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808]
@@ -320,6 +354,38 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; SSE41-NEXT:    movq %xmm1, %rax
 ; SSE41-NEXT:    retq
 ;
+; SSE42-LABEL: test_v8i64:
+; SSE42:       # %bb.0:
+; SSE42-NEXT:    movdqa %xmm0, %xmm5
+; SSE42-NEXT:    movdqa {{.*#+}} xmm4 = [9223372036854775808,9223372036854775808]
+; SSE42-NEXT:    movdqa %xmm1, %xmm6
+; SSE42-NEXT:    pxor %xmm4, %xmm6
+; SSE42-NEXT:    movdqa %xmm3, %xmm0
+; SSE42-NEXT:    pxor %xmm4, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm6, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
+; SSE42-NEXT:    movdqa %xmm5, %xmm1
+; SSE42-NEXT:    pxor %xmm4, %xmm1
+; SSE42-NEXT:    movdqa %xmm2, %xmm0
+; SSE42-NEXT:    pxor %xmm4, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm5, %xmm2
+; SSE42-NEXT:    movapd %xmm2, %xmm1
+; SSE42-NEXT:    xorpd %xmm4, %xmm1
+; SSE42-NEXT:    movapd %xmm3, %xmm0
+; SSE42-NEXT:    xorpd %xmm4, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
+; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
+; SSE42-NEXT:    movdqa %xmm3, %xmm0
+; SSE42-NEXT:    pxor %xmm4, %xmm0
+; SSE42-NEXT:    pxor %xmm1, %xmm4
+; SSE42-NEXT:    pcmpgtq %xmm0, %xmm4
+; SSE42-NEXT:    movdqa %xmm4, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
+; SSE42-NEXT:    movq %xmm1, %rax
+; SSE42-NEXT:    retq
+;
 ; AVX1-LABEL: test_v8i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
@@ -616,6 +682,62 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; SSE41-NEXT:    movq %xmm1, %rax
 ; SSE41-NEXT:    retq
 ;
+; SSE42-LABEL: test_v16i64:
+; SSE42:       # %bb.0:
+; SSE42-NEXT:    movdqa %xmm0, %xmm9
+; SSE42-NEXT:    movdqa {{.*#+}} xmm8 = [9223372036854775808,9223372036854775808]
+; SSE42-NEXT:    movdqa %xmm2, %xmm10
+; SSE42-NEXT:    pxor %xmm8, %xmm10
+; SSE42-NEXT:    movdqa %xmm6, %xmm0
+; SSE42-NEXT:    pxor %xmm8, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm10, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm6
+; SSE42-NEXT:    movdqa %xmm9, %xmm2
+; SSE42-NEXT:    pxor %xmm8, %xmm2
+; SSE42-NEXT:    movdqa %xmm4, %xmm0
+; SSE42-NEXT:    pxor %xmm8, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm2, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm9, %xmm4
+; SSE42-NEXT:    movdqa %xmm3, %xmm2
+; SSE42-NEXT:    pxor %xmm8, %xmm2
+; SSE42-NEXT:    movdqa %xmm7, %xmm0
+; SSE42-NEXT:    pxor %xmm8, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm2, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm3, %xmm7
+; SSE42-NEXT:    movdqa %xmm1, %xmm2
+; SSE42-NEXT:    pxor %xmm8, %xmm2
+; SSE42-NEXT:    movdqa %xmm5, %xmm0
+; SSE42-NEXT:    pxor %xmm8, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm2, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm1, %xmm5
+; SSE42-NEXT:    movapd %xmm5, %xmm1
+; SSE42-NEXT:    xorpd %xmm8, %xmm1
+; SSE42-NEXT:    movapd %xmm7, %xmm0
+; SSE42-NEXT:    xorpd %xmm8, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm5, %xmm7
+; SSE42-NEXT:    movapd %xmm4, %xmm1
+; SSE42-NEXT:    xorpd %xmm8, %xmm1
+; SSE42-NEXT:    movapd %xmm6, %xmm0
+; SSE42-NEXT:    xorpd %xmm8, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm4, %xmm6
+; SSE42-NEXT:    movapd %xmm6, %xmm1
+; SSE42-NEXT:    xorpd %xmm8, %xmm1
+; SSE42-NEXT:    movapd %xmm7, %xmm0
+; SSE42-NEXT:    xorpd %xmm8, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm6, %xmm7
+; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[2,3,0,1]
+; SSE42-NEXT:    movdqa %xmm7, %xmm0
+; SSE42-NEXT:    pxor %xmm8, %xmm0
+; SSE42-NEXT:    pxor %xmm1, %xmm8
+; SSE42-NEXT:    pcmpgtq %xmm0, %xmm8
+; SSE42-NEXT:    movdqa %xmm8, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm7, %xmm1
+; SSE42-NEXT:    movq %xmm1, %rax
+; SSE42-NEXT:    retq
+;
 ; AVX1-LABEL: test_v16i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [9223372036854775808,9223372036854775808]
@@ -737,12 +859,12 @@ define i32 @test_v2i32(<2 x i32> %a0) {
 ; SSE2-NEXT:    movd %xmm2, %eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v2i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE41-NEXT:    pminud %xmm0, %xmm1
-; SSE41-NEXT:    movd %xmm1, %eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v2i32:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pminud %xmm0, %xmm1
+; SSE4-NEXT:    movd %xmm1, %eax
+; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i32:
 ; AVX:       # %bb.0:
@@ -785,14 +907,14 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; SSE2-NEXT:    movd %xmm2, %eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v4i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pminud %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE41-NEXT:    pminud %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v4i32:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE4-NEXT:    pminud %xmm0, %xmm1
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE4-NEXT:    pminud %xmm1, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: test_v4i32:
 ; AVX:       # %bb.0:
@@ -847,15 +969,15 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; SSE2-NEXT:    movd %xmm2, %eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v8i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pminud %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pminud %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; SSE41-NEXT:    pminud %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v8i32:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pminud %xmm1, %xmm0
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE4-NEXT:    pminud %xmm0, %xmm1
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE4-NEXT:    pminud %xmm1, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v8i32:
 ; AVX1:       # %bb.0:
@@ -944,17 +1066,17 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; SSE2-NEXT:    movd %xmm4, %eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v16i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pminud %xmm3, %xmm1
-; SSE41-NEXT:    pminud %xmm2, %xmm1
-; SSE41-NEXT:    pminud %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
-; SSE41-NEXT:    pminud %xmm1, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE41-NEXT:    pminud %xmm0, %xmm1
-; SSE41-NEXT:    movd %xmm1, %eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v16i32:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pminud %xmm3, %xmm1
+; SSE4-NEXT:    pminud %xmm2, %xmm1
+; SSE4-NEXT:    pminud %xmm0, %xmm1
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE4-NEXT:    pminud %xmm1, %xmm0
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pminud %xmm0, %xmm1
+; SSE4-NEXT:    movd %xmm1, %eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v16i32:
 ; AVX1:       # %bb.0:
@@ -1081,21 +1203,21 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; SSE2-NEXT:    movd %xmm8, %eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v32i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pminud %xmm6, %xmm2
-; SSE41-NEXT:    pminud %xmm7, %xmm3
-; SSE41-NEXT:    pminud %xmm5, %xmm3
-; SSE41-NEXT:    pminud %xmm1, %xmm3
-; SSE41-NEXT:    pminud %xmm4, %xmm2
-; SSE41-NEXT:    pminud %xmm3, %xmm2
-; SSE41-NEXT:    pminud %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
-; SSE41-NEXT:    pminud %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE41-NEXT:    pminud %xmm0, %xmm1
-; SSE41-NEXT:    movd %xmm1, %eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v32i32:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pminud %xmm6, %xmm2
+; SSE4-NEXT:    pminud %xmm7, %xmm3
+; SSE4-NEXT:    pminud %xmm5, %xmm3
+; SSE4-NEXT:    pminud %xmm1, %xmm3
+; SSE4-NEXT:    pminud %xmm4, %xmm2
+; SSE4-NEXT:    pminud %xmm3, %xmm2
+; SSE4-NEXT:    pminud %xmm0, %xmm2
+; SSE4-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1]
+; SSE4-NEXT:    pminud %xmm2, %xmm0
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pminud %xmm0, %xmm1
+; SSE4-NEXT:    movd %xmm1, %eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v32i32:
 ; AVX1:       # %bb.0:
@@ -1169,14 +1291,14 @@ define i16 @test_v2i16(<2 x i16> %a0) {
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v2i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrld $16, %xmm1
-; SSE41-NEXT:    pminuw %xmm0, %xmm1
-; SSE41-NEXT:    movd %xmm1, %eax
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v2i16:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    movdqa %xmm0, %xmm1
+; SSE4-NEXT:    psrld $16, %xmm1
+; SSE4-NEXT:    pminuw %xmm0, %xmm1
+; SSE4-NEXT:    movd %xmm1, %eax
+; SSE4-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: test_v2i16:
 ; AVX:       # %bb.0:
@@ -1213,16 +1335,16 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v4i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; SSE41-NEXT:    pminuw %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrld $16, %xmm0
-; SSE41-NEXT:    pminuw %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v4i16:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE4-NEXT:    pminuw %xmm0, %xmm1
+; SSE4-NEXT:    movdqa %xmm1, %xmm0
+; SSE4-NEXT:    psrld $16, %xmm0
+; SSE4-NEXT:    pminuw %xmm1, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: test_v4i16:
 ; AVX:       # %bb.0:
@@ -1265,12 +1387,12 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v8i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v8i16:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    phminposuw %xmm0, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: test_v8i16:
 ; AVX:       # %bb.0:
@@ -1308,13 +1430,13 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v16i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pminuw %xmm1, %xmm0
-; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v16i16:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pminuw %xmm1, %xmm0
+; SSE4-NEXT:    phminposuw %xmm0, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v16i16:
 ; AVX1:       # %bb.0:
@@ -1372,15 +1494,15 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v32i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pminuw %xmm3, %xmm1
-; SSE41-NEXT:    pminuw %xmm2, %xmm1
-; SSE41-NEXT:    pminuw %xmm0, %xmm1
-; SSE41-NEXT:    phminposuw %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v32i16:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pminuw %xmm3, %xmm1
+; SSE4-NEXT:    pminuw %xmm2, %xmm1
+; SSE4-NEXT:    pminuw %xmm0, %xmm1
+; SSE4-NEXT:    phminposuw %xmm1, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v32i16:
 ; AVX1:       # %bb.0:
@@ -1452,19 +1574,19 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v64i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pminuw %xmm7, %xmm3
-; SSE41-NEXT:    pminuw %xmm5, %xmm3
-; SSE41-NEXT:    pminuw %xmm1, %xmm3
-; SSE41-NEXT:    pminuw %xmm6, %xmm2
-; SSE41-NEXT:    pminuw %xmm4, %xmm2
-; SSE41-NEXT:    pminuw %xmm3, %xmm2
-; SSE41-NEXT:    pminuw %xmm0, %xmm2
-; SSE41-NEXT:    phminposuw %xmm2, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v64i16:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pminuw %xmm7, %xmm3
+; SSE4-NEXT:    pminuw %xmm5, %xmm3
+; SSE4-NEXT:    pminuw %xmm1, %xmm3
+; SSE4-NEXT:    pminuw %xmm6, %xmm2
+; SSE4-NEXT:    pminuw %xmm4, %xmm2
+; SSE4-NEXT:    pminuw %xmm3, %xmm2
+; SSE4-NEXT:    pminuw %xmm0, %xmm2
+; SSE4-NEXT:    phminposuw %xmm2, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v64i16:
 ; AVX1:       # %bb.0:
@@ -1642,15 +1764,15 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v16i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrlw $8, %xmm1
-; SSE41-NEXT:    pminub %xmm0, %xmm1
-; SSE41-NEXT:    phminposuw %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v16i8:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    movdqa %xmm0, %xmm1
+; SSE4-NEXT:    psrlw $8, %xmm1
+; SSE4-NEXT:    pminub %xmm0, %xmm1
+; SSE4-NEXT:    phminposuw %xmm1, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    # kill: def $al killed $al killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX-LABEL: test_v16i8:
 ; AVX:       # %bb.0:
@@ -1691,16 +1813,16 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v32i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pminub %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrlw $8, %xmm1
-; SSE41-NEXT:    pminub %xmm0, %xmm1
-; SSE41-NEXT:    phminposuw %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v32i8:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pminub %xmm1, %xmm0
+; SSE4-NEXT:    movdqa %xmm0, %xmm1
+; SSE4-NEXT:    psrlw $8, %xmm1
+; SSE4-NEXT:    pminub %xmm0, %xmm1
+; SSE4-NEXT:    phminposuw %xmm1, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    # kill: def $al killed $al killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v32i8:
 ; AVX1:       # %bb.0:
@@ -1761,18 +1883,18 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v64i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pminub %xmm3, %xmm1
-; SSE41-NEXT:    pminub %xmm2, %xmm1
-; SSE41-NEXT:    pminub %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrlw $8, %xmm0
-; SSE41-NEXT:    pminub %xmm1, %xmm0
-; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v64i8:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pminub %xmm3, %xmm1
+; SSE4-NEXT:    pminub %xmm2, %xmm1
+; SSE4-NEXT:    pminub %xmm0, %xmm1
+; SSE4-NEXT:    movdqa %xmm1, %xmm0
+; SSE4-NEXT:    psrlw $8, %xmm0
+; SSE4-NEXT:    pminub %xmm1, %xmm0
+; SSE4-NEXT:    phminposuw %xmm0, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    # kill: def $al killed $al killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v64i8:
 ; AVX1:       # %bb.0:
@@ -1843,22 +1965,22 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
-; SSE41-LABEL: test_v128i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pminub %xmm7, %xmm3
-; SSE41-NEXT:    pminub %xmm5, %xmm3
-; SSE41-NEXT:    pminub %xmm1, %xmm3
-; SSE41-NEXT:    pminub %xmm6, %xmm2
-; SSE41-NEXT:    pminub %xmm4, %xmm2
-; SSE41-NEXT:    pminub %xmm3, %xmm2
-; SSE41-NEXT:    pminub %xmm0, %xmm2
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    psrlw $8, %xmm0
-; SSE41-NEXT:    pminub %xmm2, %xmm0
-; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
-; SSE41-NEXT:    retq
+; SSE4-LABEL: test_v128i8:
+; SSE4:       # %bb.0:
+; SSE4-NEXT:    pminub %xmm7, %xmm3
+; SSE4-NEXT:    pminub %xmm5, %xmm3
+; SSE4-NEXT:    pminub %xmm1, %xmm3
+; SSE4-NEXT:    pminub %xmm6, %xmm2
+; SSE4-NEXT:    pminub %xmm4, %xmm2
+; SSE4-NEXT:    pminub %xmm3, %xmm2
+; SSE4-NEXT:    pminub %xmm0, %xmm2
+; SSE4-NEXT:    movdqa %xmm2, %xmm0
+; SSE4-NEXT:    psrlw $8, %xmm0
+; SSE4-NEXT:    pminub %xmm2, %xmm0
+; SSE4-NEXT:    phminposuw %xmm0, %xmm0
+; SSE4-NEXT:    movd %xmm0, %eax
+; SSE4-NEXT:    # kill: def $al killed $al killed $eax
+; SSE4-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v128i8:
 ; AVX1:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll b/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll
index 8014f1f415162..8ad5c2138e38e 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll
@@ -365,33 +365,19 @@ define i1 @trunc_v8i32_v8i1(<8 x i32>) {
 }
 
 define i1 @trunc_v16i16_v16i1(<16 x i16>) {
-; SSE2-LABEL: trunc_v16i16_v16i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    packuswb %xmm1, %xmm0
-; SSE2-NEXT:    psllw $7, %xmm0
-; SSE2-NEXT:    pmovmskb %xmm0, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    xorb %al, %cl
-; SSE2-NEXT:    setnp %al
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: trunc_v16i16_v16i1:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; SSE41-NEXT:    pshufb %xmm2, %xmm1
-; SSE41-NEXT:    pshufb %xmm2, %xmm0
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE41-NEXT:    psllw $7, %xmm0
-; SSE41-NEXT:    pmovmskb %xmm0, %eax
-; SSE41-NEXT:    movl %eax, %ecx
-; SSE41-NEXT:    shrl $8, %ecx
-; SSE41-NEXT:    xorb %al, %cl
-; SSE41-NEXT:    setnp %al
-; SSE41-NEXT:    retq
+; SSE-LABEL: trunc_v16i16_v16i1:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE-NEXT:    pand %xmm2, %xmm1
+; SSE-NEXT:    pand %xmm2, %xmm0
+; SSE-NEXT:    packuswb %xmm1, %xmm0
+; SSE-NEXT:    psllw $7, %xmm0
+; SSE-NEXT:    pmovmskb %xmm0, %eax
+; SSE-NEXT:    movl %eax, %ecx
+; SSE-NEXT:    shrl $8, %ecx
+; SSE-NEXT:    xorb %al, %cl
+; SSE-NEXT:    setnp %al
+; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_v16i16_v16i1:
 ; AVX1:       # %bb.0:
@@ -775,41 +761,23 @@ define i1 @trunc_v16i32_v16i1(<16 x i32>) {
 }
 
 define i1 @trunc_v32i16_v32i1(<32 x i16>) {
-; SSE2-LABEL: trunc_v32i16_v32i1:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; SSE2-NEXT:    pand %xmm4, %xmm3
-; SSE2-NEXT:    pand %xmm4, %xmm2
-; SSE2-NEXT:    packuswb %xmm3, %xmm2
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    packuswb %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm2, %xmm0
-; SSE2-NEXT:    psllw $7, %xmm0
-; SSE2-NEXT:    pmovmskb %xmm0, %eax
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    shrl $8, %ecx
-; SSE2-NEXT:    xorb %al, %cl
-; SSE2-NEXT:    setnp %al
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: trunc_v32i16_v32i1:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; SSE41-NEXT:    pshufb %xmm4, %xmm3
-; SSE41-NEXT:    pshufb %xmm4, %xmm2
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE41-NEXT:    pshufb %xmm4, %xmm1
-; SSE41-NEXT:    pshufb %xmm4, %xmm0
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE41-NEXT:    pxor %xmm2, %xmm0
-; SSE41-NEXT:    psllw $7, %xmm0
-; SSE41-NEXT:    pmovmskb %xmm0, %eax
-; SSE41-NEXT:    movl %eax, %ecx
-; SSE41-NEXT:    shrl $8, %ecx
-; SSE41-NEXT:    xorb %al, %cl
-; SSE41-NEXT:    setnp %al
-; SSE41-NEXT:    retq
+; SSE-LABEL: trunc_v32i16_v32i1:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
+; SSE-NEXT:    pand %xmm4, %xmm3
+; SSE-NEXT:    pand %xmm4, %xmm2
+; SSE-NEXT:    packuswb %xmm3, %xmm2
+; SSE-NEXT:    pand %xmm4, %xmm1
+; SSE-NEXT:    pand %xmm4, %xmm0
+; SSE-NEXT:    packuswb %xmm1, %xmm0
+; SSE-NEXT:    pxor %xmm2, %xmm0
+; SSE-NEXT:    psllw $7, %xmm0
+; SSE-NEXT:    pmovmskb %xmm0, %eax
+; SSE-NEXT:    movl %eax, %ecx
+; SSE-NEXT:    shrl $8, %ecx
+; SSE-NEXT:    xorb %al, %cl
+; SSE-NEXT:    setnp %al
+; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_v32i16_v32i1:
 ; AVX1:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll b/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll
index c9329251de311..5ba0efdf7f50b 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll
@@ -973,10 +973,15 @@ define <16 x i8> @trunc_v4i32_shuffle(<16 x i8> %a) {
 ; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
 ; SSE41-NEXT:    retq
 ;
-; AVX-LABEL: trunc_v4i32_shuffle:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX-NEXT:    retq
+; AVX1OR2-LABEL: trunc_v4i32_shuffle:
+; AVX1OR2:       # %bb.0:
+; AVX1OR2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1OR2-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc_v4i32_shuffle:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmovdb %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
   %shuffle = shufflevector <16 x i8> %a, <16 x i8> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
   ret <16 x i8> %shuffle
 }
@@ -1964,59 +1969,37 @@ define <16 x i8> @shuffle_v16i8_02_03_04_05_06_07_00_01_10_11_12_13_14_15_08_09(
 }
 
 define <16 x i8> @PR12412(<16 x i8> %inval1, <16 x i8> %inval2) {
-; SSE2-LABEL: PR12412:
-; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    packuswb %xmm1, %xmm0
-; SSE2-NEXT:    retq
-;
-; SSSE3-LABEL: PR12412:
-; SSSE3:       # %bb.0: # %entry
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; SSSE3-NEXT:    pshufb %xmm2, %xmm1
-; SSSE3-NEXT:    pshufb %xmm2, %xmm0
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSSE3-NEXT:    retq
-;
-; SSE41-LABEL: PR12412:
-; SSE41:       # %bb.0: # %entry
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; SSE41-NEXT:    pshufb %xmm2, %xmm1
-; SSE41-NEXT:    pshufb %xmm2, %xmm0
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE41-NEXT:    retq
+; SSE-LABEL: PR12412:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE-NEXT:    pand %xmm2, %xmm1
+; SSE-NEXT:    pand %xmm2, %xmm0
+; SSE-NEXT:    packuswb %xmm1, %xmm0
+; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: PR12412:
 ; AVX1:       # %bb.0: # %entry
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: PR12412:
 ; AVX2:       # %bb.0: # %entry
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; AVX2-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512VLBW-LABEL: PR12412:
-; AVX512VLBW:       # %bb.0: # %entry
-; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; AVX512VLBW-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512VLBW-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512VLBW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512VLBW-NEXT:    retq
-;
-; AVX512VLVBMI-LABEL: PR12412:
-; AVX512VLVBMI:       # %bb.0: # %entry
-; AVX512VLVBMI-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30]
-; AVX512VLVBMI-NEXT:    vpermt2b %xmm1, %xmm2, %xmm0
-; AVX512VLVBMI-NEXT:    retq
+; AVX512VL-LABEL: PR12412:
+; AVX512VL:       # %bb.0: # %entry
+; AVX512VL-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512VL-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
 ;
 ; XOP-LABEL: PR12412:
 ; XOP:       # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-128-v4.ll b/llvm/test/CodeGen/X86/vector-shuffle-128-v4.ll
index 13ce66312aaac..1f767645921db 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-128-v4.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-128-v4.ll
@@ -1583,11 +1583,18 @@ define <4 x i32> @shuffle_v4i32_2456(<4 x i32> %a, <4 x i32> %b) {
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX-LABEL: shuffle_v4i32_2456:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,2,2]
-; AVX-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10,11]
-; AVX-NEXT:    retq
+; AVX1OR2-LABEL: shuffle_v4i32_2456:
+; AVX1OR2:       # %bb.0:
+; AVX1OR2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,2,2]
+; AVX1OR2-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[12,13,14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10,11]
+; AVX1OR2-NEXT:    retq
+;
+; AVX512VL-LABEL: shuffle_v4i32_2456:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [6,0,1,2]
+; AVX512VL-NEXT:    vpermi2d %xmm0, %xmm1, %xmm2
+; AVX512VL-NEXT:    vmovdqa %xmm2, %xmm0
+; AVX512VL-NEXT:    retq
   %s1 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
   %s2 = shufflevector <4 x i32> %s1, <4 x i32> %b, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
   ret <4 x i32> %s2
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll b/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll
index eb4e2edf157af..80345a564d114 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-128-v8.ll
@@ -2512,6 +2512,170 @@ define <8 x i16> @shuffle_v8i16_12305674(<8 x i16> %a) {
   ret <8 x i16> %shuffle
 }
 
+define <8 x i16> @shuffle_v8i16_02460246(<8 x i16> %a) {
+; SSE2-LABEL: shuffle_v8i16_02460246:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,0,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,4]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,3,2,1]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,1,3,2,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,4,5]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: shuffle_v8i16_02460246:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,0,1,4,5,8,9,12,13]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: shuffle_v8i16_02460246:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,0,1,4,5,8,9,12,13]
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: shuffle_v8i16_02460246:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,0,1,4,5,8,9,12,13]
+; AVX-NEXT:    retq
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 0, i32 2, i32 4, i32 6>
+  ret <8 x i16> %shuffle
+}
+
+define <8 x i16> @shuffle_v8i16_04040404(<8 x i16> %a) {
+; SSE2-LABEL: shuffle_v8i16_04040404:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,0]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,0,2,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,4,6,4]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: shuffle_v8i16_04040404:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,0,1,8,9,0,1,8,9,0,1,8,9]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: shuffle_v8i16_04040404:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,0,1,8,9,0,1,8,9,0,1,8,9]
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: shuffle_v8i16_04040404:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,0,1,8,9,0,1,8,9,0,1,8,9]
+; AVX-NEXT:    retq
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 4, i32 0, i32 4, i32 0, i32 4, i32 0, i32 4>
+  ret <8 x i16> %shuffle
+}
+
+define <8 x i16> @shuffle_v8i16_02468ACE(<8 x i16> %a, <8 x i16> %b) {
+; SSE2-LABEL: shuffle_v8i16_02468ACE:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: shuffle_v8i16_02468ACE:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; SSSE3-NEXT:    pshufb %xmm2, %xmm1
+; SSSE3-NEXT:    pshufb %xmm2, %xmm0
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: shuffle_v8i16_02468ACE:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; SSE41-NEXT:    pshufb %xmm2, %xmm1
+; SSE41-NEXT:    pshufb %xmm2, %xmm0
+; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: shuffle_v8i16_02468ACE:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: shuffle_v8i16_02468ACE:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; AVX2-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX2-NEXT:    retq
+;
+; AVX512VL-LABEL: shuffle_v8i16_02468ACE:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512VL-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
+;
+; XOP-LABEL: shuffle_v8i16_02468ACE:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13],xmm1[0,1,4,5,8,9,12,13]
+; XOP-NEXT:    retq
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  ret <8 x i16> %shuffle
+}
+
+define <8 x i16> @shuffle_v8i16_048C048C(<8 x i16> %a, <8 x i16> %b) {
+; SSE-LABEL: shuffle_v8i16_048C048C:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,0,2,4,5,6,7]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,0,2,4,5,6,7]
+; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: shuffle_v8i16_048C048C:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,2,0,2,4,5,6,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,0,2,4,5,6,7]
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX1-NEXT:    retq
+;
+; AVX2-SLOW-LABEL: shuffle_v8i16_048C048C:
+; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,2,0,2,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,0,2,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-SLOW-NEXT:    retq
+;
+; AVX2-FAST-LABEL: shuffle_v8i16_048C048C:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,8,9,0,1,8,9,8,9,10,11,12,13,14,15]
+; AVX2-FAST-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX2-FAST-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX2-FAST-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-FAST-NEXT:    retq
+;
+; AVX512VL-LABEL: shuffle_v8i16_048C048C:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpbroadcastq {{.*#+}} xmm2 = [3377734080528384,3377734080528384]
+; AVX512VL-NEXT:    vpermt2w %xmm1, %xmm2, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; XOP-LABEL: shuffle_v8i16_048C048C:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[0,1,8,9],xmm1[0,1,8,9],xmm0[0,1,8,9],xmm1[0,1,8,9]
+; XOP-NEXT:    retq
+  %shuffle = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 0, i32 4, i32 8, i32 12>
+  ret <8 x i16> %shuffle
+}
+
 ;
 ; Shuffle to logical bit shifts
 ;
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll
index 8c12c0d2e9fc6..7db96f42b1cb4 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll
@@ -6914,6 +6914,102 @@ define <16 x i16> @shuffle_v16i16_02_18_03_19_10_26_11_27_00_16_01_17_08_24_09_2
   ret <16 x i16> %4
 }
 
+define <16 x i16> @shuffle_v16i16_ashr_00_02_04_06_16_18_20_22_08_10_12_14_24_26_28_30(<8 x i32> %a0, <8 x i32> %a1) {
+; AVX1-LABEL: shuffle_v16i16_ashr_00_02_04_06_16_18_20_22_08_10_12_14_24_26_28_30:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpsrad $25, %xmm0, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpsrad $25, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrad $25, %xmm1, %xmm3
+; AVX1-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT:    vpsrad $25, %xmm1, %xmm1
+; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2OR512VL-LABEL: shuffle_v16i16_ashr_00_02_04_06_16_18_20_22_08_10_12_14_24_26_28_30:
+; AVX2OR512VL:       # %bb.0:
+; AVX2OR512VL-NEXT:    vpsrad $25, %ymm0, %ymm0
+; AVX2OR512VL-NEXT:    vpsrad $25, %ymm1, %ymm1
+; AVX2OR512VL-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
+; AVX2OR512VL-NEXT:    retq
+;
+; XOPAVX1-LABEL: shuffle_v16i16_ashr_00_02_04_06_16_18_20_22_08_10_12_14_24_26_28_30:
+; XOPAVX1:       # %bb.0:
+; XOPAVX1-NEXT:    vpsrad $25, %xmm0, %xmm2
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; XOPAVX1-NEXT:    vpsrad $25, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vpsrad $25, %xmm1, %xmm3
+; XOPAVX1-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; XOPAVX1-NEXT:    vpsrad $25, %xmm1, %xmm1
+; XOPAVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; XOPAVX1-NEXT:    retq
+;
+; XOPAVX2-LABEL: shuffle_v16i16_ashr_00_02_04_06_16_18_20_22_08_10_12_14_24_26_28_30:
+; XOPAVX2:       # %bb.0:
+; XOPAVX2-NEXT:    vpsrad $25, %ymm0, %ymm0
+; XOPAVX2-NEXT:    vpsrad $25, %ymm1, %ymm1
+; XOPAVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
+; XOPAVX2-NEXT:    retq
+  %1 = ashr <8 x i32> %a0, <i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25>
+  %2 = ashr <8 x i32> %a1, <i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25>
+  %3 = bitcast <8 x i32> %1 to <16 x i16>
+  %4 = bitcast <8 x i32> %2 to <16 x i16>
+  %5 = shufflevector <16 x i16> %3, <16 x i16> %4, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+  ret <16 x i16> %5
+}
+
+define <16 x i16> @shuffle_v16i16_lshr_00_02_04_06_16_18_20_22_08_10_12_14_24_26_28_30(<8 x i32> %a0, <8 x i32> %a1) {
+; AVX1-LABEL: shuffle_v16i16_lshr_00_02_04_06_16_18_20_22_08_10_12_14_24_26_28_30:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpsrld $25, %xmm0, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpsrld $25, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrld $25, %xmm1, %xmm3
+; AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT:    vpsrld $25, %xmm1, %xmm1
+; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2OR512VL-LABEL: shuffle_v16i16_lshr_00_02_04_06_16_18_20_22_08_10_12_14_24_26_28_30:
+; AVX2OR512VL:       # %bb.0:
+; AVX2OR512VL-NEXT:    vpsrld $25, %ymm0, %ymm0
+; AVX2OR512VL-NEXT:    vpsrld $25, %ymm1, %ymm1
+; AVX2OR512VL-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2OR512VL-NEXT:    retq
+;
+; XOPAVX1-LABEL: shuffle_v16i16_lshr_00_02_04_06_16_18_20_22_08_10_12_14_24_26_28_30:
+; XOPAVX1:       # %bb.0:
+; XOPAVX1-NEXT:    vpsrld $25, %xmm0, %xmm2
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; XOPAVX1-NEXT:    vpsrld $25, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vpsrld $25, %xmm1, %xmm3
+; XOPAVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; XOPAVX1-NEXT:    vpsrld $25, %xmm1, %xmm1
+; XOPAVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; XOPAVX1-NEXT:    retq
+;
+; XOPAVX2-LABEL: shuffle_v16i16_lshr_00_02_04_06_16_18_20_22_08_10_12_14_24_26_28_30:
+; XOPAVX2:       # %bb.0:
+; XOPAVX2-NEXT:    vpsrld $25, %ymm0, %ymm0
+; XOPAVX2-NEXT:    vpsrld $25, %ymm1, %ymm1
+; XOPAVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; XOPAVX2-NEXT:    retq
+  %1 = lshr <8 x i32> %a0, <i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25>
+  %2 = lshr <8 x i32> %a1, <i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25>
+  %3 = bitcast <8 x i32> %1 to <16 x i16>
+  %4 = bitcast <8 x i32> %2 to <16 x i16>
+  %5 = shufflevector <16 x i16> %3, <16 x i16> %4, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+  ret <16 x i16> %5
+}
+
 define <16 x i16> @shuffle_v16i16_04_06_07_uu_uu_06_07_05_12_14_15_uu_uu_14_15_13(<16 x i16> %a) {
 ; AVX1-LABEL: shuffle_v16i16_04_06_07_uu_uu_06_07_05_12_14_15_uu_uu_14_15_13:
 ; AVX1:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
index 343fd75044e76..5b740277a2e93 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
@@ -4753,14 +4753,14 @@ define <32 x i8> @shuffle_v32i8_00_02_04_06_08_10_12_14_32_34_36_38_40_42_44_46_
 ; AVX1-LABEL: shuffle_v32i8_00_02_04_06_08_10_12_14_32_34_36_38_40_42_44_46_16_18_20_22_24_26_28_30_48_50_52_54_56_58_60_62:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; AVX1-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
-; AVX1-NEXT:    vpshufb %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm4[0],xmm2[0]
-; AVX1-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX1-NEXT:    vpand %xmm3, %xmm4, %xmm4
+; AVX1-NEXT:    vpackuswb %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
@@ -4808,14 +4808,14 @@ define <32 x i8> @shuffle_v32i8_00_02_04_06_08_10_12_14_16_18_20_22_24_26_28_30_
 ; AVX1-LABEL: shuffle_v32i8_00_02_04_06_08_10_12_14_16_18_20_22_24_26_28_30_32_34_36_38_40_42_44_46_48_50_52_54_56_58_60_62:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; AVX1-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
+; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
@@ -4827,27 +4827,12 @@ define <32 x i8> @shuffle_v32i8_00_02_04_06_08_10_12_14_16_18_20_22_24_26_28_30_
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX2-NEXT:    retq
 ;
-; AVX512VLBW-SLOW-LABEL: shuffle_v32i8_00_02_04_06_08_10_12_14_16_18_20_22_24_26_28_30_32_34_36_38_40_42_44_46_48_50_52_54_56_58_60_62:
-; AVX512VLBW-SLOW:       # %bb.0:
-; AVX512VLBW-SLOW-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30]
-; AVX512VLBW-SLOW-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
-; AVX512VLBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
-; AVX512VLBW-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512VLBW-SLOW-NEXT:    retq
-;
-; AVX512VLBW-FAST-LABEL: shuffle_v32i8_00_02_04_06_08_10_12_14_16_18_20_22_24_26_28_30_32_34_36_38_40_42_44_46_48_50_52_54_56_58_60_62:
-; AVX512VLBW-FAST:       # %bb.0:
-; AVX512VLBW-FAST-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30]
-; AVX512VLBW-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = ymm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30,u,u,u,u,u,u,u,u]
-; AVX512VLBW-FAST-NEXT:    vmovdqa {{.*#+}} ymm0 = [0,2,5,7]
-; AVX512VLBW-FAST-NEXT:    vpermi2q %ymm1, %ymm2, %ymm0
-; AVX512VLBW-FAST-NEXT:    retq
-;
-; AVX512VLVBMI-LABEL: shuffle_v32i8_00_02_04_06_08_10_12_14_16_18_20_22_24_26_28_30_32_34_36_38_40_42_44_46_48_50_52_54_56_58_60_62:
-; AVX512VLVBMI:       # %bb.0:
-; AVX512VLVBMI-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62]
-; AVX512VLVBMI-NEXT:    vpermt2b %ymm1, %ymm2, %ymm0
-; AVX512VLVBMI-NEXT:    retq
+; AVX512VL-LABEL: shuffle_v32i8_00_02_04_06_08_10_12_14_16_18_20_22_24_26_28_30_32_34_36_38_40_42_44_46_48_50_52_54_56_58_60_62:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VL-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: shuffle_v32i8_00_02_04_06_08_10_12_14_16_18_20_22_24_26_28_30_32_34_36_38_40_42_44_46_48_50_52_54_56_58_60_62:
 ; XOPAVX1:       # %bb.0:
@@ -4894,10 +4879,10 @@ define <32 x i8> @shuffle_v32i8_shift_00_02_04_06_08_10_12_14_16_18_20_22_24_26_
 ;
 ; AVX512VLBW-LABEL: shuffle_v32i8_shift_00_02_04_06_08_10_12_14_16_18_20_22_24_26_28_30_32_34_36_38_40_42_44_46_48_50_52_54_56_58_60_62:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vpsrlw $8, %ymm0, %ymm0
-; AVX512VLBW-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; AVX512VLBW-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX512VLBW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX512VLBW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VLBW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vpsrlw $8, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
 ; AVX512VLVBMI-LABEL: shuffle_v32i8_shift_00_02_04_06_08_10_12_14_16_18_20_22_24_26_28_30_32_34_36_38_40_42_44_46_48_50_52_54_56_58_60_62:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll
index 99a5c5f63d7de..db36bdb5d9400 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll
@@ -2524,11 +2524,18 @@ define <8 x i32> @shuffle_v8i32_089abcde(<8 x i32> %a, <8 x i32> %b) {
 ; AVX2-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3,4,5,6,7]
 ; AVX2-NEXT:    retq
 ;
-; AVX512VL-LABEL: shuffle_v8i32_089abcde:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    valignd {{.*#+}} ymm1 = ymm1[7,0,1,2,3,4,5,6]
-; AVX512VL-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3,4,5,6,7]
-; AVX512VL-NEXT:    retq
+; AVX512VL-SLOW-LABEL: shuffle_v8i32_089abcde:
+; AVX512VL-SLOW:       # %bb.0:
+; AVX512VL-SLOW-NEXT:    valignd {{.*#+}} ymm1 = ymm1[7,0,1,2,3,4,5,6]
+; AVX512VL-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3,4,5,6,7]
+; AVX512VL-SLOW-NEXT:    retq
+;
+; AVX512VL-FAST-LABEL: shuffle_v8i32_089abcde:
+; AVX512VL-FAST:       # %bb.0:
+; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = [8,0,1,2,3,4,5,6]
+; AVX512VL-FAST-NEXT:    vpermi2d %ymm0, %ymm1, %ymm2
+; AVX512VL-FAST-NEXT:    vmovdqa %ymm2, %ymm0
+; AVX512VL-FAST-NEXT:    retq
   %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 0, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14>
   ret <8 x i32> %shuffle
 }
@@ -3133,8 +3140,9 @@ define <8 x i32> @add_v8i32_02468ACE_13579BDF(<8 x i32> %a, <8 x i32> %b) {
 ;
 ; AVX512VL-SLOW-LABEL: add_v8i32_02468ACE_13579BDF:
 ; AVX512VL-SLOW:       # %bb.0: # %entry
-; AVX512VL-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm0[0,2],ymm1[0,2],ymm0[4,6],ymm1[4,6]
-; AVX512VL-SLOW-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
+; AVX512VL-SLOW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VL-SLOW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm2
+; AVX512VL-SLOW-NEXT:    vpmovqd %zmm2, %ymm2
 ; AVX512VL-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm1[1,3],ymm0[5,7],ymm1[5,7]
 ; AVX512VL-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX512VL-SLOW-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
@@ -3142,8 +3150,9 @@ define <8 x i32> @add_v8i32_02468ACE_13579BDF(<8 x i32> %a, <8 x i32> %b) {
 ;
 ; AVX512VL-FAST-LABEL: add_v8i32_02468ACE_13579BDF:
 ; AVX512VL-FAST:       # %bb.0: # %entry
-; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,2,4,6,8,10,12,14]
-; AVX512VL-FAST-NEXT:    vpermi2d %ymm1, %ymm0, %ymm2
+; AVX512VL-FAST-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VL-FAST-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm2
+; AVX512VL-FAST-NEXT:    vpmovqd %zmm2, %ymm2
 ; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = [1,3,5,7,9,11,13,15]
 ; AVX512VL-FAST-NEXT:    vpermi2d %ymm1, %ymm0, %ymm3
 ; AVX512VL-FAST-NEXT:    vpaddd %ymm3, %ymm2, %ymm0
@@ -3180,8 +3189,9 @@ define <8 x i32> @add_v8i32_8ACE0246_9BDF1357(<8 x i32> %a, <8 x i32> %b) {
 ;
 ; AVX512VL-SLOW-LABEL: add_v8i32_8ACE0246_9BDF1357:
 ; AVX512VL-SLOW:       # %bb.0: # %entry
-; AVX512VL-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm1[0,2],ymm0[0,2],ymm1[4,6],ymm0[4,6]
-; AVX512VL-SLOW-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,1,3]
+; AVX512VL-SLOW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VL-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm2
+; AVX512VL-SLOW-NEXT:    vpmovqd %zmm2, %ymm2
 ; AVX512VL-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm1[1,3],ymm0[1,3],ymm1[5,7],ymm0[5,7]
 ; AVX512VL-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
 ; AVX512VL-SLOW-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
@@ -3189,8 +3199,9 @@ define <8 x i32> @add_v8i32_8ACE0246_9BDF1357(<8 x i32> %a, <8 x i32> %b) {
 ;
 ; AVX512VL-FAST-LABEL: add_v8i32_8ACE0246_9BDF1357:
 ; AVX512VL-FAST:       # %bb.0: # %entry
-; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,2,4,6,8,10,12,14]
-; AVX512VL-FAST-NEXT:    vpermi2d %ymm0, %ymm1, %ymm2
+; AVX512VL-FAST-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VL-FAST-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm2
+; AVX512VL-FAST-NEXT:    vpmovqd %zmm2, %ymm2
 ; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = [1,3,5,7,9,11,13,15]
 ; AVX512VL-FAST-NEXT:    vpermi2d %ymm0, %ymm1, %ymm3
 ; AVX512VL-FAST-NEXT:    vpaddd %ymm3, %ymm2, %ymm0
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-512-v32.ll b/llvm/test/CodeGen/X86/vector-shuffle-512-v32.ll
index 0e79116884f4d..2601c7d4172d6 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-512-v32.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-512-v32.ll
@@ -216,6 +216,58 @@ define <32 x i16> @shuffle_v32i16_0zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz(<32 x i16> %a
   ret <32 x i16> %shuffle
 }
 
+define <32 x i16> @shuffle_v32i16_ashr_00_02_04_06_32_34_36_38_08_10_12_14_40_42_44_46_16_18_20_22_48_50_52_54_24_26_28_30_56_58_60_62(<16 x i32> %a0, <16 x i32> %a1) nounwind {
+; KNL-LABEL: shuffle_v32i16_ashr_00_02_04_06_32_34_36_38_08_10_12_14_40_42_44_46_16_18_20_22_48_50_52_54_24_26_28_30_56_58_60_62:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vpsrad $25, %zmm0, %zmm0
+; KNL-NEXT:    vpsrad $25, %zmm1, %zmm1
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; KNL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; KNL-NEXT:    vpackssdw %ymm3, %ymm2, %ymm2
+; KNL-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
+; KNL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: shuffle_v32i16_ashr_00_02_04_06_32_34_36_38_08_10_12_14_40_42_44_46_16_18_20_22_48_50_52_54_24_26_28_30_56_58_60_62:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vpsrad $25, %zmm0, %zmm0
+; SKX-NEXT:    vpsrad $25, %zmm1, %zmm1
+; SKX-NEXT:    vpackssdw %zmm1, %zmm0, %zmm0
+; SKX-NEXT:    retq
+  %1 = ashr <16 x i32> %a0, <i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25>
+  %2 = ashr <16 x i32> %a1, <i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25>
+  %3 = bitcast <16 x i32> %1 to <32 x i16>
+  %4 = bitcast <16 x i32> %2 to <32 x i16>
+  %5 = shufflevector <32 x i16> %3, <32 x i16> %4, <32 x i32> <i32 0, i32 2, i32 4, i32 6, i32 32, i32 34, i32 36, i32 38, i32 8, i32 10, i32 12, i32 14, i32 40, i32 42, i32 44, i32 46, i32 16, i32 18, i32 20, i32 22, i32 48, i32 50, i32 52, i32 54, i32 24, i32 26, i32 28, i32 30, i32 56, i32 58, i32 60, i32 62>
+  ret <32 x i16> %5
+}
+
+define <32 x i16> @shuffle_v32i16_lshr_00_02_04_06_32_34_36_38_08_10_12_14_40_42_44_46_16_18_20_22_48_50_52_54_24_26_28_30_56_58_60_62(<16 x i32> %a0, <16 x i32> %a1) nounwind {
+; KNL-LABEL: shuffle_v32i16_lshr_00_02_04_06_32_34_36_38_08_10_12_14_40_42_44_46_16_18_20_22_48_50_52_54_24_26_28_30_56_58_60_62:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vpsrld $25, %zmm0, %zmm0
+; KNL-NEXT:    vpsrld $25, %zmm1, %zmm1
+; KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; KNL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; KNL-NEXT:    vpackusdw %ymm3, %ymm2, %ymm2
+; KNL-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; KNL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: shuffle_v32i16_lshr_00_02_04_06_32_34_36_38_08_10_12_14_40_42_44_46_16_18_20_22_48_50_52_54_24_26_28_30_56_58_60_62:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vpsrld $25, %zmm0, %zmm0
+; SKX-NEXT:    vpsrld $25, %zmm1, %zmm1
+; SKX-NEXT:    vpackusdw %zmm1, %zmm0, %zmm0
+; SKX-NEXT:    retq
+  %1 = lshr <16 x i32> %a0, <i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25>
+  %2 = lshr <16 x i32> %a1, <i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25>
+  %3 = bitcast <16 x i32> %1 to <32 x i16>
+  %4 = bitcast <16 x i32> %2 to <32 x i16>
+  %5 = shufflevector <32 x i16> %3, <32 x i16> %4, <32 x i32> <i32 0, i32 2, i32 4, i32 6, i32 32, i32 34, i32 36, i32 38, i32 8, i32 10, i32 12, i32 14, i32 40, i32 42, i32 44, i32 46, i32 16, i32 18, i32 20, i32 22, i32 48, i32 50, i32 52, i32 54, i32 24, i32 26, i32 28, i32 30, i32 56, i32 58, i32 60, i32 62>
+  ret <32 x i16> %5
+}
+
 define <32 x i16> @insert_dup_mem_v32i16_i32(i32* %ptr) {
 ; KNL-LABEL: insert_dup_mem_v32i16_i32:
 ; KNL:       ## %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll b/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll
index 3afb54a9d3bb8..e048f6ecf285b 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll
@@ -546,6 +546,194 @@ define <64 x i8> @shuffle_v64i8_63_64_61_66_59_68_57_70_55_72_53_74_51_76_49_78_
   ret <64 x i8> %shuffle
 }
 
+define <64 x i8> @shuffle_v64i8_ashr_00_01_04_05_08_09_12_13_64_65_68_69_72_73_76_77_16_17_20_21_24_25_28_29_80_81_84_85_88_89_92_93_32_33_36_37_40_41_44_45_96_97_100_101_104_105_108_109_48_49_52_53_56_57_60_61_112_113_116_117_120_121_124_125(<16 x i32> %a0, <16 x i32> %a1) nounwind {
+; AVX512F-LABEL: shuffle_v64i8_ashr_00_01_04_05_08_09_12_13_64_65_68_69_72_73_76_77_16_17_20_21_24_25_28_29_80_81_84_85_88_89_92_93_32_33_36_37_40_41_44_45_96_97_100_101_104_105_108_109_48_49_52_53_56_57_60_61_112_113_116_117_120_121_124_125:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrad $25, %zmm0, %zmm0
+; AVX512F-NEXT:    vpsrad $25, %zmm1, %zmm1
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512F-NEXT:    vpackssdw %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: shuffle_v64i8_ashr_00_01_04_05_08_09_12_13_64_65_68_69_72_73_76_77_16_17_20_21_24_25_28_29_80_81_84_85_88_89_92_93_32_33_36_37_40_41_44_45_96_97_100_101_104_105_108_109_48_49_52_53_56_57_60_61_112_113_116_117_120_121_124_125:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrad $25, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrad $25, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpackssdw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQ-LABEL: shuffle_v64i8_ashr_00_01_04_05_08_09_12_13_64_65_68_69_72_73_76_77_16_17_20_21_24_25_28_29_80_81_84_85_88_89_92_93_32_33_36_37_40_41_44_45_96_97_100_101_104_105_108_109_48_49_52_53_56_57_60_61_112_113_116_117_120_121_124_125:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpsrad $25, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrad $25, %zmm1, %zmm1
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512DQ-NEXT:    vpackssdw %ymm3, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512DQ-NEXT:    retq
+;
+; AVX512VBMI-LABEL: shuffle_v64i8_ashr_00_01_04_05_08_09_12_13_64_65_68_69_72_73_76_77_16_17_20_21_24_25_28_29_80_81_84_85_88_89_92_93_32_33_36_37_40_41_44_45_96_97_100_101_104_105_108_109_48_49_52_53_56_57_60_61_112_113_116_117_120_121_124_125:
+; AVX512VBMI:       # %bb.0:
+; AVX512VBMI-NEXT:    vpsrad $25, %zmm0, %zmm0
+; AVX512VBMI-NEXT:    vpsrad $25, %zmm1, %zmm1
+; AVX512VBMI-NEXT:    vpackssdw %zmm1, %zmm0, %zmm0
+; AVX512VBMI-NEXT:    retq
+  %1 = ashr <16 x i32> %a0, <i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25>
+  %2 = ashr <16 x i32> %a1, <i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25>
+  %3 = bitcast <16 x i32> %1 to <64 x i8>
+  %4 = bitcast <16 x i32> %2 to <64 x i8>
+  %5 = shufflevector <64 x i8> %3, <64 x i8> %4, <64 x i32> <i32  0, i32  1, i32  4, i32  5, i32  8, i32  9, i32 12, i32 13, i32  64, i32  65, i32  68, i32  69, i32  72, i32  73, i32  76, i32  77, i32 16, i32 17, i32 20, i32 21, i32 24, i32 25, i32 28, i32 29, i32  80, i32  81, i32  84, i32  85, i32  88, i32  89, i32  92, i32  93, i32 32, i32 33, i32 36, i32 37, i32 40, i32 41, i32 44, i32 45, i32  96, i32  97, i32 100, i32 101, i32 104, i32 105, i32 108, i32 109, i32 48, i32 49, i32 52, i32 53, i32 56, i32 57, i32 60, i32 61, i32 112, i32 113, i32 116, i32 117, i32 120, i32 121, i32 124, i32 125>
+  ret <64 x i8> %5
+}
+
+define <64 x i8> @shuffle_v64i8_ashr_00_04_08_12_64_68_72_76_00_04_08_12_64_68_72_76_16_20_24_28_80_84_88_92_16_20_24_28_80_84_88_92_32_36_40_44_96_100_104_108_32_36_40_44_96_100_104_108_48_52_56_60_112_116_120_124_48_52_56_60_112_116_120_124(<16 x i32> %a0, <16 x i32> %a1) nounwind {
+; AVX512F-LABEL: shuffle_v64i8_ashr_00_04_08_12_64_68_72_76_00_04_08_12_64_68_72_76_16_20_24_28_80_84_88_92_16_20_24_28_80_84_88_92_32_36_40_44_96_100_104_108_32_36_40_44_96_100_104_108_48_52_56_60_112_116_120_124_48_52_56_60_112_116_120_124:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrad $25, %zmm0, %zmm0
+; AVX512F-NEXT:    vpsrad $25, %zmm1, %zmm1
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512F-NEXT:    vpackssdw %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vpacksswb %ymm2, %ymm2, %ymm2
+; AVX512F-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpacksswb %ymm0, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: shuffle_v64i8_ashr_00_04_08_12_64_68_72_76_00_04_08_12_64_68_72_76_16_20_24_28_80_84_88_92_16_20_24_28_80_84_88_92_32_36_40_44_96_100_104_108_32_36_40_44_96_100_104_108_48_52_56_60_112_116_120_124_48_52_56_60_112_116_120_124:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrad $25, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrad $25, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpackssdw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpacksswb %zmm0, %zmm0, %zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQ-LABEL: shuffle_v64i8_ashr_00_04_08_12_64_68_72_76_00_04_08_12_64_68_72_76_16_20_24_28_80_84_88_92_16_20_24_28_80_84_88_92_32_36_40_44_96_100_104_108_32_36_40_44_96_100_104_108_48_52_56_60_112_116_120_124_48_52_56_60_112_116_120_124:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpsrad $25, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrad $25, %zmm1, %zmm1
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512DQ-NEXT:    vpackssdw %ymm3, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpacksswb %ymm2, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpacksswb %ymm0, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512DQ-NEXT:    retq
+;
+; AVX512VBMI-LABEL: shuffle_v64i8_ashr_00_04_08_12_64_68_72_76_00_04_08_12_64_68_72_76_16_20_24_28_80_84_88_92_16_20_24_28_80_84_88_92_32_36_40_44_96_100_104_108_32_36_40_44_96_100_104_108_48_52_56_60_112_116_120_124_48_52_56_60_112_116_120_124:
+; AVX512VBMI:       # %bb.0:
+; AVX512VBMI-NEXT:    vpsrad $25, %zmm0, %zmm0
+; AVX512VBMI-NEXT:    vpsrad $25, %zmm1, %zmm1
+; AVX512VBMI-NEXT:    vpackssdw %zmm1, %zmm0, %zmm0
+; AVX512VBMI-NEXT:    vpacksswb %zmm0, %zmm0, %zmm0
+; AVX512VBMI-NEXT:    retq
+  %1 = ashr <16 x i32> %a0, <i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25>
+  %2 = ashr <16 x i32> %a1, <i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25>
+  %3 = bitcast <16 x i32> %1 to <64 x i8>
+  %4 = bitcast <16 x i32> %2 to <64 x i8>
+  %5 = shufflevector <64 x i8> %3, <64 x i8> %4, <64 x i32> <i32  0, i32  4, i32  8, i32 12, i32  64, i32  68, i32  72, i32  76, i32  0, i32  4, i32  8, i32 12, i32  64, i32  68, i32  72, i32  76, i32 16, i32 20, i32 24, i32 28, i32  80, i32  84, i32  88, i32  92, i32 16, i32 20, i32 24, i32 28, i32  80, i32  84, i32  88, i32  92, i32 32, i32 36, i32 40, i32 44, i32  96, i32 100, i32 104, i32 108, i32 32, i32 36, i32 40, i32 44, i32  96, i32 100, i32 104, i32 108, i32 48, i32 52, i32 56, i32 60, i32 112, i32 116, i32 120, i32 124, i32 48, i32 52, i32 56, i32 60, i32 112, i32 116, i32 120, i32 124>
+  ret <64 x i8> %5
+}
+
+define <64 x i8> @shuffle_v64i8_lshr_00_01_04_05_08_09_12_13_64_65_68_69_72_73_76_77_16_17_20_21_24_25_28_29_80_81_84_85_88_89_92_93_32_33_36_37_40_41_44_45_96_97_100_101_104_105_108_109_48_49_52_53_56_57_60_61_112_113_116_117_120_121_124_125(<16 x i32> %a0, <16 x i32> %a1) nounwind {
+; AVX512F-LABEL: shuffle_v64i8_lshr_00_01_04_05_08_09_12_13_64_65_68_69_72_73_76_77_16_17_20_21_24_25_28_29_80_81_84_85_88_89_92_93_32_33_36_37_40_41_44_45_96_97_100_101_104_105_108_109_48_49_52_53_56_57_60_61_112_113_116_117_120_121_124_125:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrld $25, %zmm0, %zmm0
+; AVX512F-NEXT:    vpsrld $25, %zmm1, %zmm1
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512F-NEXT:    vpackusdw %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: shuffle_v64i8_lshr_00_01_04_05_08_09_12_13_64_65_68_69_72_73_76_77_16_17_20_21_24_25_28_29_80_81_84_85_88_89_92_93_32_33_36_37_40_41_44_45_96_97_100_101_104_105_108_109_48_49_52_53_56_57_60_61_112_113_116_117_120_121_124_125:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrld $25, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrld $25, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpackusdw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQ-LABEL: shuffle_v64i8_lshr_00_01_04_05_08_09_12_13_64_65_68_69_72_73_76_77_16_17_20_21_24_25_28_29_80_81_84_85_88_89_92_93_32_33_36_37_40_41_44_45_96_97_100_101_104_105_108_109_48_49_52_53_56_57_60_61_112_113_116_117_120_121_124_125:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpsrld $25, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrld $25, %zmm1, %zmm1
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512DQ-NEXT:    vpackusdw %ymm3, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512DQ-NEXT:    retq
+;
+; AVX512VBMI-LABEL: shuffle_v64i8_lshr_00_01_04_05_08_09_12_13_64_65_68_69_72_73_76_77_16_17_20_21_24_25_28_29_80_81_84_85_88_89_92_93_32_33_36_37_40_41_44_45_96_97_100_101_104_105_108_109_48_49_52_53_56_57_60_61_112_113_116_117_120_121_124_125:
+; AVX512VBMI:       # %bb.0:
+; AVX512VBMI-NEXT:    vpsrld $25, %zmm0, %zmm0
+; AVX512VBMI-NEXT:    vpsrld $25, %zmm1, %zmm1
+; AVX512VBMI-NEXT:    vpackusdw %zmm1, %zmm0, %zmm0
+; AVX512VBMI-NEXT:    retq
+  %1 = lshr <16 x i32> %a0, <i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25>
+  %2 = lshr <16 x i32> %a1, <i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25>
+  %3 = bitcast <16 x i32> %1 to <64 x i8>
+  %4 = bitcast <16 x i32> %2 to <64 x i8>
+  %5 = shufflevector <64 x i8> %3, <64 x i8> %4, <64 x i32> <i32  0, i32  1, i32  4, i32  5, i32  8, i32  9, i32 12, i32 13, i32  64, i32  65, i32  68, i32  69, i32  72, i32  73, i32  76, i32  77, i32 16, i32 17, i32 20, i32 21, i32 24, i32 25, i32 28, i32 29, i32  80, i32  81, i32  84, i32  85, i32  88, i32  89, i32  92, i32  93, i32 32, i32 33, i32 36, i32 37, i32 40, i32 41, i32 44, i32 45, i32  96, i32  97, i32 100, i32 101, i32 104, i32 105, i32 108, i32 109, i32 48, i32 49, i32 52, i32 53, i32 56, i32 57, i32 60, i32 61, i32 112, i32 113, i32 116, i32 117, i32 120, i32 121, i32 124, i32 125>
+  ret <64 x i8> %5
+}
+
+define <64 x i8> @shuffle_v64i8_lshr_00_04_08_12_64_68_72_76_00_04_08_12_64_68_72_76_16_20_24_28_80_84_88_092_16_20_24_28_80_84_88_92_32_36_40_44_96_100_104_108_32_36_40_44_96_100_104_108_48_52_56_60_112_116_120_124_48_52_56_60_112_116_120_124(<16 x i32> %a0, <16 x i32> %a1) nounwind {
+; AVX512F-LABEL: shuffle_v64i8_lshr_00_04_08_12_64_68_72_76_00_04_08_12_64_68_72_76_16_20_24_28_80_84_88_092_16_20_24_28_80_84_88_92_32_36_40_44_96_100_104_108_32_36_40_44_96_100_104_108_48_52_56_60_112_116_120_124_48_52_56_60_112_116_120_124:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrld $25, %zmm0, %zmm0
+; AVX512F-NEXT:    vpsrld $25, %zmm1, %zmm1
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512F-NEXT:    vpackusdw %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vpackuswb %ymm2, %ymm2, %ymm2
+; AVX512F-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpackuswb %ymm0, %ymm0, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: shuffle_v64i8_lshr_00_04_08_12_64_68_72_76_00_04_08_12_64_68_72_76_16_20_24_28_80_84_88_092_16_20_24_28_80_84_88_92_32_36_40_44_96_100_104_108_32_36_40_44_96_100_104_108_48_52_56_60_112_116_120_124_48_52_56_60_112_116_120_124:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrld $25, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrld $25, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpackusdw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpackuswb %zmm0, %zmm0, %zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQ-LABEL: shuffle_v64i8_lshr_00_04_08_12_64_68_72_76_00_04_08_12_64_68_72_76_16_20_24_28_80_84_88_092_16_20_24_28_80_84_88_92_32_36_40_44_96_100_104_108_32_36_40_44_96_100_104_108_48_52_56_60_112_116_120_124_48_52_56_60_112_116_120_124:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vpsrld $25, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpsrld $25, %zmm1, %zmm1
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
+; AVX512DQ-NEXT:    vpackusdw %ymm3, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpackuswb %ymm2, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpackuswb %ymm0, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512DQ-NEXT:    retq
+;
+; AVX512VBMI-LABEL: shuffle_v64i8_lshr_00_04_08_12_64_68_72_76_00_04_08_12_64_68_72_76_16_20_24_28_80_84_88_092_16_20_24_28_80_84_88_92_32_36_40_44_96_100_104_108_32_36_40_44_96_100_104_108_48_52_56_60_112_116_120_124_48_52_56_60_112_116_120_124:
+; AVX512VBMI:       # %bb.0:
+; AVX512VBMI-NEXT:    vpsrld $25, %zmm0, %zmm0
+; AVX512VBMI-NEXT:    vpsrld $25, %zmm1, %zmm1
+; AVX512VBMI-NEXT:    vpackusdw %zmm1, %zmm0, %zmm0
+; AVX512VBMI-NEXT:    vpackuswb %zmm0, %zmm0, %zmm0
+; AVX512VBMI-NEXT:    retq
+  %1 = lshr <16 x i32> %a0, <i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25>
+  %2 = lshr <16 x i32> %a1, <i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25>
+  %3 = bitcast <16 x i32> %1 to <64 x i8>
+  %4 = bitcast <16 x i32> %2 to <64 x i8>
+  %5 = shufflevector <64 x i8> %3, <64 x i8> %4, <64 x i32> <i32  0, i32  4, i32  8, i32 12, i32  64, i32  68, i32  72, i32  76, i32  0, i32  4, i32  8, i32 12, i32  64, i32  68, i32  72, i32  76, i32 16, i32 20, i32 24, i32 28, i32  80, i32  84, i32  88, i32  92, i32 16, i32 20, i32 24, i32 28, i32  80, i32  84, i32  88, i32  92, i32 32, i32 36, i32 40, i32 44, i32  96, i32 100, i32 104, i32 108, i32 32, i32 36, i32 40, i32 44, i32  96, i32 100, i32 104, i32 108, i32 48, i32 52, i32 56, i32 60, i32 112, i32 116, i32 120, i32 124, i32 48, i32 52, i32 56, i32 60, i32 112, i32 116, i32 120, i32 124>
+  ret <64 x i8> %5
+}
+
 define <64 x i8> @shuffle_v64i8_shift_00_02_04_06_08_10_12_14_16_18_20_22_24_26_28_30_32_34_36_38_40_42_44_46_48_50_52_54_56_58_60_62_64_66_68_70_72_74_76_78_80_82_84_86_88_90_92_94_96_98_100_102_104_106_108_110_112_114_116_118_120_122_124_126(<32 x i16> %a0, <32 x i16> %a1) {
 ; AVX512F-LABEL: shuffle_v64i8_shift_00_02_04_06_08_10_12_14_16_18_20_22_24_26_28_30_32_34_36_38_40_42_44_46_48_50_52_54_56_58_60_62_64_66_68_70_72_74_76_78_80_82_84_86_88_90_92_94_96_98_100_102_104_106_108_110_112_114_116_118_120_122_124_126:
 ; AVX512F:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-512-v8.ll b/llvm/test/CodeGen/X86/vector-shuffle-512-v8.ll
index 31acce98bcacc..1638cb7f4c58c 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-512-v8.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-512-v8.ll
@@ -192,7 +192,6 @@ define <8 x double> @shuffle_v8f64_00001111(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_81a3c5e7(<8 x double> %a, <8 x double> %b) {
-;
 ; ALL-LABEL: shuffle_v8f64_81a3c5e7:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vshufpd {{.*#+}} zmm0 = zmm1[0],zmm0[1],zmm1[2],zmm0[3],zmm1[4],zmm0[5],zmm1[6],zmm0[7]
@@ -202,7 +201,6 @@ define <8 x double> @shuffle_v8f64_81a3c5e7(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_08080808(<8 x double> %a, <8 x double> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8f64_08080808:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [0,8,0,8,0,8,0,8]
@@ -221,7 +219,6 @@ define <8 x double> @shuffle_v8f64_08080808(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_08084c4c(<8 x double> %a, <8 x double> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8f64_08084c4c:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovapd {{.*#+}} zmm2 = [0,8,0,8,4,12,4,12]
@@ -238,7 +235,6 @@ define <8 x double> @shuffle_v8f64_08084c4c(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_8823cc67(<8 x double> %a, <8 x double> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8f64_8823cc67:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovapd {{.*#+}} zmm2 = [0,0,10,11,4,4,14,15]
@@ -257,7 +253,6 @@ define <8 x double> @shuffle_v8f64_8823cc67(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_9832dc76(<8 x double> %a, <8 x double> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8f64_9832dc76:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovapd {{.*#+}} zmm2 = [1,0,11,10,5,4,15,14]
@@ -276,7 +271,6 @@ define <8 x double> @shuffle_v8f64_9832dc76(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_9810dc54(<8 x double> %a, <8 x double> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8f64_9810dc54:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovapd {{.*#+}} zmm2 = [1,0,9,8,5,4,13,12]
@@ -295,7 +289,6 @@ define <8 x double> @shuffle_v8f64_9810dc54(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_08194c5d(<8 x double> %a, <8 x double> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8f64_08194c5d:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovapd {{.*#+}} zmm2 = [0,8,1,9,4,12,5,13]
@@ -312,7 +305,6 @@ define <8 x double> @shuffle_v8f64_08194c5d(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_2a3b6e7f(<8 x double> %a, <8 x double> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8f64_2a3b6e7f:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovapd {{.*#+}} zmm2 = [2,10,3,11,6,14,7,15]
@@ -329,7 +321,6 @@ define <8 x double> @shuffle_v8f64_2a3b6e7f(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_08192a3b(<8 x double> %a, <8 x double> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8f64_08192a3b:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovapd {{.*#+}} zmm2 = [0,8,1,9,2,10,3,11]
@@ -346,7 +337,6 @@ define <8 x double> @shuffle_v8f64_08192a3b(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_08991abb(<8 x double> %a, <8 x double> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8f64_08991abb:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovapd {{.*#+}} zmm2 = [8,0,1,1,9,2,3,3]
@@ -365,7 +355,6 @@ define <8 x double> @shuffle_v8f64_08991abb(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_091b2d3f(<8 x double> %a, <8 x double> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8f64_091b2d3f:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovapd {{.*#+}} zmm2 = [0,9,1,11,2,13,3,15]
@@ -382,7 +371,6 @@ define <8 x double> @shuffle_v8f64_091b2d3f(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_09ab1def(<8 x double> %a, <8 x double> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8f64_09ab1def:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovapd {{.*#+}} zmm2 = [8,1,2,3,9,5,6,7]
@@ -401,7 +389,6 @@ define <8 x double> @shuffle_v8f64_09ab1def(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_00014445(<8 x double> %a, <8 x double> %b) {
-;
 ; ALL-LABEL: shuffle_v8f64_00014445:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermpd {{.*#+}} zmm0 = zmm0[0,0,0,1,4,4,4,5]
@@ -411,7 +398,6 @@ define <8 x double> @shuffle_v8f64_00014445(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_00204464(<8 x double> %a, <8 x double> %b) {
-;
 ; ALL-LABEL: shuffle_v8f64_00204464:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermpd {{.*#+}} zmm0 = zmm0[0,0,2,0,4,4,6,4]
@@ -421,7 +407,6 @@ define <8 x double> @shuffle_v8f64_00204464(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_03004744(<8 x double> %a, <8 x double> %b) {
-;
 ; ALL-LABEL: shuffle_v8f64_03004744:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermpd {{.*#+}} zmm0 = zmm0[0,3,0,0,4,7,4,4]
@@ -431,7 +416,6 @@ define <8 x double> @shuffle_v8f64_03004744(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_10005444(<8 x double> %a, <8 x double> %b) {
-;
 ; ALL-LABEL: shuffle_v8f64_10005444:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermpd {{.*#+}} zmm0 = zmm0[1,0,0,0,5,4,4,4]
@@ -441,7 +425,6 @@ define <8 x double> @shuffle_v8f64_10005444(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_22006644(<8 x double> %a, <8 x double> %b) {
-;
 ; ALL-LABEL: shuffle_v8f64_22006644:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermpd {{.*#+}} zmm0 = zmm0[2,2,0,0,6,6,4,4]
@@ -451,7 +434,6 @@ define <8 x double> @shuffle_v8f64_22006644(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_33307774(<8 x double> %a, <8 x double> %b) {
-;
 ; ALL-LABEL: shuffle_v8f64_33307774:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermpd {{.*#+}} zmm0 = zmm0[3,3,3,0,7,7,7,4]
@@ -461,7 +443,6 @@ define <8 x double> @shuffle_v8f64_33307774(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_32107654(<8 x double> %a, <8 x double> %b) {
-;
 ; ALL-LABEL: shuffle_v8f64_32107654:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermpd {{.*#+}} zmm0 = zmm0[3,2,1,0,7,6,5,4]
@@ -471,7 +452,6 @@ define <8 x double> @shuffle_v8f64_32107654(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_00234467(<8 x double> %a, <8 x double> %b) {
-;
 ; ALL-LABEL: shuffle_v8f64_00234467:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermilpd {{.*#+}} zmm0 = zmm0[0,0,2,3,4,4,6,7]
@@ -481,7 +461,6 @@ define <8 x double> @shuffle_v8f64_00234467(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_00224466(<8 x double> %a, <8 x double> %b) {
-;
 ; ALL-LABEL: shuffle_v8f64_00224466:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vmovddup {{.*#+}} zmm0 = zmm0[0,0,2,2,4,4,6,6]
@@ -491,7 +470,6 @@ define <8 x double> @shuffle_v8f64_00224466(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_10325476(<8 x double> %a, <8 x double> %b) {
-;
 ; ALL-LABEL: shuffle_v8f64_10325476:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermilpd {{.*#+}} zmm0 = zmm0[1,0,3,2,5,4,7,6]
@@ -501,7 +479,6 @@ define <8 x double> @shuffle_v8f64_10325476(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_11335577(<8 x double> %a, <8 x double> %b) {
-;
 ; ALL-LABEL: shuffle_v8f64_11335577:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermilpd {{.*#+}} zmm0 = zmm0[1,1,3,3,5,5,7,7]
@@ -511,7 +488,6 @@ define <8 x double> @shuffle_v8f64_11335577(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_10235467(<8 x double> %a, <8 x double> %b) {
-;
 ; ALL-LABEL: shuffle_v8f64_10235467:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermilpd {{.*#+}} zmm0 = zmm0[1,0,2,3,5,4,6,7]
@@ -521,7 +497,6 @@ define <8 x double> @shuffle_v8f64_10235467(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_10225466(<8 x double> %a, <8 x double> %b) {
-;
 ; ALL-LABEL: shuffle_v8f64_10225466:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermilpd {{.*#+}} zmm0 = zmm0[1,0,2,2,5,4,6,6]
@@ -531,7 +506,6 @@ define <8 x double> @shuffle_v8f64_10225466(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_00015444(<8 x double> %a, <8 x double> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8f64_00015444:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [0,0,0,1,5,4,4,4]
@@ -548,7 +522,6 @@ define <8 x double> @shuffle_v8f64_00015444(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_00204644(<8 x double> %a, <8 x double> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8f64_00204644:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [0,0,2,0,4,6,4,4]
@@ -565,7 +538,6 @@ define <8 x double> @shuffle_v8f64_00204644(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_03004474(<8 x double> %a, <8 x double> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8f64_03004474:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [0,3,0,0,4,4,7,4]
@@ -582,7 +554,6 @@ define <8 x double> @shuffle_v8f64_03004474(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_10004444(<8 x double> %a, <8 x double> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8f64_10004444:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [1,0,0,0,4,4,4,4]
@@ -599,7 +570,6 @@ define <8 x double> @shuffle_v8f64_10004444(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_22006446(<8 x double> %a, <8 x double> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8f64_22006446:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [2,2,0,0,6,4,4,6]
@@ -616,7 +586,6 @@ define <8 x double> @shuffle_v8f64_22006446(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_33307474(<8 x double> %a, <8 x double> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8f64_33307474:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [3,3,3,0,7,4,7,4]
@@ -633,7 +602,6 @@ define <8 x double> @shuffle_v8f64_33307474(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_32104567(<8 x double> %a, <8 x double> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8f64_32104567:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [3,2,1,0,4,5,6,7]
@@ -650,7 +618,6 @@ define <8 x double> @shuffle_v8f64_32104567(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_00236744(<8 x double> %a, <8 x double> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8f64_00236744:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [0,0,2,3,6,7,4,4]
@@ -667,7 +634,6 @@ define <8 x double> @shuffle_v8f64_00236744(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_00226644(<8 x double> %a, <8 x double> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8f64_00226644:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [0,0,2,2,6,6,4,4]
@@ -684,7 +650,6 @@ define <8 x double> @shuffle_v8f64_00226644(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_10324567(<8 x double> %a, <8 x double> %b) {
-;
 ; ALL-LABEL: shuffle_v8f64_10324567:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermilpd {{.*#+}} zmm0 = zmm0[1,0,3,2,4,5,6,7]
@@ -694,7 +659,6 @@ define <8 x double> @shuffle_v8f64_10324567(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_11334567(<8 x double> %a, <8 x double> %b) {
-;
 ; ALL-LABEL: shuffle_v8f64_11334567:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermilpd {{.*#+}} zmm0 = zmm0[1,1,3,3,4,5,6,7]
@@ -704,7 +668,6 @@ define <8 x double> @shuffle_v8f64_11334567(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_01235467(<8 x double> %a, <8 x double> %b) {
-;
 ; ALL-LABEL: shuffle_v8f64_01235467:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermilpd {{.*#+}} zmm0 = zmm0[0,1,2,3,5,4,6,7]
@@ -714,7 +677,6 @@ define <8 x double> @shuffle_v8f64_01235467(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_01235466(<8 x double> %a, <8 x double> %b) {
-;
 ; ALL-LABEL: shuffle_v8f64_01235466:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermilpd {{.*#+}} zmm0 = zmm0[0,1,2,3,5,4,6,6]
@@ -724,7 +686,6 @@ define <8 x double> @shuffle_v8f64_01235466(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_002u6u44(<8 x double> %a, <8 x double> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8f64_002u6u44:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = <0,0,2,u,6,u,4,4>
@@ -741,7 +702,6 @@ define <8 x double> @shuffle_v8f64_002u6u44(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_00uu66uu(<8 x double> %a, <8 x double> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8f64_00uu66uu:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = <0,0,u,u,6,6,u,u>
@@ -758,7 +718,6 @@ define <8 x double> @shuffle_v8f64_00uu66uu(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_103245uu(<8 x double> %a, <8 x double> %b) {
-;
 ; ALL-LABEL: shuffle_v8f64_103245uu:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermilpd {{.*#+}} zmm0 = zmm0[1,0,3,2,4,5,6,6]
@@ -768,7 +727,6 @@ define <8 x double> @shuffle_v8f64_103245uu(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_1133uu67(<8 x double> %a, <8 x double> %b) {
-;
 ; ALL-LABEL: shuffle_v8f64_1133uu67:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermilpd {{.*#+}} zmm0 = zmm0[1,1,3,3,4,4,6,7]
@@ -778,7 +736,6 @@ define <8 x double> @shuffle_v8f64_1133uu67(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_0uu354uu(<8 x double> %a, <8 x double> %b) {
-;
 ; ALL-LABEL: shuffle_v8f64_0uu354uu:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermilpd {{.*#+}} zmm0 = zmm0[0,0,2,3,5,4,6,6]
@@ -788,7 +745,6 @@ define <8 x double> @shuffle_v8f64_0uu354uu(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_uuu3uu66(<8 x double> %a, <8 x double> %b) {
-;
 ; ALL-LABEL: shuffle_v8f64_uuu3uu66:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermilpd {{.*#+}} zmm0 = zmm0[0,0,2,3,4,4,6,6]
@@ -798,7 +754,6 @@ define <8 x double> @shuffle_v8f64_uuu3uu66(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_c348cda0(<8 x double> %a, <8 x double> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8f64_c348cda0:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovapd {{.*#+}} zmm2 = [4,11,12,0,4,5,2,8]
@@ -817,7 +772,6 @@ define <8 x double> @shuffle_v8f64_c348cda0(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_f511235a(<8 x double> %a, <8 x double> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8f64_f511235a:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovapd {{.*#+}} zmm2 = [15,5,1,1,2,3,5,10]
@@ -843,8 +797,43 @@ define <8 x double> @shuffle_v8f64_1z2z5z6z(<8 x double> %a, <8 x double> %b) {
   ret <8 x double> %shuffle
 }
 
+define <8 x double> @shuffle_v8f64_23uuuuuu(<8 x double> %a0, <8 x double> %a1) {
+; ALL-LABEL: shuffle_v8f64_23uuuuuu:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; ALL-NEXT:    ret{{[l|q]}}
+  %1 = shufflevector <8 x double> %a0, <8 x double> undef, <8 x i32> <i32 2, i32 3, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
+  ret <8 x double> %1
+}
+
+define <8 x double> @shuffle_v8f64_67zzzzzz(<8 x double> %a0, <8 x double> %a1) {
+; ALL-LABEL: shuffle_v8f64_67zzzzzz:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vextractf32x4 $3, %zmm0, %xmm0
+; ALL-NEXT:    ret{{[l|q]}}
+  %1 = shufflevector <8 x double> %a0, <8 x double> zeroinitializer, <8 x i32> <i32 6, i32 7, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
+  ret <8 x double> %1
+}
+
+define <8 x double> @shuffle_v8f64_4567uuuu(<8 x double> %a0, <8 x double> %a1) {
+; ALL-LABEL: shuffle_v8f64_4567uuuu:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
+; ALL-NEXT:    ret{{[l|q]}}
+  %1 = shufflevector <8 x double> %a0, <8 x double> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 8, i32 8, i32 8>
+  ret <8 x double> %1
+}
+
+define <8 x double> @shuffle_v8f64_4567zzzz(<8 x double> %a0, <8 x double> %a1) {
+; ALL-LABEL: shuffle_v8f64_4567zzzz:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
+; ALL-NEXT:    ret{{[l|q]}}
+  %1 = shufflevector <8 x double> %a0, <8 x double> zeroinitializer, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 8, i32 8, i32 8>
+  ret <8 x double> %1
+}
+
 define <8 x i64> @shuffle_v8i64_00000000(<8 x i64> %a, <8 x i64> %b) {
-;
 ; ALL-LABEL: shuffle_v8i64_00000000:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vbroadcastsd %xmm0, %zmm0
@@ -874,7 +863,6 @@ define <8 x i64> @shuffle_v8i64_66666666(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_00000010(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_00000010:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [0,0,0,0,0,0,1,0]
@@ -891,7 +879,6 @@ define <8 x i64> @shuffle_v8i64_00000010(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_00000200(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_00000200:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [0,0,0,0,0,2,0,0]
@@ -908,7 +895,6 @@ define <8 x i64> @shuffle_v8i64_00000200(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_00003000(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_00003000:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [0,0,0,0,3,0,0,0]
@@ -925,7 +911,6 @@ define <8 x i64> @shuffle_v8i64_00003000(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_00040000(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_00040000:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [0,0,0,4,0,0,0,0]
@@ -942,7 +927,6 @@ define <8 x i64> @shuffle_v8i64_00040000(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_00500000(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_00500000:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [0,0,5,0,0,0,0,0]
@@ -959,7 +943,6 @@ define <8 x i64> @shuffle_v8i64_00500000(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_06000000(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_06000000:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [0,6,0,0,0,0,0,0]
@@ -976,7 +959,6 @@ define <8 x i64> @shuffle_v8i64_06000000(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_70000000(<8 x i64> %a, <8 x i64> %b) {
-;
 ; ALL-LABEL: shuffle_v8i64_70000000:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    movl $7, %eax
@@ -1015,7 +997,6 @@ define <8 x i64> @shuffle_v8i64_01014545_mem(<8 x i64>* %ptr, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_00112233(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_00112233:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [0,0,1,1,2,2,3,3]
@@ -1032,7 +1013,6 @@ define <8 x i64> @shuffle_v8i64_00112233(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_00001111(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_00001111:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [0,0,0,0,1,1,1,1]
@@ -1049,7 +1029,6 @@ define <8 x i64> @shuffle_v8i64_00001111(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_81a3c5e7(<8 x i64> %a, <8 x i64> %b) {
-;
 ; ALL-LABEL: shuffle_v8i64_81a3c5e7:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    movb $-86, %al
@@ -1061,7 +1040,6 @@ define <8 x i64> @shuffle_v8i64_81a3c5e7(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_08080808(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_08080808:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [0,8,0,8,0,8,0,8]
@@ -1080,7 +1058,6 @@ define <8 x i64> @shuffle_v8i64_08080808(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_08084c4c(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_08084c4c:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,8,0,8,4,12,4,12]
@@ -1097,7 +1074,6 @@ define <8 x i64> @shuffle_v8i64_08084c4c(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_8823cc67(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_8823cc67:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,0,10,11,4,4,14,15]
@@ -1116,7 +1092,6 @@ define <8 x i64> @shuffle_v8i64_8823cc67(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_9832dc76(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_9832dc76:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [1,0,11,10,5,4,15,14]
@@ -1135,7 +1110,6 @@ define <8 x i64> @shuffle_v8i64_9832dc76(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_9810dc54(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_9810dc54:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [1,0,9,8,5,4,13,12]
@@ -1154,7 +1128,6 @@ define <8 x i64> @shuffle_v8i64_9810dc54(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_08194c5d(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_08194c5d:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,8,1,9,4,12,5,13]
@@ -1171,7 +1144,6 @@ define <8 x i64> @shuffle_v8i64_08194c5d(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_2a3b6e7f(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_2a3b6e7f:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [2,10,3,11,6,14,7,15]
@@ -1188,7 +1160,6 @@ define <8 x i64> @shuffle_v8i64_2a3b6e7f(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_08192a3b(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_08192a3b:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,8,1,9,2,10,3,11]
@@ -1205,7 +1176,6 @@ define <8 x i64> @shuffle_v8i64_08192a3b(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_08991abb(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_08991abb:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [8,0,1,1,9,2,3,3]
@@ -1224,7 +1194,6 @@ define <8 x i64> @shuffle_v8i64_08991abb(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_091b2d3f(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_091b2d3f:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,9,1,11,2,13,3,15]
@@ -1241,7 +1210,6 @@ define <8 x i64> @shuffle_v8i64_091b2d3f(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_09ab1def(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_09ab1def:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [8,1,2,3,9,5,6,7]
@@ -1260,7 +1228,6 @@ define <8 x i64> @shuffle_v8i64_09ab1def(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_00014445(<8 x i64> %a, <8 x i64> %b) {
-;
 ; ALL-LABEL: shuffle_v8i64_00014445:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermpd {{.*#+}} zmm0 = zmm0[0,0,0,1,4,4,4,5]
@@ -1270,7 +1237,6 @@ define <8 x i64> @shuffle_v8i64_00014445(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_00204464(<8 x i64> %a, <8 x i64> %b) {
-;
 ; ALL-LABEL: shuffle_v8i64_00204464:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermpd {{.*#+}} zmm0 = zmm0[0,0,2,0,4,4,6,4]
@@ -1280,7 +1246,6 @@ define <8 x i64> @shuffle_v8i64_00204464(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_03004744(<8 x i64> %a, <8 x i64> %b) {
-;
 ; ALL-LABEL: shuffle_v8i64_03004744:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermpd {{.*#+}} zmm0 = zmm0[0,3,0,0,4,7,4,4]
@@ -1290,7 +1255,6 @@ define <8 x i64> @shuffle_v8i64_03004744(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_10005444(<8 x i64> %a, <8 x i64> %b) {
-;
 ; ALL-LABEL: shuffle_v8i64_10005444:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermpd {{.*#+}} zmm0 = zmm0[1,0,0,0,5,4,4,4]
@@ -1300,7 +1264,6 @@ define <8 x i64> @shuffle_v8i64_10005444(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_22006644(<8 x i64> %a, <8 x i64> %b) {
-;
 ; ALL-LABEL: shuffle_v8i64_22006644:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermpd {{.*#+}} zmm0 = zmm0[2,2,0,0,6,6,4,4]
@@ -1310,7 +1273,6 @@ define <8 x i64> @shuffle_v8i64_22006644(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_33307774(<8 x i64> %a, <8 x i64> %b) {
-;
 ; ALL-LABEL: shuffle_v8i64_33307774:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermpd {{.*#+}} zmm0 = zmm0[3,3,3,0,7,7,7,4]
@@ -1320,7 +1282,6 @@ define <8 x i64> @shuffle_v8i64_33307774(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_32107654(<8 x i64> %a, <8 x i64> %b) {
-;
 ; ALL-LABEL: shuffle_v8i64_32107654:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermpd {{.*#+}} zmm0 = zmm0[3,2,1,0,7,6,5,4]
@@ -1330,7 +1291,6 @@ define <8 x i64> @shuffle_v8i64_32107654(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_00234467(<8 x i64> %a, <8 x i64> %b) {
-;
 ; ALL-LABEL: shuffle_v8i64_00234467:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermpd {{.*#+}} zmm0 = zmm0[0,0,2,3,4,4,6,7]
@@ -1340,7 +1300,6 @@ define <8 x i64> @shuffle_v8i64_00234467(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_00224466(<8 x i64> %a, <8 x i64> %b) {
-;
 ; ALL-LABEL: shuffle_v8i64_00224466:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermilps {{.*#+}} zmm0 = zmm0[0,1,0,1,4,5,4,5,8,9,8,9,12,13,12,13]
@@ -1350,7 +1309,6 @@ define <8 x i64> @shuffle_v8i64_00224466(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_10325476(<8 x i64> %a, <8 x i64> %b) {
-;
 ; ALL-LABEL: shuffle_v8i64_10325476:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermilps {{.*#+}} zmm0 = zmm0[2,3,0,1,6,7,4,5,10,11,8,9,14,15,12,13]
@@ -1360,7 +1318,6 @@ define <8 x i64> @shuffle_v8i64_10325476(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_11335577(<8 x i64> %a, <8 x i64> %b) {
-;
 ; ALL-LABEL: shuffle_v8i64_11335577:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermilps {{.*#+}} zmm0 = zmm0[2,3,2,3,6,7,6,7,10,11,10,11,14,15,14,15]
@@ -1370,7 +1327,6 @@ define <8 x i64> @shuffle_v8i64_11335577(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_10235467(<8 x i64> %a, <8 x i64> %b) {
-;
 ; ALL-LABEL: shuffle_v8i64_10235467:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermpd {{.*#+}} zmm0 = zmm0[1,0,2,3,5,4,6,7]
@@ -1380,7 +1336,6 @@ define <8 x i64> @shuffle_v8i64_10235467(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_10225466(<8 x i64> %a, <8 x i64> %b) {
-;
 ; ALL-LABEL: shuffle_v8i64_10225466:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpermpd {{.*#+}} zmm0 = zmm0[1,0,2,2,5,4,6,6]
@@ -1390,7 +1345,6 @@ define <8 x i64> @shuffle_v8i64_10225466(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_00015444(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_00015444:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [0,0,0,1,5,4,4,4]
@@ -1407,7 +1361,6 @@ define <8 x i64> @shuffle_v8i64_00015444(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_00204644(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_00204644:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [0,0,2,0,4,6,4,4]
@@ -1424,7 +1377,6 @@ define <8 x i64> @shuffle_v8i64_00204644(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_03004474(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_03004474:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [0,3,0,0,4,4,7,4]
@@ -1441,7 +1393,6 @@ define <8 x i64> @shuffle_v8i64_03004474(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_10004444(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_10004444:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [1,0,0,0,4,4,4,4]
@@ -1458,7 +1409,6 @@ define <8 x i64> @shuffle_v8i64_10004444(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_22006446(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_22006446:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [2,2,0,0,6,4,4,6]
@@ -1475,7 +1425,6 @@ define <8 x i64> @shuffle_v8i64_22006446(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_33307474(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_33307474:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [3,3,3,0,7,4,7,4]
@@ -1492,7 +1441,6 @@ define <8 x i64> @shuffle_v8i64_33307474(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_32104567(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_32104567:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [3,2,1,0,4,5,6,7]
@@ -1509,7 +1457,6 @@ define <8 x i64> @shuffle_v8i64_32104567(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_00236744(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_00236744:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [0,0,2,3,6,7,4,4]
@@ -1526,7 +1473,6 @@ define <8 x i64> @shuffle_v8i64_00236744(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_00226644(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_00226644:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [0,0,2,2,6,6,4,4]
@@ -1543,7 +1489,6 @@ define <8 x i64> @shuffle_v8i64_00226644(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_10324567(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_10324567:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [1,0,3,2,4,5,6,7]
@@ -1560,7 +1505,6 @@ define <8 x i64> @shuffle_v8i64_10324567(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_11334567(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_11334567:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [1,1,3,3,4,5,6,7]
@@ -1577,7 +1521,6 @@ define <8 x i64> @shuffle_v8i64_11334567(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_01235467(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_01235467:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [0,1,2,3,5,4,6,7]
@@ -1594,7 +1537,6 @@ define <8 x i64> @shuffle_v8i64_01235467(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_01235466(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_01235466:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = [0,1,2,3,5,4,6,6]
@@ -1611,7 +1553,6 @@ define <8 x i64> @shuffle_v8i64_01235466(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_002u6u44(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_002u6u44:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = <0,0,2,u,6,u,4,4>
@@ -1628,7 +1569,6 @@ define <8 x i64> @shuffle_v8i64_002u6u44(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_00uu66uu(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_00uu66uu:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = <0,0,u,u,6,6,u,u>
@@ -1645,7 +1585,6 @@ define <8 x i64> @shuffle_v8i64_00uu66uu(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_103245uu(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_103245uu:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = <1,0,3,2,4,5,u,u>
@@ -1662,7 +1601,6 @@ define <8 x i64> @shuffle_v8i64_103245uu(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_1133uu67(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_1133uu67:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = <1,1,3,3,u,u,6,7>
@@ -1679,7 +1617,6 @@ define <8 x i64> @shuffle_v8i64_1133uu67(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_0uu354uu(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_0uu354uu:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = <0,u,u,3,5,4,u,u>
@@ -1696,7 +1633,6 @@ define <8 x i64> @shuffle_v8i64_0uu354uu(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_uuu3uu66(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_uuu3uu66:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovaps {{.*#+}} zmm1 = <u,u,u,3,u,u,6,6>
@@ -1713,7 +1649,6 @@ define <8 x i64> @shuffle_v8i64_uuu3uu66(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_6caa87e5(<8 x i64> %a, <8 x i64> %b) {
-;
 ; AVX512F-LABEL: shuffle_v8i64_6caa87e5:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [14,4,2,2,0,15,6,13]
@@ -1732,7 +1667,6 @@ define <8 x i64> @shuffle_v8i64_6caa87e5(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_082a4c6e(<8 x double> %a, <8 x double> %b) {
-;
 ; ALL-LABEL: shuffle_v8f64_082a4c6e:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vunpcklpd {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6]
@@ -1742,7 +1676,6 @@ define <8 x double> @shuffle_v8f64_082a4c6e(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_0z2z4z6z(<8 x double> %a, <8 x double> %b) {
-;
 ; ALL-LABEL: shuffle_v8f64_0z2z4z6z:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
@@ -1753,7 +1686,6 @@ define <8 x double> @shuffle_v8f64_0z2z4z6z(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_082a4c6e(<8 x i64> %a, <8 x i64> %b) {
-;
 ; ALL-LABEL: shuffle_v8i64_082a4c6e:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vunpcklpd {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6]
@@ -1763,7 +1695,6 @@ define <8 x i64> @shuffle_v8i64_082a4c6e(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_z8zazcze(<8 x i64> %a, <8 x i64> %b) {
-;
 ; ALL-LABEL: shuffle_v8i64_z8zazcze:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vxorps %xmm0, %xmm0, %xmm0
@@ -1774,7 +1705,6 @@ define <8 x i64> @shuffle_v8i64_z8zazcze(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_193b5d7f(<8 x double> %a, <8 x double> %b) {
-;
 ; ALL-LABEL: shuffle_v8f64_193b5d7f:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vunpckhpd {{.*#+}} zmm0 = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7]
@@ -1784,7 +1714,6 @@ define <8 x double> @shuffle_v8f64_193b5d7f(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_z9zbzdzf(<8 x double> %a, <8 x double> %b) {
-;
 ; ALL-LABEL: shuffle_v8f64_z9zbzdzf:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vxorpd %xmm0, %xmm0, %xmm0
@@ -1795,7 +1724,6 @@ define <8 x double> @shuffle_v8f64_z9zbzdzf(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_193b5d7f(<8 x i64> %a, <8 x i64> %b) {
-;
 ; ALL-LABEL: shuffle_v8i64_193b5d7f:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vunpckhpd {{.*#+}} zmm0 = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7]
@@ -1805,7 +1733,6 @@ define <8 x i64> @shuffle_v8i64_193b5d7f(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_1z3z5z7z(<8 x i64> %a, <8 x i64> %b) {
-;
 ; ALL-LABEL: shuffle_v8i64_1z3z5z7z:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
@@ -1967,7 +1894,6 @@ define <8 x double> @shuffle_v8f64_0zzzzzzz(<8 x double> %a) {
 }
 
 define <8 x i64> @shuffle_v8i64_12345678(<8 x i64> %a, <8 x i64> %b) {
-;
 ; ALL-LABEL: shuffle_v8i64_12345678:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    valignq {{.*#+}} zmm0 = zmm0[1,2,3,4,5,6,7],zmm1[0]
@@ -1977,7 +1903,6 @@ define <8 x i64> @shuffle_v8i64_12345678(<8 x i64> %a, <8 x i64> %b) {
 }
 
 define <8 x i64> @shuffle_v8i64_12345670(<8 x i64> %a) {
-;
 ; ALL-LABEL: shuffle_v8i64_12345670:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    valignq {{.*#+}} zmm0 = zmm0[1,2,3,4,5,6,7,0]
@@ -1987,7 +1912,6 @@ define <8 x i64> @shuffle_v8i64_12345670(<8 x i64> %a) {
 }
 
 define <8 x i64> @mask_shuffle_v8i64_12345678(<8 x i64> %a, <8 x i64> %b, <8 x i64> %passthru, i8 %mask) {
-;
 ; AVX512F-LABEL: mask_shuffle_v8i64_12345678:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    kmovw %edi, %k1
@@ -2009,7 +1933,6 @@ define <8 x i64> @mask_shuffle_v8i64_12345678(<8 x i64> %a, <8 x i64> %b, <8 x i
 }
 
 define <8 x i64> @mask_shuffle_v8i64_12345670(<8 x i64> %a, <8 x i64> %passthru, i8 %mask) {
-;
 ; AVX512F-LABEL: mask_shuffle_v8i64_12345670:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    kmovw %edi, %k1
@@ -2031,7 +1954,6 @@ define <8 x i64> @mask_shuffle_v8i64_12345670(<8 x i64> %a, <8 x i64> %passthru,
 }
 
 define <8 x i64> @maskz_shuffle_v8i64_12345678(<8 x i64> %a, <8 x i64> %b, i8 %mask) {
-;
 ; AVX512F-LABEL: maskz_shuffle_v8i64_12345678:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    kmovw %edi, %k1
@@ -2051,7 +1973,6 @@ define <8 x i64> @maskz_shuffle_v8i64_12345678(<8 x i64> %a, <8 x i64> %b, i8 %m
 }
 
 define <8 x i64> @maskz_shuffle_v8i64_12345670(<8 x i64> %a, i8 %mask) {
-;
 ; AVX512F-LABEL: maskz_shuffle_v8i64_12345670:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    kmovw %edi, %k1
@@ -2196,6 +2117,42 @@ define <8 x i64> @shuffle_v8i64_01234589(<8 x i64> %a, <8 x i64> %b) {
   ret <8 x i64> %shuffle
 }
 
+define <8 x i64> @shuffle_v8i64_23uuuuuu(<8 x i64> %a0, <8 x i64> %a1) {
+; ALL-LABEL: shuffle_v8i64_23uuuuuu:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; ALL-NEXT:    ret{{[l|q]}}
+  %1 = shufflevector <8 x i64> %a0, <8 x i64> undef, <8 x i32> <i32 2, i32 3, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @shuffle_v8i64_45zzzzzz(<8 x i64> %a0, <8 x i64> %a1) {
+; ALL-LABEL: shuffle_v8i64_45zzzzzz:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vextractf32x4 $2, %zmm0, %xmm0
+; ALL-NEXT:    ret{{[l|q]}}
+  %1 = shufflevector <8 x i64> %a0, <8 x i64> zeroinitializer, <8 x i32> <i32 4, i32 5, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @shuffle_v8i64_4567uuuu(<8 x i64> %a0, <8 x i64> %a1) {
+; ALL-LABEL: shuffle_v8i64_4567uuuu:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
+; ALL-NEXT:    ret{{[l|q]}}
+  %1 = shufflevector <8 x i64> %a0, <8 x i64> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 8, i32 8, i32 8>
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @shuffle_v8i64_uu67zzzz(<8 x i64> %a0, <8 x i64> %a1) {
+; ALL-LABEL: shuffle_v8i64_uu67zzzz:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
+; ALL-NEXT:    ret{{[l|q]}}
+  %1 = shufflevector <8 x i64> %a0, <8 x i64> zeroinitializer, <8 x i32> <i32 undef, i32 undef, i32 6, i32 7, i32 8, i32 8, i32 8, i32 8>
+  ret <8 x i64> %1
+}
+
 define <8 x double> @shuffle_v4f64_v8f64_22222222(<4 x double> %a) {
 ; ALL-LABEL: shuffle_v4f64_v8f64_22222222:
 ; ALL:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx2.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx2.ll
index d922de739c9d9..bbd5bcb29a980 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx2.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx2.ll
@@ -73,16 +73,16 @@ define <32 x i8> @combine_pshufb_and(<32 x i8> %a0) {
   ret <32 x i8> %2
 }
 
-define <4 x i64> @combine_permq_pshufb_as_vperm2i128(<4 x i64> %a0) {
-; X86-LABEL: combine_permq_pshufb_as_vperm2i128:
+define <4 x i64> @combine_permq_pshufb_as_vextracti128(<4 x i64> %a0) {
+; X86-LABEL: combine_permq_pshufb_as_vextracti128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],zero,zero
+; X86-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; X86-NEXT:    vpaddq {{\.LCPI.*}}, %ymm0, %ymm0
 ; X86-NEXT:    retl
 ;
-; X64-LABEL: combine_permq_pshufb_as_vperm2i128:
+; X64-LABEL: combine_permq_pshufb_as_vextracti128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],zero,zero
+; X64-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; X64-NEXT:    vpaddq {{.*}}(%rip), %ymm0, %ymm0
 ; X64-NEXT:    retq
   %1 = shufflevector <4 x i64> %a0, <4 x i64> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
@@ -93,6 +93,26 @@ define <4 x i64> @combine_permq_pshufb_as_vperm2i128(<4 x i64> %a0) {
   ret <4 x i64> %5
 }
 
+define <4 x i64> @combine_permq_pshufb_as_vmovdqa(<4 x i64> %a0) {
+; X86-LABEL: combine_permq_pshufb_as_vmovdqa:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovdqa %xmm0, %xmm0
+; X86-NEXT:    vpaddq {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: combine_permq_pshufb_as_vmovdqa:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovdqa %xmm0, %xmm0
+; X64-NEXT:    vpaddq {{.*}}(%rip), %ymm0, %ymm0
+; X64-NEXT:    retq
+  %1 = shufflevector <4 x i64> %a0, <4 x i64> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+  %2 = bitcast <4 x i64> %1 to <32 x i8>
+  %3 = call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %2, <32 x i8> <i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255>)
+  %4 = bitcast <32 x i8> %3 to <4 x i64>
+  %5 = add <4 x i64> %4, <i64 1, i64 1, i64 3, i64 3>
+  ret <4 x i64> %5
+}
+
 define <8 x i32> @combine_as_vpermd(<8 x i32> %a0) {
 ; CHECK-LABEL: combine_as_vpermd:
 ; CHECK:       # %bb.0:
@@ -117,15 +137,26 @@ define <8 x float> @combine_as_vpermps(<8 x float> %a0) {
   ret <8 x float> %3
 }
 
+define <32 x i8> @combine_permq_pshufb_as_vmovaps(<4 x i64> %a0) {
+; CHECK-LABEL: combine_permq_pshufb_as_vmovaps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm0, %xmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 = shufflevector <4 x i64> %a0, <4 x i64> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+  %2 = bitcast <4 x i64> %1 to <32 x i8>
+  %3 = call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %2, <32 x i8> <i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255>)
+  ret <32 x i8> %3
+}
+
 define <32 x i8> @combine_permq_pshufb_as_vpblendd(<4 x i64> %a0) {
 ; CHECK-LABEL: combine_permq_pshufb_as_vpblendd:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; CHECK-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; CHECK-NEXT:    ret{{[l|q]}}
   %1 = shufflevector <4 x i64> %a0, <4 x i64> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
   %2 = bitcast <4 x i64> %1 to <32 x i8>
-  %3 = call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %2, <32 x i8> <i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255>)
+  %3 = call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %2, <32 x i8> <i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>)
   ret <32 x i8> %3
 }
 
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bw.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bw.ll
index 08923cab6ebb3..17781eb922a60 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bw.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bw.ll
@@ -159,6 +159,36 @@ define <32 x i16> @combine_pshufb_as_pshufhw(<32 x i16> %a0) {
   ret <32 x i16> %1
 }
 
+define <32 x i16> @combine_vpermi2var_as_packssdw(<16 x i32> %a0, <16 x i32> %a1) nounwind {
+; CHECK-LABEL: combine_vpermi2var_as_packssdw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsrad $25, %zmm0, %zmm0
+; CHECK-NEXT:    vpsrad $25, %zmm1, %zmm1
+; CHECK-NEXT:    vpackssdw %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 = ashr <16 x i32> %a0, <i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25>
+  %2 = ashr <16 x i32> %a1, <i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25>
+  %3 = bitcast <16 x i32> %1 to <32 x i16>
+  %4 = bitcast <16 x i32> %2 to <32 x i16>
+  %5 = call <32 x i16> @llvm.x86.avx512.mask.vpermi2var.hi.512(<32 x i16> %3, <32 x i16> <i16 0, i16 2, i16 4, i16 6, i16 32, i16 34, i16 36, i16 38, i16 8, i16 10, i16 12, i16 14, i16 40, i16 42, i16 44, i16 46, i16 16, i16 18, i16 20, i16 22, i16 48, i16 50, i16 52, i16 54, i16 24, i16 26, i16 28, i16 30, i16 56, i16 58, i16 60, i16 62>, <32 x i16> %4, i32 -1)
+  ret <32 x i16> %5
+}
+
+define <32 x i16> @combine_vpermi2var_as_packusdw(<16 x i32> %a0, <16 x i32> %a1) nounwind {
+; CHECK-LABEL: combine_vpermi2var_as_packusdw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsrld $25, %zmm0, %zmm0
+; CHECK-NEXT:    vpsrld $25, %zmm1, %zmm1
+; CHECK-NEXT:    vpackusdw %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 = lshr <16 x i32> %a0, <i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25>
+  %2 = lshr <16 x i32> %a1, <i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25>
+  %3 = bitcast <16 x i32> %1 to <32 x i16>
+  %4 = bitcast <16 x i32> %2 to <32 x i16>
+  %5 = call <32 x i16> @llvm.x86.avx512.mask.vpermi2var.hi.512(<32 x i16> %3, <32 x i16> <i16 0, i16 2, i16 4, i16 6, i16 32, i16 34, i16 36, i16 38, i16 8, i16 10, i16 12, i16 14, i16 40, i16 42, i16 44, i16 46, i16 16, i16 18, i16 20, i16 22, i16 48, i16 50, i16 52, i16 54, i16 24, i16 26, i16 28, i16 30, i16 56, i16 58, i16 60, i16 62>, <32 x i16> %4, i32 -1)
+  ret <32 x i16> %5
+}
+
 define <64 x i8> @combine_pshufb_as_packsswb(<32 x i16> %a0, <32 x i16> %a1) nounwind {
 ; CHECK-LABEL: combine_pshufb_as_packsswb:
 ; CHECK:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-v1.ll b/llvm/test/CodeGen/X86/vector-shuffle-v1.ll
index 94b00fbd937e2..ab8b9bcfb7590 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-v1.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-v1.ll
@@ -62,8 +62,9 @@ define <2 x i1> @shuf2i1_1_2(<2 x i1> %a) {
 ; AVX512VL-NEXT:    vmovdqa64 %xmm0, %xmm1 {%k1} {z}
 ; AVX512VL-NEXT:    movq $-1, %rax
 ; AVX512VL-NEXT:    vmovq %rax, %xmm2
-; AVX512VL-NEXT:    vpalignr {{.*#+}} xmm1 = xmm1[8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4,5,6,7]
-; AVX512VL-NEXT:    vptestmq %xmm1, %xmm1, %k1
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,2]
+; AVX512VL-NEXT:    vpermi2q %xmm2, %xmm1, %xmm3
+; AVX512VL-NEXT:    vptestmq %xmm3, %xmm3, %k1
 ; AVX512VL-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
 ; AVX512VL-NEXT:    retq
 ;
@@ -74,8 +75,9 @@ define <2 x i1> @shuf2i1_1_2(<2 x i1> %a) {
 ; VL_BW_DQ-NEXT:    movq $-1, %rax
 ; VL_BW_DQ-NEXT:    vmovq %rax, %xmm0
 ; VL_BW_DQ-NEXT:    vpmovm2q %k0, %xmm1
-; VL_BW_DQ-NEXT:    vpalignr {{.*#+}} xmm0 = xmm1[8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4,5,6,7]
-; VL_BW_DQ-NEXT:    vpmovq2m %xmm0, %k0
+; VL_BW_DQ-NEXT:    vmovdqa {{.*#+}} xmm2 = [1,2]
+; VL_BW_DQ-NEXT:    vpermi2q %xmm0, %xmm1, %xmm2
+; VL_BW_DQ-NEXT:    vpmovq2m %xmm2, %k0
 ; VL_BW_DQ-NEXT:    vpmovm2q %k0, %xmm0
 ; VL_BW_DQ-NEXT:    retq
   %b = shufflevector <2 x i1> %a, <2 x i1> <i1 1, i1 0>, <2 x i32> <i32 1, i32 2>
diff --git a/llvm/test/CodeGen/X86/vector-trunc-math.ll b/llvm/test/CodeGen/X86/vector-trunc-math.ll
index 24418ece07a51..80e33a7aab9ee 100644
--- a/llvm/test/CodeGen/X86/vector-trunc-math.ll
+++ b/llvm/test/CodeGen/X86/vector-trunc-math.ll
@@ -383,7 +383,7 @@ define <16 x i8> @trunc_add_v16i16_v16i8(<16 x i16> %a0, <16 x i16> %a1) nounwin
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    paddw %xmm2, %xmm0
 ; SSE-NEXT:    paddw %xmm3, %xmm1
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    packuswb %xmm1, %xmm0
@@ -800,7 +800,7 @@ define <16 x i8> @trunc_add_const_v16i32_v16i8(<16 x i32> %a0) nounwind {
 define <16 x i8> @trunc_add_const_v16i16_v16i8(<16 x i16> %a0) nounwind {
 ; SSE-LABEL: trunc_add_const_v16i16_v16i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    packuswb %xmm1, %xmm0
@@ -1229,7 +1229,7 @@ define <16 x i8> @trunc_sub_v16i16_v16i8(<16 x i16> %a0, <16 x i16> %a1) nounwin
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    psubw %xmm2, %xmm0
 ; SSE-NEXT:    psubw %xmm3, %xmm1
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    packuswb %xmm1, %xmm0
@@ -1614,7 +1614,7 @@ define <16 x i8> @trunc_sub_const_v16i32_v16i8(<16 x i32> %a0) nounwind {
 define <16 x i8> @trunc_sub_const_v16i16_v16i8(<16 x i16> %a0) nounwind {
 ; SSE-LABEL: trunc_sub_const_v16i16_v16i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    packuswb %xmm1, %xmm0
@@ -2212,7 +2212,7 @@ define <16 x i8> @trunc_mul_v16i16_v16i8(<16 x i16> %a0, <16 x i16> %a1) nounwin
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pmullw %xmm2, %xmm0
 ; SSE-NEXT:    pmullw %xmm3, %xmm1
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    packuswb %xmm1, %xmm0
@@ -2698,7 +2698,7 @@ define <16 x i8> @trunc_mul_const_v16i16_v16i8(<16 x i16> %a0) nounwind {
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pmullw {{.*}}(%rip), %xmm0
 ; SSE-NEXT:    pmullw {{.*}}(%rip), %xmm1
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    packuswb %xmm1, %xmm0
@@ -3099,7 +3099,7 @@ define <16 x i8> @trunc_and_v16i32_v16i8(<16 x i32> %a0, <16 x i32> %a1) nounwin
 define <16 x i8> @trunc_and_v16i16_v16i8(<16 x i16> %a0, <16 x i16> %a1) nounwind {
 ; SSE-LABEL: trunc_and_v16i16_v16i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa {{.*#+}} xmm4 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; SSE-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
 ; SSE-NEXT:    pand %xmm4, %xmm3
 ; SSE-NEXT:    pand %xmm1, %xmm3
 ; SSE-NEXT:    pand %xmm4, %xmm2
@@ -3465,7 +3465,7 @@ define <16 x i8> @trunc_and_const_v16i32_v16i8(<16 x i32> %a0) nounwind {
 define <16 x i8> @trunc_and_const_v16i16_v16i8(<16 x i16> %a0) nounwind {
 ; SSE-LABEL: trunc_and_const_v16i16_v16i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    packuswb %xmm1, %xmm0
@@ -3866,7 +3866,7 @@ define <16 x i8> @trunc_xor_v16i16_v16i8(<16 x i16> %a0, <16 x i16> %a1) nounwin
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pxor %xmm2, %xmm0
 ; SSE-NEXT:    pxor %xmm3, %xmm1
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    packuswb %xmm1, %xmm0
@@ -4230,7 +4230,7 @@ define <16 x i8> @trunc_xor_const_v16i32_v16i8(<16 x i32> %a0) nounwind {
 define <16 x i8> @trunc_xor_const_v16i16_v16i8(<16 x i16> %a0) nounwind {
 ; SSE-LABEL: trunc_xor_const_v16i16_v16i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    packuswb %xmm1, %xmm0
@@ -4631,7 +4631,7 @@ define <16 x i8> @trunc_or_v16i16_v16i8(<16 x i16> %a0, <16 x i16> %a1) nounwind
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    por %xmm2, %xmm0
 ; SSE-NEXT:    por %xmm3, %xmm1
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    packuswb %xmm1, %xmm0
@@ -4995,7 +4995,7 @@ define <16 x i8> @trunc_or_const_v16i32_v16i8(<16 x i32> %a0) nounwind {
 define <16 x i8> @trunc_or_const_v16i16_v16i8(<16 x i16> %a0) nounwind {
 ; SSE-LABEL: trunc_or_const_v16i16_v16i8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
 ; SSE-NEXT:    pand %xmm2, %xmm1
 ; SSE-NEXT:    pand %xmm2, %xmm0
 ; SSE-NEXT:    packuswb %xmm1, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-trunc-packus.ll b/llvm/test/CodeGen/X86/vector-trunc-packus.ll
index c74b87921c442..ed26bd892e167 100644
--- a/llvm/test/CodeGen/X86/vector-trunc-packus.ll
+++ b/llvm/test/CodeGen/X86/vector-trunc-packus.ll
@@ -5211,8 +5211,10 @@ define <4 x i8> @trunc_packus_v4i32_v4i8(<4 x i32> %a0) "min-legal-vector-width"
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pminsd {{.*}}(%rip), %xmm0
 ; SSE41-NEXT:    pxor %xmm1, %xmm1
-; SSE41-NEXT:    pmaxsd %xmm1, %xmm0
-; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSE41-NEXT:    pmaxsd %xmm0, %xmm1
+; SSE41-NEXT:    packusdw %xmm0, %xmm1
+; SSE41-NEXT:    packuswb %xmm1, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_packus_v4i32_v4i8:
@@ -5220,7 +5222,8 @@ define <4 x i8> @trunc_packus_v4i32_v4i8(<4 x i32> %a0) "min-legal-vector-width"
 ; AVX1-NEXT:    vpminsd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: trunc_packus_v4i32_v4i8:
@@ -5229,7 +5232,8 @@ define <4 x i8> @trunc_packus_v4i32_v4i8(<4 x i32> %a0) "min-legal-vector-width"
 ; AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX2-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: trunc_packus_v4i32_v4i8:
@@ -5316,7 +5320,8 @@ define void @trunc_packus_v4i32_v4i8_store(<4 x i32> %a0, <4 x i8> *%p1) {
 ; SSE41-NEXT:    pminsd {{.*}}(%rip), %xmm0
 ; SSE41-NEXT:    pxor %xmm1, %xmm1
 ; SSE41-NEXT:    pmaxsd %xmm0, %xmm1
-; SSE41-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSE41-NEXT:    packusdw %xmm0, %xmm1
+; SSE41-NEXT:    packuswb %xmm0, %xmm1
 ; SSE41-NEXT:    movd %xmm1, (%rdi)
 ; SSE41-NEXT:    retq
 ;
@@ -5325,7 +5330,8 @@ define void @trunc_packus_v4i32_v4i8_store(<4 x i32> %a0, <4 x i8> *%p1) {
 ; AVX1-NEXT:    vpminsd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, (%rdi)
 ; AVX1-NEXT:    retq
 ;
@@ -5335,7 +5341,8 @@ define void @trunc_packus_v4i32_v4i8_store(<4 x i32> %a0, <4 x i8> *%p1) {
 ; AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX2-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, (%rdi)
 ; AVX2-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-trunc-ssat.ll b/llvm/test/CodeGen/X86/vector-trunc-ssat.ll
index 35a50b59fbf35..bed45538d48c1 100644
--- a/llvm/test/CodeGen/X86/vector-trunc-ssat.ll
+++ b/llvm/test/CodeGen/X86/vector-trunc-ssat.ll
@@ -2853,7 +2853,7 @@ define <4 x i8> @trunc_ssat_v4i64_v4i8(<4 x i64> %a0) {
 ; SSE2-NEXT:    pand %xmm1, %xmm3
 ; SSE2-NEXT:    pandn %xmm8, %xmm1
 ; SSE2-NEXT:    por %xmm3, %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,0,0,255,0,0,0]
 ; SSE2-NEXT:    pand %xmm2, %xmm1
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    packuswb %xmm1, %xmm0
@@ -3112,7 +3112,7 @@ define void @trunc_ssat_v4i64_v4i8_store(<4 x i64> %a0, <4 x i8> *%p1) {
 ; SSE2-NEXT:    pand %xmm1, %xmm3
 ; SSE2-NEXT:    pandn %xmm8, %xmm1
 ; SSE2-NEXT:    por %xmm3, %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,0,0,255,0,0,0]
 ; SSE2-NEXT:    pand %xmm2, %xmm1
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    packuswb %xmm1, %xmm0
@@ -5080,14 +5080,16 @@ define <4 x i8> @trunc_ssat_v4i32_v4i8(<4 x i32> %a0) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pminsd {{.*}}(%rip), %xmm0
 ; SSE41-NEXT:    pmaxsd {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSE41-NEXT:    packssdw %xmm0, %xmm0
+; SSE41-NEXT:    packsswb %xmm0, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_ssat_v4i32_v4i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpminsd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vpmaxsd {{.*}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: trunc_ssat_v4i32_v4i8:
@@ -5096,7 +5098,8 @@ define <4 x i8> @trunc_ssat_v4i32_v4i8(<4 x i32> %a0) {
 ; AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [4294967168,4294967168,4294967168,4294967168]
 ; AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: trunc_ssat_v4i32_v4i8:
@@ -5178,7 +5181,8 @@ define void @trunc_ssat_v4i32_v4i8_store(<4 x i32> %a0, <4 x i8> *%p1) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pminsd {{.*}}(%rip), %xmm0
 ; SSE41-NEXT:    pmaxsd {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSE41-NEXT:    packssdw %xmm0, %xmm0
+; SSE41-NEXT:    packsswb %xmm0, %xmm0
 ; SSE41-NEXT:    movd %xmm0, (%rdi)
 ; SSE41-NEXT:    retq
 ;
@@ -5186,7 +5190,8 @@ define void @trunc_ssat_v4i32_v4i8_store(<4 x i32> %a0, <4 x i8> *%p1) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpminsd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vpmaxsd {{.*}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, (%rdi)
 ; AVX1-NEXT:    retq
 ;
@@ -5196,7 +5201,8 @@ define void @trunc_ssat_v4i32_v4i8_store(<4 x i32> %a0, <4 x i8> *%p1) {
 ; AVX2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [4294967168,4294967168,4294967168,4294967168]
 ; AVX2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, (%rdi)
 ; AVX2-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-trunc-usat.ll b/llvm/test/CodeGen/X86/vector-trunc-usat.ll
index 8434bac380df7..4e5f833703751 100644
--- a/llvm/test/CodeGen/X86/vector-trunc-usat.ll
+++ b/llvm/test/CodeGen/X86/vector-trunc-usat.ll
@@ -3694,20 +3694,23 @@ define <4 x i8> @trunc_usat_v4i32_v4i8(<4 x i32> %a0) {
 ; SSE41-LABEL: trunc_usat_v4i32_v4i8:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pminud {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSE41-NEXT:    packusdw %xmm0, %xmm0
+; SSE41-NEXT:    packuswb %xmm0, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_usat_v4i32_v4i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: trunc_usat_v4i32_v4i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [255,255,255,255]
 ; AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX2-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: trunc_usat_v4i32_v4i8:
@@ -3776,14 +3779,16 @@ define void @trunc_usat_v4i32_v4i8_store(<4 x i32> %a0, <4 x i8> *%p1) {
 ; SSE41-LABEL: trunc_usat_v4i32_v4i8_store:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pminud {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSE41-NEXT:    packusdw %xmm0, %xmm0
+; SSE41-NEXT:    packuswb %xmm0, %xmm0
 ; SSE41-NEXT:    movd %xmm0, (%rdi)
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_usat_v4i32_v4i8_store:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, (%rdi)
 ; AVX1-NEXT:    retq
 ;
@@ -3791,7 +3796,8 @@ define void @trunc_usat_v4i32_v4i8_store(<4 x i32> %a0, <4 x i8> *%p1) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [255,255,255,255]
 ; AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX2-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, (%rdi)
 ; AVX2-NEXT:    retq
 ;
@@ -3881,24 +3887,20 @@ define <8 x i8> @trunc_usat_v8i32_v8i8(<8 x i32> %a0) {
 ; SSE41-LABEL: trunc_usat_v8i32_v8i8:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255]
-; SSE41-NEXT:    pminud %xmm2, %xmm0
 ; SSE41-NEXT:    pminud %xmm2, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
-; SSE41-NEXT:    pshufb %xmm2, %xmm1
-; SSE41-NEXT:    pshufb %xmm2, %xmm0
-; SSE41-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE41-NEXT:    pminud %xmm2, %xmm0
+; SSE41-NEXT:    packusdw %xmm1, %xmm0
+; SSE41-NEXT:    packuswb %xmm0, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_usat_v8i32_v8i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255]
-; AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX1-NEXT:    vpshufb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,255,255]
+; AVX1-NEXT:    vpminud %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpminud %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
@@ -3907,10 +3909,8 @@ define <8 x i8> @trunc_usat_v8i32_v8i8(<8 x i32> %a0) {
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255]
 ; AVX2-NEXT:    vpminud %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX2-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -4001,25 +4001,21 @@ define void @trunc_usat_v8i32_v8i8_store(<8 x i32> %a0, <8 x i8> *%p1) {
 ; SSE41-LABEL: trunc_usat_v8i32_v8i8_store:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255]
-; SSE41-NEXT:    pminud %xmm2, %xmm0
 ; SSE41-NEXT:    pminud %xmm2, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
-; SSE41-NEXT:    pshufb %xmm2, %xmm1
-; SSE41-NEXT:    pshufb %xmm2, %xmm0
-; SSE41-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE41-NEXT:    pminud %xmm2, %xmm0
+; SSE41-NEXT:    packusdw %xmm1, %xmm0
+; SSE41-NEXT:    packuswb %xmm0, %xmm0
 ; SSE41-NEXT:    movq %xmm0, (%rdi)
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_usat_v8i32_v8i8_store:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255]
-; AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpminud %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX1-NEXT:    vpshufb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,255,255]
+; AVX1-NEXT:    vpminud %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpminud %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, (%rdi)
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -4029,10 +4025,8 @@ define void @trunc_usat_v8i32_v8i8_store(<8 x i32> %a0, <8 x i8> *%p1) {
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255]
 ; AVX2-NEXT:    vpminud %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX2-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, (%rdi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-trunc.ll b/llvm/test/CodeGen/X86/vector-trunc.ll
index 94b08c9abb820..852d0174d9602 100644
--- a/llvm/test/CodeGen/X86/vector-trunc.ll
+++ b/llvm/test/CodeGen/X86/vector-trunc.ll
@@ -1026,32 +1026,14 @@ entry:
 
 ;PR25684
 define void @trunc16i16_16i8(<16 x i16> %a) {
-; SSE2-LABEL: trunc16i16_16i8:
-; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    packuswb %xmm1, %xmm0
-; SSE2-NEXT:    movdqu %xmm0, (%rax)
-; SSE2-NEXT:    retq
-;
-; SSSE3-LABEL: trunc16i16_16i8:
-; SSSE3:       # %bb.0: # %entry
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; SSSE3-NEXT:    pshufb %xmm2, %xmm1
-; SSSE3-NEXT:    pshufb %xmm2, %xmm0
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSSE3-NEXT:    movdqu %xmm0, (%rax)
-; SSSE3-NEXT:    retq
-;
-; SSE41-LABEL: trunc16i16_16i8:
-; SSE41:       # %bb.0: # %entry
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; SSE41-NEXT:    pshufb %xmm2, %xmm1
-; SSE41-NEXT:    pshufb %xmm2, %xmm0
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE41-NEXT:    movdqu %xmm0, (%rax)
-; SSE41-NEXT:    retq
+; SSE-LABEL: trunc16i16_16i8:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE-NEXT:    pand %xmm2, %xmm1
+; SSE-NEXT:    pand %xmm2, %xmm0
+; SSE-NEXT:    packuswb %xmm1, %xmm0
+; SSE-NEXT:    movdqu %xmm0, (%rax)
+; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc16i16_16i8:
 ; AVX1:       # %bb.0: # %entry
@@ -1235,44 +1217,18 @@ entry:
 }
 
 define void @trunc32i16_32i8(<32 x i16> %a) {
-; SSE2-LABEL: trunc32i16_32i8:
-; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    packuswb %xmm1, %xmm0
-; SSE2-NEXT:    pand %xmm4, %xmm3
-; SSE2-NEXT:    pand %xmm4, %xmm2
-; SSE2-NEXT:    packuswb %xmm3, %xmm2
-; SSE2-NEXT:    movdqu %xmm2, (%rax)
-; SSE2-NEXT:    movdqu %xmm0, (%rax)
-; SSE2-NEXT:    retq
-;
-; SSSE3-LABEL: trunc32i16_32i8:
-; SSSE3:       # %bb.0: # %entry
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; SSSE3-NEXT:    pshufb %xmm4, %xmm1
-; SSSE3-NEXT:    pshufb %xmm4, %xmm0
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSSE3-NEXT:    pshufb %xmm4, %xmm3
-; SSSE3-NEXT:    pshufb %xmm4, %xmm2
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSSE3-NEXT:    movdqu %xmm2, (%rax)
-; SSSE3-NEXT:    movdqu %xmm0, (%rax)
-; SSSE3-NEXT:    retq
-;
-; SSE41-LABEL: trunc32i16_32i8:
-; SSE41:       # %bb.0: # %entry
-; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; SSE41-NEXT:    pshufb %xmm4, %xmm1
-; SSE41-NEXT:    pshufb %xmm4, %xmm0
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE41-NEXT:    pshufb %xmm4, %xmm3
-; SSE41-NEXT:    pshufb %xmm4, %xmm2
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE41-NEXT:    movdqu %xmm2, (%rax)
-; SSE41-NEXT:    movdqu %xmm0, (%rax)
-; SSE41-NEXT:    retq
+; SSE-LABEL: trunc32i16_32i8:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
+; SSE-NEXT:    pand %xmm4, %xmm1
+; SSE-NEXT:    pand %xmm4, %xmm0
+; SSE-NEXT:    packuswb %xmm1, %xmm0
+; SSE-NEXT:    pand %xmm4, %xmm3
+; SSE-NEXT:    pand %xmm4, %xmm2
+; SSE-NEXT:    packuswb %xmm3, %xmm2
+; SSE-NEXT:    movdqu %xmm2, (%rax)
+; SSE-NEXT:    movdqu %xmm0, (%rax)
+; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc32i16_32i8:
 ; AVX1:       # %bb.0: # %entry
@@ -1556,10 +1512,31 @@ define <4 x i32> @trunc2x2i64_4i32(<2 x i64> %a, <2 x i64> %b) {
 ; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: trunc2x2i64_4i32:
-; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: trunc2x2i64_4i32:
+; AVX512F:       # %bb.0: # %entry
+; AVX512F-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc2x2i64_4i32:
+; AVX512VL:       # %bb.0: # %entry
+; AVX512VL-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512VL-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpmovqd %ymm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc2x2i64_4i32:
+; AVX512BW:       # %bb.0: # %entry
+; AVX512BW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc2x2i64_4i32:
+; AVX512BWVL:       # %bb.0: # %entry
+; AVX512BWVL-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512BWVL-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovqd %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
 entry:
   %0 = trunc <2 x i64> %a to <2 x i32>
   %1 = trunc <2 x i64> %b to <2 x i32>
@@ -1637,10 +1614,10 @@ define <8 x i16> @trunc2x4i32_8i16(<4 x i32> %a, <4 x i32> %b) {
 ;
 ; AVX512VL-LABEL: trunc2x4i32_8i16:
 ; AVX512VL:       # %bb.0: # %entry
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512VL-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512VL-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: trunc2x4i32_8i16:
@@ -1653,8 +1630,10 @@ define <8 x i16> @trunc2x4i32_8i16(<4 x i32> %a, <4 x i32> %b) {
 ;
 ; AVX512BWVL-LABEL: trunc2x4i32_8i16:
 ; AVX512BWVL:       # %bb.0: # %entry
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,2,4,6,8,10,12,14]
-; AVX512BWVL-NEXT:    vpermt2w %xmm1, %xmm2, %xmm0
+; AVX512BWVL-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512BWVL-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovdw %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 entry:
   %0 = trunc <4 x i32> %a to <4 x i16>
@@ -1703,45 +1682,53 @@ entry:
 }
 
 define <16 x i8> @trunc2x8i16_16i8(<8 x i16> %a, <8 x i16> %b) {
-; SSE2-LABEL: trunc2x8i16_16i8:
-; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    packuswb %xmm1, %xmm0
-; SSE2-NEXT:    retq
-;
-; SSSE3-LABEL: trunc2x8i16_16i8:
-; SSSE3:       # %bb.0: # %entry
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; SSSE3-NEXT:    pshufb %xmm2, %xmm1
-; SSSE3-NEXT:    pshufb %xmm2, %xmm0
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSSE3-NEXT:    retq
-;
-; SSE41-LABEL: trunc2x8i16_16i8:
-; SSE41:       # %bb.0: # %entry
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; SSE41-NEXT:    pshufb %xmm2, %xmm1
-; SSE41-NEXT:    pshufb %xmm2, %xmm0
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE41-NEXT:    retq
+; SSE-LABEL: trunc2x8i16_16i8:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE-NEXT:    pand %xmm2, %xmm1
+; SSE-NEXT:    pand %xmm2, %xmm0
+; SSE-NEXT:    packuswb %xmm1, %xmm0
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: trunc2x8i16_16i8:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; AVX-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: trunc2x8i16_16i8:
-; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u>
-; AVX512-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: trunc2x8i16_16i8:
+; AVX512F:       # %bb.0: # %entry
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX512F-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512F-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc2x8i16_16i8:
+; AVX512VL:       # %bb.0: # %entry
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX512VL-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc2x8i16_16i8:
+; AVX512BW:       # %bb.0: # %entry
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX512BW-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc2x8i16_16i8:
+; AVX512BWVL:       # %bb.0: # %entry
+; AVX512BWVL-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512BWVL-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
 entry:
   %0 = trunc <8 x i16> %a to <8 x i8>
   %1 = trunc <8 x i16> %b to <8 x i8>
@@ -1776,11 +1763,29 @@ define i64 @trunc8i16_i64(<8 x i16> %inval) {
 ; AVX-NEXT:    vmovq %xmm0, %rax
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: trunc8i16_i64:
-; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
-; AVX512-NEXT:    vmovq %xmm0, %rax
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: trunc8i16_i64:
+; AVX512F:       # %bb.0: # %entry
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vmovq %xmm0, %rax
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: trunc8i16_i64:
+; AVX512VL:       # %bb.0: # %entry
+; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; AVX512VL-NEXT:    vmovq %xmm0, %rax
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: trunc8i16_i64:
+; AVX512BW:       # %bb.0: # %entry
+; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT:    vmovq %xmm0, %rax
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: trunc8i16_i64:
+; AVX512BWVL:       # %bb.0: # %entry
+; AVX512BWVL-NEXT:    vpmovwb %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vmovq %xmm0, %rax
+; AVX512BWVL-NEXT:    retq
 entry:
   %0 = trunc <8 x i16> %inval to <8 x i8>
   %1 = bitcast <8 x i8> %0 to i64
diff --git a/llvm/test/CodeGen/X86/x86-interleaved-access.ll b/llvm/test/CodeGen/X86/x86-interleaved-access.ll
index e557a9d9e1623..c323db1deeac1 100644
--- a/llvm/test/CodeGen/X86/x86-interleaved-access.ll
+++ b/llvm/test/CodeGen/X86/x86-interleaved-access.ll
@@ -774,94 +774,89 @@ define <32 x i1> @interleaved_load_vf32_i8_stride4(<128 x i8>* %ptr) {
 ;
 ; AVX512-LABEL: interleaved_load_vf32_i8_stride4:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm6 = <u,u,u,u,0,4,8,12,u,u,u,u,u,u,u,u>
-; AVX512-NEXT:    vmovdqa (%rdi), %xmm10
-; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm11
-; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm12
-; AVX512-NEXT:    vmovdqa 48(%rdi), %xmm13
-; AVX512-NEXT:    vpshufb %xmm6, %xmm13, %xmm4
-; AVX512-NEXT:    vpshufb %xmm6, %xmm12, %xmm5
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm0 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512-NEXT:    vpshufb %xmm0, %xmm11, %xmm5
-; AVX512-NEXT:    vpshufb %xmm0, %xmm10, %xmm7
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm5 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm8 = xmm5[0,1],xmm4[2,3]
-; AVX512-NEXT:    vmovdqa 112(%rdi), %xmm14
-; AVX512-NEXT:    vpshufb %xmm6, %xmm14, %xmm7
-; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = mem[2,3,0,1]
-; AVX512-NEXT:    vextracti128 $1, %ymm5, %xmm5
-; AVX512-NEXT:    vpshufb %xmm6, %xmm5, %xmm6
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm9
-; AVX512-NEXT:    vmovdqa 80(%rdi), %xmm6
-; AVX512-NEXT:    vpshufb %xmm0, %xmm6, %xmm1
-; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = mem[2,3,0,1]
-; AVX512-NEXT:    vextracti128 $1, %ymm7, %xmm7
-; AVX512-NEXT:    vpshufb %xmm0, %xmm7, %xmm0
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm9[6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3],ymm0[4,5,6,7]
+; AVX512-NEXT:    vmovdqa 112(%rdi), %xmm10
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = <u,u,u,u,0,4,8,12,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm2, %xmm10, %xmm3
+; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = mem[2,3,0,1]
+; AVX512-NEXT:    vextracti128 $1, %ymm1, %xmm11
+; AVX512-NEXT:    vpshufb %xmm2, %xmm11, %xmm2
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm4
+; AVX512-NEXT:    vmovdqa 80(%rdi), %xmm12
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm5 = <0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm5, %xmm12, %xmm6
+; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = mem[2,3,0,1]
+; AVX512-NEXT:    vextracti128 $1, %ymm3, %xmm13
+; AVX512-NEXT:    vpshufb %xmm5, %xmm13, %xmm5
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
+; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7]
+; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm5
+; AVX512-NEXT:    vpmovdb %zmm5, %xmm5
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm9 = ymm5[0,1,2,3],ymm4[4,5,6,7]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm0 = <u,u,u,u,1,5,9,13,u,u,u,u,u,u,u,u>
-; AVX512-NEXT:    vpshufb %xmm0, %xmm13, %xmm1
-; AVX512-NEXT:    vpshufb %xmm0, %xmm12, %xmm2
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm14
+; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm6
+; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm7
+; AVX512-NEXT:    vmovdqa 48(%rdi), %xmm4
+; AVX512-NEXT:    vpshufb %xmm0, %xmm4, %xmm1
+; AVX512-NEXT:    vpshufb %xmm0, %xmm7, %xmm2
 ; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = <1,5,9,13,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512-NEXT:    vpshufb %xmm2, %xmm11, %xmm3
-; AVX512-NEXT:    vpshufb %xmm2, %xmm10, %xmm4
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX512-NEXT:    vpshufb %xmm2, %xmm6, %xmm3
+; AVX512-NEXT:    vpshufb %xmm2, %xmm14, %xmm5
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
 ; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
-; AVX512-NEXT:    vpshufb %xmm0, %xmm14, %xmm3
-; AVX512-NEXT:    vpshufb %xmm0, %xmm5, %xmm0
+; AVX512-NEXT:    vpshufb %xmm0, %xmm10, %xmm3
+; AVX512-NEXT:    vpshufb %xmm0, %xmm11, %xmm0
 ; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
 ; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT:    vpshufb %xmm2, %xmm6, %xmm3
-; AVX512-NEXT:    vpshufb %xmm2, %xmm7, %xmm2
+; AVX512-NEXT:    vpshufb %xmm2, %xmm12, %xmm3
+; AVX512-NEXT:    vpshufb %xmm2, %xmm13, %xmm2
 ; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
 ; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm9 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm0 = <u,u,u,u,2,6,10,14,u,u,u,u,u,u,u,u>
-; AVX512-NEXT:    vpshufb %xmm0, %xmm13, %xmm1
-; AVX512-NEXT:    vpshufb %xmm0, %xmm12, %xmm2
+; AVX512-NEXT:    vpshufb %xmm0, %xmm4, %xmm1
+; AVX512-NEXT:    vpshufb %xmm0, %xmm7, %xmm2
 ; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = <2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512-NEXT:    vpshufb %xmm2, %xmm11, %xmm3
-; AVX512-NEXT:    vpshufb %xmm2, %xmm10, %xmm4
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX512-NEXT:    vpshufb %xmm2, %xmm6, %xmm3
+; AVX512-NEXT:    vpshufb %xmm2, %xmm14, %xmm5
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
 ; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
-; AVX512-NEXT:    vpshufb %xmm0, %xmm14, %xmm3
-; AVX512-NEXT:    vpshufb %xmm0, %xmm5, %xmm0
+; AVX512-NEXT:    vpshufb %xmm0, %xmm10, %xmm3
+; AVX512-NEXT:    vpshufb %xmm0, %xmm11, %xmm0
 ; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
 ; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT:    vpshufb %xmm2, %xmm6, %xmm3
-; AVX512-NEXT:    vpshufb %xmm2, %xmm7, %xmm2
+; AVX512-NEXT:    vpshufb %xmm2, %xmm12, %xmm3
+; AVX512-NEXT:    vpshufb %xmm2, %xmm13, %xmm2
 ; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
 ; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm15 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = <u,u,u,u,3,7,11,15,u,u,u,u,u,u,u,u>
-; AVX512-NEXT:    vpshufb %xmm1, %xmm13, %xmm2
-; AVX512-NEXT:    vpshufb %xmm1, %xmm12, %xmm3
+; AVX512-NEXT:    vpshufb %xmm1, %xmm4, %xmm2
+; AVX512-NEXT:    vpshufb %xmm1, %xmm7, %xmm3
 ; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = <3,7,11,15,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512-NEXT:    vpshufb %xmm3, %xmm11, %xmm4
-; AVX512-NEXT:    vpshufb %xmm3, %xmm10, %xmm0
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3]
-; AVX512-NEXT:    vpshufb %xmm1, %xmm14, %xmm2
-; AVX512-NEXT:    vpshufb %xmm1, %xmm5, %xmm1
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; AVX512-NEXT:    vpshufb %xmm3, %xmm6, %xmm4
+; AVX512-NEXT:    vpshufb %xmm3, %xmm14, %xmm5
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm2 = xmm4[0,1],xmm2[2,3]
+; AVX512-NEXT:    vpshufb %xmm1, %xmm10, %xmm4
+; AVX512-NEXT:    vpshufb %xmm1, %xmm11, %xmm1
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
 ; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512-NEXT:    vpshufb %xmm3, %xmm6, %xmm2
-; AVX512-NEXT:    vpshufb %xmm3, %xmm7, %xmm3
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512-NEXT:    vpcmpeqb %zmm9, %zmm8, %k0
-; AVX512-NEXT:    vpcmpeqb %zmm0, %zmm15, %k1
+; AVX512-NEXT:    vpshufb %xmm3, %xmm12, %xmm4
+; AVX512-NEXT:    vpshufb %xmm3, %xmm13, %xmm3
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
+; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512-NEXT:    vpcmpeqb %zmm8, %zmm9, %k0
+; AVX512-NEXT:    vpcmpeqb %zmm1, %zmm0, %k1
 ; AVX512-NEXT:    kxnord %k1, %k0, %k0
 ; AVX512-NEXT:    vpmovm2b %k0, %zmm0
 ; AVX512-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
@@ -1462,20 +1457,15 @@ define <64 x i8> @interleaved_load_vf64_i8_stride3(<192 x i8>* %ptr){
 ; AVX512-NEXT:    vpshufb %zmm3, %zmm2, %zmm2
 ; AVX512-NEXT:    vpalignr {{.*#+}} zmm3 = zmm2[11,12,13,14,15],zmm0[0,1,2,3,4,5,6,7,8,9,10],zmm2[27,28,29,30,31],zmm0[16,17,18,19,20,21,22,23,24,25,26],zmm2[43,44,45,46,47],zmm0[32,33,34,35,36,37,38,39,40,41,42],zmm2[59,60,61,62,63],zmm0[48,49,50,51,52,53,54,55,56,57,58]
 ; AVX512-NEXT:    vpalignr {{.*#+}} zmm0 = zmm0[11,12,13,14,15],zmm1[0,1,2,3,4,5,6,7,8,9,10],zmm0[27,28,29,30,31],zmm1[16,17,18,19,20,21,22,23,24,25,26],zmm0[43,44,45,46,47],zmm1[32,33,34,35,36,37,38,39,40,41,42],zmm0[59,60,61,62,63],zmm1[48,49,50,51,52,53,54,55,56,57,58]
+; AVX512-NEXT:    vpalignr {{.*#+}} zmm1 = zmm1[11,12,13,14,15],zmm2[0,1,2,3,4,5,6,7,8,9,10],zmm1[27,28,29,30,31],zmm2[16,17,18,19,20,21,22,23,24,25,26],zmm1[43,44,45,46,47],zmm2[32,33,34,35,36,37,38,39,40,41,42],zmm1[59,60,61,62,63],zmm2[48,49,50,51,52,53,54,55,56,57,58]
 ; AVX512-NEXT:    movabsq $-576188069258921984, %rax # imm = 0xF800F800F800F800
 ; AVX512-NEXT:    kmovq %rax, %k1
-; AVX512-NEXT:    vpalignr {{.*#+}} ymm4 = ymm0[11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7,8,9,10],ymm0[27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23,24,25,26]
-; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm5
-; AVX512-NEXT:    vpalignr {{.*#+}} zmm0 {%k1} = zmm1[11,12,13,14,15],zmm2[0,1,2,3,4,5,6,7,8,9,10],zmm1[27,28,29,30,31],zmm2[16,17,18,19,20,21,22,23,24,25,26],zmm1[43,44,45,46,47],zmm2[32,33,34,35,36,37,38,39,40,41,42],zmm1[59,60,61,62,63],zmm2[48,49,50,51,52,53,54,55,56,57,58]
-; AVX512-NEXT:    vpalignr {{.*#+}} zmm1 = zmm1[11,12,13,14,15],zmm2[0,1,2,3,4,5,6,7,8,9,10],zmm1[27,28,29,30,31],zmm2[16,17,18,19,20,21,22,23,24,25,26],zmm1[43,44,45,46,47],zmm2[32,33,34,35,36,37,38,39,40,41,42],zmm1[59,60,61,62,63],zmm2[48,49,50,51,52,53,54,55,56,57,58]
+; AVX512-NEXT:    vpblendmb %zmm1, %zmm0, %zmm2 {%k1}
 ; AVX512-NEXT:    vpalignr {{.*#+}} zmm1 = zmm3[11,12,13,14,15],zmm1[0,1,2,3,4,5,6,7,8,9,10],zmm3[27,28,29,30,31],zmm1[16,17,18,19,20,21,22,23,24,25,26],zmm3[43,44,45,46,47],zmm1[32,33,34,35,36,37,38,39,40,41,42],zmm3[59,60,61,62,63],zmm1[48,49,50,51,52,53,54,55,56,57,58]
+; AVX512-NEXT:    vpaddb %zmm1, %zmm2, %zmm1
+; AVX512-NEXT:    vpalignr {{.*#+}} zmm0 = zmm0[11,12,13,14,15],zmm3[0,1,2,3,4,5,6,7,8,9,10],zmm0[27,28,29,30,31],zmm3[16,17,18,19,20,21,22,23,24,25,26],zmm0[43,44,45,46,47],zmm3[32,33,34,35,36,37,38,39,40,41,42],zmm0[59,60,61,62,63],zmm3[48,49,50,51,52,53,54,55,56,57,58]
+; AVX512-NEXT:    vpalignr {{.*#+}} zmm0 = zmm0[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25,42,43,44,45,46,47,32,33,34,35,36,37,38,39,40,41,58,59,60,61,62,63,48,49,50,51,52,53,54,55,56,57]
 ; AVX512-NEXT:    vpaddb %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vpalignr {{.*#+}} ymm1 = ymm4[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]
-; AVX512-NEXT:    vextracti64x4 $1, %zmm3, %ymm2
-; AVX512-NEXT:    vpalignr {{.*#+}} ymm2 = ymm5[11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7,8,9,10],ymm5[27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23,24,25,26]
-; AVX512-NEXT:    vpalignr {{.*#+}} ymm2 = ymm2[10,11,12,13,14,15,0,1,2,3,4,5,6,7,8,9,26,27,28,29,30,31,16,17,18,19,20,21,22,23,24,25]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
-; AVX512-NEXT:    vpaddb %zmm0, %zmm1, %zmm0
 ; AVX512-NEXT:    retq
 %wide.vec = load <192 x i8>, <192 x i8>* %ptr, align 1
 %v1 = shufflevector <192 x i8> %wide.vec, <192 x i8> undef, <64 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45, i32 48, i32 51, i32 54, i32 57, i32 60, i32 63, i32 66, i32 69, i32 72, i32 75, i32 78, i32 81, i32 84, i32 87, i32 90, i32 93, i32 96, i32 99, i32 102, i32 105, i32 108, i32 111, i32 114, i32 117, i32 120, i32 123, i32 126, i32 129, i32 132, i32 135, i32 138, i32 141, i32 144, i32 147, i32 150, i32 153, i32 156, i32 159, i32 162, i32 165, i32 168, i32 171, i32 174, i32 177, i32 180, i32 183, i32 186, i32 189>
diff --git a/llvm/test/DebugInfo/ARM/cfi-eof-prologue.mir b/llvm/test/DebugInfo/ARM/cfi-eof-prologue.mir
index ab8154edf8c1f..ea68187e32a04 100644
--- a/llvm/test/DebugInfo/ARM/cfi-eof-prologue.mir
+++ b/llvm/test/DebugInfo/ARM/cfi-eof-prologue.mir
@@ -139,6 +139,7 @@
 ---
 name:            _ZN1BC2Ev
 alignment:       2
+tracksRegLiveness: true
 liveins:         
   - { reg: '$r0' }
 frameInfo:       
@@ -154,6 +155,8 @@ stack:
       callee-saved-register: '$r4' }
 body:             |
   bb.0.entry:
+    liveins: $r0, $r4, $lr
+
     frame-setup tPUSH 14, $noreg, killed $r4, killed $lr, implicit-def $sp, implicit $sp
     frame-setup CFI_INSTRUCTION def_cfa_offset 8
     frame-setup CFI_INSTRUCTION offset $lr, -4
@@ -175,6 +178,7 @@ body:             |
 ---
 name:            _ZN1BC1Ev
 alignment:       2
+tracksRegLiveness: true
 liveins:         
   - { reg: '$r0' }
 frameInfo:       
@@ -190,6 +194,8 @@ stack:
       callee-saved-register: '$r4' }
 body:             |
   bb.0.entry:
+    liveins: $r0, $r4, $lr
+
     frame-setup tPUSH 14, $noreg, killed $r4, killed $lr, implicit-def $sp, implicit $sp
     frame-setup CFI_INSTRUCTION def_cfa_offset 8
     frame-setup CFI_INSTRUCTION offset $lr, -4
diff --git a/llvm/test/DebugInfo/MIR/ARM/larger-subregister.mir b/llvm/test/DebugInfo/MIR/ARM/larger-subregister.mir
index 6fb944e8c97ce..a9d1dd44192f9 100644
--- a/llvm/test/DebugInfo/MIR/ARM/larger-subregister.mir
+++ b/llvm/test/DebugInfo/MIR/ARM/larger-subregister.mir
@@ -29,8 +29,11 @@
   !13 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !14)
   !14 = !DICompositeType(tag: DW_TAG_structure_type, name: "CGRect", scope: !5, file: !3, line: 46, size: 128, elements: !4, runtimeLang: DW_LANG_Swift, identifier: "$sSo6CGRectVD")
 name:            'f'
+tracksRegLiveness: true
 body:             |
   bb.0:
+    liveins: $r6
+
     DBG_VALUE $s31, $noreg, !12, !DIExpression(DW_OP_LLVM_fragment, 32, 32), debug-location !10
     DBG_VALUE $q8, $noreg, !12, !DIExpression(DW_OP_LLVM_fragment, 0, 32), debug-location !10
     renamable $r0 = t2ADDri $r6, 144, 14, $noreg, $noreg, debug-location !10
diff --git a/llvm/test/DebugInfo/MIR/ARM/param-reg-const-mix.mir b/llvm/test/DebugInfo/MIR/ARM/param-reg-const-mix.mir
index 99511280f223e..83b5a6f394dc8 100644
--- a/llvm/test/DebugInfo/MIR/ARM/param-reg-const-mix.mir
+++ b/llvm/test/DebugInfo/MIR/ARM/param-reg-const-mix.mir
@@ -67,9 +67,11 @@
 ...
 ---
 name:            fn1
-tracksRegLiveness: false
+tracksRegLiveness: true
 body:             |
   bb.0.entry:
+    liveins: $r0, $r1, $r4, $lr
+
     $sp = frame-setup STMDB_UPD $sp, 14, $noreg, killed $r4, killed $lr
     $r4 = MOVr $r0, 14, $noreg, $noreg
     DBG_VALUE $r1, $noreg, !17, !DIExpression(DW_OP_LLVM_fragment, 32, 32), debug-location !18
diff --git a/llvm/test/DebugInfo/MIR/ARM/subregister-full-piece.mir b/llvm/test/DebugInfo/MIR/ARM/subregister-full-piece.mir
index fe1660d7f231d..1fa172b976456 100644
--- a/llvm/test/DebugInfo/MIR/ARM/subregister-full-piece.mir
+++ b/llvm/test/DebugInfo/MIR/ARM/subregister-full-piece.mir
@@ -39,8 +39,12 @@
   !9 = !DILocalVariable(name: "q9", scope: !5, file: !4, line: 1, type: !7)
   !20 = !DILocation(line: 0, scope: !5)
 name:            f
+tracksRegLiveness: true
 body:             |
   bb.2.for.body:
+    liveins: $r2
+
+    t2CMPri killed renamable $r2, 5, 14, $noreg, implicit-def $cpsr
     t2Bcc %bb.2.for.body, 0, killed $cpsr, debug-location !20
     DBG_VALUE $q8, $noreg, !8, !DIExpression(DW_OP_LLVM_fragment, 0, 64), debug-location !20
     DBG_VALUE $q9, $noreg, !9, !DIExpression(DW_OP_LLVM_fragment, 0, 56), debug-location !20
diff --git a/llvm/test/DebugInfo/X86/dead-store-elimination-marks-undef.ll b/llvm/test/DebugInfo/X86/dead-store-elimination-marks-undef.ll
new file mode 100644
index 0000000000000..5f9e67653d966
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/dead-store-elimination-marks-undef.ll
@@ -0,0 +1,36 @@
+; RUN: opt -mtriple=x86_64-- -S --dse %s  -o - | FileCheck %s
+; Ensure that we can mark a value as undefined when performing dead 
+; store elimination.
+; Bugzilla #45080
+
+@b = common dso_local local_unnamed_addr global i32 0, align 1
+
+define dso_local i32 @main() local_unnamed_addr !dbg !7 {
+  %1 = alloca i32, align 4
+  %2 = load i32, i32* @b, align 1, !dbg !13
+  ; CHECK: call void @llvm.dbg.value(metadata i32 undef
+  call void @llvm.dbg.value(metadata i32 %2, metadata !12, metadata !DIExpression()), !dbg !13
+  store i32 %2, i32* %1, align 4, !dbg !13
+  ret i32 0, !dbg !13
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, globals: !2, nameTableKind: None)
+!1 = !DIFile(filename: "dead-store-elimination-marks-undef.ll", directory: "/temp/bz45080")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!6 = !{!"clang version 10.0.0"}
+!7 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 3, type: !8, scopeLine: 3, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !{!12}
+!12 = !DILocalVariable(name: "l_2864", scope: !7, file: !1, line: 4, type: !10)
+!13 = !DILocation(line: 5, column: 12, scope: !7)
+
diff --git a/llvm/test/DebugInfo/X86/debug-macro-macinfo.s b/llvm/test/DebugInfo/X86/debug-macro-macinfo.s
new file mode 100644
index 0000000000000..a5337cafc9074
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/debug-macro-macinfo.s
@@ -0,0 +1,47 @@
+## This test checks that llvm-dwarfdump can dump both debug_macro and debug_macinfo
+## sections present in the same object.
+
+# RUN: llvm-mc -triple x86_64-unknown-linux -filetype=obj %s -o -| \
+# RUN:   llvm-dwarfdump -debug-macro - | FileCheck -strict-whitespace -match-full-lines %s
+
+#      CHECK:.debug_macro contents:
+# CHECK-NEXT:0x00000000:
+# CHECK-NEXT:macro header: version = 0x0005, flags = 0x02, debug_line_offset = 0x0000
+# CHECK-NEXT:DW_MACRO_start_file - lineno: 0 filenum: 0
+# CHECK-NEXT:  DW_MACRO_define_strp - lineno: 1 macro: DWARF_VERSION 5
+# CHECK-NEXT:DW_MACRO_end_file
+
+#      CHECK:.debug_macinfo contents:
+# CHECK-NEXT:0x00000000:
+# CHECK-NEXT:DW_MACINFO_start_file - lineno: 0 filenum: 2
+# CHECK-NEXT:  DW_MACINFO_define - lineno: 6 macro: DWARF_VERSION 4
+# CHECK-NEXT:DW_MACINFO_end_file
+
+	.section	.debug_macro,"",@progbits
+.Lcu_macro_begin0:
+	.short	5                      # Macro information version
+	.byte	2                       # Flags: 32 bit, debug_line_offset present
+	.long	0                       # debug_line_offset
+	.byte	3                       # DW_MACRO_start_file
+	.byte	0                       # Line Number
+	.byte	0                       # File Number
+	.byte	5                       # DW_MACRO_define_strp
+	.byte	1                       # Line Number
+	.long	.Linfo_string0          # Macro String
+	.byte	4                       # DW_MACRO_end_file
+	.byte	0                       # End Of Macro List Mark
+
+	.section	.debug_macinfo,"",@progbits
+.Lcu_macro_begin1:
+	.byte	3                       # DW_MACINFO_start_file
+	.byte	0                       # Line Number
+	.byte	2                       # File Number
+	.byte	1                       # DW_MACINFO_define
+	.byte	6                       # Line Number
+	.asciz	"DWARF_VERSION 4"      # Macro String
+	.byte	4                       # DW_MACINFO_end_file
+	.byte	0                       # End Of Macro List Mark
+
+	.section	.debug_str,"MS",@progbits,1
+.Linfo_string0:
+	.asciz	"DWARF_VERSION 5"
diff --git a/llvm/test/DebugInfo/X86/debug-macro-v5.s b/llvm/test/DebugInfo/X86/debug-macro-v5.s
new file mode 100644
index 0000000000000..b4e5772574556
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/debug-macro-v5.s
@@ -0,0 +1,54 @@
+## This test checks llvm-dwarfdump can dump debug_macro section containing
+## multiple macro units.
+
+# RUN: llvm-mc -triple x86_64-unknown-linux -filetype=obj %s -o -| \
+# RUN:   llvm-dwarfdump -debug-macro - | FileCheck -strict-whitespace -match-full-lines %s
+
+#      CHECK:.debug_macro contents:
+# CHECK-NEXT:0x00000000:
+# CHECK-NEXT:macro header: version = 0x0005, flags = 0x02, debug_line_offset = 0x0000
+# CHECK-NEXT:DW_MACRO_start_file - lineno: 0 filenum: 0
+# CHECK-NEXT:  DW_MACRO_start_file - lineno: 1 filenum: 6
+# CHECK-NEXT:    DW_MACRO_define_strp - lineno: 1 macro: FOO 5
+# CHECK-NEXT:  DW_MACRO_end_file
+# CHECK-NEXT:  DW_MACRO_undef_strp - lineno: 8 macro: WORLD1
+# CHECK-NEXT:DW_MACRO_end_file
+
+#      CHECK:0x{{.*}}:
+# CHECK-NEXT:macro header: version = 0x0005, flags = 0x00
+# CHECK-NEXT:DW_MACRO_define_strp - lineno: 0 macro: WORLD 2
+
+	.section	.debug_macro,"",@progbits
+.Lcu_macro_begin0:
+	.short	5                      # Macro information version
+	.byte	2                       # Flags: 32 bit, debug_line_offset present
+	.long	0                       # debug_line_offset
+	.byte	3                       # DW_MACRO_start_file
+	.byte	0                       # Line Number
+	.byte	0                       # File Number
+	.byte	3                       # DW_MACRO_start_file
+	.byte	1                       # Line Number
+	.byte	6                       # File Number
+	.byte	5                       # DW_MACRO_define_strp
+	.byte	1                       # Line Number
+	.long	.Linfo_string0          # Macro String
+	.byte	4                       # DW_MACRO_end_file
+	.byte	6                       # DW_MACRO_undef_strp
+	.byte	8                       # Line Number
+	.long	.Linfo_string1          # Macro String
+	.byte	4                       # DW_MACRO_end_file
+	.byte	0                       # End Of Macro List Mark
+	.short	5                      # Macro information version
+	.byte	0                       # Flags: 32 bit
+	.byte	5                       # DW_MACRO_define_strp
+	.byte	0                       # Line Number
+	.long	.Linfo_string2          # Macro String
+	.byte	0                       # End Of Macro List Mark
+
+	.section	.debug_str,"MS",@progbits,1
+.Linfo_string0:
+	.asciz	"FOO 5"
+.Linfo_string1:
+	.asciz	"WORLD1"
+.Linfo_string2:
+	.asciz	"WORLD 2"
diff --git a/llvm/test/DebugInfo/X86/instcombine-demanded-bits-salvage.ll b/llvm/test/DebugInfo/X86/instcombine-demanded-bits-salvage.ll
new file mode 100644
index 0000000000000..814a3bb8dc881
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/instcombine-demanded-bits-salvage.ll
@@ -0,0 +1,38 @@
+; RUN: opt -mtriple=x86_64-- %s -S --instcombine  -o - | FileCheck %s
+; Verify that demanded bits optimisations don't affect debuginfo 
+; variable values.
+; Bugzilla #44371
+
+@a = common dso_local local_unnamed_addr global i32 0, align 4
+
+define dso_local i32 @p() local_unnamed_addr !dbg !11 {
+entry:
+  %conv = load i32, i32* @a, align 4, !dbg !14
+  %0 = and i32 %conv, 65535, !dbg !14
+  ; CHECK: metadata !DIExpression(DW_OP_constu, 65535, DW_OP_and, DW_OP_stack_value))
+  call void @llvm.dbg.value(metadata i32 %0, metadata !15, metadata !DIExpression()), !dbg !14
+  %1 = lshr i32 %0, 12, !dbg !14
+  %2 = and i32 %1, 8, !dbg !14
+  %3 = xor i32 %2, 8, !dbg !14
+  ret i32 0, !dbg !14
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, globals: !2, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "instcomnine.ll", directory: "/temp/bz44371")
+!2 = !{}
+!3 = !{!7}
+!7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!8 = !{i32 7, !"Dwarf Version", i32 4}
+!9 = !{i32 2, !"Debug Info Version", i32 3}
+!10 = !{!""}
+!11 = distinct !DISubprogram(name: "p", scope: !1, file: !1, line: 6, type: !12, scopeLine: 6, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!12 = !DISubroutineType(types: !13)
+!13 = !{!7}
+!14 = !DILocation(line: 4, column: 35, scope: !11)
+!15 = !DILocalVariable(name: "p_28", arg: 1, scope: !11, file: !1, line: 16, type: !7)
diff --git a/llvm/test/DebugInfo/X86/licm-undef-dbg-value.ll b/llvm/test/DebugInfo/X86/licm-undef-dbg-value.ll
new file mode 100644
index 0000000000000..09a18f937b3bb
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/licm-undef-dbg-value.ll
@@ -0,0 +1,86 @@
+; RUN: opt -licm %s -S | FileCheck %s
+
+; CHECK: for.body:
+; CHECK-NEXT: llvm.dbg.value(metadata i8 undef
+
+; The load is loop invariant. Check that we leave an undef dbg.value behind
+; when licm sinks the instruction.
+
+; clang reduce.cpp -g -O2 -Xclang -disable-llvm-passes -emit-llvm -o reduce.ll
+; opt -simplifycfg -sroa -loop-rotate -o - -S reduce.ll
+; cat reduce.cpp
+; extern char a;
+; extern char b;
+; void use(char);
+; void fun() {
+;   char local = 0;
+;   for (;b;)
+;     local = a;
+;   use(local);
+; }
+
+@b = external dso_local global i8, align 1
+@a = external dso_local global i8, align 1
+
+define dso_local void @_Z3funv() !dbg !12 {
+entry:
+  call void @llvm.dbg.value(metadata i8 0, metadata !16, metadata !DIExpression()), !dbg !17
+  %0 = load i8, i8* @b, align 1, !dbg !18
+  %tobool1 = icmp ne i8 %0, 0, !dbg !18
+  br i1 %tobool1, label %for.body.lr.ph, label %for.end, !dbg !24
+
+for.body.lr.ph:                                   ; preds = %entry
+  br label %for.body, !dbg !24
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.body
+  %1 = load i8, i8* @a, align 1, !dbg !25
+  call void @llvm.dbg.value(metadata i8 %1, metadata !16, metadata !DIExpression()), !dbg !17
+  %2 = load i8, i8* @b, align 1, !dbg !18
+  %tobool = icmp ne i8 %2, 0, !dbg !18
+  br i1 %tobool, label %for.body, label %for.cond.for.end_crit_edge, !dbg !24, !llvm.loop !26
+
+for.cond.for.end_crit_edge:                       ; preds = %for.body
+  %split = phi i8 [ %1, %for.body ]
+  br label %for.end, !dbg !24
+
+for.end:                                          ; preds = %for.cond.for.end_crit_edge, %entry
+  %local.0.lcssa = phi i8 [ %split, %for.cond.for.end_crit_edge ], [ 0, %entry ], !dbg !17
+  call void @llvm.dbg.value(metadata i8 %local.0.lcssa, metadata !16, metadata !DIExpression()), !dbg !17
+  call void @_Z3usec(i8 signext %local.0.lcssa), !dbg !28
+  ret void, !dbg !29
+}
+
+declare !dbg !4 dso_local void @_Z3usec(i8 signext)
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!8, !9, !10}
+!llvm.ident = !{!11}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 11.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "reduce.cpp", directory: "/")
+!2 = !{}
+!3 = !{!4}
+!4 = !DISubprogram(name: "use", linkageName: "_Z3usec", scope: !1, file: !1, line: 3, type: !5, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{null, !7}
+!7 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!8 = !{i32 7, !"Dwarf Version", i32 4}
+!9 = !{i32 2, !"Debug Info Version", i32 3}
+!10 = !{i32 1, !"wchar_size", i32 4}
+!11 = !{!"clang version 11.0.0"}
+!12 = distinct !DISubprogram(name: "fun", linkageName: "_Z3funv", scope: !1, file: !1, line: 4, type: !13, scopeLine: 4, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !15)
+!13 = !DISubroutineType(types: !14)
+!14 = !{null}
+!15 = !{!16}
+!16 = !DILocalVariable(name: "local", scope: !12, file: !1, line: 5, type: !7)
+!17 = !DILocation(line: 0, scope: !12)
+!18 = !DILocation(line: 6, column: 9, scope: !19)
+!19 = distinct !DILexicalBlock(scope: !20, file: !1, line: 6, column: 3)
+!20 = distinct !DILexicalBlock(scope: !12, file: !1, line: 6, column: 3)
+!24 = !DILocation(line: 6, column: 3, scope: !20)
+!25 = !DILocation(line: 7, column: 13, scope: !19)
+!26 = distinct !{!26, !24, !27}
+!27 = !DILocation(line: 7, column: 13, scope: !20)
+!28 = !DILocation(line: 8, column: 3, scope: !12)
+!29 = !DILocation(line: 9, column: 1, scope: !12)
diff --git a/llvm/test/DebugInfo/X86/unsupported-dwarf64-debug-macro-v5.s b/llvm/test/DebugInfo/X86/unsupported-dwarf64-debug-macro-v5.s
new file mode 100644
index 0000000000000..a06c97ccf5e78
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/unsupported-dwarf64-debug-macro-v5.s
@@ -0,0 +1,13 @@
+## This test checks llvm-dwarfdump emits correct error diagnostics for the
+## unsupported case where DWARF64 flag is present in the debug_macro section header.
+
+# RUN: llvm-mc -triple x86_64-unknown-linux -filetype=obj %s -o -| \
+# RUN:   not llvm-dwarfdump -debug-macro - /dev/null 2>&1 | FileCheck %s
+
+# CHECK:error: DWARF64 is not supported
+
+	.section	.debug_macro,"",@progbits
+.Lcu_macro_begin0:
+	.short	5                      # Macro information version
+	.byte	3                       # Flags: 64 bit, debug_line_offset present
+	.quad	0                       # debug_line_offset
diff --git a/llvm/test/DebugInfo/X86/unsupported-opcode_operands_table-debug-macro-v5.s b/llvm/test/DebugInfo/X86/unsupported-opcode_operands_table-debug-macro-v5.s
new file mode 100644
index 0000000000000..39e557ec5c4eb
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/unsupported-opcode_operands_table-debug-macro-v5.s
@@ -0,0 +1,13 @@
+## This test checks llvm-dwarfdump emits correct error diagnostics for the
+## unsupported case where the opcode_operands_table flag is present in the
+## debug_macro section header.
+
+# RUN: llvm-mc -triple x86_64-unknown-linux -filetype=obj %s -o -| \
+# RUN:   not llvm-dwarfdump -debug-macro - /dev/null 2>&1 | FileCheck %s
+
+# CHECK:error: opcode_operands_table is not supported
+
+	.section	.debug_macro,"",@progbits
+.Lcu_macro_begin0:
+	.short	5                      # Macro information version
+	.byte	4                       # Flags: 32 bit, debug_line_offset absent, opcode_operands_table present
diff --git a/llvm/test/ExecutionEngine/MCJIT/cet-code-model-lager.ll b/llvm/test/ExecutionEngine/MCJIT/cet-code-model-lager.ll
new file mode 100644
index 0000000000000..478683b016d3d
--- /dev/null
+++ b/llvm/test/ExecutionEngine/MCJIT/cet-code-model-lager.ll
@@ -0,0 +1,32 @@
+; RUN: %lli -code-model=large %s > /dev/null
+
+@a = dso_local local_unnamed_addr global i32 1, align 4
+
+; Function Attrs: nofree noinline norecurse nounwind uwtable writeonly
+define dso_local void @ext() local_unnamed_addr #0 {
+entry:
+  store i32 0, i32* @a, align 4
+  ret void
+}
+
+; Function Attrs: nofree norecurse nounwind uwtable
+define dso_local i32 @main() local_unnamed_addr #1 {
+entry:
+  tail call fastcc void @foo()
+  %0 = load i32, i32* @a, align 4
+  ret i32 %0
+}
+
+; Function Attrs: nofree noinline norecurse nounwind uwtable writeonly
+define internal fastcc void @foo() unnamed_addr #0 {
+entry:
+  tail call void @ext()
+  ret void
+}
+
+!llvm.module.flags = !{!0, !1, !2, !3}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"cf-protection-return", i32 1}
+!2 = !{i32 4, !"cf-protection-branch", i32 1}
+!3 = !{i32 1, !"Code Model", i32 4}
diff --git a/llvm/test/ExecutionEngine/OrcLazy/Inputs/bar-return-i32-call-foo.ll b/llvm/test/ExecutionEngine/OrcLazy/Inputs/bar-return-i32-call-foo.ll
new file mode 100644
index 0000000000000..5a36041640ce6
--- /dev/null
+++ b/llvm/test/ExecutionEngine/OrcLazy/Inputs/bar-return-i32-call-foo.ll
@@ -0,0 +1,8 @@
+declare i32 @foo()
+
+define i32 @bar() {
+entry:
+  %0 = call i32 @foo()
+  ret i32 %0
+}
+
diff --git a/llvm/test/ExecutionEngine/OrcLazy/Inputs/basic-object-source.ll b/llvm/test/ExecutionEngine/OrcLazy/Inputs/foo-return-i32-0.ll
similarity index 100%
rename from llvm/test/ExecutionEngine/OrcLazy/Inputs/basic-object-source.ll
rename to llvm/test/ExecutionEngine/OrcLazy/Inputs/foo-return-i32-0.ll
diff --git a/llvm/test/ExecutionEngine/OrcLazy/Inputs/noop-main.ll b/llvm/test/ExecutionEngine/OrcLazy/Inputs/noop-main.ll
new file mode 100644
index 0000000000000..8438173f48b05
--- /dev/null
+++ b/llvm/test/ExecutionEngine/OrcLazy/Inputs/noop-main.ll
@@ -0,0 +1,7 @@
+; Can be used as an entry point for tests that rely purely on static
+; initializer side-effects.
+
+define i32 @main(i32 %argc, i8** %argv) {
+entry:
+  ret i32 0
+}
diff --git a/llvm/test/ExecutionEngine/OrcLazy/basic-object-file-loading.ll b/llvm/test/ExecutionEngine/OrcLazy/basic-object-file-loading.ll
index 0d815782b1cb0..9dc74d5241bb5 100644
--- a/llvm/test/ExecutionEngine/OrcLazy/basic-object-file-loading.ll
+++ b/llvm/test/ExecutionEngine/OrcLazy/basic-object-file-loading.ll
@@ -1,4 +1,4 @@
-; RUN: llc -filetype=obj -o %t %p/Inputs/basic-object-source.ll
+; RUN: llc -filetype=obj -o %t %p/Inputs/foo-return-i32-0.ll
 ; RUN: lli -jit-kind=orc-lazy -extra-object %t %s
 ;
 ; Check that we can load an object file and call a function in it.
diff --git a/llvm/test/ExecutionEngine/OrcLazy/global-ctors-and-dtors.ll b/llvm/test/ExecutionEngine/OrcLazy/global-ctors-and-dtors.ll
index 00b54fbf73fd7..67d392e71456a 100644
--- a/llvm/test/ExecutionEngine/OrcLazy/global-ctors-and-dtors.ll
+++ b/llvm/test/ExecutionEngine/OrcLazy/global-ctors-and-dtors.ll
@@ -1,6 +1,12 @@
-; RUN: lli -jit-kind=orc-lazy -orc-lazy-debug=funcs-to-stdout %s | FileCheck %s
+; Test that global constructors and destructors are run:
 ;
-; Test that global constructors and destructors are run.
+; RUN: lli -jit-kind=orc-lazy -orc-lazy-debug=funcs-to-stdout -extra-module %s \
+; RUN:   %S/Inputs/noop-main.ll | FileCheck %s
+;
+; Test that this is true for global constructors and destructors in other
+; JITDylibs.
+; RUN: lli -jit-kind=orc-lazy -orc-lazy-debug=funcs-to-stdout \
+; RUN:   -jd extra -extra-module %s -jd main %S/Inputs/noop-main.ll | FileCheck %s
 ;
 ; CHECK: Hello
 ; CHECK: [ {{.*}}main{{.*}} ]
@@ -22,11 +28,6 @@ entry:
 
 declare i32 @__cxa_atexit(void (i8*)*, i8*, i8*)
 
-define i32 @main(i32 %argc, i8** nocapture readnone %argv) {
-entry:
-  ret i32 0
-}
-
 define internal void @_GLOBAL__sub_I_hello.cpp() {
 entry:
   %puts.i.i.i = tail call i32 @puts(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @str, i64 0, i64 0))
diff --git a/llvm/test/ExecutionEngine/OrcLazy/static-library-support.ll b/llvm/test/ExecutionEngine/OrcLazy/static-library-support.ll
index 304160c7f787f..a13441187aa89 100644
--- a/llvm/test/ExecutionEngine/OrcLazy/static-library-support.ll
+++ b/llvm/test/ExecutionEngine/OrcLazy/static-library-support.ll
@@ -1,11 +1,13 @@
 ; This first line will generate the .o files for the next run line
-; RUN: llc -filetype=obj -o %t.o %p/Inputs/basic-object-source.ll
-; RUN: llvm-ar r %t.a %t.o
-; RUN: lli -jit-kind=orc-lazy -extra-archive %t.a %s
+; RUN: rm -rf %t && mkdir -p %t
+; RUN: llc -filetype=obj -o %t/foo.o %p/Inputs/foo-return-i32-0.ll
+; RUN: llc -filetype=obj -o %t/bar.o %p/Inputs/bar-return-i32-call-foo.ll
+; RUN: llvm-ar r %t/staticlib.a %t/foo.o %t/bar.o
+; RUN: lli -jit-kind=orc-lazy -extra-archive %t/staticlib.a %s
 
-declare i32 @foo()
+declare i32 @bar()
 
 define i32 @main() {
-  %r = call i32 @foo( )   ; <i32> [#uses=1]
+  %r = call i32 @bar()   ; <i32> [#uses=1]
   ret i32 %r
 }
diff --git a/llvm/test/FileCheck/numeric-defines-diagnostics.txt b/llvm/test/FileCheck/numeric-defines-diagnostics.txt
index da8c9cc8884e6..ddced3721f1de 100644
--- a/llvm/test/FileCheck/numeric-defines-diagnostics.txt
+++ b/llvm/test/FileCheck/numeric-defines-diagnostics.txt
@@ -28,6 +28,7 @@ NUMERRCLITRAIL-NEXT:Global define #1: #VALUE+2=10 (parsed as: {{\[\[#VALUE\+2:10
      NUMERRCLITRAIL-NEXT:                                                  ^
 
 ; Invalid format for variable.
+RUN: %ProtectFileCheckOutput \
 RUN: not FileCheck -D#,VALUE=10 --input-file %s %s 2>&1 \
 RUN:   | FileCheck %s --strict-whitespace --match-full-lines --check-prefix NUMERRCLIFMT
 
diff --git a/llvm/test/FileCheck/numeric-expression.txt b/llvm/test/FileCheck/numeric-expression.txt
index 14e3782817936..8396b6e2de11c 100644
--- a/llvm/test/FileCheck/numeric-expression.txt
+++ b/llvm/test/FileCheck/numeric-expression.txt
@@ -41,8 +41,10 @@ CHECK-LABEL: DEF FMT SPC
 CHECK-NEXT: [[# %x , VAR2a : ]]
 
 ; Numeric variable definition with unsupported matching format.
+RUN: %ProtectFileCheckOutput \
 RUN: not FileCheck --check-prefixes ERR,INVALID-FMT-SPEC1 --input-file %s %s 2>&1 \
 RUN:   | FileCheck --check-prefix INVALID-FMT-SPEC-MSG1 --strict-whitespace %s
+RUN: %ProtectFileCheckOutput \
 RUN: not FileCheck --check-prefixes ERR,INVALID-FMT-SPEC2 --input-file %s %s 2>&1 \
 RUN:   | FileCheck --check-prefix INVALID-FMT-SPEC-MSG2 --strict-whitespace %s
 
@@ -182,6 +184,7 @@ CHECK-NEXT: [[# %u, VAR2]]
 CHECK-NEXT: [[# %u, VAR3]]
 
 ; Conflicting implicit format.
+RUN: %ProtectFileCheckOutput \
 RUN: not FileCheck --check-prefixes CHECK,FMT-CONFLICT --input-file %s %s 2>&1 \
 RUN:   | FileCheck --strict-whitespace --check-prefix FMT-CONFLICT-MSG %s
 
@@ -359,6 +362,7 @@ SAME-LINE-USE-MSG2-NEXT: {{S}}AME-LINE-USE2-NEXT: {{\[\[#VAR2:VAR1\+1\]\] \[\[#V
 SAME-LINE-USE-MSG2-NEXT:        {{^}}                                         ^{{$}}
 
 ; Invalid change of format in variable redefinition.
+RUN: %ProtectFileCheckOutput \
 RUN: not FileCheck --check-prefix REDEF-NEW-FMT --input-file %s %s 2>&1 \
 RUN:   | FileCheck --strict-whitespace --check-prefix REDEF-NEW-FMT-MSG %s
 
diff --git a/llvm/test/IR/assume-builder.ll b/llvm/test/IR/assume-builder.ll
deleted file mode 100644
index 955be3a8225c9..0000000000000
--- a/llvm/test/IR/assume-builder.ll
+++ /dev/null
@@ -1,66 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes='assume-builder,verify' --enable-knowledge-retention -S %s | FileCheck %s --check-prefixes=BASIC
-; RUN: opt -passes='assume-builder,verify' --enable-knowledge-retention --assume-preserve-all -S %s | FileCheck %s --check-prefixes=ALL
-
-declare void @func(i32*, i32*)
-declare void @func_cold(i32*) cold
-declare void @func_strbool(i32*) "no-jump-tables"
-declare void @func_many(i32*) "no-jump-tables" nounwind "less-precise-fpmad" willreturn norecurse
-declare void @func_argattr(i32* align 8, i32* nonnull) nounwind
-
-define void @test(i32* %P, i32* %P1, i32* %P2, i32* %P3) {
-; BASIC-LABEL: @test(
-; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P:%.*]], i64 16), "nonnull"(i32* [[P]]) ]
-; BASIC-NEXT:    call void @func(i32* nonnull dereferenceable(16) [[P]], i32* null)
-; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P1:%.*]], i64 12), "nonnull"(i32* [[P]]) ]
-; BASIC-NEXT:    call void @func(i32* dereferenceable(12) [[P1]], i32* nonnull [[P]])
-; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "cold"(), "dereferenceable"(i32* [[P1]], i64 12) ]
-; BASIC-NEXT:    call void @func_cold(i32* dereferenceable(12) [[P1]]) #0
-; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "cold"(), "dereferenceable"(i32* [[P1]], i64 12) ]
-; BASIC-NEXT:    call void @func_cold(i32* dereferenceable(12) [[P1]])
-; BASIC-NEXT:    call void @func(i32* [[P1]], i32* [[P]])
-; BASIC-NEXT:    call void @func_strbool(i32* [[P1]])
-; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 8), "dereferenceable"(i32* [[P]], i64 16) ]
-; BASIC-NEXT:    call void @func(i32* dereferenceable(16) [[P]], i32* dereferenceable(8) [[P]])
-; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[P1]], i64 8) ]
-; BASIC-NEXT:    call void @func_many(i32* align 8 [[P1]])
-; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[P2:%.*]], i64 8), "nonnull"(i32* [[P3:%.*]]) ]
-; BASIC-NEXT:    call void @func_argattr(i32* [[P2]], i32* [[P3]])
-; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "nonnull"(i32* [[P]]), "nonnull"(i32* [[P1]]) ]
-; BASIC-NEXT:    call void @func(i32* nonnull [[P1]], i32* nonnull [[P]])
-; BASIC-NEXT:    ret void
-;
-; ALL-LABEL: @test(
-; ALL-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P:%.*]], i64 16), "nonnull"(i32* [[P]]) ]
-; ALL-NEXT:    call void @func(i32* nonnull dereferenceable(16) [[P]], i32* null)
-; ALL-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P1:%.*]], i64 12), "nonnull"(i32* [[P]]) ]
-; ALL-NEXT:    call void @func(i32* dereferenceable(12) [[P1]], i32* nonnull [[P]])
-; ALL-NEXT:    call void @llvm.assume(i1 true) [ "cold"(), "dereferenceable"(i32* [[P1]], i64 12) ]
-; ALL-NEXT:    call void @func_cold(i32* dereferenceable(12) [[P1]]) #0
-; ALL-NEXT:    call void @llvm.assume(i1 true) [ "cold"(), "dereferenceable"(i32* [[P1]], i64 12) ]
-; ALL-NEXT:    call void @func_cold(i32* dereferenceable(12) [[P1]])
-; ALL-NEXT:    call void @func(i32* [[P1]], i32* [[P]])
-; ALL-NEXT:    call void @llvm.assume(i1 true) [ "no-jump-tables"() ]
-; ALL-NEXT:    call void @func_strbool(i32* [[P1]])
-; ALL-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 8), "dereferenceable"(i32* [[P]], i64 16) ]
-; ALL-NEXT:    call void @func(i32* dereferenceable(16) [[P]], i32* dereferenceable(8) [[P]])
-; ALL-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[P1]], i64 8), "less-precise-fpmad"(), "no-jump-tables"(), "norecurse"(), "nounwind"(), "willreturn"() ]
-; ALL-NEXT:    call void @func_many(i32* align 8 [[P1]])
-; ALL-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[P2:%.*]], i64 8), "nonnull"(i32* [[P3:%.*]]), "nounwind"() ]
-; ALL-NEXT:    call void @func_argattr(i32* [[P2]], i32* [[P3]])
-; ALL-NEXT:    call void @llvm.assume(i1 true) [ "nonnull"(i32* [[P]]), "nonnull"(i32* [[P1]]) ]
-; ALL-NEXT:    call void @func(i32* nonnull [[P1]], i32* nonnull [[P]])
-; ALL-NEXT:    ret void
-;
-  call void @func(i32* nonnull dereferenceable(16) %P, i32* null)
-  call void @func(i32* dereferenceable(12) %P1, i32* nonnull %P)
-  call void @func_cold(i32* dereferenceable(12) %P1) cold
-  call void @func_cold(i32* dereferenceable(12) %P1)
-  call void @func(i32* %P1, i32* %P)
-  call void @func_strbool(i32* %P1)
-  call void @func(i32* dereferenceable(16) %P, i32* dereferenceable(8) %P)
-  call void @func_many(i32* align 8 %P1)
-  call void @func_argattr(i32* %P2, i32* %P3)
-  call void @func(i32* nonnull %P1, i32* nonnull %P)
-  ret void
-}
diff --git a/llvm/test/LTO/Resolution/X86/not-prevailing.ll b/llvm/test/LTO/Resolution/X86/not-prevailing.ll
index 7a7ca0ee784ec..174d3401a61da 100644
--- a/llvm/test/LTO/Resolution/X86/not-prevailing.ll
+++ b/llvm/test/LTO/Resolution/X86/not-prevailing.ll
@@ -6,9 +6,9 @@
 ; Check that 'foo' and 'bar' were not inlined.
 ; CHECK:      <zed>:
 ; CHECK-NEXT:  {{.*}}  pushq   %rbx
-; CHECK-NEXT:  {{.*}}  callq   0 <zed+0x6>
+; CHECK-NEXT:  {{.*}}  callq   0x6 <zed+0x6>
 ; CHECK-NEXT:  {{.*}}  movl    %eax, %ebx
-; CHECK-NEXT:  {{.*}}  callq   0 <zed+0xd>
+; CHECK-NEXT:  {{.*}}  callq   0xd <zed+0xd>
 ; CHECK-NEXT:  {{.*}}  movl    %ebx, %eax
 ; CHECK-NEXT:  {{.*}}  popq    %rbx
 ; CHECK-NEXT:  {{.*}}  retq
diff --git a/llvm/test/MC/AArch64/SVE/bfcvt-diagnostics.s b/llvm/test/MC/AArch64/SVE/bfcvt-diagnostics.s
new file mode 100644
index 0000000000000..013f15f8b6e00
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE/bfcvt-diagnostics.s
@@ -0,0 +1,27 @@
+// RUN: not llvm-mc -triple=aarch64 -mattr=+sve,bf16 2>&1 < %s| FileCheck %s
+
+bfcvt z0.s, p0/m, z1.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: bfcvt z0.s, p0/m, z1.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+bfcvt z0.h, p0/m, z1.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: bfcvt z0.h, p0/m, z1.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+bfcvt z0.h, p0/z, z1.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: bfcvt z0.h, p0/z, z1.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+bfcvt z0.h, p8/m, z1.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: bfcvt z0.h, p8/m, z1.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0.h, p0/m, z7.h
+bfcvt z0.h, p0/m, z1.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a predicated movprfx with a different element size
+// CHECK-NEXT: bfcvt z0.h, p0/m, z1.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
diff --git a/llvm/test/MC/AArch64/SVE/bfcvt.s b/llvm/test/MC/AArch64/SVE/bfcvt.s
new file mode 100644
index 0000000000000..1113ce88a8bed
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE/bfcvt.s
@@ -0,0 +1,29 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve,+bf16 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+
+bfcvt z0.H, p0/m, z1.S
+// CHECK-INST: bfcvt z0.h, p0/m, z1.s
+// CHECK-ENCODING: [0x20,0xa0,0x8a,0x65]
+// CHECK-ERROR: instruction requires: bf16 sve
+
+movprfx z0.S, p0/m, z2.S
+// CHECK-INST: movprfx z0.s, p0/m, z2.s
+// CHECK-ENCODING: [0x40,0x20,0x91,0x04]
+// CHECK-ERROR: instruction requires: sve
+
+bfcvt z0.H, p0/m, z1.S
+// CHECK-INST: bfcvt z0.h, p0/m, z1.s
+// CHECK-ENCODING: [0x20,0xa0,0x8a,0x65]
+// CHECK-ERROR: instruction requires: bf16 sve
+
+movprfx z0, z2
+// CHECK-INST: movprfx z0, z2
+// CHECK-ENCODING: [0x40,0xbc,0x20,0x04]
+// CHECK-ERROR: instruction requires: sve
+
+bfcvt z0.H, p0/m, z1.S
+// CHECK-INST: bfcvt z0.h, p0/m, z1.s
+// CHECK-ENCODING: [0x20,0xa0,0x8a,0x65]
+// CHECK-ERROR: instruction requires: bf16 sve
diff --git a/llvm/test/MC/AArch64/SVE/bfcvtnt-diagnostics.s b/llvm/test/MC/AArch64/SVE/bfcvtnt-diagnostics.s
new file mode 100644
index 0000000000000..4717204fb8620
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE/bfcvtnt-diagnostics.s
@@ -0,0 +1,27 @@
+// RUN: not llvm-mc -triple=aarch64 -mattr=+sve,bf16  2>&1 < %s| FileCheck %s
+
+bfcvtnt z0.s, p0/m, z1.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: bfcvtnt z0.s, p0/m, z1.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+bfcvtnt z0.h, p0/m, z1.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: bfcvtnt z0.h, p0/m, z1.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+bfcvtnt z0.h, p0/z, z1.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: bfcvtnt z0.h, p0/z, z1.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+bfcvtnt z0.h, p8/m, z1.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid restricted predicate register, expected p0..p7 (without element suffix)
+// CHECK-NEXT: bfcvtnt z0.h, p8/m, z1.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0.h, p0/m, z7.h
+bfcvtnt z0.h, p0/m, z1.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a predicated movprfx with a different element size
+// CHECK-NEXT: bfcvtnt z0.h, p0/m, z1.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
diff --git a/llvm/test/MC/AArch64/SVE/bfcvtnt.s b/llvm/test/MC/AArch64/SVE/bfcvtnt.s
new file mode 100644
index 0000000000000..6ab7165fbf846
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE/bfcvtnt.s
@@ -0,0 +1,29 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve,+bf16 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+
+bfcvtnt z0.H, p0/m, z1.S
+// CHECK-INST: bfcvtnt z0.h, p0/m, z1.s
+// CHECK-ENCODING: [0x20,0xa0,0x8a,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
+
+movprfx z0.S, p0/m, z2.S
+// CHECK-INST: movprfx z0.s, p0/m, z2.s
+// CHECK-ENCODING: [0x40,0x20,0x91,0x04]
+// CHECK-ERROR: instruction requires: sve
+
+bfcvtnt z0.H, p0/m, z1.S
+// CHECK-INST: bfcvtnt z0.h, p0/m, z1.s
+// CHECK-ENCODING: [0x20,0xa0,0x8a,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
+
+movprfx z0, z2
+// CHECK-INST: movprfx z0, z2
+// CHECK-ENCODING: [0x40,0xbc,0x20,0x04]
+// CHECK-ERROR: instruction requires: sve
+
+bfcvtnt z0.H, p0/m, z1.S
+// CHECK-INST: bfcvtnt z0.h, p0/m, z1.s
+// CHECK-ENCODING: [0x20,0xa0,0x8a,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
diff --git a/llvm/test/MC/AArch64/SVE/bfdot-diagnostics.s b/llvm/test/MC/AArch64/SVE/bfdot-diagnostics.s
new file mode 100644
index 0000000000000..90b441e9d75ab
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE/bfdot-diagnostics.s
@@ -0,0 +1,53 @@
+// RUN: not llvm-mc -triple=aarch64 -mattr=+sve,bf16  2>&1 < %s| FileCheck %s
+
+bfdot z0.s, z1.s, z2.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: bfdot z0.s, z1.s, z2.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+bfdot z0.h, z1.h, z2.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: bfdot z0.h, z1.h, z2.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+bfdot z0.s, z1.h, z2.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected z0.h..z7.h
+// CHECK-NEXT: bfdot z0.s, z1.h, z2.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0.s, p0/m, z7.s
+bfdot z0.s, z1.h, z2.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a predicated movprfx, suggest using unpredicated movprfx
+// CHECK-NEXT: bfdot z0.s, z1.h, z2.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+bfdot z0.s, z1.s, z2.h[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: bfdot z0.s, z1.s, z2.h[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+bfdot z0.h, z1.h, z2.h[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: bfdot z0.h, z1.h, z2.h[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+bfdot z0.s, z1.h, z2.s[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: Invalid restricted vector register, expected z0.h..z7.h
+// CHECK-NEXT: bfdot z0.s, z1.h, z2.s[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+bfdot z0.s, z1.h, z8.h[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: bfdot z0.s, z1.h, z8.h[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+bfdot z0.s, z1.h, z2.h[4]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: vector lane must be an integer in range [0, 3].
+// CHECK-NEXT: bfdot z0.s, z1.h, z2.h[4]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0.s, p0/m, z7.s
+bfdot z0.s, z1.h, z2.h[0]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a predicated movprfx, suggest using unpredicated movprfx
+// CHECK-NEXT: bfdot z0.s, z1.h, z2.h[0]
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
diff --git a/llvm/test/MC/AArch64/SVE/bfdot.s b/llvm/test/MC/AArch64/SVE/bfdot.s
new file mode 100644
index 0000000000000..d39978d797f2a
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE/bfdot.s
@@ -0,0 +1,52 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve,+bf16 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+
+bfdot z0.S, z1.H, z2.H
+// CHECK-INST: bfdot z0.s, z1.h, z2.h
+// CHECK-ENCODING: [0x20,0x80,0x62,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
+
+bfdot z0.S, z1.H, z2.H[0]
+// CHECK-INST: bfdot z0.s, z1.h, z2.h[0]
+// CHECK-ENCODING: [0x20,0x40,0x62,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
+
+bfdot z0.S, z1.H, z2.H[3]
+// CHECK-INST: bfdot z0.s, z1.h, z2.h[3]
+// CHECK-ENCODING: [0x20,0x40,0x7a,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
+
+// --------------------------------------------------------------------------//
+// Test compatibility with MOVPRFX instruction.
+
+movprfx z0, z7
+// CHECK-INST: movprfx z0, z7
+// CHECK-ENCODING: [0xe0,0xbc,0x20,0x04]
+// CHECK-ERROR: instruction requires: sve
+
+bfdot z0.S, z1.H, z2.H
+// CHECK-INST: bfdot z0.s, z1.h, z2.h
+// CHECK-ENCODING: [0x20,0x80,0x62,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
+
+movprfx z0, z7
+// CHECK-INST: movprfx z0, z7
+// CHECK-ENCODING: [0xe0,0xbc,0x20,0x04]
+// CHECK-ERROR: instruction requires: sve
+
+bfdot z0.S, z1.H, z2.H[0]
+// CHECK-INST: bfdot z0.s, z1.h, z2.h[0]
+// CHECK-ENCODING: [0x20,0x40,0x62,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
+
+movprfx z0, z7
+// CHECK-INST: movprfx z0, z7
+// CHECK-ENCODING: [0xe0,0xbc,0x20,0x04]
+// CHECK-ERROR: instruction requires: sve
+
+bfdot z0.S, z1.H, z2.H[3]
+// CHECK-INST: bfdot z0.s, z1.h, z2.h[3]
+// CHECK-ENCODING: [0x20,0x40,0x7a,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
diff --git a/llvm/test/MC/AArch64/SVE/bfmlal-diagnostics.s b/llvm/test/MC/AArch64/SVE/bfmlal-diagnostics.s
new file mode 100644
index 0000000000000..5b07c5c1ce6c9
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE/bfmlal-diagnostics.s
@@ -0,0 +1,42 @@
+// RUN: not llvm-mc -o - -triple=aarch64 -mattr=+sve,bf16  2>&1 %s | FileCheck %s
+
+bfmlalb z0.S, z1.H, z7.H[8]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: vector lane must be an integer in range [0, 7].
+// CHECK-NEXT: bfmlalb z0.S, z1.H, z7.H[8]
+// CHECK-NEXT: ^
+
+bfmlalb z0.S, z1.H, z8.H[7]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: bfmlalb z0.S, z1.H, z8.H[7]
+// CHECK-NEXT: ^
+
+bfmlalt z0.S, z1.H, z7.H[8]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: vector lane must be an integer in range [0, 7].
+// CHECK-NEXT: bfmlalt z0.S, z1.H, z7.H[8]
+// CHECK-NEXT: ^
+
+bfmlalt z0.S, z1.H, z8.H[7]
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK-NEXT: bfmlalt z0.S, z1.H, z8.H[7]
+// CHECK-NEXT: ^
+
+bfmlalt z0.S, z1.H, z7.2h[2]
+// CHECK: error: invalid vector kind qualifier
+// CHECK-NEXT: bfmlalt z0.S, z1.H, z7.2h[2]
+// CHECK-NEXT:                     ^
+
+bfmlalt z0.S, z1.H, z2.s[2]
+// CHECK: error: Invalid restricted vector register, expected z0.h..z7.h
+// CHECK-NEXT: bfmlalt z0.S, z1.H, z2.s[2]
+// CHECK-NEXT:                     ^
+
+bfmlalt z0.S, z1.s, z2.h[2]
+// CHECK: error: invalid element width
+// CHECK-NEXT: bfmlalt z0.S, z1.s, z2.h[2]
+// CHECK-NEXT:               ^
+
+movprfx z0.s, p0/m, z7.s
+bfmlalt z0.s, z1.h, z2.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a predicated movprfx
+// CHECK-NEXT: bfmlalt z0.s, z1.h, z2.h
+// CHECK-NEXT: ^
diff --git a/llvm/test/MC/AArch64/SVE/bfmlal.s b/llvm/test/MC/AArch64/SVE/bfmlal.s
new file mode 100644
index 0000000000000..d4f86d30df3a0
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE/bfmlal.s
@@ -0,0 +1,157 @@
+// RUN: llvm-mc -o - -triple=aarch64 -show-encoding -mattr=+sve,+bf16 %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -o - -triple=aarch64 -show-encoding %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+
+bfmlalb z0.S, z1.H, z2.H
+// CHECK-INST: bfmlalb z0.s, z1.h, z2.h
+// CHECK-ENCODING: [0x20,0x80,0xe2,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
+
+bfmlalt z0.S, z1.H, z2.H
+// CHECK-INST: bfmlalt z0.s, z1.h, z2.h
+// CHECK-ENCODING: [0x20,0x84,0xe2,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
+
+bfmlalb z0.S, z1.H, z2.H[0]
+// CHECK-INST: bfmlalb z0.s, z1.h, z2.h[0]
+// CHECK-ENCODING: [0x20,0x40,0xe2,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
+
+bfmlalt z0.S, z1.H, z2.H[0]
+// CHECK-INST: bfmlalt z0.s, z1.h, z2.h[0]
+// CHECK-ENCODING: [0x20,0x44,0xe2,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
+
+bfmlalb z0.S, z1.H, z2.H[7]
+// CHECK-INST: bfmlalb z0.s, z1.h, z2.h[7]
+// CHECK-ENCODING: [0x20,0x48,0xfa,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
+
+bfmlalt z0.S, z1.H, z2.H[7]
+// CHECK-INST: bfmlalt z0.s, z1.h, z2.h[7]
+// CHECK-ENCODING: [0x20,0x4c,0xfa,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
+
+bfmlalt z0.S, z1.H, z7.H[7]
+// CHECK-INST: bfmlalt z0.s, z1.h, z7.h[7]
+// CHECK-ENCODING: [0x20,0x4c,0xff,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
+
+bfmlalb z10.S, z21.H, z14.H
+// CHECK-INST: bfmlalb z10.s, z21.h, z14.h
+// CHECK-ENCODING: [0xaa,0x82,0xee,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
+
+bfmlalt z14.S, z10.H, z21.H
+// CHECK-INST: bfmlalt z14.s, z10.h, z21.h
+// CHECK-ENCODING: [0x4e,0x85,0xf5,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
+
+bfmlalb z21.s, z14.h, z3.h[2]
+// CHECK-INST: bfmlalb z21.s, z14.h, z3.h[2]
+// CHECK-ENCODING: [0xd5,0x41,0xeb,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
+
+// --------------------------------------------------------------------------//
+// Test compatibility with MOVPRFX instruction.
+
+movprfx z0, z7
+// CHECK-INST: movprfx z0, z7
+// CHECK-ENCODING: [0xe0,0xbc,0x20,0x04]
+// CHECK-ERROR: instruction requires: sve
+
+bfmlalb z0.S, z1.H, z2.H
+// CHECK-INST: bfmlalb z0.s, z1.h, z2.h
+// CHECK-ENCODING: [0x20,0x80,0xe2,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
+
+movprfx z0, z7
+// CHECK-INST: movprfx z0, z7
+// CHECK-ENCODING: [0xe0,0xbc,0x20,0x04]
+// CHECK-ERROR: instruction requires: sve
+
+bfmlalt z0.S, z1.H, z2.H
+// CHECK-INST: bfmlalt z0.s, z1.h, z2.h
+// CHECK-ENCODING: [0x20,0x84,0xe2,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
+
+movprfx z0, z7
+// CHECK-INST: movprfx z0, z7
+// CHECK-ENCODING: [0xe0,0xbc,0x20,0x04]
+// CHECK-ERROR: instruction requires: sve
+
+bfmlalb z0.S, z1.H, z2.H[0]
+// CHECK-INST: bfmlalb z0.s, z1.h, z2.h[0]
+// CHECK-ENCODING: [0x20,0x40,0xe2,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
+
+movprfx z0, z7
+// CHECK-INST: movprfx z0, z7
+// CHECK-ENCODING: [0xe0,0xbc,0x20,0x04]
+// CHECK-ERROR: instruction requires: sve
+
+bfmlalt z0.S, z1.H, z2.H[0]
+// CHECK-INST: bfmlalt z0.s, z1.h, z2.h[0]
+// CHECK-ENCODING: [0x20,0x44,0xe2,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
+
+movprfx z0, z7
+// CHECK-INST: movprfx z0, z7
+// CHECK-ENCODING: [0xe0,0xbc,0x20,0x04]
+// CHECK-ERROR: instruction requires: sve
+
+bfmlalb z0.S, z1.H, z2.H[7]
+// CHECK-INST: bfmlalb z0.s, z1.h, z2.h[7]
+// CHECK-ENCODING: [0x20,0x48,0xfa,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
+
+movprfx z0, z7
+// CHECK-INST: movprfx z0, z7
+// CHECK-ENCODING: [0xe0,0xbc,0x20,0x04]
+// CHECK-ERROR: instruction requires: sve
+
+bfmlalt z0.S, z1.H, z2.H[7]
+// CHECK-INST: bfmlalt z0.s, z1.h, z2.h[7]
+// CHECK-ENCODING: [0x20,0x4c,0xfa,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
+
+movprfx z0, z7
+// CHECK-INST: movprfx z0, z7
+// CHECK-ENCODING: [0xe0,0xbc,0x20,0x04]
+// CHECK-ERROR: instruction requires: sve
+
+bfmlalt z0.S, z1.H, z7.H[7]
+// CHECK-INST: bfmlalt z0.s, z1.h, z7.h[7]
+// CHECK-ENCODING: [0x20,0x4c,0xff,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
+
+movprfx z10, z7
+// CHECK-INST: movprfx z10, z7
+// CHECK-ENCODING: [0xea,0xbc,0x20,0x04]
+// CHECK-ERROR: instruction requires: sve
+
+bfmlalb z10.S, z21.H, z14.H
+// CHECK-INST: bfmlalb z10.s, z21.h, z14.h
+// CHECK-ENCODING: [0xaa,0x82,0xee,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
+
+movprfx z14, z7
+// CHECK-INST: movprfx z14, z7
+// CHECK-ENCODING: [0xee,0xbc,0x20,0x04]
+// CHECK-ERROR: instruction requires: sve
+
+bfmlalt z14.S, z10.H, z21.H
+// CHECK-INST: bfmlalt z14.s, z10.h, z21.h
+// CHECK-ENCODING: [0x4e,0x85,0xf5,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
+
+movprfx z21, z7
+// CHECK-INST: movprfx z21, z7
+// CHECK-ENCODING: [0xf5,0xbc,0x20,0x04]
+// CHECK-ERROR: instruction requires: sve
+
+bfmlalb z21.s, z14.h, z3.h[2]
+// CHECK-INST: bfmlalb z21.s, z14.h, z3.h[2]
+// CHECK-ENCODING: [0xd5,0x41,0xeb,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
diff --git a/llvm/test/MC/AArch64/SVE/bfmmla-diagnostics.s b/llvm/test/MC/AArch64/SVE/bfmmla-diagnostics.s
new file mode 100644
index 0000000000000..409c2c5741cab
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE/bfmmla-diagnostics.s
@@ -0,0 +1,22 @@
+// RUN: not llvm-mc -triple=aarch64 -mattr=+sve,bf16  2>&1 < %s| FileCheck %s
+
+bfmmla z0.s, z1.s, z2.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: bfmmla z0.s, z1.s, z2.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+bfmmla z0.h, z1.h, z2.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: bfmmla z0.h, z1.h, z2.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+bfmmla z0.s, z1.h, z2.s
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: bfmmla z0.s, z1.h, z2.s
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+movprfx z0.s, p0/m, z7.s
+bfmmla z0.s, z1.h, z2.h
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: instruction is unpredictable when following a predicated movprfx, suggest using unpredicated movprfx
+// CHECK-NEXT: bfmmla z0.s, z1.h, z2.h
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
diff --git a/llvm/test/MC/AArch64/SVE/bfmmla.s b/llvm/test/MC/AArch64/SVE/bfmmla.s
new file mode 100644
index 0000000000000..67a073ad6266b
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE/bfmmla.s
@@ -0,0 +1,22 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve,+bf16 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+
+bfmmla z0.S, z1.H, z2.H
+// CHECK-INST: bfmmla z0.s, z1.h, z2.h
+// CHECK-ENCODING: [0x20,0xe4,0x62,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
+
+// --------------------------------------------------------------------------//
+// Test compatibility with MOVPRFX instruction.
+
+movprfx z0, z7
+// CHECK-INST: movprfx z0, z7
+// CHECK-ENCODING: [0xe0,0xbc,0x20,0x04]
+// CHECK-ERROR: instruction requires: sve
+
+bfmmla z0.S, z1.H, z2.H
+// CHECK-INST: bfmmla z0.s, z1.h, z2.h
+// CHECK-ENCODING: [0x20,0xe4,0x62,0x64]
+// CHECK-ERROR: instruction requires: bf16 sve
diff --git a/llvm/test/MC/AArch64/armv8.6a-bf16.s b/llvm/test/MC/AArch64/armv8.6a-bf16.s
new file mode 100644
index 0000000000000..a4eaa36b51cde
--- /dev/null
+++ b/llvm/test/MC/AArch64/armv8.6a-bf16.s
@@ -0,0 +1,115 @@
+// RUN:     llvm-mc -triple aarch64 -show-encoding  -mattr=+bf16 < %s       | FileCheck %s
+// RUN:     llvm-mc -triple aarch64 -show-encoding  -mattr=+v8.6a < %s      | FileCheck %s
+// RUN: not llvm-mc -triple aarch64 -show-encoding  -mattr=-bf16  < %s 2>&1 | FileCheck %s --check-prefix=NOBF16
+// RUN: not llvm-mc -triple aarch64 -show-encoding  < %s 2>&1 | FileCheck %s --check-prefix=NOBF16
+
+
+bfdot v2.2s, v3.4h, v4.4h
+bfdot v2.4s, v3.8h, v4.8h
+// CHECK: bfdot v2.2s, v3.4h, v4.4h      // encoding: [0x62,0xfc,0x44,0x2e]
+// CHECK: bfdot v2.4s, v3.8h, v4.8h      // encoding: [0x62,0xfc,0x44,0x6e]
+// NOBF16: instruction requires: bf16
+// NOBF16-NEXT: bfdot v2.2s, v3.4h, v4.4h
+// NOBF16: instruction requires: bf16
+// NOBF16-NEXT: bfdot v2.4s, v3.8h, v4.8h
+
+bfdot  v2.2s, v3.4h, v4.2h[0]
+bfdot  v2.2s, v3.4h, v4.2h[1]
+bfdot  v2.2s, v3.4h, v4.2h[2]
+bfdot  v2.2s, v3.4h, v4.2h[3]
+// CHECK: bfdot   v2.2s, v3.4h, v4.2h[0]  // encoding: [0x62,0xf0,0x44,0x0f]
+// CHECK: bfdot   v2.2s, v3.4h, v4.2h[1]  // encoding: [0x62,0xf0,0x64,0x0f]
+// CHECK: bfdot   v2.2s, v3.4h, v4.2h[2]  // encoding: [0x62,0xf8,0x44,0x0f]
+// CHECK: bfdot   v2.2s, v3.4h, v4.2h[3]  // encoding: [0x62,0xf8,0x64,0x0f]
+// NOBF16: instruction requires: bf16
+// NOBF16-NEXT: bfdot   v2.2s, v3.4h, v4.2h[0]
+// NOBF16: instruction requires: bf16
+// NOBF16-NEXT: bfdot   v2.2s, v3.4h, v4.2h[1]
+// NOBF16: instruction requires: bf16
+// NOBF16-NEXT: bfdot   v2.2s, v3.4h, v4.2h[2]
+// NOBF16: instruction requires: bf16
+// NOBF16-NEXT: bfdot   v2.2s, v3.4h, v4.2h[3]
+
+
+bfdot v2.4s, v3.8h, v4.2h[0]
+bfdot v2.4s, v3.8h, v4.2h[1]
+bfdot v2.4s, v3.8h, v4.2h[2]
+bfdot v2.4s, v3.8h, v4.2h[3]
+// CHECK: bfdot  v2.4s, v3.8h, v4.2h[0]  // encoding: [0x62,0xf0,0x44,0x4f]
+// CHECK: bfdot  v2.4s, v3.8h, v4.2h[1]  // encoding: [0x62,0xf0,0x64,0x4f]
+// CHECK: bfdot  v2.4s, v3.8h, v4.2h[2]  // encoding: [0x62,0xf8,0x44,0x4f]
+// CHECK: bfdot  v2.4s, v3.8h, v4.2h[3]  // encoding: [0x62,0xf8,0x64,0x4f]
+// NOBF16: instruction requires: bf16
+// NOBF16-NEXT: bfdot v2.4s, v3.8h, v4.2h[0]
+// NOBF16: instruction requires: bf16
+// NOBF16-NEXT: bfdot v2.4s, v3.8h, v4.2h[1]
+// NOBF16: instruction requires: bf16
+// NOBF16-NEXT: bfdot v2.4s, v3.8h, v4.2h[2]
+// NOBF16: instruction requires: bf16
+// NOBF16-NEXT: bfdot v2.4s, v3.8h, v4.2h[3]
+
+
+bfmmla v2.4s, v3.8h, v4.8h
+bfmmla v3.4s, v4.8h, v5.8h
+// CHECK: bfmmla v2.4s, v3.8h, v4.8h   // encoding: [0x62,0xec,0x44,0x6e]
+// CHECK: bfmmla v3.4s, v4.8h, v5.8h   // encoding: [0x83,0xec,0x45,0x6e]
+// NOBF16: instruction requires: bf16
+// NOBF16-NEXT: bfmmla v2.4s, v3.8h, v4.8h
+// NOBF16: instruction requires: bf16
+// NOBF16-NEXT: bfmmla v3.4s, v4.8h, v5.8h
+
+bfcvtn  v5.4h, v5.4s
+bfcvtn2 v5.8h, v5.4s
+// CHECK: bfcvtn  v5.4h, v5.4s           // encoding: [0xa5,0x68,0xa1,0x0e]
+// CHECK: bfcvtn2 v5.8h, v5.4s           // encoding: [0xa5,0x68,0xa1,0x4e]
+// NOBF16: instruction requires: bf16
+// NOBF16-NEXT: bfcvtn  v5.4h, v5.4s
+// NOBF16: instruction requires: bf16
+// NOBF16-NEXT: bfcvtn2 v5.8h, v5.4s
+
+bfcvt  h5, s3
+// CHECK: bfcvt   h5, s3               // encoding: [0x65,0x40,0x63,0x1e]
+// NOBF16: instruction requires: bf16
+// NOBF16-NEXT: bfcvt  h5, s3
+
+bfmlalb V10.4S, V21.8h, V14.8H
+bfmlalt V21.4S, V14.8h, V10.8H
+// CHECK:       bfmlalb	v10.4s, v21.8h, v14.8h  // encoding: [0xaa,0xfe,0xce,0x2e]
+// CHECK-NEXT:  bfmlalt	v21.4s, v14.8h, v10.8h  // encoding: [0xd5,0xfd,0xca,0x6e]
+// NOBF16:      error: instruction requires: bf16
+// NOBF16-NEXT: bfmlalb V10.4S, V21.8h, V14.8H
+// NOBF16-NEXT: ^
+// NOBF16:      instruction requires: bf16
+// NOBF16-NEXT: bfmlalt V21.4S, V14.8h, V10.8H
+// NOBF16-NEXT: ^
+
+bfmlalb V14.4S, V21.8H, V10.H[1]
+bfmlalb V14.4S, V21.8H, V10.H[2]
+bfmlalb V14.4S, V21.8H, V10.H[7]
+bfmlalt V21.4S, V10.8H, V14.H[1]
+bfmlalt V21.4S, V10.8H, V14.H[2]
+bfmlalt V21.4S, V10.8H, V14.H[7]
+// CHECK:      bfmlalb v14.4s, v21.8h, v10.h[1] // encoding: [0xae,0xf2,0xda,0x0f]
+// CHECK-NEXT: bfmlalb v14.4s, v21.8h, v10.h[2] // encoding: [0xae,0xf2,0xea,0x0f]
+// CHECK-NEXT: bfmlalb v14.4s, v21.8h, v10.h[7] // encoding: [0xae,0xfa,0xfa,0x0f]
+// CHECK-NEXT: bfmlalt v21.4s, v10.8h, v14.h[1] // encoding: [0x55,0xf1,0xde,0x4f]
+// CHECK-NEXT: bfmlalt v21.4s, v10.8h, v14.h[2] // encoding: [0x55,0xf1,0xee,0x4f]
+// CHECK-NEXT: bfmlalt v21.4s, v10.8h, v14.h[7] // encoding: [0x55,0xf9,0xfe,0x4f]
+// NOBF16:      error: instruction requires: bf16
+// NOBF16-NEXT: bfmlalb V14.4S, V21.8H, V10.H[1]
+// NOBF16-NEXT: ^
+// NOBF16:      error: instruction requires: bf16
+// NOBF16-NEXT: bfmlalb V14.4S, V21.8H, V10.H[2]
+// NOBF16-NEXT: ^
+// NOBF16:      error: instruction requires: bf16
+// NOBF16-NEXT: bfmlalb V14.4S, V21.8H, V10.H[7]
+// NOBF16-NEXT: ^
+// NOBF16:      instruction requires: bf16
+// NOBF16-NEXT: bfmlalt V21.4S, V10.8H, V14.H[1]
+// NOBF16-NEXT: ^
+// NOBF16:      instruction requires: bf16
+// NOBF16-NEXT: bfmlalt V21.4S, V10.8H, V14.H[2]
+// NOBF16-NEXT: ^
+// NOBF16:      instruction requires: bf16
+// NOBF16-NEXT: bfmlalt V21.4S, V10.8H, V14.H[7]
+// NOBF16-NEXT: ^
diff --git a/llvm/test/MC/AArch64/reloc-directive-err.s b/llvm/test/MC/AArch64/reloc-directive-err.s
new file mode 100644
index 0000000000000..6eec2ae10c0a6
--- /dev/null
+++ b/llvm/test/MC/AArch64/reloc-directive-err.s
@@ -0,0 +1,6 @@
+# RUN: llvm-mc -triple=aarch64 %s 2>&1 | FileCheck --check-prefix=PRINT %s
+# RUN: not llvm-mc -filetype=obj -triple=aarch64 %s -o /dev/null 2>&1 | FileCheck %s
+
+# PRINT: .reloc 0, R_INVALID, 0
+# CHECK: {{.*}}.s:[[# @LINE+1]]:11: error: unknown relocation name
+.reloc 0, R_INVALID, 0
diff --git a/llvm/test/MC/AArch64/reloc-directive.s b/llvm/test/MC/AArch64/reloc-directive.s
index e10a90a5216be..ce0e00e193576 100644
--- a/llvm/test/MC/AArch64/reloc-directive.s
+++ b/llvm/test/MC/AArch64/reloc-directive.s
@@ -5,6 +5,9 @@
 # PRINT: .reloc 8, R_AARCH64_NONE, .data
 # PRINT: .reloc 4, R_AARCH64_NONE, foo+4
 # PRINT: .reloc 0, R_AARCH64_NONE, 8
+# PRINT: .reloc 0, R_AARCH64_ABS64, .data+2
+# PRINT: .reloc 0, R_AARCH64_TLSDESC, foo+3
+# PRINT: .reloc 0, R_AARCH64_IRELATIVE, 5
 .text
   ret
   nop
@@ -13,6 +16,10 @@
   .reloc 4, R_AARCH64_NONE, foo+4
   .reloc 0, R_AARCH64_NONE, 8
 
+  .reloc 0, R_AARCH64_ABS64, .data+2
+  .reloc 0, R_AARCH64_TLSDESC, foo+3
+  .reloc 0, R_AARCH64_IRELATIVE, 5
+
 .data
 .globl foo
 foo:
@@ -23,3 +30,6 @@ foo:
 # CHECK:      0x8 R_AARCH64_NONE .data 0x0
 # CHECK-NEXT: 0x4 R_AARCH64_NONE foo 0x4
 # CHECK-NEXT: 0x0 R_AARCH64_NONE - 0x8
+# CHECK-NEXT: 0x0 R_AARCH64_ABS64 .data 0x2
+# CHECK-NEXT: 0x0 R_AARCH64_TLSDESC foo 0x3
+# CHECK-NEXT: 0x0 R_AARCH64_IRELATIVE - 0x5
diff --git a/llvm/test/MC/ARM/bfloat16-a32-errors.s b/llvm/test/MC/ARM/bfloat16-a32-errors.s
new file mode 100644
index 0000000000000..4cfa424b7d306
--- /dev/null
+++ b/llvm/test/MC/ARM/bfloat16-a32-errors.s
@@ -0,0 +1,57 @@
+// RUN: not llvm-mc -triple arm -mattr=+bf16,-neon %s -o /dev/null 2>&1 | FileCheck %s --check-prefixes=NONEON,ALL
+// RUN: not llvm-mc -triple arm -mattr=-bf16 %s -o /dev/null 2>&1 | FileCheck %s --check-prefixes=NOBF16,ALL
+// RUN: not llvm-mc -triple arm %s -o /dev/null 2>&1 | FileCheck %s --check-prefixes=NONEON,ALL
+//
+vdot.bf16 d3, d4, d5
+vdot.bf16 q0, q1, q2
+vdot.bf16 d3, d4, d5[1]
+vdot.bf16 q0, q1, d5[1]
+vmmla.bf16 q0, q1, q2
+vcvt.bf16.f32 d1, q3
+vcvtbeq.bf16.f32 s1, s3
+vcvttne.bf16.f32 s1, s3
+// NOBF16: error: instruction requires: BFloat16 floating point extension
+// NOBF16-NEXT: vdot.bf16 d3, d4, d5
+// NOBF16-NEXT: ^
+// NOBF16-NEXT: error: instruction requires: BFloat16 floating point extension
+// NOBF16-NEXT: vdot.bf16 q0, q1, q2
+// NOBF16-NEXT: ^
+// NOBF16-NEXT: error: instruction requires: BFloat16 floating point extension
+// NOBF16-NEXT: vdot.bf16 d3, d4, d5[1]
+// NOBF16-NEXT: ^
+// NOBF16-NEXT: error: instruction requires: BFloat16 floating point extension
+// NOBF16-NEXT: vdot.bf16 q0, q1, d5[1]
+// NOBF16-NEXT: ^
+// NOBF16-NEXT: error: instruction requires: BFloat16 floating point extension
+// NOBF16-NEXT: vmmla.bf16 q0, q1, q2
+// NOBF16-NEXT: ^
+// NOBF16-NEXT: error: instruction requires: BFloat16 floating point extension
+// NOBF16-NEXT: vcvt.bf16.f32 d1, q3
+// NOBF16-NEXT: ^
+
+// NONEON: error: instruction requires: BFloat16 floating point extension NEON
+// NONEON-NEXT: vdot.bf16 d3, d4, d5
+// NONEON-NEXT: ^
+// NONEON-NEXT: error: instruction requires: BFloat16 floating point extension NEON
+// NONEON-NEXT: vdot.bf16 q0, q1, q2
+// NONEON-NEXT: ^
+// NONEON-NEXT: error: instruction requires: BFloat16 floating point extension NEON
+// NONEON-NEXT: vdot.bf16 d3, d4, d5[1]
+// NONEON-NEXT: ^
+// NONEON-NEXT: error: instruction requires: BFloat16 floating point extension NEON
+// NONEON-NEXT: vdot.bf16 q0, q1, d5[1]
+// NONEON-NEXT: ^
+// NONEON-NEXT: error: instruction requires: BFloat16 floating point extension NEON
+// NONEON-NEXT: vmmla.bf16 q0, q1, q2
+// NONEON-NEXT: ^
+// NONEON-NEXT: error: instruction requires: BFloat16 floating point extension NEON
+// NONEON-NEXT: vcvt.bf16.f32 d1, q3
+// NONEON-NEXT: ^
+
+
+// ALL-NEXT: error: instruction requires: BFloat16 floating point extension
+// ALL-NEXT: vcvtbeq.bf16.f32 s1, s3
+// ALL-NEXT: ^
+// ALL-NEXT: error: instruction requires: BFloat16 floating point extension
+// ALL-NEXT: vcvttne.bf16.f32 s1, s3
+// ALL-NEXT: ^
diff --git a/llvm/test/MC/ARM/bfloat16-a32-errors2.s b/llvm/test/MC/ARM/bfloat16-a32-errors2.s
new file mode 100644
index 0000000000000..85f1dd8cc5ce0
--- /dev/null
+++ b/llvm/test/MC/ARM/bfloat16-a32-errors2.s
@@ -0,0 +1,134 @@
+// RUN: not llvm-mc -o - -triple arm -mattr=+v8.6a -show-encoding %s 2>&1 | FileCheck %s
+vfmat.bf16 d0, d0, d0
+vfmat.bf16 d0, d0, q0
+vfmat.bf16 d0, q0, d0
+vfmat.bf16 q0, d0, d0
+vfmat.bf16 q0, q0, d0
+vfmat.bf16 q0, d0, q0
+vfmat.bf16 d0, q0, q0
+vfmat.bf16 q0, q0, q0[3]
+vfmat.bf16 q0, q0, q0[3]
+vfmat.bf16 q0, d0, d0[0]
+vfmat.bf16 d0, q0, d0[0]
+vfmat.bf16 q0, d0, d0[9]
+
+vfmab.bf16 d0, d0, d0
+vfmab.bf16 d0, d0, q0
+vfmab.bf16 d0, q0, d0
+vfmab.bf16 q0, d0, d0
+vfmab.bf16 q0, q0, d0
+vfmab.bf16 q0, d0, q0
+vfmab.bf16 d0, q0, q0
+vfmab.bf16 q0, q0, q0[3]
+vfmab.bf16 q0, q0, q0[3]
+vfmab.bf16 q0, d0, d0[0]
+vfmab.bf16 d0, q0, d0[0]
+vfmab.bf16 q0, d0, d0[9]
+
+//CHECK:error: invalid instruction
+//CHECK-NEXT:vfmat.bf16 d0, d0, d0
+//CHECK-NEXT:^
+//CHECK-NEXT:error: invalid instruction
+//CHECK-NEXT:vfmat.bf16 d0, d0, q0
+//CHECK-NEXT:^
+//CHECK-NEXT:error: invalid instruction
+//CHECK-NEXT:vfmat.bf16 d0, q0, d0
+//CHECK-NEXT:^
+//CHECK-NEXT:error: invalid instruction
+//CHECK-NEXT:vfmat.bf16 q0, d0, d0
+//CHECK-NEXT:^
+//CHECK-NEXT:error: invalid instruction, any one of the following would fix this:
+//CHECK-NEXT:vfmat.bf16 q0, q0, d0
+//CHECK-NEXT:^
+//CHECK-NEXT:note: too few operands for instruction
+//CHECK-NEXT:vfmat.bf16 q0, q0, d0
+//CHECK-NEXT:                      ^
+//CHECK-NEXT:note: operand must be a register in range [q0, q15]
+//CHECK-NEXT:vfmat.bf16 q0, q0, d0
+//CHECK-NEXT:                    ^
+//CHECK-NEXT:error: operand must be a register in range [q0, q15]
+//CHECK-NEXT:vfmat.bf16 q0, d0, q0
+//CHECK-NEXT:                ^
+//CHECK-NEXT:error: operand must be a register in range [q0, q15]
+//CHECK-NEXT:vfmat.bf16 d0, q0, q0
+//CHECK-NEXT:            ^
+//CHECK-NEXT:error: invalid instruction, any one of the following would fix this:
+//CHECK-NEXT:vfmat.bf16 q0, q0, q0[3]
+//CHECK-NEXT:^
+//CHECK-NEXT:note: operand must be a register in range [d0, d7]
+//CHECK-NEXT:vfmat.bf16 q0, q0, q0[3]
+//CHECK-NEXT:                    ^
+//CHECK-NEXT:note: too many operands for instruction
+//CHECK-NEXT:vfmat.bf16 q0, q0, q0[3]
+//CHECK-NEXT:                      ^
+//CHECK-NEXT:error: invalid instruction, any one of the following would fix this:
+//CHECK-NEXT:vfmat.bf16 q0, q0, q0[3]
+//CHECK-NEXT:^
+//CHECK-NEXT:note: operand must be a register in range [d0, d7]
+//CHECK-NEXT:vfmat.bf16 q0, q0, q0[3]
+//CHECK-NEXT:                    ^
+//CHECK-NEXT:note: too many operands for instruction
+//CHECK-NEXT:vfmat.bf16 q0, q0, q0[3]
+//CHECK-NEXT:                      ^
+//CHECK-NEXT:error: operand must be a register in range [q0, q15]
+//CHECK-NEXT:vfmat.bf16 q0, d0, d0[0]
+//CHECK-NEXT:                ^
+//CHECK-NEXT:error: operand must be a register in range [q0, q15]
+//CHECK-NEXT:vfmat.bf16 d0, q0, d0[0]
+//CHECK-NEXT:            ^
+//CHECK-NEXT:error: invalid instruction
+//CHECK-NEXT:vfmat.bf16 q0, d0, d0[9]
+//CHECK-NEXT:^
+//CHECK-NEXT:error: invalid instruction
+//CHECK-NEXT:vfmab.bf16 d0, d0, d0
+//CHECK-NEXT:^
+//CHECK-NEXT:error: invalid instruction
+//CHECK-NEXT:vfmab.bf16 d0, d0, q0
+//CHECK-NEXT:^
+//CHECK-NEXT:error: invalid instruction
+//CHECK-NEXT:vfmab.bf16 d0, q0, d0
+//CHECK-NEXT:^
+//CHECK-NEXT:error: invalid instruction
+//CHECK-NEXT:vfmab.bf16 q0, d0, d0
+//CHECK-NEXT:^
+//CHECK-NEXT:error: invalid instruction, any one of the following would fix this:
+//CHECK-NEXT:vfmab.bf16 q0, q0, d0
+//CHECK-NEXT:^
+//CHECK-NEXT:note: too few operands for instruction
+//CHECK-NEXT:vfmab.bf16 q0, q0, d0
+//CHECK-NEXT:                      ^
+//CHECK-NEXT:note: operand must be a register in range [q0, q15]
+//CHECK-NEXT:vfmab.bf16 q0, q0, d0
+//CHECK-NEXT:                    ^
+//CHECK-NEXT:error: operand must be a register in range [q0, q15]
+//CHECK-NEXT:vfmab.bf16 q0, d0, q0
+//CHECK-NEXT:                ^
+//CHECK-NEXT:error: operand must be a register in range [q0, q15]
+//CHECK-NEXT:vfmab.bf16 d0, q0, q0
+//CHECK-NEXT:            ^
+//CHECK-NEXT:error: invalid instruction, any one of the following would fix this:
+//CHECK-NEXT:vfmab.bf16 q0, q0, q0[3]
+//CHECK-NEXT:^
+//CHECK-NEXT:note: operand must be a register in range [d0, d7]
+//CHECK-NEXT:vfmab.bf16 q0, q0, q0[3]
+//CHECK-NEXT:                    ^
+//CHECK-NEXT:note: too many operands for instruction
+//CHECK-NEXT:vfmab.bf16 q0, q0, q0[3]
+//CHECK-NEXT:                      ^
+//CHECK-NEXT:error: invalid instruction, any one of the following would fix this:
+//CHECK-NEXT:vfmab.bf16 q0, q0, q0[3]
+//CHECK-NEXT:^
+//CHECK-NEXT:note: operand must be a register in range [d0, d7]
+//CHECK-NEXT:vfmab.bf16 q0, q0, q0[3]
+//CHECK-NEXT:                    ^
+//CHECK-NEXT:note: too many operands for instruction
+//CHECK-NEXT:vfmab.bf16 q0, q0, q0[3]
+//CHECK-NEXT:                      ^
+//CHECK-NEXT:error: operand must be a register in range [q0, q15]
+//CHECK-NEXT:vfmab.bf16 q0, d0, d0[0]
+//CHECK-NEXT:                ^
+//CHECK-NEXT:error: operand must be a register in range [q0, q15]
+//CHECK-NEXT:vfmab.bf16 d0, q0, d0[0]
+//CHECK-NEXT:            ^
+//CHECK-NEXT:error: invalid instruction
+//CHECK-NEXT:vfmab.bf16 q0, d0, d0[9]
diff --git a/llvm/test/MC/ARM/bfloat16-a32.s b/llvm/test/MC/ARM/bfloat16-a32.s
new file mode 100644
index 0000000000000..3ae54983e7456
--- /dev/null
+++ b/llvm/test/MC/ARM/bfloat16-a32.s
@@ -0,0 +1,55 @@
+// RUN: llvm-mc -triple arm -mattr=+bf16,+neon -show-encoding < %s | FileCheck %s  --check-prefix=CHECK
+// RUN: llvm-mc -triple arm -mattr=+v8.6a -show-encoding < %s | FileCheck %s  --check-prefix=CHECK
+
+vdot.bf16     d3, d4, d5
+// CHECK:     vdot.bf16  d3, d4, d5     @ encoding: [0x05,0x3d,0x04,0xfc]
+vdot.bf16    q0, q1, q2
+// CHECK-NEXT:     vdot.bf16 q0, q1, q2     @ encoding: [0x44,0x0d,0x02,0xfc]
+vdot.bf16     d3, d4, d5[1]
+// CHECK-NEXT:     vdot.bf16  d3, d4, d5[1] @ encoding: [0x25,0x3d,0x04,0xfe]
+vdot.bf16    q0, q1, d5[1]
+// CHECK-NEXT:     vdot.bf16  q0, q1, d5[1] @ encoding: [0x65,0x0d,0x02,0xfe]
+vmmla.bf16  q0, q1, q2
+// CHECK-NEXT:     vmmla.bf16 q0, q1, q2   @ encoding: [0x44,0x0c,0x02,0xfc]
+vcvt.bf16.f32 d1, q3
+// CHECK-NEXT:     vcvt.bf16.f32   d1, q3    @ encoding: [0x46,0x16,0xb6,0xf3]
+vcvtbeq.bf16.f32  s1, s3
+// CHECK-NEXT: vcvtbeq.bf16.f32 s1, s3       @ encoding: [0x61,0x09,0xf3,0x0e]
+vcvttne.bf16.f32 s1, s3
+// CHECK-NEXT: vcvttne.bf16.f32 s1, s3       @ encoding: [0xe1,0x09,0xf3,0x1e]
+vfmat.bf16 q0, q0, q0
+//CHECK-NEXT: vfmat.bf16      q0, q0, q0      @ encoding: [0x50,0x08,0x30,0xfc]
+vfmat.bf16 q0, q0, q15
+//CHECK-NEXT: vfmat.bf16      q0, q0, q15     @ encoding: [0x7e,0x08,0x30,0xfc]
+vfmat.bf16 q0, q15, q0
+//CHECK-NEXT: vfmat.bf16      q0, q15, q0     @ encoding: [0xd0,0x08,0x3e,0xfc]
+vfmat.bf16 q0, q15, q15
+//CHECK-NEXT: vfmat.bf16      q0, q15, q15     @ encoding: [0xfe,0x08,0x3e,0xfc]
+vfmat.bf16 q7, q0, q0
+//CHECK-NEXT: vfmat.bf16      q7, q0, q0      @ encoding: [0x50,0xe8,0x30,0xfc]
+vfmat.bf16 q8, q0, q0
+//CHECK-NEXT: vfmat.bf16      q8, q0, q0      @ encoding: [0x50,0x08,0x70,0xfc]
+vfmab.bf16 q0, q0, q0
+//CHECK-NEXT: vfmab.bf16      q0, q0, q0      @ encoding: [0x10,0x08,0x30,0xfc]
+vfmab.bf16 q0, q0, q15
+//CHECK-NEXT: vfmab.bf16      q0, q0, q15     @ encoding: [0x3e,0x08,0x30,0xfc]
+vfmab.bf16 q0, q15, q0
+//CHECK-NEXT: vfmab.bf16      q0, q15, q0     @ encoding: [0x90,0x08,0x3e,0xfc]
+vfmab.bf16 q0, q15, q15
+//CHECK-NEXT: vfmab.bf16      q0, q15, q15    @ encoding: [0xbe,0x08,0x3e,0xfc]
+vfmab.bf16 q7, q0, q0
+//CHECK-NEXT: vfmab.bf16      q7, q0, q0      @ encoding: [0x10,0xe8,0x30,0xfc]
+vfmab.bf16 q8, q0, q0
+//CHECK-NEXT: vfmab.bf16      q8, q0, q0      @ encoding: [0x10,0x08,0x70,0xfc]
+vfmat.bf16 q0, q0, d0[0]
+//CHECK-NEXT:  vfmat.bf16   q0, q0, d0[0]   @ encoding: [0x50,0x08,0x30,0xfe]
+vfmat.bf16 q0, q0, d0[3]
+//CHECK-NEXT:  vfmat.bf16   q0, q0, d0[3]   @ encoding: [0x78,0x08,0x30,0xfe]
+vfmat.bf16 q0, q0, d7[0]
+//CHECK-NEXT:  vfmat.bf16   q0, q0, d7[0]   @ encoding: [0x57,0x08,0x30,0xfe]
+vfmab.bf16 q0, q0, d0[0]
+//CHECK-NEXT:  vfmab.bf16   q0, q0, d0[0]   @ encoding: [0x10,0x08,0x30,0xfe]
+vfmab.bf16 q0, q0, d0[3]
+//CHECK-NEXT:  vfmab.bf16   q0, q0, d0[3]   @ encoding: [0x38,0x08,0x30,0xfe]
+vfmab.bf16 q0, q0, d7[0]
+//CHECK-NEXT:  vfmab.bf16   q0, q0, d7[0]   @ encoding: [0x17,0x08,0x30,0xfe]
diff --git a/llvm/test/MC/ARM/bfloat16-t32-errors.s b/llvm/test/MC/ARM/bfloat16-t32-errors.s
new file mode 100644
index 0000000000000..9684e76593eab
--- /dev/null
+++ b/llvm/test/MC/ARM/bfloat16-t32-errors.s
@@ -0,0 +1,32 @@
+// RUN: not llvm-mc -triple thumbv8 -mattr=-bf16 < %s 2>&1 | FileCheck %s
+
+vdot.bf16     d3, d4, d5
+// CHECK: instruction requires: BFloat16 floating point extension
+// CHECK-NEXT: vdot.bf16     d3, d4, d5
+
+vdot.bf16    q0, q1, q2
+// CHECK: instruction requires: BFloat16 floating point extension
+// CHECK-NEXT: vdot.bf16    q0, q1, q2
+
+vdot.bf16    d3, d4, d5[1]
+// CHECK: instruction requires: BFloat16 floating point extension
+// CHECK-NEXT: vdot.bf16    d3, d4, d5[1]
+
+vdot.bf16    q0, q1, d5[1]
+// CHECK: instruction requires: BFloat16 floating point extension
+// CHECK-NEXT: vdot.bf16    q0, q1, d5[1]
+
+vmmla.bf16  q0, q1, q2
+// CHECK: instruction requires: BFloat16 floating point extension
+// CHECK-NEXT: vmmla.bf16  q0, q1, q2
+
+vcvt.bf16.f32 d1, q3
+// CHECK: instruction requires: BFloat16 floating point extension
+// CHECK-NEXT: vcvt.bf16.f32 d1, q3
+
+vcvtbeq.bf16.f32  s1, s3
+// CHECK: note: instruction requires: BFloat16 floating point extension
+// CHECK-NEXT: vcvtbeq.bf16.f32  s1, s3
+vcvttne.bf16.f32 s1, s3
+// CHECK: note: instruction requires: BFloat16 floating point extension
+// CHECK-NEXT: vcvttne.bf16.f32 s1, s3
diff --git a/llvm/test/MC/ARM/bfloat16-t32.s b/llvm/test/MC/ARM/bfloat16-t32.s
new file mode 100644
index 0000000000000..263ff10b1db27
--- /dev/null
+++ b/llvm/test/MC/ARM/bfloat16-t32.s
@@ -0,0 +1,15 @@
+// RUN: llvm-mc -triple thumbv8 -mattr=+bf16,+neon -show-encoding < %s | FileCheck %s  --check-prefix=CHECK
+// RUN: llvm-mc -triple thumbv8 -mattr=+v8.6a -show-encoding < %s | FileCheck %s  --check-prefix=CHECK
+
+vcvt.bf16.f32 d1, q3
+// CHECK:     vcvt.bf16.f32   d1, q3    @ encoding: [0xb6,0xff,0x46,0x16]
+
+it eq
+vcvtbeq.bf16.f32  s1, s3
+// CHECK: it eq                         @ encoding: [0x08,0xbf]
+// CHECK-NEXT: vcvtbeq.bf16.f32 s1, s3  @ encoding:  [0xf3,0xee,0x61,0x09]
+
+it ne
+vcvttne.bf16.f32 s1, s3
+// CHECK: it ne                         @ encoding: [0x18,0xbf]
+// CHECK: vcvttne.bf16.f32 s1, s3       @ encoding: [0xf3,0xee,0xe1,0x09]
diff --git a/llvm/test/MC/ARM/reloc-directive-err.s b/llvm/test/MC/ARM/reloc-directive-err.s
new file mode 100644
index 0000000000000..c291fd62d2ba9
--- /dev/null
+++ b/llvm/test/MC/ARM/reloc-directive-err.s
@@ -0,0 +1,6 @@
+# RUN: llvm-mc -triple=armv7 %s 2>&1 | FileCheck --check-prefix=PRINT %s
+# RUN: not llvm-mc -filetype=obj -triple=armv7 %s -o /dev/null 2>&1 | FileCheck %s
+
+# PRINT: .reloc 0, R_INVALID, 0
+# CHECK: {{.*}}.s:[[# @LINE+1]]:11: error: unknown relocation name
+.reloc 0, R_INVALID, 0
diff --git a/llvm/test/MC/ARM/reloc-directive.s b/llvm/test/MC/ARM/reloc-directive.s
index 0855a7c2ad856..6a69c5016fda2 100644
--- a/llvm/test/MC/ARM/reloc-directive.s
+++ b/llvm/test/MC/ARM/reloc-directive.s
@@ -17,6 +17,10 @@
   .reloc 4, R_ARM_NONE, foo+4
   .reloc 0, R_ARM_NONE, 8
 
+  .reloc 0, R_ARM_ALU_PC_G0, .data+2
+  .reloc 0, R_ARM_LDR_PC_G0, foo+3
+  .reloc 0, R_ARM_THM_ALU_PREL_11_0, 5
+
 .data
 .globl foo
 foo:
@@ -27,11 +31,17 @@ foo:
 # PRINT: .reloc 8, R_ARM_NONE, .data
 # PRINT: .reloc 4, R_ARM_NONE, foo+4
 # PRINT: .reloc 0, R_ARM_NONE, 8
+# PRINT: .reloc 0, R_ARM_ALU_PC_G0, .data+2
+# PRINT: .reloc 0, R_ARM_LDR_PC_G0, foo+3
+# PRINT: .reloc 0, R_ARM_THM_ALU_PREL_11_0, 5
 
 # ARM relocations use the Elf32_Rel format. Addends are neither stored in the
 # relocation entries nor applied in the referenced locations.
 # CHECK:      0x8 R_ARM_NONE .data 0x0
 # CHECK-NEXT: 0x4 R_ARM_NONE foo 0x0
 # CHECK-NEXT: 0x0 R_ARM_NONE - 0x0
+# CHECK-NEXT: 0x0 R_ARM_ALU_PC_G0 .data 0x0
+# CHECK-NEXT: 0x0 R_ARM_LDR_PC_G0 foo 0x0
+# CHECK-NEXT: 0x0 R_ARM_THM_ALU_PREL_11_0 - 0x0
 
 # HEX: 0x00000000 00000000 00000000
diff --git a/llvm/test/MC/COFF/cv-inline-linetable-unlikely.s b/llvm/test/MC/COFF/cv-inline-linetable-unlikely.s
index 1d2e96e73eaf8..8258954535d40 100644
--- a/llvm/test/MC/COFF/cv-inline-linetable-unlikely.s
+++ b/llvm/test/MC/COFF/cv-inline-linetable-unlikely.s
@@ -26,13 +26,13 @@
 # ASM-NEXT:        4: c7 05 fc ff ff ff 00 00 00 00 movl    $0, -4(%rip)
 #  Begin inline loc (matches cv_loc below)
 # ASM-NEXT:        e: 83 3d ff ff ff ff 00          cmpl    $0, -1(%rip)
-# ASM-NEXT:       15: 75 0f                         jne     15 <g+0x26>
+# ASM-NEXT:       15: 75 0f                         jne     0x26 <g+0x26>
 #  End inline loc
 # ASM-NEXT:       17: c7 05 fc ff ff ff 00 00 00 00 movl    $0, -4(%rip)
 # ASM-NEXT:       21: 48 83 c4 28                   addq    $40, %rsp
 # ASM-NEXT:       25: c3                            retq
 #  Begin inline loc (matches cv_loc below)
-# ASM-NEXT:       26: e8 00 00 00 00                callq   0 <g+0x2b>
+# ASM-NEXT:       26: e8 00 00 00 00                callq   0x2b <g+0x2b>
 # ASM-NEXT:       2b: 0f 0b                         ud2
 #  End inline loc
 
diff --git a/llvm/test/MC/COFF/cv-loc-unreachable-2.s b/llvm/test/MC/COFF/cv-loc-unreachable-2.s
index b23ddb97282e7..2906622ddd7d7 100644
--- a/llvm/test/MC/COFF/cv-loc-unreachable-2.s
+++ b/llvm/test/MC/COFF/cv-loc-unreachable-2.s
@@ -8,9 +8,9 @@
 
 # ASM:      00000000 <_callit>:
 #   begin inline {
-# ASM-NEXT:        0: e8 00 00 00 00                calll   0 <_callit+0x5>
+# ASM-NEXT:        0: e8 00 00 00 00                calll   0x5 <_callit+0x5>
 # ASM-NEXT:        5: 85 c0                         testl   %eax, %eax
-# ASM-NEXT:        7: 75 01                         jne     1 <_callit+0xa>
+# ASM-NEXT:        7: 75 01                         jne     0xa <_callit+0xa>
 #   } end inline
 # ASM-NEXT:        9: c3                            retl
 #   begin inline {
diff --git a/llvm/test/MC/COFF/cv-loc-unreachable.s b/llvm/test/MC/COFF/cv-loc-unreachable.s
index 8c7f15d1079f5..333fb4523f5ba 100644
--- a/llvm/test/MC/COFF/cv-loc-unreachable.s
+++ b/llvm/test/MC/COFF/cv-loc-unreachable.s
@@ -19,9 +19,9 @@
 
 # ASM:      00000000 <_callit>:
 #   begin inline {
-# ASM-NEXT:        0: e8 00 00 00 00                calll   0 <_callit+0x5>
+# ASM-NEXT:        0: e8 00 00 00 00                calll   0x5 <_callit+0x5>
 # ASM-NEXT:        5: 85 c0                         testl   %eax, %eax
-# ASM-NEXT:        7: 75 01                         jne     1 <_callit+0xa>
+# ASM-NEXT:        7: 75 01                         jne     0xa <_callit+0xa>
 #   } end inline
 # ASM-NEXT:        9: c3                            retl
 #   begin inline {
diff --git a/llvm/test/MC/Disassembler/AArch64/armv8.6a-bf16.txt b/llvm/test/MC/Disassembler/AArch64/armv8.6a-bf16.txt
new file mode 100644
index 0000000000000..1d37bbcaf3865
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AArch64/armv8.6a-bf16.txt
@@ -0,0 +1,74 @@
+# RUN:     llvm-mc -triple=aarch64  -mattr=+bf16  -disassemble < %s      | FileCheck %s
+# RUN:     llvm-mc -triple=aarch64  -mattr=+v8.6a -disassemble < %s      | FileCheck %s
+# RUN: not llvm-mc -triple=aarch64  -mattr=-bf16  -disassemble < %s 2>&1 | FileCheck %s --check-prefix=NOBF16
+# RUN: not llvm-mc -triple=aarch64                -disassemble < %s 2>&1 | FileCheck %s --check-prefix=NOBF16
+
+
+[0x62,0xfc,0x44,0x2e]
+[0x62,0xfc,0x44,0x6e]
+# CHECK: bfdot   v2.2s, v3.4h, v4.4h
+# CHECK: bfdot   v2.4s, v3.8h, v4.8h
+# NOBF16: warning: invalid instruction encoding
+# NOBF16-NEXT: [0x62,0xfc,0x44,0x2e]
+# NOBF16: warning: invalid instruction encoding
+# NOBF16-NEXT: [0x62,0xfc,0x44,0x6e]
+
+[0x62,0xf0,0x44,0x4f]
+[0x62,0xf0,0x64,0x4f]
+[0x62,0xf8,0x44,0x4f]
+[0x62,0xf8,0x64,0x4f]
+# CHECK: bfdot  v2.4s, v3.8h, v4.2h[0]
+# CHECK: bfdot  v2.4s, v3.8h, v4.2h[1]
+# CHECK: bfdot  v2.4s, v3.8h, v4.2h[2]
+# CHECK: bfdot  v2.4s, v3.8h, v4.2h[3]
+# NOBF16: warning: invalid instruction encoding
+# NOBF-NEXT: [0x62,0xf0,0x44,0x4f]
+# NOBF16: warning: invalid instruction encoding
+# NOBF6-NEXT: [0x62,0xf0,0x64,0x4f]
+# NOBF16: warning: invalid instruction encoding
+# NOBF6-NEXT: [0x62,0xf8,0x44,0x4f]
+# NOBF16: warning: invalid instruction encoding
+# NOBF6-NEXT: [0x62,0xf8,0x64,0x4f]
+
+
+[0x62,0xf0,0x44,0x0f]
+[0x62,0xf0,0x64,0x0f]
+[0x62,0xf8,0x44,0x0f]
+[0x62,0xf8,0x64,0x0f]
+# CHECK: bfdot  v2.2s, v3.4h, v4.2h[0]
+# CHECK: bfdot  v2.2s, v3.4h, v4.2h[1]
+# CHECK: bfdot  v2.2s, v3.4h, v4.2h[2]
+# CHECK: bfdot  v2.2s, v3.4h, v4.2h[3]
+# NOBF16: warning: invalid instruction encoding
+# NOBF-NEXT: [0x62,0xf0,0x44,0x0f]
+# NOBF16: warning: invalid instruction encoding
+# NOBF6-NEXT: [0x62,0xf0,0x64,0x0f]
+# NOBF16: warning: invalid instruction encoding
+# NOBF6-NEXT: [0x62,0xf8,0x44,0x0f]
+# NOBF16: warning: invalid instruction encoding
+# NOBF6-NEXT: [0x62,0xf8,0x64,0x0f]
+
+
+[0x62,0xec,0x44,0x6e]
+[0x83,0xec,0x45,0x6e]
+# CHECK: bfmmla  v2.4s, v3.8h, v4.8h
+# CHECK: bfmmla  v3.4s, v4.8h, v5.8h
+# NOBF16: warning: invalid instruction encoding
+NOBF16-NEXT: [0x62,0xec,0x44,0x6e]
+# NOBF16: warning: invalid instruction encoding
+# NOBF16-NEXT: [0x83,0xec,0x45,0x6e]
+
+
+[0xa5,0x68,0xa1,0x0e]
+[0xa5,0x68,0xa1,0x4e]
+# CHECK: bfcvtn   v5.4h, v5.4s
+# CHECK: bfcvtn2  v5.8h, v5.4s
+# NOBF16: warning: invalid instruction encoding
+# NOBF16-NEXT: [0xa5,0x68,0xa1,0x0e]
+# NOBF16: warning: invalid instruction encoding
+# NOBF16-NEXT: [0xa5,0x68,0xa1,0x4e]
+
+[0x65, 0x40, 0x63, 0x1e]
+# CHECK: bfcvt  h5, s3
+# NOBF16: warning: invalid instruction encoding
+# NOBF16-NEXT: [0x65, 0x40, 0x63, 0x1e]
diff --git a/llvm/test/MC/Disassembler/ARM/bfloat16-a32_1.txt b/llvm/test/MC/Disassembler/ARM/bfloat16-a32_1.txt
new file mode 100644
index 0000000000000..be1a35f5ff009
--- /dev/null
+++ b/llvm/test/MC/Disassembler/ARM/bfloat16-a32_1.txt
@@ -0,0 +1,102 @@
+# RUN: llvm-mc -triple arm-none-linux-gnu -mattr=+bf16,+neon --disassemble < %s | FileCheck %s
+# RUN: llvm-mc -triple arm-none-linux-gnu -mattr=+v8.6a --disassemble < %s | FileCheck %s
+# RUN: llvm-mc -triple arm-none-linux-gnu -mattr=-bf16 --disassemble < %s 2>&1  | FileCheck %s --check-prefix=NOBF16
+# RUN: llvm-mc -triple arm-none-linux-gnu --disassemble < %s 2>&1  | FileCheck %s --check-prefix=NOBF16
+#
+# Tests BFloat16 instruction decodings.
+# Without BFloat16 enabled, some of these get disassembled to coprocessor instructions.
+[0x25,0x3d,0x04,0xfe]
+# CHECK: vdot.bf16 d3, d4, d5[1]
+# NOBF16: cdp2  p13, #0, c3, c4, c5, #1
+#
+[0x65,0x0d,0x02,0xfe]
+# CHECK-NEXT: vdot.bf16  q0, q1, d5[1]
+# NOBF16-NEXT: cdp2  p13, #0, c0, c2, c5, #3
+#
+[0x61,0x09,0xf3,0x0e]
+# CHECK-NEXT: vcvtbeq.bf16.f32 s1, s3
+# NOBF16-NEXT: cdpeq p9, #15, c0, c3, c1, #3
+#
+[0xe1,0x09,0xf3,0x1e]
+# CHECK-NEXT: vcvttne.bf16.f32 s1, s3
+# NOBF16-NEXT: cdpne p9, #15, c0, c3, c1, #7
+#
+[0x50,0x08,0x30,0xfc]
+# CHECK-NEXT: vfmat.bf16      q0, q0, q0
+# NOBF16-NEXT: ldc2    p8, c0, [r0], #-320
+#
+[0x7e,0x08,0x30,0xfc]
+# CHECK-NEXT: vfmat.bf16      q0, q0, q15
+# NOBF16-NEXT: ldc2    p8, c0, [r0], #-504
+#
+[0xd0,0x08,0x3e,0xfc]
+# CHECK-NEXT: vfmat.bf16      q0, q15, q0
+# NOBF16-NEXT: ldc2    p8, c0, [lr], #-832
+#
+[0xfe,0x08,0x3e,0xfc]
+# CHECK-NEXT: vfmat.bf16      q0, q15, q15
+# NOBF16-NEXT: ldc2    p8, c0, [lr], #-1016
+#
+[0xd0,0x08,0x30,0xfc]
+# CHECK-NEXT: vfmat.bf16      q0, q8, q0
+# NOBF16-NEXT: ldc2    p8, c0, [r0], #-832
+#
+[0x50,0xe8,0x30,0xfc]
+# CHECK-NEXT: vfmat.bf16      q7, q0, q0
+# NOBF16-NEXT: ldc2    p8, c14, [r0], #-320
+#
+[0x50,0x08,0x70,0xfc]
+# CHECK-NEXT: vfmat.bf16      q8, q0, q0
+# NOBF16-NEXT: ldc2l   p8, c0, [r0], #-320
+#
+[0x10,0x08,0x30,0xfc]
+# CHECK-NEXT: vfmab.bf16      q0, q0, q0
+# NOBF16-NEXT: ldc2    p8, c0, [r0], #-64
+#
+[0x3e,0x08,0x30,0xfc]
+# CHECK-NEXT: vfmab.bf16      q0, q0, q15
+# NOBF16-NEXT: ldc2    p8, c0, [r0], #-248
+#
+[0x90,0x08,0x3e,0xfc]
+# CHECK-NEXT: vfmab.bf16      q0, q15, q0
+# NOBF16-NEXT: ldc2    p8, c0, [lr], #-576
+#
+[0xbe,0x08,0x3e,0xfc]
+# CHECK-NEXT: vfmab.bf16      q0, q15, q15
+# NOBF16-NEXT: ldc2    p8, c0, [lr], #-760
+#
+[0x90,0x08,0x30,0xfc]
+# CHECK-NEXT: vfmab.bf16      q0, q8, q0
+# NOBF16-NEXT: ldc2    p8, c0, [r0], #-576
+#
+[0x10,0xe8,0x30,0xfc]
+# CHECK-NEXT: vfmab.bf16      q7, q0, q0
+# NOBF16-NEXT: ldc2    p8, c14, [r0], #-64
+#
+[0x10,0x08,0x70,0xfc]
+# CHECK-NEXT: vfmab.bf16      q8, q0, q0
+# NOBF16-NEXT: ldc2l   p8, c0, [r0], #-64
+#
+[0x50,0x08,0x30,0xfe]
+# CHECK-NEXT: vfmat.bf16 q0, q0, d0[0]
+# NOBF16-NEXT: mrc2  p8, #1, r0, c0, c0, #2
+#
+[0x78,0x08,0x30,0xfe]
+# CHECK-NEXT: vfmat.bf16 q0, q0, d0[3]
+# NOBF16-NEXT: mrc2  p8, #1, r0, c0, c8, #3
+[0x57,0x08,0x30,0xfe]
+#
+# CHECK-NEXT: vfmat.bf16 q0, q0, d7[0]
+# NOBF16-NEXT: mrc2  p8, #1, r0, c0, c7, #2
+[0x10,0x08,0x30,0xfe]
+#
+# CHECK-NEXT: vfmab.bf16 q0, q0, d0[0]
+# NOBF16-NEXT: mrc2  p8, #1, r0, c0, c0, #0
+[0x38,0x08,0x30,0xfe]
+#
+# CHECK-NEXT: vfmab.bf16 q0, q0, d0[3]
+# NOBF16-NEXT: mrc2  p8, #1, r0, c0, c8, #1
+#
+[0x17,0x08,0x30,0xfe]
+# CHECK-NEXT: vfmab.bf16 q0, q0, d7[0]
+# NOBF16-NEXT: mrc2  p8, #1, r0, c0, c7, #0
diff --git a/llvm/test/MC/Disassembler/ARM/bfloat16-a32_2.txt b/llvm/test/MC/Disassembler/ARM/bfloat16-a32_2.txt
new file mode 100644
index 0000000000000..3a02d69c18ebe
--- /dev/null
+++ b/llvm/test/MC/Disassembler/ARM/bfloat16-a32_2.txt
@@ -0,0 +1,20 @@
+# RUN: llvm-mc -triple arm-none-linux-gnu -mattr=+bf16,+neon --disassemble < %s | FileCheck %s
+# RUN: llvm-mc -triple arm-none-linux-gnu -mattr=+v8.6a --disassemble < %s | FileCheck %s
+# RUN: not llvm-mc -triple arm-none-linux-gnu -mattr=-bf16 --disassemble < %s 2>&1  | FileCheck %s --check-prefix=CHECK-NOBF16
+# RUN: not llvm-mc -triple arm-none-linux-gnu --disassemble < %s 2>&1  | FileCheck %s --check-prefix=CHECK-NOBF16
+
+[0x05,0x3d,0x04,0xfc]
+# CHECK: vdot.bf16  d3, d4, d5
+# CHECK-NOBF16: warning: invalid instruction encoding
+
+[0x44,0x0d,0x02,0xfc]
+# CHECK: vdot.bf16 q0, q1, q2
+# CHECK-NOBF16: warning: invalid instruction encoding
+
+[0x44,0x0c,0x02,0xfc]
+# CHECK: vmmla.bf16   q0, q1, q2
+# CHECK-NOBF16: warning: invalid instruction encoding
+
+[0x46,0x16,0xb6,0xf3]
+# CHECK: vcvt.bf16.f32   d1, q3
+# CHECK-ERROR:  warning: invalid instruction encoding
diff --git a/llvm/test/MC/Disassembler/ARM/bfloat16-t32.txt b/llvm/test/MC/Disassembler/ARM/bfloat16-t32.txt
new file mode 100644
index 0000000000000..e34249b33a396
--- /dev/null
+++ b/llvm/test/MC/Disassembler/ARM/bfloat16-t32.txt
@@ -0,0 +1,25 @@
+# RUN: llvm-mc -triple thumbv8-none-linux-gnu -mattr=+bf16,+neon --disassemble  < %s | FileCheck %s
+# RUN: llvm-mc -triple thumbv8-none-linux-gnu -mattr=+v8.6a --disassemble  < %s | FileCheck %s
+
+[0x04,0xfc,0x05,0x3d]
+[0x02,0xfc,0x44,0x0d]
+# CHECK:     vdot.bf16  d3, d4, d5
+# CHECK:     vdot.bf16 q0, q1, q2
+
+[0x04,0xfe,0x25,0x3d]
+# CHECK: vdot.bf16    d3, d4, d5[1]
+
+[0x02,0xfe,0x65,0x0d]
+# CHECK: vdot.bf16  q0, q1, d5[1]
+
+[0x02,0xfc,0x44,0x0c]
+# CHECK:     vmmla.bf16 q0, q1, q2
+
+[0xb6,0xff,0x46,0x16]
+# CHECK:     vcvt.bf16.f32   d1, q3
+
+[0xf3,0xee,0x61,0x09]
+# CHECK: vcvtb.bf16.f32 s1, s3
+
+[0xf3,0xee,0xe1,0x09]
+# CHECK: vcvtt.bf16.f32 s1, s3
diff --git a/llvm/test/MC/Disassembler/ARM/bfloat16-t32_errors.txt b/llvm/test/MC/Disassembler/ARM/bfloat16-t32_errors.txt
new file mode 100644
index 0000000000000..37acc32cb1d3f
--- /dev/null
+++ b/llvm/test/MC/Disassembler/ARM/bfloat16-t32_errors.txt
@@ -0,0 +1,40 @@
+# RUN: not llvm-mc -triple thumbv8-none-linux-gnu -mattr=-bf16 --disassemble < %s 2>&1  | FileCheck %s
+# RUN: not llvm-mc -triple thumbv8-none-linux-gnu --disassemble < %s 2>&1  | FileCheck %s
+
+[0x04,0xfc,0x05,0x3d]
+# CHECK: warning: invalid instruction encoding
+# CHECK-NEXT: [0x04,0xfc,0x05,0x3d]
+
+[0x02,0xfc,0x44,0x0d]
+# CHECK: warning: invalid instruction encoding
+# CHECK-NEXT: [0x02,0xfc,0x44,0x0d]
+
+
+[0x04,0xfe,0x25,0x3d]
+# CHECK: warning: invalid instruction encoding
+# CHECK-NEXT: [0x04,0xfe,0x25,0x3d]
+
+
+[0x02,0xfe,0x65,0x0d]
+# CHECK: warning: invalid instruction encoding
+# CHECK-NEXT: [0x02,0xfe,0x65,0x0d]
+
+
+[0x02,0xfc,0x44,0x0c]
+# CHECK: warning: invalid instruction encoding
+# CHECK-NEXT: [0x02,0xfc,0x44,0x0c]
+
+
+[0xb6,0xff,0x46,0x16]
+# CHECK: warning: invalid instruction encoding
+# CHECK-NEXT: [0xb6,0xff,0x46,0x16]
+
+
+[0xf3,0xee,0x61,0x09]
+# CHECK: warning: invalid instruction encoding
+# CHECK-NEXT: [0xf3,0xee,0x61,0x09]
+
+
+[0xf3,0xee,0xe1,0x09]
+# CHECK: warning: invalid instruction encoding
+# CHECK-NEXT: [0xf3,0xee,0xe1,0x09]
diff --git a/llvm/test/MC/Disassembler/PowerPC/ppc64-encoding-bookIII.txt b/llvm/test/MC/Disassembler/PowerPC/ppc64-encoding-bookIII.txt
index ae8efcf211fc3..1f5df65179f9e 100644
--- a/llvm/test/MC/Disassembler/PowerPC/ppc64-encoding-bookIII.txt
+++ b/llvm/test/MC/Disassembler/PowerPC/ppc64-encoding-bookIII.txt
@@ -57,34 +57,34 @@
 # CHECK: mtspr 275, 4                    
 0x7c 0x93 0x43 0xa6
 
-# CHECK: mtspr 280, 4                    
+# CHECK: mtasr 4                         
 0x7c 0x98 0x43 0xa6
 
-# CHECK: mfspr 4, 22                     
+# CHECK: mfdec 4                         
 0x7c 0x96 0x02 0xa6
 
-# CHECK: mtspr 22, 4                     
+# CHECK: mtdec 4                         
 0x7c 0x96 0x03 0xa6
 
 # CHECK: mfpvr 4
 0x7c 0x9f 0x42 0xa6
 
-# CHECK: mfspr 4, 25                     
+# CHECK: mfsdr1 4                        
 0x7c 0x99 0x02 0xa6
 
-# CHECK: mtspr 25, 4                     
+# CHECK: mtsdr1 4                        
 0x7c 0x99 0x03 0xa6
 
-# CHECK: mfspr 4, 26                     
+# CHECK: mfsrr0 4                        
 0x7c 0x9a 0x02 0xa6
 
-# CHECK: mtspr 26, 4                     
+# CHECK: mtsrr0 4                        
 0x7c 0x9a 0x03 0xa6
 
-# CHECK: mfspr 4, 27                     
+# CHECK: mfsrr1 4                        
 0x7c 0x9b 0x02 0xa6
 
-# CHECK: mtspr 27, 4                     
+# CHECK: mtsrr1 4                        
 0x7c 0x9b 0x03 0xa6
 
 # CHECK: slbie 4                         
diff --git a/llvm/test/MC/Disassembler/X86/x86-16.txt b/llvm/test/MC/Disassembler/X86/x86-16.txt
index 5820b5e273a81..95948e33ac683 100644
--- a/llvm/test/MC/Disassembler/X86/x86-16.txt
+++ b/llvm/test/MC/Disassembler/X86/x86-16.txt
@@ -836,3 +836,6 @@
 
 # CHECK: enqcmds (%edi), %edi
 0x67,0xf3,0x0f,0x38,0xf8,0x3f
+
+# CHECK: serialize
+0x0f 0x01 0xe8
diff --git a/llvm/test/MC/Disassembler/X86/x86-32.txt b/llvm/test/MC/Disassembler/X86/x86-32.txt
index 86157d0e8f616..9e04cc27d879e 100644
--- a/llvm/test/MC/Disassembler/X86/x86-32.txt
+++ b/llvm/test/MC/Disassembler/X86/x86-32.txt
@@ -943,3 +943,6 @@
 
 # CHECK: enqcmds 8128(%bx,%di), %ax
 0x67,0xf3,0x0f,0x38,0xf8,0x81,0xc0,0x1f
+
+# CHECK: serialize
+0x0f 0x01 0xe8
diff --git a/llvm/test/MC/Disassembler/X86/x86-64.txt b/llvm/test/MC/Disassembler/X86/x86-64.txt
index 9b4064c3f73ab..8ef1363e4c483 100644
--- a/llvm/test/MC/Disassembler/X86/x86-64.txt
+++ b/llvm/test/MC/Disassembler/X86/x86-64.txt
@@ -691,3 +691,6 @@
 
 # CHECK: enqcmds 485498096, %rax
 0xf3,0x0f,0x38,0xf8,0x04,0x25,0xf0,0x1c,0xf0,0x1c
+
+# CHECK: serialize
+0x0f 0x01 0xe8
diff --git a/llvm/test/MC/Mips/mips32r2/valid.s b/llvm/test/MC/Mips/mips32r2/valid.s
index 5b3b8ca1efd9f..d292ecc064c18 100644
--- a/llvm/test/MC/Mips/mips32r2/valid.s
+++ b/llvm/test/MC/Mips/mips32r2/valid.s
@@ -209,8 +209,7 @@ a:
         or        $12,$s0,$sp
         or        $2, 4                # CHECK: ori $2, $2, 4           # encoding: [0x34,0x42,0x00,0x04]
         pause                          # CHECK: pause # encoding:  [0x00,0x00,0x01,0x40]
-                                       # CHECK-NEXT:  # <MCInst #{{[0-9]+}} PAUSE
-                                       # CHECK-NOT    # <MCInst #{{[0-9}+}} PAUSE_MM
+                                       # CHECK-NEXT:  # <MCInst #{{[0-9]+}} PAUSE>
         pref      1, 8($5)             # CHECK: pref 1, 8($5)           # encoding: [0xcc,0xa1,0x00,0x08]
         # FIXME: Use the code generator in order to print the .set directives
         #        instead of the instruction printer.
diff --git a/llvm/test/MC/Mips/mips32r3/valid.s b/llvm/test/MC/Mips/mips32r3/valid.s
index 103073b02ddf5..137aee6811744 100644
--- a/llvm/test/MC/Mips/mips32r3/valid.s
+++ b/llvm/test/MC/Mips/mips32r3/valid.s
@@ -209,8 +209,7 @@ a:
         or        $12,$s0,$sp
         or        $2, 4                # CHECK: ori $2, $2, 4           # encoding: [0x34,0x42,0x00,0x04]
         pause                          # CHECK: pause # encoding:  [0x00,0x00,0x01,0x40]
-                                       # CHECK-NEXT:  # <MCInst #{{[0-9]+}} PAUSE
-                                       # CHECK-NOT    # <MCInst #{{[0-9}+}} PAUSE_MM
+                                       # CHECK-NEXT:  # <MCInst #{{[0-9]+}} PAUSE>
         pref      1, 8($5)             # CHECK: pref 1, 8($5)           # encoding: [0xcc,0xa1,0x00,0x08]
         # FIXME: Use the code generator in order to print the .set directives
         #        instead of the instruction printer.
diff --git a/llvm/test/MC/Mips/mips32r5/valid.s b/llvm/test/MC/Mips/mips32r5/valid.s
index 935adfb04277d..bf75a2c000bff 100644
--- a/llvm/test/MC/Mips/mips32r5/valid.s
+++ b/llvm/test/MC/Mips/mips32r5/valid.s
@@ -210,8 +210,7 @@ a:
         or        $12,$s0,$sp
         or        $2, 4                # CHECK: ori $2, $2, 4           # encoding: [0x34,0x42,0x00,0x04]
         pause                          # CHECK: pause # encoding:  [0x00,0x00,0x01,0x40]
-                                       # CHECK-NEXT:  # <MCInst #{{[0-9]+}} PAUSE
-                                       # CHECK-NOT    # <MCInst #{{[0-9}+}} PAUSE_MM
+                                       # CHECK-NEXT:  # <MCInst #{{[0-9]+}} PAUSE>
         pref      1, 8($5)             # CHECK: pref 1, 8($5)           # encoding: [0xcc,0xa1,0x00,0x08]
         # FIXME: Use the code generator in order to print the .set directives
         #        instead of the instruction printer.
diff --git a/llvm/test/MC/Mips/mips32r6/valid.s b/llvm/test/MC/Mips/mips32r6/valid.s
index 87a8467639e30..0f098a176a67c 100644
--- a/llvm/test/MC/Mips/mips32r6/valid.s
+++ b/llvm/test/MC/Mips/mips32r6/valid.s
@@ -149,8 +149,7 @@ a:
         not       $3, $4         # CHECK: not $3, $4             # encoding: [0x00,0x80,0x18,0x27]
         not       $3             # CHECK: not $3, $3             # encoding: [0x00,0x60,0x18,0x27]
         pause                    # CHECK: pause                  # encoding:  [0x00,0x00,0x01,0x40]
-                                 # CHECK-NEXT:                   # <MCInst #{{[0-9]+}} PAUSE
-                                 # CHECK-NOT                     # <MCInst #{{[0-9}+}} PAUSE_MM
+                                 # CHECK-NEXT:                   # <MCInst #{{[0-9]+}} PAUSE>
         pref    1, 8($5)         # CHECK: pref 1, 8($5)          # encoding: [0x7c,0xa1,0x04,0x35]
                                  # CHECK-NEXT:                   # <MCInst #{{.*}} PREF_R6
         # FIXME: Use the code generator in order to print the .set directives
diff --git a/llvm/test/MC/Mips/mips64r2/valid.s b/llvm/test/MC/Mips/mips64r2/valid.s
index c7696ba455e4f..e1ce6aca281b6 100644
--- a/llvm/test/MC/Mips/mips64r2/valid.s
+++ b/llvm/test/MC/Mips/mips64r2/valid.s
@@ -287,8 +287,7 @@ a:
         or        $12,$s0,$sp
         or        $2, 4                # CHECK: ori $2, $2, 4           # encoding: [0x34,0x42,0x00,0x04]
         pause                          # CHECK: pause # encoding:  [0x00,0x00,0x01,0x40]
-                                       # CHECK-NEXT:  # <MCInst #{{[0-9]+}} PAUSE
-                                       # CHECK-NOT    # <MCInst #{{[0-9}+}} PAUSE_MM
+                                       # CHECK-NEXT:  # <MCInst #{{[0-9]+}} PAUSE>
         pll.ps  $f26,$f10,$f30         # CHECK: pll.ps $f26, $f10, $f30  # encoding: [0x46,0xde,0x56,0xac]
                                        # CHECK:                          # <MCInst #{{[0-9]+}} PLL_PS64
         plu.ps  $f2,$f26,$f30          # CHECK: plu.ps $f2, $f26, $f30    # encoding: [0x46,0xde,0xd0,0xad]
diff --git a/llvm/test/MC/Mips/mips64r3/valid.s b/llvm/test/MC/Mips/mips64r3/valid.s
index 76c6f13bc21a5..fc4c58d7f84a1 100644
--- a/llvm/test/MC/Mips/mips64r3/valid.s
+++ b/llvm/test/MC/Mips/mips64r3/valid.s
@@ -281,8 +281,7 @@ a:
         or        $12,$s0,$sp
         or        $2, 4                # CHECK: ori $2, $2, 4           # encoding: [0x34,0x42,0x00,0x04]
         pause                          # CHECK: pause # encoding:  [0x00,0x00,0x01,0x40]
-                                       # CHECK-NEXT:  # <MCInst #{{[0-9]+}} PAUSE
-                                       # CHECK-NOT    # <MCInst #{{[0-9}+}} PAUSE_MM
+                                       # CHECK-NEXT:  # <MCInst #{{[0-9]+}} PAUSE>
         pll.ps  $f26,$f10,$f30         # CHECK: pll.ps $f26, $f10, $f30  # encoding: [0x46,0xde,0x56,0xac]
                                        # CHECK:                          # <MCInst #{{[0-9]+}} PLL_PS64
         plu.ps  $f2,$f26,$f30          # CHECK: plu.ps $f2, $f26, $f30    # encoding: [0x46,0xde,0xd0,0xad]
diff --git a/llvm/test/MC/Mips/mips64r5/valid.s b/llvm/test/MC/Mips/mips64r5/valid.s
index 540a832146f47..41515b13703d9 100644
--- a/llvm/test/MC/Mips/mips64r5/valid.s
+++ b/llvm/test/MC/Mips/mips64r5/valid.s
@@ -283,8 +283,7 @@ a:
         or        $12,$s0,$sp
         or        $2, 4                # CHECK: ori $2, $2, 4           # encoding: [0x34,0x42,0x00,0x04]
         pause                          # CHECK: pause # encoding:  [0x00,0x00,0x01,0x40]
-                                       # CHECK-NEXT:  # <MCInst #{{[0-9]+}} PAUSE
-                                       # CHECK-NOT    # <MCInst #{{[0-9}+}} PAUSE_MM
+                                       # CHECK-NEXT:  # <MCInst #{{[0-9]+}} PAUSE>
         pll.ps  $f26,$f10,$f30         # CHECK: pll.ps $f26, $f10, $f30  # encoding: [0x46,0xde,0x56,0xac]
                                        # CHECK:                          # <MCInst #{{[0-9]+}} PLL_PS64
         plu.ps  $f2,$f26,$f30          # CHECK: plu.ps $f2, $f26, $f30    # encoding: [0x46,0xde,0xd0,0xad]
diff --git a/llvm/test/MC/Mips/mips64r6/valid.s b/llvm/test/MC/Mips/mips64r6/valid.s
index c810a40e25289..c50bd9e31c232 100644
--- a/llvm/test/MC/Mips/mips64r6/valid.s
+++ b/llvm/test/MC/Mips/mips64r6/valid.s
@@ -213,8 +213,7 @@ a:
         not       $3             # CHECK: not $3, $3             # encoding: [0x00,0x60,0x18,0x27]
         or      $2, 4            # CHECK: ori $2, $2, 4          # encoding: [0x34,0x42,0x00,0x04]
         pause                    # CHECK: pause                  # encoding:  [0x00,0x00,0x01,0x40]
-                                 # CHECK-NEXT:                   # <MCInst #{{[0-9]+}} PAUSE
-                                 # CHECK-NOT                     # <MCInst #{{[0-9}+}} PAUSE_MM
+                                 # CHECK-NEXT:                   # <MCInst #{{[0-9]+}} PAUSE>
         pref    1, 8($5)         # CHECK: pref 1, 8($5)          # encoding: [0x7c,0xa1,0x04,0x35]
         # FIXME: Use the code generator in order to print the .set directives
         #        instead of the instruction printer.
diff --git a/llvm/test/MC/PowerPC/ppc32-reloc-directive.s b/llvm/test/MC/PowerPC/ppc32-reloc-directive.s
index 1563dd2c065db..6beeabd4db370 100644
--- a/llvm/test/MC/PowerPC/ppc32-reloc-directive.s
+++ b/llvm/test/MC/PowerPC/ppc32-reloc-directive.s
@@ -5,10 +5,16 @@
 # PRINT: .reloc 8, R_PPC_NONE, .data
 # PRINT: .reloc 4, R_PPC_NONE, foo+4
 # PRINT: .reloc 0, R_PPC_NONE, 8
+# PRINT: .reloc 0, R_PPC_ADDR32, .data+2
+# PRINT: .reloc 0, R_PPC_REL16_HI, foo+3
+# PRINT: .reloc 0, R_PPC_REL16_HA, 5
 
 # CHECK:      0x8 R_PPC_NONE .data 0x0
 # CHECK-NEXT: 0x4 R_PPC_NONE foo 0x4
 # CHECK-NEXT: 0x0 R_PPC_NONE - 0x8
+# CHECK-NEXT: 0x0 R_PPC_ADDR32 .data 0x2
+# CHECK-NEXT: 0x0 R_PPC_REL16_HI foo 0x3
+# CHECK-NEXT: 0x0 R_PPC_REL16_HA - 0x5
 
 .text
   blr
@@ -17,6 +23,9 @@
   .reloc 8, R_PPC_NONE, .data
   .reloc 4, R_PPC_NONE, foo+4
   .reloc 0, R_PPC_NONE, 8
+  .reloc 0, R_PPC_ADDR32, .data+2
+  .reloc 0, R_PPC_REL16_HI, foo+3
+  .reloc 0, R_PPC_REL16_HA, 5
 
 .data
 .globl foo
diff --git a/llvm/test/MC/PowerPC/ppc64-encoding-bookIII.s b/llvm/test/MC/PowerPC/ppc64-encoding-bookIII.s
index 9592f425dce31..503001aff0854 100644
--- a/llvm/test/MC/PowerPC/ppc64-encoding-bookIII.s
+++ b/llvm/test/MC/PowerPC/ppc64-encoding-bookIII.s
@@ -90,44 +90,44 @@
 # CHECK-LE: mtspr 263, 4                    # encoding: [0xa6,0x43,0x87,0x7c]
             mtsprg7 %r4
 
-# CHECK-BE: mtspr 280, 4                    # encoding: [0x7c,0x98,0x43,0xa6]
-# CHECK-LE: mtspr 280, 4                    # encoding: [0xa6,0x43,0x98,0x7c]
+# CHECK-BE: mtasr 4                         # encoding: [0x7c,0x98,0x43,0xa6]
+# CHECK-LE: mtasr 4                         # encoding: [0xa6,0x43,0x98,0x7c]
             mtasr %r4
 
-# CHECK-BE: mfspr 4, 22                     # encoding: [0x7c,0x96,0x02,0xa6]
-# CHECK-LE: mfspr 4, 22                     # encoding: [0xa6,0x02,0x96,0x7c]
+# CHECK-BE: mfdec 4                         # encoding: [0x7c,0x96,0x02,0xa6]
+# CHECK-LE: mfdec 4                         # encoding: [0xa6,0x02,0x96,0x7c]
             mfdec %r4
 
-# CHECK-BE: mtspr 22, 4                     # encoding: [0x7c,0x96,0x03,0xa6]
-# CHECK-LE: mtspr 22, 4                     # encoding: [0xa6,0x03,0x96,0x7c]
+# CHECK-BE: mtdec 4                         # encoding: [0x7c,0x96,0x03,0xa6]
+# CHECK-LE: mtdec 4                         # encoding: [0xa6,0x03,0x96,0x7c]
             mtdec %r4
 
 # CHECK-BE: mfpvr 4                         # encoding: [0x7c,0x9f,0x42,0xa6]
 # CHECK-LE: mfpvr 4                         # encoding: [0xa6,0x42,0x9f,0x7c]
             mfpvr %r4
 
-# CHECK-BE: mfspr 4, 25                     # encoding: [0x7c,0x99,0x02,0xa6]
-# CHECK-LE: mfspr 4, 25                     # encoding: [0xa6,0x02,0x99,0x7c]
+# CHECK-BE: mfsdr1 4                        # encoding: [0x7c,0x99,0x02,0xa6]
+# CHECK-LE: mfsdr1 4                        # encoding: [0xa6,0x02,0x99,0x7c]
             mfsdr1 %r4
 
-# CHECK-BE: mtspr 25, 4                     # encoding: [0x7c,0x99,0x03,0xa6]
-# CHECK-LE: mtspr 25, 4                     # encoding: [0xa6,0x03,0x99,0x7c]
+# CHECK-BE: mtsdr1 4                        # encoding: [0x7c,0x99,0x03,0xa6]
+# CHECK-LE: mtsdr1 4                        # encoding: [0xa6,0x03,0x99,0x7c]
             mtsdr1 %r4
 
-# CHECK-BE: mfspr 4, 26                     # encoding: [0x7c,0x9a,0x02,0xa6]
-# CHECK-LE: mfspr 4, 26                     # encoding: [0xa6,0x02,0x9a,0x7c]
+# CHECK-BE: mfsrr0 4                        # encoding: [0x7c,0x9a,0x02,0xa6]
+# CHECK-LE: mfsrr0 4                        # encoding: [0xa6,0x02,0x9a,0x7c]
             mfsrr0 %r4
 
-# CHECK-BE: mtspr 26, 4                     # encoding: [0x7c,0x9a,0x03,0xa6]
-# CHECK-LE: mtspr 26, 4                     # encoding: [0xa6,0x03,0x9a,0x7c]
+# CHECK-BE: mtsrr0 4                        # encoding: [0x7c,0x9a,0x03,0xa6]
+# CHECK-LE: mtsrr0 4                        # encoding: [0xa6,0x03,0x9a,0x7c]
             mtsrr0 %r4
 
-# CHECK-BE: mfspr 4, 27                     # encoding: [0x7c,0x9b,0x02,0xa6]
-# CHECK-LE: mfspr 4, 27                     # encoding: [0xa6,0x02,0x9b,0x7c]
+# CHECK-BE: mfsrr1 4                        # encoding: [0x7c,0x9b,0x02,0xa6]
+# CHECK-LE: mfsrr1 4                        # encoding: [0xa6,0x02,0x9b,0x7c]
             mfsrr1 %r4
 
-# CHECK-BE: mtspr 27, 4                     # encoding: [0x7c,0x9b,0x03,0xa6]
-# CHECK-LE: mtspr 27, 4                     # encoding: [0xa6,0x03,0x9b,0x7c]
+# CHECK-BE: mtsrr1 4                        # encoding: [0x7c,0x9b,0x03,0xa6]
+# CHECK-LE: mtsrr1 4                        # encoding: [0xa6,0x03,0x9b,0x7c]
             mtsrr1 %r4
 
 # CHECK-BE: slbie 4                         # encoding: [0x7c,0x00,0x23,0x64]
diff --git a/llvm/test/MC/PowerPC/ppc64-encoding-ext.s b/llvm/test/MC/PowerPC/ppc64-encoding-ext.s
index 0b7bc927c63f7..892c1eceef164 100644
--- a/llvm/test/MC/PowerPC/ppc64-encoding-ext.s
+++ b/llvm/test/MC/PowerPC/ppc64-encoding-ext.s
@@ -3443,29 +3443,29 @@
 # CHECK-BE: mfdar 2                         # encoding: [0x7c,0x53,0x02,0xa6]
 # CHECK-LE: mfdar 2                         # encoding: [0xa6,0x02,0x53,0x7c]
             mfdar 2
-# CHECK-BE: mtspr 22, 2                     # encoding: [0x7c,0x56,0x03,0xa6]
-# CHECK-LE: mtspr 22, 2                     # encoding: [0xa6,0x03,0x56,0x7c]
+# CHECK-BE: mtdec 2                         # encoding: [0x7c,0x56,0x03,0xa6]
+# CHECK-LE: mtdec 2                         # encoding: [0xa6,0x03,0x56,0x7c]
             mtdec 2
-# CHECK-BE: mfspr 2, 22                     # encoding: [0x7c,0x56,0x02,0xa6]
-# CHECK-LE: mfspr 2, 22                     # encoding: [0xa6,0x02,0x56,0x7c]
+# CHECK-BE: mfdec 2                         # encoding: [0x7c,0x56,0x02,0xa6]
+# CHECK-LE: mfdec 2                         # encoding: [0xa6,0x02,0x56,0x7c]
             mfdec 2
-# CHECK-BE: mtspr 25, 2                     # encoding: [0x7c,0x59,0x03,0xa6]
-# CHECK-LE: mtspr 25, 2                     # encoding: [0xa6,0x03,0x59,0x7c]
+# CHECK-BE: mtsdr1 2                        # encoding: [0x7c,0x59,0x03,0xa6]
+# CHECK-LE: mtsdr1 2                        # encoding: [0xa6,0x03,0x59,0x7c]
             mtsdr1 2
-# CHECK-BE: mfspr 2, 25                     # encoding: [0x7c,0x59,0x02,0xa6]
-# CHECK-LE: mfspr 2, 25                     # encoding: [0xa6,0x02,0x59,0x7c]
+# CHECK-BE: mfsdr1 2                        # encoding: [0x7c,0x59,0x02,0xa6]
+# CHECK-LE: mfsdr1 2                        # encoding: [0xa6,0x02,0x59,0x7c]
             mfsdr1 2
-# CHECK-BE: mtspr 26, 2                     # encoding: [0x7c,0x5a,0x03,0xa6]
-# CHECK-LE: mtspr 26, 2                     # encoding: [0xa6,0x03,0x5a,0x7c]
+# CHECK-BE: mtsrr0 2                        # encoding: [0x7c,0x5a,0x03,0xa6]
+# CHECK-LE: mtsrr0 2                        # encoding: [0xa6,0x03,0x5a,0x7c]
             mtsrr0 2
-# CHECK-BE: mfspr 2, 26                     # encoding: [0x7c,0x5a,0x02,0xa6]
-# CHECK-LE: mfspr 2, 26                     # encoding: [0xa6,0x02,0x5a,0x7c]
+# CHECK-BE: mfsrr0 2                        # encoding: [0x7c,0x5a,0x02,0xa6]
+# CHECK-LE: mfsrr0 2                        # encoding: [0xa6,0x02,0x5a,0x7c]
             mfsrr0 2
-# CHECK-BE: mtspr 27, 2                     # encoding: [0x7c,0x5b,0x03,0xa6]
-# CHECK-LE: mtspr 27, 2                     # encoding: [0xa6,0x03,0x5b,0x7c]
+# CHECK-BE: mtsrr1 2                        # encoding: [0x7c,0x5b,0x03,0xa6]
+# CHECK-LE: mtsrr1 2                        # encoding: [0xa6,0x03,0x5b,0x7c]
             mtsrr1 2
-# CHECK-BE: mfspr 2, 27                     # encoding: [0x7c,0x5b,0x02,0xa6]
-# CHECK-LE: mfspr 2, 27                     # encoding: [0xa6,0x02,0x5b,0x7c]
+# CHECK-BE: mfsrr1 2                        # encoding: [0x7c,0x5b,0x02,0xa6]
+# CHECK-LE: mfsrr1 2                        # encoding: [0xa6,0x02,0x5b,0x7c]
             mfsrr1 2
 # CHECK-BE: mtcfar 2                        # encoding: [0x7c,0x5c,0x03,0xa6]
 # CHECK-LE: mtcfar 2                        # encoding: [0xa6,0x03,0x5c,0x7c]
@@ -3636,8 +3636,8 @@
 # CHECK-BE: mfasr 2                          # encoding: [0x7c,0x58,0x42,0xa6]
 # CHECK-LE: mfasr 2                          # encoding: [0xa6,0x42,0x58,0x7c]
             mfasr 2
-# CHECK-BE: mtspr 280, 2                     # encoding: [0x7c,0x58,0x43,0xa6]
-# CHECK-LE: mtspr 280, 2                     # encoding: [0xa6,0x43,0x58,0x7c]
+# CHECK-BE: mtasr 2                          # encoding: [0x7c,0x58,0x43,0xa6]
+# CHECK-LE: mtasr 2                          # encoding: [0xa6,0x43,0x58,0x7c]
             mtasr 2
 
 # Load and Store Caching Inhibited Instructions
diff --git a/llvm/test/MC/PowerPC/ppc64-prefix-align.s b/llvm/test/MC/PowerPC/ppc64-prefix-align.s
index 80d2f0722a765..29594e9e33ae7 100644
--- a/llvm/test/MC/PowerPC/ppc64-prefix-align.s
+++ b/llvm/test/MC/PowerPC/ppc64-prefix-align.s
@@ -13,10 +13,10 @@
 
 beq 0, LAB1               # 4
 beq 1, LAB2               # 8
-# CHECK-BE:       0: 41 82 00 c0        bt 2, .+192
-# CHECK-BE-NEXT:  4: 41 86 00 f8        bt 6, .+248
-# CHECK-LE:       0: c0 00 82 41        bt 2, .+192
-# CHECK-LE-NEXT:  4: f8 00 86 41        bt 6, .+248
+# CHECK-BE:       0: 41 82 00 c0        bt 2, 0xc0
+# CHECK-BE-NEXT:  4: 41 86 00 f8        bt 6, 0xfc
+# CHECK-LE:       0: c0 00 82 41        bt 2, 0xc0
+# CHECK-LE-NEXT:  4: f8 00 86 41        bt 6, 0xfc
 paddi 1, 2, 8589934576, 0 # 16
 paddi 1, 2, 8589934576, 0 # 24
 paddi 1, 2, 8589934576, 0 # 32
diff --git a/llvm/test/MC/PowerPC/ppc64-reloc-directive.s b/llvm/test/MC/PowerPC/ppc64-reloc-directive.s
index 2442b085b0d79..2a03a92786b7e 100644
--- a/llvm/test/MC/PowerPC/ppc64-reloc-directive.s
+++ b/llvm/test/MC/PowerPC/ppc64-reloc-directive.s
@@ -7,10 +7,16 @@
 # PRINT: .reloc 8, R_PPC64_NONE, .data
 # PRINT: .reloc 4, R_PPC64_NONE, foo+4
 # PRINT: .reloc 0, R_PPC64_NONE, 8
+# PRINT: .reloc 0, R_PPC64_ADDR32, .data+2
+# PRINT: .reloc 0, R_PPC64_REL16_HI, foo+3
+# PRINT: .reloc 0, R_PPC64_REL16_HA, 5
 
 # CHECK:      0x8 R_PPC64_NONE .data 0x0
 # CHECK-NEXT: 0x4 R_PPC64_NONE foo 0x4
 # CHECK-NEXT: 0x0 R_PPC64_NONE - 0x8
+# CHECK-NEXT: 0x0 R_PPC64_ADDR32 .data 0x2
+# CHECK-NEXT: 0x0 R_PPC64_REL16_HI foo 0x3
+# CHECK-NEXT: 0x0 R_PPC64_REL16_HA - 0x5
 
 .text
   blr
@@ -19,6 +25,9 @@
   .reloc 8, R_PPC64_NONE, .data
   .reloc 4, R_PPC64_NONE, foo+4
   .reloc 0, R_PPC64_NONE, 8
+  .reloc 0, R_PPC64_ADDR32, .data+2
+  .reloc 0, R_PPC64_REL16_HI, foo+3
+  .reloc 0, R_PPC64_REL16_HA, 5
 
 .data
 .globl foo
diff --git a/llvm/test/MC/RISCV/attribute-arch.s b/llvm/test/MC/RISCV/attribute-arch.s
new file mode 100644
index 0000000000000..4329290b33ba8
--- /dev/null
+++ b/llvm/test/MC/RISCV/attribute-arch.s
@@ -0,0 +1,37 @@
+## Arch string without version.
+
+# RUN: llvm-mc %s -triple=riscv32 -filetype=asm | FileCheck %s
+# RUN: llvm-mc %s -triple=riscv64 -filetype=asm | FileCheck %s
+
+.attribute arch, "rv32i"
+# CHECK: attribute      5, "rv32i2p0"
+
+.attribute arch, "rv32i2"
+# CHECK: attribute      5, "rv32i2p0"
+
+.attribute arch, "rv32i2p"
+# CHECK: attribute      5, "rv32i2p0"
+
+.attribute arch, "rv32i2p0"
+# CHECK: attribute      5, "rv32i2p0"
+
+.attribute arch, "rv32i2_m2"
+# CHECK: attribute      5, "rv32i2p0_m2p0"
+
+.attribute arch, "rv32i2_ma"
+# CHECK: attribute      5, "rv32i2p0_m2p0_a2p0"
+
+.attribute arch, "rv32g"
+# CHECK: attribute      5, "rv32i2p0_m2p0_a2p0_f2p0_d2p0"
+
+.attribute arch, "rv32imafdc"
+# CHECK: attribute      5, "rv32i2p0_m2p0_a2p0_f2p0_d2p0_c2p0"
+
+.attribute arch, "rv32i2p0_mafdc"
+# CHECK: attribute      5, "rv32i2p0_m2p0_a2p0_f2p0_d2p0_c2p0"
+
+.attribute arch, "rv32ima2p0_fdc"
+# CHECK: attribute      5, "rv32i2p0_m2p0_a2p0_f2p0_d2p0_c2p0"
+
+.attribute arch, "rv32ima2p_fdc"
+# CHECK: attribute      5, "rv32i2p0_m2p0_a2p0_f2p0_d2p0_c2p0"
diff --git a/llvm/test/MC/RISCV/attribute-with-insts.s b/llvm/test/MC/RISCV/attribute-with-insts.s
new file mode 100644
index 0000000000000..9d9170a9f5aa4
--- /dev/null
+++ b/llvm/test/MC/RISCV/attribute-with-insts.s
@@ -0,0 +1,34 @@
+## Test .attribute effects.
+## We do not provide '-mattr=' and '.option rvc' and enable extensions through
+## '.attribute arch'.
+
+# RUN: llvm-mc -triple riscv32 -filetype=obj %s \
+# RUN:   | llvm-objdump --triple=riscv32 -d -M no-aliases - \
+# RUN:   | FileCheck -check-prefix=CHECK-INST %s
+
+# RUN: llvm-mc -triple riscv64 -filetype=obj %s \
+# RUN:   | llvm-objdump --triple=riscv64 -d -M no-aliases - \
+# RUN:   | FileCheck -check-prefix=CHECK-INST %s
+
+.attribute arch, "rv64i2p0_m2p0_a2p0_d2p0_c2p0"
+
+# CHECK-INST: lr.w t0, (t1)
+lr.w t0, (t1)
+
+# CHECK-INST: c.addi a3, -32
+c.addi a3, -32
+
+# CHECK-INST: fmadd.d fa0, fa1, fa2, fa3, dyn
+fmadd.d f10, f11, f12, f13, dyn
+
+# CHECK-INST: fmadd.s fa0, fa1, fa2, fa3, dyn
+fmadd.s f10, f11, f12, f13, dyn
+
+# CHECK-INST: addi ra, sp, 2
+addi ra, sp, 2
+
+# CHECK-INST: mul a4, ra, s0
+mul a4, ra, s0
+
+# CHECK-INST: addw a2, a3, a4
+addw a2, a3, a4
diff --git a/llvm/test/MC/RISCV/attribute-with-option.s b/llvm/test/MC/RISCV/attribute-with-option.s
new file mode 100644
index 0000000000000..749717cb7a550
--- /dev/null
+++ b/llvm/test/MC/RISCV/attribute-with-option.s
@@ -0,0 +1,21 @@
+## When a user specifies an architecture extension which conflicts with an
+## architecture attribute, we use the architecture attribute instead of the
+## command line option.
+##
+## This test uses option '-mattr=+e' to specify the "e" extension. However,
+## there is an architecture attribute in the file to specify rv32i. We will
+## use rv32i to assemble the file instead of rv32e.
+
+# RUN: llvm-mc %s -triple=riscv32 -mattr=+e -filetype=obj -o - \
+# RUN:   | llvm-readobj -A - | FileCheck %s
+
+.attribute arch, "rv32i2p0"
+## Invalid operand for RV32E, because x16 is an invalid register for RV32E.
+## Use RV32I to assemble, since it will not trigger an assembly error.
+lui x16, 1
+
+## Check that the architecture attribute is not overridden by the command line
+## option.
+# CHECK:      Tag: 5
+# CHECK-NEXT: TagName: arch
+# CHECK-NEXT: Value: rv32i2p0
diff --git a/llvm/test/MC/RISCV/attribute.s b/llvm/test/MC/RISCV/attribute.s
new file mode 100644
index 0000000000000..b7ad98bb6fe23
--- /dev/null
+++ b/llvm/test/MC/RISCV/attribute.s
@@ -0,0 +1,22 @@
+## Test llvm-mc could handle .attribute correctly.
+
+# RUN: llvm-mc %s -triple=riscv32 -filetype=asm | FileCheck %s
+# RUN: llvm-mc %s -triple=riscv64 -filetype=asm | FileCheck %s
+
+.attribute stack_align, 16
+# CHECK: attribute      4, 16
+
+.attribute arch, "rv32i2p0_m2p0_a2p0_c2p0"
+# CHECK: attribute      5, "rv32i2p0_m2p0_a2p0_c2p0"
+
+.attribute unaligned_access, 0
+# CHECK: attribute      6, 0
+
+.attribute priv_spec, 2
+# CHECK: attribute      8, 2
+
+.attribute priv_spec_minor, 0
+# CHECK: attribute      10, 0
+
+.attribute priv_spec_revision, 0
+# CHECK: attribute      12, 0
diff --git a/llvm/test/MC/RISCV/invalid-attribute.s b/llvm/test/MC/RISCV/invalid-attribute.s
new file mode 100644
index 0000000000000..e3cfe28fdf442
--- /dev/null
+++ b/llvm/test/MC/RISCV/invalid-attribute.s
@@ -0,0 +1,31 @@
+## Negative tests:
+##  - Feed integer value to string type attribute.
+##  - Feed string value to integer type attribute.
+##  - Invalid arch string.
+
+# RUN: not llvm-mc %s -triple=riscv32 -filetype=asm 2>&1 | FileCheck %s
+# RUN: not llvm-mc %s -triple=riscv64 -filetype=asm 2>&1 | FileCheck %s
+
+.attribute arch, "foo"
+# CHECK: [[@LINE-1]]:18: error: bad arch string foo
+
+.attribute arch, "rv32i2p0_y2p0"
+# CHECK: [[@LINE-1]]:18: error: bad arch string y2p0
+
+.attribute stack_align, "16"
+# CHECK: [[@LINE-1]]:25: error: expected numeric constant
+
+.attribute unaligned_access, "0"
+# CHECK: [[@LINE-1]]:30: error: expected numeric constant
+
+.attribute priv_spec, "2"
+# CHECK: [[@LINE-1]]:23: error: expected numeric constant
+
+.attribute priv_spec_minor, "0"
+# CHECK: [[@LINE-1]]:29: error: expected numeric constant
+
+.attribute priv_spec_revision, "0"
+# CHECK: [[@LINE-1]]:32: error: expected numeric constant
+
+.attribute arch, 30
+# CHECK: [[@LINE-1]]:18: error: expected string constant
diff --git a/llvm/test/MC/SystemZ/insn-bad-z13.s b/llvm/test/MC/SystemZ/insn-bad-z13.s
index c362b4b27ba3b..4ae4fd99fbd4d 100644
--- a/llvm/test/MC/SystemZ/insn-bad-z13.s
+++ b/llvm/test/MC/SystemZ/insn-bad-z13.s
@@ -33,8 +33,6 @@
 #CHECK: cdpt	%f0, -1(1,%r1), 0
 #CHECK: error: invalid operand
 #CHECK: cdpt	%f0, 4096(1,%r1), 0
-#CHECK: error: %r0 used in an address
-#CHECK: cdpt	%f0, 0(1,%r0), 0
 #CHECK: error: invalid use of indexed addressing
 #CHECK: cdpt	%f0, 0(%r1,%r2), 0
 #CHECK: error: unknown token in expression
@@ -48,7 +46,6 @@
 	cdpt	%f0, 0(257,%r1), 0
 	cdpt	%f0, -1(1,%r1), 0
 	cdpt	%f0, 4096(1,%r1), 0
-	cdpt	%f0, 0(1,%r0), 0
 	cdpt	%f0, 0(%r1,%r2), 0
 	cdpt	%f0, 0(-), 0
 
@@ -68,8 +65,6 @@
 #CHECK: cpdt	%f0, -1(1,%r1), 0
 #CHECK: error: invalid operand
 #CHECK: cpdt	%f0, 4096(1,%r1), 0
-#CHECK: error: %r0 used in an address
-#CHECK: cpdt	%f0, 0(1,%r0), 0
 #CHECK: error: invalid use of indexed addressing
 #CHECK: cpdt	%f0, 0(%r1,%r2), 0
 #CHECK: error: unknown token in expression
@@ -83,7 +78,6 @@
 	cpdt	%f0, 0(257,%r1), 0
 	cpdt	%f0, -1(1,%r1), 0
 	cpdt	%f0, 4096(1,%r1), 0
-	cpdt	%f0, 0(1,%r0), 0
 	cpdt	%f0, 0(%r1,%r2), 0
 	cpdt	%f0, 0(-), 0
 
@@ -103,8 +97,6 @@
 #CHECK: cpxt	%f0, -1(1,%r1), 0
 #CHECK: error: invalid operand
 #CHECK: cpxt	%f0, 4096(1,%r1), 0
-#CHECK: error: %r0 used in an address
-#CHECK: cpxt	%f0, 0(1,%r0), 0
 #CHECK: error: invalid use of indexed addressing
 #CHECK: cpxt	%f0, 0(%r1,%r2), 0
 #CHECK: error: unknown token in expression
@@ -120,7 +112,6 @@
 	cpxt	%f0, 0(257,%r1), 0
 	cpxt	%f0, -1(1,%r1), 0
 	cpxt	%f0, 4096(1,%r1), 0
-	cpxt	%f0, 0(1,%r0), 0
 	cpxt	%f0, 0(%r1,%r2), 0
 	cpxt	%f0, 0(-), 0
 	cpxt	%f15, 0(1), 0
@@ -141,8 +132,6 @@
 #CHECK: cxpt	%f0, -1(1,%r1), 0
 #CHECK: error: invalid operand
 #CHECK: cxpt	%f0, 4096(1,%r1), 0
-#CHECK: error: %r0 used in an address
-#CHECK: cxpt	%f0, 0(1,%r0), 0
 #CHECK: error: invalid use of indexed addressing
 #CHECK: cxpt	%f0, 0(%r1,%r2), 0
 #CHECK: error: unknown token in expression
@@ -158,7 +147,6 @@
 	cxpt	%f0, 0(257,%r1), 0
 	cxpt	%f0, -1(1,%r1), 0
 	cxpt	%f0, 4096(1,%r1), 0
-	cxpt	%f0, 0(1,%r0), 0
 	cxpt	%f0, 0(%r1,%r2), 0
 	cxpt	%f0, 0(-), 0
 	cxpt	%f15, 0(1), 0
@@ -1880,58 +1868,43 @@
 #CHECK: vlgv	%r0, %v0, -1, 0
 #CHECK: error: invalid operand
 #CHECK: vlgv	%r0, %v0, 4096, 0
-#CHECK: error: %r0 used in an address
-#CHECK: vlgv	%r0, %v0, 0(%r0), 0
 
 	vlgv	%r0, %v0, 0, -1
 	vlgv	%r0, %v0, 0, 16
 	vlgv	%r0, %v0, -1, 0
 	vlgv	%r0, %v0, 4096, 0
-	vlgv	%r0, %v0, 0(%r0), 0
 
 #CHECK: error: invalid operand
 #CHECK: vlgvb	%r0, %v0, -1
 #CHECK: error: invalid operand
 #CHECK: vlgvb	%r0, %v0, 4096
-#CHECK: error: %r0 used in an address
-#CHECK: vlgvb	%r0, %v0, 0(%r0)
 
 	vlgvb	%r0, %v0, -1
 	vlgvb	%r0, %v0, 4096
-	vlgvb	%r0, %v0, 0(%r0)
 
 #CHECK: error: invalid operand
 #CHECK: vlgvf	%r0, %v0, -1
 #CHECK: error: invalid operand
 #CHECK: vlgvf	%r0, %v0, 4096
-#CHECK: error: %r0 used in an address
-#CHECK: vlgvf	%r0, %v0, 0(%r0)
 
 	vlgvf	%r0, %v0, -1
 	vlgvf	%r0, %v0, 4096
-	vlgvf	%r0, %v0, 0(%r0)
 
 #CHECK: error: invalid operand
 #CHECK: vlgvg	%r0, %v0, -1
 #CHECK: error: invalid operand
 #CHECK: vlgvg	%r0, %v0, 4096
-#CHECK: error: %r0 used in an address
-#CHECK: vlgvg	%r0, %v0, 0(%r0)
 
 	vlgvg	%r0, %v0, -1
 	vlgvg	%r0, %v0, 4096
-	vlgvg	%r0, %v0, 0(%r0)
 
 #CHECK: error: invalid operand
 #CHECK: vlgvh	%r0, %v0, -1
 #CHECK: error: invalid operand
 #CHECK: vlgvh	%r0, %v0, 4096
-#CHECK: error: %r0 used in an address
-#CHECK: vlgvh	%r0, %v0, 0(%r0)
 
 	vlgvh	%r0, %v0, -1
 	vlgvh	%r0, %v0, 4096
-	vlgvh	%r0, %v0, 0(%r0)
 
 #CHECK: error: instruction requires: vector-packed-decimal
 #CHECK: vlip	%v0, 0, 0
@@ -1942,12 +1915,9 @@
 #CHECK: vll	%v0, %r0, -1
 #CHECK: error: invalid operand
 #CHECK: vll	%v0, %r0, 4096
-#CHECK: error: %r0 used in an address
-#CHECK: vll	%v0, %r0, 0(%r0)
 
 	vll	%v0, %r0, -1
 	vll	%v0, %r0, 4096
-	vll	%v0, %r0, 0(%r0)
 
 #CHECK: error: invalid operand
 #CHECK: vllez	%v0, 0, -1
@@ -2108,58 +2078,43 @@
 #CHECK: vlvg	%v0, %r0, -1, 0
 #CHECK: error: invalid operand
 #CHECK: vlvg	%v0, %r0, 4096, 0
-#CHECK: error: %r0 used in an address
-#CHECK: vlvg	%v0, %r0, 0(%r0), 0
 
 	vlvg	%v0, %r0, 0, -1
 	vlvg	%v0, %r0, 0, 16
 	vlvg	%v0, %r0, -1, 0
 	vlvg	%v0, %r0, 4096, 0
-	vlvg	%v0, %r0, 0(%r0), 0
 
 #CHECK: error: invalid operand
 #CHECK: vlvgb	%v0, %r0, -1
 #CHECK: error: invalid operand
 #CHECK: vlvgb	%v0, %r0, 4096
-#CHECK: error: %r0 used in an address
-#CHECK: vlvgb	%v0, %r0, 0(%r0)
 
 	vlvgb	%v0, %r0, -1
 	vlvgb	%v0, %r0, 4096
-	vlvgb	%v0, %r0, 0(%r0)
 
 #CHECK: error: invalid operand
 #CHECK: vlvgf	%v0, %r0, -1
 #CHECK: error: invalid operand
 #CHECK: vlvgf	%v0, %r0, 4096
-#CHECK: error: %r0 used in an address
-#CHECK: vlvgf	%v0, %r0, 0(%r0)
 
 	vlvgf	%v0, %r0, -1
 	vlvgf	%v0, %r0, 4096
-	vlvgf	%v0, %r0, 0(%r0)
 
 #CHECK: error: invalid operand
 #CHECK: vlvgg	%v0, %r0, -1
 #CHECK: error: invalid operand
 #CHECK: vlvgg	%v0, %r0, 4096
-#CHECK: error: %r0 used in an address
-#CHECK: vlvgg	%v0, %r0, 0(%r0)
 
 	vlvgg	%v0, %r0, -1
 	vlvgg	%v0, %r0, 4096
-	vlvgg	%v0, %r0, 0(%r0)
 
 #CHECK: error: invalid operand
 #CHECK: vlvgh	%v0, %r0, -1
 #CHECK: error: invalid operand
 #CHECK: vlvgh	%v0, %r0, 4096
-#CHECK: error: %r0 used in an address
-#CHECK: vlvgh	%v0, %r0, 0(%r0)
 
 	vlvgh	%v0, %r0, -1
 	vlvgh	%v0, %r0, 4096
-	vlvgh	%v0, %r0, 0(%r0)
 
 #CHECK: error: instruction requires: vector-packed-decimal
 #CHECK: vmp	%v0, %v0, %v0, 0, 0
@@ -2475,12 +2430,9 @@
 #CHECK: vstl	%v0, %r0, -1
 #CHECK: error: invalid operand
 #CHECK: vstl	%v0, %r0, 4096
-#CHECK: error: %r0 used in an address
-#CHECK: vstl	%v0, %r0, 0(%r0)
 
 	vstl	%v0, %r0, -1
 	vstl	%v0, %r0, 4096
-	vstl	%v0, %r0, 0(%r0)
 
 #CHECK: error: invalid operand
 #CHECK: vstm	%v0, %v0, -1
diff --git a/llvm/test/MC/SystemZ/insn-bad-z14.s b/llvm/test/MC/SystemZ/insn-bad-z14.s
index f3c8414af7d80..568717a85c0e1 100644
--- a/llvm/test/MC/SystemZ/insn-bad-z14.s
+++ b/llvm/test/MC/SystemZ/insn-bad-z14.s
@@ -634,25 +634,19 @@
 #CHECK: vlrl	%v0, -1, 0
 #CHECK: error: invalid operand
 #CHECK: vlrl	%v0, 4096, 0
-#CHECK: error: %r0 used in an address
-#CHECK: vlrl	%v0, 0(%r0), 0
 
 	vlrl	%v0, 0, -1
 	vlrl	%v0, 0, 256
 	vlrl	%v0, -1, 0
 	vlrl	%v0, 4096, 0
-	vlrl	%v0, 0(%r0), 0
 
 #CHECK: error: invalid operand
 #CHECK: vlrlr	%v0, %r0, -1
 #CHECK: error: invalid operand
 #CHECK: vlrlr	%v0, %r0, 4096
-#CHECK: error: %r0 used in an address
-#CHECK: vlrlr	%v0, %r0, 0(%r0)
 
 	vlrlr	%v0, %r0, -1
 	vlrlr	%v0, %r0, 4096
-	vlrlr	%v0, %r0, 0(%r0)
 
 #CHECK: error: invalid operand
 #CHECK: vmp	%v0, %v0, %v0, 0, -1
@@ -712,14 +706,11 @@
 #CHECK: vpkz	%v0, -1, 0
 #CHECK: error: invalid operand
 #CHECK: vpkz	%v0, 4096, 0
-#CHECK: error: %r0 used in an address
-#CHECK: vpkz	%v0, 0(%r0), 0
 
 	vpkz	%v0, 0, -1
 	vpkz	%v0, 0, 256
 	vpkz	%v0, -1, 0
 	vpkz	%v0, 4096, 0
-	vpkz	%v0, 0(%r0), 0
 
 #CHECK: error: invalid operand
 #CHECK: vpsop	%v0, %v0, 0, 0, -1
@@ -863,25 +854,19 @@
 #CHECK: vstrl	%v0, -1, 0
 #CHECK: error: invalid operand
 #CHECK: vstrl	%v0, 4096, 0
-#CHECK: error: %r0 used in an address
-#CHECK: vstrl	%v0, 0(%r0), 0
 
 	vstrl	%v0, 0, -1
 	vstrl	%v0, 0, 256
 	vstrl	%v0, -1, 0
 	vstrl	%v0, 4096, 0
-	vstrl	%v0, 0(%r0), 0
 
 #CHECK: error: invalid operand
 #CHECK: vstrlr	%v0, %r0, -1
 #CHECK: error: invalid operand
 #CHECK: vstrlr	%v0, %r0, 4096
-#CHECK: error: %r0 used in an address
-#CHECK: vstrlr	%v0, %r0, 0(%r0)
 
 	vstrlr	%v0, %r0, -1
 	vstrlr	%v0, %r0, 4096
-	vstrlr	%v0, %r0, 0(%r0)
 
 #CHECK: error: instruction requires: vector-enhancements-2
 #CHECK: vstrs	%v0, %v0, %v0, %v0, 0
@@ -914,14 +899,11 @@
 #CHECK: vupkz	%v0, -1, 0
 #CHECK: error: invalid operand
 #CHECK: vupkz	%v0, 4096, 0
-#CHECK: error: %r0 used in an address
-#CHECK: vupkz	%v0, 0(%r0), 0
 
 	vupkz	%v0, 0, -1
 	vupkz	%v0, 0, 256
 	vupkz	%v0, -1, 0
 	vupkz	%v0, 4096, 0
-	vupkz	%v0, 0(%r0), 0
 
 #CHECK: error: instruction requires: vector-enhancements-2
 #CHECK: wcefb	%v0, %v0, 0, 0
diff --git a/llvm/test/MC/SystemZ/insn-bad-z196.s b/llvm/test/MC/SystemZ/insn-bad-z196.s
index 33059529f74b1..04e53c3507e49 100644
--- a/llvm/test/MC/SystemZ/insn-bad-z196.s
+++ b/llvm/test/MC/SystemZ/insn-bad-z196.s
@@ -1260,56 +1260,44 @@
 #CHECK: slak	%r0,%r0,-524289
 #CHECK: error: invalid operand
 #CHECK: slak	%r0,%r0,524288
-#CHECK: error: %r0 used in an address
-#CHECK: slak	%r0,%r0,0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: slak	%r0,%r0,0(%r1,%r2)
 
 	slak	%r0,%r0,-524289
 	slak	%r0,%r0,524288
-	slak	%r0,%r0,0(%r0)
 	slak	%r0,%r0,0(%r1,%r2)
 
 #CHECK: error: invalid operand
 #CHECK: sllk	%r0,%r0,-524289
 #CHECK: error: invalid operand
 #CHECK: sllk	%r0,%r0,524288
-#CHECK: error: %r0 used in an address
-#CHECK: sllk	%r0,%r0,0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: sllk	%r0,%r0,0(%r1,%r2)
 
 	sllk	%r0,%r0,-524289
 	sllk	%r0,%r0,524288
-	sllk	%r0,%r0,0(%r0)
 	sllk	%r0,%r0,0(%r1,%r2)
 
 #CHECK: error: invalid operand
 #CHECK: srak	%r0,%r0,-524289
 #CHECK: error: invalid operand
 #CHECK: srak	%r0,%r0,524288
-#CHECK: error: %r0 used in an address
-#CHECK: srak	%r0,%r0,0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: srak	%r0,%r0,0(%r1,%r2)
 
 	srak	%r0,%r0,-524289
 	srak	%r0,%r0,524288
-	srak	%r0,%r0,0(%r0)
 	srak	%r0,%r0,0(%r1,%r2)
 
 #CHECK: error: invalid operand
 #CHECK: srlk	%r0,%r0,-524289
 #CHECK: error: invalid operand
 #CHECK: srlk	%r0,%r0,524288
-#CHECK: error: %r0 used in an address
-#CHECK: srlk	%r0,%r0,0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: srlk	%r0,%r0,0(%r1,%r2)
 
 	srlk	%r0,%r0,-524289
 	srlk	%r0,%r0,524288
-	srlk	%r0,%r0,0(%r0)
 	srlk	%r0,%r0,0(%r1,%r2)
 
 #CHECK: error: invalid operand
diff --git a/llvm/test/MC/SystemZ/insn-bad-zEC12.s b/llvm/test/MC/SystemZ/insn-bad-zEC12.s
index 552180a7c6107..b5cc6ebfd0be1 100644
--- a/llvm/test/MC/SystemZ/insn-bad-zEC12.s
+++ b/llvm/test/MC/SystemZ/insn-bad-zEC12.s
@@ -83,8 +83,6 @@
 #CHECK: cdzt	%f0, -1(1,%r1), 0
 #CHECK: error: invalid operand
 #CHECK: cdzt	%f0, 4096(1,%r1), 0
-#CHECK: error: %r0 used in an address
-#CHECK: cdzt	%f0, 0(1,%r0), 0
 #CHECK: error: invalid use of indexed addressing
 #CHECK: cdzt	%f0, 0(%r1,%r2), 0
 #CHECK: error: unknown token in expression
@@ -98,7 +96,6 @@
 	cdzt	%f0, 0(257,%r1), 0
 	cdzt	%f0, -1(1,%r1), 0
 	cdzt	%f0, 4096(1,%r1), 0
-	cdzt	%f0, 0(1,%r0), 0
 	cdzt	%f0, 0(%r1,%r2), 0
 	cdzt	%f0, 0(-), 0
 
@@ -197,8 +194,6 @@
 #CHECK: cxzt	%f0, -1(1,%r1), 0
 #CHECK: error: invalid operand
 #CHECK: cxzt	%f0, 4096(1,%r1), 0
-#CHECK: error: %r0 used in an address
-#CHECK: cxzt	%f0, 0(1,%r0), 0
 #CHECK: error: invalid use of indexed addressing
 #CHECK: cxzt	%f0, 0(%r1,%r2), 0
 #CHECK: error: unknown token in expression
@@ -214,7 +209,6 @@
 	cxzt	%f0, 0(257,%r1), 0
 	cxzt	%f0, -1(1,%r1), 0
 	cxzt	%f0, 4096(1,%r1), 0
-	cxzt	%f0, 0(1,%r0), 0
 	cxzt	%f0, 0(%r1,%r2), 0
 	cxzt	%f0, 0(-), 0
 	cxzt	%f15, 0(1), 0
@@ -235,8 +229,6 @@
 #CHECK: czdt	%f0, -1(1,%r1), 0
 #CHECK: error: invalid operand
 #CHECK: czdt	%f0, 4096(1,%r1), 0
-#CHECK: error: %r0 used in an address
-#CHECK: czdt	%f0, 0(1,%r0), 0
 #CHECK: error: invalid use of indexed addressing
 #CHECK: czdt	%f0, 0(%r1,%r2), 0
 #CHECK: error: unknown token in expression
@@ -250,7 +242,6 @@
 	czdt	%f0, 0(257,%r1), 0
 	czdt	%f0, -1(1,%r1), 0
 	czdt	%f0, 4096(1,%r1), 0
-	czdt	%f0, 0(1,%r0), 0
 	czdt	%f0, 0(%r1,%r2), 0
 	czdt	%f0, 0(-), 0
 
@@ -270,8 +261,6 @@
 #CHECK: czxt	%f0, -1(1,%r1), 0
 #CHECK: error: invalid operand
 #CHECK: czxt	%f0, 4096(1,%r1), 0
-#CHECK: error: %r0 used in an address
-#CHECK: czxt	%f0, 0(1,%r0), 0
 #CHECK: error: invalid use of indexed addressing
 #CHECK: czxt	%f0, 0(%r1,%r2), 0
 #CHECK: error: unknown token in expression
@@ -287,7 +276,6 @@
 	czxt	%f0, 0(257,%r1), 0
 	czxt	%f0, -1(1,%r1), 0
 	czxt	%f0, 4096(1,%r1), 0
-	czxt	%f0, 0(1,%r0), 0
 	czxt	%f0, 0(%r1,%r2), 0
 	czxt	%f0, 0(-), 0
 	czxt	%f15, 0(1), 0
diff --git a/llvm/test/MC/SystemZ/insn-bad.s b/llvm/test/MC/SystemZ/insn-bad.s
index 7dfbba743b60e..c330fa496a25a 100644
--- a/llvm/test/MC/SystemZ/insn-bad.s
+++ b/llvm/test/MC/SystemZ/insn-bad.s
@@ -328,10 +328,6 @@
 #CHECK: ap	0(1,%r1), -1(1,%r1)
 #CHECK: error: invalid operand
 #CHECK: ap	0(1,%r1), 4096(1,%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: ap	0(1,%r0), 0(1,%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: ap	0(1,%r1), 0(1,%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: ap	0(%r1,%r2), 0(1,%r1)
 #CHECK: error: invalid use of indexed addressing
@@ -351,8 +347,6 @@
 	ap	4096(1,%r1), 0(1,%r1)
 	ap	0(1,%r1), -1(1,%r1)
 	ap	0(1,%r1), 4096(1,%r1)
-	ap	0(1,%r0), 0(1,%r1)
-	ap	0(1,%r1), 0(1,%r0)
 	ap	0(%r1,%r2), 0(1,%r1)
 	ap	0(1,%r2), 0(%r1,%r2)
 	ap	0(-), 0(1)
@@ -1363,10 +1357,6 @@
 #CHECK: clc	0(1,%r1), -1(%r1)
 #CHECK: error: invalid operand
 #CHECK: clc	0(1,%r1), 4096(%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: clc	0(1,%r0), 0(%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: clc	0(1,%r1), 0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: clc	0(%r1,%r2), 0(%r1)
 #CHECK: error: invalid use of indexed addressing
@@ -1383,8 +1373,6 @@
 	clc	4096(1,%r1), 0(%r1)
 	clc	0(1,%r1), -1(%r1)
 	clc	0(1,%r1), 4096(%r1)
-	clc	0(1,%r0), 0(%r1)
-	clc	0(1,%r1), 0(%r0)
 	clc	0(%r1,%r2), 0(%r1)
 	clc	0(1,%r2), 0(%r1,%r2)
 	clc	0(-), 0
@@ -1904,10 +1892,6 @@
 #CHECK: cp	0(1,%r1), -1(1,%r1)
 #CHECK: error: invalid operand
 #CHECK: cp	0(1,%r1), 4096(1,%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: cp	0(1,%r0), 0(1,%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: cp	0(1,%r1), 0(1,%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: cp	0(%r1,%r2), 0(1,%r1)
 #CHECK: error: invalid use of indexed addressing
@@ -1927,8 +1911,6 @@
 	cp	4096(1,%r1), 0(1,%r1)
 	cp	0(1,%r1), -1(1,%r1)
 	cp	0(1,%r1), 4096(1,%r1)
-	cp	0(1,%r0), 0(1,%r1)
-	cp	0(1,%r1), 0(1,%r0)
 	cp	0(%r1,%r2), 0(1,%r1)
 	cp	0(1,%r2), 0(%r1,%r2)
 	cp	0(-), 0(1)
@@ -2472,10 +2454,6 @@
 #CHECK: dp	0(1,%r1), -1(1,%r1)
 #CHECK: error: invalid operand
 #CHECK: dp	0(1,%r1), 4096(1,%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: dp	0(1,%r0), 0(1,%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: dp	0(1,%r1), 0(1,%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: dp	0(%r1,%r2), 0(1,%r1)
 #CHECK: error: invalid use of indexed addressing
@@ -2495,8 +2473,6 @@
 	dp	4096(1,%r1), 0(1,%r1)
 	dp	0(1,%r1), -1(1,%r1)
 	dp	0(1,%r1), 4096(1,%r1)
-	dp	0(1,%r0), 0(1,%r1)
-	dp	0(1,%r1), 0(1,%r0)
 	dp	0(%r1,%r2), 0(1,%r1)
 	dp	0(1,%r2), 0(%r1,%r2)
 	dp	0(-), 0(1)
@@ -2611,10 +2587,6 @@
 #CHECK: ed	0(1,%r1), -1(%r1)
 #CHECK: error: invalid operand
 #CHECK: ed	0(1,%r1), 4096(%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: ed	0(1,%r0), 0(%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: ed	0(1,%r1), 0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: ed	0(%r1,%r2), 0(%r1)
 #CHECK: error: invalid use of indexed addressing
@@ -2631,8 +2603,6 @@
 	ed	4096(1,%r1), 0(%r1)
 	ed	0(1,%r1), -1(%r1)
 	ed	0(1,%r1), 4096(%r1)
-	ed	0(1,%r0), 0(%r1)
-	ed	0(1,%r1), 0(%r0)
 	ed	0(%r1,%r2), 0(%r1)
 	ed	0(1,%r2), 0(%r1,%r2)
 	ed	0(-), 0
@@ -2655,10 +2625,6 @@
 #CHECK: edmk	0(1,%r1), -1(%r1)
 #CHECK: error: invalid operand
 #CHECK: edmk	0(1,%r1), 4096(%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: edmk	0(1,%r0), 0(%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: edmk	0(1,%r1), 0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: edmk	0(%r1,%r2), 0(%r1)
 #CHECK: error: invalid use of indexed addressing
@@ -2675,8 +2641,6 @@
 	edmk	4096(1,%r1), 0(%r1)
 	edmk	0(1,%r1), -1(%r1)
 	edmk	0(1,%r1), 4096(%r1)
-	edmk	0(1,%r0), 0(%r1)
-	edmk	0(1,%r1), 0(%r0)
 	edmk	0(%r1,%r2), 0(%r1)
 	edmk	0(1,%r2), 0(%r1,%r2)
 	edmk	0(-), 0
@@ -4281,10 +4245,6 @@
 #CHECK: mp	0(1,%r1), -1(1,%r1)
 #CHECK: error: invalid operand
 #CHECK: mp	0(1,%r1), 4096(1,%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: mp	0(1,%r0), 0(1,%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: mp	0(1,%r1), 0(1,%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: mp	0(%r1,%r2), 0(1,%r1)
 #CHECK: error: invalid use of indexed addressing
@@ -4304,8 +4264,6 @@
 	mp	4096(1,%r1), 0(1,%r1)
 	mp	0(1,%r1), -1(1,%r1)
 	mp	0(1,%r1), 4096(1,%r1)
-	mp	0(1,%r0), 0(1,%r1)
-	mp	0(1,%r1), 0(1,%r0)
 	mp	0(%r1,%r2), 0(1,%r1)
 	mp	0(1,%r2), 0(%r1,%r2)
 	mp	0(-), 0(1)
@@ -4429,10 +4387,6 @@
 #CHECK: mvc	0(1,%r1), -1(%r1)
 #CHECK: error: invalid operand
 #CHECK: mvc	0(1,%r1), 4096(%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: mvc	0(1,%r0), 0(%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: mvc	0(1,%r1), 0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: mvc	0(%r1,%r2), 0(%r1)
 #CHECK: error: invalid use of indexed addressing
@@ -4449,8 +4403,6 @@
 	mvc	4096(1,%r1), 0(%r1)
 	mvc	0(1,%r1), -1(%r1)
 	mvc	0(1,%r1), 4096(%r1)
-	mvc	0(1,%r0), 0(%r1)
-	mvc	0(1,%r1), 0(%r0)
 	mvc	0(%r1,%r2), 0(%r1)
 	mvc	0(1,%r2), 0(%r1,%r2)
 	mvc	0(-), 0
@@ -4490,10 +4442,6 @@
 #CHECK: mvcin	0(1,%r1), -1(%r1)
 #CHECK: error: invalid operand
 #CHECK: mvcin	0(1,%r1), 4096(%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: mvcin	0(1,%r0), 0(%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: mvcin	0(1,%r1), 0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: mvcin	0(%r1,%r2), 0(%r1)
 #CHECK: error: invalid use of indexed addressing
@@ -4510,8 +4458,6 @@
 	mvcin	4096(1,%r1), 0(%r1)
 	mvcin	0(1,%r1), -1(%r1)
 	mvcin	0(1,%r1), 4096(%r1)
-	mvcin	0(1,%r0), 0(%r1)
-	mvcin	0(1,%r1), 0(%r0)
 	mvcin	0(%r1,%r2), 0(%r1)
 	mvcin	0(1,%r2), 0(%r1,%r2)
 	mvcin	0(-), 0
@@ -4526,10 +4472,6 @@
 #CHECK: mvck	0(%r1,%r1), -1(%r1), %r3
 #CHECK: error: invalid operand
 #CHECK: mvck	0(%r1,%r1), 4096(%r1), %r3
-#CHECK: error: %r0 used in an address
-#CHECK: mvck	0(%r1,%r0), 0(%r1), %r3
-#CHECK: error: %r0 used in an address
-#CHECK: mvck	0(%r1,%r1), 0(%r0), %r3
 #CHECK: error: invalid use of indexed addressing
 #CHECK: mvck	0(%r1,%r2), 0(%r1,%r2), %r3
 #CHECK: error: unknown token in expression
@@ -4540,8 +4482,6 @@
 	mvck	4096(%r1,%r1), 0(%r1), %r3
 	mvck	0(%r1,%r1), -1(%r1), %r3
 	mvck	0(%r1,%r1), 4096(%r1), %r3
-	mvck	0(%r1,%r0), 0(%r1), %r3
-	mvck	0(%r1,%r1), 0(%r0), %r3
 	mvck	0(%r1,%r2), 0(%r1,%r2), %r3
 	mvck	0(-), 0, %r3
 
@@ -4608,10 +4548,6 @@
 #CHECK: mvcp	0(%r1,%r1), -1(%r1), %r3
 #CHECK: error: invalid operand
 #CHECK: mvcp	0(%r1,%r1), 4096(%r1), %r3
-#CHECK: error: %r0 used in an address
-#CHECK: mvcp	0(%r1,%r0), 0(%r1), %r3
-#CHECK: error: %r0 used in an address
-#CHECK: mvcp	0(%r1,%r1), 0(%r0), %r3
 #CHECK: error: invalid use of indexed addressing
 #CHECK: mvcp	0(%r1,%r2), 0(%r1,%r2), %r3
 #CHECK: error: unknown token in expression
@@ -4622,8 +4558,6 @@
 	mvcp	4096(%r1,%r1), 0(%r1), %r3
 	mvcp	0(%r1,%r1), -1(%r1), %r3
 	mvcp	0(%r1,%r1), 4096(%r1), %r3
-	mvcp	0(%r1,%r0), 0(%r1), %r3
-	mvcp	0(%r1,%r1), 0(%r0), %r3
 	mvcp	0(%r1,%r2), 0(%r1,%r2), %r3
 	mvcp	0(-), 0, %r3
 
@@ -4637,10 +4571,6 @@
 #CHECK: mvcs	0(%r1,%r1), -1(%r1), %r3
 #CHECK: error: invalid operand
 #CHECK: mvcs	0(%r1,%r1), 4096(%r1), %r3
-#CHECK: error: %r0 used in an address
-#CHECK: mvcs	0(%r1,%r0), 0(%r1), %r3
-#CHECK: error: %r0 used in an address
-#CHECK: mvcs	0(%r1,%r1), 0(%r0), %r3
 #CHECK: error: invalid use of indexed addressing
 #CHECK: mvcs	0(%r1,%r2), 0(%r1,%r2), %r3
 #CHECK: error: unknown token in expression
@@ -4651,8 +4581,6 @@
 	mvcs	4096(%r1,%r1), 0(%r1), %r3
 	mvcs	0(%r1,%r1), -1(%r1), %r3
 	mvcs	0(%r1,%r1), 4096(%r1), %r3
-	mvcs	0(%r1,%r0), 0(%r1), %r3
-	mvcs	0(%r1,%r1), 0(%r0), %r3
 	mvcs	0(%r1,%r2), 0(%r1,%r2), %r3
 	mvcs	0(-), 0, %r3
 
@@ -4776,10 +4704,6 @@
 #CHECK: mvn	0(1,%r1), -1(%r1)
 #CHECK: error: invalid operand
 #CHECK: mvn	0(1,%r1), 4096(%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: mvn	0(1,%r0), 0(%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: mvn	0(1,%r1), 0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: mvn	0(%r1,%r2), 0(%r1)
 #CHECK: error: invalid use of indexed addressing
@@ -4796,8 +4720,6 @@
 	mvn	4096(1,%r1), 0(%r1)
 	mvn	0(1,%r1), -1(%r1)
 	mvn	0(1,%r1), 4096(%r1)
-	mvn	0(1,%r0), 0(%r1)
-	mvn	0(1,%r1), 0(%r0)
 	mvn	0(%r1,%r2), 0(%r1)
 	mvn	0(1,%r2), 0(%r1,%r2)
 	mvn	0(-), 0
@@ -4826,10 +4748,6 @@
 #CHECK: mvo	0(1,%r1), -1(1,%r1)
 #CHECK: error: invalid operand
 #CHECK: mvo	0(1,%r1), 4096(1,%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: mvo	0(1,%r0), 0(1,%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: mvo	0(1,%r1), 0(1,%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: mvo	0(%r1,%r2), 0(1,%r1)
 #CHECK: error: invalid use of indexed addressing
@@ -4849,8 +4767,6 @@
 	mvo	4096(1,%r1), 0(1,%r1)
 	mvo	0(1,%r1), -1(1,%r1)
 	mvo	0(1,%r1), 4096(1,%r1)
-	mvo	0(1,%r0), 0(1,%r1)
-	mvo	0(1,%r1), 0(1,%r0)
 	mvo	0(%r1,%r2), 0(1,%r1)
 	mvo	0(1,%r2), 0(%r1,%r2)
 	mvo	0(-), 0(1)
@@ -4873,10 +4789,6 @@
 #CHECK: mvz	0(1,%r1), -1(%r1)
 #CHECK: error: invalid operand
 #CHECK: mvz	0(1,%r1), 4096(%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: mvz	0(1,%r0), 0(%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: mvz	0(1,%r1), 0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: mvz	0(%r1,%r2), 0(%r1)
 #CHECK: error: invalid use of indexed addressing
@@ -4893,8 +4805,6 @@
 	mvz	4096(1,%r1), 0(%r1)
 	mvz	0(1,%r1), -1(%r1)
 	mvz	0(1,%r1), 4096(%r1)
-	mvz	0(1,%r0), 0(%r1)
-	mvz	0(1,%r1), 0(%r0)
 	mvz	0(%r1,%r2), 0(%r1)
 	mvz	0(1,%r2), 0(%r1,%r2)
 	mvz	0(-), 0
@@ -5021,10 +4931,6 @@
 #CHECK: nc	0(1,%r1), -1(%r1)
 #CHECK: error: invalid operand
 #CHECK: nc	0(1,%r1), 4096(%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: nc	0(1,%r0), 0(%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: nc	0(1,%r1), 0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: nc	0(%r1,%r2), 0(%r1)
 #CHECK: error: invalid use of indexed addressing
@@ -5041,8 +4947,6 @@
 	nc	4096(1,%r1), 0(%r1)
 	nc	0(1,%r1), -1(%r1)
 	nc	0(1,%r1), 4096(%r1)
-	nc	0(1,%r0), 0(%r1)
-	nc	0(1,%r1), 0(%r0)
 	nc	0(%r1,%r2), 0(%r1)
 	nc	0(1,%r2), 0(%r1,%r2)
 	nc	0(-), 0
@@ -5181,10 +5085,6 @@
 #CHECK: oc	0(1,%r1), -1(%r1)
 #CHECK: error: invalid operand
 #CHECK: oc	0(1,%r1), 4096(%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: oc	0(1,%r0), 0(%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: oc	0(1,%r1), 0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: oc	0(%r1,%r2), 0(%r1)
 #CHECK: error: invalid use of indexed addressing
@@ -5201,8 +5101,6 @@
 	oc	4096(1,%r1), 0(%r1)
 	oc	0(1,%r1), -1(%r1)
 	oc	0(1,%r1), 4096(%r1)
-	oc	0(1,%r0), 0(%r1)
-	oc	0(1,%r1), 0(%r0)
 	oc	0(%r1,%r2), 0(%r1)
 	oc	0(1,%r2), 0(%r1,%r2)
 	oc	0(-), 0
@@ -5339,10 +5237,6 @@
 #CHECK: pack	0(1,%r1), -1(1,%r1)
 #CHECK: error: invalid operand
 #CHECK: pack	0(1,%r1), 4096(1,%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: pack	0(1,%r0), 0(1,%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: pack	0(1,%r1), 0(1,%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: pack	0(%r1,%r2), 0(1,%r1)
 #CHECK: error: invalid use of indexed addressing
@@ -5362,8 +5256,6 @@
 	pack	4096(1,%r1), 0(1,%r1)
 	pack	0(1,%r1), -1(1,%r1)
 	pack	0(1,%r1), 4096(1,%r1)
-	pack	0(1,%r0), 0(1,%r1)
-	pack	0(1,%r1), 0(1,%r0)
 	pack	0(%r1,%r2), 0(1,%r1)
 	pack	0(1,%r2), 0(%r1,%r2)
 	pack	0(-), 0(1)
@@ -5441,10 +5333,6 @@
 #CHECK: pka	0(%r1), -1(1,%r1)
 #CHECK: error: invalid operand
 #CHECK: pka	0(%r1), 4096(1,%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: pka	0(%r0), 0(1,%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: pka	0(%r1), 0(1,%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: pka	0(%r1,%r2), 0(1,%r1)
 #CHECK: error: invalid use of indexed addressing
@@ -5461,8 +5349,6 @@
 	pka	4096(%r1), 0(1,%r1)
 	pka	0(%r1), -1(1,%r1)
 	pka	0(%r1), 4096(1,%r1)
-	pka	0(%r0), 0(1,%r1)
-	pka	0(%r1), 0(1,%r0)
 	pka	0(%r1,%r2), 0(1,%r1)
 	pka	0(%r2), 0(%r1,%r2)
 	pka	0, 0(-)
@@ -5485,10 +5371,6 @@
 #CHECK: pku	0(%r1), -1(1,%r1)
 #CHECK: error: invalid operand
 #CHECK: pku	0(%r1), 4096(1,%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: pku	0(%r0), 0(1,%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: pku	0(%r1), 0(1,%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: pku	0(%r1,%r2), 0(1,%r1)
 #CHECK: error: invalid use of indexed addressing
@@ -5618,28 +5500,22 @@
 #CHECK: rll	%r0,%r0,-524289
 #CHECK: error: invalid operand
 #CHECK: rll	%r0,%r0,524288
-#CHECK: error: %r0 used in an address
-#CHECK: rll	%r0,%r0,0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: rll	%r0,%r0,0(%r1,%r2)
 
 	rll	%r0,%r0,-524289
 	rll	%r0,%r0,524288
-	rll	%r0,%r0,0(%r0)
 	rll	%r0,%r0,0(%r1,%r2)
 
 #CHECK: error: invalid operand
 #CHECK: rllg	%r0,%r0,-524289
 #CHECK: error: invalid operand
 #CHECK: rllg	%r0,%r0,524288
-#CHECK: error: %r0 used in an address
-#CHECK: rllg	%r0,%r0,0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: rllg	%r0,%r0,0(%r1,%r2)
 
 	rllg	%r0,%r0,-524289
 	rllg	%r0,%r0,524288
-	rllg	%r0,%r0,0(%r0)
 	rllg	%r0,%r0,0(%r1,%r2)
 
 #CHECK: error: invalid operand
@@ -5924,28 +5800,22 @@
 #CHECK: sla	%r0,-1
 #CHECK: error: invalid operand
 #CHECK: sla	%r0,4096
-#CHECK: error: %r0 used in an address
-#CHECK: sla	%r0,0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: sla	%r0,0(%r1,%r2)
 
 	sla	%r0,-1
 	sla	%r0,4096
-	sla	%r0,0(%r0)
 	sla	%r0,0(%r1,%r2)
 
 #CHECK: error: invalid operand
 #CHECK: slag	%r0,%r0,-524289
 #CHECK: error: invalid operand
 #CHECK: slag	%r0,%r0,524288
-#CHECK: error: %r0 used in an address
-#CHECK: slag	%r0,%r0,0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: slag	%r0,%r0,0(%r1,%r2)
 
 	slag	%r0,%r0,-524289
 	slag	%r0,%r0,524288
-	slag	%r0,%r0,0(%r0)
 	slag	%r0,%r0,0(%r1,%r2)
 
 #CHECK: error: instruction requires: distinct-ops
@@ -5975,15 +5845,12 @@
 #CHECK: slda	%r0,-1
 #CHECK: error: invalid operand
 #CHECK: slda	%r0,4096
-#CHECK: error: %r0 used in an address
-#CHECK: slda	%r0,0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: slda	%r0,0(%r1,%r2)
 
 	slda	%r1,0
 	slda	%r0,-1
 	slda	%r0,4096
-	slda	%r0,0(%r0)
 	slda	%r0,0(%r1,%r2)
 
 #CHECK: error: invalid register pair
@@ -5992,15 +5859,12 @@
 #CHECK: sldl	%r0,-1
 #CHECK: error: invalid operand
 #CHECK: sldl	%r0,4096
-#CHECK: error: %r0 used in an address
-#CHECK: sldl	%r0,0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: sldl	%r0,0(%r1,%r2)
 
 	sldl	%r1,0
 	sldl	%r0,-1
 	sldl	%r0,4096
-	sldl	%r0,0(%r0)
 	sldl	%r0,0(%r1,%r2)
 
 #CHECK: error: invalid operand
@@ -6062,28 +5926,22 @@
 #CHECK: sll	%r0,-1
 #CHECK: error: invalid operand
 #CHECK: sll	%r0,4096
-#CHECK: error: %r0 used in an address
-#CHECK: sll	%r0,0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: sll	%r0,0(%r1,%r2)
 
 	sll	%r0,-1
 	sll	%r0,4096
-	sll	%r0,0(%r0)
 	sll	%r0,0(%r1,%r2)
 
 #CHECK: error: invalid operand
 #CHECK: sllg	%r0,%r0,-524289
 #CHECK: error: invalid operand
 #CHECK: sllg	%r0,%r0,524288
-#CHECK: error: %r0 used in an address
-#CHECK: sllg	%r0,%r0,0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: sllg	%r0,%r0,0(%r1,%r2)
 
 	sllg	%r0,%r0,-524289
 	sllg	%r0,%r0,524288
-	sllg	%r0,%r0,0(%r0)
 	sllg	%r0,%r0,0(%r1,%r2)
 
 #CHECK: error: instruction requires: distinct-ops
@@ -6142,10 +6000,6 @@
 #CHECK: sp	0(1,%r1), -1(1,%r1)
 #CHECK: error: invalid operand
 #CHECK: sp	0(1,%r1), 4096(1,%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: sp	0(1,%r0), 0(1,%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: sp	0(1,%r1), 0(1,%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: sp	0(%r1,%r2), 0(1,%r1)
 #CHECK: error: invalid use of indexed addressing
@@ -6165,8 +6019,6 @@
 	sp	4096(1,%r1), 0(1,%r1)
 	sp	0(1,%r1), -1(1,%r1)
 	sp	0(1,%r1), 4096(1,%r1)
-	sp	0(1,%r0), 0(1,%r1)
-	sp	0(1,%r1), 0(1,%r0)
 	sp	0(%r1,%r2), 0(1,%r1)
 	sp	0(1,%r2), 0(%r1,%r2)
 	sp	0(-), 0(1)
@@ -6256,8 +6108,6 @@
 #CHECK: sra	%r0,-1
 #CHECK: error: invalid operand
 #CHECK: sra	%r0,4096
-#CHECK: error: %r0 used in an address
-#CHECK: sra	%r0,0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: sra	%r0,0(%r1,%r2)
 
@@ -6270,8 +6120,6 @@
 #CHECK: srag	%r0,%r0,-524289
 #CHECK: error: invalid operand
 #CHECK: srag	%r0,%r0,524288
-#CHECK: error: %r0 used in an address
-#CHECK: srag	%r0,%r0,0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: srag	%r0,%r0,0(%r1,%r2)
 
@@ -6291,15 +6139,12 @@
 #CHECK: srda	%r0,-1
 #CHECK: error: invalid operand
 #CHECK: srda	%r0,4096
-#CHECK: error: %r0 used in an address
-#CHECK: srda	%r0,0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: srda	%r0,0(%r1,%r2)
 
 	srda	%r1,0
 	srda	%r0,-1
 	srda	%r0,4096
-	srda	%r0,0(%r0)
 	srda	%r0,0(%r1,%r2)
 
 #CHECK: error: invalid register pair
@@ -6308,15 +6153,12 @@
 #CHECK: srdl	%r0,-1
 #CHECK: error: invalid operand
 #CHECK: srdl	%r0,4096
-#CHECK: error: %r0 used in an address
-#CHECK: srdl	%r0,0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: srdl	%r0,0(%r1,%r2)
 
 	srdl	%r1,0
 	srdl	%r0,-1
 	srdl	%r0,4096
-	srdl	%r0,0(%r0)
 	srdl	%r0,0(%r1,%r2)
 
 #CHECK: error: invalid operand
@@ -6336,28 +6178,22 @@
 #CHECK: srl	%r0,-1
 #CHECK: error: invalid operand
 #CHECK: srl	%r0,4096
-#CHECK: error: %r0 used in an address
-#CHECK: srl	%r0,0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: srl	%r0,0(%r1,%r2)
 
 	srl	%r0,-1
 	srl	%r0,4096
-	srl	%r0,0(%r0)
 	srl	%r0,0(%r1,%r2)
 
 #CHECK: error: invalid operand
 #CHECK: srlg	%r0,%r0,-524289
 #CHECK: error: invalid operand
 #CHECK: srlg	%r0,%r0,524288
-#CHECK: error: %r0 used in an address
-#CHECK: srlg	%r0,%r0,0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: srlg	%r0,%r0,0(%r1,%r2)
 
 	srlg	%r0,%r0,-524289
 	srlg	%r0,%r0,524288
-	srlg	%r0,%r0,0(%r0)
 	srlg	%r0,%r0,0(%r1,%r2)
 
 #CHECK: error: instruction requires: distinct-ops
@@ -6410,10 +6246,6 @@
 #CHECK: srp	0(1,%r1), -1(%r1), 0
 #CHECK: error: invalid operand
 #CHECK: srp	0(1,%r1), 4096(%r1), 0
-#CHECK: error: %r0 used in an address
-#CHECK: srp	0(1,%r0), 0(%r1), 0
-#CHECK: error: %r0 used in an address
-#CHECK: srp	0(1,%r1), 0(%r0), 0
 #CHECK: error: invalid use of indexed addressing
 #CHECK: srp	0(%r1,%r2), 0(%r1), 0
 #CHECK: error: invalid use of indexed addressing
@@ -6434,8 +6266,6 @@
 	srp	4096(1,%r1), 0(%r1), 0
 	srp	0(1,%r1), -1(%r1), 0
 	srp	0(1,%r1), 4096(%r1), 0
-	srp	0(1,%r0), 0(%r1), 0
-	srp	0(1,%r1), 0(%r0), 0
 	srp	0(%r1,%r2), 0(%r1), 0
 	srp	0(1,%r2), 0(%r1,%r2), 0
 	srp	0(1), 0, -1
@@ -7242,8 +7072,6 @@
 #CHECK: tp	-1(1,%r1)
 #CHECK: error: invalid operand
 #CHECK: tp	4096(1,%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: tp	0(1,%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: tp	0(%r1,%r2)
 #CHECK: error: unknown token in expression
@@ -7255,7 +7083,6 @@
 	tp	0(17,%r1)
 	tp	-1(1,%r1)
 	tp	4096(1,%r1)
-	tp	0(1,%r0)
 	tp	0(%r1,%r2)
 	tp	0(-)
 
@@ -7305,10 +7132,6 @@
 #CHECK: tr	0(1,%r1), -1(%r1)
 #CHECK: error: invalid operand
 #CHECK: tr	0(1,%r1), 4096(%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: tr	0(1,%r0), 0(%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: tr	0(1,%r1), 0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: tr	0(%r1,%r2), 0(%r1)
 #CHECK: error: invalid use of indexed addressing
@@ -7325,8 +7148,6 @@
 	tr	4096(1,%r1), 0(%r1)
 	tr	0(1,%r1), -1(%r1)
 	tr	0(1,%r1), 4096(%r1)
-	tr	0(1,%r0), 0(%r1)
-	tr	0(1,%r1), 0(%r0)
 	tr	0(%r1,%r2), 0(%r1)
 	tr	0(1,%r2), 0(%r1,%r2)
 	tr	0(-), 0
@@ -7409,10 +7230,6 @@
 #CHECK: trt	0(1,%r1), -1(%r1)
 #CHECK: error: invalid operand
 #CHECK: trt	0(1,%r1), 4096(%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: trt	0(1,%r0), 0(%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: trt	0(1,%r1), 0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: trt	0(%r1,%r2), 0(%r1)
 #CHECK: error: invalid use of indexed addressing
@@ -7429,8 +7246,6 @@
 	trt	4096(1,%r1), 0(%r1)
 	trt	0(1,%r1), -1(%r1)
 	trt	0(1,%r1), 4096(%r1)
-	trt	0(1,%r0), 0(%r1)
-	trt	0(1,%r1), 0(%r0)
 	trt	0(%r1,%r2), 0(%r1)
 	trt	0(1,%r2), 0(%r1,%r2)
 	trt	0(-), 0
@@ -7475,10 +7290,6 @@
 #CHECK: trtr	0(1,%r1), -1(%r1)
 #CHECK: error: invalid operand
 #CHECK: trtr	0(1,%r1), 4096(%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: trtr	0(1,%r0), 0(%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: trtr	0(1,%r1), 0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: trtr	0(%r1,%r2), 0(%r1)
 #CHECK: error: invalid use of indexed addressing
@@ -7495,8 +7306,6 @@
 	trtr	4096(1,%r1), 0(%r1)
 	trtr	0(1,%r1), -1(%r1)
 	trtr	0(1,%r1), 4096(%r1)
-	trtr	0(1,%r0), 0(%r1)
-	trtr	0(1,%r1), 0(%r0)
 	trtr	0(%r1,%r2), 0(%r1)
 	trtr	0(1,%r2), 0(%r1,%r2)
 	trtr	0(-), 0
@@ -7569,10 +7378,6 @@
 #CHECK: unpk	0(1,%r1), -1(1,%r1)
 #CHECK: error: invalid operand
 #CHECK: unpk	0(1,%r1), 4096(1,%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: unpk	0(1,%r0), 0(1,%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: unpk	0(1,%r1), 0(1,%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: unpk	0(%r1,%r2), 0(1,%r1)
 #CHECK: error: invalid use of indexed addressing
@@ -7592,8 +7397,6 @@
 	unpk	4096(1,%r1), 0(1,%r1)
 	unpk	0(1,%r1), -1(1,%r1)
 	unpk	0(1,%r1), 4096(1,%r1)
-	unpk	0(1,%r0), 0(1,%r1)
-	unpk	0(1,%r1), 0(1,%r0)
 	unpk	0(%r1,%r2), 0(1,%r1)
 	unpk	0(1,%r2), 0(%r1,%r2)
 	unpk	0(-), 0(1)
@@ -7616,10 +7419,6 @@
 #CHECK: unpka	0(1,%r1), -1(%r1)
 #CHECK: error: invalid operand
 #CHECK: unpka	0(1,%r1), 4096(%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: unpka	0(1,%r0), 0(%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: unpka	0(1,%r1), 0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: unpka	0(%r1,%r2), 0(%r1)
 #CHECK: error: invalid use of indexed addressing
@@ -7636,8 +7435,6 @@
 	unpka	4096(1,%r1), 0(%r1)
 	unpka	0(1,%r1), -1(%r1)
 	unpka	0(1,%r1), 4096(%r1)
-	unpka	0(1,%r0), 0(%r1)
-	unpka	0(1,%r1), 0(%r0)
 	unpka	0(%r1,%r2), 0(%r1)
 	unpka	0(1,%r2), 0(%r1,%r2)
 	unpka	0(-), 0
@@ -7660,10 +7457,6 @@
 #CHECK: unpku	0(1,%r1), -1(%r1)
 #CHECK: error: invalid operand
 #CHECK: unpku	0(1,%r1), 4096(%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: unpku	0(1,%r0), 0(%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: unpku	0(1,%r1), 0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: unpku	0(%r1,%r2), 0(%r1)
 #CHECK: error: invalid use of indexed addressing
@@ -7680,8 +7473,6 @@
 	unpku	4096(1,%r1), 0(%r1)
 	unpku	0(1,%r1), -1(%r1)
 	unpku	0(1,%r1), 4096(%r1)
-	unpku	0(1,%r0), 0(%r1)
-	unpku	0(1,%r1), 0(%r0)
 	unpku	0(%r1,%r2), 0(%r1)
 	unpku	0(1,%r2), 0(%r1,%r2)
 	unpku	0(-), 0
@@ -7712,10 +7503,6 @@
 #CHECK: xc	0(1,%r1), -1(%r1)
 #CHECK: error: invalid operand
 #CHECK: xc	0(1,%r1), 4096(%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: xc	0(1,%r0), 0(%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: xc	0(1,%r1), 0(%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: xc	0(%r1,%r2), 0(%r1)
 #CHECK: error: invalid use of indexed addressing
@@ -7732,8 +7519,6 @@
 	xc	4096(1,%r1), 0(%r1)
 	xc	0(1,%r1), -1(%r1)
 	xc	0(1,%r1), 4096(%r1)
-	xc	0(1,%r0), 0(%r1)
-	xc	0(1,%r1), 0(%r0)
 	xc	0(%r1,%r2), 0(%r1)
 	xc	0(1,%r2), 0(%r1,%r2)
 	xc	0(-), 0
@@ -7838,10 +7623,6 @@
 #CHECK: zap	0(1,%r1), -1(1,%r1)
 #CHECK: error: invalid operand
 #CHECK: zap	0(1,%r1), 4096(1,%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: zap	0(1,%r0), 0(1,%r1)
-#CHECK: error: %r0 used in an address
-#CHECK: zap	0(1,%r1), 0(1,%r0)
 #CHECK: error: invalid use of indexed addressing
 #CHECK: zap	0(%r1,%r2), 0(1,%r1)
 #CHECK: error: invalid use of indexed addressing
@@ -7861,8 +7642,6 @@
 	zap	4096(1,%r1), 0(1,%r1)
 	zap	0(1,%r1), -1(1,%r1)
 	zap	0(1,%r1), 4096(1,%r1)
-	zap	0(1,%r0), 0(1,%r1)
-	zap	0(1,%r1), 0(1,%r0)
 	zap	0(%r1,%r2), 0(1,%r1)
 	zap	0(1,%r2), 0(%r1,%r2)
 	zap	0(-), 0(1)
diff --git a/llvm/test/MC/SystemZ/insn-good.s b/llvm/test/MC/SystemZ/insn-good.s
index 5b93ef917fd32..3c5b34abe0a9d 100644
--- a/llvm/test/MC/SystemZ/insn-good.s
+++ b/llvm/test/MC/SystemZ/insn-good.s
@@ -765,12 +765,12 @@
 	bal	%r14, 4095(%r1,%r15)
 	bal	%r15, 4095(%r15,%r1)
 
-#CHECK: balr	%r0, %r1                # encoding: [0x05,0x01]
+#CHECK: balr	%r0, %r0                # encoding: [0x05,0x00]
 #CHECK: balr	%r0, %r15               # encoding: [0x05,0x0f]
 #CHECK: balr	%r14, %r9               # encoding: [0x05,0xe9]
 #CHECK: balr	%r15, %r1               # encoding: [0x05,0xf1]
 
-	balr	%r0,%r1
+	balr	%r0,%r0
 	balr	%r0,%r15
 	balr	%r14,%r9
 	balr	%r15,%r1
@@ -789,22 +789,22 @@
 	bas	%r14, 4095(%r1,%r15)
 	bas	%r15, 4095(%r15,%r1)
 
-#CHECK: basr	%r0, %r1                # encoding: [0x0d,0x01]
+#CHECK: basr	%r0, %r0                # encoding: [0x0d,0x00]
 #CHECK: basr	%r0, %r15               # encoding: [0x0d,0x0f]
 #CHECK: basr	%r14, %r9               # encoding: [0x0d,0xe9]
 #CHECK: basr	%r15, %r1               # encoding: [0x0d,0xf1]
 
-	basr	%r0,%r1
+	basr	%r0,%r0
 	basr	%r0,%r15
 	basr	%r14,%r9
 	basr	%r15,%r1
 
-#CHECK: bassm	%r0, %r1                # encoding: [0x0c,0x01]
+#CHECK: bassm	%r0, %r0                # encoding: [0x0c,0x00]
 #CHECK: bassm	%r0, %r15               # encoding: [0x0c,0x0f]
 #CHECK: bassm	%r14, %r9               # encoding: [0x0c,0xe9]
 #CHECK: bassm	%r15, %r1               # encoding: [0x0c,0xf1]
 
-	bassm	%r0,%r1
+	bassm	%r0,%r0
 	bassm	%r0,%r15
 	bassm	%r14,%r9
 	bassm	%r15,%r1
@@ -829,12 +829,12 @@
 	bsg	%r15,%r0
 	bsg	%r7,%r8
 
-#CHECK: bsm	%r0, %r1                # encoding: [0x0b,0x01]
+#CHECK: bsm	%r0, %r0                # encoding: [0x0b,0x00]
 #CHECK: bsm	%r0, %r15               # encoding: [0x0b,0x0f]
 #CHECK: bsm	%r14, %r9               # encoding: [0x0b,0xe9]
 #CHECK: bsm	%r15, %r1               # encoding: [0x0b,0xf1]
 
-	bsm	%r0,%r1
+	bsm	%r0,%r0
 	bsm	%r0,%r15
 	bsm	%r14,%r9
 	bsm	%r15,%r1
@@ -960,95 +960,125 @@
 	bcr	0, %r15
 
 #CHECK:	bcr	1, %r7			# encoding: [0x07,0x17]
+#CHECK:	bor	%r0			# encoding: [0x07,0x10]
 #CHECK:	bor	%r15			# encoding: [0x07,0x1f]
 
 	bcr	1, %r7
+	bor	%r0
 	bor	%r15
 
 #CHECK:	bcr	2, %r7			# encoding: [0x07,0x27]
+#CHECK:	bhr	%r0			# encoding: [0x07,0x20]
 #CHECK:	bhr	%r15			# encoding: [0x07,0x2f]
 
 	bcr	2, %r7
+	bhr	%r0
 	bhr	%r15
 
 #CHECK:	bcr	3, %r7			# encoding: [0x07,0x37]
+#CHECK:	bnler	%r0			# encoding: [0x07,0x30]
 #CHECK:	bnler	%r15			# encoding: [0x07,0x3f]
 
 	bcr	3, %r7
+	bnler	%r0
 	bnler	%r15
 
 #CHECK:	bcr	4, %r7			# encoding: [0x07,0x47]
+#CHECK:	blr	%r0			# encoding: [0x07,0x40]
 #CHECK:	blr	%r15			# encoding: [0x07,0x4f]
 
 	bcr	4, %r7
+	blr	%r0
 	blr	%r15
 
 #CHECK:	bcr	5, %r7			# encoding: [0x07,0x57]
+#CHECK:	bnher	%r0			# encoding: [0x07,0x50]
 #CHECK:	bnher	%r15			# encoding: [0x07,0x5f]
 
 	bcr	5, %r7
+	bnher	%r0
 	bnher	%r15
 
 #CHECK:	bcr	6, %r7			# encoding: [0x07,0x67]
+#CHECK:	blhr	%r0			# encoding: [0x07,0x60]
 #CHECK:	blhr	%r15			# encoding: [0x07,0x6f]
 
 	bcr	6, %r7
+	blhr	%r0
 	blhr	%r15
 
 #CHECK:	bcr	7, %r7			# encoding: [0x07,0x77]
+#CHECK:	bner	%r0			# encoding: [0x07,0x70]
 #CHECK:	bner	%r15			# encoding: [0x07,0x7f]
 
 	bcr	7, %r7
+	bner	%r0
 	bner	%r15
 
 #CHECK:	bcr	8, %r7			# encoding: [0x07,0x87]
+#CHECK:	ber	%r0			# encoding: [0x07,0x80]
 #CHECK:	ber	%r15			# encoding: [0x07,0x8f]
 
 	bcr	8, %r7
+	ber	%r0
 	ber	%r15
 
 #CHECK:	bcr	9, %r7			# encoding: [0x07,0x97]
+#CHECK:	bnlhr	%r0			# encoding: [0x07,0x90]
 #CHECK:	bnlhr	%r15			# encoding: [0x07,0x9f]
 
 	bcr	9, %r7
+	bnlhr	%r0
 	bnlhr	%r15
 
 #CHECK:	bcr	10, %r7			# encoding: [0x07,0xa7]
+#CHECK:	bher	%r0			# encoding: [0x07,0xa0]
 #CHECK:	bher	%r15			# encoding: [0x07,0xaf]
 
 	bcr	10, %r7
+	bher	%r0
 	bher	%r15
 
 #CHECK:	bcr	11, %r7			# encoding: [0x07,0xb7]
+#CHECK:	bnlr	%r0			# encoding: [0x07,0xb0]
 #CHECK:	bnlr	%r15			# encoding: [0x07,0xbf]
 
 	bcr	11, %r7
+	bnlr	%r0
 	bnlr	%r15
 
 #CHECK:	bcr	12, %r7			# encoding: [0x07,0xc7]
+#CHECK:	bler	%r0			# encoding: [0x07,0xc0]
 #CHECK:	bler	%r15			# encoding: [0x07,0xcf]
 
 	bcr	12, %r7
+	bler	%r0
 	bler	%r15
 
 #CHECK:	bcr	13, %r7			# encoding: [0x07,0xd7]
+#CHECK:	bnhr	%r0			# encoding: [0x07,0xd0]
 #CHECK:	bnhr	%r15			# encoding: [0x07,0xdf]
 
 	bcr	13, %r7
+	bnhr	%r0
 	bnhr	%r15
 
 #CHECK:	bcr	14, %r7			# encoding: [0x07,0xe7]
+#CHECK:	bnor	%r0			# encoding: [0x07,0xe0]
 #CHECK:	bnor	%r15			# encoding: [0x07,0xef]
 
 	bcr	14, %r7
+	bnor	%r0
 	bnor	%r15
 
 #CHECK:	bcr	15, %r7			# encoding: [0x07,0xf7]
+#CHECK: br	%r0                     # encoding: [0x07,0xf0]
 #CHECK: br	%r1                     # encoding: [0x07,0xf1]
 #CHECK: br	%r14                    # encoding: [0x07,0xfe]
 #CHECK: br	%r15                    # encoding: [0x07,0xff]
 
 	bcr	15, %r7
+	br	%r0
 	br	%r1
 	br	%r14
 	br	%r15
@@ -8002,17 +8032,23 @@
 
 #CHECK: l	%r0, 0                  # encoding: [0x58,0x00,0x00,0x00]
 #CHECK: l	%r0, 4095               # encoding: [0x58,0x00,0x0f,0xff]
+#CHECK: l	%r0, 0(%r0)             # encoding: [0x58,0x00,0x00,0x00]
 #CHECK: l	%r0, 0(%r1)             # encoding: [0x58,0x00,0x10,0x00]
 #CHECK: l	%r0, 0(%r15)            # encoding: [0x58,0x00,0xf0,0x00]
+#CHECK: l	%r0, 4095(%r0,%r15)     # encoding: [0x58,0x00,0xff,0xff]
 #CHECK: l	%r0, 4095(%r1,%r15)     # encoding: [0x58,0x01,0xff,0xff]
+#CHECK: l	%r0, 4095(%r15,%r0)     # encoding: [0x58,0x0f,0x0f,0xff]
 #CHECK: l	%r0, 4095(%r15,%r1)     # encoding: [0x58,0x0f,0x1f,0xff]
 #CHECK: l	%r15, 0                 # encoding: [0x58,0xf0,0x00,0x00]
 
 	l	%r0, 0
 	l	%r0, 4095
+	l	%r0, 0(%r0)
 	l	%r0, 0(%r1)
 	l	%r0, 0(%r15)
+	l	%r0, 4095(%r0,%r15)
 	l	%r0, 4095(%r1,%r15)
+	l	%r0, 4095(%r15,%r0)
 	l	%r0, 4095(%r15,%r1)
 	l	%r15, 0
 
diff --git a/llvm/test/MC/SystemZ/regs-bad.s b/llvm/test/MC/SystemZ/regs-bad.s
index f4cdb69821d2e..c4d16ef4cc3d6 100644
--- a/llvm/test/MC/SystemZ/regs-bad.s
+++ b/llvm/test/MC/SystemZ/regs-bad.s
@@ -297,20 +297,3 @@
 	.cfi_offset %arid,0
 
 	.cfi_endproc
-
-#CHECK: error: %r0 used in an address
-#CHECK: sll	%r2,8(%r0)
-#CHECK: error: %r0 used in an address
-#CHECK: br	%r0
-#CHECK: error: %r0 used in an address
-#CHECK: l	%r1,8(%r0)
-#CHECK: error: %r0 used in an address
-#CHECK: l	%r1,8(%r0,%r15)
-#CHECK: error: %r0 used in an address
-#CHECK: l	%r1,8(%r15,%r0)
-
-	sll	%r2,8(%r0)
-	br	%r0
-	l	%r1,8(%r0)
-	l	%r1,8(%r0,%r15)
-	l	%r1,8(%r15,%r0)
diff --git a/llvm/test/MC/SystemZ/tokens.s b/llvm/test/MC/SystemZ/tokens.s
index 7cea21a1e5c52..177378615ed42 100644
--- a/llvm/test/MC/SystemZ/tokens.s
+++ b/llvm/test/MC/SystemZ/tokens.s
@@ -9,24 +9,18 @@
 #CHECK: foo	100(200), 300
 #CHECK: error: register expected
 #CHECK: foo	100(200,), 300
-#CHECK: error: %r0 used in an address
-#CHECK: foo	100(200,%r0), 300
 #CHECK: error: invalid instruction
 #CHECK: foo	100(200,%r1), 300
 #CHECK: error: invalid address register
 #CHECK: foo	100(%a0), 200
 #CHECK: error: invalid instruction
 #CHECK: foo	100(%r0), 200
-#CHECK: error: %r0 used in an address
-#CHECK: foo	100(%v1,%r0), 200
 #CHECK: error: invalid instruction
 #CHECK: foo	100(%v0,%r1), 200
 #CHECK: error: invalid instruction
 #CHECK: foo	100(%v31), 200
 #CHECK: error: invalid address register
 #CHECK: foo	100(%r1,%a0), 200
-#CHECK: error: %r0 used in an address
-#CHECK: foo	100(%r1,%r0), 200
 #CHECK: error: unexpected token in address
 #CHECK: foo	100(%r1,%r2, 200
 #CHECK: error: invalid instruction
@@ -68,15 +62,12 @@
 	foo	100(, 200
 	foo	100(200), 300
 	foo	100(200,), 300
-	foo	100(200,%r0), 300
 	foo	100(200,%r1), 300
 	foo	100(%a0), 200
 	foo	100(%r0), 200
-	foo	100(%v1,%r0), 200
 	foo	100(%v0,%r1), 200
 	foo	100(%v31), 200
 	foo	100(%r1,%a0), 200
-	foo	100(%r1,%r0), 200
 	foo	100(%r1,%r2, 200
 	foo	100(%r1,%r2), 200
 	foo	100(%r1,%r2)(, 200
diff --git a/llvm/test/MC/X86/AlignedBundling/misaligned-bundle-group.s b/llvm/test/MC/X86/AlignedBundling/misaligned-bundle-group.s
index 3c86caa00ccef..6962a2a65960d 100644
--- a/llvm/test/MC/X86/AlignedBundling/misaligned-bundle-group.s
+++ b/llvm/test/MC/X86/AlignedBundling/misaligned-bundle-group.s
@@ -16,8 +16,8 @@ foo:
 # CHECK-RELAX:      1a: nop
 # CHECK-RELAX:      20: nopw %cs:(%eax,%eax)
 # CHECK-RELAX:      2a: nopw %cs:(%eax,%eax)
-# CHECK-OPT:        1b: calll -4
-# CHECK-RELAX:      3b: calll -4
+# CHECK-OPT:        1b: calll 0x1c
+# CHECK-RELAX:      3b: calll 0x3c
         calll   bar # 5 bytes
         .bundle_unlock
         ret         # 1 byte
diff --git a/llvm/test/MC/X86/AlignedBundling/single-inst-bundling.s b/llvm/test/MC/X86/AlignedBundling/single-inst-bundling.s
index f15a2b34d1847..bb2efa8d16466 100644
--- a/llvm/test/MC/X86/AlignedBundling/single-inst-bundling.s
+++ b/llvm/test/MC/X86/AlignedBundling/single-inst-bundling.s
@@ -30,8 +30,8 @@ foo:
   jle     .L_ELSE
 # Due to the padding that's inserted before the addl, the jump target
 # becomes farther by one byte.
-# CHECK-OPT:     jle 5
-# CHECK-RELAX:   jle 7
+# CHECK-OPT:     jle 0x24
+# CHECK-RELAX:   jle 0x2d
 
   addl    %ebp, %eax
 # CHECK-OPT:     nop
diff --git a/llvm/test/MC/X86/align-branch-64-basic.s b/llvm/test/MC/X86/align-branch-64-basic.s
index 0b056b320a206..7186b6bc44249 100644
--- a/llvm/test/MC/X86/align-branch-64-basic.s
+++ b/llvm/test/MC/X86/align-branch-64-basic.s
@@ -103,32 +103,14 @@ test_indirect:
 bar:
   retq
 
-  # CHECK: <test_pad_via_relax>:
-  # CHECK: 200: testq
-  # CHECK: 203: jne
-  # CHECK: 209: int3
-  # note 6 byte jne which could be a 2 byte jne, but is instead
-  # expanded for padding purposes
-  # CHECK-NOT: nop
-  # CHECK: 220: callq
-  .global test_pad_via_relax
-  .p2align  5
-test_pad_via_relax:
-  testq %rax, %rax
-  jnz bar
-  .rept 23
-  int3
-  .endr
-  callq bar
-
   # This case looks really tempting to pad, but doing so for the call causes
   # the jmp to be misaligned.
   # CHECK: <test_pad_via_relax_neg1>:
-  # CHECK: 240: int3
-  # CHECK: 25a: testq
-  # CHECK: 25d: jne
-  # CHECK: 25f: nop
-  # CHECK: 260: callq
+  # CHECK: 200: int3
+  # CHECK: 21a: testq
+  # CHECK: 21d: jne
+  # CHECK: 21f: nop
+  # CHECK: 220: callq
   .global test_pad_via_relax_neg1
   .p2align  5
 test_pad_via_relax_neg1:
@@ -141,10 +123,10 @@ test_pad_via_relax_neg1:
 
   # Same as previous, but without fusion
   # CHECK: <test_pad_via_relax_neg2>:
-  # CHECK: 280: int3
-  # CHECK: 29d: jmp
-  # CHECK: 29f: nop
-  # CHECK: 2a0: callq
+  # CHECK: 240: int3
+  # CHECK: 25d: jmp
+  # CHECK: 25f: nop
+  # CHECK: 260: callq
   .global test_pad_via_relax_neg2
   .p2align  5
 test_pad_via_relax_neg2:
@@ -154,7 +136,7 @@ test_pad_via_relax_neg2:
   jmp bar2
   callq bar2
 
-bar2: 
+bar2:
 
   .section "unknown"
   .p2align 4
diff --git a/llvm/test/MC/X86/align-branch-64-pad-max-prefix.s b/llvm/test/MC/X86/align-branch-64-pad-max-prefix.s
new file mode 100644
index 0000000000000..e0bce0625b1d7
--- /dev/null
+++ b/llvm/test/MC/X86/align-branch-64-pad-max-prefix.s
@@ -0,0 +1,18 @@
+  # RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu --x86-align-branch-boundary=32 --x86-align-branch=jmp -x86-pad-max-prefix-size=5 %s | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
+  # Check instructions can be aligned correctly along with option -x86-pad-max-prefix-size=5
+
+  .text
+  .p2align 5
+  .rept 24
+  int3
+  .endr
+  # We should not increase the length of this jmp to reduce the bytes of
+  # following nops, doing so would make the jmp misaligned.
+# CHECK:      18:          jmp
+  jmp bar
+# CHECK:      1d:          nop
+# CHECK:      1e:          nop
+# CHECK:      1f:          nop
+# CHECK:      20:          int3
+  .p2align 5
+  int3
diff --git a/llvm/test/MC/X86/align-via-padding-corner.s b/llvm/test/MC/X86/align-via-padding-corner.s
new file mode 100644
index 0000000000000..cc13ff7eed7f4
--- /dev/null
+++ b/llvm/test/MC/X86/align-via-padding-corner.s
@@ -0,0 +1,29 @@
+  # RUN: llvm-mc -mcpu=skylake -filetype=obj -triple x86_64-pc-linux-gnu %s -x86-pad-max-prefix-size=5 | llvm-objdump -d - | FileCheck %s
+
+
+  # The first test check the correctness cornercase - can't add prefixes on a
+  # instruction following by a prefix.
+  .globl labeled_prefix_test
+labeled_prefix_test:
+# CHECK:       0: 2e 2e 2e 2e 2e e9 06 00 00 00    jmp
+# CHECK:       a: 3e e9 00 00 00 00                jmp
+  jmp bar
+  DS
+  jmp bar
+  .p2align 4
+bar:
+  ret
+
+  # The second test is similar to the second test - can't add prefixes on a
+  # instruction following by hardcode.
+  .p2align 5
+  .globl labeled_hardcode_test
+labeled_hardcode_test:
+# CHECK:      20: 2e 2e 2e 2e 2e e9 06 00 00 00    jmp
+# CHECK:      2a: 3e e9 00 00 00 00                jmp
+  jmp baz
+  .byte 0x3e
+  jmp baz
+  .p2align 4
+baz:
+  ret
diff --git a/llvm/test/MC/X86/align-via-padding.s b/llvm/test/MC/X86/align-via-padding.s
index 572af4b02961e..d5d86da6ddca7 100644
--- a/llvm/test/MC/X86/align-via-padding.s
+++ b/llvm/test/MC/X86/align-via-padding.s
@@ -35,12 +35,12 @@ foo:
   # fewer nops by relaxing the branch, even though we don't need to
   # CHECK: <loop_preheader>:
   # CHECK: 45: 48 85 c0                       testq %rax, %rax
-  # CHECK: 48: 2e 2e 2e 2e 0f 8e 1e 00 00 00  jle 30 <loop_exit>
+  # CHECK: 48: 2e 2e 2e 2e 0f 8e 1e 00 00 00  jle 0x70 <loop_exit>
   # CHECK: 52: 66 66 66 66 66 2e 0f 1f 84 00 00 00 00 00    	nopw	%cs:(%rax,%rax)
   # CHECK: <loop_header>:
   # CHECK: 60: 48 83 e8 01                    subq $1, %rax
   # CHECK: 64: 48 85 c0                       testq %rax, %rax
-  # CHECK: 67: 7e 07                          jle 7 <loop_exit>
+  # CHECK: 67: 7e 07                          jle 0x70 <loop_exit>
   # CHECK: 69: 2e 2e e9 f0 ff ff ff           jmp
   # CHECK: <loop_exit>:
   # CHECK: 70: c3                             retq
@@ -62,7 +62,7 @@ loop_exit:
   # Correctness cornercase - can't prefix pad jmp without having relaxed it
   # first as doing so would make the relative offset too large
   # CHECK: fd: cc                             int3
-  # CHECK: fe: eb 80                          jmp -128 <loop_exit+0x10>
+  # CHECK: fe: eb 80                          jmp 0x80 <loop_exit+0x10>
   # CHECK: 100: cc                           	int3
 .p2align 5
 .L1:
diff --git a/llvm/test/MC/X86/align-via-relaxation.s b/llvm/test/MC/X86/align-via-relaxation.s
index 7f372bba0f20c..d768232655b76 100644
--- a/llvm/test/MC/X86/align-via-relaxation.s
+++ b/llvm/test/MC/X86/align-via-relaxation.s
@@ -9,13 +9,13 @@
   # Demonstrate that we can relax instructions to provide padding, not
   # just insert nops.  jmps are being used for ease of demonstration.
   # CHECK: .text
-  # CHECK: 0: eb 1f                         jmp 31 <foo>
-  # CHECK: 2: e9 1a 00 00 00                jmp 26 <foo>
-  # CHECK: 7: e9 15 00 00 00                jmp 21 <foo>
-  # CHECK: c: e9 10 00 00 00                jmp 16 <foo>
-  # CHECK: 11: e9 0b 00 00 00               jmp 11 <foo>
-  # CHECK: 16: e9 06 00 00 00               jmp 6 <foo>
-  # CHECK: 1b: e9 01 00 00 00               jmp 1 <foo>
+  # CHECK: 0: eb 1f                         jmp 0x21 <foo>
+  # CHECK: 2: e9 1a 00 00 00                jmp 0x21 <foo>
+  # CHECK: 7: e9 15 00 00 00                jmp 0x21 <foo>
+  # CHECK: c: e9 10 00 00 00                jmp 0x21 <foo>
+  # CHECK: 11: e9 0b 00 00 00               jmp 0x21 <foo>
+  # CHECK: 16: e9 06 00 00 00               jmp 0x21 <foo>
+  # CHECK: 1b: e9 01 00 00 00               jmp 0x21 <foo>
   # CHECK: 20: cc                           int3
   .p2align 4
   jmp foo
@@ -33,7 +33,7 @@ foo:
   # Check that we're not shifting aroudn the offsets of labels - doing
   # that would require a further round of relaxation
   # CHECK: <bar>:
-  # CHECK: 22: eb fe                          jmp -2 <bar>
+  # CHECK: 22: eb fe                          jmp 0x22 <bar>
   # CHECK: 24: 66 66 66 2e 0f 1f 84 00 00 00 00 00 nopw %cs:(%rax,%rax)
   # CHECK: 30: 0f 0b                          ud2
 
@@ -48,14 +48,14 @@ nobypass:
   # fewer nops by relaxing the branch, even though we don't need to
   # CHECK: <loop_preheader>:
   # CHECK: 45: 48 85 c0                       testq %rax, %rax
-  # CHECK: 48: 0f 8e 22 00 00 00              jle 34 <loop_exit>
+  # CHECK: 48: 0f 8e 22 00 00 00              jle 0x70 <loop_exit>
   # CHECK: 4e: 66 66 66 66 66 66 2e 0f 1f 84 00 00 00 00 00 nopw %cs:(%rax,%rax)
   # CHECK: 5d: 0f 1f 00                       nopl (%rax)
   # CHECK: <loop_header>:
   # CHECK: 60: 48 83 e8 01                    subq $1, %rax
   # CHECK: 64: 48 85 c0                       testq %rax, %rax
-  # CHECK: 67: 7e 07                          jle 7 <loop_exit>
-  # CHECK: 69: e9 f2 ff ff ff                 jmp -14 <loop_header>
+  # CHECK: 67: 7e 07                          jle 0x70 <loop_exit>
+  # CHECK: 69: e9 f2 ff ff ff                 jmp 0x60 <loop_header>
   # CHECK: 6e: 66 90                          nop
   # CHECK: <loop_exit>:
   # CHECK: 70: c3                             retq
diff --git a/llvm/test/MC/X86/reloc-directive-elf-32.s b/llvm/test/MC/X86/reloc-directive-elf-32.s
index f6e321c94e0dc..06755863e5730 100644
--- a/llvm/test/MC/X86/reloc-directive-elf-32.s
+++ b/llvm/test/MC/X86/reloc-directive-elf-32.s
@@ -7,12 +7,18 @@
 # PRINT:      .reloc 2, R_386_NONE, .data
 # PRINT-NEXT: .reloc 1, R_386_NONE, foo+4
 # PRINT-NEXT: .reloc 0, R_386_NONE, 8
+# PRINT-NEXT: .reloc 0, R_386_32, .data+2
+# PRINT-NEXT: .reloc 0, R_386_IRELATIVE, foo+3
+# PRINT-NEXT: .reloc 0, R_386_GOT32X, 5
 
 # X86 relocations use the Elf32_Rel format. Addends are neither stored in the
 # relocation entries nor applied in the referenced locations.
 # CHECK:      0x2 R_386_NONE .data 0x0
 # CHECK-NEXT: 0x1 R_386_NONE foo 0x0
 # CHECK-NEXT: 0x0 R_386_NONE - 0x0
+# CHECK-NEXT: 0x0 R_386_32 .data 0x0
+# CHECK-NEXT: 0x0 R_386_IRELATIVE foo 0x0
+# CHECK-NEXT: 0x0 R_386_GOT32X - 0x0
 
 # HEX: 0x00000000 00000000 00000000
 
@@ -23,6 +29,9 @@
   .reloc 2, R_386_NONE, .data
   .reloc 1, R_386_NONE, foo+4
   .reloc 0, R_386_NONE, 8
+  .reloc 0, R_386_32, .data+2
+  .reloc 0, R_386_IRELATIVE, foo+3
+  .reloc 0, R_386_GOT32X, 5
 
 .data
 .globl foo
diff --git a/llvm/test/MC/X86/reloc-directive-elf-64.s b/llvm/test/MC/X86/reloc-directive-elf-64.s
index e9523d72949f7..30a252fcb84de 100644
--- a/llvm/test/MC/X86/reloc-directive-elf-64.s
+++ b/llvm/test/MC/X86/reloc-directive-elf-64.s
@@ -6,10 +6,16 @@
 # PRINT:      .reloc 2, R_X86_64_NONE, .data
 # PRINT-NEXT: .reloc 1, R_X86_64_NONE, foo+4
 # PRINT-NEXT: .reloc 0, R_X86_64_NONE, 8
+# PRINT-NEXT: .reloc 0, R_X86_64_64, .data+2
+# PRINT-NEXT: .reloc 0, R_X86_64_GOTPCRELX, foo+3
+# PRINT-NEXT: .reloc 0, R_X86_64_REX_GOTPCRELX, 5
 
 # CHECK:      0x2 R_X86_64_NONE .data 0x0
 # CHECK-NEXT: 0x1 R_X86_64_NONE foo 0x4
 # CHECK-NEXT: 0x0 R_X86_64_NONE - 0x8
+# CHECK-NEXT: 0x0 R_X86_64_64 .data 0x2
+# CHECK-NEXT: 0x0 R_X86_64_GOTPCRELX foo 0x3
+# CHECK-NEXT: 0x0 R_X86_64_REX_GOTPCRELX - 0x5
 
 .text
   ret
@@ -18,6 +24,9 @@
   .reloc 2, R_X86_64_NONE, .data
   .reloc 1, R_X86_64_NONE, foo+4
   .reloc 0, R_X86_64_NONE, 8
+  .reloc 0, R_X86_64_64, .data+2
+  .reloc 0, R_X86_64_GOTPCRELX, foo+3
+  .reloc 0, R_X86_64_REX_GOTPCRELX, 5
 
 .data
 .globl foo
diff --git a/llvm/test/MC/X86/x86-16.s b/llvm/test/MC/X86/x86-16.s
index b849d0aacd2b5..531a5302886c8 100644
--- a/llvm/test/MC/X86/x86-16.s
+++ b/llvm/test/MC/X86/x86-16.s
@@ -1029,3 +1029,7 @@ enqcmd  (%edi), %edi
 // CHECK: enqcmds (%edi), %edi
 // CHECK: encoding: [0x67,0xf3,0x0f,0x38,0xf8,0x3f]
 enqcmds (%edi), %edi
+
+// CHECK: serialize
+// CHECK: encoding: [0x0f,0x01,0xe8]
+serialize
diff --git a/llvm/test/MC/X86/x86-32-coverage.s b/llvm/test/MC/X86/x86-32-coverage.s
index 1403c7cbfc345..0e6d0afe63b1f 100644
--- a/llvm/test/MC/X86/x86-32-coverage.s
+++ b/llvm/test/MC/X86/x86-32-coverage.s
@@ -10876,3 +10876,7 @@ enqcmds (%bx,%di), %di
 // CHECK: enqcmds 8128(%bx,%di), %ax
 // CHECK: encoding: [0x67,0xf3,0x0f,0x38,0xf8,0x81,0xc0,0x1f]
 enqcmds 8128(%bx,%di), %ax
+
+// CHECK: serialize
+// CHECK: encoding: [0x0f,0x01,0xe8]
+serialize
diff --git a/llvm/test/MC/X86/x86-64.s b/llvm/test/MC/X86/x86-64.s
index b2251ecb4473c..8339e675be5b0 100644
--- a/llvm/test/MC/X86/x86-64.s
+++ b/llvm/test/MC/X86/x86-64.s
@@ -1877,3 +1877,7 @@ enqcmds -8192(%rdx), %rbx
 // CHECK: enqcmds 485498096, %rax
 // CHECK: encoding: [0xf3,0x0f,0x38,0xf8,0x04,0x25,0xf0,0x1c,0xf0,0x1c]
 enqcmds 485498096, %rax
+
+// CHECK: serialize
+// CHECK: encoding: [0x0f,0x01,0xe8]
+serialize
diff --git a/llvm/test/MC/X86/x86-branch-relaxation.s b/llvm/test/MC/X86/x86-branch-relaxation.s
index 0bdeb667bbfb3..8eaf32c48290c 100644
--- a/llvm/test/MC/X86/x86-branch-relaxation.s
+++ b/llvm/test/MC/X86/x86-branch-relaxation.s
@@ -24,8 +24,8 @@ baz32:
 
 	
 // CHECK16-LABEL: bar16
-// CHECK16-NEXT: e9 fe ff 	jmp	-2 <bar16+0x1>
+// CHECK16-NEXT: e9 fe ff 	jmp	0x1 <bar16+0x1>
 // CHECK32-LABEL: baz32
-// CHECK32-NEXT: e9 fc ff ff ff 	jmp	-4 <baz32+0x1>
+// CHECK32-NEXT: e9 fc ff ff ff 	jmp	0x1 <baz32+0x1>
 
 	
diff --git a/llvm/test/Object/X86/objdump-disassembly-inline-relocations.test b/llvm/test/Object/X86/objdump-disassembly-inline-relocations.test
index 34dedf9cefed5..986c652ab8ffc 100644
--- a/llvm/test/Object/X86/objdump-disassembly-inline-relocations.test
+++ b/llvm/test/Object/X86/objdump-disassembly-inline-relocations.test
@@ -42,9 +42,9 @@
 # MACHO-i386:        3:       c7 44 24 08 00 00 00 00                         movl    $0, 8(%esp)
 # MACHO-i386:        b:       c7 04 24 24 00 00 00                            movl    $36, (%esp)
 # MACHO-i386:                                e: GENERIC_RELOC_VANILLA __cstring
-# MACHO-i386:       12:       e8 1f 00 00 00                                  calll   31
+# MACHO-i386:       12:       e8 1f 00 00 00                                  calll   0x36
 # MACHO-i386:                               13: GENERIC_RELOC_VANILLA __jump_table
-# MACHO-i386:       17:       e8 15 00 00 00                                  calll   21
+# MACHO-i386:       17:       e8 15 00 00 00                                  calll   0x31
 # MACHO-i386:                               18: GENERIC_RELOC_VANILLA __jump_table
 # MACHO-i386:       1c:       8b 44 24 08                                     movl    8(%esp), %eax
 # MACHO-i386:       20:       83 c4 0c                                        addl    $12, %esp
@@ -79,9 +79,9 @@
 # ELF-i386:        3:     c7 44 24 08 00 00 00 00                         movl    $0, 8(%esp)
 # ELF-i386:        b:     c7 04 24 00 00 00 00                            movl    $0, (%esp)
 # ELF-i386:                              e: R_386_32      .rodata.str1.1
-# ELF-i386:       12:     e8 fc ff ff ff                                  calll   -4
+# ELF-i386:       12:     e8 fc ff ff ff                                  calll   0x13
 # ELF-i386:                             13: R_386_PC32    puts
-# ELF-i386:       17:     e8 fc ff ff ff                                  calll   -4
+# ELF-i386:       17:     e8 fc ff ff ff                                  calll   0x18
 # ELF-i386:                             18: R_386_PC32    SomeOtherFunction
 # ELF-i386:       1c:     8b 44 24 08                                     movl    8(%esp), %eax
 # ELF-i386:       20:     83 c4 0c                                        addl    $12, %esp
diff --git a/llvm/test/Object/X86/objdump-trivial-object.test b/llvm/test/Object/X86/objdump-trivial-object.test
index c3e080c5f88fc..b7c69d6518d8b 100644
--- a/llvm/test/Object/X86/objdump-trivial-object.test
+++ b/llvm/test/Object/X86/objdump-trivial-object.test
@@ -34,8 +34,8 @@
 # ELF-i386:        0: 83 ec 0c                        subl    $12, %esp
 # ELF-i386:        3: c7 44 24 08 00 00 00 00         movl    $0, 8(%esp)
 # ELF-i386:        b: c7 04 24 00 00 00 00            movl    $0, (%esp)
-# ELF-i386:       12: e8 fc ff ff ff                  calll   -4
-# ELF-i386:       17: e8 fc ff ff ff                  calll   -4
+# ELF-i386:       12: e8 fc ff ff ff                  calll   0x13
+# ELF-i386:       17: e8 fc ff ff ff                  calll   0x18
 # ELF-i386:       1c: 8b 44 24 08                     movl    8(%esp), %eax
 # ELF-i386:       20: 83 c4 0c                        addl    $12, %esp
 # ELF-i386:       23: c3                              ret
diff --git a/llvm/test/Object/obj2yaml.test b/llvm/test/Object/obj2yaml.test
index a5f008ffd238a..808e9fa0a34c1 100644
--- a/llvm/test/Object/obj2yaml.test
+++ b/llvm/test/Object/obj2yaml.test
@@ -659,6 +659,20 @@ Symbols:
 # ELF-AVR-NEXT:   Type:            ET_EXEC
 # ELF-AVR-NEXT:   Machine:         EM_AVR
 # ELF-AVR-NEXT:   Flags:           [ EF_AVR_ARCH_AVR2 ]
+# ELF-AVR-NEXT: ProgramHeaders:
+# ELF-AVR-NEXT:  - Type:  PT_LOAD
+# ELF-AVR-NEXT:    Flags: [ PF_X, PF_R ]
+# ELF-AVR-NEXT:    Sections:
+# ELF-AVR-NEXT:     - Section: .text
+# ELF-AVR-NEXT:     - Section: .data
+# ELF-AVR-NEXT:    Align: 0x0000000000000002
+# ELF-AVR-NEXT:  - Type:  PT_LOAD
+# ELF-AVR-NEXT:    Flags: [ PF_W, PF_R ]
+# ELF-AVR-NEXT:    Sections:
+# ELF-AVR-NEXT:     - Section: .data
+# ELF-AVR-NEXT:    VAddr: 0x0000000000800060
+# ELF-AVR-NEXT:    PAddr: 0x0000000000000004
+# ELF-AVR-NEXT:    Align: 0x0000000000000001
 # ELF-AVR-NEXT: Sections:
 # ELF-AVR-NEXT:   - Name:            .text
 # ELF-AVR-NEXT:     Type:            SHT_PROGBITS
diff --git a/llvm/test/Other/new-pm-cgprofile.ll b/llvm/test/Other/new-pm-cgprofile.ll
new file mode 100644
index 0000000000000..c7fe31ab570f3
--- /dev/null
+++ b/llvm/test/Other/new-pm-cgprofile.ll
@@ -0,0 +1,11 @@
+; RUN: opt -debug-pass-manager -passes='default<O2>' %s 2>&1 |FileCheck %s --check-prefixes=DEFAULT
+; RUN: opt -debug-pass-manager -passes='default<O2>' -enable-npm-call-graph-profile=0 %s 2>&1 |FileCheck %s --check-prefixes=OFF
+; RUN: opt -debug-pass-manager -passes='default<O2>' -enable-npm-call-graph-profile=1 %s 2>&1 |FileCheck %s --check-prefixes=ON
+;
+; DEFAULT: Running pass: CGProfilePass
+; OFF-NOT: Running pass: CGProfilePass
+; ON: Running pass: CGProfilePass
+
+define void @foo() {
+  ret void
+}
diff --git a/llvm/test/Other/opt-bisect-legacy-pass-manager.ll b/llvm/test/Other/opt-bisect-legacy-pass-manager.ll
index bf89e80d49604..d73ca80674dae 100644
--- a/llvm/test/Other/opt-bisect-legacy-pass-manager.ll
+++ b/llvm/test/Other/opt-bisect-legacy-pass-manager.ll
@@ -45,8 +45,7 @@
 ; The helper script uses this to find the optimization that inlines the call.
 ; CHECK-BISECT-INLINE-HELPER: call i32 @f2()
 ; These checks verifies that the optimization was found.
-; CHECK-BISECT-INLINE-RESULT-NOT: Last good count: 0
-; CHECK-BISECT-INLINE-RESULT: Last good count: {{[0-9]+}}
+; CHECK-BISECT-INLINE-RESULT: Last good count: {{[1-9][0-9]*}}
 
 
 ; Test a module pass.
diff --git a/llvm/test/Other/opt-override-denormal-fp-math-f32.ll b/llvm/test/Other/opt-override-denormal-fp-math-f32.ll
new file mode 100644
index 0000000000000..3bb88a66e8540
--- /dev/null
+++ b/llvm/test/Other/opt-override-denormal-fp-math-f32.ll
@@ -0,0 +1,23 @@
+; RUN: opt -S -denormal-fp-math-f32=ieee %s | FileCheck -check-prefixes=IEEE,ALL %s
+; RUN: opt -S -denormal-fp-math-f32=preserve-sign %s | FileCheck -check-prefixes=PRESERVESIGN,ALL %s
+; RUN: opt -S -denormal-fp-math-f32=positive-zero %s | FileCheck -check-prefixes=POSZERO,ALL %s
+
+; ALL: @no_denormal_fp_math_f32_attr() [[NOATTR:#[0-9]+]] {
+define i32 @no_denormal_fp_math_f32_attr() #0 {
+entry:
+  ret i32 0
+}
+
+; ALL: denormal_fp_math_attr_preserve_sign_ieee() [[ATTR:#[0-9]+]] {
+define i32 @denormal_fp_math_attr_preserve_sign_ieee() #1 {
+entry:
+  ret i32 0
+}
+
+; ALL-DAG: attributes [[ATTR]] = { nounwind "denormal-fp-math-f32"="preserve-sign,ieee" }
+; IEEE-DAG: attributes [[NOATTR]] = { nounwind "denormal-fp-math-f32"="ieee,ieee" }
+; PRESERVESIGN-DAG: attributes [[NOATTR]] = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
+; POSITIVEZERO-DAG: attributes [[NOATTR]] = { nounwind "denormal-fp-math-f32"="positive-zero,positive-zero" }
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind "denormal-fp-math-f32"="preserve-sign,ieee" }
diff --git a/llvm/test/Other/opt-override-denormal-fp-math-mixed.ll b/llvm/test/Other/opt-override-denormal-fp-math-mixed.ll
new file mode 100644
index 0000000000000..dffabbb7dccd3
--- /dev/null
+++ b/llvm/test/Other/opt-override-denormal-fp-math-mixed.ll
@@ -0,0 +1,42 @@
+; RUN: opt -S -denormal-fp-math=ieee %s | FileCheck -check-prefixes=IEEE,ALL %s
+; RUN: opt -S -denormal-fp-math=preserve-sign %s | FileCheck -check-prefixes=PRESERVESIGN,ALL %s
+; RUN: opt -S -denormal-fp-math=positive-zero %s | FileCheck -check-prefixes=POSZERO,ALL %s
+
+; RUN: opt -S -denormal-fp-math-f32=ieee %s | FileCheck -check-prefixes=IEEEF32,ALL %s
+; RUN: opt -S -denormal-fp-math-f32=preserve-sign %s | FileCheck -check-prefixes=PRESERVESIGNF32,ALL %s
+; RUN: opt -S -denormal-fp-math-f32=positive-zero %s | FileCheck -check-prefixes=POSZEROF32,ALL %s
+
+; RUN: opt -S -denormal-fp-math=ieee -denormal-fp-math-f32=ieee %s | FileCheck -check-prefixes=IEEE-BOTH,ALL %s
+; RUN: opt -S -denormal-fp-math=preserve-sign -denormal-fp-math-f32=preserve-sign %s | FileCheck -check-prefixes=PRESERVESIGN-BOTH,ALL %s
+; RUN: opt -S -denormal-fp-math=positive-zero -denormal-fp-math-f32=positive-zero %s | FileCheck -check-prefixes=POSZERO-BOTH,ALL %s
+
+
+
+; ALL: @no_denormal_fp_math_attrs() [[NOATTR:#[0-9]+]] {
+define i32 @no_denormal_fp_math_attrs() #0 {
+entry:
+  ret i32 0
+}
+
+; ALL: both_denormal_fp_math_attrs_preserve_sign_ieee() [[ATTR:#[0-9]+]] {
+define i32 @both_denormal_fp_math_attrs_preserve_sign_ieee() #1 {
+entry:
+  ret i32 0
+}
+
+; ALL-DAG: attributes [[ATTR]] = { nounwind "denormal-fp-math"="preserve-sign,ieee" "denormal-fp-math-f32"="preserve-sign,ieee" }
+
+; IEEE-DAG: attributes [[NOATTR]] = { nounwind "denormal-fp-math"="ieee,ieee" }
+; PRESERVESIGN-DAG: attributes [[NOATTR]] = { nounwind "denormal-fp-math"="preserve-sign,preserve-sign" }
+; POSITIVEZERO-DAG: attributes [[NOATTR]] = { nounwind "denormal-fp-math"="positive-zero,positive-zero" }
+
+; IEEEF32-DAG: attributes [[NOATTR]] = { nounwind "denormal-fp-math-f32"="ieee,ieee" }
+; PRESERVESIGNF32-DAG: attributes [[NOATTR]] = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
+; POSITIVEZEROF32-DAG: attributes [[NOATTR]] = { nounwind "denormal-fp-math-f32"="positive-zero,positive-zero" }
+
+; IEEE-BOTH-DAG: attributes [[NOATTR]] = { nounwind "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="ieee,ieee" }
+; PRESERVESIGN-BOTH-DAG: attributes [[NOATTR]] = { nounwind "denormal-fp-math"="preserve-sign,preserve-sign" "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
+; POSITIVEZERO-BOTH-DAG: attributes [[NOATTR]] = { nounwind "denormal-fp-math"="positive-zero,positive-zero" "denormal-fp-math-f32"="positive-zero,positive-zero" }
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind "denormal-fp-math"="preserve-sign,ieee" "denormal-fp-math-f32"="preserve-sign,ieee" }
diff --git a/llvm/test/Other/opt-override-denormal-fp-math.ll b/llvm/test/Other/opt-override-denormal-fp-math.ll
new file mode 100644
index 0000000000000..5a248205e53fa
--- /dev/null
+++ b/llvm/test/Other/opt-override-denormal-fp-math.ll
@@ -0,0 +1,23 @@
+; RUN: opt -S -denormal-fp-math=ieee %s | FileCheck -check-prefixes=IEEE,ALL %s
+; RUN: opt -S -denormal-fp-math=preserve-sign %s | FileCheck -check-prefixes=PRESERVESIGN,ALL %s
+; RUN: opt -S -denormal-fp-math=positive-zero %s | FileCheck -check-prefixes=POSZERO,ALL %s
+
+; ALL: @no_denormal_fp_math_attr() [[NOATTR:#[0-9]+]] {
+define i32 @no_denormal_fp_math_attr() #0 {
+entry:
+  ret i32 0
+}
+
+; ALL: denormal_fp_math_attr_preserve_sign_ieee() [[ATTR:#[0-9]+]] {
+define i32 @denormal_fp_math_attr_preserve_sign_ieee() #1 {
+entry:
+  ret i32 0
+}
+
+; ALL-DAG: attributes [[ATTR]] = { nounwind "denormal-fp-math"="preserve-sign,ieee" }
+; IEEE-DAG: attributes [[NOATTR]] = { nounwind "denormal-fp-math"="ieee,ieee" }
+; PRESERVESIGN-DAG: attributes [[NOATTR]] = { nounwind "denormal-fp-math"="preserve-sign,preserve-sign" }
+; POSITIVEZERO-DAG: attributes [[NOATTR]] = { nounwind "denormal-fp-math"="positive-zero,positive-zero" }
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind "denormal-fp-math"="preserve-sign,ieee" }
diff --git a/llvm/test/TableGen/address-space-patfrags.td b/llvm/test/TableGen/address-space-patfrags.td
index cf31294bfe099..232cf7920a540 100644
--- a/llvm/test/TableGen/address-space-patfrags.td
+++ b/llvm/test/TableGen/address-space-patfrags.td
@@ -81,7 +81,7 @@ def : Pat <
 
 // SDAG-NEXT: if (AddrSpace != 999)
 // SDAG-NEXT: return false;
-// SDAG-NEXT: if (cast<MemSDNode>(N)->getAlignment() < 2)
+// SDAG-NEXT: if (cast<MemSDNode>(N)->getAlign() < Align(2))
 // SDAG-NEXT: return false;
 // SDAG-NEXT: if (cast<MemSDNode>(N)->getMemoryVT() != MVT::i32) return false;
 // SDAG-NEXT: return true;
diff --git a/llvm/test/Transforms/AlignmentFromAssumptions/amdgpu-crash.ll b/llvm/test/Transforms/AlignmentFromAssumptions/amdgpu-crash.ll
new file mode 100644
index 0000000000000..065708a4d59cd
--- /dev/null
+++ b/llvm/test/Transforms/AlignmentFromAssumptions/amdgpu-crash.ll
@@ -0,0 +1,33 @@
+; Test that we don't crash.
+; RUN: opt < %s -alignment-from-assumptions -S
+; RUN: opt < %s -passes=alignment-from-assumptions -S
+
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-ni:7"
+
+%"core::str::CharIndices.29.66.90.114.138.149.165.173.181.197.205.213.229.387.398" = type { [0 x i64], i64, [0 x i64], { i8*, i8* }, [0 x i64] }
+%"unwind::libunwind::_Unwind_Exception.9.51.75.99.123.147.163.171.179.195.203.211.227.385.396" = type { [0 x i64], i64, [0 x i64], void (i32, %"unwind::libunwind::_Unwind_Exception.9.51.75.99.123.147.163.171.179.195.203.211.227.385.396"*)*, [0 x i64], [6 x i64], [0 x i64] }
+%"unwind::libunwind::_Unwind_Context.10.52.76.100.124.148.164.172.180.196.204.212.228.386.397" = type { [0 x i8] }
+
+define void @"_ZN44_$LT$$RF$T$u20$as$u20$core..fmt..Display$GT$3fmt17h7b1d039c7ff5e1feE"() {
+start:
+  %_15.i.i = alloca %"core::str::CharIndices.29.66.90.114.138.149.165.173.181.197.205.213.229.387.398", align 8, addrspace(5)
+  br label %bb12.i.i
+
+bb12.i.i:                                         ; preds = %start
+  %0 = addrspacecast %"core::str::CharIndices.29.66.90.114.138.149.165.173.181.197.205.213.229.387.398" addrspace(5)* %_15.i.i to %"core::str::CharIndices.29.66.90.114.138.149.165.173.181.197.205.213.229.387.398"*
+  %ptrint53.i.i = ptrtoint %"core::str::CharIndices.29.66.90.114.138.149.165.173.181.197.205.213.229.387.398"* %0 to i64
+  %maskedptr54.i.i = and i64 %ptrint53.i.i, 7
+  %maskcond55.i.i = icmp eq i64 %maskedptr54.i.i, 0
+  call void @llvm.assume(i1 %maskcond55.i.i)
+  br i1 undef, label %bb20.i.i, label %bb3.i.i.i.i.i.preheader.i.i
+
+bb3.i.i.i.i.i.preheader.i.i:                      ; preds = %bb12.i.i
+  %1 = getelementptr inbounds %"core::str::CharIndices.29.66.90.114.138.149.165.173.181.197.205.213.229.387.398", %"core::str::CharIndices.29.66.90.114.138.149.165.173.181.197.205.213.229.387.398"* %0, i64 0, i32 0, i64 0
+  store i64 0, i64* %1, align 8
+  unreachable
+
+bb20.i.i:                                         ; preds = %bb12.i.i
+  ret void
+}
+
+declare void @llvm.assume(i1)
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-07-02-array-indexing.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-07-02-array-indexing.ll
index bea9c33814389..65d1528151b9d 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-07-02-array-indexing.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-07-02-array-indexing.ll
@@ -13,7 +13,7 @@ define internal i32 @callee(i1 %C, i32* %A) {
 ; CHECK-NEXT:    unreachable
 ; CHECK:       F:
 ; CHECK-NEXT:    [[A_2:%.*]] = getelementptr i32, i32* null, i32 2
-; CHECK-NEXT:    [[R:%.*]] = load i32, i32* [[A_2]]
+; CHECK-NEXT:    [[R:%.*]] = load i32, i32* [[A_2]], align 4
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
 entry:
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/min-legal-vector-width.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/min-legal-vector-width.ll
index 709d065ba300b..bbaf3d6496a93 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/min-legal-vector-width.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/min-legal-vector-width.ll
@@ -8,12 +8,12 @@ target triple = "x86_64-unknown-linux-gnu"
 ; This should promote
 define internal fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer512(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #0 {
 ; CHECK-LABEL: define {{[^@]+}}@callee_avx512_legal512_prefer512_call_avx512_legal512_prefer512
-; CHECK-SAME: (<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[ARG:%.*]], <8 x i64> [[TMP0:%.*]])
+; CHECK-SAME: (<8 x i64>* noalias nocapture nofree nonnull writeonly align 64 dereferenceable(64) [[ARG:%.*]], <8 x i64> [[TMP0:%.*]])
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[ARG1_PRIV:%.*]] = alloca <8 x i64>
 ; CHECK-NEXT:    store <8 x i64> [[TMP0]], <8 x i64>* [[ARG1_PRIV]]
-; CHECK-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1_PRIV]], align 32
-; CHECK-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]], align 32
+; CHECK-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1_PRIV]], align 64
+; CHECK-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]], align 64
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -31,8 +31,8 @@ define void @avx512_legal512_prefer512_call_avx512_legal512_prefer512(<8 x i64>*
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64>* [[TMP]] to i8*
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull writeonly align 32 dereferenceable(64) [[TMP3]], i8 0, i64 32, i1 false)
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i64>, <8 x i64>* [[TMP]], align 1
-; CHECK-NEXT:    call fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer512(<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]])
-; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 32
+; CHECK-NEXT:    call fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer512(<8 x i64>* noalias nocapture nofree nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]])
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 64
 ; CHECK-NEXT:    store <8 x i64> [[TMP4]], <8 x i64>* [[ARG]], align 2
 ; CHECK-NEXT:    ret void
 ;
@@ -50,12 +50,12 @@ bb:
 ; This should promote
 define internal fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer256(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #1 {
 ; CHECK-LABEL: define {{[^@]+}}@callee_avx512_legal512_prefer256_call_avx512_legal512_prefer256
-; CHECK-SAME: (<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[ARG:%.*]], <8 x i64> [[TMP0:%.*]])
+; CHECK-SAME: (<8 x i64>* noalias nocapture nofree nonnull writeonly align 64 dereferenceable(64) [[ARG:%.*]], <8 x i64> [[TMP0:%.*]])
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[ARG1_PRIV:%.*]] = alloca <8 x i64>
 ; CHECK-NEXT:    store <8 x i64> [[TMP0]], <8 x i64>* [[ARG1_PRIV]]
-; CHECK-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1_PRIV]], align 32
-; CHECK-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]], align 32
+; CHECK-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1_PRIV]], align 64
+; CHECK-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]], align 64
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -73,8 +73,8 @@ define void @avx512_legal512_prefer256_call_avx512_legal512_prefer256(<8 x i64>*
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64>* [[TMP]] to i8*
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull writeonly align 32 dereferenceable(64) [[TMP3]], i8 0, i64 32, i1 false)
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i64>, <8 x i64>* [[TMP]], align 1
-; CHECK-NEXT:    call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer256(<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]])
-; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 32
+; CHECK-NEXT:    call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer256(<8 x i64>* noalias nocapture nofree nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]])
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 64
 ; CHECK-NEXT:    store <8 x i64> [[TMP4]], <8 x i64>* [[ARG]], align 2
 ; CHECK-NEXT:    ret void
 ;
@@ -92,12 +92,12 @@ bb:
 ; This should promote
 define internal fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer256(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #1 {
 ; CHECK-LABEL: define {{[^@]+}}@callee_avx512_legal512_prefer512_call_avx512_legal512_prefer256
-; CHECK-SAME: (<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[ARG:%.*]], <8 x i64> [[TMP0:%.*]])
+; CHECK-SAME: (<8 x i64>* noalias nocapture nofree nonnull writeonly align 64 dereferenceable(64) [[ARG:%.*]], <8 x i64> [[TMP0:%.*]])
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[ARG1_PRIV:%.*]] = alloca <8 x i64>
 ; CHECK-NEXT:    store <8 x i64> [[TMP0]], <8 x i64>* [[ARG1_PRIV]]
-; CHECK-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1_PRIV]], align 32
-; CHECK-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]], align 32
+; CHECK-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1_PRIV]], align 64
+; CHECK-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]], align 64
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -115,8 +115,8 @@ define void @avx512_legal512_prefer512_call_avx512_legal512_prefer256(<8 x i64>*
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64>* [[TMP]] to i8*
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull writeonly align 32 dereferenceable(64) [[TMP3]], i8 0, i64 32, i1 false)
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i64>, <8 x i64>* [[TMP]], align 1
-; CHECK-NEXT:    call fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer256(<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]])
-; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 32
+; CHECK-NEXT:    call fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer256(<8 x i64>* noalias nocapture nofree nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]])
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 64
 ; CHECK-NEXT:    store <8 x i64> [[TMP4]], <8 x i64>* [[ARG]], align 2
 ; CHECK-NEXT:    ret void
 ;
@@ -134,12 +134,12 @@ bb:
 ; This should promote
 define internal fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer512(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #0 {
 ; CHECK-LABEL: define {{[^@]+}}@callee_avx512_legal512_prefer256_call_avx512_legal512_prefer512
-; CHECK-SAME: (<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[ARG:%.*]], <8 x i64> [[TMP0:%.*]])
+; CHECK-SAME: (<8 x i64>* noalias nocapture nofree nonnull writeonly align 64 dereferenceable(64) [[ARG:%.*]], <8 x i64> [[TMP0:%.*]])
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[ARG1_PRIV:%.*]] = alloca <8 x i64>
 ; CHECK-NEXT:    store <8 x i64> [[TMP0]], <8 x i64>* [[ARG1_PRIV]]
-; CHECK-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1_PRIV]], align 32
-; CHECK-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]], align 32
+; CHECK-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1_PRIV]], align 64
+; CHECK-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]], align 64
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -157,8 +157,8 @@ define void @avx512_legal512_prefer256_call_avx512_legal512_prefer512(<8 x i64>*
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64>* [[TMP]] to i8*
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull writeonly align 32 dereferenceable(64) [[TMP3]], i8 0, i64 32, i1 false)
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i64>, <8 x i64>* [[TMP]], align 1
-; CHECK-NEXT:    call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer512(<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]])
-; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 32
+; CHECK-NEXT:    call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer512(<8 x i64>* noalias nocapture nofree nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]])
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 64
 ; CHECK-NEXT:    store <8 x i64> [[TMP4]], <8 x i64>* [[ARG]], align 2
 ; CHECK-NEXT:    ret void
 ;
@@ -176,10 +176,10 @@ bb:
 ; This should not promote
 define internal fastcc void @callee_avx512_legal256_prefer256_call_avx512_legal512_prefer256(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #1 {
 ; CHECK-LABEL: define {{[^@]+}}@callee_avx512_legal256_prefer256_call_avx512_legal512_prefer256
-; CHECK-SAME: (<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[ARG:%.*]], <8 x i64>* noalias nocapture nofree nonnull readonly align 32 dereferenceable(64) [[ARG1:%.*]])
+; CHECK-SAME: (<8 x i64>* noalias nocapture nofree nonnull writeonly align 64 dereferenceable(64) [[ARG:%.*]], <8 x i64>* noalias nocapture nofree nonnull readonly align 64 dereferenceable(64) [[ARG1:%.*]])
 ; CHECK-NEXT:  bb:
-; CHECK-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1]], align 32
-; CHECK-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]], align 32
+; CHECK-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1]], align 64
+; CHECK-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]], align 64
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -196,8 +196,8 @@ define void @avx512_legal256_prefer256_call_avx512_legal512_prefer256(<8 x i64>*
 ; CHECK-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64>* [[TMP]] to i8*
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull writeonly align 32 dereferenceable(64) [[TMP3]], i8 0, i64 32, i1 false)
-; CHECK-NEXT:    call fastcc void @callee_avx512_legal256_prefer256_call_avx512_legal512_prefer256(<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[TMP2]], <8 x i64>* noalias nocapture nofree nonnull readonly align 32 dereferenceable(64) [[TMP]])
-; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 32
+; CHECK-NEXT:    call fastcc void @callee_avx512_legal256_prefer256_call_avx512_legal512_prefer256(<8 x i64>* noalias nocapture nofree nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64>* noalias nocapture nofree nonnull readonly align 64 dereferenceable(64) [[TMP]])
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 64
 ; CHECK-NEXT:    store <8 x i64> [[TMP4]], <8 x i64>* [[ARG]], align 2
 ; CHECK-NEXT:    ret void
 ;
@@ -215,10 +215,10 @@ bb:
 ; This should not promote
 define internal fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal256_prefer256(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #2 {
 ; CHECK-LABEL: define {{[^@]+}}@callee_avx512_legal512_prefer256_call_avx512_legal256_prefer256
-; CHECK-SAME: (<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[ARG:%.*]], <8 x i64>* noalias nocapture nofree nonnull readonly align 32 dereferenceable(64) [[ARG1:%.*]])
+; CHECK-SAME: (<8 x i64>* noalias nocapture nofree nonnull writeonly align 64 dereferenceable(64) [[ARG:%.*]], <8 x i64>* noalias nocapture nofree nonnull readonly align 64 dereferenceable(64) [[ARG1:%.*]])
 ; CHECK-NEXT:  bb:
-; CHECK-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1]], align 32
-; CHECK-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]], align 32
+; CHECK-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1]], align 64
+; CHECK-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]], align 64
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -235,8 +235,8 @@ define void @avx512_legal512_prefer256_call_avx512_legal256_prefer256(<8 x i64>*
 ; CHECK-NEXT:    [[TMP2:%.*]] = alloca <8 x i64>, align 32
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64>* [[TMP]] to i8*
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull writeonly align 32 dereferenceable(64) [[TMP3]], i8 0, i64 32, i1 false)
-; CHECK-NEXT:    call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal256_prefer256(<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[TMP2]], <8 x i64>* noalias nocapture nofree nonnull readonly align 32 dereferenceable(64) [[TMP]])
-; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 32
+; CHECK-NEXT:    call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal256_prefer256(<8 x i64>* noalias nocapture nofree nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64>* noalias nocapture nofree nonnull readonly align 64 dereferenceable(64) [[TMP]])
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 64
 ; CHECK-NEXT:    store <8 x i64> [[TMP4]], <8 x i64>* [[ARG]], align 2
 ; CHECK-NEXT:    ret void
 ;
@@ -254,12 +254,12 @@ bb:
 ; This should promote
 define internal fastcc void @callee_avx2_legal256_prefer256_call_avx2_legal512_prefer256(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #3 {
 ; CHECK-LABEL: define {{[^@]+}}@callee_avx2_legal256_prefer256_call_avx2_legal512_prefer256
-; CHECK-SAME: (<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[ARG:%.*]], <8 x i64> [[TMP0:%.*]])
+; CHECK-SAME: (<8 x i64>* noalias nocapture nofree nonnull writeonly align 64 dereferenceable(64) [[ARG:%.*]], <8 x i64> [[TMP0:%.*]])
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[ARG1_PRIV:%.*]] = alloca <8 x i64>
 ; CHECK-NEXT:    store <8 x i64> [[TMP0]], <8 x i64>* [[ARG1_PRIV]]
-; CHECK-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1_PRIV]], align 32
-; CHECK-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]], align 32
+; CHECK-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1_PRIV]], align 64
+; CHECK-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]], align 64
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -277,8 +277,8 @@ define void @avx2_legal256_prefer256_call_avx2_legal512_prefer256(<8 x i64>* %ar
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64>* [[TMP]] to i8*
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull writeonly align 32 dereferenceable(64) [[TMP3]], i8 0, i64 32, i1 false)
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i64>, <8 x i64>* [[TMP]], align 1
-; CHECK-NEXT:    call fastcc void @callee_avx2_legal256_prefer256_call_avx2_legal512_prefer256(<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]])
-; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 32
+; CHECK-NEXT:    call fastcc void @callee_avx2_legal256_prefer256_call_avx2_legal512_prefer256(<8 x i64>* noalias nocapture nofree nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]])
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 64
 ; CHECK-NEXT:    store <8 x i64> [[TMP4]], <8 x i64>* [[ARG]], align 2
 ; CHECK-NEXT:    ret void
 ;
@@ -296,12 +296,12 @@ bb:
 ; This should promote
 define internal fastcc void @callee_avx2_legal512_prefer256_call_avx2_legal256_prefer256(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #4 {
 ; CHECK-LABEL: define {{[^@]+}}@callee_avx2_legal512_prefer256_call_avx2_legal256_prefer256
-; CHECK-SAME: (<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[ARG:%.*]], <8 x i64> [[TMP0:%.*]])
+; CHECK-SAME: (<8 x i64>* noalias nocapture nofree nonnull writeonly align 64 dereferenceable(64) [[ARG:%.*]], <8 x i64> [[TMP0:%.*]])
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[ARG1_PRIV:%.*]] = alloca <8 x i64>
 ; CHECK-NEXT:    store <8 x i64> [[TMP0]], <8 x i64>* [[ARG1_PRIV]]
-; CHECK-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1_PRIV]], align 32
-; CHECK-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]], align 32
+; CHECK-NEXT:    [[TMP:%.*]] = load <8 x i64>, <8 x i64>* [[ARG1_PRIV]], align 64
+; CHECK-NEXT:    store <8 x i64> [[TMP]], <8 x i64>* [[ARG]], align 64
 ; CHECK-NEXT:    ret void
 ;
 bb:
@@ -319,8 +319,8 @@ define void @avx2_legal512_prefer256_call_avx2_legal256_prefer256(<8 x i64>* %ar
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i64>* [[TMP]] to i8*
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull writeonly align 32 dereferenceable(64) [[TMP3]], i8 0, i64 32, i1 false)
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i64>, <8 x i64>* [[TMP]], align 1
-; CHECK-NEXT:    call fastcc void @callee_avx2_legal512_prefer256_call_avx2_legal256_prefer256(<8 x i64>* noalias nocapture nofree nonnull writeonly align 32 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]])
-; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 32
+; CHECK-NEXT:    call fastcc void @callee_avx2_legal512_prefer256_call_avx2_legal256_prefer256(<8 x i64>* noalias nocapture nofree nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]])
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i64>, <8 x i64>* [[TMP2]], align 64
 ; CHECK-NEXT:    store <8 x i64> [[TMP4]], <8 x i64>* [[ARG]], align 2
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/attrs.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/attrs.ll
index 5a322f0ac361a..6e8a2e5e78f5f 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/attrs.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/attrs.ll
@@ -19,8 +19,8 @@ define internal i32 @f(%struct.ss* byval %b, i32* byval %X, i32 %i) nounwind {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP]], align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[TMP1]], 1
 ; CHECK-NEXT:    store i32 [[TMP2]], i32* [[TMP]], align 8
-; CHECK-NEXT:    store i32 0, i32* [[X_PRIV]]
-; CHECK-NEXT:    [[L:%.*]] = load i32, i32* [[X_PRIV]]
+; CHECK-NEXT:    store i32 0, i32* [[X_PRIV]], align 4
+; CHECK-NEXT:    [[L:%.*]] = load i32, i32* [[X_PRIV]], align 4
 ; CHECK-NEXT:    [[A:%.*]] = add i32 [[L]], [[TMP2]]
 ; CHECK-NEXT:    ret i32 [[A]]
 ;
@@ -40,7 +40,7 @@ entry:
 ; Also make sure we don't drop the call zeroext attribute.
 define i32 @test(i32* %X) {
 ; CHECK-LABEL: define {{[^@]+}}@test
-; CHECK-SAME: (i32* nocapture nofree readonly [[X:%.*]])
+; CHECK-SAME: (i32* nocapture nofree readonly align 4 [[X:%.*]])
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[S:%.*]] = alloca [[STRUCT_SS:%.*]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 0
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/byval-2.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/byval-2.ll
index 39478b7cb24a9..6fdfb47a825da 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/byval-2.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/byval-2.ll
@@ -16,7 +16,7 @@ entry:
 
 define i32 @test(i32* %X) {
 ; CHECK-LABEL: define {{[^@]+}}@test
-; CHECK-SAME: (i32* nocapture nofree readonly [[X:%.*]])
+; CHECK-SAME: (i32* nocapture nofree readonly align 4 [[X:%.*]])
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[S:%.*]] = alloca [[STRUCT_SS:%.*]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr [[STRUCT_SS]], %struct.ss* [[S]], i32 0, i32 0
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/chained.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/chained.ll
index a6e8c77cefe0e..6e9f98007e460 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/chained.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/chained.ll
@@ -8,7 +8,7 @@ define internal i32 @test(i32** %x) {
 ; CHECK-LABEL: define {{[^@]+}}@test()
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[Y:%.*]] = load i32*, i32** @G2, align 8
-; CHECK-NEXT:    [[Z:%.*]] = load i32, i32* [[Y]]
+; CHECK-NEXT:    [[Z:%.*]] = load i32, i32* [[Y]], align 4
 ; CHECK-NEXT:    ret i32 [[Z]]
 ;
 entry:
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/musttail.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/musttail.ll
index dfef64a9366e0..89ee21519e78c 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/musttail.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/musttail.ll
@@ -11,8 +11,8 @@ define internal i32 @test(%T* %p) {
 ; CHECK-SAME: (%T* nocapture nofree readonly [[P:%.*]])
 ; CHECK-NEXT:    [[A_GEP:%.*]] = getelementptr [[T:%.*]], %T* [[P]], i64 0, i32 3
 ; CHECK-NEXT:    [[B_GEP:%.*]] = getelementptr [[T]], %T* [[P]], i64 0, i32 2
-; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[A_GEP]]
-; CHECK-NEXT:    [[B:%.*]] = load i32, i32* [[B_GEP]]
+; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[A_GEP]], align 4
+; CHECK-NEXT:    [[B:%.*]] = load i32, i32* [[B_GEP]], align 4
 ; CHECK-NEXT:    [[V:%.*]] = add i32 [[A]], [[B]]
 ; CHECK-NEXT:    ret i32 [[V]]
 ;
@@ -71,7 +71,7 @@ define i32 @bar(%T* %p, i32 %v) {
 ; CHECK-LABEL: define {{[^@]+}}@bar
 ; CHECK-SAME: (%T* nocapture nofree nonnull writeonly dereferenceable(4) [[P:%.*]], i32 [[V:%.*]])
 ; CHECK-NEXT:    [[I32PTR:%.*]] = getelementptr [[T:%.*]], %T* [[P]], i64 0, i32 0
-; CHECK-NEXT:    store i32 [[V]], i32* [[I32PTR]]
+; CHECK-NEXT:    store i32 [[V]], i32* [[I32PTR]], align 4
 ; CHECK-NEXT:    ret i32 0
 ;
   %i32ptr = getelementptr %T, %T* %p, i64 0, i32 0
@@ -84,8 +84,8 @@ define internal i32 @test2b(%T* %p, i32 %p2) {
 ; CHECK-SAME: (%T* nocapture nofree readonly [[P:%.*]], i32 [[P2:%.*]])
 ; CHECK-NEXT:    [[A_GEP:%.*]] = getelementptr [[T:%.*]], %T* [[P]], i64 0, i32 3
 ; CHECK-NEXT:    [[B_GEP:%.*]] = getelementptr [[T]], %T* [[P]], i64 0, i32 2
-; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[A_GEP]]
-; CHECK-NEXT:    [[B:%.*]] = load i32, i32* [[B_GEP]]
+; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[A_GEP]], align 4
+; CHECK-NEXT:    [[B:%.*]] = load i32, i32* [[B_GEP]], align 4
 ; CHECK-NEXT:    [[V:%.*]] = add i32 [[A]], [[B]]
 ; CHECK-NEXT:    [[CA:%.*]] = musttail call i32 @bar(%T* undef, i32 [[V]])
 ; CHECK-NEXT:    ret i32 [[CA]]
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/sret.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/sret.ll
index 33048a31adbca..e0cb81fbcbf3e 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/sret.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/sret.ll
@@ -10,7 +10,7 @@ define internal void @add({i32, i32}* %this, i32* sret %r) {
 ; CHECK-NEXT:    [[AP:%.*]] = getelementptr { i32, i32 }, { i32, i32 }* [[THIS]], i32 0, i32 0
 ; CHECK-NEXT:    [[BP:%.*]] = getelementptr { i32, i32 }, { i32, i32 }* [[THIS]], i32 0, i32 1
 ; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[AP]], align 8
-; CHECK-NEXT:    [[B:%.*]] = load i32, i32* [[BP]]
+; CHECK-NEXT:    [[B:%.*]] = load i32, i32* [[BP]], align 4
 ; CHECK-NEXT:    [[AB:%.*]] = add i32 [[A]], [[B]]
 ; CHECK-NEXT:    store i32 [[AB]], i32* [[R]], align 4
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/Attributor/align.ll b/llvm/test/Transforms/Attributor/align.ll
index ae8a3dbb1148e..882d0e8b8c7f6 100644
--- a/llvm/test/Transforms/Attributor/align.ll
+++ b/llvm/test/Transforms/Attributor/align.ll
@@ -552,6 +552,23 @@ define void @align_store_after_bc(i32* align 2048 %arg) {
   store i8 0, i8* %bc
   ret void
 }
+
+; Make sure we do not annotate the callee of a must-tail call with an alignment
+; we cannot also put on the caller.
+@cnd = external global i1
+define i32 @musttail_callee_1(i32* %p) {
+  %v = load i32, i32* %p, align 32
+  ret i32 %v
+}
+define i32 @musttail_caller_1(i32* %p) {
+  %c = load i1, i1* @cnd
+  br i1 %c, label %mt, label %exit
+mt:
+  %v = musttail call i32 @musttail_callee_1(i32* %p)
+  ret i32 %v
+exit:
+  ret i32 0
+}
 ; UTC_ARGS: --disable
 
 attributes #0 = { nounwind uwtable noinline }
diff --git a/llvm/test/Transforms/Attributor/callbacks.ll b/llvm/test/Transforms/Attributor/callbacks.ll
index fb0984a835a6d..12af0c8e1a7e0 100644
--- a/llvm/test/Transforms/Attributor/callbacks.ll
+++ b/llvm/test/Transforms/Attributor/callbacks.ll
@@ -43,10 +43,10 @@ entry:
 ; The others are annotated with alignment information, amongst others, or even replaced by the constants passed to the call.
 define internal void @t0_callback_callee(i32* %is_not_null, i32* %ptr, i32* %a, i64 %b, i32** %c) {
 ; CHECK-LABEL: define {{[^@]+}}@t0_callback_callee
-; CHECK-SAME: (i32* nocapture nonnull writeonly dereferenceable(4) [[IS_NOT_NULL:%.*]], i32* nocapture nonnull readonly align 8 dereferenceable(4) [[PTR:%.*]], i32* align 256 [[A:%.*]], i64 [[B:%.*]], i32** noalias nocapture nonnull readonly align 64 dereferenceable(8) [[C:%.*]])
+; CHECK-SAME: (i32* nocapture nonnull writeonly align 4 dereferenceable(4) [[IS_NOT_NULL:%.*]], i32* nocapture nonnull readonly align 8 dereferenceable(4) [[PTR:%.*]], i32* align 256 [[A:%.*]], i64 [[B:%.*]], i32** noalias nocapture nonnull readonly align 64 dereferenceable(8) [[C:%.*]])
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[PTR_VAL:%.*]] = load i32, i32* [[PTR]], align 8
-; CHECK-NEXT:    store i32 [[PTR_VAL]], i32* [[IS_NOT_NULL]]
+; CHECK-NEXT:    store i32 [[PTR_VAL]], i32* [[IS_NOT_NULL]], align 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32*, i32** [[C]], align 64
 ; CHECK-NEXT:    tail call void @t0_check(i32* align 256 [[A]], i64 99, i32* [[TMP0]])
 ; CHECK-NEXT:    ret void
@@ -96,10 +96,10 @@ entry:
 ; The others are annotated with alignment information, amongst others, or even replaced by the constants passed to the call.
 define internal void @t1_callback_callee(i32* %is_not_null, i32* %ptr, i32* %a, i64 %b, i32** %c) {
 ; CHECK-LABEL: define {{[^@]+}}@t1_callback_callee
-; CHECK-SAME: (i32* nocapture nonnull writeonly dereferenceable(4) [[IS_NOT_NULL:%.*]], i32* nocapture nonnull readonly align 8 dereferenceable(4) [[PTR:%.*]], i32* noalias nocapture align 256 [[A:%.*]], i64 [[B:%.*]], i32** noalias nocapture nonnull readonly align 64 dereferenceable(8) [[C:%.*]])
+; CHECK-SAME: (i32* nocapture nonnull writeonly align 4 dereferenceable(4) [[IS_NOT_NULL:%.*]], i32* nocapture nonnull readonly align 8 dereferenceable(4) [[PTR:%.*]], i32* noalias nocapture align 256 [[A:%.*]], i64 [[B:%.*]], i32** noalias nocapture nonnull readonly align 64 dereferenceable(8) [[C:%.*]])
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[PTR_VAL:%.*]] = load i32, i32* [[PTR]], align 8
-; CHECK-NEXT:    store i32 [[PTR_VAL]], i32* [[IS_NOT_NULL]]
+; CHECK-NEXT:    store i32 [[PTR_VAL]], i32* [[IS_NOT_NULL]], align 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32*, i32** [[C]], align 64
 ; CHECK-NEXT:    tail call void @t1_check(i32* nocapture align 256 [[A]], i64 99, i32* [[TMP0]])
 ; CHECK-NEXT:    ret void
@@ -150,10 +150,10 @@ entry:
 ; FIXME: We should derive noalias for %a and add a "fake use" of %a in all potentially synchronizing calls.
 define internal void @t2_callback_callee(i32* %is_not_null, i32* %ptr, i32* %a, i64 %b, i32** %c) {
 ; CHECK-LABEL: define {{[^@]+}}@t2_callback_callee
-; CHECK-SAME: (i32* nocapture nonnull writeonly dereferenceable(4) [[IS_NOT_NULL:%.*]], i32* nocapture nonnull readonly align 8 dereferenceable(4) [[PTR:%.*]], i32* nocapture align 256 [[A:%.*]], i64 [[B:%.*]], i32** noalias nocapture nonnull readonly align 64 dereferenceable(8) [[C:%.*]])
+; CHECK-SAME: (i32* nocapture nonnull writeonly align 4 dereferenceable(4) [[IS_NOT_NULL:%.*]], i32* nocapture nonnull readonly align 8 dereferenceable(4) [[PTR:%.*]], i32* nocapture align 256 [[A:%.*]], i64 [[B:%.*]], i32** noalias nocapture nonnull readonly align 64 dereferenceable(8) [[C:%.*]])
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[PTR_VAL:%.*]] = load i32, i32* [[PTR]], align 8
-; CHECK-NEXT:    store i32 [[PTR_VAL]], i32* [[IS_NOT_NULL]]
+; CHECK-NEXT:    store i32 [[PTR_VAL]], i32* [[IS_NOT_NULL]], align 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32*, i32** [[C]], align 64
 ; CHECK-NEXT:    tail call void @t2_check(i32* nocapture align 256 [[A]], i64 99, i32* [[TMP0]])
 ; CHECK-NEXT:    ret void
@@ -206,10 +206,10 @@ entry:
 ; FIXME: We should derive noalias for %a and add a "fake use" of %a in all potentially synchronizing calls.
 define internal void @t3_callback_callee(i32* %is_not_null, i32* %ptr, i32* %a, i64 %b, i32** %c) {
 ; CHECK-LABEL: define {{[^@]+}}@t3_callback_callee
-; CHECK-SAME: (i32* nocapture nonnull writeonly dereferenceable(4) [[IS_NOT_NULL:%.*]], i32* nocapture nonnull readonly align 8 dereferenceable(4) [[PTR:%.*]], i32* nocapture align 256 [[A:%.*]], i64 [[B:%.*]], i32** noalias nocapture nonnull readonly align 64 dereferenceable(8) [[C:%.*]])
+; CHECK-SAME: (i32* nocapture nonnull writeonly align 4 dereferenceable(4) [[IS_NOT_NULL:%.*]], i32* nocapture nonnull readonly align 8 dereferenceable(4) [[PTR:%.*]], i32* nocapture align 256 [[A:%.*]], i64 [[B:%.*]], i32** noalias nocapture nonnull readonly align 64 dereferenceable(8) [[C:%.*]])
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[PTR_VAL:%.*]] = load i32, i32* [[PTR]], align 8
-; CHECK-NEXT:    store i32 [[PTR_VAL]], i32* [[IS_NOT_NULL]]
+; CHECK-NEXT:    store i32 [[PTR_VAL]], i32* [[IS_NOT_NULL]], align 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32*, i32** [[C]], align 64
 ; CHECK-NEXT:    tail call void @t3_check(i32* nocapture align 256 [[A]], i64 99, i32* [[TMP0]])
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/Attributor/dereferenceable-1.ll b/llvm/test/Transforms/Attributor/dereferenceable-1.ll
index 459e57e5c2ef9..7a6de53f3f694 100644
--- a/llvm/test/Transforms/Attributor/dereferenceable-1.ll
+++ b/llvm/test/Transforms/Attributor/dereferenceable-1.ll
@@ -124,28 +124,28 @@ define i32* @f7_0(i32* %ptr) {
   ret i32* %ptr
 }
 
-; ATTRIBUTOR: define void @f7_1(i32* nonnull dereferenceable(4) %ptr, i1 %c)
+; ATTRIBUTOR: define void @f7_1(i32* nonnull align 4 dereferenceable(4) %ptr, i1 %c)
 define void @f7_1(i32* %ptr, i1 %c) {
 
-; ATTRIBUTOR:   %A = tail call i32 @unkown_f(i32* nonnull dereferenceable(4) %ptr)
+; ATTRIBUTOR:   %A = tail call i32 @unkown_f(i32* nonnull align 4 dereferenceable(4) %ptr)
   %A = tail call i32 @unkown_f(i32* %ptr)
 
   %ptr.0 = load i32, i32* %ptr
   ; deref 4 hold
 
 ; FIXME: this should be %B = tail call i32 @unkown_f(i32* nonnull dereferenceable(4) %ptr)
-; ATTRIBUTOR:   %B = tail call i32 @unkown_f(i32* nonnull dereferenceable(4) %ptr)
+; ATTRIBUTOR:   %B = tail call i32 @unkown_f(i32* nonnull align 4 dereferenceable(4) %ptr)
   %B = tail call i32 @unkown_f(i32* dereferenceable(1) %ptr)
 
   br i1%c, label %if.true, label %if.false
 if.true:
-; ATTRIBUTOR:   %C = tail call i32 @unkown_f(i32* nonnull dereferenceable(8) %ptr)
+; ATTRIBUTOR:   %C = tail call i32 @unkown_f(i32* nonnull align 4 dereferenceable(8) %ptr)
   %C = tail call i32 @unkown_f(i32* %ptr)
 
-; ATTRIBUTOR:   %D = tail call i32 @unkown_f(i32* nonnull dereferenceable(8) %ptr)
+; ATTRIBUTOR:   %D = tail call i32 @unkown_f(i32* nonnull align 4 dereferenceable(8) %ptr)
   %D = tail call i32 @unkown_f(i32* dereferenceable(8) %ptr)
 
-; ATTRIBUTOR:   %E = tail call i32 @unkown_f(i32* nonnull dereferenceable(8) %ptr)
+; ATTRIBUTOR:   %E = tail call i32 @unkown_f(i32* nonnull align 4 dereferenceable(8) %ptr)
   %E = tail call i32 @unkown_f(i32* %ptr)
 
   ret void
@@ -159,26 +159,26 @@ define void @f7_2(i1 %c) {
 
   %ptr =  tail call i32* @unkown_ptr()
 
-; ATTRIBUTOR:   %A = tail call i32 @unkown_f(i32* nonnull dereferenceable(4) %ptr)
+; ATTRIBUTOR:   %A = tail call i32 @unkown_f(i32* nonnull align 4 dereferenceable(4) %ptr)
   %A = tail call i32 @unkown_f(i32* %ptr)
 
   %arg_a.0 = load i32, i32* %ptr
   ; deref 4 hold
 
-; ATTRIBUTOR:   %B = tail call i32 @unkown_f(i32* nonnull dereferenceable(4) %ptr)
+; ATTRIBUTOR:   %B = tail call i32 @unkown_f(i32* nonnull align 4 dereferenceable(4) %ptr)
   %B = tail call i32 @unkown_f(i32* dereferenceable(1) %ptr)
 
   br i1%c, label %if.true, label %if.false
 if.true:
 
-; ATTRIBUTOR:   %C = tail call i32 @unkown_f(i32* nonnull dereferenceable(8) %ptr)
+; ATTRIBUTOR:   %C = tail call i32 @unkown_f(i32* nonnull align 4 dereferenceable(8) %ptr)
   %C = tail call i32 @unkown_f(i32* %ptr)
 
-; ATTRIBUTOR:   %D = tail call i32 @unkown_f(i32* nonnull dereferenceable(8) %ptr)
+; ATTRIBUTOR:   %D = tail call i32 @unkown_f(i32* nonnull align 4 dereferenceable(8) %ptr)
   %D = tail call i32 @unkown_f(i32* dereferenceable(8) %ptr)
 
   %E = tail call i32 @unkown_f(i32* %ptr)
-; ATTRIBUTOR:   %E = tail call i32 @unkown_f(i32* nonnull dereferenceable(8) %ptr)
+; ATTRIBUTOR:   %E = tail call i32 @unkown_f(i32* nonnull align 4 dereferenceable(8) %ptr)
 
   ret void
 
@@ -195,14 +195,14 @@ define i32* @f7_3() {
 
 define i32* @test_for_minus_index(i32* %p) {
 ; FIXME: This should have a return dereferenceable(8) but we need to make sure it will work in loops as well.
-; ATTRIBUTOR: define nonnull i32* @test_for_minus_index(i32* nofree nonnull writeonly "no-capture-maybe-returned" %p)
+; ATTRIBUTOR: define nonnull align 4 i32* @test_for_minus_index(i32* nofree nonnull writeonly align 4 "no-capture-maybe-returned" %p)
   %q = getelementptr inbounds i32, i32* %p, i32 -2
   store i32 1, i32* %q
   ret i32* %q
 }
 
 define void @deref_or_null_and_nonnull(i32* dereferenceable_or_null(100) %0) {
-; ATTRIBUTOR: define void @deref_or_null_and_nonnull(i32* nocapture nofree nonnull writeonly dereferenceable(100) %0)
+; ATTRIBUTOR: define void @deref_or_null_and_nonnull(i32* nocapture nofree nonnull writeonly align 4 dereferenceable(100) %0)
   store i32 1, i32* %0
   ret void
 }
diff --git a/llvm/test/Transforms/Attributor/dereferenceable-2.ll b/llvm/test/Transforms/Attributor/dereferenceable-2.ll
index f2a3e3a25f8b4..11122e081c42a 100644
--- a/llvm/test/Transforms/Attributor/dereferenceable-2.ll
+++ b/llvm/test/Transforms/Attributor/dereferenceable-2.ll
@@ -74,7 +74,7 @@ define void @gep0(i8* %unused, i8* %other, i8* %ptr) {
 ; Multiple arguments may be dereferenceable.
 
 define void @ordering(i8* %ptr1, i32* %ptr2) {
-; ATTRIBUTOR-LABEL: @ordering(i8* nocapture nofree nonnull readnone dereferenceable(3) %ptr1, i32* nocapture nofree nonnull readnone dereferenceable(8) %ptr2)
+; ATTRIBUTOR-LABEL: @ordering(i8* nocapture nofree nonnull readnone dereferenceable(3) %ptr1, i32* nocapture nofree nonnull readnone align 4 dereferenceable(8) %ptr2)
   %a20 = getelementptr i32, i32* %ptr2, i64 0
   %a12 = getelementptr i8, i8* %ptr1, i64 2
   %t12 = load i8, i8* %a12
@@ -125,7 +125,7 @@ exit:
 ; The last load may not execute, so derefenceable bytes only covers the 1st two loads.
 
 define void @partial_in_entry(i16* %ptr, i1 %cond) {
-; ATTRIBUTOR-LABEL: @partial_in_entry(i16* nocapture nofree nonnull readnone dereferenceable(4) %ptr, i1 %cond)
+; ATTRIBUTOR-LABEL: @partial_in_entry(i16* nocapture nofree nonnull readnone align 2 dereferenceable(4) %ptr, i1 %cond)
 entry:
   %arrayidx0 = getelementptr i16, i16* %ptr, i64 0
   %arrayidx1 = getelementptr i16, i16* %ptr, i64 1
@@ -144,7 +144,7 @@ exit:
 ; The 2nd and 3rd loads may never execute.
 
 define void @volatile_is_not_dereferenceable(i16* %ptr) {
-; ATTRIBUTOR-LABEL: @volatile_is_not_dereferenceable(i16* nofree %ptr)
+; ATTRIBUTOR-LABEL: @volatile_is_not_dereferenceable(i16* nofree align 2 %ptr)
   %arrayidx0 = getelementptr i16, i16* %ptr, i64 0
   %arrayidx1 = getelementptr i16, i16* %ptr, i64 1
   %arrayidx2 = getelementptr i16, i16* %ptr, i64 2
@@ -170,7 +170,7 @@ define void @atomic_is_alright(i16* %ptr) {
 declare void @may_not_return()
 
 define void @not_guaranteed_to_transfer_execution(i16* %ptr) {
-; ATTRIBUTOR-LABEL: @not_guaranteed_to_transfer_execution(i16* nocapture nofree nonnull readnone dereferenceable(2) %ptr)
+; ATTRIBUTOR-LABEL: @not_guaranteed_to_transfer_execution(i16* nocapture nofree nonnull readnone align 2 dereferenceable(2) %ptr)
   %arrayidx0 = getelementptr i16, i16* %ptr, i64 0
   %arrayidx1 = getelementptr i16, i16* %ptr, i64 1
   %arrayidx2 = getelementptr i16, i16* %ptr, i64 2
@@ -206,7 +206,7 @@ define void @multi_index_gep(<4 x i8>* %ptr) {
 ; Could round weird bitwidths down?
 
 define void @not_byte_multiple(i9* %ptr) {
-; ATTRIBUTOR-LABEL: @not_byte_multiple(i9* nocapture nofree nonnull readnone dereferenceable(2) %ptr) 
+; ATTRIBUTOR-LABEL: @not_byte_multiple(i9* nocapture nofree nonnull readnone align 2 dereferenceable(2) %ptr) 
   %arrayidx0 = getelementptr i9, i9* %ptr, i64 0
   %t0 = load i9, i9* %arrayidx0
   ret void
@@ -215,7 +215,7 @@ define void @not_byte_multiple(i9* %ptr) {
 ; Missing direct access from the pointer.
 
 define void @no_pointer_deref(i16* %ptr) {
-; ATTRIBUTOR-LABEL: @no_pointer_deref(i16* nocapture nofree readnone %ptr)
+; ATTRIBUTOR-LABEL: @no_pointer_deref(i16* nocapture nofree readnone align 2 %ptr)
   %arrayidx1 = getelementptr i16, i16* %ptr, i64 1
   %arrayidx2 = getelementptr i16, i16* %ptr, i64 2
   %t1 = load i16, i16* %arrayidx1
@@ -226,7 +226,7 @@ define void @no_pointer_deref(i16* %ptr) {
 ; Out-of-order is ok, but missing access concludes dereferenceable range.
 
 define void @non_consecutive(i32* %ptr) {
-; ATTRIBUTOR-LABEL: @non_consecutive(i32* nocapture nofree nonnull readnone dereferenceable(8) %ptr)
+; ATTRIBUTOR-LABEL: @non_consecutive(i32* nocapture nofree nonnull readnone align 4 dereferenceable(8) %ptr)
   %arrayidx1 = getelementptr i32, i32* %ptr, i64 1
   %arrayidx0 = getelementptr i32, i32* %ptr, i64 0
   %arrayidx3 = getelementptr i32, i32* %ptr, i64 3
@@ -239,7 +239,7 @@ define void @non_consecutive(i32* %ptr) {
 ; Improve on existing dereferenceable attribute.
 
 define void @more_bytes(i32* dereferenceable(8) %ptr) {
-; ATTRIBUTOR-LABEL: @more_bytes(i32* nocapture nofree nonnull readnone dereferenceable(16) %ptr)
+; ATTRIBUTOR-LABEL: @more_bytes(i32* nocapture nofree nonnull readnone align 4 dereferenceable(16) %ptr)
   %arrayidx3 = getelementptr i32, i32* %ptr, i64 3
   %arrayidx1 = getelementptr i32, i32* %ptr, i64 1
   %arrayidx0 = getelementptr i32, i32* %ptr, i64 0
@@ -254,7 +254,7 @@ define void @more_bytes(i32* dereferenceable(8) %ptr) {
 ; Improve on existing dereferenceable_or_null attribute.
 
 define void @more_bytes_and_not_null(i32* dereferenceable_or_null(8) %ptr) {
-; ATTRIBUTOR-LABEL: @more_bytes_and_not_null(i32* nocapture nofree nonnull readnone dereferenceable(16) %ptr)
+; ATTRIBUTOR-LABEL: @more_bytes_and_not_null(i32* nocapture nofree nonnull readnone align 4 dereferenceable(16) %ptr)
   %arrayidx3 = getelementptr i32, i32* %ptr, i64 3
   %arrayidx1 = getelementptr i32, i32* %ptr, i64 1
   %arrayidx0 = getelementptr i32, i32* %ptr, i64 0
@@ -269,7 +269,7 @@ define void @more_bytes_and_not_null(i32* dereferenceable_or_null(8) %ptr) {
 ; But don't pessimize existing dereferenceable attribute.
 
 define void @better_bytes(i32* dereferenceable(100) %ptr) {
-; ATTRIBUTOR-LABEL: @better_bytes(i32* nocapture nofree nonnull readnone dereferenceable(100) %ptr)
+; ATTRIBUTOR-LABEL: @better_bytes(i32* nocapture nofree nonnull readnone align 4 dereferenceable(100) %ptr)
   %arrayidx3 = getelementptr i32, i32* %ptr, i64 3
   %arrayidx1 = getelementptr i32, i32* %ptr, i64 1
   %arrayidx0 = getelementptr i32, i32* %ptr, i64 0
@@ -282,7 +282,7 @@ define void @better_bytes(i32* dereferenceable(100) %ptr) {
 }
 
 define void @bitcast(i32* %arg) {
-; ATTRIBUTOR-LABEL: @bitcast(i32* nocapture nofree nonnull readnone dereferenceable(8) %arg)
+; ATTRIBUTOR-LABEL: @bitcast(i32* nocapture nofree nonnull readnone align 4 dereferenceable(8) %arg)
   %ptr = bitcast i32* %arg to float*
   %arrayidx0 = getelementptr float, float* %ptr, i64 0
   %arrayidx1 = getelementptr float, float* %ptr, i64 1
@@ -292,7 +292,7 @@ define void @bitcast(i32* %arg) {
 }
 
 define void @bitcast_different_sizes(double* %arg1, i8* %arg2) {
-; ATTRIBUTOR-LABEL: @bitcast_different_sizes(double* nocapture nofree nonnull readnone dereferenceable(12) %arg1, i8* nocapture nofree nonnull readnone dereferenceable(16) %arg2)
+; ATTRIBUTOR-LABEL: @bitcast_different_sizes(double* nocapture nofree nonnull readnone align 4 dereferenceable(12) %arg1, i8* nocapture nofree nonnull readnone align 4 dereferenceable(16) %arg2)
   %ptr1 = bitcast double* %arg1 to float*
   %a10 = getelementptr float, float* %ptr1, i64 0
   %a11 = getelementptr float, float* %ptr1, i64 1
@@ -310,7 +310,7 @@ define void @bitcast_different_sizes(double* %arg1, i8* %arg2) {
 }
 
 define void @negative_offset(i32* %arg) {
-; ATTRIBUTOR-LABEL: @negative_offset(i32* nocapture nofree nonnull readnone dereferenceable(4) %arg)
+; ATTRIBUTOR-LABEL: @negative_offset(i32* nocapture nofree nonnull readnone align 4 dereferenceable(4) %arg)
   %ptr = bitcast i32* %arg to float*
   %arrayidx0 = getelementptr float, float* %ptr, i64 0
   %arrayidx1 = getelementptr float, float* %ptr, i64 -1
@@ -320,7 +320,7 @@ define void @negative_offset(i32* %arg) {
 }
 
 define void @stores(i32* %arg) {
-; ATTRIBUTOR-LABEL: @stores(i32* nocapture nofree nonnull writeonly dereferenceable(8) %arg)
+; ATTRIBUTOR-LABEL: @stores(i32* nocapture nofree nonnull writeonly align 4 dereferenceable(8) %arg)
   %ptr = bitcast i32* %arg to float*
   %arrayidx0 = getelementptr float, float* %ptr, i64 0
   %arrayidx1 = getelementptr float, float* %ptr, i64 1
@@ -330,7 +330,7 @@ define void @stores(i32* %arg) {
 }
 
 define void @load_store(i32* %arg) {
-; ATTRIBUTOR-LABEL: @load_store(i32* nocapture nofree nonnull writeonly dereferenceable(8) %arg)
+; ATTRIBUTOR-LABEL: @load_store(i32* nocapture nofree nonnull writeonly align 4 dereferenceable(8) %arg)
   %ptr = bitcast i32* %arg to float*
   %arrayidx0 = getelementptr float, float* %ptr, i64 0
   %arrayidx1 = getelementptr float, float* %ptr, i64 1
@@ -340,7 +340,7 @@ define void @load_store(i32* %arg) {
 }
 
 define void @different_size1(i32* %arg) {
-; ATTRIBUTOR-LABEL: @different_size1(i32* nocapture nofree nonnull writeonly dereferenceable(8) %arg)
+; ATTRIBUTOR-LABEL: @different_size1(i32* nocapture nofree nonnull writeonly align 8 dereferenceable(8) %arg)
   %arg-cast = bitcast i32* %arg to double*
   store double 0.000000e+00, double* %arg-cast
   store i32 0, i32* %arg
@@ -348,7 +348,7 @@ define void @different_size1(i32* %arg) {
 }
 
 define void @different_size2(i32* %arg) {
-; ATTRIBUTOR-LABEL: @different_size2(i32* nocapture nofree nonnull writeonly dereferenceable(8) %arg)
+; ATTRIBUTOR-LABEL: @different_size2(i32* nocapture nofree nonnull writeonly align 8 dereferenceable(8) %arg)
   store i32 0, i32* %arg
   %arg-cast = bitcast i32* %arg to double*
   store double 0.000000e+00, double* %arg-cast
diff --git a/llvm/test/Transforms/Attributor/heap_to_stack.ll b/llvm/test/Transforms/Attributor/heap_to_stack.ll
index 8dc9b945d589e..707aea7bfda89 100644
--- a/llvm/test/Transforms/Attributor/heap_to_stack.ll
+++ b/llvm/test/Transforms/Attributor/heap_to_stack.ll
@@ -76,6 +76,26 @@ define void @test3a(i8* %p) {
   ret void
 }
 
+declare noalias i8* @aligned_alloc(i64, i64)
+
+define void @test3b(i8* %p) {
+  %1 = tail call noalias i8* @aligned_alloc(i64 32, i64 128)
+  ; CHECK: %1 = alloca i8, i64 128, align 32
+  ; CHECK-NEXT: tail call void @nofree_arg_only
+  tail call void @nofree_arg_only(i8* %1, i8* %p)
+  ; CHECK-NOT: @free(i8* %1)
+  tail call void @free(i8* %1)
+  ret void
+}
+
+; leave alone non-constant alignments.
+define void @test3c(i64 %alignment) {
+  %1 = tail call noalias i8* @aligned_alloc(i64 %alignment, i64 128)
+  ; CHECK: tail call noalias i8* @aligned_alloc
+  tail call void @free(i8* %1)
+  ret void
+}
+
 declare noalias i8* @calloc(i64, i64)
 
 define void @test0() {
@@ -90,7 +110,7 @@ define void @test0() {
   ret void
 }
 
-; TEST 4 
+; TEST 4
 define void @test4() {
   %1 = tail call noalias i8* @malloc(i64 4)
   ; CHECK: %1 = alloca i8, i64 4
@@ -170,7 +190,7 @@ define void @test8() {
   store i32 10, i32* %2
   %3 = load i32, i32* %2
   tail call void @foo(i32* %2)
-  ; CHECK: @free(i8* nonnull dereferenceable(4) %1)
+  ; CHECK: @free(i8* nonnull align 4 dereferenceable(4) %1)
   tail call void @free(i8* %1)
   ret void
 }
@@ -185,7 +205,7 @@ define void @test9() {
   store i32 10, i32* %2
   %3 = load i32, i32* %2
   tail call void @foo_nounw(i32* %2)
-  ; CHECK: @free(i8* nonnull dereferenceable(4) %1)
+  ; CHECK: @free(i8* nonnull align 4 dereferenceable(4) %1)
   tail call void @free(i8* %1)
   ret void
 }
@@ -219,7 +239,7 @@ define i32 @test_lifetime() {
   ret i32 %3
 }
 
-; TEST 11 
+; TEST 11
 
 define void @test11() {
   %1 = tail call noalias i8* @malloc(i64 4)
@@ -307,7 +327,7 @@ define i32 @test13() {
   store i32 10, i32* %2
   %3 = load i32, i32* %2
   tail call void @free(i8* %1)
-  ; CHECK: tail call void @free(i8* noalias nonnull dereferenceable(4) %1)
+  ; CHECK: tail call void @free(i8* noalias nonnull align 4 dereferenceable(4) %1)
   ret i32 %3
 }
 
@@ -320,7 +340,7 @@ define i32 @test_sle() {
   store i32 10, i32* %2
   %3 = load i32, i32* %2
   tail call void @free(i8* %1)
-  ; CHECK: tail call void @free(i8* noalias nonnull dereferenceable(4) %1)
+  ; CHECK: tail call void @free(i8* noalias nonnull align 4 dereferenceable(4) %1)
   ret i32 %3
 }
 
@@ -333,7 +353,7 @@ define i32 @test_overflow() {
   store i32 10, i32* %2
   %3 = load i32, i32* %2
   tail call void @free(i8* %1)
-  ; CHECK: tail call void @free(i8* noalias nonnull dereferenceable(4) %1)
+  ; CHECK: tail call void @free(i8* noalias nonnull align 4 dereferenceable(4) %1)
   ret i32 %3
 }
 
diff --git a/llvm/test/Transforms/Attributor/misc.ll b/llvm/test/Transforms/Attributor/misc.ll
index 5f0a91128fccd..2b79bdbee21f0 100644
--- a/llvm/test/Transforms/Attributor/misc.ll
+++ b/llvm/test/Transforms/Attributor/misc.ll
@@ -89,9 +89,9 @@ entry:
 
 define internal void @foo(i32* %a) {
 ; ALL-LABEL: define {{[^@]+}}@foo
-; ALL-SAME: (i32* nocapture nofree nonnull writeonly dereferenceable(4) [[A:%.*]])
+; ALL-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[A:%.*]])
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    store i32 0, i32* [[A]]
+; ALL-NEXT:    store i32 0, i32* [[A]], align 4
 ; ALL-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/Attributor/range.ll b/llvm/test/Transforms/Attributor/range.ll
index 1e93e97634b24..efb2df556af63 100644
--- a/llvm/test/Transforms/Attributor/range.ll
+++ b/llvm/test/Transforms/Attributor/range.ll
@@ -8,8 +8,8 @@
 
 define i32 @test0(i32* %p) {
 ; CHECK-LABEL: define {{[^@]+}}@test0
-; CHECK-SAME: (i32* nocapture nofree nonnull readonly dereferenceable(4) [[P:%.*]])
-; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[P]], !range !0
+; CHECK-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P:%.*]])
+; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[P]], align 4, !range !0
 ; CHECK-NEXT:    ret i32 [[A]]
 ;
   %a = load i32, i32* %p, !range !0
@@ -19,33 +19,25 @@ define i32 @test0(i32* %p) {
 define i32 @test0-range-check(i32* %p) {
 ;
 ; OLD_PM-LABEL: define {{[^@]+}}@test0-range-check
-; OLD_PM-SAME: (i32* nocapture nofree readonly [[P:%.*]])
-; OLD_PM-NEXT:    [[A:%.*]] = tail call i32 @test0(i32* nocapture nofree readonly [[P]]) #{{[0-9]+}}, !range !0
+; OLD_PM-SAME: (i32* nocapture nofree readonly align 4 [[P:%.*]])
+; OLD_PM-NEXT:    [[A:%.*]] = tail call i32 @test0(i32* nocapture nofree readonly align 4 [[P]]) #{{[0-9]+}}, !range !0
 ; OLD_PM-NEXT:    ret i32 [[A]]
 ;
 ; NEW_PM-LABEL: define {{[^@]+}}@test0-range-check
-; NEW_PM-SAME: (i32* nocapture nofree readonly [[P:%.*]])
-; NEW_PM-NEXT:    [[A:%.*]] = tail call i32 @test0(i32* nocapture nofree readonly [[P]]) #{{[0-9]+}}, !range !0
+; NEW_PM-SAME: (i32* nocapture nofree readonly align 4 [[P:%.*]])
+; NEW_PM-NEXT:    [[A:%.*]] = tail call i32 @test0(i32* nocapture nofree readonly align 4 [[P]]) #{{[0-9]+}}, !range !0
 ; NEW_PM-NEXT:    ret i32 [[A]]
 ;
 ; CGSCC_OLD_PM-LABEL: define {{[^@]+}}@test0-range-check
-; CGSCC_OLD_PM-SAME: (i32* nocapture nofree nonnull readonly dereferenceable(4) [[P:%.*]])
-; CGSCC_OLD_PM-NEXT:    [[A:%.*]] = tail call i32 @test0(i32* nocapture nofree nonnull readonly dereferenceable(4) [[P]])
+; CGSCC_OLD_PM-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P:%.*]])
+; CGSCC_OLD_PM-NEXT:    [[A:%.*]] = tail call i32 @test0(i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P]])
 ; CGSCC_OLD_PM-NEXT:    ret i32 [[A]]
 ;
 ; CGSCC_NEW_PM-LABEL: define {{[^@]+}}@test0-range-check
-; CGSCC_NEW_PM-SAME: (i32* nocapture nofree nonnull readonly dereferenceable(4) [[P:%.*]])
-; CGSCC_NEW_PM-NEXT:    [[A:%.*]] = tail call i32 @test0(i32* nocapture nofree nonnull readonly dereferenceable(4) [[P]])
+; CGSCC_NEW_PM-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P:%.*]])
+; CGSCC_NEW_PM-NEXT:    [[A:%.*]] = tail call i32 @test0(i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P]])
 ; CGSCC_NEW_PM-NEXT:    ret i32 [[A]]
 ;
-; MODULE-LABEL: define {{[^@]+}}@test0-range-check
-; MODULE-SAME: (i32* nocapture nofree readonly [[P:%.*]])
-; MODULE-NEXT:    [[A:%.*]] = tail call i32 @test0(i32* nocapture nofree readonly [[P]]) #{{[0-9]+}}, !range !0
-; MODULE-NEXT:    ret i32 [[A]]
-; CGSCC-LABEL: define {{[^@]+}}@test0-range-check
-; CGSCC-SAME: (i32* nocapture nofree nonnull readonly dereferenceable(4) [[P:%.*]])
-; CGSCC-NEXT:    [[A:%.*]] = tail call i32 @test0(i32* nocapture nofree nonnull readonly dereferenceable(4) [[P]])
-; CGSCC-NEXT:    ret i32 [[A]]
   %a = tail call i32 @test0(i32* %p)
   ret i32 %a
 }
@@ -64,8 +56,8 @@ define void @use3(i1, i1, i1) {
 ; TEST0 icmp test
 define void @test0-icmp-check(i32* %p){
 ; OLD_PM-LABEL: define {{[^@]+}}@test0-icmp-check
-; OLD_PM-SAME: (i32* nocapture nofree readonly [[P:%.*]])
-; OLD_PM-NEXT:    [[RET:%.*]] = tail call i32 @test0(i32* nocapture nofree readonly [[P]]) #{{[0-9]+}}, !range !0
+; OLD_PM-SAME: (i32* nocapture nofree readonly align 4 [[P:%.*]])
+; OLD_PM-NEXT:    [[RET:%.*]] = tail call i32 @test0(i32* nocapture nofree readonly align 4 [[P]]) #{{[0-9]+}}, !range !0
 ; OLD_PM-NEXT:    [[CMP_EQ_2:%.*]] = icmp eq i32 [[RET]], 9
 ; OLD_PM-NEXT:    [[CMP_EQ_3:%.*]] = icmp eq i32 [[RET]], 8
 ; OLD_PM-NEXT:    [[CMP_EQ_4:%.*]] = icmp eq i32 [[RET]], 1
@@ -111,8 +103,8 @@ define void @test0-icmp-check(i32* %p){
 ; OLD_PM-NEXT:    ret void
 ;
 ; NEW_PM-LABEL: define {{[^@]+}}@test0-icmp-check
-; NEW_PM-SAME: (i32* nocapture nofree readonly [[P:%.*]])
-; NEW_PM-NEXT:    [[RET:%.*]] = tail call i32 @test0(i32* nocapture nofree readonly [[P]]) #{{[0-9]+}}, !range !0
+; NEW_PM-SAME: (i32* nocapture nofree readonly align 4 [[P:%.*]])
+; NEW_PM-NEXT:    [[RET:%.*]] = tail call i32 @test0(i32* nocapture nofree readonly align 4 [[P]]) #{{[0-9]+}}, !range !0
 ; NEW_PM-NEXT:    [[CMP_EQ_2:%.*]] = icmp eq i32 [[RET]], 9
 ; NEW_PM-NEXT:    [[CMP_EQ_3:%.*]] = icmp eq i32 [[RET]], 8
 ; NEW_PM-NEXT:    [[CMP_EQ_4:%.*]] = icmp eq i32 [[RET]], 1
@@ -158,8 +150,8 @@ define void @test0-icmp-check(i32* %p){
 ; NEW_PM-NEXT:    ret void
 ;
 ; CGSCC_OLD_PM-LABEL: define {{[^@]+}}@test0-icmp-check
-; CGSCC_OLD_PM-SAME: (i32* nocapture nofree nonnull readonly dereferenceable(4) [[P:%.*]])
-; CGSCC_OLD_PM-NEXT:    [[RET:%.*]] = tail call i32 @test0(i32* nocapture nofree nonnull readonly dereferenceable(4) [[P]])
+; CGSCC_OLD_PM-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P:%.*]])
+; CGSCC_OLD_PM-NEXT:    [[RET:%.*]] = tail call i32 @test0(i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P]])
 ; CGSCC_OLD_PM-NEXT:    [[CMP_EQ_1:%.*]] = icmp eq i32 [[RET]], 10
 ; CGSCC_OLD_PM-NEXT:    [[CMP_EQ_2:%.*]] = icmp eq i32 [[RET]], 9
 ; CGSCC_OLD_PM-NEXT:    [[CMP_EQ_3:%.*]] = icmp eq i32 [[RET]], 8
@@ -225,8 +217,8 @@ define void @test0-icmp-check(i32* %p){
 ; CGSCC_OLD_PM-NEXT:    ret void
 ;
 ; CGSCC_NEW_PM-LABEL: define {{[^@]+}}@test0-icmp-check
-; CGSCC_NEW_PM-SAME: (i32* nocapture nofree nonnull readonly dereferenceable(4) [[P:%.*]])
-; CGSCC_NEW_PM-NEXT:    [[RET:%.*]] = tail call i32 @test0(i32* nocapture nofree nonnull readonly dereferenceable(4) [[P]])
+; CGSCC_NEW_PM-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P:%.*]])
+; CGSCC_NEW_PM-NEXT:    [[RET:%.*]] = tail call i32 @test0(i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P]])
 ; CGSCC_NEW_PM-NEXT:    [[CMP_EQ_1:%.*]] = icmp eq i32 [[RET]], 10
 ; CGSCC_NEW_PM-NEXT:    [[CMP_EQ_2:%.*]] = icmp eq i32 [[RET]], 9
 ; CGSCC_NEW_PM-NEXT:    [[CMP_EQ_3:%.*]] = icmp eq i32 [[RET]], 8
@@ -291,118 +283,6 @@ define void @test0-icmp-check(i32* %p){
 ; CGSCC_NEW_PM-NEXT:    tail call void @use3(i1 [[CMP_LTE_4]], i1 [[CMP_LTE_5]], i1 [[CMP_LTE_6]])
 ; CGSCC_NEW_PM-NEXT:    ret void
 ;
-; MODULE-LABEL: define {{[^@]+}}@test0-icmp-check
-; MODULE-SAME: (i32* nocapture nofree readonly [[P:%.*]])
-; MODULE-NEXT:    [[RET:%.*]] = tail call i32 @test0(i32* nocapture nofree readonly [[P]]) #{{[0-9]+}}, !range !0
-; MODULE-NEXT:    [[CMP_EQ_2:%.*]] = icmp eq i32 [[RET]], 9
-; MODULE-NEXT:    [[CMP_EQ_3:%.*]] = icmp eq i32 [[RET]], 8
-; MODULE-NEXT:    [[CMP_EQ_4:%.*]] = icmp eq i32 [[RET]], 1
-; MODULE-NEXT:    [[CMP_EQ_5:%.*]] = icmp eq i32 [[RET]], 0
-; MODULE-NEXT:    tail call void @use3(i1 false, i1 [[CMP_EQ_2]], i1 [[CMP_EQ_3]])
-; MODULE-NEXT:    tail call void @use3(i1 [[CMP_EQ_4]], i1 [[CMP_EQ_5]], i1 false)
-; MODULE-NEXT:    [[CMP_NE_2:%.*]] = icmp ne i32 [[RET]], 9
-; MODULE-NEXT:    [[CMP_NE_3:%.*]] = icmp ne i32 [[RET]], 8
-; MODULE-NEXT:    [[CMP_NE_4:%.*]] = icmp ne i32 [[RET]], 1
-; MODULE-NEXT:    [[CMP_NE_5:%.*]] = icmp ne i32 [[RET]], 0
-; MODULE-NEXT:    tail call void @use3(i1 true, i1 [[CMP_NE_2]], i1 [[CMP_NE_3]])
-; MODULE-NEXT:    tail call void @use3(i1 [[CMP_NE_4]], i1 [[CMP_NE_5]], i1 true)
-; MODULE-NEXT:    [[CMP_UGT_3:%.*]] = icmp ugt i32 [[RET]], 8
-; MODULE-NEXT:    [[CMP_UGT_4:%.*]] = icmp ugt i32 [[RET]], 1
-; MODULE-NEXT:    [[CMP_UGT_5:%.*]] = icmp ugt i32 [[RET]], 0
-; MODULE-NEXT:    tail call void @use3(i1 false, i1 false, i1 [[CMP_UGT_3]])
-; MODULE-NEXT:    tail call void @use3(i1 [[CMP_UGT_4]], i1 [[CMP_UGT_5]], i1 false)
-; MODULE-NEXT:    [[CMP_UGE_2:%.*]] = icmp uge i32 [[RET]], 9
-; MODULE-NEXT:    [[CMP_UGE_3:%.*]] = icmp uge i32 [[RET]], 8
-; MODULE-NEXT:    [[CMP_UGE_4:%.*]] = icmp uge i32 [[RET]], 1
-; MODULE-NEXT:    tail call void @use3(i1 false, i1 [[CMP_UGE_2]], i1 [[CMP_UGE_3]])
-; MODULE-NEXT:    tail call void @use3(i1 [[CMP_UGE_4]], i1 true, i1 false)
-; MODULE-NEXT:    [[CMP_SGT_3:%.*]] = icmp sgt i32 [[RET]], 8
-; MODULE-NEXT:    [[CMP_SGT_4:%.*]] = icmp sgt i32 [[RET]], 1
-; MODULE-NEXT:    [[CMP_SGT_5:%.*]] = icmp sgt i32 [[RET]], 0
-; MODULE-NEXT:    tail call void @use3(i1 false, i1 false, i1 [[CMP_SGT_3]])
-; MODULE-NEXT:    tail call void @use3(i1 [[CMP_SGT_4]], i1 [[CMP_SGT_5]], i1 true)
-; MODULE-NEXT:    [[CMP_GTE_2:%.*]] = icmp sge i32 [[RET]], 9
-; MODULE-NEXT:    [[CMP_GTE_3:%.*]] = icmp sge i32 [[RET]], 8
-; MODULE-NEXT:    [[CMP_GTE_4:%.*]] = icmp sge i32 [[RET]], 1
-; MODULE-NEXT:    tail call void @use3(i1 false, i1 [[CMP_GTE_2]], i1 [[CMP_GTE_3]])
-; MODULE-NEXT:    tail call void @use3(i1 [[CMP_GTE_4]], i1 true, i1 true)
-; MODULE-NEXT:    [[CMP_SLT_2:%.*]] = icmp slt i32 [[RET]], 9
-; MODULE-NEXT:    [[CMP_SLT_3:%.*]] = icmp slt i32 [[RET]], 8
-; MODULE-NEXT:    [[CMP_SLT_4:%.*]] = icmp slt i32 [[RET]], 1
-; MODULE-NEXT:    tail call void @use3(i1 true, i1 [[CMP_SLT_2]], i1 [[CMP_SLT_3]])
-; MODULE-NEXT:    tail call void @use3(i1 [[CMP_SLT_4]], i1 false, i1 false)
-; MODULE-NEXT:    [[CMP_LTE_3:%.*]] = icmp sle i32 [[RET]], 8
-; MODULE-NEXT:    [[CMP_LTE_4:%.*]] = icmp sle i32 [[RET]], 1
-; MODULE-NEXT:    [[CMP_LTE_5:%.*]] = icmp sle i32 [[RET]], 0
-; MODULE-NEXT:    tail call void @use3(i1 true, i1 true, i1 [[CMP_LTE_3]])
-; MODULE-NEXT:    tail call void @use3(i1 [[CMP_LTE_4]], i1 [[CMP_LTE_5]], i1 false)
-; MODULE-NEXT:    ret void
-; CGSCC-LABEL: define {{[^@]+}}@test0-icmp-check
-; CGSCC-SAME: (i32* nocapture nofree nonnull readonly dereferenceable(4) [[P:%.*]])
-; CGSCC-NEXT:    [[RET:%.*]] = tail call i32 @test0(i32* nocapture nofree nonnull readonly dereferenceable(4) [[P]])
-; CGSCC-NEXT:    [[CMP_EQ_1:%.*]] = icmp eq i32 [[RET]], 10
-; CGSCC-NEXT:    [[CMP_EQ_2:%.*]] = icmp eq i32 [[RET]], 9
-; CGSCC-NEXT:    [[CMP_EQ_3:%.*]] = icmp eq i32 [[RET]], 8
-; CGSCC-NEXT:    [[CMP_EQ_4:%.*]] = icmp eq i32 [[RET]], 1
-; CGSCC-NEXT:    [[CMP_EQ_5:%.*]] = icmp eq i32 [[RET]], 0
-; CGSCC-NEXT:    [[CMP_EQ_6:%.*]] = icmp eq i32 [[RET]], -1
-; CGSCC-NEXT:    tail call void @use3(i1 [[CMP_EQ_1]], i1 [[CMP_EQ_2]], i1 [[CMP_EQ_3]])
-; CGSCC-NEXT:    tail call void @use3(i1 [[CMP_EQ_4]], i1 [[CMP_EQ_5]], i1 [[CMP_EQ_6]])
-; CGSCC-NEXT:    [[CMP_NE_1:%.*]] = icmp ne i32 [[RET]], 10
-; CGSCC-NEXT:    [[CMP_NE_2:%.*]] = icmp ne i32 [[RET]], 9
-; CGSCC-NEXT:    [[CMP_NE_3:%.*]] = icmp ne i32 [[RET]], 8
-; CGSCC-NEXT:    [[CMP_NE_4:%.*]] = icmp ne i32 [[RET]], 1
-; CGSCC-NEXT:    [[CMP_NE_5:%.*]] = icmp ne i32 [[RET]], 0
-; CGSCC-NEXT:    [[CMP_NE_6:%.*]] = icmp ne i32 [[RET]], -1
-; CGSCC-NEXT:    tail call void @use3(i1 [[CMP_NE_1]], i1 [[CMP_NE_2]], i1 [[CMP_NE_3]])
-; CGSCC-NEXT:    tail call void @use3(i1 [[CMP_NE_4]], i1 [[CMP_NE_5]], i1 [[CMP_NE_6]])
-; CGSCC-NEXT:    [[CMP_UGT_1:%.*]] = icmp ugt i32 [[RET]], 10
-; CGSCC-NEXT:    [[CMP_UGT_2:%.*]] = icmp ugt i32 [[RET]], 9
-; CGSCC-NEXT:    [[CMP_UGT_3:%.*]] = icmp ugt i32 [[RET]], 8
-; CGSCC-NEXT:    [[CMP_UGT_4:%.*]] = icmp ugt i32 [[RET]], 1
-; CGSCC-NEXT:    [[CMP_UGT_5:%.*]] = icmp ugt i32 [[RET]], 0
-; CGSCC-NEXT:    tail call void @use3(i1 [[CMP_UGT_1]], i1 [[CMP_UGT_2]], i1 [[CMP_UGT_3]])
-; CGSCC-NEXT:    tail call void @use3(i1 [[CMP_UGT_4]], i1 [[CMP_UGT_5]], i1 false)
-; CGSCC-NEXT:    [[CMP_UGE_1:%.*]] = icmp uge i32 [[RET]], 10
-; CGSCC-NEXT:    [[CMP_UGE_2:%.*]] = icmp uge i32 [[RET]], 9
-; CGSCC-NEXT:    [[CMP_UGE_3:%.*]] = icmp uge i32 [[RET]], 8
-; CGSCC-NEXT:    [[CMP_UGE_4:%.*]] = icmp uge i32 [[RET]], 1
-; CGSCC-NEXT:    [[CMP_UGE_6:%.*]] = icmp uge i32 [[RET]], -1
-; CGSCC-NEXT:    tail call void @use3(i1 [[CMP_UGE_1]], i1 [[CMP_UGE_2]], i1 [[CMP_UGE_3]])
-; CGSCC-NEXT:    tail call void @use3(i1 [[CMP_UGE_4]], i1 true, i1 [[CMP_UGE_6]])
-; CGSCC-NEXT:    [[CMP_SGT_1:%.*]] = icmp sgt i32 [[RET]], 10
-; CGSCC-NEXT:    [[CMP_SGT_2:%.*]] = icmp sgt i32 [[RET]], 9
-; CGSCC-NEXT:    [[CMP_SGT_3:%.*]] = icmp sgt i32 [[RET]], 8
-; CGSCC-NEXT:    [[CMP_SGT_4:%.*]] = icmp sgt i32 [[RET]], 1
-; CGSCC-NEXT:    [[CMP_SGT_5:%.*]] = icmp sgt i32 [[RET]], 0
-; CGSCC-NEXT:    [[CMP_SGT_6:%.*]] = icmp sgt i32 [[RET]], -1
-; CGSCC-NEXT:    tail call void @use3(i1 [[CMP_SGT_1]], i1 [[CMP_SGT_2]], i1 [[CMP_SGT_3]])
-; CGSCC-NEXT:    tail call void @use3(i1 [[CMP_SGT_4]], i1 [[CMP_SGT_5]], i1 [[CMP_SGT_6]])
-; CGSCC-NEXT:    [[CMP_GTE_1:%.*]] = icmp sge i32 [[RET]], 10
-; CGSCC-NEXT:    [[CMP_GTE_2:%.*]] = icmp sge i32 [[RET]], 9
-; CGSCC-NEXT:    [[CMP_GTE_3:%.*]] = icmp sge i32 [[RET]], 8
-; CGSCC-NEXT:    [[CMP_GTE_4:%.*]] = icmp sge i32 [[RET]], 1
-; CGSCC-NEXT:    [[CMP_GTE_5:%.*]] = icmp sge i32 [[RET]], 0
-; CGSCC-NEXT:    [[CMP_GTE_6:%.*]] = icmp sge i32 [[RET]], -1
-; CGSCC-NEXT:    tail call void @use3(i1 [[CMP_GTE_1]], i1 [[CMP_GTE_2]], i1 [[CMP_GTE_3]])
-; CGSCC-NEXT:    tail call void @use3(i1 [[CMP_GTE_4]], i1 [[CMP_GTE_5]], i1 [[CMP_GTE_6]])
-; CGSCC-NEXT:    [[CMP_SLT_1:%.*]] = icmp slt i32 [[RET]], 10
-; CGSCC-NEXT:    [[CMP_SLT_2:%.*]] = icmp slt i32 [[RET]], 9
-; CGSCC-NEXT:    [[CMP_SLT_3:%.*]] = icmp slt i32 [[RET]], 8
-; CGSCC-NEXT:    [[CMP_SLT_4:%.*]] = icmp slt i32 [[RET]], 1
-; CGSCC-NEXT:    [[CMP_SLT_5:%.*]] = icmp slt i32 [[RET]], 0
-; CGSCC-NEXT:    [[CMP_SLT_6:%.*]] = icmp slt i32 [[RET]], -1
-; CGSCC-NEXT:    tail call void @use3(i1 [[CMP_SLT_1]], i1 [[CMP_SLT_2]], i1 [[CMP_SLT_3]])
-; CGSCC-NEXT:    tail call void @use3(i1 [[CMP_SLT_4]], i1 [[CMP_SLT_5]], i1 [[CMP_SLT_6]])
-; CGSCC-NEXT:    [[CMP_LTE_1:%.*]] = icmp sle i32 [[RET]], 10
-; CGSCC-NEXT:    [[CMP_LTE_2:%.*]] = icmp sle i32 [[RET]], 9
-; CGSCC-NEXT:    [[CMP_LTE_3:%.*]] = icmp sle i32 [[RET]], 8
-; CGSCC-NEXT:    [[CMP_LTE_4:%.*]] = icmp sle i32 [[RET]], 1
-; CGSCC-NEXT:    [[CMP_LTE_5:%.*]] = icmp sle i32 [[RET]], 0
-; CGSCC-NEXT:    [[CMP_LTE_6:%.*]] = icmp sle i32 [[RET]], -1
-; CGSCC-NEXT:    tail call void @use3(i1 [[CMP_LTE_1]], i1 [[CMP_LTE_2]], i1 [[CMP_LTE_3]])
-; CGSCC-NEXT:    tail call void @use3(i1 [[CMP_LTE_4]], i1 [[CMP_LTE_5]], i1 [[CMP_LTE_6]])
-; CGSCC-NEXT:    ret void
   ; ret = [0, 10)
   %ret = tail call i32 @test0(i32 *%p)
 
@@ -490,8 +370,8 @@ define void @test0-icmp-check(i32* %p){
 }
 define i32 @test1(i32* %p) {
 ; CHECK-LABEL: define {{[^@]+}}@test1
-; CHECK-SAME: (i32* nocapture nofree nonnull readonly dereferenceable(4) [[P:%.*]])
-; CHECK-NEXT:    [[LOAD_10_100:%.*]] = load i32, i32* [[P]], !range !1
+; CHECK-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P:%.*]])
+; CHECK-NEXT:    [[LOAD_10_100:%.*]] = load i32, i32* [[P]], align 4, !range !1
 ; CHECK-NEXT:    [[ADD_10_THEN_20_110:%.*]] = add i32 [[LOAD_10_100]], 10
 ; CHECK-NEXT:    [[MUL_10_THEN_200_1091:%.*]] = mul i32 [[ADD_10_THEN_20_110]], 10
 ; CHECK-NEXT:    ret i32 [[MUL_10_THEN_200_1091]]
@@ -504,39 +384,29 @@ define i32 @test1(i32* %p) {
 
 define i1 @test1-check(i32* %p) {
 ; OLD_PM-LABEL: define {{[^@]+}}@test1-check
-; OLD_PM-SAME: (i32* nocapture nofree readonly [[P:%.*]])
-; OLD_PM-NEXT:    [[RES:%.*]] = tail call i32 @test1(i32* nocapture nofree readonly [[P]]) #{{[0-9]+}}, !range !2
+; OLD_PM-SAME: (i32* nocapture nofree readonly align 4 [[P:%.*]])
+; OLD_PM-NEXT:    [[RES:%.*]] = tail call i32 @test1(i32* nocapture nofree readonly align 4 [[P]]) #{{[0-9]+}}, !range !2
 ; OLD_PM-NEXT:    [[CMP:%.*]] = icmp eq i32 [[RES]], 500
 ; OLD_PM-NEXT:    ret i1 [[CMP]]
 ;
 ; NEW_PM-LABEL: define {{[^@]+}}@test1-check
-; NEW_PM-SAME: (i32* nocapture nofree readonly [[P:%.*]])
-; NEW_PM-NEXT:    [[RES:%.*]] = tail call i32 @test1(i32* nocapture nofree readonly [[P]]) #{{[0-9]+}}, !range !2
+; NEW_PM-SAME: (i32* nocapture nofree readonly align 4 [[P:%.*]])
+; NEW_PM-NEXT:    [[RES:%.*]] = tail call i32 @test1(i32* nocapture nofree readonly align 4 [[P]]) #{{[0-9]+}}, !range !2
 ; NEW_PM-NEXT:    [[CMP:%.*]] = icmp eq i32 [[RES]], 500
 ; NEW_PM-NEXT:    ret i1 [[CMP]]
 ;
 ; CGSCC_OLD_PM-LABEL: define {{[^@]+}}@test1-check
-; CGSCC_OLD_PM-SAME: (i32* nocapture nofree nonnull readonly dereferenceable(4) [[P:%.*]])
-; CGSCC_OLD_PM-NEXT:    [[RES:%.*]] = tail call i32 @test1(i32* nocapture nofree nonnull readonly dereferenceable(4) [[P]])
+; CGSCC_OLD_PM-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P:%.*]])
+; CGSCC_OLD_PM-NEXT:    [[RES:%.*]] = tail call i32 @test1(i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P]])
 ; CGSCC_OLD_PM-NEXT:    [[CMP:%.*]] = icmp eq i32 [[RES]], 500
 ; CGSCC_OLD_PM-NEXT:    ret i1 [[CMP]]
 ;
 ; CGSCC_NEW_PM-LABEL: define {{[^@]+}}@test1-check
-; CGSCC_NEW_PM-SAME: (i32* nocapture nofree nonnull readonly dereferenceable(4) [[P:%.*]])
-; CGSCC_NEW_PM-NEXT:    [[RES:%.*]] = tail call i32 @test1(i32* nocapture nofree nonnull readonly dereferenceable(4) [[P]])
+; CGSCC_NEW_PM-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P:%.*]])
+; CGSCC_NEW_PM-NEXT:    [[RES:%.*]] = tail call i32 @test1(i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P]])
 ; CGSCC_NEW_PM-NEXT:    [[CMP:%.*]] = icmp eq i32 [[RES]], 500
 ; CGSCC_NEW_PM-NEXT:    ret i1 [[CMP]]
 ;
-; MODULE-LABEL: define {{[^@]+}}@test1-check
-; MODULE-SAME: (i32* nocapture nofree readonly [[P:%.*]])
-; MODULE-NEXT:    [[RES:%.*]] = tail call i32 @test1(i32* nocapture nofree readonly [[P]]) #{{[0-9]+}}, !range !2
-; MODULE-NEXT:    [[CMP:%.*]] = icmp eq i32 [[RES]], 500
-; MODULE-NEXT:    ret i1 [[CMP]]
-; CGSCC-LABEL: define {{[^@]+}}@test1-check
-; CGSCC-SAME: (i32* nocapture nofree nonnull readonly dereferenceable(4) [[P:%.*]])
-; CGSCC-NEXT:    [[RES:%.*]] = tail call i32 @test1(i32* nocapture nofree nonnull readonly dereferenceable(4) [[P]])
-; CGSCC-NEXT:    [[CMP:%.*]] = icmp eq i32 [[RES]], 500
-; CGSCC-NEXT:    ret i1 [[CMP]]
   %res = tail call i32 @test1(i32* %p)
   %cmp = icmp eq i32 %res, 500
   ret i1 %cmp
@@ -621,29 +491,6 @@ define i32 @test2_check(i32* %p) {
 ; CGSCC_NEW_PM-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ 2, [[IF_THEN]] ], [ 3, [[IF_END]] ]
 ; CGSCC_NEW_PM-NEXT:    ret i32 [[RETVAL_0]]
 ;
-; MODULE-LABEL: define {{[^@]+}}@test2_check
-; MODULE-SAME: (i32* nocapture nofree readonly align 4 [[P:%.*]])
-; MODULE-NEXT:  entry:
-; MODULE-NEXT:    br label [[IF_THEN:%.*]]
-; MODULE:       if.then:
-; MODULE-NEXT:    br label [[RETURN:%.*]]
-; MODULE:       if.end:
-; MODULE-NEXT:    unreachable
-; MODULE:       return:
-; MODULE-NEXT:    ret i32 2
-; CGSCC-LABEL: define {{[^@]+}}@test2_check
-; CGSCC-SAME: (i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P:%.*]])
-; CGSCC-NEXT:  entry:
-; CGSCC-NEXT:    [[CALL:%.*]] = tail call i32 @test2(i32* nocapture nofree nonnull readonly align 4 dereferenceable(4) [[P]])
-; CGSCC-NEXT:    [[CMP:%.*]] = icmp slt i32 [[CALL]], 5
-; CGSCC-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-; CGSCC:       if.then:
-; CGSCC-NEXT:    br label [[RETURN:%.*]]
-; CGSCC:       if.end:
-; CGSCC-NEXT:    br label [[RETURN]]
-; CGSCC:       return:
-; CGSCC-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ 2, [[IF_THEN]] ], [ 3, [[IF_END]] ]
-; CGSCC-NEXT:    ret i32 [[RETVAL_0]]
 entry:
   %call = tail call i32 @test2(i32* %p)
   %cmp = icmp slt i32 %call, 5
@@ -1276,13 +1123,11 @@ define i32 @ret1or2(i1 %c) {
 define i1 @callee_range_1(i1 %c1, i1 %c2, i1 %c3) {
 ; OLD_PM-LABEL: define {{[^@]+}}@callee_range_1
 ; OLD_PM-SAME: (i1 [[C1:%.*]], i1 [[C2:%.*]], i1 [[C3:%.*]])
-; OLD_PM-NEXT:    [[F:%.*]] = and i1 true, true
-; OLD_PM-NEXT:    ret i1 [[F]]
+; OLD_PM-NEXT:    ret i1 true
 ;
 ; NEW_PM-LABEL: define {{[^@]+}}@callee_range_1
 ; NEW_PM-SAME: (i1 [[C1:%.*]], i1 [[C2:%.*]], i1 [[C3:%.*]])
-; NEW_PM-NEXT:    [[F:%.*]] = and i1 true, true
-; NEW_PM-NEXT:    ret i1 [[F]]
+; NEW_PM-NEXT:    ret i1 true
 ;
 ; CGSCC_OLD_PM-LABEL: define {{[^@]+}}@callee_range_1
 ; CGSCC_OLD_PM-SAME: (i1 [[C1:%.*]], i1 [[C2:%.*]], i1 [[C3:%.*]])
@@ -1367,6 +1212,76 @@ define i1 @callee_range_2(i1 %c1, i1 %c2) {
 }
 
 
+define i32 @ret100() {
+; CHECK-LABEL: define {{[^@]+}}@ret100()
+; CHECK-NEXT:    ret i32 100
+;
+  ret i32 100
+}
+
+define i1 @ctx_adjustment(i32 %V) {
+; OLD_PM-LABEL: define {{[^@]+}}@ctx_adjustment
+; OLD_PM-SAME: (i32 [[V:%.*]])
+; OLD_PM-NEXT:    [[C1:%.*]] = icmp sge i32 [[V]], 100
+; OLD_PM-NEXT:    br i1 [[C1]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; OLD_PM:       if.true:
+; OLD_PM-NEXT:    br label [[END:%.*]]
+; OLD_PM:       if.false:
+; OLD_PM-NEXT:    br label [[END]]
+; OLD_PM:       end:
+; OLD_PM-NEXT:    [[PHI:%.*]] = phi i32 [ [[V]], [[IF_TRUE]] ], [ 100, [[IF_FALSE]] ]
+; OLD_PM-NEXT:    [[C2:%.*]] = icmp sge i32 [[PHI]], 100
+; OLD_PM-NEXT:    ret i1 [[C2]]
+;
+; NEW_PM-LABEL: define {{[^@]+}}@ctx_adjustment
+; NEW_PM-SAME: (i32 [[V:%.*]])
+; NEW_PM-NEXT:    [[C1:%.*]] = icmp sge i32 [[V]], 100
+; NEW_PM-NEXT:    br i1 [[C1]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; NEW_PM:       if.true:
+; NEW_PM-NEXT:    br label [[END:%.*]]
+; NEW_PM:       if.false:
+; NEW_PM-NEXT:    br label [[END]]
+; NEW_PM:       end:
+; NEW_PM-NEXT:    ret i1 true
+;
+; CGSCC_OLD_PM-LABEL: define {{[^@]+}}@ctx_adjustment
+; CGSCC_OLD_PM-SAME: (i32 [[V:%.*]])
+; CGSCC_OLD_PM-NEXT:    [[C1:%.*]] = icmp sge i32 [[V]], 100
+; CGSCC_OLD_PM-NEXT:    br i1 [[C1]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CGSCC_OLD_PM:       if.true:
+; CGSCC_OLD_PM-NEXT:    br label [[END:%.*]]
+; CGSCC_OLD_PM:       if.false:
+; CGSCC_OLD_PM-NEXT:    br label [[END]]
+; CGSCC_OLD_PM:       end:
+; CGSCC_OLD_PM-NEXT:    [[PHI:%.*]] = phi i32 [ [[V]], [[IF_TRUE]] ], [ 100, [[IF_FALSE]] ]
+; CGSCC_OLD_PM-NEXT:    [[C2:%.*]] = icmp sge i32 [[PHI]], 100
+; CGSCC_OLD_PM-NEXT:    ret i1 [[C2]]
+;
+; CGSCC_NEW_PM-LABEL: define {{[^@]+}}@ctx_adjustment
+; CGSCC_NEW_PM-SAME: (i32 [[V:%.*]])
+; CGSCC_NEW_PM-NEXT:    [[C1:%.*]] = icmp sge i32 [[V]], 100
+; CGSCC_NEW_PM-NEXT:    br i1 [[C1]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CGSCC_NEW_PM:       if.true:
+; CGSCC_NEW_PM-NEXT:    br label [[END:%.*]]
+; CGSCC_NEW_PM:       if.false:
+; CGSCC_NEW_PM-NEXT:    br label [[END]]
+; CGSCC_NEW_PM:       end:
+; CGSCC_NEW_PM-NEXT:    ret i1 true
+;
+  %c1 = icmp sge i32 %V, 100
+  br i1 %c1, label %if.true, label %if.false
+if.true:
+  br label %end
+if.false:
+  %call = call i32 @ret100()
+  br label %end
+end:
+  %phi = phi i32 [ %V, %if.true ], [ %call, %if.false ]
+  %c2 = icmp sge i32 %phi, 100
+  ret i1 %c2
+}
+
+
 !0 = !{i32 0, i32 10}
 !1 = !{i32 10, i32 100}
 ; CHECK: !0 = !{i32 0, i32 10}
diff --git a/llvm/test/Transforms/Attributor/readattrs.ll b/llvm/test/Transforms/Attributor/readattrs.ll
index e23dc9223e49b..08c13d92b01cd 100644
--- a/llvm/test/Transforms/Attributor/readattrs.ll
+++ b/llvm/test/Transforms/Attributor/readattrs.ll
@@ -35,7 +35,7 @@ define void @test4_2(i8* %p) {
   ret void
 }
 
-; ATTRIBUTOR: define void @test5(i8** nocapture nofree nonnull writeonly dereferenceable(8) %p, i8* nofree writeonly %q)
+; ATTRIBUTOR: define void @test5(i8** nocapture nofree nonnull writeonly align 8 dereferenceable(8) %p, i8* nofree writeonly %q)
 ; Missed optz'n: we could make %q readnone, but don't break test6!
 define void @test5(i8** %p, i8* %q) {
   store i8* %q, i8** %p
@@ -43,7 +43,7 @@ define void @test5(i8** %p, i8* %q) {
 }
 
 declare void @test6_1()
-; ATTRIBUTOR: define void @test6_2(i8** nocapture nonnull writeonly dereferenceable(8) %p, i8* %q)
+; ATTRIBUTOR: define void @test6_2(i8** nocapture nonnull writeonly align 8 dereferenceable(8) %p, i8* %q)
 ; This is not a missed optz'n.
 define void @test6_2(i8** %p, i8* %q) {
   store i8* %q, i8** %p
diff --git a/llvm/test/Transforms/Attributor/value-simplify.ll b/llvm/test/Transforms/Attributor/value-simplify.ll
index ae4101ce4e18f..5a131c6f0b8ca 100644
--- a/llvm/test/Transforms/Attributor/value-simplify.ll
+++ b/llvm/test/Transforms/Attributor/value-simplify.ll
@@ -208,17 +208,18 @@ define i32* @complicated_args_inalloca() {
 
 define internal void @test_sret(%struct.X* sret %a, %struct.X** %b) {
 ; CHECK-LABEL: define {{[^@]+}}@test_sret
-; CHECK-SAME: (%struct.X* noalias nofree sret writeonly align 536870912 [[A:%.*]], %struct.X** nocapture nofree nonnull writeonly dereferenceable(8) [[B:%.*]])
-; CHECK-NEXT:    store %struct.X* [[A]], %struct.X** [[B]]
+; CHECK-SAME: (%struct.X* noalias nofree sret writeonly align 536870912 [[A:%.*]], %struct.X** nocapture nofree nonnull writeonly align 8 dereferenceable(8) [[B:%.*]])
+; CHECK-NEXT:    store %struct.X* [[A]], %struct.X** [[B]], align 8
 ; CHECK-NEXT:    ret void
 ;
   store %struct.X* %a, %struct.X** %b
   ret void
 }
+; FIXME: Alignment and dereferenceability are not propagated to the argument
 define void @complicated_args_sret(%struct.X** %b) {
 ; CHECK-LABEL: define {{[^@]+}}@complicated_args_sret
 ; CHECK-SAME: (%struct.X** nocapture nofree writeonly [[B:%.*]])
-; CHECK-NEXT:    call void @test_sret(%struct.X* noalias nofree writeonly align 536870912 null, %struct.X** nocapture nofree writeonly [[B]])
+; CHECK-NEXT:    call void @test_sret(%struct.X* noalias nofree writeonly align 536870912 null, %struct.X** nocapture nofree writeonly align 8 [[B]])
 ; CHECK-NEXT:    ret void
 ;
   call void @test_sret(%struct.X* null, %struct.X** %b)
diff --git a/llvm/test/Transforms/CallSiteSplitting/callsite-split-or-phi.ll b/llvm/test/Transforms/CallSiteSplitting/callsite-split-or-phi.ll
index cd2edb1e33559..83b265b14ed3a 100644
--- a/llvm/test/Transforms/CallSiteSplitting/callsite-split-or-phi.ll
+++ b/llvm/test/Transforms/CallSiteSplitting/callsite-split-or-phi.ll
@@ -586,3 +586,37 @@ CallSiteBB:
 End:
   ret void
 }
+
+; CHECK-LABEL: i32 @test_multiple_phis(
+; CHECK:      Header.split:
+; CHECK-NEXT:   %r2 = call i32 @callee(i32* null, i32 1, i32 5)
+; CHECK-NEXT:   br label %Tail
+
+; CHECK:      TBB.split:
+; CHECK-NEXT:   %r1 = call i32 @callee(i32* null, i32 2, i32 10)
+; CHECK-NEXT:   br label %Tail
+
+; CHECK:        Tail:
+; CHECK-NEXT:   %phi.call = phi i32 [ %r1, %TBB.split ], [ %r2, %Header.split ]
+; CHECK-NEXT:   %p.0 = phi i32 [ 0, %Header.split ], [ 99, %TBB.split ]
+; CHECK-NEXT:   %res = add i32 %phi.call, %p.0
+; CHECK-NEXT:   ret i32 %phi.call
+;
+define i32 @test_multiple_phis(i1 %c.1) {
+Header:
+  br i1 %c.1, label %Tail, label %TBB
+
+TBB:
+  br label %Tail
+
+Tail:
+  %p.0 = phi i32 [0, %Header], [99, %TBB]
+  %p.1 = phi i32[1, %Header], [2, %TBB]
+  %p.2 = phi i32 [5, %Header], [10, %TBB]
+  %r = call i32 @callee(i32* null, i32 %p.1, i32 %p.2)
+  %res = add i32 %r, %p.0
+  ret i32 %r
+
+End:
+  ret i32 10
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/extend-sink-hoist.ll b/llvm/test/Transforms/CodeGenPrepare/X86/extend-sink-hoist.ll
index 519e1ee2ce6fd..9ba178b48dac2 100644
--- a/llvm/test/Transforms/CodeGenPrepare/X86/extend-sink-hoist.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/extend-sink-hoist.ll
@@ -13,7 +13,6 @@ block1:
   %s1 = sext i64 %l1 to i128
   br label %block2
 
-; CHECK-LABEL: block2:
 ; CHECK-NEXT: sext
 ; CHECK-NEXT: load
 ; CHECK-NEXT: sext
@@ -34,7 +33,6 @@ block1:
   %l1 = load i32, i32* %mem1
   br label %block2
 
-; CHECK-LABEL: block2:
 ; CHECK-NEXT: load
 ; CHECK-NEXT: sext
 block2:
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/optimizeSelect-DT.ll b/llvm/test/Transforms/CodeGenPrepare/X86/optimizeSelect-DT.ll
index eb4e4ce431af3..be651d7eb0044 100644
--- a/llvm/test/Transforms/CodeGenPrepare/X86/optimizeSelect-DT.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/optimizeSelect-DT.ll
@@ -15,9 +15,10 @@ define i1 @PR41004(i32 %x, i32 %y, i32 %t1) {
 ; CHECK-NEXT:    br label [[SELECT_END]]
 ; CHECK:       select.end:
 ; CHECK-NEXT:    [[MUL:%.*]] = phi i32 [ [[REM]], [[SELECT_TRUE_SINK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[NEG:%.*]] = add i32 [[T1:%.*]], -1
+; CHECK-NEXT:    [[USUB:%.*]] = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 [[T1:%.*]], i32 1)
+; CHECK-NEXT:    [[NEG:%.*]] = extractvalue { i32, i1 } [[USUB]], 0
+; CHECK-NEXT:    [[TOBOOL:%.*]] = extractvalue { i32, i1 } [[USUB]], 1
 ; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[NEG]], [[MUL]]
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[T1]], 0
 ; CHECK-NEXT:    ret i1 [[TOBOOL]]
 ;
 entry:
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/tailcall-assume-xbb.ll b/llvm/test/Transforms/CodeGenPrepare/X86/tailcall-assume-xbb.ll
new file mode 100644
index 0000000000000..84720c61cb23a
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/tailcall-assume-xbb.ll
@@ -0,0 +1,48 @@
+; RUN: opt -codegenprepare -S -mtriple=x86_64-linux < %s | FileCheck %s
+
+; The ret instruction can be duplicated into BB case2 even though there is an
+; intermediate BB exit1 and call to llvm.assume.
+
+@ptr = external global i8*, align 8
+
+; CHECK:       %ret1 = tail call i8* @qux()
+; CHECK-NEXT:  ret i8* %ret1
+
+; CHECK:       %ret2 = tail call i8* @bar()
+; CHECK-NEXT:  ret i8* %ret2
+
+define i8* @foo(i64 %size, i64 %v1, i64 %v2) {
+entry:
+  %cmp1 = icmp ult i64 %size, 1025
+  br i1 %cmp1, label %if.end, label %case1
+
+case1:
+  %ret1 = tail call i8* @qux()
+  br label %exit2
+
+if.end:
+  %cmp2 = icmp ult i64 %v1, %v2
+  br i1 %cmp2, label %case3, label %case2
+
+case2:
+  %ret2 = tail call i8* @bar()
+  br label %exit1
+
+case3:
+  %ret3 = load i8*, i8** @ptr, align 8
+  br label %exit1
+
+exit1:
+  %retval1 = phi i8* [ %ret2, %case2 ], [ %ret3, %case3 ]
+  %cmp3 = icmp ne i8* %retval1, null
+  tail call void @llvm.assume(i1 %cmp3)
+  br label %exit2
+
+exit2:
+  %retval2 = phi i8* [ %ret1, %case1 ], [ %retval1, %exit1 ]
+  ret i8* %retval2
+}
+
+declare void @llvm.assume(i1)
+declare i8* @qux()
+declare i8* @bar()
diff --git a/llvm/test/Transforms/Coroutines/ArgAddr.ll b/llvm/test/Transforms/Coroutines/ArgAddr.ll
index 81cfb7db06a71..a1cac168ac402 100644
--- a/llvm/test/Transforms/Coroutines/ArgAddr.ll
+++ b/llvm/test/Transforms/Coroutines/ArgAddr.ll
@@ -1,7 +1,7 @@
 ; Need to move users of allocas that were moved into the coroutine frame after
 ; coro.begin.
-; RUN: opt < %s -O2 -enable-coroutines -S | FileCheck %s
-; RUN: opt < %s -aa-pipeline=basic-aa -passes='default<O2>' -enable-coroutines -S | FileCheck %s
+; RUN: opt < %s -preserve-alignment-assumptions-during-inlining=false -O2 -enable-coroutines -S | FileCheck %s
+; RUN: opt < %s -preserve-alignment-assumptions-during-inlining=false  -aa-pipeline=basic-aa -passes='default<O2>' -enable-coroutines -S | FileCheck %s
 
 define nonnull i8* @f(i32 %n) {
 entry:
diff --git a/llvm/test/Transforms/Coroutines/coro-alloc-with-param-O0.ll b/llvm/test/Transforms/Coroutines/coro-alloc-with-param-O0.ll
index 48bf0c52593d9..f29b5a062adb7 100644
--- a/llvm/test/Transforms/Coroutines/coro-alloc-with-param-O0.ll
+++ b/llvm/test/Transforms/Coroutines/coro-alloc-with-param-O0.ll
@@ -30,11 +30,11 @@ suspend:
 }
 
 ; See if %this was added to the frame
-; CHECK: %f_copy.Frame = type { void (%f_copy.Frame*)*, void (%f_copy.Frame*)*, i1, i1, i64 }
+; CHECK: %f_copy.Frame = type { void (%f_copy.Frame*)*, void (%f_copy.Frame*)*, i64, i1 }
 
 ; See that %this is spilled into the frame
 ; CHECK-LABEL: define i8* @f_copy(i64 %this_arg)
-; CHECK:  %this.spill.addr = getelementptr inbounds %f_copy.Frame, %f_copy.Frame* %FramePtr, i32 0, i32 4
+; CHECK:  %this.spill.addr = getelementptr inbounds %f_copy.Frame, %f_copy.Frame* %FramePtr, i32 0, i32 2
 ; CHECK:  store i64 %this_arg, i64* %this.spill.addr
 ; CHECK: ret i8* %hdl
 
diff --git a/llvm/test/Transforms/Coroutines/coro-alloc-with-param-O2.ll b/llvm/test/Transforms/Coroutines/coro-alloc-with-param-O2.ll
index bd95f4b9672d0..b3c4558a47b8a 100644
--- a/llvm/test/Transforms/Coroutines/coro-alloc-with-param-O2.ll
+++ b/llvm/test/Transforms/Coroutines/coro-alloc-with-param-O2.ll
@@ -27,11 +27,11 @@ suspend:
 }
 
 ; See if %this was added to the frame
-; CHECK: %f_direct.Frame = type { void (%f_direct.Frame*)*, void (%f_direct.Frame*)*, i1, i1, i64 }
+; CHECK: %f_direct.Frame = type { void (%f_direct.Frame*)*, void (%f_direct.Frame*)*, i64, i1 }
 
 ; See that %this is spilled into the frame
 ; CHECK-LABEL: define i8* @f_direct(i64 %this)
-; CHECK: %this.spill.addr = getelementptr inbounds %f_direct.Frame, %f_direct.Frame* %FramePtr, i32 0, i32 4
+; CHECK: %this.spill.addr = getelementptr inbounds %f_direct.Frame, %f_direct.Frame* %FramePtr, i32 0, i32 2
 ; CHECK: store i64 %this, i64* %this.spill.addr
 ; CHECK: ret i8* %hdl
 
diff --git a/llvm/test/Transforms/Coroutines/coro-catchswitch.ll b/llvm/test/Transforms/Coroutines/coro-catchswitch.ll
index 0eb4b1ee64e88..79ce5062a20d7 100644
--- a/llvm/test/Transforms/Coroutines/coro-catchswitch.ll
+++ b/llvm/test/Transforms/Coroutines/coro-catchswitch.ll
@@ -32,7 +32,7 @@ catch.dispatch:                                   ; preds = %if.else, %if.then
 ; CHECK: catch.dispatch:
 ; CHECK:  %val = phi i32 [ 2, %if.else ], [ 1, %if.then ]
 ; CHECK:  %[[Pad:.+]] = cleanuppad within none []
-; CHECK:  %val.spill.addr = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 4
+; CHECK:  %val.spill.addr = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 2
 ; CHECK:  store i32 %val, i32* %val.spill.addr
 ; CHECK:  cleanupret from %[[Pad]] unwind label %[[Switch:.+]]
 
diff --git a/llvm/test/Transforms/Coroutines/coro-debug.ll b/llvm/test/Transforms/Coroutines/coro-debug.ll
index e4a71e8108804..b624c88f99ff0 100644
--- a/llvm/test/Transforms/Coroutines/coro-debug.ll
+++ b/llvm/test/Transforms/Coroutines/coro-debug.ll
@@ -128,12 +128,12 @@ attributes #7 = { noduplicate }
 !24 = !DILocation(line: 62, column: 3, scope: !6)
 
 ; CHECK: define i8* @f(i32 %x) #0 !dbg ![[ORIG:[0-9]+]]
-; CHECK: define internal fastcc void @f.resume(%f.Frame* noalias nonnull %FramePtr) #0 !dbg ![[RESUME:[0-9]+]]
+; CHECK: define internal fastcc void @f.resume(%f.Frame* noalias nonnull align 8 dereferenceable(32) %FramePtr) #0 !dbg ![[RESUME:[0-9]+]]
 ; CHECK: entry.resume:
 ; CHECK-NEXT: call void @coro.devirt.trigger(i8* null)
 ; CHECK-NEXT: call void @llvm.dbg.declare(metadata i32* %x.addr.reload.addr, metadata ![[RESUME_VAR:[0-9]+]]
-; CHECK: define internal fastcc void @f.destroy(%f.Frame* noalias nonnull %FramePtr) #0 !dbg ![[DESTROY:[0-9]+]]
-; CHECK: define internal fastcc void @f.cleanup(%f.Frame* noalias nonnull %FramePtr) #0 !dbg ![[CLEANUP:[0-9]+]]
+; CHECK: define internal fastcc void @f.destroy(%f.Frame* noalias nonnull align 8 dereferenceable(32) %FramePtr) #0 !dbg ![[DESTROY:[0-9]+]]
+; CHECK: define internal fastcc void @f.cleanup(%f.Frame* noalias nonnull align 8 dereferenceable(32) %FramePtr) #0 !dbg ![[CLEANUP:[0-9]+]]
 
 ; CHECK: ![[ORIG]] = distinct !DISubprogram(name: "f", linkageName: "flink"
 
diff --git a/llvm/test/Transforms/Coroutines/coro-eh-aware-edge-split-01.ll b/llvm/test/Transforms/Coroutines/coro-eh-aware-edge-split-01.ll
index d8565aacb2052..4a2da113fd0c3 100644
--- a/llvm/test/Transforms/Coroutines/coro-eh-aware-edge-split-01.ll
+++ b/llvm/test/Transforms/Coroutines/coro-eh-aware-edge-split-01.ll
@@ -32,13 +32,13 @@ invoke2:
 
 ; CHECK: pad.with.phi.from.invoke2:
 ; CHECK:   %0 = cleanuppad within none []
-; CHECK:   %y.reload.addr = getelementptr inbounds %g.Frame, %g.Frame* %FramePtr, i32 0, i32 6
+; CHECK:   %y.reload.addr = getelementptr inbounds %g.Frame, %g.Frame* %FramePtr, i32 0, i32 3
 ; CHECK:   %y.reload = load i32, i32* %y.reload.addr
 ; CHECK:   cleanupret from %0 unwind label %pad.with.phi
 
 ; CHECK: pad.with.phi.from.invoke1:
 ; CHECK:   %1 = cleanuppad within none []
-; CHECK:   %x.reload.addr = getelementptr inbounds %g.Frame, %g.Frame* %FramePtr, i32 0, i32 5
+; CHECK:   %x.reload.addr = getelementptr inbounds %g.Frame, %g.Frame* %FramePtr, i32 0, i32 2
 ; CHECK:   %x.reload = load i32, i32* %x.reload.addr
 ; CHECK:   cleanupret from %1 unwind label %pad.with.phi
 
diff --git a/llvm/test/Transforms/Coroutines/coro-eh-aware-edge-split-02.ll b/llvm/test/Transforms/Coroutines/coro-eh-aware-edge-split-02.ll
index 8e9fada5c46b9..b5948e6419fa4 100644
--- a/llvm/test/Transforms/Coroutines/coro-eh-aware-edge-split-02.ll
+++ b/llvm/test/Transforms/Coroutines/coro-eh-aware-edge-split-02.ll
@@ -32,13 +32,13 @@ invoke2:
 
 ; CHECK: pad.with.phi.from.invoke2:
 ; CHECK:   %0 = cleanuppad within none []
-; CHECK:   %y.reload.addr = getelementptr inbounds %h.Frame, %h.Frame* %FramePtr, i32 0, i32 6
+; CHECK:   %y.reload.addr = getelementptr inbounds %h.Frame, %h.Frame* %FramePtr, i32 0, i32 3
 ; CHECK:   %y.reload = load i32, i32* %y.reload.addr
 ; CHECK:   cleanupret from %0 unwind label %pad.with.phi
 
 ; CHECK: pad.with.phi.from.invoke1:
 ; CHECK:   %1 = cleanuppad within none []
-; CHECK:   %x.reload.addr = getelementptr inbounds %h.Frame, %h.Frame* %FramePtr, i32 0, i32 5
+; CHECK:   %x.reload.addr = getelementptr inbounds %h.Frame, %h.Frame* %FramePtr, i32 0, i32 2
 ; CHECK:   %x.reload = load i32, i32* %x.reload.addr
 ; CHECK:   cleanupret from %1 unwind label %pad.with.phi
 
diff --git a/llvm/test/Transforms/Coroutines/coro-frame-arrayalloca.ll b/llvm/test/Transforms/Coroutines/coro-frame-arrayalloca.ll
index 56c1113f24047..8de2ac8e078ed 100644
--- a/llvm/test/Transforms/Coroutines/coro-frame-arrayalloca.ll
+++ b/llvm/test/Transforms/Coroutines/coro-frame-arrayalloca.ll
@@ -36,13 +36,13 @@ suspend:
 }
 
 ; See if the array alloca was stored as an array field.
-; CHECK-LABEL: %f.Frame = type { void (%f.Frame*)*, void (%f.Frame*)*, i1, i1, double, [4 x i32], double }
+; CHECK-LABEL: %f.Frame = type { void (%f.Frame*)*, void (%f.Frame*)*, double, double, [4 x i32], i1 }
 
 ; See if we used correct index to access prefix, data, suffix (@f)
 ; CHECK-LABEL: @f(
-; CHECK:       %prefix = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 4
-; CHECK-NEXT:  %data = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 5
-; CHECK-NEXT:  %suffix = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 6
+; CHECK:       %prefix = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 2
+; CHECK-NEXT:  %data = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 4
+; CHECK-NEXT:  %suffix = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 3
 ; CHECK-NEXT:  call void @consume.double.ptr(double* %prefix)
 ; CHECK-NEXT:  call void @consume.i32.ptr(i32* %data)
 ; CHECK-NEXT:  call void @consume.double.ptr(double* %suffix)
@@ -50,9 +50,9 @@ suspend:
 
 ; See if we used correct index to access prefix, data, suffix (@f.resume)
 ; CHECK-LABEL: @f.resume(
-; CHECK:       %[[SUFFIX:.+]] = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 6
-; CHECK:       %[[DATA:.+]] = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 5
-; CHECK:       %[[PREFIX:.+]] = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 4
+; CHECK:       %[[SUFFIX:.+]] = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 3
+; CHECK:       %[[DATA:.+]] = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 4
+; CHECK:       %[[PREFIX:.+]] = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 2
 ; CHECK:       call void @consume.double.ptr(double* %[[PREFIX]])
 ; CHECK-NEXT:  call void @consume.i32.ptr(i32* %[[DATA]])
 ; CHECK-NEXT:  call void @consume.double.ptr(double* %[[SUFFIX]])
diff --git a/llvm/test/Transforms/Coroutines/coro-frame.ll b/llvm/test/Transforms/Coroutines/coro-frame.ll
index f19e9024fc4e9..3823d661922c1 100644
--- a/llvm/test/Transforms/Coroutines/coro-frame.ll
+++ b/llvm/test/Transforms/Coroutines/coro-frame.ll
@@ -35,17 +35,17 @@ pad:
 }
 
 ; See if the float was added to the frame
-; CHECK-LABEL: %f.Frame = type { void (%f.Frame*)*, void (%f.Frame*)*, i1, i1, i64, double }
+; CHECK-LABEL: %f.Frame = type { void (%f.Frame*)*, void (%f.Frame*)*, double, i64, i1 }
 
 ; See if the float was spilled into the frame
 ; CHECK-LABEL: @f(
 ; CHECK: %r = call double @print(
-; CHECK: %r.spill.addr = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 5
+; CHECK: %r.spill.addr = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 2
 ; CHECK: store double %r, double* %r.spill.addr
 ; CHECK: ret i8* %hdl
 
-; See of the float was loaded from the frame
-; CHECK-LABEL: @f.resume(
+; See if the float was loaded from the frame
+; CHECK-LABEL: @f.resume(%f.Frame* noalias nonnull align 8
 ; CHECK: %r.reload = load double, double* %r.reload.addr
 ; CHECK: call double @print(double %r.reload)
 ; CHECK: ret void
diff --git a/llvm/test/Transforms/Coroutines/coro-heap-elide.ll b/llvm/test/Transforms/Coroutines/coro-heap-elide.ll
index fdf5d109554e1..a5315e02c7ce5 100644
--- a/llvm/test/Transforms/Coroutines/coro-heap-elide.ll
+++ b/llvm/test/Transforms/Coroutines/coro-heap-elide.ll
@@ -57,7 +57,7 @@ if.end:
 ; CHECK-LABEL: @callResume(
 define void @callResume() {
 entry:
-; CHECK: alloca %f.frame
+; CHECK: alloca [4 x i8], align 4
 ; CHECK-NOT: coro.begin
 ; CHECK-NOT: CustomAlloc
 ; CHECK: call void @may_throw()
@@ -87,7 +87,7 @@ entry:
 ; CHECK-LABEL: @callResume_with_coro_suspend_1(
 define void @callResume_with_coro_suspend_1() {
 entry:
-; CHECK: alloca %f.frame
+; CHECK: alloca [4 x i8], align 4
 ; CHECK-NOT: coro.begin
 ; CHECK-NOT: CustomAlloc
 ; CHECK: call void @may_throw()
@@ -134,7 +134,7 @@ coro.ret:
 ; CHECK-LABEL: @callResume_with_coro_suspend_2(
 define void @callResume_with_coro_suspend_2() personality i8* null {
 entry:
-; CHECK: alloca %f.frame
+; CHECK: alloca [4 x i8], align 4
 ; CHECK-NOT: coro.begin
 ; CHECK-NOT: CustomAlloc
 ; CHECK: call void @may_throw()
@@ -199,7 +199,7 @@ coro.ret:
 ; CHECK-LABEL: @callResume_with_coro_suspend_3(
 define void @callResume_with_coro_suspend_3(i8 %cond) {
 entry:
-; CHECK: alloca %f.frame
+; CHECK: alloca [4 x i8], align 4
   switch i8 %cond, label  %coro.ret [
     i8 0, label %init.suspend
     i8 1, label %coro.ret
@@ -287,7 +287,7 @@ return:
 define void @callResume_PR34897_elision(i1 %cond) {
 ; CHECK-LABEL: entry:
 entry:
-; CHECK: alloca %f.frame
+; CHECK: alloca [4 x i8], align 4
 ; CHECK: tail call void @bar(
   tail call void @bar(i8* null)
   br i1 %cond, label %if.then, label %if.else
diff --git a/llvm/test/Transforms/Coroutines/coro-materialize.ll b/llvm/test/Transforms/Coroutines/coro-materialize.ll
index 88076470f4597..2a5a7182dca99 100644
--- a/llvm/test/Transforms/Coroutines/coro-materialize.ll
+++ b/llvm/test/Transforms/Coroutines/coro-materialize.ll
@@ -34,7 +34,7 @@ suspend:
 }
 
 ; See that we only spilled one value
-; CHECK: %f.Frame = type { void (%f.Frame*)*, void (%f.Frame*)*, i1, i1, i32 }
+; CHECK: %f.Frame = type { void (%f.Frame*)*, void (%f.Frame*)*, i32, i1 }
 ; CHECK-LABEL: @f(
 
 declare i8* @llvm.coro.free(token, i8*)
diff --git a/llvm/test/Transforms/Coroutines/coro-padding.ll b/llvm/test/Transforms/Coroutines/coro-padding.ll
index f79f9c98ca746..2dbc6cbce65b3 100644
--- a/llvm/test/Transforms/Coroutines/coro-padding.ll
+++ b/llvm/test/Transforms/Coroutines/coro-padding.ll
@@ -9,7 +9,7 @@ declare void @consume(%PackedStruct*)
 
 define i8* @f() "coroutine.presplit"="1" {
 entry:
-  %data = alloca %PackedStruct, align 8
+  %data = alloca %PackedStruct, align 32
   %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
   %size = call i32 @llvm.coro.size.i32()
   %alloc = call i8* @malloc(i32 %size)
@@ -32,17 +32,17 @@ suspend:
 }
 
 ; See if the padding was inserted before PackedStruct
-; CHECK-LABEL: %f.Frame = type { void (%f.Frame*)*, void (%f.Frame*)*, i1, i1, [6 x i8], %PackedStruct }
+; CHECK-LABEL: %f.Frame = type { void (%f.Frame*)*, void (%f.Frame*)*, i1, [15 x i8], %PackedStruct }
 
-; See if we used correct index to access packed struct (padding is field 4)
+; See if we used correct index to access packed struct (padding is field 3)
 ; CHECK-LABEL: @f(
-; CHECK:       %[[DATA:.+]] = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 5
+; CHECK:       %[[DATA:.+]] = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 4
 ; CHECK-NEXT:  call void @consume(%PackedStruct* %[[DATA]])
 ; CHECK: ret i8*
 
-; See if we used correct index to access packed struct (padding is field 4)
+; See if we used correct index to access packed struct (padding is field 3)
 ; CHECK-LABEL: @f.resume(
-; CHECK:       %[[DATA:.+]] = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 5
+; CHECK:       %[[DATA:.+]] = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 4
 ; CHECK-NEXT:  call void @consume(%PackedStruct* %[[DATA]])
 ; CHECK: ret void
 
diff --git a/llvm/test/Transforms/Coroutines/coro-param-copy.ll b/llvm/test/Transforms/Coroutines/coro-param-copy.ll
index 739c4cc741815..5967a05226fdb 100644
--- a/llvm/test/Transforms/Coroutines/coro-param-copy.ll
+++ b/llvm/test/Transforms/Coroutines/coro-param-copy.ll
@@ -33,7 +33,7 @@ suspend:
 }
 
 ; See that we added both x and y to the frame.
-; CHECK: %f.Frame = type { void (%f.Frame*)*, void (%f.Frame*)*, i1, i1, i64, i64 }
+; CHECK: %f.Frame = type { void (%f.Frame*)*, void (%f.Frame*)*, i64, i64, i1 }
 
 ; See that all of the uses prior to coro-begin stays put.
 ; CHECK-LABEL: define i8* @f() {
@@ -46,10 +46,10 @@ suspend:
 
 ; See that we only copy the x as y was not modified prior to coro.begin.
 ; CHECK:  store void (%f.Frame*)* @f.destroy, void (%f.Frame*)** %destroy.addr
-; CHECK-NEXT:  %0 = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 4
+; CHECK-NEXT:  %0 = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 2
 ; CHECK-NEXT:  %1 = load i64, i64* %x.addr
 ; CHECK-NEXT:  store i64 %1, i64* %0
-; CHECK-NEXT:  %index.addr1 = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 3
+; CHECK-NEXT:  %index.addr1 = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 4
 ; CHECK-NEXT:  store i1 false, i1* %index.addr1
 ; CHECK-NEXT:  ret i8* %hdl
 
diff --git a/llvm/test/Transforms/Coroutines/coro-retcon-alloca.ll b/llvm/test/Transforms/Coroutines/coro-retcon-alloca.ll
index 17aec4eed1b23..61c21324d94a2 100644
--- a/llvm/test/Transforms/Coroutines/coro-retcon-alloca.ll
+++ b/llvm/test/Transforms/Coroutines/coro-retcon-alloca.ll
@@ -28,21 +28,20 @@ cleanup:
 
 ; CHECK-LABEL: define { i8*, i8*, i32 } @f(i8* %buffer, i32 %n)
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %buffer to i32*
-; CHECK-NEXT:    store i32 %n, i32* [[T0]], align 4
-; CHECK-NEXT:    [[ALLOC:%.*]] = tail call i8* @allocate(i32 %n)
 ; CHECK-NEXT:    [[T0:%.*]] = getelementptr inbounds i8, i8* %buffer, i64 8
-; CHECK-NEXT:    [[T1:%.*]] = bitcast i8* [[T0]] to i8**
+; CHECK-NEXT:    [[T1:%.*]] = bitcast i8* [[T0]] to i32*
+; CHECK-NEXT:    store i32 %n, i32* [[T1]], align 4
+; CHECK-NEXT:    [[ALLOC:%.*]] = tail call i8* @allocate(i32 %n)
+; CHECK-NEXT:    [[T1:%.*]] = bitcast i8* %buffer to i8**
 ; CHECK-NEXT:    store i8* [[ALLOC]], i8** [[T1]], align 8
 ; CHECK-NEXT:    [[T0:%.*]] = insertvalue { i8*, i8*, i32 } { i8* bitcast ({ i8*, i8*, i32 } (i8*, i1)* @f.resume.0 to i8*), i8* undef, i32 undef }, i8* [[ALLOC]], 1
 ; CHECK-NEXT:    [[RET:%.*]] = insertvalue { i8*, i8*, i32 } [[T0]], i32 %n, 2
 ; CHECK-NEXT:    ret { i8*, i8*, i32 } [[RET]]
 ; CHECK-NEXT:  }
 
-; CHECK-LABEL: define internal { i8*, i8*, i32 } @f.resume.0(i8* noalias nonnull %0, i1 %1)
+; CHECK-LABEL: define internal { i8*, i8*, i32 } @f.resume.0(i8* noalias nonnull align 8 dereferenceable(1024) %0, i1 %1)
 ; CHECK-NEXT:  :
-; CHECK-NEXT:    [[T0:%.*]] = getelementptr inbounds i8, i8* %0, i64 8
-; CHECK-NEXT:    [[T1:%.*]] = bitcast i8* [[T0]] to i8**
+; CHECK-NEXT:    [[T1:%.*]] = bitcast i8* %0 to i8**
 ; CHECK-NEXT:    [[ALLOC:%.*]] = load i8*, i8** [[T1]], align 8
 ; CHECK-NEXT:    tail call void @deallocate(i8* [[ALLOC]])
 ; CHECK-NEXT:    br i1 %1,
@@ -83,14 +82,14 @@ cleanup:
 ; CHECK-NEXT:    ret { i8*, i32 } [[RET]]
 ; CHECK-NEXT:  }
 
-; CHECK-LABEL: define internal { i8*, i32 } @g.resume.0(i8* noalias nonnull %0, i1 %1)
+; CHECK-LABEL: define internal { i8*, i32 } @g.resume.0(i8* noalias nonnull align 8 dereferenceable(1024) %0, i1 %1)
 ; CHECK-NEXT:  :
 ; CHECK-NEXT:    br i1 %1,
 ; CHECK:       :
 ; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %0 to i32*
-; CHECK-NEXT:    [[T1:%.*]] = load i32, i32* [[T0]], align 4
+; CHECK-NEXT:    [[T1:%.*]] = load i32, i32* [[T0]], align 8
 ; CHECK-NEXT:    %inc = add i32 [[T1]], 1
-; CHECK-NEXT:    store i32 %inc, i32* [[T0]], align 4
+; CHECK-NEXT:    store i32 %inc, i32* [[T0]], align 8
 ; CHECK-NEXT:    [[T0:%.*]] = zext i32 %inc to i64
 ; CHECK-NEXT:    [[ALLOC:%.*]] = alloca i8, i64 [[T0]], align 8
 ; CHECK-NEXT:    call void @use(i8* nonnull [[ALLOC]])
@@ -132,17 +131,17 @@ cleanup:
 ; CHECK-NEXT:    ret { i8*, i32 } [[RET]]
 ; CHECK-NEXT:  }
 
-; CHECK-LABEL: define internal { i8*, i32 } @h.resume.0(i8* noalias nonnull %0, i1 %1)
+; CHECK-LABEL: define internal { i8*, i32 } @h.resume.0(i8* noalias nonnull align 8 dereferenceable(1024) %0, i1 %1)
 ; CHECK-NEXT:  :
 ; CHECK-NEXT:    br i1 %1,
 ; CHECK:       :
 ; CHECK-NEXT:    [[NSLOT:%.*]] = bitcast i8* %0 to i32*
-; CHECK-NEXT:    [[T1:%.*]] = load i32, i32* [[NSLOT]], align 4
+; CHECK-NEXT:    [[T1:%.*]] = load i32, i32* [[NSLOT]], align 8
 ; CHECK-NEXT:    %inc = add i32 [[T1]], 1
 ; CHECK-NEXT:    [[T0:%.*]] = zext i32 %inc to i64
 ; CHECK-NEXT:    [[ALLOC:%.*]] = alloca i8, i64 [[T0]], align 8
 ; CHECK-NEXT:    call void @use(i8* nonnull [[ALLOC]])
-; CHECK-NEXT:    store i32 %inc, i32* [[NSLOT]], align 4
+; CHECK-NEXT:    store i32 %inc, i32* [[NSLOT]], align 8
 ; CHECK-NEXT:    [[RET:%.*]] = insertvalue { i8*, i32 } { i8* bitcast ({ i8*, i32 } (i8*, i1)* @h.resume.0 to i8*), i32 undef }, i32 %inc, 1
 ; CHECK-NEXT:    ret { i8*, i32 } [[RET]]
 ; CHECK:       :
@@ -180,14 +179,14 @@ loop2:
 ; CHECK-NEXT:    ret { i8*, i32 } [[RET]]
 ; CHECK-NEXT:  }
 
-; CHECK-LABEL: define internal { i8*, i32 } @i.resume.0(i8* noalias nonnull %0)
+; CHECK-LABEL: define internal { i8*, i32 } @i.resume.0(i8* noalias nonnull align 8 dereferenceable(1024) %0)
 ; CHECK-NEXT:  :
 ; CHECK-NEXT:    [[NSLOT:%.*]] = bitcast i8* %0 to i32*
-; CHECK-NEXT:    [[T1:%.*]] = load i32, i32* [[NSLOT]], align 4
+; CHECK-NEXT:    [[T1:%.*]] = load i32, i32* [[NSLOT]], align 8
 ; CHECK-NEXT:    %inc = add i32 [[T1]], 1
 ; CHECK-NEXT:    br label %loop2
 ; CHECK:       :
-; CHECK-NEXT:    store i32 %k, i32* [[NSLOT]], align 4
+; CHECK-NEXT:    store i32 %k, i32* [[NSLOT]], align 8
 ; CHECK-NEXT:    [[RET:%.*]] = insertvalue { i8*, i32 } { i8* bitcast ({ i8*, i32 } (i8*)* @i.resume.0 to i8*), i32 undef }, i32 %k, 1
 ; CHECK-NEXT:    ret { i8*, i32 } [[RET]]
 ; CHECK:       loop2:
diff --git a/llvm/test/Transforms/Coroutines/coro-retcon-once-value.ll b/llvm/test/Transforms/Coroutines/coro-retcon-once-value.ll
index ac49b22ee6bb9..6e80da87bc09f 100644
--- a/llvm/test/Transforms/Coroutines/coro-retcon-once-value.ll
+++ b/llvm/test/Transforms/Coroutines/coro-retcon-once-value.ll
@@ -45,7 +45,7 @@ cleanup:
 ; CHECK-NEXT:    ret { i8*, i32 } [[T1]]
 ; CHECK-NEXT:  }
 
-; CHECK-LABEL: define internal void @f.resume.0(i8* noalias nonnull %0, i1 zeroext %1)
+; CHECK-LABEL: define internal void @f.resume.0(i8* noalias nonnull align 8 dereferenceable(8) %0, i1 zeroext %1)
 ; CHECK-NEXT:  :
 ; CHECK-NEXT:    br i1 %1,
 ; CHECK:       :
@@ -57,7 +57,7 @@ cleanup:
 ; CHECK-NEXT:    ret void
 ; CHECK-NEXT:  }
 
-; CHECK-LABEL: define internal void @f.resume.1(i8* noalias nonnull %0, i1 zeroext %1)
+; CHECK-LABEL: define internal void @f.resume.1(i8* noalias nonnull align 8 dereferenceable(8) %0, i1 zeroext %1)
 ; CHECK-NEXT:  :
 ; CHECK-NEXT:    br i1 %1,
 ; CHECK:       :
diff --git a/llvm/test/Transforms/Coroutines/coro-retcon-once-value2.ll b/llvm/test/Transforms/Coroutines/coro-retcon-once-value2.ll
index ad49f24dc547a..4f43da03550c8 100644
--- a/llvm/test/Transforms/Coroutines/coro-retcon-once-value2.ll
+++ b/llvm/test/Transforms/Coroutines/coro-retcon-once-value2.ll
@@ -37,7 +37,7 @@ cleanup:
 ; CHECK-NEXT:    ret { i8*, i32* } [[T0]]
 ; CHECK-NEXT:  }
 
-; CHECK-LABEL: define internal void @f.resume.0(i8* noalias nonnull %0, i1 zeroext %1)
+; CHECK-LABEL: define internal void @f.resume.0(i8* noalias nonnull align 8 dereferenceable(8) %0, i1 zeroext %1)
 ; CHECK-NEXT:  :
 ; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %0 to [[FRAME_T:%.*]]**
 ; CHECK-NEXT:    [[FRAME:%.*]] = load [[FRAME_T]]*, [[FRAME_T]]** [[T0]]
diff --git a/llvm/test/Transforms/Coroutines/coro-retcon-resume-values.ll b/llvm/test/Transforms/Coroutines/coro-retcon-resume-values.ll
index ac99dd15b9882..80e8170d7ba1e 100644
--- a/llvm/test/Transforms/Coroutines/coro-retcon-resume-values.ll
+++ b/llvm/test/Transforms/Coroutines/coro-retcon-resume-values.ll
@@ -30,7 +30,7 @@ cleanup:
 ; CHECK-NEXT:    ret i8* bitcast (i8* (i8*, i32, i1)* @f.resume.0 to i8*)
 ; CHECK-NEXT:  }
 
-; CHECK-LABEL: define internal i8* @f.resume.0(i8* noalias nonnull %0, i32 %1, i1 zeroext %2)
+; CHECK-LABEL: define internal i8* @f.resume.0(i8* noalias nonnull align 4 dereferenceable(8) %0, i32 %1, i1 zeroext %2)
 ; CHECK-NEXT:  :
 ; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %0 to i32*
 ; CHECK-NEXT:    [[T1:%.*]] = load i32, i32* [[T0]], align 4
diff --git a/llvm/test/Transforms/Coroutines/coro-retcon-resume-values2.ll b/llvm/test/Transforms/Coroutines/coro-retcon-resume-values2.ll
index 43f98e958aab3..e2412b6b8a5a3 100644
--- a/llvm/test/Transforms/Coroutines/coro-retcon-resume-values2.ll
+++ b/llvm/test/Transforms/Coroutines/coro-retcon-resume-values2.ll
@@ -29,7 +29,7 @@ entry:
 ; CHECK-NEXT:    ret i8* bitcast (i8* (i8*, i32)* @f.resume.0 to i8*)
 ; CHECK-NEXT:  }
 
-; CHECK-LABEL: define internal i8* @f.resume.0(i8* noalias nonnull %0, i32 %1)
+; CHECK-LABEL: define internal i8* @f.resume.0(i8* noalias nonnull align 4 dereferenceable(8) %0, i32 %1)
 ; CHECK-NEXT:  :
 ; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %0 to [[FRAME_T:%.*]]**
 ; CHECK-NEXT:    [[FRAME:%.*]] = load [[FRAME_T]]*, [[FRAME_T]]** [[T0]]
@@ -45,7 +45,7 @@ entry:
 ; CHECK-NEXT:    ret i8* [[CONT]]
 ; CHECK-NEXT:  }
 
-; CHECK-LABEL: define internal i8* @f.resume.1(i8* noalias nonnull %0, i32 %1)
+; CHECK-LABEL: define internal i8* @f.resume.1(i8* noalias nonnull align 4 dereferenceable(8) %0, i32 %1)
 ; CHECK-NEXT:  :
 ; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %0 to [[FRAME_T:%.*]]**
 ; CHECK-NEXT:    [[FRAME:%.*]] = load [[FRAME_T]]*, [[FRAME_T]]** [[T0]]
@@ -64,7 +64,7 @@ entry:
 ; CHECK-NEXT:    ret i8* [[CONT]]
 ; CHECK-NEXT:  }
 
-; CHECK-LABEL: define internal i8* @f.resume.2(i8* noalias nonnull %0, i32 %1)
+; CHECK-LABEL: define internal i8* @f.resume.2(i8* noalias nonnull align 4 dereferenceable(8) %0, i32 %1)
 ; CHECK-NEXT:  :
 ; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* %0 to [[FRAME_T:%.*]]**
 ; CHECK-NEXT:    [[FRAME:%.*]] = load [[FRAME_T]]*, [[FRAME_T]]** [[T0]]
diff --git a/llvm/test/Transforms/Coroutines/coro-retcon-value.ll b/llvm/test/Transforms/Coroutines/coro-retcon-value.ll
index cfda73bbe754a..29ec7cda170f2 100644
--- a/llvm/test/Transforms/Coroutines/coro-retcon-value.ll
+++ b/llvm/test/Transforms/Coroutines/coro-retcon-value.ll
@@ -30,7 +30,7 @@ cleanup:
 ; CHECK-NEXT:    ret { i8*, i32 } [[RET]]
 ; CHECK-NEXT:  }
 
-; CHECK-LABEL: define internal { i8*, i32 } @f.resume.0(i8* noalias nonnull %0, i8 zeroext %1)
+; CHECK-LABEL: define internal { i8*, i32 } @f.resume.0(i8* noalias nonnull align 4 dereferenceable(8) %0, i8 zeroext %1)
 ; CHECK-NEXT:  :
 ; CHECK-NEXT:    [[T0:%.*]] = icmp eq i8 %1, 0
 ; CHECK-NEXT:    br i1 [[T0]],
diff --git a/llvm/test/Transforms/Coroutines/coro-retcon.ll b/llvm/test/Transforms/Coroutines/coro-retcon.ll
index 5cd4cb61d94cc..13283f05b2661 100644
--- a/llvm/test/Transforms/Coroutines/coro-retcon.ll
+++ b/llvm/test/Transforms/Coroutines/coro-retcon.ll
@@ -30,7 +30,7 @@ cleanup:
 ; CHECK-NEXT:    ret i8* bitcast (i8* (i8*, i1)* @f.resume.0 to i8*)
 ; CHECK-NEXT:  }
 
-; CHECK-LABEL: define internal i8* @f.resume.0(i8* noalias nonnull %0, i1 zeroext %1)
+; CHECK-LABEL: define internal i8* @f.resume.0(i8* noalias nonnull align 4 dereferenceable(8) %0, i1 zeroext %1)
 ; CHECK-NEXT:  :
 ; CHECK-NEXT:    br i1 %1,
 ; CHECK:       :
diff --git a/llvm/test/Transforms/Coroutines/coro-spill-after-phi.ll b/llvm/test/Transforms/Coroutines/coro-spill-after-phi.ll
index a22d824343366..1be173e5fc1ea 100644
--- a/llvm/test/Transforms/Coroutines/coro-spill-after-phi.ll
+++ b/llvm/test/Transforms/Coroutines/coro-spill-after-phi.ll
@@ -34,14 +34,14 @@ suspend:
 }
 
 ; Verifies that the both phis are stored correctly in the coroutine frame
-; CHECK: %f.Frame = type { void (%f.Frame*)*, void (%f.Frame*)*, i1, i1, i32, i32 }
+; CHECK: %f.Frame = type { void (%f.Frame*)*, void (%f.Frame*)*, i32, i32, i1 }
 ; CHECK-LABEL: @f(
 ; CHECK: store void (%f.Frame*)* @f.destroy, void (%f.Frame*)** %destroy.addr
 ; CHECK: %phi1 = select i1 %n, i32 0, i32 2
 ; CHECK: %phi2 = select i1 %n, i32 1, i32 3
-; CHECK: %phi2.spill.addr = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 5
+; CHECK: %phi2.spill.addr = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 3
 ; CHECK: store i32 %phi2, i32* %phi2.spill.addr
-; CHECK: %phi1.spill.addr = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 4
+; CHECK: %phi1.spill.addr = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 2
 ; CHECK: store i32 %phi1, i32* %phi1.spill.addr
 ; CHECK: ret i8* %hdl
 
diff --git a/llvm/test/Transforms/Coroutines/coro-spill-corobegin.ll b/llvm/test/Transforms/Coroutines/coro-spill-corobegin.ll
index 0caea1c90fbb8..8923e2880f282 100644
--- a/llvm/test/Transforms/Coroutines/coro-spill-corobegin.ll
+++ b/llvm/test/Transforms/Coroutines/coro-spill-corobegin.ll
@@ -38,18 +38,18 @@ suspend:
 }
 
 ; See if the i8* for coro.begin was added to f.Frame
-; CHECK-LABEL: %f.Frame = type { void (%f.Frame*)*, void (%f.Frame*)*, i1, i1, i8* }
+; CHECK-LABEL: %f.Frame = type { void (%f.Frame*)*, void (%f.Frame*)*, i8*, i1 }
 
 ; See if the g's coro.begin was spilled into the frame
 ; CHECK-LABEL: @f(
 ; CHECK: %innerid = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* bitcast ([3 x void (%g.Frame*)*]* @g.resumers to i8*))
 ; CHECK: %innerhdl = call noalias nonnull i8* @llvm.coro.begin(token %innerid, i8* null)
-; CHECK: %[[spilladdr:.+]] = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 4
+; CHECK: %[[spilladdr:.+]] = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 2
 ; CHECK: store i8* %innerhdl, i8** %[[spilladdr]]
 
 ; See if the coro.begin was loaded from the frame
 ; CHECK-LABEL: @f.resume(
-; CHECK: %[[innerhdlAddr:.+]] = getelementptr inbounds %f.Frame, %f.Frame* %{{.+}}, i32 0, i32 4
+; CHECK: %[[innerhdlAddr:.+]] = getelementptr inbounds %f.Frame, %f.Frame* %{{.+}}, i32 0, i32 2
 ; CHECK: %[[innerhdl:.+]] = load i8*, i8** %[[innerhdlAddr]]
 ; CHECK: %[[gframe:.+]] = bitcast i8* %[[innerhdl]] to %g.Frame*
 ; CHECK: %[[gvarAddr:.+]] = getelementptr inbounds %g.Frame, %g.Frame* %[[gframe]], i32 0, i32 4
diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail2.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail2.ll
new file mode 100644
index 0000000000000..2fa66c29b01b4
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-split-musttail2.ll
@@ -0,0 +1,68 @@
+; Tests that coro-split will convert coro.resume followed by a suspend to a
+; musttail call.
+; RUN: opt < %s -coro-split -S | FileCheck %s
+; RUN: opt < %s -passes=coro-split -S | FileCheck %s
+
+define void @fakeresume1(i8*)  {
+entry:
+  ret void;
+}
+
+define void @fakeresume2(i64*)  {
+entry:
+  ret void;
+}
+
+define void @g() #0 {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %alloc = call i8* @malloc(i64 16) #3
+  %vFrame = call noalias nonnull i8* @llvm.coro.begin(token %id, i8* %alloc)
+
+  %save = call token @llvm.coro.save(i8* null)
+  call fastcc void @fakeresume1(i8* null)
+
+  %suspend = call i8 @llvm.coro.suspend(token %save, i1 false)
+  switch i8 %suspend, label %exit [
+    i8 0, label %await.ready
+    i8 1, label %exit
+  ]
+await.ready:
+  %save2 = call token @llvm.coro.save(i8* null)
+  call fastcc void @fakeresume2(i64* null)
+
+  %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false)
+  switch i8 %suspend2, label %exit [
+    i8 0, label %exit
+    i8 1, label %exit
+  ]
+exit:
+  call i1 @llvm.coro.end(i8* null, i1 false)
+  ret void
+}
+
+; Verify that in the initial function resume is not marked with musttail.
+; CHECK-LABEL: @g(
+; CHECK-NOT: musttail call fastcc void @fakeresume1(i8* null)
+
+; Verify that in the resume part resume call is marked with musttail.
+; CHECK-LABEL: @g.resume(
+; CHECK: musttail call fastcc void @fakeresume2(i64* null)
+; CHECK-NEXT: ret void
+
+declare token @llvm.coro.id(i32, i8* readnone, i8* nocapture readonly, i8*) #1
+declare i1 @llvm.coro.alloc(token) #2
+declare i64 @llvm.coro.size.i64() #3
+declare i8* @llvm.coro.begin(token, i8* writeonly) #2
+declare token @llvm.coro.save(i8*) #2
+declare i8* @llvm.coro.frame() #3
+declare i8 @llvm.coro.suspend(token, i1) #2
+declare i8* @llvm.coro.free(token, i8* nocapture readonly) #1
+declare i1 @llvm.coro.end(i8*, i1) #2
+declare i8* @llvm.coro.subfn.addr(i8* nocapture readonly, i8) #1
+declare i8* @malloc(i64)
+
+attributes #0 = { "coroutine.presplit"="1" }
+attributes #1 = { argmemonly nounwind readonly }
+attributes #2 = { nounwind }
+attributes #3 = { nounwind readnone }
diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail3.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail3.ll
new file mode 100644
index 0000000000000..524fd3a62f6d1
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-split-musttail3.ll
@@ -0,0 +1,101 @@
+; Tests that coro-split will convert coro.resume followed by a suspend to a
+; musttail call.
+; RUN: opt < %s -coro-split -S | FileCheck %s
+; RUN: opt < %s -passes=coro-split -S | FileCheck %s
+
+define void @f() #0 {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %alloc = call i8* @malloc(i64 16) #3
+  %vFrame = call noalias nonnull i8* @llvm.coro.begin(token %id, i8* %alloc)
+
+  %save = call token @llvm.coro.save(i8* null)
+  %addr1 = call i8* @llvm.coro.subfn.addr(i8* null, i8 0)
+  %pv1 = bitcast i8* %addr1 to void (i8*)*
+  call fastcc void %pv1(i8* null)
+
+  %suspend = call i8 @llvm.coro.suspend(token %save, i1 false)
+  %cmp = icmp eq i8 %suspend, 0
+  br i1 %cmp, label %await.suspend, label %exit
+await.suspend:
+  %save2 = call token @llvm.coro.save(i8* null)
+  %br0 = call i8 @switch_result()
+  switch i8 %br0, label %unreach [
+    i8 0, label %await.resume3
+    i8 1, label %await.resume1
+    i8 2, label %await.resume2
+  ]
+await.resume1:
+  %hdl = call i8* @g()
+  %addr2 = call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 0)
+  %pv2 = bitcast i8* %addr2 to void (i8*)*
+  call fastcc void %pv2(i8* %hdl)
+  br label %final.suspend
+await.resume2:
+  %hdl2 = call i8* @h()
+  %addr3 = call i8* @llvm.coro.subfn.addr(i8* %hdl2, i8 0)
+  %pv3 = bitcast i8* %addr3 to void (i8*)*
+  call fastcc void %pv3(i8* %hdl2)
+  br label %final.suspend
+await.resume3:
+  %addr4 = call i8* @llvm.coro.subfn.addr(i8* null, i8 0)
+  %pv4 = bitcast i8* %addr4 to void (i8*)*
+  call fastcc void %pv4(i8* null)
+  br label %final.suspend
+final.suspend:
+  %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false)
+  %cmp2 = icmp eq i8 %suspend2, 0
+  br i1 %cmp2, label %pre.exit, label %exit
+pre.exit:
+  br label %exit
+exit:
+  call i1 @llvm.coro.end(i8* null, i1 false)
+  ret void
+unreach:
+  unreachable
+}
+
+; Verify that in the initial function resume is not marked with musttail.
+; CHECK-LABEL: @f(
+; CHECK: %[[addr1:.+]] = call i8* @llvm.coro.subfn.addr(i8* null, i8 0)
+; CHECK-NEXT: %[[pv1:.+]] = bitcast i8* %[[addr1]] to void (i8*)*
+; CHECK-NOT: musttail call fastcc void %[[pv1]](i8* null)
+
+; Verify that in the resume part resume call is marked with musttail.
+; CHECK-LABEL: @f.resume(
+; CHECK: %[[hdl:.+]] = call i8* @g()
+; CHECK-NEXT: %[[addr2:.+]] = call i8* @llvm.coro.subfn.addr(i8* %[[hdl]], i8 0)
+; CHECK-NEXT: %[[pv2:.+]] = bitcast i8* %[[addr2]] to void (i8*)*
+; CHECK-NEXT: musttail call fastcc void %[[pv2]](i8* %[[hdl]])
+; CHECK-NEXT: ret void
+; CHECK: %[[hdl2:.+]] = call i8* @h()
+; CHECK-NEXT: %[[addr3:.+]] = call i8* @llvm.coro.subfn.addr(i8* %[[hdl2]], i8 0)
+; CHECK-NEXT: %[[pv3:.+]] = bitcast i8* %[[addr3]] to void (i8*)*
+; CHECK-NEXT: musttail call fastcc void %[[pv3]](i8* %[[hdl2]])
+; CHECK-NEXT: ret void
+; CHECK: %[[addr4:.+]] = call i8* @llvm.coro.subfn.addr(i8* null, i8 0)
+; CHECK-NEXT: %[[pv4:.+]] = bitcast i8* %[[addr4]] to void (i8*)*
+; CHECK-NEXT: musttail call fastcc void %[[pv4]](i8* null)
+; CHECK-NEXT: ret void
+
+
+
+declare token @llvm.coro.id(i32, i8* readnone, i8* nocapture readonly, i8*) #1
+declare i1 @llvm.coro.alloc(token) #2
+declare i64 @llvm.coro.size.i64() #3
+declare i8* @llvm.coro.begin(token, i8* writeonly) #2
+declare token @llvm.coro.save(i8*) #2
+declare i8* @llvm.coro.frame() #3
+declare i8 @llvm.coro.suspend(token, i1) #2
+declare i8* @llvm.coro.free(token, i8* nocapture readonly) #1
+declare i1 @llvm.coro.end(i8*, i1) #2
+declare i8* @llvm.coro.subfn.addr(i8* nocapture readonly, i8) #1
+declare i8* @malloc(i64)
+declare i8 @switch_result()
+declare i8* @g()
+declare i8* @h()
+
+attributes #0 = { "coroutine.presplit"="1" }
+attributes #1 = { argmemonly nounwind readonly }
+attributes #2 = { nounwind }
+attributes #3 = { nounwind readnone }
diff --git a/llvm/test/Transforms/Coroutines/coro-swifterror.ll b/llvm/test/Transforms/Coroutines/coro-swifterror.ll
index 932e448a57193..7390bb77ca9b6 100644
--- a/llvm/test/Transforms/Coroutines/coro-swifterror.ll
+++ b/llvm/test/Transforms/Coroutines/coro-swifterror.ll
@@ -40,7 +40,7 @@ cleanup:
 ; CHECK-NEXT:    ret i8* bitcast (i8* (i8*, i1, i8**)* @f.resume.0 to i8*)
 ; CHECK-NEXT:  }
 
-; CHECK-LABEL: define internal i8* @f.resume.0(i8* noalias nonnull %0, i1 zeroext %1, i8** swifterror %2)
+; CHECK-LABEL: define internal i8* @f.resume.0(i8* noalias nonnull align 4 dereferenceable(8) %0, i1 zeroext %1, i8** swifterror %2)
 ; CHECK-NEXT:  :
 ; CHECK-NEXT:    br i1 %1,
 ; CHECK:       :
@@ -102,7 +102,7 @@ cleanup:
 ; CHECK-NEXT:    ret i8* bitcast (i8* (i8*, i1)* @g.resume.0 to i8*)
 ; CHECK-NEXT:  }
 
-; CHECK-LABEL: define internal i8* @g.resume.0(i8* noalias nonnull %0, i1 zeroext %1)
+; CHECK-LABEL: define internal i8* @g.resume.0(i8* noalias nonnull align 4 dereferenceable(8) %0, i1 zeroext %1)
 ; CHECK-NEXT:  :
 ; CHECK-NEXT:    [[ERRORSLOT:%.*]] = alloca swifterror i8*, align 4
 ; CHECK-NEXT:    br i1 %1,
diff --git a/llvm/test/Transforms/Coroutines/ex0.ll b/llvm/test/Transforms/Coroutines/ex0.ll
index 514ae867ae098..de5752387cf21 100644
--- a/llvm/test/Transforms/Coroutines/ex0.ll
+++ b/llvm/test/Transforms/Coroutines/ex0.ll
@@ -1,6 +1,6 @@
 ; First example from Doc/Coroutines.rst (two block loop)
-; RUN: opt < %s -enable-coroutines -O2 -S | FileCheck %s
-; RUN: opt < %s -enable-coroutines -aa-pipeline=basic-aa -passes='default<O2>' -S | FileCheck %s
+; RUN: opt < %s -enable-coroutines -O2 -preserve-alignment-assumptions-during-inlining=false -S | FileCheck %s
+; RUN: opt < %s -enable-coroutines -aa-pipeline=basic-aa -passes='default<O2>' -preserve-alignment-assumptions-during-inlining=false -S | FileCheck %s
 
 define i8* @f(i32 %n) {
 entry:
diff --git a/llvm/test/Transforms/Coroutines/ex1.ll b/llvm/test/Transforms/Coroutines/ex1.ll
index 161b59fb918bd..42f603865b69c 100644
--- a/llvm/test/Transforms/Coroutines/ex1.ll
+++ b/llvm/test/Transforms/Coroutines/ex1.ll
@@ -1,6 +1,6 @@
 ; First example from Doc/Coroutines.rst (one block loop)
-; RUN: opt < %s -O2 -enable-coroutines -S | FileCheck %s
-; RUN: opt < %s -aa-pipeline=basic-aa -passes='default<O2>' -enable-coroutines -S | FileCheck %s
+; RUN: opt < %s -O2 -enable-coroutines -preserve-alignment-assumptions-during-inlining=false -S | FileCheck %s
+; RUN: opt < %s -aa-pipeline=basic-aa -passes='default<O2>' -enable-coroutines -preserve-alignment-assumptions-during-inlining=false -S | FileCheck %s
 
 define i8* @f(i32 %n) {
 entry:
diff --git a/llvm/test/Transforms/Coroutines/ex5.ll b/llvm/test/Transforms/Coroutines/ex5.ll
index 84728f56b04a4..dd566505ba851 100644
--- a/llvm/test/Transforms/Coroutines/ex5.ll
+++ b/llvm/test/Transforms/Coroutines/ex5.ll
@@ -1,6 +1,6 @@
 ; Fifth example from Doc/Coroutines.rst (final suspend)
-; RUN: opt < %s -O2 -enable-coroutines -S | FileCheck %s
-; RUN: opt < %s -aa-pipeline=basic-aa -passes='default<O2>' -enable-coroutines -S | FileCheck %s
+; RUN: opt < %s -O2 -enable-coroutines -preserve-alignment-assumptions-during-inlining=false -S | FileCheck %s
+; RUN: opt < %s -aa-pipeline=basic-aa -passes='default<O2>' -enable-coroutines -preserve-alignment-assumptions-during-inlining=false -S | FileCheck %s
 
 define i8* @f(i32 %n) {
 entry:
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/merge-range-and-undef.ll b/llvm/test/Transforms/CorrelatedValuePropagation/merge-range-and-undef.ll
index 271c52af779dc..0630c94aa996f 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/merge-range-and-undef.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/merge-range-and-undef.ll
@@ -3,6 +3,8 @@
 
 ; Test case for PR44949.
 
+declare void @use(i1)
+
 ; We can remove `%res = and i64 %p, 255`, because %r = 0 and we can eliminate
 ; %p as well.
 define i64 @constant_and_undef(i1 %c1, i64 %a) {
@@ -15,6 +17,8 @@ define i64 @constant_and_undef(i1 %c1, i64 %a) {
 ; CHECK-NEXT:    [[R:%.*]] = and i64 [[A:%.*]], 0
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    call void @use(i1 true)
 ; CHECK-NEXT:    ret i64 0
 ;
 entry:
@@ -30,12 +34,15 @@ bb2:
 bb3:
   %p = phi i64 [ undef, %bb1 ], [ %r, %bb2 ]
   %res = and i64 %p, 255
+  %f.1 = icmp eq i64 %p, 1
+  call void @use(i1 %f.1)
+  %t.1 = icmp eq i64 %p, 0
+  call void @use(i1 %t.1)
   ret i64 %res
 }
 
 ; Check that we go to overdefined when merging a constant range with undef. We
 ; cannot remove '%res = and i64 %p, 255'.
-; FIXME: should not remove '%res = and i64 %p, 255'
 define i64 @constant_range_and_undef(i1 %cond, i64 %a) {
 ; CHECK-LABEL: @constant_range_and_undef(
 ; CHECK-NEXT:  entry:
@@ -48,6 +55,10 @@ define i64 @constant_range_and_undef(i1 %cond, i64 %a) {
 ; CHECK:       bb3:
 ; CHECK-NEXT:    [[P:%.*]] = phi i64 [ undef, [[BB1]] ], [ [[R]], [[BB2]] ]
 ; CHECK-NEXT:    [[RES:%.*]] = and i64 [[P]], 255
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    call void @use(i1 true)
 ; CHECK-NEXT:    ret i64 [[RES]]
 ;
 entry:
@@ -63,6 +74,13 @@ bb2:
 bb3:
   %p = phi i64 [ undef, %bb1 ], [ %r, %bb2 ]
   %res = and i64 %p, 255
+  br label %exit
+
+exit: ; CVP only simplifies based on ranges for non-local conditions.
+  %f.1 = icmp eq i64 %p, 256
+  call void @use(i1 %f.1)
+  %t.1 = icmp ne i64 %p, 256
+  call void @use(i1 %t.1)
   ret i64 %res
 }
 
@@ -87,6 +105,12 @@ define i64 @constant_range_and_undef2(i1 %c1, i1 %c2, i64 %a) {
 ; CHECK:       bb6:
 ; CHECK-NEXT:    [[P2:%.*]] = phi i64 [ [[P]], [[BB4]] ], [ [[V3]], [[BB5]] ]
 ; CHECK-NEXT:    [[RES:%.*]] = and i64 [[P2]], 255
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[F_1:%.*]] = icmp eq i64 [[P]], 256
+; CHECK-NEXT:    call void @use(i1 [[F_1]])
+; CHECK-NEXT:    [[T_1:%.*]] = icmp ne i64 [[P]], 256
+; CHECK-NEXT:    call void @use(i1 [[T_1]])
 ; CHECK-NEXT:    ret i64 [[RES]]
 ;
 entry:
@@ -114,6 +138,13 @@ bb5:
 bb6:
   %p2 = phi i64 [ %p, %bb4 ], [ %v3, %bb5 ]
   %res = and i64 %p2, 255
+  br label %exit
+
+exit: ; CVP only simplifies based on ranges for non-local conditions.
+  %f.1 = icmp eq i64 %p, 256
+  call void @use(i1 %f.1)
+  %t.1 = icmp ne i64 %p, 256
+  call void @use(i1 %t.1)
   ret i64 %res
 }
 
@@ -128,8 +159,11 @@ define i1 @constant_range_and_undef_3(i1 %cond, i64 %a) {
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    [[P:%.*]] = phi i64 [ undef, [[BB1]] ], [ [[R]], [[BB2]] ]
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i64 [[P]], 256
-; CHECK-NEXT:    ret i1 [[C]]
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    ret i1 true
 ;
 entry:
   br i1 %cond, label %bb1, label %bb2
@@ -143,7 +177,14 @@ bb2:
 
 bb3:
   %p = phi i64 [ undef, %bb1 ], [ %r, %bb2 ]
+  br label %exit
+
+exit: ; CVP only simplifies based on ranges for non-local conditions.
   %c = icmp ult i64 %p, 256
+  %f.1 = icmp eq i64 %p, 256
+  call void @use(i1 %f.1)
+  %t.1 = icmp ne i64 %p, 256
+  call void @use(i1 %t.1)
   ret i1 %c
 }
 
@@ -163,6 +204,10 @@ define i64 @constant_range_and_undef_3_incoming_v1(i1 %c1, i1 %c2, i64 %a) {
 ; CHECK:       bb4:
 ; CHECK-NEXT:    [[P:%.*]] = phi i64 [ [[R]], [[BB1]] ], [ 10, [[BB2]] ], [ undef, [[BB3]] ]
 ; CHECK-NEXT:    [[RES:%.*]] = and i64 [[P]], 255
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    call void @use(i1 true)
 ; CHECK-NEXT:    ret i64 [[RES]]
 ;
 entry:
@@ -181,6 +226,13 @@ bb3:
 bb4:
   %p = phi i64 [ %r, %bb1 ], [ 10, %bb2], [ undef, %bb3 ]
   %res = and i64 %p, 255
+  br label %exit
+
+exit: ; CVP only simplifies based on ranges for non-local conditions.
+  %f.1 = icmp eq i64 %p, 256
+  call void @use(i1 %f.1)
+  %t.1 = icmp ne i64 %p, 256
+  call void @use(i1 %t.1)
   ret i64 %res
 }
 
@@ -200,6 +252,10 @@ define i64 @constant_range_and_undef_3_incoming_v2(i1 %c1, i1 %c2, i64 %a) {
 ; CHECK:       bb4:
 ; CHECK-NEXT:    [[P:%.*]] = phi i64 [ undef, [[BB1]] ], [ 10, [[BB2]] ], [ [[R]], [[BB3]] ]
 ; CHECK-NEXT:    [[RES:%.*]] = and i64 [[P]], 255
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    call void @use(i1 true)
 ; CHECK-NEXT:    ret i64 [[RES]]
 ;
 entry:
@@ -218,6 +274,13 @@ bb3:
 bb4:
   %p = phi i64 [ undef, %bb1 ], [ 10, %bb2], [ %r, %bb3 ]
   %res = and i64 %p, 255
+  br label %exit
+
+exit: ; CVP only simplifies based on ranges for non-local conditions.
+  %f.1 = icmp eq i64 %p, 256
+  call void @use(i1 %f.1)
+  %t.1 = icmp ne i64 %p, 256
+  call void @use(i1 %t.1)
   ret i64 %res
 }
 
@@ -237,6 +300,10 @@ define i64 @constant_range_and_undef_3_incoming_v3(i1 %c1, i1 %c2, i64 %a) {
 ; CHECK:       bb4:
 ; CHECK-NEXT:    [[P:%.*]] = phi i64 [ [[R]], [[BB1]] ], [ undef, [[BB2]] ], [ 10, [[BB3]] ]
 ; CHECK-NEXT:    [[RES:%.*]] = and i64 [[P]], 255
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    call void @use(i1 true)
 ; CHECK-NEXT:    ret i64 [[RES]]
 ;
 entry:
@@ -255,10 +322,16 @@ bb3:
 bb4:
   %p = phi i64 [ %r, %bb1 ], [ undef, %bb2], [ 10, %bb3 ]
   %res = and i64 %p, 255
+  br label %exit
+
+exit: ; CVP only simplifies based on ranges for non-local conditions.
+  %f.1 = icmp eq i64 %p, 256
+  call void @use(i1 %f.1)
+  %t.1 = icmp ne i64 %p, 256
+  call void @use(i1 %t.1)
   ret i64 %res
 }
 
-
 define i64 @constant_range_and_phi_constant_undef(i1 %c1, i1 %c2, i64 %a) {
 ; CHECK-LABEL: @constant_range_and_phi_constant_undef(
 ; CHECK-NEXT:  entry:
@@ -274,6 +347,10 @@ define i64 @constant_range_and_phi_constant_undef(i1 %c1, i1 %c2, i64 %a) {
 ; CHECK-NEXT:    br label [[BB5]]
 ; CHECK:       bb5:
 ; CHECK-NEXT:    [[P:%.*]] = phi i64 [ [[R]], [[BB1]] ], [ 10, [[BB4]] ]
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    call void @use(i1 true)
 ; CHECK-NEXT:    ret i64 [[P]]
 ;
 entry:
@@ -296,5 +373,12 @@ bb4:
 bb5:
   %p = phi i64 [ %r, %bb1 ], [ %p.1, %bb4]
   %res = and i64 %p, 255
+  br label %exit
+
+exit: ; CVP only simplifies based on ranges for non-local conditions.
+  %f.1 = icmp eq i64 %p, 256
+  call void @use(i1 %f.1)
+  %t.1 = icmp ne i64 %p, 256
+  call void @use(i1 %t.1)
   ret i64 %res
 }
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/range.ll b/llvm/test/Transforms/CorrelatedValuePropagation/range.ll
index 6315e3bd74da1..634c37568ea32 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/range.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/range.ll
@@ -457,7 +457,7 @@ define i1 @test14_slt(i32 %a) {
 ; CHECK-NEXT:    br i1 [[CMP]], label [[THEN:%.*]], label [[ELSE:%.*]]
 ; CHECK:       then:
 ; CHECK-NEXT:    [[RESULT:%.*]] = or i1 false, false
-; CHECK-NEXT:    ret i1 [[RESULT]]
+; CHECK-NEXT:    ret i1 false
 ; CHECK:       else:
 ; CHECK-NEXT:    ret i1 false
 ;
@@ -508,7 +508,7 @@ define i1 @test14_sgt(i32 %a) {
 ; CHECK-NEXT:    br i1 [[CMP]], label [[THEN:%.*]], label [[ELSE:%.*]]
 ; CHECK:       then:
 ; CHECK-NEXT:    [[RESULT:%.*]] = or i1 false, false
-; CHECK-NEXT:    ret i1 [[RESULT]]
+; CHECK-NEXT:    ret i1 false
 ; CHECK:       else:
 ; CHECK-NEXT:    ret i1 false
 ;
@@ -585,7 +585,7 @@ define i1 @test14_ugt(i32 %a) {
 ; CHECK-NEXT:    br i1 [[CMP]], label [[THEN:%.*]], label [[ELSE:%.*]]
 ; CHECK:       then:
 ; CHECK-NEXT:    [[RESULT:%.*]] = or i1 false, false
-; CHECK-NEXT:    ret i1 [[RESULT]]
+; CHECK-NEXT:    ret i1 false
 ; CHECK:       else:
 ; CHECK-NEXT:    ret i1 false
 ;
@@ -629,6 +629,31 @@ else:
   ret i1 false
 }
 
+define i1 @test14_ugt_and(i32 %a) {
+; CHECK-LABEL: @test14_ugt_and(
+; CHECK-NEXT:    [[A_OFF:%.*]] = add i32 [[A:%.*]], -8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[A_OFF]], 8
+; CHECK-NEXT:    br i1 [[CMP]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[RESULT:%.*]] = and i1 false, false
+; CHECK-NEXT:    ret i1 false
+; CHECK:       else:
+; CHECK-NEXT:    ret i1 false
+;
+  %a.off = add i32 %a, -8
+  %cmp = icmp ugt i32 %a.off, 8
+  br i1 %cmp, label %then, label %else
+
+then:
+  %dead.1 = icmp eq i32 %a, 8
+  %dead.2 = icmp eq i32 %a, 16
+  %result = and i1 %dead.1, %dead.2
+  ret i1 %result
+
+else:
+  ret i1 false
+}
+
 @limit = external global i32
 define i1 @test15(i32 %a) {
 ; CHECK-LABEL: @test15(
diff --git a/llvm/test/Transforms/DeadStoreElimination/simple.ll b/llvm/test/Transforms/DeadStoreElimination/simple.ll
index 84034caee55f1..7efd3953937f0 100644
--- a/llvm/test/Transforms/DeadStoreElimination/simple.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/simple.ll
@@ -259,6 +259,8 @@ define i32 addrspace(1)* @test13_addrspacecast() {
 
 declare noalias i8* @malloc(i32)
 declare noalias i8* @calloc(i32, i32)
+declare noalias i8* @aligned_alloc(i32, i32)
+declare void @free(i8*)
 
 
 define void @test14(i32* %Q) {
@@ -272,6 +274,17 @@ define void @test14(i32* %Q) {
 
 }
 
+; Dead store on an aligned_alloc: should know that %M doesn't alias with %A.
+define i32 @test14a(i8* %M, i8 %value) {
+; CHECK-LABEL: @test14a(
+; CHECK-NOT: store
+; CHECK:     ret i32 0
+;
+  %A = tail call i8* @aligned_alloc(i32 32, i32 1024)
+  store i8 %value, i8* %A
+  tail call void @free(i8* %A)
+  ret i32 0
+}
 
 ; PR8701
 
diff --git a/llvm/test/Transforms/EarlyCSE/guards.ll b/llvm/test/Transforms/EarlyCSE/guards.ll
index de43264db6faf..3f162bd44b35e 100644
--- a/llvm/test/Transforms/EarlyCSE/guards.ll
+++ b/llvm/test/Transforms/EarlyCSE/guards.ll
@@ -1,5 +1,7 @@
-; RUN: opt -S -early-cse < %s | FileCheck %s
-; RUN: opt < %s -S -basicaa -early-cse-memssa | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -early-cse < %s | FileCheck %s --check-prefixes=CHECK,NO_ASSUME
+; RUN: opt < %s -S -basicaa -early-cse-memssa | FileCheck %s --check-prefixes=CHECK,NO_ASSUME
+; RUN: opt < %s -S -basicaa -early-cse-memssa --enable-knowledge-retention | FileCheck %s --check-prefixes=CHECK,USE_ASSUME
 
 declare void @llvm.experimental.guard(i1,...)
 
@@ -8,11 +10,17 @@ declare void @llvm.assume(i1)
 define i32 @test0(i32* %ptr, i1 %cond) {
 ; We can do store to load forwarding over a guard, since it does not
 ; clobber memory
-
-; CHECK-LABEL: @test0(
-; CHECK-NEXT:  store i32 40, i32* %ptr
-; CHECK-NEXT:  call void (i1, ...) @llvm.experimental.guard(i1 %cond) [ "deopt"() ]
-; CHECK-NEXT:  ret i32 40
+; NO_ASSUME-LABEL: @test0(
+; NO_ASSUME-NEXT:    store i32 40, i32* [[PTR:%.*]]
+; NO_ASSUME-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[COND:%.*]]) [ "deopt"() ]
+; NO_ASSUME-NEXT:    ret i32 40
+;
+; USE_ASSUME-LABEL: @test0(
+; USE_ASSUME-NEXT:    store i32 40, i32* [[PTR:%.*]]
+; USE_ASSUME-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[COND:%.*]]) [ "deopt"() ]
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    ret i32 40
+;
 
   store i32 40, i32* %ptr
   call void(i1,...) @llvm.experimental.guard(i1 %cond) [ "deopt"() ]
@@ -22,11 +30,17 @@ define i32 @test0(i32* %ptr, i1 %cond) {
 
 define i32 @test1(i32* %val, i1 %cond) {
 ; We can CSE loads over a guard, since it does not clobber memory
-
-; CHECK-LABEL: @test1(
-; CHECK-NEXT:  %val0 = load i32, i32* %val
-; CHECK-NEXT:  call void (i1, ...) @llvm.experimental.guard(i1 %cond) [ "deopt"() ]
-; CHECK-NEXT:  ret i32 0
+; NO_ASSUME-LABEL: @test1(
+; NO_ASSUME-NEXT:    [[VAL0:%.*]] = load i32, i32* [[VAL:%.*]]
+; NO_ASSUME-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[COND:%.*]]) [ "deopt"() ]
+; NO_ASSUME-NEXT:    ret i32 0
+;
+; USE_ASSUME-LABEL: @test1(
+; USE_ASSUME-NEXT:    [[VAL0:%.*]] = load i32, i32* [[VAL:%.*]]
+; USE_ASSUME-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[COND:%.*]]) [ "deopt"() ]
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[VAL]], i64 4), "nonnull"(i32* [[VAL]]) ]
+; USE_ASSUME-NEXT:    ret i32 0
+;
 
   %val0 = load i32, i32* %val
   call void(i1,...) @llvm.experimental.guard(i1 %cond) [ "deopt"() ]
@@ -37,9 +51,9 @@ define i32 @test1(i32* %val, i1 %cond) {
 
 define i32 @test2() {
 ; Guards on "true" get removed
-
 ; CHECK-LABEL: @test2(
-; CHECK-NEXT: ret i32 0
+; CHECK-NEXT:    ret i32 0
+;
   call void(i1, ...) @llvm.experimental.guard(i1 true) [ "deopt"() ]
   ret i32 0
 }
@@ -47,11 +61,11 @@ define i32 @test2() {
 define i32 @test3(i32 %val) {
 ; After a guard has executed the condition it was guarding is known to
 ; be true.
-
 ; CHECK-LABEL: @test3(
-; CHECK-NEXT:  %cond0 = icmp slt i32 %val, 40
-; CHECK-NEXT:  call void (i1, ...) @llvm.experimental.guard(i1 %cond0) [ "deopt"() ]
-; CHECK-NEXT:  ret i32 -1
+; CHECK-NEXT:    [[COND0:%.*]] = icmp slt i32 [[VAL:%.*]], 40
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[COND0]]) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 -1
+;
 
   %cond0 = icmp slt i32 %val, 40
   call void(i1,...) @llvm.experimental.guard(i1 %cond0) [ "deopt"() ]
@@ -85,23 +99,23 @@ define i32 @test3.unhandled(i32 %val) {
 
 define i32 @test4(i32 %val, i1 %c) {
 ; Same as test3, but with some control flow involved.
-
 ; CHECK-LABEL: @test4(
-; CHECK: entry:
-; CHECK-NEXT:  %cond0 = icmp slt i32 %val, 40
-; CHECK-NEXT:  call void (i1, ...) @llvm.experimental.guard(i1 %cond0
-; CHECK-NEXT:  br label %bb0
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND0:%.*]] = icmp slt i32 [[VAL:%.*]], 40
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[COND0]]) [ "deopt"() ]
+; CHECK-NEXT:    br label [[BB0:%.*]]
+; CHECK:       bb0:
+; CHECK-NEXT:    [[COND2:%.*]] = icmp ult i32 [[VAL]], 200
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[COND2]]) [ "deopt"() ]
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[LEFT:%.*]], label [[RIGHT:%.*]]
+; CHECK:       left:
+; CHECK-NEXT:    ret i32 0
+; CHECK:       right:
+; CHECK-NEXT:    ret i32 20
+;
 
-; CHECK:     bb0:
-; CHECK-NEXT:  %cond2 = icmp ult i32 %val, 200
-; CHECK-NEXT:  call void (i1, ...) @llvm.experimental.guard(i1 %cond2
-; CHECK-NEXT:  br i1 %c, label %left, label %right
 
-; CHECK:     left:
-; CHECK-NEXT:  ret i32 0
 
-; CHECK:     right:
-; CHECK-NEXT:  ret i32 20
 
 entry:
   %cond0 = icmp slt i32 %val, 40
@@ -121,29 +135,29 @@ left:
   ret i32 0
 
 right:
- ret i32 20
+  ret i32 20
 }
 
 define i32 @test5(i32 %val, i1 %c) {
 ; Same as test4, but the %left block has mutliple predecessors.
-
 ; CHECK-LABEL: @test5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND0:%.*]] = icmp slt i32 [[VAL:%.*]], 40
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[COND0]]) [ "deopt"() ]
+; CHECK-NEXT:    br label [[BB0:%.*]]
+; CHECK:       bb0:
+; CHECK-NEXT:    [[COND2:%.*]] = icmp ult i32 [[VAL]], 200
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[COND2]]) [ "deopt"() ]
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[LEFT:%.*]], label [[RIGHT:%.*]]
+; CHECK:       left:
+; CHECK-NEXT:    br label [[RIGHT]]
+; CHECK:       right:
+; CHECK-NEXT:    br label [[LEFT]]
+;
 
-; CHECK: entry:
-; CHECK-NEXT:  %cond0 = icmp slt i32 %val, 40
-; CHECK-NEXT:  call void (i1, ...) @llvm.experimental.guard(i1 %cond0
-; CHECK-NEXT:  br label %bb0
 
-; CHECK: bb0:
-; CHECK-NEXT:  %cond2 = icmp ult i32 %val, 200
-; CHECK-NEXT:  call void (i1, ...) @llvm.experimental.guard(i1 %cond2
-; CHECK-NEXT:  br i1 %c, label %left, label %right
 
-; CHECK: left:
-; CHECK-NEXT:  br label %right
 
-; CHECK: right:
-; CHECK-NEXT:  br label %left
 
 entry:
   %cond0 = icmp slt i32 %val, 40
@@ -170,11 +184,12 @@ define void @test6(i1 %c, i32* %ptr) {
 ; Check that we do not DSE over calls to @llvm.experimental.guard.
 ; Guard intrinsics do _read_ memory, so th call to guard below needs
 ; to see the store of 500 to %ptr
-
 ; CHECK-LABEL: @test6(
-; CHECK-NEXT:  store i32 500, i32* %ptr
-; CHECK-NEXT:  call void (i1, ...) @llvm.experimental.guard(i1 %c) [ "deopt"() ]
-; CHECK-NEXT:  store i32 600, i32* %ptr
+; CHECK-NEXT:    store i32 500, i32* [[PTR:%.*]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[C:%.*]]) [ "deopt"() ]
+; CHECK-NEXT:    store i32 600, i32* [[PTR]]
+; CHECK-NEXT:    ret void
+;
 
 
   store i32 500, i32* %ptr
@@ -186,11 +201,11 @@ define void @test6(i1 %c, i32* %ptr) {
 define void @test07(i32 %a, i32 %b) {
 ; Check that we are able to remove the guards on the same condition even if the
 ; condition is not being recalculated.
-
 ; CHECK-LABEL: @test07(
-; CHECK-NEXT:  %cmp = icmp eq i32 %a, %b
-; CHECK-NEXT:  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
-; CHECK-NEXT:  ret void
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[CMP]]) [ "deopt"() ]
+; CHECK-NEXT:    ret void
+;
 
   %cmp = icmp eq i32 %a, %b
   call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
@@ -202,13 +217,22 @@ define void @test07(i32 %a, i32 %b) {
 define void @test08(i32 %a, i32 %b, i32* %ptr) {
 ; Check that we deal correctly with stores when removing guards in the same
 ; block in case when the condition is not recalculated.
-
-; CHECK-LABEL: @test08(
-; CHECK-NEXT:  %cmp = icmp eq i32 %a, %b
-; CHECK-NEXT:  store i32 100, i32* %ptr
-; CHECK-NEXT:  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
-; CHECK-NEXT:  store i32 400, i32* %ptr
-; CHECK-NEXT:  ret void
+; NO_ASSUME-LABEL: @test08(
+; NO_ASSUME-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
+; NO_ASSUME-NEXT:    store i32 100, i32* [[PTR:%.*]]
+; NO_ASSUME-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[CMP]]) [ "deopt"() ]
+; NO_ASSUME-NEXT:    store i32 400, i32* [[PTR]]
+; NO_ASSUME-NEXT:    ret void
+;
+; USE_ASSUME-LABEL: @test08(
+; USE_ASSUME-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
+; USE_ASSUME-NEXT:    store i32 100, i32* [[PTR:%.*]]
+; USE_ASSUME-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[CMP]]) [ "deopt"() ]
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    store i32 400, i32* [[PTR]]
+; USE_ASSUME-NEXT:    ret void
+;
 
   %cmp = icmp eq i32 %a, %b
   store i32 100, i32* %ptr
@@ -225,22 +249,40 @@ define void @test09(i32 %a, i32 %b, i1 %c, i32* %ptr) {
 ; Similar to test08, but with more control flow.
 ; TODO: Can we get rid of the store in the end of entry given that it is
 ; post-dominated by other stores?
-
-; CHECK-LABEL: @test09(
-; CHECK:       entry:
-; CHECK-NEXT:    %cmp = icmp eq i32 %a, %b
-; CHECK-NEXT:    store i32 100, i32* %ptr
-; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
-; CHECK-NEXT:    store i32 400, i32* %ptr
-; CHECK-NEXT:    br i1 %c, label %if.true, label %if.false
-; CHECK:       if.true:
-; CHECK-NEXT:    store i32 500, i32* %ptr
-; CHECK-NEXT:    br label %merge
-; CHECK:       if.false:
-; CHECK-NEXT:    store i32 600, i32* %ptr
-; CHECK-NEXT:    br label %merge
-; CHECK:       merge:
-; CHECK-NEXT:    ret void
+; NO_ASSUME-LABEL: @test09(
+; NO_ASSUME-NEXT:  entry:
+; NO_ASSUME-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
+; NO_ASSUME-NEXT:    store i32 100, i32* [[PTR:%.*]]
+; NO_ASSUME-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[CMP]]) [ "deopt"() ]
+; NO_ASSUME-NEXT:    store i32 400, i32* [[PTR]]
+; NO_ASSUME-NEXT:    br i1 [[C:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; NO_ASSUME:       if.true:
+; NO_ASSUME-NEXT:    store i32 500, i32* [[PTR]]
+; NO_ASSUME-NEXT:    br label [[MERGE:%.*]]
+; NO_ASSUME:       if.false:
+; NO_ASSUME-NEXT:    store i32 600, i32* [[PTR]]
+; NO_ASSUME-NEXT:    br label [[MERGE]]
+; NO_ASSUME:       merge:
+; NO_ASSUME-NEXT:    ret void
+;
+; USE_ASSUME-LABEL: @test09(
+; USE_ASSUME-NEXT:  entry:
+; USE_ASSUME-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
+; USE_ASSUME-NEXT:    store i32 100, i32* [[PTR:%.*]]
+; USE_ASSUME-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[CMP]]) [ "deopt"() ]
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    store i32 400, i32* [[PTR]]
+; USE_ASSUME-NEXT:    br i1 [[C:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; USE_ASSUME:       if.true:
+; USE_ASSUME-NEXT:    store i32 500, i32* [[PTR]]
+; USE_ASSUME-NEXT:    br label [[MERGE:%.*]]
+; USE_ASSUME:       if.false:
+; USE_ASSUME-NEXT:    store i32 600, i32* [[PTR]]
+; USE_ASSUME-NEXT:    br label [[MERGE]]
+; USE_ASSUME:       merge:
+; USE_ASSUME-NEXT:    ret void
+;
 
 entry:
   %cmp = icmp eq i32 %a, %b
@@ -269,23 +311,23 @@ merge:
 
 define void @test10(i32 %a, i32 %b, i1 %c, i32* %ptr) {
 ; Make sure that non-dominating guards do not cause other guards removal.
-
 ; CHECK-LABEL: @test10(
-; CHECK:       entry:
-; CHECK-NEXT:    %cmp = icmp eq i32 %a, %b
-; CHECK-NEXT:    br i1 %c, label %if.true, label %if.false
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
 ; CHECK:       if.true:
-; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
-; CHECK-NEXT:    store i32 100, i32* %ptr
-; CHECK-NEXT:    br label %merge
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[CMP]]) [ "deopt"() ]
+; CHECK-NEXT:    store i32 100, i32* [[PTR:%.*]]
+; CHECK-NEXT:    br label [[MERGE:%.*]]
 ; CHECK:       if.false:
-; CHECK-NEXT:    store i32 200, i32* %ptr
-; CHECK-NEXT:    br label %merge
+; CHECK-NEXT:    store i32 200, i32* [[PTR]]
+; CHECK-NEXT:    br label [[MERGE]]
 ; CHECK:       merge:
-; CHECK-NEXT:    store i32 300, i32* %ptr
-; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
-; CHECK-NEXT:    store i32 400, i32* %ptr
+; CHECK-NEXT:    store i32 300, i32* [[PTR]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[CMP]]) [ "deopt"() ]
+; CHECK-NEXT:    store i32 400, i32* [[PTR]]
 ; CHECK-NEXT:    ret void
+;
 
 entry:
   %cmp = icmp eq i32 %a, %b
@@ -310,18 +352,18 @@ merge:
 
 define void @test11(i32 %a, i32 %b, i32* %ptr) {
 ; Make sure that branching condition is applied to guards.
-
 ; CHECK-LABEL: @test11(
-; CHECK:       entry:
-; CHECK-NEXT:    %cmp = icmp eq i32 %a, %b
-; CHECK-NEXT:    br i1 %cmp, label %if.true, label %if.false
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
 ; CHECK:       if.true:
-; CHECK-NEXT:    br label %merge
+; CHECK-NEXT:    br label [[MERGE:%.*]]
 ; CHECK:       if.false:
 ; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
-; CHECK-NEXT:    br label %merge
+; CHECK-NEXT:    br label [[MERGE]]
 ; CHECK:       merge:
 ; CHECK-NEXT:    ret void
+;
 
 entry:
   %cmp = icmp eq i32 %a, %b
@@ -342,11 +384,11 @@ merge:
 define void @test12(i32 %a, i32 %b) {
 ; Check that the assume marks its condition as being true (and thus allows to
 ; eliminate the dominated guards).
-
 ; CHECK-LABEL: @test12(
-; CHECK-NEXT:  %cmp = icmp eq i32 %a, %b
-; CHECK-NEXT:  call void @llvm.assume(i1 %cmp)
-; CHECK-NEXT:  ret void
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    ret void
+;
 
   %cmp = icmp eq i32 %a, %b
   call void @llvm.assume(i1 %cmp)
@@ -358,12 +400,21 @@ define void @test12(i32 %a, i32 %b) {
 
 define void @test13(i32 %a, i32 %b, i32* %ptr) {
 ; Check that we deal correctly with stores when removing guards due to assume.
-
-; CHECK-LABEL: @test13(
-; CHECK-NEXT:  %cmp = icmp eq i32 %a, %b
-; CHECK-NEXT:  call void @llvm.assume(i1 %cmp)
-; CHECK-NEXT:  store i32 400, i32* %ptr
-; CHECK-NEXT:  ret void
+; NO_ASSUME-LABEL: @test13(
+; NO_ASSUME-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
+; NO_ASSUME-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; NO_ASSUME-NEXT:    store i32 400, i32* [[PTR:%.*]]
+; NO_ASSUME-NEXT:    ret void
+;
+; USE_ASSUME-LABEL: @test13(
+; USE_ASSUME-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR:%.*]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    store i32 400, i32* [[PTR]]
+; USE_ASSUME-NEXT:    ret void
+;
 
   %cmp = icmp eq i32 %a, %b
   call void @llvm.assume(i1 %cmp)
@@ -381,21 +432,39 @@ define void @test14(i32 %a, i32 %b, i1 %c, i32* %ptr) {
 ; Similar to test13, but with more control flow.
 ; TODO: Can we get rid of the store in the end of entry given that it is
 ; post-dominated by other stores?
-
-; CHECK-LABEL: @test14(
-; CHECK:       entry:
-; CHECK-NEXT:    %cmp = icmp eq i32 %a, %b
-; CHECK-NEXT:    call void @llvm.assume(i1 %cmp)
-; CHECK-NEXT:    store i32 400, i32* %ptr
-; CHECK-NEXT:    br i1 %c, label %if.true, label %if.false
-; CHECK:       if.true:
-; CHECK-NEXT:    store i32 500, i32* %ptr
-; CHECK-NEXT:    br label %merge
-; CHECK:       if.false:
-; CHECK-NEXT:    store i32 600, i32* %ptr
-; CHECK-NEXT:    br label %merge
-; CHECK:       merge:
-; CHECK-NEXT:    ret void
+; NO_ASSUME-LABEL: @test14(
+; NO_ASSUME-NEXT:  entry:
+; NO_ASSUME-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
+; NO_ASSUME-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; NO_ASSUME-NEXT:    store i32 400, i32* [[PTR:%.*]]
+; NO_ASSUME-NEXT:    br i1 [[C:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; NO_ASSUME:       if.true:
+; NO_ASSUME-NEXT:    store i32 500, i32* [[PTR]]
+; NO_ASSUME-NEXT:    br label [[MERGE:%.*]]
+; NO_ASSUME:       if.false:
+; NO_ASSUME-NEXT:    store i32 600, i32* [[PTR]]
+; NO_ASSUME-NEXT:    br label [[MERGE]]
+; NO_ASSUME:       merge:
+; NO_ASSUME-NEXT:    ret void
+;
+; USE_ASSUME-LABEL: @test14(
+; USE_ASSUME-NEXT:  entry:
+; USE_ASSUME-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR:%.*]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    store i32 400, i32* [[PTR]]
+; USE_ASSUME-NEXT:    br i1 [[C:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; USE_ASSUME:       if.true:
+; USE_ASSUME-NEXT:    store i32 500, i32* [[PTR]]
+; USE_ASSUME-NEXT:    br label [[MERGE:%.*]]
+; USE_ASSUME:       if.false:
+; USE_ASSUME-NEXT:    store i32 600, i32* [[PTR]]
+; USE_ASSUME-NEXT:    br label [[MERGE]]
+; USE_ASSUME:       merge:
+; USE_ASSUME-NEXT:    ret void
+;
 
 entry:
   %cmp = icmp eq i32 %a, %b
@@ -425,23 +494,23 @@ merge:
 
 define void @test15(i32 %a, i32 %b, i1 %c, i32* %ptr) {
 ; Make sure that non-dominating assumes do not cause guards removal.
-
 ; CHECK-LABEL: @test15(
-; CHECK:       entry:
-; CHECK-NEXT:    %cmp = icmp eq i32 %a, %b
-; CHECK-NEXT:    br i1 %c, label %if.true, label %if.false
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
 ; CHECK:       if.true:
-; CHECK-NEXT:    call void @llvm.assume(i1 %cmp)
-; CHECK-NEXT:    store i32 100, i32* %ptr
-; CHECK-NEXT:    br label %merge
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    store i32 100, i32* [[PTR:%.*]]
+; CHECK-NEXT:    br label [[MERGE:%.*]]
 ; CHECK:       if.false:
-; CHECK-NEXT:    store i32 200, i32* %ptr
-; CHECK-NEXT:    br label %merge
+; CHECK-NEXT:    store i32 200, i32* [[PTR]]
+; CHECK-NEXT:    br label [[MERGE]]
 ; CHECK:       merge:
-; CHECK-NEXT:    store i32 300, i32* %ptr
-; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
-; CHECK-NEXT:    store i32 400, i32* %ptr
+; CHECK-NEXT:    store i32 300, i32* [[PTR]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[CMP]]) [ "deopt"() ]
+; CHECK-NEXT:    store i32 400, i32* [[PTR]]
 ; CHECK-NEXT:    ret void
+;
 
 entry:
   %cmp = icmp eq i32 %a, %b
@@ -467,12 +536,12 @@ merge:
 define void @test16(i32 %a, i32 %b) {
 ; Check that we don't bother to do anything with assumes even if we know the
 ; condition being true.
-
 ; CHECK-LABEL: @test16(
-; CHECK-NEXT:    %cmp = icmp eq i32 %a, %b
-; CHECK-NEXT:    call void @llvm.assume(i1 %cmp)
-; CHECK-NEXT:    call void @llvm.assume(i1 %cmp)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
 ; CHECK-NEXT:    ret void
+;
 
   %cmp = icmp eq i32 %a, %b
   call void @llvm.assume(i1 %cmp)
@@ -483,19 +552,19 @@ define void @test16(i32 %a, i32 %b) {
 define void @test17(i32 %a, i32 %b, i1 %c, i32* %ptr) {
 ; Check that we don't bother to do anything with assumes even if we know the
 ; condition being true or false (includes come control flow).
-
 ; CHECK-LABEL: @test17(
-; CHECK:       entry:
-; CHECK-NEXT:    %cmp = icmp eq i32 %a, %b
-; CHECK-NEXT:    br i1 %c, label %if.true, label %if.false
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
 ; CHECK:       if.true:
-; CHECK-NEXT:    call void @llvm.assume(i1 %cmp)
-; CHECK-NEXT:    br label %merge
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    br label [[MERGE:%.*]]
 ; CHECK:       if.false:
-; CHECK-NEXT:    call void @llvm.assume(i1 %cmp)
-; CHECK-NEXT:    br label %merge
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    br label [[MERGE]]
 ; CHECK:       merge:
 ; CHECK-NEXT:    ret void
+;
 
 entry:
   %cmp = icmp eq i32 %a, %b
@@ -516,11 +585,11 @@ merge:
 define void @test18(i1 %c) {
 ; Check that we don't bother to do anything with assumes even if we know the
 ; condition being true and not being an instruction.
-
 ; CHECK-LABEL: @test18(
-; CHECK-NEXT:    call void @llvm.assume(i1 %c)
-; CHECK-NEXT:    call void @llvm.assume(i1 %c)
+; CHECK-NEXT:    call void @llvm.assume(i1 [[C:%.*]])
+; CHECK-NEXT:    call void @llvm.assume(i1 [[C]])
 ; CHECK-NEXT:    ret void
+;
 
   call void @llvm.assume(i1 %c)
   call void @llvm.assume(i1 %c)
diff --git a/llvm/test/Transforms/EarlyCSE/invariant-loads.ll b/llvm/test/Transforms/EarlyCSE/invariant-loads.ll
index c3fa32d6faddd..d20521afa8a66 100644
--- a/llvm/test/Transforms/EarlyCSE/invariant-loads.ll
+++ b/llvm/test/Transforms/EarlyCSE/invariant-loads.ll
@@ -1,15 +1,27 @@
-; RUN: opt -S -early-cse < %s | FileCheck %s
-; RUN: opt -S -basicaa -early-cse-memssa < %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -early-cse < %s | FileCheck %s --check-prefixes=CHECK,NO_ASSUME
+; RUN: opt -S -basicaa -early-cse-memssa < %s | FileCheck %s --check-prefixes=CHECK,NO_ASSUME
+; RUN: opt -S -basicaa -early-cse-memssa --enable-knowledge-retention < %s | FileCheck %s --check-prefixes=CHECK,USE_ASSUME
 
 declare void @clobber_and_use(i32)
 
 define void @f_0(i32* %ptr) {
-; CHECK-LABEL: @f_0(
-; CHECK:   %val0 = load i32, i32* %ptr, !invariant.load !0
-; CHECK:   call void @clobber_and_use(i32 %val0)
-; CHECK:   call void @clobber_and_use(i32 %val0)
-; CHECK:   call void @clobber_and_use(i32 %val0)
-; CHECK:   ret void
+; NO_ASSUME-LABEL: @f_0(
+; NO_ASSUME-NEXT:    [[VAL0:%.*]] = load i32, i32* [[PTR:%.*]], !invariant.load !0
+; NO_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; NO_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; NO_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; NO_ASSUME-NEXT:    ret void
+;
+; USE_ASSUME-LABEL: @f_0(
+; USE_ASSUME-NEXT:    [[VAL0:%.*]] = load i32, i32* [[PTR:%.*]], !invariant.load !0
+; USE_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; USE_ASSUME-NEXT:    ret void
+;
 
   %val0 = load i32, i32* %ptr, !invariant.load !{}
   call void @clobber_and_use(i32 %val0)
@@ -22,11 +34,19 @@ define void @f_0(i32* %ptr) {
 
 define void @f_1(i32* %ptr) {
 ; We can forward invariant loads to non-invariant loads.
-
-; CHECK-LABEL: @f_1(
-; CHECK:   %val0 = load i32, i32* %ptr, !invariant.load !0
-; CHECK:   call void @clobber_and_use(i32 %val0)
-; CHECK:   call void @clobber_and_use(i32 %val0)
+; NO_ASSUME-LABEL: @f_1(
+; NO_ASSUME-NEXT:    [[VAL0:%.*]] = load i32, i32* [[PTR:%.*]], !invariant.load !0
+; NO_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; NO_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; NO_ASSUME-NEXT:    ret void
+;
+; USE_ASSUME-LABEL: @f_1(
+; USE_ASSUME-NEXT:    [[VAL0:%.*]] = load i32, i32* [[PTR:%.*]], !invariant.load !0
+; USE_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; USE_ASSUME-NEXT:    ret void
+;
 
   %val0 = load i32, i32* %ptr, !invariant.load !{}
   call void @clobber_and_use(i32 %val0)
@@ -37,11 +57,19 @@ define void @f_1(i32* %ptr) {
 
 define void @f_2(i32* %ptr) {
 ; We can forward a non-invariant load into an invariant load.
-
-; CHECK-LABEL: @f_2(
-; CHECK:   %val0 = load i32, i32* %ptr
-; CHECK:   call void @clobber_and_use(i32 %val0)
-; CHECK:   call void @clobber_and_use(i32 %val0)
+; NO_ASSUME-LABEL: @f_2(
+; NO_ASSUME-NEXT:    [[VAL0:%.*]] = load i32, i32* [[PTR:%.*]]
+; NO_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; NO_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; NO_ASSUME-NEXT:    ret void
+;
+; USE_ASSUME-LABEL: @f_2(
+; USE_ASSUME-NEXT:    [[VAL0:%.*]] = load i32, i32* [[PTR:%.*]]
+; USE_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; USE_ASSUME-NEXT:    ret void
+;
 
   %val0 = load i32, i32* %ptr
   call void @clobber_and_use(i32 %val0)
@@ -51,14 +79,31 @@ define void @f_2(i32* %ptr) {
 }
 
 define void @f_3(i1 %cond, i32* %ptr) {
-; CHECK-LABEL: @f_3(
+; NO_ASSUME-LABEL: @f_3(
+; NO_ASSUME-NEXT:    [[VAL0:%.*]] = load i32, i32* [[PTR:%.*]], !invariant.load !0
+; NO_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; NO_ASSUME-NEXT:    br i1 [[COND:%.*]], label [[LEFT:%.*]], label [[RIGHT:%.*]]
+; NO_ASSUME:       left:
+; NO_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; NO_ASSUME-NEXT:    ret void
+; NO_ASSUME:       right:
+; NO_ASSUME-NEXT:    ret void
+;
+; USE_ASSUME-LABEL: @f_3(
+; USE_ASSUME-NEXT:    [[VAL0:%.*]] = load i32, i32* [[PTR:%.*]], !invariant.load !0
+; USE_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; USE_ASSUME-NEXT:    br i1 [[COND:%.*]], label [[LEFT:%.*]], label [[RIGHT:%.*]]
+; USE_ASSUME:       left:
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[PTR]], i64 4), "nonnull"(i32* [[PTR]]) ]
+; USE_ASSUME-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; USE_ASSUME-NEXT:    ret void
+; USE_ASSUME:       right:
+; USE_ASSUME-NEXT:    ret void
+;
   %val0 = load i32, i32* %ptr, !invariant.load !{}
   call void @clobber_and_use(i32 %val0)
   br i1 %cond, label %left, label %right
 
-; CHECK:  %val0 = load i32, i32* %ptr, !invariant.load !0
-; CHECK: left:
-; CHECK-NEXT:  call void @clobber_and_use(i32 %val0)
 
 left:
   %val1 = load i32, i32* %ptr
@@ -72,23 +117,26 @@ right:
 define void @f_4(i1 %cond, i32* %ptr) {
 ; Negative test -- can't forward %val0 to %va1 because that'll break
 ; def-dominates-use.
-
 ; CHECK-LABEL: @f_4(
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[LEFT:%.*]], label [[MERGE:%.*]]
+; CHECK:       left:
+; CHECK-NEXT:    [[VAL0:%.*]] = load i32, i32* [[PTR:%.*]], !invariant.load !0
+; CHECK-NEXT:    call void @clobber_and_use(i32 [[VAL0]])
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[VAL1:%.*]] = load i32, i32* [[PTR]]
+; CHECK-NEXT:    call void @clobber_and_use(i32 [[VAL1]])
+; CHECK-NEXT:    ret void
+;
   br i1 %cond, label %left, label %merge
 
 left:
-; CHECK: left:
-; CHECK-NEXT:  %val0 = load i32, i32* %ptr, !invariant.load !
-; CHECK-NEXT:  call void @clobber_and_use(i32 %val0)
 
   %val0 = load i32, i32* %ptr, !invariant.load !{}
   call void @clobber_and_use(i32 %val0)
   br label %merge
 
 merge:
-; CHECK: merge:
-; CHECK-NEXT:   %val1 = load i32, i32* %ptr
-; CHECK-NEXT:   call void @clobber_and_use(i32 %val1)
 
   %val1 = load i32, i32* %ptr
   call void @clobber_and_use(i32 %val1)
@@ -100,8 +148,17 @@ merge:
 ; for the moment we chose to be conservative and just assume it's valid
 ; to restore the same unchanging value.
 define void @test_dse1(i32* %p) {
-; CHECK-LABEL: @test_dse1
-; CHECK-NOT: store
+; NO_ASSUME-LABEL: @test_dse1(
+; NO_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P:%.*]], !invariant.load !0
+; NO_ASSUME-NEXT:    call void @clobber_and_use(i32 [[V1]])
+; NO_ASSUME-NEXT:    ret void
+;
+; USE_ASSUME-LABEL: @test_dse1(
+; USE_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P:%.*]], !invariant.load !0
+; USE_ASSUME-NEXT:    call void @clobber_and_use(i32 [[V1]])
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]) ]
+; USE_ASSUME-NEXT:    ret void
+;
   %v1 = load i32, i32* %p, !invariant.load !{}
   call void @clobber_and_use(i32 %v1)
   store i32 %v1, i32* %p
@@ -110,8 +167,12 @@ define void @test_dse1(i32* %p) {
 
 ; By assumption, v1 must equal v2 (TODO)
 define void @test_false_negative_dse2(i32* %p, i32 %v2) {
-; CHECK-LABEL: @test_false_negative_dse2
-; CHECK: store
+; CHECK-LABEL: @test_false_negative_dse2(
+; CHECK-NEXT:    [[V1:%.*]] = load i32, i32* [[P:%.*]], !invariant.load !0
+; CHECK-NEXT:    call void @clobber_and_use(i32 [[V1]])
+; CHECK-NEXT:    store i32 [[V2:%.*]], i32* [[P]]
+; CHECK-NEXT:    ret void
+;
   %v1 = load i32, i32* %p, !invariant.load !{}
   call void @clobber_and_use(i32 %v1)
   store i32 %v2, i32* %p
@@ -121,12 +182,22 @@ define void @test_false_negative_dse2(i32* %p, i32 %v2) {
 ; If we remove the load, we still start an invariant scope since
 ; it lets us remove later loads not explicitly marked invariant
 define void @test_scope_start_without_load(i32* %p) {
-; CHECK-LABEL: @test_scope_start_without_load
-; CHECK:   %v1 = load i32, i32* %p
-; CHECK:   %add = add i32 %v1, %v1
-; CHECK:   call void @clobber_and_use(i32 %add)
-; CHECK:   call void @clobber_and_use(i32 %v1)
-; CHECK:   ret void
+; NO_ASSUME-LABEL: @test_scope_start_without_load(
+; NO_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P:%.*]]
+; NO_ASSUME-NEXT:    [[ADD:%.*]] = add i32 [[V1]], [[V1]]
+; NO_ASSUME-NEXT:    call void @clobber_and_use(i32 [[ADD]])
+; NO_ASSUME-NEXT:    call void @clobber_and_use(i32 [[V1]])
+; NO_ASSUME-NEXT:    ret void
+;
+; USE_ASSUME-LABEL: @test_scope_start_without_load(
+; USE_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P:%.*]]
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]) ]
+; USE_ASSUME-NEXT:    [[ADD:%.*]] = add i32 [[V1]], [[V1]]
+; USE_ASSUME-NEXT:    call void @clobber_and_use(i32 [[ADD]])
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]) ]
+; USE_ASSUME-NEXT:    call void @clobber_and_use(i32 [[V1]])
+; USE_ASSUME-NEXT:    ret void
+;
   %v1 = load i32, i32* %p
   %v2 = load i32, i32* %p, !invariant.load !{}
   %add = add i32 %v1, %v2
@@ -140,13 +211,24 @@ define void @test_scope_start_without_load(i32* %p) {
 ; with a potentially greater generation.  This hides the earlier invariant
 ; load
 define void @test_scope_restart(i32* %p) {
-; CHECK-LABEL: @test_scope_restart
-; CHECK:   %v1 = load i32, i32* %p
-; CHECK:   call void @clobber_and_use(i32 %v1)
-; CHECK:   %add = add i32 %v1, %v1
-; CHECK:   call void @clobber_and_use(i32 %add)
-; CHECK:   call void @clobber_and_use(i32 %v1)
-; CHECK:   ret void
+; NO_ASSUME-LABEL: @test_scope_restart(
+; NO_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P:%.*]], !invariant.load !0
+; NO_ASSUME-NEXT:    call void @clobber_and_use(i32 [[V1]])
+; NO_ASSUME-NEXT:    [[ADD:%.*]] = add i32 [[V1]], [[V1]]
+; NO_ASSUME-NEXT:    call void @clobber_and_use(i32 [[ADD]])
+; NO_ASSUME-NEXT:    call void @clobber_and_use(i32 [[V1]])
+; NO_ASSUME-NEXT:    ret void
+;
+; USE_ASSUME-LABEL: @test_scope_restart(
+; USE_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P:%.*]], !invariant.load !0
+; USE_ASSUME-NEXT:    call void @clobber_and_use(i32 [[V1]])
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]) ]
+; USE_ASSUME-NEXT:    [[ADD:%.*]] = add i32 [[V1]], [[V1]]
+; USE_ASSUME-NEXT:    call void @clobber_and_use(i32 [[ADD]])
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]) ]
+; USE_ASSUME-NEXT:    call void @clobber_and_use(i32 [[V1]])
+; USE_ASSUME-NEXT:    ret void
+;
   %v1 = load i32, i32* %p, !invariant.load !{}
   call void @clobber_and_use(i32 %v1)
   %v2 = load i32, i32* %p, !invariant.load !{}
diff --git a/llvm/test/Transforms/EarlyCSE/invariant.start.ll b/llvm/test/Transforms/EarlyCSE/invariant.start.ll
index b5dc9a6bff73e..8b173cf5ecfd1 100644
--- a/llvm/test/Transforms/EarlyCSE/invariant.start.ll
+++ b/llvm/test/Transforms/EarlyCSE/invariant.start.ll
@@ -1,5 +1,7 @@
-; RUN: opt < %s -S -early-cse | FileCheck %s
-; RUN: opt < %s -S -passes=early-cse | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature
+; RUN: opt < %s -S -early-cse | FileCheck %s --check-prefixes=CHECK,NO_ASSUME
+; RUN: opt < %s -S -early-cse --enable-knowledge-retention | FileCheck %s --check-prefixes=CHECK,USE_ASSUME
+; RUN: opt < %s -S -passes=early-cse | FileCheck %s --check-prefixes=CHECK,NO_ASSUME
 
 declare {}* @llvm.invariant.start.p0i8(i64, i8* nocapture) nounwind readonly
 declare void @llvm.invariant.end.p0i8({}*, i64, i8* nocapture) nounwind
@@ -7,10 +9,19 @@ declare void @llvm.invariant.end.p0i8({}*, i64, i8* nocapture) nounwind
 ; Check that we do load-load forwarding over invariant.start, since it does not
 ; clobber memory
 define i8 @test_bypass1(i8 *%P) {
-  ; CHECK-LABEL: @test_bypass1(
-  ; CHECK-NEXT: %V1 = load i8, i8* %P
-  ; CHECK-NEXT: %i = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %P)
-  ; CHECK-NEXT: ret i8 0
+; NO_ASSUME-LABEL: define {{[^@]+}}@test_bypass1
+; NO_ASSUME-SAME: (i8* [[P:%.*]])
+; NO_ASSUME-NEXT:    [[V1:%.*]] = load i8, i8* [[P]]
+; NO_ASSUME-NEXT:    [[I:%.*]] = call {}* @llvm.invariant.start.p0i8(i64 1, i8* [[P]])
+; NO_ASSUME-NEXT:    ret i8 0
+;
+; USE_ASSUME-LABEL: define {{[^@]+}}@test_bypass1
+; USE_ASSUME-SAME: (i8* [[P:%.*]])
+; USE_ASSUME-NEXT:    [[V1:%.*]] = load i8, i8* [[P]]
+; USE_ASSUME-NEXT:    [[I:%.*]] = call {}* @llvm.invariant.start.p0i8(i64 1, i8* [[P]])
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i8* [[P]], i64 1), "nonnull"(i8* [[P]]) ]
+; USE_ASSUME-NEXT:    ret i8 0
+;
 
   %V1 = load i8, i8* %P
   %i = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %P)
@@ -22,10 +33,19 @@ define i8 @test_bypass1(i8 *%P) {
 
 ; Trivial Store->load forwarding over invariant.start
 define i8 @test_bypass2(i8 *%P) {
-  ; CHECK-LABEL: @test_bypass2(
-  ; CHECK-NEXT: store i8 42, i8* %P
-  ; CHECK-NEXT: %i = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %P)
-  ; CHECK-NEXT: ret i8 42
+; NO_ASSUME-LABEL: define {{[^@]+}}@test_bypass2
+; NO_ASSUME-SAME: (i8* [[P:%.*]])
+; NO_ASSUME-NEXT:    store i8 42, i8* [[P]]
+; NO_ASSUME-NEXT:    [[I:%.*]] = call {}* @llvm.invariant.start.p0i8(i64 1, i8* [[P]])
+; NO_ASSUME-NEXT:    ret i8 42
+;
+; USE_ASSUME-LABEL: define {{[^@]+}}@test_bypass2
+; USE_ASSUME-SAME: (i8* [[P:%.*]])
+; USE_ASSUME-NEXT:    store i8 42, i8* [[P]]
+; USE_ASSUME-NEXT:    [[I:%.*]] = call {}* @llvm.invariant.start.p0i8(i64 1, i8* [[P]])
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i8* [[P]], i64 1), "nonnull"(i8* [[P]]) ]
+; USE_ASSUME-NEXT:    ret i8 42
+;
 
   store i8 42, i8* %P
   %i = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %P)
@@ -37,9 +57,19 @@ define i8 @test_bypass2(i8 *%P) {
 ; %P is valid, and the second store is actually unreachable based on semantics
 ; of invariant.start.
 define void @test_bypass3(i8* %P) {
-; CHECK-LABEL: @test_bypass3(
-; CHECK-NEXT:  %i = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %P)
-; CHECK-NEXT:  store i8 60, i8* %P
+; NO_ASSUME-LABEL: define {{[^@]+}}@test_bypass3
+; NO_ASSUME-SAME: (i8* [[P:%.*]])
+; NO_ASSUME-NEXT:    [[I:%.*]] = call {}* @llvm.invariant.start.p0i8(i64 1, i8* [[P]])
+; NO_ASSUME-NEXT:    store i8 60, i8* [[P]]
+; NO_ASSUME-NEXT:    ret void
+;
+; USE_ASSUME-LABEL: define {{[^@]+}}@test_bypass3
+; USE_ASSUME-SAME: (i8* [[P:%.*]])
+; USE_ASSUME-NEXT:    [[I:%.*]] = call {}* @llvm.invariant.start.p0i8(i64 1, i8* [[P]])
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i8* [[P]], i64 1), "nonnull"(i8* [[P]]) ]
+; USE_ASSUME-NEXT:    store i8 60, i8* [[P]]
+; USE_ASSUME-NEXT:    ret void
+;
 
   store i8 50, i8* %P
   %i = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %P)
@@ -51,12 +81,14 @@ define void @test_bypass3(i8* %P) {
 ; FIXME: Now the first store can actually be eliminated, since there is no read within
 ; the invariant region, between start and end.
 define void @test_bypass4(i8* %P) {
-
-; CHECK-LABEL: @test_bypass4(
-; CHECK-NEXT: store i8 50, i8* %P
-; CHECK-NEXT:  %i = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %P)
-; CHECK-NEXT: call void @llvm.invariant.end.p0i8({}* %i, i64 1, i8* %P)
-; CHECK-NEXT:  store i8 60, i8* %P
+; CHECK-LABEL: define {{[^@]+}}@test_bypass4
+; CHECK-SAME: (i8* [[P:%.*]])
+; CHECK-NEXT:    store i8 50, i8* [[P]]
+; CHECK-NEXT:    [[I:%.*]] = call {}* @llvm.invariant.start.p0i8(i64 1, i8* [[P]])
+; CHECK-NEXT:    call void @llvm.invariant.end.p0i8({}* [[I]], i64 1, i8* [[P]])
+; CHECK-NEXT:    store i8 60, i8* [[P]]
+; CHECK-NEXT:    ret void
+;
 
 
   store i8 50, i8* %P
@@ -72,8 +104,21 @@ declare {}* @llvm.invariant.start.p0i32(i64 %size, i32* nocapture %ptr)
 declare void @llvm.invariant.end.p0i32({}*, i64, i32* nocapture) nounwind
 
 define i32 @test_before_load(i32* %p) {
-; CHECK-LABEL: @test_before_load
-; CHECK: ret i32 0
+; NO_ASSUME-LABEL: define {{[^@]+}}@test_before_load
+; NO_ASSUME-SAME: (i32* [[P:%.*]])
+; NO_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; NO_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; NO_ASSUME-NEXT:    call void @clobber()
+; NO_ASSUME-NEXT:    ret i32 0
+;
+; USE_ASSUME-LABEL: define {{[^@]+}}@test_before_load
+; USE_ASSUME-SAME: (i32* [[P:%.*]])
+; USE_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; USE_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; USE_ASSUME-NEXT:    call void @clobber()
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]) ]
+; USE_ASSUME-NEXT:    ret i32 0
+;
   call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
   %v1 = load i32, i32* %p
   call void @clobber()
@@ -83,8 +128,21 @@ define i32 @test_before_load(i32* %p) {
 }
 
 define i32 @test_before_clobber(i32* %p) {
-; CHECK-LABEL: @test_before_clobber
-; CHECK: ret i32 0
+; NO_ASSUME-LABEL: define {{[^@]+}}@test_before_clobber
+; NO_ASSUME-SAME: (i32* [[P:%.*]])
+; NO_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; NO_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; NO_ASSUME-NEXT:    call void @clobber()
+; NO_ASSUME-NEXT:    ret i32 0
+;
+; USE_ASSUME-LABEL: define {{[^@]+}}@test_before_clobber
+; USE_ASSUME-SAME: (i32* [[P:%.*]])
+; USE_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; USE_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; USE_ASSUME-NEXT:    call void @clobber()
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]) ]
+; USE_ASSUME-NEXT:    ret i32 0
+;
   %v1 = load i32, i32* %p
   call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
   call void @clobber()
@@ -94,8 +152,23 @@ define i32 @test_before_clobber(i32* %p) {
 }
 
 define i32 @test_duplicate_scope(i32* %p) {
-; CHECK-LABEL: @test_duplicate_scope
-; CHECK: ret i32 0
+; NO_ASSUME-LABEL: define {{[^@]+}}@test_duplicate_scope
+; NO_ASSUME-SAME: (i32* [[P:%.*]])
+; NO_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; NO_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; NO_ASSUME-NEXT:    call void @clobber()
+; NO_ASSUME-NEXT:    [[TMP2:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; NO_ASSUME-NEXT:    ret i32 0
+;
+; USE_ASSUME-LABEL: define {{[^@]+}}@test_duplicate_scope
+; USE_ASSUME-SAME: (i32* [[P:%.*]])
+; USE_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; USE_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; USE_ASSUME-NEXT:    call void @clobber()
+; USE_ASSUME-NEXT:    [[TMP2:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]) ]
+; USE_ASSUME-NEXT:    ret i32 0
+;
   %v1 = load i32, i32* %p
   call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
   call void @clobber()
@@ -106,8 +179,23 @@ define i32 @test_duplicate_scope(i32* %p) {
 }
 
 define i32 @test_unanalzyable_load(i32* %p) {
-; CHECK-LABEL: @test_unanalzyable_load
-; CHECK: ret i32 0
+; NO_ASSUME-LABEL: define {{[^@]+}}@test_unanalzyable_load
+; NO_ASSUME-SAME: (i32* [[P:%.*]])
+; NO_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; NO_ASSUME-NEXT:    call void @clobber()
+; NO_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; NO_ASSUME-NEXT:    call void @clobber()
+; NO_ASSUME-NEXT:    ret i32 0
+;
+; USE_ASSUME-LABEL: define {{[^@]+}}@test_unanalzyable_load
+; USE_ASSUME-SAME: (i32* [[P:%.*]])
+; USE_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; USE_ASSUME-NEXT:    call void @clobber()
+; USE_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; USE_ASSUME-NEXT:    call void @clobber()
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]) ]
+; USE_ASSUME-NEXT:    ret i32 0
+;
   call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
   call void @clobber()
   %v1 = load i32, i32* %p
@@ -118,8 +206,15 @@ define i32 @test_unanalzyable_load(i32* %p) {
 }
 
 define i32 @test_negative_after_clobber(i32* %p) {
-; CHECK-LABEL: @test_negative_after_clobber
-; CHECK: ret i32 %sub
+; CHECK-LABEL: define {{[^@]+}}@test_negative_after_clobber
+; CHECK-SAME: (i32* [[P:%.*]])
+; CHECK-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; CHECK-NEXT:    call void @clobber()
+; CHECK-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; CHECK-NEXT:    [[V2:%.*]] = load i32, i32* [[P]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[V1]], [[V2]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
   %v1 = load i32, i32* %p
   call void @clobber()
   call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
@@ -129,8 +224,29 @@ define i32 @test_negative_after_clobber(i32* %p) {
 }
 
 define i32 @test_merge(i32* %p, i1 %cnd) {
-; CHECK-LABEL: @test_merge
-; CHECK: ret i32 0
+; NO_ASSUME-LABEL: define {{[^@]+}}@test_merge
+; NO_ASSUME-SAME: (i32* [[P:%.*]], i1 [[CND:%.*]])
+; NO_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; NO_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; NO_ASSUME-NEXT:    br i1 [[CND]], label [[MERGE:%.*]], label [[TAKEN:%.*]]
+; NO_ASSUME:       taken:
+; NO_ASSUME-NEXT:    call void @clobber()
+; NO_ASSUME-NEXT:    br label [[MERGE]]
+; NO_ASSUME:       merge:
+; NO_ASSUME-NEXT:    ret i32 0
+;
+; USE_ASSUME-LABEL: define {{[^@]+}}@test_merge
+; USE_ASSUME-SAME: (i32* [[P:%.*]], i1 [[CND:%.*]])
+; USE_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; USE_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; USE_ASSUME-NEXT:    br i1 [[CND]], label [[MERGE:%.*]], label [[TAKEN:%.*]]
+; USE_ASSUME:       taken:
+; USE_ASSUME-NEXT:    call void @clobber()
+; USE_ASSUME-NEXT:    br label [[MERGE]]
+; USE_ASSUME:       merge:
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]) ]
+; USE_ASSUME-NEXT:    ret i32 0
+;
   %v1 = load i32, i32* %p
   call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
   br i1 %cnd, label %merge, label %taken
@@ -145,8 +261,19 @@ merge:
 }
 
 define i32 @test_negative_after_mergeclobber(i32* %p, i1 %cnd) {
-; CHECK-LABEL: @test_negative_after_mergeclobber
-; CHECK: ret i32 %sub
+; CHECK-LABEL: define {{[^@]+}}@test_negative_after_mergeclobber
+; CHECK-SAME: (i32* [[P:%.*]], i1 [[CND:%.*]])
+; CHECK-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; CHECK-NEXT:    br i1 [[CND]], label [[MERGE:%.*]], label [[TAKEN:%.*]]
+; CHECK:       taken:
+; CHECK-NEXT:    call void @clobber()
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; CHECK-NEXT:    [[V2:%.*]] = load i32, i32* [[P]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[V1]], [[V2]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
   %v1 = load i32, i32* %p
   br i1 %cnd, label %merge, label %taken
 
@@ -161,10 +288,21 @@ merge:
 }
 
 ; In theory, this version could work, but earlycse is incapable of
-; merging facts along distinct paths.  
+; merging facts along distinct paths.
 define i32 @test_false_negative_merge(i32* %p, i1 %cnd) {
-; CHECK-LABEL: @test_false_negative_merge
-; CHECK: ret i32 %sub
+; CHECK-LABEL: define {{[^@]+}}@test_false_negative_merge
+; CHECK-SAME: (i32* [[P:%.*]], i1 [[CND:%.*]])
+; CHECK-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; CHECK-NEXT:    br i1 [[CND]], label [[MERGE:%.*]], label [[TAKEN:%.*]]
+; CHECK:       taken:
+; CHECK-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; CHECK-NEXT:    call void @clobber()
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[V2:%.*]] = load i32, i32* [[P]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[V1]], [[V2]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
   %v1 = load i32, i32* %p
   br i1 %cnd, label %merge, label %taken
 
@@ -179,8 +317,31 @@ merge:
 }
 
 define i32 @test_merge_unanalyzable_load(i32* %p, i1 %cnd) {
-; CHECK-LABEL: @test_merge_unanalyzable_load
-; CHECK: ret i32 0
+; NO_ASSUME-LABEL: define {{[^@]+}}@test_merge_unanalyzable_load
+; NO_ASSUME-SAME: (i32* [[P:%.*]], i1 [[CND:%.*]])
+; NO_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; NO_ASSUME-NEXT:    call void @clobber()
+; NO_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; NO_ASSUME-NEXT:    br i1 [[CND]], label [[MERGE:%.*]], label [[TAKEN:%.*]]
+; NO_ASSUME:       taken:
+; NO_ASSUME-NEXT:    call void @clobber()
+; NO_ASSUME-NEXT:    br label [[MERGE]]
+; NO_ASSUME:       merge:
+; NO_ASSUME-NEXT:    ret i32 0
+;
+; USE_ASSUME-LABEL: define {{[^@]+}}@test_merge_unanalyzable_load
+; USE_ASSUME-SAME: (i32* [[P:%.*]], i1 [[CND:%.*]])
+; USE_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; USE_ASSUME-NEXT:    call void @clobber()
+; USE_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; USE_ASSUME-NEXT:    br i1 [[CND]], label [[MERGE:%.*]], label [[TAKEN:%.*]]
+; USE_ASSUME:       taken:
+; USE_ASSUME-NEXT:    call void @clobber()
+; USE_ASSUME-NEXT:    br label [[MERGE]]
+; USE_ASSUME:       merge:
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]) ]
+; USE_ASSUME-NEXT:    ret i32 0
+;
   call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
   call void @clobber()
   %v1 = load i32, i32* %p
@@ -196,8 +357,21 @@ merge:
 }
 
 define void @test_dse_before_load(i32* %p, i1 %cnd) {
-; CHECK-LABEL: @test_dse_before_load
-; CHECK-NOT: store
+; NO_ASSUME-LABEL: define {{[^@]+}}@test_dse_before_load
+; NO_ASSUME-SAME: (i32* [[P:%.*]], i1 [[CND:%.*]])
+; NO_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; NO_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; NO_ASSUME-NEXT:    call void @clobber()
+; NO_ASSUME-NEXT:    ret void
+;
+; USE_ASSUME-LABEL: define {{[^@]+}}@test_dse_before_load
+; USE_ASSUME-SAME: (i32* [[P:%.*]], i1 [[CND:%.*]])
+; USE_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; USE_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; USE_ASSUME-NEXT:    call void @clobber()
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]) ]
+; USE_ASSUME-NEXT:    ret void
+;
   call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
   %v1 = load i32, i32* %p
   call void @clobber()
@@ -206,8 +380,21 @@ define void @test_dse_before_load(i32* %p, i1 %cnd) {
 }
 
 define void @test_dse_after_load(i32* %p, i1 %cnd) {
-; CHECK-LABEL: @test_dse_after_load
-; CHECK-NOT: store
+; NO_ASSUME-LABEL: define {{[^@]+}}@test_dse_after_load
+; NO_ASSUME-SAME: (i32* [[P:%.*]], i1 [[CND:%.*]])
+; NO_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; NO_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; NO_ASSUME-NEXT:    call void @clobber()
+; NO_ASSUME-NEXT:    ret void
+;
+; USE_ASSUME-LABEL: define {{[^@]+}}@test_dse_after_load
+; USE_ASSUME-SAME: (i32* [[P:%.*]], i1 [[CND:%.*]])
+; USE_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; USE_ASSUME-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; USE_ASSUME-NEXT:    call void @clobber()
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]) ]
+; USE_ASSUME-NEXT:    ret void
+;
   %v1 = load i32, i32* %p
   call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
   call void @clobber()
@@ -220,8 +407,17 @@ define void @test_dse_after_load(i32* %p, i1 %cnd) {
 ; typed due to the user of a Value to represent the address.  Note that other
 ; passes will canonicalize away the bitcasts in this example.
 define i32 @test_false_negative_types(i32* %p) {
-; CHECK-LABEL: @test_false_negative_types
-; CHECK: ret i32 %sub
+; CHECK-LABEL: define {{[^@]+}}@test_false_negative_types
+; CHECK-SAME: (i32* [[P:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; CHECK-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; CHECK-NEXT:    call void @clobber()
+; CHECK-NEXT:    [[PF:%.*]] = bitcast i32* [[P]] to float*
+; CHECK-NEXT:    [[V2F:%.*]] = load float, float* [[PF]]
+; CHECK-NEXT:    [[V2:%.*]] = bitcast float [[V2F]] to i32
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[V1]], [[V2]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
   call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
   %v1 = load i32, i32* %p
   call void @clobber()
@@ -233,8 +429,15 @@ define i32 @test_false_negative_types(i32* %p) {
 }
 
 define i32 @test_negative_size1(i32* %p) {
-; CHECK-LABEL: @test_negative_size1
-; CHECK: ret i32 %sub
+; CHECK-LABEL: define {{[^@]+}}@test_negative_size1
+; CHECK-SAME: (i32* [[P:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 3, i32* [[P]])
+; CHECK-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; CHECK-NEXT:    call void @clobber()
+; CHECK-NEXT:    [[V2:%.*]] = load i32, i32* [[P]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[V1]], [[V2]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
   call {}* @llvm.invariant.start.p0i32(i64 3, i32* %p)
   %v1 = load i32, i32* %p
   call void @clobber()
@@ -244,8 +447,15 @@ define i32 @test_negative_size1(i32* %p) {
 }
 
 define i32 @test_negative_size2(i32* %p) {
-; CHECK-LABEL: @test_negative_size2
-; CHECK: ret i32 %sub
+; CHECK-LABEL: define {{[^@]+}}@test_negative_size2
+; CHECK-SAME: (i32* [[P:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 0, i32* [[P]])
+; CHECK-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; CHECK-NEXT:    call void @clobber()
+; CHECK-NEXT:    [[V2:%.*]] = load i32, i32* [[P]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[V1]], [[V2]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
   call {}* @llvm.invariant.start.p0i32(i64 0, i32* %p)
   %v1 = load i32, i32* %p
   call void @clobber()
@@ -255,8 +465,16 @@ define i32 @test_negative_size2(i32* %p) {
 }
 
 define i32 @test_negative_scope(i32* %p) {
-; CHECK-LABEL: @test_negative_scope
-; CHECK: ret i32 %sub
+; CHECK-LABEL: define {{[^@]+}}@test_negative_scope
+; CHECK-SAME: (i32* [[P:%.*]])
+; CHECK-NEXT:    [[SCOPE:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; CHECK-NEXT:    call void @llvm.invariant.end.p0i32({}* [[SCOPE]], i64 4, i32* [[P]])
+; CHECK-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; CHECK-NEXT:    call void @clobber()
+; CHECK-NEXT:    [[V2:%.*]] = load i32, i32* [[P]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[V1]], [[V2]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
   %scope = call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
   call void @llvm.invariant.end.p0i32({}* %scope, i64 4, i32* %p)
   %v1 = load i32, i32* %p
@@ -267,8 +485,16 @@ define i32 @test_negative_scope(i32* %p) {
 }
 
 define i32 @test_false_negative_scope(i32* %p) {
-; CHECK-LABEL: @test_false_negative_scope
-; CHECK: ret i32 %sub
+; CHECK-LABEL: define {{[^@]+}}@test_false_negative_scope
+; CHECK-SAME: (i32* [[P:%.*]])
+; CHECK-NEXT:    [[SCOPE:%.*]] = call {}* @llvm.invariant.start.p0i32(i64 4, i32* [[P]])
+; CHECK-NEXT:    [[V1:%.*]] = load i32, i32* [[P]]
+; CHECK-NEXT:    call void @clobber()
+; CHECK-NEXT:    [[V2:%.*]] = load i32, i32* [[P]]
+; CHECK-NEXT:    call void @llvm.invariant.end.p0i32({}* [[SCOPE]], i64 4, i32* [[P]])
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[V1]], [[V2]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
   %scope = call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
   %v1 = load i32, i32* %p
   call void @clobber()
@@ -280,8 +506,19 @@ define i32 @test_false_negative_scope(i32* %p) {
 
 ; Invariant load defact starts an invariant.start scope of the appropriate size
 define i32 @test_invariant_load_scope(i32* %p) {
-; CHECK-LABEL: @test_invariant_load_scope
-; CHECK: ret i32 0
+; NO_ASSUME-LABEL: define {{[^@]+}}@test_invariant_load_scope
+; NO_ASSUME-SAME: (i32* [[P:%.*]])
+; NO_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]], !invariant.load !0
+; NO_ASSUME-NEXT:    call void @clobber()
+; NO_ASSUME-NEXT:    ret i32 0
+;
+; USE_ASSUME-LABEL: define {{[^@]+}}@test_invariant_load_scope
+; USE_ASSUME-SAME: (i32* [[P:%.*]])
+; USE_ASSUME-NEXT:    [[V1:%.*]] = load i32, i32* [[P]], !invariant.load !0
+; USE_ASSUME-NEXT:    call void @clobber()
+; USE_ASSUME-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 4), "nonnull"(i32* [[P]]) ]
+; USE_ASSUME-NEXT:    ret i32 0
+;
   %v1 = load i32, i32* %p, !invariant.load !{}
   call void @clobber()
   %v2 = load i32, i32* %p
diff --git a/llvm/test/Transforms/ExpandMemCmp/X86/memcmp.ll b/llvm/test/Transforms/ExpandMemCmp/X86/memcmp.ll
index b8cfe04d43cb8..df05021473585 100644
--- a/llvm/test/Transforms/ExpandMemCmp/X86/memcmp.ll
+++ b/llvm/test/Transforms/ExpandMemCmp/X86/memcmp.ll
@@ -165,8 +165,36 @@ define i32 @cmp6(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 
 define i32 @cmp7(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; ALL-LABEL: @cmp7(
-; ALL-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 7)
-; ALL-NEXT:    ret i32 [[CALL]]
+; ALL-NEXT:    br label [[LOADBB:%.*]]
+; ALL:       res_block:
+; ALL-NEXT:    [[PHI_SRC1:%.*]] = phi i32 [ [[TMP7:%.*]], [[LOADBB]] ], [ [[TMP16:%.*]], [[LOADBB1:%.*]] ]
+; ALL-NEXT:    [[PHI_SRC2:%.*]] = phi i32 [ [[TMP8:%.*]], [[LOADBB]] ], [ [[TMP17:%.*]], [[LOADBB1]] ]
+; ALL-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[PHI_SRC1]], [[PHI_SRC2]]
+; ALL-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 -1, i32 1
+; ALL-NEXT:    br label [[ENDBLOCK:%.*]]
+; ALL:       loadbb:
+; ALL-NEXT:    [[TMP3:%.*]] = bitcast i8* [[X:%.*]] to i32*
+; ALL-NEXT:    [[TMP4:%.*]] = bitcast i8* [[Y:%.*]] to i32*
+; ALL-NEXT:    [[TMP5:%.*]] = load i32, i32* [[TMP3]]
+; ALL-NEXT:    [[TMP6:%.*]] = load i32, i32* [[TMP4]]
+; ALL-NEXT:    [[TMP7]] = call i32 @llvm.bswap.i32(i32 [[TMP5]])
+; ALL-NEXT:    [[TMP8]] = call i32 @llvm.bswap.i32(i32 [[TMP6]])
+; ALL-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP7]], [[TMP8]]
+; ALL-NEXT:    br i1 [[TMP9]], label [[LOADBB1]], label [[RES_BLOCK:%.*]]
+; ALL:       loadbb1:
+; ALL-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i64 3
+; ALL-NEXT:    [[TMP11:%.*]] = getelementptr i8, i8* [[Y]], i64 3
+; ALL-NEXT:    [[TMP12:%.*]] = bitcast i8* [[TMP10]] to i32*
+; ALL-NEXT:    [[TMP13:%.*]] = bitcast i8* [[TMP11]] to i32*
+; ALL-NEXT:    [[TMP14:%.*]] = load i32, i32* [[TMP12]]
+; ALL-NEXT:    [[TMP15:%.*]] = load i32, i32* [[TMP13]]
+; ALL-NEXT:    [[TMP16]] = call i32 @llvm.bswap.i32(i32 [[TMP14]])
+; ALL-NEXT:    [[TMP17]] = call i32 @llvm.bswap.i32(i32 [[TMP15]])
+; ALL-NEXT:    [[TMP18:%.*]] = icmp eq i32 [[TMP16]], [[TMP17]]
+; ALL-NEXT:    br i1 [[TMP18]], label [[ENDBLOCK]], label [[RES_BLOCK]]
+; ALL:       endblock:
+; ALL-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ [[TMP2]], [[RES_BLOCK]] ]
+; ALL-NEXT:    ret i32 [[PHI_RES]]
 ;
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 7)
   ret i32 %call
@@ -304,9 +332,41 @@ define i32 @cmp10(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 }
 
 define i32 @cmp11(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
-; ALL-LABEL: @cmp11(
-; ALL-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 11)
-; ALL-NEXT:    ret i32 [[CALL]]
+; X32-LABEL: @cmp11(
+; X32-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 11)
+; X32-NEXT:    ret i32 [[CALL]]
+;
+; X64-LABEL: @cmp11(
+; X64-NEXT:    br label [[LOADBB:%.*]]
+; X64:       res_block:
+; X64-NEXT:    [[PHI_SRC1:%.*]] = phi i64 [ [[TMP7:%.*]], [[LOADBB]] ], [ [[TMP16:%.*]], [[LOADBB1:%.*]] ]
+; X64-NEXT:    [[PHI_SRC2:%.*]] = phi i64 [ [[TMP8:%.*]], [[LOADBB]] ], [ [[TMP17:%.*]], [[LOADBB1]] ]
+; X64-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[PHI_SRC1]], [[PHI_SRC2]]
+; X64-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 -1, i32 1
+; X64-NEXT:    br label [[ENDBLOCK:%.*]]
+; X64:       loadbb:
+; X64-NEXT:    [[TMP3:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64-NEXT:    [[TMP4:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64-NEXT:    [[TMP5:%.*]] = load i64, i64* [[TMP3]]
+; X64-NEXT:    [[TMP6:%.*]] = load i64, i64* [[TMP4]]
+; X64-NEXT:    [[TMP7]] = call i64 @llvm.bswap.i64(i64 [[TMP5]])
+; X64-NEXT:    [[TMP8]] = call i64 @llvm.bswap.i64(i64 [[TMP6]])
+; X64-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[TMP7]], [[TMP8]]
+; X64-NEXT:    br i1 [[TMP9]], label [[LOADBB1]], label [[RES_BLOCK:%.*]]
+; X64:       loadbb1:
+; X64-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i64 3
+; X64-NEXT:    [[TMP11:%.*]] = getelementptr i8, i8* [[Y]], i64 3
+; X64-NEXT:    [[TMP12:%.*]] = bitcast i8* [[TMP10]] to i64*
+; X64-NEXT:    [[TMP13:%.*]] = bitcast i8* [[TMP11]] to i64*
+; X64-NEXT:    [[TMP14:%.*]] = load i64, i64* [[TMP12]]
+; X64-NEXT:    [[TMP15:%.*]] = load i64, i64* [[TMP13]]
+; X64-NEXT:    [[TMP16]] = call i64 @llvm.bswap.i64(i64 [[TMP14]])
+; X64-NEXT:    [[TMP17]] = call i64 @llvm.bswap.i64(i64 [[TMP15]])
+; X64-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[TMP16]], [[TMP17]]
+; X64-NEXT:    br i1 [[TMP18]], label [[ENDBLOCK]], label [[RES_BLOCK]]
+; X64:       endblock:
+; X64-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ [[TMP2]], [[RES_BLOCK]] ]
+; X64-NEXT:    ret i32 [[PHI_RES]]
 ;
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 11)
   ret i32 %call
@@ -356,27 +416,123 @@ define i32 @cmp12(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 }
 
 define i32 @cmp13(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
-; ALL-LABEL: @cmp13(
-; ALL-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 13)
-; ALL-NEXT:    ret i32 [[CALL]]
+; X32-LABEL: @cmp13(
+; X32-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 13)
+; X32-NEXT:    ret i32 [[CALL]]
+;
+; X64-LABEL: @cmp13(
+; X64-NEXT:    br label [[LOADBB:%.*]]
+; X64:       res_block:
+; X64-NEXT:    [[PHI_SRC1:%.*]] = phi i64 [ [[TMP7:%.*]], [[LOADBB]] ], [ [[TMP16:%.*]], [[LOADBB1:%.*]] ]
+; X64-NEXT:    [[PHI_SRC2:%.*]] = phi i64 [ [[TMP8:%.*]], [[LOADBB]] ], [ [[TMP17:%.*]], [[LOADBB1]] ]
+; X64-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[PHI_SRC1]], [[PHI_SRC2]]
+; X64-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 -1, i32 1
+; X64-NEXT:    br label [[ENDBLOCK:%.*]]
+; X64:       loadbb:
+; X64-NEXT:    [[TMP3:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64-NEXT:    [[TMP4:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64-NEXT:    [[TMP5:%.*]] = load i64, i64* [[TMP3]]
+; X64-NEXT:    [[TMP6:%.*]] = load i64, i64* [[TMP4]]
+; X64-NEXT:    [[TMP7]] = call i64 @llvm.bswap.i64(i64 [[TMP5]])
+; X64-NEXT:    [[TMP8]] = call i64 @llvm.bswap.i64(i64 [[TMP6]])
+; X64-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[TMP7]], [[TMP8]]
+; X64-NEXT:    br i1 [[TMP9]], label [[LOADBB1]], label [[RES_BLOCK:%.*]]
+; X64:       loadbb1:
+; X64-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i64 5
+; X64-NEXT:    [[TMP11:%.*]] = getelementptr i8, i8* [[Y]], i64 5
+; X64-NEXT:    [[TMP12:%.*]] = bitcast i8* [[TMP10]] to i64*
+; X64-NEXT:    [[TMP13:%.*]] = bitcast i8* [[TMP11]] to i64*
+; X64-NEXT:    [[TMP14:%.*]] = load i64, i64* [[TMP12]]
+; X64-NEXT:    [[TMP15:%.*]] = load i64, i64* [[TMP13]]
+; X64-NEXT:    [[TMP16]] = call i64 @llvm.bswap.i64(i64 [[TMP14]])
+; X64-NEXT:    [[TMP17]] = call i64 @llvm.bswap.i64(i64 [[TMP15]])
+; X64-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[TMP16]], [[TMP17]]
+; X64-NEXT:    br i1 [[TMP18]], label [[ENDBLOCK]], label [[RES_BLOCK]]
+; X64:       endblock:
+; X64-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ [[TMP2]], [[RES_BLOCK]] ]
+; X64-NEXT:    ret i32 [[PHI_RES]]
 ;
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 13)
   ret i32 %call
 }
 
 define i32 @cmp14(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
-; ALL-LABEL: @cmp14(
-; ALL-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 14)
-; ALL-NEXT:    ret i32 [[CALL]]
+; X32-LABEL: @cmp14(
+; X32-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 14)
+; X32-NEXT:    ret i32 [[CALL]]
+;
+; X64-LABEL: @cmp14(
+; X64-NEXT:    br label [[LOADBB:%.*]]
+; X64:       res_block:
+; X64-NEXT:    [[PHI_SRC1:%.*]] = phi i64 [ [[TMP7:%.*]], [[LOADBB]] ], [ [[TMP16:%.*]], [[LOADBB1:%.*]] ]
+; X64-NEXT:    [[PHI_SRC2:%.*]] = phi i64 [ [[TMP8:%.*]], [[LOADBB]] ], [ [[TMP17:%.*]], [[LOADBB1]] ]
+; X64-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[PHI_SRC1]], [[PHI_SRC2]]
+; X64-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 -1, i32 1
+; X64-NEXT:    br label [[ENDBLOCK:%.*]]
+; X64:       loadbb:
+; X64-NEXT:    [[TMP3:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64-NEXT:    [[TMP4:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64-NEXT:    [[TMP5:%.*]] = load i64, i64* [[TMP3]]
+; X64-NEXT:    [[TMP6:%.*]] = load i64, i64* [[TMP4]]
+; X64-NEXT:    [[TMP7]] = call i64 @llvm.bswap.i64(i64 [[TMP5]])
+; X64-NEXT:    [[TMP8]] = call i64 @llvm.bswap.i64(i64 [[TMP6]])
+; X64-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[TMP7]], [[TMP8]]
+; X64-NEXT:    br i1 [[TMP9]], label [[LOADBB1]], label [[RES_BLOCK:%.*]]
+; X64:       loadbb1:
+; X64-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i64 6
+; X64-NEXT:    [[TMP11:%.*]] = getelementptr i8, i8* [[Y]], i64 6
+; X64-NEXT:    [[TMP12:%.*]] = bitcast i8* [[TMP10]] to i64*
+; X64-NEXT:    [[TMP13:%.*]] = bitcast i8* [[TMP11]] to i64*
+; X64-NEXT:    [[TMP14:%.*]] = load i64, i64* [[TMP12]]
+; X64-NEXT:    [[TMP15:%.*]] = load i64, i64* [[TMP13]]
+; X64-NEXT:    [[TMP16]] = call i64 @llvm.bswap.i64(i64 [[TMP14]])
+; X64-NEXT:    [[TMP17]] = call i64 @llvm.bswap.i64(i64 [[TMP15]])
+; X64-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[TMP16]], [[TMP17]]
+; X64-NEXT:    br i1 [[TMP18]], label [[ENDBLOCK]], label [[RES_BLOCK]]
+; X64:       endblock:
+; X64-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ [[TMP2]], [[RES_BLOCK]] ]
+; X64-NEXT:    ret i32 [[PHI_RES]]
 ;
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 14)
   ret i32 %call
 }
 
 define i32 @cmp15(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
-; ALL-LABEL: @cmp15(
-; ALL-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 15)
-; ALL-NEXT:    ret i32 [[CALL]]
+; X32-LABEL: @cmp15(
+; X32-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 15)
+; X32-NEXT:    ret i32 [[CALL]]
+;
+; X64-LABEL: @cmp15(
+; X64-NEXT:    br label [[LOADBB:%.*]]
+; X64:       res_block:
+; X64-NEXT:    [[PHI_SRC1:%.*]] = phi i64 [ [[TMP7:%.*]], [[LOADBB]] ], [ [[TMP16:%.*]], [[LOADBB1:%.*]] ]
+; X64-NEXT:    [[PHI_SRC2:%.*]] = phi i64 [ [[TMP8:%.*]], [[LOADBB]] ], [ [[TMP17:%.*]], [[LOADBB1]] ]
+; X64-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[PHI_SRC1]], [[PHI_SRC2]]
+; X64-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 -1, i32 1
+; X64-NEXT:    br label [[ENDBLOCK:%.*]]
+; X64:       loadbb:
+; X64-NEXT:    [[TMP3:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64-NEXT:    [[TMP4:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64-NEXT:    [[TMP5:%.*]] = load i64, i64* [[TMP3]]
+; X64-NEXT:    [[TMP6:%.*]] = load i64, i64* [[TMP4]]
+; X64-NEXT:    [[TMP7]] = call i64 @llvm.bswap.i64(i64 [[TMP5]])
+; X64-NEXT:    [[TMP8]] = call i64 @llvm.bswap.i64(i64 [[TMP6]])
+; X64-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[TMP7]], [[TMP8]]
+; X64-NEXT:    br i1 [[TMP9]], label [[LOADBB1]], label [[RES_BLOCK:%.*]]
+; X64:       loadbb1:
+; X64-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i64 7
+; X64-NEXT:    [[TMP11:%.*]] = getelementptr i8, i8* [[Y]], i64 7
+; X64-NEXT:    [[TMP12:%.*]] = bitcast i8* [[TMP10]] to i64*
+; X64-NEXT:    [[TMP13:%.*]] = bitcast i8* [[TMP11]] to i64*
+; X64-NEXT:    [[TMP14:%.*]] = load i64, i64* [[TMP12]]
+; X64-NEXT:    [[TMP15:%.*]] = load i64, i64* [[TMP13]]
+; X64-NEXT:    [[TMP16]] = call i64 @llvm.bswap.i64(i64 [[TMP14]])
+; X64-NEXT:    [[TMP17]] = call i64 @llvm.bswap.i64(i64 [[TMP15]])
+; X64-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[TMP16]], [[TMP17]]
+; X64-NEXT:    br i1 [[TMP18]], label [[ENDBLOCK]], label [[RES_BLOCK]]
+; X64:       endblock:
+; X64-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ [[TMP2]], [[RES_BLOCK]] ]
+; X64-NEXT:    ret i32 [[PHI_RES]]
 ;
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 15)
   ret i32 %call
diff --git a/llvm/test/Transforms/GVN/malloc-load-removal.ll b/llvm/test/Transforms/GVN/malloc-load-removal.ll
index 1d7a2ddc4c2d0..84f4746344ca9 100644
--- a/llvm/test/Transforms/GVN/malloc-load-removal.ll
+++ b/llvm/test/Transforms/GVN/malloc-load-removal.ll
@@ -54,3 +54,28 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK_NO_LIBCALLS: load
 ; CHECK_NO_LIBCALLS: icmp
 }
+
+declare i8* @aligned_alloc(i64, i64) nounwind
+
+define noalias i8* @test3() nounwind uwtable ssp {
+entry:
+  %call = tail call i8* @aligned_alloc(i64 256, i64 32) nounwind
+  %0 = load i8, i8* %call, align 32
+  %tobool = icmp eq i8 %0, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  store i8 0, i8* %call, align 1
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  ret i8* %call
+
+; CHECK-LABEL: @test3(
+; CHECK-NOT: load
+; CHECK-NOT: icmp
+
+; CHECK_NO_LIBCALLS-LABEL: @test3(
+; CHECK_NO_LIBCALLS: load
+; CHECK_NO_LIBCALLS: icmp
+}
diff --git a/llvm/test/CodeGen/ARM/indvar-cost.ll b/llvm/test/Transforms/IndVarSimplify/ARM/indvar-cost.ll
similarity index 87%
rename from llvm/test/CodeGen/ARM/indvar-cost.ll
rename to llvm/test/Transforms/IndVarSimplify/ARM/indvar-cost.ll
index df4c71777b964..14d873d9e3ac2 100644
--- a/llvm/test/CodeGen/ARM/indvar-cost.ll
+++ b/llvm/test/Transforms/IndVarSimplify/ARM/indvar-cost.ll
@@ -13,9 +13,6 @@ define dso_local arm_aapcscc void @arm_conv_fast_q15(i16* %pSrcA, i32 %srcALen,
 ; CHECK-T1-NEXT:    [[CMP41080:%.*]] = icmp eq i32 [[SUB]], 0
 ; CHECK-T1-NEXT:    br i1 [[CMP41080]], label [[WHILE_END13:%.*]], label [[WHILE_COND5_PREHEADER_PREHEADER:%.*]]
 ; CHECK-T1:       while.cond5.preheader.preheader:
-; CHECK-T1-NEXT:    [[TMP0:%.*]] = add i32 [[SRCALEN_SRCBLEN]], -2
-; CHECK-T1-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[TMP0]], 2
-; CHECK-T1-NEXT:    [[UMIN:%.*]] = select i1 [[TMP1]], i32 [[TMP0]], i32 2
 ; CHECK-T1-NEXT:    br label [[WHILE_COND5_PREHEADER:%.*]]
 ; CHECK-T1:       while.cond5.preheader:
 ; CHECK-T1-NEXT:    [[COUNT_01084:%.*]] = phi i32 [ [[INC:%.*]], [[WHILE_END:%.*]] ], [ 1, [[WHILE_COND5_PREHEADER_PREHEADER]] ]
@@ -29,11 +26,11 @@ define dso_local arm_aapcscc void @arm_conv_fast_q15(i16* %pSrcA, i32 %srcALen,
 ; CHECK-T1-NEXT:    [[PY_11076:%.*]] = phi i16* [ [[INCDEC_PTR8:%.*]], [[WHILE_BODY7]] ], [ [[PY_01082]], [[WHILE_COND5_PREHEADER]] ]
 ; CHECK-T1-NEXT:    [[PX_11075:%.*]] = phi i16* [ [[INCDEC_PTR:%.*]], [[WHILE_BODY7]] ], [ [[PSRCB_PSRCA]], [[WHILE_COND5_PREHEADER]] ]
 ; CHECK-T1-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i16, i16* [[PX_11075]], i32 1
-; CHECK-T1-NEXT:    [[TMP2:%.*]] = load i16, i16* [[PX_11075]], align 2
-; CHECK-T1-NEXT:    [[CONV:%.*]] = sext i16 [[TMP2]] to i32
+; CHECK-T1-NEXT:    [[TMP0:%.*]] = load i16, i16* [[PX_11075]], align 2
+; CHECK-T1-NEXT:    [[CONV:%.*]] = sext i16 [[TMP0]] to i32
 ; CHECK-T1-NEXT:    [[INCDEC_PTR8]] = getelementptr inbounds i16, i16* [[PY_11076]], i32 -1
-; CHECK-T1-NEXT:    [[TMP3:%.*]] = load i16, i16* [[PY_11076]], align 2
-; CHECK-T1-NEXT:    [[CONV9:%.*]] = sext i16 [[TMP3]] to i32
+; CHECK-T1-NEXT:    [[TMP1:%.*]] = load i16, i16* [[PY_11076]], align 2
+; CHECK-T1-NEXT:    [[CONV9:%.*]] = sext i16 [[TMP1]] to i32
 ; CHECK-T1-NEXT:    [[MUL_I:%.*]] = mul nsw i32 [[CONV9]], [[CONV]]
 ; CHECK-T1-NEXT:    [[SHR3_I:%.*]] = ashr i32 [[CONV]], 16
 ; CHECK-T1-NEXT:    [[SHR4_I:%.*]] = ashr i32 [[CONV9]], 16
@@ -45,8 +42,8 @@ define dso_local arm_aapcscc void @arm_conv_fast_q15(i16* %pSrcA, i32 %srcALen,
 ; CHECK-T1-NEXT:    br i1 [[CMP6]], label [[WHILE_END]], label [[WHILE_BODY7]]
 ; CHECK-T1:       while.end:
 ; CHECK-T1-NEXT:    [[ADD6_I_LCSSA:%.*]] = phi i32 [ [[ADD6_I]], [[WHILE_BODY7]] ]
-; CHECK-T1-NEXT:    [[TMP4:%.*]] = lshr i32 [[ADD6_I_LCSSA]], 15
-; CHECK-T1-NEXT:    [[CONV10:%.*]] = trunc i32 [[TMP4]] to i16
+; CHECK-T1-NEXT:    [[TMP2:%.*]] = lshr i32 [[ADD6_I_LCSSA]], 15
+; CHECK-T1-NEXT:    [[CONV10:%.*]] = trunc i32 [[TMP2]] to i16
 ; CHECK-T1-NEXT:    [[INCDEC_PTR11]] = getelementptr inbounds i16, i16* [[POUT_01081]], i32 1
 ; CHECK-T1-NEXT:    store i16 [[CONV10]], i16* [[POUT_01081]], align 2
 ; CHECK-T1-NEXT:    [[ADD_PTR]] = getelementptr inbounds i16, i16* [[PSRCA_PSRCB]], i32 [[COUNT_01084]]
@@ -54,19 +51,19 @@ define dso_local arm_aapcscc void @arm_conv_fast_q15(i16* %pSrcA, i32 %srcALen,
 ; CHECK-T1-NEXT:    [[DEC12]] = add i32 [[BLOCKSIZE1_01083]], -1
 ; CHECK-T1-NEXT:    [[CMP3:%.*]] = icmp ult i32 [[COUNT_01084]], 3
 ; CHECK-T1-NEXT:    [[CMP4:%.*]] = icmp ne i32 [[DEC12]], 0
-; CHECK-T1-NEXT:    [[TMP5:%.*]] = and i1 [[CMP4]], [[CMP3]]
-; CHECK-T1-NEXT:    br i1 [[TMP5]], label [[WHILE_COND5_PREHEADER]], label [[WHILE_END13_LOOPEXIT:%.*]]
+; CHECK-T1-NEXT:    [[TMP3:%.*]] = and i1 [[CMP4]], [[CMP3]]
+; CHECK-T1-NEXT:    br i1 [[TMP3]], label [[WHILE_COND5_PREHEADER]], label [[WHILE_END13_LOOPEXIT:%.*]]
 ; CHECK-T1:       while.end13.loopexit:
 ; CHECK-T1-NEXT:    [[INCDEC_PTR11_LCSSA:%.*]] = phi i16* [ [[INCDEC_PTR11]], [[WHILE_END]] ]
 ; CHECK-T1-NEXT:    [[ADD_PTR_LCSSA:%.*]] = phi i16* [ [[ADD_PTR]], [[WHILE_END]] ]
+; CHECK-T1-NEXT:    [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], [[WHILE_END]] ]
 ; CHECK-T1-NEXT:    [[DEC12_LCSSA:%.*]] = phi i32 [ [[DEC12]], [[WHILE_END]] ]
-; CHECK-T1-NEXT:    [[TMP6:%.*]] = add nuw nsw i32 [[UMIN]], 2
 ; CHECK-T1-NEXT:    br label [[WHILE_END13]]
 ; CHECK-T1:       while.end13:
 ; CHECK-T1-NEXT:    [[POUT_0_LCSSA:%.*]] = phi i16* [ [[PDST]], [[ENTRY:%.*]] ], [ [[INCDEC_PTR11_LCSSA]], [[WHILE_END13_LOOPEXIT]] ]
 ; CHECK-T1-NEXT:    [[PY_0_LCSSA:%.*]] = phi i16* [ [[PSRCA_PSRCB]], [[ENTRY]] ], [ [[ADD_PTR_LCSSA]], [[WHILE_END13_LOOPEXIT]] ]
 ; CHECK-T1-NEXT:    [[BLOCKSIZE1_0_LCSSA:%.*]] = phi i32 [ [[SUB]], [[ENTRY]] ], [ [[DEC12_LCSSA]], [[WHILE_END13_LOOPEXIT]] ]
-; CHECK-T1-NEXT:    [[COUNT_0_LCSSA:%.*]] = phi i32 [ 1, [[ENTRY]] ], [ [[TMP6]], [[WHILE_END13_LOOPEXIT]] ]
+; CHECK-T1-NEXT:    [[COUNT_0_LCSSA:%.*]] = phi i32 [ 1, [[ENTRY]] ], [ [[INC_LCSSA]], [[WHILE_END13_LOOPEXIT]] ]
 ; CHECK-T1-NEXT:    [[CMP161068:%.*]] = icmp eq i32 [[BLOCKSIZE1_0_LCSSA]], 0
 ; CHECK-T1-NEXT:    br i1 [[CMP161068]], label [[EXIT:%.*]], label [[WHILE_BODY18_PREHEADER:%.*]]
 ; CHECK-T1:       while.body18.preheader:
@@ -88,34 +85,34 @@ define dso_local arm_aapcscc void @arm_conv_fast_q15(i16* %pSrcA, i32 %srcALen,
 ; CHECK-T1-NEXT:    [[PY_31056:%.*]] = phi i16* [ [[ADD_PTR_I884:%.*]], [[WHILE_BODY23]] ], [ [[PY_21070]], [[WHILE_BODY23_PREHEADER]] ]
 ; CHECK-T1-NEXT:    [[PX_31055:%.*]] = phi i16* [ [[ADD_PTR_I890:%.*]], [[WHILE_BODY23]] ], [ [[PSRCB_PSRCA]], [[WHILE_BODY23_PREHEADER]] ]
 ; CHECK-T1-NEXT:    [[ARRAYIDX_I907:%.*]] = getelementptr inbounds i16, i16* [[PX_31055]], i32 1
-; CHECK-T1-NEXT:    [[TMP7:%.*]] = load i16, i16* [[ARRAYIDX_I907]], align 2
-; CHECK-T1-NEXT:    [[TMP8:%.*]] = load i16, i16* [[PX_31055]], align 2
+; CHECK-T1-NEXT:    [[TMP4:%.*]] = load i16, i16* [[ARRAYIDX_I907]], align 2
+; CHECK-T1-NEXT:    [[TMP5:%.*]] = load i16, i16* [[PX_31055]], align 2
 ; CHECK-T1-NEXT:    [[ADD_PTR_I912:%.*]] = getelementptr inbounds i16, i16* [[PX_31055]], i32 2
 ; CHECK-T1-NEXT:    [[ARRAYIDX_I901:%.*]] = getelementptr inbounds i16, i16* [[PY_31056]], i32 1
-; CHECK-T1-NEXT:    [[TMP9:%.*]] = load i16, i16* [[ARRAYIDX_I901]], align 2
-; CHECK-T1-NEXT:    [[TMP10:%.*]] = load i16, i16* [[PY_31056]], align 2
+; CHECK-T1-NEXT:    [[TMP6:%.*]] = load i16, i16* [[ARRAYIDX_I901]], align 2
+; CHECK-T1-NEXT:    [[TMP7:%.*]] = load i16, i16* [[PY_31056]], align 2
 ; CHECK-T1-NEXT:    [[ADD_PTR_I906:%.*]] = getelementptr inbounds i16, i16* [[PY_31056]], i32 -2
-; CHECK-T1-NEXT:    [[SHR_I892:%.*]] = sext i16 [[TMP8]] to i32
-; CHECK-T1-NEXT:    [[SHR1_I893:%.*]] = sext i16 [[TMP9]] to i32
+; CHECK-T1-NEXT:    [[SHR_I892:%.*]] = sext i16 [[TMP5]] to i32
+; CHECK-T1-NEXT:    [[SHR1_I893:%.*]] = sext i16 [[TMP6]] to i32
 ; CHECK-T1-NEXT:    [[MUL_I894:%.*]] = mul nsw i32 [[SHR1_I893]], [[SHR_I892]]
-; CHECK-T1-NEXT:    [[SHR2_I895:%.*]] = sext i16 [[TMP7]] to i32
-; CHECK-T1-NEXT:    [[SHR4_I897:%.*]] = sext i16 [[TMP10]] to i32
+; CHECK-T1-NEXT:    [[SHR2_I895:%.*]] = sext i16 [[TMP4]] to i32
+; CHECK-T1-NEXT:    [[SHR4_I897:%.*]] = sext i16 [[TMP7]] to i32
 ; CHECK-T1-NEXT:    [[MUL5_I898:%.*]] = mul nsw i32 [[SHR4_I897]], [[SHR2_I895]]
 ; CHECK-T1-NEXT:    [[ADD_I899:%.*]] = add i32 [[MUL_I894]], [[SUM_11057]]
 ; CHECK-T1-NEXT:    [[ADD6_I900:%.*]] = add i32 [[ADD_I899]], [[MUL5_I898]]
 ; CHECK-T1-NEXT:    [[ARRAYIDX_I885:%.*]] = getelementptr inbounds i16, i16* [[PX_31055]], i32 3
-; CHECK-T1-NEXT:    [[TMP11:%.*]] = load i16, i16* [[ARRAYIDX_I885]], align 2
-; CHECK-T1-NEXT:    [[TMP12:%.*]] = load i16, i16* [[ADD_PTR_I912]], align 2
+; CHECK-T1-NEXT:    [[TMP8:%.*]] = load i16, i16* [[ARRAYIDX_I885]], align 2
+; CHECK-T1-NEXT:    [[TMP9:%.*]] = load i16, i16* [[ADD_PTR_I912]], align 2
 ; CHECK-T1-NEXT:    [[ADD_PTR_I890]] = getelementptr inbounds i16, i16* [[PX_31055]], i32 4
 ; CHECK-T1-NEXT:    [[ARRAYIDX_I879:%.*]] = getelementptr inbounds i16, i16* [[PY_31056]], i32 -1
-; CHECK-T1-NEXT:    [[TMP13:%.*]] = load i16, i16* [[ARRAYIDX_I879]], align 2
-; CHECK-T1-NEXT:    [[TMP14:%.*]] = load i16, i16* [[ADD_PTR_I906]], align 2
+; CHECK-T1-NEXT:    [[TMP10:%.*]] = load i16, i16* [[ARRAYIDX_I879]], align 2
+; CHECK-T1-NEXT:    [[TMP11:%.*]] = load i16, i16* [[ADD_PTR_I906]], align 2
 ; CHECK-T1-NEXT:    [[ADD_PTR_I884]] = getelementptr inbounds i16, i16* [[PY_31056]], i32 -4
-; CHECK-T1-NEXT:    [[SHR_I870:%.*]] = sext i16 [[TMP12]] to i32
-; CHECK-T1-NEXT:    [[SHR1_I871:%.*]] = sext i16 [[TMP13]] to i32
+; CHECK-T1-NEXT:    [[SHR_I870:%.*]] = sext i16 [[TMP9]] to i32
+; CHECK-T1-NEXT:    [[SHR1_I871:%.*]] = sext i16 [[TMP10]] to i32
 ; CHECK-T1-NEXT:    [[MUL_I872:%.*]] = mul nsw i32 [[SHR1_I871]], [[SHR_I870]]
-; CHECK-T1-NEXT:    [[SHR2_I873:%.*]] = sext i16 [[TMP11]] to i32
-; CHECK-T1-NEXT:    [[SHR4_I875:%.*]] = sext i16 [[TMP14]] to i32
+; CHECK-T1-NEXT:    [[SHR2_I873:%.*]] = sext i16 [[TMP8]] to i32
+; CHECK-T1-NEXT:    [[SHR4_I875:%.*]] = sext i16 [[TMP11]] to i32
 ; CHECK-T1-NEXT:    [[MUL5_I876:%.*]] = mul nsw i32 [[SHR4_I875]], [[SHR2_I873]]
 ; CHECK-T1-NEXT:    [[ADD_I877:%.*]] = add i32 [[ADD6_I900]], [[MUL_I872]]
 ; CHECK-T1-NEXT:    [[ADD6_I878]] = add i32 [[ADD_I877]], [[MUL5_I876]]
@@ -143,11 +140,11 @@ define dso_local arm_aapcscc void @arm_conv_fast_q15(i16* %pSrcA, i32 %srcALen,
 ; CHECK-T1-NEXT:    [[PY_41064:%.*]] = phi i16* [ [[INCDEC_PTR39:%.*]], [[WHILE_BODY36]] ], [ [[ADD_PTR32]], [[WHILE_BODY36_PREHEADER]] ]
 ; CHECK-T1-NEXT:    [[PX_41063:%.*]] = phi i16* [ [[INCDEC_PTR37:%.*]], [[WHILE_BODY36]] ], [ [[PX_3_LCSSA]], [[WHILE_BODY36_PREHEADER]] ]
 ; CHECK-T1-NEXT:    [[INCDEC_PTR37]] = getelementptr inbounds i16, i16* [[PX_41063]], i32 1
-; CHECK-T1-NEXT:    [[TMP15:%.*]] = load i16, i16* [[PX_41063]], align 2
-; CHECK-T1-NEXT:    [[CONV38:%.*]] = sext i16 [[TMP15]] to i32
+; CHECK-T1-NEXT:    [[TMP12:%.*]] = load i16, i16* [[PX_41063]], align 2
+; CHECK-T1-NEXT:    [[CONV38:%.*]] = sext i16 [[TMP12]] to i32
 ; CHECK-T1-NEXT:    [[INCDEC_PTR39]] = getelementptr inbounds i16, i16* [[PY_41064]], i32 -1
-; CHECK-T1-NEXT:    [[TMP16:%.*]] = load i16, i16* [[PY_41064]], align 2
-; CHECK-T1-NEXT:    [[CONV40:%.*]] = sext i16 [[TMP16]] to i32
+; CHECK-T1-NEXT:    [[TMP13:%.*]] = load i16, i16* [[PY_41064]], align 2
+; CHECK-T1-NEXT:    [[CONV40:%.*]] = sext i16 [[TMP13]] to i32
 ; CHECK-T1-NEXT:    [[MUL_I863:%.*]] = mul nsw i32 [[CONV40]], [[CONV38]]
 ; CHECK-T1-NEXT:    [[SHR3_I864:%.*]] = ashr i32 [[CONV38]], 16
 ; CHECK-T1-NEXT:    [[SHR4_I865:%.*]] = ashr i32 [[CONV40]], 16
@@ -162,8 +159,8 @@ define dso_local arm_aapcscc void @arm_conv_fast_q15(i16* %pSrcA, i32 %srcALen,
 ; CHECK-T1-NEXT:    br label [[WHILE_END43]]
 ; CHECK-T1:       while.end43:
 ; CHECK-T1-NEXT:    [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_1_LCSSA]], [[WHILE_END31]] ], [ [[ADD6_I868_LCSSA]], [[WHILE_END43_LOOPEXIT]] ]
-; CHECK-T1-NEXT:    [[TMP17:%.*]] = lshr i32 [[SUM_2_LCSSA]], 15
-; CHECK-T1-NEXT:    [[CONV45:%.*]] = trunc i32 [[TMP17]] to i16
+; CHECK-T1-NEXT:    [[TMP14:%.*]] = lshr i32 [[SUM_2_LCSSA]], 15
+; CHECK-T1-NEXT:    [[CONV45:%.*]] = trunc i32 [[TMP14]] to i16
 ; CHECK-T1-NEXT:    [[INCDEC_PTR46]] = getelementptr inbounds i16, i16* [[POUT_11069]], i32 1
 ; CHECK-T1-NEXT:    store i16 [[CONV45]], i16* [[POUT_11069]], align 2
 ; CHECK-T1-NEXT:    [[SUB47:%.*]] = add i32 [[COUNT_11072]], -1
@@ -187,9 +184,6 @@ define dso_local arm_aapcscc void @arm_conv_fast_q15(i16* %pSrcA, i32 %srcALen,
 ; CHECK-T2-NEXT:    [[CMP41080:%.*]] = icmp eq i32 [[SUB]], 0
 ; CHECK-T2-NEXT:    br i1 [[CMP41080]], label [[WHILE_END13:%.*]], label [[WHILE_COND5_PREHEADER_PREHEADER:%.*]]
 ; CHECK-T2:       while.cond5.preheader.preheader:
-; CHECK-T2-NEXT:    [[TMP0:%.*]] = add i32 [[SRCALEN_SRCBLEN]], -2
-; CHECK-T2-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[TMP0]], 2
-; CHECK-T2-NEXT:    [[UMIN:%.*]] = select i1 [[TMP1]], i32 [[TMP0]], i32 2
 ; CHECK-T2-NEXT:    br label [[WHILE_COND5_PREHEADER:%.*]]
 ; CHECK-T2:       while.cond5.preheader:
 ; CHECK-T2-NEXT:    [[COUNT_01084:%.*]] = phi i32 [ [[INC:%.*]], [[WHILE_END:%.*]] ], [ 1, [[WHILE_COND5_PREHEADER_PREHEADER]] ]
@@ -203,11 +197,11 @@ define dso_local arm_aapcscc void @arm_conv_fast_q15(i16* %pSrcA, i32 %srcALen,
 ; CHECK-T2-NEXT:    [[PY_11076:%.*]] = phi i16* [ [[INCDEC_PTR8:%.*]], [[WHILE_BODY7]] ], [ [[PY_01082]], [[WHILE_COND5_PREHEADER]] ]
 ; CHECK-T2-NEXT:    [[PX_11075:%.*]] = phi i16* [ [[INCDEC_PTR:%.*]], [[WHILE_BODY7]] ], [ [[PSRCB_PSRCA]], [[WHILE_COND5_PREHEADER]] ]
 ; CHECK-T2-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i16, i16* [[PX_11075]], i32 1
-; CHECK-T2-NEXT:    [[TMP2:%.*]] = load i16, i16* [[PX_11075]], align 2
-; CHECK-T2-NEXT:    [[CONV:%.*]] = sext i16 [[TMP2]] to i32
+; CHECK-T2-NEXT:    [[TMP0:%.*]] = load i16, i16* [[PX_11075]], align 2
+; CHECK-T2-NEXT:    [[CONV:%.*]] = sext i16 [[TMP0]] to i32
 ; CHECK-T2-NEXT:    [[INCDEC_PTR8]] = getelementptr inbounds i16, i16* [[PY_11076]], i32 -1
-; CHECK-T2-NEXT:    [[TMP3:%.*]] = load i16, i16* [[PY_11076]], align 2
-; CHECK-T2-NEXT:    [[CONV9:%.*]] = sext i16 [[TMP3]] to i32
+; CHECK-T2-NEXT:    [[TMP1:%.*]] = load i16, i16* [[PY_11076]], align 2
+; CHECK-T2-NEXT:    [[CONV9:%.*]] = sext i16 [[TMP1]] to i32
 ; CHECK-T2-NEXT:    [[MUL_I:%.*]] = mul nsw i32 [[CONV9]], [[CONV]]
 ; CHECK-T2-NEXT:    [[SHR3_I:%.*]] = ashr i32 [[CONV]], 16
 ; CHECK-T2-NEXT:    [[SHR4_I:%.*]] = ashr i32 [[CONV9]], 16
@@ -219,8 +213,8 @@ define dso_local arm_aapcscc void @arm_conv_fast_q15(i16* %pSrcA, i32 %srcALen,
 ; CHECK-T2-NEXT:    br i1 [[CMP6]], label [[WHILE_END]], label [[WHILE_BODY7]]
 ; CHECK-T2:       while.end:
 ; CHECK-T2-NEXT:    [[ADD6_I_LCSSA:%.*]] = phi i32 [ [[ADD6_I]], [[WHILE_BODY7]] ]
-; CHECK-T2-NEXT:    [[TMP4:%.*]] = lshr i32 [[ADD6_I_LCSSA]], 15
-; CHECK-T2-NEXT:    [[CONV10:%.*]] = trunc i32 [[TMP4]] to i16
+; CHECK-T2-NEXT:    [[TMP2:%.*]] = lshr i32 [[ADD6_I_LCSSA]], 15
+; CHECK-T2-NEXT:    [[CONV10:%.*]] = trunc i32 [[TMP2]] to i16
 ; CHECK-T2-NEXT:    [[INCDEC_PTR11]] = getelementptr inbounds i16, i16* [[POUT_01081]], i32 1
 ; CHECK-T2-NEXT:    store i16 [[CONV10]], i16* [[POUT_01081]], align 2
 ; CHECK-T2-NEXT:    [[ADD_PTR]] = getelementptr inbounds i16, i16* [[PSRCA_PSRCB]], i32 [[COUNT_01084]]
@@ -228,19 +222,19 @@ define dso_local arm_aapcscc void @arm_conv_fast_q15(i16* %pSrcA, i32 %srcALen,
 ; CHECK-T2-NEXT:    [[DEC12]] = add i32 [[BLOCKSIZE1_01083]], -1
 ; CHECK-T2-NEXT:    [[CMP3:%.*]] = icmp ult i32 [[COUNT_01084]], 3
 ; CHECK-T2-NEXT:    [[CMP4:%.*]] = icmp ne i32 [[DEC12]], 0
-; CHECK-T2-NEXT:    [[TMP5:%.*]] = and i1 [[CMP4]], [[CMP3]]
-; CHECK-T2-NEXT:    br i1 [[TMP5]], label [[WHILE_COND5_PREHEADER]], label [[WHILE_END13_LOOPEXIT:%.*]]
+; CHECK-T2-NEXT:    [[TMP3:%.*]] = and i1 [[CMP4]], [[CMP3]]
+; CHECK-T2-NEXT:    br i1 [[TMP3]], label [[WHILE_COND5_PREHEADER]], label [[WHILE_END13_LOOPEXIT:%.*]]
 ; CHECK-T2:       while.end13.loopexit:
 ; CHECK-T2-NEXT:    [[INCDEC_PTR11_LCSSA:%.*]] = phi i16* [ [[INCDEC_PTR11]], [[WHILE_END]] ]
 ; CHECK-T2-NEXT:    [[ADD_PTR_LCSSA:%.*]] = phi i16* [ [[ADD_PTR]], [[WHILE_END]] ]
+; CHECK-T2-NEXT:    [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], [[WHILE_END]] ]
 ; CHECK-T2-NEXT:    [[DEC12_LCSSA:%.*]] = phi i32 [ [[DEC12]], [[WHILE_END]] ]
-; CHECK-T2-NEXT:    [[TMP6:%.*]] = add nuw nsw i32 [[UMIN]], 2
 ; CHECK-T2-NEXT:    br label [[WHILE_END13]]
 ; CHECK-T2:       while.end13:
 ; CHECK-T2-NEXT:    [[POUT_0_LCSSA:%.*]] = phi i16* [ [[PDST]], [[ENTRY:%.*]] ], [ [[INCDEC_PTR11_LCSSA]], [[WHILE_END13_LOOPEXIT]] ]
 ; CHECK-T2-NEXT:    [[PY_0_LCSSA:%.*]] = phi i16* [ [[PSRCA_PSRCB]], [[ENTRY]] ], [ [[ADD_PTR_LCSSA]], [[WHILE_END13_LOOPEXIT]] ]
 ; CHECK-T2-NEXT:    [[BLOCKSIZE1_0_LCSSA:%.*]] = phi i32 [ [[SUB]], [[ENTRY]] ], [ [[DEC12_LCSSA]], [[WHILE_END13_LOOPEXIT]] ]
-; CHECK-T2-NEXT:    [[COUNT_0_LCSSA:%.*]] = phi i32 [ 1, [[ENTRY]] ], [ [[TMP6]], [[WHILE_END13_LOOPEXIT]] ]
+; CHECK-T2-NEXT:    [[COUNT_0_LCSSA:%.*]] = phi i32 [ 1, [[ENTRY]] ], [ [[INC_LCSSA]], [[WHILE_END13_LOOPEXIT]] ]
 ; CHECK-T2-NEXT:    [[CMP161068:%.*]] = icmp eq i32 [[BLOCKSIZE1_0_LCSSA]], 0
 ; CHECK-T2-NEXT:    br i1 [[CMP161068]], label [[EXIT:%.*]], label [[WHILE_BODY18_PREHEADER:%.*]]
 ; CHECK-T2:       while.body18.preheader:
@@ -262,34 +256,34 @@ define dso_local arm_aapcscc void @arm_conv_fast_q15(i16* %pSrcA, i32 %srcALen,
 ; CHECK-T2-NEXT:    [[PY_31056:%.*]] = phi i16* [ [[ADD_PTR_I884:%.*]], [[WHILE_BODY23]] ], [ [[PY_21070]], [[WHILE_BODY23_PREHEADER]] ]
 ; CHECK-T2-NEXT:    [[PX_31055:%.*]] = phi i16* [ [[ADD_PTR_I890:%.*]], [[WHILE_BODY23]] ], [ [[PSRCB_PSRCA]], [[WHILE_BODY23_PREHEADER]] ]
 ; CHECK-T2-NEXT:    [[ARRAYIDX_I907:%.*]] = getelementptr inbounds i16, i16* [[PX_31055]], i32 1
-; CHECK-T2-NEXT:    [[TMP7:%.*]] = load i16, i16* [[ARRAYIDX_I907]], align 2
-; CHECK-T2-NEXT:    [[TMP8:%.*]] = load i16, i16* [[PX_31055]], align 2
+; CHECK-T2-NEXT:    [[TMP4:%.*]] = load i16, i16* [[ARRAYIDX_I907]], align 2
+; CHECK-T2-NEXT:    [[TMP5:%.*]] = load i16, i16* [[PX_31055]], align 2
 ; CHECK-T2-NEXT:    [[ADD_PTR_I912:%.*]] = getelementptr inbounds i16, i16* [[PX_31055]], i32 2
 ; CHECK-T2-NEXT:    [[ARRAYIDX_I901:%.*]] = getelementptr inbounds i16, i16* [[PY_31056]], i32 1
-; CHECK-T2-NEXT:    [[TMP9:%.*]] = load i16, i16* [[ARRAYIDX_I901]], align 2
-; CHECK-T2-NEXT:    [[TMP10:%.*]] = load i16, i16* [[PY_31056]], align 2
+; CHECK-T2-NEXT:    [[TMP6:%.*]] = load i16, i16* [[ARRAYIDX_I901]], align 2
+; CHECK-T2-NEXT:    [[TMP7:%.*]] = load i16, i16* [[PY_31056]], align 2
 ; CHECK-T2-NEXT:    [[ADD_PTR_I906:%.*]] = getelementptr inbounds i16, i16* [[PY_31056]], i32 -2
-; CHECK-T2-NEXT:    [[SHR_I892:%.*]] = sext i16 [[TMP8]] to i32
-; CHECK-T2-NEXT:    [[SHR1_I893:%.*]] = sext i16 [[TMP9]] to i32
+; CHECK-T2-NEXT:    [[SHR_I892:%.*]] = sext i16 [[TMP5]] to i32
+; CHECK-T2-NEXT:    [[SHR1_I893:%.*]] = sext i16 [[TMP6]] to i32
 ; CHECK-T2-NEXT:    [[MUL_I894:%.*]] = mul nsw i32 [[SHR1_I893]], [[SHR_I892]]
-; CHECK-T2-NEXT:    [[SHR2_I895:%.*]] = sext i16 [[TMP7]] to i32
-; CHECK-T2-NEXT:    [[SHR4_I897:%.*]] = sext i16 [[TMP10]] to i32
+; CHECK-T2-NEXT:    [[SHR2_I895:%.*]] = sext i16 [[TMP4]] to i32
+; CHECK-T2-NEXT:    [[SHR4_I897:%.*]] = sext i16 [[TMP7]] to i32
 ; CHECK-T2-NEXT:    [[MUL5_I898:%.*]] = mul nsw i32 [[SHR4_I897]], [[SHR2_I895]]
 ; CHECK-T2-NEXT:    [[ADD_I899:%.*]] = add i32 [[MUL_I894]], [[SUM_11057]]
 ; CHECK-T2-NEXT:    [[ADD6_I900:%.*]] = add i32 [[ADD_I899]], [[MUL5_I898]]
 ; CHECK-T2-NEXT:    [[ARRAYIDX_I885:%.*]] = getelementptr inbounds i16, i16* [[PX_31055]], i32 3
-; CHECK-T2-NEXT:    [[TMP11:%.*]] = load i16, i16* [[ARRAYIDX_I885]], align 2
-; CHECK-T2-NEXT:    [[TMP12:%.*]] = load i16, i16* [[ADD_PTR_I912]], align 2
+; CHECK-T2-NEXT:    [[TMP8:%.*]] = load i16, i16* [[ARRAYIDX_I885]], align 2
+; CHECK-T2-NEXT:    [[TMP9:%.*]] = load i16, i16* [[ADD_PTR_I912]], align 2
 ; CHECK-T2-NEXT:    [[ADD_PTR_I890]] = getelementptr inbounds i16, i16* [[PX_31055]], i32 4
 ; CHECK-T2-NEXT:    [[ARRAYIDX_I879:%.*]] = getelementptr inbounds i16, i16* [[PY_31056]], i32 -1
-; CHECK-T2-NEXT:    [[TMP13:%.*]] = load i16, i16* [[ARRAYIDX_I879]], align 2
-; CHECK-T2-NEXT:    [[TMP14:%.*]] = load i16, i16* [[ADD_PTR_I906]], align 2
+; CHECK-T2-NEXT:    [[TMP10:%.*]] = load i16, i16* [[ARRAYIDX_I879]], align 2
+; CHECK-T2-NEXT:    [[TMP11:%.*]] = load i16, i16* [[ADD_PTR_I906]], align 2
 ; CHECK-T2-NEXT:    [[ADD_PTR_I884]] = getelementptr inbounds i16, i16* [[PY_31056]], i32 -4
-; CHECK-T2-NEXT:    [[SHR_I870:%.*]] = sext i16 [[TMP12]] to i32
-; CHECK-T2-NEXT:    [[SHR1_I871:%.*]] = sext i16 [[TMP13]] to i32
+; CHECK-T2-NEXT:    [[SHR_I870:%.*]] = sext i16 [[TMP9]] to i32
+; CHECK-T2-NEXT:    [[SHR1_I871:%.*]] = sext i16 [[TMP10]] to i32
 ; CHECK-T2-NEXT:    [[MUL_I872:%.*]] = mul nsw i32 [[SHR1_I871]], [[SHR_I870]]
-; CHECK-T2-NEXT:    [[SHR2_I873:%.*]] = sext i16 [[TMP11]] to i32
-; CHECK-T2-NEXT:    [[SHR4_I875:%.*]] = sext i16 [[TMP14]] to i32
+; CHECK-T2-NEXT:    [[SHR2_I873:%.*]] = sext i16 [[TMP8]] to i32
+; CHECK-T2-NEXT:    [[SHR4_I875:%.*]] = sext i16 [[TMP11]] to i32
 ; CHECK-T2-NEXT:    [[MUL5_I876:%.*]] = mul nsw i32 [[SHR4_I875]], [[SHR2_I873]]
 ; CHECK-T2-NEXT:    [[ADD_I877:%.*]] = add i32 [[ADD6_I900]], [[MUL_I872]]
 ; CHECK-T2-NEXT:    [[ADD6_I878]] = add i32 [[ADD_I877]], [[MUL5_I876]]
@@ -317,11 +311,11 @@ define dso_local arm_aapcscc void @arm_conv_fast_q15(i16* %pSrcA, i32 %srcALen,
 ; CHECK-T2-NEXT:    [[PY_41064:%.*]] = phi i16* [ [[INCDEC_PTR39:%.*]], [[WHILE_BODY36]] ], [ [[ADD_PTR32]], [[WHILE_BODY36_PREHEADER]] ]
 ; CHECK-T2-NEXT:    [[PX_41063:%.*]] = phi i16* [ [[INCDEC_PTR37:%.*]], [[WHILE_BODY36]] ], [ [[PX_3_LCSSA]], [[WHILE_BODY36_PREHEADER]] ]
 ; CHECK-T2-NEXT:    [[INCDEC_PTR37]] = getelementptr inbounds i16, i16* [[PX_41063]], i32 1
-; CHECK-T2-NEXT:    [[TMP15:%.*]] = load i16, i16* [[PX_41063]], align 2
-; CHECK-T2-NEXT:    [[CONV38:%.*]] = sext i16 [[TMP15]] to i32
+; CHECK-T2-NEXT:    [[TMP12:%.*]] = load i16, i16* [[PX_41063]], align 2
+; CHECK-T2-NEXT:    [[CONV38:%.*]] = sext i16 [[TMP12]] to i32
 ; CHECK-T2-NEXT:    [[INCDEC_PTR39]] = getelementptr inbounds i16, i16* [[PY_41064]], i32 -1
-; CHECK-T2-NEXT:    [[TMP16:%.*]] = load i16, i16* [[PY_41064]], align 2
-; CHECK-T2-NEXT:    [[CONV40:%.*]] = sext i16 [[TMP16]] to i32
+; CHECK-T2-NEXT:    [[TMP13:%.*]] = load i16, i16* [[PY_41064]], align 2
+; CHECK-T2-NEXT:    [[CONV40:%.*]] = sext i16 [[TMP13]] to i32
 ; CHECK-T2-NEXT:    [[MUL_I863:%.*]] = mul nsw i32 [[CONV40]], [[CONV38]]
 ; CHECK-T2-NEXT:    [[SHR3_I864:%.*]] = ashr i32 [[CONV38]], 16
 ; CHECK-T2-NEXT:    [[SHR4_I865:%.*]] = ashr i32 [[CONV40]], 16
@@ -336,8 +330,8 @@ define dso_local arm_aapcscc void @arm_conv_fast_q15(i16* %pSrcA, i32 %srcALen,
 ; CHECK-T2-NEXT:    br label [[WHILE_END43]]
 ; CHECK-T2:       while.end43:
 ; CHECK-T2-NEXT:    [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_1_LCSSA]], [[WHILE_END31]] ], [ [[ADD6_I868_LCSSA]], [[WHILE_END43_LOOPEXIT]] ]
-; CHECK-T2-NEXT:    [[TMP17:%.*]] = lshr i32 [[SUM_2_LCSSA]], 15
-; CHECK-T2-NEXT:    [[CONV45:%.*]] = trunc i32 [[TMP17]] to i16
+; CHECK-T2-NEXT:    [[TMP14:%.*]] = lshr i32 [[SUM_2_LCSSA]], 15
+; CHECK-T2-NEXT:    [[CONV45:%.*]] = trunc i32 [[TMP14]] to i16
 ; CHECK-T2-NEXT:    [[INCDEC_PTR46]] = getelementptr inbounds i16, i16* [[POUT_11069]], i32 1
 ; CHECK-T2-NEXT:    store i16 [[CONV45]], i16* [[POUT_11069]], align 2
 ; CHECK-T2-NEXT:    [[SUB47:%.*]] = add i32 [[COUNT_11072]], -1
diff --git a/llvm/test/CodeGen/ARM/indvar-unroll-imm-cost.ll b/llvm/test/Transforms/IndVarSimplify/ARM/indvar-unroll-imm-cost.ll
similarity index 100%
rename from llvm/test/CodeGen/ARM/indvar-unroll-imm-cost.ll
rename to llvm/test/Transforms/IndVarSimplify/ARM/indvar-unroll-imm-cost.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/ARM/lit.local.cfg b/llvm/test/Transforms/IndVarSimplify/ARM/lit.local.cfg
new file mode 100644
index 0000000000000..236e1d3441665
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/ARM/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'ARM' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/IndVarSimplify/2009-04-14-shorten_iv_vars.ll b/llvm/test/Transforms/IndVarSimplify/X86/2009-04-14-shorten_iv_vars.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/2009-04-14-shorten_iv_vars.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/2009-04-14-shorten_iv_vars.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/2009-04-15-shorten-iv-vars-2.ll b/llvm/test/Transforms/IndVarSimplify/X86/2009-04-15-shorten-iv-vars-2.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/2009-04-15-shorten-iv-vars-2.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/2009-04-15-shorten-iv-vars-2.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/2011-11-15-multiexit.ll b/llvm/test/Transforms/IndVarSimplify/X86/2011-11-15-multiexit.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/2011-11-15-multiexit.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/2011-11-15-multiexit.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/deterministic-scev-verify.ll b/llvm/test/Transforms/IndVarSimplify/X86/deterministic-scev-verify.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/deterministic-scev-verify.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/deterministic-scev-verify.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/eliminate-trunc.ll b/llvm/test/Transforms/IndVarSimplify/X86/eliminate-trunc.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/eliminate-trunc.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/eliminate-trunc.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/huge_muls.ll b/llvm/test/Transforms/IndVarSimplify/X86/huge_muls.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/huge_muls.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/huge_muls.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/indvar-debug-value.ll b/llvm/test/Transforms/IndVarSimplify/X86/indvar-debug-value.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/indvar-debug-value.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/indvar-debug-value.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/indvar-debug-value2.ll b/llvm/test/Transforms/IndVarSimplify/X86/indvar-debug-value2.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/indvar-debug-value2.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/indvar-debug-value2.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/inner-loop-by-latch-cond.ll b/llvm/test/Transforms/IndVarSimplify/X86/inner-loop-by-latch-cond.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/inner-loop-by-latch-cond.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/inner-loop-by-latch-cond.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/inner-loop.ll b/llvm/test/Transforms/IndVarSimplify/X86/inner-loop.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/inner-loop.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/inner-loop.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/iv-widen.ll b/llvm/test/Transforms/IndVarSimplify/X86/iv-widen.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/iv-widen.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/iv-widen.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/X86/lit.local.cfg b/llvm/test/Transforms/IndVarSimplify/X86/lit.local.cfg
new file mode 100644
index 0000000000000..c8625f4d9d248
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/X86/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'X86' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/IndVarSimplify/loop-invariant-conditions.ll b/llvm/test/Transforms/IndVarSimplify/X86/loop-invariant-conditions.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/loop-invariant-conditions.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/loop-invariant-conditions.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/loop_evaluate10.ll b/llvm/test/Transforms/IndVarSimplify/X86/loop_evaluate10.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/loop_evaluate10.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/loop_evaluate10.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/overflow-intrinsics.ll b/llvm/test/Transforms/IndVarSimplify/X86/overflow-intrinsics.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/overflow-intrinsics.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/overflow-intrinsics.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/polynomial-expand.ll b/llvm/test/Transforms/IndVarSimplify/X86/polynomial-expand.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/polynomial-expand.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/polynomial-expand.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/pr24356.ll b/llvm/test/Transforms/IndVarSimplify/X86/pr24356.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/pr24356.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/pr24356.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/pr24804.ll b/llvm/test/Transforms/IndVarSimplify/X86/pr24804.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/pr24804.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/pr24804.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/pr24956.ll b/llvm/test/Transforms/IndVarSimplify/X86/pr24956.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/pr24956.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/pr24956.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/pr25047.ll b/llvm/test/Transforms/IndVarSimplify/X86/pr25047.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/pr25047.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/pr25047.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/pr25051.ll b/llvm/test/Transforms/IndVarSimplify/X86/pr25051.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/pr25051.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/pr25051.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/pr25360.ll b/llvm/test/Transforms/IndVarSimplify/X86/pr25360.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/pr25360.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/pr25360.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/pr25421.ll b/llvm/test/Transforms/IndVarSimplify/X86/pr25421.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/pr25421.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/pr25421.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/pr25576.ll b/llvm/test/Transforms/IndVarSimplify/X86/pr25576.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/pr25576.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/pr25576.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/pr26973.ll b/llvm/test/Transforms/IndVarSimplify/X86/pr26973.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/pr26973.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/pr26973.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/pr27133.ll b/llvm/test/Transforms/IndVarSimplify/X86/pr27133.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/pr27133.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/pr27133.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/pr28935.ll b/llvm/test/Transforms/IndVarSimplify/X86/pr28935.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/pr28935.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/pr28935.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/pr35406.ll b/llvm/test/Transforms/IndVarSimplify/X86/pr35406.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/pr35406.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/pr35406.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/replace-iv-with-loop-invariant.ll b/llvm/test/Transforms/IndVarSimplify/X86/replace-iv-with-loop-invariant.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/replace-iv-with-loop-invariant.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/replace-iv-with-loop-invariant.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/scev-phi-debug-info.ll b/llvm/test/Transforms/IndVarSimplify/X86/scev-phi-debug-info.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/scev-phi-debug-info.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/scev-phi-debug-info.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/uglygep.ll b/llvm/test/Transforms/IndVarSimplify/X86/uglygep.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/uglygep.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/uglygep.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/variable-stride-ivs-1.ll b/llvm/test/Transforms/IndVarSimplify/X86/variable-stride-ivs-1.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/variable-stride-ivs-1.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/variable-stride-ivs-1.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/verify-scev.ll b/llvm/test/Transforms/IndVarSimplify/X86/verify-scev.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/verify-scev.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/verify-scev.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/widen-nsw.ll b/llvm/test/Transforms/IndVarSimplify/X86/widen-nsw.ll
similarity index 100%
rename from llvm/test/Transforms/IndVarSimplify/widen-nsw.ll
rename to llvm/test/Transforms/IndVarSimplify/X86/widen-nsw.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/do-recompute-if-cheap.ll b/llvm/test/Transforms/IndVarSimplify/dont-recompute.ll
similarity index 91%
rename from llvm/test/Transforms/IndVarSimplify/do-recompute-if-cheap.ll
rename to llvm/test/Transforms/IndVarSimplify/dont-recompute.ll
index 869f55e1d6dc5..91abc06529a13 100644
--- a/llvm/test/Transforms/IndVarSimplify/do-recompute-if-cheap.ll
+++ b/llvm/test/Transforms/IndVarSimplify/dont-recompute.ll
@@ -36,8 +36,8 @@ define void @test(i32 %m) nounwind uwtable {
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 186
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
 ; CHECK:       for.end:
-; CHECK-NEXT:    [[TMP0:%.*]] = mul i32 [[M]], 186
-; CHECK-NEXT:    tail call void @func(i32 [[TMP0]])
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ]
+; CHECK-NEXT:    tail call void @func(i32 [[ADD_LCSSA]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -70,8 +70,8 @@ define i32 @test2(i32 %m) nounwind uwtable {
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 186
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
 ; CHECK:       for.end:
-; CHECK-NEXT:    [[TMP0:%.*]] = mul i32 [[M]], 186
-; CHECK-NEXT:    ret i32 [[TMP0]]
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ]
+; CHECK-NEXT:    ret i32 [[ADD_LCSSA]]
 ;
 entry:
   br label %for.body
@@ -102,8 +102,8 @@ define void @test3(i32 %m) nounwind uwtable {
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 186
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
 ; CHECK:       for.end:
-; CHECK-NEXT:    [[TMP0:%.*]] = mul i32 [[M]], 186
-; CHECK-NEXT:    tail call void @func(i32 [[TMP0]])
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ]
+; CHECK-NEXT:    tail call void @func(i32 [[ADD_LCSSA]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -142,8 +142,8 @@ define void @test4(i32 %m) nounwind uwtable {
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 186
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
 ; CHECK:       for.end:
-; CHECK-NEXT:    [[TMP0:%.*]] = mul i32 [[M]], 186
-; CHECK-NEXT:    [[SOFT_USE:%.*]] = add i32 [[TMP0]], 123
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[SOFT_USE:%.*]] = add i32 [[ADD_LCSSA]], 123
 ; CHECK-NEXT:    tail call void @func(i32 [[SOFT_USE]])
 ; CHECK-NEXT:    ret void
 ;
@@ -179,8 +179,8 @@ define void @test5(i32 %m) nounwind uwtable {
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 186
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
 ; CHECK:       for.end:
-; CHECK-NEXT:    [[TMP0:%.*]] = mul i32 [[M]], 186
-; CHECK-NEXT:    tail call void @func(i32 [[TMP0]])
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ]
+; CHECK-NEXT:    tail call void @func(i32 [[ADD_LCSSA]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -216,8 +216,8 @@ define void @test6(i32 %m, i32* %p) nounwind uwtable {
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 186
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
 ; CHECK:       for.end:
-; CHECK-NEXT:    [[TMP0:%.*]] = mul i32 [[M]], 186
-; CHECK-NEXT:    tail call void @func(i32 [[TMP0]])
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ]
+; CHECK-NEXT:    tail call void @func(i32 [[ADD_LCSSA]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/IndVarSimplify/elim-extend.ll b/llvm/test/Transforms/IndVarSimplify/elim-extend.ll
index 3387cfa9bbbd0..14c9c99903ce4 100644
--- a/llvm/test/Transforms/IndVarSimplify/elim-extend.ll
+++ b/llvm/test/Transforms/IndVarSimplify/elim-extend.ll
@@ -143,7 +143,8 @@ define void @nestedIV(i8* %address, i32 %limit) nounwind {
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], [[TMP0]]
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[INNERLOOP]], label [[INNEREXIT:%.*]]
 ; CHECK:       innerexit:
-; CHECK-NEXT:    [[TMP4:%.*]] = trunc i64 [[TMP0]] to i32
+; CHECK-NEXT:    [[INNERCOUNT_LCSSA_WIDE:%.*]] = phi i64 [ [[INDVARS_IV_NEXT]], [[INNERLOOP]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i64 [[INNERCOUNT_LCSSA_WIDE]] to i32
 ; CHECK-NEXT:    br label [[OUTERMERGE]]
 ; CHECK:       outermerge:
 ; CHECK-NEXT:    [[INNERCOUNT_MERGE]] = phi i32 [ [[TMP4]], [[INNEREXIT]] ], [ [[INNERCOUNT]], [[INNERPREHEADER]] ]
diff --git a/llvm/test/Transforms/IndVarSimplify/lrev-existing-umin.ll b/llvm/test/Transforms/IndVarSimplify/lrev-existing-umin.ll
index 5fc2b3068f196..6ef7a5ce9e263 100644
--- a/llvm/test/Transforms/IndVarSimplify/lrev-existing-umin.ll
+++ b/llvm/test/Transforms/IndVarSimplify/lrev-existing-umin.ll
@@ -26,7 +26,8 @@ define void @f(i32 %length.i.88, i32 %length.i, i8* %tmp12, i32 %tmp10, i8* %tmp
 ; CHECK-NEXT:    [[TMP23:%.*]] = icmp slt i32 [[TMP22]], [[TMP14]]
 ; CHECK-NEXT:    br i1 [[TMP23]], label [[NOT_ZERO11]], label [[MAIN_EXIT_SELECTOR:%.*]]
 ; CHECK:       main.exit.selector:
-; CHECK-NEXT:    [[TMP24:%.*]] = icmp slt i32 [[TMP14]], [[LENGTH_I]]
+; CHECK-NEXT:    [[TMP22_LCSSA:%.*]] = phi i32 [ [[TMP22]], [[NOT_ZERO11]] ]
+; CHECK-NEXT:    [[TMP24:%.*]] = icmp slt i32 [[TMP22_LCSSA]], [[LENGTH_I]]
 ; CHECK-NEXT:    br i1 [[TMP24]], label [[NOT_ZERO11_POSTLOOP]], label [[LEAVE:%.*]]
 ; CHECK:       leave:
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/IndVarSimplify/pr28705.ll b/llvm/test/Transforms/IndVarSimplify/pr28705.ll
index b431a28a4888d..a6fed805dc65b 100644
--- a/llvm/test/Transforms/IndVarSimplify/pr28705.ll
+++ b/llvm/test/Transforms/IndVarSimplify/pr28705.ll
@@ -16,14 +16,14 @@ define void @foo(i32 %sub.ptr.div.i, i8* %ref.i1174) local_unnamed_addr {
 ; CHECK:       for.body650.lr.ph:
 ; CHECK-NEXT:    br label [[FOR_BODY650:%.*]]
 ; CHECK:       loopexit:
-; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[DOTSROA_SPECULATED]], 1
+; CHECK-NEXT:    [[INC_I_I_LCSSA:%.*]] = phi i32 [ [[INC_I_I:%.*]], [[FOR_BODY650]] ]
 ; CHECK-NEXT:    br label [[XZ_EXIT]]
 ; CHECK:       XZ.exit:
-; CHECK-NEXT:    [[DB_SROA_9_0_LCSSA:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[TMP0]], [[LOOPEXIT:%.*]] ]
+; CHECK-NEXT:    [[DB_SROA_9_0_LCSSA:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[INC_I_I_LCSSA]], [[LOOPEXIT:%.*]] ]
 ; CHECK-NEXT:    br label [[END:%.*]]
 ; CHECK:       for.body650:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[FOR_BODY650_LR_PH]] ], [ [[INC655:%.*]], [[FOR_BODY650]] ]
-; CHECK-NEXT:    [[IV2:%.*]] = phi i32 [ 1, [[FOR_BODY650_LR_PH]] ], [ [[INC_I_I:%.*]], [[FOR_BODY650]] ]
+; CHECK-NEXT:    [[IV2:%.*]] = phi i32 [ 1, [[FOR_BODY650_LR_PH]] ], [ [[INC_I_I]], [[FOR_BODY650]] ]
 ; CHECK-NEXT:    [[ARRAYIDX_I_I1105:%.*]] = getelementptr inbounds i8, i8* [[REF_I1174:%.*]], i32 [[IV2]]
 ; CHECK-NEXT:    store i8 7, i8* [[ARRAYIDX_I_I1105]], align 1
 ; CHECK-NEXT:    [[INC_I_I]] = add nuw nsw i32 [[IV2]], 1
diff --git a/llvm/test/Transforms/IndVarSimplify/pr39673.ll b/llvm/test/Transforms/IndVarSimplify/pr39673.ll
index 4ae9f4532d50c..7fb90a90071ac 100644
--- a/llvm/test/Transforms/IndVarSimplify/pr39673.ll
+++ b/llvm/test/Transforms/IndVarSimplify/pr39673.ll
@@ -72,8 +72,8 @@ define i16 @dom_argument(i16 %arg1, i16 %arg2) {
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i16 [[L2_ADD]], 2
 ; CHECK-NEXT:    br i1 [[CMP2]], label [[LOOP2]], label [[LOOP2_END:%.*]]
 ; CHECK:       loop2.end:
-; CHECK-NEXT:    [[TMP0:%.*]] = add i16 [[ARG2]], 2
-; CHECK-NEXT:    ret i16 [[TMP0]]
+; CHECK-NEXT:    [[K2_ADD_LCSSA:%.*]] = phi i16 [ [[K2_ADD]], [[LOOP2]] ]
+; CHECK-NEXT:    ret i16 [[K2_ADD_LCSSA]]
 ;
 entry:
   br label %loop1
@@ -121,8 +121,8 @@ define i16 @dummy_phi_outside_loop(i16 %arg) {
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i16 [[L2_ADD]], 2
 ; CHECK-NEXT:    br i1 [[CMP2]], label [[LOOP2]], label [[LOOP2_END:%.*]]
 ; CHECK:       loop2.end:
-; CHECK-NEXT:    [[TMP0:%.*]] = add i16 [[DUMMY]], 2
-; CHECK-NEXT:    ret i16 [[TMP0]]
+; CHECK-NEXT:    [[K2_ADD_LCSSA:%.*]] = phi i16 [ [[K2_ADD]], [[LOOP2]] ]
+; CHECK-NEXT:    ret i16 [[K2_ADD_LCSSA]]
 ;
 entry:
   br label %loop2.preheader
@@ -166,8 +166,8 @@ define i16 @neg_loop_carried(i16 %arg) {
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i16 [[L2_ADD]], 2
 ; CHECK-NEXT:    br i1 [[CMP2]], label [[LOOP2]], label [[LOOP2_END:%.*]]
 ; CHECK:       loop2.end:
-; CHECK-NEXT:    [[TMP1:%.*]] = add i16 [[TMP0]], 2
-; CHECK-NEXT:    ret i16 [[TMP1]]
+; CHECK-NEXT:    [[K2_ADD_LCSSA:%.*]] = phi i16 [ [[K2_ADD]], [[LOOP2]] ]
+; CHECK-NEXT:    ret i16 [[K2_ADD_LCSSA]]
 ;
 entry:
   br label %loop1
diff --git a/llvm/test/Transforms/Inline/AMDGPU/inline-target-cpu.ll b/llvm/test/Transforms/Inline/AMDGPU/inline-target-cpu.ll
index 87330c72d442c..961933bdd6efd 100644
--- a/llvm/test/Transforms/Inline/AMDGPU/inline-target-cpu.ll
+++ b/llvm/test/Transforms/Inline/AMDGPU/inline-target-cpu.ll
@@ -15,89 +15,74 @@ define i32 @target_cpu_call_no_target_cpu() #1 {
 
 ; CHECK-LABEL: @target_cpu_target_features_call_no_target_cpu(
 ; CHECK-NEXT: ret i32 0
-define i32 @target_cpu_target_features_call_no_target_cpu() #2 {
+define i32 @target_cpu_target_features_call_no_target_cpu() {
   %call = call i32 @func_no_target_cpu()
   ret i32 %call
 }
 
-; CHECK-LABEL: @fp32_denormals(
-define i32 @fp32_denormals() #3 {
-  ret i32 0
-}
-
-; CHECK-LABEL: @no_fp32_denormals_call_f32_denormals(
-; CHECK-NEXT: call i32 @fp32_denormals()
-define i32 @no_fp32_denormals_call_f32_denormals() #4 {
-  %call = call i32 @fp32_denormals()
-  ret i32 %call
-}
-
 ; Make sure gfx9 can call unspecified functions because of movrel
 ; feature change.
 ; CHECK-LABEL: @gfx9_target_features_call_no_target_cpu(
 ; CHECK-NEXT: ret i32 0
-define i32 @gfx9_target_features_call_no_target_cpu() #5 {
+define i32 @gfx9_target_features_call_no_target_cpu() #2 {
   %call = call i32 @func_no_target_cpu()
   ret i32 %call
 }
 
-define i32 @func_no_halfrate64ops() #6 {
+define i32 @func_no_halfrate64ops() #3 {
   ret i32 0
 }
 
-define i32 @func_with_halfrate64ops() #7 {
+define i32 @func_with_halfrate64ops() #4 {
   ret i32 0
 }
 
 ; CHECK-LABEL: @call_func_without_halfrate64ops(
 ; CHECK-NEXT: ret i32 0
-define i32 @call_func_without_halfrate64ops() #7 {
+define i32 @call_func_without_halfrate64ops() #4 {
   %call = call i32 @func_no_halfrate64ops()
   ret i32 %call
 }
 
 ; CHECK-LABEL: @call_func_with_halfrate64ops(
 ; CHECK-NEXT: ret i32 0
-define i32 @call_func_with_halfrate64ops() #6 {
+define i32 @call_func_with_halfrate64ops() #3 {
   %call = call i32 @func_with_halfrate64ops()
   ret i32 %call
 }
 
-define i32 @func_no_loadstoreopt() #8 {
+define i32 @func_no_loadstoreopt() #5 {
   ret i32 0
 }
 
-define i32 @func_with_loadstoreopt() #9 {
+define i32 @func_with_loadstoreopt() #6 {
   ret i32 0
 }
 
 ; CHECK-LABEL: @call_func_without_loadstoreopt(
 ; CHECK-NEXT: ret i32 0
-define i32 @call_func_without_loadstoreopt() #9 {
+define i32 @call_func_without_loadstoreopt() #6 {
   %call = call i32 @func_no_loadstoreopt()
   ret i32 %call
 }
 
-define i32 @enable_codeobjectv3() #10 {
+define i32 @enable_codeobjectv3() #7 {
   ret i32 999
 }
 
 ; CHECK-LABEL: @disable_codeobjectv3_call_codeobjectv3(
 ; CHECK-NEXT: ret i32 999
-define i32 @disable_codeobjectv3_call_codeobjectv3() #11 {
+define i32 @disable_codeobjectv3_call_codeobjectv3() #8 {
   %call = call i32 @enable_codeobjectv3()
   ret i32 %call
 }
 
 attributes #0 = { nounwind }
 attributes #1 = { nounwind "target-cpu"="fiji" }
-attributes #2 = { nounwind "target-cpu"="fiji" "target-features"="+fp32-denormals" }
-attributes #3 = { nounwind "target-features"="+fp32-denormals" }
-attributes #4 = { nounwind "target-features"="-fp32-denormals" }
-attributes #5 = { nounwind "target-cpu"="gfx900" }
-attributes #6 = { nounwind "target-features"="-half-rate-64-ops" }
-attributes #7 = { nounwind "target-features"="+half-rate-64-ops" }
-attributes #8 = { nounwind "target-features"="-load-store-opt" }
-attributes #9 = { nounwind "target-features"="+load-store-opt" }
-attributes #10 = { nounwind "target-features"="+code-object-v3" }
-attributes #11 = { nounwind "target-features"="-code-object-v3" }
+attributes #2 = { nounwind "target-cpu"="gfx900" }
+attributes #3 = { nounwind "target-features"="-half-rate-64-ops" }
+attributes #4 = { nounwind "target-features"="+half-rate-64-ops" }
+attributes #5 = { nounwind "target-features"="-load-store-opt" }
+attributes #6 = { nounwind "target-features"="+load-store-opt" }
+attributes #7 = { nounwind "target-features"="+code-object-v3" }
+attributes #8 = { nounwind "target-features"="-code-object-v3" }
diff --git a/llvm/test/Transforms/Inline/inline_call.ll b/llvm/test/Transforms/Inline/inline_call.ll
new file mode 100644
index 0000000000000..fb000f0c805f2
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline_call.ll
@@ -0,0 +1,71 @@
+; Check the optimizer doesn't crash at inlining the function top and all of its callees are inlined.
+; RUN: opt < %s -O3 -S | FileCheck %s
+
+define dso_local void (...)* @second(i8** %p) {
+entry:
+  %p.addr = alloca i8**, align 8
+  store i8** %p, i8*** %p.addr, align 8
+  %tmp = load i8**, i8*** %p.addr, align 8
+  %tmp1 = load i8*, i8** %tmp, align 8
+  %tmp2 = bitcast i8* %tmp1 to void (...)*
+  ret void (...)* %tmp2
+}
+
+define dso_local void @top()  {
+entry:
+  ; CHECK: {{.*}} = {{.*}} call {{.*}} @ext
+  ; CHECK-NOT: {{.*}} = {{.*}} call {{.*}} @third
+  ; CHECK-NOT: {{.*}} = {{.*}} call {{.*}} @second
+  ; CHECK-NOT: {{.*}} = {{.*}} call {{.*}} @wrapper
+  %q = alloca i8*, align 8
+  store i8* bitcast (void ()* @third to i8*), i8** %q, align 8
+  %tmp = call void (...)* @second(i8** %q)
+  ; The call to 'wrapper' here is to ensure that its function attributes
+  ; i.e., returning its parameter and having no side effect, will be decuded
+  ; before the next round of inlining happens to 'top' to expose the bug.
+  %call =  call void (...)* @wrapper(void (...)* %tmp) 
+  ; The indirect call here is to confuse the alias analyzer so that
+  ; an incomplete graph will be built during the first round of inlining.
+  ; This allows the current function to be processed before the actual 
+  ; callee, i.e., the function 'run', is processed. Once it's simplified to 
+  ; a direct call, it also enables an additional round of inlining with all
+  ; function attributes deduced. 
+  call void (...) %call()
+  ret void
+}
+
+define dso_local void (...)* @gen() {
+entry:
+  %call = call void (...)* (...) @ext()
+  ret void (...)* %call
+}
+
+declare dso_local void (...)* @ext(...) 
+
+define dso_local void (...)* @wrapper(void (...)* %fn) {
+entry:
+  ret void (...)* %fn
+}
+
+define dso_local void @run(void (...)* %fn) {
+entry:
+  %fn.addr = alloca void (...)*, align 8
+  %f = alloca void (...)*, align 8
+  store void (...)* %fn, void (...)** %fn.addr, align 8
+  %tmp = load void (...)*, void (...)** %fn.addr, align 8
+  %call = call void (...)* @wrapper(void (...)* %tmp)
+  store void (...)* %call, void (...)** %f, align 8
+  %tmp1 = load void (...)*, void (...)** %f, align 8
+  call void (...) %tmp1()
+  ret void
+}
+
+define dso_local void @third() {
+entry:
+  %f = alloca void (...)*, align 8
+  %call = call void (...)* @gen()
+  store void (...)* %call, void (...)** %f, align 8
+  %tmp = load void (...)*, void (...)** %f, align 8
+  call void @run(void (...)* %tmp)
+  ret void
+}
\ No newline at end of file
diff --git a/llvm/test/Transforms/Inline/ret_attr_update.ll b/llvm/test/Transforms/Inline/ret_attr_update.ll
new file mode 100644
index 0000000000000..647029e13459a
--- /dev/null
+++ b/llvm/test/Transforms/Inline/ret_attr_update.ll
@@ -0,0 +1,223 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -inline-threshold=0 -always-inline -S | FileCheck %s
+; RUN: opt < %s -passes=always-inline -S | FileCheck %s
+
+declare i8* @foo(i8*) argmemonly nounwind
+
+define i8* @callee(i8 *%p) alwaysinline {
+; CHECK-LABEL: @callee(
+; CHECK-NEXT:    [[R:%.*]] = call i8* @foo(i8* noalias [[P:%.*]])
+; CHECK-NEXT:    ret i8* [[R]]
+;
+  %r = call i8* @foo(i8* noalias %p)
+  ret i8* %r
+}
+
+define i8* @caller(i8* %ptr, i64 %x) {
+; CHECK-LABEL: @caller(
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, i8* [[PTR:%.*]], i64 [[X:%.*]]
+; CHECK-NEXT:    [[R_I:%.*]] = call nonnull i8* @foo(i8* noalias [[GEP]])
+; CHECK-NEXT:    ret i8* [[R_I]]
+;
+  %gep = getelementptr inbounds i8, i8* %ptr, i64 %x
+  %p = call nonnull i8* @callee(i8* %gep)
+  ret i8* %p
+}
+
+declare void @llvm.experimental.guard(i1,...)
+; Cannot add nonnull attribute to foo
+; because the guard is a throwing call
+define internal i8* @callee_with_throwable(i8* %p) alwaysinline {
+  %r = call i8* @foo(i8* %p)
+  %cond = icmp ne i8* %r, null
+  call void (i1, ...) @llvm.experimental.guard(i1 %cond) [ "deopt"() ]
+  ret i8* %r
+}
+
+declare i8* @bar(i8*) readonly nounwind
+; Here also we cannot add nonnull attribute to the call bar.
+define internal i8* @callee_with_explicit_control_flow(i8* %p) alwaysinline {
+  %r = call i8* @bar(i8* %p)
+  %cond = icmp ne i8* %r, null
+  br i1 %cond, label %ret, label %orig
+
+ret:
+  ret i8* %r
+
+orig:
+  ret i8* %p
+}
+
+define i8* @caller2(i8* %ptr, i64 %x, i1 %cond) {
+; CHECK-LABEL: @caller2(
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, i8* [[PTR:%.*]], i64 [[X:%.*]]
+; CHECK-NEXT:    [[R_I:%.*]] = call i8* @foo(i8* [[GEP]])
+; CHECK-NEXT:    [[COND_I:%.*]] = icmp ne i8* [[R_I]], null
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[COND_I]]) [ "deopt"() ]
+; CHECK-NEXT:    [[R_I1:%.*]] = call i8* @bar(i8* [[GEP]])
+; CHECK-NEXT:    [[COND_I2:%.*]] = icmp ne i8* [[R_I1]], null
+; CHECK-NEXT:    br i1 [[COND_I2]], label [[RET_I:%.*]], label [[ORIG_I:%.*]]
+; CHECK:       ret.i:
+; CHECK-NEXT:    br label [[CALLEE_WITH_EXPLICIT_CONTROL_FLOW_EXIT:%.*]]
+; CHECK:       orig.i:
+; CHECK-NEXT:    br label [[CALLEE_WITH_EXPLICIT_CONTROL_FLOW_EXIT]]
+; CHECK:       callee_with_explicit_control_flow.exit:
+; CHECK-NEXT:    [[Q3:%.*]] = phi i8* [ [[R_I1]], [[RET_I]] ], [ [[GEP]], [[ORIG_I]] ]
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[PRET:%.*]], label [[QRET:%.*]]
+; CHECK:       pret:
+; CHECK-NEXT:    ret i8* [[R_I]]
+; CHECK:       qret:
+; CHECK-NEXT:    ret i8* [[Q3]]
+;
+  %gep = getelementptr inbounds i8, i8* %ptr, i64 %x
+  %p = call nonnull i8* @callee_with_throwable(i8* %gep)
+  %q = call nonnull i8* @callee_with_explicit_control_flow(i8* %gep)
+  br i1 %cond, label %pret, label %qret
+
+pret:
+  ret i8* %p
+
+qret:
+  ret i8* %q
+}
+
+define internal i8* @callee3(i8 *%p) alwaysinline {
+  %r = call noalias i8* @foo(i8* %p)
+  ret i8* %r
+}
+
+; add the deref attribute to the existing attributes on foo.
+define i8* @caller3(i8* %ptr, i64 %x) {
+; CHECK-LABEL: @caller3(
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, i8* [[PTR:%.*]], i64 [[X:%.*]]
+; CHECK-NEXT:    [[R_I:%.*]] = call noalias dereferenceable_or_null(12) i8* @foo(i8* [[GEP]])
+; CHECK-NEXT:    ret i8* [[R_I]]
+;
+  %gep = getelementptr inbounds i8, i8* %ptr, i64 %x
+  %p = call dereferenceable_or_null(12) i8* @callee3(i8* %gep)
+  ret i8* %p
+}
+
+declare i8* @inf_loop_call(i8*) nounwind
+; We cannot propagate attributes to foo because we do not know whether inf_loop_call
+; will return execution.
+define internal i8* @callee_with_sideeffect_callsite(i8* %p) alwaysinline {
+  %r = call i8* @foo(i8* %p)
+  %v = call i8* @inf_loop_call(i8* %p)
+  ret i8* %r
+}
+
+; do not add deref attribute to foo
+define i8* @test4(i8* %ptr, i64 %x) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, i8* [[PTR:%.*]], i64 [[X:%.*]]
+; CHECK-NEXT:    [[R_I:%.*]] = call i8* @foo(i8* [[GEP]])
+; CHECK-NEXT:    [[V_I:%.*]] = call i8* @inf_loop_call(i8* [[GEP]])
+; CHECK-NEXT:    ret i8* [[R_I]]
+;
+  %gep = getelementptr inbounds i8, i8* %ptr, i64 %x
+  %p = call dereferenceable_or_null(12) i8* @callee_with_sideeffect_callsite(i8* %gep)
+  ret i8* %p
+}
+
+declare i8* @baz(i8*) nounwind readonly
+define internal i8* @callee5(i8* %p) alwaysinline {
+  %r = call i8* @foo(i8* %p)
+  %v = call i8* @baz(i8* %p)
+  ret i8* %r
+}
+
+; add the deref attribute to foo.
+define i8* @test5(i8* %ptr, i64 %x) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, i8* [[PTR:%.*]], i64 [[X:%.*]]
+; CHECK-NEXT:    [[R_I:%.*]] = call dereferenceable_or_null(12) i8* @foo(i8* [[GEP]])
+; CHECK-NEXT:    [[V_I:%.*]] = call i8* @baz(i8* [[GEP]])
+; CHECK-NEXT:    ret i8* [[R_I]]
+;
+  %gep = getelementptr inbounds i8, i8* %ptr, i64 %x
+  %s = call dereferenceable_or_null(12) i8* @callee5(i8* %gep)
+  ret i8* %s
+}
+
+; deref attributes have different values on the callee and the call feeding into
+; the return.
+; AttrBuilder chooses the already existing value and does not overwrite it.
+define internal i8* @callee6(i8* %p) alwaysinline {
+  %r = call dereferenceable_or_null(16) i8* @foo(i8* %p)
+  %v = call i8* @baz(i8* %p)
+  ret i8* %r
+}
+
+
+define i8* @test6(i8* %ptr, i64 %x) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, i8* [[PTR:%.*]], i64 [[X:%.*]]
+; CHECK-NEXT:    [[R_I:%.*]] = call dereferenceable_or_null(16) i8* @foo(i8* [[GEP]])
+; CHECK-NEXT:    [[V_I:%.*]] = call i8* @baz(i8* [[GEP]])
+; CHECK-NEXT:    ret i8* [[R_I]]
+;
+  %gep = getelementptr inbounds i8, i8* %ptr, i64 %x
+  %s = call dereferenceable_or_null(12) i8* @callee6(i8* %gep)
+  ret i8* %s
+}
+
+; We add the attributes from the callee to both the calls below.
+define internal i8* @callee7(i8 *%ptr, i1 %cond) alwaysinline {
+  br i1 %cond, label %pass, label %fail
+
+pass:
+  %r = call i8* @foo(i8* noalias %ptr)
+  ret i8* %r
+
+fail:
+  %s = call i8* @baz(i8* %ptr)
+  ret i8* %s
+}
+
+define void @test7(i8* %ptr, i64 %x, i1 %cond) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, i8* [[PTR:%.*]], i64 [[X:%.*]]
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[PASS_I:%.*]], label [[FAIL_I:%.*]]
+; CHECK:       pass.i:
+; CHECK-NEXT:    [[R_I:%.*]] = call nonnull i8* @foo(i8* noalias [[GEP]])
+; CHECK-NEXT:    br label [[CALLEE7_EXIT:%.*]]
+; CHECK:       fail.i:
+; CHECK-NEXT:    [[S_I:%.*]] = call nonnull i8* @baz(i8* [[GEP]])
+; CHECK-NEXT:    br label [[CALLEE7_EXIT]]
+; CHECK:       callee7.exit:
+; CHECK-NEXT:    [[T1:%.*]] = phi i8* [ [[R_I]], [[PASS_I]] ], [ [[S_I]], [[FAIL_I]] ]
+; CHECK-NEXT:    call void @snort(i8* [[T1]])
+; CHECK-NEXT:    ret void
+;
+
+  %gep = getelementptr inbounds i8, i8* %ptr, i64 %x
+  %t = call nonnull i8* @callee7(i8* %gep, i1 %cond)
+  call void @snort(i8* %t)
+  ret void
+}
+declare void @snort(i8*)
+
+declare i32 @intrinsic(i8*) nounwind argmemonly
+
+define internal i32 @callee8(i8* %ptr) alwaysinline {
+  %r = call i32 @intrinsic(i8* noalias %ptr)
+  ret i32 %r
+}
+
+
+; signext is an attribute specific to the target ABI and not the
+; callee/callsite.
+; We cannot propagate that attribute to another call since it can be invalid at
+; that call.
+define i32 @test8(i8* %ptr, i64 %x) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, i8* [[PTR:%.*]], i64 [[X:%.*]]
+; CHECK-NEXT:    [[R_I:%.*]] = call i32 @intrinsic(i8* noalias [[GEP]])
+; CHECK-NEXT:    ret i32 [[R_I]]
+;
+
+  %gep = getelementptr inbounds i8, i8* %ptr, i64 %x
+  %t = call signext i32 @callee8(i8* %gep)
+  ret i32 %t
+}
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
index 1c622a32f30f7..032ca08ffc2b9 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
@@ -2378,6 +2378,65 @@ define i64 @fcmp_constant_to_rhs_olt(float %x) {
   ret i64 %result
 }
 
+; --------------------------------------------------------------------
+; llvm.amdgcn.ballot
+; --------------------------------------------------------------------
+
+declare i64 @llvm.amdgcn.ballot.i64(i1) nounwind readnone convergent
+declare i32 @llvm.amdgcn.ballot.i32(i1) nounwind readnone convergent
+
+define i64 @ballot_nocombine_64(i1 %i) {
+; CHECK-LABEL: @ballot_nocombine_64(
+; CHECK-NEXT:    %b = call i64 @llvm.amdgcn.ballot.i64(i1 %i)
+; CHECK-NEXT:    ret i64 %b
+;
+  %b = call i64 @llvm.amdgcn.ballot.i64(i1 %i)
+  ret i64 %b
+}
+
+define i64 @ballot_zero_64() {
+; CHECK-LABEL: @ballot_zero_64(
+; CHECK-NEXT:    ret i64 0
+;
+  %b = call i64 @llvm.amdgcn.ballot.i64(i1 0)
+  ret i64 %b
+}
+
+define i64 @ballot_one_64() {
+; CHECK-LABEL: @ballot_one_64(
+; CHECK-NEXT:    %b = call i64 @llvm.read_register.i64(metadata !0) [[CONVERGENT]]
+; CHECK-NEXT:    ret i64 %b
+;
+  %b = call i64 @llvm.amdgcn.ballot.i64(i1 1)
+  ret i64 %b
+}
+
+define i32 @ballot_nocombine_32(i1 %i) {
+; CHECK-LABEL: @ballot_nocombine_32(
+; CHECK-NEXT:    %b = call i32 @llvm.amdgcn.ballot.i32(i1 %i)
+; CHECK-NEXT:    ret i32 %b
+;
+  %b = call i32 @llvm.amdgcn.ballot.i32(i1 %i)
+  ret i32 %b
+}
+
+define i32 @ballot_zero_32() {
+; CHECK-LABEL: @ballot_zero_32(
+; CHECK-NEXT:    ret i32 0
+;
+  %b = call i32 @llvm.amdgcn.ballot.i32(i1 0)
+  ret i32 %b
+}
+
+define i32 @ballot_one_32() {
+; CHECK-LABEL: @ballot_one_32(
+; CHECK-NEXT:    %b = call i32 @llvm.read_register.i32(metadata !1) [[CONVERGENT]]
+; CHECK-NEXT:    ret i32 %b
+;
+  %b = call i32 @llvm.amdgcn.ballot.i32(i1 1)
+  ret i32 %b
+}
+
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.wqm.vote
 ; --------------------------------------------------------------------
diff --git a/llvm/test/Transforms/InstCombine/ExtractCast.ll b/llvm/test/Transforms/InstCombine/ExtractCast.ll
index 9a8872f2fb1f3..9dc9396837c2c 100644
--- a/llvm/test/Transforms/InstCombine/ExtractCast.ll
+++ b/llvm/test/Transforms/InstCombine/ExtractCast.ll
@@ -1,27 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instcombine -S -o - | FileCheck %s
 
-; CHECK-LABEL: @a(
 define i32 @a(<4 x i64> %I) {
-entry:
-; CHECK-NOT: trunc <4 x i64>
-        %J = trunc <4 x i64> %I to <4 x i32>
-        %K = extractelement <4 x i32> %J, i32 3
-; CHECK: extractelement <4 x i64>
-; CHECK: trunc i64
-; CHECK: ret
-        ret i32 %K
+; CHECK-LABEL: @a(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i64> [[I:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[K:%.*]] = extractelement <8 x i32> [[TMP1]], i32 6
+; CHECK-NEXT:    ret i32 [[K]]
+;
+  %J = trunc <4 x i64> %I to <4 x i32>
+  %K = extractelement <4 x i32> %J, i32 3
+  ret i32 %K
 }
 
-
-; CHECK-LABEL: @b(
 define i32 @b(<4 x float> %I) {
-entry:
-; CHECK-NOT: fptosi <4 x float>
-        %J = fptosi <4 x float> %I to <4 x i32>
-        %K = extractelement <4 x i32> %J, i32 3
-; CHECK: extractelement <4 x float>
-; CHECK: fptosi float
-; CHECK: ret
-        ret i32 %K
+; CHECK-LABEL: @b(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[I:%.*]], i32 3
+; CHECK-NEXT:    [[K:%.*]] = fptosi float [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[K]]
+;
+  %J = fptosi <4 x float> %I to <4 x i32>
+  %K = extractelement <4 x i32> %J, i32 3
+  ret i32 %K
 }
 
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-sse.ll b/llvm/test/Transforms/InstCombine/X86/x86-sse.ll
index d58b083df2b6a..d7eb0dfe716de 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-sse.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-sse.ll
@@ -119,6 +119,17 @@ define float @test_add_ss_1(float %a, float %b) {
   ret float %7
 }
 
+define float @test_add_ss_2(float %a) {
+; CHECK-LABEL: @test_add_ss_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd float [[A:%.*]], [[A]]
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %1 = insertelement <4 x float> zeroinitializer, float %a, i32 0
+  %2 = tail call <4 x float> @llvm.x86.sse.add.ss(<4 x float> %1, <4 x float> %1)
+  %3 = extractelement <4 x float> %2, i32 0
+  ret float %3
+}
+
 define float @test_sub_ss_0(float %a, float %b) {
 ; CHECK-LABEL: @test_sub_ss_0(
 ; CHECK-NEXT:    [[TMP1:%.*]] = fsub float [[A:%.*]], [[B:%.*]]
@@ -151,6 +162,17 @@ define float @test_sub_ss_2(float %a, float %b) {
   ret float %7
 }
 
+define float @test_sub_ss_3(float %a) {
+; CHECK-LABEL: @test_sub_ss_3(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub float [[A:%.*]], [[A]]
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %1 = insertelement <4 x float> zeroinitializer, float %a, i32 0
+  %2 = tail call <4 x float> @llvm.x86.sse.sub.ss(<4 x float> %1, <4 x float> %1)
+  %3 = extractelement <4 x float> %2, i32 0
+  ret float %3
+}
+
 define float @test_mul_ss_0(float %a, float %b) {
 ; CHECK-LABEL: @test_mul_ss_0(
 ; CHECK-NEXT:    [[TMP1:%.*]] = fmul float [[A:%.*]], [[B:%.*]]
@@ -183,6 +205,17 @@ define float @test_mul_ss_3(float %a, float %b) {
   ret float %7
 }
 
+define float @test_mul_ss_4(float %a) {
+; CHECK-LABEL: @test_mul_ss_4(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul float [[A:%.*]], [[A]]
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %1 = insertelement <4 x float> zeroinitializer, float %a, i32 0
+  %2 = tail call <4 x float> @llvm.x86.sse.mul.ss(<4 x float> %1, <4 x float> %1)
+  %3 = extractelement <4 x float> %2, i32 0
+  ret float %3
+}
+
 define float @test_div_ss_0(float %a, float %b) {
 ; CHECK-LABEL: @test_div_ss_0(
 ; CHECK-NEXT:    [[TMP1:%.*]] = fdiv float [[A:%.*]], [[B:%.*]]
@@ -215,6 +248,17 @@ define float @test_div_ss_1(float %a, float %b) {
   ret float %7
 }
 
+define float @test_div_ss_2(float %a) {
+; CHECK-LABEL: @test_div_ss_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv float [[A:%.*]], [[A]]
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %1 = insertelement <4 x float> zeroinitializer, float %a, i32 0
+  %2 = tail call <4 x float> @llvm.x86.sse.div.ss(<4 x float> %1, <4 x float> %1)
+  %3 = extractelement <4 x float> %2, i32 0
+  ret float %3
+}
+
 define <4 x float> @test_min_ss(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: @test_min_ss(
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.sse.min.ss(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]])
@@ -262,6 +306,19 @@ define float @test_min_ss_2(float %a, float %b) {
   ret float %7
 }
 
+define float @test_min_ss_3(float %a) {
+; CHECK-LABEL: @test_min_ss_3(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> <float undef, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>, float [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <4 x float> @llvm.x86.sse.min.ss(<4 x float> [[TMP1]], <4 x float> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; CHECK-NEXT:    ret float [[TMP3]]
+;
+  %1 = insertelement <4 x float> zeroinitializer, float %a, i32 0
+  %2 = tail call <4 x float> @llvm.x86.sse.min.ss(<4 x float> %1, <4 x float> %1)
+  %3 = extractelement <4 x float> %2, i32 0
+  ret float %3
+}
+
 define <4 x float> @test_max_ss(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: @test_max_ss(
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.sse.max.ss(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]])
@@ -309,6 +366,19 @@ define float @test_max_ss_3(float %a, float %b) {
   ret float %7
 }
 
+define float @test_max_ss_4(float %a) {
+; CHECK-LABEL: @test_max_ss_4(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> <float undef, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>, float [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <4 x float> @llvm.x86.sse.max.ss(<4 x float> [[TMP1]], <4 x float> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; CHECK-NEXT:    ret float [[TMP3]]
+;
+  %1 = insertelement <4 x float> zeroinitializer, float %a, i32 0
+  %2 = tail call <4 x float> @llvm.x86.sse.max.ss(<4 x float> %1, <4 x float> %1)
+  %3 = extractelement <4 x float> %2, i32 0
+  ret float %3
+}
+
 define <4 x float> @test_cmp_ss(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: @test_cmp_ss(
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], i8 0)
@@ -356,6 +426,19 @@ define float @test_cmp_ss_1(float %a, float %b) {
   ret float %7
 }
 
+define float @test_cmp_ss_2(float %a) {
+; CHECK-LABEL: @test_cmp_ss_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> <float undef, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>, float [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> [[TMP1]], <4 x float> [[TMP1]], i8 3)
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; CHECK-NEXT:    ret float [[TMP3]]
+;
+  %1 = insertelement <4 x float> zeroinitializer, float %a, i32 0
+  %2 = tail call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> %1, <4 x float> %1, i8 3)
+  %3 = extractelement <4 x float> %2, i32 0
+  ret float %3
+}
+
 define i32 @test_comieq_ss_0(float %a, float %b) {
 ; CHECK-LABEL: @test_comieq_ss_0(
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-sse2.ll b/llvm/test/Transforms/InstCombine/X86/x86-sse2.ll
index 721097e016f20..dee3da0b0e1c1 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-sse2.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-sse2.ll
@@ -4,7 +4,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 define double @test_sqrt_sd_0(double %a) {
 ; CHECK-LABEL: @test_sqrt_sd_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.sqrt.f64(double %a)
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.sqrt.f64(double [[A:%.*]])
 ; CHECK-NEXT:    ret double [[TMP1]]
 ;
   %1 = insertelement <2 x double> undef, double %a, i32 0
@@ -27,7 +27,7 @@ define double @test_sqrt_sd_1(double %a) {
 
 define double @test_add_sd_0(double %a, double %b) {
 ; CHECK-LABEL: @test_add_sd_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = fadd double %a, %b
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd double [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    ret double [[TMP1]]
 ;
   %1 = insertelement <2 x double> undef, double %a, i32 0
@@ -52,9 +52,20 @@ define double @test_add_sd_1(double %a, double %b) {
   ret double %6
 }
 
+define double @test_add_sd_2(double %a) {
+; CHECK-LABEL: @test_add_sd_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd double [[A:%.*]], [[A]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %1 = insertelement <2 x double> zeroinitializer, double %a, i32 0
+  %2 = tail call <2 x double> @llvm.x86.sse2.add.sd(<2 x double> %1, <2 x double> %1)
+  %3 = extractelement <2 x double> %2, i32 0
+  ret double %3
+}
+
 define double @test_sub_sd_0(double %a, double %b) {
 ; CHECK-LABEL: @test_sub_sd_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = fsub double %a, %b
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub double [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    ret double [[TMP1]]
 ;
   %1 = insertelement <2 x double> undef, double %a, i32 0
@@ -79,9 +90,20 @@ define double @test_sub_sd_1(double %a, double %b) {
   ret double %6
 }
 
+define double @test_sub_sd_2(double %a) {
+; CHECK-LABEL: @test_sub_sd_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub double [[A:%.*]], [[A]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %1 = insertelement <2 x double> zeroinitializer, double %a, i32 0
+  %2 = tail call <2 x double> @llvm.x86.sse2.sub.sd(<2 x double> %1, <2 x double> %1)
+  %3 = extractelement <2 x double> %2, i32 0
+  ret double %3
+}
+
 define double @test_mul_sd_0(double %a, double %b) {
 ; CHECK-LABEL: @test_mul_sd_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = fmul double %a, %b
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul double [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    ret double [[TMP1]]
 ;
   %1 = insertelement <2 x double> undef, double %a, i32 0
@@ -106,9 +128,20 @@ define double @test_mul_sd_1(double %a, double %b) {
   ret double %6
 }
 
+define double @test_mul_sd_2(double %a) {
+; CHECK-LABEL: @test_mul_sd_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul double [[A:%.*]], [[A]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %1 = insertelement <2 x double> zeroinitializer, double %a, i32 0
+  %2 = tail call <2 x double> @llvm.x86.sse2.mul.sd(<2 x double> %1, <2 x double> %1)
+  %3 = extractelement <2 x double> %2, i32 0
+  ret double %3
+}
+
 define double @test_div_sd_0(double %a, double %b) {
 ; CHECK-LABEL: @test_div_sd_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = fdiv double %a, %b
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv double [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    ret double [[TMP1]]
 ;
   %1 = insertelement <2 x double> undef, double %a, i32 0
@@ -133,9 +166,20 @@ define double @test_div_sd_1(double %a, double %b) {
   ret double %6
 }
 
+define double @test_div_sd_2(double %a) {
+; CHECK-LABEL: @test_div_sd_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv double [[A:%.*]], [[A]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %1 = insertelement <2 x double> zeroinitializer, double %a, i32 0
+  %2 = tail call <2 x double> @llvm.x86.sse2.div.sd(<2 x double> %1, <2 x double> %1)
+  %3 = extractelement <2 x double> %2, i32 0
+  ret double %3
+}
+
 define <2 x double> @test_min_sd(<2 x double> %a, <2 x double> %b) {
 ; CHECK-LABEL: @test_min_sd(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.sse2.min.sd(<2 x double> %a, <2 x double> %b)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.sse2.min.sd(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]])
 ; CHECK-NEXT:    ret <2 x double> [[TMP1]]
 ;
   %1 = insertelement <2 x double> %b, double 2.000000e+00, i32 1
@@ -145,8 +189,8 @@ define <2 x double> @test_min_sd(<2 x double> %a, <2 x double> %b) {
 
 define double @test_min_sd_0(double %a, double %b) {
 ; CHECK-LABEL: @test_min_sd_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double %b, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call <2 x double> @llvm.x86.sse2.min.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i32 0
 ; CHECK-NEXT:    ret double [[TMP4]]
@@ -173,9 +217,22 @@ define double @test_min_sd_1(double %a, double %b) {
   ret double %6
 }
 
+define double @test_min_sd_2(double %a) {
+; CHECK-LABEL: @test_min_sd_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> <double undef, double 0.000000e+00>, double [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <2 x double> @llvm.x86.sse2.min.sd(<2 x double> [[TMP1]], <2 x double> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[TMP2]], i32 0
+; CHECK-NEXT:    ret double [[TMP3]]
+;
+  %1 = insertelement <2 x double> zeroinitializer, double %a, i32 0
+  %2 = tail call <2 x double> @llvm.x86.sse2.min.sd(<2 x double> %1, <2 x double> %1)
+  %3 = extractelement <2 x double> %2, i32 0
+  ret double %3
+}
+
 define <2 x double> @test_max_sd(<2 x double> %a, <2 x double> %b) {
 ; CHECK-LABEL: @test_max_sd(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.sse2.max.sd(<2 x double> %a, <2 x double> %b)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.sse2.max.sd(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]])
 ; CHECK-NEXT:    ret <2 x double> [[TMP1]]
 ;
   %1 = insertelement <2 x double> %b, double 2.000000e+00, i32 1
@@ -185,8 +242,8 @@ define <2 x double> @test_max_sd(<2 x double> %a, <2 x double> %b) {
 
 define double @test_max_sd_0(double %a, double %b) {
 ; CHECK-LABEL: @test_max_sd_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double %b, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call <2 x double> @llvm.x86.sse2.max.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i32 0
 ; CHECK-NEXT:    ret double [[TMP4]]
@@ -213,9 +270,22 @@ define double @test_max_sd_1(double %a, double %b) {
   ret double %6
 }
 
+define double @test_max_sd_2(double %a) {
+; CHECK-LABEL: @test_max_sd_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> <double undef, double 0.000000e+00>, double [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <2 x double> @llvm.x86.sse2.max.sd(<2 x double> [[TMP1]], <2 x double> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[TMP2]], i32 0
+; CHECK-NEXT:    ret double [[TMP3]]
+;
+  %1 = insertelement <2 x double> zeroinitializer, double %a, i32 0
+  %2 = tail call <2 x double> @llvm.x86.sse2.max.sd(<2 x double> %1, <2 x double> %1)
+  %3 = extractelement <2 x double> %2, i32 0
+  ret double %3
+}
+
 define <2 x double> @test_cmp_sd(<2 x double> %a, <2 x double> %b) {
 ; CHECK-LABEL: @test_cmp_sd(
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %a, <2 x double> %b, i8 0)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], i8 0)
 ; CHECK-NEXT:    ret <2 x double> [[TMP1]]
 ;
   %1 = insertelement <2 x double> %b, double 2.000000e+00, i32 1
@@ -225,8 +295,8 @@ define <2 x double> @test_cmp_sd(<2 x double> %a, <2 x double> %b) {
 
 define double @test_cmp_sd_0(double %a, double %b) {
 ; CHECK-LABEL: @test_cmp_sd_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double %b, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]], i8 0)
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i32 0
 ; CHECK-NEXT:    ret double [[TMP4]]
@@ -253,10 +323,23 @@ define double @test_cmp_sd_1(double %a, double %b) {
   ret double %6
 }
 
+define double @test_cmp_sd_2(double %a) {
+; CHECK-LABEL: @test_cmp_sd_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> <double undef, double 0.000000e+00>, double [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> [[TMP1]], <2 x double> [[TMP1]], i8 3)
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[TMP2]], i32 0
+; CHECK-NEXT:    ret double [[TMP3]]
+;
+  %1 = insertelement <2 x double> zeroinitializer, double %a, i32 0
+  %2 = tail call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %1, <2 x double> %1, i8 3)
+  %3 = extractelement <2 x double> %2, i32 0
+  ret double %3
+}
+
 define i32 @test_comieq_sd_0(double %a, double %b) {
 ; CHECK-LABEL: @test_comieq_sd_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double %b, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.comieq.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
@@ -270,8 +353,8 @@ define i32 @test_comieq_sd_0(double %a, double %b) {
 
 define i32 @test_comige_sd_0(double %a, double %b) {
 ; CHECK-LABEL: @test_comige_sd_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double %b, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.comige.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
@@ -285,8 +368,8 @@ define i32 @test_comige_sd_0(double %a, double %b) {
 
 define i32 @test_comigt_sd_0(double %a, double %b) {
 ; CHECK-LABEL: @test_comigt_sd_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double %b, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.comigt.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
@@ -300,8 +383,8 @@ define i32 @test_comigt_sd_0(double %a, double %b) {
 
 define i32 @test_comile_sd_0(double %a, double %b) {
 ; CHECK-LABEL: @test_comile_sd_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double %b, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.comile.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
@@ -315,8 +398,8 @@ define i32 @test_comile_sd_0(double %a, double %b) {
 
 define i32 @test_comilt_sd_0(double %a, double %b) {
 ; CHECK-LABEL: @test_comilt_sd_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double %b, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.comilt.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
@@ -330,8 +413,8 @@ define i32 @test_comilt_sd_0(double %a, double %b) {
 
 define i32 @test_comineq_sd_0(double %a, double %b) {
 ; CHECK-LABEL: @test_comineq_sd_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double %b, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.comineq.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
@@ -345,8 +428,8 @@ define i32 @test_comineq_sd_0(double %a, double %b) {
 
 define i32 @test_ucomieq_sd_0(double %a, double %b) {
 ; CHECK-LABEL: @test_ucomieq_sd_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double %b, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.ucomieq.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
@@ -360,8 +443,8 @@ define i32 @test_ucomieq_sd_0(double %a, double %b) {
 
 define i32 @test_ucomige_sd_0(double %a, double %b) {
 ; CHECK-LABEL: @test_ucomige_sd_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double %b, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.ucomige.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
@@ -375,8 +458,8 @@ define i32 @test_ucomige_sd_0(double %a, double %b) {
 
 define i32 @test_ucomigt_sd_0(double %a, double %b) {
 ; CHECK-LABEL: @test_ucomigt_sd_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double %b, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.ucomigt.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
@@ -390,8 +473,8 @@ define i32 @test_ucomigt_sd_0(double %a, double %b) {
 
 define i32 @test_ucomile_sd_0(double %a, double %b) {
 ; CHECK-LABEL: @test_ucomile_sd_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double %b, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.ucomile.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
@@ -405,8 +488,8 @@ define i32 @test_ucomile_sd_0(double %a, double %b) {
 
 define i32 @test_ucomilt_sd_0(double %a, double %b) {
 ; CHECK-LABEL: @test_ucomilt_sd_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double %b, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.ucomilt.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
@@ -420,8 +503,8 @@ define i32 @test_ucomilt_sd_0(double %a, double %b) {
 
 define i32 @test_ucomineq_sd_0(double %a, double %b) {
 ; CHECK-LABEL: @test_ucomineq_sd_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double %b, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.ucomineq.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-sse41.ll b/llvm/test/Transforms/InstCombine/X86/x86-sse41.ll
index f95b1b4d552d2..eb4a011b13052 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-sse41.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-sse41.ll
@@ -42,6 +42,19 @@ define double @test_round_sd_1(double %a, double %b) {
   ret double %6
 }
 
+define double @test_round_sd_2(double %a) {
+; CHECK-LABEL: @test_round_sd_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <2 x double> @llvm.x86.sse41.round.sd(<2 x double> undef, <2 x double> [[TMP1]], i32 10)
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[TMP2]], i32 0
+; CHECK-NEXT:    ret double [[TMP3]]
+;
+  %1 = insertelement <2 x double> zeroinitializer, double %a, i32 0
+  %2 = tail call <2 x double> @llvm.x86.sse41.round.sd(<2 x double> %1, <2 x double> %1, i32 10)
+  %3 = extractelement <2 x double> %2, i32 0
+  ret double %3
+}
+
 define <4 x float> @test_round_ss(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: @test_round_ss(
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.sse41.round.ss(<4 x float> <float undef, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>, <4 x float> [[B:%.*]], i32 10)
@@ -94,5 +107,18 @@ define float @test_round_ss_2(float %a, float %b) {
   ret float %r
 }
 
+define float @test_round_ss_3(float %a) {
+; CHECK-LABEL: @test_round_ss_3(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <4 x float> @llvm.x86.sse41.round.ss(<4 x float> undef, <4 x float> [[TMP1]], i32 10)
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; CHECK-NEXT:    ret float [[TMP3]]
+;
+  %1 = insertelement <4 x float> zeroinitializer, float %a, i32 0
+  %2 = tail call <4 x float> @llvm.x86.sse41.round.ss(<4 x float> %1, <4 x float> %1, i32 10)
+  %3 = extractelement <4 x float> %2, i32 0
+  ret float %3
+}
+
 declare <2 x double> @llvm.x86.sse41.round.sd(<2 x double>, <2 x double>, i32) nounwind readnone
 declare <4 x float> @llvm.x86.sse41.round.ss(<4 x float>, <4 x float>, i32) nounwind readnone
diff --git a/llvm/test/Transforms/InstCombine/assume.ll b/llvm/test/Transforms/InstCombine/assume.ll
index 8abb379e537e1..66d58485a7940 100644
--- a/llvm/test/Transforms/InstCombine/assume.ll
+++ b/llvm/test/Transforms/InstCombine/assume.ll
@@ -289,9 +289,9 @@ define i1 @nonnull3(i32** %a, i1 %control) {
 ; CHECK-LABEL: @nonnull3(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32* [[LOAD]], null
 ; CHECK-NEXT:    br i1 [[CONTROL:%.*]], label [[TAKEN:%.*]], label [[NOT_TAKEN:%.*]]
 ; CHECK:       taken:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32* [[LOAD]], null
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
 ; CHECK-NEXT:    ret i1 false
 ; CHECK:       not_taken:
@@ -414,6 +414,117 @@ define i32 @PR40940(<4 x i8> %x) {
   ret i32 %t2
 }
 
+define i1 @nonnull3A(i32** %a, i1 %control) {
+; CHECK-LABEL: @nonnull3A(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8
+; CHECK-NEXT:    br i1 [[CONTROL:%.*]], label [[TAKEN:%.*]], label [[NOT_TAKEN:%.*]]
+; CHECK:       taken:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32* [[LOAD]], null
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    ret i1 true
+; CHECK:       not_taken:
+; CHECK-NEXT:    [[RVAL_2:%.*]] = icmp sgt i32* [[LOAD]], null
+; CHECK-NEXT:    ret i1 [[RVAL_2]]
+;
+entry:
+  %load = load i32*, i32** %a
+  %cmp = icmp ne i32* %load, null
+  br i1 %control, label %taken, label %not_taken
+taken:
+  call void @llvm.assume(i1 %cmp)
+  ret i1 %cmp
+not_taken:
+  call void @llvm.assume(i1 %cmp)
+  %rval.2 = icmp sgt i32* %load, null
+  ret i1 %rval.2
+}
+
+define i1 @nonnull3B(i32** %a, i1 %control) {
+; CHECK-LABEL: @nonnull3B(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[CONTROL:%.*]], label [[TAKEN:%.*]], label [[NOT_TAKEN:%.*]]
+; CHECK:       taken:
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32* [[LOAD]], null
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]]) [ "nonnull"(i32* [[LOAD]]), "nonnull"(i1 [[CMP]]) ]
+; CHECK-NEXT:    ret i1 true
+; CHECK:       not_taken:
+; CHECK-NEXT:    ret i1 [[CONTROL]]
+;
+entry:
+  %load = load i32*, i32** %a
+  %cmp = icmp ne i32* %load, null
+  br i1 %control, label %taken, label %not_taken
+taken:
+  call void @llvm.assume(i1 %cmp) ["nonnull"(i32* %load), "nonnull"(i1 %cmp)]
+  ret i1 %cmp
+not_taken:
+  call void @llvm.assume(i1 %cmp) ["nonnull"(i32* %load), "nonnull"(i1 %cmp)]
+  ret i1 %control
+}
+
+declare i1 @tmp1(i1)
+
+define i1 @nonnull3C(i32** %a, i1 %control) {
+; CHECK-LABEL: @nonnull3C(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[CONTROL:%.*]], label [[TAKEN:%.*]], label [[NOT_TAKEN:%.*]]
+; CHECK:       taken:
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32* [[LOAD]], null
+; CHECK-NEXT:    [[CMP2:%.*]] = call i1 @tmp1(i1 [[CMP]])
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 [[CMP2]]
+; CHECK:       not_taken:
+; CHECK-NEXT:    ret i1 [[CONTROL]]
+;
+entry:
+  %load = load i32*, i32** %a
+  %cmp = icmp ne i32* %load, null
+  br i1 %control, label %taken, label %not_taken
+taken:
+  %cmp2 = call i1 @tmp1(i1 %cmp)
+  br label %exit
+exit:
+  ; FIXME: this shouldn't be dropped because it is still dominated by the new position of %load
+  call void @llvm.assume(i1 %cmp) ["nonnull"(i32* %load), "nonnull"(i1 %cmp)]
+  ret i1 %cmp2
+not_taken:
+  call void @llvm.assume(i1 %cmp)
+  ret i1 %control
+}
+
+define i1 @nonnull3D(i32** %a, i1 %control) {
+; CHECK-LABEL: @nonnull3D(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[CONTROL:%.*]], label [[TAKEN:%.*]], label [[NOT_TAKEN:%.*]]
+; CHECK:       taken:
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32*, i32** [[A:%.*]], align 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32* [[LOAD]], null
+; CHECK-NEXT:    [[CMP2:%.*]] = call i1 @tmp1(i1 [[CMP]])
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 [[CMP2]]
+; CHECK:       not_taken:
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "ignore"(i32* undef), "ignore"(i1 undef), "nonnull"(i1 [[CONTROL]]) ]
+; CHECK-NEXT:    ret i1 [[CONTROL]]
+;
+entry:
+  %load = load i32*, i32** %a
+  %cmp = icmp ne i32* %load, null
+  br i1 %control, label %taken, label %not_taken
+taken:
+  %cmp2 = call i1 @tmp1(i1 %cmp)
+  br label %exit
+exit:
+  ret i1 %cmp2
+not_taken:
+  call void @llvm.assume(i1 %cmp) ["nonnull"(i32* %load), "nonnull"(i1 %cmp), "nonnull"(i1 %control)]
+  ret i1 %control
+}
+
 declare void @llvm.dbg.value(metadata, metadata, metadata)
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/Transforms/InstCombine/deref-alloc-fns.ll b/llvm/test/Transforms/InstCombine/deref-alloc-fns.ll
index efaf1cd1604d5..9d9964de2e000 100644
--- a/llvm/test/Transforms/InstCombine/deref-alloc-fns.ll
+++ b/llvm/test/Transforms/InstCombine/deref-alloc-fns.ll
@@ -7,6 +7,7 @@ declare noalias i8* @realloc(i8* nocapture, i64)
 declare noalias nonnull i8* @_Znam(i64) ; throwing version of 'new'
 declare noalias nonnull i8* @_Znwm(i64) ; throwing version of 'new'
 declare noalias i8* @strdup(i8*)
+declare noalias i8* @aligned_alloc(i64, i64)
 
 @.str = private unnamed_addr constant [6 x i8] c"hello\00", align 1
 
@@ -28,6 +29,31 @@ define noalias i8* @malloc_constant_size() {
   ret i8* %call
 }
 
+define noalias i8* @aligned_alloc_constant_size() {
+; CHECK-LABEL: @aligned_alloc_constant_size(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call noalias align 32 dereferenceable_or_null(512) i8* @aligned_alloc(i64 32, i64 512)
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = tail call noalias i8* @aligned_alloc(i64 32, i64 512)
+  ret i8* %call
+}
+
+declare noalias i8* @foo(i8*, i8*, i8*)
+
+define noalias i8* @aligned_alloc_dynamic_args(i64 %align, i64 %size) {
+; CHECK-LABEL: @aligned_alloc_dynamic_args(
+; CHECK-NEXT:    tail call noalias dereferenceable_or_null(1024) i8* @aligned_alloc(i64 %{{.*}}, i64 1024)
+; CHECK-NEXT:    tail call noalias i8* @aligned_alloc(i64 0, i64 1024)
+; CHECK-NEXT:    tail call noalias i8* @aligned_alloc(i64 32, i64 %{{.*}})
+;
+  %call = tail call noalias i8* @aligned_alloc(i64 %align, i64 1024)
+  %call_1 = tail call noalias i8* @aligned_alloc(i64 0, i64 1024)
+  %call_2 = tail call noalias i8* @aligned_alloc(i64 32, i64 %size)
+
+  call i8* @foo(i8* %call, i8* %call_1, i8* %call_2)
+  ret i8* %call
+}
+
 define noalias i8* @malloc_constant_size2() {
 ; CHECK-LABEL: @malloc_constant_size2(
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call noalias dereferenceable_or_null(80) i8* @malloc(i64 40)
@@ -46,7 +72,6 @@ define noalias i8* @malloc_constant_size3() {
   ret i8* %call
 }
 
-
 define noalias i8* @malloc_constant_zero_size() {
 ; CHECK-LABEL: @malloc_constant_zero_size(
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call noalias i8* @malloc(i64 0)
diff --git a/llvm/test/Transforms/InstCombine/double-float-shrink-1.ll b/llvm/test/Transforms/InstCombine/double-float-shrink-1.ll
index 330e0abb36b85..65439eb1429f8 100644
--- a/llvm/test/Transforms/InstCombine/double-float-shrink-1.ll
+++ b/llvm/test/Transforms/InstCombine/double-float-shrink-1.ll
@@ -529,12 +529,14 @@ define float @max1(float %a, float %b) {
 
 define float @fake_fmin(float %a, float %b) {
 ; CHECK-LABEL: @fake_fmin(
-; CHECK-NEXT:    [[C:%.*]] = fpext float [[A:%.*]] to fp128
-; CHECK-NEXT:    [[D:%.*]] = fpext float [[B:%.*]] to fp128
-; ISC99-NEXT:    [[E:%.*]] = call nsz fp128 @llvm.minnum.f128(fp128 [[C]], fp128 [[D]])
+; ISC99-NEXT:    [[MIN:%.*]] = call nsz float @llvm.minnum.f32(float %a, float %b)
+; ISC99-NEXT:    ret float [[MIN]]
+
+; ISC89-NEXT:    [[C:%.*]] = fpext float [[A:%.*]] to fp128
+; ISC89-NEXT:    [[D:%.*]] = fpext float [[B:%.*]] to fp128
 ; ISC89-NEXT:    [[E:%.*]] = call fp128 @fmin(fp128 [[C]], fp128 [[D]])
-; CHECK-NEXT:    [[F:%.*]] = fptrunc fp128 [[E]] to float
-; CHECK-NEXT:    ret float [[F]]
+; ISC89-NEXT:    [[F:%.*]] = fptrunc fp128 [[E]] to float
+; ISC89-NEXT:    ret float [[F]]
 ;
   %c = fpext float %a to fp128
   %d = fpext float %b to fp128
diff --git a/llvm/test/Transforms/InstCombine/gepphigep.ll b/llvm/test/Transforms/InstCombine/gepphigep.ll
index cc90d714be734..7d3fe949ede26 100644
--- a/llvm/test/Transforms/InstCombine/gepphigep.ll
+++ b/llvm/test/Transforms/InstCombine/gepphigep.ll
@@ -1,4 +1,4 @@
-; RUN: opt -instcombine -S  < %s | FileCheck %s
+; RUN: opt -instcombine -instcombine-infinite-loop-threshold=3 -S < %s | FileCheck %s
 
 %struct1 = type { %struct2*, i32, i32, i32 }
 %struct2 = type { i32, i32 }
diff --git a/llvm/test/Transforms/InstCombine/icmp.ll b/llvm/test/Transforms/InstCombine/icmp.ll
index 9e283510799b9..41bf661b092e8 100644
--- a/llvm/test/Transforms/InstCombine/icmp.ll
+++ b/llvm/test/Transforms/InstCombine/icmp.ll
@@ -3606,3 +3606,94 @@ define <2 x i32> @Op1Negated_Vec(<2 x i32> %x) {
   %cond = select <2 x i1> %cmp, <2 x i32> %sub, <2 x i32> %x
   ret <2 x i32> %cond
 }
+
+define i1 @signbit_bitcast_fpext(float %x) {
+; CHECK-LABEL: @signbit_bitcast_fpext(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[R:%.*]] = icmp slt i32 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %f = fpext float %x to double
+  %b = bitcast double %f to i64
+  %r = icmp slt i64 %b, 0
+  ret i1 %r
+}
+
+define <2 x i1> @signbit_bitcast_fpext_vec(<2 x half> %x) {
+; CHECK-LABEL: @signbit_bitcast_fpext_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x half> [[X:%.*]] to <2 x i16>
+; CHECK-NEXT:    [[R:%.*]] = icmp slt <2 x i16> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %f = fpext <2 x half> %x to <2 x float>
+  %b = bitcast <2 x float> %f to <2 x i32>
+  %r = icmp ugt <2 x i32> %b, <i32 2147483647, i32 2147483647>
+  ret <2 x i1> %r
+}
+
+define i1 @signbit_bitcast_fptrunc(float %x) {
+; CHECK-LABEL: @signbit_bitcast_fptrunc(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[R:%.*]] = icmp sgt i32 [[TMP1]], -1
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %f = fptrunc float %x to half
+  %b = bitcast half %f to i16
+  %r = icmp ult i16 %b, 32768
+  ret i1 %r
+}
+
+define <2 x i1> @signbit_bitcast_fptrunc_vec(<2 x double> %x) {
+; CHECK-LABEL: @signbit_bitcast_fptrunc_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x double> [[X:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[R:%.*]] = icmp sgt <2 x i64> [[TMP1]], <i64 -1, i64 -1>
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %f = fptrunc <2 x double> %x to <2 x half>
+  %b = bitcast <2 x half> %f to <2 x i16>
+  %r = icmp sge <2 x i16> %b, zeroinitializer
+  ret <2 x i1> %r
+}
+
+define i1 @signbit_bitcast_fpext_wrong_cmp(float %x) {
+; CHECK-LABEL: @signbit_bitcast_fpext_wrong_cmp(
+; CHECK-NEXT:    [[F:%.*]] = fpext float [[X:%.*]] to double
+; CHECK-NEXT:    [[B:%.*]] = bitcast double [[F]] to i64
+; CHECK-NEXT:    [[R:%.*]] = icmp slt i64 [[B]], 1
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %f = fpext float %x to double
+  %b = bitcast double %f to i64
+  %r = icmp slt i64 %b, 1
+  ret i1 %r
+}
+
+define <4 x i1> @signbit_bitcast_fpext_vec_wrong_bitcast(<2 x half> %x) {
+; CHECK-LABEL: @signbit_bitcast_fpext_vec_wrong_bitcast(
+; CHECK-NEXT:    [[F:%.*]] = fpext <2 x half> [[X:%.*]] to <2 x float>
+; CHECK-NEXT:    [[B:%.*]] = bitcast <2 x float> [[F]] to <4 x i16>
+; CHECK-NEXT:    [[R:%.*]] = icmp sgt <4 x i16> [[B]], <i16 -1, i16 -1, i16 -1, i16 -1>
+; CHECK-NEXT:    ret <4 x i1> [[R]]
+;
+  %f = fpext <2 x half> %x to <2 x float>
+  %b = bitcast <2 x float> %f to <4 x i16>
+  %r = icmp sgt <4 x i16> %b, <i16 -1, i16 -1, i16 -1, i16 -1>
+  ret <4 x i1> %r
+}
+
+declare void @use_i64(i64)
+
+define i1 @signbit_bitcast_fpext_extra_use(float %x, i64* %p) {
+; CHECK-LABEL: @signbit_bitcast_fpext_extra_use(
+; CHECK-NEXT:    [[F:%.*]] = fpext float [[X:%.*]] to double
+; CHECK-NEXT:    [[B:%.*]] = bitcast double [[F]] to i64
+; CHECK-NEXT:    call void @use_i64(i64 [[B]])
+; CHECK-NEXT:    [[R:%.*]] = icmp slt i64 [[B]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %f = fpext float %x to double
+  %b = bitcast double %f to i64
+  call void @use_i64(i64 %b)
+  %r = icmp slt i64 %b, 0
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/limit-max-iterations.ll b/llvm/test/Transforms/InstCombine/limit-max-iterations.ll
index af027998edac3..a29166c042564 100644
--- a/llvm/test/Transforms/InstCombine/limit-max-iterations.ll
+++ b/llvm/test/Transforms/InstCombine/limit-max-iterations.ll
@@ -2,43 +2,38 @@
 ; RUN: opt < %s -instcombine --instcombine-max-iterations=0 -S | FileCheck %s --check-prefix=ZERO
 ; RUN: opt < %s -instcombine --instcombine-max-iterations=1 -S | FileCheck %s --check-prefix=ONE
 ; RUN: opt < %s -instcombine -S | FileCheck %s --check-prefix=FIXPOINT
-; RUN: not --crash opt < %s -instcombine -S --instcombine-infinite-loop-threshold=3 2>&1 | FileCheck %s --check-prefix=LOOP
+; RUN: not --crash opt < %s -instcombine -S --instcombine-infinite-loop-threshold=2 2>&1 | FileCheck %s --check-prefix=LOOP
 
-; Based on xor-of-icmps-with-extra-uses.ll. This requires multiple iterations of
+; Based on builtin-dynamic-object-size.ll. This requires multiple iterations of
 ; InstCombine to reach a fixpoint.
 
-define i1 @v0_select_of_consts(i32 %X, i32* %selected) {
-; ZERO-LABEL: @v0_select_of_consts(
-; ZERO-NEXT:    [[COND0:%.*]] = icmp sgt i32 [[X:%.*]], 32767
-; ZERO-NEXT:    [[COND1:%.*]] = icmp sgt i32 [[X]], -32768
-; ZERO-NEXT:    [[SELECT:%.*]] = select i1 [[COND0]], i32 32767, i32 -32768
-; ZERO-NEXT:    store i32 [[SELECT]], i32* [[SELECTED:%.*]]
-; ZERO-NEXT:    [[RES:%.*]] = xor i1 [[COND0]], [[COND1]]
-; ZERO-NEXT:    ret i1 [[RES]]
-
-; ONE-LABEL: @v0_select_of_consts(
-; ONE-NEXT:    [[COND0:%.*]] = icmp sle i32 [[X:%.*]], 32767
-; ONE-NEXT:    [[COND0_NOT:%.*]] = xor i1 [[COND0]], true
-; ONE-NEXT:    [[COND1:%.*]] = icmp sgt i32 [[X]], -32768
-; ONE-NEXT:    [[SELECT:%.*]] = select i1 [[COND0_NOT]], i32 32767, i32 -32768
-; ONE-NEXT:    store i32 [[SELECT]], i32* [[SELECTED:%.*]], align 4
-; ONE-NEXT:    [[TMP1:%.*]] = and i1 [[COND0]], [[COND1]]
-; ONE-NEXT:    ret i1 [[TMP1]]
-
-; FIXPOINT-LABEL: @v0_select_of_consts(
-; FIXPOINT-NEXT:    [[COND0_INV:%.*]] = icmp sgt i32 [[X:%.*]], 32767
-; FIXPOINT-NEXT:    [[SELECT:%.*]] = select i1 [[COND0_INV]], i32 32767, i32 -32768
-; FIXPOINT-NEXT:    store i32 [[SELECT]], i32* [[SELECTED:%.*]], align 4
-; FIXPOINT-NEXT:    [[X_OFF:%.*]] = add i32 [[X]], 32767
-; FIXPOINT-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X_OFF]], 65535
-; FIXPOINT-NEXT:    ret i1 [[TMP1]]
-
-; LOOP: LLVM ERROR: Instruction Combining seems stuck in an infinite loop after 3 iterations.
-
-  %cond0 = icmp sgt i32 %X, 32767
-  %cond1 = icmp sgt i32 %X, -32768
-  %select = select i1 %cond0, i32 32767, i32 -32768
-  store i32 %select, i32* %selected
-  %res = xor i1 %cond0, %cond1
-  ret i1 %res
+define i64 @weird_identity_but_ok(i64 %sz) {
+; ZERO-LABEL: @weird_identity_but_ok(
+; ZERO-NEXT:  entry:
+; ZERO-NEXT:    [[CALL:%.*]] = tail call i8* @malloc(i64 [[SZ:%.*]])
+; ZERO-NEXT:    [[CALC_SIZE:%.*]] = tail call i64 @llvm.objectsize.i64.p0i8(i8* [[CALL]], i1 false, i1 true, i1 true)
+; ZERO-NEXT:    tail call void @free(i8* [[CALL]])
+; ZERO-NEXT:    ret i64 [[CALC_SIZE]]
+;
+; ONE-LABEL: @weird_identity_but_ok(
+; ONE-NEXT:  entry:
+; ONE-NEXT:    [[TMP0:%.*]] = sub i64 [[SZ:%.*]], 0
+; ONE-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[SZ]], 0
+; ONE-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i64 0, i64 [[TMP0]]
+; ONE-NEXT:    ret i64 [[TMP2]]
+;
+; FIXPOINT-LABEL: @weird_identity_but_ok(
+; FIXPOINT-NEXT:  entry:
+; FIXPOINT-NEXT:    ret i64 [[SZ:%.*]]
+;
+; LOOP: LLVM ERROR: Instruction Combining seems stuck in an infinite loop after 2 iterations.
+entry:
+  %call = tail call i8* @malloc(i64 %sz)
+  %calc_size = tail call i64 @llvm.objectsize.i64.p0i8(i8* %call, i1 false, i1 true, i1 true)
+  tail call void @free(i8* %call)
+  ret i64 %calc_size
 }
+
+declare i64 @llvm.objectsize.i64.p0i8(i8*, i1, i1, i1)
+declare i8* @malloc(i64)
+declare void @free(i8*)
diff --git a/llvm/test/Transforms/InstCombine/malloc-free-delete.ll b/llvm/test/Transforms/InstCombine/malloc-free-delete.ll
index 7f6d7b4b28326..2f4447b651189 100644
--- a/llvm/test/Transforms/InstCombine/malloc-free-delete.ll
+++ b/llvm/test/Transforms/InstCombine/malloc-free-delete.ll
@@ -13,8 +13,19 @@ define i32 @main(i32 %argc, i8** %argv) {
   ret i32 0
 }
 
+define i32 @dead_aligned_alloc(i32 %size, i32 %alignment, i8 %value) {
+; CHECK-LABEL: @dead_aligned_alloc(
+; CHECK-NEXT:    ret i32 0
+;
+  %aligned_allocation = tail call i8* @aligned_alloc(i32 %alignment, i32 %size)
+  store i8 %value, i8* %aligned_allocation
+  tail call void @free(i8* %aligned_allocation)
+  ret i32 0
+}
+
 declare noalias i8* @calloc(i32, i32) nounwind
 declare noalias i8* @malloc(i32)
+declare noalias i8* @aligned_alloc(i32, i32)
 declare void @free(i8*)
 
 define i1 @foo() {
diff --git a/llvm/test/Transforms/InstCombine/max-of-nots.ll b/llvm/test/Transforms/InstCombine/max-of-nots.ll
index 7757c70a22b48..c6639ec20fec5 100644
--- a/llvm/test/Transforms/InstCombine/max-of-nots.ll
+++ b/llvm/test/Transforms/InstCombine/max-of-nots.ll
@@ -358,3 +358,99 @@ define <2 x i32> @max_of_min_vec(<2 x i32> %a) {
   ret <2 x i32> %s1
 }
 
+declare void @use(i8, i8, i8, i8)
+
+define void @cmyk(i8 %r, i8 %g, i8 %b) {
+; CHECK-LABEL: @cmyk(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i8 [[R:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i8 [[R]], i8 [[B]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i8 [[TMP2]], [[G:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i8 [[TMP2]], i8 [[G]]
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i8 [[TMP4]], -1
+; CHECK-NEXT:    [[CK:%.*]] = sub i8 [[TMP4]], [[R]]
+; CHECK-NEXT:    [[MK:%.*]] = sub i8 [[TMP4]], [[G]]
+; CHECK-NEXT:    [[YK:%.*]] = sub i8 [[TMP4]], [[B]]
+; CHECK-NEXT:    call void @use(i8 [[CK]], i8 [[MK]], i8 [[YK]], i8 [[TMP5]])
+; CHECK-NEXT:    ret void
+;
+  %notr = xor i8 %r, -1
+  %notg = xor i8 %g, -1
+  %notb = xor i8 %b, -1
+  %cmp_gr = icmp slt i8 %g, %r
+  %cmp_br = icmp slt i8 %notr, %notb
+  %min_br = select i1 %cmp_br, i8 %notr, i8 %notb
+  %cmp_gb = icmp slt i8 %notg, %notb
+  %min_gb = select i1 %cmp_gb, i8 %notg, i8 %notb
+  %k = select i1 %cmp_gr, i8 %min_br, i8 %min_gb
+  %ck = sub i8 %notr, %k
+  %mk = sub i8 %notg, %k
+  %yk = sub i8 %notb, %k
+  call void @use(i8 %ck, i8 %mk, i8 %yk, i8 %k)
+  ret void
+}
+
+define void @cmyk2(i8 %r, i8 %g, i8 %b) {
+; CHECK-LABEL: @cmyk2(
+; CHECK-NEXT:    [[NOTR:%.*]] = xor i8 [[R:%.*]], -1
+; CHECK-NEXT:    [[NOTG:%.*]] = xor i8 [[G:%.*]], -1
+; CHECK-NEXT:    [[NOTB:%.*]] = xor i8 [[B:%.*]], -1
+; CHECK-NEXT:    [[CMP_GR:%.*]] = icmp slt i8 [[G]], [[R]]
+; CHECK-NEXT:    [[CMP_BR:%.*]] = icmp slt i8 [[B]], [[R]]
+; CHECK-NEXT:    [[MIN_BR:%.*]] = select i1 [[CMP_BR]], i8 [[NOTR]], i8 [[NOTB]]
+; CHECK-NEXT:    [[CMP_BG:%.*]] = icmp slt i8 [[B]], [[G]]
+; CHECK-NEXT:    [[MIN_BG:%.*]] = select i1 [[CMP_BG]], i8 [[NOTG]], i8 [[NOTB]]
+; CHECK-NEXT:    [[K:%.*]] = select i1 [[CMP_GR]], i8 [[MIN_BR]], i8 [[MIN_BG]]
+; CHECK-NEXT:    [[CK:%.*]] = sub i8 [[NOTR]], [[K]]
+; CHECK-NEXT:    [[MK:%.*]] = sub i8 [[NOTG]], [[K]]
+; CHECK-NEXT:    [[YK:%.*]] = sub i8 [[NOTB]], [[K]]
+; CHECK-NEXT:    call void @use(i8 [[CK]], i8 [[MK]], i8 [[YK]], i8 [[K]])
+; CHECK-NEXT:    ret void
+;
+  %notr = xor i8 %r, -1
+  %notg = xor i8 %g, -1
+  %notb = xor i8 %b, -1
+  %cmp_gr = icmp slt i8 %g, %r
+  %cmp_br = icmp slt i8 %b, %r
+  %min_br = select i1 %cmp_br, i8 %notr, i8 %notb
+  %cmp_bg = icmp slt i8 %b, %g
+  %min_bg = select i1 %cmp_bg, i8 %notg, i8 %notb
+  %k = select i1 %cmp_gr, i8 %min_br, i8 %min_bg
+  %ck = sub i8 %notr, %k
+  %mk = sub i8 %notg, %k
+  %yk = sub i8 %notb, %k
+  call void @use(i8 %ck, i8 %mk, i8 %yk, i8 %k)
+  ret void
+}
+
+define void @cmyk3(i8 %r, i8 %g, i8 %b) {
+; CHECK-LABEL: @cmyk3(
+; CHECK-NEXT:    [[NOTR:%.*]] = xor i8 [[R:%.*]], -1
+; CHECK-NEXT:    [[NOTG:%.*]] = xor i8 [[G:%.*]], -1
+; CHECK-NEXT:    [[NOTB:%.*]] = xor i8 [[B:%.*]], -1
+; CHECK-NEXT:    [[CMP_GR:%.*]] = icmp ult i8 [[G]], [[R]]
+; CHECK-NEXT:    [[CMP_BR:%.*]] = icmp ult i8 [[B]], [[R]]
+; CHECK-NEXT:    [[SEL_RB:%.*]] = select i1 [[CMP_BR]], i8 [[NOTR]], i8 [[NOTB]]
+; CHECK-NEXT:    [[CMP_BG:%.*]] = icmp ult i8 [[B]], [[G]]
+; CHECK-NEXT:    [[SEL_GB:%.*]] = select i1 [[CMP_BG]], i8 [[NOTG]], i8 [[NOTB]]
+; CHECK-NEXT:    [[K:%.*]] = select i1 [[CMP_GR]], i8 [[SEL_RB]], i8 [[SEL_GB]]
+; CHECK-NEXT:    [[CK:%.*]] = sub i8 [[NOTR]], [[K]]
+; CHECK-NEXT:    [[MK:%.*]] = sub i8 [[NOTG]], [[K]]
+; CHECK-NEXT:    [[YK:%.*]] = sub i8 [[NOTB]], [[K]]
+; CHECK-NEXT:    tail call void @use(i8 [[CK]], i8 [[MK]], i8 [[YK]], i8 [[K]])
+; CHECK-NEXT:    ret void
+;
+  %notr = xor i8 %r, -1
+  %notg = xor i8 %g, -1
+  %notb = xor i8 %b, -1
+  %cmp_gr = icmp ult i8 %g, %r
+  %cmp_br = icmp ult i8 %b, %r
+  %sel_rb = select i1 %cmp_br, i8 %notr, i8 %notb
+  %cmp_bg = icmp ult i8 %b, %g
+  %sel_gb = select i1 %cmp_bg, i8 %notg, i8 %notb
+  %k = select i1 %cmp_gr, i8 %sel_rb, i8 %sel_gb
+  %ck = sub i8 %notr, %k
+  %mk = sub i8 %notg, %k
+  %yk = sub i8 %notb, %k
+  tail call void @use(i8 %ck, i8 %mk, i8 %yk, i8 %k)
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/maximum.ll b/llvm/test/Transforms/InstCombine/maximum.ll
index a60d0a6685afa..2439f7cd5f502 100644
--- a/llvm/test/Transforms/InstCombine/maximum.ll
+++ b/llvm/test/Transforms/InstCombine/maximum.ll
@@ -361,3 +361,27 @@ define float @unary_neg_neg_extra_use_x_and_y(float %x, float %y) {
   call void @use(float %negy)
   ret float %r
 }
+
+define float @reduce_precision(float %x, float %y) {
+; CHECK-LABEL: @reduce_precision(
+; CHECK-NEXT:    [[MAXIMUM:%.*]] = call float @llvm.maximum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    ret float [[MAXIMUM]]
+;
+  %x.ext = fpext float %x to double
+  %y.ext = fpext float %y to double
+  %maximum = call double @llvm.maximum.f64(double %x.ext, double %y.ext)
+  %trunc = fptrunc double %maximum to float
+  ret float %trunc
+}
+
+define float @reduce_precision_fmf(float %x, float %y) {
+; CHECK-LABEL: @reduce_precision_fmf(
+; CHECK-NEXT:    [[MAXIMUM:%.*]] = call nnan float @llvm.maximum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    ret float [[MAXIMUM]]
+;
+  %x.ext = fpext float %x to double
+  %y.ext = fpext float %y to double
+  %maximum = call nnan double @llvm.maximum.f64(double %x.ext, double %y.ext)
+  %trunc = fptrunc double %maximum to float
+  ret float %trunc
+}
diff --git a/llvm/test/Transforms/InstCombine/maxnum.ll b/llvm/test/Transforms/InstCombine/maxnum.ll
index c89e30598db44..df3c75c7e53f6 100644
--- a/llvm/test/Transforms/InstCombine/maxnum.ll
+++ b/llvm/test/Transforms/InstCombine/maxnum.ll
@@ -355,3 +355,61 @@ define float @unary_neg_neg_extra_use_x_and_y(float %x, float %y) {
   call void @use(float %negy)
   ret float %r
 }
+
+define float @reduce_precision(float %x, float %y) {
+; CHECK-LABEL: @reduce_precision(
+; CHECK-NEXT:    [[MAXNUM:%.*]] = call float @llvm.maxnum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    ret float [[MAXNUM]]
+;
+  %x.ext = fpext float %x to double
+  %y.ext = fpext float %y to double
+  %maxnum = call double @llvm.maxnum.f64(double %x.ext, double %y.ext)
+  %trunc = fptrunc double %maxnum to float
+  ret float %trunc
+}
+
+define float @reduce_precision_fmf(float %x, float %y) {
+; CHECK-LABEL: @reduce_precision_fmf(
+; CHECK-NEXT:    [[MAXNUM:%.*]] = call nnan float @llvm.maxnum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    ret float [[MAXNUM]]
+;
+  %x.ext = fpext float %x to double
+  %y.ext = fpext float %y to double
+  %maxnum = call nnan double @llvm.maxnum.f64(double %x.ext, double %y.ext)
+  %trunc = fptrunc double %maxnum to float
+  ret float %trunc
+}
+
+define float @reduce_precision_multi_use_0(float %x, float %y) {
+; CHECK-LABEL: @reduce_precision_multi_use_0(
+; CHECK-NEXT:    [[X_EXT:%.*]] = fpext float [[X:%.*]] to double
+; CHECK-NEXT:    [[Y_EXT:%.*]] = fpext float [[Y:%.*]] to double
+; CHECK-NEXT:    store double [[X_EXT]], double* undef, align 8
+; CHECK-NEXT:    [[MAXNUM:%.*]] = call double @llvm.maxnum.f64(double [[X_EXT]], double [[Y_EXT]])
+; CHECK-NEXT:    [[TRUNC:%.*]] = fptrunc double [[MAXNUM]] to float
+; CHECK-NEXT:    ret float [[TRUNC]]
+;
+  %x.ext = fpext float %x to double
+  %y.ext = fpext float %y to double
+  store double %x.ext, double* undef
+  %maxnum = call double @llvm.maxnum.f64(double %x.ext, double %y.ext)
+  %trunc = fptrunc double %maxnum to float
+  ret float %trunc
+}
+
+define float @reduce_precision_multi_use_1(float %x, float %y) {
+; CHECK-LABEL: @reduce_precision_multi_use_1(
+; CHECK-NEXT:    [[X_EXT:%.*]] = fpext float [[X:%.*]] to double
+; CHECK-NEXT:    [[Y_EXT:%.*]] = fpext float [[Y:%.*]] to double
+; CHECK-NEXT:    store double [[Y_EXT]], double* undef, align 8
+; CHECK-NEXT:    [[MAXNUM:%.*]] = call double @llvm.maxnum.f64(double [[X_EXT]], double [[Y_EXT]])
+; CHECK-NEXT:    [[TRUNC:%.*]] = fptrunc double [[MAXNUM]] to float
+; CHECK-NEXT:    ret float [[TRUNC]]
+;
+  %x.ext = fpext float %x to double
+  %y.ext = fpext float %y to double
+  store double %y.ext, double* undef
+  %maxnum = call double @llvm.maxnum.f64(double %x.ext, double %y.ext)
+  %trunc = fptrunc double %maxnum to float
+  ret float %trunc
+}
diff --git a/llvm/test/Transforms/InstCombine/minimum.ll b/llvm/test/Transforms/InstCombine/minimum.ll
index 158e44ce9cc53..a8b6aefe52c49 100644
--- a/llvm/test/Transforms/InstCombine/minimum.ll
+++ b/llvm/test/Transforms/InstCombine/minimum.ll
@@ -149,8 +149,9 @@ define float @minimum_f32_val_nan(float %x) {
 
 define float @minimum_f32_1_minimum_val_p0(float %x) {
 ; CHECK-LABEL: @minimum_f32_1_minimum_val_p0(
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.minimum.f32(float %x, float 0.000000e+00)
-; CHECK-NEXT: ret float [[RES]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.minimum.f32(float [[X:%.*]], float 0.000000e+00)
+; CHECK-NEXT:    ret float [[TMP1]]
+;
   %y = call float @llvm.minimum.f32(float %x, float 0.0)
   %z = call float @llvm.minimum.f32(float %y, float 1.0)
   ret float %z
@@ -158,8 +159,9 @@ define float @minimum_f32_1_minimum_val_p0(float %x) {
 
 define float @minimum_f32_1_minimum_p0_val_fast(float %x) {
 ; CHECK-LABEL: @minimum_f32_1_minimum_p0_val_fast(
-; CHECK-NEXT: [[RES:%.*]] = call fast float @llvm.minimum.f32(float %x, float 0.000000e+00)
-; CHECK-NEXT: ret float [[RES]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast float @llvm.minimum.f32(float [[X:%.*]], float 0.000000e+00)
+; CHECK-NEXT:    ret float [[TMP1]]
+;
   %y = call float @llvm.minimum.f32(float 0.0, float %x)
   %z = call fast float @llvm.minimum.f32(float %y, float 1.0)
   ret float %z
@@ -167,8 +169,9 @@ define float @minimum_f32_1_minimum_p0_val_fast(float %x) {
 
 define float @minimum_f32_1_minimum_p0_val_nnan_ninf(float %x) {
 ; CHECK-LABEL: @minimum_f32_1_minimum_p0_val_nnan_ninf(
-; CHECK-NEXT: [[RES:%.*]] = call nnan ninf float @llvm.minimum.f32(float %x, float 0.000000e+00)
-; CHECK-NEXT: ret float [[RES]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call nnan ninf float @llvm.minimum.f32(float [[X:%.*]], float 0.000000e+00)
+; CHECK-NEXT:    ret float [[TMP1]]
+;
   %y = call float @llvm.minimum.f32(float 0.0, float %x)
   %z = call nnan ninf float @llvm.minimum.f32(float %y, float 1.0)
   ret float %z
@@ -176,8 +179,9 @@ define float @minimum_f32_1_minimum_p0_val_nnan_ninf(float %x) {
 
 define float @minimum_f32_p0_minimum_val_n0(float %x) {
 ; CHECK-LABEL: @minimum_f32_p0_minimum_val_n0(
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.minimum.f32(float %x, float -0.000000e+00)
-; CHECK-NEXT: ret float [[RES]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.minimum.f32(float [[X:%.*]], float -0.000000e+00)
+; CHECK-NEXT:    ret float [[TMP1]]
+;
   %y = call float @llvm.minimum.f32(float %x, float -0.0)
   %z = call float @llvm.minimum.f32(float %y, float 0.0)
   ret float %z
@@ -185,8 +189,9 @@ define float @minimum_f32_p0_minimum_val_n0(float %x) {
 
 define float @minimum_f32_1_minimum_p0_val(float %x) {
 ; CHECK-LABEL: @minimum_f32_1_minimum_p0_val(
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.minimum.f32(float %x, float 0.000000e+00)
-; CHECK-NEXT: ret float [[RES]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.minimum.f32(float [[X:%.*]], float 0.000000e+00)
+; CHECK-NEXT:    ret float [[TMP1]]
+;
   %y = call float @llvm.minimum.f32(float 0.0, float %x)
   %z = call float @llvm.minimum.f32(float %y, float 1.0)
   ret float %z
@@ -194,8 +199,9 @@ define float @minimum_f32_1_minimum_p0_val(float %x) {
 
 define <2 x float> @minimum_f32_1_minimum_val_p0_val_v2f32(<2 x float> %x) {
 ; CHECK-LABEL: @minimum_f32_1_minimum_val_p0_val_v2f32(
-; CHECK-NEXT: [[RES:%.*]] = call <2 x float> @llvm.minimum.v2f32(<2 x float> %x, <2 x float> zeroinitializer)
-; CHECK-NEXT: ret <2 x float> [[RES]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x float> @llvm.minimum.v2f32(<2 x float> [[X:%.*]], <2 x float> zeroinitializer)
+; CHECK-NEXT:    ret <2 x float> [[TMP1]]
+;
   %y = call <2 x float> @llvm.minimum.v2f32(<2 x float> %x, <2 x float> zeroinitializer)
   %z = call <2 x float> @llvm.minimum.v2f32(<2 x float> %y, <2 x float><float 1.0, float 1.0>)
   ret <2 x float> %z
@@ -386,3 +392,27 @@ define double @unary_neg_neg_extra_use_x_and_y(double %x, double %y) {
   call void @use(double %negy)
   ret double %r
 }
+
+define float @reduce_precision(float %x, float %y) {
+; CHECK-LABEL: @reduce_precision(
+; CHECK-NEXT:    [[MINIMUM:%.*]] = call float @llvm.minimum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    ret float [[MINIMUM]]
+;
+  %x.ext = fpext float %x to double
+  %y.ext = fpext float %y to double
+  %minimum = call double @llvm.minimum.f64(double %x.ext, double %y.ext)
+  %trunc = fptrunc double %minimum to float
+  ret float %trunc
+}
+
+define float @reduce_precision_fmf(float %x, float %y) {
+; CHECK-LABEL: @reduce_precision_fmf(
+; CHECK-NEXT:    [[MINIMUM:%.*]] = call nnan float @llvm.minimum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    ret float [[MINIMUM]]
+;
+  %x.ext = fpext float %x to double
+  %y.ext = fpext float %y to double
+  %minimum = call nnan double @llvm.minimum.f64(double %x.ext, double %y.ext)
+  %trunc = fptrunc double %minimum to float
+  ret float %trunc
+}
diff --git a/llvm/test/Transforms/InstCombine/minnum.ll b/llvm/test/Transforms/InstCombine/minnum.ll
index f65d055c66e63..1c3e5a803c0df 100644
--- a/llvm/test/Transforms/InstCombine/minnum.ll
+++ b/llvm/test/Transforms/InstCombine/minnum.ll
@@ -393,3 +393,60 @@ define double @unary_neg_neg_extra_use_x_and_y(double %x, double %y) {
   ret double %r
 }
 
+define float @reduce_precision(float %x, float %y) {
+; CHECK-LABEL: @reduce_precision(
+; CHECK-NEXT:    [[MINNUM:%.*]] = call float @llvm.minnum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    ret float [[MINNUM]]
+;
+  %x.ext = fpext float %x to double
+  %y.ext = fpext float %y to double
+  %minnum = call double @llvm.minnum.f64(double %x.ext, double %y.ext)
+  %trunc = fptrunc double %minnum to float
+  ret float %trunc
+}
+
+define float @reduce_precision_fmf(float %x, float %y) {
+; CHECK-LABEL: @reduce_precision_fmf(
+; CHECK-NEXT:    [[MINNUM:%.*]] = call nnan float @llvm.minnum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    ret float [[MINNUM]]
+;
+  %x.ext = fpext float %x to double
+  %y.ext = fpext float %y to double
+  %minnum = call nnan double @llvm.minnum.f64(double %x.ext, double %y.ext)
+  %trunc = fptrunc double %minnum to float
+  ret float %trunc
+}
+
+define float @reduce_precision_multi_use_0(float %x, float %y) {
+; CHECK-LABEL: @reduce_precision_multi_use_0(
+; CHECK-NEXT:    [[X_EXT:%.*]] = fpext float [[X:%.*]] to double
+; CHECK-NEXT:    [[Y_EXT:%.*]] = fpext float [[Y:%.*]] to double
+; CHECK-NEXT:    store double [[X_EXT]], double* undef, align 8
+; CHECK-NEXT:    [[MINNUM:%.*]] = call double @llvm.minnum.f64(double [[X_EXT]], double [[Y_EXT]])
+; CHECK-NEXT:    [[TRUNC:%.*]] = fptrunc double [[MINNUM]] to float
+; CHECK-NEXT:    ret float [[TRUNC]]
+;
+  %x.ext = fpext float %x to double
+  %y.ext = fpext float %y to double
+  store double %x.ext, double* undef
+  %minnum = call double @llvm.minnum.f64(double %x.ext, double %y.ext)
+  %trunc = fptrunc double %minnum to float
+  ret float %trunc
+}
+
+define float @reduce_precision_multi_use_1(float %x, float %y) {
+; CHECK-LABEL: @reduce_precision_multi_use_1(
+; CHECK-NEXT:    [[X_EXT:%.*]] = fpext float [[X:%.*]] to double
+; CHECK-NEXT:    [[Y_EXT:%.*]] = fpext float [[Y:%.*]] to double
+; CHECK-NEXT:    store double [[Y_EXT]], double* undef, align 8
+; CHECK-NEXT:    [[MINNUM:%.*]] = call double @llvm.minnum.f64(double [[X_EXT]], double [[Y_EXT]])
+; CHECK-NEXT:    [[TRUNC:%.*]] = fptrunc double [[MINNUM]] to float
+; CHECK-NEXT:    ret float [[TRUNC]]
+;
+  %x.ext = fpext float %x to double
+  %y.ext = fpext float %y to double
+  store double %y.ext, double* undef
+  %minnum = call double @llvm.minnum.f64(double %x.ext, double %y.ext)
+  %trunc = fptrunc double %minnum to float
+  ret float %trunc
+}
diff --git a/llvm/test/Transforms/InstCombine/pr33689_same_bitwidth.ll b/llvm/test/Transforms/InstCombine/pr33689_same_bitwidth.ll
index e5dd019b9b519..35f0e5caadc6e 100644
--- a/llvm/test/Transforms/InstCombine/pr33689_same_bitwidth.ll
+++ b/llvm/test/Transforms/InstCombine/pr33689_same_bitwidth.ll
@@ -10,12 +10,12 @@ target datalayout = "p:16:16"
 @a = external global i16
 @b = external global i16*
 
-define void @f() {
+define void @f(i1 %cond) {
 ; CHECK-LABEL: @f(
 ; CHECK-NEXT:  bb0:
 ; CHECK-NEXT:    [[TMP12:%.*]] = alloca [2 x i32], align 8
 ; CHECK-NEXT:    [[TMP12_SUB:%.*]] = getelementptr inbounds [2 x i32], [2 x i32]* [[TMP12]], i16 0, i16 0
-; CHECK-NEXT:    br i1 undef, label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
 ; CHECK:       bb1:
 ; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint [2 x i32]* [[TMP12]] to i16
 ; CHECK-NEXT:    store i16 [[TMP8]], i16* @a, align 2
@@ -34,7 +34,7 @@ bb0:
   %useless3 = bitcast %i64_t* %tmp1 to i16*
   %useless4 = getelementptr inbounds i16, i16* %useless3, i16 undef
   %useless5 = bitcast i16* %useless4 to i32*
-  br i1 undef, label %bb1, label %bb2
+  br i1 %cond, label %bb1, label %bb2
 
 bb1:                                              ; preds = %bb0
   %useless6 = insertvalue [1 x i32*] undef, i32* %tmp2, 0
diff --git a/llvm/test/Transforms/InstCombine/shuffle-cast.ll b/llvm/test/Transforms/InstCombine/shuffle-cast.ll
new file mode 100644
index 0000000000000..e4b21ff9e1118
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/shuffle-cast.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S -data-layout="e" | FileCheck %s --check-prefixes=ANY,LE
+; RUN: opt < %s -instcombine -S -data-layout="E" | FileCheck %s --check-prefixes=ANY,BE
+
+define <4 x i16> @trunc_little_endian(<4 x i32> %x) {
+; ANY-LABEL: @trunc_little_endian(
+; ANY-NEXT:    [[B:%.*]] = bitcast <4 x i32> [[X:%.*]] to <8 x i16>
+; ANY-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[B]], <8 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; ANY-NEXT:    ret <4 x i16> [[R]]
+;
+  %b = bitcast <4 x i32> %x to <8 x i16>
+  %r = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  ret <4 x i16> %r
+}
+
+define <4 x i16> @trunc_big_endian(<4 x i32> %x) {
+; ANY-LABEL: @trunc_big_endian(
+; ANY-NEXT:    [[B:%.*]] = bitcast <4 x i32> [[X:%.*]] to <8 x i16>
+; ANY-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[B]], <8 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; ANY-NEXT:    ret <4 x i16> [[R]]
+;
+  %b = bitcast <4 x i32> %x to <8 x i16>
+  %r = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  ret <4 x i16> %r
+}
+
+declare void @use_v8i16(<8 x i16>)
+
+define <2 x i16> @trunc_little_endian_extra_use(<2 x i64> %x) {
+; ANY-LABEL: @trunc_little_endian_extra_use(
+; ANY-NEXT:    [[B:%.*]] = bitcast <2 x i64> [[X:%.*]] to <8 x i16>
+; ANY-NEXT:    call void @use_v8i16(<8 x i16> [[B]])
+; ANY-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[B]], <8 x i16> undef, <2 x i32> <i32 0, i32 4>
+; ANY-NEXT:    ret <2 x i16> [[R]]
+;
+  %b = bitcast <2 x i64> %x to <8 x i16>
+  call void @use_v8i16(<8 x i16> %b)
+  %r = shufflevector <8 x i16> %b, <8 x i16> undef, <2 x i32> <i32 0, i32 4>
+  ret <2 x i16> %r
+}
+
+declare void @use_v12i11(<12 x i11>)
+
+define <4 x i11> @trunc_big_endian_extra_use(<4 x i33> %x) {
+; ANY-LABEL: @trunc_big_endian_extra_use(
+; ANY-NEXT:    [[B:%.*]] = bitcast <4 x i33> [[X:%.*]] to <12 x i11>
+; ANY-NEXT:    call void @use_v12i11(<12 x i11> [[B]])
+; ANY-NEXT:    [[R:%.*]] = shufflevector <12 x i11> [[B]], <12 x i11> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+; ANY-NEXT:    ret <4 x i11> [[R]]
+;
+  %b = bitcast <4 x i33> %x to <12 x i11>
+  call void @use_v12i11(<12 x i11> %b)
+  %r = shufflevector <12 x i11> %b, <12 x i11> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+  ret <4 x i11> %r
+}
+
diff --git a/llvm/test/Transforms/InstCombine/shufflevec-bitcast.ll b/llvm/test/Transforms/InstCombine/shufflevec-bitcast.ll
index 0f0365a07fb40..5af4f50a191fd 100644
--- a/llvm/test/Transforms/InstCombine/shufflevec-bitcast.ll
+++ b/llvm/test/Transforms/InstCombine/shufflevec-bitcast.ll
@@ -1,12 +1,18 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 
-define void @test(<16 x i8> %w, i32* %o1, float* %o2) {
-
-; CHECK:       %v.bc = bitcast <16 x i8> %w to <4 x i32>
-; CHECK-NEXT:  %v.extract = extractelement <4 x i32> %v.bc, i32 3
-; CHECK-NEXT:  %v.bc{{[0-9]*}} = bitcast <16 x i8> %w to <4 x float>
-; CHECK-NEXT:  %v.extract{{[0-9]*}} = extractelement <4 x float> %v.bc{{[0-9]*}}, i32 3
+declare void @use(<4 x i16>)
 
+define void @test(<16 x i8> %w, i32* %o1, float* %o2) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:    [[V_BC:%.*]] = bitcast <16 x i8> [[W:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[V_EXTRACT:%.*]] = extractelement <4 x i32> [[V_BC]], i32 3
+; CHECK-NEXT:    [[V_BC1:%.*]] = bitcast <16 x i8> [[W]] to <4 x float>
+; CHECK-NEXT:    [[V_EXTRACT2:%.*]] = extractelement <4 x float> [[V_BC1]], i32 3
+; CHECK-NEXT:    store i32 [[V_EXTRACT]], i32* [[O1:%.*]], align 4
+; CHECK-NEXT:    store float [[V_EXTRACT2]], float* [[O2:%.*]], align 4
+; CHECK-NEXT:    ret void
+;
   %v = shufflevector <16 x i8> %w, <16 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
   %f = bitcast <4 x i8> %v to float
   %i = bitcast <4 x i8> %v to i32
@@ -14,3 +20,91 @@ define void @test(<16 x i8> %w, i32* %o1, float* %o2) {
   store float %f, float* %o2, align 4
   ret void
 }
+
+; Shuffle-of-bitcast-splat --> splat-bitcast
+
+define <4 x i16> @splat_bitcast_operand(<8 x i8> %x) {
+; CHECK-LABEL: @splat_bitcast_operand(
+; CHECK-NEXT:    [[S1:%.*]] = shufflevector <8 x i8> [[X:%.*]], <8 x i8> undef, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[S2:%.*]] = bitcast <8 x i8> [[S1]] to <4 x i16>
+; CHECK-NEXT:    ret <4 x i16> [[S2]]
+;
+  %s1 = shufflevector <8 x i8> %x, <8 x i8> undef, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %bc = bitcast <8 x i8> %s1 to <4 x i16>
+  %s2 = shufflevector <4 x i16> %bc, <4 x i16> undef, <4 x i32> <i32 0, i32 2, i32 1, i32 0>
+  ret <4 x i16> %s2
+}
+
+; Shuffle-of-bitcast-splat --> splat-bitcast
+
+define <4 x i16> @splat_bitcast_operand_uses(<8 x i8> %x) {
+; CHECK-LABEL: @splat_bitcast_operand_uses(
+; CHECK-NEXT:    [[S1:%.*]] = shufflevector <8 x i8> [[X:%.*]], <8 x i8> undef, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[BC:%.*]] = bitcast <8 x i8> [[S1]] to <4 x i16>
+; CHECK-NEXT:    call void @use(<4 x i16> [[BC]])
+; CHECK-NEXT:    [[S2:%.*]] = bitcast <8 x i8> [[S1]] to <4 x i16>
+; CHECK-NEXT:    ret <4 x i16> [[S2]]
+;
+  %s1 = shufflevector <8 x i8> %x, <8 x i8> undef, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %bc = bitcast <8 x i8> %s1 to <4 x i16>
+  call void @use(<4 x i16> %bc)
+  %s2 = shufflevector <4 x i16> %bc, <4 x i16> undef, <4 x i32> <i32 0, i32 2, i32 1, i32 0>
+  ret <4 x i16> %s2
+}
+
+; Shuffle-of-bitcast-splat --> splat-bitcast
+
+define <4 x i32> @splat_bitcast_operand_same_size_src_elt(<4 x float> %x) {
+; CHECK-LABEL: @splat_bitcast_operand_same_size_src_elt(
+; CHECK-NEXT:    [[S1:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> undef, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    [[S2:%.*]] = bitcast <4 x float> [[S1]] to <4 x i32>
+; CHECK-NEXT:    ret <4 x i32> [[S2]]
+;
+  %s1 = shufflevector <4 x float> %x, <4 x float> undef, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  %bc = bitcast <4 x float> %s1 to <4 x i32>
+  %s2 = shufflevector <4 x i32> %bc, <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 1, i32 0>
+  ret <4 x i32> %s2
+}
+
+; Scaled mask is inverse of first mask.
+
+define <4 x i32> @shuf_bitcast_operand(<16 x i8> %x) {
+; CHECK-LABEL: @shuf_bitcast_operand(
+; CHECK-NEXT:    [[S2:%.*]] = bitcast <16 x i8> [[X:%.*]] to <4 x i32>
+; CHECK-NEXT:    ret <4 x i32> [[S2]]
+;
+  %s1 = shufflevector <16 x i8> %x, <16 x i8> undef, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+  %bc = bitcast <16 x i8> %s1 to <4 x i32>
+  %s2 = shufflevector <4 x i32> %bc, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  ret <4 x i32> %s2
+}
+
+; TODO: Could allow fold for length-changing shuffles.
+
+define <5 x i16> @splat_bitcast_operand_change_type(<8 x i8> %x) {
+; CHECK-LABEL: @splat_bitcast_operand_change_type(
+; CHECK-NEXT:    [[S1:%.*]] = shufflevector <8 x i8> [[X:%.*]], <8 x i8> undef, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[BC:%.*]] = bitcast <8 x i8> [[S1]] to <4 x i16>
+; CHECK-NEXT:    [[S2:%.*]] = shufflevector <4 x i16> [[BC]], <4 x i16> undef, <5 x i32> <i32 0, i32 2, i32 1, i32 0, i32 3>
+; CHECK-NEXT:    ret <5 x i16> [[S2]]
+;
+  %s1 = shufflevector <8 x i8> %x, <8 x i8> undef, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %bc = bitcast <8 x i8> %s1 to <4 x i16>
+  %s2 = shufflevector <4 x i16> %bc, <4 x i16> undef, <5 x i32> <i32 0, i32 2, i32 1, i32 0, i32 3>
+  ret <5 x i16> %s2
+}
+
+; TODO: Could allow fold for cast to narrow element.
+
+define <4 x i16> @splat_bitcast_operand_wider_src_elt(<2 x i32> %x) {
+; CHECK-LABEL: @splat_bitcast_operand_wider_src_elt(
+; CHECK-NEXT:    [[S1:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> undef, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:    [[BC:%.*]] = bitcast <2 x i32> [[S1]] to <4 x i16>
+; CHECK-NEXT:    [[S2:%.*]] = shufflevector <4 x i16> [[BC]], <4 x i16> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT:    ret <4 x i16> [[S2]]
+;
+  %s1 = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+  %bc = bitcast <2 x i32> %s1 to <4 x i16>
+  %s2 = shufflevector <4 x i16> %bc, <4 x i16> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  ret <4 x i16> %s2
+}
diff --git a/llvm/test/Transforms/InstCombine/sub-of-negatible.ll b/llvm/test/Transforms/InstCombine/sub-of-negatible.ll
index 01084312c9f8a..897242e4837b6 100644
--- a/llvm/test/Transforms/InstCombine/sub-of-negatible.ll
+++ b/llvm/test/Transforms/InstCombine/sub-of-negatible.ll
@@ -159,8 +159,8 @@ define i8 @n8(i8 %x, i1 %y, i8 %z) {
 ; x - (y - z) -> x - y + z -> x + (z - y)
 define i8 @t9(i8 %x, i8 %y) {
 ; CHECK-LABEL: @t9(
-; CHECK-NEXT:    [[T01:%.*]] = sub i8 [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    ret i8 [[T01]]
+; CHECK-NEXT:    [[T0_NEG:%.*]] = sub i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i8 [[T0_NEG]]
 ;
   %t0 = sub i8 %y, %x
   %t1 = sub i8 0, %t0
@@ -375,3 +375,51 @@ define i8 @n21(i8 %x, i16 %y) {
   %t2 = sub i8 %x, %t1
   ret i8 %t2
 }
+
+define i4 @negate_xor(i4 %x) {
+; CHECK-LABEL: @negate_xor(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i4 [[X:%.*]], -6
+; CHECK-NEXT:    [[O_NEG:%.*]] = add i4 [[TMP1]], 1
+; CHECK-NEXT:    ret i4 [[O_NEG]]
+;
+  %o = xor i4 %x, 5
+  %r = sub i4 0, %o
+  ret i4 %r
+}
+
+define <2 x i4> @negate_xor_vec(<2 x i4> %x) {
+; CHECK-LABEL: @negate_xor_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i4> [[X:%.*]], <i4 -6, i4 5>
+; CHECK-NEXT:    [[O_NEG:%.*]] = add <2 x i4> [[TMP1]], <i4 1, i4 1>
+; CHECK-NEXT:    ret <2 x i4> [[O_NEG]]
+;
+  %o = xor <2 x i4> %x, <i4 5, i4 10>
+  %r = sub <2 x i4> zeroinitializer, %o
+  ret <2 x i4> %r
+}
+
+define i8 @negate_xor_use(i8 %x) {
+; CHECK-LABEL: @negate_xor_use(
+; CHECK-NEXT:    [[O:%.*]] = xor i8 [[X:%.*]], 5
+; CHECK-NEXT:    call void @use8(i8 [[O]])
+; CHECK-NEXT:    [[R:%.*]] = sub i8 0, [[O]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %o = xor i8 %x, 5
+  call void @use8(i8 %o)
+  %r = sub i8 0, %o
+  ret i8 %r
+}
+
+define i4 @negate_shl_xor(i4 %x, i4 %y) {
+; CHECK-LABEL: @negate_shl_xor(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i4 [[X:%.*]], -6
+; CHECK-NEXT:    [[O_NEG:%.*]] = add i4 [[TMP1]], 1
+; CHECK-NEXT:    [[S_NEG:%.*]] = shl i4 [[O_NEG]], [[Y:%.*]]
+; CHECK-NEXT:    ret i4 [[S_NEG]]
+;
+  %o = xor i4 %x, 5
+  %s = shl i4 %o, %y
+  %r = sub i4 0, %s
+  ret i4 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/trunc-extractelement.ll b/llvm/test/Transforms/InstCombine/trunc-extractelement.ll
new file mode 100644
index 0000000000000..c44577d1f1d1b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/trunc-extractelement.ll
@@ -0,0 +1,178 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S -data-layout="e" | FileCheck %s --check-prefixes=ANY,LE
+; RUN: opt < %s -instcombine -S -data-layout="E" | FileCheck %s --check-prefixes=ANY,BE
+
+define i32 @shrinkExtractElt_i64_to_i32_0(<3 x i64> %x) {
+; LE-LABEL: @shrinkExtractElt_i64_to_i32_0(
+; LE-NEXT:    [[TMP1:%.*]] = bitcast <3 x i64> [[X:%.*]] to <6 x i32>
+; LE-NEXT:    [[T:%.*]] = extractelement <6 x i32> [[TMP1]], i32 0
+; LE-NEXT:    ret i32 [[T]]
+;
+; BE-LABEL: @shrinkExtractElt_i64_to_i32_0(
+; BE-NEXT:    [[TMP1:%.*]] = bitcast <3 x i64> [[X:%.*]] to <6 x i32>
+; BE-NEXT:    [[T:%.*]] = extractelement <6 x i32> [[TMP1]], i32 1
+; BE-NEXT:    ret i32 [[T]]
+;
+  %e = extractelement <3 x i64> %x, i32 0
+  %t = trunc i64 %e to i32
+  ret i32 %t
+}
+
+define i32 @shrinkExtractElt_i64_to_i32_1(<3 x i64> %x) {
+; LE-LABEL: @shrinkExtractElt_i64_to_i32_1(
+; LE-NEXT:    [[TMP1:%.*]] = bitcast <3 x i64> [[X:%.*]] to <6 x i32>
+; LE-NEXT:    [[T:%.*]] = extractelement <6 x i32> [[TMP1]], i32 2
+; LE-NEXT:    ret i32 [[T]]
+;
+; BE-LABEL: @shrinkExtractElt_i64_to_i32_1(
+; BE-NEXT:    [[TMP1:%.*]] = bitcast <3 x i64> [[X:%.*]] to <6 x i32>
+; BE-NEXT:    [[T:%.*]] = extractelement <6 x i32> [[TMP1]], i32 3
+; BE-NEXT:    ret i32 [[T]]
+;
+  %e = extractelement <3 x i64> %x, i32 1
+  %t = trunc i64 %e to i32
+  ret i32 %t
+}
+
+define i32 @shrinkExtractElt_i64_to_i32_2(<3 x i64> %x) {
+; LE-LABEL: @shrinkExtractElt_i64_to_i32_2(
+; LE-NEXT:    [[TMP1:%.*]] = bitcast <3 x i64> [[X:%.*]] to <6 x i32>
+; LE-NEXT:    [[T:%.*]] = extractelement <6 x i32> [[TMP1]], i32 4
+; LE-NEXT:    ret i32 [[T]]
+;
+; BE-LABEL: @shrinkExtractElt_i64_to_i32_2(
+; BE-NEXT:    [[TMP1:%.*]] = bitcast <3 x i64> [[X:%.*]] to <6 x i32>
+; BE-NEXT:    [[T:%.*]] = extractelement <6 x i32> [[TMP1]], i32 5
+; BE-NEXT:    ret i32 [[T]]
+;
+  %e = extractelement <3 x i64> %x, i32 2
+  %t = trunc i64 %e to i32
+  ret i32 %t
+}
+
+define i16 @shrinkExtractElt_i64_to_i16_0(<3 x i64> %x) {
+; LE-LABEL: @shrinkExtractElt_i64_to_i16_0(
+; LE-NEXT:    [[TMP1:%.*]] = bitcast <3 x i64> [[X:%.*]] to <12 x i16>
+; LE-NEXT:    [[T:%.*]] = extractelement <12 x i16> [[TMP1]], i32 0
+; LE-NEXT:    ret i16 [[T]]
+;
+; BE-LABEL: @shrinkExtractElt_i64_to_i16_0(
+; BE-NEXT:    [[TMP1:%.*]] = bitcast <3 x i64> [[X:%.*]] to <12 x i16>
+; BE-NEXT:    [[T:%.*]] = extractelement <12 x i16> [[TMP1]], i32 3
+; BE-NEXT:    ret i16 [[T]]
+;
+  %e = extractelement <3 x i64> %x, i16 0
+  %t = trunc i64 %e to i16
+  ret i16 %t
+}
+
+define i16 @shrinkExtractElt_i64_to_i16_1(<3 x i64> %x) {
+; LE-LABEL: @shrinkExtractElt_i64_to_i16_1(
+; LE-NEXT:    [[TMP1:%.*]] = bitcast <3 x i64> [[X:%.*]] to <12 x i16>
+; LE-NEXT:    [[T:%.*]] = extractelement <12 x i16> [[TMP1]], i32 4
+; LE-NEXT:    ret i16 [[T]]
+;
+; BE-LABEL: @shrinkExtractElt_i64_to_i16_1(
+; BE-NEXT:    [[TMP1:%.*]] = bitcast <3 x i64> [[X:%.*]] to <12 x i16>
+; BE-NEXT:    [[T:%.*]] = extractelement <12 x i16> [[TMP1]], i32 7
+; BE-NEXT:    ret i16 [[T]]
+;
+  %e = extractelement <3 x i64> %x, i16 1
+  %t = trunc i64 %e to i16
+  ret i16 %t
+}
+
+define i16 @shrinkExtractElt_i64_to_i16_2(<3 x i64> %x) {
+; LE-LABEL: @shrinkExtractElt_i64_to_i16_2(
+; LE-NEXT:    [[TMP1:%.*]] = bitcast <3 x i64> [[X:%.*]] to <12 x i16>
+; LE-NEXT:    [[T:%.*]] = extractelement <12 x i16> [[TMP1]], i32 8
+; LE-NEXT:    ret i16 [[T]]
+;
+; BE-LABEL: @shrinkExtractElt_i64_to_i16_2(
+; BE-NEXT:    [[TMP1:%.*]] = bitcast <3 x i64> [[X:%.*]] to <12 x i16>
+; BE-NEXT:    [[T:%.*]] = extractelement <12 x i16> [[TMP1]], i32 11
+; BE-NEXT:    ret i16 [[T]]
+;
+  %e = extractelement <3 x i64> %x, i16 2
+  %t = trunc i64 %e to i16
+  ret i16 %t
+}
+
+; Crazy types may be ok.
+define i11 @shrinkExtractElt_i33_to_11_2(<3 x i33> %x) {
+; LE-LABEL: @shrinkExtractElt_i33_to_11_2(
+; LE-NEXT:    [[TMP1:%.*]] = bitcast <3 x i33> [[X:%.*]] to <9 x i11>
+; LE-NEXT:    [[T:%.*]] = extractelement <9 x i11> [[TMP1]], i32 6
+; LE-NEXT:    ret i11 [[T]]
+;
+; BE-LABEL: @shrinkExtractElt_i33_to_11_2(
+; BE-NEXT:    [[TMP1:%.*]] = bitcast <3 x i33> [[X:%.*]] to <9 x i11>
+; BE-NEXT:    [[T:%.*]] = extractelement <9 x i11> [[TMP1]], i32 8
+; BE-NEXT:    ret i11 [[T]]
+;
+  %e = extractelement <3 x i33> %x, i16 2
+  %t = trunc i33 %e to i11
+  ret i11 %t
+}
+
+; Do not optimize if it would result in an invalid bitcast instruction.
+define i13 @shrinkExtractElt_i67_to_i13_2(<3 x i67> %x) {
+; ANY-LABEL: @shrinkExtractElt_i67_to_i13_2(
+; ANY-NEXT:    [[E:%.*]] = extractelement <3 x i67> [[X:%.*]], i459 2
+; ANY-NEXT:    [[T:%.*]] = trunc i67 [[E]] to i13
+; ANY-NEXT:    ret i13 [[T]]
+;
+  %e = extractelement <3 x i67> %x, i459 2
+  %t = trunc i67 %e to i13
+  ret i13 %t
+}
+
+; Do not optimize if the bitcast instruction would be valid, but the
+; transform would be wrong.
+define i30 @shrinkExtractElt_i40_to_i30_1(<3 x i40> %x) {
+; ANY-LABEL: @shrinkExtractElt_i40_to_i30_1(
+; ANY-NEXT:    [[E:%.*]] = extractelement <3 x i40> [[X:%.*]], i32 1
+; ANY-NEXT:    [[T:%.*]] = trunc i40 [[E]] to i30
+; ANY-NEXT:    ret i30 [[T]]
+;
+  %e = extractelement <3 x i40> %x, i32 1
+  %t = trunc i40 %e to i30
+  ret i30 %t
+}
+
+; Do not canonicalize if that would increase the instruction count.
+declare void @use(i64)
+define i16 @shrinkExtractElt_i64_to_i16_2_extra_use(<3 x i64> %x) {
+; ANY-LABEL: @shrinkExtractElt_i64_to_i16_2_extra_use(
+; ANY-NEXT:    [[E:%.*]] = extractelement <3 x i64> [[X:%.*]], i64 2
+; ANY-NEXT:    call void @use(i64 [[E]])
+; ANY-NEXT:    [[T:%.*]] = trunc i64 [[E]] to i16
+; ANY-NEXT:    ret i16 [[T]]
+;
+  %e = extractelement <3 x i64> %x, i64 2
+  call void @use(i64 %e)
+  %t = trunc i64 %e to i16
+  ret i16 %t
+}
+
+; Check to ensure PR45314 remains fixed.
+define <4 x i64> @PR45314(<4 x i64> %x) {
+; LE-LABEL: @PR45314(
+; LE-NEXT:    [[TMP1:%.*]] = bitcast <4 x i64> [[X:%.*]] to <8 x i32>
+; LE-NEXT:    [[S:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> zeroinitializer
+; LE-NEXT:    [[B:%.*]] = bitcast <8 x i32> [[S]] to <4 x i64>
+; LE-NEXT:    ret <4 x i64> [[B]]
+;
+; BE-LABEL: @PR45314(
+; BE-NEXT:    [[TMP1:%.*]] = bitcast <4 x i64> [[X:%.*]] to <8 x i32>
+; BE-NEXT:    [[S:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; BE-NEXT:    [[B:%.*]] = bitcast <8 x i32> [[S]] to <4 x i64>
+; BE-NEXT:    ret <4 x i64> [[B]]
+;
+  %e = extractelement <4 x i64> %x, i32 0
+  %t = trunc i64 %e to i32
+  %i = insertelement <8 x i32> undef, i32 %t, i32 0
+  %s = shufflevector <8 x i32> %i, <8 x i32> undef, <8 x i32> zeroinitializer
+  %b = bitcast <8 x i32> %s to <4 x i64>
+  ret <4 x i64> %b
+}
diff --git a/llvm/test/Transforms/InstCombine/vector-xor.ll b/llvm/test/Transforms/InstCombine/vector-xor.ll
index c10e56a6561a2..52e904902b51f 100644
--- a/llvm/test/Transforms/InstCombine/vector-xor.ll
+++ b/llvm/test/Transforms/InstCombine/vector-xor.ll
@@ -140,7 +140,7 @@ define <4 x i32> @test_v4i32_not_ashr_negative_const(<4 x i32> %a0) {
 
 define <4 x i32> @test_v4i32_not_ashr_negative_const_undef(<4 x i32> %a0) {
 ; CHECK-LABEL: @test_v4i32_not_ashr_negative_const_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i32> <i32 2, i32 4, i32 undef, i32 8>, [[A0:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i32> <i32 2, i32 4, i32 0, i32 8>, [[A0:%.*]]
 ; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 ;
   %1 = ashr <4 x i32> <i32 -3, i32 -5, i32 undef, i32 -9>, %a0
diff --git a/llvm/test/Transforms/InstCombine/xor-of-icmps-with-extra-uses.ll b/llvm/test/Transforms/InstCombine/xor-of-icmps-with-extra-uses.ll
index a795ead93e546..fef919678742c 100644
--- a/llvm/test/Transforms/InstCombine/xor-of-icmps-with-extra-uses.ll
+++ b/llvm/test/Transforms/InstCombine/xor-of-icmps-with-extra-uses.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -instcombine -S | FileCheck %s
+; RUN: opt < %s -instcombine -instcombine-infinite-loop-threshold=2 -S | FileCheck %s
 
 ; These xor-of-icmps could be replaced with and-of-icmps, but %cond0 has extra
 ; uses, so we don't consider it, even though some cases are freely invertible.
diff --git a/llvm/test/Transforms/InstSimplify/compare.ll b/llvm/test/Transforms/InstSimplify/compare.ll
index 86e1fef22f602..5b997a5fc491d 100644
--- a/llvm/test/Transforms/InstSimplify/compare.ll
+++ b/llvm/test/Transforms/InstSimplify/compare.ll
@@ -135,7 +135,6 @@ define i1 @gep9(i8* %ptr) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    ret i1 true
 ;
-
 entry:
   %first1 = getelementptr inbounds i8, i8* %ptr, i32 0
   %first2 = getelementptr inbounds i8, i8* %first1, i32 1
@@ -156,7 +155,6 @@ define i1 @gep10(i8* %ptr) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    ret i1 true
 ;
-
 entry:
   %first1 = getelementptr inbounds i8, i8* %ptr, i32 -2
   %first2 = getelementptr inbounds i8, i8* %first1, i32 44
@@ -173,7 +171,6 @@ define i1 @gep11(i8* %ptr) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    ret i1 true
 ;
-
 entry:
   %first1 = getelementptr inbounds i8, i8* %ptr, i32 -2
   %last1 = getelementptr inbounds i8, i8* %ptr, i32 48
@@ -191,7 +188,6 @@ define i1 @gep12(i8* %ptr) {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8* [[FIRST1]], [[LAST2]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
-
 entry:
   %first1 = getelementptr inbounds i8, i8* %ptr, i32 -2
   %last1 = getelementptr inbounds i8, i8* %ptr, i32 48
@@ -278,7 +274,6 @@ define i1 @gep16_no_null_opt(i8* %ptr, i32 %a) #0 {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8* [[X]], null
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
-; CHECK getelementptr inbounds i8, i8* %ptr, i32 %b
   %b = or i32 %a, 1
   %x = getelementptr inbounds i8, i8* %ptr, i32 %b
   %cmp = icmp eq i8* %x, null
diff --git a/llvm/test/Transforms/InstSimplify/constfold-constrained.ll b/llvm/test/Transforms/InstSimplify/constfold-constrained.ll
new file mode 100644
index 0000000000000..d53de99fe0b5e
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/constfold-constrained.ll
@@ -0,0 +1,244 @@
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+
+; Verify that floor(10.1) is folded to 10.0 when the exception behavior is 'ignore'.
+define double @floor_01() #0 {
+entry:
+  %result = call double @llvm.experimental.constrained.floor.f64(
+                                               double 1.010000e+01,
+                                               metadata !"fpexcept.ignore") #0
+  ret double %result
+  ; CHECK-LABEL: @floor_01
+  ; CHECK: ret double 1.000000e+01
+}
+
+; Verify that floor(-10.1) is folded to -11.0 when the exception behavior is not 'ignore'.
+define double @floor_02() #0 {
+entry:
+  %result = call double @llvm.experimental.constrained.floor.f64(
+                                               double -1.010000e+01,
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+  ; CHECK-LABEL: @floor_02
+  ; CHECK: ret double -1.100000e+01
+}
+
+; Verify that ceil(10.1) is folded to 11.0 when the exception behavior is 'ignore'.
+define double @ceil_01() #0 {
+entry:
+  %result = call double @llvm.experimental.constrained.ceil.f64(
+                                               double 1.010000e+01,
+                                               metadata !"fpexcept.ignore") #0
+  ret double %result
+  ; CHECK-LABEL: @ceil_01
+  ; CHECK: ret double 1.100000e+01
+}
+
+; Verify that ceil(-10.1) is folded to -10.0 when the exception behavior is not 'ignore'.
+define double @ceil_02() #0 {
+entry:
+  %result = call double @llvm.experimental.constrained.ceil.f64(
+                                               double -1.010000e+01,
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+  ; CHECK-LABEL: @ceil_02
+  ; CHECK: ret double -1.000000e+01
+}
+
+; Verify that trunc(10.1) is folded to 10.0 when the exception behavior is 'ignore'.
+define double @trunc_01() #0 {
+entry:
+  %result = call double @llvm.experimental.constrained.trunc.f64(
+                                               double 1.010000e+01,
+                                               metadata !"fpexcept.ignore") #0
+  ret double %result
+  ; CHECK-LABEL: @trunc_01
+  ; CHECK: ret double 1.000000e+01
+}
+
+; Verify that trunc(-10.1) is folded to -10.0 when the exception behavior is NOT 'ignore'.
+define double @trunc_02() #0 {
+entry:
+  %result = call double @llvm.experimental.constrained.trunc.f64(
+                                               double -1.010000e+01,
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+  ; CHECK-LABEL: @trunc_02
+  ; CHECK: ret double -1.000000e+01
+}
+
+; Verify that round(10.5) is folded to 11.0 when the exception behavior is 'ignore'.
+define double @round_01() #0 {
+entry:
+  %result = call double @llvm.experimental.constrained.round.f64(
+                                               double 1.050000e+01,
+                                               metadata !"fpexcept.ignore") #0
+  ret double %result
+  ; CHECK-LABEL: @round_01
+  ; CHECK: ret double 1.100000e+01
+}
+
+; Verify that floor(-10.5) is folded to -11.0 when the exception behavior is NOT 'ignore'.
+define double @round_02() #0 {
+entry:
+  %result = call double @llvm.experimental.constrained.round.f64(
+                                               double -1.050000e+01,
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+  ; CHECK-LABEL: @round_02
+  ; CHECK: ret double -1.100000e+01
+}
+
+; Verify that nearbyint(10.5) is folded to 11.0 when the rounding mode is 'upward'.
+define double @nearbyint_01() #0 {
+entry:
+  %result = call double @llvm.experimental.constrained.nearbyint.f64(
+                                               double 1.050000e+01,
+                                               metadata !"round.upward",
+                                               metadata !"fpexcept.ignore") #0
+  ret double %result
+  ; CHECK-LABEL: @nearbyint_01
+  ; CHECK: ret double 1.100000e+01
+}
+
+; Verify that nearbyint(10.5) is folded to 10.0 when the rounding mode is 'downward'.
+define double @nearbyint_02() #0 {
+entry:
+  %result = call double @llvm.experimental.constrained.nearbyint.f64(
+                                               double 1.050000e+01,
+                                               metadata !"round.downward",
+                                               metadata !"fpexcept.maytrap") #0
+  ret double %result
+  ; CHECK-LABEL: @nearbyint_02
+  ; CHECK: ret double 1.000000e+01
+}
+
+; Verify that nearbyint(10.5) is folded to 10.0 when the rounding mode is 'towardzero'.
+define double @nearbyint_03() #0 {
+entry:
+  %result = call double @llvm.experimental.constrained.nearbyint.f64(
+                                               double 1.050000e+01,
+                                               metadata !"round.towardzero",
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+  ; CHECK-LABEL: @nearbyint_03
+  ; CHECK: ret double 1.000000e+01
+}
+
+; Verify that nearbyint(10.5) is folded to 10.0 when the rounding mode is 'tonearest'.
+define double @nearbyint_04() #0 {
+entry:
+  %result = call double @llvm.experimental.constrained.nearbyint.f64(
+                                               double 1.050000e+01,
+                                               metadata !"round.tonearest",
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+  ; CHECK-LABEL: @nearbyint_04
+  ; CHECK: ret double 1.000000e+01
+}
+
+; Verify that nearbyint(10.5) is NOT folded if the rounding mode is 'dynamic'.
+define double @nearbyint_05() #0 {
+entry:
+  %result = call double @llvm.experimental.constrained.nearbyint.f64(
+                                               double 1.050000e+01,
+                                               metadata !"round.dynamic",
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+  ; CHECK-LABEL: @nearbyint_05
+  ; CHECK: [[VAL:%.+]] = {{.*}}call double @llvm.experimental.constrained.nearbyint
+  ; CHECK: ret double [[VAL]]
+}
+
+; Verify that trunc(SNAN) is NOT folded if the exception behavior mode is not 'ignore'.
+define double @nonfinite_01() #0 {
+entry:
+  %result = call double @llvm.experimental.constrained.trunc.f64(
+                                               double 0x7ff4000000000000,
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+  ; CHECK-LABEL: @nonfinite_01
+  ; CHECK: [[VAL:%.+]] = {{.*}}call double @llvm.experimental.constrained.trunc
+  ; CHECK: ret double [[VAL]]
+}
+
+; Verify that trunc(SNAN) is folded to QNAN if the exception behavior mode is 'ignore'.
+define double @nonfinite_02() #0 {
+entry:
+  %result = call double @llvm.experimental.constrained.trunc.f64(
+                                               double 0x7ff4000000000000,
+                                               metadata !"fpexcept.ignore") #0
+  ret double %result
+  ; CHECK-LABEL: @nonfinite_02
+  ; CHECK: ret double 0x7FF8000000000000
+}
+
+; Verify that trunc(QNAN) is folded even if the exception behavior mode is not 'ignore'.
+define double @nonfinite_03() #0 {
+entry:
+  %result = call double @llvm.experimental.constrained.trunc.f64(
+                                               double 0x7ff8000000000000,
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+  ; CHECK-LABEL: @nonfinite_03
+  ; CHECK: ret double 0x7FF8000000000000
+}
+
+; Verify that trunc(+Inf) is folded even if the exception behavior mode is not 'ignore'.
+define double @nonfinite_04() #0 {
+entry:
+  %result = call double @llvm.experimental.constrained.trunc.f64(
+                                               double 0x7ff0000000000000,
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+  ; CHECK-LABEL: @nonfinite_04
+  ; CHECK: ret double 0x7FF0000000000000
+}
+
+; Verify that rint(10) is folded to 10.0 when the rounding mode is 'tonearest'.
+define double @rint_01() #0 {
+entry:
+  %result = call double @llvm.experimental.constrained.rint.f64(
+                                               double 1.000000e+01,
+                                               metadata !"round.tonearest",
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+  ; CHECK-LABEL: @rint_01
+  ; CHECK: ret double 1.000000e+01
+}
+
+; Verify that rint(10.1) is NOT folded to 10.0 when the exception behavior is 'strict'.
+define double @rint_02() #0 {
+entry:
+  %result = call double @llvm.experimental.constrained.rint.f64(
+                                               double 1.010000e+01,
+                                               metadata !"round.tonearest",
+                                               metadata !"fpexcept.strict") #0
+  ret double %result
+  ; CHECK-LABEL: @rint_02
+  ; CHECK: [[VAL:%.+]] = {{.*}}call double @llvm.experimental.constrained.rint
+  ; CHECK: ret double [[VAL]]
+}
+
+; Verify that rint(10.1) is folded to 10.0 when the exception behavior is not 'strict'.
+define double @rint_03() #0 {
+entry:
+  %result = call double @llvm.experimental.constrained.rint.f64(
+                                               double 1.010000e+01,
+                                               metadata !"round.tonearest",
+                                               metadata !"fpexcept.maytrap") #0
+  ret double %result
+  ; CHECK-LABEL: @rint_03
+  ; CHECK: ret double 1.000000e+01
+}
+
+
+attributes #0 = { strictfp }
+
+declare double @llvm.experimental.constrained.nearbyint.f64(double, metadata, metadata)
+declare double @llvm.experimental.constrained.floor.f64(double, metadata)
+declare double @llvm.experimental.constrained.ceil.f64(double, metadata)
+declare double @llvm.experimental.constrained.trunc.f64(double, metadata)
+declare double @llvm.experimental.constrained.round.f64(double, metadata)
+declare double @llvm.experimental.constrained.rint.f64(double, metadata, metadata)
+
diff --git a/llvm/test/Transforms/JumpThreading/is_constant.ll b/llvm/test/Transforms/JumpThreading/is_constant.ll
new file mode 100644
index 0000000000000..12d0451ef0956
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/is_constant.ll
@@ -0,0 +1,40 @@
+; RUN: opt -jump-threading -S -verify < %s | FileCheck %s
+
+; This is reduced from the Linux net/ipv4/tcp.c file built with ASAN. We
+; don't want jump threading to split up a basic block which has a PHI node with
+; at least one constant incoming value, whose value is used by an is.constant
+; intrinsic with non-local uses. It could lead to later passes no DCE'ing
+; invalid paths.
+
+; CHECK-LABEL:    define void @test1(
+; CHECK-LABEL:    bb_cond:
+; CHECK-NOT:        %sext = phi i64 [ %var, %entry ]
+; CHECK-NEXT:       %sext = phi i64 [ 24, %bb_constant ], [ %var, %entry ]
+; CHECK-NEXT:       %cond2 = icmp
+; CHECK-NEXT:       call i1 @llvm.is.constant.i64(
+
+define void @test1(i32 %a, i64 %var) {
+entry:
+  %cond1 = icmp ugt i32 %a, 24
+  br i1 %cond1, label %bb_constant, label %bb_cond
+
+bb_constant:
+  br label %bb_cond
+
+bb_cond:
+  %sext = phi i64 [ 24, %bb_constant ], [ %var, %entry ]
+  %cond2 = icmp ugt i64 %sext, 24
+  %is_constant = call i1 @llvm.is.constant.i64(i64 %sext)
+  br i1 %cond2, label %bb_then, label %bb_else
+
+bb_then:
+  unreachable
+
+bb_else:
+  unreachable
+}
+
+; Function Attrs: nounwind readnone willreturn
+declare i1 @llvm.is.constant.i64(i64) #0
+
+attributes #0 = { convergent nounwind readnone willreturn }
diff --git a/llvm/test/Transforms/JumpThreading/select-unfold-msan.ll b/llvm/test/Transforms/JumpThreading/select-unfold-msan.ll
new file mode 100644
index 0000000000000..ea336e0f0f7ed
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/select-unfold-msan.ll
@@ -0,0 +1,28 @@
+; PR45220
+; RUN: opt -S -jump-threading < %s | FileCheck %s
+
+declare i1 @NOP()
+
+define dso_local i32 @f(i1 %b, i1 %u) sanitize_memory {
+entry:
+  br i1 %b, label %if.end, label %if.else
+
+if.else:
+  %call = call i1 @NOP()
+  br label %if.end
+
+if.end:
+; Check that both selects in this BB are still in place,
+; and were not replaced with a conditional branch.
+; CHECK:      phi
+; CHECK-NEXT: phi
+; CHECK-NEXT: select
+; CHECK-NEXT: select
+; CHECK-NEXT: ret
+  %u1 = phi i1 [ true, %if.else ], [ %u, %entry ]
+  %v = phi i1 [ %call, %if.else ], [ false, %entry ]
+  %s = select i1 %u1, i32 22, i32 0
+  %v1 = select i1 %v, i32 %s, i32 42
+  ret i32 %v1
+}
+
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores.ll
index 8302ad9562f52..250a51a27b442 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores.ll
@@ -111,7 +111,7 @@ define amdgpu_kernel void @merge_global_store_2_constants_i32_f32(i32 addrspace(
 }
 
 ; CHECK-LABEL: @merge_global_store_2_constants_f32_i32
-; CHECK  store <2 x float> <float 4.000000e+00, float 0x370EC00000000000>, <2 x float> addrspace(1)* %{{[0-9]+$}}
+; CHECK: store <2 x i32> <i32 1082130432, i32 123>, <2 x i32> addrspace(1)*
 define amdgpu_kernel void @merge_global_store_2_constants_f32_i32(float addrspace(1)* %out) #0 {
   %out.gep.1 = getelementptr float, float addrspace(1)* %out, i32 1
   %out.gep.1.bc = bitcast float addrspace(1)* %out.gep.1 to i32 addrspace(1)*
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/vect-ptr-ptr-size-mismatch.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/vect-ptr-ptr-size-mismatch.ll
index 1cb8d14f17786..3f24f5c9e53d1 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/vect-ptr-ptr-size-mismatch.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/vect-ptr-ptr-size-mismatch.ll
@@ -55,7 +55,7 @@ entry:
 }
 
 ; CHECK-LABEL: @ext_ptr
-; CHECK  load <2 x i32>
+; CHECK: load <2 x i32>
 define void @ext_ptr(i32 addrspace(5)* %p) {
 entry:
   %gep1 = getelementptr inbounds i32, i32 addrspace(5)* %p, i64 0
@@ -68,7 +68,7 @@ entry:
 }
 
 ; CHECK-LABEL: @shrink_ptr
-; CHECK  load <2 x i32>
+; CHECK: load <2 x i32>
 define void @shrink_ptr(i32* %p) {
 entry:
   %gep1 = getelementptr inbounds i32, i32* %p, i64 0
diff --git a/llvm/test/Transforms/LoopRotate/pr35210.ll b/llvm/test/Transforms/LoopRotate/pr35210.ll
index 982a4c0c8e595..6fddeeeb90893 100644
--- a/llvm/test/Transforms/LoopRotate/pr35210.ll
+++ b/llvm/test/Transforms/LoopRotate/pr35210.ll
@@ -62,46 +62,37 @@
 ; MSSA-NEXT: Running analysis: PostDominatorTreeAnalysis on f
 ; MSSA-NEXT: Finished llvm::Function pass manager run.
 
-
 ; CHECK-LABEL: define i8 @f() {
-; CHECK-NEXT : entry:
-; CHECK-NEXT :   br label %bb
-; CHECK-NEXT :
-; CHECK-NEXT : bb:                                               ; preds = %bb, %entry
-; CHECK-NEXT :   %mode.0 = phi i8 [ 0, %entry ], [ %indvar.next, %bb ]
-; CHECK-NEXT :   %tmp5 = icmp eq i8 %mode.0, 1
-; CHECK-NEXT :   %indvar.next = add i8 %mode.0, 1
-; CHECK-NEXT :   br i1 %tmp5, label %bb5, label %bb
-; CHECK-NEXT :
-; CHECK-NEXT : bb5:                                              ; preds = %bb
-; CHECK-NEXT :   tail call void @raise_exception() #0
-; CHECK-NEXT :   unreachable
-; CHECK-NEXT : }
-; CHECK-NEXT :
-; CHECK-NEXT : ; Function Attrs: noreturn
-; CHECK-NEXT : declare void @raise_exception() #0
-; CHECK-NEXT :
-; CHECK-NEXT : attributes #0 = { noreturn }
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label %bb
+; CHECK:       bb:                                               ; preds = %bb, %entry
+; CHECK-NEXT:    %mode.0 = phi i8 [ 0, %entry ], [ %indvar.next, %bb ]
+; CHECK-NEXT:    %tmp5 = icmp eq i8 %mode.0, 1
+; CHECK-NEXT:    %indvar.next = add i8 %mode.0, 1
+; CHECK-NEXT:    br i1 %tmp5, label %bb5, label %bb
+; CHECK:       bb5:                                              ; preds = %bb
+; CHECK-NEXT:    tail call void @raise_exception() #0
+; CHECK-NEXT:    unreachable
+; CHECK-NEXT:  }
+; CHECK:       ; Function Attrs: noreturn
+; CHECK:       declare void @raise_exception() #0
+; CHECK:       attributes #0 = { noreturn }
 
 ; MSSA-LABEL: define i8 @f() {
-; MSSA-NEXT : entry:
-; MSSA-NEXT :   br label %bb
-; MSSA-NEXT :
-; MSSA-NEXT : bb:                                               ; preds = %bb, %entry
-; MSSA-NEXT :   %mode.0 = phi i8 [ 0, %entry ], [ %indvar.next, %bb ]
-; MSSA-NEXT :   %tmp5 = icmp eq i8 %mode.0, 1
-; MSSA-NEXT :   %indvar.next = add i8 %mode.0, 1
-; MSSA-NEXT :   br i1 %tmp5, label %bb5, label %bb
-; MSSA-NEXT :
-; MSSA-NEXT : bb5:                                              ; preds = %bb
-; MSSA-NEXT :   tail call void @raise_exception() #0
-; MSSA-NEXT :   unreachable
-; MSSA-NEXT : }
-; MSSA-NEXT :
-; MSSA-NEXT : ; Function Attrs: noreturn
-; MSSA-NEXT : declare void @raise_exception() #0
-; MSSA-NEXT :
-; MSSA-NEXT : attributes #0 = { noreturn }
+; MSSA-NEXT:  entry:
+; MSSA-NEXT:    br label %bb
+; MSSA:       bb:                                               ; preds = %bb, %entry
+; MSSA-NEXT:    %mode.0 = phi i8 [ 0, %entry ], [ %indvar.next, %bb ]
+; MSSA-NEXT:    %tmp5 = icmp eq i8 %mode.0, 1
+; MSSA-NEXT:    %indvar.next = add i8 %mode.0, 1
+; MSSA-NEXT:    br i1 %tmp5, label %bb5, label %bb
+; MSSA:       bb5:                                              ; preds = %bb
+; MSSA-NEXT:    tail call void @raise_exception() #0
+; MSSA-NEXT:    unreachable
+; MSSA-NEXT:  }
+; MSSA:       ; Function Attrs: noreturn
+; MSSA:       declare void @raise_exception() #0
+; MSSA:       attributes #0 = { noreturn }
 
 define i8 @f() {
 entry:
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-insns-2.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-insns-2.ll
index 239cc02335067..613e65d0c6500 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-insns-2.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-insns-2.ll
@@ -10,9 +10,9 @@
 ; INSN:   getelementptr i32, i32* %x, i64 %indvars.iv
 ; INSN:   getelementptr i32, i32* %y, i64 %indvars.iv
 ; INSN:   getelementptr i32, i32* %q, i64 %indvars.iv
-; REGS    %lsr.iv4 = phi
-; REGS    %lsr.iv2 = phi
-; REGS    %lsr.iv1 = phi
+; REGS:   %lsr.iv4 = phi
+; REGS:   %lsr.iv2 = phi
+; REGS:   %lsr.iv1 = phi
 ; REGS:   getelementptr i32, i32* %lsr.iv1, i64 1
 ; REGS:   getelementptr i32, i32* %lsr.iv2, i64 1
 ; REGS:   getelementptr i32, i32* %lsr.iv4, i64 1
diff --git a/llvm/test/Transforms/LoopVectorize/X86/svml-calls.ll b/llvm/test/Transforms/LoopVectorize/X86/svml-calls.ll
index 8ff62f178d09f..c1af2cb9f413e 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/svml-calls.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/svml-calls.ll
@@ -28,6 +28,12 @@ declare float @logf(float) #0
 declare double @llvm.log.f64(double) #0
 declare float @llvm.log.f32(float) #0
 
+declare double @exp2(double) #0
+declare float @exp2f(float) #0
+declare double @llvm.exp2.f64(double) #0
+declare float @llvm.exp2.f32(float) #0
+declare double @__exp2_finite(double) #0
+declare float @__exp2f_finite(float) #0
 
 define void @sin_f64(double* nocapture %varray) {
 ; CHECK-LABEL: @sin_f64(
@@ -497,5 +503,142 @@ for.end:
   ret void
 }
 
-attributes #0 = { nounwind readnone }
+define void @exp2_f64(double* nocapture %varray) {
+; CHECK-LABEL: @exp2_f64(
+; CHECK:    [[TMP5:%.*]] = call <4 x double> @__svml_exp24(<4 x double> [[TMP4:%.*]])
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @exp2(double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @exp2_f32(float* nocapture %varray) {
+; CHECK-LABEL: @exp2_f32(
+; CHECK:    [[TMP5:%.*]] = call <4 x float> @__svml_exp2f4(<4 x float> [[TMP4:%.*]])
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @exp2f(float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
 
+for.end:
+  ret void
+}
+
+define void @exp2_f64_intrinsic(double* nocapture %varray) {
+; CHECK-LABEL: @exp2_f64_intrinsic(
+; CHECK:    [[TMP5:%.*]] = call <4 x double> @__svml_exp24(<4 x double> [[TMP4:%.*]])
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @llvm.exp2.f64(double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @exp2_f32_intrinsic(float* nocapture %varray) {
+; CHECK-LABEL: @exp2_f32_intrinsic(
+; CHECK:    [[TMP5:%.*]] = call <4 x float> @__svml_exp2f4(<4 x float> [[TMP4:%.*]])
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @llvm.exp2.f32(float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @exp2f_finite(float* nocapture %varray) {
+; CHECK-LABEL: @exp2f_finite(
+; CHECK:    [[TMP5:%.*]] = call <4 x float> @__svml_exp2f4(<4 x float> [[TMP4:%.*]])
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @__exp2f_finite(float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @exp2_finite(double* nocapture %varray) {
+; CHECK-LABEL: @exp2_finite(
+; CHECK:    [[TMP5:%.*]] = call <4 x double> @__svml_exp24(<4 x double> [[TMP4:%.*]])
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @__exp2_finite(double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+attributes #0 = { nounwind readnone }
diff --git a/llvm/test/Transforms/LoopVectorize/pr45259.ll b/llvm/test/Transforms/LoopVectorize/pr45259.ll
new file mode 100644
index 0000000000000..3d1370b14a0f4
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/pr45259.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -S -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 | FileCheck %s
+
+; Check that we can vectorize this loop without crashing.
+
+; CHECK-LABEL: define {{.*}} @widget(
+; CHECK: [[vecInd:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>
+; CHECK-NEXT: add <4 x i8> [[vecInd]], <i8 1, i8 1, i8 1, i8 1>
+
+define i8 @widget(i8* %arr, i8 %t9) {
+bb:
+  br label %bb6
+
+bb6:
+  %t1.0 = phi i8* [ %arr, %bb ], [ null, %bb6 ]
+  %c = call i1 @cond()
+  br i1 %c, label %for.preheader, label %bb6
+
+for.preheader:
+  br label %for.body
+
+for.body:
+  %iv = phi i8 [ %iv.next, %for.body ], [ 0, %for.preheader ]
+  %iv.next = add i8 %iv, 1
+  %ptr = getelementptr inbounds i8, i8* %arr, i8 %iv.next
+  %t3.i = icmp slt i8 %iv.next, %t9
+  %t3.i8 = zext i1 %t3.i to i8
+  store i8 %t3.i8, i8* %ptr
+  %ec = icmp eq i8* %t1.0, %ptr
+  br i1 %ec, label %for.exit, label %for.body
+
+for.exit:
+  %iv.next.lcssa = phi i8 [ %iv.next, %for.body ]
+  ret i8 %iv.next.lcssa
+}
+
+declare i1 @cond()
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/const-gep.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/const-gep.ll
new file mode 100644
index 0000000000000..c417ba719412e
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/const-gep.ll
@@ -0,0 +1,85 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck  %s
+
+; Make sure we correctly lower in the presence of getelementptr constant
+; expressions.
+
+@foo = global [5 x <4 x double>] zeroinitializer, align 16
+
+define void @test(i32 %r, i32 %c) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[R_ADDR:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[C_ADDR:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i32 [[R:%.*]], i32* [[R_ADDR]], align 4
+; CHECK-NEXT:    store i32 [[C:%.*]], i32* [[C_ADDR]], align 4
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <2 x double>, <2 x double>* bitcast ([5 x <4 x double>]* @foo to <2 x double>*), align 8
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr ([5 x <4 x double>], [5 x <4 x double>]* @foo, i32 0, i32 0, i32 2) to <2 x double>*), align 8
+; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x double> [[COL_LOAD]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> undef, double [[TMP0]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <1 x double> [[BLOCK]], [[SPLAT_SPLAT]]
+; CHECK-NEXT:    [[BLOCK2:%.*]] = shufflevector <2 x double> [[COL_LOAD1]], <2 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[COL_LOAD]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT3:%.*]] = insertelement <1 x double> undef, double [[TMP2]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT4:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT3]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul <1 x double> [[BLOCK2]], [[SPLAT_SPLAT4]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <1 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x double> [[TMP4]], <1 x double> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP5]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:    [[BLOCK5:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x double> [[COL_LOAD]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT6:%.*]] = insertelement <1 x double> undef, double [[TMP7]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT7:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT6]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = fmul <1 x double> [[BLOCK5]], [[SPLAT_SPLAT7]]
+; CHECK-NEXT:    [[BLOCK8:%.*]] = shufflevector <2 x double> [[COL_LOAD1]], <2 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x double> [[COL_LOAD]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT9:%.*]] = insertelement <1 x double> undef, double [[TMP9]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT10:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT9]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = fmul <1 x double> [[BLOCK8]], [[SPLAT_SPLAT10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = fadd <1 x double> [[TMP8]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x double> [[TMP11]], <1 x double> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> [[TMP12]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[BLOCK11:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x double> [[COL_LOAD1]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT12:%.*]] = insertelement <1 x double> undef, double [[TMP14]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT13:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT12]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = fmul <1 x double> [[BLOCK11]], [[SPLAT_SPLAT13]]
+; CHECK-NEXT:    [[BLOCK14:%.*]] = shufflevector <2 x double> [[COL_LOAD1]], <2 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x double> [[COL_LOAD1]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT15:%.*]] = insertelement <1 x double> undef, double [[TMP16]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT16:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT15]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = fmul <1 x double> [[BLOCK14]], [[SPLAT_SPLAT16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = fadd <1 x double> [[TMP15]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x double> [[TMP18]], <1 x double> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP19]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:    [[BLOCK17:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <2 x double> [[COL_LOAD1]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT18:%.*]] = insertelement <1 x double> undef, double [[TMP21]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT19:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT18]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = fmul <1 x double> [[BLOCK17]], [[SPLAT_SPLAT19]]
+; CHECK-NEXT:    [[BLOCK20:%.*]] = shufflevector <2 x double> [[COL_LOAD1]], <2 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <2 x double> [[COL_LOAD1]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT21:%.*]] = insertelement <1 x double> undef, double [[TMP23]], i32 0
+; CHECK-NEXT:    [[SPLAT_SPLAT22:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT21]], <1 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = fmul <1 x double> [[BLOCK20]], [[SPLAT_SPLAT22]]
+; CHECK-NEXT:    [[TMP25:%.*]] = fadd <1 x double> [[TMP22]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x double> [[TMP25]], <1 x double> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <2 x double> [[TMP20]], <2 x double> [[TMP26]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    store <2 x double> [[COL_LOAD]], <2 x double>* bitcast (double* getelementptr inbounds ([5 x <4 x double>], [5 x <4 x double>]* @foo, i64 0, i64 2, i32 0) to <2 x double>*), align 8
+; CHECK-NEXT:    store <2 x double> [[COL_LOAD1]], <2 x double>* bitcast (double* getelementptr ([5 x <4 x double>], [5 x <4 x double>]* @foo, i64 0, i64 2, i32 2) to <2 x double>*), align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %r.addr = alloca i32, align 4
+  %c.addr = alloca i32, align 4
+  store i32 %r, i32* %r.addr, align 4
+  store i32 %c, i32* %c.addr, align 4
+  %0 = load <4 x double>, <4 x double>* getelementptr inbounds ([5 x <4 x double>], [5 x <4 x double>]* @foo, i64 0, i64 0), align 16
+  %mul = call <4 x double> @llvm.matrix.multiply(<4 x double> %0, <4 x double> %0, i32 2, i32 2, i32 2)
+  store <4 x double> %0, <4 x double>* getelementptr inbounds ([5 x <4 x double>], [5 x <4 x double>]* @foo, i64 0, i64 2), align 16
+  ret void
+}
+
+declare <4 x double> @llvm.matrix.multiply(<4 x double>, <4 x double>, i32, i32, i32)
diff --git a/llvm/test/Transforms/NewGVN/malloc-load-removal.ll b/llvm/test/Transforms/NewGVN/malloc-load-removal.ll
index 72f4839a5545d..c62bac950df25 100644
--- a/llvm/test/Transforms/NewGVN/malloc-load-removal.ll
+++ b/llvm/test/Transforms/NewGVN/malloc-load-removal.ll
@@ -54,3 +54,28 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK_NO_LIBCALLS: load
 ; CHECK_NO_LIBCALLS: icmp
 }
+
+declare i8* @aligned_alloc(i64, i64) nounwind
+
+define noalias i8* @test3() nounwind uwtable ssp {
+entry:
+  %call = tail call i8* @aligned_alloc(i64 256, i64 32) nounwind
+  %0 = load i8, i8* %call, align 32
+  %tobool = icmp eq i8 %0, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  store i8 0, i8* %call, align 1
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  ret i8* %call
+
+; CHECK-LABEL: @test3(
+; CHECK-NOT: load
+; CHECK-NOT: icmp
+
+; CHECK_NO_LIBCALLS-LABEL: @test3(
+; CHECK_NO_LIBCALLS: load
+; CHECK_NO_LIBCALLS: icmp
+}
diff --git a/llvm/test/Transforms/OpenMP/add_attributes.ll b/llvm/test/Transforms/OpenMP/add_attributes.ll
index df93c70866aa6..017bc1382e06b 100644
--- a/llvm/test/Transforms/OpenMP/add_attributes.ll
+++ b/llvm/test/Transforms/OpenMP/add_attributes.ll
@@ -9,6 +9,7 @@ target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16
 
 %struct.omp_lock_t = type { i8* }
 %struct.omp_nest_lock_t = type { i8* }
+%struct.ident_t = type { i32, i32, i32, i32, i8* }
 
 define void @call_all(i32 %schedule, %struct.omp_lock_t* %lock, i32 %lock_hint, %struct.omp_nest_lock_t* %nest_lock, i32 %i, i8* %s, i64 %st, i8* %vp, double %d, i32 %proc_bind, i64 %allocator_handle, i8* %cp, i64 %event_handle, i32 %pause_resource) {
 entry:
@@ -460,6 +461,40 @@ declare dso_local i32 @omp_pause_resource_all(i32)
 
 declare dso_local i32 @omp_get_supported_active_levels()
 
+declare void @__kmpc_barrier(%struct.ident_t*, i32)
+
+declare i32 @__kmpc_cancel(%struct.ident_t*, i32, i32)
+
+declare i32 @__kmpc_cancel_barrier(%struct.ident_t*, i32)
+
+declare void @__kmpc_flush(%struct.ident_t*)
+
+declare i32 @__kmpc_global_thread_num(%struct.ident_t*)
+
+declare void @__kmpc_fork_call(%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...)
+
+declare i32 @__kmpc_omp_taskwait(%struct.ident_t*, i32)
+
+declare i32 @__kmpc_omp_taskyield(%struct.ident_t*, i32, i32)
+
+declare void @__kmpc_push_num_threads(%struct.ident_t*, i32, i32)
+
+declare void @__kmpc_push_proc_bind(%struct.ident_t*, i32, i32)
+
+declare void @__kmpc_serialized_parallel(%struct.ident_t*, i32)
+
+declare void @__kmpc_end_serialized_parallel(%struct.ident_t*, i32)
+
+declare i32 @__kmpc_master(%struct.ident_t*, i32)
+
+declare void @__kmpc_end_master(%struct.ident_t*, i32)
+
+declare void @__kmpc_critical(%struct.ident_t*, i32, [8 x i32]*)
+
+declare void @__kmpc_critical_with_hint(%struct.ident_t*, i32, [8 x i32]*, i32)
+
+declare void @__kmpc_end_critical(%struct.ident_t*, i32, [8 x i32]*)
+
 ; CHECK: ; Function Attrs: nounwind
 ; CHECK-NEXT: declare dso_local void @omp_set_num_threads(i32)
 
@@ -685,67 +720,118 @@ declare dso_local i32 @omp_get_supported_active_levels()
 ; CHECK: ; Function Attrs: nounwind
 ; CHECK-NEXT: declare dso_local i32 @omp_get_supported_active_levels() #0
 
-; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind writeonly
+; CHECK: Function Attrs: inaccessiblemem_or_argmemonly
+; CHECK-NEXT: declare void @__kmpc_barrier(%struct.ident_t*, i32)
+
+; CHECK: Function Attrs: inaccessiblemem_or_argmemonly
+; CHECK-NEXT: declare i32 @__kmpc_cancel(%struct.ident_t*, i32, i32)
+
+; CHECK: Function Attrs: inaccessiblemem_or_argmemonly
+; CHECK-NEXT: declare i32 @__kmpc_cancel_barrier(%struct.ident_t*, i32)
+
+; CHECK: Function Attrs: inaccessiblemem_or_argmemonly
+; CHECK-NEXT: declare void @__kmpc_flush(%struct.ident_t*)
+
+; CHECK: Function Attrs: nounwind
+; CHECK-NEXT: declare i32 @__kmpc_global_thread_num(%struct.ident_t*)
+
+; CHECK: Function Attrs: nounwind
+; CHECK-NEXT: declare void @__kmpc_fork_call(%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...)
+
+; CHECK: Function Attrs: inaccessiblemem_or_argmemonly
+; CHECK-NEXT: declare i32 @__kmpc_omp_taskwait(%struct.ident_t*, i32)
+
+; CHECK: Function Attrs: inaccessiblemem_or_argmemonly
+; CHECK-NEXT: declare i32 @__kmpc_omp_taskyield(%struct.ident_t*, i32, i32)
+
+; CHECK: Function Attrs: inaccessiblemem_or_argmemonly
+; CHECK-NEXT: declare void @__kmpc_push_num_threads(%struct.ident_t*, i32, i32)
+
+; CHECK: Function Attrs: inaccessiblemem_or_argmemonly
+; CHECK-NEXT: declare void @__kmpc_push_proc_bind(%struct.ident_t*, i32, i32)
+
+; CHECK: Function Attrs: inaccessiblemem_or_argmemonly
+; CHECK-NEXT: declare void @__kmpc_serialized_parallel(%struct.ident_t*, i32)
+
+; CHECK: Function Attrs: inaccessiblemem_or_argmemonly
+; CHECK-NEXT: declare void @__kmpc_end_serialized_parallel(%struct.ident_t*, i32)
+
+; CHECK: Function Attrs: inaccessiblemem_or_argmemonly
+; CHECK-NEXT: declare i32 @__kmpc_master(%struct.ident_t*, i32)
+
+; CHECK: Function Attrs: inaccessiblemem_or_argmemonly
+; CHECK-NEXT: declare void @__kmpc_end_master(%struct.ident_t*, i32)
+
+; CHECK: Function Attrs: inaccessiblemem_or_argmemonly
+; CHECK-NEXT: declare void @__kmpc_critical(%struct.ident_t*, i32, [8 x i32]*)
+
+; CHECK: Function Attrs: inaccessiblemem_or_argmemonly
+; CHECK-NEXT: declare void @__kmpc_critical_with_hint(%struct.ident_t*, i32, [8 x i32]*, i32)
+
+; CHECK: Function Attrs: inaccessiblemem_or_argmemonly
+; CHECK-NEXT: declare void @__kmpc_end_critical(%struct.ident_t*, i32, [8 x i32]*)
+
+; OPTIMISTIC: ; Function Attrs: inaccessiblememonly nofree nosync nounwind writeonly
 ; OPTIMISTIC-NEXT: declare dso_local void @omp_set_num_threads(i32)
 
-; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind writeonly
+; OPTIMISTIC: ; Function Attrs: inaccessiblememonly nofree nosync nounwind writeonly
 ; OPTIMISTIC-NEXT: declare dso_local void @omp_set_dynamic(i32)
 
-; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind writeonly
+; OPTIMISTIC: ; Function Attrs: inaccessiblememonly nofree nosync nounwind writeonly
 ; OPTIMISTIC-NEXT: declare dso_local void @omp_set_nested(i32)
 
-; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind writeonly
+; OPTIMISTIC: ; Function Attrs: inaccessiblememonly nofree nosync nounwind writeonly
 ; OPTIMISTIC-NEXT: declare dso_local void @omp_set_max_active_levels(i32)
 
-; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind writeonly
+; OPTIMISTIC: ; Function Attrs: inaccessiblememonly nofree nosync nounwind writeonly
 ; OPTIMISTIC-NEXT: declare dso_local void @omp_set_schedule(i32, i32)
 
-; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind readonly
+; OPTIMISTIC: ; Function Attrs: inaccessiblememonly nofree nosync nounwind readonly
 ; OPTIMISTIC-NEXT: declare dso_local i32 @omp_get_num_threads() #1
 
 ; OPTIMISTIC-NOT: Function Attrs
 ; OPTIMISTIC: declare dso_local void @use_int(i32)
 
-; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind readonly
+; OPTIMISTIC: ; Function Attrs: inaccessiblememonly nofree nosync nounwind readonly
 ; OPTIMISTIC-NEXT: declare dso_local i32 @omp_get_dynamic() #1
 
-; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind readonly
+; OPTIMISTIC: ; Function Attrs: inaccessiblememonly nofree nosync nounwind readonly
 ; OPTIMISTIC-NEXT: declare dso_local i32 @omp_get_nested() #1
 
-; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind readonly
+; OPTIMISTIC: ; Function Attrs: inaccessiblememonly nofree nosync nounwind readonly
 ; OPTIMISTIC-NEXT: declare dso_local i32 @omp_get_max_threads() #1
 
-; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind readonly
+; OPTIMISTIC: ; Function Attrs: inaccessiblememonly nofree nosync nounwind readonly
 ; OPTIMISTIC-NEXT: declare dso_local i32 @omp_get_thread_num() #1
 
-; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind readonly
+; OPTIMISTIC: ; Function Attrs: inaccessiblememonly nofree nosync nounwind readonly
 ; OPTIMISTIC-NEXT: declare dso_local i32 @omp_get_num_procs() #1
 
-; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind readonly
+; OPTIMISTIC: ; Function Attrs: inaccessiblememonly nofree nosync nounwind readonly
 ; OPTIMISTIC-NEXT: declare dso_local i32 @omp_in_parallel() #1
 
-; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind readonly
+; OPTIMISTIC: ; Function Attrs: inaccessiblememonly nofree nosync nounwind readonly
 ; OPTIMISTIC-NEXT: declare dso_local i32 @omp_in_final() #1
 
-; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind readonly
+; OPTIMISTIC: ; Function Attrs: inaccessiblememonly nofree nosync nounwind readonly
 ; OPTIMISTIC-NEXT: declare dso_local i32 @omp_get_active_level() #1
 
-; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind readonly
+; OPTIMISTIC: ; Function Attrs: inaccessiblememonly nofree nosync nounwind readonly
 ; OPTIMISTIC-NEXT: declare dso_local i32 @omp_get_level() #1
 
-; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind readonly
+; OPTIMISTIC: ; Function Attrs: inaccessiblememonly nofree nosync nounwind readonly
 ; OPTIMISTIC-NEXT: declare dso_local i32 @omp_get_ancestor_thread_num(i32) #1
 
-; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind readonly
+; OPTIMISTIC: ; Function Attrs: inaccessiblememonly nofree nosync nounwind readonly
 ; OPTIMISTIC-NEXT: declare dso_local i32 @omp_get_team_size(i32) #1
 
-; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind readonly
+; OPTIMISTIC: ; Function Attrs: inaccessiblememonly nofree nosync nounwind readonly
 ; OPTIMISTIC-NEXT: declare dso_local i32 @omp_get_thread_limit() #1
 
-; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind readonly
+; OPTIMISTIC: ; Function Attrs: inaccessiblememonly nofree nosync nounwind readonly
 ; OPTIMISTIC-NEXT: declare dso_local i32 @omp_get_max_active_levels() #1
 
-; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind
+; OPTIMISTIC: ; Function Attrs: inaccessiblemem_or_argmemonly nofree nosync nounwind
 ; OPTIMISTIC-NEXT: declare dso_local void @omp_get_schedule(i32* nocapture writeonly, i32* nocapture writeonly) #2
 
 ; OPTIMISTIC-NOT: Function Attrs
@@ -814,7 +900,7 @@ declare dso_local i32 @omp_get_supported_active_levels()
 ; OPTIMISTIC-NOT: Function Attrs
 ; OPTIMISTIC: declare dso_local i32 @omp_get_team_num()
 
-; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind readonly
+; OPTIMISTIC: ; Function Attrs: inaccessiblememonly nofree nosync nounwind readonly
 ; OPTIMISTIC-NEXT: declare dso_local i32 @omp_get_cancellation() #1
 
 ; OPTIMISTIC-NOT: Function Attrs
@@ -844,25 +930,25 @@ declare dso_local i32 @omp_get_supported_active_levels()
 ; OPTIMISTIC-NOT: Function Attrs
 ; OPTIMISTIC: declare dso_local i32 @omp_get_device_num()
 
-; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind readonly
+; OPTIMISTIC: ; Function Attrs: inaccessiblememonly nofree nosync nounwind readonly
 ; OPTIMISTIC-NEXT: declare dso_local i32 @omp_get_proc_bind() #1
 
-; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind readonly
+; OPTIMISTIC: ; Function Attrs: inaccessiblememonly nofree nosync nounwind readonly
 ; OPTIMISTIC-NEXT: declare dso_local i32 @omp_get_num_places() #1
 
 ; OPTIMISTIC-NOT: Function Attrs
 ; OPTIMISTIC: declare dso_local i32 @omp_get_place_num_procs(i32)
 
-; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind
+; OPTIMISTIC: ; Function Attrs: inaccessiblemem_or_argmemonly nofree nosync nounwind
 ; OPTIMISTIC-NEXT: declare dso_local void @omp_get_place_proc_ids(i32, i32* nocapture writeonly) #2
 
-; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind readonly
+; OPTIMISTIC: ; Function Attrs: inaccessiblememonly nofree nosync nounwind readonly
 ; OPTIMISTIC-NEXT: declare dso_local i32 @omp_get_place_num() #1
 
-; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind readonly
+; OPTIMISTIC: ; Function Attrs: inaccessiblememonly nofree nosync nounwind readonly
 ; OPTIMISTIC-NEXT: declare dso_local i32 @omp_get_partition_num_places() #1
 
-; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind readonly
+; OPTIMISTIC: ; Function Attrs: inaccessiblememonly nofree nosync nounwind readonly
 ; OPTIMISTIC-NEXT: declare dso_local void @omp_get_partition_place_nums(i32*) #1
 
 ; OPTIMISTIC-NOT: Function Attrs
@@ -907,5 +993,8 @@ declare dso_local i32 @omp_get_supported_active_levels()
 ; OPTIMISTIC-NOT: Function Attrs
 ; OPTIMISTIC: declare dso_local i32 @omp_pause_resource_all(i32)
 
-; OPTIMISTIC: ; Function Attrs: nofree nosync nounwind readonly
+; OPTIMISTIC: ; Function Attrs: inaccessiblememonly nofree nosync nounwind readonly
 ; OPTIMISTIC-NEXT: declare dso_local i32 @omp_get_supported_active_levels() #1
+
+; OPTIMISTIC: ; Function Attrs: inaccessiblememonly nofree nosync nounwind readonly
+; OPTIMISTIC-NEXT: declare i32 @__kmpc_global_thread_num(%struct.ident_t*)
diff --git a/llvm/test/Transforms/OpenMP/parallel_deletion.ll b/llvm/test/Transforms/OpenMP/parallel_deletion.ll
index 82e0295eb6a87..ee316b56794a1 100644
--- a/llvm/test/Transforms/OpenMP/parallel_deletion.ll
+++ b/llvm/test/Transforms/OpenMP/parallel_deletion.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature
 ; RUN: opt -S -attributor -openmpopt -attributor-disable=false < %s | FileCheck %s
 ; RUN: opt -S -passes='attributor,cgscc(openmpopt)' -attributor-disable=false < %s | FileCheck %s
 ;
@@ -142,7 +142,7 @@ define void @delete_parallel_2() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[TMP:%.*]] = bitcast i32* [[A]] to i8*
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull align 4 dereferenceable(4) [[TMP]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull align 4 dereferenceable(4) [[TMP]]) #0
 ; CHECK-NEXT:    store i32 0, i32* [[A]], align 4
 ; CHECK-NEXT:    call void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(%struct.ident_t* nonnull align 8 dereferenceable(24) @0, i32 1, void (i32*, i32*, ...)* nonnull bitcast (void (i32*, i32*, i32*)* @.omp_outlined..3 to void (i32*, i32*, ...)*), i32* nocapture nofree nonnull align 4 dereferenceable(4) [[A]])
 ; CHECK-NEXT:    call void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(%struct.ident_t* nonnull align 8 dereferenceable(24) @0, i32 1, void (i32*, i32*, ...)* nonnull bitcast (void (i32*, i32*, i32*)* @.omp_outlined..4 to void (i32*, i32*, ...)*), i32* nocapture nonnull align 4 dereferenceable(4) [[A]])
diff --git a/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp3.ll b/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp3.ll
index 15fa0398b8be8..eb457973be0ca 100644
--- a/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp3.ll
+++ b/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp3.ll
@@ -6,9 +6,15 @@
 
 ; Test to make sure importing and dead stripping works in the
 ; case where the target is a local function that also indirectly calls itself.
-; RUN: llvm-lto2 run -thinlto-threads=1 -save-temps -o %t3 %t.bc %t2.bc -r %t.bc,fptr,plx -r %t.bc,main,plx -r %t2.bc,_Z6updatei,pl -r %t2.bc,fptr,l -print-imports 2>&1 | FileCheck %s --check-prefix=IMPORTS
+; RUN: llvm-lto2 run -thinlto-threads=1 -save-temps -o %t3 %t.bc %t2.bc -r %t.bc,fptr,plx -r %t.bc,main,plx -r %t2.bc,_Z6updatei,pl -r %t2.bc,fptr,l
+; RUN: llvm-dis %t3.1.3.import.bc -o - | FileCheck %s --check-prefix=IMPORTS
+
 ; Make sure we import the promted indirectly called target
-; IMPORTS: Import _ZL3foov.llvm.0
+; IMPORTS: void @_ZL3foov.llvm.0()
+
+; RUN: llvm-dis %t3.2.4.opt.bc -o - | FileCheck %s --check-prefix=ICALL-PROM
+; ICALL-PROM:   br i1 %{{[0-9]+}}, label %tailrecurse, label %if.false.orig_indirect,
+
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
@@ -19,7 +25,6 @@ target triple = "x86_64-unknown-linux-gnu"
 define i32 @main() local_unnamed_addr #0 !prof !34 {
 entry:
   %0 = load void ()*, void ()** @fptr, align 8
-; ICALL-PROM:   br i1 %{{[0-9]+}}, label %if.true.direct_targ, label %if.false.orig_indirect
   tail call void %0(), !prof !40
   ret i32 0
 }
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/shuffle.ll b/llvm/test/Transforms/PhaseOrdering/X86/shuffle.ll
new file mode 100644
index 0000000000000..8510eede50a36
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/X86/shuffle.ll
@@ -0,0 +1,340 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -O2                   -S -mattr=avx < %s | FileCheck %s
+; RUN: opt -passes='default<O2>' -S -mattr=avx < %s | FileCheck %s
+
+target triple = "x86_64--"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Eliminate redundant shuffles
+
+define <2 x i64> @shuffle_32_add_16_shuffle_32_masks_are_eq(<2 x i64> %v) {
+; CHECK-LABEL: @shuffle_32_add_16_shuffle_32_masks_are_eq(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[V:%.*]] to <8 x i16>
+; CHECK-NEXT:    [[TMP2:%.*]] = shl <8 x i16> [[TMP1]], <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+; CHECK-NEXT:    [[BC5:%.*]] = bitcast <8 x i16> [[TMP2]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[BC5]]
+;
+  %bc0 = bitcast <2 x i64> %v to <4 x i32>
+  %shuffle = shufflevector <4 x i32> %bc0, <4 x i32> zeroinitializer, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %bc1 = bitcast <4 x i32> %shuffle to <2 x i64>
+  %bc2 = bitcast <2 x i64> %bc1 to <8 x i16>
+  %add.i = add <8 x i16> %bc2, %bc2
+  %bc3 = bitcast <8 x i16> %add.i to <2 x i64>
+  %bc4 = bitcast <2 x i64> %bc3 to <4 x i32>
+  %shuffle4 = shufflevector <4 x i32> %bc4, <4 x i32> zeroinitializer, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %bc5 = bitcast <4 x i32> %shuffle4 to <2 x i64>
+  ret <2 x i64> %bc5
+}
+
+; Eliminate redundant shuffles
+
+define <2 x i64> @shuffle_32_add_8_shuffle_32_masks_are_eq(<2 x i64> %v) {
+; CHECK-LABEL: @shuffle_32_add_8_shuffle_32_masks_are_eq(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[V:%.*]] to <16 x i8>
+; CHECK-NEXT:    [[TMP2:%.*]] = shl <16 x i8> [[TMP1]], <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+; CHECK-NEXT:    [[BC5:%.*]] = bitcast <16 x i8> [[TMP2]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[BC5]]
+;
+  %bc0 = bitcast <2 x i64> %v to <4 x i32>
+  %shuffle = shufflevector <4 x i32> %bc0, <4 x i32> zeroinitializer, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %bc1 = bitcast <4 x i32> %shuffle to <2 x i64>
+  %bc2 = bitcast <2 x i64> %bc1 to <16 x i8>
+  %add.i = add <16 x i8> %bc2, %bc2
+  %bc3 = bitcast <16 x i8> %add.i to <2 x i64>
+  %bc4 = bitcast <2 x i64> %bc3 to <4 x i32>
+  %shuffle4 = shufflevector <4 x i32> %bc4, <4 x i32> zeroinitializer, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %bc5 = bitcast <4 x i32> %shuffle4 to <2 x i64>
+  ret <2 x i64> %bc5
+}
+
+; TODO: Eliminate redundant shuffles
+
+define <2 x i64> @shuffle_8_add_32_shuffle_8_masks_are_eq(<2 x i64> %v) {
+; CHECK-LABEL: @shuffle_8_add_32_shuffle_8_masks_are_eq(
+; CHECK-NEXT:    [[BC0:%.*]] = bitcast <2 x i64> [[V:%.*]] to <16 x i8>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <16 x i8> [[BC0]], <16 x i8> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[BC2:%.*]] = bitcast <16 x i8> [[SHUFFLE]] to <4 x i32>
+; CHECK-NEXT:    [[ADD_I:%.*]] = shl <4 x i32> [[BC2]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[BC4:%.*]] = bitcast <4 x i32> [[ADD_I]] to <16 x i8>
+; CHECK-NEXT:    [[SHUFFLE4:%.*]] = shufflevector <16 x i8> [[BC4]], <16 x i8> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[BC5:%.*]] = bitcast <16 x i8> [[SHUFFLE4]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[BC5]]
+;
+  %bc0 = bitcast <2 x i64> %v to <16 x i8>
+  %shuffle = shufflevector <16 x i8> %bc0, <16 x i8> zeroinitializer, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %bc1 = bitcast <16 x i8> %shuffle to <2 x i64>
+  %bc2 = bitcast <2 x i64> %bc1 to <4 x i32>
+  %add.i = add <4 x i32> %bc2, %bc2
+  %bc3 = bitcast <4 x i32> %add.i to <2 x i64>
+  %bc4 = bitcast <2 x i64> %bc3 to <16 x i8>
+  %shuffle4 = shufflevector <16 x i8> %bc4, <16 x i8> zeroinitializer, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %bc5 = bitcast <16 x i8> %shuffle4 to <2 x i64>
+  ret <2 x i64> %bc5
+}
+
+; Single shuffle should sink.
+
+define <8 x i16> @shuffle_32_add_16_masks_are_eq(<4 x i32> %v1, <4 x i32> %v2) {
+; CHECK-LABEL: @shuffle_32_add_16_masks_are_eq(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[V1:%.*]] to <8 x i16>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[V2:%.*]] to <8 x i16>
+; CHECK-NEXT:    [[TMP3:%.*]] = add <8 x i16> [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[ADD:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
+; CHECK-NEXT:    ret <8 x i16> [[ADD]]
+;
+
+  %shuffle1 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
+  %shuffle2 = shufflevector <4 x i32> %v2, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
+  %bc1 = bitcast <4 x i32> %shuffle1 to <8 x i16>
+  %bc2 = bitcast <4 x i32> %shuffle2 to <8 x i16>
+  %add = add <8 x i16> %bc1, %bc2
+  ret <8 x i16> %add
+}
+
+; Sink single shuffle.
+
+define <16 x i8> @shuffle_32_add_8_masks_are_eq(<4 x i32> %v1, <4 x i32> %v2) {
+; CHECK-LABEL: @shuffle_32_add_8_masks_are_eq(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[V1:%.*]] to <16 x i8>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[V2:%.*]] to <16 x i8>
+; CHECK-NEXT:    [[TMP3:%.*]] = add <16 x i8> [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[ADD:%.*]] = shufflevector <16 x i8> [[TMP3]], <16 x i8> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <16 x i8> [[ADD]]
+;
+
+  %shuffle1 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
+  %shuffle2 = shufflevector <4 x i32> %v2, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
+  %bc1 = bitcast <4 x i32> %shuffle1 to <16 x i8>
+  %bc2 = bitcast <4 x i32> %shuffle2 to <16 x i8>
+  %add = add <16 x i8> %bc1, %bc2
+  ret <16 x i8> %add
+}
+
+; Sink single shuffle.
+
+define <16 x i8> @shuffle_16_add_8_masks_are_eq(<8 x i16> %v1, <8 x i16> %v2) {
+; CHECK-LABEL: @shuffle_16_add_8_masks_are_eq(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i16> [[V1:%.*]] to <16 x i8>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i16> [[V2:%.*]] to <16 x i8>
+; CHECK-NEXT:    [[TMP3:%.*]] = add <16 x i8> [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[ADD:%.*]] = shufflevector <16 x i8> [[TMP3]], <16 x i8> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1, i32 8, i32 9, i32 10, i32 11, i32 14, i32 15, i32 12, i32 13>
+; CHECK-NEXT:    ret <16 x i8> [[ADD]]
+;
+
+  %shuffle1 = shufflevector <8 x i16> %v1, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 1, i32 0, i32 4, i32 5, i32 7, i32 6>
+  %shuffle2 = shufflevector <8 x i16> %v2, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 1, i32 0, i32 4, i32 5, i32 7, i32 6>
+  %bc1 = bitcast <8 x i16> %shuffle1 to <16 x i8>
+  %bc2 = bitcast <8 x i16> %shuffle2 to <16 x i8>
+  %add = add <16 x i8> %bc1, %bc2
+  ret <16 x i8> %add
+}
+
+; TODO: Sink single shuffle.
+
+define <4 x i32> @shuffle_16_add_32_masks_are_eq_and_can_be_converted_up(<8 x i16> %v1, <8 x i16> %v2) {
+; CHECK-LABEL: @shuffle_16_add_32_masks_are_eq_and_can_be_converted_up(
+; CHECK-NEXT:    [[SHUFFLE1:%.*]] = shufflevector <8 x i16> [[V1:%.*]], <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[V2:%.*]], <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[BC1:%.*]] = bitcast <8 x i16> [[SHUFFLE1]] to <4 x i32>
+; CHECK-NEXT:    [[BC2:%.*]] = bitcast <8 x i16> [[SHUFFLE2]] to <4 x i32>
+; CHECK-NEXT:    [[ADD:%.*]] = add <4 x i32> [[BC2]], [[BC1]]
+; CHECK-NEXT:    ret <4 x i32> [[ADD]]
+;
+
+  %shuffle1 = shufflevector <8 x i16> %v1, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+  %shuffle2 = shufflevector <8 x i16> %v2, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+  %bc1 = bitcast <8 x i16> %shuffle1 to <4 x i32>
+  %bc2 = bitcast <8 x i16> %shuffle2 to <4 x i32>
+  %add = add <4 x i32> %bc1, %bc2
+  ret <4 x i32> %add
+}
+
+; TODO: Sink single shuffle.
+
+define <4 x i32> @shuffle_8_add_32_masks_are_eq_and_can_be_converted_up(<16 x i8> %v1, <16 x i8> %v2) {
+; CHECK-LABEL: @shuffle_8_add_32_masks_are_eq_and_can_be_converted_up(
+; CHECK-NEXT:    [[SHUFFLE1:%.*]] = shufflevector <16 x i8> [[V1:%.*]], <16 x i8> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <16 x i8> [[V2:%.*]], <16 x i8> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[BC1:%.*]] = bitcast <16 x i8> [[SHUFFLE1]] to <4 x i32>
+; CHECK-NEXT:    [[BC2:%.*]] = bitcast <16 x i8> [[SHUFFLE2]] to <4 x i32>
+; CHECK-NEXT:    [[ADD:%.*]] = add <4 x i32> [[BC2]], [[BC1]]
+; CHECK-NEXT:    ret <4 x i32> [[ADD]]
+;
+
+  %shuffle1 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %shuffle2 = shufflevector <16 x i8> %v2, <16 x i8> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %bc1 = bitcast <16 x i8> %shuffle1 to <4 x i32>
+  %bc2 = bitcast <16 x i8> %shuffle2 to <4 x i32>
+  %add = add <4 x i32> %bc1, %bc2
+  ret <4 x i32> %add
+}
+
+; shuffle<8 x i16>( bitcast<8 x i16>( shuffle<4 x i32>(v)))
+; TODO: Squash shuffles and widen type?
+
+define <8 x i16> @shuffle_32_bitcast_16_shuffle_16_can_be_converted_up(<4 x i32> %v1) {
+; CHECK-LABEL: @shuffle_32_bitcast_16_shuffle_16_can_be_converted_up(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[V1:%.*]] to <8 x i16>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[TMP2]], <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
+; CHECK-NEXT:    ret <8 x i16> [[SHUFFLE2]]
+;
+
+  %shuffle1 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
+  %bc1 = bitcast <4 x i32> %shuffle1 to <8 x i16>
+  %shuffle2 = shufflevector <8 x i16> %bc1, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
+  ret <8 x i16> %shuffle2
+}
+
+; shuffle<8 x i16>( bitcast<8 x i16>( shuffle<4 x i32>(v)))
+; TODO: Squash shuffles?
+
+define <8 x i16> @shuffle_32_bitcast_16_shuffle_16_can_not_be_converted_up(<4 x i32> %v1) {
+; CHECK-LABEL: @shuffle_32_bitcast_16_shuffle_16_can_not_be_converted_up(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[V1:%.*]] to <8 x i16>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[TMP2]], <8 x i16> undef, <8 x i32> <i32 5, i32 4, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <8 x i16> [[SHUFFLE2]]
+;
+
+  %shuffle1 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
+  %bc1 = bitcast <4 x i32> %shuffle1 to <8 x i16>
+  %shuffle2 = shufflevector <8 x i16> %bc1, <8 x i16> undef, <8 x i32> <i32 5, i32 4, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+  ret <8 x i16> %shuffle2
+}
+
+; shuffle<16 x i8>( bitcast<16 x i8>( shuffle<4 x i32>(v)))
+; TODO: Squash shuffles and widen type?
+
+define <16 x i8> @shuffle_32_bitcast_8_shuffle_8_can_be_converted_up(<4 x i32> %v1) {
+; CHECK-LABEL: @shuffle_32_bitcast_8_shuffle_8_can_be_converted_up(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[V1:%.*]] to <16 x i8>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <16 x i8> [[TMP2]], <16 x i8> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <16 x i8> [[SHUFFLE2]]
+;
+
+  %shuffle1 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
+  %bc1 = bitcast <4 x i32> %shuffle1 to <16 x i8>
+  %shuffle2 = shufflevector <16 x i8> %bc1, <16 x i8> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+  ret <16 x i8> %shuffle2
+}
+
+; shuffle<16 x i8>( bitcast<16 x i8>( shuffle<4 x i32>(v)))
+; TODO: Squash shuffles?
+
+define <16 x i8> @shuffle_32_bitcast_8_shuffle_8_can_not_be_converted_up(<4 x i32> %v1) {
+; CHECK-LABEL: @shuffle_32_bitcast_8_shuffle_8_can_not_be_converted_up(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[V1:%.*]] to <16 x i8>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <16 x i8> [[TMP2]], <16 x i8> undef, <16 x i32> <i32 5, i32 4, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    ret <16 x i8> [[SHUFFLE2]]
+;
+
+  %shuffle1 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
+  %bc1 = bitcast <4 x i32> %shuffle1 to <16 x i8>
+  %shuffle2 = shufflevector <16 x i8> %bc1, <16 x i8> undef, <16 x i32> <i32 5, i32 4, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x i8> %shuffle2
+}
+
+; shuffle<4 x i32>( bitcast<4 x i32>( shuffle<16 x i8>(v)))
+; TODO: Narrow, squash shuffles, and widen type?
+
+define <4 x i32> @shuffle_8_bitcast_32_shuffle_32_can_be_converted_up(<16 x i8> %v1) {
+; CHECK-LABEL: @shuffle_8_bitcast_32_shuffle_32_can_be_converted_up(
+; CHECK-NEXT:    [[SHUFFLE1:%.*]] = shufflevector <16 x i8> [[V1:%.*]], <16 x i8> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[BC1:%.*]] = bitcast <16 x i8> [[SHUFFLE1]] to <4 x i32>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <4 x i32> [[BC1]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
+; CHECK-NEXT:    ret <4 x i32> [[SHUFFLE2]]
+;
+
+  %shuffle1 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+  %bc1 = bitcast <16 x i8> %shuffle1 to <4 x i32>
+  %shuffle2 = shufflevector <4 x i32> %bc1, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
+  ret <4 x i32> %shuffle2
+}
+
+; shuffle<4 x i32>( bitcast<4 x i32>( shuffle<8 x i16>(v)))
+; TODO: Narrow, squash shuffles, and widen type?
+
+define <4 x i32> @shuffle_16_bitcast_32_shuffle_32_can_be_converted_up(<8 x i16> %v1) {
+; CHECK-LABEL: @shuffle_16_bitcast_32_shuffle_32_can_be_converted_up(
+; CHECK-NEXT:    [[SHUFFLE1:%.*]] = shufflevector <8 x i16> [[V1:%.*]], <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
+; CHECK-NEXT:    [[BC1:%.*]] = bitcast <8 x i16> [[SHUFFLE1]] to <4 x i32>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <4 x i32> [[BC1]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
+; CHECK-NEXT:    ret <4 x i32> [[SHUFFLE2]]
+;
+
+  %shuffle1 = shufflevector <8 x i16> %v1, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
+  %bc1 = bitcast <8 x i16> %shuffle1 to <4 x i32>
+  %shuffle2 = shufflevector <4 x i32> %bc1, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
+  ret <4 x i32> %shuffle2
+}
+
+; shuffle<4 x i32>( bitcast<4 x i32>( shuffle<16 x i8>(v)))
+; TODO: Narrow and squash shuffles?
+
+define <4 x i32> @shuffle_8_bitcast_32_shuffle_32_can_not_be_converted_up(<16 x i8> %v1) {
+; CHECK-LABEL: @shuffle_8_bitcast_32_shuffle_32_can_not_be_converted_up(
+; CHECK-NEXT:    [[SHUFFLE1:%.*]] = shufflevector <16 x i8> [[V1:%.*]], <16 x i8> undef, <16 x i32> <i32 9, i32 8, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[BC1:%.*]] = bitcast <16 x i8> [[SHUFFLE1]] to <4 x i32>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <4 x i32> [[BC1]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
+; CHECK-NEXT:    ret <4 x i32> [[SHUFFLE2]]
+;
+
+  %shuffle1 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> <i32 9, i32 8, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+  %bc1 = bitcast <16 x i8> %shuffle1 to <4 x i32>
+  %shuffle2 = shufflevector <4 x i32> %bc1, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
+  ret <4 x i32> %shuffle2
+}
+
+; shuffle<4 x i32>( bitcast<4 x i32>( shuffle<8 x i16>(v)))
+; TODO: Narrow and squash shuffles?
+
+define <4 x i32> @shuffle_16_bitcast_32_shuffle_32_can_not_be_converted_up(<8 x i16> %v1) {
+; CHECK-LABEL: @shuffle_16_bitcast_32_shuffle_32_can_not_be_converted_up(
+; CHECK-NEXT:    [[SHUFFLE1:%.*]] = shufflevector <8 x i16> [[V1:%.*]], <8 x i16> undef, <8 x i32> <i32 5, i32 4, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
+; CHECK-NEXT:    [[BC1:%.*]] = bitcast <8 x i16> [[SHUFFLE1]] to <4 x i32>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <4 x i32> [[BC1]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
+; CHECK-NEXT:    ret <4 x i32> [[SHUFFLE2]]
+;
+
+  %shuffle1 = shufflevector <8 x i16> %v1, <8 x i16> undef, <8 x i32> <i32 5, i32 4, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
+  %bc1 = bitcast <8 x i16> %shuffle1 to <4 x i32>
+  %shuffle2 = shufflevector <4 x i32> %bc1, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 0>
+  ret <4 x i32> %shuffle2
+}
+
+; shuffle<8 x i16>( bitcast<8 x i16>( shuffle<16 x i8>(v)))
+; TODO: Narrow, squash shuffles, and widen type?
+
+define <8 x i16> @shuffle_8_bitcast_16_shuffle_16_can__be_converted_up(<16 x i8> %v1) {
+; CHECK-LABEL: @shuffle_8_bitcast_16_shuffle_16_can__be_converted_up(
+; CHECK-NEXT:    [[SHUFFLE1:%.*]] = shufflevector <16 x i8> [[V1:%.*]], <16 x i8> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[BC1:%.*]] = bitcast <16 x i8> [[SHUFFLE1]] to <8 x i16>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[BC1]], <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
+; CHECK-NEXT:    ret <8 x i16> [[SHUFFLE2]]
+;
+
+  %shuffle1 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+  %bc1 = bitcast <16 x i8> %shuffle1 to <8 x i16>
+  %shuffle2 = shufflevector <8 x i16> %bc1, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
+  ret <8 x i16> %shuffle2
+}
+
+; shuffle<8 x i16>( bitcast<8 x i16>( shuffle<16 x i8>(v)))
+; TODO: Narrow and squash shuffles?
+
+define <8 x i16> @shuffle_8_bitcast_16_shuffle_16_can_not_be_converted_up(<16 x i8> %v1) {
+; CHECK-LABEL: @shuffle_8_bitcast_16_shuffle_16_can_not_be_converted_up(
+; CHECK-NEXT:    [[SHUFFLE1:%.*]] = shufflevector <16 x i8> [[V1:%.*]], <16 x i8> undef, <16 x i32> <i32 9, i32 8, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[BC1:%.*]] = bitcast <16 x i8> [[SHUFFLE1]] to <8 x i16>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <8 x i16> [[BC1]], <8 x i16> undef, <8 x i32> <i32 5, i32 4, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
+; CHECK-NEXT:    ret <8 x i16> [[SHUFFLE2]]
+;
+
+  %shuffle1 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> <i32 9, i32 8, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+  %bc1 = bitcast <16 x i8> %shuffle1 to <8 x i16>
+  %shuffle2 = shufflevector <8 x i16> %bc1, <8 x i16> undef, <8 x i32> <i32 5, i32 4, i32 6, i32 7, i32 2, i32 3, i32 0, i32 1>
+  ret <8 x i16> %shuffle2
+}
diff --git a/llvm/test/Transforms/PhaseOrdering/minmax.ll b/llvm/test/Transforms/PhaseOrdering/minmax.ll
new file mode 100644
index 0000000000000..6ddf49193f09e
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/minmax.ll
@@ -0,0 +1,65 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -O1                   -S < %s  | FileCheck %s --check-prefixes=ANY,OLDPM
+; RUN: opt -passes='default<O1>' -S < %s  | FileCheck %s --check-prefixes=ANY,NEWPM
+
+; This is an important benchmark for color-space-conversion.
+; It should reduce to contain only 1 'not' op.
+
+declare void @use(i8, i8, i8, i8)
+
+define void @cmyk(i8 %r, i8 %g, i8 %b) {
+; ANY-LABEL: @cmyk(
+; ANY-NEXT:  entry:
+; ANY-NEXT:    [[TMP0:%.*]] = icmp sgt i8 [[R:%.*]], [[B:%.*]]
+; ANY-NEXT:    [[TMP1:%.*]] = select i1 [[TMP0]], i8 [[R]], i8 [[B]]
+; ANY-NEXT:    [[TMP2:%.*]] = icmp sgt i8 [[TMP1]], [[G:%.*]]
+; ANY-NEXT:    [[TMP3:%.*]] = select i1 [[TMP2]], i8 [[TMP1]], i8 [[G]]
+; ANY-NEXT:    [[TMP4:%.*]] = xor i8 [[TMP3]], -1
+; ANY-NEXT:    [[SUB31:%.*]] = sub i8 [[TMP3]], [[R]]
+; ANY-NEXT:    [[SUB35:%.*]] = sub i8 [[TMP3]], [[G]]
+; ANY-NEXT:    [[SUB39:%.*]] = sub i8 [[TMP3]], [[B]]
+; ANY-NEXT:    call void @use(i8 [[SUB31]], i8 [[SUB35]], i8 [[SUB39]], i8 [[TMP4]])
+; ANY-NEXT:    ret void
+;
+entry:
+  %conv = sext i8 %r to i32
+  %sub = sub nsw i32 255, %conv
+  %conv1 = trunc i32 %sub to i8
+  %conv2 = sext i8 %g to i32
+  %sub3 = sub nsw i32 255, %conv2
+  %conv4 = trunc i32 %sub3 to i8
+  %conv5 = sext i8 %b to i32
+  %sub6 = sub nsw i32 255, %conv5
+  %conv7 = trunc i32 %sub6 to i8
+  %conv8 = sext i8 %conv1 to i32
+  %conv9 = sext i8 %conv4 to i32
+  %cmp = icmp slt i32 %conv8, %conv9
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %conv12 = sext i8 %conv7 to i32
+  %cmp13 = icmp slt i32 %conv8, %conv12
+  %cond = select i1 %cmp13, i32 %conv8, i32 %conv12
+  %conv17 = trunc i32 %cond to i8
+  br label %if.end
+
+if.else:
+  %conv19 = sext i8 %conv7 to i32
+  %cmp20 = icmp slt i32 %conv9, %conv19
+  %cond27 = select i1 %cmp20, i32 %conv9, i32 %conv19
+  %conv28 = trunc i32 %cond27 to i8
+  br label %if.end
+
+if.end:
+  %k.0 = phi i8 [ %conv17, %if.then ], [ %conv28, %if.else ]
+  %conv30 = sext i8 %k.0 to i32
+  %sub31 = sub nsw i32 %conv8, %conv30
+  %conv32 = trunc i32 %sub31 to i8
+  %sub35 = sub nsw i32 %conv9, %conv30
+  %conv36 = trunc i32 %sub35 to i8
+  %conv37 = sext i8 %conv7 to i32
+  %sub39 = sub nsw i32 %conv37, %conv30
+  %conv40 = trunc i32 %sub39 to i8
+  call void @use(i8 %conv32, i8 %conv36, i8 %conv40, i8 %k.0)
+  ret void
+}
diff --git a/llvm/test/Transforms/PhaseOrdering/vector-trunc.ll b/llvm/test/Transforms/PhaseOrdering/vector-trunc.ll
new file mode 100644
index 0000000000000..494b9a7a6c0bc
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/vector-trunc.ll
@@ -0,0 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -O2                   -S -data-layout="e" < %s | FileCheck %s --check-prefixes=ANY,OLDPM
+; RUN: opt -passes='default<O2>' -S -data-layout="e" < %s | FileCheck %s --check-prefixes=ANY,NEWPM
+
+define <4 x i16> @truncate(<4 x i32> %x) {
+; ANY-LABEL: @truncate(
+; ANY-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[X:%.*]] to <8 x i16>
+; ANY-NEXT:    [[V3:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; ANY-NEXT:    ret <4 x i16> [[V3]]
+;
+  %x0 = extractelement <4 x i32> %x, i32 0
+  %t0 = trunc i32 %x0 to i16
+  %v0 = insertelement <4 x i16> undef, i16 %t0, i32 0
+  %x1 = extractelement <4 x i32> %x, i32 1
+  %t1 = trunc i32 %x1 to i16
+  %v1 = insertelement <4 x i16> %v0, i16 %t1, i32 1
+  %x2 = extractelement <4 x i32> %x, i32 2
+  %t2 = trunc i32 %x2 to i16
+  %v2 = insertelement <4 x i16> %v1, i16 %t2, i32 2
+  %x3 = extractelement <4 x i32> %x, i32 3
+  %t3 = trunc i32 %x3 to i16
+  %v3 = insertelement <4 x i16> %v2, i16 %t3, i32 3
+  ret <4 x i16> %v3
+}
diff --git a/llvm/test/Transforms/SCCP/conditions-ranges-with-undef.ll b/llvm/test/Transforms/SCCP/conditions-ranges-with-undef.ll
new file mode 100644
index 0000000000000..4f2f66f26402e
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/conditions-ranges-with-undef.ll
@@ -0,0 +1,272 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -ipsccp -S | FileCheck %s
+
+declare void @use(i1)
+
+; We can simplify the conditions in the true block, because the condition
+; allows us to replace all uses of %a in the block with a  constant.
+define void @val_undef_eq() {
+; CHECK-LABEL: @val_undef_eq(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A:%.*]] = add i32 undef, 0
+; CHECK-NEXT:    [[BC_1:%.*]] = icmp eq i32 [[A]], 10
+; CHECK-NEXT:    br i1 [[BC_1]], label [[TRUE:%.*]], label [[FALSE:%.*]]
+; CHECK:       true:
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    ret void
+; CHECK:       false:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %a = add i32 undef, 0
+  %bc.1 = icmp eq i32 %a, 10
+  br i1 %bc.1, label %true, label %false
+
+true:
+  %f.1 = icmp ne i32 %a, 10
+  call void @use(i1 %f.1)
+  %f.2 = icmp eq i32 %a,  10
+  call void @use(i1 %f.2)
+  ret void
+
+false:
+  ret void
+}
+
+declare void @use.i32(i32)
+
+; It is not allowed to use the range information from the condition to remove
+; %a.127 = and ... in the true block, as %a could be undef.
+define void @val_undef_range() {
+; CHECK-LABEL: @val_undef_range(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A:%.*]] = add i32 undef, 0
+; CHECK-NEXT:    [[BC_1:%.*]] = icmp ult i32 [[A]], 127
+; CHECK-NEXT:    br i1 [[BC_1]], label [[TRUE:%.*]], label [[FALSE:%.*]]
+; CHECK:       true:
+; CHECK-NEXT:    [[F_1:%.*]] = icmp eq i32 [[A]], 128
+; CHECK-NEXT:    call void @use(i1 [[F_1]])
+; CHECK-NEXT:    [[A_127:%.*]] = and i32 [[A]], 127
+; CHECK-NEXT:    call void @use.i32(i32 [[A_127]])
+; CHECK-NEXT:    ret void
+; CHECK:       false:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %a = add i32 undef, 0
+  %bc.1 = icmp ult i32 %a, 127
+  br i1 %bc.1, label %true, label %false
+
+true:
+  %f.1 = icmp eq i32 %a, 128
+  call void @use(i1 %f.1)
+
+  %a.127 = and i32 %a, 127
+  call void @use.i32(i32 %a.127)
+  ret void
+
+false:
+  ret void
+}
+
+; All uses of %p can be replaced by a constant (10).
+define void @val_singlecrfromundef_range(i1 %cond) {
+; CHECK-LABEL: @val_singlecrfromundef_range(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[INC1:%.*]], label [[INC2:%.*]]
+; CHECK:       inc1:
+; CHECK-NEXT:    br label [[IF:%.*]]
+; CHECK:       inc2:
+; CHECK-NEXT:    br label [[IF]]
+; CHECK:       if:
+; CHECK-NEXT:    br label [[TRUE:%.*]]
+; CHECK:       true:
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    call void @use.i32(i32 10)
+; CHECK-NEXT:    ret void
+;
+entry:
+
+  br i1 %cond, label %inc1, label %inc2
+
+inc1:
+  br label %if
+
+inc2:
+  br label %if
+
+if:
+  %p = phi i32 [ 10, %inc1 ], [ undef, %inc2 ]
+  %bc.1 = icmp ult i32 %p, 127
+  br i1 %bc.1, label %true, label %false
+
+true:
+  %f.1 = icmp eq i32 %p, 128
+  call void @use(i1 %f.1)
+
+  %p.127 = and i32 %p, 127
+  call void @use.i32(i32 %p.127)
+  ret void
+
+false:
+  ret void
+}
+
+
+; It is not allowed to use the information from the condition ([0, 128))
+; to remove a.127.2 = and i32 %p, 127, as %p might be undef.
+define void @val_undef_to_cr_to_overdef_range(i32 %a, i1 %cond) {
+; CHECK-LABEL: @val_undef_to_cr_to_overdef_range(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A_127:%.*]] = and i32 [[A:%.*]], 127
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[INC1:%.*]], label [[INC2:%.*]]
+; CHECK:       inc1:
+; CHECK-NEXT:    br label [[IF:%.*]]
+; CHECK:       inc2:
+; CHECK-NEXT:    br label [[IF]]
+; CHECK:       if:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ [[A_127]], [[INC1]] ], [ undef, [[INC2]] ]
+; CHECK-NEXT:    [[BC_1:%.*]] = icmp ult i32 [[P]], 100
+; CHECK-NEXT:    br i1 [[BC_1]], label [[TRUE:%.*]], label [[FALSE:%.*]]
+; CHECK:       true:
+; CHECK-NEXT:    [[F_1:%.*]] = icmp eq i32 [[P]], 128
+; CHECK-NEXT:    call void @use(i1 [[F_1]])
+; CHECK-NEXT:    [[P_127:%.*]] = and i32 [[P]], 127
+; CHECK-NEXT:    call void @use.i32(i32 [[P_127]])
+; CHECK-NEXT:    ret void
+; CHECK:       false:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %a.127 = and i32 %a, 127
+  br i1 %cond, label %inc1, label %inc2
+
+inc1:
+  br label %if
+
+inc2:
+  br label %if
+
+if:
+  %p = phi i32 [ %a.127, %inc1 ], [ undef, %inc2 ]
+  %bc.1 = icmp ult i32 %p, 100
+  br i1 %bc.1, label %true, label %false
+
+true:
+  %f.1 = icmp eq i32 %p, 128
+  call void @use(i1 %f.1)
+
+  %p.127 = and i32 %p, 127
+  call void @use.i32(i32 %p.127)
+  ret void
+
+false:
+  ret void
+}
+
+; All uses of %p can be replaced by a constant (10), we are allowed to use it
+; as a bound too.
+define void @bound_singlecrfromundef(i32 %a, i1 %cond) {
+; CHECK-LABEL: @bound_singlecrfromundef(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[PRED:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[PRED]]
+; CHECK:       pred:
+; CHECK-NEXT:    [[BC_1:%.*]] = icmp ugt i32 [[A:%.*]], 10
+; CHECK-NEXT:    br i1 [[BC_1]], label [[TRUE:%.*]], label [[FALSE:%.*]]
+; CHECK:       true:
+; CHECK-NEXT:    [[F_1:%.*]] = icmp eq i32 [[A]], 5
+; CHECK-NEXT:    call void @use(i1 [[F_1]])
+; CHECK-NEXT:    [[T_1:%.*]] = icmp ne i32 [[A]], 5
+; CHECK-NEXT:    call void @use(i1 [[T_1]])
+; CHECK-NEXT:    [[A_127:%.*]] = and i32 [[A]], 127
+; CHECK-NEXT:    call void @use.i32(i32 [[A_127]])
+; CHECK-NEXT:    ret void
+; CHECK:       false:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 %cond, label %bb1, label %bb2
+
+bb1:
+  br label %pred
+
+bb2:
+  br label %pred
+
+pred:
+  %p = phi i32 [ undef, %bb1 ], [ 10, %bb2 ]
+  %bc.1 = icmp ugt i32 %a, %p
+  br i1 %bc.1, label %true, label %false
+
+true:
+  %f.1 = icmp eq i32 %a, 5
+  call void @use(i1 %f.1)
+
+  %t.1 = icmp ne i32 %a,  5
+  call void @use(i1 %t.1)
+
+  %a.127 = and i32 %a, 127
+  call void @use.i32(i32 %a.127)
+
+  ret void
+
+false:
+  ret void
+}
+
+; It is not allowed to use the information from %p as a bound, because an
+; incoming value is undef.
+define void @bound_range_and_undef(i32 %a, i1 %cond) {
+; CHECK-LABEL: @bound_range_and_undef(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A_10:%.*]] = and i32 [[A:%.*]], 127
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[PRED:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[PRED]]
+; CHECK:       pred:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ [[A_10]], [[BB1]] ], [ undef, [[BB2]] ]
+; CHECK-NEXT:    [[BC_1:%.*]] = icmp ugt i32 [[A]], [[P]]
+; CHECK-NEXT:    br i1 [[BC_1]], label [[TRUE:%.*]], label [[FALSE:%.*]]
+; CHECK:       true:
+; CHECK-NEXT:    [[F_1:%.*]] = icmp eq i32 [[A]], 300
+; CHECK-NEXT:    call void @use(i1 [[F_1]])
+; CHECK-NEXT:    [[A_127_2:%.*]] = and i32 [[P]], 127
+; CHECK-NEXT:    call void @use.i32(i32 [[A_127_2]])
+; CHECK-NEXT:    ret void
+; CHECK:       false:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %a.10 = and i32 %a, 127
+  br i1 %cond, label %bb1, label %bb2
+
+bb1:
+  br label %pred
+
+bb2:
+  br label %pred
+
+pred:
+  %p = phi i32 [ %a.10, %bb1 ], [ undef, %bb2 ]
+  %bc.1 = icmp ugt i32 %a, %p
+  br i1 %bc.1, label %true, label %false
+
+true:
+  %f.1 = icmp eq i32 %a, 300
+  call void @use(i1 %f.1)
+
+  %a.127.2 = and i32 %p, 127
+  call void @use.i32(i32 %a.127.2)
+
+  ret void
+
+false:
+  ret void
+}
diff --git a/llvm/test/Transforms/SCCP/conditions-ranges.ll b/llvm/test/Transforms/SCCP/conditions-ranges.ll
index 345e521d35c76..b722854abeb53 100644
--- a/llvm/test/Transforms/SCCP/conditions-ranges.ll
+++ b/llvm/test/Transforms/SCCP/conditions-ranges.ll
@@ -232,136 +232,6 @@ false:
   ret void
 }
 
-define void @loop.1() {
-entry:
-  br label %for.cond
-
-for.cond:                                         ; preds = %for.cond.cleanup13, %if.then
-  %i.0 = phi i32 [ 0, %entry ], [ %inc27, %for.cond.cleanup13 ]
-  %cmp9 = icmp sle i32 %i.0, 3
-  br i1 %cmp9, label %for.body, label %for.cond.cleanup
-
-for.cond.cleanup:                                 ; preds = %for.cond
-  ret void
-
-for.body:                                         ; preds = %for.cond
-  br label %for.cond11
-
-for.cond11:                                       ; preds = %arrayctor.cont21, %for.body
-   br label %for.cond.cleanup13
-
-for.cond.cleanup13:                               ; preds = %for.cond11
-  %inc27 = add nsw i32 %i.0, 1
-  br label %for.cond
-}
-
-
-define void @loop() {
-; CHECK-LABEL: @loop(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[FOR_COND:%.*]]
-; CHECK:       for.cond:
-; CHECK-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC27:%.*]], [[FOR_COND_CLEANUP13:%.*]] ]
-; CHECK-NEXT:    [[CMP9:%.*]] = icmp sle i32 [[I_0]], 3
-; CHECK-NEXT:    br i1 [[CMP9]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
-; CHECK:       for.cond.cleanup:
-; CHECK-NEXT:    ret void
-; CHECK:       for.body:
-; CHECK-NEXT:    br label [[FOR_COND11:%.*]]
-; CHECK:       for.cond11:
-; CHECK-NEXT:    [[J_0:%.*]] = phi i32 [ 0, [[FOR_BODY]] ], [ [[INC:%.*]], [[FOR_BODY14:%.*]] ]
-; CHECK-NEXT:    [[CMP12:%.*]] = icmp slt i32 [[J_0]], 2
-; CHECK-NEXT:    br i1 [[CMP12]], label [[FOR_BODY14]], label [[FOR_COND_CLEANUP13]]
-; CHECK:       for.cond.cleanup13:
-; CHECK-NEXT:    [[INC27]] = add nsw i32 [[I_0]], 1
-; CHECK-NEXT:    br label [[FOR_COND]]
-; CHECK:       for.body14:
-; CHECK-NEXT:    [[INC]] = add nsw i32 [[J_0]], 1
-; CHECK-NEXT:    br label [[FOR_COND11]]
-;
-entry:
-  br label %for.cond
-
-for.cond:                                         ; preds = %for.cond.cleanup13, %if.then
-  %i.0 = phi i32 [ 0, %entry ], [ %inc27, %for.cond.cleanup13 ]
-  %cmp9 = icmp sle i32 %i.0, 3
-  br i1 %cmp9, label %for.body, label %for.cond.cleanup
-
-for.cond.cleanup:                                 ; preds = %for.cond
-  ret void
-
-for.body:                                         ; preds = %for.cond
-  br label %for.cond11
-
-for.cond11:                                       ; preds = %arrayctor.cont21, %for.body
-  %j.0 = phi i32 [ 0, %for.body ], [ %inc, %for.body14 ]
-  %cmp12 = icmp slt i32 %j.0, 2
-  br i1 %cmp12, label %for.body14, label %for.cond.cleanup13
-
-for.cond.cleanup13:                               ; preds = %for.cond11
-  %inc27 = add nsw i32 %i.0, 1
-  br label %for.cond
-
-for.body14:
-  %inc = add nsw i32 %j.0, 1
-  br label %for.cond11
-}
-
-define i32 @udiv_1(i64 %sz) {
-; CHECK-LABEL: @udiv_1(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 4088, [[SZ:%.*]]
-; CHECK-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_END:%.*]]
-; CHECK:       cond.true:
-; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 4088, [[SZ]]
-; CHECK-NEXT:    br label [[COND_END]]
-; CHECK:       cond.end:
-; CHECK-NEXT:    [[COND:%.*]] = phi i64 [ [[DIV]], [[COND_TRUE]] ], [ 1, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[CONV:%.*]] = trunc i64 [[COND]] to i32
-; CHECK-NEXT:    ret i32 [[CONV]]
-;
-entry:
-  %cmp = icmp ugt i64 4088, %sz
-  br i1 %cmp, label %cond.true, label %cond.end
-
-cond.true:                                        ; preds = %entry
-  %div = udiv i64 4088, %sz
-  br label %cond.end
-
-cond.end:                                         ; preds = %entry, %cond.true
-  %cond = phi i64 [ %div, %cond.true ], [ 1, %entry ]
-  %conv = trunc i64 %cond to i32
-  ret i32 %conv
-}
-
-; Same as @udiv_1, but with the condition switched.
-define i32 @udiv_2(i64 %sz) {
-; CHECK-LABEL: @udiv_2(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[SZ:%.*]], 4088
-; CHECK-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_END:%.*]]
-; CHECK:       cond.true:
-; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 4088, [[SZ]]
-; CHECK-NEXT:    br label [[COND_END]]
-; CHECK:       cond.end:
-; CHECK-NEXT:    [[COND:%.*]] = phi i64 [ [[DIV]], [[COND_TRUE]] ], [ 1, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[CONV:%.*]] = trunc i64 [[COND]] to i32
-; CHECK-NEXT:    ret i32 [[CONV]]
-;
-entry:
-  %cmp = icmp ugt i64 %sz, 4088
-  br i1 %cmp, label %cond.true, label %cond.end
-
-cond.true:                                        ; preds = %entry
-  %div = udiv i64 4088, %sz
-  br label %cond.end
-
-cond.end:                                         ; preds = %entry, %cond.true
-  %cond = phi i64 [ %div, %cond.true ], [ 1, %entry ]
-  %conv = trunc i64 %cond to i32
-  ret i32 %conv
-}
-
 ; Test with 2 unrelated nested conditions.
 define void @f7_nested_conds(i32* %a, i32 %b) {
 ; CHECK-LABEL: @f7_nested_conds(
@@ -710,3 +580,273 @@ true: ; %b in [0, 256)
 false:
   ret void
 }
+
+define void @f11_contradiction(i32 %a, i32 %b) {
+; CHECK-LABEL: @f11_contradiction(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BC_1:%.*]] = icmp eq i32 [[B:%.*]], 10
+; CHECK-NEXT:    br i1 [[BC_1]], label [[TRUE:%.*]], label [[FALSE:%.*]]
+; CHECK:       true:
+; CHECK-NEXT:    br label [[FALSE]]
+; CHECK:       false:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %bc.1 = icmp eq i32 %b, 10
+  br i1 %bc.1, label %true, label %false
+
+true: ; %b in [10, 11)
+  %bc.2 = icmp eq i32 %b, 20
+  br i1 %bc.2, label %true.2, label %false
+
+true.2:
+  %f.1 = icmp eq i32 %b, 256
+  call void @use(i1 %f.1)
+  %f.2 = icmp ne i32 %b, 300
+  call void @use(i1 %f.2)
+  ret void
+
+false:
+  ret void
+}
+
+define void @f12_float(float %b) {
+; CHECK-LABEL: @f12_float(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BC_1:%.*]] = fcmp olt float 1.000000e+00, [[B:%.*]]
+; CHECK-NEXT:    br i1 [[BC_1]], label [[TRUE:%.*]], label [[FALSE:%.*]]
+; CHECK:       true:
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    ret void
+; CHECK:       false:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %a = fadd float 0.0, 1.0
+  %bc.1 = fcmp olt float %a, %b
+  br i1 %bc.1, label %true, label %false
+
+true: ; %b in [10, 11)
+  %f.1 = fcmp one float %a, 1.0
+  call void @use(i1 %f.1)
+
+  %t.1 = fcmp oeq float %a, 1.0
+  call void @use(i1 %t.1)
+  ret void
+
+false:
+  ret void
+}
+
+
+@A = global i32 17
+@B = global i32 17
+
+define void @f13_constexpr1() {
+; CHECK-LABEL: @f13_constexpr1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BC_1:%.*]] = icmp eq i32 add (i32 ptrtoint (i32* @A to i32), i32 10), 55
+; CHECK-NEXT:    br i1 [[BC_1]], label [[TRUE:%.*]], label [[FALSE:%.*]]
+; CHECK:       true:
+; CHECK-NEXT:    [[F_1:%.*]] = icmp eq i32 add (i32 ptrtoint (i32* @A to i32), i32 10), 10
+; CHECK-NEXT:    call void @use(i1 [[F_1]])
+; CHECK-NEXT:    [[F_2:%.*]] = icmp eq i32 add (i32 ptrtoint (i32* @A to i32), i32 10), 55
+; CHECK-NEXT:    call void @use(i1 [[F_2]])
+; CHECK-NEXT:    ret void
+; CHECK:       false:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %a = add i32 ptrtoint (i32* @A to i32), 10
+  %bc.1 = icmp eq i32 %a, 55
+  br i1 %bc.1, label %true, label %false
+
+true: ; %b in [10, 11)
+  %f.1 = icmp eq i32 %a, 10
+  call void @use(i1 %f.1)
+  %f.2 = icmp eq i32 %a, 55
+  call void @use(i1 %f.2)
+
+  ret void
+
+false:
+  ret void
+}
+
+; TODO: can we fold the compares in the true block?
+define void @f14_constexpr2() {
+; CHECK-LABEL: @f14_constexpr2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 icmp eq (i32 ptrtoint (i32* @A to i32), i32 ptrtoint (i32* @B to i32)), label [[TRUE:%.*]], label [[FALSE:%.*]]
+; CHECK:       true:
+; CHECK-NEXT:    call void @use(i1 icmp ne (i32 ptrtoint (i32* @A to i32), i32 ptrtoint (i32* @B to i32)))
+; CHECK-NEXT:    call void @use(i1 icmp eq (i32 ptrtoint (i32* @A to i32), i32 ptrtoint (i32* @B to i32)))
+; CHECK-NEXT:    ret void
+; CHECK:       false:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %a = add i32 ptrtoint (i32* @A to i32), 0
+  %b = add i32 ptrtoint (i32* @B to i32), 0
+  %bc.1 = icmp eq i32 %a, %b
+  br i1 %bc.1, label %true, label %false
+
+true: ; %b in [10, 11)
+  %f.1 = icmp ne i32 %a, %b
+  call void @use(i1 %f.1)
+  %f.2 = icmp eq i32 %a,  %b
+  call void @use(i1 %f.2)
+
+  ret void
+
+false:
+  ret void
+}
+
+define void @loop_1() {
+; CHECK-LABEL: @loop_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC27:%.*]], [[FOR_COND_CLEANUP13:%.*]] ]
+; CHECK-NEXT:    [[CMP9:%.*]] = icmp sle i32 [[I_0]], 3
+; CHECK-NEXT:    br i1 [[CMP9]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    ret void
+; CHECK:       for.body:
+; CHECK-NEXT:    br label [[FOR_COND11:%.*]]
+; CHECK:       for.cond11:
+; CHECK-NEXT:    br label [[FOR_COND_CLEANUP13]]
+; CHECK:       for.cond.cleanup13:
+; CHECK-NEXT:    [[INC27]] = add nsw i32 [[I_0]], 1
+; CHECK-NEXT:    br label [[FOR_COND]]
+;
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond.cleanup13, %if.then
+  %i.0 = phi i32 [ 0, %entry ], [ %inc27, %for.cond.cleanup13 ]
+  %cmp9 = icmp sle i32 %i.0, 3
+  br i1 %cmp9, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  ret void
+
+for.body:                                         ; preds = %for.cond
+  br label %for.cond11
+
+for.cond11:                                       ; preds = %arrayctor.cont21, %for.body
+  br label %for.cond.cleanup13
+
+for.cond.cleanup13:                               ; preds = %for.cond11
+  %inc27 = add nsw i32 %i.0, 1
+  br label %for.cond
+}
+
+
+define void @loop() {
+; CHECK-LABEL: @loop(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC27:%.*]], [[FOR_COND_CLEANUP13:%.*]] ]
+; CHECK-NEXT:    [[CMP9:%.*]] = icmp sle i32 [[I_0]], 3
+; CHECK-NEXT:    br i1 [[CMP9]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    ret void
+; CHECK:       for.body:
+; CHECK-NEXT:    br label [[FOR_COND11:%.*]]
+; CHECK:       for.cond11:
+; CHECK-NEXT:    [[J_0:%.*]] = phi i32 [ 0, [[FOR_BODY]] ], [ [[INC:%.*]], [[FOR_BODY14:%.*]] ]
+; CHECK-NEXT:    [[CMP12:%.*]] = icmp slt i32 [[J_0]], 2
+; CHECK-NEXT:    br i1 [[CMP12]], label [[FOR_BODY14]], label [[FOR_COND_CLEANUP13]]
+; CHECK:       for.cond.cleanup13:
+; CHECK-NEXT:    [[INC27]] = add nsw i32 [[I_0]], 1
+; CHECK-NEXT:    br label [[FOR_COND]]
+; CHECK:       for.body14:
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[J_0]], 1
+; CHECK-NEXT:    br label [[FOR_COND11]]
+;
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond.cleanup13, %if.then
+  %i.0 = phi i32 [ 0, %entry ], [ %inc27, %for.cond.cleanup13 ]
+  %cmp9 = icmp sle i32 %i.0, 3
+  br i1 %cmp9, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  ret void
+
+for.body:                                         ; preds = %for.cond
+  br label %for.cond11
+
+for.cond11:                                       ; preds = %arrayctor.cont21, %for.body
+  %j.0 = phi i32 [ 0, %for.body ], [ %inc, %for.body14 ]
+  %cmp12 = icmp slt i32 %j.0, 2
+  br i1 %cmp12, label %for.body14, label %for.cond.cleanup13
+
+for.cond.cleanup13:                               ; preds = %for.cond11
+  %inc27 = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.body14:
+  %inc = add nsw i32 %j.0, 1
+  br label %for.cond11
+}
+
+define i32 @udiv_1(i64 %sz) {
+; CHECK-LABEL: @udiv_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 4088, [[SZ:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_END:%.*]]
+; CHECK:       cond.true:
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 4088, [[SZ]]
+; CHECK-NEXT:    br label [[COND_END]]
+; CHECK:       cond.end:
+; CHECK-NEXT:    [[COND:%.*]] = phi i64 [ [[DIV]], [[COND_TRUE]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[CONV:%.*]] = trunc i64 [[COND]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+entry:
+  %cmp = icmp ugt i64 4088, %sz
+  br i1 %cmp, label %cond.true, label %cond.end
+
+cond.true:                                        ; preds = %entry
+  %div = udiv i64 4088, %sz
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i64 [ %div, %cond.true ], [ 1, %entry ]
+  %conv = trunc i64 %cond to i32
+  ret i32 %conv
+}
+
+; Same as @udiv_1, but with the condition switched.
+define i32 @udiv_2(i64 %sz) {
+; CHECK-LABEL: @udiv_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[SZ:%.*]], 4088
+; CHECK-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_END:%.*]]
+; CHECK:       cond.true:
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 4088, [[SZ]]
+; CHECK-NEXT:    br label [[COND_END]]
+; CHECK:       cond.end:
+; CHECK-NEXT:    [[COND:%.*]] = phi i64 [ [[DIV]], [[COND_TRUE]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[CONV:%.*]] = trunc i64 [[COND]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+entry:
+  %cmp = icmp ugt i64 %sz, 4088
+  br i1 %cmp, label %cond.true, label %cond.end
+
+cond.true:                                        ; preds = %entry
+  %div = udiv i64 4088, %sz
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i64 [ %div, %cond.true ], [ 1, %entry ]
+  %conv = trunc i64 %cond to i32
+  ret i32 %conv
+}
diff --git a/llvm/test/Transforms/SCCP/ip-ranges-casts.ll b/llvm/test/Transforms/SCCP/ip-ranges-casts.ll
new file mode 100644
index 0000000000000..c7a53c1189774
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/ip-ranges-casts.ll
@@ -0,0 +1,313 @@
+; RUN: opt < %s -ipsccp -S | FileCheck %s
+
+; x = [100, 301)
+define internal i1 @f.trunc(i32 %x) {
+; CHECK-LABEL: define internal i1 @f.trunc(i32 %x) {
+; CHECK-NEXT:    %t.1 = trunc i32 %x to i16
+; CHECK-NEXT:    %c.2 = icmp sgt i16 %t.1, 299
+; CHECK-NEXT:    %c.4 = icmp slt i16 %t.1, 101
+; CHECK-NEXT:    %res.1 = add i1 false, %c.2
+; CHECK-NEXT:    %res.2 = add i1 %res.1, false
+; CHECK-NEXT:    %res.3 = add i1 %res.2, %c.4
+; CHECK-NEXT:    %t.2 = trunc i32 %x to i8
+; CHECK-NEXT:    %c.5 = icmp sgt i8 %t.2, 44
+; CHECK-NEXT:    %c.6 = icmp sgt i8 %t.2, 43
+; CHECK-NEXT:    %c.7 = icmp slt i8 %t.2, 100
+; CHECK-NEXT:    %c.8 = icmp slt i8 %t.2, 101
+; CHECK-NEXT:    %res.4 = add i1 %res.3, %c.5
+; CHECK-NEXT:    %res.5 = add i1 %res.4, %c.6
+; CHECK-NEXT:    %res.6 = add i1 %res.5, %c.7
+; CHECK-NEXT:    %res.7 = add i1 %res.6, %c.8
+; CHECK-NEXT:    ret i1 %res.7
+
+  %t.1 = trunc i32 %x to i16
+  %c.1 = icmp sgt i16 %t.1, 300
+  %c.2 = icmp sgt i16 %t.1, 299
+  %c.3 = icmp slt i16 %t.1, 100
+  %c.4 = icmp slt i16 %t.1, 101
+  %res.1 = add i1 %c.1, %c.2
+  %res.2 = add i1 %res.1, %c.3
+  %res.3 = add i1 %res.2, %c.4
+  %t.2 = trunc i32 %x to i8
+  %c.5 = icmp sgt i8 %t.2, 300
+  %c.6 = icmp sgt i8 %t.2, 299
+  %c.7 = icmp slt i8 %t.2, 100
+  %c.8 = icmp slt i8 %t.2, 101
+  %res.4 = add i1 %res.3, %c.5
+  %res.5 = add i1 %res.4, %c.6
+  %res.6 = add i1 %res.5, %c.7
+  %res.7 = add i1 %res.6, %c.8
+  ret i1 %res.7
+}
+
+define i1 @caller1() {
+; CHECK-LABEL:  define i1 @caller1() {
+; CHECK-NEXT:    %call.1 = tail call i1 @f.trunc(i32 100)
+; CHECK-NEXT:    %call.2 = tail call i1 @f.trunc(i32 300)
+; CHECK-NEXT:    %res = and i1 %call.1, %call.2
+; CHECK-NEXT:    ret i1 %res
+;
+  %call.1 = tail call i1 @f.trunc(i32 100)
+  %call.2 = tail call i1 @f.trunc(i32 300)
+  %res = and i1 %call.1, %call.2
+  ret i1 %res
+}
+
+
+; x = [100, 301)
+define internal i1 @f.zext(i32 %x, i32 %y) {
+; CHECK-LABEL: define internal i1 @f.zext(i32 %x, i32 %y) {
+; CHECK-NEXT:    %t.1 = zext i32 %x to i64
+; CHECK-NEXT:    %c.2 = icmp sgt i64 %t.1, 299
+; CHECK-NEXT:    %c.4 = icmp slt i64 %t.1, 101
+; CHECK-NEXT:    %res.1 = add i1 false, %c.2
+; CHECK-NEXT:    %res.2 = add i1 %res.1, false
+; CHECK-NEXT:    %res.3 = add i1 %res.2, %c.4
+; CHECK-NEXT:    %t.2 = zext i32 %y to i64
+; CHECK-NEXT:    %c.5 = icmp sgt i64 %t.2, 300
+; CHECK-NEXT:    %c.6 = icmp sgt i64 %t.2, 299
+; CHECK-NEXT:    %c.8 = icmp slt i64 %t.2, 1
+; CHECK-NEXT:    %res.4 = add i1 %res.3, %c.5
+; CHECK-NEXT:    %res.5 = add i1 %res.4, %c.6
+; CHECK-NEXT:    %res.6 = add i1 %res.5, false
+; CHECK-NEXT:    %res.7 = add i1 %res.6, %c.8
+; CHECK-NEXT:    ret i1 %res.7
+
+  %t.1 = zext i32 %x to i64
+  %c.1 = icmp sgt i64 %t.1, 300
+  %c.2 = icmp sgt i64 %t.1, 299
+  %c.3 = icmp slt i64 %t.1, 100
+  %c.4 = icmp slt i64 %t.1, 101
+  %res.1 = add i1 %c.1, %c.2
+  %res.2 = add i1 %res.1, %c.3
+  %res.3 = add i1 %res.2, %c.4
+  %t.2 = zext i32 %y to i64
+  %c.5 = icmp sgt i64 %t.2, 300
+  %c.6 = icmp sgt i64 %t.2, 299
+  %c.7 = icmp slt i64 %t.2, 0
+  %c.8 = icmp slt i64 %t.2, 1
+  %res.4 = add i1 %res.3, %c.5
+  %res.5 = add i1 %res.4, %c.6
+  %res.6 = add i1 %res.5, %c.7
+  %res.7 = add i1 %res.6, %c.8
+  ret i1 %res.7
+}
+
+define i1 @caller.zext() {
+; CHECK-LABEL:  define i1 @caller.zext() {
+; CHECK-NEXT:    %call.1 = tail call i1 @f.zext(i32 100, i32 -120)
+; CHECK-NEXT:    %call.2 = tail call i1 @f.zext(i32 300, i32 900)
+; CHECK-NEXT:    %res = and i1 %call.1, %call.2
+; CHECK-NEXT:    ret i1 %res
+;
+  %call.1 = tail call i1 @f.zext(i32 100, i32 -120)
+  %call.2 = tail call i1 @f.zext(i32 300, i32 900)
+  %res = and i1 %call.1, %call.2
+  ret i1 %res
+}
+
+; x = [100, 301)
+define internal i1 @f.sext(i32 %x, i32 %y) {
+; CHECK-LABEL: define internal i1 @f.sext(i32 %x, i32 %y) {
+; CHECK-NEXT:    %t.1 = sext i32 %x to i64
+; CHECK-NEXT:    %c.2 = icmp sgt i64 %t.1, 299
+; CHECK-NEXT:    %c.4 = icmp slt i64 %t.1, 101
+; CHECK-NEXT:    %res.1 = add i1 false, %c.2
+; CHECK-NEXT:    %res.2 = add i1 %res.1, false
+; CHECK-NEXT:    %res.3 = add i1 %res.2, %c.4
+; CHECK-NEXT:    %t.2 = sext i32 %y to i64
+; CHECK-NEXT:    %c.6 = icmp sgt i64 %t.2, 899
+; CHECK-NEXT:    %c.8 = icmp slt i64 %t.2, -119
+; CHECK-NEXT:    %res.4 = add i1 %res.3, false
+; CHECK-NEXT:    %res.5 = add i1 %res.4, %c.6
+; CHECK-NEXT:    %res.6 = add i1 %res.5, false
+; CHECK-NEXT:    %res.7 = add i1 %res.6, %c.8
+; CHECK-NEXT:    ret i1 %res.7
+;
+  %t.1 = sext i32 %x to i64
+  %c.1 = icmp sgt i64 %t.1, 300
+  %c.2 = icmp sgt i64 %t.1, 299
+  %c.3 = icmp slt i64 %t.1, 100
+  %c.4 = icmp slt i64 %t.1, 101
+  %res.1 = add i1 %c.1, %c.2
+  %res.2 = add i1 %res.1, %c.3
+  %res.3 = add i1 %res.2, %c.4
+  %t.2 = sext i32 %y to i64
+  %c.5 = icmp sgt i64 %t.2, 900
+  %c.6 = icmp sgt i64 %t.2, 899
+  %c.7 = icmp slt i64 %t.2, -120
+  %c.8 = icmp slt i64 %t.2, -119
+  %res.4 = add i1 %res.3, %c.5
+  %res.5 = add i1 %res.4, %c.6
+  %res.6 = add i1 %res.5, %c.7
+  %res.7 = add i1 %res.6, %c.8
+  ret i1 %res.7
+}
+
+define i1 @caller.sext() {
+; CHECK-LABEL:  define i1 @caller.sext() {
+; CHECK-NEXT:    %call.1 = tail call i1 @f.sext(i32 100, i32 -120)
+; CHECK-NEXT:    %call.2 = tail call i1 @f.sext(i32 300, i32 900)
+; CHECK-NEXT:    %res = and i1 %call.1, %call.2
+; CHECK-NEXT:    ret i1 %res
+;
+  %call.1 = tail call i1 @f.sext(i32 100, i32 -120)
+  %call.2 = tail call i1 @f.sext(i32 300, i32 900)
+  %res = and i1 %call.1, %call.2
+  ret i1 %res
+}
+
+; There's nothing we can do besides going to the full range or overdefined.
+define internal i1 @f.fptosi(i32 %x) {
+; CHECK-LABEL: define internal i1 @f.fptosi(i32 %x) {
+; CHECK-NEXT:    %to.double = sitofp i32 %x to double
+; CHECK-NEXT:    %add = fadd double 0.000000e+00, %to.double
+; CHECK-NEXT:    %to.i32 = fptosi double %add to i32
+; CHECK-NEXT:    %c.1 = icmp sgt i32 %to.i32, 300
+; CHECK-NEXT:    %c.2 = icmp sgt i32 %to.i32, 299
+; CHECK-NEXT:    %c.3 = icmp slt i32 %to.i32, 100
+; CHECK-NEXT:    %c.4 = icmp slt i32 %to.i32, 101
+; CHECK-NEXT:    %res.1 = add i1 %c.1, %c.2
+; CHECK-NEXT:    %res.2 = add i1 %res.1, %c.3
+; CHECK-NEXT:    %res.3 = add i1 %res.2, %c.4
+; CHECK-NEXT:    ret i1 %res.3
+;
+  %to.double = sitofp i32 %x to double
+  %add = fadd double 0.000000e+00, %to.double
+  %to.i32 = fptosi double %add to i32
+  %c.1 = icmp sgt i32 %to.i32, 300
+  %c.2 = icmp sgt i32 %to.i32, 299
+  %c.3 = icmp slt i32 %to.i32, 100
+  %c.4 = icmp slt i32 %to.i32, 101
+  %res.1 = add i1 %c.1, %c.2
+  %res.2 = add i1 %res.1, %c.3
+  %res.3 = add i1 %res.2, %c.4
+  ret i1 %res.3
+}
+
+define i1 @caller.fptosi() {
+; CHECK-LABEL:  define i1 @caller.fptosi() {
+; CHECK-NEXT:    %call.1 = tail call i1 @f.fptosi(i32 100)
+; CHECK-NEXT:    %call.2 = tail call i1 @f.fptosi(i32 300)
+; CHECK-NEXT:    %res = and i1 %call.1, %call.2
+; CHECK-NEXT:    ret i1 %res
+;
+  %call.1 = tail call i1 @f.fptosi(i32 100)
+  %call.2 = tail call i1 @f.fptosi(i32 300)
+  %res = and i1 %call.1, %call.2
+  ret i1 %res
+}
+
+; There's nothing we can do besides going to the full range or overdefined.
+define internal i1 @f.fpext(i16 %x) {
+; CHECK-LABEL: define internal i1 @f.fpext(i16 %x) {
+; CHECK-NEXT:    %to.float = sitofp i16 %x to float
+; CHECK-NEXT:    %to.double = fpext float %to.float to double
+; CHECK-NEXT:    %to.i64 = fptoui float %to.float to i64
+; CHECK-NEXT:    %c.1 = icmp sgt i64 %to.i64, 300
+; CHECK-NEXT:    %c.2 = icmp sgt i64 %to.i64, 299
+; CHECK-NEXT:    %c.3 = icmp slt i64 %to.i64, 100
+; CHECK-NEXT:    %c.4 = icmp slt i64 %to.i64, 101
+; CHECK-NEXT:    %res.1 = add i1 %c.1, %c.2
+; CHECK-NEXT:    %res.2 = add i1 %res.1, %c.3
+; CHECK-NEXT:    %res.3 = add i1 %res.2, %c.4
+; CHECK-NEXT:    ret i1 %res.3
+;
+  %to.float = sitofp i16 %x to float
+  %to.double = fpext float %to.float  to double
+  %to.i64= fptoui float %to.float to i64
+  %c.1 = icmp sgt i64 %to.i64, 300
+  %c.2 = icmp sgt i64 %to.i64, 299
+  %c.3 = icmp slt i64 %to.i64, 100
+  %c.4 = icmp slt i64 %to.i64, 101
+  %res.1 = add i1 %c.1, %c.2
+  %res.2 = add i1 %res.1, %c.3
+  %res.3 = add i1 %res.2, %c.4
+  ret i1 %res.3
+}
+
+; There's nothing we can do besides going to the full range or overdefined.
+define i1 @caller.fpext() {
+; CHECK-LABEL:  define i1 @caller.fpext() {
+; CHECK-NEXT:    %call.1 = tail call i1 @f.fpext(i16 100)
+; CHECK-NEXT:    %call.2 = tail call i1 @f.fpext(i16 300)
+; CHECK-NEXT:    %res = and i1 %call.1, %call.2
+; CHECK-NEXT:    ret i1 %res
+;
+  %call.1 = tail call i1 @f.fpext(i16 100)
+  %call.2 = tail call i1 @f.fpext(i16 300)
+  %res = and i1 %call.1, %call.2
+  ret i1 %res
+}
+
+; There's nothing we can do besides going to the full range or overdefined.
+define internal i1 @f.inttoptr.ptrtoint(i64 %x) {
+; CHECK-LABEL: define internal i1 @f.inttoptr.ptrtoint(i64 %x) {
+; CHECK-NEXT:    %to.ptr = inttoptr i64 %x to i8*
+; CHECK-NEXT:    %to.i64 = ptrtoint i8* %to.ptr to i64
+; CHECK-NEXT:    %c.1 = icmp sgt i64 %to.i64, 300
+; CHECK-NEXT:    %c.2 = icmp sgt i64 %to.i64, 299
+; CHECK-NEXT:    %c.3 = icmp slt i64 %to.i64, 100
+; CHECK-NEXT:    %c.4 = icmp slt i64 %to.i64, 101
+; CHECK-NEXT:    %res.1 = add i1 %c.1, %c.2
+; CHECK-NEXT:    %res.2 = add i1 %res.1, %c.3
+; CHECK-NEXT:    %res.3 = add i1 %res.2, %c.4
+; CHECK-NEXT:    ret i1 %res.3
+;
+  %to.ptr = inttoptr i64 %x to i8*
+  %to.i64 = ptrtoint i8* %to.ptr to i64
+  %c.1 = icmp sgt i64 %to.i64, 300
+  %c.2 = icmp sgt i64 %to.i64, 299
+  %c.3 = icmp slt i64 %to.i64, 100
+  %c.4 = icmp slt i64 %to.i64, 101
+  %res.1 = add i1 %c.1, %c.2
+  %res.2 = add i1 %res.1, %c.3
+  %res.3 = add i1 %res.2, %c.4
+  ret i1 %res.3
+}
+
+define i1 @caller.inttoptr.ptrtoint() {
+; CHECK-LABEL:  define i1 @caller.inttoptr.ptrtoint() {
+; CHECK-NEXT:    %call.1 = tail call i1 @f.inttoptr.ptrtoint(i64 100)
+; CHECK-NEXT:    %call.2 = tail call i1 @f.inttoptr.ptrtoint(i64 300)
+; CHECK-NEXT:    %res = and i1 %call.1, %call.2
+; CHECK-NEXT:    ret i1 %res
+;
+  %call.1 = tail call i1 @f.inttoptr.ptrtoint(i64 100)
+  %call.2 = tail call i1 @f.inttoptr.ptrtoint(i64 300)
+  %res = and i1 %call.1, %call.2
+  ret i1 %res
+}
+
+; Make sure we do not create constant ranges for int to fp casts.
+define i1 @int_range_to_double_cast(i32 %a) {
+; CHECK-LABEL: define i1 @int_range_to_double_cast(i32 %a)
+; CHECK-NEXT:    %r = and i32 %a, 255
+; CHECK-NEXT:    %tmp4 = sitofp i32 %r to double
+; CHECK-NEXT:    %tmp10 = fadd double 0.000000e+00, %tmp4
+; CHECK-NEXT:    %tmp11 = fcmp olt double %tmp4, %tmp10
+; CHECK-NEXT:    ret i1 %tmp11
+;
+  %r = and i32 %a, 255
+  %tmp4 = sitofp i32 %r to double
+  %tmp10 = fadd double 0.000000e+00, %tmp4
+  %tmp11 = fcmp olt double %tmp4, %tmp10
+  ret i1 %tmp11
+}
+
+; Make sure we do not use ranges to propagate info from vectors.
+define i16 @vector_binop_and_cast() {
+; CHECK-LABEL: define i16 @vector_binop_and_cast(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %vecinit7 = insertelement <8 x i16> <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>, i16 undef, i32 0
+; CHECK-NEXT:    %rem = srem <8 x i16> <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>, %vecinit7
+; CHECK-NEXT:    %0 = bitcast <8 x i16> %rem to i128
+; CHECK-NEXT:    %1 = trunc i128 %0 to i16
+; CHECK-NEXT:    ret i16 %1
+entry:
+  %vecinit7 = insertelement <8 x i16> <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>, i16 undef, i32 0
+  %rem = srem <8 x i16> <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>, %vecinit7
+  %0 = bitcast <8 x i16> %rem to i128
+  %1 = trunc i128 %0 to i16
+  ret i16 %1
+}
diff --git a/llvm/test/Transforms/SCCP/phi-cycle.ll b/llvm/test/Transforms/SCCP/phi-cycle.ll
new file mode 100644
index 0000000000000..59c0ca2a6ff50
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/phi-cycle.ll
@@ -0,0 +1,33 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -ipsccp -S %s | FileCheck %s
+
+declare i1 @cond()
+
+define i32 @test() {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[C_1:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP]], label [[LATCH_2:%.*]]
+; CHECK:       latch.2:
+; CHECK-NEXT:    [[C_2:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_2]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  br label %loop
+
+loop:
+  %p = phi i32 [ undef, %entry ], [ 0, %latch.2 ], [ %p, %loop]
+  %c.1 = call i1 @cond()
+  br i1 %c.1, label %loop, label %latch.2
+
+latch.2:
+  %c.2 = call i1 @cond()
+  br i1 %c.2, label %loop, label %exit
+
+exit:
+  ret i32 %p
+}
diff --git a/llvm/test/Transforms/SCCP/range-and-ip.ll b/llvm/test/Transforms/SCCP/range-and-ip.ll
index d3e7b2f54d322..c9f9e765b4e26 100644
--- a/llvm/test/Transforms/SCCP/range-and-ip.ll
+++ b/llvm/test/Transforms/SCCP/range-and-ip.ll
@@ -14,9 +14,7 @@ define i1 @constant_and_undef(i64 %a) {
 ; CHECK:       bb2:
 ; CHECK-NEXT:    [[RANGE:%.*]] = and i64 [[A:%.*]], 255
 ; CHECK-NEXT:    [[C_3:%.*]] = call i1 @f1(i64 [[RANGE]])
-; CHECK-NEXT:    [[R_1:%.*]] = and i1 [[C_1]], [[C_2]]
-; CHECK-NEXT:    [[R_2:%.*]] = and i1 [[R_1]], [[C_3]]
-; CHECK-NEXT:    ret i1 [[R_2]]
+; CHECK-NEXT:    ret i1 true
 ;
   %c.1 = call i1 @f1(i64 undef)
   br label %bb1
@@ -37,9 +35,8 @@ declare void @sideeffect(i1, i64 %a)
 
 define internal i1 @f1(i64 %r) {
 ; CHECK-LABEL: define {{.*}} @f1(
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i64 [[R:%.*]], 256
-; CHECK-NEXT:    call void @sideeffect(i1 [[C]], i64 [[R]])
-; CHECK-NEXT:    ret i1 [[C]]
+; CHECK-NEXT:    call void @sideeffect(i1 true, i64 [[R:%.*]])
+; CHECK-NEXT:    ret i1 undef
 ;
   %c = icmp ult i64 %r, 256
   call void @sideeffect(i1 %c, i64 %r)
diff --git a/llvm/test/Transforms/SCCP/range-and.ll b/llvm/test/Transforms/SCCP/range-and.ll
index 3b349692db704..0bdc36f79f49e 100644
--- a/llvm/test/Transforms/SCCP/range-and.ll
+++ b/llvm/test/Transforms/SCCP/range-and.ll
@@ -128,7 +128,7 @@ bb3:
   ret i64 %res
 }
 
-define i1 @constant_range_and_255_100(i1 %cond, i64 %a) {
+define i64 @constant_range_and_255_100(i1 %cond, i64 %a) {
 ; CHECK-LABEL: @constant_range_and_255_100(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 [[COND:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
@@ -141,7 +141,8 @@ define i1 @constant_range_and_255_100(i1 %cond, i64 %a) {
 ; CHECK:       bb3:
 ; CHECK-NEXT:    [[P:%.*]] = phi i64 [ [[R_1]], [[BB1]] ], [ [[R_2]], [[BB2]] ]
 ; CHECK-NEXT:    [[P_AND:%.*]] = and i64 [[P]], 512
-; CHECK-NEXT:    ret i1 true
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    ret i64 [[P_AND]]
 ;
 entry:
   br i1 %cond, label %bb1, label %bb2
@@ -158,7 +159,8 @@ bb3:
   %p = phi i64 [ %r.1, %bb1 ], [ %r.2, %bb2 ]
   %p.and = and i64 %p, 512
   %c = icmp ult i64 %p.and, 256
-  ret i1 %c
+  call void @use(i1 %c)
+  ret i64 %p.and
 }
 
 
@@ -224,8 +226,7 @@ define i1 @constant_range_and_undef_3(i1 %cond, i64 %a) {
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    [[P:%.*]] = phi i64 [ undef, [[BB1]] ], [ [[R]], [[BB2]] ]
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i64 [[P]], 256
-; CHECK-NEXT:    ret i1 [[C]]
+; CHECK-NEXT:    ret i1 true
 ;
 entry:
   br i1 %cond, label %bb1, label %bb2
@@ -254,8 +255,7 @@ define i1 @constant_range_and_undef_3_switched_incoming(i1 %cond, i64 %a) {
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    [[P:%.*]] = phi i64 [ [[R]], [[BB1]] ], [ undef, [[BB2]] ]
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i64 [[P]], 256
-; CHECK-NEXT:    ret i1 [[C]]
+; CHECK-NEXT:    ret i1 true
 ;
 entry:
   br i1 %cond, label %bb1, label %bb2
diff --git a/llvm/test/Transforms/SROA/scalable-vectors.ll b/llvm/test/Transforms/SROA/scalable-vectors.ll
new file mode 100644
index 0000000000000..bda54e25b945d
--- /dev/null
+++ b/llvm/test/Transforms/SROA/scalable-vectors.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -sroa -S | FileCheck %s
+; RUN: opt < %s -passes=sroa -S | FileCheck %s
+
+; This test checks that SROA runs mem2reg on scalable vectors.
+
+define <vscale x 16 x i1> @alloca_nxv16i1(<vscale x 16 x i1> %pg) {
+; CHECK-LABEL: alloca_nxv16i1
+; CHECK-NEXT: ret <vscale x 16 x i1> %pg
+  %pg.addr = alloca <vscale x 16 x i1>
+  store <vscale x 16 x i1> %pg, <vscale x 16 x i1>* %pg.addr
+  %1 = load <vscale x 16 x i1>, <vscale x 16 x i1>* %pg.addr
+  ret <vscale x 16 x i1> %1
+}
+
+define <vscale x 16 x i8> @alloca_nxv16i8(<vscale x 16 x i8> %vec) {
+; CHECK-LABEL: alloca_nxv16i8
+; CHECK-NEXT: ret <vscale x 16 x i8> %vec
+  %vec.addr = alloca <vscale x 16 x i8>
+  store <vscale x 16 x i8> %vec, <vscale x 16 x i8>* %vec.addr
+  %1 = load <vscale x 16 x i8>, <vscale x 16 x i8>* %vec.addr
+  ret <vscale x 16 x i8> %1
+}
+
+; Test scalable alloca that can't be promoted. Mem2Reg only considers
+; non-volatile loads and stores for promotion.
+define <vscale x 16 x i8> @unpromotable_alloca(<vscale x 16 x i8> %vec) {
+; CHECK-LABEL: unpromotable_alloca
+; CHECK-NEXT: %vec.addr = alloca <vscale x 16 x i8>
+; CHECK-NEXT: store volatile <vscale x 16 x i8> %vec, <vscale x 16 x i8>* %vec.addr
+; CHECK-NEXT: %1 = load volatile <vscale x 16 x i8>, <vscale x 16 x i8>* %vec.addr
+; CHECK-NEXT: ret <vscale x 16 x i8> %1
+  %vec.addr = alloca <vscale x 16 x i8>
+  store volatile <vscale x 16 x i8> %vec, <vscale x 16 x i8>* %vec.addr
+  %1 = load volatile <vscale x 16 x i8>, <vscale x 16 x i8>* %vec.addr
+  ret <vscale x 16 x i8> %1
+}
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/inline.md5extbinary.afdo b/llvm/test/Transforms/SampleProfile/Inputs/inline.md5extbinary.afdo
new file mode 100644
index 0000000000000..c827d1b825f42
Binary files /dev/null and b/llvm/test/Transforms/SampleProfile/Inputs/inline.md5extbinary.afdo differ
diff --git a/llvm/test/Transforms/SampleProfile/profile-format.ll b/llvm/test/Transforms/SampleProfile/profile-format.ll
index 4f7ae0a06a417..c42fb5a5e0e5e 100644
--- a/llvm/test/Transforms/SampleProfile/profile-format.ll
+++ b/llvm/test/Transforms/SampleProfile/profile-format.ll
@@ -4,6 +4,8 @@
 ; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.compactbinary.afdo -S | FileCheck %s
 ; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/inline.extbinary.afdo -S | FileCheck %s
 ; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.extbinary.afdo -S | FileCheck %s
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/inline.md5extbinary.afdo -S | FileCheck %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.md5extbinary.afdo -S | FileCheck %s
 
 ; Original C++ test case
 ;
diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/pr45371-find-either-reset.ll b/llvm/test/Transforms/SeparateConstOffsetFromGEP/pr45371-find-either-reset.ll
new file mode 100644
index 0000000000000..efe426b718eb3
--- /dev/null
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/pr45371-find-either-reset.ll
@@ -0,0 +1,23 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -separate-const-offset-from-gep < %s | FileCheck %s
+
+@e = external global [4000 x i8], align 1
+
+define void @find_either_reset() {
+; CHECK-LABEL: @find_either_reset(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 65536, undef
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc i32 [[SUB]] to i8
+; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[TMP0]], 96
+; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i8 0 to i64
+; CHECK-NEXT:    [[IDXPROM1:%.*]] = sext i8 [[TMP1]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [4000 x i8], [4000 x i8]* @e, i64 [[IDXPROM]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    ret void
+;
+entry:
+  %sub = sub nsw i32 65536, undef
+  %0 = trunc i32 %sub to i8
+  %1 = add i8 %0, -4000
+  %arrayidx = getelementptr inbounds [4000 x i8], [4000 x i8]* @e, i8 0, i8 %1
+  ret void
+}
diff --git a/llvm/test/Transforms/StructurizeCFG/workarounds/needs-unified-loop-exits.ll b/llvm/test/Transforms/StructurizeCFG/workarounds/needs-unified-loop-exits.ll
new file mode 100644
index 0000000000000..32565503f73e6
--- /dev/null
+++ b/llvm/test/Transforms/StructurizeCFG/workarounds/needs-unified-loop-exits.ll
@@ -0,0 +1,173 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -unify-loop-exits -structurizecfg -S | FileCheck %s
+
+; The structurizer uses an RPO traversal over a region, along with a
+; manual hack that is meant to sort ensure that blocks within a loop
+; are all visited before visiting blocks outside the loop. But this
+; does not always work as expected. For example the results are
+; incorrect when multiple nested loops are involved.
+
+; The workaround for this is to unify loop exits. Each loop now
+; becomes an SESE region with a single header and a single exit. The
+; structurizer is a region pass, and it no longer sees the entire loop
+; nest in a single region. More importantly, for each loop, the only
+; block reachable outside the loop is the region exit, which avoids
+; any confusion in the hacked RPO traversal.
+
+; In the function below, B1 is an exiting block in outer loop H1. It's
+; successor inside the loop is the header of another loop H2. Due to
+; the incorrect traversal, B1 dominates all the blocks in the
+; structurized program, except the header H1.
+
+define void @exiting-block(i1 %PredH1, i1 %PredB2, i1 %PredB1, i1 %PredH2) {
+; CHECK-LABEL: @exiting-block(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PREDH1_INV:%.*]] = xor i1 [[PREDH1:%.*]], true
+; CHECK-NEXT:    [[PREDB2_INV:%.*]] = xor i1 [[PREDB2:%.*]], true
+; CHECK-NEXT:    br label [[H1:%.*]]
+; CHECK:       H1:
+; CHECK-NEXT:    br i1 [[PREDH1_INV]], label [[B1:%.*]], label [[FLOW3:%.*]]
+; CHECK:       Flow3:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i1 [ [[PREDB1:%.*]], [[B1]] ], [ [[PREDH1]], [[H1]] ]
+; CHECK-NEXT:    br i1 [[TMP0]], label [[H2:%.*]], label [[FLOW4:%.*]]
+; CHECK:       H2:
+; CHECK-NEXT:    br i1 [[PREDH2:%.*]], label [[B2:%.*]], label [[FLOW:%.*]]
+; CHECK:       B2:
+; CHECK-NEXT:    br i1 [[PREDB2_INV]], label [[L2:%.*]], label [[FLOW2:%.*]]
+; CHECK:       Flow:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i1 [ false, [[FLOW2]] ], [ true, [[H2]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = phi i1 [ [[TMP4:%.*]], [[FLOW2]] ], [ true, [[H2]] ]
+; CHECK-NEXT:    br i1 [[TMP2]], label [[LOOP_EXIT_GUARD1:%.*]], label [[H2]]
+; CHECK:       L2:
+; CHECK-NEXT:    br label [[FLOW2]]
+; CHECK:       L1:
+; CHECK-NEXT:    br label [[FLOW5:%.*]]
+; CHECK:       B1:
+; CHECK-NEXT:    br label [[FLOW3]]
+; CHECK:       C:
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+; CHECK:       Flow5:
+; CHECK-NEXT:    [[TMP3:%.*]] = phi i1 [ false, [[L1:%.*]] ], [ true, [[LOOP_EXIT_GUARD1]] ]
+; CHECK-NEXT:    br label [[FLOW4]]
+; CHECK:       loop.exit.guard:
+; CHECK-NEXT:    br i1 [[TMP5:%.*]], label [[C:%.*]], label [[EXIT]]
+; CHECK:       Flow2:
+; CHECK-NEXT:    [[TMP4]] = phi i1 [ false, [[L2]] ], [ true, [[B2]] ]
+; CHECK-NEXT:    br label [[FLOW]]
+; CHECK:       Flow4:
+; CHECK-NEXT:    [[TMP5]] = phi i1 [ false, [[FLOW5]] ], [ true, [[FLOW3]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = phi i1 [ [[TMP3]], [[FLOW5]] ], [ true, [[FLOW3]] ]
+; CHECK-NEXT:    br i1 [[TMP6]], label [[LOOP_EXIT_GUARD:%.*]], label [[H1]]
+; CHECK:       loop.exit.guard1:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[L1]], label [[FLOW5]]
+;
+entry:
+  br label %H1
+
+H1:                                               ; preds = %L1, %entry
+  br i1 %PredH1, label %H2, label %B1
+
+H2:                                               ; preds = %B1, %L2, %H1
+  br i1 %PredH2, label %B2, label %L1
+
+B2:                                               ; preds = %H2
+  br i1 %PredB2, label %exit, label %L2
+
+L2:                                               ; preds = %B2
+  br label %H2
+
+L1:                                               ; preds = %H2
+  br label %H1
+
+B1:                                               ; preds = %H1
+  br i1 %PredB1, label %H2, label %C
+
+C:                                                ; preds = %B1
+  br label %exit
+
+exit:                                             ; preds = %C, %B2
+  ret void
+}
+
+; The function below has three nested loops. Due to the incorrect
+; traversal, H2 dominates H3 in the structurized program, and the
+; backedge from L13 to H3 has no equivalent path.
+
+define void @incorrect-backedge(i1 %PredH2, i1 %PredH3, i1 %PredL2, i1 %PredL13, i1 %PredL1)
+; CHECK-LABEL: @incorrect-backedge(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PREDH2_INV:%.*]] = xor i1 [[PREDH2:%.*]], true
+; CHECK-NEXT:    [[PREDL2_INV:%.*]] = xor i1 [[PREDL2:%.*]], true
+; CHECK-NEXT:    [[PREDH3_INV:%.*]] = xor i1 [[PREDH3:%.*]], true
+; CHECK-NEXT:    [[PREDL13_INV:%.*]] = xor i1 [[PREDL13:%.*]], true
+; CHECK-NEXT:    br label [[H1:%.*]]
+; CHECK:       H1:
+; CHECK-NEXT:    br label [[H2:%.*]]
+; CHECK:       H2:
+; CHECK-NEXT:    br i1 [[PREDH2_INV]], label [[H3:%.*]], label [[FLOW4:%.*]]
+; CHECK:       H3:
+; CHECK-NEXT:    br i1 [[PREDH3_INV]], label [[L2:%.*]], label [[FLOW:%.*]]
+; CHECK:       L2:
+; CHECK-NEXT:    br i1 [[PREDL2_INV]], label [[L13:%.*]], label [[FLOW3:%.*]]
+; CHECK:       Flow:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i1 [ false, [[FLOW3]] ], [ true, [[H3]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i1 [ [[TMP6:%.*]], [[FLOW3]] ], [ true, [[H3]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = phi i1 [ [[TMP7:%.*]], [[FLOW3]] ], [ true, [[H3]] ]
+; CHECK-NEXT:    br i1 [[TMP2]], label [[LOOP_EXIT_GUARD2:%.*]], label [[H3]]
+; CHECK:       L13:
+; CHECK-NEXT:    br label [[FLOW3]]
+; CHECK:       Flow5:
+; CHECK-NEXT:    [[TMP3:%.*]] = phi i1 [ [[TMP8:%.*]], [[LOOP_EXIT_GUARD1:%.*]] ], [ true, [[LOOP_EXIT_GUARD:%.*]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = phi i1 [ false, [[LOOP_EXIT_GUARD1]] ], [ true, [[LOOP_EXIT_GUARD]] ]
+; CHECK-NEXT:    br i1 [[TMP4]], label [[L1:%.*]], label [[FLOW6:%.*]]
+; CHECK:       L1:
+; CHECK-NEXT:    br label [[FLOW6]]
+; CHECK:       Flow6:
+; CHECK-NEXT:    [[TMP5:%.*]] = phi i1 [ [[PREDL1:%.*]], [[L1]] ], [ [[TMP3]], [[FLOW5:%.*]] ]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[H1]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+; CHECK:       loop.exit.guard:
+; CHECK-NEXT:    br i1 [[TMP11:%.*]], label [[LOOP_EXIT_GUARD1]], label [[FLOW5]]
+; CHECK:       loop.exit.guard1:
+; CHECK-NEXT:    br label [[FLOW5]]
+; CHECK:       Flow3:
+; CHECK-NEXT:    [[TMP6]] = phi i1 [ true, [[L13]] ], [ false, [[L2]] ]
+; CHECK-NEXT:    [[TMP7]] = phi i1 [ [[PREDL13_INV]], [[L13]] ], [ true, [[L2]] ]
+; CHECK-NEXT:    br label [[FLOW]]
+; CHECK:       Flow4:
+; CHECK-NEXT:    [[TMP8]] = phi i1 [ [[TMP0]], [[LOOP_EXIT_GUARD2]] ], [ false, [[H2]] ]
+; CHECK-NEXT:    [[TMP9:%.*]] = phi i1 [ false, [[LOOP_EXIT_GUARD2]] ], [ true, [[H2]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = phi i1 [ [[TMP1]], [[LOOP_EXIT_GUARD2]] ], [ true, [[H2]] ]
+; CHECK-NEXT:    [[TMP11]] = xor i1 [[TMP9]], true
+; CHECK-NEXT:    br i1 [[TMP10]], label [[LOOP_EXIT_GUARD]], label [[H2]]
+; CHECK:       loop.exit.guard2:
+; CHECK-NEXT:    br label [[FLOW4]]
+;
+{
+entry:
+  br label %H1
+
+H1:
+  br label %H2
+
+H2:
+  br i1 %PredH2, label %L1, label %H3
+
+H3:
+  br i1 %PredH3, label %exit, label %L2
+
+L2:
+  br i1 %PredL2, label %H2, label %L13
+
+L13:
+  br i1 %PredL13, label %H3, label %H1
+
+L1:
+  br i1 %PredL1, label %exit, label %H1
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/UnifyLoopExits/basic.ll b/llvm/test/Transforms/UnifyLoopExits/basic.ll
new file mode 100644
index 0000000000000..11fa32c862438
--- /dev/null
+++ b/llvm/test/Transforms/UnifyLoopExits/basic.ll
@@ -0,0 +1,109 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -unify-loop-exits -S | FileCheck %s
+
+define void @loop_1(i1 %PredEntry, i1 %PredB, i1 %PredC, i1 %PredD) {
+; CHECK-LABEL: @loop_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[PREDENTRY:%.*]], label [[A:%.*]], label [[G:%.*]]
+; CHECK:       A:
+; CHECK-NEXT:    br label [[B:%.*]]
+; CHECK:       B:
+; CHECK-NEXT:    br i1 [[PREDB:%.*]], label [[C:%.*]], label [[LOOP_EXIT_GUARD:%.*]]
+; CHECK:       C:
+; CHECK-NEXT:    br i1 [[PREDC:%.*]], label [[D:%.*]], label [[LOOP_EXIT_GUARD]]
+; CHECK:       D:
+; CHECK-NEXT:    br i1 [[PREDD:%.*]], label [[A]], label [[LOOP_EXIT_GUARD]]
+; CHECK:       E:
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       F:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       G:
+; CHECK-NEXT:    br label [[F:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+; CHECK:       loop.exit.guard:
+; CHECK-NEXT:    [[GUARD_E:%.*]] = phi i1 [ true, [[B]] ], [ false, [[C]] ], [ false, [[D]] ]
+; CHECK-NEXT:    br i1 [[GUARD_E]], label [[E:%.*]], label [[F]]
+;
+entry:
+  br i1 %PredEntry, label %A, label %G
+
+A:
+  br label %B
+
+B:
+  br i1 %PredB, label %C, label %E
+
+C:
+  br i1 %PredC, label %D, label %F
+
+D:
+  br i1 %PredD, label %A, label %F
+
+E:
+  br label %exit
+
+F:
+  br label %exit
+
+G:
+  br label %F
+
+exit:
+  ret void
+}
+
+define void @loop_2(i1 %PredA, i1 %PredB, i1 %PredC) {
+; CHECK-LABEL: @loop_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[A:%.*]]
+; CHECK:       A:
+; CHECK-NEXT:    br i1 [[PREDA:%.*]], label [[B:%.*]], label [[LOOP_EXIT_GUARD:%.*]]
+; CHECK:       B:
+; CHECK-NEXT:    br i1 [[PREDB:%.*]], label [[C:%.*]], label [[LOOP_EXIT_GUARD]]
+; CHECK:       C:
+; CHECK-NEXT:    br i1 [[PREDC:%.*]], label [[D:%.*]], label [[LOOP_EXIT_GUARD]]
+; CHECK:       D:
+; CHECK-NEXT:    br label [[A]]
+; CHECK:       X:
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       Y:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       Z:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+; CHECK:       loop.exit.guard:
+; CHECK-NEXT:    [[GUARD_X:%.*]] = phi i1 [ true, [[A]] ], [ false, [[B]] ], [ false, [[C]] ]
+; CHECK-NEXT:    [[GUARD_Y:%.*]] = phi i1 [ false, [[A]] ], [ true, [[B]] ], [ false, [[C]] ]
+; CHECK-NEXT:    br i1 [[GUARD_X]], label [[X:%.*]], label [[LOOP_EXIT_GUARD1:%.*]]
+; CHECK:       loop.exit.guard1:
+; CHECK-NEXT:    br i1 [[GUARD_Y]], label [[Y:%.*]], label [[Z:%.*]]
+;
+entry:
+  br label %A
+
+A:
+  br i1 %PredA, label %B, label %X
+
+B:
+  br i1 %PredB, label %C, label %Y
+
+C:
+  br i1 %PredC, label %D, label %Z
+
+D:
+  br label %A
+
+X:
+  br label %exit
+
+Y:
+  br label %exit
+
+Z:
+  br label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/UnifyLoopExits/nested.ll b/llvm/test/Transforms/UnifyLoopExits/nested.ll
new file mode 100644
index 0000000000000..e5d04dccbbdf6
--- /dev/null
+++ b/llvm/test/Transforms/UnifyLoopExits/nested.ll
@@ -0,0 +1,80 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -unify-loop-exits -S | FileCheck %s
+
+define void @nested(i1 %PredB3, i1 %PredB4, i1 %PredA4, i1 %PredA3, i32 %X, i32 %Y, i32 %Z) {
+; CHECK-LABEL: @nested(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[A1:%.*]]
+; CHECK:       A1:
+; CHECK-NEXT:    br label [[B1:%.*]]
+; CHECK:       B1:
+; CHECK-NEXT:    br label [[B2:%.*]]
+; CHECK:       B2:
+; CHECK-NEXT:    [[X_INC:%.*]] = add i32 [[X:%.*]], 1
+; CHECK-NEXT:    br label [[B3:%.*]]
+; CHECK:       B3:
+; CHECK-NEXT:    br i1 [[PREDB3:%.*]], label [[B4:%.*]], label [[LOOP_EXIT_GUARD1:%.*]]
+; CHECK:       B4:
+; CHECK-NEXT:    br i1 [[PREDB4:%.*]], label [[B1]], label [[LOOP_EXIT_GUARD1]]
+; CHECK:       A2:
+; CHECK-NEXT:    br label [[A4:%.*]]
+; CHECK:       A3:
+; CHECK-NEXT:    br label [[A4]]
+; CHECK:       A4:
+; CHECK-NEXT:    [[A4_PHI:%.*]] = phi i32 [ [[Y:%.*]], [[A3:%.*]] ], [ [[X_INC_MOVED:%.*]], [[A2:%.*]] ]
+; CHECK-NEXT:    br i1 [[PREDA4:%.*]], label [[LOOP_EXIT_GUARD:%.*]], label [[A5:%.*]]
+; CHECK:       A5:
+; CHECK-NEXT:    br i1 [[PREDA3:%.*]], label [[LOOP_EXIT_GUARD]], label [[A1]]
+; CHECK:       C:
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[EXIT_PHI:%.*]] = phi i32 [ [[Z:%.*]], [[C:%.*]] ], [ [[EXIT_PHI_MOVED:%.*]], [[LOOP_EXIT_GUARD]] ]
+; CHECK-NEXT:    ret void
+; CHECK:       loop.exit.guard:
+; CHECK-NEXT:    [[GUARD_C:%.*]] = phi i1 [ true, [[A4]] ], [ false, [[A5]] ]
+; CHECK-NEXT:    [[EXIT_PHI_MOVED]] = phi i32 [ undef, [[A4]] ], [ [[A4_PHI]], [[A5]] ]
+; CHECK-NEXT:    br i1 [[GUARD_C]], label [[C]], label [[EXIT]]
+; CHECK:       loop.exit.guard1:
+; CHECK-NEXT:    [[GUARD_A3:%.*]] = phi i1 [ true, [[B3]] ], [ false, [[B4]] ]
+; CHECK-NEXT:    [[X_INC_MOVED]] = phi i32 [ [[X_INC]], [[B3]] ], [ [[X_INC]], [[B4]] ]
+; CHECK-NEXT:    br i1 [[GUARD_A3]], label [[A3]], label [[A2]]
+;
+entry:
+  br label %A1
+
+A1:
+  br label %B1
+
+B1:
+  br label %B2
+
+B2:
+  %X.inc = add i32 %X, 1
+  br label %B3
+
+B3:
+  br i1 %PredB3, label %B4, label %A3
+
+B4:
+  br i1 %PredB4, label %B1, label %A2
+
+A2:
+  br label %A4
+
+A3:
+  br label %A4
+
+A4:
+  %A4.phi = phi i32 [%Y, %A3], [%X.inc, %A2]
+  br i1 %PredA4, label %C, label %A5
+
+A5:
+  br i1 %PredA3, label %exit, label %A1
+
+C:
+  br label %exit
+
+exit:
+  %exit.phi = phi i32 [%A4.phi, %A5], [%Z, %C]
+  ret void
+}
diff --git a/llvm/test/Transforms/UnifyLoopExits/restore-ssa.ll b/llvm/test/Transforms/UnifyLoopExits/restore-ssa.ll
new file mode 100644
index 0000000000000..2db170cd48896
--- /dev/null
+++ b/llvm/test/Transforms/UnifyLoopExits/restore-ssa.ll
@@ -0,0 +1,238 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -unify-loop-exits -S | FileCheck %s
+
+; Loop consists of A and B:
+; - A is the header
+; - A and B are exiting blocks
+; - C and return are exit blocks.
+; Pattern: Value (%mytmp42) defined in exiting block (A) and used in
+;          exit block (return).
+;          The relevant code uses DT::dominates(Value,
+;          BasicBlock). This is misnamed because it actually checks
+;          strict dominance, causing the pattern to be miscompiled
+;          (the use receives an undef value).
+define i32 @exiting-used-in-exit(i32* %arg1, i32* %arg2) local_unnamed_addr align 2 {
+; CHECK-LABEL: @exiting-used-in-exit(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[A:%.*]]
+; CHECK:       A:
+; CHECK-NEXT:    [[MYTMP42:%.*]] = load i32, i32* [[ARG1:%.*]], align 4
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[MYTMP42]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[B:%.*]], label [[LOOP_EXIT_GUARD:%.*]]
+; CHECK:       B:
+; CHECK-NEXT:    [[MYTMP41:%.*]] = load i32, i32* [[ARG2:%.*]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[MYTMP41]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[A]], label [[LOOP_EXIT_GUARD]]
+; CHECK:       C:
+; CHECK-NEXT:    [[INC:%.*]] = add i32 [[MYTMP41_MOVED:%.*]], 1
+; CHECK-NEXT:    br label [[RETURN:%.*]]
+; CHECK:       return:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ [[INC]], [[C:%.*]] ], [ [[PHI_MOVED:%.*]], [[LOOP_EXIT_GUARD]] ]
+; CHECK-NEXT:    ret i32 [[PHI]]
+; CHECK:       loop.exit.guard:
+; CHECK-NEXT:    [[GUARD_RETURN:%.*]] = phi i1 [ true, [[A]] ], [ false, [[B]] ]
+; CHECK-NEXT:    [[PHI_MOVED]] = phi i32 [ [[MYTMP42]], [[A]] ], [ undef, [[B]] ]
+; CHECK-NEXT:    [[MYTMP41_MOVED]] = phi i32 [ undef, [[A]] ], [ [[MYTMP41]], [[B]] ]
+; CHECK-NEXT:    br i1 [[GUARD_RETURN]], label [[RETURN]], label [[C]]
+;
+entry:
+  br label %A
+
+A:
+  %mytmp42 = load i32, i32* %arg1, align 4
+  %cmp1 = icmp slt i32 %mytmp42, 0
+  br i1 %cmp1, label %B, label %return
+
+B:
+  %mytmp41 = load i32, i32* %arg2, align 4
+  %cmp = icmp slt i32 %mytmp41, 0
+  br i1 %cmp, label %A, label %C
+
+C:
+  %inc = add i32 %mytmp41, 1
+  br label %return
+
+return:
+  %phi = phi i32 [ %inc, %C ], [ %mytmp42, %A ]
+  ret i32 %phi
+}
+
+; Loop consists of A, B and C:
+; - A is the header
+; - A and C are exiting blocks
+; - B is an "internal" block that dominates exiting block C
+; - D and return are exit blocks.
+; Pattern: Value (%mytmp41) defined in internal block (B) and used in an
+;          exit block (D).
+define i32 @internal-used-in-exit(i32* %arg1, i32* %arg2) local_unnamed_addr align 2 {
+; CHECK-LABEL: @internal-used-in-exit(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MYTMP42:%.*]] = load i32, i32* [[ARG1:%.*]], align 4
+; CHECK-NEXT:    br label [[A:%.*]]
+; CHECK:       A:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[MYTMP42]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[B:%.*]], label [[LOOP_EXIT_GUARD:%.*]]
+; CHECK:       B:
+; CHECK-NEXT:    [[MYTMP41:%.*]] = load i32, i32* [[ARG2:%.*]], align 4
+; CHECK-NEXT:    br label [[C:%.*]]
+; CHECK:       C:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[MYTMP42]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[A]], label [[LOOP_EXIT_GUARD]]
+; CHECK:       D:
+; CHECK-NEXT:    [[INC:%.*]] = add i32 [[MYTMP41_MOVED:%.*]], 1
+; CHECK-NEXT:    br label [[RETURN:%.*]]
+; CHECK:       return:
+; CHECK-NEXT:    ret i32 0
+; CHECK:       loop.exit.guard:
+; CHECK-NEXT:    [[GUARD_RETURN:%.*]] = phi i1 [ true, [[A]] ], [ false, [[C]] ]
+; CHECK-NEXT:    [[MYTMP41_MOVED]] = phi i32 [ undef, [[A]] ], [ [[MYTMP41]], [[C]] ]
+; CHECK-NEXT:    br i1 [[GUARD_RETURN]], label [[RETURN]], label [[D:%.*]]
+;
+entry:
+  %mytmp42 = load i32, i32* %arg1, align 4
+  br label %A
+
+A:
+  %cmp1 = icmp slt i32 %mytmp42, 0
+  br i1 %cmp1, label %B, label %return
+
+B:
+  %mytmp41 = load i32, i32* %arg2, align 4
+  br label %C
+
+C:
+  %cmp = icmp slt i32 %mytmp42, 0
+  br i1 %cmp, label %A, label %D
+
+D:
+  %inc = add i32 %mytmp41, 1
+  br label %return
+
+return:
+  ret i32 0
+}
+
+; Loop consists of A, B and C:
+; - A is the header
+; - A and C are exiting blocks
+; - B is an "internal" block that dominates exiting block C
+; - D and return are exit blocks.
+; Pattern: %return contains a phi node that receives values from
+;          %entry, %A and %D. This mixes all the special cases in a single phi.
+define i32 @mixed-use-in-exit(i32* %arg1, i32* %arg2) local_unnamed_addr align 2 {
+; CHECK-LABEL: @mixed-use-in-exit(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MYTMP42:%.*]] = load i32, i32* [[ARG1:%.*]], align 4
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[MYTMP42]], 0
+; CHECK-NEXT:    br i1 [[CMP2]], label [[A:%.*]], label [[RETURN:%.*]]
+; CHECK:       A:
+; CHECK-NEXT:    [[MYTMP43:%.*]] = add i32 [[MYTMP42]], 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[MYTMP42]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[B:%.*]], label [[LOOP_EXIT_GUARD:%.*]]
+; CHECK:       B:
+; CHECK-NEXT:    [[MYTMP41:%.*]] = load i32, i32* [[ARG2:%.*]], align 4
+; CHECK-NEXT:    br label [[C:%.*]]
+; CHECK:       C:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[MYTMP42]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[A]], label [[LOOP_EXIT_GUARD]]
+; CHECK:       D:
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ [[MYTMP41_MOVED:%.*]], [[D:%.*]] ], [ [[MYTMP42]], [[ENTRY:%.*]] ], [ [[PHI_MOVED:%.*]], [[LOOP_EXIT_GUARD]] ]
+; CHECK-NEXT:    ret i32 [[PHI]]
+; CHECK:       loop.exit.guard:
+; CHECK-NEXT:    [[GUARD_RETURN:%.*]] = phi i1 [ true, [[A]] ], [ false, [[C]] ]
+; CHECK-NEXT:    [[PHI_MOVED]] = phi i32 [ [[MYTMP43]], [[A]] ], [ undef, [[C]] ]
+; CHECK-NEXT:    [[MYTMP41_MOVED]] = phi i32 [ undef, [[A]] ], [ [[MYTMP41]], [[C]] ]
+; CHECK-NEXT:    br i1 [[GUARD_RETURN]], label [[RETURN]], label [[D]]
+;
+entry:
+  %mytmp42 = load i32, i32* %arg1, align 4
+  %cmp2 = icmp slt i32 %mytmp42, 0
+  br i1 %cmp2, label %A, label %return
+
+A:
+  %mytmp43 = add i32 %mytmp42, 1
+  %cmp1 = icmp slt i32 %mytmp42, 0
+  br i1 %cmp1, label %B, label %return
+
+B:
+  %mytmp41 = load i32, i32* %arg2, align 4
+  br label %C
+
+C:
+  %cmp = icmp slt i32 %mytmp42, 0
+  br i1 %cmp, label %A, label %D
+
+D:
+  br label %return
+
+return:
+  %phi = phi i32 [ %mytmp41, %D ], [ %mytmp43, %A ], [%mytmp42, %entry]
+  ret i32 %phi
+}
+
+; Loop consists of A, B and C:
+; - A is the header
+; - A and C are exiting blocks
+; - B is an "internal" block that dominates exiting block C
+; - D and E are exit blocks.
+; Pattern: Value (%mytmp41) defined in internal block (B) and used in a
+;          downstream block not related to the loop (return). The use
+;          is a phi where the incoming block for %mytmp41 is not related
+;          to the loop (D).
+;          This pattern does not involve either the exiting blocks or
+;          the exit blocks, which catches any such assumptions built
+;          into the SSA reconstruction phase.
+define i32 @phi-via-external-block(i32* %arg1, i32* %arg2) local_unnamed_addr align 2 {
+; CHECK-LABEL: @phi-via-external-block(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MYTMP42:%.*]] = load i32, i32* [[ARG1:%.*]], align 4
+; CHECK-NEXT:    br label [[A:%.*]]
+; CHECK:       A:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[MYTMP42]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[B:%.*]], label [[LOOP_EXIT_GUARD:%.*]]
+; CHECK:       B:
+; CHECK-NEXT:    [[MYTMP41:%.*]] = load i32, i32* [[ARG2:%.*]], align 4
+; CHECK-NEXT:    br label [[C:%.*]]
+; CHECK:       C:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[MYTMP42]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[A]], label [[LOOP_EXIT_GUARD]]
+; CHECK:       D:
+; CHECK-NEXT:    br label [[RETURN:%.*]]
+; CHECK:       E:
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ [[MYTMP41_MOVED:%.*]], [[D:%.*]] ], [ [[MYTMP42]], [[E:%.*]] ]
+; CHECK-NEXT:    ret i32 [[PHI]]
+; CHECK:       loop.exit.guard:
+; CHECK-NEXT:    [[GUARD_E:%.*]] = phi i1 [ true, [[A]] ], [ false, [[C]] ]
+; CHECK-NEXT:    [[MYTMP41_MOVED]] = phi i32 [ undef, [[A]] ], [ [[MYTMP41]], [[C]] ]
+; CHECK-NEXT:    br i1 [[GUARD_E]], label [[E]], label [[D]]
+;
+entry:
+  %mytmp42 = load i32, i32* %arg1, align 4
+  br label %A
+
+A:
+  %cmp1 = icmp slt i32 %mytmp42, 0
+  br i1 %cmp1, label %B, label %E
+
+B:
+  %mytmp41 = load i32, i32* %arg2, align 4
+  br label %C
+
+C:
+  %cmp = icmp slt i32 %mytmp42, 0
+  br i1 %cmp, label %A, label %D
+
+D:
+  br label %return
+
+E:
+  br label %return
+
+return:
+  %phi = phi i32 [ %mytmp41, %D ], [ %mytmp42, %E ]
+  ret i32 %phi
+}
diff --git a/llvm/test/Transforms/UnifyLoopExits/switch.ll b/llvm/test/Transforms/UnifyLoopExits/switch.ll
new file mode 100644
index 0000000000000..618e54d95a7e1
--- /dev/null
+++ b/llvm/test/Transforms/UnifyLoopExits/switch.ll
@@ -0,0 +1,68 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -unify-loop-exits -S | FileCheck %s
+
+define void @loop_1(i32 %Value, i1 %PredEntry, i1 %PredD) {
+; CHECK-LABEL: @loop_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[PREDENTRY:%.*]], label [[A:%.*]], label [[G:%.*]]
+; CHECK:       A:
+; CHECK-NEXT:    br label [[B:%.*]]
+; CHECK:       B:
+; CHECK-NEXT:    br label [[NODEBLOCK:%.*]]
+; CHECK:       NodeBlock:
+; CHECK-NEXT:    [[PIVOT:%.*]] = icmp slt i32 [[VALUE:%.*]], 1
+; CHECK-NEXT:    br i1 [[PIVOT]], label [[LEAFBLOCK:%.*]], label [[LEAFBLOCK1:%.*]]
+; CHECK:       LeafBlock1:
+; CHECK-NEXT:    [[SWITCHLEAF2:%.*]] = icmp eq i32 [[VALUE]], 1
+; CHECK-NEXT:    br i1 [[SWITCHLEAF2]], label [[D:%.*]], label [[LOOP_EXIT_GUARD:%.*]]
+; CHECK:       LeafBlock:
+; CHECK-NEXT:    [[SWITCHLEAF:%.*]] = icmp eq i32 [[VALUE]], 0
+; CHECK-NEXT:    br i1 [[SWITCHLEAF]], label [[C:%.*]], label [[LOOP_EXIT_GUARD]]
+; CHECK:       C:
+; CHECK-NEXT:    br label [[D]]
+; CHECK:       D:
+; CHECK-NEXT:    br i1 [[PREDD:%.*]], label [[A]], label [[LOOP_EXIT_GUARD]]
+; CHECK:       NewDefault:
+; CHECK-NEXT:    br label [[X:%.*]]
+; CHECK:       X:
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       Y:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       G:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+; CHECK:       loop.exit.guard:
+; CHECK-NEXT:    [[GUARD_NEWDEFAULT:%.*]] = phi i1 [ true, [[LEAFBLOCK1]] ], [ true, [[LEAFBLOCK]] ], [ false, [[D]] ]
+; CHECK-NEXT:    br i1 [[GUARD_NEWDEFAULT]], label [[NEWDEFAULT:%.*]], label [[Y:%.*]]
+;
+entry:
+  br i1 %PredEntry, label %A, label %G
+
+A:
+  br label %B
+
+B:
+  switch i32 %Value, label %X [
+  i32 0, label %C
+  i32 1, label %D
+  ]
+
+C:
+  br label %D
+
+D:
+  br i1 %PredD, label %A, label %Y
+
+X:
+  br label %exit
+
+Y:
+  br label %exit
+
+G:
+  br label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/Util/assume-builder.ll b/llvm/test/Transforms/Util/assume-builder.ll
new file mode 100644
index 0000000000000..b39f5de6bdf32
--- /dev/null
+++ b/llvm/test/Transforms/Util/assume-builder.ll
@@ -0,0 +1,324 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes='assume-builder,verify' --enable-knowledge-retention -S %s | FileCheck %s --check-prefixes=BASIC
+; RUN: opt -passes='assume-builder,verify' --enable-knowledge-retention --assume-preserve-all -S %s | FileCheck %s --check-prefixes=ALL
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+declare void @func(i32*, i32*)
+declare void @func_cold(i32*) cold
+declare void @func_strbool(i32*) "no-jump-tables"
+declare void @func_many(i32*) "no-jump-tables" nounwind "less-precise-fpmad" willreturn norecurse
+declare void @func_argattr(i32* align 8, i32* nonnull) nounwind
+
+define void @test(i32* %P, i32* %P1, i32* %P2, i32* %P3) {
+; BASIC-LABEL: @test(
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P:%.*]], i64 16), "nonnull"(i32* [[P]]) ]
+; BASIC-NEXT:    call void @func(i32* nonnull dereferenceable(16) [[P]], i32* null)
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P1:%.*]], i64 12), "nonnull"(i32* [[P]]) ]
+; BASIC-NEXT:    call void @func(i32* dereferenceable(12) [[P1]], i32* nonnull [[P]])
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "cold"(), "dereferenceable"(i32* [[P1]], i64 12) ]
+; BASIC-NEXT:    call void @func_cold(i32* dereferenceable(12) [[P1]]) #0
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "cold"(), "dereferenceable"(i32* [[P1]], i64 12) ]
+; BASIC-NEXT:    call void @func_cold(i32* dereferenceable(12) [[P1]])
+; BASIC-NEXT:    call void @func(i32* [[P1]], i32* [[P]])
+; BASIC-NEXT:    call void @func_strbool(i32* [[P1]])
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 8), "dereferenceable"(i32* [[P]], i64 16) ]
+; BASIC-NEXT:    call void @func(i32* dereferenceable(16) [[P]], i32* dereferenceable(8) [[P]])
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[P1]], i64 8) ]
+; BASIC-NEXT:    call void @func_many(i32* align 8 [[P1]])
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[P2:%.*]], i64 8), "nonnull"(i32* [[P3:%.*]]) ]
+; BASIC-NEXT:    call void @func_argattr(i32* [[P2]], i32* [[P3]])
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "nonnull"(i32* [[P]]), "nonnull"(i32* [[P1]]) ]
+; BASIC-NEXT:    call void @func(i32* nonnull [[P1]], i32* nonnull [[P]])
+; BASIC-NEXT:    ret void
+;
+; ALL-LABEL: @test(
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P:%.*]], i64 16), "nonnull"(i32* [[P]]) ]
+; ALL-NEXT:    call void @func(i32* nonnull dereferenceable(16) [[P]], i32* null)
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P1:%.*]], i64 12), "nonnull"(i32* [[P]]) ]
+; ALL-NEXT:    call void @func(i32* dereferenceable(12) [[P1]], i32* nonnull [[P]])
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "cold"(), "dereferenceable"(i32* [[P1]], i64 12) ]
+; ALL-NEXT:    call void @func_cold(i32* dereferenceable(12) [[P1]]) #0
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "cold"(), "dereferenceable"(i32* [[P1]], i64 12) ]
+; ALL-NEXT:    call void @func_cold(i32* dereferenceable(12) [[P1]])
+; ALL-NEXT:    call void @func(i32* [[P1]], i32* [[P]])
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "no-jump-tables"() ]
+; ALL-NEXT:    call void @func_strbool(i32* [[P1]])
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32* [[P]], i64 8), "dereferenceable"(i32* [[P]], i64 16) ]
+; ALL-NEXT:    call void @func(i32* dereferenceable(16) [[P]], i32* dereferenceable(8) [[P]])
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[P1]], i64 8), "less-precise-fpmad"(), "no-jump-tables"(), "norecurse"(), "nounwind"(), "willreturn"() ]
+; ALL-NEXT:    call void @func_many(i32* align 8 [[P1]])
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[P2:%.*]], i64 8), "nonnull"(i32* [[P3:%.*]]), "nounwind"() ]
+; ALL-NEXT:    call void @func_argattr(i32* [[P2]], i32* [[P3]])
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "nonnull"(i32* [[P]]), "nonnull"(i32* [[P1]]) ]
+; ALL-NEXT:    call void @func(i32* nonnull [[P1]], i32* nonnull [[P]])
+; ALL-NEXT:    ret void
+;
+  call void @func(i32* nonnull dereferenceable(16) %P, i32* null)
+  call void @func(i32* dereferenceable(12) %P1, i32* nonnull %P)
+  call void @func_cold(i32* dereferenceable(12) %P1) cold
+  call void @func_cold(i32* dereferenceable(12) %P1)
+  call void @func(i32* %P1, i32* %P)
+  call void @func_strbool(i32* %P1)
+  call void @func(i32* dereferenceable(16) %P, i32* dereferenceable(8) %P)
+  call void @func_many(i32* align 8 %P1)
+  call void @func_argattr(i32* %P2, i32* %P3)
+  call void @func(i32* nonnull %P1, i32* nonnull %P)
+  ret void
+}
+
+%struct.S = type { i32, i8, i32* }
+
+define i32 @test2(%struct.S* %0, i32* %1, i8* %2) {
+; BASIC-LABEL: @test2(
+; BASIC-NEXT:    [[TMP4:%.*]] = alloca %struct.S*, align 8
+; BASIC-NEXT:    [[TMP5:%.*]] = alloca i32*, align 8
+; BASIC-NEXT:    [[TMP6:%.*]] = alloca i8*, align 8
+; BASIC-NEXT:    [[TMP7:%.*]] = alloca [[STRUCT_S:%.*]], align 8
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "align"(%struct.S** [[TMP4]], i64 8), "dereferenceable"(%struct.S** [[TMP4]], i64 8), "nonnull"(%struct.S** [[TMP4]]) ]
+; BASIC-NEXT:    store %struct.S* [[TMP0:%.*]], %struct.S** [[TMP4]], align 8
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32** [[TMP5]], i64 8), "dereferenceable"(i32** [[TMP5]], i64 8), "nonnull"(i32** [[TMP5]]) ]
+; BASIC-NEXT:    store i32* [[TMP1:%.*]], i32** [[TMP5]], align 8
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i8** [[TMP6]], i64 8), "nonnull"(i8** [[TMP6]]) ]
+; BASIC-NEXT:    store i8* [[TMP2:%.*]], i8** [[TMP6]]
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32** [[TMP5]], i64 8), "dereferenceable"(i32** [[TMP5]], i64 8), "nonnull"(i32** [[TMP5]]) ]
+; BASIC-NEXT:    [[TMP8:%.*]] = load i32*, i32** [[TMP5]], align 8
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[TMP8]], i64 4), "dereferenceable"(i32* [[TMP8]], i64 4), "nonnull"(i32* [[TMP8]]) ]
+; BASIC-NEXT:    [[TMP9:%.*]] = load i32, i32* [[TMP8]], align 4
+; BASIC-NEXT:    [[TMP10:%.*]] = trunc i32 [[TMP9]] to i8
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "align"(i8** [[TMP6]], i64 8), "dereferenceable"(i8** [[TMP6]], i64 8), "nonnull"(i8** [[TMP6]]) ]
+; BASIC-NEXT:    [[TMP11:%.*]] = load i8*, i8** [[TMP6]], align 8
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i8* [[TMP11]], i64 1), "nonnull"(i8* [[TMP11]]) ]
+; BASIC-NEXT:    store i8 [[TMP10]], i8* [[TMP11]], align 1
+; BASIC-NEXT:    [[TMP12:%.*]] = bitcast %struct.S* [[TMP7]] to i8*
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(%struct.S** [[TMP4]], i64 8), "nonnull"(%struct.S** [[TMP4]]) ]
+; BASIC-NEXT:    [[TMP13:%.*]] = load %struct.S*, %struct.S** [[TMP4]]
+; BASIC-NEXT:    [[TMP14:%.*]] = bitcast %struct.S* [[TMP13]] to i8*
+; BASIC-NEXT:    [[TMP15:%.*]] = bitcast %struct.S* [[TMP7]] to i8*
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "align"(%struct.S** [[TMP4]], i64 8), "dereferenceable"(%struct.S** [[TMP4]], i64 8), "nonnull"(%struct.S** [[TMP4]]) ]
+; BASIC-NEXT:    [[TMP16:%.*]] = load %struct.S*, %struct.S** [[TMP4]], align 8
+; BASIC-NEXT:    [[TMP17:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.S* [[TMP16]], i32 0, i32 0
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[TMP17]], i64 8), "dereferenceable"(i32* [[TMP17]], i64 4), "nonnull"(i32* [[TMP17]]) ]
+; BASIC-NEXT:    [[TMP18:%.*]] = load i32, i32* [[TMP17]], align 8
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "align"(%struct.S** [[TMP4]], i64 8), "dereferenceable"(%struct.S** [[TMP4]], i64 8), "nonnull"(%struct.S** [[TMP4]]) ]
+; BASIC-NEXT:    [[TMP19:%.*]] = load %struct.S*, %struct.S** [[TMP4]], align 8
+; BASIC-NEXT:    [[TMP20:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.S* [[TMP19]], i32 0, i32 1
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "align"(i8* [[TMP20]], i64 4), "dereferenceable"(i8* [[TMP20]], i64 1), "nonnull"(i8* [[TMP20]]) ]
+; BASIC-NEXT:    [[TMP21:%.*]] = load i8, i8* [[TMP20]], align 4
+; BASIC-NEXT:    [[TMP22:%.*]] = sext i8 [[TMP21]] to i32
+; BASIC-NEXT:    [[TMP23:%.*]] = add nsw i32 [[TMP18]], [[TMP22]]
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "align"(%struct.S** [[TMP4]], i64 8), "dereferenceable"(%struct.S** [[TMP4]], i64 8), "nonnull"(%struct.S** [[TMP4]]) ]
+; BASIC-NEXT:    [[TMP24:%.*]] = load %struct.S*, %struct.S** [[TMP4]], align 8
+; BASIC-NEXT:    [[TMP25:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.S* [[TMP24]], i32 0, i32 2
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32** [[TMP25]], i64 8), "dereferenceable"(i32** [[TMP25]], i64 8), "nonnull"(i32** [[TMP25]]) ]
+; BASIC-NEXT:    [[TMP26:%.*]] = load i32*, i32** [[TMP25]], align 8
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[TMP26]], i64 4), "dereferenceable"(i32* [[TMP26]], i64 4), "nonnull"(i32* [[TMP26]]) ]
+; BASIC-NEXT:    [[TMP27:%.*]] = load i32, i32* [[TMP26]], align 4
+; BASIC-NEXT:    [[TMP28:%.*]] = add nsw i32 [[TMP23]], [[TMP27]]
+; BASIC-NEXT:    ret i32 [[TMP28]]
+;
+; ALL-LABEL: @test2(
+; ALL-NEXT:    [[TMP4:%.*]] = alloca %struct.S*, align 8
+; ALL-NEXT:    [[TMP5:%.*]] = alloca i32*, align 8
+; ALL-NEXT:    [[TMP6:%.*]] = alloca i8*, align 8
+; ALL-NEXT:    [[TMP7:%.*]] = alloca [[STRUCT_S:%.*]], align 8
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "align"(%struct.S** [[TMP4]], i64 8), "dereferenceable"(%struct.S** [[TMP4]], i64 8), "nonnull"(%struct.S** [[TMP4]]) ]
+; ALL-NEXT:    store %struct.S* [[TMP0:%.*]], %struct.S** [[TMP4]], align 8
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32** [[TMP5]], i64 8), "dereferenceable"(i32** [[TMP5]], i64 8), "nonnull"(i32** [[TMP5]]) ]
+; ALL-NEXT:    store i32* [[TMP1:%.*]], i32** [[TMP5]], align 8
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i8** [[TMP6]], i64 8), "nonnull"(i8** [[TMP6]]) ]
+; ALL-NEXT:    store i8* [[TMP2:%.*]], i8** [[TMP6]]
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32** [[TMP5]], i64 8), "dereferenceable"(i32** [[TMP5]], i64 8), "nonnull"(i32** [[TMP5]]) ]
+; ALL-NEXT:    [[TMP8:%.*]] = load i32*, i32** [[TMP5]], align 8
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[TMP8]], i64 4), "dereferenceable"(i32* [[TMP8]], i64 4), "nonnull"(i32* [[TMP8]]) ]
+; ALL-NEXT:    [[TMP9:%.*]] = load i32, i32* [[TMP8]], align 4
+; ALL-NEXT:    [[TMP10:%.*]] = trunc i32 [[TMP9]] to i8
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "align"(i8** [[TMP6]], i64 8), "dereferenceable"(i8** [[TMP6]], i64 8), "nonnull"(i8** [[TMP6]]) ]
+; ALL-NEXT:    [[TMP11:%.*]] = load i8*, i8** [[TMP6]], align 8
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i8* [[TMP11]], i64 1), "nonnull"(i8* [[TMP11]]) ]
+; ALL-NEXT:    store i8 [[TMP10]], i8* [[TMP11]], align 1
+; ALL-NEXT:    [[TMP12:%.*]] = bitcast %struct.S* [[TMP7]] to i8*
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(%struct.S** [[TMP4]], i64 8), "nonnull"(%struct.S** [[TMP4]]) ]
+; ALL-NEXT:    [[TMP13:%.*]] = load %struct.S*, %struct.S** [[TMP4]]
+; ALL-NEXT:    [[TMP14:%.*]] = bitcast %struct.S* [[TMP13]] to i8*
+; ALL-NEXT:    [[TMP15:%.*]] = bitcast %struct.S* [[TMP7]] to i8*
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "align"(%struct.S** [[TMP4]], i64 8), "dereferenceable"(%struct.S** [[TMP4]], i64 8), "nonnull"(%struct.S** [[TMP4]]) ]
+; ALL-NEXT:    [[TMP16:%.*]] = load %struct.S*, %struct.S** [[TMP4]], align 8
+; ALL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.S* [[TMP16]], i32 0, i32 0
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[TMP17]], i64 8), "dereferenceable"(i32* [[TMP17]], i64 4), "nonnull"(i32* [[TMP17]]) ]
+; ALL-NEXT:    [[TMP18:%.*]] = load i32, i32* [[TMP17]], align 8
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "align"(%struct.S** [[TMP4]], i64 8), "dereferenceable"(%struct.S** [[TMP4]], i64 8), "nonnull"(%struct.S** [[TMP4]]) ]
+; ALL-NEXT:    [[TMP19:%.*]] = load %struct.S*, %struct.S** [[TMP4]], align 8
+; ALL-NEXT:    [[TMP20:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.S* [[TMP19]], i32 0, i32 1
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "align"(i8* [[TMP20]], i64 4), "dereferenceable"(i8* [[TMP20]], i64 1), "nonnull"(i8* [[TMP20]]) ]
+; ALL-NEXT:    [[TMP21:%.*]] = load i8, i8* [[TMP20]], align 4
+; ALL-NEXT:    [[TMP22:%.*]] = sext i8 [[TMP21]] to i32
+; ALL-NEXT:    [[TMP23:%.*]] = add nsw i32 [[TMP18]], [[TMP22]]
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "align"(%struct.S** [[TMP4]], i64 8), "dereferenceable"(%struct.S** [[TMP4]], i64 8), "nonnull"(%struct.S** [[TMP4]]) ]
+; ALL-NEXT:    [[TMP24:%.*]] = load %struct.S*, %struct.S** [[TMP4]], align 8
+; ALL-NEXT:    [[TMP25:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.S* [[TMP24]], i32 0, i32 2
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32** [[TMP25]], i64 8), "dereferenceable"(i32** [[TMP25]], i64 8), "nonnull"(i32** [[TMP25]]) ]
+; ALL-NEXT:    [[TMP26:%.*]] = load i32*, i32** [[TMP25]], align 8
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[TMP26]], i64 4), "dereferenceable"(i32* [[TMP26]], i64 4), "nonnull"(i32* [[TMP26]]) ]
+; ALL-NEXT:    [[TMP27:%.*]] = load i32, i32* [[TMP26]], align 4
+; ALL-NEXT:    [[TMP28:%.*]] = add nsw i32 [[TMP23]], [[TMP27]]
+; ALL-NEXT:    ret i32 [[TMP28]]
+;
+  %4 = alloca %struct.S*, align 8
+  %5 = alloca i32*, align 8
+  %6 = alloca i8*, align 8
+  %7 = alloca %struct.S, align 8
+  store %struct.S* %0, %struct.S** %4, align 8
+  store i32* %1, i32** %5, align 8
+  store i8* %2, i8** %6
+  %8 = load i32*, i32** %5, align 8
+  %9 = load i32, i32* %8, align 4
+  %10 = trunc i32 %9 to i8
+  %11 = load i8*, i8** %6, align 8
+  store i8 %10, i8* %11, align 1
+  %12 = bitcast %struct.S* %7 to i8*
+  %13 = load %struct.S*, %struct.S** %4
+  %14 = bitcast %struct.S* %13 to i8*
+  %15 = bitcast %struct.S* %7 to i8*
+  %16 = load %struct.S*, %struct.S** %4, align 8
+  %17 = getelementptr inbounds %struct.S, %struct.S* %16, i32 0, i32 0
+  %18 = load i32, i32* %17, align 8
+  %19 = load %struct.S*, %struct.S** %4, align 8
+  %20 = getelementptr inbounds %struct.S, %struct.S* %19, i32 0, i32 1
+  %21 = load i8, i8* %20, align 4
+  %22 = sext i8 %21 to i32
+  %23 = add nsw i32 %18, %22
+  %24 = load %struct.S*, %struct.S** %4, align 8
+  %25 = getelementptr inbounds %struct.S, %struct.S* %24, i32 0, i32 2
+  %26 = load i32*, i32** %25, align 8
+  %27 = load i32, i32* %26, align 4
+  %28 = add nsw i32 %23, %27
+  ret i32 %28
+}
+
+define i32 @test3(%struct.S* %0, i32* %1, i8* %2) "null-pointer-is-valid"="true" {
+; BASIC-LABEL: @test3(
+; BASIC-NEXT:    [[TMP4:%.*]] = alloca %struct.S*, align 8
+; BASIC-NEXT:    [[TMP5:%.*]] = alloca i32*, align 8
+; BASIC-NEXT:    [[TMP6:%.*]] = alloca i8*, align 8
+; BASIC-NEXT:    [[TMP7:%.*]] = alloca [[STRUCT_S:%.*]], align 8
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "align"(%struct.S** [[TMP4]], i64 8), "dereferenceable"(%struct.S** [[TMP4]], i64 8) ]
+; BASIC-NEXT:    store %struct.S* [[TMP0:%.*]], %struct.S** [[TMP4]], align 8
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32** [[TMP5]], i64 8), "dereferenceable"(i32** [[TMP5]], i64 8) ]
+; BASIC-NEXT:    store i32* [[TMP1:%.*]], i32** [[TMP5]], align 8
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "align"(i8** [[TMP6]], i64 8), "dereferenceable"(i8** [[TMP6]], i64 8) ]
+; BASIC-NEXT:    store i8* [[TMP2:%.*]], i8** [[TMP6]], align 8
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32** [[TMP5]], i64 8), "dereferenceable"(i32** [[TMP5]], i64 8) ]
+; BASIC-NEXT:    [[TMP8:%.*]] = load i32*, i32** [[TMP5]], align 8
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[TMP8]], i64 4), "dereferenceable"(i32* [[TMP8]], i64 4) ]
+; BASIC-NEXT:    [[TMP9:%.*]] = load i32, i32* [[TMP8]], align 4
+; BASIC-NEXT:    [[TMP10:%.*]] = trunc i32 [[TMP9]] to i8
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i8** [[TMP6]], i64 8) ]
+; BASIC-NEXT:    [[TMP11:%.*]] = load i8*, i8** [[TMP6]]
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i8* [[TMP11]], i64 1) ]
+; BASIC-NEXT:    store i8 [[TMP10]], i8* [[TMP11]], align 1
+; BASIC-NEXT:    [[TMP12:%.*]] = bitcast %struct.S* [[TMP7]] to i8*
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "align"(%struct.S** [[TMP4]], i64 8), "dereferenceable"(%struct.S** [[TMP4]], i64 8) ]
+; BASIC-NEXT:    [[TMP13:%.*]] = load %struct.S*, %struct.S** [[TMP4]], align 8
+; BASIC-NEXT:    [[TMP14:%.*]] = bitcast %struct.S* [[TMP13]] to i8*
+; BASIC-NEXT:    [[TMP15:%.*]] = bitcast %struct.S* [[TMP7]] to i8*
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "align"(%struct.S** [[TMP4]], i64 8), "dereferenceable"(%struct.S** [[TMP4]], i64 8) ]
+; BASIC-NEXT:    [[TMP16:%.*]] = load %struct.S*, %struct.S** [[TMP4]], align 8
+; BASIC-NEXT:    [[TMP17:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.S* [[TMP16]], i32 0, i32 0
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[TMP17]], i64 8), "dereferenceable"(i32* [[TMP17]], i64 4) ]
+; BASIC-NEXT:    [[TMP18:%.*]] = load i32, i32* [[TMP17]], align 8
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "align"(%struct.S** [[TMP4]], i64 8), "dereferenceable"(%struct.S** [[TMP4]], i64 8) ]
+; BASIC-NEXT:    [[TMP19:%.*]] = load %struct.S*, %struct.S** [[TMP4]], align 8
+; BASIC-NEXT:    [[TMP20:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.S* [[TMP19]], i32 0, i32 1
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "align"(i8* [[TMP20]], i64 4), "dereferenceable"(i8* [[TMP20]], i64 1) ]
+; BASIC-NEXT:    [[TMP21:%.*]] = load i8, i8* [[TMP20]], align 4
+; BASIC-NEXT:    [[TMP22:%.*]] = sext i8 [[TMP21]] to i32
+; BASIC-NEXT:    [[TMP23:%.*]] = add nsw i32 [[TMP18]], [[TMP22]]
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "align"(%struct.S** [[TMP4]], i64 8), "dereferenceable"(%struct.S** [[TMP4]], i64 8) ]
+; BASIC-NEXT:    [[TMP24:%.*]] = load %struct.S*, %struct.S** [[TMP4]], align 8
+; BASIC-NEXT:    [[TMP25:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.S* [[TMP24]], i32 0, i32 2
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32** [[TMP25]], i64 8) ]
+; BASIC-NEXT:    [[TMP26:%.*]] = load i32*, i32** [[TMP25]]
+; BASIC-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[TMP26]], i64 4), "dereferenceable"(i32* [[TMP26]], i64 4) ]
+; BASIC-NEXT:    [[TMP27:%.*]] = load i32, i32* [[TMP26]], align 4
+; BASIC-NEXT:    [[TMP28:%.*]] = add nsw i32 [[TMP23]], [[TMP27]]
+; BASIC-NEXT:    ret i32 [[TMP28]]
+;
+; ALL-LABEL: @test3(
+; ALL-NEXT:    [[TMP4:%.*]] = alloca %struct.S*, align 8
+; ALL-NEXT:    [[TMP5:%.*]] = alloca i32*, align 8
+; ALL-NEXT:    [[TMP6:%.*]] = alloca i8*, align 8
+; ALL-NEXT:    [[TMP7:%.*]] = alloca [[STRUCT_S:%.*]], align 8
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "align"(%struct.S** [[TMP4]], i64 8), "dereferenceable"(%struct.S** [[TMP4]], i64 8) ]
+; ALL-NEXT:    store %struct.S* [[TMP0:%.*]], %struct.S** [[TMP4]], align 8
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32** [[TMP5]], i64 8), "dereferenceable"(i32** [[TMP5]], i64 8) ]
+; ALL-NEXT:    store i32* [[TMP1:%.*]], i32** [[TMP5]], align 8
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "align"(i8** [[TMP6]], i64 8), "dereferenceable"(i8** [[TMP6]], i64 8) ]
+; ALL-NEXT:    store i8* [[TMP2:%.*]], i8** [[TMP6]], align 8
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32** [[TMP5]], i64 8), "dereferenceable"(i32** [[TMP5]], i64 8) ]
+; ALL-NEXT:    [[TMP8:%.*]] = load i32*, i32** [[TMP5]], align 8
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[TMP8]], i64 4), "dereferenceable"(i32* [[TMP8]], i64 4) ]
+; ALL-NEXT:    [[TMP9:%.*]] = load i32, i32* [[TMP8]], align 4
+; ALL-NEXT:    [[TMP10:%.*]] = trunc i32 [[TMP9]] to i8
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i8** [[TMP6]], i64 8) ]
+; ALL-NEXT:    [[TMP11:%.*]] = load i8*, i8** [[TMP6]]
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i8* [[TMP11]], i64 1) ]
+; ALL-NEXT:    store i8 [[TMP10]], i8* [[TMP11]], align 1
+; ALL-NEXT:    [[TMP12:%.*]] = bitcast %struct.S* [[TMP7]] to i8*
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "align"(%struct.S** [[TMP4]], i64 8), "dereferenceable"(%struct.S** [[TMP4]], i64 8) ]
+; ALL-NEXT:    [[TMP13:%.*]] = load %struct.S*, %struct.S** [[TMP4]], align 8
+; ALL-NEXT:    [[TMP14:%.*]] = bitcast %struct.S* [[TMP13]] to i8*
+; ALL-NEXT:    [[TMP15:%.*]] = bitcast %struct.S* [[TMP7]] to i8*
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "align"(%struct.S** [[TMP4]], i64 8), "dereferenceable"(%struct.S** [[TMP4]], i64 8) ]
+; ALL-NEXT:    [[TMP16:%.*]] = load %struct.S*, %struct.S** [[TMP4]], align 8
+; ALL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.S* [[TMP16]], i32 0, i32 0
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[TMP17]], i64 8), "dereferenceable"(i32* [[TMP17]], i64 4) ]
+; ALL-NEXT:    [[TMP18:%.*]] = load i32, i32* [[TMP17]], align 8
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "align"(%struct.S** [[TMP4]], i64 8), "dereferenceable"(%struct.S** [[TMP4]], i64 8) ]
+; ALL-NEXT:    [[TMP19:%.*]] = load %struct.S*, %struct.S** [[TMP4]], align 8
+; ALL-NEXT:    [[TMP20:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.S* [[TMP19]], i32 0, i32 1
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "align"(i8* [[TMP20]], i64 4), "dereferenceable"(i8* [[TMP20]], i64 1) ]
+; ALL-NEXT:    [[TMP21:%.*]] = load i8, i8* [[TMP20]], align 4
+; ALL-NEXT:    [[TMP22:%.*]] = sext i8 [[TMP21]] to i32
+; ALL-NEXT:    [[TMP23:%.*]] = add nsw i32 [[TMP18]], [[TMP22]]
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "align"(%struct.S** [[TMP4]], i64 8), "dereferenceable"(%struct.S** [[TMP4]], i64 8) ]
+; ALL-NEXT:    [[TMP24:%.*]] = load %struct.S*, %struct.S** [[TMP4]], align 8
+; ALL-NEXT:    [[TMP25:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.S* [[TMP24]], i32 0, i32 2
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(i32** [[TMP25]], i64 8) ]
+; ALL-NEXT:    [[TMP26:%.*]] = load i32*, i32** [[TMP25]]
+; ALL-NEXT:    call void @llvm.assume(i1 true) [ "align"(i32* [[TMP26]], i64 4), "dereferenceable"(i32* [[TMP26]], i64 4) ]
+; ALL-NEXT:    [[TMP27:%.*]] = load i32, i32* [[TMP26]], align 4
+; ALL-NEXT:    [[TMP28:%.*]] = add nsw i32 [[TMP23]], [[TMP27]]
+; ALL-NEXT:    ret i32 [[TMP28]]
+;
+  %4 = alloca %struct.S*, align 8
+  %5 = alloca i32*, align 8
+  %6 = alloca i8*, align 8
+  %7 = alloca %struct.S, align 8
+  store %struct.S* %0, %struct.S** %4, align 8
+  store i32* %1, i32** %5, align 8
+  store i8* %2, i8** %6, align 8
+  %8 = load i32*, i32** %5, align 8
+  %9 = load i32, i32* %8, align 4
+  %10 = trunc i32 %9 to i8
+  %11 = load i8*, i8** %6
+  store i8 %10, i8* %11, align 1
+  %12 = bitcast %struct.S* %7 to i8*
+  %13 = load %struct.S*, %struct.S** %4, align 8
+  %14 = bitcast %struct.S* %13 to i8*
+  %15 = bitcast %struct.S* %7 to i8*
+  %16 = load %struct.S*, %struct.S** %4, align 8
+  %17 = getelementptr inbounds %struct.S, %struct.S* %16, i32 0, i32 0
+  %18 = load i32, i32* %17, align 8
+  %19 = load %struct.S*, %struct.S** %4, align 8
+  %20 = getelementptr inbounds %struct.S, %struct.S* %19, i32 0, i32 1
+  %21 = load i8, i8* %20, align 4
+  %22 = sext i8 %21 to i32
+  %23 = add nsw i32 %18, %22
+  %24 = load %struct.S*, %struct.S** %4, align 8
+  %25 = getelementptr inbounds %struct.S, %struct.S* %24, i32 0, i32 2
+  %26 = load i32*, i32** %25
+  %27 = load i32, i32* %26, align 4
+  %28 = add nsw i32 %23, %27
+  ret i32 %28
+}
diff --git a/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-labels.ll b/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-labels.ll
new file mode 100644
index 0000000000000..ed48add3b115a
--- /dev/null
+++ b/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-labels.ll
@@ -0,0 +1,28 @@
+; RUN: opt -S -strip-nonlinetable-debuginfo %s -o - | FileCheck %s
+; CHECK: define void @f()
+define void @f() !dbg !4 {
+entry:
+; CHECK-NOT: llvm.dbg.label
+  call void @llvm.dbg.label(metadata !12), !dbg !11
+  ret void, !dbg !11
+}
+
+declare void @llvm.dbg.label(metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "LLVM", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2)
+!1 = !DIFile(filename: "f.c", directory: "/")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !5, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: false, unit: !0, retainedNodes: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{null}
+!7 = !{i32 2, !"Dwarf Version", i32 2}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"PIC Level", i32 2}
+!10 = !{!"LLVM"}
+!11 = !DILocation(line: 1, column: 24, scope: !4)
+; CHECK-NOT: DILabel
+!12 = !DILabel(scope: !4, name: "entry", file: !1, line: 1)
diff --git a/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-localvars.ll b/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-localvars.ll
index c2cb87acb2a0b..c2642c312ccdc 100644
--- a/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-localvars.ll
+++ b/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-localvars.ll
@@ -3,14 +3,18 @@
 define void @f() !dbg !4 {
 entry:
   %i = alloca i32, align 4
-  ; CHECK-NOT: llvm.dbg.declare
+  ; CHECK-NOT: llvm.dbg.{{addr|declare|value}}
   call void @llvm.dbg.declare(metadata i32* %i, metadata !11, metadata !13), !dbg !14
+  call void @llvm.dbg.addr(metadata i32* %i, metadata !16, metadata !13), !dbg !14
   store i32 42, i32* %i, align 4, !dbg !14
+  call void @llvm.dbg.value(metadata i32 0, metadata !16, metadata !13), !dbg !15
   ret void, !dbg !15
 }
 
 ; Function Attrs: nounwind readnone
+declare void @llvm.dbg.addr(metadata, metadata, metadata)
 declare void @llvm.dbg.declare(metadata, metadata, metadata)
+declare void @llvm.dbg.value(metadata, metadata, metadata)
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!7, !8, !9}
@@ -22,6 +26,7 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata)
 ; CHECK: ![[F]] = distinct !DISubprogram(name: "f"
 ; CHECK-NOT: retainedNodes:
 ; CHECK-NOT: distinct !DISubprogram(name: "f"
+; CHECK-NOT: DILocalVariable
 !4 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !5, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: false, unit: !0, retainedNodes: !2)
 !5 = !DISubroutineType(types: !6)
 !6 = !{null}
@@ -34,3 +39,4 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata)
 !13 = !DIExpression()
 !14 = !DILocation(line: 1, column: 16, scope: !4)
 !15 = !DILocation(line: 1, column: 24, scope: !4)
+!16 = !DILocalVariable(name: "j", scope: !4, file: !1, line: 1, type: !12)
diff --git a/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll b/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll
index cc3d35ff9d71a..bffd6aba570a8 100644
--- a/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll
@@ -418,9 +418,9 @@ define float @ext14_ext15_fmul_v16f32(<16 x float> %x) {
 
 define <4 x float> @ins_bo_ext_ext(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: @ins_bo_ext_ext(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 3, i32 undef>
-; CHECK-NEXT:    [[TMP2:%.*]] = fadd <4 x float> [[A]], [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 2
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 undef, i32 2>
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd <4 x float> [[TMP1]], [[A]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i64 3
 ; CHECK-NEXT:    [[V3:%.*]] = insertelement <4 x float> [[B:%.*]], float [[TMP3]], i32 3
 ; CHECK-NEXT:    ret <4 x float> [[V3]]
 ;
@@ -431,6 +431,9 @@ define <4 x float> @ins_bo_ext_ext(<4 x float> %a, <4 x float> %b) {
   ret <4 x float> %v3
 }
 
+; TODO: This is conservatively left to extract from the lower index value,
+;       but it is likely that extracting from index 3 is the better option.
+
 define <4 x float> @ins_bo_ext_ext_uses(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: @ins_bo_ext_ext_uses(
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 3, i32 undef>
@@ -452,13 +455,13 @@ define <4 x float> @PR34724(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: @PR34724(
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 3, i32 undef>
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[B:%.*]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[B]], <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 3, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[B]], <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 undef, i32 2>
 ; CHECK-NEXT:    [[TMP4:%.*]] = fadd <4 x float> [[A]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP4]], i32 2
 ; CHECK-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[B]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[TMP6]], i32 0
-; CHECK-NEXT:    [[TMP8:%.*]] = fadd <4 x float> [[B]], [[TMP3]]
-; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x float> [[TMP8]], i32 2
+; CHECK-NEXT:    [[TMP8:%.*]] = fadd <4 x float> [[TMP3]], [[B]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x float> [[TMP8]], i64 3
 ; CHECK-NEXT:    [[V1:%.*]] = insertelement <4 x float> undef, float [[TMP5]], i32 1
 ; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x float> [[V1]], float [[TMP7]], i32 2
 ; CHECK-NEXT:    [[V3:%.*]] = insertelement <4 x float> [[V2]], float [[TMP9]], i32 3
diff --git a/llvm/test/Transforms/VectorCombine/X86/extract-cmp.ll b/llvm/test/Transforms/VectorCombine/X86/extract-cmp.ll
index 807bb800e9976..6f6f6d0f9674a 100644
--- a/llvm/test/Transforms/VectorCombine/X86/extract-cmp.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/extract-cmp.ll
@@ -161,9 +161,9 @@ define <4 x i1> @ins_fcmp_ext_ext(<4 x float> %a, <4 x i1> %b) {
 ; SSE-NEXT:    ret <4 x i1> [[R]]
 ;
 ; AVX-LABEL: @ins_fcmp_ext_ext(
-; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> undef, <4 x i32> <i32 undef, i32 2, i32 undef, i32 undef>
-; AVX-NEXT:    [[TMP2:%.*]] = fcmp ugt <4 x float> [[TMP1]], [[A]]
-; AVX-NEXT:    [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 1, i32 undef>
+; AVX-NEXT:    [[TMP2:%.*]] = fcmp ugt <4 x float> [[A]], [[TMP1]]
+; AVX-NEXT:    [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i32 2
 ; AVX-NEXT:    [[R:%.*]] = insertelement <4 x i1> [[B:%.*]], i1 [[TMP3]], i32 2
 ; AVX-NEXT:    ret <4 x i1> [[R]]
 ;
@@ -176,9 +176,9 @@ define <4 x i1> @ins_fcmp_ext_ext(<4 x float> %a, <4 x i1> %b) {
 
 define <4 x i1> @ins_icmp_ext_ext(<4 x i32> %a, <4 x i1> %b) {
 ; CHECK-LABEL: @ins_icmp_ext_ext(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> undef, <4 x i32> <i32 undef, i32 undef, i32 3, i32 undef>
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp ule <4 x i32> [[A]], [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i32 2
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> undef, <4 x i32> <i32 undef, i32 undef, i32 undef, i32 2>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ule <4 x i32> [[TMP1]], [[A]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i64 3
 ; CHECK-NEXT:    [[R:%.*]] = insertelement <4 x i1> [[B:%.*]], i1 [[TMP3]], i32 3
 ; CHECK-NEXT:    ret <4 x i1> [[R]]
 ;
diff --git a/llvm/test/Transforms/VectorCombine/X86/shuffle.ll b/llvm/test/Transforms/VectorCombine/X86/shuffle.ll
index 78d628615766f..5a758d78d9375 100644
--- a/llvm/test/Transforms/VectorCombine/X86/shuffle.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/shuffle.ll
@@ -2,28 +2,65 @@
 ; RUN: opt < %s -vector-combine -S -mtriple=x86_64-- -mattr=SSE2 | FileCheck %s --check-prefixes=CHECK,SSE
 ; RUN: opt < %s -vector-combine -S -mtriple=x86_64-- -mattr=AVX2 | FileCheck %s --check-prefixes=CHECK,AVX
 
+; x86 does not have a cheap v16i8 shuffle until SSSE3 (pshufb)
+
 define <16 x i8> @bitcast_shuf_narrow_element(<4 x i32> %v) {
-; CHECK-LABEL: @bitcast_shuf_narrow_element(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[V:%.*]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    [[R:%.*]] = bitcast <4 x i32> [[SHUF]] to <16 x i8>
-; CHECK-NEXT:    ret <16 x i8> [[R]]
+; SSE-LABEL: @bitcast_shuf_narrow_element(
+; SSE-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[V:%.*]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; SSE-NEXT:    [[R:%.*]] = bitcast <4 x i32> [[SHUF]] to <16 x i8>
+; SSE-NEXT:    ret <16 x i8> [[R]]
+;
+; AVX-LABEL: @bitcast_shuf_narrow_element(
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[V:%.*]] to <16 x i8>
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> undef, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+; AVX-NEXT:    ret <16 x i8> [[TMP2]]
 ;
   %shuf = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   %r = bitcast <4 x i32> %shuf to <16 x i8>
   ret <16 x i8> %r
 }
 
+; v4f32 is the same cost as v4i32, so this always works
+
 define <4 x float> @bitcast_shuf_same_size(<4 x i32> %v) {
 ; CHECK-LABEL: @bitcast_shuf_same_size(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[V:%.*]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    [[R:%.*]] = bitcast <4 x i32> [[SHUF]] to <4 x float>
-; CHECK-NEXT:    ret <4 x float> [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[V:%.*]] to <4 x float>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <4 x float> [[TMP2]]
 ;
   %shuf = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   %r = bitcast <4 x i32> %shuf to <4 x float>
   ret <4 x float> %r
 }
 
+; Negative test - length-changing shuffle
+
+define <16 x i8> @bitcast_shuf_narrow_element_wrong_size(<2 x i32> %v) {
+; CHECK-LABEL: @bitcast_shuf_narrow_element_wrong_size(
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i32> [[V:%.*]], <2 x i32> undef, <4 x i32> <i32 1, i32 0, i32 1, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = bitcast <4 x i32> [[SHUF]] to <16 x i8>
+; CHECK-NEXT:    ret <16 x i8> [[R]]
+;
+  %shuf = shufflevector <2 x i32> %v, <2 x i32> undef, <4 x i32> <i32 1, i32 0, i32 1, i32 0>
+  %r = bitcast <4 x i32> %shuf to <16 x i8>
+  ret <16 x i8> %r
+}
+
+; Negative test - must cast to vector type
+
+define i128 @bitcast_shuf_narrow_element_wrong_type(<4 x i32> %v) {
+; CHECK-LABEL: @bitcast_shuf_narrow_element_wrong_type(
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[V:%.*]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = bitcast <4 x i32> [[SHUF]] to i128
+; CHECK-NEXT:    ret i128 [[R]]
+;
+  %shuf = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %r = bitcast <4 x i32> %shuf to i128
+  ret i128 %r
+}
+
+; Negative test - but might want to try this
+
 define <4 x i32> @bitcast_shuf_wide_element(<8 x i16> %v) {
 ; CHECK-LABEL: @bitcast_shuf_wide_element(
 ; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <8 x i16> [[V:%.*]], <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 2, i32 3, i32 2, i32 3>
@@ -37,6 +74,8 @@ define <4 x i32> @bitcast_shuf_wide_element(<8 x i16> %v) {
 
 declare void @use(<4 x i32>)
 
+; Negative test - don't create an extra shuffle
+
 define <16 x i8> @bitcast_shuf_uses(<4 x i32> %v) {
 ; CHECK-LABEL: @bitcast_shuf_uses(
 ; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[V:%.*]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
@@ -51,15 +90,25 @@ define <16 x i8> @bitcast_shuf_uses(<4 x i32> %v) {
 }
 
 define <2 x i64> @PR35454_1(<2 x i64> %v) {
-; CHECK-LABEL: @PR35454_1(
-; CHECK-NEXT:    [[BC:%.*]] = bitcast <2 x i64> [[V:%.*]] to <4 x i32>
-; CHECK-NEXT:    [[PERMIL:%.*]] = shufflevector <4 x i32> [[BC]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    [[BC1:%.*]] = bitcast <4 x i32> [[PERMIL]] to <16 x i8>
-; CHECK-NEXT:    [[ADD:%.*]] = shl <16 x i8> [[BC1]], <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
-; CHECK-NEXT:    [[BC2:%.*]] = bitcast <16 x i8> [[ADD]] to <4 x i32>
-; CHECK-NEXT:    [[PERMIL1:%.*]] = shufflevector <4 x i32> [[BC2]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    [[BC3:%.*]] = bitcast <4 x i32> [[PERMIL1]] to <2 x i64>
-; CHECK-NEXT:    ret <2 x i64> [[BC3]]
+; SSE-LABEL: @PR35454_1(
+; SSE-NEXT:    [[BC:%.*]] = bitcast <2 x i64> [[V:%.*]] to <4 x i32>
+; SSE-NEXT:    [[PERMIL:%.*]] = shufflevector <4 x i32> [[BC]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; SSE-NEXT:    [[BC1:%.*]] = bitcast <4 x i32> [[PERMIL]] to <16 x i8>
+; SSE-NEXT:    [[ADD:%.*]] = shl <16 x i8> [[BC1]], <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+; SSE-NEXT:    [[BC2:%.*]] = bitcast <16 x i8> [[ADD]] to <4 x i32>
+; SSE-NEXT:    [[PERMIL1:%.*]] = shufflevector <4 x i32> [[BC2]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; SSE-NEXT:    [[BC3:%.*]] = bitcast <4 x i32> [[PERMIL1]] to <2 x i64>
+; SSE-NEXT:    ret <2 x i64> [[BC3]]
+;
+; AVX-LABEL: @PR35454_1(
+; AVX-NEXT:    [[BC:%.*]] = bitcast <2 x i64> [[V:%.*]] to <4 x i32>
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[BC]] to <16 x i8>
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> undef, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+; AVX-NEXT:    [[ADD:%.*]] = shl <16 x i8> [[TMP2]], <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+; AVX-NEXT:    [[BC2:%.*]] = bitcast <16 x i8> [[ADD]] to <4 x i32>
+; AVX-NEXT:    [[PERMIL1:%.*]] = shufflevector <4 x i32> [[BC2]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; AVX-NEXT:    [[BC3:%.*]] = bitcast <4 x i32> [[PERMIL1]] to <2 x i64>
+; AVX-NEXT:    ret <2 x i64> [[BC3]]
 ;
   %bc = bitcast <2 x i64> %v to <4 x i32>
   %permil = shufflevector <4 x i32> %bc, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
@@ -72,15 +121,25 @@ define <2 x i64> @PR35454_1(<2 x i64> %v) {
 }
 
 define <2 x i64> @PR35454_2(<2 x i64> %v) {
-; CHECK-LABEL: @PR35454_2(
-; CHECK-NEXT:    [[BC:%.*]] = bitcast <2 x i64> [[V:%.*]] to <4 x i32>
-; CHECK-NEXT:    [[PERMIL:%.*]] = shufflevector <4 x i32> [[BC]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    [[BC1:%.*]] = bitcast <4 x i32> [[PERMIL]] to <8 x i16>
-; CHECK-NEXT:    [[ADD:%.*]] = shl <8 x i16> [[BC1]], <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
-; CHECK-NEXT:    [[BC2:%.*]] = bitcast <8 x i16> [[ADD]] to <4 x i32>
-; CHECK-NEXT:    [[PERMIL1:%.*]] = shufflevector <4 x i32> [[BC2]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    [[BC3:%.*]] = bitcast <4 x i32> [[PERMIL1]] to <2 x i64>
-; CHECK-NEXT:    ret <2 x i64> [[BC3]]
+; SSE-LABEL: @PR35454_2(
+; SSE-NEXT:    [[BC:%.*]] = bitcast <2 x i64> [[V:%.*]] to <4 x i32>
+; SSE-NEXT:    [[PERMIL:%.*]] = shufflevector <4 x i32> [[BC]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; SSE-NEXT:    [[BC1:%.*]] = bitcast <4 x i32> [[PERMIL]] to <8 x i16>
+; SSE-NEXT:    [[ADD:%.*]] = shl <8 x i16> [[BC1]], <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+; SSE-NEXT:    [[BC2:%.*]] = bitcast <8 x i16> [[ADD]] to <4 x i32>
+; SSE-NEXT:    [[PERMIL1:%.*]] = shufflevector <4 x i32> [[BC2]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; SSE-NEXT:    [[BC3:%.*]] = bitcast <4 x i32> [[PERMIL1]] to <2 x i64>
+; SSE-NEXT:    ret <2 x i64> [[BC3]]
+;
+; AVX-LABEL: @PR35454_2(
+; AVX-NEXT:    [[BC:%.*]] = bitcast <2 x i64> [[V:%.*]] to <4 x i32>
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[BC]] to <8 x i16>
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <8 x i32> <i32 6, i32 7, i32 4, i32 5, i32 2, i32 3, i32 0, i32 1>
+; AVX-NEXT:    [[ADD:%.*]] = shl <8 x i16> [[TMP2]], <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+; AVX-NEXT:    [[BC2:%.*]] = bitcast <8 x i16> [[ADD]] to <4 x i32>
+; AVX-NEXT:    [[PERMIL1:%.*]] = shufflevector <4 x i32> [[BC2]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; AVX-NEXT:    [[BC3:%.*]] = bitcast <4 x i32> [[PERMIL1]] to <2 x i64>
+; AVX-NEXT:    ret <2 x i64> [[BC3]]
 ;
   %bc = bitcast <2 x i64> %v to <4 x i32>
   %permil = shufflevector <4 x i32> %bc, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
diff --git a/llvm/test/Unit/lit.site.cfg.py.in b/llvm/test/Unit/lit.site.cfg.py.in
index 1fef001be6271..3358ccf89eed4 100644
--- a/llvm/test/Unit/lit.site.cfg.py.in
+++ b/llvm/test/Unit/lit.site.cfg.py.in
@@ -2,12 +2,12 @@
 
 import sys
 
-config.llvm_src_root = "@LLVM_SOURCE_DIR@"
-config.llvm_obj_root = "@LLVM_BINARY_DIR@"
-config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
+config.llvm_src_root = path("@LLVM_SOURCE_DIR@")
+config.llvm_obj_root = path("@LLVM_BINARY_DIR@")
+config.llvm_tools_dir = path("@LLVM_TOOLS_DIR@")
 config.llvm_build_mode = "@LLVM_BUILD_MODE@"
 config.enable_shared = @ENABLE_SHARED@
-config.shlibdir = "@SHLIBDIR@"
+config.shlibdir = path("@SHLIBDIR@")
 
 # Support substitution of the tools_dir and build_mode with user parameters.
 # This is used when we can't determine the tool dir at configuration time.
@@ -20,4 +20,5 @@ except KeyError:
     lit_config.fatal("unable to find %r parameter, use '--param=%s=VALUE'" % (key,key))
 
 # Let the main config do the real work.
-lit_config.load_config(config, "@LLVM_SOURCE_DIR@/test/Unit/lit.cfg.py")
+lit_config.load_config(
+    config, os.path.join(config.llvm_src_root, "test/Unit/lit.cfg.py"))
diff --git a/llvm/test/Verifier/matrix-intrinsics.ll b/llvm/test/Verifier/matrix-intrinsics.ll
new file mode 100644
index 0000000000000..d2f23ab7894ed
--- /dev/null
+++ b/llvm/test/Verifier/matrix-intrinsics.ll
@@ -0,0 +1,40 @@
+; RUN: not llvm-as < %s -o /dev/null 2>&1 | FileCheck %s
+
+declare <4 x float> @llvm.matrix.transpose.v4f32(<4 x float>, i32, i32)
+define <4 x float> @transpose(<4 x float> %m) {
+; CHECK: assembly parsed, but does not verify as correct!
+; CHECK-NEXT: result of a matrix operation does not fit in the returned vector
+; CHECK-NEXT: result of a matrix operation does not fit in the returned vector
+  %result.1 = call <4 x float> @llvm.matrix.transpose.v4f32(<4 x float> %m, i32 3, i32 2)
+  %result.2 = call <4 x float> @llvm.matrix.transpose.v4f32(<4 x float> %result.1, i32 2, i32 1)
+  ret <4 x float> %result.2
+}
+
+declare <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float>, <4 x float>, i32, i32, i32)
+define <4 x float> @multiply(<4 x float> %m) {
+; CHECK-NEXT: result of a matrix operation does not fit in the returned vector
+; CHECK-NEXT: result of a matrix operation does not fit in the returned vector
+  %result.1 = call <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float> %m, <4 x float> %m, i32 3, i32 2, i32 2)
+  %result.2 = call <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float> %result.1, <4 x float> %m, i32 2, i32 2, i32 1)
+  ret <4 x float> %result.2
+}
+
+declare <4 x float> @llvm.matrix.columnwise.load.v4f32.p0v4f32(<4 x float>*, i32, i32, i32)
+declare <6 x float> @llvm.matrix.columnwise.load.v6f32.p0v6f32(<6 x float>*, i32, i32, i32)
+define <4 x float> @columnwise_load(<4 x float>* %m, <6 x float>* %n) {
+; CHECK-NEXT: result of a matrix operation does not fit in the returned vector
+; CHECK-NEXT: result of a matrix operation does not fit in the returned vector
+  %result.1 = call <4 x float> @llvm.matrix.columnwise.load.v4f32.p0v4f32(<4 x float>* %m, i32 2, i32 1, i32 2)
+  %result.2 = call <6 x float> @llvm.matrix.columnwise.load.v6f32.p0v6f32(<6 x float>* %n, i32 2, i32 3, i32 3)
+  ret <4 x float> %result.1
+}
+
+declare void @llvm.matrix.columnwise.store.v4f32.p0v4f32(<4 x float>, <4 x float>*, i32, i32, i32)
+declare void @llvm.matrix.columnwise.store.v6f32.p0v6f32(<6 x float>, <6 x float>*, i32, i32, i32)
+define void @columnwise_store(<4 x float>* %m, <6 x float>* %n) {
+; CHECK-NEXT: result of a matrix operation does not fit in the returned vector
+; CHECK-NEXT: result of a matrix operation does not fit in the returned vector
+  call void @llvm.matrix.columnwise.store.v4f32.p0v4f32(<4 x float> zeroinitializer, <4 x float>* %m, i32 2, i32 1, i32 2)
+  call void @llvm.matrix.columnwise.store.v6f32.p0v6f32(<6 x float> zeroinitializer, <6 x float>* %n, i32 2, i32 3, i32 3)
+  ret void
+}
diff --git a/llvm/test/lit.site.cfg.py.in b/llvm/test/lit.site.cfg.py.in
index 6f4d5f7908281..2af0ec3360896 100644
--- a/llvm/test/lit.site.cfg.py.in
+++ b/llvm/test/lit.site.cfg.py.in
@@ -4,14 +4,14 @@ import sys
 
 config.host_triple = "@LLVM_HOST_TRIPLE@"
 config.target_triple = "@TARGET_TRIPLE@"
-config.llvm_src_root = "@LLVM_SOURCE_DIR@"
-config.llvm_obj_root = "@LLVM_BINARY_DIR@"
-config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
-config.llvm_lib_dir = "@LLVM_LIBRARY_DIR@"
-config.llvm_shlib_dir = "@SHLIBDIR@"
+config.llvm_src_root = path("@LLVM_SOURCE_DIR@")
+config.llvm_obj_root = path("@LLVM_BINARY_DIR@")
+config.llvm_tools_dir = path("@LLVM_TOOLS_DIR@")
+config.llvm_lib_dir = path("@LLVM_LIBRARY_DIR@")
+config.llvm_shlib_dir = path("@SHLIBDIR@")
 config.llvm_shlib_ext = "@SHLIBEXT@"
 config.llvm_exe_ext = "@EXEEXT@"
-config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@"
+config.lit_tools_dir = path("@LLVM_LIT_TOOLS_DIR@")
 config.python_executable = "@PYTHON_EXECUTABLE@"
 config.gold_executable = "@GOLD_EXECUTABLE@"
 config.ld64_executable = "@LD64_EXECUTABLE@"
@@ -63,4 +63,5 @@ import lit.llvm
 lit.llvm.initialize(lit_config, config)
 
 # Let the main config do the real work.
-lit_config.load_config(config, "@LLVM_SOURCE_DIR@/test/lit.cfg.py")
+lit_config.load_config(
+    config, os.path.join(config.llvm_src_root, "test/lit.cfg.py"))
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/sometimes_deleted_function.ll b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/sometimes_deleted_function.ll
index 6b82c49f018e3..adb1ec9154f72 100644
--- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/sometimes_deleted_function.ll
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/sometimes_deleted_function.ll
@@ -1,9 +1,13 @@
-; RUN: opt -S < %s | FileCheck %s --check-prefixes=ALL,FIRST
-; RUN: opt -S -globalopt < %s | FileCheck %s --check-prefixes=ALL,SECOND
+; RUN: opt -S < %s | FileCheck %s --check-prefixes=ALL,ALL_BUT_TWO,ALL_BUT_THREE,ALL_BUT_FOUR,ONE_AND_TWO,ONE_AND_THREE,ONE_AND_FOUR,ONE
+; RUN: opt -S -globalopt < %s | FileCheck %s --check-prefixes=ALL,ALL_BUT_ONE,ALL_BUT_THREE,ALL_BUT_FOUR,ONE_AND_TWO,TWO_AND_THREE,TWO_AND_FOUR,TWO
+; RUN: opt -S -instsimplify < %s | FileCheck %s --check-prefixes=ALL,ALL_BUT_ONE,ALL_BUT_TWO,ALL_BUT_FOUR,ONE_AND_THREE,TWO_AND_THREE,THREE_AND_FOUR,THREE
+; RUN: opt -S < %s | FileCheck %s --check-prefixes=ALL,ALL_BUT_ONE,ALL_BUT_TWO,ALL_BUT_THREE,ONE_AND_FOUR,TWO_AND_FOUR,THREE_AND_FOUR,FOUR
 ;
-; Make sure we use FIRST to check for @sometimes_here as ALL does not work.
+; Make sure we don't use anything to check for @sometimes_here that contains "ALL" or "TWO".
+; Also verify we use "ONE_AND_FOUR" for the unmodified @sometimes_here version and "THREE" for the version without the add.
 
 define internal void @sometimes_here() {
+  %c = add i32 undef, undef
   ret void
 }
 
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/sometimes_deleted_function.ll.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/sometimes_deleted_function.ll.expected
index f9374257162fb..c42845f8d859d 100644
--- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/sometimes_deleted_function.ll.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/sometimes_deleted_function.ll.expected
@@ -1,13 +1,21 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S < %s | FileCheck %s --check-prefixes=ALL,FIRST
-; RUN: opt -S -globalopt < %s | FileCheck %s --check-prefixes=ALL,SECOND
+; RUN: opt -S < %s | FileCheck %s --check-prefixes=ALL,ALL_BUT_TWO,ALL_BUT_THREE,ALL_BUT_FOUR,ONE_AND_TWO,ONE_AND_THREE,ONE_AND_FOUR,ONE
+; RUN: opt -S -globalopt < %s | FileCheck %s --check-prefixes=ALL,ALL_BUT_ONE,ALL_BUT_THREE,ALL_BUT_FOUR,ONE_AND_TWO,TWO_AND_THREE,TWO_AND_FOUR,TWO
+; RUN: opt -S -instsimplify < %s | FileCheck %s --check-prefixes=ALL,ALL_BUT_ONE,ALL_BUT_TWO,ALL_BUT_FOUR,ONE_AND_THREE,TWO_AND_THREE,THREE_AND_FOUR,THREE
+; RUN: opt -S < %s | FileCheck %s --check-prefixes=ALL,ALL_BUT_ONE,ALL_BUT_TWO,ALL_BUT_THREE,ONE_AND_FOUR,TWO_AND_FOUR,THREE_AND_FOUR,FOUR
 ;
-; Make sure we use FIRST to check for @sometimes_here as ALL does not work.
+; Make sure we don't use anything to check for @sometimes_here that contains "ALL" or "TWO".
+; Also verify we use "ONE_AND_FOUR" for the unmodified @sometimes_here version and "THREE" for the version without the add.
 
 define internal void @sometimes_here() {
-; FIRST-LABEL: @sometimes_here(
-; FIRST-NEXT:    ret void
+; ONE_AND_FOUR-LABEL: @sometimes_here(
+; ONE_AND_FOUR-NEXT:    [[C:%.*]] = add i32 undef, undef
+; ONE_AND_FOUR-NEXT:    ret void
 ;
+; THREE-LABEL: @sometimes_here(
+; THREE-NEXT:    ret void
+;
+  %c = add i32 undef, undef
   ret void
 }
 
diff --git a/llvm/test/tools/llvm-cov/Inputs/binary-formats.canonical.json b/llvm/test/tools/llvm-cov/Inputs/binary-formats.canonical.json
index f8bfcede1fd7e..e6eb0bd3fba01 100644
--- a/llvm/test/tools/llvm-cov/Inputs/binary-formats.canonical.json
+++ b/llvm/test/tools/llvm-cov/Inputs/binary-formats.canonical.json
@@ -4,8 +4,8 @@ CHECK-SAME:  "files":[
 CHECK-SAME:    {"expansions":[],
 CHECK-SAME:     "filename":"/tmp/binary-formats.c",
 CHECK-SAME:     "segments":
-CHECK-SAME:        4,40,100,true,true
-CHECK-SAME:        4,42,0,false,false
+CHECK-SAME:        4,40,100,true,true,false
+CHECK-SAME:        4,42,0,false,false,false
 CHECK-SAME:     "summary":{"functions":{"count":1,"covered":1,"percent":100},
 CHECK-SAME:                "instantiations":{"count":1,"covered":1,"percent":100},
 CHECK-SAME:                "lines":{"count":1,"covered":1,"percent":100},
@@ -24,4 +24,4 @@ CHECK-SAME:     "lines":{"count":1,"covered":1,"percent":100},
 CHECK-SAME:     "regions":{"count":1,"covered":1,"notcovered":0,"percent":100}}}
 CHECK-SAME: ],
 CHECK-SAME: "type":"llvm.coverage.json.export"
-CHECK-SAME: "version":"2.0.0"
+CHECK-SAME: "version":"2.0.1"
diff --git a/llvm/test/tools/llvm-cov/misssing-profdata.test b/llvm/test/tools/llvm-cov/misssing-profdata.test
new file mode 100644
index 0000000000000..44faeac740719
--- /dev/null
+++ b/llvm/test/tools/llvm-cov/misssing-profdata.test
@@ -0,0 +1,2 @@
+RUN: not llvm-cov show -instr-profile=%t.nonexistent %t.nonexistent 2>&1 | FileCheck %s
+CHECK: Could not read profile data
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/Inputs/i386_macho_with_debug.yaml b/llvm/test/tools/llvm-dwarfdump/X86/Inputs/i386_macho_with_debug.yaml
new file mode 100644
index 0000000000000..ad03961b060e0
--- /dev/null
+++ b/llvm/test/tools/llvm-dwarfdump/X86/Inputs/i386_macho_with_debug.yaml
@@ -0,0 +1,88 @@
+--- !mach-o
+FileHeader:
+  magic:           0xFEEDFACE
+  cputype:         0x00000007
+  cpusubtype:      0x00000003
+  filetype:        0x00000001
+  ncmds:           4
+  sizeofcmds:      312
+  flags:           0x00002000
+LoadCommands:
+  - cmd:             LC_SEGMENT
+    cmdsize:         192
+    segname:         ''
+    vmaddr:          0
+    vmsize:          72
+    fileoff:         340
+    filesize:        72
+    maxprot:         7
+    initprot:        7
+    nsects:          2
+    flags:           0
+    Sections:
+      - sectname:        __text
+        segname:         __TEXT
+        addr:            0x0000000000000000
+        size:            18
+        offset:          0x00000154
+        align:           4
+        reloff:          0x00000000
+        nreloc:          0
+        flags:           0x80000400
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+      - sectname:        __debug_info
+        segname:         __DWARF
+        addr:            0x0000000000000014
+        size:            52
+        offset:          0x00000168
+        align:           2
+        reloff:          0x00000000
+        nreloc:          0
+        flags:           0x6800000B
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+  - cmd:             LC_VERSION_MIN_MACOSX
+    cmdsize:         16
+    version:         656384
+    sdk:             0
+  - cmd:             LC_SYMTAB
+    cmdsize:         24
+    symoff:          412
+    nsyms:           1
+    stroff:          424
+    strsize:         8
+  - cmd:             LC_DYSYMTAB
+    cmdsize:         80
+    ilocalsym:       0
+    nlocalsym:       0
+    iextdefsym:      0
+    nextdefsym:      1
+    iundefsym:       1
+    nundefsym:       0
+    tocoff:          0
+    ntoc:            0
+    modtaboff:       0
+    nmodtab:         0
+    extrefsymoff:    0
+    nextrefsyms:     0
+    indirectsymoff:  0
+    nindirectsyms:   0
+    extreloff:       0
+    nextrel:         0
+    locreloff:       0
+    nlocrel:         0
+LinkEditData:
+  NameList:
+    - n_strx:          1
+      n_type:          0x0F
+      n_sect:          1
+      n_desc:          0
+      n_value:         0
+  StringTable:
+    - ''
+    - _main
+    - ''
+...
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/Inputs/x86_64_macho_with_debug.yaml b/llvm/test/tools/llvm-dwarfdump/X86/Inputs/x86_64_macho_with_debug.yaml
new file mode 100644
index 0000000000000..9333794939beb
--- /dev/null
+++ b/llvm/test/tools/llvm-dwarfdump/X86/Inputs/x86_64_macho_with_debug.yaml
@@ -0,0 +1,89 @@
+--- !mach-o
+FileHeader:
+  magic:           0xFEEDFACF
+  cputype:         0x01000007
+  cpusubtype:      0x00000003
+  filetype:        0x00000001
+  ncmds:           4
+  sizeofcmds:      352
+  flags:           0x00002000
+  reserved:        0x00000000
+LoadCommands:
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         232
+    segname:         ''
+    vmaddr:          0
+    vmsize:          80
+    fileoff:         384
+    filesize:        80
+    maxprot:         7
+    initprot:        7
+    nsects:          2
+    flags:           0
+    Sections:
+      - sectname:        __text
+        segname:         __TEXT
+        addr:            0x0000000000000000
+        size:            15
+        offset:          0x00000180
+        align:           4
+        reloff:          0x00000000
+        nreloc:          0
+        flags:           0x80000400
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+      - sectname:        __debug_info
+        segname:         __DWARF
+        addr:            0x0000000000000010
+        size:            64
+        offset:          0x00000190
+        align:           3
+        reloff:          0x00000000
+        nreloc:          0
+        flags:           0x6800000B
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+  - cmd:             LC_VERSION_MIN_MACOSX
+    cmdsize:         16
+    version:         656384
+    sdk:             0
+  - cmd:             LC_SYMTAB
+    cmdsize:         24
+    symoff:          464
+    nsyms:           1
+    stroff:          480
+    strsize:         8
+  - cmd:             LC_DYSYMTAB
+    cmdsize:         80
+    ilocalsym:       0
+    nlocalsym:       0
+    iextdefsym:      0
+    nextdefsym:      1
+    iundefsym:       1
+    nundefsym:       0
+    tocoff:          0
+    ntoc:            0
+    modtaboff:       0
+    nmodtab:         0
+    extrefsymoff:    0
+    nextrefsyms:     0
+    indirectsymoff:  0
+    nindirectsyms:   0
+    extreloff:       0
+    nextrel:         0
+    locreloff:       0
+    nlocrel:         0
+LinkEditData:
+  NameList:
+    - n_strx:          1
+      n_type:          0x0F
+      n_sect:          1
+      n_desc:          0
+      n_value:         0
+  StringTable:
+    - ''
+    - _main
+    - ''
+...
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/section_sizes_archive.test b/llvm/test/tools/llvm-dwarfdump/X86/section_sizes_archive.test
new file mode 100644
index 0000000000000..7280046043a2f
--- /dev/null
+++ b/llvm/test/tools/llvm-dwarfdump/X86/section_sizes_archive.test
@@ -0,0 +1,66 @@
+## Check how llvm-dwarfdump calculates section sizes
+## with --show-section-sizes on an archive.
+
+# RUN: rm -rf %t && mkdir -p %t
+# RUN: yaml2obj --docnum=1 %s -o %t/1.o
+# RUN: yaml2obj --docnum=2 %s -o %t/2.o
+
+# RUN: rm -f %t.a
+# RUN: llvm-ar rc %t.a %t/1.o %t/2.o
+
+# RUN: llvm-dwarfdump --show-section-sizes %t.a \
+# RUN:    | FileCheck %s -DARCHIVE=%t.a --match-full-lines --strict-whitespace
+
+#      CHECK:----------------------------------------------------
+# CHECK-NEXT:file: [[ARCHIVE]](1.o)
+# CHECK-NEXT:----------------------------------------------------
+# CHECK-NEXT:SECTION      SIZE (b)
+# CHECK-NEXT:-----------  --------
+# CHECK-NEXT:.debug_info        17 (3.17%)
+# CHECK-NEXT:.debug_line        19 (3.54%)
+# CHECK-EMPTY:
+# CHECK-NEXT: Total Size: 36  (6.72%)
+# CHECK-NEXT: Total File Size: 536
+# CHECK-NEXT:----------------------------------------------------
+# CHECK-NEXT:----------------------------------------------------
+# CHECK-NEXT:file: [[ARCHIVE]](2.o)
+# CHECK-NEXT:----------------------------------------------------
+# CHECK-NEXT:SECTION      SIZE (b)
+# CHECK-NEXT:-----------  --------
+# CHECK-NEXT:.debug_loc          1 (0.20%)
+# CHECK-NEXT:.debug_line        13 (2.54%)
+# CHECK-EMPTY:
+# CHECK-NEXT: Total Size: 14  (2.73%)
+# CHECK-NEXT: Total File Size: 512
+# CHECK-NEXT:----------------------------------------------------
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_X86_64
+Sections:
+- Name:    .text1
+  Type:    SHT_PROGBITS
+- Name: .debug_info
+  Type: SHT_PROGBITS
+  Size: 17
+- Name: .debug_line
+  Type: SHT_PROGBITS
+  Size: 19
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_X86_64
+Sections:
+- Name:    .text2
+  Type:    SHT_PROGBITS
+- Name: .debug_loc
+  Type: SHT_PROGBITS
+  Size: 1
+- Name: .debug_line
+  Type: SHT_PROGBITS
+  Size: 13
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/section_sizes_coff.test b/llvm/test/tools/llvm-dwarfdump/X86/section_sizes_coff.test
new file mode 100644
index 0000000000000..7d72e02b946de
--- /dev/null
+++ b/llvm/test/tools/llvm-dwarfdump/X86/section_sizes_coff.test
@@ -0,0 +1,43 @@
+## Check how llvm-dwarfdump calculates section sizes
+## with --show-section-sizes for COFF objects.
+
+# RUN: yaml2obj %s | llvm-dwarfdump - --show-section-sizes \
+# RUN:   | FileCheck %s --match-full-lines --strict-whitespace
+
+#      CHECK:----------------------------------------------------
+# CHECK-NEXT:file: {{.*}}
+# CHECK-NEXT:----------------------------------------------------
+# CHECK-NEXT:SECTION        SIZE (b)
+# CHECK-NEXT:-------------  --------
+# CHECK-NEXT:.debug_info           2 (0.70%)
+# CHECK-NEXT:.debug_abbrev         1 (0.35%)
+# CHECK-NEXT:.debug_str            1 (0.35%)
+# CHECK-EMPTY:
+# CHECK-NEXT: Total Size: 4  (1.40%)
+# CHECK-NEXT: Total File Size: 286
+# CHECK-NEXT:----------------------------------------------------
+
+--- !COFF
+header:
+  Machine:         IMAGE_FILE_MACHINE_AMD64
+sections:
+  - Name:            .bss
+    Characteristics: []
+    SectionData:     ''
+  - Name:            .debug_str
+    Characteristics: []
+    SectionData:     00
+  - Name:            .debug_abbrev
+    Characteristics: []
+    Alignment:       1
+    SectionData:     00
+  - Name:            .debug_info
+    Characteristics: []
+    SectionData:     1111
+## This is a debug section following the Mach-O naming style, and is used
+## to show that such sections are not included in the report.
+  - Name:            __debug_foo
+    Characteristics: []
+    SectionData:     00
+symbols:
+...
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/section_sizes_elf.test b/llvm/test/tools/llvm-dwarfdump/X86/section_sizes_elf.test
new file mode 100644
index 0000000000000..ad1763faaa322
--- /dev/null
+++ b/llvm/test/tools/llvm-dwarfdump/X86/section_sizes_elf.test
@@ -0,0 +1,55 @@
+## Check how llvm-dwarfdump calculates section sizes
+## with --show-section-sizes for ELF objects.
+
+# RUN: yaml2obj %s | llvm-dwarfdump - --show-section-sizes \
+# RUN:   | FileCheck %s --match-full-lines --strict-whitespace
+
+#      CHECK:----------------------------------------------------
+# CHECK-NEXT:file: {{.*}}
+# CHECK-NEXT:----------------------------------------------------
+# CHECK-NEXT:SECTION          SIZE (b)
+# CHECK-NEXT:---------------  --------
+# CHECK-NEXT:.debug_info            17 (1.62%)
+# CHECK-NEXT:.debug_loc              1 (0.10%)
+# CHECK-NEXT:.debug_type            26 (2.48%)
+# CHECK-NEXT:.debug_foo            100 (9.54%)
+# CHECK-NEXT:.debug_info.dwo         9 (0.86%)
+# CHECK-NEXT:.debug_line            19 (1.81%)
+# CHECK-EMPTY:
+# CHECK-NEXT: Total Size: 172  (16.41%)
+# CHECK-NEXT: Total File Size: 1048
+# CHECK-NEXT:----------------------------------------------------
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_X86_64
+Sections:
+  - Name: .debug_info
+    Type: SHT_PROGBITS
+    Size: 17
+  - Name: .debug_line
+    Type: SHT_PROGBITS
+    Size: 19
+  - Name: .debug_loc
+    Type: SHT_PROGBITS
+    Size: 1
+  - Name: .debug_type
+    Type: SHT_PROGBITS
+    Size: 13
+  - Name: .debug_type [1]
+    Type: SHT_PROGBITS
+    Size: 13
+  - Name: .debug_foo
+    Type: SHT_PROGBITS
+    Size: 100
+  - Name: .debug_info.dwo
+    Type: SHT_PROGBITS
+    Size: 9
+## This is a debug section following the Mach-O naming style, and is used
+## to show that such sections are not included in the report.
+  - Name: __debug_bar
+    Type: SHT_PROGBITS
+    Size: 1
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/section_sizes_fat_binary.test b/llvm/test/tools/llvm-dwarfdump/X86/section_sizes_fat_binary.test
new file mode 100644
index 0000000000000..d0b8c43a0f4cb
--- /dev/null
+++ b/llvm/test/tools/llvm-dwarfdump/X86/section_sizes_fat_binary.test
@@ -0,0 +1,31 @@
+## Check how llvm-dwarfdump calculates section sizes
+## with --show-section-sizes on a fat binary.
+
+# RUN: yaml2obj %p/Inputs/i386_macho_with_debug.yaml -o %t-i386.o
+# RUN: yaml2obj %p/Inputs/x86_64_macho_with_debug.yaml -o %t-x86_64.o
+
+# RUN: llvm-lipo %t-i386.o %t-x86_64.o -create -output %t.o
+
+# RUN: llvm-dwarfdump --show-section-sizes %t.o \
+# RUN:    | FileCheck %s -DFILE=%t.o --match-full-lines --strict-whitespace
+
+#      CHECK:----------------------------------------------------
+# CHECK-NEXT:file: [[FILE]](i386)
+# CHECK-NEXT:----------------------------------------------------
+# CHECK-NEXT:SECTION       SIZE (b)
+# CHECK-NEXT:------------  --------
+# CHECK-NEXT:__debug_info        52 (12.04%)
+# CHECK-EMPTY:
+# CHECK-NEXT: Total Size: 52  (12.04%)
+# CHECK-NEXT: Total File Size: 432
+# CHECK-NEXT:----------------------------------------------------
+# CHECK-NEXT:----------------------------------------------------
+# CHECK-NEXT:file: [[FILE]](x86_64)
+# CHECK-NEXT:----------------------------------------------------
+# CHECK-NEXT:SECTION       SIZE (b)
+# CHECK-NEXT:------------  --------
+# CHECK-NEXT:__debug_info        64 (13.11%)
+# CHECK-EMPTY:
+# CHECK-NEXT: Total Size: 64  (13.11%)
+# CHECK-NEXT: Total File Size: 488
+# CHECK-NEXT:----------------------------------------------------
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/section_sizes_macho.test b/llvm/test/tools/llvm-dwarfdump/X86/section_sizes_macho.test
new file mode 100644
index 0000000000000..517416e4ca640
--- /dev/null
+++ b/llvm/test/tools/llvm-dwarfdump/X86/section_sizes_macho.test
@@ -0,0 +1,106 @@
+## Check how llvm-dwarfdump calculates section sizes
+## with --show-section-sizes for Mach-O objects.
+
+# RUN: yaml2obj %s | llvm-dwarfdump - --show-section-sizes \
+# RUN:   | FileCheck %s --match-full-lines --strict-whitespace
+
+#      CHECK:----------------------------------------------------
+# CHECK-NEXT:file: {{.*}}
+# CHECK-NEXT:----------------------------------------------------
+# CHECK-NEXT:SECTION       SIZE (b)
+# CHECK-NEXT:------------  --------
+# CHECK-NEXT:__debug_info         4 (0.93%)
+# CHECK-EMPTY:
+# CHECK-NEXT: Total Size: 4  (0.93%)
+# CHECK-NEXT: Total File Size: 432
+# CHECK-NEXT:----------------------------------------------------
+
+--- !mach-o
+FileHeader:
+  magic:           0xFEEDFACF
+  cputype:         0x01000007
+  cpusubtype:      0x00000003
+  filetype:        0x00000001
+  ncmds:           4
+  sizeofcmds:      360
+  flags:           0x00002000
+  reserved:        0x00000000
+LoadCommands:
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         232
+    segname:         ''
+    vmaddr:          0
+    vmsize:          4
+    fileoff:         392
+    filesize:        0
+    maxprot:         7
+    initprot:        7
+    nsects:          2
+    flags:           0
+    Sections:
+## This is a debug section following the ELF naming style, and is used
+## to show that such sections are not included in the report.
+      - sectname:        .debug_line
+        segname:         __DWARF
+        addr:            0x0000000000000000
+        size:            0
+        offset:          0x00000188
+        align:           0
+        reloff:          0x00000000
+        nreloc:          0
+        flags:           0x80000000
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+        content:         ''
+      - sectname:        __debug_info
+        segname:         __DWARF
+        addr:            0x0000000000000000
+        size:            4
+        offset:          0x00000000
+        align:           2
+        reloff:          0x00000000
+        nreloc:          0
+        flags:           0x00000001
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+  - cmd:             LC_BUILD_VERSION
+    cmdsize:         24
+    platform:        1
+    minos:           658944
+    sdk:             658944
+    ntools:          0
+  - cmd:             LC_SYMTAB
+    cmdsize:         24
+    symoff:          392
+    nsyms:           2
+    stroff:          424
+    strsize:         8
+  - cmd:             LC_DYSYMTAB
+    cmdsize:         80
+    ilocalsym:       0
+    nlocalsym:       1
+    iextdefsym:      1
+    nextdefsym:      0
+    iundefsym:       1
+    nundefsym:       1
+    tocoff:          0
+    ntoc:            0
+    modtaboff:       0
+    nmodtab:         0
+    extrefsymoff:    0
+    nextrefsyms:     0
+    indirectsymoff:  0
+    nindirectsyms:   0
+    extreloff:       0
+    nextrel:         0
+    locreloff:       0
+    nlocrel:         0
+LinkEditData:
+  StringTable:
+    - ''
+    - _b
+    - _a
+    - ''
+...
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/section_sizes_no_debug_sections.test b/llvm/test/tools/llvm-dwarfdump/X86/section_sizes_no_debug_sections.test
new file mode 100644
index 0000000000000..e272b1f64bdee
--- /dev/null
+++ b/llvm/test/tools/llvm-dwarfdump/X86/section_sizes_no_debug_sections.test
@@ -0,0 +1,29 @@
+## Check how llvm-dwarfdump calculates section sizes with
+## --show-section-sizes in the case there isn't any debug section.
+
+# RUN: yaml2obj %s | llvm-dwarfdump - --show-section-sizes \
+# RUN:   | FileCheck %s --match-full-lines --strict-whitespace
+
+#      CHECK:----------------------------------------------------
+# CHECK-NEXT:file: {{.*}}
+# CHECK-NEXT:----------------------------------------------------
+# CHECK-NEXT:SECTION  SIZE (b)
+# CHECK-NEXT:-------  --------
+# CHECK-EMPTY:
+# CHECK-NEXT: Total Size: 0  (0.00%)
+# CHECK-NEXT: Total File Size: 456
+# CHECK-NEXT:----------------------------------------------------
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_X86_64
+Sections:
+  - Name: .foo
+    Type: SHT_PROGBITS
+    Size: 17
+  - Name: .bar
+    Type: SHT_PROGBITS
+    Size: 19
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/statistics.ll b/llvm/test/tools/llvm-dwarfdump/X86/statistics.ll
index 1830e27837dd4..aa4bcaf121a61 100644
--- a/llvm/test/tools/llvm-dwarfdump/X86/statistics.ll
+++ b/llvm/test/tools/llvm-dwarfdump/X86/statistics.ll
@@ -48,7 +48,11 @@
 ; CHECK: "scope bytes covered":
 ; CHECK: "total function size":[[FUNCSIZE:[0-9]+]]
 ; CHECK: "total inlined function size":[[INLINESIZE:[0-9]+]]
-
+; CHECK: "size of __debug_info":380
+; CHECK: "size of __debug_loc":35
+; CHECK: "size of __debug_abbrev":303
+; CHECK: "size of __debug_line":117
+; CHECK: "size of __debug_str":204
 
 ; ModuleID = '/tmp/quality.cpp'
 source_filename = "/tmp/quality.cpp"
diff --git a/llvm/test/tools/llvm-ifs/Inputs/strong-mismatch-size.ifs b/llvm/test/tools/llvm-ifs/Inputs/strong-mismatch-size.ifs
index 9afb08802726a..30b7cda9b548c 100644
--- a/llvm/test/tools/llvm-ifs/Inputs/strong-mismatch-size.ifs
+++ b/llvm/test/tools/llvm-ifs/Inputs/strong-mismatch-size.ifs
@@ -1,8 +1,8 @@
 # NOTE: Used by weak-mismatch.ifs
---- !experimental-ifs-v1
-IfsVersion: 1.0
+--- !experimental-ifs-v2
+IfsVersion: 2.0
 Triple: x86_64-unknown-linux-gnu
 ObjectFileFormat: ELF
 Symbols:
-  foobar: { Type: Object, Size: 2 }
+  - { Name: foobar, Type: Object, Size: 2 }
 ...
diff --git a/llvm/test/tools/llvm-ifs/Inputs/strong-mismatch-type.ifs b/llvm/test/tools/llvm-ifs/Inputs/strong-mismatch-type.ifs
index 8fc550a644cb2..3f8d54c7e536f 100644
--- a/llvm/test/tools/llvm-ifs/Inputs/strong-mismatch-type.ifs
+++ b/llvm/test/tools/llvm-ifs/Inputs/strong-mismatch-type.ifs
@@ -1,8 +1,8 @@
 # NOTE: Used by weak-mismatch.ifs
---- !experimental-ifs-v1
-IfsVersion: 1.0
+--- !experimental-ifs-v2
+IfsVersion: 2.0
 Triple: x86_64-unknown-linux-gnu
 ObjectFileFormat: ELF
 Symbols:
-  foobar: { Type: Func }
+  - { Name: foobar, Type: Func }
 ...
diff --git a/llvm/test/tools/llvm-ifs/conflict-header-format.ifs b/llvm/test/tools/llvm-ifs/conflict-header-format.ifs
index 40ae9c0526f25..4e26fb8080e8c 100644
--- a/llvm/test/tools/llvm-ifs/conflict-header-format.ifs
+++ b/llvm/test/tools/llvm-ifs/conflict-header-format.ifs
@@ -5,10 +5,10 @@
 # CHECK-IFS-NEXT: Filenames:
 # CHECK-IFS-NEXT: ObjectFileFormat Values: TBD ELF
 
---- !experimental-ifs-v1
-IfsVersion:      1.0
+--- !experimental-ifs-v2
+IfsVersion: 2.0
 Triple:          x86_64-apple-unknown
 ObjectFileFormat: TBD
 Symbols:
-  a:               { Type: Func }
+  - { Name: a, Type: Func }
 ...
diff --git a/llvm/test/tools/llvm-ifs/conflict-header-triple.ifs b/llvm/test/tools/llvm-ifs/conflict-header-triple.ifs
index 15bddc6a15f7a..9ce04b8b3f315 100644
--- a/llvm/test/tools/llvm-ifs/conflict-header-triple.ifs
+++ b/llvm/test/tools/llvm-ifs/conflict-header-triple.ifs
@@ -5,10 +5,10 @@
 # CHECK-IFS-NEXT: Filenames:
 # CHECK-IFS-NEXT: Triple Values: mips-unknown-linux x86_64-unknown-linux-gnu
 
---- !experimental-ifs-v1
-IfsVersion:      1.0
+--- !experimental-ifs-v2
+IfsVersion: 2.0
 Triple:          mips-unknown-linux
 ObjectFileFormat: ELF
 Symbols:
-  a:               { Type: Func }
+  - { Name: a, Type: Func }
 ...
diff --git a/llvm/test/tools/llvm-ifs/conflict-header-version.ifs b/llvm/test/tools/llvm-ifs/conflict-header-version.ifs
index addf9943441b9..ecdeb311f8601 100644
--- a/llvm/test/tools/llvm-ifs/conflict-header-version.ifs
+++ b/llvm/test/tools/llvm-ifs/conflict-header-version.ifs
@@ -5,12 +5,12 @@
 # RUN: FileCheck %s --check-prefixes=CHECK-IFS2
 
 # CHECK-IFS: error: Interface Stub: IfsVersion Mismatch.
-# CHECK-IFS2: error: Interface Stub: Bad IfsVersion: 0.0, llvm-ifs supported version: 1.2.
+# CHECK-IFS2: error: Interface Stub: Bad IfsVersion: 0.0, llvm-ifs supported version: 2.0.
 
---- !experimental-ifs-v1
+--- !experimental-ifs-v2
 IfsVersion:      0.0
 Triple:          x86_64-unknown-linux-gnu
 ObjectFileFormat: ELF
 Symbols:
-  a:               { Type: Func }
+  - { Name: a, Type: Func }
 ...
diff --git a/llvm/test/tools/llvm-ifs/conflict-size.ifs b/llvm/test/tools/llvm-ifs/conflict-size.ifs
index 173ce268c741c..5e0fcafd55db7 100644
--- a/llvm/test/tools/llvm-ifs/conflict-size.ifs
+++ b/llvm/test/tools/llvm-ifs/conflict-size.ifs
@@ -7,10 +7,10 @@
 # CHECK-IFS-NEXT: Filename:
 # CHECK-IFS-NEXT: Size Values: 1 4
 
---- !experimental-ifs-v1
-IfsVersion: 1.0
+--- !experimental-ifs-v2
+IfsVersion: 2.0
 Triple: x86_64-unknown-linux-gnu
 ObjectFileFormat: ELF
 Symbols:
-  b: { Type: Object, Size: 1 }
+  - { Name: b, Type: Object, Size: 1 }
 ...
diff --git a/llvm/test/tools/llvm-ifs/conflict-type.ifs b/llvm/test/tools/llvm-ifs/conflict-type.ifs
index c518be4e14115..1a10ea79a41c1 100644
--- a/llvm/test/tools/llvm-ifs/conflict-type.ifs
+++ b/llvm/test/tools/llvm-ifs/conflict-type.ifs
@@ -7,10 +7,10 @@
 # CHECK-IFS-NEXT: Filename:
 # CHECK-IFS-NEXT: Type Values: Object Func
 
---- !experimental-ifs-v1
-IfsVersion: 1.0
+--- !experimental-ifs-v2
+IfsVersion: 2.0
 Triple: x86_64-unknown-linux-gnu
 ObjectFileFormat: ELF
 Symbols:
-  a: { Type: Object, Size: 1 }
+  - { Name: a, Type: Object, Size: 1 }
 ...
diff --git a/llvm/test/tools/llvm-ifs/conflict-weak.ifs b/llvm/test/tools/llvm-ifs/conflict-weak.ifs
index 823b8f1866c36..23eb73d7535fa 100644
--- a/llvm/test/tools/llvm-ifs/conflict-weak.ifs
+++ b/llvm/test/tools/llvm-ifs/conflict-weak.ifs
@@ -2,12 +2,12 @@
 # RUN: FileCheck %s --check-prefixes=CHECK-IFS
 
 # CHECK-IFS: Symbols:
-# CHECK-IFS-NEXT: a: { Type: Func, Weak: true }
+# CHECK-IFS-NEXT: - { Name: a, Type: Func, Weak: true }
 
---- !experimental-ifs-v1
-IfsVersion: 1.0
+--- !experimental-ifs-v2
+IfsVersion: 2.0
 Triple: x86_64-unknown-linux-gnu
 ObjectFileFormat: ELF
 Symbols:
-  a: { Type: Func, Weak: true }
+  - { Name: a, Type: Func, Weak: true }
 ...
diff --git a/llvm/test/tools/llvm-ifs/default-empty.ifs b/llvm/test/tools/llvm-ifs/default-empty.ifs
index 9848f418f58c6..c61f29a379765 100644
--- a/llvm/test/tools/llvm-ifs/default-empty.ifs
+++ b/llvm/test/tools/llvm-ifs/default-empty.ifs
@@ -1,25 +1,25 @@
 # RUN: llvm-ifs -action write-ifs -o - %s | FileCheck --check-prefixes=CHECK-DEFAULT %s
 # RUN: llvm-ifs -action write-ifs -o - %s %S/weak.ifs | FileCheck --check-prefixes=CHECK-MERGE %s
 
-# CHECK-DEFAULT: --- !experimental-ifs-v1
-# CHECK-DEFAULT-NEXT: IfsVersion:      1.2
+# CHECK-DEFAULT: --- !experimental-ifs-v2
+# CHECK-DEFAULT-NEXT: IfsVersion: 2.0
 # CHECK-DEFAULT-NEXT: Triple:          ''
 # CHECK-DEFAULT-NEXT: ObjectFileFormat: ELF
-# CHECK-DEFAULT-NEXT: Symbols:         {}
+# CHECK-DEFAULT-NEXT: Symbols:         []
 # CHECK-DEFAULT-NEXT: ...
 
-# CHECK-MERGE: --- !experimental-ifs-v1
-# CHECK-MERGE-NEXT: IfsVersion: 1.0
+# CHECK-MERGE: --- !experimental-ifs-v2
+# CHECK-MERGE-NEXT: IfsVersion: 2.0
 # CHECK-MERGE-NEXT: Triple: x86_64-unknown-linux-gnu
 # CHECK-MERGE-NEXT: ObjectFileFormat: ELF
 # CHECK-MERGE-NEXT: Symbols:
-# CHECK-MERGE-DAG:   _Z8weakFuncv: { Type: Func, Weak: true }
-# CHECK-MERGE-DAG:   _Z10strongFuncv: { Type: Func }
+# CHECK-MERGE-DAG:  - { Name: _Z8weakFuncv, Type: Func, Weak: true }
+# CHECK-MERGE-DAG:  - { Name: _Z10strongFuncv, Type: Func }
 # CHECK-MERGE: ...
 
---- !experimental-ifs-v1
-IfsVersion:      1.2
+--- !experimental-ifs-v2
+IfsVersion: 2.0
 Triple:          ''
 ObjectFileFormat: ELF
-Symbols:         {}
+Symbols:         []
 ...
diff --git a/llvm/test/tools/llvm-ifs/empty1.ifs b/llvm/test/tools/llvm-ifs/empty1.ifs
new file mode 100644
index 0000000000000..d237dd7ea10ac
--- /dev/null
+++ b/llvm/test/tools/llvm-ifs/empty1.ifs
@@ -0,0 +1,15 @@
+# RUN: llvm-ifs -action write-ifs -o - %s | FileCheck %s
+
+# CHECK: --- !experimental-ifs-v2
+# CHECK-NEXT: IfsVersion: 2.0
+# CHECK-NEXT: Triple: x86_64-unknown-linux-gnu
+# CHECK-NEXT: ObjectFileFormat: ELF
+# CHECK-NEXT: Symbols: []
+# CHECK: ...
+
+--- !experimental-ifs-v2
+IfsVersion:      2.0
+Triple:          x86_64-unknown-linux-gnu
+ObjectFileFormat: ELF
+Symbols:         []
+...
diff --git a/llvm/test/tools/llvm-ifs/empty2.ifs b/llvm/test/tools/llvm-ifs/empty2.ifs
new file mode 100644
index 0000000000000..a294c777bbf9b
--- /dev/null
+++ b/llvm/test/tools/llvm-ifs/empty2.ifs
@@ -0,0 +1,15 @@
+# RUN: llvm-ifs -action write-ifs -o - %s | FileCheck %s
+
+# CHECK: --- !experimental-ifs-v2
+# CHECK-NEXT: IfsVersion: 2.0
+# CHECK-NEXT: Triple: x86_64-unknown-linux-gnu
+# CHECK-NEXT: ObjectFileFormat: ELF
+# CHECK-NEXT: Symbols: []
+# CHECK: ...
+
+--- !experimental-ifs-v2
+IfsVersion:      2.0
+Triple:          x86_64-unknown-linux-gnu
+ObjectFileFormat: ELF
+Symbols:
+...
diff --git a/llvm/test/tools/llvm-ifs/func.ifs b/llvm/test/tools/llvm-ifs/func.ifs
index 496e26241922b..d6d85782b2e37 100644
--- a/llvm/test/tools/llvm-ifs/func.ifs
+++ b/llvm/test/tools/llvm-ifs/func.ifs
@@ -10,13 +10,13 @@
 # RUN: llvm-ifs -action write-ifs -o - %s %s | \
 # RUN: FileCheck %s --check-prefixes=CHECK-MERGE-IFS
 
-# CHECK-IFS: --- !experimental-ifs-v1
-# CHECK-IFS-NEXT: IfsVersion:      1.0
+# CHECK-IFS: --- !experimental-ifs-v2
+# CHECK-IFS-NEXT: IfsVersion: 2.0
 # CHECK-IFS-NEXT: Triple:          x86_64-unknown-linux-gnu
 # CHECK-IFS-NEXT: ObjectFileFormat: ELF
 # CHECK-IFS-NEXT: Symbols:
-# CHECK-IFS-DAG:   a:               { Type: Func }
-# CHECK-IFS-DAG:   b:               { Type: Object, Size: 4 }
+# CHECK-IFS-DAG:   - { Name: a, Type: Func }
+# CHECK-IFS-DAG:   - { Name: b, Type: Object, Size: 4 }
 # CHECK-IFS: ...
 
 # CHECK-ELF: ELF Header:
@@ -39,18 +39,18 @@
 # CHECK-DARWIN-TBD3-NEXT: ...
 
 # Here we are testing to see if two identical symbols will merge.
-# CHECK-MERGE-IFS: --- !experimental-ifs-v1
-# CHECK-MERGE-IFS-NEXT: IfsVersion:      1.0
+# CHECK-MERGE-IFS: --- !experimental-ifs-v2
+# CHECK-MERGE-IFS-NEXT: IfsVersion: 2.0
 # CHECK-MERGE-IFS-NEXT: Triple:          x86_64-unknown-linux-gnu
 # CHECK-MERGE-IFS-NEXT: ObjectFileFormat: ELF
 # CHECK-MERGE-IFS-NEXT: Symbols:
-# CHECK-MERGE-IFS-NEXT:   a:               { Type: Func }
+# CHECK-MERGE-IFS-NEXT:   - { Name: a, Type: Func }
 # CHECK-MERGE-IFS-NEXT: ...
 
---- !experimental-ifs-v1
-IfsVersion: 1.0
+--- !experimental-ifs-v2
+IfsVersion: 2.0
 Triple: x86_64-unknown-linux-gnu
 ObjectFileFormat: ELF
 Symbols:
-  a: { Type: Func }
+  - { Name: a, Type: Func }
 ...
diff --git a/llvm/test/tools/llvm-ifs/ios-tbd.ifs b/llvm/test/tools/llvm-ifs/ios-tbd.ifs
index 13671b02c5cb3..5b21aedf6500b 100644
--- a/llvm/test/tools/llvm-ifs/ios-tbd.ifs
+++ b/llvm/test/tools/llvm-ifs/ios-tbd.ifs
@@ -13,10 +13,10 @@
 # CHECK-NEXT:     symbols:         [ __Z3fooi ]
 # CHECK-NEXT: ...
 
---- !experimental-ifs-v1
-IfsVersion: 1.0
+--- !experimental-ifs-v2
+IfsVersion: 2.0
 Triple: arm64-apple-ios
 ObjectFileFormat: TBD
 Symbols:
-  __Z3fooi: { Type: Func }
+  - { Name: __Z3fooi, Type: Func }
 ...
diff --git a/llvm/test/tools/llvm-ifs/macos-tbd.ifs b/llvm/test/tools/llvm-ifs/macos-tbd.ifs
index bd84806fb2199..b04828b2a39d5 100644
--- a/llvm/test/tools/llvm-ifs/macos-tbd.ifs
+++ b/llvm/test/tools/llvm-ifs/macos-tbd.ifs
@@ -13,10 +13,10 @@
 # CHECK-NEXT:     symbols:         [ __Z3fooi ]
 # CHECK-NEXT: ...
 
---- !experimental-ifs-v1
-IfsVersion: 1.0
+--- !experimental-ifs-v2
+IfsVersion: 2.0
 Triple: arm64-apple-macosx
 ObjectFileFormat: TBD
 Symbols:
-  __Z3fooi: { Type: Func }
+  - { Name: __Z3fooi, Type: Func }
 ...
diff --git a/llvm/test/tools/llvm-ifs/object-function-size-weak-combo.ifs b/llvm/test/tools/llvm-ifs/object-function-size-weak-combo.ifs
index b6328fbc58d91..769f423f328a0 100644
--- a/llvm/test/tools/llvm-ifs/object-function-size-weak-combo.ifs
+++ b/llvm/test/tools/llvm-ifs/object-function-size-weak-combo.ifs
@@ -4,17 +4,17 @@
 # RUN: llvm-ifs -action write-bin -o - %s %S/func.ifs %S/object.ifs %S/weak.ifs | \
 # RUN: llvm-readelf --all | FileCheck %s --check-prefixes=CHECK-ELF
 
-# CHECK-IFS: --- !experimental-ifs-v1
-# CHECK-IFS-NEXT: IfsVersion:      1.0
+# CHECK-IFS: --- !experimental-ifs-v2
+# CHECK-IFS-NEXT: IfsVersion: 2.0
 # CHECK-IFS-NEXT: Triple:          x86_64-unknown-linux-gnu
 # CHECK-IFS-NEXT: ObjectFileFormat: ELF
 # CHECK-IFS-NEXT: Symbols:
-# CHECK-IFS-DAG:   e:               { Type: Object, Size: 8 }
-# CHECK-IFS-DAG:   a:               { Type: Func }
-# CHECK-IFS-DAG:   f:               { Type: Object, Size: 2 }
-# CHECK-IFS-DAG:   _Z10strongFuncv: { Type: Func }
-# CHECK-IFS-DAG:   _Z8weakFuncv:    { Type: Func, Weak: true }
-# CHECK-IFS-DAG:   b:               { Type: Object, Size: 4 }
+# CHECK-IFS-DAG:   - { Name: e,              Type: Object, Size: 8 }
+# CHECK-IFS-DAG:   - { Name: a,              Type: Func }
+# CHECK-IFS-DAG:   - { Name: f,              Type: Object, Size: 2 }
+# CHECK-IFS-DAG:   - { Name: _Z10strongFuncv, Type: Func }
+# CHECK-IFS-DAG:   - { Name: _Z8weakFuncv,   Type: Func, Weak: true }
+# CHECK-IFS-DAG:   - { Name: b,              Type: Object, Size: 4 }
 # CHECK-IFS: ...
 
 # CHECK-ELF: FUNC    GLOBAL DEFAULT  1 _Z10strongFuncv
@@ -24,11 +24,11 @@
 # CHECK-ELF: OBJECT  GLOBAL DEFAULT  1 e
 # CHECK-ELF: OBJECT  GLOBAL DEFAULT  1 f
 
---- !experimental-ifs-v1
-IfsVersion:      1.0
+--- !experimental-ifs-v2
+IfsVersion: 2.0
 Triple:          x86_64-unknown-linux-gnu
 ObjectFileFormat: ELF
 Symbols:
-  e:               { Type: Object, Size: 8 }
-  f:               { Type: Object, Size: 2 }
+  - { Name: e, Type: Object, Size: 8 }
+  - { Name: f, Type: Object, Size: 2 }
 ...
diff --git a/llvm/test/tools/llvm-ifs/object.ifs b/llvm/test/tools/llvm-ifs/object.ifs
index 733cc38001d36..c4823c20fce25 100644
--- a/llvm/test/tools/llvm-ifs/object.ifs
+++ b/llvm/test/tools/llvm-ifs/object.ifs
@@ -4,12 +4,12 @@
 # RUN: llvm-ifs -action write-bin -o - %s | \
 # RUN: llvm-readelf --all | FileCheck %s --check-prefixes=CHECK-ELF
 
-# CHECK-IFS: --- !experimental-ifs-v1
-# CHECK-IFS-NEXT: IfsVersion:      1.0
+# CHECK-IFS: --- !experimental-ifs-v2
+# CHECK-IFS-NEXT: IfsVersion: 2.0
 # CHECK-IFS-NEXT: Triple:          x86_64-unknown-linux-gnu
 # CHECK-IFS-NEXT: ObjectFileFormat: ELF
 # CHECK-IFS-NEXT: Symbols:
-# CHECK-IFS-NEXT:   b:               { Type: Object, Size: 4 }
+# CHECK-IFS-NEXT:   - { Name: b, Type: Object, Size: 4 }
 # CHECK-IFS-NEXT: ...
 
 # CHECK-ELF: ELF Header:
@@ -19,10 +19,10 @@
 # CHECK-ELF-NOT:   FUNC    GLOBAL DEFAULT  1 a
 # CHECK-ELF:   OBJECT  GLOBAL DEFAULT  1 b
 
---- !experimental-ifs-v1
-IfsVersion: 1.0
+--- !experimental-ifs-v2
+IfsVersion: 2.0
 Triple: x86_64-unknown-linux-gnu
 ObjectFileFormat: ELF
 Symbols:
-  b: { Type: Object, Size: 4 }
+  - { Name: b, Type: Object, Size: 4 }
 ...
diff --git a/llvm/test/tools/llvm-ifs/strong.ifs b/llvm/test/tools/llvm-ifs/strong.ifs
index bdc930fbaaa3d..ccc1f9e5d8b64 100644
--- a/llvm/test/tools/llvm-ifs/strong.ifs
+++ b/llvm/test/tools/llvm-ifs/strong.ifs
@@ -1,17 +1,17 @@
 # RUN: llvm-ifs -action write-ifs -o - %s %S/strong.ifs | FileCheck %s --check-prefixes=CHECK-IFS
 
-# CHECK-IFS: --- !experimental-ifs-v1
-# CHECK-IFS-NEXT: IfsVersion: 1.0
+# CHECK-IFS: --- !experimental-ifs-v2
+# CHECK-IFS-NEXT: IfsVersion: 2.0
 # CHECK-IFS-NEXT: Triple: x86_64-unknown-linux-gnu
 # CHECK-IFS-NEXT: ObjectFileFormat: ELF
 # CHECK-IFS-NEXT: Symbols:
-# CHECK-IFS-DAG:   _Z8weakFuncv: { Type: Func }
+# CHECK-IFS-DAG:   - { Name: _Z8weakFuncv, Type: Func }
 # CHECK-IFS: ...
 
---- !experimental-ifs-v1
-IfsVersion: 1.0
+--- !experimental-ifs-v2
+IfsVersion: 2.0
 Triple: x86_64-unknown-linux-gnu
 ObjectFileFormat: ELF
 Symbols:
-  _Z8weakFuncv: { Type: Func }
+  - { Name: _Z8weakFuncv, Type: Func }
 ...
diff --git a/llvm/test/tools/llvm-ifs/tvos-tbd.ifs b/llvm/test/tools/llvm-ifs/tvos-tbd.ifs
index 08c8478c1daf2..6db01bf6162fb 100644
--- a/llvm/test/tools/llvm-ifs/tvos-tbd.ifs
+++ b/llvm/test/tools/llvm-ifs/tvos-tbd.ifs
@@ -13,10 +13,10 @@
 # CHECK-NEXT:     symbols:         [ __Z3fooi ]
 # CHECK-NEXT: ...
 
---- !experimental-ifs-v1
-IfsVersion: 1.0
+--- !experimental-ifs-v2
+IfsVersion: 2.0
 Triple: arm64-apple-tvos
 ObjectFileFormat: TBD
 Symbols:
-  __Z3fooi: { Type: Func }
+  - { Name: __Z3fooi, Type: Func }
 ...
diff --git a/llvm/test/tools/llvm-ifs/version-ok.ifs b/llvm/test/tools/llvm-ifs/version-ok.ifs
index fd150ee77d559..646b8624feb1d 100644
--- a/llvm/test/tools/llvm-ifs/version-ok.ifs
+++ b/llvm/test/tools/llvm-ifs/version-ok.ifs
@@ -1,9 +1,9 @@
 # RUN: llvm-ifs -action write-ifs -o - %s %S/object.ifs
 
---- !experimental-ifs-v1
-IfsVersion:      1.1
+--- !experimental-ifs-v2
+IfsVersion: 2.0
 Triple:          x86_64-unknown-linux-gnu
 ObjectFileFormat: ELF
 Symbols:
-  a:               { Type: Func }
+  - { Name: a, Type: Func }
 ...
diff --git a/llvm/test/tools/llvm-ifs/watchos-tbd.ifs b/llvm/test/tools/llvm-ifs/watchos-tbd.ifs
index 74a9d962a3e01..fcb9142652021 100644
--- a/llvm/test/tools/llvm-ifs/watchos-tbd.ifs
+++ b/llvm/test/tools/llvm-ifs/watchos-tbd.ifs
@@ -13,10 +13,10 @@
 # CHECK-NEXT:     symbols:         [ __Z3fooi ]
 # CHECK-NEXT: ...
 
---- !experimental-ifs-v1
-IfsVersion: 1.0
+--- !experimental-ifs-v2
+IfsVersion: 2.0
 Triple: arm64-apple-watchos
 ObjectFileFormat: TBD
 Symbols:
-  __Z3fooi: { Type: Func }
+  - { Name: __Z3fooi, Type: Func }
 ...
diff --git a/llvm/test/tools/llvm-ifs/weak-mismatch.ifs b/llvm/test/tools/llvm-ifs/weak-mismatch.ifs
index 15abc2064cc21..cf45dff8c0628 100644
--- a/llvm/test/tools/llvm-ifs/weak-mismatch.ifs
+++ b/llvm/test/tools/llvm-ifs/weak-mismatch.ifs
@@ -10,10 +10,10 @@
 # CHECK-TYPE-NEXT: Filename:
 # CHECK-TYPE-NEXT: Type Values: Object Func
 
---- !experimental-ifs-v1
-IfsVersion: 1.0
+--- !experimental-ifs-v2
+IfsVersion: 2.0
 Triple: x86_64-unknown-linux-gnu
 ObjectFileFormat: ELF
 Symbols:
-  foobar: { Type: Object, Size: 1, Weak: true }
+  - { Name: foobar, Type: Object, Size: 1, Weak: true }
 ...
diff --git a/llvm/test/tools/llvm-ifs/weak.ifs b/llvm/test/tools/llvm-ifs/weak.ifs
index a7441be1c1f22..bf80910505301 100644
--- a/llvm/test/tools/llvm-ifs/weak.ifs
+++ b/llvm/test/tools/llvm-ifs/weak.ifs
@@ -1,19 +1,19 @@
 # RUN: llvm-ifs -action write-ifs -o - %s | FileCheck %s --check-prefixes=CHECK-IFS
 
-# CHECK-IFS: --- !experimental-ifs-v1
-# CHECK-IFS-NEXT: IfsVersion: 1.0
+# CHECK-IFS: --- !experimental-ifs-v2
+# CHECK-IFS-NEXT: IfsVersion: 2.0
 # CHECK-IFS-NEXT: Triple: x86_64-unknown-linux-gnu
 # CHECK-IFS-NEXT: ObjectFileFormat: ELF
 # CHECK-IFS-NEXT: Symbols:
-# CHECK-IFS-DAG:   _Z8weakFuncv: { Type: Func, Weak: true }
-# CHECK-IFS-DAG:   _Z10strongFuncv: { Type: Func }
+# CHECK-IFS-DAG:   - { Name: _Z8weakFuncv, Type: Func, Weak: true }
+# CHECK-IFS-DAG:   - { Name: _Z10strongFuncv, Type: Func }
 # CHECK-IFS: ...
 
---- !experimental-ifs-v1
-IfsVersion: 1.0
+--- !experimental-ifs-v2
+IfsVersion: 2.0
 Triple: x86_64-unknown-linux-gnu
 ObjectFileFormat: ELF
 Symbols:
-  _Z8weakFuncv: { Type: Func, Weak: true }
-  _Z10strongFuncv: { Type: Func }
+  - { Name: _Z8weakFuncv, Type: Func, Weak: true }
+  - { Name: _Z10strongFuncv, Type: Func }
 ...
diff --git a/llvm/test/tools/llvm-objdump/ELF/AMDGPU/source-lines.ll b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/source-lines.ll
index 6258d48748e25..7b8a53a2626a3 100644
--- a/llvm/test/tools/llvm-objdump/ELF/AMDGPU/source-lines.ll
+++ b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/source-lines.ll
@@ -68,7 +68,7 @@ entry:
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
-attributes #0 = { noinline nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+fp64-fp16-denormals,-fp32-denormals" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
 
 !llvm.dbg.cu = !{!0}
diff --git a/llvm/test/tools/llvm-objdump/ELF/PowerPC/branch-offset.s b/llvm/test/tools/llvm-objdump/ELF/PowerPC/branch-offset.s
index e78e4c16eab55..bc0b682530b86 100644
--- a/llvm/test/tools/llvm-objdump/ELF/PowerPC/branch-offset.s
+++ b/llvm/test/tools/llvm-objdump/ELF/PowerPC/branch-offset.s
@@ -8,10 +8,10 @@
 # RUN: llvm-objdump -d --no-show-raw-insn %t.64.o | FileCheck --check-prefixes=ELF64,CHECK %s
 
 # CHECK-LABEL: <bl>:
-# ELF32-NEXT:   bl .-4
-# ELF64-NEXT:   bl .-4
-# CHECK-NEXT:   bl .+0
-# CHECK-NEXT:   bl .+4
+# ELF32-NEXT:   bl 0xfffffffc
+# ELF64-NEXT:   bl 0xfffffffffffffffc
+# CHECK-NEXT:   bl 0x4
+# CHECK-NEXT:   bl 0xc
 
 bl:
   bl .-4
@@ -19,9 +19,9 @@ bl:
   bl .+4
 
 # CHECK-LABEL: <b>:
-# CHECK-NEXT:   b .+67108860
-# CHECK-NEXT:   b .+0
-# CHECK-NEXT:   b .+4
+# CHECK-NEXT:   b 0x8
+# CHECK-NEXT:   b 0x10
+# CHECK-NEXT:   b 0x18
 
 b:
   b .-4
@@ -29,7 +29,9 @@ b:
   b .+4
 
 # CHECK-LABEL: <bt>:
-# CHECK-NEXT:   bt 2, .+65532
+# CHECK-NEXT:   18: bt 2, 0x14
+# CHECK-NEXT:   1c: bt 1, 0x20
 
 bt:
   bt 2, .-4
+  bgt .+4
diff --git a/llvm/test/tools/llvm-objdump/ELF/call-absolute-symbol.test b/llvm/test/tools/llvm-objdump/ELF/call-absolute-symbol.test
index f1c6ff025d706..424daf31bfff4 100644
--- a/llvm/test/tools/llvm-objdump/ELF/call-absolute-symbol.test
+++ b/llvm/test/tools/llvm-objdump/ELF/call-absolute-symbol.test
@@ -1,3 +1,3 @@
 // REQUIRES: x86_64-linux
 // RUN: llvm-objdump -d %p/Inputs/call-absolute-symbol.elf-x86_64 | FileCheck %s
-CHECK: 201000:	e8 fb f0 df ff 	callq	-2100997 <foo>
+CHECK: 201000:	e8 fb f0 df ff 	callq	0x100 <foo>
diff --git a/llvm/test/tools/llvm-objdump/MachO/hex-displacement.test b/llvm/test/tools/llvm-objdump/MachO/hex-displacement.test
index 541cca53869bd..d8d481cec043b 100644
--- a/llvm/test/tools/llvm-objdump/MachO/hex-displacement.test
+++ b/llvm/test/tools/llvm-objdump/MachO/hex-displacement.test
@@ -3,4 +3,4 @@
 
 # RUN: llvm-objdump -d %p/Inputs/hello.exe.macho-i386 | FileCheck %s
 
-# CHECK: 1f47:   e8 00 00 00 00  calll   0 <_main+0xc>
+# CHECK: 1f47:   e8 00 00 00 00  calll   0x1f4c <_main+0xc>
diff --git a/llvm/test/tools/llvm-objdump/RISCV/lit.local.cfg b/llvm/test/tools/llvm-objdump/RISCV/lit.local.cfg
new file mode 100644
index 0000000000000..c63820126f8ca
--- /dev/null
+++ b/llvm/test/tools/llvm-objdump/RISCV/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'RISCV' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/tools/llvm-objdump/RISCV/unknown-arch-attr.test b/llvm/test/tools/llvm-objdump/RISCV/unknown-arch-attr.test
new file mode 100644
index 0000000000000..23e10f8a110a2
--- /dev/null
+++ b/llvm/test/tools/llvm-objdump/RISCV/unknown-arch-attr.test
@@ -0,0 +1,38 @@
+## Handle unrecognized arch attributes.
+## Encode an unrecognized arch feature into an object file and try to decode it.
+## The expected behavior is to ignore the unrecognized arch feature and
+## continue to process the following arch features.
+##
+## The object file has the "rv32i2p0_x1p0_m2p0" arch feature. "x1p0" is an
+## unrecognized architecture extension. llvm-objdump will ignore it and decode
+## "mul" instruction correctly according to "m2p0" in the arch feature.
+##
+## This test cannot be assembly because the test needs an unrecognized arch
+## feature and `llvm-mc` will filter out the unrecognized arch feature.
+
+# RUN: yaml2obj %s -D BITS=32 -o %t.32.o
+# RUN: llvm-objdump -d %t.32.o \
+# RUN:   | FileCheck %s --check-prefixes=DISASM
+# RUN: yaml2obj %s -D BITS=64 -o %t.64.o
+# RUN: llvm-objdump -d %t.64.o \
+# RUN:   | FileCheck %s --check-prefixes=DISASM
+
+# DISASM: mul a0, a1, a2
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS[[BITS]]
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_RISCV
+Sections:
+  - Name:    .text
+    Type:    SHT_PROGBITS
+    Flags:   [ SHF_ALLOC, SHF_EXECINSTR ]
+## The content is the encoding of "mul a0, a1, a2".
+## The encoding could be decoded only when the "m" extension is enabled.
+    Content: 3385C502
+  - Name:    .riscv.attributes
+    Type:    SHT_RISCV_ATTRIBUTES
+## The content is the encoding of the arch feature "rv32i2p0_x1p0_m2p0"
+    Content: 412300000072697363760001190000000572763332693270305F783170305F6D32703000
diff --git a/llvm/test/tools/llvm-objdump/X86/coff-disassemble-export.test b/llvm/test/tools/llvm-objdump/X86/coff-disassemble-export.test
index 5cd01bcfa70d8..f05a1513b27a8 100644
--- a/llvm/test/tools/llvm-objdump/X86/coff-disassemble-export.test
+++ b/llvm/test/tools/llvm-objdump/X86/coff-disassemble-export.test
@@ -2,7 +2,7 @@
 // RUN:   FileCheck %s
 
 // CHECK-LABEL: <g>:
-// CHECK: calll 8 <f>
+// CHECK: calll 0x10001010 <f>
 
 // CHECK-LABEL: f:
-// CHECK: calll -24 <g>
+// CHECK: calll 0x10001000 <g>
diff --git a/llvm/test/tools/llvm-objdump/X86/demangle.s b/llvm/test/tools/llvm-objdump/X86/demangle.s
index d555bbf5ac74f..f6523f2e931bb 100644
--- a/llvm/test/tools/llvm-objdump/X86/demangle.s
+++ b/llvm/test/tools/llvm-objdump/X86/demangle.s
@@ -11,7 +11,7 @@
 ## Check the case when relocations are inlined into disassembly.
 # RUN: llvm-objdump -d -r --demangle %t | FileCheck %s --check-prefix=INLINE
 # INLINE:      <foo()>:
-# INLINE-NEXT:  0: {{.*}}  callq   0 <_Z3foov+0x5>
+# INLINE-NEXT:  0: {{.*}}  callq   0x5 <_Z3foov+0x5>
 # INLINE-NEXT:  0000000000000001:  R_X86_64_PLT32 foo()-0x4
 
 .text
diff --git a/llvm/test/tools/llvm-objdump/X86/elf-disassemble-no-symtab.test b/llvm/test/tools/llvm-objdump/X86/elf-disassemble-no-symtab.test
index a5bb43f95041d..61c268d218ebf 100644
--- a/llvm/test/tools/llvm-objdump/X86/elf-disassemble-no-symtab.test
+++ b/llvm/test/tools/llvm-objdump/X86/elf-disassemble-no-symtab.test
@@ -8,7 +8,7 @@
 # CHECK:       Disassembly of section .text:
 # CHECK-EMPTY:
 # CHECK-NEXT:  0000000000004000 <.text>:
-# CHECK-NEXT:    4000: e8 42 00 00 00                callq   66 <.text+0x47>
+# CHECK-NEXT:    4000: e8 42 00 00 00                callq   0x4047 <.text+0x47>
 
 --- !ELF
 FileHeader:
diff --git a/llvm/test/tools/llvm-objdump/X86/elf-disassemble-relocs.test b/llvm/test/tools/llvm-objdump/X86/elf-disassemble-relocs.test
index a7d77f863eb70..593735a6a1026 100644
--- a/llvm/test/tools/llvm-objdump/X86/elf-disassemble-relocs.test
+++ b/llvm/test/tools/llvm-objdump/X86/elf-disassemble-relocs.test
@@ -4,10 +4,10 @@
 # RUN: yaml2obj %s --docnum=1 -o %t1.o
 # RUN: llvm-objdump %t1.o -d -r | FileCheck %s --implicit-check-not="RELOCATION RECORDS"
 
-# CHECK:      0: e8 00 00 00 00                callq   0 <.text+0x5>
+# CHECK:      0: e8 00 00 00 00                callq   0x5 <.text+0x5>
 # CHECK-NEXT:          0000000000000001:  R_X86_64_PC32        foo-0x4
 # CHECK-NEXT:          0000000000000002:  R_X86_64_NONE        bar+0x8
-# CHECK-NEXT: 5: e8 00 00 00 00                callq   0 <.text+0xa>
+# CHECK-NEXT: 5: e8 00 00 00 00                callq   0xa <.text+0xa>
 # CHECK-NEXT:          0000000000000006:  R_X86_64_PLT32       foo+0x1
 
 --- !ELF
diff --git a/llvm/test/tools/llvm-objdump/X86/elf-disassemble-symbol-references.yaml b/llvm/test/tools/llvm-objdump/X86/elf-disassemble-symbol-references.yaml
index 92c82e7519d61..f1470c8479781 100644
--- a/llvm/test/tools/llvm-objdump/X86/elf-disassemble-symbol-references.yaml
+++ b/llvm/test/tools/llvm-objdump/X86/elf-disassemble-symbol-references.yaml
@@ -3,9 +3,9 @@
 # RUN: llvm-objdump %t -d | FileCheck %s --check-prefix=EXEC
 
 # EXEC: Disassembly of section .text1:
-# EXEC:     4000: e8 00 00 00 00                callq   0 <third>
+# EXEC:     4000: e8 00 00 00 00                callq   0x4005 <third>
 # EXEC: Disassembly of section .text2:
-# EXEC:     4005: e8 12 34 56 78                callq   2018915346 <fourth+0x78563412>
+# EXEC:     4005: e8 12 34 56 78                callq   0x7856741c <fourth+0x78563412>
 
 --- !ELF
 FileHeader:
@@ -48,17 +48,17 @@ Symbols:
 # REL:      Disassembly of section .text1:
 # REL-EMPTY:
 # REL-NEXT: 0000000000000000 <.text1>:
-# REL-NEXT:        0: e8 00 00 00 00                callq   0 <.text1+0x5>
+# REL-NEXT:        0: e8 00 00 00 00                callq   0x5 <.text1+0x5>
 # REL-EMPTY:
 # REL-NEXT: Disassembly of section .text2:
 # REL-EMPTY:
 # REL-NEXT: 0000000000000000 <.text2>:
-# REL-NEXT:        0: e8 00 00 00 00                callq   0 <sym2>
+# REL-NEXT:        0: e8 00 00 00 00                callq   0x5 <sym2>
 # REL-EMPTY:
 # REL-NEXT: Disassembly of section .text3:
 # REL-EMPTY:
 # REL-NEXT: 0000000000000000 <.text3>:
-# REL-NEXT:        0: e8 00 00 00 00                callq   0 <.text3+0x5>
+# REL-NEXT:        0: e8 00 00 00 00                callq   0x5 <.text3+0x5>
 
 --- !ELF
 FileHeader:
diff --git a/llvm/test/tools/llvm-objdump/X86/section-filter-relocs.test b/llvm/test/tools/llvm-objdump/X86/section-filter-relocs.test
index a00e34ee7510f..002bd9349d163 100644
--- a/llvm/test/tools/llvm-objdump/X86/section-filter-relocs.test
+++ b/llvm/test/tools/llvm-objdump/X86/section-filter-relocs.test
@@ -10,7 +10,7 @@
 # DISASM:       Disassembly of section .text:
 # DISASM-EMPTY:
 # DISASM-NEXT:  0000000000000400 <.text>:
-# DISASM-NEXT:  400: e8 00 00 00 00                callq   0 <.text+0x5>
+# DISASM-NEXT:  400: e8 00 00 00 00                callq   0x405 <.text+0x5>
 # RELOC-NEXT:                      00000401:  R_X86_64_PC32        foo+0x1
 # RELOC-NEXT:                      00000401:  R_X86_64_GOT32       foo
 # DISASM:       Disassembly of section .rodata:
diff --git a/llvm/test/tools/llvm-objdump/XCOFF/disassemble-all.test b/llvm/test/tools/llvm-objdump/XCOFF/disassemble-all.test
index 195d4f72eb655..1dee2aa2d52a1 100644
--- a/llvm/test/tools/llvm-objdump/XCOFF/disassemble-all.test
+++ b/llvm/test/tools/llvm-objdump/XCOFF/disassemble-all.test
@@ -1,22 +1,28 @@
+# REQUIRES: powerpc-registered-target
+
 # RUN: llvm-objdump -D %p/Inputs/xcoff-section-headers.o | \
-# RUN: FileCheck %s
+# RUN:   FileCheck --check-prefixes=CHECK,ONLY-D %s
+
+# RUN: llvm-objdump -D -r %p/Inputs/xcoff-section-headers.o | \
+# RUN:   FileCheck --check-prefixes=CHECK,WITH-R %s
+
+## xcoff-section-headers.o Compiled with IBM XL C/C++ for AIX, V16.1.0
+## compiler command: xlc -qtls -o xcoff-section-headers.o -c test.c
+## test.c:
+## int a;
+## int b = 12345;
+## __thread int c;
+## __thread double d = 3.14159;
+##
+## int func(void)  {
+##   return a;
+## }
 
-# xcoff-section-headers.o Compiled with IBM XL C/C++ for AIX, V16.1.0
-# compiler command: xlc -qtls -o xcoff-section-headers.o -c test.c
-# test.c:
-# int a;
-# int b = 12345;
-# __thread int c;
-# __thread double d = 3.14159;
-#
-# int func(void)  {
-#   return a;
-# }
-; REQUIRES: powerpc-registered-target
-CHECK: Inputs/xcoff-section-headers.o:	file format aixcoff-rs6000
-CHECK: Disassembly of section .text:
-CHECK: 00000000 <.text>:
+CHECK:        Inputs/xcoff-section-headers.o:	file format aixcoff-rs6000
+CHECK:        Disassembly of section .text:
+CHECK:        00000000 <.text>:
 CHECK-NEXT:        0: 80 62 00 04                  	lwz 3, 4(2)
+WITH-R-NEXT:                         00000002:  R_TOC        a
 CHECK-NEXT:        4: 80 63 00 00                  	lwz 3, 0(3)
 CHECK-NEXT:        8: 4e 80 00 20                  	blr
 CHECK-NEXT:        c: 00 00 00 00                  	<unknown>
@@ -26,30 +32,31 @@ CHECK-NEXT:       18: 00 00 00 0c                  	<unknown>
 CHECK-NEXT:       1c: 00 04 66 75                  	<unknown>
 CHECK-NEXT:       20: 6e 63 00 00                  	xoris 3, 19, 0
 CHECK-NEXT: 		...
-CHECK: Disassembly of section .data:
-CHECK: 00000080 <func>:
+CHECK:        Disassembly of section .data:
+CHECK:        00000080 <func>:
 CHECK-NEXT:       80: 00 00 00 94                  	<unknown>
-CHECK: 00000084 <a>:
+CHECK:        00000084 <a>:
 CHECK-NEXT:       84: 00 00 00 a4                  	<unknown>
-CHECK: 00000088 <b>:
+CHECK:        00000088 <b>:
 CHECK-NEXT:       88: 00 00 00 a0                  	<unknown>
-CHECK: 0000008c <c>:
+CHECK:        0000008c <c>:
 CHECK-NEXT:       8c: 00 00 00 08                  	<unknown>
-CHECK: 00000090 <d>:
-CHECK-NEXT:       90: 00 00 00 00                  	<unknown>
-CHECK: 00000094 <func>:
+CHECK:        00000090 <d>:
+ONLY-D-NEXT:      90: 00 00 00 00                  	<unknown>
+WITH-R-NEXT:      ...
+CHECK:        00000094 <func>:
 CHECK-NEXT:       94: 00 00 00 00                  	<unknown>
 CHECK-NEXT:       98: 00 00 00 80                  	<unknown>
 CHECK-NEXT:       9c: 00 00 00 00                  	<unknown>
-CHECK: 000000a0 <b>:
+CHECK:        000000a0 <b>:
 CHECK-NEXT:       a0: 00 00 30 39                  	<unknown>
-CHECK: Disassembly of section .bss:
-CHECK: 000000a4 <a>:
-CHECK-NEXT: ...
-CHECK: Disassembly of section .tdata:
-CHECK: 00000000 <d>:
-CHECK-NEXT:        0: 40 09 21 f9                  	bdnzfl	9, .+8696
+CHECK:        Disassembly of section .bss:
+CHECK:        000000a4 <a>:
+CHECK-NEXT:        ...
+CHECK:        Disassembly of section .tdata:
+CHECK:        00000000 <d>:
+CHECK-NEXT:        0: 40 09 21 f9                  	bdnzfl	9, 0x21f8
 CHECK-NEXT:        4: f0 1b 86 6e                  	<unknown>
-CHECK: Disassembly of section .tbss:
-CHECK: 00000008 <c>:
-CHECK-NEXT: ...
+CHECK:        Disassembly of section .tbss:
+CHECK:        00000008 <c>:
+CHECK-NEXT:        ...
diff --git a/llvm/test/tools/llvm-objdump/XCOFF/print-reloc.test b/llvm/test/tools/llvm-objdump/XCOFF/print-reloc.test
new file mode 100644
index 0000000000000..ee821360e5cbb
--- /dev/null
+++ b/llvm/test/tools/llvm-objdump/XCOFF/print-reloc.test
@@ -0,0 +1,31 @@
+# REQUIRES: powerpc-registered-target
+
+# RUN: llvm-objdump -r %p/Inputs/xcoff-section-headers.o | \
+# RUN:   FileCheck --match-full-lines --strict-whitespace %s
+
+## xcoff-section-headers.o Compiled with IBM XL C/C++ for AIX, V16.1.0
+## compiler command: xlc -qtls -o xcoff-section-headers.o -c test.c
+## test.c:
+## int a;
+## int b = 12345;
+## __thread int c;
+## __thread double d = 3.14159;
+##
+## int func(void)  {
+##   return a;
+## }
+
+      CHECK:RELOCATION RECORDS FOR [.text]:
+ CHECK-NEXT:OFFSET   TYPE                     VALUE
+ CHECK-NEXT:00000002 R_TOC                    a
+CHECK-EMPTY:
+      CHECK:RELOCATION RECORDS FOR [.data]:
+ CHECK-NEXT:OFFSET   TYPE                     VALUE
+ CHECK-NEXT:00000000 R_POS                    func
+ CHECK-NEXT:00000004 R_POS                    a
+ CHECK-NEXT:00000008 R_POS                    b
+ CHECK-NEXT:0000000c R_TLS                    c
+ CHECK-NEXT:00000010 R_TLS                    d
+ CHECK-NEXT:00000014 R_POS                    .func
+ CHECK-NEXT:00000018 R_POS                    TOC
+CHECK-EMPTY:
diff --git a/llvm/test/tools/llvm-profdata/roundtrip.test b/llvm/test/tools/llvm-profdata/roundtrip.test
index 7af76e0a58224..ce20d633a74ac 100644
--- a/llvm/test/tools/llvm-profdata/roundtrip.test
+++ b/llvm/test/tools/llvm-profdata/roundtrip.test
@@ -16,3 +16,11 @@ RUN: llvm-profdata merge --sample --binary -output=%t.4.profdata %S/Inputs/sampl
 RUN: llvm-profdata merge --sample --extbinary -output=%t.5.profdata %t.4.profdata
 RUN: llvm-profdata merge --sample --text -output=%t.4.proftext %t.5.profdata
 RUN: diff -b %t.4.proftext %S/Inputs/sample-profile.proftext
+# Trip from text --> extbinary --> md5text
+# Trip from text --> compbinary --> md5text
+# Compare the two md5 texts
+RUN: llvm-profdata merge --sample --compbinary -output=%t.6.profdata %S/Inputs/sample-profile.proftext
+RUN: llvm-profdata merge --sample --text -output=%t.6.proftext %S/Inputs/sample-profile.proftext
+RUN: llvm-profdata merge --sample --extbinary -use-md5 -output=%t.7.profdata %S/Inputs/sample-profile.proftext
+RUN: llvm-profdata merge --sample --text -output=%t.7.proftext %S/Inputs/sample-profile.proftext
+RUN: diff -b %t.6.proftext %t.7.proftext
diff --git a/llvm/test/tools/llvm-rc/Inputs/tag-menu.rc b/llvm/test/tools/llvm-rc/Inputs/tag-menu.rc
index 33ba0d5026891..cce9bc2970317 100644
--- a/llvm/test/tools/llvm-rc/Inputs/tag-menu.rc
+++ b/llvm/test/tools/llvm-rc/Inputs/tag-menu.rc
@@ -54,7 +54,7 @@ BEGIN
     MENUITEM "a&bcde", 2
     MENUITEM "ab&cde", 3
     MENUITEM "abc&de", 4
-    MENUITEM "abcd&e", 5
+    MENUITEM "abcd&e", -1
   }
 END
 
diff --git a/llvm/test/tools/llvm-rc/tag-menu.test b/llvm/test/tools/llvm-rc/tag-menu.test
index 5c73d0ea198b8..58c856b700f8b 100644
--- a/llvm/test/tools/llvm-rc/tag-menu.test
+++ b/llvm/test/tools/llvm-rc/tag-menu.test
@@ -69,7 +69,7 @@
 ; MENU-NEXT:   0060: 61002600 62006300 64006500 00000000  |a.&.b.c.d.e.....|
 ; MENU-NEXT:   0070: 03006100 62002600 63006400 65000000  |..a.b.&.c.d.e...|
 ; MENU-NEXT:   0080: 00000400 61006200 63002600 64006500  |....a.b.c.&.d.e.|
-; MENU-NEXT:   0090: 00008000 05006100 62006300 64002600  |......a.b.c.d.&.|
+; MENU-NEXT:   0090: 00008000 FFFF6100 62006300 64002600  |......a.b.c.d.&.|
 ; MENU-NEXT:   00A0: 65000000                             |e...|
 ; MENU-NEXT: )
 
diff --git a/llvm/test/tools/llvm-readobj/ELF/RISCV/attribute.s b/llvm/test/tools/llvm-readobj/ELF/RISCV/attribute.s
new file mode 100644
index 0000000000000..5ae7bfcb0290e
--- /dev/null
+++ b/llvm/test/tools/llvm-readobj/ELF/RISCV/attribute.s
@@ -0,0 +1,44 @@
+## Test llvm-readobj & llvm-readelf can decode RISC-V attributes correctly.
+
+# RUN: llvm-mc -triple riscv32 -filetype obj -o %t.rv32.o %s
+# RUN: llvm-mc -triple riscv64 -filetype obj -o %t.rv64.o %s
+# RUN: llvm-readobj --arch-specific %t.rv32.o \
+# RUN:   | FileCheck %s --check-prefix=CHECK-OBJ
+# RUN: llvm-readelf -A %t.rv32.o \
+# RUN:   | FileCheck %s --check-prefix=CHECK-OBJ
+# RUN: llvm-readobj --arch-specific %t.rv64.o \
+# RUN:   | FileCheck %s --check-prefix=CHECK-OBJ
+# RUN: llvm-readelf -A %t.rv64.o \
+# RUN:   | FileCheck %s --check-prefix=CHECK-OBJ
+
+.attribute  Tag_stack_align, 16
+# CHECK-OBJ:      Tag: 4
+# CHECK-OBJ-NEXT: Value: 16
+# CHECK-OBJ-NEXT: TagName: stack_align
+# CHECK-OBJ-NEXT: Description: Stack alignment is 16-bytes
+
+.attribute  Tag_arch, "rv32i2p0_m2p0_a2p0_c2p0"
+# CHECK-OBJ:      Tag: 5
+# CHECK-OBJ-NEXT: TagName: arch
+# CHECK-OBJ-NEXT: Value: rv32i2p0_m2p0_a2p0_c2p0
+
+.attribute  Tag_unaligned_access, 0
+# CHECK-OBJ:      Tag: 6
+# CHECK-OBJ-NEXT: Value: 0
+# CHECK-OBJ-NEXT: TagName: unaligned_access
+# CHECK-OBJ-NEXT: Description: No unaligned access
+
+.attribute  Tag_priv_spec, 2
+# CHECK-OBJ:      Tag: 8
+# CHECK-OBJ-NEXT: TagName: priv_spec
+# CHECK-OBJ-NEXT: Value: 2
+
+.attribute  Tag_priv_spec_minor, 0
+# CHECK-OBJ:      Tag: 10
+# CHECK-OBJ-NEXT: TagName: priv_spec_minor
+# CHECK-OBJ-NEXT: Value: 0
+
+.attribute  Tag_priv_spec_revision, 0
+# CHECK-OBJ:      Tag: 12
+# CHECK-OBJ-NEXT: TagName: priv_spec_revision
+# CHECK-OBJ-NEXT: Value: 0
diff --git a/llvm/test/tools/llvm-readobj/ELF/RISCV/invalid-attr-section-size.test b/llvm/test/tools/llvm-readobj/ELF/RISCV/invalid-attr-section-size.test
new file mode 100644
index 0000000000000..524134e1579b8
--- /dev/null
+++ b/llvm/test/tools/llvm-readobj/ELF/RISCV/invalid-attr-section-size.test
@@ -0,0 +1,20 @@
+## This test case is used to ensure the error code is caught by llvm-readobj.
+
+# RUN: yaml2obj %s -D BITS=32 -o %t.32.o
+# RUN: llvm-readobj -A %t.32.o 2>&1 | FileCheck -DFILE=%t %s
+# RUN: yaml2obj %s -D BITS=64 -o %t.64.o
+# RUN: llvm-readobj -A %t.64.o 2>&1 | FileCheck -DFILE=%t %s
+
+# CHECK: warning: '[[FILE]].{{32|64}}.o': invalid section length 0 at offset 0x1
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS[[BITS]]
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_RISCV
+Sections:
+  - Name:    .riscv.attributes
+    Type:    SHT_RISCV_ATTRIBUTES
+## Version: 'A'(0x41), section length: 0
+    Content: 4100000000
diff --git a/llvm/test/tools/llvm-readobj/ELF/RISCV/invalid-attr-version.test b/llvm/test/tools/llvm-readobj/ELF/RISCV/invalid-attr-version.test
new file mode 100644
index 0000000000000..9a4d81bcc4f14
--- /dev/null
+++ b/llvm/test/tools/llvm-readobj/ELF/RISCV/invalid-attr-version.test
@@ -0,0 +1,21 @@
+## This test case is used to ensure llvm-readobj checks the version of
+## attribute sections correctly.
+
+# RUN: yaml2obj %s -D BITS=32 -o %t.32.o
+# RUN: llvm-readobj -A %t.32.o 2>&1 | FileCheck -DFILE=%t %s
+# RUN: yaml2obj %s -D BITS=64 -o %t.64.o
+# RUN: llvm-readobj -A %t.64.o 2>&1 | FileCheck -DFILE=%t %s
+
+# CHECK: warning: '[[FILE]].{{32|64}}.o': unrecognised FormatVersion: 0x42
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS[[BITS]]
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_RISCV
+Sections:
+  - Name:    .riscv.attributes
+    Type:    SHT_RISCV_ATTRIBUTES
+## Version: 'B'
+    Content: 42
diff --git a/llvm/test/tools/llvm-readobj/ELF/RISCV/lit.local.cfg b/llvm/test/tools/llvm-readobj/ELF/RISCV/lit.local.cfg
new file mode 100644
index 0000000000000..c63820126f8ca
--- /dev/null
+++ b/llvm/test/tools/llvm-readobj/ELF/RISCV/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'RISCV' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/tools/llvm-readobj/ELF/RISCV/section-types.test b/llvm/test/tools/llvm-readobj/ELF/RISCV/section-types.test
new file mode 100644
index 0000000000000..d1f35306afb25
--- /dev/null
+++ b/llvm/test/tools/llvm-readobj/ELF/RISCV/section-types.test
@@ -0,0 +1,21 @@
+## Show that all RISCV specific section types are properly printed for both
+## LLVM and GNU styles.
+
+# RUN: yaml2obj %s -o %t-riscv.o
+# RUN: llvm-readobj --section-headers %t-riscv.o | FileCheck %s --check-prefix=LLVM
+# RUN: llvm-readelf --section-headers %t-riscv.o | FileCheck %s --check-prefix=GNU
+
+# LLVM:      Name: .riscv.attributes (1)
+# LLVM-NEXT: Type: SHT_RISCV_ATTRIBUTES (0x70000003)
+
+# GNU: [ 1] .riscv.attributes RISCV_ATTRIBUTES
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_RISCV
+Sections:
+  - Name: .riscv.attributes
+    Type: SHT_RISCV_ATTRIBUTES
diff --git a/llvm/test/tools/llvm-readobj/ELF/RISCV/validate-attr-section.test b/llvm/test/tools/llvm-readobj/ELF/RISCV/validate-attr-section.test
new file mode 100644
index 0000000000000..66a5a7a8d31f3
--- /dev/null
+++ b/llvm/test/tools/llvm-readobj/ELF/RISCV/validate-attr-section.test
@@ -0,0 +1,17 @@
+## We only implement attribute section printing for little-endian encoding.
+
+# RUN: yaml2obj %s -o %t.o
+# RUN: llvm-readobj -A %t.o | FileCheck %s
+
+# CHECK: Attributes not implemented.
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+## Test big-endian encoding.
+  Data:    ELFDATA2MSB
+  Type:    ET_REL
+  Machine: EM_RISCV
+Sections:
+  - Name: .riscv.attributes
+    Type: SHT_RISCV_ATTRIBUTES
diff --git a/llvm/test/tools/llvm-readobj/ELF/hash-histogram.test b/llvm/test/tools/llvm-readobj/ELF/hash-histogram.test
index fd4de9ab9384e..f7551b481a863 100644
--- a/llvm/test/tools/llvm-readobj/ELF/hash-histogram.test
+++ b/llvm/test/tools/llvm-readobj/ELF/hash-histogram.test
@@ -1,20 +1,30 @@
 ## Here we test the --elf-hash-histogram command line option.
 
-## This test case checks how we built a histogram for a GNU hash section.
-## We check both 32-bit and 64-bit inputs.
+## This test case checks how we built histograms for hash sections.
 
 # RUN: yaml2obj --docnum=1 -D BITS=32 %s -o %t1-32.o
-# RUN: llvm-readelf --elf-hash-histogram %t1-32.o | FileCheck %s --check-prefix=GNU-HASH
+# RUN: llvm-readelf --elf-hash-histogram %t1-32.o | FileCheck %s --check-prefix=HIST
+
+## Test --histogram and -I aliases.
+# RUN: llvm-readelf --histogram %t1-32.o | FileCheck %s --check-prefix=HIST
+# RUN: llvm-readelf -I %t1-32.o | FileCheck %s --check-prefix=HIST
 
 # RUN: yaml2obj --docnum=1 -D BITS=64 %s -o %t1-64.o
-# RUN: llvm-readelf --elf-hash-histogram %t1-64.o | FileCheck %s --check-prefix=GNU-HASH
+# RUN: llvm-readelf --elf-hash-histogram %t1-64.o | FileCheck %s --check-prefix=HIST
 
-# GNU-HASH:      Histogram for `.gnu.hash' bucket list length (total of 3 buckets)
-# GNU-HASH-NEXT:  Length  Number     % of total  Coverage
-# GNU-HASH-NEXT:       0  1          ( 33.3%)       0.0%
-# GNU-HASH-NEXT:       1  1          ( 33.3%)      25.0%
-# GNU-HASH-NEXT:       2  0          (  0.0%)      25.0%
-# GNU-HASH-NEXT:       3  1          ( 33.3%)     100.0%
+# HIST:      Histogram for bucket list length (total of 3 buckets)
+# HIST-NEXT:  Length  Number     % of total  Coverage
+# HIST-NEXT:       0  2          ( 66.7%)       0.0%
+# HIST-NEXT:       1  0          (  0.0%)       0.0%
+# HIST-NEXT:       2  0          (  0.0%)       0.0%
+# HIST-NEXT:       3  1          ( 33.3%)     100.0%
+# HIST-NEXT: Histogram for `.gnu.hash' bucket list length (total of 3 buckets)
+# HIST-NEXT:  Length  Number     % of total  Coverage
+# HIST-NEXT:       0  1          ( 33.3%)       0.0%
+# HIST-NEXT:       1  1          ( 33.3%)      25.0%
+# HIST-NEXT:       2  0          (  0.0%)      25.0%
+# HIST-NEXT:       3  1          ( 33.3%)     100.0%
+# HIST-NOT:  {{.}}
 
 --- !ELF
 FileHeader:
@@ -23,6 +33,11 @@ FileHeader:
   Type:    ET_DYN
   Machine: EM_386
 Sections:
+  - Name:   .hash
+    Type:   SHT_HASH
+    Flags:  [ SHF_ALLOC ]
+    Bucket: [ 6, 4, 5 ]
+    Chain:  [ 0, 0, 1, 0, 2 ]
   - Name:  .gnu.hash
     Type:  SHT_GNU_HASH
     Flags: [ SHF_ALLOC ]
@@ -36,8 +51,11 @@ Sections:
     Type:  SHT_DYNAMIC
     Flags: [ SHF_WRITE, SHF_ALLOC ]
     Entries:
-      - Tag:   DT_GNU_HASH
+      - Tag:   DT_HASH
         Value: 0x0
+      - Tag:   DT_GNU_HASH
+## sizeof(.hash) == 0x28.
+        Value: 0x28
       - Tag:   DT_NULL
         Value: 0x0
 DynamicSymbols:
@@ -48,6 +66,7 @@ DynamicSymbols:
 ProgramHeaders:
   - Type:  PT_LOAD
     Sections:
+      - Section: .hash
       - Section: .gnu.hash
       - Section: .dynamic
 
@@ -93,3 +112,98 @@ ProgramHeaders:
     Sections:
       - Section: .hash
       - Section: .dynamic
+
+## Each SHT_HASH section starts with two 32-bit fields: nbucket and nchain.
+## Check we report an error when a DT_HASH value points to data that has size less than 8 bytes.
+
+# RUN: yaml2obj --docnum=3 %s -o %t3.o
+# RUN: llvm-readelf --elf-hash-histogram %t3.o 2>&1 | FileCheck %s --check-prefix=ERR1 -DFILE=%t3.o
+
+# ERR1: warning: '[[FILE]]': the hash table at offset 0x2b1 goes past the end of the file (0x2b8){{$}}
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_X86_64
+Sections:
+  - Name:   .hash
+    Type:   SHT_HASH
+    Flags:  [ SHF_ALLOC ]
+    Bucket: [ 0 ]
+    Chain:  [ 0 ]
+  - Name:  .dynamic
+    Type:  SHT_DYNAMIC
+    Flags: [ SHF_WRITE, SHF_ALLOC ]
+    Entries:
+      - Tag:   DT_HASH
+        Value: 0x239
+      - Tag:   DT_NULL
+        Value: 0x0
+DynamicSymbols: []
+ProgramHeaders:
+  - Type:     PT_LOAD
+    FileSize: 0x23a
+    Sections:
+      - Section: .hash
+      - Section: .dynamic
+
+## Check we report a warning when the hash table goes past the end of the file.
+
+## Case A.1: the hash table ends right before the EOF. We have a broken nbucket
+##           field that has a value larger than the number of buckets.
+# RUN: yaml2obj --docnum=4 %s -o %t4.1.o -DNBUCKET=0x5d -DNCHAIN=0x1
+# RUN: llvm-readelf --elf-hash-histogram %t4.1.o 2>&1 | \
+# RUN:   FileCheck %s --implicit-check-not={{.}} --allow-empty
+
+## Case A.2: the hash table ends 1 byte past the EOF. We have a broken nbucket
+##           field that has a value larger than the number of buckets.
+# RUN: yaml2obj --docnum=4 %s -o %t4.2.o -DNBUCKET=0x5e -DNCHAIN=0x1
+# RUN: llvm-readelf --elf-hash-histogram %t4.2.o 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=ERR2 -DFILE=%t4.2.o --implicit-check-not="warning:"
+# ERR2: warning: '[[FILE]]': the hash table at offset 0x54 goes past the end of the file (0x1d4), nbucket = 94, nchain = 1{{$}}
+
+## Case B.1: the hash table ends right before the EOF. We have a broken nchain
+##           field that has a value larger than the number of chains.
+# RUN: yaml2obj --docnum=4 %s -o %t4.3.o -DNBUCKET=0x1 -DNCHAIN=0x5d
+# RUN: llvm-readelf --elf-hash-histogram %t4.3.o 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=ERR3 -DFILE=%t4.3.o --implicit-check-not="warning:"
+# ERR3: warning: '[[FILE]]': hash table nchain (93) differs from symbol count derived from SHT_DYNSYM section header (1){{$}}
+
+## Case B.2: the hash table ends 1 byte past the EOF. We have a broken nchain
+##           field that has a value larger than the number of chains.
+# RUN: yaml2obj --docnum=4 %s -o %t4.4.o -DNBUCKET=0x1 -DNCHAIN=0x5e
+# RUN: llvm-readelf --elf-hash-histogram %t4.4.o 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=ERR4 -DFILE=%t4.4.o --implicit-check-not="warning:"
+# ERR4: warning: '[[FILE]]': hash table nchain (94) differs from symbol count derived from SHT_DYNSYM section header (1){{$}}
+# ERR4: warning: '[[FILE]]': the hash table at offset 0x54 goes past the end of the file (0x1d4), nbucket = 1, nchain = 94{{$}}
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS32
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_X86_64
+Sections:
+  - Name:    .hash
+    Type:    SHT_HASH
+    Flags:   [ SHF_ALLOC ]
+    Bucket:  [ 0 ]
+    NBucket: [[NBUCKET]]
+    Chain:   [ 0 ]
+    NChain:  [[NCHAIN]]
+  - Name:  .dynamic
+    Type:  SHT_DYNAMIC
+    Flags: [ SHF_WRITE, SHF_ALLOC ]
+    Entries:
+      - Tag:   DT_HASH
+        Value: 0x0
+      - Tag:   DT_NULL
+        Value: 0x0
+DynamicSymbols: []
+ProgramHeaders:
+  - Type: PT_LOAD
+    Sections:
+      - Section: .hash
+      - Section: .dynamic
diff --git a/llvm/test/tools/llvm-readobj/ELF/hash-symbols.test b/llvm/test/tools/llvm-readobj/ELF/hash-symbols.test
index 0348444b82224..40e680e3b7511 100644
--- a/llvm/test/tools/llvm-readobj/ELF/hash-symbols.test
+++ b/llvm/test/tools/llvm-readobj/ELF/hash-symbols.test
@@ -2,10 +2,13 @@
 
 ## Check the output when both .hash and .gnu.hash sections are present.
 
-# RUN: yaml2obj --docnum=1 %s -o %t1-32.so
+# RUN: yaml2obj --docnum=1 -DBITS=32 %s -o %t1-32.so
 # RUN: llvm-readelf --hash-symbols %t1-32.so \
 # RUN:   | FileCheck %s --strict-whitespace --match-full-lines --check-prefix HASH-32
 
+# RUN: yaml2obj --docnum=1 -DBITS=64 %s -o %t1-64.so
+# RUN: llvm-readelf --hash-symbols %t1-64.so | FileCheck %s --check-prefix HASH-64
+
 # HASH-32: Symbol table of .hash for image:
 # HASH-32-NEXT:  Num Buc:    Value  Size   Type   Bind Vis      Ndx Name
 # HASH-32-NEXT:    1   0: 00000000     0 NOTYPE  GLOBAL DEFAULT UND ccc
@@ -22,75 +25,6 @@
 # HASH-32-NEXT:    5   2: 00001001     0 NOTYPE  WEAK   DEFAULT   1 bbb
 # HASH-32-NOT: {{.}}
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS32
-  Data:    ELFDATA2LSB
-  Type:    ET_DYN
-  Machine: EM_386
-Sections:
-  - Name:    .hash
-    Type:    SHT_HASH
-    Flags:   [ SHF_ALLOC ]
-    Link:    .dynsym
-    Bucket:  [ 1, 0, 0 ]
-    Chain:   [ 0, 5, 4, 2, 0, 3 ]
-  - Name:    .gnu.hash
-    Type:    SHT_GNU_HASH
-    Flags:   [ SHF_ALLOC ]
-    Link:    .dynsym
-    Header:
-      SymNdx: 0x2
-      Shift2: 0x0
-    BloomFilter: [ 0x0 ]
-    HashBuckets: [ 0x0, 0x2, 0x4 ]
-    HashValues:  [ 0x0B885C68, 0x0B886991, 0x0B886DF4, 0x0B8860CB ]
-  - Name:  .dynamic
-    Type:  SHT_DYNAMIC
-    Flags: [ SHF_ALLOC ]
-    Link:  .dynstr
-    Entries:
-## PT_LOAD's p_vaddr is 0x0. PT_LOAD's p_offset = 0x1bc. DT_HASH value is 0x0. 
-## llvm-readelf will read .hash content from p_offset + (p_vaddr - DT_HASH value) = 0x1bc.
-## This matches the file offset of the .hash section.
-      - Tag:   DT_HASH
-        Value: 0x0000000000000000
-      - Tag:   DT_GNU_HASH
-## PT_LOAD's p_vaddr is 0x0. PT_LOAD's p_offset = 0x1bc. DT_GNU_HASH value is 0x2c (size of .hash = 0x2c).
-## llvm-readelf will read .gnu.hash content from p_offset + (p_vaddr - DT_GNU_HASH value) = 0x1e8.
-## This matches the file offset of the .gnu.hash section.
-        Value: 0x000000000000002C
-      - Tag:   DT_NULL
-        Value: 0x0000000000000000
-DynamicSymbols:
-  - Name:    ccc
-    Binding: STB_GLOBAL
-  - Name:    aaa
-    Section: .hash
-    Binding: STB_GLOBAL
-    Value:   0x0000000000001000
-  - Name:    ddd
-    Index:   SHN_ABS
-    Binding: STB_GLOBAL
-    Value:   0x0000000000000001
-  - Name:    eee
-    Section: .gnu.hash
-    Binding: STB_GLOBAL
-  - Name:    bbb
-    Section: .hash
-    Binding: STB_WEAK
-    Value:   0x0000000000001001
-ProgramHeaders:
-  - Type:  PT_LOAD
-    Flags: [ PF_R, PF_X ]
-    Sections:
-      - Section: .hash
-      - Section: .gnu.hash
-      - Section: .dynamic
-
-# RUN: yaml2obj --docnum=2 %s -o %t1-64.so
-# RUN: llvm-readelf --hash-symbols %t1-64.so | FileCheck %s --check-prefix HASH-64
-
 # HASH-64:      Symbol table of .hash for image:
 # HASH-64-NEXT:  Num Buc:    Value          Size   Type   Bind Vis      Ndx Name
 # HASH-64-NEXT:    1   0: 0000000000000000     0 NOTYPE  GLOBAL DEFAULT UND ccc
@@ -109,7 +43,7 @@ ProgramHeaders:
 
 --- !ELF
 FileHeader:
-  Class:   ELFCLASS64
+  Class:   ELFCLASS[[BITS]]
   Data:    ELFDATA2LSB
   Type:    ET_DYN
   Machine: EM_X86_64
@@ -128,23 +62,23 @@ Sections:
       SymNdx: 0x2
       Shift2: 0x0
     BloomFilter: [ 0x0 ]
-    HashBuckets: [ 0x00000000, 0x00000002, 0x00000004 ]
+    HashBuckets: [ 0x0, 0x2, 0x4 ]
     HashValues:  [ 0x0B885C68, 0x0B886991, 0x0B886DF4, 0x0B8860CB ]
   - Name:  .dynamic
     Type:  SHT_DYNAMIC
     Flags: [ SHF_ALLOC ]
     Link:  .dynstr
     Entries:
+## PT_LOAD's p_vaddr is 0x0. DT_HASH value is 0x0. 
+## llvm-readelf will read .hash content from PT_LOAD's p_offset + (DT_HASH value - p_vaddr).
+## This matches the file offset of the .hash section.
       - Tag:   DT_HASH
-## PT_LOAD's p_vaddr is 0x0. PT_LOAD's p_offset = 0x2b8. DT_HASH value is 0x0. 
-## llvm-readelf will read .hash content from p_offset + (p_vaddr - DT_HASH value) = 0x2b8.
-## This matches the file offset of the .hash section. 
         Value: 0x0000000000000000
       - Tag:   DT_GNU_HASH
-## PT_LOAD's p_vaddr is 0x0. PT_LOAD's p_offset = 0x2b8. DT_GNU_HASH value is 0x2c (size of .hash = 0x2c).
-## llvm-readelf will read .gnu.hash content from p_offset + (p_vaddr - DT_GNU_HASH value) = 0x2e4.
+## PT_LOAD's p_vaddr is 0x0. DT_GNU_HASH value is 0x2c (size of .hash = 0x2c).
+## llvm-readelf will read .gnu.hash content from PT_LOAD's p_offset + (DT_GNU_HASH value - p_vaddr).
 ## This matches the file offset of the .gnu.hash section.
-        Value: 0x000000000000002c
+        Value: 0x000000000000002C
       - Tag:   DT_NULL
         Value: 0x0000000000000000
 DynamicSymbols:
@@ -175,7 +109,7 @@ ProgramHeaders:
 
 ## Check the output when only .hash section is present.
 
-# RUN: yaml2obj --docnum=3 %s -o %t2-32.so
+# RUN: yaml2obj --docnum=2 %s -o %t2-32.so
 # RUN: llvm-readelf --hash-symbols %t2-32.so \
 # RUN:   | FileCheck %s --strict-whitespace --match-full-lines --check-prefix ONLY-HASH-32
 
@@ -237,7 +171,7 @@ ProgramHeaders:
 
 ## Check the output when only .gnu.hash section is present.
 
-# RUN: yaml2obj --docnum=4 %s -o %t3-32.so
+# RUN: yaml2obj --docnum=3 %s -o %t3-32.so
 # RUN: llvm-readelf --hash-symbols %t3-32.so \
 # RUN:   | FileCheck %s --strict-whitespace --match-full-lines --check-prefix ONLY-GNUHASH-32
 
@@ -301,7 +235,7 @@ ProgramHeaders:
       - Section: .dynamic
 
 ## Show that if there are no hash sections, we do not print anything.
-# RUN: yaml2obj --docnum=5 %s -o %t4.so
+# RUN: yaml2obj --docnum=4 %s -o %t4.so
 # RUN: llvm-readelf --hash-symbols %t4.so \
 # RUN:   | FileCheck %s --check-prefix NO-HASH --allow-empty
 
@@ -377,8 +311,8 @@ ProgramHeaders:
 ## Show that we report a warning for a hash table which contains an entry of
 ## the bucket array pointing to a cycle.
 
-# RUN: yaml2obj --docnum=6 %s -o %t6.so
-# RUN: llvm-readelf --hash-symbols %t6.so 2>&1 | FileCheck %s -DFILE=%t6.so --check-prefix=BROKEN
+# RUN: yaml2obj --docnum=5 %s -o %t5.so
+# RUN: llvm-readelf --hash-symbols %t5.so 2>&1 | FileCheck %s -DFILE=%t5.so --check-prefix=BROKEN
 
 # BROKEN:      Symbol table of .hash for image:
 # BROKEN-NEXT:  Num Buc:    Value  Size   Type   Bind Vis      Ndx Name
@@ -413,3 +347,104 @@ ProgramHeaders:
     Sections:
       - Section: .hash
       - Section: .dynamic
+
+## Each SHT_HASH section starts with two 32-bit fields: nbucket and nchain.
+## Check we report an error when a DT_HASH value points to data that has size less than 8 bytes.
+
+# RUN: yaml2obj --docnum=6 %s -o %t6.o
+# RUN: llvm-readelf --hash-symbols %t6.o 2>&1 | FileCheck %s --check-prefix=ERR1 -DFILE=%t6.o
+
+# ERR1: warning: '[[FILE]]': the hash table at offset 0x2b1 goes past the end of the file (0x2b8){{$}}
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_X86_64
+Sections:
+  - Name:   .hash
+    Type:   SHT_HASH
+    Flags:  [ SHF_ALLOC ]
+    Bucket: [ 0 ]
+    Chain:  [ 0 ]
+  - Name:  .dynamic
+    Type:  SHT_DYNAMIC
+    Flags: [ SHF_WRITE, SHF_ALLOC ]
+    Entries:
+      - Tag:   DT_HASH
+        Value: 0x239
+      - Tag:   DT_NULL
+        Value: 0x0
+DynamicSymbols: []
+ProgramHeaders:
+  - Type:     PT_LOAD
+    FileSize: 0x23a
+    Sections:
+      - Section: .hash
+      - Section: .dynamic
+
+## Check we report a warning when the hash table goes past the end of the file.
+
+## Case A.1: the hash table ends right before the EOF. We have a broken nbucket
+##           field that has a value larger than the number of buckets.
+# RUN: yaml2obj --docnum=7 %s -o %t7.1.o -DNBUCKET=0x5d -DNCHAIN=0x1
+# RUN: llvm-readelf --hash-symbols %t7.1.o 2>&1 | FileCheck %s --check-prefix=NOERR1
+# NOERR1:            Symbol table of .hash for image:
+# NOERR1-NEXT:         Num Buc:    Value  Size   Type   Bind Vis      Ndx Name
+# NOERR1-NEXT-EMPTY:
+
+## Case A.2: the hash table ends 1 byte past the EOF. We have a broken nbucket
+##           field that has a value larger than the number of buckets.
+# RUN: yaml2obj --docnum=7 %s -o %t7.2.o -DNBUCKET=0x5e -DNCHAIN=0x1
+# RUN: llvm-readelf --hash-symbols %t7.2.o 2>&1 | FileCheck %s --check-prefix=ERR2 -DFILE=%t7.2.o
+# ERR2:      Symbol table of .hash for image:
+# ERR2-NEXT: warning: '[[FILE]]': the hash table at offset 0x54 goes past the end of the file (0x1d4), nbucket = 94, nchain = 1{{$}}
+# ERR2-NOT:  {{.}}
+
+## Case B.1: the hash table ends right before the EOF. We have a broken nchain
+##           field that has a value larger than the number of chains.
+# RUN: yaml2obj --docnum=7 %s -o %t7.3.o -DNBUCKET=0x1 -DNCHAIN=0x5d
+# RUN: llvm-readelf --hash-symbols %t7.3.o 2>&1 | \
+# RUN:   FileCheck %s --implicit-check-not="warning:" --check-prefix=NOERR2 -DFILE=%t7.3.o
+# NOERR2:      warning: '[[FILE]]': hash table nchain (93) differs from symbol count derived from SHT_DYNSYM section header (1)
+# NOERR2:      Symbol table of .hash for image:
+# NOERR2-NEXT: Num Buc: Value Size Type Bind Vis Ndx Name
+# NOERR2-NOT:  {{.}}
+
+## Case B.2: the hash table ends 1 byte past the EOF. We have a broken nchain
+##           field that has a value larger than the number of chains.
+# RUN: yaml2obj --docnum=7 %s -o %t7.4.o -DNBUCKET=0x1 -DNCHAIN=0x5e
+# RUN: llvm-readelf --hash-symbols %t7.4.o 2>&1 | FileCheck %s --check-prefix=ERR3 -DFILE=%t7.4.o
+# ERR3:      Symbol table of .hash for image:
+# ERR3-NEXT: warning: '[[FILE]]': the hash table at offset 0x54 goes past the end of the file (0x1d4), nbucket = 1, nchain = 94{{$}}
+# ERR3-NOT:  {{.}}
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS32
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_X86_64
+Sections:
+  - Name:    .hash
+    Type:    SHT_HASH
+    Flags:   [ SHF_ALLOC ]
+    Bucket:  [ 0 ]
+    NBucket: [[NBUCKET]]
+    Chain:   [ 0 ]
+    NChain:  [[NCHAIN]]
+  - Name:  .dynamic
+    Type:  SHT_DYNAMIC
+    Flags: [ SHF_WRITE, SHF_ALLOC ]
+    Entries:
+      - Tag:   DT_HASH
+        Value: 0x0
+      - Tag:   DT_NULL
+        Value: 0x0
+DynamicSymbols: []
+ProgramHeaders:
+  - Type: PT_LOAD
+    Sections:
+      - Section: .hash
+      - Section: .dynamic
diff --git a/llvm/test/tools/llvm-readobj/ELF/hash-table.test b/llvm/test/tools/llvm-readobj/ELF/hash-table.test
index 2abfdd01baf23..8cbe615eee22e 100644
--- a/llvm/test/tools/llvm-readobj/ELF/hash-table.test
+++ b/llvm/test/tools/llvm-readobj/ELF/hash-table.test
@@ -115,3 +115,133 @@ ProgramHeaders:
     VAddr: 0x1010
     Sections:
       - Section: .dynamic
+
+## Each SHT_HASH section starts with two 32-bit fields: nbucket and nchain.
+## Check we report an error when a DT_HASH value points to data that has size less than 8 bytes.
+
+# RUN: yaml2obj --docnum=4 %s -o %t4.o
+# RUN: llvm-readelf --hash-table %t4.o 2>&1 | FileCheck %s --check-prefix=ERR1 -DFILE=%t4.o
+# RUN: llvm-readobj --hash-table %t4.o 2>&1 | FileCheck %s --check-prefix=ERR1 -DFILE=%t4.o
+
+# ERR1:      HashTable {
+# ERR1-NEXT:  warning: '[[FILE]]': the hash table at offset 0x2b1 goes past the end of the file (0x2b8){{$}}
+# ERR1-NEXT: }
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_X86_64
+Sections:
+  - Name:   .hash
+    Type:   SHT_HASH
+    Flags:  [ SHF_ALLOC ]
+    Bucket: [ 0 ]
+    Chain:  [ 0 ]
+  - Name:  .dynamic
+    Type:  SHT_DYNAMIC
+    Flags: [ SHF_WRITE, SHF_ALLOC ]
+    Entries:
+      - Tag:   DT_HASH
+        Value: 0x239
+      - Tag:   DT_NULL
+        Value: 0x0
+DynamicSymbols: []
+ProgramHeaders:
+  - Type:     PT_LOAD
+    FileSize: 0x23a
+    Sections:
+      - Section: .hash
+      - Section: .dynamic
+
+## Check we report a warning when the hash table goes past the end of the file.
+
+## Case A.1: the hash table ends right before the EOF. We have a broken nbucket
+##           field that has a value larger than the number of buckets.
+# RUN: yaml2obj --docnum=5 %s -o %t5.1.o -DNBUCKET=0x5d -DNCHAIN=0x1
+# RUN: llvm-readelf --hash-table %t5.1.o 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=NOERR1 --implicit-check-not="warning:"
+# RUN: llvm-readobj --hash-table %t5.1.o 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=NOERR1 --implicit-check-not="warning:"
+
+# NOERR1:      HashTable {
+# NOERR1-NEXT:  Num Buckets: 93
+# NOERR1-NEXT:  Num Chains: 1
+## Here we would dump the rest of the file as buckets array because we have a broken nbucket field.
+## No need to check what we dump, we only want to test that we have no unexpected warnings/crashes.
+# NOERR1-NEXT:  Buckets:
+# NOERR1-NEXT:  Chains: [0]
+# NOERR1-NEXT: }
+
+## Case A.2: the hash table ends 1 byte past the EOF. We have a broken nbucket
+##           field that has a value larger than the number of buckets.
+# RUN: yaml2obj --docnum=5 %s -o %t5.2.o -DNBUCKET=0x5e -DNCHAIN=0x1
+# RUN: llvm-readelf --hash-table %t5.2.o 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=ERR2 -DFILE=%t5.2.o --implicit-check-not="warning:"
+# RUN: llvm-readobj --hash-table %t5.2.o 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=ERR2 -DFILE=%t5.2.o --implicit-check-not="warning:"
+
+# ERR2:      HashTable {
+# ERR2-NEXT:  warning: '[[FILE]]': the hash table at offset 0x54 goes past the end of the file (0x1d4), nbucket = 94, nchain = 1{{$}}
+# ERR2-NEXT: }
+
+## Case B.1: the hash table ends right before the EOF. We have a broken nchain
+##           field that has a value larger than the number of chains.
+# RUN: yaml2obj --docnum=5 %s -o %t5.3.o -DNBUCKET=0x1 -DNCHAIN=0x5d
+# RUN: llvm-readelf --hash-table %t5.3.o 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=NOERR2 -DFILE=%t5.3.o --implicit-check-not="warning:"
+# RUN: llvm-readobj --hash-table %t5.3.o 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=NOERR2 -DFILE=%t5.3.o --implicit-check-not="warning:"
+
+# NOERR2:      warning: '[[FILE]]': hash table nchain (93) differs from symbol count derived from SHT_DYNSYM section header (1)
+# NOERR2:      HashTable {
+# NOERR2-NEXT:   Num Buckets: 1
+# NOERR2-NEXT:   Num Chains: 93
+# NOERR2-NEXT:   Buckets: [0]
+## Here we would dump the rest of the file as chain array because we have a broken nchain field.
+## No need to check what we dump, we only want to test that we have no unexpected warnings/crashes.
+# NOERR2-NEXT:   Chains:
+# NOERR2-NEXT: }
+
+## Case B.2: the hash table ends 1 byte past the EOF. We have a broken nchain
+##           field that has a value larger than the number of chains.
+# RUN: yaml2obj --docnum=5 %s -o %t5.4.o -DNBUCKET=0x1 -DNCHAIN=0x5e
+# RUN: llvm-readelf --hash-table %t5.4.o 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=ERR3 -DFILE=%t5.4.o --implicit-check-not="warning:"
+# RUN: llvm-readobj --hash-table %t5.4.o 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=ERR3 -DFILE=%t5.4.o --implicit-check-not="warning:"
+
+# ERR3:      warning: '[[FILE]]': hash table nchain (94) differs from symbol count derived from SHT_DYNSYM section header (1)
+# ERR3:      HashTable {
+# ERR3-NEXT:  warning: '[[FILE]]': the hash table at offset 0x54 goes past the end of the file (0x1d4), nbucket = 1, nchain = 94{{$}}
+# ERR3-NEXT: }
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS32
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_X86_64
+Sections:
+  - Name:    .hash
+    Type:    SHT_HASH
+    Flags:   [ SHF_ALLOC ]
+    Bucket:  [ 0 ]
+    NBucket: [[NBUCKET]]
+    Chain:   [ 0 ]
+    NChain:  [[NCHAIN]]
+  - Name:  .dynamic
+    Type:  SHT_DYNAMIC
+    Flags: [ SHF_WRITE, SHF_ALLOC ]
+    Entries:
+      - Tag:   DT_HASH
+        Value: 0x0
+      - Tag:   DT_NULL
+        Value: 0x0
+DynamicSymbols: []
+ProgramHeaders:
+  - Type: PT_LOAD
+    Sections:
+      - Section: .hash
+      - Section: .dynamic
diff --git a/llvm/test/tools/llvm-readobj/ELF/loadname.test b/llvm/test/tools/llvm-readobj/ELF/loadname.test
index 7bd64fd27fa7f..a8eaab4b8ad7d 100644
--- a/llvm/test/tools/llvm-readobj/ELF/loadname.test
+++ b/llvm/test/tools/llvm-readobj/ELF/loadname.test
@@ -1,6 +1,8 @@
 ## Check we are able to dump library soname properly.
 
-# RUN: yaml2obj %s -o %t.o
+## Test a valid object case first. We set 'FileSize' to 0x48, because this is a no-op,
+## i.e. this value would be set if we had no 'FileSize' at all.
+# RUN: yaml2obj -DDTSTRTABVAL=0x0 -DPHDRFILESIZE="0x48" %s -o %t.o
 # RUN: llvm-readobj %t.o | FileCheck %s --check-prefix LLVM
 # RUN: llvm-readelf --dynamic-table %t.o | FileCheck %s --check-prefix GNU
 
@@ -33,7 +35,7 @@ Sections:
     Link:  .dynstr
     Entries:
       - Tag:   DT_STRTAB
-        Value: 0x0000000000000000
+        Value: [[DTSTRTABVAL]]
       - Tag:   DT_STRSZ
         Value: 0x0000000000000007
       - Tag:   DT_SONAME
@@ -41,9 +43,23 @@ Sections:
       - Tag:   DT_NULL
         Value: 0x0000000000000000
 ProgramHeaders:
-  - Type:  PT_LOAD
-    Flags: [ PF_R ]
-    VAddr: 0x0000
+  - Type:     PT_LOAD
+    Flags:    [ PF_R ]
+    VAddr:    0x0000
+    FileSize: [[PHDRFILESIZE]]
     Sections:
       - Section: .dynstr
       - Section: .dynamic
+
+## Check we do not crash when an object contains a DT_STRTAB entry whose address
+## is past the end of the object.
+## Note that we have to set p_filesz for PT_LOAD larger than DT_STRTAB value
+## to trigger this particular warning.
+
+# RUN: yaml2obj -DDTSTRTABVAL=0xFFFE -DPHDRFILESIZE=0xFFFF %s -o %t.err.1.o
+# RUN: llvm-readobj %t.err.1.o 2>&1 | FileCheck %s -DFILE=%t.err.1.o --check-prefixes=BROKEN-OFFSET,BROKEN-OFFSET-LLVM
+# RUN: llvm-readelf --dynamic-table %t.err.1.o 2>&1 | FileCheck %s -DFILE=%t.err.1.o --check-prefixes=BROKEN-OFFSET,BROKEN-OFFSET-GNU
+
+# BROKEN-OFFSET:      warning: '[[FILE]]': Unable to parse DT_STRTAB: can't map virtual address 0xfffe to the segment with index 1: the segment ends at 0x10077, which is greater than the file size (0x228)
+# BROKEN-OFFSET-LLVM: LoadName: <String table is empty or was not found>
+# BROKEN-OFFSET-GNU:  0x000000000000000e (SONAME) Library soname: [<String table is empty or was not found>]
diff --git a/llvm/test/tools/llvm-symbolizer/relativenames.s b/llvm/test/tools/llvm-symbolizer/relativenames.s
new file mode 100644
index 0000000000000..96b89cb004bf4
--- /dev/null
+++ b/llvm/test/tools/llvm-symbolizer/relativenames.s
@@ -0,0 +1,18 @@
+# REQUIRES: x86-registered-target
+
+foo:
+    nop
+
+# RUN: llvm-mc --filetype=obj --triple=x86_64-pc-linux %s -o %t.o -g
+
+# RUN: llvm-symbolizer 0 --relativenames --obj=%t.o \
+# RUN:    | FileCheck %s -DDIR=%p --check-prefix=RELATIVENAMES
+
+## Ensure last option wins.
+# RUN: llvm-symbolizer 0 --basenames --relativenames --obj=%t.o \
+# RUN:    | FileCheck %s -DDIR=%p --check-prefix=RELATIVENAMES
+# RUN: llvm-symbolizer 0 --relativenames --basenames --obj=%t.o \
+# RUN:    | FileCheck %s --check-prefix=BASENAMES
+
+# RELATIVENAMES: [[DIR]]{{\\|/}}relativenames.s:4
+# BASENAMES: {{^}}relativenames.s:4
diff --git a/llvm/test/tools/obj2yaml/program-headers.yaml b/llvm/test/tools/obj2yaml/program-headers.yaml
new file mode 100644
index 0000000000000..1c784f17b6e5e
--- /dev/null
+++ b/llvm/test/tools/obj2yaml/program-headers.yaml
@@ -0,0 +1,421 @@
+## Show that obj2yaml is able to dump program headers.
+
+## Part I. Base check. All simple cases that look OK as a part of a single large test live here.
+
+# RUN: yaml2obj %s -o %t1
+
+## Show the layout of the object before we dump it using obj2yaml.
+## The check is here to make it clear what the layout should look like.
+# RUN: llvm-readelf --segments %t1 | FileCheck %s --check-prefix=SEGMENT-MAPPING
+
+# SEGMENT-MAPPING:      Program Headers:
+# SEGMENT-MAPPING-NEXT:   Type      Offset   VirtAddr           PhysAddr           FileSiz  MemSiz   Flg Align
+# SEGMENT-MAPPING-NEXT:   LOAD      0x000000 0x0000000000000000 0x0000000000000000 0x000281 0x000281 R   0x1000
+# SEGMENT-MAPPING-NEXT:   LOAD      0x000281 0x0000000000001000 0x0000000000001000 0x000010 0x000010 R E 0x1000
+# SEGMENT-MAPPING-NEXT:   LOAD      0x000291 0x0000000000002000 0x0000000000002000 0x000009 0x000009 R   0x1000
+# SEGMENT-MAPPING-NEXT:   LOAD      0x00029a 0x0000000000003ef0 0x0000000000003ef0 0x000011 0x000011 RW  0x1000
+# SEGMENT-MAPPING-NEXT:   DYNAMIC   0x00029a 0x0000000000003ef0 0x0000000000003ef0 0x000010 0x000010 RW  0x8
+# SEGMENT-MAPPING-NEXT:   GNU_RELRO 0x00029a 0x0000000000003ef0 0x0000000000003ef0 0x000010 0x000010 R   0x1
+# SEGMENT-MAPPING-NEXT:   LOAD      0x000000 0x0000000000004000 0x0000000000004000 0x000000 0x000000 R   0x1
+# SEGMENT-MAPPING-NEXT:   LOAD      0x000248 0x00000000000001a0 0x00000000000001a0 0x000020 0x000020 R   0x1
+# SEGMENT-MAPPING-NEXT:   LOAD      0x000248 0x00000000000001a0 0x00000000000001a0 0x000020 0x000020 R   0x1
+# SEGMENT-MAPPING:      Section to Segment mapping:
+# SEGMENT-MAPPING-NEXT:  Segment Sections...
+# SEGMENT-MAPPING-NEXT:   00     .hash .gnu.hash .dynsym .dynstr {{$}}
+# SEGMENT-MAPPING-NEXT:   01     .foo .zed {{$}}
+# SEGMENT-MAPPING-NEXT:   02     .foo .baz {{$}}
+# SEGMENT-MAPPING-NEXT:   03     .dynamic .dynamic.tail {{$}}
+# SEGMENT-MAPPING-NEXT:   04     .dynamic {{$}}
+# SEGMENT-MAPPING-NEXT:   05     .dynamic {{$}}
+# SEGMENT-MAPPING-NEXT:   06{{ *$}}
+# SEGMENT-MAPPING-NEXT:   07     .gnu.hash {{$}}
+# SEGMENT-MAPPING-NEXT:   08     .gnu.hash {{$}}
+# SEGMENT-MAPPING-NEXT:   None   .symtab .strtab .shstrtab {{$}}
+
+## Check that obj2yaml produced a correct program headers description.
+
+# RUN: obj2yaml %t1 | FileCheck %s --check-prefix=YAML
+
+# YAML:      ProgramHeaders:
+# YAML-NEXT:  - Type:  PT_LOAD
+# YAML-NEXT:    Flags: [ PF_R ]
+# YAML-NEXT:    Sections:
+# YAML-NEXT:      - Section: .hash
+# YAML-NEXT:      - Section: .gnu.hash
+# YAML-NEXT:      - Section: .dynsym
+# YAML-NEXT:      - Section: .dynstr
+# YAML-NEXT:    Align: 0x0000000000001000
+# YAML-NEXT:  - Type:  PT_LOAD
+# YAML-NEXT:    Flags: [ PF_X, PF_R ]
+# YAML-NEXT:    Sections:
+# YAML-NEXT:      - Section: .foo
+# YAML-NEXT:      - Section: .zed
+# YAML-NEXT:    VAddr: 0x0000000000001000
+# YAML-NEXT:    Align: 0x0000000000001000
+# YAML-NEXT:  - Type:  PT_LOAD
+# YAML-NEXT:    Flags: [ PF_R ]
+# YAML-NEXT:    Sections:
+# YAML-NEXT:      - Section: '.foo [1]'
+# YAML-NEXT:      - Section: .baz
+# YAML-NEXT:    VAddr: 0x0000000000002000
+# YAML-NEXT:    Align: 0x0000000000001000
+# YAML-NEXT:  - Type:  PT_LOAD
+# YAML-NEXT:    Flags: [ PF_W, PF_R ]
+# YAML-NEXT:    Sections:
+# YAML-NEXT:      - Section: .dynamic
+# YAML-NEXT:      - Section: .dynamic.tail
+# YAML-NEXT:    VAddr: 0x0000000000003EF0
+# YAML-NEXT:    Align: 0x0000000000001000
+# YAML-NEXT:  - Type:  PT_DYNAMIC
+# YAML-NEXT:    Flags: [ PF_W, PF_R ]
+# YAML-NEXT:    Sections:
+# YAML-NEXT:      - Section: .dynamic
+# YAML-NEXT:    VAddr: 0x0000000000003EF0
+# YAML-NEXT:    Align: 0x0000000000000008
+# YAML-NEXT:  - Type:  PT_GNU_RELRO
+# YAML-NEXT:    Flags: [ PF_R ]
+# YAML-NEXT:    Sections:
+# YAML-NEXT:      - Section: .dynamic
+# YAML-NEXT:    VAddr: 0x0000000000003EF0
+# YAML-NEXT:    Align: 0x0000000000000001
+# YAML-NEXT:  - Type:  PT_LOAD
+# YAML-NEXT:    Flags: [ PF_R ]
+# YAML-NEXT:    VAddr: 0x0000000000004000
+# YAML-NEXT:    Align: 0x0000000000000001
+# YAML-NEXT:  - Type:  PT_LOAD
+# YAML-NEXT:    Flags: [ PF_R ]
+# YAML-NEXT:    Sections:
+# YAML-NEXT:      - Section: .gnu.hash
+# YAML-NEXT:    VAddr: 0x00000000000001A0
+# YAML-NEXT:    Align: 0x0000000000000001
+# YAML-NEXT:  - Type:  PT_LOAD
+# YAML-NEXT:    Flags: [ PF_R ]
+# YAML-NEXT:    Sections:
+# YAML-NEXT:      - Section: .gnu.hash
+# YAML-NEXT:    VAddr: 0x00000000000001A0
+# YAML-NEXT:    Align: 0x0000000000000001
+# YAML-NEXT: Sections:
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_X86_64
+ProgramHeaders:
+## Check we can create a PT_LOAD with arbitrary (we used .hash, .gnu.hash)
+## and implicit sections (we use .dynsym, .dynstr). It also checks that the
+## SHT_NULL section at index 0 is not included in the segment.
+  - Type:  PT_LOAD
+    Flags: [ PF_R ]
+    Sections:
+      - Section: .hash
+      - Section: .gnu.hash
+      - Section: .dynsym
+      - Section: .dynstr
+    Align:  0x1000
+    Offset: 0x0
+## Check we can create a PT_LOAD with a different set of properties and sections.
+  - Type:  PT_LOAD
+    Flags: [ PF_X, PF_R ]
+    Sections:
+      - Section: .foo
+      - Section: .zed
+    VAddr: 0x1000
+    Align: 0x1000
+## Create a PT_LOAD to demonstate we are able to refer to output sections with the same name.
+  - Type:  PT_LOAD
+    Flags: [ PF_R ]
+    Sections:
+      - Section: '.foo [1]'
+      - Section: .baz
+    VAddr: 0x2000
+    Align: 0x1000
+## Show we can create a writeable PT_LOAD segment and put an arbitrary section into it.
+## Here we test both regular (SHT_PROGBITS) and a special section (SHT_DYNAMIC).
+  - Type:  PT_LOAD
+    Flags: [ PF_W, PF_R ]
+    Sections:
+      - Section: .dynamic
+      - Section: .dynamic.tail
+    VAddr: 0x3EF0
+    Align: 0x1000
+## Show we can create a nested dynamic segment and put a section into it.
+  - Type:  PT_DYNAMIC
+    Flags: [ PF_W, PF_R ]
+    Sections:
+      - Section: .dynamic
+    VAddr: 0x3EF0
+    Align: 0x8
+## Show we can create a relro segment and put a section into it.
+## We used .dynamic here and in tests above to demonstrate that
+## we can place a section in any number of segments.
+  - Type:  PT_GNU_RELRO
+    Flags: [ PF_R ]
+    Sections:
+      - Section: .dynamic
+    VAddr: 0x3EF0
+    Align: 0x1
+## Show we can dump a standalone empty segment.
+  - Type:  PT_LOAD
+    Flags: [ PF_R ]
+    Sections: [ ]
+    VAddr: 0x4000
+    Align: 0x1
+## ELF specification says that loadable segment entries in the
+## program header are sorted by virtual address.
+## Show we can dump an out of order segment.
+  - Type:  PT_LOAD
+    Flags: [ PF_R ]
+    Sections:
+      - Section: .gnu.hash
+    VAddr: 0x1A0
+    Align: 0x1
+## Test we are able to dump duplicated segments.
+## We use a segment that is the same as the previous one for this.
+  - Type:  PT_LOAD
+    Flags: [ PF_R ]
+    Sections:
+      - Section: .gnu.hash
+    VAddr: 0x1A0
+    Align: 0x1
+Sections:
+  - Name:    .hash
+    Type:    SHT_PROGBITS
+    Flags:   [ SHF_ALLOC ]
+    Address: 0x190
+    Size:    0x10
+  - Name:    .gnu.hash
+    Type:    SHT_PROGBITS
+    Flags:   [ SHF_ALLOC ]
+    Address: 0x1A0
+    Size:    0x20
+  - Name:    .dynsym
+    Type:    SHT_DYNSYM
+    Flags:   [ SHF_ALLOC ]
+    Address: 0x1C0
+    Link:    .dynstr
+    EntSize: 0x18
+  - Name:    .dynstr
+    Type:    SHT_STRTAB
+    Flags:   [ SHF_ALLOC ]
+    Address: 0x1D8
+  - Name:    .foo
+    Type:    SHT_PROGBITS
+    Flags:   [ SHF_ALLOC, SHF_EXECINSTR ]
+    Address: 0x1000
+    Size:    0x8
+  - Name:    .zed
+    Type:    SHT_PROGBITS
+    Flags:   [ SHF_ALLOC, SHF_EXECINSTR ]
+    Address: 0x1008
+    Size:    0x8
+  - Name:    '.foo [1]'
+    Type:    SHT_PROGBITS
+    Flags:   [ SHF_ALLOC ]
+    Address: 0x2000
+    Size:    0x8
+  - Name:    .baz
+    Type:    SHT_PROGBITS
+    Flags:   [ SHF_ALLOC ]
+    Address: 0x2008
+    Size:    0x1
+  - Name:    .dynamic
+    Type:    SHT_DYNAMIC
+    Flags:   [ SHF_WRITE, SHF_ALLOC ]
+    Address: 0x0000000000003EF0
+    Link:    .dynstr
+    Entries:
+      - Tag:   DT_NULL
+        Value: 0x0
+  - Name:    .dynamic.tail
+    Type:    SHT_PROGBITS
+    Flags:   [ SHF_WRITE, SHF_ALLOC ]
+    Content: "FE"
+Symbols: []
+DynamicSymbols: []
+
+## Part II. More specific tests.
+
+## Check we are able to dump segments that are empty or
+## contain empty sections.
+# RUN: yaml2obj --docnum=2 %s -o %t2
+# RUN: obj2yaml %t2 | FileCheck %s --check-prefix=EMPTY
+
+# EMPTY:      - Type:  PT_LOAD
+# EMPTY-NEXT:   Flags: [ PF_W, PF_R ]
+# EMPTY-NEXT:   Sections:
+# EMPTY-NEXT:     - Section: .empty.tls.start
+# EMPTY-NEXT:     - Section: .section.1
+# EMPTY-NEXT:     - Section: .empty.tls.middle
+# EMPTY-NEXT:     - Section: .section.2
+# EMPTY-NEXT:     - Section: .empty.tls.end
+# EMPTY-NEXT:   VAddr: 0x0000000000001000
+# EMPTY-NEXT:   Align: 0x0000000000001000
+# EMPTY-NEXT: - Type:            PT_TLS
+# EMPTY-NEXT:   Flags:           [ PF_W, PF_R ]
+# EMPTY-NEXT:   Sections:
+# EMPTY-NEXT:     - Section:         .empty.tls.start
+# EMPTY-NEXT:   VAddr:           0x0000000000001000
+# EMPTY-NEXT:   Align:           0x0000000000000001
+# EMPTY-NEXT: - Type:            PT_TLS
+# EMPTY-NEXT:   Flags:           [ PF_W, PF_R ]
+# EMPTY-NEXT:   Sections:
+# EMPTY-NEXT:     - Section:         .empty.tls.middle
+# EMPTY-NEXT:   VAddr:           0x0000000000001100
+# EMPTY-NEXT:   Align:           0x0000000000000001
+# EMPTY-NEXT: - Type:            PT_TLS
+# EMPTY-NEXT:   Flags:           [ PF_W, PF_R ]
+# EMPTY-NEXT:   Sections:
+# EMPTY-NEXT:     - Section:         .empty.tls.end
+# EMPTY-NEXT:   VAddr:           0x0000000000001200
+# EMPTY-NEXT:   Align:           0x0000000000000001
+# EMPTY-NEXT: Sections:
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_X86_64
+ProgramHeaders:
+  - Type:  PT_LOAD
+    Flags: [ PF_W, PF_R ]
+    Sections:
+      - Section: .empty.tls.start
+      - Section: .section.1
+      - Section: .empty.tls.middle
+      - Section: .section.2
+      - Section: .empty.tls.end
+    VAddr: 0x1000
+    Align: 0x1000
+  - Type:  PT_TLS
+    Flags: [ PF_W, PF_R ]
+    Sections:
+      - Section: .empty.tls.start
+    VAddr: 0x1000
+    Align: 0x1
+  - Type:  PT_TLS
+    Flags: [ PF_W, PF_R ]
+    Sections:
+      - Section: .empty.tls.middle
+    VAddr: 0x1100
+    Align: 0x1
+  - Type:  PT_TLS
+    Flags: [ PF_W, PF_R ]
+    Sections:
+      - Section: .empty.tls.end
+    VAddr: 0x1200
+    Align: 0x1
+Sections:
+  - Name:    .empty.tls.start
+    Type:    SHT_PROGBITS
+    Flags:   [ SHF_ALLOC, SHF_TLS ]
+    Size:    0x0
+    Address: 0x1000
+  - Name:  .section.1
+    Type:  SHT_PROGBITS
+    Flags: [ SHF_ALLOC ]
+    Size:  0x100
+  - Name:  .empty.tls.middle
+    Type:  SHT_PROGBITS
+    Flags: [ SHF_ALLOC, SHF_TLS ]
+    Size:  0x0
+  - Name:  .section.2
+    Type:  SHT_PROGBITS
+    Flags: [ SHF_ALLOC ]
+    Size:  0x100
+  - Name:  .empty.tls.end
+    Type:  SHT_PROGBITS
+    Flags: [ SHF_ALLOC, SHF_TLS ]
+    Size:  0x0
+
+## Document we are able to dump misaligned segments.
+## I.e. segments where (p_offset % p_align) != (p_vaddr % p_align).
+# RUN: yaml2obj --docnum=3 %s -o %t3
+# RUN: llvm-readelf --segments --sections %t3 | FileCheck %s --check-prefix=MISALIGNED-READELF
+# RUN: obj2yaml %t3 | FileCheck %s --check-prefix=MISALIGNED-YAML
+
+## As a misaligned p_offset value we use (`.foo` section offset - 1).
+# MISALIGNED-READELF:      [Nr] Name Type     Address          Off
+# MISALIGNED-READELF:      [ 1] .foo PROGBITS 0000000000001000 000078
+# MISALIGNED-READELF:      Type Offset   VirtAddr           PhysAddr           FileSiz  MemSiz   Flg Align
+# MISALIGNED-READELF-NEXT: LOAD 0x000077 0x0000000000001000 0x0000000000001000 0x000078 0x000078 R   0x1000
+
+# MISALIGNED-YAML:      ProgramHeaders:
+# MISALIGNED-YAML-NEXT:  - Type:  PT_LOAD
+# MISALIGNED-YAML-NEXT:    Flags: [ PF_R ]
+# MISALIGNED-YAML-NEXT:    Sections:
+# MISALIGNED-YAML-NEXT:     - Section: .foo
+# MISALIGNED-YAML-NEXT:    VAddr: 0x0000000000001000
+# MISALIGNED-YAML-NEXT:    Align: 0x0000000000001000
+# MISALIGNED-YAML-NEXT: Sections:
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_X86_64
+ProgramHeaders:
+  - Type:  PT_LOAD
+    Flags: [ PF_R ]
+    Sections:
+      - Section: .foo
+    VAddr:  0x1000
+    Align:  0x1000
+    Offset: 0x000077
+Sections:
+  - Name:    .foo
+    Type:    SHT_PROGBITS
+    Flags:   [ SHF_ALLOC ]
+    Size:    0x77
+    Address: 0x1000
+
+## Test we include non-allocatable sections in segments.
+## We also document that SHT_NULL sections are not considered to be inside a segment.
+# RUN: yaml2obj --docnum=4 %s -o %t4
+# RUN: obj2yaml %t4 | FileCheck %s --check-prefix=NON-ALLOC
+
+# NON-ALLOC:      ProgramHeaders:
+# NON-ALLOC-NEXT:   - Type:  PT_LOAD
+# NON-ALLOC-NEXT:     Flags: [ PF_R ]
+# NON-ALLOC-NEXT:     Sections:
+# NON-ALLOC-NEXT:      - Section: .alloc.1
+# NON-ALLOC-NEXT:      - Section: .non-alloc.1
+# NON-ALLOC-NEXT:      - Section: .alloc.2
+# NON-ALLOC-NEXT:     VAddr: 0x0000000000001000
+# NON-ALLOC-NEXT:     Align: 0x0000000000000001
+# NON-ALLOC-NEXT: Sections:
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_X86_64
+ProgramHeaders:
+  - Type:  PT_LOAD
+    Flags: [ PF_R ]
+    Sections:
+      - Section: .alloc.1
+      - Section: .alloc.2
+    VAddr:  0x1000
+Sections:
+  - Name:    .alloc.1
+    Type:    SHT_PROGBITS
+    Flags:   [ SHF_ALLOC ]
+    Size:    0x100
+    Address: 0x1000
+  - Name:    .non-alloc.1
+    Type:    SHT_PROGBITS
+    Flags:   [ ]
+    Size:    0x10
+  - Name:    .non-alloc.2
+    Type:    SHT_NULL
+    Flags:   [ ]
+    Size:    0x10
+  - Name:    .alloc.2
+    Type:    SHT_PROGBITS
+    Flags:   [ SHF_ALLOC ]
+    Size:    0x1
diff --git a/llvm/test/tools/yaml2obj/ELF/hash-section.yaml b/llvm/test/tools/yaml2obj/ELF/hash-section.yaml
index 4aad9c11fd592..2274e4c9a7c9f 100644
--- a/llvm/test/tools/yaml2obj/ELF/hash-section.yaml
+++ b/llvm/test/tools/yaml2obj/ELF/hash-section.yaml
@@ -276,3 +276,39 @@ Sections:
     Type: SHT_HASH
     Size: 0x1
     Chain: [ 1 ]
+
+## Check we can override "nbucket" and "nchain" values of a SHT_HASH section using "NBucket"
+## and "NChain" tags. Check that the section size is unaffected when we do this.
+
+# RUN: yaml2obj --docnum=14 %s -o %t14
+# RUN: llvm-readobj --sections --section-data %t14 | FileCheck %s --check-prefix=OVERRIDE
+
+# OVERRIDE:      Name: .hash
+# OVERRIDE-NEXT: Type: SHT_HASH
+# OVERRIDE-NEXT: Flags [
+# OVERRIDE-NEXT: ]
+# OVERRIDE-NEXT: Address: 0x0
+# OVERRIDE-NEXT: Offset: 0x34
+# OVERRIDE-NEXT: Size: 28
+# OVERRIDE-NEXT: Link: 0
+# OVERRIDE-NEXT: Info: 0
+# OVERRIDE-NEXT: AddressAlignment: 0
+# OVERRIDE-NEXT: EntrySize: 0
+# OVERRIDE-NEXT: SectionData (
+# OVERRIDE-NEXT:   0000: AA000000 BB000000 01000000 02000000
+# OVERRIDE-NEXT:   0010: 03000000 04000000 05000000
+# OVERRIDE-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS32
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_386
+Sections:
+  - Name:   .hash
+    Type:   SHT_HASH
+    Bucket: [ 1, 2 ]
+    Chain:  [ 3, 4, 5 ]
+    NBucket: 0xAA
+    NChain:  0xBB
diff --git a/llvm/tools/gold/gold-plugin.cpp b/llvm/tools/gold/gold-plugin.cpp
index 2db5716171816..6d2369f52c42e 100644
--- a/llvm/tools/gold/gold-plugin.cpp
+++ b/llvm/tools/gold/gold-plugin.cpp
@@ -28,6 +28,7 @@
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/TargetSelect.h"
+#include "llvm/Support/Threading.h"
 #include "llvm/Support/raw_ostream.h"
 #include <list>
 #include <map>
@@ -134,11 +135,9 @@ namespace options {
   };
   static OutputType TheOutputType = OT_NORMAL;
   static unsigned OptLevel = 2;
-  // Default parallelism of 0 used to indicate that user did not specify.
-  // Actual parallelism default value depends on implementation.
   // Currently only affects ThinLTO, where the default is the max cores in the
-  // system.
-  static unsigned Parallelism = 0;
+  // system. See llvm::get_threadpool_strategy() for acceptable values.
+  static std::string Parallelism;
   // Default regular LTO codegen parallelism (number of partitions).
   static unsigned ParallelCodeGenParallelismLevel = 1;
 #ifdef NDEBUG
@@ -272,8 +271,10 @@ namespace options {
         message(LDPL_FATAL, "Optimization level must be between 0 and 3");
       OptLevel = opt[1] - '0';
     } else if (opt.startswith("jobs=")) {
-      if (StringRef(opt_ + 5).getAsInteger(10, Parallelism))
-        message(LDPL_FATAL, "Invalid parallelism level: %s", opt_ + 5);
+      StringRef Num(opt_ + 5);
+      if (!get_threadpool_strategy(Num))
+        message(LDPL_FATAL, "Invalid parallelism level: %s", Num.data());
+      Parallelism = Num.str();
     } else if (opt.startswith("lto-partitions=")) {
       if (opt.substr(strlen("lto-partitions="))
               .getAsInteger(10, ParallelCodeGenParallelismLevel))
@@ -877,14 +878,15 @@ static std::unique_ptr<LTO> createLTO(IndexWriteCallback OnIndexWrite,
   Conf.PTO.LoopVectorization = options::OptLevel > 1;
   Conf.PTO.SLPVectorization = options::OptLevel > 1;
 
-  if (options::Parallelism)
-    Backend = createInProcessThinBackend(options::Parallelism);
   if (options::thinlto_index_only) {
     std::string OldPrefix, NewPrefix;
     getThinLTOOldAndNewPrefix(OldPrefix, NewPrefix);
     Backend = createWriteIndexesThinBackend(OldPrefix, NewPrefix,
                                             options::thinlto_emit_imports_files,
                                             LinkedObjectsFile, OnIndexWrite);
+  } else {
+    Backend = createInProcessThinBackend(
+        llvm::heavyweight_hardware_concurrency(options::Parallelism));
   }
 
   Conf.OverrideTriple = options::triple;
diff --git a/llvm/tools/llc/llc.cpp b/llvm/tools/llc/llc.cpp
index 76dd071bd2c71..cd02971b24944 100644
--- a/llvm/tools/llc/llc.cpp
+++ b/llvm/tools/llc/llc.cpp
@@ -319,6 +319,7 @@ int main(int argc, char **argv) {
   initializeScalarizeMaskedMemIntrinPass(*Registry);
   initializeExpandReductionsPass(*Registry);
   initializeHardwareLoopsPass(*Registry);
+  initializeTransformUtils(*Registry);
 
   // Initialize debugging passes.
   initializeScavengerTestPass(*Registry);
diff --git a/llvm/tools/llvm-cov/CodeCoverage.cpp b/llvm/tools/llvm-cov/CodeCoverage.cpp
index 625e2342e4bb3..b2024e9ee3311 100644
--- a/llvm/tools/llvm-cov/CodeCoverage.cpp
+++ b/llvm/tools/llvm-cov/CodeCoverage.cpp
@@ -867,8 +867,8 @@ int CodeCoverageTool::doShow(int argc, const char **argv,
   }
 
   sys::fs::file_status Status;
-  if (sys::fs::status(PGOFilename, Status)) {
-    error("profdata file error: can not get the file status. \n");
+  if (std::error_code EC = sys::fs::status(PGOFilename, Status)) {
+    error("Could not read profile data!", EC.message());
     return 1;
   }
 
diff --git a/llvm/tools/llvm-cov/CoverageExporterJson.cpp b/llvm/tools/llvm-cov/CoverageExporterJson.cpp
index ba8ff5c8fe523..c5d1b248bd96a 100644
--- a/llvm/tools/llvm-cov/CoverageExporterJson.cpp
+++ b/llvm/tools/llvm-cov/CoverageExporterJson.cpp
@@ -53,7 +53,7 @@
 #include <utility>
 
 /// The semantic version combined as a string.
-#define LLVM_COVERAGE_EXPORT_JSON_STR "2.0.0"
+#define LLVM_COVERAGE_EXPORT_JSON_STR "2.0.1"
 
 /// Unique type identifier for JSON coverage export.
 #define LLVM_COVERAGE_EXPORT_JSON_TYPE_STR "llvm.coverage.json.export"
@@ -72,8 +72,9 @@ int64_t clamp_uint64_to_int64(uint64_t u) {
 }
 
 json::Array renderSegment(const coverage::CoverageSegment &Segment) {
-  return json::Array({Segment.Line, Segment.Col, clamp_uint64_to_int64(Segment.Count),
-                      Segment.HasCount, Segment.IsRegionEntry});
+  return json::Array({Segment.Line, Segment.Col,
+                      clamp_uint64_to_int64(Segment.Count), Segment.HasCount,
+                      Segment.IsRegionEntry, Segment.IsGapRegion});
 }
 
 json::Array renderRegion(const coverage::CountedRegion &Region) {
diff --git a/llvm/tools/llvm-dwarfdump/CMakeLists.txt b/llvm/tools/llvm-dwarfdump/CMakeLists.txt
index 77620e0faaf81..58d8114cbe799 100644
--- a/llvm/tools/llvm-dwarfdump/CMakeLists.txt
+++ b/llvm/tools/llvm-dwarfdump/CMakeLists.txt
@@ -8,6 +8,7 @@ set(LLVM_LINK_COMPONENTS
   )
 
 add_llvm_tool(llvm-dwarfdump
+  SectionSizes.cpp
   Statistics.cpp
   llvm-dwarfdump.cpp
   )
diff --git a/llvm/tools/llvm-dwarfdump/SectionSizes.cpp b/llvm/tools/llvm-dwarfdump/SectionSizes.cpp
new file mode 100644
index 0000000000000..21bbee57e19d1
--- /dev/null
+++ b/llvm/tools/llvm-dwarfdump/SectionSizes.cpp
@@ -0,0 +1,120 @@
+//===-- SectionSizes.cpp - Debug section sizes ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SectionSizes.h"
+
+#define DEBUG_TYPE "dwarfdump"
+
+using namespace llvm;
+using namespace object;
+
+static size_t getNameColumnWidth(const SectionSizes &Sizes,
+                                 const StringRef SectionNameTitle) {
+  // The minimum column width should be the size of "SECTION".
+  size_t Width = SectionNameTitle.size();
+  for (const auto &DebugSec : Sizes.DebugSectionSizes) {
+    StringRef SectionName = DebugSec.getKey();
+    Width = std::max(Width, SectionName.size());
+  }
+  return Width;
+}
+
+static size_t getSizeColumnWidth(const SectionSizes &Sizes,
+                                 const StringRef SectionSizeTitle) {
+  // The minimum column width should be the size of the column title.
+  size_t Width = SectionSizeTitle.size();
+  for (const auto &DebugSec : Sizes.DebugSectionSizes) {
+    size_t NumWidth = std::to_string(DebugSec.getValue()).size();
+    Width = std::max(Width, NumWidth);
+  }
+  return Width;
+}
+
+static void prettyPrintSectionSizes(const ObjectFile &Obj,
+                                    const SectionSizes &Sizes,
+                                    raw_ostream &OS) {
+  const StringRef SectionNameTitle = "SECTION";
+  const StringRef SectionSizeTitle = "SIZE (b)";
+
+  size_t NameColWidth = getNameColumnWidth(Sizes, SectionNameTitle);
+  size_t SizeColWidth = getSizeColumnWidth(Sizes, SectionSizeTitle);
+
+  OS << "----------------------------------------------------" << '\n';
+  OS << SectionNameTitle;
+  size_t SectionNameTitleWidth = SectionNameTitle.size();
+  for (unsigned i = 0; i < (NameColWidth - SectionNameTitleWidth) + 2; i++)
+    OS << " ";
+  OS << SectionSizeTitle << '\n';
+  for (unsigned i = 0; i < NameColWidth; i++)
+    OS << "-";
+  OS << "  ";
+
+  for (unsigned i = 0; i < SizeColWidth; i++)
+    OS << "-";
+  OS << '\n';
+
+  for (const auto &DebugSec : Sizes.DebugSectionSizes) {
+    OS << left_justify(DebugSec.getKey(), NameColWidth) << "  ";
+
+    auto NumBytes = std::to_string(DebugSec.getValue());
+    OS << right_justify(NumBytes, SizeColWidth) << " ("
+       << format("%0.2f", DebugSec.getValue() /
+                              static_cast<double>(Sizes.TotalObjectSize) * 100)
+       << "%)\n";
+  }
+
+  OS << '\n';
+  OS << " Total Size: " << Sizes.TotalDebugSectionsSize << "  ("
+     << format("%0.2f", Sizes.TotalDebugSectionsSize /
+                            static_cast<double>(Sizes.TotalObjectSize) * 100)
+     << "%)\n";
+  OS << " Total File Size: " << Sizes.TotalObjectSize << '\n';
+  OS << "----------------------------------------------------" << '\n';
+}
+
+void llvm::calculateSectionSizes(const ObjectFile &Obj, SectionSizes &Sizes,
+                                 const Twine &Filename) {
+  // Get total size.
+  Sizes.TotalObjectSize = Obj.getData().size();
+
+  for (const SectionRef &Section : Obj.sections()) {
+    StringRef SectionName;
+    if (Expected<StringRef> NameOrErr = Section.getName())
+      SectionName = *NameOrErr;
+    else
+      WithColor::defaultWarningHandler(
+          createFileError(Filename, NameOrErr.takeError()));
+
+    LLVM_DEBUG(dbgs() << SectionName.str() << ": " << Section.getSize()
+                      << '\n');
+
+    if (!Section.isDebugSection(SectionName))
+      continue;
+
+    Sizes.TotalDebugSectionsSize += Section.getSize();
+    Sizes.DebugSectionSizes[SectionName] += Section.getSize();
+  }
+}
+
+bool collectObjectSectionSizes(ObjectFile &Obj, DWARFContext & /*DICtx*/,
+                               const Twine &Filename, raw_ostream &OS) {
+  SectionSizes Sizes;
+
+  // Get the section sizes.
+  calculateSectionSizes(Obj, Sizes, Filename);
+
+  OS << "----------------------------------------------------\n";
+  OS << "file: " << Filename.str() << '\n';
+
+  prettyPrintSectionSizes(Obj, Sizes, OS);
+
+  // TODO: If the input file is an archive, print the cumulative summary of all
+  // files from the archive.
+
+  return true;
+}
diff --git a/llvm/tools/llvm-dwarfdump/SectionSizes.h b/llvm/tools/llvm-dwarfdump/SectionSizes.h
new file mode 100644
index 0000000000000..1664b1a236d62
--- /dev/null
+++ b/llvm/tools/llvm-dwarfdump/SectionSizes.h
@@ -0,0 +1,38 @@
+//===- SectionSizes.h - Debug section sizes ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===-----------------------------------------------------------------------===/
+
+#ifndef LLVM_TOOLS_SECTION_SIZES_H
+#define LLVM_TOOLS_SECTION_SIZES_H
+
+#include "llvm/DebugInfo/DIContext.h"
+#include "llvm/DebugInfo/DWARF/DWARFContext.h"
+#include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/WithColor.h"
+
+namespace llvm {
+
+using SectionSizeMap = StringMap<uint64_t>;
+
+/// Holds cumulative section sizes for an object file.
+struct SectionSizes {
+  /// Map of .debug section names and their sizes across all such-named
+  /// sections.
+  SectionSizeMap DebugSectionSizes;
+  /// Total number of bytes of all sections.
+  uint64_t TotalObjectSize = 0;
+  /// Total number of bytes of all debug sections.
+  uint64_t TotalDebugSectionsSize = 0;
+};
+
+/// Calculate the section sizes.
+void calculateSectionSizes(const object::ObjectFile &Obj, SectionSizes &Sizes,
+                           const Twine &Filename);
+
+} // end namespace llvm
+
+#endif // LLVM_TOOLS_SECTION_SIZES_H
diff --git a/llvm/tools/llvm-dwarfdump/Statistics.cpp b/llvm/tools/llvm-dwarfdump/Statistics.cpp
index 6ba6a7037136e..ca94426519fd4 100644
--- a/llvm/tools/llvm-dwarfdump/Statistics.cpp
+++ b/llvm/tools/llvm-dwarfdump/Statistics.cpp
@@ -15,6 +15,8 @@
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/JSON.h"
 
+#include "SectionSizes.h"
+
 #define DEBUG_TYPE "dwarfdump"
 using namespace llvm;
 using namespace object;
@@ -468,6 +470,7 @@ static void printDatum(raw_ostream &OS, const char *Key, json::Value Value) {
   OS << ",\"" << Key << "\":" << Value;
   LLVM_DEBUG(llvm::dbgs() << Key << ": " << Value << '\n');
 }
+
 static void printLocationStats(raw_ostream &OS,
                                const char *Key,
                                std::vector<unsigned> &LocationStats) {
@@ -491,6 +494,12 @@ static void printLocationStats(raw_ostream &OS,
   LLVM_DEBUG(llvm::dbgs() << Key << " with 100% of its scope covered: "
                           << LocationStats[NumOfCoverageCategories - 1]);
 }
+
+static void printSectionSizes(raw_ostream &OS, const SectionSizes &Sizes) {
+  for (const auto &DebugSec : Sizes.DebugSectionSizes)
+    OS << ",\"size of " << DebugSec.getKey() << "\":" << DebugSec.getValue();
+}
+
 /// \}
 
 /// Collect debug info quality metrics for an entire DIContext.
@@ -512,6 +521,10 @@ bool collectStatsForObjectFile(ObjectFile &Obj, DWARFContext &DICtx,
       collectStatsRecursive(CUDie, "/", "g", 0, 0, Statistics, GlobalStats,
                             LocStats);
 
+  /// Collect the sizes of debug sections.
+  SectionSizes Sizes;
+  calculateSectionSizes(Obj, Sizes, Filename);
+
   /// The version number should be increased every time the algorithm is changed
   /// (including bug fixes). New metrics may be added without increasing the
   /// version.
@@ -602,6 +615,7 @@ bool collectStatsForObjectFile(ObjectFile &Obj, DWARFContext &DICtx,
   printDatum(OS, "vars with binary location", VarWithLoc);
   printDatum(OS, "total variables procesed by location statistics",
              LocStats.NumVarParam);
+  printSectionSizes(OS, Sizes);
   printLocationStats(OS, "variables", LocStats.VarParamLocStats);
   printLocationStats(OS, "variables (excluding the debug entry values)",
                      LocStats.VarParamNonEntryValLocStats);
diff --git a/llvm/tools/llvm-dwarfdump/llvm-dwarfdump.cpp b/llvm/tools/llvm-dwarfdump/llvm-dwarfdump.cpp
index 28ede2ef5029d..e559de1059cf1 100644
--- a/llvm/tools/llvm-dwarfdump/llvm-dwarfdump.cpp
+++ b/llvm/tools/llvm-dwarfdump/llvm-dwarfdump.cpp
@@ -208,6 +208,11 @@ static cl::opt<bool>
     Statistics("statistics",
                cl::desc("Emit JSON-formatted debug info quality metrics."),
                cat(DwarfDumpCategory));
+static cl::opt<bool>
+    ShowSectionSizes("show-section-sizes",
+                     cl::desc("Show the sizes of all debug sections, "
+                              "expressed in bytes."),
+                     cat(DwarfDumpCategory));
 static opt<bool> Verify("verify", desc("Verify the DWARF debug info."),
                         cat(DwarfDumpCategory));
 static opt<bool> Quiet("quiet", desc("Use with -verify to not emit to STDOUT."),
@@ -413,6 +418,9 @@ static bool lookup(ObjectFile &Obj, DWARFContext &DICtx, uint64_t Address,
 bool collectStatsForObjectFile(ObjectFile &Obj, DWARFContext &DICtx,
                                const Twine &Filename, raw_ostream &OS);
 
+bool collectObjectSectionSizes(ObjectFile &Obj, DWARFContext & /*DICtx*/,
+                               const Twine &Filename, raw_ostream &OS);
+
 static bool dumpObjectFile(ObjectFile &Obj, DWARFContext &DICtx,
                            const Twine &Filename, raw_ostream &OS) {
   logAllUnhandledErrors(DICtx.loadRegisterInfo(Obj), errs(),
@@ -635,12 +643,16 @@ int main(int argc, char **argv) {
           return handleFile(Object, verifyObjectFile, OutputFile.os());
         }))
       return 1;
-  } else if (Statistics)
+  } else if (Statistics) {
     for (auto Object : Objects)
       handleFile(Object, collectStatsForObjectFile, OutputFile.os());
-  else
+  } else if (ShowSectionSizes) {
+    for (auto Object : Objects)
+      handleFile(Object, collectObjectSectionSizes, OutputFile.os());
+  } else {
     for (auto Object : Objects)
       handleFile(Object, dumpObjectFile, OutputFile.os());
+  }
 
   return EXIT_SUCCESS;
 }
diff --git a/llvm/tools/llvm-dwp/llvm-dwp.cpp b/llvm/tools/llvm-dwp/llvm-dwp.cpp
index 51b3470afee45..2a1781983c4ec 100644
--- a/llvm/tools/llvm-dwp/llvm-dwp.cpp
+++ b/llvm/tools/llvm-dwp/llvm-dwp.cpp
@@ -219,7 +219,7 @@ struct UnitIndexEntry {
 static StringRef getSubsection(StringRef Section,
                                const DWARFUnitIndex::Entry &Entry,
                                DWARFSectionKind Kind) {
-  const auto *Off = Entry.getOffset(Kind);
+  const auto *Off = Entry.getContribution(Kind);
   if (!Off)
     return StringRef();
   return Section.substr(Off->Offset, Off->Length);
@@ -231,7 +231,7 @@ static void addAllTypesFromDWP(
     const UnitIndexEntry &TUEntry, uint32_t &TypesOffset) {
   Out.SwitchSection(OutputTypes);
   for (const DWARFUnitIndex::Entry &E : TUIndex.getRows()) {
-    auto *I = E.getOffsets();
+    auto *I = E.getContributions();
     if (!I)
       continue;
     auto P = TypeIndexEntries.insert(std::make_pair(E.getSignature(), TUEntry));
@@ -599,7 +599,7 @@ static Error write(MCStreamer &Out, ArrayRef<std::string> Inputs) {
       return make_error<DWPError>("failed to parse cu_index");
 
     for (const DWARFUnitIndex::Entry &E : CUIndex.getRows()) {
-      auto *I = E.getOffsets();
+      auto *I = E.getContributions();
       if (!I)
         continue;
       auto P = IndexEntries.insert(std::make_pair(E.getSignature(), CurEntry));
diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkResult.h b/llvm/tools/llvm-exegesis/lib/BenchmarkResult.h
index 1788e7490685f..d4bad347f6047 100644
--- a/llvm/tools/llvm-exegesis/lib/BenchmarkResult.h
+++ b/llvm/tools/llvm-exegesis/lib/BenchmarkResult.h
@@ -68,8 +68,7 @@ struct InstructionBenchmark {
   // The number of instructions inside the repeated snippet. For example, if a
   // snippet of 3 instructions is repeated 4 times, this is 12.
   int NumRepetitions = 0;
-  enum RepetitionModeE { Duplicate, Loop };
-  RepetitionModeE RepetitionMode;
+  enum RepetitionModeE { Duplicate, Loop, AggregateMin };
   // Note that measurements are per instruction.
   std::vector<BenchmarkMeasure> Measurements;
   std::string Error;
diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp
index 5b15bc29c60d3..be778f29f52f0 100644
--- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp
+++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp
@@ -14,6 +14,7 @@
 #include "Error.h"
 #include "MCInstrDescView.h"
 #include "PerfHelper.h"
+#include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
@@ -81,7 +82,8 @@ class FunctionExecutorImpl : public BenchmarkRunner::FunctionExecutor {
 
 Expected<InstructionBenchmark> BenchmarkRunner::runConfiguration(
     const BenchmarkCode &BC, unsigned NumRepetitions,
-    const SnippetRepetitor &Repetitor, bool DumpObjectToDisk) const {
+    ArrayRef<std::unique_ptr<const SnippetRepetitor>> Repetitors,
+    bool DumpObjectToDisk) const {
   InstructionBenchmark InstrBenchmark;
   InstrBenchmark.Mode = Mode;
   InstrBenchmark.CpuName = std::string(State.getTargetMachine().getTargetCPU());
@@ -94,70 +96,113 @@ Expected<InstructionBenchmark> BenchmarkRunner::runConfiguration(
 
   InstrBenchmark.Key = BC.Key;
 
-  // Assemble at least kMinInstructionsForSnippet instructions by repeating the
-  // snippet for debug/analysis. This is so that the user clearly understands
-  // that the inside instructions are repeated.
-  constexpr const int kMinInstructionsForSnippet = 16;
-  {
-    SmallString<0> Buffer;
-    raw_svector_ostream OS(Buffer);
-    if (Error E = assembleToStream(
-            State.getExegesisTarget(), State.createTargetMachine(), BC.LiveIns,
-            BC.Key.RegisterInitialValues,
-            Repetitor.Repeat(Instructions, kMinInstructionsForSnippet), OS)) {
-      return std::move(E);
+  // If we end up having an error, and we've previously succeeded with
+  // some other Repetitor, we want to discard the previous measurements.
+  struct ClearBenchmarkOnReturn {
+    ClearBenchmarkOnReturn(InstructionBenchmark *IB) : IB(IB) {}
+    ~ClearBenchmarkOnReturn() {
+      if (Clear)
+        IB->Measurements.clear();
+    }
+    void disarm() { Clear = false; }
+
+  private:
+    InstructionBenchmark *const IB;
+    bool Clear = true;
+  };
+  ClearBenchmarkOnReturn CBOR(&InstrBenchmark);
+
+  for (const std::unique_ptr<const SnippetRepetitor> &Repetitor : Repetitors) {
+    // Assemble at least kMinInstructionsForSnippet instructions by repeating
+    // the snippet for debug/analysis. This is so that the user clearly
+    // understands that the inside instructions are repeated.
+    constexpr const int kMinInstructionsForSnippet = 16;
+    {
+      SmallString<0> Buffer;
+      raw_svector_ostream OS(Buffer);
+      if (Error E = assembleToStream(
+              State.getExegesisTarget(), State.createTargetMachine(),
+              BC.LiveIns, BC.Key.RegisterInitialValues,
+              Repetitor->Repeat(Instructions, kMinInstructionsForSnippet),
+              OS)) {
+        return std::move(E);
+      }
+      const ExecutableFunction EF(State.createTargetMachine(),
+                                  getObjectFromBuffer(OS.str()));
+      const auto FnBytes = EF.getFunctionBytes();
+      InstrBenchmark.AssembledSnippet.insert(
+          InstrBenchmark.AssembledSnippet.end(), FnBytes.begin(),
+          FnBytes.end());
     }
-    const ExecutableFunction EF(State.createTargetMachine(),
-                                getObjectFromBuffer(OS.str()));
-    const auto FnBytes = EF.getFunctionBytes();
-    InstrBenchmark.AssembledSnippet.assign(FnBytes.begin(), FnBytes.end());
-  }
 
-  // Assemble NumRepetitions instructions repetitions of the snippet for
-  // measurements.
-  const auto Filler =
-      Repetitor.Repeat(Instructions, InstrBenchmark.NumRepetitions);
+    // Assemble NumRepetitions instructions repetitions of the snippet for
+    // measurements.
+    const auto Filler =
+        Repetitor->Repeat(Instructions, InstrBenchmark.NumRepetitions);
+
+    object::OwningBinary<object::ObjectFile> ObjectFile;
+    if (DumpObjectToDisk) {
+      auto ObjectFilePath = writeObjectFile(BC, Filler);
+      if (Error E = ObjectFilePath.takeError()) {
+        InstrBenchmark.Error = toString(std::move(E));
+        return InstrBenchmark;
+      }
+      outs() << "Check generated assembly with: /usr/bin/objdump -d "
+             << *ObjectFilePath << "\n";
+      ObjectFile = getObjectFromFile(*ObjectFilePath);
+    } else {
+      SmallString<0> Buffer;
+      raw_svector_ostream OS(Buffer);
+      if (Error E = assembleToStream(
+              State.getExegesisTarget(), State.createTargetMachine(),
+              BC.LiveIns, BC.Key.RegisterInitialValues, Filler, OS)) {
+        return std::move(E);
+      }
+      ObjectFile = getObjectFromBuffer(OS.str());
+    }
 
-  object::OwningBinary<object::ObjectFile> ObjectFile;
-  if (DumpObjectToDisk) {
-    auto ObjectFilePath = writeObjectFile(BC, Filler);
-    if (Error E = ObjectFilePath.takeError()) {
+    const FunctionExecutorImpl Executor(State, std::move(ObjectFile),
+                                        Scratch.get());
+    auto NewMeasurements = runMeasurements(Executor);
+    if (Error E = NewMeasurements.takeError()) {
+      if (!E.isA<SnippetCrash>())
+        return std::move(E);
       InstrBenchmark.Error = toString(std::move(E));
       return InstrBenchmark;
     }
-    outs() << "Check generated assembly with: /usr/bin/objdump -d "
-           << *ObjectFilePath << "\n";
-    ObjectFile = getObjectFromFile(*ObjectFilePath);
-  } else {
-    SmallString<0> Buffer;
-    raw_svector_ostream OS(Buffer);
-    if (Error E = assembleToStream(State.getExegesisTarget(),
-                                   State.createTargetMachine(), BC.LiveIns,
-                                   BC.Key.RegisterInitialValues, Filler, OS)) {
-      return std::move(E);
+    assert(InstrBenchmark.NumRepetitions > 0 && "invalid NumRepetitions");
+    for (BenchmarkMeasure &BM : *NewMeasurements) {
+      // Scale the measurements by instruction.
+      BM.PerInstructionValue /= InstrBenchmark.NumRepetitions;
+      // Scale the measurements by snippet.
+      BM.PerSnippetValue *= static_cast<double>(Instructions.size()) /
+                            InstrBenchmark.NumRepetitions;
+    }
+    if (InstrBenchmark.Measurements.empty()) {
+      InstrBenchmark.Measurements = std::move(*NewMeasurements);
+      continue;
     }
-    ObjectFile = getObjectFromBuffer(OS.str());
-  }
 
-  const FunctionExecutorImpl Executor(State, std::move(ObjectFile),
-                                      Scratch.get());
-  auto Measurements = runMeasurements(Executor);
-  if (Error E = Measurements.takeError()) {
-    if (!E.isA<SnippetCrash>())
-      return std::move(E);
-    InstrBenchmark.Error = toString(std::move(E));
-    return InstrBenchmark;
-  }
-  InstrBenchmark.Measurements = std::move(*Measurements);
-  assert(InstrBenchmark.NumRepetitions > 0 && "invalid NumRepetitions");
-  for (BenchmarkMeasure &BM : InstrBenchmark.Measurements) {
-    // Scale the measurements by instruction.
-    BM.PerInstructionValue /= InstrBenchmark.NumRepetitions;
-    // Scale the measurements by snippet.
-    BM.PerSnippetValue *= static_cast<double>(Instructions.size()) /
-                          InstrBenchmark.NumRepetitions;
+    assert(Repetitors.size() > 1 && !InstrBenchmark.Measurements.empty() &&
+           "We're in an 'min' repetition mode, and need to aggregate new "
+           "result to the existing result.");
+    assert(InstrBenchmark.Measurements.size() == NewMeasurements->size() &&
+           "Expected to have identical number of measurements.");
+    for (auto I : zip(InstrBenchmark.Measurements, *NewMeasurements)) {
+      BenchmarkMeasure &Measurement = std::get<0>(I);
+      BenchmarkMeasure &NewMeasurement = std::get<1>(I);
+      assert(Measurement.Key == NewMeasurement.Key &&
+             "Expected measurements to be symmetric");
+
+      Measurement.PerInstructionValue = std::min(
+          Measurement.PerInstructionValue, NewMeasurement.PerInstructionValue);
+      Measurement.PerSnippetValue =
+          std::min(Measurement.PerSnippetValue, NewMeasurement.PerSnippetValue);
+    }
   }
 
+  // We successfully measured everything, so don't discard the results.
+  CBOR.disarm();
   return InstrBenchmark;
 }
 
diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h
index 1a8866705d089..b0fdb34450ee3 100644
--- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h
+++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h
@@ -40,7 +40,7 @@ class BenchmarkRunner {
 
   Expected<InstructionBenchmark>
   runConfiguration(const BenchmarkCode &Configuration, unsigned NumRepetitions,
-                   const SnippetRepetitor &Repetitor,
+                   ArrayRef<std::unique_ptr<const SnippetRepetitor>> Repetitors,
                    bool DumpObjectToDisk) const;
 
   // Scratch space to run instructions that touch memory.
diff --git a/llvm/tools/llvm-exegesis/lib/SnippetRepetitor.cpp b/llvm/tools/llvm-exegesis/lib/SnippetRepetitor.cpp
index ba618ac3f97e3..c866e972a1c46 100644
--- a/llvm/tools/llvm-exegesis/lib/SnippetRepetitor.cpp
+++ b/llvm/tools/llvm-exegesis/lib/SnippetRepetitor.cpp
@@ -110,6 +110,8 @@ SnippetRepetitor::Create(InstructionBenchmark::RepetitionModeE Mode,
     return std::make_unique<DuplicateSnippetRepetitor>(State);
   case InstructionBenchmark::Loop:
     return std::make_unique<LoopSnippetRepetitor>(State);
+  case InstructionBenchmark::AggregateMin:
+    break;
   }
   llvm_unreachable("Unknown RepetitionModeE enum");
 }
diff --git a/llvm/tools/llvm-exegesis/llvm-exegesis.cpp b/llvm/tools/llvm-exegesis/llvm-exegesis.cpp
index 3adc9f0ca19ee..ce3a31c12d4ae 100644
--- a/llvm/tools/llvm-exegesis/llvm-exegesis.cpp
+++ b/llvm/tools/llvm-exegesis/llvm-exegesis.cpp
@@ -86,10 +86,14 @@ static cl::opt<exegesis::InstructionBenchmark::ModeE> BenchmarkMode(
 static cl::opt<exegesis::InstructionBenchmark::RepetitionModeE> RepetitionMode(
     "repetition-mode", cl::desc("how to repeat the instruction snippet"),
     cl::cat(BenchmarkOptions),
-    cl::values(clEnumValN(exegesis::InstructionBenchmark::Duplicate,
-                          "duplicate", "Duplicate the snippet"),
-               clEnumValN(exegesis::InstructionBenchmark::Loop, "loop",
-                          "Loop over the snippet")));
+    cl::values(
+        clEnumValN(exegesis::InstructionBenchmark::Duplicate, "duplicate",
+                   "Duplicate the snippet"),
+        clEnumValN(exegesis::InstructionBenchmark::Loop, "loop",
+                   "Loop over the snippet"),
+        clEnumValN(exegesis::InstructionBenchmark::AggregateMin, "min",
+                   "All of the above and take the minimum of measurements")),
+    cl::init(exegesis::InstructionBenchmark::Duplicate));
 
 static cl::opt<unsigned>
     NumRepetitions("num-repetitions",
@@ -285,7 +289,22 @@ void benchmarkMain() {
 
   const auto Opcodes = getOpcodesOrDie(State.getInstrInfo());
 
-  const auto Repetitor = SnippetRepetitor::Create(RepetitionMode, State);
+  SmallVector<std::unique_ptr<const SnippetRepetitor>, 2> Repetitors;
+  if (RepetitionMode != InstructionBenchmark::RepetitionModeE::AggregateMin)
+    Repetitors.emplace_back(SnippetRepetitor::Create(RepetitionMode, State));
+  else {
+    for (InstructionBenchmark::RepetitionModeE RepMode :
+         {InstructionBenchmark::RepetitionModeE::Duplicate,
+          InstructionBenchmark::RepetitionModeE::Loop})
+      Repetitors.emplace_back(SnippetRepetitor::Create(RepMode, State));
+  }
+
+  BitVector AllReservedRegs;
+  llvm::for_each(Repetitors,
+                 [&AllReservedRegs](
+                     const std::unique_ptr<const SnippetRepetitor> &Repetitor) {
+                   AllReservedRegs |= Repetitor->getReservedRegs();
+                 });
 
   std::vector<BenchmarkCode> Configurations;
   if (!Opcodes.empty()) {
@@ -298,8 +317,8 @@ void benchmarkMain() {
                << ": ignoring instruction without sched class\n";
         continue;
       }
-      auto ConfigsForInstr =
-          generateSnippets(State, Opcode, Repetitor->getReservedRegs());
+
+      auto ConfigsForInstr = generateSnippets(State, Opcode, AllReservedRegs);
       if (!ConfigsForInstr) {
         logAllUnhandledErrors(
             ConfigsForInstr.takeError(), errs(),
@@ -324,7 +343,7 @@ void benchmarkMain() {
 
   for (const BenchmarkCode &Conf : Configurations) {
     InstructionBenchmark Result = ExitOnErr(Runner->runConfiguration(
-        Conf, NumRepetitions, *Repetitor, DumpObjectToDisk));
+        Conf, NumRepetitions, Repetitors, DumpObjectToDisk));
     ExitOnFileError(BenchmarkFile, Result.writeYaml(State, BenchmarkFile));
   }
   exegesis::pfm::pfmTerminate();
diff --git a/llvm/tools/llvm-ifs/llvm-ifs.cpp b/llvm/tools/llvm-ifs/llvm-ifs.cpp
index 3b0d2ee725ffa..0d1a7518dad3b 100644
--- a/llvm/tools/llvm-ifs/llvm-ifs.cpp
+++ b/llvm/tools/llvm-ifs/llvm-ifs.cpp
@@ -26,6 +26,7 @@
 #include "llvm/TextAPI/MachO/TextAPIWriter.h"
 #include <set>
 #include <string>
+#include <vector>
 
 using namespace llvm;
 using namespace llvm::yaml;
@@ -34,8 +35,8 @@ using namespace llvm::MachO;
 #define DEBUG_TYPE "llvm-ifs"
 
 namespace {
-const VersionTuple IFSVersionCurrent(1, 2);
-}
+const VersionTuple IFSVersionCurrent(2, 0);
+} // end anonymous namespace
 
 static cl::opt<std::string> Action("action", cl::desc("<llvm-ifs action>"),
                                    cl::value_desc("write-ifs | write-bin"),
@@ -76,6 +77,7 @@ std::string getTypeName(IFSSymbolType Type) {
 }
 
 struct IFSSymbol {
+  IFSSymbol() = default;
   IFSSymbol(std::string SymbolName) : Name(SymbolName) {}
   std::string Name;
   uint64_t Size;
@@ -85,6 +87,8 @@ struct IFSSymbol {
   bool operator<(const IFSSymbol &RHS) const { return Name < RHS.Name; }
 };
 
+LLVM_YAML_IS_SEQUENCE_VECTOR(IFSSymbol)
+
 namespace llvm {
 namespace yaml {
 /// YAML traits for IFSSymbolType.
@@ -124,6 +128,7 @@ template <> struct ScalarTraits<VersionTuple> {
 /// YAML traits for IFSSymbol.
 template <> struct MappingTraits<IFSSymbol> {
   static void mapping(IO &IO, IFSSymbol &Symbol) {
+    IO.mapRequired("Name", Symbol.Name);
     IO.mapRequired("Type", Symbol.Type);
     // The need for symbol size depends on the symbol type.
     if (Symbol.Type == IFSSymbolType::NoType)
@@ -140,20 +145,6 @@ template <> struct MappingTraits<IFSSymbol> {
   static const bool flow = true;
 };
 
-/// YAML traits for set of IFSSymbols.
-template <> struct CustomMappingTraits<std::set<IFSSymbol>> {
-  static void inputOne(IO &IO, StringRef Key, std::set<IFSSymbol> &Set) {
-    std::string Name = Key.str();
-    IFSSymbol Sym(Name);
-    IO.mapRequired(Name.c_str(), Sym);
-    Set.insert(Sym);
-  }
-
-  static void output(IO &IO, std::set<IFSSymbol> &Set) {
-    for (auto &Sym : Set)
-      IO.mapRequired(Sym.Name.c_str(), const_cast<IFSSymbol &>(Sym));
-  }
-};
 } // namespace yaml
 } // namespace llvm
 
@@ -167,7 +158,7 @@ class IFSStub {
   std::string ObjectFileFormat;
   Optional<std::string> SOName;
   std::vector<std::string> NeededLibs;
-  std::set<IFSSymbol> Symbols;
+  std::vector<IFSSymbol> Symbols;
 
   IFSStub() = default;
   IFSStub(const IFSStub &Stub)
@@ -186,14 +177,18 @@ namespace yaml {
 /// YAML traits for IFSStub objects.
 template <> struct MappingTraits<IFSStub> {
   static void mapping(IO &IO, IFSStub &Stub) {
-    if (!IO.mapTag("!experimental-ifs-v1", true))
+    if (!IO.mapTag("!experimental-ifs-v2", true))
       IO.setError("Not a .ifs YAML file.");
+
+    auto OldContext = IO.getContext();
+    IO.setContext(&Stub);
     IO.mapRequired("IfsVersion", Stub.IfsVersion);
     IO.mapOptional("Triple", Stub.Triple);
     IO.mapOptional("ObjectFileFormat", Stub.ObjectFileFormat);
     IO.mapOptional("SOName", Stub.SOName);
     IO.mapOptional("NeededLibs", Stub.NeededLibs);
     IO.mapRequired("Symbols", Stub.Symbols);
+    IO.setContext(&OldContext);
   }
 };
 } // namespace yaml
@@ -218,7 +213,7 @@ static Expected<std::unique_ptr<IFSStub>> readInputFile(StringRef FilePath) {
   return std::move(Stub);
 }
 
-int writeTbdStub(const llvm::Triple &T, const std::set<IFSSymbol> &Symbols,
+int writeTbdStub(const llvm::Triple &T, const std::vector<IFSSymbol> &Symbols,
                  const StringRef Format, raw_ostream &Out) {
 
   auto PlatformKindOrError =
@@ -280,7 +275,7 @@ int writeTbdStub(const llvm::Triple &T, const std::set<IFSSymbol> &Symbols,
   return 0;
 }
 
-int writeElfStub(const llvm::Triple &T, const std::set<IFSSymbol> &Symbols,
+int writeElfStub(const llvm::Triple &T, const std::vector<IFSSymbol> &Symbols,
                  const StringRef Format, raw_ostream &Out) {
   SmallString<0> Storage;
   Storage.clear();
@@ -387,8 +382,8 @@ int writeIfso(const IFSStub &Stub, bool IsWriteIfs, raw_ostream &Out) {
 
 // TODO: Drop ObjectFileFormat, it can be subsumed from the triple.
 // New Interface Stubs Yaml Format:
-// --- !experimental-ifs-v1
-// IfsVersion:      1.0
+// --- !experimental-ifs-v2
+// IfsVersion: 2.0
 // Triple:          <llvm triple>
 // ObjectFileFormat: <ELF | others not yet supported>
 // Symbols:
@@ -517,7 +512,7 @@ int main(int argc, char *argv[]) {
     }
 
   for (auto &Entry : SymbolMap)
-    Stub.Symbols.insert(Entry.second);
+    Stub.Symbols.push_back(Entry.second);
 
   std::error_code SysErr;
 
diff --git a/llvm/tools/llvm-lipo/llvm-lipo.cpp b/llvm/tools/llvm-lipo/llvm-lipo.cpp
index e75327ee53abe..8c2740d8c94d1 100644
--- a/llvm/tools/llvm-lipo/llvm-lipo.cpp
+++ b/llvm/tools/llvm-lipo/llvm-lipo.cpp
@@ -763,7 +763,6 @@ static void extractSlice(ArrayRef<OwningBinary<Binary>> InputBinaries,
     reportError("input file " +
                 InputBinaries.front().getBinary()->getFileName() +
                 " must be a fat file when the -extract option is specified");
-    exit(EXIT_FAILURE);
   }
 
   SmallVector<std::unique_ptr<MachOObjectFile>, 2> ExtractedObjects;
diff --git a/llvm/tools/llvm-lto2/llvm-lto2.cpp b/llvm/tools/llvm-lto2/llvm-lto2.cpp
index dda596be0ce66..6d78082f980e7 100644
--- a/llvm/tools/llvm-lto2/llvm-lto2.cpp
+++ b/llvm/tools/llvm-lto2/llvm-lto2.cpp
@@ -68,9 +68,10 @@ static cl::opt<bool>
                                        "distributed backend case"));
 
 // Default to using all available threads in the system, but using only one
-// thread per core, as indicated by the usage of
-// heavyweight_hardware_concurrency() in the InProcessThinBackend constructor.
-static cl::opt<int> Threads("thinlto-threads", cl::init(0));
+// thread per core (no SMT).
+// Use -thinlto-threads=all to use hardware_concurrency() instead, which means
+// to use all hardware threads or cores in the system.
+static cl::opt<std::string> Threads("thinlto-threads");
 
 static cl::list<std::string> SymbolResolutions(
     "r",
@@ -286,7 +287,8 @@ static int run(int argc, char **argv) {
                                             /* LinkedObjectsFile */ nullptr,
                                             /* OnWrite */ {});
   else
-    Backend = createInProcessThinBackend(Threads);
+    Backend = createInProcessThinBackend(
+        llvm::heavyweight_hardware_concurrency(Threads));
   LTO Lto(std::move(Conf), std::move(Backend));
 
   bool HasErrors = false;
diff --git a/llvm/tools/llvm-objdump/CMakeLists.txt b/llvm/tools/llvm-objdump/CMakeLists.txt
index d8f320e83196b..91c28dda06020 100644
--- a/llvm/tools/llvm-objdump/CMakeLists.txt
+++ b/llvm/tools/llvm-objdump/CMakeLists.txt
@@ -20,6 +20,7 @@ add_llvm_tool(llvm-objdump
   ELFDump.cpp
   MachODump.cpp
   WasmDump.cpp
+  XCOFFDump.cpp
   )
 
 if(HAVE_LIBXAR)
diff --git a/llvm/tools/llvm-objdump/COFFDump.cpp b/llvm/tools/llvm-objdump/COFFDump.cpp
index 60b0f5a3cbd15..aea0d266f98dd 100644
--- a/llvm/tools/llvm-objdump/COFFDump.cpp
+++ b/llvm/tools/llvm-objdump/COFFDump.cpp
@@ -14,6 +14,8 @@
 ///
 //===----------------------------------------------------------------------===//
 
+#include "COFFDump.h"
+
 #include "llvm-objdump.h"
 #include "llvm/Demangle/Demangle.h"
 #include "llvm/Object/COFF.h"
@@ -24,10 +26,10 @@
 #include "llvm/Support/WithColor.h"
 #include "llvm/Support/raw_ostream.h"
 
+using namespace llvm;
 using namespace llvm::object;
 using namespace llvm::Win64EH;
 
-namespace llvm {
 // Returns the name of the unwind code.
 static StringRef getUnwindCodeTypeName(uint8_t Code) {
   switch(Code) {
@@ -473,9 +475,9 @@ static bool getPDataSection(const COFFObjectFile *Obj,
   return false;
 }
 
-Error getCOFFRelocationValueString(const COFFObjectFile *Obj,
-                                         const RelocationRef &Rel,
-                                         SmallVectorImpl<char> &Result) {
+Error objdump::getCOFFRelocationValueString(const COFFObjectFile *Obj,
+                                            const RelocationRef &Rel,
+                                            SmallVectorImpl<char> &Result) {
   symbol_iterator SymI = Rel.getSymbol();
   Expected<StringRef> SymNameOrErr = SymI->getName();
   if (!SymNameOrErr)
@@ -596,7 +598,7 @@ static void printRuntimeFunctionRels(const COFFObjectFile *Obj,
   printWin64EHUnwindInfo(UI);
 }
 
-void printCOFFUnwindInfo(const COFFObjectFile *Obj) {
+void objdump::printCOFFUnwindInfo(const COFFObjectFile *Obj) {
   if (Obj->getMachine() != COFF::IMAGE_FILE_MACHINE_AMD64) {
     WithColor::error(errs(), "llvm-objdump")
         << "unsupported image machine type "
@@ -625,7 +627,7 @@ void printCOFFUnwindInfo(const COFFObjectFile *Obj) {
   }
 }
 
-void printCOFFFileHeader(const object::ObjectFile *Obj) {
+void objdump::printCOFFFileHeader(const object::ObjectFile *Obj) {
   const COFFObjectFile *file = dyn_cast<const COFFObjectFile>(Obj);
   printTLSDirectory(file);
   printLoadConfiguration(file);
@@ -633,7 +635,7 @@ void printCOFFFileHeader(const object::ObjectFile *Obj) {
   printExportTable(file);
 }
 
-void printCOFFSymbolTable(const object::COFFImportFile *i) {
+void objdump::printCOFFSymbolTable(const object::COFFImportFile *i) {
   unsigned Index = 0;
   bool IsCode = i->getCOFFImportHeader()->getType() == COFF::IMPORT_CODE;
 
@@ -656,7 +658,7 @@ void printCOFFSymbolTable(const object::COFFImportFile *i) {
   }
 }
 
-void printCOFFSymbolTable(const COFFObjectFile *coff) {
+void objdump::printCOFFSymbolTable(const COFFObjectFile *coff) {
   for (unsigned SI = 0, SE = coff->getNumberOfSymbols(); SI != SE; ++SI) {
     Expected<COFFSymbolRef> Symbol = coff->getSymbol(SI);
     if (!Symbol)
@@ -735,4 +737,3 @@ void printCOFFSymbolTable(const COFFObjectFile *coff) {
     }
   }
 }
-} // namespace llvm
diff --git a/llvm/tools/llvm-objdump/COFFDump.h b/llvm/tools/llvm-objdump/COFFDump.h
new file mode 100644
index 0000000000000..21f97bdeb83cd
--- /dev/null
+++ b/llvm/tools/llvm-objdump/COFFDump.h
@@ -0,0 +1,37 @@
+//===-- COFFDump.h ----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TOOLS_LLVM_OBJDUMP_COFFDUMP_H
+#define LLVM_TOOLS_LLVM_OBJDUMP_COFFDUMP_H
+
+#include "llvm/ADT/SmallVector.h"
+
+namespace llvm {
+
+class Error;
+
+namespace object {
+class COFFObjectFile;
+class COFFImportFile;
+class ObjectFile;
+class RelocationRef;
+} // namespace object
+
+namespace objdump {
+Error getCOFFRelocationValueString(const object::COFFObjectFile *Obj,
+                                   const object::RelocationRef &Rel,
+                                   llvm::SmallVectorImpl<char> &Result);
+
+void printCOFFUnwindInfo(const object::COFFObjectFile *O);
+void printCOFFFileHeader(const object::ObjectFile *O);
+void printCOFFSymbolTable(const object::COFFImportFile *I);
+void printCOFFSymbolTable(const object::COFFObjectFile *O);
+} // namespace objdump
+} // namespace llvm
+
+#endif
diff --git a/llvm/tools/llvm-objdump/XCOFFDump.cpp b/llvm/tools/llvm-objdump/XCOFFDump.cpp
new file mode 100644
index 0000000000000..67123910a9d20
--- /dev/null
+++ b/llvm/tools/llvm-objdump/XCOFFDump.cpp
@@ -0,0 +1,34 @@
+//===-- XCOFFDump.cpp - XCOFF-specific dumper -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file implements the XCOFF-specific dumper for llvm-objdump.
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm-objdump.h"
+#include "llvm/Object/XCOFFObjectFile.h"
+
+using namespace llvm::object;
+
+llvm::Error llvm::getXCOFFRelocationValueString(const XCOFFObjectFile *Obj,
+                                                const RelocationRef &Rel,
+                                                SmallVectorImpl<char> &Result) {
+  symbol_iterator SymI = Rel.getSymbol();
+  if (SymI == Obj->symbol_end())
+    return make_error<GenericBinaryError>(
+        "invalid symbol reference in relocation entry",
+        object_error::parse_failed);
+
+  Expected<StringRef> SymNameOrErr = SymI->getName();
+  if (!SymNameOrErr)
+    return SymNameOrErr.takeError();
+  StringRef SymName = *SymNameOrErr;
+  Result.append(SymName.begin(), SymName.end());
+  return Error::success();
+}
diff --git a/llvm/tools/llvm-objdump/llvm-objdump.cpp b/llvm/tools/llvm-objdump/llvm-objdump.cpp
index 1cfd06d2cef5e..6f284159cb216 100644
--- a/llvm/tools/llvm-objdump/llvm-objdump.cpp
+++ b/llvm/tools/llvm-objdump/llvm-objdump.cpp
@@ -16,6 +16,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm-objdump.h"
+#include "COFFDump.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetOperations.h"
@@ -45,6 +46,7 @@
 #include "llvm/Object/MachO.h"
 #include "llvm/Object/MachOUniversal.h"
 #include "llvm/Object/ObjectFile.h"
+#include "llvm/Object/XCOFFObjectFile.h"
 #include "llvm/Object/Wasm.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
@@ -71,6 +73,7 @@
 #include <utility>
 
 using namespace llvm::object;
+using namespace llvm::objdump;
 
 namespace llvm {
 
@@ -500,6 +503,8 @@ static Error getRelocationValueString(const RelocationRef &Rel,
     return getWasmRelocationValueString(Wasm, Rel, Result);
   if (auto *MachO = dyn_cast<MachOObjectFile>(Obj))
     return getMachORelocationValueString(MachO, Rel, Result);
+  if (auto *XCOFF = dyn_cast<XCOFFObjectFile>(Obj))
+    return getXCOFFRelocationValueString(XCOFF, Rel, Result);
   llvm_unreachable("unknown object file format");
 }
 
@@ -735,9 +740,15 @@ class PrettyPrinter {
     unsigned Column = OS.tell() - Start;
     OS.indent(Column < TabStop - 1 ? TabStop - 1 - Column : 7 - Column % 8);
 
-    if (MI)
-      IP.printInst(MI, Address.Address, "", STI, OS);
-    else
+    if (MI) {
+      // See MCInstPrinter::printInst. On targets where a PC relative immediate
+      // is relative to the next instruction and the length of a MCInst is
+      // difficult to measure (x86), this is the address of the next
+      // instruction.
+      uint64_t Addr =
+          Address.Address + (STI.getTargetTriple().isX86() ? Bytes.size() : 0);
+      IP.printInst(MI, Addr, "", STI, OS);
+    } else
       OS << "\t<unknown>";
   }
 };
@@ -1202,11 +1213,13 @@ static void disassembleObject(const Target *TheTarget, const ObjectFile *Obj,
   StringSaver Saver(A);
   addPltEntries(Obj, AllSymbols, Saver);
 
-  // Create a mapping from virtual address to section.
+  // Create a mapping from virtual address to section. An empty section can
+  // cause more than one section at the same address. Use a stable sort to
+  // stabilize the output.
   std::vector<std::pair<uint64_t, SectionRef>> SectionAddresses;
   for (SectionRef Sec : Obj->sections())
     SectionAddresses.emplace_back(Sec.getAddress(), Sec);
-  array_pod_sort(SectionAddresses.begin(), SectionAddresses.end());
+  stable_sort(SectionAddresses);
 
   // Linked executables (.exe and .dll files) typically don't include a real
   // symbol table but they might contain an export table.
@@ -1236,11 +1249,12 @@ static void disassembleObject(const Target *TheTarget, const ObjectFile *Obj,
   }
 
   // Sort all the symbols, this allows us to use a simple binary search to find
-  // a symbol near an address.
+  // Multiple symbols can have the same address. Use a stable sort to stabilize
+  // the output.
   StringSet<> FoundDisasmSymbolSet;
   for (std::pair<const SectionRef, SectionSymbolsTy> &SecSyms : AllSymbols)
-    array_pod_sort(SecSyms.second.begin(), SecSyms.second.end());
-  array_pod_sort(AbsoluteSymbols.begin(), AbsoluteSymbols.end());
+    stable_sort(SecSyms.second);
+  stable_sort(AbsoluteSymbols);
 
   for (const SectionRef &Section : ToolSectionFilter(*Obj)) {
     if (FilterSections.empty() && !DisassembleAll &&
@@ -1421,7 +1435,7 @@ static void disassembleObject(const Target *TheTarget, const ObjectFile *Obj,
         // them. Otherwise we might want to skip zero bytes we see.
         if (!DisassembleZeroes) {
           uint64_t MaxOffset = End - Index;
-          // For -reloc: print zero blocks patched by relocations, so that
+          // For --reloc: print zero blocks patched by relocations, so that
           // relocations can be shown in the dump.
           if (RelCur != RelEnd)
             MaxOffset = RelCur->getOffset() - Index;
@@ -1632,6 +1646,7 @@ static void disassembleObject(const ObjectFile *Obj, bool InlineRelocs) {
     reportError(Obj->getFileName(),
                 "no instruction printer for target " + TripleName);
   IP->setPrintImmHex(PrintImmHex);
+  IP->setPrintBranchImmAsAddress(true);
 
   PrettyPrinter &PIP = selectPrettyPrinter(Triple(TripleName));
   SourcePrinter SP(Obj, TheTarget->getName());
diff --git a/llvm/tools/llvm-objdump/llvm-objdump.h b/llvm/tools/llvm-objdump/llvm-objdump.h
index 43ce02ae0bc27..f834f11def396 100644
--- a/llvm/tools/llvm-objdump/llvm-objdump.h
+++ b/llvm/tools/llvm-objdump/llvm-objdump.h
@@ -18,13 +18,12 @@ namespace llvm {
 class StringRef;
 
 namespace object {
-class COFFObjectFile;
-class COFFImportFile;
 class ELFObjectFileBase;
 class ELFSectionRef;
 class MachOObjectFile;
 class MachOUniversalBinary;
 class RelocationRef;
+class XCOFFObjectFile;
 }
 
 extern cl::opt<bool> Demangle;
@@ -96,22 +95,21 @@ SectionFilter ToolSectionFilter(llvm::object::ObjectFile const &O,
 Error getELFRelocationValueString(const object::ELFObjectFileBase *Obj,
                                   const object::RelocationRef &Rel,
                                   llvm::SmallVectorImpl<char> &Result);
-Error getCOFFRelocationValueString(const object::COFFObjectFile *Obj,
-                                   const object::RelocationRef &Rel,
-                                   llvm::SmallVectorImpl<char> &Result);
 Error getWasmRelocationValueString(const object::WasmObjectFile *Obj,
                                    const object::RelocationRef &RelRef,
                                    llvm::SmallVectorImpl<char> &Result);
 Error getMachORelocationValueString(const object::MachOObjectFile *Obj,
                                     const object::RelocationRef &RelRef,
                                     llvm::SmallVectorImpl<char> &Result);
+Error getXCOFFRelocationValueString(const object::XCOFFObjectFile *Obj,
+                                    const object::RelocationRef &RelRef,
+                                    llvm::SmallVectorImpl<char> &Result);
 
 uint64_t getELFSectionLMA(const object::ELFSectionRef& Sec);
 
 bool isRelocAddressLess(object::RelocationRef A, object::RelocationRef B);
 void parseInputMachO(StringRef Filename);
 void parseInputMachO(object::MachOUniversalBinary *UB);
-void printCOFFUnwindInfo(const object::COFFObjectFile *O);
 void printMachOUnwindInfo(const object::MachOObjectFile *O);
 void printMachOExportsTrie(const object::MachOObjectFile *O);
 void printMachORebaseTable(object::MachOObjectFile *O);
@@ -121,9 +119,6 @@ void printMachOWeakBindTable(object::MachOObjectFile *O);
 void printELFFileHeader(const object::ObjectFile *O);
 void printELFDynamicSection(const object::ObjectFile *Obj);
 void printELFSymbolVersionInfo(const object::ObjectFile *Obj);
-void printCOFFFileHeader(const object::ObjectFile *O);
-void printCOFFSymbolTable(const object::COFFImportFile *I);
-void printCOFFSymbolTable(const object::COFFObjectFile *O);
 void printMachOFileHeader(const object::ObjectFile *O);
 void printMachOLoadCommands(const object::ObjectFile *O);
 void printWasmFileHeader(const object::ObjectFile *O);
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index f05c7e637cd55..90ce8400c845c 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -448,7 +448,7 @@ static void handleExtBinaryWriter(sampleprof::SampleProfileWriter &Writer,
                                   ProfileFormat OutputFormat,
                                   MemoryBuffer *Buffer,
                                   sampleprof::ProfileSymbolList &WriterList,
-                                  bool CompressAllSections) {
+                                  bool CompressAllSections, bool UseMD5) {
   populateProfileSymbolList(Buffer, WriterList);
   if (WriterList.size() > 0 && OutputFormat != PF_Ext_Binary)
     warn("Profile Symbol list is not empty but the output format is not "
@@ -457,22 +457,24 @@ static void handleExtBinaryWriter(sampleprof::SampleProfileWriter &Writer,
   Writer.setProfileSymbolList(&WriterList);
 
   if (CompressAllSections) {
-    if (OutputFormat != PF_Ext_Binary) {
+    if (OutputFormat != PF_Ext_Binary)
       warn("-compress-all-section is ignored. Specify -extbinary to enable it");
-    } else {
-      auto ExtBinaryWriter =
-          static_cast<sampleprof::SampleProfileWriterExtBinary *>(&Writer);
-      ExtBinaryWriter->setToCompressAllSections();
-    }
+    else
+      Writer.setToCompressAllSections();
+  }
+  if (UseMD5) {
+    if (OutputFormat != PF_Ext_Binary)
+      warn("-use-md5 is ignored. Specify -extbinary to enable it");
+    else
+      Writer.setUseMD5();
   }
 }
 
-static void mergeSampleProfile(const WeightedFileVector &Inputs,
-                               SymbolRemapper *Remapper,
-                               StringRef OutputFilename,
-                               ProfileFormat OutputFormat,
-                               StringRef ProfileSymbolListFile,
-                               bool CompressAllSections, FailureMode FailMode) {
+static void
+mergeSampleProfile(const WeightedFileVector &Inputs, SymbolRemapper *Remapper,
+                   StringRef OutputFilename, ProfileFormat OutputFormat,
+                   StringRef ProfileSymbolListFile, bool CompressAllSections,
+                   bool UseMD5, FailureMode FailMode) {
   using namespace sampleprof;
   StringMap<FunctionSamples> ProfileMap;
   SmallVector<std::unique_ptr<sampleprof::SampleProfileReader>, 5> Readers;
@@ -529,7 +531,7 @@ static void mergeSampleProfile(const WeightedFileVector &Inputs,
   // Make sure Buffer lives as long as WriterList.
   auto Buffer = getInputFileBuf(ProfileSymbolListFile);
   handleExtBinaryWriter(*Writer, OutputFormat, Buffer.get(), WriterList,
-                        CompressAllSections);
+                        CompressAllSections, UseMD5);
   Writer->write(ProfileMap);
 }
 
@@ -657,6 +659,10 @@ static int merge_main(int argc, const char *argv[]) {
       "compress-all-sections", cl::init(false), cl::Hidden,
       cl::desc("Compress all sections when writing the profile (only "
                "meaningful for -extbinary)"));
+  cl::opt<bool> UseMD5(
+      "use-md5", cl::init(false), cl::Hidden,
+      cl::desc("Choose to use MD5 to represent string in name table (only "
+               "meaningful for -extbinary)"));
 
   cl::ParseCommandLineOptions(argc, argv, "LLVM profile data merger\n");
 
@@ -691,7 +697,7 @@ static int merge_main(int argc, const char *argv[]) {
   else
     mergeSampleProfile(WeightedInputs, Remapper.get(), OutputFilename,
                        OutputFormat, ProfileSymbolListFile, CompressAllSections,
-                       FailureMode);
+                       UseMD5, FailureMode);
 
   return 0;
 }
diff --git a/llvm/tools/llvm-rc/ResourceFileWriter.cpp b/llvm/tools/llvm-rc/ResourceFileWriter.cpp
index 44b3fe143ce89..615474b4a8646 100644
--- a/llvm/tools/llvm-rc/ResourceFileWriter.cpp
+++ b/llvm/tools/llvm-rc/ResourceFileWriter.cpp
@@ -1180,8 +1180,10 @@ Error ResourceFileWriter::writeMenuDefinition(
 
   if (auto *MenuItemPtr = dyn_cast<MenuItem>(DefPtr)) {
     writeInt<uint16_t>(Flags);
-    RETURN_IF_ERROR(
-        checkNumberFits<uint16_t>(MenuItemPtr->Id, "MENUITEM action ID"));
+    // Some resource files use -1, i.e. UINT32_MAX, for empty menu items.
+    if (MenuItemPtr->Id != static_cast<uint32_t>(-1))
+      RETURN_IF_ERROR(
+          checkNumberFits<uint16_t>(MenuItemPtr->Id, "MENUITEM action ID"));
     writeInt<uint16_t>(MenuItemPtr->Id);
     RETURN_IF_ERROR(writeCString(MenuItemPtr->Name));
     return Error::success();
diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index defa4a3cc1095..ca05f99aa715e 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -52,6 +52,8 @@
 #include "llvm/Support/LEB128.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/MipsABIFlags.h"
+#include "llvm/Support/RISCVAttributeParser.h"
+#include "llvm/Support/RISCVAttributes.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
@@ -305,6 +307,8 @@ template <typename ELFT> class ELFDumper : public ObjDumper {
   };
   mutable SmallVector<Optional<VersionEntry>, 16> VersionMap;
 
+  std::unordered_set<std::string> Warnings;
+
 public:
   Elf_Dyn_Range dynamic_table() const {
     // A valid .dynamic section contains an array of entries terminated
@@ -367,6 +371,9 @@ template <typename ELFT> class ELFDumper : public ObjDumper {
 
   Expected<std::pair<const Elf_Sym *, std::string>>
   getRelocationTarget(const Elf_Shdr *SymTab, const Elf_Rela &R) const;
+
+  std::function<Error(const Twine &Msg)> WarningHandler;
+  void reportUniqueWarning(Error Err) const;
 };
 
 template <class ELFT>
@@ -459,12 +466,12 @@ ELFDumper<ELFT>::getVersionTable(const Elf_Shdr *Sec, ArrayRef<Elf_Sym> *SymTab,
   Expected<std::pair<ArrayRef<Elf_Sym>, StringRef>> SymTabOrErr =
       getLinkAsSymtab(Obj, Sec, SecNdx, SHT_DYNSYM);
   if (!SymTabOrErr) {
-    ELFDumperStyle->reportUniqueWarning(SymTabOrErr.takeError());
+    reportUniqueWarning(SymTabOrErr.takeError());
     return *VersionsOrErr;
   }
 
   if (SymTabOrErr->first.size() != VersionsOrErr->size())
-    ELFDumperStyle->reportUniqueWarning(
+    reportUniqueWarning(
         createError("SHT_GNU_versym section with index " + Twine(SecNdx) +
                     ": the number of entries (" + Twine(VersionsOrErr->size()) +
                     ") does not match the number of symbols (" +
@@ -578,7 +585,7 @@ ELFDumper<ELFT>::getVersionDependencies(const Elf_Shdr *Sec) const {
   StringRef StrTab;
   Expected<StringRef> StrTabOrErr = getLinkAsStrtab(Obj, Sec, SecNdx);
   if (!StrTabOrErr)
-    ELFDumperStyle->reportUniqueWarning(StrTabOrErr.takeError());
+    reportUniqueWarning(StrTabOrErr.takeError());
   else
     StrTab = *StrTabOrErr;
 
@@ -707,14 +714,6 @@ template <typename ELFT> class DumpStyle {
 
   DumpStyle(ELFDumper<ELFT> *Dumper) : Dumper(Dumper) {
     FileName = this->Dumper->getElfObject()->getFileName();
-
-    // Dumper reports all non-critical errors as warnings.
-    // It does not print the same warning more than once.
-    WarningHandler = [this](const Twine &Msg) {
-      if (Warnings.insert(Msg.str()).second)
-        reportWarning(createError(Msg), FileName);
-      return Error::success();
-    };
   }
 
   virtual ~DumpStyle() = default;
@@ -767,14 +766,11 @@ template <typename ELFT> class DumpStyle {
   virtual void printMipsABIFlags(const ELFObjectFile<ELFT> *Obj) = 0;
   const ELFDumper<ELFT> *dumper() const { return Dumper; }
 
-  void reportUniqueWarning(Error Err) const;
-
 protected:
-  std::function<Error(const Twine &Msg)> WarningHandler;
+  void reportUniqueWarning(Error Err) const;
   StringRef FileName;
 
 private:
-  std::unordered_set<std::string> Warnings;
   const ELFDumper<ELFT> *Dumper;
 };
 
@@ -898,13 +894,18 @@ template <typename ELFT> class GNUStyle : public DumpStyle<ELFT> {
 };
 
 template <class ELFT>
-void DumpStyle<ELFT>::reportUniqueWarning(Error Err) const {
+void ELFDumper<ELFT>::reportUniqueWarning(Error Err) const {
   handleAllErrors(std::move(Err), [&](const ErrorInfoBase &EI) {
     cantFail(WarningHandler(EI.message()),
              "WarningHandler should always return ErrorSuccess");
   });
 }
 
+template <class ELFT>
+void DumpStyle<ELFT>::reportUniqueWarning(Error Err) const {
+  this->dumper()->reportUniqueWarning(std::move(Err));
+}
+
 template <typename ELFT> class LLVMStyle : public DumpStyle<ELFT> {
 public:
   TYPEDEF_ELF_TYPES(ELFT)
@@ -1156,12 +1157,12 @@ std::string ELFDumper<ELFT>::getFullSymbolName(const Elf_Sym *Symbol,
     Expected<unsigned> SectionIndex =
         getSymbolSectionIndex(Symbol, Syms.begin());
     if (!SectionIndex) {
-      ELFDumperStyle->reportUniqueWarning(SectionIndex.takeError());
+      reportUniqueWarning(SectionIndex.takeError());
       return "<?>";
     }
     Expected<StringRef> NameOrErr = getSymbolSectionName(Symbol, *SectionIndex);
     if (!NameOrErr) {
-      ELFDumperStyle->reportUniqueWarning(NameOrErr.takeError());
+      reportUniqueWarning(NameOrErr.takeError());
       return ("<section " + Twine(*SectionIndex) + ">").str();
     }
     return std::string(*NameOrErr);
@@ -1173,7 +1174,7 @@ std::string ELFDumper<ELFT>::getFullSymbolName(const Elf_Sym *Symbol,
   bool IsDefault;
   Expected<StringRef> VersionOrErr = getSymbolVersion(&*Symbol, IsDefault);
   if (!VersionOrErr) {
-    ELFDumperStyle->reportUniqueWarning(VersionOrErr.takeError());
+    reportUniqueWarning(VersionOrErr.takeError());
     return SymbolName + "@<corrupt>";
   }
 
@@ -1984,6 +1985,14 @@ ELFDumper<ELFT>::ELFDumper(const object::ELFObjectFile<ELFT> *ObjF,
     : ObjDumper(Writer), ObjF(ObjF), DynRelRegion(ObjF->getFileName()),
       DynRelaRegion(ObjF->getFileName()), DynRelrRegion(ObjF->getFileName()),
       DynPLTRelRegion(ObjF->getFileName()), DynamicTable(ObjF->getFileName()) {
+  // Dumper reports all non-critical errors as warnings.
+  // It does not print the same warning more than once.
+  WarningHandler = [this](const Twine &Msg) {
+    if (Warnings.insert(Msg.str()).second)
+      reportWarning(createError(Msg), this->ObjF->getFileName());
+    return Error::success();
+  };
+
   if (opts::Output == opts::GNU)
     ELFDumperStyle.reset(new GNUStyle<ELFT>(Writer, this));
   else
@@ -2174,7 +2183,7 @@ void ELFDumper<ELFT>::parseDynamicTable(const ELFFile<ELFT> *Obj) {
     // locate .dynsym at runtime. The location we find in the section header
     // and the location we find here should match.
     if (DynSymFromTable && DynSymFromTable->Addr != DynSymRegion->Addr)
-      ELFDumperStyle->reportUniqueWarning(
+      reportUniqueWarning(
           createError("SHT_DYNSYM section header and DT_SYMTAB disagree about "
                       "the location of the dynamic symbol table"));
 
@@ -2184,7 +2193,7 @@ void ELFDumper<ELFT>::parseDynamicTable(const ELFFile<ELFT> *Obj) {
     // according to the section header.
     if (HashTable &&
         HashTable->nchain != DynSymRegion->Size / DynSymRegion->EntSize)
-      ELFDumperStyle->reportUniqueWarning(createError(
+      reportUniqueWarning(createError(
           "hash table nchain (" + Twine(HashTable->nchain) +
           ") differs from symbol count derived from SHT_DYNSYM section "
           "header (" +
@@ -2598,9 +2607,35 @@ template <class ELFT> void ELFDumper<ELFT>::printNeededLibraries() {
     W.startLine() << L << "\n";
 }
 
+template <class ELFT>
+static bool checkHashTable(const ELFFile<ELFT> *Obj,
+                           const typename ELFT::Hash *H, StringRef FileName) {
+  auto WarnAndReturn = [&](uint64_t Off, const Twine &Msg = "") {
+    reportWarning(createError("the hash table at offset 0x" +
+                              Twine::utohexstr(Off) +
+                              " goes past the end of the file (0x" +
+                              Twine::utohexstr(Obj->getBufSize()) + ")" + Msg),
+                  FileName);
+    return false;
+  };
+
+  // Each SHT_HASH section starts from two 32-bit fields: nbucket and nchain.
+  const unsigned HeaderSize = 2 * sizeof(typename ELFT::Word);
+  const uint64_t SecOffset = (const uint8_t *)H - Obj->base();
+  if (Obj->getBufSize() - SecOffset < HeaderSize)
+    return WarnAndReturn(SecOffset);
+
+  if (Obj->getBufSize() - SecOffset - HeaderSize <
+      ((uint64_t)H->nbucket + H->nchain) * sizeof(typename ELFT::Word))
+    return WarnAndReturn(SecOffset, ", nbucket = " + Twine(H->nbucket) +
+                                        ", nchain = " + Twine(H->nchain));
+  return true;
+}
+
 template <typename ELFT> void ELFDumper<ELFT>::printHashTable() {
   DictScope D(W, "HashTable");
-  if (!HashTable)
+  if (!HashTable ||
+      !checkHashTable(ObjF->getELFFile(), HashTable, ObjF->getFileName()))
     return;
   W.printNumber("Num Buckets", HashTable->nbucket);
   W.printNumber("Num Chains", HashTable->nchain);
@@ -2671,6 +2706,7 @@ template <class ELFT> void ELFDumper<ELFT>::printArchSpecificInfo() {
   const ELFFile<ELFT> *Obj = ObjF->getELFFile();
   switch (Obj->getHeader()->e_machine) {
   case EM_ARM:
+  case EM_RISCV:
     printAttributes();
     break;
   case EM_MIPS: {
@@ -2691,40 +2727,45 @@ template <class ELFT> void ELFDumper<ELFT>::printArchSpecificInfo() {
   }
 }
 
-template <class ELFT> void ELFDumper<ELFT>::printAttributes() {
-  W.startLine() << "Attributes not implemented.\n";
-}
-
 namespace {
 
-template <> void ELFDumper<ELF32LE>::printAttributes() {
-  const ELFFile<ELF32LE> *Obj = ObjF->getELFFile();
-  if (Obj->getHeader()->e_machine != EM_ARM) {
+template <class ELFT> void ELFDumper<ELFT>::printAttributes() {
+  const ELFFile<ELFT> *Obj = ObjF->getELFFile();
+  if (!Obj->isLE()) {
     W.startLine() << "Attributes not implemented.\n";
     return;
   }
 
+  const unsigned Machine = Obj->getHeader()->e_machine;
+  assert((Machine == EM_ARM || Machine == EM_RISCV) &&
+         "Attributes not implemented.");
+
   DictScope BA(W, "BuildAttributes");
-  for (const ELFO::Elf_Shdr &Sec :
-       unwrapOrError(ObjF->getFileName(), Obj->sections())) {
-    if (Sec.sh_type != ELF::SHT_ARM_ATTRIBUTES)
+  for (const auto &Sec : unwrapOrError(ObjF->getFileName(), Obj->sections())) {
+    if (Sec.sh_type != ELF::SHT_ARM_ATTRIBUTES &&
+        Sec.sh_type != ELF::SHT_RISCV_ATTRIBUTES)
       continue;
 
     ArrayRef<uint8_t> Contents =
         unwrapOrError(ObjF->getFileName(), Obj->getSectionContents(&Sec));
-    if (Contents[0] != ARMBuildAttrs::Format_Version) {
-      errs() << "unrecognised FormatVersion: 0x"
-             << Twine::utohexstr(Contents[0]) << '\n';
+    if (Contents[0] != ELFAttrs::Format_Version) {
+      reportWarning(createError(Twine("unrecognised FormatVersion: 0x") +
+                                Twine::utohexstr(Contents[0])),
+                    ObjF->getFileName());
       continue;
     }
-
     W.printHex("FormatVersion", Contents[0]);
     if (Contents.size() == 1)
       continue;
 
-    // TODO: Print error and delete the redundant FormatVersion check above.
-    if (Error E = ARMAttributeParser(&W).parse(Contents, support::little))
-      consumeError(std::move(E));
+    // TODO: Delete the redundant FormatVersion check above.
+    if (Machine == EM_ARM) {
+      if (Error E = ARMAttributeParser(&W).parse(Contents, support::little))
+        reportWarning(std::move(E), ObjF->getFileName());
+    } else if (Machine == EM_RISCV) {
+      if (Error E = RISCVAttributeParser(&W).parse(Contents, support::little))
+        reportWarning(std::move(E), ObjF->getFileName());
+    }
   }
 }
 
@@ -3562,6 +3603,11 @@ static std::string getSectionTypeString(unsigned Arch, unsigned Type) {
       return "MIPS_ABIFLAGS";
     }
     break;
+  case EM_RISCV:
+    switch (Type) {
+    case SHT_RISCV_ATTRIBUTES:
+      return "RISCV_ATTRIBUTES";
+    }
   }
   switch (Type) {
   case SHT_NULL:
@@ -3677,7 +3723,7 @@ void GNUStyle<ELFT>::printSectionHeaders(const ELFO *Obj) {
   const ELFObjectFile<ELFT> *ElfObj = this->dumper()->getElfObject();
   StringRef SecStrTable = unwrapOrError<StringRef>(
       ElfObj->getFileName(),
-      Obj->getSectionStringTable(Sections, this->WarningHandler));
+      Obj->getSectionStringTable(Sections, this->dumper()->WarningHandler));
   size_t SectionIndex = 0;
   for (const Elf_Shdr &Sec : Sections) {
     Fields[0].Str = to_string(SectionIndex);
@@ -3879,9 +3925,7 @@ template <class ELFT> void GNUStyle<ELFT>::printHashSymbols(const ELFO *Obj) {
   auto StringTable = this->dumper()->getDynamicStringTable();
   auto DynSyms = this->dumper()->dynamic_symbols();
 
-  // Try printing .hash
-  if (auto SysVHash = this->dumper()->getHashTable()) {
-    OS << "\n Symbol table of .hash for image:\n";
+  auto PrintHashTable = [&](const Elf_Hash *SysVHash) {
     if (ELFT::Is64Bits)
       OS << "  Num Buc:    Value          Size   Type   Bind Vis      Ndx Name";
     else
@@ -3910,6 +3954,12 @@ template <class ELFT> void GNUStyle<ELFT>::printHashSymbols(const ELFO *Obj) {
         Visited[Ch] = true;
       }
     }
+  };
+
+  if (const Elf_Hash *SysVHash = this->dumper()->getHashTable()) {
+    OS << "\n Symbol table of .hash for image:\n";
+    if (checkHashTable(Obj, SysVHash, this->FileName))
+      PrintHashTable(SysVHash);
   }
 
   // Try printing .gnu.hash
@@ -4432,6 +4482,9 @@ template <class ELFT>
 void GNUStyle<ELFT>::printHashHistogram(const ELFFile<ELFT> *Obj) {
   // Print histogram for .hash section
   if (const Elf_Hash *HashTable = this->dumper()->getHashTable()) {
+    if (!checkHashTable(Obj, HashTable, this->FileName))
+      return;
+
     size_t NBucket = HashTable->nbucket;
     size_t NChain = HashTable->nchain;
     ArrayRef<Elf_Word> Buckets = HashTable->buckets();
@@ -5845,7 +5898,7 @@ void LLVMStyle<ELFT>::printSectionHeaders(const ELFO *Obj) {
   for (const Elf_Shdr &Sec : Sections) {
     StringRef Name = "<?>";
     if (Expected<StringRef> SecNameOrErr =
-            Obj->getSectionName(&Sec, this->WarningHandler))
+            Obj->getSectionName(&Sec, this->dumper()->WarningHandler))
       Name = *SecNameOrErr;
     else
       this->reportUniqueWarning(SecNameOrErr.takeError());
diff --git a/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp b/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp
index 8f4a93fcc25d2..8d46bd2cb6270 100644
--- a/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp
+++ b/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp
@@ -71,13 +71,15 @@ static cl::alias
     ClPrintInliningAliasInlines("inlines", cl::desc("Alias for -inlining"),
                                 cl::NotHidden, cl::aliasopt(ClPrintInlining));
 
-// -basenames, -s
 static cl::opt<bool> ClBasenames("basenames", cl::init(false),
                                  cl::desc("Strip directory names from paths"));
 static cl::alias ClBasenamesShort("s", cl::desc("Alias for -basenames"),
                                   cl::NotHidden, cl::aliasopt(ClBasenames));
 
-// -demangle, -C, -no-demangle
+static cl::opt<bool>
+    ClRelativenames("relativenames", cl::init(false),
+                    cl::desc("Strip the compilation directory from paths"));
+
 static cl::opt<bool>
 ClDemangle("demangle", cl::init(true), cl::desc("Demangle function names"));
 static cl::alias
@@ -91,7 +93,6 @@ static cl::opt<std::string> ClDefaultArch("default-arch", cl::init(""),
                                           cl::desc("Default architecture "
                                                    "(for multi-arch objects)"));
 
-// -obj, -exe, -e
 static cl::opt<std::string>
 ClBinaryName("obj", cl::init(""),
              cl::desc("Path to object file to be symbolized (if not provided, "
@@ -112,7 +113,6 @@ ClDsymHint("dsym-hint", cl::ZeroOrMore,
            cl::desc("Path to .dSYM bundles to search for debug info for the "
                     "object files"));
 
-// -print-address, -addresses, -a
 static cl::opt<bool>
 ClPrintAddress("print-address", cl::init(false),
                cl::desc("Show address before line information"));
@@ -123,7 +123,6 @@ static cl::alias
 ClPrintAddressAliasA("a", cl::desc("Alias for -print-address"),
                      cl::NotHidden, cl::aliasopt(ClPrintAddress), cl::Grouping);
 
-// -pretty-print, -p
 static cl::opt<bool>
     ClPrettyPrint("pretty-print", cl::init(false),
                   cl::desc("Make the output more human friendly"));
@@ -138,7 +137,6 @@ static cl::opt<int> ClPrintSourceContextLines(
 static cl::opt<bool> ClVerbose("verbose", cl::init(false),
                                cl::desc("Print verbose line info"));
 
-// -adjust-vma
 static cl::opt<uint64_t>
     ClAdjustVMA("adjust-vma", cl::init(0), cl::value_desc("offset"),
                 cl::desc("Add specified offset to object file addresses"));
@@ -197,30 +195,30 @@ static bool parseCommand(StringRef InputString, Command &Cmd,
     // If no cmd, assume it's CODE.
     Cmd = Command::Code;
   }
-  const char *pos = InputString.data();
+  const char *Pos = InputString.data();
   // Skip delimiters and parse input filename (if needed).
   if (ClBinaryName.empty()) {
-    pos += strspn(pos, kDelimiters);
-    if (*pos == '"' || *pos == '\'') {
-      char quote = *pos;
-      pos++;
-      const char *end = strchr(pos, quote);
-      if (!end)
+    Pos += strspn(Pos, kDelimiters);
+    if (*Pos == '"' || *Pos == '\'') {
+      char Quote = *Pos;
+      Pos++;
+      const char *End = strchr(Pos, Quote);
+      if (!End)
         return false;
-      ModuleName = std::string(pos, end - pos);
-      pos = end + 1;
+      ModuleName = std::string(Pos, End - Pos);
+      Pos = End + 1;
     } else {
-      int name_length = strcspn(pos, kDelimiters);
-      ModuleName = std::string(pos, name_length);
-      pos += name_length;
+      int NameLength = strcspn(Pos, kDelimiters);
+      ModuleName = std::string(Pos, NameLength);
+      Pos += NameLength;
     }
   } else {
     ModuleName = ClBinaryName;
   }
   // Skip delimiters and parse module offset.
-  pos += strspn(pos, kDelimiters);
-  int offset_length = strcspn(pos, kDelimiters);
-  return !StringRef(pos, offset_length).getAsInteger(0, ModuleOffset);
+  Pos += strspn(Pos, kDelimiters);
+  int OffsetLength = strcspn(Pos, kDelimiters);
+  return !StringRef(Pos, OffsetLength).getAsInteger(0, ModuleOffset);
 }
 
 static void symbolizeInput(StringRef InputString, LLVMSymbolizer &Symbolizer,
@@ -310,8 +308,12 @@ int main(int argc, char **argv) {
   Opts.DWPName = ClDwpName;
   Opts.DebugFileDirectory = ClDebugFileDirectory;
   Opts.PathStyle = DILineInfoSpecifier::FileLineInfoKind::AbsoluteFilePath;
-  if (ClBasenames)
+  // If both --basenames and --relativenames are specified then pick the last
+  // one.
+  if (ClBasenames.getPosition() > ClRelativenames.getPosition())
     Opts.PathStyle = DILineInfoSpecifier::FileLineInfoKind::BaseNameOnly;
+  else if (ClRelativenames)
+    Opts.PathStyle = DILineInfoSpecifier::FileLineInfoKind::RelativeFilePath;
 
   for (const auto &hint : ClDsymHint) {
     if (sys::path::extension(hint) == ".dSYM") {
diff --git a/llvm/tools/obj2yaml/elf2yaml.cpp b/llvm/tools/obj2yaml/elf2yaml.cpp
index ac3eefdeaace6..464f899352f48 100644
--- a/llvm/tools/obj2yaml/elf2yaml.cpp
+++ b/llvm/tools/obj2yaml/elf2yaml.cpp
@@ -51,6 +51,9 @@ class ELFDumper {
   const object::ELFFile<ELFT> &Obj;
   ArrayRef<Elf_Word> ShndxTable;
 
+  Expected<std::vector<ELFYAML::ProgramHeader>>
+  dumpProgramHeaders(ArrayRef<std::unique_ptr<ELFYAML::Chunk>> Sections);
+
   Error dumpSymbols(const Elf_Shdr *Symtab,
                     std::vector<ELFYAML::Symbol> &Symbols);
   Error dumpSymbol(const Elf_Sym *Sym, const Elf_Shdr *SymTab,
@@ -88,6 +91,10 @@ class ELFDumper {
   Expected<ELFYAML::MipsABIFlags *> dumpMipsABIFlags(const Elf_Shdr *Shdr);
   Expected<ELFYAML::StackSizesSection *>
   dumpStackSizesSection(const Elf_Shdr *Shdr);
+  Expected<ELFYAML::RawContentSection *>
+  dumpPlaceholderSection(const Elf_Shdr *Shdr);
+
+  bool shouldPrintSection(const ELFYAML::Section &S, const Elf_Shdr &SHdr);
 
 public:
   ELFDumper(const object::ELFFile<ELFT> &O);
@@ -112,6 +119,12 @@ ELFDumper<ELFT>::getUniquedSectionName(const Elf_Shdr *Sec) {
   if (!NameOrErr)
     return NameOrErr;
   StringRef Name = *NameOrErr;
+  // In some specific cases we might have more than one section without a
+  // name (sh_name == 0). It normally doesn't happen, but when we have this case
+  // it doesn't make sense to uniquify their names and add noise to the output.
+  if (Name.empty())
+    return "";
+
   std::string &Ret = SectionNames[SecIndex];
 
   auto It = UsedSectionNames.insert({Name, 0});
@@ -157,6 +170,32 @@ ELFDumper<ELFT>::getUniquedSymbolName(const Elf_Sym *Sym, StringRef StrTable,
   return Name;
 }
 
+template <class ELFT>
+bool ELFDumper<ELFT>::shouldPrintSection(const ELFYAML::Section &S,
+                                         const Elf_Shdr &SHdr) {
+  // We only print the SHT_NULL section at index 0 when it
+  // has at least one non-null field, because yaml2obj
+  // normally creates the zero section at index 0 implicitly.
+  if (S.Type == ELF::SHT_NULL && (&SHdr == &Sections[0])) {
+    const uint8_t *Begin = reinterpret_cast<const uint8_t *>(&SHdr);
+    const uint8_t *End = Begin + sizeof(Elf_Shdr);
+    return std::find_if(Begin, End, [](uint8_t V) { return V != 0; }) != End;
+  }
+
+  // Normally we use "Symbols:" and "DynamicSymbols:" to describe contents of
+  // symbol tables. We also build and emit corresponding string tables
+  // implicitly. But sometimes it is important to preserve positions and virtual
+  // addresses of allocatable sections, e.g. for creating program headers.
+  // Generally we are trying to reduce noise in the YAML output. Because
+  // of that we do not print non-allocatable versions of such sections and
+  // assume they are placed at the end.
+  if (S.Type == ELF::SHT_STRTAB || S.Type == ELF::SHT_SYMTAB ||
+      S.Type == ELF::SHT_DYNSYM)
+    return S.Flags.getValueOr(ELFYAML::ELF_SHF(0)) & ELF::SHF_ALLOC;
+
+  return true;
+}
+
 template <class ELFT> Expected<ELFYAML::Object *> ELFDumper<ELFT>::dump() {
   auto Y = std::make_unique<ELFYAML::Object>();
 
@@ -227,15 +266,83 @@ template <class ELFT> Expected<ELFYAML::Object *> ELFDumper<ELFT>::dump() {
       return std::move(E);
   }
 
-  if (Expected<std::vector<std::unique_ptr<ELFYAML::Chunk>>> ChunksOrErr =
-          dumpSections())
-    Y->Chunks = std::move(*ChunksOrErr);
-  else
+  // We dump all sections first. It is simple and allows us to verify that all
+  // sections are valid and also to generalize the code. But we are not going to
+  // keep all of them in the final output (see comments for
+  // 'shouldPrintSection()'). Undesired chunks will be removed later.
+  Expected<std::vector<std::unique_ptr<ELFYAML::Chunk>>> ChunksOrErr =
+      dumpSections();
+  if (!ChunksOrErr)
     return ChunksOrErr.takeError();
+  std::vector<std::unique_ptr<ELFYAML::Chunk>> Chunks = std::move(*ChunksOrErr);
+
+  // Dump program headers.
+  Expected<std::vector<ELFYAML::ProgramHeader>> PhdrsOrErr =
+      dumpProgramHeaders(Chunks);
+  if (!PhdrsOrErr)
+    return PhdrsOrErr.takeError();
+  Y->ProgramHeaders = std::move(*PhdrsOrErr);
 
+  llvm::erase_if(Chunks, [this](const std::unique_ptr<ELFYAML::Chunk> &C) {
+    const ELFYAML::Section &S = cast<ELFYAML::Section>(*C.get());
+    return !shouldPrintSection(S, Sections[S.OriginalSecNdx]);
+  });
+
+  Y->Chunks = std::move(Chunks);
   return Y.release();
 }
 
+template <class ELFT>
+static bool isInSegment(const ELFYAML::Section &Sec,
+                        const typename ELFT::Shdr &SHdr,
+                        const typename ELFT::Phdr &Phdr) {
+  if (Sec.Type == ELF::SHT_NULL)
+    return false;
+  return SHdr.sh_offset >= Phdr.p_offset &&
+         (SHdr.sh_offset + SHdr.sh_size <= Phdr.p_offset + Phdr.p_filesz);
+}
+
+template <class ELFT>
+Expected<std::vector<ELFYAML::ProgramHeader>>
+ELFDumper<ELFT>::dumpProgramHeaders(
+    ArrayRef<std::unique_ptr<ELFYAML::Chunk>> Chunks) {
+  std::vector<ELFYAML::ProgramHeader> Ret;
+  Expected<typename ELFT::PhdrRange> PhdrsOrErr = Obj.program_headers();
+  if (!PhdrsOrErr)
+    return PhdrsOrErr.takeError();
+
+  for (const typename ELFT::Phdr &Phdr : *PhdrsOrErr) {
+    ELFYAML::ProgramHeader PH;
+    PH.Type = Phdr.p_type;
+    PH.Flags = Phdr.p_flags;
+    PH.VAddr = Phdr.p_vaddr;
+    PH.PAddr = Phdr.p_paddr;
+    PH.Align = static_cast<llvm::yaml::Hex64>(Phdr.p_align);
+
+    // Here we match sections with segments.
+    // It is not possible to have a non-Section chunk, because
+    // obj2yaml does not create Fill chunks.
+    for (const std::unique_ptr<ELFYAML::Chunk> &C : Chunks) {
+      ELFYAML::Section &S = cast<ELFYAML::Section>(*C.get());
+      if (isInSegment<ELFT>(S, Sections[S.OriginalSecNdx], Phdr))
+        PH.Sections.push_back({S.Name});
+    }
+
+    Ret.push_back(PH);
+  }
+
+  return Ret;
+}
+
+template <class ELFT>
+Expected<ELFYAML::RawContentSection *>
+ELFDumper<ELFT>::dumpPlaceholderSection(const Elf_Shdr *Shdr) {
+  auto S = std::make_unique<ELFYAML::RawContentSection>();
+  if (Error E = dumpCommonSection(Shdr, *S.get()))
+    return std::move(E);
+  return S.release();
+}
+
 template <class ELFT>
 Expected<std::vector<std::unique_ptr<ELFYAML::Chunk>>>
 ELFDumper<ELFT>::dumpSections() {
@@ -288,6 +395,13 @@ ELFDumper<ELFT>::dumpSections() {
     case ELF::SHT_LLVM_CALL_GRAPH_PROFILE:
       return
           [this](const Elf_Shdr *S) { return dumpCallGraphProfileSection(S); };
+    case ELF::SHT_STRTAB:
+    case ELF::SHT_SYMTAB:
+    case ELF::SHT_DYNSYM:
+      // The contents of these sections are described by other parts of the YAML
+      // file. But we still want to dump them, because their properties can be
+      // important. See comments for 'shouldPrintSection()' for more details.
+      return [this](const Elf_Shdr *S) { return dumpPlaceholderSection(S); };
     default:
       return nullptr;
     }
@@ -303,37 +417,6 @@ ELFDumper<ELFT>::dumpSections() {
       continue;
     }
 
-    if (Sec.sh_type == ELF::SHT_STRTAB || Sec.sh_type == ELF::SHT_SYMTAB ||
-        Sec.sh_type == ELF::SHT_DYNSYM) {
-      // The contents of these sections are described by other parts of the YAML
-      // file. We still dump them so that their positions in the section header
-      // table are correctly recorded. We only dump allocatable section because
-      // their positions and addresses are important, e.g. for creating program
-      // headers. Some sections, like .symtab or .strtab normally are not
-      // allocatable and do not have virtual addresses. We want to avoid noise
-      // in the YAML output and assume that they are placed at the end.
-      if (Sec.sh_flags & ELF::SHF_ALLOC) {
-        auto S = std::make_unique<ELFYAML::RawContentSection>();
-        if (Error E = dumpCommonSection(&Sec, *S.get()))
-          return std::move(E);
-        if (Error E = Add(S.release()))
-          return std::move(E);
-      }
-      continue;
-    }
-
-    if (Sec.sh_type == ELF::SHT_NULL) {
-      // We only dump the SHT_NULL section at index 0 when it
-      // has at least one non-null field, because yaml2obj
-      // normally creates the zero section at index 0 implicitly.
-      if (&Sec == &Sections[0]) {
-        const uint8_t *Begin = reinterpret_cast<const uint8_t *>(&Sec);
-        const uint8_t *End = Begin + sizeof(Elf_Shdr);
-        if (std::find_if(Begin, End, [](uint8_t V) { return V != 0; }) == End)
-          continue;
-      }
-    }
-
     // Recognize some special SHT_PROGBITS sections by name.
     if (Sec.sh_type == ELF::SHT_PROGBITS) {
       auto NameOrErr = getUniquedSectionName(&Sec);
@@ -484,6 +567,8 @@ Error ELFDumper<ELFT>::dumpCommonSection(const Elf_Shdr *Shdr,
   if (Shdr->sh_entsize != getDefaultShEntSize<ELFT>(S.Type))
     S.EntSize = static_cast<llvm::yaml::Hex64>(Shdr->sh_entsize);
 
+  S.OriginalSecNdx = Shdr - &Sections[0];
+
   auto NameOrErr = getUniquedSectionName(Shdr);
   if (!NameOrErr)
     return NameOrErr.takeError();
diff --git a/llvm/tools/obj2yaml/wasm2yaml.cpp b/llvm/tools/obj2yaml/wasm2yaml.cpp
index ea7a1e983bd43..689c231b0dd76 100644
--- a/llvm/tools/obj2yaml/wasm2yaml.cpp
+++ b/llvm/tools/obj2yaml/wasm2yaml.cpp
@@ -262,6 +262,18 @@ ErrorOr<WasmYAML::Object *> WasmDumper::dump() {
       S = std::move(MemorySec);
       break;
     }
+    case wasm::WASM_SEC_EVENT: {
+      auto EventSec = std::make_unique<WasmYAML::EventSection>();
+      for (auto &Event : Obj.events()) {
+        WasmYAML::Event E;
+        E.Index = Event.Index;
+        E.Attribute = Event.Type.Attribute;
+        E.SigIndex = Event.Type.SigIndex;
+        EventSec->Events.push_back(E);
+      }
+      S = std::move(EventSec);
+      break;
+    }
     case wasm::WASM_SEC_GLOBAL: {
       auto GlobalSec = std::make_unique<WasmYAML::GlobalSection>();
       for (auto &Global : Obj.globals()) {
@@ -275,18 +287,6 @@ ErrorOr<WasmYAML::Object *> WasmDumper::dump() {
       S = std::move(GlobalSec);
       break;
     }
-    case wasm::WASM_SEC_EVENT: {
-      auto EventSec = std::make_unique<WasmYAML::EventSection>();
-      for (auto &Event : Obj.events()) {
-        WasmYAML::Event E;
-        E.Index = Event.Index;
-        E.Attribute = Event.Type.Attribute;
-        E.SigIndex = Event.Type.SigIndex;
-        EventSec->Events.push_back(E);
-      }
-      S = std::move(EventSec);
-      break;
-    }
     case wasm::WASM_SEC_START: {
       auto StartSec = std::make_unique<WasmYAML::StartSection>();
       StartSec->StartFunction = Obj.startFunction();
diff --git a/llvm/tools/opt-viewer/opt-viewer.py b/llvm/tools/opt-viewer/opt-viewer.py
index e98781595accb..714fccc8df967 100755
--- a/llvm/tools/opt-viewer/opt-viewer.py
+++ b/llvm/tools/opt-viewer/opt-viewer.py
@@ -3,9 +3,9 @@
 from __future__ import print_function
 
 import argparse
-import cgi
 import errno
 import functools
+import html
 import io
 from multiprocessing import cpu_count
 import os.path
@@ -197,7 +197,7 @@ def __init__(self, output_dir, should_display_hotness, max_hottest_remarks_on_in
         self.max_hottest_remarks_on_index = max_hottest_remarks_on_index
 
     def render_entry(self, r, odd):
-        escaped_name = cgi.escape(r.DemangledFunctionName)
+        escaped_name = html.escape(r.DemangledFunctionName)
         print(u'''
 <tr>
 <td class=\"column-entry-{odd}\"><a href={r.Link}>{r.DebugLocString}</a></td>
diff --git a/llvm/tools/opt-viewer/optrecord.py b/llvm/tools/opt-viewer/optrecord.py
index 2dff0c847c77e..6a53e13f4c2b8 100644
--- a/llvm/tools/opt-viewer/optrecord.py
+++ b/llvm/tools/opt-viewer/optrecord.py
@@ -11,7 +11,7 @@
     print("For faster parsing, you may want to install libYAML for PyYAML")
     from yaml import Loader
 
-import cgi
+import html
 from collections import defaultdict
 import fnmatch
 import functools
@@ -159,7 +159,7 @@ def getArgString(self, mapping):
         (key, value) = list(mapping.items())[0]
 
         if key == 'Caller' or key == 'Callee' or key == 'DirectCallee':
-            value = cgi.escape(self.demangle(value))
+            value = html.escape(self.demangle(value))
 
         if dl and key != 'Caller':
             dl_dict = dict(list(dl))
diff --git a/llvm/unittests/ADT/CMakeLists.txt b/llvm/unittests/ADT/CMakeLists.txt
index 53d18b3732717..771d1600ca6d1 100644
--- a/llvm/unittests/ADT/CMakeLists.txt
+++ b/llvm/unittests/ADT/CMakeLists.txt
@@ -73,6 +73,7 @@ add_llvm_unittest(ADTTests
   TinyPtrVectorTest.cpp
   TripleTest.cpp
   TwineTest.cpp
+  WaymarkingTest.cpp
   )
 
 target_link_libraries(ADTTests PRIVATE LLVMTestingSupport)
diff --git a/llvm/unittests/ADT/FunctionRefTest.cpp b/llvm/unittests/ADT/FunctionRefTest.cpp
index 0f744c1032e92..669b87dbf8e41 100644
--- a/llvm/unittests/ADT/FunctionRefTest.cpp
+++ b/llvm/unittests/ADT/FunctionRefTest.cpp
@@ -38,4 +38,14 @@ TEST(FunctionRefTest, Copy) {
   EXPECT_EQ(1, Y());
 }
 
+TEST(FunctionRefTest, BadCopy) {
+  auto A = [] { return 1; };
+  function_ref<int()> X;
+  function_ref<int()> Y = A;
+  function_ref<int()> Z = static_cast<const function_ref<int()> &&>(Y);
+  X = Z;
+  Y = nullptr;
+  ASSERT_EQ(1, X());
 }
+
+} // namespace
diff --git a/llvm/unittests/ADT/StringSetTest.cpp b/llvm/unittests/ADT/StringSetTest.cpp
index 17bfa1dd18d23..f3ec217779ec7 100644
--- a/llvm/unittests/ADT/StringSetTest.cpp
+++ b/llvm/unittests/ADT/StringSetTest.cpp
@@ -41,4 +41,15 @@ TEST_F(StringSetTest, InsertAndCountStringMapEntry) {
   Element->Destroy();
 }
 
+TEST_F(StringSetTest, EmptyString) {
+  // Verify that the empty string can by successfully inserted
+  StringSet<> Set;
+  size_t Count = Set.count("");
+  EXPECT_EQ(Count, 0UL);
+
+  Set.insert("");
+  Count = Set.count("");
+  EXPECT_EQ(Count, 1UL);
+}
+
 } // end anonymous namespace
diff --git a/llvm/unittests/ADT/WaymarkingTest.cpp b/llvm/unittests/ADT/WaymarkingTest.cpp
new file mode 100644
index 0000000000000..0a6b66e8d21ce
--- /dev/null
+++ b/llvm/unittests/ADT/WaymarkingTest.cpp
@@ -0,0 +1,142 @@
+//===- llvm/unittest/IR/WaymarkTest.cpp - Waymarking unit tests -----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/Waymarking.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+static const int N = 100;
+
+// Get the Waymarking Tag of the pointer.
+static int tag(int *P) {
+  return static_cast<int>(reinterpret_cast<uintptr_t>(P) &
+                          uintptr_t(alignof(int *) - 1));
+}
+
+// Get the actual pointer, by stripping the Waymarking Tag.
+static int *ref(int *P) {
+  return reinterpret_cast<int *>(reinterpret_cast<uintptr_t>(P) &
+                                 ~uintptr_t(alignof(int *) - 1));
+}
+
+static int **createArray(int Len) {
+  int **A = new int *[Len];
+  for (int I = 0; I < Len; ++I)
+    A[I] = new int(I);
+  return A;
+}
+
+static void freeArray(int **A, int Len) {
+  for (int I = 0; I < Len; ++I)
+    delete ref(A[I]);
+  delete[] A;
+}
+
+// Verify the values stored in the array are as expected, and did not change due
+// to fillWaymarks.
+static void verifyArrayValues(int **A, int Begin, int End) {
+  for (int I = Begin; I < End; ++I)
+    EXPECT_EQ(I, *ref(A[I]));
+}
+
+static void verifyArrayValues(int **A, int Len) {
+  verifyArrayValues(A, 0, Len);
+}
+
+// Verify that we can follow the waymarks to the array's head from each element
+// of the array.
+static void verifyFollowWaymarks(int **A, int Len) {
+  for (int I = 0; I < Len; ++I) {
+    int **P = followWaymarks(A + I);
+    EXPECT_EQ(A, P);
+  }
+}
+
+namespace {
+
+// Test filling and following the waymarks of a single array.
+TEST(WaymarkingTest, SingleHead) {
+  const int N2 = 2 * N;
+  int **volatile A = createArray(N2);
+
+  // Fill the first half of the array with waymarks.
+  fillWaymarks(A, A + N, 0);
+  verifyArrayValues(A, N2);
+
+  verifyFollowWaymarks(A, N);
+
+  // Fill the rest of the waymarks (continuing from where we stopped).
+  fillWaymarks(A + N, A + N2, N);
+  verifyArrayValues(A, N2);
+
+  verifyFollowWaymarks(A, N);
+
+  freeArray(A, N2);
+}
+
+// Test filling and following the waymarks of an array split into several
+// different sections of waymarks (treated just like separate arrays).
+TEST(WaymarkingTest, MultiHead) {
+  const int N2 = 2 * N;
+  const int N3 = 3 * N;
+  int **volatile A = createArray(N3);
+
+  // Separate the array into 3 sections of waymarks.
+  fillWaymarks(A, A + N, 0);
+  fillWaymarks(A + N, A + N2, 0);
+  fillWaymarks(A + N2, A + N3, 0);
+  verifyArrayValues(A, N3);
+
+  verifyFollowWaymarks(A, N);
+  verifyFollowWaymarks(A + N, N2 - N);
+  verifyFollowWaymarks(A + N2, N3 - N2);
+
+  freeArray(A, N3);
+}
+
+// Test reseting (value and tag of) elements inside an array of waymarks.
+TEST(WaymarkingTest, Reset) {
+  int **volatile A = createArray(N);
+
+  fillWaymarks(A, A + N, 0);
+  verifyArrayValues(A, N);
+
+  const int N2 = N / 2;
+  const int N3 = N / 3;
+  const int N4 = N / 4;
+
+  // Reset specific elements and check that the tag remains the same.
+  int T2 = tag(A[N2]);
+  delete ref(A[N2]);
+  A[N2] = new int(N2);
+  fillWaymarks(A + N2, A + N2 + 1, N2);
+  verifyArrayValues(A, N2, N2 + 1);
+  EXPECT_EQ(T2, tag(A[N2]));
+
+  int T3 = tag(A[N3]);
+  delete ref(A[N3]);
+  A[N3] = new int(N3);
+  fillWaymarks(A + N3, A + N3 + 1, N3);
+  verifyArrayValues(A, N3, N3 + 1);
+  EXPECT_EQ(T3, tag(A[N3]));
+
+  int T4 = tag(A[N4]);
+  delete ref(A[N4]);
+  A[N4] = new int(N4);
+  fillWaymarks(A + N4, A + N4 + 1, N4);
+  verifyArrayValues(A, N4, N4 + 1);
+  EXPECT_EQ(T4, tag(A[N4]));
+
+  verifyArrayValues(A, N);
+  verifyFollowWaymarks(A, N);
+
+  freeArray(A, N);
+}
+
+} // end anonymous namespace
diff --git a/llvm/unittests/IR/KnowledgeRetentionTest.cpp b/llvm/unittests/Analysis/AssumeBundleQueriesTest.cpp
similarity index 98%
rename from llvm/unittests/IR/KnowledgeRetentionTest.cpp
rename to llvm/unittests/Analysis/AssumeBundleQueriesTest.cpp
index aca801d2b296e..f6f4849a94e81 100644
--- a/llvm/unittests/IR/KnowledgeRetentionTest.cpp
+++ b/llvm/unittests/Analysis/AssumeBundleQueriesTest.cpp
@@ -1,4 +1,4 @@
-//===- KnowledgeRetention.h - utilities to preserve informations *- C++ -*-===//
+//===- AssumeBundleQueriesTest.cpp ------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/IR/KnowledgeRetention.h"
+#include "llvm/Analysis/AssumeBundleQueries.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/IR/CallSite.h"
 #include "llvm/IR/LLVMContext.h"
@@ -14,6 +14,7 @@
 #include "llvm/Support/Regex.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
 #include "gtest/gtest.h"
 #include <random>
 
@@ -22,10 +23,6 @@ using namespace llvm;
 extern cl::opt<bool> ShouldPreserveAllAttributes;
 extern cl::opt<bool> EnableKnowledgeRetention;
 
-static IntrinsicInst *buildAssumeFromInst(Instruction *I) {
-  return cast_or_null<IntrinsicInst>(BuildAssumeFromInst(I));
-}
-
 static void RunTest(
     StringRef Head, StringRef Tail,
     std::vector<std::pair<StringRef, llvm::function_ref<void(Instruction *)>>>
@@ -480,8 +477,7 @@ static void RunRandTest(uint64_t Seed, int Size, int MinCount, int MaxCount,
   }
 
   auto *Assume = cast<IntrinsicInst>(IntrinsicInst::Create(
-      FnAssume, ArrayRef<Value *>({ConstantInt::getTrue(C)}),
-      std::move(OpBundle)));
+      FnAssume, ArrayRef<Value *>({ConstantInt::getTrue(C)}), OpBundle));
   Assume->insertBefore(&F->begin()->front());
   RetainedKnowledgeMap Map;
   fillMapFromAssume(*Assume, Map);
diff --git a/llvm/unittests/Analysis/CMakeLists.txt b/llvm/unittests/Analysis/CMakeLists.txt
index 6fabd940a74af..254b2893cac5b 100644
--- a/llvm/unittests/Analysis/CMakeLists.txt
+++ b/llvm/unittests/Analysis/CMakeLists.txt
@@ -9,6 +9,7 @@ set(LLVM_LINK_COMPONENTS
 add_llvm_unittest(AnalysisTests
   AliasAnalysisTest.cpp
   AliasSetTrackerTest.cpp
+  AssumeBundleQueriesTest.cpp
   BasicAliasAnalysisTest.cpp
   BlockFrequencyInfoTest.cpp
   BranchProbabilityInfoTest.cpp
diff --git a/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp b/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
index af0ee631ee122..f21081467c1eb 100644
--- a/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
+++ b/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
@@ -96,6 +96,7 @@ TEST_F(TargetLibraryInfoTest, ValidProto) {
       "declare float @acoshf(float)\n"
       "declare x86_fp80 @acoshl(x86_fp80)\n"
       "declare x86_fp80 @acosl(x86_fp80)\n"
+      "declare i8* @aligned_alloc(i64, i64)\n"
       "declare double @asin(double)\n"
       "declare float @asinf(float)\n"
       "declare double @asinh(double)\n"
diff --git a/llvm/unittests/Analysis/VectorUtilsTest.cpp b/llvm/unittests/Analysis/VectorUtilsTest.cpp
index d471e79842ca7..1a06b0994bc08 100644
--- a/llvm/unittests/Analysis/VectorUtilsTest.cpp
+++ b/llvm/unittests/Analysis/VectorUtilsTest.cpp
@@ -100,10 +100,10 @@ TEST_F(BasicTest, isSplat) {
 
 TEST_F(BasicTest, scaleShuffleMask) {
   SmallVector<int, 16> ScaledMask;
-  scaleShuffleMask<int>(1, {3,2,0,-2}, ScaledMask);
-  EXPECT_EQ(makeArrayRef<int>(ScaledMask), makeArrayRef<int>({3,2,0,-2}));
-  scaleShuffleMask<int>(4, {3,2,0,-1}, ScaledMask);
-  EXPECT_EQ(makeArrayRef<int>(ScaledMask), makeArrayRef<int>({12,13,14,15,8,9,10,11,0,1,2,3,-1,-1,-1,-1}));
+  scaleShuffleMask(1, {3,2,0,-2}, ScaledMask);
+  EXPECT_EQ(makeArrayRef(ScaledMask), makeArrayRef({3,2,0,-2}));
+  scaleShuffleMask(4, {3,2,0,-1}, ScaledMask);
+  EXPECT_EQ(makeArrayRef(ScaledMask), makeArrayRef({12,13,14,15,8,9,10,11,0,1,2,3,-1,-1,-1,-1}));
 }
 
 TEST_F(BasicTest, getSplatIndex) {
diff --git a/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp b/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp
index 244cb75c3e297..ffdd108cd853b 100644
--- a/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp
@@ -2578,7 +2578,7 @@ TEST_F(AArch64GISelMITest, BitcastLoad) {
   DefineLegalizerInfo(A, {});
 
   MachineMemOperand *MMO = B.getMF().getMachineMemOperand(
-    MachinePointerInfo(), MachineMemOperand::MOLoad, 4, 4);
+      MachinePointerInfo(), MachineMemOperand::MOLoad, 4, Align(4));
   auto Load = B.buildLoad(V4S8, Ptr, *MMO);
 
   AInfo Info(MF->getSubtarget());
@@ -2611,7 +2611,7 @@ TEST_F(AArch64GISelMITest, BitcastStore) {
   DefineLegalizerInfo(A, {});
 
   MachineMemOperand *MMO = B.getMF().getMachineMemOperand(
-    MachinePointerInfo(), MachineMemOperand::MOStore, 4, 4);
+      MachinePointerInfo(), MachineMemOperand::MOStore, 4, Align(4));
   auto Val = B.buildUndef(V4S8);
   auto Store = B.buildStore(Val, Ptr, *MMO);
 
@@ -2722,4 +2722,30 @@ TEST_F(AArch64GISelMITest, BitcastBitOps) {
   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 }
 
+TEST_F(AArch64GISelMITest, CreateLibcall) {
+  setUp();
+  if (!TM)
+    return;
+
+  DefineLegalizerInfo(A, {});
+
+  AInfo Info(MF->getSubtarget());
+  DummyGISelObserver Observer;
+
+  LLVMContext &Ctx = MF->getFunction().getContext();
+  auto *RetTy = Type::getVoidTy(Ctx);
+
+  EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
+            createLibcall(B, "abort", {{}, RetTy}, {}, CallingConv::C));
+
+  auto CheckStr = R"(
+  CHECK: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+  CHECK: BL &abort
+  CHECK: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+  )";
+
+  // Check
+  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
+}
+
 } // namespace
diff --git a/llvm/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp b/llvm/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp
index 4d766cd42bee7..737e66273a35b 100644
--- a/llvm/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp
@@ -302,11 +302,10 @@ TEST_F(AArch64GISelMITest, BuildAtomicRMW) {
   SmallVector<Register, 4> Copies;
   collectCopies(Copies, MF);
 
-  MachineMemOperand *MMO =
-    MF->getMachineMemOperand(
+  MachineMemOperand *MMO = MF->getMachineMemOperand(
       MachinePointerInfo(),
-      MachineMemOperand::MOLoad | MachineMemOperand::MOStore,
-      8, 8, AAMDNodes(), nullptr, SyncScope::System, AtomicOrdering::Unordered);
+      MachineMemOperand::MOLoad | MachineMemOperand::MOStore, 8, Align(8),
+      AAMDNodes(), nullptr, SyncScope::System, AtomicOrdering::Unordered);
 
   auto Ptr = B.buildUndef(P0);
   B.buildAtomicRMWFAdd(S64, Ptr, Copies[0], *MMO);
diff --git a/llvm/unittests/CodeGen/GlobalISel/PatternMatchTest.cpp b/llvm/unittests/CodeGen/GlobalISel/PatternMatchTest.cpp
index 172eca46b4a9a..6a54084cacafb 100644
--- a/llvm/unittests/CodeGen/GlobalISel/PatternMatchTest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/PatternMatchTest.cpp
@@ -137,6 +137,14 @@ TEST_F(AArch64GISelMITest, MatchBinaryOp) {
   EXPECT_TRUE(match);
   EXPECT_EQ(Src0, Copies[0]);
   EXPECT_EQ(Src1, TruncCopy1.getReg(0));
+
+  // Match shl, and make sure a different shift amount type works.
+  auto Shl = B.buildShl(s64, Copies[0], TruncCopy1);
+  match = mi_match(Shl.getReg(0), *MRI,
+                   m_GShl(m_Reg(Src0), m_Reg(Src1)));
+  EXPECT_TRUE(match);
+  EXPECT_EQ(Src0, Copies[0]);
+  EXPECT_EQ(Src1, TruncCopy1.getReg(0));
 }
 
 TEST_F(AArch64GISelMITest, MatchICmp) {
diff --git a/llvm/unittests/CodeGen/MachineInstrTest.cpp b/llvm/unittests/CodeGen/MachineInstrTest.cpp
index 4151ef7a6d007..33baaf62efdf7 100644
--- a/llvm/unittests/CodeGen/MachineInstrTest.cpp
+++ b/llvm/unittests/CodeGen/MachineInstrTest.cpp
@@ -50,8 +50,8 @@ TEST(IsIdenticalToTest, DifferentDefs) {
       {0, 0, MCOI::OPERAND_REGISTER, 0},
       {0, 1 << MCOI::OptionalDef, MCOI::OPERAND_REGISTER, 0}};
   MCInstrDesc MCID = {
-      0, NumOps,  NumDefs, 0,      0, 1ULL << MCID::HasOptionalDef,
-      0, nullptr, nullptr, OpInfo, 0, nullptr};
+      0, NumOps,  NumDefs, 0,     0, 1ULL << MCID::HasOptionalDef,
+      0, nullptr, nullptr, OpInfo};
 
   // Create two MIs with different virtual reg defs and the same uses.
   unsigned VirtualDef1 = -42; // The value doesn't matter, but the sign does.
@@ -121,8 +121,8 @@ TEST(MachineInstrExpressionTraitTest, IsEqualAgreesWithGetHashValue) {
       {0, 0, MCOI::OPERAND_REGISTER, 0},
       {0, 1 << MCOI::OptionalDef, MCOI::OPERAND_REGISTER, 0}};
   MCInstrDesc MCID = {
-      0, NumOps,  NumDefs, 0,      0, 1ULL << MCID::HasOptionalDef,
-      0, nullptr, nullptr, OpInfo, 0, nullptr};
+      0, NumOps,  NumDefs, 0,     0, 1ULL << MCID::HasOptionalDef,
+      0, nullptr, nullptr, OpInfo};
 
   // Define a series of instructions with different kinds of operands and make
   // sure that the hash function is consistent with isEqual for various
@@ -196,8 +196,7 @@ TEST(MachineInstrPrintingTest, DebugLocPrinting) {
   auto MF = createMachineFunction(Ctx, Mod);
 
   MCOperandInfo OpInfo{0, 0, MCOI::OPERAND_REGISTER, 0};
-  MCInstrDesc MCID = {0, 1,       1,       0,       0, 0,
-                      0, nullptr, nullptr, &OpInfo, 0, nullptr};
+  MCInstrDesc MCID = {0, 1, 1, 0, 0, 0, 0, nullptr, nullptr, &OpInfo};
 
   DIFile *DIF = DIFile::getDistinct(Ctx, "filename", "");
   DISubprogram *DIS = DISubprogram::getDistinct(
@@ -224,8 +223,7 @@ TEST(MachineInstrSpan, DistanceBegin) {
   auto MF = createMachineFunction(Ctx, Mod);
   auto MBB = MF->CreateMachineBasicBlock();
 
-  MCInstrDesc MCID = {0, 0,       0,       0,       0, 0,
-                      0, nullptr, nullptr, nullptr, 0, nullptr};
+  MCInstrDesc MCID = {0, 0, 0, 0, 0, 0, 0, nullptr, nullptr, nullptr};
 
   auto MII = MBB->begin();
   MachineInstrSpan MIS(MII, MBB);
@@ -242,8 +240,7 @@ TEST(MachineInstrSpan, DistanceEnd) {
   auto MF = createMachineFunction(Ctx, Mod);
   auto MBB = MF->CreateMachineBasicBlock();
 
-  MCInstrDesc MCID = {0, 0,       0,       0,       0, 0,
-                      0, nullptr, nullptr, nullptr, 0, nullptr};
+  MCInstrDesc MCID = {0, 0, 0, 0, 0, 0, 0, nullptr, nullptr, nullptr};
 
   auto MII = MBB->end();
   MachineInstrSpan MIS(MII, MBB);
@@ -258,14 +255,13 @@ TEST(MachineInstrExtraInfo, AddExtraInfo) {
   LLVMContext Ctx;
   Module Mod("Module", Ctx);
   auto MF = createMachineFunction(Ctx, Mod);
-  MCInstrDesc MCID = {0, 0,       0,       0,       0, 0,
-                      0, nullptr, nullptr, nullptr, 0, nullptr};
+  MCInstrDesc MCID = {0, 0, 0, 0, 0, 0, 0, nullptr, nullptr, nullptr};
 
   auto MI = MF->CreateMachineInstr(MCID, DebugLoc());
   auto MAI = MCAsmInfo();
   auto MC = createMCContext(&MAI);
   auto MMO = MF->getMachineMemOperand(MachinePointerInfo(),
-                                      MachineMemOperand::MOLoad, 8, 8);
+                                      MachineMemOperand::MOLoad, 8, Align(8));
   SmallVector<MachineMemOperand *, 2> MMOs;
   MMOs.push_back(MMO);
   MCSymbol *Sym1 = MC->createTempSymbol("pre_label", false);
@@ -306,14 +302,13 @@ TEST(MachineInstrExtraInfo, ChangeExtraInfo) {
   LLVMContext Ctx;
   Module Mod("Module", Ctx);
   auto MF = createMachineFunction(Ctx, Mod);
-  MCInstrDesc MCID = {0, 0,       0,       0,       0, 0,
-                      0, nullptr, nullptr, nullptr, 0, nullptr};
+  MCInstrDesc MCID = {0, 0, 0, 0, 0, 0, 0, nullptr, nullptr, nullptr};
 
   auto MI = MF->CreateMachineInstr(MCID, DebugLoc());
   auto MAI = MCAsmInfo();
   auto MC = createMCContext(&MAI);
   auto MMO = MF->getMachineMemOperand(MachinePointerInfo(),
-                                      MachineMemOperand::MOLoad, 8, 8);
+                                      MachineMemOperand::MOLoad, 8, Align(8));
   SmallVector<MachineMemOperand *, 2> MMOs;
   MMOs.push_back(MMO);
   MCSymbol *Sym1 = MC->createTempSymbol("pre_label", false);
@@ -344,14 +339,13 @@ TEST(MachineInstrExtraInfo, RemoveExtraInfo) {
   LLVMContext Ctx;
   Module Mod("Module", Ctx);
   auto MF = createMachineFunction(Ctx, Mod);
-  MCInstrDesc MCID = {0, 0,       0,       0,       0, 0,
-                      0, nullptr, nullptr, nullptr, 0, nullptr};
+  MCInstrDesc MCID = {0, 0, 0, 0, 0, 0, 0, nullptr, nullptr, nullptr};
 
   auto MI = MF->CreateMachineInstr(MCID, DebugLoc());
   auto MAI = MCAsmInfo();
   auto MC = createMCContext(&MAI);
   auto MMO = MF->getMachineMemOperand(MachinePointerInfo(),
-                                      MachineMemOperand::MOLoad, 8, 8);
+                                      MachineMemOperand::MOLoad, 8, Align(8));
   SmallVector<MachineMemOperand *, 2> MMOs;
   MMOs.push_back(MMO);
   MMOs.push_back(MMO);
diff --git a/llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp b/llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp
index ef97f2cd40a40..a8f85d061bf7a 100644
--- a/llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp
+++ b/llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp
@@ -91,6 +91,62 @@ TEST_F(CoreAPIsStandardTest, EmptyLookup) {
   EXPECT_TRUE(OnCompletionRun) << "OnCompletion was not run for empty query";
 }
 
+TEST_F(CoreAPIsStandardTest, ResolveUnrequestedSymbol) {
+  // Test that all symbols in a MaterializationUnit materialize corretly when
+  // only a subset of symbols is looked up.
+  // The aim here is to ensure that we're not relying on the query to set up
+  // state needed to materialize the unrequested symbols.
+
+  cantFail(JD.define(std::make_unique<SimpleMaterializationUnit>(
+      SymbolFlagsMap({{Foo, FooSym.getFlags()}, {Bar, BarSym.getFlags()}}),
+      [this](MaterializationResponsibility R) {
+        cantFail(R.notifyResolved({{Foo, FooSym}, {Bar, BarSym}}));
+        cantFail(R.notifyEmitted());
+      })));
+
+  auto Result =
+      cantFail(ES.lookup(makeJITDylibSearchOrder(&JD), SymbolLookupSet({Foo})));
+  EXPECT_EQ(Result.size(), 1U) << "Unexpected number of results";
+  EXPECT_TRUE(Result.count(Foo)) << "Expected result for \"Foo\"";
+}
+
+TEST_F(CoreAPIsStandardTest, MaterializationSideEffctsOnlyTest) {
+  // Test that basic materialization-side-effects-only symbols work as expected:
+  // that they can be emitted without being resolved, that queries for them
+  // don't return until they're emitted, and that they don't appear in query
+  // results.
+
+  Optional<MaterializationResponsibility> FooR;
+  Optional<SymbolMap> Result;
+
+  cantFail(JD.define(std::make_unique<SimpleMaterializationUnit>(
+      SymbolFlagsMap(
+          {{Foo, JITSymbolFlags::Exported |
+                     JITSymbolFlags::MaterializationSideEffectsOnly}}),
+      [&](MaterializationResponsibility R) { FooR.emplace(std::move(R)); })));
+
+  ES.lookup(
+      LookupKind::Static, makeJITDylibSearchOrder(&JD),
+      SymbolLookupSet({Foo}, SymbolLookupFlags::WeaklyReferencedSymbol),
+      SymbolState::Ready,
+      [&](Expected<SymbolMap> LookupResult) {
+        if (LookupResult)
+          Result = std::move(*LookupResult);
+        else
+          ADD_FAILURE() << "Unexpected lookup error: "
+                        << toString(LookupResult.takeError());
+      },
+      NoDependenciesToRegister);
+
+  EXPECT_FALSE(Result) << "Lookup returned unexpectedly";
+  EXPECT_TRUE(FooR) << "Lookup failed to trigger materialization";
+  EXPECT_THAT_ERROR(FooR->notifyEmitted(), Succeeded())
+      << "Emission of materialization-side-effects-only symbol failed";
+
+  EXPECT_TRUE(Result) << "Lookup failed to return";
+  EXPECT_TRUE(Result->empty()) << "Lookup result contained unexpected value";
+}
+
 TEST_F(CoreAPIsStandardTest, RemoveSymbolsTest) {
   // Test that:
   // (1) Missing symbols generate a SymbolsNotFound error.
diff --git a/llvm/unittests/IR/CFGBuilder.cpp b/llvm/unittests/IR/CFGBuilder.cpp
index 424a83bb635be..3583ab2a8a557 100644
--- a/llvm/unittests/IR/CFGBuilder.cpp
+++ b/llvm/unittests/IR/CFGBuilder.cpp
@@ -269,8 +269,8 @@ TEST(CFGBuilder, Rebuild) {
 
 static_assert(is_trivially_copyable<succ_iterator>::value,
               "trivially copyable");
-static_assert(is_trivially_copyable<succ_const_iterator>::value,
+static_assert(is_trivially_copyable<const_succ_iterator>::value,
               "trivially copyable");
 static_assert(is_trivially_copyable<succ_range>::value, "trivially copyable");
-static_assert(is_trivially_copyable<succ_const_range>::value,
+static_assert(is_trivially_copyable<const_succ_range>::value,
               "trivially copyable");
diff --git a/llvm/unittests/IR/CMakeLists.txt b/llvm/unittests/IR/CMakeLists.txt
index 05df08d46b29b..d0068517f09a2 100644
--- a/llvm/unittests/IR/CMakeLists.txt
+++ b/llvm/unittests/IR/CMakeLists.txt
@@ -24,7 +24,6 @@ add_llvm_unittest(IRTests
   IRBuilderTest.cpp
   InstructionsTest.cpp
   IntrinsicsTest.cpp
-  KnowledgeRetentionTest.cpp
   LegacyPassManagerTest.cpp
   MDBuilderTest.cpp
   ManglerTest.cpp
diff --git a/llvm/unittests/IR/InstructionsTest.cpp b/llvm/unittests/IR/InstructionsTest.cpp
index c2f70724337c7..f49c75a6015f8 100644
--- a/llvm/unittests/IR/InstructionsTest.cpp
+++ b/llvm/unittests/IR/InstructionsTest.cpp
@@ -197,6 +197,12 @@ TEST(InstructionsTest, CastInst) {
   Type *V2Int32Ty = VectorType::get(Int32Ty, 2);
   Type *V2Int64Ty = VectorType::get(Int64Ty, 2);
   Type *V4Int16Ty = VectorType::get(Int16Ty, 4);
+  Type *V1Int16Ty = VectorType::get(Int16Ty, 1);
+
+  Type *VScaleV2Int32Ty = VectorType::get(Int32Ty, 2, true);
+  Type *VScaleV2Int64Ty = VectorType::get(Int64Ty, 2, true);
+  Type *VScaleV4Int16Ty = VectorType::get(Int16Ty, 4, true);
+  Type *VScaleV1Int16Ty = VectorType::get(Int16Ty, 1, true);
 
   Type *Int32PtrTy = PointerType::get(Int32Ty, 0);
   Type *Int64PtrTy = PointerType::get(Int64Ty, 0);
@@ -207,11 +213,15 @@ TEST(InstructionsTest, CastInst) {
   Type *V2Int32PtrAS1Ty = VectorType::get(Int32PtrAS1Ty, 2);
   Type *V2Int64PtrAS1Ty = VectorType::get(Int64PtrAS1Ty, 2);
   Type *V4Int32PtrAS1Ty = VectorType::get(Int32PtrAS1Ty, 4);
+  Type *VScaleV4Int32PtrAS1Ty = VectorType::get(Int32PtrAS1Ty, 4, true);
   Type *V4Int64PtrAS1Ty = VectorType::get(Int64PtrAS1Ty, 4);
 
   Type *V2Int64PtrTy = VectorType::get(Int64PtrTy, 2);
   Type *V2Int32PtrTy = VectorType::get(Int32PtrTy, 2);
+  Type *VScaleV2Int32PtrTy = VectorType::get(Int32PtrTy, 2, true);
   Type *V4Int32PtrTy = VectorType::get(Int32PtrTy, 4);
+  Type *VScaleV4Int32PtrTy = VectorType::get(Int32PtrTy, 4, true);
+  Type *VScaleV4Int64PtrTy = VectorType::get(Int64PtrTy, 4, true);
 
   const Constant* c8 = Constant::getNullValue(V8x8Ty);
   const Constant* c64 = Constant::getNullValue(V8x64Ty);
@@ -286,6 +296,75 @@ TEST(InstructionsTest, CastInst) {
                                      Constant::getNullValue(V2Int32PtrTy),
                                      V4Int32PtrAS1Ty));
 
+  // Address space cast of fixed/scalable vectors of pointers to scalable/fixed
+  // vector of pointers.
+  EXPECT_FALSE(CastInst::castIsValid(
+      Instruction::AddrSpaceCast, Constant::getNullValue(VScaleV4Int32PtrAS1Ty),
+      V4Int32PtrTy));
+  EXPECT_FALSE(CastInst::castIsValid(Instruction::AddrSpaceCast,
+                                     Constant::getNullValue(V4Int32PtrTy),
+                                     VScaleV4Int32PtrAS1Ty));
+  // Address space cast of scalable vectors of pointers to scalable vector of
+  // pointers.
+  EXPECT_FALSE(CastInst::castIsValid(
+      Instruction::AddrSpaceCast, Constant::getNullValue(VScaleV4Int32PtrAS1Ty),
+      VScaleV2Int32PtrTy));
+  EXPECT_FALSE(CastInst::castIsValid(Instruction::AddrSpaceCast,
+                                     Constant::getNullValue(VScaleV2Int32PtrTy),
+                                     VScaleV4Int32PtrAS1Ty));
+  EXPECT_TRUE(CastInst::castIsValid(Instruction::AddrSpaceCast,
+                                    Constant::getNullValue(VScaleV4Int64PtrTy),
+                                    VScaleV4Int32PtrAS1Ty));
+  // Same number of lanes, different address space.
+  EXPECT_TRUE(CastInst::castIsValid(
+      Instruction::AddrSpaceCast, Constant::getNullValue(VScaleV4Int32PtrAS1Ty),
+      VScaleV4Int32PtrTy));
+  // Same number of lanes, same address space.
+  EXPECT_FALSE(CastInst::castIsValid(Instruction::AddrSpaceCast,
+                                     Constant::getNullValue(VScaleV4Int64PtrTy),
+                                     VScaleV4Int32PtrTy));
+
+  // Bit casting fixed/scalable vector to scalable/fixed vectors.
+  EXPECT_FALSE(CastInst::castIsValid(Instruction::BitCast,
+                                     Constant::getNullValue(V2Int32Ty),
+                                     VScaleV2Int32Ty));
+  EXPECT_FALSE(CastInst::castIsValid(Instruction::BitCast,
+                                     Constant::getNullValue(V2Int64Ty),
+                                     VScaleV2Int64Ty));
+  EXPECT_FALSE(CastInst::castIsValid(Instruction::BitCast,
+                                     Constant::getNullValue(V4Int16Ty),
+                                     VScaleV4Int16Ty));
+  EXPECT_FALSE(CastInst::castIsValid(Instruction::BitCast,
+                                     Constant::getNullValue(VScaleV2Int32Ty),
+                                     V2Int32Ty));
+  EXPECT_FALSE(CastInst::castIsValid(Instruction::BitCast,
+                                     Constant::getNullValue(VScaleV2Int64Ty),
+                                     V2Int64Ty));
+  EXPECT_FALSE(CastInst::castIsValid(Instruction::BitCast,
+                                     Constant::getNullValue(VScaleV4Int16Ty),
+                                     V4Int16Ty));
+
+  // Bit casting scalable vectors to scalable vectors.
+  EXPECT_TRUE(CastInst::castIsValid(Instruction::BitCast,
+                                    Constant::getNullValue(VScaleV4Int16Ty),
+                                    VScaleV2Int32Ty));
+  EXPECT_TRUE(CastInst::castIsValid(Instruction::BitCast,
+                                    Constant::getNullValue(VScaleV2Int32Ty),
+                                    VScaleV4Int16Ty));
+  EXPECT_FALSE(CastInst::castIsValid(Instruction::BitCast,
+                                     Constant::getNullValue(VScaleV2Int64Ty),
+                                     VScaleV2Int32Ty));
+  EXPECT_FALSE(CastInst::castIsValid(Instruction::BitCast,
+                                     Constant::getNullValue(VScaleV2Int32Ty),
+                                     VScaleV2Int64Ty));
+
+  // Bitcasting to/from <vscale x 1 x Ty>
+  EXPECT_FALSE(CastInst::castIsValid(Instruction::BitCast,
+                                     Constant::getNullValue(VScaleV1Int16Ty),
+                                     V1Int16Ty));
+  EXPECT_FALSE(CastInst::castIsValid(Instruction::BitCast,
+                                     Constant::getNullValue(V1Int16Ty),
+                                     VScaleV1Int16Ty));
 
   // Check that assertion is not hit when creating a cast with a vector of
   // pointers
diff --git a/llvm/unittests/IR/PatternMatch.cpp b/llvm/unittests/IR/PatternMatch.cpp
index cb11d1ec71f2d..186746bca791d 100644
--- a/llvm/unittests/IR/PatternMatch.cpp
+++ b/llvm/unittests/IR/PatternMatch.cpp
@@ -991,31 +991,32 @@ TEST_F(PatternMatchTest, VectorOps) {
   EXPECT_TRUE(match(EX3, m_ExtractElement(m_Constant(), m_ConstantInt())));
 
   // Test matching shufflevector
-  EXPECT_TRUE(match(SI1, m_ShuffleVector(m_Value(), m_Undef(), m_Zero())));
-  EXPECT_TRUE(match(SI2, m_ShuffleVector(m_Value(A), m_Value(B), m_Value(C))));
+  ArrayRef<int> Mask;
+  EXPECT_TRUE(match(SI1, m_ShuffleVector(m_Value(), m_Undef(), m_ZeroMask())));
+  EXPECT_TRUE(
+      match(SI2, m_ShuffleVector(m_Value(A), m_Value(B), m_Mask(Mask))));
   EXPECT_TRUE(A == VI3);
   EXPECT_TRUE(B == VI4);
-  EXPECT_TRUE(C == IdxVec);
   A = B = C = nullptr; // reset
 
   // Test matching the vector splat pattern
   EXPECT_TRUE(match(
       SI1,
       m_ShuffleVector(m_InsertElement(m_Undef(), m_SpecificInt(1), m_Zero()),
-                      m_Undef(), m_Zero())));
+                      m_Undef(), m_ZeroMask())));
   EXPECT_FALSE(match(
       SI3, m_ShuffleVector(m_InsertElement(m_Undef(), m_Value(), m_Zero()),
-                           m_Undef(), m_Zero())));
+                           m_Undef(), m_ZeroMask())));
   EXPECT_FALSE(match(
       SI4, m_ShuffleVector(m_InsertElement(m_Undef(), m_Value(), m_Zero()),
-                           m_Undef(), m_Zero())));
+                           m_Undef(), m_ZeroMask())));
   EXPECT_TRUE(match(
       SP1,
       m_ShuffleVector(m_InsertElement(m_Undef(), m_SpecificInt(2), m_Zero()),
-                      m_Undef(), m_Zero())));
+                      m_Undef(), m_ZeroMask())));
   EXPECT_TRUE(match(
       SP2, m_ShuffleVector(m_InsertElement(m_Undef(), m_Value(A), m_Zero()),
-                           m_Undef(), m_Zero())));
+                           m_Undef(), m_ZeroMask())));
   EXPECT_TRUE(A == Val);
 }
 
diff --git a/llvm/unittests/MC/AMDGPU/DwarfRegMappings.cpp b/llvm/unittests/MC/AMDGPU/DwarfRegMappings.cpp
index e416ece9aaa85..aea69bb7297a2 100644
--- a/llvm/unittests/MC/AMDGPU/DwarfRegMappings.cpp
+++ b/llvm/unittests/MC/AMDGPU/DwarfRegMappings.cpp
@@ -65,10 +65,10 @@ TEST(AMDGPUDwarfRegMappingTests, TestWave32DwarfRegMapping) {
     if (TM && TM->getMCRegisterInfo()) {
       auto MRI = TM->getMCRegisterInfo();
       // Wave32 Dwarf register mapping test numbers
-      // PC_32 => 0, EXEC_MASK_32 => 1, S0 => 32, S63 => 95,
+      // PC_64 => 16, EXEC_MASK_32 => 1, S0 => 32, S63 => 95,
       // S64 => 1088, S105 => 1129, V0 => 1536, V255 => 1791,
       // A0 => 2048, A255 => 2303
-      for (int llvmReg : {0, 1, 32, 95, 1088, 1129, 1536, 1791, 2048, 2303}) {
+      for (int llvmReg : {16, 1, 32, 95, 1088, 1129, 1536, 1791, 2048, 2303}) {
         MCRegister PCReg(*MRI->getLLVMRegNum(llvmReg, false));
         EXPECT_EQ(llvmReg, MRI->getDwarfRegNum(PCReg, false));
       }
diff --git a/llvm/unittests/ProfileData/SampleProfTest.cpp b/llvm/unittests/ProfileData/SampleProfTest.cpp
index 866a9ec21f7a8..3aaba056c6187 100644
--- a/llvm/unittests/ProfileData/SampleProfTest.cpp
+++ b/llvm/unittests/ProfileData/SampleProfTest.cpp
@@ -77,11 +77,13 @@ struct SampleProfTest : ::testing::Test {
     OS->close();
   }
 
-  void testRoundTrip(SampleProfileFormat Format, bool Remap) {
+  void testRoundTrip(SampleProfileFormat Format, bool Remap, bool UseMD5) {
     SmallVector<char, 128> ProfilePath;
     ASSERT_TRUE(NoError(llvm::sys::fs::createTemporaryFile("profile", "", ProfilePath)));
     StringRef Profile(ProfilePath.data(), ProfilePath.size());
     createWriter(Format, Profile);
+    if (Format == SampleProfileFormat::SPF_Ext_Binary && UseMD5)
+      static_cast<SampleProfileWriterExtBinary *>(Writer.get())->setUseMD5();
 
     StringRef FooName("_Z3fooi");
     FunctionSamples FooSamples;
@@ -168,7 +170,7 @@ struct SampleProfTest : ::testing::Test {
 
     FunctionSamples *ReadFooSamples = Reader->getSamplesFor(FooName);
     ASSERT_TRUE(ReadFooSamples != nullptr);
-    if (Format != SampleProfileFormat::SPF_Compact_Binary) {
+    if (!UseMD5) {
       ASSERT_EQ("_Z3fooi", ReadFooSamples->getName());
     }
     ASSERT_EQ(7711u, ReadFooSamples->getTotalSamples());
@@ -176,7 +178,7 @@ struct SampleProfTest : ::testing::Test {
 
     FunctionSamples *ReadBarSamples = Reader->getSamplesFor(BarName);
     ASSERT_TRUE(ReadBarSamples != nullptr);
-    if (Format != SampleProfileFormat::SPF_Compact_Binary) {
+    if (!UseMD5) {
       ASSERT_EQ("_Z3bari", ReadBarSamples->getName());
     }
     ASSERT_EQ(20301u, ReadBarSamples->getTotalSamples());
@@ -205,10 +207,10 @@ struct SampleProfTest : ::testing::Test {
 
     std::string MconstructGUID;
     StringRef MconstructRep =
-        getRepInFormat(MconstructName, Format, MconstructGUID);
+        getRepInFormat(MconstructName, UseMD5, MconstructGUID);
     std::string StringviewGUID;
     StringRef StringviewRep =
-        getRepInFormat(StringviewName, Format, StringviewGUID);
+        getRepInFormat(StringviewName, UseMD5, StringviewGUID);
     ASSERT_EQ(1000u, CTMap.get()[MconstructRep]);
     ASSERT_EQ(437u, CTMap.get()[StringviewRep]);
 
@@ -333,31 +335,35 @@ struct SampleProfTest : ::testing::Test {
 };
 
 TEST_F(SampleProfTest, roundtrip_text_profile) {
-  testRoundTrip(SampleProfileFormat::SPF_Text, false);
+  testRoundTrip(SampleProfileFormat::SPF_Text, false, false);
 }
 
 TEST_F(SampleProfTest, roundtrip_raw_binary_profile) {
-  testRoundTrip(SampleProfileFormat::SPF_Binary, false);
+  testRoundTrip(SampleProfileFormat::SPF_Binary, false, false);
 }
 
 TEST_F(SampleProfTest, roundtrip_compact_binary_profile) {
-  testRoundTrip(SampleProfileFormat::SPF_Compact_Binary, false);
+  testRoundTrip(SampleProfileFormat::SPF_Compact_Binary, false, true);
 }
 
 TEST_F(SampleProfTest, roundtrip_ext_binary_profile) {
-  testRoundTrip(SampleProfileFormat::SPF_Ext_Binary, false);
+  testRoundTrip(SampleProfileFormat::SPF_Ext_Binary, false, false);
+}
+
+TEST_F(SampleProfTest, roundtrip_md5_ext_binary_profile) {
+  testRoundTrip(SampleProfileFormat::SPF_Ext_Binary, false, true);
 }
 
 TEST_F(SampleProfTest, remap_text_profile) {
-  testRoundTrip(SampleProfileFormat::SPF_Text, true);
+  testRoundTrip(SampleProfileFormat::SPF_Text, true, false);
 }
 
 TEST_F(SampleProfTest, remap_raw_binary_profile) {
-  testRoundTrip(SampleProfileFormat::SPF_Binary, true);
+  testRoundTrip(SampleProfileFormat::SPF_Binary, true, false);
 }
 
 TEST_F(SampleProfTest, remap_ext_binary_profile) {
-  testRoundTrip(SampleProfileFormat::SPF_Ext_Binary, true);
+  testRoundTrip(SampleProfileFormat::SPF_Ext_Binary, true, false);
 }
 
 TEST_F(SampleProfTest, sample_overflow_saturation) {
diff --git a/llvm/unittests/Support/ARMAttributeParser.cpp b/llvm/unittests/Support/ARMAttributeParser.cpp
index 76a0407df6a6b..88e3ce389a344 100644
--- a/llvm/unittests/Support/ARMAttributeParser.cpp
+++ b/llvm/unittests/Support/ARMAttributeParser.cpp
@@ -1,5 +1,6 @@
 #include "llvm/Support/ARMAttributeParser.h"
 #include "llvm/Support/ARMBuildAttributes.h"
+#include "llvm/Support/ELFAttributes.h"
 #include "gtest/gtest.h"
 #include <string>
 
@@ -36,8 +37,8 @@ bool testBuildAttr(unsigned Tag, unsigned Value,
   ARMAttributeParser Parser;
   cantFail(Parser.parse(Bytes, support::little));
 
-  return (Parser.hasAttribute(ExpectedTag) &&
-    Parser.getAttributeValue(ExpectedTag) == ExpectedValue);
+  Optional<unsigned> Attr = Parser.getAttributeValue(ExpectedTag);
+  return Attr.hasValue() && Attr.getValue() == ExpectedValue;
 }
 
 void testParseError(ArrayRef<uint8_t> bytes, const char *msg) {
@@ -47,34 +48,8 @@ void testParseError(ArrayRef<uint8_t> bytes, const char *msg) {
 }
 
 bool testTagString(unsigned Tag, const char *name) {
-  return ARMBuildAttrs::AttrTypeAsString(Tag).str() == name;
-}
-
-TEST(ARMAttributeParser, UnrecognizedFormatVersion) {
-  static const uint8_t bytes[] = {1};
-  testParseError(bytes, "unrecognized format-version: 0x1");
-}
-
-TEST(ARMAttributeParser, InvalidSubsectionLength) {
-  static const uint8_t bytes[] = {'A', 3, 0, 0, 0};
-  testParseError(bytes, "invalid subsection length 3 at offset 0x1");
-}
-
-TEST(ARMAttributeParser, UnrecognizedVendorName) {
-  static const uint8_t bytes[] = {'A', 7, 0, 0, 0, 'x', 'y', 0};
-  testParseError(bytes, "unrecognized vendor-name: xy");
-}
-
-TEST(ARMAttributeParser, InvalidAttributeSize) {
-  static const uint8_t bytes[] = {'A', 15,  0, 0, 0, 'a', 'e', 'a',
-                                  'b', 'i', 0, 4, 4, 0,   0,   0};
-  testParseError(bytes, "invalid attribute size 4 at offset 0xb");
-}
-
-TEST(ARMAttributeParser, UnrecognizedTag) {
-  static const uint8_t bytes[] = {'A', 15,  0, 0, 0, 'a', 'e', 'a',
-                                  'b', 'i', 0, 4, 5, 0,   0,   0};
-  testParseError(bytes, "unrecognized tag 0x4 at offset 0xb");
+  return ELFAttrs::attrTypeAsString(Tag, ARMBuildAttrs::ARMAttributeTags)
+             .str() == name;
 }
 
 TEST(ARMAttributeParser, UnknownCPU_arch) {
diff --git a/llvm/unittests/Support/CMakeLists.txt b/llvm/unittests/Support/CMakeLists.txt
index 6ffa8c1d6f64b..2710114a3660c 100644
--- a/llvm/unittests/Support/CMakeLists.txt
+++ b/llvm/unittests/Support/CMakeLists.txt
@@ -28,6 +28,7 @@ add_llvm_unittest(SupportTests
   DJBTest.cpp
   EndianStreamTest.cpp
   EndianTest.cpp
+  ELFAttributeParserTest.cpp
   ErrnoTest.cpp
   ErrorOrTest.cpp
   ErrorTest.cpp
@@ -60,6 +61,7 @@ add_llvm_unittest(SupportTests
   RegexTest.cpp
   ReverseIterationTest.cpp
   ReplaceFileTest.cpp
+  RISCVAttributeParserTest.cpp
   ScaledNumberTest.cpp
   SimpleTableTest.cpp
   SourceMgrTest.cpp
diff --git a/llvm/unittests/Support/ELFAttributeParserTest.cpp b/llvm/unittests/Support/ELFAttributeParserTest.cpp
new file mode 100644
index 0000000000000..8234d4ee176f2
--- /dev/null
+++ b/llvm/unittests/Support/ELFAttributeParserTest.cpp
@@ -0,0 +1,63 @@
+//===----- unittests/ELFAttributeParserTest.cpp ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/ELFAttributeParser.h"
+#include "llvm/Support/ELFAttributes.h"
+#include "gtest/gtest.h"
+#include <string>
+
+using namespace llvm;
+
+static const TagNameMap emptyTagNameMap;
+
+// This class is used to test the common part of the ELF attribute section.
+class AttributeHeaderParser : public ELFAttributeParser {
+  Error handler(uint64_t tag, bool &handled) {
+    // Treat all attributes as handled.
+    handled = true;
+    return Error::success();
+  }
+
+public:
+  AttributeHeaderParser(ScopedPrinter *printer)
+      : ELFAttributeParser(printer, emptyTagNameMap, "test") {}
+  AttributeHeaderParser() : ELFAttributeParser(emptyTagNameMap, "test") {}
+};
+
+static void testParseError(ArrayRef<uint8_t> bytes, const char *msg) {
+  AttributeHeaderParser parser;
+  Error e = parser.parse(bytes, support::little);
+  EXPECT_STREQ(toString(std::move(e)).c_str(), msg);
+}
+
+TEST(AttributeHeaderParser, UnrecognizedFormatVersion) {
+  static const uint8_t bytes[] = {1};
+  testParseError(bytes, "unrecognized format-version: 0x1");
+}
+
+TEST(AttributeHeaderParser, InvalidSectionLength) {
+  static const uint8_t bytes[] = {'A', 3, 0, 0, 0};
+  testParseError(bytes, "invalid section length 3 at offset 0x1");
+}
+
+TEST(AttributeHeaderParser, UnrecognizedVendorName) {
+  static const uint8_t bytes[] = {'A', 7, 0, 0, 0, 'x', 'y', 0};
+  testParseError(bytes, "unrecognized vendor-name: xy");
+}
+
+TEST(AttributeHeaderParser, UnrecognizedTag) {
+  static const uint8_t bytes[] = {'A', 14, 0, 0, 0, 't', 'e', 's',
+                                  't', 0,  4, 5, 0, 0,   0};
+  testParseError(bytes, "unrecognized tag 0x4 at offset 0xa");
+}
+
+TEST(AttributeHeaderParser, InvalidAttributeSize) {
+  static const uint8_t bytes[] = {'A', 14, 0, 0, 0, 't', 'e', 's',
+                                  't', 0,  1, 4, 0, 0,   0};
+  testParseError(bytes, "invalid attribute size 4 at offset 0xa");
+}
diff --git a/llvm/unittests/Support/FileCollectorTest.cpp b/llvm/unittests/Support/FileCollectorTest.cpp
index 42a4877dba391..a81ce7130f2f2 100644
--- a/llvm/unittests/Support/FileCollectorTest.cpp
+++ b/llvm/unittests/Support/FileCollectorTest.cpp
@@ -120,6 +120,41 @@ TEST(FileCollectorTest, addFile) {
   EXPECT_FALSE(FileCollector.hasSeen("/path/to/d"));
 }
 
+TEST(FileCollectorTest, addDirectory) {
+  ScopedDir file_root("file_root", true);
+
+  llvm::SmallString<128> aaa = file_root.Path;
+  llvm::sys::path::append(aaa, "aaa");
+  ScopedFile a(aaa.str());
+
+  llvm::SmallString<128> bbb = file_root.Path;
+  llvm::sys::path::append(bbb, "bbb");
+  ScopedFile b(bbb.str());
+
+  llvm::SmallString<128> ccc = file_root.Path;
+  llvm::sys::path::append(ccc, "ccc");
+  ScopedFile c(ccc.str());
+
+  std::string root_fs = std::string(file_root.Path.str());
+  TestingFileCollector FileCollector(root_fs, root_fs);
+
+  FileCollector.addDirectory(file_root.Path);
+
+  // Make sure the root is correct.
+  EXPECT_EQ(FileCollector.Root, root_fs);
+
+  // Make sure we've seen all the added files.
+  EXPECT_TRUE(FileCollector.hasSeen(a.Path));
+  EXPECT_TRUE(FileCollector.hasSeen(b.Path));
+  EXPECT_TRUE(FileCollector.hasSeen(c.Path));
+
+  // Make sure we've only seen the added files.
+  llvm::SmallString<128> ddd = file_root.Path;
+  llvm::sys::path::append(ddd, "ddd");
+  ScopedFile d(ddd.str());
+  EXPECT_FALSE(FileCollector.hasSeen(d.Path));
+}
+
 TEST(FileCollectorTest, copyFiles) {
   ScopedDir file_root("file_root", true);
   ScopedFile a(file_root + "/aaa");
diff --git a/llvm/unittests/Support/Path.cpp b/llvm/unittests/Support/Path.cpp
index 671966b52dd0c..1a2ac1818eb04 100644
--- a/llvm/unittests/Support/Path.cpp
+++ b/llvm/unittests/Support/Path.cpp
@@ -1254,21 +1254,14 @@ TEST(Support, ReplacePathPrefix) {
   path::replace_path_prefix(Path, OldPrefix, EmptyPrefix);
   EXPECT_EQ(Path, "/foo");
   Path = Path2;
-  path::replace_path_prefix(Path, OldPrefix, EmptyPrefix, path::Style::native,
-                            true);
+  path::replace_path_prefix(Path, OldPrefixSep, EmptyPrefix);
   EXPECT_EQ(Path, "foo");
   Path = Path3;
-  path::replace_path_prefix(Path, OldPrefix, NewPrefix, path::Style::native,
-                            false);
+  path::replace_path_prefix(Path, OldPrefix, NewPrefix);
   EXPECT_EQ(Path, "/newnew/foo");
   Path = Path3;
-  path::replace_path_prefix(Path, OldPrefix, NewPrefix, path::Style::native,
-                            true);
-  EXPECT_EQ(Path, "/oldnew/foo");
-  Path = Path3;
-  path::replace_path_prefix(Path, OldPrefixSep, NewPrefix, path::Style::native,
-                            true);
-  EXPECT_EQ(Path, "/oldnew/foo");
+  path::replace_path_prefix(Path, OldPrefix, NewPrefix2);
+  EXPECT_EQ(Path, "/longernewnew/foo");
   Path = Path1;
   path::replace_path_prefix(Path, EmptyPrefix, NewPrefix);
   EXPECT_EQ(Path, "/new/foo");
@@ -1279,13 +1272,8 @@ TEST(Support, ReplacePathPrefix) {
   path::replace_path_prefix(Path, OldPrefix, NewPrefix);
   EXPECT_EQ(Path, "/new/");
   Path = OldPrefix;
-  path::replace_path_prefix(Path, OldPrefixSep, NewPrefix, path::Style::native,
-                            false);
+  path::replace_path_prefix(Path, OldPrefixSep, NewPrefix);
   EXPECT_EQ(Path, "/old");
-  Path = OldPrefix;
-  path::replace_path_prefix(Path, OldPrefixSep, NewPrefix, path::Style::native,
-                            true);
-  EXPECT_EQ(Path, "/new");
 }
 
 TEST_F(FileSystemTest, OpenFileForRead) {
diff --git a/llvm/unittests/Support/RISCVAttributeParserTest.cpp b/llvm/unittests/Support/RISCVAttributeParserTest.cpp
new file mode 100644
index 0000000000000..1458af604a5c5
--- /dev/null
+++ b/llvm/unittests/Support/RISCVAttributeParserTest.cpp
@@ -0,0 +1,70 @@
+//===----- unittests/RISCVAttributeParserTest.cpp -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "llvm/Support/RISCVAttributeParser.h"
+#include "llvm/Support/ARMBuildAttributes.h"
+#include "llvm/Support/ELFAttributes.h"
+#include "gtest/gtest.h"
+#include <string>
+
+using namespace llvm;
+
+struct RISCVAttributeSection {
+  unsigned Tag;
+  unsigned Value;
+
+  RISCVAttributeSection(unsigned tag, unsigned value)
+      : Tag(tag), Value(value) {}
+
+  void write(raw_ostream &OS) {
+    OS.flush();
+    // length = length + "riscv\0" + TagFile + ByteSize + Tag + Value;
+    // length = 17 bytes
+
+    OS << 'A' << (uint8_t)17 << (uint8_t)0 << (uint8_t)0 << (uint8_t)0;
+    OS << "riscv" << '\0';
+    OS << (uint8_t)1 << (uint8_t)7 << (uint8_t)0 << (uint8_t)0 << (uint8_t)0;
+    OS << (uint8_t)Tag << (uint8_t)Value;
+  }
+};
+
+static bool testAttribute(unsigned Tag, unsigned Value, unsigned ExpectedTag,
+                          unsigned ExpectedValue) {
+  std::string buffer;
+  raw_string_ostream OS(buffer);
+  RISCVAttributeSection Section(Tag, Value);
+  Section.write(OS);
+  ArrayRef<uint8_t> Bytes(reinterpret_cast<const uint8_t *>(OS.str().c_str()),
+                          OS.str().size());
+
+  RISCVAttributeParser Parser;
+  cantFail(Parser.parse(Bytes, support::little));
+
+  Optional<unsigned> Attr = Parser.getAttributeValue(ExpectedTag);
+  return Attr.hasValue() && Attr.getValue() == ExpectedValue;
+}
+
+static bool testTagString(unsigned Tag, const char *name) {
+  return ELFAttrs::attrTypeAsString(Tag, RISCVAttrs::RISCVAttributeTags)
+             .str() == name;
+}
+
+TEST(StackAlign, testAttribute) {
+  EXPECT_TRUE(testTagString(4, "Tag_stack_align"));
+  EXPECT_TRUE(
+      testAttribute(4, 4, RISCVAttrs::STACK_ALIGN, RISCVAttrs::ALIGN_4));
+  EXPECT_TRUE(
+      testAttribute(4, 16, RISCVAttrs::STACK_ALIGN, RISCVAttrs::ALIGN_16));
+}
+
+TEST(UnalignedAccess, testAttribute) {
+  EXPECT_TRUE(testTagString(6, "Tag_unaligned_access"));
+  EXPECT_TRUE(testAttribute(6, 0, RISCVAttrs::UNALIGNED_ACCESS,
+                            RISCVAttrs::NOT_ALLOWED));
+  EXPECT_TRUE(
+      testAttribute(6, 1, RISCVAttrs::UNALIGNED_ACCESS, RISCVAttrs::ALLOWED));
+}
diff --git a/llvm/unittests/Support/TargetParserTest.cpp b/llvm/unittests/Support/TargetParserTest.cpp
index 2d68b4c7511dd..825a127e3e08f 100644
--- a/llvm/unittests/Support/TargetParserTest.cpp
+++ b/llvm/unittests/Support/TargetParserTest.cpp
@@ -26,9 +26,9 @@ const char *ARMArch[] = {
     "armv7e-m",    "armv7em",      "armv8-a",     "armv8",        "armv8a",
     "armv8l",      "armv8.1-a",    "armv8.1a",    "armv8.2-a",    "armv8.2a",
     "armv8.3-a",   "armv8.3a",     "armv8.4-a",   "armv8.4a",     "armv8.5-a",
-    "armv8.5a",     "armv8-r",     "armv8r",      "armv8-m.base", "armv8m.base",
-    "armv8-m.main", "armv8m.main", "iwmmxt",      "iwmmxt2",      "xscale",
-    "armv8.1-m.main",
+    "armv8.5a",     "armv8.6-a",   "armv8.6a",     "armv8-r",     "armv8r",
+    "armv8-m.base", "armv8m.base", "armv8-m.main", "armv8m.main", "iwmmxt",
+    "iwmmxt2",      "xscale",      "armv8.1-m.main",
 };
 
 bool testARMCPU(StringRef CPUName, StringRef ExpectedArch,
@@ -410,6 +410,9 @@ TEST(TargetParserTest, testARMArch) {
   EXPECT_TRUE(
       testARMArch("armv8.5-a", "generic", "v8.5a",
                           ARMBuildAttrs::CPUArch::v8_A));
+  EXPECT_TRUE(
+      testARMArch("armv8.6-a", "generic", "v8.6a",
+                          ARMBuildAttrs::CPUArch::v8_A));
   EXPECT_TRUE(
       testARMArch("armv8-r", "cortex-r52", "v8r",
                           ARMBuildAttrs::CPUArch::v8_R));
@@ -678,7 +681,7 @@ TEST(TargetParserTest, ARMparseArchEndianAndISA) {
       "v7",   "v7a",    "v7ve",  "v7hl",   "v7l",   "v7-r",   "v7r",   "v7-m",
       "v7m",  "v7k",    "v7s",   "v7e-m",  "v7em",  "v8-a",   "v8",    "v8a",
       "v8l",  "v8.1-a", "v8.1a", "v8.2-a", "v8.2a", "v8.3-a", "v8.3a", "v8.4-a",
-      "v8.4a", "v8.5-a","v8.5a", "v8-r",   "v8m.base", "v8m.main", "v8.1m.main"
+      "v8.4a", "v8.5-a","v8.5a", "v8.6-a", "v8.6a", "v8-r",   "v8m.base", "v8m.main", "v8.1m.main"
   };
 
   for (unsigned i = 0; i < array_lengthof(Arch); i++) {
@@ -743,6 +746,7 @@ TEST(TargetParserTest, ARMparseArchProfile) {
     case ARM::ArchKind::ARMV8_3A:
     case ARM::ArchKind::ARMV8_4A:
     case ARM::ArchKind::ARMV8_5A:
+    case ARM::ArchKind::ARMV8_6A:
       EXPECT_EQ(ARM::ProfileKind::A, ARM::parseArchProfile(ARMArch[i]));
       break;
     default:
@@ -1008,6 +1012,8 @@ TEST(TargetParserTest, testAArch64Arch) {
                               ARMBuildAttrs::CPUArch::v8_A));
   EXPECT_TRUE(testAArch64Arch("armv8.5-a", "generic", "v8.5a",
                               ARMBuildAttrs::CPUArch::v8_A));
+  EXPECT_TRUE(testAArch64Arch("armv8.6-a", "generic", "v8.6a",
+                              ARMBuildAttrs::CPUArch::v8_A));
 }
 
 bool testAArch64Extension(StringRef CPUName, AArch64::ArchKind AK,
diff --git a/llvm/unittests/Support/VirtualFileSystemTest.cpp b/llvm/unittests/Support/VirtualFileSystemTest.cpp
index 8ac454b511e8f..f4e999dedd679 100644
--- a/llvm/unittests/Support/VirtualFileSystemTest.cpp
+++ b/llvm/unittests/Support/VirtualFileSystemTest.cpp
@@ -2200,20 +2200,14 @@ TEST_F(VFSFromYAMLTest, YAMLVFSWriterTest) {
   ScopedDir _g(TestDirectory + "/g");
   ScopedFile _h(TestDirectory + "/h", "");
 
-  // This test exposes a bug/shortcoming in the YAMLVFSWriter. Below we call
-  // addFileMapping for _a and _e, which causes _ab and _ef not to exists in
-  // the deserialized file system, because _a and _e got emitted as regular
-  // files. The counter example is _c, if we only call addFileMapping for _cd,
-  // things work as expected.
-
   vfs::YAMLVFSWriter VFSWriter;
-  VFSWriter.addFileMapping(_a.Path, "//root/a");
+  VFSWriter.addDirectoryMapping(_a.Path, "//root/a");
   VFSWriter.addFileMapping(_ab.Path, "//root/a/b");
   VFSWriter.addFileMapping(_cd.Path, "//root/c/d");
-  VFSWriter.addFileMapping(_e.Path, "//root/e");
-  VFSWriter.addFileMapping(_ef.Path, "//root/e/f");
+  VFSWriter.addDirectoryMapping(_e.Path, "//root/e");
+  VFSWriter.addDirectoryMapping(_ef.Path, "//root/e/f");
   VFSWriter.addFileMapping(_g.Path, "//root/g");
-  VFSWriter.addFileMapping(_h.Path, "//root/h");
+  VFSWriter.addDirectoryMapping(_h.Path, "//root/h");
 
   std::string Buffer;
   raw_string_ostream OS(Buffer);
@@ -2236,11 +2230,11 @@ TEST_F(VFSFromYAMLTest, YAMLVFSWriterTest) {
   ASSERT_TRUE(FS.get() != nullptr);
 
   EXPECT_TRUE(FS->exists(_a.Path));
-  EXPECT_FALSE(FS->exists(_ab.Path)); // FIXME: See explanation above.
+  EXPECT_TRUE(FS->exists(_ab.Path));
   EXPECT_TRUE(FS->exists(_c.Path));
   EXPECT_TRUE(FS->exists(_cd.Path));
   EXPECT_TRUE(FS->exists(_e.Path));
-  EXPECT_FALSE(FS->exists(_ef.Path)); // FIXME: See explanation above.
+  EXPECT_TRUE(FS->exists(_ef.Path));
   EXPECT_TRUE(FS->exists(_g.Path));
   EXPECT_TRUE(FS->exists(_h.Path));
 }
diff --git a/llvm/unittests/Target/AMDGPU/DwarfRegMappings.cpp b/llvm/unittests/Target/AMDGPU/DwarfRegMappings.cpp
index 87facdee061dc..fca919993dc79 100644
--- a/llvm/unittests/Target/AMDGPU/DwarfRegMappings.cpp
+++ b/llvm/unittests/Target/AMDGPU/DwarfRegMappings.cpp
@@ -75,10 +75,11 @@ TEST(AMDGPUDwarfRegMappingTests, TestWave32DwarfRegMapping) {
       auto MRI = ST.getRegisterInfo();
       if (MRI) {
         // Wave32 Dwarf register mapping test numbers
-        // PC_32 => 0, EXEC_MASK_32 => 1, S0 => 32, S63 => 95,
+        // PC_64 => 16, EXEC_MASK_32 => 1, S0 => 32, S63 => 95,
         // S64 => 1088, S105 => 1129, V0 => 1536, V255 => 1791,
         // A0 => 2048, A255 => 2303
-        for (int llvmReg : {0, 1, 32, 95, 1088, 1129, 1536, 1791, 2048, 2303}) {
+        for (int llvmReg :
+             {16, 1, 32, 95, 1088, 1129, 1536, 1791, 2048, 2303}) {
           MCRegister PCReg(*MRI->getLLVMRegNum(llvmReg, false));
           EXPECT_EQ(llvmReg, MRI->getDwarfRegNum(PCReg, false));
         }
diff --git a/llvm/unittests/Target/ARM/MachineInstrTest.cpp b/llvm/unittests/Target/ARM/MachineInstrTest.cpp
index 90f84d4e5aa1f..36d87dcc2462d 100644
--- a/llvm/unittests/Target/ARM/MachineInstrTest.cpp
+++ b/llvm/unittests/Target/ARM/MachineInstrTest.cpp
@@ -10,6 +10,103 @@
 
 using namespace llvm;
 
+TEST(MachineInstructionDoubleWidthResult, IsCorrect) {
+  using namespace ARM;
+
+  auto DoubleWidthResult = [](unsigned Opcode) {
+    switch (Opcode) {
+    default:
+      break;
+    case MVE_VMULLBp16:
+    case MVE_VMULLBp8:
+    case MVE_VMULLBs16:
+    case MVE_VMULLBs32:
+    case MVE_VMULLBs8:
+    case MVE_VMULLBu16:
+    case MVE_VMULLBu32:
+    case MVE_VMULLBu8:
+    case MVE_VMULLTp16:
+    case MVE_VMULLTp8:
+    case MVE_VMULLTs16:
+    case MVE_VMULLTs32:
+    case MVE_VMULLTs8:
+    case MVE_VMULLTu16:
+    case MVE_VMULLTu32:
+    case MVE_VMULLTu8:
+    case MVE_VQDMULL_qr_s16bh:
+    case MVE_VQDMULL_qr_s16th:
+    case MVE_VQDMULL_qr_s32bh:
+    case MVE_VQDMULL_qr_s32th:
+    case MVE_VQDMULLs16bh:
+    case MVE_VQDMULLs16th:
+    case MVE_VQDMULLs32bh:
+    case MVE_VQDMULLs32th:
+    case MVE_VMOVLs16bh:
+    case MVE_VMOVLs16th:
+    case MVE_VMOVLs8bh:
+    case MVE_VMOVLs8th:
+    case MVE_VMOVLu16bh:
+    case MVE_VMOVLu16th:
+    case MVE_VMOVLu8bh:
+    case MVE_VMOVLu8th:
+    case MVE_VSHLL_imms16bh:
+    case MVE_VSHLL_imms16th:
+    case MVE_VSHLL_imms8bh:
+    case MVE_VSHLL_imms8th:
+    case MVE_VSHLL_immu16bh:
+    case MVE_VSHLL_immu16th:
+    case MVE_VSHLL_immu8bh:
+    case MVE_VSHLL_immu8th:
+    case MVE_VSHLL_lws16bh:
+    case MVE_VSHLL_lws16th:
+    case MVE_VSHLL_lws8bh:
+    case MVE_VSHLL_lws8th:
+    case MVE_VSHLL_lwu16bh:
+    case MVE_VSHLL_lwu16th:
+    case MVE_VSHLL_lwu8bh:
+    case MVE_VSHLL_lwu8th:
+      return true;
+    }
+    return false;
+  };
+
+  LLVMInitializeARMTargetInfo();
+  LLVMInitializeARMTarget();
+  LLVMInitializeARMTargetMC();
+
+  auto TT(Triple::normalize("thumbv8.1m.main-arm-none-eabi"));
+  std::string Error;
+  const Target *T = TargetRegistry::lookupTarget(TT, Error);
+  if (!T) {
+    dbgs() << Error;
+    return;
+  }
+
+  TargetOptions Options;
+  auto TM = std::unique_ptr<LLVMTargetMachine>(
+    static_cast<LLVMTargetMachine*>(
+      T->createTargetMachine(TT, "generic", "", Options, None, None,
+                             CodeGenOpt::Default)));
+  ARMSubtarget ST(TM->getTargetTriple(), std::string(TM->getTargetCPU()),
+                  std::string(TM->getTargetFeatureString()),
+                  *static_cast<const ARMBaseTargetMachine *>(TM.get()), false);
+  const ARMBaseInstrInfo *TII = ST.getInstrInfo();
+  auto MII = TM->getMCInstrInfo();
+
+  for (unsigned i = 0; i < ARM::INSTRUCTION_LIST_END; ++i) {
+    const MCInstrDesc &Desc = TII->get(i);
+
+    uint64_t Flags = Desc.TSFlags;
+    if ((Flags & ARMII::DomainMask) != ARMII::DomainMVE)
+      continue;
+
+    bool Valid = (Flags & ARMII::DoubleWidthResult) != 0;
+    ASSERT_EQ(DoubleWidthResult(i), Valid)
+              << MII->getName(i)
+              << ": mismatched expectation for tail-predicated safety\n";
+  }
+}
+
 TEST(MachineInstructionHorizontalReduction, IsCorrect) {
   using namespace ARM;
 
@@ -159,7 +256,6 @@ TEST(MachineInstructionHorizontalReduction, IsCorrect) {
     uint64_t Flags = Desc.TSFlags;
     if ((Flags & ARMII::DomainMask) != ARMII::DomainMVE)
       continue;
-
     bool Valid = (Flags & ARMII::HorizontalReduction) != 0;
     ASSERT_EQ(HorizontalReduction(i), Valid)
               << MII->getName(i)
@@ -311,6 +407,18 @@ TEST(MachineInstrValidTailPredication, IsCorrect) {
     case MVE_VADD_qr_i16:
     case MVE_VADD_qr_i32:
     case MVE_VADD_qr_i8:
+    case MVE_VADDVs16acc:
+    case MVE_VADDVs16no_acc:
+    case MVE_VADDVs32acc:
+    case MVE_VADDVs32no_acc:
+    case MVE_VADDVs8acc:
+    case MVE_VADDVs8no_acc:
+    case MVE_VADDVu16acc:
+    case MVE_VADDVu16no_acc:
+    case MVE_VADDVu32acc:
+    case MVE_VADDVu32no_acc:
+    case MVE_VADDVu8acc:
+    case MVE_VADDVu8no_acc:
     case MVE_VADDf16:
     case MVE_VADDf32:
     case MVE_VADDi16:
@@ -828,59 +936,14 @@ TEST(MachineInstrValidTailPredication, IsCorrect) {
 
 TEST(MachineInstr, HasSideEffects) {
   using namespace ARM;
-  unsigned Opcodes[] = {
-      // MVE Loads/Stores
-      MVE_VLDRBS16,        MVE_VLDRBS16_post,   MVE_VLDRBS16_pre,
-      MVE_VLDRBS16_rq,     MVE_VLDRBS32,        MVE_VLDRBS32_post,
-      MVE_VLDRBS32_pre,    MVE_VLDRBS32_rq,     MVE_VLDRBU16,
-      MVE_VLDRBU16_post,   MVE_VLDRBU16_pre,    MVE_VLDRBU16_rq,
-      MVE_VLDRBU32,        MVE_VLDRBU32_post,   MVE_VLDRBU32_pre,
-      MVE_VLDRBU32_rq,     MVE_VLDRBU8,         MVE_VLDRBU8_post,
-      MVE_VLDRBU8_pre,     MVE_VLDRBU8_rq,      MVE_VLDRDU64_qi,
-      MVE_VLDRDU64_qi_pre, MVE_VLDRDU64_rq,     MVE_VLDRDU64_rq_u,
-      MVE_VLDRHS32,        MVE_VLDRHS32_post,   MVE_VLDRHS32_pre,
-      MVE_VLDRHS32_rq,     MVE_VLDRHS32_rq_u,   MVE_VLDRHU16,
-      MVE_VLDRHU16_post,   MVE_VLDRHU16_pre,    MVE_VLDRHU16_rq,
-      MVE_VLDRHU16_rq_u,   MVE_VLDRHU32,        MVE_VLDRHU32_post,
-      MVE_VLDRHU32_pre,    MVE_VLDRHU32_rq,     MVE_VLDRHU32_rq_u,
-      MVE_VLDRWU32,        MVE_VLDRWU32_post,   MVE_VLDRWU32_pre,
-      MVE_VLDRWU32_qi,     MVE_VLDRWU32_qi_pre, MVE_VLDRWU32_rq,
-      MVE_VLDRWU32_rq_u,   MVE_VLD20_16,        MVE_VLD20_16_wb,
-      MVE_VLD20_32,        MVE_VLD20_32_wb,     MVE_VLD20_8,
-      MVE_VLD20_8_wb,      MVE_VLD21_16,        MVE_VLD21_16_wb,
-      MVE_VLD21_32,        MVE_VLD21_32_wb,     MVE_VLD21_8,
-      MVE_VLD21_8_wb,      MVE_VLD40_16,        MVE_VLD40_16_wb,
-      MVE_VLD40_32,        MVE_VLD40_32_wb,     MVE_VLD40_8,
-      MVE_VLD40_8_wb,      MVE_VLD41_16,        MVE_VLD41_16_wb,
-      MVE_VLD41_32,        MVE_VLD41_32_wb,     MVE_VLD41_8,
-      MVE_VLD41_8_wb,      MVE_VLD42_16,        MVE_VLD42_16_wb,
-      MVE_VLD42_32,        MVE_VLD42_32_wb,     MVE_VLD42_8,
-      MVE_VLD42_8_wb,      MVE_VLD43_16,        MVE_VLD43_16_wb,
-      MVE_VLD43_32,        MVE_VLD43_32_wb,     MVE_VLD43_8,
-      MVE_VLD43_8_wb,      MVE_VSTRB16,         MVE_VSTRB16_post,
-      MVE_VSTRB16_pre,     MVE_VSTRB16_rq,      MVE_VSTRB32,
-      MVE_VSTRB32_post,    MVE_VSTRB32_pre,     MVE_VSTRB32_rq,
-      MVE_VSTRB8_rq,       MVE_VSTRBU8,         MVE_VSTRBU8_post,
-      MVE_VSTRBU8_pre,     MVE_VSTRD64_qi,      MVE_VSTRD64_qi_pre,
-      MVE_VSTRD64_rq,      MVE_VSTRD64_rq_u,    MVE_VSTRH16_rq,
-      MVE_VSTRH16_rq_u,    MVE_VSTRH32,         MVE_VSTRH32_post,
-      MVE_VSTRH32_pre,     MVE_VSTRH32_rq,      MVE_VSTRH32_rq_u,
-      MVE_VSTRHU16,        MVE_VSTRHU16_post,   MVE_VSTRHU16_pre,
-      MVE_VSTRW32_qi,      MVE_VSTRW32_qi_pre,  MVE_VSTRW32_rq,
-      MVE_VSTRW32_rq_u,    MVE_VSTRWU32,        MVE_VSTRWU32_post,
-      MVE_VSTRWU32_pre,    MVE_VST20_16,        MVE_VST20_16_wb,
-      MVE_VST20_32,        MVE_VST20_32_wb,     MVE_VST20_8,
-      MVE_VST20_8_wb,      MVE_VST21_16,        MVE_VST21_16_wb,
-      MVE_VST21_32,        MVE_VST21_32_wb,     MVE_VST21_8,
-      MVE_VST21_8_wb,      MVE_VST40_16,        MVE_VST40_16_wb,
-      MVE_VST40_32,        MVE_VST40_32_wb,     MVE_VST40_8,
-      MVE_VST40_8_wb,      MVE_VST41_16,        MVE_VST41_16_wb,
-      MVE_VST41_32,        MVE_VST41_32_wb,     MVE_VST41_8,
-      MVE_VST41_8_wb,      MVE_VST42_16,        MVE_VST42_16_wb,
-      MVE_VST42_32,        MVE_VST42_32_wb,     MVE_VST42_8,
-      MVE_VST42_8_wb,      MVE_VST43_16,        MVE_VST43_16_wb,
-      MVE_VST43_32,        MVE_VST43_32_wb,     MVE_VST43_8,
-      MVE_VST43_8_wb,
+  std::set<unsigned> UnpredictableOpcodes = {
+      MVE_VCTP8,     MVE_VCTP16,    MVE_VCTP32,    MVE_VCTP64,    MVE_VPST,
+      MVE_VPTv16i8,  MVE_VPTv8i16,  MVE_VPTv4i32,  MVE_VPTv16i8r, MVE_VPTv8i16r,
+      MVE_VPTv4i32r, MVE_VPTv16s8,  MVE_VPTv8s16,  MVE_VPTv4s32,  MVE_VPTv16s8r,
+      MVE_VPTv8s16r, MVE_VPTv4s32r, MVE_VPTv16u8,  MVE_VPTv8u16,  MVE_VPTv4u32,
+      MVE_VPTv16u8r, MVE_VPTv8u16r, MVE_VPTv4u32r, MVE_VPTv8f16,  MVE_VPTv4f32,
+      MVE_VPTv8f16r, MVE_VPTv4f32r, MVE_VADC,      MVE_VADCI,     MVE_VSBC,
+      MVE_VSBCI,     MVE_VSHLC,
   };
 
   LLVMInitializeARMTargetInfo();
@@ -905,8 +968,13 @@ TEST(MachineInstr, HasSideEffects) {
   const ARMBaseInstrInfo *TII = ST.getInstrInfo();
   auto MII = TM->getMCInstrInfo();
 
-  for (unsigned Op : Opcodes) {
+  for (unsigned Op = 0; Op < ARM::INSTRUCTION_LIST_END; ++Op) {
     const MCInstrDesc &Desc = TII->get(Op);
+    if ((Desc.TSFlags & ARMII::DomainMask) != ARMII::DomainMVE)
+      continue;
+    if (UnpredictableOpcodes.count(Op))
+      continue;
+
     ASSERT_FALSE(Desc.hasUnmodeledSideEffects())
         << MII->getName(Op) << " has unexpected side effects";
   }
diff --git a/llvm/unittests/TextAPI/TextStubV3Tests.cpp b/llvm/unittests/TextAPI/TextStubV3Tests.cpp
index 6e7c087296e62..d59104e1367ab 100644
--- a/llvm/unittests/TextAPI/TextStubV3Tests.cpp
+++ b/llvm/unittests/TextAPI/TextStubV3Tests.cpp
@@ -117,6 +117,127 @@ TEST(TBDv3, ReadFile) {
       std::equal(Exports.begin(), Exports.end(), std::begin(TBDv3Symbols)));
 }
 
+TEST(TBDv3, ReadMultipleDocuments) {
+  static const char TBDv3Inlines[] =
+      "--- !tapi-tbd-v3\n"
+      "archs: [ armv7, arm64 ]\n"
+      "uuids: [ 'armv7: 00000000-0000-0000-0000-000000000000',\n"
+      "         'arm64: 11111111-1111-1111-1111-111111111111']\n"
+      "platform: ios\n"
+      "install-name: Test.dylib\n"
+      "current-version: 2.3.4\n"
+      "compatibility-version: 1.0\n"
+      "swift-abi-version: 1.1\n"
+      "parent-umbrella: Umbrella.dylib\n"
+      "exports:\n"
+      "  - archs: [ armv7, arm64 ]\n"
+      "    allowable-clients: [ clientA ]\n"
+      "    re-exports: [ /usr/lib/libfoo.dylib,\n"
+      "                  TestInline.dylib ]\n"
+      "    symbols: [ _sym1, _sym2, _sym3, _sym4, $ld$hide$os9.0$_sym1 ]\n"
+      "    objc-classes: [ class1, class2 ]\n"
+      "    objc-eh-types: [ class1 ]\n"
+      "    objc-ivars: [ class1._ivar1, class1._ivar2 ]\n"
+      "    weak-def-symbols: [ _weak1, _weak2 ]\n"
+      "    thread-local-symbols: [ _tlv1, _tlv3 ]\n"
+      "  - archs: [ armv7 ]\n"
+      "    symbols: [ _sym5 ]\n"
+      "    objc-classes: [ class3 ]\n"
+      "    objc-ivars: [ class1._ivar3 ]\n"
+      "    weak-def-symbols: [ _weak3 ]\n"
+      "    thread-local-symbols: [ _tlv3 ]\n"
+      "--- !tapi-tbd-v3\n"
+      "archs: [ armv7, arm64 ]\n"
+      "uuids: [ 'armv7: 00000000-0000-0000-0000-000000000000',\n"
+      "         'arm64: 11111111-1111-1111-1111-111111111111']\n"
+      "platform: ios\n"
+      "install-name: TestInline.dylib\n"
+      "swift-abi-version: 1.1\n"
+      "exports:\n"
+      "  - archs: [ armv7, arm64 ]\n"
+      "    symbols: [ _sym5, _sym6 ]\n"
+      "...\n";
+
+  auto Result = TextAPIReader::get(MemoryBufferRef(TBDv3Inlines, "Test.tbd"));
+  EXPECT_TRUE(!!Result);
+  auto File = std::move(Result.get());
+  EXPECT_EQ(File->documents().size(), 1U);
+  EXPECT_EQ(FileType::TBD_V3, File->getFileType());
+  auto Archs = AK_armv7 | AK_arm64;
+  auto Platform = PlatformKind::iOS;
+  TargetList Targets;
+  for (auto &&arch : Archs)
+    Targets.emplace_back(Target(arch, Platform));
+  EXPECT_EQ(Archs, File->getArchitectures());
+  UUIDs Uuids = {{Target(AK_armv7, PlatformKind::unknown),
+                  "00000000-0000-0000-0000-000000000000"},
+                 {Target(AK_arm64, PlatformKind::unknown),
+                  "11111111-1111-1111-1111-111111111111"}};
+  EXPECT_EQ(Uuids, File->uuids());
+  EXPECT_EQ(File->getPlatforms().size(), 1U);
+  EXPECT_EQ(Platform, *File->getPlatforms().begin());
+  EXPECT_EQ(std::string("Test.dylib"), File->getInstallName());
+  EXPECT_EQ(PackedVersion(2, 3, 4), File->getCurrentVersion());
+  EXPECT_EQ(PackedVersion(1, 0, 0), File->getCompatibilityVersion());
+  EXPECT_EQ(2U, File->getSwiftABIVersion());
+  EXPECT_EQ(ObjCConstraintType::Retain_Release, File->getObjCConstraint());
+  EXPECT_TRUE(File->isTwoLevelNamespace());
+  EXPECT_TRUE(File->isApplicationExtensionSafe());
+  EXPECT_FALSE(File->isInstallAPI());
+  InterfaceFileRef Client("clientA", Targets);
+  const std::vector<InterfaceFileRef> Reexports = {
+      InterfaceFileRef("/usr/lib/libfoo.dylib", Targets),
+      InterfaceFileRef("TestInline.dylib", Targets)};
+  EXPECT_EQ(1U, File->allowableClients().size());
+  EXPECT_EQ(Client, File->allowableClients().front());
+  EXPECT_EQ(2U, File->reexportedLibraries().size());
+  EXPECT_EQ(Reexports, File->reexportedLibraries());
+
+  ExportedSymbolSeq Exports;
+  for (const auto *Sym : File->symbols()) {
+    EXPECT_FALSE(Sym->isWeakReferenced());
+    EXPECT_FALSE(Sym->isUndefined());
+    Exports.emplace_back(ExportedSymbol{Sym->getKind(), Sym->getName().str(),
+                                        Sym->isWeakDefined(),
+                                        Sym->isThreadLocalValue()});
+  }
+  llvm::sort(Exports.begin(), Exports.end());
+
+  EXPECT_EQ(sizeof(TBDv3Symbols) / sizeof(ExportedSymbol), Exports.size());
+  EXPECT_TRUE(
+      std::equal(Exports.begin(), Exports.end(), std::begin(TBDv3Symbols)));
+
+  // Check Second Document
+  Exports.clear();
+  auto Document = File->documents().front();
+  EXPECT_EQ(FileType::TBD_V3, Document->getFileType());
+  EXPECT_EQ(Archs, Document->getArchitectures());
+  EXPECT_EQ(Uuids, Document->uuids());
+  EXPECT_EQ(Platform, *Document->getPlatforms().begin());
+  EXPECT_EQ(std::string("TestInline.dylib"), Document->getInstallName());
+  EXPECT_EQ(PackedVersion(1, 0, 0), Document->getCurrentVersion());
+  EXPECT_EQ(PackedVersion(1, 0, 0), Document->getCompatibilityVersion());
+  EXPECT_EQ(2U, Document->getSwiftABIVersion());
+
+  for (const auto *Sym : Document->symbols()) {
+    EXPECT_FALSE(Sym->isWeakReferenced());
+    EXPECT_FALSE(Sym->isUndefined());
+    Exports.emplace_back(ExportedSymbol{Sym->getKind(), Sym->getName().str(),
+                                        Sym->isWeakDefined(),
+                                        Sym->isThreadLocalValue()});
+  }
+  llvm::sort(Exports.begin(), Exports.end());
+
+  ExportedSymbolSeq DocumentSymbols{
+      {SymbolKind::GlobalSymbol, "_sym5", false, false},
+      {SymbolKind::GlobalSymbol, "_sym6", false, false},
+  };
+
+  EXPECT_EQ(DocumentSymbols.size(), Exports.size());
+  EXPECT_TRUE(
+      std::equal(Exports.begin(), Exports.end(), DocumentSymbols.begin()));
+}
+
 TEST(TBDv3, WriteFile) {
   static const char TBDv3File3[] =
       "--- !tapi-tbd-v3\n"
@@ -171,6 +292,87 @@ TEST(TBDv3, WriteFile) {
   EXPECT_STREQ(TBDv3File3, Buffer.c_str());
 }
 
+TEST(TBDv3, WriteMultipleDocuments) {
+  static const char TBDv3Inlines[] =
+      "--- !tapi-tbd-v3\n"
+      "archs:           [ i386, x86_64 ]\n"
+      "platform:        zippered\n"
+      "install-name:    '/usr/lib/libfoo.dylib'\n"
+      "current-version: 1.2.3\n"
+      "compatibility-version: 0\n"
+      "swift-abi-version: 5\n"
+      "exports:\n"
+      "  - archs:           [ x86_64 ]\n"
+      "    allowable-clients: [ clientA ]\n"
+      "    re-exports:      [ '/usr/lib/libbar.dylib' ]\n"
+      "  - archs:           [ i386, x86_64 ]\n"
+      "    symbols:         [ _sym1 ]\n"
+      "    objc-classes:    [ Class1 ]\n"
+      "    objc-eh-types:   [ Class1 ]\n"
+      "    objc-ivars:      [ Class1._ivar1 ]\n"
+      "    weak-def-symbols: [ _sym2 ]\n"
+      "    thread-local-symbols: [ _symA ]\n"
+      "--- !tapi-tbd-v3\n"
+      "archs:           [ i386 ]\n"
+      "platform:        macosx\n"
+      "install-name:    '/usr/lib/libbar.dylib'\n"
+      "current-version: 0\n"
+      "compatibility-version: 0\n"
+      "swift-abi-version: 5\n"
+      "objc-constraint: none\n"
+      "exports:\n"
+      "  - archs:           [ i386 ]\n"
+      "    symbols:         [ _sym3, _sym4 ]\n"
+      "...\n";
+
+  InterfaceFile File;
+  TargetList Targets;
+  for (auto &&arch : AK_i386 | AK_x86_64) {
+    Targets.emplace_back(Target(arch, PlatformKind::macOS));
+    Targets.emplace_back(Target(arch, PlatformKind::macCatalyst));
+  }
+  File.addTargets(Targets);
+  File.setPath("libfoo.dylib");
+  File.setInstallName("/usr/lib/libfoo.dylib");
+  File.setFileType(FileType::TBD_V3);
+  File.setCurrentVersion(PackedVersion(1, 2, 3));
+  File.setTwoLevelNamespace();
+  File.setApplicationExtensionSafe();
+  File.setSwiftABIVersion(5);
+  File.setObjCConstraint(ObjCConstraintType::Retain_Release);
+  File.addAllowableClient("clientA", Targets[2]);
+  File.addReexportedLibrary("/usr/lib/libbar.dylib", Targets[2]);
+  File.addSymbol(SymbolKind::GlobalSymbol, "_sym1", Targets);
+  File.addSymbol(SymbolKind::GlobalSymbol, "_sym2", Targets,
+                 SymbolFlags::WeakDefined);
+  File.addSymbol(SymbolKind::GlobalSymbol, "_symA", Targets,
+                 SymbolFlags::ThreadLocalValue);
+  File.addSymbol(SymbolKind::ObjectiveCClass, "Class1", Targets);
+  File.addSymbol(SymbolKind::ObjectiveCClassEHType, "Class1", Targets);
+  File.addSymbol(SymbolKind::ObjectiveCInstanceVariable, "Class1._ivar1",
+                 Targets);
+
+  // Inline document
+  InterfaceFile Document;
+  Targets = {Target(AK_i386, PlatformKind::macOS)};
+  Document.addTargets(Targets);
+  Document.setPath("libbar.dylib");
+  Document.setInstallName("/usr/lib/libbar.dylib");
+  Document.setFileType(FileType::TBD_V3);
+  Document.setTwoLevelNamespace();
+  Document.setApplicationExtensionSafe();
+  Document.setSwiftABIVersion(5);
+  Document.addSymbol(SymbolKind::GlobalSymbol, "_sym3", Targets);
+  Document.addSymbol(SymbolKind::GlobalSymbol, "_sym4", Targets);
+  File.addDocument(std::make_shared<InterfaceFile>(std::move(Document)));
+  
+  SmallString<4096> Buffer;
+  raw_svector_ostream OS(Buffer);
+  auto Result = TextAPIWriter::writeToStream(OS, File);
+  EXPECT_FALSE(Result);
+  EXPECT_STREQ(TBDv3Inlines, Buffer.c_str());
+}
+
 TEST(TBDv3, Platform_macOS) {
   static const char TBDv3PlatformMacOS[] = "--- !tapi-tbd-v3\n"
                                            "archs: [ x86_64 ]\n"
diff --git a/llvm/unittests/TextAPI/TextStubV4Tests.cpp b/llvm/unittests/TextAPI/TextStubV4Tests.cpp
index cecc6d19f50ed..1e8bea4c9dc72 100644
--- a/llvm/unittests/TextAPI/TextStubV4Tests.cpp
+++ b/llvm/unittests/TextAPI/TextStubV4Tests.cpp
@@ -17,24 +17,11 @@
 using namespace llvm;
 using namespace llvm::MachO;
 
-static ExportedSymbol TBDv4ExportedSymbols[] = {
-    {SymbolKind::GlobalSymbol, "_symA", false, false},
-    {SymbolKind::GlobalSymbol, "_symAB", false, false},
-    {SymbolKind::GlobalSymbol, "_symB", false, false},
-};
-
-static ExportedSymbol TBDv4ReexportedSymbols[] = {
-    {SymbolKind::GlobalSymbol, "_symC", false, false},
-};
-
-static ExportedSymbol TBDv4UndefinedSymbols[] = {
-    {SymbolKind::GlobalSymbol, "_symD", false, false},
-};
 
 namespace TBDv4 {
 
 TEST(TBDv4, ReadFile) {
-  static const char tbd_v4_file[] =
+  static const char TBDv4File[] =
       "--- !tapi-tbd\n"
       "tbd-version: 4\n"
       "targets:  [ i386-macos, x86_64-macos, x86_64-ios ]\n"
@@ -89,7 +76,7 @@ TEST(TBDv4, ReadFile) {
       "    thread-local-symbols: []\n"
       "...\n";
 
-  auto Result = TextAPIReader::get(MemoryBufferRef(tbd_v4_file, "Test.tbd"));
+  auto Result = TextAPIReader::get(MemoryBufferRef(TBDv4File, "Test.tbd"));
   EXPECT_TRUE(!!Result);
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V4, File->getFileType());
@@ -141,22 +128,213 @@ TEST(TBDv4, ReadFile) {
   llvm::sort(Reexports.begin(), Reexports.end());
   llvm::sort(Undefineds.begin(), Undefineds.end());
 
-  EXPECT_EQ(sizeof(TBDv4ExportedSymbols) / sizeof(ExportedSymbol),
+  static ExportedSymbol ExpectedExportedSymbols[] = {
+      {SymbolKind::GlobalSymbol, "_symA", false, false},
+      {SymbolKind::GlobalSymbol, "_symAB", false, false},
+      {SymbolKind::GlobalSymbol, "_symB", false, false},
+  };
+
+  static ExportedSymbol ExpectedReexportedSymbols[] = {
+      {SymbolKind::GlobalSymbol, "_symC", false, false},
+  };
+
+  static ExportedSymbol ExpectedUndefinedSymbols[] = {
+      {SymbolKind::GlobalSymbol, "_symD", false, false},
+  };
+
+  EXPECT_EQ(sizeof(ExpectedExportedSymbols) / sizeof(ExportedSymbol),
             Exports.size());
-  EXPECT_EQ(sizeof(TBDv4ReexportedSymbols) / sizeof(ExportedSymbol),
+  EXPECT_EQ(sizeof(ExpectedReexportedSymbols) / sizeof(ExportedSymbol),
             Reexports.size());
-  EXPECT_EQ(sizeof(TBDv4UndefinedSymbols) / sizeof(ExportedSymbol),
+  EXPECT_EQ(sizeof(ExpectedUndefinedSymbols) / sizeof(ExportedSymbol),
             Undefineds.size());
   EXPECT_TRUE(std::equal(Exports.begin(), Exports.end(),
-                         std::begin(TBDv4ExportedSymbols)));
+                         std::begin(ExpectedExportedSymbols)));
   EXPECT_TRUE(std::equal(Reexports.begin(), Reexports.end(),
-                         std::begin(TBDv4ReexportedSymbols)));
+                         std::begin(ExpectedReexportedSymbols)));
   EXPECT_TRUE(std::equal(Undefineds.begin(), Undefineds.end(),
-                         std::begin(TBDv4UndefinedSymbols)));
+                         std::begin(ExpectedUndefinedSymbols)));
+}
+
+TEST(TBDv4, ReadMultipleDocuments) {
+  static const char TBDv4Inlines[] =
+      "--- !tapi-tbd\n"
+      "tbd-version: 4\n"
+      "targets: [ i386-macos, i386-maccatalyst, x86_64-macos, "
+      "x86_64-maccatalyst ]\n"
+      "uuids:\n"
+      "  - target: i386-macos\n"
+      "    value: 00000000-0000-0000-0000-000000000000\n"
+      "  - target: i386-maccatalyst\n"
+      "    value: 00000000-0000-0000-0000-000000000002\n"
+      "  - target: x86_64-macos\n"
+      "    value: 11111111-1111-1111-1111-111111111111\n"
+      "  - target: x86_64-maccatalyst\n"
+      "    value: 11111111-1111-1111-1111-111111111112\n"
+      "install-name: /System/Library/Frameworks/Umbrella.framework/Umbrella\n"
+      "parent-umbrella:\n"
+      "  - targets: [ i386-macos, x86_64-macos ]\n"
+      "    umbrella: System\n"
+      "reexported-libraries:\n"
+      "  - targets: [ i386-macos, x86_64-macos ]\n"
+      "    libraries: [ /System/Library/Frameworks/A.framework/A ]\n"
+      "--- !tapi-tbd\n"
+      "tbd-version: 4\n"
+      "targets:  [ i386-macos, x86_64-macos ]\n"
+      "uuids:\n"
+      "  - target: i386-macos\n"
+      "    value: 20000000-0000-0000-0000-000000000000\n"
+      "  - target: x86_64-macos\n"
+      "    value: 21111111-1111-1111-1111-111111111111\n"
+      "flags: [ flat_namespace ]\n"
+      "install-name: /System/Library/Frameworks/A.framework/A\n"
+      "current-version: 1.2.3\n"
+      "compatibility-version: 1.2\n"
+      "swift-abi-version: 5\n"
+      "exports:\n"
+      "  - targets: [ i386-macos ]\n"
+      "    symbols: [ _symA ]\n"
+      "    objc-classes: []\n"
+      "    objc-eh-types: []\n"
+      "    objc-ivars: []\n"
+      "    weak-symbols: []\n"
+      "    thread-local-symbols: []\n"
+      "  - targets: [ x86_64-macos ]\n"
+      "    symbols: [_symAB]\n"
+      "reexports:\n"
+      "  - targets: [ i386-macos ]\n"
+      "    symbols: [_symC]\n"
+      "    objc-classes: []\n"
+      "    objc-eh-types: []\n"
+      "    objc-ivars: []\n"
+      "    weak-symbols: []\n"
+      "    thread-local-symbols: []\n"
+      "undefineds:\n"
+      "  - targets: [ i386-macos ]\n"
+      "    symbols: [ _symD ]\n"
+      "    objc-classes: []\n"
+      "    objc-eh-types: []\n"
+      "    objc-ivars: []\n"
+      "    weak-symbols: []\n"
+      "    thread-local-symbols: []\n"
+      "...\n";
+
+  PlatformSet Platforms;
+  Platforms.insert(PlatformKind::macOS);
+  Platforms.insert(PlatformKind::macCatalyst);
+  ArchitectureSet Archs = AK_i386 | AK_x86_64;
+  TargetList Targets;
+  for (auto &&Arch : Archs)
+    for (auto &&Platform : Platforms)
+      Targets.emplace_back(Target(Arch, Platform));
+  UUIDs Uuids = {
+      {Targets[0], "00000000-0000-0000-0000-000000000000"},
+      {Targets[1], "00000000-0000-0000-0000-000000000002"},
+      {Targets[2], "11111111-1111-1111-1111-111111111111"},
+      {Targets[3], "11111111-1111-1111-1111-111111111112"},
+  };
+
+  auto Result = TextAPIReader::get(MemoryBufferRef(TBDv4Inlines, "Test.tbd"));
+  EXPECT_TRUE(!!Result);
+  auto File = std::move(Result.get());
+  EXPECT_EQ(FileType::TBD_V4, File->getFileType());
+  EXPECT_EQ(Archs, File->getArchitectures());
+  EXPECT_EQ(Uuids, File->uuids());
+  EXPECT_EQ(Platforms, File->getPlatforms());
+  EXPECT_EQ(
+      std::string("/System/Library/Frameworks/Umbrella.framework/Umbrella"),
+      File->getInstallName());
+  EXPECT_TRUE(File->isTwoLevelNamespace());
+  EXPECT_TRUE(File->isApplicationExtensionSafe());
+  EXPECT_FALSE(File->isInstallAPI());
+  EXPECT_EQ(PackedVersion(1, 0, 0), File->getCurrentVersion());
+  EXPECT_EQ(PackedVersion(1, 0, 0), File->getCompatibilityVersion());
+  InterfaceFileRef reexport("/System/Library/Frameworks/A.framework/A",
+                            {Targets[0], Targets[2]});
+  EXPECT_EQ(1U, File->reexportedLibraries().size());
+  EXPECT_EQ(reexport, File->reexportedLibraries().front());
+  ExportedSymbolSeq Exports;
+  for (const auto *Sym : File->symbols()) {
+    EXPECT_FALSE(Sym->isWeakReferenced());
+    EXPECT_FALSE(Sym->isUndefined());
+    Exports.emplace_back(ExportedSymbol{Sym->getKind(), Sym->getName().str(),
+                                        Sym->isWeakDefined(),
+                                        Sym->isThreadLocalValue()});
+  }
+  EXPECT_EQ(0U, Exports.size());
+
+  // Check Inlined Document
+  Exports.clear();
+  Targets.clear();
+  Uuids.clear();
+  PlatformKind Platform = PlatformKind::macOS;
+  for (auto &&Arch : Archs)
+    Targets.emplace_back(Target(Arch, Platform));
+  Uuids = {
+      {Targets[0], "20000000-0000-0000-0000-000000000000"},
+      {Targets[1], "21111111-1111-1111-1111-111111111111"},
+  };
+
+  auto Document = File->documents().front();
+  EXPECT_EQ(FileType::TBD_V4, Document->getFileType());
+  EXPECT_EQ(Archs, Document->getArchitectures());
+  EXPECT_EQ(Uuids, Document->uuids());
+  EXPECT_EQ(1U, Document->getPlatforms().size());
+  EXPECT_EQ(Platform, *(Document->getPlatforms().begin()));
+  EXPECT_EQ(std::string("/System/Library/Frameworks/A.framework/A"),
+            Document->getInstallName());
+  EXPECT_EQ(PackedVersion(1, 2, 3), Document->getCurrentVersion());
+  EXPECT_EQ(PackedVersion(1, 2, 0), Document->getCompatibilityVersion());
+  EXPECT_EQ(5U, Document->getSwiftABIVersion());
+  EXPECT_FALSE(Document->isTwoLevelNamespace());
+  EXPECT_TRUE(Document->isApplicationExtensionSafe());
+  EXPECT_FALSE(Document->isInstallAPI());
+
+  ExportedSymbolSeq Reexports, Undefineds;
+  for (const auto *Sym : Document->symbols()) {
+    ExportedSymbol Temp =
+        ExportedSymbol{Sym->getKind(), std::string(Sym->getName()),
+                       Sym->isWeakDefined(), Sym->isThreadLocalValue()};
+    EXPECT_FALSE(Sym->isWeakReferenced());
+    if (Sym->isUndefined())
+      Undefineds.emplace_back(std::move(Temp));
+    else
+      Sym->isReexported() ? Reexports.emplace_back(std::move(Temp))
+                          : Exports.emplace_back(std::move(Temp));
+  }
+  llvm::sort(Exports.begin(), Exports.end());
+  llvm::sort(Reexports.begin(), Reexports.end());
+  llvm::sort(Undefineds.begin(), Undefineds.end());
+
+  static ExportedSymbol ExpectedExportedSymbols[] = {
+      {SymbolKind::GlobalSymbol, "_symA", false, false},
+      {SymbolKind::GlobalSymbol, "_symAB", false, false},
+  };
+
+  static ExportedSymbol ExpectedReexportedSymbols[] = {
+      {SymbolKind::GlobalSymbol, "_symC", false, false},
+  };
+
+  static ExportedSymbol ExpectedUndefinedSymbols[] = {
+      {SymbolKind::GlobalSymbol, "_symD", false, false},
+  };
+
+  EXPECT_EQ(sizeof(ExpectedExportedSymbols) / sizeof(ExportedSymbol),
+            Exports.size());
+  EXPECT_EQ(sizeof(ExpectedReexportedSymbols) / sizeof(ExportedSymbol),
+            Reexports.size());
+  EXPECT_EQ(sizeof(ExpectedUndefinedSymbols) / sizeof(ExportedSymbol),
+            Undefineds.size());
+  EXPECT_TRUE(std::equal(Exports.begin(), Exports.end(),
+                         std::begin(ExpectedExportedSymbols)));
+  EXPECT_TRUE(std::equal(Reexports.begin(), Reexports.end(),
+                         std::begin(ExpectedReexportedSymbols)));
+  EXPECT_TRUE(std::equal(Undefineds.begin(), Undefineds.end(),
+                         std::begin(ExpectedUndefinedSymbols)));
 }
 
 TEST(TBDv4, WriteFile) {
-  static const char tbd_v4_file[] =
+  static const char TBDv4File[] =
       "--- !tapi-tbd\n"
       "tbd-version:     4\n"
       "targets:         [ i386-macos, x86_64-ios-simulator ]\n"
@@ -215,11 +393,94 @@ TEST(TBDv4, WriteFile) {
   raw_svector_ostream OS(Buffer);
   auto Result = TextAPIWriter::writeToStream(OS, File);
   EXPECT_FALSE(Result);
-  EXPECT_STREQ(tbd_v4_file, Buffer.c_str());
+  EXPECT_STREQ(TBDv4File, Buffer.c_str());
+}
+
+TEST(TBDv4, WriteMultipleDocuments) {
+  static const char TBDv4Inlines[] =
+      "--- !tapi-tbd\n"
+      "tbd-version:     4\n"
+      "targets:         [ i386-maccatalyst, x86_64-maccatalyst ]\n"
+      "uuids:\n"
+      "  - target:          i386-maccatalyst\n"
+      "    value:           00000000-0000-0000-0000-000000000002\n"
+      "  - target:          x86_64-maccatalyst\n"
+      "    value:           11111111-1111-1111-1111-111111111112\n"
+      "install-name:    "
+      "'/System/Library/Frameworks/Umbrella.framework/Umbrella'\n"
+      "reexported-libraries:\n"
+      "  - targets:         [ i386-maccatalyst, x86_64-maccatalyst ]\n"
+      "    libraries:       [ '/System/Library/Frameworks/A.framework/A' ]\n"
+      "--- !tapi-tbd\n"
+      "tbd-version:     4\n"
+      "targets:         [ i386-maccatalyst, x86_64-maccatalyst ]\n"
+      "uuids:\n"
+      "  - target:          i386-maccatalyst\n"
+      "    value:           00000000-0000-0000-0000-000000000000\n"
+      "  - target:          x86_64-maccatalyst\n"
+      "    value:           11111111-1111-1111-1111-111111111111\n"
+      "install-name:    '/System/Library/Frameworks/A.framework/A'\n"
+      "exports:\n"
+      "  - targets:         [ i386-maccatalyst ]\n"
+      "    weak-symbols:    [ _symC ]\n"
+      "  - targets:         [ i386-maccatalyst, x86_64-maccatalyst ]\n"
+      "    symbols:         [ _symA ]\n"
+      "    objc-classes:    [ Class1 ]\n"
+      "  - targets:         [ x86_64-maccatalyst ]\n"
+      "    symbols:         [ _symAB ]\n"
+      "...\n";
+
+  InterfaceFile File;
+  PlatformKind Platform = PlatformKind::macCatalyst;
+  TargetList Targets = {
+      Target(AK_i386, Platform),
+      Target(AK_x86_64, Platform),
+  };
+  UUIDs Uuids = {{Targets[0], "00000000-0000-0000-0000-000000000002"},
+                 {Targets[1], "11111111-1111-1111-1111-111111111112"}};
+  File.setInstallName("/System/Library/Frameworks/Umbrella.framework/Umbrella");
+  File.setFileType(FileType::TBD_V4);
+  File.addTargets(Targets);
+  File.addUUID(Uuids[0].first, Uuids[0].second);
+  File.addUUID(Uuids[1].first, Uuids[1].second);
+  File.setCompatibilityVersion(PackedVersion(1, 0, 0));
+  File.setCurrentVersion(PackedVersion(1, 0, 0));
+  File.setTwoLevelNamespace();
+  File.setApplicationExtensionSafe(true);
+  File.addReexportedLibrary("/System/Library/Frameworks/A.framework/A",
+                            Targets[0]);
+  File.addReexportedLibrary("/System/Library/Frameworks/A.framework/A",
+                            Targets[1]);
+
+  // Write Second Document
+  Uuids = {{Targets[0], "00000000-0000-0000-0000-000000000000"},
+           {Targets[1], "11111111-1111-1111-1111-111111111111"}};
+  InterfaceFile Document;
+  Document.setInstallName("/System/Library/Frameworks/A.framework/A");
+  Document.setFileType(FileType::TBD_V4);
+  Document.addTargets(Targets);
+  Document.addUUID(Uuids[0].first, Uuids[0].second);
+  Document.addUUID(Uuids[1].first, Uuids[1].second);
+  Document.setCompatibilityVersion(PackedVersion(1, 0, 0));
+  Document.setCurrentVersion(PackedVersion(1, 0, 0));
+  Document.setTwoLevelNamespace();
+  Document.setApplicationExtensionSafe(true);
+  Document.addSymbol(SymbolKind::GlobalSymbol, "_symA", Targets);
+  Document.addSymbol(SymbolKind::GlobalSymbol, "_symAB", {Targets[1]});
+  Document.addSymbol(SymbolKind::GlobalSymbol, "_symC", {Targets[0]},
+                     SymbolFlags::WeakDefined);
+  Document.addSymbol(SymbolKind::ObjectiveCClass, "Class1", Targets);
+  File.addDocument(std::make_shared<InterfaceFile>(std::move(Document)));
+
+  SmallString<4096> Buffer;
+  raw_svector_ostream OS(Buffer);
+  auto Result = TextAPIWriter::writeToStream(OS, File);
+  EXPECT_FALSE(Result);
+  EXPECT_STREQ(TBDv4Inlines, Buffer.c_str());
 }
 
 TEST(TBDv4, MultipleTargets) {
-  static const char tbd_multiple_targets[] =
+  static const char TBDv4MultipleTargets[] =
       "--- !tapi-tbd\n"
       "tbd-version: 4\n"
       "targets: [ i386-maccatalyst, x86_64-tvos, arm64-ios ]\n"
@@ -227,7 +488,7 @@ TEST(TBDv4, MultipleTargets) {
       "...\n";
 
   auto Result =
-      TextAPIReader::get(MemoryBufferRef(tbd_multiple_targets, "Test.tbd"));
+      TextAPIReader::get(MemoryBufferRef(TBDv4MultipleTargets, "Test.tbd"));
   EXPECT_TRUE(!!Result);
   PlatformSet Platforms;
   Platforms.insert(PlatformKind::macCatalyst);
@@ -244,12 +505,12 @@ TEST(TBDv4, MultipleTargets) {
   raw_svector_ostream OS(Buffer);
   auto WriteResult = TextAPIWriter::writeToStream(OS, *File);
   EXPECT_TRUE(!WriteResult);
-  EXPECT_EQ(stripWhitespace(tbd_multiple_targets),
+  EXPECT_EQ(stripWhitespace(TBDv4MultipleTargets),
             stripWhitespace(Buffer.c_str()));
 }
 
 TEST(TBDv4, MultipleTargetsSameArch) {
-  static const char tbd_targets_same_arch[] =
+  static const char TBDv4TargetsSameArch[] =
       "--- !tapi-tbd\n"
       "tbd-version: 4\n"
       "targets: [ x86_64-tvos , x86_64-maccatalyst ]\n"
@@ -257,7 +518,7 @@ TEST(TBDv4, MultipleTargetsSameArch) {
       "...\n";
 
   auto Result =
-      TextAPIReader::get(MemoryBufferRef(tbd_targets_same_arch, "Test.tbd"));
+      TextAPIReader::get(MemoryBufferRef(TBDv4TargetsSameArch, "Test.tbd"));
   EXPECT_TRUE(!!Result);
   PlatformSet Platforms;
   Platforms.insert(PlatformKind::tvOS);
@@ -273,12 +534,12 @@ TEST(TBDv4, MultipleTargetsSameArch) {
   raw_svector_ostream OS(Buffer);
   auto WriteResult = TextAPIWriter::writeToStream(OS, *File);
   EXPECT_TRUE(!WriteResult);
-  EXPECT_EQ(stripWhitespace(tbd_targets_same_arch),
+  EXPECT_EQ(stripWhitespace(TBDv4TargetsSameArch),
             stripWhitespace(Buffer.c_str()));
 }
 
 TEST(TBDv4, MultipleTargetsSamePlatform) {
-  static const char tbd_multiple_targets_same_platform[] =
+  static const char TBDv4MultipleTargetsSamePlatform[] =
       "--- !tapi-tbd\n"
       "tbd-version: 4\n"
       "targets: [ armv7k-ios , arm64-ios]\n"
@@ -286,7 +547,7 @@ TEST(TBDv4, MultipleTargetsSamePlatform) {
       "...\n";
 
   auto Result = TextAPIReader::get(
-      MemoryBufferRef(tbd_multiple_targets_same_platform, "Test.tbd"));
+      MemoryBufferRef(TBDv4MultipleTargetsSamePlatform, "Test.tbd"));
   EXPECT_TRUE(!!Result);
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V4, File->getFileType());
@@ -298,12 +559,12 @@ TEST(TBDv4, MultipleTargetsSamePlatform) {
   raw_svector_ostream OS(Buffer);
   auto WriteResult = TextAPIWriter::writeToStream(OS, *File);
   EXPECT_TRUE(!WriteResult);
-  EXPECT_EQ(stripWhitespace(tbd_multiple_targets_same_platform),
+  EXPECT_EQ(stripWhitespace(TBDv4MultipleTargetsSamePlatform),
             stripWhitespace(Buffer.c_str()));
 }
 
 TEST(TBDv4, Target_maccatalyst) {
-  static const char tbd_target_maccatalyst[] =
+  static const char TBDv4TargetMacCatalyst[] =
       "--- !tapi-tbd\n"
       "tbd-version: 4\n"
       "targets: [  x86_64-maccatalyst ]\n"
@@ -311,7 +572,7 @@ TEST(TBDv4, Target_maccatalyst) {
       "...\n";
 
   auto Result =
-      TextAPIReader::get(MemoryBufferRef(tbd_target_maccatalyst, "Test.tbd"));
+      TextAPIReader::get(MemoryBufferRef(TBDv4TargetMacCatalyst, "Test.tbd"));
   EXPECT_TRUE(!!Result);
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V4, File->getFileType());
@@ -323,19 +584,19 @@ TEST(TBDv4, Target_maccatalyst) {
   raw_svector_ostream OS(Buffer);
   auto WriteResult = TextAPIWriter::writeToStream(OS, *File);
   EXPECT_TRUE(!WriteResult);
-  EXPECT_EQ(stripWhitespace(tbd_target_maccatalyst),
+  EXPECT_EQ(stripWhitespace(TBDv4TargetMacCatalyst),
             stripWhitespace(Buffer.c_str()));
 }
 
 TEST(TBDv4, Target_x86_ios) {
-  static const char tbd_target_x86_ios[] = "--- !tapi-tbd\n"
-                                           "tbd-version: 4\n"
-                                           "targets: [  x86_64-ios ]\n"
-                                           "install-name: Test.dylib\n"
-                                           "...\n";
+  static const char TBDv4Targetx86iOS[] = "--- !tapi-tbd\n"
+                                          "tbd-version: 4\n"
+                                          "targets: [  x86_64-ios ]\n"
+                                          "install-name: Test.dylib\n"
+                                          "...\n";
 
   auto Result =
-      TextAPIReader::get(MemoryBufferRef(tbd_target_x86_ios, "Test.tbd"));
+      TextAPIReader::get(MemoryBufferRef(TBDv4Targetx86iOS, "Test.tbd"));
   EXPECT_TRUE(!!Result);
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V4, File->getFileType());
@@ -347,19 +608,19 @@ TEST(TBDv4, Target_x86_ios) {
   raw_svector_ostream OS(Buffer);
   auto WriteResult = TextAPIWriter::writeToStream(OS, *File);
   EXPECT_TRUE(!WriteResult);
-  EXPECT_EQ(stripWhitespace(tbd_target_x86_ios),
+  EXPECT_EQ(stripWhitespace(TBDv4Targetx86iOS),
             stripWhitespace(Buffer.c_str()));
 }
 
 TEST(TBDv4, Target_arm_bridgeOS) {
-  static const char tbd_platform_bridgeos[] = "--- !tapi-tbd\n"
+  static const char TBDv4PlatformBridgeOS[] = "--- !tapi-tbd\n"
                                               "tbd-version: 4\n"
                                               "targets: [  armv7k-bridgeos ]\n"
                                               "install-name: Test.dylib\n"
                                               "...\n";
 
   auto Result =
-      TextAPIReader::get(MemoryBufferRef(tbd_platform_bridgeos, "Test.tbd"));
+      TextAPIReader::get(MemoryBufferRef(TBDv4PlatformBridgeOS, "Test.tbd"));
   EXPECT_TRUE(!!Result);
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V4, File->getFileType());
@@ -371,18 +632,19 @@ TEST(TBDv4, Target_arm_bridgeOS) {
   raw_svector_ostream OS(Buffer);
   auto WriteResult = TextAPIWriter::writeToStream(OS, *File);
   EXPECT_TRUE(!WriteResult);
-  EXPECT_EQ(stripWhitespace(tbd_platform_bridgeos),
+  EXPECT_EQ(stripWhitespace(TBDv4PlatformBridgeOS),
             stripWhitespace(Buffer.c_str()));
 }
 
 TEST(TBDv4, Target_arm_iOS) {
-  static const char tbdv4_arm64e[] = "--- !tapi-tbd\n"
-                                     "tbd-version: 4\n"
-                                     "targets: [  arm64e-ios ]\n"
-                                     "install-name: Test.dylib\n"
-                                     "...\n";
+  static const char TBDv4ArchArm64e[] = "--- !tapi-tbd\n"
+                                        "tbd-version: 4\n"
+                                        "targets: [  arm64e-ios ]\n"
+                                        "install-name: Test.dylib\n"
+                                        "...\n";
 
-  auto Result = TextAPIReader::get(MemoryBufferRef(tbdv4_arm64e, "Test.tbd"));
+  auto Result =
+      TextAPIReader::get(MemoryBufferRef(TBDv4ArchArm64e, "Test.tbd"));
   EXPECT_TRUE(!!Result);
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V4, File->getFileType());
@@ -394,17 +656,18 @@ TEST(TBDv4, Target_arm_iOS) {
   raw_svector_ostream OS(Buffer);
   auto WriteResult = TextAPIWriter::writeToStream(OS, *File);
   EXPECT_TRUE(!WriteResult);
-  EXPECT_EQ(stripWhitespace(tbdv4_arm64e), stripWhitespace(Buffer.c_str()));
+  EXPECT_EQ(stripWhitespace(TBDv4ArchArm64e), stripWhitespace(Buffer.c_str()));
 }
 
 TEST(TBDv4, Target_x86_macos) {
-  static const char tbd_x86_macos[] = "--- !tapi-tbd\n"
-                                      "tbd-version: 4\n"
-                                      "targets: [  x86_64-macos ]\n"
-                                      "install-name: Test.dylib\n"
-                                      "...\n";
+  static const char TBDv4Targetx86MacOS[] = "--- !tapi-tbd\n"
+                                            "tbd-version: 4\n"
+                                            "targets: [  x86_64-macos ]\n"
+                                            "install-name: Test.dylib\n"
+                                            "...\n";
 
-  auto Result = TextAPIReader::get(MemoryBufferRef(tbd_x86_macos, "Test.tbd"));
+  auto Result =
+      TextAPIReader::get(MemoryBufferRef(TBDv4Targetx86MacOS, "Test.tbd"));
   EXPECT_TRUE(!!Result);
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V4, File->getFileType());
@@ -416,18 +679,20 @@ TEST(TBDv4, Target_x86_macos) {
   raw_svector_ostream OS(Buffer);
   auto WriteResult = TextAPIWriter::writeToStream(OS, *File);
   EXPECT_TRUE(!WriteResult);
-  EXPECT_EQ(stripWhitespace(tbd_x86_macos), stripWhitespace(Buffer.c_str()));
+  EXPECT_EQ(stripWhitespace(TBDv4Targetx86MacOS),
+            stripWhitespace(Buffer.c_str()));
 }
 
 TEST(TBDv4, Target_x86_ios_simulator) {
-  static const char tbd_x86_ios_sim[] = "--- !tapi-tbd\n"
-                                        "tbd-version: 4\n"
-                                        "targets: [  x86_64-ios-simulator  ]\n"
-                                        "install-name: Test.dylib\n"
-                                        "...\n";
+  static const char TBDv4Targetx86iOSSim[] =
+      "--- !tapi-tbd\n"
+      "tbd-version: 4\n"
+      "targets: [  x86_64-ios-simulator  ]\n"
+      "install-name: Test.dylib\n"
+      "...\n";
 
   auto Result =
-      TextAPIReader::get(MemoryBufferRef(tbd_x86_ios_sim, "Test.tbd"));
+      TextAPIReader::get(MemoryBufferRef(TBDv4Targetx86iOSSim, "Test.tbd"));
   EXPECT_TRUE(!!Result);
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V4, File->getFileType());
@@ -439,19 +704,19 @@ TEST(TBDv4, Target_x86_ios_simulator) {
   raw_svector_ostream OS(Buffer);
   auto WriteResult = TextAPIWriter::writeToStream(OS, *File);
   EXPECT_TRUE(!WriteResult);
-  EXPECT_EQ(stripWhitespace(tbd_x86_ios_sim), stripWhitespace(Buffer.c_str()));
+  EXPECT_EQ(stripWhitespace(TBDv4Targetx86iOSSim),
+            stripWhitespace(Buffer.c_str()));
 }
 
 TEST(TBDv4, Target_x86_tvos_simulator) {
-  static const char tbd_x86_tvos_sim[] =
-      "--- !tapi-tbd\n"
-      "tbd-version: 4\n"
-      "targets: [  x86_64-tvos-simulator  ]\n"
-      "install-name: Test.dylib\n"
-      "...\n";
+  static const char TBDv4x86tvOSSim[] = "--- !tapi-tbd\n"
+                                        "tbd-version: 4\n"
+                                        "targets: [  x86_64-tvos-simulator  ]\n"
+                                        "install-name: Test.dylib\n"
+                                        "...\n";
 
   auto Result =
-      TextAPIReader::get(MemoryBufferRef(tbd_x86_tvos_sim, "Test.tbd"));
+      TextAPIReader::get(MemoryBufferRef(TBDv4x86tvOSSim, "Test.tbd"));
   EXPECT_TRUE(!!Result);
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V4, File->getFileType());
@@ -463,11 +728,11 @@ TEST(TBDv4, Target_x86_tvos_simulator) {
   raw_svector_ostream OS(Buffer);
   auto WriteResult = TextAPIWriter::writeToStream(OS, *File);
   EXPECT_TRUE(!WriteResult);
-  EXPECT_EQ(stripWhitespace(tbd_x86_tvos_sim), stripWhitespace(Buffer.c_str()));
+  EXPECT_EQ(stripWhitespace(TBDv4x86tvOSSim), stripWhitespace(Buffer.c_str()));
 }
 
 TEST(TBDv4, Target_i386_watchos_simulator) {
-  static const char tbd_i386_watchos_sim[] =
+  static const char TBDv4i386watchOSSim[] =
       "--- !tapi-tbd\n"
       "tbd-version: 4\n"
       "targets: [  i386-watchos-simulator  ]\n"
@@ -475,7 +740,7 @@ TEST(TBDv4, Target_i386_watchos_simulator) {
       "...\n";
 
   auto Result =
-      TextAPIReader::get(MemoryBufferRef(tbd_i386_watchos_sim, "Test.tbd"));
+      TextAPIReader::get(MemoryBufferRef(TBDv4i386watchOSSim, "Test.tbd"));
   EXPECT_TRUE(!!Result);
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V4, File->getFileType());
@@ -487,19 +752,20 @@ TEST(TBDv4, Target_i386_watchos_simulator) {
   raw_svector_ostream OS(Buffer);
   auto WriteResult = TextAPIWriter::writeToStream(OS, *File);
   EXPECT_TRUE(!WriteResult);
-  EXPECT_EQ(stripWhitespace(tbd_i386_watchos_sim),
+  EXPECT_EQ(stripWhitespace(TBDv4i386watchOSSim),
             stripWhitespace(Buffer.c_str()));
 }
 
 TEST(TBDv4, Swift_1) {
-  static const char tbd_swift_1[] = "--- !tapi-tbd\n"
-                                    "tbd-version: 4\n"
-                                    "targets: [  x86_64-macos ]\n"
-                                    "install-name: Test.dylib\n"
-                                    "swift-abi-version: 1\n"
-                                    "...\n";
+  static const char TBDv4SwiftVersion1[] = "--- !tapi-tbd\n"
+                                           "tbd-version: 4\n"
+                                           "targets: [  x86_64-macos ]\n"
+                                           "install-name: Test.dylib\n"
+                                           "swift-abi-version: 1\n"
+                                           "...\n";
 
-  auto Result = TextAPIReader::get(MemoryBufferRef(tbd_swift_1, "Test.tbd"));
+  auto Result =
+      TextAPIReader::get(MemoryBufferRef(TBDv4SwiftVersion1, "Test.tbd"));
   EXPECT_TRUE(!!Result);
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V4, File->getFileType());
@@ -509,14 +775,14 @@ TEST(TBDv4, Swift_1) {
 }
 
 TEST(TBDv4, Swift_2) {
-  static const char tbd_v4_swift_2[] = "--- !tapi-tbd\n"
-                                       "tbd-version: 4\n"
-                                       "targets: [  x86_64-macos ]\n"
-                                       "install-name: Test.dylib\n"
-                                       "swift-abi-version: 2\n"
-                                       "...\n";
-
-  auto Result = TextAPIReader::get(MemoryBufferRef(tbd_v4_swift_2, "Test.tbd"));
+  static const char TBDv4Swift2[] = "--- !tapi-tbd\n"
+                                    "tbd-version: 4\n"
+                                    "targets: [  x86_64-macos ]\n"
+                                    "install-name: Test.dylib\n"
+                                    "swift-abi-version: 2\n"
+                                    "...\n";
+
+  auto Result = TextAPIReader::get(MemoryBufferRef(TBDv4Swift2, "Test.tbd"));
   EXPECT_TRUE(!!Result);
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V4, File->getFileType());
@@ -526,14 +792,15 @@ TEST(TBDv4, Swift_2) {
 }
 
 TEST(TBDv4, Swift_5) {
-  static const char tbd_swift_5[] = "--- !tapi-tbd\n"
-                                    "tbd-version: 4\n"
-                                    "targets: [  x86_64-macos ]\n"
-                                    "install-name: Test.dylib\n"
-                                    "swift-abi-version: 5\n"
-                                    "...\n";
+  static const char TBDv4SwiftVersion5[] = "--- !tapi-tbd\n"
+                                           "tbd-version: 4\n"
+                                           "targets: [  x86_64-macos ]\n"
+                                           "install-name: Test.dylib\n"
+                                           "swift-abi-version: 5\n"
+                                           "...\n";
 
-  auto Result = TextAPIReader::get(MemoryBufferRef(tbd_swift_5, "Test.tbd"));
+  auto Result =
+      TextAPIReader::get(MemoryBufferRef(TBDv4SwiftVersion5, "Test.tbd"));
   EXPECT_TRUE(!!Result);
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V4, File->getFileType());
@@ -543,18 +810,20 @@ TEST(TBDv4, Swift_5) {
   raw_svector_ostream OS(Buffer);
   auto WriteResult = TextAPIWriter::writeToStream(OS, *File);
   EXPECT_TRUE(!WriteResult);
-  EXPECT_EQ(stripWhitespace(tbd_swift_5), stripWhitespace(Buffer.c_str()));
+  EXPECT_EQ(stripWhitespace(TBDv4SwiftVersion5),
+            stripWhitespace(Buffer.c_str()));
 }
 
 TEST(TBDv4, Swift_99) {
-  static const char tbd_swift_99[] = "--- !tapi-tbd\n"
-                                     "tbd-version: 4\n"
-                                     "targets: [  x86_64-macos ]\n"
-                                     "install-name: Test.dylib\n"
-                                     "swift-abi-version: 99\n"
-                                     "...\n";
-
-  auto Result = TextAPIReader::get(MemoryBufferRef(tbd_swift_99, "Test.tbd"));
+  static const char TBDv4SwiftVersion99[] = "--- !tapi-tbd\n"
+                                            "tbd-version: 4\n"
+                                            "targets: [  x86_64-macos ]\n"
+                                            "install-name: Test.dylib\n"
+                                            "swift-abi-version: 99\n"
+                                            "...\n";
+
+  auto Result =
+      TextAPIReader::get(MemoryBufferRef(TBDv4SwiftVersion99, "Test.tbd"));
   EXPECT_TRUE(!!Result);
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V4, File->getFileType());
@@ -564,19 +833,19 @@ TEST(TBDv4, Swift_99) {
   raw_svector_ostream OS(Buffer);
   auto WriteResult = TextAPIWriter::writeToStream(OS, *File);
   EXPECT_TRUE(!WriteResult);
-  EXPECT_EQ(stripWhitespace(tbd_swift_99), stripWhitespace(Buffer.c_str()));
+  EXPECT_EQ(stripWhitespace(TBDv4SwiftVersion99),
+            stripWhitespace(Buffer.c_str()));
 }
 
 TEST(TBDv4, InvalidArchitecture) {
-  static const char tbd_file_unknown_architecture[] =
-      "--- !tapi-tbd\n"
-      "tbd-version: 4\n"
-      "targets: [ foo-macos ]\n"
-      "install-name: Test.dylib\n"
-      "...\n";
+  static const char TBDv4UnknownArch[] = "--- !tapi-tbd\n"
+                                         "tbd-version: 4\n"
+                                         "targets: [ foo-macos ]\n"
+                                         "install-name: Test.dylib\n"
+                                         "...\n";
 
-  auto Result = TextAPIReader::get(
-      MemoryBufferRef(tbd_file_unknown_architecture, "Test.tbd"));
+  auto Result =
+      TextAPIReader::get(MemoryBufferRef(TBDv4UnknownArch, "Test.tbd"));
   EXPECT_FALSE(!!Result);
   auto errorMessage = toString(Result.takeError());
   EXPECT_EQ("malformed file\nTest.tbd:3:12: error: unknown "
@@ -586,14 +855,14 @@ TEST(TBDv4, InvalidArchitecture) {
 }
 
 TEST(TBDv4, InvalidPlatform) {
-  static const char tbd_file_invalid_platform[] = "--- !tapi-tbd\n"
-                                                  "tbd-version: 4\n"
-                                                  "targets: [ x86_64-maos ]\n"
-                                                  "install-name: Test.dylib\n"
-                                                  "...\n";
+  static const char TBDv4FInvalidPlatform[] = "--- !tapi-tbd\n"
+                                              "tbd-version: 4\n"
+                                              "targets: [ x86_64-maos ]\n"
+                                              "install-name: Test.dylib\n"
+                                              "...\n";
 
-  auto Result = TextAPIReader::get(
-      MemoryBufferRef(tbd_file_invalid_platform, "Test.tbd"));
+  auto Result =
+      TextAPIReader::get(MemoryBufferRef(TBDv4FInvalidPlatform, "Test.tbd"));
   EXPECT_FALSE(!!Result);
   auto errorMessage = toString(Result.takeError());
   EXPECT_EQ("malformed file\nTest.tbd:3:12: error: unknown platform\ntargets: "
@@ -603,12 +872,12 @@ TEST(TBDv4, InvalidPlatform) {
 }
 
 TEST(TBDv4, MalformedFile1) {
-  static const char malformed_file1[] = "--- !tapi-tbd\n"
-                                        "tbd-version: 4\n"
-                                        "...\n";
+  static const char TBDv4MalformedFile1[] = "--- !tapi-tbd\n"
+                                            "tbd-version: 4\n"
+                                            "...\n";
 
   auto Result =
-      TextAPIReader::get(MemoryBufferRef(malformed_file1, "Test.tbd"));
+      TextAPIReader::get(MemoryBufferRef(TBDv4MalformedFile1, "Test.tbd"));
   EXPECT_FALSE(!!Result);
   auto errorMessage = toString(Result.takeError());
   ASSERT_EQ("malformed file\nTest.tbd:2:1: error: missing required key "
@@ -617,14 +886,14 @@ TEST(TBDv4, MalformedFile1) {
 }
 
 TEST(TBDv4, MalformedFile2) {
-  static const char malformed_file2[] = "--- !tapi-tbd\n"
-                                        "tbd-version: 4\n"
-                                        "targets: [ x86_64-macos ]\n"
-                                        "install-name: Test.dylib\n"
-                                        "foobar: \"unsupported key\"\n";
+  static const char TBDv4MalformedFile2[] = "--- !tapi-tbd\n"
+                                            "tbd-version: 4\n"
+                                            "targets: [ x86_64-macos ]\n"
+                                            "install-name: Test.dylib\n"
+                                            "foobar: \"unsupported key\"\n";
 
   auto Result =
-      TextAPIReader::get(MemoryBufferRef(malformed_file2, "Test.tbd"));
+      TextAPIReader::get(MemoryBufferRef(TBDv4MalformedFile2, "Test.tbd"));
   EXPECT_FALSE(!!Result);
   auto errorMessage = toString(Result.takeError());
   ASSERT_EQ(
@@ -634,15 +903,15 @@ TEST(TBDv4, MalformedFile2) {
 }
 
 TEST(TBDv4, MalformedFile3) {
-  static const char tbd_v4_swift_1_1[] = "--- !tapi-tbd\n"
-                                         "tbd-version: 4\n"
-                                         "targets: [  x86_64-macos ]\n"
-                                         "install-name: Test.dylib\n"
-                                         "swift-abi-version: 1.1\n"
-                                         "...\n";
+  static const char TBDv4MalformedSwift[] = "--- !tapi-tbd\n"
+                                            "tbd-version: 4\n"
+                                            "targets: [  x86_64-macos ]\n"
+                                            "install-name: Test.dylib\n"
+                                            "swift-abi-version: 1.1\n"
+                                            "...\n";
 
   auto Result =
-      TextAPIReader::get(MemoryBufferRef(tbd_v4_swift_1_1, "Test.tbd"));
+      TextAPIReader::get(MemoryBufferRef(TBDv4MalformedSwift, "Test.tbd"));
   EXPECT_FALSE(!!Result);
   auto errorMessage = toString(Result.takeError());
   EXPECT_EQ("malformed file\nTest.tbd:5:20: error: invalid Swift ABI "
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
index 5269a9a17543f..dfa3d0ee73322 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
@@ -94,7 +94,7 @@ TEST_F(VPlanHCFGTest, testBuildHCFGInnerLoop) {
   Plan->addVPValue(&*F->arg_begin());
   std::string FullDump;
   raw_string_ostream(FullDump) << *Plan;
-  EXPECT_EQ(R"(digraph VPlan {
+  const char *ExpectedStr = R"(digraph VPlan {
 graph [labelloc=t, fontsize=30; label="Vectorization Plan"]
 node [shape=rect, fontname=Courier, fontsize=30]
 edge [fontname=Courier, fontsize=30]
@@ -125,8 +125,8 @@ compound=true
     ]
   }
 }
-)",
-            FullDump);
+)";
+  EXPECT_EQ(ExpectedStr, FullDump);
 
   LoopVectorizationLegality::InductionList Inductions;
   SmallPtrSet<Instruction *, 1> DeadInstructions;
@@ -169,28 +169,18 @@ TEST_F(VPlanHCFGTest, testVPInstructionToVPRecipesInner) {
   EXPECT_EQ(1u, Entry->getNumSuccessors());
 
   VPBasicBlock *VecBB = Entry->getSingleSuccessor()->getEntryBasicBlock();
-  EXPECT_EQ(6u, VecBB->size());
+  EXPECT_EQ(7u, VecBB->size());
   EXPECT_EQ(2u, VecBB->getNumPredecessors());
   EXPECT_EQ(2u, VecBB->getNumSuccessors());
 
   auto Iter = VecBB->begin();
-  auto *Phi = dyn_cast<VPWidenPHIRecipe>(&*Iter++);
-  EXPECT_NE(nullptr, Phi);
-
-  auto *Idx = dyn_cast<VPWidenGEPRecipe>(&*Iter++);
-  EXPECT_NE(nullptr, Idx);
-
-  auto *Load = dyn_cast<VPWidenMemoryInstructionRecipe>(&*Iter++);
-  EXPECT_NE(nullptr, Load);
-
-  auto *Add = dyn_cast<VPWidenRecipe>(&*Iter++);
-  EXPECT_NE(nullptr, Add);
-
-  auto *Store = dyn_cast<VPWidenMemoryInstructionRecipe>(&*Iter++);
-  EXPECT_NE(nullptr, Store);
-
-  auto *LastWiden = dyn_cast<VPWidenRecipe>(&*Iter++);
-  EXPECT_NE(nullptr, LastWiden);
+  EXPECT_NE(nullptr, dyn_cast<VPWidenPHIRecipe>(&*Iter++));
+  EXPECT_NE(nullptr, dyn_cast<VPWidenGEPRecipe>(&*Iter++));
+  EXPECT_NE(nullptr, dyn_cast<VPWidenMemoryInstructionRecipe>(&*Iter++));
+  EXPECT_NE(nullptr, dyn_cast<VPWidenRecipe>(&*Iter++));
+  EXPECT_NE(nullptr, dyn_cast<VPWidenMemoryInstructionRecipe>(&*Iter++));
+  EXPECT_NE(nullptr, dyn_cast<VPWidenRecipe>(&*Iter++));
+  EXPECT_NE(nullptr, dyn_cast<VPWidenRecipe>(&*Iter++));
   EXPECT_EQ(VecBB->end(), Iter);
 }
 
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index ce0e0017fa85e..73e01fade9a97 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -209,7 +209,7 @@ TEST(VPBasicBlockTest, print) {
   std::string FullDump;
   raw_string_ostream(FullDump) << Plan;
 
-  EXPECT_EQ(R"(digraph VPlan {
+  const char *ExpectedStr = R"(digraph VPlan {
 graph [labelloc=t, fontsize=30; label="Vectorization Plan"]
 node [shape=rect, fontname=Courier, fontsize=30]
 edge [fontname=Courier, fontsize=30]
@@ -227,8 +227,8 @@ compound=true
       "EMIT ret vp<%2>\l"
   ]
 }
-)",
-            FullDump);
+)";
+  EXPECT_EQ(ExpectedStr, FullDump);
 
   {
     std::string I3Dump;
diff --git a/llvm/utils/TableGen/AsmMatcherEmitter.cpp b/llvm/utils/TableGen/AsmMatcherEmitter.cpp
index 01133c13ae63f..3d63059dcb8bf 100644
--- a/llvm/utils/TableGen/AsmMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/AsmMatcherEmitter.cpp
@@ -3863,7 +3863,7 @@ void AsmMatcherEmitter::run(raw_ostream &OS) {
     OS << "    std::string Info;\n";
     OS << "    if (!getParser().getTargetParser().\n";
     OS << "        getTargetOptions().MCNoDeprecatedWarn &&\n";
-    OS << "        MII.get(Inst.getOpcode()).getDeprecatedInfo(Inst, getSTI(), Info)) {\n";
+    OS << "        MII.getDeprecatedInfo(Inst, getSTI(), Info)) {\n";
     OS << "      SMLoc Loc = ((" << Target.getName()
        << "Operand&)*Operands[0]).getStartLoc();\n";
     OS << "      getParser().Warning(Loc, Info, None);\n";
diff --git a/llvm/utils/TableGen/AsmWriterEmitter.cpp b/llvm/utils/TableGen/AsmWriterEmitter.cpp
index b8913c36e0022..72c7baa139a9d 100644
--- a/llvm/utils/TableGen/AsmWriterEmitter.cpp
+++ b/llvm/utils/TableGen/AsmWriterEmitter.cpp
@@ -783,7 +783,7 @@ void AsmWriterEmitter::EmitPrintAliasInstruction(raw_ostream &O) {
   // before it can be matched to the mnemonic.
   std::map<std::string, std::vector<IAPrinter>> IAPrinterMap;
 
-  std::vector<std::string> PrintMethods;
+  std::vector<std::pair<std::string, bool>> PrintMethods;
 
   // A list of MCOperandPredicates for all operands in use, and the reverse map
   std::vector<const Record*> MCOpPredicates;
@@ -855,11 +855,16 @@ void AsmWriterEmitter::EmitPrintAliasInstruction(raw_ostream &O) {
           if (Rec->isSubClassOf("RegisterOperand") ||
               Rec->isSubClassOf("Operand")) {
             StringRef PrintMethod = Rec->getValueAsString("PrintMethod");
+            bool IsPCRel =
+                Rec->getValueAsString("OperandType") == "OPERAND_PCREL";
             if (PrintMethod != "" && PrintMethod != "printOperand") {
-              PrintMethodIdx =
-                  llvm::find(PrintMethods, PrintMethod) - PrintMethods.begin();
+              PrintMethodIdx = llvm::find_if(PrintMethods,
+                                             [&](auto &X) {
+                                               return X.first == PrintMethod;
+                                             }) -
+                               PrintMethods.begin();
               if (static_cast<unsigned>(PrintMethodIdx) == PrintMethods.size())
-                PrintMethods.push_back(std::string(PrintMethod));
+                PrintMethods.emplace_back(std::string(PrintMethod), IsPCRel);
             }
           }
 
@@ -985,7 +990,8 @@ void AsmWriterEmitter::EmitPrintAliasInstruction(raw_ostream &O) {
 
   HeaderO << "bool " << Target.getName() << ClassName
           << "::printAliasInstr(const MCInst"
-          << " *MI, " << (PassSubtarget ? "const MCSubtargetInfo &STI, " : "")
+          << " *MI, uint64_t Address, "
+          << (PassSubtarget ? "const MCSubtargetInfo &STI, " : "")
           << "raw_ostream &OS) {\n";
 
   std::string PatternsForOpcode;
@@ -1148,7 +1154,7 @@ void AsmWriterEmitter::EmitPrintAliasInstruction(raw_ostream &O) {
   O << "          ++I;\n";
   O << "          int OpIdx = AsmString[I++] - 1;\n";
   O << "          int PrintMethodIdx = AsmString[I++] - 1;\n";
-  O << "          printCustomAliasOperand(MI, OpIdx, PrintMethodIdx, ";
+  O << "          printCustomAliasOperand(MI, Address, OpIdx, PrintMethodIdx, ";
   O << (PassSubtarget ? "STI, " : "");
   O << "OS);\n";
   O << "        } else\n";
@@ -1170,7 +1176,7 @@ void AsmWriterEmitter::EmitPrintAliasInstruction(raw_ostream &O) {
 
   O << "void " << Target.getName() << ClassName << "::"
     << "printCustomAliasOperand(\n"
-    << "         const MCInst *MI, unsigned OpIdx,\n"
+    << "         const MCInst *MI, uint64_t Address, unsigned OpIdx,\n"
     << "         unsigned PrintMethodIdx,\n"
     << (PassSubtarget ? "         const MCSubtargetInfo &STI,\n" : "")
     << "         raw_ostream &OS) {\n";
@@ -1184,7 +1190,8 @@ void AsmWriterEmitter::EmitPrintAliasInstruction(raw_ostream &O) {
 
     for (unsigned i = 0; i < PrintMethods.size(); ++i) {
       O << "  case " << i << ":\n"
-        << "    " << PrintMethods[i] << "(MI, OpIdx, "
+        << "    " << PrintMethods[i].first << "(MI, "
+        << (PrintMethods[i].second ? "Address, " : "") << "OpIdx, "
         << (PassSubtarget ? "STI, " : "") << "OS);\n"
         << "    break;\n";
     }
diff --git a/llvm/utils/TableGen/AsmWriterInst.cpp b/llvm/utils/TableGen/AsmWriterInst.cpp
index c26e0e4211832..24d29ffc28e5d 100644
--- a/llvm/utils/TableGen/AsmWriterInst.cpp
+++ b/llvm/utils/TableGen/AsmWriterInst.cpp
@@ -36,6 +36,8 @@ std::string AsmWriterOperand::getCode(bool PassSubtarget) const {
     return Str;
 
   std::string Result = Str + "(MI";
+  if (PCRel)
+    Result += ", Address";
   if (MIOpNo != ~0U)
     Result += ", " + utostr(MIOpNo);
   if (PassSubtarget)
@@ -179,7 +181,9 @@ AsmWriterInst::AsmWriterInst(const CodeGenInstruction &CGI, unsigned CGIIndex,
         CGIOperandList::OperandInfo OpInfo = CGI.Operands[OpNo];
 
         unsigned MIOp = OpInfo.MIOperandNo;
-        Operands.emplace_back(OpInfo.PrinterMethodName, MIOp, Modifier);
+        Operands.emplace_back(OpInfo.PrinterMethodName, MIOp, Modifier,
+                              AsmWriterOperand::isMachineInstrOperand,
+                              OpInfo.OperandType == "MCOI::OPERAND_PCREL");
       }
       LastEmitted = VarEnd;
     }
diff --git a/llvm/utils/TableGen/AsmWriterInst.h b/llvm/utils/TableGen/AsmWriterInst.h
index a59112efea44f..366c9eca664f0 100644
--- a/llvm/utils/TableGen/AsmWriterInst.h
+++ b/llvm/utils/TableGen/AsmWriterInst.h
@@ -48,6 +48,8 @@ namespace llvm {
     /// an operand, specified with syntax like ${opname:modifier}.
     std::string MiModifier;
 
+    bool PCRel = false;
+
     // To make VS STL happy
     AsmWriterOperand(OpType op = isLiteralTextOperand):OperandType(op) {}
 
@@ -55,11 +57,11 @@ namespace llvm {
                      OpType op = isLiteralTextOperand)
     : OperandType(op), Str(LitStr) {}
 
-    AsmWriterOperand(const std::string &Printer,
-                     unsigned _MIOpNo,
+    AsmWriterOperand(const std::string &Printer, unsigned _MIOpNo,
                      const std::string &Modifier,
-                     OpType op = isMachineInstrOperand)
-    : OperandType(op), MIOpNo(_MIOpNo), Str(Printer), MiModifier(Modifier) {}
+                     OpType op = isMachineInstrOperand, bool PCRel = false)
+        : OperandType(op), MIOpNo(_MIOpNo), Str(Printer), MiModifier(Modifier),
+          PCRel(PCRel) {}
 
     bool operator!=(const AsmWriterOperand &Other) const {
       if (OperandType != Other.OperandType || Str != Other.Str) return true;
diff --git a/llvm/utils/TableGen/CodeGenDAGPatterns.cpp b/llvm/utils/TableGen/CodeGenDAGPatterns.cpp
index db61139d4eb59..6fdc116721f33 100644
--- a/llvm/utils/TableGen/CodeGenDAGPatterns.cpp
+++ b/llvm/utils/TableGen/CodeGenDAGPatterns.cpp
@@ -999,9 +999,9 @@ std::string TreePredicateFn::getPredCode() const {
 
     int64_t MinAlign = getMinAlignment();
     if (MinAlign > 0) {
-      Code += "if (cast<MemSDNode>(N)->getAlignment() < ";
+      Code += "if (cast<MemSDNode>(N)->getAlign() < Align(";
       Code += utostr(MinAlign);
-      Code += ")\nreturn false;\n";
+      Code += "))\nreturn false;\n";
     }
 
     Record *MemoryVT = getMemoryVT();
diff --git a/llvm/utils/TableGen/CodeGenSchedule.cpp b/llvm/utils/TableGen/CodeGenSchedule.cpp
index 09cbb234f8932..67583c736cd2f 100644
--- a/llvm/utils/TableGen/CodeGenSchedule.cpp
+++ b/llvm/utils/TableGen/CodeGenSchedule.cpp
@@ -1082,15 +1082,14 @@ void CodeGenSchedModels::createInstRWClass(Record *InstRWDef) {
           for (Record *RWD : RWDefs) {
             if (RWD->getValueAsDef("SchedModel") == RWModelDef &&
                 RWModelDef->getValueAsBit("FullInstRWOverlapCheck")) {
-              for (Record *Inst : InstDefs) {
-                PrintFatalError
-                    (InstRWDef->getLoc(),
-                     "Overlapping InstRW definition for \"" +
-                     Inst->getName() +
-                     "\" also matches previous \"" +
-                     RWD->getValue("Instrs")->getValue()->getAsString() +
-                     "\".");
-              }
+              assert(!InstDefs.empty()); // Checked at function start.
+              PrintFatalError
+                  (InstRWDef->getLoc(),
+                   "Overlapping InstRW definition for \"" +
+                   InstDefs.front()->getName() +
+                   "\" also matches previous \"" +
+                   RWD->getValue("Instrs")->getValue()->getAsString() +
+                   "\".");
             }
           }
           LLVM_DEBUG(dbgs() << "InstRW: Reuse SC " << OldSCIdx << ":"
@@ -1118,15 +1117,14 @@ void CodeGenSchedModels::createInstRWClass(Record *InstRWDef) {
       Record *RWModelDef = InstRWDef->getValueAsDef("SchedModel");
       for (Record *OldRWDef : SchedClasses[OldSCIdx].InstRWs) {
         if (OldRWDef->getValueAsDef("SchedModel") == RWModelDef) {
-          for (Record *InstDef : InstDefs) {
-            PrintFatalError
-                (InstRWDef->getLoc(),
-                 "Overlapping InstRW definition for \"" +
-                 InstDef->getName() +
-                 "\" also matches previous \"" +
-                 OldRWDef->getValue("Instrs")->getValue()->getAsString() +
-                 "\".");
-          }
+          assert(!InstDefs.empty()); // Checked at function start.
+          PrintFatalError
+              (InstRWDef->getLoc(),
+               "Overlapping InstRW definition for \"" +
+               InstDefs.front()->getName() +
+               "\" also matches previous \"" +
+               OldRWDef->getValue("Instrs")->getValue()->getAsString() +
+               "\".");
         }
         assert(OldRWDef != InstRWDef &&
                "SchedClass has duplicate InstRW def");
diff --git a/llvm/utils/TableGen/InstrInfoEmitter.cpp b/llvm/utils/TableGen/InstrInfoEmitter.cpp
index c857d36f90142..2da2bec112f55 100644
--- a/llvm/utils/TableGen/InstrInfoEmitter.cpp
+++ b/llvm/utils/TableGen/InstrInfoEmitter.cpp
@@ -581,15 +581,66 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
     OS << InstrNames.get(std::string(Inst->TheDef->getName())) << "U, ";
     ++Num;
   }
-
   OS << "\n};\n\n";
 
+  bool HasDeprecationFeatures =
+      llvm::any_of(NumberedInstructions, [](const CodeGenInstruction *Inst) {
+        return !Inst->HasComplexDeprecationPredicate &&
+               !Inst->DeprecatedReason.empty();
+      });
+  if (HasDeprecationFeatures) {
+    OS << "extern const uint8_t " << TargetName
+       << "InstrDeprecationFeatures[] = {";
+    Num = 0;
+    for (const CodeGenInstruction *Inst : NumberedInstructions) {
+      if (Num % 8 == 0)
+        OS << "\n    ";
+      if (!Inst->HasComplexDeprecationPredicate &&
+          !Inst->DeprecatedReason.empty())
+        OS << Target.getInstNamespace() << "::" << Inst->DeprecatedReason
+           << ", ";
+      else
+        OS << "uint8_t(-1), ";
+      ++Num;
+    }
+    OS << "\n};\n\n";
+  }
+
+  bool HasComplexDeprecationInfos =
+      llvm::any_of(NumberedInstructions, [](const CodeGenInstruction *Inst) {
+        return Inst->HasComplexDeprecationPredicate;
+      });
+  if (HasComplexDeprecationInfos) {
+    OS << "extern const MCInstrInfo::ComplexDeprecationPredicate " << TargetName
+       << "InstrComplexDeprecationInfos[] = {";
+    Num = 0;
+    for (const CodeGenInstruction *Inst : NumberedInstructions) {
+      if (Num % 8 == 0)
+        OS << "\n    ";
+      if (Inst->HasComplexDeprecationPredicate)
+        // Emit a function pointer to the complex predicate method.
+        OS << "&get" << Inst->DeprecatedReason << "DeprecationInfo, ";
+      else
+        OS << "nullptr, ";
+      ++Num;
+    }
+    OS << "\n};\n\n";
+  }
+
   // MCInstrInfo initialization routine.
   OS << "static inline void Init" << TargetName
      << "MCInstrInfo(MCInstrInfo *II) {\n";
-  OS << "  II->InitMCInstrInfo(" << TargetName << "Insts, "
-     << TargetName << "InstrNameIndices, " << TargetName << "InstrNameData, "
-     << NumberedInstructions.size() << ");\n}\n\n";
+  OS << "  II->InitMCInstrInfo(" << TargetName << "Insts, " << TargetName
+     << "InstrNameIndices, " << TargetName << "InstrNameData, ";
+  if (HasDeprecationFeatures)
+    OS << TargetName << "InstrDeprecationFeatures, ";
+  else
+    OS << "nullptr, ";
+  if (HasComplexDeprecationInfos)
+    OS << TargetName << "InstrComplexDeprecationInfos, ";
+  else
+    OS << "nullptr, ";
+  OS << NumberedInstructions.size() << ");\n}\n\n";
 
   OS << "} // end namespace llvm\n";
 
@@ -629,12 +680,28 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
   OS << "extern const MCInstrDesc " << TargetName << "Insts[];\n";
   OS << "extern const unsigned " << TargetName << "InstrNameIndices[];\n";
   OS << "extern const char " << TargetName << "InstrNameData[];\n";
+  if (HasDeprecationFeatures)
+    OS << "extern const uint8_t " << TargetName
+       << "InstrDeprecationFeatures[];\n";
+  if (HasComplexDeprecationInfos)
+    OS << "extern const MCInstrInfo::ComplexDeprecationPredicate " << TargetName
+       << "InstrComplexDeprecationInfos[];\n";
   OS << ClassName << "::" << ClassName
-     << "(int CFSetupOpcode, int CFDestroyOpcode, int CatchRetOpcode, int ReturnOpcode)\n"
-     << "  : TargetInstrInfo(CFSetupOpcode, CFDestroyOpcode, CatchRetOpcode, ReturnOpcode) {\n"
+     << "(int CFSetupOpcode, int CFDestroyOpcode, int CatchRetOpcode, int "
+        "ReturnOpcode)\n"
+     << "  : TargetInstrInfo(CFSetupOpcode, CFDestroyOpcode, CatchRetOpcode, "
+        "ReturnOpcode) {\n"
      << "  InitMCInstrInfo(" << TargetName << "Insts, " << TargetName
-     << "InstrNameIndices, " << TargetName << "InstrNameData, "
-     << NumberedInstructions.size() << ");\n}\n";
+     << "InstrNameIndices, " << TargetName << "InstrNameData, ";
+  if (HasDeprecationFeatures)
+    OS << TargetName << "InstrDeprecationFeatures, ";
+  else
+    OS << "nullptr, ";
+  if (HasComplexDeprecationInfos)
+    OS << TargetName << "InstrComplexDeprecationInfos, ";
+  else
+    OS << "nullptr, ";
+  OS << NumberedInstructions.size() << ");\n}\n";
   OS << "} // end namespace llvm\n";
 
   OS << "#endif // GET_INSTRINFO_CTOR_DTOR\n\n";
@@ -745,18 +812,6 @@ void InstrInfoEmitter::emitRecord(const CodeGenInstruction &Inst, unsigned Num,
   else
     OS << "OperandInfo" << OpInfo.find(OperandInfo)->second;
 
-  if (Inst.HasComplexDeprecationPredicate)
-    // Emit a function pointer to the complex predicate method.
-    OS << ", -1 "
-       << ",&get" << Inst.DeprecatedReason << "DeprecationInfo";
-  else if (!Inst.DeprecatedReason.empty())
-    // Emit the Subtarget feature.
-    OS << ", " << Target.getInstNamespace() << "::" << Inst.DeprecatedReason
-       << " ,nullptr";
-  else
-    // Instruction isn't deprecated.
-    OS << ", -1 ,nullptr";
-
   OS << " },  // Inst #" << Num << " = " << Inst.TheDef->getName() << "\n";
 }
 
diff --git a/llvm/utils/TableGen/RISCVCompressInstEmitter.cpp b/llvm/utils/TableGen/RISCVCompressInstEmitter.cpp
index e64a2327f0455..f298e639bf7fc 100644
--- a/llvm/utils/TableGen/RISCVCompressInstEmitter.cpp
+++ b/llvm/utils/TableGen/RISCVCompressInstEmitter.cpp
@@ -85,7 +85,7 @@ class RISCVCompressInstEmitter {
     MapKind Kind;
     union {
       unsigned Operand; // Operand number mapped to.
-      uint64_t Imm;     // Integer immediate value.
+      int64_t Imm;      // Integer immediate value.
       Record *Reg;      // Physical register.
     } Data;
     int TiedOpIdx = -1; // Tied operand index within the instruction.
diff --git a/llvm/utils/UpdateTestChecks/asm.py b/llvm/utils/UpdateTestChecks/asm.py
index ea416ed25bdc5..68f826ec85715 100644
--- a/llvm/utils/UpdateTestChecks/asm.py
+++ b/llvm/utils/UpdateTestChecks/asm.py
@@ -140,10 +140,6 @@ class string:
     r'^\s*(\.Lfunc_end[0-9]+:\n|end_function)',
     flags=(re.M | re.S))
 
-
-SCRUB_LOOP_COMMENT_RE = re.compile(
-    r'# =>This Inner Loop Header:.*|# in Loop:.*', flags=re.M)
-
 SCRUB_X86_SHUFFLES_RE = (
     re.compile(
         r'^(\s*\w+) [^#\n]+#+ ((?:[xyz]mm\d+|mem)( \{%k\d+\}( \{z\})?)? = .*)$',
@@ -234,7 +230,7 @@ def scrub_asm_powerpc(asm, args):
   # Expand the tabs used for indentation.
   asm = string.expandtabs(asm, 2)
   # Stripe unimportant comments, but leave the token '#' in place.
-  asm = SCRUB_LOOP_COMMENT_RE.sub(r'#', asm)
+  asm = common.SCRUB_LOOP_COMMENT_RE.sub(r'#', asm)
   # Strip trailing whitespace.
   asm = common.SCRUB_TRAILING_WHITESPACE_RE.sub(r'', asm)
   return asm
diff --git a/llvm/utils/UpdateTestChecks/common.py b/llvm/utils/UpdateTestChecks/common.py
index 3186628b50a26..e35f4b10c3ba1 100644
--- a/llvm/utils/UpdateTestChecks/common.py
+++ b/llvm/utils/UpdateTestChecks/common.py
@@ -283,22 +283,17 @@ def add_checks(output_lines, comment_marker, prefix_list, func_dict, func_name,
   # prefix_blacklist is constructed, we can now emit the output
   for p in prefix_list:
     checkprefixes = p[0]
-    saved_output = None
     for checkprefix in checkprefixes:
       if checkprefix in printed_prefixes:
         break
 
-      # prefix is blacklisted. We remember the output as we might need it later but we will not emit anything for the prefix.
+      # Check if the prefix is blacklisted.
       if checkprefix in prefix_blacklist:
-          if not saved_output and func_name in func_dict[checkprefix]:
-              saved_output = func_dict[checkprefix][func_name]
-          continue
+        continue
 
-      # If we do not have output for this prefix but there is one saved, we go ahead with this prefix and the saved output.
+      # If we do not have output for this prefix we skip it.
       if not func_dict[checkprefix][func_name]:
-        if not saved_output:
-            continue
-        func_dict[checkprefix][func_name] = saved_output
+        continue
 
       # Add some space between different check prefixes, but not after the last
       # check line (before the test code).
diff --git a/llvm/utils/clang-parse-diagnostics-file b/llvm/utils/clang-parse-diagnostics-file
index 59b13f306505a..1f720c34544a7 100755
--- a/llvm/utils/clang-parse-diagnostics-file
+++ b/llvm/utils/clang-parse-diagnostics-file
@@ -66,6 +66,10 @@ Utility for dumping Clang-style logged diagnostics.\
     for file_diags in diags:
         file = file_diags.get('main-file')
 
+        # Diagnostics from modules don't have a main-file listed.
+        if not file:
+            file = '<module-includes>'
+
         # Ignore diagnostics for 'conftest.c', which is the file autoconf uses
         # for its tests (which frequently will have warnings).
         if os.path.basename(file) == 'conftest.c':
diff --git a/llvm/utils/gn/build/sync_source_lists_from_cmake.py b/llvm/utils/gn/build/sync_source_lists_from_cmake.py
index 0b13a8afc7163..46a779ef376c7 100755
--- a/llvm/utils/gn/build/sync_source_lists_from_cmake.py
+++ b/llvm/utils/gn/build/sync_source_lists_from_cmake.py
@@ -27,13 +27,13 @@ def patch_gn_file(gn_file, add, remove):
     with open(gn_file) as f:
         gn_contents = f.read()
 
-    srcs_tok = 'sources = ['
+    srcs_tok = 'sources = [\n'
     tokloc = gn_contents.find(srcs_tok)
 
     if tokloc == -1: raise ValueError(gn_file + ': Failed to find source list')
     if gn_contents.find(srcs_tok, tokloc + 1) != -1:
         raise ValueError(gn_file + ': Multiple source lists')
-    if gn_file.find('# NOSORT', 0, tokloc) != -1:
+    if gn_contents.find('# NOSORT', 0, tokloc) != -1:
         raise ValueError(gn_file + ': Found # NOSORT, needs manual merge')
 
     tokloc += len(srcs_tok)
diff --git a/llvm/utils/gn/build/write_cmake_config.py b/llvm/utils/gn/build/write_cmake_config.py
index ef706488d19a7..4fb9031795e66 100755
--- a/llvm/utils/gn/build/write_cmake_config.py
+++ b/llvm/utils/gn/build/write_cmake_config.py
@@ -22,7 +22,7 @@
         #define FOO 0
 
 2.) #cmakedefine FOO [...]
-    Checks if key FOO is set to a truthy in value, and depending on that prints
+    Checks if key FOO is set to a truthy value, and depending on that prints
     one of the following two lines:
 
         #define FOO [...]
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clangd/refactor/tweaks/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clangd/refactor/tweaks/BUILD.gn
index 993adce4bb5c0..687b8720cc3ea 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clangd/refactor/tweaks/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clangd/refactor/tweaks/BUILD.gn
@@ -12,6 +12,7 @@ source_set("tweaks") {
   ]
   include_dirs = [ "../.." ]
   sources = [
+    "AddUsing.cpp",
     "AnnotateHighlightings.cpp",
     "DefineInline.cpp",
     "DefineOutline.cpp",
diff --git a/llvm/utils/gn/secondary/clang/include/clang/Config/BUILD.gn b/llvm/utils/gn/secondary/clang/include/clang/Config/BUILD.gn
index cc2c4e19ad492..7fbfb46a41c54 100644
--- a/llvm/utils/gn/secondary/clang/include/clang/Config/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/include/clang/Config/BUILD.gn
@@ -36,6 +36,7 @@ write_cmake_config("Config") {
     "ENABLE_X86_RELAX_RELOCATIONS=",
     "ENABLE_EXPERIMENTAL_NEW_PASS_MANAGER=",
     "CLANG_ENABLE_OBJC_REWRITER=1",  # FIXME: flag?
+    "CLANG_SYSTEMZ_DEFAULT_ARCH=z10",
   ]
 
   if (clang_enable_arcmt) {
diff --git a/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
index aa1fc8fbb39b9..f1c4bfd113324 100644
--- a/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
@@ -46,7 +46,7 @@ copy("arm_headers") {
     ":arm_sve",
   ]
   sources = []
-  foreach (dep, deps) {
+  foreach(dep, deps) {
     sources += get_target_outputs(dep)
   }
   outputs = [ "$clang_resource_dir/include/{{source_file_part}}" ]
@@ -55,61 +55,65 @@ copy("arm_headers") {
 copy("Headers") {
   deps = [ ":arm_headers" ]
 
-  # NOSORT
-  # Tell `gn format` to not reorder the sources list: Its order matches CMake,
-  # and the ordering is alphabetical but ignores leading underscores.
   sources = [
-    "arm_cmse.h",
+    "__clang_cuda_builtin_vars.h",
+    "__clang_cuda_cmath.h",
+    "__clang_cuda_complex_builtins.h",
+    "__clang_cuda_device_functions.h",
+    "__clang_cuda_intrinsics.h",
+    "__clang_cuda_libdevice_declares.h",
+    "__clang_cuda_math.h",
+    "__clang_cuda_math_forward_declares.h",
+    "__clang_cuda_runtime_wrapper.h",
+    "__stddef_max_align_t.h",
+    "__wmmintrin_aes.h",
+    "__wmmintrin_pclmul.h",
     "adxintrin.h",
     "altivec.h",
     "ammintrin.h",
+    "arm64intr.h",
     "arm_acle.h",
+    "arm_cmse.h",
     "armintr.h",
-    "arm64intr.h",
     "avx2intrin.h",
     "avx512bf16intrin.h",
-    "avx512bwintrin.h",
     "avx512bitalgintrin.h",
-    "avx512vlbitalgintrin.h",
+    "avx512bwintrin.h",
     "avx512cdintrin.h",
-    "avx512vpopcntdqintrin.h",
     "avx512dqintrin.h",
     "avx512erintrin.h",
     "avx512fintrin.h",
     "avx512ifmaintrin.h",
     "avx512ifmavlintrin.h",
     "avx512pfintrin.h",
+    "avx512vbmi2intrin.h",
     "avx512vbmiintrin.h",
     "avx512vbmivlintrin.h",
-    "avx512vbmi2intrin.h",
-    "avx512vlvbmi2intrin.h",
     "avx512vlbf16intrin.h",
+    "avx512vlbitalgintrin.h",
     "avx512vlbwintrin.h",
     "avx512vlcdintrin.h",
     "avx512vldqintrin.h",
     "avx512vlintrin.h",
-    "avx512vp2intersectintrin.h",
+    "avx512vlvbmi2intrin.h",
+    "avx512vlvnniintrin.h",
     "avx512vlvp2intersectintrin.h",
-    "avx512vpopcntdqvlintrin.h",
     "avx512vnniintrin.h",
-    "avx512vlvnniintrin.h",
+    "avx512vp2intersectintrin.h",
+    "avx512vpopcntdqintrin.h",
+    "avx512vpopcntdqvlintrin.h",
     "avxintrin.h",
     "bmi2intrin.h",
     "bmiintrin.h",
-    "__clang_cuda_builtin_vars.h",
-    "__clang_cuda_cmath.h",
-    "__clang_cuda_complex_builtins.h",
-    "__clang_cuda_device_functions.h",
-    "__clang_cuda_intrinsics.h",
-    "__clang_cuda_libdevice_declares.h",
-    "__clang_cuda_math_forward_declares.h",
-    "__clang_cuda_runtime_wrapper.h",
     "cetintrin.h",
     "cldemoteintrin.h",
-    "clzerointrin.h",
-    "cpuid.h",
     "clflushoptintrin.h",
     "clwbintrin.h",
+    "clzerointrin.h",
+    "cpuid.h",
+    "cuda_wrappers/algorithm",
+    "cuda_wrappers/complex",
+    "cuda_wrappers/new",
     "emmintrin.h",
     "enqcmdintrin.h",
     "f16cintrin.h",
@@ -130,24 +134,36 @@ copy("Headers") {
     "lwpintrin.h",
     "lzcntintrin.h",
     "mm3dnow.h",
-    "mmintrin.h",
     "mm_malloc.h",
+    "mmintrin.h",
     "module.modulemap",
     "movdirintrin.h",
     "msa.h",
     "mwaitxintrin.h",
     "nmmintrin.h",
-    "opencl-c.h",
     "opencl-c-base.h",
+    "opencl-c.h",
+    "openmp_wrappers/__clang_openmp_math.h",
+    "openmp_wrappers/__clang_openmp_math_declares.h",
+    "openmp_wrappers/cmath",
+    "openmp_wrappers/math.h",
     "pconfigintrin.h",
     "pkuintrin.h",
     "pmmintrin.h",
     "popcntintrin.h",
+    "ppc_wrappers/emmintrin.h",
+    "ppc_wrappers/mm_malloc.h",
+    "ppc_wrappers/mmintrin.h",
+    "ppc_wrappers/pmmintrin.h",
+    "ppc_wrappers/smmintrin.h",
+    "ppc_wrappers/tmmintrin.h",
+    "ppc_wrappers/xmmintrin.h",
     "prfchwintrin.h",
     "ptwriteintrin.h",
     "rdseedintrin.h",
     "rtmintrin.h",
     "s390intrin.h",
+    "serializeintrin.h",
     "sgxintrin.h",
     "shaintrin.h",
     "smmintrin.h",
@@ -156,7 +172,6 @@ copy("Headers") {
     "stdatomic.h",
     "stdbool.h",
     "stddef.h",
-    "__stddef_max_align_t.h",
     "stdint.h",
     "stdnoreturn.h",
     "tbmintrin.h",
@@ -169,10 +184,9 @@ copy("Headers") {
     "vecintrin.h",
     "vpclmulqdqintrin.h",
     "waitpkgintrin.h",
+    "wasm_simd128.h",
     "wbnoinvdintrin.h",
     "wmmintrin.h",
-    "__wmmintrin_aes.h",
-    "__wmmintrin_pclmul.h",
     "x86intrin.h",
     "xmmintrin.h",
     "xopintrin.h",
@@ -181,20 +195,6 @@ copy("Headers") {
     "xsaveoptintrin.h",
     "xsavesintrin.h",
     "xtestintrin.h",
-    "cuda_wrappers/algorithm",
-    "cuda_wrappers/complex",
-    "cuda_wrappers/new",
-    "ppc_wrappers/mmintrin.h",
-    "ppc_wrappers/xmmintrin.h",
-    "ppc_wrappers/mm_malloc.h",
-    "ppc_wrappers/emmintrin.h",
-    "ppc_wrappers/pmmintrin.h",
-    "ppc_wrappers/smmintrin.h",
-    "ppc_wrappers/tmmintrin.h",
-    "openmp_wrappers/math.h",
-    "openmp_wrappers/cmath",
-    "openmp_wrappers/__clang_openmp_math.h",
-    "openmp_wrappers/__clang_openmp_math_declares.h",
   ]
   outputs = [ "$clang_resource_dir/include/{{source_target_relative}}" ]
 }
diff --git a/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Frontend/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Frontend/BUILD.gn
index e4654bf14944b..cfae4227c06bb 100644
--- a/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Frontend/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Frontend/BUILD.gn
@@ -14,8 +14,9 @@ static_library("Frontend") {
   ]
   sources = [
     "AnalysisConsumer.cpp",
-    "CheckerRegistration.cpp",
+    "AnalyzerHelpFlags.cpp",
     "CheckerRegistry.cpp",
+    "CreateCheckerManager.cpp",
     "FrontendActions.cpp",
     "ModelConsumer.cpp",
     "ModelInjector.cpp",
diff --git a/llvm/utils/gn/secondary/clang/test/BUILD.gn b/llvm/utils/gn/secondary/clang/test/BUILD.gn
index 874891e89c81a..84ac2e893cdf5 100644
--- a/llvm/utils/gn/secondary/clang/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/test/BUILD.gn
@@ -12,7 +12,10 @@ template("write_lit_config") {
     input = invoker.input
     output = invoker.output
     values = [
-      "LIT_SITE_CFG_IN_HEADER=## Autogenerated from $input, do not edit",
+      # FIXME: Write relative paths for path()s.
+      "LIT_SITE_CFG_IN_HEADER=" +
+          "## Autogenerated from $input, do not edit\n\n" +
+          "def path(p): return p if p else \"\"",
       "CLANG_BINARY_DIR=" +
           rebase_path(get_label_info("//clang", "target_out_dir")),
       "CLANG_SOURCE_DIR=" + rebase_path("//clang"),
diff --git a/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/BUILD.gn
index 1185a3165c78a..382f5e7066385 100644
--- a/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/BUILD.gn
+++ b/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/BUILD.gn
@@ -89,9 +89,8 @@ source_set("cxx_wrapper_sources") {
 
 config("scudo_config") {
   include_dirs = [
+    ".",
     "include",
-    "//compiler-rt/lib/scudo/standalone",
-    "//compiler-rt/lib/scudo/standalone/include",
   ]
   if (current_os == "android") {
     cflags = [ "-fno-emulated-tls" ]
diff --git a/llvm/utils/gn/secondary/lld/Common/BUILD.gn b/llvm/utils/gn/secondary/lld/Common/BUILD.gn
index 02f91cde67453..125cad30a32e0 100644
--- a/llvm/utils/gn/secondary/lld/Common/BUILD.gn
+++ b/llvm/utils/gn/secondary/lld/Common/BUILD.gn
@@ -39,7 +39,6 @@ static_library("Common") {
     "Reproduce.cpp",
     "Strings.cpp",
     "TargetOptionsCommandFlags.cpp",
-    "Threads.cpp",
     "Timer.cpp",
     "Version.cpp",
   ]
diff --git a/llvm/utils/gn/secondary/lld/MachO/BUILD.gn b/llvm/utils/gn/secondary/lld/MachO/BUILD.gn
new file mode 100644
index 0000000000000..16a21037914dc
--- /dev/null
+++ b/llvm/utils/gn/secondary/lld/MachO/BUILD.gn
@@ -0,0 +1,32 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("Options") {
+  visibility = [ ":MachO2" ]
+  args = [ "-gen-opt-parser-defs" ]
+}
+
+static_library("MachO2") {
+  output_name = "lldMachO2"
+  configs += [ "//llvm/utils/gn/build:lld_code" ]
+  deps = [
+    ":Options",
+    "//lld/Common",
+    "//llvm/lib/BinaryFormat",
+    "//llvm/lib/Object",
+    "//llvm/lib/Option",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target:TargetsToBuild",
+  ]
+  include_dirs = [ "." ]
+  sources = [
+    "Arch/X86_64.cpp",
+    "Driver.cpp",
+    "InputFiles.cpp",
+    "InputSection.cpp",
+    "OutputSegment.cpp",
+    "SymbolTable.cpp",
+    "Symbols.cpp",
+    "Target.cpp",
+    "Writer.cpp",
+  ]
+}
diff --git a/llvm/utils/gn/secondary/lld/tools/lld/BUILD.gn b/llvm/utils/gn/secondary/lld/tools/lld/BUILD.gn
index f79e2b7883a9f..746def906f31a 100644
--- a/llvm/utils/gn/secondary/lld/tools/lld/BUILD.gn
+++ b/llvm/utils/gn/secondary/lld/tools/lld/BUILD.gn
@@ -27,6 +27,7 @@ executable("lld") {
   deps = [
     "//lld/COFF",
     "//lld/ELF",
+    "//lld/MachO:MachO2",
     "//lld/MinGW",
     "//lld/lib/Driver",
     "//lld/wasm",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn
index 82bf99de1bc9b..fba090761c42d 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn
@@ -18,6 +18,7 @@ static_library("Analysis") {
     "AliasAnalysisSummary.cpp",
     "AliasSetTracker.cpp",
     "Analysis.cpp",
+    "AssumeBundleQueries.cpp",
     "AssumptionCache.cpp",
     "BasicAliasAnalysis.cpp",
     "BlockFrequencyInfo.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/IR/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/IR/BUILD.gn
index 55a81edf5acb5..aa70596afe81c 100644
--- a/llvm/utils/gn/secondary/llvm/lib/IR/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/IR/BUILD.gn
@@ -43,7 +43,6 @@ static_library("IR") {
     "Instruction.cpp",
     "Instructions.cpp",
     "IntrinsicInst.cpp",
-    "KnowledgeRetention.cpp",
     "LLVMContext.cpp",
     "LLVMContextImpl.cpp",
     "LLVMRemarkStreamer.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/MC/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/MC/BUILD.gn
index bac3549860fb8..e0c7701e98964 100644
--- a/llvm/utils/gn/secondary/llvm/lib/MC/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/MC/BUILD.gn
@@ -34,6 +34,7 @@ static_library("MC") {
     "MCInstPrinter.cpp",
     "MCInstrAnalysis.cpp",
     "MCInstrDesc.cpp",
+    "MCInstrInfo.cpp",
     "MCLabel.cpp",
     "MCLinkerOptimizationHint.cpp",
     "MCMachOStreamer.cpp",
@@ -54,6 +55,7 @@ static_library("MC") {
     "MCSubtargetInfo.cpp",
     "MCSymbol.cpp",
     "MCSymbolELF.cpp",
+    "MCSymbolXCOFF.cpp",
     "MCTargetOptions.cpp",
     "MCTargetOptionsCommandFlags.cpp",
     "MCValue.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
index b1200b6c60d29..13ffdaedfb66c 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
@@ -65,6 +65,8 @@ static_library("Support") {
     "Debug.cpp",
     "DebugCounter.cpp",
     "DeltaAlgorithm.cpp",
+    "ELFAttributeParser.cpp",
+    "ELFAttributes.cpp",
     "Error.cpp",
     "ErrorHandling.cpp",
     "FileCheck.cpp",
@@ -98,6 +100,8 @@ static_library("Support") {
     "Parallel.cpp",
     "PluginLoader.cpp",
     "PrettyStackTrace.cpp",
+    "RISCVAttributeParser.cpp",
+    "RISCVAttributes.cpp",
     "RWMutex.cpp",
     "RandomNumberGenerator.cpp",
     "Regex.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
index 43b58f2d61359..aa13890398267 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
@@ -197,6 +197,7 @@ static_library("LLVMAMDGPUCodeGen") {
     "SIPeepholeSDWA.cpp",
     "SIPostRABundler.cpp",
     "SIPreAllocateWWMRegs.cpp",
+    "SIPreEmitPeephole.cpp",
     "SIRegisterInfo.cpp",
     "SIRemoveShortExecBranches.cpp",
     "SIShrinkInstructions.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/X86/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/X86/BUILD.gn
index e0b596375309a..46b9242ca5a3d 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/X86/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/X86/BUILD.gn
@@ -98,6 +98,7 @@ static_library("LLVMX86CodeGen") {
     "X86ISelDAGToDAG.cpp",
     "X86ISelLowering.cpp",
     "X86IndirectBranchTracking.cpp",
+    "X86IndirectThunks.cpp",
     "X86InsertPrefetch.cpp",
     "X86InsertWait.cpp",
     "X86InstrFMA3Info.cpp",
@@ -106,14 +107,16 @@ static_library("LLVMX86CodeGen") {
     "X86InstructionSelector.cpp",
     "X86InterleavedAccess.cpp",
     "X86LegalizerInfo.cpp",
+    "X86LoadValueInjectionLoadHardening.cpp",
+    "X86LoadValueInjectionRetHardening.cpp",
     "X86MCInstLower.cpp",
     "X86MachineFunctionInfo.cpp",
     "X86MacroFusion.cpp",
     "X86OptimizeLEAs.cpp",
     "X86PadShortFunction.cpp",
+    "X86PartialReduction.cpp",
     "X86RegisterBankInfo.cpp",
     "X86RegisterInfo.cpp",
-    "X86RetpolineThunks.cpp",
     "X86SelectionDAGInfo.cpp",
     "X86ShuffleDecodeConstantPool.cpp",
     "X86SpeculativeLoadHardening.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/targets.gni b/llvm/utils/gn/secondary/llvm/lib/Target/targets.gni
index 2590432529b6a..102040c2fa82d 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/targets.gni
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/targets.gni
@@ -68,9 +68,9 @@ foreach(target, llvm_targets_to_build) {
     llvm_build_WebAssembly = true
   } else if (target == "X86") {
     llvm_build_X86 = true
-  } else if (target == "AVR" || target == "Hexagon" ||
-             target == "Lanai" || target == "NVPTX" || target == "RISCV" ||
-             target == "Sparc" || target == "SystemZ") {
+  } else if (target == "AVR" || target == "Hexagon" || target == "Lanai" ||
+             target == "NVPTX" || target == "RISCV" || target == "Sparc" ||
+             target == "SystemZ") {
     # Nothing to do.
   } else {
     all_targets_string = ""
diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn
index 0618c57f3f5fc..3a132cca253f4 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn
@@ -9,6 +9,7 @@ static_library("Utils") {
     "AMDGPUEmitPrintf.cpp",
     "ASanStackFrameLayout.cpp",
     "AddDiscriminators.cpp",
+    "AssumeBundleBuilder.cpp",
     "BasicBlockUtils.cpp",
     "BreakCriticalEdges.cpp",
     "BuildLibCalls.cpp",
@@ -69,6 +70,7 @@ static_library("Utils") {
     "StripNonLineTableDebugInfo.cpp",
     "SymbolRewriter.cpp",
     "UnifyFunctionExitNodes.cpp",
+    "UnifyLoopExits.cpp",
     "Utils.cpp",
     "VNCoercion.cpp",
     "ValueMapper.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/test/BUILD.gn b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
index 0f7d90d1ab60f..c7740ac4cc99b 100644
--- a/llvm/utils/gn/secondary/llvm/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
@@ -14,7 +14,10 @@ template("write_lit_config") {
     input = invoker.input
     output = invoker.output
     values = [
-      "LIT_SITE_CFG_IN_HEADER=## Autogenerated from $input, do not edit",
+      # FIXME: Write relative paths for path()s.
+      "LIT_SITE_CFG_IN_HEADER=" +
+          "## Autogenerated from $input, do not edit\n\n" +
+          "def path(p): return p if p else \"\"",
       "ENABLE_SHARED=0",
       "LLVM_BINARY_DIR=" +
           rebase_path(get_label_info("//llvm", "target_out_dir")),
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-dwarfdump/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-dwarfdump/BUILD.gn
index b99086fd4dc20..10f2d50d00433 100644
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-dwarfdump/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-dwarfdump/BUILD.gn
@@ -8,6 +8,7 @@ executable("llvm-dwarfdump") {
     "//llvm/lib/Target:AllTargetsInfos",
   ]
   sources = [
+    "SectionSizes.cpp",
     "Statistics.cpp",
     "llvm-dwarfdump.cpp",
   ]
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-objdump/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-objdump/BUILD.gn
index 06bbf56ccad32..c8ba176dd34fb 100644
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-objdump/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-objdump/BUILD.gn
@@ -39,6 +39,7 @@ executable("llvm-objdump") {
     "ELFDump.cpp",
     "MachODump.cpp",
     "WasmDump.cpp",
+    "XCOFFDump.cpp",
     "llvm-objdump.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/llvm/unittests/ADT/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/ADT/BUILD.gn
index c7ee1ad377ff5..0c58814c952a8 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/ADT/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/ADT/BUILD.gn
@@ -80,5 +80,6 @@ unittest("ADTTests") {
     "TinyPtrVectorTest.cpp",
     "TripleTest.cpp",
     "TwineTest.cpp",
+    "WaymarkingTest.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/llvm/unittests/Analysis/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Analysis/BUILD.gn
index d73088885aa86..03c5dc3f25039 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/Analysis/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/Analysis/BUILD.gn
@@ -11,6 +11,7 @@ unittest("AnalysisTests") {
   sources = [
     "AliasAnalysisTest.cpp",
     "AliasSetTrackerTest.cpp",
+    "AssumeBundleQueriesTest.cpp",
     "BasicAliasAnalysisTest.cpp",
     "BlockFrequencyInfoTest.cpp",
     "BranchProbabilityInfoTest.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/IR/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/IR/BUILD.gn
index 92b3f4e37810e..97dd0da6f92b6 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/IR/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/IR/BUILD.gn
@@ -26,7 +26,6 @@ unittest("IRTests") {
     "IRBuilderTest.cpp",
     "InstructionsTest.cpp",
     "IntrinsicsTest.cpp",
-    "KnowledgeRetentionTest.cpp",
     "LegacyPassManagerTest.cpp",
     "MDBuilderTest.cpp",
     "ManglerTest.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/MC/AMDGPU/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/MC/AMDGPU/BUILD.gn
index 8d1763e65f0ce..ccfe716e8ed78 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/MC/AMDGPU/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/MC/AMDGPU/BUILD.gn
@@ -9,6 +9,7 @@ unittest("AMDGPUDwarfTests") {
     "//llvm/lib/Target/AMDGPU/TargetInfo",
   ]
   sources = [
+    # Make `gn format` not collapse this, for sync_source_lists_from_cmake.py.
     "DwarfRegMappings.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
index 0ba402cf9e815..c25323955b969 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
@@ -29,6 +29,7 @@ unittest("SupportTests") {
     "DataExtractorTest.cpp",
     "DebugCounterTest.cpp",
     "DebugTest.cpp",
+    "ELFAttributeParserTest.cpp",
     "EndianStreamTest.cpp",
     "EndianTest.cpp",
     "ErrnoTest.cpp",
@@ -59,6 +60,7 @@ unittest("SupportTests") {
     "Path.cpp",
     "ProcessTest.cpp",
     "ProgramTest.cpp",
+    "RISCVAttributeParserTest.cpp",
     "RegexTest.cpp",
     "ReplaceFileTest.cpp",
     "ReverseIterationTest.cpp",
diff --git a/llvm/utils/lit/lit/LitConfig.py b/llvm/utils/lit/lit/LitConfig.py
index 58011b5986bf6..9ec634388cc17 100644
--- a/llvm/utils/lit/lit/LitConfig.py
+++ b/llvm/utils/lit/lit/LitConfig.py
@@ -66,6 +66,19 @@ def __init__(self, progname, path, quiet,
         self.maxIndividualTestTime = maxIndividualTestTime
         self.parallelism_groups = parallelism_groups
         self.echo_all_commands = echo_all_commands
+        self._recursiveExpansionLimit = None
+
+    @property
+    def recursiveExpansionLimit(self):
+        return self._recursiveExpansionLimit
+
+    @recursiveExpansionLimit.setter
+    def recursiveExpansionLimit(self, value):
+        if value is not None and not isinstance(value, int):
+            self.fatal('recursiveExpansionLimit must be either None or an integer (got <{}>)'.format(value))
+        if isinstance(value, int) and value < 0:
+            self.fatal('recursiveExpansionLimit must be a non-negative integer (got <{}>)'.format(value))
+        self._recursiveExpansionLimit = value
 
     @property
     def maxIndividualTestTime(self):
diff --git a/llvm/utils/lit/lit/Test.py b/llvm/utils/lit/lit/Test.py
index 000bcf8fc38fe..c809783dae7e6 100644
--- a/llvm/utils/lit/lit/Test.py
+++ b/llvm/utils/lit/lit/Test.py
@@ -230,6 +230,20 @@ def __init__(self, suite, path_in_suite, config, file_path = None):
     def setResult(self, result):
         assert self.result is None, "result already set"
         assert isinstance(result, Result), "unexpected result type"
+        try:
+            expected_to_fail = self.isExpectedToFail()
+        except ValueError as err:
+            # Syntax error in an XFAIL line.
+            result.code = UNRESOLVED
+            result.output = str(err)
+        else:
+            if expected_to_fail:
+                # pass -> unexpected pass
+                if result.code is PASS:
+                    result.code = XPASS
+                # fail -> expected fail
+                elif result.code is FAIL:
+                    result.code = XFAIL
         self.result = result
 
     def isFailure(self):
diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
index 4ee3b673c28f9..e227dc8fd4ded 100644
--- a/llvm/utils/lit/lit/TestRunner.py
+++ b/llvm/utils/lit/lit/TestRunner.py
@@ -147,7 +147,7 @@ def __init__(self, command, stdout, stderr, exitCode, timeoutReached,
         self.exitCode = exitCode
         self.timeoutReached = timeoutReached
         self.outputFiles = list(outputFiles)
-               
+
 def executeShCmd(cmd, shenv, results, timeout=0):
     """
         Wrapper around _executeShCmd that handles
@@ -844,7 +844,7 @@ def _executeShCmd(cmd, shenv, results, timeoutHelper):
                         data = None
                     if data is not None:
                         output_files.append((name, path, data))
-            
+
         results.append(ShellCommandResult(
             cmd.commands[i], out, err, res, timeoutHelper.timeoutReached(),
             output_files))
@@ -917,7 +917,7 @@ def executeScriptInternal(test, litConfig, tmpBase, commands, cwd):
                 else:
                     out += data
                 out += "\n"
-                    
+
         if result.stdout.strip():
             out += '# command output:\n%s\n' % (result.stdout,)
         if result.stderr.strip():
@@ -1147,10 +1147,18 @@ def memoized(x):
 def _caching_re_compile(r):
     return re.compile(r)
 
-def applySubstitutions(script, substitutions):
-    """Apply substitutions to the script.  Allow full regular expression syntax.
+def applySubstitutions(script, substitutions, recursion_limit=None):
+    """
+    Apply substitutions to the script.  Allow full regular expression syntax.
     Replace each matching occurrence of regular expression pattern a with
-    substitution b in line ln."""
+    substitution b in line ln.
+
+    If a substitution expands into another substitution, it is expanded
+    recursively until the line has no more expandable substitutions. If
+    the line can still can be substituted after being substituted
+    `recursion_limit` times, it is an error. If the `recursion_limit` is
+    `None` (the default), no recursive substitution is performed at all.
+    """
     def processLine(ln):
         # Apply substitutions
         for a,b in substitutions:
@@ -1167,9 +1175,28 @@ def processLine(ln):
 
         # Strip the trailing newline and any extra whitespace.
         return ln.strip()
+
+    def processLineToFixedPoint(ln):
+        assert isinstance(recursion_limit, int) and recursion_limit >= 0
+        origLine = ln
+        steps = 0
+        processed = processLine(ln)
+        while processed != ln and steps < recursion_limit:
+            ln = processed
+            processed = processLine(ln)
+            steps += 1
+
+        if processed != ln:
+            raise ValueError("Recursive substitution of '%s' did not complete "
+                             "in the provided recursion limit (%s)" % \
+                             (origLine, recursion_limit))
+
+        return processed
+
     # Note Python 3 map() gives an iterator rather than a list so explicitly
     # convert to list before returning.
-    return list(map(processLine, script))
+    process = processLine if recursion_limit is None else processLineToFixedPoint
+    return list(map(process, script))
 
 
 class ParserKind(object):
@@ -1180,7 +1207,7 @@ class ParserKind(object):
     TAG: A keyword taking no value. Ex 'END.'
     COMMAND: A keyword taking a list of shell commands. Ex 'RUN:'
     LIST: A keyword taking a comma-separated list of values.
-    BOOLEAN_EXPR: A keyword taking a comma-separated list of 
+    BOOLEAN_EXPR: A keyword taking a comma-separated list of
         boolean expressions. Ex 'XFAIL:'
     INTEGER: A keyword taking a single integer. Ex 'ALLOW_RETRIES:'
     CUSTOM: A keyword with custom parsing semantics.
@@ -1375,7 +1402,7 @@ def parseIntegratedTestScript(test, additional_parsers=[],
         IntegratedTestKeywordParser('END.', ParserKind.TAG)
     ]
     keyword_parsers = {p.keyword: p for p in builtin_parsers}
-    
+
     # Install user-defined additional parsers.
     for parser in additional_parsers:
         if not isinstance(parser, IntegratedTestKeywordParser):
@@ -1385,7 +1412,7 @@ def parseIntegratedTestScript(test, additional_parsers=[],
             raise ValueError("Parser for keyword '%s' already exists"
                              % parser.keyword)
         keyword_parsers[parser.keyword] = parser
-        
+
     # Collect the test lines from the script.
     sourcepath = test.getSourcePath()
     for line_number, command_type, ln in \
@@ -1450,25 +1477,43 @@ def parseIntegratedTestScript(test, additional_parsers=[],
 
 
 def _runShTest(test, litConfig, useExternalSh, script, tmpBase):
+    def runOnce(execdir):
+        if useExternalSh:
+            res = executeScript(test, litConfig, tmpBase, script, execdir)
+        else:
+            res = executeScriptInternal(test, litConfig, tmpBase, script, execdir)
+        if isinstance(res, lit.Test.Result):
+            return res
+
+        out,err,exitCode,timeoutInfo = res
+        if exitCode == 0:
+            status = Test.PASS
+        else:
+            if timeoutInfo is None:
+                status = Test.FAIL
+            else:
+                status = Test.TIMEOUT
+        return out,err,exitCode,timeoutInfo,status
+
     # Create the output directory if it does not already exist.
     lit.util.mkdir_p(os.path.dirname(tmpBase))
 
+    # Re-run failed tests up to test.allowed_retries times.
     execdir = os.path.dirname(test.getExecPath())
-    if useExternalSh:
-        res = executeScript(test, litConfig, tmpBase, script, execdir)
-    else:
-        res = executeScriptInternal(test, litConfig, tmpBase, script, execdir)
-    if isinstance(res, lit.Test.Result):
-        return res
+    attempts = test.allowed_retries + 1
+    for i in range(attempts):
+        res = runOnce(execdir)
+        if isinstance(res, lit.Test.Result):
+            return res
 
-    out,err,exitCode,timeoutInfo = res
-    if exitCode == 0:
-        status = Test.PASS
-    else:
-        if timeoutInfo is None:
-            status = Test.FAIL
-        else:
-            status = Test.TIMEOUT
+        out,err,exitCode,timeoutInfo,status = res
+        if status != Test.FAIL:
+            break
+
+    # If we had to run the test more than once, count it as a flaky pass. These
+    # will be printed separately in the test summary.
+    if i > 0 and status == Test.PASS:
+        status = Test.FLAKYPASS
 
     # Form the output log.
     output = """Script:\n--\n%s\n--\nExit Code: %d\n""" % (
@@ -1506,16 +1551,7 @@ def executeShTest(test, litConfig, useExternalSh,
     substitutions = list(extra_substitutions)
     substitutions += getDefaultSubstitutions(test, tmpDir, tmpBase,
                                              normalize_slashes=useExternalSh)
-    script = applySubstitutions(script, substitutions)
+    script = applySubstitutions(script, substitutions,
+                                recursion_limit=litConfig.recursiveExpansionLimit)
 
-    # Re-run failed tests up to test.allowed_retries times.
-    attempts = test.allowed_retries + 1
-    for i in range(attempts):
-        res = _runShTest(test, litConfig, useExternalSh, script, tmpBase)
-        if res.code != Test.FAIL:
-            break
-    # If we had to run the test more than once, count it as a flaky pass. These
-    # will be printed separately in the test summary.
-    if i > 0 and res.code == Test.PASS:
-        res.code = Test.FLAKYPASS
-    return res
+    return _runShTest(test, litConfig, useExternalSh, script, tmpBase)
diff --git a/llvm/utils/lit/lit/__init__.py b/llvm/utils/lit/lit/__init__.py
index d364464d7789f..6bf0132d42528 100644
--- a/llvm/utils/lit/lit/__init__.py
+++ b/llvm/utils/lit/lit/__init__.py
@@ -6,4 +6,3 @@
 __version__ = '.'.join(str(v) for v in __versioninfo__) + 'dev'
 
 __all__ = []
-from .main import main
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 402fadb12d6c5..656a783d3dca5 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -5,18 +5,17 @@
 
 import lit.util
 
+
 def parse_args():
-    parser = argparse.ArgumentParser()
+    parser = argparse.ArgumentParser(prog='lit')
     parser.add_argument('test_paths',
             nargs='+',
             metavar="TEST_PATH",
             help='File or path to include in the test suite')
 
-    parser.add_argument("--version",
-            dest="show_version",
-            help="Show version and exit",
-            version="lit " + lit.__version__,
-            action="version")
+    parser.add_argument('--version',
+            action='version',
+            version='%(prog)s ' + lit.__version__)
 
     parser.add_argument("-j", "--threads", "--workers",
             dest="workers",
@@ -134,7 +133,7 @@ def parse_args():
             metavar="REGEX",
             type=_case_insensitive_regex,
             help="Only run tests with paths matching the given regular expression",
-            default=os.environ.get("LIT_FILTER"))
+            default=os.environ.get("LIT_FILTER", ".*"))
     selection_group.add_argument("--num-shards", # TODO(yln): --shards N/M
             dest="numShards",
             metavar="M",
@@ -189,12 +188,15 @@ def parse_args():
 
     return opts
 
+
 def _positive_int(arg):
     return _int(arg, 'positive', lambda i: i > 0)
 
+
 def _non_negative_int(arg):
     return _int(arg, 'non-negative', lambda i: i >= 0)
 
+
 def _int(arg, kind, pred):
     desc = "requires {} integer, but found '{}'"
     try:
@@ -205,6 +207,7 @@ def _int(arg, kind, pred):
         raise _error(desc, kind, arg)
     return i
 
+
 def _case_insensitive_regex(arg):
     import re
     try:
@@ -212,6 +215,7 @@ def _case_insensitive_regex(arg):
     except re.error as reason:
         raise _error("invalid regular expression: '{}', {}", arg, reason)
 
+
 def _error(desc, *args):
     msg = desc.format(*args)
     return argparse.ArgumentTypeError(msg)
diff --git a/llvm/utils/lit/lit/formats/shtest.py b/llvm/utils/lit/lit/formats/shtest.py
index fdc9bd0241f3f..e1be48cbd37f1 100644
--- a/llvm/utils/lit/lit/formats/shtest.py
+++ b/llvm/utils/lit/lit/formats/shtest.py
@@ -17,9 +17,14 @@ class ShTest(FileBasedTest):
     The ShTest files contain some number of shell-like command pipelines, along
     with assertions about what should be in the output.
     """
-    def __init__(self, execute_external=False):
+    def __init__(self, execute_external=False, extra_substitutions=[],
+                 preamble_commands=[]):
         self.execute_external = execute_external
+        self.extra_substitutions = extra_substitutions
+        self.preamble_commands = preamble_commands
 
     def execute(self, test, litConfig):
         return lit.TestRunner.executeShTest(test, litConfig,
-                                            self.execute_external)
+                                            self.execute_external,
+                                            self.extra_substitutions,
+                                            self.preamble_commands)
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index c530d3485ffba..fb45995a6280f 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -20,7 +20,6 @@
 
 def main(builtin_params={}):
     opts = lit.cl_arguments.parse_args()
-
     params = create_params(builtin_params, opts.user_params)
     is_windows = platform.system() == 'Windows'
 
@@ -58,22 +57,19 @@ def main(builtin_params={}):
         print_suites_or_tests(discovered_tests, opts)
         return
 
-    if opts.filter:
-        filtered_tests = [t for t in discovered_tests if
-                          opts.filter.search(t.getFullName())]
-        if not filtered_tests:
-            sys.stderr.write('error: filter did not match any tests '
-                             '(of %d discovered).  ' % len(discovered_tests))
-            if opts.allow_empty_runs:
-                sys.stderr.write('Suppressing error because '
-                                 "'--allow-empty-runs' was specified.\n")
-                sys.exit(0)
-            else:
-                sys.stderr.write("Use '--allow-empty-runs' to suppress this "
-                                 'error.\n')
-                sys.exit(2)
-    else:
-        filtered_tests = discovered_tests
+    filtered_tests = [t for t in discovered_tests if
+                      opts.filter.search(t.getFullName())]
+    if not filtered_tests:
+        sys.stderr.write('error: filter did not match any tests '
+                         '(of %d discovered).  ' % len(discovered_tests))
+        if opts.allow_empty_runs:
+            sys.stderr.write("Suppressing error because '--allow-empty-runs' "
+                             'was specified.\n')
+            sys.exit(0)
+        else:
+            sys.stderr.write("Use '--allow-empty-runs' to suppress this "
+                             'error.\n')
+            sys.exit(2)
 
     determine_order(filtered_tests, opts.order)
 
@@ -85,8 +81,7 @@ def main(builtin_params={}):
                              'Consider decreasing the number of shards.\n')
             sys.exit(0)
 
-    if opts.max_tests:
-        filtered_tests = filtered_tests[:opts.max_tests]
+    filtered_tests = filtered_tests[:opts.max_tests]
 
     opts.workers = min(len(filtered_tests), opts.workers)
 
@@ -191,24 +186,34 @@ def filter_by_shard(tests, run, shards, lit_config):
     return selected_tests
 
 
-def run_tests(tests, lit_config, opts, numTotalTests):
-    display = lit.display.create_display(opts, len(tests), numTotalTests,
+def run_tests(tests, lit_config, opts, discovered_tests):
+    display = lit.display.create_display(opts, len(tests), discovered_tests,
                                          opts.workers)
     def progress_callback(test):
         display.update(test)
         if opts.order == 'failing-first':
             touch_file(test)
 
-    run = lit.run.create_run(tests, lit_config, opts.workers, progress_callback,
-                             opts.max_failures, opts.timeout)
+    run = lit.run.Run(tests, lit_config, opts.workers, progress_callback,
+                      opts.max_failures, opts.timeout)
 
     display.print_header()
+
+    interrupted = False
+    error = None
     try:
         execute_in_tmp_dir(run, lit_config)
-        display.clear(interrupted=False)
     except KeyboardInterrupt:
-        display.clear(interrupted=True)
-        print(' [interrupted by user]')
+        interrupted = True
+        error = '  interrupted by user'
+    except lit.run.MaxFailuresError:
+        error = 'warning: reached maximum number of test failures'
+    except lit.run.TimeoutError:
+        error = 'warning: reached timeout'
+
+    display.clear(interrupted)
+    if error:
+        sys.stderr.write('%s, skipping remaining tests\n' % error)
 
 
 def execute_in_tmp_dir(run, lit_config):
@@ -227,10 +232,6 @@ def execute_in_tmp_dir(run, lit_config):
                 'TEMP': tmp_dir,
                 'TEMPDIR': tmp_dir,
                 })
-    # FIXME: If Python does not exit cleanly, this directory will not be cleaned
-    # up. We should consider writing the lit pid into the temp directory,
-    # scanning for stale temp directories, and deleting temp directories whose
-    # lit process has died.
     try:
         run.execute()
     finally:
diff --git a/llvm/utils/lit/lit/run.py b/llvm/utils/lit/lit/run.py
index 65495ed9dd352..fca3fb47fd5e2 100644
--- a/llvm/utils/lit/lit/run.py
+++ b/llvm/utils/lit/lit/run.py
@@ -14,23 +14,24 @@ def acquire(self): pass
     def release(self): pass
 
 
-def create_run(tests, lit_config, workers, progress_callback, max_failures,
-               timeout):
-    assert workers > 0
-    if workers == 1:
-        return SerialRun(tests, lit_config, progress_callback, max_failures, timeout)
-    return ParallelRun(tests, lit_config, progress_callback, max_failures, timeout, workers)
+class MaxFailuresError(Exception):
+    pass
+class TimeoutError(Exception):
+    pass
 
 
 class Run(object):
     """A concrete, configured testing run."""
 
-    def __init__(self, tests, lit_config, progress_callback, max_failures, timeout):
+    def __init__(self, tests, lit_config, workers, progress_callback,
+                 max_failures, timeout):
         self.tests = tests
         self.lit_config = lit_config
+        self.workers = workers
         self.progress_callback = progress_callback
         self.max_failures = max_failures
         self.timeout = timeout
+        assert workers > 0
 
     def execute(self):
         """
@@ -50,8 +51,7 @@ def execute(self):
         computed. Tests which were not actually executed (for any reason) will
         be given an UNRESOLVED result.
         """
-        self.failure_count = 0
-        self.hit_max_failures = False
+        self.failures = 0
 
         # Larger timeouts (one year, positive infinity) don't work on Windows.
         one_week = 7 * 24 * 60 * 60  # days * hours * minutes * seconds
@@ -65,44 +65,6 @@ def execute(self):
             if test.result is None:
                 test.setResult(lit.Test.Result(lit.Test.UNRESOLVED, '', 0.0))
 
-    # TODO(yln): as the comment says.. this is racing with the main thread waiting
-    # for results
-    def _process_result(self, test, result):
-        # Don't add any more test results after we've hit the maximum failure
-        # count.  Otherwise we're racing with the main thread, which is going
-        # to terminate the process pool soon.
-        if self.hit_max_failures:
-            return
-
-        test.setResult(result)
-
-        # Use test.isFailure() for correct XFAIL and XPASS handling
-        if test.isFailure():
-            self.failure_count += 1
-            if self.failure_count == self.max_failures:
-                self.hit_max_failures = True
-
-        self.progress_callback(test)
-
-
-class SerialRun(Run):
-    def __init__(self, tests, lit_config, progress_callback, max_failures, timeout):
-        super(SerialRun, self).__init__(tests, lit_config, progress_callback, max_failures, timeout)
-
-    def _execute(self, deadline):
-        # TODO(yln): ignores deadline
-        for test in self.tests:
-            result = lit.worker._execute(test, self.lit_config)
-            self._process_result(test, result)
-            if self.hit_max_failures:
-                break
-
-
-class ParallelRun(Run):
-    def __init__(self, tests, lit_config, progress_callback, max_failures, timeout, workers):
-        super(ParallelRun, self).__init__(tests, lit_config, progress_callback, max_failures, timeout)
-        self.workers = workers
-
     def _execute(self, deadline):
         semaphores = {
             k: NopSemaphore() if v is None else
@@ -111,34 +73,36 @@ def _execute(self, deadline):
 
         self._increase_process_limit()
 
-        # Start a process pool. Copy over the data shared between all test runs.
-        # FIXME: Find a way to capture the worker process stderr. If the user
-        # interrupts the workers before we make it into our task callback, they
-        # will each raise a KeyboardInterrupt exception and print to stderr at
-        # the same time.
         pool = multiprocessing.Pool(self.workers, lit.worker.initialize,
                                     (self.lit_config, semaphores))
 
-        self._install_win32_signal_handler(pool)
-
         async_results = [
             pool.apply_async(lit.worker.execute, args=[test],
-                callback=lambda r, t=test: self._process_result(t, r))
+                             callback=self.progress_callback)
             for test in self.tests]
         pool.close()
 
-        for ar in async_results:
-            timeout = deadline - time.time()
+        try:
+            self._wait_for(async_results, deadline)
+        except:
+            pool.terminate()
+            raise
+        finally:
+            pool.join()
+
+    def _wait_for(self, async_results, deadline):
+        timeout = deadline - time.time()
+        for idx, ar in enumerate(async_results):
             try:
-                ar.get(timeout)
+                test = ar.get(timeout)
             except multiprocessing.TimeoutError:
-                # TODO(yln): print timeout error
-                pool.terminate()
-                break
-            if self.hit_max_failures:
-                pool.terminate()
-                break
-        pool.join()
+                raise TimeoutError()
+            else:
+                self.tests[idx] = test
+                if test.isFailure():
+                    self.failures += 1
+                    if self.failures == self.max_failures:
+                        raise MaxFailuresError()
 
     # TODO(yln): interferes with progress bar
     # Some tests use threads internally, and at least on Linux each of these
@@ -164,13 +128,3 @@ def _increase_process_limit(self):
             # Warn, unless this is Windows, in which case this is expected.
             if os.name != 'nt':
                 self.lit_config.warning('Failed to raise process limit: %s' % ex)
-
-    def _install_win32_signal_handler(self, pool):
-        if lit.util.win32api is not None:
-            def console_ctrl_handler(type):
-                print('\nCtrl-C detected, terminating.')
-                pool.terminate()
-                pool.join()
-                lit.util.abort_now()
-                return True
-            lit.util.win32api.SetConsoleCtrlHandler(console_ctrl_handler, True)
diff --git a/llvm/utils/lit/lit/util.py b/llvm/utils/lit/lit/util.py
index fa752d0e8d6ea..ab51988e21ecd 100644
--- a/llvm/utils/lit/lit/util.py
+++ b/llvm/utils/lit/lit/util.py
@@ -479,17 +479,3 @@ def killProcessAndChildren(pid):
             psutilProc.kill()
         except psutil.NoSuchProcess:
             pass
-
-
-try:
-    import win32api
-except ImportError:
-    win32api = None
-
-def abort_now():
-    """Abort the current process without doing any exception teardown"""
-    sys.stdout.flush()
-    if win32api:
-        win32api.TerminateProcess(win32api.GetCurrentProcess(), 3)
-    else:
-        os.kill(0, 9)
diff --git a/llvm/utils/lit/lit/worker.py b/llvm/utils/lit/lit/worker.py
index 02fcc20caaf20..95a8578afac99 100644
--- a/llvm/utils/lit/lit/worker.py
+++ b/llvm/utils/lit/lit/worker.py
@@ -5,15 +5,18 @@
 For efficiency, we copy all data needed to execute all tests into each worker
 and store it in global variables. This reduces the cost of each task.
 """
+import signal
 import time
 import traceback
 
 import lit.Test
 import lit.util
 
+
 _lit_config = None
 _parallelism_semaphores = None
 
+
 def initialize(lit_config, parallelism_semaphores):
     """Copy data shared by all test executions into worker processes"""
     global _lit_config
@@ -21,6 +24,12 @@ def initialize(lit_config, parallelism_semaphores):
     _lit_config = lit_config
     _parallelism_semaphores = parallelism_semaphores
 
+    # We use the following strategy for dealing with Ctrl+C/KeyboardInterrupt in
+    # subprocesses created by the multiprocessing.Pool.
+    # https://noswap.com/blog/python-multiprocessing-keyboardinterrupt
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+
 def execute(test):
     """Run one test in a multiprocessing.Pool
 
@@ -30,14 +39,11 @@ def execute(test):
     Arguments and results of this function are pickled, so they should be cheap
     to copy.
     """
-    try:
-        return _execute_in_parallelism_group(test, _lit_config,
-                                             _parallelism_semaphores)
-    except KeyboardInterrupt:
-        # If a worker process gets an interrupt, abort it immediately.
-        lit.util.abort_now()
-    except:
-        traceback.print_exc()
+    result = _execute_in_parallelism_group(test, _lit_config,
+                                           _parallelism_semaphores)
+    test.setResult(result)
+    return test
+
 
 def _execute_in_parallelism_group(test, lit_config, parallelism_semaphores):
     pg = test.config.parallelism_group
@@ -56,41 +62,16 @@ def _execute_in_parallelism_group(test, lit_config, parallelism_semaphores):
 
 
 def _execute(test, lit_config):
-    """Execute one test"""
     start = time.time()
     result = _execute_test_handle_errors(test, lit_config)
-    end = time.time()
-
-    result.elapsed = end - start
-    resolve_result_code(result, test)
-
+    result.elapsed = time.time() - start
     return result
 
 
-# TODO(yln): is this the right place to deal with this?
-# isExpectedToFail() only works after the test has been executed.
-def resolve_result_code(result, test):
-    try:
-        expected_to_fail = test.isExpectedToFail()
-    except ValueError as e:
-        # Syntax error in an XFAIL line.
-        result.code = lit.Test.UNRESOLVED
-        result.output = str(e)
-    else:
-        if expected_to_fail:
-            # pass -> unexpected pass
-            if result.code is lit.Test.PASS:
-                result.code = lit.Test.XPASS
-            # fail -> expected fail
-            if result.code is lit.Test.FAIL:
-                result.code = lit.Test.XFAIL
-
-
 def _execute_test_handle_errors(test, lit_config):
     try:
-        return _adapt_result(test.config.test_format.execute(test, lit_config))
-    except KeyboardInterrupt:
-        raise
+        result = test.config.test_format.execute(test, lit_config)
+        return _adapt_result(result)
     except:
         if lit_config.debug:
             raise
diff --git a/llvm/utils/lit/setup.py b/llvm/utils/lit/setup.py
index 9d26b2ed71c73..9224ec1467588 100644
--- a/llvm/utils/lit/setup.py
+++ b/llvm/utils/lit/setup.py
@@ -71,7 +71,7 @@
     packages = find_packages(),
     entry_points = {
         'console_scripts': [
-            'lit = lit:main',
+            'lit = lit.main:main',
             ],
         }
 )
diff --git a/llvm/utils/lit/tests/Inputs/shtest-inject/lit.cfg b/llvm/utils/lit/tests/Inputs/shtest-inject/lit.cfg
index 65a02e0081a25..dfec55a3d427d 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-inject/lit.cfg
+++ b/llvm/utils/lit/tests/Inputs/shtest-inject/lit.cfg
@@ -1,17 +1,12 @@
 import lit
 
-class CustomFormat(lit.formats.TestFormat):
-    def execute(self, test, litConfig):
-        commands = [
-            'echo "THIS WAS"',
-            'echo "INJECTED"'
-        ]
-        return lit.TestRunner.executeShTest(test, litConfig,
-                                            useExternalSh=False,
-                                            preamble_commands=commands)
+preamble_commands = [
+    'echo "THIS WAS"',
+    'echo "INJECTED"'
+];
 
 config.name = 'shtest-inject'
 config.suffixes = ['.txt']
-config.test_format = CustomFormat()
+config.test_format = lit.formats.ShTest(preamble_commands=preamble_commands)
 config.test_source_root = None
 config.test_exec_root = None
diff --git a/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/does-not-substitute-no-limit/lit.cfg b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/does-not-substitute-no-limit/lit.cfg
new file mode 100644
index 0000000000000..80c5f88c2ebd8
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/does-not-substitute-no-limit/lit.cfg
@@ -0,0 +1,10 @@
+import lit.formats
+config.name = 'does-not-substitute-no-limit'
+config.suffixes = ['.py']
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
+
+config.substitutions = [("%rec1", "STOP"), ("%rec2", "%rec1"),
+                        ("%rec3", "%rec2"), ("%rec4", "%rec3"),
+                        ("%rec5", "%rec4")]
diff --git a/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/does-not-substitute-no-limit/test.py b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/does-not-substitute-no-limit/test.py
new file mode 100644
index 0000000000000..eb115d6f73cd6
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/does-not-substitute-no-limit/test.py
@@ -0,0 +1 @@
+# RUN: echo %rec5
diff --git a/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/does-not-substitute-within-limit/lit.cfg b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/does-not-substitute-within-limit/lit.cfg
new file mode 100644
index 0000000000000..8fce89741b667
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/does-not-substitute-within-limit/lit.cfg
@@ -0,0 +1,12 @@
+import lit.formats
+config.name = 'does-not-substitute-within-limit'
+config.suffixes = ['.py']
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
+
+config.substitutions = [("%rec1", "STOP"), ("%rec2", "%rec1"),
+                        ("%rec3", "%rec2"), ("%rec4", "%rec3"),
+                        ("%rec5", "%rec4")]
+
+lit_config.recursiveExpansionLimit = 2
diff --git a/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/does-not-substitute-within-limit/test.py b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/does-not-substitute-within-limit/test.py
new file mode 100644
index 0000000000000..eb115d6f73cd6
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/does-not-substitute-within-limit/test.py
@@ -0,0 +1 @@
+# RUN: echo %rec5
diff --git a/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/negative-integer/lit.cfg b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/negative-integer/lit.cfg
new file mode 100644
index 0000000000000..0037b34bb54f1
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/negative-integer/lit.cfg
@@ -0,0 +1,8 @@
+import lit.formats
+config.name = 'negative-integer'
+config.suffixes = ['.py']
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
+
+lit_config.recursiveExpansionLimit = -4
diff --git a/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/negative-integer/test.py b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/negative-integer/test.py
new file mode 100644
index 0000000000000..b80b60b7a2794
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/negative-integer/test.py
@@ -0,0 +1 @@
+# RUN: true
diff --git a/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/not-an-integer/lit.cfg b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/not-an-integer/lit.cfg
new file mode 100644
index 0000000000000..3eb36d8663ddd
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/not-an-integer/lit.cfg
@@ -0,0 +1,8 @@
+import lit.formats
+config.name = 'not-an-integer'
+config.suffixes = ['.py']
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
+
+lit_config.recursiveExpansionLimit = "not-an-integer"
diff --git a/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/not-an-integer/test.py b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/not-an-integer/test.py
new file mode 100644
index 0000000000000..b80b60b7a2794
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/not-an-integer/test.py
@@ -0,0 +1 @@
+# RUN: true
diff --git a/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/set-to-none/lit.cfg b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/set-to-none/lit.cfg
new file mode 100644
index 0000000000000..1dfc57a809441
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/set-to-none/lit.cfg
@@ -0,0 +1,8 @@
+import lit.formats
+config.name = 'set-to-none'
+config.suffixes = ['.py']
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
+
+lit_config.recursiveExpansionLimit = None
diff --git a/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/set-to-none/test.py b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/set-to-none/test.py
new file mode 100644
index 0000000000000..b80b60b7a2794
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/set-to-none/test.py
@@ -0,0 +1 @@
+# RUN: true
diff --git a/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/substitutes-within-limit/lit.cfg b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/substitutes-within-limit/lit.cfg
new file mode 100644
index 0000000000000..aa66c0e856eb3
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/substitutes-within-limit/lit.cfg
@@ -0,0 +1,12 @@
+import lit.formats
+config.name = 'substitutes-within-limit'
+config.suffixes = ['.py']
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
+
+config.substitutions = [("%rec1", "STOP"), ("%rec2", "%rec1"),
+                        ("%rec3", "%rec2"), ("%rec4", "%rec3"),
+                        ("%rec5", "%rec4")]
+
+lit_config.recursiveExpansionLimit = 5
diff --git a/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/substitutes-within-limit/test.py b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/substitutes-within-limit/test.py
new file mode 100644
index 0000000000000..eb115d6f73cd6
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-recursive-substitution/substitutes-within-limit/test.py
@@ -0,0 +1 @@
+# RUN: echo %rec5
diff --git a/llvm/utils/lit/tests/shtest-inject.py b/llvm/utils/lit/tests/shtest-inject.py
index f51f083f3990c..9f9ff60979215 100644
--- a/llvm/utils/lit/tests/shtest-inject.py
+++ b/llvm/utils/lit/tests/shtest-inject.py
@@ -1,5 +1,4 @@
-# Check that we can inject commands at the beginning of a ShTest using a custom
-# test format.
+# Check that we can inject commands at the beginning of a ShTest.
 
 # RUN: %{lit} -j 1 %{inputs}/shtest-inject/test-empty.txt --show-all | FileCheck --check-prefix=CHECK-TEST1 %s
 #
diff --git a/llvm/utils/lit/tests/shtest-recursive-substitution.py b/llvm/utils/lit/tests/shtest-recursive-substitution.py
new file mode 100644
index 0000000000000..1b3d5f4c44519
--- /dev/null
+++ b/llvm/utils/lit/tests/shtest-recursive-substitution.py
@@ -0,0 +1,23 @@
+# Check that the config.recursiveExpansionLimit is picked up and will cause
+# lit substitutions to be expanded recursively.
+
+# RUN: %{lit} -j 1 %{inputs}/shtest-recursive-substitution/substitutes-within-limit --show-all | FileCheck --check-prefix=CHECK-TEST1 %s
+# CHECK-TEST1: PASS: substitutes-within-limit :: test.py
+# CHECK-TEST1: $ "echo" "STOP"
+
+# RUN: not %{lit} -j 1 %{inputs}/shtest-recursive-substitution/does-not-substitute-within-limit --show-all | FileCheck --check-prefix=CHECK-TEST2 %s
+# CHECK-TEST2: UNRESOLVED: does-not-substitute-within-limit :: test.py
+# CHECK-TEST2: ValueError: Recursive substitution of
+
+# RUN: %{lit} -j 1 %{inputs}/shtest-recursive-substitution/does-not-substitute-no-limit --show-all | FileCheck --check-prefix=CHECK-TEST3 %s
+# CHECK-TEST3: PASS: does-not-substitute-no-limit :: test.py
+# CHECK-TEST3: $ "echo" "%rec4"
+
+# RUN: not %{lit} -j 1 %{inputs}/shtest-recursive-substitution/not-an-integer --show-all 2>&1 | FileCheck --check-prefix=CHECK-TEST4 %s
+# CHECK-TEST4: recursiveExpansionLimit must be either None or an integer
+
+# RUN: not %{lit} -j 1 %{inputs}/shtest-recursive-substitution/negative-integer --show-all 2>&1 | FileCheck --check-prefix=CHECK-TEST5 %s
+# CHECK-TEST5: recursiveExpansionLimit must be a non-negative integer
+
+# RUN: %{lit} -j 1 %{inputs}/shtest-recursive-substitution/set-to-none --show-all | FileCheck --check-prefix=CHECK-TEST6 %s
+# CHECK-TEST6: PASS: set-to-none :: test.py
diff --git a/llvm/utils/lit/tests/unit/TestRunner.py b/llvm/utils/lit/tests/unit/TestRunner.py
index 4f33fce648850..5b5703c2e17da 100644
--- a/llvm/utils/lit/tests/unit/TestRunner.py
+++ b/llvm/utils/lit/tests/unit/TestRunner.py
@@ -199,6 +199,74 @@ def custom_parse(line_number, line, output):
         except BaseException as e:
             self.fail("CUSTOM_NO_PARSER: raised the wrong exception: %r" % e)
 
+class TestApplySubtitutions(unittest.TestCase):
+    def test_simple(self):
+        script = ["echo %bar"]
+        substitutions = [("%bar", "hello")]
+        result = lit.TestRunner.applySubstitutions(script, substitutions)
+        self.assertEqual(result, ["echo hello"])
+
+    def test_multiple_substitutions(self):
+        script = ["echo %bar %baz"]
+        substitutions = [("%bar", "hello"),
+                         ("%baz", "world"),
+                         ("%useless", "shouldnt expand")]
+        result = lit.TestRunner.applySubstitutions(script, substitutions)
+        self.assertEqual(result, ["echo hello world"])
+
+    def test_multiple_script_lines(self):
+        script = ["%cxx %compile_flags -c -o %t.o",
+                  "%cxx %link_flags %t.o -o %t.exe"]
+        substitutions = [("%cxx", "clang++"),
+                         ("%compile_flags", "-std=c++11 -O3"),
+                         ("%link_flags", "-lc++")]
+        result = lit.TestRunner.applySubstitutions(script, substitutions)
+        self.assertEqual(result, ["clang++ -std=c++11 -O3 -c -o %t.o",
+                                  "clang++ -lc++ %t.o -o %t.exe"])
+
+    def test_recursive_substitution_real(self):
+        script = ["%build %s"]
+        substitutions = [("%cxx", "clang++"),
+                         ("%compile_flags", "-std=c++11 -O3"),
+                         ("%link_flags", "-lc++"),
+                         ("%build", "%cxx %compile_flags %link_flags %s -o %t.exe")]
+        result = lit.TestRunner.applySubstitutions(script, substitutions, recursion_limit=3)
+        self.assertEqual(result, ["clang++ -std=c++11 -O3 -lc++ %s -o %t.exe %s"])
+
+    def test_recursive_substitution_limit(self):
+        script = ["%rec5"]
+        # Make sure the substitutions are not in an order where the global
+        # substitution would appear to be recursive just because they are
+        # processed in the right order.
+        substitutions = [("%rec1", "STOP"), ("%rec2", "%rec1"),
+                         ("%rec3", "%rec2"), ("%rec4", "%rec3"), ("%rec5", "%rec4")]
+        for limit in [5, 6, 7]:
+            result = lit.TestRunner.applySubstitutions(script, substitutions, recursion_limit=limit)
+            self.assertEqual(result, ["STOP"])
+
+    def test_recursive_substitution_limit_exceeded(self):
+        script = ["%rec5"]
+        substitutions = [("%rec1", "STOP"), ("%rec2", "%rec1"),
+                         ("%rec3", "%rec2"), ("%rec4", "%rec3"), ("%rec5", "%rec4")]
+        for limit in [0, 1, 2, 3, 4]:
+            try:
+                lit.TestRunner.applySubstitutions(script, substitutions, recursion_limit=limit)
+                self.fail("applySubstitutions should have raised an exception")
+            except ValueError:
+                pass
+
+    def test_recursive_substitution_invalid_value(self):
+        script = ["%rec5"]
+        substitutions = [("%rec1", "STOP"), ("%rec2", "%rec1"),
+                         ("%rec3", "%rec2"), ("%rec4", "%rec3"), ("%rec5", "%rec4")]
+        for limit in [-1, -2, -3, "foo"]:
+            try:
+                lit.TestRunner.applySubstitutions(script, substitutions, recursion_limit=limit)
+                self.fail("applySubstitutions should have raised an exception")
+            except AssertionError:
+                pass
+
+
 if __name__ == '__main__':
     TestIntegratedTestKeywordParser.load_keyword_parser_lit_tests()
     unittest.main(verbosity=2)
diff --git a/llvm/utils/lit/tests/usage.py b/llvm/utils/lit/tests/usage.py
index d168c5eff9e3c..77b3573c53750 100644
--- a/llvm/utils/lit/tests/usage.py
+++ b/llvm/utils/lit/tests/usage.py
@@ -1,6 +1,7 @@
-# Basic sanity check that usage works.
+# Basic sanity check for `--help` and `--version` options.
 #
-# RUN: %{lit} --help > %t.out
-# RUN: FileCheck < %t.out %s
+# RUN: %{lit} --help         | FileCheck %s --check-prefix=HELP
+# RUN: %{lit} --version 2>&1 | FileCheck %s --check-prefix=VERSION
 #
-# CHECK: usage: lit.py [-h]
+# HELP: usage: lit [-h]
+# VERSION: lit {{[0-9]+\.[0-9]+\.[0-9]+[a-zA-Z0-9]*}}
diff --git a/llvm/utils/lit/tests/version.py b/llvm/utils/lit/tests/version.py
deleted file mode 100644
index 1d5e152ddd3b2..0000000000000
--- a/llvm/utils/lit/tests/version.py
+++ /dev/null
@@ -1,5 +0,0 @@
-# Basic sanity check that --version works.
-#
-# RUN: %{lit} --version 2>&1 | FileCheck %s
-#
-# CHECK: lit {{[0-9]+\.[0-9]+\.[0-9]+[a-zA-Z0-9]*}}
diff --git a/llvm/utils/llvm-locstats/llvm-locstats.py b/llvm/utils/llvm-locstats/llvm-locstats.py
index 7b2f706fca943..dec87f9caf7d3 100755
--- a/llvm/utils/llvm-locstats/llvm-locstats.py
+++ b/llvm/utils/llvm-locstats/llvm-locstats.py
@@ -121,14 +121,16 @@ def draw_location_diff(self, locstats_to_compare):
     ax = fig.add_subplot(111)
     init_plot(plt)
 
+    comparison_keys = list(coverage_buckets())
     ax.bar(buckets, self.variables_coverage_map.values(), align='edge',
-           tick_label=self.variables_coverage_map.keys(), width=0.4,
+           width=0.4,
            label='variables of {}'.format(self.file_name))
     ax.bar(buckets_to_compare,
            locstats_to_compare.variables_coverage_map.values(),
            color='r', align='edge', width=-0.4,
-           tick_label=locstats_to_compare.variables_coverage_map.keys(),
            label='variables of {}'.format(locstats_to_compare.file_name))
+    ax.set_xticks(range(len(comparison_keys)))
+    ax.set_xticklabels(comparison_keys)
 
     props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)
     plt.text(0.02, 0.88,
diff --git a/mlir/cmake/modules/AddMLIR.cmake b/mlir/cmake/modules/AddMLIR.cmake
index 2adb8f2f29351..7449f54ea8771 100644
--- a/mlir/cmake/modules/AddMLIR.cmake
+++ b/mlir/cmake/modules/AddMLIR.cmake
@@ -1,5 +1,5 @@
 function(mlir_tablegen ofn)
-  tablegen(MLIR ${ARGV} "-I${MLIR_MAIN_SRC_DIR}" "-I${MLIR_INCLUDE_DIR}")
+  tablegen(MLIR ${ARGV})
   set(TABLEGEN_OUTPUT ${TABLEGEN_OUTPUT} ${CMAKE_CURRENT_BINARY_DIR}/${ofn}
       PARENT_SCOPE)
 endfunction()
diff --git a/mlir/docs/ConversionToLLVMDialect.md b/mlir/docs/ConversionToLLVMDialect.md
index f1917e8ac7069..15d09cb4b87a6 100644
--- a/mlir/docs/ConversionToLLVMDialect.md
+++ b/mlir/docs/ConversionToLLVMDialect.md
@@ -374,12 +374,14 @@ overridden by the user.*
 
 ### C-compatible wrapper emission
 
-In practical cases, it may be desirable to have externally-facing functions
-with a single attribute corresponding to a MemRef argument. When interfacing
-with LLVM IR produced from C, the code needs to respect the corresponding
-calling convention. The conversion to the LLVM dialect provides an option to
-generate wrapper functions that take memref descriptors as pointers-to-struct
-compatible with data types produced by Clang when compiling C sources.
+In practical cases, it may be desirable to have externally-facing functions with
+a single attribute corresponding to a MemRef argument. When interfacing with
+LLVM IR produced from C, the code needs to respect the corresponding calling
+convention. The conversion to the LLVM dialect provides an option to generate
+wrapper functions that take memref descriptors as pointers-to-struct compatible
+with data types produced by Clang when compiling C sources. The generation of
+such wrapper functions can additionally be controlled at a function granularity
+by setting the `llvm.emit_c_interface` unit attribute.
 
 More specifically, a memref argument is converted into a pointer-to-struct
 argument of type `{T*, T*, i64, i64[N], i64[N]}*` in the wrapper function, where
@@ -394,7 +396,7 @@ struct MemRefDescriptor {
   T *aligned;
   intptr_t offset;
   intptr_t sizes[N];
-  intptr_t stides[N];
+  intptr_t strides[N];
 };
 ```
 
diff --git a/mlir/docs/CreatingADialect.md b/mlir/docs/CreatingADialect.md
index e8a8d40ff0965..a1ecb6770aa96 100644
--- a/mlir/docs/CreatingADialect.md
+++ b/mlir/docs/CreatingADialect.md
@@ -126,7 +126,7 @@ mlir/lib/Conversion/GPUCommon.
 Each conversion typically exists in a separate library, declared with
 add_mlir_conversion_library().  Conversion libraries typically depend
 on their source and target dialects, but may also depend on other
-dialects (e.g. MLIRStandard).  Typically this dependence is specifid
+dialects (e.g. MLIRStandard).  Typically this dependence is specified
 using target_link_libraries() and the PUBLIC keyword.  For instance:
 
 ```cmake
diff --git a/mlir/docs/DeclarativeRewrites.md b/mlir/docs/DeclarativeRewrites.md
index 0faae5a726f85..82b97e04a8c0a 100644
--- a/mlir/docs/DeclarativeRewrites.md
+++ b/mlir/docs/DeclarativeRewrites.md
@@ -407,7 +407,7 @@ def : Pat<(OneAttrOp $attr),
 ```
 
 In the above, `$_self` is substituted by the attribute bound by `$attr`, which
-is `OnAttrOp`'s array attribute.
+is `OneAttrOp`'s array attribute.
 
 Positional placeholders will be substituted by the `dag` object parameters at
 the `NativeCodeCall` use site. For example, if we define `SomeCall :
@@ -469,7 +469,7 @@ value. Instead we can use multiple result patterns:
 
 ```tablegen
 def : Pattern<(AddIOp $lhs, $rhs),
-              [(StoreOp (AllocOp:$mem (ShapeOp %lhs)), (AddIOp $lhs, $rhs)),
+              [(StoreOp (AllocOp:$mem (ShapeOp $lhs)), (AddIOp $lhs, $rhs)),
                (LoadOp $mem)];
 ```
 
diff --git a/mlir/docs/Diagnostics.md b/mlir/docs/Diagnostics.md
index c60dffb022e01..d33c35dd3e4be 100644
--- a/mlir/docs/Diagnostics.md
+++ b/mlir/docs/Diagnostics.md
@@ -93,8 +93,8 @@ DiagnosticEngine engine = ctx->getDiagEngine();
 // or failure if the diagnostic should be propagated to the previous handlers.
 DiagnosticEngine::HandlerID id = engine.registerHandler(
     [](Diagnostic &diag) -> LogicalResult {
-  bool should_propage_diagnostic = ...;
-  return failure(should_propage_diagnostic);
+  bool should_propagate_diagnostic = ...;
+  return failure(should_propagate_diagnostic);
 });
 
 
diff --git a/mlir/docs/Dialects/Affine.md b/mlir/docs/Dialects/Affine.md
index 4a7d5c30eda76..4e4c7edf0e285 100644
--- a/mlir/docs/Dialects/Affine.md
+++ b/mlir/docs/Dialects/Affine.md
@@ -1,4 +1,4 @@
-# Affine Dialect
+# 'affine' Dialect
 
 This dialect provides a powerful abstraction for affine operations and analyses.
 
@@ -295,140 +295,9 @@ affine.if #set42(%i, %j)[%M, %N] {
 
 ## Operations
 
-#### 'affine.apply' operation
+[include "Dialects/AffineOps.md"]
 
-Syntax:
-
-```
-operation ::= ssa-id `=` `affine.apply` affine-map-attribute dim-and-symbol-use-list
-```
-
-The `affine.apply` operation applies an
-[affine mapping](#affine-expressions) to a list of SSA values,
-yielding a single SSA value. The number of dimension and symbol arguments to
-affine.apply must be equal to the respective number of dimensional and symbolic
-inputs to the affine mapping; the `affine.apply` operation always returns one
-value. The input operands and result must all have 'index' type.
-
-Example:
-
-```mlir
-#map10 = affine_map<(d0, d1) -> (d0 floordiv 8 + d1 floordiv 128)>
-...
-%1 = affine.apply #map10 (%s, %t)
-
-// Inline example.
-%2 = affine.apply affine_map<(i)[s0] -> (i+s0)> (%42)[%n]
-```
-
-#### 'affine.for' operation
-
-Syntax:
-
-```
-operation   ::= `affine.for` ssa-id `=` lower-bound `to` upper-bound
-                      (`step` integer-literal)? `{` op* `}`
-
-lower-bound ::= `max`? affine-map-attribute dim-and-symbol-use-list | shorthand-bound
-upper-bound ::= `min`? affine-map-attribute dim-and-symbol-use-list | shorthand-bound
-shorthand-bound ::= ssa-id | `-`? integer-literal
-```
-
-The `affine.for` operation represents an affine loop nest. It has one region
-containing its body. This region must contain one block that terminates with
-[`affine.terminator`](#affineterminator-operation). *Note:* when `affine.for` is
-printed in custom format, the terminator is omitted. The block has one argument
-of [`index`](../LangRef.md#index-type) type that represents the induction
-variable of the loop.
-
-The `affine.for` operation executes its body a number of times iterating from a
-lower bound to an upper bound by a stride. The stride, represented by `step`, is
-a positive constant integer which defaults to "1" if not present. The lower and
-upper bounds specify a half-open range: the range includes the lower bound but
-does not include the upper bound.
-
-The lower and upper bounds of a `affine.for` operation are represented as an
-application of an affine mapping to a list of SSA values passed to the map. The
-[same restrictions](#restrictions-on-dimensions-and-symbols) hold for these SSA
-values as for all bindings of SSA values to dimensions and symbols.
-
-The affine mappings for the bounds may return multiple results, in which case
-the `max`/`min` keywords are required (for the lower/upper bound respectively),
-and the bound is the maximum/minimum of the returned values. There is no
-semantic ambiguity, but MLIR syntax requires the use of these keywords to make
-things more obvious to human readers.
-
-Many upper and lower bounds are simple, so MLIR accepts two custom form
-syntaxes: the form that accepts a single 'ssa-id' (e.g. `%N`) is shorthand for
-applying that SSA value to a function that maps a single symbol to itself, e.g.,
-`()[s]->(s)()[%N]`. The integer literal form (e.g. `-42`) is shorthand for a
-nullary mapping function that returns the constant value (e.g. `()->(-42)()`).
-
-Example showing reverse iteration of the inner loop:
-
-```mlir
-#map57 = affine_map<(d0)[s0] -> (s0 - d0 - 1)>
-
-func @simple_example(%A: memref<?x?xf32>, %B: memref<?x?xf32>) {
-  %N = dim %A, 0 : memref<?x?xf32>
-  affine.for %i = 0 to %N step 1 {
-    affine.for %j = 0 to %N {   // implicitly steps by 1
-      %0 = affine.apply #map57(%j)[%N]
-      %tmp = call @F1(%A, %i, %0) : (memref<?x?xf32>, index, index)->(f32)
-      call @F2(%tmp, %B, %i, %0) : (f32, memref<?x?xf32>, index, index)->()
-    }
-  }
-  return
-}
-```
-
-#### 'affine.if' operation
-
-Syntax:
-
-```
-operation    ::= `affine.if` if-op-cond `{` op* `}` (`else` `{` op* `}`)?
-if-op-cond ::= integer-set-attr dim-and-symbol-use-list
-```
-
-The `affine.if` operation restricts execution to a subset of the loop iteration
-space defined by an integer set (a conjunction of affine constraints). A single
-`affine.if` may end with an optional `else` clause.
-
-The condition of the `affine.if` is represented by an
-[integer set](#integer-sets) (a conjunction of affine constraints),
-and the SSA values bound to the dimensions and symbols in the integer set. The
-[same restrictions](#restrictions-on-dimensions-and-symbols) hold for these SSA
-values as for all bindings of SSA values to dimensions and symbols.
-
-The `affine.if` operation contains two regions for the "then" and "else"
-clauses. The latter may be empty (i.e. contain no blocks), meaning the absence
-of the else clause. When non-empty, both regions must contain exactly one block
-terminating with [`affine.terminator`](#affineterminator-operation). *Note:*
-when `affine.if` is printed in custom format, the terminator is omitted. These
-blocks must not have any arguments.
-
-Example:
-
-```mlir
-#set = affine_set<(d0, d1)[s0]: (d0 - 10 >= 0, s0 - d0 - 9 >= 0,
-                                 d1 - 10 >= 0, s0 - d1 - 9 >= 0)>
-func @reduced_domain_example(%A, %X, %N) : (memref<10xi32>, i32, i32) {
-  affine.for %i = 0 to %N {
-     affine.for %j = 0 to %N {
-       %0 = affine.apply #map42(%j)
-       %tmp = call @S1(%X, %i, %0)
-       affine.if #set(%i, %j)[%N] {
-          %1 = affine.apply #map43(%i, %j)
-          call @S2(%tmp, %A, %i, %1)
-       }
-    }
-  }
-  return
-}
-```
-
-#### 'affine.load' operation
+### 'affine.load' operation
 
 Syntax:
 
@@ -458,7 +327,7 @@ Example:
 
 ```
 
-#### 'affine.store' operation
+### 'affine.store' operation
 
 Syntax:
 
@@ -488,7 +357,7 @@ Example:
 
 ```
 
-#### 'affine.dma_start' operation
+### 'affine.dma_start' operation
 
 Syntax:
 
@@ -519,7 +388,6 @@ specified. The value of 'num_elements' must be a multiple of
 Example:
 
 ```mlir
-
 For example, a DmaStartOp operation that transfers 256 elements of a memref
 '%src' in memory space 0 at indices [%i + 3, %j] to memref '%dst' in memory
 space 1 at indices [%k + 7, %l], would be specified as follows:
@@ -537,10 +405,9 @@ space 1 at indices [%k + 7, %l], would be specified as follows:
 
   affine.dma_start %src[%i, %j], %dst[%k, %l], %tag[%idx], %num_elements,
     %stride, %num_elt_per_stride : ...
-
 ```
 
-#### 'affine.dma_wait' operation
+### 'affine.dma_wait' operation
 
 Syntax:
 
@@ -558,54 +425,9 @@ associated with the DMA operation. For example:
 Example:
 
 ```mlir
-
-  affine.dma_start %src[%i, %j], %dst[%k, %l], %tag[%index], %num_elements :
-    memref<2048xf32, 0>, memref<256xf32, 1>, memref<1xi32, 2>
-  ...
-  ...
-  affine.dma_wait %tag[%index], %num_elements : memref<1xi32, 2>
-
-```
-
-#### 'affine.min' operation
-
-Syntax:
-
-```
-operation ::= ssa-id `=` `affine.min` affine-map-attribute dim-and-symbol-use-list
-```
-
-The `affine.min` operation applies an
-[affine mapping](#affine-expressions) to a list of SSA values, and returns the
-minimum value of all result expressions. The number of dimension and symbol
-arguments to affine.min must be equal to the respective number of dimensional
-and symbolic inputs to the affine mapping; the `affine.min` operation always
-returns one value. The input operands and result must all have 'index' type.
-
-Example:
-
-```mlir
-
-%0 = affine.min affine_map<(d0)[s0] -> (1000, d0 + 512, s0)> (%arg0)[%arg1]
-
-```
-
-#### `affine.terminator` operation
-
-Syntax:
-
-```
-operation ::= `"affine.terminator"() : () -> ()`
+affine.dma_start %src[%i, %j], %dst[%k, %l], %tag[%index], %num_elements :
+  memref<2048xf32, 0>, memref<256xf32, 1>, memref<1xi32, 2>
+...
+...
+affine.dma_wait %tag[%index], %num_elements : memref<1xi32, 2>
 ```
-
-Affine terminator is a special terminator operation for blocks inside affine
-loops ([`affine.for`](#affinefor-operation)) and branches
-([`affine.if`](#affineif-operation)). It unconditionally transmits the control
-flow to the successor of the operation enclosing the region.
-
-*Rationale*: bodies of affine operations are [blocks](../LangRef.md#blocks) that
-must have terminators. Loops and branches represent structured control flow and
-should not accept arbitrary branches as terminators.
-
-This operation does _not_ have a custom syntax. However, affine control
-operations omit the terminator in their custom syntax for brevity.
diff --git a/mlir/docs/Dialects/GPU.md b/mlir/docs/Dialects/GPU.md
index 7dcd8f6053c42..fc2391dd1ed92 100644
--- a/mlir/docs/Dialects/GPU.md
+++ b/mlir/docs/Dialects/GPU.md
@@ -1,4 +1,4 @@
-# GPU Dialect
+# 'gpu' Dialect
 
 Note: this dialect is more likely to change than others in the near future; use
 with caution.
@@ -36,97 +36,4 @@ structure and representing analysis results in the IR.
 
 ## Operations
 
-### `gpu.block_dim`
-
-Returns the number of threads in the thread block (aka the block size) along the
-x, y, or z `dimension`.
-
-Example:
-
-```mlir
-  %bDimX = "gpu.block_dim"() {dimension = "x"} : () -> (index)
-```
-
-### `gpu.block_id`
-
-Returns the block id, i.e. the index of the current block within the grid along
-the x, y, or z `dimension`.
-
-Example:
-
-```mlir
-  %bIdY = "gpu.block_id"() {dimension = "y"} : () -> (index)
-```
-
-### `gpu.grid_dim`
-
-Returns the number of thread blocks in the grid along the x, y, or z
-`dimension`.
-
-Example:
-
-```mlir
-  %gDimZ = "gpu.grid_dim"() {dimension = "z"} : () -> (index)
-```
-
-### `gpu.thread_id`
-
-Returns the thread id, i.e. the index of the current thread within the block
-along the x, y, or z `dimension`.
-
-Example:
-
-```mlir
-  %tIdX = "gpu.thread_id"() {dimension = "x"} : () -> (index)
-```
-
-### `gpu.yield`
-
-Is a special terminator operation for blocks inside regions in gpu ops. It
-returns values to the immediately enclosing gpu op.
-
-Example:
-
-```mlir
-gpu.yield %f0, %f1 : f32, f32
-```
-
-### `gpu.all_reduce`
-
-The "all_reduce" op reduces the value of every work item across a local
-workgroup. The result is equal for all work items of a workgroup.
-
-For example, both
-
-```mlir
-%1 = "gpu.all_reduce"(%0) ({}) { op = "add" } : (f32) -> (f32)
-%2 = "gpu.all_reduce"(%0) ({
-^bb(%lhs : f32, %rhs : f32):
-  %sum = addf %lhs, %rhs : f32
-  "gpu.yield"(%sum) : (f32) -> ()
-}) : (f32) -> (f32)
-```
-
-compute the sum of each work item's %0 value. The first version specifies the
-accumulation as operation, whereas the second version specifies the accumulation
-as code region. The accumulation operation must either be `add` or `mul`.
-
-Either none or all work items of a workgroup need to execute this op
-in convergence.
-
-### `gpu.barrier`
-
-The "barrier" op synchronizes all work items of a workgroup. It is used
-to coordinate communication between the work items of the workgroup.
-
-```mlir
-gpu.barrier
-```
-
-waits until all work items in the workgroup have reached this point and all
-memory accesses made by these work items prior to the op are visible to all work
-items in the workgroup. Data hazards between work items accessing the same
-memory can be avoided by synchronizing work items in-between these accesses.
-
-Either none or all work items of a workgroup need to execute this op
-in convergence.
+[include "Dialects/GPUOps.md"]
diff --git a/mlir/docs/Dialects/LLVM.md b/mlir/docs/Dialects/LLVM.md
index 00d0fa02fece6..e512675565fd0 100644
--- a/mlir/docs/Dialects/LLVM.md
+++ b/mlir/docs/Dialects/LLVM.md
@@ -1,4 +1,4 @@
-# LLVM IR Dialect
+# 'llvm' Dialect
 
 This dialect wraps the LLVM IR types and instructions into MLIR types and
 operations. It provides several additional operations that are necessary to
@@ -79,6 +79,32 @@ llvm.func internal @internal_func() {
 
 ```
 
+#### Attribute pass-through
+
+An LLVM IR dialect function provides a mechanism to forward function-level
+attributes to LLVM IR using the `passthrough` attribute. This is an array
+attribute containing either string attributes or array attributes. In the former
+case, the value of the string is interpreted as the name of LLVM IR function
+attribute. In the latter case, the array is expected to contain exactly two
+string attributes, the first corresponding to the name of LLVM IR function
+attribute, and the second corresponding to its value. Note that even integer
+LLVM IR function attributes have their value represented in the string form.
+
+Example:
+
+```mlir
+llvm.func @func() attributes {
+  passthrough = ["noinline",           // value-less attribute
+                 ["alignstack", "4"],  // integer attribute with value
+                 ["other", "attr"]]    // attrbute unknown to LLVM
+} {
+  llvm.return
+}
+```
+
+If the attribute is not known to LLVM IR, it will be attached as a string
+attribute.
+
 ### LLVM IR operations
 
 The following operations are currently supported. The semantics of these
diff --git a/mlir/docs/Dialects/Linalg.md b/mlir/docs/Dialects/Linalg.md
index 6255f23ae4b80..878ce8f11523c 100644
--- a/mlir/docs/Dialects/Linalg.md
+++ b/mlir/docs/Dialects/Linalg.md
@@ -1,4 +1,4 @@
-# Linalg Dialect
+# 'linalg' Dialect
 
 [TOC]
 
@@ -29,7 +29,7 @@ performed on the Linalg IR and that have influenced its design:
 1. Tiled Producer-Consumer Fusion with Parametric Tile-And-Fuse.
 1. Map to Parallel and Reduction Loops and Hardware.
 1. Vectorization: Rewrite in Vector Form.
-1. Lower to Loops (Affine and/or Generic).
+1. Lower to Loops (Affine, Generic and Parallel).
 1. Lower to Library Calls or Special Instructions, Intrinsics or ISA.
 1. Partially Lower to Iterations Over a Finer-Grained Linalg Op.
 
@@ -241,7 +241,7 @@ example:
   (i, j) -> (i, j),
   (i, j) -> (i, j)
 }
-#attrs = {args_in: 1, args_out: 1, indexings: #indexing_maps}
+#attrs = {args_in: 2, args_out: 1, indexings: #indexing_maps}
 func @example(%A: memref<?x?xf32>, %B: memref<?x?xf32>, %C: memref<?x?xf32>) {
   linalg.generic #attrs (%A, %B, %C) {
     ^bb0(%a: f32, %b: f32):
@@ -295,7 +295,7 @@ example:
   (i, j) -> (i, j),
   (i, j) -> (i, j)
 }
-#attrs = {args_in: 1, args_out: 1, indexings: #indexing_maps, fun: #fun_attr}
+#attrs = {args_in: 2, args_out: 1, indexings: #indexing_maps, fun: #fun_attr}
 func @example(%A: memref<?x?xf32>, %B: memref<?x?xf32>, %C: memref<?x?xf32>) {
   linalg.generic #attrs (%A, %B, %C) {
     ^bb0(%a: f32, %b: f32):
@@ -469,3 +469,7 @@ These key questions (and much more) should be really thought of in the general
 context of MLIR in which different levels of IR interoperate seamlessly. In
 practice, it is not necessary (or beneficial) to try and solve all problems in the
 same IR.
+
+## Operations
+
+[include "Dialects/LinalgOps.md"]
diff --git a/mlir/docs/Dialects/SPIR-V.md b/mlir/docs/Dialects/SPIR-V.md
index e1e772b43a71a..acf56f137e16e 100644
--- a/mlir/docs/Dialects/SPIR-V.md
+++ b/mlir/docs/Dialects/SPIR-V.md
@@ -1,4 +1,4 @@
-# SPIR-V Dialect
+# 'spv' Dialect
 
 This document describes the design of the SPIR-V dialect in MLIR. It lists
 various design choices we made for modeling different SPIR-V mechanisms, and
@@ -15,6 +15,8 @@ Vulkan and OpenCL. It is fully defined in a
 [human-readable specification][SpirvSpec]; the syntax of various SPIR-V
 instructions are encoded in a [machine-readable grammar][SpirvGrammar].
 
+[TOC]
+
 ## Design Guidelines
 
 SPIR-V is a binary intermediate language that serves dual purpose: on one side,
@@ -459,8 +461,9 @@ can be represented in the dialect as
 ```
 
 Operation documentation is written in each op's Op Definition Spec using
-TableGen. A markdown version of the doc can be found at
-[mlir.llvm.org][LlvmMlirSpirvDoc] or generated using `mlir-tblgen -gen-doc`.
+TableGen. A markdown version of the doc can be generated using
+`mlir-tblgen -gen-doc` and is attached in the
+[Operation definitions](#operation-definitions) section.
 
 ### Ops from extended instruction sets
 
@@ -1224,6 +1227,10 @@ conversion][MlirDialectConversionSignatureConversion] might be needed as well.
 operations contained within its region are valid operations in the SPIR-V
 dialect.
 
+## Operation definitions
+
+[include "Dialects/SPIRVOps.md"]
+
 [Spirv]: https://www.khronos.org/registry/spir-v/
 [SpirvSpec]: https://www.khronos.org/registry/spir-v/specs/unified1/SPIRV.html
 [SpirvLogicalLayout]: https://www.khronos.org/registry/spir-v/specs/unified1/SPIRV.html#_a_id_logicallayout_a_logical_layout_of_a_module
@@ -1270,7 +1277,6 @@ dialect.
 [GitHubDialectTracking]: https://github.com/tensorflow/mlir/issues/302
 [GitHubLoweringTracking]: https://github.com/tensorflow/mlir/issues/303
 [GenSpirvUtilsPy]: https://github.com/llvm/llvm-project/blob/master/mlir/utils/spirv/gen_spirv_dialect.py
-[LlvmMlirSpirvDoc]: ../Dialects/SPIRVOps/
 [CustomTypeAttrTutorial]: ../DefiningAttributesAndTypes/
 [VulkanSpirv]: https://renderdoc.org/vkspec_chunked/chap40.html#spirvenv
 [VulkanShaderInterface]: https://renderdoc.org/vkspec_chunked/chap14.html#interfaces-resources
diff --git a/mlir/docs/Dialects/Standard.md b/mlir/docs/Dialects/Standard.md
index e956b649ac628..a0f1f0ffe134c 100644
--- a/mlir/docs/Dialects/Standard.md
+++ b/mlir/docs/Dialects/Standard.md
@@ -1,4 +1,4 @@
-# Standard Dialect
+# 'std' Dialect
 
 This dialect provides documentation for operations within the Standard dialect.
 
@@ -7,221 +7,9 @@ and should be split into multiple more-focused dialects accordingly.
 
 [TOC]
 
-TODO: shape, which returns a 1D tensor, and can take an unknown rank tensor as
-input.
+## Operations
 
-TODO: rank, which returns an index.
-
-## Terminator operations
-
-Terminator operations are required at the end of each block. They may contain a
-list of successors, i.e. other blocks to which the control flow will proceed.
-
-### 'br' terminator operation
-
-Syntax:
-
-```
-operation ::= `br` successor
-successor ::= bb-id branch-use-list?
-branch-use-list ::= `(` ssa-use-list `:` type-list-no-parens `)`
-```
-
-The `br` terminator operation represents an unconditional jump to a target
-block. The count and types of operands to the branch must align with the
-arguments in the target block.
-
-The MLIR branch operation is not allowed to target the entry block for a region.
-
-### 'cond_br' terminator operation
-
-Syntax:
-
-```
-operation ::= `cond_br` ssa-use `,` successor `,` successor
-```
-
-The `cond_br` terminator operation represents a conditional branch on a boolean
-(1-bit integer) value. If the bit is set, then the first destination is jumped
-to; if it is false, the second destination is chosen. The count and types of
-operands must align with the arguments in the corresponding target blocks.
-
-The MLIR conditional branch operation is not allowed to target the entry block
-for a region. The two destinations of the conditional branch operation are
-allowed to be the same.
-
-The following example illustrates a function with a conditional branch operation
-that targets the same block:
-
-```mlir
-func @select(i32, i32, i1) -> i32 {
-^bb0(%a : i32, %b :i32, %flag : i1) :
-    // Both targets are the same, operands differ
-    cond_br %flag, ^bb1(%a : i32), ^bb1(%b : i32)
-
-^bb1(%x : i32) :
-    return %x : i32
-}
-```
-
-### 'return' terminator operation
-
-Syntax:
-
-```
-operation ::= `return` (ssa-use-list `:` type-list-no-parens)?
-```
-
-The `return` terminator operation represents the completion of a function, and
-produces the result values. The count and types of the operands must match the
-result types of the enclosing function. It is legal for multiple blocks in a
-single function to return.
-
-## Core Operations
-
-### 'call' operation
-
-Syntax:
-
-```
-operation ::=
-    (ssa-id `=`)? `call` symbol-ref-id `(` ssa-use-list? `)` `:` function-type
-```
-
-The `call` operation represents a direct call to a function. The operands and
-result types of the call must match the specified function type. The callee is
-encoded as a function attribute named "callee".
-
-Example:
-
-```mlir
-// Calling the function my_add.
-%31 = call @my_add(%0, %1) : (tensor<16xf32>, tensor<16xf32>) -> tensor<16xf32>
-```
-
-### 'call_indirect' operation
-
-Syntax:
-
-```
-operation ::= `call_indirect` ssa-use `(` ssa-use-list? `)` `:` function-type
-```
-
-The `call_indirect` operation represents an indirect call to a value of function
-type. Functions are first class types in MLIR, and may be passed as arguments
-and merged together with block arguments. The operands and result types of the
-call must match the specified function type.
-
-Function values can be created with the
-[`constant` operation](#constant-operation).
-
-Example:
-
-```mlir
-%31 = call_indirect %15(%0, %1)
-        : (tensor<16xf32>, tensor<16xf32>) -> tensor<16xf32>
-```
-
-### 'dim' operation
-
-Syntax:
-
-```
-operation ::= ssa-id `=` `dim` ssa-id `,` integer-literal `:` type
-```
-
-The `dim` operation takes a memref or tensor operand and a dimension index, and
-returns an [`index`](../LangRef.md#index-type) that is the size of that
-dimension.
-
-The `dim` operation is represented with a single integer attribute named
-`index`, and the type specifies the type of the memref or tensor operand.
-
-Examples:
-
-```mlir
-// Always returns 4, can be constant folded:
-%x = dim %A, 0 : tensor<4 x ? x f32>
-
-// Returns the dynamic dimension of %A.
-%y = dim %A, 1 : tensor<4 x ? x f32>
-
-// Equivalent generic form:
-%x = "std.dim"(%A) {index = 0 : i64} : (tensor<4 x ? x f32>) -> index
-%y = "std.dim"(%A) {index = 1 : i64} : (tensor<4 x ? x f32>) -> index
-```
-
-## Memory Operations
-
-### 'alloc' operation
-
-Syntax:
-
-```
-operation ::= ssa-id `=` `alloc` dim-and-symbol-use-list `:` memref-type
-```
-
-Allocates a new memref of specified type. Values required for dynamic dimension
-sizes are passed as arguments in parentheses (in the same order in which they
-appear in the shape signature of the memref) while the symbols required by the
-layout map are passed in the square brackets in lexicographical order. If no
-layout maps are specified in the memref, then an identity mapping is used.
-
-The buffer referenced by a memref type is created by the `alloc` operation, and
-destroyed by the `dealloc` operation.
-
-Example:
-
-```mlir
-// Allocating memref for a fully static shape.
-%A = alloc() : memref<1024x64xf32, #layout_map0, memspace0>
-
-// %M, %N, %x, %y are SSA values of integer type.  M and N are bound to the
-// two unknown dimensions of the type and x/y are bound to symbols in
-// #layout_map1.
-%B = alloc(%M, %N)[%x, %y] : memref<?x?xf32, #layout_map1, memspace1>
-```
-
-### 'alloc_static' operation
-
-Syntax:
-
-```
-operation ::=
-    ssa-id `=` `alloc_static` `(` integer-literal `)` :  memref-type
-```
-
-Allocates a new memref of specified type with a fixed base pointer location in
-memory. 'alloc_static' does not support types that have dynamic shapes or that
-require dynamic symbols in their layout function (use the
-[`alloc` operation](#alloc-operation) in those cases).
-
-Example:
-
-```mlir
-%A = alloc_static(0x1232a00) : memref<1024 x 64 x f32, #layout_map0, memspace0>
-```
-
-The `alloc_static` operation is used to represent code after buffer allocation
-has been performed.
-
-### 'dealloc' operation
-
-Syntax:
-
-```
-operation ::= `dealloc` ssa-use `:` memref-type
-```
-
-Delineates the end of the lifetime of the memory corresponding to a memref
-allocation. It is paired with an [`alloc`](#alloc-operation) or
-[`alloc_static`](#alloc-static-operation) operation.
-
-Example:
-
-```mlir
-dealloc %A : memref<128 x f32, #layout, memspace0>
-```
+[include "Dialects/StandardOps.md"]
 
 ### 'dma_start' operation
 
@@ -286,880 +74,3 @@ Example:
 ```mlir
 dma_wait %tag[%idx], %size : memref<1 x i32, affine_map<(d0) -> (d0)>, 4>
 ```
-
-### 'extract_element' operation
-
-Syntax:
-
-```
-operation ::= ssa-id `=` `extract_element` ssa-use `[` ssa-use-list `]` `:` type
-```
-
-The `extract_element` op reads a tensor or vector and returns one element from
-it specified by an index list. The output of the 'extract_element' is a new
-value with the same type as the elements of the tensor or vector. The arity of
-indices matches the rank of the accessed value (i.e., if a tensor is of rank 3,
-then 3 indices are required for the extract. The indices should all be of
-`index` type.
-
-Examples:
-
-```mlir
-%3 = extract_element %v[%1, %2] : vector<4x4xi32>
-%4 = extract_element %t[%1, %2] : tensor<4x4xi32>
-%5 = extract_element %ut[%1, %2] : tensor<*xi32>
-```
-
-### 'load' operation
-
-Syntax:
-
-```
-operation ::= ssa-id `=` `load` ssa-use `[` ssa-use-list `]` `:` memref-type
-```
-
-The `load` op reads an element from a memref specified by an index list. The
-output of load is a new value with the same type as the elements of the memref.
-The arity of indices is the rank of the memref (i.e., if the memref loaded from
-is of rank 3, then 3 indices are required for the load following the memref
-identifier).
-
-In an `affine.if` or `affine.for` body, the indices of a load are restricted to
-SSA values bound to surrounding loop induction variables,
-[symbols](../LangRef.md#dimensions-and-symbols), results of a
-[`constant` operation](#constant-operation), or the result of an `affine.apply`
-operation that can in turn take as arguments all of the aforementioned SSA
-values or the recursively result of such an `affine.apply` operation.
-
-Example:
-
-```mlir
-%1 = affine.apply affine_map<(d0, d1) -> (3*d0)> (%i, %j)
-%2 = affine.apply affine_map<(d0, d1) -> (d1+1)> (%i, %j)
-%12 = load %A[%1, %2] : memref<8x?xi32, #layout, memspace0>
-
-// Example of an indirect load (treated as non-affine)
-%3 = affine.apply affine_map<(d0) -> (2*d0 + 1)>(%12)
-%13 = load %A[%3, %2] : memref<4x?xi32, #layout, memspace0>
-```
-
-**Context:** The `load` and `store` operations are specifically crafted to fully
-resolve a reference to an element of a memref, and (in affine `affine.if` and
-`affine.for` operations) the compiler can follow use-def chains (e.g. through
-[`affine.apply`](Affine.md#affineapply-operation) operations) to precisely
-analyze references at compile-time using polyhedral techniques. This is possible
-because of the
-[restrictions on dimensions and symbols](Affine.md#restrictions-on-dimensions-and-symbols)
-in these contexts.
-
-### 'splat' operation
-
-Syntax:
-
-```
-operation ::= ssa-id `=` `splat` ssa-use `:` ( vector-type | tensor-type )
-```
-
-Broadcast the operand to all elements of the result vector or tensor. The
-operand has to be of either integer or float type. When the result is a tensor,
-it has to be statically shaped.
-
-Example:
-
-```mlir
-  %s = load %A[%i] : memref<128xf32>
-  %v = splat %s : vector<4xf32>
-  %t = splat %s : tensor<8x16xi32>
-```
-
-TODO: This operation is easy to extend to broadcast to dynamically shaped
-tensors in the same way dynamically shaped memrefs are handled.
-```mlir
-// Broadcasts %s to a 2-d dynamically shaped tensor, with %m, %n binding
-// to the sizes of the two dynamic dimensions.
-%m = "foo"() : () -> (index)
-%n = "bar"() : () -> (index)
-%t = splat %s [%m, %n] : tensor<?x?xi32>
-```
-
-### 'store' operation
-
-Syntax:
-
-```
-operation ::= `store` ssa-use `,` ssa-use `[` ssa-use-list `]` `:` memref-type
-```
-
-Store value to memref location given by indices. The value stored should have
-the same type as the elemental type of the memref. The number of arguments
-provided within brackets need to match the rank of the memref.
-
-In an affine context, the indices of a store are restricted to SSA values bound
-to surrounding loop induction variables,
-[symbols](Affine.md#restrictions-on-dimensions-and-symbols), results of a
-[`constant` operation](#constant-operation), or the result of an
-[`affine.apply`](Affine.md#affineapply-operation) operation that can in turn
-take as arguments all of the aforementioned SSA values or the recursively result
-of such an `affine.apply` operation.
-
-Example:
-
-```mlir
-store %100, %A[%1, 1023] : memref<4x?xf32, #layout, memspace0>
-```
-
-**Context:** The `load` and `store` operations are specifically crafted to fully
-resolve a reference to an element of a memref, and (in polyhedral `affine.if`
-and `affine.for` operations) the compiler can follow use-def chains (e.g.
-through [`affine.apply`](Affine.md#affineapply-operation) operations) to
-precisely analyze references at compile-time using polyhedral techniques. This
-is possible because of the
-[restrictions on dimensions and symbols](Affine.md#restrictions-on-dimensions-and-symbols)
-in these contexts.
-
-### 'tensor_load' operation
-
-Syntax:
-
-```
-operation ::= ssa-id `=` `tensor_load` ssa-use-and-type
-```
-
-Create a tensor from a memref, making an independent copy of the element data.
-The result value is a tensor whose shape and element type match the memref
-operand.
-
-Example:
-
-```mlir
-// Produces a value of tensor<4x?xf32> type.
-%12 = tensor_load %10 : memref<4x?xf32, #layout, memspace0>
-```
-
-### 'tensor_store' operation
-
-Syntax:
-
-```
-operation ::= `tensor_store` ssa-use `,` ssa-use `:` memref-type
-```
-
-Stores the contents of a tensor into a memref. The first operand is a value of
-tensor type, the second operand is a value of memref type. The shapes and
-element types of these must match, and are specified by the memref type.
-
-Example:
-
-```mlir
-%9 = dim %8, 1 : tensor<4x?xf32>
-%10 = alloc(%9) : memref<4x?xf32, #layout, memspace0>
-tensor_store %8, %10 : memref<4x?xf32, #layout, memspace0>
-```
-
-## Unary Operations
-
-### 'absf' operation
-
-Syntax:
-
-```
-operation ::= ssa-id `=` `absf` ssa-use `:` type
-```
-
-Examples:
-
-```mlir
-// Scalar absolute value.
-%a = absf %b : f64
-
-// SIMD vector element-wise absolute value.
-%f = absf %g : vector<4xf32>
-
-// Tensor element-wise absolute value.
-%x = absf %y : tensor<4x?xf8>
-```
-
-The `absf` operation computes the absolute value. It takes one operand and
-returns one result of the same type. This type may be a float scalar type, a
-vector whose element type is float, or a tensor of floats. It has no standard
-attributes.
-
-### 'ceilf' operation
-
-Syntax:
-
-```
-operation ::= ssa-id `=` `ceilf` ssa-use `:` type
-```
-
-Examples:
-
-```mlir
-// Scalar ceiling value.
-%a = ceilf %b : f64
-
-// SIMD vector element-wise ceiling value.
-%f = ceilf %g : vector<4xf32>
-
-// Tensor element-wise ceiling value.
-%x = ceilf %y : tensor<4x?xf8>
-```
-
-The `ceilf` operation computes the ceiling of a given value. It takes one
-operand and returns one result of the same type. This type may be a float
-scalar type, a vector whose element type is float, or a tensor of floats. It
-has no standard attributes.
-
-### 'cos' operation
-
-Syntax:
-
-```
-operation ::= ssa-id `=` `cos` ssa-use `:` type
-```
-
-Examples:
-
-```mlir
-// Scalar cosine value.
-%a = cos %b : f64
-
-// SIMD vector element-wise cosine value.
-%f = cos %g : vector<4xf32>
-
-// Tensor element-wise cosine value.
-%x = cos %y : tensor<4x?xf8>
-```
-
-The `cos` operation computes the cosine of a given value. It takes one operand
-and returns one result of the same type. This type may be a float scalar type,
-a vector whose element type is float, or a tensor of floats. It has no standard
-attributes.
-
-### 'exp' operation
-
-Syntax:
-
-```
-operation ::= ssa-id `=` `exp` ssa-use `:` type
-```
-
-Examples:
-
-```mlir
-// Scalar natural exponential.
-%a = exp %b : f64
-
-// SIMD vector element-wise natural exponential.
-%f = exp %g : vector<4xf32>
-
-// Tensor element-wise natural exponential.
-%x = exp %y : tensor<4x?xf8>
-```
-
-The `exp` operation takes one operand and returns one result of the same type.
-This type may be a float scalar type, a vector whose element type is float, or a
-tensor of floats. It has no standard attributes.
-
-### 'negf' operation
-
-Syntax:
-
-```
-operation ::= ssa-id `=` `negf` ssa-use `:` type
-```
-
-Examples:
-
-```mlir
-// Scalar negation value.
-%a = negf %b : f64
-
-// SIMD vector element-wise negation value.
-%f = negf %g : vector<4xf32>
-
-// Tensor element-wise negation value.
-%x = negf %y : tensor<4x?xf8>
-```
-
-The `negf` operation computes the negation of a given value. It takes one
-operand and returns one result of the same type. This type may be a float
-scalar type, a vector whose element type is float, or a tensor of floats. It
-has no standard attributes.
-
-### 'sqrt' operation
-
-Syntax:
-
-```
-operation ::= ssa-id `=` `sqrt` ssa-use `:` type
-```
-
-Examples:
-
-```mlir
-// Scalar square root value.
-%a = sqrt %b : f64
-// SIMD vector element-wise square root value.
-%f = sqrt %g : vector<4xf32>
-// Tensor element-wise square root value.
-%x = sqrt %y : tensor<4x?xf32>
-```
-
-### 'tanh' operation
-
-Syntax:
-
-```
-operation ::= ssa-id `=` `tanh` ssa-use `:` type
-```
-
-Examples:
-
-```mlir
-// Scalar hyperbolic tangent value.
-%a = tanh %b : f64
-
-// SIMD vector element-wise hyperbolic tangent value.
-%f = tanh %g : vector<4xf32>
-
-// Tensor element-wise hyperbolic tangent value.
-%x = tanh %y : tensor<4x?xf8>
-```
-
-The `tanh` operation computes the hyperbolic tangent. It takes one operand and
-returns one result of the same type. This type may be a float scalar type, a
-vector whose element type is float, or a tensor of floats. It has no standard
-attributes.
-
-## Arithmetic Operations
-
-Basic arithmetic in MLIR is specified by standard operations described in this
-section.
-
-### 'addi' operation
-
-Syntax:
-
-```
-operation ::= ssa-id `=` `addi` ssa-use `,` ssa-use `:` type
-```
-
-Examples:
-
-```mlir
-// Scalar addition.
-%a = addi %b, %c : i64
-
-// SIMD vector element-wise addition, e.g. for Intel SSE.
-%f = addi %g, %h : vector<4xi32>
-
-// Tensor element-wise addition.
-%x = addi %y, %z : tensor<4x?xi8>
-```
-
-The `addi` operation takes two operands and returns one result, each of these is
-required to be the same type. This type may be an integer scalar type, a vector
-whose element type is integer, or a tensor of integers. It has no standard
-attributes.
-
-### 'addf' operation
-
-Syntax:
-
-```
-operation ::= ssa-id `=` `addf` ssa-use `,` ssa-use `:` type
-```
-
-Examples:
-
-```mlir
-// Scalar addition.
-%a = addf %b, %c : f64
-
-// SIMD vector addition, e.g. for Intel SSE.
-%f = addf %g, %h : vector<4xf32>
-
-// Tensor addition.
-%x = addf %y, %z : tensor<4x?xbf16>
-```
-
-The `addf` operation takes two operands and returns one result, each of these is
-required to be the same type. This type may be a floating point scalar type, a
-vector whose element type is a floating point type, or a floating point tensor.
-
-It has no standard attributes.
-
-TODO: In the distant future, this will accept optional attributes for fast math,
-contraction, rounding mode, and other controls.
-
-### 'and' operation
-
-Bitwise integer and.
-
-Syntax:
-
-```
-operation ::= ssa-id `=` `and` ssa-use `,` ssa-use `:` type
-```
-
-Examples:
-
-```mlir
-// Scalar integer bitwise and.
-%a = and %b, %c : i64
-
-// SIMD vector element-wise bitwise integer and.
-%f = and %g, %h : vector<4xi32>
-
-// Tensor element-wise bitwise integer and.
-%x = and %y, %z : tensor<4x?xi8>
-```
-
-The `and` operation takes two operands and returns one result, each of these is
-required to be the same type. This type may be an integer scalar type, a vector
-whose element type is integer, or a tensor of integers. It has no standard
-attributes.
-
-### 'cmpi' operation
-
-Syntax:
-
-```
-operation ::= ssa-id `=` `cmpi` string-literal `,` ssa-id `,` ssa-id `:` type
-```
-
-Examples:
-
-```mlir
-// Custom form of scalar "signed less than" comparison.
-%x = cmpi "slt", %lhs, %rhs : i32
-
-// Generic form of the same operation.
-%x = "std.cmpi"(%lhs, %rhs) {predicate = 2 : i64} : (i32, i32) -> i1
-
-// Custom form of vector equality comparison.
-%x = cmpi "eq", %lhs, %rhs : vector<4xi64>
-
-// Generic form of the same operation.
-%x = "std.cmpi"(%lhs, %rhs) {predicate = 0 : i64}
-    : (vector<4xi64>, vector<4xi64>) -> vector<4xi1>
-```
-
-The `cmpi` operation is a generic comparison for integer-like types. Its two
-arguments can be integers, vectors or tensors thereof as long as their types
-match. The operation produces an i1 for the former case, a vector or a tensor of
-i1 with the same shape as inputs in the other cases.
-
-Its first argument is an attribute that defines which type of comparison is
-performed. The following comparisons are supported:
-
--   equal (mnemonic: `"eq"`; integer value: `0`)
--   not equal (mnemonic: `"ne"`; integer value: `1`)
--   signed less than (mnemonic: `"slt"`; integer value: `2`)
--   signed less than or equal (mnemonic: `"sle"`; integer value: `3`)
--   signed greater than (mnemonic: `"sgt"`; integer value: `4`)
--   signed greater than or equal (mnemonic: `"sge"`; integer value: `5`)
--   unsigned less than (mnemonic: `"ult"`; integer value: `6`)
--   unsigned less than or equal (mnemonic: `"ule"`; integer value: `7`)
--   unsigned greater than (mnemonic: `"ugt"`; integer value: `8`)
--   unsigned greater than or equal (mnemonic: `"uge"`; integer value: `9`)
-
-The result is `1` if the comparison is true and `0` otherwise. For vector or
-tensor operands, the comparison is performed elementwise and the element of the
-result indicates whether the comparison is true for the operand elements with
-the same indices as those of the result.
-
-Note: while the custom assembly form uses strings, the actual underlying
-attribute has integer type (or rather enum class in C++ code) as seen from the
-generic assembly form. String literals are used to improve readability of the IR
-by humans.
-
-This operation only applies to integer-like operands, but not floats. The main
-reason being that comparison operations have diverging sets of attributes:
-integers require sign specification while floats require various floating
-point-related particularities, e.g., `-ffast-math` behavior, IEEE754 compliance,
-etc
-([rationale](../Rationale.md#splitting-floating-point-vs-integer-operations)).
-The type of comparison is specified as attribute to avoid introducing ten
-similar operations, taking into account that they are often implemented using
-the same operation downstream
-([rationale](../Rationale.md#specifying-comparison-kind-as-attribute)). The
-separation between signed and unsigned order comparisons is necessary because of
-integers being signless. The comparison operation must know how to interpret
-values with the foremost bit being set: negatives in two's complement or large
-positives
-([rationale](../Rationale.md#specifying-sign-in-integer-comparison-operations)).
-
-### 'constant' operation
-
-Syntax:
-
-```
-operation ::= ssa-id `=` `constant` attribute-value `:` type
-```
-
-The `constant` operation produces an SSA value equal to some constant specified
-by an attribute. This is the way that MLIR uses to form simple integer and
-floating point constants, as well as more exotic things like references to
-functions and (TODO!) tensor/vector constants.
-
-The `constant` operation is represented with a single attribute named "value".
-The type specifies the result type of the operation.
-
-Examples:
-
-```mlir
-// Integer constant
-%1 = constant 42 : i32
-
-// Reference to function @myfn.
-%3 = constant @myfn : (tensor<16xf32>, f32) -> tensor<16xf32>
-
-// Equivalent generic forms
-%1 = "std.constant"() {value = 42 : i32} : () -> i32
-%3 = "std.constant"() {value = @myfn}
-   : () -> ((tensor<16xf32>, f32) -> tensor<16xf32>)
-
-```
-
-MLIR does not allow direct references to functions in SSA operands because the
-compiler is multithreaded, and disallowing SSA values to directly reference a
-function simplifies this
-([rationale](../Rationale.md#multithreading-the-compiler)).
-
-### 'copysign' operation
-
-Syntax:
-
-```
-operation ::= ssa-id `=` `copysign` ssa-use `:` type
-```
-
-Examples:
-
-```mlir
-// Scalar copysign value.
-%a = copysign %b %c : f64
-
-// SIMD vector element-wise copysign value.
-%f = copysign %g %h : vector<4xf32>
-
-// Tensor element-wise copysign value.
-%x = copysign %y %z : tensor<4x?xf8>
-```
-
-The `copysign` returns a value with the magnitude of the first operand and the
-sign of the second operand. It takes two operands and returns one result of the
-same type. This type may be a float scalar type, a vector whose element type is
-float, or a tensor of floats. It has no standard attributes.
-
-### 'divis' operation
-
-Signed integer division. Rounds towards zero. Treats the leading bit as sign,
-i.e. `6 / -2 = -3`.
-
-Note: the semantics of division by zero or signed division overflow (minimum
-value divided by -1) is TBD; do NOT assume any specific behavior.
-
-Syntax:
-
-```
-operation ::= ssa-id `=` `divis` ssa-use `,` ssa-use `:` type
-```
-
-Examples:
-
-```mlir
-// Scalar signed integer division.
-%a = divis %b, %c : i64
-
-// SIMD vector element-wise division.
-%f = divis %g, %h : vector<4xi32>
-
-// Tensor element-wise integer division.
-%x = divis %y, %z : tensor<4x?xi8>
-```
-
-The `divis` operation takes two operands and returns one result, each of these
-is required to be the same type. This type may be an integer scalar type, a
-vector whose element type is integer, or a tensor of integers. It has no
-standard attributes.
-
-### 'diviu' operation
-
-Unsigned integer division. Rounds towards zero. Treats the leading bit as the
-most significant, i.e. for `i16` given two's complement representation, `6 /
--2 = 6 / (2^16 - 2) = 0`.
-
-Note: the semantics of division by zero is TBD; do NOT assume any specific
-behavior.
-
-Syntax:
-
-```
-operation ::= ssa-id `=` `diviu` ssa-use `,` ssa-use `:` type
-```
-
-Examples:
-
-```mlir
-// Scalar unsigned integer division.
-%a = diviu %b, %c : i64
-
-// SIMD vector element-wise division.
-%f = diviu %g, %h : vector<4xi32>
-
-// Tensor element-wise integer division.
-%x = diviu %y, %z : tensor<4x?xi8>
-```
-
-The `diviu` operation takes two operands and returns one result, each of these
-is required to be the same type. This type may be an integer scalar type, a
-vector whose element type is integer, or a tensor of integers. It has no
-standard attributes.
-
-### 'memref_cast' operation
-
-Syntax:
-
-```
-operation ::= ssa-id `=` `memref_cast` ssa-use `:` type `to` type
-```
-
-Examples:
-
-```mlir
-// Discard static dimension information.
-%3 = memref_cast %2 : memref<4x?xf32> to memref<?x?xf32>
-
-// Convert to a type with more known dimensions.
-%4 = memref_cast %3 : memref<?x?xf32> to memref<4x?xf32>
-
-// Convert to a type with unknown rank.
-%5 = memref_cast %3 : memref<?x?xf32> to memref<*xf32>
-
-// Convert to a type with static rank.
-%6 = memref_cast %5 : memref<*xf32> to memref<?x?xf32>
-```
-
-Convert a memref from one type to an equivalent type without changing any data
-elements. The types are equivalent if 1. they both have the same static rank,
-same element type, same mappings, same address space. The operation is invalid
-if converting to a mismatching constant dimension, or 2. exactly one of the
-operands have an unknown rank, and they both have the same element type and same
-address space. The operation is invalid if both operands are of dynamic rank or
-if converting to a mismatching static rank.
-
-### 'mulf' operation
-
-Syntax:
-
-```
-operation ::= ssa-id `=` `mulf` ssa-use `,` ssa-use `:` type
-```
-
-Examples:
-
-```mlir
-// Scalar multiplication.
-%a = mulf %b, %c : f64
-
-// SIMD pointwise vector multiplication, e.g. for Intel SSE.
-%f = mulf %g, %h : vector<4xf32>
-
-// Tensor pointwise multiplication.
-%x = mulf %y, %z : tensor<4x?xbf16>
-```
-
-The `mulf` operation takes two operands and returns one result, each of these is
-required to be the same type. This type may be a floating point scalar type, a
-vector whose element type is a floating point type, or a floating point tensor.
-
-It has no standard attributes.
-
-TODO: In the distant future, this will accept optional attributes for fast math,
-contraction, rounding mode, and other controls.
-
-### 'or' operation
-
-Bitwise integer or.
-
-Syntax:
-
-```
-operation ::= ssa-id `=` `or` ssa-use `,` ssa-use `:` type
-```
-
-Examples:
-
-```mlir
-// Scalar integer bitwise or.
-%a = or %b, %c : i64
-
-// SIMD vector element-wise bitwise integer or.
-%f = or %g, %h : vector<4xi32>
-
-// Tensor element-wise bitwise integer or.
-%x = or %y, %z : tensor<4x?xi8>
-```
-
-The `or` operation takes two operands and returns one result, each of these is
-required to be the same type. This type may be an integer scalar type, a vector
-whose element type is integer, or a tensor of integers. It has no standard
-attributes.
-
-### 'remis' operation
-
-Signed integer division remainder. Treats the leading bit as sign, i.e. `6 %
--2 = 0`.
-
-Note: the semantics of division by zero is TBD; do NOT assume any specific
-behavior.
-
-Syntax:
-
-```
-operation ::= ssa-id `=` `remis` ssa-use `,` ssa-use `:` type
-```
-
-Examples:
-
-```mlir
-// Scalar signed integer division remainder.
-%a = remis %b, %c : i64
-
-// SIMD vector element-wise division remainder.
-%f = remis %g, %h : vector<4xi32>
-
-// Tensor element-wise integer division remainder.
-%x = remis %y, %z : tensor<4x?xi8>
-```
-
-The `remis` operation takes two operands and returns one result, each of these
-is required to be the same type. This type may be an integer scalar type, a
-vector whose element type is integer, or a tensor of integers. It has no
-standard attributes.
-
-### 'remiu' operation
-
-Unsigned integer division remainder. Treats the leading bit as the most
-significant, i.e. for `i16`, `6 % -2 = 6 % (2^16 - 2) = 6`.
-
-Note: the semantics of division by zero is TBD; do NOT assume any specific
-behavior.
-
-Syntax:
-
-```
-operation ::= ssa-id `=` `remiu` ssa-use `,` ssa-use `:` type
-```
-
-Examples:
-
-```mlir
-// Scalar unsigned integer division remainder.
-%a = remiu %b, %c : i64
-
-// SIMD vector element-wise division remainder.
-%f = remiu %g, %h : vector<4xi32>
-
-// Tensor element-wise integer division remainder.
-%x = remiu %y, %z : tensor<4x?xi8>
-```
-
-The `remiu` operation takes two operands and returns one result, each of these
-is required to be the same type. This type may be an integer scalar type, a
-vector whose element type is integer, or a tensor of integers. It has no
-standard attributes.
-
-### 'select' operation
-
-Syntax:
-
-```
-operation ::= ssa-id `=` `select` ssa-use `,` ssa-use `,` ssa-use `:` type
-```
-
-Examples:
-
-```mlir
-// Custom form of scalar selection.
-%x = select %cond, %true, %false : i32
-
-// Generic form of the same operation.
-%x = "std.select"(%cond, %true, %false) : (i1, i32, i32) -> i32
-
-// Vector selection is element-wise
-%vx = "std.select"(%vcond, %vtrue, %vfalse)
-    : (vector<42xi1>, vector<42xf32>, vector<42xf32>) -> vector<42xf32>
-```
-
-The `select` operation chooses one value based on a binary condition supplied as
-its first operand. If the value of the first operand is `1`, the second operand
-is chosen, otherwise the third operand is chosen. The second and the third
-operand must have the same type.
-
-The operation applies to vectors and tensors elementwise given the _shape_ of
-all operands is identical. The choice is made for each element individually
-based on the value at the same position as the element in the condition operand.
-
-The `select` operation combined with [`cmpi`](#cmpi-operation) can be used to
-implement `min` and `max` with signed or unsigned comparison semantics.
-
-### 'tensor_cast' operation
-
-Syntax:
-
-```
-operation ::= ssa-id `=` `tensor_cast` ssa-use `:` type `to` type
-```
-
-Examples:
-
-```mlir
-// Convert from unknown rank to rank 2 with unknown dimension sizes.
-%2 = "std.tensor_cast"(%1) : (tensor<*xf32>) -> tensor<?x?xf32>
-%2 = tensor_cast %1 : tensor<*xf32> to tensor<?x?xf32>
-
-// Convert to a type with more known dimensions.
-%3 = "std.tensor_cast"(%2) : (tensor<?x?xf32>) -> tensor<4x?xf32>
-
-// Discard static dimension and rank information.
-%4 = "std.tensor_cast"(%3) : (tensor<4x?xf32>) -> tensor<?x?xf32>
-%5 = "std.tensor_cast"(%4) : (tensor<?x?xf32>) -> tensor<*xf32>
-```
-
-Convert a tensor from one type to an equivalent type without changing any data
-elements. The source and destination types must both be tensor types with the
-same element type. If both are ranked, then the rank should be the same and
-static dimensions should match. The operation is invalid if converting to a
-mismatching constant dimension.
-
-### 'xor' operation
-
-Bitwise integer xor.
-
-Syntax:
-
-```
-operation ::= ssa-id `=` `xor` ssa-use, ssa-use `:` type
-```
-
-Examples:
-
-```mlir
-// Scalar integer bitwise xor.
-%a = xor %b, %c : i64
-
-// SIMD vector element-wise bitwise integer xor.
-%f = xor %g, %h : vector<4xi32>
-
-// Tensor element-wise bitwise integer xor.
-%x = xor %y, %z : tensor<4x?xi8>
-```
-
-The `xor` operation takes two operands and returns one result, each of these is
-required to be the same type. This type may be an integer scalar type, a vector
-whose element type is integer, or a tensor of integers. It has no standard
-attributes.
diff --git a/mlir/docs/Dialects/Vector.md b/mlir/docs/Dialects/Vector.md
index 10de1731a2f24..910a9cd2464a3 100644
--- a/mlir/docs/Dialects/Vector.md
+++ b/mlir/docs/Dialects/Vector.md
@@ -1,4 +1,4 @@
-# Vector Dialect
+# 'vector' Dialect
 
 [TOC]
 
diff --git a/mlir/docs/OpDefinitions.md b/mlir/docs/OpDefinitions.md
index 29269e508bb08..5d7f679073092 100644
--- a/mlir/docs/OpDefinitions.md
+++ b/mlir/docs/OpDefinitions.md
@@ -710,7 +710,7 @@ to:
    -  Note that `attr-dict` does not overlap with individual attributes. These
       attributes will simply be elided when printing the attribute dictionary.
 
-##### Type Inferrence
+##### Type Inference
 
 One requirement of the format is that the types of operands and results must
 always be present. In certain instances, the type of a variable may be deduced
diff --git a/mlir/docs/Passes.md b/mlir/docs/Passes.md
index 78ea257b57bc3..866f3666d7d00 100644
--- a/mlir/docs/Passes.md
+++ b/mlir/docs/Passes.md
@@ -4,295 +4,46 @@ This document describes the available MLIR passes and their contracts.
 
 [TOC]
 
-## Affine control lowering (`-lower-affine`)
+## General Transformation Passes
 
-Convert operations related to affine control into a graph of blocks using
-operations from the standard dialect.
+[include "GeneralPasses.md"]
 
-Loop statements are converted to a subgraph of blocks (initialization, condition
-checking, subgraph of body blocks) with loop induction variable being passed as
-the block argument of the condition checking block. Conditional statements are
-converted to a subgraph of blocks (chain of condition checking with
-short-circuit logic, subgraphs of 'then' and 'else' body blocks). `affine.apply`
-operations are converted into sequences of primitive arithmetic operations that
-have the same effect, using operands of the `index` type. Consequently, named
-maps and sets may be removed from the module.
+## Conversion Passes
 
-For example, `%r = affine.apply (d0, d1)[s0] -> (d0 + 2*d1 + s0)(%d0, %d1)[%s0]`
-can be converted into:
+[include "ConversionPasses.md"]
 
-```mlir
-%d0 = <...>
-%d1 = <...>
-%s0 = <...>
-%0 = constant 2 : index
-%1 = muli %0, %d1
-%2 = addi %d0, %1
-%r = addi %2, %s0
-```
+## Quantizer Passes
 
-### Input invariant
+[include "QuantizerPasses.md"]
 
--   no `Tensor` types;
+## `affine` Dialect Passes
 
-These restrictions may be lifted in the future.
+[include "AffinePasses.md"]
 
-### Output IR
+## `fxpmath` Dialect Passes
 
-Functions with `affine.for` and `affine.if` operations eliminated. These
-functions may contain operations from the Standard dialect in addition to those
-already present before the pass.
+[include "FxpMathPasses.md"]
 
-### Invariants
+## `gpu` Dialect Passes
 
--   Functions without a body are not modified.
--   The semantics of the other functions is preserved.
--   Individual operations other than those mentioned above are not modified if
-    they do not depend on the loop iterator value or on the result of
-    `affine.apply`.
+[include "GPUPasses.md"]
 
-## Conversion from Standard to LLVM IR dialect (`-convert-std-to-llvm`)
+## `linalg` Dialect Passes
 
-Convert standard operations into the LLVM IR dialect operations.
+[include "LinalgPasses.md"]
 
-### Input invariant
+## `llvm` Dialect Passes
 
--   operations including: arithmetic on integers and floats, constants, direct
-    calls, returns and branches;
--   no `tensor` types;
--   all `vector` are one-dimensional;
--   all blocks are reachable by following the successors of the first basic
-    block;
+[include "LLVMPasses.md"]
 
-If other operations are present and their results are required by the LLVM IR
-dialect operations, the pass will fail.  Any LLVM IR operations or types already
-present in the IR will be kept as is.
+## `loop` Dialect Passes
 
-### Output IR
+[include "LoopPasses.md"]
 
-Functions converted to LLVM IR. Function arguments types are converted
-one-to-one. Function results are converted one-to-one and, in case more than 1
-value is returned, packed into an LLVM IR struct type. Function calls and
-returns are updated accordingly. Block argument types are updated to use LLVM IR
-types.
+## `quant` Dialect Passes
 
-## Data Copy DMA generation (`-affine-data-copy-generate`)
+[include "QuantPasses.md"]
 
-Replaces all loads and stores on memref's living in 'slowMemorySpace' by
-introducing DMA operations (strided DMA if necessary) to transfer data to/from
-`fastMemorySpace` and rewriting the original load's/store's to instead
-load/store from the allocated fast memory buffers. Additional options specify
-the identifier corresponding to the fast memory space and the amount of fast
-memory space available. The pass traverses through the nesting structure,
-recursing to inner levels if necessary to determine at what depth DMA transfers
-need to be placed so that the allocated buffers fit within the memory capacity
-provided. If this is not possible (for example, when the elemental type itself
-is of size larger than the DMA capacity), an error with location information is
-emitted. The DMA transfers are also hoisted up past all loops with respect to
-which the transfers are invariant.
+## `spv` Dialect Passes
 
-Input
-
-```mlir
-func @loop_nest_tiled() -> memref<256x1024xf32> {
-  %0 = alloc() : memref<256x1024xf32>
-  affine.for %i0 = 0 to 256 step 32 {
-    affine.for %i1 = 0 to 1024 step 32 {
-      affine.for %i2 = (d0) -> (d0)(%i0) to (d0) -> (d0 + 32)(%i0) {
-        affine.for %i3 = (d0) -> (d0)(%i1) to (d0) -> (d0 + 32)(%i1) {
-          %1 = affine.load %0[%i2, %i3] : memref<256x1024xf32>
-        }
-      }
-    }
-  }
-  return %0 : memref<256x1024xf32>
-}
-```
-
-Output (with flags: -affine-data-copy-generate -affine-data-copy-generate-fast-mem-space=2)
-
-```mlir
-module {
-  func @loop_nest_tiled() -> memref<256x1024xf32> {
-    %c262144 = constant 262144 : index
-    %c0 = constant 0 : index
-    %0 = alloc() : memref<256x1024xf32>
-    %1 = alloc() : memref<256x1024xf32, 2>
-    %2 = alloc() : memref<1xi32>
-    affine.dma_start %0[%c0, %c0], %1[%c0, %c0], %2[%c0], %c262144 : memref<256x1024xf32>, memref<256x1024xf32, 2>, memref<1xi32>
-    affine.dma_wait %2[%c0], %c262144 : memref<1xi32>
-    affine.for %arg0 = 0 to 256 step 32 {
-      affine.for %arg1 = 0 to 1024 step 32 {
-        affine.for %arg2 = #map1(%arg0) to #map2(%arg0) {
-          affine.for %arg3 = #map1(%arg1) to #map2(%arg1) {
-            %3 = affine.load %1[%arg2, %arg3] : memref<256x1024xf32, 2>
-          }
-        }
-      }
-    }
-    dealloc %2 : memref<1xi32>
-    dealloc %1 : memref<256x1024xf32, 2>
-    return %0 : memref<256x1024xf32>
-  }
-}
-```
-
-## Loop tiling (`-affine-loop-tile`)
-
-Performs tiling or blocking of loop nests. It currently works on perfect loop
-nests.
-
-## Loop unroll (`-affine-loop-unroll`)
-
-This pass implements loop unrolling. It is able to unroll loops with arbitrary
-bounds, and generate a cleanup loop when necessary.
-
-## Loop unroll and jam (`-affine-loop-unroll-jam`)
-
-This pass implements unroll and jam for loops. It works on both perfect or
-imperfect loop nests.
-
-## Loop fusion (`-affine-loop-fusion`)
-
-Performs fusion of loop nests using a slicing-based approach. The fused loop
-nests, when possible, are rewritten to access significantly smaller local
-buffers instead of the original memref's, and the latter are often
-either completely optimized away or contracted. This transformation leads to
-enhanced locality and lower memory footprint through the elimination or
-contraction of temporaries / intermediate memref's. These benefits are sometimes
-achieved at the expense of redundant computation through a cost model that
-evaluates available choices such as the depth at which a source slice should be
-materialized in the designation slice.
-
-## Memref bound checking (`-memref-bound-check`)
-
-Checks all load's and store's on memref's for out of bound accesses, and reports
-any out of bound accesses (both overrun and underrun) with location information.
-
-```mlir
-test/Transforms/memref-bound-check.mlir:19:13: error: 'load' op memref out of upper bound access along dimension #2
-      %x  = load %A[%idx0, %idx1] : memref<9 x 9 x i32>
-            ^
-test/Transforms/memref-bound-check.mlir:19:13: error: 'load' op memref out of lower bound access along dimension #2
-      %x  = load %A[%idx0, %idx1] : memref<9 x 9 x i32>
-            ^
-```
-
-## Memref dataflow optimization (`-memref-dataflow-opt`)
-
-This pass performs store to load forwarding for memref's to eliminate memory
-accesses and potentially the entire memref if all its accesses are forwarded.
-
-Input
-
-```mlir
-func @store_load_affine_apply() -> memref<10x10xf32> {
-  %cf7 = constant 7.0 : f32
-  %m = alloc() : memref<10x10xf32>
-  affine.for %i0 = 0 to 10 {
-    affine.for %i1 = 0 to 10 {
-      affine.store %cf7, %m[%i0, %i1] : memref<10x10xf32>
-      %v0 = affine.load %m[%i0, %i1] : memref<10x10xf32>
-      %v1 = addf %v0, %v0 : f32
-    }
-  }
-  return %m : memref<10x10xf32>
-}
-```
-
-Output
-
-```mlir
-module {
-  func @store_load_affine_apply() -> memref<10x10xf32> {
-    %cst = constant 7.000000e+00 : f32
-    %0 = alloc() : memref<10x10xf32>
-    affine.for %arg0 = 0 to 10 {
-      affine.for %arg1 = 0 to 10 {
-        affine.store %cst, %0[%arg0, %arg1] : memref<10x10xf32>
-        %1 = addf %cst, %cst : f32
-      }
-    }
-    return %0 : memref<10x10xf32>
-  }
-}
-
-```
-
-## Memref dependence analysis (`-memref-dependence-check`)
-
-This pass performs dependence analysis to determine dependences between pairs of
-memory operations (load's and store's) on memref's. Dependence analysis exploits
-polyhedral information available (affine maps, expressions, and affine.apply
-operations) to precisely represent dependences using affine constraints, while
-also computing dependence vectors from them, where each component of the
-dependence vector provides a lower and an upper bound on the dependence distance
-along the corresponding dimension.
-
-```mlir
-test/Transforms/memref-dataflow-opt.mlir:232:7: note: dependence from 2 to 1 at depth 1 = ([1, 1], [-inf, +inf])
-      store %cf9, %m[%idx] : memref<10xf32>
-```
-
-## Pipeline data transfer (`-affine-pipeline-data-transfer`)
-
-This pass performs a transformation to overlap non-blocking DMA operations in a
-loop with computations through double buffering. This is achieved by advancing
-dma_start operations with respect to other operations.
-
-Input
-
-```mlir
-func @pipelinedatatransfer() {
-  %0 = alloc() : memref<256xf32>
-  %1 = alloc() : memref<32xf32, 1>
-  %2 = alloc() : memref<1xf32>
-  %c0 = constant 0 : index
-  %c128 = constant 128 : index
-  affine.for %i0 = 0 to 8 {
-    affine.dma_start %0[%i0], %1[%i0], %2[%c0], %c128 : memref<256xf32>, memref<32xf32, 1>, memref<1xf32>
-    affine.dma_wait %2[%c0], %c128 : memref<1xf32>
-    %3 = affine.load %1[%i0] : memref<32xf32, 1>
-    %4 = "compute"(%3) : (f32) -> f32
-    affine.store %4, %1[%i0] : memref<32xf32, 1>
-  }
-  return
-}
-```
-
-Output
-
-```mlir
-module {
-  func @pipelinedatatransfer() {
-    %c8 = constant 8 : index
-    %c0 = constant 0 : index
-    %0 = alloc() : memref<256xf32>
-    %c0_0 = constant 0 : index
-    %c128 = constant 128 : index
-    %1 = alloc() : memref<2x32xf32, 1>
-    %2 = alloc() : memref<2x1xf32>
-    affine.dma_start %0[%c0], %1[%c0 mod 2, %c0], %2[%c0 mod 2, symbol(%c0_0)], %c128 : memref<256xf32>, memref<2x32xf32, 1>, memref<2x1xf32>
-    affine.for %arg0 = 1 to 8 {
-      affine.dma_start %0[%arg0], %1[%arg0 mod 2, %arg0], %2[%arg0 mod 2, symbol(%c0_0)], %c128 : memref<256xf32>, memref<2x32xf32, 1>, memref<2x1xf32>
-      %8 = affine.apply #map3(%arg0)
-      %9 = affine.apply #map4(%8)
-      %10 = affine.apply #map4(%8)
-      affine.dma_wait %2[%8 mod 2, symbol(%c0_0)], %c128 : memref<2x1xf32>
-      %11 = affine.load %1[%8 mod 2, %8] : memref<2x32xf32, 1>
-      %12 = "compute"(%11) : (f32) -> f32
-      affine.store %12, %1[%8 mod 2, %8] : memref<2x32xf32, 1>
-    }
-    %3 = affine.apply #map3(%c8)
-    %4 = affine.apply #map4(%3)
-    %5 = affine.apply #map4(%3)
-    affine.dma_wait %2[%3 mod 2, symbol(%c0_0)], %c128 : memref<2x1xf32>
-    %6 = affine.load %1[%3 mod 2, %3] : memref<2x32xf32, 1>
-    %7 = "compute"(%6) : (f32) -> f32
-    affine.store %7, %1[%3 mod 2, %3] : memref<2x32xf32, 1>
-    dealloc %2 : memref<2x1xf32>
-    dealloc %1 : memref<2x32xf32, 1>
-    return
-  }
-}
-```
+[include "SPIRVPasses.md"]
diff --git a/mlir/docs/Rationale.md b/mlir/docs/Rationale.md
index ff475fd849e27..e76bae7bb6459 100644
--- a/mlir/docs/Rationale.md
+++ b/mlir/docs/Rationale.md
@@ -202,21 +202,21 @@ and described in
 interest
 [starts here](https://www.google.com/url?q=https://youtu.be/Ntj8ab-5cvE?t%3D596&sa=D&ust=1529450150971000&usg=AFQjCNFQHEWL7m8q3eO-1DiKw9zqC2v24Q).
 
-### Index type disallowed in vector/tensor/memref types
-
-Index types are not allowed as elements of `vector`, `tensor` or `memref` type.
-Index types are intended to be used for platform-specific "size" values and may
-appear in subscripts, sizes of aggregate types and affine expressions. They are
-also tightly coupled with `affine.apply` and affine.load/store operations;
-having `index` type is a necessary precondition of a value to be acceptable by
-these operations. While it may be useful to have `memref<?xindex>` to express
-indirect accesses, e.g. sparse matrix manipulations or lookup tables, it creates
-problems MLIR is not ready to address yet. MLIR needs to internally store
-constants of aggregate types and emit code operating on values of those types,
-which are subject to target-specific size and alignment constraints.  Since MLIR
-does not have a target description mechanism at the moment, it cannot reliably
-emit such code. Moreover, some platforms may not support vectors of type
-equivalent to `index`.
+### Index type disallowed in vector/memref types
+
+Index types are not allowed as elements of `vector` and `memref` types. Index
+types are intended to be used for platform-specific "size" values and may appear
+in subscripts, sizes of aggregate types and affine expressions. They are also
+tightly coupled with `affine.apply` and affine.load/store operations; having
+`index` type is a necessary precondition of a value to be acceptable by these
+operations. While it may be useful to have `memref<?xindex>` to express indirect
+accesses, e.g. sparse matrix manipulations or lookup tables, it creates problems
+MLIR is not ready to address yet. MLIR needs to internally store constants of
+aggregate types and emit code operating on values of those types, which are
+subject to target-specific size and alignment constraints. Since MLIR does not
+have a target description mechanism at the moment, it cannot reliably emit such
+code. Moreover, some platforms may not support vectors of type equivalent to
+`index`.
 
 Indirect access use cases can be alternatively supported by providing and
 `index_cast` instruction that allows for conversion between `index` and
@@ -224,6 +224,11 @@ fixed-width integer types, at the SSA value level. It has an additional benefit
 of supporting smaller integer types, e.g. `i8` or `i16`, for small indices
 instead of (presumably larger) `index` type.
 
+Index types are allowed as element types of `tensor` types. The `tensor` type
+specifically abstracts the target-specific aspects that intersect with the
+code-generation-related/lowering-related concerns explained above. In fact, the
+`tensor` type even allows dialect-specific types as element types.
+
 ### Bit width of a non-primitive types and `index` is undefined
 
 The bit width of a compound type is not defined by MLIR, it may be defined by a
diff --git a/mlir/docs/RationaleLinalgDialect.md b/mlir/docs/RationaleLinalgDialect.md
index 0085e1ff7a09d..01e205741fd44 100644
--- a/mlir/docs/RationaleLinalgDialect.md
+++ b/mlir/docs/RationaleLinalgDialect.md
@@ -311,7 +311,7 @@ with NVCC from a textual representation. While this was a pragmatic
 short-term solution it made it hard to perform low-level rewrites that
 would have helped with register reuse in the ***compute-bound regime***.
 - The same reliance on emitting CUDA code made it difficult to
-create cost models when time came. This made it artifically harder to
+create cost models when time came. This made it artificially harder to
 prune out bad solutions than necessary. This resulted in excessive
 runtime evaluation, as reported in the paper [Machine Learning Systems
 are Stuck in a Rut](https://dl.acm.org/doi/10.1145/3317550.3321441).
diff --git a/mlir/examples/toy/Ch3/CMakeLists.txt b/mlir/examples/toy/Ch3/CMakeLists.txt
index 4dab5e4d0626b..ef70dcbac309d 100644
--- a/mlir/examples/toy/Ch3/CMakeLists.txt
+++ b/mlir/examples/toy/Ch3/CMakeLists.txt
@@ -1,3 +1,4 @@
+include_directories(include)
 add_subdirectory(include)
 
 set(LLVM_LINK_COMPONENTS
@@ -5,7 +6,7 @@ set(LLVM_LINK_COMPONENTS
   )
 
 set(LLVM_TARGET_DEFINITIONS mlir/ToyCombine.td)
-mlir_tablegen(ToyCombine.inc -gen-rewriters "-I${CMAKE_CURRENT_SOURCE_DIR}/include")
+mlir_tablegen(ToyCombine.inc -gen-rewriters)
 add_public_tablegen_target(ToyCh3CombineIncGen)
 
 add_toy_chapter(toyc-ch3
@@ -20,7 +21,6 @@ add_toy_chapter(toyc-ch3
   ToyCh3CombineIncGen
   )
 
-include_directories(include/)
 include_directories(${CMAKE_CURRENT_BINARY_DIR})
 include_directories(${CMAKE_CURRENT_BINARY_DIR}/include/)
 target_link_libraries(toyc-ch3
diff --git a/mlir/examples/toy/Ch4/CMakeLists.txt b/mlir/examples/toy/Ch4/CMakeLists.txt
index 3589b10645a62..ae30a691894e2 100644
--- a/mlir/examples/toy/Ch4/CMakeLists.txt
+++ b/mlir/examples/toy/Ch4/CMakeLists.txt
@@ -1,3 +1,4 @@
+include_directories(include)
 add_subdirectory(include)
 
 set(LLVM_LINK_COMPONENTS
@@ -5,7 +6,7 @@ set(LLVM_LINK_COMPONENTS
   )
 
 set(LLVM_TARGET_DEFINITIONS mlir/ToyCombine.td)
-mlir_tablegen(ToyCombine.inc -gen-rewriters "-I${CMAKE_CURRENT_SOURCE_DIR}/include")
+mlir_tablegen(ToyCombine.inc -gen-rewriters)
 add_public_tablegen_target(ToyCh4CombineIncGen)
 
 add_toy_chapter(toyc-ch4
@@ -22,7 +23,6 @@ add_toy_chapter(toyc-ch4
   ToyCh4CombineIncGen
   )
 
-include_directories(include/)
 include_directories(${CMAKE_CURRENT_BINARY_DIR})
 include_directories(${CMAKE_CURRENT_BINARY_DIR}/include/)
 target_link_libraries(toyc-ch4
diff --git a/mlir/examples/toy/Ch4/include/toy/CMakeLists.txt b/mlir/examples/toy/Ch4/include/toy/CMakeLists.txt
index 798d0df1d8d68..7f60477fc272d 100644
--- a/mlir/examples/toy/Ch4/include/toy/CMakeLists.txt
+++ b/mlir/examples/toy/Ch4/include/toy/CMakeLists.txt
@@ -1,6 +1,6 @@
 set(LLVM_TARGET_DEFINITIONS Ops.td)
-mlir_tablegen(Ops.h.inc -gen-op-decls "-I${CMAKE_CURRENT_SOURCE_DIR}/..")
-mlir_tablegen(Ops.cpp.inc -gen-op-defs "-I${CMAKE_CURRENT_SOURCE_DIR}/..")
+mlir_tablegen(Ops.h.inc -gen-op-decls)
+mlir_tablegen(Ops.cpp.inc -gen-op-defs)
 add_public_tablegen_target(ToyCh4OpsIncGen)
 
 set(LLVM_TARGET_DEFINITIONS ShapeInferenceInterface.td)
diff --git a/mlir/examples/toy/Ch5/CMakeLists.txt b/mlir/examples/toy/Ch5/CMakeLists.txt
index c3627cf11cd71..ba3a88e03c0b9 100644
--- a/mlir/examples/toy/Ch5/CMakeLists.txt
+++ b/mlir/examples/toy/Ch5/CMakeLists.txt
@@ -1,3 +1,4 @@
+include_directories(include)
 add_subdirectory(include)
 
 set(LLVM_LINK_COMPONENTS
@@ -5,7 +6,7 @@ set(LLVM_LINK_COMPONENTS
   )
 
 set(LLVM_TARGET_DEFINITIONS mlir/ToyCombine.td)
-mlir_tablegen(ToyCombine.inc -gen-rewriters "-I${CMAKE_CURRENT_SOURCE_DIR}/include")
+mlir_tablegen(ToyCombine.inc -gen-rewriters)
 add_public_tablegen_target(ToyCh5CombineIncGen)
 
 add_toy_chapter(toyc-ch5
@@ -23,7 +24,6 @@ add_toy_chapter(toyc-ch5
   ToyCh5CombineIncGen
   )
 
-include_directories(include/)
 include_directories(${CMAKE_CURRENT_BINARY_DIR})
 include_directories(${CMAKE_CURRENT_BINARY_DIR}/include/)
 get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
diff --git a/mlir/examples/toy/Ch5/include/toy/CMakeLists.txt b/mlir/examples/toy/Ch5/include/toy/CMakeLists.txt
index aaa932896d0f1..e8bd1fc0bc2ef 100644
--- a/mlir/examples/toy/Ch5/include/toy/CMakeLists.txt
+++ b/mlir/examples/toy/Ch5/include/toy/CMakeLists.txt
@@ -1,6 +1,6 @@
 set(LLVM_TARGET_DEFINITIONS Ops.td)
-mlir_tablegen(Ops.h.inc -gen-op-decls "-I${CMAKE_CURRENT_SOURCE_DIR}/..")
-mlir_tablegen(Ops.cpp.inc -gen-op-defs "-I${CMAKE_CURRENT_SOURCE_DIR}/..")
+mlir_tablegen(Ops.h.inc -gen-op-decls)
+mlir_tablegen(Ops.cpp.inc -gen-op-defs)
 add_public_tablegen_target(ToyCh5OpsIncGen)
 
 set(LLVM_TARGET_DEFINITIONS ShapeInferenceInterface.td)
diff --git a/mlir/examples/toy/Ch6/CMakeLists.txt b/mlir/examples/toy/Ch6/CMakeLists.txt
index 5d39a9ab5fc15..be797c6c1e965 100644
--- a/mlir/examples/toy/Ch6/CMakeLists.txt
+++ b/mlir/examples/toy/Ch6/CMakeLists.txt
@@ -1,3 +1,4 @@
+include_directories(include)
 add_subdirectory(include)
 
 set(LLVM_LINK_COMPONENTS
@@ -6,7 +7,7 @@ set(LLVM_LINK_COMPONENTS
   )
 
 set(LLVM_TARGET_DEFINITIONS mlir/ToyCombine.td)
-mlir_tablegen(ToyCombine.inc -gen-rewriters "-I${CMAKE_CURRENT_SOURCE_DIR}/include")
+mlir_tablegen(ToyCombine.inc -gen-rewriters)
 add_public_tablegen_target(ToyCh6CombineIncGen)
 
 add_toy_chapter(toyc-ch6
@@ -25,7 +26,6 @@ add_toy_chapter(toyc-ch6
   ToyCh6CombineIncGen
   )
 
-include_directories(include/)
 include_directories(${CMAKE_CURRENT_BINARY_DIR})
 include_directories(${CMAKE_CURRENT_BINARY_DIR}/include/)
 get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
diff --git a/mlir/examples/toy/Ch6/include/toy/CMakeLists.txt b/mlir/examples/toy/Ch6/include/toy/CMakeLists.txt
index aecf11fab6c94..c6adf5a15a736 100644
--- a/mlir/examples/toy/Ch6/include/toy/CMakeLists.txt
+++ b/mlir/examples/toy/Ch6/include/toy/CMakeLists.txt
@@ -1,6 +1,6 @@
 set(LLVM_TARGET_DEFINITIONS Ops.td)
-mlir_tablegen(Ops.h.inc -gen-op-decls "-I${CMAKE_CURRENT_SOURCE_DIR}/..")
-mlir_tablegen(Ops.cpp.inc -gen-op-defs "-I${CMAKE_CURRENT_SOURCE_DIR}/..")
+mlir_tablegen(Ops.h.inc -gen-op-decls)
+mlir_tablegen(Ops.cpp.inc -gen-op-defs)
 add_public_tablegen_target(ToyCh6OpsIncGen)
 
 set(LLVM_TARGET_DEFINITIONS ShapeInferenceInterface.td)
diff --git a/mlir/examples/toy/Ch7/CMakeLists.txt b/mlir/examples/toy/Ch7/CMakeLists.txt
index 760052ee840f0..9a9f335d3a920 100644
--- a/mlir/examples/toy/Ch7/CMakeLists.txt
+++ b/mlir/examples/toy/Ch7/CMakeLists.txt
@@ -1,3 +1,4 @@
+include_directories(include)
 add_subdirectory(include)
 
 set(LLVM_LINK_COMPONENTS
@@ -6,7 +7,7 @@ set(LLVM_LINK_COMPONENTS
   )
 
 set(LLVM_TARGET_DEFINITIONS mlir/ToyCombine.td)
-mlir_tablegen(ToyCombine.inc -gen-rewriters "-I${CMAKE_CURRENT_SOURCE_DIR}/include")
+mlir_tablegen(ToyCombine.inc -gen-rewriters)
 add_public_tablegen_target(ToyCh7CombineIncGen)
 
 add_toy_chapter(toyc-ch7
@@ -25,7 +26,6 @@ add_toy_chapter(toyc-ch7
   ToyCh7CombineIncGen
   )
 
-include_directories(include/)
 include_directories(${CMAKE_CURRENT_BINARY_DIR})
 include_directories(${CMAKE_CURRENT_BINARY_DIR}/include/)
 get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
diff --git a/mlir/examples/toy/Ch7/include/toy/CMakeLists.txt b/mlir/examples/toy/Ch7/include/toy/CMakeLists.txt
index fa30bd2e8e03e..43eb23bf93b8c 100644
--- a/mlir/examples/toy/Ch7/include/toy/CMakeLists.txt
+++ b/mlir/examples/toy/Ch7/include/toy/CMakeLists.txt
@@ -1,6 +1,6 @@
 set(LLVM_TARGET_DEFINITIONS Ops.td)
-mlir_tablegen(Ops.h.inc -gen-op-decls "-I${CMAKE_CURRENT_SOURCE_DIR}/..")
-mlir_tablegen(Ops.cpp.inc -gen-op-defs "-I${CMAKE_CURRENT_SOURCE_DIR}/..")
+mlir_tablegen(Ops.h.inc -gen-op-decls)
+mlir_tablegen(Ops.cpp.inc -gen-op-defs)
 add_public_tablegen_target(ToyCh7OpsIncGen)
 
 set(LLVM_TARGET_DEFINITIONS ShapeInferenceInterface.td)
diff --git a/mlir/include/mlir/Analysis/AffineStructures.h b/mlir/include/mlir/Analysis/AffineStructures.h
index 5d99320c28601..cd31952308346 100644
--- a/mlir/include/mlir/Analysis/AffineStructures.h
+++ b/mlir/include/mlir/Analysis/AffineStructures.h
@@ -210,6 +210,21 @@ class FlatAffineConstraints {
                                      ValueRange operands, bool eq,
                                      bool lower = true);
 
+  /// Returns the bound for the identifier at `pos` from the inequality at
+  /// `ineqPos` as a 1-d affine value map (affine map + operands). The returned
+  /// affine value map can either be a lower bound or an upper bound depending
+  /// on the sign of atIneq(ineqPos, pos). Asserts if the row at `ineqPos` does
+  /// not involve the `pos`th identifier.
+  void getIneqAsAffineValueMap(unsigned pos, unsigned ineqPos,
+                               AffineValueMap &vmap,
+                               MLIRContext *context) const;
+
+  /// Returns the constraint system as an integer set. Returns a null integer
+  /// set if the system has no constraints, or if an integer set couldn't be
+  /// constructed as a result of a local variable's explicit representation not
+  /// being known and such a local variable appearing in any of the constraints.
+  IntegerSet getAsIntegerSet(MLIRContext *context) const;
+
   /// Computes the lower and upper bounds of the first 'num' dimensional
   /// identifiers (starting at 'offset') as an affine map of the remaining
   /// identifiers (dimensional and symbolic). This method is able to detect
@@ -446,15 +461,17 @@ class FlatAffineConstraints {
   /// affine expressions involving only the symbolic identifiers. `lb` and
   /// `ub` (along with the `boundFloorDivisor`) are set to represent the lower
   /// and upper bound associated with the constant difference: `lb`, `ub` have
-  /// the coefficients, and boundFloorDivisor, their divisor.
+  /// the coefficients, and boundFloorDivisor, their divisor. `minLbPos` and
+  /// `minUbPos` if non-null are set to the position of the constant lower bound
+  /// and upper bound respectively (to the same if they are from an equality).
   /// Ex: if the lower bound is [(s0 + s2 - 1) floordiv 32] for a system with
-  /// three symbolic identifiers, *lb = [1, 0, 1], boundDivisor = 32. See
-  /// comments at function definition for examples.
-  Optional<int64_t>
-  getConstantBoundOnDimSize(unsigned pos,
-                            SmallVectorImpl<int64_t> *lb = nullptr,
-                            int64_t *boundFloorDivisor = nullptr,
-                            SmallVectorImpl<int64_t> *ub = nullptr) const;
+  /// three symbolic identifiers, *lb = [1, 0, 1], lbDivisor = 32. See comments
+  /// at function definition for examples.
+  Optional<int64_t> getConstantBoundOnDimSize(
+      unsigned pos, SmallVectorImpl<int64_t> *lb = nullptr,
+      int64_t *boundFloorDivisor = nullptr,
+      SmallVectorImpl<int64_t> *ub = nullptr, unsigned *minLbPos = nullptr,
+      unsigned *minUbPos = nullptr) const;
 
   /// Returns the constant lower bound for the pos^th identifier if there is
   /// one; None otherwise.
@@ -476,6 +493,20 @@ class FlatAffineConstraints {
                         unsigned symStartPos, ArrayRef<AffineExpr> localExprs,
                         MLIRContext *context) const;
 
+  /// Gather positions of all lower and upper bounds of the identifier at `pos`,
+  /// and optionally any equalities on it. In addition, the bounds are to be
+  /// independent of identifiers in position range [`offset`, `offset` + `num`).
+  void
+  getLowerAndUpperBoundIndices(unsigned pos,
+                               SmallVectorImpl<unsigned> *lbIndices,
+                               SmallVectorImpl<unsigned> *ubIndices,
+                               SmallVectorImpl<unsigned> *eqIndices = nullptr,
+                               unsigned offset = 0, unsigned num = 0) const;
+
+  /// Removes constraints that are independent of (i.e., do not have a
+  /// coefficient for) for identifiers in the range [pos, pos + num).
+  void removeIndependentConstraints(unsigned pos, unsigned num);
+
   /// Returns true if the set can be trivially detected as being
   /// hyper-rectangular on the specified contiguous set of identifiers.
   bool isHyperRectangular(unsigned pos, unsigned num) const;
@@ -484,7 +515,8 @@ class FlatAffineConstraints {
   /// that can be detected as redundant as a result of differing only in their
   /// constant term part. A constraint of the form <non-negative constant> >= 0
   /// is considered trivially true. This method is a linear time method on the
-  /// constraints, does a single scan, and updates in place.
+  /// constraints, does a single scan, and updates in place. It also normalizes
+  /// constraints by their GCD and performs GCD tightening on inequalities.
   void removeTrivialRedundancy();
 
   /// A more expensive check to detect redundant inequalities thatn
diff --git a/mlir/include/mlir/Analysis/Dominance.h b/mlir/include/mlir/Analysis/Dominance.h
index 99e9eae231892..44d8e02a66701 100644
--- a/mlir/include/mlir/Analysis/Dominance.h
+++ b/mlir/include/mlir/Analysis/Dominance.h
@@ -34,12 +34,20 @@ template <bool IsPostDom> class DominanceInfoBase {
   /// Recalculate the dominance info.
   void recalculate(Operation *op);
 
+  /// Finds the nearest common dominator block for the two given blocks a
+  /// and b. If no common dominator can be found, this function will return
+  /// nullptr.
+  Block *findNearestCommonDominator(Block *a, Block *b) const;
+
   /// Get the root dominance node of the given region.
   DominanceInfoNode *getRootNode(Region *region) {
     assert(dominanceInfos.count(region) != 0);
     return dominanceInfos[region]->getRootNode();
   }
 
+  /// Return the dominance node from the Region containing block A.
+  DominanceInfoNode *getNode(Block *a);
+
 protected:
   using super = DominanceInfoBase<IsPostDom>;
 
@@ -82,9 +90,6 @@ class DominanceInfo : public detail::DominanceInfoBase</*IsPostDom=*/false> {
     return super::properlyDominates(a, b);
   }
 
-  /// Return the dominance node from the Region containing block A.
-  DominanceInfoNode *getNode(Block *a);
-
   /// Update the internal DFS numbers for the dominance nodes.
   void updateDFSNumbers();
 };
diff --git a/mlir/include/mlir/Analysis/LoopAnalysis.h b/mlir/include/mlir/Analysis/LoopAnalysis.h
index 5141df736a9cc..7ed19ef99f871 100644
--- a/mlir/include/mlir/Analysis/LoopAnalysis.h
+++ b/mlir/include/mlir/Analysis/LoopAnalysis.h
@@ -82,7 +82,7 @@ bool isVectorizableLoopBody(AffineForOp loop, int *memRefDim,
 /// 'def' and all its uses have the same shift factor.
 // TODO(mlir-team): extend this to check for memory-based dependence
 // violation when we have the support.
-bool isInstwiseShiftValid(AffineForOp forOp, ArrayRef<uint64_t> shifts);
+bool isOpwiseShiftValid(AffineForOp forOp, ArrayRef<uint64_t> shifts);
 } // end namespace mlir
 
 #endif // MLIR_ANALYSIS_LOOP_ANALYSIS_H
diff --git a/mlir/include/mlir/Analysis/Utils.h b/mlir/include/mlir/Analysis/Utils.h
index 3a40b85cea05b..9b69c6f61a4b9 100644
--- a/mlir/include/mlir/Analysis/Utils.h
+++ b/mlir/include/mlir/Analysis/Utils.h
@@ -290,6 +290,12 @@ Optional<int64_t> getMemoryFootprintBytes(AffineForOp forOp,
 /// Returns true if `forOp' is a parallel loop.
 bool isLoopParallel(AffineForOp forOp);
 
+/// Simplify the integer set by simplifying the underlying affine expressions by
+/// flattening and some simple inference. Also, drop any duplicate constraints.
+/// Returns the simplified integer set. This method runs in time linear in the
+/// number of constraints.
+IntegerSet simplifyIntegerSet(IntegerSet set);
+
 } // end namespace mlir
 
 #endif // MLIR_ANALYSIS_UTILS_H
diff --git a/mlir/include/mlir/CMakeLists.txt b/mlir/include/mlir/CMakeLists.txt
index 4754391dc39fd..594dc6180f504 100644
--- a/mlir/include/mlir/CMakeLists.txt
+++ b/mlir/include/mlir/CMakeLists.txt
@@ -1,3 +1,6 @@
+add_subdirectory(Conversion)
 add_subdirectory(Dialect)
 add_subdirectory(IR)
 add_subdirectory(Interfaces)
+add_subdirectory(Quantizer)
+add_subdirectory(Transforms)
diff --git a/mlir/include/mlir/Conversion/CMakeLists.txt b/mlir/include/mlir/Conversion/CMakeLists.txt
new file mode 100644
index 0000000000000..d4ce2634f4505
--- /dev/null
+++ b/mlir/include/mlir/Conversion/CMakeLists.txt
@@ -0,0 +1,6 @@
+
+set(LLVM_TARGET_DEFINITIONS Passes.td)
+mlir_tablegen(Passes.h.inc -gen-pass-decls)
+add_public_tablegen_target(MLIRConversionPassIncGen)
+
+add_mlir_doc(Passes -gen-pass-doc ConversionPasses ./)
diff --git a/mlir/include/mlir/Conversion/LoopsToGPU/LoopsToGPUPass.h b/mlir/include/mlir/Conversion/LoopsToGPU/LoopsToGPUPass.h
index d5f48d29ea6c8..049e8538d746c 100644
--- a/mlir/include/mlir/Conversion/LoopsToGPU/LoopsToGPUPass.h
+++ b/mlir/include/mlir/Conversion/LoopsToGPU/LoopsToGPUPass.h
@@ -27,6 +27,7 @@ class Pass;
 /// calling the conversion.
 std::unique_ptr<OpPassBase<FuncOp>>
 createSimpleLoopsToGPUPass(unsigned numBlockDims, unsigned numThreadDims);
+std::unique_ptr<OpPassBase<FuncOp>> createSimpleLoopsToGPUPass();
 
 /// Create a pass that converts every loop operation within the body of the
 /// FuncOp into a GPU launch. The number of workgroups and workgroup size for
@@ -37,6 +38,7 @@ createSimpleLoopsToGPUPass(unsigned numBlockDims, unsigned numThreadDims);
 std::unique_ptr<OpPassBase<FuncOp>>
 createLoopToGPUPass(ArrayRef<int64_t> numWorkGroups,
                     ArrayRef<int64_t> workGroupSize);
+std::unique_ptr<OpPassBase<FuncOp>> createLoopToGPUPass();
 
 /// Creates a pass that converts loop.parallel operations into a gpu.launch
 /// operation. The mapping of loop dimensions to launch dimensions is derived
diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td
new file mode 100644
index 0000000000000..5553655fafaeb
--- /dev/null
+++ b/mlir/include/mlir/Conversion/Passes.td
@@ -0,0 +1,267 @@
+//===-- Passes.td - Conversion pass definition file --------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_CONVERSION_PASSES
+#define MLIR_CONVERSION_PASSES
+
+include "mlir/Pass/PassBase.td"
+
+//===----------------------------------------------------------------------===//
+// AffineToStandard
+//===----------------------------------------------------------------------===//
+
+def ConvertAffineToStandard : Pass<"lower-affine"> {
+  let summary = "Lower Affine operations to a combination of Standard and Loop "
+                "operations";
+  let description = [{
+
+    Convert operations from the affine dialect into operations from the loop and
+    standard dialects.
+
+    `affine.for` operations are converted to `loop.for` operations that are free
+    of certain structural restrictions (on their bounds and step). `affine.if`
+    is similarly converted to the `loop.if` operation. `affine.apply` operations
+    are converted into sequences of primitive arithmetic operations from the
+    standard dialect that have the same effect, using operands of the `index`
+    type. Consequently, named maps and sets thare are no longer in use may be
+    removed from the module.
+
+    For example, `%r = affine.apply affine_map<(d0, d1)[s0] -> (d0 + 2*d1 +
+    s0)>(%d0, %d1)[%s0]`
+    can be converted into:
+
+    ```mlir
+    %d0 = <...>
+    %d1 = <...>
+    %s0 = <...>
+    %0 = constant 2 : index
+    %1 = muli %0, %d1
+    %2 = addi %d0, %1
+    %r = addi %2, %s0
+    ```
+
+    #### Input invariant
+
+    -   no `Tensor` types;
+
+    These restrictions may be lifted in the future.
+
+    #### Output IR
+
+    Functions with `affine.for` and `affine.if` operations eliminated. These
+    functions may contain operations from the Standard dialect in addition to
+    those already present before the pass.
+
+    #### Invariants
+
+    -   Functions without a body are not modified.
+    -   The semantics of the other functions is preserved.
+    -   Individual operations other than those mentioned above are not modified
+        if they do not depend on the loop iterator value or on the result of
+        `affine.apply`.
+  }];
+  let constructor = "mlir::createLowerAffinePass()";
+}
+
+//===----------------------------------------------------------------------===//
+// AVX512ToLLVM
+//===----------------------------------------------------------------------===//
+
+def ConvertAVX512ToLLVM : Pass<"convert-avx512-to-llvm"> {
+  let summary = "Convert the operations from the avx512 dialect into the LLVM "
+                "dialect";
+  let constructor = "mlir::createConvertAVX512ToLLVMPass()";
+}
+
+//===----------------------------------------------------------------------===//
+// GPUToCUDA
+//===----------------------------------------------------------------------===//
+
+def ConvertGpuLaunchFuncToCudaCalls : Pass<"launch-func-to-cuda"> {
+  let summary = "Convert all launch_func ops to CUDA runtime calls";
+  let constructor = "mlir::createConvertGpuLaunchFuncToCudaCallsPass()";
+}
+
+//===----------------------------------------------------------------------===//
+// GPUToNVVM
+//===----------------------------------------------------------------------===//
+
+def ConvertGpuOpsToNVVMOps : Pass<"convert-gpu-to-nvvm"> {
+  let summary = "Generate NVVM operations for gpu operations";
+  let constructor = "mlir::createLowerGpuOpsToNVVMOpsPass()";
+}
+
+//===----------------------------------------------------------------------===//
+// GPUToROCDL
+//===----------------------------------------------------------------------===//
+
+def ConvertGpuOpsToROCDLOps : Pass<"convert-gpu-to-rocdl"> {
+  let summary = "Generate ROCDL operations for gpu operations";
+  let constructor = "mlir::createLowerGpuOpsToROCDLOpsPass()";
+}
+
+//===----------------------------------------------------------------------===//
+// GPUToSPIRV
+//===----------------------------------------------------------------------===//
+
+def ConvertGPUToSPIRV : Pass<"convert-gpu-to-spirv"> {
+  let summary = "Convert GPU dialect to SPIR-V dialect";
+  let constructor = "mlir::createConvertGPUToSPIRVPass()";
+}
+
+//===----------------------------------------------------------------------===//
+// GPUToVulkan
+//===----------------------------------------------------------------------===//
+
+def ConvertGpuLaunchFuncToVulkanLaunchFunc
+    : Pass<"convert-gpu-launch-to-vulkan-launch"> {
+  let summary = "Convert gpu.launch_func to vulkanLaunch external call";
+  let constructor = "mlir::createConvertGpuLaunchFuncToVulkanLaunchFuncPass()";
+}
+
+def ConvertVulkanLaunchFuncToVulkanCalls : Pass<"launch-func-to-vulkan"> {
+  let summary = "Convert vulkanLaunch external call to Vulkan runtime external "
+                "calls";
+  let constructor = "mlir::createConvertVulkanLaunchFuncToVulkanCallsPass()";
+}
+
+//===----------------------------------------------------------------------===//
+// LinalgToLLVM
+//===----------------------------------------------------------------------===//
+
+def ConvertLinalgToLLVM : Pass<"convert-linalg-to-llvm"> {
+  let summary = "Convert the operations from the linalg dialect into the LLVM "
+                "dialect";
+  let constructor = "mlir::createConvertLinalgToLLVMPass()";
+}
+
+//===----------------------------------------------------------------------===//
+// LinalgToSPIRV
+//===----------------------------------------------------------------------===//
+
+def ConvertLinalgToSPIRV : Pass<"convert-linalg-to-spirv"> {
+  let summary = "Convert Linalg ops to SPIR-V ops";
+  let constructor = "mlir::createLinalgToSPIRVPass()";
+}
+
+//===----------------------------------------------------------------------===//
+// LoopToStandard
+//===----------------------------------------------------------------------===//
+
+def ConvertLoopToStandard : Pass<"convert-loop-to-std"> {
+  let summary = "Convert Loop dialect to Standard dialect, replacing structured"
+                " control flow with a CFG";
+  let constructor = "mlir::createLowerToCFGPass()";
+}
+
+//===----------------------------------------------------------------------===//
+// LoopsToGPU
+//===----------------------------------------------------------------------===//
+
+def ConvertSimpleLoopsToGPU : Pass<"convert-loops-to-gpu"> {
+  let summary = "Convert top-level loops to GPU kernels";
+  let constructor = "mlir::createSimpleLoopsToGPUPass()";
+  let options = [
+    Option<"numBlockDims", "gpu-block-dims", "unsigned", /*default=*/"1u",
+           "Number of GPU block dimensions for mapping">,
+    Option<"numThreadDims", "gpu-thread-dims", "unsigned", /*default=*/"1u",
+           "Number of GPU thread dimensions for mapping">
+  ];
+}
+
+def ConvertLoopsToGPU : Pass<"convert-loop-op-to-gpu"> {
+  let summary = "Convert top-level loop::ForOp to GPU kernels";
+  let constructor = "mlir::createLoopToGPUPass()";
+  let options = [
+    ListOption<"numWorkGroups", "gpu-num-workgroups", "int64_t",
+               "Num workgroups in the GPU launch",
+               "llvm::cl::ZeroOrMore, llvm::cl::MiscFlags::CommaSeparated">,
+    ListOption<"workGroupSize", "gpu-workgroup-size", "int64_t",
+               "Workgroup Size in the GPU launch",
+               "llvm::cl::ZeroOrMore, llvm::cl::MiscFlags::CommaSeparated">
+  ];
+}
+
+def ConvertParallelLoopToGpu : Pass<"convert-parallel-loops-to-gpu"> {
+  let summary = "Convert mapped loop.parallel ops to gpu launch operations";
+  let constructor = "mlir::createParallelLoopToGpuPass()";
+}
+
+//===----------------------------------------------------------------------===//
+// StandardToLLVM
+//===----------------------------------------------------------------------===//
+
+def ConvertStandardToLLVM : Pass<"convert-std-to-llvm"> {
+  let summary = "Convert scalar and vector operations from the Standard to the "
+                "LLVM dialect";
+  let description = [{
+    Convert standard operations into the LLVM IR dialect operations.
+
+    #### Input invariant
+
+    -   operations including: arithmetic on integers and floats, constants,
+        direct calls, returns and branches;
+    -   no `tensor` types;
+    -   all `vector` are one-dimensional;
+    -   all blocks are reachable by following the successors of the first basic
+        block;
+
+    If other operations are present and their results are required by the LLVM
+    IR dialect operations, the pass will fail.  Any LLVM IR operations or types
+    already present in the IR will be kept as is.
+
+    #### Output IR
+
+    Functions converted to LLVM IR. Function arguments types are converted
+    one-to-one. Function results are converted one-to-one and, in case more than
+    1 value is returned, packed into an LLVM IR struct type. Function calls and
+    returns are updated accordingly. Block argument types are updated to use
+    LLVM IR types.
+  }];
+  let constructor = "mlir::createLowerToLLVMPass()";
+  let options = [
+    Option<"useAlloca", "use-alloca", "bool", /*default=*/"false",
+           "Use `alloca` instead of `call @malloc` for converting std.alloc">,
+    Option<"useBarePtrCallConv", "use-bare-ptr-memref-call-conv", "bool",
+           /*default=*/"false",
+           "Replace FuncOp's MemRef arguments with bare pointers to the MemRef "
+           "element types">,
+    Option<"emitCWrappers", "emit-c-wrappers", "bool", /*default=*/"false",
+           "Emit wrappers for C-compatible pointer-to-struct memref "
+           "descriptors">,
+    Option<"indexBitwidth", "index-bitwidth", "unsigned",
+           /*default=*/"kDeriveIndexBitwidthFromDataLayout",
+           "Bitwidth of the index type, 0 to use size of machine word">,
+  ];
+}
+
+//===----------------------------------------------------------------------===//
+// StandardToSPIRV
+//===----------------------------------------------------------------------===//
+
+def LegalizeStandardForSPIRV : Pass<"legalize-std-for-spirv"> {
+  let summary = "Legalize standard ops for SPIR-V lowering";
+  let constructor = "mlir::createLegalizeStdOpsForSPIRVLoweringPass()";
+}
+
+def ConvertStandardToSPIRV : Pass<"convert-std-to-spirv"> {
+  let summary = "Convert Standard Ops to SPIR-V dialect";
+  let constructor = "mlir::createConvertStandardToSPIRVPass()";
+}
+
+//===----------------------------------------------------------------------===//
+// VectorToLLVM
+//===----------------------------------------------------------------------===//
+
+def ConvertVectorToLLVM : Pass<"convert-vector-to-llvm"> {
+  let summary = "Lower the operations from the vector dialect into the LLVM "
+                "dialect";
+  let constructor = "mlir::createConvertVectorToLLVMPass()";
+}
+
+#endif // MLIR_CONVERSION_PASSES
diff --git a/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h b/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h
index 2e6a513f10dba..e1add00c5d990 100644
--- a/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h
+++ b/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h
@@ -42,6 +42,10 @@ struct LLVMTypeConverterCustomization {
   /// Customize the type conversion of function arguments.
   CustomCallback funcArgConverter;
 
+  /// Used to determine the bitwidth of the LLVM integer type that the index
+  /// type gets lowered to. Defaults to deriving the size from the data layout.
+  unsigned indexBitwidth;
+
   /// Initialize customization to default callbacks.
   LLVMTypeConverterCustomization();
 };
@@ -90,6 +94,9 @@ class LLVMTypeConverter : public TypeConverter {
   /// to each of the MLIR types converted with `convertType`.
   Type packFunctionResults(ArrayRef<Type> types);
 
+  /// Returns the MLIR context.
+  MLIRContext &getContext();
+
   /// Returns the LLVM context.
   llvm::LLVMContext &getLLVMContext();
 
@@ -119,6 +126,13 @@ class LLVMTypeConverter : public TypeConverter {
                                    ArrayRef<Value> values,
                                    Location loc) override;
 
+  /// Gets the LLVM representation of the index type. The returned type is an
+  /// integer type with the size confgured for this type converter.
+  LLVM::LLVMType getIndexType();
+
+  /// Gets the bitwidth of the index type when converted to LLVM.
+  unsigned getIndexTypeBitwidth() { return customizations.indexBitwidth; }
+
 protected:
   /// LLVM IR module used to parse/create types.
   llvm::Module *module;
@@ -178,10 +192,6 @@ class LLVMTypeConverter : public TypeConverter {
   // Convert a 1D vector type into an LLVM vector type.
   Type convertVectorType(VectorType type);
 
-  // Get the LLVM representation of the index type based on the bitwidth of the
-  // pointer as defined by the data layout of the module.
-  LLVM::LLVMType getIndexType();
-
   /// Callbacks for customizing the type conversion.
   LLVMTypeConverterCustomization customizations;
 };
@@ -356,6 +366,7 @@ class UnrankedMemRefDescriptor : public StructBuilder {
   /// `unpack`.
   static unsigned getNumUnpackedValues() { return 2; }
 };
+
 /// Base class for operation conversions targeting the LLVM IR dialect. Provides
 /// conversion patterns with access to an LLVMTypeConverter.
 class ConvertToLLVMPattern : public ConversionPattern {
@@ -364,11 +375,107 @@ class ConvertToLLVMPattern : public ConversionPattern {
                        LLVMTypeConverter &typeConverter,
                        PatternBenefit benefit = 1);
 
+  /// Returns the LLVM dialect.
+  LLVM::LLVMDialect &getDialect() const;
+
+  /// Returns the LLVM IR context.
+  llvm::LLVMContext &getContext() const;
+
+  /// Returns the LLVM IR module associated with the LLVM dialect.
+  llvm::Module &getModule() const;
+
+  /// Gets the MLIR type wrapping the LLVM integer type whose bit width is
+  /// defined by the used type converter.
+  LLVM::LLVMType getIndexType() const;
+
+  /// Gets the MLIR type wrapping the LLVM void type.
+  LLVM::LLVMType getVoidType() const;
+
+  /// Get the MLIR type wrapping the LLVM i8* type.
+  LLVM::LLVMType getVoidPtrType() const;
+
+  /// Create an LLVM dialect operation defining the given index constant.
+  Value createIndexConstant(ConversionPatternRewriter &builder, Location loc,
+                            uint64_t value) const;
+
 protected:
   /// Reference to the type converter, with potential extensions.
   LLVMTypeConverter &typeConverter;
 };
 
+/// Utility class for operation conversions targeting the LLVM dialect that
+/// match exactly one source operation.
+template <typename OpTy>
+class ConvertOpToLLVMPattern : public ConvertToLLVMPattern {
+public:
+  ConvertOpToLLVMPattern(LLVMTypeConverter &typeConverter,
+                         PatternBenefit benefit = 1)
+      : ConvertToLLVMPattern(OpTy::getOperationName(),
+                             &typeConverter.getContext(), typeConverter,
+                             benefit) {}
+};
+
+namespace LLVM {
+namespace detail {
+/// Replaces the given operaiton "op" with a new operation of type "targetOp"
+/// and given operands.
+LogicalResult oneToOneRewrite(Operation *op, StringRef targetOp,
+                              ValueRange operands,
+                              LLVMTypeConverter &typeConverter,
+                              ConversionPatternRewriter &rewriter);
+
+LogicalResult vectorOneToOneRewrite(Operation *op, StringRef targetOp,
+                                    ValueRange operands,
+                                    LLVMTypeConverter &typeConverter,
+                                    ConversionPatternRewriter &rewriter);
+} // namespace detail
+} // namespace LLVM
+
+/// Generic implementation of one-to-one conversion from "SourceOp" to
+/// "TargetOp" where the latter belongs to the LLVM dialect or an equivalent.
+/// Upholds a convention that multi-result operations get converted into an
+/// operation returning the LLVM IR structure type, in which case individual
+/// values must be extacted from using LLVM::ExtractValueOp before being used.
+template <typename SourceOp, typename TargetOp>
+class OneToOneConvertToLLVMPattern : public ConvertOpToLLVMPattern<SourceOp> {
+public:
+  using ConvertOpToLLVMPattern<SourceOp>::ConvertOpToLLVMPattern;
+  using Super = OneToOneConvertToLLVMPattern<SourceOp, TargetOp>;
+
+  /// Converts the type of the result to an LLVM type, pass operands as is,
+  /// preserve attributes.
+  LogicalResult
+  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    return LLVM::detail::oneToOneRewrite(op, TargetOp::getOperationName(),
+                                         operands, this->typeConverter,
+                                         rewriter);
+  }
+};
+
+/// Basic lowering implementation for rewriting from Ops to LLVM Dialect Ops
+/// with one result. This supports higher-dimensional vector types.
+template <typename SourceOp, typename TargetOp>
+class VectorConvertToLLVMPattern : public ConvertOpToLLVMPattern<SourceOp> {
+public:
+  using ConvertOpToLLVMPattern<SourceOp>::ConvertOpToLLVMPattern;
+  using Super = VectorConvertToLLVMPattern<SourceOp, TargetOp>;
+
+  LogicalResult
+  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    static_assert(
+        std::is_base_of<OpTrait::OneResult<SourceOp>, SourceOp>::value,
+        "expected single result op");
+    static_assert(std::is_base_of<OpTrait::SameOperandsAndResultType<SourceOp>,
+                                  SourceOp>::value,
+                  "expected same operands and result type");
+    return LLVM::detail::vectorOneToOneRewrite(op, TargetOp::getOperationName(),
+                                               operands, this->typeConverter,
+                                               rewriter);
+  }
+};
+
 /// Derived class that automatically populates legalization information for
 /// different LLVM ops.
 class LLVMConversionTarget : public ConversionTarget {
diff --git a/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h b/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h
index dd815b381cc93..c3c104bf9d21c 100644
--- a/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h
+++ b/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h
@@ -53,13 +53,18 @@ void populateStdToLLVMBarePtrConversionPatterns(
     LLVMTypeConverter &converter, OwningRewritePatternList &patterns,
     bool useAlloca = false);
 
+/// Value to pass as bitwidth for the index type when the converter is expected
+/// to derive the bitwith from the LLVM data layout.
+static constexpr unsigned kDeriveIndexBitwidthFromDataLayout = 0;
+
 /// Creates a pass to convert the Standard dialect into the LLVMIR dialect.
 /// By default stdlib malloc/free are used for allocating MemRef payloads.
 /// Specifying `useAlloca-true` emits stack allocations instead. In the future
 /// this may become an enum when we have concrete uses for other options.
-std::unique_ptr<OpPassBase<ModuleOp>>
-createLowerToLLVMPass(bool useAlloca = false, bool useBarePtrCallConv = false,
-                      bool emitCWrappers = false);
+std::unique_ptr<OpPassBase<ModuleOp>> createLowerToLLVMPass(
+    bool useAlloca = false, bool useBarePtrCallConv = false,
+    bool emitCWrappers = false,
+    unsigned indexBitwidth = kDeriveIndexBitwidthFromDataLayout);
 
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Conversion/VectorToLoops/ConvertVectorToLoops.h b/mlir/include/mlir/Conversion/VectorToLoops/ConvertVectorToLoops.h
index 5c45a3c32ca06..e96b7cfe76e54 100644
--- a/mlir/include/mlir/Conversion/VectorToLoops/ConvertVectorToLoops.h
+++ b/mlir/include/mlir/Conversion/VectorToLoops/ConvertVectorToLoops.h
@@ -5,23 +5,18 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+
 #ifndef MLIR_CONVERSION_VECTORTOLLVM_CONVERTVECTORTOLOOPS_H_
 #define MLIR_CONVERSION_VECTORTOLLVM_CONVERTVECTORTOLOOPS_H_
 
-#include "mlir/Transforms/DialectConversion.h"
-
 namespace mlir {
 class MLIRContext;
-class ModuleOp;
-template <typename T> class OpPassBase;
+class OwningRewritePatternList;
 
 /// Collect a set of patterns to convert from the Vector dialect to loops + std.
 void populateVectorToAffineLoopsConversionPatterns(
     MLIRContext *context, OwningRewritePatternList &patterns);
 
-/// Create a pass to convert vector operations to affine loops + std dialect.
-OpPassBase<ModuleOp> *createLowerVectorToLoopsPass();
-
 } // namespace mlir
 
 #endif // MLIR_CONVERSION_VECTORTOLLVM_CONVERTVECTORTOLOOPS_H_
diff --git a/mlir/include/mlir/Dialect/Affine/CMakeLists.txt b/mlir/include/mlir/Dialect/Affine/CMakeLists.txt
index f33061b2d87cf..404c926f60eda 100644
--- a/mlir/include/mlir/Dialect/Affine/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/Affine/CMakeLists.txt
@@ -1 +1,7 @@
 add_subdirectory(IR)
+
+set(LLVM_TARGET_DEFINITIONS Passes.td)
+mlir_tablegen(Passes.h.inc -gen-pass-decls)
+add_public_tablegen_target(MLIRAffinePassIncGen)
+
+add_mlir_doc(Passes -gen-pass-doc AffinePasses ./)
diff --git a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td
index 6994b5f17661d..aedd21d3f6e7e 100644
--- a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td
+++ b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td
@@ -44,22 +44,23 @@ def ImplicitAffineTerminator
 def AffineApplyOp : Affine_Op<"apply", [NoSideEffect]> {
   let summary = "affine apply operation";
   let description = [{
-    The affine.apply operation applies an affine mapping to a list of SSA
-    values, yielding a single SSA value. The number of dimension and symbol
-    arguments to affine.apply must be equal to the respective number of
-    dimensional and symbolic inputs to the affine mapping; the affine mapping
-    has to be one-dimensional, and so the affine.apply operation always returns
-    one value. The input operands and result must all have ‘index’ type.
+    The affine.apply operation applies an [affine mapping](#affine-expressions)
+    to a list of SSA values, yielding a single SSA value. The number of
+    dimension and symbol arguments to `affine.apply` must be equal to the
+    respective number of dimensional and symbolic inputs to the affine mapping;
+    the affine mapping has to be one-dimensional, and so the `affine.apply`
+    operation always returns one value. The input operands and result must all
+    have ‘index’ type.
 
     Example:
 
     ```mlir
-      #map10 = affine_map<(d0, d1) -> (d0 floordiv 8 + d1 floordiv 128)>
-      ...
-      %1 = affine.apply #map10 (%s, %t)
+    #map10 = affine_map<(d0, d1) -> (d0 floordiv 8 + d1 floordiv 128)>
+    ...
+    %1 = affine.apply #map10 (%s, %t)
 
-      // Inline example.
-      %2 = affine.apply affine_map<(i)[s0] -> (i+s0)> (%42)[%n]
+    // Inline example.
+    %2 = affine.apply affine_map<(i)[s0] -> (i+s0)> (%42)[%n]
     ```
   }];
   let arguments = (ins AffineMapAttr:$map, Variadic<Index>:$mapOperands);
@@ -100,33 +101,66 @@ def AffineForOp : Affine_Op<"for",
      DeclareOpInterfaceMethods<LoopLikeOpInterface>]> {
   let summary = "for operation";
   let description = [{
-    The "affine.for" operation represents an affine loop nest, defining an SSA
-    value for its induction variable. It has one region capturing the loop body.
-    The induction variable is represented as a argument of this region. This SSA
-    value always has type index, which is the size of the machine word. The
-    stride, represented by step, is a positive constant integer which defaults
-    to "1" if not present. The lower and upper bounds specify a half-open range:
-    the range includes the lower bound but does not include the upper bound.
-
-    The body region must contain exactly one block that terminates with
-    "affine.terminator".  Calling AffineForOp::build will create such region
-    and insert the terminator, so will the parsing even in cases if it is absent
-    from the custom format.
-
-    The lower and upper bounds of a for operation are represented as an
+    Syntax:
+
+    ```
+    operation   ::= `affine.for` ssa-id `=` lower-bound `to` upper-bound
+                    (`step` integer-literal)? `{` op* `}`
+
+    lower-bound ::= `max`? affine-map-attribute dim-and-symbol-use-list | shorthand-bound
+    upper-bound ::= `min`? affine-map-attribute dim-and-symbol-use-list | shorthand-bound
+    shorthand-bound ::= ssa-id | `-`? integer-literal
+    ```
+
+    The `affine.for` operation represents an affine loop nest. It has one region
+    containing its body. This region must contain one block that terminates with
+    [`affine.terminator`](#affineterminator-operation). *Note:* when
+    `affine.for` is printed in custom format, the terminator is omitted. The
+    block has one argument of [`index`](../LangRef.md#index-type) type that
+    represents the induction variable of the loop.
+
+    The `affine.for` operation executes its body a number of times iterating
+    from a lower bound to an upper bound by a stride. The stride, represented by
+    `step`, is a positive constant integer which defaults to "1" if not present.
+    The lower and upper bounds specify a half-open range: the range includes the
+    lower bound but does not include the upper bound.
+
+    The lower and upper bounds of a `affine.for` operation are represented as an
     application of an affine mapping to a list of SSA values passed to the map.
-    The same restrictions hold for these SSA values as for all bindings of SSA
-    values to dimensions and symbols. The affine mappings for the bounds may
-    return multiple results, in which case the max/min keywords are required
-    (for the lower/upper bound respectively), and the bound is the
-    maximum/minimum of the returned values.
+    The [same restrictions](#restrictions-on-dimensions-and-symbols) hold for
+    these SSA values as for all bindings of SSA values to dimensions and
+    symbols.
 
-    Example:
+    The affine mappings for the bounds may return multiple results, in which
+    case the `max`/`min` keywords are required (for the lower/upper bound
+    respectively), and the bound is the maximum/minimum of the returned values.
+    There is no semantic ambiguity, but MLIR syntax requires the use of these
+    keywords to make things more obvious to human readers.
 
-      affine.for %i = 1 to 10 {
-        ...
-      }
+    Many upper and lower bounds are simple, so MLIR accepts two custom form
+    syntaxes: the form that accepts a single 'ssa-id' (e.g. `%N`) is shorthand
+    for applying that SSA value to a function that maps a single symbol to
+    itself, e.g., `()[s]->(s)()[%N]`. The integer literal form (e.g. `-42`) is
+    shorthand for a nullary mapping function that returns the constant value
+    (e.g. `()->(-42)()`).
+
+    Example showing reverse iteration of the inner loop:
 
+    ```mlir
+    #map57 = affine_map<(d0)[s0] -> (s0 - d0 - 1)>
+
+    func @simple_example(%A: memref<?x?xf32>, %B: memref<?x?xf32>) {
+      %N = dim %A, 0 : memref<?x?xf32>
+      affine.for %i = 0 to %N step 1 {
+        affine.for %j = 0 to %N {   // implicitly steps by 1
+          %0 = affine.apply #map57(%j)[%N]
+          %tmp = call @F1(%A, %i, %0) : (memref<?x?xf32>, index, index)->(f32)
+          call @F2(%tmp, %B, %i, %0) : (f32, memref<?x?xf32>, index, index)->()
+        }
+      }
+      return
+    }
+    ```
   }];
   let arguments = (ins Variadic<AnyType>);
   let regions = (region SizedRegion<1>:$region);
@@ -236,23 +270,51 @@ def AffineIfOp : Affine_Op<"if",
                            [ImplicitAffineTerminator, RecursiveSideEffects]> {
   let summary = "if-then-else operation";
   let description = [{
-    The "if" operation represents an if-then-else construct for conditionally
-    executing two regions of code. The operands to an if operation are an
-    IntegerSet condition and a set of symbol/dimension operands to the
-    condition set. The operation produces no results. For example:
-
-       affine.if #set(%i)  {
-         ...
-       } else {
-         ...
-       }
-
-    The 'else' blocks to the if operation are optional, and may be omitted. For
-    example:
-
-       affine.if #set(%i)  {
-         ...
-       }
+    Syntax:
+
+    ```
+    operation  ::= `affine.if` if-op-cond `{` op* `}` (`else` `{` op* `}`)?
+    if-op-cond ::= integer-set-attr dim-and-symbol-use-list
+    ```
+
+    The `affine.if` operation restricts execution to a subset of the loop
+    iteration space defined by an integer set (a conjunction of affine
+    constraints). A single `affine.if` may end with an optional `else` clause.
+
+    The condition of the `affine.if` is represented by an
+    [integer set](#integer-sets) (a conjunction of affine constraints),
+    and the SSA values bound to the dimensions and symbols in the integer set.
+    The [same restrictions](#restrictions-on-dimensions-and-symbols) hold for
+    these SSA values as for all bindings of SSA values to dimensions and
+    symbols.
+
+    The `affine.if` operation contains two regions for the "then" and "else"
+    clauses. The latter may be empty (i.e. contain no blocks), meaning the
+    absence of the else clause. When non-empty, both regions must contain
+    exactly one block terminating with
+    [`affine.terminator`](#affineterminator-operation). *Note:* when `affine.if`
+    is printed in custom format, the terminator is omitted. These blocks must
+    not have any arguments.
+
+    Example:
+
+    ```mlir
+    #set = affine_set<(d0, d1)[s0]: (d0 - 10 >= 0, s0 - d0 - 9 >= 0,
+                                     d1 - 10 >= 0, s0 - d1 - 9 >= 0)>
+    func @reduced_domain_example(%A, %X, %N) : (memref<10xi32>, i32, i32) {
+      affine.for %i = 0 to %N {
+         affine.for %j = 0 to %N {
+           %0 = affine.apply #map42(%j)
+           %tmp = call @S1(%X, %i, %0)
+           affine.if #set(%i, %j)[%N] {
+              %1 = affine.apply #map43(%i, %j)
+              call @S2(%tmp, %A, %i, %1)
+           }
+        }
+      }
+      return
+    }
+    ```
   }];
   let arguments = (ins Variadic<AnyType>);
   let regions = (region SizedRegion<1>:$thenRegion, AnyRegion:$elseRegion);
@@ -275,6 +337,16 @@ def AffineIfOp : Affine_Op<"if",
     /// list of AffineIf is not resizable.
     void setConditional(IntegerSet set, ValueRange operands);
 
+    Block *getThenBlock() {
+      assert(!thenRegion().empty() && "Unexpected empty 'then' region.");
+      return &thenRegion().front();
+    }
+
+    Block *getElseBlock() {
+      assert(!elseRegion().empty() && "Empty 'else' region.");
+      return &elseRegion().front();
+    }
+
     OpBuilder getThenBodyBuilder() {
       assert(!thenRegion().empty() && "Unexpected empty 'then' region.");
       Block &body = thenRegion().front();
@@ -318,12 +390,24 @@ class AffineMinMaxOpBase<string mnemonic, list<OpTrait> traits = []> :
 def AffineMinOp : AffineMinMaxOpBase<"min", [NoSideEffect]> {
   let summary = "min operation";
   let description = [{
-    The "min" operation computes the minimum value result from a multi-result
-    affine map.
+    Syntax:
+
+    ```
+    operation ::= ssa-id `=` `affine.min` affine-map-attribute dim-and-symbol-use-list
+    ```
+
+    The `affine.min` operation applies an [affine mapping](#affine-expressions)
+    to a list of SSA values, and returns the minimum value of all result
+    expressions. The number of dimension and symbol arguments to `affine.min`
+    must be equal to the respective number of dimensional and symbolic inputs to
+    the affine mapping; the `affine.min` operation always returns one value. The
+    input operands and result must all have 'index' type.
 
     Example:
 
-       %0 = affine.min (d0) -> (1000, d0 + 512) (%i0) : index
+    ```mlir
+    %0 = affine.min affine_map<(d0)[s0] -> (1000, d0 + 512, s0)> (%arg0)[%arg1]
+    ```
   }];
 }
 
@@ -335,7 +419,9 @@ def AffineMaxOp : AffineMinMaxOpBase<"max", [NoSideEffect]> {
 
     Example:
 
-       %0 = affine.max (d0) -> (1000, d0 + 512) (%i0) : index
+    ```mlir
+    %0 = affine.max (d0) -> (1000, d0 + 512) (%i0) : index
+    ```
   }];
 }
 
@@ -365,9 +451,9 @@ def AffineParallelOp : Affine_Op<"parallel", [ImplicitAffineTerminator]> {
     Example:
 
     ```mlir
-      affine.parallel (%i, %j) = (0, 0) to (10, 10) step (1, 1) {
-        ...
-      }
+    affine.parallel (%i, %j) = (0, 0) to (10, 10) step (1, 1) {
+      ...
+    }
     ```
   }];
 
@@ -401,7 +487,7 @@ def AffineParallelOp : Affine_Op<"parallel", [ImplicitAffineTerminator]> {
 
     /// Get ranges as constants, may fail in dynamic case.
     Optional<SmallVector<int64_t, 8>> getConstantRanges();
-    
+
     Block *getBody();
     OpBuilder getBodyBuilder();
     void setSteps(ArrayRef<int64_t> newSteps);
@@ -420,8 +506,9 @@ def AffinePrefetchOp : Affine_Op<"prefetch"> {
     a read/write specifier, a locality hint, and a cache type specifier as shown
     below:
 
-      affine.prefetch %0[%i, %j + 5], read, locality<3>, data
-                          : memref<400x400xi32>
+    ```mlir
+    affine.prefetch %0[%i, %j + 5], read, locality<3>, data : memref<400x400xi32>
+    ```
 
     The read/write specifier is either 'read' or 'write', the locality hint
     specifier ranges from locality<0> (no locality) to locality<3> (extremely
@@ -491,9 +578,20 @@ def AffineTerminatorOp :
     Affine_Op<"terminator", [NoSideEffect, Terminator]> {
   let summary = "affine terminator operation";
   let description = [{
+    Syntax:
+
+    ```
+    operation ::= `"affine.terminator"() : () -> ()`
+    ```
+
     Affine terminator is a special terminator operation for blocks inside affine
-    loops and branches. It unconditionally transmits the control flow to the
-    successor of the operation enclosing the region.
+    loops ([`affine.for`](#affinefor-operation)) and branches
+    ([`affine.if`](#affineif-operation)). It unconditionally transmits the
+    control flow to the successor of the operation enclosing the region.
+
+    *Rationale*: bodies of affine operations are [blocks](../LangRef.md#blocks)
+    that must have terminators. Loops and branches represent structured control
+    flow and should not accept arbitrary branches as terminators.
 
     This operation does _not_ have a custom syntax. However, affine control
     operations omit the terminator in their custom syntax for brevity.
diff --git a/mlir/include/mlir/Dialect/Affine/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/Affine/IR/CMakeLists.txt
index 4a7144ef96e4b..1fd2a505d8507 100644
--- a/mlir/include/mlir/Dialect/Affine/IR/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/Affine/IR/CMakeLists.txt
@@ -1,2 +1,2 @@
 add_mlir_dialect(AffineOps affine)
-add_mlir_doc(AffineOps -gen-dialect-doc AffineDialect Dialects/)
+add_mlir_doc(AffineOps -gen-op-doc AffineOps Dialects/)
diff --git a/mlir/include/mlir/Dialect/Affine/Passes.h b/mlir/include/mlir/Dialect/Affine/Passes.h
index 735c6c0360f5d..75ff4a33649d6 100644
--- a/mlir/include/mlir/Dialect/Affine/Passes.h
+++ b/mlir/include/mlir/Dialect/Affine/Passes.h
@@ -42,10 +42,14 @@ std::unique_ptr<OpPassBase<FuncOp>> createAffineDataCopyGenerationPass(
     unsigned slowMemorySpace, unsigned fastMemorySpace,
     unsigned tagMemorySpace = 0, int minDmaTransferSize = 1024,
     uint64_t fastMemCapacityBytes = std::numeric_limits<uint64_t>::max());
+/// Overload relying on pass options for initialization.
+std::unique_ptr<OpPassBase<FuncOp>> createAffineDataCopyGenerationPass();
 
 /// Creates a pass to perform tiling on loop nests.
 std::unique_ptr<OpPassBase<FuncOp>>
 createLoopTilingPass(uint64_t cacheSizeBytes);
+/// Overload relying on pass options for initialization.
+std::unique_ptr<OpPassBase<FuncOp>> createLoopTilingPass();
 
 /// Creates a loop unrolling pass with the provided parameters.
 /// 'getUnrollFactor' is a function callback for clients to supply a function
@@ -67,6 +71,8 @@ createLoopUnrollAndJamPass(int unrollJamFactor = -1);
 /// target-independent, n-D super-vector abstraction.
 std::unique_ptr<OpPassBase<FuncOp>>
 createSuperVectorizePass(ArrayRef<int64_t> virtualVectorSize);
+/// Overload relying on pass options for initialization.
+std::unique_ptr<OpPassBase<FuncOp>> createSuperVectorizePass();
 
 } // end namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/Affine/Passes.td b/mlir/include/mlir/Dialect/Affine/Passes.td
new file mode 100644
index 0000000000000..4ae53571d1f4d
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Affine/Passes.td
@@ -0,0 +1,70 @@
+//===-- Passes.td - Affine pass definition file ------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains definitions for passes within the Affine/ directory.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_AFFINE_PASSES
+#define MLIR_DIALECT_AFFINE_PASSES
+
+include "mlir/Pass/PassBase.td"
+
+def AffineDataCopyGeneration : Pass<"affine-data-copy-generate"> {
+  let summary = "Generate explicit copying for affine memory operations";
+  let constructor = "mlir::createAffineDataCopyGenerationPass()";
+}
+
+def AffineLoopInvariantCodeMotion : Pass<"affine-loop-invariant-code-motion"> {
+  let summary = "Hoist loop invariant instructions outside of affine loops";
+  let constructor = "mlir::createAffineLoopInvariantCodeMotionPass()";
+}
+
+def AffineLoopTiling : Pass<"affine-loop-tile"> {
+  let summary = "Tile affine loop nests";
+  let constructor = "mlir::createLoopTilingPass()";
+}
+
+def AffineLoopUnroll : Pass<"affine-loop-unroll"> {
+  let summary = "Unroll affine loops";
+  let constructor = "mlir::createLoopUnrollPass()";
+}
+
+def AffineLoopUnrollAndJam : Pass<"affine-loop-unroll-jam"> {
+  let summary = "Unroll and jam affine loops";
+  let constructor = "mlir::createLoopUnrollAndJamPass()";
+}
+
+def AffineVectorize : Pass<"affine-super-vectorize"> {
+  let summary = "Vectorize to a target independent n-D vector abstraction";
+  let constructor = "mlir::createSuperVectorizePass()";
+  let options = [
+    ListOption<"vectorSizes", "virtual-vector-size", "int64_t",
+               "Specify an n-D virtual vector size for vectorization",
+               "llvm::cl::ZeroOrMore, llvm::cl::MiscFlags::CommaSeparated">,
+    // Optionally, the fixed mapping from loop to fastest varying MemRef
+    // dimension for all the MemRefs within a loop pattern:
+    //   the index represents the loop depth, the value represents the k^th
+    //   fastest varying memory dimension.
+    // This is voluntarily restrictive and is meant to precisely target a
+    // particular loop/op pair, for testing purposes.
+    ListOption<"fastestVaryingPattern", "test-fastest-varying", "int64_t",
+               "Specify a 1-D, 2-D or 3-D pattern of fastest varying memory "
+               "dimensions to match. See defaultPatterns in Vectorize.cpp for "
+               "a description and examples. This is used for testing purposes",
+               "llvm::cl::ZeroOrMore, llvm::cl::MiscFlags::CommaSeparated">
+  ];
+}
+
+def SimplifyAffineStructures : Pass<"simplify-affine-structures"> {
+  let summary = "Simplify affine expressions in maps/sets and normalize "
+                "memrefs";
+  let constructor = "mlir::createSimplifyAffineStructuresPass()";
+}
+
+#endif // MLIR_DIALECT_AFFINE_PASSES
diff --git a/mlir/include/mlir/Dialect/FxpMathOps/CMakeLists.txt b/mlir/include/mlir/Dialect/FxpMathOps/CMakeLists.txt
index 2a493d6c1b204..b2d6dc660dbbf 100644
--- a/mlir/include/mlir/Dialect/FxpMathOps/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/FxpMathOps/CMakeLists.txt
@@ -1,2 +1,8 @@
 add_mlir_dialect(FxpMathOps fxpmath)
 add_mlir_doc(FxpMathOps -gen-dialect-doc FxpMathDialect Dialects/)
+
+set(LLVM_TARGET_DEFINITIONS Passes.td)
+mlir_tablegen(Passes.h.inc -gen-pass-decls)
+add_public_tablegen_target(MLIRFxpMathPassIncGen)
+
+add_mlir_doc(Passes -gen-pass-doc FxpMathPasses ./)
diff --git a/mlir/include/mlir/Dialect/FxpMathOps/Passes.h b/mlir/include/mlir/Dialect/FxpMathOps/Passes.h
index 1039bcc9cb344..bec2e74f095f9 100644
--- a/mlir/include/mlir/Dialect/FxpMathOps/Passes.h
+++ b/mlir/include/mlir/Dialect/FxpMathOps/Passes.h
@@ -13,6 +13,8 @@
 #ifndef MLIR_DIALECT_FXPMATHOPS_PASSES_H
 #define MLIR_DIALECT_FXPMATHOPS_PASSES_H
 
+#include <memory>
+
 namespace mlir {
 class FuncOp;
 template <typename T> class OpPassBase;
@@ -23,11 +25,11 @@ namespace fxpmath {
 /// arithmetic. This will leave unrecognized real math ops as-is and is
 /// typically followed by a pass that lowers any unrecognized ops to a pure
 /// floating point form.
-OpPassBase<FuncOp> *createLowerUniformRealMathPass();
+std::unique_ptr<OpPassBase<FuncOp>> createLowerUniformRealMathPass();
 
 /// Creates a pass that lowers uniform-quantized qcast/dcast ops to equivalent
 /// operations that perform quantize/dequantize.
-OpPassBase<FuncOp> *createLowerUniformCastsPass();
+std::unique_ptr<OpPassBase<FuncOp>> createLowerUniformCastsPass();
 
 } // namespace fxpmath
 } // namespace mlir
diff --git a/mlir/include/mlir/Dialect/FxpMathOps/Passes.td b/mlir/include/mlir/Dialect/FxpMathOps/Passes.td
new file mode 100644
index 0000000000000..254d200029f5e
--- /dev/null
+++ b/mlir/include/mlir/Dialect/FxpMathOps/Passes.td
@@ -0,0 +1,24 @@
+//===-- Passes.td - FxpMath pass definition file -----------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_FXPMATH_PASSES
+#define MLIR_DIALECT_FXPMATH_PASSES
+
+include "mlir/Pass/PassBase.td"
+
+def FxpMathLowerUniformCasts : Pass<"fxpmath-lower-uniform-casts"> {
+  let summary = "Lowers uniform-quantized casts";
+  let constructor = "mlir::fxpmath::createLowerUniformCastsPass()";
+}
+
+def FxpMathLowerUniformRealMath : Pass<"fxpmath-lower-uniform-real-math"> {
+  let summary = "Lowers uniform-quantized real math ops to integer arithmetic";
+  let constructor = "mlir::fxpmath::createLowerUniformRealMathPass()";
+}
+
+#endif // MLIR_DIALECT_FXPMATH_PASSES
diff --git a/mlir/include/mlir/Dialect/GPU/CMakeLists.txt b/mlir/include/mlir/Dialect/GPU/CMakeLists.txt
index d341303d62da2..6c80b4c8e3b98 100644
--- a/mlir/include/mlir/Dialect/GPU/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/GPU/CMakeLists.txt
@@ -1,5 +1,5 @@
-add_mlir_dialect(GPUOps gpu GPUOps)
-add_mlir_doc(GPUOps -gen-dialect-doc GPUDialect Dialects/)
+add_mlir_dialect(GPUOps gpu)
+add_mlir_doc(GPUOps -gen-op-doc GPUOps Dialects/)
 
 set(LLVM_TARGET_DEFINITIONS ParallelLoopMapperAttr.td)
 mlir_tablegen(ParallelLoopMapperAttr.h.inc -gen-struct-attr-decls)
@@ -10,3 +10,9 @@ set(LLVM_TARGET_DEFINITIONS ParallelLoopMapperAttr.td)
 mlir_tablegen(ParallelLoopMapperEnums.h.inc -gen-enum-decls)
 mlir_tablegen(ParallelLoopMapperEnums.cpp.inc -gen-enum-defs)
 add_public_tablegen_target(MLIRParallelLoopMapperEnumsGen)
+
+set(LLVM_TARGET_DEFINITIONS Passes.td)
+mlir_tablegen(Passes.h.inc -gen-pass-decls)
+add_public_tablegen_target(MLIRGPUPassIncGen)
+
+add_mlir_doc(Passes -gen-pass-doc GPUPasses ./)
diff --git a/mlir/include/mlir/Dialect/GPU/GPUOps.td b/mlir/include/mlir/Dialect/GPU/GPUOps.td
index 6feaf82405f0f..981f397564554 100644
--- a/mlir/include/mlir/Dialect/GPU/GPUOps.td
+++ b/mlir/include/mlir/Dialect/GPU/GPUOps.td
@@ -36,10 +36,54 @@ class GPU_IndexOp<string mnemonic, list<OpTrait> traits = []> :
   let verifier = [{ return ::verifyIndexOp(*this); }];
 }
 
-def GPU_BlockDimOp : GPU_IndexOp<"block_dim">;
-def GPU_BlockIdOp : GPU_IndexOp<"block_id">;
-def GPU_GridDimOp : GPU_IndexOp<"grid_dim">;
-def GPU_ThreadIdOp : GPU_IndexOp<"thread_id">;
+def GPU_BlockDimOp : GPU_IndexOp<"block_dim"> {
+  let description = [{
+    Returns the number of threads in the thread block (aka the block size) along
+    the x, y, or z `dimension`.
+
+    Example:
+
+    ```mlir
+    %bDimX = "gpu.block_dim"() {dimension = "x"} : () -> (index)
+    ```
+  }];
+}
+def GPU_BlockIdOp : GPU_IndexOp<"block_id"> {
+  let description = [{
+    Returns the block id, i.e. the index of the current block within the grid
+    along the x, y, or z `dimension`.
+
+    Example:
+
+    ```mlir
+    %bIdY = "gpu.block_id"() {dimension = "y"} : () -> (index)
+    ```
+  }];
+}
+def GPU_GridDimOp : GPU_IndexOp<"grid_dim"> {
+  let description = [{
+    Returns the number of thread blocks in the grid along the x, y, or z
+    `dimension`.
+
+    Example:
+
+    ```mlir
+    %gDimZ = "gpu.grid_dim"() {dimension = "z"} : () -> (index)
+    ```
+  }];
+}
+def GPU_ThreadIdOp : GPU_IndexOp<"thread_id"> {
+  let description = [{
+    Returns the thread id, i.e. the index of the current thread within the block
+    along the x, y, or z `dimension`.
+
+    Example:
+
+    ```mlir
+    %tIdX = "gpu.thread_id"() {dimension = "x"} : () -> (index)
+    ```
+  }];
+}
 
 def GPU_GPUFuncOp : GPU_Op<"func", [FunctionLike, IsolatedFromAbove, Symbol]> {
   let summary = "Function executable on a GPU";
@@ -443,7 +487,7 @@ def GPU_ReturnOp : GPU_Op<"return", [HasParent<"GPUFuncOp">, NoSideEffect,
   let description = [{
     A terminator operation for regions that appear in the body of  `gpu.func`
     functions. The operands to the `gpu.return` are the result values returned
-    by an incovation of the `gpu.func`.
+    by an invocation of the `gpu.func`.
   }];
 
   let builders = [OpBuilder<"Builder *builder, OperationState &result", " // empty">];
@@ -471,13 +515,14 @@ def GPU_YieldOp : GPU_Op<"yield", [NoSideEffect, Terminator]>,
     Arguments<(ins Variadic<AnyType>:$values)> {
   let summary = "GPU yield operation";
   let description = [{
-    "gpu.yield" is a special terminator operation for blocks inside regions
+    gpu.yield` is a special terminator operation for blocks inside regions
     in gpu ops. It returns values to the immediately enclosing gpu op.
 
     Example:
 
-       ```gpu.yield %f0, %f1 : f32, f32
-       ```
+    ```mlir
+    gpu.yield %f0, %f1 : f32, f32
+    ```
   }];
 }
 
@@ -509,18 +554,20 @@ def GPU_AllReduceOp : GPU_Op<"all_reduce",
     Results<(outs AnyType)> {
   let summary = "Reduce values among workgroup.";
   let description = [{
-    The "all_reduce" op reduces the value of every work item across a local
+    The `all_reduce` op reduces the value of every work item across a local
     workgroup. The result is equal for all work items of a workgroup.
 
     For example, both
+
+    ```mlir
+    %1 = "gpu.all_reduce"(%0) ({}) { op = "add" } : (f32) -> (f32)
+    %2 = "gpu.all_reduce"(%0) ({
+    ^bb(%lhs : f32, %rhs : f32):
+      %sum = addf %lhs, %rhs : f32
+      "gpu.yield"(%sum) : (f32) -> ()
+    }) : (f32) -> (f32)
     ```
-      %1 = "gpu.all_reduce"(%0) ({}) { op = "add" } : (f32) -> (f32)
-      %2 = "gpu.all_reduce"(%0) ({
-      ^bb(%lhs : f32, %rhs : f32):
-        %sum = addf %lhs, %rhs : f32
-        "gpu.yield"(%sum) : (f32) -> ()
-      }) : (f32) -> (f32)
-    ```
+
     compute the sum of each work item's %0 value. The first version specifies
     the accumulation as operation, whereas the second version specifies the
     accumulation as code region. The accumulation operation must be one of:
@@ -550,11 +597,13 @@ def GPU_ShuffleOp : GPU_Op<"shuffle", [NoSideEffect]>,
     The "shuffle" op moves values to a different invocation within the same
     subgroup.
 
-    For example
-    ```
-      %1, %2 = gpu.shuffle %0, %offset, %width xor : f32
+    Example:
+
+    ```mlir
+    %1, %2 = gpu.shuffle %0, %offset, %width xor : f32
     ```
-    for lane k returns the value from lane `k ^ offset` and `true` if that lane
+
+    For lane k returns the value from lane `k ^ offset` and `true` if that lane
     is smaller than %width. Otherwise it returns an unspecified value and
     `false`. A lane is the index of an invocation relative to its subgroup.
 
@@ -574,9 +623,10 @@ def GPU_BarrierOp : GPU_Op<"barrier"> {
     The "barrier" op synchronizes all work items of a workgroup. It is used
     to coordinate communication between the work items of the workgroup.
 
+    ```mlir
+    gpu.barrier
     ```
-      gpu.barrier
-    ```
+
     waits until all work items in the workgroup have reached this point
     and all memory accesses made by these work items prior to the op are
     visible to all work items in the workgroup. Data hazards between work items
diff --git a/mlir/include/mlir/Dialect/GPU/Passes.td b/mlir/include/mlir/Dialect/GPU/Passes.td
new file mode 100644
index 0000000000000..5636243082974
--- /dev/null
+++ b/mlir/include/mlir/Dialect/GPU/Passes.td
@@ -0,0 +1,19 @@
+//===-- Passes.td - GPU pass definition file ---------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_GPU_PASSES
+#define MLIR_DIALECT_GPU_PASSES
+
+include "mlir/Pass/PassBase.td"
+
+def GpuKernelOutlining : Pass<"gpu-kernel-outlining"> {
+  let summary = "Outline gpu.launch bodies to kernel functions";
+  let constructor = "mlir::createGpuKernelOutliningPass()";
+}
+
+#endif // MLIR_DIALECT_GPU_PASSES
diff --git a/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt b/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
index d7e581b1b9492..cc4fd1bafc720 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
@@ -1,3 +1,5 @@
+add_subdirectory(Transforms)
+
 set(LLVM_TARGET_DEFINITIONS LLVMOps.td)
 mlir_tablegen(LLVMOps.h.inc -gen-op-decls)
 mlir_tablegen(LLVMOps.cpp.inc -gen-op-defs)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
index 53dea5bec65a5..54b2e9429bc3b 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
@@ -663,7 +663,8 @@ def LLVM_LLVMFuncOp
     : LLVM_ZeroResultOp<"func", [IsolatedFromAbove, FunctionLike, Symbol]>,
       Arguments<(ins DefaultValuedAttr<Linkage,
                                        "Linkage::External">:$linkage,
-                     OptionalAttr<FlatSymbolRefAttr>:$personality)> {
+                     OptionalAttr<FlatSymbolRefAttr>:$personality,
+                     OptionalAttr<ArrayAttr>:$passthrough)> {
   let summary = "LLVM dialect function, has wrapped LLVM IR function type";
 
   let regions = (region AnyRegion:$body);
@@ -771,6 +772,7 @@ class LLVM_TernarySameArgsIntrinsicOp<string func, list<OpTrait> traits = []> :
 def LLVM_CopySignOp : LLVM_BinarySameArgsIntrinsicOp<"copysign">;
 def LLVM_CosOp : LLVM_UnaryIntrinsicOp<"cos">;
 def LLVM_ExpOp : LLVM_UnaryIntrinsicOp<"exp">;
+def LLVM_Exp2Op : LLVM_UnaryIntrinsicOp<"exp2">;
 def LLVM_FAbsOp : LLVM_UnaryIntrinsicOp<"fabs">;
 def LLVM_FCeilOp : LLVM_UnaryIntrinsicOp<"ceil">;
 def LLVM_FMAOp : LLVM_TernarySameArgsIntrinsicOp<"fma">;
@@ -864,7 +866,7 @@ def LLVM_MatrixMultiplyOp
 
 /// Create a llvm.matrix.transpose call, transposing a `rows` x `columns` 2-D
 /// `matrix`, as specified in the LLVM MatrixBuilder.
-def LLVM_MatrixTranposeOp
+def LLVM_MatrixTransposeOp
     : LLVM_OneResultOp<"intr.matrix.transpose">,
       Arguments<(ins LLVM_Type:$matrix, I32Attr:$rows, I32Attr:$columns)> {
   string llvmBuilder = [{
diff --git a/mlir/include/mlir/Dialect/LLVMIR/Transforms/CMakeLists.txt b/mlir/include/mlir/Dialect/LLVMIR/Transforms/CMakeLists.txt
new file mode 100644
index 0000000000000..a2fd81c23e11b
--- /dev/null
+++ b/mlir/include/mlir/Dialect/LLVMIR/Transforms/CMakeLists.txt
@@ -0,0 +1,5 @@
+set(LLVM_TARGET_DEFINITIONS Passes.td)
+mlir_tablegen(Passes.h.inc -gen-pass-decls)
+add_public_tablegen_target(MLIRLLVMPassIncGen)
+
+add_mlir_doc(Passes -gen-pass-doc LLVMPasses ./)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/Transforms/Passes.td b/mlir/include/mlir/Dialect/LLVMIR/Transforms/Passes.td
new file mode 100644
index 0000000000000..0dc193e794f52
--- /dev/null
+++ b/mlir/include/mlir/Dialect/LLVMIR/Transforms/Passes.td
@@ -0,0 +1,19 @@
+//===-- Passes.td - LLVM pass definition file --------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_LLVMIR_TRANSFORMS_PASSES
+#define MLIR_DIALECT_LLVMIR_TRANSFORMS_PASSES
+
+include "mlir/Pass/PassBase.td"
+
+def LLVMLegalizeForExport : Pass<"llvm-legalize-for-export"> {
+  let summary = "Legalize LLVM dialect to be convertible to LLVM IR";
+  let constructor = "mlir::LLVM::createLegalizeForExportPass()";
+}
+
+#endif // MLIR_DIALECT_LLVMIR_TRANSFORMS_PASSES
diff --git a/mlir/include/mlir/Dialect/Linalg/Analysis/DependenceAnalysis.h b/mlir/include/mlir/Dialect/Linalg/Analysis/DependenceAnalysis.h
index 34de176a998ef..e40d63661b777 100644
--- a/mlir/include/mlir/Dialect/Linalg/Analysis/DependenceAnalysis.h
+++ b/mlir/include/mlir/Dialect/Linalg/Analysis/DependenceAnalysis.h
@@ -63,6 +63,7 @@ class LinalgDependenceGraph {
   using dependence_range = iterator_range<dependence_iterator>;
 
   enum DependenceType { RAR = 0, RAW, WAR, WAW, NumTypes };
+  static StringRef getDependenceTypeStr(DependenceType depType);
 
   // Builds a linalg dependence graph for the ops of type LinalgOp under `f`.
   static LinalgDependenceGraph buildDependenceGraph(Aliases &aliases, FuncOp f);
diff --git a/mlir/include/mlir/Dialect/Linalg/CMakeLists.txt b/mlir/include/mlir/Dialect/Linalg/CMakeLists.txt
index 9f57627c321fb..076c2dfbccb5c 100644
--- a/mlir/include/mlir/Dialect/Linalg/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/Linalg/CMakeLists.txt
@@ -1,2 +1,8 @@
 add_subdirectory(IR)
 add_subdirectory(Transforms)
+
+set(LLVM_TARGET_DEFINITIONS Passes.td)
+mlir_tablegen(Passes.h.inc -gen-pass-decls)
+add_public_tablegen_target(MLIRLinalgPassIncGen)
+
+add_mlir_doc(Passes -gen-pass-doc LinalgPasses ./)
diff --git a/mlir/include/mlir/Dialect/Linalg/EDSC/Builders.h b/mlir/include/mlir/Dialect/Linalg/EDSC/Builders.h
index 73ccffdf80fb1..66cd8c002ff2c 100644
--- a/mlir/include/mlir/Dialect/Linalg/EDSC/Builders.h
+++ b/mlir/include/mlir/Dialect/Linalg/EDSC/Builders.h
@@ -161,31 +161,34 @@ void macRegionBuilder(ArrayRef<BlockArgument> args);
 
 /// Unary pointwise operation (with broadcast) entry point.
 using UnaryPointwiseOpBuilder = function_ref<Value(ValueHandle)>;
-Operation *linalg_pointwise(UnaryPointwiseOpBuilder unaryOp,
-                            StructuredIndexed I, StructuredIndexed O);
+Operation *linalg_generic_pointwise(UnaryPointwiseOpBuilder unaryOp,
+                                    StructuredIndexed I, StructuredIndexed O);
 
 /// Build a linalg.pointwise with all `parallel` iterators and a region that
 /// computes `O = tanh(I)`. The client is responsible for specifying the proper
 /// indexings when creating the StructuredIndexed.
-Operation *linalg_pointwise_tanh(StructuredIndexed I, StructuredIndexed O);
+Operation *linalg_generic_pointwise_tanh(StructuredIndexed I,
+                                         StructuredIndexed O);
 
 /// Binary pointwise operation (with broadcast) entry point.
 using BinaryPointwiseOpBuilder = function_ref<Value(ValueHandle, ValueHandle)>;
-Operation *linalg_pointwise(BinaryPointwiseOpBuilder binaryOp,
-                            StructuredIndexed I1, StructuredIndexed I2,
-                            StructuredIndexed O);
+Operation *linalg_generic_pointwise(BinaryPointwiseOpBuilder binaryOp,
+                                    StructuredIndexed I1, StructuredIndexed I2,
+                                    StructuredIndexed O);
 
 /// Build a linalg.pointwise with all `parallel` iterators and a region that
 /// computes `O = I1 + I2`. The client is responsible for specifying the proper
 /// indexings when creating the StructuredIndexed.
-Operation *linalg_pointwise_add(StructuredIndexed I1, StructuredIndexed I2,
-                                StructuredIndexed O);
+Operation *linalg_generic_pointwise_add(StructuredIndexed I1,
+                                        StructuredIndexed I2,
+                                        StructuredIndexed O);
 
 /// Build a linalg.pointwise with all `parallel` iterators and a region that
 /// computes `O = max(I1, I2)`. The client is responsible for specifying the
 /// proper indexings when creating the StructuredIndexed.
-Operation *linalg_pointwise_max(StructuredIndexed I1, StructuredIndexed I2,
-                                StructuredIndexed O);
+Operation *linalg_generic_pointwise_max(StructuredIndexed I1,
+                                        StructuredIndexed I2,
+                                        StructuredIndexed O);
 
 // TODO(ntv): Implement more useful pointwise operations on a per-need basis.
 
@@ -198,8 +201,9 @@ using MatmulRegionBuilder = function_ref<void(ArrayRef<BlockArgument> args)>;
 ///    |
 ///    |  C(m, n) += A(m, k) * B(k, n)
 /// ```
-Operation *linalg_matmul(ValueHandle vA, ValueHandle vB, ValueHandle vC,
-                         MatmulRegionBuilder regionBuilder = macRegionBuilder);
+Operation *
+linalg_generic_matmul(ValueHandle vA, ValueHandle vB, ValueHandle vC,
+                      MatmulRegionBuilder regionBuilder = macRegionBuilder);
 
 /// Build a linalg.generic, under the current ScopedContext, at the current
 /// insert point, that computes:
@@ -209,8 +213,9 @@ Operation *linalg_matmul(ValueHandle vA, ValueHandle vB, ValueHandle vC,
 ///    |  C(m, n) = sum_k(A(m, k) * B(k, n))
 /// ```
 /// and returns the tensor `C`.
-Operation *linalg_matmul(ValueHandle vA, ValueHandle vB, RankedTensorType tC,
-                         MatmulRegionBuilder regionBuilder = mulRegionBuilder);
+Operation *
+linalg_generic_matmul(ValueHandle vA, ValueHandle vB, RankedTensorType tC,
+                      MatmulRegionBuilder regionBuilder = mulRegionBuilder);
 
 /// Build a linalg.generic, under the current ScopedContext, at the current
 /// insert point, that computes:
@@ -220,15 +225,17 @@ Operation *linalg_matmul(ValueHandle vA, ValueHandle vB, RankedTensorType tC,
 ///    |  D(m, n) = C(m, n) + sum_k(A(m, k) * B(k, n))
 /// ```
 /// and returns the tensor `D`.
-Operation *linalg_matmul(ValueHandle vA, ValueHandle vB, ValueHandle vC,
-                         RankedTensorType tD,
-                         MatmulRegionBuilder regionBuilder = macRegionBuilder);
+Operation *
+linalg_generic_matmul(ValueHandle vA, ValueHandle vB, ValueHandle vC,
+                      RankedTensorType tD,
+                      MatmulRegionBuilder regionBuilder = macRegionBuilder);
 
 template <typename Container>
-Operation *linalg_matmul(Container values,
-                         MatmulRegionBuilder regionBuilder = macRegionBuilder) {
+Operation *
+linalg_generic_matmul(Container values,
+                      MatmulRegionBuilder regionBuilder = macRegionBuilder) {
   assert(values.size() == 3 && "Expected exactly 3 values");
-  return linalg_matmul(values[0], values[1], values[2], regionBuilder);
+  return linalg_generic_matmul(values[0], values[1], values[2], regionBuilder);
 }
 
 /// Build a linalg.generic, under the current ScopedContext, at the current
@@ -253,15 +260,17 @@ Operation *linalg_matmul(Container values,
 /// For now `...` must be empty (i.e. only 2-D convolutions are supported).
 ///
 // TODO(ntv) Extend convolution rank with some template magic.
-Operation *linalg_conv_nhwc(ValueHandle vI, ValueHandle vW, ValueHandle vO,
-                            ArrayRef<int> strides = {},
-                            ArrayRef<int> dilations = {});
+Operation *linalg_generic_conv_nhwc(ValueHandle vI, ValueHandle vW,
+                                    ValueHandle vO, ArrayRef<int> strides = {},
+                                    ArrayRef<int> dilations = {});
 
 template <typename Container>
-Operation *linalg_conv_nhwc(Container values, ArrayRef<int> strides = {},
-                            ArrayRef<int> dilations = {}) {
+Operation *linalg_generic_conv_nhwc(Container values,
+                                    ArrayRef<int> strides = {},
+                                    ArrayRef<int> dilations = {}) {
   assert(values.size() == 3 && "Expected exactly 3 values");
-  return linalg_conv_nhwc(values[0], values[1], values[2], strides, dilations);
+  return linalg_generic_conv_nhwc(values[0], values[1], values[2], strides,
+                                  dilations);
 }
 
 /// Build a linalg.generic, under the current ScopedContext, at the current
@@ -286,18 +295,20 @@ Operation *linalg_conv_nhwc(Container values, ArrayRef<int> strides = {},
 /// For now `...` must be empty (i.e. only 2-D convolutions are supported).
 ///
 // TODO(ntv) Extend convolution rank with some template magic.
-Operation *linalg_dilated_conv_nhwc(ValueHandle vI, ValueHandle vW,
-                                    ValueHandle vO, int depth_multiplier = 1,
-                                    ArrayRef<int> strides = {},
-                                    ArrayRef<int> dilations = {});
+Operation *linalg_generic_dilated_conv_nhwc(ValueHandle vI, ValueHandle vW,
+                                            ValueHandle vO,
+                                            int depth_multiplier = 1,
+                                            ArrayRef<int> strides = {},
+                                            ArrayRef<int> dilations = {});
 
 template <typename Container>
-Operation *linalg_dilated_conv_nhwc(Container values, int depth_multiplier,
-                                    ArrayRef<int> strides = {},
-                                    ArrayRef<int> dilations = {}) {
+Operation *linalg_generic_dilated_conv_nhwc(Container values,
+                                            int depth_multiplier,
+                                            ArrayRef<int> strides = {},
+                                            ArrayRef<int> dilations = {}) {
   assert(values.size() == 3 && "Expected exactly 3 values");
-  return linalg_dilated_conv_nhwc(values[0], values[1], values[2],
-                                  depth_multiplier, strides, dilations);
+  return linalg_generic_dilated_conv_nhwc(values[0], values[1], values[2],
+                                          depth_multiplier, strides, dilations);
 }
 
 } // namespace ops
diff --git a/mlir/include/mlir/Dialect/Linalg/EDSC/Intrinsics.h b/mlir/include/mlir/Dialect/Linalg/EDSC/Intrinsics.h
index 98ff016182fa1..dedc18934b844 100644
--- a/mlir/include/mlir/Dialect/Linalg/EDSC/Intrinsics.h
+++ b/mlir/include/mlir/Dialect/Linalg/EDSC/Intrinsics.h
@@ -26,7 +26,10 @@ ValueHandle ValueHandle::create(OperationFolder *folder, Args... args) {
 
 namespace intrinsics {
 using linalg_copy = OperationBuilder<linalg::CopyOp>;
+using linalg_dot = OperationBuilder<linalg::DotOp>;
 using linalg_fill = OperationBuilder<linalg::FillOp>;
+using linalg_matmul = OperationBuilder<linalg::MatmulOp>;
+using linalg_matvec = OperationBuilder<linalg::MatvecOp>;
 using linalg_range = ValueBuilder<linalg::RangeOp>;
 using linalg_reshape = ValueBuilder<linalg::ReshapeOp>;
 using linalg_slice = ValueBuilder<linalg::SliceOp>;
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/Linalg/IR/CMakeLists.txt
index 2c8c33ff62a64..41035ed76bbab 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/Linalg/IR/CMakeLists.txt
@@ -1,5 +1,6 @@
 add_mlir_dialect(LinalgOps linalg)
-add_mlir_doc(LinalgDoc -gen-dialect-doc LinalgDialect Dialects/)
+add_mlir_doc(LinalgDoc -gen-op-doc LinalgOps Dialects/)
+
 set(LLVM_TARGET_DEFINITIONS LinalgStructuredOps.td)
 mlir_tablegen(LinalgStructuredOps.h.inc -gen-op-decls)
 mlir_tablegen(LinalgStructuredOps.cpp.inc -gen-op-defs)
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.h b/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.h
index 7756a08d5cb22..77d9d9fc26313 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.h
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.h
@@ -29,6 +29,9 @@ namespace mlir {
 namespace linalg {
 
 class ConvOp;
+class PoolingMaxOp;
+class PoolingMinOp;
+class PoolingSumOp;
 
 /// Returns the name mangled library call name to disambiguate between different
 /// overloads at the C level. The name mangling scheme is basic and uses MLIR
@@ -60,12 +63,13 @@ std::string generateLibraryCallName(Operation *op);
 SmallVector<AffineExpr, 4> makeAffineDimExprs(unsigned num, unsigned &startIdx,
                                               MLIRContext *context);
 
-/// Builds the indexing expressions for a ConvOp `op`. Returns the vector of
-/// AffineMaps representing:
-///   `stride[i] * xs[i] + dilation[i] * zs[i] - pad_low[i]`
-SmallVector<AffineExpr, 4> weightedConvInputIndex(ConvOp op,
-                                                  ArrayRef<AffineExpr> xs,
-                                                  ArrayRef<AffineExpr> zs);
+/// Builds the indexing expressions for a ConvOp/PoolingOp `op`. Returns the
+/// vector of AffineMaps representing:
+///   `stride[i] * outputDims[i] + dilation[i] * windowDims[i] - pad_low[i]`
+template <typename PoolingOp>
+extern SmallVector<AffineExpr, 4>
+weightedPoolingInputIndex(PoolingOp op, ArrayRef<AffineExpr> outputDims,
+                          ArrayRef<AffineExpr> windowDims);
 
 /// Returns `maybeMap.get()` if `maybeMap` is set, otherwise returns the
 /// symbol-less identity map of `rank`.
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td
index dc0c03f26cc84..bf0e1dd487707 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td
@@ -44,7 +44,7 @@ def Linalg_RangeOp :
     Example:
 
     ```mlir
-      %3 = linalg.range %0:%1:%2 : !linalg.range
+    %3 = linalg.range %0:%1:%2 : !linalg.range
     ````
   }];
   let builders = [OpBuilder<
@@ -91,15 +91,15 @@ def Linalg_ReshapeOp : Linalg_Op<"reshape", [NoSideEffect]>,
     Examples:
 
     ```mlir
-       // Dimension collapse (i, j) -> i' and k -> k'
-       %1 = linalg.reshape %0 [(i, j, k) -> (i, j), (i, j, k) -> (k)] :
-         memref<?x?x?xf32, stride_spec> into memref<?x?xf32, stride_spec_2>
+    // Dimension collapse (i, j) -> i' and k -> k'
+    %1 = linalg.reshape %0 [(i, j, k) -> (i, j), (i, j, k) -> (k)] :
+      memref<?x?x?xf32, stride_spec> into memref<?x?xf32, stride_spec_2>
     ```
 
     ```mlir
-       // Dimension expansion i -> (i', j') and (k) -> (k')
-       %1 = linalg.reshape %0 [(i, j, k) -> (i, j), (i, j, k) -> (k)] :
-         memref<?x?xf32, stride_spec> into memref<?x?x?xf32, stride_spec_2>
+    // Dimension expansion i -> (i', j') and (k) -> (k')
+    %1 = linalg.reshape %0 [(i, j, k) -> (i, j), (i, j, k) -> (k)] :
+      memref<?x?xf32, stride_spec> into memref<?x?x?xf32, stride_spec_2>
     ```
   }];
 
@@ -151,22 +151,22 @@ def Linalg_SliceOp : Linalg_Op<"slice", [NoSideEffect]>,
       1. rank-preserving `slice`:
 
       ```mlir
-        %4 = linalg.slice %0[%1, %2] : memref<?x?xf32, stride_spec>,
-          !linalg.range, !linalg.range, memref<?x?xf32, stride_spec>
-       ```
+      %4 = linalg.slice %0[%1, %2] : memref<?x?xf32, stride_spec>,
+        !linalg.range, !linalg.range, memref<?x?xf32, stride_spec>
+      ```
 
       2. rank-reducing `slice` (from 2-D to 1-D):
 
       ```mlir
-        %4 = linalg.slice %0[%1, %2] : memref<?x?xf32, stride_spec>,
-          index, !linalg.range, memref<?x?xf32, stride_spec>
+      %4 = linalg.slice %0[%1, %2] : memref<?x?xf32, stride_spec>,
+        index, !linalg.range, memref<?x?xf32, stride_spec>
       ```
 
       3. rank-reducing `slice` (from 2-D to 0-D):
 
       ```mlir
-        %4 = linalg.slice %0[%1, %2] : memref<?x?xf32, stride_spec>,
-          index, index, memref<?x?xf32, stride_spec>
+      %4 = linalg.slice %0[%1, %2] : memref<?x?xf32, stride_spec>,
+        index, index, memref<?x?xf32, stride_spec>
       ```
   }];
 
@@ -210,7 +210,7 @@ def Linalg_TransposeOp : Linalg_Op<"transpose", [NoSideEffect]>,
     Example:
 
     ```mlir
-       %1 = linalg.transpose %0 (i, j) -> (j, i) : memref<?x?xf32, stride_spec>
+    %1 = linalg.transpose %0 (i, j) -> (j, i) : memref<?x?xf32, stride_spec>
     ```
   }];
 
@@ -245,7 +245,7 @@ def Linalg_YieldOp : Linalg_Op<"yield", [NoSideEffect, Terminator]>,
     Example:
 
     ```mlir
-       linalg.yield %f0, %f1 : f32, f32
+    linalg.yield %f0, %f1 : f32, f32
     ```
   }];
 }
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
index 5c8590fc60635..31b89bc1b2bf8 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
@@ -61,44 +61,48 @@ def CopyOp : LinalgStructured_Op<"copy", [NInputs<1>, NOutputs<1>]> {
     Copies the data in the input view into the output view.
 
     Usage:
-      ```mlir
-      linalg.copy(%arg0, %arg1) : memref<?xf32, stride_specification>,
-                                  memref<?xf32, stride_specification>
-      ```
+
+    ```mlir
+    linalg.copy(%arg0, %arg1) : memref<?xf32, stride_specification>,
+                                memref<?xf32, stride_specification>
+    ```
 
     One possible lowering to loop form is:
-      ```mlir
-      %0 = linalg.dim %arg0, 0 : index
-      loop.for %i0 = %c0 to %0 step %c1 {
-        %1 = load %arg0[%i0] : memref<?xf32, stride_specification>
-        store %1, %arg1[%i0] : memref<?xf32, stride_specification>
-      }
-      ```
+
+    ```mlir
+    %0 = linalg.dim %arg0, 0 : index
+    loop.for %i0 = %c0 to %0 step %c1 {
+      %1 = load %arg0[%i0] : memref<?xf32, stride_specification>
+      store %1, %arg1[%i0] : memref<?xf32, stride_specification>
+    }
+    ```
 
     Optionally, can take `input_permutation` and `output_permutation` attributes
     to reorder the dimensions of the input and output views.
 
     Usage:
-      ```mlir
-      linalg.copy(%arg0, %arg1) {inputPermutation : (i, j, k) -> (i, k, j),
-                                 outputPermutation : (i, j, k) -> (k, j, i)} :
-        memref<?x?x?xf32, stride_specification>,
-        memref<?x?x?xf32, stride_specification>
-     ```
+
+    ```mlir
+    linalg.copy(%arg0, %arg1) {inputPermutation : (i, j, k) -> (i, k, j),
+                               outputPermutation : (i, j, k) -> (k, j, i)} :
+      memref<?x?x?xf32, stride_specification>,
+      memref<?x?x?xf32, stride_specification>
+    ```
 
     One possible lowering to loop form is:
-      ```mlir
-      %0 = linalg.dim %arg0, 0
-      %1 = linalg.dim %arg0, 1
-      %2 = linalg.dim %arg0, 2
-      loop.for %i0 = %c0 to %{{.*}} step %c1 {
-        loop.for %i1 = %c0 to %{{.*}} step %c1 {
-          loop.for %i2 = %c0 to %{{.*}} step %c1 {
-            %3 = load %arg0[%i0, %i2, %i1] :
-                    memref<?x?x?xf32, stride_specification>
-            store %3, %arg1[%i2, %i1, %i0] :
-                    memref<?x?x?xf32, stride_specification>
-      ```
+
+    ```mlir
+    %0 = linalg.dim %arg0, 0
+    %1 = linalg.dim %arg0, 1
+    %2 = linalg.dim %arg0, 2
+    loop.for %i0 = %c0 to %{{.*}} step %c1 {
+      loop.for %i1 = %c0 to %{{.*}} step %c1 {
+        loop.for %i2 = %c0 to %{{.*}} step %c1 {
+          %3 = load %arg0[%i0, %i2, %i1] :
+                  memref<?x?x?xf32, stride_specification>
+          store %3, %arg1[%i2, %i1, %i0] :
+                  memref<?x?x?xf32, stride_specification>
+    ```
 
     The views are expected to be compatible for correctness but this is not
     enforced at the moment.
@@ -247,7 +251,69 @@ def MatmulOp : LinalgStructured_Op<"matmul", [NInputs<2>, NOutputs<1>]> {
   let hasFolder = 1;
 }
 
-def ConvOp : LinalgStructured_Op<"conv", [NInputs<2>, NOutputs<1>]> {
+/// A base class for pooling operation such as conv. The arguments must contain
+/// optional arguments `strides`, `dilations` and `padding` with following type:
+///   OptionalAttr<I64ArrayAttr>:$strides
+///   OptionalAttr<I64ArrayAttr>:$dilations
+///   OptionalAttr<I64ElementsAttr>:$padding
+/// `stirdes` denotes the step of each window along the dimension.
+class PoolingBase_Op<string mnemonic, list<OpTrait> props>
+  : LinalgStructured_Op<mnemonic, props> {
+  let description = [{
+    Performs an N-D pooling operation similarly to the description in the TF
+    documentation:
+    https://www.tensorflow.org/api_docs/python/tf/nn/pool
+
+    Different from the description, this operation doesn't perform on batch and
+    channel. It only takes tensors of rank `N`.
+
+    ```
+      output[x[0], ..., x[N-1]] =
+        REDUCE_{z[0], ..., z[N-1]}
+          input[
+                x[0] * strides[0] - pad_before[0] + dilation_rate[0]*z[0],
+                ...
+                x[N-1]*strides[N-1] - pad_before[N-1] + dilation_rate[N-1]*z[N-1]
+                ],
+    ```
+
+    The required optional arguments are:
+      - strides: an i64 array specifying the stride (i.e. step) for window
+        loops.
+      - dilations: an i64 array specifying the filter upsampling/input
+        downsampling rate
+      - padding: an i64 array of pairs (low, high) specifying the number of
+        elements to pad along a dimension.
+
+    If strides or dilations attributes are missing then the default value is
+    one for each of the input dimensions. Similarly, padding values are zero
+    for both low and high in each of the dimensions, if not specified.
+  }];
+
+  code commonUtils = libraryCallName # [{
+    int64_t getStride(unsigned i) {
+      assert(i < getNumWindowLoops());
+      if (!strides().hasValue()) return 1;
+      return strides()->getValue()[i]
+        .cast<IntegerAttr>().getValue().getSExtValue();
+    }
+
+    int64_t getDilation(unsigned i) {
+      assert(i < getNumWindowLoops());
+      if (!dilations().hasValue()) return 1;
+      return dilations()->getValue()[i]
+        .cast<IntegerAttr>().getValue().getSExtValue();
+    }
+
+    int64_t getLowPad(unsigned i) {
+      assert(i < getNumWindowLoops());
+      if (!padding().hasValue()) return 0;
+      return padding().getValue().getValue<int64_t>({i, 0});
+    }
+  }];
+}
+
+def ConvOp : PoolingBase_Op<"conv", [NInputs<2>, NOutputs<1>]> {
 
   let description = [{
     Generic n-D convolution as described in the TF documentation:
@@ -278,7 +344,7 @@ def ConvOp : LinalgStructured_Op<"conv", [NInputs<2>, NOutputs<1>]> {
                    OptionalAttr<I64ArrayAttr>:$dilations,
                    OptionalAttr<I64ElementsAttr>:$padding);
 
-  let extraClassDeclaration = libraryCallName # [{
+  let extraClassDeclaration = commonUtils # [{
     // TODO(ntv) extend to support more than 1 dimensions and potentially
     // grouping too.
     unsigned getNumBatchDimensions() { return 1; }
@@ -305,30 +371,10 @@ def ConvOp : LinalgStructured_Op<"conv", [NInputs<2>, NOutputs<1>]> {
       return iters;
     }
 
-    int64_t getStride(unsigned i) {
-      assert(i < getNumWindowLoops());
-      if (!strides().hasValue()) return 1;
-      return strides()->getValue()[i]
-        .cast<IntegerAttr>().getValue().getSExtValue();
-    }
-
-    int64_t getDilation(unsigned i) {
-      assert(i < getNumWindowLoops());
-      if (!dilations().hasValue()) return 1;
-      return dilations()->getValue()[i]
-        .cast<IntegerAttr>().getValue().getSExtValue();
-    }
-
-    int64_t getLowPad(unsigned i) {
-      assert(i < getNumWindowLoops());
-      if (!padding().hasValue()) return 0;
-      return padding().getValue().getValue<int64_t>({i, 0});
-    }
-
     //   F(z0, ..., zN-1, q, k) *
     //     I(b, x0 + z0 - pad_low_0, ..., xN-1 + zN-1 - pad_low_N-1, q)
     //   ->  O(b, x0, ..., xN-1, k)
-    // for N equal to `nWindow`. If there is no padding attirbute, it will be
+    // for N equal to `nWindow`. If there is no padding attribute, it will be
     // ignored.
     llvm::Optional<SmallVector<AffineMap, 8>> referenceIndexingMaps() {
       MLIRContext *context = getContext();
@@ -354,7 +400,7 @@ def ConvOp : LinalgStructured_Op<"conv", [NInputs<2>, NOutputs<1>]> {
       // Window reduction dims: sum_{z[0], ..., z[N-1], q}
       auto zs = makeAffineDimExprs(nWin, idx, context);
       // Construct the weighedSum expression.
-      auto ws = weightedConvInputIndex(*this, xs, zs);
+      auto ws = weightedPoolingInputIndex(*this, xs, zs);
       return SmallVector<AffineMap, 8>{
         // filter[z[0], ..., z[N-1], q, k]
         AffineMap::get(idx, 0, concat(concat(zs, qs), ks)),
@@ -374,6 +420,86 @@ def ConvOp : LinalgStructured_Op<"conv", [NInputs<2>, NOutputs<1>]> {
   let hasFolder = 1;
 }
 
+class SingleInputPoolingBase_Op<string mnemonic>
+  : PoolingBase_Op<mnemonic, [NInputs<2>, NOutputs<1>]> {
+  let description = [{
+    A base class for single input pooling function.
+
+    TODO: Figure out a better way to handle window dimensions, i.e., eliminate
+    the fake memref.
+    The window dimensions are specified by argument `windowDims`. The i-th
+    dimension in the shape of `windowDims` denotes the size of the window along
+    dimension i. For example, if the window size is 2x3, then a memref<2x3>
+    should be passed to the operation as `windowDims`.
+  }];
+
+  let arguments = (ins AnyStridedMemRef:$input,
+                   AnyStridedMemRef:$windowDims,
+                   AnyStridedMemRef:$output,
+                   OptionalAttr<I64ArrayAttr>:$strides,
+                   OptionalAttr<I64ArrayAttr>:$dilations,
+                   OptionalAttr<I64ElementsAttr>:$padding);
+
+  let extraClassDeclaration = commonUtils# [{
+    llvm::Optional<SmallVector<StringRef, 8>> referenceIterators() {
+      // Outer parallel loops are always the number of output dimensions.
+      unsigned nPar = getOutputShapedType(0).getRank();
+      // The window loops has the same number loops with output dimensions.
+      unsigned nWin = nPar;
+      SmallVector<StringRef, 8> iters(nPar, getParallelIteratorTypeName());
+      iters.reserve(nPar + nWin);
+      iters.append(nWin, getWindowIteratorTypeName());
+      return iters;
+    }
+
+    llvm::Optional<SmallVector<AffineMap, 8>> referenceIndexingMaps() {
+      MLIRContext *context = getContext();
+      auto nPar = getNumParallelLoops();
+      auto nWin = getNumWindowLoops();
+      assert(nWin > 0 && "expected at least one window dimension");
+      unsigned idx = 0;
+      auto outputDims = makeAffineDimExprs(nPar, idx, context);
+      auto windowDims = makeAffineDimExprs(nWin, idx, context);
+      // Construct the weighedSum expression.
+      auto inputDims =
+          weightedPoolingInputIndex(*this, outputDims, windowDims);
+      return SmallVector<AffineMap, 8>{
+        // input
+        AffineMap::get(idx, 0, inputDims),
+        // windowDims
+        AffineMap::get(idx, 0, windowDims),
+        // output
+        AffineMap::get(idx, 0, outputDims)
+        };
+    }
+  }];
+
+  let verifier = [{ return ::verify(*this); }];
+
+  let hasFolder = 1;
+}
+
+def PoolingMaxOp: SingleInputPoolingBase_Op<"pooling_max"> {
+  let description = [{
+    Takes max op as pooling operation, i.e., it samples the maximum value in the
+    window.
+  }];
+}
+
+def PoolingMinOp: SingleInputPoolingBase_Op<"pooling_min"> {
+  let description = [{
+    Takes min op as pooling operation, i.e., it samples the minimum value in the
+    window.
+  }];
+}
+
+def PoolingSumOp: SingleInputPoolingBase_Op<"pooling_sum"> {
+  let description = [{
+    Takes add op as pooling operation, i.e., it accumulates the values in the
+    window.
+  }];
+}
+
 //===----------------------------------------------------------------------===//
 // Generic Linalg ops.
 //===----------------------------------------------------------------------===//
@@ -441,10 +567,10 @@ def GenericOp : GenericOpBase<"generic"> {
     specified as attributes. In pretty form, a linalg.generic op is written as:
 
       ```mlir
-        linalg.generic #trait_attribute %A, %B, %C {other-attributes} :
-          memref<?x?xf32, stride_specification>,
-          memref<?x?xf32, stride_specification>,
-          memref<?x?xf32, stride_specification>
+      linalg.generic #trait_attribute %A, %B, %C {other-attributes} :
+        memref<?x?xf32, stride_specification>,
+        memref<?x?xf32, stride_specification>,
+        memref<?x?xf32, stride_specification>
       ```
 
     Where #trait_attributes is an alias of a dictionary attribute containing:
@@ -474,41 +600,41 @@ def GenericOp : GenericOpBase<"generic"> {
     Example:
     Defining a #matmul_trait attribute in MLIR can be done as follows:
       ```mlir
-        func @fma(%a: f32, %b: f32, %c: f32) -> f32 {
-          %d = mulf %a, %b: f32
-          %e = addf %c, %d: f32
-          return %e: f32
-        }
-        #matmul_accesses = [
-          (m, n, k) -> (m, k),
-          (m, n, k) -> (k, n),
-          (m, n, k) -> (m, n)
-        ]
-        #matmul_trait = {
-          doc = "C(m, n) += A(m, k) * B(k, n)",
-          fun = @fma,
-          indexing_maps = #matmul_accesses,
-          library_call = "linalg_matmul",
-          n_views = [2, 1],
-          iterator_types = ["parallel", "parallel", "reduction"]
-        }
+      func @fma(%a: f32, %b: f32, %c: f32) -> f32 {
+        %d = mulf %a, %b: f32
+        %e = addf %c, %d: f32
+        return %e: f32
+      }
+      #matmul_accesses = [
+        (m, n, k) -> (m, k),
+        (m, n, k) -> (k, n),
+        (m, n, k) -> (m, n)
+      ]
+      #matmul_trait = {
+        doc = "C(m, n) += A(m, k) * B(k, n)",
+        fun = @fma,
+        indexing_maps = #matmul_accesses,
+        library_call = "linalg_matmul",
+        n_views = [2, 1],
+        iterator_types = ["parallel", "parallel", "reduction"]
+      }
       ```
 
     And can be reused in multiple places as:
       ```mlir
-        linalg.generic #matmul_trait %A, %B, %C [other-attributes] :
-          memref<?x?xf32, stride_specification>,
-          memref<?x?xf32, stride_specification>,
-          memref<?x?xf32, stride_specification>
+      linalg.generic #matmul_trait %A, %B, %C [other-attributes] :
+        memref<?x?xf32, stride_specification>,
+        memref<?x?xf32, stride_specification>,
+        memref<?x?xf32, stride_specification>
       ```
 
     This may lower to either:
       ```mlir
-        call @linalg_matmul(%A, %B, %C) :
-          (memref<?x?xf32, stride_specification>,
-           memref<?x?xf32, stride_specification>,
-           memref<?x?xf32, stride_specification>)
-          -> ()
+      call @linalg_matmul(%A, %B, %C) :
+        (memref<?x?xf32, stride_specification>,
+         memref<?x?xf32, stride_specification>,
+         memref<?x?xf32, stride_specification>)
+        -> ()
       ```
 
     or IR resembling:
@@ -532,10 +658,10 @@ def GenericOp : GenericOpBase<"generic"> {
     mixing input and output ranked tensor values with input and output memrefs.
 
     ```mlir
-      %C = linalg.generic #trait_attribute %A, %B {other-attributes} :
-        tensor<?x?xf32>,
-        memref<?x?xf32, stride_specification>
-        -> (tensor<?x?xf32>)
+    %C = linalg.generic #trait_attribute %A, %B {other-attributes} :
+      tensor<?x?xf32>,
+      memref<?x?xf32, stride_specification>
+      -> (tensor<?x?xf32>)
     ```
 
     In this case, the number of outputs (args_out) must match the sum of (1) the
@@ -568,10 +694,10 @@ def IndexedGenericOp : GenericOpBase<"indexed_generic"> {
     written as:
 
       ```mlir
-        linalg.indexed_generic #trait_attribute %A, %B, %C {other-attributes} :
-          memref<?x?xf32, stride_specification>,
-          memref<?x?xf32, stride_specification>,
-          memref<?x?xf32, stride_specification>
+      linalg.indexed_generic #trait_attribute %A, %B, %C {other-attributes} :
+        memref<?x?xf32, stride_specification>,
+        memref<?x?xf32, stride_specification>,
+        memref<?x?xf32, stride_specification>
       ```
 
     Where #trait_attributes is an alias of a dictionary attribute containing:
@@ -600,49 +726,53 @@ def IndexedGenericOp : GenericOpBase<"indexed_generic"> {
 
     Example:
     Defining a #matmul_trait attribute in MLIR can be done as follows:
-      ```mlir
-        func @fma(%offset_m: index, %offset_n: index, %offset_k: index,
-                  %a: f32, %b: f32, %c: f32)
-          -> f32
-        {
-          "some_optional_condition"(%offset_m, %offset_n, %offset_k)
-          %d = mulf %a, %b: f32
-          %e = addf %c, %d: f32
-          return %e: f32
-        }
-        #matmul_accesses = [
-          (m, n, k) -> (m, k),
-          (m, n, k) -> (k, n),
-          (m, n, k) -> (m, n)
-        ]
-        #matmul_trait = {
-          doc = "C(m, n) += A(m, k) * B(k, n)",
-          fun = @fma,
-          indexing_maps = #matmul_accesses,
-          library_call = "linalg_matmul",
-          n_views = [2, 1],
-          iterator_types = ["parallel", "parallel", "reduction"]
-        }
-      ```
+
+    ```mlir
+    func @fma(%offset_m: index, %offset_n: index, %offset_k: index,
+              %a: f32, %b: f32, %c: f32)
+      -> f32
+    {
+      "some_optional_condition"(%offset_m, %offset_n, %offset_k)
+      %d = mulf %a, %b: f32
+      %e = addf %c, %d: f32
+      return %e: f32
+    }
+    #matmul_accesses = [
+      (m, n, k) -> (m, k),
+      (m, n, k) -> (k, n),
+      (m, n, k) -> (m, n)
+    ]
+    #matmul_trait = {
+      doc = "C(m, n) += A(m, k) * B(k, n)",
+      fun = @fma,
+      indexing_maps = #matmul_accesses,
+      library_call = "linalg_matmul",
+      n_views = [2, 1],
+      iterator_types = ["parallel", "parallel", "reduction"]
+    }
+    ```
 
     And can be reused in multiple places as:
-      ```mlir
-        linalg.indexed_generic #matmul_trait %A, %B, %C [other-attributes] :
-          memref<?x?xf32, stride_specification>,
-          memref<?x?xf32, stride_specification>,
-          memref<?x?xf32, stride_specification>
-      ```
+
+    ```mlir
+    linalg.indexed_generic #matmul_trait %A, %B, %C [other-attributes] :
+      memref<?x?xf32, stride_specification>,
+      memref<?x?xf32, stride_specification>,
+      memref<?x?xf32, stride_specification>
+    ```
 
     This may lower to either:
-      ```mlir
-        call @linalg_matmul(%offset_m, %offset_n, %offset_k, %A, %B, %C) :
-          (memref<?x?xf32, stride_specification>,
-           memref<?x?xf32, stride_specification>,
-           memref<?x?xf32, stride_specification>)
-          -> ()
-      ```
+
+    ```mlir
+    call @linalg_matmul(%offset_m, %offset_n, %offset_k, %A, %B, %C) :
+      (memref<?x?xf32, stride_specification>,
+       memref<?x?xf32, stride_specification>,
+       memref<?x?xf32, stride_specification>)
+      -> ()
+    ```
 
     or IR resembling:
+
     ```mlir
     loop.for %m = %c0 to %M step %c1 {
       loop.for %n = %c0 to %N step %c1 {
@@ -664,10 +794,10 @@ def IndexedGenericOp : GenericOpBase<"indexed_generic"> {
     memrefs.
 
     ```mlir
-      %C = linalg.indexed_generic #trait_attribute %A, %B {other-attributes}
-      : tensor<?x?xf32>,
-        memref<?x?xf32, stride_specification>
-        -> (tensor<?x?xf32>)
+    %C = linalg.indexed_generic #trait_attribute %A, %B {other-attributes}
+    : tensor<?x?xf32>,
+      memref<?x?xf32, stride_specification>
+      -> (tensor<?x?xf32>)
     ```
 
     In this case, the number of outputs (args_out) must match the sum of (1) the
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOpsInterface.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOpsInterface.td
index 902b118ac7852..46fb9881aba56 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOpsInterface.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOpsInterface.td
@@ -100,6 +100,10 @@ def LinalgStructuredInterface : OpInterface<"LinalgOp"> {
     //===------------------------------------------------------------------===//
     // Input and Output arguments handling.
     //===------------------------------------------------------------------===//
+    InterfaceMethod<
+      "Return one single buffer at position `$i`.",
+      "Value", "getBuffer", (ins "unsigned":$i)
+    >,
     InterfaceMethod<
       "Return the number of inputs and outputs, irrespective of their buffer "
       "or tensor type.",
@@ -119,12 +123,13 @@ def LinalgStructuredInterface : OpInterface<"LinalgOp"> {
     // Other interface methods.
     //===------------------------------------------------------------------===//
     InterfaceMethod<
-      "Return the reference iterators for this named op (if any are specied). "
-      "These reference iterators are used to specify the default behavior of "
-      "the op. Typically this would be a static method but in order to allow "
-      "rank-polymorphic ops, this needs to be per object instance. Named ops "
-      "must define referenceIterators, even if empty for the 0-D case. "
-      "Generic ops on the other hand have a None `referenceIterators`",
+      "Return the reference iterators for this named op (if any are "
+      "specified). These reference iterators are used to specify the default "
+      "behavior of the op. Typically this would be a static method but in "
+      "order to allow rank-polymorphic ops, this needs to be per object "
+      "instance. Named ops must define referenceIterators, even if empty for "
+      "the 0-D case. Generic ops on the other hand have a None "
+      "`referenceIterators`",
       "llvm::Optional<SmallVector<StringRef, 8>>", "referenceIterators"
     >,
     InterfaceMethod<
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgTraits.h b/mlir/include/mlir/Dialect/Linalg/IR/LinalgTraits.h
index f546d3670b6ac..b13b6d268226c 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgTraits.h
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgTraits.h
@@ -184,6 +184,10 @@ class StructuredOpTraits
   //==========================================================================//
   // Input and Output arguments handling.
   //==========================================================================//
+  Value getBuffer(unsigned i) {
+    assert(i < getNumInputsAndOutputBuffers() && "overflowing buffers index");
+    return this->getOperation()->getOperand(i);
+  }
   /// Return the number of inputs and outputs, irrespective of their buffer or
   /// tensor type.
   unsigned getNumInputsAndOutputs() { return nInputs() + nOutputs(); }
diff --git a/mlir/include/mlir/Dialect/Linalg/Passes.h b/mlir/include/mlir/Dialect/Linalg/Passes.h
index 61f88c10470a7..9f52e360c7fb6 100644
--- a/mlir/include/mlir/Dialect/Linalg/Passes.h
+++ b/mlir/include/mlir/Dialect/Linalg/Passes.h
@@ -20,8 +20,10 @@ namespace mlir {
 class FuncOp;
 class ModuleOp;
 template <typename T> class OpPassBase;
+class Pass;
 
 std::unique_ptr<OpPassBase<FuncOp>> createLinalgFusionPass();
+std::unique_ptr<Pass> createLinalgFusionOfTensorOpsPass();
 
 std::unique_ptr<OpPassBase<FuncOp>>
 createLinalgTilingPass(ArrayRef<int64_t> tileSizes = {});
@@ -31,6 +33,7 @@ createLinalgTilingToParallelLoopsPass(ArrayRef<int64_t> tileSizes = {});
 
 std::unique_ptr<OpPassBase<FuncOp>>
 createLinalgPromotionPass(bool dynamicBuffers);
+std::unique_ptr<OpPassBase<FuncOp>> createLinalgPromotionPass();
 
 /// Create a pass to convert Linalg operations to loop.for loops and
 /// std.load/std.store accesses.
diff --git a/mlir/include/mlir/Dialect/Linalg/Passes.td b/mlir/include/mlir/Dialect/Linalg/Passes.td
new file mode 100644
index 0000000000000..210ad1092c668
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Linalg/Passes.td
@@ -0,0 +1,70 @@
+//===-- Passes.td - Linalg pass definition file ------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_LINALG_PASSES
+#define MLIR_DIALECT_LINALG_PASSES
+
+include "mlir/Pass/PassBase.td"
+
+def LinalgFusion : Pass<"linalg-fusion"> {
+  let summary = "Fuse operations in the linalg dialect";
+  let constructor = "mlir::createLinalgFusionPass()";
+}
+
+def LinalgFusionOfTensorOps : Pass<"linalg-fusion-for-tensor-ops"> {
+  let summary = "Fuse operations on RankedTensorType in linalg dialect";
+  let constructor = "mlir::createLinalgFusionOfTensorOpsPass()";
+}
+
+def LinalgLowerToAffineLoops : Pass<"convert-linalg-to-affine-loops"> {
+  let summary = "Lower the operations from the linalg dialect into affine "
+                "loops";
+  let constructor = "mlir::createConvertLinalgToAffineLoopsPass()";
+}
+
+def LinalgLowerToLoops : Pass<"convert-linalg-to-loops"> {
+  let summary = "Lower the operations from the linalg dialect into loops";
+  let constructor = "mlir::createConvertLinalgToLoopsPass()";
+}
+
+def LinalgLowerToParallelLoops : Pass<"convert-linalg-to-parallel-loops"> {
+  let summary = "Lower the operations from the linalg dialect into parallel "
+                "loops";
+  let constructor = "mlir::createConvertLinalgToParallelLoopsPass()";
+}
+
+def LinalgPromotion : Pass<"linalg-promote-subviews"> {
+  let summary = "Promote subview ops to local buffers";
+  let constructor = "mlir::createLinalgPromotionPass()";
+  let options = [
+    Option<"dynamicBuffers", "test-promote-dynamic", "bool",
+           /*default=*/"false", "Test generation of dynamic promoted buffers">
+  ];
+}
+
+def LinalgTiling : Pass<"linalg-tile"> {
+  let summary = "Tile operations in the linalg dialect";
+  let constructor = "mlir::createLinalgTilingPass()";
+  let options = [
+    ListOption<"tileSizes", "linalg-tile-sizes", "int64_t",
+               "Test generation of dynamic promoted buffers",
+               "llvm::cl::ZeroOrMore, llvm::cl::MiscFlags::CommaSeparated">
+  ];
+}
+
+def LinalgTilingToParallelLoops : Pass<"linalg-tile-to-parallel-loops"> {
+  let summary = "Tile operations in the linalg dialect to parallel loops";
+  let constructor = "mlir::createLinalgTilingToParallelLoopsPass()";
+  let options = [
+    ListOption<"tileSizes", "linalg-tile-sizes", "int64_t",
+               "Test generation of dynamic promoted buffers",
+               "llvm::cl::ZeroOrMore, llvm::cl::MiscFlags::CommaSeparated">
+  ];
+}
+
+#endif // MLIR_DIALECT_LINALG_PASSES
diff --git a/mlir/include/mlir/Dialect/LoopOps/CMakeLists.txt b/mlir/include/mlir/Dialect/LoopOps/CMakeLists.txt
index 4a838cc1d52da..2627cbb542dae 100644
--- a/mlir/include/mlir/Dialect/LoopOps/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/LoopOps/CMakeLists.txt
@@ -1,2 +1,8 @@
 add_mlir_dialect(LoopOps loop)
 add_mlir_doc(LoopOps -gen-dialect-doc LoopDialect Dialects/)
+
+set(LLVM_TARGET_DEFINITIONS Passes.td)
+mlir_tablegen(Passes.h.inc -gen-pass-decls)
+add_public_tablegen_target(MLIRLoopPassIncGen)
+
+add_mlir_doc(Passes -gen-pass-doc LoopPasses ./)
diff --git a/mlir/include/mlir/Dialect/LoopOps/LoopOps.td b/mlir/include/mlir/Dialect/LoopOps/LoopOps.td
index 7b010729831b8..c51bf6f3c7b93 100644
--- a/mlir/include/mlir/Dialect/LoopOps/LoopOps.td
+++ b/mlir/include/mlir/Dialect/LoopOps/LoopOps.td
@@ -57,12 +57,12 @@ def ForOp : Loop_Op<"for",
     cases when it is absent from the custom format. For example:
 
     ```mlir
-       loop.for %iv = %lb to %ub step %step {
-         ... // body
-       }
+    loop.for %iv = %lb to %ub step %step {
+      ... // body
+    }
     ```
 
-    "loop.for" can also operate on loop-carried variables and returns the final
+    `loop.for` can also operate on loop-carried variables and returns the final
     values after loop termination. The initial values of the variables are
     passed as additional SSA operands to the "loop.for" following the 3 loop
     control SSA values mentioned above (lower bound, upper bound and step). The
@@ -120,7 +120,7 @@ def ForOp : Loop_Op<"for",
       }
       return %sum : f32
     }
-  ```
+    ```
   }];
   let arguments = (ins Index:$lowerBound,
                        Index:$upperBound,
@@ -142,10 +142,10 @@ def ForOp : Loop_Op<"for",
     OpBuilder getBodyBuilder() {
       return OpBuilder(getBody(), std::prev(getBody()->end()));
     }
-    iterator_range<Block::args_iterator> getRegionIterArgs() {
+    Block::BlockArgListType getRegionIterArgs() {
       return getBody()->getArguments().drop_front();
     }
-    iterator_range<Operation::operand_iterator> getIterOperands() {
+    Operation::operand_range getIterOperands() {
       return getOperands().drop_front(getNumControlOperands());
     }
 
@@ -174,44 +174,47 @@ def IfOp : Loop_Op<"if",
       [SingleBlockImplicitTerminator<"YieldOp">, RecursiveSideEffects]> {
   let summary = "if-then-else operation";
   let description = [{
-    The "loop.if" operation represents an if-then-else construct for
+    The `loop.if` operation represents an if-then-else construct for
     conditionally executing two regions of code. The operand to an if operation
     is a boolean value. For example:
 
     ```mlir
-       loop.if %b  {
-         ...
-       } else {
-         ...
-       }
+    loop.if %b  {
+      ...
+    } else {
+      ...
+    }
     ```
 
-    "loop.if" may also return results that are defined in its regions. The
+    `loop.if` may also return results that are defined in its regions. The
     values defined are determined by which execution path is taken.
-    For example:
+
+    Example:
+
     ```mlir
-       %x, %y = loop.if %b -> (f32, f32) {
-         %x_true = ...
-         %y_true = ...
-         loop.yield %x_true, %y_true : f32, f32
-       } else {
-         %x_false = ...
-         %y_false = ...
-         loop.yield %x_false, %y_false : f32, f32
-       }
+    %x, %y = loop.if %b -> (f32, f32) {
+      %x_true = ...
+      %y_true = ...
+      loop.yield %x_true, %y_true : f32, f32
+    } else {
+      %x_false = ...
+      %y_false = ...
+      loop.yield %x_false, %y_false : f32, f32
+    }
     ```
 
-    "loop.if" regions are always terminated with "loop.yield". If "loop.if"
+    `loop.if` regions are always terminated with "loop.yield". If "loop.if"
     defines no values, the "loop.yield" can be left out, and will be inserted
     implicitly. Otherwise, it must be explicit.
     Also, if "loop.if" defines one or more values, the 'else' block cannot be
     omitted.
 
-    For example:
+    Example:
+
     ```mlir
-       loop.if %b  {
-         ...
-       }
+    loop.if %b  {
+      ...
+    }
     ```
   }];
   let arguments = (ins I1:$condition);
@@ -243,7 +246,9 @@ def IfOp : Loop_Op<"if",
 }
 
 def ParallelOp : Loop_Op<"parallel",
-    [AttrSizedOperandSegments, SingleBlockImplicitTerminator<"YieldOp">]> {
+    [AttrSizedOperandSegments,
+     DeclareOpInterfaceMethods<LoopLikeOpInterface>,
+     SingleBlockImplicitTerminator<"YieldOp">]> {
   let summary = "parallel for operation";
   let description = [{
     The "loop.parallel" operation represents a loop nest taking 4 groups of SSA
@@ -256,7 +261,7 @@ def ParallelOp : Loop_Op<"parallel",
     The lower and upper bounds specify a half-open range: the range includes
     the lower bound but does not include the upper bound. The initial values
     have the same types as results of "loop.parallel". If there are no results,
-    the keyword `init` can be omitted. 
+    the keyword `init` can be omitted.
 
     Semantically we require that the iteration space can be iterated in any
     order, and the loop body can be executed in parallel. If there are data
@@ -274,19 +279,20 @@ def ParallelOp : Loop_Op<"parallel",
     The body region must contain exactly one block that terminates with
     "loop.yield" without operands. Parsing ParallelOp will create such a region
     and insert the terminator when it is absent from the custom format.
-    For example:
+
+    Example:
 
     ```mlir
-       loop.parallel (%iv) = (%lb) to (%ub) step (%step) -> f32 {
-         %zero = constant 0.0 : f32
-         loop.reduce(%zero) : f32 {
-           ^bb0(%lhs : f32, %rhs: f32):
-             %res = addf %lhs, %rhs : f32
-             loop.reduce.return %res : f32
-         }
-       }
+    loop.parallel (%iv) = (%lb) to (%ub) step (%step) -> f32 {
+      %zero = constant 0.0 : f32
+      loop.reduce(%zero) : f32 {
+        ^bb0(%lhs : f32, %rhs: f32):
+          %res = addf %lhs, %rhs : f32
+          loop.reduce.return %res : f32
+      }
+    }
     ```
-   }];
+  }];
 
   let arguments = (ins Variadic<Index>:$lowerBound,
                        Variadic<Index>:$upperBound,
@@ -307,7 +313,7 @@ def ParallelOp : Loop_Op<"parallel",
     unsigned getNumInductionVars() {
       return getBody()->getNumArguments();
     }
-    iterator_range<Block::args_iterator> getInductionVars() {
+    Block::BlockArgListType getInductionVars() {
       return getBody()->getArguments();
     }
     unsigned getNumLoops() { return step().size(); }
@@ -343,14 +349,13 @@ def ReduceOp : Loop_Op<"reduce", [HasParent<"ParallelOp">]> {
     Example:
 
     ```mlir
-      %operand = constant 1.0 : f32
-      loop.reduce(%operand) : f32 {
-        ^bb0(%lhs : f32, %rhs: f32):
-          %res = addf %lhs, %rhs : f32
-          loop.reduce.return %res : f32
-      }
+    %operand = constant 1.0 : f32
+    loop.reduce(%operand) : f32 {
+      ^bb0(%lhs : f32, %rhs: f32):
+        %res = addf %lhs, %rhs : f32
+        loop.reduce.return %res : f32
+    }
     ```
-
   }];
 
   let skipDefaultBuilders = 1;
@@ -373,7 +378,7 @@ def ReduceReturnOp :
     the operand of "loop.reduce". Example for the custom format:
 
     ```mlir
-      loop.reduce.return %res : f32
+    loop.reduce.return %res : f32
     ```
   }];
 
diff --git a/mlir/include/mlir/Dialect/LoopOps/Passes.td b/mlir/include/mlir/Dialect/LoopOps/Passes.td
new file mode 100644
index 0000000000000..444dcfe222015
--- /dev/null
+++ b/mlir/include/mlir/Dialect/LoopOps/Passes.td
@@ -0,0 +1,34 @@
+//===-- Passes.td - Loop pass definition file --------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_LOOP_PASSES
+#define MLIR_DIALECT_LOOP_PASSES
+
+include "mlir/Pass/PassBase.td"
+
+def LoopParallelLoopFusion : Pass<"parallel-loop-fusion"> {
+  let summary = "Fuse adjacent parallel loops";
+  let constructor = "mlir::createParallelLoopFusionPass()";
+}
+
+def LoopParallelLoopSpecialization : Pass<"parallel-loop-specialization"> {
+  let summary = "Specialize parallel loops for vectorization";
+  let constructor = "mlir::createParallelLoopSpecializationPass()";
+}
+
+def LoopParallelLoopTiling : Pass<"parallel-loop-tiling"> {
+  let summary = "Tile parallel loops";
+  let constructor = "mlir::createParallelLoopTilingPass()";
+  let options = [
+    ListOption<"tileSizes", "parallel-loop-tile-sizes", "int64_t",
+               "Factors to tile parallel loops by",
+               "llvm::cl::ZeroOrMore, llvm::cl::MiscFlags::CommaSeparated">
+  ];
+}
+
+#endif // MLIR_DIALECT_LOOP_PASSES
diff --git a/mlir/include/mlir/Dialect/Quant/CMakeLists.txt b/mlir/include/mlir/Dialect/Quant/CMakeLists.txt
index b18726736e942..1a48e4928b334 100644
--- a/mlir/include/mlir/Dialect/Quant/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/Quant/CMakeLists.txt
@@ -1,2 +1,8 @@
 add_mlir_dialect(QuantOps quant)
 add_mlir_doc(QuantOps -gen-dialect-doc QuantDialect Dialects/)
+
+set(LLVM_TARGET_DEFINITIONS Passes.td)
+mlir_tablegen(Passes.h.inc -gen-pass-decls)
+add_public_tablegen_target(MLIRQuantPassIncGen)
+
+add_mlir_doc(Passes -gen-pass-doc QuantPasses ./)
diff --git a/mlir/include/mlir/Dialect/Quant/Passes.td b/mlir/include/mlir/Dialect/Quant/Passes.td
new file mode 100644
index 0000000000000..f55a430069771
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Quant/Passes.td
@@ -0,0 +1,26 @@
+//===-- Passes.td - Quant pass definition file -------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_QUANT_PASSES
+#define MLIR_DIALECT_QUANT_PASSES
+
+include "mlir/Pass/PassBase.td"
+
+def QuantConvertConst : Pass<"quant-convert-const"> {
+  let summary = "Converts constants followed by qbarrier to actual quantized "
+                "values";
+  let constructor = "mlir::quant::createConvertConstPass()";
+}
+
+def QuantConvertSimulatedQuant : Pass<"quant-convert-simulated-quantization"> {
+  let summary = "Converts training-time simulated quantization ops to "
+                "corresponding quantize/dequantize casts";
+  let constructor = "mlir::quant::createConvertSimulatedQuantPass()";
+}
+
+#endif // MLIR_DIALECT_QUANT_PASSES
diff --git a/mlir/include/mlir/Dialect/Quant/QuantOps.td b/mlir/include/mlir/Dialect/Quant/QuantOps.td
index 57e37534efb72..d1a12d8ca6029 100644
--- a/mlir/include/mlir/Dialect/Quant/QuantOps.td
+++ b/mlir/include/mlir/Dialect/Quant/QuantOps.td
@@ -92,7 +92,7 @@ def quant_QuantizeRegionOp : quant_Op<"region", [
     IsolatedFromAbove,
     SingleBlockImplicitTerminator<"ReturnOp">]> {
   let summary = [{
-    The `region operation wraps high-precision ops as a logical low-precision
+    The `region` operation wraps high-precision ops as a logical low-precision
     quantized kernel.
   }];
 
@@ -119,14 +119,15 @@ def quant_ReturnOp : quant_Op<"return", [Terminator]> {
 
 def quant_ConstFakeQuant : quant_Op<"const_fake_quant",
                                     [SameOperandsAndResultType, NoSideEffect]> {
-  let summary =
-      "Simulates the effect of uniform quantization with const range.";
+  let summary = [{
+    Simulates the effect of uniform quantization with const range.
+  }];
 
   let description = [{
     Given a const min, max, num_bits and narrow_range attribute, applies the
     same uniform quantization simulation as is done by the TensorFlow
     fake_quant_with_min_max_args op. See the fakeQuantAttrsToType() utility
-    method and the quant-convert-simulated-quantization pass for futher details.
+    method and the quant-convert-simulated-quantization pass for further details.
   }];
 
   let arguments = (ins
@@ -148,14 +149,15 @@ def quant_ConstFakeQuant : quant_Op<"const_fake_quant",
 
 def quant_ConstFakeQuantPerAxis : quant_Op<"const_fake_quant_per_axis",
                                     [SameOperandsAndResultType, NoSideEffect]> {
-  let summary =
-      "Simulates the effect of per axis uniform quantization with const range.";
+  let summary = [{
+    Simulates the effect of per axis uniform quantization with const range.
+  }];
 
   let description = [{
     Given a const min, max, num_bits and narrow_range attribute, applies the
     same per axis uniform quantization simulation as is done by the TensorFlow
     fake_quant_with_min_max_vars_per_channel op. See the fakeQuantAttrsToType()
-    utility method and the quant-convert-simulated-quantization pass for futher
+    utility method and the quant-convert-simulated-quantization pass for further
     details.
   }];
 
@@ -179,8 +181,7 @@ def quant_ConstFakeQuantPerAxis : quant_Op<"const_fake_quant_per_axis",
 }
 
 def quant_StatisticsRefOp : quant_Op<"stats_ref", [SameOperandsAndResultType]> {
-  let summary =
-      "Indicates that statistics are resolved by reference.";
+  let summary = "Indicates that statistics are resolved by reference.";
 
   let description = [{
     This op acts as an identity that, when encountered at runtime, should result
@@ -198,8 +199,7 @@ def quant_StatisticsRefOp : quant_Op<"stats_ref", [SameOperandsAndResultType]> {
 }
 
 def quant_StatisticsOp : quant_Op<"stats", [SameOperandsAndResultType]> {
-  let summary =
-      "Identity op which associates statistics with the value.";
+  let summary = "Identity op which associates statistics with the value.";
 
   let description = [{
     Associates statistics about the runtime ranges of values observed for
@@ -213,8 +213,11 @@ def quant_StatisticsOp : quant_Op<"stats", [SameOperandsAndResultType]> {
     `layerStats` must be a rank 1 tensor: [2]
     `axisStats` must be a rank 2 tensor: [N, 2], where N=the slice size
       splitted by the `axis` dimension. For example:
-      <?x?x3x2>, axis=3 => N=2
-      <?x?x3x2>, axis=2 => N=6
+
+    ```
+    <?x?x3x2>, axis=3 => N=2
+    <?x?x3x2>, axis=2 => N=6
+    ```
   }];
 
   let arguments = (ins
@@ -263,8 +266,9 @@ def quant_StatisticsOp : quant_Op<"stats", [SameOperandsAndResultType]> {
 }
 
 def quant_CoupledRefOp : quant_Op<"coupled_ref", [SameOperandsAndResultType]> {
-  let summary =
-      "Indicates that one point of the computation is coupled to another.";
+  let summary = [{
+    Indicates that one point of the computation is coupled to another.
+  }];
 
   let description = [{
     Ordinarily, relationships between ops for the purposes of determining
diff --git a/mlir/include/mlir/Dialect/SPIRV/CMakeLists.txt b/mlir/include/mlir/Dialect/SPIRV/CMakeLists.txt
index 771d4c1a43bbb..15f4a4dfe8478 100644
--- a/mlir/include/mlir/Dialect/SPIRV/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/SPIRV/CMakeLists.txt
@@ -1,5 +1,5 @@
 add_mlir_dialect(SPIRVOps spv)
-add_mlir_doc(SPIRVOps -gen-dialect-doc SPIRVDialect Dialects/)
+add_mlir_doc(SPIRVOps -gen-op-doc SPIRVOps Dialects/)
 
 set(LLVM_TARGET_DEFINITIONS SPIRVBase.td)
 mlir_tablegen(SPIRVEnums.h.inc -gen-enum-decls)
@@ -30,3 +30,9 @@ set(LLVM_TARGET_DEFINITIONS TargetAndABI.td)
 mlir_tablegen(TargetAndABI.h.inc -gen-struct-attr-decls)
 mlir_tablegen(TargetAndABI.cpp.inc -gen-struct-attr-defs)
 add_public_tablegen_target(MLIRSPIRVTargetAndABIIncGen)
+
+set(LLVM_TARGET_DEFINITIONS Passes.td)
+mlir_tablegen(Passes.h.inc -gen-pass-decls)
+add_public_tablegen_target(MLIRSPIRVPassIncGen)
+
+add_mlir_doc(Passes -gen-pass-doc SPIRVPasses ./)
diff --git a/mlir/include/mlir/Dialect/SPIRV/Passes.td b/mlir/include/mlir/Dialect/SPIRV/Passes.td
new file mode 100644
index 0000000000000..a038499557974
--- /dev/null
+++ b/mlir/include/mlir/Dialect/SPIRV/Passes.td
@@ -0,0 +1,30 @@
+//===-- Passes.td - SPIRV pass definition file -------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_SPIRV_PASSES
+#define MLIR_DIALECT_SPIRV_PASSES
+
+include "mlir/Pass/PassBase.td"
+
+def SPIRVCompositeTypeLayout : Pass<"decorate-spirv-composite-type-layout"> {
+  let summary = "Decorate SPIR-V composite type with layout info";
+  let constructor = "mlir::spirv::createDecorateSPIRVCompositeTypeLayoutPass()";
+}
+
+def SPIRVLowerABIAttributes : Pass<"spirv-lower-abi-attrs"> {
+  let summary = "Decorate SPIR-V composite type with layout info";
+  let constructor = "mlir::spirv::createLowerABIAttributesPass()";
+}
+
+def SPIRVUpdateVCE : Pass<"spirv-update-vce"> {
+  let summary = "Deduce and attach minimal (version, capabilities, extensions) "
+                "requirements to spv.module ops";
+  let constructor = "mlir::spirv::createUpdateVersionCapabilityExtensionPass()";
+}
+
+#endif // MLIR_DIALECT_SPIRV_PASSES
diff --git a/mlir/include/mlir/Dialect/SPIRV/SPIRVArithmeticOps.td b/mlir/include/mlir/Dialect/SPIRV/SPIRVArithmeticOps.td
index 5605e1a784bfb..2ff30e4be2596 100644
--- a/mlir/include/mlir/Dialect/SPIRV/SPIRVArithmeticOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/SPIRVArithmeticOps.td
@@ -44,16 +44,16 @@ def SPV_FAddOp : SPV_ArithmeticBinaryOp<"FAdd", SPV_Float, [Commutative]> {
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
     ```
     float-scalar-vector-type ::= float-type |
                                  `vector<` integer-literal `x` float-type `>`
     fadd-op ::= ssa-id `=` `spv.FAdd` ssa-use, ssa-use
                           `:` float-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.FAdd %0, %1 : f32
     %5 = spv.FAdd %2, %3 : vector<4xf32>
     ```
@@ -74,7 +74,7 @@ def SPV_FDivOp : SPV_ArithmeticBinaryOp<"FDiv", SPV_Float, []> {
      Results are computed per component.  The resulting value is undefined
     if Operand 2 is 0.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
     ```
     float-scalar-vector-type ::= float-type |
                                  `vector<` integer-literal `x` float-type `>`
@@ -82,9 +82,9 @@ def SPV_FDivOp : SPV_ArithmeticBinaryOp<"FDiv", SPV_Float, []> {
                           `:` float-scalar-vector-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.FDiv %0, %1 : f32
     %5 = spv.FDiv %2, %3 : vector<4xf32>
     ```
@@ -109,16 +109,16 @@ def SPV_FModOp : SPV_ArithmeticBinaryOp<"FMod", SPV_Float, []> {
     1 divided by Operand 2 where if r ≠ 0, the sign of r is the same as the
     sign of Operand 2.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
     ```
     float-scalar-vector-type ::= float-type |
                                  `vector<` integer-literal `x` float-type `>`
     fmod-op ::= ssa-id `=` `spv.FMod` ssa-use, ssa-use
                           `:` float-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.FMod %0, %1 : f32
     %5 = spv.FMod %2, %3 : vector<4xf32>
     ```
@@ -138,7 +138,7 @@ def SPV_FMulOp : SPV_ArithmeticBinaryOp<"FMul", SPV_Float, [Commutative]> {
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     float-scalar-vector-type ::= float-type |
@@ -147,9 +147,9 @@ def SPV_FMulOp : SPV_ArithmeticBinaryOp<"FMul", SPV_Float, [Commutative]> {
                           `:` float-scalar-vector-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.FMul %0, %1 : f32
     %5 = spv.FMul %2, %3 : vector<4xf32>
     ```
@@ -159,7 +159,12 @@ def SPV_FMulOp : SPV_ArithmeticBinaryOp<"FMul", SPV_Float, [Commutative]> {
 // -----
 
 def SPV_FNegateOp : SPV_ArithmeticUnaryOp<"FNegate", SPV_Float, []> {
-  let summary = "Floating-point subtract of Operand from zero.";
+  let summary = [{
+    Inverts the sign bit of Operand. (Note, however, that OpFNegate is still
+    considered a floating-point instruction, and so is subject to the
+    general floating-point rules regarding, for example, subnormals and NaN
+    propagation).
+  }];
 
   let description = [{
     Result Type must be a scalar or vector of floating-point type.
@@ -168,7 +173,7 @@ def SPV_FNegateOp : SPV_ArithmeticUnaryOp<"FNegate", SPV_Float, []> {
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     float-scalar-vector-type ::= float-type |
@@ -176,9 +181,9 @@ def SPV_FNegateOp : SPV_ArithmeticUnaryOp<"FNegate", SPV_Float, []> {
     fmul-op ::= `spv.FNegate` ssa-use `:` float-scalar-vector-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %1 = spv.FNegate %0 : f32
     %3 = spv.FNegate %2 : vector<4xf32>
     ```
@@ -203,7 +208,7 @@ def SPV_FRemOp : SPV_ArithmeticBinaryOp<"FRem", SPV_Float, []> {
     1 divided by Operand 2 where if r ≠ 0, the sign of r is the same as the
     sign of Operand 1.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
     ```
     float-scalar-vector-type ::= float-type |
                                  `vector<` integer-literal `x` float-type `>`
@@ -211,9 +216,9 @@ def SPV_FRemOp : SPV_ArithmeticBinaryOp<"FRem", SPV_Float, []> {
                           `:` float-scalar-vector-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.FRemOp %0, %1 : f32
     %5 = spv.FRemOp %2, %3 : vector<4xf32>
     ```
@@ -233,7 +238,7 @@ def SPV_FSubOp : SPV_ArithmeticBinaryOp<"FSub", SPV_Float, []> {
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
     ```
     float-scalar-vector-type ::= float-type |
                                  `vector<` integer-literal `x` float-type `>`
@@ -241,9 +246,9 @@ def SPV_FSubOp : SPV_ArithmeticBinaryOp<"FSub", SPV_Float, []> {
                           `:` float-scalar-vector-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.FRemOp %0, %1 : f32
     %5 = spv.FRemOp %2, %3 : vector<4xf32>
     ```
@@ -268,7 +273,7 @@ def SPV_IAddOp : SPV_ArithmeticBinaryOp<"IAdd", SPV_Integer, [Commutative]> {
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
     ```
     integer-scalar-vector-type ::= integer-type |
                                  `vector<` integer-literal `x` integer-type `>`
@@ -276,9 +281,9 @@ def SPV_IAddOp : SPV_ArithmeticBinaryOp<"IAdd", SPV_Integer, [Commutative]> {
                           `:` integer-scalar-vector-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.IAdd %0, %1 : i32
     %5 = spv.IAdd %2, %3 : vector<4xi32>
 
@@ -306,7 +311,7 @@ def SPV_IMulOp : SPV_ArithmeticBinaryOp<"IMul", SPV_Integer, [Commutative]> {
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
     ```
     integer-scalar-vector-type ::= integer-type |
                                  `vector<` integer-literal `x` integer-type `>`
@@ -314,9 +319,9 @@ def SPV_IMulOp : SPV_ArithmeticBinaryOp<"IMul", SPV_Integer, [Commutative]> {
                           `:` integer-scalar-vector-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.IMul %0, %1 : i32
     %5 = spv.IMul %2, %3 : vector<4xi32>
 
@@ -344,7 +349,7 @@ def SPV_ISubOp : SPV_ArithmeticBinaryOp<"ISub", SPV_Integer, []> {
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
     ```
     integer-scalar-vector-type ::= integer-type |
                                  `vector<` integer-literal `x` integer-type `>`
@@ -352,9 +357,9 @@ def SPV_ISubOp : SPV_ArithmeticBinaryOp<"ISub", SPV_Integer, []> {
                           `:` integer-scalar-vector-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.ISub %0, %1 : i32
     %5 = spv.ISub %2, %3 : vector<4xi32>
 
@@ -379,7 +384,7 @@ def SPV_SDivOp : SPV_ArithmeticBinaryOp<"SDiv", SPV_Integer, []> {
      Results are computed per component.  The resulting value is undefined
     if Operand 2 is 0.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
     ```
     integer-scalar-vector-type ::= integer-type |
                                  `vector<` integer-literal `x` integer-type `>`
@@ -387,9 +392,9 @@ def SPV_SDivOp : SPV_ArithmeticBinaryOp<"SDiv", SPV_Integer, []> {
                            `:` integer-scalar-vector-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.SDiv %0, %1 : i32
     %5 = spv.SDiv %2, %3 : vector<4xi32>
 
@@ -417,16 +422,16 @@ def SPV_SModOp : SPV_ArithmeticBinaryOp<"SMod", SPV_Integer, []> {
     1 divided by Operand 2 where if r ≠ 0, the sign of r is the same as the
     sign of Operand 2.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
     ```
     integer-scalar-vector-type ::= integer-type |
                                  `vector<` integer-literal `x` integer-type `>`
     smod-op ::= ssa-id `=` `spv.SMod` ssa-use, ssa-use
                            `:` integer-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.SMod %0, %1 : i32
     %5 = spv.SMod %2, %3 : vector<4xi32>
 
@@ -454,16 +459,16 @@ def SPV_SRemOp : SPV_ArithmeticBinaryOp<"SRem", SPV_Integer, []> {
     1 divided by Operand 2 where if r ≠ 0, the sign of r is the same as the
     sign of Operand 1.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
     ```
     integer-scalar-vector-type ::= integer-type |
                                  `vector<` integer-literal `x` integer-type `>`
     srem-op ::= ssa-id `=` `spv.SRem` ssa-use, ssa-use
                            `:` integer-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.SRem %0, %1 : i32
     %5 = spv.SRem %2, %3 : vector<4xi32>
 
@@ -486,16 +491,16 @@ def SPV_UDivOp : SPV_ArithmeticBinaryOp<"UDiv", SPV_Integer, []> {
      Results are computed per component.  The resulting value is undefined
     if Operand 2 is 0.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
     ```
     integer-scalar-vector-type ::= integer-type |
                                  `vector<` integer-literal `x` integer-type `>`
     udiv-op ::= ssa-id `=` `spv.UDiv` ssa-use, ssa-use
                            `:` integer-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.UDiv %0, %1 : i32
     %5 = spv.UDiv %2, %3 : vector<4xi32>
 
@@ -518,16 +523,16 @@ def SPV_UModOp : SPV_ArithmeticBinaryOp<"UMod", SPV_Integer> {
      Results are computed per component.  The resulting value is undefined
     if Operand 2 is 0.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
     ```
     integer-scalar-vector-type ::= integer-type |
                                  `vector<` integer-literal `x` integer-type `>`
     umod-op ::= ssa-id `=` `spv.UMod` ssa-use, ssa-use
                            `:` integer-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.UMod %0, %1 : i32
     %5 = spv.UMod %2, %3 : vector<4xi32>
 
diff --git a/mlir/include/mlir/Dialect/SPIRV/SPIRVAtomicOps.td b/mlir/include/mlir/Dialect/SPIRV/SPIRVAtomicOps.td
index 8aeb5d64797e5..13eddf687f5fd 100644
--- a/mlir/include/mlir/Dialect/SPIRV/SPIRVAtomicOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/SPIRVAtomicOps.td
@@ -82,7 +82,7 @@ def SPV_AtomicAndOp : SPV_AtomicUpdateWithValueOp<"AtomicAnd", []> {
 
     Memory must be a valid memory Scope.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     scope ::= `"CrossDevice"` | `"Device"` | `"Workgroup"` | ...
@@ -94,9 +94,9 @@ def SPV_AtomicAndOp : SPV_AtomicUpdateWithValueOp<"AtomicAnd", []> {
                         ssa-use `,` ssa-use `:` spv-pointer-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %0 = spv.AtomicAnd "Device" "None" %pointer, %value :
                        !spv.ptr<i32, StorageBuffer>
     ```
@@ -113,7 +113,7 @@ def SPV_AtomicCompareExchangeWeakOp : SPV_Op<"AtomicCompareExchangeWeak", []> {
 
     Memory must be a valid memory Scope.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     atomic-compare-exchange-weak-op ::=
@@ -122,9 +122,9 @@ def SPV_AtomicCompareExchangeWeakOp : SPV_Op<"AtomicCompareExchangeWeak", []> {
                                         `:` spv-pointer-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %0 = spv.AtomicCompareExchangeWeak "Workgroup" "Acquire" "None"
                                        %pointer, %value, %comparator
                                        : !spv.ptr<i32, WorkGroup>
@@ -176,7 +176,7 @@ def SPV_AtomicIAddOp : SPV_AtomicUpdateWithValueOp<"AtomicIAdd", []> {
 
     Memory must be a valid memory Scope.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     atomic-iadd-op ::=
@@ -184,9 +184,9 @@ def SPV_AtomicIAddOp : SPV_AtomicUpdateWithValueOp<"AtomicIAdd", []> {
                          ssa-use `,` ssa-use `:` spv-pointer-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %0 = spv.AtomicIAdd "Device" "None" %pointer, %value :
                         !spv.ptr<i32, StorageBuffer>
     ```
@@ -216,7 +216,7 @@ def SPV_AtomicIDecrementOp : SPV_AtomicUpdateOp<"AtomicIDecrement", []> {
 
     Memory must be a valid memory Scope.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     atomic-idecrement-op ::=
@@ -224,9 +224,9 @@ def SPV_AtomicIDecrementOp : SPV_AtomicUpdateOp<"AtomicIDecrement", []> {
                                `:` spv-pointer-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %0 = spv.AtomicIDecrement "Device" "None" %pointer :
                               !spv.ptr<i32, StorageBuffer>
     ```
@@ -255,7 +255,7 @@ def SPV_AtomicIIncrementOp : SPV_AtomicUpdateOp<"AtomicIIncrement", []> {
 
     Memory must be a valid memory Scope.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     atomic-iincrement-op ::=
@@ -263,9 +263,9 @@ def SPV_AtomicIIncrementOp : SPV_AtomicUpdateOp<"AtomicIIncrement", []> {
                                `:` spv-pointer-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %0 = spv.AtomicIncrement "Device" "None" %pointer :
                              !spv.ptr<i32, StorageBuffer>
     ```
@@ -297,7 +297,7 @@ def SPV_AtomicISubOp : SPV_AtomicUpdateWithValueOp<"AtomicISub", []> {
 
     Memory must be a valid memory Scope.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     atomic-isub-op ::=
@@ -305,9 +305,9 @@ def SPV_AtomicISubOp : SPV_AtomicUpdateWithValueOp<"AtomicISub", []> {
                          ssa-use `,` ssa-use `:` spv-pointer-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %0 = spv.AtomicISub "Device" "None" %pointer, %value :
                         !spv.ptr<i32, StorageBuffer>
     ```
@@ -338,7 +338,7 @@ def SPV_AtomicOrOp : SPV_AtomicUpdateWithValueOp<"AtomicOr", []> {
 
     Memory must be a valid memory Scope.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     atomic-or-op ::=
@@ -346,9 +346,9 @@ def SPV_AtomicOrOp : SPV_AtomicUpdateWithValueOp<"AtomicOr", []> {
                        ssa-use `,` ssa-use `:` spv-pointer-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %0 = spv.AtomicOr "Device" "None" %pointer, %value :
                       !spv.ptr<i32, StorageBuffer>
     ```
@@ -380,7 +380,7 @@ def SPV_AtomicSMaxOp : SPV_AtomicUpdateWithValueOp<"AtomicSMax", []> {
 
     Memory must be a valid memory Scope.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     atomic-smax-op ::=
@@ -388,9 +388,9 @@ def SPV_AtomicSMaxOp : SPV_AtomicUpdateWithValueOp<"AtomicSMax", []> {
                          ssa-use `,` ssa-use `:` spv-pointer-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %0 = spv.AtomicSMax "Device" "None" %pointer, %value :
                         !spv.ptr<i32, StorageBuffer>
     ```
@@ -422,7 +422,7 @@ def SPV_AtomicSMinOp : SPV_AtomicUpdateWithValueOp<"AtomicSMin", []> {
 
     Memory must be a valid memory Scope.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     atomic-smin-op ::=
@@ -430,9 +430,9 @@ def SPV_AtomicSMinOp : SPV_AtomicUpdateWithValueOp<"AtomicSMin", []> {
                          ssa-use `,` ssa-use `:` spv-pointer-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %0 = spv.AtomicSMin "Device" "None" %pointer, %value :
                         !spv.ptr<i32, StorageBuffer>
     ```
@@ -464,7 +464,7 @@ def SPV_AtomicUMaxOp : SPV_AtomicUpdateWithValueOp<"AtomicUMax", []> {
 
     Memory must be a valid memory Scope.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     atomic-umax-op ::=
@@ -472,9 +472,9 @@ def SPV_AtomicUMaxOp : SPV_AtomicUpdateWithValueOp<"AtomicUMax", []> {
                          ssa-use `,` ssa-use `:` spv-pointer-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %0 = spv.AtomicUMax "Device" "None" %pointer, %value :
                         !spv.ptr<i32, StorageBuffer>
     ```
@@ -506,7 +506,7 @@ def SPV_AtomicUMinOp : SPV_AtomicUpdateWithValueOp<"AtomicUMin", []> {
 
     Memory must be a valid memory Scope.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     atomic-umin-op ::=
@@ -514,9 +514,9 @@ def SPV_AtomicUMinOp : SPV_AtomicUpdateWithValueOp<"AtomicUMin", []> {
                          ssa-use `,` ssa-use `:` spv-pointer-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %0 = spv.AtomicUMin "Device" "None" %pointer, %value :
                         !spv.ptr<i32, StorageBuffer>
     ```
@@ -548,7 +548,7 @@ def SPV_AtomicXorOp : SPV_AtomicUpdateWithValueOp<"AtomicXor", []> {
 
     Memory must be a valid memory Scope.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     atomic-xor-op ::=
@@ -556,9 +556,9 @@ def SPV_AtomicXorOp : SPV_AtomicUpdateWithValueOp<"AtomicXor", []> {
                         ssa-use `,` ssa-use `:` spv-pointer-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %0 = spv.AtomicXor "Device" "None" %pointer, %value :
                        !spv.ptr<i32, StorageBuffer>
     ```
diff --git a/mlir/include/mlir/Dialect/SPIRV/SPIRVAttributes.h b/mlir/include/mlir/Dialect/SPIRV/SPIRVAttributes.h
index 4e1eb1271b597..f7cbbbe75757c 100644
--- a/mlir/include/mlir/Dialect/SPIRV/SPIRVAttributes.h
+++ b/mlir/include/mlir/Dialect/SPIRV/SPIRVAttributes.h
@@ -92,7 +92,7 @@ class VerCapExtAttr
 };
 
 /// An attribute that specifies the target version, allowed extensions and
-/// capabilities, and resource limits. These information describles a SPIR-V
+/// capabilities, and resource limits. These information describes a SPIR-V
 /// target environment.
 class TargetEnvAttr
     : public Attribute::AttrBase<TargetEnvAttr, Attribute,
diff --git a/mlir/include/mlir/Dialect/SPIRV/SPIRVBase.td b/mlir/include/mlir/Dialect/SPIRV/SPIRVBase.td
index b6715dc9fcd7a..56813c403a678 100644
--- a/mlir/include/mlir/Dialect/SPIRV/SPIRVBase.td
+++ b/mlir/include/mlir/Dialect/SPIRV/SPIRVBase.td
@@ -3285,7 +3285,7 @@ def SPV_OpcodeAttr :
 class SPV_Op<string mnemonic, list<OpTrait> traits = []> :
     Op<SPIRV_Dialect, mnemonic, !listconcat(traits, [
          // TODO(antiagainst): We don't need all of the following traits for
-         // every op; only the suitabble ones should be added automatically
+         // every op; only the suitable ones should be added automatically
          // after ODS supports dialect-specific contents.
          DeclareOpInterfaceMethods<QueryMinVersionInterface>,
          DeclareOpInterfaceMethods<QueryMaxVersionInterface>,
diff --git a/mlir/include/mlir/Dialect/SPIRV/SPIRVBitOps.td b/mlir/include/mlir/Dialect/SPIRV/SPIRVBitOps.td
index d89fcd7229154..378fb30587ad0 100644
--- a/mlir/include/mlir/Dialect/SPIRV/SPIRVBitOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/SPIRVBitOps.td
@@ -76,7 +76,7 @@ def SPV_BitCountOp : SPV_BitUnaryOp<"BitCount", []> {
     The result is the unsigned value that is the number of bits in Base that
     are 1.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     integer-scalar-vector-type ::= integer-type |
@@ -85,9 +85,9 @@ def SPV_BitCountOp : SPV_BitUnaryOp<"BitCount", []> {
                                `:` integer-scalar-vector-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.BitCount %0: i32
     %3 = spv.BitCount %1: vector<4xi32>
     ```
@@ -133,7 +133,7 @@ def SPV_BitFieldInsertOp : SPV_Op<"BitFieldInsert",
     The resulting value is undefined if Count or Offset or their sum is
     greater than the number of bits in the result.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     integer-scalar-vector-type ::= integer-type |
@@ -144,9 +144,9 @@ def SPV_BitFieldInsertOp : SPV_Op<"BitFieldInsert",
                                       `,` integer-type `,` integer-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %0 = spv.BitFieldInsert %base, %insert, %offset, %count : vector<3xi32>, i8, i8
     ```
   }];
@@ -204,7 +204,7 @@ def SPV_BitFieldSExtractOp : SPV_BitFieldExtractOp<"BitFieldSExtract", []> {
     The resulting value is undefined if Count or Offset or their sum is
     greater than the number of bits in the result.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     integer-scalar-vector-type ::= integer-type |
@@ -215,9 +215,9 @@ def SPV_BitFieldSExtractOp : SPV_BitFieldExtractOp<"BitFieldSExtract", []> {
                                          `,` integer-type `,` integer-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %0 = spv.BitFieldSExtract %base, %offset, %count : vector<3xi32>, i8, i8
     ```
   }];
@@ -240,7 +240,7 @@ def SPV_BitFieldUExtractOp : SPV_BitFieldExtractOp<"BitFieldUExtract", []> {
     that there is no sign extension. The remaining bits of the result will
     all be 0.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     integer-scalar-vector-type ::= integer-type |
@@ -251,9 +251,9 @@ def SPV_BitFieldUExtractOp : SPV_BitFieldExtractOp<"BitFieldUExtract", []> {
                                          `,` integer-type `,` integer-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %0 = spv.BitFieldUExtract %base, %offset, %count : vector<3xi32>, i8, i8
     ```
   }];
@@ -281,7 +281,7 @@ def SPV_BitReverseOp : SPV_BitUnaryOp<"BitReverse", []> {
     The bit-number n of the result will be taken from bit-number Width - 1 -
     n of Base, where Width is the OpTypeInt operand of the Result Type.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     integer-scalar-vector-type ::= integer-type |
@@ -290,9 +290,9 @@ def SPV_BitReverseOp : SPV_BitUnaryOp<"BitReverse", []> {
                                  `:` integer-scalar-vector-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.BitReverse %0 : i32
     %3 = spv.BitReverse %1 : vector<4xi32>
     ```
@@ -322,7 +322,7 @@ def SPV_BitwiseAndOp : SPV_BitBinaryOp<"BitwiseAnd", [Commutative]> {
     They must have the same number of components as Result Type. They must
     have the same component width as Result Type.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     integer-scalar-vector-type ::= integer-type |
@@ -331,9 +331,9 @@ def SPV_BitwiseAndOp : SPV_BitBinaryOp<"BitwiseAnd", [Commutative]> {
                                   `:` integer-scalar-vector-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.BitwiseAnd %0, %1 : i32
     %2 = spv.BitwiseAnd %0, %1 : vector<4xi32>
     ```
@@ -363,7 +363,7 @@ def SPV_BitwiseOrOp : SPV_BitBinaryOp<"BitwiseOr", [Commutative]> {
     They must have the same number of components as Result Type. They must
     have the same component width as Result Type.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     integer-scalar-vector-type ::= integer-type |
@@ -372,9 +372,9 @@ def SPV_BitwiseOrOp : SPV_BitBinaryOp<"BitwiseOr", [Commutative]> {
                                   `:` integer-scalar-vector-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.BitwiseOr %0, %1 : i32
     %2 = spv.BitwiseOr %0, %1 : vector<4xi32>
     ```
@@ -404,7 +404,7 @@ def SPV_BitwiseXorOp : SPV_BitBinaryOp<"BitwiseXor", [Commutative]> {
     They must have the same number of components as Result Type. They must
     have the same component width as Result Type.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     integer-scalar-vector-type ::= integer-type |
@@ -413,9 +413,9 @@ def SPV_BitwiseXorOp : SPV_BitBinaryOp<"BitwiseXor", [Commutative]> {
                                   `:` integer-scalar-vector-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.BitwiseXor %0, %1 : i32
     %2 = spv.BitwiseXor %0, %1 : vector<4xi32>
     ```
@@ -453,7 +453,7 @@ def SPV_ShiftLeftLogicalOp : SPV_ShiftOp<"ShiftLeftLogical", []> {
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     integer-scalar-vector-type ::= integer-type |
@@ -464,9 +464,9 @@ def SPV_ShiftLeftLogicalOp : SPV_ShiftOp<"ShiftLeftLogical", []> {
                                           integer-scalar-vector-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.ShiftLeftLogical %0, %1 : i32, i16
     %5 = spv.ShiftLeftLogical %3, %4 : vector<3xi32>, vector<3xi16>
     ```
@@ -501,7 +501,7 @@ def SPV_ShiftRightArithmeticOp : SPV_ShiftOp<"ShiftRightArithmetic", []> {
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     integer-scalar-vector-type ::= integer-type |
@@ -512,9 +512,9 @@ def SPV_ShiftRightArithmeticOp : SPV_ShiftOp<"ShiftRightArithmetic", []> {
                                               integer-scalar-vector-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.ShiftRightArithmetic %0, %1 : i32, i16
     %5 = spv.ShiftRightArithmetic %3, %4 : vector<3xi32>, vector<3xi16>
     ```
@@ -550,7 +550,7 @@ def SPV_ShiftRightLogicalOp : SPV_ShiftOp<"ShiftRightLogical", []> {
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     integer-scalar-vector-type ::= integer-type |
@@ -561,9 +561,9 @@ def SPV_ShiftRightLogicalOp : SPV_ShiftOp<"ShiftRightLogical", []> {
                                            integer-scalar-vector-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.ShiftRightLogical %0, %1 : i32, i16
     %5 = spv.ShiftRightLogical %3, %4 : vector<3xi32>, vector<3xi16>
     ```
@@ -591,7 +591,7 @@ def SPV_NotOp : SPV_BitUnaryOp<"Not", []> {
     have the same number of components as Result Type.  The component width
     must equal the component width in Result Type.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     integer-scalar-vector-type ::= integer-type |
@@ -599,9 +599,9 @@ def SPV_NotOp : SPV_BitUnaryOp<"Not", []> {
     not-op ::= ssa-id `=` `spv.BitNot` ssa-use `:` integer-scalar-vector-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.Not %0 : i32
     %3 = spv.Not %1 : vector<4xi32>
     ```
diff --git a/mlir/include/mlir/Dialect/SPIRV/SPIRVCastOps.td b/mlir/include/mlir/Dialect/SPIRV/SPIRVCastOps.td
index f3af975ee52ee..e5de294e7fb6e 100644
--- a/mlir/include/mlir/Dialect/SPIRV/SPIRVCastOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/SPIRVCastOps.td
@@ -62,16 +62,16 @@ def SPV_BitcastOp : SPV_Op<"Bitcast", [NoSideEffect]> {
     component of S (mapping to multiple components of L) maps its lower-
     ordered bits to the lower-numbered components of L.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     bitcast-op ::= ssa-id `=` `spv.Bitcast` ssa-use
                    `:` operand-type `to` result-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %1 = spv.Bitcast %0 : f32 to i32
     %1 = spv.Bitcast %0 : vector<2xf32> to i64
     %1 = spv.Bitcast %0 : !spv.ptr<f32, Function> to !spv.ptr<i32, Function>
@@ -108,16 +108,16 @@ def SPV_ConvertFToSOp : SPV_CastOp<"ConvertFToS", SPV_Integer, SPV_Float, []> {
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     convert-f-to-s-op ::= ssa-id `=` `spv.ConvertFToSOp` ssa-use
                           `:` operand-type `to` result-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %1 = spv.ConvertFToS %0 : f32 to i32
     %3 = spv.ConvertFToS %2 : vector<3xf32> to vector<3xi32>
     ```
@@ -141,16 +141,16 @@ def SPV_ConvertFToUOp : SPV_CastOp<"ConvertFToU", SPV_Integer, SPV_Float, []> {
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     convert-f-to-u-op ::= ssa-id `=` `spv.ConvertFToUOp` ssa-use
                           `:` operand-type `to` result-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %1 = spv.ConvertFToU %0 : f32 to i32
     %3 = spv.ConvertFToU %2 : vector<3xf32> to vector<3xi32>
     ```
@@ -172,16 +172,16 @@ def SPV_ConvertSToFOp : SPV_CastOp<"ConvertSToF", SPV_Float, SPV_Integer, []> {
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     convert-s-to-f-op ::= ssa-id `=` `spv.ConvertSToFOp` ssa-use
                           `:` operand-type `to` result-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %1 = spv.ConvertSToF %0 : i32 to f32
     %3 = spv.ConvertSToF %2 : vector<3xi32> to vector<3xf32>
     ```
@@ -203,16 +203,16 @@ def SPV_ConvertUToFOp : SPV_CastOp<"ConvertUToF", SPV_Float, SPV_Integer, []> {
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     convert-u-to-f-op ::= ssa-id `=` `spv.ConvertUToFOp` ssa-use
                           `:` operand-type `to` result-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %1 = spv.ConvertUToF %0 : i32 to f32
     %3 = spv.ConvertUToF %2 : vector<3xi32> to vector<3xf32>
     ```
@@ -236,16 +236,16 @@ def SPV_FConvertOp : SPV_CastOp<"FConvert", SPV_Float, SPV_Float, []> {
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     f-convert-op ::= ssa-id `=` `spv.FConvertOp` ssa-use
                      `:` operand-type `to` result-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %1 = spv.FConvertOp %0 : f32 to f64
     %3 = spv.FConvertOp %2 : vector<3xf32> to vector<3xf64>
     ```
@@ -270,16 +270,16 @@ def SPV_SConvertOp : SPV_CastOp<"SConvert", SPV_Integer, SPV_Integer, []> {
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     s-convert-op ::= ssa-id `=` `spv.SConvertOp` ssa-use
                      `:` operand-type `to` result-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %1 = spv.SConvertOp %0 : i32 to i64
     %3 = spv.SConvertOp %2 : vector<3xi32> to vector<3xi64>
     ```
@@ -305,16 +305,16 @@ def SPV_UConvertOp : SPV_CastOp<"UConvert", SPV_Integer, SPV_Integer, []> {
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     u-convert-op ::= ssa-id `=` `spv.UConvertOp` ssa-use
                  `:` operand-type `to` result-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %1 = spv.UConvertOp %0 : i32 to i64
     %3 = spv.UConvertOp %2 : vector<3xi32> to vector<3xi64>
     ```
diff --git a/mlir/include/mlir/Dialect/SPIRV/SPIRVCompositeOps.td b/mlir/include/mlir/Dialect/SPIRV/SPIRVCompositeOps.td
index 8f8073f6a1e68..8e83221023c4e 100644
--- a/mlir/include/mlir/Dialect/SPIRV/SPIRVCompositeOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/SPIRVCompositeOps.td
@@ -44,16 +44,16 @@ def SPV_CompositeConstructOp : SPV_Op<"CompositeConstruct", [NoSideEffect]> {
     type of the result. When constructing a vector, there must be at least
     two Constituent operands.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     composite-construct-op ::= ssa-id `=` `spv.CompositeConstruct`
                                (ssa-use (`,` ssa-use)* )? `:` composite-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %0 = spv.CompositeConstruct %1, %2, %3 : vector<3xf32>
     ```
   }];
@@ -83,7 +83,7 @@ def SPV_CompositeExtractOp : SPV_Op<"CompositeExtract", [NoSideEffect]> {
     bounds.  All composite constituents use zero-based numbering, as
     described by their OpType… instruction.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     composite-extract-op ::= ssa-id `=` `spv.CompositeExtract` ssa-use
@@ -91,9 +91,9 @@ def SPV_CompositeExtractOp : SPV_Op<"CompositeExtract", [NoSideEffect]> {
                              `:` composite-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %0 = spv.Variable : !spv.ptr<!spv.array<4x!spv.array<4xf32>>, Function>
     %1 = spv.Load "Function" %0 ["Volatile"] : !spv.array<4x!spv.array<4xf32>>
     %2 = spv.CompositeExtract %1[1 : i32] : !spv.array<4x!spv.array<4xf32>>
@@ -138,7 +138,7 @@ def SPV_CompositeInsertOp : SPV_Op<"CompositeInsert", [NoSideEffect]> {
     numbering, as described by their OpType… instruction. The type of the
     part selected to modify must match the type of Object.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     composite-insert-op ::= ssa-id `=` `spv.CompositeInsert` ssa-use, ssa-use
@@ -146,9 +146,9 @@ def SPV_CompositeInsertOp : SPV_Op<"CompositeInsert", [NoSideEffect]> {
                             `:` object-type `into` composite-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %0 = spv.CompositeInsert %object, %composite[1 : i32] : f32 into !spv.array<4xf32>
     ```
   }];
diff --git a/mlir/include/mlir/Dialect/SPIRV/SPIRVControlFlowOps.td b/mlir/include/mlir/Dialect/SPIRV/SPIRVControlFlowOps.td
index f800e349f9ac6..bb39d6fbf9f83 100644
--- a/mlir/include/mlir/Dialect/SPIRV/SPIRVControlFlowOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/SPIRVControlFlowOps.td
@@ -29,7 +29,7 @@ def SPV_BranchOp : SPV_Op<"Branch", [
   let description = [{
     This instruction must be the last instruction in a block.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     branch-op ::= `spv.Branch` successor
@@ -37,9 +37,9 @@ def SPV_BranchOp : SPV_Op<"Branch", [
     branch-use-list ::= `(` ssa-use-list `:` type-list-no-parens `)`
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     spv.Branch ^target
     spv.Branch ^target(%0, %1: i32, f32)
     ```
@@ -105,7 +105,7 @@ def SPV_BranchConditionalOp : SPV_Op<"BranchConditional", [
 
     This instruction must be the last instruction in a block.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     branch-conditional-op ::= `spv.BranchConditional` ssa-use
@@ -115,9 +115,9 @@ def SPV_BranchConditionalOp : SPV_Op<"BranchConditional", [
     branch-use-list ::= `(` ssa-use-list `:` type-list-no-parens `)`
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     spv.BranchConditional %condition, ^true_branch, ^false_branch
     spv.BranchConditional %condition, ^true_branch(%0: i32), ^false_branch(%1: i32)
     ```
@@ -219,16 +219,16 @@ def SPV_FunctionCallOp : SPV_Op<"FunctionCall", [
     information: Result Type must match the Return Type of the function, and
     the calling argument types must match the formal parameter types.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     function-call-op ::= `spv.FunctionCall` function-id `(` ssa-use-list `)`
                      `:` function-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     spv.FunctionCall @f_void(%arg0) : (i32) ->  ()
     %0 = spv.FunctionCall @f_iadd(%arg0, %arg1) : (i32, i32) -> i32
     ```
@@ -348,7 +348,7 @@ def SPV_ReturnOp : SPV_Op<"Return", [InFunctionScope, NoSideEffect,
   let description = [{
     This instruction must be the last instruction in a block.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     return-op ::= `spv.Return`
@@ -370,7 +370,7 @@ def SPV_UnreachableOp : SPV_Op<"Unreachable", [InFunctionScope, Terminator]> {
   let description = [{
     This instruction must be the last instruction in a block.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     unreachable-op ::= `spv.Unreachable`
@@ -397,15 +397,15 @@ def SPV_ReturnValueOp : SPV_Op<"ReturnValue", [InFunctionScope, NoSideEffect,
 
     This instruction must be the last instruction in a block.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     return-value-op ::= `spv.ReturnValue` ssa-use `:` spirv-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     spv.ReturnValue %0 : f32
     ```
   }];
diff --git a/mlir/include/mlir/Dialect/SPIRV/SPIRVGLSLOps.td b/mlir/include/mlir/Dialect/SPIRV/SPIRVGLSLOps.td
index 260e954371b12..65108db71dc0d 100644
--- a/mlir/include/mlir/Dialect/SPIRV/SPIRVGLSLOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/SPIRVGLSLOps.td
@@ -91,16 +91,16 @@ def SPV_GLSLFAbsOp : SPV_GLSLUnaryArithmeticOp<"FAbs", 4, SPV_Float> {
     Result Type and the type of x must be the same type. Results are computed
     per component.
 
-    ### Custom assembly format
+    <!-- End of AutoGen section -->
     ```
     float-scalar-vector-type ::= float-type |
                                  `vector<` integer-literal `x` float-type `>`
     abs-op ::= ssa-id `=` `spv.GLSL.FAbs` ssa-use `:`
                float-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.GLSL.FAbs %0 : f32
     %3 = spv.GLSL.FAbs %1 : vector<3xf16>
     ```
@@ -120,16 +120,16 @@ def SPV_GLSLSAbsOp : SPV_GLSLUnaryArithmeticOp<"SAbs", 5, SPV_Integer> {
     types. Result Type and operand types must have the same number of components
     with the same component width. Results are computed per component.
 
-    ### Custom assembly format
+    <!-- End of AutoGen section -->
     ```
     integer-scalar-vector-type ::= integer-type |
                                    `vector<` integer-literal `x` integer-type `>`
     abs-op ::= ssa-id `=` `spv.GLSL.SAbs` ssa-use `:`
                integer-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.GLSL.SAbs %0 : i32
     %3 = spv.GLSL.SAbs %1 : vector<3xi16>
     ```
@@ -151,16 +151,16 @@ def SPV_GLSLCeilOp : SPV_GLSLUnaryArithmeticOp<"Ceil", 9, SPV_Float> {
     Result Type and the type of x must be the same type. Results are computed
     per component.
 
-    ### Custom assembly format
+    <!-- End of AutoGen section -->
     ```
     float-scalar-vector-type ::= float-type |
                                  `vector<` integer-literal `x` float-type `>`
     ceil-op ::= ssa-id `=` `spv.GLSL.Ceil` ssa-use `:`
                 float-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.GLSL.Ceil %0 : f32
     %3 = spv.GLSL.Ceil %1 : vector<3xf16>
     ```
@@ -181,7 +181,7 @@ def SPV_GLSLCosOp : SPV_GLSLUnaryArithmeticOp<"Cos", 14, SPV_Float16or32> {
     Result Type and the type of x must be the same type. Results are computed
     per component.
 
-    ### Custom assembly format
+    <!-- End of AutoGen section -->
     ```
     restricted-float-scalar-type ::=  `f16` | `f32`
     restricted-float-scalar-vector-type ::=
@@ -190,9 +190,9 @@ def SPV_GLSLCosOp : SPV_GLSLUnaryArithmeticOp<"Cos", 14, SPV_Float16or32> {
     cos-op ::= ssa-id `=` `spv.GLSL.Cos` ssa-use `:`
                restricted-float-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.GLSL.Cos %0 : f32
     %3 = spv.GLSL.Cos %1 : vector<3xf16>
     ```
@@ -213,7 +213,7 @@ def SPV_GLSLSinOp : SPV_GLSLUnaryArithmeticOp<"Sin", 13, SPV_Float16or32> {
     Result Type and the type of x must be the same type. Results are computed
     per component.
 
-    ### Custom assembly format
+    <!-- End of AutoGen section -->
     ```
     restricted-float-scalar-type ::=  `f16` | `f32`
     restricted-float-scalar-vector-type ::=
@@ -222,9 +222,9 @@ def SPV_GLSLSinOp : SPV_GLSLUnaryArithmeticOp<"Sin", 13, SPV_Float16or32> {
     sin-op ::= ssa-id `=` `spv.GLSL.Sin` ssa-use `:`
                restricted-float-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.GLSL.Sin %0 : f32
     %3 = spv.GLSL.Sin %1 : vector<3xf16>
     ```
@@ -245,7 +245,7 @@ def SPV_GLSLExpOp : SPV_GLSLUnaryArithmeticOp<"Exp", 27, SPV_Float16or32> {
     Result Type and the type of x must be the same type. Results are
     computed per component.";
 
-    ### Custom assembly format
+    <!-- End of AutoGen section -->
     ```
     restricted-float-scalar-type ::=  `f16` | `f32`
     restricted-float-scalar-vector-type ::=
@@ -254,9 +254,9 @@ def SPV_GLSLExpOp : SPV_GLSLUnaryArithmeticOp<"Exp", 27, SPV_Float16or32> {
     exp-op ::= ssa-id `=` `spv.GLSL.Exp` ssa-use `:`
                restricted-float-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.GLSL.Exp %0 : f32
     %3 = spv.GLSL.Exp %1 : vector<3xf16>
     ```
@@ -278,16 +278,16 @@ def SPV_GLSLFloorOp : SPV_GLSLUnaryArithmeticOp<"Floor", 8, SPV_Float> {
     Result Type and the type of x must be the same type. Results are computed
     per component.
 
-    ### Custom assembly format
+    <!-- End of AutoGen section -->
     ```
     float-scalar-vector-type ::= float-type |
                                  `vector<` integer-literal `x` float-type `>`
     floor-op ::= ssa-id `=` `spv.GLSL.Floor` ssa-use `:`
                 float-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.GLSL.Floor %0 : f32
     %3 = spv.GLSL.Floor %1 : vector<3xf16>
     ```
@@ -308,16 +308,16 @@ def SPV_GLSLInverseSqrtOp : SPV_GLSLUnaryArithmeticOp<"InverseSqrt", 32, SPV_Flo
     Result Type and the type of x must be the same type. Results are computed
     per component.
 
-    ### Custom assembly format
+    <!-- End of AutoGen section -->
     ```
     float-scalar-vector-type ::= float-type |
                                  `vector<` integer-literal `x` float-type `>`
     rsqrt-op ::= ssa-id `=` `spv.GLSL.InverseSqrt` ssa-use `:`
                  float-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.GLSL.InverseSqrt %0 : f32
     %3 = spv.GLSL.InverseSqrt %1 : vector<3xf16>
     ```
@@ -339,7 +339,7 @@ def SPV_GLSLLogOp : SPV_GLSLUnaryArithmeticOp<"Log", 28, SPV_Float16or32> {
     Result Type and the type of x must be the same type. Results are computed
     per component.
 
-    ### Custom assembly format
+    <!-- End of AutoGen section -->
     ```
     restricted-float-scalar-type ::=  `f16` | `f32`
     restricted-float-scalar-vector-type ::=
@@ -348,9 +348,9 @@ def SPV_GLSLLogOp : SPV_GLSLUnaryArithmeticOp<"Log", 28, SPV_Float16or32> {
     log-op ::= ssa-id `=` `spv.GLSL.Log` ssa-use `:`
                restricted-float-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.GLSL.Log %0 : f32
     %3 = spv.GLSL.Log %1 : vector<3xf16>
     ```
@@ -372,16 +372,16 @@ def SPV_GLSLFMaxOp : SPV_GLSLBinaryArithmeticOp<"FMax", 40, SPV_Float> {
     Result Type and the type of all operands must be the same
     type. Results are computed per component.
 
-    ### Custom assembly format
+    <!-- End of AutoGen section -->
     ```
     float-scalar-vector-type ::= float-type |
                                  `vector<` integer-literal `x` float-type `>`
     fmax-op ::= ssa-id `=` `spv.GLSL.FMax` ssa-use `:`
                 float-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.GLSL.FMax %0, %1 : f32
     %3 = spv.GLSL.FMax %0, %1 : vector<3xf16>
     ```
@@ -402,16 +402,16 @@ def SPV_GLSLSMaxOp : SPV_GLSLBinaryArithmeticOp<"SMax", 42, SPV_Integer> {
     components with the same component width. Results are computed per
     component.
 
-    ### Custom assembly format
+    <!-- End of AutoGen section -->
     ```
     integer-scalar-vector-type ::= integer-type |
                                    `vector<` integer-literal `x` integer-type `>`
     smax-op ::= ssa-id `=` `spv.GLSL.SMax` ssa-use `:`
                 integer-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.GLSL.SMax %0, %1 : i32
     %3 = spv.GLSL.SMax %0, %1 : vector<3xi16>
     ```
@@ -433,16 +433,16 @@ def SPV_GLSLFMinOp : SPV_GLSLBinaryArithmeticOp<"FMin", 37, SPV_Float> {
     Result Type and the type of all operands must be the same type. Results are
     computed per component.
 
-    ### Custom assembly format
+    <!-- End of AutoGen section -->
     ```
     float-scalar-vector-type ::= float-type |
                                  `vector<` integer-literal `x` float-type `>`
     fmin-op ::= ssa-id `=` `spv.GLSL.FMin` ssa-use `:`
                 float-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.GLSL.FMin %0, %1 : f32
     %3 = spv.GLSL.FMin %0, %1 : vector<3xf16>
     ```
@@ -463,16 +463,16 @@ def SPV_GLSLSMinOp : SPV_GLSLBinaryArithmeticOp<"SMin", 39, SPV_Integer> {
     components with the same component width. Results are computed per
     component.
 
-    ### Custom assembly format
+    <!-- End of AutoGen section -->
     ```
     integer-scalar-vector-type ::= integer-type |
                                    `vector<` integer-literal `x` integer-type `>`
     smin-op ::= ssa-id `=` `spv.GLSL.SMin` ssa-use `:`
                 integer-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.GLSL.SMin %0, %1 : i32
     %3 = spv.GLSL.SMin %0, %1 : vector<3xi16>
     ```
@@ -493,16 +493,16 @@ def SPV_GLSLFSignOp : SPV_GLSLUnaryArithmeticOp<"FSign", 6, SPV_Float> {
     Result Type and the type of x must be the same type. Results are computed
     per component.
 
-    ### Custom assembly format
+    <!-- End of AutoGen section -->
     ```
     float-scalar-vector-type ::= float-type |
                                  `vector<` integer-literal `x` float-type `>`
     sign-op ::= ssa-id `=` `spv.GLSL.FSign` ssa-use `:`
                 float-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.GLSL.FSign %0 : f32
     %3 = spv.GLSL.FSign %1 : vector<3xf16>
     ```
@@ -522,16 +522,16 @@ def SPV_GLSLSSignOp : SPV_GLSLUnaryArithmeticOp<"SSign", 7, SPV_Integer> {
     types. Result Type and operand types must have the same number of components
     with the same component width. Results are computed per component.
 
-    ### Custom assembly format
+    <!-- End of AutoGen section -->
     ```
     integer-scalar-vector-type ::= integer-type |
                                    `vector<` integer-literal `x` integer-type `>`
     sign-op ::= ssa-id `=` `spv.GLSL.SSign` ssa-use `:`
                 integer-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.GLSL.SSign %0 : i32
     %3 = spv.GLSL.SSign %1 : vector<3xi16>
     ```
@@ -552,16 +552,16 @@ def SPV_GLSLSqrtOp : SPV_GLSLUnaryArithmeticOp<"Sqrt", 31, SPV_Float> {
     Result Type and the type of x must be the same type. Results are computed
     per component.
 
-    ### Custom assembly format
+    <!-- End of AutoGen section -->
     ```
     float-scalar-vector-type ::= float-type |
                                  `vector<` integer-literal `x` float-type `>`
     sqrt-op ::= ssa-id `=` `spv.GLSL.Sqrt` ssa-use `:`
                 float-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.GLSL.Sqrt %0 : f32
     %3 = spv.GLSL.Sqrt %1 : vector<3xf16>
     ```
@@ -582,7 +582,7 @@ def SPV_GLSLTanhOp : SPV_GLSLUnaryArithmeticOp<"Tanh", 21, SPV_Float16or32> {
     Result Type and the type of x must be the same type. Results are computed
     per component.
 
-    ### Custom assembly format
+    <!-- End of AutoGen section -->
     ```
     restricted-float-scalar-type ::=  `f16` | `f32`
     restricted-float-scalar-vector-type ::=
@@ -591,9 +591,9 @@ def SPV_GLSLTanhOp : SPV_GLSLUnaryArithmeticOp<"Tanh", 21, SPV_Float16or32> {
     tanh-op ::= ssa-id `=` `spv.GLSL.Tanh` ssa-use `:`
                 restricted-float-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.GLSL.Tanh %0 : f32
     %3 = spv.GLSL.Tanh %1 : vector<3xf16>
     ```
diff --git a/mlir/include/mlir/Dialect/SPIRV/SPIRVGroupOps.td b/mlir/include/mlir/Dialect/SPIRV/SPIRVGroupOps.td
index e80d6e00c76aa..832e03cba75cc 100644
--- a/mlir/include/mlir/Dialect/SPIRV/SPIRVGroupOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/SPIRVGroupOps.td
@@ -35,16 +35,16 @@ def SPV_SubgroupBallotKHROp : SPV_Op<"SubgroupBallotKHR", []> {
     higher bit number of the last bitmask needed to represent all bits of the
     subgroup invocations.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     subgroup-ballot-op ::= ssa-id `=` `spv.SubgroupBallotKHR`
                                 ssa-use `:` `vector` `<` 4 `x` `i32` `>`
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %0 = spv.SubgroupBallotKHR %predicate : vector<4xi32>
     ```
   }];
diff --git a/mlir/include/mlir/Dialect/SPIRV/SPIRVLogicalOps.td b/mlir/include/mlir/Dialect/SPIRV/SPIRVLogicalOps.td
index a28c843c2023b..11ebd52439332 100644
--- a/mlir/include/mlir/Dialect/SPIRV/SPIRVLogicalOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/SPIRVLogicalOps.td
@@ -52,7 +52,7 @@ def SPV_FOrdEqualOp : SPV_LogicalBinaryOp<"FOrdEqual", SPV_Float, [Commutative]>
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     float-scalar-vector-type ::= float-type |
@@ -60,9 +60,9 @@ def SPV_FOrdEqualOp : SPV_LogicalBinaryOp<"FOrdEqual", SPV_Float, [Commutative]>
     fordequal-op ::= ssa-id `=` `spv.FOrdEqual` ssa-use, ssa-use
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.FOrdEqual %0, %1 : f32
     %5 = spv.FOrdEqual %2, %3 : vector<4xf32>
     ```
@@ -86,7 +86,7 @@ def SPV_FOrdGreaterThanOp : SPV_LogicalBinaryOp<"FOrdGreaterThan", SPV_Float, []
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     float-scalar-vector-type ::= float-type |
@@ -94,9 +94,9 @@ def SPV_FOrdGreaterThanOp : SPV_LogicalBinaryOp<"FOrdGreaterThan", SPV_Float, []
     fordgt-op ::= ssa-id `=` `spv.FOrdGreaterThan` ssa-use, ssa-use
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.FOrdGreaterThan %0, %1 : f32
     %5 = spv.FOrdGreaterThan %2, %3 : vector<4xf32>
     ```
@@ -120,7 +120,7 @@ def SPV_FOrdGreaterThanEqualOp : SPV_LogicalBinaryOp<"FOrdGreaterThanEqual", SPV
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     float-scalar-vector-type ::= float-type |
@@ -128,9 +128,9 @@ def SPV_FOrdGreaterThanEqualOp : SPV_LogicalBinaryOp<"FOrdGreaterThanEqual", SPV
     fordgte-op ::= ssa-id `=` `spv.FOrdGreaterThanEqual` ssa-use, ssa-use
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.FOrdGreaterThanEqual %0, %1 : f32
     %5 = spv.FOrdGreaterThanEqual %2, %3 : vector<4xf32>
     ```
@@ -154,7 +154,7 @@ def SPV_FOrdLessThanOp : SPV_LogicalBinaryOp<"FOrdLessThan", SPV_Float, []> {
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     float-scalar-vector-type ::= float-type |
@@ -162,9 +162,9 @@ def SPV_FOrdLessThanOp : SPV_LogicalBinaryOp<"FOrdLessThan", SPV_Float, []> {
     fordlt-op ::= ssa-id `=` `spv.FOrdLessThan` ssa-use, ssa-use
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.FOrdLessThan %0, %1 : f32
     %5 = spv.FOrdLessThan %2, %3 : vector<4xf32>
     ```
@@ -188,7 +188,7 @@ def SPV_FOrdLessThanEqualOp : SPV_LogicalBinaryOp<"FOrdLessThanEqual", SPV_Float
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     float-scalar-vector-type ::= float-type |
@@ -196,9 +196,9 @@ def SPV_FOrdLessThanEqualOp : SPV_LogicalBinaryOp<"FOrdLessThanEqual", SPV_Float
     fordlte-op ::= ssa-id `=` `spv.FOrdLessThanEqual` ssa-use, ssa-use
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.FOrdLessThanEqual %0, %1 : f32
     %5 = spv.FOrdLessThanEqual %2, %3 : vector<4xf32>
     ```
@@ -219,7 +219,7 @@ def SPV_FOrdNotEqualOp : SPV_LogicalBinaryOp<"FOrdNotEqual", SPV_Float, [Commuta
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     float-scalar-vector-type ::= float-type |
@@ -227,9 +227,9 @@ def SPV_FOrdNotEqualOp : SPV_LogicalBinaryOp<"FOrdNotEqual", SPV_Float, [Commuta
     fordneq-op ::= ssa-id `=` `spv.FOrdNotEqual` ssa-use, ssa-use
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.FOrdNotEqual %0, %1 : f32
     %5 = spv.FOrdNotEqual %2, %3 : vector<4xf32>
     ```
@@ -250,7 +250,7 @@ def SPV_FUnordEqualOp : SPV_LogicalBinaryOp<"FUnordEqual", SPV_Float, [Commutati
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     float-scalar-vector-type ::= float-type |
@@ -258,9 +258,9 @@ def SPV_FUnordEqualOp : SPV_LogicalBinaryOp<"FUnordEqual", SPV_Float, [Commutati
     funordequal-op ::= ssa-id `=` `spv.FUnordEqual` ssa-use, ssa-use
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.FUnordEqual %0, %1 : f32
     %5 = spv.FUnordEqual %2, %3 : vector<4xf32>
     ```
@@ -284,7 +284,7 @@ def SPV_FUnordGreaterThanOp : SPV_LogicalBinaryOp<"FUnordGreaterThan", SPV_Float
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     float-scalar-vector-type ::= float-type |
@@ -292,9 +292,9 @@ def SPV_FUnordGreaterThanOp : SPV_LogicalBinaryOp<"FUnordGreaterThan", SPV_Float
     funordgt-op ::= ssa-id `=` `spv.FUnordGreaterThan` ssa-use, ssa-use
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.FUnordGreaterThan %0, %1 : f32
     %5 = spv.FUnordGreaterThan %2, %3 : vector<4xf32>
     ```
@@ -318,7 +318,7 @@ def SPV_FUnordGreaterThanEqualOp : SPV_LogicalBinaryOp<"FUnordGreaterThanEqual",
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     float-scalar-vector-type ::= float-type |
@@ -326,9 +326,9 @@ def SPV_FUnordGreaterThanEqualOp : SPV_LogicalBinaryOp<"FUnordGreaterThanEqual",
     funordgte-op ::= ssa-id `=` `spv.FUnordGreaterThanEqual` ssa-use, ssa-use
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.FUnordGreaterThanEqual %0, %1 : f32
     %5 = spv.FUnordGreaterThanEqual %2, %3 : vector<4xf32>
     ```
@@ -352,7 +352,7 @@ def SPV_FUnordLessThanOp : SPV_LogicalBinaryOp<"FUnordLessThan", SPV_Float, []>
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     float-scalar-vector-type ::= float-type |
@@ -360,9 +360,9 @@ def SPV_FUnordLessThanOp : SPV_LogicalBinaryOp<"FUnordLessThan", SPV_Float, []>
     funordlt-op ::= ssa-id `=` `spv.FUnordLessThan` ssa-use, ssa-use
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.FUnordLessThan %0, %1 : f32
     %5 = spv.FUnordLessThan %2, %3 : vector<4xf32>
     ```
@@ -386,7 +386,7 @@ def SPV_FUnordLessThanEqualOp : SPV_LogicalBinaryOp<"FUnordLessThanEqual", SPV_F
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     float-scalar-vector-type ::= float-type |
@@ -394,9 +394,9 @@ def SPV_FUnordLessThanEqualOp : SPV_LogicalBinaryOp<"FUnordLessThanEqual", SPV_F
     funordlte-op ::= ssa-id `=` `spv.FUnordLessThanEqual` ssa-use, ssa-use
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.FUnordLessThanEqual %0, %1 : f32
     %5 = spv.FUnordLessThanEqual %2, %3 : vector<4xf32>
     ```
@@ -417,7 +417,7 @@ def SPV_FUnordNotEqualOp : SPV_LogicalBinaryOp<"FUnordNotEqual", SPV_Float, [Com
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     float-scalar-vector-type ::= float-type |
@@ -425,9 +425,9 @@ def SPV_FUnordNotEqualOp : SPV_LogicalBinaryOp<"FUnordNotEqual", SPV_Float, [Com
     funordneq-op ::= ssa-id `=` `spv.FUnordNotEqual` ssa-use, ssa-use
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.FUnordNotEqual %0, %1 : f32
     %5 = spv.FUnordNotEqual %2, %3 : vector<4xf32>
     ```
@@ -448,16 +448,16 @@ def SPV_IEqualOp : SPV_LogicalBinaryOp<"IEqual", SPV_Integer, [Commutative]> {
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
     ```
     integer-scalar-vector-type ::= integer-type |
                                  `vector<` integer-literal `x` integer-type `>`
     iequal-op ::= ssa-id `=` `spv.IEqual` ssa-use, ssa-use
                              `:` integer-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.IEqual %0, %1 : i32
     %5 = spv.IEqual %2, %3 : vector<4xi32>
 
@@ -479,16 +479,16 @@ def SPV_INotEqualOp : SPV_LogicalBinaryOp<"INotEqual", SPV_Integer, [Commutative
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
     ```
     integer-scalar-vector-type ::= integer-type |
                                  `vector<` integer-literal `x` integer-type `>`
     inot-equal-op ::= ssa-id `=` `spv.INotEqual` ssa-use, ssa-use
                                  `:` integer-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.INotEqual %0, %1 : i32
     %5 = spv.INotEqual %2, %3 : vector<4xi32>
 
@@ -513,16 +513,16 @@ def SPV_LogicalAndOp : SPV_LogicalBinaryOp<"LogicalAnd", SPV_Bool, [Commutative]
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     logical-and ::= `spv.LogicalAnd` ssa-use `,` ssa-use
                     `:` operand-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.LogicalAnd %0, %1 : i1
     %2 = spv.LogicalAnd %0, %1 : vector<4xi1>
     ```
@@ -548,16 +548,16 @@ def SPV_LogicalEqualOp : SPV_LogicalBinaryOp<"LogicalEqual", SPV_Bool, [Commutat
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     logical-equal ::= `spv.LogicalEqual` ssa-use `,` ssa-use
                       `:` operand-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.LogicalEqual %0, %1 : i1
     %2 = spv.LogicalEqual %0, %1 : vector<4xi1>
     ```
@@ -578,15 +578,15 @@ def SPV_LogicalNotOp : SPV_LogicalUnaryOp<"LogicalNot", SPV_Bool, []> {
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     logical-not ::= `spv.LogicalNot` ssa-use `:` operand-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.LogicalNot %0 : i1
     %2 = spv.LogicalNot %0 : vector<4xi1>
     ```
@@ -612,16 +612,16 @@ def SPV_LogicalNotEqualOp : SPV_LogicalBinaryOp<"LogicalNotEqual", SPV_Bool, [Co
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     logical-not-equal ::= `spv.LogicalNotEqual` ssa-use `,` ssa-use
                           `:` operand-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.LogicalNotEqual %0, %1 : i1
     %2 = spv.LogicalNotEqual %0, %1 : vector<4xi1>
     ```
@@ -645,16 +645,16 @@ def SPV_LogicalOrOp : SPV_LogicalBinaryOp<"LogicalOr", SPV_Bool, [Commutative]>
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     logical-or ::= `spv.LogicalOr` ssa-use `,` ssa-use
                     `:` operand-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %2 = spv.LogicalOr %0, %1 : i1
     %2 = spv.LogicalOr %0, %1 : vector<4xi1>
     ```
@@ -679,16 +679,16 @@ def SPV_SGreaterThanOp : SPV_LogicalBinaryOp<"SGreaterThan", SPV_Integer, []> {
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
     ```
     integer-scalar-vector-type ::= integer-type |
                                  `vector<` integer-literal `x` integer-type `>`
     sgreater-than-op ::= ssa-id `=` `spv.SGreaterThan` ssa-use, ssa-use
                                     `:` integer-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.SGreaterThan %0, %1 : i32
     %5 = spv.SGreaterThan %2, %3 : vector<4xi32>
 
@@ -713,14 +713,14 @@ def SPV_SGreaterThanEqualOp : SPV_LogicalBinaryOp<"SGreaterThanEqual", SPV_Integ
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
     ```
     integer-scalar-vector-type ::= integer-type |
                                  `vector<` integer-literal `x` integer-type `>`
     sgreater-than-equal-op ::= ssa-id `=` `spv.SGreaterThanEqual` ssa-use, ssa-use
                                           `:` integer-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
     ```
     %4 = spv.SGreaterThanEqual %0, %1 : i32
@@ -746,16 +746,16 @@ def SPV_SLessThanOp : SPV_LogicalBinaryOp<"SLessThan", SPV_Integer, []> {
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
     ```
     integer-scalar-vector-type ::= integer-type |
                                  `vector<` integer-literal `x` integer-type `>`
     sless-than-op ::= ssa-id `=` `spv.SLessThan` ssa-use, ssa-use
                                  `:` integer-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.SLessThan %0, %1 : i32
     %5 = spv.SLessThan %2, %3 : vector<4xi32>
 
@@ -780,16 +780,16 @@ def SPV_SLessThanEqualOp : SPV_LogicalBinaryOp<"SLessThanEqual", SPV_Integer, []
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
     ```
     integer-scalar-vector-type ::= integer-type |
                                  `vector<` integer-literal `x` integer-type `>`
     sless-than-equal-op ::= ssa-id `=` `spv.SLessThanEqual` ssa-use, ssa-use
                                        `:` integer-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.SLessThanEqual %0, %1 : i32
     %5 = spv.SLessThanEqual %2, %3 : vector<4xi32>
 
@@ -822,7 +822,7 @@ def SPV_SelectOp : SPV_Op<"Select",
     component in the result is taken from Object 1, otherwise it is taken
     from Object 2.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     scalar-type ::= integer-type | float-type | boolean-type
@@ -835,9 +835,9 @@ def SPV_SelectOp : SPV_Op<"Select",
                   `:` select-condition-type `,` select-object-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %3 = spv.Select %0, %1, %2 : i1, f32
     %3 = spv.Select %0, %1, %2 : i1, vector<3xi32>
     %3 = spv.Select %0, %1, %2 : vector<3xi1>, vector<3xf32>
@@ -879,16 +879,16 @@ def SPV_UGreaterThanOp : SPV_LogicalBinaryOp<"UGreaterThan", SPV_Integer, []> {
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
     ```
     integer-scalar-vector-type ::= integer-type |
                                  `vector<` integer-literal `x` integer-type `>`
     ugreater-than-op ::= ssa-id `=` `spv.UGreaterThan` ssa-use, ssa-use
                                     `:` integer-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.UGreaterThan %0, %1 : i32
     %5 = spv.UGreaterThan %2, %3 : vector<4xi32>
 
@@ -913,16 +913,16 @@ def SPV_UGreaterThanEqualOp : SPV_LogicalBinaryOp<"UGreaterThanEqual", SPV_Integ
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
     ```
     integer-scalar-vector-type ::= integer-type |
                                  `vector<` integer-literal `x` integer-type `>`
     ugreater-than-equal-op ::= ssa-id `=` `spv.UGreaterThanEqual` ssa-use, ssa-use
                                           `:` integer-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.UGreaterThanEqual %0, %1 : i32
     %5 = spv.UGreaterThanEqual %2, %3 : vector<4xi32>
 
@@ -946,16 +946,16 @@ def SPV_ULessThanOp : SPV_LogicalBinaryOp<"ULessThan", SPV_Integer, []> {
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
     ```
     integer-scalar-vector-type ::= integer-type |
                                  `vector<` integer-literal `x` integer-type `>`
     uless-than-op ::= ssa-id `=` `spv.ULessThan` ssa-use, ssa-use
                                  `:` integer-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.ULessThan %0, %1 : i32
     %5 = spv.ULessThan %2, %3 : vector<4xi32>
 
@@ -981,16 +981,16 @@ def SPV_ULessThanEqualOp :
 
      Results are computed per component.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
     ```
     integer-scalar-vector-type ::= integer-type |
                                  `vector<` integer-literal `x` integer-type `>`
     uless-than-equal-op ::= ssa-id `=` `spv.ULessThanEqual` ssa-use, ssa-use
                                        `:` integer-scalar-vector-type
     ```
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %4 = spv.ULessThanEqual %0, %1 : i32
     %5 = spv.ULessThanEqual %2, %3 : vector<4xi32>
 
diff --git a/mlir/include/mlir/Dialect/SPIRV/SPIRVNonUniformOps.td b/mlir/include/mlir/Dialect/SPIRV/SPIRVNonUniformOps.td
index 9d7366add3989..b3ba0f6d0b761 100644
--- a/mlir/include/mlir/Dialect/SPIRV/SPIRVNonUniformOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/SPIRVNonUniformOps.td
@@ -67,7 +67,7 @@ def SPV_GroupNonUniformBallotOp : SPV_Op<"GroupNonUniformBallot", []> {
 
     Predicate must be a Boolean type.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     scope ::= `"Workgroup"` | `"Subgroup"`
@@ -75,9 +75,9 @@ def SPV_GroupNonUniformBallotOp : SPV_Op<"GroupNonUniformBallot", []> {
                               ssa-use `:` `vector` `<` 4 `x` `integer-type` `>`
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %0 = spv.GroupNonUniformBallot "SubGroup" %predicate : vector<4xi32>
     ```
   }];
@@ -116,7 +116,7 @@ def SPV_GroupNonUniformElectOp : SPV_Op<"GroupNonUniformElect", []> {
 
     Execution must be Workgroup or Subgroup Scope.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     scope ::= `"Workgroup"` | `"Subgroup"`
@@ -124,9 +124,9 @@ def SPV_GroupNonUniformElectOp : SPV_Op<"GroupNonUniformElect", []> {
                              `:` `i1`
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %0 = spv.GroupNonUniformElect : i1
     ```
   }];
@@ -180,7 +180,7 @@ def SPV_GroupNonUniformFAddOp :
     a power of 2. If ClusterSize is greater than the declared SubGroupSize,
     executing this instruction results in undefined behavior.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     scope ::= `"Workgroup"` | `"Subgroup"`
@@ -192,9 +192,9 @@ def SPV_GroupNonUniformFAddOp :
                             `:` float-scalar-vector-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %four = spv.constant 4 : i32
     %scalar = ... : f32
     %vector = ... : vector<4xf32>
@@ -234,7 +234,7 @@ def SPV_GroupNonUniformFMaxOp :
     a power of 2. If ClusterSize is greater than the declared SubGroupSize,
     executing this instruction results in undefined behavior.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     scope ::= `"Workgroup"` | `"Subgroup"`
@@ -246,9 +246,9 @@ def SPV_GroupNonUniformFMaxOp :
                             `:` float-scalar-vector-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %four = spv.constant 4 : i32
     %scalar = ... : f32
     %vector = ... : vector<4xf32>
@@ -288,7 +288,7 @@ def SPV_GroupNonUniformFMinOp :
     a power of 2. If ClusterSize is greater than the declared SubGroupSize,
     executing this instruction results in undefined behavior.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     scope ::= `"Workgroup"` | `"Subgroup"`
@@ -300,9 +300,9 @@ def SPV_GroupNonUniformFMinOp :
                             `:` float-scalar-vector-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %four = spv.constant 4 : i32
     %scalar = ... : f32
     %vector = ... : vector<4xf32>
@@ -339,7 +339,7 @@ def SPV_GroupNonUniformFMulOp :
     a power of 2. If ClusterSize is greater than the declared SubGroupSize,
     executing this instruction results in undefined behavior.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     scope ::= `"Workgroup"` | `"Subgroup"`
@@ -351,9 +351,9 @@ def SPV_GroupNonUniformFMulOp :
                             `:` float-scalar-vector-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %four = spv.constant 4 : i32
     %scalar = ... : f32
     %vector = ... : vector<4xf32>
@@ -388,7 +388,7 @@ def SPV_GroupNonUniformIAddOp :
     a power of 2. If ClusterSize is greater than the declared SubGroupSize,
     executing this instruction results in undefined behavior.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     scope ::= `"Workgroup"` | `"Subgroup"`
@@ -400,9 +400,9 @@ def SPV_GroupNonUniformIAddOp :
                             `:` integer-scalar-vector-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %four = spv.constant 4 : i32
     %scalar = ... : i32
     %vector = ... : vector<4xi32>
@@ -437,7 +437,7 @@ def SPV_GroupNonUniformIMulOp :
     a power of 2. If ClusterSize is greater than the declared SubGroupSize,
     executing this instruction results in undefined behavior.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     scope ::= `"Workgroup"` | `"Subgroup"`
@@ -449,9 +449,9 @@ def SPV_GroupNonUniformIMulOp :
                             `:` integer-scalar-vector-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %four = spv.constant 4 : i32
     %scalar = ... : i32
     %vector = ... : vector<4xi32>
@@ -486,7 +486,7 @@ def SPV_GroupNonUniformSMaxOp :
     a power of 2. If ClusterSize is greater than the declared SubGroupSize,
     executing this instruction results in undefined behavior.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     scope ::= `"Workgroup"` | `"Subgroup"`
@@ -498,9 +498,9 @@ def SPV_GroupNonUniformSMaxOp :
                             `:` integer-scalar-vector-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %four = spv.constant 4 : i32
     %scalar = ... : i32
     %vector = ... : vector<4xi32>
@@ -535,7 +535,7 @@ def SPV_GroupNonUniformSMinOp :
     a power of 2. If ClusterSize is greater than the declared SubGroupSize,
     executing this instruction results in undefined behavior.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     scope ::= `"Workgroup"` | `"Subgroup"`
@@ -547,9 +547,9 @@ def SPV_GroupNonUniformSMinOp :
                             `:` integer-scalar-vector-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %four = spv.constant 4 : i32
     %scalar = ... : i32
     %vector = ... : vector<4xi32>
@@ -585,7 +585,7 @@ def SPV_GroupNonUniformUMaxOp :
     a power of 2. If ClusterSize is greater than the declared SubGroupSize,
     executing this instruction results in undefined behavior.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     scope ::= `"Workgroup"` | `"Subgroup"`
@@ -597,9 +597,9 @@ def SPV_GroupNonUniformUMaxOp :
                             `:` integer-scalar-vector-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %four = spv.constant 4 : i32
     %scalar = ... : i32
     %vector = ... : vector<4xi32>
@@ -635,7 +635,7 @@ def SPV_GroupNonUniformUMinOp :
     a power of 2. If ClusterSize is greater than the declared SubGroupSize,
     executing this instruction results in undefined behavior.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     scope ::= `"Workgroup"` | `"Subgroup"`
@@ -647,9 +647,9 @@ def SPV_GroupNonUniformUMinOp :
                             `:` integer-scalar-vector-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %four = spv.constant 4 : i32
     %scalar = ... : i32
     %vector = ... : vector<4xi32>
diff --git a/mlir/include/mlir/Dialect/SPIRV/SPIRVOps.td b/mlir/include/mlir/Dialect/SPIRV/SPIRVOps.td
index f9796817e38b4..71bf7bdceb3b5 100644
--- a/mlir/include/mlir/Dialect/SPIRV/SPIRVOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/SPIRVOps.td
@@ -67,16 +67,16 @@ def SPV_AccessChainOp : SPV_Op<"AccessChain", [NoSideEffect]> {
 
     - must be an OpConstant when indexing into a structure.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
     ```
     access-chain-op ::= ssa-id `=` `spv.AccessChain` ssa-use
                         `[` ssa-use (',' ssa-use)* `]`
                         `:` pointer-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %0 = "spv.constant"() { value = 1: i32} : () -> i32
     %1 = spv.Variable : !spv.ptr<!spv.struct<f32, !spv.array<4xf32>>, Function>
     %2 = spv.AccessChain %1[%0] : !spv.ptr<!spv.struct<f32, !spv.array<4xf32>>, Function>
@@ -134,7 +134,7 @@ def SPV_ControlBarrierOp : SPV_Op<"ControlBarrier", []> {
     OpControlBarrier will be visible to any other invocation after return
     from that OpControlBarrier.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     scope ::= `"CrossDevice"` | `"Device"` | `"Workgroup"` | ...
@@ -144,9 +144,9 @@ def SPV_ControlBarrierOp : SPV_Op<"ControlBarrier", []> {
     control-barrier-op ::= `spv.ControlBarrier` scope, scope, memory-semantics
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     spv.ControlBarrier "Workgroup", "Device", "Acquire|UniformMemory"
 
     ```
@@ -184,7 +184,7 @@ def SPV_ExecutionModeOp : SPV_Op<"ExecutionMode", [InModuleScope]> {
     mode that takes no Extra Operands, or takes Extra Operands that are not
     <id> operands.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     execution-mode ::= "Invocations" | "SpacingEqual" |
@@ -194,9 +194,9 @@ def SPV_ExecutionModeOp : SPV_Op<"ExecutionMode", [InModuleScope]> {
                           (integer-literal (`, ` integer-literal)* )?
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     spv.ExecutionMode @foo "ContractionOff"
     spv.ExecutionMode @bar "LocalSizeHint", 3, 4, 5
     ```
@@ -237,7 +237,7 @@ def SPV_LoadOp : SPV_Op<"Load", []> {
     literal. If not present, it is the same as specifying the memory operand
     None.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     memory-access ::= `"None"` | `"Volatile"` | `"Aligned", ` integer-literal
@@ -247,9 +247,9 @@ def SPV_LoadOp : SPV_Op<"Load", []> {
                 (`[` memory-access `]`)? ` : ` spirv-element-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %0 = spv.Variable : !spv.ptr<f32, Function>
     %1 = spv.Load "Function" %0 : f32
     %2 = spv.Load "Function" %0 ["Volatile"] : f32
@@ -296,7 +296,7 @@ def SPV_MemoryBarrierOp : SPV_Op<"MemoryBarrier", []> {
     To execute both a memory barrier and a control barrier, see
     OpControlBarrier.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     scope ::= `"CrossDevice"` | `"Device"` | `"Workgroup"` | ...
@@ -306,9 +306,9 @@ def SPV_MemoryBarrierOp : SPV_Op<"MemoryBarrier", []> {
     memory-barrier-op ::= `spv.MemoryBarrier` scope, memory-semantics
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     spv.MemoryBarrier "Device", "Acquire|UniformMemory"
 
     ```
@@ -343,21 +343,22 @@ def SPV_StoreOp : SPV_Op<"Store", []> {
     literal. If not present, it is the same as specifying the memory operand
     None.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     store-op ::= `spv.Store ` storage-class ssa-use `, ` ssa-use `, `
                   (`[` memory-access `]`)? `:` spirv-element-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %0 = spv.Variable : !spv.ptr<f32, Function>
     %1 = spv.FMul ... : f32
     spv.Store "Function" %0, %1 : f32
     spv.Store "Function" %0, %1 ["Volatile"] : f32
     spv.Store "Function" %0, %1 ["Aligned", 4] : f32
+    ```
   }];
 
   let arguments = (ins
@@ -367,6 +368,8 @@ def SPV_StoreOp : SPV_Op<"Store", []> {
     OptionalAttr<I32Attr>:$alignment
   );
 
+  let results = (outs);
+
   let builders = [
     OpBuilder<"Builder *builder, OperationState &state, "
       "Value ptr, Value value, ArrayRef<NamedAttribute> namedAttrs = {}", [{
@@ -375,8 +378,6 @@ def SPV_StoreOp : SPV_Op<"Store", []> {
       state.addAttributes(namedAttrs);
     }]>
   ];
-
-  let results = (outs);
 }
 
 // -----
@@ -391,15 +392,15 @@ def SPV_UndefOp : SPV_Op<"undef", []> {
     bit pattern or abstract value resulting in possibly different concrete,
     abstract, or opaque values.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     undef-op ::= `spv.undef` `:` spirv-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %0 = spv.undef : f32
     %1 = spv.undef : !spv.struct<!spv.array<4 x vector<4xi32>>>
     ```
@@ -441,7 +442,7 @@ def SPV_VariableOp : SPV_Op<"Variable", []> {
     instruction. Initializer must have the same type as the type pointed to
     by Result Type.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     variable-op ::= ssa-id `=` `spv.Variable` (`init(` ssa-use `)`)?
@@ -454,9 +455,9 @@ def SPV_VariableOp : SPV_Op<"Variable", []> {
     descriptor set and binding number. `built_in` specifies SPIR-V
     BuiltIn decoration associated with the op.
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %0 = spv.constant ...
 
     %1 = spv.Variable : !spv.ptr<f32, Function>
diff --git a/mlir/include/mlir/Dialect/SPIRV/SPIRVStructureOps.td b/mlir/include/mlir/Dialect/SPIRV/SPIRVStructureOps.td
index ed8f0b1d38831..3bcbdc8069a15 100644
--- a/mlir/include/mlir/Dialect/SPIRV/SPIRVStructureOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/SPIRVStructureOps.td
@@ -33,16 +33,16 @@ def SPV_AddressOfOp : SPV_Op<"_address_of", [InFunctionScope, NoSideEffect]> {
     type, this op returns a pointer type as well, and the type is the same as
     the variable referenced.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     spv-address-of-op ::= ssa-id `=` `spv._address_of` symbol-ref-id
                                      `:` spirv-pointer-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %0 = spv._address_of @global_var : !spv.ptr<f32, Input>
     ```
   }];
@@ -86,16 +86,16 @@ def SPV_ConstantOp : SPV_Op<"constant", [ConstantLike, NoSideEffect]> {
     and this op is purely mechanical; so it can be scoped to the binary
     (de)serialization process.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     spv-constant-op ::= ssa-id `=` `spv.constant` attribute-value
                         (`:` spirv-type)?
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %0 = spv.constant true
     %1 = spv.constant dense<[2, 3]> : vector<2xf32>
     %2 = spv.constant [dense<3.0> : vector<2xf32>] : !spv.array<1xvector<2xf32>>
@@ -159,7 +159,7 @@ def SPV_EntryPointOp : SPV_Op<"EntryPoint", [InModuleScope]> {
     storage classes are all storage classes used in declaring all
     global variables referenced by the entry point’s call tree.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     execution-model ::= "Vertex" | "TesellationControl" |
@@ -169,9 +169,9 @@ def SPV_EntryPointOp : SPV_Op<"EntryPoint", [InModuleScope]> {
                        symbol-reference (`, ` symbol-reference)*
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     spv.EntryPoint "GLCompute" @foo
     spv.EntryPoint "Kernel" @foo, @var1, @var2
 
@@ -214,7 +214,7 @@ def SPV_FuncOp : SPV_Op<"func", [
     This op itself takes no operands and generates no results. Its region
     can take zero or more arguments and return zero or one values.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     spv-function-control ::= "None" | "Inline" | "DontInline" | ...
@@ -222,9 +222,9 @@ def SPV_FuncOp : SPV_Op<"func", [
                          spv-function-control region
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     spv.func @foo() -> () "None" { ... }
     spv.func @bar() -> () "Inline|Pure" { ... }
     ```
@@ -299,7 +299,7 @@ def SPV_GlobalVariableOp : SPV_Op<"globalVariable", [InModuleScope, Symbol]> {
     `spv.globalVariable` operation in module scope. Initializer must
     have the same type as the type of the defined symbol.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     variable-op ::= `spv.globalVariable` spirv-type symbol-ref-id
@@ -313,9 +313,9 @@ def SPV_GlobalVariableOp : SPV_Op<"globalVariable", [InModuleScope, Symbol]> {
     descriptor set and binding number. `built_in` specifies SPIR-V
     BuiltIn decoration associated with the op.
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     spv.globalVariable @var0 : !spv.ptr<f32, Input> @var0
     spv.globalVariable @var1 initializer(@var0) : !spv.ptr<f32, Output>
     spv.globalVariable @var2 bind(1, 2) : !spv.ptr<f32, Uniform>
@@ -379,7 +379,7 @@ def SPV_ModuleOp : SPV_Op<"module",
     This op has only one region, which only contains one block. The block
     must be terminated via the `spv._module_end` op.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     addressing-model ::= `Logical` | `Physical32` | `Physical64` | ...
@@ -390,9 +390,9 @@ def SPV_ModuleOp : SPV_Op<"module",
                       region
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     spv.module Logical GLSL450  {}
 
     spv.module Logical Vulkan
@@ -477,16 +477,16 @@ def SPV_ReferenceOfOp : SPV_Op<"_reference_of", [NoSideEffect]> {
     for modelling purpose in the SPIR-V dialect. This op's return type is
     the same as the specialization constant.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     spv-reference-of-op ::= ssa-id `=` `spv._reference_of` symbol-ref-id
                                        `:` spirv-scalar-type
     ```
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     %0 = spv._reference_of @spec_const : f32
     ```
   }];
@@ -522,7 +522,7 @@ def SPV_SpecConstantOp : SPV_Op<"specConstant", [InModuleScope, Symbol]> {
     `OpSpecConstantComposite` and `OpSpecConstantOp` are modelled with
     separate ops.
 
-    ### Custom assembly form
+    <!-- End of AutoGen section -->
 
     ```
     spv-spec-constant-op ::= `spv.specConstant` symbol-ref-id
@@ -533,9 +533,9 @@ def SPV_SpecConstantOp : SPV_Op<"specConstant", [InModuleScope, Symbol]> {
     where `spec_id` specifies the SPIR-V SpecId decoration associated with
     the op.
 
-    For example:
+    #### Example:
 
-    ```
+    ```mlir
     spv.specConstant @spec_const1 = true
     spv.specConstant @spec_const2 spec_id(5) = 42 : i32
     ```
diff --git a/mlir/include/mlir/Dialect/Shape/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/Shape/IR/CMakeLists.txt
index 6f4d28c339c8a..e759c9ad22b09 100644
--- a/mlir/include/mlir/Dialect/Shape/IR/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/Shape/IR/CMakeLists.txt
@@ -1,5 +1,2 @@
-set(LLVM_TARGET_DEFINITIONS ShapeOps.td)
-mlir_tablegen(ShapeOps.h.inc -gen-op-decls)
-mlir_tablegen(ShapeOps.cpp.inc -gen-op-defs)
-mlir_tablegen(ShapeOpsDialect.h.inc -gen-dialect-decls)
-add_public_tablegen_target(MLIRShapeOpsIncGen)
+add_mlir_dialect(ShapeOps shape)
+add_mlir_doc(ShapeOps -gen-dialect-doc ShapeDialect Dialects/)
diff --git a/mlir/include/mlir/Dialect/Shape/IR/Shape.h b/mlir/include/mlir/Dialect/Shape/IR/Shape.h
index 37ea17e2bfec4..0134ba9381ac0 100644
--- a/mlir/include/mlir/Dialect/Shape/IR/Shape.h
+++ b/mlir/include/mlir/Dialect/Shape/IR/Shape.h
@@ -16,6 +16,8 @@
 
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/OpDefinition.h"
+#include "mlir/IR/OpImplementation.h"
+#include "mlir/Interfaces/InferTypeOpInterface.h"
 #include "mlir/Interfaces/SideEffects.h"
 
 namespace mlir {
diff --git a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
index da2b964ce6aaf..77516b9f29cde 100644
--- a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
+++ b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
@@ -14,6 +14,7 @@
 #define SHAPE_OPS
 
 include "mlir/IR/OpBase.td"
+include "mlir/Interfaces/InferTypeOpInterface.td"
 include "mlir/Interfaces/SideEffects.td"
 
 // TODO(jpienaar): Move to base.
@@ -34,7 +35,7 @@ def ShapeDialect : Dialect {
     shapes as input, return the invalid shape if one of its operands is an
     invalid shape. This avoids flagging multiple errors for one verification
     failure. The dialect itself does not specify how errors should be combined
-    (there are multiple different options, from always chosing first operand,
+    (there are multiple different options, from always choosing first operand,
     concatting etc. on how to combine them).
   }];
 
@@ -168,17 +169,37 @@ def Shape_ConstantOp : Shape_Op<"constant", []> {
   let parser = [{ return ::parse$cppClass(parser, result); }];
 }
 
-def Shape_CreateShapeOp : Shape_Op<"create_shape", []> {
-  let summary = "Creates a shape descriptor from a tensor";
+def Shape_FromExtentTensorOp : Shape_Op<"from_extent_tensor", []> {
+  let summary = "Creates a shape from a tensor of extents";
   let description = [{
-    Creates a shape from a 1D integral tensor. The rank equals the number of
-    elements in the tensor, and extent matches the values of the elements.
+    Creates a shape from a 1D integral tensor of extents. The rank of the
+    resulting shape equals the number of elements in the tensor, and the
+    extents match the values of the elements.
   }];
 
   let arguments = (ins I32Tensor:$input);
   let results = (outs Shape_ShapeType:$result);
 }
 
+def Shape_ToExtentTensorOp : Shape_Op<"to_tensor", []> {
+  let summary = "Creates a dimension tensor from a shape";
+  // TODO: Think more about the error situation. Perhaps factor out the
+  // error detection into a separate op so downstream consumers can control
+  // their error behavior. Then this op would assume that the input has
+  // been properly checked to not be an error (and could thus be a
+  // NoSideEffect op).
+  let description = [{
+    Converts a shape to a 1D integral tensor of extents. The number of elements
+    in the tensor equals the rank of the shape, and the elements equal the
+    extents of the shape.
+
+    If the shape represents an error, then this op currently aborts the program.
+  }];
+
+  let arguments = (ins Shape_ShapeType:$input);
+  let results = (outs I32Tensor:$result);
+}
+
 def Shape_JoinOp : Shape_Op<"join", []> {
   let summary = "Returns the least general shape.size of its operands";
   let description = [{
@@ -299,4 +320,50 @@ def Shape_DebugPrintOp : Shape_Op<"debug_print", []> {
   let results =  (outs Shape_ShapeOrSizeType:$output);
 }
 
+def Shape_SplitAtOp : Shape_Op<"split_at",
+    [DeclareOpInterfaceMethods<InferTypeOpInterface>]> {
+  let summary = "Splits a shape at a given index.";
+  let description = [{
+    Splits a shape at a given dimension `index`, returning two shapes.
+    If `index` is negative, it is treated as indexing from the back of the
+    shape. This negative-handling behavior is important when handling unranked
+    shapes, where the positive index is not necessarily knowable due to a
+    dynamic number of leading dimensions.
+
+    Examples:
+    - split_at([4,5,6], index=0) -> [], [4,5,6]
+    - split_at([4,5,6], index=1) -> [4], [5,6]
+    - split_at([4,5,6], index=2) -> [4,5], [6]
+    - split_at([4,5,6], index=3) -> [4,5,6], []
+    - split_at([4,5,6], index=4) -> error
+    - split_at([4,5,6], index=-1) -> [4,5], [6]
+    - split_at([4,5,6], index=-2) -> [4], [5,6]
+    - split_at([4,5,6], index=-3) -> [], [4,5,6]
+    - split_at([4,5,6], index=-4) -> error
+
+    Requires:
+    - `index` is in the range [-rank(operand),rank(operand)]
+  }];
+
+  let arguments = (ins Shape_ShapeType:$operand, I32:$index);
+  let results = (outs Shape_ShapeType:$head, Shape_ShapeType:$tail);
+}
+
+def Shape_ConcatOp : Shape_Op<"concat",
+    [DeclareOpInterfaceMethods<InferTypeOpInterface>]> {
+  let summary = "Concatenates two shapes.";
+  let description = [{
+    Creates a shape whose dimensions consist of first the dimensions from `lhs`
+    followed by the dimensions of `rhs`.
+
+    Example:
+    concat([2,3], [4,5]) -> [2,3,4,5]
+    concat([], []) -> []
+    concat([], [4,5,6]) -> [4,5,6]
+  }];
+
+  let arguments = (ins Shape_ShapeType:$lhs, Shape_ShapeType:$rhs);
+  let results = (outs Shape_ShapeType:$result);
+}
+
 #endif // SHAPE_OPS
diff --git a/mlir/include/mlir/Dialect/StandardOps/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/StandardOps/IR/CMakeLists.txt
index 9abc8430c16b0..b9178c5a0db34 100644
--- a/mlir/include/mlir/Dialect/StandardOps/IR/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/StandardOps/IR/CMakeLists.txt
@@ -5,3 +5,5 @@ mlir_tablegen(OpsDialect.h.inc -gen-dialect-decls)
 mlir_tablegen(OpsEnums.h.inc -gen-enum-decls)
 mlir_tablegen(OpsEnums.cpp.inc -gen-enum-defs)
 add_public_tablegen_target(MLIRStandardOpsIncGen)
+
+add_mlir_doc(Ops -gen-op-doc StandardOps Dialects/)
diff --git a/mlir/include/mlir/Dialect/StandardOps/IR/Ops.td b/mlir/include/mlir/Dialect/StandardOps/IR/Ops.td
index 9ca5465f3787b..d61e74fcfea62 100644
--- a/mlir/include/mlir/Dialect/StandardOps/IR/Ops.td
+++ b/mlir/include/mlir/Dialect/StandardOps/IR/Ops.td
@@ -133,8 +133,20 @@ def AbsFOp : FloatUnaryOp<"absf"> {
   let description = [{
     The `absf` operation computes the absolute value. It takes one operand and
     returns one result of the same type. This type may be a float scalar type,
-    a vector whose element type is float, or a tensor of floats. It has no
-    standard attributes.
+    a vector whose element type is float, or a tensor of floats.
+
+    Example:
+
+    ```mlir
+    // Scalar absolute value.
+    %a = absf %b : f64
+
+    // SIMD vector element-wise absolute value.
+    %f = absf %g : vector<4xf32>
+
+    // Tensor element-wise absolute value.
+    %x = absf %y : tensor<4x?xf8>
+    ```
   }];
 }
 
@@ -144,6 +156,34 @@ def AbsFOp : FloatUnaryOp<"absf"> {
 
 def AddFOp : FloatArithmeticOp<"addf"> {
   let summary = "floating point addition operation";
+  let description = [{
+    Syntax:
+
+    ```
+    operation ::= ssa-id `=` `std.addf` ssa-use `,` ssa-use `:` type
+    ```
+
+    The `addf` operation takes two operands and returns one result, each of
+    these is required to be the same type. This type may be a floating point
+    scalar type, a vector whose element type is a floating point type, or a
+    floating point tensor.
+
+    Example:
+
+    ```mlir
+    // Scalar addition.
+    %a = addf %b, %c : f64
+
+    // SIMD vector addition, e.g. for Intel SSE.
+    %f = addf %g, %h : vector<4xf32>
+
+    // Tensor addition.
+    %x = addf %y, %z : tensor<4x?xbf16>
+    ```
+
+    TODO: In the distant future, this will accept optional attributes for fast
+    math, contraction, rounding mode, and other controls.
+  }];
   let hasFolder = 1;
 }
 
@@ -153,6 +193,31 @@ def AddFOp : FloatArithmeticOp<"addf"> {
 
 def AddIOp : IntArithmeticOp<"addi", [Commutative]> {
   let summary = "integer addition operation";
+  let description = [{
+    Syntax:
+
+    ```
+    operation ::= ssa-id `=` `std.addi` ssa-use `,` ssa-use `:` type
+    ```
+
+    The `addi` operation takes two operands and returns one result, each of
+    these is required to be the same type. This type may be an integer scalar
+    type, a vector whose element type is integer, or a tensor of integers. It
+    has no standard attributes.
+
+    Example:
+
+    ```mlir
+    // Scalar addition.
+    %a = addi %b, %c : i64
+
+    // SIMD vector element-wise addition, e.g. for Intel SSE.
+    %f = addi %g, %h : vector<4xi32>
+
+    // Tensor element-wise addition.
+    %x = addi %y, %z : tensor<4x?xi8>
+    ```
+  }];
   let hasFolder = 1;
 }
 
@@ -163,32 +228,42 @@ def AddIOp : IntArithmeticOp<"addi", [Commutative]> {
 def AllocOp : Std_Op<"alloc"> {
   let summary = "memory allocation operation";
   let description = [{
-    The "alloc" operation allocates a region of memory, as specified by its
-    memref type. For example:
+    The `alloc` operation allocates a region of memory, as specified by its
+    memref type.
+
+    Example:
 
-      %0 = alloc() : memref<8x64xf32, (d0, d1) -> (d0, d1), 1>
+    ```mlir
+    %0 = alloc() : memref<8x64xf32, (d0, d1) -> (d0, d1), 1>
+    ```
 
     The optional list of dimension operands are bound to the dynamic dimensions
     specified in its memref type. In the example below, the ssa value '%d' is
     bound to the second dimension of the memref (which is dynamic).
 
-      %0 = alloc(%d) : memref<8x?xf32, (d0, d1) -> (d0, d1), 1>
+    ```mlir
+    %0 = alloc(%d) : memref<8x?xf32, (d0, d1) -> (d0, d1), 1>
+    ```
 
     The optional list of symbol operands are bound to the symbols of the
     memrefs affine map. In the example below, the ssa value '%s' is bound to
     the symbol 's0' in the affine map specified in the allocs memref type.
 
-      %0 = alloc()[%s] : memref<8x64xf32, (d0, d1)[s0] -> ((d0 + s0), d1), 1>
+    ```mlir
+    %0 = alloc()[%s] : memref<8x64xf32, (d0, d1)[s0] -> ((d0 + s0), d1), 1>
+    ```
 
     This operation returns a single ssa value of memref type, which can be used
     by subsequent load and store operations.
 
     The optional `alignment` attribute may be specified to ensure that the
     region of memory that will be indexed is aligned at the specified byte
-    boundary. TODO(b/144281289) optional alignment attribute to MemRefType.
+    boundary.
 
-      %0 = alloc()[%s] {alignment = 8} :
-        memref<8x64xf32, (d0, d1)[s0] -> ((d0 + s0), d1), 1>
+    ```mlir
+    %0 = alloc()[%s] {alignment = 8} :
+      memref<8x64xf32, (d0, d1)[s0] -> ((d0 + s0), d1), 1>
+    ```
   }];
 
   let arguments = (ins Variadic<Index>:$value,
@@ -238,6 +313,31 @@ def AllocOp : Std_Op<"alloc"> {
 
 def AndOp : IntArithmeticOp<"and", [Commutative]> {
   let summary = "integer binary and";
+  let description = [{
+    Syntax:
+
+    ```
+    operation ::= ssa-id `=` `std.and` ssa-use `,` ssa-use `:` type
+    ```
+
+    The `and` operation takes two operands and returns one result, each of these
+    is required to be the same type. This type may be an integer scalar type, a
+    vector whose element type is integer, or a tensor of integers. It has no
+    standard attributes.
+
+    Example:
+
+    ```mlir
+    // Scalar integer bitwise and.
+    %a = and %b, %c : i64
+
+    // SIMD vector element-wise bitwise integer and.
+    %f = and %g, %h : vector<4xi32>
+
+    // Tensor element-wise bitwise integer and.
+    %x = and %y, %z : tensor<4x?xi8>
+    ```
+  }];
   let hasFolder = 1;
 }
 
@@ -249,7 +349,7 @@ def AssumeAlignmentOp : Std_Op<"assume_alignment"> {
   let summary =
       "assertion that gives alignment information to the input memref";
   let description = [{
-    The assume alignment operation takes a memref and a integer of alignment
+    The `assume_alignment` operation takes a memref and a integer of alignment
     value, and internally annotates the buffer with the given alignment. If
     the buffer isn't aligned to the given alignment, the behavior is undefined.
 
@@ -296,7 +396,7 @@ def AtomicRMWOp : Std_Op<"atomic_rmw", [
     ]> {
   let summary = "atomic read-modify-write operation";
   let description = [{
-    The "atomic_rmw" operation provides a way to perform a read-modify-write
+    The `atomic_rmw` operation provides a way to perform a read-modify-write
     sequence that is free from data races. The kind enumeration specifies the
     modification to perform. The value operand represents the new value to be
     applied during the modification. The memref operand represents the buffer
@@ -307,7 +407,7 @@ def AtomicRMWOp : Std_Op<"atomic_rmw", [
     Example:
 
     ```mlir
-      %x = atomic_rmw "addf" %value, %I[%i] : (f32, memref<10xf32>) -> f32
+    %x = atomic_rmw "addf" %value, %I[%i] : (f32, memref<10xf32>) -> f32
     ```
   }];
 
@@ -338,15 +438,19 @@ def BranchOp : Std_Op<"br",
     [DeclareOpInterfaceMethods<BranchOpInterface>, NoSideEffect, Terminator]> {
   let summary = "branch operation";
   let description = [{
-    The "br" operation represents a branch operation in a function.
+    The `br` operation represents a branch operation in a function.
     The operation takes variable number of operands and produces no results.
     The operand number and types for each successor must match the arguments of
-    the block successor. For example:
+    the block successor.
+
+    Example:
 
-      ^bb2:
-        %2 = call @someFn()
-        br ^bb3(%2 : tensor<*xf32>)
-      ^bb3(%3: tensor<*xf32>):
+    ```mlir
+    ^bb2:
+      %2 = call @someFn()
+      br ^bb3(%2 : tensor<*xf32>)
+    ^bb3(%3: tensor<*xf32>):
+    ```
   }];
 
   let arguments = (ins Variadic<AnyType>:$destOperands);
@@ -382,12 +486,16 @@ def BranchOp : Std_Op<"br",
 def CallOp : Std_Op<"call", [CallOpInterface]> {
   let summary = "call operation";
   let description = [{
-    The "call" operation represents a direct call to a function that is within
-    the same symbol scope as the call.  The operands and result types of the
+    The `call` operation represents a direct call to a function that is within
+    the same symbol scope as the call. The operands and result types of the
     call must match the specified function type. The callee is encoded as a
-    function attribute named "callee".
+    symbol reference attribute named "callee".
 
-      %2 = call @my_add(%0, %1) : (f32, f32) -> f32
+    Example:
+
+    ```mlir
+    %2 = call @my_add(%0, %1) : (f32, f32) -> f32
+    ```
   }];
 
   let arguments = (ins FlatSymbolRefAttr:$callee, Variadic<AnyType>:$operands);
@@ -450,12 +558,20 @@ def CallIndirectOp : Std_Op<"call_indirect", [
     ]> {
   let summary = "indirect call operation";
   let description = [{
-    The "call_indirect" operation represents an indirect call to a value of
-    function type.  Functions are first class types in MLIR, and may be passed
-    as arguments and merged together with block arguments.  The operands
-    and result types of the call must match the specified function type.
+    The `call_indirect` operation represents an indirect call to a value of
+    function type. Functions are first class types in MLIR, and may be passed as
+    arguments and merged together with block arguments. The operands and result
+    types of the call must match the specified function type.
+
+    Function values can be created with the
+    [`constant` operation](#constant-operation).
+
+    Example:
 
-      %3 = call_indirect %2(%0, %1) : (f32, f32) -> f32
+    ```mlir
+    %31 = call_indirect %15(%0, %1)
+            : (tensor<16xf32>, tensor<16xf32>) -> tensor<16xf32>
+    ```
   }];
 
   let arguments = (ins FunctionType:$callee, Variadic<AnyType>:$operands);
@@ -497,10 +613,29 @@ def CallIndirectOp : Std_Op<"call_indirect", [
 def CeilFOp : FloatUnaryOp<"ceilf"> {
   let summary = "ceiling of the specified value";
   let description = [{
+    Syntax:
+
+    ```
+    operation ::= ssa-id `=` `std.ceilf` ssa-use `:` type
+    ```
+
     The `ceilf` operation computes the ceiling of a given value. It takes one
     operand and returns one result of the same type. This type may be a float
     scalar type, a vector whose element type is float, or a tensor of floats.
     It has no standard attributes.
+
+    Example:
+
+    ```mlir
+    // Scalar ceiling value.
+    %a = ceilf %b : f64
+
+    // SIMD vector element-wise ceiling value.
+    %f = ceilf %g : vector<4xf32>
+
+    // Tensor element-wise ceiling value.
+    %x = ceilf %y : tensor<4x?xf8>
+    ```
   }];
 }
 
@@ -543,7 +678,7 @@ def CmpFOp : Std_Op<"cmpf",
        "lhs", "result", "getI1SameShape($_self)">]> {
   let summary = "floating-point comparison operation";
   let description = [{
-    The "cmpf" operation compares its two operands according to the float
+    The `cmpf` operation compares its two operands according to the float
     comparison rules and the predicate specified by the respective attribute.
     The predicate defines the type of comparison: (un)orderedness, (in)equality
     and signed less/greater than (or equal to) as well as predicates that are
@@ -559,9 +694,13 @@ def CmpFOp : Std_Op<"cmpf",
     attribute is merely a syntactic sugar and is converted to an integer
     attribute by the parser.
 
-      %r1 = cmpf "oeq" %0, %1 : f32
-      %r2 = cmpf "ult" %0, %1 : tensor<42x42xf64>
-      %r3 = "std.cmpf"(%0, %1) {predicate: 0} : (f8, f8) -> i1
+    Example:
+
+    ```mlir
+    %r1 = cmpf "oeq" %0, %1 : f32
+    %r2 = cmpf "ult" %0, %1 : tensor<42x42xf64>
+    %r3 = "std.cmpf"(%0, %1) {predicate: 0} : (f8, f8) -> i1
+    ```
   }];
 
   let arguments = (ins
@@ -623,24 +762,67 @@ def CmpIOp : Std_Op<"cmpi",
        "lhs", "result", "getI1SameShape($_self)">]> {
   let summary = "integer comparison operation";
   let description = [{
-    The "cmpi" operation compares its two operands according to the integer
-    comparison rules and the predicate specified by the respective attribute.
-    The predicate defines the type of comparison: (in)equality, (un)signed
-    less/greater than (or equal to).  The operands must have the same type, and
-    this type must be an integer type, a vector or a tensor thereof.  The result
-    is an i1, or a vector/tensor thereof having the same shape as the inputs.
-    Since integers are signless, the predicate also explicitly indicates
-    whether to interpret the operands as signed or unsigned integers for
-    less/greater than comparisons.  For the sake of readability by humans,
-    custom assembly form for the operation uses a string-typed attribute for
-    the predicate.  The value of this attribute corresponds to lower-cased name
-    of the predicate constant, e.g., "slt" means "signed less than".  The string
-    representation of the attribute is merely a syntactic sugar and is converted
-    to an integer attribute by the parser.
-
-      %r1 = cmpi "eq" %0, %1 : i32
-      %r2 = cmpi "slt" %0, %1 : tensor<42x42xi64>
-      %r3 = "std.cmpi"(%0, %1){predicate: 0} : (i8, i8) -> i1
+    The `cmpi` operation is a generic comparison for integer-like types. Its two
+    arguments can be integers, vectors or tensors thereof as long as their types
+    match. The operation produces an i1 for the former case, a vector or a
+    tensor of i1 with the same shape as inputs in the other cases.
+
+    Its first argument is an attribute that defines which type of comparison is
+    performed. The following comparisons are supported:
+
+    -   equal (mnemonic: `"eq"`; integer value: `0`)
+    -   not equal (mnemonic: `"ne"`; integer value: `1`)
+    -   signed less than (mnemonic: `"slt"`; integer value: `2`)
+    -   signed less than or equal (mnemonic: `"sle"`; integer value: `3`)
+    -   signed greater than (mnemonic: `"sgt"`; integer value: `4`)
+    -   signed greater than or equal (mnemonic: `"sge"`; integer value: `5`)
+    -   unsigned less than (mnemonic: `"ult"`; integer value: `6`)
+    -   unsigned less than or equal (mnemonic: `"ule"`; integer value: `7`)
+    -   unsigned greater than (mnemonic: `"ugt"`; integer value: `8`)
+    -   unsigned greater than or equal (mnemonic: `"uge"`; integer value: `9`)
+
+    The result is `1` if the comparison is true and `0` otherwise. For vector or
+    tensor operands, the comparison is performed elementwise and the element of
+    the result indicates whether the comparison is true for the operand elements
+    with the same indices as those of the result.
+
+    Note: while the custom assembly form uses strings, the actual underlying
+    attribute has integer type (or rather enum class in C++ code) as seen from
+    the generic assembly form. String literals are used to improve readability
+    of the IR by humans.
+
+    This operation only applies to integer-like operands, but not floats. The
+    main reason being that comparison operations have diverging sets of
+    attributes: integers require sign specification while floats require various
+    floating point-related particularities, e.g., `-ffast-math` behavior,
+    IEEE754 compliance, etc
+    ([rationale](../Rationale.md#splitting-floating-point-vs-integer-operations)).
+    The type of comparison is specified as attribute to avoid introducing ten
+    similar operations, taking into account that they are often implemented
+    using the same operation downstream
+    ([rationale](../Rationale.md#specifying-comparison-kind-as-attribute)). The
+    separation between signed and unsigned order comparisons is necessary
+    because of integers being signless. The comparison operation must know how
+    to interpret values with the foremost bit being set: negatives in two's
+    complement or large positives
+    ([rationale](../Rationale.md#specifying-sign-in-integer-comparison-operations)).
+
+    Example:
+
+    ```mlir
+    // Custom form of scalar "signed less than" comparison.
+    %x = cmpi "slt", %lhs, %rhs : i32
+
+    // Generic form of the same operation.
+    %x = "std.cmpi"(%lhs, %rhs) {predicate = 2 : i64} : (i32, i32) -> i1
+
+    // Custom form of vector equality comparison.
+    %x = cmpi "eq", %lhs, %rhs : vector<4xi64>
+
+    // Generic form of the same operation.
+    %x = "std.cmpi"(%lhs, %rhs) {predicate = 0 : i64}
+        : (vector<4xi64>, vector<4xi64>) -> vector<4xi1>
+    ```
   }];
 
   let arguments = (ins
@@ -682,18 +864,30 @@ def CondBranchOp : Std_Op<"cond_br",
      NoSideEffect, Terminator]> {
   let summary = "conditional branch operation";
   let description = [{
-    The "cond_br" operation represents a conditional branch operation in a
-    function. The operation takes variable number of operands and produces
-    no results. The operand number and types for each successor must match the
-    arguments of the block successor. For example:
+    The `cond_br` terminator operation represents a conditional branch on a
+    boolean (1-bit integer) value. If the bit is set, then the first destination
+    is jumped to; if it is false, the second destination is chosen. The count
+    and types of operands must align with the arguments in the corresponding
+    target blocks.
+
+    The MLIR conditional branch operation is not allowed to target the entry
+    block for a region. The two destinations of the conditional branch operation
+    are allowed to be the same.
+
+    The following example illustrates a function with a conditional branch
+    operation that targets the same block.
+
+    Example:
+
+    ```mlir
+    func @select(%a: i32, %b: i32, %flag: i1) -> i32 {
+      // Both targets are the same, operands differ
+      cond_br %flag, ^bb1(%a : i32), ^bb1(%b : i32)
 
-      ^bb0:
-         %0 = extract_element %arg0[] : tensor<i1>
-         cond_br %0, ^bb1, ^bb2
-      ^bb1:
-         ...
-      ^bb2:
-         ...
+    ^bb1(%x : i32) :
+      return %x : i32
+    }
+    ```
   }];
 
   let arguments = (ins I1:$condition,
@@ -799,6 +993,38 @@ def CondBranchOp : Std_Op<"cond_br",
 def ConstantOp : Std_Op<"constant",
     [ConstantLike, NoSideEffect, DeclareOpInterfaceMethods<OpAsmOpInterface>]> {
   let summary = "constant";
+  let description = [{
+    Syntax:
+
+    ```
+    operation ::= ssa-id `=` `std.constant` attribute-value `:` type
+    ```
+
+    The `constant` operation produces an SSA value equal to some constant
+    specified by an attribute. This is the way that MLIR uses to form simple
+    integer and floating point constants, as well as more exotic things like
+    references to functions and tensor/vector constants.
+
+    Example:
+
+    ```mlir
+    // Integer constant
+    %1 = constant 42 : i32
+
+    // Reference to function @myfn.
+    %3 = constant @myfn : (tensor<16xf32>, f32) -> tensor<16xf32>
+
+    // Equivalent generic forms
+    %1 = "std.constant"() {value = 42 : i32} : () -> i32
+    %3 = "std.constant"() {value = @myfn}
+       : () -> ((tensor<16xf32>, f32) -> tensor<16xf32>)
+    ```
+
+    MLIR does not allow direct references to functions in SSA operands because
+    the compiler is multithreaded, and disallowing SSA values to directly
+    reference a function simplifies this
+    ([rationale](../Rationale.md#multithreading-the-compiler)).
+  }];
 
   let arguments = (ins AnyAttr:$value);
   let results = (outs AnyType);
@@ -825,11 +1051,30 @@ def ConstantOp : Std_Op<"constant",
 def CopySignOp : FloatArithmeticOp<"copysign"> {
   let summary = "A copysign operation";
   let description = [{
+    Syntax:
+
+    ```
+    operation ::= ssa-id `=` `std.copysign` ssa-use `:` type
+    ```
+
     The `copysign` returns a value with the magnitude of the first operand and
     the sign of the second operand. It takes two operands and returns one
     result of the same type. This type may be a float scalar type, a vector
     whose element type is float, or a tensor of floats. It has no standard
     attributes.
+
+    Example:
+
+    ```mlir
+    // Scalar copysign value.
+    %a = copysign %b %c : f64
+
+    // SIMD vector element-wise copysign value.
+    %f = copysign %g %h : vector<4xf32>
+
+    // Tensor element-wise copysign value.
+    %x = copysign %y %z : tensor<4x?xf8>
+    ```
   }];
 }
 
@@ -840,10 +1085,29 @@ def CopySignOp : FloatArithmeticOp<"copysign"> {
 def CosOp : FloatUnaryOp<"cos"> {
   let summary = "cosine of the specified value";
   let description = [{
+    Syntax:
+
+    ```
+    operation ::= ssa-id `=` `std.cos` ssa-use `:` type
+    ```
+
     The `cos` operation computes the cosine of a given value. It takes one
     operand and returns one result of the same type. This type may be a float
     scalar type, a vector whose element type is float, or a tensor of floats.
     It has no standard attributes.
+
+    Example:
+
+    ```mlir
+    // Scalar cosine value.
+    %a = cos %b : f64
+
+    // SIMD vector element-wise cosine value.
+    %f = cos %g : vector<4xf32>
+
+    // Tensor element-wise cosine value.
+    %x = cos %y : tensor<4x?xf8>
+    ```
   }];
 }
 
@@ -854,14 +1118,17 @@ def CosOp : FloatUnaryOp<"cos"> {
 def DeallocOp : Std_Op<"dealloc"> {
   let summary = "memory deallocation operation";
   let description = [{
-    The "dealloc" operation frees the region of memory referenced by a memref
-    which was originally created by the "alloc" operation.
-    The "dealloc" operation should not be called on memrefs which alias an
-    alloc'd memref (i.e. memrefs returned by the "view" and "reshape"
-    operations).
+    The `dealloc` operation frees the region of memory referenced by a memref
+    which was originally created by the `alloc` operation.
+    The `dealloc` operation should not be called on memrefs which alias an
+    alloc'd memref (e.g. memrefs returned by `view` operations).
 
-      %0 = alloc() : memref<8x64xf32, (d0, d1) -> (d0, d1), 1>
-      dealloc %0 : memref<8x64xf32, (d0, d1) -> (d0, d1), 1>
+    Example:
+
+    ```mlir
+    %0 = alloc() : memref<8x64xf32, (d0, d1) -> (d0, d1), 1>
+    dealloc %0 : memref<8x64xf32, (d0, d1) -> (d0, d1), 1>
+    ```
   }];
 
   let arguments = (ins AnyMemRef:$memref);
@@ -878,11 +1145,32 @@ def DeallocOp : Std_Op<"dealloc"> {
 def DimOp : Std_Op<"dim", [NoSideEffect]> {
   let summary = "dimension index operation";
   let description = [{
-    The "dim" operation takes a memref or tensor operand and returns an "index".
-    It requires a single integer attribute named "index". It returns the size
-    of the specified dimension. For example:
+    Syntax:
+
+    ```
+    operation ::= ssa-id `=` `std.dim` ssa-id `,` integer-literal `:` type
+    ```
+
+    The `dim` operation takes a memref or tensor operand and a dimension index,
+    and returns an [`index`](../LangRef.md#index-type) that is the size of that
+    dimension.
+
+    The `dim` operation is represented with a single integer attribute named
+    `index`, and the type specifies the type of the memref or tensor operand.
 
-      %1 = dim %0, 2 : tensor<?x?x?xf32>
+    Example:
+
+    ```mlir
+    // Always returns 4, can be constant folded:
+    %x = dim %A, 0 : tensor<4 x ? x f32>
+
+    // Returns the dynamic dimension of %A.
+    %y = dim %A, 1 : tensor<4 x ? x f32>
+
+    // Equivalent generic form:
+    %x = "std.dim"(%A) {index = 0 : i64} : (tensor<4 x ? x f32>) -> index
+    %y = "std.dim"(%A) {index = 1 : i64} : (tensor<4 x ? x f32>) -> index
+    ```
   }];
 
   let arguments = (ins AnyTypeOf<[AnyMemRef, AnyTensor],
@@ -921,6 +1209,38 @@ def DivFOp : FloatArithmeticOp<"divf"> {
 
 def ExpOp : FloatUnaryOp<"exp"> {
   let summary = "base-e exponential of the specified value";
+  let description = [{
+    Syntax:
+
+    ```
+    operation ::= ssa-id `=` `std.exp` ssa-use `:` type
+    ```
+
+    The `exp` operation takes one operand and returns one result of the same
+    type. This type may be a float scalar type, a vector whose element type is
+    float, or a tensor of floats. It has no standard attributes.
+
+    Example:
+
+    ```mlir
+    // Scalar natural exponential.
+    %a = exp %b : f64
+
+    // SIMD vector element-wise natural exponential.
+    %f = exp %g : vector<4xf32>
+
+    // Tensor element-wise natural exponential.
+    %x = exp %y : tensor<4x?xf8>
+    ```
+  }];
+}
+
+//===----------------------------------------------------------------------===//
+// ExpOp
+//===----------------------------------------------------------------------===//
+
+def Exp2Op : FloatUnaryOp<"exp2"> {
+  let summary = "base-2 exponential of the specified value";
 }
 
 //===----------------------------------------------------------------------===//
@@ -934,14 +1254,20 @@ def ExtractElementOp : Std_Op<"extract_element",
                     "$_self.cast<ShapedType>().getElementType()">]> {
   let summary = "element extract operation";
   let description = [{
-    The "extract_element" op reads a tensor or vector and returns one element
-    from it specified by an index list. The output of extract is a new value
-    with the same type as the elements of the tensor or vector. The arity of
-    indices matches the rank of the accessed value (i.e., if a tensor is of rank
-    3, then 3 indices are required for the extract).  The indices should all be
-    of index type. For example:
+    The `extract_element` op reads a tensor or vector and returns one element
+    from it specified by an index list. The output of the 'extract_element' is a
+    new value with the same type as the elements of the tensor or vector. The
+    arity of indices matches the rank of the accessed value (i.e., if a tensor
+    is of rank 3, then 3 indices are required for the extract. The indices
+    should all be of `index` type.
 
-      %3 = extract_element %0[%1, %2] : vector<4x4xi32>
+    Example:
+
+    ```mlir
+    %3 = extract_element %v[%1, %2] : vector<4x4xi32>
+    %4 = extract_element %t[%1, %2] : tensor<4x4xi32>
+    %5 = extract_element %ut[%1, %2] : tensor<*xi32>
+    ```
   }];
 
   let arguments = (ins AnyTypeOf<[AnyVector, AnyTensor]>:$aggregate,
@@ -1021,9 +1347,9 @@ def FPTruncOp : CastOp<"fptrunc">, Arguments<(ins AnyType:$in)> {
 def IndexCastOp : CastOp<"index_cast">, Arguments<(ins AnyType:$in)> {
   let summary = "cast between index and integer types";
   let description = [{
-    Casts between integer scalars and 'index' scalars.  Index is an integer of
-    platform-specific bit width.  If casting to a wider integer, the value is
-    sign-extended.  If casting to a narrower integer, the value is truncated.
+    Casts between integer scalars and 'index' scalars. Index is an integer of
+    platform-specific bit width. If casting to a wider integer, the value is
+    sign-extended. If casting to a narrower integer, the value is truncated.
   }];
 
   let extraClassDeclaration = [{
@@ -1045,13 +1371,40 @@ def LoadOp : Std_Op<"load",
                      "$_self.cast<MemRefType>().getElementType()">]> {
   let summary = "load operation";
   let description = [{
-    The "load" op reads an element from a memref specified by an index list. The
+    The `load` op reads an element from a memref specified by an index list. The
     output of load is a new value with the same type as the elements of the
     memref. The arity of indices is the rank of the memref (i.e., if the memref
     loaded from is of rank 3, then 3 indices are required for the load following
-    the memref identifier). For example:
+    the memref identifier).
+
+    In an `affine.if` or `affine.for` body, the indices of a load are restricted
+    to SSA values bound to surrounding loop induction variables,
+    [symbols](../LangRef.md#dimensions-and-symbols), results of a
+    [`constant` operation](#constant-operation), or the result of an
+    `affine.apply` operation that can in turn take as arguments all of the
+    aforementioned SSA values or the recursively result of such an
+    `affine.apply` operation.
+
+    Example:
 
-      %3 = load %0[%1, %1] : memref<4x4xi32>
+    ```mlir
+    %1 = affine.apply affine_map<(d0, d1) -> (3*d0)> (%i, %j)
+    %2 = affine.apply affine_map<(d0, d1) -> (d1+1)> (%i, %j)
+    %12 = load %A[%1, %2] : memref<8x?xi32, #layout, memspace0>
+
+    // Example of an indirect load (treated as non-affine)
+    %3 = affine.apply affine_map<(d0) -> (2*d0 + 1)>(%12)
+    %13 = load %A[%3, %2] : memref<4x?xi32, #layout, memspace0>
+    ```
+
+    **Context:** The `load` and `store` operations are specifically crafted to
+    fully resolve a reference to an element of a memref, and (in affine
+    `affine.if` and `affine.for` operations) the compiler can follow use-def
+    chains (e.g. through [`affine.apply`](Affine.md#affineapply-operation)
+    operations) to precisely analyze references at compile-time using polyhedral
+    techniques. This is possible because of the
+    [restrictions on dimensions and symbols](Affine.md#restrictions-on-dimensions-and-symbols)
+    in these contexts.
   }];
 
   let arguments = (ins Arg<AnyMemRef, "the reference to load from",
@@ -1106,7 +1459,13 @@ def Log2Op : FloatUnaryOp<"log2"> {
 def MemRefCastOp : CastOp<"memref_cast"> {
   let summary = "memref cast operation";
   let description = [{
-    The "memref_cast" operation converts a memref from one type to an equivalent
+    Syntax:
+
+    ```
+    operation ::= ssa-id `=` `std.memref_cast` ssa-use `:` type `to` type
+    ```
+
+    The `memref_cast` operation converts a memref from one type to an equivalent
     type with a compatible shape. The source and destination types are
     compatible if:
     a. both are ranked memref types with the same element type, affine mappings,
@@ -1118,30 +1477,36 @@ def MemRefCastOp : CastOp<"memref_cast"> {
     disagree with resultant destination size.
 
     Example:
-    Assert that the input dynamic shape matches the destination static shape.
-       %2 = memref_cast %1 : memref<?x?xf32> to memref<4x4xf32>
-    Erase static shape information, replacing it with dynamic information.
-       %3 = memref_cast %1 : memref<4xf32> to memref<?xf32>
-
-    The same holds true for offsets and strides.
 
-    Assert that the input dynamic shape matches the destination static stride.
-       %4 = memref_cast %1 : memref<12x4xf32, offset:?, strides: [?, ?]> to
-                             memref<12x4xf32, offset:5, strides: [4, 1]>
-    Erase static offset and stride information, replacing it with
-    dynamic information.
-       %5 = memref_cast %1 : memref<12x4xf32, offset:5, strides: [4, 1]> to
-                             memref<12x4xf32, offset:?, strides: [?, ?]>
+    ```mlir
+    // Assert that the input dynamic shape matches the destination static shape.
+    %2 = memref_cast %1 : memref<?x?xf32> to memref<4x4xf32>
+    // Erase static shape information, replacing it with dynamic information.
+    %3 = memref_cast %1 : memref<4xf32> to memref<?xf32>
+
+    // The same holds true for offsets and strides.
+
+    // Assert that the input dynamic shape matches the destination static stride.
+    %4 = memref_cast %1 : memref<12x4xf32, offset:?, strides: [?, ?]> to
+                          memref<12x4xf32, offset:5, strides: [4, 1]>
+    // Erase static offset and stride information, replacing it with
+    // dynamic information.
+    %5 = memref_cast %1 : memref<12x4xf32, offset:5, strides: [4, 1]> to
+                          memref<12x4xf32, offset:?, strides: [?, ?]>
+    ```
 
     b. either or both memref types are unranked with the same element type, and
     address space.
 
     Example:
+
+    ```mlir
     Cast to concrete shape.
         %4 = memref_cast %1 : memref<*xf32> to memref<4x?xf32>
 
     Erase rank information.
         %5 = memref_cast %1 : memref<4x?xf32> to memref<*xf32>
+    ```
   }];
 
   let arguments = (ins AnyRankedOrUnrankedMemRef:$source);
@@ -1163,6 +1528,34 @@ def MemRefCastOp : CastOp<"memref_cast"> {
 
 def MulFOp : FloatArithmeticOp<"mulf"> {
   let summary = "floating point multiplication operation";
+  let description = [{
+    Syntax:
+
+    ```
+    operation ::= ssa-id `=` `std.mulf` ssa-use `,` ssa-use `:` type
+    ```
+
+    The `mulf` operation takes two operands and returns one result, each of
+    these is required to be the same type. This type may be a floating point
+    scalar type, a vector whose element type is a floating point type, or a
+    floating point tensor.
+
+    Example:
+
+    ```mlir
+    // Scalar multiplication.
+    %a = mulf %b, %c : f64
+
+    // SIMD pointwise vector multiplication, e.g. for Intel SSE.
+    %f = mulf %g, %h : vector<4xf32>
+
+    // Tensor pointwise multiplication.
+    %x = mulf %y, %z : tensor<4x?xbf16>
+    ```
+
+    TODO: In the distant future, this will accept optional attributes for fast
+    math, contraction, rounding mode, and other controls.
+  }];
   let hasFolder = 1;
 }
 
@@ -1182,10 +1575,29 @@ def MulIOp : IntArithmeticOp<"muli", [Commutative]> {
 def NegFOp : FloatUnaryOp<"negf"> {
   let summary = "floating point negation";
   let description = [{
+    Syntax:
+
+    ```
+    operation ::= ssa-id `=` `negf` ssa-use `:` type
+    ```
+
     The `negf` operation computes the negation of a given value. It takes one
     operand and returns one result of the same type. This type may be a float
     scalar type, a vector whose element type is float, or a tensor of floats.
     It has no standard attributes.
+
+    Example:
+
+    ```mlir
+    // Scalar negation value.
+    %a = negf %b : f64
+
+    // SIMD vector element-wise negation value.
+    %f = negf %g : vector<4xf32>
+
+    // Tensor element-wise negation value.
+    %x = negf %y : tensor<4x?xf8>
+    ```
   }];
 }
 
@@ -1195,6 +1607,31 @@ def NegFOp : FloatUnaryOp<"negf"> {
 
 def OrOp : IntArithmeticOp<"or", [Commutative]> {
   let summary = "integer binary or";
+  let description = [{
+    Syntax:
+
+    ```
+    operation ::= ssa-id `=` `or` ssa-use `,` ssa-use `:` type
+    ```
+
+    The `or` operation takes two operands and returns one result, each of these
+    is required to be the same type. This type may be an integer scalar type, a
+    vector whose element type is integer, or a tensor of integers. It has no
+    standard attributes.
+
+    Example:
+
+    ```mlir
+    // Scalar integer bitwise or.
+    %a = or %b, %c : i64
+
+    // SIMD vector element-wise bitwise integer or.
+    %f = or %g, %h : vector<4xi32>
+
+    // Tensor element-wise bitwise integer or.
+    %x = or %y, %z : tensor<4x?xi8>
+    ```
+  }];
   let hasFolder = 1;
 }
 
@@ -1210,7 +1647,9 @@ def PrefetchOp : Std_Op<"prefetch"> {
     read/write specifier, a locality hint, and a cache type specifier as shown
     below:
 
-      prefetch %0[%i, %j], read, locality<3>, data : memref<400x400xi32>
+    ```mlir
+    prefetch %0[%i, %j], read, locality<3>, data : memref<400x400xi32>
+    ```
 
     The read/write specifier is either 'read' or 'write', the locality hint
     ranges from locality<0> (no locality) to locality<3> (extremely local keep
@@ -1258,9 +1697,13 @@ def PrefetchOp : Std_Op<"prefetch"> {
 def RankOp : Std_Op<"rank", [NoSideEffect]> {
   let summary = "rank operation";
   let description = [{
-    The "rank" operation takes a tensor operand and returns its rank.
+    The `rank` operation takes a tensor operand and returns its rank.
+
+    Example:
 
-      %1 = rank %0 : index
+    ```mlir
+    %1 = rank %0 : index
+    ```
   }];
 
   let arguments = (ins AnyTensor);
@@ -1293,14 +1736,19 @@ def ReturnOp : Std_Op<"return", [NoSideEffect, HasParent<"FuncOp">,
                                  Terminator]> {
   let summary = "return operation";
   let description = [{
-    The "return" operation represents a return operation within a function.
+    The `return` operation represents a return operation within a function.
     The operation takes variable number of operands and produces no results.
     The operand number and types must match the signature of the function
-    that contains the operation. For example:
+    that contains the operation.
 
-      func @foo() : (i32, f8) {
+    Example:
+
+    ```mlir
+    func @foo() : (i32, f8) {
       ...
       return %0, %1 : i32, f8
+    }
+    ```
   }];
 
   let arguments = (ins Variadic<AnyType>:$operands);
@@ -1337,16 +1785,32 @@ def SelectOp : Std_Op<"select", [NoSideEffect, SameOperandsAndResultShape,
                      "getI1SameShape($_self)">]> {
   let summary = "select operation";
   let description = [{
-    The "select" operation chooses one value based on a binary condition
-    supplied as its first operand. If the value of the first operand is 1, the
-    second operand is chosen, otherwise the third operand is chosen. The second
-    and the third operand must have the same type. The operation applies
-    elementwise to vectors and tensors.  The shape of all arguments must be
-    identical. For example, the maximum operation is obtained by combining
-    "select" with "cmpi" as follows.
+    The `select` operation chooses one value based on a binary condition
+    supplied as its first operand. If the value of the first operand is `1`,
+    the second operand is chosen, otherwise the third operand is chosen.
+    The second and the third operand must have the same type.
+
+    The operation applies to vectors and tensors elementwise given the _shape_
+    of all operands is identical. The choice is made for each element
+    individually based on the value at the same position as the element in the
+    condition operand.
+
+    The `select` operation combined with [`cmpi`](#std-cmpi) can be used
+    to implement `min` and `max` with signed or unsigned comparison semantics.
 
-      %2 = cmpi "gt" %0, %1 : i32         // %2 is i1
-      %3 = select %2, %0, %1 : i32
+    Example:
+
+    ```mlir
+    // Custom form of scalar selection.
+    %x = select %cond, %true, %false : i32
+
+    // Generic form of the same operation.
+    %x = "std.select"(%cond, %true, %false) : (i1, i32, i32) -> i32
+
+    // Vector selection is element-wise
+    %vx = "std.select"(%vcond, %vtrue, %vfalse)
+        : (vector<42xi1>, vector<42xf32>, vector<42xf32>) -> vector<42xf32>
+    ```
   }];
 
   let arguments = (ins BoolLike:$condition,
@@ -1385,9 +1849,13 @@ def ShiftLeftOp : IntArithmeticOp<"shift_left"> {
     The shift_left operation shifts an integer value to the left by a variable
     amount. The low order bits are filled with zeros.
 
-      %1 = constant 5 : i8                       // %1 is 0b00000101
-      %2 = constant 3 : i8
-      %3 = shift_left %1, %2 : (i8, i8) -> i8    // %3 is 0b00101000
+    Example:
+
+    ```mlir
+    %1 = constant 5 : i8                       // %1 is 0b00000101
+    %2 = constant 3 : i8
+    %3 = shift_left %1, %2 : (i8, i8) -> i8    // %3 is 0b00101000
+    ```
   }];
 }
 
@@ -1397,6 +1865,32 @@ def ShiftLeftOp : IntArithmeticOp<"shift_left"> {
 
 def SignedDivIOp : IntArithmeticOp<"divi_signed"> {
   let summary = "signed integer division operation";
+  let description = [{
+    Syntax:
+
+    ```
+    operation ::= ssa-id `=` `divi_signed` ssa-use `,` ssa-use `:` type
+    ```
+
+    Signed integer division. Rounds towards zero. Treats the leading bit as
+    sign, i.e. `6 / -2 = -3`.
+
+    Note: the semantics of division by zero or signed division overflow (minimum
+    value divided by -1) is TBD; do NOT assume any specific behavior.
+
+    Example:
+
+    ```mlir
+    // Scalar signed integer division.
+    %a = divis %b, %c : i64
+
+    // SIMD vector element-wise division.
+    %f = divis %g, %h : vector<4xi32>
+
+    // Tensor element-wise integer division.
+    %x = divis %y, %z : tensor<4x?xi8>
+    ```
+  }];
   let hasFolder = 1;
 }
 
@@ -1406,6 +1900,32 @@ def SignedDivIOp : IntArithmeticOp<"divi_signed"> {
 
 def SignedRemIOp : IntArithmeticOp<"remi_signed"> {
   let summary = "signed integer division remainder operation";
+  let description = [{
+    Syntax:
+
+    ```
+    operation ::= ssa-id `=` `std.remi_signed` ssa-use `,` ssa-use `:` type
+    ```
+
+    Signed integer division remainder. Treats the leading bit as sign, i.e. `6 %
+    -2 = 0`.
+
+    Note: the semantics of division by zero is TBD; do NOT assume any specific
+    behavior.
+
+    Example:
+
+    ```mlir
+    // Scalar signed integer division remainder.
+    %a = remis %b, %c : i64
+
+    // SIMD vector element-wise division remainder.
+    %f = remis %g, %h : vector<4xi32>
+
+    // Tensor element-wise integer division remainder.
+    %x = remis %y, %z : tensor<4x?xi8>
+    ```
+  }];
   let hasFolder = 1;
 }
 
@@ -1421,11 +1941,15 @@ def SignedShiftRightOp : IntArithmeticOp<"shift_right_signed"> {
     bits in the output are filled with copies of the most-significant bit
     of the shifted value (which means that the sign of the value is preserved).
 
-      %1 = constant 160 : i8                             // %1 is 0b10100000
-      %2 = constant 3 : i8
-      %3 = shift_right_signed %1, %2 : (i8, i8) -> i8    // %3 is 0b11110100
-      %4 = constant 96 : i8                              // %4 is 0b01100000
-      %5 = shift_right_signed %4, %2 : (i8, i8) -> i8    // %5 is 0b00001100
+    Example:
+
+    ```mlir
+    %1 = constant 160 : i8                             // %1 is 0b10100000
+    %2 = constant 3 : i8
+    %3 = shift_right_signed %1, %2 : (i8, i8) -> i8    // %3 is 0b11110100
+    %4 = constant 96 : i8                              // %4 is 0b01100000
+    %5 = shift_right_signed %4, %2 : (i8, i8) -> i8    // %5 is 0b00001100
+    ```
   }];
 }
 
@@ -1443,12 +1967,16 @@ def SignExtendIOp : Std_Op<"sexti",
     The top-most (N - M) bits of the output are filled with copies
     of the most-significant bit of the input.
 
-      %1 = constant 5 : i3            // %1 is 0b101
-      %2 = sexti %1 : i3 to i6        // %2 is 0b111101
-      %3 = constant 2 : i3            // %3 is 0b010
-      %4 = sexti %3 : i3 to i6        // %4 is 0b000010
+    Example:
+
+    ```mlir
+    %1 = constant 5 : i3            // %1 is 0b101
+    %2 = sexti %1 : i3 to i6        // %2 is 0b111101
+    %3 = constant 2 : i3            // %3 is 0b010
+    %4 = sexti %3 : i3 to i6        // %4 is 0b000010
 
-      %5 = sexti %0 : vector<2 x i32> to vector<2 x i64>
+    %5 = sexti %0 : vector<2 x i32> to vector<2 x i64>
+    ```
   }];
 
   let arguments = (ins SignlessIntegerLike:$value);
@@ -1500,24 +2028,28 @@ def SplatOp : Std_Op<"splat", [NoSideEffect,
                     "$_self.cast<ShapedType>().getElementType()">]> {
   let summary = "splat or broadcast operation";
   let description = [{
-    The "splat" op reads a value of integer or float type and broadcasts it into
-    a vector or a tensor. The output of splat is thus a new value of either
-    vector or tensor type with elemental type being its operand's type.
-    When the result is a tensor, it has to be statically shaped.
+    Broadcast the operand to all elements of the result vector or tensor. The
+    operand has to be of either integer or float type. When the result is a
+    tensor, it has to be statically shaped.
+
+    Example:
 
-      %1 = splat %0 : vector<8xi32>
-      %2 = splat %0 : tensor<4x8xi32>
+    ```mlir
+    %s = load %A[%i] : memref<128xf32>
+    %v = splat %s : vector<4xf32>
+    %t = splat %s : tensor<8x16xi32>
+    ```
 
-    TODO: Extend this operation to broadcast to dynamically shaped tensors in
-    the same way dynamically shaped memrefs are handled.
+    TODO: This operation is easy to extend to broadcast to dynamically shaped
+    tensors in the same way dynamically shaped memrefs are handled.
 
+    ```mlir
     // Broadcasts %s to a 2-d dynamically shaped tensor, with %m, %n binding
     // to the sizes of the two dynamic dimensions.
-
-      %m = "foo"() : () -> (index)
-      %n = "bar"() : () -> (index)
-      %t = splat %s [%m, %n] : tensor<?x?xi32>
-
+    %m = "foo"() : () -> (index)
+    %n = "bar"() : () -> (index)
+    %t = splat %s [%m, %n] : tensor<?x?xi32>
+    ```
   }];
 
   let arguments = (ins AnyTypeOf<[AnySignlessInteger, AnyFloat],
@@ -1545,6 +2077,17 @@ def SqrtOp : FloatUnaryOp<"sqrt"> {
     returns one result of the same type. This type may be a float scalar type, a
     vector whose element type is float, or a tensor of floats. It has no standard
     attributes.
+
+    Example:
+
+    ```mlir
+    // Scalar square root value.
+    %a = sqrt %b : f64
+    // SIMD vector element-wise square root value.
+    %f = sqrt %g : vector<4xf32>
+    // Tensor element-wise square root value.
+    %x = sqrt %y : tensor<4x?xf32>
+    ```
   }];
 }
 
@@ -1558,14 +2101,32 @@ def StoreOp : Std_Op<"store",
                      "$_self.cast<MemRefType>().getElementType()">]> {
   let summary = "store operation";
   let description = [{
-    The "store" op writes an element to a memref specified by an index list.
-    The arity of indices is the rank of the memref (i.e. if the memref being
-    stored to is of rank 3, then 3 indices are required for the store following
-    the memref identifier). The store operation does not produce a result.
+    Store a value to a memref location given by indices. The value stored should
+    have the same type as the elemental type of the memref. The number of
+    arguments provided within brackets need to match the rank of the memref.
+
+    In an affine context, the indices of a store are restricted to SSA values
+    bound to surrounding loop induction variables,
+    [symbols](Affine.md#restrictions-on-dimensions-and-symbols), results of a
+    [`constant` operation](#constant-operation), or the result of an
+    [`affine.apply`](Affine.md#affineapply-operation) operation that can in turn
+    take as arguments all of the aforementioned SSA values or the recursively
+    result of such an `affine.apply` operation.
+
+    Example:
 
-    In the following example, the ssa value '%v' is stored in memref '%A' at
-    indices [%i, %j]:
-      store %v, %A[%i, %j] : memref<4x128xf32, (d0, d1) -> (d0, d1), 0>
+    ```mlir
+    store %100, %A[%1, 1023] : memref<4x?xf32, #layout, memspace0>
+    ```
+
+    **Context:** The `load` and `store` operations are specifically crafted to
+    fully resolve a reference to an element of a memref, and (in polyhedral
+    `affine.if` and `affine.for` operations) the compiler can follow use-def
+    chains (e.g. through [`affine.apply`](Affine.md#affineapply-operation)
+    operations) to precisely analyze references at compile-time using polyhedral
+    techniques. This is possible because of the
+    [restrictions on dimensions and symbols](Affine.md#restrictions-on-dimensions-and-symbols)
+    in these contexts.
   }];
 
   let arguments = (ins AnyType:$value,
@@ -1645,86 +2206,93 @@ def SubViewOp : Std_Op<"subview", [AttrSizedOperandSegments, NoSideEffect]> {
 
     Example 1:
 
-      %0 = alloc() : memref<64x4xf32, (d0, d1) -> (d0 * 4 + d1)>
+    ```mlir
+    %0 = alloc() : memref<64x4xf32, (d0, d1) -> (d0 * 4 + d1)>
 
-      // Create a sub-view of "base" memref '%0' with offset arguments '%c0',
-      // dynamic sizes for each dimension, and stride arguments '%c1'.
-      %1 = subview %0[%c0, %c0][%size0, %size1][%c1, %c1]
-        : memref<64x4xf32, (d0, d1) -> (d0 * 4 + d1) > to
-          memref<?x?xf32, (d0, d1)[s0, s1] -> (d0 * s1 + d1 + s0)>
+    // Create a sub-view of "base" memref '%0' with offset arguments '%c0',
+    // dynamic sizes for each dimension, and stride arguments '%c1'.
+    %1 = subview %0[%c0, %c0][%size0, %size1][%c1, %c1]
+      : memref<64x4xf32, (d0, d1) -> (d0 * 4 + d1) > to
+        memref<?x?xf32, (d0, d1)[s0, s1] -> (d0 * s1 + d1 + s0)>
+    ```
 
     Example 2:
 
-      %0 = alloc() : memref<8x16x4xf32, (d0, d1, d1) -> (d0 * 64 + d1 * 4 + d2)>
-
-      // Create a sub-view of "base" memref '%0' with dynamic offsets, sizes,
-      // and strides.
-      // Note that dynamic offsets are represented by the linearized dynamic
-      // offset symbol 's0' in the subview memref layout map, and that the
-      // dynamic strides operands, after being applied to the base memref
-      // strides in each dimension, are represented in the view memref layout
-      // map as symbols 's1', 's2' and 's3'.
-      %1 = subview %0[%i, %j, %k][%size0, %size1, %size2][%x, %y, %z]
-        : memref<8x16x4xf32, (d0, d1, d2) -> (d0 * 64 + d1 * 4 + d2)> to
-          memref<?x?x?xf32,
-            (d0, d1, d2)[s0, s1, s2, s3] -> (d0 * s1 + d1 * s2 + d2 * s3 + s0)>
+    ```mlir
+    %0 = alloc() : memref<8x16x4xf32, (d0, d1, d1) -> (d0 * 64 + d1 * 4 + d2)>
+
+    // Create a sub-view of "base" memref '%0' with dynamic offsets, sizes,
+    // and strides.
+    // Note that dynamic offsets are represented by the linearized dynamic
+    // offset symbol 's0' in the subview memref layout map, and that the
+    // dynamic strides operands, after being applied to the base memref
+    // strides in each dimension, are represented in the view memref layout
+    // map as symbols 's1', 's2' and 's3'.
+    %1 = subview %0[%i, %j, %k][%size0, %size1, %size2][%x, %y, %z]
+      : memref<8x16x4xf32, (d0, d1, d2) -> (d0 * 64 + d1 * 4 + d2)> to
+        memref<?x?x?xf32,
+          (d0, d1, d2)[s0, s1, s2, s3] -> (d0 * s1 + d1 * s2 + d2 * s3 + s0)>
+    ```
 
     Example 3:
 
-      %0 = alloc() : memref<8x16x4xf32, (d0, d1, d1) -> (d0 * 64 + d1 * 4 + d2)>
+    ```mlir
+    %0 = alloc() : memref<8x16x4xf32, (d0, d1, d1) -> (d0 * 64 + d1 * 4 + d2)>
 
-      // Subview with constant offsets, sizes and strides.
-      %1 = subview %0[][][]
-        : memref<8x16x4xf32, (d0, d1, d2) -> (d0 * 64 + d1 * 4 + d2)> to
-          memref<4x4x4xf32, (d0, d1, d2) -> (d0 * 16 + d1 * 4 + d2 + 8)>
+    // Subview with constant offsets, sizes and strides.
+    %1 = subview %0[][][]
+      : memref<8x16x4xf32, (d0, d1, d2) -> (d0 * 64 + d1 * 4 + d2)> to
+        memref<4x4x4xf32, (d0, d1, d2) -> (d0 * 16 + d1 * 4 + d2 + 8)>
+    ```
 
     Example 4:
 
-      %0 = alloc(%arg0, %arg1) : memref<?x?xf32>
-
-      // Subview with constant size, but dynamic offsets and
-      // strides. The resulting memref has a static shape, but if the
-      // base memref has an affine map to describe the layout, the result
-      // memref also uses an affine map to describe the layout. The
-      // strides of the result memref is computed as follows:
-      //
-      // Let #map1 represents the layout of the base memref, and #map2
-      // represents the layout of the result memref. A #mapsubview can be
-      // constructed to map an index from the result memref to the base
-      // memref (note that the description below uses more convenient
-      // naming for symbols, while in affine maps, symbols are
-      // represented as unsigned numbers that identify that symbol in the
-      // given affine map.
-      //
-      // #mapsubview = (d0, d1)[o0, o1, t0, t1] -> (d0 * t0 + o0, d1 * t1 + o1)
-      //
-      // where, o0, o1, ... are offsets, and t0, t1, ... are strides. Then,
-      //
-      // #map2 = #map1.compose(#mapsubview)
-      //
-      // If the layout map is represented as
-      //
-      // #map1 = (d0, d1)[s0, s1, s2] -> (d0 * s1 + d1 * s2 + s0)
-      //
-      // then,
-      //
-      // #map2 = (d0, d1)[s0, s1, s2, o0, o1, t0, t1] ->
-      //              (d0 * s1 * t0 + d1 * s2 * t1 + o0 * s1 + o1 * s2 + s0)
-      //
-      // Representing this canonically
-      //
-      // #map2 = (d0, d1)[r0, r1, r2] -> (d0 * r1 + d1 * r2 + r0)
-      //
-      // where, r0 = o0 * s1 + o1 * s2 + s0, r1 = s1 * t0, r2 = s2 * t1.
-      %1 = subview %0[%i, %j][][%x, %y] :
-        : memref<?x?xf32, (d0, d1)[s0, s1, s2] -> (d0 * s1 + d1 * s2 + s0)> to
-          memref<4x4xf32, (d0, d1)[r0, r1, r2] -> (d0 * r1 + d1 * r2 + r0)>
-
-      // Note that the subview op does not guarantee that the result
-      // memref is "inbounds" w.r.t to base memref. It is upto the client
-      // to ensure that the subview is accessed in a manner that is
-      // in-bounds.
-
+    ```mlir
+    %0 = alloc(%arg0, %arg1) : memref<?x?xf32>
+
+    // Subview with constant size, but dynamic offsets and
+    // strides. The resulting memref has a static shape, but if the
+    // base memref has an affine map to describe the layout, the result
+    // memref also uses an affine map to describe the layout. The
+    // strides of the result memref is computed as follows:
+    //
+    // Let #map1 represents the layout of the base memref, and #map2
+    // represents the layout of the result memref. A #mapsubview can be
+    // constructed to map an index from the result memref to the base
+    // memref (note that the description below uses more convenient
+    // naming for symbols, while in affine maps, symbols are
+    // represented as unsigned numbers that identify that symbol in the
+    // given affine map.
+    //
+    // #mapsubview = (d0, d1)[o0, o1, t0, t1] -> (d0 * t0 + o0, d1 * t1 + o1)
+    //
+    // where, o0, o1, ... are offsets, and t0, t1, ... are strides. Then,
+    //
+    // #map2 = #map1.compose(#mapsubview)
+    //
+    // If the layout map is represented as
+    //
+    // #map1 = (d0, d1)[s0, s1, s2] -> (d0 * s1 + d1 * s2 + s0)
+    //
+    // then,
+    //
+    // #map2 = (d0, d1)[s0, s1, s2, o0, o1, t0, t1] ->
+    //              (d0 * s1 * t0 + d1 * s2 * t1 + o0 * s1 + o1 * s2 + s0)
+    //
+    // Representing this canonically
+    //
+    // #map2 = (d0, d1)[r0, r1, r2] -> (d0 * r1 + d1 * r2 + r0)
+    //
+    // where, r0 = o0 * s1 + o1 * s2 + s0, r1 = s1 * t0, r2 = s2 * t1.
+    %1 = subview %0[%i, %j][][%x, %y] :
+      : memref<?x?xf32, (d0, d1)[s0, s1, s2] -> (d0 * s1 + d1 * s2 + s0)> to
+        memref<4x4xf32, (d0, d1)[r0, r1, r2] -> (d0 * r1 + d1 * r2 + r0)>
+
+    // Note that the subview op does not guarantee that the result
+    // memref is "inbounds" w.r.t to base memref. It is upto the client
+    // to ensure that the subview is accessed in a manner that is
+    // in-bounds.
+    ```
     }
   }];
 
@@ -1797,10 +2365,29 @@ def SubViewOp : Std_Op<"subview", [AttrSizedOperandSegments, NoSideEffect]> {
 def TanhOp : FloatUnaryOp<"tanh"> {
   let summary = "hyperbolic tangent of the specified value";
   let description = [{
+    Syntax:
+
+    ```
+    operation ::= ssa-id `=` `std.tanh` ssa-use `:` type
+    ```
+
     The `tanh` operation computes the hyperbolic tangent. It takes one operand
     and returns one result of the same type. This type may be a float scalar
     type, a vector whose element type is float, or a tensor of floats. It has
     no standard attributes.
+
+    Example:
+
+    ```mlir
+    // Scalar hyperbolic tangent value.
+    %a = tanh %b : f64
+
+    // SIMD vector element-wise hyperbolic tangent value.
+    %f = tanh %g : vector<4xf32>
+
+    // Tensor element-wise hyperbolic tangent value.
+    %x = tanh %y : tensor<4x?xf8>
+    ```
   }];
 }
 
@@ -1811,14 +2398,32 @@ def TanhOp : FloatUnaryOp<"tanh"> {
 def TensorCastOp : CastOp<"tensor_cast"> {
   let summary = "tensor cast operation";
   let description = [{
-    The "tensor_cast" operation converts a tensor from one type to an equivalent
-    type without changing any data elements.  The source and destination types
-    must both be tensor types with the same element type.  If both are ranked
-    then the rank should be the same and static dimensions should match.  The
-    operation is invalid if converting to a mismatching constant dimension.
+    Syntax:
+
+    ```
+    operation ::= ssa-id `=` `std.tensor_cast` ssa-use `:` type `to` type
+    ```
+
+    Convert a tensor from one type to an equivalent type without changing any
+    data elements. The source and destination types must both be tensor types
+    with the same element type. If both are ranked, then the rank should be the
+    same and static dimensions should match. The operation is invalid if
+    converting to a mismatching constant dimension.
+
+    Example:
+
+    ```mlir
+    // Convert from unknown rank to rank 2 with unknown dimension sizes.
+    %2 = "std.tensor_cast"(%1) : (tensor<*xf32>) -> tensor<?x?xf32>
+    %2 = tensor_cast %1 : tensor<*xf32> to tensor<?x?xf32>
 
-    Convert from unknown rank to rank 2 with unknown dimension sizes.
-       %2 = tensor_cast %1 : tensor<*xf32> to tensor<?x?xf32>
+    // Convert to a type with more known dimensions.
+    %3 = "std.tensor_cast"(%2) : (tensor<?x?xf32>) -> tensor<4x?xf32>
+
+    // Discard static dimension and rank information.
+    %4 = "std.tensor_cast"(%3) : (tensor<4x?xf32>) -> tensor<?x?xf32>
+    %5 = "std.tensor_cast"(%4) : (tensor<?x?xf32>) -> tensor<*xf32>
+    ```
   }];
 
   let arguments = (ins AnyTensor);
@@ -1845,12 +2450,16 @@ def TensorLoadOp : Std_Op<"tensor_load",
                     "getTensorTypeFromMemRefType($_self)">]> {
   let summary = "tensor load operation";
   let description = [{
-    The "tensor_load" operation creates a tensor from a memref, making an
-    independent copy of the element data. The result value is a tensor whose
-    shape and element type match the memref operand.
+    Create a tensor from a memref, making an independent copy of the element
+    data. The result value is a tensor whose shape and element type match the
+    memref operand.
+
+    Example:
 
-    Produce a value of tensor<4x?xf32> type.
-       %12 = tensor_load %10 : memref<4x?xf32, #layout, memspace0>
+    ```mlir
+    // Produces a value of tensor<4x?xf32> type.
+    %12 = tensor_load %10 : memref<4x?xf32, #layout, memspace0>
+    ```
   }];
 
   let arguments = (ins Arg<AnyMemRef, "the reference to load from",
@@ -1887,15 +2496,17 @@ def TensorStoreOp : Std_Op<"tensor_store",
                     "getTensorTypeFromMemRefType($_self)">]> {
   let summary = "tensor store operation";
   let description = [{
-    The "tensor_store" operation stores the contents of a tensor into a memref.
-    The first operand is a value of tensor type, the second operand is a value
-    of memref type. The shapes and element types of these must match, and are
-    specified by the memref type.
+    Stores the contents of a tensor into a memref. The first operand is a value
+    of tensor type, the second operand is a value of memref type. The shapes and
+    element types of these must match, and are specified by the memref type.
 
     Example:
-       %9 = dim %8, 1 : tensor<4x?xf32>
-       %10 = alloc(%9) : memref<4x?xf32, #layout, memspace0>
-       tensor_store %8, %10 : memref<4x?xf32, #layout, memspace0>
+
+    ```mlir
+    %9 = dim %8, 1 : tensor<4x?xf32>
+    %10 = alloc(%9) : memref<4x?xf32, #layout, memspace0>
+    tensor_store %8, %10 : memref<4x?xf32, #layout, memspace0>
+    ```
   }];
 
   let arguments = (ins AnyTensor:$tensor,
@@ -1919,11 +2530,15 @@ def TruncateIOp : Std_Op<"trunci", [NoSideEffect, SameOperandsAndResultShape]> {
     bit-width must be smaller than the input bit-width (N < M).
     The top-most (N - M) bits of the input are discarded.
 
+    Example:
+
+    ```mlir
       %1 = constant 21 : i5           // %1 is 0b10101
       %2 = trunci %1 : i5 to i4       // %2 is 0b0101
       %3 = trunci %1 : i5 to i3       // %3 is 0b101
 
       %5 = trunci %0 : vector<2 x i32> to vector<2 x i16>
+    ```
   }];
 
   let arguments = (ins SignlessIntegerLike:$value);
@@ -1949,6 +2564,32 @@ def TruncateIOp : Std_Op<"trunci", [NoSideEffect, SameOperandsAndResultShape]> {
 
 def UnsignedDivIOp : IntArithmeticOp<"divi_unsigned"> {
   let summary = "unsigned integer division operation";
+  let description = [{
+    Syntax:
+    ```
+    operation ::= ssa-id `=` `std.divi_unsigned` ssa-use `,` ssa-use `:` type
+    ```
+
+    Unsigned integer division. Rounds towards zero. Treats the leading bit as
+    the most significant, i.e. for `i16` given two's complement representation,
+    `6 / -2 = 6 / (2^16 - 2) = 0`.
+
+    Note: the semantics of division by zero is TBD; do NOT assume any specific
+    behavior.
+
+    Example:
+
+    ```mlir
+    // Scalar unsigned integer division.
+    %a = diviu %b, %c : i64
+
+    // SIMD vector element-wise division.
+    %f = diviu %g, %h : vector<4xi32>
+
+    // Tensor element-wise integer division.
+    %x = diviu %y, %z : tensor<4x?xi8>
+    ```
+  }];
   let hasFolder = 1;
 }
 
@@ -1958,6 +2599,32 @@ def UnsignedDivIOp : IntArithmeticOp<"divi_unsigned"> {
 
 def UnsignedRemIOp : IntArithmeticOp<"remi_unsigned"> {
   let summary = "unsigned integer division remainder operation";
+  let description = [{
+    Syntax:
+
+    ```
+    operation ::= ssa-id `=` `std.remi_unsigned` ssa-use `,` ssa-use `:` type
+    ```
+
+    Unsigned integer division remainder. Treats the leading bit as the most
+    significant, i.e. for `i16`, `6 % -2 = 6 % (2^16 - 2) = 6`.
+
+    Note: the semantics of division by zero is TBD; do NOT assume any specific
+    behavior.
+
+    Example:
+
+    ```mlir
+    // Scalar unsigned integer division remainder.
+    %a = remiu %b, %c : i64
+
+    // SIMD vector element-wise division remainder.
+    %f = remiu %g, %h : vector<4xi32>
+
+    // Tensor element-wise integer division remainder.
+    %x = remiu %y, %z : tensor<4x?xi8>
+    ```
+  }];
   let hasFolder = 1;
 }
 
@@ -1972,9 +2639,13 @@ def UnsignedShiftRightOp : IntArithmeticOp<"shift_right_unsigned"> {
     a variable amount. The integer is interpreted as unsigned. The high order
     bits are always filled with zeros.
 
-      %1 = constant 160 : i8                               // %1 is 0b10100000
-      %2 = constant 3 : i8
-      %3 = shift_right_unsigned %1, %2 : (i8, i8) -> i8    // %3 is 0b00010100
+    Example:
+
+    ```mlir
+    %1 = constant 160 : i8                               // %1 is 0b10100000
+    %2 = constant 3 : i8
+    %3 = shift_right_unsigned %1, %2 : (i8, i8) -> i8    // %3 is 0b00010100
+    ```
   }];
 }
 
@@ -1994,6 +2665,9 @@ def ViewOp : Std_Op<"view", [NoSideEffect]> {
     *) A dynamic size operand must be specified for each dynamic dimension
        in the resulting view memref type.
 
+    Example:
+
+    ```mlir
     // Allocate a flat 1D/i8 memref.
     %0 = alloc() : memref<2048xi8>
 
@@ -2016,6 +2690,7 @@ def ViewOp : Std_Op<"view", [NoSideEffect]> {
     %3 = view %0[%offset_1024][%size0, %size1]
       : memref<2048xi8> to memref<?x?x4xf32,
         (d0, d1, d2)[s0, s1] -> (d0 * s1 + d1 * 4 + d2 + s0)>
+    ```
   }];
 
   let arguments = (ins MemRefRankOf<[I8], [1]>:$source,
@@ -2050,6 +2725,25 @@ def ViewOp : Std_Op<"view", [NoSideEffect]> {
 
 def XOrOp : IntArithmeticOp<"xor", [Commutative]> {
   let summary = "integer binary xor";
+  let description = [{
+    The `xor` operation takes two operands and returns one result, each of these
+    is required to be the same type. This type may be an integer scalar type, a
+    vector whose element type is integer, or a tensor of integers. It has no
+    standard attributes.
+
+    Example:
+
+    ```mlir
+    // Scalar integer bitwise xor.
+    %a = xor %b, %c : i64
+
+    // SIMD vector element-wise bitwise integer xor.
+    %f = xor %g, %h : vector<4xi32>
+
+    // Tensor element-wise bitwise integer xor.
+    %x = xor %y, %z : tensor<4x?xi8>
+    ```
+  }];
   let hasFolder = 1;
 }
 
@@ -2065,12 +2759,16 @@ def ZeroExtendIOp : Std_Op<"zexti", [NoSideEffect, SameOperandsAndResultShape]>
     bit-width must be larger than the input bit-width (N > M).
     The top-most (N - M) bits of the output are filled with zeros.
 
+    Example:
+
+    ```mlir
       %1 = constant 5 : i3            // %1 is 0b101
       %2 = zexti %1 : i3 to i6        // %2 is 0b000101
       %3 = constant 2 : i3            // %3 is 0b010
       %4 = zexti %3 : i3 to i6        // %4 is 0b000010
 
       %5 = zexti %0 : vector<2 x i32> to vector<2 x i64>
+    ```
   }];
 
   let arguments = (ins SignlessIntegerLike:$value);
diff --git a/mlir/include/mlir/Dialect/Utils/StructuredOpsUtils.h b/mlir/include/mlir/Dialect/Utils/StructuredOpsUtils.h
index d54791a65410a..bb37bb28a18cc 100644
--- a/mlir/include/mlir/Dialect/Utils/StructuredOpsUtils.h
+++ b/mlir/include/mlir/Dialect/Utils/StructuredOpsUtils.h
@@ -72,6 +72,15 @@ constexpr StringRef getFunAttrName() { return "fun"; }
 /// function that implements the structured op.
 constexpr StringRef getLibraryCallAttrName() { return "library_call"; }
 
+/// Attribute name for the StrArrayAttr which encodes the value of strides.
+constexpr StringRef getStridesAttrName() { return "strides"; }
+
+/// Attribute name for the StrArrayAttr which encodes the value of dilations.
+constexpr StringRef getDilationsAttrName() { return "dilations"; }
+
+/// Attribute name for the StrArrayAttr which encodes the value of paddings.
+constexpr StringRef getPaddingAttrName() { return "padding"; }
+
 /// Use to encode that a particular iterator type has parallel semantics.
 constexpr StringRef getParallelIteratorTypeName() { return "parallel"; }
 
diff --git a/mlir/include/mlir/Dialect/Vector/EDSC/Builders.h b/mlir/include/mlir/Dialect/Vector/EDSC/Builders.h
index 024ae93a8f326..396053f63213f 100644
--- a/mlir/include/mlir/Dialect/Vector/EDSC/Builders.h
+++ b/mlir/include/mlir/Dialect/Vector/EDSC/Builders.h
@@ -44,7 +44,7 @@ Value vector_contraction(StructuredIndexed A, StructuredIndexed B,
 /// Prerequisites:
 /// A, B and C capture values of proper vector types. For instance
 /// `A: vector<4x8xf32>`, `B: vector<8x16f32>` and `C: vector<4x16xf32>`.
-Value vector_matmul(Value A, Value B, Value C);
+Value vector_contraction_matmul(Value A, Value B, Value C);
 
 } // namespace ops
 } // namespace edsc
diff --git a/mlir/include/mlir/Dialect/Vector/EDSC/Intrinsics.h b/mlir/include/mlir/Dialect/Vector/EDSC/Intrinsics.h
index c307721607df7..79ab479c6133e 100644
--- a/mlir/include/mlir/Dialect/Vector/EDSC/Intrinsics.h
+++ b/mlir/include/mlir/Dialect/Vector/EDSC/Intrinsics.h
@@ -16,6 +16,7 @@ namespace intrinsics {
 
 using vector_broadcast = ValueBuilder<vector::BroadcastOp>;
 using vector_contract = ValueBuilder<vector::ContractionOp>;
+using vector_matmul = ValueBuilder<vector::MatmulOp>;
 using vector_print = OperationBuilder<vector::PrintOp>;
 
 } // namespace intrinsics
diff --git a/mlir/include/mlir/Dialect/Vector/VectorOps.td b/mlir/include/mlir/Dialect/Vector/VectorOps.td
index 2e895e63ba27e..aac766f38a9c2 100644
--- a/mlir/include/mlir/Dialect/Vector/VectorOps.td
+++ b/mlir/include/mlir/Dialect/Vector/VectorOps.td
@@ -87,58 +87,59 @@ def Vector_ContractionOp :
     and acc arguments. An indexing map attribute specifies a mapping from each
     iterator in the iterator type list, to each dimension of an N-D vector.
 
-    Examples:
+    Example:
+
     ```mlir
-      // Simple dot product (K = 0).
-      #contraction_accesses = [
-       affine_map<(i) -> (i)>,
-       affine_map<(i) -> (i)>,
-       affine_map<(i) -> ()>
-      ]
-      #contraction_trait = {
-        indexing_maps = #contraction_accesses,
-        iterator_types = ["reduction"]
-      }
-      %3 = vector.contract #contraction_trait %0, %1, %2
-        : vector<10xf32>, vector<10xf32> into f32
-
-      // 2D vector contraction with one contracting dimension (matmul, K = 2).
-      #contraction_accesses = [
-        affine_map<(i, j, k) -> (i, k)>,
-        affine_map<(i, j, k) -> (k, j)>,
-        affine_map<(i, j, k) -> (i, j)>
-      ]
-      #contraction_trait = {
-        indexing_maps = #contraction_accesses,
-        iterator_types = ["parallel", "parallel", "reduction"]
-      }
+    // Simple dot product (K = 0).
+    #contraction_accesses = [
+     affine_map<(i) -> (i)>,
+     affine_map<(i) -> (i)>,
+     affine_map<(i) -> ()>
+    ]
+    #contraction_trait = {
+      indexing_maps = #contraction_accesses,
+      iterator_types = ["reduction"]
+    }
+    %3 = vector.contract #contraction_trait %0, %1, %2
+      : vector<10xf32>, vector<10xf32> into f32
+
+    // 2D vector contraction with one contracting dimension (matmul, K = 2).
+    #contraction_accesses = [
+      affine_map<(i, j, k) -> (i, k)>,
+      affine_map<(i, j, k) -> (k, j)>,
+      affine_map<(i, j, k) -> (i, j)>
+    ]
+    #contraction_trait = {
+      indexing_maps = #contraction_accesses,
+      iterator_types = ["parallel", "parallel", "reduction"]
+    }
 
-      %3 = vector.contract #contraction_trait %0, %1, %2
-        : vector<4x3xf32>, vector<3x7xf32> into vector<4x7xf32>
-
-      // 4D to 3D vector contraction with two contracting dimensions and
-      // one batch dimension (K = 3).
-      #contraction_accesses = [
-        affine_map<(b0, f0, f1, c0, c1) -> (c0, b0, c1, f0)>,
-        affine_map<(b0, f0, f1, c0, c1) -> (b0, c1, c0, f1)>,
-        affine_map<(b0, f0, f1, c0, c1) -> (b0, f0, f1)>
-      ]
-      #contraction_trait = {
-        indexing_maps = #contraction_accesses,
-        iterator_types = ["parallel", "parallel", "parallel",
-                          "reduction", "reduction"]
-      }
+    %3 = vector.contract #contraction_trait %0, %1, %2
+      : vector<4x3xf32>, vector<3x7xf32> into vector<4x7xf32>
+
+    // 4D to 3D vector contraction with two contracting dimensions and
+    // one batch dimension (K = 3).
+    #contraction_accesses = [
+      affine_map<(b0, f0, f1, c0, c1) -> (c0, b0, c1, f0)>,
+      affine_map<(b0, f0, f1, c0, c1) -> (b0, c1, c0, f1)>,
+      affine_map<(b0, f0, f1, c0, c1) -> (b0, f0, f1)>
+    ]
+    #contraction_trait = {
+      indexing_maps = #contraction_accesses,
+      iterator_types = ["parallel", "parallel", "parallel",
+                        "reduction", "reduction"]
+    }
 
-      %4 = vector.contract #contraction_trait %0, %1, %2
-          : vector<7x8x16x15xf32>, vector<8x16x7x5xf32> into vector<8x15x5xf32>
+    %4 = vector.contract #contraction_trait %0, %1, %2
+        : vector<7x8x16x15xf32>, vector<8x16x7x5xf32> into vector<8x15x5xf32>
 
-      // 4D vector contraction with two contracting dimensions and optional
-      // vector mask arguments.
-      %lhs_mask = vector.constant_mask [7, 8, 16, 15] : vector<7x8x16x15xi1>
-      %rhs_mask = vector.constant_mask [8, 16, 7, 5] : vector<8x16x7x5xi1>
+    // 4D vector contraction with two contracting dimensions and optional
+    // vector mask arguments.
+    %lhs_mask = vector.constant_mask [7, 8, 16, 15] : vector<7x8x16x15xi1>
+    %rhs_mask = vector.constant_mask [8, 16, 7, 5] : vector<8x16x7x5xi1>
 
-      %5 = vector.contract #contraction_trait %0, %1, %2, %lhs_mask, %rhs_mask
-         : vector<7x8x16x15xf32>, vector<8x16x7x5xf32> into vector<8x15x8x5xf32>
+    %5 = vector.contract #contraction_trait %0, %1, %2, %lhs_mask, %rhs_mask
+       : vector<7x8x16x15xf32>, vector<8x16x7x5xf32> into vector<8x15x8x5xf32>
     ```
   }];
   let builders = [OpBuilder<
@@ -203,13 +204,14 @@ def Vector_ReductionOp :
 
     http://llvm.org/docs/LangRef.html#experimental-vector-reduction-intrinsics
 
-    Examples:
+    Example:
+
     ```mlir
-      %1 = vector.reduction "add", %0 : vector<16xf32> into f32
+    %1 = vector.reduction "add", %0 : vector<16xf32> into f32
 
-      %3 = vector.reduction "xor", %2 : vector<4xi32> into i32
+    %3 = vector.reduction "xor", %2 : vector<4xi32> into i32
 
-      %4 = vector.reduction "mul", %0, %1 : vector<16xf32> into f32
+    %4 = vector.reduction "mul", %0, %1 : vector<16xf32> into f32
     ```
   }];
   let extraClassDeclaration = [{
@@ -247,11 +249,12 @@ def Vector_BroadcastOp :
     dimension of 1. These rules imply that any scalar broadcast (k=0) to any
     shaped vector with the same element type is always legal.
 
-    Examples:
+    Example:
+
     ```mlir
-      %0 = constant 0.0 : f32
-      %1 = vector.broadcast %0 : f32 to vector<16xf32>
-      %2 = vector.broadcast %1 : vector<16xf32> to vector<4x16xf32>
+    %0 = constant 0.0 : f32
+    %1 = vector.broadcast %0 : f32 to vector<16xf32>
+    %2 = vector.broadcast %1 : vector<16xf32> to vector<4x16xf32>
     ```
   }];
   let extraClassDeclaration = [{
@@ -290,7 +293,8 @@ def Vector_ShuffleOp :
       mask values must be within range, viz. given two k-D operands v1 and v2
       above, all mask values are in the range [0,s_1+t_1)
 
-    Examples:
+    Example:
+
     ```mlir
     %0 = vector.shuffle %a, %b[0, 3]
                : vector<2xf32>, vector<2xf32>       ; yields vector<2xf32>
@@ -298,7 +302,6 @@ def Vector_ShuffleOp :
                : vector<2x16xf32>, vector<1x16xf32> ; yields vector<3x16xf32>
     %2 = vector.shuffle %a, %b[3, 2, 1, 0]
                : vector<2xf32>, vector<2xf32>       ; yields vector<4xf32>
-
     ```
   }];
   let builders = [OpBuilder<"Builder *builder, OperationState &result,"
@@ -333,9 +336,10 @@ def Vector_ExtractElementOp :
     https://llvm.org/docs/LangRef.html#extractelement-instruction
 
     Example:
+
     ```mlir
-      %c = constant 15 : i32
-      %1 = vector.extractelement %0[%c : i32]: vector<16xf32>
+    %c = constant 15 : i32
+    %1 = vector.extractelement %0[%c : i32]: vector<16xf32>
     ```
   }];
   let extraClassDeclaration = [{
@@ -360,10 +364,11 @@ def Vector_ExtractOp :
     Takes an n-D vector and a k-D position and extracts the (n-k)-D vector at
     the proper position. Degenerates to an element type in the 0-D case.
 
-    Examples:
+    Example:
+
     ```mlir
-      %1 = vector.extract %0[3]: vector<4x8x16xf32>
-      %2 = vector.extract %0[3, 3, 3]: vector<4x8x16xf32>
+    %1 = vector.extract %0[3]: vector<4x8x16xf32>
+    %2 = vector.extract %0[3, 3, 3]: vector<4x8x16xf32>
     ```
   }];
   let builders = [OpBuilder<
@@ -396,19 +401,20 @@ def Vector_ExtractSlicesOp :
     linear index of the slice w.r.t the unrolling scheme represented by 'sizes'.
     Currently, only unit strides are supported.
 
-    Examples:
+    Example:
+
     ```mlir
-      %0 = vector.transfer_read ...: vector<4x2xf32>
+    %0 = vector.transfer_read ...: vector<4x2xf32>
 
-      %1 = vector.extract_slices %0, [2, 2], [1, 1]
-        : vector<4x2xf32> into tuple<vector<2x2xf32>, vector<2x2xf32>>
+    %1 = vector.extract_slices %0, [2, 2], [1, 1]
+      : vector<4x2xf32> into tuple<vector<2x2xf32>, vector<2x2xf32>>
 
-      // Example with partial slices at dimension boundaries.
-      %2 = vector.transfer_read ...: vector<4x3xf32>
+    // Example with partial slices at dimension boundaries.
+    %2 = vector.transfer_read ...: vector<4x3xf32>
 
-      %3 = vector.extract_slices %2, [2, 2], [1, 1]
-        : vector<4x3xf32> into tuple<vector<2x2xf32>, vector<2x1xf32>,
-                                     vector<2x2xf32>, vector<2x1xf32>>
+    %3 = vector.extract_slices %2, [2, 2], [1, 1]
+      : vector<4x3xf32> into tuple<vector<2x2xf32>, vector<2x1xf32>,
+                                   vector<2x2xf32>, vector<2x1xf32>>
     ```
   }];
   let builders = [OpBuilder<
@@ -449,8 +455,9 @@ def Vector_FMAOp :
     to the `llvm.fma.*` intrinsic.
 
     Example:
+
     ```mlir
-      %3 = vector.fma %0, %1, %2: vector<8x16xf32>
+    %3 = vector.fma %0, %1, %2: vector<8x16xf32>
     ```
   }];
   // Fully specified by traits.
@@ -483,10 +490,11 @@ def Vector_InsertElementOp :
     https://llvm.org/docs/LangRef.html#insertelement-instruction
 
     Example:
+
     ```mlir
-      %c = constant 15 : i32
-      %f = constant 0.0f : f32
-      %1 = vector.insertelement %f, %0[%c : i32]: vector<16xf32>
+    %c = constant 15 : i32
+    %f = constant 0.0f : f32
+    %1 = vector.insertelement %f, %0[%c : i32]: vector<16xf32>
     ```
   }];
   let extraClassDeclaration = [{
@@ -515,12 +523,11 @@ def Vector_InsertOp :
     and inserts the n-D source into the (n+k)-D destination at the proper
     position. Degenerates to a scalar source type when n = 0.
 
-    Examples:
+    Example:
+
     ```mlir
-      %2 = vector.insert %0, %1[3]:
-        vector<8x16xf32> into vector<4x8x16xf32>
-      %5 = vector.insert %3, %4[3, 3, 3]:
-        f32 into vector<4x8x16xf32>
+    %2 = vector.insert %0, %1[3] : vector<8x16xf32> into vector<4x8x16xf32>
+    %5 = vector.insert %3, %4[3, 3, 3] : f32 into vector<4x8x16xf32>
     ```
   }];
   let assemblyFormat = [{
@@ -558,22 +565,23 @@ def Vector_InsertSlicesOp :
     linear index of the slice w.r.t the unrolling scheme represented by 'sizes'.
     Currently, only unit strides are supported.
 
-    Examples:
+    Example:
+
     ```mlir
-      %0 = vector.extract_slices %0, [2, 2], [1, 1]
-        : vector<4x2xf32> into tuple<vector<2x2xf32>, vector<2x2xf32>>
+    %0 = vector.extract_slices %0, [2, 2], [1, 1]
+      : vector<4x2xf32> into tuple<vector<2x2xf32>, vector<2x2xf32>>
 
-      %1 = vector.insert_slices %0, [2, 2], [1, 1]
-        : tuple<vector<2x2xf32>, vector<2x2xf32>> into vector<4x2xf32>
+    %1 = vector.insert_slices %0, [2, 2], [1, 1]
+      : tuple<vector<2x2xf32>, vector<2x2xf32>> into vector<4x2xf32>
 
-      // Example with partial slices at dimension boundaries.
-      %3 = vector.extract_slices %2, [2, 2], [1, 1]
-        : vector<4x3xf32> into tuple<vector<2x2xf32>, vector<2x1xf32>,
-                                     vector<2x2xf32>, vector<2x1xf32>>
+    // Example with partial slices at dimension boundaries.
+    %3 = vector.extract_slices %2, [2, 2], [1, 1]
+      : vector<4x3xf32> into tuple<vector<2x2xf32>, vector<2x1xf32>,
+                                   vector<2x2xf32>, vector<2x1xf32>>
 
-      %4 = vector.insert_slices %3, [2, 2], [1, 1]
-        : tuple<vector<2x2xf32>, vector<2x1xf32>,
-                vector<2x2xf32>, vector<2x1xf32>> into vector<4x3xf32>
+    %4 = vector.insert_slices %3, [2, 2], [1, 1]
+      : tuple<vector<2x2xf32>, vector<2x1xf32>,
+              vector<2x2xf32>, vector<2x1xf32>> into vector<4x3xf32>
     ```
   }];
 
@@ -616,11 +624,12 @@ def Vector_InsertStridedSliceOp :
     the last k-D dimensions contain the k-D source vector elements strided at
     the proper location as specified by the offsets.
 
-    Examples:
+    Example:
+
     ```mlir
-      %2 = vector.insert_strided_slice %0, %1
-          {offsets = [0, 0, 2], strides = [1, 1]}:
-        vector<2x4xf32> into vector<16x4x8xf32>
+    %2 = vector.insert_strided_slice %0, %1
+        {offsets = [0, 0, 2], strides = [1, 1]}:
+      vector<2x4xf32> into vector<16x4x8xf32>
     ```
   }];
 
@@ -658,14 +667,15 @@ def Vector_OuterProductOp :
     the LLVMIR dialect, this form emits `llvm.intr.fma`, which is guaranteed to
     lower to actual `fma` instructions on x86.
 
-    Examples:
-    ```mlir
-      %2 = vector.outerproduct %0, %1: vector<4xf32>, vector<8xf32>
-      return %2: vector<4x8xf32>
+    Example:
 
-      %3 = vector.outerproduct %0, %1, %2:
-        vector<4xf32>, vector<8xf32>, vector<4x8xf32>
-      return %3: vector<4x8xf32>
+    ```
+    %2 = vector.outerproduct %0, %1: vector<4xf32>, vector<8xf32>
+    return %2: vector<4x8xf32>
+
+    %3 = vector.outerproduct %0, %1, %2:
+      vector<4xf32>, vector<8xf32>, vector<4x8xf32>
+    return %3: vector<4x8xf32>
     ```
   }];
   let extraClassDeclaration = [{
@@ -708,8 +718,8 @@ def Vector_ReshapeOp :
     In the examples below, valid data elements are represented by an alphabetic
     character, and undefined data elements are represented by '-'.
 
-    Example:
-    ```mlir
+    Example
+
       vector<1x8xf32> with valid data shape [6], fixed vector sizes [8]
 
                 input: [a, b, c, d, e, f]
@@ -718,9 +728,8 @@ def Vector_ReshapeOp :
 
         vector layout: [a, b, c, d, e, f, -, -]
 
-    ```
-    Example:
-    ```mlir
+    Example
+
       vector<2x8xf32> with valid data shape [10], fixed vector sizes [8]
 
                 input: [a, b, c, d, e, f, g, h, i, j]
@@ -729,9 +738,9 @@ def Vector_ReshapeOp :
 
         vector layout: [[a, b, c, d, e, f, g, h],
                         [i, j, -, -, -, -, -, -]]
-    ```
-    Example:
-    ```mlir
+
+    Example
+
       vector<2x2x2x3xf32> with valid data shape [3, 5], fixed vector sizes
       [2, 3]
 
@@ -750,9 +759,9 @@ def Vector_ReshapeOp :
                           [-, -, -]]
                          [[n, o, -],
                           [-, -, -]]]]
-    ```
-    Example:
-    ```mlir
+
+    Example
+
       %1 = vector.reshape %0, [%c3, %c6], [%c2, %c9], [4]
         : vector<3x2x4xf32> to vector<2x3x4xf32>
 
@@ -776,7 +785,6 @@ def Vector_ReshapeOp :
                        [[j, k, l, m],
                         [n, o, p, q],
                         [r, -, -, -]]]
-    ```
   }];
 
   let extraClassDeclaration = [{
@@ -828,16 +836,17 @@ def Vector_StridedSliceOp :
     attribute. The returned subvector contains the elements starting at offset
     `offsets` and ending at `offsets + sizes`.
 
-    Examples:
+    Example:
+
     ```mlir
-      %1 = vector.strided_slice %0
-          {offsets = [0, 2], sizes = [2, 4], strides = [1, 1]}:
-        vector<4x8x16xf32> to vector<2x4x16xf32>
-    ```
+    %1 = vector.strided_slice %0
+        {offsets = [0, 2], sizes = [2, 4], strides = [1, 1]}:
+      vector<4x8x16xf32> to vector<2x4x16xf32>
 
     // TODO(ntv) Evolve to a range form syntax similar to:
     %1 = vector.strided_slice %0[0:2:1][2:4:1]
       vector<4x8x16xf32> to vector<2x4x16xf32>
+    ```
   }];
   let builders = [OpBuilder<
     "Builder *builder, OperationState &result, Value source, " #
@@ -948,12 +957,13 @@ def Vector_TransferReadOp :
     implemented using a warp-shuffle if loop `j` were mapped to `threadIdx.x`.
 
     Syntax
-    ```mlir
+    ```
     operation ::= ssa-id `=` `vector.transfer_read` ssa-use-list
       `{` attribute-entry `} :` memref-type `,` vector-type
     ```
 
-    Examples:
+    Example:
+
     ```mlir
     // Read the slice `%A[%i0, %i1:%i1+256, %i2:%i2+32]` into vector<32x256xf32>
     // and pad with %f0 to handle the boundary case:
@@ -1026,14 +1036,7 @@ def Vector_TransferWriteOp :
     valid. Different lowerings may be pertinent depending on the hardware
     support.
 
-    Syntax:
-
-    ```mlir
-    operation ::= `vector.transfer_write` ssa-use-list `{` attribute-entry `} :
-      ` vector-type ', ' memref-type '
-    ```
-
-    Examples:
+    Example:
 
     ```mlir
     // write vector<16x32x64xf32> into the slice
@@ -1099,7 +1102,7 @@ def Vector_ShapeCastOp :
     2-D MLIR vector to a 1-D flattened LLVM vector.shape_cast lowering to LLVM
     is supported in that particular case, for now.
 
-    Examples:
+    Example:
 
     ```mlir
     // Example casting to a lower vector rank.
@@ -1139,7 +1142,7 @@ def Vector_TypeCastOp :
 
     Syntax:
 
-    ```mlir
+    ```
     operation ::= `vector.type_cast` ssa-use : memref-type to memref-type
     ```
 
@@ -1184,20 +1187,20 @@ def Vector_ConstantMaskOp :
     (otherwise element values are set to 0).
 
     Example:
-    ```
-      create a constant vector mask of size 4x3xi1 with elements in range
-      0 <= row <= 2 and 0 <= col <= 1 are set to 1 (others to 0).
-
-      %1 = vector.constant_mask [3, 2] : vector<4x3xi1>
-
-      print %1
-                    columns
-                  0    1    2
-                |------------
-              0 | 1    1    0
-        rows  1 | 1    1    0
-              2 | 1    1    0
-              3 | 0    0    0
+
+    ```mlir
+    // create a constant vector mask of size 4x3xi1 with elements in range
+    // 0 <= row <= 2 and 0 <= col <= 1 are set to 1 (others to 0).
+    %1 = vector.constant_mask [3, 2] : vector<4x3xi1>
+
+    print %1
+                  columns
+                0    1    2
+              |------------
+            0 | 1    1    0
+      rows  1 | 1    1    0
+            2 | 1    1    0
+            3 | 0    0    0
     ```
   }];
 
@@ -1221,20 +1224,20 @@ def Vector_CreateMaskOp :
     (otherwise element values are set to 0).
 
     Example:
-    ```
-      create a vector mask of size 4x3xi1 where elements in range
-      0 <= row <= 2 and 0 <= col <= 1 are set to 1 (others to 0).
-
-      %1 = vector.create_mask %c3, %c2 : vector<4x3xi1>
-
-      print %1
-                    columns
-                  0    1    2
-                |------------
-              0 | 1    1    0
-        rows  1 | 1    1    0
-              2 | 1    1    0
-              3 | 0    0    0
+
+    ```mlir
+    // create a vector mask of size 4x3xi1 where elements in range
+    // 0 <= row <= 2 and 0 <= col <= 1 are set to 1 (others to 0).
+    %1 = vector.create_mask %c3, %c2 : vector<4x3xi1>
+
+    print %1
+                  columns
+                0    1    2
+              |------------
+            0 | 1    1    0
+      rows  1 | 1    1    0
+            2 | 1    1    0
+            3 | 0    0    0
     ```
   }];
 
@@ -1254,16 +1257,17 @@ def Vector_TupleOp :
     transformation and should be removed before lowering to lower-level
     dialects.
 
-    Examples:
-    ```mlir
-      %0 = vector.transfer_read ... : vector<2x2xf32>
-      %1 = vector.transfer_read ... : vector<2x1xf32>
-      %2 = vector.transfer_read ... : vector<2x2xf32>
-      %3 = vector.transfer_read ... : vector<2x1xf32>
 
-      %4 = vector.tuple %0, %1, %2, %3
-        : vector<2x2xf32>, vector<2x1xf32>, vector<2x2xf32>, vector<2x1xf32>
+    Example:
 
+    ```mlir
+    %0 = vector.transfer_read ... : vector<2x2xf32>
+    %1 = vector.transfer_read ... : vector<2x1xf32>
+    %2 = vector.transfer_read ... : vector<2x2xf32>
+    %3 = vector.transfer_read ... : vector<2x1xf32>
+
+    %4 = vector.tuple %0, %1, %2, %3
+      : vector<2x2xf32>, vector<2x1xf32>, vector<2x2xf32>, vector<2x1xf32>
     ```
   }];
 
@@ -1285,16 +1289,19 @@ def Vector_TransposeOp :
     Takes a n-D vector and returns the transposed n-D vector defined by
     the permutation of ranks in the n-sized integer array attribute.
     In the operation
+
     ```mlir
-      %1 = vector.tranpose %0, [i_1, .., i_n]
-        : vector<d_1 x .. x d_n x f32>
-        to vector<d_trans[0] x .. x d_trans[n-1] x f32>
+    %1 = vector.transpose %0, [i_1, .., i_n]
+      : vector<d_1 x .. x d_n x f32>
+      to vector<d_trans[0] x .. x d_trans[n-1] x f32>
     ```
+
     the transp array [i_1, .., i_n] must be a permutation of [0, .., n-1].
 
     Example:
+
     ```mlir
-    %1 = vector.tranpose %0, [1, 0] : vector<2x3xf32> to vector<3x2xf32>
+    %1 = vector.transpose %0, [1, 0] : vector<2x3xf32> to vector<3x2xf32>
 
      [ [a, b, c],       [ [a, d],
        [d, e, f] ]  ->    [b, e],
@@ -1308,10 +1315,13 @@ def Vector_TransposeOp :
     VectorType getResultType() {
       return result().getType().cast<VectorType>();
     }
+    void getTransp(SmallVectorImpl<int64_t> &results);
   }];
   let assemblyFormat = [{
     $vector `,` $transp attr-dict `:` type($vector) `to` type($result)
   }];
+  let hasCanonicalizer = 1;
+  let hasFolder = 1;
 }
 
 def Vector_TupleGetOp :
@@ -1326,14 +1336,15 @@ def Vector_TupleGetOp :
     transformation and should be removed before lowering to lower-level
     dialects.
 
-    Examples:
+    Example:
+
     ```mlir
-      %4 = vector.tuple %0, %1, %2, %3
-        : vector<2x2xf32>, vector<2x1xf32>, vector<2x2xf32>, vector<2x1xf32>>
+    %4 = vector.tuple %0, %1, %2, %3
+      : vector<2x2xf32>, vector<2x1xf32>, vector<2x2xf32>, vector<2x1xf32>>
 
-      %5 = vector.tuple_get %4, 1
-        : tuple<vector<2x2xf32>, vector<2x1xf32>,
-                vector<2x2xf32>, vector<2x1xf32>>
+    %5 = vector.tuple_get %4, 1
+      : tuple<vector<2x2xf32>, vector<2x1xf32>,
+              vector<2x2xf32>, vector<2x1xf32>>
     ```
   }];
 
@@ -1356,21 +1367,22 @@ def Vector_PrintOp :
     Prints the source vector (or scalar) to stdout in human readable
     format (for testing and debugging). No return value.
 
-    Examples:
+    Example:
+
     ```mlir
-      %0 = constant 0.0 : f32
-      %1 = vector.broadcast %0 : f32 to vector<4xf32>
-      vector.print %1 : vector<4xf32>
+    %0 = constant 0.0 : f32
+    %1 = vector.broadcast %0 : f32 to vector<4xf32>
+    vector.print %1 : vector<4xf32>
 
-      when lowered to LLVM, the vector print is unrolled into
-      elementary printing method calls that at runtime will yield
+    when lowered to LLVM, the vector print is unrolled into
+    elementary printing method calls that at runtime will yield
 
-      ( 0.0, 0.0, 0.0, 0.0 )
+    ( 0.0, 0.0, 0.0, 0.0 )
 
-      on stdout when linked with a small runtime support library,
-      which only needs to provide a few printing methods (single
-      value for all data types, opening/closing bracket, comma,
-      newline).
+    on stdout when linked with a small runtime support library,
+    which only needs to provide a few printing methods (single
+    value for all data types, opening/closing bracket, comma,
+    newline).
     ```
   }];
   let verifier = ?;
@@ -1421,9 +1433,9 @@ def Vector_MatmulOp : Vector_Op<"matrix_multiply", [NoSideEffect,
     Example:
 
     ```mlir
-      %C = vector.matrix_multiply %A, %B
-        { lhs_rows = 4: i32, lhs_columns = 16: i32 , rhs_columns = 3: i32 } :
-        (vector<64xf64>, vector<48xf64>) -> vector<12xf64>
+    %C = vector.matrix_multiply %A, %B
+      { lhs_rows = 4: i32, lhs_columns = 16: i32 , rhs_columns = 3: i32 } :
+      (vector<64xf64>, vector<48xf64>) -> vector<12xf64>
     ```
   }];
   let builders = [
diff --git a/mlir/include/mlir/Dialect/Vector/VectorUtils.h b/mlir/include/mlir/Dialect/Vector/VectorUtils.h
index 4bc03e4943fd6..35527542d639a 100644
--- a/mlir/include/mlir/Dialect/Vector/VectorUtils.h
+++ b/mlir/include/mlir/Dialect/Vector/VectorUtils.h
@@ -31,6 +31,9 @@ class VectorType;
 SmallVector<int64_t, 4> computeStrides(ArrayRef<int64_t> shape,
                                        ArrayRef<int64_t> sizes);
 
+/// Computes and returns the linearized index of 'offsets' w.r.t. 'basis'.
+int64_t linearize(ArrayRef<int64_t> offsets, ArrayRef<int64_t> basis);
+
 /// Given the slice strides together with a linear index in the dimension
 /// space, returns the vector-space offsets in each dimension for a
 /// de-linearized index.
diff --git a/mlir/include/mlir/Dialect/VectorOps/VectorOps.td b/mlir/include/mlir/Dialect/VectorOps/VectorOps.td
deleted file mode 100644
index ef1fde00e5ec2..0000000000000
--- a/mlir/include/mlir/Dialect/VectorOps/VectorOps.td
+++ /dev/null
@@ -1,1402 +0,0 @@
-//===- VectorOps.td - Vector op definitions ---------------*- tablegen -*-====//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// Defines MLIR vector operations.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef VECTOR_OPS
-#define VECTOR_OPS
-
-include "mlir/Dialect/Affine/IR/AffineOpsBase.td"
-include "mlir/Interfaces/SideEffects.td"
-
-
-def VectorOps_Dialect : Dialect {
-  let name = "vector";
-  let cppNamespace = "vector";
-  let hasConstantMaterializer = 1;
-}
-
-// Base class for Vector dialect ops.
-class Vector_Op<string mnemonic, list<OpTrait> traits = []> :
-    Op<VectorOps_Dialect, mnemonic, traits> {
-  // For every vector op, there needs to be a:
-  //   * void print(OpAsmPrinter &p, ${C++ class of Op} op)
-  //   * LogicalResult verify(${C++ class of Op} op)
-  //   * ParseResult parse${C++ class of Op}(OpAsmParser &parser,
-  //                                         OperationState &result)
-  // functions.
-  let printer = [{ return ::print(p, *this); }];
-  let verifier = [{ return ::verify(*this); }];
-  let parser = [{ return ::parse$cppClass(parser, result); }];
-}
-
-// TODO(andydavis, ntv) Add an attribute to specify a different algebra
-// with operators other than the current set: {*, +}.
-def Vector_ContractionOp :
-  Vector_Op<"contract", [NoSideEffect,
-     PredOpTrait<"first operand lhs and result have same element type",
-                 TCresVTEtIsSameAsOpBase<0, 0>>,
-     PredOpTrait<"second operand rhs and result have same element type",
-                 TCresVTEtIsSameAsOpBase<0, 1>>,
-     PredOpTrait<"third operand acc and result have same element type",
-                 TCresVTEtIsSameAsOpBase<0, 1>>]>,
-    Arguments<(ins AnyVector:$lhs, AnyVector:$rhs, AnyType:$acc,
-               Variadic<VectorOf<[I1]>>:$masks,
-               AffineMapArrayAttr:$indexing_maps, ArrayAttr:$iterator_types)>,
-    Results<(outs AnyType)> {
-  let summary = "vector contraction operation";
-  let description = [{
-    Computes the sum of products of vector elements along contracting
-    dimension pairs from 2 vectors of rank M and N respectively, adds this
-    intermediate result to the accumulator argument of rank K, and returns a
-    vector result of rank K (where K = num_lhs_free_dims + num_rhs_free_dims +
-    num_batch_dims (see dimension type descriptions below)). For K = 0 (no
-    free or batch dimensions), the accumulator and output are a scalar.
-
-    Optional vector mask arguments (produced by CreateMaskOp or ConstantMaskOp)
-    specify the dynamic dimension sizes of valid data within the lhs/rhs vector
-    arguments.
-
-    An iterator type attribute list must be specified, where each element of
-    the list represents an iterator with one of the following types:
-
-    *) "reduction": reduction dimensions are present in the lhs and rhs
-                    arguments but not in the output (and accumulator
-                    argument). These are the dimensions along which the vector
-                    contraction op computes the sum of products, and
-                    contracting dimension pair dimension sizes must match
-                    between lhs/rhs.
-    *) "parallel": Batch dimensions are iterator type "parallel", and
-                   are non-contracting dimensions present in the lhs, rhs and
-                   output. The lhs/rhs co-iterate along the batch dimensions,
-                   which should be expressed in their indexing maps.
-
-                   Free dimensions are iterator type "parallel", and are
-                   non-contraction, non-batch dimensions accessed by either the
-                   lhs or rhs (but not both). The lhs and rhs free dimensions
-                   are unrelated to each other and do not co-iterate, which
-                   should be expressed in their indexing maps.
-
-    An indexing map attribute list must be specified with an entry for lhs, rhs
-    and acc arguments. An indexing map attribute specifies a mapping from each
-    iterator in the iterator type list, to each dimension of an N-D vector.
-
-    Examples:
-
-      // Simple dot product (K = 0).
-      #contraction_accesses = [
-       affine_map<(i) -> (i)>,
-       affine_map<(i) -> (i)>,
-       affine_map<(i) -> ()>
-      ]
-      #contraction_trait = {
-        indexing_maps = #contraction_accesses,
-        iterator_types = ["reduction"]
-      }
-      %3 = vector.contract #contraction_trait %0, %1, %2
-        : vector<10xf32>, vector<10xf32> into f32
-
-      // 2D vector contraction with one contracting dimension (matmul, K = 2).
-      #contraction_accesses = [
-        affine_map<(i, j, k) -> (i, k)>,
-        affine_map<(i, j, k) -> (k, j)>,
-        affine_map<(i, j, k) -> (i, j)>
-      ]
-      #contraction_trait = {
-        indexing_maps = #contraction_accesses,
-        iterator_types = ["parallel", "parallel", "reduction"]
-      }
-
-      %3 = vector.contract #contraction_trait %0, %1, %2
-        : vector<4x3xf32>, vector<3x7xf32> into vector<4x7xf32>
-
-      // 4D to 3D vector contraction with two contracting dimensions and
-      // one batch dimension (K = 3).
-      #contraction_accesses = [
-        affine_map<(b0, f0, f1, c0, c1) -> (c0, b0, c1, f0)>,
-        affine_map<(b0, f0, f1, c0, c1) -> (b0, c1, c0, f1)>,
-        affine_map<(b0, f0, f1, c0, c1) -> (b0, f0, f1)>
-      ]
-      #contraction_trait = {
-        indexing_maps = #contraction_accesses,
-        iterator_types = ["parallel", "parallel", "parallel",
-                          "reduction", "reduction"]
-      }
-
-      %4 = vector.contract #contraction_trait %0, %1, %2
-          : vector<7x8x16x15xf32>, vector<8x16x7x5xf32> into vector<8x15x5xf32>
-
-      // 4D vector contraction with two contracting dimensions and optional
-      // vector mask arguments.
-      %lhs_mask = vector.constant_mask [7, 8, 16, 15] : vector<7x8x16x15xi1>
-      %rhs_mask = vector.constant_mask [8, 16, 7, 5] : vector<8x16x7x5xi1>
-
-      %5 = vector.contract #contraction_trait %0, %1, %2, %lhs_mask, %rhs_mask
-         : vector<7x8x16x15xf32>, vector<8x16x7x5xf32> into vector<8x15x8x5xf32>
-  }];
-  let builders = [OpBuilder<
-    "Builder *builder, OperationState &result, Value lhs, Value rhs, "
-    "Value acc, ArrayAttr indexingMaps, ArrayAttr iteratorTypes">,
-    OpBuilder<
-      "Builder *builder, OperationState &result, Value lhs, Value rhs, "
-      "Value acc, ArrayRef<ArrayRef<AffineExpr>> indexingExprs, "
-      "ArrayRef<StringRef> iteratorTypes">];
-  let extraClassDeclaration = [{
-    VectorType getLhsType() {
-      return lhs().getType().cast<VectorType>();
-    }
-    VectorType getRhsType() {
-      return rhs().getType().cast<VectorType>();
-    }
-    Type getAccType() { return acc().getType(); }
-    VectorType getLHSVectorMaskType() {
-      if (llvm::size(masks()) != 2) return VectorType();
-      return getOperand(3).getType().cast<VectorType>();
-    }
-    VectorType getRHSVectorMaskType() {
-      if (llvm::size(masks()) != 2) return VectorType();
-      return getOperand(4).getType().cast<VectorType>();
-    }
-    Type getResultType() { return getResult().getType(); }
-    ArrayRef<StringRef> getTraitAttrNames();
-    SmallVector<AffineMap, 4> getIndexingMaps();
-    static unsigned getAccOperandIndex() { return 2; }
-
-    // Returns the bounds of each dimension in the iteration space spanned
-    // by the iterator types of this operation.
-    void getIterationBounds(SmallVectorImpl<int64_t> &iterationBounds);
-
-    // Returns a list of index maps, where there is a list entry for each
-    // op indexing map attribute (i.e. one for each input and output, with
-    // the output listed last). Each index map, maps from this operations
-    // iteration space, to vector dimensions of the maps input/output.
-    void getIterationIndexMap(
-      std::vector<DenseMap<int64_t, int64_t>> &iterationIndexMap);
-
-    std::vector<std::pair<int64_t, int64_t>> getContractingDimMap();
-    std::vector<std::pair<int64_t, int64_t>> getBatchDimMap();
-  }];
-}
-
-def Vector_ReductionOp :
-  Vector_Op<"reduction", [NoSideEffect,
-     PredOpTrait<"source operand and result have same element type",
-                 TCresVTEtIsSameAsOpBase<0, 0>>]>,
-    Arguments<(ins StrAttr:$kind, AnyVector:$vector, Variadic<AnyType>:$acc)>,
-    Results<(outs AnyType:$dest)> {
-  let summary = "reduction operation";
-  let description = [{
-    Reduces an 1-D vector "horizontally" into a scalar using the given
-    operation (add/mul/min/max for int/fp and and/or/xor for int only).
-    Some reductions (add/mul for fp) also allow an optional fused
-    accumulator.
-
-    Note that these operations are restricted to 1-D vectors to remain
-    close to the corresponding LLVM intrinsics:
-
-    http://llvm.org/docs/LangRef.html#experimental-vector-reduction-intrinsics
-
-    Examples:
-    ```
-      %1 = vector.reduction "add", %0 : vector<16xf32> into f32
-
-      %3 = vector.reduction "xor", %2 : vector<4xi32> into i32
-
-      %4 = vector.reduction "mul", %0, %1 : vector<16xf32> into f32
-    ```
-  }];
-  let extraClassDeclaration = [{
-    VectorType getVectorType() {
-      return vector().getType().cast<VectorType>();
-    }
-  }];
-}
-
-def Vector_BroadcastOp :
-  Vector_Op<"broadcast", [NoSideEffect,
-     PredOpTrait<"source operand and result have same element type",
-                 TCresVTEtIsSameAsOpBase<0, 0>>]>,
-    Arguments<(ins AnyType:$source)>,
-    Results<(outs AnyVector:$vector)> {
-  let summary = "broadcast operation";
-  let description = [{
-    Broadcasts the scalar or k-D vector value in the source operand
-    to a n-D result vector such that the broadcast makes sense, i.e.,
-    the source operand is duplicated to match the given rank and sizes
-    in the result vector. The legality rules are:
-    * the source operand must have the same element type as the result type
-    * a k-D vector <s_1 x .. x s_k x type> can be broadcast to
-      a n-D vector <t_1 x .. x t_n x type> if
-       * k <= n, and
-       * the sizes in the trailing dimensions n-k < i <= n with j=i+k-n
-          match exactly as s_j = t_i or s_j = 1:
-       ```
-           t_1 x   ..  t_n-k x t_n-k+1 x .. x t_i x .. x t_n
-                               s_1     x .. x s_j x .. x s_k
-               <duplication>         <potential stretch>
-       ```
-    The source operand is duplicated over all the missing leading dimensions
-    and stretched over the trailing dimensions where the source has a non-equal
-    dimension of 1. These rules imply that any scalar broadcast (k=0) to any
-    shaped vector with the same element type is always legal.
-
-    Examples:
-    ```
-      %0 = constant 0.0 : f32
-      %1 = vector.broadcast %0 : f32 to vector<16xf32>
-      %2 = vector.broadcast %1 : vector<16xf32> to vector<4x16xf32>
-    ```
-  }];
-  let extraClassDeclaration = [{
-    Type getSourceType() { return source().getType(); }
-    VectorType getVectorType() {
-      return vector().getType().cast<VectorType>();
-    }
-  }];
-  let assemblyFormat = "$source attr-dict `:` type($source) `to` type($vector)";
-}
-
-def Vector_ShuffleOp :
-  Vector_Op<"shuffle", [NoSideEffect,
-     PredOpTrait<"first operand v1 and result have same element type",
-                 TCresVTEtIsSameAsOpBase<0, 0>>,
-     PredOpTrait<"second operand v2 and result have same element type",
-                 TCresVTEtIsSameAsOpBase<0, 1>>]>,
-     Arguments<(ins AnyVector:$v1, AnyVector:$v2, I64ArrayAttr:$mask)>,
-     Results<(outs AnyVector:$vector)> {
-  let summary = "shuffle operation";
-  let description = [{
-    The shuffle operation constructs a permutation (or duplication) of elements
-    from two input vectors, returning a vector with the same element type as
-    the input and a length that is the same as the shuffle mask. The two input
-    vectors must have the same element type, rank, and trailing dimension sizes
-    and shuffles their values in the leading dimension (which may differ in size)
-    according to the given mask. The legality rules are:
-    * the two operands must have the same element type as the result
-    * the two operands and the result must have the same rank and trailing
-      dimension sizes, viz. given two k-D operands
-              v1 : <s_1 x s_2 x .. x s_k x type> and
-              v2 : <t_1 x t_2 x .. x t_k x type>
-      we have s_i = t_i for all 1 < i <= k
-    * the mask length equals the leading dimension size of the result
-    * numbering the input vector indices left to right across the operands, all
-      mask values must be within range, viz. given two k-D operands v1 and v2
-      above, all mask values are in the range [0,s_1+t_1)
-
-    Examples:
-    ```
-    %0 = vector.shuffle %a, %b[0, 3]
-               : vector<2xf32>, vector<2xf32>       ; yields vector<2xf32>
-    %1 = vector.shuffle %c, %b[0, 1, 2]
-               : vector<2x16xf32>, vector<1x16xf32> ; yields vector<3x16xf32>
-    %2 = vector.shuffle %a, %b[3, 2, 1, 0]
-               : vector<2xf32>, vector<2xf32>       ; yields vector<4xf32>
-
-    ```
-  }];
-  let builders = [OpBuilder<"Builder *builder, OperationState &result,"
-                            "Value v1, Value v2, ArrayRef<int64_t>">];
-  let extraClassDeclaration = [{
-    static StringRef getMaskAttrName() { return "mask"; }
-    VectorType getV1VectorType() {
-      return v1().getType().cast<VectorType>();
-    }
-    VectorType getV2VectorType() {
-      return v2().getType().cast<VectorType>();
-    }
-    VectorType getVectorType() {
-      return vector().getType().cast<VectorType>();
-    }
-  }];
-}
-
-def Vector_ExtractElementOp :
-  Vector_Op<"extractelement", [NoSideEffect,
-     TypesMatchWith<"result type matches element type of vector operand",
-                    "vector", "result",
-                    "$_self.cast<ShapedType>().getElementType()">]>,
-    Arguments<(ins AnyVector:$vector, AnySignlessInteger:$position)>,
-    Results<(outs AnyType:$result)> {
-  let summary = "extractelement operation";
-  let description = [{
-    Takes an 1-D vector and a dynamic index position and extracts the
-    scalar at that position. Note that this instruction resembles
-    vector.extract, but is restricted to 1-D vectors and relaxed
-    to dynamic indices. It is meant to be closer to LLVM's version:
-    https://llvm.org/docs/LangRef.html#extractelement-instruction
-
-    Example:
-    ```
-      %c = constant 15 : i32
-      %1 = vector.extractelement %0[%c : i32]: vector<16xf32>
-    ```
-  }];
-  let extraClassDeclaration = [{
-    VectorType getVectorType() {
-      return vector().getType().cast<VectorType>();
-    }
-  }];
-
-  let assemblyFormat = [{
-    $vector `[` $position `:` type($position) `]` attr-dict `:` type($vector)
-  }];
-}
-
-def Vector_ExtractOp :
-  Vector_Op<"extract", [NoSideEffect,
-     PredOpTrait<"operand and result have same element type",
-                 TCresVTEtIsSameAsOpBase<0, 0>>]>,
-    Arguments<(ins AnyVector:$vector, I64ArrayAttr:$position)>,
-    Results<(outs AnyType)> {
-  let summary = "extract operation";
-  let description = [{
-    Takes an n-D vector and a k-D position and extracts the (n-k)-D vector at
-    the proper position. Degenerates to an element type in the 0-D case.
-
-    Examples:
-    ```
-      %1 = vector.extract %0[3]: vector<4x8x16xf32>
-      %2 = vector.extract %0[3, 3, 3]: vector<4x8x16xf32>
-    ```
-  }];
-  let builders = [OpBuilder<
-    "Builder *builder, OperationState &result, Value source,"
-    "ArrayRef<int64_t>">];
-  let extraClassDeclaration = [{
-    static StringRef getPositionAttrName() { return "position"; }
-    VectorType getVectorType() {
-      return vector().getType().cast<VectorType>();
-    }
-  }];
-}
-
-def Vector_ExtractSlicesOp :
-  Vector_Op<"extract_slices", [NoSideEffect]>,
-    Arguments<(ins AnyVector:$vector, I64ArrayAttr:$sizes,
-                   I64ArrayAttr:$strides)>,
-    Results<(outs TupleOf<[AnyVector]>)> {
-  let summary = "vector extract slices operation";
-  let description = [{
-    Takes an N-d vector and returns a tuple of vector slices of 'vector',
-    based on 'sizes' and 'strides' parameters.
-
-    The arguments 'sizes' and 'strides' represent a specification for
-    generating the unrolling of 'vector' shape, which has all slices of shape
-    'sizes' except for slices at dimension boundaries when 'vector' dimension
-    sizes are not a multiple of 'sizes'.
-
-    Each slice is returned at the tuple element index corresponding to the
-    linear index of the slice w.r.t the unrolling scheme represented by 'sizes'.
-    Currently, only unit strides are supported.
-
-    Examples:
-    ```
-      %0 = vector.transfer_read ...: vector<4x2xf32>
-
-      %1 = vector.extract_slices %0, [2, 2], [1, 1]
-        : vector<4x2xf32> into tuple<vector<2x2xf32>, vector<2x2xf32>>
-
-      // Example with partial slices at dimension boundaries.
-      %2 = vector.transfer_read ...: vector<4x3xf32>
-
-      %3 = vector.extract_slices %2, [2, 2], [1, 1]
-        : vector<4x3xf32> into tuple<vector<2x2xf32>, vector<2x1xf32>,
-                                     vector<2x2xf32>, vector<2x1xf32>>
-    ```
-  }];
-  let builders = [OpBuilder<
-    "Builder *builder, OperationState &result, TupleType tupleType, " #
-    "Value vector, ArrayRef<int64_t> sizes, " #
-    "ArrayRef<int64_t> strides">];
-  let extraClassDeclaration = [{
-    VectorType getSourceVectorType() {
-      return vector().getType().cast<VectorType>();
-    }
-    TupleType getResultTupleType() {
-      return getResult().getType().cast<TupleType>();
-    }
-    void getSizes(SmallVectorImpl<int64_t> &results);
-    void getStrides(SmallVectorImpl<int64_t> &results);
-    static StringRef getSizesAttrName() { return "sizes"; }
-    static StringRef getStridesAttrName() { return "strides"; }
-  }];
-  let assemblyFormat = [{
-    $vector `,` $sizes `,` $strides attr-dict `:` type($vector) `into`
-    type(results)
-  }];
-}
-
-def Vector_FMAOp :
-  Op<VectorOps_Dialect, "fma", [NoSideEffect,
-                             AllTypesMatch<["lhs", "rhs", "acc", "result"]>]>,
-    Arguments<(ins AnyVector:$lhs, AnyVector:$rhs, AnyVector:$acc)>,
-    Results<(outs AnyVector:$result)> {
-  let summary = "vector fused multiply-add";
-  let description = [{
-    Multiply-add expressions operate on n-D vectors and compute a fused
-    pointwise multiply-and-accumulate: `$result = `$lhs * $rhs + $acc`.
-    All operands and result have the same vector type. The semantics
-    of the operation correspond to those of the `llvm.fma`
-    [intrinsic](https://llvm.org/docs/LangRef.html#int-fma). In the
-    particular case of lowering to LLVM, this is guaranteed to lower
-    to the `llvm.fma.*` intrinsic.
-
-    Example:
-
-    ```
-      %3 = vector.fma %0, %1, %2: vector<8x16xf32>
-    ```
-  }];
-  // Fully specified by traits.
-  let verifier = ?;
-  let assemblyFormat = "$lhs `,` $rhs `,` $acc attr-dict `:` type($lhs)";
-  let builders = [OpBuilder<
-    "Builder *b, OperationState &result, Value lhs, Value rhs, Value acc",
-    "build(b, result, lhs.getType(), lhs, rhs, acc);">];
-  let extraClassDeclaration = [{
-    VectorType getVectorType() { return lhs().getType().cast<VectorType>(); }
-  }];
-}
-
-def Vector_InsertElementOp :
-  Vector_Op<"insertelement", [NoSideEffect,
-     TypesMatchWith<"source operand type matches element type of result",
-                    "result", "source",
-                    "$_self.cast<ShapedType>().getElementType()">,
-     AllTypesMatch<["dest", "result"]>]>,
-     Arguments<(ins AnyType:$source, AnyVector:$dest,
-                    AnySignlessInteger:$position)>,
-     Results<(outs AnyVector:$result)> {
-  let summary = "insertelement operation";
-  let description = [{
-    Takes a scalar source, an 1-D destination vector and a dynamic index
-    position and inserts the source into the destination at the proper
-    position.  Note that this instruction resembles vector.insert, but
-    is restricted to 1-D vectors and relaxed to dynamic indices. It is
-    meant to be closer to LLVM's version:
-    https://llvm.org/docs/LangRef.html#insertelement-instruction
-
-    Example:
-    ```
-      %c = constant 15 : i32
-      %f = constant 0.0f : f32
-      %1 = vector.insertelement %f, %0[%c : i32]: vector<16xf32>
-    ```
-  }];
-  let extraClassDeclaration = [{
-    Type getSourceType() { return source().getType(); }
-    VectorType getDestVectorType() {
-      return dest().getType().cast<VectorType>();
-    }
-  }];
-
-  let assemblyFormat = [{
-    $source `,` $dest `[` $position `:` type($position) `]` attr-dict `:`
-    type($result)
-  }];
-}
-
-def Vector_InsertOp :
-  Vector_Op<"insert", [NoSideEffect,
-     PredOpTrait<"source operand and result have same element type",
-                 TCresVTEtIsSameAsOpBase<0, 0>>,
-     AllTypesMatch<["dest", "res"]>]>,
-     Arguments<(ins AnyType:$source, AnyVector:$dest, I64ArrayAttr:$position)>,
-     Results<(outs AnyVector:$res)> {
-  let summary = "insert operation";
-  let description = [{
-    Takes an n-D source vector, an (n+k)-D destination vector and a k-D position
-    and inserts the n-D source into the (n+k)-D destination at the proper
-    position. Degenerates to a scalar source type when n = 0.
-
-    Examples:
-    ```
-      %2 = vector.insert %0, %1[3]:
-        vector<8x16xf32> into vector<4x8x16xf32>
-      %5 = vector.insert %3, %4[3, 3, 3]:
-        f32 into vector<4x8x16xf32>
-    ```
-  }];
-  let assemblyFormat = [{
-    $source `,` $dest $position attr-dict `:` type($source) `into` type($dest)
-  }];
-
-  let builders = [OpBuilder<
-    "Builder *builder, OperationState &result, Value source, " #
-    "Value dest, ArrayRef<int64_t>">];
-  let extraClassDeclaration = [{
-    static StringRef getPositionAttrName() { return "position"; }
-    Type getSourceType() { return source().getType(); }
-    VectorType getDestVectorType() {
-      return dest().getType().cast<VectorType>();
-    }
-  }];
-}
-
-def Vector_InsertSlicesOp :
-  Vector_Op<"insert_slices", [NoSideEffect]>,
-    Arguments<(ins TupleOf<[AnyVector]>:$vectors, I64ArrayAttr:$sizes,
-                   I64ArrayAttr:$strides)>,
-    Results<(outs AnyVector)> {
-  let summary = "vector insert slices operation";
-  let description = [{
-    Takes a tuple of vector slices and inserts them into the vector result
-    according to the 'sizes' and 'strides' parameters.
-
-    The arguments 'sizes' and 'strides' represent a specification for
-    generating the unrolling of 'vector' shape, which has all slices of shape
-    'sizes' except for slices at dimension boundaries when 'vector' dimension
-    sizes are not a multiple of 'sizes'.
-
-    Each slice in 'vectors' is at the tuple element index corresponding to the
-    linear index of the slice w.r.t the unrolling scheme represented by 'sizes'.
-    Currently, only unit strides are supported.
-
-    Examples:
-    ```
-      %0 = vector.extract_slices %0, [2, 2], [1, 1]
-        : vector<4x2xf32> into tuple<vector<2x2xf32>, vector<2x2xf32>>
-
-      %1 = vector.insert_slices %0, [2, 2], [1, 1]
-        : tuple<vector<2x2xf32>, vector<2x2xf32>> into vector<4x2xf32>
-
-      // Example with partial slices at dimension boundaries.
-      %3 = vector.extract_slices %2, [2, 2], [1, 1]
-        : vector<4x3xf32> into tuple<vector<2x2xf32>, vector<2x1xf32>,
-                                     vector<2x2xf32>, vector<2x1xf32>>
-
-      %4 = vector.insert_slices %3, [2, 2], [1, 1]
-        : tuple<vector<2x2xf32>, vector<2x1xf32>,
-                vector<2x2xf32>, vector<2x1xf32>> into vector<4x3xf32>
-    ```
-  }];
-
-  let extraClassDeclaration = [{
-    TupleType getSourceTupleType() {
-      return vectors().getType().cast<TupleType>();
-    }
-    VectorType getResultVectorType() {
-      return getResult().getType().cast<VectorType>();
-    }
-    void getSizes(SmallVectorImpl<int64_t> &results);
-    void getStrides(SmallVectorImpl<int64_t> &results);
-    static StringRef getSizesAttrName() { return "sizes"; }
-    static StringRef getStridesAttrName() { return "strides"; }
-  }];
-  let assemblyFormat = [{
-    $vectors `,` $sizes `,` $strides attr-dict `:` type($vectors) `into`
-    type(results)
-  }];
-}
-
-def Vector_InsertStridedSliceOp :
-  Vector_Op<"insert_strided_slice", [NoSideEffect,
-    PredOpTrait<"operand #0 and result have same element type",
-                 TCresVTEtIsSameAsOpBase<0, 0>>,
-    AllTypesMatch<["dest", "res"]>]>,
-    Arguments<(ins AnyVector:$source, AnyVector:$dest, I64ArrayAttr:$offsets,
-               I64ArrayAttr:$strides)>,
-    Results<(outs AnyVector:$res)> {
-  let summary = "strided_slice operation";
-  let description = [{
-    Takes a k-D source vector, an n-D destination vector (n >= k), n-sized
-    `offsets` integer array attribute, a k-sized `strides` integer array attribute
-    and inserts the k-D source vector as a strided subvector at the proper offset
-    into the n-D destination vector.
-
-    At the moment strides must contain only 1s.
-
-    Returns an n-D vector that is a copy of the n-D destination vector in which
-    the last k-D dimensions contain the k-D source vector elements strided at
-    the proper location as specified by the offsets.
-
-    Examples:
-    ```
-      %2 = vector.insert_strided_slice %0, %1
-          {offsets = [0, 0, 2], strides = [1, 1]}:
-        vector<2x4xf32> into vector<16x4x8xf32>
-    ```
-  }];
-
-  let assemblyFormat = [{
-    $source `,` $dest attr-dict `:` type($source) `into` type($dest)
-  }];
-
-  let builders = [OpBuilder<
-    "Builder *builder, OperationState &result, Value source, Value dest, " #
-    "ArrayRef<int64_t> offsets, ArrayRef<int64_t> strides">];
-  let extraClassDeclaration = [{
-    static StringRef getOffsetsAttrName() { return "offsets"; }
-    static StringRef getStridesAttrName() { return "strides"; }
-    VectorType getSourceVectorType() {
-      return source().getType().cast<VectorType>();
-    }
-    VectorType getDestVectorType() {
-      return dest().getType().cast<VectorType>();
-    }
-  }];
-}
-
-def Vector_OuterProductOp :
-  Vector_Op<"outerproduct", [NoSideEffect, SameOperandsAndResultElementType]>,
-    Arguments<(ins AnyVector:$lhs, AnyVector:$rhs, Variadic<AnyVector>:$acc)>,
-    Results<(outs AnyVector)> {
-  let summary = "vector outerproduct with optional fused add";
-  let description = [{
-    Takes 2 1-D vectors and returns the 2-D vector containing the outer-product.
-
-    An optional extra 2-D vector argument may be specified in which case the
-    operation returns the sum of the outer-product and the extra vector. In this
-    multiply-accumulate scenario, the rounding mode is that obtained by
-    guaranteeing that a fused-multiply add operation is emitted. When lowered to
-    the LLVMIR dialect, this form emits `llvm.intr.fma`, which is guaranteed to
-    lower to actual `fma` instructions on x86.
-
-    Examples:
-
-    ```
-      %2 = vector.outerproduct %0, %1: vector<4xf32>, vector<8xf32>
-      return %2: vector<4x8xf32>
-
-      %3 = vector.outerproduct %0, %1, %2:
-        vector<4xf32>, vector<8xf32>, vector<4x8xf32>
-      return %3: vector<4x8xf32>
-    ```
-  }];
-  let extraClassDeclaration = [{
-    VectorType getOperandVectorTypeLHS() {
-      return lhs().getType().cast<VectorType>();
-    }
-    VectorType getOperandVectorTypeRHS() {
-      return rhs().getType().cast<VectorType>();
-    }
-    VectorType getOperandVectorTypeACC() {
-      return (llvm::size(acc()) == 0) ? VectorType() :
-        (*acc().begin()).getType().cast<VectorType>();
-    }
-    VectorType getVectorType() {
-      return getResult().getType().cast<VectorType>();
-    }
-  }];
-}
-
-// TODO(andydavis) Add transformation which decomposes ReshapeOp into an
-// optimized sequence of vector rotate/shuffle/select operations.
-def Vector_ReshapeOp :
-  Vector_Op<"reshape", [AttrSizedOperandSegments, NoSideEffect]>,
-    Arguments<(ins AnyVector:$vector, Variadic<Index>:$input_shape,
-               Variadic<Index>:$output_shape,
-               I64ArrayAttr:$fixed_vector_sizes)>,
-    Results<(outs AnyVector:$result)> {
-  let summary = "vector reshape operation";
-  let description = [{
-    Reshapes its vector operand from 'input_shape' to 'output_shape' maintaining
-    fixed vector dimension 'fixed_vector_sizes' on the innermost vector
-    dimensions.
-
-    The parameters 'input_shape' and 'output_shape' represent valid data shapes
-    across fixed vector shapes. For example, if a vector has a valid data
-    shape [6] with fixed vector size [8], then the valid data elements are
-    assumed to be stored at the beginning of the vector with the remaining
-    vector elements undefined.
-
-    In the examples below, valid data elements are represented by an alphabetic
-    character, and undefined data elements are represented by '-'.
-
-    Example
-
-      vector<1x8xf32> with valid data shape [6], fixed vector sizes [8]
-
-                input: [a, b, c, d, e, f]
-
-           layout map: (d0) -> (d0 floordiv 8, d0 mod 8)
-
-        vector layout: [a, b, c, d, e, f, -, -]
-
-    Example
-
-      vector<2x8xf32> with valid data shape [10], fixed vector sizes [8]
-
-                input: [a, b, c, d, e, f, g, h, i, j]
-
-           layout map: (d0) -> (d0 floordiv 8, d0 mod 8)
-
-        vector layout: [[a, b, c, d, e, f, g, h],
-                        [i, j, -, -, -, -, -, -]]
-
-    Example
-
-      vector<2x2x2x3xf32> with valid data shape [3, 5], fixed vector sizes
-      [2, 3]
-
-                input: [[a, b, c, d, e],
-                        [f, g, h, i, j],
-                        [k, l, m, n, o]]
-
-           layout map: (d0, d1) -> (d0 floordiv 3, d1 floordiv 5,
-                                    d0 mod 3, d1 mod 5)
-
-        vector layout: [[[[a, b, c],
-                          [f, g, h]]
-                         [[d, e, -],
-                          [i, j, -]]],
-                        [[[k, l, m],
-                          [-, -, -]]
-                         [[n, o, -],
-                          [-, -, -]]]]
-
-    Example
-
-      %1 = vector.reshape %0, [%c3, %c6], [%c2, %c9], [4]
-        : vector<3x2x4xf32> to vector<2x3x4xf32>
-
-             input: [[a, b, c, d, e, f],
-                     [g, h, i, j, k, l],
-                     [m, n, o, p, q, r]]
-
-        layout map: (d0, d1) -> (d0, d1 floordiv 4, d1 mod 4)
-
-
-      Input vector:  [[[a, b, c, d],
-                       [e, f, -, -]],
-                      [[g, h, i, j],
-                       [k, l, -, -]],
-                      [[m, n, o, p],
-                       [q, r, -, -]]]
-
-      Output vector:  [[[a, b, c, d],
-                        [e, f, g, h],
-                        [i, -, -, -]],
-                       [[j, k, l, m],
-                        [n, o, p, q],
-                        [r, -, -, -]]]
-  }];
-
-  let extraClassDeclaration = [{
-    VectorType getInputVectorType() {
-      return vector().getType().cast<VectorType>();
-    }
-    VectorType getOutputVectorType() {
-      return getResult().getType().cast<VectorType>();
-    }
-
-    /// Returns as integer value the number of input shape operands.
-    int64_t getNumInputShapeSizes() { return input_shape().size(); }
-
-    /// Returns as integer value the number of output shape operands.
-    int64_t getNumOutputShapeSizes() { return output_shape().size(); }
-
-    void getFixedVectorSizes(SmallVectorImpl<int64_t> &results);
-
-    static StringRef getFixedVectorSizesAttrName() {
-      return "fixed_vector_sizes";
-    }
-    static StringRef getInputShapeAttrName() { return "input_shape"; }
-    static StringRef getOutputShapeAttrName() { return "output_shape"; }
-  }];
-
-  let assemblyFormat = [{
-    $vector `,` `[` $input_shape `]` `,` `[` $output_shape `]` `,`
-    $fixed_vector_sizes attr-dict `:` type($vector) `to` type($result)
-  }];
-}
-
-def Vector_StridedSliceOp :
-  Vector_Op<"strided_slice", [NoSideEffect,
-    PredOpTrait<"operand and result have same element type",
-                 TCresVTEtIsSameAsOpBase<0, 0>>]>,
-    Arguments<(ins AnyVector:$vector, I64ArrayAttr:$offsets,
-               I64ArrayAttr:$sizes, I64ArrayAttr:$strides)>,
-    Results<(outs AnyVector)> {
-  let summary = "strided_slice operation";
-  let description = [{
-    Takes an n-D vector, k-D `offsets` integer array attribute, a k-sized
-    `sizes` integer array attribute, a k-sized `strides` integer array
-    attribute and extracts the n-D subvector at the proper offset.
-
-    At the moment strides must contain only 1s.
-    // TODO(ntv) support non-1 strides.
-
-    Returns an n-D vector where the first k-D dimensions match the `sizes`
-    attribute. The returned subvector contains the elements starting at offset
-    `offsets` and ending at `offsets + sizes`.
-
-    Examples:
-    ```
-      %1 = vector.strided_slice %0
-          {offsets = [0, 2], sizes = [2, 4], strides = [1, 1]}:
-        vector<4x8x16xf32> to vector<2x4x16xf32>
-    ```
-
-    // TODO(ntv) Evolve to a range form syntax similar to:
-    %1 = vector.strided_slice %0[0:2:1][2:4:1]
-      vector<4x8x16xf32> to vector<2x4x16xf32>
-  }];
-  let builders = [OpBuilder<
-    "Builder *builder, OperationState &result, Value source, " #
-    "ArrayRef<int64_t> offsets, ArrayRef<int64_t> sizes, " #
-    "ArrayRef<int64_t> strides">];
-  let extraClassDeclaration = [{
-    static StringRef getOffsetsAttrName() { return "offsets"; }
-    static StringRef getSizesAttrName() { return "sizes"; }
-    static StringRef getStridesAttrName() { return "strides"; }
-    VectorType getVectorType(){ return vector().getType().cast<VectorType>(); }
-    void getOffsets(SmallVectorImpl<int64_t> &results);
-  }];
-  let hasCanonicalizer = 1;
-  let assemblyFormat = "$vector attr-dict `:` type($vector) `to` type(results)";
-}
-
-def Vector_TransferReadOp :
-  Vector_Op<"transfer_read">,
-    Arguments<(ins AnyMemRef:$memref, Variadic<Index>:$indices,
-               AffineMapAttr:$permutation_map, AnyType:$padding)>,
-    Results<(outs AnyVector:$vector)> {
-
-  let summary = "Reads a supervector from memory into an SSA vector value.";
-
-  let description = [{
-    The `vector.transfer_read` op performs a blocking read from a slice within
-    a [MemRef](../LangRef.md#memref-type) supplied as its first operand
-    into a [vector](../LangRef.md#vector-type) of the same base elemental type.
-
-    A memref operand with vector element type, must have its vector element
-    type match a suffix (shape and element type) of the vector (e.g.
-    memref<3x2x6x4x3xf32>, vector<1x1x4x3xf32>).
-
-    The slice is further defined by a full-rank index within the MemRef,
-    supplied as the operands `2 .. 1 + rank(memref)`. The permutation_map
-    [attribute](../LangRef.md#attributes) is an
-    [affine-map](Affine.md#affine-maps) which specifies the transposition on the
-    slice to match the vector shape. The size of the slice is specified by the
-    size of the vector, given as the return type. An `ssa-value` of the same
-    elemental type as the MemRef is provided as the last operand to specify
-    padding in the case of out-of-bounds accesses. This operation is called
-    'read' by opposition to 'load' because the super-vector granularity is
-    generally not representable with a single hardware register.
-    A `vector.transfer_read` is thus a mid-level
-    abstraction that supports super-vectorization with non-effecting padding for
-    full-tile-only code.
-
-    More precisely, let's dive deeper into the permutation_map for the following
-    MLIR:
-
-    ```mlir
-    vector.transfer_read %A[%expr1, %expr2, %expr3, %expr4]
-      { permutation_map : (d0,d1,d2,d3) -> (d2,0,d0) } :
-      memref<?x?x?x?xf32>, vector<3x4x5xf32>
-    ```
-
-    This operation always reads a slice starting at `%A[%expr1, %expr2, %expr3,
-    %expr4]`. The size of the slice is 3 along d2 and 5 along d0, so the slice
-    is: `%A[%expr1 : %expr1 + 5, %expr2, %expr3:%expr3 + 3, %expr4]`
-
-    That slice needs to be read into a `vector<3x4x5xf32>`. Since the
-    permutation map is not full rank, there must be a broadcast along vector
-    dimension `1`.
-
-    A notional lowering of vector.transfer_read could generate code resembling:
-
-    ```mlir
-    // %expr1, %expr2, %expr3, %expr4 defined before this point
-    %tmp = alloc() : vector<3x4x5xf32>
-    %view_in_tmp = "element_type_cast"(%tmp) : memref<1xvector<3x4x5xf32>>
-    for %i = 0 to 3 {
-      affine.for %j = 0 to 4 {
-        affine.for %k = 0 to 5 {
-          %a = load %A[%expr1 + %k, %expr2, %expr3 + %i, %expr4] :
-            memref<?x?x?x?xf32>
-          store %tmp[%i, %j, %k] : vector<3x4x5xf32>
-    }}}
-    %c0 = constant 0 : index
-    %vec = load %view_in_tmp[%c0] : vector<3x4x5xf32>
-    ```
-
-    On a GPU one could then map `i`, `j`, `k` to blocks and threads. Notice that
-    the temporary storage footprint is `3 * 5` values but `3 * 4 * 5` values are
-    actually transferred between `%A` and `%tmp`.
-
-    Alternatively, if a notional vector broadcast operation were available, the
-    lowered code would resemble:
-
-    ```mlir
-    // %expr1, %expr2, %expr3, %expr4 defined before this point
-    %tmp = alloc() : vector<3x4x5xf32>
-    %view_in_tmp = "element_type_cast"(%tmp) : memref<1xvector<3x4x5xf32>>
-    for %i = 0 to 3 {
-      affine.for %k = 0 to 5 {
-        %a = load %A[%expr1 + %k, %expr2, %expr3 + %i, %expr4] :
-          memref<?x?x?x?xf32>
-        store %tmp[%i, 0, %k] : vector<3x4x5xf32>
-    }}
-    %c0 = constant 0 : index
-    %tmpvec = load %view_in_tmp[%c0] : vector<3x4x5xf32>
-    %vec = broadcast %tmpvec, 1 : vector<3x4x5xf32>
-    ```
-
-    where `broadcast` broadcasts from element 0 to all others along the
-    specified dimension. This time, the temporary storage footprint is `3 * 5`
-    values which is the same amount of data as the `3 * 5` values transferred.
-    An additional `1` broadcast is required. On a GPU this broadcast could be
-    implemented using a warp-shuffle if loop `j` were mapped to `threadIdx.x`.
-
-    Syntax
-    ```
-    operation ::= ssa-id `=` `vector.transfer_read` ssa-use-list
-      `{` attribute-entry `} :` memref-type `,` vector-type
-    ```
-
-    Examples:
-
-    ```mlir
-    // Read the slice `%A[%i0, %i1:%i1+256, %i2:%i2+32]` into vector<32x256xf32>
-    // and pad with %f0 to handle the boundary case:
-    %f0 = constant 0.0f : f32
-    for %i0 = 0 to %0 {
-      affine.for %i1 = 0 to %1 step 256 {
-        affine.for %i2 = 0 to %2 step 32 {
-          %v = vector.transfer_read %A[%i0, %i1, %i2], (%f0)
-               {permutation_map: (d0, d1, d2) -> (d2, d1)} :
-               memref<?x?x?xf32>, vector<32x256xf32>
-    }}}
-
-    // Read the slice `%A[%i0, %i1]` (i.e. the element `%A[%i0, %i1]`) into
-    // vector<128xf32>. The underlying implementation will require a 1-D vector
-    // broadcast:
-    for %i0 = 0 to %0 {
-      affine.for %i1 = 0 to %1 {
-        %3 = vector.transfer_read %A[%i0, %i1]
-             {permutation_map: (d0, d1) -> (0)} :
-             memref<?x?xf32>, vector<128xf32>
-      }
-    }
-
-    // Read from a memref with vector element type.
-    %4 = vector.transfer_read %arg1[%c3, %c3], %vf0
-      {permutation_map = (d0, d1)->(d0, d1)}
-        : memref<?x?xvector<4x3xf32>>, vector<1x1x4x3xf32>
-    ```
-  }];
-
-  let extraClassDeclaration = [{
-    MemRefType getMemRefType() {
-      return memref().getType().cast<MemRefType>();
-    }
-    VectorType getVectorType() {
-      return vector().getType().cast<VectorType>();
-    }
-  }];
-}
-
-def Vector_TransferWriteOp :
-  Vector_Op<"transfer_write">,
-    Arguments<(ins AnyVector:$vector, AnyMemRef:$memref,
-               Variadic<Index>:$indices,
-               AffineMapAttr:$permutation_map)> {
-
-  let summary = "The vector.transfer_write op writes a supervector to memory.";
-
-  let description = [{
-    The `vector.transfer_write` performs a blocking write from a
-    [vector](../LangRef.md#vector-type), supplied as its first operand, into a
-    slice within a [MemRef](../LangRef.md#memref-type) of the same base
-    elemental type, supplied as its second operand.
-
-    A vector memref operand must have its vector element type match a suffix
-    (shape and element type) of the vector (e.g. memref<3x2x6x4x3xf32>,
-    vector<1x1x4x3xf32>).
-
-    The slice is further defined by a full-rank index within the MemRef,
-    supplied as the operands `3 .. 2 + rank(memref)`.
-    The permutation_map [attribute](../LangRef.md#attributes) is an
-    [affine-map](Affine.md#affine-maps) which specifies the transposition on the
-    slice to match the vector shape. The size of the slice is specified by the
-    size of the vector. This operation is called 'write' by opposition to
-    'store' because the super-vector granularity is generally not representable
-    with a single hardware register. A `vector.transfer_write` is thus a
-    mid-level abstraction that supports super-vectorization with non-effecting
-    padding for full-tile-only code. It is the responsibility of
-    `vector.transfer_write`'s implementation to ensure the memory writes are
-    valid. Different lowerings may be pertinent depending on the hardware
-    support.
-
-    Syntax:
-
-    ```
-    operation ::= `vector.transfer_write` ssa-use-list `{` attribute-entry `} :
-      ` vector-type ', ' memref-type '
-    ```
-
-    Examples:
-
-    ```mlir
-    // write vector<16x32x64xf32> into the slice
-    //   `%A[%i0, %i1:%i1+32, %i2:%i2+64, %i3:%i3+16]`:
-    for %i0 = 0 to %0 {
-      affine.for %i1 = 0 to %1 step 32 {
-        affine.for %i2 = 0 to %2 step 64 {
-          affine.for %i3 = 0 to %3 step 16 {
-            %val = `ssa-value` : vector<16x32x64xf32>
-            vector.transfer_write %val, %A[%i0, %i1, %i2, %i3]
-              {permutation_map: (d0, d1, d2, d3) -> (d3, d1, d2)} :
-              vector<16x32x64xf32>, memref<?x?x?x?xf32>
-    }}}}
-
-    // write to a memref with vector element type.
-    vector.transfer_write %4, %arg1[%c3, %c3]
-      {permutation_map = (d0, d1)->(d0, d1)}
-        : vector<1x1x4x3xf32>, memref<?x?xvector<4x3xf32>>
-    ```
-  }];
-
-  let extraClassDeclaration = [{
-    VectorType getVectorType() {
-      return vector().getType().cast<VectorType>();
-    }
-    MemRefType getMemRefType() {
-      return memref().getType().cast<MemRefType>();
-    }
-  }];
-  let assemblyFormat = [{
-    $vector `,` $memref `[` $indices `]` attr-dict `:` type($vector) `,`
-    type($memref)
-  }];
-}
-
-def Vector_ShapeCastOp :
-  Vector_Op<"shape_cast", [NoSideEffect]>,
-    Arguments<(ins AnyTypeOf<[AnyVector, TupleOf<[AnyVector]>]>:$source)>,
-    Results<(outs AnyTypeOf<[AnyVector, TupleOf<[AnyVector]>]>:$result)> {
-  let summary = "shape_cast casts between vector shapes";
-  let description = [{
-    The shape_cast operation casts between an n-D source vector shape and
-    a k-D result vector shape (the element type remains the same).
-
-    If reducing rank (n > k), result dimension sizes must be a product
-    of contiguous source dimension sizes.
-    If expanding rank (n < k), source dimensions must factor into a
-    contiguous sequence of destination dimension sizes.
-    Each source dim is expanded (or contiguous sequence of source dims combined)
-    in source dimension list order (i.e. 0 <= i < n), to produce a contiguous
-    sequence of result dims (or a single result dim), in result dimension list
-    order (i.e. 0 <= j < k). The product of all source dimension sizes and all
-    result dimension sizes must match.
-
-    If the source/result types are a tuple of vectors, the casting operation
-    described above is applied to each source/result tuple element pair.
-
-    It is currently assumed that this operation does not require moving data,
-    and that it will be folded away before lowering vector operations.
-
-    There is an exception to the folding expectation when targeting
-    llvm.intr.matrix operations. We need a type conversion back and forth from a
-    2-D MLIR vector to a 1-D flattened LLVM vector.shape_cast lowering to LLVM
-    is supported in that particular case, for now.
-
-    Examples:
-
-    ```mlir
-    // Example casting to a lower vector rank.
-    %1 = vector.shape_cast %0 : vector<5x1x4x3xf32> to vector<20x3xf32>
-
-    // Example casting to a higher vector rank.
-    %3 = vector.shape_cast %2 : vector<10x12x8xf32> to vector<5x2x3x4x8xf32>
-
-    // Example casting a tuple of vectors of same rank, where tuple elements
-    // may have different shapes.
-    %5 = vector.shape_cast %4 : tuple<vector<3x4x2xf32>, vector<3x3x2xf32>> to
-                                tuple<vector<12x2xf32>, vector<9x2xf32>>
-    ```
-  }];
-  let extraClassDeclaration = [{
-    VectorType getSourceVectorType() {
-      return source().getType().cast<VectorType>();
-    }
-    VectorType getResultVectorType() {
-      return getResult().getType().cast<VectorType>();
-    }
-  }];
-  let assemblyFormat = "$source attr-dict `:` type($source) `to` type($result)";
-}
-
-def Vector_TypeCastOp :
-  Vector_Op<"type_cast", [NoSideEffect]>,
-    Arguments<(ins StaticShapeMemRefOf<[AnyType]>:$memref)>,
-    Results<(outs AnyMemRef)> {
-  let summary = "type_cast op converts a scalar memref to a vector memref";
-  let description = [{
-    Performs a conversion from a memref with scalar element to a memref with a
-    *single* vector element, copying the shape of the memref to the vector. This
-    is the minimal viable operation that is required to makeke
-    super-vectorization operational. It can be seen as a special case of the
-    `view` operation but scoped in the super-vectorization context.
-
-    Syntax:
-
-    ```
-    operation ::= `vector.type_cast` ssa-use : memref-type to memref-type
-    ```
-
-    Example:
-
-    ```mlir
-    %A  = alloc() : memref<5x4x3xf32>
-    %VA = vector.type_cast %A : memref<5x4x3xf32> to memref<vector<5x4x3xf32>>
-    ```
-  }];
-
-  let builders = [OpBuilder<
-    "Builder *builder, OperationState &result, Value source">];
-
-  let parser = [{
-    return impl::parseCastOp(parser, result);
-  }];
-
-  let extraClassDeclaration = [{
-    MemRefType getMemRefType() {
-      return memref().getType().cast<MemRefType>();
-    }
-    MemRefType getResultMemRefType() {
-      return getResult().getType().cast<MemRefType>();
-    }
-  }];
-}
-
-def Vector_ConstantMaskOp :
-  Vector_Op<"constant_mask", [NoSideEffect]>,
-    Arguments<(ins I64ArrayAttr:$mask_dim_sizes)>,
-    Results<(outs VectorOf<[I1]>)> {
-  let summary = "creates a constant vector mask";
-  let description = [{
-    Creates and returns a vector mask where elements of the result vector
-    are set to '0' or '1', based on whether the element indices are contained
-    within a hyper-rectangular region specified by the 'mask_dim_sizes'
-    array attribute argument. Each element of the 'mask_dim_sizes' array,
-    specifies an exclusive upper bound [0, mask-dim-size-element-value)
-    for a unique dimension in the vector result. The conjunction of the ranges
-    define a hyper-rectangular region within which elements values are set to 1
-    (otherwise element values are set to 0).
-
-    Example: create a constant vector mask of size 4x3xi1 with elements in range
-             0 <= row <= 2 and 0 <= col <= 1 are set to 1 (others to 0).
-
-      %1 = vector.constant_mask [3, 2] : vector<4x3xi1>
-
-      print %1
-                    columns
-                  0    1    2
-                |------------
-              0 | 1    1    0
-        rows  1 | 1    1    0
-              2 | 1    1    0
-              3 | 0    0    0
-  }];
-
-  let extraClassDeclaration = [{
-    static StringRef getMaskDimSizesAttrName() { return "mask_dim_sizes"; }
-  }];
-  let assemblyFormat = "$mask_dim_sizes attr-dict `:` type(results)";
-}
-
-def Vector_CreateMaskOp :
-  Vector_Op<"create_mask", [NoSideEffect]>,
-    Arguments<(ins Variadic<Index>:$operands)>, Results<(outs VectorOf<[I1]>)> {
-  let summary = "creates a vector mask";
-  let description = [{
-    Creates and returns a vector mask where elements of the result vector
-    are set to '0' or '1', based on whether the element indices are contained
-    within a hyper-rectangular region specified by the operands. Specifically,
-    each operand specifies a range [0, operand-value) for a unique dimension in
-    the vector result. The conjunction of the operand ranges define a
-    hyper-rectangular region within which elements values are set to 1
-    (otherwise element values are set to 0).
-
-    Example: create a vector mask of size 4x3xi1 where elements in range
-             0 <= row <= 2 and 0 <= col <= 1 are set to 1 (others to 0).
-
-      %1 = vector.create_mask %c3, %c2 : vector<4x3xi1>
-
-      print %1
-                    columns
-                  0    1    2
-                |------------
-              0 | 1    1    0
-        rows  1 | 1    1    0
-              2 | 1    1    0
-              3 | 0    0    0
-  }];
-
-  let hasCanonicalizer = 1;
-  let assemblyFormat = "$operands attr-dict `:` type(results)";
-}
-
-def Vector_TupleOp :
-  Vector_Op<"tuple", [NoSideEffect]>,
-    Arguments<(ins Variadic<AnyVector>:$vectors)>,
-    Results<(outs TupleOf<[AnyVector]>)> {
-  let summary = "make tuple of vectors operation";
-  let description = [{
-    Returns a tuple of its operands 'vectors'.
-
-    Note that this operation is used during the vector op unrolling
-    transformation and should be removed before lowering to lower-level
-    dialects.
-
-
-    Examples:
-    ```
-      %0 = vector.transfer_read ... : vector<2x2xf32>
-      %1 = vector.transfer_read ... : vector<2x1xf32>
-      %2 = vector.transfer_read ... : vector<2x2xf32>
-      %3 = vector.transfer_read ... : vector<2x1xf32>
-
-      %4 = vector.tuple %0, %1, %2, %3
-        : vector<2x2xf32>, vector<2x1xf32>, vector<2x2xf32>, vector<2x1xf32>
-
-    ```
-  }];
-
-  let extraClassDeclaration = [{
-    TupleType getResultTupleType() {
-      return getResult().getType().cast<TupleType>();
-    }
-  }];
-}
-
-def Vector_TupleGetOp :
-  Vector_Op<"tuple_get", [NoSideEffect]>,
-    Arguments<(ins TupleOf<[AnyVector]>:$vectors, APIntAttr:$index)>,
-    Results<(outs AnyVector)> {
-  let summary = "vector tuple get operation";
-  let description = [{
-    Returns the tuple element of 'vectors' at 'index'.
-
-    Note that this operation is used during the vector op unrolling
-    transformation and should be removed before lowering to lower-level
-    dialects.
-
-    Examples:
-    ```
-      %4 = vector.tuple %0, %1, %2, %3
-        : vector<2x2xf32>, vector<2x1xf32>, vector<2x2xf32>, vector<2x1xf32>>
-
-      %5 = vector.tuple_get %4, 1
-        : tuple<vector<2x2xf32>, vector<2x1xf32>,
-                vector<2x2xf32>, vector<2x1xf32>>
-    ```
-  }];
-
-  let extraClassDeclaration = [{
-    VectorType getResultVectorType() {
-      return getResult().getType().cast<VectorType>();
-    }
-    int64_t getIndex() {
-      return getAttrOfType<IntegerAttr>("index").getValue().getSExtValue();
-    }
-    static StringRef getIndexAttrName() { return "index"; }
-  }];
-  let hasFolder = 1;
-}
-
-def Vector_PrintOp :
-  Vector_Op<"print", []>, Arguments<(ins AnyType:$source)> {
-  let summary = "print operation (for testing and debugging)";
-  let description = [{
-    Prints the source vector (or scalar) to stdout in human readable
-    format (for testing and debugging). No return value.
-
-    Examples:
-    ```
-      %0 = constant 0.0 : f32
-      %1 = vector.broadcast %0 : f32 to vector<4xf32>
-      vector.print %1 : vector<4xf32>
-
-      when lowered to LLVM, the vector print is unrolled into
-      elementary printing method calls that at runtime will yield
-
-      ( 0.0, 0.0, 0.0, 0.0 )
-
-      on stdout when linked with a small runtime support library,
-      which only needs to provide a few printing methods (single
-      value for all data types, opening/closing bracket, comma,
-      newline).
-    ```
-  }];
-  let verifier = ?;
-  let extraClassDeclaration = [{
-    Type getPrintType() {
-      return source().getType();
-    }
-  }];
-  let assemblyFormat = "$source attr-dict `:` type($source)";
-}
-
-//===----------------------------------------------------------------------===//
-// Ops used for supporting progressive lowering and conversion type changes.
-//===----------------------------------------------------------------------===//
-
-/// Vector dialect matrix multiplication op that operates on flattened 1-D
-/// MLIR vectors. This is the counterpart of llvm.matrix.multiply in MLIR.
-/// This may seem redundant with vector.contract but it serves the purposes of
-/// more progressive lowering and localized type conversion on the path:
-///   `vector<...x...xf32> -> vector<...xf32> -> !llvm<... x float>`.
-def Vector_MatmulOp : Vector_Op<"matrix_multiply", [NoSideEffect,
-        PredOpTrait<"lhs operand and result have same element type",
-                    TCresVTEtIsSameAsOpBase<0, 0>>,
-        PredOpTrait<"rhs operand and result have same element type",
-                    TCresVTEtIsSameAsOpBase<0, 1>>]>,
-      Arguments<(
-        // TODO(ntv, fhahn): tighten vector element types that make sense.
-        ins VectorOfRankAndType<[1],
-              [AnySignlessInteger, AnySignedInteger, AnyFloat]>:$lhs,
-            VectorOfRankAndType<[1],
-              [AnySignlessInteger, AnySignedInteger, AnyFloat]>:$rhs,
-            I32Attr:$lhs_rows, I32Attr:$lhs_columns, I32Attr:$rhs_columns)>,
-      Results<(
-        outs VectorOfRankAndType<[1],
-               [AnySignlessInteger, AnySignedInteger, AnyFloat]>:$res)>
-{
-  let summary = "Vector matrix multiplication op that operates on flattened 1-D"
-    " MLIR vectors";
-  let description = [{
-    This is the counterpart of llvm.matrix.multiply in MLIR. It serves the
-    purposes of more progressive lowering and localized type conversion.
-
-    The ‘vector.matrix_multiply’ op treats `lhs` as matrix with <lhs_rows> rows
-    and <lhs_columns> columns, `rhs` as matrix with <lhs_columns> rows and
-    <rhs_columns> and multiplies them. The result matrix is returned embedded in
-    the result vector.
-
-    Example:
-
-    ```
-      %C = vector.matrix_multiply %A, %B
-        { lhs_rows = 4: i32, lhs_columns = 16: i32 , rhs_columns = 3: i32 } :
-        (vector<64xf64>, vector<48xf64>) -> vector<12xf64>
-    ```
-  }];
-  let builders = [
-   OpBuilder<"Builder *builder, OperationState &result, Value lhs, Value rhs, "
-             "unsigned lhsRows, unsigned lhsColumns, unsigned rhsColumns",
-   [{
-     result.addOperands({lhs, rhs});
-     result.addAttribute("lhs_rows", builder->getI32IntegerAttr(lhsRows));
-     result.addAttribute("lhs_columns", builder->getI32IntegerAttr(lhsColumns));
-     result.addAttribute("rhs_columns", builder->getI32IntegerAttr(rhsColumns));
-     result.addTypes(VectorType::get(lhsRows * lhsColumns,
-       lhs.getType().cast<VectorType>().getElementType()));
-   }]>,
-  ];
-  let verifier = ?;
-  let assemblyFormat = "$lhs `,` $rhs attr-dict "
-    "`:` `(` type($lhs) `,` type($rhs) `)` `->` type($res)";
-}
-
-#endif // VECTOR_OPS
diff --git a/mlir/include/mlir/ExecutionEngine/CRunnerUtils.h b/mlir/include/mlir/ExecutionEngine/CRunnerUtils.h
index c1ab7425fec68..020c66e07e854 100644
--- a/mlir/include/mlir/ExecutionEngine/CRunnerUtils.h
+++ b/mlir/include/mlir/ExecutionEngine/CRunnerUtils.h
@@ -20,6 +20,7 @@
 #ifdef mlir_c_runner_utils_EXPORTS
 /* We are building this library */
 #define MLIR_CRUNNERUTILS_EXPORT __declspec(dllexport)
+#define MLIR_CRUNNERUTILS_DEFINE_FUNCTIONS
 #else
 /* We are using this library */
 #define MLIR_CRUNNERUTILS_EXPORT __declspec(dllimport)
@@ -27,6 +28,7 @@
 #endif // MLIR_CRUNNERUTILS_EXPORT
 #else
 #define MLIR_CRUNNERUTILS_EXPORT
+#define MLIR_CRUNNERUTILS_DEFINE_FUNCTIONS
 #endif // _WIN32
 
 #include <cstdint>
diff --git a/mlir/include/mlir/IR/AffineMap.h b/mlir/include/mlir/IR/AffineMap.h
index 14deb85fb2f03..ff65da62a1150 100644
--- a/mlir/include/mlir/IR/AffineMap.h
+++ b/mlir/include/mlir/IR/AffineMap.h
@@ -49,9 +49,16 @@ class AffineMap {
   static AffineMap get(unsigned dimCount, unsigned symbolCount,
                        MLIRContext *context);
 
+  /// Returns an affine map with `dimCount` dimensions and `symbolCount` symbols
+  /// mapping to the given results. The array of results cannot be empty.
   static AffineMap get(unsigned dimCount, unsigned symbolCount,
                        ArrayRef<AffineExpr> results);
 
+  /// Returns an affine map with `dimCount` dimensions and `symbolCount` mapping
+  /// to the given results, where the number of results can be zero.
+  static AffineMap get(unsigned dimCount, unsigned symbolCount,
+                       ArrayRef<AffineExpr> results, MLIRContext *context);
+
   /// Returns a single constant result affine map.
   static AffineMap getConstantMap(int64_t val, MLIRContext *context);
 
@@ -208,9 +215,13 @@ struct MutableAffineMap {
   MLIRContext *context;
 };
 
-/// Simplify an affine map by simplifying its underlying AffineExpr results.
+/// Simplifies an affine map by simplifying its underlying AffineExpr results.
 AffineMap simplifyAffineMap(AffineMap map);
 
+/// Returns a map with the same dimension and symbol count as `map`, but whose
+/// results are the unique affine expressions of `map`.
+AffineMap removeDuplicateExprs(AffineMap map);
+
 /// Returns a map of codomain to domain dimensions such that the first codomain
 /// dimension for a particular domain dimension is selected.
 /// Returns an empty map if the input map is empty or if `map` is not invertible
diff --git a/mlir/include/mlir/IR/Builders.h b/mlir/include/mlir/IR/Builders.h
index 5b42132d463ab..75f49e86d10a2 100644
--- a/mlir/include/mlir/IR/Builders.h
+++ b/mlir/include/mlir/IR/Builders.h
@@ -188,13 +188,23 @@ class OpBuilder : public Builder {
     setInsertionPoint(op);
   }
 
-  explicit OpBuilder(Block *block) : OpBuilder(block, block->end()) {}
-
   OpBuilder(Block *block, Block::iterator insertPoint)
       : OpBuilder(block->getParent()) {
     setInsertionPoint(block, insertPoint);
   }
 
+  /// Create a builder and set the insertion point to before the first operation
+  /// in the block but still inside th block.
+  static OpBuilder atBlockBegin(Block *block) {
+    return OpBuilder(block, block->begin());
+  }
+
+  /// Create a builder and set the insertion point to after the last operation
+  /// in the block but still inside the block.
+  static OpBuilder atBlockEnd(Block *block) {
+    return OpBuilder(block, block->end());
+  }
+
   /// This class represents a saved insertion point.
   class InsertPoint {
   public:
@@ -288,13 +298,15 @@ class OpBuilder : public Builder {
   /// Insert the given operation at the current insertion point and return it.
   virtual Operation *insert(Operation *op);
 
-  /// Add new block and set the insertion point to the end of it. The block is
-  /// inserted at the provided insertion point of 'parent'.
-  Block *createBlock(Region *parent, Region::iterator insertPt = {});
+  /// Add new block with 'argTypes' arguments and set the insertion point to the
+  /// end of it. The block is inserted at the provided insertion point of
+  /// 'parent'.
+  virtual Block *createBlock(Region *parent, Region::iterator insertPt = {},
+                             TypeRange argTypes = llvm::None);
 
-  /// Add new block and set the insertion point to the end of it. The block is
-  /// placed before 'insertBefore'.
-  Block *createBlock(Block *insertBefore);
+  /// Add new block with 'argTypes' arguments and set the insertion point to the
+  /// end of it. The block is placed before 'insertBefore'.
+  Block *createBlock(Block *insertBefore, TypeRange argTypes = llvm::None);
 
   /// Returns the current block of the builder.
   Block *getBlock() const { return block; }
diff --git a/mlir/include/mlir/IR/Diagnostics.h b/mlir/include/mlir/IR/Diagnostics.h
index f366935d61f92..20fd366398d37 100644
--- a/mlir/include/mlir/IR/Diagnostics.h
+++ b/mlir/include/mlir/IR/Diagnostics.h
@@ -547,7 +547,8 @@ class SourceMgrDiagnosticHandler : public ScopedDiagnosticHandler {
   ~SourceMgrDiagnosticHandler();
 
   /// Emit the given diagnostic information with the held source manager.
-  void emitDiagnostic(Location loc, Twine message, DiagnosticSeverity kind);
+  void emitDiagnostic(Location loc, Twine message, DiagnosticSeverity kind,
+                      bool displaySourceLine = true);
 
 protected:
   /// Emit the given diagnostic with the held source manager.
diff --git a/mlir/include/mlir/IR/MLIRContext.h b/mlir/include/mlir/IR/MLIRContext.h
index 1cfeeeee5b2eb..6a65c4e2085ca 100644
--- a/mlir/include/mlir/IR/MLIRContext.h
+++ b/mlir/include/mlir/IR/MLIRContext.h
@@ -49,6 +49,12 @@ class MLIRContext {
     return static_cast<T *>(getRegisteredDialect(T::getDialectNamespace()));
   }
 
+  /// Return true if we allow to create operation for unregistered dialects.
+  bool allowsUnregisteredDialects();
+
+  /// Enables creating operations in unregistered dialects.
+  void allowUnregisteredDialects(bool allow = true);
+
   /// Return information about all registered operations.  This isn't very
   /// efficient: typically you should ask the operations about their properties
   /// directly.
diff --git a/mlir/include/mlir/IR/OpImplementation.h b/mlir/include/mlir/IR/OpImplementation.h
index f471e6ca0cc58..0a26025441121 100644
--- a/mlir/include/mlir/IR/OpImplementation.h
+++ b/mlir/include/mlir/IR/OpImplementation.h
@@ -452,6 +452,9 @@ class OpAsmParser {
   /// Parse a single operand.
   virtual ParseResult parseOperand(OperandType &result) = 0;
 
+  /// Parse a single operand if present.
+  virtual OptionalParseResult parseOptionalOperand(OperandType &result) = 0;
+
   /// These are the supported delimiters around operand lists and region
   /// argument lists, used by parseOperandList and parseRegionArgumentList.
   enum class Delimiter {
diff --git a/mlir/include/mlir/IR/StandardTypes.h b/mlir/include/mlir/IR/StandardTypes.h
index 1c18e21006dc4..b9774eb22de1c 100644
--- a/mlir/include/mlir/IR/StandardTypes.h
+++ b/mlir/include/mlir/IR/StandardTypes.h
@@ -330,7 +330,7 @@ class TensorType : public ShapedType {
     // element type within that dialect.
     return type.isa<ComplexType>() || type.isa<FloatType>() ||
            type.isa<IntegerType>() || type.isa<OpaqueType>() ||
-           type.isa<VectorType>() ||
+           type.isa<VectorType>() || type.isa<IndexType>() ||
            (type.getKind() > Type::Kind::LAST_STANDARD_TYPE);
   }
 
diff --git a/mlir/include/mlir/InitAllPasses.h b/mlir/include/mlir/InitAllPasses.h
index b983ecb478ddb..08d267a9cf18d 100644
--- a/mlir/include/mlir/InitAllPasses.h
+++ b/mlir/include/mlir/InitAllPasses.h
@@ -22,11 +22,13 @@
 #include "mlir/Conversion/GPUToVulkan/ConvertGPUToVulkanPass.h"
 #include "mlir/Conversion/LinalgToLLVM/LinalgToLLVM.h"
 #include "mlir/Conversion/LinalgToSPIRV/LinalgToSPIRVPass.h"
+#include "mlir/Conversion/LoopToStandard/ConvertLoopToStandard.h"
 #include "mlir/Conversion/LoopsToGPU/LoopsToGPUPass.h"
+#include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h"
 #include "mlir/Conversion/StandardToSPIRV/ConvertStandardToSPIRVPass.h"
+#include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h"
 #include "mlir/Dialect/Affine/Passes.h"
 #include "mlir/Dialect/FxpMathOps/Passes.h"
-#include "mlir/Dialect/FxpMathOps/Passes.h"
 #include "mlir/Dialect/GPU/Passes.h"
 #include "mlir/Dialect/LLVMIR/Transforms/LegalizeForExport.h"
 #include "mlir/Dialect/Linalg/Passes.h"
@@ -36,6 +38,8 @@
 #include "mlir/Quantizer/Transforms/Passes.h"
 #include "mlir/Transforms/LocationSnapshot.h"
 #include "mlir/Transforms/Passes.h"
+#include "mlir/Transforms/ViewOpGraph.h"
+#include "mlir/Transforms/ViewRegionGraph.h"
 
 #include <cstdlib>
 
@@ -49,93 +53,47 @@ namespace mlir {
 // individual passes.
 // The global registry is interesting to interact with the command-line tools.
 inline void registerAllPasses() {
-  // At the moment we still rely on global initializers for registering passes,
-  // but we may not do it in the future.
-  // We must reference the passes in such a way that compilers will not
-  // delete it all as dead code, even with whole program optimization,
-  // yet is effectively a NO-OP. As the compiler isn't smart enough
-  // to know that getenv() never returns -1, this will do the job.
-  if (std::getenv("bar") != (char *)-1)
-    return;
-
   // Init general passes
-  createCanonicalizerPass();
-  createCSEPass();
-  createSuperVectorizePass({});
-  createLoopUnrollPass();
-  createLoopUnrollAndJamPass();
-  createSimplifyAffineStructuresPass();
-  createLoopFusionPass();
-  createLoopInvariantCodeMotionPass();
-  createAffineLoopInvariantCodeMotionPass();
-  createPipelineDataTransferPass();
-  createLowerAffinePass();
-  createLoopTilingPass(0);
-  createLoopCoalescingPass();
-  createAffineDataCopyGenerationPass(0, 0);
-  createMemRefDataFlowOptPass();
-  createStripDebugInfoPass();
-  createPrintOpStatsPass();
-  createInlinerPass();
-  createSymbolDCEPass();
-  createLocationSnapshotPass({});
-
-  // AVX512
-  createConvertAVX512ToLLVMPass();
-
-  // GPUtoRODCLPass
-  createLowerGpuOpsToROCDLOpsPass();
-
-  // FxpOpsDialect passes
-  fxpmath::createLowerUniformRealMathPass();
-  fxpmath::createLowerUniformCastsPass();
+#define GEN_PASS_REGISTRATION
+#include "mlir/Transforms/Passes.h.inc"
 
-  // GPU
-  createGpuKernelOutliningPass();
-  createSimpleLoopsToGPUPass(0, 0);
-  createLoopToGPUPass({}, {});
+  // Conversion passes
+#define GEN_PASS_REGISTRATION
+#include "mlir/Conversion/Passes.h.inc"
 
-  // CUDA
-  createConvertGpuLaunchFuncToCudaCallsPass();
-  createLowerGpuOpsToNVVMOpsPass();
+  // Affine
+#define GEN_PASS_REGISTRATION
+#include "mlir/Dialect/Affine/Passes.h.inc"
+
+  // FxpMath
+#define GEN_PASS_REGISTRATION
+#include "mlir/Dialect/FxpMathOps/Passes.h.inc"
+
+  // GPU
+#define GEN_PASS_REGISTRATION
+#include "mlir/Dialect/GPU/Passes.h.inc"
 
   // Linalg
-  createLinalgFusionPass();
-  createLinalgTilingPass();
-  createLinalgTilingToParallelLoopsPass();
-  createLinalgPromotionPass(0);
-  createConvertLinalgToLoopsPass();
-  createConvertLinalgToParallelLoopsPass();
-  createConvertLinalgToAffineLoopsPass();
-  createConvertLinalgToLLVMPass();
+#define GEN_PASS_REGISTRATION
+#include "mlir/Dialect/Linalg/Passes.h.inc"
 
   // LLVM
-  LLVM::createLegalizeForExportPass();
+#define GEN_PASS_REGISTRATION
+#include "mlir/Dialect/LLVMIR/Transforms/Passes.h.inc"
 
-  // LoopOps
-  createParallelLoopFusionPass();
-  createParallelLoopSpecializationPass();
-  createParallelLoopTilingPass();
+  // Loop
+#define GEN_PASS_REGISTRATION
+#include "mlir/Dialect/LoopOps/Passes.h.inc"
 
-  // QuantOps
-  quant::createConvertSimulatedQuantPass();
-  quant::createConvertConstPass();
-  quantizer::createAddDefaultStatsPass();
-  quantizer::createRemoveInstrumentationPass();
-  quantizer::registerInferQuantizedTypesPass();
+  // Quant
+#define GEN_PASS_REGISTRATION
+#include "mlir/Dialect/Quant/Passes.h.inc"
+#define GEN_PASS_REGISTRATION
+#include "mlir/Quantizer/Transforms/Passes.h.inc"
 
   // SPIR-V
-  spirv::createDecorateSPIRVCompositeTypeLayoutPass();
-  spirv::createLowerABIAttributesPass();
-  spirv::createUpdateVersionCapabilityExtensionPass();
-  createConvertGPUToSPIRVPass();
-  createConvertStandardToSPIRVPass();
-  createLegalizeStdOpsForSPIRVLoweringPass();
-  createLinalgToSPIRVPass();
-
-  // Vulkan
-  createConvertGpuLaunchFuncToVulkanLaunchFuncPass();
-  createConvertVulkanLaunchFuncToVulkanCallsPass();
+#define GEN_PASS_REGISTRATION
+#include "mlir/Dialect/SPIRV/Passes.h.inc"
 }
 
 } // namespace mlir
diff --git a/mlir/include/mlir/Interfaces/InferTypeOpInterface.td b/mlir/include/mlir/Interfaces/InferTypeOpInterface.td
index aec8c0a626acf..fb6bdc220c83f 100644
--- a/mlir/include/mlir/Interfaces/InferTypeOpInterface.td
+++ b/mlir/include/mlir/Interfaces/InferTypeOpInterface.td
@@ -100,7 +100,8 @@ def InferShapedTypeOpInterface : OpInterface<"InferShapedTypeOpInterface"> {
     InterfaceMethod<
       /*desc=*/[{Reify the shape computation for the operation.
 
-      Insert operations using the given OpBulder that computes the result shape.
+      Insert operations using the given OpBuilder that computes the result
+      shape.
       }],
       /*retTy=*/"LogicalResult",
       /*methodName=*/"reifyReturnTypeShapes",
diff --git a/mlir/include/mlir/Pass/PassBase.td b/mlir/include/mlir/Pass/PassBase.td
new file mode 100644
index 0000000000000..29478047f7572
--- /dev/null
+++ b/mlir/include/mlir/Pass/PassBase.td
@@ -0,0 +1,85 @@
+//===-- PassBase.td - Base pass definition file ------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains definitions for defining pass registration and other
+// mechanisms.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_PASS_PASSBASE
+#define MLIR_PASS_PASSBASE
+
+//===----------------------------------------------------------------------===//
+// Options
+//===----------------------------------------------------------------------===//
+
+class Option<string varName, string arg, string valueType, string default,
+             string desc, string additionalFlags = ""> {
+  // The name for the C++ option variable.
+  string cppName = varName;
+
+  // The command line argument to use for this option.
+  string argument = arg;
+
+  // The C++ type of the option.
+  string type = valueType;
+
+  // The default value of the option. "" corresponds to no default.
+  string defaultValue = default;
+
+  // A description for this option.
+  string description = desc;
+
+  // A set of additional flags to pass along to the option constructor.
+  string additionalOptFlags = additionalFlags;
+}
+
+class ListOption<string varName, string arg, string valueType,
+                 string desc, string additionalFlags = "">
+  : Option<varName, arg, valueType, /*default=*/"", desc, additionalFlags> {}
+
+//===----------------------------------------------------------------------===//
+// Statistics
+//===----------------------------------------------------------------------===//
+
+class Statistic<string varName, string statName, string desc> {
+  // The C++ variable name for the statistic.
+  string cppName = varName;
+
+  // The displayed name of the statistic, similar to the argument of an option.
+  string name = statName;
+
+  // The description of the statistic.
+  string description = desc;
+}
+
+//===----------------------------------------------------------------------===//
+// Pass
+//===----------------------------------------------------------------------===//
+
+class Pass<string passArg> {
+  // The command line argument of the pass.
+  string argument = passArg;
+
+  // A short 1-line summary of the pass.
+  string summary = "";
+
+  // A human readable description of the pass.
+  string description = "";
+
+  // A C++ constructor call to create an instance of this pass.
+  code constructor = [{}];
+
+  // A set of options provided by this pass.
+  list<Option> options = [];
+
+  // A set of statistics provided by this pass.
+  list<Statistic> statistics = [];
+}
+
+#endif // MLIR_PASS_PASSBASE
diff --git a/mlir/include/mlir/Pass/PassRegistry.h b/mlir/include/mlir/Pass/PassRegistry.h
index fab118d229c6f..31c09c1259192 100644
--- a/mlir/include/mlir/Pass/PassRegistry.h
+++ b/mlir/include/mlir/Pass/PassRegistry.h
@@ -122,7 +122,7 @@ void registerPassPipeline(
 
 /// Register a specific dialect pass allocator function with the system,
 /// typically used through the PassRegistration template.
-void registerPass(StringRef arg, StringRef description, const PassID *passID,
+void registerPass(StringRef arg, StringRef description,
                   const PassAllocatorFunction &function);
 
 /// PassRegistration provides a global initializer that registers a Pass
@@ -138,7 +138,7 @@ void registerPass(StringRef arg, StringRef description, const PassID *passID,
 template <typename ConcretePass> struct PassRegistration {
   PassRegistration(StringRef arg, StringRef description,
                    const PassAllocatorFunction &constructor) {
-    registerPass(arg, description, PassID::getID<ConcretePass>(), constructor);
+    registerPass(arg, description, constructor);
   }
 
   PassRegistration(StringRef arg, StringRef description)
diff --git a/mlir/include/mlir/Quantizer/CMakeLists.txt b/mlir/include/mlir/Quantizer/CMakeLists.txt
new file mode 100644
index 0000000000000..e31af32661164
--- /dev/null
+++ b/mlir/include/mlir/Quantizer/CMakeLists.txt
@@ -0,0 +1 @@
+add_subdirectory(Transforms)
diff --git a/mlir/include/mlir/Quantizer/Transforms/CMakeLists.txt b/mlir/include/mlir/Quantizer/Transforms/CMakeLists.txt
new file mode 100644
index 0000000000000..b828c72243317
--- /dev/null
+++ b/mlir/include/mlir/Quantizer/Transforms/CMakeLists.txt
@@ -0,0 +1,6 @@
+
+set(LLVM_TARGET_DEFINITIONS Passes.td)
+mlir_tablegen(Passes.h.inc -gen-pass-decls)
+add_public_tablegen_target(MLIRQuantizerPassIncGen)
+
+add_mlir_doc(Passes -gen-pass-doc QuantizerPasses ./)
diff --git a/mlir/include/mlir/Quantizer/Transforms/Passes.h b/mlir/include/mlir/Quantizer/Transforms/Passes.h
index 3cd0475d2f8ee..5877dc525a942 100644
--- a/mlir/include/mlir/Quantizer/Transforms/Passes.h
+++ b/mlir/include/mlir/Quantizer/Transforms/Passes.h
@@ -27,9 +27,7 @@ class TargetConfiguration;
 std::unique_ptr<OpPassBase<ModuleOp>>
 createInferQuantizedTypesPass(SolverContext &solverContext,
                               const TargetConfiguration &config);
-
-/// Registers the InferQuantizedTypes pass with the global registry.
-void registerInferQuantizedTypesPass();
+std::unique_ptr<OpPassBase<ModuleOp>> createInferQuantizedTypesPass();
 
 /// Creates a pass which removes any instrumentation and hint ops which have
 /// no effect on final runtime.
diff --git a/mlir/include/mlir/Quantizer/Transforms/Passes.td b/mlir/include/mlir/Quantizer/Transforms/Passes.td
new file mode 100644
index 0000000000000..e89674478b17c
--- /dev/null
+++ b/mlir/include/mlir/Quantizer/Transforms/Passes.td
@@ -0,0 +1,31 @@
+//===-- Passes.td - Quantizer pass definition file ---------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_QUANTIZER_TRANSFORMS_PASSES
+#define MLIR_QUANTIZER_TRANSFORMS_PASSES
+
+include "mlir/Pass/PassBase.td"
+
+def QuantizerAddDefaultStats : Pass<"quantizer-add-default-stats-test"> {
+  let summary = "Add default (dummy) statistics to all ops that can benefit "
+                "from runtime statistics";
+  let constructor = "mlir::quantizer::createAddDefaultStatsPass()";
+}
+
+def QuantizerInferQuantizedTypes : Pass<"quantizer-infer-quantized-types"> {
+  let summary = "Infer quantized types for a module";
+  let constructor = "mlir::quantizer::createInferQuantizedTypesPass()";
+}
+
+def QuantizerRemoveInstrumentation : Pass<"quantizer-remove-instrumentation"> {
+  let summary = "Remove instrumentation and hints which have no effect on "
+                "final execution";
+  let constructor = "mlir::quantizer::createRemoveInstrumentationPass()";
+}
+
+#endif // MLIR_QUANTIZER_TRANSFORMS_PASSES
diff --git a/mlir/include/mlir/Support/MlirOptMain.h b/mlir/include/mlir/Support/MlirOptMain.h
index 7eb6a433fd13d..f235ea3fa520f 100644
--- a/mlir/include/mlir/Support/MlirOptMain.h
+++ b/mlir/include/mlir/Support/MlirOptMain.h
@@ -26,6 +26,6 @@ LogicalResult MlirOptMain(llvm::raw_ostream &os,
                           std::unique_ptr<llvm::MemoryBuffer> buffer,
                           const PassPipelineCLParser &passPipeline,
                           bool splitInputFile, bool verifyDiagnostics,
-                          bool verifyPasses);
+                          bool verifyPasses, bool allowUnregisteredDialects);
 
 } // end namespace mlir
diff --git a/mlir/include/mlir/TableGen/Pass.h b/mlir/include/mlir/TableGen/Pass.h
new file mode 100644
index 0000000000000..d8b5eaa043135
--- /dev/null
+++ b/mlir/include/mlir/TableGen/Pass.h
@@ -0,0 +1,111 @@
+//===- Pass.h - TableGen pass definitions -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TABLEGEN_PASS_H_
+#define MLIR_TABLEGEN_PASS_H_
+
+#include "mlir/Support/LLVM.h"
+#include <vector>
+
+namespace llvm {
+class Record;
+} // end namespace llvm
+
+namespace mlir {
+namespace tblgen {
+//===----------------------------------------------------------------------===//
+// PassOption
+//===----------------------------------------------------------------------===//
+class PassOption {
+public:
+  explicit PassOption(const llvm::Record *def) : def(def) {}
+
+  /// Return the name for the C++ option variable.
+  StringRef getCppVariableName() const;
+
+  /// Return the command line argument to use for this option.
+  StringRef getArgument() const;
+
+  /// Return the C++ type of the option.
+  StringRef getType() const;
+
+  /// Return the default value of the option.
+  Optional<StringRef> getDefaultValue() const;
+
+  /// Return the description for this option.
+  StringRef getDescription() const;
+
+  /// Return the additional flags passed to the option constructor.
+  Optional<StringRef> getAdditionalFlags() const;
+
+  /// Flag indicating if this is a list option.
+  bool isListOption() const;
+
+private:
+  const llvm::Record *def;
+};
+
+//===----------------------------------------------------------------------===//
+// PassStatistic
+//===----------------------------------------------------------------------===//
+class PassStatistic {
+public:
+  explicit PassStatistic(const llvm::Record *def) : def(def) {}
+
+  /// Return the name for the C++ statistic variable.
+  StringRef getCppVariableName() const;
+
+  /// Return the name of the statistic.
+  StringRef getName() const;
+
+  /// Return the description for this statistic.
+  StringRef getDescription() const;
+
+private:
+  const llvm::Record *def;
+};
+
+//===----------------------------------------------------------------------===//
+// Pass
+//===----------------------------------------------------------------------===//
+
+/// Wrapper class providing helper methods for Passes defined in TableGen.
+class Pass {
+public:
+  explicit Pass(const llvm::Record *def);
+
+  /// Return the command line argument of the pass.
+  StringRef getArgument() const;
+
+  /// Return the short 1-line summary of the pass.
+  StringRef getSummary() const;
+
+  /// Return the description of the pass.
+  StringRef getDescription() const;
+
+  /// Return the C++ constructor call to create an instance of this pass.
+  StringRef getConstructor() const;
+
+  /// Return the options provided by this pass.
+  ArrayRef<PassOption> getOptions() const;
+
+  /// Return the statistics provided by this pass.
+  ArrayRef<PassStatistic> getStatistics() const;
+
+  const llvm::Record *getDef() const { return def; }
+
+private:
+  const llvm::Record *def;
+  std::vector<PassOption> options;
+  std::vector<PassStatistic> statistics;
+};
+
+} // end namespace tblgen
+} // end namespace mlir
+
+#endif // MLIR_TABLEGEN_PASS_H_
diff --git a/mlir/include/mlir/Transforms/CMakeLists.txt b/mlir/include/mlir/Transforms/CMakeLists.txt
new file mode 100644
index 0000000000000..706193188eddd
--- /dev/null
+++ b/mlir/include/mlir/Transforms/CMakeLists.txt
@@ -0,0 +1,6 @@
+
+set(LLVM_TARGET_DEFINITIONS Passes.td)
+mlir_tablegen(Passes.h.inc -gen-pass-decls)
+add_public_tablegen_target(MLIRTransformsPassIncGen)
+
+add_mlir_doc(Passes -gen-pass-doc GeneralPasses ./)
diff --git a/mlir/include/mlir/Transforms/DialectConversion.h b/mlir/include/mlir/Transforms/DialectConversion.h
index 776007347c5e7..9ab3a715e0abe 100644
--- a/mlir/include/mlir/Transforms/DialectConversion.h
+++ b/mlir/include/mlir/Transforms/DialectConversion.h
@@ -344,6 +344,10 @@ class ConversionPatternRewriter final : public PatternRewriter {
   /// otherwise an assert will be issued.
   void eraseOp(Operation *op) override;
 
+  /// PatternRewriter hook for creating a new block with the given arguments.
+  Block *createBlock(Region *parent, Region::iterator insertPt = {},
+                     TypeRange argTypes = llvm::None) override;
+
   /// PatternRewriter hook for splitting a block into two parts.
   Block *splitBlock(Block *block, Block::iterator before) override;
 
diff --git a/mlir/include/mlir/Transforms/LocationSnapshot.h b/mlir/include/mlir/Transforms/LocationSnapshot.h
index e4c911ae07d67..9012494cd46fc 100644
--- a/mlir/include/mlir/Transforms/LocationSnapshot.h
+++ b/mlir/include/mlir/Transforms/LocationSnapshot.h
@@ -58,6 +58,8 @@ LogicalResult generateLocationsFromIR(StringRef fileName, StringRef tag,
 std::unique_ptr<Pass> createLocationSnapshotPass(OpPrintingFlags flags,
                                                  StringRef fileName = "",
                                                  StringRef tag = "");
+/// Overload utilizing pass options for initialization.
+std::unique_ptr<Pass> createLocationSnapshotPass();
 
 } // end namespace mlir
 
diff --git a/mlir/include/mlir/Transforms/LoopUtils.h b/mlir/include/mlir/Transforms/LoopUtils.h
index 479b7ce81a42c..3bc34f1444e05 100644
--- a/mlir/include/mlir/Transforms/LoopUtils.h
+++ b/mlir/include/mlir/Transforms/LoopUtils.h
@@ -24,10 +24,12 @@ class AffineForOp;
 class FuncOp;
 class OpBuilder;
 class Value;
+class ValueRange;
 struct MemRefRegion;
 
 namespace loop {
 class ForOp;
+class ParallelOp;
 } // end namespace loop
 
 /// Unrolls this for operation completely if the trip count is known to be
@@ -70,29 +72,22 @@ LogicalResult promoteIfSingleIteration(AffineForOp forOp);
 /// their body into the containing Block.
 void promoteSingleIterationLoops(FuncOp f);
 
-/// Computes the cleanup loop lower bound of the loop being unrolled with
-/// the specified unroll factor; this bound will also be upper bound of the main
-/// part of the unrolled loop. Computes the bound as an AffineMap with its
-/// operands or a null map when the trip count can't be expressed as an affine
-/// expression.
-void getCleanupLoopLowerBound(AffineForOp forOp, unsigned unrollFactor,
-                              AffineMap *map, SmallVectorImpl<Value> *operands,
-                              OpBuilder &builder);
-
-/// Skew the operations in the body of an affine.for operation with the
-/// specified operation-wise shifts. The shifts are with respect to the
-/// original execution order, and are multiplied by the loop 'step' before being
-/// applied. If `unrollPrologueEpilogue` is set, fully unroll the prologue and
-/// epilogue loops when possible.
+/// Skew the operations in an affine.for's body with the specified
+/// operation-wise shifts. The shifts are with respect to the original execution
+/// order, and are multiplied by the loop 'step' before being applied. If
+/// `unrollPrologueEpilogue` is set, fully unroll the prologue and epilogue
+/// loops when possible.
 LLVM_NODISCARD
 LogicalResult affineForOpBodySkew(AffineForOp forOp, ArrayRef<uint64_t> shifts,
                                   bool unrollPrologueEpilogue = false);
 
 /// Tiles the specified band of perfectly nested loops creating tile-space loops
-/// and intra-tile loops. A band is a contiguous set of loops.
+/// and intra-tile loops. A band is a contiguous set of loops. `tiledNest` when
+/// non-null is set to the loops of the tiled nest from outermost to innermost.
 LLVM_NODISCARD
 LogicalResult tileCodeGen(MutableArrayRef<AffineForOp> band,
-                          ArrayRef<unsigned> tileSizes);
+                          ArrayRef<unsigned> tileSizes,
+                          SmallVectorImpl<AffineForOp> *tiledNest = nullptr);
 
 /// Performs loop interchange on 'forOpA' and 'forOpB'. Requires that 'forOpA'
 /// and 'forOpB' are part of a perfectly nested sequence of loops.
@@ -104,11 +99,15 @@ void interchangeLoops(AffineForOp forOpA, AffineForOp forOpB);
 bool isValidLoopInterchangePermutation(ArrayRef<AffineForOp> loops,
                                        ArrayRef<unsigned> loopPermMap);
 
-/// Performs a sequence of loop interchanges on perfectly nested 'loops', as
-/// specified by permutation 'loopPermMap' (loop 'i' in 'loops' is mapped to
-/// location 'j = 'loopPermMap[i]' after the loop interchange).
-unsigned interchangeLoops(ArrayRef<AffineForOp> loops,
-                          ArrayRef<unsigned> loopPermMap);
+/// Performs a loop permutation on a perfectly nested loop nest `inputNest`
+/// (where the contained loops appear from outer to inner) as specified by the
+/// permutation `permMap`: loop 'i' in `inputNest` is mapped to location
+/// 'loopPermMap[i]', where positions 0, 1, ... are from the outermost position
+/// to inner. Returns the position in `inputNest` of the AffineForOp that
+/// becomes the new outermost loop of this nest. This method always succeeds,
+/// asserts out on invalid input / specifications.
+unsigned permuteLoops(MutableArrayRef<AffineForOp> inputNest,
+                      ArrayRef<unsigned> permMap);
 
 // Sinks all sequential loops to the innermost levels (while preserving
 // relative order among them) and moves all parallel loops to the
@@ -116,11 +115,6 @@ unsigned interchangeLoops(ArrayRef<AffineForOp> loops,
 // Returns AffineForOp of the root of the new loop nest after loop interchanges.
 AffineForOp sinkSequentialLoops(AffineForOp forOp);
 
-/// Sinks 'forOp' by 'loopDepth' levels by performing a series of loop
-/// interchanges. Requires that 'forOp' is part of a perfect nest with
-/// 'loopDepth' AffineForOps consecutively nested under it.
-void sinkLoop(AffineForOp forOp, unsigned loopDepth);
-
 /// Performs tiling fo imperfectly nested loops (with interchange) by
 /// strip-mining the `forOps` by `sizes` and sinking them, in their order of
 /// occurrence in `forOps`, under each of the `targets`.
@@ -226,6 +220,12 @@ TileLoops extractFixedOuterLoops(loop::ForOp rootFOrOp,
 /// independent of any loop induction variable involved in the nest.
 void coalesceLoops(MutableArrayRef<loop::ForOp> loops);
 
+/// Take the ParallelLoop and for each set of dimension indices, combine them
+/// into a single dimension. combinedDimensions must contain each index into
+/// loops exactly once.
+void collapseParallelLoops(loop::ParallelOp loops,
+                           ArrayRef<std::vector<unsigned>> combinedDimensions);
+
 /// Maps `forOp` for execution on a parallel grid of virtual `processorIds` of
 /// size given by `numProcessors`. This is achieved by embedding the SSA values
 /// corresponding to `processorIds` and `numProcessors` into the bounds and step
@@ -264,6 +264,29 @@ void mapLoopToProcessorIds(loop::ForOp forOp, ArrayRef<Value> processorId,
 void gatherLoops(FuncOp func,
                  std::vector<SmallVector<AffineForOp, 2>> &depthToLoops);
 
+/// Creates an AffineForOp while ensuring that the lower and upper bounds are
+/// canonicalized, i.e., unused and duplicate operands are removed, any constant
+/// operands propagated/folded in, and duplicate bound maps dropped.
+AffineForOp createCanonicalizedAffineForOp(OpBuilder b, Location loc,
+                                           ValueRange lbOperands,
+                                           AffineMap lbMap,
+                                           ValueRange ubOperands,
+                                           AffineMap ubMap, int64_t step = 1);
+
+/// Separates full tiles from partial tiles for a perfect nest `nest` by
+/// generating a conditional guard that selects between the full tile version
+/// and the partial tile version using an AffineIfOp. The original loop nest
+/// is replaced by this guarded two version form.
+///
+///    affine.if (cond)
+///      // full_tile
+///    else
+///      // partial tile
+///
+LogicalResult
+separateFullTiles(MutableArrayRef<AffineForOp> nest,
+                  SmallVectorImpl<AffineForOp> *fullTileNest = nullptr);
+
 } // end namespace mlir
 
 #endif // MLIR_TRANSFORMS_LOOP_UTILS_H
diff --git a/mlir/include/mlir/Transforms/Passes.h b/mlir/include/mlir/Transforms/Passes.h
index 28b29eb29ad49..d9aee6e1f2c21 100644
--- a/mlir/include/mlir/Transforms/Passes.h
+++ b/mlir/include/mlir/Transforms/Passes.h
@@ -57,6 +57,10 @@ std::unique_ptr<OpPassBase<FuncOp>> createLowerAffinePass();
 /// bounds into a single loop.
 std::unique_ptr<OpPassBase<FuncOp>> createLoopCoalescingPass();
 
+/// Creates a pass that transforms a single ParallelLoop over N induction
+/// variables into another ParallelLoop over less than N induction variables.
+std::unique_ptr<Pass> createParallelLoopCollapsingPass();
+
 /// Creates a pass to perform optimizations relying on memref dataflow such as
 /// store to load forwarding, elimination of dead stores, and dead allocs.
 std::unique_ptr<OpPassBase<FuncOp>> createMemRefDataFlowOptPass();
@@ -64,7 +68,7 @@ std::unique_ptr<OpPassBase<FuncOp>> createMemRefDataFlowOptPass();
 /// Creates a pass to strip debug information from a function.
 std::unique_ptr<Pass> createStripDebugInfoPass();
 
-/// Creates a pass which prints the list of ops and the number of occurences in
+/// Creates a pass which prints the list of ops and the number of occurrences in
 /// the module.
 std::unique_ptr<OpPassBase<ModuleOp>> createPrintOpStatsPass();
 
diff --git a/mlir/include/mlir/Transforms/Passes.td b/mlir/include/mlir/Transforms/Passes.td
new file mode 100644
index 0000000000000..0873004532800
--- /dev/null
+++ b/mlir/include/mlir/Transforms/Passes.td
@@ -0,0 +1,220 @@
+//===-- Passes.td - Transforms pass definition file --------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains definitions for passes within the Transforms/ directory.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TRANSFORMS_PASSES
+#define MLIR_TRANSFORMS_PASSES
+
+include "mlir/Pass/PassBase.td"
+
+def AffinePipelineDataTransfer : Pass<
+    "affine-pipeline-data-transfer"> {
+  let summary = "Pipeline non-blocking data transfers between explicitly "
+                "managed levels of the memory hierarchy";
+  let description = [{
+    This pass performs a transformation to overlap non-blocking DMA operations
+    in a loop with computations through double buffering. This is achieved by
+    advancing dma_start operations with respect to other operations.
+
+    Input
+
+    ```mlir
+    func @pipelinedatatransfer() {
+      %0 = alloc() : memref<256xf32>
+      %1 = alloc() : memref<32xf32, 1>
+      %2 = alloc() : memref<1xf32>
+      %c0 = constant 0 : index
+      %c128 = constant 128 : index
+      affine.for %i0 = 0 to 8 {
+        affine.dma_start %0[%i0], %1[%i0], %2[%c0], %c128 : memref<256xf32>, memref<32xf32, 1>, memref<1xf32>
+        affine.dma_wait %2[%c0], %c128 : memref<1xf32>
+        %3 = affine.load %1[%i0] : memref<32xf32, 1>
+        %4 = "compute"(%3) : (f32) -> f32
+        affine.store %4, %1[%i0] : memref<32xf32, 1>
+      }
+      return
+    }
+    ```
+
+    Output
+
+    ```mlir
+    module {
+      func @pipelinedatatransfer() {
+        %c8 = constant 8 : index
+        %c0 = constant 0 : index
+        %0 = alloc() : memref<256xf32>
+        %c0_0 = constant 0 : index
+        %c128 = constant 128 : index
+        %1 = alloc() : memref<2x32xf32, 1>
+        %2 = alloc() : memref<2x1xf32>
+        affine.dma_start %0[%c0], %1[%c0 mod 2, %c0], %2[%c0 mod 2, symbol(%c0_0)], %c128 : memref<256xf32>, memref<2x32xf32, 1>, memref<2x1xf32>
+        affine.for %arg0 = 1 to 8 {
+          affine.dma_start %0[%arg0], %1[%arg0 mod 2, %arg0], %2[%arg0 mod 2, symbol(%c0_0)], %c128 : memref<256xf32>, memref<2x32xf32, 1>, memref<2x1xf32>
+          %8 = affine.apply #map3(%arg0)
+          %9 = affine.apply #map4(%8)
+          %10 = affine.apply #map4(%8)
+          affine.dma_wait %2[%8 mod 2, symbol(%c0_0)], %c128 : memref<2x1xf32>
+          %11 = affine.load %1[%8 mod 2, %8] : memref<2x32xf32, 1>
+          %12 = "compute"(%11) : (f32) -> f32
+          affine.store %12, %1[%8 mod 2, %8] : memref<2x32xf32, 1>
+        }
+        %3 = affine.apply #map3(%c8)
+        %4 = affine.apply #map4(%3)
+        %5 = affine.apply #map4(%3)
+        affine.dma_wait %2[%3 mod 2, symbol(%c0_0)], %c128 : memref<2x1xf32>
+        %6 = affine.load %1[%3 mod 2, %3] : memref<2x32xf32, 1>
+        %7 = "compute"(%6) : (f32) -> f32
+        affine.store %7, %1[%3 mod 2, %3] : memref<2x32xf32, 1>
+        dealloc %2 : memref<2x1xf32>
+        dealloc %1 : memref<2x32xf32, 1>
+        return
+      }
+    }
+    ```
+  }];
+  let constructor = "mlir::createPipelineDataTransferPass()";
+}
+
+def AffineLoopFusion : Pass<"affine-loop-fusion"> {
+  let summary = "Fuse affine loop nests";
+  let constructor = "mlir::createLoopFusionPass()";
+}
+
+def Canonicalizer : Pass<"canonicalize"> {
+  let summary = "Canonicalize operations";
+  let constructor = "mlir::createCanonicalizerPass()";
+}
+
+def CSE : Pass<"cse"> {
+  let summary = "Eliminate common sub-expressions";
+  let constructor = "mlir::createCSEPass()";
+  let statistics = [
+    Statistic<"numCSE", "num-cse'd", "Number of operations CSE'd">,
+    Statistic<"numDCE", "num-dce'd", "Number of operations DCE'd">
+  ];
+}
+
+def Inliner : Pass<"inline"> {
+  let summary = "Inline function calls";
+  let constructor = "mlir::createInlinerPass()";
+}
+
+def LocationSnapshot : Pass<"snapshot-op-locations"> {
+  let summary = "Generate new locations from the current IR";
+  let constructor = "mlir::createLocationSnapshotPass()";
+  let options = [
+    Option<"fileName", "filename", "std::string", /*default=*/"",
+           "The filename to print the generated IR.">,
+    Option<"tag", "tag", "std::string", /*default=*/"",
+           "A tag to use when fusing the new locations with the "
+           "original. If unset, the locations are replaced.">,
+  ];
+}
+
+def LoopCoalescing : Pass<"loop-coalescing"> {
+  let summary = "Coalesce nested loops with independent bounds into a single "
+                "loop";
+  let constructor = "mlir::createLoopCoalescingPass()";
+}
+
+def LoopInvariantCodeMotion : Pass<"loop-invariant-code-motion"> {
+  let summary = "Hoist loop invariant instructions outside of the loop";
+  let constructor = "mlir::createLoopInvariantCodeMotionPass()";
+}
+
+def MemRefDataFlowOpt : Pass<"memref-dataflow-opt"> {
+  let summary = "Perform store/load forwarding for memrefs";
+  let description = [{
+    This pass performs store to load forwarding for memref's to eliminate memory
+    accesses and potentially the entire memref if all its accesses are
+    forwarded.
+
+    Input
+
+    ```mlir
+    func @store_load_affine_apply() -> memref<10x10xf32> {
+      %cf7 = constant 7.0 : f32
+      %m = alloc() : memref<10x10xf32>
+      affine.for %i0 = 0 to 10 {
+        affine.for %i1 = 0 to 10 {
+          affine.store %cf7, %m[%i0, %i1] : memref<10x10xf32>
+          %v0 = affine.load %m[%i0, %i1] : memref<10x10xf32>
+          %v1 = addf %v0, %v0 : f32
+        }
+      }
+      return %m : memref<10x10xf32>
+    }
+    ```
+
+    Output
+
+    ```mlir
+    module {
+      func @store_load_affine_apply() -> memref<10x10xf32> {
+        %cst = constant 7.000000e+00 : f32
+        %0 = alloc() : memref<10x10xf32>
+        affine.for %arg0 = 0 to 10 {
+          affine.for %arg1 = 0 to 10 {
+            affine.store %cst, %0[%arg0, %arg1] : memref<10x10xf32>
+            %1 = addf %cst, %cst : f32
+          }
+        }
+        return %0 : memref<10x10xf32>
+      }
+    }
+    ```
+  }];
+  let constructor = "mlir::createMemRefDataFlowOptPass()";
+}
+
+def ParallelLoopCollapsing : Pass<"parallel-loop-collapsing"> {
+  let summary = "Collapse parallel loops to use less induction variables";
+  let constructor = "mlir::createParallelLoopCollapsingPass()";
+  let options = [
+    ListOption<"clCollapsedIndices0", "collapsed-indices-0", "unsigned",
+               "Which loop indices to combine 0th loop index",
+               "llvm::cl::MiscFlags::CommaSeparated">,
+    ListOption<"clCollapsedIndices1", "collapsed-indices-1", "unsigned",
+               "Which loop indices to combine into the position 1 loop index",
+               "llvm::cl::MiscFlags::CommaSeparated">,
+    ListOption<"clCollapsedIndices2", "collapsed-indices-2", "unsigned",
+               "Which loop indices to combine into the position 2 loop index",
+               "llvm::cl::MiscFlags::CommaSeparated">,
+  ];
+}
+
+def PrintCFG : Pass<"print-cfg-graph"> {
+  let summary = "Print CFG graph per-Region";
+  let constructor = "mlir::createPrintCFGGraphPass()";
+}
+
+def PrintOpStats : Pass<"print-op-stats"> {
+  let summary = "Print statistics of operations";
+  let constructor = "mlir::createPrintOpStatsPass()";
+}
+
+def PrintOp : Pass<"print-op-graph"> {
+  let summary = "Print op graph per-Region";
+  let constructor = "mlir::createPrintOpGraphPass()";
+}
+
+def StripDebugInfo : Pass<"strip-debuginfo"> {
+  let summary = "Strip debug info from all operations";
+  let constructor = "mlir::createStripDebugInfoPass()";
+}
+
+def SymbolDCE : Pass<"symbol-dce"> {
+  let summary = "Eliminate dead symbols";
+  let constructor = "mlir::createSymbolDCEPass()";
+}
+
+#endif // MLIR_TRANSFORMS_PASSES
diff --git a/mlir/lib/Analysis/AffineAnalysis.cpp b/mlir/lib/Analysis/AffineAnalysis.cpp
index 56a0d72bb387c..2abd3811ab0c0 100644
--- a/mlir/lib/Analysis/AffineAnalysis.cpp
+++ b/mlir/lib/Analysis/AffineAnalysis.cpp
@@ -12,16 +12,13 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Analysis/AffineAnalysis.h"
-#include "mlir/Analysis/AffineStructures.h"
 #include "mlir/Analysis/Utils.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Affine/IR/AffineValueMap.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/IR/AffineExprVisitor.h"
-#include "mlir/IR/Builders.h"
 #include "mlir/IR/Function.h"
 #include "mlir/IR/IntegerSet.h"
-#include "mlir/IR/Operation.h"
 #include "mlir/Support/MathExtras.h"
 #include "mlir/Support/STLExtras.h"
 #include "llvm/ADT/DenseMap.h"
diff --git a/mlir/lib/Analysis/AffineStructures.cpp b/mlir/lib/Analysis/AffineStructures.cpp
index 6ebc673c3100b..947087c549e24 100644
--- a/mlir/lib/Analysis/AffineStructures.cpp
+++ b/mlir/lib/Analysis/AffineStructures.cpp
@@ -726,15 +726,14 @@ LogicalResult FlatAffineConstraints::addAffineForOpDomain(AffineForOp forOp) {
 // equality (isEq=true) or inequality (isEq=false) constraints.
 // Returns true and sets row found in search in 'rowIdx'.
 // Returns false otherwise.
-static bool
-findConstraintWithNonZeroAt(const FlatAffineConstraints &constraints,
-                            unsigned colIdx, bool isEq, unsigned *rowIdx) {
+static bool findConstraintWithNonZeroAt(const FlatAffineConstraints &cst,
+                                        unsigned colIdx, bool isEq,
+                                        unsigned *rowIdx) {
+  assert(colIdx < cst.getNumCols() && "position out of bounds");
   auto at = [&](unsigned rowIdx) -> int64_t {
-    return isEq ? constraints.atEq(rowIdx, colIdx)
-                : constraints.atIneq(rowIdx, colIdx);
+    return isEq ? cst.atEq(rowIdx, colIdx) : cst.atIneq(rowIdx, colIdx);
   };
-  unsigned e =
-      isEq ? constraints.getNumEqualities() : constraints.getNumInequalities();
+  unsigned e = isEq ? cst.getNumEqualities() : cst.getNumInequalities();
   for (*rowIdx = 0; *rowIdx < e; ++(*rowIdx)) {
     if (at(*rowIdx) != 0) {
       return true;
@@ -1201,38 +1200,58 @@ static bool detectAsMod(const FlatAffineConstraints &cst, unsigned pos,
   return false;
 }
 
-/// Gather all lower and upper bounds of the identifier at `pos`. The bounds are
-/// to be independent of [offset, offset + num) identifiers.
-static void getLowerAndUpperBoundIndices(const FlatAffineConstraints &cst,
-                                         unsigned pos,
-                                         SmallVectorImpl<unsigned> *lbIndices,
-                                         SmallVectorImpl<unsigned> *ubIndices,
-                                         unsigned offset = 0,
-                                         unsigned num = 0) {
-  assert(pos < cst.getNumIds() && "invalid position");
+/// Gather all lower and upper bounds of the identifier at `pos`, and
+/// optionally any equalities on it. In addition, the bounds are to be
+/// independent of identifiers in position range [`offset`, `offset` + `num`).
+void FlatAffineConstraints::getLowerAndUpperBoundIndices(
+    unsigned pos, SmallVectorImpl<unsigned> *lbIndices,
+    SmallVectorImpl<unsigned> *ubIndices, SmallVectorImpl<unsigned> *eqIndices,
+    unsigned offset, unsigned num) const {
+  assert(pos < getNumIds() && "invalid position");
+  assert(offset + num < getNumCols() && "invalid range");
 
-  // Gather all lower bounds and upper bounds of the variable. Since the
-  // canonical form c_1*x_1 + c_2*x_2 + ... + c_0 >= 0, a constraint is a lower
-  // bound for x_i if c_i >= 1, and an upper bound if c_i <= -1.
-  for (unsigned r = 0, e = cst.getNumInequalities(); r < e; r++) {
-    // The bounds are to be independent of [offset, offset + num) columns.
+  // Checks for a constraint that has a non-zero coeff for the identifiers in
+  // the position range [offset, offset + num) while ignoring `pos`.
+  auto containsConstraintDependentOnRange = [&](unsigned r, bool isEq) {
     unsigned c, f;
+    auto cst = isEq ? getEquality(r) : getInequality(r);
     for (c = offset, f = offset + num; c < f; ++c) {
       if (c == pos)
         continue;
-      if (cst.atIneq(r, c) != 0)
+      if (cst[c] != 0)
         break;
     }
-    if (c < f)
+    return c < f;
+  };
+
+  // Gather all lower bounds and upper bounds of the variable. Since the
+  // canonical form c_1*x_1 + c_2*x_2 + ... + c_0 >= 0, a constraint is a lower
+  // bound for x_i if c_i >= 1, and an upper bound if c_i <= -1.
+  for (unsigned r = 0, e = getNumInequalities(); r < e; r++) {
+    // The bounds are to be independent of [offset, offset + num) columns.
+    if (containsConstraintDependentOnRange(r, /*isEq=*/false))
       continue;
-    if (cst.atIneq(r, pos) >= 1) {
+    if (atIneq(r, pos) >= 1) {
       // Lower bound.
       lbIndices->push_back(r);
-    } else if (cst.atIneq(r, pos) <= -1) {
+    } else if (atIneq(r, pos) <= -1) {
       // Upper bound.
       ubIndices->push_back(r);
     }
   }
+
+  // An equality is both a lower and upper bound. Record any equalities
+  // involving the pos^th identifier.
+  if (!eqIndices)
+    return;
+
+  for (unsigned r = 0, e = getNumEqualities(); r < e; r++) {
+    if (atEq(r, pos) == 0)
+      continue;
+    if (containsConstraintDependentOnRange(r, /*isEq=*/true))
+      continue;
+    eqIndices->push_back(r);
+  }
 }
 
 /// Check if the pos^th identifier can be expressed as a floordiv of an affine
@@ -1248,7 +1267,7 @@ static bool detectAsFloorDiv(const FlatAffineConstraints &cst, unsigned pos,
   assert(pos < cst.getNumIds() && "invalid position");
 
   SmallVector<unsigned, 4> lbIndices, ubIndices;
-  getLowerAndUpperBoundIndices(cst, pos, &lbIndices, &ubIndices);
+  cst.getLowerAndUpperBoundIndices(pos, &lbIndices, &ubIndices);
 
   // Check if any lower bound, upper bound pair is of the form:
   // divisor * id >=  expr - (divisor - 1)    <-- Lower bound for 'id'
@@ -1377,7 +1396,7 @@ std::pair<AffineMap, AffineMap> FlatAffineConstraints::getLowerAndUpperBound(
          "incorrect local exprs count");
 
   SmallVector<unsigned, 4> lbIndices, ubIndices;
-  getLowerAndUpperBoundIndices(*this, pos + offset, &lbIndices, &ubIndices);
+  getLowerAndUpperBoundIndices(pos + offset, &lbIndices, &ubIndices);
 
   /// Add to 'b' from 'a' in set [0, offset) U [offset + num, symbStartPos).
   auto addCoeffs = [&](ArrayRef<int64_t> a, SmallVectorImpl<int64_t> &b) {
@@ -1873,9 +1892,23 @@ void FlatAffineConstraints::removeEquality(unsigned pos) {
   std::copy(equalities.begin() + inputIndex,
             equalities.begin() + inputIndex + numElemsToCopy,
             equalities.begin() + outputIndex);
+  assert(equalities.size() >= numReservedCols);
   equalities.resize(equalities.size() - numReservedCols);
 }
 
+void FlatAffineConstraints::removeInequality(unsigned pos) {
+  unsigned numInequalities = getNumInequalities();
+  assert(pos < numInequalities && "invalid position");
+  unsigned outputIndex = pos * numReservedCols;
+  unsigned inputIndex = (pos + 1) * numReservedCols;
+  unsigned numElemsToCopy = (numInequalities - pos - 1) * numReservedCols;
+  std::copy(inequalities.begin() + inputIndex,
+            inequalities.begin() + inputIndex + numElemsToCopy,
+            inequalities.begin() + outputIndex);
+  assert(inequalities.size() >= numReservedCols);
+  inequalities.resize(inequalities.size() - numReservedCols);
+}
+
 /// Finds an equality that equates the specified identifier to a constant.
 /// Returns the position of the equality row. If 'symbolic' is set to true,
 /// symbols are also treated like a constant, i.e., an affine function of the
@@ -1952,14 +1985,22 @@ void FlatAffineConstraints::constantFoldIdRange(unsigned pos, unsigned num) {
 //       ceil(s0 - 7 / 8) = floor(s0 / 8)).
 Optional<int64_t> FlatAffineConstraints::getConstantBoundOnDimSize(
     unsigned pos, SmallVectorImpl<int64_t> *lb, int64_t *boundFloorDivisor,
-    SmallVectorImpl<int64_t> *ub) const {
+    SmallVectorImpl<int64_t> *ub, unsigned *minLbPos,
+    unsigned *minUbPos) const {
   assert(pos < getNumDimIds() && "Invalid identifier position");
-  assert(getNumLocalIds() == 0);
 
   // Find an equality for 'pos'^th identifier that equates it to some function
   // of the symbolic identifiers (+ constant).
   int eqPos = findEqualityToConstant(*this, pos, /*symbolic=*/true);
   if (eqPos != -1) {
+    auto eq = getEquality(eqPos);
+    // If the equality involves a local var, punt for now.
+    // TODO: this can be handled in the future by using the explicit
+    // representation of the local vars.
+    if (!std::all_of(eq.begin() + getNumDimAndSymbolIds(), eq.end() - 1,
+                     [](int64_t coeff) { return coeff == 0; }))
+      return None;
+
     // This identifier can only take a single value.
     if (lb) {
       // Set lb to that symbolic value.
@@ -1980,6 +2021,10 @@ Optional<int64_t> FlatAffineConstraints::getConstantBoundOnDimSize(
              "both lb and divisor or none should be provided");
       *boundFloorDivisor = 1;
     }
+    if (minLbPos)
+      *minLbPos = eqPos;
+    if (minUbPos)
+      *minUbPos = eqPos;
     return 1;
   }
 
@@ -2000,8 +2045,8 @@ Optional<int64_t> FlatAffineConstraints::getConstantBoundOnDimSize(
   // the bounds can only involve symbolic (and local) identifiers. Since the
   // canonical form c_1*x_1 + c_2*x_2 + ... + c_0 >= 0, a constraint is a lower
   // bound for x_i if c_i >= 1, and an upper bound if c_i <= -1.
-  getLowerAndUpperBoundIndices(*this, pos, &lbIndices, &ubIndices,
-                               /*offset=*/0,
+  getLowerAndUpperBoundIndices(pos, &lbIndices, &ubIndices,
+                               /*eqIndices=*/nullptr, /*offset=*/0,
                                /*num=*/getNumDimIds());
 
   Optional<int64_t> minDiff = None;
@@ -2055,6 +2100,10 @@ Optional<int64_t> FlatAffineConstraints::getConstantBoundOnDimSize(
     // the constant term for the lower bound.
     (*lb)[getNumSymbolIds()] += atIneq(minLbPosition, pos) - 1;
   }
+  if (minLbPos)
+    *minLbPos = minLbPosition;
+  if (minUbPos)
+    *minUbPos = minUbPosition;
   return minDiff;
 }
 
@@ -2191,13 +2240,16 @@ void FlatAffineConstraints::dump() const { print(llvm::errs()); }
 //  Uses a DenseSet to hash and detect duplicates followed by a linear scan to
 //  remove duplicates in place.
 void FlatAffineConstraints::removeTrivialRedundancy() {
-  SmallDenseSet<ArrayRef<int64_t>, 8> rowSet;
+  GCDTightenInequalities();
+  normalizeConstraintsByGCD();
 
   // A map used to detect redundancy stemming from constraints that only differ
   // in their constant term. The value stored is <row position, const term>
   // for a given row.
   SmallDenseMap<ArrayRef<int64_t>, std::pair<unsigned, int64_t>>
       rowsWithoutConstTerm;
+  // To unique rows.
+  SmallDenseSet<ArrayRef<int64_t>, 8> rowSet;
 
   // Check if constraint is of the form <non-negative-constant> >= 0.
   auto isTriviallyValid = [&](unsigned r) -> bool {
@@ -2690,3 +2742,183 @@ FlatAffineConstraints::unionBoundingBox(const FlatAffineConstraints &otherCst) {
 
   return success();
 }
+
+/// Compute an explicit representation for local vars. For all systems coming
+/// from MLIR integer sets, maps, or expressions where local vars were
+/// introduced to model floordivs and mods, this always succeeds.
+static LogicalResult computeLocalVars(const FlatAffineConstraints &cst,
+                                      SmallVectorImpl<AffineExpr> &memo,
+                                      MLIRContext *context) {
+  unsigned numDims = cst.getNumDimIds();
+  unsigned numSyms = cst.getNumSymbolIds();
+
+  // Initialize dimensional and symbolic identifiers.
+  for (unsigned i = 0; i < numDims; i++)
+    memo[i] = getAffineDimExpr(i, context);
+  for (unsigned i = numDims, e = numDims + numSyms; i < e; i++)
+    memo[i] = getAffineSymbolExpr(i - numDims, context);
+
+  bool changed;
+  do {
+    // Each time `changed` is true at the end of this iteration, one or more
+    // local vars would have been detected as floordivs and set in memo; so the
+    // number of null entries in memo[...] strictly reduces; so this converges.
+    changed = false;
+    for (unsigned i = 0, e = cst.getNumLocalIds(); i < e; ++i)
+      if (!memo[numDims + numSyms + i] &&
+          detectAsFloorDiv(cst, /*pos=*/numDims + numSyms + i, context, memo))
+        changed = true;
+  } while (changed);
+
+  ArrayRef<AffineExpr> localExprs =
+      ArrayRef<AffineExpr>(memo).take_back(cst.getNumLocalIds());
+  return success(
+      llvm::all_of(localExprs, [](AffineExpr expr) { return expr; }));
+}
+
+void FlatAffineConstraints::getIneqAsAffineValueMap(
+    unsigned pos, unsigned ineqPos, AffineValueMap &vmap,
+    MLIRContext *context) const {
+  unsigned numDims = getNumDimIds();
+  unsigned numSyms = getNumSymbolIds();
+
+  assert(pos < numDims && "invalid position");
+  assert(ineqPos < getNumInequalities() && "invalid inequality position");
+
+  // Get expressions for local vars.
+  SmallVector<AffineExpr, 8> memo(getNumIds(), AffineExpr());
+  if (failed(computeLocalVars(*this, memo, context)))
+    assert(false &&
+           "one or more local exprs do not have an explicit representation");
+  auto localExprs = ArrayRef<AffineExpr>(memo).take_back(getNumLocalIds());
+
+  // Compute the AffineExpr lower/upper bound for this inequality.
+  ArrayRef<int64_t> inequality = getInequality(ineqPos);
+  SmallVector<int64_t, 8> bound;
+  bound.reserve(getNumCols() - 1);
+  // Everything other than the coefficient at `pos`.
+  bound.append(inequality.begin(), inequality.begin() + pos);
+  bound.append(inequality.begin() + pos + 1, inequality.end());
+
+  if (inequality[pos] > 0)
+    // Lower bound.
+    std::transform(bound.begin(), bound.end(), bound.begin(),
+                   std::negate<int64_t>());
+  else
+    // Upper bound (which is exclusive).
+    bound.back() += 1;
+
+  // Convert to AffineExpr (tree) form.
+  auto boundExpr = getAffineExprFromFlatForm(bound, numDims - 1, numSyms,
+                                             localExprs, context);
+
+  // Get the values to bind to this affine expr (all dims and symbols).
+  SmallVector<Value, 4> operands;
+  getIdValues(0, pos, &operands);
+  SmallVector<Value, 4> trailingOperands;
+  getIdValues(pos + 1, getNumDimAndSymbolIds(), &trailingOperands);
+  operands.append(trailingOperands.begin(), trailingOperands.end());
+  vmap.reset(AffineMap::get(numDims - 1, numSyms, boundExpr), operands);
+}
+
+/// Returns true if the pos^th column is all zero for both inequalities and
+/// equalities..
+static bool isColZero(const FlatAffineConstraints &cst, unsigned pos) {
+  unsigned rowPos;
+  return !findConstraintWithNonZeroAt(cst, pos, /*isEq=*/false, &rowPos) &&
+         !findConstraintWithNonZeroAt(cst, pos, /*isEq=*/true, &rowPos);
+}
+
+IntegerSet FlatAffineConstraints::getAsIntegerSet(MLIRContext *context) const {
+  if (getNumConstraints() == 0)
+    // Return universal set (always true): 0 == 0.
+    return IntegerSet::get(getNumDimIds(), getNumSymbolIds(),
+                           getAffineConstantExpr(/*constant=*/0, context),
+                           /*eqFlags=*/true);
+
+  // Construct local references.
+  SmallVector<AffineExpr, 8> memo(getNumIds(), AffineExpr());
+
+  if (failed(computeLocalVars(*this, memo, context))) {
+    // Check if the local variables without an explicit representation have
+    // zero coefficients everywhere.
+    for (unsigned i = getNumDimAndSymbolIds(), e = getNumIds(); i < e; ++i) {
+      if (!memo[i] && !isColZero(*this, /*pos=*/i)) {
+        LLVM_DEBUG(llvm::dbgs() << "one or more local exprs do not have an "
+                                   "explicit representation");
+        return IntegerSet();
+      }
+    }
+  }
+
+  ArrayRef<AffineExpr> localExprs =
+      ArrayRef<AffineExpr>(memo).take_back(getNumLocalIds());
+
+  // Construct the IntegerSet from the equalities/inequalities.
+  unsigned numDims = getNumDimIds();
+  unsigned numSyms = getNumSymbolIds();
+
+  SmallVector<bool, 16> eqFlags(getNumConstraints());
+  std::fill(eqFlags.begin(), eqFlags.begin() + getNumEqualities(), true);
+  std::fill(eqFlags.begin() + getNumEqualities(), eqFlags.end(), false);
+
+  SmallVector<AffineExpr, 8> exprs;
+  exprs.reserve(getNumConstraints());
+
+  for (unsigned i = 0, e = getNumEqualities(); i < e; ++i)
+    exprs.push_back(getAffineExprFromFlatForm(getEquality(i), numDims, numSyms,
+                                              localExprs, context));
+  for (unsigned i = 0, e = getNumInequalities(); i < e; ++i)
+    exprs.push_back(getAffineExprFromFlatForm(getInequality(i), numDims,
+                                              numSyms, localExprs, context));
+  return IntegerSet::get(numDims, numSyms, exprs, eqFlags);
+}
+
+/// Find positions of inequalities and equalities that do not have a coefficient
+/// for [pos, pos + num) identifiers.
+static void getIndependentConstraints(const FlatAffineConstraints &cst,
+                                      unsigned pos, unsigned num,
+                                      SmallVectorImpl<unsigned> &nbIneqIndices,
+                                      SmallVectorImpl<unsigned> &nbEqIndices) {
+  assert(pos < cst.getNumIds() && "invalid start position");
+  assert(pos + num <= cst.getNumIds() && "invalid limit");
+
+  for (unsigned r = 0, e = cst.getNumInequalities(); r < e; r++) {
+    // The bounds are to be independent of [offset, offset + num) columns.
+    unsigned c;
+    for (c = pos; c < pos + num; ++c) {
+      if (cst.atIneq(r, c) != 0)
+        break;
+    }
+    if (c == pos + num)
+      nbIneqIndices.push_back(r);
+  }
+
+  for (unsigned r = 0, e = cst.getNumEqualities(); r < e; r++) {
+    // The bounds are to be independent of [offset, offset + num) columns.
+    unsigned c;
+    for (c = pos; c < pos + num; ++c) {
+      if (cst.atEq(r, c) != 0)
+        break;
+    }
+    if (c == pos + num)
+      nbEqIndices.push_back(r);
+  }
+}
+
+void FlatAffineConstraints::removeIndependentConstraints(unsigned pos,
+                                                         unsigned num) {
+  assert(pos + num <= getNumIds() && "invalid range");
+
+  // Remove constraints that are independent of these identifiers.
+  SmallVector<unsigned, 4> nbIneqIndices, nbEqIndices;
+  getIndependentConstraints(*this, /*pos=*/0, num, nbIneqIndices, nbEqIndices);
+
+  // Iterate in reverse so that indices don't have to be updated.
+  // TODO: This method can be made more efficient (because removal of each
+  // inequality leads to much shifting/copying in the underlying buffer).
+  for (auto nbIndex : llvm::reverse(nbIneqIndices))
+    removeInequality(nbIndex);
+  for (auto nbIndex : llvm::reverse(nbEqIndices))
+    removeEquality(nbIndex);
+}
diff --git a/mlir/lib/Analysis/Dominance.cpp b/mlir/lib/Analysis/Dominance.cpp
index b2f0930cd81af..1b5f4cd917a27 100644
--- a/mlir/lib/Analysis/Dominance.cpp
+++ b/mlir/lib/Analysis/Dominance.cpp
@@ -13,6 +13,7 @@
 
 #include "mlir/Analysis/Dominance.h"
 #include "mlir/IR/Operation.h"
+#include "llvm/ADT/DenseMap.h"
 #include "llvm/Support/GenericDomTreeConstruction.h"
 
 using namespace mlir;
@@ -43,6 +44,99 @@ void DominanceInfoBase<IsPostDom>::recalculate(Operation *op) {
   });
 }
 
+/// Walks up the list of containers of the given block and calls the
+/// user-defined traversal function for every pair of a region and block that
+/// could be found during traversal. If the user-defined function returns true
+/// for a given pair, traverseAncestors will return the current block. Nullptr
+/// otherwise.
+template <typename FuncT>
+Block *traverseAncestors(Block *block, const FuncT &func) {
+  // Invoke the user-defined traversal function in the beginning for the current
+  // block.
+  if (func(block))
+    return block;
+
+  Region *region = block->getParent();
+  while (region) {
+    Operation *ancestor = region->getParentOp();
+    // If we have reached to top... return.
+    if (!ancestor || !(block = ancestor->getBlock()))
+      break;
+
+    // Update the nested region using the new ancestor block.
+    region = block->getParent();
+
+    // Invoke the user-defined traversal function and check whether we can
+    // already return.
+    if (func(block))
+      return block;
+  }
+  return nullptr;
+}
+
+/// Tries to update the given block references to live in the same region by
+/// exploring the relationship of both blocks with respect to their regions.
+static bool tryGetBlocksInSameRegion(Block *&a, Block *&b) {
+  // If both block do not live in the same region, we will have to check their
+  // parent operations.
+  if (a->getParent() == b->getParent())
+    return true;
+
+  // Iterate over all ancestors of a and insert them into the map. This allows
+  // for efficient lookups to find a commonly shared region.
+  llvm::SmallDenseMap<Region *, Block *, 4> ancestors;
+  traverseAncestors(a, [&](Block *block) {
+    ancestors[block->getParent()] = block;
+    return false;
+  });
+
+  // Try to find a common ancestor starting with regionB.
+  b = traverseAncestors(
+      b, [&](Block *block) { return ancestors.count(block->getParent()) > 0; });
+
+  // If there is no match, we will not be able to find a common dominator since
+  // both regions do not share a common parent region.
+  if (!b)
+    return false;
+
+  // We have found a common parent region. Update block a to refer to this
+  // region.
+  auto it = ancestors.find(b->getParent());
+  assert(it != ancestors.end());
+  a = it->second;
+  return true;
+}
+
+template <bool IsPostDom>
+Block *
+DominanceInfoBase<IsPostDom>::findNearestCommonDominator(Block *a,
+                                                         Block *b) const {
+  // If either a or b are null, then conservatively return nullptr.
+  if (!a || !b)
+    return nullptr;
+
+  // Try to find blocks that are in the same region.
+  if (!tryGetBlocksInSameRegion(a, b))
+    return nullptr;
+
+  // Get and verify dominance information of the common parent region.
+  Region *parentRegion = a->getParent();
+  auto infoAIt = dominanceInfos.find(parentRegion);
+  if (infoAIt == dominanceInfos.end())
+    return nullptr;
+
+  // Since the blocks live in the same region, we can rely on already
+  // existing dominance functionality.
+  return infoAIt->second->findNearestCommonDominator(a, b);
+}
+
+template <bool IsPostDom>
+DominanceInfoNode *DominanceInfoBase<IsPostDom>::getNode(Block *a) {
+  auto *region = a->getParent();
+  assert(dominanceInfos.count(region) != 0);
+  return dominanceInfos[region]->getNode(a);
+}
+
 /// Return true if the specified block A properly dominates block B.
 template <bool IsPostDom>
 bool DominanceInfoBase<IsPostDom>::properlyDominates(Block *a, Block *b) {
@@ -57,21 +151,17 @@ bool DominanceInfoBase<IsPostDom>::properlyDominates(Block *a, Block *b) {
   // If both blocks are not in the same region, 'a' properly dominates 'b' if
   // 'b' is defined in an operation region that (recursively) ends up being
   // dominated by 'a'. Walk up the list of containers enclosing B.
-  auto *regionA = a->getParent(), *regionB = b->getParent();
-  if (regionA != regionB) {
-    Operation *bAncestor;
-    do {
-      bAncestor = regionB->getParentOp();
-      // If 'bAncestor' is the top level region, then 'a' is a block that post
-      // dominates 'b'.
-      if (!bAncestor || !bAncestor->getBlock())
-        return IsPostDom;
-
-      regionB = bAncestor->getBlock()->getParent();
-    } while (regionA != regionB);
+  auto *regionA = a->getParent();
+  if (regionA != b->getParent()) {
+    b = traverseAncestors(
+        b, [&](Block *block) { return block->getParent() == regionA; });
+
+    // If we could not find a valid block b then it is either a not a dominator
+    // or a post dominator.
+    if (!b)
+      return IsPostDom;
 
     // Check to see if the ancestor of 'b' is the same block as 'a'.
-    b = bAncestor->getBlock();
     if (a == b)
       return true;
   }
@@ -132,12 +222,6 @@ bool DominanceInfo::properlyDominates(Value a, Operation *b) {
   return dominates(a.cast<BlockArgument>().getOwner(), b->getBlock());
 }
 
-DominanceInfoNode *DominanceInfo::getNode(Block *a) {
-  auto *region = a->getParent();
-  assert(dominanceInfos.count(region) != 0);
-  return dominanceInfos[region]->getNode(a);
-}
-
 void DominanceInfo::updateDFSNumbers() {
   for (auto &iter : dominanceInfos)
     iter.second->updateDFSNumbers();
diff --git a/mlir/lib/Analysis/LoopAnalysis.cpp b/mlir/lib/Analysis/LoopAnalysis.cpp
index 9b5725188faea..40e176367d18b 100644
--- a/mlir/lib/Analysis/LoopAnalysis.cpp
+++ b/mlir/lib/Analysis/LoopAnalysis.cpp
@@ -30,9 +30,6 @@ using namespace mlir;
 /// expression is simplified before returning. This method only utilizes map
 /// composition to construct lower and upper bounds before computing the trip
 /// count expressions.
-// TODO(mlir-team): this should be moved into 'Transforms/' and be replaced by a
-// pure analysis method relying on FlatAffineConstraints; the latter will also
-// be more powerful (since both inequalities and equalities will be considered).
 void mlir::buildTripCountMapAndOperands(
     AffineForOp forOp, AffineMap *tripCountMap,
     SmallVectorImpl<Value> *tripCountOperands) {
@@ -270,8 +267,8 @@ static bool isContiguousAccess(Value iv, LoadOrStoreOp memoryOp,
   return true;
 }
 
-template <typename LoadOrStoreOpPointer>
-static bool isVectorElement(LoadOrStoreOpPointer memoryOp) {
+template <typename LoadOrStoreOp>
+static bool isVectorElement(LoadOrStoreOp memoryOp) {
   auto memRefType = memoryOp.getMemRefType();
   return memRefType.getElementType().template isa<VectorType>();
 }
@@ -351,7 +348,7 @@ bool mlir::isVectorizableLoopBody(AffineForOp loop,
 /// 'def' and all its uses have the same shift factor.
 // TODO(mlir-team): extend this to check for memory-based dependence violation
 // when we have the support.
-bool mlir::isInstwiseShiftValid(AffineForOp forOp, ArrayRef<uint64_t> shifts) {
+bool mlir::isOpwiseShiftValid(AffineForOp forOp, ArrayRef<uint64_t> shifts) {
   auto *forBody = forOp.getBody();
   assert(shifts.size() == forBody->getOperations().size());
 
diff --git a/mlir/lib/Analysis/Utils.cpp b/mlir/lib/Analysis/Utils.cpp
index 9e400e4b6a3c4..bef227e83564b 100644
--- a/mlir/lib/Analysis/Utils.cpp
+++ b/mlir/lib/Analysis/Utils.cpp
@@ -17,6 +17,7 @@
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Affine/IR/AffineValueMap.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
+#include "mlir/IR/IntegerSet.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
@@ -1007,3 +1008,15 @@ bool mlir::isLoopParallel(AffineForOp forOp) {
   }
   return true;
 }
+
+IntegerSet mlir::simplifyIntegerSet(IntegerSet set) {
+  FlatAffineConstraints fac(set);
+  if (fac.isEmpty())
+    return IntegerSet::getEmptySet(set.getNumDims(), set.getNumSymbols(),
+                                   set.getContext());
+  fac.removeTrivialRedundancy();
+
+  auto simplifiedSet = fac.getAsIntegerSet(set.getContext());
+  assert(simplifiedSet && "guaranteed to succeed while roundtripping");
+  return simplifiedSet;
+}
diff --git a/mlir/lib/Analysis/Verifier.cpp b/mlir/lib/Analysis/Verifier.cpp
index 161feb426c2ff..e15d3513b7de5 100644
--- a/mlir/lib/Analysis/Verifier.cpp
+++ b/mlir/lib/Analysis/Verifier.cpp
@@ -198,9 +198,18 @@ LogicalResult OperationVerifier::verifyOperation(Operation &op) {
       it = dialectAllowsUnknownOps
                .try_emplace(dialectPrefix, dialect->allowsUnknownOperations())
                .first;
-    // Otherwise, conservatively allow unknown operations.
-    else
+    // Otherwise, unregistered dialects (when allowed by the context)
+    // conservatively allow unknown operations.
+    else {
+      if (!op.getContext()->allowsUnregisteredDialects() && !op.getDialect())
+        return op.emitOpError()
+               << "created with unregistered dialect. If this is "
+                  "intended, please call allowUnregisteredDialects() on the "
+                  "MLIRContext, or use -allow-unregistered-dialect with "
+                  "mlir-opt";
+
       it = dialectAllowsUnknownOps.try_emplace(dialectPrefix, true).first;
+    }
   }
 
   if (!it->second) {
diff --git a/mlir/lib/Conversion/AVX512ToLLVM/CMakeLists.txt b/mlir/lib/Conversion/AVX512ToLLVM/CMakeLists.txt
index 5573f6ca1618c..9df0d4fde7f1e 100644
--- a/mlir/lib/Conversion/AVX512ToLLVM/CMakeLists.txt
+++ b/mlir/lib/Conversion/AVX512ToLLVM/CMakeLists.txt
@@ -3,6 +3,9 @@ add_mlir_conversion_library(MLIRAVX512ToLLVM
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/AVX512ToLLVM
+
+  DEPENDS
+  MLIRConversionPassIncGen
 )
 
 set(LIBS
diff --git a/mlir/lib/Conversion/AVX512ToLLVM/ConvertAVX512ToLLVM.cpp b/mlir/lib/Conversion/AVX512ToLLVM/ConvertAVX512ToLLVM.cpp
index af29714eb69a8..91f3cc933a020 100644
--- a/mlir/lib/Conversion/AVX512ToLLVM/ConvertAVX512ToLLVM.cpp
+++ b/mlir/lib/Conversion/AVX512ToLLVM/ConvertAVX512ToLLVM.cpp
@@ -164,6 +164,10 @@ void mlir::populateAVX512ToLLVMConversionPatterns(
 
 namespace {
 struct ConvertAVX512ToLLVMPass : public ModulePass<ConvertAVX512ToLLVMPass> {
+/// Include the generated pass utilities.
+#define GEN_PASS_ConvertAVX512ToLLVM
+#include "mlir/Conversion/Passes.h.inc"
+
   void runOnModule() override;
 };
 } // namespace
@@ -191,7 +195,3 @@ void ConvertAVX512ToLLVMPass::runOnModule() {
 std::unique_ptr<OpPassBase<ModuleOp>> mlir::createConvertAVX512ToLLVMPass() {
   return std::make_unique<ConvertAVX512ToLLVMPass>();
 }
-
-static PassRegistration<ConvertAVX512ToLLVMPass> pass(
-    "convert-avx512-to-llvm",
-    "Convert the operations from the avx512 dialect into the LLVM dialect");
diff --git a/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp b/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
index bb434abdc26c7..56928da2c633d 100644
--- a/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
+++ b/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
@@ -258,7 +258,7 @@ static Value buildMinMaxReductionSeq(Location loc, CmpIPredicate predicate,
   return value;
 }
 
-/// Emit instructions that correspond to computing the maximum value amoung the
+/// Emit instructions that correspond to computing the maximum value among the
 /// values of a (potentially) multi-output affine map applied to `operands`.
 static Value lowerAffineMapMax(OpBuilder &builder, Location loc, AffineMap map,
                                ValueRange operands) {
@@ -267,7 +267,7 @@ static Value lowerAffineMapMax(OpBuilder &builder, Location loc, AffineMap map,
   return nullptr;
 }
 
-/// Emit instructions that correspond to computing the minimum value amoung the
+/// Emit instructions that correspond to computing the minimum value among the
 /// values of a (potentially) multi-output affine map applied to `operands`.
 static Value lowerAffineMapMin(OpBuilder &builder, Location loc, AffineMap map,
                                ValueRange operands) {
@@ -578,6 +578,10 @@ void mlir::populateAffineToStdConversionPatterns(
 
 namespace {
 class LowerAffinePass : public FunctionPass<LowerAffinePass> {
+/// Include the generated pass utilities.
+#define GEN_PASS_ConvertAffineToStandard
+#include "mlir/Conversion/Passes.h.inc"
+
   void runOnFunction() override {
     OwningRewritePatternList patterns;
     populateAffineToStdConversionPatterns(patterns, &getContext());
@@ -594,7 +598,3 @@ class LowerAffinePass : public FunctionPass<LowerAffinePass> {
 std::unique_ptr<OpPassBase<FuncOp>> mlir::createLowerAffinePass() {
   return std::make_unique<LowerAffinePass>();
 }
-
-static PassRegistration<LowerAffinePass>
-    pass("lower-affine",
-         "Lower If, For, AffineApply operations to primitive equivalents");
diff --git a/mlir/lib/Conversion/AffineToStandard/CMakeLists.txt b/mlir/lib/Conversion/AffineToStandard/CMakeLists.txt
index 5613b28e3418f..821a1deb0a153 100644
--- a/mlir/lib/Conversion/AffineToStandard/CMakeLists.txt
+++ b/mlir/lib/Conversion/AffineToStandard/CMakeLists.txt
@@ -3,6 +3,9 @@ add_mlir_conversion_library(MLIRAffineToStandard
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/AffineToStandard
+
+  DEPENDS
+  MLIRConversionPassIncGen
 )
 target_link_libraries(
   MLIRAffineToStandard
diff --git a/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h b/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h
index 3ab1fc48cf72a..6d4b99a77d7df 100644
--- a/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h
+++ b/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h
@@ -34,16 +34,12 @@ struct GPUIndexIntrinsicOpLowering : public ConvertToLLVMPattern {
         .Default(invalid);
   }
 
-  static unsigned getIndexBitWidth(LLVMTypeConverter &type_converter) {
-    auto dialect = type_converter.getDialect();
-    return dialect->getLLVMModule().getDataLayout().getPointerSizeInBits();
-  }
-
 public:
-  explicit GPUIndexIntrinsicOpLowering(LLVMTypeConverter &lowering_)
+  explicit GPUIndexIntrinsicOpLowering(LLVMTypeConverter &typeConverter)
       : ConvertToLLVMPattern(Op::getOperationName(),
-                             lowering_.getDialect()->getContext(), lowering_),
-        indexBitwidth(getIndexBitWidth(lowering_)) {}
+                             typeConverter.getDialect()->getContext(),
+                             typeConverter),
+        indexBitwidth(typeConverter.getIndexTypeBitwidth()) {}
 
   // Convert the kernel arguments to an LLVM type, preserve the rest.
   LogicalResult
diff --git a/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h b/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
index eb5628d18e46f..c7bbb6d989c8b 100644
--- a/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
+++ b/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
@@ -95,24 +95,6 @@ struct OpToFuncCallLowering : public ConvertToLLVMPattern {
   const std::string f64Func;
 };
 
-namespace gpu {
-/// Returns a predicate to be used with addDynamicallyLegalOp. The predicate
-/// returns false for calls to the provided intrinsics and true otherwise.
-inline std::function<bool(Operation *)>
-filterIllegalLLVMIntrinsics(ArrayRef<StringRef> intrinsics, MLIRContext *ctx) {
-  SmallVector<StringRef, 4> illegalIds(intrinsics.begin(), intrinsics.end());
-  return [illegalIds](Operation *op) -> bool {
-    LLVM::CallOp callOp = dyn_cast<LLVM::CallOp>(op);
-    if (!callOp || !callOp.callee())
-      return true;
-    StringRef callee = callOp.callee().getValue();
-    return !llvm::any_of(illegalIds, [callee](StringRef intrinsic) {
-      return callee.equals(intrinsic);
-    });
-  };
-}
-} // namespace gpu
-
 } // namespace mlir
 
 #endif // MLIR_CONVERSION_GPUCOMMON_OPTOFUNCCALLLOWERING_H_
diff --git a/mlir/lib/Conversion/GPUToCUDA/CMakeLists.txt b/mlir/lib/Conversion/GPUToCUDA/CMakeLists.txt
index c8f0ded6bc192..a4c98e555322b 100644
--- a/mlir/lib/Conversion/GPUToCUDA/CMakeLists.txt
+++ b/mlir/lib/Conversion/GPUToCUDA/CMakeLists.txt
@@ -15,7 +15,12 @@ if (MLIR_CUDA_CONVERSIONS_ENABLED)
   )
 endif()
 
-add_mlir_conversion_library(MLIRGPUtoCUDATransforms ${SOURCES})
+add_mlir_conversion_library(MLIRGPUtoCUDATransforms
+  ${SOURCES}
+
+  DEPENDS
+  MLIRConversionPassIncGen
+)
 target_link_libraries(MLIRGPUtoCUDATransforms
   PUBLIC
   ${NVPTX_LIBS}
diff --git a/mlir/lib/Conversion/GPUToCUDA/ConvertLaunchFuncToCudaCalls.cpp b/mlir/lib/Conversion/GPUToCUDA/ConvertLaunchFuncToCudaCalls.cpp
index 7a275b42df605..38c092a2eaf0f 100644
--- a/mlir/lib/Conversion/GPUToCUDA/ConvertLaunchFuncToCudaCalls.cpp
+++ b/mlir/lib/Conversion/GPUToCUDA/ConvertLaunchFuncToCudaCalls.cpp
@@ -63,6 +63,10 @@ namespace {
 class GpuLaunchFuncToCudaCallsPass
     : public ModulePass<GpuLaunchFuncToCudaCallsPass> {
 private:
+/// Include the generated pass utilities.
+#define GEN_PASS_ConvertGpuLaunchFuncToCudaCalls
+#include "mlir/Conversion/Passes.h.inc"
+
   LLVM::LLVMDialect *getLLVMDialect() { return llvmDialect; }
 
   llvm::LLVMContext &getLLVMContext() {
@@ -465,7 +469,3 @@ std::unique_ptr<mlir::OpPassBase<mlir::ModuleOp>>
 mlir::createConvertGpuLaunchFuncToCudaCallsPass() {
   return std::make_unique<GpuLaunchFuncToCudaCallsPass>();
 }
-
-static PassRegistration<GpuLaunchFuncToCudaCallsPass>
-    pass("launch-func-to-cuda",
-         "Convert all launch_func ops to CUDA runtime calls");
diff --git a/mlir/lib/Conversion/GPUToNVVM/CMakeLists.txt b/mlir/lib/Conversion/GPUToNVVM/CMakeLists.txt
index 75d82e80260be..b7c583d571699 100644
--- a/mlir/lib/Conversion/GPUToNVVM/CMakeLists.txt
+++ b/mlir/lib/Conversion/GPUToNVVM/CMakeLists.txt
@@ -6,6 +6,7 @@ add_mlir_conversion_library(MLIRGPUtoNVVMTransforms
   LowerGpuOpsToNVVMOps.cpp
 
   DEPENDS
+  MLIRConversionPassIncGen
   MLIRGPUToNVVMIncGen
   )
 
diff --git a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
index e929caac6133f..d8297c76ba9a4 100644
--- a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
+++ b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
@@ -248,6 +248,10 @@ struct GPUReturnOpLowering : public ConvertToLLVMPattern {
 class LowerGpuOpsToNVVMOpsPass
     : public OperationPass<LowerGpuOpsToNVVMOpsPass, gpu::GPUModuleOp> {
 public:
+/// Include the generated pass utilities.
+#define GEN_PASS_ConvertGpuOpsToNVVMOps
+#include "mlir/Conversion/Passes.h.inc"
+
   void runOnOperation() override {
     gpu::GPUModuleOp m = getOperation();
 
@@ -279,8 +283,6 @@ class LowerGpuOpsToNVVMOpsPass
                         LLVM::LogOp, LLVM::Log10Op, LLVM::Log2Op>();
     target.addIllegalOp<FuncOp>();
     target.addLegalDialect<NVVM::NVVMDialect>();
-    target.addDynamicallyLegalOp<mlir::LLVM::CallOp>(
-        gpu::filterIllegalLLVMIntrinsics({"tanh", "tanhf"}, m.getContext()));
     // TODO(csigg): Remove once we support replacing non-root ops.
     target.addLegalOp<gpu::YieldOp, gpu::GPUModuleOp, gpu::ModuleEndOp>();
     if (failed(applyPartialConversion(m, target, patterns, &converter)))
@@ -326,6 +328,3 @@ std::unique_ptr<OpPassBase<gpu::GPUModuleOp>>
 mlir::createLowerGpuOpsToNVVMOpsPass() {
   return std::make_unique<LowerGpuOpsToNVVMOpsPass>();
 }
-
-static PassRegistration<LowerGpuOpsToNVVMOpsPass>
-    pass("convert-gpu-to-nvvm", "Generate NVVM operations for gpu operations");
diff --git a/mlir/lib/Conversion/GPUToROCDL/CMakeLists.txt b/mlir/lib/Conversion/GPUToROCDL/CMakeLists.txt
index 9acc53a4e788d..142734c3ef89b 100644
--- a/mlir/lib/Conversion/GPUToROCDL/CMakeLists.txt
+++ b/mlir/lib/Conversion/GPUToROCDL/CMakeLists.txt
@@ -1,5 +1,8 @@
 add_mlir_conversion_library(MLIRGPUtoROCDLTransforms
   LowerGpuOpsToROCDLOps.cpp
+
+  DEPENDS
+  MLIRConversionPassIncGen
   )
 target_link_libraries(MLIRGPUtoROCDLTransforms
   PUBLIC
diff --git a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
index 238821ec8dc3a..b89a1704e9f58 100644
--- a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
+++ b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
@@ -34,6 +34,10 @@ namespace {
 class LowerGpuOpsToROCDLOpsPass
     : public OperationPass<LowerGpuOpsToROCDLOpsPass, gpu::GPUModuleOp> {
 public:
+/// Include the generated pass utilities.
+#define GEN_PASS_ConvertGpuOpsToROCDLOps
+#include "mlir/Conversion/Passes.h.inc"
+
   void runOnOperation() override {
     gpu::GPUModuleOp m = getOperation();
 
@@ -71,8 +75,6 @@ class LowerGpuOpsToROCDLOpsPass
     target.addLegalDialect<LLVM::LLVMDialect, ROCDL::ROCDLDialect>();
     target.addIllegalOp<LLVM::CosOp, LLVM::ExpOp, LLVM::FAbsOp, LLVM::FCeilOp,
                         LLVM::LogOp, LLVM::Log10Op, LLVM::Log2Op>();
-    target.addDynamicallyLegalOp<LLVM::CallOp>(
-        gpu::filterIllegalLLVMIntrinsics({"tanh", "tanhf"}, m.getContext()));
     target.addIllegalOp<FuncOp>();
     if (failed(applyPartialConversion(m, target, patterns, &converter)))
       signalPassFailure();
@@ -86,6 +88,3 @@ mlir::createLowerGpuOpsToROCDLOpsPass() {
   return std::make_unique<LowerGpuOpsToROCDLOpsPass>();
 }
 
-static PassRegistration<LowerGpuOpsToROCDLOpsPass>
-    pass("convert-gpu-to-rocdl",
-         "Generate ROCDL operations for gpu operations");
diff --git a/mlir/lib/Conversion/GPUToSPIRV/CMakeLists.txt b/mlir/lib/Conversion/GPUToSPIRV/CMakeLists.txt
index a0c5b6a1c32cf..f473a3e04e518 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/CMakeLists.txt
+++ b/mlir/lib/Conversion/GPUToSPIRV/CMakeLists.txt
@@ -7,6 +7,7 @@ add_mlir_conversion_library(MLIRGPUtoSPIRVTransforms
   ConvertGPUToSPIRVPass.cpp
 
   DEPENDS
+  MLIRConversionPassIncGen
   MLIRGPUToSPIRVIncGen
   )
 
diff --git a/mlir/lib/Conversion/GPUToSPIRV/ConvertGPUToSPIRV.cpp b/mlir/lib/Conversion/GPUToSPIRV/ConvertGPUToSPIRV.cpp
index 5483c2330c20d..5d380a2f3aa2f 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/ConvertGPUToSPIRV.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/ConvertGPUToSPIRV.cpp
@@ -69,7 +69,7 @@ class LaunchConfigConversion : public SPIRVOpLowering<SourceOp> {
 
 /// This is separate because in Vulkan workgroup size is exposed to shaders via
 /// a constant with WorkgroupSize decoration. So here we cannot generate a
-/// builtin variable; instead the infromation in the `spv.entry_point_abi`
+/// builtin variable; instead the information in the `spv.entry_point_abi`
 /// attribute on the surrounding FuncOp is used to replace the gpu::BlockDimOp.
 class WorkGroupSizeConversion : public SPIRVOpLowering<gpu::BlockDimOp> {
 public:
@@ -349,10 +349,18 @@ LogicalResult GPUFuncOpConversion::matchAndRewrite(
   if (!gpu::GPUDialect::isKernel(funcOp))
     return failure();
 
-  // TODO(antiagainst): we are dictating the ABI by ourselves here; it should be
-  // specified outside.
   SmallVector<spirv::InterfaceVarABIAttr, 4> argABI;
   for (auto argIndex : llvm::seq<unsigned>(0, funcOp.getNumArguments())) {
+    // If the ABI is already specified, use it.
+    auto abiAttr = funcOp.getArgAttrOfType<spirv::InterfaceVarABIAttr>(
+        argIndex, spirv::getInterfaceVarABIAttrName());
+    if (abiAttr) {
+      argABI.push_back(abiAttr);
+      continue;
+    }
+    // todo(ravishankarm): Use the "default ABI". Remove this in a follow up
+    // CL. Staging this to make this easy to revert in case of breakages out of
+    // tree.
     Optional<spirv::StorageClass> sc;
     if (funcOp.getArgument(argIndex).getType().isIntOrIndexOrFloat())
       sc = spirv::StorageClass::StorageBuffer;
diff --git a/mlir/lib/Conversion/GPUToSPIRV/ConvertGPUToSPIRVPass.cpp b/mlir/lib/Conversion/GPUToSPIRV/ConvertGPUToSPIRVPass.cpp
index 272eb163ab69b..1102ef182c5fa 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/ConvertGPUToSPIRVPass.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/ConvertGPUToSPIRVPass.cpp
@@ -10,6 +10,7 @@
 // into a spv.module operation
 //
 //===----------------------------------------------------------------------===//
+
 #include "mlir/Conversion/GPUToSPIRV/ConvertGPUToSPIRVPass.h"
 #include "mlir/Conversion/GPUToSPIRV/ConvertGPUToSPIRV.h"
 #include "mlir/Conversion/StandardToSPIRV/ConvertStandardToSPIRV.h"
@@ -19,7 +20,6 @@
 #include "mlir/Dialect/SPIRV/SPIRVLowering.h"
 #include "mlir/Dialect/SPIRV/SPIRVOps.h"
 #include "mlir/Pass/Pass.h"
-#include "mlir/Pass/PassRegistry.h"
 
 using namespace mlir;
 
@@ -34,6 +34,10 @@ namespace {
 ///
 /// 2) Lower the body of the spirv::ModuleOp.
 struct GPUToSPIRVPass : public ModulePass<GPUToSPIRVPass> {
+/// Include the generated pass utilities.
+#define GEN_PASS_ConvertGpuToSPIRV
+#include "mlir/Conversion/Passes.h.inc"
+
   void runOnModule() override;
 };
 } // namespace
@@ -70,6 +74,3 @@ void GPUToSPIRVPass::runOnModule() {
 std::unique_ptr<OpPassBase<ModuleOp>> mlir::createConvertGPUToSPIRVPass() {
   return std::make_unique<GPUToSPIRVPass>();
 }
-
-static PassRegistration<GPUToSPIRVPass>
-    pass("convert-gpu-to-spirv", "Convert GPU dialect to SPIR-V dialect");
diff --git a/mlir/lib/Conversion/GPUToVulkan/CMakeLists.txt b/mlir/lib/Conversion/GPUToVulkan/CMakeLists.txt
index 847c9c5031e99..ecfc2d75d5f36 100644
--- a/mlir/lib/Conversion/GPUToVulkan/CMakeLists.txt
+++ b/mlir/lib/Conversion/GPUToVulkan/CMakeLists.txt
@@ -1,6 +1,9 @@
 add_mlir_conversion_library(MLIRGPUtoVulkanTransforms
   ConvertLaunchFuncToVulkanCalls.cpp
   ConvertGPULaunchFuncToVulkanLaunchFunc.cpp
+
+  DEPENDS
+  MLIRConversionPassIncGen
   )
 
 target_link_libraries(MLIRGPUtoVulkanTransforms
diff --git a/mlir/lib/Conversion/GPUToVulkan/ConvertGPULaunchFuncToVulkanLaunchFunc.cpp b/mlir/lib/Conversion/GPUToVulkan/ConvertGPULaunchFuncToVulkanLaunchFunc.cpp
index 26833fd4daa75..823860ba25893 100644
--- a/mlir/lib/Conversion/GPUToVulkan/ConvertGPULaunchFuncToVulkanLaunchFunc.cpp
+++ b/mlir/lib/Conversion/GPUToVulkan/ConvertGPULaunchFuncToVulkanLaunchFunc.cpp
@@ -40,6 +40,10 @@ namespace {
 class ConvertGpuLaunchFuncToVulkanLaunchFunc
     : public ModulePass<ConvertGpuLaunchFuncToVulkanLaunchFunc> {
 public:
+/// Include the generated pass utilities.
+#define GEN_PASS_ConvertGpuLaunchFuncToVulkanLaunchFunc
+#include "mlir/Conversion/Passes.h.inc"
+
   void runOnModule() override;
 
 private:
@@ -48,14 +52,15 @@ class ConvertGpuLaunchFuncToVulkanLaunchFunc
   LogicalResult createBinaryShader(ModuleOp module,
                                    std::vector<char> &binaryShader);
 
-  /// Converts the given `luanchOp` to vulkan launch call.
+  /// Converts the given `launchOp` to vulkan launch call.
   void convertGpuLaunchFunc(gpu::LaunchFuncOp launchOp);
 
   /// Checks where the given type is supported by Vulkan runtime.
   bool isSupportedType(Type type) {
     // TODO(denis0x0D): Handle other types.
     if (auto memRefType = type.dyn_cast_or_null<MemRefType>())
-      return memRefType.hasRank() && memRefType.getRank() == 1;
+      return memRefType.hasRank() &&
+             (memRefType.getRank() >= 1 && memRefType.getRank() <= 3);
     return false;
   }
 
@@ -98,7 +103,8 @@ LogicalResult ConvertGpuLaunchFuncToVulkanLaunchFunc::declareVulkanLaunchFunc(
 
   // Check that all operands have supported types except those for the launch
   // configuration.
-  for (auto type : llvm::drop_begin(vulkanLaunchTypes, 6)) {
+  for (auto type :
+       llvm::drop_begin(vulkanLaunchTypes, gpu::LaunchOp::kNumConfigOperands)) {
     if (!isSupportedType(type))
       return launchOp.emitError() << type << " is unsupported to run on Vulkan";
   }
@@ -167,7 +173,3 @@ std::unique_ptr<mlir::OpPassBase<mlir::ModuleOp>>
 mlir::createConvertGpuLaunchFuncToVulkanLaunchFuncPass() {
   return std::make_unique<ConvertGpuLaunchFuncToVulkanLaunchFunc>();
 }
-
-static PassRegistration<ConvertGpuLaunchFuncToVulkanLaunchFunc>
-    pass("convert-gpu-launch-to-vulkan-launch",
-         "Convert gpu.launch_func to vulkanLaunch external call");
diff --git a/mlir/lib/Conversion/GPUToVulkan/ConvertLaunchFuncToVulkanCalls.cpp b/mlir/lib/Conversion/GPUToVulkan/ConvertLaunchFuncToVulkanCalls.cpp
index f1dc52e5f8563..ebc8ded483ff9 100644
--- a/mlir/lib/Conversion/GPUToVulkan/ConvertLaunchFuncToVulkanCalls.cpp
+++ b/mlir/lib/Conversion/GPUToVulkan/ConvertLaunchFuncToVulkanCalls.cpp
@@ -24,10 +24,13 @@
 #include "mlir/Pass/Pass.h"
 
 #include "llvm/ADT/SmallString.h"
+#include "llvm/Support/FormatVariadic.h"
 
 using namespace mlir;
 
 static constexpr const char *kBindMemRef1DFloat = "bindMemRef1DFloat";
+static constexpr const char *kBindMemRef2DFloat = "bindMemRef2DFloat";
+static constexpr const char *kBindMemRef3DFloat = "bindMemRef3DFloat";
 static constexpr const char *kCInterfaceVulkanLaunch =
     "_mlir_ciface_vulkanLaunch";
 static constexpr const char *kDeinitVulkan = "deinitVulkan";
@@ -57,6 +60,10 @@ namespace {
 class VulkanLaunchFuncToVulkanCallsPass
     : public ModulePass<VulkanLaunchFuncToVulkanCallsPass> {
 private:
+/// Include the generated pass utilities.
+#define GEN_PASS_ConvertVulkanLaunchFuncToVulkanCalls
+#include "mlir/Conversion/Passes.h.inc"
+
   LLVM::LLVMDialect *getLLVMDialect() { return llvmDialect; }
 
   llvm::LLVMContext &getLLVMContext() {
@@ -70,10 +77,12 @@ class VulkanLaunchFuncToVulkanCallsPass
     llvmPointerType = LLVM::LLVMType::getInt8PtrTy(llvmDialect);
     llvmInt32Type = LLVM::LLVMType::getInt32Ty(llvmDialect);
     llvmInt64Type = LLVM::LLVMType::getInt64Ty(llvmDialect);
-    initializeMemRefTypes();
+    llvmMemRef1DFloat = getMemRefType(1);
+    llvmMemRef2DFloat = getMemRefType(2);
+    llvmMemRef3DFloat = getMemRefType(3);
   }
 
-  void initializeMemRefTypes() {
+  LLVM::LLVMType getMemRefType(uint32_t rank) {
     // According to the MLIR doc memref argument is converted into a
     // pointer-to-struct argument of type:
     // template <typename Elem, size_t Rank>
@@ -85,14 +94,15 @@ class VulkanLaunchFuncToVulkanCallsPass
     //   int64_t strides[Rank]; // omitted when rank == 0
     // };
     auto llvmPtrToFloatType = getFloatType().getPointerTo();
-    auto llvmArrayOneElementSizeType =
-        LLVM::LLVMType::getArrayTy(getInt64Type(), 1);
+    auto llvmArrayRankElementSizeType =
+        LLVM::LLVMType::getArrayTy(getInt64Type(), rank);
 
-    // Create a type `!llvm<"{ float*, float*, i64, [1 x i64], [1 x i64]}">`.
-    llvmMemRef1DFloat = LLVM::LLVMType::getStructTy(
+    // Create a type
+    // `!llvm<"{ float*, float*, i64, [`rank` x i64], [`rank` x i64]}">`.
+    return LLVM::LLVMType::getStructTy(
         llvmDialect,
         {llvmPtrToFloatType, llvmPtrToFloatType, getInt64Type(),
-         llvmArrayOneElementSizeType, llvmArrayOneElementSizeType});
+         llvmArrayRankElementSizeType, llvmArrayRankElementSizeType});
   }
 
   LLVM::LLVMType getFloatType() { return llvmFloatType; }
@@ -101,6 +111,8 @@ class VulkanLaunchFuncToVulkanCallsPass
   LLVM::LLVMType getInt32Type() { return llvmInt32Type; }
   LLVM::LLVMType getInt64Type() { return llvmInt64Type; }
   LLVM::LLVMType getMemRef1DFloat() { return llvmMemRef1DFloat; }
+  LLVM::LLVMType getMemRef2DFloat() { return llvmMemRef2DFloat; }
+  LLVM::LLVMType getMemRef3DFloat() { return llvmMemRef3DFloat; }
 
   /// Creates a LLVM global for the given `name`.
   Value createEntryPointNameConstant(StringRef name, Location loc,
@@ -134,6 +146,9 @@ class VulkanLaunchFuncToVulkanCallsPass
   /// Collects SPIRV attributes from the given `vulkanLaunchCallOp`.
   void collectSPIRVAttributes(LLVM::CallOp vulkanLaunchCallOp);
 
+  /// Deduces a rank from the given 'ptrToMemRefDescriptor`.
+  LogicalResult deduceMemRefRank(Value ptrToMemRefDescriptor, uint32_t &rank);
+
 public:
   void runOnModule() override;
 
@@ -145,6 +160,8 @@ class VulkanLaunchFuncToVulkanCallsPass
   LLVM::LLVMType llvmInt32Type;
   LLVM::LLVMType llvmInt64Type;
   LLVM::LLVMType llvmMemRef1DFloat;
+  LLVM::LLVMType llvmMemRef2DFloat;
+  LLVM::LLVMType llvmMemRef3DFloat;
 
   // TODO: Use an associative array to support multiple vulkan launch calls.
   std::pair<StringAttr, StringAttr> spirvAttributes;
@@ -212,14 +229,52 @@ void VulkanLaunchFuncToVulkanCallsPass::createBindMemRefCalls(
     // Create LLVM constant for the descriptor binding index.
     Value descriptorBinding = builder.create<LLVM::ConstantOp>(
         loc, getInt32Type(), builder.getI32IntegerAttr(en.index()));
+
+    auto ptrToMemRefDescriptor = en.value();
+    uint32_t rank = 0;
+    if (failed(deduceMemRefRank(ptrToMemRefDescriptor, rank))) {
+      cInterfaceVulkanLaunchCallOp.emitError()
+          << "invalid memref descriptor " << ptrToMemRefDescriptor.getType();
+      return signalPassFailure();
+    }
+
+    auto symbolName = llvm::formatv("bindMemRef{0}DFloat", rank).str();
     // Create call to `bindMemRef`.
     builder.create<LLVM::CallOp>(
         loc, ArrayRef<Type>{getVoidType()},
-        // TODO: Add support for memref with other ranks.
-        builder.getSymbolRefAttr(kBindMemRef1DFloat),
+        builder.getSymbolRefAttr(
+            StringRef(symbolName.data(), symbolName.size())),
         ArrayRef<Value>{vulkanRuntime, descriptorSet, descriptorBinding,
-                        en.value()});
+                        ptrToMemRefDescriptor});
+  }
+}
+
+LogicalResult
+VulkanLaunchFuncToVulkanCallsPass::deduceMemRefRank(Value ptrToMemRefDescriptor,
+                                                    uint32_t &rank) {
+  auto llvmPtrDescriptorTy =
+      ptrToMemRefDescriptor.getType().dyn_cast<LLVM::LLVMType>();
+  if (!llvmPtrDescriptorTy)
+    return failure();
+
+  auto llvmDescriptorTy = llvmPtrDescriptorTy.getPointerElementTy();
+  // template <typename Elem, size_t Rank>
+  // struct {
+  //   Elem *allocated;
+  //   Elem *aligned;
+  //   int64_t offset;
+  //   int64_t sizes[Rank]; // omitted when rank == 0
+  //   int64_t strides[Rank]; // omitted when rank == 0
+  // };
+  if (!llvmDescriptorTy || !llvmDescriptorTy.isStructTy())
+    return failure();
+  if (llvmDescriptorTy.getStructNumElements() == 3) {
+    rank = 0;
+    return success();
   }
+
+  rank = llvmDescriptorTy.getStructElementType(3).getArrayNumElements();
+  return success();
 }
 
 void VulkanLaunchFuncToVulkanCallsPass::declareVulkanFunctions(Location loc) {
@@ -268,6 +323,26 @@ void VulkanLaunchFuncToVulkanCallsPass::declareVulkanFunctions(Location loc) {
                                       /*isVarArg=*/false));
   }
 
+  if (!module.lookupSymbol(kBindMemRef2DFloat)) {
+    builder.create<LLVM::LLVMFuncOp>(
+        loc, kBindMemRef2DFloat,
+        LLVM::LLVMType::getFunctionTy(getVoidType(),
+                                      {getPointerType(), getInt32Type(),
+                                       getInt32Type(),
+                                       getMemRef2DFloat().getPointerTo()},
+                                      /*isVarArg=*/false));
+  }
+
+  if (!module.lookupSymbol(kBindMemRef3DFloat)) {
+    builder.create<LLVM::LLVMFuncOp>(
+        loc, kBindMemRef3DFloat,
+        LLVM::LLVMType::getFunctionTy(getVoidType(),
+                                      {getPointerType(), getInt32Type(),
+                                       getInt32Type(),
+                                       getMemRef3DFloat().getPointerTo()},
+                                      /*isVarArg=*/false));
+  }
+
   if (!module.lookupSymbol(kInitVulkan)) {
     builder.create<LLVM::LLVMFuncOp>(
         loc, kInitVulkan,
@@ -365,7 +440,3 @@ std::unique_ptr<mlir::OpPassBase<mlir::ModuleOp>>
 mlir::createConvertVulkanLaunchFuncToVulkanCallsPass() {
   return std::make_unique<VulkanLaunchFuncToVulkanCallsPass>();
 }
-
-static PassRegistration<VulkanLaunchFuncToVulkanCallsPass>
-    pass("launch-func-to-vulkan",
-         "Convert vulkanLaunch external call to Vulkan runtime external calls");
diff --git a/mlir/lib/Conversion/LinalgToLLVM/CMakeLists.txt b/mlir/lib/Conversion/LinalgToLLVM/CMakeLists.txt
index 4f21ca208bfdc..72e0966f447b6 100644
--- a/mlir/lib/Conversion/LinalgToLLVM/CMakeLists.txt
+++ b/mlir/lib/Conversion/LinalgToLLVM/CMakeLists.txt
@@ -3,6 +3,9 @@ add_mlir_conversion_library(MLIRLinalgToLLVM
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/LinalgToLLVM
+
+  DEPENDS
+  MLIRConversionPassIncGen
 )
 set(LIBS
   MLIRAffineToStandard
diff --git a/mlir/lib/Conversion/LinalgToLLVM/LinalgToLLVM.cpp b/mlir/lib/Conversion/LinalgToLLVM/LinalgToLLVM.cpp
index 577b134fa5ed5..cb66ae9f50132 100644
--- a/mlir/lib/Conversion/LinalgToLLVM/LinalgToLLVM.cpp
+++ b/mlir/lib/Conversion/LinalgToLLVM/LinalgToLLVM.cpp
@@ -400,8 +400,13 @@ static FlatSymbolRefAttr getLibraryCallSymbolRef(Operation *op,
   // Insert before module terminator.
   rewriter.setInsertionPoint(module.getBody(),
                              std::prev(module.getBody()->end()));
-  rewriter.create<FuncOp>(op->getLoc(), fnNameAttr.getValue(), libFnType,
-                          ArrayRef<NamedAttribute>{});
+  FuncOp funcOp =
+      rewriter.create<FuncOp>(op->getLoc(), fnNameAttr.getValue(), libFnType,
+                              ArrayRef<NamedAttribute>{});
+  // Insert a function attribute that will trigger the emission of the
+  // corresponding `_mlir_ciface_xxx` interface so that external libraries see
+  // a normalized ABI. This interface is added during std to llvm conversion.
+  funcOp.setAttr("llvm.emit_c_interface", UnitAttr::get(op->getContext()));
   return fnNameAttr;
 }
 
@@ -524,12 +529,21 @@ populateLinalgToStandardConversionPatterns(OwningRewritePatternList &patterns,
                                            MLIRContext *ctx) {
   // TODO(ntv) ConvOp conversion needs to export a descriptor with relevant
   // attribute values such as kernel striding and dilation.
-  patterns.insert<CopyTransposeConversion, LinalgOpConversion<ConvOp>,
-                  LinalgOpConversion<CopyOp>, LinalgOpConversion<DotOp>,
-                  LinalgOpConversion<FillOp>, LinalgOpConversion<GenericOp>,
-                  LinalgOpConversion<IndexedGenericOp>,
-                  LinalgOpConversion<MatmulOp>, LinalgOpConversion<MatvecOp>>(
-      ctx);
+  // clang-format off
+  patterns.insert<
+      CopyTransposeConversion,
+      LinalgOpConversion<ConvOp>,
+      LinalgOpConversion<PoolingMaxOp>,
+      LinalgOpConversion<PoolingMinOp>,
+      LinalgOpConversion<PoolingSumOp>,
+      LinalgOpConversion<CopyOp>,
+      LinalgOpConversion<DotOp>,
+      LinalgOpConversion<FillOp>,
+      LinalgOpConversion<GenericOp>,
+      LinalgOpConversion<IndexedGenericOp>,
+      LinalgOpConversion<MatmulOp>,
+      LinalgOpConversion<MatvecOp>>(ctx);
+  // clang-format on
 }
 
 } // namespace
@@ -548,6 +562,10 @@ void mlir::populateLinalgToLLVMConversionPatterns(
 
 namespace {
 struct ConvertLinalgToLLVMPass : public ModulePass<ConvertLinalgToLLVMPass> {
+/// Include the generated pass utilities.
+#define GEN_PASS_ConvertLinalgToLLVM
+#include "mlir/Conversion/Passes.h.inc"
+
   void runOnModule() override;
 };
 } // namespace
@@ -560,8 +578,7 @@ void ConvertLinalgToLLVMPass::runOnModule() {
   LLVMTypeConverter converter(&getContext());
   populateAffineToStdConversionPatterns(patterns, &getContext());
   populateLoopToStdConversionPatterns(patterns, &getContext());
-  populateStdToLLVMConversionPatterns(converter, patterns, /*useAlloca=*/false,
-                                      /*emitCWrappers=*/true);
+  populateStdToLLVMConversionPatterns(converter, patterns);
   populateVectorToLLVMMatrixConversionPatterns(converter, patterns);
   populateVectorToLLVMConversionPatterns(converter, patterns);
   populateLinalgToStandardConversionPatterns(patterns, &getContext());
@@ -578,7 +595,3 @@ void ConvertLinalgToLLVMPass::runOnModule() {
 std::unique_ptr<OpPassBase<ModuleOp>> mlir::createConvertLinalgToLLVMPass() {
   return std::make_unique<ConvertLinalgToLLVMPass>();
 }
-
-static PassRegistration<ConvertLinalgToLLVMPass> pass(
-    "convert-linalg-to-llvm",
-    "Convert the operations from the linalg dialect into the LLVM dialect");
diff --git a/mlir/lib/Conversion/LinalgToSPIRV/CMakeLists.txt b/mlir/lib/Conversion/LinalgToSPIRV/CMakeLists.txt
index 52a813470f022..f0cc532233665 100644
--- a/mlir/lib/Conversion/LinalgToSPIRV/CMakeLists.txt
+++ b/mlir/lib/Conversion/LinalgToSPIRV/CMakeLists.txt
@@ -5,6 +5,9 @@ add_mlir_conversion_library(MLIRLinalgToSPIRVTransforms
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/SPIRV
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/IR
+
+  DEPENDS
+  MLIRConversionPassIncGen
   )
 
 target_link_libraries(MLIRLinalgToSPIRVTransforms
diff --git a/mlir/lib/Conversion/LinalgToSPIRV/LinalgToSPIRV.cpp b/mlir/lib/Conversion/LinalgToSPIRV/LinalgToSPIRV.cpp
index 90ec624a8cc14..c4dcefc38d6eb 100644
--- a/mlir/lib/Conversion/LinalgToSPIRV/LinalgToSPIRV.cpp
+++ b/mlir/lib/Conversion/LinalgToSPIRV/LinalgToSPIRV.cpp
@@ -70,7 +70,7 @@ SingleWorkgroupReduction::matchAsPerformingReduction(
   if (!genericOp.hasBufferSemantics())
     return llvm::None;
 
-  // Make sure this is reudction with one input and one output.
+  // Make sure this is reduction with one input and one output.
   if (genericOp.args_in().getZExtValue() != 1 ||
       genericOp.args_out().getZExtValue() != 1)
     return llvm::None;
diff --git a/mlir/lib/Conversion/LinalgToSPIRV/LinalgToSPIRVPass.cpp b/mlir/lib/Conversion/LinalgToSPIRV/LinalgToSPIRVPass.cpp
index 4477c070796ef..0962746c486a6 100644
--- a/mlir/lib/Conversion/LinalgToSPIRV/LinalgToSPIRVPass.cpp
+++ b/mlir/lib/Conversion/LinalgToSPIRV/LinalgToSPIRVPass.cpp
@@ -17,6 +17,10 @@ using namespace mlir;
 namespace {
 /// A pass converting MLIR Linalg ops into SPIR-V ops.
 class LinalgToSPIRVPass : public ModulePass<LinalgToSPIRVPass> {
+/// Include the generated pass utilities.
+#define GEN_PASS_ConvertLinalgToSPIRV
+#include "mlir/Conversion/Passes.h.inc"
+
   void runOnModule() override;
 };
 } // namespace
@@ -46,6 +50,3 @@ void LinalgToSPIRVPass::runOnModule() {
 std::unique_ptr<OpPassBase<ModuleOp>> mlir::createLinalgToSPIRVPass() {
   return std::make_unique<LinalgToSPIRVPass>();
 }
-
-static PassRegistration<LinalgToSPIRVPass>
-    pass("convert-linalg-to-spirv", "Convert Linalg ops to SPIR-V ops");
diff --git a/mlir/lib/Conversion/LoopToStandard/CMakeLists.txt b/mlir/lib/Conversion/LoopToStandard/CMakeLists.txt
index d0401d80fe939..d4c7528475835 100644
--- a/mlir/lib/Conversion/LoopToStandard/CMakeLists.txt
+++ b/mlir/lib/Conversion/LoopToStandard/CMakeLists.txt
@@ -3,6 +3,9 @@ add_mlir_conversion_library(MLIRLoopToStandard
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/LoopToStandard
+
+  DEPENDS
+  MLIRConversionPassIncGen
 )
 target_link_libraries(
   MLIRLoopToStandard
diff --git a/mlir/lib/Conversion/LoopToStandard/LoopToStandard.cpp b/mlir/lib/Conversion/LoopToStandard/LoopToStandard.cpp
index 178759ad67d6f..e72c83027611b 100644
--- a/mlir/lib/Conversion/LoopToStandard/LoopToStandard.cpp
+++ b/mlir/lib/Conversion/LoopToStandard/LoopToStandard.cpp
@@ -31,6 +31,10 @@ using namespace mlir::loop;
 namespace {
 
 struct LoopToStandardPass : public OperationPass<LoopToStandardPass> {
+/// Include the generated pass utilities.
+#define GEN_PASS_ConvertLoopToStandard
+#include "mlir/Conversion/Passes.h.inc"
+
   void runOnOperation() override;
 };
 
@@ -301,8 +305,8 @@ ParallelLowering::matchAndRewrite(ParallelOp parallelOp,
       // A loop is constructed with an empty "yield" terminator by default.
       // Replace it with another "yield" that forwards the results of the nested
       // loop to the parent loop. We need to explicitly make sure the new
-      // terminator is the last operation in the block because further transfoms
-      // rely on this.
+      // terminator is the last operation in the block because further
+      // transforms rely on this.
       rewriter.setInsertionPointToEnd(rewriter.getInsertionBlock());
       rewriter.replaceOpWithNewOp<YieldOp>(
           rewriter.getInsertionBlock()->getTerminator(), forOp.getResults());
@@ -364,7 +368,3 @@ void LoopToStandardPass::runOnOperation() {
 std::unique_ptr<Pass> mlir::createLowerToCFGPass() {
   return std::make_unique<LoopToStandardPass>();
 }
-
-static PassRegistration<LoopToStandardPass>
-    pass("convert-loop-to-std", "Convert Loop dialect to Standard dialect, "
-                                "replacing structured control flow with a CFG");
diff --git a/mlir/lib/Conversion/LoopsToGPU/CMakeLists.txt b/mlir/lib/Conversion/LoopsToGPU/CMakeLists.txt
index 9a460bcf71655..2c62755eebc84 100644
--- a/mlir/lib/Conversion/LoopsToGPU/CMakeLists.txt
+++ b/mlir/lib/Conversion/LoopsToGPU/CMakeLists.txt
@@ -4,6 +4,9 @@ add_mlir_conversion_library(MLIRLoopsToGPU
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/LoopsToGPU
+
+  DEPENDS
+  MLIRConversionPassIncGen
 )
 target_link_libraries(MLIRLoopsToGPU
   PUBLIC
diff --git a/mlir/lib/Conversion/LoopsToGPU/LoopsToGPU.cpp b/mlir/lib/Conversion/LoopsToGPU/LoopsToGPU.cpp
index b9c81ea45592e..4b94833c0feaa 100644
--- a/mlir/lib/Conversion/LoopsToGPU/LoopsToGPU.cpp
+++ b/mlir/lib/Conversion/LoopsToGPU/LoopsToGPU.cpp
@@ -624,7 +624,7 @@ static LogicalResult processParallelLoop(
       // If there was also a bound, insert that, too.
       // TODO(herhut): Check that we do not assign bounds twice.
       if (annotation.bound().getValue()) {
-        // We pass as the single opererand to the bound-map the number of
+        // We pass as the single operand to the bound-map the number of
         // iterations, which is (upperBound - lowerBound) ceilDiv step. To
         // support inner loops with dynamic upper bounds (as generated by e.g.
         // tiling), try to derive a max for the bounds. If the used bound for
@@ -774,7 +774,7 @@ ParallelToGpuLaunchLowering::matchAndRewrite(ParallelOp parallelOp,
     // Now walk over the body and clone it.
     // TODO: This is only correct if there either is no further loop.parallel
     //       nested or this code is side-effect free. Otherwise we might need
-    //       predication. We are overly consertaive for now and only allow
+    //       predication. We are overly conservative for now and only allow
     //       side-effects in the innermost scope.
     if (auto nestedParallel = dyn_cast<ParallelOp>(op)) {
       // Before entering a nested scope, make sure there have been no
diff --git a/mlir/lib/Conversion/LoopsToGPU/LoopsToGPUPass.cpp b/mlir/lib/Conversion/LoopsToGPU/LoopsToGPUPass.cpp
index 9a5e2a608df98..98a855c105c35 100644
--- a/mlir/lib/Conversion/LoopsToGPU/LoopsToGPUPass.cpp
+++ b/mlir/lib/Conversion/LoopsToGPU/LoopsToGPUPass.cpp
@@ -29,6 +29,10 @@ namespace {
 // GPU launch operations.  Nested launches are not allowed, so this does not
 // walk the function recursively to avoid considering nested loops.
 struct ForLoopMapper : public FunctionPass<ForLoopMapper> {
+/// Include the generated pass utilities.
+#define GEN_PASS_ConvertSimpleLoopsToGPU
+#include "mlir/Conversion/Passes.h.inc"
+
   ForLoopMapper() = default;
   ForLoopMapper(const ForLoopMapper &) {}
   ForLoopMapper(unsigned numBlockDims, unsigned numThreadDims) {
@@ -50,15 +54,6 @@ struct ForLoopMapper : public FunctionPass<ForLoopMapper> {
         }
       }
   }
-
-  Option<unsigned> numBlockDims{
-      *this, "gpu-block-dims",
-      llvm::cl::desc("Number of GPU block dimensions for mapping"),
-      llvm::cl::init(1u)};
-  Option<unsigned> numThreadDims{
-      *this, "gpu-thread-dims",
-      llvm::cl::desc("Number of GPU thread dimensions for mapping"),
-      llvm::cl::init(1u)};
 };
 
 // A pass that traverses top-level loops in the function and convertes them to
@@ -68,6 +63,10 @@ struct ForLoopMapper : public FunctionPass<ForLoopMapper> {
 // to be perfectly nested upto depth equal to size of `workGroupSize`.
 struct ImperfectlyNestedForLoopMapper
     : public FunctionPass<ImperfectlyNestedForLoopMapper> {
+/// Include the generated pass utilities.
+#define GEN_PASS_ConvertLoopsToGPU
+#include "mlir/Conversion/Passes.h.inc"
+
   ImperfectlyNestedForLoopMapper() = default;
   ImperfectlyNestedForLoopMapper(const ImperfectlyNestedForLoopMapper &) {}
   ImperfectlyNestedForLoopMapper(ArrayRef<int64_t> numWorkGroups,
@@ -103,17 +102,13 @@ struct ImperfectlyNestedForLoopMapper
       }
     }
   }
-  ListOption<int64_t> numWorkGroups{
-      *this, "gpu-num-workgroups",
-      llvm::cl::desc("Num workgroups in the GPU launch"), llvm::cl::ZeroOrMore,
-      llvm::cl::MiscFlags::CommaSeparated};
-  ListOption<int64_t> workGroupSize{
-      *this, "gpu-workgroup-size",
-      llvm::cl::desc("Workgroup Size in the GPU launch"), llvm::cl::ZeroOrMore,
-      llvm::cl::MiscFlags::CommaSeparated};
 };
 
 struct ParallelLoopToGpuPass : public OperationPass<ParallelLoopToGpuPass> {
+/// Include the generated pass utilities.
+#define GEN_PASS_ConvertParallelLoopToGpu
+#include "mlir/Conversion/Passes.h.inc"
+
   void runOnOperation() override {
     OwningRewritePatternList patterns;
     populateParallelLoopToGPUPatterns(patterns, &getContext());
@@ -135,6 +130,9 @@ mlir::createSimpleLoopsToGPUPass(unsigned numBlockDims,
                                  unsigned numThreadDims) {
   return std::make_unique<ForLoopMapper>(numBlockDims, numThreadDims);
 }
+std::unique_ptr<OpPassBase<FuncOp>> mlir::createSimpleLoopsToGPUPass() {
+  return std::make_unique<ForLoopMapper>();
+}
 
 std::unique_ptr<OpPassBase<FuncOp>>
 mlir::createLoopToGPUPass(ArrayRef<int64_t> numWorkGroups,
@@ -142,18 +140,10 @@ mlir::createLoopToGPUPass(ArrayRef<int64_t> numWorkGroups,
   return std::make_unique<ImperfectlyNestedForLoopMapper>(numWorkGroups,
                                                           workGroupSize);
 }
+std::unique_ptr<OpPassBase<FuncOp>> mlir::createLoopToGPUPass() {
+  return std::make_unique<ImperfectlyNestedForLoopMapper>();
+}
 
 std::unique_ptr<Pass> mlir::createParallelLoopToGpuPass() {
   return std::make_unique<ParallelLoopToGpuPass>();
 }
-
-static PassRegistration<ForLoopMapper>
-    registration(PASS_NAME, "Convert top-level loops to GPU kernels");
-
-static PassRegistration<ImperfectlyNestedForLoopMapper>
-    loopOpToGPU(LOOPOP_TO_GPU_PASS_NAME,
-                "Convert top-level loop::ForOp to GPU kernels");
-
-static PassRegistration<ParallelLoopToGpuPass>
-    pass("convert-parallel-loops-to-gpu", "Convert mapped loop.parallel ops"
-                                          " to gpu launch operations.");
diff --git a/mlir/lib/Conversion/StandardToLLVM/CMakeLists.txt b/mlir/lib/Conversion/StandardToLLVM/CMakeLists.txt
index 3a5b2f6ba06c6..ef7ad11d93ef4 100644
--- a/mlir/lib/Conversion/StandardToLLVM/CMakeLists.txt
+++ b/mlir/lib/Conversion/StandardToLLVM/CMakeLists.txt
@@ -3,6 +3,9 @@ add_mlir_conversion_library(MLIRStandardToLLVM
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/StandardToLLVM
+
+  DEPENDS
+  MLIRConversionPassIncGen
 )
 target_link_libraries(
   MLIRStandardToLLVM
diff --git a/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp b/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp
index e353e933a8ae5..12fc8b177c17f 100644
--- a/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp
+++ b/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp
@@ -52,9 +52,9 @@ static LLVM::LLVMType unwrap(Type type) {
 }
 
 /// Initialize customization to default callbacks.
-LLVMTypeConverterCustomization::LLVMTypeConverterCustomization() {
-  funcArgConverter = structFuncArgTypeConverter;
-}
+LLVMTypeConverterCustomization::LLVMTypeConverterCustomization()
+    : funcArgConverter(structFuncArgTypeConverter),
+      indexBitwidth(kDeriveIndexBitwidthFromDataLayout) {}
 
 /// Callback to convert function argument types. It converts a MemRef function
 /// argument to a list of non-aggregate types containing descriptor
@@ -133,6 +133,9 @@ LLVMTypeConverter::LLVMTypeConverter(
       customizations(customs) {
   assert(llvmDialect && "LLVM IR dialect is not registered");
   module = &llvmDialect->getLLVMModule();
+  if (customizations.indexBitwidth == kDeriveIndexBitwidthFromDataLayout)
+    customizations.indexBitwidth =
+        module->getDataLayout().getPointerSizeInBits();
 
   // Register conversions for the standard types.
   addConversion([&](FloatType type) { return convertFloatType(type); });
@@ -148,14 +151,18 @@ LLVMTypeConverter::LLVMTypeConverter(
   addConversion([](LLVM::LLVMType type) { return type; });
 }
 
+/// Returns the MLIR context.
+MLIRContext &LLVMTypeConverter::getContext() {
+  return *getDialect()->getContext();
+}
+
 /// Get the LLVM context.
 llvm::LLVMContext &LLVMTypeConverter::getLLVMContext() {
   return module->getContext();
 }
 
 LLVM::LLVMType LLVMTypeConverter::getIndexType() {
-  return LLVM::LLVMType::getIntNTy(
-      llvmDialect, module->getDataLayout().getPointerSizeInBits());
+  return LLVM::LLVMType::getIntNTy(llvmDialect, getIndexTypeBitwidth());
 }
 
 Type LLVMTypeConverter::convertIndexType(IndexType type) {
@@ -699,52 +706,34 @@ void UnrankedMemRefDescriptor::unpack(OpBuilder &builder, Location loc,
   results.push_back(d.memRefDescPtr(builder, loc));
 }
 
-namespace {
-// Base class for Standard to LLVM IR op conversions.  Matches the Op type
-// provided as template argument.  Carries a reference to the LLVM dialect in
-// case it is necessary for rewriters.
-template <typename SourceOp>
-class LLVMLegalizationPattern : public ConvertToLLVMPattern {
-public:
-  // Construct a conversion pattern.
-  explicit LLVMLegalizationPattern(LLVM::LLVMDialect &dialect_,
-                                   LLVMTypeConverter &typeConverter_)
-      : ConvertToLLVMPattern(SourceOp::getOperationName(),
-                             dialect_.getContext(), typeConverter_),
-        dialect(dialect_) {}
-
-  // Get the LLVM IR dialect.
-  LLVM::LLVMDialect &getDialect() const { return dialect; }
-  // Get the LLVM context.
-  llvm::LLVMContext &getContext() const { return dialect.getLLVMContext(); }
-  // Get the LLVM module in which the types are constructed.
-  llvm::Module &getModule() const { return dialect.getLLVMModule(); }
-
-  // Get the MLIR type wrapping the LLVM integer type whose bit width is defined
-  // by the pointer size used in the LLVM module.
-  LLVM::LLVMType getIndexType() const {
-    return LLVM::LLVMType::getIntNTy(
-        &dialect, getModule().getDataLayout().getPointerSizeInBits());
-  }
+LLVM::LLVMDialect &ConvertToLLVMPattern::getDialect() const {
+  return *typeConverter.getDialect();
+}
 
-  LLVM::LLVMType getVoidType() const {
-    return LLVM::LLVMType::getVoidTy(&dialect);
-  }
+llvm::LLVMContext &ConvertToLLVMPattern::getContext() const {
+  return typeConverter.getLLVMContext();
+}
 
-  // Get the MLIR type wrapping the LLVM i8* type.
-  LLVM::LLVMType getVoidPtrType() const {
-    return LLVM::LLVMType::getInt8PtrTy(&dialect);
-  }
+llvm::Module &ConvertToLLVMPattern::getModule() const {
+  return getDialect().getLLVMModule();
+}
 
-  // Create an LLVM IR pseudo-operation defining the given index constant.
-  Value createIndexConstant(ConversionPatternRewriter &builder, Location loc,
-                            uint64_t value) const {
-    return createIndexAttrConstant(builder, loc, getIndexType(), value);
-  }
+LLVM::LLVMType ConvertToLLVMPattern::getIndexType() const {
+  return typeConverter.getIndexType();
+}
 
-protected:
-  LLVM::LLVMDialect &dialect;
-};
+LLVM::LLVMType ConvertToLLVMPattern::getVoidType() const {
+  return LLVM::LLVMType::getVoidTy(&getDialect());
+}
+
+LLVM::LLVMType ConvertToLLVMPattern::getVoidPtrType() const {
+  return LLVM::LLVMType::getInt8PtrTy(&getDialect());
+}
+
+Value ConvertToLLVMPattern::createIndexConstant(
+    ConversionPatternRewriter &builder, Location loc, uint64_t value) const {
+  return createIndexAttrConstant(builder, loc, getIndexType(), value);
+}
 
 /// Only retain those attributes that are not constructed by
 /// `LLVMFuncOp::build`. If `filterArgAttrs` is set, also filter out argument
@@ -876,9 +865,11 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc,
   builder.create<LLVM::ReturnOp>(loc, call.getResults());
 }
 
-struct FuncOpConversionBase : public LLVMLegalizationPattern<FuncOp> {
+namespace {
+
+struct FuncOpConversionBase : public ConvertOpToLLVMPattern<FuncOp> {
 protected:
-  using LLVMLegalizationPattern<FuncOp>::LLVMLegalizationPattern;
+  using ConvertOpToLLVMPattern<FuncOp>::ConvertOpToLLVMPattern;
   using UnsignedTypePair = std::pair<unsigned, Type>;
 
   // Gather the positions and types of memref-typed arguments in a given
@@ -941,10 +932,10 @@ struct FuncOpConversionBase : public LLVMLegalizationPattern<FuncOp> {
 /// FuncOp legalization pattern that converts MemRef arguments to pointers to
 /// MemRef descriptors (LLVM struct data types) containing all the MemRef type
 /// information.
+static constexpr StringRef kEmitIfaceAttrName = "llvm.emit_c_interface";
 struct FuncOpConversion : public FuncOpConversionBase {
-  FuncOpConversion(LLVM::LLVMDialect &dialect, LLVMTypeConverter &converter,
-                   bool emitCWrappers)
-      : FuncOpConversionBase(dialect, converter), emitWrappers(emitCWrappers) {}
+  FuncOpConversion(LLVMTypeConverter &converter, bool emitCWrappers)
+      : FuncOpConversionBase(converter), emitWrappers(emitCWrappers) {}
 
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
@@ -952,7 +943,7 @@ struct FuncOpConversion : public FuncOpConversionBase {
     auto funcOp = cast<FuncOp>(op);
 
     auto newFuncOp = convertFuncOpToLLVMFuncOp(funcOp, rewriter);
-    if (emitWrappers) {
+    if (emitWrappers || funcOp.getAttrOfType<UnitAttr>(kEmitIfaceAttrName)) {
       if (newFuncOp.isExternal())
         wrapExternalFunction(rewriter, op->getLoc(), typeConverter, funcOp,
                              newFuncOp);
@@ -1022,7 +1013,6 @@ struct BarePtrFuncOpConversion : public FuncOpConversionBase {
 };
 
 //////////////// Support for Lowering operations on n-D vectors ////////////////
-namespace {
 // Helper struct to "unroll" operations on n-D vectors in terms of operations on
 // 1-D LLVM vectors.
 struct NDVectorTypeInfo {
@@ -1100,75 +1090,48 @@ void nDVectorIterate(const NDVectorTypeInfo &info, OpBuilder &builder,
 }
 ////////////// End Support for Lowering operations on n-D vectors //////////////
 
-// Basic lowering implementation for one-to-one rewriting from Standard Ops to
-// LLVM Dialect Ops.
-template <typename SourceOp, typename TargetOp>
-struct OneToOneLLVMOpLowering : public LLVMLegalizationPattern<SourceOp> {
-  using LLVMLegalizationPattern<SourceOp>::LLVMLegalizationPattern;
-  using Super = OneToOneLLVMOpLowering<SourceOp, TargetOp>;
-
-  // Convert the type of the result to an LLVM type, pass operands as is,
-  // preserve attributes.
-  LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
-                  ConversionPatternRewriter &rewriter) const override {
-    unsigned numResults = op->getNumResults();
-
-    Type packedType;
-    if (numResults != 0) {
-      packedType =
-          this->typeConverter.packFunctionResults(op->getResultTypes());
-      if (!packedType)
-        return failure();
-    }
-
-    auto newOp = rewriter.create<TargetOp>(op->getLoc(), packedType, operands,
-                                           op->getAttrs());
-
-    // If the operation produced 0 or 1 result, return them immediately.
-    if (numResults == 0)
-      return rewriter.eraseOp(op), success();
-    if (numResults == 1)
-      return rewriter.replaceOp(op, newOp.getOperation()->getResult(0)),
-             success();
-
-    // Otherwise, it had been converted to an operation producing a structure.
-    // Extract individual results from the structure and return them as list.
-    SmallVector<Value, 4> results;
-    results.reserve(numResults);
-    for (unsigned i = 0; i < numResults; ++i) {
-      auto type = this->typeConverter.convertType(op->getResult(i).getType());
-      results.push_back(rewriter.create<LLVM::ExtractValueOp>(
-          op->getLoc(), type, newOp.getOperation()->getResult(0),
-          rewriter.getI64ArrayAttr(i)));
-    }
-    rewriter.replaceOp(op, results);
-    return success();
+/// Replaces the given operaiton "op" with a new operation of type "targetOp"
+/// and given operands.
+LogicalResult LLVM::detail::oneToOneRewrite(
+    Operation *op, StringRef targetOp, ValueRange operands,
+    LLVMTypeConverter &typeConverter, ConversionPatternRewriter &rewriter) {
+  unsigned numResults = op->getNumResults();
+
+  Type packedType;
+  if (numResults != 0) {
+    packedType = typeConverter.packFunctionResults(op->getResultTypes());
+    if (!packedType)
+      return failure();
   }
-};
-
-template <typename SourceOp, unsigned OpCount>
-struct OpCountValidator {
-  static_assert(
-      std::is_base_of<
-          typename OpTrait::NOperands<OpCount>::template Impl<SourceOp>,
-          SourceOp>::value,
-      "wrong operand count");
-};
 
-template <typename SourceOp>
-struct OpCountValidator<SourceOp, 1> {
-  static_assert(std::is_base_of<OpTrait::OneOperand<SourceOp>, SourceOp>::value,
-                "expected a single operand");
-};
-
-template <typename SourceOp, unsigned OpCount>
-void ValidateOpCount() {
-  OpCountValidator<SourceOp, OpCount>();
+  // Create the operation through state since we don't know its C++ type.
+  OperationState state(op->getLoc(), targetOp);
+  state.addTypes(packedType);
+  state.addOperands(operands);
+  state.addAttributes(op->getAttrs());
+  Operation *newOp = rewriter.createOperation(state);
+
+  // If the operation produced 0 or 1 result, return them immediately.
+  if (numResults == 0)
+    return rewriter.eraseOp(op), success();
+  if (numResults == 1)
+    return rewriter.replaceOp(op, newOp->getResult(0)), success();
+
+  // Otherwise, it had been converted to an operation producing a structure.
+  // Extract individual results from the structure and return them as list.
+  SmallVector<Value, 4> results;
+  results.reserve(numResults);
+  for (unsigned i = 0; i < numResults; ++i) {
+    auto type = typeConverter.convertType(op->getResult(i).getType());
+    results.push_back(rewriter.create<LLVM::ExtractValueOp>(
+        op->getLoc(), type, newOp->getResult(0), rewriter.getI64ArrayAttr(i)));
+  }
+  rewriter.replaceOp(op, results);
+  return success();
 }
 
-static LogicalResult HandleMultidimensionalVectors(
-    Operation *op, ArrayRef<Value> operands, LLVMTypeConverter &typeConverter,
+static LogicalResult handleMultidimensionalVectors(
+    Operation *op, ValueRange operands, LLVMTypeConverter &typeConverter,
     std::function<Value(LLVM::LLVMType, ValueRange)> createOperand,
     ConversionPatternRewriter &rewriter) {
   auto vectorType = op->getResult(0).getType().dyn_cast<VectorType>();
@@ -1197,161 +1160,75 @@ static LogicalResult HandleMultidimensionalVectors(
   return success();
 }
 
-// Basic lowering implementation for rewriting from Standard Ops to LLVM Dialect
-// Ops for N-ary ops with one result. This supports higher-dimensional vector
-// types.
-template <typename SourceOp, typename TargetOp, unsigned OpCount>
-struct NaryOpLLVMOpLowering : public LLVMLegalizationPattern<SourceOp> {
-  using LLVMLegalizationPattern<SourceOp>::LLVMLegalizationPattern;
-  using Super = NaryOpLLVMOpLowering<SourceOp, TargetOp, OpCount>;
-
-  // Convert the type of the result to an LLVM type, pass operands as is,
-  // preserve attributes.
-  LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
-                  ConversionPatternRewriter &rewriter) const override {
-    ValidateOpCount<SourceOp, OpCount>();
-    static_assert(
-        std::is_base_of<OpTrait::OneResult<SourceOp>, SourceOp>::value,
-        "expected single result op");
-    static_assert(std::is_base_of<OpTrait::SameOperandsAndResultType<SourceOp>,
-                                  SourceOp>::value,
-                  "expected same operands and result type");
-
-    // Cannot convert ops if their operands are not of LLVM type.
-    for (Value operand : operands) {
-      if (!operand || !operand.getType().isa<LLVM::LLVMType>())
-        return failure();
-    }
+LogicalResult LLVM::detail::vectorOneToOneRewrite(
+    Operation *op, StringRef targetOp, ValueRange operands,
+    LLVMTypeConverter &typeConverter, ConversionPatternRewriter &rewriter) {
+  assert(!operands.empty());
 
-    auto llvmArrayTy = operands[0].getType().cast<LLVM::LLVMType>();
+  // Cannot convert ops if their operands are not of LLVM type.
+  if (!llvm::all_of(operands.getTypes(),
+                    [](Type t) { return t.isa<LLVM::LLVMType>(); }))
+    return failure();
 
-    if (!llvmArrayTy.isArrayTy()) {
-      auto newOp = rewriter.create<TargetOp>(
-          op->getLoc(), operands[0].getType(), operands, op->getAttrs());
-      rewriter.replaceOp(op, newOp.getResult());
-      return success();
-    }
+  auto llvmArrayTy = operands[0].getType().cast<LLVM::LLVMType>();
+  if (!llvmArrayTy.isArrayTy())
+    return oneToOneRewrite(op, targetOp, operands, typeConverter, rewriter);
 
-    if (succeeded(HandleMultidimensionalVectors(
-            op, operands, this->typeConverter,
-            [&](LLVM::LLVMType llvmVectorTy, ValueRange operands) {
-              return rewriter.create<TargetOp>(op->getLoc(), llvmVectorTy,
-                                               operands, op->getAttrs());
-            },
-            rewriter)))
-      return success();
-    return failure();
-  }
-};
+  auto callback = [op, targetOp, &rewriter](LLVM::LLVMType llvmVectorTy,
+                                            ValueRange operands) {
+    OperationState state(op->getLoc(), targetOp);
+    state.addTypes(llvmVectorTy);
+    state.addOperands(operands);
+    state.addAttributes(op->getAttrs());
+    return rewriter.createOperation(state)->getResult(0);
+  };
 
-template <typename SourceOp, typename TargetOp>
-using UnaryOpLLVMOpLowering = NaryOpLLVMOpLowering<SourceOp, TargetOp, 1>;
-template <typename SourceOp, typename TargetOp>
-using BinaryOpLLVMOpLowering = NaryOpLLVMOpLowering<SourceOp, TargetOp, 2>;
+  return handleMultidimensionalVectors(op, operands, typeConverter, callback,
+                                       rewriter);
+}
 
-// Specific lowerings.
-// FIXME: this should be tablegen'ed.
-struct AbsFOpLowering : public UnaryOpLLVMOpLowering<AbsFOp, LLVM::FAbsOp> {
-  using Super::Super;
-};
-struct CeilFOpLowering : public UnaryOpLLVMOpLowering<CeilFOp, LLVM::FCeilOp> {
-  using Super::Super;
-};
-struct CosOpLowering : public UnaryOpLLVMOpLowering<CosOp, LLVM::CosOp> {
-  using Super::Super;
-};
-struct ExpOpLowering : public UnaryOpLLVMOpLowering<ExpOp, LLVM::ExpOp> {
-  using Super::Super;
-};
-struct LogOpLowering : public UnaryOpLLVMOpLowering<LogOp, LLVM::LogOp> {
-  using Super::Super;
-};
-struct Log10OpLowering : public UnaryOpLLVMOpLowering<Log10Op, LLVM::Log10Op> {
-  using Super::Super;
-};
-struct Log2OpLowering : public UnaryOpLLVMOpLowering<Log2Op, LLVM::Log2Op> {
-  using Super::Super;
-};
-struct NegFOpLowering : public UnaryOpLLVMOpLowering<NegFOp, LLVM::FNegOp> {
-  using Super::Super;
-};
-struct AddIOpLowering : public BinaryOpLLVMOpLowering<AddIOp, LLVM::AddOp> {
-  using Super::Super;
-};
-struct SubIOpLowering : public BinaryOpLLVMOpLowering<SubIOp, LLVM::SubOp> {
-  using Super::Super;
-};
-struct MulIOpLowering : public BinaryOpLLVMOpLowering<MulIOp, LLVM::MulOp> {
-  using Super::Super;
-};
-struct SignedDivIOpLowering
-    : public BinaryOpLLVMOpLowering<SignedDivIOp, LLVM::SDivOp> {
-  using Super::Super;
-};
-struct SqrtOpLowering : public UnaryOpLLVMOpLowering<SqrtOp, LLVM::SqrtOp> {
-  using Super::Super;
-};
-struct UnsignedDivIOpLowering
-    : public BinaryOpLLVMOpLowering<UnsignedDivIOp, LLVM::UDivOp> {
-  using Super::Super;
-};
-struct SignedRemIOpLowering
-    : public BinaryOpLLVMOpLowering<SignedRemIOp, LLVM::SRemOp> {
-  using Super::Super;
-};
-struct UnsignedRemIOpLowering
-    : public BinaryOpLLVMOpLowering<UnsignedRemIOp, LLVM::URemOp> {
-  using Super::Super;
-};
-struct AndOpLowering : public BinaryOpLLVMOpLowering<AndOp, LLVM::AndOp> {
-  using Super::Super;
-};
-struct OrOpLowering : public BinaryOpLLVMOpLowering<OrOp, LLVM::OrOp> {
-  using Super::Super;
-};
-struct XOrOpLowering : public BinaryOpLLVMOpLowering<XOrOp, LLVM::XOrOp> {
-  using Super::Super;
-};
-struct AddFOpLowering : public BinaryOpLLVMOpLowering<AddFOp, LLVM::FAddOp> {
-  using Super::Super;
-};
-struct SubFOpLowering : public BinaryOpLLVMOpLowering<SubFOp, LLVM::FSubOp> {
-  using Super::Super;
-};
-struct MulFOpLowering : public BinaryOpLLVMOpLowering<MulFOp, LLVM::FMulOp> {
-  using Super::Super;
-};
-struct DivFOpLowering : public BinaryOpLLVMOpLowering<DivFOp, LLVM::FDivOp> {
-  using Super::Super;
-};
-struct RemFOpLowering : public BinaryOpLLVMOpLowering<RemFOp, LLVM::FRemOp> {
-  using Super::Super;
-};
-struct CopySignOpLowering
-    : public BinaryOpLLVMOpLowering<CopySignOp, LLVM::CopySignOp> {
-  using Super::Super;
-};
-struct SelectOpLowering
-    : public OneToOneLLVMOpLowering<SelectOp, LLVM::SelectOp> {
-  using Super::Super;
-};
-struct ConstLLVMOpLowering
-    : public OneToOneLLVMOpLowering<ConstantOp, LLVM::ConstantOp> {
-  using Super::Super;
-};
-struct ShiftLeftOpLowering
-    : public OneToOneLLVMOpLowering<ShiftLeftOp, LLVM::ShlOp> {
-  using Super::Super;
-};
-struct SignedShiftRightOpLowering
-    : public OneToOneLLVMOpLowering<SignedShiftRightOp, LLVM::AShrOp> {
-  using Super::Super;
-};
-struct UnsignedShiftRightOpLowering
-    : public OneToOneLLVMOpLowering<UnsignedShiftRightOp, LLVM::LShrOp> {
-  using Super::Super;
-};
+namespace {
+// Straightforward lowerings.
+using AbsFOpLowering = VectorConvertToLLVMPattern<AbsFOp, LLVM::FAbsOp>;
+using AddFOpLowering = VectorConvertToLLVMPattern<AddFOp, LLVM::FAddOp>;
+using AddIOpLowering = VectorConvertToLLVMPattern<AddIOp, LLVM::AddOp>;
+using AndOpLowering = VectorConvertToLLVMPattern<AndOp, LLVM::AndOp>;
+using CeilFOpLowering = VectorConvertToLLVMPattern<CeilFOp, LLVM::FCeilOp>;
+using ConstLLVMOpLowering =
+    OneToOneConvertToLLVMPattern<ConstantOp, LLVM::ConstantOp>;
+using CopySignOpLowering =
+    VectorConvertToLLVMPattern<CopySignOp, LLVM::CopySignOp>;
+using CosOpLowering = VectorConvertToLLVMPattern<CosOp, LLVM::CosOp>;
+using DivFOpLowering = VectorConvertToLLVMPattern<DivFOp, LLVM::FDivOp>;
+using ExpOpLowering = VectorConvertToLLVMPattern<ExpOp, LLVM::ExpOp>;
+using Exp2OpLowering = VectorConvertToLLVMPattern<Exp2Op, LLVM::Exp2Op>;
+using Log10OpLowering = VectorConvertToLLVMPattern<Log10Op, LLVM::Log10Op>;
+using Log2OpLowering = VectorConvertToLLVMPattern<Log2Op, LLVM::Log2Op>;
+using LogOpLowering = VectorConvertToLLVMPattern<LogOp, LLVM::LogOp>;
+using MulFOpLowering = VectorConvertToLLVMPattern<MulFOp, LLVM::FMulOp>;
+using MulIOpLowering = VectorConvertToLLVMPattern<MulIOp, LLVM::MulOp>;
+using NegFOpLowering = VectorConvertToLLVMPattern<NegFOp, LLVM::FNegOp>;
+using OrOpLowering = VectorConvertToLLVMPattern<OrOp, LLVM::OrOp>;
+using RemFOpLowering = VectorConvertToLLVMPattern<RemFOp, LLVM::FRemOp>;
+using SelectOpLowering = OneToOneConvertToLLVMPattern<SelectOp, LLVM::SelectOp>;
+using ShiftLeftOpLowering =
+    OneToOneConvertToLLVMPattern<ShiftLeftOp, LLVM::ShlOp>;
+using SignedDivIOpLowering =
+    VectorConvertToLLVMPattern<SignedDivIOp, LLVM::SDivOp>;
+using SignedRemIOpLowering =
+    VectorConvertToLLVMPattern<SignedRemIOp, LLVM::SRemOp>;
+using SignedShiftRightOpLowering =
+    OneToOneConvertToLLVMPattern<SignedShiftRightOp, LLVM::AShrOp>;
+using SqrtOpLowering = VectorConvertToLLVMPattern<SqrtOp, LLVM::SqrtOp>;
+using SubFOpLowering = VectorConvertToLLVMPattern<SubFOp, LLVM::FSubOp>;
+using SubIOpLowering = VectorConvertToLLVMPattern<SubIOp, LLVM::SubOp>;
+using UnsignedDivIOpLowering =
+    VectorConvertToLLVMPattern<UnsignedDivIOp, LLVM::UDivOp>;
+using UnsignedRemIOpLowering =
+    VectorConvertToLLVMPattern<UnsignedRemIOp, LLVM::URemOp>;
+using UnsignedShiftRightOpLowering =
+    OneToOneConvertToLLVMPattern<UnsignedShiftRightOp, LLVM::LShrOp>;
+using XOrOpLowering = VectorConvertToLLVMPattern<XOrOp, LLVM::XOrOp>;
 
 // Check if the MemRefType `type` is supported by the lowering. We currently
 // only support memrefs with identity maps.
@@ -1373,13 +1250,11 @@ static bool isSupportedMemRefType(MemRefType type) {
 // Alignment is obtained by allocating `alignment - 1` more bytes than requested
 // and shifting the aligned pointer relative to the allocated memory. If
 // alignment is unspecified, the two pointers are equal.
-struct AllocOpLowering : public LLVMLegalizationPattern<AllocOp> {
-  using LLVMLegalizationPattern<AllocOp>::LLVMLegalizationPattern;
+struct AllocOpLowering : public ConvertOpToLLVMPattern<AllocOp> {
+  using ConvertOpToLLVMPattern<AllocOp>::ConvertOpToLLVMPattern;
 
-  AllocOpLowering(LLVM::LLVMDialect &dialect_, LLVMTypeConverter &converter,
-                  bool useAlloca = false)
-      : LLVMLegalizationPattern<AllocOp>(dialect_, converter),
-        useAlloca(useAlloca) {}
+  explicit AllocOpLowering(LLVMTypeConverter &converter, bool useAlloca = false)
+      : ConvertOpToLLVMPattern<AllocOp>(converter), useAlloca(useAlloca) {}
 
   LogicalResult match(Operation *op) const override {
     MemRefType type = cast<AllocOp>(op).getType();
@@ -1569,10 +1444,10 @@ struct AllocOpLowering : public LLVMLegalizationPattern<AllocOp> {
 // A CallOp automatically promotes MemRefType to a sequence of alloca/store and
 // passes the pointer to the MemRef across function boundaries.
 template <typename CallOpType>
-struct CallOpInterfaceLowering : public LLVMLegalizationPattern<CallOpType> {
-  using LLVMLegalizationPattern<CallOpType>::LLVMLegalizationPattern;
+struct CallOpInterfaceLowering : public ConvertOpToLLVMPattern<CallOpType> {
+  using ConvertOpToLLVMPattern<CallOpType>::ConvertOpToLLVMPattern;
   using Super = CallOpInterfaceLowering<CallOpType>;
-  using Base = LLVMLegalizationPattern<CallOpType>;
+  using Base = ConvertOpToLLVMPattern<CallOpType>;
 
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
@@ -1639,13 +1514,12 @@ struct CallIndirectOpLowering : public CallOpInterfaceLowering<CallIndirectOp> {
 // A `dealloc` is converted into a call to `free` on the underlying data buffer.
 // The memref descriptor being an SSA value, there is no need to clean it up
 // in any way.
-struct DeallocOpLowering : public LLVMLegalizationPattern<DeallocOp> {
-  using LLVMLegalizationPattern<DeallocOp>::LLVMLegalizationPattern;
+struct DeallocOpLowering : public ConvertOpToLLVMPattern<DeallocOp> {
+  using ConvertOpToLLVMPattern<DeallocOp>::ConvertOpToLLVMPattern;
 
-  DeallocOpLowering(LLVM::LLVMDialect &dialect_, LLVMTypeConverter &converter,
-                    bool useAlloca = false)
-      : LLVMLegalizationPattern<DeallocOp>(dialect_, converter),
-        useAlloca(useAlloca) {}
+  explicit DeallocOpLowering(LLVMTypeConverter &converter,
+                             bool useAlloca = false)
+      : ConvertOpToLLVMPattern<DeallocOp>(converter), useAlloca(useAlloca) {}
 
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
@@ -1680,8 +1554,8 @@ struct DeallocOpLowering : public LLVMLegalizationPattern<DeallocOp> {
 };
 
 // A `rsqrt` is converted into `1 / sqrt`.
-struct RsqrtOpLowering : public LLVMLegalizationPattern<RsqrtOp> {
-  using LLVMLegalizationPattern<RsqrtOp>::LLVMLegalizationPattern;
+struct RsqrtOpLowering : public ConvertOpToLLVMPattern<RsqrtOp> {
+  using ConvertOpToLLVMPattern<RsqrtOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
@@ -1716,79 +1590,26 @@ struct RsqrtOpLowering : public LLVMLegalizationPattern<RsqrtOp> {
     if (!vectorType)
       return failure();
 
-    if (succeeded(HandleMultidimensionalVectors(
-            op, operands, typeConverter,
-            [&](LLVM::LLVMType llvmVectorTy, ValueRange operands) {
-              auto splatAttr = SplatElementsAttr::get(
-                  mlir::VectorType::get({llvmVectorTy.getUnderlyingType()
-                                             ->getVectorNumElements()},
-                                        floatType),
-                  floatOne);
-              auto one = rewriter.create<LLVM::ConstantOp>(loc, llvmVectorTy,
-                                                           splatAttr);
-              auto sqrt =
-                  rewriter.create<LLVM::SqrtOp>(loc, llvmVectorTy, operands[0]);
-              return rewriter.create<LLVM::FDivOp>(loc, llvmVectorTy, one,
-                                                   sqrt);
-            },
-            rewriter)))
-      return success();
-    return failure();
-  }
-};
-
-// A `tanh` is converted into a call to the `tanh` function.
-struct TanhOpLowering : public LLVMLegalizationPattern<TanhOp> {
-  using LLVMLegalizationPattern<TanhOp>::LLVMLegalizationPattern;
-
-  LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
-                  ConversionPatternRewriter &rewriter) const override {
-
-    using LLVMFuncOpT = LLVM::LLVMFuncOp;
-    using LLVMTypeT = LLVM::LLVMType;
-
-    OperandAdaptor<TanhOp> transformed(operands);
-    LLVMTypeT operandType =
-        transformed.operand().getType().dyn_cast<LLVM::LLVMType>();
-
-    if (!operandType)
-      return failure();
-
-    std::string functionName;
-    if (operandType.isFloatTy())
-      functionName = "tanhf";
-    else if (operandType.isDoubleTy())
-      functionName = "tanh";
-    else
-      return failure();
-
-    // Get a reference to the tanh function, inserting it if necessary.
-    Operation *tanhFunc =
-        SymbolTable::lookupNearestSymbolFrom(op, functionName);
-
-    LLVMFuncOpT tanhLLVMFunc;
-    if (tanhFunc) {
-      tanhLLVMFunc = cast<LLVMFuncOpT>(tanhFunc);
-    } else {
-      PatternRewriter::InsertionGuard insertGuard(rewriter);
-      auto module = op->getParentOfType<ModuleOp>();
-      rewriter.setInsertionPointToStart(module.getBody());
-      tanhLLVMFunc = rewriter.create<LLVMFuncOpT>(
-          module.getLoc(), functionName,
-          LLVMTypeT::getFunctionTy(operandType, operandType,
-                                   /*isVarArg=*/false));
-    }
-
-    rewriter.replaceOpWithNewOp<LLVM::CallOp>(
-        op, operandType, rewriter.getSymbolRefAttr(tanhLLVMFunc),
-        transformed.operand());
-    return success();
+    return handleMultidimensionalVectors(
+        op, operands, typeConverter,
+        [&](LLVM::LLVMType llvmVectorTy, ValueRange operands) {
+          auto splatAttr = SplatElementsAttr::get(
+              mlir::VectorType::get(
+                  {llvmVectorTy.getUnderlyingType()->getVectorNumElements()},
+                  floatType),
+              floatOne);
+          auto one =
+              rewriter.create<LLVM::ConstantOp>(loc, llvmVectorTy, splatAttr);
+          auto sqrt =
+              rewriter.create<LLVM::SqrtOp>(loc, llvmVectorTy, operands[0]);
+          return rewriter.create<LLVM::FDivOp>(loc, llvmVectorTy, one, sqrt);
+        },
+        rewriter);
   }
 };
 
-struct MemRefCastOpLowering : public LLVMLegalizationPattern<MemRefCastOp> {
-  using LLVMLegalizationPattern<MemRefCastOp>::LLVMLegalizationPattern;
+struct MemRefCastOpLowering : public ConvertOpToLLVMPattern<MemRefCastOp> {
+  using ConvertOpToLLVMPattern<MemRefCastOp>::ConvertOpToLLVMPattern;
 
   LogicalResult match(Operation *op) const override {
     auto memRefCastOp = cast<MemRefCastOp>(op);
@@ -1883,8 +1704,8 @@ struct MemRefCastOpLowering : public LLVMLegalizationPattern<MemRefCastOp> {
 };
 
 struct DialectCastOpLowering
-    : public LLVMLegalizationPattern<LLVM::DialectCastOp> {
-  using LLVMLegalizationPattern<LLVM::DialectCastOp>::LLVMLegalizationPattern;
+    : public ConvertOpToLLVMPattern<LLVM::DialectCastOp> {
+  using ConvertOpToLLVMPattern<LLVM::DialectCastOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
@@ -1902,8 +1723,8 @@ struct DialectCastOpLowering
 
 // A `dim` is converted to a constant for static sizes and to an access to the
 // size stored in the memref descriptor for dynamic sizes.
-struct DimOpLowering : public LLVMLegalizationPattern<DimOp> {
-  using LLVMLegalizationPattern<DimOp>::LLVMLegalizationPattern;
+struct DimOpLowering : public ConvertOpToLLVMPattern<DimOp> {
+  using ConvertOpToLLVMPattern<DimOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
@@ -1930,8 +1751,8 @@ struct DimOpLowering : public LLVMLegalizationPattern<DimOp> {
 // to supported MemRef types.  Provides functionality to emit code accessing a
 // specific element of the underlying data buffer.
 template <typename Derived>
-struct LoadStoreOpLowering : public LLVMLegalizationPattern<Derived> {
-  using LLVMLegalizationPattern<Derived>::LLVMLegalizationPattern;
+struct LoadStoreOpLowering : public ConvertOpToLLVMPattern<Derived> {
+  using ConvertOpToLLVMPattern<Derived>::ConvertOpToLLVMPattern;
   using Base = LoadStoreOpLowering<Derived>;
 
   LogicalResult match(Operation *op) const override {
@@ -2079,8 +1900,8 @@ struct PrefetchOpLowering : public LoadStoreOpLowering<PrefetchOp> {
 // an integer.  If the bit width of the source and target integer types is the
 // same, just erase the cast.  If the target type is wider, sign-extend the
 // value, otherwise truncate it.
-struct IndexCastOpLowering : public LLVMLegalizationPattern<IndexCastOp> {
-  using LLVMLegalizationPattern<IndexCastOp>::LLVMLegalizationPattern;
+struct IndexCastOpLowering : public ConvertOpToLLVMPattern<IndexCastOp> {
+  using ConvertOpToLLVMPattern<IndexCastOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
@@ -2114,8 +1935,8 @@ static LLVMPredType convertCmpPredicate(StdPredType pred) {
   return static_cast<LLVMPredType>(pred);
 }
 
-struct CmpIOpLowering : public LLVMLegalizationPattern<CmpIOp> {
-  using LLVMLegalizationPattern<CmpIOp>::LLVMLegalizationPattern;
+struct CmpIOpLowering : public ConvertOpToLLVMPattern<CmpIOp> {
+  using ConvertOpToLLVMPattern<CmpIOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
@@ -2133,8 +1954,8 @@ struct CmpIOpLowering : public LLVMLegalizationPattern<CmpIOp> {
   }
 };
 
-struct CmpFOpLowering : public LLVMLegalizationPattern<CmpFOp> {
-  using LLVMLegalizationPattern<CmpFOp>::LLVMLegalizationPattern;
+struct CmpFOpLowering : public ConvertOpToLLVMPattern<CmpFOp> {
+  using ConvertOpToLLVMPattern<CmpFOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
@@ -2153,39 +1974,40 @@ struct CmpFOpLowering : public LLVMLegalizationPattern<CmpFOp> {
 };
 
 struct SIToFPLowering
-    : public OneToOneLLVMOpLowering<SIToFPOp, LLVM::SIToFPOp> {
+    : public OneToOneConvertToLLVMPattern<SIToFPOp, LLVM::SIToFPOp> {
   using Super::Super;
 };
 
-struct FPExtLowering : public OneToOneLLVMOpLowering<FPExtOp, LLVM::FPExtOp> {
+struct FPExtLowering
+    : public OneToOneConvertToLLVMPattern<FPExtOp, LLVM::FPExtOp> {
   using Super::Super;
 };
 
 struct FPTruncLowering
-    : public OneToOneLLVMOpLowering<FPTruncOp, LLVM::FPTruncOp> {
+    : public OneToOneConvertToLLVMPattern<FPTruncOp, LLVM::FPTruncOp> {
   using Super::Super;
 };
 
 struct SignExtendIOpLowering
-    : public OneToOneLLVMOpLowering<SignExtendIOp, LLVM::SExtOp> {
+    : public OneToOneConvertToLLVMPattern<SignExtendIOp, LLVM::SExtOp> {
   using Super::Super;
 };
 
 struct TruncateIOpLowering
-    : public OneToOneLLVMOpLowering<TruncateIOp, LLVM::TruncOp> {
+    : public OneToOneConvertToLLVMPattern<TruncateIOp, LLVM::TruncOp> {
   using Super::Super;
 };
 
 struct ZeroExtendIOpLowering
-    : public OneToOneLLVMOpLowering<ZeroExtendIOp, LLVM::ZExtOp> {
+    : public OneToOneConvertToLLVMPattern<ZeroExtendIOp, LLVM::ZExtOp> {
   using Super::Super;
 };
 
 // Base class for LLVM IR lowering terminator operations with successors.
 template <typename SourceOp, typename TargetOp>
 struct OneToOneLLVMTerminatorLowering
-    : public LLVMLegalizationPattern<SourceOp> {
-  using LLVMLegalizationPattern<SourceOp>::LLVMLegalizationPattern;
+    : public ConvertOpToLLVMPattern<SourceOp> {
+  using ConvertOpToLLVMPattern<SourceOp>::ConvertOpToLLVMPattern;
   using Super = OneToOneLLVMTerminatorLowering<SourceOp, TargetOp>;
 
   LogicalResult
@@ -2203,8 +2025,8 @@ struct OneToOneLLVMTerminatorLowering
 // can only return 0 or 1 value, we pack multiple values into a structure type.
 // Emit `UndefOp` followed by `InsertValueOp`s to create such structure if
 // necessary before returning it
-struct ReturnOpLowering : public LLVMLegalizationPattern<ReturnOp> {
-  using LLVMLegalizationPattern<ReturnOp>::LLVMLegalizationPattern;
+struct ReturnOpLowering : public ConvertOpToLLVMPattern<ReturnOp> {
+  using ConvertOpToLLVMPattern<ReturnOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
@@ -2252,8 +2074,8 @@ struct CondBranchOpLowering
 
 // The Splat operation is lowered to an insertelement + a shufflevector
 // operation. Splat to only 1-d vector result types are lowered.
-struct SplatOpLowering : public LLVMLegalizationPattern<SplatOp> {
-  using LLVMLegalizationPattern<SplatOp>::LLVMLegalizationPattern;
+struct SplatOpLowering : public ConvertOpToLLVMPattern<SplatOp> {
+  using ConvertOpToLLVMPattern<SplatOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
@@ -2286,8 +2108,8 @@ struct SplatOpLowering : public LLVMLegalizationPattern<SplatOp> {
 // The Splat operation is lowered to an insertelement + a shufflevector
 // operation. Splat to only 2+-d vector result types are lowered by the
 // SplatNdOpLowering, the 1-d case is handled by SplatOpLowering.
-struct SplatNdOpLowering : public LLVMLegalizationPattern<SplatOp> {
-  using LLVMLegalizationPattern<SplatOp>::LLVMLegalizationPattern;
+struct SplatNdOpLowering : public ConvertOpToLLVMPattern<SplatOp> {
+  using ConvertOpToLLVMPattern<SplatOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
@@ -2340,8 +2162,8 @@ struct SplatNdOpLowering : public LLVMLegalizationPattern<SplatOp> {
 ///   2. Updates to the descriptor to introduce the data ptr, offset, size
 ///      and stride.
 /// The subview op is replaced by the descriptor.
-struct SubViewOpLowering : public LLVMLegalizationPattern<SubViewOp> {
-  using LLVMLegalizationPattern<SubViewOp>::LLVMLegalizationPattern;
+struct SubViewOpLowering : public ConvertOpToLLVMPattern<SubViewOp> {
+  using ConvertOpToLLVMPattern<SubViewOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
@@ -2468,8 +2290,8 @@ struct SubViewOpLowering : public LLVMLegalizationPattern<SubViewOp> {
 ///   2. Updates to the descriptor to introduce the data ptr, offset, size
 ///      and stride.
 /// The view op is replaced by the descriptor.
-struct ViewOpLowering : public LLVMLegalizationPattern<ViewOp> {
-  using LLVMLegalizationPattern<ViewOp>::LLVMLegalizationPattern;
+struct ViewOpLowering : public ConvertOpToLLVMPattern<ViewOp> {
+  using ConvertOpToLLVMPattern<ViewOp>::ConvertOpToLLVMPattern;
 
   // Build and return the value for the idx^th shape dimension, either by
   // returning the constant shape dimension or counting the proper dynamic size.
@@ -2585,8 +2407,8 @@ struct ViewOpLowering : public LLVMLegalizationPattern<ViewOp> {
 };
 
 struct AssumeAlignmentOpLowering
-    : public LLVMLegalizationPattern<AssumeAlignmentOp> {
-  using LLVMLegalizationPattern<AssumeAlignmentOp>::LLVMLegalizationPattern;
+    : public ConvertOpToLLVMPattern<AssumeAlignmentOp> {
+  using ConvertOpToLLVMPattern<AssumeAlignmentOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
@@ -2807,6 +2629,7 @@ void mlir::populateStdToLLVMNonMemoryConversionPatterns(
       DialectCastOpLowering,
       DivFOpLowering,
       ExpOpLowering,
+      Exp2OpLowering,
       LogOpLowering,
       Log10OpLowering,
       Log2OpLowering,
@@ -2833,13 +2656,12 @@ void mlir::populateStdToLLVMNonMemoryConversionPatterns(
       SqrtOpLowering,
       SubFOpLowering,
       SubIOpLowering,
-      TanhOpLowering,
       TruncateIOpLowering,
       UnsignedDivIOpLowering,
       UnsignedRemIOpLowering,
       UnsignedShiftRightOpLowering,
       XOrOpLowering,
-      ZeroExtendIOpLowering>(*converter.getDialect(), converter);
+      ZeroExtendIOpLowering>(converter);
   // clang-format on
 }
 
@@ -2854,19 +2676,17 @@ void mlir::populateStdToLLVMMemoryConversionPatters(
       MemRefCastOpLowering,
       StoreOpLowering,
       SubViewOpLowering,
-      ViewOpLowering>(*converter.getDialect(), converter);
+      ViewOpLowering>(converter);
   patterns.insert<
       AllocOpLowering,
-      DeallocOpLowering>(
-        *converter.getDialect(), converter, useAlloca);
+      DeallocOpLowering>(converter, useAlloca);
   // clang-format on
 }
 
 void mlir::populateStdToLLVMDefaultFuncOpConversionPattern(
     LLVMTypeConverter &converter, OwningRewritePatternList &patterns,
     bool emitCWrappers) {
-  patterns.insert<FuncOpConversion>(*converter.getDialect(), converter,
-                                    emitCWrappers);
+  patterns.insert<FuncOpConversion>(converter, emitCWrappers);
 }
 
 void mlir::populateStdToLLVMConversionPatterns(
@@ -2880,7 +2700,7 @@ void mlir::populateStdToLLVMConversionPatterns(
 
 static void populateStdToLLVMBarePtrFuncOpConversionPattern(
     LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
-  patterns.insert<BarePtrFuncOpConversion>(*converter.getDialect(), converter);
+  patterns.insert<BarePtrFuncOpConversion>(converter);
 }
 
 void mlir::populateStdToLLVMBarePtrConversionPatterns(
@@ -2957,12 +2777,17 @@ LLVMTypeConverter::promoteMemRefDescriptors(Location loc, ValueRange opOperands,
 namespace {
 /// A pass converting MLIR operations into the LLVM IR dialect.
 struct LLVMLoweringPass : public ModulePass<LLVMLoweringPass> {
+/// Include the generated pass utilities.
+#define GEN_PASS_ConvertStandardToLLVM
+#include "mlir/Conversion/Passes.h.inc"
+
   /// Creates an LLVM lowering pass.
-  explicit LLVMLoweringPass(bool useAlloca, bool useBarePtrCallConv,
-                            bool emitCWrappers) {
+  LLVMLoweringPass(bool useAlloca, bool useBarePtrCallConv, bool emitCWrappers,
+                   unsigned indexBitwidth) {
     this->useAlloca = useAlloca;
     this->useBarePtrCallConv = useBarePtrCallConv;
     this->emitCWrappers = emitCWrappers;
+    this->indexBitwidth = indexBitwidth;
   }
   explicit LLVMLoweringPass() {}
   LLVMLoweringPass(const LLVMLoweringPass &pass) {}
@@ -2982,6 +2807,7 @@ struct LLVMLoweringPass : public ModulePass<LLVMLoweringPass> {
     LLVMTypeConverterCustomization customs;
     customs.funcArgConverter = useBarePtrCallConv ? barePtrFuncArgTypeConverter
                                                   : structFuncArgTypeConverter;
+    customs.indexBitwidth = indexBitwidth;
     LLVMTypeConverter typeConverter(&getContext(), customs);
 
     OwningRewritePatternList patterns;
@@ -2996,25 +2822,6 @@ struct LLVMLoweringPass : public ModulePass<LLVMLoweringPass> {
     if (failed(applyPartialConversion(m, target, patterns, &typeConverter)))
       signalPassFailure();
   }
-
-  /// Use `alloca` instead of `call @malloc` for converting std.alloc.
-  Option<bool> useAlloca{
-      *this, "use-alloca",
-      llvm::cl::desc("Replace emission of malloc/free by alloca"),
-      llvm::cl::init(false)};
-
-  /// Convert memrefs to bare pointers in function signatures.
-  Option<bool> useBarePtrCallConv{
-      *this, "use-bare-ptr-memref-call-conv",
-      llvm::cl::desc("Replace FuncOp's MemRef arguments with "
-                     "bare pointers to the MemRef element types"),
-      llvm::cl::init(false)};
-
-  /// Emit wrappers for C-compatible pointer-to-struct memref descriptors.
-  Option<bool> emitCWrappers{
-      *this, "emit-c-wrappers",
-      llvm::cl::desc("Emit C-compatible wrapper functions"),
-      llvm::cl::init(false)};
 };
 } // end namespace
 
@@ -3022,15 +2829,12 @@ mlir::LLVMConversionTarget::LLVMConversionTarget(MLIRContext &ctx)
     : ConversionTarget(ctx) {
   this->addLegalDialect<LLVM::LLVMDialect>();
   this->addIllegalOp<LLVM::DialectCastOp>();
+  this->addIllegalOp<TanhOp>();
 }
 
 std::unique_ptr<OpPassBase<ModuleOp>>
 mlir::createLowerToLLVMPass(bool useAlloca, bool useBarePtrCallConv,
-                            bool emitCWrappers) {
+                            bool emitCWrappers, unsigned indexBitwidth) {
   return std::make_unique<LLVMLoweringPass>(useAlloca, useBarePtrCallConv,
-                                            emitCWrappers);
+                                            emitCWrappers, indexBitwidth);
 }
-
-static PassRegistration<LLVMLoweringPass>
-    pass(PASS_NAME, "Convert scalar and vector operations from the "
-                    "Standard to the LLVM dialect");
diff --git a/mlir/lib/Conversion/StandardToSPIRV/CMakeLists.txt b/mlir/lib/Conversion/StandardToSPIRV/CMakeLists.txt
index 6d940eaf024e3..bb249078d62c0 100644
--- a/mlir/lib/Conversion/StandardToSPIRV/CMakeLists.txt
+++ b/mlir/lib/Conversion/StandardToSPIRV/CMakeLists.txt
@@ -6,6 +6,9 @@ add_mlir_conversion_library(MLIRStandardToSPIRVTransforms
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/SPIRV
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/IR
+
+  DEPENDS
+  MLIRConversionPassIncGen
   )
 
 target_link_libraries(MLIRStandardToSPIRVTransforms
diff --git a/mlir/lib/Conversion/StandardToSPIRV/ConvertStandardToSPIRVPass.cpp b/mlir/lib/Conversion/StandardToSPIRV/ConvertStandardToSPIRVPass.cpp
index efccd168d6ea8..ab7dd8546995d 100644
--- a/mlir/lib/Conversion/StandardToSPIRV/ConvertStandardToSPIRVPass.cpp
+++ b/mlir/lib/Conversion/StandardToSPIRV/ConvertStandardToSPIRVPass.cpp
@@ -23,6 +23,10 @@ namespace {
 /// A pass converting MLIR Standard operations into the SPIR-V dialect.
 class ConvertStandardToSPIRVPass
     : public ModulePass<ConvertStandardToSPIRVPass> {
+/// Include the generated pass utilities.
+#define GEN_PASS_ConvertStandardToSPIRV
+#include "mlir/Conversion/Passes.h.inc"
+
   void runOnModule() override;
 };
 } // namespace
@@ -48,6 +52,3 @@ void ConvertStandardToSPIRVPass::runOnModule() {
 std::unique_ptr<OpPassBase<ModuleOp>> mlir::createConvertStandardToSPIRVPass() {
   return std::make_unique<ConvertStandardToSPIRVPass>();
 }
-
-static PassRegistration<ConvertStandardToSPIRVPass>
-    pass("convert-std-to-spirv", "Convert Standard Ops to SPIR-V dialect");
diff --git a/mlir/lib/Conversion/StandardToSPIRV/LegalizeStandardForSPIRV.cpp b/mlir/lib/Conversion/StandardToSPIRV/LegalizeStandardForSPIRV.cpp
index 3af01e025801d..381087fbdf762 100644
--- a/mlir/lib/Conversion/StandardToSPIRV/LegalizeStandardForSPIRV.cpp
+++ b/mlir/lib/Conversion/StandardToSPIRV/LegalizeStandardForSPIRV.cpp
@@ -161,6 +161,10 @@ void mlir::populateStdLegalizationPatternsForSPIRVLowering(
 
 namespace {
 struct SPIRVLegalization final : public OperationPass<SPIRVLegalization> {
+/// Include the generated pass utilities.
+#define GEN_PASS_LegalizeStandardForSPIRV
+#include "mlir/Conversion/Passes.h.inc"
+
   void runOnOperation() override;
 };
 } // namespace
@@ -175,6 +179,3 @@ void SPIRVLegalization::runOnOperation() {
 std::unique_ptr<Pass> mlir::createLegalizeStdOpsForSPIRVLoweringPass() {
   return std::make_unique<SPIRVLegalization>();
 }
-
-static PassRegistration<SPIRVLegalization>
-    pass("legalize-std-for-spirv", "Legalize standard ops for SPIR-V lowering");
diff --git a/mlir/lib/Conversion/VectorToLLVM/CMakeLists.txt b/mlir/lib/Conversion/VectorToLLVM/CMakeLists.txt
index 734af0b96a39d..ffc6da95da7ec 100644
--- a/mlir/lib/Conversion/VectorToLLVM/CMakeLists.txt
+++ b/mlir/lib/Conversion/VectorToLLVM/CMakeLists.txt
@@ -3,6 +3,9 @@ add_mlir_conversion_library(MLIRVectorToLLVM
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/VectorToLLVM
+
+  DEPENDS
+  MLIRConversionPassIncGen
 )
 set(LIBS
   MLIRLLVMIR
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index 2b3020d046a2f..d5a4f86d2ca9e 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -1119,6 +1119,10 @@ void mlir::populateVectorToLLVMMatrixConversionPatterns(
 
 namespace {
 struct LowerVectorToLLVMPass : public ModulePass<LowerVectorToLLVMPass> {
+/// Include the generated pass utilities.
+#define GEN_PASS_ConvertVectorToLLVM
+#include "mlir/Conversion/Passes.h.inc"
+
   void runOnModule() override;
 };
 } // namespace
@@ -1153,7 +1157,3 @@ void LowerVectorToLLVMPass::runOnModule() {
 std::unique_ptr<OpPassBase<ModuleOp>> mlir::createConvertVectorToLLVMPass() {
   return std::make_unique<LowerVectorToLLVMPass>();
 }
-
-static PassRegistration<LowerVectorToLLVMPass>
-    pass("convert-vector-to-llvm",
-         "Lower the operations from the vector dialect into the LLVM dialect");
diff --git a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
index 7bf8ab4e07a58..a92c621fa96e9 100644
--- a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
@@ -1383,7 +1383,10 @@ static LogicalResult canonicalizeLoopBounds(AffineForOp forOp) {
   auto prevUbMap = ubMap;
 
   canonicalizeMapAndOperands(&lbMap, &lbOperands);
+  lbMap = removeDuplicateExprs(lbMap);
+
   canonicalizeMapAndOperands(&ubMap, &ubOperands);
+  ubMap = removeDuplicateExprs(ubMap);
 
   // Any canonicalization change always leads to updated map(s).
   if (lbMap == prevLbMap && ubMap == prevUbMap)
@@ -2136,7 +2139,7 @@ LogicalResult AffinePrefetchOp::fold(ArrayRef<Attribute> cstOperands,
 
 void AffineParallelOp::build(Builder *builder, OperationState &result,
                              ArrayRef<int64_t> ranges) {
-  // Default initalize empty maps.
+  // Default initialize empty maps.
   auto lbMap = AffineMap::get(builder->getContext());
   auto ubMap = AffineMap::get(builder->getContext());
   // If there are ranges, set each to [0, N).
diff --git a/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp b/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
index 4f6d453fb56b2..3101a2ec66943 100644
--- a/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/AffineDataCopyGeneration.cpp
@@ -21,11 +21,10 @@
 
 #include "mlir/Analysis/Utils.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
+#include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/Dialect/Affine/Passes.h"
-#include "mlir/IR/Builders.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/LoopUtils.h"
-#include "mlir/Transforms/Utils.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
@@ -76,6 +75,10 @@ namespace {
 // are strided. Check for strided stores.
 struct AffineDataCopyGeneration
     : public FunctionPass<AffineDataCopyGeneration> {
+/// Include the generated pass utilities.
+#define GEN_PASS_AffineDataCopyGeneration
+#include "mlir/Dialect/Affine/Passes.h.inc"
+
   explicit AffineDataCopyGeneration(
       unsigned slowMemorySpace = 0,
       unsigned fastMemorySpace = clFastMemorySpace, unsigned tagMemorySpace = 0,
@@ -137,6 +140,9 @@ std::unique_ptr<OpPassBase<FuncOp>> mlir::createAffineDataCopyGenerationPass(
       slowMemorySpace, fastMemorySpace, tagMemorySpace, minDmaTransferSize,
       fastMemCapacityBytes);
 }
+std::unique_ptr<OpPassBase<FuncOp>> mlir::createAffineDataCopyGenerationPass() {
+  return std::make_unique<AffineDataCopyGeneration>();
+}
 
 /// Generate copies for this block. The block is partitioned into separate
 /// ranges: each range is either a sequence of one or more operations starting
@@ -262,7 +268,3 @@ void AffineDataCopyGeneration::runOnFunction() {
     nest->walk([](AffineForOp forOp) { promoteIfSingleIteration(forOp); });
   }
 }
-
-static PassRegistration<AffineDataCopyGeneration>
-    pass("affine-data-copy-generate",
-         "Generate explicit copying for memory operations");
diff --git a/mlir/lib/Dialect/Affine/Transforms/AffineLoopInvariantCodeMotion.cpp b/mlir/lib/Dialect/Affine/Transforms/AffineLoopInvariantCodeMotion.cpp
index 746aceb8090a8..12fdb37bf8417 100644
--- a/mlir/lib/Dialect/Affine/Transforms/AffineLoopInvariantCodeMotion.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/AffineLoopInvariantCodeMotion.cpp
@@ -42,6 +42,10 @@ namespace {
 /// TODO: This code should be removed once the new LICM pass can handle its
 ///       uses.
 struct LoopInvariantCodeMotion : public FunctionPass<LoopInvariantCodeMotion> {
+/// Include the generated pass utilities.
+#define GEN_PASS_AffineLoopInvariantCodeMotion
+#include "mlir/Dialect/Affine/Passes.h.inc"
+
   void runOnFunction() override;
   void runOnAffineForOp(AffineForOp forOp);
 };
@@ -232,7 +236,3 @@ std::unique_ptr<OpPassBase<FuncOp>>
 mlir::createAffineLoopInvariantCodeMotionPass() {
   return std::make_unique<LoopInvariantCodeMotion>();
 }
-
-static PassRegistration<LoopInvariantCodeMotion>
-    pass("affine-loop-invariant-code-motion",
-         "Hoist loop invariant instructions outside of the loop");
diff --git a/mlir/lib/Dialect/Affine/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Affine/Transforms/CMakeLists.txt
index 89abbd521deca..8337369657768 100644
--- a/mlir/lib/Dialect/Affine/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/Affine/Transforms/CMakeLists.txt
@@ -12,6 +12,7 @@ add_mlir_dialect_library(MLIRAffineTransforms
 
   DEPENDS
   MLIRAffineOpsIncGen
+  MLIRAffinePassIncGen
   MLIRLoopLikeInterfaceIncGen
   )
 target_link_libraries(MLIRAffineTransforms
diff --git a/mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp b/mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp
index 3f08315170c8f..2f5eea7606a9f 100644
--- a/mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp
@@ -15,7 +15,9 @@
 #include "mlir/Analysis/LoopAnalysis.h"
 #include "mlir/Analysis/Utils.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
+#include "mlir/Dialect/Affine/IR/AffineValueMap.h"
 #include "mlir/Dialect/Affine/Passes.h"
+#include "mlir/IR/BlockAndValueMapping.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/LoopUtils.h"
@@ -33,6 +35,12 @@ static llvm::cl::opt<unsigned long long>
                    llvm::cl::desc("Set size of cache to tile for in KiB"),
                    llvm::cl::cat(clOptionsCategory));
 
+// Separate full and partial tiles.
+static llvm::cl::opt<bool>
+    clSeparate("affine-tile-separate",
+               llvm::cl::desc("Separate full and partial tiles"),
+               llvm::cl::cat(clOptionsCategory));
+
 // Tile size to use for all loops (overrides -tile-sizes if provided).
 static llvm::cl::opt<unsigned>
     clTileSize("affine-tile-size",
@@ -51,6 +59,10 @@ namespace {
 
 /// A pass to perform loop tiling on all suitable loop nests of a Function.
 struct LoopTiling : public FunctionPass<LoopTiling> {
+/// Include the generated pass utilities.
+#define GEN_PASS_AffineLoopTiling
+#include "mlir/Dialect/Affine/Passes.h.inc"
+
   explicit LoopTiling(uint64_t cacheSizeBytes = kDefaultCacheMemCapacity,
                       bool avoidMaxMinBounds = true)
       : cacheSizeBytes(cacheSizeBytes), avoidMaxMinBounds(avoidMaxMinBounds) {}
@@ -77,6 +89,9 @@ std::unique_ptr<OpPassBase<FuncOp>>
 mlir::createLoopTilingPass(uint64_t cacheSizeBytes) {
   return std::make_unique<LoopTiling>(cacheSizeBytes);
 }
+std::unique_ptr<OpPassBase<FuncOp>> mlir::createLoopTilingPass() {
+  return std::make_unique<LoopTiling>();
+}
 
 // Move the loop body of AffineForOp 'src' from 'src' into the specified
 // location in destination's body, ignoring the terminator.
@@ -176,11 +191,12 @@ constructTiledIndexSetHyperRect(MutableArrayRef<AffineForOp> origLoops,
 /// and intra-tile loops. A band is a contiguous set of loops.
 //  TODO(bondhugula): handle non hyper-rectangular spaces.
 LogicalResult mlir::tileCodeGen(MutableArrayRef<AffineForOp> band,
-                                ArrayRef<unsigned> tileSizes) {
+                                ArrayRef<unsigned> tileSizes,
+                                SmallVectorImpl<AffineForOp> *tiledNest) {
+  // Check if the supplied for op's are all successively nested.
   assert(!band.empty() && "no loops in band");
   assert(band.size() == tileSizes.size() && "Too few/many tile sizes");
 
-  // Check if the supplied for op's are all successively nested.
   for (unsigned i = 1, e = band.size(); i < e; i++)
     assert(band[i].getParentOp() == band[i - 1] && "not a perfect nest / band");
 
@@ -248,6 +264,9 @@ LogicalResult mlir::tileCodeGen(MutableArrayRef<AffineForOp> band,
   // Erase the old loop nest.
   rootAffineForOp.erase();
 
+  if (tiledNest)
+    *tiledNest = std::move(tiledLoops);
+
   return success();
 }
 
@@ -393,12 +412,18 @@ void LoopTiling::runOnFunction() {
         diag << tSize << ' ';
       diag << "]\n";
     }
-    if (failed(tileCodeGen(band, tileSizes)))
+    SmallVector<AffineForOp, 6> tiledNest;
+    if (failed(tileCodeGen(band, tileSizes, &tiledNest)))
       return signalPassFailure();
+
+    // Separate full and partial tiles.
+    if (clSeparate) {
+      auto intraTileLoops =
+          MutableArrayRef<AffineForOp>(tiledNest).drop_front(band.size());
+      separateFullTiles(intraTileLoops);
+    }
   }
 }
 
 constexpr unsigned LoopTiling::kDefaultTileSize;
 constexpr uint64_t LoopTiling::kDefaultCacheMemCapacity;
-
-static PassRegistration<LoopTiling> pass("affine-loop-tile", "Tile loop nests");
diff --git a/mlir/lib/Dialect/Affine/Transforms/LoopUnroll.cpp b/mlir/lib/Dialect/Affine/Transforms/LoopUnroll.cpp
index 40187a7ebae7f..7f33630d3b8ab 100644
--- a/mlir/lib/Dialect/Affine/Transforms/LoopUnroll.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/LoopUnroll.cpp
@@ -50,11 +50,19 @@ static llvm::cl::opt<unsigned> clUnrollFullThreshold(
     llvm::cl::cat(clOptionsCategory));
 
 namespace {
+
+// TODO: this is really a test pass and should be moved out of dialect
+// transforms.
+
 /// Loop unrolling pass. Unrolls all innermost loops unless full unrolling and a
 /// full unroll threshold was specified, in which case, fully unrolls all loops
 /// with trip count less than the specified threshold. The latter is for testing
 /// purposes, especially for testing outer loop unrolling.
 struct LoopUnroll : public FunctionPass<LoopUnroll> {
+/// Include the generated pass utilities.
+#define GEN_PASS_AffineUnroll
+#include "mlir/Dialect/Affine/Passes.h.inc"
+
   const Optional<unsigned> unrollFactor;
   const Optional<bool> unrollFull;
   // Callback to obtain unroll factors; if this has a callable target, takes
@@ -76,44 +84,32 @@ struct LoopUnroll : public FunctionPass<LoopUnroll> {
 };
 } // end anonymous namespace
 
-void LoopUnroll::runOnFunction() {
-  // Gathers all innermost loops through a post order pruned walk.
-  struct InnermostLoopGatherer {
-    // Store innermost loops as we walk.
-    std::vector<AffineForOp> loops;
-
-    void walkPostOrder(FuncOp f) {
-      for (auto &b : f)
-        walkPostOrder(b.begin(), b.end());
-    }
-
-    bool walkPostOrder(Block::iterator Start, Block::iterator End) {
-      bool hasInnerLoops = false;
-      // We need to walk all elements since all innermost loops need to be
-      // gathered as opposed to determining whether this list has any inner
-      // loops or not.
-      while (Start != End)
-        hasInnerLoops |= walkPostOrder(&(*Start++));
-      return hasInnerLoops;
-    }
-    bool walkPostOrder(Operation *opInst) {
-      bool hasInnerLoops = false;
-      for (auto &region : opInst->getRegions())
-        for (auto &block : region)
-          hasInnerLoops |= walkPostOrder(block.begin(), block.end());
-      if (isa<AffineForOp>(opInst)) {
-        if (!hasInnerLoops)
-          loops.push_back(cast<AffineForOp>(opInst));
-        return true;
-      }
-      return hasInnerLoops;
-    }
-  };
+/// Returns true if no other affine.for ops are nested within.
+static bool isInnermostAffineForOp(AffineForOp forOp) {
+  // Only for the innermost affine.for op's.
+  bool isInnermost = true;
+  forOp.walk([&](AffineForOp thisForOp) {
+    // Since this is a post order walk, we are able to conclude here.
+    isInnermost = (thisForOp == forOp);
+    return WalkResult::interrupt();
+  });
+  return isInnermost;
+}
 
+/// Gathers loops that have no affine.for's nested within.
+static void gatherInnermostLoops(FuncOp f,
+                                 SmallVectorImpl<AffineForOp> &loops) {
+  f.walk([&](AffineForOp forOp) {
+    if (isInnermostAffineForOp(forOp))
+      loops.push_back(forOp);
+  });
+}
+
+void LoopUnroll::runOnFunction() {
   if (clUnrollFull.getNumOccurrences() > 0 &&
       clUnrollFullThreshold.getNumOccurrences() > 0) {
     // Store short loops as we walk.
-    std::vector<AffineForOp> loops;
+    SmallVector<AffineForOp, 4> loops;
 
     // Gathers all loops with trip count <= minTripCount. Do a post order walk
     // so that loops are gathered from innermost to outermost (or else unrolling
@@ -133,10 +129,10 @@ void LoopUnroll::runOnFunction() {
                                 : 1;
   // If the call back is provided, we will recurse until no loops are found.
   FuncOp func = getFunction();
+  SmallVector<AffineForOp, 4> loops;
   for (unsigned i = 0; i < numRepetitions || getUnrollFactor; i++) {
-    InnermostLoopGatherer ilg;
-    ilg.walkPostOrder(func);
-    auto &loops = ilg.loops;
+    loops.clear();
+    gatherInnermostLoops(func, loops);
     if (loops.empty())
       break;
     bool unrolled = false;
@@ -177,5 +173,3 @@ std::unique_ptr<OpPassBase<FuncOp>> mlir::createLoopUnrollPass(
       unrollFactor == -1 ? None : Optional<unsigned>(unrollFactor),
       unrollFull == -1 ? None : Optional<bool>(unrollFull), getUnrollFactor);
 }
-
-static PassRegistration<LoopUnroll> pass("affine-loop-unroll", "Unroll loops");
diff --git a/mlir/lib/Dialect/Affine/Transforms/LoopUnrollAndJam.cpp b/mlir/lib/Dialect/Affine/Transforms/LoopUnrollAndJam.cpp
index 218a6a2ce23db..5f419a83d6cfd 100644
--- a/mlir/lib/Dialect/Affine/Transforms/LoopUnrollAndJam.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/LoopUnrollAndJam.cpp
@@ -61,6 +61,10 @@ namespace {
 /// Loop unroll jam pass. Currently, this just unroll jams the first
 /// outer loop in a Function.
 struct LoopUnrollAndJam : public FunctionPass<LoopUnrollAndJam> {
+/// Include the generated pass utilities.
+#define GEN_PASS_AffineLoopUnrollAndJam
+#include "mlir/Dialect/Affine/Passes.h.inc"
+
   Optional<unsigned> unrollJamFactor;
   static const unsigned kDefaultUnrollJamFactor = 4;
 
@@ -100,6 +104,3 @@ LogicalResult LoopUnrollAndJam::runOnAffineForOp(AffineForOp forOp) {
   // Unroll and jam by four otherwise.
   return loopUnrollJamByFactor(forOp, kDefaultUnrollJamFactor);
 }
-
-static PassRegistration<LoopUnrollAndJam> pass("affine-loop-unroll-jam",
-                                               "Unroll and jam loops");
diff --git a/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineStructures.cpp b/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineStructures.cpp
index 60ad1545d3502..fc58b19656fe9 100644
--- a/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineStructures.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineStructures.cpp
@@ -10,13 +10,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Analysis/AffineStructures.h"
+#include "mlir/Analysis/Utils.h"
+#include "mlir/Dialect/Affine/Passes.h"
 #include "mlir/IR/IntegerSet.h"
 #include "mlir/Pass/Pass.h"
-#include "mlir/Dialect/Affine/IR/AffineValueMap.h"
-#include "mlir/Dialect/Affine/IR/AffineOps.h"
-#include "mlir/Dialect/Affine/Passes.h"
-#include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/Transforms/Utils.h"
 
 #define DEBUG_TYPE "simplify-affine-structure"
@@ -31,6 +28,10 @@ namespace {
 /// identity layout ones.
 struct SimplifyAffineStructures
     : public FunctionPass<SimplifyAffineStructures> {
+/// Include the generated pass utilities.
+#define GEN_PASS_SimplifyAffineStructures
+#include "mlir/Dialect/Affine/Passes.h.inc"
+
   void runOnFunction() override;
 
   /// Utility to simplify an affine attribute and update its entry in the parent
@@ -58,15 +59,7 @@ struct SimplifyAffineStructures
     op->setAttr(name, simplified);
   }
 
-  /// Performs basic integer set simplifications. Checks if it's empty, and
-  /// replaces it with the canonical empty set if it is.
-  IntegerSet simplify(IntegerSet set) {
-    FlatAffineConstraints fac(set);
-    if (fac.isEmpty())
-      return IntegerSet::getEmptySet(set.getNumDims(), set.getNumSymbols(),
-                                     &getContext());
-    return set;
-  }
+  IntegerSet simplify(IntegerSet set) { return simplifyIntegerSet(set); }
 
   /// Performs basic affine map simplifications.
   AffineMap simplify(AffineMap map) {
@@ -105,7 +98,3 @@ void SimplifyAffineStructures::runOnFunction() {
     normalizeMemRef(allocOp);
   }
 }
-
-static PassRegistration<SimplifyAffineStructures>
-    pass("simplify-affine-structures",
-         "Simplify affine expressions in maps/sets and normalize memrefs");
diff --git a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
index d9a9ad969e511..3959459822251 100644
--- a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
@@ -574,29 +574,14 @@ namespace {
 /// Base state for the vectorize pass.
 /// Command line arguments are preempted by non-empty pass arguments.
 struct Vectorize : public FunctionPass<Vectorize> {
+/// Include the generated pass utilities.
+#define GEN_PASS_AffineVectorize
+#include "mlir/Dialect/Affine/Passes.h.inc"
+
   Vectorize() = default;
   Vectorize(const Vectorize &) {}
   Vectorize(ArrayRef<int64_t> virtualVectorSize);
   void runOnFunction() override;
-
-  /// The virtual vector size that we vectorize to.
-  ListOption<int64_t> vectorSizes{
-      *this, "virtual-vector-size",
-      llvm::cl::desc("Specify an n-D virtual vector size for vectorization"),
-      llvm::cl::ZeroOrMore, llvm::cl::CommaSeparated};
-  /// Optionally, the fixed mapping from loop to fastest varying MemRef
-  /// dimension for all the MemRefs within a loop pattern:
-  ///   the index represents the loop depth, the value represents the k^th
-  ///   fastest varying memory dimension.
-  /// This is voluntarily restrictive and is meant to precisely target a
-  /// particular loop/op pair, for testing purposes.
-  ListOption<int64_t> fastestVaryingPattern{
-      *this, "test-fastest-varying",
-      llvm::cl::desc(
-          "Specify a 1-D, 2-D or 3-D pattern of fastest varying memory"
-          " dimensions to match. See defaultPatterns in Vectorize.cpp for a"
-          " description and examples. This is used for testing purposes"),
-      llvm::cl::ZeroOrMore, llvm::cl::CommaSeparated};
 };
 
 } // end anonymous namespace
@@ -1271,7 +1256,6 @@ std::unique_ptr<OpPassBase<FuncOp>>
 mlir::createSuperVectorizePass(ArrayRef<int64_t> virtualVectorSize) {
   return std::make_unique<Vectorize>(virtualVectorSize);
 }
-
-static PassRegistration<Vectorize>
-    pass("affine-super-vectorize",
-         "Vectorize to a target independent n-D vector abstraction");
+std::unique_ptr<OpPassBase<FuncOp>> mlir::createSuperVectorizePass() {
+  return std::make_unique<Vectorize>();
+}
diff --git a/mlir/lib/Dialect/FxpMathOps/CMakeLists.txt b/mlir/lib/Dialect/FxpMathOps/CMakeLists.txt
index 45c05cf6f3a80..751db7af86f71 100644
--- a/mlir/lib/Dialect/FxpMathOps/CMakeLists.txt
+++ b/mlir/lib/Dialect/FxpMathOps/CMakeLists.txt
@@ -7,11 +7,12 @@ add_mlir_dialect_library(MLIRFxpMathOps
 
   DEPENDS
   MLIRFxpMathOpsIncGen
+  MLIRFxpMathPassIncGen
   )
 
 target_link_libraries(MLIRFxpMathOps
   PUBLIC
-  MLIRQuantOps
+  MLIRQuant
   MLIRIR
   MLIRPass
   MLIRSideEffects
diff --git a/mlir/lib/Dialect/FxpMathOps/Transforms/LowerUniformRealMath.cpp b/mlir/lib/Dialect/FxpMathOps/Transforms/LowerUniformRealMath.cpp
index 2a876a332ea6c..4136566201bd2 100644
--- a/mlir/lib/Dialect/FxpMathOps/Transforms/LowerUniformRealMath.cpp
+++ b/mlir/lib/Dialect/FxpMathOps/Transforms/LowerUniformRealMath.cpp
@@ -21,13 +21,20 @@ using namespace mlir::fxpmath::detail;
 using namespace mlir::quant;
 
 namespace {
-
 struct LowerUniformRealMathPass
     : public FunctionPass<LowerUniformRealMathPass> {
+/// Include the generated pass utilities.
+#define GEN_PASS_FxpMathLowerUniformRealMath
+#include "mlir/Dialect/FxpMathOps/Passes.h.inc"
+
   void runOnFunction() override;
 };
 
 struct LowerUniformCastsPass : public FunctionPass<LowerUniformCastsPass> {
+/// Include the generated pass utilities.
+#define GEN_PASS_FxpMathLowerUniformCasts
+#include "mlir/Dialect/FxpMathOps/Passes.h.inc"
+
   void runOnFunction() override;
 };
 
@@ -364,14 +371,11 @@ void LowerUniformRealMathPass::runOnFunction() {
   applyPatternsGreedily(fn, patterns);
 }
 
-OpPassBase<FuncOp> *mlir::fxpmath::createLowerUniformRealMathPass() {
-  return new LowerUniformRealMathPass();
+std::unique_ptr<OpPassBase<FuncOp>>
+mlir::fxpmath::createLowerUniformRealMathPass() {
+  return std::make_unique<LowerUniformRealMathPass>();
 }
 
-static PassRegistration<LowerUniformRealMathPass> lowerUniformRealMathPass(
-    "fxpmath-lower-uniform-real-math",
-    "Lowers uniform-quantized real math ops to integer arithmetic.");
-
 //===----------------------------------------------------------------------===//
 // LowerUniformCasts pass
 //===----------------------------------------------------------------------===//
@@ -384,10 +388,7 @@ void LowerUniformCastsPass::runOnFunction() {
   applyPatternsGreedily(fn, patterns);
 }
 
-OpPassBase<FuncOp> *mlir::fxpmath::createLowerUniformCastsPass() {
-  return new LowerUniformCastsPass();
+std::unique_ptr<OpPassBase<FuncOp>>
+mlir::fxpmath::createLowerUniformCastsPass() {
+  return std::make_unique<LowerUniformCastsPass>();
 }
-
-static PassRegistration<LowerUniformCastsPass>
-    lowerUniformCastsPass("fxpmath-lower-uniform-casts",
-                          "Lowers uniform-quantized casts.");
diff --git a/mlir/lib/Dialect/GPU/CMakeLists.txt b/mlir/lib/Dialect/GPU/CMakeLists.txt
index e71a018a451c3..ad63b3669409d 100644
--- a/mlir/lib/Dialect/GPU/CMakeLists.txt
+++ b/mlir/lib/Dialect/GPU/CMakeLists.txt
@@ -10,6 +10,7 @@ add_mlir_dialect_library(MLIRGPU
 
   DEPENDS
   MLIRGPUOpsIncGen
+  MLIRGPUPassIncGen
   MLIRParallelLoopMapperAttrGen
   MLIRParallelLoopMapperEnumsGen
   )
diff --git a/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp b/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
index 99029b010ba32..d20f23de78eca 100644
--- a/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
@@ -205,7 +205,6 @@ static void convertToLaunchFuncOp(gpu::LaunchOp launchOp,
 }
 
 namespace {
-
 /// Pass that moves the kernel of each LaunchOp into its separate nested module.
 ///
 /// This pass moves the kernel code of each LaunchOp into a function created
@@ -217,6 +216,10 @@ namespace {
 /// symbol of the cubin accessor function.
 class GpuKernelOutliningPass : public ModulePass<GpuKernelOutliningPass> {
 public:
+/// Include the generated pass utilities.
+#define GEN_PASS_GpuKernelOutlining
+#include "mlir/Dialect/GPU/Passes.h.inc"
+
   void runOnModule() override {
     SymbolTable symbolTable(getModule());
     bool modified = false;
@@ -300,7 +303,3 @@ class GpuKernelOutliningPass : public ModulePass<GpuKernelOutliningPass> {
 std::unique_ptr<OpPassBase<ModuleOp>> mlir::createGpuKernelOutliningPass() {
   return std::make_unique<GpuKernelOutliningPass>();
 }
-
-static PassRegistration<GpuKernelOutliningPass>
-    pass("gpu-kernel-outlining",
-         "Outline gpu.launch bodies to kernel functions.");
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/CMakeLists.txt b/mlir/lib/Dialect/LLVMIR/Transforms/CMakeLists.txt
index 6b37b060de6c8..216586e6242d0 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/CMakeLists.txt
@@ -1,5 +1,8 @@
 add_mlir_dialect_library(MLIRLLVMIRTransforms
   LegalizeForExport.cpp
+
+  DEPENDS
+  MLIRLLVMPassIncGen
   )
 
 target_link_libraries(MLIRLLVMIRTransforms
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/LegalizeForExport.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/LegalizeForExport.cpp
index fce4045f1aa3e..663f0c11432b3 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/LegalizeForExport.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/LegalizeForExport.cpp
@@ -58,6 +58,10 @@ void mlir::LLVM::ensureDistinctSuccessors(Operation *op) {
 
 namespace {
 struct LegalizeForExportPass : public OperationPass<LegalizeForExportPass> {
+/// Include the generated pass utilities.
+#define GEN_PASS_LLVMLegalizeForExport
+#include "mlir/Dialect/LLVMIR/Transforms/Passes.h.inc"
+
   void runOnOperation() override {
     LLVM::ensureDistinctSuccessors(getOperation());
   }
@@ -67,7 +71,3 @@ struct LegalizeForExportPass : public OperationPass<LegalizeForExportPass> {
 std::unique_ptr<Pass> LLVM::createLegalizeForExportPass() {
   return std::make_unique<LegalizeForExportPass>();
 }
-
-static PassRegistration<LegalizeForExportPass>
-    pass("llvm-legalize-for-export",
-         "Legalize LLVM dialect to be convertible to LLVM IR");
diff --git a/mlir/lib/Dialect/Linalg/Analysis/DependenceAnalysis.cpp b/mlir/lib/Dialect/Linalg/Analysis/DependenceAnalysis.cpp
index bf52039a6dc1d..90ce8fd6bb0b8 100644
--- a/mlir/lib/Dialect/Linalg/Analysis/DependenceAnalysis.cpp
+++ b/mlir/lib/Dialect/Linalg/Analysis/DependenceAnalysis.cpp
@@ -24,24 +24,6 @@ using namespace mlir::linalg;
 
 using llvm::dbgs;
 
-#ifndef NDEBUG
-static StringRef toStringRef(LinalgDependenceGraph::DependenceType dt) {
-  switch (dt) {
-  case LinalgDependenceGraph::DependenceType::RAW:
-    return "RAW";
-  case LinalgDependenceGraph::DependenceType::RAR:
-    return "RAR";
-  case LinalgDependenceGraph::DependenceType::WAR:
-    return "WAR";
-  case LinalgDependenceGraph::DependenceType::WAW:
-    return "WAW";
-  default:
-    break;
-  }
-  llvm_unreachable("Unexpected DependenceType");
-}
-#endif
-
 Value Aliases::find(Value v) {
   if (v.isa<BlockArgument>())
     return v;
@@ -76,6 +58,22 @@ Value Aliases::find(Value v) {
   }
 }
 
+StringRef LinalgDependenceGraph::getDependenceTypeStr(DependenceType depType) {
+  switch (depType) {
+  case LinalgDependenceGraph::DependenceType::RAW:
+    return "RAW";
+  case LinalgDependenceGraph::DependenceType::RAR:
+    return "RAR";
+  case LinalgDependenceGraph::DependenceType::WAR:
+    return "WAR";
+  case LinalgDependenceGraph::DependenceType::WAW:
+    return "WAW";
+  default:
+    break;
+  }
+  llvm_unreachable("Unexpected DependenceType");
+}
+
 LinalgDependenceGraph
 LinalgDependenceGraph::buildDependenceGraph(Aliases &aliases, FuncOp f) {
   SmallVector<Operation *, 8> linalgOps;
@@ -100,7 +98,7 @@ LinalgDependenceGraph::LinalgDependenceGraph(Aliases &aliases,
 void LinalgDependenceGraph::addDependenceElem(DependenceType dt,
                                               LinalgOpView indexingOpView,
                                               LinalgOpView dependentOpView) {
-  LLVM_DEBUG(dbgs() << "\nAdd dep type " << toStringRef(dt) << ":\t"
+  LLVM_DEBUG(dbgs() << "\nAdd dep type " << getDependenceTypeStr(dt) << ":\t"
                     << *indexingOpView.op << " -> " << *dependentOpView.op);
   dependencesFromGraphs[dt][indexingOpView.op].push_back(
       LinalgDependenceGraphElem{dependentOpView, indexingOpView.view});
@@ -227,8 +225,8 @@ LinalgDependenceGraph::findOperationsWithCoveringDependences(
         continue;
       auto *op = dependence.dependentOpView.op;
       LLVM_DEBUG(dbgs() << "\n***Found covering dependence of type "
-                        << toStringRef(dt) << ": " << *src << " -> " << *op
-                        << " on " << dependence.indexingView);
+                        << getDependenceTypeStr(dt) << ": " << *src << " -> "
+                        << *op << " on " << dependence.indexingView);
       res.push_back(op);
     }
   }
diff --git a/mlir/lib/Dialect/Linalg/EDSC/Builders.cpp b/mlir/lib/Dialect/Linalg/EDSC/Builders.cpp
index 198c7fc698dd2..10c18107fd8e6 100644
--- a/mlir/lib/Dialect/Linalg/EDSC/Builders.cpp
+++ b/mlir/lib/Dialect/Linalg/EDSC/Builders.cpp
@@ -221,9 +221,8 @@ void mlir::edsc::ops::macRegionBuilder(ArrayRef<BlockArgument> args) {
   linalg_yield((c + a * b).getValue());
 }
 
-Operation *mlir::edsc::ops::linalg_pointwise(UnaryPointwiseOpBuilder unaryOp,
-                                             StructuredIndexed I,
-                                             StructuredIndexed O) {
+Operation *mlir::edsc::ops::linalg_generic_pointwise(
+    UnaryPointwiseOpBuilder unaryOp, StructuredIndexed I, StructuredIndexed O) {
   SmallVector<IteratorType, 4> iterTypes(O.getExprs().size(),
                                          IteratorType::Parallel);
   if (O.getType().isa<RankedTensorType>()) {
@@ -242,18 +241,17 @@ Operation *mlir::edsc::ops::linalg_pointwise(UnaryPointwiseOpBuilder unaryOp,
   return makeGenericLinalgOp(iterTypes, {I}, {O}, fun);
 }
 
-Operation *mlir::edsc::ops::linalg_pointwise_tanh(StructuredIndexed I,
-                                                  StructuredIndexed O) {
+Operation *mlir::edsc::ops::linalg_generic_pointwise_tanh(StructuredIndexed I,
+                                                          StructuredIndexed O) {
   UnaryPointwiseOpBuilder unOp(
       [](ValueHandle a) -> Value { return std_tanh(a); });
-  return linalg_pointwise(unOp, I, O);
+  return linalg_generic_pointwise(unOp, I, O);
 }
 
 /// Binary pointwise operation (with broadcast) entry point.
-Operation *mlir::edsc::ops::linalg_pointwise(BinaryPointwiseOpBuilder binaryOp,
-                                             StructuredIndexed I1,
-                                             StructuredIndexed I2,
-                                             StructuredIndexed O) {
+Operation *mlir::edsc::ops::linalg_generic_pointwise(
+    BinaryPointwiseOpBuilder binaryOp, StructuredIndexed I1,
+    StructuredIndexed I2, StructuredIndexed O) {
   SmallVector<IteratorType, 4> iterTypes(O.getExprs().size(),
                                          IteratorType::Parallel);
   if (O.getType().isa<RankedTensorType>()) {
@@ -272,28 +270,29 @@ Operation *mlir::edsc::ops::linalg_pointwise(BinaryPointwiseOpBuilder binaryOp,
   return makeGenericLinalgOp(iterTypes, {I1, I2}, {O}, fun);
 }
 
-Operation *mlir::edsc::ops::linalg_pointwise_add(StructuredIndexed I1,
-                                                 StructuredIndexed I2,
-                                                 StructuredIndexed O) {
+Operation *mlir::edsc::ops::linalg_generic_pointwise_add(StructuredIndexed I1,
+                                                         StructuredIndexed I2,
+                                                         StructuredIndexed O) {
   using edsc::op::operator+;
   BinaryPointwiseOpBuilder binOp(
       [](ValueHandle a, ValueHandle b) -> Value { return a + b; });
-  return linalg_pointwise(binOp, I1, I2, O);
+  return linalg_generic_pointwise(binOp, I1, I2, O);
 }
 
-Operation *mlir::edsc::ops::linalg_pointwise_max(StructuredIndexed I1,
-                                                 StructuredIndexed I2,
-                                                 StructuredIndexed O) {
+Operation *mlir::edsc::ops::linalg_generic_pointwise_max(StructuredIndexed I1,
+                                                         StructuredIndexed I2,
+                                                         StructuredIndexed O) {
   BinaryPointwiseOpBuilder binOp([](ValueHandle a, ValueHandle b) -> Value {
     using edsc::op::operator>;
     return std_select(a > b, a, b).getValue();
   });
-  return linalg_pointwise(binOp, I1, I2, O);
+  return linalg_generic_pointwise(binOp, I1, I2, O);
 }
 
-Operation *mlir::edsc::ops::linalg_matmul(ValueHandle vA, ValueHandle vB,
-                                          ValueHandle vC,
-                                          MatmulRegionBuilder regionBuilder) {
+Operation *
+mlir::edsc::ops::linalg_generic_matmul(ValueHandle vA, ValueHandle vB,
+                                       ValueHandle vC,
+                                       MatmulRegionBuilder regionBuilder) {
   // clang-format off
   AffineExpr m, n, k;
   bindDims(ScopedContext::getContext(), m, n, k);
@@ -306,9 +305,10 @@ Operation *mlir::edsc::ops::linalg_matmul(ValueHandle vA, ValueHandle vB,
   // clang-format on
 }
 
-Operation *mlir::edsc::ops::linalg_matmul(ValueHandle vA, ValueHandle vB,
-                                          RankedTensorType tC,
-                                          MatmulRegionBuilder regionBuilder) {
+Operation *
+mlir::edsc::ops::linalg_generic_matmul(ValueHandle vA, ValueHandle vB,
+                                       RankedTensorType tC,
+                                       MatmulRegionBuilder regionBuilder) {
   // clang-format off
   AffineExpr m, n, k;
   bindDims(ScopedContext::getContext(), m, n, k);
@@ -321,9 +321,10 @@ Operation *mlir::edsc::ops::linalg_matmul(ValueHandle vA, ValueHandle vB,
   // clang-format on
 }
 
-Operation *mlir::edsc::ops::linalg_matmul(ValueHandle vA, ValueHandle vB,
-                                          ValueHandle vC, RankedTensorType tD,
-                                          MatmulRegionBuilder regionBuilder) {
+Operation *
+mlir::edsc::ops::linalg_generic_matmul(ValueHandle vA, ValueHandle vB,
+                                       ValueHandle vC, RankedTensorType tD,
+                                       MatmulRegionBuilder regionBuilder) {
   // clang-format off
   AffineExpr m, n, k;
   bindDims(ScopedContext::getContext(), m, n, k);
@@ -336,10 +337,11 @@ Operation *mlir::edsc::ops::linalg_matmul(ValueHandle vA, ValueHandle vB,
   // clang-format on
 }
 
-Operation *mlir::edsc::ops::linalg_conv_nhwc(ValueHandle vI, ValueHandle vW,
-                                             ValueHandle vO,
-                                             ArrayRef<int> strides,
-                                             ArrayRef<int> dilations) {
+Operation *mlir::edsc::ops::linalg_generic_conv_nhwc(ValueHandle vI,
+                                                     ValueHandle vW,
+                                                     ValueHandle vO,
+                                                     ArrayRef<int> strides,
+                                                     ArrayRef<int> dilations) {
   MLIRContext *ctx = ScopedContext::getContext();
   // TODO(ntv) some template magic to make everything rank-polymorphic.
   assert((dilations.empty() || dilations.size() == 2) && "only 2-D conv atm");
@@ -370,7 +372,7 @@ Operation *mlir::edsc::ops::linalg_conv_nhwc(ValueHandle vI, ValueHandle vW,
   // clang-format on
 }
 
-Operation *mlir::edsc::ops::linalg_dilated_conv_nhwc(
+Operation *mlir::edsc::ops::linalg_generic_dilated_conv_nhwc(
     ValueHandle vI, ValueHandle vW, ValueHandle vO, int depth_multiplier,
     ArrayRef<int> strides, ArrayRef<int> dilations) {
   MLIRContext *ctx = ScopedContext::getContext();
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
index aa340e55e8b57..24dcf73709431 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
@@ -140,7 +140,6 @@ static void printGenericOp(OpAsmPrinter &p, GenericOpType op) {
     p.printRegion(op.region());
   p.printOptionalAttrDict(op.getAttrs(), attrNames);
   p << ": " << op.getOperandTypes();
-
   auto outputTensorTypes = op.getResultTypes();
   if (!outputTensorTypes.empty())
     p << " -> " << outputTensorTypes;
@@ -827,8 +826,10 @@ static LogicalResult verify(CopyOp op) {
   return success();
 }
 
-static LogicalResult
-verifyStrideOrDilation(ConvOp op, ArrayRef<Attribute> attrs, bool isStride) {
+template <typename LinalgPoolingOp>
+static LogicalResult verifyStrideOrDilation(LinalgPoolingOp op,
+                                            ArrayRef<Attribute> attrs,
+                                            bool isStride) {
   auto strideOrDilation = isStride ? "stride" : "dilation";
   if (attrs.size() != op.getNumWindowLoops())
     return op.emitOpError("expects num ")
@@ -860,6 +861,41 @@ static LogicalResult verify(ConvOp op) {
   return success();
 }
 
+template <typename PoolingOp>
+static LogicalResult verifySingleInputPoolingOp(PoolingOp op) {
+  auto inputType = op.input().getType().template cast<MemRefType>();
+  auto outputType = op.output().getType().template cast<MemRefType>();
+  if (outputType.getElementType() != inputType.getElementType())
+    return op.emitOpError("expects memref elemental types to match");
+
+  auto windowDimsType = op.windowDims().getType().template cast<MemRefType>();
+  if (outputType.getRank() != inputType.getRank() ||
+      outputType.getRank() != windowDimsType.getRank())
+    return op.emitOpError("expects memref ranks to match");
+
+  if (auto strides = op.strides()) {
+    if (failed(
+            verifyStrideOrDilation(op, strides->getValue(), /*isStride=*/true)))
+      return failure();
+  }
+  if (auto dilations = op.dilations()) {
+    if (failed(verifyStrideOrDilation(op, dilations->getValue(),
+                                      /*isStride=*/false)))
+      return failure();
+  }
+  return success();
+}
+
+static LogicalResult verify(PoolingMaxOp op) {
+  return verifySingleInputPoolingOp(op);
+}
+static LogicalResult verify(PoolingMinOp op) {
+  return verifySingleInputPoolingOp(op);
+}
+static LogicalResult verify(PoolingSumOp op) {
+  return verifySingleInputPoolingOp(op);
+}
+
 namespace mlir {
 namespace linalg {
 
@@ -894,21 +930,34 @@ mlir::linalg::makeAffineDimExprs(unsigned num, unsigned &startIdx,
   return res;
 }
 
+template <typename PoolingOp>
 SmallVector<AffineExpr, 4>
-mlir::linalg::weightedConvInputIndex(ConvOp op, ArrayRef<AffineExpr> xs,
-                                     ArrayRef<AffineExpr> zs) {
-  assert(xs.size() == zs.size());
+mlir::linalg::weightedPoolingInputIndex(PoolingOp op,
+                                        ArrayRef<AffineExpr> outputDims,
+                                        ArrayRef<AffineExpr> windowDims) {
+  assert(outputDims.size() == windowDims.size());
   SmallVector<AffineExpr, 4> res;
-  res.reserve(xs.size());
-  for (unsigned i = 0, e = xs.size(); i < e; ++i) {
+  res.reserve(outputDims.size());
+  for (unsigned i = 0, e = outputDims.size(); i < e; ++i) {
     // TODO(ntv): add a level of indirection to linalg.generic.
-    auto expr =
-        op.getStride(i) * xs[i] + op.getDilation(i) * zs[i] - op.getLowPad(i);
+    auto expr = op.getStride(i) * outputDims[i] +
+                op.getDilation(i) * windowDims[i] - op.getLowPad(i);
     res.push_back(expr);
   }
   return res;
 }
 
+#define INSTANTIATE_WEIGHTED_POOLING_INPUT_INDEX(OP_TYPE)                      \
+  template SmallVector<AffineExpr, 4>                                          \
+  mlir::linalg::weightedPoolingInputIndex<OP_TYPE>(                            \
+      OP_TYPE op, ArrayRef<AffineExpr> outputDims,                             \
+      ArrayRef<AffineExpr> windowDims);
+
+INSTANTIATE_WEIGHTED_POOLING_INPUT_INDEX(ConvOp)
+INSTANTIATE_WEIGHTED_POOLING_INPUT_INDEX(PoolingMaxOp)
+INSTANTIATE_WEIGHTED_POOLING_INPUT_INDEX(PoolingMinOp)
+INSTANTIATE_WEIGHTED_POOLING_INPUT_INDEX(PoolingSumOp)
+
 SmallVector<AffineExpr, 4> mlir::linalg::concat(ArrayRef<AffineExpr> a,
                                                 ArrayRef<AffineExpr> b) {
   auto rangeA = llvm::make_range(a.begin(), a.end());
@@ -959,6 +1008,18 @@ LogicalResult ConvOp::fold(ArrayRef<Attribute>,
                            SmallVectorImpl<OpFoldResult> &) {
   return foldMemRefCast(*this);
 }
+LogicalResult PoolingMaxOp::fold(ArrayRef<Attribute>,
+                                 SmallVectorImpl<OpFoldResult> &) {
+  return foldMemRefCast(*this);
+}
+LogicalResult PoolingMinOp::fold(ArrayRef<Attribute>,
+                                 SmallVectorImpl<OpFoldResult> &) {
+  return foldMemRefCast(*this);
+}
+LogicalResult PoolingSumOp::fold(ArrayRef<Attribute>,
+                                 SmallVectorImpl<OpFoldResult> &) {
+  return foldMemRefCast(*this);
+}
 LogicalResult CopyOp::fold(ArrayRef<Attribute>,
                            SmallVectorImpl<OpFoldResult> &) {
   return foldMemRefCast(*this);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt
index 349c2d6980acf..93f7142ce2df1 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt
@@ -10,6 +10,7 @@ add_mlir_dialect_library(MLIRLinalgTransforms
 
   DEPENDS
   intrinsics_gen
+  MLIRLinalgPassIncGen
   MLIRLinalgTransformPatternsIncGen
   )
 target_link_libraries(MLIRLinalgTransforms
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp b/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
index b6af16c979c35..d3c56851a3851 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
@@ -157,9 +157,9 @@ static LinalgOp fuse(Value producedView, LinalgOp producer, LinalgOp consumer,
   }
 
   auto subView = dyn_cast_or_null<SubViewOp>(
-      consumer.getInput(consumerIdx).getDefiningOp());
-  auto slice =
-      dyn_cast_or_null<SliceOp>(consumer.getInput(consumerIdx).getDefiningOp());
+      consumer.getBuffer(consumerIdx).getDefiningOp());
+  auto slice = dyn_cast_or_null<SliceOp>(
+      consumer.getBuffer(consumerIdx).getDefiningOp());
   assert(subView || slice);
   (void)subView;
   (void)slice;
@@ -274,16 +274,15 @@ bool mlir::linalg::isFusableInto(const LinalgDependenceGraph &graph,
   return true;
 }
 
-// Only consider RAW atm.
-Optional<FusionInfo> mlir::linalg::fuseProducerOf(
-    OpBuilder &b, LinalgOp consumer, unsigned consumerIdx,
-    const LinalgDependenceGraph &graph, OperationFolder *folder) {
+static Optional<FusionInfo>
+fuseProducerOfDep(OpBuilder &b, LinalgOp consumer, unsigned consumerIdx,
+                  const LinalgDependenceGraph &graph, OperationFolder *folder,
+                  LinalgDependenceGraph::DependenceType depType) {
   assert(consumer.hasBufferSemantics() &&
          "expected linalg op with buffer semantics");
   LLVM_DEBUG(dbgs() << "\nStart examining consumer: "
                     << *consumer.getOperation());
-  for (auto dependence : graph.getDependencesInto(
-           consumer, LinalgDependenceGraph::DependenceType::RAW)) {
+  for (auto dependence : graph.getDependencesInto(consumer, depType)) {
     LLVM_DEBUG(dbgs() << "\n***Consider producer:\t"
                       << *dependence.dependentOpView.op << "\n");
     auto producer = cast<LinalgOp>(dependence.dependentOpView.op);
@@ -294,7 +293,7 @@ Optional<FusionInfo> mlir::linalg::fuseProducerOf(
 
     // Check that the dependence is indeed on the input `consumerIdx` view.
     auto consumedView = dependence.indexingView;
-    if (consumer.getInput(consumerIdx) != consumedView)
+    if (consumer.getBuffer(consumerIdx) != consumedView)
       continue;
 
     // Consumer consumes this view, `isStructurallyFusableProducer` also checks
@@ -302,9 +301,10 @@ Optional<FusionInfo> mlir::linalg::fuseProducerOf(
     auto producedView = dependence.dependentOpView.view;
     auto producerIdx = producer.getIndexOfOutputBuffer(producedView).getValue();
     // `consumerIdx` and `producerIdx` exist by construction.
-    LLVM_DEBUG(dbgs() << "\nRAW producer: " << *producer.getOperation()
-                      << " view: " << producedView
-                      << " output index: " << producerIdx);
+    LLVM_DEBUG(dbgs() << "\n"
+                      << LinalgDependenceGraph::getDependenceTypeStr(depType)
+                      << "producer: " << *producer.getOperation() << " view: "
+                      << producedView << " output index: " << producerIdx);
 
     // Must be a subview or a slice to guarantee there are loops we can fuse
     // into.
@@ -332,6 +332,22 @@ Optional<FusionInfo> mlir::linalg::fuseProducerOf(
   return llvm::None;
 }
 
+// Only consider RAW and WAW atm.
+Optional<FusionInfo> mlir::linalg::fuseProducerOf(
+    OpBuilder &b, LinalgOp consumer, unsigned consumerIdx,
+    const LinalgDependenceGraph &graph, OperationFolder *folder) {
+  SmallVector<LinalgDependenceGraph::DependenceType, 4> deps = {
+      LinalgDependenceGraph::DependenceType::RAW,
+      LinalgDependenceGraph::DependenceType::WAW,
+  };
+  for (auto dep : deps) {
+    if (auto res =
+            fuseProducerOfDep(b, consumer, consumerIdx, graph, folder, dep))
+      return res;
+  }
+  return llvm::None;
+}
+
 /// Checks if two Generic ops are fusible, when one is a producer and another is
 /// a consumer (with the result of the producer being the `consumerIdx` operand
 /// of the consumer).
@@ -498,7 +514,8 @@ static void fuseLinalgOpsGreedily(FuncOp f) {
   // The current naive and expensive reconstruction of the graph should be
   // removed.
   for (auto *op : llvm::reverse(linalgOps)) {
-    for (unsigned id = 0, e = LinalgOp(op).getNumInputs(); id < e; ++id) {
+    for (unsigned id = 0, e = LinalgOp(op).getNumInputsAndOutputBuffers();
+         id < e; ++id) {
       linalg::Aliases aliases;
       linalg::LinalgDependenceGraph graph(aliases, linalgOps);
       if (auto info = fuseProducerOf(b, op, id, graph, &folder)) {
@@ -551,6 +568,10 @@ struct FuseGenericTensorOps : public OpRewritePattern<GenericOp> {
 
 /// Pass that fuses generic ops on tensors. Used only for testing.
 struct FusionOfTensorOpsPass : public OperationPass<FusionOfTensorOpsPass> {
+/// Include the generated pass utilities.
+#define GEN_PASS_LinalgFusionOfTensorOps
+#include "mlir/Dialect/Linalg/Passes.h.inc"
+
   void runOnOperation() override {
     OwningRewritePatternList patterns;
     Operation *op = getOperation();
@@ -560,6 +581,10 @@ struct FusionOfTensorOpsPass : public OperationPass<FusionOfTensorOpsPass> {
 };
 
 struct LinalgFusionPass : public FunctionPass<LinalgFusionPass> {
+/// Include the generated pass utilities.
+#define GEN_PASS_LinalgFusion
+#include "mlir/Dialect/Linalg/Passes.h.inc"
+
   void runOnFunction() override { fuseLinalgOpsGreedily(getFunction()); }
 };
 } // namespace
@@ -568,9 +593,6 @@ std::unique_ptr<OpPassBase<FuncOp>> mlir::createLinalgFusionPass() {
   return std::make_unique<LinalgFusionPass>();
 }
 
-static PassRegistration<LinalgFusionPass>
-    pass("linalg-fusion", "Fuse operations in the linalg dialect");
-
-static PassRegistration<FusionOfTensorOpsPass>
-    tensorOpsPass("linalg-fusion-for-tensor-ops",
-                  "Fuse operations on RankedTensorType in linalg dialect");
+std::unique_ptr<Pass> mlir::createLinalgFusionOfTensorOpsPass() {
+  return std::make_unique<FusionOfTensorOpsPass>();
+}
diff --git a/mlir/lib/Dialect/Linalg/Transforms/LinalgToLoops.cpp b/mlir/lib/Dialect/Linalg/Transforms/LinalgToLoops.cpp
index 5e1748cc47c00..4e26080eec536 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/LinalgToLoops.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/LinalgToLoops.cpp
@@ -101,11 +101,28 @@ static void inlineRegionAndEmitStdStore(OpType op,
   assert(isa<YieldOp>(terminator) &&
          "expected an yield op in the end of the region");
   for (unsigned i = 0, e = terminator.getNumOperands(); i < e; ++i) {
-    std_store(map.lookup(terminator.getOperand(i)), outputBuffers[i],
+    std_store(map.lookupOrDefault(terminator.getOperand(i)), outputBuffers[i],
               indexing[i]);
   }
 }
 
+// Returns a pair that contains input indices and output indices of a
+// SingleInputPoolingOp `op`.
+template <typename SingleInputPoolingOp>
+static std::pair<SmallVector<ValueHandle, 8>, SmallVector<ValueHandle, 8>>
+getInputAndOutputIndices(ArrayRef<Value> allIvs, SingleInputPoolingOp op) {
+  auto &b = ScopedContext::getBuilder();
+  auto loc = ScopedContext::getLocation();
+  auto mapsRange = op.indexing_maps().template getAsRange<AffineMapAttr>();
+  auto maps =
+      functional::map([](AffineMapAttr a) { return a.getValue(); }, mapsRange);
+  SmallVector<ValueHandle, 8> iIdx(
+      makeCanonicalAffineApplies(b, loc, maps[0], allIvs));
+  SmallVector<ValueHandle, 8> oIdx(
+      makeCanonicalAffineApplies(b, loc, maps[2], allIvs));
+  return {iIdx, oIdx};
+}
+
 namespace {
 template <typename IndexedValueType, typename LinalgOpType>
 class LinalgScopedEmitter {};
@@ -203,7 +220,7 @@ template <typename IndexedValueType>
 class LinalgScopedEmitter<IndexedValueType, ConvOp> {
 public:
   /// Returns the input value of convOp. If the indices in `imIdx` is out of
-  /// boundrary, returns 0 instead.
+  /// boundary, returns 0 instead.
   static ValueHandle getConvOpInput(ConvOp convOp, IndexedValueType im,
                                     ArrayRef<ValueHandle> imIdx) {
     // TODO(ntv): add a level of indirection to linalg.generic.
@@ -234,7 +251,7 @@ class LinalgScopedEmitter<IndexedValueType, ConvOp> {
       ValueHandle rightBound = std_dim(convOp.input(), idx);
       conds.push_back(conds.back() || (dim >= rightBound));
 
-      // When padding is involed, the indices will only be shifted to negative,
+      // When padding is involved, the indices will only be shifted to negative,
       // so having a max op is enough.
       auto maxMap = AffineMap::get(/*dimCount=*/1, 0,
                                    {getAffineDimExpr(/*position=*/0, context),
@@ -273,6 +290,57 @@ class LinalgScopedEmitter<IndexedValueType, ConvOp> {
   }
 };
 
+template <typename IndexedValueType>
+class LinalgScopedEmitter<IndexedValueType, PoolingMaxOp> {
+public:
+  static void emitScalarImplementation(ArrayRef<Value> allIvs,
+                                       PoolingMaxOp op) {
+    auto indices = getInputAndOutputIndices(allIvs, op);
+    ValueHandleArray iIdx(indices.first);
+    ValueHandleArray oIdx(indices.second);
+
+    // Emit scalar form.
+    ValueHandle lhs = std_load(op.output(), oIdx);
+    ValueHandle rhs = std_load(op.input(), iIdx);
+    using edsc::op::operator>;
+    ValueHandle maxValue = std_select(lhs > rhs, lhs, rhs);
+    std_store(maxValue, op.output(), oIdx);
+  }
+};
+
+template <typename IndexedValueType>
+class LinalgScopedEmitter<IndexedValueType, PoolingMinOp> {
+public:
+  static void emitScalarImplementation(ArrayRef<Value> allIvs,
+                                       PoolingMinOp op) {
+    auto indices = getInputAndOutputIndices(allIvs, op);
+    ValueHandleArray iIdx(indices.first);
+    ValueHandleArray oIdx(indices.second);
+
+    // Emit scalar form.
+    ValueHandle lhs = std_load(op.output(), oIdx);
+    ValueHandle rhs = std_load(op.input(), iIdx);
+    using edsc::op::operator<;
+    ValueHandle minValue = std_select(lhs < rhs, lhs, rhs);
+    std_store(minValue, op.output(), oIdx);
+  }
+};
+
+template <typename IndexedValueType>
+class LinalgScopedEmitter<IndexedValueType, PoolingSumOp> {
+public:
+  static void emitScalarImplementation(ArrayRef<Value> allIvs,
+                                       PoolingSumOp op) {
+    auto indices = getInputAndOutputIndices(allIvs, op);
+    SmallVector<ValueHandle, 8> iIdx = indices.first;
+    SmallVector<ValueHandle, 8> oIdx = indices.second;
+    IndexedValueType input(op.input()), output(op.output());
+
+    // Emit scalar form.
+    output(oIdx) += input(iIdx);
+  }
+};
+
 // Emits the MLIR for the scalar part of the generic op by:
 //   1. Emitting std_load and std_store ops for each input and output
 //      view in order. This is achieved by applying the appropriate input or
@@ -572,14 +640,6 @@ void FillRewritePatterns(OwningRewritePatternList &patterns, MLIRContext *ctx) {
                      >::build(patterns, ctx);
 }
 
-namespace {
-template <typename LoopType, typename IndexedValueType>
-struct LowerLinalgToLoopsPass
-    : public FunctionPass<LowerLinalgToLoopsPass<LoopType, IndexedValueType>> {
-  void runOnFunction() override;
-};
-} // namespace
-
 // Local folding pattern for AffineApplyOp that we can apply greedily.
 // This replaces AffineApplyOp by the proper value in cases where the associated
 // map is trivial. A trivial map here is defined as a map with a single result
@@ -619,8 +679,7 @@ struct FoldAffineOp : public RewritePattern {
 } // namespace
 
 template <typename LoopType, typename IndexedValueType>
-void LowerLinalgToLoopsPass<LoopType, IndexedValueType>::runOnFunction() {
-  auto *context = &this->getContext();
+static void lowerLinalgToLoopsImpl(Operation *op, MLIRContext *context) {
   OwningRewritePatternList patterns;
   // Canonicalization and folding patterns applied greedily allow cleaning up
   // the emitted IR on the fly.
@@ -630,24 +689,54 @@ void LowerLinalgToLoopsPass<LoopType, IndexedValueType>::runOnFunction() {
   AffineApplyOp::getCanonicalizationPatterns(patterns, context);
   patterns.insert<FoldAffineOp>(context);
   // Just apply the patterns greedily.
-  applyPatternsGreedily(this->getFunction(), patterns);
+  applyPatternsGreedily(op, patterns);
 }
 
+namespace {
+struct LowerToAffineLoops : public FunctionPass<LowerToAffineLoops> {
+/// Include the generated pass utilities.
+#define GEN_PASS_LinalgLowerToAffineLoops
+#include "mlir/Dialect/Linalg/Passes.h.inc"
+
+  void runOnFunction() override {
+    lowerLinalgToLoopsImpl<AffineForOp, AffineIndexedValue>(getFunction(),
+                                                            &getContext());
+  }
+};
+struct LowerToLoops : public FunctionPass<LowerToLoops> {
+/// Include the generated pass utilities.
+#define GEN_PASS_LinalgLowerToLoops
+#include "mlir/Dialect/Linalg/Passes.h.inc"
+
+  void runOnFunction() override {
+    lowerLinalgToLoopsImpl<loop::ForOp, StdIndexedValue>(getFunction(),
+                                                         &getContext());
+  }
+};
+struct LowerToParallelLoops : public FunctionPass<LowerToParallelLoops> {
+/// Include the generated pass utilities.
+#define GEN_PASS_LinalgLowerToParallelLoops
+#include "mlir/Dialect/Linalg/Passes.h.inc"
+
+  void runOnFunction() override {
+    lowerLinalgToLoopsImpl<loop::ParallelOp, StdIndexedValue>(getFunction(),
+                                                              &getContext());
+  }
+};
+} // namespace
+
 std::unique_ptr<OpPassBase<FuncOp>> mlir::createConvertLinalgToLoopsPass() {
-  return std::make_unique<
-      LowerLinalgToLoopsPass<loop::ForOp, StdIndexedValue>>();
+  return std::make_unique<LowerToLoops>();
 }
 
 std::unique_ptr<OpPassBase<FuncOp>>
 mlir::createConvertLinalgToParallelLoopsPass() {
-  return std::make_unique<
-      LowerLinalgToLoopsPass<loop::ParallelOp, StdIndexedValue>>();
+  return std::make_unique<LowerToParallelLoops>();
 }
 
 std::unique_ptr<OpPassBase<FuncOp>>
 mlir::createConvertLinalgToAffineLoopsPass() {
-  return std::make_unique<
-      LowerLinalgToLoopsPass<AffineForOp, AffineIndexedValue>>();
+  return std::make_unique<LowerToAffineLoops>();
 }
 
 /// Emits a loop nest of `loop.for` with the proper body for `op`.
@@ -688,6 +777,9 @@ INSTANTIATE_LINALG_OP_TO_LOOPS(DotOp)
 INSTANTIATE_LINALG_OP_TO_LOOPS(MatvecOp)
 INSTANTIATE_LINALG_OP_TO_LOOPS(MatmulOp)
 INSTANTIATE_LINALG_OP_TO_LOOPS(ConvOp)
+INSTANTIATE_LINALG_OP_TO_LOOPS(PoolingMaxOp)
+INSTANTIATE_LINALG_OP_TO_LOOPS(PoolingMinOp)
+INSTANTIATE_LINALG_OP_TO_LOOPS(PoolingSumOp)
 INSTANTIATE_LINALG_OP_TO_LOOPS(GenericOp)
 INSTANTIATE_LINALG_OP_TO_LOOPS(IndexedGenericOp)
 
@@ -696,19 +788,3 @@ INSTANTIATE_LINALG_OP_TO_LOOPS(IndexedGenericOp)
 template LogicalResult
 mlir::linalg::linalgOpToParallelLoops<GenericOp>(PatternRewriter &rewriter,
                                                  Operation *op);
-
-static PassRegistration<LowerLinalgToLoopsPass<loop::ForOp, StdIndexedValue>>
-    structuredLoopsPass(
-        "convert-linalg-to-loops",
-        "Lower the operations from the linalg dialect into loops");
-
-static PassRegistration<
-    LowerLinalgToLoopsPass<loop::ParallelOp, StdIndexedValue>>
-    parallelLoopsPass(
-        "convert-linalg-to-parallel-loops",
-        "Lower the operations from the linalg dialect into parallel loops");
-
-static PassRegistration<LowerLinalgToLoopsPass<AffineForOp, AffineIndexedValue>>
-    affineLoopsPass(
-        "convert-linalg-to-affine-loops",
-        "Lower the operations from the linalg dialect into affine loops");
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp b/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
index 8a6b5cf8b5daf..6603507a1cdff 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
@@ -231,6 +231,10 @@ static void promoteSubViews(FuncOp f, bool dynamicBuffers) {
 
 namespace {
 struct LinalgPromotionPass : public FunctionPass<LinalgPromotionPass> {
+/// Include the generated pass utilities.
+#define GEN_PASS_LinalgPromotion
+#include "mlir/Dialect/Linalg/Passes.h.inc"
+
   LinalgPromotionPass() = default;
   LinalgPromotionPass(const LinalgPromotionPass &) {}
   LinalgPromotionPass(bool dynamicBuffers) {
@@ -240,11 +244,6 @@ struct LinalgPromotionPass : public FunctionPass<LinalgPromotionPass> {
   void runOnFunction() override {
     promoteSubViews(getFunction(), dynamicBuffers);
   }
-
-  Option<bool> dynamicBuffers{
-      *this, "test-promote-dynamic",
-      llvm::cl::desc("Test generation of dynamic promoted buffers"),
-      llvm::cl::init(false)};
 };
 } // namespace
 
@@ -252,6 +251,6 @@ std::unique_ptr<OpPassBase<FuncOp>>
 mlir::createLinalgPromotionPass(bool dynamicBuffers) {
   return std::make_unique<LinalgPromotionPass>(dynamicBuffers);
 }
-
-static PassRegistration<LinalgPromotionPass>
-    pass("linalg-promote-subviews", "promote subview ops to local buffers");
+std::unique_ptr<OpPassBase<FuncOp>> mlir::createLinalgPromotionPass() {
+  return std::make_unique<LinalgPromotionPass>();
+}
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
index 2d9ca16c63b69..919a7d53f479b 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
@@ -336,9 +336,12 @@ Optional<TiledLinalgOp> static tileLinalgOpImpl(OpBuilder &b, LinalgOp op,
          "expected matching number of tile sizes and loops");
 
   if (auto convOp = dyn_cast<linalg::ConvOp>(op.getOperation())) {
-    // TODO(ntv): add a level of indirection to linalg.generic.
-    if (convOp.padding())
-      llvm_unreachable("Unexpected conv with padding");
+    // For conv op only support tiling along batch dimension (which is the first
+    // loop).
+    if (convOp.padding() &&
+        !llvm::all_of(tileSizes.drop_front(),
+                      [](Value val) { return isZero(val); }))
+      return llvm::None;
   }
 
   // If permutation is empty, use the identity. Build the permutation map
@@ -420,12 +423,6 @@ tileLinalgOpImpl(OpBuilder &b, LinalgOp op, ArrayRef<int64_t> tileSizes,
   if (tileSizes.empty())
     return llvm::None;
 
-  if (auto convOp = dyn_cast<linalg::ConvOp>(op.getOperation())) {
-    // TODO(ntv): add a level of indirection to linalg.generic.
-    if (convOp.padding())
-      llvm_unreachable("Unexpected conv with padding");
-  }
-
   // The following uses the convention that "tiling by zero" skips tiling a
   // particular dimension. This convention is significantly simpler to handle
   // instead of adjusting affine maps to account for missing dimensions.
@@ -436,6 +433,14 @@ tileLinalgOpImpl(OpBuilder &b, LinalgOp op, ArrayRef<int64_t> tileSizes,
   if (llvm::all_of(tileSizes, [](int64_t v) { return v == 0; }))
     return llvm::None;
 
+  if (auto convOp = dyn_cast<linalg::ConvOp>(op.getOperation())) {
+    // For conv op only support tiling along batch dimension (which is the first
+    // loop).
+    if (convOp.padding() && !llvm::all_of(tileSizes.drop_front(),
+                                          [](int64_t val) { return val == 0; }))
+      return llvm::None;
+  }
+
   // Create a builder for tile size constants.
   OpBuilder::InsertionGuard g(b);
   b.setInsertionPoint(op);
@@ -502,41 +507,47 @@ static void tileLinalgOps(FuncOp f, ArrayRef<int64_t> tileSizes) {
 }
 
 namespace {
+struct LinalgTilingPass : public FunctionPass<LinalgTilingPass> {
+/// Include the generated pass utilities.
+#define GEN_PASS_LinalgTiling
+#include "mlir/Dialect/Linalg/Passes.h.inc"
 
-template <typename LoopTy>
-struct LinalgTilingPass : public FunctionPass<LinalgTilingPass<LoopTy>> {
   LinalgTilingPass() = default;
   LinalgTilingPass(const LinalgTilingPass &) {}
   LinalgTilingPass(ArrayRef<int64_t> sizes) {
-    this->tileSizes->assign(sizes.begin(), sizes.end());
+    tileSizes->assign(sizes.begin(), sizes.end());
   }
 
   void runOnFunction() override {
-    tileLinalgOps<LoopTy>(this->getFunction(), tileSizes);
+    tileLinalgOps<loop::ForOp>(getFunction(), tileSizes);
   }
+};
 
-  Pass::ListOption<int64_t> tileSizes{
-      *this, "linalg-tile-sizes",
-      llvm::cl::desc("Tile sizes by which to tile linalg operations"),
-      llvm::cl::ZeroOrMore, llvm::cl::MiscFlags::CommaSeparated};
+struct LinalgTilingToParallelLoopsPass
+    : public FunctionPass<LinalgTilingToParallelLoopsPass> {
+/// Include the generated pass utilities.
+#define GEN_PASS_LinalgTilingToParallelLoops
+#include "mlir/Dialect/Linalg/Passes.h.inc"
+
+  LinalgTilingToParallelLoopsPass() = default;
+  LinalgTilingToParallelLoopsPass(const LinalgTilingToParallelLoopsPass &) {}
+  LinalgTilingToParallelLoopsPass(ArrayRef<int64_t> sizes) {
+    tileSizes->assign(sizes.begin(), sizes.end());
+  }
+
+  void runOnFunction() override {
+    tileLinalgOps<loop::ParallelOp>(getFunction(), tileSizes);
+  }
 };
 
 } // namespace
 
 std::unique_ptr<OpPassBase<FuncOp>>
 mlir::createLinalgTilingPass(ArrayRef<int64_t> tileSizes) {
-  return std::make_unique<LinalgTilingPass<loop::ForOp>>(tileSizes);
+  return std::make_unique<LinalgTilingPass>(tileSizes);
 }
 
 std::unique_ptr<OpPassBase<FuncOp>>
 mlir::createLinalgTilingToParallelLoopsPass(ArrayRef<int64_t> tileSizes) {
-  return std::make_unique<LinalgTilingPass<loop::ParallelOp>>(tileSizes);
+  return std::make_unique<LinalgTilingToParallelLoopsPass>(tileSizes);
 }
-
-static PassRegistration<LinalgTilingPass<loop::ForOp>>
-    tiling_pass("linalg-tile", "Tile operations in the linalg dialect");
-
-static PassRegistration<LinalgTilingPass<loop::ParallelOp>>
-    tiling_to_parallel_loops(
-        "linalg-tile-to-parallel-loops",
-        "Tile operations in the linalg dialect to parallel loops");
diff --git a/mlir/lib/Dialect/LoopOps/LoopOps.cpp b/mlir/lib/Dialect/LoopOps/LoopOps.cpp
index 3d7ee3846c22d..c62507fa8aafa 100644
--- a/mlir/lib/Dialect/LoopOps/LoopOps.cpp
+++ b/mlir/lib/Dialect/LoopOps/LoopOps.cpp
@@ -182,7 +182,7 @@ bool ForOp::isDefinedOutsideOfLoop(Value value) {
 
 LogicalResult ForOp::moveOutOfLoop(ArrayRef<Operation *> ops) {
   for (auto op : ops)
-    op->moveBefore(this->getOperation());
+    op->moveBefore(*this);
   return success();
 }
 
@@ -191,8 +191,8 @@ ForOp mlir::loop::getForInductionVarOwner(Value val) {
   if (!ivArg)
     return ForOp();
   assert(ivArg.getOwner() && "unlinked block argument");
-  auto *containingInst = ivArg.getOwner()->getParentOp();
-  return dyn_cast_or_null<ForOp>(containingInst);
+  auto *containingOp = ivArg.getOwner()->getParentOp();
+  return dyn_cast_or_null<ForOp>(containingOp);
 }
 
 //===----------------------------------------------------------------------===//
@@ -459,13 +459,25 @@ static void print(OpAsmPrinter &p, ParallelOp op) {
       op.getAttrs(), /*elidedAttrs=*/ParallelOp::getOperandSegmentSizeAttr());
 }
 
+Region &ParallelOp::getLoopBody() { return region(); }
+
+bool ParallelOp::isDefinedOutsideOfLoop(Value value) {
+  return !region().isAncestor(value.getParentRegion());
+}
+
+LogicalResult ParallelOp::moveOutOfLoop(ArrayRef<Operation *> ops) {
+  for (auto op : ops)
+    op->moveBefore(*this);
+  return success();
+}
+
 ParallelOp mlir::loop::getParallelForInductionVarOwner(Value val) {
   auto ivArg = val.dyn_cast<BlockArgument>();
   if (!ivArg)
     return ParallelOp();
   assert(ivArg.getOwner() && "unlinked block argument");
-  auto *containingInst = ivArg.getOwner()->getParentOp();
-  return dyn_cast<ParallelOp>(containingInst);
+  auto *containingOp = ivArg.getOwner()->getParentOp();
+  return dyn_cast<ParallelOp>(containingOp);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/LoopOps/Transforms/CMakeLists.txt b/mlir/lib/Dialect/LoopOps/Transforms/CMakeLists.txt
index 2ec44b4722987..e3ec12b4b21dc 100644
--- a/mlir/lib/Dialect/LoopOps/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/LoopOps/Transforms/CMakeLists.txt
@@ -5,6 +5,9 @@ add_mlir_dialect_library(MLIRLoopOpsTransforms
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/LoopOps
+
+  DEPENDS
+  MLIRLoopPassIncGen
   )
 target_link_libraries(MLIRLoopOpsTransforms
   PUBLIC
diff --git a/mlir/lib/Dialect/LoopOps/Transforms/ParallelLoopFusion.cpp b/mlir/lib/Dialect/LoopOps/Transforms/ParallelLoopFusion.cpp
index b84cfa5aa78c3..b59f1fc3c8c91 100644
--- a/mlir/lib/Dialect/LoopOps/Transforms/ParallelLoopFusion.cpp
+++ b/mlir/lib/Dialect/LoopOps/Transforms/ParallelLoopFusion.cpp
@@ -160,8 +160,11 @@ void mlir::loop::naivelyFuseParallelOps(Region &region) {
 }
 
 namespace {
-
 struct ParallelLoopFusion : public OperationPass<ParallelLoopFusion> {
+/// Include the generated pass utilities.
+#define GEN_PASS_LoopParallelLoopFusion
+#include "mlir/Dialect/LoopOps/Passes.h.inc"
+
   void runOnOperation() override {
     for (Region &region : getOperation()->getRegions())
       naivelyFuseParallelOps(region);
@@ -173,6 +176,3 @@ struct ParallelLoopFusion : public OperationPass<ParallelLoopFusion> {
 std::unique_ptr<Pass> mlir::createParallelLoopFusionPass() {
   return std::make_unique<ParallelLoopFusion>();
 }
-
-static PassRegistration<ParallelLoopFusion>
-    pass("parallel-loop-fusion", "Fuse adjacent parallel loops.");
diff --git a/mlir/lib/Dialect/LoopOps/Transforms/ParallelLoopSpecialization.cpp b/mlir/lib/Dialect/LoopOps/Transforms/ParallelLoopSpecialization.cpp
index c692c0174f0c6..471d5e53e7d00 100644
--- a/mlir/lib/Dialect/LoopOps/Transforms/ParallelLoopSpecialization.cpp
+++ b/mlir/lib/Dialect/LoopOps/Transforms/ParallelLoopSpecialization.cpp
@@ -61,6 +61,10 @@ static void specializeLoopForUnrolling(ParallelOp op) {
 namespace {
 struct ParallelLoopSpecialization
     : public FunctionPass<ParallelLoopSpecialization> {
+/// Include the generated pass utilities.
+#define GEN_PASS_LoopParallelLoopSpecialization
+#include "mlir/Dialect/LoopOps/Passes.h.inc"
+
   void runOnFunction() override {
     getFunction().walk([](ParallelOp op) { specializeLoopForUnrolling(op); });
   }
@@ -70,7 +74,3 @@ struct ParallelLoopSpecialization
 std::unique_ptr<Pass> mlir::createParallelLoopSpecializationPass() {
   return std::make_unique<ParallelLoopSpecialization>();
 }
-
-static PassRegistration<ParallelLoopSpecialization>
-    pass("parallel-loop-specialization",
-         "Specialize parallel loops for vectorization.");
diff --git a/mlir/lib/Dialect/LoopOps/Transforms/ParallelLoopTiling.cpp b/mlir/lib/Dialect/LoopOps/Transforms/ParallelLoopTiling.cpp
index 85fd241cee7e3..8b63d00902911 100644
--- a/mlir/lib/Dialect/LoopOps/Transforms/ParallelLoopTiling.cpp
+++ b/mlir/lib/Dialect/LoopOps/Transforms/ParallelLoopTiling.cpp
@@ -102,8 +102,12 @@ static bool getInnermostNestedLoops(Block *block,
 
 namespace {
 struct ParallelLoopTiling : public FunctionPass<ParallelLoopTiling> {
+/// Include the generated pass utilities.
+#define GEN_PASS_LoopParallelLoopTiling
+#include "mlir/Dialect/LoopOps/Passes.h.inc"
+
   ParallelLoopTiling() = default;
-  ParallelLoopTiling(const ParallelLoopTiling &) {} // tileSize is non-copyable.
+  ParallelLoopTiling(const ParallelLoopTiling &) {}
   explicit ParallelLoopTiling(ArrayRef<int64_t> tileSizes) {
     this->tileSizes = tileSizes;
   }
@@ -117,11 +121,6 @@ struct ParallelLoopTiling : public FunctionPass<ParallelLoopTiling> {
       tileParallelLoop(pLoop, tileSizes);
     }
   }
-
-  ListOption<int64_t> tileSizes{
-      *this, "parallel-loop-tile-sizes",
-      llvm::cl::desc("factors to tile parallel loops by"), llvm::cl::ZeroOrMore,
-      llvm::cl::MiscFlags::CommaSeparated};
 };
 } // namespace
 
@@ -129,6 +128,3 @@ std::unique_ptr<Pass>
 mlir::createParallelLoopTilingPass(ArrayRef<int64_t> tileSizes) {
   return std::make_unique<ParallelLoopTiling>(tileSizes);
 }
-
-static PassRegistration<ParallelLoopTiling> pass("parallel-loop-tiling",
-                                                 "Tile parallel loops.");
diff --git a/mlir/lib/Dialect/Quant/CMakeLists.txt b/mlir/lib/Dialect/Quant/CMakeLists.txt
index aa014ab50666c..4f83829622065 100644
--- a/mlir/lib/Dialect/Quant/CMakeLists.txt
+++ b/mlir/lib/Dialect/Quant/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_mlir_dialect_library(MLIRQuantOps
+add_mlir_dialect_library(MLIRQuant
   IR/QuantOps.cpp
   IR/QuantTypes.cpp
   IR/TypeDetail.h
@@ -14,8 +14,9 @@ add_mlir_dialect_library(MLIRQuantOps
 
   DEPENDS
   MLIRQuantOpsIncGen
+  MLIRQuantPassIncGen
   )
-target_link_libraries(MLIRQuantOps
+target_link_libraries(MLIRQuant
   PUBLIC
   MLIRIR
   MLIRPass
diff --git a/mlir/lib/Dialect/Quant/Transforms/ConvertConst.cpp b/mlir/lib/Dialect/Quant/Transforms/ConvertConst.cpp
index 2598e8cf50138..3017346a9acc4 100644
--- a/mlir/lib/Dialect/Quant/Transforms/ConvertConst.cpp
+++ b/mlir/lib/Dialect/Quant/Transforms/ConvertConst.cpp
@@ -21,9 +21,11 @@ using namespace mlir;
 using namespace mlir::quant;
 
 namespace {
+struct ConvertConstPass : public FunctionPass<ConvertConstPass> {
+/// Include the generated pass utilities.
+#define GEN_PASS_QuantConvertConst
+#include "mlir/Dialect/Quant/Passes.h.inc"
 
-class ConvertConstPass : public FunctionPass<ConvertConstPass> {
-public:
   void runOnFunction() override;
 };
 
@@ -106,7 +108,3 @@ void ConvertConstPass::runOnFunction() {
 std::unique_ptr<OpPassBase<FuncOp>> mlir::quant::createConvertConstPass() {
   return std::make_unique<ConvertConstPass>();
 }
-
-static PassRegistration<ConvertConstPass>
-    pass("quant-convert-const",
-         "Converts constants followed by qbarrier to actual quantized values");
diff --git a/mlir/lib/Dialect/Quant/Transforms/ConvertSimQuant.cpp b/mlir/lib/Dialect/Quant/Transforms/ConvertSimQuant.cpp
index c921aeafda904..b76cee6a412c9 100644
--- a/mlir/lib/Dialect/Quant/Transforms/ConvertSimQuant.cpp
+++ b/mlir/lib/Dialect/Quant/Transforms/ConvertSimQuant.cpp
@@ -19,10 +19,12 @@ using namespace mlir;
 using namespace mlir::quant;
 
 namespace {
-
-class ConvertSimulatedQuantPass
+struct ConvertSimulatedQuantPass
     : public FunctionPass<ConvertSimulatedQuantPass> {
-public:
+/// Include the generated pass utilities.
+#define GEN_PASS_QuantConvertSimulatedQuant
+#include "mlir/Dialect/Quant/Passes.h.inc"
+
   void runOnFunction() override;
 };
 
@@ -142,8 +144,3 @@ std::unique_ptr<OpPassBase<FuncOp>>
 mlir::quant::createConvertSimulatedQuantPass() {
   return std::make_unique<ConvertSimulatedQuantPass>();
 }
-
-static PassRegistration<ConvertSimulatedQuantPass>
-    pass("quant-convert-simulated-quantization",
-         "Converts training-time simulated quantization ops to corresponding "
-         "quantize/dequantize casts.");
diff --git a/mlir/lib/Dialect/SPIRV/CMakeLists.txt b/mlir/lib/Dialect/SPIRV/CMakeLists.txt
index afb7497b9e7c7..e6834ca2ef3e4 100644
--- a/mlir/lib/Dialect/SPIRV/CMakeLists.txt
+++ b/mlir/lib/Dialect/SPIRV/CMakeLists.txt
@@ -1,3 +1,7 @@
+if (MSVC)
+  set_source_files_properties(SPIRVDialect.cpp PROPERTIES COMPILE_FLAGS /bigobj)
+endif()
+
 set(LLVM_TARGET_DEFINITIONS SPIRVCanonicalization.td)
 mlir_tablegen(SPIRVCanonicalization.inc -gen-rewriters)
 add_public_tablegen_target(MLIRSPIRVCanonicalizationIncGen)
diff --git a/mlir/lib/Dialect/SPIRV/SPIRVDialect.cpp b/mlir/lib/Dialect/SPIRV/SPIRVDialect.cpp
index 8ed417cad58d8..92ca31de6213d 100644
--- a/mlir/lib/Dialect/SPIRV/SPIRVDialect.cpp
+++ b/mlir/lib/Dialect/SPIRV/SPIRVDialect.cpp
@@ -751,7 +751,7 @@ Attribute SPIRVDialect::parseAttribute(DialectAsmParser &parser,
   if (attrKind == spirv::VerCapExtAttr::getKindName())
     return parseVerCapExtAttr(parser);
 
-  parser.emitError(parser.getNameLoc(), "unknown SPIR-V attriubte kind: ")
+  parser.emitError(parser.getNameLoc(), "unknown SPIR-V attribute kind: ")
       << attrKind;
   return {};
 }
diff --git a/mlir/lib/Dialect/SPIRV/SPIRVLowering.cpp b/mlir/lib/Dialect/SPIRV/SPIRVLowering.cpp
index 3fd987b0e5657..520a1f36f18ce 100644
--- a/mlir/lib/Dialect/SPIRV/SPIRVLowering.cpp
+++ b/mlir/lib/Dialect/SPIRV/SPIRVLowering.cpp
@@ -18,6 +18,7 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/Debug.h"
 
+#include <cstdint>
 #include <functional>
 
 #define DEBUG_TYPE "mlir-spirv-lowering"
@@ -99,9 +100,9 @@ Type SPIRVTypeConverter::getIndexType(MLIRContext *context) {
 
 /// Mapping between SPIR-V storage classes to memref memory spaces.
 ///
-/// Note: memref does not have a defined smenatics for each memory space; it
+/// Note: memref does not have a defined semantics for each memory space; it
 /// depends on the context where it is used. There are no particular reasons
-/// behind the number assigments; we try to follow NVVM conventions and largely
+/// behind the number assignments; we try to follow NVVM conventions and largely
 /// give common storage classes a smaller number. The hope is use symbolic
 /// memory space representation eventually after memref supports it.
 // TODO(antiagainst): swap Generic and StorageBuffer assignment to be more akin
@@ -136,6 +137,8 @@ SPIRVTypeConverter::getMemorySpaceForStorageClass(spirv::StorageClass storage) {
 
   switch (storage) { STORAGE_SPACE_MAP_LIST(STORAGE_SPACE_MAP_FN) }
 #undef STORAGE_SPACE_MAP_FN
+  llvm_unreachable("unhandled storage class!");
+  return UINT32_MAX;
 }
 
 Optional<spirv::StorageClass>
@@ -236,7 +239,7 @@ convertScalarType(const spirv::TargetEnv &targetEnv, spirv::ScalarType type,
   // bitwidth given this is a scalar type.
   // TODO(antiagainst): We are unconditionally converting the bitwidth here,
   // this might be okay for non-interface types (i.e., types used in
-  // Priviate/Function storage classes), but not for interface types (i.e.,
+  // Private/Function storage classes), but not for interface types (i.e.,
   // types used in StorageBuffer/Uniform/PushConstant/etc. storage classes).
   // This is because the later actually affects the ABI contract with the
   // runtime. So we may want to expose a control on SPIRVTypeConverter to fail
diff --git a/mlir/lib/Dialect/SPIRV/SPIRVOps.cpp b/mlir/lib/Dialect/SPIRV/SPIRVOps.cpp
index f6b862156c49e..841b5e0ace647 100644
--- a/mlir/lib/Dialect/SPIRV/SPIRVOps.cpp
+++ b/mlir/lib/Dialect/SPIRV/SPIRVOps.cpp
@@ -2076,7 +2076,7 @@ void spirv::LoopOp::addEntryAndMergeBlock() {
   body().push_back(new Block());
   auto *mergeBlock = new Block();
   body().push_back(mergeBlock);
-  OpBuilder builder(mergeBlock);
+  OpBuilder builder = OpBuilder::atBlockEnd(mergeBlock);
 
   // Add a spv._merge op into the merge block.
   builder.create<spirv::MergeOp>(getLoc());
@@ -2373,7 +2373,7 @@ void spirv::SelectionOp::addMergeBlock() {
   assert(body().empty() && "entry and merge block already exist");
   auto *mergeBlock = new Block();
   body().push_back(mergeBlock);
-  OpBuilder builder(mergeBlock);
+  OpBuilder builder = OpBuilder::atBlockEnd(mergeBlock);
 
   // Add a spv._merge op into the merge block.
   builder.create<spirv::MergeOp>(getLoc());
diff --git a/mlir/lib/Dialect/SPIRV/Serialization/Deserializer.cpp b/mlir/lib/Dialect/SPIRV/Serialization/Deserializer.cpp
index fbba3595db8fc..82be16847594b 100644
--- a/mlir/lib/Dialect/SPIRV/Serialization/Deserializer.cpp
+++ b/mlir/lib/Dialect/SPIRV/Serialization/Deserializer.cpp
@@ -553,11 +553,11 @@ LogicalResult Deserializer::processHeader() {
       MIN_VERSION_CASE(5);
 #undef MIN_VERSION_CASE
     default:
-      return emitError(unknownLoc, "unspported SPIR-V minor version: ")
+      return emitError(unknownLoc, "unsupported SPIR-V minor version: ")
              << minorVersion;
     }
   } else {
-    return emitError(unknownLoc, "unspported SPIR-V major version: ")
+    return emitError(unknownLoc, "unsupported SPIR-V major version: ")
            << majorVersion;
   }
 
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/CMakeLists.txt b/mlir/lib/Dialect/SPIRV/Transforms/CMakeLists.txt
index f31c4836e7f7e..e388069d52eea 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/SPIRV/Transforms/CMakeLists.txt
@@ -5,6 +5,9 @@ add_mlir_dialect_library(MLIRSPIRVTransforms
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/SPIRV
+
+  DEPENDS
+  MLIRSPIRVPassIncGen
   )
 
 target_link_libraries(MLIRSPIRVTransforms
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/DecorateSPIRVCompositeTypeLayoutPass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/DecorateSPIRVCompositeTypeLayoutPass.cpp
index 0645408398b6a..79ed81956f084 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/DecorateSPIRVCompositeTypeLayoutPass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/DecorateSPIRVCompositeTypeLayoutPass.cpp
@@ -117,7 +117,3 @@ std::unique_ptr<OpPassBase<ModuleOp>>
 mlir::spirv::createDecorateSPIRVCompositeTypeLayoutPass() {
   return std::make_unique<DecorateSPIRVCompositeTypeLayoutPass>();
 }
-
-static PassRegistration<DecorateSPIRVCompositeTypeLayoutPass>
-    pass("decorate-spirv-composite-type-layout",
-         "Decorate SPIR-V composite type with layout info");
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
index 1ca9cad977af0..9cb2bfe1e1fc5 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
@@ -264,6 +264,3 @@ std::unique_ptr<OpPassBase<spirv::ModuleOp>>
 mlir::spirv::createLowerABIAttributesPass() {
   return std::make_unique<LowerABIAttributesPass>();
 }
-
-static PassRegistration<LowerABIAttributesPass>
-    pass("spirv-lower-abi-attrs", "Lower SPIR-V ABI Attributes");
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/UpdateVCEPass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/UpdateVCEPass.cpp
index 201adbbd38374..ebb5b6eda83ac 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/UpdateVCEPass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/UpdateVCEPass.cpp
@@ -177,8 +177,3 @@ std::unique_ptr<OpPassBase<spirv::ModuleOp>>
 mlir::spirv::createUpdateVersionCapabilityExtensionPass() {
   return std::make_unique<UpdateVCEPass>();
 }
-
-static PassRegistration<UpdateVCEPass>
-    pass("spirv-update-vce",
-         "Deduce and attach minimal (version, capabilities, extensions) "
-         "requirements to spv.module ops");
diff --git a/mlir/lib/Dialect/Shape/CMakeLists.txt b/mlir/lib/Dialect/Shape/CMakeLists.txt
index 485a3c710abb5..982f73385076e 100644
--- a/mlir/lib/Dialect/Shape/CMakeLists.txt
+++ b/mlir/lib/Dialect/Shape/CMakeLists.txt
@@ -9,6 +9,7 @@ add_mlir_dialect_library(MLIRShape
   )
 target_link_libraries(MLIRShape
   PUBLIC
+  MLIRInferTypeOpInterface
   MLIRIR
   MLIRSideEffects
   LLVMSupport
diff --git a/mlir/lib/Dialect/Shape/IR/Shape.cpp b/mlir/lib/Dialect/Shape/IR/Shape.cpp
index f7f69a64826b1..85798ce9eff44 100644
--- a/mlir/lib/Dialect/Shape/IR/Shape.cpp
+++ b/mlir/lib/Dialect/Shape/IR/Shape.cpp
@@ -106,6 +106,33 @@ static ParseResult parseConstantOp(OpAsmParser &parser,
 
 static LogicalResult verify(ConstantOp &op) { return success(); }
 
+//===----------------------------------------------------------------------===//
+// SplitAtOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult SplitAtOp::inferReturnTypes(
+    MLIRContext *context, Optional<Location> location, ValueRange operands,
+    ArrayRef<NamedAttribute> attributes, RegionRange regions,
+    SmallVectorImpl<Type> &inferredReturnTypes) {
+  auto shapeType = ShapeType::get(context);
+  inferredReturnTypes.push_back(shapeType);
+  inferredReturnTypes.push_back(shapeType);
+  return success();
+}
+
+//===----------------------------------------------------------------------===//
+// ConcatOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult ConcatOp::inferReturnTypes(
+    MLIRContext *context, Optional<Location> location, ValueRange operands,
+    ArrayRef<NamedAttribute> attributes, RegionRange regions,
+    SmallVectorImpl<Type> &inferredReturnTypes) {
+  auto shapeType = ShapeType::get(context);
+  inferredReturnTypes.push_back(shapeType);
+  return success();
+}
+
 namespace mlir {
 namespace shape {
 
diff --git a/mlir/lib/Dialect/Traits.cpp b/mlir/lib/Dialect/Traits.cpp
index 352e26fd8080d..dc721adc7472f 100644
--- a/mlir/lib/Dialect/Traits.cpp
+++ b/mlir/lib/Dialect/Traits.cpp
@@ -202,7 +202,7 @@ LogicalResult OpTrait::impl::verifyCompatibleOperandBroadcast(Operation *op) {
   auto rankedResults = make_filter_range(
       op->getResultTypes(), [](Type t) { return t.isa<RankedTensorType>(); });
 
-  // If all of the results are unranked then no further verfication.
+  // If all of the results are unranked then no further verification.
   if (rankedResults.empty())
     return success();
 
diff --git a/mlir/lib/Dialect/Vector/EDSC/Builders.cpp b/mlir/lib/Dialect/Vector/EDSC/Builders.cpp
index b1f94655ab28b..d2436ef8ae62c 100644
--- a/mlir/lib/Dialect/Vector/EDSC/Builders.cpp
+++ b/mlir/lib/Dialect/Vector/EDSC/Builders.cpp
@@ -30,7 +30,7 @@ Value mlir::edsc::ops::vector_contraction(
       ArrayRef<StringRef>{functional::map(toString, iteratorTypes)});
 }
 
-Value mlir::edsc::ops::vector_matmul(Value A, Value B, Value C) {
+Value mlir::edsc::ops::vector_contraction_matmul(Value A, Value B, Value C) {
   AffineExpr m, n, k;
   bindDims(ScopedContext::getContext(), m, n, k);
   return vector_contraction(StructuredIndexed(A, {m, k}),
diff --git a/mlir/lib/Dialect/Vector/VectorOps.cpp b/mlir/lib/Dialect/Vector/VectorOps.cpp
index 816aaf9f59482..19aa52f8a4617 100644
--- a/mlir/lib/Dialect/Vector/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/VectorOps.cpp
@@ -1524,6 +1524,23 @@ static LogicalResult verify(TupleOp op) { return success(); }
 // TransposeOp
 //===----------------------------------------------------------------------===//
 
+// Eliminates transpose operations, which produce values identical to their
+// input values. This happens when the dimensions of the input vector remain in
+// their original order after the transpose operation.
+OpFoldResult TransposeOp::fold(ArrayRef<Attribute> operands) {
+  SmallVector<int64_t, 4> transp;
+  getTransp(transp);
+
+  // Check if the permutation of the dimensions contains sequential values:
+  // {0, 1, 2, ...}.
+  for (int64_t i = 0, e = transp.size(); i < e; i++) {
+    if (transp[i] != i)
+      return {};
+  }
+
+  return vector();
+}
+
 static LogicalResult verify(TransposeOp op) {
   VectorType vectorType = op.getVectorType();
   VectorType resultType = op.getResultType();
@@ -1549,6 +1566,59 @@ static LogicalResult verify(TransposeOp op) {
   return success();
 }
 
+namespace {
+
+// Rewrites two back-to-back TransposeOp operations into a single TransposeOp.
+class TransposeFolder final : public OpRewritePattern<TransposeOp> {
+public:
+  using OpRewritePattern<TransposeOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(TransposeOp transposeOp,
+                                PatternRewriter &rewriter) const override {
+    // Wrapper around TransposeOp::getTransp() for cleaner code.
+    auto getPermutation = [](TransposeOp transpose) {
+      SmallVector<int64_t, 4> permutation;
+      transpose.getTransp(permutation);
+      return permutation;
+    };
+
+    // Composes two permutations: result[i] = permutation1[permutation2[i]].
+    auto composePermutations = [](ArrayRef<int64_t> permutation1,
+                                  ArrayRef<int64_t> permutation2) {
+      SmallVector<int64_t, 4> result;
+      for (auto index : permutation2)
+        result.push_back(permutation1[index]);
+      return result;
+    };
+
+    // Return if the input of 'transposeOp' is not defined by another transpose.
+    TransposeOp parentTransposeOp =
+        dyn_cast_or_null<TransposeOp>(transposeOp.vector().getDefiningOp());
+    if (!parentTransposeOp)
+      return failure();
+
+    SmallVector<int64_t, 4> permutation = composePermutations(
+        getPermutation(parentTransposeOp), getPermutation(transposeOp));
+    // Replace 'transposeOp' with a new transpose operation.
+    rewriter.replaceOpWithNewOp<TransposeOp>(
+        transposeOp, transposeOp.getResult().getType(),
+        parentTransposeOp.vector(),
+        vector::getVectorSubscriptAttr(rewriter, permutation));
+    return success();
+  }
+};
+
+} // end anonymous namespace
+
+void TransposeOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
+                                              MLIRContext *context) {
+  results.insert<TransposeFolder>(context);
+}
+
+void TransposeOp::getTransp(SmallVectorImpl<int64_t> &results) {
+  populateFromInt64AttrArray(transp(), results);
+}
+
 //===----------------------------------------------------------------------===//
 // TupleGetOp
 //===----------------------------------------------------------------------===//
@@ -1683,7 +1753,8 @@ void CreateMaskOp::getCanonicalizationPatterns(
 
 void mlir::vector::populateVectorToVectorCanonicalizationPatterns(
     OwningRewritePatternList &patterns, MLIRContext *context) {
-  patterns.insert<CreateMaskFolder, StridedSliceConstantMaskFolder>(context);
+  patterns.insert<CreateMaskFolder, StridedSliceConstantMaskFolder,
+                  TransposeFolder>(context);
 }
 
 namespace mlir {
diff --git a/mlir/lib/Dialect/Vector/VectorTransforms.cpp b/mlir/lib/Dialect/Vector/VectorTransforms.cpp
index ef3484d31a3cc..dbb0bf4437006 100644
--- a/mlir/lib/Dialect/Vector/VectorTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/VectorTransforms.cpp
@@ -69,15 +69,6 @@ static int64_t computeMaxLinearIndex(ArrayRef<int64_t> basis) {
   return res;
 }
 
-/// Computes and returns the linearized index of 'offsets' w.r.t. 'basis'.
-static int64_t linearize(ArrayRef<int64_t> offsets, ArrayRef<int64_t> basis) {
-  assert(offsets.size() == basis.size());
-  int64_t linearIndex = 0;
-  for (unsigned idx = 0, e = basis.size(); idx < e; ++idx)
-    linearIndex += offsets[idx] * basis[idx];
-  return linearIndex;
-}
-
 // Clones `op` into a new operations that takes `operands` and returns
 // `resultTypes`.
 static Operation *cloneOpWithOperandsAndTypes(PatternRewriter &builder,
@@ -270,7 +261,7 @@ struct VectorState {
 //                           insertslice
 //                                |
 
-// TODO(andydavis) Add the following canonicalization/simplifcation patterns:
+// TODO(andydavis) Add the following canonicalization/simplification patterns:
 // *) Add pattern which matches InsertStridedSlice -> StridedSlice and forwards
 //    InsertStridedSlice operand to StridedSlice.
 // *) Add pattern which matches SourceOp -> StridedSlice -> UserOp which checks
@@ -683,6 +674,99 @@ struct ShapeCastOpDecomposer : public OpRewritePattern<vector::ShapeCastOp> {
   }
 };
 
+/// Returns the producer Value of the same type as 'consumerValue', by tracking
+/// the tuple index and offsets of the consumer vector value through the
+/// chain of operations (TupleGetOp, InsertSlicesOp, ExtractSlicesOp, TupleOp)
+/// from consumer to producer. Each operation in the chain is structured, and
+/// so the tuple index and offsets can be mapped from result to input, while
+/// visiting each operation in the chain.
+/// Returns nullptr on failure.
+static Value getProducerValue(Value consumerValue) {
+  auto consumerVectorType = consumerValue.getType().cast<VectorType>();
+  // A tupleIndex == -1 indicates that 'offsets' are w.r.t a vector type.
+  int64_t tupleIndex = -1;
+  SmallVector<int64_t, 4> offsets(consumerVectorType.getRank(), 0);
+  auto *op = consumerValue.getDefiningOp();
+  while (op != nullptr) {
+    if (auto tupleGetOp = dyn_cast<vector::TupleGetOp>(op)) {
+      assert(tupleIndex == -1 && "TupleGetOp must have vector result type");
+
+      // Update 'tupleIndex' and next defining 'op' to visit.
+      tupleIndex = tupleGetOp.getIndex();
+      op = tupleGetOp.vectors().getDefiningOp();
+    } else if (auto extractSlicesOp = dyn_cast<vector::ExtractSlicesOp>(op)) {
+      assert(tupleIndex >= 0);
+
+      // Compute slice strides for 'extractSlicesOp'.
+      SmallVector<int64_t, 4> sizes;
+      extractSlicesOp.getSizes(sizes);
+      auto sliceStrides = computeStrides(
+          extractSlicesOp.getSourceVectorType().getShape(), sizes);
+
+      // Compute 'elementOffsets' into 'extractSlicesOp' input vector type,
+      // of 'extractSlicesOp' result vector tuple element at 'tupleIndex'.
+      auto vectorOffsets = delinearize(sliceStrides, tupleIndex);
+      auto elementOffsets =
+          computeElementOffsetsFromVectorSliceOffsets(sizes, vectorOffsets);
+
+      // Add 'elementOffsets' to 'offsets' so that 'offsets' are now relative
+      // to the 'extractSlicesOp' input vector type.
+      assert(offsets.size() == elementOffsets.size());
+      for (unsigned i = 0, e = offsets.size(); i < e; ++i)
+        offsets[i] += elementOffsets[i];
+
+      // Clear 'tupleIndex' and update next defining 'op' to visit.
+      tupleIndex = -1;
+      op = extractSlicesOp.vector().getDefiningOp();
+    } else if (auto insertSlicesOp = dyn_cast<vector::InsertSlicesOp>(op)) {
+      assert(tupleIndex == -1);
+
+      // Compute slice strides for 'insertSlicesOp'.
+      SmallVector<int64_t, 4> sizes;
+      insertSlicesOp.getSizes(sizes);
+      auto sliceStrides = computeStrides(
+          insertSlicesOp.getResultVectorType().getShape(), sizes);
+
+      // Compute 'vectorOffsets' of 'insertSlicesOp' input vector slice,
+      // of 'insertSlicesOp' result vector type at 'offsets'.
+      SmallVector<int64_t, 4> vectorOffsets(offsets.size());
+      assert(offsets.size() == sizes.size());
+      for (unsigned i = 0, e = offsets.size(); i < e; ++i)
+        vectorOffsets[i] = offsets[i] / sizes[i];
+
+      // Compute the source tuple element index.
+      tupleIndex = linearize(vectorOffsets, sliceStrides);
+
+      // Subtract 'elementOffsets' from 'offsets' so that 'offsets' are now
+      // relative to input tuple element vector type at 'tupleIndex'.
+      auto elementOffsets =
+          computeElementOffsetsFromVectorSliceOffsets(sizes, vectorOffsets);
+      assert(offsets.size() == elementOffsets.size());
+      for (unsigned i = 0, e = offsets.size(); i < e; ++i) {
+        offsets[i] -= elementOffsets[i];
+        assert(offsets[i] >= 0);
+      }
+
+      // Update next defining 'op' to visit.
+      op = insertSlicesOp.vectors().getDefiningOp();
+    } else if (auto tupleOp = dyn_cast<vector::TupleOp>(op)) {
+      assert(tupleIndex >= 0);
+
+      // Return tuple element 'value' at 'tupleIndex' if it matches type.
+      auto value = tupleOp.getOperand(tupleIndex);
+      if (value.getType() == consumerVectorType)
+        return value;
+
+      // Update 'tupleIndex' and next defining 'op' to visit.
+      tupleIndex = -1;
+      op = value.getDefiningOp();
+    } else {
+      break;
+    }
+  }
+  return nullptr;
+}
+
 /// ShapeCastOpFolder folds cancelling ShapeCastOps away.
 //
 // Example:
@@ -740,28 +824,11 @@ struct TupleGetFolderOp : public OpRewritePattern<vector::TupleGetOp> {
 
   LogicalResult matchAndRewrite(vector::TupleGetOp tupleGetOp,
                                 PatternRewriter &rewriter) const override {
-    // Return if 'tupleGetOp.vectors' arg was not defined by ExtractSlicesOp.
-    auto extractSlicesOp = dyn_cast_or_null<vector::ExtractSlicesOp>(
-        tupleGetOp.vectors().getDefiningOp());
-    if (!extractSlicesOp)
-      return failure();
-
-    // Return if 'extractSlicesOp.vector' arg was not defined by InsertSlicesOp.
-    auto insertSlicesOp = dyn_cast_or_null<vector::InsertSlicesOp>(
-        extractSlicesOp.vector().getDefiningOp());
-    if (!insertSlicesOp)
-      return failure();
-
-    // Return if 'insertSlicesOp.vectors' arg was not defined by TupleOp.
-    auto tupleOp = dyn_cast_or_null<vector::TupleOp>(
-        insertSlicesOp.vectors().getDefiningOp());
-    if (!tupleOp)
-      return failure();
-
-    // Forward Value from 'tupleOp' at 'tupleGetOp.index'.
-    Value tupleValue = tupleOp.getOperand(tupleGetOp.getIndex());
-    rewriter.replaceOp(tupleGetOp, tupleValue);
-    return success();
+    if (auto producer = getProducerValue(tupleGetOp.getResult())) {
+      rewriter.replaceOp(tupleGetOp, producer);
+      return success();
+    }
+    return failure();
   }
 };
 
diff --git a/mlir/lib/Dialect/Vector/VectorUtils.cpp b/mlir/lib/Dialect/Vector/VectorUtils.cpp
index f929dddd6d8d6..9038b7ad66173 100644
--- a/mlir/lib/Dialect/Vector/VectorUtils.cpp
+++ b/mlir/lib/Dialect/Vector/VectorUtils.cpp
@@ -28,10 +28,10 @@
 
 using llvm::SetVector;
 
-namespace mlir {
+using namespace mlir;
 
-SmallVector<int64_t, 4> computeStrides(ArrayRef<int64_t> shape,
-                                       ArrayRef<int64_t> sizes) {
+SmallVector<int64_t, 4> mlir::computeStrides(ArrayRef<int64_t> shape,
+                                             ArrayRef<int64_t> sizes) {
   int64_t rank = shape.size();
   // Compute the count for each dimension.
   SmallVector<int64_t, 4> sliceDimCounts(rank);
@@ -45,8 +45,16 @@ SmallVector<int64_t, 4> computeStrides(ArrayRef<int64_t> shape,
   return sliceStrides;
 }
 
-SmallVector<int64_t, 4> delinearize(ArrayRef<int64_t> sliceStrides,
-                                    int64_t index) {
+int64_t mlir::linearize(ArrayRef<int64_t> offsets, ArrayRef<int64_t> basis) {
+  assert(offsets.size() == basis.size());
+  int64_t linearIndex = 0;
+  for (unsigned idx = 0, e = basis.size(); idx < e; ++idx)
+    linearIndex += offsets[idx] * basis[idx];
+  return linearIndex;
+}
+
+SmallVector<int64_t, 4> mlir::delinearize(ArrayRef<int64_t> sliceStrides,
+                                          int64_t index) {
   int64_t rank = sliceStrides.size();
   SmallVector<int64_t, 4> vectorOffsets(rank);
   for (int64_t r = 0; r < rank; ++r) {
@@ -57,16 +65,15 @@ SmallVector<int64_t, 4> delinearize(ArrayRef<int64_t> sliceStrides,
   return vectorOffsets;
 }
 
-SmallVector<int64_t, 4>
-computeElementOffsetsFromVectorSliceOffsets(ArrayRef<int64_t> sizes,
-                                            ArrayRef<int64_t> vectorOffsets) {
+SmallVector<int64_t, 4> mlir::computeElementOffsetsFromVectorSliceOffsets(
+    ArrayRef<int64_t> sizes, ArrayRef<int64_t> vectorOffsets) {
   return functional::zipMap([](int64_t v1, int64_t v2) { return v1 * v2; },
                             vectorOffsets, sizes);
 }
 
-SmallVector<int64_t, 4> computeSliceSizes(ArrayRef<int64_t> shape,
-                                          ArrayRef<int64_t> sizes,
-                                          ArrayRef<int64_t> elementOffsets) {
+SmallVector<int64_t, 4>
+mlir::computeSliceSizes(ArrayRef<int64_t> shape, ArrayRef<int64_t> sizes,
+                        ArrayRef<int64_t> elementOffsets) {
   int64_t rank = shape.size();
   SmallVector<int64_t, 4> sliceSizes(rank);
   for (unsigned r = 0; r < rank; ++r)
@@ -74,8 +81,8 @@ SmallVector<int64_t, 4> computeSliceSizes(ArrayRef<int64_t> shape,
   return sliceSizes;
 }
 
-Optional<SmallVector<int64_t, 4>> shapeRatio(ArrayRef<int64_t> superShape,
-                                             ArrayRef<int64_t> subShape) {
+Optional<SmallVector<int64_t, 4>> mlir::shapeRatio(ArrayRef<int64_t> superShape,
+                                                   ArrayRef<int64_t> subShape) {
   if (superShape.size() < subShape.size()) {
     return Optional<SmallVector<int64_t, 4>>();
   }
@@ -114,8 +121,8 @@ Optional<SmallVector<int64_t, 4>> shapeRatio(ArrayRef<int64_t> superShape,
   return SmallVector<int64_t, 4>{result.rbegin(), result.rend()};
 }
 
-Optional<SmallVector<int64_t, 4>> shapeRatio(VectorType superVectorType,
-                                             VectorType subVectorType) {
+Optional<SmallVector<int64_t, 4>> mlir::shapeRatio(VectorType superVectorType,
+                                                   VectorType subVectorType) {
   assert(superVectorType.getElementType() == subVectorType.getElementType() &&
          "vector types must be of the same elemental type");
   return shapeRatio(superVectorType.getShape(), subVectorType.getShape());
@@ -201,9 +208,9 @@ static SetVector<Operation *> getEnclosingforOps(Operation *op) {
   return getParentsOfType<AffineForOp>(op);
 }
 
-AffineMap
-makePermutationMap(Operation *op, ArrayRef<Value> indices,
-                   const DenseMap<Operation *, unsigned> &loopToVectorDim) {
+AffineMap mlir::makePermutationMap(
+    Operation *op, ArrayRef<Value> indices,
+    const DenseMap<Operation *, unsigned> &loopToVectorDim) {
   DenseMap<Operation *, unsigned> enclosingLoopToVectorDim;
   auto enclosingLoops = getEnclosingforOps(op);
   for (auto *forInst : enclosingLoops) {
@@ -212,7 +219,7 @@ makePermutationMap(Operation *op, ArrayRef<Value> indices,
       enclosingLoopToVectorDim.insert(*it);
     }
   }
-  return makePermutationMap(indices, enclosingLoopToVectorDim);
+  return ::makePermutationMap(indices, enclosingLoopToVectorDim);
 }
 
 bool matcher::operatesOnSuperVectorsOf(Operation &op,
@@ -275,4 +282,3 @@ bool matcher::operatesOnSuperVectorsOf(Operation &op,
   return true;
 }
 
-} // namespace mlir
diff --git a/mlir/lib/Dialect/VectorOps/VectorTransforms.cpp b/mlir/lib/Dialect/VectorOps/VectorTransforms.cpp
deleted file mode 100644
index e853c76d0dba3..0000000000000
--- a/mlir/lib/Dialect/VectorOps/VectorTransforms.cpp
+++ /dev/null
@@ -1,1349 +0,0 @@
-//===- VectorToLoops.cpp - Conversion within the Vector dialect -----------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements target-independent rewrites as 1->N patterns.
-//
-//===----------------------------------------------------------------------===//
-
-#include <type_traits>
-
-#include "mlir/Dialect/Affine/IR/AffineOps.h"
-#include "mlir/Dialect/StandardOps/IR/Ops.h"
-#include "mlir/Dialect/Utils/StructuredOpsUtils.h"
-#include "mlir/Dialect/VectorOps/VectorOps.h"
-#include "mlir/Dialect/VectorOps/VectorTransforms.h"
-#include "mlir/Dialect/VectorOps/VectorUtils.h"
-#include "mlir/IR/AffineExpr.h"
-#include "mlir/IR/AffineMap.h"
-#include "mlir/IR/Attributes.h"
-#include "mlir/IR/Builders.h"
-#include "mlir/IR/Function.h"
-#include "mlir/IR/Location.h"
-#include "mlir/IR/Matchers.h"
-#include "mlir/IR/Module.h"
-#include "mlir/IR/OperationSupport.h"
-#include "mlir/IR/PatternMatch.h"
-#include "mlir/IR/Types.h"
-#include "mlir/Support/Functional.h"
-#include "mlir/Support/STLExtras.h"
-
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/raw_ostream.h"
-
-#define DEBUG_TYPE "vector-to-vector"
-
-using namespace mlir;
-using llvm::dbgs;
-using mlir::functional::zipMap;
-
-static llvm::cl::OptionCategory clOptionsCategory(DEBUG_TYPE " options");
-
-static llvm::cl::opt<bool> lowerToLLVMMatrixIntrinsics(
-    "vector-lower-matrix-intrinsics",
-    llvm::cl::desc("Lower vector.contract to llvm.intr.matrix.multiply"),
-    llvm::cl::init(false), llvm::cl::cat(clOptionsCategory));
-
-/// Given a shape with sizes greater than 0 along all dimensions,
-/// returns the distance, in number of elements, between a slice in a dimension
-/// and the next slice in the same dimension.
-///   e.g. shape[3, 4, 5] -> linearization_basis[20, 5, 1]
-static SmallVector<int64_t, 8> computeStrides(ArrayRef<int64_t> shape) {
-  if (shape.empty())
-    return {};
-  SmallVector<int64_t, 8> tmp;
-  tmp.reserve(shape.size());
-  int64_t running = 1;
-  for (auto size : llvm::reverse(shape)) {
-    assert(size > 0 && "size must be nonnegative");
-    tmp.push_back(running);
-    running *= size;
-  }
-  return SmallVector<int64_t, 8>(tmp.rbegin(), tmp.rend());
-}
-
-static int64_t computeMaxLinearIndex(ArrayRef<int64_t> basis) {
-  if (basis.empty())
-    return 0;
-  int64_t res = 1;
-  for (auto b : basis)
-    res *= b;
-  return res;
-}
-
-/// Computes and returns the linearized index of 'offsets' w.r.t. 'basis'.
-static int64_t linearize(ArrayRef<int64_t> offsets, ArrayRef<int64_t> basis) {
-  assert(offsets.size() == basis.size());
-  int64_t linearIndex = 0;
-  for (unsigned idx = 0, e = basis.size(); idx < e; ++idx)
-    linearIndex += offsets[idx] * basis[idx];
-  return linearIndex;
-}
-
-// Clones `op` into a new operations that takes `operands` and returns
-// `resultTypes`.
-static Operation *cloneOpWithOperandsAndTypes(PatternRewriter &builder,
-                                              Location loc, Operation *op,
-                                              ArrayRef<Value> operands,
-                                              ArrayRef<Type> resultTypes) {
-  OperationState res(loc, op->getName().getStringRef(), operands, resultTypes,
-                     op->getAttrs());
-  return builder.createOperation(res);
-}
-
-// Populates 'resultElements[indexMap[i]]' with elements from 'inputElements[i]'
-// for each index 'i' in inputElements with a valid mapping in 'indexMap'.
-static void getMappedElements(const DenseMap<int64_t, int64_t> &indexMap,
-                              ArrayRef<int64_t> inputElements,
-                              SmallVectorImpl<int64_t> &resultElements) {
-  assert(indexMap.size() == resultElements.size());
-  assert(inputElements.size() >= resultElements.size());
-  for (unsigned i = 0, e = inputElements.size(); i < e; ++i) {
-    auto it = indexMap.find(i);
-    if (it != indexMap.end())
-      resultElements[it->second] = inputElements[i];
-  }
-}
-
-// Returns a tuple type with vector element types for each resulting slice
-// of 'vectorType' unrolled by 'sizes' and 'strides'.
-// TODO(andydavis) Move this to a utility function and share it with
-// Extract/InsertSlicesOp verification.
-static TupleType generateExtractSlicesOpResultType(VectorType vectorType,
-                                                   ArrayRef<int64_t> sizes,
-                                                   ArrayRef<int64_t> strides,
-                                                   PatternRewriter &builder) {
-  assert(llvm::all_of(strides, [](int64_t s) { return s == 1; }));
-  assert(static_cast<int64_t>(sizes.size()) == vectorType.getRank());
-  assert(static_cast<int64_t>(strides.size()) == vectorType.getRank());
-
-  // Compute shape ratio of 'shape' and 'sizes'.
-  auto shape = vectorType.getShape();
-  auto maybeDimSliceCounts = shapeRatio(shape, sizes);
-  assert(maybeDimSliceCounts.hasValue());
-  auto sliceDimCounts = *maybeDimSliceCounts;
-
-  // Compute strides w.r.t number of slices in each dimension.
-  auto sliceStrides = computeStrides(sliceDimCounts);
-  int64_t sliceCount = computeMaxLinearIndex(sliceDimCounts);
-  SmallVector<Type, 4> vectorTypes(sliceCount);
-  for (unsigned i = 0; i < sliceCount; ++i) {
-    auto vectorOffsets = delinearize(sliceStrides, i);
-    auto elementOffsets =
-        computeElementOffsetsFromVectorSliceOffsets(sizes, vectorOffsets);
-    auto sliceSizes = computeSliceSizes(shape, sizes, elementOffsets);
-    // Create Vector type and add to 'vectorTypes[i]'.
-    vectorTypes[i] = VectorType::get(sliceSizes, vectorType.getElementType());
-  }
-  return TupleType::get(vectorTypes, builder.getContext());
-}
-
-// UnrolledVectorState aggregates per-operand/result vector state required for
-// unrolling.
-struct UnrolledVectorState {
-  SmallVector<int64_t, 4> unrolledShape;
-  SmallVector<int64_t, 4> unrollFactors;
-  SmallVector<int64_t, 8> basis;
-  int64_t numInstances;
-  Value slicesTuple;
-};
-
-// Populates 'state' with unrolled shape, unroll factors, basis and
-// num unrolled instances for 'vectorType'.
-static void initUnrolledVectorState(VectorType vectorType, Value initValue,
-                                    const DenseMap<int64_t, int64_t> &indexMap,
-                                    ArrayRef<int64_t> targetShape,
-                                    UnrolledVectorState &state,
-                                    PatternRewriter &builder) {
-  // Compute unrolled shape of 'vectorType'.
-  state.unrolledShape.resize(vectorType.getRank());
-  getMappedElements(indexMap, targetShape, state.unrolledShape);
-  // Compute unroll factors for unrolled shape.
-  auto maybeUnrollFactors =
-      shapeRatio(vectorType.getShape(), state.unrolledShape);
-  assert(maybeUnrollFactors.hasValue());
-  state.unrollFactors = *maybeUnrollFactors;
-  // Compute 'basis' and 'numInstances' based on 'state.unrollFactors'.
-  state.basis = computeStrides(state.unrollFactors);
-  state.numInstances = computeMaxLinearIndex(state.unrollFactors);
-  state.slicesTuple = nullptr;
-  if (initValue != nullptr) {
-    // Create ExtractSlicesOp.
-    SmallVector<int64_t, 4> sizes(state.unrolledShape);
-    SmallVector<int64_t, 4> strides(state.unrollFactors.size(), 1);
-    auto tupleType =
-        generateExtractSlicesOpResultType(vectorType, sizes, strides, builder);
-    state.slicesTuple = builder.create<vector::ExtractSlicesOp>(
-        initValue.getLoc(), tupleType, initValue, sizes, strides);
-  }
-}
-
-// Computes and returns the linear index of the unrolled vector at
-// 'vectorOffsets' within the vector represented by 'state'.
-static int64_t
-getUnrolledVectorLinearIndex(UnrolledVectorState &state,
-                             ArrayRef<int64_t> vectorOffsets,
-                             DenseMap<int64_t, int64_t> &indexMap) {
-  // Compute vector offsets.
-  SmallVector<int64_t, 4> sliceOffsets(state.unrolledShape.size());
-  getMappedElements(indexMap, vectorOffsets, sliceOffsets);
-  // Compute and return linear index of 'sliceOffsets' w.r.t 'state.basis'.
-  return linearize(sliceOffsets, state.basis);
-}
-
-// Returns an unrolled vector at 'vectorOffsets' within the vector
-// represented by 'state'. The vector is created from a slice of 'initValue'
-// if not present in 'cache'.
-static Value getOrCreateUnrolledVectorSlice(
-    Location loc, UnrolledVectorState &state, ArrayRef<int64_t> vectorOffsets,
-    ArrayRef<int64_t> offsets, DenseMap<int64_t, int64_t> &indexMap,
-    Value initValue, SmallVectorImpl<Value> &cache, PatternRewriter &builder) {
-  // Compute slice offsets.
-  SmallVector<int64_t, 4> sliceOffsets(state.unrolledShape.size());
-  getMappedElements(indexMap, offsets, sliceOffsets);
-  // TODO(b/144845578) Support non-1 strides.
-  SmallVector<int64_t, 4> sliceStrides(state.unrolledShape.size(), 1);
-  // Compute linear index of 'sliceOffsets' w.r.t 'state.basis'.
-  int64_t sliceLinearIndex =
-      getUnrolledVectorLinearIndex(state, vectorOffsets, indexMap);
-  assert(sliceLinearIndex < static_cast<int64_t>(cache.size()));
-  auto valueSlice = cache[sliceLinearIndex];
-  if (valueSlice == nullptr) {
-    // Return tuple element at 'sliceLinearIndex'.
-    auto tupleIndex = builder.getI64IntegerAttr(sliceLinearIndex);
-    auto initValueType = initValue.getType().cast<VectorType>();
-    auto vectorType =
-        VectorType::get(state.unrolledShape, initValueType.getElementType());
-    // Initialize 'cache' with slice from 'initValue'.
-    valueSlice = builder.create<vector::TupleGetOp>(
-        loc, vectorType, state.slicesTuple, tupleIndex);
-    // Store value back to 'cache'.
-    cache[sliceLinearIndex] = valueSlice;
-  }
-  return valueSlice;
-}
-
-// VectorState aggregates per-operand/result vector state required for
-// creating slices of vector operands, and clones of the operation being
-// unrolled.
-struct VectorState {
-  // The type of this vector.
-  VectorType type;
-  // Map from iteration space index to vector dimension index.
-  DenseMap<int64_t, int64_t> indexMap;
-  // Index of this value in operation's operand list (-1 if not an operand).
-  int64_t operandIndex = -1;
-  // Accumulator iterator flag.
-  bool isAcc = false;
-};
-
-//
-// unrollSingleResultStructuredOp
-//
-// Returns a value representing the result of structured operation 'op'
-// with iteration bounds 'iterationBounds' unrolled to 'targetShape'.
-// A list of VectorState objects must be specified in 'vectors', where
-// each VectorState in the list represents a vector operand or vector result
-// (if the operation does not have an accumulator operand).
-// The VectorState at index 'resultIndex' in the list must be the state
-// associated with the operations single result (i.e. either its accumulator
-// operand or vector result value).
-//
-// Example:
-//
-//  // Before unrolling
-//
-//   operand0                operand1                operand2
-//       \                      |                      /
-//        -------------------- opA --------------------
-//
-//  // After unrolling by 2
-//
-//   operand0                operand1                operand2
-//   /      \                /      \                /      \
-// slice00  slice01       slice10  slice11        slice20  slice21
-//   \         |            |          |            /          |
-//    -------------------- opA0 --------------------           |
-//             |            |          |                       |
-//              \           |          |                      /
-//               -------------------- opA1 -------------------
-//                          |          |
-//                           \        /
-//                           insertslice
-//                                |
-
-// TODO(andydavis) Add the following canonicalization/simplifcation patterns:
-// *) Add pattern which matches InsertStridedSlice -> StridedSlice and forwards
-//    InsertStridedSlice operand to StridedSlice.
-// *) Add pattern which matches SourceOp -> StridedSlice -> UserOp which checks
-//    if there are duplicate identical StridedSlice ops from SourceOp, and
-//    rewrites itself to use the first duplicate. This transformation should
-//    cause users of identifical StridedSlice ops to reuse the same StridedSlice
-//    operation, and leave the duplicate StridedSlice ops with no users
-//    (removable with DCE).
-
-// TODO(andydavis) Generalize this to support structured ops beyond
-// vector ContractionOp, and merge it with 'unrollSingleResultOpMatchingType'
-static Value unrollSingleResultStructuredOp(Operation *op,
-                                            ArrayRef<int64_t> iterationBounds,
-                                            std::vector<VectorState> &vectors,
-                                            unsigned resultIndex,
-                                            ArrayRef<int64_t> targetShape,
-                                            PatternRewriter &builder) {
-  auto shapedType = op->getResult(0).getType().dyn_cast_or_null<ShapedType>();
-  if (!shapedType || !shapedType.hasStaticShape())
-    assert(false && "Expected a statically shaped result type");
-
-  // Compute unroll factors for 'iterationBounds' based on 'targetShape'
-  auto maybeUnrollFactors = shapeRatio(iterationBounds, targetShape);
-  if (!maybeUnrollFactors.hasValue())
-    assert(false && "Failed to compute unroll factors for target shape");
-  auto unrollFactors = *maybeUnrollFactors;
-
-  // Compute unrolled vector state for each vector in 'vectors'.
-  unsigned numVectors = vectors.size();
-  SmallVector<UnrolledVectorState, 3> unrolledVectorState(numVectors);
-  for (unsigned i = 0; i < numVectors; ++i) {
-    int64_t operandIndex = vectors[i].operandIndex;
-    auto operand = operandIndex >= 0 ? op->getOperand(operandIndex) : nullptr;
-    initUnrolledVectorState(vectors[i].type, operand, vectors[i].indexMap,
-                            targetShape, unrolledVectorState[i], builder);
-  }
-  // Compute number of total unrolled instances.
-  auto numUnrolledInstances = computeMaxLinearIndex(unrollFactors);
-  auto sliceStrides = computeStrides(unrollFactors);
-
-  auto &resultValueState = unrolledVectorState[resultIndex];
-  auto unrolledResultType = VectorType::get(resultValueState.unrolledShape,
-                                            shapedType.getElementType());
-
-  // Initialize caches for intermediate vector results.
-  std::vector<SmallVector<Value, 4>> caches(numVectors);
-  for (unsigned i = 0; i < numVectors; ++i)
-    caches[i].resize(unrolledVectorState[i].numInstances);
-
-  // Unroll 'numUnrolledInstances' of 'op', storing results in 'caches'.
-  for (unsigned i = 0; i < numUnrolledInstances; ++i) {
-    auto vectorOffsets = delinearize(sliceStrides, i);
-    auto elementOffsets =
-        computeElementOffsetsFromVectorSliceOffsets(targetShape, vectorOffsets);
-    // Get cached slice (or create slice) for each operand at 'offsets'.
-    SmallVector<Value, 3> operands;
-    operands.resize(op->getNumOperands());
-    for (unsigned i = 0; i < numVectors; ++i) {
-      int64_t operandIndex = vectors[i].operandIndex;
-      if (operandIndex < 0)
-        continue; // Output
-      auto operand = op->getOperand(operandIndex);
-      operands[operandIndex] = getOrCreateUnrolledVectorSlice(
-          op->getLoc(), unrolledVectorState[i], vectorOffsets, elementOffsets,
-          vectors[i].indexMap, operand, caches[i], builder);
-    }
-    // Create op on sliced vector arguments.
-    auto resultVector =
-        cloneOpWithOperandsAndTypes(builder, op->getLoc(), op, operands,
-                                    unrolledResultType)
-            ->getResult(0);
-
-    // Compute linear result index.
-    int64_t linearIndex = getUnrolledVectorLinearIndex(
-        resultValueState, vectorOffsets, vectors[resultIndex].indexMap);
-    // Update result cache at 'linearIndex'.
-    caches[resultIndex][linearIndex] = resultVector;
-  }
-
-  // Create TupleOp of unrolled result vectors.
-  SmallVector<Type, 4> vectorTupleTypes(resultValueState.numInstances);
-  SmallVector<Value, 4> vectorTupleValues(resultValueState.numInstances);
-  for (unsigned i = 0; i < resultValueState.numInstances; ++i) {
-    vectorTupleTypes[i] = caches[resultIndex][i].getType().cast<VectorType>();
-    vectorTupleValues[i] = caches[resultIndex][i];
-  }
-  TupleType tupleType = builder.getTupleType(vectorTupleTypes);
-  Value tupleOp = builder.create<vector::TupleOp>(op->getLoc(), tupleType,
-                                                  vectorTupleValues);
-
-  // Create InsertSlicesOp(Tuple(result_vectors)).
-  auto resultVectorType = op->getResult(0).getType().cast<VectorType>();
-  SmallVector<int64_t, 4> sizes(resultValueState.unrolledShape);
-  SmallVector<int64_t, 4> strides(resultValueState.unrollFactors.size(), 1);
-
-  Value insertSlicesOp = builder.create<vector::InsertSlicesOp>(
-      op->getLoc(), resultVectorType, tupleOp, builder.getI64ArrayAttr(sizes),
-      builder.getI64ArrayAttr(strides));
-  return insertSlicesOp;
-}
-
-static void getVectorContractionOpUnrollState(
-    vector::ContractionOp contractionOp, ArrayRef<int64_t> targetShape,
-    SmallVectorImpl<int64_t> &iterationBounds,
-    std::vector<VectorState> &vectors, unsigned &resultIndex) {
-  // Get contraction op iteration bounds.
-  contractionOp.getIterationBounds(iterationBounds);
-  assert(iterationBounds.size() == targetShape.size());
-  // Get map from iteration space index to lhs/rhs/result shape index.
-  std::vector<DenseMap<int64_t, int64_t>> iterationIndexMapList;
-  contractionOp.getIterationIndexMap(iterationIndexMapList);
-  unsigned numIterators = iterationIndexMapList.size();
-  vectors.resize(numIterators);
-  unsigned accOperandIndex = vector::ContractionOp::getAccOperandIndex();
-  for (unsigned i = 0; i < numIterators; ++i) {
-    vectors[i].type = contractionOp.getOperand(i).getType().cast<VectorType>();
-    vectors[i].indexMap = iterationIndexMapList[i];
-    vectors[i].operandIndex = i;
-    vectors[i].isAcc = i == accOperandIndex ? true : false;
-  }
-
-  if (llvm::size(contractionOp.masks()) == 2) {
-    // Add vectors for lhs/rhs vector mask arguments. Masks have the
-    // same vector shape lhs/rhs args, so copy their index maps.
-    vectors.push_back({contractionOp.getLHSVectorMaskType(),
-                       vectors[0].indexMap, accOperandIndex + 1, false});
-    vectors.push_back({contractionOp.getRHSVectorMaskType(),
-                       vectors[1].indexMap, accOperandIndex + 2, false});
-  }
-  // Unroll 'op' 'iterationBounds' to 'targetShape'.
-  // TODO(andydavis) Use linalg style 'args_in'/'args_out' to partition
-  // 'vectors' instead of 'resultIndex'.
-  resultIndex = accOperandIndex;
-}
-
-static void
-getVectorElementwiseOpUnrollState(Operation *op, ArrayRef<int64_t> targetShape,
-                                  SmallVectorImpl<int64_t> &iterationBounds,
-                                  std::vector<VectorState> &vectors,
-                                  unsigned &resultIndex) {
-  // Verify that operation and operands all have the same vector shape.
-  auto resultType = op->getResult(0).getType().dyn_cast_or_null<VectorType>();
-  assert(resultType && "Expected op with vector result type");
-  auto resultShape = resultType.getShape();
-  // Verify that all operands have the same vector type as result.
-  assert(llvm::all_of(op->getOperandTypes(),
-                      [=](Type type) { return type == resultType; }));
-  // Populate 'iterationBounds' with 'resultShape' for elementwise operations.
-  iterationBounds.assign(resultShape.begin(), resultShape.end());
-
-  // Create trivial elementwise identity index map based on 'resultShape'.
-  DenseMap<int64_t, int64_t> indexMap;
-  indexMap.reserve(resultShape.size());
-  for (unsigned i = 0; i < resultShape.size(); ++i)
-    indexMap[i] = i;
-
-  // Create VectorState each operand and single result.
-  unsigned numVectors = op->getNumOperands() + op->getNumResults();
-  vectors.resize(numVectors);
-  for (unsigned i = 0; i < op->getNumOperands(); ++i)
-    vectors[i] = {resultType, indexMap, i, false};
-  vectors[numVectors - 1] = {resultType, indexMap, -1, false};
-  resultIndex = numVectors - 1;
-}
-
-// Entry point for unrolling declarative pattern rewrites.
-SmallVector<Value, 1> mlir::vector::unrollSingleResultOpMatchingType(
-    PatternRewriter &builder, Operation *op, ArrayRef<int64_t> targetShape) {
-  assert(op->getNumResults() == 1 && "Expected single result operation");
-
-  // Populate 'iterationBounds', 'vectors' and 'resultIndex' to unroll 'op'.
-  SmallVector<int64_t, 6> iterationBounds;
-  std::vector<VectorState> vectors;
-  unsigned resultIndex;
-
-  if (auto contractionOp = dyn_cast<vector::ContractionOp>(op)) {
-    // Populate state for vector ContractionOp.
-    getVectorContractionOpUnrollState(contractionOp, targetShape,
-                                      iterationBounds, vectors, resultIndex);
-  } else {
-    // Populate state for vector elementwise op.
-    getVectorElementwiseOpUnrollState(op, targetShape, iterationBounds, vectors,
-                                      resultIndex);
-  }
-
-  // Unroll 'op' with 'iterationBounds' to 'targetShape'.
-  return SmallVector<Value, 1>{unrollSingleResultStructuredOp(
-      op, iterationBounds, vectors, resultIndex, targetShape, builder)};
-}
-
-/// Generates slices of 'vectorType' according to 'sizes' and 'strides, and
-/// calls 'fn' with linear index and indices for each slice.
-static void
-generateTransferOpSlices(Type memrefElementType, VectorType vectorType,
-                         TupleType tupleType, ArrayRef<int64_t> sizes,
-                         ArrayRef<int64_t> strides, ArrayRef<Value> indices,
-                         PatternRewriter &rewriter,
-                         function_ref<void(unsigned, ArrayRef<Value>)> fn) {
-  // Compute strides w.r.t. to slice counts in each dimension.
-  auto maybeDimSliceCounts = shapeRatio(vectorType.getShape(), sizes);
-  assert(maybeDimSliceCounts.hasValue());
-  auto sliceDimCounts = *maybeDimSliceCounts;
-  auto sliceStrides = computeStrides(sliceDimCounts);
-
-  int64_t numSlices = tupleType.size();
-  unsigned numSliceIndices = indices.size();
-  // Compute 'indexOffset' at which to update 'indices', which is equal
-  // to the memref rank (indices.size) minus the effective 'vectorRank'.
-  // The effective 'vectorRank', is equal to the rank of the vector type
-  // minus the rank of the memref vector element type (if it has one).
-  //
-  // For example:
-  //
-  //   Given memref type 'memref<6x2x1xvector<2x4xf32>>' and vector
-  //   transfer_read/write ops which read/write vectors of type
-  //   'vector<2x1x2x4xf32>'. The memref rank is 3, and the effective
-  //   vector rank is 4 - 2 = 2, and so 'indexOffset' = 3 - 2 = 1.
-  //
-  unsigned vectorRank = vectorType.getRank();
-  if (auto memrefVectorElementType = memrefElementType.dyn_cast<VectorType>()) {
-    assert(vectorRank >= memrefVectorElementType.getRank());
-    vectorRank -= memrefVectorElementType.getRank();
-  }
-  unsigned indexOffset = numSliceIndices - vectorRank;
-
-  auto *ctx = rewriter.getContext();
-  for (unsigned i = 0; i < numSlices; ++i) {
-    auto vectorOffsets = delinearize(sliceStrides, i);
-    auto elementOffsets =
-        computeElementOffsetsFromVectorSliceOffsets(sizes, vectorOffsets);
-    // Compute 'sliceIndices' by adding 'sliceOffsets[i]' to 'indices[i]'.
-    SmallVector<Value, 4> sliceIndices(numSliceIndices);
-    for (unsigned j = 0; j < numSliceIndices; ++j) {
-      if (j < indexOffset) {
-        sliceIndices[j] = indices[j];
-      } else {
-        auto expr = getAffineDimExpr(0, ctx) +
-                    getAffineConstantExpr(elementOffsets[j - indexOffset], ctx);
-        auto map = AffineMap::get(/*dimCount=*/1, /*symbolCount=*/0, expr);
-        sliceIndices[j] = rewriter.create<AffineApplyOp>(
-            indices[j].getLoc(), map, ArrayRef<Value>(indices[j]));
-      }
-    }
-    // Call 'fn' to generate slice 'i' at 'sliceIndices'.
-    fn(i, sliceIndices);
-  }
-}
-
-/// Returns true if 'map' is a suffix of an identity affine map, false
-/// otherwise. Example: affine_map<(d0, d1, d2, d3) -> (d2, d3)>
-static bool isIdentitySuffix(AffineMap map) {
-  if (map.getNumDims() < map.getNumResults())
-    return false;
-  ArrayRef<AffineExpr> results = map.getResults();
-  Optional<int> lastPos;
-  for (unsigned i = 0, e = map.getNumResults(); i < e; ++i) {
-    auto expr = results[i].dyn_cast<AffineDimExpr>();
-    if (!expr)
-      return false;
-    int currPos = static_cast<int>(expr.getPosition());
-    if (lastPos.hasValue() && currPos != lastPos.getValue() + 1)
-      return false;
-    lastPos = currPos;
-  }
-  return true;
-}
-
-namespace {
-
-// Splits vector TransferReadOp into smaller TransferReadOps based on slicing
-// scheme of its unique ExtractSlicesOp user.
-struct SplitTransferReadOp : public OpRewritePattern<vector::TransferReadOp> {
-  using OpRewritePattern<vector::TransferReadOp>::OpRewritePattern;
-
-  PatternMatchResult matchAndRewrite(vector::TransferReadOp xferReadOp,
-                                     PatternRewriter &rewriter) const override {
-    // TODO(andydavis, ntv) Support splitting TransferReadOp with non-identity
-    // permutation maps. Repurpose code from MaterializeVectors transformation.
-    if (!isIdentitySuffix(xferReadOp.permutation_map()))
-      return matchFailure();
-    // Return unless the unique 'xferReadOp' user is an ExtractSlicesOp.
-    Value xferReadResult = xferReadOp.getResult();
-    auto extractSlicesOp =
-        dyn_cast<vector::ExtractSlicesOp>(*xferReadResult.getUsers().begin());
-    if (!xferReadResult.hasOneUse() || !extractSlicesOp)
-      return matchFailure();
-
-    // Get 'sizes' and 'strides' parameters from ExtractSlicesOp user.
-    auto sourceVectorType = extractSlicesOp.getSourceVectorType();
-    auto resultTupleType = extractSlicesOp.getResultTupleType();
-    SmallVector<int64_t, 4> sizes;
-    extractSlicesOp.getSizes(sizes);
-    SmallVector<int64_t, 4> strides;
-    extractSlicesOp.getStrides(strides);
-    assert(llvm::all_of(strides, [](int64_t s) { return s == 1; }));
-
-    Location loc = xferReadOp.getLoc();
-    auto memrefElementType =
-        xferReadOp.memref().getType().cast<MemRefType>().getElementType();
-    int64_t numSlices = resultTupleType.size();
-    SmallVector<Value, 4> vectorTupleValues(numSlices);
-    SmallVector<Value, 4> indices(xferReadOp.indices().begin(),
-                                  xferReadOp.indices().end());
-    auto createSlice = [&](unsigned index, ArrayRef<Value> sliceIndices) {
-      // Get VectorType for slice 'i'.
-      auto sliceVectorType = resultTupleType.getType(index);
-      // Create split TransferReadOp for 'sliceUser'.
-      vectorTupleValues[index] = rewriter.create<vector::TransferReadOp>(
-          loc, sliceVectorType, xferReadOp.memref(), sliceIndices,
-          xferReadOp.permutation_map(), xferReadOp.padding());
-    };
-    generateTransferOpSlices(memrefElementType, sourceVectorType,
-                             resultTupleType, sizes, strides, indices, rewriter,
-                             createSlice);
-
-    // Create tuple of splice xfer read operations.
-    Value tupleOp = rewriter.create<vector::TupleOp>(loc, resultTupleType,
-                                                     vectorTupleValues);
-    // Replace 'xferReadOp' with result 'insertSlicesResult'.
-    rewriter.replaceOpWithNewOp<vector::InsertSlicesOp>(
-        xferReadOp, sourceVectorType, tupleOp, extractSlicesOp.sizes(),
-        extractSlicesOp.strides());
-    return matchSuccess();
-  }
-};
-
-// Splits vector TransferWriteOp into smaller TransferWriteOps for each source.
-struct SplitTransferWriteOp : public OpRewritePattern<vector::TransferWriteOp> {
-  using OpRewritePattern<vector::TransferWriteOp>::OpRewritePattern;
-
-  PatternMatchResult matchAndRewrite(vector::TransferWriteOp xferWriteOp,
-                                     PatternRewriter &rewriter) const override {
-    // TODO(andydavis, ntv) Support splitting TransferWriteOp with non-identity
-    // permutation maps. Repurpose code from MaterializeVectors transformation.
-    if (!isIdentitySuffix(xferWriteOp.permutation_map()))
-      return matchFailure();
-    // Return unless the 'xferWriteOp' 'vector' operand is an 'InsertSlicesOp'.
-    auto *vectorDefOp = xferWriteOp.vector().getDefiningOp();
-    auto insertSlicesOp = dyn_cast_or_null<vector::InsertSlicesOp>(vectorDefOp);
-    if (!insertSlicesOp)
-      return matchFailure();
-
-    // Get TupleOp operand of 'insertSlicesOp'.
-    auto tupleOp = dyn_cast_or_null<vector::TupleOp>(
-        insertSlicesOp.vectors().getDefiningOp());
-    if (!tupleOp)
-      return matchFailure();
-
-    // Get 'sizes' and 'strides' parameters from InsertSlicesOp user.
-    auto sourceTupleType = insertSlicesOp.getSourceTupleType();
-    auto resultVectorType = insertSlicesOp.getResultVectorType();
-    SmallVector<int64_t, 4> sizes;
-    insertSlicesOp.getSizes(sizes);
-    SmallVector<int64_t, 4> strides;
-    insertSlicesOp.getStrides(strides);
-
-    Location loc = xferWriteOp.getLoc();
-    auto memrefElementType =
-        xferWriteOp.memref().getType().cast<MemRefType>().getElementType();
-    SmallVector<Value, 4> indices(xferWriteOp.indices().begin(),
-                                  xferWriteOp.indices().end());
-    auto createSlice = [&](unsigned index, ArrayRef<Value> sliceIndices) {
-      // Create split TransferWriteOp for source vector 'tupleOp.operand[i]'.
-      rewriter.create<vector::TransferWriteOp>(
-          loc, tupleOp.getOperand(index), xferWriteOp.memref(), sliceIndices,
-          xferWriteOp.permutation_map());
-    };
-    generateTransferOpSlices(memrefElementType, resultVectorType,
-                             sourceTupleType, sizes, strides, indices, rewriter,
-                             createSlice);
-
-    // Erase old 'xferWriteOp'.
-    rewriter.eraseOp(xferWriteOp);
-    return matchSuccess();
-  }
-};
-
-/// Decomposes ShapeCastOp on tuple-of-vectors to multiple ShapeCastOps, each
-/// on vector types.
-struct ShapeCastOpDecomposer : public OpRewritePattern<vector::ShapeCastOp> {
-  using OpRewritePattern<vector::ShapeCastOp>::OpRewritePattern;
-
-  PatternMatchResult matchAndRewrite(vector::ShapeCastOp shapeCastOp,
-                                     PatternRewriter &rewriter) const override {
-    // Check if 'shapeCastOp' has tuple source/result type.
-    auto sourceTupleType =
-        shapeCastOp.source().getType().dyn_cast_or_null<TupleType>();
-    auto resultTupleType =
-        shapeCastOp.result().getType().dyn_cast_or_null<TupleType>();
-    if (!sourceTupleType || !resultTupleType)
-      return matchFailure();
-    assert(sourceTupleType.size() == resultTupleType.size());
-
-    // Create single-vector ShapeCastOp for each source tuple element.
-    Location loc = shapeCastOp.getLoc();
-    SmallVector<Value, 8> resultElements;
-    resultElements.reserve(resultTupleType.size());
-    for (unsigned i = 0, e = sourceTupleType.size(); i < e; ++i) {
-      auto sourceElement = rewriter.create<vector::TupleGetOp>(
-          loc, sourceTupleType.getType(i), shapeCastOp.source(),
-          rewriter.getI64IntegerAttr(i));
-      resultElements.push_back(rewriter.create<vector::ShapeCastOp>(
-          loc, resultTupleType.getType(i), sourceElement));
-    }
-
-    // Replace 'shapeCastOp' with tuple of 'resultElements'.
-    rewriter.replaceOpWithNewOp<vector::TupleOp>(shapeCastOp, resultTupleType,
-                                                 resultElements);
-    return matchSuccess();
-  }
-};
-
-/// ShapeCastOpFolder folds cancelling ShapeCastOps away.
-//
-// Example:
-//
-//  The following MLIR with cancelling ShapeCastOps:
-//
-//   %0 = source : vector<5x4x2xf32>
-//   %1 = shape_cast %0 : vector<5x4x2xf32> to vector<20x2xf32>
-//   %2 = shape_cast %1 : vector<20x2xf32> to vector<5x4x2xf32>
-//   %3 = user %2 : vector<5x4x2xf32>
-//
-//  Should canonicalize to the following:
-//
-//   %0 = source : vector<5x4x2xf32>
-//   %1 = user %0 : vector<5x4x2xf32>
-//
-struct ShapeCastOpFolder : public OpRewritePattern<vector::ShapeCastOp> {
-  using OpRewritePattern<vector::ShapeCastOp>::OpRewritePattern;
-
-  PatternMatchResult matchAndRewrite(vector::ShapeCastOp shapeCastOp,
-                                     PatternRewriter &rewriter) const override {
-    // Check if 'shapeCastOp' has vector source/result type.
-    auto sourceVectorType =
-        shapeCastOp.source().getType().dyn_cast_or_null<VectorType>();
-    auto resultVectorType =
-        shapeCastOp.result().getType().dyn_cast_or_null<VectorType>();
-    if (!sourceVectorType || !resultVectorType)
-      return matchFailure();
-
-    // Check if shape cast op source operand is also a shape cast op.
-    auto sourceShapeCastOp = dyn_cast_or_null<vector::ShapeCastOp>(
-        shapeCastOp.source().getDefiningOp());
-    if (!sourceShapeCastOp)
-      return matchFailure();
-    auto operandSourceVectorType =
-        sourceShapeCastOp.source().getType().cast<VectorType>();
-    auto operandResultVectorType =
-        sourceShapeCastOp.result().getType().cast<VectorType>();
-
-    // Check if shape cast operations invert each other.
-    if (operandSourceVectorType != resultVectorType ||
-        operandResultVectorType != sourceVectorType)
-      return matchFailure();
-
-    rewriter.replaceOp(shapeCastOp, sourceShapeCastOp.source());
-    return matchSuccess();
-  }
-};
-
-// Patter rewrite which forward tuple elements to their users.
-// User(TupleGetOp(ExtractSlicesOp(InsertSlicesOp(TupleOp(Producer)))))
-//   -> User(Producer)
-struct TupleGetFolderOp : public OpRewritePattern<vector::TupleGetOp> {
-  using OpRewritePattern<vector::TupleGetOp>::OpRewritePattern;
-
-  PatternMatchResult matchAndRewrite(vector::TupleGetOp tupleGetOp,
-                                     PatternRewriter &rewriter) const override {
-    // Return if 'tupleGetOp.vectors' arg was not defined by ExtractSlicesOp.
-    auto extractSlicesOp = dyn_cast_or_null<vector::ExtractSlicesOp>(
-        tupleGetOp.vectors().getDefiningOp());
-    if (!extractSlicesOp)
-      return matchFailure();
-
-    // Return if 'extractSlicesOp.vector' arg was not defined by InsertSlicesOp.
-    auto insertSlicesOp = dyn_cast_or_null<vector::InsertSlicesOp>(
-        extractSlicesOp.vector().getDefiningOp());
-    if (!insertSlicesOp)
-      return matchFailure();
-
-    // Return if 'insertSlicesOp.vectors' arg was not defined by TupleOp.
-    auto tupleOp = dyn_cast_or_null<vector::TupleOp>(
-        insertSlicesOp.vectors().getDefiningOp());
-    if (!tupleOp)
-      return matchFailure();
-
-    // Forward Value from 'tupleOp' at 'tupleGetOp.index'.
-    Value tupleValue = tupleOp.getOperand(tupleGetOp.getIndex());
-    rewriter.replaceOp(tupleGetOp, tupleValue);
-    return matchSuccess();
-  }
-};
-
-/// Progressive lowering of ExtractSlicesOp to tuple of StridedSliceOp.
-/// One:
-///   %x = vector.extract_slices %0
-/// is replaced by:
-///   %a = vector.strided_slice %0
-///   %b = vector.strided_slice %0
-///   ..
-///   %x = vector.tuple %a, %b, ..
-class ExtractSlicesOpLowering
-    : public OpRewritePattern<vector::ExtractSlicesOp> {
-public:
-  using OpRewritePattern<vector::ExtractSlicesOp>::OpRewritePattern;
-
-  PatternMatchResult matchAndRewrite(vector::ExtractSlicesOp op,
-                                     PatternRewriter &rewriter) const override {
-    auto loc = op.getLoc();
-
-    VectorType vectorType = op.getSourceVectorType();
-    auto shape = vectorType.getShape();
-
-    SmallVector<int64_t, 4> sizes;
-    op.getSizes(sizes);
-    SmallVector<int64_t, 4> strides;
-    op.getStrides(strides); // all-ones at the moment
-
-    // For each element in the tuple, generate the proper strided slice.
-    TupleType tupleType = op.getResultTupleType();
-    int64_t tupleSize = tupleType.size();
-    SmallVector<Value, 4> tupleValues(tupleSize);
-    auto sliceStrides = computeStrides(shape, sizes);
-    for (int64_t i = 0; i < tupleSize; ++i) {
-      auto vectorOffsets = delinearize(sliceStrides, i);
-      auto elementOffsets =
-          computeElementOffsetsFromVectorSliceOffsets(sizes, vectorOffsets);
-      auto sliceSizes = computeSliceSizes(shape, sizes, elementOffsets);
-      // Insert in tuple.
-      tupleValues[i] = rewriter.create<vector::StridedSliceOp>(
-          loc, op.vector(), elementOffsets, sliceSizes, strides);
-    }
-
-    rewriter.replaceOpWithNewOp<vector::TupleOp>(op, tupleType, tupleValues);
-    return matchSuccess();
-  }
-};
-
-/// Progressive lowering of InsertSlicesOp to series of InsertStridedSliceOp.
-/// One:
-///   %x = vector.insert_slices %0
-/// is replaced by:
-///   %r0 = vector.splat 0
-//    %t1 = vector.tuple_get %0, 0
-///   %r1 = vector.insert_strided_slice %r0, %t1
-//    %t2 = vector.tuple_get %0, 1
-///   %r2 = vector.insert_strided_slice %r1, %t2
-///   ..
-///   %x  = ..
-class InsertSlicesOpLowering : public OpRewritePattern<vector::InsertSlicesOp> {
-public:
-  using OpRewritePattern<vector::InsertSlicesOp>::OpRewritePattern;
-
-  PatternMatchResult matchAndRewrite(vector::InsertSlicesOp op,
-                                     PatternRewriter &rewriter) const override {
-    auto loc = op.getLoc();
-
-    VectorType vectorType = op.getResultVectorType();
-    auto shape = vectorType.getShape();
-
-    SmallVector<int64_t, 4> sizes;
-    op.getSizes(sizes);
-    SmallVector<int64_t, 4> strides;
-    op.getStrides(strides); // all-ones at the moment
-
-    // Prepare result.
-    auto elemType = vectorType.getElementType();
-    Value zero = rewriter.create<ConstantOp>(loc, elemType,
-                                             rewriter.getZeroAttr(elemType));
-    Value result = rewriter.create<SplatOp>(loc, vectorType, zero);
-
-    // For each element in the tuple, extract the proper strided slice.
-    TupleType tupleType = op.getSourceTupleType();
-    int64_t tupleSize = tupleType.size();
-    auto sliceStrides = computeStrides(shape, sizes);
-    for (int64_t i = 0; i < tupleSize; ++i) {
-      auto vectorOffsets = delinearize(sliceStrides, i);
-      auto elementOffsets =
-          computeElementOffsetsFromVectorSliceOffsets(sizes, vectorOffsets);
-      // Extract from tuple into the result.
-      auto index = rewriter.getI64IntegerAttr(i);
-      auto tupleGet = rewriter.create<vector::TupleGetOp>(
-          loc, tupleType.getType(i), op.getOperand(), index);
-      result = rewriter.create<vector::InsertStridedSliceOp>(
-          loc, tupleGet, result, elementOffsets, strides);
-    }
-
-    rewriter.replaceOp(op, result);
-    return matchSuccess();
-  }
-};
-
-/// Progressive lowering of OuterProductOp.
-/// One:
-///   %x = vector.outerproduct %lhs, %rhs, %acc
-/// is replaced by:
-///   %z = zero-result
-///   %0 = vector.extract %lhs[0]
-///   %1 = vector.broadcast %0
-///   %2 = vector.extract %acc[0]
-///   %3 = vector.fma %1, %arg1, %2
-///   %4 = vector.insert %3, %z[0]
-///   ..
-///   %x = vector.insert %.., %..[N-1]
-///
-class OuterProductOpLowering : public OpRewritePattern<vector::OuterProductOp> {
-public:
-  using OpRewritePattern<vector::OuterProductOp>::OpRewritePattern;
-
-  PatternMatchResult matchAndRewrite(vector::OuterProductOp op,
-                                     PatternRewriter &rewriter) const override {
-    auto loc = op.getLoc();
-
-    VectorType rhsType = op.getOperandVectorTypeRHS();
-    VectorType resType = op.getVectorType();
-    Type eltType = resType.getElementType();
-    Value acc = (op.acc().empty()) ? nullptr : op.acc()[0];
-
-    Value zero = rewriter.create<ConstantOp>(loc, eltType,
-                                             rewriter.getZeroAttr(eltType));
-    Value result = rewriter.create<SplatOp>(loc, resType, zero);
-    for (int64_t d = 0, e = resType.getDimSize(0); d < e; ++d) {
-      auto pos = rewriter.getI64ArrayAttr(d);
-      Value x = rewriter.create<vector::ExtractOp>(loc, eltType, op.lhs(), pos);
-      Value b = rewriter.create<vector::BroadcastOp>(loc, rhsType, x);
-      Value m;
-      if (acc) {
-        Value z = rewriter.create<vector::ExtractOp>(loc, rhsType, acc, pos);
-        m = rewriter.create<vector::FMAOp>(loc, b, op.rhs(), z);
-      } else {
-        m = rewriter.create<MulFOp>(loc, b, op.rhs());
-      }
-      result = rewriter.create<vector::InsertOp>(loc, resType, m, result, pos);
-    }
-    rewriter.replaceOp(op, result);
-    return matchSuccess();
-  }
-};
-
-/// Progressive lowering of ContractionOp.
-/// One:
-///   %x = vector.contract with at least one free/batch dimension
-/// is replaced by:
-///   %a = vector.contract with one less free/batch dimension
-///   %b = vector.contract with one less free/batch dimension
-///   ..
-///   %x = combine %a %b ..
-/// until a pure contraction is reached (no free/batch dimensions),
-/// which is replaced by a fma/reduction op.
-///
-/// TODO(ajcbik): break down into transpose/reshape/cast ops
-///               when they become available to avoid code dup
-/// TODO(ajcbik): investigate lowering order impact on performance
-class ContractionOpLowering : public OpRewritePattern<vector::ContractionOp> {
-public:
-  using OpRewritePattern<vector::ContractionOp>::OpRewritePattern;
-
-  PatternMatchResult matchAndRewrite(vector::ContractionOp op,
-                                     PatternRewriter &rewriter) const override {
-    // TODO(ajcbik): implement masks
-    if (llvm::size(op.masks()) != 0)
-      return matchFailure();
-
-    // TODO(ntv, ajcbik): implement benefits, cost models, separate this out in
-    // a new pattern.
-    // TODO(ntv, fhahn): once row-major mode is available in LLVM's matrix
-    // intrinsics, use that.
-    if (lowerToLLVMMatrixIntrinsics &&
-        isColumnMajorMatmul(op.indexing_maps())) {
-      VectorType lhsType = op.getLhsType();
-      VectorType rhsType = op.getRhsType();
-      Type flattenedLHSType =
-          VectorType::get(lhsType.getNumElements(), lhsType.getElementType());
-      Type flattenedRHSType =
-          VectorType::get(rhsType.getNumElements(), rhsType.getElementType());
-      auto lhs = rewriter.create<vector::ShapeCastOp>(
-          op.getLoc(), flattenedLHSType, op.lhs());
-      auto rhs = rewriter.create<vector::ShapeCastOp>(
-          op.getLoc(), flattenedRHSType, op.rhs());
-
-      unsigned lhsRows = op.getLhsType().getShape()[0];
-      unsigned lhsColumns = op.getLhsType().getShape()[1];
-      unsigned rhsColumns = op.getRhsType().getShape()[1];
-      Value mul = rewriter.create<vector::MatmulOp>(
-          op.getLoc(), lhs, rhs, lhsRows, lhsColumns, rhsColumns);
-      mul = rewriter.create<vector::ShapeCastOp>(op.getLoc(),
-                                                 op.acc().getType(), mul);
-      Type elementType = op.getLhsType().getElementType();
-      assert(elementType.isIntOrFloat());
-      if (elementType.isa<IntegerType>())
-        rewriter.replaceOpWithNewOp<AddIOp>(op, op.acc(), mul);
-      else
-        rewriter.replaceOpWithNewOp<AddFOp>(op, op.acc(), mul);
-      return matchSuccess();
-    }
-
-    // Find first batch dimension in LHS/RHS, and lower when found.
-    std::vector<std::pair<int64_t, int64_t>> batchDimMap = op.getBatchDimMap();
-    if (!batchDimMap.empty()) {
-      int64_t lhsIndex = batchDimMap[0].first;
-      int64_t rhsIndex = batchDimMap[0].second;
-      rewriter.replaceOp(op, lowerParallel(op, lhsIndex, rhsIndex, rewriter));
-      return matchSuccess();
-    }
-
-    // Collect contracting dimensions.
-    std::vector<std::pair<int64_t, int64_t>> contractingDimMap =
-        op.getContractingDimMap();
-    DenseSet<int64_t> lhsContractingDimSet;
-    DenseSet<int64_t> rhsContractingDimSet;
-    for (auto &dimPair : contractingDimMap) {
-      lhsContractingDimSet.insert(dimPair.first);
-      rhsContractingDimSet.insert(dimPair.second);
-    }
-
-    // Find first free dimension in LHS, and lower when found.
-    VectorType lhsType = op.getLhsType();
-    for (int64_t lhsIndex = 0, e = lhsType.getRank(); lhsIndex < e;
-         ++lhsIndex) {
-      if (lhsContractingDimSet.count(lhsIndex) == 0) {
-        rewriter.replaceOp(
-            op, lowerParallel(op, lhsIndex, /*rhsIndex=*/-1, rewriter));
-        return matchSuccess();
-      }
-    }
-
-    // Find first free dimension in RHS, and lower when found.
-    VectorType rhsType = op.getRhsType();
-    for (int64_t rhsIndex = 0, e = rhsType.getRank(); rhsIndex < e;
-         ++rhsIndex) {
-      if (rhsContractingDimSet.count(rhsIndex) == 0) {
-        rewriter.replaceOp(
-            op, lowerParallel(op, /*lhsIndex=*/-1, rhsIndex, rewriter));
-        return matchSuccess();
-      }
-    }
-
-    // Lower the first remaining reduction dimension.
-    if (!contractingDimMap.empty()) {
-      rewriter.replaceOp(op, lowerReduction(op, rewriter));
-      return matchSuccess();
-    }
-
-    return matchFailure();
-  }
-
-private:
-  // Lower one parallel dimension.
-  // TODO(ajcbik): consider reusing existing contract unrolling
-  Value lowerParallel(vector::ContractionOp op, int64_t lhsIndex,
-                      int64_t rhsIndex, PatternRewriter &rewriter) const {
-    VectorType lhsType = op.getLhsType();
-    VectorType rhsType = op.getRhsType();
-    VectorType resType = op.getResultType().cast<VectorType>();
-    // Find the iterator type index and result index.
-    SmallVector<AffineMap, 4> iMap = op.getIndexingMaps();
-    int64_t iterIndex = -1;
-    int64_t dimSize = -1;
-    if (lhsIndex >= 0) {
-      iterIndex =
-          iMap[0].getResult(lhsIndex).cast<AffineDimExpr>().getPosition();
-      assert((rhsIndex < 0 || iterIndex == iMap[1]
-                                               .getResult(rhsIndex)
-                                               .cast<AffineDimExpr>()
-                                               .getPosition()) &&
-             "parallel index should be free in LHS or batch in LHS/RHS");
-      dimSize = lhsType.getDimSize(lhsIndex);
-    } else {
-      assert(rhsIndex >= 0 && "missing parallel index");
-      iterIndex =
-          iMap[1].getResult(rhsIndex).cast<AffineDimExpr>().getPosition();
-      dimSize = rhsType.getDimSize(rhsIndex);
-    }
-    assert(iterIndex >= 0 && "parallel index not listed in operand mapping");
-    Optional<int64_t> lookup = getResultIndex(iMap[2], iterIndex);
-    assert(lookup.hasValue() && "parallel index not listed in reduction");
-    int64_t resIndex = lookup.getValue();
-    // Construct new iterator types and affine map array attribute.
-    SmallVector<AffineMap, 4> lowIndexingMaps;
-    lowIndexingMaps.push_back(adjustMap(iMap[0], iterIndex, rewriter));
-    lowIndexingMaps.push_back(adjustMap(iMap[1], iterIndex, rewriter));
-    lowIndexingMaps.push_back(adjustMap(iMap[2], iterIndex, rewriter));
-    auto lowAffine = rewriter.getAffineMapArrayAttr(lowIndexingMaps);
-    auto lowIter =
-        rewriter.getArrayAttr(adjustIter(op.iterator_types(), iterIndex));
-    // Unroll into a series of lower dimensional vector.contract ops.
-    Location loc = op.getLoc();
-    Value result = zeroVector(loc, resType, rewriter);
-    for (int64_t d = 0; d < dimSize; ++d) {
-      auto lhs = reshapeLoad(loc, op.lhs(), lhsType, lhsIndex, d, rewriter);
-      auto rhs = reshapeLoad(loc, op.rhs(), rhsType, rhsIndex, d, rewriter);
-      auto acc = reshapeLoad(loc, op.acc(), resType, resIndex, d, rewriter);
-      Value lowContract = rewriter.create<vector::ContractionOp>(
-          loc, lhs, rhs, acc, lowAffine, lowIter);
-      result = reshapeStore(loc, lowContract, result, resType, resIndex, d,
-                            rewriter);
-    }
-    return result;
-  }
-
-  // Lower one reduction dimension.
-  Value lowerReduction(vector::ContractionOp op,
-                       PatternRewriter &rewriter) const {
-    auto loc = op.getLoc();
-    VectorType lhsType = op.getLhsType();
-    VectorType rhsType = op.getRhsType();
-    Type resType = op.getResultType();
-    assert(!resType.isa<VectorType>());
-    // Use iterator index 0.
-    int64_t iterIndex = 0;
-    SmallVector<AffineMap, 4> iMap = op.getIndexingMaps();
-    Optional<int64_t> lookupLhs = getResultIndex(iMap[0], iterIndex);
-    Optional<int64_t> lookupRhs = getResultIndex(iMap[1], iterIndex);
-    assert(lookupLhs.hasValue() && "missing LHS parallel index");
-    assert(lookupRhs.hasValue() && "missing RHS parallel index");
-    int64_t lhsIndex = lookupLhs.getValue();
-    int64_t rhsIndex = lookupRhs.getValue();
-    int64_t dimSize = lhsType.getDimSize(lhsIndex);
-    assert(dimSize == rhsType.getDimSize(rhsIndex) && "corrupt shape");
-    // Base case.
-    if (lhsType.getRank() == 1) {
-      assert(rhsType.getRank() == 1 && "corrupt contraction");
-      Value zero = zeroVector(loc, lhsType, rewriter);
-      Value fma = rewriter.create<vector::FMAOp>(loc, op.lhs(), op.rhs(), zero);
-      StringAttr kind = rewriter.getStringAttr("add");
-      return rewriter.create<vector::ReductionOp>(loc, resType, kind, fma,
-                                                  op.acc());
-    }
-    // Construct new iterator types and affine map array attribute.
-    SmallVector<AffineMap, 4> lowIndexingMaps;
-    lowIndexingMaps.push_back(adjustMap(iMap[0], iterIndex, rewriter));
-    lowIndexingMaps.push_back(adjustMap(iMap[1], iterIndex, rewriter));
-    lowIndexingMaps.push_back(adjustMap(iMap[2], iterIndex, rewriter));
-    auto lowAffine = rewriter.getAffineMapArrayAttr(lowIndexingMaps);
-    auto lowIter =
-        rewriter.getArrayAttr(adjustIter(op.iterator_types(), iterIndex));
-    // Unroll into a series of lower dimensional vector.contract ops.
-    // By feeding the initial accumulator into the first contraction,
-    // and the result of each contraction into the next, eventually
-    // the sum of all reductions is computed.
-    Value result = op.acc();
-    for (int64_t d = 0; d < dimSize; ++d) {
-      auto lhs = reshapeLoad(loc, op.lhs(), lhsType, lhsIndex, d, rewriter);
-      auto rhs = reshapeLoad(loc, op.rhs(), rhsType, rhsIndex, d, rewriter);
-      result = rewriter.create<vector::ContractionOp>(loc, lhs, rhs, result,
-                                                      lowAffine, lowIter);
-    }
-    return result;
-  }
-
-  // Helper method to construct a zero vector.
-  static Value zeroVector(Location loc, VectorType vType,
-                          PatternRewriter &rewriter) {
-    Type eltType = vType.getElementType();
-    Value zero = rewriter.create<ConstantOp>(loc, eltType,
-                                             rewriter.getZeroAttr(eltType));
-    return rewriter.create<SplatOp>(loc, vType, zero);
-  }
-
-  // Helper to find an index in an affine map.
-  static Optional<int64_t> getResultIndex(AffineMap map, int64_t index) {
-    for (int64_t i = 0, e = map.getNumResults(); i < e; ++i) {
-      int64_t idx = map.getResult(i).cast<AffineDimExpr>().getPosition();
-      if (idx == index)
-        return i;
-    }
-    return None;
-  }
-
-  // Helper to construct iterator types with one index removed.
-  static SmallVector<Attribute, 4> adjustIter(ArrayAttr iteratorTypes,
-                                              int64_t index) {
-    SmallVector<Attribute, 4> results;
-    for (auto it : llvm::enumerate(iteratorTypes)) {
-      int64_t idx = it.index();
-      if (idx == index)
-        continue;
-      results.push_back(it.value());
-    }
-    return results;
-  }
-
-  // Helper to construct an affine map with one index removed.
-  static AffineMap adjustMap(AffineMap map, int64_t index,
-                             PatternRewriter &rewriter) {
-    auto *ctx = rewriter.getContext();
-    SmallVector<AffineExpr, 4> results;
-    for (int64_t i = 0, e = map.getNumResults(); i < e; ++i) {
-      int64_t idx = map.getResult(i).cast<AffineDimExpr>().getPosition();
-      if (idx == index)
-        continue;
-      // Re-insert remaining indices, but renamed when occurring
-      // after the removed index.
-      auto targetExpr = getAffineDimExpr(idx < index ? idx : idx - 1, ctx);
-      results.push_back(targetExpr);
-    }
-    // The (...) -> () affine map has its own factory method.
-    return results.empty() ? AffineMap::get(map.getNumDims() - 1, 0, ctx)
-                           : AffineMap::get(map.getNumDims() - 1, 0, results);
-  }
-
-  // Helper to drop dimension from vector type.
-  static Type adjustType(VectorType tp, int64_t index) {
-    int64_t rank = tp.getRank();
-    Type eltType = tp.getElementType();
-    if (rank == 1) {
-      assert(index == 0 && "index for scalar result out of bounds");
-      return eltType;
-    }
-    SmallVector<int64_t, 4> adjustedShape;
-    for (int64_t i = 0; i < rank; ++i) {
-      // Omit dimension at the given index.
-      if (i == index)
-        continue;
-      // Otherwise, add dimension back.
-      adjustedShape.push_back(tp.getDimSize(i));
-    }
-    return VectorType::get(adjustedShape, eltType);
-  }
-
-  // Helper method to possibly drop a dimension in a load.
-  // TODO(ajcbik): use a reshaping vector load (and share lowering code)
-  static Value reshapeLoad(Location loc, Value val, VectorType type,
-                           int64_t index, int64_t pos,
-                           PatternRewriter &rewriter) {
-    if (index == -1)
-      return val;
-    Type lowType = adjustType(type, 0);
-    // At extraction dimension?
-    if (index == 0) {
-      auto posAttr = rewriter.getI64ArrayAttr(pos);
-      return rewriter.create<vector::ExtractOp>(loc, lowType, val, posAttr);
-    }
-    // Unroll leading dimensions.
-    VectorType vType = lowType.cast<VectorType>();
-    VectorType resType = adjustType(type, index).cast<VectorType>();
-    Value result = zeroVector(loc, resType, rewriter);
-    for (int64_t d = 0, e = resType.getDimSize(0); d < e; d++) {
-      auto posAttr = rewriter.getI64ArrayAttr(d);
-      Value ext = rewriter.create<vector::ExtractOp>(loc, vType, val, posAttr);
-      Value load = reshapeLoad(loc, ext, vType, index - 1, pos, rewriter);
-      result = rewriter.create<vector::InsertOp>(loc, resType, load, result,
-                                                 posAttr);
-    }
-    return result;
-  }
-
-  // Helper method to possibly drop a dimension in a store.
-  // TODO(ajcbik): use a reshaping vector store (and share lowering code)
-  static Value reshapeStore(Location loc, Value val, Value result,
-                            VectorType type, int64_t index, int64_t pos,
-                            PatternRewriter &rewriter) {
-    // Unmodified?
-    if (index == -1)
-      return val;
-    // At insertion dimension?
-    if (index == 0) {
-      auto posAttr = rewriter.getI64ArrayAttr(pos);
-      return rewriter.create<vector::InsertOp>(loc, type, val, result, posAttr);
-    }
-    // Unroll leading dimensions.
-    Type lowType = adjustType(type, 0);
-    VectorType vType = lowType.cast<VectorType>();
-    Type insType = adjustType(vType, 0);
-    for (int64_t d = 0, e = type.getDimSize(0); d < e; d++) {
-      auto posAttr = rewriter.getI64ArrayAttr(d);
-      Value ext =
-          rewriter.create<vector::ExtractOp>(loc, vType, result, posAttr);
-      Value ins =
-          rewriter.create<vector::ExtractOp>(loc, insType, val, posAttr);
-      Value sto = reshapeStore(loc, ins, ext, vType, index - 1, pos, rewriter);
-      result =
-          rewriter.create<vector::InsertOp>(loc, type, sto, result, posAttr);
-    }
-    return result;
-  }
-};
-
-/// ShapeOp 2D -> 1D downcast serves the purpose of flattening 2-D to 1-D
-/// vectors progressively on the way to target llvm.matrix intrinsics.
-/// This iterates over the most major dimension of the 2-D vector and performs
-/// rewrites into:
-///   vector.extract from 2-D + vector.insert_strided_slice offset into 1-D
-class ShapeCastOp2DDownCastRewritePattern
-    : public OpRewritePattern<vector::ShapeCastOp> {
-public:
-  using OpRewritePattern<vector::ShapeCastOp>::OpRewritePattern;
-
-  PatternMatchResult matchAndRewrite(vector::ShapeCastOp op,
-                                     PatternRewriter &rewriter) const override {
-    auto sourceVectorType = op.getSourceVectorType();
-    auto resultVectorType = op.getResultVectorType();
-    if (sourceVectorType.getRank() != 2 || resultVectorType.getRank() != 1)
-      return matchFailure();
-
-    auto loc = op.getLoc();
-    auto elemType = sourceVectorType.getElementType();
-    Value zero = rewriter.create<ConstantOp>(loc, elemType,
-                                             rewriter.getZeroAttr(elemType));
-    Value desc = rewriter.create<SplatOp>(loc, resultVectorType, zero);
-    unsigned mostMinorVectorSize = sourceVectorType.getShape()[1];
-    for (int64_t i = 0, e = sourceVectorType.getShape().front(); i != e; ++i) {
-      Value vec = rewriter.create<vector::ExtractOp>(loc, op.source(), i);
-      desc = rewriter.create<vector::InsertStridedSliceOp>(
-          loc, vec, desc,
-          /*offsets=*/i * mostMinorVectorSize, /*strides=*/1);
-    }
-    rewriter.replaceOp(op, desc);
-    return matchSuccess();
-  }
-};
-
-/// ShapeOp 1D -> 2D upcast serves the purpose of unflattening 2-D from 1-D
-/// vectors progressively on the way from targeting llvm.matrix intrinsics.
-/// This iterates over the most major dimension of the 2-D vector and performs
-/// rewrites into:
-///   vector.strided_slice from 1-D + vector.insert into 2-D
-class ShapeCastOp2DUpCastRewritePattern
-    : public OpRewritePattern<vector::ShapeCastOp> {
-public:
-  using OpRewritePattern<vector::ShapeCastOp>::OpRewritePattern;
-
-  PatternMatchResult matchAndRewrite(vector::ShapeCastOp op,
-                                     PatternRewriter &rewriter) const override {
-    auto sourceVectorType = op.getSourceVectorType();
-    auto resultVectorType = op.getResultVectorType();
-    if (sourceVectorType.getRank() != 1 || resultVectorType.getRank() != 2)
-      return matchFailure();
-
-    auto loc = op.getLoc();
-    auto elemType = sourceVectorType.getElementType();
-    Value zero = rewriter.create<ConstantOp>(loc, elemType,
-                                             rewriter.getZeroAttr(elemType));
-    Value desc = rewriter.create<SplatOp>(loc, resultVectorType, zero);
-    unsigned mostMinorVectorSize = resultVectorType.getShape()[1];
-    for (int64_t i = 0, e = resultVectorType.getShape().front(); i != e; ++i) {
-      Value vec = rewriter.create<vector::StridedSliceOp>(
-          loc, op.source(), /*offsets=*/i * mostMinorVectorSize,
-          /*sizes=*/mostMinorVectorSize,
-          /*strides=*/1);
-      desc = rewriter.create<vector::InsertOp>(loc, vec, desc, i);
-    }
-    rewriter.replaceOp(op, desc);
-    return matchSuccess();
-  }
-};
-
-} // namespace
-
-// TODO(andydavis) Add pattern to rewrite ExtractSlices(ConstantMaskOp).
-// TODO(andydavis) Add this as DRR pattern.
-void mlir::vector::populateVectorToVectorTransformationPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *context) {
-  patterns.insert<ShapeCastOpDecomposer, ShapeCastOpFolder, SplitTransferReadOp,
-                  SplitTransferWriteOp, TupleGetFolderOp>(context);
-}
-
-void mlir::vector::populateVectorSlicesLoweringPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *context) {
-  patterns.insert<ExtractSlicesOpLowering, InsertSlicesOpLowering>(context);
-}
-
-void mlir::vector::populateVectorContractLoweringPatterns(
-    OwningRewritePatternList &patterns, MLIRContext *context) {
-  patterns.insert<ContractionOpLowering, ShapeCastOp2DDownCastRewritePattern,
-                  ShapeCastOp2DUpCastRewritePattern, OuterProductOpLowering>(
-      context);
-}
diff --git a/mlir/lib/Dialect/VectorOps/VectorUtils.cpp b/mlir/lib/Dialect/VectorOps/VectorUtils.cpp
deleted file mode 100644
index 46a990080a4f2..0000000000000
--- a/mlir/lib/Dialect/VectorOps/VectorUtils.cpp
+++ /dev/null
@@ -1,278 +0,0 @@
-//===- VectorUtils.cpp - MLIR Utilities for VectorOps   ------------------===//
-//
-// Part of the MLIR Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements utility methods for working with the VectorOps dialect.
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/Dialect/VectorOps/VectorUtils.h"
-#include "mlir/Analysis/LoopAnalysis.h"
-#include "mlir/Dialect/Affine/IR/AffineOps.h"
-#include "mlir/Dialect/StandardOps/IR/Ops.h"
-#include "mlir/Dialect/VectorOps/VectorOps.h"
-#include "mlir/IR/Builders.h"
-#include "mlir/IR/IntegerSet.h"
-#include "mlir/IR/Operation.h"
-#include "mlir/Support/Functional.h"
-#include "mlir/Support/LLVM.h"
-#include "mlir/Support/MathExtras.h"
-#include "mlir/Support/STLExtras.h"
-
-#include "llvm/ADT/DenseSet.h"
-#include "llvm/ADT/SetVector.h"
-
-using llvm::SetVector;
-
-namespace mlir {
-
-SmallVector<int64_t, 4> computeStrides(ArrayRef<int64_t> shape,
-                                       ArrayRef<int64_t> sizes) {
-  int64_t rank = shape.size();
-  // Compute the count for each dimension.
-  SmallVector<int64_t, 4> sliceDimCounts(rank);
-  for (int64_t r = 0; r < rank; ++r)
-    sliceDimCounts[r] = ceilDiv(shape[r], sizes[r]);
-  // Use that to compute the slice stride for each dimension.
-  SmallVector<int64_t, 4> sliceStrides(rank);
-  sliceStrides[rank - 1] = 1;
-  for (int64_t r = rank - 2; r >= 0; --r)
-    sliceStrides[r] = sliceStrides[r + 1] * sliceDimCounts[r + 1];
-  return sliceStrides;
-}
-
-SmallVector<int64_t, 4> delinearize(ArrayRef<int64_t> sliceStrides,
-                                    int64_t index) {
-  int64_t rank = sliceStrides.size();
-  SmallVector<int64_t, 4> vectorOffsets(rank);
-  for (int64_t r = 0; r < rank; ++r) {
-    assert(sliceStrides[r] > 0);
-    vectorOffsets[r] = index / sliceStrides[r];
-    index %= sliceStrides[r];
-  }
-  return vectorOffsets;
-}
-
-SmallVector<int64_t, 4>
-computeElementOffsetsFromVectorSliceOffsets(ArrayRef<int64_t> sizes,
-                                            ArrayRef<int64_t> vectorOffsets) {
-  return functional::zipMap([](int64_t v1, int64_t v2) { return v1 * v2; },
-                            vectorOffsets, sizes);
-}
-
-SmallVector<int64_t, 4> computeSliceSizes(ArrayRef<int64_t> shape,
-                                          ArrayRef<int64_t> sizes,
-                                          ArrayRef<int64_t> elementOffsets) {
-  int64_t rank = shape.size();
-  SmallVector<int64_t, 4> sliceSizes(rank);
-  for (unsigned r = 0; r < rank; ++r)
-    sliceSizes[r] = std::min(sizes[r], shape[r] - elementOffsets[r]);
-  return sliceSizes;
-}
-
-Optional<SmallVector<int64_t, 4>> shapeRatio(ArrayRef<int64_t> superShape,
-                                             ArrayRef<int64_t> subShape) {
-  if (superShape.size() < subShape.size()) {
-    return Optional<SmallVector<int64_t, 4>>();
-  }
-
-  // Starting from the end, compute the integer divisors.
-  // Set the boolean `divides` if integral division is not possible.
-  std::vector<int64_t> result;
-  result.reserve(superShape.size());
-  bool divides = true;
-  auto divide = [&divides, &result](int superSize, int subSize) {
-    assert(superSize > 0 && "superSize must be > 0");
-    assert(subSize > 0 && "subSize must be > 0");
-    divides &= (superSize % subSize == 0);
-    result.push_back(superSize / subSize);
-  };
-  functional::zipApply(
-      divide, SmallVector<int64_t, 8>{superShape.rbegin(), superShape.rend()},
-      SmallVector<int64_t, 8>{subShape.rbegin(), subShape.rend()});
-
-  // If integral division does not occur, return and let the caller decide.
-  if (!divides) {
-    return None;
-  }
-
-  // At this point we computed the ratio (in reverse) for the common
-  // size. Fill with the remaining entries from the super-vector shape (still in
-  // reverse).
-  int commonSize = subShape.size();
-  std::copy(superShape.rbegin() + commonSize, superShape.rend(),
-            std::back_inserter(result));
-
-  assert(result.size() == superShape.size() &&
-         "super to sub shape ratio is not of the same size as the super rank");
-
-  // Reverse again to get it back in the proper order and return.
-  return SmallVector<int64_t, 4>{result.rbegin(), result.rend()};
-}
-
-Optional<SmallVector<int64_t, 4>> shapeRatio(VectorType superVectorType,
-                                             VectorType subVectorType) {
-  assert(superVectorType.getElementType() == subVectorType.getElementType() &&
-         "vector types must be of the same elemental type");
-  return shapeRatio(superVectorType.getShape(), subVectorType.getShape());
-}
-
-/// Constructs a permutation map from memref indices to vector dimension.
-///
-/// The implementation uses the knowledge of the mapping of enclosing loop to
-/// vector dimension. `enclosingLoopToVectorDim` carries this information as a
-/// map with:
-///   - keys representing "vectorized enclosing loops";
-///   - values representing the corresponding vector dimension.
-/// The algorithm traverses "vectorized enclosing loops" and extracts the
-/// at-most-one MemRef index that is invariant along said loop. This index is
-/// guaranteed to be at most one by construction: otherwise the MemRef is not
-/// vectorizable.
-/// If this invariant index is found, it is added to the permutation_map at the
-/// proper vector dimension.
-/// If no index is found to be invariant, 0 is added to the permutation_map and
-/// corresponds to a vector broadcast along that dimension.
-///
-/// Returns an empty AffineMap if `enclosingLoopToVectorDim` is empty,
-/// signalling that no permutation map can be constructed given
-/// `enclosingLoopToVectorDim`.
-///
-/// Examples can be found in the documentation of `makePermutationMap`, in the
-/// header file.
-static AffineMap makePermutationMap(
-    ArrayRef<Value> indices,
-    const DenseMap<Operation *, unsigned> &enclosingLoopToVectorDim) {
-  if (enclosingLoopToVectorDim.empty())
-    return AffineMap();
-  MLIRContext *context =
-      enclosingLoopToVectorDim.begin()->getFirst()->getContext();
-  using functional::makePtrDynCaster;
-  using functional::map;
-  SmallVector<AffineExpr, 4> perm(enclosingLoopToVectorDim.size(),
-                                  getAffineConstantExpr(0, context));
-
-  for (auto kvp : enclosingLoopToVectorDim) {
-    assert(kvp.second < perm.size());
-    auto invariants = getInvariantAccesses(
-        cast<AffineForOp>(kvp.first).getInductionVar(), indices);
-    unsigned numIndices = indices.size();
-    unsigned countInvariantIndices = 0;
-    for (unsigned dim = 0; dim < numIndices; ++dim) {
-      if (!invariants.count(indices[dim])) {
-        assert(perm[kvp.second] == getAffineConstantExpr(0, context) &&
-               "permutationMap already has an entry along dim");
-        perm[kvp.second] = getAffineDimExpr(dim, context);
-      } else {
-        ++countInvariantIndices;
-      }
-    }
-    assert((countInvariantIndices == numIndices ||
-            countInvariantIndices == numIndices - 1) &&
-           "Vectorization prerequisite violated: at most 1 index may be "
-           "invariant wrt a vectorized loop");
-  }
-  return AffineMap::get(indices.size(), 0, perm);
-}
-
-/// Implementation detail that walks up the parents and records the ones with
-/// the specified type.
-/// TODO(ntv): could also be implemented as a collect parents followed by a
-/// filter and made available outside this file.
-template <typename T>
-static SetVector<Operation *> getParentsOfType(Operation *op) {
-  SetVector<Operation *> res;
-  auto *current = op;
-  while (auto *parent = current->getParentOp()) {
-    if (auto typedParent = dyn_cast<T>(parent)) {
-      assert(res.count(parent) == 0 && "Already inserted");
-      res.insert(parent);
-    }
-    current = parent;
-  }
-  return res;
-}
-
-/// Returns the enclosing AffineForOp, from closest to farthest.
-static SetVector<Operation *> getEnclosingforOps(Operation *op) {
-  return getParentsOfType<AffineForOp>(op);
-}
-
-AffineMap
-makePermutationMap(Operation *op, ArrayRef<Value> indices,
-                   const DenseMap<Operation *, unsigned> &loopToVectorDim) {
-  DenseMap<Operation *, unsigned> enclosingLoopToVectorDim;
-  auto enclosingLoops = getEnclosingforOps(op);
-  for (auto *forInst : enclosingLoops) {
-    auto it = loopToVectorDim.find(forInst);
-    if (it != loopToVectorDim.end()) {
-      enclosingLoopToVectorDim.insert(*it);
-    }
-  }
-  return makePermutationMap(indices, enclosingLoopToVectorDim);
-}
-
-bool matcher::operatesOnSuperVectorsOf(Operation &op,
-                                       VectorType subVectorType) {
-  // First, extract the vector type and distinguish between:
-  //   a. ops that *must* lower a super-vector (i.e. vector.transfer_read,
-  //      vector.transfer_write); and
-  //   b. ops that *may* lower a super-vector (all other ops).
-  // The ops that *may* lower a super-vector only do so if the super-vector to
-  // sub-vector ratio exists. The ops that *must* lower a super-vector are
-  // explicitly checked for this property.
-  /// TODO(ntv): there should be a single function for all ops to do this so we
-  /// do not have to special case. Maybe a trait, or just a method, unclear atm.
-  bool mustDivide = false;
-  (void)mustDivide;
-  VectorType superVectorType;
-  if (auto read = dyn_cast<vector::TransferReadOp>(op)) {
-    superVectorType = read.getVectorType();
-    mustDivide = true;
-  } else if (auto write = dyn_cast<vector::TransferWriteOp>(op)) {
-    superVectorType = write.getVectorType();
-    mustDivide = true;
-  } else if (op.getNumResults() == 0) {
-    if (!isa<ReturnOp>(op)) {
-      op.emitError("NYI: assuming only return operations can have 0 "
-                   " results at this point");
-    }
-    return false;
-  } else if (op.getNumResults() == 1) {
-    if (auto v = op.getResult(0).getType().dyn_cast<VectorType>()) {
-      superVectorType = v;
-    } else {
-      // Not a vector type.
-      return false;
-    }
-  } else {
-    // Not a vector.transfer and has more than 1 result, fail hard for now to
-    // wake us up when something changes.
-    op.emitError("NYI: operation has more than 1 result");
-    return false;
-  }
-
-  // Get the ratio.
-  auto ratio = shapeRatio(superVectorType, subVectorType);
-
-  // Sanity check.
-  assert((ratio.hasValue() || !mustDivide) &&
-         "vector.transfer operation in which super-vector size is not an"
-         " integer multiple of sub-vector size");
-
-  // This catches cases that are not strictly necessary to have multiplicity but
-  // still aren't divisible by the sub-vector shape.
-  // This could be useful information if we wanted to reshape at the level of
-  // the vector type (but we would have to look at the compute and distinguish
-  // between parallel, reduction and possibly other cases.
-  if (!ratio.hasValue()) {
-    return false;
-  }
-
-  return true;
-}
-
-} // namespace mlir
diff --git a/mlir/lib/ExecutionEngine/CRunnerUtils.cpp b/mlir/lib/ExecutionEngine/CRunnerUtils.cpp
index 7faf3f9d42439..ad5be24378ceb 100644
--- a/mlir/lib/ExecutionEngine/CRunnerUtils.cpp
+++ b/mlir/lib/ExecutionEngine/CRunnerUtils.cpp
@@ -17,6 +17,8 @@
 #include <cinttypes>
 #include <cstdio>
 
+#ifdef MLIR_CRUNNERUTILS_DEFINE_FUNCTIONS
+
 // Small runtime support "lib" for vector.print lowering.
 // By providing elementary printing methods only, this
 // library can remain fully unaware of low-level implementation
@@ -29,3 +31,5 @@ extern "C" void print_open() { fputs("( ", stdout); }
 extern "C" void print_close() { fputs(" )", stdout); }
 extern "C" void print_comma() { fputs(", ", stdout); }
 extern "C" void print_newline() { fputc('\n', stdout); }
+
+#endif
diff --git a/mlir/lib/ExecutionEngine/RunnerUtils.cpp b/mlir/lib/ExecutionEngine/RunnerUtils.cpp
index af75b10763195..7991eca61994f 100644
--- a/mlir/lib/ExecutionEngine/RunnerUtils.cpp
+++ b/mlir/lib/ExecutionEngine/RunnerUtils.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file imlpements basic functions to debug structured MLIR types at
+// This file implements basic functions to debug structured MLIR types at
 // runtime. Entities in this file may not be compatible with targets without a
 // C++ runtime. These may be progressively migrated to CRunnerUtils.cpp over
 // time.
@@ -15,6 +15,10 @@
 
 #include "mlir/ExecutionEngine/RunnerUtils.h"
 
+#ifndef _WIN32
+#include <sys/time.h>
+#endif // _WIN32
+
 extern "C" void _mlir_ciface_print_memref_vector_4x4xf32(
     StridedMemRefType<Vector2D<4, 4, float>, 2> *M) {
   impl::printMemRef(*M);
@@ -103,3 +107,22 @@ extern "C" void
 _mlir_ciface_print_memref_4d_f32(StridedMemRefType<float, 4> *M) {
   impl::printMemRef(*M);
 }
+
+/// Prints GFLOPS rating.
+extern "C" void print_flops(double flops) {
+  fprintf(stderr, "%lf GFLOPS\n", flops / 1.0E9);
+}
+
+/// Returns the number of seconds since Epoch 1970-01-01 00:00:00 +0000 (UTC).
+extern "C" double rtclock() {
+#ifndef _WIN32
+  struct timeval tp;
+  int stat = gettimeofday(&tp, NULL);
+  if (stat != 0)
+    fprintf(stderr, "Error returning time from gettimeofday: %d\n", stat);
+  return (tp.tv_sec + tp.tv_usec * 1.0e-6);
+#else
+  fprintf(stderr, "Timing utility not implemented on Windows\n");
+  return 0.0;
+#endif // _WIN32
+}
diff --git a/mlir/lib/IR/AffineMap.cpp b/mlir/lib/IR/AffineMap.cpp
index 00feb08db89ad..7e5036e37bab1 100644
--- a/mlir/lib/IR/AffineMap.cpp
+++ b/mlir/lib/IR/AffineMap.cpp
@@ -8,7 +8,6 @@
 
 #include "mlir/IR/AffineMap.h"
 #include "AffineMapDetail.h"
-#include "mlir/IR/AffineExpr.h"
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/StandardTypes.h"
 #include "mlir/Support/Functional.h"
@@ -326,6 +325,15 @@ AffineMap mlir::simplifyAffineMap(AffineMap map) {
   return AffineMap::get(map.getNumDims(), map.getNumSymbols(), exprs);
 }
 
+AffineMap mlir::removeDuplicateExprs(AffineMap map) {
+  auto results = map.getResults();
+  SmallVector<AffineExpr, 4> uniqueExprs(results.begin(), results.end());
+  uniqueExprs.erase(std::unique(uniqueExprs.begin(), uniqueExprs.end()),
+                    uniqueExprs.end());
+  return AffineMap::get(map.getNumDims(), map.getNumSymbols(), uniqueExprs,
+                        map.getContext());
+}
+
 AffineMap mlir::inversePermutation(AffineMap map) {
   if (map.isEmpty())
     return map;
diff --git a/mlir/lib/IR/AsmPrinter.cpp b/mlir/lib/IR/AsmPrinter.cpp
index bf7905a147819..bc680cfbc9800 100644
--- a/mlir/lib/IR/AsmPrinter.cpp
+++ b/mlir/lib/IR/AsmPrinter.cpp
@@ -898,7 +898,7 @@ class ModulePrinter {
     mlir::interleaveComma(c, os, each_fn);
   }
 
-  /// This enum descripes the different kinds of elision for the type of an
+  /// This enum describes the different kinds of elision for the type of an
   /// attribute when printing it.
   enum class AttrTypeElision {
     /// The type must not be elided,
@@ -2025,7 +2025,7 @@ class OperationPrinter : public ModulePrinter, private OpAsmPrinter {
     state->getSSANameState().shadowRegionArgs(region, namesToUse);
   }
 
-  /// Print the given affine map with the smybol and dimension operands printed
+  /// Print the given affine map with the symbol and dimension operands printed
   /// inline with the map.
   void printAffineMapOfSSAIds(AffineMapAttr mapAttr,
                               ValueRange operands) override;
diff --git a/mlir/lib/IR/Builders.cpp b/mlir/lib/IR/Builders.cpp
index 23536651f9749..c8d5ea6b6ca98 100644
--- a/mlir/lib/IR/Builders.cpp
+++ b/mlir/lib/IR/Builders.cpp
@@ -339,24 +339,28 @@ Operation *OpBuilder::insert(Operation *op) {
   return op;
 }
 
-/// Add new block and set the insertion point to the end of it. The block is
-/// inserted at the provided insertion point of 'parent'.
-Block *OpBuilder::createBlock(Region *parent, Region::iterator insertPt) {
+/// Add new block with 'argTypes' arguments and set the insertion point to the
+/// end of it. The block is inserted at the provided insertion point of
+/// 'parent'.
+Block *OpBuilder::createBlock(Region *parent, Region::iterator insertPt,
+                              TypeRange argTypes) {
   assert(parent && "expected valid parent region");
   if (insertPt == Region::iterator())
     insertPt = parent->end();
 
   Block *b = new Block();
+  b->addArguments(argTypes);
   parent->getBlocks().insert(insertPt, b);
   setInsertionPointToEnd(b);
   return b;
 }
 
-/// Add new block and set the insertion point to the end of it.  The block is
-/// placed before 'insertBefore'.
-Block *OpBuilder::createBlock(Block *insertBefore) {
+/// Add new block with 'argTypes' arguments and set the insertion point to the
+/// end of it.  The block is placed before 'insertBefore'.
+Block *OpBuilder::createBlock(Block *insertBefore, TypeRange argTypes) {
   assert(insertBefore && "expected valid insertion block");
-  return createBlock(insertBefore->getParent(), Region::iterator(insertBefore));
+  return createBlock(insertBefore->getParent(), Region::iterator(insertBefore),
+                     argTypes);
 }
 
 /// Create an operation given the fields represented as an OperationState.
diff --git a/mlir/lib/IR/Diagnostics.cpp b/mlir/lib/IR/Diagnostics.cpp
index 3e8999fcf8bba..1acb7e7f5931b 100644
--- a/mlir/lib/IR/Diagnostics.cpp
+++ b/mlir/lib/IR/Diagnostics.cpp
@@ -435,7 +435,8 @@ SourceMgrDiagnosticHandler::SourceMgrDiagnosticHandler(llvm::SourceMgr &mgr,
 SourceMgrDiagnosticHandler::~SourceMgrDiagnosticHandler() {}
 
 void SourceMgrDiagnosticHandler::emitDiagnostic(Location loc, Twine message,
-                                                DiagnosticSeverity kind) {
+                                                DiagnosticSeverity kind,
+                                                bool displaySourceLine) {
   // Extract a file location from this loc.
   auto fileLoc = getFileLineColLoc(loc);
 
@@ -449,41 +450,52 @@ void SourceMgrDiagnosticHandler::emitDiagnostic(Location loc, Twine message,
     return mgr.PrintMessage(os, llvm::SMLoc(), getDiagKind(kind), strOS.str());
   }
 
-  // Otherwise, try to convert the file location to an SMLoc.
-  auto smloc = convertLocToSMLoc(*fileLoc);
-  if (smloc.isValid())
-    return mgr.PrintMessage(os, smloc, getDiagKind(kind), message);
+  // Otherwise if we are displaying the source line, try to convert the file
+  // location to an SMLoc.
+  if (displaySourceLine) {
+    auto smloc = convertLocToSMLoc(*fileLoc);
+    if (smloc.isValid())
+      return mgr.PrintMessage(os, smloc, getDiagKind(kind), message);
+  }
 
   // If the conversion was unsuccessful, create a diagnostic with the file
-  // information.
-  llvm::SMDiagnostic diag(fileLoc->getFilename(), getDiagKind(kind),
-                          message.str());
+  // information. We manually combine the line and column to avoid asserts in
+  // the constructor of SMDiagnostic that takes a location.
+  std::string locStr;
+  llvm::raw_string_ostream locOS(locStr);
+  locOS << fileLoc->getFilename() << ":" << fileLoc->getLine() << ":"
+        << fileLoc->getColumn();
+  llvm::SMDiagnostic diag(locOS.str(), getDiagKind(kind), message.str());
   diag.print(nullptr, os);
 }
 
 /// Emit the given diagnostic with the held source manager.
 void SourceMgrDiagnosticHandler::emitDiagnostic(Diagnostic &diag) {
   // Emit the diagnostic.
-  auto loc = diag.getLocation();
+  Location loc = diag.getLocation();
   emitDiagnostic(loc, diag.str(), diag.getSeverity());
 
   // If the diagnostic location was a call site location, then print the call
   // stack as well.
   if (auto callLoc = getCallSiteLoc(loc)) {
     // Print the call stack while valid, or until the limit is reached.
-    Location callerLoc = callLoc->getCaller();
+    loc = callLoc->getCaller();
     for (unsigned curDepth = 0; curDepth < callStackLimit; ++curDepth) {
-      emitDiagnostic(callerLoc, "called from", DiagnosticSeverity::Note);
-      callLoc = getCallSiteLoc(callerLoc);
-      if (!callLoc)
+      emitDiagnostic(loc, "called from", DiagnosticSeverity::Note);
+      if ((callLoc = getCallSiteLoc(loc)))
+        loc = callLoc->getCaller();
+      else
         break;
-      callerLoc = callLoc->getCaller();
     }
   }
 
-  // Emit each of the notes.
-  for (auto &note : diag.getNotes())
-    emitDiagnostic(note.getLocation(), note.str(), note.getSeverity());
+  // Emit each of the notes. Only display the source code if the location is
+  // different from the previous location.
+  for (auto &note : diag.getNotes()) {
+    emitDiagnostic(note.getLocation(), note.str(), note.getSeverity(),
+                   /*displaySourceLine=*/loc != note.getLocation());
+    loc = note.getLocation();
+  }
 }
 
 /// Get a memory buffer for the given file, or nullptr if one is not found.
diff --git a/mlir/lib/IR/Function.cpp b/mlir/lib/IR/Function.cpp
index 320dea4d5dc24..1933c4df18bd8 100644
--- a/mlir/lib/IR/Function.cpp
+++ b/mlir/lib/IR/Function.cpp
@@ -9,12 +9,7 @@
 #include "mlir/IR/Function.h"
 #include "mlir/IR/BlockAndValueMapping.h"
 #include "mlir/IR/Builders.h"
-#include "mlir/IR/Diagnostics.h"
-#include "mlir/IR/Dialect.h"
 #include "mlir/IR/FunctionImplementation.h"
-#include "mlir/IR/MLIRContext.h"
-#include "mlir/IR/Module.h"
-#include "mlir/IR/OpImplementation.h"
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SmallString.h"
diff --git a/mlir/lib/IR/IntegerSet.cpp b/mlir/lib/IR/IntegerSet.cpp
index 2e848b93355ae..bb1f2920a492f 100644
--- a/mlir/lib/IR/IntegerSet.cpp
+++ b/mlir/lib/IR/IntegerSet.cpp
@@ -8,7 +8,6 @@
 
 #include "mlir/IR/IntegerSet.h"
 #include "IntegerSetDetail.h"
-#include "mlir/IR/AffineExpr.h"
 
 using namespace mlir;
 using namespace mlir::detail;
diff --git a/mlir/lib/IR/MLIRContext.cpp b/mlir/lib/IR/MLIRContext.cpp
index 198becaed5cc1..3875b57775bcb 100644
--- a/mlir/lib/IR/MLIRContext.cpp
+++ b/mlir/lib/IR/MLIRContext.cpp
@@ -161,6 +161,15 @@ class MLIRContextImpl {
   //===--------------------------------------------------------------------===//
   DiagnosticEngine diagEngine;
 
+  //===--------------------------------------------------------------------===//
+  // Options
+  //===--------------------------------------------------------------------===//
+
+  /// In most cases, creating operation in unregistered dialect is not desired
+  /// and indicate a misconfiguration of the compiler. This option enables to
+  /// detect such use cases
+  bool allowUnregisteredDialects = false;
+
   //===--------------------------------------------------------------------===//
   // Other
   //===--------------------------------------------------------------------===//
@@ -349,6 +358,14 @@ void Dialect::registerDialect(MLIRContext *context) {
   impl.dialects.insert(insertPt, std::move(dialect));
 }
 
+bool MLIRContext::allowsUnregisteredDialects() {
+  return impl->allowUnregisteredDialects;
+}
+
+void MLIRContext::allowUnregisteredDialects(bool allowing) {
+  impl->allowUnregisteredDialects = allowing;
+}
+
 /// Return information about all registered operations.  This isn't very
 /// efficient, typically you should ask the operations about their properties
 /// directly.
@@ -458,7 +475,7 @@ static Dialect &lookupDialectForSymbol(MLIRContext *ctx,
   return *it->second;
 }
 
-/// Returns the storage unqiuer used for constructing type storage instances.
+/// Returns the storage uniquer used for constructing type storage instances.
 /// This should not be used directly.
 StorageUniquer &MLIRContext::getTypeUniquer() { return getImpl().typeUniquer; }
 
@@ -613,7 +630,7 @@ AffineMap AffineMap::get(MLIRContext *context) {
 
 AffineMap AffineMap::get(unsigned dimCount, unsigned symbolCount,
                          MLIRContext *context) {
-  return getImpl(dimCount, /*symbolCount=*/0, /*results=*/{}, context);
+  return getImpl(dimCount, symbolCount, /*results=*/{}, context);
 }
 
 AffineMap AffineMap::get(unsigned dimCount, unsigned symbolCount,
@@ -623,6 +640,11 @@ AffineMap AffineMap::get(unsigned dimCount, unsigned symbolCount,
   return getImpl(dimCount, symbolCount, results, results[0].getContext());
 }
 
+AffineMap AffineMap::get(unsigned dimCount, unsigned symbolCount,
+                         ArrayRef<AffineExpr> results, MLIRContext *context) {
+  return getImpl(dimCount, symbolCount, results, context);
+}
+
 //===----------------------------------------------------------------------===//
 // Integer Sets: these are allocated into the bump pointer, and are immutable.
 // Unlike AffineMap's, these are uniqued only if they are small.
diff --git a/mlir/lib/IR/Operation.cpp b/mlir/lib/IR/Operation.cpp
index 0d207fd1a6aa3..58ebf29ceebd3 100644
--- a/mlir/lib/IR/Operation.cpp
+++ b/mlir/lib/IR/Operation.cpp
@@ -8,11 +8,7 @@
 
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/BlockAndValueMapping.h"
-#include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/Dialect.h"
-#include "mlir/IR/Function.h"
-#include "mlir/IR/MLIRContext.h"
-#include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/StandardTypes.h"
diff --git a/mlir/lib/Parser/Parser.cpp b/mlir/lib/Parser/Parser.cpp
index a29b34b570d0d..e68867b1e44e0 100644
--- a/mlir/lib/Parser/Parser.cpp
+++ b/mlir/lib/Parser/Parser.cpp
@@ -1759,6 +1759,33 @@ static Optional<APFloat> buildHexadecimalFloatLiteral(Parser *p, FloatType type,
   return APFloat(type.getFloatSemantics(), apInt);
 }
 
+/// Construct an APint from a parsed value, a known attribute type and
+/// sign.
+static Optional<APInt> buildAttributeAPInt(Type type, bool isNegative,
+                                           uint64_t value) {
+  // We have the integer literal as an uint64_t in val, now convert it into an
+  // APInt and check that we don't overflow.
+  int width = type.isIndex() ? 64 : type.getIntOrFloatBitWidth();
+  APInt apInt(width, value, isNegative);
+  if (apInt != value)
+    return llvm::None;
+
+  if (isNegative) {
+    // The value is negative, we have an overflow if the sign bit is not set
+    // in the negated apInt.
+    apInt.negate();
+    if (!apInt.isSignBitSet())
+      return llvm::None;
+  } else if ((type.isSignedInteger() || type.isIndex()) &&
+             apInt.isSignBitSet()) {
+    // The value is a positive signed integer or index,
+    // we have an overflow if the sign bit is set.
+    return llvm::None;
+  }
+
+  return apInt;
+}
+
 /// Parse a decimal or a hexadecimal literal, which can be either an integer
 /// or a float attribute.
 Attribute Parser::parseDecOrHexAttr(Type type, bool isNegative) {
@@ -1809,19 +1836,12 @@ Attribute Parser::parseDecOrHexAttr(Type type, bool isNegative) {
     return nullptr;
   }
 
-  // Parse the integer literal.
-  int width = type.isIndex() ? 64 : type.getIntOrFloatBitWidth();
-  APInt apInt(width, *val, isNegative);
-  if (apInt != *val)
-    return emitError(loc, "integer constant out of range for attribute"),
-           nullptr;
+  Optional<APInt> apInt = buildAttributeAPInt(type, isNegative, *val);
 
-  // Otherwise construct an integer attribute.
-  if (isNegative ? (int64_t)-val.getValue() >= 0 : (int64_t)val.getValue() < 0)
+  if (!apInt)
     return emitError(loc, "integer constant out of range for attribute"),
            nullptr;
-
-  return builder.getIntegerAttr(type, isNegative ? -apInt : apInt);
+  return builder.getIntegerAttr(type, *apInt);
 }
 
 /// Parse elements values stored within a hex etring. On success, the values are
@@ -2038,16 +2058,15 @@ DenseElementsAttr TensorLiteralParser::getIntAttr(llvm::SMLoc loc,
 
     // Create APInt values for each element with the correct bitwidth.
     auto val = token.getUInt64IntegerValue();
-    if (!val.hasValue() || (isNegative ? (int64_t)-val.getValue() >= 0
-                                       : (int64_t)val.getValue() < 0)) {
+    if (!val.hasValue()) {
       p.emitError(tokenLoc, "integer constant out of range for attribute");
       return nullptr;
     }
-    APInt apInt(eltTy.getWidth(), val.getValue(), isNegative);
-    if (apInt != val.getValue())
+    Optional<APInt> apInt = buildAttributeAPInt(eltTy, isNegative, *val);
+    if (!apInt)
       return (p.emitError(tokenLoc, "integer constant out of range for type"),
               nullptr);
-    intElements.push_back(isNegative ? -apInt : apInt);
+    intElements.push_back(*apInt);
   }
 
   return DenseElementsAttr::get(type, intElements);
@@ -2121,7 +2140,7 @@ DenseElementsAttr TensorLiteralParser::getHexAttr(llvm::SMLoc loc,
   if (parseElementAttrHexValues(p, hexStorage.getValue(), data))
     return nullptr;
 
-  // Check that the size of the hex data correpsonds to the size of the type, or
+  // Check that the size of the hex data corresponds to the size of the type, or
   // a splat of the type.
   // TODO: bf16 is currently stored as a double, this should be removed when
   // APFloat properly supports it.
@@ -4234,6 +4253,13 @@ class CustomOpAsmParser : public OpAsmParser {
     return success();
   }
 
+  /// Parse a single operand if present.
+  OptionalParseResult parseOptionalOperand(OperandType &result) override {
+    if (parser.getToken().is(Token::percent_identifier))
+      return parseOperand(result);
+    return llvm::None;
+  }
+
   /// Parse zero or more SSA comma-separated operand references with a specified
   /// surrounding delimiter, and an optional required operand count.
   ParseResult parseOperandList(SmallVectorImpl<OperandType> &result,
diff --git a/mlir/lib/Pass/PassRegistry.cpp b/mlir/lib/Pass/PassRegistry.cpp
index 0ecf0266eb5a7..74ac36ae8c59c 100644
--- a/mlir/lib/Pass/PassRegistry.cpp
+++ b/mlir/lib/Pass/PassRegistry.cpp
@@ -96,8 +96,9 @@ PassInfo::PassInfo(StringRef arg, StringRef description, const PassID *passID,
           }) {}
 
 void mlir::registerPass(StringRef arg, StringRef description,
-                        const PassID *passID,
                         const PassAllocatorFunction &function) {
+  // TODO: We should use the 'arg' as the lookup key instead of the pass id.
+  const PassID *passID = function()->getPassID();
   PassInfo passInfo(arg, description, passID, function);
   bool inserted = passRegistry->try_emplace(passID, passInfo).second;
   assert(inserted && "Pass registered multiple times");
diff --git a/mlir/lib/Quantizer/CMakeLists.txt b/mlir/lib/Quantizer/CMakeLists.txt
index 9dc0a794e3810..15459304d5357 100644
--- a/mlir/lib/Quantizer/CMakeLists.txt
+++ b/mlir/lib/Quantizer/CMakeLists.txt
@@ -14,7 +14,7 @@ add_mlir_library(MLIRQuantizerSupport
 target_link_libraries(MLIRQuantizerSupport
   PUBLIC
   MLIRIR
-  MLIRQuantOps
+  MLIRQuant
   MLIRSupport
   MLIRStandardOps
   LLVMSupport
@@ -34,7 +34,7 @@ target_link_libraries(MLIRQuantizerFxpMathConfig
   PUBLIC
   MLIRIR
   MLIRFxpMathOps
-  MLIRQuantOps
+  MLIRQuant
   MLIRQuantizerSupport
   MLIRStandardOps
   MLIRSupport
@@ -48,13 +48,16 @@ add_mlir_library(MLIRQuantizerTransforms
   Transforms/RemoveInstrumentationPass.cpp
 
   ADDITIONAL_HEADER_DIRS
+
+  DEPENDS
+  MLIRQuantizerPassIncGen
   )
 target_link_libraries(MLIRQuantizerTransforms
   PUBLIC
   MLIRIR
   MLIRQuantizerFxpMathConfig
   MLIRQuantizerSupport
-  MLIRQuantOps
+  MLIRQuant
   MLIRPass
   LLVMSupport
   )
diff --git a/mlir/lib/Quantizer/Transforms/AddDefaultStatsTestPass.cpp b/mlir/lib/Quantizer/Transforms/AddDefaultStatsTestPass.cpp
index 7a9627dd144e8..238b0e850b881 100644
--- a/mlir/lib/Quantizer/Transforms/AddDefaultStatsTestPass.cpp
+++ b/mlir/lib/Quantizer/Transforms/AddDefaultStatsTestPass.cpp
@@ -29,9 +29,12 @@ using namespace mlir::quantizer;
 using namespace mlir::quant;
 
 namespace {
-
 class AddDefaultStatsPass : public FunctionPass<AddDefaultStatsPass> {
 public:
+/// Include the generated pass utilities.
+#define GEN_PASS_QuantizerAddDefaultStats
+#include "mlir/Quantizer/Transforms/Passes.h.inc"
+
   AddDefaultStatsPass() = default;
   AddDefaultStatsPass(SolverContext &solverContext,
                       const TargetConfiguration &config)
@@ -113,8 +116,3 @@ std::unique_ptr<OpPassBase<FuncOp>>
 mlir::quantizer::createAddDefaultStatsPass() {
   return std::make_unique<AddDefaultStatsPass>();
 }
-
-static PassRegistration<AddDefaultStatsPass> pass(
-    "quantizer-add-default-stats-test",
-    "Adds default (dummy) statistics to all ops that can benefit from "
-    "runtime statistics. This is meant to help in early stage bootstrapping.");
diff --git a/mlir/lib/Quantizer/Transforms/InferQuantizedTypesPass.cpp b/mlir/lib/Quantizer/Transforms/InferQuantizedTypesPass.cpp
index 1876b33b84214..49d98ef695c08 100644
--- a/mlir/lib/Quantizer/Transforms/InferQuantizedTypesPass.cpp
+++ b/mlir/lib/Quantizer/Transforms/InferQuantizedTypesPass.cpp
@@ -71,13 +71,17 @@ struct DOTGraphTraits<const CAGSlice *>
 } // end namespace llvm
 
 namespace {
-
 class InferQuantizedTypesPass : public ModulePass<InferQuantizedTypesPass> {
 public:
+/// Include the generated pass utilities.
+#define GEN_PASS_QuantizerInferQuantizedTypes
+#include "mlir/Quantizer/Transforms/Passes.h.inc"
+
   InferQuantizedTypesPass() = default;
   InferQuantizedTypesPass(SolverContext &solverContext,
                           const TargetConfiguration &config)
       : explicitSolverContext(&solverContext), explicitConfig(&config) {}
+
   void runOnModule() override;
   void runWithConfig(SolverContext &solverContext,
                      const TargetConfiguration &config);
@@ -282,12 +286,7 @@ mlir::quantizer::createInferQuantizedTypesPass(
     SolverContext &solverContext, const TargetConfiguration &config) {
   return std::make_unique<InferQuantizedTypesPass>(solverContext, config);
 }
-void mlir::quantizer::registerInferQuantizedTypesPass() {
-  // Do nothing, this will be enough to force link this file and the static
-  // registration will kick-in. This is temporary while we're refactoring pass
-  // registration to move away from static constructors.
+std::unique_ptr<OpPassBase<ModuleOp>>
+mlir::quantizer::createInferQuantizedTypesPass() {
+  return std::make_unique<InferQuantizedTypesPass>();
 }
-
-static PassRegistration<InferQuantizedTypesPass>
-    pass("quantizer-infer-quantized-types",
-         "Infers quantized types for a module");
diff --git a/mlir/lib/Quantizer/Transforms/RemoveInstrumentationPass.cpp b/mlir/lib/Quantizer/Transforms/RemoveInstrumentationPass.cpp
index 0112753c49fc7..6c348d3922541 100644
--- a/mlir/lib/Quantizer/Transforms/RemoveInstrumentationPass.cpp
+++ b/mlir/lib/Quantizer/Transforms/RemoveInstrumentationPass.cpp
@@ -23,9 +23,12 @@ using namespace mlir::quantizer;
 using namespace mlir::quant;
 
 namespace {
-
 class RemoveInstrumentationPass
     : public FunctionPass<RemoveInstrumentationPass> {
+/// Include the generated pass utilities.
+#define GEN_PASS_QuantizerRemoveInstrumentation
+#include "mlir/Quantizer/Transforms/Passes.h.inc"
+
   void runOnFunction() override;
 };
 
@@ -61,8 +64,3 @@ std::unique_ptr<OpPassBase<FuncOp>>
 mlir::quantizer::createRemoveInstrumentationPass() {
   return std::make_unique<RemoveInstrumentationPass>();
 }
-
-static PassRegistration<RemoveInstrumentationPass>
-    pass("quantizer-remove-instrumentation",
-         "Removes instrumentation and hints which have no effect on final "
-         "execution");
diff --git a/mlir/lib/Support/MlirOptMain.cpp b/mlir/lib/Support/MlirOptMain.cpp
index 5db824d21ad20..53e336a24b471 100644
--- a/mlir/lib/Support/MlirOptMain.cpp
+++ b/mlir/lib/Support/MlirOptMain.cpp
@@ -67,6 +67,7 @@ static LogicalResult performActions(raw_ostream &os, bool verifyDiagnostics,
 static LogicalResult processBuffer(raw_ostream &os,
                                    std::unique_ptr<MemoryBuffer> ownedBuffer,
                                    bool verifyDiagnostics, bool verifyPasses,
+                                   bool allowUnregisteredDialects,
                                    const PassPipelineCLParser &passPipeline) {
   // Tell sourceMgr about this buffer, which is what the parser will pick up.
   SourceMgr sourceMgr;
@@ -74,6 +75,7 @@ static LogicalResult processBuffer(raw_ostream &os,
 
   // Parse the input file.
   MLIRContext context;
+  context.allowUnregisteredDialects(allowUnregisteredDialects);
 
   // If we are in verify diagnostics mode then we have a lot of work to do,
   // otherwise just perform the actions without worrying about it.
@@ -100,7 +102,8 @@ LogicalResult mlir::MlirOptMain(raw_ostream &os,
                                 std::unique_ptr<MemoryBuffer> buffer,
                                 const PassPipelineCLParser &passPipeline,
                                 bool splitInputFile, bool verifyDiagnostics,
-                                bool verifyPasses) {
+                                bool verifyPasses,
+                                bool allowUnregisteredDialects) {
   // The split-input-file mode is a very specific mode that slices the file
   // up into small pieces and checks each independently.
   if (splitInputFile)
@@ -108,10 +111,11 @@ LogicalResult mlir::MlirOptMain(raw_ostream &os,
         std::move(buffer),
         [&](std::unique_ptr<MemoryBuffer> chunkBuffer, raw_ostream &os) {
           return processBuffer(os, std::move(chunkBuffer), verifyDiagnostics,
-                               verifyPasses, passPipeline);
+                               verifyPasses, allowUnregisteredDialects,
+                               passPipeline);
         },
         os);
 
   return processBuffer(os, std::move(buffer), verifyDiagnostics, verifyPasses,
-                       passPipeline);
+                       allowUnregisteredDialects, passPipeline);
 }
diff --git a/mlir/lib/TableGen/CMakeLists.txt b/mlir/lib/TableGen/CMakeLists.txt
index 4c6ac720f0ea6..08384657f94f6 100644
--- a/mlir/lib/TableGen/CMakeLists.txt
+++ b/mlir/lib/TableGen/CMakeLists.txt
@@ -8,6 +8,7 @@ add_llvm_library(LLVMMLIRTableGen
   OpClass.cpp
   OpInterfaces.cpp
   OpTrait.cpp
+  Pass.cpp
   Pattern.cpp
   Predicate.cpp
   SideEffects.cpp
diff --git a/mlir/lib/TableGen/Pass.cpp b/mlir/lib/TableGen/Pass.cpp
new file mode 100644
index 0000000000000..63f31c99bc90d
--- /dev/null
+++ b/mlir/lib/TableGen/Pass.cpp
@@ -0,0 +1,90 @@
+//===- Pass.cpp - Pass related classes ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/TableGen/Pass.h"
+#include "llvm/TableGen/Record.h"
+
+using namespace mlir;
+using namespace mlir::tblgen;
+
+//===----------------------------------------------------------------------===//
+// PassOption
+//===----------------------------------------------------------------------===//
+
+StringRef PassOption::getCppVariableName() const {
+  return def->getValueAsString("cppName");
+}
+
+StringRef PassOption::getArgument() const {
+  return def->getValueAsString("argument");
+}
+
+StringRef PassOption::getType() const { return def->getValueAsString("type"); }
+
+Optional<StringRef> PassOption::getDefaultValue() const {
+  StringRef defaultVal = def->getValueAsString("defaultValue");
+  return defaultVal.empty() ? Optional<StringRef>() : defaultVal;
+}
+
+StringRef PassOption::getDescription() const {
+  return def->getValueAsString("description");
+}
+
+Optional<StringRef> PassOption::getAdditionalFlags() const {
+  StringRef additionalFlags = def->getValueAsString("additionalOptFlags");
+  return additionalFlags.empty() ? Optional<StringRef>() : additionalFlags;
+}
+
+bool PassOption::isListOption() const {
+  return def->isSubClassOf("ListOption");
+}
+
+//===----------------------------------------------------------------------===//
+// PassStatistic
+//===----------------------------------------------------------------------===//
+
+StringRef PassStatistic::getCppVariableName() const {
+  return def->getValueAsString("cppName");
+}
+
+StringRef PassStatistic::getName() const {
+  return def->getValueAsString("name");
+}
+
+StringRef PassStatistic::getDescription() const {
+  return def->getValueAsString("description");
+}
+
+//===----------------------------------------------------------------------===//
+// Pass
+//===----------------------------------------------------------------------===//
+
+Pass::Pass(const llvm::Record *def) : def(def) {
+  for (auto *init : def->getValueAsListOfDefs("options"))
+    options.push_back(PassOption(init));
+  for (auto *init : def->getValueAsListOfDefs("statistics"))
+    statistics.push_back(PassStatistic(init));
+}
+
+StringRef Pass::getArgument() const {
+  return def->getValueAsString("argument");
+}
+
+StringRef Pass::getSummary() const { return def->getValueAsString("summary"); }
+
+StringRef Pass::getDescription() const {
+  return def->getValueAsString("description");
+}
+
+StringRef Pass::getConstructor() const {
+  return def->getValueAsString("constructor");
+}
+
+ArrayRef<PassOption> Pass::getOptions() const { return options; }
+
+ArrayRef<PassStatistic> Pass::getStatistics() const { return statistics; }
diff --git a/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp b/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp
index 8a71762e956db..6f051c049c830 100644
--- a/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp
+++ b/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp
@@ -446,7 +446,7 @@ Value Importer::processValue(llvm::Value *value) {
   // We don't expect to see instructions in dominator order. If we haven't seen
   // this instruction yet, create an unknown op and remap it later.
   if (isa<llvm::Instruction>(value)) {
-    OperationState state(UnknownLoc::get(context), "unknown");
+    OperationState state(UnknownLoc::get(context), "llvm.unknown");
     LLVMType type = processType(value->getType());
     if (!type)
       return nullptr;
diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index 8bc76870e8f70..da58eb7559afd 100644
--- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -92,8 +92,12 @@ llvm::Constant *ModuleTranslation::getLLVMConstant(llvm::Type *llvmType,
     emitError(loc, "struct types are not supported in constants");
     return nullptr;
   }
+  // For integer types, we allow a mismatch in sizes as the index type in
+  // MLIR might have a different size than the index type in the LLVM module.
   if (auto intAttr = attr.dyn_cast<IntegerAttr>())
-    return llvm::ConstantInt::get(llvmType, intAttr.getValue());
+    return llvm::ConstantInt::get(
+        llvmType,
+        intAttr.getValue().sextOrTrunc(llvmType->getIntegerBitWidth()));
   if (auto boolAttr = attr.dyn_cast<BoolAttr>())
     return llvm::ConstantInt::get(llvmType, boolAttr.getValue());
   if (auto floatAttr = attr.dyn_cast<FloatAttr>())
@@ -103,7 +107,7 @@ llvm::Constant *ModuleTranslation::getLLVMConstant(llvm::Type *llvmType,
         functionMapping.lookup(funcAttr.getValue()), llvmType);
   if (auto splatAttr = attr.dyn_cast<SplatElementsAttr>()) {
     auto *sequentialType = cast<llvm::SequentialType>(llvmType);
-    auto elementType = sequentialType->getElementType();
+    auto *elementType = sequentialType->getElementType();
     uint64_t numElements = sequentialType->getNumElements();
     // Splat value is a scalar. Extract it only if the element type is not
     // another sequence type. The recursion terminates because each step removes
@@ -119,7 +123,7 @@ llvm::Constant *ModuleTranslation::getLLVMConstant(llvm::Type *llvmType,
       return llvm::ConstantVector::getSplat(
           llvm::ElementCount(numElements, /*Scalable=*/false), child);
     if (llvmType->isArrayTy()) {
-      auto arrayType = llvm::ArrayType::get(elementType, numElements);
+      auto *arrayType = llvm::ArrayType::get(elementType, numElements);
       SmallVector<llvm::Constant *, 8> constants(numElements, child);
       return llvm::ConstantArray::get(arrayType, constants);
     }
@@ -556,6 +560,83 @@ static llvm::SetVector<Block *> topologicalSort(LLVMFuncOp f) {
   return blocks;
 }
 
+/// Attempts to add an attribute identified by `key`, optionally with the given
+/// `value` to LLVM function `llvmFunc`. Reports errors at `loc` if any. If the
+/// attribute has a kind known to LLVM IR, create the attribute of this kind,
+/// otherwise keep it as a string attribute. Performs additional checks for
+/// attributes known to have or not have a value in order to avoid assertions
+/// inside LLVM upon construction.
+static LogicalResult checkedAddLLVMFnAttribute(Location loc,
+                                               llvm::Function *llvmFunc,
+                                               StringRef key,
+                                               StringRef value = StringRef()) {
+  auto kind = llvm::Attribute::getAttrKindFromName(key);
+  if (kind == llvm::Attribute::None) {
+    llvmFunc->addFnAttr(key, value);
+    return success();
+  }
+
+  if (llvm::Attribute::doesAttrKindHaveArgument(kind)) {
+    if (value.empty())
+      return emitError(loc) << "LLVM attribute '" << key << "' expects a value";
+
+    int result;
+    if (!value.getAsInteger(/*Radix=*/0, result))
+      llvmFunc->addFnAttr(
+          llvm::Attribute::get(llvmFunc->getContext(), kind, result));
+    else
+      llvmFunc->addFnAttr(key, value);
+    return success();
+  }
+
+  if (!value.empty())
+    return emitError(loc) << "LLVM attribute '" << key
+                          << "' does not expect a value, found '" << value
+                          << "'";
+
+  llvmFunc->addFnAttr(kind);
+  return success();
+}
+
+/// Attaches the attributes listed in the given array attribute to `llvmFunc`.
+/// Reports error to `loc` if any and returns immediately. Expects `attributes`
+/// to be an array attribute containing either string attributes, treated as
+/// value-less LLVM attributes, or array attributes containing two string
+/// attributes, with the first string being the name of the corresponding LLVM
+/// attribute and the second string beings its value. Note that even integer
+/// attributes are expected to have their values expressed as strings.
+static LogicalResult
+forwardPassthroughAttributes(Location loc, Optional<ArrayAttr> attributes,
+                             llvm::Function *llvmFunc) {
+  if (!attributes)
+    return success();
+
+  for (Attribute attr : *attributes) {
+    if (auto stringAttr = attr.dyn_cast<StringAttr>()) {
+      if (failed(
+              checkedAddLLVMFnAttribute(loc, llvmFunc, stringAttr.getValue())))
+        return failure();
+      continue;
+    }
+
+    auto arrayAttr = attr.dyn_cast<ArrayAttr>();
+    if (!arrayAttr || arrayAttr.size() != 2)
+      return emitError(loc)
+             << "expected 'passthrough' to contain string or array attributes";
+
+    auto keyAttr = arrayAttr[0].dyn_cast<StringAttr>();
+    auto valueAttr = arrayAttr[1].dyn_cast<StringAttr>();
+    if (!keyAttr || !valueAttr)
+      return emitError(loc)
+             << "expected arrays within 'passthrough' to contain two strings";
+
+    if (failed(checkedAddLLVMFnAttribute(loc, llvmFunc, keyAttr.getValue(),
+                                         valueAttr.getValue())))
+      return failure();
+  }
+  return success();
+}
+
 LogicalResult ModuleTranslation::convertOneFunction(LLVMFuncOp func) {
   // Clear the block and value mappings, they are only relevant within one
   // function.
@@ -633,9 +714,13 @@ LogicalResult ModuleTranslation::convertFunctions() {
     llvm::FunctionCallee llvmFuncCst = llvmModule->getOrInsertFunction(
         function.getName(),
         cast<llvm::FunctionType>(function.getType().getUnderlyingType()));
-    assert(isa<llvm::Function>(llvmFuncCst.getCallee()));
-    functionMapping[function.getName()] =
-        cast<llvm::Function>(llvmFuncCst.getCallee());
+    llvm::Function *llvmFunc = cast<llvm::Function>(llvmFuncCst.getCallee());
+    functionMapping[function.getName()] = llvmFunc;
+
+    // Forward the pass-through attributes to LLVM.
+    if (failed(forwardPassthroughAttributes(function.getLoc(),
+                                            function.passthrough(), llvmFunc)))
+      return failure();
   }
 
   // Convert functions.
diff --git a/mlir/lib/Transforms/CMakeLists.txt b/mlir/lib/Transforms/CMakeLists.txt
index 8d3c5f46f8c4b..49704229ad14b 100644
--- a/mlir/lib/Transforms/CMakeLists.txt
+++ b/mlir/lib/Transforms/CMakeLists.txt
@@ -11,6 +11,7 @@ add_mlir_library(MLIRTransforms
   LoopInvariantCodeMotion.cpp
   MemRefDataFlowOpt.cpp
   OpStats.cpp
+  ParallelLoopCollapsing.cpp
   PipelineDataTransfer.cpp
   StripDebugInfo.cpp
   SymbolDCE.cpp
@@ -22,6 +23,7 @@ add_mlir_library(MLIRTransforms
 
   DEPENDS
   MLIRStandardOpsIncGen
+  MLIRTransformsPassIncGen
   )
 
 target_link_libraries(MLIRTransforms
diff --git a/mlir/lib/Transforms/CSE.cpp b/mlir/lib/Transforms/CSE.cpp
index 42ba7153f5cdd..67890f63dcd97 100644
--- a/mlir/lib/Transforms/CSE.cpp
+++ b/mlir/lib/Transforms/CSE.cpp
@@ -74,6 +74,10 @@ struct SimpleOperationInfo : public llvm::DenseMapInfo<Operation *> {
 namespace {
 /// Simple common sub-expression elimination.
 struct CSE : public OperationPass<CSE> {
+/// Include the generated pass utilities.
+#define GEN_PASS_CSE
+#include "mlir/Transforms/Passes.h.inc"
+
   CSE() = default;
   CSE(const CSE &) {}
 
@@ -114,10 +118,6 @@ struct CSE : public OperationPass<CSE> {
 private:
   /// Operations marked as dead and to be erased.
   std::vector<Operation *> opsToErase;
-
-  /// Statistics for CSE.
-  Statistic numCSE{this, "num-cse'd", "Number of operations CSE'd"};
-  Statistic numDCE{this, "num-dce'd", "Number of operations trivially DCE'd"};
 };
 } // end anonymous namespace
 
@@ -262,5 +262,3 @@ void CSE::runOnOperation() {
 }
 
 std::unique_ptr<Pass> mlir::createCSEPass() { return std::make_unique<CSE>(); }
-
-static PassRegistration<CSE> pass("cse", "Eliminate common sub-expressions");
diff --git a/mlir/lib/Transforms/Canonicalizer.cpp b/mlir/lib/Transforms/Canonicalizer.cpp
index f9fa1cdc4a3a8..964fc7f665005 100644
--- a/mlir/lib/Transforms/Canonicalizer.cpp
+++ b/mlir/lib/Transforms/Canonicalizer.cpp
@@ -11,15 +11,19 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/Passes.h"
+
 using namespace mlir;
 
 namespace {
 /// Canonicalize operations in nested regions.
 struct Canonicalizer : public OperationPass<Canonicalizer> {
+/// Include the generated pass utilities.
+#define GEN_PASS_Canonicalizer
+#include "mlir/Transforms/Passes.h.inc"
+
   void runOnOperation() override {
     OwningRewritePatternList patterns;
 
@@ -40,6 +44,3 @@ struct Canonicalizer : public OperationPass<Canonicalizer> {
 std::unique_ptr<Pass> mlir::createCanonicalizerPass() {
   return std::make_unique<Canonicalizer>();
 }
-
-static PassRegistration<Canonicalizer> pass("canonicalize",
-                                            "Canonicalize operations");
diff --git a/mlir/lib/Transforms/DialectConversion.cpp b/mlir/lib/Transforms/DialectConversion.cpp
index 19304b3fb73f8..725f5f4bb16e5 100644
--- a/mlir/lib/Transforms/DialectConversion.cpp
+++ b/mlir/lib/Transforms/DialectConversion.cpp
@@ -585,6 +585,9 @@ struct ConversionPatternRewriterImpl {
   /// PatternRewriter hook for replacing the results of an operation.
   void replaceOp(Operation *op, ValueRange newValues);
 
+  /// Notifies that a block was created.
+  void notifyCreatedBlock(Block *block);
+
   /// Notifies that a block was split.
   void notifySplitBlock(Block *block, Block *continuation);
 
@@ -804,6 +807,10 @@ void ConversionPatternRewriterImpl::replaceOp(Operation *op,
   markNestedOpsIgnored(op);
 }
 
+void ConversionPatternRewriterImpl::notifyCreatedBlock(Block *block) {
+  blockActions.push_back(BlockAction::getCreate(block));
+}
+
 void ConversionPatternRewriterImpl::notifySplitBlock(Block *block,
                                                      Block *continuation) {
   blockActions.push_back(BlockAction::getSplit(continuation, block));
@@ -910,6 +917,15 @@ Value ConversionPatternRewriter::getRemappedValue(Value key) {
   return impl->mapping.lookupOrDefault(key);
 }
 
+/// PatternRewriter hook for creating a new block with the given arguments.
+Block *ConversionPatternRewriter::createBlock(Region *parent,
+                                              Region::iterator insertPtr,
+                                              TypeRange argTypes) {
+  Block *block = PatternRewriter::createBlock(parent, insertPtr, argTypes);
+  impl->notifyCreatedBlock(block);
+  return block;
+}
+
 /// PatternRewriter hook for splitting a block into two parts.
 Block *ConversionPatternRewriter::splitBlock(Block *block,
                                              Block::iterator before) {
diff --git a/mlir/lib/Transforms/Inliner.cpp b/mlir/lib/Transforms/Inliner.cpp
index ea48582dc52a9..04ae30faf2aa3 100644
--- a/mlir/lib/Transforms/Inliner.cpp
+++ b/mlir/lib/Transforms/Inliner.cpp
@@ -590,6 +590,10 @@ static void inlineSCC(Inliner &inliner, CGUseList &useList,
 
 namespace {
 struct InlinerPass : public OperationPass<InlinerPass> {
+/// Include the generated pass utilities.
+#define GEN_PASS_Inliner
+#include "mlir/Transforms/Passes.h.inc"
+
   void runOnOperation() override {
     CallGraph &cg = getAnalysis<CallGraph>();
     auto *context = &getContext();
@@ -622,5 +626,3 @@ struct InlinerPass : public OperationPass<InlinerPass> {
 std::unique_ptr<Pass> mlir::createInlinerPass() {
   return std::make_unique<InlinerPass>();
 }
-
-static PassRegistration<InlinerPass> pass("inline", "Inline function calls");
diff --git a/mlir/lib/Transforms/LocationSnapshot.cpp b/mlir/lib/Transforms/LocationSnapshot.cpp
index 3d1ce828f1f28..4f54e13e76fd8 100644
--- a/mlir/lib/Transforms/LocationSnapshot.cpp
+++ b/mlir/lib/Transforms/LocationSnapshot.cpp
@@ -123,8 +123,11 @@ LogicalResult mlir::generateLocationsFromIR(StringRef fileName, StringRef tag,
 }
 
 namespace {
-class LocationSnapshotPass : public OperationPass<LocationSnapshotPass> {
-public:
+struct LocationSnapshotPass : public OperationPass<LocationSnapshotPass> {
+/// Include the generated pass utilities.
+#define GEN_PASS_LocationSnapshot
+#include "mlir/Transforms/Passes.h.inc"
+
   LocationSnapshotPass() = default;
   LocationSnapshotPass(const LocationSnapshotPass &) {}
   LocationSnapshotPass(OpPrintingFlags flags, StringRef fileName, StringRef tag)
@@ -139,14 +142,6 @@ class LocationSnapshotPass : public OperationPass<LocationSnapshotPass> {
       return signalPassFailure();
   }
 
-  Option<std::string> fileName{
-      *this, "filename",
-      llvm::cl::desc("The filename to print the generated IR.")};
-  Option<std::string> tag{
-      *this, "tag",
-      llvm::cl::desc("A tag to use when fusing the new locations with the "
-                     "original. If unset, the locations are replaced.")};
-
   /// The printing flags to use when creating the snapshot.
   OpPrintingFlags flags;
 };
@@ -157,6 +152,6 @@ std::unique_ptr<Pass> mlir::createLocationSnapshotPass(OpPrintingFlags flags,
                                                        StringRef tag) {
   return std::make_unique<LocationSnapshotPass>(flags, fileName, tag);
 }
-
-static PassRegistration<LocationSnapshotPass>
-    reg("snapshot-op-locations", "generate new locations from the current IR");
+std::unique_ptr<Pass> mlir::createLocationSnapshotPass() {
+  return std::make_unique<LocationSnapshotPass>();
+}
diff --git a/mlir/lib/Transforms/LoopCoalescing.cpp b/mlir/lib/Transforms/LoopCoalescing.cpp
index cfd5ae88bee70..322b3b92c52c6 100644
--- a/mlir/lib/Transforms/LoopCoalescing.cpp
+++ b/mlir/lib/Transforms/LoopCoalescing.cpp
@@ -19,8 +19,11 @@
 using namespace mlir;
 
 namespace {
-class LoopCoalescingPass : public FunctionPass<LoopCoalescingPass> {
-public:
+struct LoopCoalescingPass : public FunctionPass<LoopCoalescingPass> {
+/// Include the generated pass utilities.
+#define GEN_PASS_LoopCoalescing
+#include "mlir/Transforms/Passes.h.inc"
+
   void runOnFunction() override {
     FuncOp func = getFunction();
 
@@ -89,7 +92,3 @@ class LoopCoalescingPass : public FunctionPass<LoopCoalescingPass> {
 std::unique_ptr<OpPassBase<FuncOp>> mlir::createLoopCoalescingPass() {
   return std::make_unique<LoopCoalescingPass>();
 }
-
-static PassRegistration<LoopCoalescingPass>
-    reg(PASS_NAME,
-        "coalesce nested loops with independent bounds into a single loop");
diff --git a/mlir/lib/Transforms/LoopFusion.cpp b/mlir/lib/Transforms/LoopFusion.cpp
index 2f08f95261f27..55b45c0595a63 100644
--- a/mlir/lib/Transforms/LoopFusion.cpp
+++ b/mlir/lib/Transforms/LoopFusion.cpp
@@ -68,7 +68,6 @@ static llvm::cl::opt<unsigned long long> clFusionLocalBufThreshold(
     llvm::cl::cat(clOptionsCategory));
 
 namespace {
-
 /// Loop fusion pass. This pass currently supports a greedy fusion policy,
 /// which fuses loop nests with single-writer/single-reader memref dependences
 /// with the goal of improving locality.
@@ -79,6 +78,10 @@ namespace {
 // and add support for more general loop fusion algorithms.
 
 struct LoopFusion : public FunctionPass<LoopFusion> {
+/// Include the generated pass utilities.
+#define GEN_PASS_AffineLoopFusion
+#include "mlir/Transforms/Passes.h.inc"
+
   LoopFusion(unsigned fastMemorySpace = 0, uint64_t localBufSizeThreshold = 0,
              bool maximalFusion = false)
       : localBufSizeThreshold(localBufSizeThreshold),
@@ -1973,6 +1976,3 @@ void LoopFusion::runOnFunction() {
     GreedyFusion(&g, localBufSizeThreshold, fastMemorySpace, maximalFusion)
         .run();
 }
-
-static PassRegistration<LoopFusion> pass("affine-loop-fusion",
-                                         "Fuse loop nests");
diff --git a/mlir/lib/Transforms/LoopInvariantCodeMotion.cpp b/mlir/lib/Transforms/LoopInvariantCodeMotion.cpp
index e616663e11fcd..7407676d58771 100644
--- a/mlir/lib/Transforms/LoopInvariantCodeMotion.cpp
+++ b/mlir/lib/Transforms/LoopInvariantCodeMotion.cpp
@@ -28,7 +28,10 @@ using namespace mlir;
 namespace {
 /// Loop invariant code motion (LICM) pass.
 struct LoopInvariantCodeMotion : public OperationPass<LoopInvariantCodeMotion> {
-public:
+/// Include the generated pass utilities.
+#define GEN_PASS_LoopInvariantCodeMotion
+#include "mlir/Transforms/Passes.h.inc"
+
   void runOnOperation() override;
 };
 } // end anonymous namespace
@@ -121,7 +124,3 @@ void LoopInvariantCodeMotion::runOnOperation() {
 std::unique_ptr<Pass> mlir::createLoopInvariantCodeMotionPass() {
   return std::make_unique<LoopInvariantCodeMotion>();
 }
-
-static PassRegistration<LoopInvariantCodeMotion>
-    pass("loop-invariant-code-motion",
-         "Hoist loop invariant instructions outside of the loop");
diff --git a/mlir/lib/Transforms/MemRefDataFlowOpt.cpp b/mlir/lib/Transforms/MemRefDataFlowOpt.cpp
index eaf5c744723ce..e251edaf38cba 100644
--- a/mlir/lib/Transforms/MemRefDataFlowOpt.cpp
+++ b/mlir/lib/Transforms/MemRefDataFlowOpt.cpp
@@ -28,7 +28,6 @@
 using namespace mlir;
 
 namespace {
-
 // The store to load forwarding relies on three conditions:
 //
 // 1) they need to have mathematically equivalent affine access functions
@@ -62,6 +61,10 @@ namespace {
 // than dealloc) remain.
 //
 struct MemRefDataFlowOpt : public FunctionPass<MemRefDataFlowOpt> {
+/// Include the generated pass utilities.
+#define GEN_PASS_MemRefDataFlowOpt
+#include "mlir/Transforms/Passes.h.inc"
+
   void runOnFunction() override;
 
   void forwardStoreToLoad(AffineLoadOp loadOp);
@@ -222,6 +225,3 @@ void MemRefDataFlowOpt::runOnFunction() {
     defInst->erase();
   }
 }
-
-static PassRegistration<MemRefDataFlowOpt>
-    pass("memref-dataflow-opt", "Perform store/load forwarding for memrefs");
diff --git a/mlir/lib/Transforms/OpStats.cpp b/mlir/lib/Transforms/OpStats.cpp
index 0377b719f6d26..b7832f580dd42 100644
--- a/mlir/lib/Transforms/OpStats.cpp
+++ b/mlir/lib/Transforms/OpStats.cpp
@@ -19,6 +19,10 @@ using namespace mlir;
 
 namespace {
 struct PrintOpStatsPass : public ModulePass<PrintOpStatsPass> {
+/// Include the generated pass utilities.
+#define GEN_PASS_PrintOpStats
+#include "mlir/Transforms/Passes.h.inc"
+
   explicit PrintOpStatsPass(raw_ostream &os = llvm::errs()) : os(os) {}
 
   // Prints the resultant operation statistics post iterating over the module.
@@ -84,6 +88,3 @@ void PrintOpStatsPass::printSummary() {
 std::unique_ptr<OpPassBase<ModuleOp>> mlir::createPrintOpStatsPass() {
   return std::make_unique<PrintOpStatsPass>();
 }
-
-static PassRegistration<PrintOpStatsPass>
-    pass("print-op-stats", "Print statistics of operations");
diff --git a/mlir/lib/Transforms/ParallelLoopCollapsing.cpp b/mlir/lib/Transforms/ParallelLoopCollapsing.cpp
new file mode 100644
index 0000000000000..29aded6465c77
--- /dev/null
+++ b/mlir/lib/Transforms/ParallelLoopCollapsing.cpp
@@ -0,0 +1,52 @@
+//===- ParallelLoopCollapsing.cpp - Pass collapsing parallel loop indices -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/LoopOps/LoopOps.h"
+#include "mlir/Dialect/StandardOps/IR/Ops.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Transforms/LoopUtils.h"
+#include "mlir/Transforms/Passes.h"
+#include "mlir/Transforms/RegionUtils.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+
+#define DEBUG_TYPE "parallel-loop-collapsing"
+
+using namespace mlir;
+
+namespace {
+struct ParallelLoopCollapsing : public OperationPass<ParallelLoopCollapsing> {
+/// Include the generated pass utilities.
+#define GEN_PASS_ParallelLoopCollapsing
+#include "mlir/Transforms/Passes.h.inc"
+
+  ParallelLoopCollapsing() = default;
+  ParallelLoopCollapsing(const ParallelLoopCollapsing &) {}
+  void runOnOperation() override {
+    Operation *module = getOperation();
+
+    module->walk([&](loop::ParallelOp op) {
+      // The common case for GPU dialect will be simplifying the ParallelOp to 3
+      // arguments, so we do that here to simplify things.
+      llvm::SmallVector<std::vector<unsigned>, 3> combinedLoops;
+      if (clCollapsedIndices0.size())
+        combinedLoops.push_back(clCollapsedIndices0);
+      if (clCollapsedIndices1.size())
+        combinedLoops.push_back(clCollapsedIndices1);
+      if (clCollapsedIndices2.size())
+        combinedLoops.push_back(clCollapsedIndices2);
+      collapseParallelLoops(op, combinedLoops);
+    });
+  }
+};
+
+} // namespace
+
+std::unique_ptr<Pass> mlir::createParallelLoopCollapsingPass() {
+  return std::make_unique<ParallelLoopCollapsing>();
+}
diff --git a/mlir/lib/Transforms/PipelineDataTransfer.cpp b/mlir/lib/Transforms/PipelineDataTransfer.cpp
index df7bafc4b90b0..4c10303365400 100644
--- a/mlir/lib/Transforms/PipelineDataTransfer.cpp
+++ b/mlir/lib/Transforms/PipelineDataTransfer.cpp
@@ -28,8 +28,11 @@
 using namespace mlir;
 
 namespace {
-
 struct PipelineDataTransfer : public FunctionPass<PipelineDataTransfer> {
+/// Include the generated pass utilities.
+#define GEN_PASS_AffinePipelineDataTransfer
+#include "mlir/Transforms/Passes.h.inc"
+
   void runOnFunction() override;
   void runOnAffineForOp(AffineForOp forOp);
 
@@ -342,7 +345,7 @@ void PipelineDataTransfer::runOnAffineForOp(AffineForOp forOp) {
       instShiftMap[&op] = 1;
 
   // Get shifts stored in map.
-  std::vector<uint64_t> shifts(forOp.getBody()->getOperations().size());
+  SmallVector<uint64_t, 8> shifts(forOp.getBody()->getOperations().size());
   unsigned s = 0;
   for (auto &op : forOp.getBody()->without_terminator()) {
     assert(instShiftMap.find(&op) != instShiftMap.end());
@@ -355,7 +358,7 @@ void PipelineDataTransfer::runOnAffineForOp(AffineForOp forOp) {
     });
   }
 
-  if (!isInstwiseShiftValid(forOp, shifts)) {
+  if (!isOpwiseShiftValid(forOp, shifts)) {
     // Violates dependences.
     LLVM_DEBUG(llvm::dbgs() << "Shifts invalid - unexpected\n";);
     return;
@@ -366,8 +369,3 @@ void PipelineDataTransfer::runOnAffineForOp(AffineForOp forOp) {
     return;
   }
 }
-
-static PassRegistration<PipelineDataTransfer> pass(
-    "affine-pipeline-data-transfer",
-    "Pipeline non-blocking data transfers between explicitly managed levels of "
-    "the memory hierarchy");
diff --git a/mlir/lib/Transforms/StripDebugInfo.cpp b/mlir/lib/Transforms/StripDebugInfo.cpp
index 1a8b213febad7..d3420cfc35a13 100644
--- a/mlir/lib/Transforms/StripDebugInfo.cpp
+++ b/mlir/lib/Transforms/StripDebugInfo.cpp
@@ -15,6 +15,10 @@ using namespace mlir;
 
 namespace {
 struct StripDebugInfo : public OperationPass<StripDebugInfo> {
+/// Include the generated pass utilities.
+#define GEN_PASS_StripDebugInfo
+#include "mlir/Transforms/Passes.h.inc"
+
   void runOnOperation() override;
 };
 } // end anonymous namespace
@@ -29,6 +33,3 @@ void StripDebugInfo::runOnOperation() {
 std::unique_ptr<Pass> mlir::createStripDebugInfoPass() {
   return std::make_unique<StripDebugInfo>();
 }
-
-static PassRegistration<StripDebugInfo>
-    pass("strip-debuginfo", "Strip debug info from all operations");
diff --git a/mlir/lib/Transforms/SymbolDCE.cpp b/mlir/lib/Transforms/SymbolDCE.cpp
index 7d1d112c1a816..7513e34af388a 100644
--- a/mlir/lib/Transforms/SymbolDCE.cpp
+++ b/mlir/lib/Transforms/SymbolDCE.cpp
@@ -18,6 +18,10 @@ using namespace mlir;
 
 namespace {
 struct SymbolDCE : public OperationPass<SymbolDCE> {
+/// Include the generated pass utilities.
+#define GEN_PASS_SymbolDCE
+#include "mlir/Transforms/Passes.h.inc"
+
   void runOnOperation() override;
 
   /// Compute the liveness of the symbols within the given symbol table.
@@ -156,5 +160,3 @@ LogicalResult SymbolDCE::computeLiveness(Operation *symbolTableOp,
 std::unique_ptr<Pass> mlir::createSymbolDCEPass() {
   return std::make_unique<SymbolDCE>();
 }
-
-static PassRegistration<SymbolDCE> pass("symbol-dce", "Eliminate dead symbols");
diff --git a/mlir/lib/Transforms/Utils/LoopUtils.cpp b/mlir/lib/Transforms/Utils/LoopUtils.cpp
index a1c888b35af8c..42649ae336c09 100644
--- a/mlir/lib/Transforms/Utils/LoopUtils.cpp
+++ b/mlir/lib/Transforms/Utils/LoopUtils.cpp
@@ -17,10 +17,12 @@
 #include "mlir/Analysis/SliceAnalysis.h"
 #include "mlir/Analysis/Utils.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
+#include "mlir/Dialect/Affine/IR/AffineValueMap.h"
 #include "mlir/Dialect/LoopOps/LoopOps.h"
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/BlockAndValueMapping.h"
 #include "mlir/IR/Function.h"
+#include "mlir/IR/IntegerSet.h"
 #include "mlir/Transforms/RegionUtils.h"
 #include "mlir/Transforms/Utils.h"
 #include "llvm/ADT/DenseMap.h"
@@ -36,20 +38,29 @@ using namespace mlir;
 using llvm::SetVector;
 using llvm::SmallMapVector;
 
+namespace {
+// This structure is to pass and return sets of loop parameters without
+// confusing the order.
+struct LoopParams {
+  Value lowerBound;
+  Value upperBound;
+  Value step;
+};
+} // namespace
+
 /// Computes the cleanup loop lower bound of the loop being unrolled with
 /// the specified unroll factor; this bound will also be upper bound of the main
 /// part of the unrolled loop. Computes the bound as an AffineMap with its
 /// operands or a null map when the trip count can't be expressed as an affine
 /// expression.
-void mlir::getCleanupLoopLowerBound(AffineForOp forOp, unsigned unrollFactor,
-                                    AffineMap *map,
-                                    SmallVectorImpl<Value> *operands,
-                                    OpBuilder &b) {
+static void getCleanupLoopLowerBound(AffineForOp forOp, unsigned unrollFactor,
+                                     AffineMap &map,
+                                     SmallVectorImpl<Value> &operands) {
   auto lbMap = forOp.getLowerBoundMap();
 
   // Single result lower bound map only.
   if (lbMap.getNumResults() != 1) {
-    *map = AffineMap();
+    map = AffineMap();
     return;
   }
 
@@ -59,11 +70,11 @@ void mlir::getCleanupLoopLowerBound(AffineForOp forOp, unsigned unrollFactor,
 
   // Sometimes the trip count cannot be expressed as an affine expression.
   if (!tripCountMap) {
-    *map = AffineMap();
+    map = AffineMap();
     return;
   }
 
-  unsigned step = forOp.getStep();
+  OpBuilder b(forOp);
   auto lb = b.create<AffineApplyOp>(forOp.getLoc(), lbMap,
                                     forOp.getLowerBoundOperands());
 
@@ -74,6 +85,7 @@ void mlir::getCleanupLoopLowerBound(AffineForOp forOp, unsigned unrollFactor,
   // these affine.apply's make up the cleanup loop lower bound.
   SmallVector<AffineExpr, 4> bumpExprs(tripCountMap.getNumResults());
   SmallVector<Value, 4> bumpValues(tripCountMap.getNumResults());
+  int64_t step = forOp.getStep();
   for (unsigned i = 0, e = tripCountMap.getNumResults(); i < e; i++) {
     auto tripCountExpr = tripCountMap.getResult(i);
     bumpExprs[i] = (tripCountExpr - tripCountExpr % unrollFactor) * step;
@@ -87,19 +99,19 @@ void mlir::getCleanupLoopLowerBound(AffineForOp forOp, unsigned unrollFactor,
   for (unsigned i = 0, e = bumpExprs.size(); i < e; i++)
     newUbExprs[i] = b.getAffineDimExpr(0) + b.getAffineDimExpr(i + 1);
 
-  operands->clear();
-  operands->push_back(lb);
-  operands->append(bumpValues.begin(), bumpValues.end());
-  *map = AffineMap::get(1 + tripCountMap.getNumResults(), 0, newUbExprs);
+  operands.clear();
+  operands.push_back(lb);
+  operands.append(bumpValues.begin(), bumpValues.end());
+  map = AffineMap::get(1 + tripCountMap.getNumResults(), 0, newUbExprs);
   // Simplify the map + operands.
-  fullyComposeAffineMapAndOperands(map, operands);
-  *map = simplifyAffineMap(*map);
-  canonicalizeMapAndOperands(map, operands);
+  fullyComposeAffineMapAndOperands(&map, &operands);
+  map = simplifyAffineMap(map);
+  canonicalizeMapAndOperands(&map, &operands);
   // Remove any affine.apply's that became dead from the simplification above.
-  for (auto v : bumpValues) {
+  for (auto v : bumpValues)
     if (v.use_empty())
       v.getDefiningOp()->erase();
-  }
+
   if (lb.use_empty())
     lb.erase();
 }
@@ -109,16 +121,15 @@ void mlir::getCleanupLoopLowerBound(AffineForOp forOp, unsigned unrollFactor,
 // TODO(bondhugula): extend this for arbitrary affine bounds.
 LogicalResult mlir::promoteIfSingleIteration(AffineForOp forOp) {
   Optional<uint64_t> tripCount = getConstantTripCount(forOp);
-  if (!tripCount.hasValue() || tripCount.getValue() != 1)
+  if (!tripCount || tripCount.getValue() != 1)
     return failure();
 
-  // TODO(mlir-team): there is no builder for a max.
   if (forOp.getLowerBoundMap().getNumResults() != 1)
     return failure();
 
   // Replaces all IV uses to its single iteration value.
   auto iv = forOp.getInductionVar();
-  Operation *op = forOp.getOperation();
+  auto *parentBlock = forOp.getOperation()->getBlock();
   if (!iv.use_empty()) {
     if (forOp.hasConstantLowerBound()) {
       OpBuilder topBuilder(forOp.getParentOfType<FuncOp>().getBody());
@@ -128,7 +139,7 @@ LogicalResult mlir::promoteIfSingleIteration(AffineForOp forOp) {
     } else {
       auto lbOperands = forOp.getLowerBoundOperands();
       auto lbMap = forOp.getLowerBoundMap();
-      OpBuilder builder(op->getBlock(), Block::iterator(op));
+      OpBuilder builder(parentBlock, Block::iterator(forOp));
       if (lbMap == builder.getDimIdentityMap()) {
         // No need of generating an affine.apply.
         iv.replaceAllUsesWith(lbOperands[0]);
@@ -139,17 +150,16 @@ LogicalResult mlir::promoteIfSingleIteration(AffineForOp forOp) {
       }
     }
   }
-  // Move the loop body operations, except for terminator, to the loop's
+  // Move the loop body operations, except for its terminator, to the loop's
   // containing block.
-  auto *block = op->getBlock();
-  forOp.getBody()->getOperations().back().erase();
-  block->getOperations().splice(Block::iterator(op),
-                                forOp.getBody()->getOperations());
+  forOp.getBody()->back().erase();
+  parentBlock->getOperations().splice(Block::iterator(forOp),
+                                      forOp.getBody()->getOperations());
   forOp.erase();
   return success();
 }
 
-/// Promotes all single iteration for op's in the FuncOp, i.e., moves
+/// Promotes all single iteration 'for' ops in `f`, i.e., moves
 /// their body into the containing Block.
 void mlir::promoteSingleIterationLoops(FuncOp f) {
   // Gathers all innermost loops through a post order pruned walk.
@@ -221,6 +231,8 @@ static AffineForOp generateShiftedLoop(
 LogicalResult mlir::affineForOpBodySkew(AffineForOp forOp,
                                         ArrayRef<uint64_t> shifts,
                                         bool unrollPrologueEpilogue) {
+  assert(forOp.getBody()->getOperations().size() == shifts.size() &&
+         "too few/many shifts");
   if (forOp.getBody()->begin() == std::prev(forOp.getBody()->end()))
     return success();
 
@@ -235,20 +247,17 @@ LogicalResult mlir::affineForOpBodySkew(AffineForOp forOp,
   }
   uint64_t tripCount = mayBeConstTripCount.getValue();
 
-  assert(isInstwiseShiftValid(forOp, shifts) &&
+  assert(isOpwiseShiftValid(forOp, shifts) &&
          "shifts will lead to an invalid transformation\n");
 
   int64_t step = forOp.getStep();
 
-  unsigned numChildInsts = forOp.getBody()->getOperations().size();
+  unsigned numChildOps = shifts.size();
 
   // Do a linear time (counting) sort for the shifts.
-  uint64_t maxShift = 0;
-  for (unsigned i = 0; i < numChildInsts; i++) {
-    maxShift = std::max(maxShift, shifts[i]);
-  }
-  // Such large shifts are not the typical use case.
-  if (maxShift >= numChildInsts) {
+  uint64_t maxShift = *std::max_element(shifts.begin(), shifts.end());
+  if (maxShift >= numChildOps) {
+    // Large shifts are not the typical use case.
     forOp.emitWarning("not shifting because shifts are unrealistically large");
     return success();
   }
@@ -277,13 +286,13 @@ LogicalResult mlir::affineForOpBodySkew(AffineForOp forOp,
 
   auto origLbMap = forOp.getLowerBoundMap();
   uint64_t lbShift = 0;
-  OpBuilder b(forOp.getOperation());
+  OpBuilder b(forOp);
   for (uint64_t d = 0, e = sortedOpGroups.size(); d < e; ++d) {
     // If nothing is shifted by d, continue.
     if (sortedOpGroups[d].empty())
       continue;
     if (!opGroupQueue.empty()) {
-      assert(d >= 1 &&
+      assert(d > 0 &&
              "Queue expected to be empty when the first block is found");
       // The interval for which the loop needs to be generated here is:
       // [lbShift, min(lbShift + tripCount, d)) and the body of the
@@ -331,8 +340,7 @@ LogicalResult mlir::affineForOpBodySkew(AffineForOp forOp,
 
   if (unrollPrologueEpilogue && prologue)
     loopUnrollFull(prologue);
-  if (unrollPrologueEpilogue && !epilogue &&
-      epilogue.getOperation() != prologue.getOperation())
+  if (unrollPrologueEpilogue && !epilogue && epilogue != prologue)
     loopUnrollFull(epilogue);
 
   return success();
@@ -377,9 +385,8 @@ LogicalResult mlir::loopUnrollFull(AffineForOp forOp) {
   Optional<uint64_t> mayBeConstantTripCount = getConstantTripCount(forOp);
   if (mayBeConstantTripCount.hasValue()) {
     uint64_t tripCount = mayBeConstantTripCount.getValue();
-    if (tripCount == 1) {
+    if (tripCount == 1)
       return promoteIfSingleIteration(forOp);
-    }
     return loopUnrollByFactor(forOp, tripCount);
   }
   return failure();
@@ -401,14 +408,14 @@ LogicalResult mlir::loopUnrollUpToFactor(AffineForOp forOp,
 /// is successfully unrolled.
 LogicalResult mlir::loopUnrollByFactor(AffineForOp forOp,
                                        uint64_t unrollFactor) {
-  assert(unrollFactor >= 1 && "unroll factor should be >= 1");
+  assert(unrollFactor > 0 && "unroll factor should be positive");
 
   if (unrollFactor == 1)
     return promoteIfSingleIteration(forOp);
 
-  if (forOp.getBody()->empty() ||
-      forOp.getBody()->begin() == std::prev(forOp.getBody()->end()))
-    return failure();
+  // Nothing in the loop body other than the terminator.
+  if (has_single_element(forOp.getBody()->getOperations()))
+    return success();
 
   // Loops where the lower bound is a max expression isn't supported for
   // unrolling since the trip count can be expressed as an affine function when
@@ -426,20 +433,19 @@ LogicalResult mlir::loopUnrollByFactor(AffineForOp forOp,
     return failure();
 
   // Generate the cleanup loop if trip count isn't a multiple of unrollFactor.
-  Operation *op = forOp.getOperation();
   if (getLargestDivisorOfTripCount(forOp) % unrollFactor != 0) {
-    OpBuilder builder(op->getBlock(), ++Block::iterator(op));
-    auto cleanupForInst = cast<AffineForOp>(builder.clone(*op));
+    OpBuilder builder(forOp.getOperation()->getBlock(),
+                      std::next(Block::iterator(forOp)));
+    auto cleanupForOp = cast<AffineForOp>(builder.clone(*forOp));
     AffineMap cleanupMap;
     SmallVector<Value, 4> cleanupOperands;
-    getCleanupLoopLowerBound(forOp, unrollFactor, &cleanupMap, &cleanupOperands,
-                             builder);
+    getCleanupLoopLowerBound(forOp, unrollFactor, cleanupMap, cleanupOperands);
     assert(cleanupMap &&
            "cleanup loop lower bound map for single result lower bound maps "
            "can always be determined");
-    cleanupForInst.setLowerBound(cleanupOperands, cleanupMap);
+    cleanupForOp.setLowerBound(cleanupOperands, cleanupMap);
     // Promote the loop body up if this has turned into a single iteration loop.
-    promoteIfSingleIteration(cleanupForInst);
+    promoteIfSingleIteration(cleanupForOp);
 
     // Adjust upper bound of the original loop; this is the same as the lower
     // bound of the cleanup loop.
@@ -458,7 +464,7 @@ LogicalResult mlir::loopUnrollByFactor(AffineForOp forOp,
   // so that we know what to clone (since we are doing this in-place).
   Block::iterator srcBlockEnd = std::prev(forOp.getBody()->end(), 2);
 
-  // Unroll the contents of 'forOp' (append unrollFactor-1 additional copies).
+  // Unroll the contents of 'forOp' (append unrollFactor - 1 additional copies).
   auto forOpIV = forOp.getInductionVar();
   for (unsigned i = 1; i < unrollFactor; i++) {
     BlockAndValueMapping operandMap;
@@ -489,7 +495,6 @@ LogicalResult mlir::loopUnrollByFactor(AffineForOp forOp,
 LogicalResult mlir::loopUnrollJamUpToFactor(AffineForOp forOp,
                                             uint64_t unrollJamFactor) {
   Optional<uint64_t> mayBeConstantTripCount = getConstantTripCount(forOp);
-
   if (mayBeConstantTripCount.hasValue() &&
       mayBeConstantTripCount.getValue() < unrollJamFactor)
     return loopUnrollJamByFactor(forOp, mayBeConstantTripCount.getValue());
@@ -512,6 +517,7 @@ LogicalResult mlir::loopUnrollJamByFactor(AffineForOp forOp,
         for (auto &block : region)
           walk(block);
     }
+
     void walk(Block &block) {
       for (auto it = block.begin(), e = std::prev(block.end()); it != e;) {
         auto subBlockStart = it;
@@ -519,21 +525,21 @@ LogicalResult mlir::loopUnrollJamByFactor(AffineForOp forOp,
           ++it;
         if (it != subBlockStart)
           subBlocks.push_back({subBlockStart, std::prev(it)});
-        // Process all for insts that appear next.
+        // Process all for ops that appear next.
         while (it != e && isa<AffineForOp>(&*it))
           walk(&*it++);
       }
     }
   };
 
-  assert(unrollJamFactor >= 1 && "unroll jam factor should be >= 1");
+  assert(unrollJamFactor > 0 && "unroll jam factor should be positive");
 
   if (unrollJamFactor == 1)
     return promoteIfSingleIteration(forOp);
 
-  if (forOp.getBody()->empty() ||
-      forOp.getBody()->begin() == std::prev(forOp.getBody()->end()))
-    return failure();
+  // Nothing in the loop body other than the terminator.
+  if (has_single_element(forOp.getBody()->getOperations()))
+    return success();
 
   // Loops where both lower and upper bounds are multi-result maps won't be
   // unrolled (since the trip can't be expressed as an affine function in
@@ -547,28 +553,29 @@ LogicalResult mlir::loopUnrollJamByFactor(AffineForOp forOp,
   Optional<uint64_t> mayBeConstantTripCount = getConstantTripCount(forOp);
   // If the trip count is lower than the unroll jam factor, no unroll jam.
   if (mayBeConstantTripCount.hasValue() &&
-      mayBeConstantTripCount.getValue() < unrollJamFactor)
+      mayBeConstantTripCount.getValue() < unrollJamFactor) {
+    LLVM_DEBUG(llvm::dbgs() << "[failed] trip count < unroll-jam factor\n");
     return failure();
-
-  auto *forInst = forOp.getOperation();
+  }
 
   // Gather all sub-blocks to jam upon the loop being unrolled.
   JamBlockGatherer jbg;
-  jbg.walk(forInst);
+  jbg.walk(forOp);
   auto &subBlocks = jbg.subBlocks;
 
   // Generate the cleanup loop if trip count isn't a multiple of
   // unrollJamFactor.
   if (getLargestDivisorOfTripCount(forOp) % unrollJamFactor != 0) {
     // Insert the cleanup loop right after 'forOp'.
-    OpBuilder builder(forInst->getBlock(), std::next(Block::iterator(forInst)));
-    auto cleanupAffineForOp = cast<AffineForOp>(builder.clone(*forInst));
+    OpBuilder builder(forOp.getOperation()->getBlock(),
+                      std::next(Block::iterator(forOp)));
+    auto cleanupAffineForOp = cast<AffineForOp>(builder.clone(*forOp));
     // Adjust the lower bound of the cleanup loop; its upper bound is the same
     // as the original loop's upper bound.
     AffineMap cleanupMap;
     SmallVector<Value, 4> cleanupOperands;
-    getCleanupLoopLowerBound(forOp, unrollJamFactor, &cleanupMap,
-                             &cleanupOperands, builder);
+    getCleanupLoopLowerBound(forOp, unrollJamFactor, cleanupMap,
+                             cleanupOperands);
     cleanupAffineForOp.setLowerBound(cleanupOperands, cleanupMap);
 
     // Promote the cleanup loop if it has turned into a single iteration loop.
@@ -587,7 +594,7 @@ LogicalResult mlir::loopUnrollJamByFactor(AffineForOp forOp,
   // Unroll and jam (appends unrollJamFactor - 1 additional copies).
   for (unsigned i = unrollJamFactor - 1; i >= 1; --i) {
     // Operand map persists across all sub-blocks.
-    BlockAndValueMapping operandMapping;
+    BlockAndValueMapping operandMap;
     for (auto &subBlock : subBlocks) {
       // Builder to insert unroll-jammed bodies. Insert right at the end of
       // sub-block.
@@ -600,13 +607,12 @@ LogicalResult mlir::loopUnrollJamByFactor(AffineForOp forOp,
         auto d0 = builder.getAffineDimExpr(0);
         auto bumpMap = AffineMap::get(1, 0, {d0 + i * step});
         auto ivUnroll =
-            builder.create<AffineApplyOp>(forInst->getLoc(), bumpMap, forOpIV);
-        operandMapping.map(forOpIV, ivUnroll);
+            builder.create<AffineApplyOp>(forOp.getLoc(), bumpMap, forOpIV);
+        operandMap.map(forOpIV, ivUnroll);
       }
       // Clone the sub-block being unroll-jammed.
-      for (auto it = subBlock.first; it != std::next(subBlock.second); ++it) {
-        builder.clone(*it, operandMapping);
-      }
+      for (auto it = subBlock.first; it != std::next(subBlock.second); ++it)
+        builder.clone(*it, operandMap);
     }
   }
 
@@ -618,8 +624,6 @@ LogicalResult mlir::loopUnrollJamByFactor(AffineForOp forOp,
 /// Performs loop interchange on 'forOpA' and 'forOpB', where 'forOpB' is
 /// nested within 'forOpA' as the only non-terminator operation in its block.
 void mlir::interchangeLoops(AffineForOp forOpA, AffineForOp forOpB) {
-  auto *forOpAInst = forOpA.getOperation();
-
   assert(&*forOpA.getBody()->begin() == forOpB.getOperation());
   auto &forOpABody = forOpA.getBody()->getOperations();
   auto &forOpBBody = forOpB.getBody()->getOperations();
@@ -627,16 +631,17 @@ void mlir::interchangeLoops(AffineForOp forOpA, AffineForOp forOpB) {
   // 1) Splice forOpA's non-terminator operations (which is just forOpB) just
   // before forOpA (in ForOpA's parent's block) this should leave 'forOpA's
   // body containing only the terminator.
-  forOpAInst->getBlock()->getOperations().splice(Block::iterator(forOpAInst),
-                                                 forOpABody, forOpABody.begin(),
-                                                 std::prev(forOpABody.end()));
+  forOpA.getOperation()->getBlock()->getOperations().splice(
+      Block::iterator(forOpA), forOpABody, forOpABody.begin(),
+      std::prev(forOpABody.end()));
   // 2) Splice forOpB's non-terminator operations into the beginning of forOpA's
   // body (this leaves forOpB's body containing only the terminator).
   forOpABody.splice(forOpABody.begin(), forOpBBody, forOpBBody.begin(),
                     std::prev(forOpBBody.end()));
   // 3) Splice forOpA into the beginning of forOpB's body.
-  forOpBBody.splice(forOpBBody.begin(), forOpAInst->getBlock()->getOperations(),
-                    Block::iterator(forOpAInst));
+  forOpBBody.splice(forOpBBody.begin(),
+                    forOpA.getOperation()->getBlock()->getOperations(),
+                    Block::iterator(forOpA));
 }
 
 // Checks each dependence component against the permutation to see if the
@@ -688,23 +693,88 @@ bool mlir::isValidLoopInterchangePermutation(ArrayRef<AffineForOp> loops,
   return checkLoopInterchangeDependences(depCompsVec, loops, loopPermMap);
 }
 
-/// Performs a sequence of loop interchanges of loops in perfectly nested
-/// sequence of loops in 'loops', as specified by permutation in 'loopPermMap'.
-unsigned mlir::interchangeLoops(ArrayRef<AffineForOp> loops,
-                                ArrayRef<unsigned> loopPermMap) {
-  Optional<unsigned> loopNestRootIndex;
-  for (int i = loops.size() - 1; i >= 0; --i) {
-    int permIndex = static_cast<int>(loopPermMap[i]);
-    // Store the index of the for loop which will be the new loop nest root.
-    if (permIndex == 0)
-      loopNestRootIndex = i;
-    if (permIndex > i) {
-      // Sink loop 'i' by 'permIndex - i' levels deeper into the loop nest.
-      sinkLoop(loops[i], permIndex - i);
+/// Return true if `loops` is a perfect nest.
+static bool LLVM_ATTRIBUTE_UNUSED isPerfectlyNested(ArrayRef<AffineForOp> loops) {
+  auto outerLoop = loops.front();
+  for (auto loop : loops.drop_front()) {
+    auto parentForOp = dyn_cast<AffineForOp>(loop.getParentOp());
+    // parentForOp's body should be just this loop and the terminator.
+    if (parentForOp != outerLoop ||
+        parentForOp.getBody()->getOperations().size() != 2)
+      return false;
+    outerLoop = loop;
+  }
+  return true;
+}
+
+// input[i] should move from position i -> permMap[i]. Returns the position in
+// `input` that becomes the new outermost loop.
+unsigned mlir::permuteLoops(MutableArrayRef<AffineForOp> input,
+                            ArrayRef<unsigned> permMap) {
+  assert(input.size() == permMap.size() && "invalid permutation map size");
+  // Check whether the permutation spec is valid. This is a small vector - we'll
+  // just sort and check if it's iota.
+  SmallVector<unsigned, 4> checkPermMap(permMap.begin(), permMap.end());
+  llvm::sort(checkPermMap);
+  if (llvm::any_of(llvm::enumerate(checkPermMap),
+                   [](const auto &en) { return en.value() != en.index(); }))
+    assert(false && "invalid permutation map");
+
+  // Nothing to do.
+  if (input.size() < 2)
+    return 0;
+
+  assert(isPerfectlyNested(input) && "input not perfectly nested");
+
+  // Compute the inverse mapping, invPermMap: since input[i] goes to position
+  // permMap[i], position i of the permuted nest is at input[invPermMap[i]].
+  SmallVector<std::pair<unsigned, unsigned>, 4> invPermMap;
+  for (unsigned i = 0, e = input.size(); i < e; ++i)
+    invPermMap.push_back({permMap[i], i});
+  llvm::sort(invPermMap);
+
+  // Move the innermost loop body to the loop that would be the innermost in the
+  // permuted nest (only if the innermost loop is going to change).
+  if (permMap.back() != input.size() - 1) {
+    auto *destBody = input[invPermMap.back().second].getBody();
+    auto *srcBody = input.back().getBody();
+    destBody->getOperations().splice(destBody->begin(),
+                                     srcBody->getOperations(), srcBody->begin(),
+                                     std::prev(srcBody->end()));
+  }
+
+  // We'll move each loop in `input` in the reverse order so that its body is
+  // empty when we are moving it; this incurs zero copies and no erasing.
+  for (int i = input.size() - 1; i >= 0; --i) {
+    // If this has to become the outermost loop after permutation, add it to the
+    // parent block of the original root.
+    if (permMap[i] == 0) {
+      // If the root remains the same, nothing to do.
+      if (i == 0)
+        continue;
+      // Make input[i] the new outermost loop moving it into parentBlock.
+      auto *parentBlock = input[0].getOperation()->getBlock();
+      parentBlock->getOperations().splice(
+          Block::iterator(input[0]),
+          input[i].getOperation()->getBlock()->getOperations(),
+          Block::iterator(input[i]));
+      continue;
     }
+
+    // If the parent in the permuted order is the same as in the original,
+    // nothing to do.
+    unsigned parentPosInInput = invPermMap[permMap[i] - 1].second;
+    if (i > 0 && static_cast<unsigned>(i - 1) == parentPosInInput)
+      continue;
+
+    // Move input[i] to its surrounding loop in the transformed nest.
+    auto *destBody = input[parentPosInInput].getBody();
+    destBody->getOperations().splice(
+        destBody->begin(), input[i].getOperation()->getBlock()->getOperations(),
+        Block::iterator(input[i]));
   }
-  assert(loopNestRootIndex.hasValue());
-  return loopNestRootIndex.getValue();
+
+  return invPermMap[0].second;
 }
 
 // Sinks all sequential loops to the innermost levels (while preserving
@@ -758,19 +828,10 @@ AffineForOp mlir::sinkSequentialLoops(AffineForOp forOp) {
   if (!checkLoopInterchangeDependences(depCompsVec, loops, loopPermMap))
     return forOp;
   // Perform loop interchange according to permutation 'loopPermMap'.
-  unsigned loopNestRootIndex = interchangeLoops(loops, loopPermMap);
+  unsigned loopNestRootIndex = permuteLoops(loops, loopPermMap);
   return loops[loopNestRootIndex];
 }
 
-/// Performs a series of loop interchanges to sink 'forOp' 'loopDepth' levels
-/// deeper in the loop nest.
-void mlir::sinkLoop(AffineForOp forOp, unsigned loopDepth) {
-  for (unsigned i = 0; i < loopDepth; ++i) {
-    AffineForOp nextForOp = cast<AffineForOp>(forOp.getBody()->front());
-    interchangeLoops(forOp, nextForOp);
-  }
-}
-
 // Factors out common behavior to add a new `iv` (resp. `iv` + `offset`) to the
 // lower (resp. upper) loop bound. When called for both the lower and upper
 // bounds, the resulting IR resembles:
@@ -805,8 +866,8 @@ stripmineSink(AffineForOp forOp, uint64_t factor,
   auto scaledStep = originalStep * factor;
   forOp.setStep(scaledStep);
 
-  auto *op = forOp.getOperation();
-  OpBuilder b(op->getBlock(), ++Block::iterator(op));
+  OpBuilder b(forOp.getOperation()->getBlock(),
+              std::next(Block::iterator(forOp)));
 
   // Lower-bound map creation.
   auto lbMap = forOp.getLowerBoundMap();
@@ -1094,69 +1155,77 @@ replaceAllUsesExcept(Value orig, Value replacement,
   }
 }
 
-// Transform a loop with a strictly positive step
-//   for %i = %lb to %ub step %s
-// into a 0-based loop with step 1
-//   for %ii = 0 to ceildiv(%ub - %lb, %s) step 1 {
-//     %i = %ii * %s + %lb
-// Insert the induction variable remapping in the body of `inner`, which is
-// expected to be either `loop` or another loop perfectly nested under `loop`.
-// Insert the definition of new bounds immediate before `outer`, which is
-// expected to be either `loop` or its parent in the loop nest.
-static void normalizeLoop(loop::ForOp loop, loop::ForOp outer,
-                          loop::ForOp inner) {
-  OpBuilder builder(outer);
-  Location loc = loop.getLoc();
-
+/// Return the new lower bound, upper bound, and step in that order. Insert any
+/// additional bounds calculations before the given builder and any additional
+/// conversion back to the original loop induction value inside the given Block.
+static LoopParams normalizeLoop(OpBuilder &boundsBuilder,
+                                OpBuilder &insideLoopBuilder, Location loc,
+                                Value lowerBound, Value upperBound, Value step,
+                                Value inductionVar) {
   // Check if the loop is already known to have a constant zero lower bound or
   // a constant one step.
   bool isZeroBased = false;
   if (auto ubCst =
-          dyn_cast_or_null<ConstantIndexOp>(loop.lowerBound().getDefiningOp()))
+          dyn_cast_or_null<ConstantIndexOp>(lowerBound.getDefiningOp()))
     isZeroBased = ubCst.getValue() == 0;
 
   bool isStepOne = false;
-  if (auto stepCst =
-          dyn_cast_or_null<ConstantIndexOp>(loop.step().getDefiningOp()))
+  if (auto stepCst = dyn_cast_or_null<ConstantIndexOp>(step.getDefiningOp()))
     isStepOne = stepCst.getValue() == 1;
 
-  if (isZeroBased && isStepOne)
-    return;
-
   // Compute the number of iterations the loop executes: ceildiv(ub - lb, step)
   // assuming the step is strictly positive.  Update the bounds and the step
   // of the loop to go from 0 to the number of iterations, if necessary.
   // TODO(zinenko): introduce support for negative steps or emit dynamic asserts
   // on step positivity, whatever gets implemented first.
-  Value diff =
-      builder.create<SubIOp>(loc, loop.upperBound(), loop.lowerBound());
-  Value numIterations = ceilDivPositive(builder, loc, diff, loop.step());
-  loop.setUpperBound(numIterations);
-
-  Value lb = loop.lowerBound();
-  if (!isZeroBased) {
-    Value cst0 = builder.create<ConstantIndexOp>(loc, 0);
-    loop.setLowerBound(cst0);
-  }
+  if (isZeroBased && isStepOne)
+    return {/*lowerBound=*/lowerBound, /*upperBound=*/upperBound,
+            /*step=*/step};
 
-  Value step = loop.step();
-  if (!isStepOne) {
-    Value cst1 = builder.create<ConstantIndexOp>(loc, 1);
-    loop.setStep(cst1);
-  }
+  Value diff = boundsBuilder.create<SubIOp>(loc, upperBound, lowerBound);
+  Value newUpperBound = ceilDivPositive(boundsBuilder, loc, diff, step);
+
+  Value newLowerBound =
+      isZeroBased ? lowerBound : boundsBuilder.create<ConstantIndexOp>(loc, 0);
+  Value newStep =
+      isStepOne ? step : boundsBuilder.create<ConstantIndexOp>(loc, 1);
 
   // Insert code computing the value of the original loop induction variable
   // from the "normalized" one.
-  builder.setInsertionPointToStart(inner.getBody());
   Value scaled =
-      isStepOne ? loop.getInductionVar()
-                : builder.create<MulIOp>(loc, loop.getInductionVar(), step);
+      isStepOne ? inductionVar
+                : insideLoopBuilder.create<MulIOp>(loc, inductionVar, step);
   Value shifted =
-      isZeroBased ? scaled : builder.create<AddIOp>(loc, scaled, lb);
+      isZeroBased ? scaled
+                  : insideLoopBuilder.create<AddIOp>(loc, scaled, lowerBound);
 
   SmallPtrSet<Operation *, 2> preserve{scaled.getDefiningOp(),
                                        shifted.getDefiningOp()};
-  replaceAllUsesExcept(loop.getInductionVar(), shifted, preserve);
+  replaceAllUsesExcept(inductionVar, shifted, preserve);
+  return {/*lowerBound=*/newLowerBound, /*upperBound=*/newUpperBound,
+          /*step=*/newStep};
+}
+
+/// Transform a loop with a strictly positive step
+///   for %i = %lb to %ub step %s
+/// into a 0-based loop with step 1
+///   for %ii = 0 to ceildiv(%ub - %lb, %s) step 1 {
+///     %i = %ii * %s + %lb
+/// Insert the induction variable remapping in the body of `inner`, which is
+/// expected to be either `loop` or another loop perfectly nested under `loop`.
+/// Insert the definition of new bounds immediate before `outer`, which is
+/// expected to be either `loop` or its parent in the loop nest.
+static void normalizeLoop(loop::ForOp loop, loop::ForOp outer,
+                          loop::ForOp inner) {
+  OpBuilder builder(outer);
+  OpBuilder innerBuilder = OpBuilder::atBlockBegin(inner.getBody());
+  auto loopPieces =
+      normalizeLoop(builder, innerBuilder, loop.getLoc(), loop.lowerBound(),
+                    loop.upperBound(), loop.step(), loop.getInductionVar());
+
+  loop.setLowerBound(loopPieces.lowerBound);
+  loop.setUpperBound(loopPieces.upperBound);
+  loop.setStep(loopPieces.step);
 }
 
 void mlir::coalesceLoops(MutableArrayRef<loop::ForOp> loops) {
@@ -1214,6 +1283,87 @@ void mlir::coalesceLoops(MutableArrayRef<loop::ForOp> loops) {
   second.erase();
 }
 
+void mlir::collapseParallelLoops(
+    loop::ParallelOp loops,
+    ArrayRef<std::vector<unsigned>> combinedDimensions) {
+  OpBuilder outsideBuilder(loops);
+  Location loc = loops.getLoc();
+
+  // Normalize ParallelOp's iteration pattern.
+  SmallVector<Value, 3> normalizedLowerBounds;
+  SmallVector<Value, 3> normalizedSteps;
+  SmallVector<Value, 3> normalizedUpperBounds;
+  for (unsigned i = 0, e = loops.getNumLoops(); i < e; ++i) {
+    OpBuilder insideLoopBuilder = OpBuilder::atBlockBegin(loops.getBody());
+    auto resultBounds =
+        normalizeLoop(outsideBuilder, insideLoopBuilder, loc,
+                      loops.lowerBound()[i], loops.upperBound()[i],
+                      loops.step()[i], loops.getBody()->getArgument(i));
+
+    normalizedLowerBounds.push_back(resultBounds.lowerBound);
+    normalizedUpperBounds.push_back(resultBounds.upperBound);
+    normalizedSteps.push_back(resultBounds.step);
+  }
+
+  // Combine iteration spaces.
+  SmallVector<Value, 3> lowerBounds;
+  SmallVector<Value, 3> steps;
+  SmallVector<Value, 3> upperBounds;
+  auto cst0 = outsideBuilder.create<ConstantIndexOp>(loc, 0);
+  auto cst1 = outsideBuilder.create<ConstantIndexOp>(loc, 1);
+  for (unsigned i = 0, e = combinedDimensions.size(); i < e; ++i) {
+    Value newUpperBound = outsideBuilder.create<ConstantIndexOp>(loc, 1);
+    for (auto idx : combinedDimensions[i]) {
+      newUpperBound = outsideBuilder.create<MulIOp>(loc, newUpperBound,
+                                                    normalizedUpperBounds[idx]);
+    }
+    lowerBounds.push_back(cst0);
+    steps.push_back(cst1);
+    upperBounds.push_back(newUpperBound);
+  }
+
+  // Create new ParallelLoop with conversions to the original induction values.
+  // The loop below uses divisions to get the relevant range of values in the
+  // new induction value that represent each range of the original induction
+  // value. The remainders then determine based on that range, which iteration
+  // of the original induction value this represents. This is a normalized value
+  // that is un-normalized already by the previous logic.
+  auto newPloop = outsideBuilder.create<loop::ParallelOp>(loc, lowerBounds,
+                                                          upperBounds, steps);
+  OpBuilder insideBuilder(newPloop.region());
+  for (unsigned i = 0, e = combinedDimensions.size(); i < e; ++i) {
+    Value previous = newPloop.getBody()->getArgument(i);
+    unsigned numberCombinedDimensions = combinedDimensions[i].size();
+    // Iterate over all except the last induction value.
+    for (unsigned j = 0, e = numberCombinedDimensions - 1; j < e; ++j) {
+      unsigned idx = combinedDimensions[i][j];
+
+      // Determine the current induction value's current loop iteration
+      Value iv = insideBuilder.create<SignedRemIOp>(loc, previous,
+                                                    normalizedUpperBounds[idx]);
+      replaceAllUsesInRegionWith(loops.getBody()->getArgument(idx), iv,
+                                 loops.region());
+
+      // Remove the effect of the current induction value to prepare for the
+      // next value.
+      previous = insideBuilder.create<SignedDivIOp>(
+          loc, previous, normalizedUpperBounds[idx + 1]);
+    }
+
+    // The final induction value is just the remaining value.
+    unsigned idx = combinedDimensions[i][numberCombinedDimensions - 1];
+    replaceAllUsesInRegionWith(loops.getBody()->getArgument(idx), previous,
+                               loops.region());
+  }
+
+  // Replace the old loop with the new loop.
+  loops.getBody()->back().erase();
+  newPloop.getBody()->getOperations().splice(
+      Block::iterator(newPloop.getBody()->back()),
+      loops.getBody()->getOperations());
+  loops.erase();
+}
+
 void mlir::mapLoopToProcessorIds(loop::ForOp forOp, ArrayRef<Value> processorId,
                                  ArrayRef<Value> numProcessors) {
   assert(processorId.size() == numProcessors.size());
@@ -1665,16 +1815,16 @@ static LogicalResult generateCopy(
 
 /// Construct the memref region to just include the entire memref. Returns false
 /// dynamic shaped memref's for now. `numParamLoopIVs` is the number of
-/// enclosing loop IVs of opInst (starting from the outermost) that the region
+/// enclosing loop IVs of `op` (starting from the outermost) that the region
 /// is parametric on.
-static bool getFullMemRefAsRegion(Operation *opInst, unsigned numParamLoopIVs,
+static bool getFullMemRefAsRegion(Operation *op, unsigned numParamLoopIVs,
                                   MemRefRegion *region) {
   unsigned rank;
-  if (auto loadOp = dyn_cast<AffineLoadOp>(opInst)) {
+  if (auto loadOp = dyn_cast<AffineLoadOp>(op)) {
     rank = loadOp.getMemRefType().getRank();
     region->memref = loadOp.getMemRef();
     region->setWrite(false);
-  } else if (auto storeOp = dyn_cast<AffineStoreOp>(opInst)) {
+  } else if (auto storeOp = dyn_cast<AffineStoreOp>(op)) {
     rank = storeOp.getMemRefType().getRank();
     region->memref = storeOp.getMemRef();
     region->setWrite(true);
@@ -1691,7 +1841,7 @@ static bool getFullMemRefAsRegion(Operation *opInst, unsigned numParamLoopIVs,
   // Just get the first numSymbols IVs, which the memref region is parametric
   // on.
   SmallVector<AffineForOp, 4> ivs;
-  getLoopIVs(*opInst, &ivs);
+  getLoopIVs(*op, &ivs);
   ivs.resize(numParamLoopIVs);
   SmallVector<Value, 4> symbols;
   extractForInductionVars(ivs, &symbols);
@@ -1969,3 +2119,216 @@ void mlir::gatherLoops(FuncOp func,
     depthToLoops.pop_back();
   }
 }
+
+// TODO: if necessary, this can be extended to also compose in any
+// affine.applys, fold to constant if all result dimensions of the map are
+// constant (canonicalizeMapAndOperands below already does this for single
+// result bound maps), and use simplifyMap to perform algebraic simplication.
+AffineForOp mlir::createCanonicalizedAffineForOp(
+    OpBuilder b, Location loc, ValueRange lbOperands, AffineMap lbMap,
+    ValueRange ubOperands, AffineMap ubMap, int64_t step) {
+  SmallVector<Value, 4> lowerOperands(lbOperands);
+  SmallVector<Value, 4> upperOperands(ubOperands);
+
+  fullyComposeAffineMapAndOperands(&lbMap, &lowerOperands);
+  canonicalizeMapAndOperands(&lbMap, &lowerOperands);
+  lbMap = removeDuplicateExprs(lbMap);
+  fullyComposeAffineMapAndOperands(&ubMap, &upperOperands);
+  canonicalizeMapAndOperands(&ubMap, &upperOperands);
+  ubMap = removeDuplicateExprs(ubMap);
+
+  return b.create<AffineForOp>(loc, lowerOperands, lbMap, upperOperands, ubMap,
+                               step);
+}
+
+/// Creates an AffineIfOp that encodes the conditional to choose between
+/// the constant trip count version and an unknown trip count version of this
+/// nest of loops. This is used to separate partial and full tiles if `loops`
+/// has the intra-tile loops. The affine.if op is inserted at the builder
+/// insertion point of `b`.
+static AffineIfOp createSeparationCondition(MutableArrayRef<AffineForOp> loops,
+                                            OpBuilder b) {
+  if (loops.empty())
+    return nullptr;
+
+  auto *context = loops[0].getContext();
+
+  FlatAffineConstraints cst;
+  getIndexSet(loops, &cst);
+
+  // Remove constraints that are independent of these loop IVs.
+  cst.removeIndependentConstraints(/*pos=*/0, /*num=*/loops.size());
+
+  // Construct the constraint set representing the guard for full tiles. The
+  // lower bound (and upper bound) corresponding to the full tile should be
+  // larger (and resp. smaller) than any other lower (or upper bound).
+  SmallVector<int64_t, 8> fullTileLb, fullTileUb;
+  for (auto loop : loops) {
+    (void) loop;
+    // TODO: Non-unit stride is not an issue to generalize to.
+    assert(loop.getStep() == 1 && "point loop step expected to be one");
+    // Mark everything symbols for the purpose of finding a constant diff pair.
+    cst.setDimSymbolSeparation(/*newSymbolCount=*/cst.getNumDimAndSymbolIds() -
+                               1);
+    unsigned fullTileLbPos, fullTileUbPos;
+    if (!cst.getConstantBoundOnDimSize(0, /*lb=*/nullptr,
+                                       /*lbFloorDivisor=*/nullptr,
+                                       /*ub=*/nullptr, &fullTileLbPos,
+                                       &fullTileUbPos)) {
+      LLVM_DEBUG(llvm::dbgs() << "Can't get constant diff pair for a loop\n");
+      return nullptr;
+    }
+
+    SmallVector<unsigned, 4> lbIndices, ubIndices;
+    cst.getLowerAndUpperBoundIndices(/*pos=*/0, &lbIndices, &ubIndices);
+
+    auto fLb = cst.getInequality(fullTileLbPos);
+    auto fUb = cst.getInequality(fullTileUbPos);
+    fullTileLb.assign(fLb.begin(), fLb.end());
+    fullTileUb.assign(fUb.begin(), fUb.end());
+
+    // Full tile lower bound should be >= than any other lower bound.
+    for (auto lbIndex : lbIndices)
+      for (unsigned i = 0, e = cst.getNumCols(); i < e; ++i)
+        cst.atIneq(lbIndex, i) = fullTileLb[i] - cst.atIneq(lbIndex, i);
+
+    // Full tile upper bound should be <= any other upper bound.
+    for (auto ubIndex : ubIndices)
+      for (unsigned i = 0, e = cst.getNumCols(); i < e; ++i)
+        cst.atIneq(ubIndex, i) -= fullTileUb[i];
+
+    cst.removeId(0);
+  }
+
+  // The previous step leads to all zeros for the full tile lb and ub position
+  // itself; remove those and any other duplicates / trivial redundancies.
+  cst.removeTrivialRedundancy();
+
+  // Turn everything into dims conservatively since we earlier turned all
+  // trailing ids past point loop IV into symbols. Some of these could be outer
+  // loop IVs; we'll canonicalize anyway.
+  cst.setDimSymbolSeparation(0);
+
+  IntegerSet ifCondSet = cst.getAsIntegerSet(context);
+  // ifCondSet can be null if cst was empty -- this can happen if all loops
+  // in the nest have constant trip counts.
+  if (!ifCondSet)
+    return nullptr;
+
+  SmallVector<Value, 4> setOperands;
+  cst.getIdValues(0, cst.getNumDimAndSymbolIds(), &setOperands);
+  canonicalizeSetAndOperands(&ifCondSet, &setOperands);
+  return b.create<AffineIfOp>(loops[0].getLoc(), ifCondSet, setOperands,
+                              /*withElseRegion=*/true);
+}
+
+/// Create the full tile loop nest (along with its body).
+static LogicalResult
+createFullTiles(MutableArrayRef<AffineForOp> inputNest,
+                SmallVectorImpl<AffineForOp> &fullTileLoops, OpBuilder b) {
+  fullTileLoops.reserve(inputNest.size());
+
+  // For each loop in the original nest identify a lower/upper bound pair such
+  // that their difference is a constant.
+  FlatAffineConstraints cst;
+  for (auto loop : inputNest) {
+    // TODO: straightforward to generalize to a non-unit stride.
+    if (loop.getStep() != 1) {
+      LLVM_DEBUG(llvm::dbgs()
+                 << "[tile separation] non-unit stride not implemented\n");
+      return failure();
+    }
+    getIndexSet({loop}, &cst);
+    // We will mark everything other than this loop IV as symbol for getting a
+    // pair of <lb, ub> with a constant difference.
+    cst.setDimSymbolSeparation(cst.getNumDimAndSymbolIds() - 1);
+    unsigned lbPos, ubPos;
+    if (!cst.getConstantBoundOnDimSize(/*pos=*/0, /*lb=*/nullptr,
+                                       /*lbDivisor=*/nullptr, /*ub=*/nullptr,
+                                       &lbPos, &ubPos) ||
+        lbPos == ubPos) {
+      LLVM_DEBUG(llvm::dbgs() << "[tile separation] Can't get constant diff / "
+                                 "equalities not yet handled\n");
+      return failure();
+    }
+
+    // Set all identifiers as dimensions uniformly since some of those marked as
+    // symbols above could be outer loop IVs (corresponding tile space IVs).
+    cst.setDimSymbolSeparation(/*newSymbolCount=*/0);
+
+    AffineValueMap lbVmap, ubVmap;
+    cst.getIneqAsAffineValueMap(/*pos=*/0, lbPos, lbVmap, b.getContext());
+    cst.getIneqAsAffineValueMap(/*pos=*/0, ubPos, ubVmap, b.getContext());
+    AffineForOp fullTileLoop = createCanonicalizedAffineForOp(
+        b, loop.getLoc(), lbVmap.getOperands(), lbVmap.getAffineMap(),
+        ubVmap.getOperands(), ubVmap.getAffineMap());
+    b = fullTileLoop.getBodyBuilder();
+    fullTileLoops.push_back(fullTileLoop);
+  }
+
+  // Add the body for the full tile loop nest.
+  BlockAndValueMapping operandMap;
+  for (auto loopEn : llvm::enumerate(inputNest))
+    operandMap.map(loopEn.value().getInductionVar(),
+                   fullTileLoops[loopEn.index()].getInductionVar());
+  b = fullTileLoops.back().getBodyBuilder();
+  for (auto &op : inputNest.back().getBody()->without_terminator())
+    b.clone(op, operandMap);
+  return success();
+}
+
+LogicalResult
+mlir::separateFullTiles(MutableArrayRef<AffineForOp> inputNest,
+                        SmallVectorImpl<AffineForOp> *fullTileNest) {
+  if (inputNest.empty())
+    return success();
+
+  auto firstLoop = inputNest[0];
+
+  // Each successive for op has to be nested in the other.
+  auto prevLoop = firstLoop;
+  for (auto loop : inputNest.drop_front(1)) {
+    assert(loop.getParentOp() == prevLoop && "input not contiguously nested");
+    prevLoop = loop;
+  }
+
+  // Create the full tile loop nest.
+  SmallVector<AffineForOp, 4> fullTileLoops;
+  OpBuilder b(firstLoop);
+  if (failed(createFullTiles(inputNest, fullTileLoops, b))) {
+    if (!fullTileLoops.empty())
+      fullTileLoops.front().erase();
+    return failure();
+  }
+
+  // Create and insert the version select right before the root of the nest.
+  b = OpBuilder(firstLoop);
+  AffineIfOp ifOp = createSeparationCondition(inputNest, b);
+  if (!ifOp) {
+    fullTileLoops.front().erase();
+    LLVM_DEBUG(llvm::dbgs() << "All tiles are full tiles, or failure creating "
+                               "separation condition\n");
+    return failure();
+  }
+
+  // Move the full tile into the then block.
+  Block *thenBlock = ifOp.getThenBlock();
+  AffineForOp outermostFullTileLoop = fullTileLoops[0];
+  thenBlock->getOperations().splice(
+      std::prev(thenBlock->end()),
+      outermostFullTileLoop.getOperation()->getBlock()->getOperations(),
+      Block::iterator(outermostFullTileLoop));
+
+  // Move the partial tile into the else block. The partial tile is the same as
+  // the original loop nest.
+  Block *elseBlock = ifOp.getElseBlock();
+  elseBlock->getOperations().splice(
+      std::prev(elseBlock->end()),
+      firstLoop.getOperation()->getBlock()->getOperations(),
+      Block::iterator(firstLoop));
+
+  if (fullTileNest)
+    *fullTileNest = std::move(fullTileLoops);
+
+  return success();
+}
diff --git a/mlir/lib/Transforms/ViewOpGraph.cpp b/mlir/lib/Transforms/ViewOpGraph.cpp
index e5083b512f5c7..fcaff9a0b0690 100644
--- a/mlir/lib/Transforms/ViewOpGraph.cpp
+++ b/mlir/lib/Transforms/ViewOpGraph.cpp
@@ -101,6 +101,10 @@ namespace {
 // Note: this is a module pass only to avoid interleaving on the same ostream
 // due to multi-threading over functions.
 struct PrintOpPass : public ModulePass<PrintOpPass> {
+/// Include the generated pass utilities.
+#define GEN_PASS_PrintOpGraph
+#include "mlir/Transforms/Passes.h.inc"
+
   explicit PrintOpPass(raw_ostream &os = llvm::errs(), bool short_names = false,
                        const Twine &title = "")
       : os(os), title(title.str()), short_names(short_names) {}
@@ -161,6 +165,3 @@ mlir::createPrintOpGraphPass(raw_ostream &os, bool shortNames,
                              const Twine &title) {
   return std::make_unique<PrintOpPass>(os, shortNames, title);
 }
-
-static PassRegistration<PrintOpPass> pass("print-op-graph",
-                                          "Print op graph per region");
diff --git a/mlir/lib/Transforms/ViewRegionGraph.cpp b/mlir/lib/Transforms/ViewRegionGraph.cpp
index f9e2929b8ada3..cf9ff6d8077e4 100644
--- a/mlir/lib/Transforms/ViewRegionGraph.cpp
+++ b/mlir/lib/Transforms/ViewRegionGraph.cpp
@@ -61,6 +61,10 @@ void mlir::Region::viewGraph() { viewGraph("region"); }
 
 namespace {
 struct PrintCFGPass : public FunctionPass<PrintCFGPass> {
+/// Include the generated pass utilities.
+#define GEN_PASS_PrintCFG
+#include "mlir/Transforms/Passes.h.inc"
+
   PrintCFGPass(raw_ostream &os = llvm::errs(), bool shortNames = false,
                const Twine &title = "")
       : os(os), shortNames(shortNames), title(title.str()) {}
@@ -80,6 +84,3 @@ mlir::createPrintCFGGraphPass(raw_ostream &os, bool shortNames,
                               const Twine &title) {
   return std::make_unique<PrintCFGPass>(os, shortNames, title);
 }
-
-static PassRegistration<PrintCFGPass> pass("print-cfg-graph",
-                                           "Print CFG graph per Function");
diff --git a/mlir/test/Analysis/test-dominance.mlir b/mlir/test/Analysis/test-dominance.mlir
new file mode 100644
index 0000000000000..3e3678b40468a
--- /dev/null
+++ b/mlir/test/Analysis/test-dominance.mlir
@@ -0,0 +1,207 @@
+// RUN: mlir-opt %s -test-print-dominance -split-input-file 2>&1 | FileCheck %s --dump-input-on-failure
+
+// CHECK-LABEL: Testing : func_condBranch
+func @func_condBranch(%cond : i1) {
+  cond_br %cond, ^bb1, ^bb2
+^bb1:
+  br ^exit
+^bb2:
+  br ^exit
+^exit:
+  return
+}
+// CHECK-LABEL: --- DominanceInfo ---
+// CHECK-NEXT: Nearest(0, 0) = 0
+// CHECK-NEXT: Nearest(0, 1) = 0
+// CHECK-NEXT: Nearest(0, 2) = 0
+// CHECK-NEXT: Nearest(0, 3) = 0
+// CHECK: Nearest(1, 0) = 0
+// CHECK-NEXT: Nearest(1, 1) = 1
+// CHECK-NEXT: Nearest(1, 2) = 0
+// CHECK-NEXT: Nearest(1, 3) = 0
+// CHECK: Nearest(2, 0) = 0
+// CHECK-NEXT: Nearest(2, 1) = 0
+// CHECK-NEXT: Nearest(2, 2) = 2
+// CHECK-NEXT: Nearest(2, 3) = 0
+// CHECK: Nearest(3, 0) = 0
+// CHECK-NEXT: Nearest(3, 1) = 0
+// CHECK-NEXT: Nearest(3, 2) = 0
+// CHECK-NEXT: Nearest(3, 3) = 3
+// CHECK-LABEL: --- PostDominanceInfo ---
+// CHECK-NEXT: Nearest(0, 0) = 0
+// CHECK-NEXT: Nearest(0, 1) = 3
+// CHECK-NEXT: Nearest(0, 2) = 3
+// CHECK-NEXT: Nearest(0, 3) = 3
+// CHECK: Nearest(1, 0) = 3
+// CHECK-NEXT: Nearest(1, 1) = 1
+// CHECK-NEXT: Nearest(1, 2) = 3
+// CHECK-NEXT: Nearest(1, 3) = 3
+// CHECK: Nearest(2, 0) = 3
+// CHECK-NEXT: Nearest(2, 1) = 3
+// CHECK-NEXT: Nearest(2, 2) = 2
+// CHECK-NEXT: Nearest(2, 3) = 3
+// CHECK: Nearest(3, 0) = 3
+// CHECK-NEXT: Nearest(3, 1) = 3
+// CHECK-NEXT: Nearest(3, 2) = 3
+// CHECK-NEXT: Nearest(3, 3) = 3
+
+// -----
+
+// CHECK-LABEL: Testing : func_loop
+func @func_loop(%arg0 : i32, %arg1 : i32) {
+  br ^loopHeader(%arg0 : i32)
+^loopHeader(%counter : i32):
+  %lessThan = cmpi "slt", %counter, %arg1 : i32
+  cond_br %lessThan, ^loopBody, ^exit
+^loopBody:
+  %const0 = constant 1 : i32
+  %inc = addi %counter, %const0 : i32
+  br ^loopHeader(%inc : i32)
+^exit:
+  return
+}
+// CHECK-LABEL: --- DominanceInfo ---
+// CHECK: Nearest(1, 0) = 0
+// CHECK-NEXT: Nearest(1, 1) = 1
+// CHECK-NEXT: Nearest(1, 2) = 1
+// CHECK-NEXT: Nearest(1, 3) = 1
+// CHECK: Nearest(2, 0) = 0
+// CHECK-NEXT: Nearest(2, 1) = 1
+// CHECK-NEXT: Nearest(2, 2) = 2
+// CHECK-NEXT: Nearest(2, 3) = 1
+// CHECK: Nearest(3, 0) = 0
+// CHECK-NEXT: Nearest(3, 1) = 1
+// CHECK-NEXT: Nearest(3, 2) = 1
+// CHECK-NEXT: Nearest(3, 3) = 3
+// CHECK-LABEL: --- PostDominanceInfo ---
+// CHECK: Nearest(1, 0) = 1
+// CHECK-NEXT: Nearest(1, 1) = 1
+// CHECK-NEXT: Nearest(1, 2) = 1
+// CHECK-NEXT: Nearest(1, 3) = 3
+// CHECK: Nearest(2, 0) = 1
+// CHECK-NEXT: Nearest(2, 1) = 1
+// CHECK-NEXT: Nearest(2, 2) = 2
+// CHECK-NEXT: Nearest(2, 3) = 3
+// CHECK: Nearest(3, 0) = 3
+// CHECK-NEXT: Nearest(3, 1) = 3
+// CHECK-NEXT: Nearest(3, 2) = 3
+// CHECK-NEXT: Nearest(3, 3) = 3
+
+// -----
+
+// CHECK-LABEL: Testing : nested_region
+func @nested_region(%arg0 : index, %arg1 : index, %arg2 : index) {
+  loop.for %arg3 = %arg0 to %arg1 step %arg2 { }
+  return
+}
+
+// CHECK-LABEL: --- DominanceInfo ---
+// CHECK-NEXT: Nearest(0, 0) = 0
+// CHECK-NEXT: Nearest(0, 1) = 1
+// CHECK: Nearest(1, 0) = 1
+// CHECK-NEXT: Nearest(1, 1) = 1
+// CHECK-LABEL: --- PostDominanceInfo ---
+// CHECK-NEXT: Nearest(0, 0) = 0
+// CHECK-NEXT: Nearest(0, 1) = 1
+// CHECK: Nearest(1, 0) = 1
+// CHECK-NEXT: Nearest(1, 1) = 1
+
+// -----
+
+// CHECK-LABEL: Testing : nested_region2
+func @nested_region2(%arg0 : index, %arg1 : index, %arg2 : index) {
+  loop.for %arg3 = %arg0 to %arg1 step %arg2 {
+    loop.for %arg4 = %arg0 to %arg1 step %arg2 {
+      loop.for %arg5 = %arg0 to %arg1 step %arg2 { }
+    }
+  }
+  return
+}
+// CHECK-LABEL: --- DominanceInfo ---
+// CHECK: Nearest(1, 0) = 1
+// CHECK-NEXT: Nearest(1, 1) = 1
+// CHECK-NEXT: Nearest(1, 2) = 2
+// CHECK-NEXT: Nearest(1, 3) = 3
+// CHECK: Nearest(2, 0) = 2
+// CHECK-NEXT: Nearest(2, 1) = 2
+// CHECK-NEXT: Nearest(2, 2) = 2
+// CHECK-NEXT: Nearest(2, 3) = 3
+// CHECK: Nearest(3, 0) = 3
+// CHECK-NEXT: Nearest(3, 1) = 3
+// CHECK-NEXT: Nearest(3, 2) = 3
+// CHECK-NEXT: Nearest(3, 3) = 3
+// CHECK-LABEL: --- PostDominanceInfo ---
+// CHECK-NEXT: Nearest(0, 0) = 0
+// CHECK-NEXT: Nearest(0, 1) = 1
+// CHECK-NEXT: Nearest(0, 2) = 2
+// CHECK-NEXT: Nearest(0, 3) = 3
+// CHECK: Nearest(1, 0) = 1
+// CHECK-NEXT: Nearest(1, 1) = 1
+// CHECK-NEXT: Nearest(1, 2) = 2
+// CHECK-NEXT: Nearest(1, 3) = 3
+// CHECK: Nearest(2, 0) = 2
+// CHECK-NEXT: Nearest(2, 1) = 2
+// CHECK-NEXT: Nearest(2, 2) = 2
+// CHECK-NEXT: Nearest(2, 3) = 3
+
+// -----
+
+// CHECK-LABEL: Testing : func_loop_nested_region
+func @func_loop_nested_region(
+  %arg0 : i32,
+  %arg1 : i32,
+  %arg2 : index,
+  %arg3 : index,
+  %arg4 : index) {
+  br ^loopHeader(%arg0 : i32)
+^loopHeader(%counter : i32):
+  %lessThan = cmpi "slt", %counter, %arg1 : i32
+  cond_br %lessThan, ^loopBody, ^exit
+^loopBody:
+  %const0 = constant 1 : i32
+  %inc = addi %counter, %const0 : i32
+  loop.for %arg5 = %arg2 to %arg3 step %arg4 {
+    loop.for %arg6 = %arg2 to %arg3 step %arg4 { }
+  }
+  br ^loopHeader(%inc : i32)
+^exit:
+  return
+}
+// CHECK-LABEL: --- DominanceInfo ---
+// CHECK: Nearest(2, 0) = 0
+// CHECK-NEXT: Nearest(2, 1) = 1
+// CHECK-NEXT: Nearest(2, 2) = 2
+// CHECK-NEXT: Nearest(2, 3) = 2
+// CHECK-NEXT: Nearest(2, 4) = 2
+// CHECK-NEXT: Nearest(2, 5) = 1
+// CHECK: Nearest(3, 0) = 0
+// CHECK-NEXT: Nearest(3, 1) = 1
+// CHECK-NEXT: Nearest(3, 2) = 2
+// CHECK-NEXT: Nearest(3, 3) = 3
+// CHECK-NEXT: Nearest(3, 4) = 4
+// CHECK-NEXT: Nearest(3, 5) = 1
+// CHECK: Nearest(4, 0) = 0
+// CHECK-NEXT: Nearest(4, 1) = 1
+// CHECK-NEXT: Nearest(4, 2) = 2
+// CHECK-NEXT: Nearest(4, 3) = 4
+// CHECK-NEXT: Nearest(4, 4) = 4
+// CHECK-NEXT: Nearest(4, 5) = 1
+// CHECK-LABEL: --- PostDominanceInfo ---
+// CHECK: Nearest(2, 0) = 1
+// CHECK-NEXT: Nearest(2, 1) = 1
+// CHECK-NEXT: Nearest(2, 2) = 2
+// CHECK-NEXT: Nearest(2, 3) = 2
+// CHECK-NEXT: Nearest(2, 4) = 2
+// CHECK-NEXT: Nearest(2, 5) = 5
+// CHECK: Nearest(3, 0) = 1
+// CHECK-NEXT: Nearest(3, 1) = 1
+// CHECK-NEXT: Nearest(3, 2) = 2
+// CHECK-NEXT: Nearest(3, 3) = 3
+// CHECK-NEXT: Nearest(3, 4) = 4
+// CHECK-NEXT: Nearest(3, 5) = 5
+// CHECK: Nearest(4, 0) = 1
+// CHECK-NEXT: Nearest(4, 1) = 1
+// CHECK-NEXT: Nearest(4, 2) = 2
+// CHECK-NEXT: Nearest(4, 3) = 4
+// CHECK-NEXT: Nearest(4, 4) = 4
+// CHECK-NEXT: Nearest(4, 5) = 5
\ No newline at end of file
diff --git a/mlir/test/CMakeLists.txt b/mlir/test/CMakeLists.txt
index 97584f81d2701..91bad3331dcad 100644
--- a/mlir/test/CMakeLists.txt
+++ b/mlir/test/CMakeLists.txt
@@ -10,6 +10,7 @@ llvm_canonicalize_cmake_booleans(
 # Passed to lit.site.cfg.py.in to set up the path where to find the libraries
 # for linalg integration tests.
 set(MLIR_DIALECT_LINALG_INTEGRATION_TEST_LIB_DIR ${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
+set(MLIR_RUNNER_UTILS_DIR ${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
 
 # Passed to lit.site.cfg.py.in to set up the path where to find the libraries
 # for the mlir cuda runner tests.
diff --git a/mlir/test/Transforms/lower-affine.mlir b/mlir/test/Conversion/AffineToStandard/lower-affine.mlir
similarity index 98%
rename from mlir/test/Transforms/lower-affine.mlir
rename to mlir/test/Conversion/AffineToStandard/lower-affine.mlir
index 18b410ae7f58d..18ef4e201c8d1 100644
--- a/mlir/test/Transforms/lower-affine.mlir
+++ b/mlir/test/Conversion/AffineToStandard/lower-affine.mlir
@@ -344,7 +344,7 @@ func @loop_min_max(%N : index) {
   return
 }
 
-#map_7_values = affine_map<(i) -> (i, i, i, i, i, i, i)>
+#map_7_values = affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d0, d1, d2, d3, d4, d5, d6)>
 
 // Check that the "min" (cmpi "slt" + select) reduction sequence is emitted
 // correctly for a an affine map with 7 results.
@@ -369,8 +369,8 @@ func @loop_min_max(%N : index) {
 // CHECK-NEXT:   }
 // CHECK-NEXT:   return
 // CHECK-NEXT: }
-func @min_reduction_tree(%v : index) {
-  affine.for %i = 0 to min #map_7_values(%v)[] {
+func @min_reduction_tree(%v1 : index, %v2 : index, %v3 : index, %v4 : index, %v5 : index, %v6 : index, %v7 : index) {
+  affine.for %i = 0 to min #map_7_values(%v1, %v2, %v3, %v4, %v5, %v6, %v7)[] {
     call @body(%i) : (index) -> ()
   }
   return
@@ -619,4 +619,4 @@ func @affine_max(%arg0: index, %arg1: index) -> index{
   // CHECK: select %[[cmp]], %[[first]], %[[second]]
   %0 = affine.max affine_map<(d0,d1) -> (d0 - d1, d1 - d0)>(%arg0, %arg1)
   return %0 : index
-}
\ No newline at end of file
+}
diff --git a/mlir/test/Conversion/GPUToCUDA/lower-launch-func-to-cuda.mlir b/mlir/test/Conversion/GPUToCUDA/lower-launch-func-to-cuda.mlir
index bb02b5d74b53f..17244ce9f066c 100644
--- a/mlir/test/Conversion/GPUToCUDA/lower-launch-func-to-cuda.mlir
+++ b/mlir/test/Conversion/GPUToCUDA/lower-launch-func-to-cuda.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s --launch-func-to-cuda | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s --launch-func-to-cuda | FileCheck %s
 
 module attributes {gpu.container_module} {
 
diff --git a/mlir/test/Conversion/GPUToNVVM/memory-attrbution.mlir b/mlir/test/Conversion/GPUToNVVM/memory-attrbution.mlir
index c6d080fde2aa6..68b615725ad43 100644
--- a/mlir/test/Conversion/GPUToNVVM/memory-attrbution.mlir
+++ b/mlir/test/Conversion/GPUToNVVM/memory-attrbution.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt --convert-gpu-to-nvvm --split-input-file %s | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect --convert-gpu-to-nvvm --split-input-file %s | FileCheck %s
 
 gpu.module @kernel {
   // CHECK-LABEL:  llvm.func @private
diff --git a/mlir/test/Conversion/GPUToSPIRV/simple.mlir b/mlir/test/Conversion/GPUToSPIRV/simple.mlir
index 3076cd04b9fe1..9cf0f5045d731 100644
--- a/mlir/test/Conversion/GPUToSPIRV/simple.mlir
+++ b/mlir/test/Conversion/GPUToSPIRV/simple.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -split-input-file -convert-gpu-to-spirv -verify-diagnostics %s -o - | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect -split-input-file -convert-gpu-to-spirv -verify-diagnostics %s -o - | FileCheck %s
 
 module attributes {gpu.container_module} {
   gpu.module @kernels {
diff --git a/mlir/test/Conversion/StandardToLLVM/calling-convention.mlir b/mlir/test/Conversion/StandardToLLVM/calling-convention.mlir
index 348afff339673..87bdab2680f99 100644
--- a/mlir/test/Conversion/StandardToLLVM/calling-convention.mlir
+++ b/mlir/test/Conversion/StandardToLLVM/calling-convention.mlir
@@ -1,4 +1,5 @@
 // RUN: mlir-opt -convert-std-to-llvm='emit-c-wrappers=1' %s | FileCheck %s
+// RUN: mlir-opt -convert-std-to-llvm %s | FileCheck %s --check-prefix=EMIT_C_ATTRIBUTE
 
 // This tests the default memref calling convention and the emission of C
 // wrappers. We don't need to separate runs because the wrapper-emission
@@ -72,6 +73,7 @@ func @caller() {
 }
 
 // CHECK-LABEL: @callee
+// EMIT_C_ATTRIBUTE-LABEL: @callee
 func @callee(%arg0: memref<?xf32>, %arg1: index) {
   %0 = load %arg0[%arg1] : memref<?xf32>
   return
@@ -93,3 +95,17 @@ func @callee(%arg0: memref<?xf32>, %arg1: index) {
   // Forward the descriptor components to the call.
   // CHECK: llvm.call @callee(%[[ALLOC]], %[[ALIGN]], %[[OFFSET]], %[[SIZE]], %[[STRIDE]], %{{.*}}) : (!llvm<"float*">, !llvm<"float*">, !llvm.i64, !llvm.i64, !llvm.i64, !llvm.i64) -> ()
 
+//   EMIT_C_ATTRIBUTE-NOT: @mlir_ciface_callee
+
+// CHECK-LABEL: @other_callee
+// EMIT_C_ATTRIBUTE-LABEL: @other_callee
+func @other_callee(%arg0: memref<?xf32>, %arg1: index) attributes { llvm.emit_c_interface } {
+  %0 = load %arg0[%arg1] : memref<?xf32>
+  return
+}
+
+// CHECK: @_mlir_ciface_other_callee
+// CHECK:   llvm.call @other_callee
+
+// EMIT_C_ATTRIBUTE: @_mlir_ciface_other_callee
+// EMIT_C_ATTRIBUTE:   llvm.call @other_callee
diff --git a/mlir/test/Conversion/StandardToLLVM/convert-to-llvmir.mlir b/mlir/test/Conversion/StandardToLLVM/convert-to-llvmir.mlir
index 699ea31836a5c..9c8d47db05b83 100644
--- a/mlir/test/Conversion/StandardToLLVM/convert-to-llvmir.mlir
+++ b/mlir/test/Conversion/StandardToLLVM/convert-to-llvmir.mlir
@@ -1,4 +1,5 @@
 // RUN: mlir-opt -convert-std-to-llvm %s -split-input-file | FileCheck %s
+// RUN: mlir-opt -convert-std-to-llvm='index-bitwidth=32' %s -split-input-file | FileCheck --check-prefix=CHECK32 %s
 
 // CHECK-LABEL: func @empty() {
 // CHECK-NEXT:  llvm.return
@@ -12,15 +13,21 @@ func @empty() {
 func @body(index)
 
 // CHECK-LABEL: func @simple_loop() {
+// CHECK32-LABEL: func @simple_loop() {
 func @simple_loop() {
 ^bb0:
 // CHECK-NEXT:  llvm.br ^bb1
+// CHECK32-NEXT:  llvm.br ^bb1
   br ^bb1
 
 // CHECK-NEXT: ^bb1:	// pred: ^bb0
 // CHECK-NEXT:  {{.*}} = llvm.mlir.constant(1 : index) : !llvm.i64
 // CHECK-NEXT:  {{.*}} = llvm.mlir.constant(42 : index) : !llvm.i64
 // CHECK-NEXT:  llvm.br ^bb2({{.*}} : !llvm.i64)
+// CHECK32-NEXT: ^bb1:	// pred: ^bb0
+// CHECK32-NEXT:  {{.*}} = llvm.mlir.constant(1 : index) : !llvm.i32
+// CHECK32-NEXT:  {{.*}} = llvm.mlir.constant(42 : index) : !llvm.i32
+// CHECK32-NEXT:  llvm.br ^bb2({{.*}} : !llvm.i32)
 ^bb1:	// pred: ^bb0
   %c1 = constant 1 : index
   %c42 = constant 42 : index
@@ -29,6 +36,9 @@ func @simple_loop() {
 // CHECK:      ^bb2({{.*}}: !llvm.i64):	// 2 preds: ^bb1, ^bb3
 // CHECK-NEXT:  {{.*}} = llvm.icmp "slt" {{.*}}, {{.*}} : !llvm.i64
 // CHECK-NEXT:  llvm.cond_br {{.*}}, ^bb3, ^bb4
+// CHECK32:      ^bb2({{.*}}: !llvm.i32):	// 2 preds: ^bb1, ^bb3
+// CHECK32-NEXT:  {{.*}} = llvm.icmp "slt" {{.*}}, {{.*}} : !llvm.i32
+// CHECK32-NEXT:  llvm.cond_br {{.*}}, ^bb3, ^bb4
 ^bb2(%0: index):	// 2 preds: ^bb1, ^bb3
   %1 = cmpi "slt", %0, %c42 : index
   cond_br %1, ^bb3, ^bb4
@@ -38,6 +48,11 @@ func @simple_loop() {
 // CHECK-NEXT:  {{.*}} = llvm.mlir.constant(1 : index) : !llvm.i64
 // CHECK-NEXT:  {{.*}} = llvm.add {{.*}}, {{.*}} : !llvm.i64
 // CHECK-NEXT:  llvm.br ^bb2({{.*}} : !llvm.i64)
+// CHECK32:      ^bb3:	// pred: ^bb2
+// CHECK32-NEXT:  llvm.call @body({{.*}}) : (!llvm.i32) -> ()
+// CHECK32-NEXT:  {{.*}} = llvm.mlir.constant(1 : index) : !llvm.i32
+// CHECK32-NEXT:  {{.*}} = llvm.add {{.*}}, {{.*}} : !llvm.i32
+// CHECK32-NEXT:  llvm.br ^bb2({{.*}} : !llvm.i32)
 ^bb3:	// pred: ^bb2
   call @body(%0) : (index) -> ()
   %c1_0 = constant 1 : index
@@ -81,13 +96,18 @@ func @ml_caller() {
 }
 
 // CHECK-LABEL: func @body_args(!llvm.i64) -> !llvm.i64
+// CHECK32-LABEL: func @body_args(!llvm.i32) -> !llvm.i32
 func @body_args(index) -> index
 // CHECK-LABEL: func @other(!llvm.i64, !llvm.i32) -> !llvm.i32
+// CHECK32-LABEL: func @other(!llvm.i32, !llvm.i32) -> !llvm.i32
 func @other(index, i32) -> i32
 
 // CHECK-LABEL: func @func_args(%arg0: !llvm.i32, %arg1: !llvm.i32) -> !llvm.i32 {
 // CHECK-NEXT:  {{.*}} = llvm.mlir.constant(0 : i32) : !llvm.i32
 // CHECK-NEXT:  llvm.br ^bb1
+// CHECK32-LABEL: func @func_args(%arg0: !llvm.i32, %arg1: !llvm.i32) -> !llvm.i32 {
+// CHECK32-NEXT:  {{.*}} = llvm.mlir.constant(0 : i32) : !llvm.i32
+// CHECK32-NEXT:  llvm.br ^bb1
 func @func_args(i32, i32) -> i32 {
 ^bb0(%arg0: i32, %arg1: i32):
   %c0_i32 = constant 0 : i32
@@ -97,6 +117,10 @@ func @func_args(i32, i32) -> i32 {
 // CHECK-NEXT:  {{.*}} = llvm.mlir.constant(0 : index) : !llvm.i64
 // CHECK-NEXT:  {{.*}} = llvm.mlir.constant(42 : index) : !llvm.i64
 // CHECK-NEXT:  llvm.br ^bb2({{.*}} : !llvm.i64)
+// CHECK32-NEXT: ^bb1:	// pred: ^bb0
+// CHECK32-NEXT:  {{.*}} = llvm.mlir.constant(0 : index) : !llvm.i32
+// CHECK32-NEXT:  {{.*}} = llvm.mlir.constant(42 : index) : !llvm.i32
+// CHECK32-NEXT:  llvm.br ^bb2({{.*}} : !llvm.i32)
 ^bb1:	// pred: ^bb0
   %c0 = constant 0 : index
   %c42 = constant 42 : index
@@ -105,6 +129,9 @@ func @func_args(i32, i32) -> i32 {
 // CHECK-NEXT: ^bb2({{.*}}: !llvm.i64):	// 2 preds: ^bb1, ^bb3
 // CHECK-NEXT:  {{.*}} = llvm.icmp "slt" {{.*}}, {{.*}} : !llvm.i64
 // CHECK-NEXT:  llvm.cond_br {{.*}}, ^bb3, ^bb4
+// CHECK32-NEXT: ^bb2({{.*}}: !llvm.i32):	// 2 preds: ^bb1, ^bb3
+// CHECK32-NEXT:  {{.*}} = llvm.icmp "slt" {{.*}}, {{.*}} : !llvm.i32
+// CHECK32-NEXT:  llvm.cond_br {{.*}}, ^bb3, ^bb4
 ^bb2(%0: index):	// 2 preds: ^bb1, ^bb3
   %1 = cmpi "slt", %0, %c42 : index
   cond_br %1, ^bb3, ^bb4
@@ -117,6 +144,14 @@ func @func_args(i32, i32) -> i32 {
 // CHECK-NEXT:  {{.*}} = llvm.mlir.constant(1 : index) : !llvm.i64
 // CHECK-NEXT:  {{.*}} = llvm.add {{.*}}, {{.*}} : !llvm.i64
 // CHECK-NEXT:  llvm.br ^bb2({{.*}} : !llvm.i64)
+// CHECK32-NEXT: ^bb3:	// pred: ^bb2
+// CHECK32-NEXT:  {{.*}} = llvm.call @body_args({{.*}}) : (!llvm.i32) -> !llvm.i32
+// CHECK32-NEXT:  {{.*}} = llvm.call @other({{.*}}, %arg0) : (!llvm.i32, !llvm.i32) -> !llvm.i32
+// CHECK32-NEXT:  {{.*}} = llvm.call @other({{.*}}, {{.*}}) : (!llvm.i32, !llvm.i32) -> !llvm.i32
+// CHECK32-NEXT:  {{.*}} = llvm.call @other({{.*}}, %arg1) : (!llvm.i32, !llvm.i32) -> !llvm.i32
+// CHECK32-NEXT:  {{.*}} = llvm.mlir.constant(1 : index) : !llvm.i32
+// CHECK32-NEXT:  {{.*}} = llvm.add {{.*}}, {{.*}} : !llvm.i32
+// CHECK32-NEXT:  llvm.br ^bb2({{.*}} : !llvm.i32)
 ^bb3:	// pred: ^bb2
   %2 = call @body_args(%0) : (index) -> index
   %3 = call @other(%2, %arg0) : (index, i32) -> i32
@@ -130,6 +165,10 @@ func @func_args(i32, i32) -> i32 {
 // CHECK-NEXT:  {{.*}} = llvm.mlir.constant(0 : index) : !llvm.i64
 // CHECK-NEXT:  {{.*}} = llvm.call @other({{.*}}, {{.*}}) : (!llvm.i64, !llvm.i32) -> !llvm.i32
 // CHECK-NEXT:  llvm.return {{.*}} : !llvm.i32
+// CHECK32-NEXT: ^bb4:	// pred: ^bb2
+// CHECK32-NEXT:  {{.*}} = llvm.mlir.constant(0 : index) : !llvm.i32
+// CHECK32-NEXT:  {{.*}} = llvm.call @other({{.*}}, {{.*}}) : (!llvm.i32, !llvm.i32) -> !llvm.i32
+// CHECK32-NEXT:  llvm.return {{.*}} : !llvm.i32
 ^bb4:	// pred: ^bb2
   %c0_0 = constant 0 : index
   %7 = call @other(%c0_0, %c0_i32) : (index, i32) -> i32
@@ -137,12 +176,15 @@ func @func_args(i32, i32) -> i32 {
 }
 
 // CHECK-LABEL: func @pre(!llvm.i64)
+// CHECK32-LABEL: func @pre(!llvm.i32)
 func @pre(index)
 
 // CHECK-LABEL: func @body2(!llvm.i64, !llvm.i64)
+// CHECK32-LABEL: func @body2(!llvm.i32, !llvm.i32)
 func @body2(index, index)
 
 // CHECK-LABEL: func @post(!llvm.i64)
+// CHECK32-LABEL: func @post(!llvm.i32)
 func @post(index)
 
 // CHECK-LABEL: func @imperfectly_nested_loops() {
@@ -326,14 +368,19 @@ func @get_i64() -> (i64)
 // CHECK-LABEL: func @get_f32() -> !llvm.float
 func @get_f32() -> (f32)
 // CHECK-LABEL: func @get_memref() -> !llvm<"{ float*, float*, i64, [4 x i64], [4 x i64] }">
+// CHECK32-LABEL: func @get_memref() -> !llvm<"{ float*, float*, i32, [4 x i32], [4 x i32] }">
 func @get_memref() -> (memref<42x?x10x?xf32>)
 
 // CHECK-LABEL: func @multireturn() -> !llvm<"{ i64, float, { float*, float*, i64, [4 x i64], [4 x i64] } }"> {
+// CHECK32-LABEL: func @multireturn() -> !llvm<"{ i64, float, { float*, float*, i32, [4 x i32], [4 x i32] } }"> {
 func @multireturn() -> (i64, f32, memref<42x?x10x?xf32>) {
 ^bb0:
 // CHECK-NEXT:  {{.*}} = llvm.call @get_i64() : () -> !llvm.i64
 // CHECK-NEXT:  {{.*}} = llvm.call @get_f32() : () -> !llvm.float
 // CHECK-NEXT:  {{.*}} = llvm.call @get_memref() : () -> !llvm<"{ float*, float*, i64, [4 x i64], [4 x i64] }">
+// CHECK32-NEXT:  {{.*}} = llvm.call @get_i64() : () -> !llvm.i64
+// CHECK32-NEXT:  {{.*}} = llvm.call @get_f32() : () -> !llvm.float
+// CHECK32-NEXT:  {{.*}} = llvm.call @get_memref() : () -> !llvm<"{ float*, float*, i32, [4 x i32], [4 x i32] }">
   %0 = call @get_i64() : () -> (i64)
   %1 = call @get_f32() : () -> (f32)
   %2 = call @get_memref() : () -> (memref<42x?x10x?xf32>)
@@ -342,17 +389,27 @@ func @multireturn() -> (i64, f32, memref<42x?x10x?xf32>) {
 // CHECK-NEXT:  {{.*}} = llvm.insertvalue {{.*}}, {{.*}}[1] : !llvm<"{ i64, float, { float*, float*, i64, [4 x i64], [4 x i64] } }">
 // CHECK-NEXT:  {{.*}} = llvm.insertvalue {{.*}}, {{.*}}[2] : !llvm<"{ i64, float, { float*, float*, i64, [4 x i64], [4 x i64] } }">
 // CHECK-NEXT:  llvm.return {{.*}} : !llvm<"{ i64, float, { float*, float*, i64, [4 x i64], [4 x i64] } }">
+// CHECK32-NEXT:  {{.*}} = llvm.mlir.undef : !llvm<"{ i64, float, { float*, float*, i32, [4 x i32], [4 x i32] } }">
+// CHECK32-NEXT:  {{.*}} = llvm.insertvalue {{.*}}, {{.*}}[0] : !llvm<"{ i64, float, { float*, float*, i32, [4 x i32], [4 x i32] } }">
+// CHECK32-NEXT:  {{.*}} = llvm.insertvalue {{.*}}, {{.*}}[1] : !llvm<"{ i64, float, { float*, float*, i32, [4 x i32], [4 x i32] } }">
+// CHECK32-NEXT:  {{.*}} = llvm.insertvalue {{.*}}, {{.*}}[2] : !llvm<"{ i64, float, { float*, float*, i32, [4 x i32], [4 x i32] } }">
+// CHECK32-NEXT:  llvm.return {{.*}} : !llvm<"{ i64, float, { float*, float*, i32, [4 x i32], [4 x i32] } }">
   return %0, %1, %2 : i64, f32, memref<42x?x10x?xf32>
 }
 
 
 // CHECK-LABEL: func @multireturn_caller() {
+// CHECK32-LABEL: func @multireturn_caller() {
 func @multireturn_caller() {
 ^bb0:
 // CHECK-NEXT:  {{.*}} = llvm.call @multireturn() : () -> !llvm<"{ i64, float, { float*, float*, i64, [4 x i64], [4 x i64] } }">
 // CHECK-NEXT:  {{.*}} = llvm.extractvalue {{.*}}[0] : !llvm<"{ i64, float, { float*, float*, i64, [4 x i64], [4 x i64] } }">
 // CHECK-NEXT:  {{.*}} = llvm.extractvalue {{.*}}[1] : !llvm<"{ i64, float, { float*, float*, i64, [4 x i64], [4 x i64] } }">
 // CHECK-NEXT:  {{.*}} = llvm.extractvalue {{.*}}[2] : !llvm<"{ i64, float, { float*, float*, i64, [4 x i64], [4 x i64] } }">
+// CHECK32-NEXT:  {{.*}} = llvm.call @multireturn() : () -> !llvm<"{ i64, float, { float*, float*, i32, [4 x i32], [4 x i32] } }">
+// CHECK32-NEXT:  {{.*}} = llvm.extractvalue {{.*}}[0] : !llvm<"{ i64, float, { float*, float*, i32, [4 x i32], [4 x i32] } }">
+// CHECK32-NEXT:  {{.*}} = llvm.extractvalue {{.*}}[1] : !llvm<"{ i64, float, { float*, float*, i32, [4 x i32], [4 x i32] } }">
+// CHECK32-NEXT:  {{.*}} = llvm.extractvalue {{.*}}[2] : !llvm<"{ i64, float, { float*, float*, i32, [4 x i32], [4 x i32] } }">
   %0:3 = call @multireturn() : () -> (i64, f32, memref<42x?x10x?xf32>)
   %1 = constant 42 : i64
 // CHECK:       {{.*}} = llvm.add {{.*}}, {{.*}} : !llvm.i64
@@ -407,43 +464,41 @@ func @ops(f32, f32, i32, i32, f64) -> (f32, i32) {
 // CHECK-NEXT:  %2 = llvm.icmp "slt" %arg2, %1 : !llvm.i32
   %2 = cmpi "slt", %arg2, %1 : i32
 // CHECK-NEXT:  %3 = llvm.sdiv %arg2, %arg3 : !llvm.i32
-  %4 = divi_signed %arg2, %arg3 : i32
+  %3 = divi_signed %arg2, %arg3 : i32
 // CHECK-NEXT:  %4 = llvm.udiv %arg2, %arg3 : !llvm.i32
-  %5 = divi_unsigned %arg2, %arg3 : i32
+  %4 = divi_unsigned %arg2, %arg3 : i32
 // CHECK-NEXT:  %5 = llvm.srem %arg2, %arg3 : !llvm.i32
-  %6 = remi_signed %arg2, %arg3 : i32
+  %5 = remi_signed %arg2, %arg3 : i32
 // CHECK-NEXT:  %6 = llvm.urem %arg2, %arg3 : !llvm.i32
-  %7 = remi_unsigned %arg2, %arg3 : i32
+  %6 = remi_unsigned %arg2, %arg3 : i32
 // CHECK-NEXT:  %7 = llvm.select %2, %arg2, %arg3 : !llvm.i1, !llvm.i32
-  %8 = select %2, %arg2, %arg3 : i32
+  %7 = select %2, %arg2, %arg3 : i32
 // CHECK-NEXT:  %8 = llvm.fdiv %arg0, %arg1 : !llvm.float
-  %9 = divf %arg0, %arg1 : f32
+  %8 = divf %arg0, %arg1 : f32
 // CHECK-NEXT:  %9 = llvm.frem %arg0, %arg1 : !llvm.float
-  %10 = remf %arg0, %arg1 : f32
+  %9 = remf %arg0, %arg1 : f32
 // CHECK-NEXT: %10 = llvm.and %arg2, %arg3 : !llvm.i32
-  %11 = and %arg2, %arg3 : i32
+  %10 = and %arg2, %arg3 : i32
 // CHECK-NEXT: %11 = llvm.or %arg2, %arg3 : !llvm.i32
-  %12 = or %arg2, %arg3 : i32
+  %11 = or %arg2, %arg3 : i32
 // CHECK-NEXT: %12 = llvm.xor %arg2, %arg3 : !llvm.i32
-  %13 = xor %arg2, %arg3 : i32
+  %12 = xor %arg2, %arg3 : i32
 // CHECK-NEXT: %13 = "llvm.intr.exp"(%arg0) : (!llvm.float) -> !llvm.float
-  %14 = std.exp %arg0 : f32
-// CHECK-NEXT: %14 = llvm.call @tanhf(%arg0) : (!llvm.float) -> !llvm.float
-  %15 = std.tanh %arg0 : f32
+  %13 = std.exp %arg0 : f32
+// CHECK-NEXT: %14 = "llvm.intr.exp2"(%arg0) : (!llvm.float) -> !llvm.float
+  %14 = std.exp2 %arg0 : f32
 // CHECK-NEXT: %15 = llvm.mlir.constant(7.900000e-01 : f64) : !llvm.double
-  %16 = constant 7.9e-01 : f64
-// CHECK-NEXT: %16 = llvm.call @tanh(%15) : (!llvm.double) -> !llvm.double
-  %17 = std.tanh %16 : f64
-// CHECK-NEXT: %17 = llvm.shl %arg2, %arg3 : !llvm.i32
-  %18 = shift_left %arg2, %arg3 : i32
-// CHECK-NEXT: %18 = llvm.ashr %arg2, %arg3 : !llvm.i32
-  %19 = shift_right_signed %arg2, %arg3 : i32
-// CHECK-NEXT: %19 = llvm.lshr %arg2, %arg3 : !llvm.i32
-  %20 = shift_right_unsigned %arg2, %arg3 : i32
+  %15 = constant 7.9e-01 : f64
+// CHECK-NEXT: %16 = llvm.shl %arg2, %arg3 : !llvm.i32
+  %16 = shift_left %arg2, %arg3 : i32
+// CHECK-NEXT: %17 = llvm.ashr %arg2, %arg3 : !llvm.i32
+  %17 = shift_right_signed %arg2, %arg3 : i32
+// CHECK-NEXT: %18 = llvm.lshr %arg2, %arg3 : !llvm.i32
+  %18 = shift_right_unsigned %arg2, %arg3 : i32
 // CHECK-NEXT: %{{[0-9]+}} = "llvm.intr.sqrt"(%arg0) : (!llvm.float) -> !llvm.float
-  %21 = std.sqrt %arg0 : f32
+  %19 = std.sqrt %arg0 : f32
 // CHECK-NEXT: %{{[0-9]+}} = "llvm.intr.sqrt"(%arg4) : (!llvm.double) -> !llvm.double
-  %22 = std.sqrt %arg4 : f64
+  %20 = std.sqrt %arg4 : f64
   return %0, %4 : f32, i32
 }
 
@@ -744,9 +799,16 @@ func @view(%arg0 : index, %arg1 : index, %arg2 : index) {
 // CHECK:         %[[ARG0:[a-zA-Z0-9]*]]: !llvm.i64,
 // CHECK:         %[[ARG1:[a-zA-Z0-9]*]]: !llvm.i64,
 // CHECK:         %[[ARG2:.*]]: !llvm.i64)
+// CHECK32-LABEL: func @subview(
+// CHECK32-COUNT-2: !llvm<"float*">,
+// CHECK32-COUNT-5: {{%[a-zA-Z0-9]*}}: !llvm.i32,
+// CHECK32:         %[[ARG0:[a-zA-Z0-9]*]]: !llvm.i32,
+// CHECK32:         %[[ARG1:[a-zA-Z0-9]*]]: !llvm.i32,
+// CHECK32:         %[[ARG2:.*]]: !llvm.i32)
 func @subview(%0 : memref<64x4xf32, affine_map<(d0, d1) -> (d0 * 4 + d1)>>, %arg0 : index, %arg1 : index, %arg2 : index) {
   // The last "insertvalue" that populates the memref descriptor from the function arguments.
   // CHECK: %[[MEMREF:.*]] = llvm.insertvalue %{{.*}}, %{{.*}}[4, 1]
+  // CHECK32: %[[MEMREF:.*]] = llvm.insertvalue %{{.*}}, %{{.*}}[4, 1]
 
   // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
   // CHECK: %[[DESC0:.*]] = llvm.insertvalue %{{.*}}, %[[DESC]][0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
@@ -765,15 +827,34 @@ func @subview(%0 : memref<64x4xf32, affine_map<(d0, d1) -> (d0 * 4 + d1)>>, %arg
   // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[ARG0]], %[[DESC4]][3, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
   // CHECK: %[[DESCSTRIDE0:.*]] = llvm.mul %[[ARG0]], %[[STRIDE0]] : !llvm.i64
   // CHECK: llvm.insertvalue %[[DESCSTRIDE0]], %[[DESC5]][4, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK32: %[[DESC:.*]] = llvm.mlir.undef : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC0:.*]] = llvm.insertvalue %{{.*}}, %[[DESC]][0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC1:.*]] = llvm.insertvalue %{{.*}}, %[[DESC0]][1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEMREF]][4, 0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEMREF]][4, 1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[OFF:.*]] = llvm.extractvalue %[[MEMREF]][2] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[OFFINC:.*]] = llvm.mul %[[ARG0]], %[[STRIDE0]] : !llvm.i32
+  // CHECK32: %[[OFF1:.*]] = llvm.add %[[OFF]], %[[OFFINC]] : !llvm.i32
+  // CHECK32: %[[OFFINC1:.*]] = llvm.mul %[[ARG1]], %[[STRIDE1]] : !llvm.i32
+  // CHECK32: %[[OFF2:.*]] = llvm.add %[[OFF1]], %[[OFFINC1]] : !llvm.i32
+  // CHECK32: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC3:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC2]][3, 1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESCSTRIDE1:.*]] = llvm.mul %[[ARG1]], %[[STRIDE1]] : !llvm.i32
+  // CHECK32: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE1]], %[[DESC3]][4, 1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC5:.*]] = llvm.insertvalue %[[ARG0]], %[[DESC4]][3, 0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESCSTRIDE0:.*]] = llvm.mul %[[ARG0]], %[[STRIDE0]] : !llvm.i32
+
   %1 = subview %0[%arg0, %arg1][%arg0, %arg1][%arg0, %arg1] :
     memref<64x4xf32, affine_map<(d0, d1) -> (d0 * 4 + d1)>> to memref<?x?xf32, affine_map<(d0, d1)[s0, s1, s2] -> (d0 * s1 + d1 * s2 + s0)>>
   return
 }
 
 // CHECK-LABEL: func @subview_const_size(
+// CHECK32-LABEL: func @subview_const_size(
 func @subview_const_size(%0 : memref<64x4xf32, affine_map<(d0, d1) -> (d0 * 4 + d1)>>, %arg0 : index, %arg1 : index, %arg2 : index) {
   // The last "insertvalue" that populates the memref descriptor from the function arguments.
   // CHECK: %[[MEMREF:.*]] = llvm.insertvalue %{{.*}}, %{{.*}}[4, 1]
+  // CHECK32: %[[MEMREF:.*]] = llvm.insertvalue %{{.*}}, %{{.*}}[4, 1]
 
   // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
   // CHECK: %[[DESC0:.*]] = llvm.insertvalue %{{.*}}, %[[DESC]][0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
@@ -794,15 +875,36 @@ func @subview_const_size(%0 : memref<64x4xf32, affine_map<(d0, d1) -> (d0 * 4 +
   // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[CST4]], %[[DESC4]][3, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
   // CHECK: %[[DESCSTRIDE0:.*]] = llvm.mul %[[ARG0]], %[[STRIDE0]] : !llvm.i64
   // CHECK: llvm.insertvalue %[[DESCSTRIDE0]], %[[DESC5]][4, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK32: %[[DESC:.*]] = llvm.mlir.undef : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC0:.*]] = llvm.insertvalue %{{.*}}, %[[DESC]][0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC1:.*]] = llvm.insertvalue %{{.*}}, %[[DESC0]][1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEMREF]][4, 0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEMREF]][4, 1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[CST4:.*]] = llvm.mlir.constant(4 : i64)
+  // CHECK32: %[[CST2:.*]] = llvm.mlir.constant(2 : i64)
+  // CHECK32: %[[OFF:.*]] = llvm.extractvalue %[[MEMREF]][2] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[OFFINC:.*]] = llvm.mul %[[ARG0]], %[[STRIDE0]] : !llvm.i32
+  // CHECK32: %[[OFF1:.*]] = llvm.add %[[OFF]], %[[OFFINC]] : !llvm.i32
+  // CHECK32: %[[OFFINC1:.*]] = llvm.mul %[[ARG1]], %[[STRIDE1]] : !llvm.i32
+  // CHECK32: %[[OFF2:.*]] = llvm.add %[[OFF1]], %[[OFFINC1]] : !llvm.i32
+  // CHECK32: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC3:.*]] = llvm.insertvalue %[[CST2]], %[[DESC2]][3, 1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESCSTRIDE1:.*]] = llvm.mul %[[ARG1]], %[[STRIDE1]] : !llvm.i32
+  // CHECK32: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE1]], %[[DESC3]][4, 1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC5:.*]] = llvm.insertvalue %[[CST4]], %[[DESC4]][3, 0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESCSTRIDE0:.*]] = llvm.mul %[[ARG0]], %[[STRIDE0]] : !llvm.i32
+  // CHECK32: llvm.insertvalue %[[DESCSTRIDE0]], %[[DESC5]][4, 0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
   %1 = subview %0[%arg0, %arg1][][%arg0, %arg1] :
     memref<64x4xf32, affine_map<(d0, d1) -> (d0 * 4 + d1)>> to memref<4x2xf32, affine_map<(d0, d1)[s0, s1, s2] -> (d0 * s1 + d1 * s2 + s0)>>
   return
 }
 
 // CHECK-LABEL: func @subview_const_stride(
+// CHECK32-LABEL: func @subview_const_stride(
 func @subview_const_stride(%0 : memref<64x4xf32, affine_map<(d0, d1) -> (d0 * 4 + d1)>>, %arg0 : index, %arg1 : index, %arg2 : index) {
   // The last "insertvalue" that populates the memref descriptor from the function arguments.
   // CHECK: %[[MEMREF:.*]] = llvm.insertvalue %{{.*}}, %{{.*}}[4, 1]
+  // CHECK32: %[[MEMREF:.*]] = llvm.insertvalue %{{.*}}, %{{.*}}[4, 1]
 
   // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
   // CHECK: %[[DESC0:.*]] = llvm.insertvalue %{{.*}}, %[[DESC]][0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
@@ -821,15 +923,34 @@ func @subview_const_stride(%0 : memref<64x4xf32, affine_map<(d0, d1) -> (d0 * 4
   // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[ARG0]], %[[DESC4]][3, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
   // CHECK: %[[CST4:.*]] = llvm.mlir.constant(4 : i64)
   // CHECK: llvm.insertvalue %[[CST4]], %[[DESC5]][4, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK32: %[[DESC:.*]] = llvm.mlir.undef : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC0:.*]] = llvm.insertvalue %{{.*}}, %[[DESC]][0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC1:.*]] = llvm.insertvalue %{{.*}}, %[[DESC0]][1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEMREF]][4, 0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEMREF]][4, 1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[OFF:.*]] = llvm.extractvalue %[[MEMREF]][2] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[OFFINC:.*]] = llvm.mul %[[ARG0]], %[[STRIDE0]] : !llvm.i32
+  // CHECK32: %[[OFF1:.*]] = llvm.add %[[OFF]], %[[OFFINC]] : !llvm.i32
+  // CHECK32: %[[OFFINC1:.*]] = llvm.mul %[[ARG1]], %[[STRIDE1]] : !llvm.i32
+  // CHECK32: %[[OFF2:.*]] = llvm.add %[[OFF1]], %[[OFFINC1]] : !llvm.i32
+  // CHECK32: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC3:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC2]][3, 1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[CST2:.*]] = llvm.mlir.constant(2 : i64)
+  // CHECK32: %[[DESC4:.*]] = llvm.insertvalue %[[CST2]], %[[DESC3]][4, 1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC5:.*]] = llvm.insertvalue %[[ARG0]], %[[DESC4]][3, 0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[CST4:.*]] = llvm.mlir.constant(4 : i64)
+  // CHECK32: llvm.insertvalue %[[CST4]], %[[DESC5]][4, 0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
   %1 = subview %0[%arg0, %arg1][%arg0, %arg1][] :
     memref<64x4xf32, affine_map<(d0, d1) -> (d0 * 4 + d1)>> to memref<?x?xf32, affine_map<(d0, d1)[s0] -> (d0 * 4 + d1 * 2 + s0)>>
   return
 }
 
 // CHECK-LABEL: func @subview_const_stride_and_offset(
+// CHECK32-LABEL: func @subview_const_stride_and_offset(
 func @subview_const_stride_and_offset(%0 : memref<64x4xf32, affine_map<(d0, d1) -> (d0 * 4 + d1)>>) {
   // The last "insertvalue" that populates the memref descriptor from the function arguments.
   // CHECK: %[[MEMREF:.*]] = llvm.insertvalue %{{.*}}, %{{.*}}[4, 1]
+  // CHECK32: %[[MEMREF:.*]] = llvm.insertvalue %{{.*}}, %{{.*}}[4, 1]
 
   // CHECK: %[[DESC:.*]] = llvm.mlir.undef : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
   // CHECK: %[[DESC0:.*]] = llvm.insertvalue %{{.*}}, %[[DESC]][0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
@@ -846,6 +967,21 @@ func @subview_const_stride_and_offset(%0 : memref<64x4xf32, affine_map<(d0, d1)
   // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[CST62]], %[[DESC4]][3, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
   // CHECK: %[[CST4:.*]] = llvm.mlir.constant(4 : i64)
   // CHECK: llvm.insertvalue %[[CST4]], %[[DESC5]][4, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }">
+  // CHECK32: %[[DESC:.*]] = llvm.mlir.undef : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC0:.*]] = llvm.insertvalue %{{.*}}, %[[DESC]][0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC1:.*]] = llvm.insertvalue %{{.*}}, %[[DESC0]][1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEMREF]][4, 0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEMREF]][4, 1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[CST62:.*]] = llvm.mlir.constant(62 : i64)
+  // CHECK32: %[[CST3:.*]] = llvm.mlir.constant(3 : i64)
+  // CHECK32: %[[CST8:.*]] = llvm.mlir.constant(8 : index)
+  // CHECK32: %[[DESC2:.*]] = llvm.insertvalue %[[CST8]], %[[DESC1]][2] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC3:.*]] = llvm.insertvalue %[[CST3]], %[[DESC2]][3, 1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[CST1:.*]] = llvm.mlir.constant(1 : i64)
+  // CHECK32: %[[DESC4:.*]] = llvm.insertvalue %[[CST1]], %[[DESC3]][4, 1] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[DESC5:.*]] = llvm.insertvalue %[[CST62]], %[[DESC4]][3, 0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
+  // CHECK32: %[[CST4:.*]] = llvm.mlir.constant(4 : i64)
+  // CHECK32: llvm.insertvalue %[[CST4]], %[[DESC5]][4, 0] : !llvm<"{ float*, float*, i32, [2 x i32], [2 x i32] }">
   %1 = subview %0[][][] :
     memref<64x4xf32, affine_map<(d0, d1) -> (d0 * 4 + d1)>> to memref<62x3xf32, affine_map<(d0, d1) -> (d0 * 4 + d1 + 8)>>
   return
@@ -853,22 +989,6 @@ func @subview_const_stride_and_offset(%0 : memref<64x4xf32, affine_map<(d0, d1)
 
 // -----
 
-module {
-  func @check_tanh_func_added_only_once_to_symbol_table(%f: f32, %lf: f64) -> () {
-    %f0 = std.tanh %f : f32
-    %f1 = std.tanh %f0 : f32
-    %lf0 = std.tanh %lf : f64
-    %lf1 = std.tanh %lf0 : f64
-    return
-  }
-// CHECK: module {
-// CHECK: llvm.func @tanh(!llvm.double) -> !llvm.double
-// CHECK: llvm.func @tanhf(!llvm.float) -> !llvm.float
-// CHECK-LABEL: func @check_tanh_func_added_only_once_to_symbol_table
-}
-
-// -----
-
 // CHECK-LABEL: func @atomic_rmw
 func @atomic_rmw(%I : memref<10xi32>, %ival : i32, %F : memref<10xf32>, %fval : f32, %i : index) {
   atomic_rmw "assign" %fval, %F[%i] : (f32, memref<10xf32>) -> f32
diff --git a/mlir/test/Conversion/StandardToLLVM/standard-to-llvm.mlir b/mlir/test/Conversion/StandardToLLVM/standard-to-llvm.mlir
index 6aec959314841..b6bc81311abf1 100644
--- a/mlir/test/Conversion/StandardToLLVM/standard-to-llvm.mlir
+++ b/mlir/test/Conversion/StandardToLLVM/standard-to-llvm.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -convert-std-to-llvm -split-input-file -verify-diagnostics | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -convert-std-to-llvm -split-input-file -verify-diagnostics | FileCheck %s
 
 // CHECK-LABEL: func @address_space(
 // CHECK-SAME:    !llvm<"float addrspace(7)*">
diff --git a/mlir/test/Conversion/StandardToSPIRV/std-to-spirv.mlir b/mlir/test/Conversion/StandardToSPIRV/std-to-spirv.mlir
index 91219acc0bd54..59e1c8016a547 100644
--- a/mlir/test/Conversion/StandardToSPIRV/std-to-spirv.mlir
+++ b/mlir/test/Conversion/StandardToSPIRV/std-to-spirv.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -split-input-file -convert-std-to-spirv %s -o - | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect -split-input-file -convert-std-to-spirv %s -o - | FileCheck %s
 
 //===----------------------------------------------------------------------===//
 // std arithmetic ops
diff --git a/mlir/test/Conversion/StandardToSPIRV/std-types-to-spirv.mlir b/mlir/test/Conversion/StandardToSPIRV/std-types-to-spirv.mlir
index 81911bd1a6332..8ad15cf111041 100644
--- a/mlir/test/Conversion/StandardToSPIRV/std-types-to-spirv.mlir
+++ b/mlir/test/Conversion/StandardToSPIRV/std-types-to-spirv.mlir
@@ -510,7 +510,7 @@ func @dynamic_dim_memref(%arg0: memref<8x?xi32>) { return }
 // Tensor types
 //===----------------------------------------------------------------------===//
 
-// Check that tensor element types are kept untouched with proper capabilites.
+// Check that tensor element types are kept untouched with proper capabilities.
 module attributes {
   spv.target_env = #spv.target_env<
     #spv.vce<v1.0, [Int8, Int16, Int64, Float16, Float64], []>,
diff --git a/mlir/test/Dialect/Affine/SuperVectorize/compose_maps.mlir b/mlir/test/Dialect/Affine/SuperVectorize/compose_maps.mlir
index ca1641762838c..126a176fa8362 100644
--- a/mlir/test/Dialect/Affine/SuperVectorize/compose_maps.mlir
+++ b/mlir/test/Dialect/Affine/SuperVectorize/compose_maps.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -affine-super-vectorizer-test -compose-maps 2>&1 |  FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -affine-super-vectorizer-test -compose-maps 2>&1 |  FileCheck %s
 
 // For all these cases, the test traverses the `test_affine_map` ops and
 // composes them in order one-by-one.
diff --git a/mlir/test/Dialect/Affine/affine-data-copy.mlir b/mlir/test/Dialect/Affine/affine-data-copy.mlir
index e9543e5280b9b..48e7908aac0a9 100644
--- a/mlir/test/Dialect/Affine/affine-data-copy.mlir
+++ b/mlir/test/Dialect/Affine/affine-data-copy.mlir
@@ -1,6 +1,6 @@
 // RUN: mlir-opt %s -split-input-file -affine-data-copy-generate -affine-data-copy-generate-dma=false -affine-data-copy-generate-fast-mem-space=0 -affine-data-copy-generate-skip-non-unit-stride-loops | FileCheck %s
 // Small buffer size to trigger fine copies.
-// RUN: mlir-opt %s -affine-data-copy-generate -affine-data-copy-generate-dma=false -affine-data-copy-generate-fast-mem-space=0 -affine-data-copy-generate-fast-mem-capacity=1 | FileCheck --check-prefix=CHECK-SMALL %s
+// RUN: mlir-opt %s -split-input-file -affine-data-copy-generate -affine-data-copy-generate-dma=false -affine-data-copy-generate-fast-mem-space=0 -affine-data-copy-generate-fast-mem-capacity=1 | FileCheck --check-prefix=CHECK-SMALL %s
 
 // Test affine data copy with a memref filter. We use a test pass that invokes
 // affine data copy utility on the input loop nest.
@@ -14,12 +14,14 @@
 // footprint -- so that one could write a definite test case and not have to
 // update it each time something related to the cost functions change.
 
-#map0 = affine_map<(d0) -> (d0)>
-#map1 = affine_map<(d0) -> (d0 + 128)>
+#id = affine_map<(d0) -> (d0)>
+#ub = affine_map<(d0) -> (d0 + 128)>
 
 // Map used to index the original memref while copying.
 // CHECK-DAG: [[MEM_IDX_MAP:map[0-9]+]] = affine_map<(d0, d1) -> (d0 + d1)>
 // Map used to index the buffer while computing.
+// CHECK-DAG: [[MAP_IDENTITY:map[0-9]+]] = affine_map<(d0) -> (d0)>
+// CHECK-DAG: [[MAP_PLUS_128:map[0-9]+]] = affine_map<(d0) -> (d0 + 128)>
 // CHECK-DAG: [[BUF_IDX_MAP:map[0-9]+]] = affine_map<(d0, d1, d2, d3) -> (-d0 + d2, -d1 + d3)>
 
 // CHECK-LABEL: func @matmul
@@ -28,9 +30,9 @@ func @matmul(%A: memref<4096x4096xf32>, %B: memref<4096x4096xf32>, %C: memref<40
   affine.for %i = 0 to 4096 step 128 {
     affine.for %j = 0 to 4096 step 128 {
       affine.for %k = 0 to 4096 step 128 {
-        affine.for %ii = #map0(%i) to #map1(%i) {
-          affine.for %jj = #map0(%j) to #map1(%j) {
-            affine.for %kk = #map0(%k) to #map1(%k) {
+        affine.for %ii = #id(%i) to #ub(%i) {
+          affine.for %jj = #id(%j) to #ub(%j) {
+            affine.for %kk = #id(%k) to #ub(%k) {
               %5 = affine.load %A[%ii, %kk] : memref<4096x4096xf32>
               %6 = affine.load %B[%kk, %jj] : memref<4096x4096xf32>
               %7 = affine.load %C[%ii, %jj] : memref<4096x4096xf32>
@@ -55,10 +57,10 @@ func @matmul(%A: memref<4096x4096xf32>, %B: memref<4096x4096xf32>, %C: memref<40
 // The result matrix's copy gets hoisted out.
 // Result matrix copy-in.
 // CHECK:     affine.for %{{.*}} = 0 to 128 {
-// CHECK:       %{{.*}} = affine.apply #[[MEM_IDX_MAP]](%{{.*}}, %{{.*}})
+// CHECK:       affine.apply #[[MEM_IDX_MAP]](%{{.*}}, %{{.*}})
 // CHECK:       affine.for %{{.*}} = 0 to 128 {
-// CHECK:         %{{.*}} = affine.apply #[[MEM_IDX_MAP]](%{{.*}}, %{{.*}})
-// CHECK:         %{{.*}} = affine.load %{{.*}}[%{{.*}}, %{{.*}}] : memref<4096x4096xf32>
+// CHECK:         affine.apply #[[MEM_IDX_MAP]](%{{.*}}, %{{.*}})
+// CHECK:         affine.load %{{.*}}[%{{.*}}, %{{.*}}] : memref<4096x4096xf32>
 // CHECK:         affine.store %{{.*}}, [[BUFC]][%{{.*}}, %{{.*}}] : memref<128x128xf32>
 // CHECK:       }
 // CHECK:     }
@@ -67,10 +69,10 @@ func @matmul(%A: memref<4096x4096xf32>, %B: memref<4096x4096xf32>, %C: memref<40
 // CHECK:     affine.for %{{.*}} = 0 to 4096 step 128 {
 // CHECK:      [[BUFA:%[0-9]+]] = alloc() : memref<128x128xf32>
 // CHECK:       affine.for %{{.*}} = 0 to 128 {
-// CHECK:         %{{.*}} = affine.apply #[[MEM_IDX_MAP]](%{{.*}}, %{{.*}})
+// CHECK:         affine.apply #[[MEM_IDX_MAP]](%{{.*}}, %{{.*}})
 // CHECK:         affine.for %{{.*}} = 0 to 128 {
-// CHECK:           %{{.*}} = affine.apply #[[MEM_IDX_MAP]](%{{.*}}, %{{.*}})
-// CHECK:           %{{.*}} = affine.load %{{.*}}[%{{.*}}, %{{.*}}] : memref<4096x4096xf32>
+// CHECK:           affine.apply #[[MEM_IDX_MAP]](%{{.*}}, %{{.*}})
+// CHECK:           affine.load %{{.*}}[%{{.*}}, %{{.*}}] : memref<4096x4096xf32>
 // CHECK:           affine.store %{{.*}}, [[BUFA]][%{{.*}}, %{{.*}}] : memref<128x128xf32>
 // CHECK:         }
 // CHECK:       }
@@ -78,23 +80,23 @@ func @matmul(%A: memref<4096x4096xf32>, %B: memref<4096x4096xf32>, %C: memref<40
 // RHS matrix copy-in.
 // CHECK:       [[BUFB:%[0-9]+]] = alloc() : memref<128x128xf32>
 // CHECK:       affine.for %{{.*}} = 0 to 128 {
-// CHECK:         %{{.*}} = affine.apply #[[MEM_IDX_MAP]](%{{.*}}, %{{.*}})
+// CHECK:         affine.apply #[[MEM_IDX_MAP]](%{{.*}}, %{{.*}})
 // CHECK:         affine.for %{{.*}} = 0 to 128 {
-// CHECK:           %{{.*}} = affine.apply #[[MEM_IDX_MAP]](%{{.*}}, %{{.*}})
-// CHECK:           %{{.*}} = affine.load %{{.*}}[%{{.*}}, %{{.*}}] : memref<4096x4096xf32>
+// CHECK:           affine.apply #[[MEM_IDX_MAP]](%{{.*}}, %{{.*}})
+// CHECK:           affine.load %{{.*}}[%{{.*}}, %{{.*}}] : memref<4096x4096xf32>
 // CHECK:           affine.store %{{.*}}, [[BUFB]][%{{.*}}, %{{.*}}] : memref<128x128xf32>
 // CHECK:         }
 // CHECK:       }
 
 // Computation on the fast buffers.
-// CHECK:       affine.for %{{.*}} = #map7(%{{.*}}) to #map8(%{{.*}}) {
-// CHECK:         affine.for %{{.*}} = #map7(%{{.*}}) to #map8(%{{.*}}) {
-// CHECK:           affine.for %{{.*}} = #map7(%{{.*}}) to #map8(%{{.*}}) {
-// CHECK:             %{{.*}} = affine.load [[BUFA]][-%{{.*}} + %{{.*}}, -%{{.*}} + %{{.*}}] : memref<128x128xf32>
-// CHECK:             %{{.*}} = affine.load [[BUFB]][-%{{.*}} + %{{.*}}, -%{{.*}} + %{{.*}}] : memref<128x128xf32>
-// CHECK:             %{{.*}} = affine.load [[BUFC]][-%{{.*}} + %{{.*}}, -%{{.*}} + %{{.*}}] : memref<128x128xf32>
-// CHECK:             %{{.*}} = mulf %{{.*}}, %{{.*}} : f32
-// CHECK:             %{{.*}} = addf %{{.*}}, %{{.*}} : f32
+// CHECK:       affine.for %{{.*}} = #[[MAP_IDENTITY]](%{{.*}}) to #[[MAP_PLUS_128]](%{{.*}}) {
+// CHECK:         affine.for %{{.*}} = #[[MAP_IDENTITY]](%{{.*}}) to #[[MAP_PLUS_128]](%{{.*}}) {
+// CHECK:           affine.for %{{.*}} = #[[MAP_IDENTITY]](%{{.*}}) to #[[MAP_PLUS_128]](%{{.*}}) {
+// CHECK:             affine.load [[BUFA]][-%{{.*}} + %{{.*}}, -%{{.*}} + %{{.*}}] : memref<128x128xf32>
+// CHECK:             affine.load [[BUFB]][-%{{.*}} + %{{.*}}, -%{{.*}} + %{{.*}}] : memref<128x128xf32>
+// CHECK:             affine.load [[BUFC]][-%{{.*}} + %{{.*}}, -%{{.*}} + %{{.*}}] : memref<128x128xf32>
+// CHECK:             mulf %{{.*}}, %{{.*}} : f32
+// CHECK:             addf %{{.*}}, %{{.*}} : f32
 // CHECK:             affine.store %{{.*}}, [[BUFC]][-%{{.*}} + %{{.*}}, -%{{.*}} + %{{.*}}] : memref<128x128xf32>
 // CHECK:           }
 // CHECK:         }
@@ -102,14 +104,14 @@ func @matmul(%A: memref<4096x4096xf32>, %B: memref<4096x4096xf32>, %C: memref<40
 // CHECK:       dealloc [[BUFB]] : memref<128x128xf32>
 // CHECK:       dealloc [[BUFA]] : memref<128x128xf32>
 // CHECK:     }
-// CHECK:     %{{.*}} = affine.apply #map0(%{{.*}}, %{{.*}})
-// CHECK:     %{{.*}} = affine.apply #map1(%{{.*}}, %{{.*}})
+// CHECK:     affine.apply #map{{.*}}(%{{.*}}, %{{.*}})
+// CHECK:     affine.apply #map{{.*}}(%{{.*}}, %{{.*}})
 
 // Result matrix copy out.
 // CHECK:     affine.for %{{.*}} = 0 to 128 {
-// CHECK:       %{{.*}} = affine.apply #[[MEM_IDX_MAP]](%{{.*}}, %{{.*}})
+// CHECK:       affine.apply #[[MEM_IDX_MAP]](%{{.*}}, %{{.*}})
 // CHECK:       affine.for %{{.*}} = 0 to 128 {
-// CHECK:         %{{.*}} = affine.apply #[[MEM_IDX_MAP]](%{{.*}}, %{{.*}})
+// CHECK:         affine.apply #[[MEM_IDX_MAP]](%{{.*}}, %{{.*}})
 // CHECK:         [[BUFA]] = affine.load [[BUFC]][%{{.*}}, %{{.*}}] : memref<128x128xf32>
 // CHECK:         store [[BUFA]], %{{.*}}[%{{.*}}, %{{.*}}] : memref<4096x4096xf32>
 // CHECK:       }
@@ -125,11 +127,11 @@ func @matmul(%A: memref<4096x4096xf32>, %B: memref<4096x4096xf32>, %C: memref<40
 //  FILTER-NOT:   alloc()
 //      FILTER:   affine.for %{{.*}} = 0 to 128 {
 //      FILTER:     affine.for %{{.*}} = 0 to 4096 {
-//      FILTER:     affine.for %{{.*}} = 0 to 4096 step 128 {
-// FILTER-NEXT:       affine.for %{{.*}} = 0 to 4096 step 128 {
+//      FILTER:   affine.for %{{.*}} = 0 to 4096 step 128 {
+// FILTER-NEXT:     affine.for %{{.*}} = 0 to 4096 step 128 {
+// FILTER-NEXT:       affine.for %{{.*}} = #map{{.*}}(%{{.*}}) to #map{{.*}}(%{{.*}}) {
 // FILTER-NEXT:         affine.for %{{.*}} = #map{{.*}}(%{{.*}}) to #map{{.*}}(%{{.*}}) {
 // FILTER-NEXT:           affine.for %{{.*}} = #map{{.*}}(%{{.*}}) to #map{{.*}}(%{{.*}}) {
-// FILTER-NEXT:             affine.for %{{.*}} = #map{{.*}}(%{{.*}}) to #map{{.*}}(%{{.*}}) {
 //      FILTER:   dealloc %1 : memref<128x4096xf32>
 //  FILTER-NOT:   dealloc %1 : memref<128x4096xf32>
 
@@ -157,32 +159,32 @@ func @foo(%arg0: memref<1024x1024xf32>, %arg1: memref<1024x1024xf32>, %arg2: mem
 }
 // CHECK-SMALL: affine.for %arg{{.*}} = 0 to 1024 {
 // CHECK-SMALL:   affine.for %arg{{.*}} = 0 to 1024 {
-// CHECK-SMALL:     %{{.*}} = affine.apply #map{{.*}}(%arg{{.*}}, %arg{{.*}})
-// CHECK-SMALL:     %{{.*}} = affine.apply #map{{.*}}(%arg{{.*}}, %arg{{.*}})
-// CHECK-SMALL:     %{{.*}} = alloc() : memref<1x1xf32>
-// CHECK-SMALL:     %{{.*}} = affine.apply #map{{.*}}(%arg{{.*}}, %c0{{.*}})
-// CHECK-SMALL:     %{{.*}} = affine.apply #map{{.*}}(%arg{{.*}}, %c0{{.*}})
-// CHECK-SMALL:     %{{.*}} = affine.load %arg{{.*}}[%{{.*}}, %{{.*}}] : memref<1024x1024xf32>
+// CHECK-SMALL:     affine.apply #map{{.*}}(%arg{{.*}}, %arg{{.*}})
+// CHECK-SMALL:     affine.apply #map{{.*}}(%arg{{.*}}, %arg{{.*}})
+// CHECK-SMALL:     alloc() : memref<1x1xf32>
+// CHECK-SMALL:     affine.apply #map{{.*}}(%arg{{.*}}, %c0{{.*}})
+// CHECK-SMALL:     affine.apply #map{{.*}}(%arg{{.*}}, %c0{{.*}})
+// CHECK-SMALL:     affine.load %arg{{.*}}[%{{.*}}, %{{.*}}] : memref<1024x1024xf32>
 // CHECK-SMALL:     affine.store %{{.*}}, %{{.*}}[%c0{{.*}}, %c0{{.*}}] : memref<1x1xf32>
 // CHECK-SMALL:     affine.for %arg{{.*}} = 0 to 1024 {
-// CHECK-SMALL:       %{{.*}} = affine.apply #map{{.*}}(%arg{{.*}}, %arg{{.*}})
-// CHECK-SMALL:       %{{.*}} = affine.apply #map{{.*}}(%arg{{.*}}, %arg{{.*}})
-// CHECK-SMALL:       %{{.*}} = alloc() : memref<1x1xf32>
-// CHECK-SMALL:       %{{.*}} = affine.apply #map{{.*}}(%arg{{.*}}, %c0{{.*}})
-// CHECK-SMALL:       %{{.*}} = affine.apply #map{{.*}}(%arg{{.*}}, %c0{{.*}})
-// CHECK-SMALL:       %{{.*}} = affine.load %arg{{.*}}[%{{.*}}, %{{.*}}] : memref<1024x1024xf32>
+// CHECK-SMALL:       affine.apply #map{{.*}}(%arg{{.*}}, %arg{{.*}})
+// CHECK-SMALL:       affine.apply #map{{.*}}(%arg{{.*}}, %arg{{.*}})
+// CHECK-SMALL:       alloc() : memref<1x1xf32>
+// CHECK-SMALL:       affine.apply #map{{.*}}(%arg{{.*}}, %c0{{.*}})
+// CHECK-SMALL:       affine.apply #map{{.*}}(%arg{{.*}}, %c0{{.*}})
+// CHECK-SMALL:       affine.load %arg{{.*}}[%{{.*}}, %{{.*}}] : memref<1024x1024xf32>
 // CHECK-SMALL:       affine.store %{{.*}}, %{{.*}}[%c0{{.*}}, %c0{{.*}}] : memref<1x1xf32>
-// CHECK-SMALL:       %{{.*}} = affine.load %{{.*}}[0, 0] : memref<1x1xf32>
-// CHECK-SMALL:       %{{.*}} = affine.load %{{.*}}[0, 0] : memref<1x1xf32>
-// CHECK-SMALL:       %{{.*}} = addf %{{.*}}, %{{.*}} : f32
+// CHECK-SMALL:       affine.load %{{.*}}[0, 0] : memref<1x1xf32>
+// CHECK-SMALL:       affine.load %{{.*}}[0, 0] : memref<1x1xf32>
+// CHECK-SMALL:       addf %{{.*}}, %{{.*}} : f32
 // CHECK-SMALL:       affine.store %{{.*}}, %{{.*}}[0, 0] : memref<1x1xf32>
 // CHECK-SMALL:       dealloc %{{.*}} : memref<1x1xf32>
 // CHECK-SMALL:     }
-// CHECK-SMALL:     %{{.*}} = affine.apply #map{{.*}}(%arg{{.*}}, %arg{{.*}})
-// CHECK-SMALL:     %{{.*}} = affine.apply #map{{.*}}(%arg{{.*}}, %arg{{.*}})
-// CHECK-SMALL:     %{{.*}} = affine.apply #map{{.*}}(%arg{{.*}}, %c0{{.*}})
-// CHECK-SMALL:     %{{.*}} = affine.apply #map{{.*}}(%arg{{.*}}, %c0{{.*}})
-// CHECK-SMALL:     %{{.*}} = affine.load %{{.*}}[%c0{{.*}}, %c0{{.*}}] : memref<1x1xf32>
+// CHECK-SMALL:     affine.apply #map{{.*}}(%arg{{.*}}, %arg{{.*}})
+// CHECK-SMALL:     affine.apply #map{{.*}}(%arg{{.*}}, %arg{{.*}})
+// CHECK-SMALL:     affine.apply #map{{.*}}(%arg{{.*}}, %c0{{.*}})
+// CHECK-SMALL:     affine.apply #map{{.*}}(%arg{{.*}}, %c0{{.*}})
+// CHECK-SMALL:     affine.load %{{.*}}[%c0{{.*}}, %c0{{.*}}] : memref<1x1xf32>
 // CHECK-SMALL:     affine.store %{{.*}}, %arg{{.*}}[%{{.*}}, %{{.*}}] : memref<1024x1024xf32>
 // CHECK-SMALL:     dealloc %{{.*}} : memref<1x1xf32>
 // CHECK-SMALL:   }
@@ -200,6 +202,7 @@ func @foo(%arg0: memref<1024x1024xf32>, %arg1: memref<1024x1024xf32>, %arg2: mem
 // FILTER-NEXT:     affine.for %{{.*}} = 0 to 1024 {
 //      FILTER: dealloc %{{.*}} : memref<1024x1024xf32>
 //  FILTER-NOT: dealloc
+//  FILTER:     return
 
 // CHeck that only one memref is copied, because for-memref-region is enabled
 // (and the first ever encountered load is analyzed).
@@ -211,4 +214,4 @@ func @foo(%arg0: memref<1024x1024xf32>, %arg1: memref<1024x1024xf32>, %arg2: mem
 // MEMREF_REGION-NEXT:   affine.for %{{.*}} = 0 to 1024 {
 // MEMREF_REGION-NEXT:     affine.for %{{.*}} = 0 to 1024 {
 //      MEMREF_REGION: dealloc %{{.*}} : memref<1024x1024xf32>
-//  MEMREF_REGION-NOT: dealloc
+// MEMREF_REGION-NOT: dealloc
diff --git a/mlir/test/Dialect/Affine/canonicalize.mlir b/mlir/test/Dialect/Affine/canonicalize.mlir
index ede438eb0bd4d..90437ac7ce5f2 100644
--- a/mlir/test/Dialect/Affine/canonicalize.mlir
+++ b/mlir/test/Dialect/Affine/canonicalize.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -split-input-file -pass-pipeline='func(canonicalize)' | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -split-input-file -pass-pipeline='func(canonicalize)' | FileCheck %s
 
 // Affine maps for test case: compose_affine_maps_1dto2d_no_symbols
 // CHECK-DAG: [[MAP0:#map[0-9]+]] = affine_map<(d0) -> (d0 - 1)>
@@ -594,3 +594,15 @@ func @rep(%arg0 : index, %arg1 : index) -> index {
   %1 = affine.min #map2(%0)[%arg1]
   return %1 : index
 }
+
+// -----
+// CHECK-DAG: #[[lb:.*]] = affine_map<()[s0] -> (s0)>
+// CHECK-DAG: #[[ub:.*]] = affine_map<()[s0] -> (s0 + 2)>
+
+func @drop_duplicate_bounds(%N : index) {
+  // affine.for %i = max #lb(%arg0) to min #ub(%arg0)
+  affine.for %i = max affine_map<(d0) -> (d0, d0)>(%N) to min affine_map<(d0) -> (d0 + 2, d0 + 2)>(%N) {
+    "foo"() : () -> ()
+  }
+  return
+}
diff --git a/mlir/test/Dialect/Affine/dma-generate.mlir b/mlir/test/Dialect/Affine/dma-generate.mlir
index b1e71e694690b..3b90cc6ec0a0f 100644
--- a/mlir/test/Dialect/Affine/dma-generate.mlir
+++ b/mlir/test/Dialect/Affine/dma-generate.mlir
@@ -1,5 +1,5 @@
-// RUN: mlir-opt %s -split-input-file -affine-data-copy-generate -affine-data-copy-generate-dma -affine-data-copy-generate-fast-mem-space=2 -affine-data-copy-generate-skip-non-unit-stride-loops -verify-diagnostics | FileCheck %s
-// RUN: mlir-opt %s -split-input-file -affine-data-copy-generate -affine-data-copy-generate-dma -affine-data-copy-generate-fast-mem-capacity=16 -affine-data-copy-generate-fast-mem-space=2 | FileCheck %s --check-prefix FAST-MEM-16KB
+// RUN: mlir-opt -allow-unregistered-dialect %s -split-input-file -affine-data-copy-generate -affine-data-copy-generate-dma -affine-data-copy-generate-fast-mem-space=2 -affine-data-copy-generate-skip-non-unit-stride-loops -verify-diagnostics | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -split-input-file -affine-data-copy-generate -affine-data-copy-generate-dma -affine-data-copy-generate-fast-mem-capacity=16 -affine-data-copy-generate-fast-mem-space=2 | FileCheck %s --check-prefix FAST-MEM-16KB
 
 // We run most test cases with -copy-skip-non-unit-stride-loops to allow testing
 // DMA generation at inner levels easily - since the DMA generation would
@@ -22,28 +22,28 @@ func @loop_nest_1d() {
   %B = alloc() : memref<512 x f32>
   %F = alloc() : memref<256 x f32, 2>
   // First DMA buffer.
-  // CHECK:  %{{.*}} = alloc() : memref<256xf32>
-  // CHECK:  %{{.*}} = alloc() : memref<256xf32, 2>
+  // CHECK:  alloc() : memref<256xf32>
+  // CHECK:  alloc() : memref<256xf32, 2>
   // Tag for first DMA.
-  // CHECK:  %{{.*}} = alloc() : memref<1xi32>
+  // CHECK:  alloc() : memref<1xi32>
   // First DMA transfer.
   // CHECK:  affine.dma_start %{{.*}}[%{{.*}}], %{{.*}}[%{{.*}}], %{{.*}}[%{{.*}}], %{{.*}} : memref<256xf32>, memref<256xf32, 2>, memref<1xi32>
   // CHECK:  affine.dma_wait %{{.*}}[%{{.*}}], %{{.*}} : memref<1xi32>
   // Second DMA buffer.
-  // CHECK:  %{{.*}} = alloc() : memref<256xf32, 2>
+  // CHECK:  alloc() : memref<256xf32, 2>
   // Tag for second DMA.
-  // CHECK:  %{{.*}} = alloc() : memref<1xi32>
+  // CHECK:  alloc() : memref<1xi32>
   // Second DMA transfer.
   // CHECK:       affine.dma_start %{{.*}}[%{{.*}}], %{{.*}}[%{{.*}}], %{{.*}}[%{{.*}}], %{{.*}} : memref<512xf32>, memref<256xf32, 2>, memref<1xi32>
   // CHECK-NEXT:  affine.dma_wait %{{.*}}[%{{.*}}], %{{.*}} : memref<1xi32>
   // CHECK: affine.for %{{.*}} = 0 to 256 {
-      // CHECK-NEXT: %{{.*}} = affine.load %{{.*}}[%{{.*}}] : memref<256xf32, 2>
-      // CHECK:      %{{.*}} = affine.apply [[MAP_PLUS_256]](%{{.*}})
+      // CHECK-NEXT: affine.load %{{.*}}[%{{.*}}] : memref<256xf32, 2>
+      // CHECK:      affine.apply [[MAP_PLUS_256]](%{{.*}})
       // Buffer for '%{{.*}}' in faster memref space is smaller size: 256xf32
       // Affine map for 'affine.load %{{.*}}' is composed: %{{.*}} + 256 - 256 = %{{.*}}.
       // CHECK-NEXT: %{{.*}} = affine.load %{{.*}}[%{{.*}}] : memref<256xf32, 2>
       // Already in faster memory space.
-      // CHECK:     %{{.*}} = affine.load %{{.*}}[%{{.*}}] : memref<256xf32, 2>
+      // CHECK:     affine.load %{{.*}}[%{{.*}}] : memref<256xf32, 2>
   // CHECK-NEXT: }
   // CHECK-NEXT: dealloc %{{.*}} : memref<1xi32>
   // CHECK-NEXT: dealloc %{{.*}} : memref<256xf32, 2>
@@ -83,20 +83,20 @@ func @loop_nest_1d() {
 // CHECK-NEXT:    affine.for %{{.*}} = 0 to 32 {
 // CHECK-NEXT:      affine.for %{{.*}} = 0 to 32 {
 // CHECK-NEXT:        affine.for %{{.*}} = 0 to 16 {
-// CHECK-NEXT:          %{{.*}} = affine.apply #map{{[0-9]+}}(%{{.*}}, %{{.*}})
+// CHECK-NEXT:          affine.apply #map{{[0-9]+}}(%{{.*}}, %{{.*}})
 // CHECK-NEXT:          %{{.*}} = affine.load [[BUFB]][%{{.*}} * 16 + %{{.*}}, %{{.*}}] : memref<512x32xf32, 2>
 // CHECK-NEXT:          "foo"(%{{.*}}) : (f32) -> ()
 // CHECK-NEXT:        }
 // CHECK-NEXT:        affine.for %{{.*}} = 0 to 16 {
-// CHECK-NEXT:          %{{.*}} = affine.apply #map{{[0-9]+}}(%{{.*}}, %{{.*}})
-// CHECK-NEXT:          %{{.*}} = affine.load [[BUFA]][%{{.*}} * 16 + %{{.*}}, %{{.*}}] : memref<512x32xf32, 2>
+// CHECK-NEXT:          affine.apply #map{{[0-9]+}}(%{{.*}}, %{{.*}})
+// CHECK-NEXT:          affine.load [[BUFA]][%{{.*}} * 16 + %{{.*}}, %{{.*}}] : memref<512x32xf32, 2>
 // CHECK-NEXT:          "bar"(%{{.*}}) : (f32) -> ()
 // CHECK-NEXT:        }
 // CHECK-NEXT:        affine.for %{{.*}} = 0 to 16 {
-// CHECK-NEXT:          %{{.*}} = "abc_compute"() : () -> f32
-// CHECK-NEXT:          %{{.*}} = affine.apply #map{{[0-9]+}}(%{{.*}}, %{{.*}})
-// CHECK-NEXT:          %{{.*}} = affine.load [[BUFC]][%{{.*}} * 16 + %{{.*}}, %{{.*}}] : memref<512x32xf32, 2>
-// CHECK-NEXT:          %{{.*}} = "addf32"(%{{.*}}, %{{.*}}) : (f32, f32) -> f32
+// CHECK-NEXT:          "abc_compute"() : () -> f32
+// CHECK-NEXT:          affine.apply #map{{[0-9]+}}(%{{.*}}, %{{.*}})
+// CHECK-NEXT:          affine.load [[BUFC]][%{{.*}} * 16 + %{{.*}}, %{{.*}}] : memref<512x32xf32, 2>
+// CHECK-NEXT:          "addf32"(%{{.*}}, %{{.*}}) : (f32, f32) -> f32
 // CHECK-NEXT:          affine.store %{{.*}}, [[BUFC]][%{{.*}} * 16 + %{{.*}}, %{{.*}}] : memref<512x32xf32, 2>
 // CHECK-NEXT:        }
 // CHECK-NEXT:        "foobar"() : () -> ()
@@ -153,11 +153,11 @@ func @loop_nest_high_d(%A: memref<512 x 32 x f32>,
 // region within a 256 x 8 memref.
 //
 // CHECK-LABEL: func @loop_nest_modulo() {
-// CHECK:       %{{.*}} = alloc() : memref<256x8xf32>
+// CHECK:       alloc() : memref<256x8xf32>
 // CHECK-NEXT:    affine.for %{{.*}} = 0 to 32 step 4 {
-// CHECK-NEXT:      %{{.*}} = affine.apply #map{{[0-9]+}}(%{{.*}})
-// CHECK-NEXT:      %{{.*}} = alloc() : memref<1x2xf32, 2>
-// CHECK-NEXT:      %{{.*}} = alloc() : memref<1xi32>
+// CHECK-NEXT:      affine.apply #map{{[0-9]+}}(%{{.*}})
+// CHECK-NEXT:      alloc() : memref<1x2xf32, 2>
+// CHECK-NEXT:      alloc() : memref<1xi32>
 // Composition of the affine map for '%{{.*}}' causes '%{{.*}}' to be added as a symbol.
 // CHECK-NEXT:      affine.dma_start %{{.*}}[%{{.*}}, 0], %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}}[%{{.*}}], %{{.*}} : memref<256x8xf32>, memref<1x2xf32, 2>, memref<1xi32>
 // CHECK-NEXT:      affine.dma_wait %{{.*}}[%{{.*}}], %{{.*}} : memref<1xi32>
@@ -191,8 +191,8 @@ func @loop_nest_tiled() -> memref<256x1024xf32> {
   %0 = alloc() : memref<256x1024xf32>
   affine.for %i0 = 0 to 256 step 32 {
     affine.for %i1 = 0 to 1024 step 32 {
-// CHECK:      %{{.*}} = alloc() : memref<32x32xf32, 2>
-// CHECK-NEXT: %{{.*}} = alloc() : memref<1xi32>
+// CHECK:      alloc() : memref<32x32xf32, 2>
+// CHECK-NEXT: alloc() : memref<1xi32>
 // Strided DMA here: 32 x 32 tile in a 256 x 1024 memref.
 // CHECK-NEXT: affine.dma_start %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}}[%{{.*}}], %{{.*}}, %{{.*}}, %{{.*}} : memref<256x1024xf32>, memref<32x32xf32, 2>, memref<1xi32>
 // CHECK-NEXT: affine.dma_wait
@@ -200,7 +200,7 @@ func @loop_nest_tiled() -> memref<256x1024xf32> {
 // CHECK-NEXT:   affine.for %{{.*}} = #map
       affine.for %i2 = affine_map<(d0) -> (d0)>(%i0) to affine_map<(d0) -> (d0 + 32)>(%i0) {
         affine.for %i3 = affine_map<(d0) -> (d0)>(%i1) to affine_map<(d0) -> (d0 + 32)>(%i1) {
-          // CHECK: %{{.*}} = affine.load %{{.*}}[-%{{.*}} + %{{.*}}, -%{{.*}} + %{{.*}}] : memref<32x32xf32, 2>
+          // CHECK: affine.load %{{.*}}[-%{{.*}} + %{{.*}}, -%{{.*}} + %{{.*}}] : memref<32x32xf32, 2>
           %1 = affine.load %0[%i2, %i3] : memref<256x1024xf32>
         } // CHECK-NEXT: }
       }
@@ -215,14 +215,14 @@ func @loop_nest_tiled() -> memref<256x1024xf32> {
 func @dma_constant_dim_access(%A : memref<100x100xf32>) {
   %one = constant 1 : index
   %N = constant 100 : index
-  // CHECK:      %{{.*}} = alloc() : memref<1x100xf32, 2>
-  // CHECK-NEXT: %{{.*}} = alloc() : memref<1xi32>
+  // CHECK:      alloc() : memref<1x100xf32, 2>
+  // CHECK-NEXT: alloc() : memref<1xi32>
   // No strided DMA needed here.
   // CHECK:      affine.dma_start %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}}[%{{.*}}], %{{.*}}  : memref<100x100xf32>, memref<1x100xf32, 2>,
   // CHECK-NEXT: affine.dma_wait %{{.*}}[%{{.*}}], %{{.*}} : memref<1xi32>
   affine.for %i = 0 to 100 {
     affine.for %j = 0 to affine_map<()[s0] -> (s0)> ()[%N] {
-      // CHECK: %{{.*}} = affine.load %{{.*}}[0, %{{.*}}] : memref<1x100xf32, 2>
+      // CHECK: affine.load %{{.*}}[0, %{{.*}}] : memref<1x100xf32, 2>
       affine.load %A[%one, %j] : memref<100 x 100 x f32>
     }
   }
@@ -243,14 +243,14 @@ func @dma_with_symbolic_accesses(%A : memref<100x100xf32>, %M : index) {
     }
   }
   return
-// CHECK:       %{{.*}} = alloc() : memref<100x100xf32, 2>
-// CHECK-NEXT:  %{{.*}} = alloc() : memref<1xi32>
+// CHECK:       alloc() : memref<100x100xf32, 2>
+// CHECK-NEXT:  alloc() : memref<1xi32>
 // CHECK-NEXT:  affine.dma_start %{{.*}}[0, symbol(%{{.*}}) + 9], %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}}[%{{.*}}], %{{.*}}
 // CHECK-NEXT:  affine.dma_wait %{{.*}}[%{{.*}}], %{{.*}}
 // CHECK-NEXT:  affine.for %{{.*}} = 0 to 100 {
 // CHECK-NEXT:    affine.for %{{.*}} = 0 to 100 {
-// CHECK-NEXT:      %{{.*}} = affine.apply [[MAP_SYM_SHIFT]](%{{.*}}, %{{.*}})[%{{.*}}, %{{.*}}]
-// CHECK-NEXT:      %{{.*}} = affine.load %{{.*}}[%{{.*}}, %{{.*}}] : memref<100x100xf32, 2>
+// CHECK-NEXT:      affine.apply [[MAP_SYM_SHIFT]](%{{.*}}, %{{.*}})[%{{.*}}, %{{.*}}]
+// CHECK-NEXT:      affine.load %{{.*}}[%{{.*}}, %{{.*}}] : memref<100x100xf32, 2>
 // CHECK-NEXT:    }
 // CHECK-NEXT:  }
 // CHECK:       return
@@ -263,8 +263,8 @@ func @dma_with_symbolic_loop_bounds(%A : memref<100x100xf32>, %M : index, %N: in
   %K = constant 9 : index
 // The buffer size can't be bound by a constant smaller than the original
 // memref size; so the DMA buffer is the entire 100x100.
-// CHECK:       %{{.*}} = alloc() : memref<100x100xf32, 2>
-// CHECK-NEXT:  %{{.*}} = alloc() : memref<1xi32>
+// CHECK:       alloc() : memref<100x100xf32, 2>
+// CHECK-NEXT:  alloc() : memref<1xi32>
 // CHECK-NEXT:  affine.dma_start %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}}[%{{.*}}], %{{.*}} : memref<100x100xf32>, memref<100x100xf32, 2>, memref<1xi32>
 // CHECK-NEXT:  affine.dma_wait %{{.*}}[%{{.*}}], %{{.*}} : memref<1xi32>
   affine.for %i = 0 to 100 {
@@ -286,7 +286,7 @@ func @dma_unknown_size(%arg0: memref<?x?xf32>) {
     affine.for %j = 0 to %N {
       // If this loop nest isn't tiled, the access requires a non-constant DMA
       // size -- not yet implemented.
-      // CHECK: %{{.*}} = affine.load %{{.*}}[%{{.*}}, %{{.*}}] : memref<?x?xf32>
+      // CHECK: affine.load %{{.*}}[%{{.*}}, %{{.*}}] : memref<?x?xf32>
       affine.load %arg0[%i, %j] : memref<? x ? x f32>
       // expected-error@-6 {{copy generation failed for one or more memref's in this block}}
     }
@@ -306,7 +306,7 @@ func @dma_memref_3d(%arg0: memref<1024x1024x1024xf32>) {
         %idz = affine.apply affine_map<(d0) -> (d0 mod 128)>(%k)
         // DMA with nested striding (or emulating with loop around strided DMA)
         // not yet implemented.
-        // CHECK: %{{.*}} = affine.load %{{.*}}[%{{.*}}, %{{.*}}, %{{.*}}] : memref<1024x1024x1024xf32>
+        // CHECK: affine.load %{{.*}}[%{{.*}}, %{{.*}}, %{{.*}}] : memref<1024x1024x1024xf32>
         %v = affine.load %arg0[%idx, %idy, %idz] : memref<1024 x 1024 x 1024 x f32>
         // expected-error@-10 {{copy generation failed for one or more memref's in this block}}
       }
@@ -351,22 +351,22 @@ func @multi_load_store_union() {
   }
   return
 }
-// CHECK:       %{{.*}} = alloc() : memref<512x512xf32>
-// CHECK-NEXT:  %{{.*}} = alloc() : memref<382x446xf32, 2>
-// CHECK-NEXT:  %{{.*}} = alloc() : memref<1xi32>
+// CHECK:       alloc() : memref<512x512xf32>
+// CHECK-NEXT:  alloc() : memref<382x446xf32, 2>
+// CHECK-NEXT:  alloc() : memref<1xi32>
 // CHECK-NEXT:  affine.dma_start %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}}[%{{.*}}], %{{.*}}, %{{.*}}, %{{.*}} : memref<512x512xf32>, memref<382x446xf32, 2>, memref<1xi32>
 // CHECK-NEXT:  affine.dma_wait %{{.*}}[%{{.*}}], %{{.*}} : memref<1xi32>
-// CHECK-NEXT:  %{{.*}} = alloc() : memref<1xi32>
+// CHECK-NEXT:  alloc() : memref<1xi32>
 // CHECK-NEXT:  affine.for %{{.*}} = 0 to 256 {
 // CHECK-NEXT:    affine.for %{{.*}} = 0 to 256 {
-// CHECK-NEXT:      %{{.*}} = affine.apply [[MAP_PLUS_64]](%{{.*}})
-// CHECK-NEXT:      %{{.*}} = affine.apply [[MAP_PLUS_128]](%{{.*}})
-// CHECK-NEXT:      %{{.*}} = affine.apply [[MAP_PLUS_2]](%{{.*}})
-// CHECK-NEXT:      %{{.*}} = affine.apply [[MAP_PLUS_2]](%{{.*}})
-// CHECK-NEXT:      %{{.*}} = affine.load %{{.*}}[%{{.*}}, %{{.*}} + 126] : memref<382x446xf32, 2>
-// CHECK-NEXT:      %{{.*}} = affine.load %{{.*}}[%{{.*}} + 62, %{{.*}}] : memref<382x446xf32, 2>
-// CHECK-NEXT:      %{{.*}} = affine.apply [[MAP_PLUS_128]](%{{.*}})
-// CHECK-NEXT:      %{{.*}} = affine.apply [[MAP_PLUS_192]](%{{.*}})
+// CHECK-NEXT:      affine.apply [[MAP_PLUS_64]](%{{.*}})
+// CHECK-NEXT:      affine.apply [[MAP_PLUS_128]](%{{.*}})
+// CHECK-NEXT:      affine.apply [[MAP_PLUS_2]](%{{.*}})
+// CHECK-NEXT:      affine.apply [[MAP_PLUS_2]](%{{.*}})
+// CHECK-NEXT:      affine.load %{{.*}}[%{{.*}}, %{{.*}} + 126] : memref<382x446xf32, 2>
+// CHECK-NEXT:      affine.load %{{.*}}[%{{.*}} + 62, %{{.*}}] : memref<382x446xf32, 2>
+// CHECK-NEXT:      affine.apply [[MAP_PLUS_128]](%{{.*}})
+// CHECK-NEXT:      affine.apply [[MAP_PLUS_192]](%{{.*}})
 // CHECK-NEXT:      affine.store %{{.*}}, %{{.*}}[%{{.*}}, %{{.*}} + 190] : memref<382x446xf32, 2>
 // CHECK-NEXT:      affine.store %{{.*}}, %{{.*}}[%{{.*}} + 126, %{{.*}}] : memref<382x446xf32, 2>
 // CHECK-NEXT:    }
@@ -396,29 +396,29 @@ func @dma_loop_straightline_interspersed() {
 }
 // There are three regions here - the 'load' preceding the loop, the loop
 // itself, and the operations appearing after the loop.
-// CHECK:       %{{.*}} = alloc() : memref<256xf32>
-// CHECK-NEXT:  %{{.*}} = alloc() : memref<1xf32, 2>
-// CHECK-NEXT:  %{{.*}} = alloc() : memref<1xi32>
+// CHECK:       alloc() : memref<256xf32>
+// CHECK-NEXT:  alloc() : memref<1xf32, 2>
+// CHECK-NEXT:  alloc() : memref<1xi32>
 // CHECK-NEXT:  affine.dma_start %{{.*}}[%{{.*}}], %{{.*}}[%{{.*}}], %{{.*}}[%{{.*}}], %{{.*}} : memref<256xf32>, memref<1xf32, 2>, memref<1xi32>
 // CHECK-NEXT:  affine.dma_wait %{{.*}}[%{{.*}}], %{{.*}} : memref<1xi32>
-// CHECK-NEXT:  %{{.*}} = affine.load %{{.*}}[0] : memref<1xf32, 2>
+// CHECK-NEXT:  affine.load %{{.*}}[0] : memref<1xf32, 2>
 // CHECK-NEXT:  dealloc %{{.*}} : memref<1xi32>
 // CHECK-NEXT:  dealloc %{{.*}} : memref<1xf32, 2>
-// CHECK-NEXT:  %{{.*}} = alloc() : memref<254xf32, 2>
-// CHECK-NEXT:  %{{.*}} = alloc() : memref<1xi32>
+// CHECK-NEXT:  alloc() : memref<254xf32, 2>
+// CHECK-NEXT:  alloc() : memref<1xi32>
 // CHECK-NEXT:  affine.dma_start %{{.*}}[%{{.*}}], %{{.*}}[%{{.*}}], %{{.*}}[%{{.*}}], %{{.*}} : memref<256xf32>, memref<254xf32, 2>, memref<1xi32>
 // CHECK-NEXT:  affine.dma_wait %{{.*}}[%{{.*}}], %{{.*}} : memref<1xi32>
 // CHECK-NEXT:  affine.for %{{.*}} = 1 to 255 {
-// CHECK-NEXT:    %{{.*}} = affine.load %{{.*}}[%{{.*}} - 1] : memref<254xf32, 2>
+// CHECK-NEXT:    affine.load %{{.*}}[%{{.*}} - 1] : memref<254xf32, 2>
 // CHECK-NEXT:  }
 // CHECK-NEXT:  dealloc %{{.*}} : memref<1xi32>
 // CHECK-NEXT:  dealloc %{{.*}} : memref<254xf32, 2>
-// CHECK-NEXT:  %{{.*}} = alloc() : memref<256xf32, 2>
-// CHECK-NEXT:  %{{.*}} = alloc() : memref<1xi32>
+// CHECK-NEXT:  alloc() : memref<256xf32, 2>
+// CHECK-NEXT:  alloc() : memref<1xi32>
 // CHECK-NEXT:  affine.dma_start %{{.*}}[%{{.*}}], %{{.*}}[%{{.*}}], %{{.*}}[%{{.*}}], %{{.*}} : memref<256xf32>, memref<256xf32, 2>, memref<1xi32>
 // CHECK-NEXT:  affine.dma_wait %{{.*}}[%{{.*}}], %{{.*}} : memref<1xi32>
-// CHECK-NEXT:  %{{.*}} = alloc() : memref<1xi32>
-// CHECK-NEXT:  %{{.*}} = affine.load %{{.*}}[255] : memref<256xf32, 2>
+// CHECK-NEXT:  alloc() : memref<1xi32>
+// CHECK-NEXT:  affine.load %{{.*}}[255] : memref<256xf32, 2>
 // CHECK-NEXT:  affine.store %{{.*}}, %{{.*}}[0] : memref<256xf32, 2>
 // CHECK-NEXT:  affine.dma_start %{{.*}}[%{{.*}}], %{{.*}}[%{{.*}}], %{{.*}}[%{{.*}}], %{{.*}} : memref<256xf32, 2>, memref<256xf32>, memref<1xi32>
 // CHECK-NEXT:  affine.dma_wait %{{.*}}[%{{.*}}], %{{.*}} : memref<1xi32>
@@ -449,9 +449,9 @@ func @dma_mixed_loop_blocks() {
 // CHECK:       affine.dma_start [[MEM]][%{{.*}}, %{{.*}}], [[BUF]][%{{.*}}, %{{.*}}], [[TAG]][%{{.*}}], %{{.*}} : memref<256x256xvector<8xf32>>, memref<256x256xvector<8xf32>, 2>, memref<1xi32>
 // CHECK-NEXT:  affine.dma_wait [[TAG]][%{{.*}}], %{{.*}} : memref<1xi32>
 // CHECK-NEXT:  affine.for %{{.*}} = 0 to 256 {
-// CHECK:         %{{.*}} = affine.load [[BUF]][0, 0] : memref<256x256xvector<8xf32>, 2>
+// CHECK:         affine.load [[BUF]][0, 0] : memref<256x256xvector<8xf32>, 2>
 // CHECK:         affine.for %{{.*}} = 0 to 256 {
-// CHECK-NEXT:      %{{.*}} = affine.load [[BUF]][%{{.*}}, %{{.*}}] : memref<256x256xvector<8xf32>, 2>
+// CHECK-NEXT:      affine.load [[BUF]][%{{.*}}, %{{.*}}] : memref<256x256xvector<8xf32>, 2>
 
 // -----
 
@@ -469,7 +469,7 @@ func @relative_loop_bounds(%arg0: memref<1027xf32>) {
 // CHECK-NEXT: [[MEM:%[0-9]+]] = alloc() : memref<1xi32>
 // CHECK-NEXT: affine.for %{{.*}} = 0 to 1024 {
 // CHECK-NEXT:    affine.for %{{.*}} = {{#map[0-9]+}}(%{{.*}}) to {{#map[0-9]+}}(%{{.*}}) {
-// CHECK-NEXT:      %{{.*}} = constant 0.000000e+00 : f32
+// CHECK-NEXT:      constant 0.000000e+00 : f32
 // CHECK-NEXT:      affine.store %{{.*}}, [[BUF]][%{{.*}}] : memref<1027xf32, 2>
 // CHECK-NEXT:    }
 // CHECK-NEXT:  }
@@ -496,16 +496,16 @@ func @test_read_write_region_union() {
   return
 }
 
-// CHECK:       %{{.*}} = alloc() : memref<256xf32>
-// CHECK-NEXT:  %{{.*}} = alloc() : memref<85xf32, 2>
-// CHECK-NEXT:  %{{.*}} = alloc() : memref<1xi32>
+// CHECK:       alloc() : memref<256xf32>
+// CHECK-NEXT:  alloc() : memref<85xf32, 2>
+// CHECK-NEXT:  alloc() : memref<1xi32>
 // CHECK-NEXT:  affine.dma_start %{{.*}}[%{{.*}}], %{{.*}}[%{{.*}}], %{{.*}}[%{{.*}}], %{{.*}} : memref<256xf32>, memref<85xf32, 2>, memref<1xi32>
 // CHECK-NEXT:  affine.dma_wait %{{.*}}[%{{.*}}], %{{.*}} : memref<1xi32>
-// CHECK-NEXT:  %{{.*}} = alloc() : memref<1xi32>
+// CHECK-NEXT:  alloc() : memref<1xi32>
 // CHECK-NEXT:  affine.for %{{.*}} = 0 to 10 {
-// CHECK-NEXT:    %{{.*}} = affine.apply [[MAP_READ_OFFSET]](%{{.*}})
-// CHECK-NEXT:    %{{.*}} = affine.apply [[MAP_WRITE_OFFSET]](%{{.*}})
-// CHECK-NEXT:    %{{.*}} = affine.load %{{.*}}[%{{.*}} + 75] : memref<85xf32, 2>
+// CHECK-NEXT:    affine.apply [[MAP_READ_OFFSET]](%{{.*}})
+// CHECK-NEXT:    affine.apply [[MAP_WRITE_OFFSET]](%{{.*}})
+// CHECK-NEXT:    affine.load %{{.*}}[%{{.*}} + 75] : memref<85xf32, 2>
 // CHECK-NEXT:    affine.store %{{.*}}, %{{.*}}[%{{.*}}] : memref<85xf32, 2>
 // CHECK-NEXT:  }
 // CHECK-NEXT:  affine.dma_start %{{.*}}[%{{.*}}], %{{.*}}[%{{.*}}], %{{.*}}[%{{.*}}], %{{.*}} : memref<85xf32, 2>, memref<256xf32>, memref<1xi32>
@@ -567,8 +567,8 @@ func @test_memref_bounds(%arg0: memref<4x4x16x1xvector<8x128xf32>>, %arg1: memre
   return %arg1, %arg2 : memref<144x9xvector<8x128xf32>>, memref<2xvector<8x128xf32>>
 }
 
-// CHECK:       %{{.*}} = alloc() : memref<4x4x16x1xvector<8x128xf32>, 2>
-// CHECK-NEXT:  %{{.*}} = alloc() : memref<1xi32>
+// CHECK:       alloc() : memref<4x4x16x1xvector<8x128xf32>, 2>
+// CHECK-NEXT:  alloc() : memref<1xi32>
 // CHECK-NEXT:  affine.dma_start %{{.*}}[%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}], %{{.*}}[%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}], %{{.*}}[%{{.*}}], %{{.*}} : memref<4x4x16x1xvector<8x128xf32>>, memref<4x4x16x1xvector<8x128xf32>, 2>, memref<1xi32>
 // CHECK-NEXT:  affine.dma_wait %{{.*}}[%{{.*}}], %{{.*}} : memref<1xi32>
 
diff --git a/mlir/test/Dialect/Affine/inlining.mlir b/mlir/test/Dialect/Affine/inlining.mlir
index 43b6c2452071a..a83ef37a1d55b 100644
--- a/mlir/test/Dialect/Affine/inlining.mlir
+++ b/mlir/test/Dialect/Affine/inlining.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -inline -mlir-disable-inline-simplify | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -inline -mlir-disable-inline-simplify | FileCheck %s
 
 // Basic test that functions within affine operations are inlined.
 func @func_with_affine_ops(%N: index) {
diff --git a/mlir/test/Dialect/Affine/invalid.mlir b/mlir/test/Dialect/Affine/invalid.mlir
index ca0baa442caa5..4ad8b7ed82470 100644
--- a/mlir/test/Dialect/Affine/invalid.mlir
+++ b/mlir/test/Dialect/Affine/invalid.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -split-input-file -verify-diagnostics
+// RUN: mlir-opt -allow-unregistered-dialect %s -split-input-file -verify-diagnostics
 
 // -----
 
diff --git a/mlir/test/Dialect/Affine/loop-permute.mlir b/mlir/test/Dialect/Affine/loop-permute.mlir
new file mode 100644
index 0000000000000..98eadfaabc9e4
--- /dev/null
+++ b/mlir/test/Dialect/Affine/loop-permute.mlir
@@ -0,0 +1,47 @@
+// RUN: mlir-opt -allow-unregistered-dialect %s -test-loop-permutation="permutation-map=1,2,0" | FileCheck %s --check-prefix=CHECK-120
+// RUN: mlir-opt -allow-unregistered-dialect %s -test-loop-permutation="permutation-map=1,0,2" | FileCheck %s --check-prefix=CHECK-102
+// RUN: mlir-opt -allow-unregistered-dialect %s -test-loop-permutation="permutation-map=0,1,2" | FileCheck %s --check-prefix=CHECK-012
+// RUN: mlir-opt -allow-unregistered-dialect %s -test-loop-permutation="permutation-map=0,2,1" | FileCheck %s --check-prefix=CHECK-021
+// RUN: mlir-opt -allow-unregistered-dialect %s -test-loop-permutation="permutation-map=2,0,1" | FileCheck %s --check-prefix=CHECK-201
+// RUN: mlir-opt -allow-unregistered-dialect %s -test-loop-permutation="permutation-map=2,1,0" | FileCheck %s --check-prefix=CHECK-210
+
+// CHECK-120-LABEL: func @permute
+func @permute(%U0 : index, %U1 : index, %U2 : index) {
+  "abc"() : () -> ()
+  affine.for %arg0 = 0 to %U0 {
+    affine.for %arg1 = 0 to %U1 {
+      affine.for %arg2 = 0 to %U2 {
+        "foo"(%arg0, %arg1) : (index, index) -> ()
+        "bar"(%arg2) : (index) -> ()
+      }
+    }
+  }
+  "xyz"() : () -> ()
+  return
+}
+// CHECK-120:      "abc"
+// CHECK-120-NEXT: affine.for
+// CHECK-120-NEXT:   affine.for
+// CHECK-120-NEXT:     affine.for
+// CHECK-120-NEXT:       "foo"(%arg4, %arg5)
+// CHECK-120-NEXT:       "bar"(%arg3)
+// CHECK-120-NEXT:     }
+// CHECK-120-NEXT:   }
+// CHECK-120-NEXT: }
+// CHECK-120-NEXT: "xyz"
+// CHECK-120-NEXT: return
+
+// CHECK-102:      "foo"(%arg4, %arg3)
+// CHECK-102-NEXT: "bar"(%arg5)
+
+// CHECK-012:      "foo"(%arg3, %arg4)
+// CHECK-012-NEXT: "bar"(%arg5)
+
+// CHECK-021:      "foo"(%arg3, %arg5)
+// CHECK-021-NEXT: "bar"(%arg4)
+
+// CHECK-210:      "foo"(%arg5, %arg4)
+// CHECK-210-NEXT: "bar"(%arg3)
+
+// CHECK-201:      "foo"(%arg5, %arg3)
+// CHECK-201-NEXT: "bar"(%arg4)
diff --git a/mlir/test/Dialect/Affine/loop-tiling.mlir b/mlir/test/Dialect/Affine/loop-tiling.mlir
index 2f8223b37eebc..110233aa23a2f 100644
--- a/mlir/test/Dialect/Affine/loop-tiling.mlir
+++ b/mlir/test/Dialect/Affine/loop-tiling.mlir
@@ -1,5 +1,6 @@
-// RUN: mlir-opt %s -split-input-file  -affine-loop-tile -affine-tile-size=32 | FileCheck %s
-// RUN: mlir-opt %s -split-input-file -affine-loop-tile -affine-tile-cache-size=512 | FileCheck %s --check-prefix=MODEL
+// RUN: mlir-opt -allow-unregistered-dialect %s -split-input-file  -affine-loop-tile -affine-tile-size=32 | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -split-input-file -affine-loop-tile -affine-tile-cache-size=512 | FileCheck %s --check-prefix=MODEL
+// RUN: mlir-opt -allow-unregistered-dialect %s -split-input-file -affine-loop-tile -affine-tile-size=32 -affine-tile-separate | FileCheck %s --check-prefix=SEPARATE
 
 // -----
 
@@ -169,6 +170,8 @@ func @tile_with_loop_upper_bounds_in_two_symbols(%arg0: memref<?xf32>, %limit: i
 
 // -----
 
+// CHECK-LABEL: func @trip_count_1
+// SEPARATE-LABEL: func @trip_count_1
 func @trip_count_1(%arg0: memref<196608x1xf32>, %arg1: memref<196608x1xf32>)
     -> memref<196608x1xf32> {
   affine.for %i1 = 0 to 196608 {
@@ -177,8 +180,65 @@ func @trip_count_1(%arg0: memref<196608x1xf32>, %arg1: memref<196608x1xf32>)
       affine.store %4, %arg1[%i1, %i3] : memref<196608x1xf32>
     }
   }
+  // CHECK: affine.load %{{.*}}[%{{.*}}, %{{.*}}] : memref<196608x1xf32>
   return %arg1 : memref<196608x1xf32>
 }
+// SEPARATE: return
 
-// CHECK: %{{.*}} = affine.load %{{.*}}[%{{.*}}, %{{.*}}] : memref<196608x1xf32>
+// -----
+
+func @separate_full_tile_2d(%M : index, %N : index) {
+  affine.for %i0 = 0 to %M {
+    affine.for %i1 = 0 to %N {
+      "foo"() : () -> ()
+    }
+  }
+  return
+}
+
+// SEPARATE-DAG: #[[SEP_COND:.*]] = affine_set<(d0, d1)[s0, s1] : (-d0 + s0 - 32 >= 0, -d1 + s1 - 32 >= 0)>
+// SEPARATE-DAG: #[[LB:.*]] = affine_map<(d0) -> (d0)>
+// SEPARATE-DAG: #[[FULL_TILE_UB:.*]] = affine_map<(d0) -> (d0 + 32)>
+// SEPARATE-DAG: #[[PART_TILE_UB:.*]] = affine_map<(d0)[s0] -> (d0 + 32, s0)>
+
+// SEPARATE:       affine.for %arg2
+// SEPARATE-NEXT:    affine.for %arg3
+// SEPARATE-NEXT:      affine.if #[[SEP_COND]](%arg2, %arg3)[%arg0, %arg1] {
+// SEPARATE-NEXT:        affine.for %arg4 = #[[LB]](%arg2) to #[[FULL_TILE_UB]](%arg2) {
+// SEPARATE-NEXT:          affine.for %arg5 = #[[LB]](%arg3) to #[[FULL_TILE_UB]](%arg3) {
+// SEPARATE-NEXT:           "foo"
+// SEPARATE-NEXT:          }
+// SEPARATE-NEXT:        }
+// SEPARATE-NEXT:      } else {
+// SEPARATE-NEXT:        affine.for %arg4 = #[[LB]](%arg2) to min #[[PART_TILE_UB]](%arg2)[%arg0] {
+// SEPARATE-NEXT:          affine.for %arg5 = #[[LB]](%arg3) to min #[[PART_TILE_UB]](%arg3)[%arg1] {
+// SEPARATE-NEXT:           "foo"
+// SEPARATE-NEXT:          }
+// SEPARATE-NEXT:        }
+// SEPARATE-NEXT:      }
+// SEPARATE-NEXT:    }
+// SEPARATE-NEXT:  }
+// SEPARATE-NEXT:  return
+
+// -----
+
+func @separate_full_tile_1d_max_min(%M : index, %N : index, %P : index, %Q : index) {
+  affine.for %i0 = max affine_map<(d0, d1) -> (d0, d1)>  (%M, %N) to min affine_map< (d0, d1) -> (d0, d1)> (%P, %Q) {
+  }
+  return
+}
 
+// SEPARATE-DAG: #[[SEP_COND:.*]] = affine_set<(d0)[s0, s1] : (-d0 + s0 - 32 >= 0, -d0 + s1 - 32 >= 0)>
+// SEPARATE-DAG: #[[TILE_LB:.*]] = affine_map<(d0) -> (d0)>
+// SEPARATE-DAG: #[[FULL_TILE_UB:.*]] = affine_map<(d0) -> (d0 + 32)>
+// SEPARATE-DAG: #[[PARTIAL_TILE_UB:.*]] = affine_map<(d0, d1, d2) -> (d2 + 32, d0, d1)>
+
+// SEPARATE:         affine.for %arg4
+// SEPARATE-NEXT:      affine.if #[[SEP_COND]](%arg4)[%arg2, %arg3] {
+// SEPARATE-NEXT:        affine.for %arg5 = #[[TILE_LB]](%arg4) to #[[FULL_TILE_UB]](%arg4) {
+// SEPARATE-NEXT:        }
+// SEPARATE-NEXT:      } else {
+// SEPARATE-NEXT:        affine.for %arg5 = #[[TILE_LB]](%arg4) to min #[[PARTIAL_TILE_UB]](%arg2, %arg3, %arg4) {
+// SEPARATE-NEXT:        }
+// SEPARATE-NEXT:      }
+// SEPARATE-NEXT:    }
diff --git a/mlir/test/Dialect/Affine/ops.mlir b/mlir/test/Dialect/Affine/ops.mlir
index c48da1da4759a..cd42980f87c97 100644
--- a/mlir/test/Dialect/Affine/ops.mlir
+++ b/mlir/test/Dialect/Affine/ops.mlir
@@ -1,5 +1,5 @@
-// RUN: mlir-opt -split-input-file %s | FileCheck %s
-// RUN: mlir-opt %s -mlir-print-op-generic | FileCheck -check-prefix=GENERIC %s
+// RUN: mlir-opt -allow-unregistered-dialect -split-input-file %s | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -mlir-print-op-generic | FileCheck -check-prefix=GENERIC %s
 
 // Check that the attributes for the affine operations are round-tripped.
 // Check that `affine.terminator` is visible in the generic form.
diff --git a/mlir/test/Dialect/Affine/parallelism-detection.mlir b/mlir/test/Dialect/Affine/parallelism-detection.mlir
index fa08a896868fb..0788e6f8fb208 100644
--- a/mlir/test/Dialect/Affine/parallelism-detection.mlir
+++ b/mlir/test/Dialect/Affine/parallelism-detection.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -test-detect-parallel -split-input-file -verify-diagnostics | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -test-detect-parallel -split-input-file -verify-diagnostics | FileCheck %s
 
 // CHECK-LABEL: func @loop_nest_3d_outer_two_parallel
 func @loop_nest_3d_outer_two_parallel(%N : index) {
diff --git a/mlir/test/Dialect/Affine/simplify-affine-structures.mlir b/mlir/test/Dialect/Affine/simplify-affine-structures.mlir
index 89f37d0b6c926..c8d2fd4c0a969 100644
--- a/mlir/test/Dialect/Affine/simplify-affine-structures.mlir
+++ b/mlir/test/Dialect/Affine/simplify-affine-structures.mlir
@@ -1,7 +1,7 @@
-// RUN: mlir-opt %s -simplify-affine-structures | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -split-input-file -simplify-affine-structures | FileCheck %s
 
 // CHECK-DAG: [[SET_EMPTY_2D:#set[0-9]+]] = affine_set<(d0, d1) : (1 == 0)>
-// CHECK-DAG: #set1 = affine_set<(d0, d1) : (d0 - 100 == 0, d1 - 10 == 0, -d0 + 100 >= 0, d1 >= 0, d1 + 101 >= 0)>
+// CHECK-DAG: #set1 = affine_set<(d0, d1) : (d0 - 100 == 0, d1 - 10 == 0, -d0 + 100 >= 0, d1 >= 0)>
 // CHECK-DAG: #set2 = affine_set<(d0, d1)[s0, s1] : (1 == 0)>
 // CHECK-DAG: #set3 = affine_set<(d0, d1)[s0, s1] : (d0 * 7 + d1 * 5 + s0 * 11 + s1 == 0, d0 * 5 - d1 * 11 + s0 * 7 + s1 == 0, d0 * 11 + d1 * 7 - s0 * 5 + s1 == 0, d0 * 7 + d1 * 5 + s0 * 11 + s1 == 0)>
 // CHECK-DAG: [[SET_EMPTY_1D:#set[0-9]+]] = affine_set<(d0) : (1 == 0)>
@@ -236,3 +236,23 @@ func @test_empty_set(%N : index) {
 
   return
 }
+
+// -----
+
+// CHECK-DAG: #[[SET1:.*]] = affine_set<(d0, d1) : (d0 >= 0, -d0 + 50 >= 0)
+// CHECK-DAG: #[[SET2:.*]] = affine_set<(d0, d1) : (1 == 0)
+// CHECK-DAG: #[[SET3:.*]] = affine_set<(d0, d1) : (0 == 0)
+
+// CHECK-LABEL: func @simplify_set
+func @simplify_set(%a : index, %b : index) {
+  // CHECK: affine.if #[[SET1]]
+  affine.if affine_set<(d0, d1) : (d0 - d1 + d1 + d0 >= 0, 2 >= 0, d0 >= 0, -d0 + 50 >= 0, -d0 + 100 >= 0)>(%a, %b) {
+  }
+  // CHECK: affine.if #[[SET2]]
+  affine.if affine_set<(d0, d1) : (d0 mod 2 - 1 == 0, d0 - 2 * (d0 floordiv 2) == 0)>(%a, %b) {
+  }
+  // CHECK: affine.if #[[SET3]]
+  affine.if affine_set<(d0, d1) : (1 >= 0, 3 >= 0)>(%a, %b) {
+  }
+	return
+}
diff --git a/mlir/test/Dialect/Affine/slicing-utils.mlir b/mlir/test/Dialect/Affine/slicing-utils.mlir
index 251e400a9b524..defb49a538d02 100644
--- a/mlir/test/Dialect/Affine/slicing-utils.mlir
+++ b/mlir/test/Dialect/Affine/slicing-utils.mlir
@@ -1,6 +1,6 @@
-// RUN: mlir-opt %s -affine-super-vectorizer-test -forward-slicing=true 2>&1 | FileCheck %s --check-prefix=FWD
-// RUN: mlir-opt %s -affine-super-vectorizer-test -backward-slicing=true 2>&1 | FileCheck %s --check-prefix=BWD
-// RUN: mlir-opt %s -affine-super-vectorizer-test -slicing=true 2>&1 | FileCheck %s --check-prefix=FWDBWD
+// RUN: mlir-opt -allow-unregistered-dialect %s -affine-super-vectorizer-test -forward-slicing=true 2>&1 | FileCheck %s --check-prefix=FWD
+// RUN: mlir-opt -allow-unregistered-dialect %s -affine-super-vectorizer-test -backward-slicing=true 2>&1 | FileCheck %s --check-prefix=BWD
+// RUN: mlir-opt -allow-unregistered-dialect %s -affine-super-vectorizer-test -slicing=true 2>&1 | FileCheck %s --check-prefix=FWDBWD
 
 ///   1       2      3      4
 ///   |_______|      |______|
diff --git a/mlir/test/Dialect/Affine/unroll-jam.mlir b/mlir/test/Dialect/Affine/unroll-jam.mlir
index 60de5f17508d4..9d23f7a6aa673 100644
--- a/mlir/test/Dialect/Affine/unroll-jam.mlir
+++ b/mlir/test/Dialect/Affine/unroll-jam.mlir
@@ -1,5 +1,5 @@
-// RUN: mlir-opt %s -affine-loop-unroll-jam -unroll-jam-factor=2 | FileCheck %s
-// RUN: mlir-opt %s -affine-loop-unroll-jam -unroll-jam-factor=4 | FileCheck --check-prefix=UJAM-FOUR %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -affine-loop-unroll-jam -unroll-jam-factor=2 | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -affine-loop-unroll-jam -unroll-jam-factor=4 | FileCheck --check-prefix=UJAM-FOUR %s
 
 // CHECK-DAG: [[MAP_PLUS_1:#map[0-9]+]] = affine_map<(d0) -> (d0 + 1)>
 // CHECK-DAG: [[MAP_DIV_OFFSET:#map[0-9]+]] = affine_map<()[s0] -> (((s0 - 1) floordiv 2) * 2 + 1)>
diff --git a/mlir/test/Dialect/Affine/unroll.mlir b/mlir/test/Dialect/Affine/unroll.mlir
index 3414aa1e0d5b2..7a352ea07471f 100644
--- a/mlir/test/Dialect/Affine/unroll.mlir
+++ b/mlir/test/Dialect/Affine/unroll.mlir
@@ -1,7 +1,7 @@
-// RUN: mlir-opt %s -affine-loop-unroll -unroll-full | FileCheck %s --check-prefix UNROLL-FULL
-// RUN: mlir-opt %s -affine-loop-unroll -unroll-full -unroll-full-threshold=2 | FileCheck %s --check-prefix SHORT
-// RUN: mlir-opt %s -affine-loop-unroll -unroll-factor=4 | FileCheck %s --check-prefix UNROLL-BY-4
-// RUN: mlir-opt %s -affine-loop-unroll -unroll-factor=1 | FileCheck %s --check-prefix UNROLL-BY-1
+// RUN: mlir-opt -allow-unregistered-dialect %s -affine-loop-unroll -unroll-full | FileCheck %s --check-prefix UNROLL-FULL
+// RUN: mlir-opt -allow-unregistered-dialect %s -affine-loop-unroll -unroll-full -unroll-full-threshold=2 | FileCheck %s --check-prefix SHORT
+// RUN: mlir-opt -allow-unregistered-dialect %s -affine-loop-unroll -unroll-factor=4 | FileCheck %s --check-prefix UNROLL-BY-4
+// RUN: mlir-opt -allow-unregistered-dialect %s -affine-loop-unroll -unroll-factor=1 | FileCheck %s --check-prefix UNROLL-BY-1
 
 // UNROLL-FULL-DAG: [[MAP0:#map[0-9]+]] = affine_map<(d0) -> (d0 + 1)>
 // UNROLL-FULL-DAG: [[MAP1:#map[0-9]+]] = affine_map<(d0) -> (d0 + 2)>
diff --git a/mlir/test/Dialect/GPU/ops.mlir b/mlir/test/Dialect/GPU/ops.mlir
index 60364ca6e5079..196513b90c621 100644
--- a/mlir/test/Dialect/GPU/ops.mlir
+++ b/mlir/test/Dialect/GPU/ops.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s | FileCheck %s
 
 module attributes {gpu.container_module} {
 
diff --git a/mlir/test/Dialect/GPU/outlining.mlir b/mlir/test/Dialect/GPU/outlining.mlir
index 7d9a6011eb667..0d8734716a472 100644
--- a/mlir/test/Dialect/GPU/outlining.mlir
+++ b/mlir/test/Dialect/GPU/outlining.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -gpu-kernel-outlining -split-input-file -verify-diagnostics %s | FileCheck %s -dump-input-on-failure
+// RUN: mlir-opt -allow-unregistered-dialect -gpu-kernel-outlining -split-input-file -verify-diagnostics %s | FileCheck %s -dump-input-on-failure
 
 // CHECK: module attributes {gpu.container_module}
 
diff --git a/mlir/test/Dialect/GPU/promotion.mlir b/mlir/test/Dialect/GPU/promotion.mlir
index c06174e0fcded..bb5a93c420f5e 100644
--- a/mlir/test/Dialect/GPU/promotion.mlir
+++ b/mlir/test/Dialect/GPU/promotion.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -test-gpu-memory-promotion -split-input-file %s | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect -test-gpu-memory-promotion -split-input-file %s | FileCheck %s
 
 module @foo attributes {gpu.kernel_module} {
   // Verify that the attribution was indeed introduced
diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir
index 710328c1cfc89..0562cd4701e2b 100644
--- a/mlir/test/Dialect/LLVMIR/invalid.mlir
+++ b/mlir/test/Dialect/LLVMIR/invalid.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -split-input-file -verify-diagnostics
+// RUN: mlir-opt -allow-unregistered-dialect %s -split-input-file -verify-diagnostics
 
 // expected-error@+1{{llvm.noalias argument attribute of non boolean type}}
 func @invalid_noalias(%arg0: !llvm.i32 {llvm.noalias = 3}) {
diff --git a/mlir/test/Dialect/Linalg/fusion.mlir b/mlir/test/Dialect/Linalg/fusion.mlir
index f844f76a3da61..82ef196d0d979 100644
--- a/mlir/test/Dialect/Linalg/fusion.mlir
+++ b/mlir/test/Dialect/Linalg/fusion.mlir
@@ -41,12 +41,11 @@ func @f1(%A: memref<?x?xf32, offset: 0, strides: [?, 1]>,
 }
 // CHECK-LABEL: func @f1
 // CHECK:   (%[[A:.*]]:{{.*}}, %[[B:.*]]:{{.*}}, %[[C:.*]]:{{.*}}, %[[D:.*]]:{{.*}}, %[[E:.*]]:{{.*}})
-// No RAW dependences, the pass does not fuse RAR atm.
-// CHECK: linalg.matmul
 // CHECK: loop.for
 // CHECK:   loop.for
 // CHECK:     loop.for
 // CHECK:       linalg.matmul
+// CHECK:       linalg.matmul
 
 // -----
 
@@ -334,15 +333,13 @@ func @f6(%A: memref<?x?xf32, offset: 0, strides: [?, ?]>,
 }
 // CHECK-LABEL: func @f6
 // CHECK:  (%[[A:.*]]:{{.*}}, %[[B:.*]]:{{.*}}, %[[C:.*]]:{{.*}}, %[[D:.*]]:{{.*}}, %[[E:.*]]:{{.*}})
-// Cannot fuse C due to interleaved read of C that would be bypassed.
-// Cannot fuse E (WAW).
-// CHECK:  linalg.matmul
-// CHECK:  linalg.matmul
+// Fuse the producer of E (WAW) then the producer of C (WAR).
 // CHECK:  loop.for
 // CHECK:    loop.for
 // CHECK:      loop.for
 // CHECK:        linalg.matmul
-// CHECK-NOT:      linalg.matmul
+// CHECK:        linalg.matmul
+// CHECK:        linalg.matmul
 
 // -----
 
@@ -785,3 +782,53 @@ func @fusion_of_three(%arg0: memref<100x10xf32>,
 // CHECK:       linalg.generic
 // CHECK:         exp
 // CHECK:         linalg.yield
+
+// -----
+
+#map0 = affine_map<(d0, d1, d2) -> (d0, d1 - d2)>
+#map1 = affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3, s4] -> (d0 * s1 + s0 + d1 * s2 + d2 * s3 + d3 * s4)>
+#map2 = affine_map<()[s0] -> (s0 + 3)>
+
+func @fill_and_conv(%arg0: memref<1x4x5x1xf32>, %arg1: memref<2x3x1x1xf32>, %arg2: memref<1x4x5x1xf32>) {
+  %cst = constant 0.000000e+00 : f32
+  linalg.fill(%arg2, %cst) : memref<1x4x5x1xf32>, f32
+
+  %c4 = constant 4 : index
+  %c1 = constant 1 : index
+  %c0 = constant 0 : index
+  %c2 = constant 2 : index
+  %c3 = constant 3 : index
+  %4 = dim %arg1, 0 : memref<2x3x1x1xf32>
+  %5 = dim %arg1, 1 : memref<2x3x1x1xf32>
+  %6 = dim %arg0, 0 : memref<1x4x5x1xf32>
+  %7 = dim %arg0, 1 : memref<1x4x5x1xf32>
+  %8 = dim %arg0, 3 : memref<1x4x5x1xf32>
+  %9 = dim %arg2, 0 : memref<1x4x5x1xf32>
+  %10 = dim %arg2, 1 : memref<1x4x5x1xf32>
+  %11 = dim %arg2, 2 : memref<1x4x5x1xf32>
+  %12 = dim %arg2, 3 : memref<1x4x5x1xf32>
+  %13 = linalg.range %c0 : %6 : %c2 : !linalg.range
+  %14 = linalg.range %c0 : %10 : %c3 : !linalg.range
+  loop.for %arg3 = %c0 to %6 step %c2 {
+    loop.for %arg4 = %c0 to %10 step %c3 {
+      %15 = affine.min #map0(%c2, %c1, %arg3)
+      %16 = affine.apply #map2()[%7]
+      %17 = affine.min #map0(%16, %c4, %arg4)
+      %18 = dim %arg0, 2 : memref<1x4x5x1xf32>
+      %19 = dim %arg0, 3 : memref<1x4x5x1xf32>
+      %20 = subview %arg0[%arg3, %arg4, %c0, %c0] [%15, %17, %18, %19] [%c1, %c1, %c1, %c1] : memref<1x4x5x1xf32> to memref<?x?x?x?xf32, #map1>
+      %21 = affine.min #map0(%c2, %c1, %arg3)
+      %22 = affine.min #map0(%c3, %c4, %arg4)
+      %23 = dim %arg2, 2 : memref<1x4x5x1xf32>
+      %24 = dim %arg2, 3 : memref<1x4x5x1xf32>
+      %25 = subview %arg2[%arg3, %arg4, %c0, %c0] [%21, %22, %23, %24] [%c1, %c1, %c1, %c1] : memref<1x4x5x1xf32> to memref<?x?x?x?xf32, #map1>
+      linalg.conv(%arg1, %20, %25) {dilations = [1, 1], strides = [1, 1]} : memref<2x3x1x1xf32>, memref<?x?x?x?xf32, #map1>, memref<?x?x?x?xf32, #map1>
+    }
+  }
+  return
+}
+// CHECK-LABEL: func @fill_and_conv
+// CHECK: loop.for
+// CHECK:   loop.for
+// CHECK:     linalg.fill
+// CHECK:     linalg.conv
diff --git a/mlir/test/Dialect/Linalg/invalid.mlir b/mlir/test/Dialect/Linalg/invalid.mlir
index 59e4a764afcc9..7a8291504ae67 100644
--- a/mlir/test/Dialect/Linalg/invalid.mlir
+++ b/mlir/test/Dialect/Linalg/invalid.mlir
@@ -513,3 +513,14 @@ func @reshape(%arg0: memref<?x?x?xf32>) {
   %0 = linalg.reshape %arg0 [affine_map<(i, j, k) -> (i, j)>, affine_map<(i, j, k) -> (k)>] :
     memref<?x?x?xf32> into memref<?x?xf32, affine_map<(d0, d1)[s0] -> (d0 * s0 + d1)>>
 }
+
+// -----
+
+func @pooling_rank_mismatch(%arg0: memref<?x?x?xf32>,
+                            %arg1: memref<2x3xf32>,
+                            %arg2: memref<?x?x?xf32>) {
+  // expected-error @+1 {{expects memref ranks to match}}
+  linalg.pooling_max(%arg0, %arg1, %arg2) {strides = [2, 1, 2]}:
+    memref<?x?x?xf32>, memref<2x3xf32>, memref<?x?x?xf32>
+  return
+}
diff --git a/mlir/test/Dialect/Linalg/loops.mlir b/mlir/test/Dialect/Linalg/loops.mlir
index b2cd61791d66d..1bd0cf61dd24f 100644
--- a/mlir/test/Dialect/Linalg/loops.mlir
+++ b/mlir/test/Dialect/Linalg/loops.mlir
@@ -9,6 +9,7 @@
 // CHECK-DAG: #[[strided4D:.*]] = affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3] -> (d0 * s1 + s0 + d1 * s2 + d2 * s3 + d3)>
 // CHECK-DAG: #[[clampMinMap:.*]] = affine_map<(d0) -> (d0, 0)>
 
+// CHECK-DAG: #[[Stride1Dilation1:.*]] = affine_map<(d0, d1) -> (d0  + d1)>
 // CHECK-DAG: #[[Stride2Dilation1:.*]] = affine_map<(d0, d1) -> (d0 * 2 + d1)>
 // CHECK-DAG: #[[Stride2Dilation4:.*]] = affine_map<(d0, d1) -> (d0 * 2 + d1 * 4)>
 // CHECK-DAG: #[[Stride3Dilation5:.*]] = affine_map<(d0, d1) -> (d0 * 3 + d1 * 5)>
@@ -251,6 +252,75 @@ func @conv_padding(%arg0: memref<?x?x?x?xf32>,
 //       CHECK:                 %{{.*}} = addf %{{.*}}, %{{.*}} : f32
 //       CHECK:                 store %{{.*}}, %{{.*}}[%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}] : memref<?x?x?x?xf32>
 
+func @pooling_max(%arg0: memref<?x?xf32>,
+                  %arg1: memref<?x?xi32>,
+                  %arg2: memref<?x?xf32>) {
+  linalg.pooling_max(%arg0, %arg1, %arg2) { strides = [2, 1] }:
+    memref<?x?xf32>, memref<?x?xi32>, memref<?x?xf32>
+  return
+}
+// CHECK-LABEL: func @pooling_max
+//       CHECK:   %[[WX:.*]] = dim %arg1, 0 : memref<?x?xi32>
+//       CHECK:   %[[WY:.*]] = dim %arg1, 1 : memref<?x?xi32>
+//       CHECK:   %[[OX:.*]] = dim %arg2, 0 : memref<?x?xf32>
+//       CHECK:   %[[OY:.*]] = dim %arg2, 1 : memref<?x?xf32>
+//       CHECK:   loop.for %{{.*}} = %{{.*}} to %[[OX]] step %{{.*}} {
+//       CHECK:     loop.for %{{.*}} = %{{.*}} to %[[OY]] step %{{.*}} {
+//       CHECK:       loop.for %{{.*}} = %{{.*}} to %[[WX]] step %{{.*}} {
+//       CHECK:         loop.for %{{.*}} = %{{.*}} to %[[WY]] step %{{.*}} {
+//       CHECK:           %[[IX:.*]] = affine.apply #[[Stride2Dilation1]](%{{.*}}, %{{.*}})
+//       CHECK:           %[[IY:.*]] = affine.apply #[[Stride1Dilation1]](%{{.*}}, %{{.*}})
+//       CHECK:           %{{.*}} = load %{{.*}}[%{{.*}}, %{{.*}}] : memref<?x?xf32>
+//       CHECK:           %{{.*}} = load %{{.*}}[%[[IX]], %[[IY]]] : memref<?x?xf32>
+//       CHECK:           %[[RES:.*]] = select %{{.*}}, %{{.*}}, %{{.*}} : f32
+//       CHECK:           store %[[RES]], %{{.*}}[%{{.*}}, %{{.*}}] : memref<?x?xf32>
+
+func @pooling_min(%arg0: memref<?x?xf32>,
+                  %arg1: memref<?x?xi32>,
+                  %arg2: memref<?x?xf32>) {
+  linalg.pooling_min(%arg0, %arg1, %arg2) { strides = [2, 1] }:
+    memref<?x?xf32>, memref<?x?xi32>, memref<?x?xf32>
+  return
+}
+// CHECK-LABEL: func @pooling_min
+//       CHECK:   %[[WX:.*]] = dim %arg1, 0 : memref<?x?xi32>
+//       CHECK:   %[[WY:.*]] = dim %arg1, 1 : memref<?x?xi32>
+//       CHECK:   %[[OX:.*]] = dim %arg2, 0 : memref<?x?xf32>
+//       CHECK:   %[[OY:.*]] = dim %arg2, 1 : memref<?x?xf32>
+//       CHECK:   loop.for %{{.*}} = %{{.*}} to %[[OX]] step %{{.*}} {
+//       CHECK:     loop.for %{{.*}} = %{{.*}} to %[[OY]] step %{{.*}} {
+//       CHECK:       loop.for %{{.*}} = %{{.*}} to %[[WX]] step %{{.*}} {
+//       CHECK:         loop.for %{{.*}} = %{{.*}} to %[[WY]] step %{{.*}} {
+//       CHECK:           %[[IX:.*]] = affine.apply #[[Stride2Dilation1]](%{{.*}}, %{{.*}})
+//       CHECK:           %[[IY:.*]] = affine.apply #[[Stride1Dilation1]](%{{.*}}, %{{.*}})
+//       CHECK:           %{{.*}} = load %{{.*}}[%{{.*}}, %{{.*}}] : memref<?x?xf32>
+//       CHECK:           %{{.*}} = load %{{.*}}[%[[IX]], %[[IY]]] : memref<?x?xf32>
+//       CHECK:           %[[RES:.*]] = select %{{.*}}, %{{.*}}, %{{.*}} : f32
+//       CHECK:           store %[[RES]], %{{.*}}[%{{.*}}, %{{.*}}] : memref<?x?xf32>
+
+func @pooling_sum(%arg0: memref<?x?xf32>,
+                  %arg1: memref<?x?xi32>,
+                  %arg2: memref<?x?xf32>) {
+  linalg.pooling_sum(%arg0, %arg1, %arg2) { strides = [2, 1] }:
+    memref<?x?xf32>, memref<?x?xi32>, memref<?x?xf32>
+  return
+}
+// CHECK-LABEL: func @pooling_sum
+//       CHECK:   %[[WX:.*]] = dim %arg1, 0 : memref<?x?xi32>
+//       CHECK:   %[[WY:.*]] = dim %arg1, 1 : memref<?x?xi32>
+//       CHECK:   %[[OX:.*]] = dim %arg2, 0 : memref<?x?xf32>
+//       CHECK:   %[[OY:.*]] = dim %arg2, 1 : memref<?x?xf32>
+//       CHECK:   loop.for %{{.*}} = %{{.*}} to %[[OX]] step %{{.*}} {
+//       CHECK:     loop.for %{{.*}} = %{{.*}} to %[[OY]] step %{{.*}} {
+//       CHECK:       loop.for %{{.*}} = %{{.*}} to %[[WX]] step %{{.*}} {
+//       CHECK:         loop.for %{{.*}} = %{{.*}} to %[[WY]] step %{{.*}} {
+//       CHECK:           %[[IX:.*]] = affine.apply #[[Stride2Dilation1]](%{{.*}}, %{{.*}})
+//       CHECK:           %[[IY:.*]] = affine.apply #[[Stride1Dilation1]](%{{.*}}, %{{.*}})
+//       CHECK:           %[[RHS:.*]] = load %{{.*}}[%[[IX]], %[[IY]]] : memref<?x?xf32>
+//       CHECK:           %[[LHS:.*]] = load %{{.*}}[%{{.*}}, %{{.*}}] : memref<?x?xf32>
+//       CHECK:           %[[RES:.*]] = addf %[[LHS]], %[[RHS]] : f32
+//       CHECK:           store %[[RES]], %{{.*}}[%{{.*}}, %{{.*}}] : memref<?x?xf32>
+
 func @foo(%0: f32, %1: f32, %2: f32) -> (f32, f32) {
   %f0 = constant 0.0 : f32
   return %f0, %f0 : f32, f32
@@ -522,3 +592,24 @@ func @indexed_generic_op_1D_reduce(%arg0: memref<?xf32>,
 // CHECK:   %[[d:.*]] = select %{{.*}}, %[[b]], %[[c]]
 // CHECK:   %[[e:.*]] = addf %[[a]], %[[d]]
 // CHECK:   store %[[e]], %[[ARG2]][]
+
+#trait_const_fill = {
+  args_in = 0,
+  args_out = 1,
+  indexing_maps = [affine_map<(i) -> (i)>],
+  iterator_types = ["parallel"],
+  library_call = "some_external_fn"
+}
+func @generic_const_init(%arg0: memref<?xf32>) {
+	%cst = constant 1.0 : f32
+  linalg.generic #trait_const_fill %arg0 {
+    ^bb0(%arg1: f32):   // no predecessors
+      linalg.yield %cst : f32
+    }: memref<?xf32>
+    return
+}
+// CHECK-LABEL: @generic_const_init
+// CHECK-SAME: %[[ARG0:.*]]: memref<?xf32>
+// CHECK: %[[CONST:.*]] = constant 1.000000e+00 : f32
+// CHECK: loop.for %[[i:.*]] = {{.*}}
+// CHECK:   store %[[CONST]], %[[ARG0]]
diff --git a/mlir/test/Dialect/Linalg/roundtrip.mlir b/mlir/test/Dialect/Linalg/roundtrip.mlir
index 468fad45dd902..05d35f8f43e4d 100644
--- a/mlir/test/Dialect/Linalg/roundtrip.mlir
+++ b/mlir/test/Dialect/Linalg/roundtrip.mlir
@@ -244,6 +244,48 @@ func @conv_padding(%arg0: memref<?x?x?x?xf32>,
 
 // -----
 
+func @pooling_max(%arg0: memref<?x?x?xf32>,
+                  %arg1: memref<?x?x?xi32>,
+                  %arg2: memref<?x?x?xf32>) {
+  linalg.pooling_max(%arg0, %arg1, %arg2) {strides = [2, 1, 2]}:
+    memref<?x?x?xf32>, memref<?x?x?xi32>, memref<?x?x?xf32>
+  return
+}
+// CHECK-LABEL: func @pooling_max
+//       CHECK:   linalg.pooling_max(%{{.*}}, %{{.*}}, %{{.*}})
+//  CHECK-SAME:   {strides = [2, 1, 2]}
+//  CHECK-SAME:   memref<?x?x?xf32>, memref<?x?x?xi32>, memref<?x?x?xf32>
+
+// -----
+
+func @pooling_min(%arg0: memref<?x?x?xf32>,
+                  %arg1: memref<?x?x?xi32>,
+                  %arg2: memref<?x?x?xf32>) {
+  linalg.pooling_min(%arg0, %arg1, %arg2) {strides = [2, 1, 2]}:
+    memref<?x?x?xf32>, memref<?x?x?xi32>, memref<?x?x?xf32>
+  return
+}
+// CHECK-LABEL: func @pooling_min
+//       CHECK:   linalg.pooling_min(%{{.*}}, %{{.*}}, %{{.*}})
+//  CHECK-SAME:   {strides = [2, 1, 2]}
+//  CHECK-SAME:   memref<?x?x?xf32>, memref<?x?x?xi32>, memref<?x?x?xf32>
+
+// -----
+
+func @pooling_sum(%arg0: memref<?x?x?xf32>,
+                  %arg1: memref<?x?x?xi32>,
+                  %arg2: memref<?x?x?xf32>) {
+  linalg.pooling_sum(%arg0, %arg1, %arg2) {strides = [2, 1, 2]}:
+    memref<?x?x?xf32>, memref<?x?x?xi32>, memref<?x?x?xf32>
+  return
+}
+// CHECK-LABEL: func @pooling_sum
+//       CHECK:   linalg.pooling_sum(%{{.*}}, %{{.*}}, %{{.*}})
+//  CHECK-SAME:   {strides = [2, 1, 2]}
+//  CHECK-SAME:   memref<?x?x?xf32>, memref<?x?x?xi32>, memref<?x?x?xf32>
+
+// -----
+
 // CHECK-DAG: #[[strided2D:.*]] = affine_map<(d0, d1)[s0, s1] -> (d0 * s1 + s0 + d1)>
 // CHECK-DAG: #[[strided3D:.*]] = affine_map<(d0, d1, d2)[s0, s1, s2] -> (d0 * s1 + s0 + d1 * s2 + d2)>
 
diff --git a/mlir/test/Dialect/Linalg/tile_conv_padding.mlir b/mlir/test/Dialect/Linalg/tile_conv_padding.mlir
new file mode 100644
index 0000000000000..a704233e5deb2
--- /dev/null
+++ b/mlir/test/Dialect/Linalg/tile_conv_padding.mlir
@@ -0,0 +1,40 @@
+// RUN: mlir-opt %s -linalg-tile="linalg-tile-sizes=2,3,0,0,4" | FileCheck %s -check-prefix=TILE-23004
+// RUN: mlir-opt %s -linalg-tile="linalg-tile-sizes=2" | FileCheck %s -check-prefix=TILE-20000
+
+// TILE-23004-DAG: #[[strided4D:.*]] = affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3] -> (d0 * s1 + s0 + d1 * s2 + d2 * s3 + d3)>
+// TILE-20000-DAG: #[[strided4D:.*]] = affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3] -> (d0 * s1 + s0 + d1 * s2 + d2 * s3 + d3)>
+// TILE-20000-DAG: #[[minmap:.*]] = affine_map<(d0, d1, d2) -> (d0, d1 - d2)>
+// TILE-20000-DAG: #[[subviewstride:.*]] = affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3, s4] -> (d0 * s1 + s0 + d1 * s2 + d2 * s3 + d3 * s4)>
+
+func @conv_padding(%arg0: memref<?x?x?x?xf32, offset: ?, strides: [?, ?, ?, 1]>, %arg1: memref<?x?x?x?xf32, offset: ?, strides: [?, ?, ?, 1]>, %arg2: memref<?x?x?x?xf32, offset: ?, strides: [?, ?, ?, 1]>) {
+  linalg.conv(%arg0, %arg1, %arg2) {dilations = [10, 20], padding = dense<[[1, 1], [0, 1]]> : tensor<2x2xi64>, strides = [30, 40]} : memref<?x?x?x?xf32, offset: ?, strides: [?, ?, ?, 1]>, memref<?x?x?x?xf32, offset: ?, strides: [?, ?, ?, 1]>, memref<?x?x?x?xf32, offset: ?, strides: [?, ?, ?, 1]>
+  return
+}
+// TILE-23004-LABEL: func @conv_padding(
+//  TILE-23004-SAME:   %[[ARG0:[a-zA-Z0-9_]*]]: memref<?x?x?x?xf32, #[[strided4D]]>
+//  TILE-23004-SAME:   %[[ARG1:[a-zA-Z0-9_]*]]: memref<?x?x?x?xf32, #[[strided4D]]>
+//  TILE-23004-SAME:   %[[ARG2:[a-zA-Z0-9_]*]]: memref<?x?x?x?xf32, #[[strided4D]]>)
+//       TILE-23004:         linalg.conv(%[[ARG0]], %[[ARG1]], %[[ARG2]])
+
+// TILE-20000-LABEL: func @conv_padding(
+//  TILE-20000-SAME: %[[ARG0:[a-zA-Z0-9_]*]]: memref<?x?x?x?xf32, #[[strided4D]]>
+//  TILE-20000-SAME: %[[ARG1:[a-zA-Z0-9_]*]]: memref<?x?x?x?xf32, #[[strided4D]]>
+//  TILE-20000-SAME: %[[ARG2:[a-zA-Z0-9_]*]]: memref<?x?x?x?xf32, #[[strided4D]]>)
+//   TILE-20000-DAG:   %[[C0:.*]] = constant 0 : index
+//   TILE-20000-DAG:   %[[C1:.*]] = constant 1 : index
+//   TILE-20000-DAG:   %[[C2:.*]] = constant 2 : index
+//       TILE-20000:   %[[B:.*]] = dim %[[ARG1]], 0
+//       TILE-20000:   loop.for %[[ivI:.*]] = %[[C0]] to %[[B]] step %[[C2]] {
+//       TILE-20000:     %[[DIM10:.*]] = dim %[[ARG1]], 0
+//       TILE-20000:     %[[EXTENT:.*]] = affine.min #[[minmap]](%[[C2]], %[[DIM10]], %[[ivI]])
+//       TILE-20000:     %[[DIM11:.*]] = dim %[[ARG1]], 1
+//       TILE-20000:     %[[DIM12:.*]] = dim %[[ARG1]], 2
+//       TILE-20000:     %[[DIM13:.*]] = dim %[[ARG1]], 3
+//       TILE-20000:     %[[SUBVIEW1:.*]] = subview %[[ARG1]][%[[ivI]], %[[C0]], %[[C0]], %[[C0]]] [%[[EXTENT]], %[[DIM11]], %[[DIM12]], %[[DIM13]]]
+//       TILE-20000:     %[[DIM20:.*]] = dim %[[ARG2]], 0
+//       TILE-20000:     %[[EXTENT:.*]] = affine.min #[[minmap]](%[[C2]], %[[DIM20]], %[[ivI]])
+//       TILE-20000:     %[[DIM21:.*]] = dim %[[ARG2]], 1
+//       TILE-20000:     %[[DIM22:.*]] = dim %[[ARG2]], 2
+//       TILE-20000:     %[[DIM23:.*]] = dim %[[ARG2]], 3
+//       TILE-20000:     %[[SUBVIEW2:.*]] = subview %[[ARG2]][%[[ivI]], %[[C0]], %[[C0]], %[[C0]]] [%[[EXTENT]], %[[DIM21]], %[[DIM22]], %[[DIM23]]]
+//       TILE-20000:     linalg.conv(%[[ARG0]], %[[SUBVIEW1]], %[[SUBVIEW2]])
diff --git a/mlir/test/Dialect/Loops/invalid.mlir b/mlir/test/Dialect/Loops/invalid.mlir
index 6962387b946c7..e827a78f2b56a 100644
--- a/mlir/test/Dialect/Loops/invalid.mlir
+++ b/mlir/test/Dialect/Loops/invalid.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -split-input-file -verify-diagnostics
+// RUN: mlir-opt -allow-unregistered-dialect %s -split-input-file -verify-diagnostics
 
 func @loop_for_lb(%arg0: f32, %arg1: index) {
   // expected-error@+1 {{operand #0 must be index}}
diff --git a/mlir/test/Dialect/Loops/parallel-loop-fusion.mlir b/mlir/test/Dialect/Loops/parallel-loop-fusion.mlir
index 6625c124cbdbd..13993304c62a8 100644
--- a/mlir/test/Dialect/Loops/parallel-loop-fusion.mlir
+++ b/mlir/test/Dialect/Loops/parallel-loop-fusion.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -pass-pipeline='func(parallel-loop-fusion)' -split-input-file | FileCheck %s --dump-input-on-failure
+// RUN: mlir-opt -allow-unregistered-dialect %s -pass-pipeline='func(parallel-loop-fusion)' -split-input-file | FileCheck %s --dump-input-on-failure
 
 func @fuse_empty_loops() {
   %c2 = constant 2 : index
diff --git a/mlir/test/Dialect/Quant/parse-any.mlir b/mlir/test/Dialect/Quant/parse-any.mlir
index 90976e8d43742..8dbd62c3a2312 100644
--- a/mlir/test/Dialect/Quant/parse-any.mlir
+++ b/mlir/test/Dialect/Quant/parse-any.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -split-input-file | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -split-input-file | FileCheck %s
 
 // -----
 // CHECK-LABEL: parseFullySpecified
diff --git a/mlir/test/Dialect/Quant/parse-ops-invalid.mlir b/mlir/test/Dialect/Quant/parse-ops-invalid.mlir
index 272c53070c7e1..ead636fb19852 100644
--- a/mlir/test/Dialect/Quant/parse-ops-invalid.mlir
+++ b/mlir/test/Dialect/Quant/parse-ops-invalid.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -split-input-file -verify-diagnostics
+// RUN: mlir-opt -allow-unregistered-dialect %s -split-input-file -verify-diagnostics
 
 // -----
 func @invalidStatisticsMismatchedLayerType(%arg0: tensor<8x4x3xf32>) ->
diff --git a/mlir/test/Dialect/Quant/parse-uniform.mlir b/mlir/test/Dialect/Quant/parse-uniform.mlir
index 82d2ae6564994..0e609a77d1fcf 100644
--- a/mlir/test/Dialect/Quant/parse-uniform.mlir
+++ b/mlir/test/Dialect/Quant/parse-uniform.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -split-input-file | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -split-input-file | FileCheck %s
 
 // -----
 // All per-layer params specified:
diff --git a/mlir/test/Dialect/Quant/quant_region.mlir b/mlir/test/Dialect/Quant/quant_region.mlir
index ee874211a7acb..7a7b6440e94a0 100644
--- a/mlir/test/Dialect/Quant/quant_region.mlir
+++ b/mlir/test/Dialect/Quant/quant_region.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -split-input-file -verify-diagnostics %s | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect -split-input-file -verify-diagnostics %s | FileCheck %s
 
 // CHECK-LABEL: @source
 func @source(%arg0: tensor<4xf32>, %arg1: tensor<4xf32>, %arg2: tensor<4xf32>) -> (tensor<4xf32>) {
diff --git a/mlir/test/Dialect/SPIRV/canonicalize.mlir b/mlir/test/Dialect/SPIRV/canonicalize.mlir
index 72857d7d7de24..f8c3bdebda394 100644
--- a/mlir/test/Dialect/SPIRV/canonicalize.mlir
+++ b/mlir/test/Dialect/SPIRV/canonicalize.mlir
@@ -273,7 +273,7 @@ func @const_fold_scalar_imul_flow() -> (i32, i32, i32) {
   %c1 = spv.constant 2 : i32
   %c2 = spv.constant 4 : i32
   %c3 = spv.constant 4294967295 : i32  // 2^32 - 1 : 0xffff ffff
-  %c4 = spv.constant -2147483649 : i32 // -2^31 - 1: 0x7fff ffff
+  %c4 = spv.constant 2147483647 : i32  // 2^31 - 1 : 0x7fff ffff
 
   // (0xffff ffff << 1) = 0x1 ffff fffe -> 0xffff fffe
   // CHECK: %[[CST2:.*]] = spv.constant -2
@@ -331,7 +331,7 @@ func @const_fold_scalar_isub_flow() -> (i32, i32, i32, i32) {
   %c1 = spv.constant 0 : i32
   %c2 = spv.constant 1 : i32
   %c3 = spv.constant 4294967295 : i32  // 2^32 - 1 : 0xffff ffff
-  %c4 = spv.constant -2147483649 : i32 // -2^31 - 1: 0x7fff ffff
+  %c4 = spv.constant 2147483647 : i32  // 2^31     : 0x7fff ffff
   %c5 = spv.constant -1 : i32          //          : 0xffff ffff
   %c6 = spv.constant -2 : i32          //          : 0xffff fffe
 
diff --git a/mlir/test/Dialect/SPIRV/control-flow-ops.mlir b/mlir/test/Dialect/SPIRV/control-flow-ops.mlir
index 9eab38cbcb05a..4951171bbca0b 100644
--- a/mlir/test/Dialect/SPIRV/control-flow-ops.mlir
+++ b/mlir/test/Dialect/SPIRV/control-flow-ops.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -split-input-file -verify-diagnostics %s | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect -split-input-file -verify-diagnostics %s | FileCheck %s
 
 //===----------------------------------------------------------------------===//
 // spv.Branch
diff --git a/mlir/test/Dialect/SPIRV/ops.mlir b/mlir/test/Dialect/SPIRV/ops.mlir
index aee4ff221dd9b..5cf91c0b09b96 100644
--- a/mlir/test/Dialect/SPIRV/ops.mlir
+++ b/mlir/test/Dialect/SPIRV/ops.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -split-input-file -verify-diagnostics %s | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect -split-input-file -verify-diagnostics %s | FileCheck %s
 
 //===----------------------------------------------------------------------===//
 // spv.AccessChain
diff --git a/mlir/test/Dialect/SPIRV/structure-ops.mlir b/mlir/test/Dialect/SPIRV/structure-ops.mlir
index 8a51ad5dc06d1..8e0f447223e4f 100644
--- a/mlir/test/Dialect/SPIRV/structure-ops.mlir
+++ b/mlir/test/Dialect/SPIRV/structure-ops.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -split-input-file -verify-diagnostics %s | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect -split-input-file -verify-diagnostics %s | FileCheck %s
 
 //===----------------------------------------------------------------------===//
 // spv._address_of
diff --git a/mlir/test/Dialect/Vector/canonicalize.mlir b/mlir/test/Dialect/Vector/canonicalize.mlir
index 8dca47515cede..61163a704d643 100644
--- a/mlir/test/Dialect/Vector/canonicalize.mlir
+++ b/mlir/test/Dialect/Vector/canonicalize.mlir
@@ -87,3 +87,75 @@ func @strided_slice_of_constant_mask() -> (vector<2x1xi1>) {
   // CHECK: vector.constant_mask [1, 1] : vector<2x1xi1>
   return %1 : vector<2x1xi1>
 }
+
+// -----
+
+// CHECK-LABEL: transpose_1D_identity
+// CHECK-SAME: ([[ARG:%.*]]: vector<4xf32>)
+func @transpose_1D_identity(%arg : vector<4xf32>) -> vector<4xf32> {
+  // CHECK-NOT: transpose
+  %0 = vector.transpose %arg, [0] : vector<4xf32> to vector<4xf32>
+  // CHECK-NEXT: return [[ARG]]
+  return %0 : vector<4xf32>
+}
+
+// -----
+
+// CHECK-LABEL: transpose_2D_identity
+// CHECK-SAME: ([[ARG:%.*]]: vector<4x3xf32>)
+func @transpose_2D_identity(%arg : vector<4x3xf32>) -> vector<4x3xf32> {
+  // CHECK-NOT: transpose
+  %0 = vector.transpose %arg, [0, 1] : vector<4x3xf32> to vector<4x3xf32>
+  // CHECK-NEXT: return [[ARG]]
+  return %0 : vector<4x3xf32>
+}
+
+// -----
+
+// CHECK-LABEL: transpose_3D_identity
+// CHECK-SAME: ([[ARG:%.*]]: vector<4x3x2xf32>)
+func @transpose_3D_identity(%arg : vector<4x3x2xf32>) -> vector<4x3x2xf32> {
+  // CHECK-NOT: transpose
+  %0 = vector.transpose %arg, [0, 1, 2] : vector<4x3x2xf32> to vector<4x3x2xf32>
+  // CHECK-NEXT: return [[ARG]]
+  return %0 : vector<4x3x2xf32>
+}
+
+// -----
+
+// CHECK-LABEL: transpose_2D_sequence
+// CHECK-SAME: ([[ARG:%.*]]: vector<4x3xf32>)
+func @transpose_2D_sequence(%arg : vector<4x3xf32>) -> vector<4x3xf32> {
+  // CHECK-NOT: transpose
+  %0 = vector.transpose %arg, [1, 0] : vector<4x3xf32> to vector<3x4xf32>
+  %1 = vector.transpose %0, [0, 1] : vector<3x4xf32> to vector<3x4xf32>
+  %2 = vector.transpose %1, [1, 0] : vector<3x4xf32> to vector<4x3xf32>
+  %3 = vector.transpose %2, [0, 1] : vector<4x3xf32> to vector<4x3xf32>
+  // CHECK: [[ADD:%.*]] = addf [[ARG]], [[ARG]]
+  %4 = addf %2, %3 : vector<4x3xf32>
+  // CHECK-NEXT: return [[ADD]]
+  return %4 : vector<4x3xf32>
+}
+
+// -----
+
+// CHECK-LABEL: transpose_3D_sequence
+// CHECK-SAME: ([[ARG:%.*]]: vector<4x3x2xf32>)
+func @transpose_3D_sequence(%arg : vector<4x3x2xf32>) -> vector<4x3x2xf32> {
+  // CHECK: [[T0:%.*]] = vector.transpose [[ARG]], [2, 1, 0]
+  %0 = vector.transpose %arg, [1, 2, 0] : vector<4x3x2xf32> to vector<3x2x4xf32>
+  %1 = vector.transpose %0, [1, 0, 2] : vector<3x2x4xf32> to vector<2x3x4xf32>
+  // CHECK-NOT: transpose
+  %2 = vector.transpose %1, [2, 1, 0] : vector<2x3x4xf32> to vector<4x3x2xf32>
+  %3 = vector.transpose %2, [2, 1, 0] : vector<4x3x2xf32> to vector<2x3x4xf32>
+  // CHECK: [[MUL:%.*]] = mulf [[T0]], [[T0]]
+  %4 = mulf %1, %3 : vector<2x3x4xf32>
+  // CHECK: [[T5:%.*]] = vector.transpose [[MUL]], [2, 1, 0]
+  %5 = vector.transpose %4, [2, 1, 0] : vector<2x3x4xf32> to vector<4x3x2xf32>
+  // CHECK-NOT: transpose
+  %6 = vector.transpose %3, [2, 1, 0] : vector<2x3x4xf32> to vector<4x3x2xf32>
+  // CHECK: [[ADD:%.*]] = addf [[T5]], [[ARG]]
+  %7 = addf %5, %6 : vector<4x3x2xf32>
+  // CHECK-NEXT: return [[ADD]]
+  return %7 : vector<4x3x2xf32>
+}
diff --git a/mlir/test/Dialect/Vector/vector-transforms.mlir b/mlir/test/Dialect/Vector/vector-transforms.mlir
index 7582758384ce5..082afbaff0b0a 100644
--- a/mlir/test/Dialect/Vector/vector-transforms.mlir
+++ b/mlir/test/Dialect/Vector/vector-transforms.mlir
@@ -313,6 +313,95 @@ func @tuple_get(%arg0: vector<4xf32>, %arg1: vector<8xf32>) -> vector<8xf32> {
   return %1 : vector<8xf32>
 }
 
+// CHECK-LABEL: func @tuple_get_producer_consumer
+// CHECK-SAME: %[[A0:.*0]]: vector<2x4xf32>,
+// CHECK-SAME: %[[A1:.*1]]: vector<2x4xf32>,
+// CHECK-SAME: %[[A2:.*2]]: vector<2x4xf32>,
+// CHECK-SAME: %[[A3:.*3]]: vector<2x4xf32>,
+// CHECK-SAME: %[[A4:.*4]]: vector<2x4xf32>,
+// CHECK-SAME: %[[A5:.*5]]: vector<2x4xf32>,
+// CHECK-SAME: %[[A6:.*6]]: vector<2x4xf32>,
+// CHECK-SAME: %[[A7:.*7]]: vector<2x4xf32>
+//      CHECK: return %[[A7]] : vector<2x4xf32>
+
+func @tuple_get_producer_consumer(
+  %arg0 : vector<2x4xf32>, %arg1 : vector<2x4xf32>,
+  %arg2 : vector<2x4xf32>, %arg3 : vector<2x4xf32>,
+  %arg4 : vector<2x4xf32>, %arg5 : vector<2x4xf32>,
+  %arg6 : vector<2x4xf32>, %arg7 : vector<2x4xf32>) -> vector<2x4xf32> {
+  %0 = vector.tuple %arg0, %arg1, %arg2, %arg3, %arg4, %arg5, %arg6, %arg7
+    : vector<2x4xf32>, vector<2x4xf32>, vector<2x4xf32>, vector<2x4xf32>,
+      vector<2x4xf32>, vector<2x4xf32>, vector<2x4xf32>, vector<2x4xf32>
+  // %arg7 == %0 at tupleIndex = 7, offsets = [0, 0]
+  %1 = vector.insert_slices %0, [2, 4], [1, 1]
+    : tuple<vector<2x4xf32>, vector<2x4xf32>, vector<2x4xf32>, vector<2x4xf32>,
+            vector<2x4xf32>, vector<2x4xf32>, vector<2x4xf32>, vector<2x4xf32>>
+      into vector<4x16xf32>
+  // %arg7 == %1 at tupleIndex = -1, offsets = [2, 12]
+  %2 = vector.extract_slices %1, [4, 8], [1, 1]
+    : vector<4x16xf32> into tuple<vector<4x8xf32>, vector<4x8xf32>>
+  // %arg7 == %2 at tupleIndex = 1, offsets = [2, 4]
+  %3 = vector.tuple_get %2, 1 : tuple<vector<4x8xf32>, vector<4x8xf32>>
+  // %arg7 == %3 at tupleIndex = -1, offsets = [2, 4]
+  %4 = vector.extract_slices %3, [2, 4], [1, 1]
+    : vector<4x8xf32> into
+      tuple<vector<2x4xf32>, vector<2x4xf32>, vector<2x4xf32>, vector<2x4xf32>>
+  // %arg7 == %4 at tupleIndex = 3, offsets = [0, 0]
+  %5 = vector.tuple_get %4, 3
+    : tuple<vector<2x4xf32>, vector<2x4xf32>, vector<2x4xf32>, vector<2x4xf32>>
+  // %arg7 == %5
+  return %5 : vector<2x4xf32>
+}
+
+// CHECK-LABEL: func @tuple_get_producer_consumer_swizzle
+// CHECK-SAME: %[[A0:.*0]]: vector<2x4xf32>,
+// CHECK-SAME: %[[A1:.*1]]: vector<2x4xf32>,
+// CHECK-SAME: %[[A2:.*2]]: vector<2x4xf32>,
+// CHECK-SAME: %[[A3:.*3]]: vector<2x4xf32>,
+// CHECK-SAME: %[[A4:.*4]]: vector<2x4xf32>,
+// CHECK-SAME: %[[A5:.*5]]: vector<2x4xf32>,
+// CHECK-SAME: %[[A6:.*6]]: vector<2x4xf32>,
+// CHECK-SAME: %[[A7:.*7]]: vector<2x4xf32>
+//      CHECK: return %[[A7]] : vector<2x4xf32>
+
+func @tuple_get_producer_consumer_swizzle(
+  %arg0 : vector<2x4xf32>, %arg1 : vector<2x4xf32>,
+  %arg2 : vector<2x4xf32>, %arg3 : vector<2x4xf32>,
+  %arg4 : vector<2x4xf32>, %arg5 : vector<2x4xf32>,
+  %arg6 : vector<2x4xf32>, %arg7 : vector<2x4xf32>) -> vector<2x4xf32> {
+  %0 = vector.tuple %arg0, %arg1, %arg2, %arg3, %arg4, %arg5, %arg6, %arg7
+    : vector<2x4xf32>, vector<2x4xf32>, vector<2x4xf32>, vector<2x4xf32>,
+      vector<2x4xf32>, vector<2x4xf32>, vector<2x4xf32>, vector<2x4xf32>
+  // %arg7 == %0 at tupleIndex = 7, offsets = [0, 0]
+  %1 = vector.insert_slices %0, [2, 4], [1, 1]
+    : tuple<vector<2x4xf32>, vector<2x4xf32>, vector<2x4xf32>, vector<2x4xf32>,
+            vector<2x4xf32>, vector<2x4xf32>, vector<2x4xf32>, vector<2x4xf32>>
+      into vector<4x16xf32>
+  // %arg7 == %1 at tupleIndex = -1, offsets = [2, 12]
+  %2 = vector.extract_slices %1, [4, 8], [1, 1]
+    : vector<4x16xf32> into tuple<vector<4x8xf32>, vector<4x8xf32>>
+  // %arg7 == %2 at tupleIndex = 1, offsets = [2, 4]
+
+  // Extract tuple elements.
+  %3 = vector.tuple_get %2, 0 : tuple<vector<4x8xf32>, vector<4x8xf32>>
+  %4 = vector.tuple_get %2, 1 : tuple<vector<4x8xf32>, vector<4x8xf32>>
+  // %arg7 == %4 at tupleIndex = -1, offsets = [2, 4]
+
+  // Swizzle tuple elements.
+  %5 = vector.tuple %4, %3 : vector<4x8xf32>, vector<4x8xf32>
+  // %arg7 == %5 at tupleIndex = 0, offsets = [2, 4]
+  %6 = vector.tuple_get %5, 0 : tuple<vector<4x8xf32>, vector<4x8xf32>>
+  // %arg7 == %6 at tupleIndex = -1, offsets = [2, 4]
+  %7 = vector.extract_slices %6, [2, 4], [1, 1]
+    : vector<4x8xf32> into
+      tuple<vector<2x4xf32>, vector<2x4xf32>, vector<2x4xf32>, vector<2x4xf32>>
+  // %arg7 == %7 at tupleIndex = 3, offsets = [0, 0]
+  %8 = vector.tuple_get %7, 3
+    : tuple<vector<2x4xf32>, vector<2x4xf32>, vector<2x4xf32>, vector<2x4xf32>>
+  // %arg7 == %8
+  return %8 : vector<2x4xf32>
+}
+
 // CHECK-LABEL: func @vector_transfers_vector_element_type
 //      CHECK: %[[C0:.*]] = constant 0 : index
 //      CHECK: %[[C1:.*]] = constant 1 : index
diff --git a/mlir/test/EDSC/builder-api-test.cpp b/mlir/test/EDSC/builder-api-test.cpp
index a9bb0b62dbdf9..0c725e98fa3b5 100644
--- a/mlir/test/EDSC/builder-api-test.cpp
+++ b/mlir/test/EDSC/builder-api-test.cpp
@@ -47,6 +47,7 @@ static MLIRContext &globalContext() {
   }();
   (void)init_once;
   static thread_local MLIRContext context;
+  context.allowUnregisteredDialects();
   return context;
 }
 
@@ -803,7 +804,7 @@ TEST_FUNC(affine_if_op) {
 }
 
 // clang-format off
-// CHECK-LABEL: func @linalg_pointwise
+// CHECK-LABEL: func @linalg_generic_pointwise
 //       CHECK:   linalg.generic {args_in = 2 : i64, args_out = 1 : i64,
 // CHECK-SAME: indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>, affine_map<(d0, d1) -> (d0, d1)>, affine_map<(d0, d1) -> (d0, d1)>],
 // CHECK-SAME: iterator_types = ["parallel", "parallel"]}
@@ -821,14 +822,14 @@ TEST_FUNC(affine_if_op) {
 //       CHECK:     tanh
 //       CHECK:   }: memref<?x?xf32>, memref<?x?xf32>
 // clang-format on
-TEST_FUNC(linalg_pointwise_test) {
+TEST_FUNC(linalg_generic_pointwise_test) {
   using namespace edsc;
   using namespace edsc::ops;
 
   auto f32Type = FloatType::getF32(&globalContext());
   auto memrefType = MemRefType::get(
       {ShapedType::kDynamicSize, ShapedType::kDynamicSize}, f32Type, {}, 0);
-  auto f = makeFunction("linalg_pointwise", {},
+  auto f = makeFunction("linalg_generic_pointwise", {},
                         {memrefType, memrefType, memrefType});
 
   OpBuilder builder(f.getBody());
@@ -837,16 +838,16 @@ TEST_FUNC(linalg_pointwise_test) {
   AffineExpr i, j;
   bindDims(&globalContext(), i, j);
   StructuredIndexed SA(A), SB(B), SC(C);
-  linalg_pointwise_add(SA({i, j}), SB({i, j}), SC({i, j}));
-  linalg_pointwise_max(SA({i, j}), SB({i, j}), SC({i, j}));
-  linalg_pointwise_tanh(SA({i, j}), SC({i, j}));
+  linalg_generic_pointwise_add(SA({i, j}), SB({i, j}), SC({i, j}));
+  linalg_generic_pointwise_max(SA({i, j}), SB({i, j}), SC({i, j}));
+  linalg_generic_pointwise_tanh(SA({i, j}), SC({i, j}));
 
   f.print(llvm::outs());
   f.erase();
 }
 
 // clang-format off
-// CHECK-LABEL: func @linalg_matmul
+// CHECK-LABEL: func @linalg_generic_matmul
 //       CHECK:   linalg.generic {args_in = 2 : i64, args_out = 1 : i64,
 // CHECK-SAME: indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d2)>, affine_map<(d0, d1, d2) -> (d2, d1)>, affine_map<(d0, d1, d2) -> (d0, d1)>],
 // CHECK-SAME: iterator_types = ["parallel", "parallel", "reduction"]}
@@ -856,7 +857,7 @@ TEST_FUNC(linalg_pointwise_test) {
 //       CHECK:     linalg.yield %[[a4]] : f32
 //       CHECK:   }: memref<?x?xf32>, memref<?x?xf32>, memref<?x?xf32>
 // clang-format on
-TEST_FUNC(linalg_matmul_test) {
+TEST_FUNC(linalg_generic_matmul_test) {
   using namespace edsc;
   using namespace edsc::ops;
 
@@ -864,18 +865,18 @@ TEST_FUNC(linalg_matmul_test) {
   auto memrefType = MemRefType::get(
       {ShapedType::kDynamicSize, ShapedType::kDynamicSize}, f32Type, {}, 0);
   auto f =
-      makeFunction("linalg_matmul", {}, {memrefType, memrefType, memrefType});
+      makeFunction("linalg_generic_matmul", {}, {memrefType, memrefType, memrefType});
 
   OpBuilder builder(f.getBody());
   ScopedContext scope(builder, f.getLoc());
-  linalg_matmul(makeValueHandles(llvm::to_vector<3>(f.getArguments())));
+  linalg_generic_matmul(makeValueHandles(llvm::to_vector<3>(f.getArguments())));
 
   f.print(llvm::outs());
   f.erase();
 }
 
 // clang-format off
-// CHECK-LABEL: func @linalg_conv_nhwc
+// CHECK-LABEL: func @linalg_generic_conv_nhwc
 //       CHECK:   linalg.generic {args_in = 2 : i64, args_out = 1 : i64,
 // CHECK-SAME: indexing_maps = [affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d0, d2 * 3 + d4 * 5, d3 * 4 + d5 * 6, d6)>,
 // CHECK-SAME: affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d4, d5, d6, d1)>,
@@ -887,7 +888,7 @@ TEST_FUNC(linalg_matmul_test) {
 //       CHECK:     linalg.yield %[[a4]] : f32
 //       CHECK:   }: memref<?x?x?x?xf32>, memref<?x?x?x?xf32>, memref<?x?x?x?xf32>
 // clang-format on
-TEST_FUNC(linalg_conv_nhwc) {
+TEST_FUNC(linalg_generic_conv_nhwc) {
   using namespace edsc;
   using namespace edsc::ops;
 
@@ -896,12 +897,12 @@ TEST_FUNC(linalg_conv_nhwc) {
       MemRefType::get({ShapedType::kDynamicSize, ShapedType::kDynamicSize,
                        ShapedType::kDynamicSize, ShapedType::kDynamicSize},
                       f32Type, {}, 0);
-  auto f = makeFunction("linalg_conv_nhwc", {},
+  auto f = makeFunction("linalg_generic_conv_nhwc", {},
                         {memrefType, memrefType, memrefType});
 
   OpBuilder builder(f.getBody());
   ScopedContext scope(builder, f.getLoc());
-  linalg_conv_nhwc(makeValueHandles(llvm::to_vector<3>(f.getArguments())),
+  linalg_generic_conv_nhwc(makeValueHandles(llvm::to_vector<3>(f.getArguments())),
                    /*strides=*/{3, 4}, /*dilations=*/{5, 6});
 
   f.print(llvm::outs());
@@ -909,7 +910,7 @@ TEST_FUNC(linalg_conv_nhwc) {
 }
 
 // clang-format off
-// CHECK-LABEL: func @linalg_dilated_conv_nhwc
+// CHECK-LABEL: func @linalg_generic_dilated_conv_nhwc
 //       CHECK:   linalg.generic {args_in = 2 : i64, args_out = 1 : i64,
 // CHECK-SAME: indexing_maps = [affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d0, d3 * 3 + d5 * 5, d4 * 4 + d6 * 6, d2)>,
 // CHECK-SAME: affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d5, d6, d2, d1)>,
@@ -921,7 +922,7 @@ TEST_FUNC(linalg_conv_nhwc) {
 //       CHECK:     linalg.yield %[[a4]] : f32
 //       CHECK:   }: memref<?x?x?x?xf32>, memref<?x?x?x?xf32>, memref<?x?x?x?xf32>
 // clang-format on
-TEST_FUNC(linalg_dilated_conv_nhwc) {
+TEST_FUNC(linalg_generic_dilated_conv_nhwc) {
   using namespace edsc;
   using namespace edsc::ops;
 
@@ -930,12 +931,12 @@ TEST_FUNC(linalg_dilated_conv_nhwc) {
       MemRefType::get({ShapedType::kDynamicSize, ShapedType::kDynamicSize,
                        ShapedType::kDynamicSize, ShapedType::kDynamicSize},
                       f32Type, {}, 0);
-  auto f = makeFunction("linalg_dilated_conv_nhwc", {},
+  auto f = makeFunction("linalg_generic_dilated_conv_nhwc", {},
                         {memrefType, memrefType, memrefType});
 
   OpBuilder builder(f.getBody());
   ScopedContext scope(builder, f.getLoc());
-  linalg_dilated_conv_nhwc(makeValueHandles(f.getArguments()),
+  linalg_generic_dilated_conv_nhwc(makeValueHandles(f.getArguments()),
                            /*depth_multiplier=*/7,
                            /*strides=*/{3, 4}, /*dilations=*/{5, 6});
 
@@ -1018,11 +1019,11 @@ TEST_FUNC(linalg_tensors_test) {
   AffineExpr i, j;
   bindDims(&globalContext(), i, j);
   StructuredIndexed SA(A), SB(B), SC(tensorType);
-  linalg_pointwise_add(SA({i, j}), SB({i, j}), SC({i, j}));
-  linalg_pointwise_max(SA({i, j}), SB({i, j}), SC({i, j}));
-  linalg_pointwise_tanh(SA({i, j}), SC({i, j}));
-  Value o1 = linalg_matmul(A, B, tensorType)->getResult(0);
-  linalg_matmul(A, B, ValueHandle(o1), tensorType);
+  linalg_generic_pointwise_add(SA({i, j}), SB({i, j}), SC({i, j}));
+  linalg_generic_pointwise_max(SA({i, j}), SB({i, j}), SC({i, j}));
+  linalg_generic_pointwise_tanh(SA({i, j}), SC({i, j}));
+  Value o1 = linalg_generic_matmul(A, B, tensorType)->getResult(0);
+  linalg_generic_matmul(A, B, ValueHandle(o1), tensorType);
 
   f.print(llvm::outs());
   f.erase();
@@ -1066,9 +1067,9 @@ TEST_FUNC(memref_vector_matmul_test) {
   ValueHandle A(f.getArgument(0)), B(f.getArgument(1)), C(f.getArgument(2));
   auto contractionBuilder = [](ArrayRef<BlockArgument> args) {
     assert(args.size() == 3 && "expected 3 block arguments");
-    (linalg_yield(vector_matmul(args[0], args[1], args[2])));
+    (linalg_yield(vector_contraction_matmul(args[0], args[1], args[2])));
   };
-  linalg_matmul(A, B, C, contractionBuilder);
+  linalg_generic_matmul(A, B, C, contractionBuilder);
 
   f.print(llvm::outs());
   f.erase();
diff --git a/mlir/test/IR/affine-map.mlir b/mlir/test/IR/affine-map.mlir
index d4c80049b7423..f953c00e2043c 100644
--- a/mlir/test/IR/affine-map.mlir
+++ b/mlir/test/IR/affine-map.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s | FileCheck %s
 
 // Identity maps used in trivial compositions in MemRefs are optimized away.
 // CHECK-NOT: #map{{[0-9]+}} = affine_map<(d0, d1) -> (d0, d1)>
diff --git a/mlir/test/IR/attribute.mlir b/mlir/test/IR/attribute.mlir
index 50d4e8cd14540..a5133abfda2c7 100644
--- a/mlir/test/IR/attribute.mlir
+++ b/mlir/test/IR/attribute.mlir
@@ -33,6 +33,83 @@ func @int_attrs_pass() {
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// Check that the maximum and minimum integer attribute values are
+// representable and preserved during a round-trip.
+//===----------------------------------------------------------------------===//
+
+func @int_attrs_pass() {
+  "test.in_range_attrs"() {
+    // CHECK: attr_00 = -128 : i8
+    attr_00 = -128 : i8,
+    // CHECK-SAME: attr_01 = 127 : i8
+    attr_01 = 127 : i8,
+    // CHECK-SAME: attr_02 = -128 : si8
+    attr_02 = -128 : si8,
+    // CHECK-SAME: attr_03 = 127 : si8
+    attr_03 = 127 : si8,
+    // CHECK-SAME: attr_04 = 255 : ui8
+    attr_04 = 255 : ui8,
+    // CHECK-SAME: attr_05 = -32768 : i16
+    attr_05 = -32768 : i16,
+    // CHECK-SAME: attr_06 = 32767 : i16
+    attr_06 = 32767 : i16,
+    // CHECK-SAME: attr_07 = -32768 : si16
+    attr_07 = -32768 : si16,
+    // CHECK-SAME: attr_08 = 32767 : si16
+    attr_08 = 32767 : si16,
+    // CHECK-SAME: attr_09 = 65535 : ui16
+    attr_09 = 65535 : ui16,
+    // CHECK-SAME: attr_10 = -2147483647 : i32
+    attr_10 = -2147483647 : i32,
+    // CHECK-SAME: attr_11 = 2147483646 : i32
+    attr_11 = 2147483646 : i32,
+    // CHECK-SAME: attr_12 = -2147483647 : si32
+    attr_12 = -2147483647 : si32,
+    // CHECK-SAME: attr_13 = 2147483646 : si32
+    attr_13 = 2147483646 : si32,
+    // CHECK-SAME: attr_14 = 4294967295 : ui32
+    attr_14 = 4294967295 : ui32,
+    // CHECK-SAME: attr_15 = -9223372036854775808 : i64
+    attr_15 = -9223372036854775808 : i64,
+    // CHECK-SAME: attr_16 = 9223372036854775807 : i64
+    attr_16 = 9223372036854775807 : i64,
+    // CHECK-SAME: attr_17 = -9223372036854775808 : si64
+    attr_17 = -9223372036854775808 : si64,
+    // CHECK-SAME: attr_18 = 9223372036854775807 : si64
+    attr_18 = 9223372036854775807 : si64,
+    // CHECK-SAME: attr_19 = 18446744073709551615 : ui64
+    attr_19 = 18446744073709551615 : ui64
+  } : () -> ()
+
+  return
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// Check that positive values larger than 2^n-1 for signless integers
+// are mapped to their negative signed counterpart. This behaviour is
+// undocumented in the language specification, but it is what the
+// parser currently does.
+//===----------------------------------------------------------------------===//
+
+func @int_attrs_pass() {
+  "test.i8_attr"() {
+    // CHECK: attr_00 = -1 : i8
+    attr_00 = 255 : i8,
+    // CHECK-SAME: attr_01 = -1 : i16
+    attr_01 = 65535 : i16,
+    // CHECK-SAME: attr_02 = -1 : i32
+    attr_02 = 4294967295 : i32,
+    // CHECK-SAME: attr_03 = -1 : i64
+    attr_03 = 18446744073709551615 : i64
+  } : () -> ()
+  return
+}
+// -----
+
+
 func @wrong_int_attrs_signedness_fail() {
   // expected-error @+1 {{'si32_attr' failed to satisfy constraint: 32-bit signed integer attribute}}
   "test.int_attrs"() {
diff --git a/mlir/test/IR/core-ops.mlir b/mlir/test/IR/core-ops.mlir
index 454ecfc537913..b50ffa65f1790 100644
--- a/mlir/test/IR/core-ops.mlir
+++ b/mlir/test/IR/core-ops.mlir
@@ -1,8 +1,8 @@
-// RUN: mlir-opt %s | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s | FileCheck %s
 // Verify the printed output can be parsed.
-// RUN: mlir-opt %s | mlir-opt | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s | mlir-opt -allow-unregistered-dialect | FileCheck %s
 // Verify the generic form can be parsed.
-// RUN: mlir-opt -mlir-print-op-generic %s | mlir-opt | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect -mlir-print-op-generic %s | mlir-opt -allow-unregistered-dialect | FileCheck %s
 
 // CHECK: #map0 = affine_map<(d0) -> (d0 + 1)>
 
diff --git a/mlir/test/IR/dense-elements-hex.mlir b/mlir/test/IR/dense-elements-hex.mlir
index f2ce955357fb4..bfd1f1b27e7fe 100644
--- a/mlir/test/IR/dense-elements-hex.mlir
+++ b/mlir/test/IR/dense-elements-hex.mlir
@@ -1,5 +1,5 @@
-// RUN: mlir-opt %s -verify-diagnostics -split-input-file -mlir-print-elementsattrs-with-hex-if-larger=1 | FileCheck %s --check-prefix=HEX
-// RUN: mlir-opt %s -verify-diagnostics -split-input-file | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -verify-diagnostics -split-input-file -mlir-print-elementsattrs-with-hex-if-larger=1 | FileCheck %s --check-prefix=HEX
+// RUN: mlir-opt -allow-unregistered-dialect %s -verify-diagnostics -split-input-file | FileCheck %s
 
 // HEX: dense<"0x00000000000024400000000000001440"> : tensor<2xf64>
 "foo.op"() {dense.attr = dense<[10.0, 5.0]> : tensor<2xf64>} : () -> ()
diff --git a/mlir/test/IR/diagnostic-handler.mlir b/mlir/test/IR/diagnostic-handler.mlir
index d3108e33f78fb..198d5e84e119c 100644
--- a/mlir/test/IR/diagnostic-handler.mlir
+++ b/mlir/test/IR/diagnostic-handler.mlir
@@ -5,9 +5,9 @@
 
 // Emit the first available call stack in the fused location.
 func @constant_out_of_range() {
-  // CHECK: mysource1: error: 'std.constant' op requires attribute's type ('i64') to match op's return type ('i1')
-  // CHECK-NEXT: mysource2: note: called from
-  // CHECK-NEXT: mysource3: note: called from
+  // CHECK: mysource1:0:0: error: 'std.constant' op requires attribute's type ('i64') to match op's return type ('i1')
+  // CHECK-NEXT: mysource2:1:0: note: called from
+  // CHECK-NEXT: mysource3:2:0: note: called from
   %x = "std.constant"() {value = 100} : () -> i1 loc(fused["bar", callsite("foo"("mysource1":0:0) at callsite("mysource2":1:0 at "mysource3":2:0))])
   return
 }
diff --git a/mlir/test/IR/invalid-ops.mlir b/mlir/test/IR/invalid-ops.mlir
index 5f311f34c45e2..3f0804aff96ab 100644
--- a/mlir/test/IR/invalid-ops.mlir
+++ b/mlir/test/IR/invalid-ops.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -split-input-file -verify-diagnostics
+// RUN: mlir-opt -allow-unregistered-dialect %s -split-input-file -verify-diagnostics
 
 func @dim(tensor<1xf32>) {
 ^bb(%0: tensor<1xf32>):
diff --git a/mlir/test/IR/invalid.mlir b/mlir/test/IR/invalid.mlir
index 2c008be930fd7..cbd97cd63b54e 100644
--- a/mlir/test/IR/invalid.mlir
+++ b/mlir/test/IR/invalid.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -split-input-file -verify-diagnostics
+// RUN: mlir-opt -allow-unregistered-dialect %s -split-input-file -verify-diagnostics
 
 // Check different error cases.
 // -----
@@ -23,10 +23,6 @@ func @indexvector(vector<4 x index>) -> () // expected-error {{vector elements m
 
 func @indexmemref(memref<? x index>) -> () // expected-error {{invalid memref element type}}
 
-// -----
-
-func @indextensor(tensor<4 x index>) -> () // expected-error {{invalid tensor element type}}
-
 // -----
 // Test no map in memref type.
 func @memrefs(memref<2x4xi8, >) // expected-error {{expected list element}}
@@ -1231,3 +1227,243 @@ func @negative_value_in_unsigned_vector_attr() {
   // expected-error @+1 {{expected unsigned integer elements, but parsed negative value}}
   "foo"() {bar = dense<[5, -5]> : vector<2xui32>} : () -> ()
 }
+
+// -----
+
+func @large_bound() {
+  "test.out_of_range_attribute"() {
+    // expected-error @+1 {{integer constant out of range for attribute}}
+    attr = -129 : i8
+  } : () -> ()
+  return
+}
+
+// -----
+
+func @large_bound() {
+  "test.out_of_range_attribute"() {
+    // expected-error @+1 {{integer constant out of range for attribute}}
+    attr = 256 : i8
+  } : () -> ()
+  return
+}
+
+// -----
+
+func @large_bound() {
+  "test.out_of_range_attribute"() {
+    // expected-error @+1 {{integer constant out of range for attribute}}
+    attr = -129 : si8
+  } : () -> ()
+  return
+}
+
+// -----
+
+func @large_bound() {
+  "test.out_of_range_attribute"() {
+    // expected-error @+1 {{integer constant out of range for attribute}}
+    attr = 129 : si8
+  } : () -> ()
+  return
+}
+
+// -----
+
+func @large_bound() {
+  "test.out_of_range_attribute"() {
+    // expected-error @+1 {{negative integer literal not valid for unsigned integer type}}
+    attr = -1 : ui8
+  } : () -> ()
+  return
+}
+
+// -----
+
+func @large_bound() {
+  "test.out_of_range_attribute"() {
+    // expected-error @+1 {{integer constant out of range for attribute}}
+    attr = 256 : ui8
+  } : () -> ()
+  return
+}
+
+// -----
+
+func @large_bound() {
+  "test.out_of_range_attribute"() {
+    // expected-error @+1 {{integer constant out of range for attribute}}
+    attr = -32769 : i16
+  } : () -> ()
+  return
+}
+
+// -----
+
+func @large_bound() {
+  "test.out_of_range_attribute"() {
+    // expected-error @+1 {{integer constant out of range for attribute}}
+    attr = 65536 : i16
+  } : () -> ()
+  return
+}
+
+// -----
+
+func @large_bound() {
+  "test.out_of_range_attribute"() {
+    // expected-error @+1 {{integer constant out of range for attribute}}
+    attr = -32769 : si16
+  } : () -> ()
+  return
+}
+
+// -----
+
+func @large_bound() {
+  "test.out_of_range_attribute"() {
+    // expected-error @+1 {{integer constant out of range for attribute}}
+    attr = 32768 : si16
+  } : () -> ()
+  return
+}
+
+// -----
+
+func @large_bound() {
+  "test.out_of_range_attribute"() {
+    // expected-error @+1 {{negative integer literal not valid for unsigned integer type}}
+    attr = -1 : ui16
+  } : () -> ()
+  return
+}
+
+// -----
+
+func @large_bound() {
+  "test.out_of_range_attribute"() {
+    // expected-error @+1 {{integer constant out of range for attribute}}
+    attr = 65536: ui16
+  } : () -> ()
+  return
+}
+
+// -----
+
+func @large_bound() {
+  "test.out_of_range_attribute"() {
+    // expected-error @+1 {{integer constant out of range for attribute}}
+    attr = -2147483649 : i32
+  } : () -> ()
+  return
+}
+
+// -----
+
+func @large_bound() {
+  "test.out_of_range_attribute"() {
+    // expected-error @+1 {{integer constant out of range for attribute}}
+    attr = 4294967296 : i32
+  } : () -> ()
+  return
+}
+
+// -----
+
+func @large_bound() {
+  "test.out_of_range_attribute"() {
+    // expected-error @+1 {{integer constant out of range for attribute}}
+    attr = -2147483649 : si32
+  } : () -> ()
+  return
+}
+
+// -----
+
+func @large_bound() {
+  "test.out_of_range_attribute"() {
+    // expected-error @+1 {{integer constant out of range for attribute}}
+    attr = 2147483648 : si32
+  } : () -> ()
+  return
+}
+
+// -----
+
+func @large_bound() {
+  "test.out_of_range_attribute"() {
+    // expected-error @+1 {{negative integer literal not valid for unsigned integer type}}
+    attr = -1 : ui32
+  } : () -> ()
+  return
+}
+
+// -----
+
+func @large_bound() {
+  "test.out_of_range_attribute"() {
+    // expected-error @+1 {{integer constant out of range for attribute}}
+    attr = 4294967296 : ui32
+  } : () -> ()
+  return
+}
+
+// -----
+
+func @large_bound() {
+  "test.out_of_range_attribute"() {
+    // expected-error @+1 {{integer constant out of range for attribute}}
+    attr = -9223372036854775809 : i64
+  } : () -> ()
+  return
+}
+
+// -----
+
+func @large_bound() {
+  "test.out_of_range_attribute"() {
+    // expected-error @+1 {{integer constant out of range for attribute}}
+    attr = 18446744073709551616 : i64
+  } : () -> ()
+  return
+}
+
+// -----
+
+func @large_bound() {
+  "test.out_of_range_attribute"() {
+    // expected-error @+1 {{integer constant out of range for attribute}}
+    attr = -9223372036854775809 : si64
+  } : () -> ()
+  return
+}
+
+// -----
+
+func @large_bound() {
+  "test.out_of_range_attribute"() {
+    // expected-error @+1 {{integer constant out of range for attribute}}
+    attr = 9223372036854775808 : si64
+  } : () -> ()
+  return
+}
+
+// -----
+
+func @large_bound() {
+  "test.out_of_range_attribute"() {
+    // expected-error @+1 {{negative integer literal not valid for unsigned integer type}}
+    attr = -1 : ui64
+  } : () -> ()
+  return
+}
+
+// -----
+
+func @large_bound() {
+  "test.out_of_range_attribute"() {
+    // expected-error @+1 {{integer constant out of range for attribute}}
+    attr = 18446744073709551616 : ui64
+  } : () -> ()
+  return
+}
diff --git a/mlir/test/IR/locations.mlir b/mlir/test/IR/locations.mlir
index b0039dde15e38..6227ba59e6ec0 100644
--- a/mlir/test/IR/locations.mlir
+++ b/mlir/test/IR/locations.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -mlir-print-debuginfo | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -mlir-print-debuginfo | FileCheck %s
 // This test verifies that debug locations are round-trippable.
 
 #set0 = affine_set<(d0) : (1 == 0)>
diff --git a/mlir/test/IR/module-op.mlir b/mlir/test/IR/module-op.mlir
index 84a404b1acfac..2d4ea839c1378 100644
--- a/mlir/test/IR/module-op.mlir
+++ b/mlir/test/IR/module-op.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -split-input-file -mlir-print-debuginfo | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -split-input-file -mlir-print-debuginfo | FileCheck %s
 
 // CHECK: module {
 module {
diff --git a/mlir/test/IR/op-stats.mlir b/mlir/test/IR/op-stats.mlir
index 63abe6d12675f..5c5b54d961b0a 100644
--- a/mlir/test/IR/op-stats.mlir
+++ b/mlir/test/IR/op-stats.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -print-op-stats %s -o=/dev/null 2>&1 | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect -print-op-stats %s -o=/dev/null 2>&1 | FileCheck %s
 
 func @main(tensor<4xf32>, tensor<4xf32>) -> tensor<4xf32> {
 ^bb0(%arg0: tensor<4xf32>, %arg1: tensor<4xf32>):
diff --git a/mlir/test/IR/opaque_locations.mlir b/mlir/test/IR/opaque_locations.mlir
index 3718d345197d0..b33ca4cfea2dd 100644
--- a/mlir/test/IR/opaque_locations.mlir
+++ b/mlir/test/IR/opaque_locations.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -test-opaque-loc -mlir-print-debuginfo | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -test-opaque-loc -mlir-print-debuginfo | FileCheck %s
 // This test verifies that debug opaque locations can be printed.
 
 #set0 = affine_set<(d0) : (1 == 0)>
diff --git a/mlir/test/IR/parser.mlir b/mlir/test/IR/parser.mlir
index 45ee1e1d89cd3..a42bf3238ea36 100644
--- a/mlir/test/IR/parser.mlir
+++ b/mlir/test/IR/parser.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s | FileCheck %s
 
 // CHECK-DAG: #map{{[0-9]+}} = affine_map<(d0, d1, d2, d3, d4)[s0] -> (d0, d1, d2, d4, d3)>
 #map0 = affine_map<(d0, d1, d2, d3, d4)[s0] -> (d0, d1, d2, d4, d3)>
diff --git a/mlir/test/IR/pretty-locations.mlir b/mlir/test/IR/pretty-locations.mlir
index 116bfa51233e6..79d282ff73151 100644
--- a/mlir/test/IR/pretty-locations.mlir
+++ b/mlir/test/IR/pretty-locations.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -mlir-print-debuginfo -mlir-pretty-debuginfo | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -mlir-print-debuginfo -mlir-pretty-debuginfo | FileCheck %s
 
 #set0 = affine_set<(d0) : (1 == 0)>
 
diff --git a/mlir/test/IR/pretty-region-args.mlir b/mlir/test/IR/pretty-region-args.mlir
index 6e980fa2c3fb9..caed3691fa955 100644
--- a/mlir/test/IR/pretty-region-args.mlir
+++ b/mlir/test/IR/pretty-region-args.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s | FileCheck %s
 
 // CHECK-LABEL: func @custom_region_names
 func @custom_region_names() -> () {
diff --git a/mlir/test/IR/print-op-local-scope.mlir b/mlir/test/IR/print-op-local-scope.mlir
index b6f36052d70a4..93b25fca943eb 100644
--- a/mlir/test/IR/print-op-local-scope.mlir
+++ b/mlir/test/IR/print-op-local-scope.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -mlir-print-local-scope | FileCheck %s --dump-input-on-failure
+// RUN: mlir-opt -allow-unregistered-dialect %s -mlir-print-local-scope | FileCheck %s --dump-input-on-failure
 
 // CHECK: "foo.op"() : () -> memref<?xf32, affine_map<(d0) -> (d0 * 2)>>
 "foo.op"() : () -> (memref<?xf32, affine_map<(d0) -> (2*d0)>>)
diff --git a/mlir/test/IR/region.mlir b/mlir/test/IR/region.mlir
index 27e19d612e2d2..a32371cb7155e 100644
--- a/mlir/test/IR/region.mlir
+++ b/mlir/test/IR/region.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -split-input-file -verify-diagnostics | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -split-input-file -verify-diagnostics | FileCheck %s
 
 //===----------------------------------------------------------------------===//
 // Test the number of regions
diff --git a/mlir/test/IR/test-symbol-dce.mlir b/mlir/test/IR/test-symbol-dce.mlir
index cc75fecef8774..9193c4716c3c0 100644
--- a/mlir/test/IR/test-symbol-dce.mlir
+++ b/mlir/test/IR/test-symbol-dce.mlir
@@ -1,5 +1,5 @@
-// RUN: mlir-opt %s -symbol-dce -split-input-file -verify-diagnostics | FileCheck %s
-// RUN: mlir-opt %s -pass-pipeline="module(symbol-dce)" -split-input-file | FileCheck %s --check-prefix=NESTED
+// RUN: mlir-opt -allow-unregistered-dialect %s -symbol-dce -split-input-file -verify-diagnostics | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -pass-pipeline="module(symbol-dce)" -split-input-file | FileCheck %s --check-prefix=NESTED
 
 // Check that trivially dead and trivially live non-nested cases are handled.
 
diff --git a/mlir/test/IR/test-symbol-rauw.mlir b/mlir/test/IR/test-symbol-rauw.mlir
index 25d22e4322567..2924c20df7322 100644
--- a/mlir/test/IR/test-symbol-rauw.mlir
+++ b/mlir/test/IR/test-symbol-rauw.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -test-symbol-rauw -split-input-file | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -test-symbol-rauw -split-input-file | FileCheck %s
 
 // Symbol references to the module itself don't affect uses of symbols within
 // its table.
diff --git a/mlir/test/IR/test-symbol-uses.mlir b/mlir/test/IR/test-symbol-uses.mlir
index f95ac6b2d701c..17f4a3054f9fe 100644
--- a/mlir/test/IR/test-symbol-uses.mlir
+++ b/mlir/test/IR/test-symbol-uses.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -test-symbol-uses -split-input-file -verify-diagnostics
+// RUN: mlir-opt -allow-unregistered-dialect %s -test-symbol-uses -split-input-file -verify-diagnostics
 
 // Symbol references to the module itself don't affect uses of symbols within
 // its table.
diff --git a/mlir/test/IR/traits.mlir b/mlir/test/IR/traits.mlir
index 655fbb89b2d48..a076d8f103080 100644
--- a/mlir/test/IR/traits.mlir
+++ b/mlir/test/IR/traits.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -split-input-file -verify-diagnostics | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -split-input-file -verify-diagnostics | FileCheck %s
 
 // CHECK: succeededSameOperandsElementType
 func @succeededSameOperandsElementType(%t10x10 : tensor<10x10xf32>, %t1f: tensor<1xf32>, %v1: vector<1xf32>, %t1i: tensor<1xi32>, %sf: f32) {
diff --git a/mlir/test/IR/wrapping_op.mlir b/mlir/test/IR/wrapping_op.mlir
index 92918f116889e..661fb04518f40 100644
--- a/mlir/test/IR/wrapping_op.mlir
+++ b/mlir/test/IR/wrapping_op.mlir
@@ -1,5 +1,5 @@
-// RUN: mlir-opt %s | FileCheck %s
-// RUN: mlir-opt -mlir-print-op-generic -mlir-print-debuginfo %s | FileCheck %s --check-prefix=CHECK-GENERIC
+// RUN: mlir-opt -allow-unregistered-dialect %s | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect -mlir-print-op-generic -mlir-print-debuginfo %s | FileCheck %s --check-prefix=CHECK-GENERIC
 
 // CHECK-LABEL: func @wrapping_op
 // CHECK-GENERIC: "func"
diff --git a/mlir/test/Target/llvmir-intrinsics.mlir b/mlir/test/Target/llvmir-intrinsics.mlir
index 7be5e5fd596e5..c332bc23b814e 100644
--- a/mlir/test/Target/llvmir-intrinsics.mlir
+++ b/mlir/test/Target/llvmir-intrinsics.mlir
@@ -27,6 +27,15 @@ llvm.func @exp_test(%arg0: !llvm.float, %arg1: !llvm<"<8 x float>">) {
   llvm.return
 }
 
+// CHECK-LABEL: @exp2_test
+llvm.func @exp2_test(%arg0: !llvm.float, %arg1: !llvm<"<8 x float>">) {
+  // CHECK: call float @llvm.exp2.f32
+  "llvm.intr.exp2"(%arg0) : (!llvm.float) -> !llvm.float
+  // CHECK: call <8 x float> @llvm.exp2.v8f32
+  "llvm.intr.exp2"(%arg1) : (!llvm<"<8 x float>">) -> !llvm<"<8 x float>">
+  llvm.return
+}
+
 // CHECK-LABEL: @log_test
 llvm.func @log_test(%arg0: !llvm.float, %arg1: !llvm<"<8 x float>">) {
   // CHECK: call float @llvm.log.f32
diff --git a/mlir/test/Target/llvmir-invalid.mlir b/mlir/test/Target/llvmir-invalid.mlir
index dce4189f733fe..692975b2cf08d 100644
--- a/mlir/test/Target/llvmir-invalid.mlir
+++ b/mlir/test/Target/llvmir-invalid.mlir
@@ -17,3 +17,23 @@ llvm.func @no_nested_struct() -> !llvm<"[2 x [2 x [2 x {i32}]]]"> {
 
 // expected-error @+1 {{unsupported constant value}}
 llvm.mlir.global internal constant @test([2.5, 7.4]) : !llvm<"[2 x double]">
+
+// -----
+
+// expected-error @+1 {{LLVM attribute 'noinline' does not expect a value}}
+llvm.func @passthrough_unexpected_value() attributes {passthrough = [["noinline", "42"]]}
+
+// -----
+
+// expected-error @+1 {{LLVM attribute 'alignstack' expects a value}}
+llvm.func @passthrough_expected_value() attributes {passthrough = ["alignstack"]}
+
+// -----
+
+// expected-error @+1 {{expected 'passthrough' to contain string or array attributes}}
+llvm.func @passthrough_wrong_type() attributes {passthrough = [42]}
+
+// -----
+
+// expected-error @+1 {{expected arrays within 'passthrough' to contain two strings}}
+llvm.func @passthrough_wrong_type() attributes {passthrough = [[42, 42]]}
diff --git a/mlir/test/Target/llvmir.mlir b/mlir/test/Target/llvmir.mlir
index 59c43b82cca5f..3909a8a31228f 100644
--- a/mlir/test/Target/llvmir.mlir
+++ b/mlir/test/Target/llvmir.mlir
@@ -1200,4 +1200,16 @@ llvm.func @callFenceInst() {
   // CHECK: fence release
   llvm.fence syncscope("") release
   llvm.return
-}
\ No newline at end of file
+}
+
+// CHECK-LABEL @passthrough
+// CHECK: #[[ATTR_GROUP:[0-9]*]]
+llvm.func @passthrough() attributes {passthrough = ["noinline", ["alignstack", "4"], "null-pointer-is-valid", ["foo", "bar"]]} {
+  llvm.return
+}
+
+// CHECK: attributes #[[ATTR_GROUP]] = {
+// CHECK-DAG: noinline
+// CHECK-DAG: alignstack=4
+// CHECK-DAG: "null-pointer-is-valid"
+// CHECK-DAG: "foo"="bar"
diff --git a/mlir/test/Transforms/canonicalize-dce.mlir b/mlir/test/Transforms/canonicalize-dce.mlir
index eee83559979f8..b93af002823a8 100644
--- a/mlir/test/Transforms/canonicalize-dce.mlir
+++ b/mlir/test/Transforms/canonicalize-dce.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -split-input-file -pass-pipeline='func(canonicalize)' | FileCheck %s --dump-input=fail
+// RUN: mlir-opt -allow-unregistered-dialect %s -split-input-file -pass-pipeline='func(canonicalize)' | FileCheck %s --dump-input=fail
 
 // Test case: Simple case of deleting a dead pure op.
 
diff --git a/mlir/test/Transforms/canonicalize.mlir b/mlir/test/Transforms/canonicalize.mlir
index 877693b78c87a..5e54c62625620 100644
--- a/mlir/test/Transforms/canonicalize.mlir
+++ b/mlir/test/Transforms/canonicalize.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -pass-pipeline='func(canonicalize)' -split-input-file | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -pass-pipeline='func(canonicalize)' -split-input-file | FileCheck %s
 
 // CHECK-LABEL: func @test_subi_zero
 func @test_subi_zero(%arg0: i32) -> i32 {
diff --git a/mlir/test/Transforms/constant-fold.mlir b/mlir/test/Transforms/constant-fold.mlir
index b156ca918750c..304c9875d6fef 100644
--- a/mlir/test/Transforms/constant-fold.mlir
+++ b/mlir/test/Transforms/constant-fold.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -split-input-file -test-constant-fold | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -split-input-file -test-constant-fold | FileCheck %s
 
 // -----
 
diff --git a/mlir/test/Transforms/cse.mlir b/mlir/test/Transforms/cse.mlir
index 8e526176a0261..9be014d8a909a 100644
--- a/mlir/test/Transforms/cse.mlir
+++ b/mlir/test/Transforms/cse.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -pass-pipeline='func(cse)' | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -pass-pipeline='func(cse)' | FileCheck %s
 
 // CHECK-DAG: #map0 = affine_map<(d0) -> (d0 mod 2)>
 #map0 = affine_map<(d0) -> (d0 mod 2)>
diff --git a/mlir/test/Transforms/inlining-dce.mlir b/mlir/test/Transforms/inlining-dce.mlir
index d9c8bf983c195..73b6489127d8f 100644
--- a/mlir/test/Transforms/inlining-dce.mlir
+++ b/mlir/test/Transforms/inlining-dce.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -inline | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -inline | FileCheck %s
 
 // This file tests the callgraph dead code elimination performed by the inliner.
 
diff --git a/mlir/test/Transforms/location-snapshot.mlir b/mlir/test/Transforms/location-snapshot.mlir
index 4153998a5a90e..cde2018ba939a 100644
--- a/mlir/test/Transforms/location-snapshot.mlir
+++ b/mlir/test/Transforms/location-snapshot.mlir
@@ -1,5 +1,5 @@
-// RUN: mlir-opt -snapshot-op-locations='filename=%/t' -mlir-print-debuginfo %s | FileCheck %s -DFILE=%/t
-// RUN: mlir-opt -snapshot-op-locations='filename=%/t tag='tagged'' -mlir-print-debuginfo %s | FileCheck %s --check-prefix=TAG -DFILE=%/t
+// RUN: mlir-opt -allow-unregistered-dialect -snapshot-op-locations='filename=%/t' -mlir-print-debuginfo %s | FileCheck %s -DFILE=%/t
+// RUN: mlir-opt -allow-unregistered-dialect -snapshot-op-locations='filename=%/t tag='tagged'' -mlir-print-debuginfo %s | FileCheck %s --check-prefix=TAG -DFILE=%/t
 
 // CHECK-LABEL: func @function
 // CHECK-NEXT: loc("[[FILE]]":{{[0-9]+}}:{{[0-9]+}})
diff --git a/mlir/test/Transforms/loop-coalescing.mlir b/mlir/test/Transforms/loop-coalescing.mlir
index d10cf19543eee..b25f91214c955 100644
--- a/mlir/test/Transforms/loop-coalescing.mlir
+++ b/mlir/test/Transforms/loop-coalescing.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -loop-coalescing %s | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect -loop-coalescing %s | FileCheck %s
 
 // CHECK-LABEL: @one_3d_nest
 func @one_3d_nest() {
diff --git a/mlir/test/Transforms/loop-fusion-dependence-check.mlir b/mlir/test/Transforms/loop-fusion-dependence-check.mlir
index 4b5c77839fb22..3d8f8017826fd 100644
--- a/mlir/test/Transforms/loop-fusion-dependence-check.mlir
+++ b/mlir/test/Transforms/loop-fusion-dependence-check.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -test-loop-fusion -test-loop-fusion-dependence-check -split-input-file -verify-diagnostics | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -test-loop-fusion -test-loop-fusion-dependence-check -split-input-file -verify-diagnostics | FileCheck %s
 
 // -----
 
diff --git a/mlir/test/Transforms/loop-fusion.mlir b/mlir/test/Transforms/loop-fusion.mlir
index 89140e42bd8fc..d1e1587ea2fe0 100644
--- a/mlir/test/Transforms/loop-fusion.mlir
+++ b/mlir/test/Transforms/loop-fusion.mlir
@@ -1,5 +1,5 @@
-// RUN: mlir-opt %s -affine-loop-fusion -split-input-file | FileCheck %s
-// RUN: mlir-opt %s -affine-loop-fusion -fusion-maximal -split-input-file | FileCheck %s --check-prefix=MAXIMAL
+// RUN: mlir-opt -allow-unregistered-dialect %s -affine-loop-fusion -split-input-file | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -affine-loop-fusion -fusion-maximal -split-input-file | FileCheck %s --check-prefix=MAXIMAL
 
 // TODO(andydavis) Add more tests:
 // *) Add nested fusion test cases when non-constant loop bound support is
diff --git a/mlir/test/Transforms/loop-invariant-code-motion.mlir b/mlir/test/Transforms/loop-invariant-code-motion.mlir
index 5873532fc2544..494bfa45345c3 100644
--- a/mlir/test/Transforms/loop-invariant-code-motion.mlir
+++ b/mlir/test/Transforms/loop-invariant-code-motion.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -loop-invariant-code-motion -split-input-file | FileCheck %s
+// RUN: mlir-opt %s  -split-input-file -loop-invariant-code-motion | FileCheck %s
 
 func @nested_loops_both_having_invariant_code() {
   %m = alloc() : memref<10xf32>
@@ -25,6 +25,8 @@ func @nested_loops_both_having_invariant_code() {
   return
 }
 
+// -----
+
 func @nested_loops_code_invariant_to_both() {
   %m = alloc() : memref<10xf32>
   %cf7 = constant 7.0 : f32
@@ -44,6 +46,8 @@ func @nested_loops_code_invariant_to_both() {
   return
 }
 
+// -----
+
 func @single_loop_nothing_invariant() {
   %m1 = alloc() : memref<10xf32>
   %m2 = alloc() : memref<10xf32>
@@ -65,6 +69,8 @@ func @single_loop_nothing_invariant() {
   return
 }
 
+// -----
+
 func @invariant_code_inside_affine_if() {
   %m = alloc() : memref<10xf32>
   %cf8 = constant 8.0 : f32
@@ -81,7 +87,7 @@ func @invariant_code_inside_affine_if() {
   // CHECK: %0 = alloc() : memref<10xf32>
   // CHECK-NEXT: %cst = constant 8.000000e+00 : f32
   // CHECK-NEXT: affine.for %arg0 = 0 to 10 {
-  // CHECK-NEXT: %1 = affine.apply #map3(%arg0)
+  // CHECK-NEXT: %1 = affine.apply #map0(%arg0)
   // CHECK-NEXT: affine.if #set0(%arg0, %1) {
   // CHECK-NEXT: %2 = addf %cst, %cst : f32
   // CHECK-NEXT: affine.store %2, %0[%arg0] : memref<10xf32>
@@ -91,6 +97,8 @@ func @invariant_code_inside_affine_if() {
   return
 }
 
+// -----
+
 func @invariant_affine_if() {
   %m = alloc() : memref<10xf32>
   %cf8 = constant 8.0 : f32
@@ -114,6 +122,8 @@ func @invariant_affine_if() {
   return
 }
 
+// -----
+
 func @invariant_affine_if2() {
   %m = alloc() : memref<10xf32>
   %cf8 = constant 8.0 : f32
@@ -139,6 +149,8 @@ func @invariant_affine_if2() {
   return
 }
 
+// -----
+
 func @invariant_affine_nested_if() {
   %m = alloc() : memref<10xf32>
   %cf8 = constant 8.0 : f32
@@ -169,6 +181,8 @@ func @invariant_affine_nested_if() {
   return
 }
 
+// -----
+
 func @invariant_affine_nested_if_else() {
   %m = alloc() : memref<10xf32>
   %cf8 = constant 8.0 : f32
@@ -205,6 +219,8 @@ func @invariant_affine_nested_if_else() {
   return
 }
 
+// -----
+
 func @invariant_loop_dialect() {
   %ci0 = constant 0 : index
   %ci10 = constant 10 : index
@@ -226,6 +242,8 @@ func @invariant_loop_dialect() {
   return
 }
 
+// -----
+
 func @variant_loop_dialect() {
   %ci0 = constant 0 : index
   %ci10 = constant 10 : index
@@ -244,3 +262,33 @@ func @variant_loop_dialect() {
 
   return
 }
+
+// -----
+
+func @parallel_loop_with_invariant() {
+  %c0 = constant 0 : index
+  %c10 = constant 10 : index
+  %c1 = constant 1 : index
+  %c7 = constant 7 : i32
+  %c8 = constant 8 : i32
+  loop.parallel (%arg0, %arg1) = (%c0, %c0) to (%c10, %c10) step (%c1, %c1) {
+      %v0 = addi %c7, %c8 : i32
+      %v3 = addi %arg0, %arg1 : index
+  }
+
+  // CHECK-LABEL: func @parallel_loop_with_invariant
+  // CHECK: %c0 = constant 0 : index
+  // CHECK-NEXT: %c10 = constant 10 : index
+  // CHECK-NEXT: %c1 = constant 1 : index
+  // CHECK-NEXT: %c7_i32 = constant 7 : i32
+  // CHECK-NEXT: %c8_i32 = constant 8 : i32
+  // CHECK-NEXT: addi %c7_i32, %c8_i32 : i32
+  // CHECK-NEXT: loop.parallel (%arg0, %arg1) = (%c0, %c0) to (%c10, %c10) step (%c1, %c1)
+  // CHECK-NEXT:   addi %arg0, %arg1 : index
+  // CHECK-NEXT:   yield
+  // CHECK-NEXT: }
+  // CHECK-NEXT: return
+
+  return
+}
+
diff --git a/mlir/test/Transforms/memref-dataflow-opt.mlir b/mlir/test/Transforms/memref-dataflow-opt.mlir
index 0fe8f715c0e7b..deb193e02bdaf 100644
--- a/mlir/test/Transforms/memref-dataflow-opt.mlir
+++ b/mlir/test/Transforms/memref-dataflow-opt.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -memref-dataflow-opt | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -memref-dataflow-opt | FileCheck %s
 
 // CHECK-DAG: [[MAP0:#map[0-9]+]] = affine_map<(d0, d1) -> (d1 + 1)>
 // CHECK-DAG: [[MAP1:#map[0-9]+]] = affine_map<(d0, d1) -> (d0)>
diff --git a/mlir/test/Transforms/parallel-loop-collapsing.mlir b/mlir/test/Transforms/parallel-loop-collapsing.mlir
new file mode 100644
index 0000000000000..6fcb78cf4c567
--- /dev/null
+++ b/mlir/test/Transforms/parallel-loop-collapsing.mlir
@@ -0,0 +1,51 @@
+// RUN: mlir-opt -allow-unregistered-dialect %s -pass-pipeline='func(parallel-loop-collapsing{collapsed-indices-0=0,3 collapsed-indices-1=1,4 collapsed-indices-2=2}, canonicalize)' | FileCheck %s
+
+func @parallel_many_dims() {
+  %c0 = constant 0 : index
+  %c1 = constant 1 : index
+  %c2 = constant 2 : index
+  %c3 = constant 3 : index
+  %c4 = constant 4 : index
+  %c5 = constant 5 : index
+  %c6 = constant 6 : index
+  %c7 = constant 7 : index
+  %c8 = constant 8 : index
+  %c9 = constant 9 : index
+  %c10 = constant 10 : index
+  %c11 = constant 11 : index
+  %c12 = constant 12 : index
+  %c13 = constant 13 : index
+  %c14 = constant 14 : index
+
+  loop.parallel (%i0, %i1, %i2, %i3, %i4) = (%c0, %c3, %c6, %c9, %c12) to (%c2, %c5, %c8, %c11, %c14)
+                                          step (%c1, %c4, %c7, %c10, %c13) {
+    %result = "magic.op"(%i0, %i1, %i2, %i3, %i4): (index, index, index, index, index) -> index
+  }
+  return
+}
+
+// CHECK-LABEL: func @parallel_many_dims() {
+// CHECK:         [[C6:%.*]] = constant 6 : index
+// CHECK:         [[C7:%.*]] = constant 7 : index
+// CHECK:         [[C9:%.*]] = constant 9 : index
+// CHECK:         [[C10:%.*]] = constant 10 : index
+// CHECK:         [[C12:%.*]] = constant 12 : index
+// CHECK:         [[C13:%.*]] = constant 13 : index
+// CHECK:         [[C3:%.*]] = constant 3 : index
+// CHECK:         [[C0:%.*]] = constant 0 : index
+// CHECK:         [[C1:%.*]] = constant 1 : index
+// CHECK:         [[C2:%.*]] = constant 2 : index
+// CHECK:         loop.parallel ([[NEW_I0:%.*]], [[NEW_I1:%.*]], [[NEW_I2:%.*]]) = ([[C0]], [[C0]], [[C0]]) to ([[C2]], [[C1]], [[C1]]) step ([[C1]], [[C1]], [[C1]]) {
+// CHECK:           [[I0:%.*]] = remi_signed [[NEW_I0]], [[C2]] : index
+// CHECK:           [[I3_COUNT:%.*]] = divi_signed [[NEW_I0]], [[C1]] : index
+// CHECK:           [[I4_COUNT:%.*]] = divi_signed [[NEW_I1]], [[C1]] : index
+// CHECK:           [[VAL_16:%.*]] = muli [[I4_COUNT]], [[C13]] : index
+// CHECK:           [[I4:%.*]] = addi [[VAL_16]], [[C12]] : index
+// CHECK:           [[VAL_18:%.*]] = muli [[I3_COUNT]], [[C10]] : index
+// CHECK:           [[I3:%.*]] = addi [[VAL_18]], [[C9]] : index
+// CHECK:           [[VAL_20:%.*]] = muli [[NEW_I2]], [[C7]] : index
+// CHECK:           [[I2:%.*]] = addi [[VAL_20]], [[C6]] : index
+// CHECK:           "magic.op"([[I0]], [[C3]], [[I2]], [[I3]], [[I4]]) : (index, index, index, index, index) -> index
+// CHECK:           loop.yield
+// CHECK-NEXT:    }
+// CHECK-NEXT:    return
diff --git a/mlir/test/Transforms/parametric-mapping.mlir b/mlir/test/Transforms/parametric-mapping.mlir
index fdfd8cf526ed7..a7981e5beddf8 100644
--- a/mlir/test/Transforms/parametric-mapping.mlir
+++ b/mlir/test/Transforms/parametric-mapping.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -test-mapping-to-processing-elements %s | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect -test-mapping-to-processing-elements %s | FileCheck %s
 
 // CHECK-LABEL: @map1d
 //       CHECK: (%[[lb:.*]]: index, %[[ub:.*]]: index, %[[step:.*]]: index) {
diff --git a/mlir/test/Transforms/pipeline-data-transfer.mlir b/mlir/test/Transforms/pipeline-data-transfer.mlir
index a2ff126255209..842c1d6210e7c 100644
--- a/mlir/test/Transforms/pipeline-data-transfer.mlir
+++ b/mlir/test/Transforms/pipeline-data-transfer.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -split-input-file -affine-pipeline-data-transfer | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -split-input-file -affine-pipeline-data-transfer | FileCheck %s
 
 // -----
 
diff --git a/mlir/test/Transforms/single-parallel-loop-collapsing.mlir b/mlir/test/Transforms/single-parallel-loop-collapsing.mlir
new file mode 100644
index 0000000000000..6662411eaf148
--- /dev/null
+++ b/mlir/test/Transforms/single-parallel-loop-collapsing.mlir
@@ -0,0 +1,34 @@
+// RUN: mlir-opt -allow-unregistered-dialect %s -pass-pipeline='func(parallel-loop-collapsing{collapsed-indices-0=0,1}, canonicalize)' | FileCheck %s
+
+func @collapse_to_single() {
+  %c0 = constant 3 : index
+  %c1 = constant 7 : index
+  %c2 = constant 11 : index
+  %c3 = constant 29 : index
+  %c4 = constant 3 : index
+  %c5 = constant 4 : index
+  loop.parallel (%i0, %i1) = (%c0, %c1) to (%c2, %c3) step (%c4, %c5) {
+    %result = "magic.op"(%i0, %i1): (index, index) -> index
+  }
+  return
+}
+
+// CHECK-LABEL: func @collapse_to_single() {
+// CHECK:         [[C7:%.*]] = constant 7 : index
+// CHECK:         [[C4:%.*]] = constant 4 : index
+// CHECK:         [[C18:%.*]] = constant 18 : index
+// CHECK:         [[C3:%.*]] = constant 3 : index
+// CHECK:         [[C6:%.*]] = constant 6 : index
+// CHECK:         [[C0:%.*]] = constant 0 : index
+// CHECK:         [[C1:%.*]] = constant 1 : index
+// CHECK:         loop.parallel ([[NEW_I:%.*]]) = ([[C0]]) to ([[C18]]) step ([[C1]]) {
+// CHECK:           [[I0_COUNT:%.*]] = remi_signed [[NEW_I]], [[C3]] : index
+// CHECK:           [[I1_COUNT:%.*]] = divi_signed [[NEW_I]], [[C6]] : index
+// CHECK:           [[VAL_10:%.*]] = muli [[I1_COUNT]], [[C4]] : index
+// CHECK:           [[I1:%.*]] = addi [[VAL_10]], [[C7]] : index
+// CHECK:           [[VAL_12:%.*]] = muli [[I0_COUNT]], [[C3]] : index
+// CHECK:           [[I0:%.*]] = addi [[VAL_12]], [[C3]] : index
+// CHECK:           "magic.op"([[I0]], [[I1]]) : (index, index) -> index
+// CHECK:           loop.yield
+// CHECK-NEXT:    }
+// CHECK-NEXT:    return
diff --git a/mlir/test/Transforms/strip-debuginfo.mlir b/mlir/test/Transforms/strip-debuginfo.mlir
index 49bb22052f9a9..f1bf62e1d172a 100644
--- a/mlir/test/Transforms/strip-debuginfo.mlir
+++ b/mlir/test/Transforms/strip-debuginfo.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -mlir-print-debuginfo -strip-debuginfo | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -mlir-print-debuginfo -strip-debuginfo | FileCheck %s
 // This test verifies that debug locations are stripped.
 
 #set0 = affine_set<(d0) : (1 == 0)>
diff --git a/mlir/test/Transforms/test-canonicalize.mlir b/mlir/test/Transforms/test-canonicalize.mlir
index 920fd8c5f989c..0cd308d378c13 100644
--- a/mlir/test/Transforms/test-canonicalize.mlir
+++ b/mlir/test/Transforms/test-canonicalize.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -pass-pipeline='func(canonicalize)' | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -pass-pipeline='func(canonicalize)' | FileCheck %s
 
 // CHECK-LABEL: func @remove_op_with_inner_ops_pattern
 func @remove_op_with_inner_ops_pattern() {
diff --git a/mlir/test/Transforms/test-inlining.mlir b/mlir/test/Transforms/test-inlining.mlir
index d3a1c7ca46bff..099410e0ab221 100644
--- a/mlir/test/Transforms/test-inlining.mlir
+++ b/mlir/test/Transforms/test-inlining.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -test-inline | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -test-inline | FileCheck %s
 
 // CHECK-LABEL: func @inline_with_arg
 func @inline_with_arg(%arg0 : i32) -> i32 {
diff --git a/mlir/test/Transforms/test-legalizer-analysis.mlir b/mlir/test/Transforms/test-legalizer-analysis.mlir
index a53813a4baff6..cd0b936ada78e 100644
--- a/mlir/test/Transforms/test-legalizer-analysis.mlir
+++ b/mlir/test/Transforms/test-legalizer-analysis.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -test-legalize-patterns -verify-diagnostics -test-legalize-mode=analysis %s | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect -test-legalize-patterns -verify-diagnostics -test-legalize-mode=analysis %s | FileCheck %s
 // expected-remark@-2 {{op 'module' is legalizable}}
 // expected-remark@-3 {{op 'module_terminator' is legalizable}}
 
diff --git a/mlir/test/Transforms/test-legalizer-full.mlir b/mlir/test/Transforms/test-legalizer-full.mlir
index 6bbda4aad2bab..7652de4f67674 100644
--- a/mlir/test/Transforms/test-legalizer-full.mlir
+++ b/mlir/test/Transforms/test-legalizer-full.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -test-legalize-patterns -test-legalize-mode=full -split-input-file -verify-diagnostics %s | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect -test-legalize-patterns -test-legalize-mode=full -split-input-file -verify-diagnostics %s | FileCheck %s
 
 // CHECK-LABEL: func @multi_level_mapping
 func @multi_level_mapping() {
diff --git a/mlir/test/Transforms/test-legalizer.mlir b/mlir/test/Transforms/test-legalizer.mlir
index dd8330626551f..3305e017d5b31 100644
--- a/mlir/test/Transforms/test-legalizer.mlir
+++ b/mlir/test/Transforms/test-legalizer.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -split-input-file -test-legalize-patterns -verify-diagnostics %s | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect -split-input-file -test-legalize-patterns -verify-diagnostics %s | FileCheck %s
 
 // CHECK-LABEL: verifyDirectPattern
 func @verifyDirectPattern() -> i32 {
@@ -130,6 +130,19 @@ func @remove_foldable_op(%arg0 : i32) -> (i32) {
   return %0 : i32
 }
 
+// CHECK-LABEL: @create_block
+func @create_block() {
+  "test.container"() ({
+    // Check that we created a block with arguments.
+    // CHECK-NOT: test.create_block
+    // CHECK: ^{{.*}}(%{{.*}}: i32, %{{.*}}: i32):
+    // CHECK: test.finish
+    "test.create_block"() : () -> ()
+    "test.finish"() : () -> ()
+  }) : () -> ()
+  return
+}
+
 // -----
 
 func @fail_to_convert_illegal_op() -> i32 {
@@ -163,3 +176,17 @@ func @fail_to_convert_region() {
   }) : () -> ()
   return
 }
+
+// -----
+
+// CHECK-LABEL: @create_illegal_block
+func @create_illegal_block() {
+  "test.container"() ({
+    // Check that we can undo block creation, i.e. that the block was removed.
+    // CHECK: test.create_illegal_block
+    // CHECK-NOT: ^{{.*}}(%{{.*}}: i32, %{{.*}}: i32):
+    "test.create_illegal_block"() : () -> ()
+    "test.finish"() : () -> ()
+  }) : () -> ()
+  return
+}
diff --git a/mlir/test/lib/CMakeLists.txt b/mlir/test/lib/CMakeLists.txt
index 591847802f7be..3e69c7514d8f2 100644
--- a/mlir/test/lib/CMakeLists.txt
+++ b/mlir/test/lib/CMakeLists.txt
@@ -2,5 +2,4 @@ add_subdirectory(DeclarativeTransforms)
 add_subdirectory(Dialect)
 add_subdirectory(IR)
 add_subdirectory(Pass)
-add_subdirectory(TestDialect)
 add_subdirectory(Transforms)
diff --git a/mlir/test/lib/Dialect/Affine/CMakeLists.txt b/mlir/test/lib/Dialect/Affine/CMakeLists.txt
index 0d6cd2aa9edcc..fb1346a8fb5b8 100644
--- a/mlir/test/lib/Dialect/Affine/CMakeLists.txt
+++ b/mlir/test/lib/Dialect/Affine/CMakeLists.txt
@@ -1,5 +1,6 @@
 add_llvm_library(MLIRAffineTransformsTestPasses
   TestAffineDataCopy.cpp
+  TestLoopPermutation.cpp
   TestParallelismDetection.cpp
   TestVectorizationUtils.cpp
 
diff --git a/mlir/test/lib/Dialect/Affine/TestLoopPermutation.cpp b/mlir/test/lib/Dialect/Affine/TestLoopPermutation.cpp
new file mode 100644
index 0000000000000..87b2e620f7d6a
--- /dev/null
+++ b/mlir/test/lib/Dialect/Affine/TestLoopPermutation.cpp
@@ -0,0 +1,67 @@
+//===- TestLoopPermutation.cpp - Test affine loop permutation -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a pass to test the affine for op permutation utility.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Analysis/Utils.h"
+#include "mlir/Dialect/Affine/IR/AffineOps.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Transforms/LoopUtils.h"
+#include "mlir/Transforms/Passes.h"
+
+#define PASS_NAME "test-loop-permutation"
+
+using namespace mlir;
+
+static llvm::cl::OptionCategory clOptionsCategory(PASS_NAME " options");
+
+namespace {
+
+/// This pass applies the permutation on the first maximal perfect nest.
+struct TestLoopPermutation : public FunctionPass<TestLoopPermutation> {
+  TestLoopPermutation() = default;
+  TestLoopPermutation(const TestLoopPermutation &pass){};
+
+  void runOnFunction() override;
+
+private:
+  /// Permutation specifying loop i is mapped to permList[i] in
+  /// transformed nest (with i going from outermost to innermost).
+  ListOption<unsigned> permList{*this, "permutation-map",
+                                llvm::cl::desc("Specify the loop permutation"),
+                                llvm::cl::OneOrMore, llvm::cl::CommaSeparated};
+};
+
+} // end anonymous namespace
+
+void TestLoopPermutation::runOnFunction() {
+  // Get the first maximal perfect nest.
+  SmallVector<AffineForOp, 6> nest;
+  for (auto &op : getFunction().front()) {
+    if (auto forOp = dyn_cast<AffineForOp>(op)) {
+      getPerfectlyNestedLoops(nest, forOp);
+      break;
+    }
+  }
+
+  // Nothing to do.
+  if (nest.size() < 2)
+    return;
+
+  SmallVector<unsigned, 4> permMap(permList.begin(), permList.end());
+  permuteLoops(nest, permMap);
+}
+
+namespace mlir {
+void registerTestLoopPermutationPass() {
+  PassRegistration<TestLoopPermutation>(
+      PASS_NAME, "Tests affine loop permutation utility");
+}
+} // namespace mlir
diff --git a/mlir/test/lib/Dialect/CMakeLists.txt b/mlir/test/lib/Dialect/CMakeLists.txt
index 160fe9c203da9..9008b86314be0 100644
--- a/mlir/test/lib/Dialect/CMakeLists.txt
+++ b/mlir/test/lib/Dialect/CMakeLists.txt
@@ -1,2 +1,3 @@
 add_subdirectory(Affine)
 add_subdirectory(SPIRV)
+add_subdirectory(Test)
diff --git a/mlir/test/lib/TestDialect/CMakeLists.txt b/mlir/test/lib/Dialect/Test/CMakeLists.txt
similarity index 100%
rename from mlir/test/lib/TestDialect/CMakeLists.txt
rename to mlir/test/lib/Dialect/Test/CMakeLists.txt
diff --git a/mlir/test/lib/TestDialect/TestDialect.cpp b/mlir/test/lib/Dialect/Test/TestDialect.cpp
similarity index 100%
rename from mlir/test/lib/TestDialect/TestDialect.cpp
rename to mlir/test/lib/Dialect/Test/TestDialect.cpp
diff --git a/mlir/test/lib/TestDialect/TestDialect.h b/mlir/test/lib/Dialect/Test/TestDialect.h
similarity index 100%
rename from mlir/test/lib/TestDialect/TestDialect.h
rename to mlir/test/lib/Dialect/Test/TestDialect.h
diff --git a/mlir/test/lib/TestDialect/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td
similarity index 100%
rename from mlir/test/lib/TestDialect/TestOps.td
rename to mlir/test/lib/Dialect/Test/TestOps.td
diff --git a/mlir/test/lib/TestDialect/TestPatterns.cpp b/mlir/test/lib/Dialect/Test/TestPatterns.cpp
similarity index 91%
rename from mlir/test/lib/TestDialect/TestPatterns.cpp
rename to mlir/test/lib/Dialect/Test/TestPatterns.cpp
index 0b73f09c1943a..23d650e154791 100644
--- a/mlir/test/lib/TestDialect/TestPatterns.cpp
+++ b/mlir/test/lib/Dialect/Test/TestPatterns.cpp
@@ -183,6 +183,41 @@ struct TestRegionRewriteUndo : public RewritePattern {
     return success();
   }
 };
+/// A simple pattern that creates a block at the end of the parent region of the
+/// matched operation.
+struct TestCreateBlock : public RewritePattern {
+  TestCreateBlock(MLIRContext *ctx)
+      : RewritePattern("test.create_block", /*benefit=*/1, ctx) {}
+
+  LogicalResult matchAndRewrite(Operation *op,
+                                PatternRewriter &rewriter) const final {
+    Region &region = *op->getParentRegion();
+    Type i32Type = rewriter.getIntegerType(32);
+    rewriter.createBlock(&region, region.end(), {i32Type, i32Type});
+    rewriter.create<TerminatorOp>(op->getLoc());
+    rewriter.replaceOp(op, {});
+    return success();
+  }
+};
+
+/// A simple pattern that creates a block containing an invalid operaiton in
+/// order to trigger the block creation undo mechanism.
+struct TestCreateIllegalBlock : public RewritePattern {
+  TestCreateIllegalBlock(MLIRContext *ctx)
+      : RewritePattern("test.create_illegal_block", /*benefit=*/1, ctx) {}
+
+  LogicalResult matchAndRewrite(Operation *op,
+                                PatternRewriter &rewriter) const final {
+    Region &region = *op->getParentRegion();
+    Type i32Type = rewriter.getIntegerType(32);
+    rewriter.createBlock(&region, region.end(), {i32Type, i32Type});
+    // Create an illegal op to ensure the conversion fails.
+    rewriter.create<ILLegalOpF>(op->getLoc(), i32Type);
+    rewriter.create<TerminatorOp>(op->getLoc());
+    rewriter.replaceOp(op, {});
+    return success();
+  }
+};
 
 //===----------------------------------------------------------------------===//
 // Type-Conversion Rewrite Testing
@@ -373,12 +408,12 @@ struct TestLegalizePatternDriver
     TestTypeConverter converter;
     mlir::OwningRewritePatternList patterns;
     populateWithGenerated(&getContext(), &patterns);
-    patterns
-        .insert<TestRegionRewriteBlockMovement, TestRegionRewriteUndo,
-                TestPassthroughInvalidOp, TestSplitReturnType,
-                TestChangeProducerTypeI32ToF32, TestChangeProducerTypeF32ToF64,
-                TestChangeProducerTypeF32ToInvalid, TestUpdateConsumerType,
-                TestNonRootReplacement>(&getContext());
+    patterns.insert<
+        TestRegionRewriteBlockMovement, TestRegionRewriteUndo, TestCreateBlock,
+        TestCreateIllegalBlock, TestPassthroughInvalidOp, TestSplitReturnType,
+        TestChangeProducerTypeI32ToF32, TestChangeProducerTypeF32ToF64,
+        TestChangeProducerTypeF32ToInvalid, TestUpdateConsumerType,
+        TestNonRootReplacement>(&getContext());
     patterns.insert<TestDropOpSignatureConversion>(&getContext(), converter);
     mlir::populateFuncOpTypeConversionPattern(patterns, &getContext(),
                                               converter);
@@ -388,7 +423,8 @@ struct TestLegalizePatternDriver
     // Define the conversion target used for the test.
     ConversionTarget target(getContext());
     target.addLegalOp<ModuleOp, ModuleTerminatorOp>();
-    target.addLegalOp<LegalOpA, LegalOpB, TestCastOp, TestValidOp>();
+    target.addLegalOp<LegalOpA, LegalOpB, TestCastOp, TestValidOp,
+                      TerminatorOp>();
     target
         .addIllegalOp<ILLegalOpF, TestRegionBuilderOp, TestOpWithRegionFold>();
     target.addDynamicallyLegalOp<TestReturnOp>([](TestReturnOp op) {
diff --git a/mlir/test/lib/TestDialect/lit.local.cfg b/mlir/test/lib/Dialect/Test/lit.local.cfg
similarity index 100%
rename from mlir/test/lib/TestDialect/lit.local.cfg
rename to mlir/test/lib/Dialect/Test/lit.local.cfg
diff --git a/mlir/test/lib/IR/CMakeLists.txt b/mlir/test/lib/IR/CMakeLists.txt
index bf0f8eed44100..c4e4ebc9b88ee 100644
--- a/mlir/test/lib/IR/CMakeLists.txt
+++ b/mlir/test/lib/IR/CMakeLists.txt
@@ -6,8 +6,8 @@ add_llvm_library(MLIRTestIR
 
   ADDITIONAL_HEADER_DIRS
   )
-include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../TestDialect)
-include_directories(${CMAKE_CURRENT_BINARY_DIR}/../TestDialect)
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../Dialect/Test)
+include_directories(${CMAKE_CURRENT_BINARY_DIR}/../Dialect/Test)
 
 target_link_libraries(MLIRTestIR
   PUBLIC
diff --git a/mlir/test/lib/Transforms/CMakeLists.txt b/mlir/test/lib/Transforms/CMakeLists.txt
index b4726439c83f7..904a47221ac19 100644
--- a/mlir/test/lib/Transforms/CMakeLists.txt
+++ b/mlir/test/lib/Transforms/CMakeLists.txt
@@ -3,6 +3,7 @@ add_llvm_library(MLIRTestTransforms
   TestCallGraph.cpp
   TestConstantFold.cpp
   TestConvertGPUKernelToCubin.cpp
+  TestDominance.cpp
   TestLoopFusion.cpp
   TestGpuMemoryPromotion.cpp
   TestGpuParallelLoopMapping.cpp
@@ -27,8 +28,8 @@ add_llvm_library(MLIRTestTransforms
   MLIRTestVectorTransformPatternsIncGen
 )
 
-include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../TestDialect)
-include_directories(${CMAKE_CURRENT_BINARY_DIR}/../TestDialect)
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../Dialect/Test)
+include_directories(${CMAKE_CURRENT_BINARY_DIR}/../Dialect/Test)
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../DeclarativeTransforms)
 include_directories(${CMAKE_CURRENT_BINARY_DIR}/../DeclarativeTransforms)
 
diff --git a/mlir/test/lib/Transforms/TestDominance.cpp b/mlir/test/lib/Transforms/TestDominance.cpp
new file mode 100644
index 0000000000000..784bb1f405647
--- /dev/null
+++ b/mlir/test/lib/Transforms/TestDominance.cpp
@@ -0,0 +1,90 @@
+//===- TestDominance.cpp - Test dominance construction and information
+//-------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains test passes for constructing and resolving dominance
+// information.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Analysis/Dominance.h"
+#include "mlir/Pass/Pass.h"
+
+using namespace mlir;
+
+namespace {
+
+/// Helper class to print dominance information.
+class DominanceTest {
+public:
+  /// Constructs a new test instance using the given operation.
+  DominanceTest(Operation *operation) : operation(operation) {
+    // Create unique ids for each block.
+    operation->walk([&](Operation *nested) {
+      if (blockIds.count(nested->getBlock()) > 0)
+        return;
+      blockIds.insert({nested->getBlock(), blockIds.size()});
+    });
+  }
+
+  /// Prints dominance information of all blocks.
+  template <typename DominanceT>
+  void printDominance(DominanceT &dominanceInfo) {
+    DenseSet<Block *> parentVisited;
+    operation->walk([&](Operation *op) {
+      Block *block = op->getBlock();
+      if (!parentVisited.insert(block).second)
+        return;
+
+      DenseSet<Block *> visited;
+      operation->walk([&](Operation *nested) {
+        Block *nestedBlock = nested->getBlock();
+        if (!visited.insert(nestedBlock).second)
+          return;
+        llvm::errs() << "Nearest(" << blockIds[block] << ", "
+                     << blockIds[nestedBlock] << ") = ";
+        Block *dom =
+            dominanceInfo.findNearestCommonDominator(block, nestedBlock);
+        if (dom)
+          llvm::errs() << blockIds[dom];
+        else
+          llvm::errs() << "<no dom>";
+        llvm::errs() << "\n";
+      });
+    });
+  }
+
+private:
+  Operation *operation;
+  DenseMap<Block *, size_t> blockIds;
+};
+
+struct TestDominancePass : public FunctionPass<TestDominancePass> {
+
+  void runOnFunction() override {
+    llvm::errs() << "Testing : " << getFunction().getName() << "\n";
+    DominanceTest dominanceTest(getFunction());
+
+    // Print dominance information.
+    llvm::errs() << "--- DominanceInfo ---\n";
+    dominanceTest.printDominance(getAnalysis<DominanceInfo>());
+
+    llvm::errs() << "--- PostDominanceInfo ---\n";
+    dominanceTest.printDominance(getAnalysis<PostDominanceInfo>());
+  }
+};
+
+} // end anonymous namespace
+
+namespace mlir {
+void registerTestDominancePass() {
+  PassRegistration<TestDominancePass>(
+      "test-print-dominance",
+      "Print the dominance information for multiple regions.");
+}
+} // namespace mlir
diff --git a/mlir/test/lib/Transforms/TestGpuParallelLoopMapping.cpp b/mlir/test/lib/Transforms/TestGpuParallelLoopMapping.cpp
index df96d9ce96eac..d7bbdbb94cba5 100644
--- a/mlir/test/lib/Transforms/TestGpuParallelLoopMapping.cpp
+++ b/mlir/test/lib/Transforms/TestGpuParallelLoopMapping.cpp
@@ -18,7 +18,7 @@ using namespace mlir;
 
 namespace {
 /// Simple pass for testing the mapping of parallel loops to hardware ids using
-/// a greedy mapping stratgegy.
+/// a greedy mapping strategy.
 class TestGpuGreedyParallelLoopMappingPass
     : public OperationPass<TestGpuGreedyParallelLoopMappingPass, FuncOp> {
   void runOnOperation() override {
diff --git a/mlir/test/lit.cfg.py b/mlir/test/lit.cfg.py
index 62015568670b6..84a4de057c135 100644
--- a/mlir/test/lit.cfg.py
+++ b/mlir/test/lit.cfg.py
@@ -66,8 +66,9 @@
     ToolSubst('toy-ch3', unresolved='ignore'),
     ToolSubst('toy-ch4', unresolved='ignore'),
     ToolSubst('toy-ch5', unresolved='ignore'),
-    ToolSubst('%linalg_test_lib_dir', config.linalg_test_lib_dir, unresolved='ignore'),
     ToolSubst('%cuda_wrapper_library_dir', config.cuda_wrapper_library_dir, unresolved='ignore'),
+    ToolSubst('%linalg_test_lib_dir', config.linalg_test_lib_dir, unresolved='ignore'),
+    ToolSubst('%mlir_runner_utils_dir', config.mlir_runner_utils_dir, unresolved='ignore'),
     ToolSubst('%vulkan_wrapper_library_dir', config.vulkan_wrapper_library_dir, unresolved='ignore')
 ])
 
diff --git a/mlir/test/lit.site.cfg.py.in b/mlir/test/lit.site.cfg.py.in
index 5e18083a8b5c9..dafb1c9a3eb86 100644
--- a/mlir/test/lit.site.cfg.py.in
+++ b/mlir/test/lit.site.cfg.py.in
@@ -30,6 +30,7 @@ config.llvm_host_triple = '@LLVM_HOST_TRIPLE@'
 config.host_arch = "@HOST_ARCH@"
 config.mlir_src_root = "@MLIR_SOURCE_DIR@"
 config.mlir_obj_root = "@MLIR_BINARY_DIR@"
+config.mlir_runner_utils_dir = "@MLIR_RUNNER_UTILS_DIR@"
 config.mlir_tools_dir = "@MLIR_TOOLS_DIR@"
 config.linalg_test_lib_dir = "@MLIR_DIALECT_LINALG_INTEGRATION_TEST_LIB_DIR@"
 config.build_examples = @LLVM_BUILD_EXAMPLES@
diff --git a/mlir/test/mlir-cpu-runner/sgemm_naive_codegen.mlir b/mlir/test/mlir-cpu-runner/sgemm_naive_codegen.mlir
new file mode 100644
index 0000000000000..eadf51d9e2ee0
--- /dev/null
+++ b/mlir/test/mlir-cpu-runner/sgemm_naive_codegen.mlir
@@ -0,0 +1,71 @@
+// RUN: mlir-opt -convert-linalg-to-loops -lower-affine -convert-loop-to-std -convert-std-to-llvm %s | mlir-cpu-runner -O3 -e main -entry-point-result=void -shared-libs=%mlir_runner_utils_dir/libmlir_runner_utils%shlibext | FileCheck %s
+
+func @main() {
+  %A = alloc() : memref<64x64xf32>
+  %B = alloc() : memref<64x64xf32>
+  %C = alloc() : memref<64x64xf32>
+
+  %cf1 = constant 1.00000e+00 : f32
+
+  linalg.fill(%A, %cf1) : memref<64x64xf32>, f32
+  linalg.fill(%B, %cf1) : memref<64x64xf32>, f32
+
+  %reps = constant 1 : index
+
+  %t_start = call @rtclock() : () -> f64
+  affine.for %arg0 = 0 to 5 {
+    linalg.fill(%C, %cf1) : memref<64x64xf32>, f32
+    call @sgemm_naive(%A, %B, %C) : (memref<64x64xf32>, memref<64x64xf32>, memref<64x64xf32>) -> ()
+  }
+  %t_end = call @rtclock() : () -> f64
+  %t = subf %t_end, %t_start : f64
+
+  %pC = memref_cast %C : memref<64x64xf32> to memref<*xf32>
+  call @print_memref_f32(%pC) : (memref<*xf32>) -> ()
+
+  %M = dim %C, 0 : memref<64x64xf32>
+  %N = dim %C, 1 : memref<64x64xf32>
+  %K = dim %A, 1 : memref<64x64xf32>
+
+  %f1 = muli %M, %N : index
+  %f2 = muli %f1, %K : index
+
+  // 2*M*N*K.
+  %c2 = constant 2 : index
+  %f3 = muli %c2, %f2 : index
+  %num_flops = muli %reps, %f3 : index
+  %num_flops_i = index_cast %num_flops : index to i64
+  %num_flops_f = sitofp %num_flops_i : i64 to f64
+  %flops = divf %num_flops_f, %t : f64
+  call @print_flops(%flops) : (f64) -> ()
+
+  return
+}
+// CHECK: 65,   65,   65,
+
+func @sgemm_naive(%arg0: memref<64x64xf32>, %arg1: memref<64x64xf32>, %arg2: memref<64x64xf32>) {
+  %c0 = constant 0 : index
+  affine.for %arg3 = 0 to 64 {
+    affine.for %arg4 = 0 to 64 {
+      %m = alloc() : memref<1xf32>
+      %v = affine.load %arg2[%arg3, %arg4] : memref<64x64xf32>
+      affine.store %v, %m[%c0] : memref<1xf32>
+      affine.for %arg5 = 0 to 64 {
+        %3 = affine.load %arg0[%arg3, %arg5] : memref<64x64xf32>
+        %4 = affine.load %arg1[%arg5, %arg4] : memref<64x64xf32>
+        %5 = affine.load %m[0] : memref<1xf32>
+        %6 = mulf %3, %4 : f32
+        %7 = addf %6, %5 : f32
+        affine.store %7, %m[0] : memref<1xf32>
+      }
+      %s = affine.load %m[%c0] : memref<1xf32>
+      affine.store %s, %arg2[%arg3, %arg4] : memref<64x64xf32>
+      dealloc %m : memref<1xf32>
+    }
+  }
+  return
+}
+
+func @print_flops(f64)
+func @rtclock() -> f64
+func @print_memref_f32(memref<*xf32>)
diff --git a/mlir/test/mlir-cpu-runner/unranked_memref.mlir b/mlir/test/mlir-cpu-runner/unranked_memref.mlir
index 7447e9d6e02b2..13d4db9c90c67 100644
--- a/mlir/test/mlir-cpu-runner/unranked_memref.mlir
+++ b/mlir/test/mlir-cpu-runner/unranked_memref.mlir
@@ -55,5 +55,5 @@ func @main() -> () {
     return
 }
 
-func @print_memref_i8(memref<*xi8>)
-func @print_memref_f32(memref<*xf32>)
+func @print_memref_i8(memref<*xi8>) attributes { llvm.emit_c_interface }
+func @print_memref_f32(memref<*xf32>) attributes { llvm.emit_c_interface }
diff --git a/mlir/test/mlir-cpu-runner/utils.mlir b/mlir/test/mlir-cpu-runner/utils.mlir
index 2a5692035db8a..d3ab6177eb65c 100644
--- a/mlir/test/mlir-cpu-runner/utils.mlir
+++ b/mlir/test/mlir-cpu-runner/utils.mlir
@@ -50,7 +50,7 @@ func @print_3d() {
 //    PRINT-3D-NEXT: 2,    2,    4,    2,    2
 //    PRINT-3D-NEXT: 2,    2,    2,    2,    2
 
-func @print_memref_f32(memref<*xf32>)
+func @print_memref_f32(memref<*xf32>) attributes { llvm.emit_c_interface }
 
 !vector_type_C = type vector<4x4xf32>
 !matrix_type_CC = type memref<1x1x!vector_type_C>
@@ -71,4 +71,4 @@ func @vector_splat_2d() {
 // PRINT-VECTOR-SPLAT-2D: Memref base@ = {{.*}} rank = 2 offset = 0 sizes = [1, 1] strides = [1, 1] data =
 // PRINT-VECTOR-SPLAT-2D-NEXT: [((10, 10, 10, 10),   (10, 10, 10, 10),   (10, 10, 10, 10),   (10, 10, 10, 10))]
 
-func @print_memref_vector_4x4xf32(memref<?x?x!vector_type_C>)
+func @print_memref_vector_4x4xf32(memref<?x?x!vector_type_C>) attributes { llvm.emit_c_interface }
diff --git a/mlir/test/mlir-tblgen/llvm-intrinsics.td b/mlir/test/mlir-tblgen/llvm-intrinsics.td
index e3f90954908dd..575877e86c301 100644
--- a/mlir/test/mlir-tblgen/llvm-intrinsics.td
+++ b/mlir/test/mlir-tblgen/llvm-intrinsics.td
@@ -5,7 +5,7 @@
 // writing a local test source. We filter out platform-specific intrinsic
 // includes from the main file to avoid unnecessary dependencies and decrease
 // the test cost. The command-line flags further ensure a specific intrinsic is
-// processed and we only check the ouptut below.
+// processed and we only check the output below.
 // We also verify emission of type specialization for overloadable intrinsics.
 //
 // RUN: cat %S/../../../llvm/include/llvm/IR/Intrinsics.td \
diff --git a/mlir/test/mlir-tblgen/op-format.mlir b/mlir/test/mlir-tblgen/op-format.mlir
index 6c53527d649fd..5e0e484ae3e26 100644
--- a/mlir/test/mlir-tblgen/op-format.mlir
+++ b/mlir/test/mlir-tblgen/op-format.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s | mlir-opt -verify-diagnostics | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s | mlir-opt -allow-unregistered-dialect -verify-diagnostics | FileCheck %s
 
 // CHECK: %[[I64:.*]] =
 %i64 = "foo.op"() : () -> (i64)
diff --git a/mlir/test/mlir-tblgen/op-result.td b/mlir/test/mlir-tblgen/op-result.td
index 76484c8c7fb5b..c8bdd15cf2f12 100644
--- a/mlir/test/mlir-tblgen/op-result.td
+++ b/mlir/test/mlir-tblgen/op-result.td
@@ -38,6 +38,10 @@ def OpC : NS_Op<"three_normal_result_op", []> {
 // CHECK-NEXT:   odsState.addTypes(resultType1)
 // CHECK-NEXT:   odsState.addTypes(z)
 
+// CHECK:      void OpC::build(Builder *odsBuilder, OperationState &odsState, ArrayRef<Type> resultTypes) {
+// CHECK-NEXT:   assert(resultTypes.size() == 3u && "mismatched number of results");
+// CHECK-NEXT:   odsState.addTypes(resultTypes);
+
 def IntegerTypeAttr : TypeAttrBase<"IntegerType", "Integer type attribute">;
 def OpD : NS_Op<"type_attr_as_result_type", [FirstAttrDerivedResultType]> {
   let arguments = (ins I32:$x, IntegerTypeAttr:$attr, F32Attr:$f32);
diff --git a/mlir/test/mlir-vulkan-runner/mulf.mlir b/mlir/test/mlir-vulkan-runner/mulf.mlir
new file mode 100644
index 0000000000000..dc962108cbc3a
--- /dev/null
+++ b/mlir/test/mlir-vulkan-runner/mulf.mlir
@@ -0,0 +1,52 @@
+// RUN: mlir-vulkan-runner %s --shared-libs=%vulkan_wrapper_library_dir/libvulkan-runtime-wrappers%shlibext,%linalg_test_lib_dir/libmlir_runner_utils%shlibext --entry-point-result=void | FileCheck %s
+
+// CHECK-COUNT-4: [6, 6, 6, 6]
+module attributes {
+  gpu.container_module,
+  spv.target_env = #spv.target_env<
+    #spv.vce<v1.0, [Shader], [SPV_KHR_storage_buffer_storage_class]>,
+    {max_compute_workgroup_invocations = 128 : i32,
+     max_compute_workgroup_size = dense<[128, 128, 64]> : vector<3xi32>}>
+} {
+  gpu.module @kernels {
+    gpu.func @kernel_mul(%arg0 : memref<4x4xf32>, %arg1 : memref<4x4xf32>, %arg2 : memref<4x4xf32>)
+      attributes {gpu.kernel, spv.entry_point_abi = {local_size = dense<[1, 1, 1]>: vector<3xi32>}} {
+      %x = "gpu.block_id"() {dimension = "x"} : () -> index
+      %y = "gpu.block_id"() {dimension = "y"} : () -> index
+      %1 = load %arg0[%x, %y] : memref<4x4xf32>
+      %2 = load %arg1[%x, %y] : memref<4x4xf32>
+      %3 = mulf %1, %2 : f32
+      store %3, %arg2[%x, %y] : memref<4x4xf32>
+      gpu.return
+    }
+  }
+
+  func @main() {
+    %arg0 = alloc() : memref<4x4xf32>
+    %arg1 = alloc() : memref<4x4xf32>
+    %arg2 = alloc() : memref<4x4xf32>
+    %0 = constant 0 : i32
+    %1 = constant 1 : i32
+    %2 = constant 2 : i32
+    %value0 = constant 0.0 : f32
+    %value1 = constant 2.0 : f32
+    %value2 = constant 3.0 : f32
+    %arg3 = memref_cast %arg0 : memref<4x4xf32> to memref<?x?xf32>
+    %arg4 = memref_cast %arg1 : memref<4x4xf32> to memref<?x?xf32>
+    %arg5 = memref_cast %arg2 : memref<4x4xf32> to memref<?x?xf32>
+    call @fillResource2DFloat(%arg3, %value1) : (memref<?x?xf32>, f32) -> ()
+    call @fillResource2DFloat(%arg4, %value2) : (memref<?x?xf32>, f32) -> ()
+    call @fillResource2DFloat(%arg5, %value0) : (memref<?x?xf32>, f32) -> ()
+
+    %cst1 = constant 1 : index
+    %cst4 = constant 4 : index
+    "gpu.launch_func"(%cst4, %cst4, %cst1, %cst1, %cst1, %cst1, %arg0, %arg1, %arg2) { kernel = "kernel_mul", kernel_module = @kernels }
+        : (index, index, index, index, index, index, memref<4x4xf32>, memref<4x4xf32>, memref<4x4xf32>) -> ()
+    %arg6 = memref_cast %arg5 : memref<?x?xf32> to memref<*xf32>
+    call @print_memref_f32(%arg6) : (memref<*xf32>) -> ()
+    return
+  }
+  func @fillResource2DFloat(%0 : memref<?x?xf32>, %1 : f32)
+  func @print_memref_f32(%ptr : memref<*xf32>)
+}
+
diff --git a/mlir/test/mlir-vulkan-runner/subf.mlir b/mlir/test/mlir-vulkan-runner/subf.mlir
new file mode 100644
index 0000000000000..82dec1243740c
--- /dev/null
+++ b/mlir/test/mlir-vulkan-runner/subf.mlir
@@ -0,0 +1,54 @@
+// RUN: mlir-vulkan-runner %s --shared-libs=%vulkan_wrapper_library_dir/libvulkan-runtime-wrappers%shlibext,%linalg_test_lib_dir/libmlir_runner_utils%shlibext --entry-point-result=void | FileCheck %s
+
+// CHECK-COUNT-32: [2.2, 2.2, 2.2, 2.2]
+module attributes {
+  gpu.container_module,
+  spv.target_env = #spv.target_env<
+    #spv.vce<v1.0, [Shader], [SPV_KHR_storage_buffer_storage_class]>,
+    {max_compute_workgroup_invocations = 128 : i32,
+     max_compute_workgroup_size = dense<[128, 128, 64]> : vector<3xi32>}>
+} {
+  gpu.module @kernels {
+    gpu.func @kernel_sub(%arg0 : memref<8x4x4xf32>, %arg1 : memref<4x4xf32>, %arg2 : memref<8x4x4xf32>)
+      attributes {gpu.kernel, spv.entry_point_abi = {local_size = dense<[1, 1, 1]>: vector<3xi32>}} {
+      %x = "gpu.block_id"() {dimension = "x"} : () -> index
+      %y = "gpu.block_id"() {dimension = "y"} : () -> index
+      %z = "gpu.block_id"() {dimension = "z"} : () -> index
+      %1 = load %arg0[%x, %y, %z] : memref<8x4x4xf32>
+      %2 = load %arg1[%y, %z] : memref<4x4xf32>
+      %3 = subf %1, %2 : f32
+      store %3, %arg2[%x, %y, %z] : memref<8x4x4xf32>
+      gpu.return
+    }
+  }
+
+  func @main() {
+    %arg0 = alloc() : memref<8x4x4xf32>
+    %arg1 = alloc() : memref<4x4xf32>
+    %arg2 = alloc() : memref<8x4x4xf32>
+    %0 = constant 0 : i32
+    %1 = constant 1 : i32
+    %2 = constant 2 : i32
+    %value0 = constant 0.0 : f32
+    %value1 = constant 3.3 : f32
+    %value2 = constant 1.1 : f32
+    %arg3 = memref_cast %arg0 : memref<8x4x4xf32> to memref<?x?x?xf32>
+    %arg4 = memref_cast %arg1 : memref<4x4xf32> to memref<?x?xf32>
+    %arg5 = memref_cast %arg2 : memref<8x4x4xf32> to memref<?x?x?xf32>
+    call @fillResource3DFloat(%arg3, %value1) : (memref<?x?x?xf32>, f32) -> ()
+    call @fillResource2DFloat(%arg4, %value2) : (memref<?x?xf32>, f32) -> ()
+    call @fillResource3DFloat(%arg5, %value0) : (memref<?x?x?xf32>, f32) -> ()
+
+    %cst1 = constant 1 : index
+    %cst4 = constant 4 : index
+    %cst8 = constant 8 : index
+    "gpu.launch_func"(%cst8, %cst4, %cst4, %cst1, %cst1, %cst1, %arg0, %arg1, %arg2) { kernel = "kernel_sub", kernel_module = @kernels }
+        : (index, index, index, index, index, index, memref<8x4x4xf32>, memref<4x4xf32>, memref<8x4x4xf32>) -> ()
+    %arg6 = memref_cast %arg5 : memref<?x?x?xf32> to memref<*xf32>
+    call @print_memref_f32(%arg6) : (memref<*xf32>) -> ()
+    return
+  }
+  func @fillResource2DFloat(%0 : memref<?x?xf32>, %1 : f32)
+  func @fillResource3DFloat(%0 : memref<?x?x?xf32>, %1 : f32)
+  func @print_memref_f32(%ptr : memref<*xf32>)
+}
diff --git a/mlir/test/mlir-vulkan-runner/time.mlir b/mlir/test/mlir-vulkan-runner/time.mlir
new file mode 100644
index 0000000000000..f69b4feec37f7
--- /dev/null
+++ b/mlir/test/mlir-vulkan-runner/time.mlir
@@ -0,0 +1,57 @@
+// RUN: mlir-vulkan-runner %s --shared-libs=%vulkan_wrapper_library_dir/libvulkan-runtime-wrappers%shlibext,%linalg_test_lib_dir/libmlir_runner_utils%shlibext --entry-point-result=void | FileCheck %s
+
+// CHECK: Compute shader execution time
+// CHECK: Command buffer submit time
+// CHECK: Wait idle time
+
+module attributes {
+  gpu.container_module,
+  spv.target_env = #spv.target_env<
+    #spv.vce<v1.0, [Shader], [SPV_KHR_storage_buffer_storage_class]>,
+    {max_compute_workgroup_invocations = 128 : i32,
+     max_compute_workgroup_size = dense<[128, 128, 64]> : vector<3xi32>}>
+} {
+  gpu.module @kernels {
+    gpu.func @kernel_add(%arg0 : memref<16384xf32>, %arg1 : memref<16384xf32>, %arg2 : memref<16384xf32>)
+      attributes {gpu.kernel, spv.entry_point_abi = {local_size = dense<[128, 1, 1]>: vector<3xi32>}} {
+      %bid = "gpu.block_id"() {dimension = "x"} : () -> index
+      %tid = "gpu.thread_id"() {dimension = "x"} : () -> index
+      %cst = constant 128 : index
+      %b = muli %bid, %cst : index
+      %0 = addi %b, %tid : index
+      %1 = load %arg0[%0] : memref<16384xf32>
+      %2 = load %arg1[%0] : memref<16384xf32>
+      %3 = addf %1, %2 : f32
+      store %3, %arg2[%0] : memref<16384xf32>
+      gpu.return
+    }
+  }
+
+  func @main() {
+    %arg0 = alloc() : memref<16384xf32>
+    %arg1 = alloc() : memref<16384xf32>
+    %arg2 = alloc() : memref<16384xf32>
+    %0 = constant 0 : i32
+    %1 = constant 1 : i32
+    %2 = constant 2 : i32
+    %value0 = constant 0.0 : f32
+    %value1 = constant 1.1 : f32
+    %value2 = constant 2.2 : f32
+    %arg3 = memref_cast %arg0 : memref<16384xf32> to memref<?xf32>
+    %arg4 = memref_cast %arg1 : memref<16384xf32> to memref<?xf32>
+    %arg5 = memref_cast %arg2 : memref<16384xf32> to memref<?xf32>
+    call @fillResource1DFloat(%arg3, %value1) : (memref<?xf32>, f32) -> ()
+    call @fillResource1DFloat(%arg4, %value2) : (memref<?xf32>, f32) -> ()
+    call @fillResource1DFloat(%arg5, %value0) : (memref<?xf32>, f32) -> ()
+
+    %cst1 = constant 1 : index
+    %cst128 = constant 128 : index
+    "gpu.launch_func"(%cst128, %cst1, %cst1, %cst128, %cst1, %cst1, %arg0, %arg1, %arg2) { kernel = "kernel_add", kernel_module = @kernels }
+        : (index, index, index, index, index, index, memref<16384xf32>, memref<16384xf32>, memref<16384xf32>) -> ()
+    %arg6 = memref_cast %arg5 : memref<?xf32> to memref<*xf32>
+    return
+  }
+  func @fillResource1DFloat(%0 : memref<?xf32>, %1 : f32)
+  func @print_memref_f32(%ptr : memref<*xf32>)
+}
+
diff --git a/mlir/tools/mlir-opt/CMakeLists.txt b/mlir/tools/mlir-opt/CMakeLists.txt
index 2ae36d466dd45..60bf5a1d87f3d 100644
--- a/mlir/tools/mlir-opt/CMakeLists.txt
+++ b/mlir/tools/mlir-opt/CMakeLists.txt
@@ -47,16 +47,6 @@ target_link_libraries(MLIRMlirOptMain
 add_llvm_tool(mlir-opt
  mlir-opt.cpp
 )
-# Manually expand the target library, since our MLIR libraries
-# aren't plugged into the LLVM dependency tracking. If we don't
-# do this then we can't insert the CodeGen library after ourselves
-llvm_expand_pseudo_components(TARGET_LIBS AllTargetsCodeGens)
-# Prepend LLVM in front of every target, this is how the library
-# are named with CMake
-SET(targets_to_link)
-FOREACH(t ${TARGET_LIBS})
-  LIST(APPEND targets_to_link "LLVM${t}")
-ENDFOREACH(t)
 
 llvm_update_compile_flags(mlir-opt)
 target_link_libraries(mlir-opt PRIVATE ${LIBS} ${targets_to_link})
diff --git a/mlir/tools/mlir-opt/mlir-opt.cpp b/mlir/tools/mlir-opt/mlir-opt.cpp
index ff0f49f987b60..0aca9f6d37d2a 100644
--- a/mlir/tools/mlir-opt/mlir-opt.cpp
+++ b/mlir/tools/mlir-opt/mlir-opt.cpp
@@ -39,9 +39,11 @@ void registerSimpleParametricTilingPass();
 void registerSymbolTestPasses();
 void registerTestAffineDataCopyPass();
 void registerTestAllReduceLoweringPass();
+void registerTestLoopPermutationPass();
 void registerTestCallGraphPass();
 void registerTestConstantFold();
 void registerTestConvertGPUKernelToCubinPass();
+void registerTestDominancePass();
 void registerTestFunc();
 void registerTestGpuMemoryPromotionPass();
 void registerTestLinalgTransforms();
@@ -83,6 +85,10 @@ static cl::opt<bool>
                  cl::desc("Run the verifier after each transformation pass"),
                  cl::init(true));
 
+static cl::opt<bool> allowUnregisteredDialects(
+    "allow-unregistered-dialect",
+    cl::desc("Allow operation with no registered dialects"), cl::init(false));
+
 void registerTestPasses() {
   registerConvertToTargetEnvPass();
   registerInliner();
@@ -95,11 +101,13 @@ void registerTestPasses() {
   registerSymbolTestPasses();
   registerTestAffineDataCopyPass();
   registerTestAllReduceLoweringPass();
+  registerTestLoopPermutationPass();
   registerTestCallGraphPass();
   registerTestConstantFold();
 #if MLIR_CUDA_CONVERSIONS_ENABLED
   registerTestConvertGPUKernelToCubinPass();
 #endif
+  registerTestDominancePass();
   registerTestFunc();
   registerTestGpuMemoryPromotionPass();
   registerTestLinalgTransforms();
@@ -159,5 +167,6 @@ int main(int argc, char **argv) {
   }
 
   return failed(MlirOptMain(output->os(), std::move(file), passPipeline,
-                            splitInputFile, verifyDiagnostics, verifyPasses));
+                            splitInputFile, verifyDiagnostics, verifyPasses,
+                            allowUnregisteredDialects));
 }
diff --git a/mlir/tools/mlir-tblgen/CMakeLists.txt b/mlir/tools/mlir-tblgen/CMakeLists.txt
index b7628cff11f82..19e6230e54ac8 100644
--- a/mlir/tools/mlir-tblgen/CMakeLists.txt
+++ b/mlir/tools/mlir-tblgen/CMakeLists.txt
@@ -13,6 +13,8 @@ add_tablegen(mlir-tblgen MLIR
   OpDocGen.cpp
   OpFormatGen.cpp
   OpInterfacesGen.cpp
+  PassGen.cpp
+  PassDocGen.cpp
   RewriterGen.cpp
   SPIRVUtilsGen.cpp
   StructsGen.cpp
diff --git a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
index 47eef13ad586f..f262f12ddf2d7 100644
--- a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
@@ -702,6 +702,11 @@ void OpEmitter::genSeparateArgParamBuilder() {
       }
       return;
     case TypeParamKind::Collective:
+      body << "  "
+           << "assert(resultTypes.size() "
+           << (op.getNumVariadicResults() == 0 ? "==" : ">=") << " "
+           << (op.getNumResults() - op.getNumVariadicResults())
+           << "u && \"mismatched number of results\");\n";
       body << "  " << builderOpState << ".addTypes(resultTypes);\n";
       return;
     };
diff --git a/mlir/tools/mlir-tblgen/OpFormatGen.cpp b/mlir/tools/mlir-tblgen/OpFormatGen.cpp
index 525981e03c86b..b9303979d3f4b 100644
--- a/mlir/tools/mlir-tblgen/OpFormatGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpFormatGen.cpp
@@ -1234,12 +1234,12 @@ class FormatParser {
     Optional<StringRef> transformer;
   };
 
-  /// Given the values of an `AllTypesMatch` trait, check for inferrable type
+  /// Given the values of an `AllTypesMatch` trait, check for inferable type
   /// resolution.
   void handleAllTypesMatchConstraint(
       ArrayRef<StringRef> values,
       llvm::StringMap<TypeResolutionInstance> &variableTyResolver);
-  /// Check for inferrable type resolution given all operands, and or results,
+  /// Check for inferable type resolution given all operands, and or results,
   /// have the same type. If 'includeResults' is true, the results also have the
   /// same type as all of the operands.
   void handleSameTypesConstraint(
diff --git a/mlir/tools/mlir-tblgen/PassDocGen.cpp b/mlir/tools/mlir-tblgen/PassDocGen.cpp
new file mode 100644
index 0000000000000..d55468a0f386d
--- /dev/null
+++ b/mlir/tools/mlir-tblgen/PassDocGen.cpp
@@ -0,0 +1,80 @@
+//===- PassDocGen.cpp - MLIR pass documentation generator -----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// PassDocGen uses the description of passes to generate documentation.
+//
+//===----------------------------------------------------------------------===//
+
+#include "DocGenUtilities.h"
+#include "mlir/TableGen/GenInfo.h"
+#include "mlir/TableGen/Pass.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/TableGen/Record.h"
+
+using namespace mlir;
+using namespace mlir::tblgen;
+
+/// Emit the documentation for the given pass.
+static void emitDoc(const Pass &pass, raw_ostream &os) {
+  os << llvm::formatv("### `-{0}`: {1}\n", pass.getArgument(),
+                      pass.getSummary());
+  emitDescription(pass.getDescription(), os);
+
+  // Handle the options of the pass.
+  ArrayRef<PassOption> options = pass.getOptions();
+  if (!options.empty()) {
+    os << "\n#### Options\n```\n";
+    size_t longestOption = 0;
+    for (const PassOption &option : options)
+      longestOption = std::max(option.getArgument().size(), longestOption);
+    for (const PassOption &option : options) {
+      os << "-" << option.getArgument();
+      os.indent(longestOption - option.getArgument().size())
+          << " : " << option.getDescription() << "\n";
+    }
+    os << "```\n";
+  }
+
+  // Handle the statistics of the pass.
+  ArrayRef<PassStatistic> stats = pass.getStatistics();
+  if (!stats.empty()) {
+    os << "\n#### Statistics\n```\n";
+    size_t longestStat = 0;
+    for (const PassStatistic &stat : stats)
+      longestStat = std::max(stat.getName().size(), longestStat);
+    for (const PassStatistic &stat : stats) {
+      os << stat.getName();
+      os.indent(longestStat - stat.getName().size())
+          << " : " << stat.getDescription() << "\n";
+    }
+    os << "```\n";
+  }
+}
+
+static void emitDocs(const llvm::RecordKeeper &recordKeeper, raw_ostream &os) {
+  os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
+  auto passDefs = recordKeeper.getAllDerivedDefinitions("Pass");
+
+  // Collect the registered passes, sorted by argument name.
+  SmallVector<Pass, 16> passes(passDefs.begin(), passDefs.end());
+  SmallVector<Pass *, 16> sortedPasses(llvm::make_pointer_range(passes));
+  llvm::array_pod_sort(sortedPasses.begin(), sortedPasses.end(),
+                       [](Pass *const *lhs, Pass *const *rhs) {
+                         return (*lhs)->getArgument().compare(
+                             (*rhs)->getArgument());
+                       });
+  for (Pass *pass : sortedPasses)
+    emitDoc(*pass, os);
+}
+
+static mlir::GenRegistration
+    genRegister("gen-pass-doc", "Generate pass documentation",
+                [](const llvm::RecordKeeper &records, raw_ostream &os) {
+                  emitDocs(records, os);
+                  return false;
+                });
diff --git a/mlir/tools/mlir-tblgen/PassGen.cpp b/mlir/tools/mlir-tblgen/PassGen.cpp
new file mode 100644
index 0000000000000..4cda353953ade
--- /dev/null
+++ b/mlir/tools/mlir-tblgen/PassGen.cpp
@@ -0,0 +1,139 @@
+//===- Pass.cpp - MLIR pass registration generator ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// PassGen uses the description of passes to generate base classes for passes
+// and command line registration.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/TableGen/GenInfo.h"
+#include "mlir/TableGen/Pass.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
+
+using namespace mlir;
+using namespace mlir::tblgen;
+
+//===----------------------------------------------------------------------===//
+// GEN: Pass base class generation
+//===----------------------------------------------------------------------===//
+
+/// The code snippet used to generate the start of a pass base class.
+///
+/// {0}: The def name of the pass record.
+/// {1}: The command line argument for the pass.
+const char *const passDeclBegin = R"(
+//===----------------------------------------------------------------------===//
+// {0}
+//===----------------------------------------------------------------------===//
+#ifdef GEN_PASS_{0}
+  /// Returns the command-line argument attached to this pass.
+  static StringRef getPassArgument() { return "{1}"; }
+)";
+
+/// The code snippet used to generate the end of a pass base class.
+///
+/// {0}: The def name of the pass record.
+const char *const passDeclEnd = R"(
+#undef GEN_PASS_{0}
+#endif // GEN_PASS_{0}
+)";
+
+/// Emit the declarations for each of the pass options.
+static void emitPassOptionDecls(const Pass &pass, raw_ostream &os) {
+  for (const PassOption &opt : pass.getOptions()) {
+    os.indent(2) << "Pass::" << (opt.isListOption() ? "ListOption" : "Option");
+
+    os << llvm::formatv("<{0}> {1}{{*this, \"{2}\", llvm::cl::desc(\"{3}\")",
+                        opt.getType(), opt.getCppVariableName(),
+                        opt.getArgument(), opt.getDescription());
+    if (Optional<StringRef> defaultVal = opt.getDefaultValue())
+      os << ", llvm::cl::init(" << defaultVal << ")";
+    if (Optional<StringRef> additionalFlags = opt.getAdditionalFlags())
+      os << ", " << *additionalFlags;
+    os << "};\n";
+  }
+}
+
+/// Emit the declarations for each of the pass statistics.
+static void emitPassStatisticDecls(const Pass &pass, raw_ostream &os) {
+  for (const PassStatistic &stat : pass.getStatistics()) {
+    os << llvm::formatv("  Pass::Statistic {0}{{this, \"{1}\", \"{2}\"};\n",
+                        stat.getCppVariableName(), stat.getName(),
+                        stat.getDescription());
+  }
+}
+
+static void emitPassDecl(const Pass &pass, raw_ostream &os) {
+  StringRef defName = pass.getDef()->getName();
+  os << llvm::formatv(passDeclBegin, defName, pass.getArgument());
+  emitPassOptionDecls(pass, os);
+  emitPassStatisticDecls(pass, os);
+  os << llvm::formatv(passDeclEnd, defName);
+}
+
+//===----------------------------------------------------------------------===//
+// GEN: Pass registration generation
+//===----------------------------------------------------------------------===//
+
+/// Emit the code for registering each of the given passes with the global
+/// PassRegistry.
+static void emitRegistration(ArrayRef<Pass> passes, raw_ostream &os) {
+  os << "#ifdef GEN_PASS_REGISTRATION\n";
+  for (const Pass &pass : passes) {
+    os << llvm::formatv("#define GEN_PASS_REGISTRATION_{0}\n",
+                        pass.getDef()->getName());
+  }
+  os << "#endif // GEN_PASS_REGISTRATION\n";
+
+  for (const Pass &pass : passes) {
+    os << llvm::formatv("#ifdef GEN_PASS_REGISTRATION_{0}\n",
+                        pass.getDef()->getName());
+    os << llvm::formatv("::mlir::registerPass(\"{0}\", \"{1}\", []() -> "
+                        "std::unique_ptr<Pass> {{ return {2}; });\n",
+                        pass.getArgument(), pass.getSummary(),
+                        pass.getConstructor());
+    os << llvm::formatv("#endif // GEN_PASS_REGISTRATION_{0}\n",
+                        pass.getDef()->getName());
+    os << llvm::formatv("#undef GEN_PASS_REGISTRATION_{0}\n",
+                        pass.getDef()->getName());
+  }
+
+  os << "#ifdef GEN_PASS_REGISTRATION\n";
+  for (const Pass &pass : passes) {
+    os << llvm::formatv("#undef GEN_PASS_REGISTRATION_{0}\n",
+                        pass.getDef()->getName());
+  }
+  os << "#endif // GEN_PASS_REGISTRATION\n";
+  os << "#undef GEN_PASS_REGISTRATION\n";
+}
+
+//===----------------------------------------------------------------------===//
+// GEN: Registration hooks
+//===----------------------------------------------------------------------===//
+
+static void emitDecls(const llvm::RecordKeeper &recordKeeper, raw_ostream &os) {
+  os << "/* Autogenerated by mlir-tblgen; don't manually edit */\n";
+
+  std::vector<Pass> passes;
+  for (const auto *def : recordKeeper.getAllDerivedDefinitions("Pass")) {
+    Pass pass(def);
+    passes.push_back(pass);
+    emitPassDecl(pass, os);
+  }
+  emitRegistration(passes, os);
+}
+
+static mlir::GenRegistration
+    genRegister("gen-pass-decls", "Generate operation documentation",
+                [](const llvm::RecordKeeper &records, raw_ostream &os) {
+                  emitDecls(records, os);
+                  return false;
+                });
diff --git a/mlir/tools/mlir-translate/mlir-translate.cpp b/mlir/tools/mlir-translate/mlir-translate.cpp
index cd1b226deff7d..1efcaec9cd63a 100644
--- a/mlir/tools/mlir-translate/mlir-translate.cpp
+++ b/mlir/tools/mlir-translate/mlir-translate.cpp
@@ -73,6 +73,7 @@ int main(int argc, char **argv) {
   auto processBuffer = [&](std::unique_ptr<llvm::MemoryBuffer> ownedBuffer,
                            raw_ostream &os) {
     MLIRContext context;
+    context.allowUnregisteredDialects();
     llvm::SourceMgr sourceMgr;
     sourceMgr.AddNewSourceBuffer(std::move(ownedBuffer), llvm::SMLoc());
 
diff --git a/mlir/tools/mlir-vulkan-runner/VulkanRuntime.cpp b/mlir/tools/mlir-vulkan-runner/VulkanRuntime.cpp
index d85db945dd966..77b12b1d9c5fe 100644
--- a/mlir/tools/mlir-vulkan-runner/VulkanRuntime.cpp
+++ b/mlir/tools/mlir-vulkan-runner/VulkanRuntime.cpp
@@ -138,7 +138,7 @@ LogicalResult VulkanRuntime::destroy() {
   // For each descriptor set.
   for (auto &deviceMemoryBufferMapPair : deviceMemoryBufferMap) {
     auto &deviceMemoryBuffers = deviceMemoryBufferMapPair.second;
-    // For each descirptor binding.
+    // For each descriptor binding.
     for (auto &memoryBuffer : deviceMemoryBuffers) {
       vkFreeMemory(device, memoryBuffer.deviceMemory, nullptr);
       vkDestroyBuffer(device, memoryBuffer.buffer, nullptr);
@@ -254,14 +254,14 @@ LogicalResult VulkanRuntime::createDevice() {
   if (failed(getBestComputeQueue()))
     return failure();
 
-  const float queuePrioritory = 1.0f;
+  const float queuePriority = 1.0f;
   VkDeviceQueueCreateInfo deviceQueueCreateInfo = {};
   deviceQueueCreateInfo.sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO;
   deviceQueueCreateInfo.pNext = nullptr;
   deviceQueueCreateInfo.flags = 0;
   deviceQueueCreateInfo.queueFamilyIndex = queueFamilyIndex;
   deviceQueueCreateInfo.queueCount = 1;
-  deviceQueueCreateInfo.pQueuePriorities = &queuePrioritory;
+  deviceQueueCreateInfo.pQueuePriorities = &queuePriority;
 
   // Structure specifying parameters of a newly created device.
   VkDeviceCreateInfo deviceCreateInfo = {};
@@ -599,7 +599,7 @@ LogicalResult VulkanRuntime::createDescriptorPool() {
 
 LogicalResult VulkanRuntime::allocateDescriptorSets() {
   VkDescriptorSetAllocateInfo descriptorSetAllocateInfo = {};
-  // Size of desciptor sets and descriptor layout sets is the same.
+  // Size of descriptor sets and descriptor layout sets is the same.
   descriptorSets.resize(descriptorSetLayouts.size());
   descriptorSetAllocateInfo.sType =
       VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO;
@@ -631,7 +631,7 @@ LogicalResult VulkanRuntime::setWriteDescriptors() {
       VkWriteDescriptorSet wSet = {};
       wSet.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET;
       wSet.pNext = nullptr;
-      // Descirptor set.
+      // Descriptor set.
       wSet.dstSet = *descriptorSetIt;
       wSet.dstBinding = memoryBuffer.bindingIndex;
       wSet.dstArrayElement = 0;
diff --git a/mlir/tools/mlir-vulkan-runner/VulkanRuntime.h b/mlir/tools/mlir-vulkan-runner/VulkanRuntime.h
index cf266f9349bd2..270665160447e 100644
--- a/mlir/tools/mlir-vulkan-runner/VulkanRuntime.h
+++ b/mlir/tools/mlir-vulkan-runner/VulkanRuntime.h
@@ -57,7 +57,7 @@ struct NumWorkGroups {
 struct DescriptorSetInfo {
   /// Index of a descriptor set in descriptor sets.
   DescriptorSetIndex descriptorSet{0};
-  /// Number of desriptors in a set.
+  /// Number of descriptors in a set.
   uint32_t descriptorSize{0};
   /// Type of a descriptor set.
   VkDescriptorType descriptorType{VK_DESCRIPTOR_TYPE_MAX_ENUM};
diff --git a/mlir/tools/mlir-vulkan-runner/vulkan-runtime-wrappers.cpp b/mlir/tools/mlir-vulkan-runner/vulkan-runtime-wrappers.cpp
index 52c11dad8c7e0..4c428ef0349aa 100644
--- a/mlir/tools/mlir-vulkan-runner/vulkan-runtime-wrappers.cpp
+++ b/mlir/tools/mlir-vulkan-runner/vulkan-runtime-wrappers.cpp
@@ -111,9 +111,46 @@ void bindMemRef1DFloat(void *vkRuntimeManager, DescriptorSetIndex setIndex,
       ->setResourceData(setIndex, bindIndex, memBuffer);
 }
 
+/// Binds the given 2D float memref to the given descriptor set and descriptor
+/// index.
+void bindMemRef2DFloat(void *vkRuntimeManager, DescriptorSetIndex setIndex,
+                       BindingIndex bindIndex,
+                       MemRefDescriptor<float, 2> *ptr) {
+  VulkanHostMemoryBuffer memBuffer{
+      ptr->allocated,
+      static_cast<uint32_t>(ptr->sizes[0] * ptr->sizes[1] * sizeof(float))};
+  reinterpret_cast<VulkanRuntimeManager *>(vkRuntimeManager)
+      ->setResourceData(setIndex, bindIndex, memBuffer);
+}
+
+/// Binds the given 3D float memref to the given descriptor set and descriptor
+/// index.
+void bindMemRef3DFloat(void *vkRuntimeManager, DescriptorSetIndex setIndex,
+                       BindingIndex bindIndex,
+                       MemRefDescriptor<float, 3> *ptr) {
+  VulkanHostMemoryBuffer memBuffer{
+      ptr->allocated, static_cast<uint32_t>(ptr->sizes[0] * ptr->sizes[1] *
+                                            ptr->sizes[2] * sizeof(float))};
+  reinterpret_cast<VulkanRuntimeManager *>(vkRuntimeManager)
+      ->setResourceData(setIndex, bindIndex, memBuffer);
+}
+
 /// Fills the given 1D float memref with the given float value.
 void _mlir_ciface_fillResource1DFloat(MemRefDescriptor<float, 1> *ptr, // NOLINT
                                       float value) {
   std::fill_n(ptr->allocated, ptr->sizes[0], value);
 }
+
+/// Fills the given 2D float memref with the given float value.
+void _mlir_ciface_fillResource2DFloat(MemRefDescriptor<float, 2> *ptr, // NOLINT
+                                      float value) {
+  std::fill_n(ptr->allocated, ptr->sizes[0] * ptr->sizes[1], value);
+}
+
+/// Fills the given 3D float memref with the given float value.
+void _mlir_ciface_fillResource3DFloat(MemRefDescriptor<float, 3> *ptr, // NOLINT
+                                      float value) {
+  std::fill_n(ptr->allocated, ptr->sizes[0] * ptr->sizes[1] * ptr->sizes[2],
+              value);
+}
 }
diff --git a/mlir/unittests/IR/OperationSupportTest.cpp b/mlir/unittests/IR/OperationSupportTest.cpp
index ad3cac4c967c6..b1cd5ef098dc6 100644
--- a/mlir/unittests/IR/OperationSupportTest.cpp
+++ b/mlir/unittests/IR/OperationSupportTest.cpp
@@ -14,15 +14,16 @@
 using namespace mlir;
 using namespace mlir::detail;
 
-namespace {
-Operation *createOp(MLIRContext *context, bool resizableOperands,
-                    ArrayRef<Value> operands = llvm::None,
-                    ArrayRef<Type> resultTypes = llvm::None) {
+static Operation *createOp(MLIRContext *context, bool resizableOperands,
+                           ArrayRef<Value> operands = llvm::None,
+                           ArrayRef<Type> resultTypes = llvm::None) {
+  context->allowUnregisteredDialects();
   return Operation::create(
       UnknownLoc::get(context), OperationName("foo.bar", context), resultTypes,
       operands, llvm::None, llvm::None, 0, resizableOperands);
 }
 
+namespace {
 TEST(OperandStorageTest, NonResizable) {
   MLIRContext context;
   Builder builder(&context);
diff --git a/mlir/utils/spirv/gen_spirv_dialect.py b/mlir/utils/spirv/gen_spirv_dialect.py
index 096aa37910031..2c735907063ad 100755
--- a/mlir/utils/spirv/gen_spirv_dialect.py
+++ b/mlir/utils/spirv/gen_spirv_dialect.py
@@ -245,6 +245,11 @@ def get_availability_spec(enum_case, capability_mapping, for_op, for_cap):
   """
   assert not (for_op and for_cap), 'cannot set both for_op and for_cap'
 
+  DEFAULT_MIN_VERSION = 'MinVersion<SPV_V_1_0>'
+  DEFAULT_MAX_VERSION = 'MaxVersion<SPV_V_1_5>'
+  DEFAULT_CAP = 'Capability<[]>'
+  DEFAULT_EXT = 'Extension<[]>'
+
   min_version = enum_case.get('version', '')
   if min_version == 'None':
     min_version = ''
@@ -253,7 +258,7 @@ def get_availability_spec(enum_case, capability_mapping, for_op, for_cap):
   # TODO(antiagainst): delete this once ODS can support dialect-specific content
   # and we can use omission to mean no requirements.
   if for_op and not min_version:
-    min_version = 'MinVersion<SPV_V_1_0>'
+    min_version = DEFAULT_MIN_VERSION
 
   max_version = enum_case.get('lastVersion', '')
   if max_version:
@@ -261,7 +266,7 @@ def get_availability_spec(enum_case, capability_mapping, for_op, for_cap):
   # TODO(antiagainst): delete this once ODS can support dialect-specific content
   # and we can use omission to mean no requirements.
   if for_op and not max_version:
-    max_version = 'MaxVersion<SPV_V_1_5>'
+    max_version = DEFAULT_MAX_VERSION
 
   exts = enum_case.get('extensions', [])
   if exts:
@@ -272,11 +277,11 @@ def get_availability_spec(enum_case, capability_mapping, for_op, for_cap):
     # under such case should be interpreted as this symbol is introduced as
     # a core symbol since the given version, rather than a minimal version
     # requirement.
-    min_version = 'MinVersion<SPV_V_1_0>' if for_op else ''
+    min_version = DEFAULT_MIN_VERSION if for_op else ''
   # TODO(antiagainst): delete this once ODS can support dialect-specific content
   # and we can use omission to mean no requirements.
   if for_op and not exts:
-    exts = 'Extension<[]>'
+    exts = DEFAULT_EXT
 
   caps = enum_case.get('capabilities', [])
   implies = ''
@@ -303,10 +308,16 @@ def get_availability_spec(enum_case, capability_mapping, for_op, for_cap):
   # TODO(antiagainst): delete this once ODS can support dialect-specific content
   # and we can use omission to mean no requirements.
   if for_op and not caps:
-    caps = 'Capability<[]>'
+    caps = DEFAULT_CAP
 
   avail = ''
-  if min_version or max_version or caps or exts:
+  # Compose availability spec if any of the requirements is not empty.
+  # For ops, because we have a default in SPV_Op class, omit if the spec
+  # is the same.
+  if (min_version or max_version or caps or exts) and not (
+      for_op and min_version == DEFAULT_MIN_VERSION and
+      max_version == DEFAULT_MAX_VERSION and caps == DEFAULT_CAP and
+      exts == DEFAULT_EXT):
     joined_spec = ',\n    '.join(
         [e for e in [min_version, max_version, exts, caps] if e])
     avail = '{} availability = [\n    {}\n  ];'.format(
@@ -574,19 +585,19 @@ def map_spec_operand_to_ods_argument(operand):
   return '{}:${}'.format(arg_type, name)
 
 
-def get_description(text, assembly):
+def get_description(text, appendix):
   """Generates the description for the given SPIR-V instruction.
 
   Arguments:
     - text: Textual description of the operation as string.
-    - assembly: Custom Assembly format with example as string.
+    - appendix: Additional contents to attach in description as string,
+                includking IR examples, and others.
 
   Returns:
     - A string that corresponds to the description of the Tablegen op.
   """
-  fmt_str = ('{text}\n\n    ### Custom assembly ' 'form\n{assembly}\n  ')
-  return fmt_str.format(
-      text=text, assembly=assembly)
+  fmt_str = '{text}\n\n    <!-- End of AutoGen section -->\n{appendix}\n  '
+  return fmt_str.format(text=text, appendix=appendix)
 
 
 def get_op_definition(instruction, doc, existing_info, capability_mapping):
@@ -617,8 +628,6 @@ def get_op_definition(instruction, doc, existing_info, capability_mapping):
 
   opname = instruction['opname'][2:]
   category_args = existing_info.get('category_args', '')
-  # Make sure we have ', ' to separate the category arguments from traits
-  category_args = category_args.rstrip(', ') + ', '
 
   if '\n' in doc:
     summary, text = doc.split('\n', 1)
@@ -644,9 +653,13 @@ def get_op_definition(instruction, doc, existing_info, capability_mapping):
   operands = instruction.get('operands', [])
 
   # Op availability
-  avail = get_availability_spec(instruction, capability_mapping, True, False)
-  if avail:
-    avail = '\n\n  {0}'.format(avail)
+  avail = ''
+  # We assume other instruction categories has a base availability spec, so
+  # only add this if this is directly using SPV_Op as the base.
+  if inst_category == 'Op':
+    avail = get_availability_spec(instruction, capability_mapping, True, False)
+    if avail:
+      avail = '\n\n  {0}'.format(avail)
 
   # Set op's result
   results = ''
@@ -673,8 +686,8 @@ def get_op_definition(instruction, doc, existing_info, capability_mapping):
   if description is None:
     assembly = '\n    ```\n'\
                '    [TODO]\n'\
-               '    ```\n\n'\
-               '    For example:\n\n'\
+               '    ```mlir\n\n'\
+               '    #### Example:\n\n'\
                '    ```\n'\
                '    [TODO]\n' \
                '    ```'
@@ -727,7 +740,7 @@ def get_string_between_nested(base, start, end):
   Returns:
     - The substring if found
     - The part of the base after end of the substring. Is the base string itself
-      if the substring wasnt found.
+      if the substring wasn't found.
   """
   split = base.split(start, 1)
   if len(split) == 2:
@@ -738,6 +751,8 @@ def get_string_between_nested(base, start, end):
     while unmatched_start > 0 and index < len(rest):
       if rest[index:].startswith(end):
         unmatched_start -= 1
+        if unmatched_start == 0:
+          break
         index += len(end)
       elif rest[index:].startswith(start):
         unmatched_start += 1
@@ -748,7 +763,7 @@ def get_string_between_nested(base, start, end):
     assert index < len(rest), \
            'cannot find end "{end}" while extracting substring '\
            'starting with "{start}"'.format(start=start, end=end)
-    return rest[:index - len(end)].rstrip(end), rest[index:]
+    return rest[:index], rest[index + len(end):]
   return '', split[0]
 
 
@@ -775,12 +790,12 @@ def extract_td_op_info(op_def):
   inst_category = inst_category[0] if len(inst_category) == 1 else 'Op'
 
   # Get category_args
-  op_tmpl_params = get_string_between_nested(op_def, '<', '>')[0]
+  op_tmpl_params, _ = get_string_between_nested(op_def, '<', '>')
   opstringname, rest = get_string_between(op_tmpl_params, '"', '"')
   category_args = rest.split('[', 1)[0]
 
   # Get traits
-  traits, _ = get_string_between(rest, '[', ']')
+  traits, _ = get_string_between_nested(rest, '[', ']')
 
   # Get description
   description, rest = get_string_between(op_def, 'let description = [{\n',
diff --git a/openmp/libomptarget/deviceRTLs/amdgcn/src/target_impl.h b/openmp/libomptarget/deviceRTLs/amdgcn/src/target_impl.h
index c2d5e329f0a5d..04755a6a3e73f 100644
--- a/openmp/libomptarget/deviceRTLs/amdgcn/src/target_impl.h
+++ b/openmp/libomptarget/deviceRTLs/amdgcn/src/target_impl.h
@@ -18,6 +18,8 @@
 
 #include "amdgcn_interface.h"
 
+#include <assert.h>
+#include <inttypes.h>
 #include <stddef.h>
 #include <stdint.h>
 
diff --git a/openmp/libomptarget/deviceRTLs/common/debug.h b/openmp/libomptarget/deviceRTLs/common/debug.h
index b232490c43c2c..d39ef9d052448 100644
--- a/openmp/libomptarget/deviceRTLs/common/debug.h
+++ b/openmp/libomptarget/deviceRTLs/common/debug.h
@@ -127,7 +127,6 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #if OMPTARGET_NVPTX_DEBUG || OMPTARGET_NVPTX_TEST || OMPTARGET_NVPTX_WARNING
-#include <stdio.h>
 #include "common/support.h"
 
 template <typename... Arguments>
@@ -138,7 +137,6 @@ NOINLINE static void log(const char *fmt, Arguments... parameters) {
 
 #endif
 #if OMPTARGET_NVPTX_TEST
-#include <assert.h>
 
 template <typename... Arguments>
 NOINLINE static void check(bool cond, const char *fmt,
diff --git a/openmp/libomptarget/deviceRTLs/common/omptarget.h b/openmp/libomptarget/deviceRTLs/common/omptarget.h
index d17edb05e6aae..7985691bb8ef9 100644
--- a/openmp/libomptarget/deviceRTLs/common/omptarget.h
+++ b/openmp/libomptarget/deviceRTLs/common/omptarget.h
@@ -14,11 +14,6 @@
 #ifndef OMPTARGET_H
 #define OMPTARGET_H
 
-// std includes
-#include <inttypes.h>
-#include <math.h>
-
-// local includes
 #include "target_impl.h"
 #include "common/debug.h"     // debug
 #include "interface.h" // interfaces with omp, compiler, and user
diff --git a/openmp/libomptarget/deviceRTLs/common/src/data_sharing.cu b/openmp/libomptarget/deviceRTLs/common/src/data_sharing.cu
index c259c7707895a..f6523c8ce8aa2 100644
--- a/openmp/libomptarget/deviceRTLs/common/src/data_sharing.cu
+++ b/openmp/libomptarget/deviceRTLs/common/src/data_sharing.cu
@@ -11,7 +11,6 @@
 //===----------------------------------------------------------------------===//
 #include "common/omptarget.h"
 #include "target_impl.h"
-#include <stdio.h>
 
 // Return true if this is the master thread.
 INLINE static bool IsMasterThread(bool isSPMDExecutionMode) {
diff --git a/openmp/libomptarget/deviceRTLs/nvptx/src/target_impl.h b/openmp/libomptarget/deviceRTLs/nvptx/src/target_impl.h
index 04d510b11591f..1b966510ec7ef 100644
--- a/openmp/libomptarget/deviceRTLs/nvptx/src/target_impl.h
+++ b/openmp/libomptarget/deviceRTLs/nvptx/src/target_impl.h
@@ -12,7 +12,10 @@
 #ifndef _TARGET_IMPL_H_
 #define _TARGET_IMPL_H_
 
+#include <assert.h>
 #include <cuda.h>
+#include <inttypes.h>
+#include <stdio.h>
 #include <stdlib.h>
 
 #include "nvptx_interface.h"
diff --git a/openmp/libomptarget/plugins/common/elf_common.c b/openmp/libomptarget/plugins/common/elf_common.c
index 60e1e4fda4800..b0efd1abc1489 100644
--- a/openmp/libomptarget/plugins/common/elf_common.c
+++ b/openmp/libomptarget/plugins/common/elf_common.c
@@ -45,6 +45,7 @@ static inline int32_t elf_check_machine(__tgt_device_image *image,
   // Check if ELF is the right kind.
   if (elf_kind(e) != ELF_K_ELF) {
     DP("Unexpected ELF type!\n");
+    elf_end(e);
     return 0;
   }
   Elf64_Ehdr *eh64 = elf64_getehdr(e);
diff --git a/openmp/runtime/src/kmp_lock.cpp b/openmp/runtime/src/kmp_lock.cpp
index 2cc9e08278c4e..8e20c6c2c8ee7 100644
--- a/openmp/runtime/src/kmp_lock.cpp
+++ b/openmp/runtime/src/kmp_lock.cpp
@@ -1239,6 +1239,9 @@ __kmp_acquire_queuing_lock_timed_template(kmp_queuing_lock_t *lck,
       KMP_MB();
       // ToDo: Use __kmp_wait_sleep or similar when blocktime != inf
       KMP_WAIT(spin_here_p, FALSE, KMP_EQ, lck);
+      // Synchronize writes to both runtime thread structures
+      // and writes in user code.
+      KMP_MB();
 
 #ifdef DEBUG_QUEUING_LOCKS
       TRACE_LOCK(gtid + 1, "acq spin");
diff --git a/openmp/runtime/test/tasking/bug_nested_proxy_task.c b/openmp/runtime/test/tasking/bug_nested_proxy_task.c
index f70e9044a63bf..b4e8a54602878 100644
--- a/openmp/runtime/test/tasking/bug_nested_proxy_task.c
+++ b/openmp/runtime/test/tasking/bug_nested_proxy_task.c
@@ -2,6 +2,10 @@
 // The runtime currently does not get dependency information from GCC.
 // UNSUPPORTED: gcc
 
+// Very flaky on openmp-clang-x86_64-linux-debian.
+// https://bugs.llvm.org/show_bug.cgi?id=45397
+// UNSUPPORTED: linux
+
 #include <stdio.h>
 #include <omp.h>
 #include <pthread.h>
diff --git a/openmp/runtime/test/tasking/bug_proxy_task_dep_waiting.c b/openmp/runtime/test/tasking/bug_proxy_task_dep_waiting.c
index c07f399d2b43e..d48314ba1408b 100644
--- a/openmp/runtime/test/tasking/bug_proxy_task_dep_waiting.c
+++ b/openmp/runtime/test/tasking/bug_proxy_task_dep_waiting.c
@@ -2,6 +2,10 @@
 // The runtime currently does not get dependency information from GCC.
 // UNSUPPORTED: gcc
 
+// Very flaky on openmp-clang-x86_64-linux-debian.
+// https://bugs.llvm.org/show_bug.cgi?id=45397
+// UNSUPPORTED: linux
+
 #include <stdio.h>
 #include <omp.h>
 #include <pthread.h>
diff --git a/openmp/runtime/test/tasking/omp_task_depend_resize_hashmap.c b/openmp/runtime/test/tasking/omp_task_depend_resize_hashmap.c
index 971f8c0126b0c..58cca562f5947 100644
--- a/openmp/runtime/test/tasking/omp_task_depend_resize_hashmap.c
+++ b/openmp/runtime/test/tasking/omp_task_depend_resize_hashmap.c
@@ -3,6 +3,11 @@
 // This test is known to be fragile on NetBSD kernel at the moment,
 // https://bugs.llvm.org/show_bug.cgi?id=42020.
 // UNSUPPORTED: netbsd
+
+// Very flaky on openmp-clang-x86_64-linux-debian.
+// https://bugs.llvm.org/show_bug.cgi?id=45397
+// UNSUPPORTED: linux
+
 #include<omp.h>
 #include<stdlib.h>
 #include<string.h>
diff --git a/openmp/runtime/test/tasking/omp_task_red_taskloop.c b/openmp/runtime/test/tasking/omp_task_red_taskloop.c
index 89c66256da736..bcc4b84ef8ce4 100644
--- a/openmp/runtime/test/tasking/omp_task_red_taskloop.c
+++ b/openmp/runtime/test/tasking/omp_task_red_taskloop.c
@@ -36,7 +36,7 @@ printf("th %d passed bar0\n", th_gen);
   for (i = 1; i < 4; ++i) {
     bar(i);
 printf("th %d (gen by th %d) passed bar%d in taskloop\n", omp_get_thread_num(), th_gen, i);
-//  #pragma omp task in_reduction(+:r)
+  #pragma omp task in_reduction(+:r)
     r += i;
   }
   return 0;
diff --git a/utils/arcanist/clang-format.sh b/utils/arcanist/clang-format.sh
new file mode 100755
index 0000000000000..3f3dcdbb1686d
--- /dev/null
+++ b/utils/arcanist/clang-format.sh
@@ -0,0 +1,59 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# "script-and-regex.regex": "/^(?P<severity>.*?)\n(?P<message>.*?)\n(?P<line>\\d),(?P<char>\\d)(\n(?P<original>.*?)>>>>\n(?P<replacement>.*?)<<<<?)$/s",
+
+# Arcanist linter that invokes clang-format.
+# stdout from this script is parsed into a regex and used by Arcanist.
+# https://secure.phabricator.com/book/phabricator/article/arcanist_lint_script_and_regex/
+
+# To skip running all linters when creating/updating a diff, use `arc diff --nolint`.
+
+if ! hash clang-format-diff >/dev/null; then
+  # advice severity level is completely non-disruptive.
+  # switch to warning or error if you want to prompt the user.
+  echo "advice"
+  echo "clang-format-diff not found in user's PATH; not linting file."
+  echo "===="
+  exit 0
+fi
+
+src_file="${1}"
+original_file="$(mktemp)"
+formatted_file="$(mktemp)"
+readonly src_file
+readonly original_file
+readonly formatted_file
+cp -p "${src_file}" "${original_file}"
+cp -p "${src_file}" "${formatted_file}"
+
+cleanup() {
+  rc=$?
+  rm "${formatted_file}" "${original_file}"
+  exit ${rc}
+}
+trap 'cleanup' INT HUP QUIT TERM EXIT
+
+# Arcanist can filter out lint messages for unchanged lines, but for that, we
+# need to generate line by line lint messages. Instead, we generate one lint
+# message on line 1, char 1 with file content edited using clang-format-diff.
+if git rev-parse --git-dir >/dev/null; then
+  arc_base_commit=$(arc which --show-base)
+  # An alternative is to use git-clang-format.
+  git diff -U0 --no-color "${arc_base_commit}"| clang-format-diff -style file -i -p1
+else
+  svn diff --diff-cmd=diff -x -U0 "${src_file}" | clang-format-diff -style LLVM -i
+fi
+
+cp -p "${src_file}" "${formatted_file}"
+cp -p "${original_file}" "${src_file}"
+if ! diff -q "${src_file}" "${formatted_file}" > /dev/null ; then
+  echo "autofix"
+  echo "clang-format suggested style edits found:"
+  echo "1,1"  # line,char of start of replacement.
+  cat "${src_file}"
+  echo ">>>>"
+  cat "${formatted_file}"
+  echo "<<<<"
+fi